summaryrefslogtreecommitdiffstats
path: root/contrib
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
commit133a45c109da5310add55824db21af5239951f93 (patch)
treeba6ac4c0a950a0dda56451944315d66409923918 /contrib
parentInitial commit. (diff)
downloadrspamd-133a45c109da5310add55824db21af5239951f93.tar.xz
rspamd-133a45c109da5310add55824db21af5239951f93.zip
Adding upstream version 3.8.1.upstream/3.8.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'contrib')
-rw-r--r--contrib/DEPENDENCY_INFO.md41
-rw-r--r--contrib/aho-corasick/CMakeLists.txt14
-rw-r--r--contrib/aho-corasick/LICENSE165
-rw-r--r--contrib/aho-corasick/README.md67
-rw-r--r--contrib/aho-corasick/_acism.h114
-rw-r--r--contrib/aho-corasick/acism.c124
-rw-r--r--contrib/aho-corasick/acism.h53
-rw-r--r--contrib/aho-corasick/acism_create.c382
-rw-r--r--contrib/ankerl/LICENSE21
-rw-r--r--contrib/ankerl/svector.h999
-rw-r--r--contrib/ankerl/unordered_dense.h2032
-rw-r--r--contrib/backward-cpp/BackwardConfig.cmake249
-rw-r--r--contrib/backward-cpp/CMakeLists.txt88
-rw-r--r--contrib/backward-cpp/backward.cpp42
-rw-r--r--contrib/backward-cpp/backward.hpp4471
-rw-r--r--contrib/cdb/CMakeLists.txt8
-rw-r--r--contrib/cdb/cdb.h161
-rw-r--r--contrib/cdb/cdb_find.c147
-rw-r--r--contrib/cdb/cdb_init.c140
-rw-r--r--contrib/cdb/cdb_make.c524
-rw-r--r--contrib/doctest/CMakeLists.txt63
-rw-r--r--contrib/doctest/LICENSE.txt21
-rw-r--r--contrib/doctest/README.md153
-rw-r--r--contrib/doctest/doctest/doctest.h6580
-rw-r--r--contrib/doctest/doctest/extensions/doctest_mpi.h120
-rw-r--r--contrib/doctest/doctest/extensions/doctest_util.h34
-rw-r--r--contrib/doctest/doctest/extensions/mpi_reporter.h236
-rw-r--r--contrib/doctest/doctest/parts/doctest.cpp3866
-rw-r--r--contrib/doctest/doctest/parts/doctest_fwd.h2706
-rw-r--r--contrib/elastic/kibana.json138
-rw-r--r--contrib/elastic/rspamd_template.json149
-rw-r--r--contrib/exim/dlfunc-json/README10
-rw-r--r--contrib/exim/dlfunc-json/exim-example.txt75
-rw-r--r--contrib/exim/dlfunc-json/rspamd.c576
-rw-r--r--contrib/exim/local_scan.c700
-rw-r--r--contrib/exim/local_scan.c.in377
-rw-r--r--contrib/exim/patch-exim-src_spam.c.diff338
-rw-r--r--contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff332
-rw-r--r--contrib/exim/shutdown.patch14
-rw-r--r--contrib/expected/COPYING121
-rw-r--r--contrib/expected/expected.hpp2326
-rw-r--r--contrib/fastutf8/CMakeLists.txt11
-rw-r--r--contrib/fastutf8/LICENSE22
-rw-r--r--contrib/fastutf8/avx2.c314
-rw-r--r--contrib/fastutf8/fastutf8.c160
-rw-r--r--contrib/fastutf8/fastutf8.h65
-rw-r--r--contrib/fastutf8/sse41.c272
-rw-r--r--contrib/fmt/LICENSE.rst27
-rw-r--r--contrib/fmt/README.rst506
-rw-r--r--contrib/fmt/include/fmt/args.h234
-rw-r--r--contrib/fmt/include/fmt/chrono.h2267
-rw-r--r--contrib/fmt/include/fmt/color.h633
-rw-r--r--contrib/fmt/include/fmt/compile.h607
-rw-r--r--contrib/fmt/include/fmt/core.h2951
-rw-r--r--contrib/fmt/include/fmt/format-inl.h1681
-rw-r--r--contrib/fmt/include/fmt/format.h4735
-rw-r--r--contrib/fmt/include/fmt/locale.h2
-rw-r--r--contrib/fmt/include/fmt/os.h451
-rw-r--r--contrib/fmt/include/fmt/ostream.h209
-rw-r--r--contrib/fmt/include/fmt/posix.h2
-rw-r--r--contrib/fmt/include/fmt/printf.h679
-rw-r--r--contrib/fmt/include/fmt/ranges.h732
-rw-r--r--contrib/fmt/include/fmt/std.h349
-rw-r--r--contrib/fmt/include/fmt/xchar.h259
-rw-r--r--contrib/fpconv/CMakeLists.txt7
-rw-r--r--contrib/fpconv/LICENSE23
-rw-r--r--contrib/fpconv/fpconv.c480
-rw-r--r--contrib/fpconv/fpconv.h35
-rw-r--r--contrib/fpconv/powers.h87
-rw-r--r--contrib/frozen/AUTHORS3
-rw-r--r--contrib/frozen/CMakeLists.txt12
-rw-r--r--contrib/frozen/LICENSE202
-rw-r--r--contrib/frozen/include/frozen/algorithm.h197
-rw-r--r--contrib/frozen/include/frozen/bits/algorithms.h229
-rw-r--r--contrib/frozen/include/frozen/bits/basic_types.h200
-rw-r--r--contrib/frozen/include/frozen/bits/constexpr_assert.h40
-rw-r--r--contrib/frozen/include/frozen/bits/defines.h58
-rw-r--r--contrib/frozen/include/frozen/bits/elsa.h50
-rw-r--r--contrib/frozen/include/frozen/bits/exceptions.h39
-rw-r--r--contrib/frozen/include/frozen/bits/pmh.h240
-rw-r--r--contrib/frozen/include/frozen/bits/version.h30
-rw-r--r--contrib/frozen/include/frozen/map.h323
-rw-r--r--contrib/frozen/include/frozen/random.h90
-rw-r--r--contrib/frozen/include/frozen/set.h220
-rw-r--r--contrib/frozen/include/frozen/string.h152
-rw-r--r--contrib/frozen/include/frozen/unordered_map.h197
-rw-r--r--contrib/frozen/include/frozen/unordered_set.h147
-rw-r--r--contrib/fu2/LICENSE.txt23
-rw-r--r--contrib/fu2/include/function2/function2.hpp1792
-rw-r--r--contrib/google-ced/CMakeLists.txt26
-rw-r--r--contrib/google-ced/LICENSE202
-rw-r--r--contrib/google-ced/ced_c.cc25
-rw-r--r--contrib/google-ced/ced_c.h32
-rw-r--r--contrib/google-ced/compact_enc_det.cc5719
-rw-r--r--contrib/google-ced/compact_enc_det.h83
-rw-r--r--contrib/google-ced/compact_enc_det_generated_tables.h6326
-rw-r--r--contrib/google-ced/compact_enc_det_generated_tables2.h856
-rw-r--r--contrib/google-ced/compact_enc_det_hint_code.cc169
-rw-r--r--contrib/google-ced/compact_enc_det_hint_code.h45
-rw-r--r--contrib/google-ced/detail_head_string.inc152
-rw-r--r--contrib/google-ced/util/basictypes.h331
-rw-r--r--contrib/google-ced/util/case_insensitive_hash.h88
-rw-r--r--contrib/google-ced/util/commandlineflags.h39
-rw-r--r--contrib/google-ced/util/encodings/encodings.cc891
-rw-r--r--contrib/google-ced/util/encodings/encodings.h299
-rw-r--r--contrib/google-ced/util/encodings/encodings.pb.h181
-rw-r--r--contrib/google-ced/util/encodings/encodings_unittest.cc34
-rw-r--r--contrib/google-ced/util/languages/languages.cc349
-rw-r--r--contrib/google-ced/util/languages/languages.h381
-rw-r--r--contrib/google-ced/util/languages/languages.pb.h191
-rw-r--r--contrib/google-ced/util/logging.h25
-rw-r--r--contrib/google-ced/util/port.h53
-rw-r--r--contrib/google-ced/util/string_util.h61
-rw-r--r--contrib/google-ced/util/varsetter.h66
-rw-r--r--contrib/hiredis/CMakeLists.txt11
-rw-r--r--contrib/hiredis/COPYING29
-rw-r--r--contrib/hiredis/README.md392
-rw-r--r--contrib/hiredis/adapters/libev.h147
-rw-r--r--contrib/hiredis/adapters/libevent.h108
-rw-r--r--contrib/hiredis/async.c693
-rw-r--r--contrib/hiredis/async.h135
-rw-r--r--contrib/hiredis/dict.c340
-rw-r--r--contrib/hiredis/dict.h126
-rw-r--r--contrib/hiredis/fmacros.h21
-rw-r--r--contrib/hiredis/hiredis.c1025
-rw-r--r--contrib/hiredis/hiredis.h242
-rw-r--r--contrib/hiredis/net.c464
-rw-r--r--contrib/hiredis/net.h53
-rw-r--r--contrib/hiredis/read.c525
-rw-r--r--contrib/hiredis/read.h116
-rw-r--r--contrib/hiredis/sds.c1095
-rw-r--r--contrib/hiredis/sds.h108
-rw-r--r--contrib/http-parser/CMakeLists.txt8
-rw-r--r--contrib/http-parser/LICENSE-MIT23
-rw-r--r--contrib/http-parser/http_parser.c2268
-rw-r--r--contrib/http-parser/http_parser.h321
-rw-r--r--contrib/kann/CMakeLists.txt16
-rw-r--r--contrib/kann/LICENSE.txt24
-rw-r--r--contrib/kann/kann.c992
-rw-r--r--contrib/kann/kann.h240
-rw-r--r--contrib/kann/kautodiff.c2460
-rw-r--r--contrib/kann/kautodiff.h256
-rw-r--r--contrib/languages-data/af.json1
-rw-r--r--contrib/languages-data/an.json1
-rw-r--r--contrib/languages-data/ar.json1
-rw-r--r--contrib/languages-data/bg.json1
-rw-r--r--contrib/languages-data/bn.json1
-rw-r--r--contrib/languages-data/br.json1
-rw-r--r--contrib/languages-data/cs.json1
-rw-r--r--contrib/languages-data/cy.json1
-rw-r--r--contrib/languages-data/da.json1
-rw-r--r--contrib/languages-data/de.json1
-rw-r--r--contrib/languages-data/en.json1
-rw-r--r--contrib/languages-data/es.json1
-rw-r--r--contrib/languages-data/et.json1
-rw-r--r--contrib/languages-data/eu.json1
-rw-r--r--contrib/languages-data/fa.json1
-rw-r--r--contrib/languages-data/fi.json1
-rw-r--r--contrib/languages-data/fr.json1
-rw-r--r--contrib/languages-data/ga.json1
-rw-r--r--contrib/languages-data/hi.json1
-rw-r--r--contrib/languages-data/hr.json1
-rw-r--r--contrib/languages-data/hu.json1
-rw-r--r--contrib/languages-data/id.json1
-rw-r--r--contrib/languages-data/is.json1
-rw-r--r--contrib/languages-data/it.json1
-rw-r--r--contrib/languages-data/lt.json1
-rw-r--r--contrib/languages-data/lv.json1
-rw-r--r--contrib/languages-data/mr.json1
-rw-r--r--contrib/languages-data/ms.json1
-rw-r--r--contrib/languages-data/ne.json1
-rw-r--r--contrib/languages-data/nl.json1
-rw-r--r--contrib/languages-data/no.json1
-rw-r--r--contrib/languages-data/pa.json1
-rw-r--r--contrib/languages-data/pl.json1
-rw-r--r--contrib/languages-data/pt.json1
-rw-r--r--contrib/languages-data/ro.json1
-rw-r--r--contrib/languages-data/ru.json1
-rw-r--r--contrib/languages-data/sl.json1
-rw-r--r--contrib/languages-data/so.json1
-rw-r--r--contrib/languages-data/sq.json1
-rw-r--r--contrib/languages-data/sr.json1
-rw-r--r--contrib/languages-data/stop_words4153
-rw-r--r--contrib/languages-data/sv.json1
-rw-r--r--contrib/languages-data/sw.json1
-rw-r--r--contrib/languages-data/tr.json1
-rw-r--r--contrib/languages-data/uk.json1
-rw-r--r--contrib/languages-data/ur.json1
-rw-r--r--contrib/languages-data/vi.json1
-rw-r--r--contrib/lc-btrie/CMakeLists.txt6
-rw-r--r--contrib/lc-btrie/btrie.c2644
-rw-r--r--contrib/lc-btrie/btrie.h83
-rw-r--r--contrib/libev/CMakeLists.txt82
-rw-r--r--contrib/libev/Changes529
-rw-r--r--contrib/libev/LICENSE37
-rw-r--r--contrib/libev/config.h.in112
-rw-r--r--contrib/libev/ev++.h816
-rw-r--r--contrib/libev/ev.c5678
-rw-r--r--contrib/libev/ev.h849
-rw-r--r--contrib/libev/ev_epoll.c298
-rw-r--r--contrib/libev/ev_iouring.c697
-rw-r--r--contrib/libev/ev_kqueue.c224
-rw-r--r--contrib/libev/ev_linuxaio.c620
-rw-r--r--contrib/libev/ev_poll.c156
-rw-r--r--contrib/libev/ev_port.c192
-rw-r--r--contrib/libev/ev_select.c316
-rw-r--r--contrib/libev/ev_vars.h249
-rw-r--r--contrib/libev/ev_win32.c162
-rw-r--r--contrib/libev/ev_wrap.h272
-rw-r--r--contrib/libottery/CMakeLists.txt11
-rw-r--r--contrib/libottery/aes_cryptobox.c181
-rw-r--r--contrib/libottery/chacha_cryptobox.c64
-rw-r--r--contrib/libottery/chacha_merged.c218
-rw-r--r--contrib/libottery/chacha_merged_ecrypt.h133
-rw-r--r--contrib/libottery/ottery-internal.h335
-rw-r--r--contrib/libottery/ottery-threading.h96
-rw-r--r--contrib/libottery/ottery.c847
-rw-r--r--contrib/libottery/ottery.h143
-rw-r--r--contrib/libottery/ottery_common.h351
-rw-r--r--contrib/libottery/ottery_cpuinfo.c88
-rw-r--r--contrib/libottery/ottery_entropy.c112
-rw-r--r--contrib/libottery/ottery_entropy_cryptgenrandom.c51
-rw-r--r--contrib/libottery/ottery_entropy_egd.c75
-rw-r--r--contrib/libottery/ottery_entropy_rdrand.c58
-rw-r--r--contrib/libottery/ottery_entropy_urandom.c117
-rw-r--r--contrib/libottery/ottery_global.c116
-rw-r--r--contrib/libottery/ottery_nolock.h190
-rw-r--r--contrib/libottery/ottery_st.h184
-rw-r--r--contrib/libottery/ottery_version.h.in28
-rw-r--r--contrib/librdns/CMakeLists.txt11
-rw-r--r--contrib/librdns/compression.c168
-rw-r--r--contrib/librdns/compression.h48
-rw-r--r--contrib/librdns/curve.c890
-rw-r--r--contrib/librdns/dns_private.h338
-rw-r--r--contrib/librdns/logger.c53
-rw-r--r--contrib/librdns/logger.h48
-rw-r--r--contrib/librdns/packet.c288
-rw-r--r--contrib/librdns/packet.h61
-rw-r--r--contrib/librdns/parse.c461
-rw-r--r--contrib/librdns/parse.h65
-rw-r--r--contrib/librdns/punycode.c303
-rw-r--r--contrib/librdns/punycode.h59
-rw-r--r--contrib/librdns/rdns.h493
-rw-r--r--contrib/librdns/rdns_curve.h75
-rw-r--r--contrib/librdns/rdns_ev.h235
-rw-r--r--contrib/librdns/rdns_event.h231
-rw-r--r--contrib/librdns/ref.h71
-rw-r--r--contrib/librdns/resolver.c1577
-rw-r--r--contrib/librdns/upstream.h282
-rw-r--r--contrib/librdns/util.c1035
-rw-r--r--contrib/librdns/util.h99
-rw-r--r--contrib/libucl/CMakeLists.txt19
-rw-r--r--contrib/libucl/README.md397
-rw-r--r--contrib/libucl/khash.h682
-rw-r--r--contrib/libucl/kvec.h161
-rw-r--r--contrib/libucl/lua_ucl.c1574
-rw-r--r--contrib/libucl/lua_ucl.h109
-rw-r--r--contrib/libucl/tree.h219
-rw-r--r--contrib/libucl/ucl.h1650
-rw-r--r--contrib/libucl/ucl_chartable.h268
-rw-r--r--contrib/libucl/ucl_emitter.c1189
-rw-r--r--contrib/libucl/ucl_emitter_streamline.c173
-rw-r--r--contrib/libucl/ucl_hash.c498
-rw-r--r--contrib/libucl/ucl_hash.h114
-rw-r--r--contrib/libucl/ucl_internal.h666
-rw-r--r--contrib/libucl/ucl_msgpack.c1615
-rw-r--r--contrib/libucl/ucl_parser.c3212
-rw-r--r--contrib/libucl/ucl_schema.c1104
-rw-r--r--contrib/libucl/ucl_sexp.c226
-rw-r--r--contrib/libucl/ucl_util.c3952
-rw-r--r--contrib/lua-argparse/LICENSE20
-rw-r--r--contrib/lua-argparse/argparse.lua2100
-rw-r--r--contrib/lua-bit/CMakeLists.txt4
-rw-r--r--contrib/lua-bit/bit.c198
-rw-r--r--contrib/lua-fun/COPYING.md27
-rw-r--r--contrib/lua-fun/fun.lua1076
-rw-r--r--contrib/lua-lpeg/CMakeLists.txt9
-rw-r--r--contrib/lua-lpeg/LICENSE19
-rw-r--r--contrib/lua-lpeg/lpcap.c555
-rw-r--r--contrib/lua-lpeg/lpcap.h57
-rw-r--r--contrib/lua-lpeg/lpcode.c1014
-rw-r--r--contrib/lua-lpeg/lpcode.h40
-rw-r--r--contrib/lua-lpeg/lpegre.lua267
-rw-r--r--contrib/lua-lpeg/lpprint.c244
-rw-r--r--contrib/lua-lpeg/lpprint.h36
-rw-r--r--contrib/lua-lpeg/lptree.c1319
-rw-r--r--contrib/lua-lpeg/lptree.h77
-rw-r--r--contrib/lua-lpeg/lptypes.h154
-rw-r--r--contrib/lua-lpeg/lpvm.c383
-rw-r--r--contrib/lua-lpeg/lpvm.h58
-rw-r--r--contrib/lua-lupa/LICENSE21
-rw-r--r--contrib/lua-lupa/README.md179
-rw-r--r--contrib/lua-lupa/lupa.lua1810
-rw-r--r--contrib/lua-tableshape/LICENSE19
-rw-r--r--contrib/lua-tableshape/tableshape.lua2324
-rw-r--r--contrib/mumhash/mum.h392
-rw-r--r--contrib/publicsuffix/effective_tld_names.dat14206
-rw-r--r--contrib/publicsuffix/idn.pl20
-rw-r--r--contrib/replxx/CMakeLists.txt82
-rw-r--r--contrib/replxx/LICENSE.md63
-rw-r--r--contrib/replxx/README.md119
-rw-r--r--contrib/replxx/include/replxx.h575
-rw-r--r--contrib/replxx/include/replxx.hxx615
-rw-r--r--contrib/replxx/src/conversion.cxx134
-rw-r--r--contrib/replxx/src/conversion.hxx35
-rw-r--r--contrib/replxx/src/escape.cxx890
-rw-r--r--contrib/replxx/src/escape.hxx37
-rw-r--r--contrib/replxx/src/history.cxx402
-rw-r--r--contrib/replxx/src/history.hxx141
-rw-r--r--contrib/replxx/src/killring.hxx78
-rw-r--r--contrib/replxx/src/prompt.cxx144
-rw-r--r--contrib/replxx/src/prompt.hxx45
-rw-r--r--contrib/replxx/src/replxx.cxx648
-rw-r--r--contrib/replxx/src/replxx_impl.cxx2248
-rw-r--r--contrib/replxx/src/replxx_impl.hxx280
-rw-r--r--contrib/replxx/src/terminal.cxx742
-rw-r--r--contrib/replxx/src/terminal.hxx94
-rw-r--r--contrib/replxx/src/unicodestring.hxx201
-rw-r--r--contrib/replxx/src/utf8string.hxx94
-rw-r--r--contrib/replxx/src/util.cxx158
-rw-r--r--contrib/replxx/src/util.hxx25
-rw-r--r--contrib/replxx/src/wcwidth.cpp296
-rw-r--r--contrib/replxx/src/windows.cxx144
-rw-r--r--contrib/replxx/src/windows.hxx44
-rw-r--r--contrib/snowball/.gitignore5
-rw-r--r--contrib/snowball/.travis.yml4
-rw-r--r--contrib/snowball/AUTHORS27
-rw-r--r--contrib/snowball/CMakeLists.txt70
-rw-r--r--contrib/snowball/NEWS407
-rw-r--r--contrib/snowball/README5
-rw-r--r--contrib/snowball/algorithms/arabic.sbl561
-rw-r--r--contrib/snowball/algorithms/basque.sbl149
-rw-r--r--contrib/snowball/algorithms/catalan.sbl202
-rw-r--r--contrib/snowball/algorithms/danish.sbl93
-rw-r--r--contrib/snowball/algorithms/dutch.sbl164
-rw-r--r--contrib/snowball/algorithms/english.sbl229
-rw-r--r--contrib/snowball/algorithms/finnish.sbl197
-rw-r--r--contrib/snowball/algorithms/french.sbl254
-rw-r--r--contrib/snowball/algorithms/german.sbl139
-rw-r--r--contrib/snowball/algorithms/german2.sbl145
-rw-r--r--contrib/snowball/algorithms/greek.sbl706
-rw-r--r--contrib/snowball/algorithms/hindi.sbl323
-rw-r--r--contrib/snowball/algorithms/hungarian.sbl241
-rw-r--r--contrib/snowball/algorithms/indonesian.sbl192
-rw-r--r--contrib/snowball/algorithms/irish.sbl151
-rw-r--r--contrib/snowball/algorithms/italian.sbl195
-rw-r--r--contrib/snowball/algorithms/kraaij_pohlmann.sbl240
-rw-r--r--contrib/snowball/algorithms/lithuanian.sbl373
-rw-r--r--contrib/snowball/algorithms/lovins.sbl208
-rw-r--r--contrib/snowball/algorithms/nepali.sbl92
-rw-r--r--contrib/snowball/algorithms/norwegian.sbl80
-rw-r--r--contrib/snowball/algorithms/porter.sbl139
-rw-r--r--contrib/snowball/algorithms/portuguese.sbl218
-rw-r--r--contrib/snowball/algorithms/romanian.sbl236
-rw-r--r--contrib/snowball/algorithms/russian.sbl221
-rw-r--r--contrib/snowball/algorithms/serbian.sbl2378
-rw-r--r--contrib/snowball/algorithms/spanish.sbl230
-rw-r--r--contrib/snowball/algorithms/swedish.sbl72
-rw-r--r--contrib/snowball/algorithms/tamil.sbl405
-rw-r--r--contrib/snowball/algorithms/turkish.sbl470
-rw-r--r--contrib/snowball/charsets/ISO-8859-2.sbl98
-rw-r--r--contrib/snowball/charsets/KOI8-R.sbl74
-rw-r--r--contrib/snowball/charsets/cp850.sbl130
-rw-r--r--contrib/snowball/compiler/analyser.c1380
-rw-r--r--contrib/snowball/compiler/driver.c574
-rw-r--r--contrib/snowball/compiler/generator.c1725
-rw-r--r--contrib/snowball/compiler/header.h411
-rw-r--r--contrib/snowball/compiler/space.c287
-rw-r--r--contrib/snowball/compiler/syswords.h86
-rw-r--r--contrib/snowball/compiler/syswords2.h13
-rw-r--r--contrib/snowball/compiler/tokeniser.c567
-rw-r--r--contrib/snowball/doc/TODO15
-rw-r--r--contrib/snowball/doc/libstemmer_c_README125
-rw-r--r--contrib/snowball/doc/libstemmer_java_README40
-rw-r--r--contrib/snowball/include/libstemmer.h78
-rw-r--r--contrib/snowball/libstemmer/libstemmer_c.in96
-rwxr-xr-xcontrib/snowball/libstemmer/mkmodules.pl267
-rw-r--r--contrib/snowball/libstemmer/modules.txt58
-rw-r--r--contrib/snowball/libstemmer/modules_utf8.txt49
-rw-r--r--contrib/snowball/runtime/api.c58
-rw-r--r--contrib/snowball/runtime/api.h32
-rw-r--r--contrib/snowball/runtime/header.h59
-rw-r--r--contrib/snowball/runtime/utilities.c503
-rw-r--r--contrib/t1ha/CMakeLists.txt6
-rw-r--r--contrib/t1ha/LICENSE21
-rw-r--r--contrib/t1ha/t1ha.h321
-rw-r--r--contrib/t1ha/t1ha1.c158
-rw-r--r--contrib/t1ha/t1ha2.c326
-rw-r--r--contrib/t1ha/t1ha_bits.h1171
-rw-r--r--contrib/uthash/uthash.h951
-rw-r--r--contrib/uthash/utlist.h766
-rw-r--r--contrib/uthash/utstring.h415
-rw-r--r--contrib/xxhash/CMakeLists.txt13
-rw-r--r--contrib/xxhash/LICENSE24
-rw-r--r--contrib/xxhash/xxh3.h55
-rw-r--r--contrib/xxhash/xxhash.c43
-rw-r--r--contrib/xxhash/xxhash.h5580
-rw-r--r--contrib/zstd/CHANGELOG555
-rw-r--r--contrib/zstd/CMakeLists.txt27
-rw-r--r--contrib/zstd/LICENSE30
-rw-r--r--contrib/zstd/PATENTS33
-rw-r--r--contrib/zstd/README.md94
-rw-r--r--contrib/zstd/bits.h175
-rw-r--r--contrib/zstd/bitstream.h437
-rw-r--r--contrib/zstd/clevels.h134
-rw-r--r--contrib/zstd/compiler.h354
-rw-r--r--contrib/zstd/cpu.h213
-rw-r--r--contrib/zstd/debug.c24
-rw-r--r--contrib/zstd/debug.h107
-rw-r--r--contrib/zstd/divsufsort.c1913
-rw-r--r--contrib/zstd/divsufsort.h67
-rw-r--r--contrib/zstd/entropy_common.c340
-rw-r--r--contrib/zstd/error_private.c63
-rw-r--r--contrib/zstd/error_private.h159
-rw-r--r--contrib/zstd/error_public.h59
-rw-r--r--contrib/zstd/fse.h639
-rw-r--r--contrib/zstd/fse_compress.c624
-rw-r--r--contrib/zstd/fse_decompress.c311
-rw-r--r--contrib/zstd/hist.c181
-rw-r--r--contrib/zstd/hist.h75
-rw-r--r--contrib/zstd/huf.h273
-rw-r--r--contrib/zstd/huf_compress.c1435
-rw-r--r--contrib/zstd/huf_decompress.c1882
-rw-r--r--contrib/zstd/mem.h435
-rw-r--r--contrib/zstd/pool.c371
-rw-r--r--contrib/zstd/pool.h90
-rw-r--r--contrib/zstd/portability_macros.h156
-rw-r--r--contrib/zstd/zdict.h474
-rw-r--r--contrib/zstd/zstd.h2974
-rw-r--r--contrib/zstd/zstd_common.c83
-rw-r--r--contrib/zstd/zstd_compress.c6927
-rw-r--r--contrib/zstd/zstd_compress.h307
-rw-r--r--contrib/zstd/zstd_compress_internal.h1478
-rw-r--r--contrib/zstd/zstd_compress_literals.c235
-rw-r--r--contrib/zstd/zstd_compress_literals.h39
-rw-r--r--contrib/zstd/zstd_compress_sequences.c442
-rw-r--r--contrib/zstd/zstd_compress_sequences.h54
-rw-r--r--contrib/zstd/zstd_compress_superblock.c577
-rw-r--r--contrib/zstd/zstd_compress_superblock.h32
-rw-r--r--contrib/zstd/zstd_cwksp.h678
-rw-r--r--contrib/zstd/zstd_ddict.c243
-rw-r--r--contrib/zstd/zstd_ddict.h44
-rw-r--r--contrib/zstd/zstd_decompress.c2352
-rw-r--r--contrib/zstd/zstd_decompress_block.c2193
-rw-r--r--contrib/zstd/zstd_decompress_block.h68
-rw-r--r--contrib/zstd/zstd_decompress_internal.h238
-rw-r--r--contrib/zstd/zstd_deps.h111
-rw-r--r--contrib/zstd/zstd_double_fast.c758
-rw-r--r--contrib/zstd/zstd_double_fast.h39
-rw-r--r--contrib/zstd/zstd_errors.h114
-rw-r--r--contrib/zstd/zstd_fast.c960
-rw-r--r--contrib/zstd/zstd_fast.h38
-rw-r--r--contrib/zstd/zstd_internal.h397
-rw-r--r--contrib/zstd/zstd_lazy.c2127
-rw-r--r--contrib/zstd/zstd_lazy.h127
-rw-r--r--contrib/zstd/zstd_ldm.c724
-rw-r--r--contrib/zstd/zstd_ldm.h117
-rw-r--r--contrib/zstd/zstd_ldm_geartab.h106
-rw-r--r--contrib/zstd/zstd_opt.c1470
-rw-r--r--contrib/zstd/zstd_opt.h56
-rw-r--r--contrib/zstd/zstd_trace.h163
461 files changed, 225380 insertions, 0 deletions
diff --git a/contrib/DEPENDENCY_INFO.md b/contrib/DEPENDENCY_INFO.md
new file mode 100644
index 0000000..300a38c
--- /dev/null
+++ b/contrib/DEPENDENCY_INFO.md
@@ -0,0 +1,41 @@
+# Rspamd Dependency Info
+
+| Name | Version | License | Patched | Notes |
+|------------------------|---------|---------------------|---------|---------------------|
+| aho-corasick | ? | LGPL-3.0 | YES | lowercase support |
+| cdb | 1.1.0 | Public Domain / CC0 | NO | |
+| hiredis | 0.13.3 | BSD-3-Clause | YES | many changes |
+| libev | 4.33 | BSD-2-Clause | YES | many changes |
+| lc-btrie | ? | BSD-3-Clause | YES | mempool support |
+| libottery | ? | Public Domain / CC0 | YES | many changes |
+| librdns | ? | BSD-2-Clause | YES | |
+| libucl | ? | BSD-2-Clause | YES | |
+| replxx | 6d93360 | BSD-2-Clause | YES | libicu usage |
+| lua-argparse | 0.7.1 | MIT | NO | |
+| lua-bit | 1.0.2 | MIT | YES | build fixes |
+| lua-fun | ? | MIT | YES | rspamd text |
+| lua-lpeg | 1.0 | MIT | YES | rspamd text + alloc |
+| lua-moses | ? | MIT | NO | |
+| lua-lupa | ? | MIT | NO | |
+| lua-tableshape | 2.6.0 | MIT | NO | |
+| mumhash | ? | MIT | NO | |
+| ngx-http-parser | 2.2.0 | MIT | YES | spamc support |
+| Mozilla-PublicSuffix | ? | MIT | NO | |
+| snowball | ? | BSD-3-Clause | NO | |
+| t1ha | ? | Zlib | NO | |
+| uthash | 1.9.8 | BSD | YES | |
+| xxhash | 0.8.1 | BSD | NO | |
+| zstd | 1.5.4 | BSD | YES | build fixes only |
+| google-ced | 37529e6 | Apache 2 | YES | build fixes |
+| kann | ? | MIT | YES | blas/lapack changes |
+| fpconv | ? | Boost | YES | many changes |
+| fastutf8 | ? | MIT | YES | many changes |
+| expected | v1.0 | Public Domain / CC0 | NO | |
+| frozen | 1.0.1 | Apache 2 | NO | |
+| fmt | 10.0.0 | MIT | NO | |
+| doctest | 2.4.6 | MIT | NO | |
+| function2 | 4.1.0 | Boost | NO | |
+| ankerl/svector | 1.0.2 | MIT | NO | |
+| ankerl/unordered_dense | 4.4.0 | MIT | NO | |
+| backward-cpp | 1.6 | MIT | NO | |
+
diff --git a/contrib/aho-corasick/CMakeLists.txt b/contrib/aho-corasick/CMakeLists.txt
new file mode 100644
index 0000000..f47dbed
--- /dev/null
+++ b/contrib/aho-corasick/CMakeLists.txt
@@ -0,0 +1,14 @@
+SET(AHOCORASICSRC acism_create.c
+ acism.c)
+
+IF(NOT GPL_RSPAMD_BINARY)
+ ADD_LIBRARY(rspamd-actrie SHARED ${AHOCORASICSRC})
+ target_link_libraries(rspamd-actrie glib-2.0)
+ target_link_libraries(rspamd-actrie "${RSPAMD_REQUIRED_LIBRARIES}")
+
+ INSTALL(TARGETS rspamd-actrie
+ LIBRARY DESTINATION ${RSPAMD_LIBDIR})
+ELSE()
+ ADD_LIBRARY(rspamd-actrie STATIC ${AHOCORASICSRC})
+ target_link_libraries(rspamd-actrie glib-2.0)
+ENDIF()
diff --git a/contrib/aho-corasick/LICENSE b/contrib/aho-corasick/LICENSE
new file mode 100644
index 0000000..65c5ca8
--- /dev/null
+++ b/contrib/aho-corasick/LICENSE
@@ -0,0 +1,165 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/contrib/aho-corasick/README.md b/contrib/aho-corasick/README.md
new file mode 100644
index 0000000..c245d6f
--- /dev/null
+++ b/contrib/aho-corasick/README.md
@@ -0,0 +1,67 @@
+aho-corasick
+==
+
+Aho-Corasick parallel string search, using interleaved arrays.
+
+Mischa Sandberg mischasan@gmail.com
+
+ACISM is an implementation of Aho-Corasick parallel string search,
+using an Interleaved State-transition Matrix.
+It combines the fastest possible Aho-Corasick implementation,
+with the smallest possible data structure (!).
+
+FEATURES
+--------
+
+* Fast. No hashing, no tree traversal; just a straight look-up equivalent to
+ matrix[state, input-byte] per input character.
+
+* Tiny. On average, the whole data structure (mostly the array) takes about 2-3 bytes per
+ input pattern byte. The original set of pattern strings can be reverse-generated from the machine.
+
+* Shareable. The state machine contains no pointers, so it can be compiled once,
+ then memory-mapped by many processes.
+
+* Searches byte vectors, not null-terminated strings.
+ Suitable for searching machine code as much as searching text.
+
+* DOS-proof. Well, that's an attribute of Aho-Corasick,
+ so no real points for that.
+
+* Stream-ready. The state can be saved between calls to search data.
+
+DOCUMENTATION
+-------------
+
+The GoogleDocs description is at http://goo.gl/lE6zG
+I originally called it "psearch", but found that name was overused by other authors.
+
+LICENSE
+-------
+
+Though I've had strong suggestions to go with BSD license, I'm going with GPL2 until I figure out
+how to keep in touch with people who download and use the code. Hence the "CONTACT ME IF..." line in the license.
+
+GETTING STARTED
+---------------
+
+Download the source, type "gmake".
+"gmake install" exports lib/libacism.a, include/acism.h and bin/acism_x.
+"acism_x.c" is a good example of calling acism_create and acism_scan/acism_more.
+
+(If you're interested in the GNUmakefile and rules.mk,
+ check my blog posts on non-recursive make, at mischasan.wordpress.com.)
+
+HISTORY
+-------
+
+The interleaved-array approach was tried and discarded in the late 70's, because the compile time was O(n^2).
+acism_create beats the problem with a "hint" array that tracks the restart points for searches.
+That, plus discarding the original idea of how to get maximal density, resulted in the tiny-fast win-win.
+
+ACKNOWLEDGEMENTS
+----------------
+
+I'd like to thank Mike Shannon, who wanted to see a machine built to make best use of L1/L2 cache.
+The change to do that doubled performance on hardware with a much larger cache than the matrix.
+Go figure.
diff --git a/contrib/aho-corasick/_acism.h b/contrib/aho-corasick/_acism.h
new file mode 100644
index 0000000..3993594
--- /dev/null
+++ b/contrib/aho-corasick/_acism.h
@@ -0,0 +1,114 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License Version 3 as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+#ifndef _ACISM_H
+#define _ACISM_H
+
+#include <stdint.h>
+#include <stdlib.h> // malloc
+#include <string.h> // memcpy
+
+typedef int (*qsort_cmp)(const void *, const void *);
+
+// "Width" specifier for different plats
+#if __LONG_MAX__ == 9223372036854775807LL
+# ifdef __APPLE__
+# define F64 "ll"
+# else
+# define F64 "l"
+# endif
+#elif __LONG_MAX__ == 2147483647L || defined(_LONG_LONG) || defined(__sun) // AIX 6.1 ...
+# define F64 "ll"
+#else
+//XXX Assuming F64 is "ll" for VS
+# define F64 "ll"
+#endif
+
+#ifndef ACISM_SIZE
+# define ACISM_SIZE 4
+#endif
+
+#if ACISM_SIZE == 8
+typedef uint64_t TRAN, STATE, STRNO;
+# define SYM_BITS 9U
+# define SYM_MASK 511
+# define FNO F64
+#else
+typedef uint32_t TRAN, STATE, STRNO;
+# define SYM_BITS psp->sym_bits
+# define SYM_MASK psp->sym_mask
+# define FNO
+#endif
+
+typedef uint16_t SYMBOL;
+typedef unsigned _SYMBOL; // An efficient stacklocal SYMBOL
+
+#define BACK ((SYMBOL)0)
+#define ROOT ((STATE) 0)
+
+// MATCH and SUFFIX are the top 2 bits of a TRAN:
+enum {
+ IS_MATCH = (TRAN)1 << (8*sizeof(TRAN) - 1),
+ IS_SUFFIX = (TRAN)1 << (8*sizeof(TRAN) - 2),
+ T_FLAGS = IS_MATCH | IS_SUFFIX
+};
+
+typedef struct { STATE state; STRNO strno; } STRASH;
+
+struct acism {
+ TRAN* tranv;
+ STRASH* hashv;
+ unsigned flags;
+# define IS_MMAP 1
+
+#if ACISM_SIZE < 8
+ TRAN sym_mask;
+ unsigned sym_bits;
+#endif
+ unsigned hash_mod; // search hashv starting at (state + sym) % hash_mod.
+ unsigned hash_size; // #(hashv): hash_mod plus the overflows past [hash_mod-1]
+ unsigned tran_size; // #(tranv)
+ unsigned nsyms, nchars, nstrs, maxlen;
+ SYMBOL symv[256];
+};
+
+#include "acism.h"
+
+// p_size: size of tranv + hashv
+static inline size_t p_size(ACISM const *psp)
+{ return psp->hash_size * sizeof*psp->hashv
+ + psp->tran_size * sizeof*psp->tranv; }
+
+static inline unsigned p_hash(ACISM const *psp, STATE s)
+{ return s * 107 % psp->hash_mod; }
+
+static inline void set_tranv(ACISM *psp, void *mem)
+{ psp->hashv = (STRASH*)&(psp->tranv = (TRAN*)mem)[psp->tran_size]; }
+
+// TRAN accessors. For ACISM_SIZE=8, SYM_{BITS,MASK} do not use psp.
+
+static inline TRAN p_tran(ACISM const *psp, STATE s, _SYMBOL sym)
+{ return psp->tranv[s + sym] ^ sym; }
+
+static inline _SYMBOL t_sym(ACISM const *psp, TRAN t) { (void)psp; return t & SYM_MASK; }
+static inline STATE t_next(ACISM const *psp, TRAN t) { (void)psp; return (t & ~T_FLAGS) >> SYM_BITS; }
+static inline int t_isleaf(ACISM const *psp, TRAN t) { return t_next(psp, t) >= psp->tran_size; }
+static inline int t_strno(ACISM const *psp, TRAN t) { return t_next(psp, t) - psp->tran_size; }
+static inline _SYMBOL t_valid(ACISM const *psp, TRAN t) { return !t_sym(psp, t); }
+
+#endif//_ACISM_H \ No newline at end of file
diff --git a/contrib/aho-corasick/acism.c b/contrib/aho-corasick/acism.c
new file mode 100644
index 0000000..e2b48a5
--- /dev/null
+++ b/contrib/aho-corasick/acism.c
@@ -0,0 +1,124 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License Version as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#include <glib.h>
+
+#include "_acism.h"
+#include "unix-std.h"
+
+#define BACK ((SYMBOL)0)
+#define ROOT ((STATE) 0)
+extern const guchar lc_map[256];
+
+int
+acism_lookup(ac_trie_t const *psp, const char *text, size_t len,
+ ACISM_ACTION *cb, void *context, int *statep, bool caseless)
+{
+ char const *cp = text, *endp = cp + len;
+ uint8_t s;
+ STATE state = *statep;
+ int ret = 0;
+
+ while (cp < endp) {
+ s = caseless ? lc_map[(guint8)*cp++] : *cp++;
+ _SYMBOL sym = psp->symv[s];
+ if (!sym) {
+ // Input byte is not in any pattern string.
+ state = ROOT;
+ continue;
+ }
+
+ // Search for a valid transition from this (state, sym),
+ // following the backref chain.
+
+ TRAN next;
+ while (!t_valid(psp, next = p_tran(psp, state, sym)) && state != ROOT) {
+ TRAN back = p_tran(psp, state, BACK);
+ state = t_valid(psp, back) ? t_next(psp, back) : ROOT;
+ }
+
+ if (!t_valid(psp, next))
+ continue;
+
+ if (!(next & (IS_MATCH | IS_SUFFIX))) {
+ // No complete match yet; keep going.
+ state = t_next(psp, next);
+ continue;
+ }
+
+ // At this point, one or more patterns have matched.
+ // Find all matches by following the backref chain.
+ // A valid node for (sym) with no SUFFIX flag marks the
+ // end of the suffix chain.
+ // In the same backref traversal, find a new (state),
+ // if the original transition is to a leaf.
+
+ STATE s = state;
+
+ // Initially state is ROOT. The chain search saves the
+ // first state from which the next char has a transition.
+ state = t_isleaf(psp, next) ? 0 : t_next(psp, next);
+
+ while (1) {
+
+ if (t_valid(psp, next)) {
+
+ if (next & IS_MATCH) {
+ unsigned strno, ss = s + sym, i;
+ if (t_isleaf(psp, psp->tranv[ss])) {
+ strno = t_strno(psp, psp->tranv[ss]);
+ } else {
+ for (i = p_hash(psp, ss); psp->hashv[i].state != ss; ++i);
+ strno = psp->hashv[i].strno;
+ }
+
+ if ((ret = cb(strno, cp - text, context)))
+ goto EXIT;
+ }
+
+ if (!state && !t_isleaf(psp, next))
+ state = t_next(psp, next);
+ if ( state && !(next & IS_SUFFIX))
+ break;
+ }
+
+ if (s == ROOT)
+ break;
+
+ TRAN b = p_tran(psp, s, BACK);
+ s = t_valid(psp, b) ? t_next(psp, b) : ROOT;
+ next = p_tran(psp, s, sym);
+ }
+ }
+EXIT:
+ *statep = state;
+ return ret;
+}
+
+void
+acism_destroy(ac_trie_t *psp)
+{
+ if (!psp) return;
+ if (psp->flags & IS_MMAP)
+ munmap((char*)psp->tranv - sizeof(ac_trie_t),
+ sizeof(ac_trie_t) + p_size(psp));
+ else g_free(psp->tranv);
+ g_free(psp);
+}
+//EOF
diff --git a/contrib/aho-corasick/acism.h b/contrib/aho-corasick/acism.h
new file mode 100644
index 0000000..1e03176
--- /dev/null
+++ b/contrib/aho-corasick/acism.h
@@ -0,0 +1,53 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+** Copyright (C) 2015 Vsevolod Stakhov <vsevolod@rspamd.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#ifndef ACISM_H
+#define ACISM_H
+
+#include "config.h"
+// "acism" uses MEMREF {ptr,len} bytevec structs for "string" args,
+// rather than NUL-terminated "C" strings.
+
+typedef struct ac_trie_pat_s { char const *ptr; size_t len; } ac_trie_pat_t;
+
+typedef struct acism ac_trie_t;
+typedef struct acism ACISM;
+typedef struct ac_trie_pat_s MEMREF;
+
+ac_trie_t* acism_create(ac_trie_pat_t const *strv, int nstrs);
+void acism_destroy(ac_trie_t*);
+
+// For each match, acism_scan calls its ACISM_ACTION fn,
+// giving it the strv[] index of the matched string,
+// and the text[] offset of the byte PAST the end of the string.
+// If ACISM_ACTION returns 0, search continues; otherwise,
+// acism_more returns that nonzero value immediately.
+
+typedef int (ACISM_ACTION)(int strnum, int textpos, void *context);
+
+// If sequential blocks of (text) are passed to repeated acism_more calls,
+// then search continues where the previous acism_more left off --
+// string matches can cross block boundaries.
+// *state should initially be (0).
+
+int acism_lookup(ac_trie_t const *psp, const char *text, size_t len,
+ ACISM_ACTION *cb, void *context, int *statep, bool caseless);
+
+#endif//ACISM_H
diff --git a/contrib/aho-corasick/acism_create.c b/contrib/aho-corasick/acism_create.c
new file mode 100644
index 0000000..2d4439f
--- /dev/null
+++ b/contrib/aho-corasick/acism_create.c
@@ -0,0 +1,382 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License Version 3 as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+#include "_acism.h"
+
+typedef enum { BASE=2, USED=1 } USES;
+
+typedef struct tnode {
+ struct tnode *child, *next, *back;
+ // nrefs was used in "prune_backlinks".
+ // It will be used again in "curtail".
+ unsigned nrefs;
+ STATE state;
+ STRNO match;
+ SYMBOL sym;
+ char is_suffix; // "bool"
+} TNODE;
+
+//--------------|---------------------------------------------
+// bitwid: 1+floor(log2(u))
+static inline int bitwid(unsigned u)
+{
+ int ret = !!u;
+ if (u & 0xFFFF0000) u >>= 16, ret += 16;
+ if (u & 0x0000FF00) u >>= 8, ret += 8;
+ if (u & 0x000000F0) u >>= 4, ret += 4;
+ if (u & 0x0000000C) u >>= 2, ret += 2;
+ if (u & 0x00000002) ret++;
+ return ret;
+}
+
+static void fill_symv(ACISM*, MEMREF const*, int ns);
+static int create_tree(TNODE*, SYMBOL const*symv, MEMREF const*strv, int nstrs);
+static void add_backlinks(TNODE*, TNODE**, TNODE**);
+static int interleave(TNODE*, int nnodes, int nsyms, TNODE**, TNODE**);
+static void fill_tranv(ACISM*, TNODE const*);
+static void fill_hashv(ACISM*, TNODE const*, int nn);
+
+static TNODE* find_child(TNODE*, SYMBOL);
+
+// (ns) is either a STATE, or a (STRNO + tran_size)
+static inline void
+set_tran(ACISM *psp, STATE s, SYMBOL sym, int match, int suffix, TRAN ns)
+{
+ psp->tranv[s + sym] = sym | (match ? IS_MATCH : 0)
+ | (suffix ? IS_SUFFIX : 0)
+ | (ns << SYM_BITS);
+}
+
+// Track statistics for construction
+#ifdef ACISM_STATS
+typedef struct { long long val; const char *name; } PSSTAT;
+extern PSSTAT psstat[];
+# define NOTE(n) (psstat[__LINE__] = (PSSTAT) {n, #n})
+# define HIT(id) (psstat[__LINE__].val++, psstat[__LINE__].name = id)
+#else
+# define NOTE(n) (void)0
+# define HIT(id) (void)0
+#endif //ACISM_STATS
+
+//--------------|---------------------------------------------
+ACISM*
+acism_create(MEMREF const* strv, int nstrs)
+{
+ TNODE **v1 = NULL, **v2 = NULL;
+ ACISM *psp = g_malloc0(sizeof*psp);
+
+ fill_symv(psp, strv, nstrs);
+ TNODE *troot = g_malloc0((psp->nchars + 1) * sizeof(*troot));
+
+ int nnodes = create_tree(troot, psp->symv, strv, nstrs);
+ NOTE(nnodes);
+
+ // v1, v2: breadth-first work vectors for add_backlink and interleave.
+ int i = (nstrs + 1) * sizeof(TNODE);
+ add_backlinks(troot, v1 = g_malloc0(i), v2 = g_malloc0(i));
+
+ int nhash = 0;
+ TNODE* tp = troot + nnodes;
+ while (--tp > troot)
+ nhash += tp->match && tp->child;
+
+ // Calculate each node's offset in tranv[]:
+ psp->tran_size = interleave(troot, nnodes, psp->nsyms, v1, v2);
+ if (bitwid(psp->tran_size + nstrs - 1) + SYM_BITS > sizeof(TRAN)*8 - 2)
+ goto FAIL;
+
+ if (nhash) {
+ // Hash table is for match info of non-leaf nodes (only).
+ // Set hash_size for p_size(psp):
+ psp->hash_mod = nhash * 5 / 4 + 1;
+ // Initially oversize the table for overflows without wraparound.
+ psp->hash_size = psp->hash_mod + nhash;
+ }
+
+ set_tranv(psp, g_malloc0(p_size(psp) + sizeof(TRAN)));
+ if (!psp->tranv) goto FAIL;
+ fill_tranv(psp, troot);
+ // The root state (0) must not look like a valid backref.
+ // Any symbol value other than (0) in tranv[0] ensures that.
+ psp->tranv[0] = 1;
+
+ if (nhash) {
+ fill_hashv(psp, troot, nnodes);
+ // Adjust hash_size to include trailing overflows
+ // but trim trailing empty slots.
+ psp->hash_size = psp->hash_mod;
+ while ( psp->hashv[psp->hash_size].state) ++psp->hash_size;
+ while (!psp->hashv[psp->hash_size - 1].state) --psp->hash_size;
+ set_tranv(psp, g_realloc(psp->tranv, p_size(psp)));
+ }
+
+ // Diagnostics/statistics only:
+ psp->nstrs = nstrs;
+ for (i = psp->maxlen = 0; i < nstrs; ++i)
+ if (psp->maxlen < strv[i].len) psp->maxlen = strv[i].len;
+
+ goto DONE;
+ FAIL: acism_destroy(psp), psp = NULL;
+ DONE: free(troot), free(v1), free(v2);
+ return psp;
+}
+
+typedef struct { int freq, rank; } FRANK;
+static int frcmp(FRANK*a, FRANK*b) { return a->freq - b->freq; }
+
+static void
+fill_symv(ACISM *psp, MEMREF const *strv, int nstrs)
+{
+ int i, j;
+ FRANK frv[256];
+
+ for (i = 0; i < 256; ++i) frv[i] = (FRANK){0,i};
+ for (i = 0; i < nstrs; ++i)
+ for (psp->nchars += j = strv[i].len; --j >= 0;)
+ frv[(uint8_t)strv[i].ptr[j]].freq++;
+
+ qsort(frv, 256, sizeof*frv, (qsort_cmp)frcmp);
+
+ for (i = 256; --i >= 0 && frv[i].freq;)
+ psp->symv[frv[i].rank] = ++psp->nsyms;
+ ++psp->nsyms;
+
+#if ACISM_SIZE < 8
+ psp->sym_bits = bitwid(psp->nsyms);
+ psp->sym_mask = ~((~0u) << psp->sym_bits);
+#endif
+}
+
+static int
+create_tree(TNODE *Tree, SYMBOL const *symv, MEMREF const *strv, int nstrs)
+{
+ int i, j;
+ TNODE *nextp = Tree + 1;
+
+ for (i = 0; i < nstrs; ++i) {
+ TNODE *tp = Tree;
+
+ for (j = 0; tp->child && j < (int)strv[i].len; ++j) {
+ SYMBOL sym = symv[(uint8_t)strv[i].ptr[j]];
+
+ if (sym < tp->child->sym) {
+ // Prep to insert new node before tp->child
+ nextp->next = tp->child;
+ break;
+ }
+
+ tp = tp->child;
+ while (tp->next && sym >= tp->next->sym)
+ tp = tp->next;
+
+ // Insert new sibling after tp
+ if (sym > tp->sym) {
+ nextp->next = tp->next;
+ tp = tp->next = nextp++;
+ tp->sym = sym;
+ tp->back = Tree;
+ }
+ }
+
+ for (; j < (int) strv[i].len; ++j) {
+ tp = tp->child = nextp++;
+ tp->sym = symv[(uint8_t)strv[i].ptr[j]];
+ tp->back = Tree;
+ }
+
+ tp->match = i + 1; // Encode strno as nonzero
+ }
+
+ return nextp - Tree;
+}
+
+static void
+add_backlinks(TNODE *troot, TNODE **v1, TNODE **v2)
+{
+ TNODE *tp, **tmp;
+
+ for (tp = troot->child, tmp = v1; tp; tp = tp->next)
+ *tmp++ = tp;
+ *tmp = NULL;
+
+ while (*v1) {
+ TNODE **spp = v1, **dpp = v2, *srcp, *dstp;
+
+ while ((srcp = *spp++)) {
+ for (dstp = srcp->child; dstp; dstp = dstp->next) {
+ TNODE *bp = NULL;
+ if (dstp->child)
+ *dpp++ = dstp;
+
+ // Go through the parent (srcp) node's backlink chain,
+ // looking for a useful backlink for the child (dstp).
+ // If the parent (srcp) has a backlink to (tp),
+ // and (tp) has a child matching the transition sym
+ // for (srcp -> dstp), then it is a useful backlink
+ // for the child (dstp).
+ // Note that backlinks do not point at the suffix match;
+ // they point at the PARENT of that match.
+
+ for (tp = srcp->back; tp; tp = tp->back)
+ if ((bp = find_child(tp, dstp->sym)))
+ break;
+ if (!bp)
+ bp = troot;
+
+ dstp->back = dstp->child ? bp : tp ? tp : troot;
+ dstp->back->nrefs++;
+ dstp->is_suffix = bp->match || bp->is_suffix;
+ }
+ }
+ *dpp = 0;
+ tmp = v1; v1 = v2; v2 = tmp;
+ }
+}
+
+static int
+interleave(TNODE *troot, int nnodes, int nsyms, TNODE **v1, TNODE **v2)
+{
+ unsigned usev_size = nnodes + nsyms;
+ char *usev = g_malloc0(usev_size * sizeof(*usev));
+ STATE last_trans = 0, startv[257][2];
+ TNODE *cp, **tmp;
+
+ memset(startv, 0, nsyms * sizeof*startv);
+
+ // Iterate through one level of the Tree at a time.
+ // That srsly improves locality (L1-cache use).
+
+ v1[0] = troot, v1[1] = NULL;
+ for (; *v1; tmp = v1, v1 = v2, v2 = tmp) {
+ TNODE **srcp = v1, **dstp = v2, *tp;
+ while ((tp = *srcp++)) {
+ if (!tp->child) continue;
+
+ HIT("nonleaf");
+ if (tp->back == troot) tp->back = NULL; // simplify tests.
+ cp = tp->child;
+
+ STATE pos, *startp = &startv[cp->sym][!!tp->back];
+ while ((cp = cp->next)) {
+ STATE *newp = &startv[cp->sym][!!tp->back];
+ if (*startp < *newp) startp = newp;
+ }
+
+ // If (tp) has a backref, we need a slot at offset 0
+ // that is free as a base AND to be used (filled in).
+ char need = tp->back ? BASE|USED : BASE;
+ for (pos = *startp;; ++pos) {
+ if (usev[pos] & need) {
+ HIT("inner loop");
+ continue;
+ }
+
+ for (cp = tp->child; cp; cp = cp->next) {
+ HIT("child loop");
+ if (usev[pos + cp->sym] & USED) break;
+ }
+
+ // No child needs an in-use slot? We're done.
+ if (!cp) break;
+ }
+ tp->state = pos;
+
+ // Mark node's base and children as used:
+ usev[pos] |= need;
+ STATE last = 0; // Make compiler happy
+ int nkids = 0;
+ for (cp = tp->child; cp; *dstp++ = cp, cp = cp->next, ++nkids)
+ usev[last = pos + cp->sym] |= USED;
+
+ // This is a HEURISTIC for advancing search for other nodes
+ *startp += (pos - *startp) / nkids;
+
+ if (last_trans < last) {
+ last_trans = last;
+ if (last + nsyms >= usev_size) {
+ usev = g_realloc(usev, usev_size << 1);
+ memset(usev + usev_size, 0, usev_size);
+ usev_size <<= 1;
+ }
+ }
+ }
+
+ *dstp = NULL;
+ }
+
+ free(usev);
+ return last_trans + 1;
+}
+
+static void
+fill_hashv(ACISM *psp, TNODE const treev[], int nnodes)
+{
+ STRASH *sv = g_malloc0(psp->hash_mod * sizeof*sv), *sp = sv;
+ int i;
+
+ // First pass: insert without resolving collisions.
+ for (i = 0; i < nnodes; ++i) {
+ STATE base = treev[i].state;
+ TNODE const *tp;
+ for (tp = treev[i].child; tp; tp = tp->next) {
+ if (tp->match && tp->child) {
+ STATE state = base + tp->sym;
+ STRASH *hp = &psp->hashv[p_hash(psp, state)];
+ *(hp->state ? sp++ : hp) = (STRASH){state, tp->match - 1};
+ }
+ }
+ }
+
+ while (--sp >= sv) {
+ HIT("hash collisions");
+ for (i = p_hash(psp, sp->state); psp->hashv[i].state; ++i)
+ HIT("hash displacements");
+ psp->hashv[i] = *sp;
+ }
+
+ free(sv);
+}
+
+static void
+fill_tranv(ACISM *psp, TNODE const*tp)
+{
+ TNODE const *cp = tp->child;
+
+ if (cp && tp->back)
+ set_tran(psp, tp->state, 0, 0, 0, tp->back->state);
+
+ for (; cp; cp = cp->next) {
+ //NOTE: cp->match is (strno+1) so that !cp->match means "no match".
+ set_tran(psp, tp->state, cp->sym, cp->match, cp->is_suffix,
+ cp->child ? cp->state : cp->match - 1 + psp->tran_size);
+ if (cp->child)
+ fill_tranv(psp, cp);
+ }
+}
+
+static TNODE *
+find_child(TNODE *tp, SYMBOL sym)
+{
+ for (tp = tp->child; tp && tp->sym < sym; tp = tp->next);
+ return tp && tp->sym == sym ? tp : NULL;
+}
+
+#ifdef ACISM_STATS
+PSSTAT psstat[__LINE__] = {{__LINE__,0}};
+#endif//ACISM_STATS
+//EOF \ No newline at end of file
diff --git a/contrib/ankerl/LICENSE b/contrib/ankerl/LICENSE
new file mode 100644
index 0000000..c4d1a0e
--- /dev/null
+++ b/contrib/ankerl/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Martin Leitner-Ankerl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/ankerl/svector.h b/contrib/ankerl/svector.h
new file mode 100644
index 0000000..dbd075b
--- /dev/null
+++ b/contrib/ankerl/svector.h
@@ -0,0 +1,999 @@
+// ┌─┐┬ ┬┌─┐┌─┐┌┬┐┌─┐┬─┐ Compact SVO optimized vector C++17 or higher
+// └─┐└┐┌┘├┤ │ │ │ │├┬┘ Version 1.0.2
+// └─┘ └┘ └─┘└─┘ ┴ └─┘┴└─ https://github.com/martinus/svector
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef ANKERL_SVECTOR_H
+#define ANKERL_SVECTOR_H
+
+// see https://semver.org/spec/v2.0.0.html
+#define ANKERL_SVECTOR_VERSION_MAJOR 1 // incompatible API changes
+#define ANKERL_SVECTOR_VERSION_MINOR 0 // add functionality in a backwards compatible manner
+#define ANKERL_SVECTOR_VERSION_PATCH 2 // backwards compatible bug fixes
+
+// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
+#define ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
+#define ANKERL_SVECTOR_VERSION_CONCAT(major, minor, patch) ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch)
+#define ANKERL_SVECTOR_NAMESPACE \
+ ANKERL_SVECTOR_VERSION_CONCAT(ANKERL_SVECTOR_VERSION_MAJOR, ANKERL_SVECTOR_VERSION_MINOR, ANKERL_SVECTOR_VERSION_PATCH)
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <initializer_list>
+#include <iterator>
+#include <limits>
+#include <memory>
+#include <new>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+namespace ankerl {
+inline namespace ANKERL_SVECTOR_NAMESPACE {
+namespace detail {
+
+template <typename Condition, typename T = void>
+using enable_if_t = typename std::enable_if<Condition::value, T>::type;
+
+template <typename It>
+using is_input_iterator = std::is_base_of<std::input_iterator_tag, typename std::iterator_traits<It>::iterator_category>;
+
+constexpr auto round_up(size_t n, size_t multiple) -> size_t {
+ return ((n + (multiple - 1)) / multiple) * multiple;
+}
+
+template <typename T>
+constexpr auto cx_min(T a, T b) -> T {
+ return a < b ? a : b;
+}
+
+template <typename T>
+constexpr auto cx_max(T a, T b) -> T {
+ return a > b ? a : b;
+}
+
+template <typename T>
+constexpr auto alignment_of_svector() -> size_t {
+ return cx_max(sizeof(void*), std::alignment_of_v<T>);
+}
+
+/**
+ * @brief Calculates sizeof(svector<T, N>) for a given type and inline capacity
+ */
+template <typename T>
+constexpr auto size_of_svector(size_t min_inline_capacity) -> size_t {
+ // + 1 for one byte size in direct mode
+ return round_up(sizeof(T) * min_inline_capacity + 1, alignment_of_svector<T>());
+}
+
+/**
+ * @brief Calculates how many T we can actually store inside of an svector without increasing its sizeof().
+ *
+ * E.g. svector<char, 1> could store 7 bytes even though 1 is specified. This makes sure we don't waste any
+ * of the padding.
+ */
+template <typename T>
+constexpr auto automatic_capacity(size_t min_inline_capacity) -> size_t {
+ return cx_min((size_of_svector<T>(min_inline_capacity) - 1U) / sizeof(T), size_t{127});
+}
+
+/**
+ * Holds size & capacity, a glorified struct.
+ */
+class header {
+ size_t m_size{};
+ size_t const m_capacity;
+
+public:
+ inline explicit header(size_t capacity)
+ : m_capacity{capacity} {}
+
+ [[nodiscard]] inline auto size() const -> size_t {
+ return m_size;
+ }
+
+ [[nodiscard]] inline auto capacity() const -> size_t {
+ return m_capacity;
+ }
+
+ inline void size(size_t s) {
+ m_size = s;
+ }
+};
+
+/**
+ * @brief Holds header (size+capacity) plus an arbitrary number of T.
+ *
+ * To make storage compact, we don't actually store a pointer to T. We don't have to
+ * because we know exactly at which location it begins.
+ */
+template <typename T>
+struct storage : public header {
+ static constexpr auto alignment_of_t = std::alignment_of_v<T>;
+ static constexpr auto max_alignment = std::max(std::alignment_of_v<header>, std::alignment_of_v<T>);
+ static constexpr auto offset_to_data = detail::round_up(sizeof(header), alignment_of_t);
+ static_assert(max_alignment <= __STDCPP_DEFAULT_NEW_ALIGNMENT__);
+
+ explicit storage(size_t capacity)
+ : header(capacity) {}
+
+ auto data() -> T* {
+ auto ptr_to_data = reinterpret_cast<std::byte*>(this) + offset_to_data;
+ return std::launder(reinterpret_cast<T*>(ptr_to_data));
+ }
+
+ /**
+ * @brief Allocates space for storage plus capacity*T objects.
+ *
+ * Checks to make sure that allocation won't overflow.
+ *
+ * @param capacity Number of T to allocate.
+ * @return storage<T>*
+ */
+ static auto alloc(size_t capacity) -> storage<T>* {
+ // make sure we don't overflow!
+ auto mem = sizeof(T) * capacity;
+ if (mem < capacity) {
+ throw std::bad_alloc();
+ }
+ if (offset_to_data + mem < mem) {
+ throw std::bad_alloc();
+ }
+ mem += offset_to_data;
+ if (static_cast<uint64_t>(mem) > static_cast<uint64_t>(std::numeric_limits<std::ptrdiff_t>::max())) {
+ throw std::bad_alloc();
+ }
+
+ void* ptr = ::operator new(offset_to_data + sizeof(T) * capacity);
+ if (nullptr == ptr) {
+ throw std::bad_alloc();
+ }
+ // use void* to ensure we don't use an overload for T*
+ return new (ptr) storage<T>(capacity);
+ }
+};
+
+} // namespace detail
+
+template <typename T, size_t MinInlineCapacity>
+class svector {
+ static_assert(MinInlineCapacity <= 127, "sorry, can't have more than 127 direct elements");
+ static constexpr auto N = detail::automatic_capacity<T>(MinInlineCapacity);
+
+ enum class direction { direct, indirect };
+
+ /**
+ * A buffer to hold the data of the svector Depending on direct/indirect mode, the content it holds is like so:
+ *
+ * direct:
+ * m_data[0] & 1: lowest bit is 1 for direct mode.
+ * m_data[0] >> 1: size for direct mode
+ * Then 0-X bytes unused (padding), and then the actual inline T data.
+ * indirect:
+ * m_data[0] & 1: lowest bit is 0 for indirect mode
+ * m_data[0..7]: stores an uintptr_t, which points to the indirect data.
+ */
+ alignas(detail::alignment_of_svector<T>()) std::array<uint8_t, detail::size_of_svector<T>(MinInlineCapacity)> m_data;
+
+ // direct mode ///////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto is_direct() const -> bool {
+ return (m_data[0] & 1U) != 0U;
+ }
+
+ [[nodiscard]] auto direct_size() const -> size_t {
+ return m_data[0] >> 1U;
+ }
+
+ // sets size of direct mode and mode to direct too.
+ constexpr void set_direct_and_size(size_t s) {
+ m_data[0] = (s << 1U) | 1U;
+ }
+
+ [[nodiscard]] auto direct_data() -> T* {
+ return std::launder(reinterpret_cast<T*>(m_data.data() + std::alignment_of_v<T>));
+ }
+
+ // indirect mode /////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto indirect() -> detail::storage<T>* {
+ detail::storage<T>* ptr; // NOLINT(cppcoreguidelines-init-variables)
+ std::memcpy(&ptr, m_data.data(), sizeof(ptr));
+ return ptr;
+ }
+
+ [[nodiscard]] auto indirect() const -> detail::storage<T> const* {
+ return const_cast<svector*>(this)->indirect(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ void set_indirect(detail::storage<T>* ptr) {
+ std::memcpy(m_data.data(), &ptr, sizeof(ptr));
+
+ // safety check to guarantee the lowest bit is 0
+ if (is_direct()) {
+ throw std::bad_alloc(); // LCOV_EXCL_LINE
+ }
+ }
+
+ // helpers ///////////////////////////////////////////////////////////////
+
+ /**
+ * @brief Moves size objects from source_ptr to target_ptr, and destroys what remains in source_ptr.
+ *
+ * Assumes data is not overlapping
+ */
+ static void uninitialized_move_and_destroy(T* source_ptr, T* target_ptr, size_t size) {
+ if constexpr (std::is_trivially_copyable_v<T>) {
+ std::memcpy(target_ptr, source_ptr, size * sizeof(T));
+ } else {
+ std::uninitialized_move_n(source_ptr, size, target_ptr);
+ std::destroy_n(source_ptr, size);
+ }
+ }
+
+ /**
+ * @brief Reallocates all data when capacity changes.
+ *
+ * if new_capacity <= N chooses direct memory, otherwise indirect.
+ */
+ void realloc(size_t new_capacity) {
+ if (new_capacity <= N) {
+ // put everything into direct storage
+ if (is_direct()) {
+ // direct -> direct: nothing to do!
+ return;
+ }
+
+ // indirect -> direct
+ auto* storage = indirect();
+ uninitialized_move_and_destroy(storage->data(), direct_data(), storage->size());
+ set_direct_and_size(storage->size());
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ } else {
+ // put everything into indirect storage
+ auto* storage = detail::storage<T>::alloc(new_capacity);
+ if (is_direct()) {
+ // direct -> indirect
+ uninitialized_move_and_destroy(data<direction::direct>(), storage->data(), size<direction::direct>());
+ storage->size(size<direction::direct>());
+ } else {
+ // indirect -> indirect
+ uninitialized_move_and_destroy(data<direction::indirect>(), storage->data(), size<direction::indirect>());
+ storage->size(size<direction::indirect>());
+ auto* storage = indirect();
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ }
+ set_indirect(storage);
+ }
+ }
+
+ /**
+ * @brief Doubles starting_capacity until it is >= size_to_fit.
+ */
+ [[nodiscard]] static auto calculate_new_capacity(size_t size_to_fit, size_t starting_capacity) -> size_t {
+ if (size_to_fit > max_size()) {
+ // not enough space
+ throw std::bad_alloc();
+ }
+
+ if (size_to_fit == 0) {
+ // special handling for 0 so N==0 works
+ return starting_capacity;
+ }
+ // start with at least 1, so N==0 works
+ auto new_capacity = std::max<size_t>(1, starting_capacity);
+
+ // double capacity until its large enough, but make sure we don't overflow
+ while (new_capacity < size_to_fit && new_capacity * 2 > new_capacity) {
+ new_capacity *= 2;
+ }
+ if (new_capacity < size_to_fit) {
+ // got an overflow, set capacity to max
+ new_capacity = max_size();
+ }
+ return std::min(new_capacity, max_size());
+ }
+
+ template <direction D>
+ [[nodiscard]] auto capacity() const -> size_t {
+ if constexpr (D == direction::direct) {
+ return N;
+ } else {
+ return indirect()->capacity();
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto size() const -> size_t {
+ if constexpr (D == direction::direct) {
+ return direct_size();
+ } else {
+ return indirect()->size();
+ }
+ }
+
+ template <direction D>
+ void set_size(size_t s) {
+ if constexpr (D == direction::direct) {
+ set_direct_and_size(s);
+ } else {
+ indirect()->size(s);
+ }
+ }
+
+ void set_size(size_t s) {
+ if (is_direct()) {
+ set_size<direction::direct>(s);
+ } else {
+ set_size<direction::indirect>(s);
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto data() -> T* {
+ if constexpr (D == direction::direct) {
+ return direct_data();
+ } else {
+ return indirect()->data();
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto data() const -> T const* {
+ return const_cast<svector*>(this)->data<D>(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <direction D>
+ void pop_back() {
+ if constexpr (std::is_trivially_destructible_v<T>) {
+ set_size<D>(size<D>() - 1);
+ } else {
+ auto s = size<D>() - 1;
+ (data<D>() + s)->~T();
+ set_size<D>(s);
+ }
+ }
+
+ /**
+ * @brief We need variadic arguments so we can either use copy ctor or default ctor
+ */
+ template <direction D, class... Args>
+ void resize_after_reserve(size_t count, Args&&... args) {
+ auto current_size = size<D>();
+ if (current_size > count) {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ auto* d = data<D>();
+ std::destroy(d + count, d + current_size);
+ }
+ } else {
+ auto* d = data<D>();
+ for (auto ptr = d + current_size, end = d + count; ptr != end; ++ptr) {
+ new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...);
+ }
+ }
+ set_size<D>(count);
+ }
+
+ // Makes sure that to is not past the end iterator
+ template <direction D>
+ auto erase_checked_end(T const* cfrom, T const* to) -> T* {
+ auto* const erase_begin = const_cast<T*>(cfrom); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ auto* const container_end = data<D>() + size<D>();
+ auto* const erase_end = std::min(const_cast<T*>(to), container_end); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+
+ std::move(erase_end, container_end, erase_begin);
+ auto const num_erased = std::distance(erase_begin, erase_end);
+ std::destroy(container_end - num_erased, container_end);
+ set_size<D>(size<D>() - num_erased);
+ return erase_begin;
+ }
+
+ template <typename It>
+ void assign(It first, It last, std::input_iterator_tag /*unused*/) {
+ clear();
+
+ // TODO this can be made faster, e.g. by setting size only when finished.
+ while (first != last) {
+ push_back(*first);
+ ++first;
+ }
+ }
+
+ template <typename It>
+ void assign(It first, It last, std::forward_iterator_tag /*unused*/) {
+ clear();
+
+ auto s = std::distance(first, last);
+ reserve(s);
+ std::uninitialized_copy(first, last, data());
+ set_size(s);
+ }
+
+ // precondition: all uninitialized
+ void do_move_assign(svector&& other) {
+ if (!other.is_direct()) {
+ // take other's memory, even when empty
+ set_indirect(other.indirect());
+ } else {
+ auto* other_ptr = other.data<direction::direct>();
+ auto s = other.size<direction::direct>();
+ auto* other_end = other_ptr + s;
+
+ std::uninitialized_move(other_ptr, other_end, data<direction::direct>());
+ std::destroy(other_ptr, other_end);
+ set_size(s);
+ }
+ other.set_direct_and_size(0);
+ }
+
+ /**
+ * @brief Shifts data [source_begin, source_end( to the right, starting on target_begin.
+ *
+ * Preconditions:
+ * * contiguous memory
+ * * source_begin <= target_begin
+ * * source_end onwards is uninitialized memory
+ *
+ * Destroys then empty elements in [source_begin, source_end(
+ */
+ static void shift_right(T* source_begin, T* source_end, T* target_begin) {
+ // 1. uninitialized moves
+ auto const num_moves = std::distance(source_begin, source_end);
+ auto const target_end = target_begin + num_moves;
+ auto const num_uninitialized_move = std::min(num_moves, std::distance(source_end, target_end));
+ std::uninitialized_move(source_end - num_uninitialized_move, source_end, target_end - num_uninitialized_move);
+ std::move_backward(source_begin, source_end - num_uninitialized_move, target_end - num_uninitialized_move);
+ std::destroy(source_begin, std::min(source_end, target_begin));
+ }
+
+ template <direction D>
+ [[nodiscard]] auto make_uninitialized_space_new(size_t s, T* p, size_t count) -> T* {
+ auto target = svector();
+ // we know target is indirect because we're increasing capacity
+ target.reserve(s + count);
+
+ // move everything [begin, pos[
+ auto* target_pos = std::uninitialized_move(data<D>(), p, target.template data<direction::indirect>());
+
+ // move everything [pos, end]
+ std::uninitialized_move(p, data<D>() + s, target_pos + count);
+
+ target.template set_size<direction::indirect>(s + count);
+ *this = std::move(target);
+ return target_pos;
+ }
+
+ template <direction D>
+ [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* {
+ auto* const p = const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ auto s = size<D>();
+ if (s + count > capacity<D>()) {
+ return make_uninitialized_space_new<D>(s, p, count);
+ }
+
+ shift_right(p, data<D>() + s, p + count);
+ set_size<D>(s + count);
+ return p;
+ }
+
+ // makes space for uninitialized data of cout elements. Also updates size.
+ [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* {
+ if (is_direct()) {
+ return make_uninitialized_space<direction::direct>(pos, count);
+ }
+ return make_uninitialized_space<direction::indirect>(pos, count);
+ }
+
+ void destroy() {
+ auto const is_dir = is_direct();
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ T* ptr = nullptr;
+ size_t s = 0;
+ if (is_dir) {
+ ptr = data<direction::direct>();
+ s = size<direction::direct>();
+ } else {
+ ptr = data<direction::indirect>();
+ s = size<direction::indirect>();
+ }
+ std::destroy_n(ptr, s);
+ }
+ if (!is_dir) {
+ auto* storage = indirect();
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ }
+ set_direct_and_size(0);
+ }
+
+ // performs a const_cast so we don't need this implementation twice
+ template <direction D>
+ auto at(size_t idx) -> T& {
+ if (idx >= size<D>()) {
+ throw std::out_of_range{"svector: idx out of range"};
+ }
+ auto* ptr = const_cast<T*>(data<D>() + idx); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ return *ptr;
+ } // LCOV_EXCL_LINE why is this single } marked as not covered? gcov bug?
+
+public:
+ using value_type = T;
+ using size_type = size_t;
+ using difference_type = std::ptrdiff_t;
+ using reference = value_type&;
+ using const_reference = value_type const&;
+ using pointer = T*;
+ using const_pointer = T const*;
+ using iterator = T*;
+ using const_iterator = T const*;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+ svector() {
+ set_direct_and_size(0);
+ }
+
+ svector(size_t count, T const& value)
+ : svector() {
+ resize(count, value);
+ }
+
+ explicit svector(size_t count)
+ : svector() {
+ reserve(count);
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count);
+ } else {
+ resize_after_reserve<direction::indirect>(count);
+ }
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ svector(InputIt first, InputIt last)
+ : svector() {
+ assign(first, last);
+ }
+
+ svector(svector const& other)
+ : svector() {
+ auto s = other.size();
+ reserve(s);
+ std::uninitialized_copy(other.begin(), other.end(), begin());
+ set_size(s);
+ }
+
+ svector(svector&& other) noexcept
+ : svector() {
+ do_move_assign(std::move(other));
+ }
+
+ svector(std::initializer_list<T> init)
+ : svector(init.begin(), init.end()) {}
+
+ ~svector() {
+ destroy();
+ }
+
+ void assign(size_t count, T const& value) {
+ clear();
+ resize(count, value);
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ void assign(InputIt first, InputIt last) {
+ assign(first, last, typename std::iterator_traits<InputIt>::iterator_category());
+ }
+
+ void assign(std::initializer_list<T> l) {
+ assign(l.begin(), l.end());
+ }
+
+ auto operator=(svector const& other) -> svector& {
+ if (&other == this) {
+ return *this;
+ }
+
+ assign(other.begin(), other.end());
+ return *this;
+ }
+
+ auto operator=(svector&& other) noexcept -> svector& {
+ if (&other == this) {
+ // It doesn't seem to be required to do self-check, but let's do it anyways to be safe
+ return *this;
+ }
+ destroy();
+ do_move_assign(std::move(other));
+ return *this;
+ }
+
+ auto operator=(std::initializer_list<T> l) -> svector& {
+ assign(l.begin(), l.end());
+ return *this;
+ }
+
+ void resize(size_t count) {
+ if (count > capacity()) {
+ reserve(count);
+ }
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count);
+ } else {
+ resize_after_reserve<direction::indirect>(count);
+ }
+ }
+
+ void resize(size_t count, T const& value) {
+ if (count > capacity()) {
+ reserve(count);
+ }
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count, value);
+ } else {
+ resize_after_reserve<direction::indirect>(count, value);
+ }
+ }
+
+ void reserve(size_t s) {
+ auto old_capacity = capacity();
+ auto new_capacity = calculate_new_capacity(s, old_capacity);
+ if (new_capacity > old_capacity) {
+ realloc(new_capacity);
+ }
+ }
+
+ [[nodiscard]] auto capacity() const -> size_t {
+ if (is_direct()) {
+ return capacity<direction::direct>();
+ }
+ return capacity<direction::indirect>();
+ }
+
+ [[nodiscard]] auto size() const -> size_t {
+ if (is_direct()) {
+ return size<direction::direct>();
+ }
+ return size<direction::indirect>();
+ }
+
+ [[nodiscard]] auto data() -> T* {
+ if (is_direct()) {
+ return direct_data();
+ }
+ return indirect()->data();
+ }
+
+ [[nodiscard]] auto data() const -> T const* {
+ return const_cast<svector*>(this)->data(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <class... Args>
+ auto emplace_back(Args&&... args) -> T& {
+ size_t c; // NOLINT(cppcoreguidelines-init-variables)
+ size_t s; // NOLINT(cppcoreguidelines-init-variables)
+ bool is_dir = is_direct();
+ if (is_dir) {
+ c = capacity<direction::direct>();
+ s = size<direction::direct>();
+ } else {
+ c = capacity<direction::indirect>();
+ s = size<direction::indirect>();
+ }
+
+ if (s == c) {
+ auto new_capacity = calculate_new_capacity(s + 1, c);
+ realloc(new_capacity);
+ // reallocation happened, so we definitely are now in indirect mode
+ is_dir = false;
+ }
+
+ T* ptr; // NOLINT(cppcoreguidelines-init-variables)
+ if (is_dir) {
+ ptr = data<direction::direct>() + s;
+ set_size<direction::direct>(s + 1);
+ } else {
+ ptr = data<direction::indirect>() + s;
+ set_size<direction::indirect>(s + 1);
+ }
+ return *new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...);
+ }
+
+ void push_back(T const& value) {
+ emplace_back(value);
+ }
+
+ void push_back(T&& value) {
+ emplace_back(std::move(value));
+ }
+
+ [[nodiscard]] auto operator[](size_t idx) const -> T const& {
+ return *(data() + idx);
+ }
+
+ [[nodiscard]] auto operator[](size_t idx) -> T& {
+ return *(data() + idx);
+ }
+
+ auto at(size_t idx) -> T& {
+ if (is_direct()) {
+ return at<direction::direct>(idx);
+ }
+ return at<direction::indirect>(idx);
+ }
+
+ auto at(size_t idx) const -> T const& {
+ return const_cast<svector*>(this)->at(idx); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ [[nodiscard]] auto begin() const -> T const* {
+ return data();
+ }
+
+ [[nodiscard]] auto cbegin() const -> T const* {
+ return begin();
+ }
+
+ [[nodiscard]] auto begin() -> T* {
+ return data();
+ }
+
+ [[nodiscard]] auto end() -> T* {
+ if (is_direct()) {
+ return data<direction::direct>() + size<direction::direct>();
+ }
+ return data<direction::indirect>() + size<direction::indirect>();
+ }
+
+ [[nodiscard]] auto end() const -> T const* {
+ return const_cast<svector*>(this)->end(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ [[nodiscard]] auto cend() const -> T const* {
+ return end();
+ }
+
+ [[nodiscard]] auto rbegin() -> reverse_iterator {
+ return reverse_iterator{end()};
+ }
+
+ [[nodiscard]] auto rbegin() const -> const_reverse_iterator {
+ return crbegin();
+ }
+
+ [[nodiscard]] auto crbegin() const -> const_reverse_iterator {
+ return const_reverse_iterator{end()};
+ }
+
+ [[nodiscard]] auto rend() -> reverse_iterator {
+ return reverse_iterator{begin()};
+ }
+
+ [[nodiscard]] auto rend() const -> const_reverse_iterator {
+ return crend();
+ }
+
+ [[nodiscard]] auto crend() const -> const_reverse_iterator {
+ return const_reverse_iterator{begin()};
+ }
+
+ [[nodiscard]] auto front() const -> T const& {
+ return *data();
+ }
+
+ [[nodiscard]] auto front() -> T& {
+ return *data();
+ }
+
+ [[nodiscard]] auto back() -> T& {
+ if (is_direct()) {
+ return *(data<direction::direct>() + size<direction::direct>() - 1);
+ }
+ return *(data<direction::indirect>() + size<direction::indirect>() - 1);
+ }
+
+ [[nodiscard]] auto back() const -> T const& {
+ return const_cast<svector*>(this)->back(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ void clear() {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ std::destroy(begin(), end());
+ }
+
+ if (is_direct()) {
+ set_size<direction::direct>(0);
+ } else {
+ set_size<direction::indirect>(0);
+ }
+ }
+
+ [[nodiscard]] auto empty() const -> bool {
+ return 0U == size();
+ }
+
+ void pop_back() {
+ if (is_direct()) {
+ pop_back<direction::direct>();
+ } else {
+ pop_back<direction::indirect>();
+ }
+ }
+
+ [[nodiscard]] static auto max_size() -> size_t {
+ return std::numeric_limits<std::ptrdiff_t>::max();
+ }
+
+ void swap(svector& other) {
+ // TODO we could try to do the minimum number of moves
+ std::swap(*this, other);
+ }
+
+ void shrink_to_fit() {
+ // per the standard we wouldn't need to do anything here. But since we are so nice,
+ // let's do the shrink.
+ auto const c = capacity();
+ auto const s = size();
+ if (s >= c) {
+ return;
+ }
+
+ auto new_capacity = calculate_new_capacity(s, N);
+ if (new_capacity == c) {
+ // nothing change!
+ return;
+ }
+
+ realloc(new_capacity);
+ }
+
+ template <class... Args>
+ auto emplace(const_iterator pos, Args&&... args) -> iterator {
+ auto* p = make_uninitialized_space(pos, 1);
+ return new (static_cast<void*>(p)) T(std::forward<Args>(args)...);
+ }
+
+ auto insert(const_iterator pos, T const& value) -> iterator {
+ return emplace(pos, value);
+ }
+
+ auto insert(const_iterator pos, T&& value) -> iterator {
+ return emplace(pos, std::move(value));
+ }
+
+ auto insert(const_iterator pos, size_t count, T const& value) -> iterator {
+ auto* p = make_uninitialized_space(pos, count);
+ std::uninitialized_fill_n(p, count, value);
+ return p;
+ }
+
+ template <typename It>
+ auto insert(const_iterator pos, It first, It last, std::input_iterator_tag /*unused*/) {
+ if (!(first != last)) {
+ return const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ // just input_iterator_tag makes this very slow. Let's do the same as the STL.
+ if (pos == end()) {
+ auto s = size();
+ while (first != last) {
+ emplace_back(*first);
+ ++first;
+ }
+ return begin() + s;
+ }
+
+ auto tmp = svector(first, last);
+ return insert(pos, std::make_move_iterator(tmp.begin()), std::make_move_iterator(tmp.end()));
+ }
+
+ template <typename It>
+ auto insert(const_iterator pos, It first, It last, std::forward_iterator_tag /*unused*/) -> iterator {
+ auto* p = make_uninitialized_space(pos, std::distance(first, last));
+ std::uninitialized_copy(first, last, p);
+ return p;
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ auto insert(const_iterator pos, InputIt first, InputIt last) -> iterator {
+ return insert(pos, first, last, typename std::iterator_traits<InputIt>::iterator_category());
+ }
+
+ auto insert(const_iterator pos, std::initializer_list<T> l) -> iterator {
+ return insert(pos, l.begin(), l.end());
+ }
+
+ auto erase(const_iterator pos) -> iterator {
+ return erase(pos, pos + 1);
+ }
+
+ auto erase(const_iterator first, const_iterator last) -> iterator {
+ if (is_direct()) {
+ return erase_checked_end<direction::direct>(first, last);
+ }
+ return erase_checked_end<direction::indirect>(first, last);
+ }
+};
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator==(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::equal(a.begin(), a.end(), b.begin(), b.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator!=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a == b);
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator<(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator>=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a < b);
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator>(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::lexicographical_compare(b.begin(), b.end(), a.begin(), a.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator<=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a > b);
+}
+
+} // namespace ANKERL_SVECTOR_NAMESPACE
+} // namespace ankerl
+
+// NOLINTNEXTLINE(cert-dcl58-cpp)
+namespace std {
+inline namespace ANKERL_SVECTOR_NAMESPACE {
+
+template <class T, size_t N, class U>
+constexpr auto erase(ankerl::svector<T, N>& sv, U const& value) -> typename ankerl::svector<T, N>::size_type {
+ auto* removed_begin = std::remove(sv.begin(), sv.end(), value);
+ auto num_removed = std::distance(removed_begin, sv.end());
+ sv.erase(removed_begin, sv.end());
+ return num_removed;
+}
+
+template <class T, size_t N, class Pred>
+constexpr auto erase_if(ankerl::svector<T, N>& sv, Pred pred) -> typename ankerl::svector<T, N>::size_type {
+ auto* removed_begin = std::remove_if(sv.begin(), sv.end(), pred);
+ auto num_removed = std::distance(removed_begin, sv.end());
+ sv.erase(removed_begin, sv.end());
+ return num_removed;
+}
+
+} // namespace ANKERL_SVECTOR_NAMESPACE
+} // namespace std
+
+#endif
diff --git a/contrib/ankerl/unordered_dense.h b/contrib/ankerl/unordered_dense.h
new file mode 100644
index 0000000..2aaacd6
--- /dev/null
+++ b/contrib/ankerl/unordered_dense.h
@@ -0,0 +1,2032 @@
+///////////////////////// ankerl::unordered_dense::{map, set} /////////////////////////
+
+// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion.
+// Version 4.4.0
+// https://github.com/martinus/unordered_dense
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022-2023 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef ANKERL_UNORDERED_DENSE_H
+#define ANKERL_UNORDERED_DENSE_H
+
+// see https://semver.org/spec/v2.0.0.html
+#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes
+#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality
+#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes
+
+// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch)
+#define ANKERL_UNORDERED_DENSE_NAMESPACE \
+ ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \
+ ANKERL_UNORDERED_DENSE_VERSION_MAJOR, ANKERL_UNORDERED_DENSE_VERSION_MINOR, ANKERL_UNORDERED_DENSE_VERSION_PATCH)
+
+#if defined(_MSVC_LANG)
+# define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG
+#else
+# define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus
+#endif
+
+#if defined(__GNUC__)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__))
+#elif defined(_MSC_VER)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop))
+#endif
+
+// exceptions
+#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage)
+#else
+# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage)
+#endif
+#ifdef _MSC_VER
+# define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline)
+#else
+# define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline))
+#endif
+
+// defined in unordered_dense.cpp
+#if !defined(ANKERL_UNORDERED_DENSE_EXPORT)
+# define ANKERL_UNORDERED_DENSE_EXPORT
+#endif
+
+#if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L
+# error ankerl::unordered_dense requires C++17 or higher
+#else
+# include <array> // for array
+# include <cstdint> // for uint64_t, uint32_t, uint8_t, UINT64_C
+# include <cstring> // for size_t, memcpy, memset
+# include <functional> // for equal_to, hash
+# include <initializer_list> // for initializer_list
+# include <iterator> // for pair, distance
+# include <limits> // for numeric_limits
+# include <memory> // for allocator, allocator_traits, shared_ptr
+# include <optional> // for optional
+# include <stdexcept> // for out_of_range
+# include <string> // for basic_string
+# include <string_view> // for basic_string_view, hash
+# include <tuple> // for forward_as_tuple
+# include <type_traits> // for enable_if_t, declval, conditional_t, ena...
+# include <utility> // for forward, exchange, pair, as_const, piece...
+# include <vector> // for vector
+# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0
+# include <cstdlib> // for abort
+# endif
+
+# if defined(__has_include)
+# if __has_include(<memory_resource>)
+# define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage)
+# include <memory_resource> // for polymorphic_allocator
+# elif __has_include(<experimental/memory_resource>)
+# define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage)
+# include <experimental/memory_resource> // for polymorphic_allocator
+# endif
+# endif
+
+# if defined(_MSC_VER) && defined(_M_X64)
+# include <intrin.h>
+# pragma intrinsic(_umul128)
+# endif
+
+# if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+# define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage)
+# else
+# define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
+# endif
+
+namespace ankerl::unordered_dense {
+inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE {
+
+namespace detail {
+
+# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS()
+
+// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
+// inlinings more difficult. Throws are also generally the slow path.
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() {
+ throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found");
+}
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() {
+ throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size");
+}
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() {
+ throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements");
+}
+
+# else
+
+[[noreturn]] inline void on_error_key_not_found() {
+ abort();
+}
+[[noreturn]] inline void on_error_bucket_overflow() {
+ abort();
+}
+[[noreturn]] inline void on_error_too_many_elements() {
+ abort();
+}
+
+# endif
+
+} // namespace detail
+
+// hash ///////////////////////////////////////////////////////////////////////
+
+// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash
+// No big-endian support (because different values on different machines don't matter),
+// hardcodes seed and the secret, reformats the code, and clang-tidy fixes.
+namespace detail::wyhash {
+
+inline void mum(uint64_t* a, uint64_t* b) {
+# if defined(__SIZEOF_INT128__)
+ __uint128_t r = *a;
+ r *= *b;
+ *a = static_cast<uint64_t>(r);
+ *b = static_cast<uint64_t>(r >> 64U);
+# elif defined(_MSC_VER) && defined(_M_X64)
+ *a = _umul128(*a, *b, b);
+# else
+ uint64_t ha = *a >> 32U;
+ uint64_t hb = *b >> 32U;
+ uint64_t la = static_cast<uint32_t>(*a);
+ uint64_t lb = static_cast<uint32_t>(*b);
+ uint64_t hi{};
+ uint64_t lo{};
+ uint64_t rh = ha * hb;
+ uint64_t rm0 = ha * lb;
+ uint64_t rm1 = hb * la;
+ uint64_t rl = la * lb;
+ uint64_t t = rl + (rm0 << 32U);
+ auto c = static_cast<uint64_t>(t < rl);
+ lo = t + (rm1 << 32U);
+ c += static_cast<uint64_t>(lo < t);
+ hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c;
+ *a = lo;
+ *b = hi;
+# endif
+}
+
+// multiply and xor mix function, aka MUM
+[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t {
+ mum(&a, &b);
+ return a ^ b;
+}
+
+// read functions. WARNING: we don't care about endianness, so results are different on big endian!
+[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t {
+ uint64_t v{};
+ std::memcpy(&v, p, 8U);
+ return v;
+}
+
+[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t {
+ uint32_t v{};
+ std::memcpy(&v, p, 4);
+ return v;
+}
+
+// reads 1, 2, or 3 bytes
+[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t {
+ return (static_cast<uint64_t>(p[0]) << 16U) | (static_cast<uint64_t>(p[k >> 1U]) << 8U) | p[k - 1];
+}
+
+[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t {
+ static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f),
+ UINT64_C(0xe7037ed1a0b428db),
+ UINT64_C(0x8ebc6af09c88c6e3),
+ UINT64_C(0x589965cc75374cc3)};
+
+ auto const* p = static_cast<uint8_t const*>(key);
+ uint64_t seed = secret[0];
+ uint64_t a{};
+ uint64_t b{};
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) {
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) {
+ a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U));
+ b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U));
+ } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) {
+ a = r3(p, len);
+ b = 0;
+ } else {
+ a = 0;
+ b = 0;
+ }
+ } else {
+ size_t i = len;
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) {
+ uint64_t see1 = seed;
+ uint64_t see2 = seed;
+ do {
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
+ see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1);
+ see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2);
+ p += 48;
+ i -= 48;
+ } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48));
+ seed ^= see1 ^ see2;
+ }
+ while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) {
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
+ i -= 16;
+ p += 16;
+ }
+ a = r8(p + i - 16);
+ b = r8(p + i - 8);
+ }
+
+ return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed));
+}
+
+[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t {
+ return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15));
+}
+
+} // namespace detail::wyhash
+
+ANKERL_UNORDERED_DENSE_EXPORT template <typename T, typename Enable = void>
+struct hash {
+ auto operator()(T const& obj) const noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>())))
+ -> uint64_t {
+ return std::hash<T>{}(obj);
+ }
+};
+
+template <typename CharT>
+struct hash<std::basic_string<CharT>> {
+ using is_avalanching = void;
+ auto operator()(std::basic_string<CharT> const& str) const noexcept -> uint64_t {
+ return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size());
+ }
+};
+
+template <typename CharT>
+struct hash<std::basic_string_view<CharT>> {
+ using is_avalanching = void;
+ auto operator()(std::basic_string_view<CharT> const& sv) const noexcept -> uint64_t {
+ return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size());
+ }
+};
+
+template <class T>
+struct hash<T*> {
+ using is_avalanching = void;
+ auto operator()(T* ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr));
+ }
+};
+
+template <class T>
+struct hash<std::unique_ptr<T>> {
+ using is_avalanching = void;
+ auto operator()(std::unique_ptr<T> const& ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
+ }
+};
+
+template <class T>
+struct hash<std::shared_ptr<T>> {
+ using is_avalanching = void;
+ auto operator()(std::shared_ptr<T> const& ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
+ }
+};
+
+template <typename Enum>
+struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
+ using is_avalanching = void;
+ auto operator()(Enum e) const noexcept -> uint64_t {
+ using underlying = typename std::underlying_type_t<Enum>;
+ return detail::wyhash::hash(static_cast<underlying>(e));
+ }
+};
+
+template <typename... Args>
+struct tuple_hash_helper {
+ // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest.
+ // If it isn't an integral we need to hash it.
+ template <typename Arg>
+ [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t {
+ if constexpr (std::is_integral_v<Arg> || std::is_enum_v<Arg>) {
+ return static_cast<uint64_t>(arg);
+ } else {
+ return hash<Arg>{}(arg);
+ }
+ }
+
+ [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t {
+ return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69});
+ }
+
+ // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If
+ // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized
+ // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer.
+ template <typename T, std::size_t... Idx>
+ [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence<Idx...>) noexcept -> uint64_t {
+ auto h = uint64_t{};
+ ((h = mix64(h, to64(std::get<Idx>(t)))), ...);
+ return h;
+ }
+};
+
+template <typename... Args>
+struct hash<std::tuple<Args...>> : tuple_hash_helper<Args...> {
+ using is_avalanching = void;
+ auto operator()(std::tuple<Args...> const& t) const noexcept -> uint64_t {
+ return tuple_hash_helper<Args...>::calc_hash(t, std::index_sequence_for<Args...>{});
+ }
+};
+
+template <typename A, typename B>
+struct hash<std::pair<A, B>> : tuple_hash_helper<A, B> {
+ using is_avalanching = void;
+ auto operator()(std::pair<A, B> const& t) const noexcept -> uint64_t {
+ return tuple_hash_helper<A, B>::calc_hash(t, std::index_sequence_for<A, B>{});
+ }
+};
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \
+ template <> \
+ struct hash<T> { \
+ using is_avalanching = void; \
+ auto operator()(T const& obj) const noexcept -> uint64_t { \
+ return detail::wyhash::hash(static_cast<uint64_t>(obj)); \
+ } \
+ }
+
+# if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wuseless-cast"
+# endif
+// see https://en.cppreference.com/w/cpp/utility/hash
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char);
+# if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t)
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t);
+# endif
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long);
+
+# if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic pop
+# endif
+
+// bucket_type //////////////////////////////////////////////////////////
+
+namespace bucket_type {
+
+struct standard {
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
+
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
+ uint32_t m_value_idx; // index into the m_values vector.
+};
+
+ANKERL_UNORDERED_DENSE_PACK(struct big {
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
+
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
+ size_t m_value_idx; // index into the m_values vector.
+});
+
+} // namespace bucket_type
+
+namespace detail {
+
+struct nonesuch {};
+
+template <class Default, class AlwaysVoid, template <class...> class Op, class... Args>
+struct detector {
+ using value_t = std::false_type;
+ using type = Default;
+};
+
+template <class Default, template <class...> class Op, class... Args>
+struct detector<Default, std::void_t<Op<Args...>>, Op, Args...> {
+ using value_t = std::true_type;
+ using type = Op<Args...>;
+};
+
+template <template <class...> class Op, class... Args>
+using is_detected = typename detail::detector<detail::nonesuch, void, Op, Args...>::value_t;
+
+template <template <class...> class Op, class... Args>
+constexpr bool is_detected_v = is_detected<Op, Args...>::value;
+
+template <typename T>
+using detect_avalanching = typename T::is_avalanching;
+
+template <typename T>
+using detect_is_transparent = typename T::is_transparent;
+
+template <typename T>
+using detect_iterator = typename T::iterator;
+
+template <typename T>
+using detect_reserve = decltype(std::declval<T&>().reserve(size_t{}));
+
+// enable_if helpers
+
+template <typename Mapped>
+constexpr bool is_map_v = !std::is_void_v<Mapped>;
+
+// clang-format off
+template <typename Hash, typename KeyEqual>
+constexpr bool is_transparent_v = is_detected_v<detect_is_transparent, Hash> && is_detected_v<detect_is_transparent, KeyEqual>;
+// clang-format on
+
+template <typename From, typename To1, typename To2>
+constexpr bool is_neither_convertible_v = !std::is_convertible_v<From, To1> && !std::is_convertible_v<From, To2>;
+
+template <typename T>
+constexpr bool has_reserve = is_detected_v<detect_reserve, T>;
+
+// base type for map has mapped_type
+template <class T>
+struct base_table_type_map {
+ using mapped_type = T;
+};
+
+// base type for set doesn't have mapped_type
+struct base_table_type_set {};
+
+} // namespace detail
+
+// Very much like std::deque, but faster for indexing (in most cases). As of now this doesn't implement the full std::vector
+// API, but merely what's necessary to work as an underlying container for ankerl::unordered_dense::{map, set}.
+// It allocates blocks of equal size and puts them into the m_blocks vector. That means it can grow simply by adding a new
+// block to the back of m_blocks, and doesn't double its size like an std::vector. The disadvantage is that memory is not
+// linear and thus there is one more indirection necessary for indexing.
+template <typename T, typename Allocator = std::allocator<T>, size_t MaxSegmentSizeBytes = 4096>
+class segmented_vector {
+ template <bool IsConst>
+ class iter_t;
+
+public:
+ using allocator_type = Allocator;
+ using pointer = typename std::allocator_traits<allocator_type>::pointer;
+ using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
+ using difference_type = typename std::allocator_traits<allocator_type>::difference_type;
+ using value_type = T;
+ using size_type = std::size_t;
+ using reference = T&;
+ using const_reference = T const&;
+ using iterator = iter_t<false>;
+ using const_iterator = iter_t<true>;
+
+private:
+ using vec_alloc = typename std::allocator_traits<Allocator>::template rebind_alloc<pointer>;
+ std::vector<pointer, vec_alloc> m_blocks{};
+ size_t m_size{};
+
+ // Calculates the maximum number for x in (s << x) <= max_val
+ static constexpr auto num_bits_closest(size_t max_val, size_t s) -> size_t {
+ auto f = size_t{0};
+ while (s << (f + 1) <= max_val) {
+ ++f;
+ }
+ return f;
+ }
+
+ using self_t = segmented_vector<T, Allocator, MaxSegmentSizeBytes>;
+ static constexpr auto num_bits = num_bits_closest(MaxSegmentSizeBytes, sizeof(T));
+ static constexpr auto num_elements_in_block = 1U << num_bits;
+ static constexpr auto mask = num_elements_in_block - 1U;
+
+ /**
+ * Iterator class doubles as const_iterator and iterator
+ */
+ template <bool IsConst>
+ class iter_t {
+ using ptr_t = typename std::conditional_t<IsConst, segmented_vector::const_pointer const*, segmented_vector::pointer*>;
+ ptr_t m_data{};
+ size_t m_idx{};
+
+ template <bool B>
+ friend class iter_t;
+
+ public:
+ using difference_type = segmented_vector::difference_type;
+ using value_type = T;
+ using reference = typename std::conditional_t<IsConst, value_type const&, value_type&>;
+ using pointer = typename std::conditional_t<IsConst, segmented_vector::const_pointer, segmented_vector::pointer>;
+ using iterator_category = std::forward_iterator_tag;
+
+ iter_t() noexcept = default;
+
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
+ constexpr iter_t(iter_t<OtherIsConst> const& other) noexcept
+ : m_data(other.m_data)
+ , m_idx(other.m_idx) {}
+
+ constexpr iter_t(ptr_t data, size_t idx) noexcept
+ : m_data(data)
+ , m_idx(idx) {}
+
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ constexpr auto operator=(iter_t<OtherIsConst> const& other) noexcept -> iter_t& {
+ m_data = other.m_data;
+ m_idx = other.m_idx;
+ return *this;
+ }
+
+ constexpr auto operator++() noexcept -> iter_t& {
+ ++m_idx;
+ return *this;
+ }
+
+ constexpr auto operator+(difference_type diff) noexcept -> iter_t {
+ return {m_data, static_cast<size_t>(static_cast<difference_type>(m_idx) + diff)};
+ }
+
+ template <bool OtherIsConst>
+ constexpr auto operator-(iter_t<OtherIsConst> const& other) noexcept -> difference_type {
+ return static_cast<difference_type>(m_idx) - static_cast<difference_type>(other.m_idx);
+ }
+
+ constexpr auto operator*() const noexcept -> reference {
+ return m_data[m_idx >> num_bits][m_idx & mask];
+ }
+
+ constexpr auto operator->() const noexcept -> pointer {
+ return &m_data[m_idx >> num_bits][m_idx & mask];
+ }
+
+ template <bool O>
+ constexpr auto operator==(iter_t<O> const& o) const noexcept -> bool {
+ return m_idx == o.m_idx;
+ }
+
+ template <bool O>
+ constexpr auto operator!=(iter_t<O> const& o) const noexcept -> bool {
+ return !(*this == o);
+ }
+ };
+
+ // slow path: need to allocate a new segment every once in a while
+ void increase_capacity() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ pointer block = std::allocator_traits<Allocator>::allocate(ba, num_elements_in_block);
+ m_blocks.push_back(block);
+ }
+
+ // Moves everything from other
+ void append_everything_from(segmented_vector&& other) {
+ reserve(size() + other.size());
+ for (auto&& o : other) {
+ emplace_back(std::move(o));
+ }
+ }
+
+ // Copies everything from other
+ void append_everything_from(segmented_vector const& other) {
+ reserve(size() + other.size());
+ for (auto const& o : other) {
+ emplace_back(o);
+ }
+ }
+
+ void dealloc() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ for (auto ptr : m_blocks) {
+ std::allocator_traits<Allocator>::deallocate(ba, ptr, num_elements_in_block);
+ }
+ }
+
+ [[nodiscard]] static constexpr auto calc_num_blocks_for_capacity(size_t capacity) {
+ return (capacity + num_elements_in_block - 1U) / num_elements_in_block;
+ }
+
+public:
+ segmented_vector() = default;
+
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
+ segmented_vector(Allocator alloc)
+ : m_blocks(vec_alloc(alloc)) {}
+
+ segmented_vector(segmented_vector&& other, Allocator alloc)
+ : segmented_vector(alloc) {
+ *this = std::move(other);
+ }
+
+ segmented_vector(segmented_vector const& other, Allocator alloc)
+ : m_blocks(vec_alloc(alloc)) {
+ append_everything_from(other);
+ }
+
+ segmented_vector(segmented_vector&& other) noexcept
+ : segmented_vector(std::move(other), get_allocator()) {}
+
+ segmented_vector(segmented_vector const& other) {
+ append_everything_from(other);
+ }
+
+ auto operator=(segmented_vector const& other) -> segmented_vector& {
+ if (this == &other) {
+ return *this;
+ }
+ clear();
+ append_everything_from(other);
+ return *this;
+ }
+
+ auto operator=(segmented_vector&& other) noexcept -> segmented_vector& {
+ clear();
+ dealloc();
+ if (other.get_allocator() == get_allocator()) {
+ m_blocks = std::move(other.m_blocks);
+ m_size = std::exchange(other.m_size, {});
+ } else {
+ // make sure to construct with other's allocator!
+ m_blocks = std::vector<pointer, vec_alloc>(vec_alloc(other.get_allocator()));
+ append_everything_from(std::move(other));
+ }
+ return *this;
+ }
+
+ ~segmented_vector() {
+ clear();
+ dealloc();
+ }
+
+ [[nodiscard]] constexpr auto size() const -> size_t {
+ return m_size;
+ }
+
+ [[nodiscard]] constexpr auto capacity() const -> size_t {
+ return m_blocks.size() * num_elements_in_block;
+ }
+
+ // Indexing is highly performance critical
+ [[nodiscard]] constexpr auto operator[](size_t i) const noexcept -> T const& {
+ return m_blocks[i >> num_bits][i & mask];
+ }
+
+ [[nodiscard]] constexpr auto operator[](size_t i) noexcept -> T& {
+ return m_blocks[i >> num_bits][i & mask];
+ }
+
+ [[nodiscard]] constexpr auto begin() -> iterator {
+ return {m_blocks.data(), 0U};
+ }
+ [[nodiscard]] constexpr auto begin() const -> const_iterator {
+ return {m_blocks.data(), 0U};
+ }
+ [[nodiscard]] constexpr auto cbegin() const -> const_iterator {
+ return {m_blocks.data(), 0U};
+ }
+
+ [[nodiscard]] constexpr auto end() -> iterator {
+ return {m_blocks.data(), m_size};
+ }
+ [[nodiscard]] constexpr auto end() const -> const_iterator {
+ return {m_blocks.data(), m_size};
+ }
+ [[nodiscard]] constexpr auto cend() const -> const_iterator {
+ return {m_blocks.data(), m_size};
+ }
+
+ [[nodiscard]] constexpr auto back() -> reference {
+ return operator[](m_size - 1);
+ }
+ [[nodiscard]] constexpr auto back() const -> const_reference {
+ return operator[](m_size - 1);
+ }
+
+ void pop_back() {
+ back().~T();
+ --m_size;
+ }
+
+ [[nodiscard]] auto empty() const {
+ return 0 == m_size;
+ }
+
+ void reserve(size_t new_capacity) {
+ m_blocks.reserve(calc_num_blocks_for_capacity(new_capacity));
+ while (new_capacity > capacity()) {
+ increase_capacity();
+ }
+ }
+
+ [[nodiscard]] auto get_allocator() const -> allocator_type {
+ return allocator_type{m_blocks.get_allocator()};
+ }
+
+ template <class... Args>
+ auto emplace_back(Args&&... args) -> reference {
+ if (m_size == capacity()) {
+ increase_capacity();
+ }
+ auto* ptr = static_cast<void*>(&operator[](m_size));
+ auto& ref = *new (ptr) T(std::forward<Args>(args)...);
+ ++m_size;
+ return ref;
+ }
+
+ void clear() {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ for (size_t i = 0, s = size(); i < s; ++i) {
+ operator[](i).~T();
+ }
+ }
+ m_size = 0;
+ }
+
+ void shrink_to_fit() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ auto num_blocks_required = calc_num_blocks_for_capacity(m_size);
+ while (m_blocks.size() > num_blocks_required) {
+ std::allocator_traits<Allocator>::deallocate(ba, m_blocks.back(), num_elements_in_block);
+ m_blocks.pop_back();
+ }
+ m_blocks.shrink_to_fit();
+ }
+};
+
+namespace detail {
+
+// This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set.
+template <class Key,
+ class T, // when void, treat it as a set.
+ class Hash,
+ class KeyEqual,
+ class AllocatorOrContainer,
+ class Bucket,
+ bool IsSegmented>
+class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, base_table_type_set> {
+ using underlying_value_type = typename std::conditional_t<is_map_v<T>, std::pair<Key, T>, Key>;
+ using underlying_container_type = std::conditional_t<IsSegmented,
+ segmented_vector<underlying_value_type, AllocatorOrContainer>,
+ std::vector<underlying_value_type, AllocatorOrContainer>>;
+
+public:
+ using value_container_type = std::
+ conditional_t<is_detected_v<detect_iterator, AllocatorOrContainer>, AllocatorOrContainer, underlying_container_type>;
+
+private:
+ using bucket_alloc =
+ typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
+ using bucket_alloc_traits = std::allocator_traits<bucket_alloc>;
+
+ static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets
+ static constexpr float default_max_load_factor = 0.8F;
+
+public:
+ using key_type = Key;
+ using value_type = typename value_container_type::value_type;
+ using size_type = typename value_container_type::size_type;
+ using difference_type = typename value_container_type::difference_type;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using allocator_type = typename value_container_type::allocator_type;
+ using reference = typename value_container_type::reference;
+ using const_reference = typename value_container_type::const_reference;
+ using pointer = typename value_container_type::pointer;
+ using const_pointer = typename value_container_type::const_pointer;
+ using const_iterator = typename value_container_type::const_iterator;
+ using iterator = std::conditional_t<is_map_v<T>, typename value_container_type::iterator, const_iterator>;
+ using bucket_type = Bucket;
+
+private:
+ using value_idx_type = decltype(Bucket::m_value_idx);
+ using dist_and_fingerprint_type = decltype(Bucket::m_dist_and_fingerprint);
+
+ static_assert(std::is_trivially_destructible_v<Bucket>, "assert there's no need to call destructor / std::destroy");
+ static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");
+
+ value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes.
+ using bucket_pointer = typename std::allocator_traits<bucket_alloc>::pointer;
+ bucket_pointer m_buckets{};
+ size_t m_num_buckets = 0;
+ size_t m_max_bucket_capacity = 0;
+ float m_max_load_factor = default_max_load_factor;
+ Hash m_hash{};
+ KeyEqual m_equal{};
+ uint8_t m_shifts = initial_shifts;
+
+ [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
+ return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets)
+ ? 0
+ : static_cast<value_idx_type>(bucket_idx + 1U);
+ }
+
+ // Helper to access bucket through pointer types
+ [[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& {
+ return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset));
+ }
+
+ // use the dist_inc and dist_dec functions so that uint16_t types work without warning
+ [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
+ return static_cast<dist_and_fingerprint_type>(x + Bucket::dist_inc);
+ }
+
+ [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
+ return static_cast<dist_and_fingerprint_type>(x - Bucket::dist_inc);
+ }
+
+ // The goal of mixed_hash is to always produce a high quality 64bit hash.
+ template <typename K>
+ [[nodiscard]] constexpr auto mixed_hash(K const& key) const -> uint64_t {
+ if constexpr (is_detected_v<detect_avalanching, Hash>) {
+ // we know that the hash is good because is_avalanching.
+ if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) {
+ // 32bit hash and is_avalanching => multiply with a constant to avalanche bits upwards
+ return m_hash(key) * UINT64_C(0x9ddfea08eb382d69);
+ } else {
+ // 64bit and is_avalanching => only use the hash itself.
+ return m_hash(key);
+ }
+ } else {
+ // not is_avalanching => apply wyhash
+ return wyhash::hash(m_hash(key));
+ }
+ }
+
+ [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type {
+ return Bucket::dist_inc | (static_cast<dist_and_fingerprint_type>(hash) & Bucket::fingerprint_mask);
+ }
+
+ [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type {
+ return static_cast<value_idx_type>(hash >> m_shifts);
+ }
+
+ [[nodiscard]] static constexpr auto get_key(value_type const& vt) -> key_type const& {
+ if constexpr (is_map_v<T>) {
+ return vt.first;
+ } else {
+ return vt;
+ }
+ }
+
+ template <typename K>
+ [[nodiscard]] auto next_while_less(K const& key) const -> Bucket {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint < at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+ return {dist_and_fingerprint, bucket_idx};
+ }
+
+ void place_and_shift_up(Bucket bucket, value_idx_type place) {
+ while (0 != at(m_buckets, place).m_dist_and_fingerprint) {
+ bucket = std::exchange(at(m_buckets, place), bucket);
+ bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint);
+ place = next(place);
+ }
+ at(m_buckets, place) = bucket;
+ }
+
+ [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t {
+ return (std::min)(max_bucket_count(), size_t{1} << (64U - shifts));
+ }
+
+ [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t {
+ auto shifts = initial_shifts;
+ while (shifts > 0 && static_cast<size_t>(static_cast<float>(calc_num_buckets(shifts)) * max_load_factor()) < s) {
+ --shifts;
+ }
+ return shifts;
+ }
+
+ // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS
+ void copy_buckets(table const& other) {
+ // assumes m_values has already the correct data copied over.
+ if (empty()) {
+ // when empty, at least allocate an initial buckets and clear them.
+ allocate_buckets_from_shift();
+ clear_buckets();
+ } else {
+ m_shifts = other.m_shifts;
+ allocate_buckets_from_shift();
+ std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count());
+ }
+ }
+
+ /**
+ * True when no element can be added any more without increasing the size
+ */
+ [[nodiscard]] auto is_full() const -> bool {
+ return size() > m_max_bucket_capacity;
+ }
+
+ void deallocate_buckets() {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ if (nullptr != m_buckets) {
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
+ m_buckets = nullptr;
+ }
+ m_num_buckets = 0;
+ m_max_bucket_capacity = 0;
+ }
+
+ void allocate_buckets_from_shift() {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ m_num_buckets = calc_num_buckets(m_shifts);
+ m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets);
+ if (m_num_buckets == max_bucket_count()) {
+ // reached the maximum, make sure we can use each bucket
+ m_max_bucket_capacity = max_bucket_count();
+ } else {
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor());
+ }
+ }
+
+ void clear_buckets() {
+ if (m_buckets != nullptr) {
+ std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count());
+ }
+ }
+
+ void clear_and_fill_buckets_from_values() {
+ clear_buckets();
+ for (value_idx_type value_idx = 0, end_idx = static_cast<value_idx_type>(m_values.size()); value_idx < end_idx;
+ ++value_idx) {
+ auto const& key = get_key(m_values[value_idx]);
+ auto [dist_and_fingerprint, bucket] = next_while_less(key);
+
+ // we know for certain that key has not yet been inserted, so no need to check it.
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket);
+ }
+ }
+
+ void increase_size() {
+ if (m_max_bucket_capacity == max_bucket_count()) {
+ // remove the value again, we can't add it!
+ m_values.pop_back();
+ on_error_bucket_overflow();
+ }
+ --m_shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+
+ template <typename Op>
+ void do_erase(value_idx_type bucket_idx, Op handle_erased_value) {
+ auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx;
+
+ // shift down until either empty or an element with correct spot is found
+ auto next_bucket_idx = next(bucket_idx);
+ while (at(m_buckets, next_bucket_idx).m_dist_and_fingerprint >= Bucket::dist_inc * 2) {
+ at(m_buckets, bucket_idx) = {dist_dec(at(m_buckets, next_bucket_idx).m_dist_and_fingerprint),
+ at(m_buckets, next_bucket_idx).m_value_idx};
+ bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx));
+ }
+ at(m_buckets, bucket_idx) = {};
+ handle_erased_value(std::move(m_values[value_idx_to_remove]));
+
+ // update m_values
+ if (value_idx_to_remove != m_values.size() - 1) {
+ // no luck, we'll have to replace the value with the last one and update the index accordingly
+ auto& val = m_values[value_idx_to_remove];
+ val = std::move(m_values.back());
+
+ // update the values_idx of the moved entry. No need to play the info game, just look until we find the values_idx
+ auto mh = mixed_hash(get_key(val));
+ bucket_idx = bucket_idx_from_hash(mh);
+
+ auto const values_idx_back = static_cast<value_idx_type>(m_values.size() - 1);
+ while (values_idx_back != at(m_buckets, bucket_idx).m_value_idx) {
+ bucket_idx = next(bucket_idx);
+ }
+ at(m_buckets, bucket_idx).m_value_idx = value_idx_to_remove;
+ }
+ m_values.pop_back();
+ }
+
+ template <typename K, typename Op>
+ auto do_erase_key(K&& key, Op handle_erased_value) -> size_t {
+ if (empty()) {
+ return 0;
+ }
+
+ auto [dist_and_fingerprint, bucket_idx] = next_while_less(key);
+
+ while (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ !m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ return 0;
+ }
+ do_erase(bucket_idx, handle_erased_value);
+ return 1;
+ }
+
+ template <class K, class M>
+ auto do_insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
+ auto it_isinserted = try_emplace(std::forward<K>(key), std::forward<M>(mapped));
+ if (!it_isinserted.second) {
+ it_isinserted.first->second = std::forward<M>(mapped);
+ }
+ return it_isinserted;
+ }
+
+ template <typename... Args>
+ auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args)
+ -> std::pair<iterator, bool> {
+
+ // emplace the new value. If that throws an exception, no harm done; index is still in a valid state
+ m_values.emplace_back(std::forward<Args>(args)...);
+
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
+ increase_size();
+ } else {
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ }
+
+ // place element and shift up until we find an empty spot
+ return {begin() + static_cast<difference_type>(value_idx), true};
+ }
+
+ template <typename K, typename... Args>
+ auto do_try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (true) {
+ auto* bucket = &at(m_buckets, bucket_idx);
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return {begin() + static_cast<difference_type>(bucket->m_value_idx), false};
+ }
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
+ return do_place_element(dist_and_fingerprint,
+ bucket_idx,
+ std::piecewise_construct,
+ std::forward_as_tuple(std::forward<K>(key)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+ }
+
+ template <typename K>
+ auto do_find(K const& key) -> iterator {
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) {
+ return end();
+ }
+
+ auto mh = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(mh);
+ auto bucket_idx = bucket_idx_from_hash(mh);
+ auto* bucket = &at(m_buckets, bucket_idx);
+
+ // unrolled loop. *Always* check a few directly, then enter the loop. This is faster.
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+
+ while (true) {
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
+ return end();
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+ }
+ }
+
+ template <typename K>
+ auto do_find(K const& key) const -> const_iterator {
+ return const_cast<table*>(this)->do_find(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto do_at(K const& key) -> Q& {
+ if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) {
+ return it->second;
+ }
+ on_error_key_not_found();
+ }
+
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto do_at(K const& key) const -> Q const& {
+ return const_cast<table*>(this)->at(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+public:
+ explicit table(size_t bucket_count,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc_or_container = allocator_type())
+ : m_values(alloc_or_container)
+ , m_hash(hash)
+ , m_equal(equal) {
+ if (0 != bucket_count) {
+ reserve(bucket_count);
+ } else {
+ allocate_buckets_from_shift();
+ clear_buckets();
+ }
+ }
+
+ table()
+ : table(0) {}
+
+ table(size_t bucket_count, allocator_type const& alloc)
+ : table(bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ table(size_t bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(bucket_count, hash, KeyEqual(), alloc) {}
+
+ explicit table(allocator_type const& alloc)
+ : table(0, Hash(), KeyEqual(), alloc) {}
+
+ template <class InputIt>
+ table(InputIt first,
+ InputIt last,
+ size_type bucket_count = 0,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc = allocator_type())
+ : table(bucket_count, hash, equal, alloc) {
+ insert(first, last);
+ }
+
+ template <class InputIt>
+ table(InputIt first, InputIt last, size_type bucket_count, allocator_type const& alloc)
+ : table(first, last, bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ template <class InputIt>
+ table(InputIt first, InputIt last, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(first, last, bucket_count, hash, KeyEqual(), alloc) {}
+
+ table(table const& other)
+ : table(other, other.m_values.get_allocator()) {}
+
+ table(table const& other, allocator_type const& alloc)
+ : m_values(other.m_values, alloc)
+ , m_max_load_factor(other.m_max_load_factor)
+ , m_hash(other.m_hash)
+ , m_equal(other.m_equal) {
+ copy_buckets(other);
+ }
+
+ table(table&& other) noexcept
+ : table(std::move(other), other.m_values.get_allocator()) {}
+
+ table(table&& other, allocator_type const& alloc) noexcept
+ : m_values(alloc) {
+ *this = std::move(other);
+ }
+
+ table(std::initializer_list<value_type> ilist,
+ size_t bucket_count = 0,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc = allocator_type())
+ : table(bucket_count, hash, equal, alloc) {
+ insert(ilist);
+ }
+
+ table(std::initializer_list<value_type> ilist, size_type bucket_count, allocator_type const& alloc)
+ : table(ilist, bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(init, bucket_count, hash, KeyEqual(), alloc) {}
+
+ ~table() {
+ if (nullptr != m_buckets) {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
+ }
+ }
+
+ auto operator=(table const& other) -> table& {
+ if (&other != this) {
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
+ m_values = other.m_values;
+ m_max_load_factor = other.m_max_load_factor;
+ m_hash = other.m_hash;
+ m_equal = other.m_equal;
+ m_shifts = initial_shifts;
+ copy_buckets(other);
+ }
+ return *this;
+ }
+
+ auto operator=(table&& other) noexcept(noexcept(std::is_nothrow_move_assignable_v<value_container_type> &&
+ std::is_nothrow_move_assignable_v<Hash> &&
+ std::is_nothrow_move_assignable_v<KeyEqual>)) -> table& {
+ if (&other != this) {
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
+ m_values = std::move(other.m_values);
+ other.m_values.clear();
+
+ // we can only reuse m_buckets when both maps have the same allocator!
+ if (get_allocator() == other.get_allocator()) {
+ m_buckets = std::exchange(other.m_buckets, nullptr);
+ m_num_buckets = std::exchange(other.m_num_buckets, 0);
+ m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
+ m_shifts = std::exchange(other.m_shifts, initial_shifts);
+ m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
+ m_hash = std::exchange(other.m_hash, {});
+ m_equal = std::exchange(other.m_equal, {});
+ other.allocate_buckets_from_shift();
+ other.clear_buckets();
+ } else {
+ // set max_load_factor *before* copying the other's buckets, so we have the same
+ // behavior
+ m_max_load_factor = other.m_max_load_factor;
+
+ // copy_buckets sets m_buckets, m_num_buckets, m_max_bucket_capacity, m_shifts
+ copy_buckets(other);
+ // clear's the other's buckets so other is now already usable.
+ other.clear_buckets();
+ m_hash = other.m_hash;
+ m_equal = other.m_equal;
+ }
+ // map "other" is now already usable, it's empty.
+ }
+ return *this;
+ }
+
+ auto operator=(std::initializer_list<value_type> ilist) -> table& {
+ clear();
+ insert(ilist);
+ return *this;
+ }
+
+ auto get_allocator() const noexcept -> allocator_type {
+ return m_values.get_allocator();
+ }
+
+ // iterators //////////////////////////////////////////////////////////////
+
+ auto begin() noexcept -> iterator {
+ return m_values.begin();
+ }
+
+ auto begin() const noexcept -> const_iterator {
+ return m_values.begin();
+ }
+
+ auto cbegin() const noexcept -> const_iterator {
+ return m_values.cbegin();
+ }
+
+ auto end() noexcept -> iterator {
+ return m_values.end();
+ }
+
+ auto cend() const noexcept -> const_iterator {
+ return m_values.cend();
+ }
+
+ auto end() const noexcept -> const_iterator {
+ return m_values.end();
+ }
+
+ // capacity ///////////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto empty() const noexcept -> bool {
+ return m_values.empty();
+ }
+
+ [[nodiscard]] auto size() const noexcept -> size_t {
+ return m_values.size();
+ }
+
+ [[nodiscard]] static constexpr auto max_size() noexcept -> size_t {
+ if constexpr ((std::numeric_limits<value_idx_type>::max)() == (std::numeric_limits<size_t>::max)()) {
+ return size_t{1} << (sizeof(value_idx_type) * 8 - 1);
+ } else {
+ return size_t{1} << (sizeof(value_idx_type) * 8);
+ }
+ }
+
+ // modifiers //////////////////////////////////////////////////////////////
+
+ void clear() {
+ m_values.clear();
+ clear_buckets();
+ }
+
+ auto insert(value_type const& value) -> std::pair<iterator, bool> {
+ return emplace(value);
+ }
+
+ auto insert(value_type&& value) -> std::pair<iterator, bool> {
+ return emplace(std::move(value));
+ }
+
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
+ auto insert(P&& value) -> std::pair<iterator, bool> {
+ return emplace(std::forward<P>(value));
+ }
+
+ auto insert(const_iterator /*hint*/, value_type const& value) -> iterator {
+ return insert(value).first;
+ }
+
+ auto insert(const_iterator /*hint*/, value_type&& value) -> iterator {
+ return insert(std::move(value)).first;
+ }
+
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
+ auto insert(const_iterator /*hint*/, P&& value) -> iterator {
+ return insert(std::forward<P>(value)).first;
+ }
+
+ template <class InputIt>
+ void insert(InputIt first, InputIt last) {
+ while (first != last) {
+ insert(*first);
+ ++first;
+ }
+ }
+
+ void insert(std::initializer_list<value_type> ilist) {
+ insert(ilist.begin(), ilist.end());
+ }
+
+ // nonstandard API: *this is emptied.
+ // Also see "A Standard flat_map" https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p0429r9.pdf
+ auto extract() && -> value_container_type {
+ return std::move(m_values);
+ }
+
+ // nonstandard API:
+ // Discards the internally held container and replaces it with the one passed. Erases non-unique elements.
+ auto replace(value_container_type&& container) {
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(container.size() > max_size())) {
+ on_error_too_many_elements();
+ }
+ auto shifts = calc_shifts_for_size(container.size());
+ if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ }
+ clear_buckets();
+
+ m_values = std::move(container);
+
+ // can't use clear_and_fill_buckets_from_values() because container elements might not be unique
+ auto value_idx = value_idx_type{};
+
+ // loop until we reach the end of the container. duplicated entries will be replaced with back().
+ while (value_idx != static_cast<value_idx_type>(m_values.size())) {
+ auto const& key = get_key(m_values[value_idx]);
+
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ bool key_found = false;
+ while (true) {
+ auto const& bucket = at(m_buckets, bucket_idx);
+ if (dist_and_fingerprint > bucket.m_dist_and_fingerprint) {
+ break;
+ }
+ if (dist_and_fingerprint == bucket.m_dist_and_fingerprint &&
+ m_equal(key, get_key(m_values[bucket.m_value_idx]))) {
+ key_found = true;
+ break;
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ if (key_found) {
+ if (value_idx != static_cast<value_idx_type>(m_values.size() - 1)) {
+ m_values[value_idx] = std::move(m_values.back());
+ }
+ m_values.pop_back();
+ } else {
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ ++value_idx;
+ }
+ }
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(Key const& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(key, std::forward<M>(mapped));
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(Key&& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped));
+ }
+
+ template <typename K,
+ typename M,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped));
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, Key const& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(key, std::forward<M>(mapped)).first;
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, Key&& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped)).first;
+ }
+
+ template <typename K,
+ typename M,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, K&& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)).first;
+ }
+
+ // Single arguments for unordered_set can be used without having to construct the value_type
+ template <class K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<!is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto emplace(K&& key) -> std::pair<iterator, bool> {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ m_equal(key, m_values[at(m_buckets, bucket_idx).m_value_idx])) {
+ // found it, return without ever actually creating anything
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ // value is new, insert element first, so when exception happens we are in a valid state
+ return do_place_element(dist_and_fingerprint, bucket_idx, std::forward<K>(key));
+ }
+
+ template <class... Args>
+ auto emplace(Args&&... args) -> std::pair<iterator, bool> {
+ // we have to instantiate the value_type to be able to access the key.
+ // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop.
+ auto& key = get_key(m_values.emplace_back(std::forward<Args>(args)...));
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
+ m_values.pop_back(); // value was already there, so get rid of it
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ // value is new, place the bucket and shift up until we find an empty spot
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
+ // increase_size just rehashes all the data we have in m_values
+ increase_size();
+ } else {
+ // place element and shift up until we find an empty spot
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ }
+ return {begin() + static_cast<difference_type>(value_idx), true};
+ }
+
+ template <class... Args>
+ auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator {
+ return emplace(std::forward<Args>(args)...).first;
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(Key const& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(key, std::forward<Args>(args)...);
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(Key&& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...);
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator {
+ return do_try_emplace(key, std::forward<Args>(args)...).first;
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator {
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...).first;
+ }
+
+ template <
+ typename K,
+ typename... Args,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
+ bool> = true>
+ auto try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...);
+ }
+
+ template <
+ typename K,
+ typename... Args,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
+ bool> = true>
+ auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator {
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...).first;
+ }
+
+ auto erase(iterator it) -> iterator {
+ auto hash = mixed_hash(get_key(*it));
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
+ bucket_idx = next(bucket_idx);
+ }
+
+ do_erase(bucket_idx, [](value_type&& /*unused*/) {
+ });
+ return begin() + static_cast<difference_type>(value_idx_to_remove);
+ }
+
+ auto extract(iterator it) -> value_type {
+ auto hash = mixed_hash(get_key(*it));
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
+ bucket_idx = next(bucket_idx);
+ }
+
+ auto tmp = std::optional<value_type>{};
+ do_erase(bucket_idx, [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return std::move(tmp).value();
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto erase(const_iterator it) -> iterator {
+ return erase(begin() + (it - cbegin()));
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto extract(const_iterator it) -> value_type {
+ return extract(begin() + (it - cbegin()));
+ }
+
+ auto erase(const_iterator first, const_iterator last) -> iterator {
+ auto const idx_first = first - cbegin();
+ auto const idx_last = last - cbegin();
+ auto const first_to_last = std::distance(first, last);
+ auto const last_to_end = std::distance(last, cend());
+
+ // remove elements from left to right which moves elements from the end back
+ auto const mid = idx_first + (std::min)(first_to_last, last_to_end);
+ auto idx = idx_first;
+ while (idx != mid) {
+ erase(begin() + idx);
+ ++idx;
+ }
+
+ // all elements from the right are moved, now remove the last element until all done
+ idx = idx_last;
+ while (idx != mid) {
+ --idx;
+ erase(begin() + idx);
+ }
+
+ return begin() + idx_first;
+ }
+
+ auto erase(Key const& key) -> size_t {
+ return do_erase_key(key, [](value_type&& /*unused*/) {
+ });
+ }
+
+ auto extract(Key const& key) -> std::optional<value_type> {
+ auto tmp = std::optional<value_type>{};
+ do_erase_key(key, [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return tmp;
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto erase(K&& key) -> size_t {
+ return do_erase_key(std::forward<K>(key), [](value_type&& /*unused*/) {
+ });
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto extract(K&& key) -> std::optional<value_type> {
+ auto tmp = std::optional<value_type>{};
+ do_erase_key(std::forward<K>(key), [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return tmp;
+ }
+
+ void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v<value_container_type> &&
+ std::is_nothrow_swappable_v<Hash> && std::is_nothrow_swappable_v<KeyEqual>)) {
+ using std::swap;
+ swap(other, *this);
+ }
+
+ // lookup /////////////////////////////////////////////////////////////////
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto at(key_type const& key) -> Q& {
+ return do_at(key);
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto at(K const& key) -> Q& {
+ return do_at(key);
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto at(key_type const& key) const -> Q const& {
+ return do_at(key);
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto at(K const& key) const -> Q const& {
+ return do_at(key);
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto operator[](Key const& key) -> Q& {
+ return try_emplace(key).first->second;
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto operator[](Key&& key) -> Q& {
+ return try_emplace(std::move(key)).first->second;
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto operator[](K&& key) -> Q& {
+ return try_emplace(std::forward<K>(key)).first->second;
+ }
+
+ auto count(Key const& key) const -> size_t {
+ return find(key) == end() ? 0 : 1;
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto count(K const& key) const -> size_t {
+ return find(key) == end() ? 0 : 1;
+ }
+
+ auto find(Key const& key) -> iterator {
+ return do_find(key);
+ }
+
+ auto find(Key const& key) const -> const_iterator {
+ return do_find(key);
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto find(K const& key) -> iterator {
+ return do_find(key);
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto find(K const& key) const -> const_iterator {
+ return do_find(key);
+ }
+
+ auto contains(Key const& key) const -> bool {
+ return find(key) != end();
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto contains(K const& key) const -> bool {
+ return find(key) != end();
+ }
+
+ auto equal_range(Key const& key) -> std::pair<iterator, iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ auto equal_range(const Key& key) const -> std::pair<const_iterator, const_iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto equal_range(K const& key) -> std::pair<iterator, iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto equal_range(K const& key) const -> std::pair<const_iterator, const_iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ // bucket interface ///////////////////////////////////////////////////////
+
+ auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
+ return m_num_buckets;
+ }
+
+ static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
+ return max_size();
+ }
+
+ // hash policy ////////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto load_factor() const -> float {
+ return bucket_count() ? static_cast<float>(size()) / static_cast<float>(bucket_count()) : 0.0F;
+ }
+
+ [[nodiscard]] auto max_load_factor() const -> float {
+ return m_max_load_factor;
+ }
+
+ void max_load_factor(float ml) {
+ m_max_load_factor = ml;
+ if (m_num_buckets != max_bucket_count()) {
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
+ }
+ }
+
+ void rehash(size_t count) {
+ count = (std::min)(count, max_size());
+ auto shifts = calc_shifts_for_size((std::max)(count, size()));
+ if (shifts != m_shifts) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ m_values.shrink_to_fit();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+ }
+
+ void reserve(size_t capa) {
+ capa = (std::min)(capa, max_size());
+ if constexpr (has_reserve<value_container_type>) {
+ // std::deque doesn't have reserve(). Make sure we only call when available
+ m_values.reserve(capa);
+ }
+ auto shifts = calc_shifts_for_size((std::max)(capa, size()));
+ if (0 == m_num_buckets || shifts < m_shifts) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+ }
+
+ // observers //////////////////////////////////////////////////////////////
+
+ auto hash_function() const -> hasher {
+ return m_hash;
+ }
+
+ auto key_eq() const -> key_equal {
+ return m_equal;
+ }
+
+ // nonstandard API: expose the underlying values container
+ [[nodiscard]] auto values() const noexcept -> value_container_type const& {
+ return m_values;
+ }
+
+ // non-member functions ///////////////////////////////////////////////////
+
+ friend auto operator==(table const& a, table const& b) -> bool {
+ if (&a == &b) {
+ return true;
+ }
+ if (a.size() != b.size()) {
+ return false;
+ }
+ for (auto const& b_entry : b) {
+ auto it = a.find(get_key(b_entry));
+ if constexpr (is_map_v<T>) {
+ // map: check that key is here, then also check that value is the same
+ if (a.end() == it || !(b_entry.second == it->second)) {
+ return false;
+ }
+ } else {
+ // set: only check that the key is here
+ if (a.end() == it) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ friend auto operator!=(table const& a, table const& b) -> bool {
+ return !(a == b);
+ }
+};
+
+} // namespace detail
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
+ class Bucket = bucket_type::standard>
+using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
+ class Bucket = bucket_type::standard>
+using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<Key>,
+ class Bucket = bucket_type::standard>
+using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
+
+# if defined(ANKERL_UNORDERED_DENSE_PMR)
+
+namespace pmr {
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using map =
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_map =
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, true>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using set = detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_set =
+ detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, true>;
+
+} // namespace pmr
+
+# endif
+
+// deduction guides ///////////////////////////////////////////////////////////
+
+// deduction guides for alias templates are only possible since C++20
+// see https://en.cppreference.com/w/cpp/language/class_template_argument_deduction
+
+} // namespace ANKERL_UNORDERED_DENSE_NAMESPACE
+} // namespace ankerl::unordered_dense
+
+// std extensions /////////////////////////////////////////////////////////////
+
+namespace std { // NOLINT(cert-dcl58-cpp)
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash,
+ class KeyEqual,
+ class AllocatorOrContainer,
+ class Bucket,
+ class Pred,
+ bool IsSegmented>
+// NOLINTNEXTLINE(cert-dcl58-cpp)
+auto erase_if(ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>& map,
+ Pred pred) -> size_t {
+ using map_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>;
+
+ // going back to front because erase() invalidates the end iterator
+ auto const old_size = map.size();
+ auto idx = old_size;
+ while (idx) {
+ --idx;
+ auto it = map.begin() + static_cast<typename map_t::difference_type>(idx);
+ if (pred(*it)) {
+ map.erase(it);
+ }
+ }
+
+ return old_size - map.size();
+}
+
+} // namespace std
+
+#endif
+#endif
diff --git a/contrib/backward-cpp/BackwardConfig.cmake b/contrib/backward-cpp/BackwardConfig.cmake
new file mode 100644
index 0000000..4cec0be
--- /dev/null
+++ b/contrib/backward-cpp/BackwardConfig.cmake
@@ -0,0 +1,249 @@
+#
+# BackwardMacros.cmake
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+# OPTIONS
+###############################################################################
+
+set(STACK_WALKING_UNWIND TRUE CACHE BOOL
+ "Use compiler's unwind API")
+set(STACK_WALKING_BACKTRACE FALSE CACHE BOOL
+ "Use backtrace from (e)glibc for stack walking")
+set(STACK_WALKING_LIBUNWIND FALSE CACHE BOOL
+ "Use libunwind for stack walking")
+
+set(STACK_DETAILS_AUTO_DETECT TRUE CACHE BOOL
+ "Auto detect backward's stack details dependencies")
+
+set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE CACHE BOOL
+ "Use backtrace from (e)glibc for symbols resolution")
+set(STACK_DETAILS_DW FALSE CACHE BOOL
+ "Use libdw to read debug info")
+set(STACK_DETAILS_BFD FALSE CACHE BOOL
+ "Use libbfd to read debug info")
+set(STACK_DETAILS_DWARF FALSE CACHE BOOL
+ "Use libdwarf/libelf to read debug info")
+
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND NOT DEFINED BACKWARD_TESTS)
+ # If this is a top level CMake project, we most lixely want the tests
+ set(BACKWARD_TESTS ON CACHE BOOL "Enable tests")
+else()
+ set(BACKWARD_TESTS OFF CACHE BOOL "Enable tests")
+endif()
+###############################################################################
+# CONFIGS
+###############################################################################
+include(FindPackageHandleStandardArgs)
+
+if (STACK_WALKING_LIBUNWIND)
+ # libunwind works on the macOS without having to add special include
+ # paths or libraries
+ if (NOT APPLE)
+ find_path(LIBUNWIND_INCLUDE_DIR NAMES "libunwind.h")
+ find_library(LIBUNWIND_LIBRARY unwind)
+
+ if (LIBUNWIND_LIBRARY)
+ include(CheckSymbolExists)
+ check_symbol_exists(UNW_INIT_SIGNAL_FRAME libunwind.h HAVE_UNW_INIT_SIGNAL_FRAME)
+ if (NOT HAVE_UNW_INIT_SIGNAL_FRAME)
+ message(STATUS "libunwind does not support unwinding from signal handler frames")
+ endif()
+ endif()
+
+ set(LIBUNWIND_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_DIR})
+ set(LIBDWARF_LIBRARIES ${LIBUNWIND_LIBRARY})
+ find_package_handle_standard_args(libunwind DEFAULT_MSG
+ LIBUNWIND_LIBRARY LIBUNWIND_INCLUDE_DIR)
+ mark_as_advanced(LIBUNWIND_INCLUDE_DIR LIBUNWIND_LIBRARY)
+ list(APPEND _BACKWARD_LIBRARIES ${LIBUNWIND_LIBRARY})
+ endif()
+
+ # Disable other unwinders if libunwind is found
+ set(STACK_WALKING_UNWIND FALSE)
+ set(STACK_WALKING_BACKTRACE FALSE)
+endif()
+
+if (${STACK_DETAILS_AUTO_DETECT})
+ if(NOT CMAKE_VERSION VERSION_LESS 3.17)
+ set(_name_mismatched_arg NAME_MISMATCHED)
+ endif()
+ # find libdw
+ find_path(LIBDW_INCLUDE_DIR NAMES "elfutils/libdw.h" "elfutils/libdwfl.h")
+ find_library(LIBDW_LIBRARY dw)
+ set(LIBDW_INCLUDE_DIRS ${LIBDW_INCLUDE_DIR} )
+ set(LIBDW_LIBRARIES ${LIBDW_LIBRARY} )
+ find_package_handle_standard_args(libdw ${_name_mismatched_arg}
+ REQUIRED_VARS LIBDW_LIBRARY LIBDW_INCLUDE_DIR)
+ mark_as_advanced(LIBDW_INCLUDE_DIR LIBDW_LIBRARY)
+
+ # find libbfd
+ find_path(LIBBFD_INCLUDE_DIR NAMES "bfd.h")
+ find_path(LIBDL_INCLUDE_DIR NAMES "dlfcn.h")
+ find_library(LIBBFD_LIBRARY bfd)
+ find_library(LIBDL_LIBRARY dl)
+ set(LIBBFD_INCLUDE_DIRS ${LIBBFD_INCLUDE_DIR} ${LIBDL_INCLUDE_DIR})
+ set(LIBBFD_LIBRARIES ${LIBBFD_LIBRARY} ${LIBDL_LIBRARY})
+ find_package_handle_standard_args(libbfd ${_name_mismatched_arg}
+ REQUIRED_VARS LIBBFD_LIBRARY LIBBFD_INCLUDE_DIR
+ LIBDL_LIBRARY LIBDL_INCLUDE_DIR)
+ mark_as_advanced(LIBBFD_INCLUDE_DIR LIBBFD_LIBRARY
+ LIBDL_INCLUDE_DIR LIBDL_LIBRARY)
+
+ # find libdwarf
+ find_path(LIBDWARF_INCLUDE_DIR NAMES "libdwarf.h" PATH_SUFFIXES libdwarf)
+ find_path(LIBELF_INCLUDE_DIR NAMES "libelf.h")
+ find_path(LIBDL_INCLUDE_DIR NAMES "dlfcn.h")
+ find_library(LIBDWARF_LIBRARY dwarf)
+ find_library(LIBELF_LIBRARY elf)
+ find_library(LIBDL_LIBRARY dl)
+ set(LIBDWARF_INCLUDE_DIRS ${LIBDWARF_INCLUDE_DIR} ${LIBELF_INCLUDE_DIR} ${LIBDL_INCLUDE_DIR})
+ set(LIBDWARF_LIBRARIES ${LIBDWARF_LIBRARY} ${LIBELF_LIBRARY} ${LIBDL_LIBRARY})
+ find_package_handle_standard_args(libdwarf ${_name_mismatched_arg}
+ REQUIRED_VARS LIBDWARF_LIBRARY LIBDWARF_INCLUDE_DIR
+ LIBELF_LIBRARY LIBELF_INCLUDE_DIR
+ LIBDL_LIBRARY LIBDL_INCLUDE_DIR)
+ mark_as_advanced(LIBDWARF_INCLUDE_DIR LIBDWARF_LIBRARY
+ LIBELF_INCLUDE_DIR LIBELF_LIBRARY
+ LIBDL_INCLUDE_DIR LIBDL_LIBRARY)
+
+ if (LIBDW_FOUND)
+ LIST(APPEND _BACKWARD_INCLUDE_DIRS ${LIBDW_INCLUDE_DIRS})
+ LIST(APPEND _BACKWARD_LIBRARIES ${LIBDW_LIBRARIES})
+ set(STACK_DETAILS_DW TRUE)
+ set(STACK_DETAILS_BFD FALSE)
+ set(STACK_DETAILS_DWARF FALSE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE)
+ elseif(LIBBFD_FOUND)
+ LIST(APPEND _BACKWARD_INCLUDE_DIRS ${LIBBFD_INCLUDE_DIRS})
+ LIST(APPEND _BACKWARD_LIBRARIES ${LIBBFD_LIBRARIES})
+
+ # If we attempt to link against static bfd, make sure to link its dependencies, too
+ get_filename_component(bfd_lib_ext "${LIBBFD_LIBRARY}" EXT)
+ if (bfd_lib_ext STREQUAL "${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ if (NOT APPLE)
+ list(APPEND _BACKWARD_LIBRARIES iberty z)
+ endif()
+ endif()
+
+ set(STACK_DETAILS_DW FALSE)
+ set(STACK_DETAILS_BFD TRUE)
+ set(STACK_DETAILS_DWARF FALSE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE)
+ elseif(LIBDWARF_FOUND)
+ LIST(APPEND _BACKWARD_INCLUDE_DIRS ${LIBDWARF_INCLUDE_DIRS})
+ LIST(APPEND _BACKWARD_LIBRARIES ${LIBDWARF_LIBRARIES})
+
+ set(STACK_DETAILS_DW FALSE)
+ set(STACK_DETAILS_BFD FALSE)
+ set(STACK_DETAILS_DWARF TRUE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE)
+ else()
+ set(STACK_DETAILS_DW FALSE)
+ set(STACK_DETAILS_BFD FALSE)
+ set(STACK_DETAILS_DWARF FALSE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL TRUE)
+ endif()
+else()
+ if (STACK_DETAILS_DW)
+ LIST(APPEND _BACKWARD_LIBRARIES dw)
+ endif()
+
+ if (STACK_DETAILS_BFD)
+ LIST(APPEND _BACKWARD_LIBRARIES bfd dl)
+ endif()
+
+ if (STACK_DETAILS_DWARF)
+ LIST(APPEND _BACKWARD_LIBRARIES dwarf elf)
+ endif()
+endif()
+
+macro(map_definitions var_prefix define_prefix)
+ foreach(def ${ARGN})
+ if (${${var_prefix}${def}})
+ LIST(APPEND _BACKWARD_DEFINITIONS "${define_prefix}${def}=1")
+ else()
+ LIST(APPEND _BACKWARD_DEFINITIONS "${define_prefix}${def}=0")
+ endif()
+ endforeach()
+endmacro()
+
+if (NOT _BACKWARD_DEFINITIONS)
+ map_definitions("STACK_WALKING_" "BACKWARD_HAS_" UNWIND LIBUNWIND BACKTRACE)
+ map_definitions("STACK_DETAILS_" "BACKWARD_HAS_" BACKTRACE_SYMBOL DW BFD DWARF)
+endif()
+
+if(WIN32)
+ list(APPEND _BACKWARD_LIBRARIES dbghelp psapi)
+ if(MINGW)
+ set(MINGW_MSVCR_LIBRARY "msvcr90$<$<CONFIG:DEBUG>:d>" CACHE STRING "Mingw MSVC runtime import library")
+ list(APPEND _BACKWARD_LIBRARIES ${MINGW_MSVCR_LIBRARY})
+ endif()
+endif()
+
+set(BACKWARD_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}")
+
+set(BACKWARD_HAS_EXTERNAL_LIBRARIES FALSE)
+set(FIND_PACKAGE_REQUIRED_VARS BACKWARD_INCLUDE_DIR)
+if(DEFINED _BACKWARD_LIBRARIES)
+ set(BACKWARD_HAS_EXTERNAL_LIBRARIES TRUE)
+ list(APPEND FIND_PACKAGE_REQUIRED_VARS _BACKWARD_LIBRARIES)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Backward
+ REQUIRED_VARS ${FIND_PACKAGE_REQUIRED_VARS}
+)
+list(APPEND _BACKWARD_INCLUDE_DIRS ${BACKWARD_INCLUDE_DIR})
+
+macro(add_backward target)
+ target_include_directories(${target} PRIVATE ${BACKWARD_INCLUDE_DIRS})
+ set_property(TARGET ${target} APPEND PROPERTY COMPILE_DEFINITIONS ${BACKWARD_DEFINITIONS})
+ set_property(TARGET ${target} APPEND PROPERTY LINK_LIBRARIES ${BACKWARD_LIBRARIES})
+endmacro()
+
+set(BACKWARD_INCLUDE_DIRS ${_BACKWARD_INCLUDE_DIRS} CACHE INTERNAL "_BACKWARD_INCLUDE_DIRS")
+set(BACKWARD_DEFINITIONS ${_BACKWARD_DEFINITIONS} CACHE INTERNAL "BACKWARD_DEFINITIONS")
+set(BACKWARD_LIBRARIES ${_BACKWARD_LIBRARIES} CACHE INTERNAL "BACKWARD_LIBRARIES")
+mark_as_advanced(BACKWARD_INCLUDE_DIRS BACKWARD_DEFINITIONS BACKWARD_LIBRARIES)
+
+# Expand each definition in BACKWARD_DEFINITIONS to its own cmake var and export
+# to outer scope
+foreach(var ${BACKWARD_DEFINITIONS})
+ string(REPLACE "=" ";" var_as_list ${var})
+ list(GET var_as_list 0 var_name)
+ list(GET var_as_list 1 var_value)
+ set(${var_name} ${var_value})
+ mark_as_advanced(${var_name})
+endforeach()
+
+if (NOT TARGET Backward::Backward)
+ add_library(Backward::Backward INTERFACE IMPORTED)
+ set_target_properties(Backward::Backward PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${BACKWARD_INCLUDE_DIRS}"
+ INTERFACE_COMPILE_DEFINITIONS "${BACKWARD_DEFINITIONS}"
+ )
+ if(BACKWARD_HAS_EXTERNAL_LIBRARIES)
+ set_target_properties(Backward::Backward PROPERTIES
+ INTERFACE_LINK_LIBRARIES "${BACKWARD_LIBRARIES}"
+ )
+ endif()
+endif()
diff --git a/contrib/backward-cpp/CMakeLists.txt b/contrib/backward-cpp/CMakeLists.txt
new file mode 100644
index 0000000..038c505
--- /dev/null
+++ b/contrib/backward-cpp/CMakeLists.txt
@@ -0,0 +1,88 @@
+#
+# CMakeLists.txt
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+cmake_minimum_required(VERSION 3.0)
+project(backward CXX)
+
+# Introduce variables:
+# * CMAKE_INSTALL_LIBDIR
+# * CMAKE_INSTALL_BINDIR
+# * CMAKE_INSTALL_INCLUDEDIR
+include(GNUInstallDirs)
+
+include(BackwardConfig.cmake)
+
+# check if compiler is nvcc or nvcc_wrapper
+set(COMPILER_IS_NVCC false)
+get_filename_component(COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+if (COMPILER_NAME MATCHES "^nvcc")
+ set(COMPILER_IS_NVCC true)
+endif()
+
+if (DEFINED ENV{OMPI_CXX} OR DEFINED ENV{MPICH_CXX})
+ if ( ($ENV{OMPI_CXX} MATCHES "nvcc") OR ($ENV{MPICH_CXX} MATCHES "nvcc") )
+ set(COMPILER_IS_NVCC true)
+ endif()
+endif()
+
+# set CXX standard
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+set(CMAKE_CXX_STANDARD 11)
+if (${COMPILER_IS_NVCC})
+ # GNU CXX extensions are not supported by nvcc
+ set(CMAKE_CXX_EXTENSIONS OFF)
+endif()
+
+###############################################################################
+# COMPILER FLAGS
+###############################################################################
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_COMPILER_IS_GNUCXX)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra")
+ if (NOT ${COMPILER_IS_NVCC})
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic-errors")
+ endif()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
+endif()
+
+###############################################################################
+# BACKWARD OBJECT
+###############################################################################
+
+add_library(backward_object OBJECT backward.cpp)
+target_compile_definitions(backward_object PRIVATE ${BACKWARD_DEFINITIONS})
+target_include_directories(backward_object PRIVATE ${BACKWARD_INCLUDE_DIRS})
+set(BACKWARD_ENABLE $<TARGET_OBJECTS:backward_object> CACHE STRING
+ "Link with this object to setup backward automatically")
+
+
+###############################################################################
+# BACKWARD LIBRARY (Includes backward.cpp)
+###############################################################################
+option(BACKWARD_SHARED "Build dynamic backward-cpp shared lib" OFF)
+
+if(BACKWARD_SHARED)
+ set(libtype SHARED)
+endif()
+add_library(backward ${libtype} backward.cpp)
+target_compile_definitions(backward PUBLIC ${BACKWARD_DEFINITIONS})
+target_include_directories(backward PUBLIC ${BACKWARD_INCLUDE_DIRS}) \ No newline at end of file
diff --git a/contrib/backward-cpp/backward.cpp b/contrib/backward-cpp/backward.cpp
new file mode 100644
index 0000000..110441c
--- /dev/null
+++ b/contrib/backward-cpp/backward.cpp
@@ -0,0 +1,42 @@
+// Pick your poison.
+//
+// On GNU/Linux, you have few choices to get the most out of your stack trace.
+//
+// By default you get:
+// - object filename
+// - function name
+//
+// In order to add:
+// - source filename
+// - line and column numbers
+// - source code snippet (assuming the file is accessible)
+
+// Install one of the following libraries then uncomment one of the macro (or
+// better, add the detection of the lib and the macro definition in your build
+// system)
+
+// - apt-get install libdw-dev ...
+// - g++/clang++ -ldw ...
+// #define BACKWARD_HAS_DW 1
+
+// - apt-get install binutils-dev ...
+// - g++/clang++ -lbfd ...
+// #define BACKWARD_HAS_BFD 1
+
+// - apt-get install libdwarf-dev ...
+// - g++/clang++ -ldwarf ...
+// #define BACKWARD_HAS_DWARF 1
+
+// Regardless of the library you choose to read the debug information,
+// for potentially more detailed stack traces you can use libunwind
+// - apt-get install libunwind-dev
+// - g++/clang++ -lunwind
+// #define BACKWARD_HAS_LIBUNWIND 1
+
+#include "backward.hpp"
+
+namespace backward {
+
+backward::SignalHandling sh;
+
+} // namespace backward
diff --git a/contrib/backward-cpp/backward.hpp b/contrib/backward-cpp/backward.hpp
new file mode 100644
index 0000000..ca09b72
--- /dev/null
+++ b/contrib/backward-cpp/backward.hpp
@@ -0,0 +1,4471 @@
+/*
+ * backward.hpp
+ * Copyright 2013 Google Inc. All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef H_6B9572DA_A64B_49E6_B234_051480991C89
+#define H_6B9572DA_A64B_49E6_B234_051480991C89
+
+#ifndef __cplusplus
+#error "It's not going to compile without a C++ compiler..."
+#endif
+
+#if defined(BACKWARD_CXX11)
+#elif defined(BACKWARD_CXX98)
+#else
+#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
+#define BACKWARD_CXX11
+#define BACKWARD_ATLEAST_CXX11
+#define BACKWARD_ATLEAST_CXX98
+#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)
+#define BACKWARD_ATLEAST_CXX17
+#endif
+#else
+#define BACKWARD_CXX98
+#define BACKWARD_ATLEAST_CXX98
+#endif
+#endif
+
+// You can define one of the following (or leave it to the auto-detection):
+//
+// #define BACKWARD_SYSTEM_LINUX
+// - specialization for linux
+//
+// #define BACKWARD_SYSTEM_DARWIN
+// - specialization for Mac OS X 10.5 and later.
+//
+// #define BACKWARD_SYSTEM_WINDOWS
+// - specialization for Windows (Clang 9 and MSVC2017)
+//
+// #define BACKWARD_SYSTEM_UNKNOWN
+// - placebo implementation, does nothing.
+//
+#if defined(BACKWARD_SYSTEM_LINUX)
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+#elif defined(BACKWARD_SYSTEM_UNKNOWN)
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+#else
+#if defined(__linux) || defined(__linux__)
+#define BACKWARD_SYSTEM_LINUX
+#elif defined(__APPLE__)
+#define BACKWARD_SYSTEM_DARWIN
+#elif defined(_WIN32)
+#define BACKWARD_SYSTEM_WINDOWS
+#else
+#define BACKWARD_SYSTEM_UNKNOWN
+#endif
+#endif
+
+#define NOINLINE __attribute__((noinline))
+
+#include <algorithm>
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <limits>
+#include <new>
+#include <sstream>
+#include <streambuf>
+#include <string>
+#include <vector>
+#include <exception>
+#include <iterator>
+
+#if defined(BACKWARD_SYSTEM_LINUX)
+
+// On linux, backtrace can back-trace or "walk" the stack using the following
+// libraries:
+//
+// #define BACKWARD_HAS_UNWIND 1
+// - unwind comes from libgcc, but I saw an equivalent inside clang itself.
+// - with unwind, the stacktrace is as accurate as it can possibly be, since
+// this is used by the C++ runtine in gcc/clang for stack unwinding on
+// exception.
+// - normally libgcc is already linked to your program by default.
+//
+// #define BACKWARD_HAS_LIBUNWIND 1
+// - libunwind provides, in some cases, a more accurate stacktrace as it knows
+// to decode signal handler frames and lets us edit the context registers when
+// unwinding, allowing stack traces over bad function references.
+//
+// #define BACKWARD_HAS_BACKTRACE == 1
+// - backtrace seems to be a little bit more portable than libunwind, but on
+// linux, it uses unwind anyway, but abstract away a tiny information that is
+// sadly really important in order to get perfectly accurate stack traces.
+// - backtrace is part of the (e)glib library.
+//
+// The default is:
+// #define BACKWARD_HAS_UNWIND == 1
+//
+// Note that only one of the define should be set to 1 at a time.
+//
+#if BACKWARD_HAS_UNWIND == 1
+#elif BACKWARD_HAS_LIBUNWIND == 1
+#elif BACKWARD_HAS_BACKTRACE == 1
+#else
+#undef BACKWARD_HAS_UNWIND
+#define BACKWARD_HAS_UNWIND 1
+#undef BACKWARD_HAS_LIBUNWIND
+#define BACKWARD_HAS_LIBUNWIND 0
+#undef BACKWARD_HAS_BACKTRACE
+#define BACKWARD_HAS_BACKTRACE 0
+#endif
+
+// On linux, backward can extract detailed information about a stack trace
+// using one of the following libraries:
+//
+// #define BACKWARD_HAS_DW 1
+// - libdw gives you the most juicy details out of your stack traces:
+// - object filename
+// - function name
+// - source filename
+// - line and column numbers
+// - source code snippet (assuming the file is accessible)
+// - variable names (if not optimized out)
+// - variable values (not supported by backward-cpp)
+// - You need to link with the lib "dw":
+// - apt-get install libdw-dev
+// - g++/clang++ -ldw ...
+//
+// #define BACKWARD_HAS_BFD 1
+// - With libbfd, you get a fair amount of details:
+// - object filename
+// - function name
+// - source filename
+// - line numbers
+// - source code snippet (assuming the file is accessible)
+// - You need to link with the lib "bfd":
+// - apt-get install binutils-dev
+// - g++/clang++ -lbfd ...
+//
+// #define BACKWARD_HAS_DWARF 1
+// - libdwarf gives you the most juicy details out of your stack traces:
+// - object filename
+// - function name
+// - source filename
+// - line and column numbers
+// - source code snippet (assuming the file is accessible)
+// - variable names (if not optimized out)
+// - variable values (not supported by backward-cpp)
+// - You need to link with the lib "dwarf":
+// - apt-get install libdwarf-dev
+// - g++/clang++ -ldwarf ...
+//
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+// - backtrace provides minimal details for a stack trace:
+// - object filename
+// - function name
+// - backtrace is part of the (e)glib library.
+//
+// The default is:
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+//
+// Note that only one of the define should be set to 1 at a time.
+//
+#if BACKWARD_HAS_DW == 1
+#elif BACKWARD_HAS_BFD == 1
+#elif BACKWARD_HAS_DWARF == 1
+#elif BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+#else
+#undef BACKWARD_HAS_DW
+#define BACKWARD_HAS_DW 0
+#undef BACKWARD_HAS_BFD
+#define BACKWARD_HAS_BFD 0
+#undef BACKWARD_HAS_DWARF
+#define BACKWARD_HAS_DWARF 0
+#undef BACKWARD_HAS_BACKTRACE_SYMBOL
+#define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+#endif
+
+#include <cxxabi.h>
+#include <fcntl.h>
+#ifdef __ANDROID__
+// Old Android API levels define _Unwind_Ptr in both link.h and
+// unwind.h Rename the one in link.h as we are not going to be using
+// it
+#define _Unwind_Ptr _Unwind_Ptr_Custom
+#include <link.h>
+#undef _Unwind_Ptr
+#else
+#include <link.h>
+#endif
+#if defined(__ppc__) || defined(__powerpc) || defined(__powerpc__) || \
+ defined(__POWERPC__)
+// Linux kernel header required for the struct pt_regs definition
+// to access the NIP (Next Instruction Pointer) register value
+#include <asm/ptrace.h>
+#endif
+#include <signal.h>
+#include <sys/stat.h>
+#include <syscall.h>
+#include <unistd.h>
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#undef _GNU_SOURCE
+#else
+#include <dlfcn.h>
+#endif
+
+#if BACKWARD_HAS_BFD == 1
+// NOTE: defining PACKAGE{,_VERSION} is required before including
+// bfd.h on some platforms, see also:
+// https://sourceware.org/bugzilla/show_bug.cgi?id=14243
+#ifndef PACKAGE
+#define PACKAGE
+#endif
+#ifndef PACKAGE_VERSION
+#define PACKAGE_VERSION
+#endif
+#include <bfd.h>
+#endif
+
+#if BACKWARD_HAS_DW == 1
+#include <dwarf.h>
+#include <elfutils/libdw.h>
+#include <elfutils/libdwfl.h>
+#endif
+
+#if BACKWARD_HAS_DWARF == 1
+#include <algorithm>
+#include <dwarf.h>
+#include <libdwarf.h>
+#include <libelf.h>
+#include <map>
+#endif
+
+#if (BACKWARD_HAS_BACKTRACE == 1) || (BACKWARD_HAS_BACKTRACE_SYMBOL == 1)
+// then we shall rely on backtrace
+#include <execinfo.h>
+#endif
+
+#endif // defined(BACKWARD_SYSTEM_LINUX)
+
+#if defined(BACKWARD_SYSTEM_DARWIN)
+// On Darwin, backtrace can back-trace or "walk" the stack using the following
+// libraries:
+//
+// #define BACKWARD_HAS_UNWIND 1
+// - unwind comes from libgcc, but I saw an equivalent inside clang itself.
+// - with unwind, the stacktrace is as accurate as it can possibly be, since
+// this is used by the C++ runtine in gcc/clang for stack unwinding on
+// exception.
+// - normally libgcc is already linked to your program by default.
+//
+// #define BACKWARD_HAS_LIBUNWIND 1
+// - libunwind comes from clang, which implements an API compatible version.
+// - libunwind provides, in some cases, a more accurate stacktrace as it knows
+// to decode signal handler frames and lets us edit the context registers when
+// unwinding, allowing stack traces over bad function references.
+//
+// #define BACKWARD_HAS_BACKTRACE == 1
+// - backtrace is available by default, though it does not produce as much
+// information as another library might.
+//
+// The default is:
+// #define BACKWARD_HAS_UNWIND == 1
+//
+// Note that only one of the define should be set to 1 at a time.
+//
+#if BACKWARD_HAS_UNWIND == 1
+#elif BACKWARD_HAS_BACKTRACE == 1
+#elif BACKWARD_HAS_LIBUNWIND == 1
+#else
+#undef BACKWARD_HAS_UNWIND
+#define BACKWARD_HAS_UNWIND 1
+#undef BACKWARD_HAS_BACKTRACE
+#define BACKWARD_HAS_BACKTRACE 0
+#undef BACKWARD_HAS_LIBUNWIND
+#define BACKWARD_HAS_LIBUNWIND 0
+#endif
+
+// On Darwin, backward can extract detailed information about a stack trace
+// using one of the following libraries:
+//
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+// - backtrace provides minimal details for a stack trace:
+// - object filename
+// - function name
+//
+// The default is:
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+//
+#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+#else
+#undef BACKWARD_HAS_BACKTRACE_SYMBOL
+#define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+#endif
+
+#include <cxxabi.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#if (BACKWARD_HAS_BACKTRACE == 1) || (BACKWARD_HAS_BACKTRACE_SYMBOL == 1)
+#include <execinfo.h>
+#endif
+#endif // defined(BACKWARD_SYSTEM_DARWIN)
+
+#if defined(BACKWARD_SYSTEM_WINDOWS)
+
+#include <condition_variable>
+#include <mutex>
+#include <thread>
+
+#include <basetsd.h>
+typedef SSIZE_T ssize_t;
+
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <winnt.h>
+
+#include <psapi.h>
+#include <signal.h>
+
+#ifndef __clang__
+#undef NOINLINE
+#define NOINLINE __declspec(noinline)
+#endif
+
+#ifdef _MSC_VER
+#pragma comment(lib, "psapi.lib")
+#pragma comment(lib, "dbghelp.lib")
+#endif
+
+// Comment / packing is from stackoverflow:
+// https://stackoverflow.com/questions/6205981/windows-c-stack-trace-from-a-running-app/28276227#28276227
+// Some versions of imagehlp.dll lack the proper packing directives themselves
+// so we need to do it.
+#pragma pack(push, before_imagehlp, 8)
+#include <imagehlp.h>
+#pragma pack(pop, before_imagehlp)
+
+// TODO maybe these should be undefined somewhere else?
+#undef BACKWARD_HAS_UNWIND
+#undef BACKWARD_HAS_BACKTRACE
+#if BACKWARD_HAS_PDB_SYMBOL == 1
+#else
+#undef BACKWARD_HAS_PDB_SYMBOL
+#define BACKWARD_HAS_PDB_SYMBOL 1
+#endif
+
+#endif
+
+#if BACKWARD_HAS_UNWIND == 1
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#include <unwind.h>
+#undef _GNU_SOURCE
+#else
+#include <unwind.h>
+#endif
+// while gcc's unwind.h defines something like that:
+// extern _Unwind_Ptr _Unwind_GetIP (struct _Unwind_Context *);
+// extern _Unwind_Ptr _Unwind_GetIPInfo (struct _Unwind_Context *, int *);
+//
+// clang's unwind.h defines something like this:
+// uintptr_t _Unwind_GetIP(struct _Unwind_Context* __context);
+//
+// Even if the _Unwind_GetIPInfo can be linked to, it is not declared, worse we
+// cannot just redeclare it because clang's unwind.h doesn't define _Unwind_Ptr
+// anyway.
+//
+// Luckily we can play on the fact that the guard macros have a different name:
+#ifdef __CLANG_UNWIND_H
+// In fact, this function still comes from libgcc (on my different linux boxes,
+// clang links against libgcc).
+#include <inttypes.h>
+extern "C" uintptr_t _Unwind_GetIPInfo(_Unwind_Context *, int *);
+#endif
+
+#endif // BACKWARD_HAS_UNWIND == 1
+
+#if BACKWARD_HAS_LIBUNWIND == 1
+#define UNW_LOCAL_ONLY
+#include <libunwind.h>
+#endif // BACKWARD_HAS_LIBUNWIND == 1
+
+#ifdef BACKWARD_ATLEAST_CXX11
+#include <unordered_map>
+#include <utility> // for std::swap
+namespace backward {
+namespace details {
+template <typename K, typename V> struct hashtable {
+ typedef std::unordered_map<K, V> type;
+};
+using std::move;
+} // namespace details
+} // namespace backward
+#else // NOT BACKWARD_ATLEAST_CXX11
+#define nullptr NULL
+#define override
+#include <map>
+namespace backward {
+namespace details {
+template <typename K, typename V> struct hashtable {
+ typedef std::map<K, V> type;
+};
+template <typename T> const T &move(const T &v) { return v; }
+template <typename T> T &move(T &v) { return v; }
+} // namespace details
+} // namespace backward
+#endif // BACKWARD_ATLEAST_CXX11
+
+namespace backward {
+namespace details {
+#if defined(BACKWARD_SYSTEM_WINDOWS)
+const char kBackwardPathDelimiter[] = ";";
+#else
+const char kBackwardPathDelimiter[] = ":";
+#endif
+} // namespace details
+} // namespace backward
+
+namespace backward {
+
+namespace system_tag {
+struct linux_tag; // seems that I cannot call that "linux" because the name
+// is already defined... so I am adding _tag everywhere.
+struct darwin_tag;
+struct windows_tag;
+struct unknown_tag;
+
+#if defined(BACKWARD_SYSTEM_LINUX)
+typedef linux_tag current_tag;
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+typedef darwin_tag current_tag;
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+typedef windows_tag current_tag;
+#elif defined(BACKWARD_SYSTEM_UNKNOWN)
+typedef unknown_tag current_tag;
+#else
+#error "May I please get my system defines?"
+#endif
+} // namespace system_tag
+
+namespace trace_resolver_tag {
+#if defined(BACKWARD_SYSTEM_LINUX)
+struct libdw;
+struct libbfd;
+struct libdwarf;
+struct backtrace_symbol;
+
+#if BACKWARD_HAS_DW == 1
+typedef libdw current;
+#elif BACKWARD_HAS_BFD == 1
+typedef libbfd current;
+#elif BACKWARD_HAS_DWARF == 1
+typedef libdwarf current;
+#elif BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+typedef backtrace_symbol current;
+#else
+#error "You shall not pass, until you know what you want."
+#endif
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+struct backtrace_symbol;
+
+#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+typedef backtrace_symbol current;
+#else
+#error "You shall not pass, until you know what you want."
+#endif
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+struct pdb_symbol;
+#if BACKWARD_HAS_PDB_SYMBOL == 1
+typedef pdb_symbol current;
+#else
+#error "You shall not pass, until you know what you want."
+#endif
+#endif
+} // namespace trace_resolver_tag
+
+namespace details {
+
+template <typename T> struct rm_ptr { typedef T type; };
+
+template <typename T> struct rm_ptr<T *> { typedef T type; };
+
+template <typename T> struct rm_ptr<const T *> { typedef const T type; };
+
+template <typename R, typename T, R (*F)(T)> struct deleter {
+ template <typename U> void operator()(U &ptr) const { (*F)(ptr); }
+};
+
+template <typename T> struct default_delete {
+ void operator()(T &ptr) const { delete ptr; }
+};
+
+template <typename T, typename Deleter = deleter<void, void *, &::free>>
+class handle {
+ struct dummy;
+ T _val;
+ bool _empty;
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ handle(const handle &) = delete;
+ handle &operator=(const handle &) = delete;
+#endif
+
+public:
+ ~handle() {
+ if (!_empty) {
+ Deleter()(_val);
+ }
+ }
+
+ explicit handle() : _val(), _empty(true) {}
+ explicit handle(T val) : _val(val), _empty(false) {
+ if (!_val)
+ _empty = true;
+ }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ handle(handle &&from) : _empty(true) { swap(from); }
+ handle &operator=(handle &&from) {
+ swap(from);
+ return *this;
+ }
+#else
+ explicit handle(const handle &from) : _empty(true) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<handle &>(from));
+ }
+ handle &operator=(const handle &from) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<handle &>(from));
+ return *this;
+ }
+#endif
+
+ void reset(T new_val) {
+ handle tmp(new_val);
+ swap(tmp);
+ }
+
+ void update(T new_val) {
+ _val = new_val;
+ _empty = !static_cast<bool>(new_val);
+ }
+
+ operator const dummy *() const {
+ if (_empty) {
+ return nullptr;
+ }
+ return reinterpret_cast<const dummy *>(_val);
+ }
+ T get() { return _val; }
+ T release() {
+ _empty = true;
+ return _val;
+ }
+ void swap(handle &b) {
+ using std::swap;
+ swap(b._val, _val); // can throw, we are safe here.
+ swap(b._empty, _empty); // should not throw: if you cannot swap two
+ // bools without throwing... It's a lost cause anyway!
+ }
+
+ T &operator->() { return _val; }
+ const T &operator->() const { return _val; }
+
+ typedef typename rm_ptr<T>::type &ref_t;
+ typedef const typename rm_ptr<T>::type &const_ref_t;
+ ref_t operator*() { return *_val; }
+ const_ref_t operator*() const { return *_val; }
+ ref_t operator[](size_t idx) { return _val[idx]; }
+
+ // Watch out, we've got a badass over here
+ T *operator&() {
+ _empty = false;
+ return &_val;
+ }
+};
+
+// Default demangler implementation (do nothing).
+template <typename TAG> struct demangler_impl {
+ static std::string demangle(const char *funcname) { return funcname; }
+};
+
+#if defined(BACKWARD_SYSTEM_LINUX) || defined(BACKWARD_SYSTEM_DARWIN)
+
+template <> struct demangler_impl<system_tag::current_tag> {
+ demangler_impl() : _demangle_buffer_length(0) {}
+
+ std::string demangle(const char *funcname) {
+ using namespace details;
+ char *result = abi::__cxa_demangle(funcname, _demangle_buffer.get(),
+ &_demangle_buffer_length, nullptr);
+ if (result) {
+ _demangle_buffer.update(result);
+ return result;
+ }
+ return funcname;
+ }
+
+private:
+ details::handle<char *> _demangle_buffer;
+ size_t _demangle_buffer_length;
+};
+
+#endif // BACKWARD_SYSTEM_LINUX || BACKWARD_SYSTEM_DARWIN
+
+struct demangler : public demangler_impl<system_tag::current_tag> {};
+
+// Split a string on the platform's PATH delimiter. Example: if delimiter
+// is ":" then:
+// "" --> []
+// ":" --> ["",""]
+// "::" --> ["","",""]
+// "/a/b/c" --> ["/a/b/c"]
+// "/a/b/c:/d/e/f" --> ["/a/b/c","/d/e/f"]
+// etc.
+inline std::vector<std::string> split_source_prefixes(const std::string &s) {
+ std::vector<std::string> out;
+ size_t last = 0;
+ size_t next = 0;
+ size_t delimiter_size = sizeof(kBackwardPathDelimiter) - 1;
+ while ((next = s.find(kBackwardPathDelimiter, last)) != std::string::npos) {
+ out.push_back(s.substr(last, next - last));
+ last = next + delimiter_size;
+ }
+ if (last <= s.length()) {
+ out.push_back(s.substr(last));
+ }
+ return out;
+}
+
+} // namespace details
+
+/*************** A TRACE ***************/
+
+struct Trace {
+ void *addr;
+ size_t idx;
+
+ Trace() : addr(nullptr), idx(0) {}
+
+ explicit Trace(void *_addr, size_t _idx) : addr(_addr), idx(_idx) {}
+};
+
+struct ResolvedTrace : public Trace {
+
+ struct SourceLoc {
+ std::string function;
+ std::string filename;
+ unsigned line;
+ unsigned col;
+
+ SourceLoc() : line(0), col(0) {}
+
+ bool operator==(const SourceLoc &b) const {
+ return function == b.function && filename == b.filename &&
+ line == b.line && col == b.col;
+ }
+
+ bool operator!=(const SourceLoc &b) const { return !(*this == b); }
+ };
+
+ // In which binary object this trace is located.
+ std::string object_filename;
+
+ // The function in the object that contain the trace. This is not the same
+ // as source.function which can be an function inlined in object_function.
+ std::string object_function;
+
+ // The source location of this trace. It is possible for filename to be
+ // empty and for line/col to be invalid (value 0) if this information
+ // couldn't be deduced, for example if there is no debug information in the
+ // binary object.
+ SourceLoc source;
+
+ // An optionals list of "inliners". All the successive sources location
+ // from where the source location of the trace (the attribute right above)
+ // is inlined. It is especially useful when you compiled with optimization.
+ typedef std::vector<SourceLoc> source_locs_t;
+ source_locs_t inliners;
+
+ ResolvedTrace() : Trace() {}
+ ResolvedTrace(const Trace &mini_trace) : Trace(mini_trace) {}
+};
+
+/*************** STACK TRACE ***************/
+
+// default implemention.
+template <typename TAG> class StackTraceImpl {
+public:
+ size_t size() const { return 0; }
+ Trace operator[](size_t) const { return Trace(); }
+ size_t load_here(size_t = 0) { return 0; }
+ size_t load_from(void *, size_t = 0, void * = nullptr, void * = nullptr) {
+ return 0;
+ }
+ size_t thread_id() const { return 0; }
+ void skip_n_firsts(size_t) {}
+};
+
+class StackTraceImplBase {
+public:
+ StackTraceImplBase()
+ : _thread_id(0), _skip(0), _context(nullptr), _error_addr(nullptr) {}
+
+ size_t thread_id() const { return _thread_id; }
+
+ void skip_n_firsts(size_t n) { _skip = n; }
+
+protected:
+ void load_thread_info() {
+#ifdef BACKWARD_SYSTEM_LINUX
+#ifndef __ANDROID__
+ _thread_id = static_cast<size_t>(syscall(SYS_gettid));
+#else
+ _thread_id = static_cast<size_t>(gettid());
+#endif
+ if (_thread_id == static_cast<size_t>(getpid())) {
+ // If the thread is the main one, let's hide that.
+ // I like to keep little secret sometimes.
+ _thread_id = 0;
+ }
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+ _thread_id = reinterpret_cast<size_t>(pthread_self());
+ if (pthread_main_np() == 1) {
+ // If the thread is the main one, let's hide that.
+ _thread_id = 0;
+ }
+#endif
+ }
+
+ void set_context(void *context) { _context = context; }
+ void *context() const { return _context; }
+
+ void set_error_addr(void *error_addr) { _error_addr = error_addr; }
+ void *error_addr() const { return _error_addr; }
+
+ size_t skip_n_firsts() const { return _skip; }
+
+private:
+ size_t _thread_id;
+ size_t _skip;
+ void *_context;
+ void *_error_addr;
+};
+
+class StackTraceImplHolder : public StackTraceImplBase {
+public:
+ size_t size() const {
+ return (_stacktrace.size() >= skip_n_firsts())
+ ? _stacktrace.size() - skip_n_firsts()
+ : 0;
+ }
+ Trace operator[](size_t idx) const {
+ if (idx >= size()) {
+ return Trace();
+ }
+ return Trace(_stacktrace[idx + skip_n_firsts()], idx);
+ }
+ void *const *begin() const {
+ if (size()) {
+ return &_stacktrace[skip_n_firsts()];
+ }
+ return nullptr;
+ }
+
+protected:
+ std::vector<void *> _stacktrace;
+};
+
+#if BACKWARD_HAS_UNWIND == 1
+
+namespace details {
+
+template <typename F> class Unwinder {
+public:
+ size_t operator()(F &f, size_t depth) {
+ _f = &f;
+ _index = -1;
+ _depth = depth;
+ _Unwind_Backtrace(&this->backtrace_trampoline, this);
+ return static_cast<size_t>(_index);
+ }
+
+private:
+ F *_f;
+ ssize_t _index;
+ size_t _depth;
+
+ static _Unwind_Reason_Code backtrace_trampoline(_Unwind_Context *ctx,
+ void *self) {
+ return (static_cast<Unwinder *>(self))->backtrace(ctx);
+ }
+
+ _Unwind_Reason_Code backtrace(_Unwind_Context *ctx) {
+ if (_index >= 0 && static_cast<size_t>(_index) >= _depth)
+ return _URC_END_OF_STACK;
+
+ int ip_before_instruction = 0;
+ uintptr_t ip = _Unwind_GetIPInfo(ctx, &ip_before_instruction);
+
+ if (!ip_before_instruction) {
+ // calculating 0-1 for unsigned, looks like a possible bug to sanitiziers,
+ // so let's do it explicitly:
+ if (ip == 0) {
+ ip = std::numeric_limits<uintptr_t>::max(); // set it to 0xffff... (as
+ // from casting 0-1)
+ } else {
+ ip -= 1; // else just normally decrement it (no overflow/underflow will
+ // happen)
+ }
+ }
+
+ if (_index >= 0) { // ignore first frame.
+ (*_f)(static_cast<size_t>(_index), reinterpret_cast<void *>(ip));
+ }
+ _index += 1;
+ return _URC_NO_REASON;
+ }
+};
+
+template <typename F> size_t unwind(F f, size_t depth) {
+ Unwinder<F> unwinder;
+ return unwinder(f, depth);
+}
+
+} // namespace details
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ NOINLINE
+ size_t load_here(size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_thread_info();
+ set_context(context);
+ set_error_addr(error_addr);
+ if (depth == 0) {
+ return 0;
+ }
+ _stacktrace.resize(depth);
+ size_t trace_cnt = details::unwind(callback(*this), depth);
+ _stacktrace.resize(trace_cnt);
+ skip_n_firsts(0);
+ return size();
+ }
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+
+private:
+ struct callback {
+ StackTraceImpl &self;
+ callback(StackTraceImpl &_self) : self(_self) {}
+
+ void operator()(size_t idx, void *addr) { self._stacktrace[idx] = addr; }
+ };
+};
+
+#elif BACKWARD_HAS_LIBUNWIND == 1
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ __attribute__((noinline)) size_t load_here(size_t depth = 32,
+ void *_context = nullptr,
+ void *_error_addr = nullptr) {
+ set_context(_context);
+ set_error_addr(_error_addr);
+ load_thread_info();
+ if (depth == 0) {
+ return 0;
+ }
+ _stacktrace.resize(depth + 1);
+
+ int result = 0;
+
+ unw_context_t ctx;
+ size_t index = 0;
+
+ // Add the tail call. If the Instruction Pointer is the crash address it
+ // means we got a bad function pointer dereference, so we "unwind" the
+ // bad pointer manually by using the return address pointed to by the
+ // Stack Pointer as the Instruction Pointer and letting libunwind do
+ // the rest
+
+ if (context()) {
+ ucontext_t *uctx = reinterpret_cast<ucontext_t *>(context());
+#ifdef REG_RIP // x86_64
+ if (uctx->uc_mcontext.gregs[REG_RIP] ==
+ reinterpret_cast<greg_t>(error_addr())) {
+ uctx->uc_mcontext.gregs[REG_RIP] =
+ *reinterpret_cast<size_t *>(uctx->uc_mcontext.gregs[REG_RSP]);
+ }
+ _stacktrace[index] =
+ reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_RIP]);
+ ++index;
+ ctx = *reinterpret_cast<unw_context_t *>(uctx);
+#elif defined(REG_EIP) // x86_32
+ if (uctx->uc_mcontext.gregs[REG_EIP] ==
+ reinterpret_cast<greg_t>(error_addr())) {
+ uctx->uc_mcontext.gregs[REG_EIP] =
+ *reinterpret_cast<size_t *>(uctx->uc_mcontext.gregs[REG_ESP]);
+ }
+ _stacktrace[index] =
+ reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_EIP]);
+ ++index;
+ ctx = *reinterpret_cast<unw_context_t *>(uctx);
+#elif defined(__arm__)
+ // libunwind uses its own context type for ARM unwinding.
+ // Copy the registers from the signal handler's context so we can
+ // unwind
+ unw_getcontext(&ctx);
+ ctx.regs[UNW_ARM_R0] = uctx->uc_mcontext.arm_r0;
+ ctx.regs[UNW_ARM_R1] = uctx->uc_mcontext.arm_r1;
+ ctx.regs[UNW_ARM_R2] = uctx->uc_mcontext.arm_r2;
+ ctx.regs[UNW_ARM_R3] = uctx->uc_mcontext.arm_r3;
+ ctx.regs[UNW_ARM_R4] = uctx->uc_mcontext.arm_r4;
+ ctx.regs[UNW_ARM_R5] = uctx->uc_mcontext.arm_r5;
+ ctx.regs[UNW_ARM_R6] = uctx->uc_mcontext.arm_r6;
+ ctx.regs[UNW_ARM_R7] = uctx->uc_mcontext.arm_r7;
+ ctx.regs[UNW_ARM_R8] = uctx->uc_mcontext.arm_r8;
+ ctx.regs[UNW_ARM_R9] = uctx->uc_mcontext.arm_r9;
+ ctx.regs[UNW_ARM_R10] = uctx->uc_mcontext.arm_r10;
+ ctx.regs[UNW_ARM_R11] = uctx->uc_mcontext.arm_fp;
+ ctx.regs[UNW_ARM_R12] = uctx->uc_mcontext.arm_ip;
+ ctx.regs[UNW_ARM_R13] = uctx->uc_mcontext.arm_sp;
+ ctx.regs[UNW_ARM_R14] = uctx->uc_mcontext.arm_lr;
+ ctx.regs[UNW_ARM_R15] = uctx->uc_mcontext.arm_pc;
+
+ // If we have crashed in the PC use the LR instead, as this was
+ // a bad function dereference
+ if (reinterpret_cast<unsigned long>(error_addr()) ==
+ uctx->uc_mcontext.arm_pc) {
+ ctx.regs[UNW_ARM_R15] =
+ uctx->uc_mcontext.arm_lr - sizeof(unsigned long);
+ }
+ _stacktrace[index] = reinterpret_cast<void *>(ctx.regs[UNW_ARM_R15]);
+ ++index;
+#elif defined(__APPLE__) && defined(__x86_64__)
+ unw_getcontext(&ctx);
+ // OS X's implementation of libunwind uses its own context object
+ // so we need to convert the passed context to libunwind's format
+ // (information about the data layout taken from unw_getcontext.s
+ // in Apple's libunwind source
+ ctx.data[0] = uctx->uc_mcontext->__ss.__rax;
+ ctx.data[1] = uctx->uc_mcontext->__ss.__rbx;
+ ctx.data[2] = uctx->uc_mcontext->__ss.__rcx;
+ ctx.data[3] = uctx->uc_mcontext->__ss.__rdx;
+ ctx.data[4] = uctx->uc_mcontext->__ss.__rdi;
+ ctx.data[5] = uctx->uc_mcontext->__ss.__rsi;
+ ctx.data[6] = uctx->uc_mcontext->__ss.__rbp;
+ ctx.data[7] = uctx->uc_mcontext->__ss.__rsp;
+ ctx.data[8] = uctx->uc_mcontext->__ss.__r8;
+ ctx.data[9] = uctx->uc_mcontext->__ss.__r9;
+ ctx.data[10] = uctx->uc_mcontext->__ss.__r10;
+ ctx.data[11] = uctx->uc_mcontext->__ss.__r11;
+ ctx.data[12] = uctx->uc_mcontext->__ss.__r12;
+ ctx.data[13] = uctx->uc_mcontext->__ss.__r13;
+ ctx.data[14] = uctx->uc_mcontext->__ss.__r14;
+ ctx.data[15] = uctx->uc_mcontext->__ss.__r15;
+ ctx.data[16] = uctx->uc_mcontext->__ss.__rip;
+
+ // If the IP is the same as the crash address we have a bad function
+ // dereference The caller's address is pointed to by %rsp, so we
+ // dereference that value and set it to be the next frame's IP.
+ if (uctx->uc_mcontext->__ss.__rip ==
+ reinterpret_cast<__uint64_t>(error_addr())) {
+ ctx.data[16] =
+ *reinterpret_cast<__uint64_t *>(uctx->uc_mcontext->__ss.__rsp);
+ }
+ _stacktrace[index] = reinterpret_cast<void *>(ctx.data[16]);
+ ++index;
+#elif defined(__APPLE__)
+ unw_getcontext(&ctx)
+ // TODO: Convert the ucontext_t to libunwind's unw_context_t like
+ // we do in 64 bits
+ if (ctx.uc_mcontext->__ss.__eip ==
+ reinterpret_cast<greg_t>(error_addr())) {
+ ctx.uc_mcontext->__ss.__eip = ctx.uc_mcontext->__ss.__esp;
+ }
+ _stacktrace[index] =
+ reinterpret_cast<void *>(ctx.uc_mcontext->__ss.__eip);
+ ++index;
+#endif
+ }
+
+ unw_cursor_t cursor;
+ if (context()) {
+#if defined(UNW_INIT_SIGNAL_FRAME)
+ result = unw_init_local2(&cursor, &ctx, UNW_INIT_SIGNAL_FRAME);
+#else
+ result = unw_init_local(&cursor, &ctx);
+#endif
+ } else {
+ unw_getcontext(&ctx);
+ ;
+ result = unw_init_local(&cursor, &ctx);
+ }
+
+ if (result != 0)
+ return 1;
+
+ unw_word_t ip = 0;
+
+ while (index <= depth && unw_step(&cursor) > 0) {
+ result = unw_get_reg(&cursor, UNW_REG_IP, &ip);
+ if (result == 0) {
+ _stacktrace[index] = reinterpret_cast<void *>(--ip);
+ ++index;
+ }
+ }
+ --index;
+
+ _stacktrace.resize(index + 1);
+ skip_n_firsts(0);
+ return size();
+ }
+
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ _stacktrace[i] = (void *)((uintptr_t)_stacktrace[i]);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+};
+
+#elif defined(BACKWARD_HAS_BACKTRACE)
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ NOINLINE
+ size_t load_here(size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ set_context(context);
+ set_error_addr(error_addr);
+ load_thread_info();
+ if (depth == 0) {
+ return 0;
+ }
+ _stacktrace.resize(depth + 1);
+ size_t trace_cnt = backtrace(&_stacktrace[0], _stacktrace.size());
+ _stacktrace.resize(trace_cnt);
+ skip_n_firsts(1);
+ return size();
+ }
+
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ _stacktrace[i] = (void *)((uintptr_t)_stacktrace[i] + 1);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+};
+
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ // We have to load the machine type from the image info
+ // So we first initialize the resolver, and it tells us this info
+ void set_machine_type(DWORD machine_type) { machine_type_ = machine_type; }
+ void set_context(CONTEXT *ctx) { ctx_ = ctx; }
+ void set_thread_handle(HANDLE handle) { thd_ = handle; }
+
+ NOINLINE
+ size_t load_here(size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ set_context(static_cast<CONTEXT*>(context));
+ set_error_addr(error_addr);
+ CONTEXT localCtx; // used when no context is provided
+
+ if (depth == 0) {
+ return 0;
+ }
+
+ if (!ctx_) {
+ ctx_ = &localCtx;
+ RtlCaptureContext(ctx_);
+ }
+
+ if (!thd_) {
+ thd_ = GetCurrentThread();
+ }
+
+ HANDLE process = GetCurrentProcess();
+
+ STACKFRAME64 s;
+ memset(&s, 0, sizeof(STACKFRAME64));
+
+ // TODO: 32 bit context capture
+ s.AddrStack.Mode = AddrModeFlat;
+ s.AddrFrame.Mode = AddrModeFlat;
+ s.AddrPC.Mode = AddrModeFlat;
+#ifdef _M_X64
+ s.AddrPC.Offset = ctx_->Rip;
+ s.AddrStack.Offset = ctx_->Rsp;
+ s.AddrFrame.Offset = ctx_->Rbp;
+#else
+ s.AddrPC.Offset = ctx_->Eip;
+ s.AddrStack.Offset = ctx_->Esp;
+ s.AddrFrame.Offset = ctx_->Ebp;
+#endif
+
+ if (!machine_type_) {
+#ifdef _M_X64
+ machine_type_ = IMAGE_FILE_MACHINE_AMD64;
+#else
+ machine_type_ = IMAGE_FILE_MACHINE_I386;
+#endif
+ }
+
+ for (;;) {
+ // NOTE: this only works if PDBs are already loaded!
+ SetLastError(0);
+ if (!StackWalk64(machine_type_, process, thd_, &s, ctx_, NULL,
+ SymFunctionTableAccess64, SymGetModuleBase64, NULL))
+ break;
+
+ if (s.AddrReturn.Offset == 0)
+ break;
+
+ _stacktrace.push_back(reinterpret_cast<void *>(s.AddrPC.Offset));
+
+ if (size() >= depth)
+ break;
+ }
+
+ return size();
+ }
+
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+
+private:
+ DWORD machine_type_ = 0;
+ HANDLE thd_ = 0;
+ CONTEXT *ctx_ = nullptr;
+};
+
+#endif
+
+class StackTrace : public StackTraceImpl<system_tag::current_tag> {};
+
+/*************** TRACE RESOLVER ***************/
+
+class TraceResolverImplBase {
+public:
+ virtual ~TraceResolverImplBase() {}
+
+ virtual void load_addresses(void *const*addresses, int address_count) {
+ (void)addresses;
+ (void)address_count;
+ }
+
+ template <class ST> void load_stacktrace(ST &st) {
+ load_addresses(st.begin(), (int)st.size());
+ }
+
+ virtual ResolvedTrace resolve(ResolvedTrace t) { return t; }
+
+protected:
+ std::string demangle(const char *funcname) {
+ return _demangler.demangle(funcname);
+ }
+
+private:
+ details::demangler _demangler;
+};
+
+template <typename TAG> class TraceResolverImpl;
+
+#ifdef BACKWARD_SYSTEM_UNKNOWN
+
+template <> class TraceResolverImpl<system_tag::unknown_tag>
+ : public TraceResolverImplBase {};
+
+#endif
+
+#ifdef BACKWARD_SYSTEM_LINUX
+
+class TraceResolverLinuxBase : public TraceResolverImplBase {
+public:
+ TraceResolverLinuxBase()
+ : argv0_(get_argv0()), exec_path_(read_symlink("/proc/self/exe")) {}
+ std::string resolve_exec_path(Dl_info &symbol_info) const {
+ // mutates symbol_info.dli_fname to be filename to open and returns filename
+ // to display
+ if (symbol_info.dli_fname == argv0_) {
+ // dladdr returns argv[0] in dli_fname for symbols contained in
+ // the main executable, which is not a valid path if the
+ // executable was found by a search of the PATH environment
+ // variable; In that case, we actually open /proc/self/exe, which
+ // is always the actual executable (even if it was deleted/replaced!)
+ // but display the path that /proc/self/exe links to.
+ // However, this right away reduces probability of successful symbol
+ // resolution, because libbfd may try to find *.debug files in the
+ // same dir, in case symbols are stripped. As a result, it may try
+ // to find a file /proc/self/<exe_name>.debug, which obviously does
+ // not exist. /proc/self/exe is a last resort. First load attempt
+ // should go for the original executable file path.
+ symbol_info.dli_fname = "/proc/self/exe";
+ return exec_path_;
+ } else {
+ return symbol_info.dli_fname;
+ }
+ }
+
+private:
+ std::string argv0_;
+ std::string exec_path_;
+
+ static std::string get_argv0() {
+ std::string argv0;
+ std::ifstream ifs("/proc/self/cmdline");
+ std::getline(ifs, argv0, '\0');
+ return argv0;
+ }
+
+ static std::string read_symlink(std::string const &symlink_path) {
+ std::string path;
+ path.resize(100);
+
+ while (true) {
+ ssize_t len =
+ ::readlink(symlink_path.c_str(), &*path.begin(), path.size());
+ if (len < 0) {
+ return "";
+ }
+ if (static_cast<size_t>(len) == path.size()) {
+ path.resize(path.size() * 2);
+ } else {
+ path.resize(static_cast<std::string::size_type>(len));
+ break;
+ }
+ }
+
+ return path;
+ }
+};
+
+template <typename STACKTRACE_TAG> class TraceResolverLinuxImpl;
+
+#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::backtrace_symbol>
+ : public TraceResolverLinuxBase {
+public:
+ void load_addresses(void *const*addresses, int address_count) override {
+ if (address_count == 0) {
+ return;
+ }
+ _symbols.reset(backtrace_symbols(addresses, address_count));
+ }
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ char *filename = _symbols[trace.idx];
+ char *funcname = filename;
+ while (*funcname && *funcname != '(') {
+ funcname += 1;
+ }
+ trace.object_filename.assign(filename,
+ funcname); // ok even if funcname is the ending
+ // \0 (then we assign entire string)
+
+ if (*funcname) { // if it's not end of string (e.g. from last frame ip==0)
+ funcname += 1;
+ char *funcname_end = funcname;
+ while (*funcname_end && *funcname_end != ')' && *funcname_end != '+') {
+ funcname_end += 1;
+ }
+ *funcname_end = '\0';
+ trace.object_function = this->demangle(funcname);
+ trace.source.function = trace.object_function; // we cannot do better.
+ }
+ return trace;
+ }
+
+private:
+ details::handle<char **> _symbols;
+};
+
+#endif // BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+
+#if BACKWARD_HAS_BFD == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::libbfd>
+ : public TraceResolverLinuxBase {
+public:
+ TraceResolverLinuxImpl() : _bfd_loaded(false) {}
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ Dl_info symbol_info;
+
+ // trace.addr is a virtual address in memory pointing to some code.
+ // Let's try to find from which loaded object it comes from.
+ // The loaded object can be yourself btw.
+ if (!dladdr(trace.addr, &symbol_info)) {
+ return trace; // dat broken trace...
+ }
+
+ // Now we get in symbol_info:
+ // .dli_fname:
+ // pathname of the shared object that contains the address.
+ // .dli_fbase:
+ // where the object is loaded in memory.
+ // .dli_sname:
+ // the name of the nearest symbol to trace.addr, we expect a
+ // function name.
+ // .dli_saddr:
+ // the exact address corresponding to .dli_sname.
+
+ if (symbol_info.dli_sname) {
+ trace.object_function = demangle(symbol_info.dli_sname);
+ }
+
+ if (!symbol_info.dli_fname) {
+ return trace;
+ }
+
+ trace.object_filename = resolve_exec_path(symbol_info);
+ bfd_fileobject *fobj;
+ // Before rushing to resolution need to ensure the executable
+ // file still can be used. For that compare inode numbers of
+ // what is stored by the executable's file path, and in the
+ // dli_fname, which not necessarily equals to the executable.
+ // It can be a shared library, or /proc/self/exe, and in the
+ // latter case has drawbacks. See the exec path resolution for
+ // details. In short - the dli object should be used only as
+ // the last resort.
+ // If inode numbers are equal, it is known dli_fname and the
+ // executable file are the same. This is guaranteed by Linux,
+ // because if the executable file is changed/deleted, it will
+ // be done in a new inode. The old file will be preserved in
+ // /proc/self/exe, and may even have inode 0. The latter can
+ // happen if the inode was actually reused, and the file was
+ // kept only in the main memory.
+ //
+ struct stat obj_stat;
+ struct stat dli_stat;
+ if (stat(trace.object_filename.c_str(), &obj_stat) == 0 &&
+ stat(symbol_info.dli_fname, &dli_stat) == 0 &&
+ obj_stat.st_ino == dli_stat.st_ino) {
+ // The executable file, and the shared object containing the
+ // address are the same file. Safe to use the original path.
+ // this is preferable. Libbfd will search for stripped debug
+ // symbols in the same directory.
+ fobj = load_object_with_bfd(trace.object_filename);
+ } else{
+ // The original object file was *deleted*! The only hope is
+ // that the debug symbols are either inside the shared
+ // object file, or are in the same directory, and this is
+ // not /proc/self/exe.
+ fobj = nullptr;
+ }
+ if (fobj == nullptr || !fobj->handle) {
+ fobj = load_object_with_bfd(symbol_info.dli_fname);
+ if (!fobj->handle) {
+ return trace;
+ }
+ }
+
+ find_sym_result *details_selected; // to be filled.
+
+ // trace.addr is the next instruction to be executed after returning
+ // from the nested stack frame. In C++ this usually relate to the next
+ // statement right after the function call that leaded to a new stack
+ // frame. This is not usually what you want to see when printing out a
+ // stacktrace...
+ find_sym_result details_call_site =
+ find_symbol_details(fobj, trace.addr, symbol_info.dli_fbase);
+ details_selected = &details_call_site;
+
+#if BACKWARD_HAS_UNWIND == 0
+ // ...this is why we also try to resolve the symbol that is right
+ // before the return address. If we are lucky enough, we will get the
+ // line of the function that was called. But if the code is optimized,
+ // we might get something absolutely not related since the compiler
+ // can reschedule the return address with inline functions and
+ // tail-call optimisation (among other things that I don't even know
+ // or cannot even dream about with my tiny limited brain).
+ find_sym_result details_adjusted_call_site = find_symbol_details(
+ fobj, (void *)(uintptr_t(trace.addr) - 1), symbol_info.dli_fbase);
+
+ // In debug mode, we should always get the right thing(TM).
+ if (details_call_site.found && details_adjusted_call_site.found) {
+ // Ok, we assume that details_adjusted_call_site is a better estimation.
+ details_selected = &details_adjusted_call_site;
+ trace.addr = (void *)(uintptr_t(trace.addr) - 1);
+ }
+
+ if (details_selected == &details_call_site && details_call_site.found) {
+ // we have to re-resolve the symbol in order to reset some
+ // internal state in BFD... so we can call backtrace_inliners
+ // thereafter...
+ details_call_site =
+ find_symbol_details(fobj, trace.addr, symbol_info.dli_fbase);
+ }
+#endif // BACKWARD_HAS_UNWIND
+
+ if (details_selected->found) {
+ if (details_selected->filename) {
+ trace.source.filename = details_selected->filename;
+ }
+ trace.source.line = details_selected->line;
+
+ if (details_selected->funcname) {
+ // this time we get the name of the function where the code is
+ // located, instead of the function were the address is
+ // located. In short, if the code was inlined, we get the
+ // function correspoding to the code. Else we already got in
+ // trace.function.
+ trace.source.function = demangle(details_selected->funcname);
+
+ if (!symbol_info.dli_sname) {
+ // for the case dladdr failed to find the symbol name of
+ // the function, we might as well try to put something
+ // here.
+ trace.object_function = trace.source.function;
+ }
+ }
+
+ // Maybe the source of the trace got inlined inside the function
+ // (trace.source.function). Let's see if we can get all the inlined
+ // calls along the way up to the initial call site.
+ trace.inliners = backtrace_inliners(fobj, *details_selected);
+
+#if 0
+ if (trace.inliners.size() == 0) {
+ // Maybe the trace was not inlined... or maybe it was and we
+ // are lacking the debug information. Let's try to make the
+ // world better and see if we can get the line number of the
+ // function (trace.source.function) now.
+ //
+ // We will get the location of where the function start (to be
+ // exact: the first instruction that really start the
+ // function), not where the name of the function is defined.
+ // This can be quite far away from the name of the function
+ // btw.
+ //
+ // If the source of the function is the same as the source of
+ // the trace, we cannot say if the trace was really inlined or
+ // not. However, if the filename of the source is different
+ // between the function and the trace... we can declare it as
+ // an inliner. This is not 100% accurate, but better than
+ // nothing.
+
+ if (symbol_info.dli_saddr) {
+ find_sym_result details = find_symbol_details(fobj,
+ symbol_info.dli_saddr,
+ symbol_info.dli_fbase);
+
+ if (details.found) {
+ ResolvedTrace::SourceLoc diy_inliner;
+ diy_inliner.line = details.line;
+ if (details.filename) {
+ diy_inliner.filename = details.filename;
+ }
+ if (details.funcname) {
+ diy_inliner.function = demangle(details.funcname);
+ } else {
+ diy_inliner.function = trace.source.function;
+ }
+ if (diy_inliner != trace.source) {
+ trace.inliners.push_back(diy_inliner);
+ }
+ }
+ }
+ }
+#endif
+ }
+
+ return trace;
+ }
+
+private:
+ bool _bfd_loaded;
+
+ typedef details::handle<bfd *,
+ details::deleter<bfd_boolean, bfd *, &bfd_close>>
+ bfd_handle_t;
+
+ typedef details::handle<asymbol **> bfd_symtab_t;
+
+ struct bfd_fileobject {
+ bfd_handle_t handle;
+ bfd_vma base_addr;
+ bfd_symtab_t symtab;
+ bfd_symtab_t dynamic_symtab;
+ };
+
+ typedef details::hashtable<std::string, bfd_fileobject>::type fobj_bfd_map_t;
+ fobj_bfd_map_t _fobj_bfd_map;
+
+ bfd_fileobject *load_object_with_bfd(const std::string &filename_object) {
+ using namespace details;
+
+ if (!_bfd_loaded) {
+ using namespace details;
+ bfd_init();
+ _bfd_loaded = true;
+ }
+
+ fobj_bfd_map_t::iterator it = _fobj_bfd_map.find(filename_object);
+ if (it != _fobj_bfd_map.end()) {
+ return &it->second;
+ }
+
+ // this new object is empty for now.
+ bfd_fileobject *r = &_fobj_bfd_map[filename_object];
+
+ // we do the work temporary in this one;
+ bfd_handle_t bfd_handle;
+
+ int fd = open(filename_object.c_str(), O_RDONLY);
+ bfd_handle.reset(bfd_fdopenr(filename_object.c_str(), "default", fd));
+ if (!bfd_handle) {
+ close(fd);
+ return r;
+ }
+
+ if (!bfd_check_format(bfd_handle.get(), bfd_object)) {
+ return r; // not an object? You lose.
+ }
+
+ if ((bfd_get_file_flags(bfd_handle.get()) & HAS_SYMS) == 0) {
+ return r; // that's what happen when you forget to compile in debug.
+ }
+
+ ssize_t symtab_storage_size = bfd_get_symtab_upper_bound(bfd_handle.get());
+
+ ssize_t dyn_symtab_storage_size =
+ bfd_get_dynamic_symtab_upper_bound(bfd_handle.get());
+
+ if (symtab_storage_size <= 0 && dyn_symtab_storage_size <= 0) {
+ return r; // weird, is the file is corrupted?
+ }
+
+ bfd_symtab_t symtab, dynamic_symtab;
+ ssize_t symcount = 0, dyn_symcount = 0;
+
+ if (symtab_storage_size > 0) {
+ symtab.reset(static_cast<bfd_symbol **>(
+ malloc(static_cast<size_t>(symtab_storage_size))));
+ symcount = bfd_canonicalize_symtab(bfd_handle.get(), symtab.get());
+ }
+
+ if (dyn_symtab_storage_size > 0) {
+ dynamic_symtab.reset(static_cast<bfd_symbol **>(
+ malloc(static_cast<size_t>(dyn_symtab_storage_size))));
+ dyn_symcount = bfd_canonicalize_dynamic_symtab(bfd_handle.get(),
+ dynamic_symtab.get());
+ }
+
+ if (symcount <= 0 && dyn_symcount <= 0) {
+ return r; // damned, that's a stripped file that you got there!
+ }
+
+ r->handle = move(bfd_handle);
+ r->symtab = move(symtab);
+ r->dynamic_symtab = move(dynamic_symtab);
+ return r;
+ }
+
+ struct find_sym_result {
+ bool found;
+ const char *filename;
+ const char *funcname;
+ unsigned int line;
+ };
+
+ struct find_sym_context {
+ TraceResolverLinuxImpl *self;
+ bfd_fileobject *fobj;
+ void *addr;
+ void *base_addr;
+ find_sym_result result;
+ };
+
+ find_sym_result find_symbol_details(bfd_fileobject *fobj, void *addr,
+ void *base_addr) {
+ find_sym_context context;
+ context.self = this;
+ context.fobj = fobj;
+ context.addr = addr;
+ context.base_addr = base_addr;
+ context.result.found = false;
+ bfd_map_over_sections(fobj->handle.get(), &find_in_section_trampoline,
+ static_cast<void *>(&context));
+ return context.result;
+ }
+
+ static void find_in_section_trampoline(bfd *, asection *section, void *data) {
+ find_sym_context *context = static_cast<find_sym_context *>(data);
+ context->self->find_in_section(
+ reinterpret_cast<bfd_vma>(context->addr),
+ reinterpret_cast<bfd_vma>(context->base_addr), context->fobj, section,
+ context->result);
+ }
+
+ void find_in_section(bfd_vma addr, bfd_vma base_addr, bfd_fileobject *fobj,
+ asection *section, find_sym_result &result) {
+ if (result.found)
+ return;
+
+#ifdef bfd_get_section_flags
+ if ((bfd_get_section_flags(fobj->handle.get(), section) & SEC_ALLOC) == 0)
+#else
+ if ((bfd_section_flags(section) & SEC_ALLOC) == 0)
+#endif
+ return; // a debug section is never loaded automatically.
+
+#ifdef bfd_get_section_vma
+ bfd_vma sec_addr = bfd_get_section_vma(fobj->handle.get(), section);
+#else
+ bfd_vma sec_addr = bfd_section_vma(section);
+#endif
+#ifdef bfd_get_section_size
+ bfd_size_type size = bfd_get_section_size(section);
+#else
+ bfd_size_type size = bfd_section_size(section);
+#endif
+
+ // are we in the boundaries of the section?
+ if (addr < sec_addr || addr >= sec_addr + size) {
+ addr -= base_addr; // oups, a relocated object, lets try again...
+ if (addr < sec_addr || addr >= sec_addr + size) {
+ return;
+ }
+ }
+
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
+#endif
+ if (!result.found && fobj->symtab) {
+ result.found = bfd_find_nearest_line(
+ fobj->handle.get(), section, fobj->symtab.get(), addr - sec_addr,
+ &result.filename, &result.funcname, &result.line);
+ }
+
+ if (!result.found && fobj->dynamic_symtab) {
+ result.found = bfd_find_nearest_line(
+ fobj->handle.get(), section, fobj->dynamic_symtab.get(),
+ addr - sec_addr, &result.filename, &result.funcname, &result.line);
+ }
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+ }
+
+ ResolvedTrace::source_locs_t
+ backtrace_inliners(bfd_fileobject *fobj, find_sym_result previous_result) {
+ // This function can be called ONLY after a SUCCESSFUL call to
+ // find_symbol_details. The state is global to the bfd_handle.
+ ResolvedTrace::source_locs_t results;
+ while (previous_result.found) {
+ find_sym_result result;
+ result.found = bfd_find_inliner_info(fobj->handle.get(), &result.filename,
+ &result.funcname, &result.line);
+
+ if (result
+ .found) /* and not (
+ cstrings_eq(previous_result.filename,
+ result.filename) and
+ cstrings_eq(previous_result.funcname, result.funcname)
+ and result.line == previous_result.line
+ )) */
+ {
+ ResolvedTrace::SourceLoc src_loc;
+ src_loc.line = result.line;
+ if (result.filename) {
+ src_loc.filename = result.filename;
+ }
+ if (result.funcname) {
+ src_loc.function = demangle(result.funcname);
+ }
+ results.push_back(src_loc);
+ }
+ previous_result = result;
+ }
+ return results;
+ }
+
+ bool cstrings_eq(const char *a, const char *b) {
+ if (!a || !b) {
+ return false;
+ }
+ return strcmp(a, b) == 0;
+ }
+};
+#endif // BACKWARD_HAS_BFD == 1
+
+#if BACKWARD_HAS_DW == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::libdw>
+ : public TraceResolverLinuxBase {
+public:
+ TraceResolverLinuxImpl() : _dwfl_handle_initialized(false) {}
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ using namespace details;
+
+ Dwarf_Addr trace_addr = (Dwarf_Addr)trace.addr;
+
+ if (!_dwfl_handle_initialized) {
+ // initialize dwfl...
+ _dwfl_cb.reset(new Dwfl_Callbacks);
+ _dwfl_cb->find_elf = &dwfl_linux_proc_find_elf;
+ _dwfl_cb->find_debuginfo = &dwfl_standard_find_debuginfo;
+ _dwfl_cb->debuginfo_path = 0;
+
+ _dwfl_handle.reset(dwfl_begin(_dwfl_cb.get()));
+ _dwfl_handle_initialized = true;
+
+ if (!_dwfl_handle) {
+ return trace;
+ }
+
+ // ...from the current process.
+ dwfl_report_begin(_dwfl_handle.get());
+ int r = dwfl_linux_proc_report(_dwfl_handle.get(), getpid());
+ dwfl_report_end(_dwfl_handle.get(), NULL, NULL);
+ if (r < 0) {
+ return trace;
+ }
+ }
+
+ if (!_dwfl_handle) {
+ return trace;
+ }
+
+ // find the module (binary object) that contains the trace's address.
+ // This is not using any debug information, but the addresses ranges of
+ // all the currently loaded binary object.
+ Dwfl_Module *mod = dwfl_addrmodule(_dwfl_handle.get(), trace_addr);
+ if (mod) {
+ // now that we found it, lets get the name of it, this will be the
+ // full path to the running binary or one of the loaded library.
+ const char *module_name = dwfl_module_info(mod, 0, 0, 0, 0, 0, 0, 0);
+ if (module_name) {
+ trace.object_filename = module_name;
+ }
+ // We also look after the name of the symbol, equal or before this
+ // address. This is found by walking the symtab. We should get the
+ // symbol corresponding to the function (mangled) containing the
+ // address. If the code corresponding to the address was inlined,
+ // this is the name of the out-most inliner function.
+ const char *sym_name = dwfl_module_addrname(mod, trace_addr);
+ if (sym_name) {
+ trace.object_function = demangle(sym_name);
+ }
+ }
+
+ // now let's get serious, and find out the source location (file and
+ // line number) of the address.
+
+ // This function will look in .debug_aranges for the address and map it
+ // to the location of the compilation unit DIE in .debug_info and
+ // return it.
+ Dwarf_Addr mod_bias = 0;
+ Dwarf_Die *cudie = dwfl_module_addrdie(mod, trace_addr, &mod_bias);
+
+#if 1
+ if (!cudie) {
+ // Sadly clang does not generate the section .debug_aranges, thus
+ // dwfl_module_addrdie will fail early. Clang doesn't either set
+ // the lowpc/highpc/range info for every compilation unit.
+ //
+ // So in order to save the world:
+ // for every compilation unit, we will iterate over every single
+ // DIEs. Normally functions should have a lowpc/highpc/range, which
+ // we will use to infer the compilation unit.
+
+ // note that this is probably badly inefficient.
+ while ((cudie = dwfl_module_nextcu(mod, cudie, &mod_bias))) {
+ Dwarf_Die die_mem;
+ Dwarf_Die *fundie =
+ find_fundie_by_pc(cudie, trace_addr - mod_bias, &die_mem);
+ if (fundie) {
+ break;
+ }
+ }
+ }
+#endif
+
+//#define BACKWARD_I_DO_NOT_RECOMMEND_TO_ENABLE_THIS_HORRIBLE_PIECE_OF_CODE
+#ifdef BACKWARD_I_DO_NOT_RECOMMEND_TO_ENABLE_THIS_HORRIBLE_PIECE_OF_CODE
+ if (!cudie) {
+ // If it's still not enough, lets dive deeper in the shit, and try
+ // to save the world again: for every compilation unit, we will
+ // load the corresponding .debug_line section, and see if we can
+ // find our address in it.
+
+ Dwarf_Addr cfi_bias;
+ Dwarf_CFI *cfi_cache = dwfl_module_eh_cfi(mod, &cfi_bias);
+
+ Dwarf_Addr bias;
+ while ((cudie = dwfl_module_nextcu(mod, cudie, &bias))) {
+ if (dwarf_getsrc_die(cudie, trace_addr - bias)) {
+
+ // ...but if we get a match, it might be a false positive
+ // because our (address - bias) might as well be valid in a
+ // different compilation unit. So we throw our last card on
+ // the table and lookup for the address into the .eh_frame
+ // section.
+
+ handle<Dwarf_Frame *> frame;
+ dwarf_cfi_addrframe(cfi_cache, trace_addr - cfi_bias, &frame);
+ if (frame) {
+ break;
+ }
+ }
+ }
+ }
+#endif
+
+ if (!cudie) {
+ return trace; // this time we lost the game :/
+ }
+
+ // Now that we have a compilation unit DIE, this function will be able
+ // to load the corresponding section in .debug_line (if not already
+ // loaded) and hopefully find the source location mapped to our
+ // address.
+ Dwarf_Line *srcloc = dwarf_getsrc_die(cudie, trace_addr - mod_bias);
+
+ if (srcloc) {
+ const char *srcfile = dwarf_linesrc(srcloc, 0, 0);
+ if (srcfile) {
+ trace.source.filename = srcfile;
+ }
+ int line = 0, col = 0;
+ dwarf_lineno(srcloc, &line);
+ dwarf_linecol(srcloc, &col);
+ trace.source.line = line;
+ trace.source.col = col;
+ }
+
+ deep_first_search_by_pc(cudie, trace_addr - mod_bias,
+ inliners_search_cb(trace));
+ if (trace.source.function.size() == 0) {
+ // fallback.
+ trace.source.function = trace.object_function;
+ }
+
+ return trace;
+ }
+
+private:
+ typedef details::handle<Dwfl *, details::deleter<void, Dwfl *, &dwfl_end>>
+ dwfl_handle_t;
+ details::handle<Dwfl_Callbacks *, details::default_delete<Dwfl_Callbacks *>>
+ _dwfl_cb;
+ dwfl_handle_t _dwfl_handle;
+ bool _dwfl_handle_initialized;
+
+ // defined here because in C++98, template function cannot take locally
+ // defined types... grrr.
+ struct inliners_search_cb {
+ void operator()(Dwarf_Die *die) {
+ switch (dwarf_tag(die)) {
+ const char *name;
+ case DW_TAG_subprogram:
+ if ((name = dwarf_diename(die))) {
+ trace.source.function = name;
+ }
+ break;
+
+ case DW_TAG_inlined_subroutine:
+ ResolvedTrace::SourceLoc sloc;
+ Dwarf_Attribute attr_mem;
+
+ if ((name = dwarf_diename(die))) {
+ sloc.function = name;
+ }
+ if ((name = die_call_file(die))) {
+ sloc.filename = name;
+ }
+
+ Dwarf_Word line = 0, col = 0;
+ dwarf_formudata(dwarf_attr(die, DW_AT_call_line, &attr_mem), &line);
+ dwarf_formudata(dwarf_attr(die, DW_AT_call_column, &attr_mem), &col);
+ sloc.line = (unsigned)line;
+ sloc.col = (unsigned)col;
+
+ trace.inliners.push_back(sloc);
+ break;
+ };
+ }
+ ResolvedTrace &trace;
+ inliners_search_cb(ResolvedTrace &t) : trace(t) {}
+ };
+
+ static bool die_has_pc(Dwarf_Die *die, Dwarf_Addr pc) {
+ Dwarf_Addr low, high;
+
+ // continuous range
+ if (dwarf_hasattr(die, DW_AT_low_pc) && dwarf_hasattr(die, DW_AT_high_pc)) {
+ if (dwarf_lowpc(die, &low) != 0) {
+ return false;
+ }
+ if (dwarf_highpc(die, &high) != 0) {
+ Dwarf_Attribute attr_mem;
+ Dwarf_Attribute *attr = dwarf_attr(die, DW_AT_high_pc, &attr_mem);
+ Dwarf_Word value;
+ if (dwarf_formudata(attr, &value) != 0) {
+ return false;
+ }
+ high = low + value;
+ }
+ return pc >= low && pc < high;
+ }
+
+ // non-continuous range.
+ Dwarf_Addr base;
+ ptrdiff_t offset = 0;
+ while ((offset = dwarf_ranges(die, offset, &base, &low, &high)) > 0) {
+ if (pc >= low && pc < high) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static Dwarf_Die *find_fundie_by_pc(Dwarf_Die *parent_die, Dwarf_Addr pc,
+ Dwarf_Die *result) {
+ if (dwarf_child(parent_die, result) != 0) {
+ return 0;
+ }
+
+ Dwarf_Die *die = result;
+ do {
+ switch (dwarf_tag(die)) {
+ case DW_TAG_subprogram:
+ case DW_TAG_inlined_subroutine:
+ if (die_has_pc(die, pc)) {
+ return result;
+ }
+ };
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ dwarf_formflag(dwarf_attr(die, DW_AT_declaration, &attr_mem),
+ &declaration);
+ if (!declaration) {
+ // let's be curious and look deeper in the tree,
+ // function are not necessarily at the first level, but
+ // might be nested inside a namespace, structure etc.
+ Dwarf_Die die_mem;
+ Dwarf_Die *indie = find_fundie_by_pc(die, pc, &die_mem);
+ if (indie) {
+ *result = die_mem;
+ return result;
+ }
+ }
+ } while (dwarf_siblingof(die, result) == 0);
+ return 0;
+ }
+
+ template <typename CB>
+ static bool deep_first_search_by_pc(Dwarf_Die *parent_die, Dwarf_Addr pc,
+ CB cb) {
+ Dwarf_Die die_mem;
+ if (dwarf_child(parent_die, &die_mem) != 0) {
+ return false;
+ }
+
+ bool branch_has_pc = false;
+ Dwarf_Die *die = &die_mem;
+ do {
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ dwarf_formflag(dwarf_attr(die, DW_AT_declaration, &attr_mem),
+ &declaration);
+ if (!declaration) {
+ // let's be curious and look deeper in the tree, function are
+ // not necessarily at the first level, but might be nested
+ // inside a namespace, structure, a function, an inlined
+ // function etc.
+ branch_has_pc = deep_first_search_by_pc(die, pc, cb);
+ }
+ if (!branch_has_pc) {
+ branch_has_pc = die_has_pc(die, pc);
+ }
+ if (branch_has_pc) {
+ cb(die);
+ }
+ } while (dwarf_siblingof(die, &die_mem) == 0);
+ return branch_has_pc;
+ }
+
+ static const char *die_call_file(Dwarf_Die *die) {
+ Dwarf_Attribute attr_mem;
+ Dwarf_Word file_idx = 0;
+
+ dwarf_formudata(dwarf_attr(die, DW_AT_call_file, &attr_mem), &file_idx);
+
+ if (file_idx == 0) {
+ return 0;
+ }
+
+ Dwarf_Die die_mem;
+ Dwarf_Die *cudie = dwarf_diecu(die, &die_mem, 0, 0);
+ if (!cudie) {
+ return 0;
+ }
+
+ Dwarf_Files *files = 0;
+ size_t nfiles;
+ dwarf_getsrcfiles(cudie, &files, &nfiles);
+ if (!files) {
+ return 0;
+ }
+
+ return dwarf_filesrc(files, file_idx, 0, 0);
+ }
+};
+#endif // BACKWARD_HAS_DW == 1
+
+#if BACKWARD_HAS_DWARF == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::libdwarf>
+ : public TraceResolverLinuxBase {
+public:
+ TraceResolverLinuxImpl() : _dwarf_loaded(false) {}
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ // trace.addr is a virtual address in memory pointing to some code.
+ // Let's try to find from which loaded object it comes from.
+ // The loaded object can be yourself btw.
+
+ Dl_info symbol_info;
+ int dladdr_result = 0;
+#if defined(__GLIBC__)
+ link_map *link_map;
+ // We request the link map so we can get information about offsets
+ dladdr_result =
+ dladdr1(trace.addr, &symbol_info, reinterpret_cast<void **>(&link_map),
+ RTLD_DL_LINKMAP);
+#else
+ // Android doesn't have dladdr1. Don't use the linker map.
+ dladdr_result = dladdr(trace.addr, &symbol_info);
+#endif
+ if (!dladdr_result) {
+ return trace; // dat broken trace...
+ }
+
+ // Now we get in symbol_info:
+ // .dli_fname:
+ // pathname of the shared object that contains the address.
+ // .dli_fbase:
+ // where the object is loaded in memory.
+ // .dli_sname:
+ // the name of the nearest symbol to trace.addr, we expect a
+ // function name.
+ // .dli_saddr:
+ // the exact address corresponding to .dli_sname.
+ //
+ // And in link_map:
+ // .l_addr:
+ // difference between the address in the ELF file and the address
+ // in memory
+ // l_name:
+ // absolute pathname where the object was found
+
+ if (symbol_info.dli_sname) {
+ trace.object_function = demangle(symbol_info.dli_sname);
+ }
+
+ if (!symbol_info.dli_fname) {
+ return trace;
+ }
+
+ trace.object_filename = resolve_exec_path(symbol_info);
+ dwarf_fileobject &fobj = load_object_with_dwarf(symbol_info.dli_fname);
+ if (!fobj.dwarf_handle) {
+ return trace; // sad, we couldn't load the object :(
+ }
+
+#if defined(__GLIBC__)
+ // Convert the address to a module relative one by looking at
+ // the module's loading address in the link map
+ Dwarf_Addr address = reinterpret_cast<uintptr_t>(trace.addr) -
+ reinterpret_cast<uintptr_t>(link_map->l_addr);
+#else
+ Dwarf_Addr address = reinterpret_cast<uintptr_t>(trace.addr);
+#endif
+
+ if (trace.object_function.empty()) {
+ symbol_cache_t::iterator it = fobj.symbol_cache.lower_bound(address);
+
+ if (it != fobj.symbol_cache.end()) {
+ if (it->first != address) {
+ if (it != fobj.symbol_cache.begin()) {
+ --it;
+ }
+ }
+ trace.object_function = demangle(it->second.c_str());
+ }
+ }
+
+ // Get the Compilation Unit DIE for the address
+ Dwarf_Die die = find_die(fobj, address);
+
+ if (!die) {
+ return trace; // this time we lost the game :/
+ }
+
+ // libdwarf doesn't give us direct access to its objects, it always
+ // allocates a copy for the caller. We keep that copy alive in a cache
+ // and we deallocate it later when it's no longer required.
+ die_cache_entry &die_object = get_die_cache(fobj, die);
+ if (die_object.isEmpty())
+ return trace; // We have no line section for this DIE
+
+ die_linemap_t::iterator it = die_object.line_section.lower_bound(address);
+
+ if (it != die_object.line_section.end()) {
+ if (it->first != address) {
+ if (it == die_object.line_section.begin()) {
+ // If we are on the first item of the line section
+ // but the address does not match it means that
+ // the address is below the range of the DIE. Give up.
+ return trace;
+ } else {
+ --it;
+ }
+ }
+ } else {
+ return trace; // We didn't find the address.
+ }
+
+ // Get the Dwarf_Line that the address points to and call libdwarf
+ // to get source file, line and column info.
+ Dwarf_Line line = die_object.line_buffer[it->second];
+ Dwarf_Error error = DW_DLE_NE;
+
+ char *filename;
+ if (dwarf_linesrc(line, &filename, &error) == DW_DLV_OK) {
+ trace.source.filename = std::string(filename);
+ dwarf_dealloc(fobj.dwarf_handle.get(), filename, DW_DLA_STRING);
+ }
+
+ Dwarf_Unsigned number = 0;
+ if (dwarf_lineno(line, &number, &error) == DW_DLV_OK) {
+ trace.source.line = number;
+ } else {
+ trace.source.line = 0;
+ }
+
+ if (dwarf_lineoff_b(line, &number, &error) == DW_DLV_OK) {
+ trace.source.col = number;
+ } else {
+ trace.source.col = 0;
+ }
+
+ std::vector<std::string> namespace_stack;
+ deep_first_search_by_pc(fobj, die, address, namespace_stack,
+ inliners_search_cb(trace, fobj, die));
+
+ dwarf_dealloc(fobj.dwarf_handle.get(), die, DW_DLA_DIE);
+
+ return trace;
+ }
+
+public:
+ static int close_dwarf(Dwarf_Debug dwarf) {
+ return dwarf_finish(dwarf, NULL);
+ }
+
+private:
+ bool _dwarf_loaded;
+
+ typedef details::handle<int, details::deleter<int, int, &::close>>
+ dwarf_file_t;
+
+ typedef details::handle<Elf *, details::deleter<int, Elf *, &elf_end>>
+ dwarf_elf_t;
+
+ typedef details::handle<Dwarf_Debug,
+ details::deleter<int, Dwarf_Debug, &close_dwarf>>
+ dwarf_handle_t;
+
+ typedef std::map<Dwarf_Addr, int> die_linemap_t;
+
+ typedef std::map<Dwarf_Off, Dwarf_Off> die_specmap_t;
+
+ struct die_cache_entry {
+ die_specmap_t spec_section;
+ die_linemap_t line_section;
+ Dwarf_Line *line_buffer;
+ Dwarf_Signed line_count;
+ Dwarf_Line_Context line_context;
+
+ inline bool isEmpty() {
+ return line_buffer == NULL || line_count == 0 || line_context == NULL ||
+ line_section.empty();
+ }
+
+ die_cache_entry() : line_buffer(0), line_count(0), line_context(0) {}
+
+ ~die_cache_entry() {
+ if (line_context) {
+ dwarf_srclines_dealloc_b(line_context);
+ }
+ }
+ };
+
+ typedef std::map<Dwarf_Off, die_cache_entry> die_cache_t;
+
+ typedef std::map<uintptr_t, std::string> symbol_cache_t;
+
+ struct dwarf_fileobject {
+ dwarf_file_t file_handle;
+ dwarf_elf_t elf_handle;
+ dwarf_handle_t dwarf_handle;
+ symbol_cache_t symbol_cache;
+
+ // Die cache
+ die_cache_t die_cache;
+ die_cache_entry *current_cu;
+ };
+
+ typedef details::hashtable<std::string, dwarf_fileobject>::type
+ fobj_dwarf_map_t;
+ fobj_dwarf_map_t _fobj_dwarf_map;
+
+ static bool cstrings_eq(const char *a, const char *b) {
+ if (!a || !b) {
+ return false;
+ }
+ return strcmp(a, b) == 0;
+ }
+
+ dwarf_fileobject &load_object_with_dwarf(const std::string &filename_object) {
+
+ if (!_dwarf_loaded) {
+ // Set the ELF library operating version
+ // If that fails there's nothing we can do
+ _dwarf_loaded = elf_version(EV_CURRENT) != EV_NONE;
+ }
+
+ fobj_dwarf_map_t::iterator it = _fobj_dwarf_map.find(filename_object);
+ if (it != _fobj_dwarf_map.end()) {
+ return it->second;
+ }
+
+ // this new object is empty for now
+ dwarf_fileobject &r = _fobj_dwarf_map[filename_object];
+
+ dwarf_file_t file_handle;
+ file_handle.reset(open(filename_object.c_str(), O_RDONLY));
+ if (file_handle.get() < 0) {
+ return r;
+ }
+
+ // Try to get an ELF handle. We need to read the ELF sections
+ // because we want to see if there is a .gnu_debuglink section
+ // that points to a split debug file
+ dwarf_elf_t elf_handle;
+ elf_handle.reset(elf_begin(file_handle.get(), ELF_C_READ, NULL));
+ if (!elf_handle) {
+ return r;
+ }
+
+ const char *e_ident = elf_getident(elf_handle.get(), 0);
+ if (!e_ident) {
+ return r;
+ }
+
+ // Get the number of sections
+ // We use the new APIs as elf_getshnum is deprecated
+ size_t shdrnum = 0;
+ if (elf_getshdrnum(elf_handle.get(), &shdrnum) == -1) {
+ return r;
+ }
+
+ // Get the index to the string section
+ size_t shdrstrndx = 0;
+ if (elf_getshdrstrndx(elf_handle.get(), &shdrstrndx) == -1) {
+ return r;
+ }
+
+ std::string debuglink;
+ // Iterate through the ELF sections to try to get a gnu_debuglink
+ // note and also to cache the symbol table.
+ // We go the preprocessor way to avoid having to create templated
+ // classes or using gelf (which might throw a compiler error if 64 bit
+ // is not supported
+#define ELF_GET_DATA(ARCH) \
+ Elf_Scn *elf_section = 0; \
+ Elf_Data *elf_data = 0; \
+ Elf##ARCH##_Shdr *section_header = 0; \
+ Elf_Scn *symbol_section = 0; \
+ size_t symbol_count = 0; \
+ size_t symbol_strings = 0; \
+ Elf##ARCH##_Sym *symbol = 0; \
+ const char *section_name = 0; \
+ \
+ while ((elf_section = elf_nextscn(elf_handle.get(), elf_section)) != NULL) { \
+ section_header = elf##ARCH##_getshdr(elf_section); \
+ if (section_header == NULL) { \
+ return r; \
+ } \
+ \
+ if ((section_name = elf_strptr(elf_handle.get(), shdrstrndx, \
+ section_header->sh_name)) == NULL) { \
+ return r; \
+ } \
+ \
+ if (cstrings_eq(section_name, ".gnu_debuglink")) { \
+ elf_data = elf_getdata(elf_section, NULL); \
+ if (elf_data && elf_data->d_size > 0) { \
+ debuglink = \
+ std::string(reinterpret_cast<const char *>(elf_data->d_buf)); \
+ } \
+ } \
+ \
+ switch (section_header->sh_type) { \
+ case SHT_SYMTAB: \
+ symbol_section = elf_section; \
+ symbol_count = section_header->sh_size / section_header->sh_entsize; \
+ symbol_strings = section_header->sh_link; \
+ break; \
+ \
+ /* We use .dynsyms as a last resort, we prefer .symtab */ \
+ case SHT_DYNSYM: \
+ if (!symbol_section) { \
+ symbol_section = elf_section; \
+ symbol_count = section_header->sh_size / section_header->sh_entsize; \
+ symbol_strings = section_header->sh_link; \
+ } \
+ break; \
+ } \
+ } \
+ \
+ if (symbol_section && symbol_count && symbol_strings) { \
+ elf_data = elf_getdata(symbol_section, NULL); \
+ symbol = reinterpret_cast<Elf##ARCH##_Sym *>(elf_data->d_buf); \
+ for (size_t i = 0; i < symbol_count; ++i) { \
+ int type = ELF##ARCH##_ST_TYPE(symbol->st_info); \
+ if (type == STT_FUNC && symbol->st_value > 0) { \
+ r.symbol_cache[symbol->st_value] = std::string( \
+ elf_strptr(elf_handle.get(), symbol_strings, symbol->st_name)); \
+ } \
+ ++symbol; \
+ } \
+ }
+
+ if (e_ident[EI_CLASS] == ELFCLASS32) {
+ ELF_GET_DATA(32)
+ } else if (e_ident[EI_CLASS] == ELFCLASS64) {
+ // libelf might have been built without 64 bit support
+#if __LIBELF64
+ ELF_GET_DATA(64)
+#endif
+ }
+
+ if (!debuglink.empty()) {
+ // We have a debuglink section! Open an elf instance on that
+ // file instead. If we can't open the file, then return
+ // the elf handle we had already opened.
+ dwarf_file_t debuglink_file;
+ debuglink_file.reset(open(debuglink.c_str(), O_RDONLY));
+ if (debuglink_file.get() > 0) {
+ dwarf_elf_t debuglink_elf;
+ debuglink_elf.reset(elf_begin(debuglink_file.get(), ELF_C_READ, NULL));
+
+ // If we have a valid elf handle, return the new elf handle
+ // and file handle and discard the original ones
+ if (debuglink_elf) {
+ elf_handle = move(debuglink_elf);
+ file_handle = move(debuglink_file);
+ }
+ }
+ }
+
+ // Ok, we have a valid ELF handle, let's try to get debug symbols
+ Dwarf_Debug dwarf_debug;
+ Dwarf_Error error = DW_DLE_NE;
+ dwarf_handle_t dwarf_handle;
+
+ int dwarf_result = dwarf_elf_init(elf_handle.get(), DW_DLC_READ, NULL, NULL,
+ &dwarf_debug, &error);
+
+ // We don't do any special handling for DW_DLV_NO_ENTRY specially.
+ // If we get an error, or the file doesn't have debug information
+ // we just return.
+ if (dwarf_result != DW_DLV_OK) {
+ return r;
+ }
+
+ dwarf_handle.reset(dwarf_debug);
+
+ r.file_handle = move(file_handle);
+ r.elf_handle = move(elf_handle);
+ r.dwarf_handle = move(dwarf_handle);
+
+ return r;
+ }
+
+ die_cache_entry &get_die_cache(dwarf_fileobject &fobj, Dwarf_Die die) {
+ Dwarf_Error error = DW_DLE_NE;
+
+ // Get the die offset, we use it as the cache key
+ Dwarf_Off die_offset;
+ if (dwarf_dieoffset(die, &die_offset, &error) != DW_DLV_OK) {
+ die_offset = 0;
+ }
+
+ die_cache_t::iterator it = fobj.die_cache.find(die_offset);
+
+ if (it != fobj.die_cache.end()) {
+ fobj.current_cu = &it->second;
+ return it->second;
+ }
+
+ die_cache_entry &de = fobj.die_cache[die_offset];
+ fobj.current_cu = &de;
+
+ Dwarf_Addr line_addr;
+ Dwarf_Small table_count;
+
+ // The addresses in the line section are not fully sorted (they might
+ // be sorted by block of code belonging to the same file), which makes
+ // it necessary to do so before searching is possible.
+ //
+ // As libdwarf allocates a copy of everything, let's get the contents
+ // of the line section and keep it around. We also create a map of
+ // program counter to line table indices so we can search by address
+ // and get the line buffer index.
+ //
+ // To make things more difficult, the same address can span more than
+ // one line, so we need to keep the index pointing to the first line
+ // by using insert instead of the map's [ operator.
+
+ // Get the line context for the DIE
+ if (dwarf_srclines_b(die, 0, &table_count, &de.line_context, &error) ==
+ DW_DLV_OK) {
+ // Get the source lines for this line context, to be deallocated
+ // later
+ if (dwarf_srclines_from_linecontext(de.line_context, &de.line_buffer,
+ &de.line_count,
+ &error) == DW_DLV_OK) {
+
+ // Add all the addresses to our map
+ for (int i = 0; i < de.line_count; i++) {
+ if (dwarf_lineaddr(de.line_buffer[i], &line_addr, &error) !=
+ DW_DLV_OK) {
+ line_addr = 0;
+ }
+ de.line_section.insert(std::pair<Dwarf_Addr, int>(line_addr, i));
+ }
+ }
+ }
+
+ // For each CU, cache the function DIEs that contain the
+ // DW_AT_specification attribute. When building with -g3 the function
+ // DIEs are separated in declaration and specification, with the
+ // declaration containing only the name and parameters and the
+ // specification the low/high pc and other compiler attributes.
+ //
+ // We cache those specifications so we don't skip over the declarations,
+ // because they have no pc, and we can do namespace resolution for
+ // DWARF function names.
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Die current_die = 0;
+ if (dwarf_child(die, &current_die, &error) == DW_DLV_OK) {
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+
+ Dwarf_Half tag_value;
+ dwarf_tag(current_die, &tag_value, &error);
+
+ if (tag_value == DW_TAG_subprogram ||
+ tag_value == DW_TAG_inlined_subroutine) {
+
+ Dwarf_Bool has_attr = 0;
+ if (dwarf_hasattr(current_die, DW_AT_specification, &has_attr,
+ &error) == DW_DLV_OK) {
+ if (has_attr) {
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(current_die, DW_AT_specification, &attr_mem,
+ &error) == DW_DLV_OK) {
+ Dwarf_Off spec_offset = 0;
+ if (dwarf_formref(attr_mem, &spec_offset, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Off spec_die_offset;
+ if (dwarf_dieoffset(current_die, &spec_die_offset, &error) ==
+ DW_DLV_OK) {
+ de.spec_section[spec_offset] = spec_die_offset;
+ }
+ }
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ }
+ }
+
+ int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (result == DW_DLV_ERROR) {
+ break;
+ } else if (result == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ current_die = sibling_die;
+ }
+ }
+ return de;
+ }
+
+ static Dwarf_Die get_referenced_die(Dwarf_Debug dwarf, Dwarf_Die die,
+ Dwarf_Half attr, bool global) {
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Attribute attr_mem;
+
+ Dwarf_Die found_die = NULL;
+ if (dwarf_attr(die, attr, &attr_mem, &error) == DW_DLV_OK) {
+ Dwarf_Off offset;
+ int result = 0;
+ if (global) {
+ result = dwarf_global_formref(attr_mem, &offset, &error);
+ } else {
+ result = dwarf_formref(attr_mem, &offset, &error);
+ }
+
+ if (result == DW_DLV_OK) {
+ if (dwarf_offdie(dwarf, offset, &found_die, &error) != DW_DLV_OK) {
+ found_die = NULL;
+ }
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ return found_die;
+ }
+
+ static std::string get_referenced_die_name(Dwarf_Debug dwarf, Dwarf_Die die,
+ Dwarf_Half attr, bool global) {
+ Dwarf_Error error = DW_DLE_NE;
+ std::string value;
+
+ Dwarf_Die found_die = get_referenced_die(dwarf, die, attr, global);
+
+ if (found_die) {
+ char *name;
+ if (dwarf_diename(found_die, &name, &error) == DW_DLV_OK) {
+ if (name) {
+ value = std::string(name);
+ }
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ }
+ dwarf_dealloc(dwarf, found_die, DW_DLA_DIE);
+ }
+
+ return value;
+ }
+
+ // Returns a spec DIE linked to the passed one. The caller should
+ // deallocate the DIE
+ static Dwarf_Die get_spec_die(dwarf_fileobject &fobj, Dwarf_Die die) {
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Off die_offset;
+ if (fobj.current_cu &&
+ dwarf_die_CU_offset(die, &die_offset, &error) == DW_DLV_OK) {
+ die_specmap_t::iterator it =
+ fobj.current_cu->spec_section.find(die_offset);
+
+ // If we have a DIE that completes the current one, check if
+ // that one has the pc we are looking for
+ if (it != fobj.current_cu->spec_section.end()) {
+ Dwarf_Die spec_die = 0;
+ if (dwarf_offdie(dwarf, it->second, &spec_die, &error) == DW_DLV_OK) {
+ return spec_die;
+ }
+ }
+ }
+
+ // Maybe we have an abstract origin DIE with the function information?
+ return get_referenced_die(fobj.dwarf_handle.get(), die,
+ DW_AT_abstract_origin, true);
+ }
+
+ static bool die_has_pc(dwarf_fileobject &fobj, Dwarf_Die die, Dwarf_Addr pc) {
+ Dwarf_Addr low_pc = 0, high_pc = 0;
+ Dwarf_Half high_pc_form = 0;
+ Dwarf_Form_Class return_class;
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ bool has_lowpc = false;
+ bool has_highpc = false;
+ bool has_ranges = false;
+
+ if (dwarf_lowpc(die, &low_pc, &error) == DW_DLV_OK) {
+ // If we have a low_pc check if there is a high pc.
+ // If we don't have a high pc this might mean we have a base
+ // address for the ranges list or just an address.
+ has_lowpc = true;
+
+ if (dwarf_highpc_b(die, &high_pc, &high_pc_form, &return_class, &error) ==
+ DW_DLV_OK) {
+ // We do have a high pc. In DWARF 4+ this is an offset from the
+ // low pc, but in earlier versions it's an absolute address.
+
+ has_highpc = true;
+ // In DWARF 2/3 this would be a DW_FORM_CLASS_ADDRESS
+ if (return_class == DW_FORM_CLASS_CONSTANT) {
+ high_pc = low_pc + high_pc;
+ }
+
+ // We have low and high pc, check if our address
+ // is in that range
+ return pc >= low_pc && pc < high_pc;
+ }
+ } else {
+ // Reset the low_pc, in case dwarf_lowpc failing set it to some
+ // undefined value.
+ low_pc = 0;
+ }
+
+ // Check if DW_AT_ranges is present and search for the PC in the
+ // returned ranges list. We always add the low_pc, as it not set it will
+ // be 0, in case we had a DW_AT_low_pc and DW_AT_ranges pair
+ bool result = false;
+
+ Dwarf_Attribute attr;
+ if (dwarf_attr(die, DW_AT_ranges, &attr, &error) == DW_DLV_OK) {
+
+ Dwarf_Off offset;
+ if (dwarf_global_formref(attr, &offset, &error) == DW_DLV_OK) {
+ Dwarf_Ranges *ranges;
+ Dwarf_Signed ranges_count = 0;
+ Dwarf_Unsigned byte_count = 0;
+
+ if (dwarf_get_ranges_a(dwarf, offset, die, &ranges, &ranges_count,
+ &byte_count, &error) == DW_DLV_OK) {
+ has_ranges = ranges_count != 0;
+ for (int i = 0; i < ranges_count; i++) {
+ if (ranges[i].dwr_addr1 != 0 &&
+ pc >= ranges[i].dwr_addr1 + low_pc &&
+ pc < ranges[i].dwr_addr2 + low_pc) {
+ result = true;
+ break;
+ }
+ }
+ dwarf_ranges_dealloc(dwarf, ranges, ranges_count);
+ }
+ }
+ }
+
+ // Last attempt. We might have a single address set as low_pc.
+ if (!result && low_pc != 0 && pc == low_pc) {
+ result = true;
+ }
+
+ // If we don't have lowpc, highpc and ranges maybe this DIE is a
+ // declaration that relies on a DW_AT_specification DIE that happens
+ // later. Use the specification cache we filled when we loaded this CU.
+ if (!result && (!has_lowpc && !has_highpc && !has_ranges)) {
+ Dwarf_Die spec_die = get_spec_die(fobj, die);
+ if (spec_die) {
+ result = die_has_pc(fobj, spec_die, pc);
+ dwarf_dealloc(dwarf, spec_die, DW_DLA_DIE);
+ }
+ }
+
+ return result;
+ }
+
+ static void get_type(Dwarf_Debug dwarf, Dwarf_Die die, std::string &type) {
+ Dwarf_Error error = DW_DLE_NE;
+
+ Dwarf_Die child = 0;
+ if (dwarf_child(die, &child, &error) == DW_DLV_OK) {
+ get_type(dwarf, child, type);
+ }
+
+ if (child) {
+ type.insert(0, "::");
+ dwarf_dealloc(dwarf, child, DW_DLA_DIE);
+ }
+
+ char *name;
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ type.insert(0, std::string(name));
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ } else {
+ type.insert(0, "<unknown>");
+ }
+ }
+
+ static std::string get_type_by_signature(Dwarf_Debug dwarf, Dwarf_Die die) {
+ Dwarf_Error error = DW_DLE_NE;
+
+ Dwarf_Sig8 signature;
+ Dwarf_Bool has_attr = 0;
+ if (dwarf_hasattr(die, DW_AT_signature, &has_attr, &error) == DW_DLV_OK) {
+ if (has_attr) {
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(die, DW_AT_signature, &attr_mem, &error) == DW_DLV_OK) {
+ if (dwarf_formsig8(attr_mem, &signature, &error) != DW_DLV_OK) {
+ return std::string("<no type signature>");
+ }
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ }
+
+ Dwarf_Unsigned next_cu_header;
+ Dwarf_Sig8 tu_signature;
+ std::string result;
+ bool found = false;
+
+ while (dwarf_next_cu_header_d(dwarf, 0, 0, 0, 0, 0, 0, 0, &tu_signature, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+
+ if (strncmp(signature.signature, tu_signature.signature, 8) == 0) {
+ Dwarf_Die type_cu_die = 0;
+ if (dwarf_siblingof_b(dwarf, 0, 0, &type_cu_die, &error) == DW_DLV_OK) {
+ Dwarf_Die child_die = 0;
+ if (dwarf_child(type_cu_die, &child_die, &error) == DW_DLV_OK) {
+ get_type(dwarf, child_die, result);
+ found = !result.empty();
+ dwarf_dealloc(dwarf, child_die, DW_DLA_DIE);
+ }
+ dwarf_dealloc(dwarf, type_cu_die, DW_DLA_DIE);
+ }
+ }
+ }
+
+ if (found) {
+ while (dwarf_next_cu_header_d(dwarf, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ // Reset the cu header state. Unfortunately, libdwarf's
+ // next_cu_header API keeps its own iterator per Dwarf_Debug
+ // that can't be reset. We need to keep fetching elements until
+ // the end.
+ }
+ } else {
+ // If we couldn't resolve the type just print out the signature
+ std::ostringstream string_stream;
+ string_stream << "<0x" << std::hex << std::setfill('0');
+ for (int i = 0; i < 8; ++i) {
+ string_stream << std::setw(2) << std::hex
+ << (int)(unsigned char)(signature.signature[i]);
+ }
+ string_stream << ">";
+ result = string_stream.str();
+ }
+ return result;
+ }
+
+ struct type_context_t {
+ bool is_const;
+ bool is_typedef;
+ bool has_type;
+ bool has_name;
+ std::string text;
+
+ type_context_t()
+ : is_const(false), is_typedef(false), has_type(false), has_name(false) {
+ }
+ };
+
+ // Types are resolved from right to left: we get the variable name first
+ // and then all specifiers (like const or pointer) in a chain of DW_AT_type
+ // DIEs. Call this function recursively until we get a complete type
+ // string.
+ static void set_parameter_string(dwarf_fileobject &fobj, Dwarf_Die die,
+ type_context_t &context) {
+ char *name;
+ Dwarf_Error error = DW_DLE_NE;
+
+ // typedefs contain also the base type, so we skip it and only
+ // print the typedef name
+ if (!context.is_typedef) {
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ if (!context.text.empty()) {
+ context.text.insert(0, " ");
+ }
+ context.text.insert(0, std::string(name));
+ dwarf_dealloc(fobj.dwarf_handle.get(), name, DW_DLA_STRING);
+ }
+ } else {
+ context.is_typedef = false;
+ context.has_type = true;
+ if (context.is_const) {
+ context.text.insert(0, "const ");
+ context.is_const = false;
+ }
+ }
+
+ bool next_type_is_const = false;
+ bool is_keyword = true;
+
+ Dwarf_Half tag = 0;
+ Dwarf_Bool has_attr = 0;
+ if (dwarf_tag(die, &tag, &error) == DW_DLV_OK) {
+ switch (tag) {
+ case DW_TAG_structure_type:
+ case DW_TAG_union_type:
+ case DW_TAG_class_type:
+ case DW_TAG_enumeration_type:
+ context.has_type = true;
+ if (dwarf_hasattr(die, DW_AT_signature, &has_attr, &error) ==
+ DW_DLV_OK) {
+ // If we have a signature it means the type is defined
+ // in .debug_types, so we need to load the DIE pointed
+ // at by the signature and resolve it
+ if (has_attr) {
+ std::string type =
+ get_type_by_signature(fobj.dwarf_handle.get(), die);
+ if (context.is_const)
+ type.insert(0, "const ");
+
+ if (!context.text.empty())
+ context.text.insert(0, " ");
+ context.text.insert(0, type);
+ }
+
+ // Treat enums like typedefs, and skip printing its
+ // base type
+ context.is_typedef = (tag == DW_TAG_enumeration_type);
+ }
+ break;
+ case DW_TAG_const_type:
+ next_type_is_const = true;
+ break;
+ case DW_TAG_pointer_type:
+ context.text.insert(0, "*");
+ break;
+ case DW_TAG_reference_type:
+ context.text.insert(0, "&");
+ break;
+ case DW_TAG_restrict_type:
+ context.text.insert(0, "restrict ");
+ break;
+ case DW_TAG_rvalue_reference_type:
+ context.text.insert(0, "&&");
+ break;
+ case DW_TAG_volatile_type:
+ context.text.insert(0, "volatile ");
+ break;
+ case DW_TAG_typedef:
+ // Propagate the const-ness to the next type
+ // as typedefs are linked to its base type
+ next_type_is_const = context.is_const;
+ context.is_typedef = true;
+ context.has_type = true;
+ break;
+ case DW_TAG_base_type:
+ context.has_type = true;
+ break;
+ case DW_TAG_formal_parameter:
+ context.has_name = true;
+ break;
+ default:
+ is_keyword = false;
+ break;
+ }
+ }
+
+ if (!is_keyword && context.is_const) {
+ context.text.insert(0, "const ");
+ }
+
+ context.is_const = next_type_is_const;
+
+ Dwarf_Die ref =
+ get_referenced_die(fobj.dwarf_handle.get(), die, DW_AT_type, true);
+ if (ref) {
+ set_parameter_string(fobj, ref, context);
+ dwarf_dealloc(fobj.dwarf_handle.get(), ref, DW_DLA_DIE);
+ }
+
+ if (!context.has_type && context.has_name) {
+ context.text.insert(0, "void ");
+ context.has_type = true;
+ }
+ }
+
+ // Resolve the function return type and parameters
+ static void set_function_parameters(std::string &function_name,
+ std::vector<std::string> &ns,
+ dwarf_fileobject &fobj, Dwarf_Die die) {
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Die current_die = 0;
+ std::string parameters;
+ bool has_spec = true;
+ // Check if we have a spec DIE. If we do we use it as it contains
+ // more information, like parameter names.
+ Dwarf_Die spec_die = get_spec_die(fobj, die);
+ if (!spec_die) {
+ has_spec = false;
+ spec_die = die;
+ }
+
+ std::vector<std::string>::const_iterator it = ns.begin();
+ std::string ns_name;
+ for (it = ns.begin(); it < ns.end(); ++it) {
+ ns_name.append(*it).append("::");
+ }
+
+ if (!ns_name.empty()) {
+ function_name.insert(0, ns_name);
+ }
+
+ // See if we have a function return type. It can be either on the
+ // current die or in its spec one (usually true for inlined functions)
+ std::string return_type =
+ get_referenced_die_name(dwarf, die, DW_AT_type, true);
+ if (return_type.empty()) {
+ return_type = get_referenced_die_name(dwarf, spec_die, DW_AT_type, true);
+ }
+ if (!return_type.empty()) {
+ return_type.append(" ");
+ function_name.insert(0, return_type);
+ }
+
+ if (dwarf_child(spec_die, &current_die, &error) == DW_DLV_OK) {
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+
+ Dwarf_Half tag_value;
+ dwarf_tag(current_die, &tag_value, &error);
+
+ if (tag_value == DW_TAG_formal_parameter) {
+ // Ignore artificial (ie, compiler generated) parameters
+ bool is_artificial = false;
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(current_die, DW_AT_artificial, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Bool flag = 0;
+ if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) {
+ is_artificial = flag != 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (!is_artificial) {
+ type_context_t context;
+ set_parameter_string(fobj, current_die, context);
+
+ if (parameters.empty()) {
+ parameters.append("(");
+ } else {
+ parameters.append(", ");
+ }
+ parameters.append(context.text);
+ }
+ }
+
+ int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (result == DW_DLV_ERROR) {
+ break;
+ } else if (result == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ current_die = sibling_die;
+ }
+ }
+ if (parameters.empty())
+ parameters = "(";
+ parameters.append(")");
+
+ // If we got a spec DIE we need to deallocate it
+ if (has_spec)
+ dwarf_dealloc(dwarf, spec_die, DW_DLA_DIE);
+
+ function_name.append(parameters);
+ }
+
+ // defined here because in C++98, template function cannot take locally
+ // defined types... grrr.
+ struct inliners_search_cb {
+ void operator()(Dwarf_Die die, std::vector<std::string> &ns) {
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Half tag_value;
+ Dwarf_Attribute attr_mem;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+
+ dwarf_tag(die, &tag_value, &error);
+
+ switch (tag_value) {
+ char *name;
+ case DW_TAG_subprogram:
+ if (!trace.source.function.empty())
+ break;
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ trace.source.function = std::string(name);
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ } else {
+ // We don't have a function name in this DIE.
+ // Check if there is a referenced non-defining
+ // declaration.
+ trace.source.function =
+ get_referenced_die_name(dwarf, die, DW_AT_abstract_origin, true);
+ if (trace.source.function.empty()) {
+ trace.source.function =
+ get_referenced_die_name(dwarf, die, DW_AT_specification, true);
+ }
+ }
+
+ // Append the function parameters, if available
+ set_function_parameters(trace.source.function, ns, fobj, die);
+
+ // If the object function name is empty, it's possible that
+ // there is no dynamic symbol table (maybe the executable
+ // was stripped or not built with -rdynamic). See if we have
+ // a DWARF linkage name to use instead. We try both
+ // linkage_name and MIPS_linkage_name because the MIPS tag
+ // was the unofficial one until it was adopted in DWARF4.
+ // Old gcc versions generate MIPS_linkage_name
+ if (trace.object_function.empty()) {
+ details::demangler demangler;
+
+ if (dwarf_attr(die, DW_AT_linkage_name, &attr_mem, &error) !=
+ DW_DLV_OK) {
+ if (dwarf_attr(die, DW_AT_MIPS_linkage_name, &attr_mem, &error) !=
+ DW_DLV_OK) {
+ break;
+ }
+ }
+
+ char *linkage;
+ if (dwarf_formstring(attr_mem, &linkage, &error) == DW_DLV_OK) {
+ trace.object_function = demangler.demangle(linkage);
+ dwarf_dealloc(dwarf, linkage, DW_DLA_STRING);
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ break;
+
+ case DW_TAG_inlined_subroutine:
+ ResolvedTrace::SourceLoc sloc;
+
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ sloc.function = std::string(name);
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ } else {
+ // We don't have a name for this inlined DIE, it could
+ // be that there is an abstract origin instead.
+ // Get the DW_AT_abstract_origin value, which is a
+ // reference to the source DIE and try to get its name
+ sloc.function =
+ get_referenced_die_name(dwarf, die, DW_AT_abstract_origin, true);
+ }
+
+ set_function_parameters(sloc.function, ns, fobj, die);
+
+ std::string file = die_call_file(dwarf, die, cu_die);
+ if (!file.empty())
+ sloc.filename = file;
+
+ Dwarf_Unsigned number = 0;
+ if (dwarf_attr(die, DW_AT_call_line, &attr_mem, &error) == DW_DLV_OK) {
+ if (dwarf_formudata(attr_mem, &number, &error) == DW_DLV_OK) {
+ sloc.line = number;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (dwarf_attr(die, DW_AT_call_column, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ if (dwarf_formudata(attr_mem, &number, &error) == DW_DLV_OK) {
+ sloc.col = number;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ trace.inliners.push_back(sloc);
+ break;
+ };
+ }
+ ResolvedTrace &trace;
+ dwarf_fileobject &fobj;
+ Dwarf_Die cu_die;
+ inliners_search_cb(ResolvedTrace &t, dwarf_fileobject &f, Dwarf_Die c)
+ : trace(t), fobj(f), cu_die(c) {}
+ };
+
+ static Dwarf_Die find_fundie_by_pc(dwarf_fileobject &fobj,
+ Dwarf_Die parent_die, Dwarf_Addr pc,
+ Dwarf_Die result) {
+ Dwarf_Die current_die = 0;
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+
+ if (dwarf_child(parent_die, &current_die, &error) != DW_DLV_OK) {
+ return NULL;
+ }
+
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+ Dwarf_Half tag_value;
+ dwarf_tag(current_die, &tag_value, &error);
+
+ switch (tag_value) {
+ case DW_TAG_subprogram:
+ case DW_TAG_inlined_subroutine:
+ if (die_has_pc(fobj, current_die, pc)) {
+ return current_die;
+ }
+ };
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(current_die, DW_AT_declaration, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Bool flag = 0;
+ if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) {
+ declaration = flag != 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (!declaration) {
+ // let's be curious and look deeper in the tree, functions are
+ // not necessarily at the first level, but might be nested
+ // inside a namespace, structure, a function, an inlined
+ // function etc.
+ Dwarf_Die die_mem = 0;
+ Dwarf_Die indie = find_fundie_by_pc(fobj, current_die, pc, die_mem);
+ if (indie) {
+ result = die_mem;
+ return result;
+ }
+ }
+
+ int res = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (res == DW_DLV_ERROR) {
+ return NULL;
+ } else if (res == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != parent_die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ current_die = sibling_die;
+ }
+ return NULL;
+ }
+
+ template <typename CB>
+ static bool deep_first_search_by_pc(dwarf_fileobject &fobj,
+ Dwarf_Die parent_die, Dwarf_Addr pc,
+ std::vector<std::string> &ns, CB cb) {
+ Dwarf_Die current_die = 0;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+
+ if (dwarf_child(parent_die, &current_die, &error) != DW_DLV_OK) {
+ return false;
+ }
+
+ bool branch_has_pc = false;
+ bool has_namespace = false;
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+
+ Dwarf_Half tag;
+ if (dwarf_tag(current_die, &tag, &error) == DW_DLV_OK) {
+ if (tag == DW_TAG_namespace || tag == DW_TAG_class_type) {
+ char *ns_name = NULL;
+ if (dwarf_diename(current_die, &ns_name, &error) == DW_DLV_OK) {
+ if (ns_name) {
+ ns.push_back(std::string(ns_name));
+ } else {
+ ns.push_back("<unknown>");
+ }
+ dwarf_dealloc(dwarf, ns_name, DW_DLA_STRING);
+ } else {
+ ns.push_back("<unknown>");
+ }
+ has_namespace = true;
+ }
+ }
+
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ if (tag != DW_TAG_class_type &&
+ dwarf_attr(current_die, DW_AT_declaration, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Bool flag = 0;
+ if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) {
+ declaration = flag != 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (!declaration) {
+ // let's be curious and look deeper in the tree, function are
+ // not necessarily at the first level, but might be nested
+ // inside a namespace, structure, a function, an inlined
+ // function etc.
+ branch_has_pc = deep_first_search_by_pc(fobj, current_die, pc, ns, cb);
+ }
+
+ if (!branch_has_pc) {
+ branch_has_pc = die_has_pc(fobj, current_die, pc);
+ }
+
+ if (branch_has_pc) {
+ cb(current_die, ns);
+ }
+
+ int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (result == DW_DLV_ERROR) {
+ return false;
+ } else if (result == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != parent_die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ if (has_namespace) {
+ has_namespace = false;
+ ns.pop_back();
+ }
+ current_die = sibling_die;
+ }
+
+ if (has_namespace) {
+ ns.pop_back();
+ }
+ return branch_has_pc;
+ }
+
+ static std::string die_call_file(Dwarf_Debug dwarf, Dwarf_Die die,
+ Dwarf_Die cu_die) {
+ Dwarf_Attribute attr_mem;
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Unsigned file_index;
+
+ std::string file;
+
+ if (dwarf_attr(die, DW_AT_call_file, &attr_mem, &error) == DW_DLV_OK) {
+ if (dwarf_formudata(attr_mem, &file_index, &error) != DW_DLV_OK) {
+ file_index = 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+
+ if (file_index == 0) {
+ return file;
+ }
+
+ char **srcfiles = 0;
+ Dwarf_Signed file_count = 0;
+ if (dwarf_srcfiles(cu_die, &srcfiles, &file_count, &error) == DW_DLV_OK) {
+ if (file_count > 0 && file_index <= static_cast<Dwarf_Unsigned>(file_count)) {
+ file = std::string(srcfiles[file_index - 1]);
+ }
+
+ // Deallocate all strings!
+ for (int i = 0; i < file_count; ++i) {
+ dwarf_dealloc(dwarf, srcfiles[i], DW_DLA_STRING);
+ }
+ dwarf_dealloc(dwarf, srcfiles, DW_DLA_LIST);
+ }
+ }
+ return file;
+ }
+
+ Dwarf_Die find_die(dwarf_fileobject &fobj, Dwarf_Addr addr) {
+ // Let's get to work! First see if we have a debug_aranges section so
+ // we can speed up the search
+
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Arange *aranges;
+ Dwarf_Signed arange_count;
+
+ Dwarf_Die returnDie;
+ bool found = false;
+ if (dwarf_get_aranges(dwarf, &aranges, &arange_count, &error) !=
+ DW_DLV_OK) {
+ aranges = NULL;
+ }
+
+ if (aranges) {
+ // We have aranges. Get the one where our address is.
+ Dwarf_Arange arange;
+ if (dwarf_get_arange(aranges, arange_count, addr, &arange, &error) ==
+ DW_DLV_OK) {
+
+ // We found our address. Get the compilation-unit DIE offset
+ // represented by the given address range.
+ Dwarf_Off cu_die_offset;
+ if (dwarf_get_cu_die_offset(arange, &cu_die_offset, &error) ==
+ DW_DLV_OK) {
+ // Get the DIE at the offset returned by the aranges search.
+ // We set is_info to 1 to specify that the offset is from
+ // the .debug_info section (and not .debug_types)
+ int dwarf_result =
+ dwarf_offdie_b(dwarf, cu_die_offset, 1, &returnDie, &error);
+
+ found = dwarf_result == DW_DLV_OK;
+ }
+ dwarf_dealloc(dwarf, arange, DW_DLA_ARANGE);
+ }
+ }
+
+ if (found)
+ return returnDie; // The caller is responsible for freeing the die
+
+ // The search for aranges failed. Try to find our address by scanning
+ // all compilation units.
+ Dwarf_Unsigned next_cu_header;
+ Dwarf_Half tag = 0;
+ returnDie = 0;
+
+ while (!found &&
+ dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+
+ if (returnDie)
+ dwarf_dealloc(dwarf, returnDie, DW_DLA_DIE);
+
+ if (dwarf_siblingof(dwarf, 0, &returnDie, &error) == DW_DLV_OK) {
+ if ((dwarf_tag(returnDie, &tag, &error) == DW_DLV_OK) &&
+ tag == DW_TAG_compile_unit) {
+ if (die_has_pc(fobj, returnDie, addr)) {
+ found = true;
+ }
+ }
+ }
+ }
+
+ if (found) {
+ while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ // Reset the cu header state. Libdwarf's next_cu_header API
+ // keeps its own iterator per Dwarf_Debug that can't be reset.
+ // We need to keep fetching elements until the end.
+ }
+ }
+
+ if (found)
+ return returnDie;
+
+ // We couldn't find any compilation units with ranges or a high/low pc.
+ // Try again by looking at all DIEs in all compilation units.
+ Dwarf_Die cudie;
+ while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ if (dwarf_siblingof(dwarf, 0, &cudie, &error) == DW_DLV_OK) {
+ Dwarf_Die die_mem = 0;
+ Dwarf_Die resultDie = find_fundie_by_pc(fobj, cudie, addr, die_mem);
+
+ if (resultDie) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (found) {
+ while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ // Reset the cu header state. Libdwarf's next_cu_header API
+ // keeps its own iterator per Dwarf_Debug that can't be reset.
+ // We need to keep fetching elements until the end.
+ }
+ }
+
+ if (found)
+ return cudie;
+
+ // We failed.
+ return NULL;
+ }
+};
+#endif // BACKWARD_HAS_DWARF == 1
+
+template <>
+class TraceResolverImpl<system_tag::linux_tag>
+ : public TraceResolverLinuxImpl<trace_resolver_tag::current> {};
+
+#endif // BACKWARD_SYSTEM_LINUX
+
+#ifdef BACKWARD_SYSTEM_DARWIN
+
+template <typename STACKTRACE_TAG> class TraceResolverDarwinImpl;
+
+template <>
+class TraceResolverDarwinImpl<trace_resolver_tag::backtrace_symbol>
+ : public TraceResolverImplBase {
+public:
+ void load_addresses(void *const*addresses, int address_count) override {
+ if (address_count == 0) {
+ return;
+ }
+ _symbols.reset(backtrace_symbols(addresses, address_count));
+ }
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ // parse:
+ // <n> <file> <addr> <mangled-name> + <offset>
+ char *filename = _symbols[trace.idx];
+
+ // skip "<n> "
+ while (*filename && *filename != ' ')
+ filename++;
+ while (*filename == ' ')
+ filename++;
+
+ // find start of <mangled-name> from end (<file> may contain a space)
+ char *p = filename + strlen(filename) - 1;
+ // skip to start of " + <offset>"
+ while (p > filename && *p != ' ')
+ p--;
+ while (p > filename && *p == ' ')
+ p--;
+ while (p > filename && *p != ' ')
+ p--;
+ while (p > filename && *p == ' ')
+ p--;
+ char *funcname_end = p + 1;
+
+ // skip to start of "<manged-name>"
+ while (p > filename && *p != ' ')
+ p--;
+ char *funcname = p + 1;
+
+ // skip to start of " <addr> "
+ while (p > filename && *p == ' ')
+ p--;
+ while (p > filename && *p != ' ')
+ p--;
+ while (p > filename && *p == ' ')
+ p--;
+
+ // skip "<file>", handling the case where it contains a
+ char *filename_end = p + 1;
+ if (p == filename) {
+ // something went wrong, give up
+ filename_end = filename + strlen(filename);
+ funcname = filename_end;
+ }
+ trace.object_filename.assign(
+ filename, filename_end); // ok even if filename_end is the ending \0
+ // (then we assign entire string)
+
+ if (*funcname) { // if it's not end of string
+ *funcname_end = '\0';
+
+ trace.object_function = this->demangle(funcname);
+ trace.object_function += " ";
+ trace.object_function += (funcname_end + 1);
+ trace.source.function = trace.object_function; // we cannot do better.
+ }
+ return trace;
+ }
+
+private:
+ details::handle<char **> _symbols;
+};
+
+template <>
+class TraceResolverImpl<system_tag::darwin_tag>
+ : public TraceResolverDarwinImpl<trace_resolver_tag::current> {};
+
+#endif // BACKWARD_SYSTEM_DARWIN
+
+#ifdef BACKWARD_SYSTEM_WINDOWS
+
+// Load all symbol info
+// Based on:
+// https://stackoverflow.com/questions/6205981/windows-c-stack-trace-from-a-running-app/28276227#28276227
+
+struct module_data {
+ std::string image_name;
+ std::string module_name;
+ void *base_address;
+ DWORD load_size;
+};
+
+class get_mod_info {
+ HANDLE process;
+ static const int buffer_length = 4096;
+
+public:
+ get_mod_info(HANDLE h) : process(h) {}
+
+ module_data operator()(HMODULE module) {
+ module_data ret;
+ char temp[buffer_length];
+ MODULEINFO mi;
+
+ GetModuleInformation(process, module, &mi, sizeof(mi));
+ ret.base_address = mi.lpBaseOfDll;
+ ret.load_size = mi.SizeOfImage;
+
+ GetModuleFileNameExA(process, module, temp, sizeof(temp));
+ ret.image_name = temp;
+ GetModuleBaseNameA(process, module, temp, sizeof(temp));
+ ret.module_name = temp;
+ std::vector<char> img(ret.image_name.begin(), ret.image_name.end());
+ std::vector<char> mod(ret.module_name.begin(), ret.module_name.end());
+ SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address,
+ ret.load_size);
+ return ret;
+ }
+};
+
+template <> class TraceResolverImpl<system_tag::windows_tag>
+ : public TraceResolverImplBase {
+public:
+ TraceResolverImpl() {
+
+ HANDLE process = GetCurrentProcess();
+
+ std::vector<module_data> modules;
+ DWORD cbNeeded;
+ std::vector<HMODULE> module_handles(1);
+ SymInitialize(process, NULL, false);
+ DWORD symOptions = SymGetOptions();
+ symOptions |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME;
+ SymSetOptions(symOptions);
+ EnumProcessModules(process, &module_handles[0],
+ module_handles.size() * sizeof(HMODULE), &cbNeeded);
+ module_handles.resize(cbNeeded / sizeof(HMODULE));
+ EnumProcessModules(process, &module_handles[0],
+ module_handles.size() * sizeof(HMODULE), &cbNeeded);
+ std::transform(module_handles.begin(), module_handles.end(),
+ std::back_inserter(modules), get_mod_info(process));
+ void *base = modules[0].base_address;
+ IMAGE_NT_HEADERS *h = ImageNtHeader(base);
+ image_type = h->FileHeader.Machine;
+ }
+
+ static const int max_sym_len = 255;
+ struct symbol_t {
+ SYMBOL_INFO sym;
+ char buffer[max_sym_len];
+ } sym;
+
+ DWORD64 displacement;
+
+ ResolvedTrace resolve(ResolvedTrace t) override {
+ HANDLE process = GetCurrentProcess();
+
+ char name[256];
+
+ memset(&sym, 0, sizeof(sym));
+ sym.sym.SizeOfStruct = sizeof(SYMBOL_INFO);
+ sym.sym.MaxNameLen = max_sym_len;
+
+ if (!SymFromAddr(process, (ULONG64)t.addr, &displacement, &sym.sym)) {
+ // TODO: error handling everywhere
+ char* lpMsgBuf;
+ DWORD dw = GetLastError();
+
+ if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+ FORMAT_MESSAGE_FROM_SYSTEM |
+ FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL, dw, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+ (char*)&lpMsgBuf, 0, NULL)) {
+ std::fprintf(stderr, "%s\n", lpMsgBuf);
+ LocalFree(lpMsgBuf);
+ }
+
+ // abort();
+ }
+ UnDecorateSymbolName(sym.sym.Name, (PSTR)name, 256, UNDNAME_COMPLETE);
+
+ DWORD offset = 0;
+ IMAGEHLP_LINE line;
+ if (SymGetLineFromAddr(process, (ULONG64)t.addr, &offset, &line)) {
+ t.object_filename = line.FileName;
+ t.source.filename = line.FileName;
+ t.source.line = line.LineNumber;
+ t.source.col = offset;
+ }
+
+ t.source.function = name;
+ t.object_filename = "";
+ t.object_function = name;
+
+ return t;
+ }
+
+ DWORD machine_type() const { return image_type; }
+
+private:
+ DWORD image_type;
+};
+
+#endif
+
+class TraceResolver : public TraceResolverImpl<system_tag::current_tag> {};
+
+/*************** CODE SNIPPET ***************/
+
+class SourceFile {
+public:
+ typedef std::vector<std::pair<unsigned, std::string>> lines_t;
+
+ SourceFile() {}
+ SourceFile(const std::string &path) {
+ // 1. If BACKWARD_CXX_SOURCE_PREFIXES is set then assume it contains
+ // a colon-separated list of path prefixes. Try prepending each
+ // to the given path until a valid file is found.
+ const std::vector<std::string> &prefixes = get_paths_from_env_variable();
+ for (size_t i = 0; i < prefixes.size(); ++i) {
+ // Double slashes (//) should not be a problem.
+ std::string new_path = prefixes[i] + '/' + path;
+ _file.reset(new std::ifstream(new_path.c_str()));
+ if (is_open())
+ break;
+ }
+ // 2. If no valid file found then fallback to opening the path as-is.
+ if (!_file || !is_open()) {
+ _file.reset(new std::ifstream(path.c_str()));
+ }
+ }
+ bool is_open() const { return _file->is_open(); }
+
+ lines_t &get_lines(unsigned line_start, unsigned line_count, lines_t &lines) {
+ using namespace std;
+ // This function make uses of the dumbest algo ever:
+ // 1) seek(0)
+ // 2) read lines one by one and discard until line_start
+ // 3) read line one by one until line_start + line_count
+ //
+ // If you are getting snippets many time from the same file, it is
+ // somewhat a waste of CPU, feel free to benchmark and propose a
+ // better solution ;)
+
+ _file->clear();
+ _file->seekg(0);
+ string line;
+ unsigned line_idx;
+
+ for (line_idx = 1; line_idx < line_start; ++line_idx) {
+ std::getline(*_file, line);
+ if (!*_file) {
+ return lines;
+ }
+ }
+
+ // think of it like a lambda in C++98 ;)
+ // but look, I will reuse it two times!
+ // What a good boy am I.
+ struct isspace {
+ bool operator()(char c) { return std::isspace(c); }
+ };
+
+ bool started = false;
+ for (; line_idx < line_start + line_count; ++line_idx) {
+ getline(*_file, line);
+ if (!*_file) {
+ return lines;
+ }
+ if (!started) {
+ if (std::find_if(line.begin(), line.end(), not_isspace()) == line.end())
+ continue;
+ started = true;
+ }
+ lines.push_back(make_pair(line_idx, line));
+ }
+
+ lines.erase(
+ std::find_if(lines.rbegin(), lines.rend(), not_isempty()).base(),
+ lines.end());
+ return lines;
+ }
+
+ lines_t get_lines(unsigned line_start, unsigned line_count) {
+ lines_t lines;
+ return get_lines(line_start, line_count, lines);
+ }
+
+ // there is no find_if_not in C++98, lets do something crappy to
+ // workaround.
+ struct not_isspace {
+ bool operator()(char c) { return !std::isspace(c); }
+ };
+ // and define this one here because C++98 is not happy with local defined
+ // struct passed to template functions, fuuuu.
+ struct not_isempty {
+ bool operator()(const lines_t::value_type &p) {
+ return !(std::find_if(p.second.begin(), p.second.end(), not_isspace()) ==
+ p.second.end());
+ }
+ };
+
+ void swap(SourceFile &b) { _file.swap(b._file); }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ SourceFile(SourceFile &&from) : _file(nullptr) { swap(from); }
+ SourceFile &operator=(SourceFile &&from) {
+ swap(from);
+ return *this;
+ }
+#else
+ explicit SourceFile(const SourceFile &from) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<SourceFile &>(from));
+ }
+ SourceFile &operator=(const SourceFile &from) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<SourceFile &>(from));
+ return *this;
+ }
+#endif
+
+private:
+ details::handle<std::ifstream *, details::default_delete<std::ifstream *>>
+ _file;
+
+ std::vector<std::string> get_paths_from_env_variable_impl() {
+ std::vector<std::string> paths;
+ const char *prefixes_str = std::getenv("BACKWARD_CXX_SOURCE_PREFIXES");
+ if (prefixes_str && prefixes_str[0]) {
+ paths = details::split_source_prefixes(prefixes_str);
+ }
+ return paths;
+ }
+
+ const std::vector<std::string> &get_paths_from_env_variable() {
+ static std::vector<std::string> paths = get_paths_from_env_variable_impl();
+ return paths;
+ }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ SourceFile(const SourceFile &) = delete;
+ SourceFile &operator=(const SourceFile &) = delete;
+#endif
+};
+
+class SnippetFactory {
+public:
+ typedef SourceFile::lines_t lines_t;
+
+ lines_t get_snippet(const std::string &filename, unsigned line_start,
+ unsigned context_size) {
+
+ SourceFile &src_file = get_src_file(filename);
+ unsigned start = line_start - context_size / 2;
+ return src_file.get_lines(start, context_size);
+ }
+
+ lines_t get_combined_snippet(const std::string &filename_a, unsigned line_a,
+ const std::string &filename_b, unsigned line_b,
+ unsigned context_size) {
+ SourceFile &src_file_a = get_src_file(filename_a);
+ SourceFile &src_file_b = get_src_file(filename_b);
+
+ lines_t lines =
+ src_file_a.get_lines(line_a - context_size / 4, context_size / 2);
+ src_file_b.get_lines(line_b - context_size / 4, context_size / 2, lines);
+ return lines;
+ }
+
+ lines_t get_coalesced_snippet(const std::string &filename, unsigned line_a,
+ unsigned line_b, unsigned context_size) {
+ SourceFile &src_file = get_src_file(filename);
+
+ using std::max;
+ using std::min;
+ unsigned a = min(line_a, line_b);
+ unsigned b = max(line_a, line_b);
+
+ if ((b - a) < (context_size / 3)) {
+ return src_file.get_lines((a + b - context_size + 1) / 2, context_size);
+ }
+
+ lines_t lines = src_file.get_lines(a - context_size / 4, context_size / 2);
+ src_file.get_lines(b - context_size / 4, context_size / 2, lines);
+ return lines;
+ }
+
+private:
+ typedef details::hashtable<std::string, SourceFile>::type src_files_t;
+ src_files_t _src_files;
+
+ SourceFile &get_src_file(const std::string &filename) {
+ src_files_t::iterator it = _src_files.find(filename);
+ if (it != _src_files.end()) {
+ return it->second;
+ }
+ SourceFile &new_src_file = _src_files[filename];
+ new_src_file = SourceFile(filename);
+ return new_src_file;
+ }
+};
+
+/*************** PRINTER ***************/
+
+namespace ColorMode {
+enum type { automatic, never, always };
+}
+
+class cfile_streambuf : public std::streambuf {
+public:
+ cfile_streambuf(FILE *_sink) : sink(_sink) {}
+ int_type underflow() override { return traits_type::eof(); }
+ int_type overflow(int_type ch) override {
+ if (traits_type::not_eof(ch) && fputc(ch, sink) != EOF) {
+ return ch;
+ }
+ return traits_type::eof();
+ }
+
+ std::streamsize xsputn(const char_type *s, std::streamsize count) override {
+ return static_cast<std::streamsize>(
+ fwrite(s, sizeof *s, static_cast<size_t>(count), sink));
+ }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+public:
+ cfile_streambuf(const cfile_streambuf &) = delete;
+ cfile_streambuf &operator=(const cfile_streambuf &) = delete;
+#else
+private:
+ cfile_streambuf(const cfile_streambuf &);
+ cfile_streambuf &operator=(const cfile_streambuf &);
+#endif
+
+private:
+ FILE *sink;
+ std::vector<char> buffer;
+};
+
+#ifdef BACKWARD_SYSTEM_LINUX
+
+namespace Color {
+enum type { yellow = 33, purple = 35, reset = 39 };
+} // namespace Color
+
+class Colorize {
+public:
+ Colorize(std::ostream &os) : _os(os), _reset(false), _enabled(false) {}
+
+ void activate(ColorMode::type mode) { _enabled = mode == ColorMode::always; }
+
+ void activate(ColorMode::type mode, FILE *fp) { activate(mode, fileno(fp)); }
+
+ void set_color(Color::type ccode) {
+ if (!_enabled)
+ return;
+
+ // I assume that the terminal can handle basic colors. Seriously I
+ // don't want to deal with all the termcap shit.
+ _os << "\033[" << static_cast<int>(ccode) << "m";
+ _reset = (ccode != Color::reset);
+ }
+
+ ~Colorize() {
+ if (_reset) {
+ set_color(Color::reset);
+ }
+ }
+
+private:
+ void activate(ColorMode::type mode, int fd) {
+ activate(mode == ColorMode::automatic && isatty(fd) ? ColorMode::always
+ : mode);
+ }
+
+ std::ostream &_os;
+ bool _reset;
+ bool _enabled;
+};
+
+#else // ndef BACKWARD_SYSTEM_LINUX
+
+namespace Color {
+enum type { yellow = 0, purple = 0, reset = 0 };
+} // namespace Color
+
+class Colorize {
+public:
+ Colorize(std::ostream &) {}
+ void activate(ColorMode::type) {}
+ void activate(ColorMode::type, FILE *) {}
+ void set_color(Color::type) {}
+};
+
+#endif // BACKWARD_SYSTEM_LINUX
+
+class Printer {
+public:
+ bool snippet;
+ ColorMode::type color_mode;
+ bool address;
+ bool object;
+ int inliner_context_size;
+ int trace_context_size;
+
+ Printer()
+ : snippet(true), color_mode(ColorMode::automatic), address(false),
+ object(false), inliner_context_size(5), trace_context_size(7) {}
+
+ template <typename ST> FILE *print(ST &st, FILE *fp = stderr) {
+ cfile_streambuf obuf(fp);
+ std::ostream os(&obuf);
+ Colorize colorize(os);
+ colorize.activate(color_mode, fp);
+ print_stacktrace(st, os, colorize);
+ return fp;
+ }
+
+ template <typename ST> std::ostream &print(ST &st, std::ostream &os) {
+ Colorize colorize(os);
+ colorize.activate(color_mode);
+ print_stacktrace(st, os, colorize);
+ return os;
+ }
+
+ template <typename IT>
+ FILE *print(IT begin, IT end, FILE *fp = stderr, size_t thread_id = 0) {
+ cfile_streambuf obuf(fp);
+ std::ostream os(&obuf);
+ Colorize colorize(os);
+ colorize.activate(color_mode, fp);
+ print_stacktrace(begin, end, os, thread_id, colorize);
+ return fp;
+ }
+
+ template <typename IT>
+ std::ostream &print(IT begin, IT end, std::ostream &os,
+ size_t thread_id = 0) {
+ Colorize colorize(os);
+ colorize.activate(color_mode);
+ print_stacktrace(begin, end, os, thread_id, colorize);
+ return os;
+ }
+
+ TraceResolver const &resolver() const { return _resolver; }
+
+private:
+ TraceResolver _resolver;
+ SnippetFactory _snippets;
+
+ template <typename ST>
+ void print_stacktrace(ST &st, std::ostream &os, Colorize &colorize) {
+ print_header(os, st.thread_id());
+ _resolver.load_stacktrace(st);
+ for (size_t trace_idx = st.size(); trace_idx > 0; --trace_idx) {
+ print_trace(os, _resolver.resolve(st[trace_idx - 1]), colorize);
+ }
+ }
+
+ template <typename IT>
+ void print_stacktrace(IT begin, IT end, std::ostream &os, size_t thread_id,
+ Colorize &colorize) {
+ print_header(os, thread_id);
+ for (; begin != end; ++begin) {
+ print_trace(os, *begin, colorize);
+ }
+ }
+
+ void print_header(std::ostream &os, size_t thread_id) {
+ os << "Stack trace (most recent call last)";
+ if (thread_id) {
+ os << " in thread " << thread_id;
+ }
+ os << ":\n";
+ }
+
+ void print_trace(std::ostream &os, const ResolvedTrace &trace,
+ Colorize &colorize) {
+ os << "#" << std::left << std::setw(2) << trace.idx << std::right;
+ bool already_indented = true;
+
+ if (!trace.source.filename.size() || object) {
+ os << " Object \"" << trace.object_filename << "\", at " << trace.addr
+ << ", in " << trace.object_function << "\n";
+ already_indented = false;
+ }
+
+ for (size_t inliner_idx = trace.inliners.size(); inliner_idx > 0;
+ --inliner_idx) {
+ if (!already_indented) {
+ os << " ";
+ }
+ const ResolvedTrace::SourceLoc &inliner_loc =
+ trace.inliners[inliner_idx - 1];
+ print_source_loc(os, " | ", inliner_loc);
+ if (snippet) {
+ print_snippet(os, " | ", inliner_loc, colorize, Color::purple,
+ inliner_context_size);
+ }
+ already_indented = false;
+ }
+
+ if (trace.source.filename.size()) {
+ if (!already_indented) {
+ os << " ";
+ }
+ print_source_loc(os, " ", trace.source, trace.addr);
+ if (snippet) {
+ print_snippet(os, " ", trace.source, colorize, Color::yellow,
+ trace_context_size);
+ }
+ }
+ }
+
+ void print_snippet(std::ostream &os, const char *indent,
+ const ResolvedTrace::SourceLoc &source_loc,
+ Colorize &colorize, Color::type color_code,
+ int context_size) {
+ using namespace std;
+ typedef SnippetFactory::lines_t lines_t;
+
+ lines_t lines = _snippets.get_snippet(source_loc.filename, source_loc.line,
+ static_cast<unsigned>(context_size));
+
+ for (lines_t::const_iterator it = lines.begin(); it != lines.end(); ++it) {
+ if (it->first == source_loc.line) {
+ colorize.set_color(color_code);
+ os << indent << ">";
+ } else {
+ os << indent << " ";
+ }
+ os << std::setw(4) << it->first << ": " << it->second << "\n";
+ if (it->first == source_loc.line) {
+ colorize.set_color(Color::reset);
+ }
+ }
+ }
+
+ void print_source_loc(std::ostream &os, const char *indent,
+ const ResolvedTrace::SourceLoc &source_loc,
+ void *addr = nullptr) {
+ os << indent << "Source \"" << source_loc.filename << "\", line "
+ << source_loc.line << ", in " << source_loc.function;
+
+ if (address && addr != nullptr) {
+ os << " [" << addr << "]";
+ }
+ os << "\n";
+ }
+};
+
+/*************** SIGNALS HANDLING ***************/
+
+#if defined(BACKWARD_SYSTEM_LINUX) || defined(BACKWARD_SYSTEM_DARWIN)
+
+class SignalHandling {
+public:
+ static std::vector<int> make_default_signals() {
+ const int posix_signals[] = {
+ // Signals for which the default action is "Core".
+ SIGABRT, // Abort signal from abort(3)
+ SIGBUS, // Bus error (bad memory access)
+ SIGFPE, // Floating point exception
+ SIGILL, // Illegal Instruction
+ SIGIOT, // IOT trap. A synonym for SIGABRT
+ SIGQUIT, // Quit from keyboard
+ SIGSEGV, // Invalid memory reference
+ SIGSYS, // Bad argument to routine (SVr4)
+ SIGTRAP, // Trace/breakpoint trap
+ SIGXCPU, // CPU time limit exceeded (4.2BSD)
+ SIGXFSZ, // File size limit exceeded (4.2BSD)
+#if defined(BACKWARD_SYSTEM_DARWIN)
+ SIGEMT, // emulation instruction executed
+#endif
+ };
+ return std::vector<int>(posix_signals,
+ posix_signals +
+ sizeof posix_signals / sizeof posix_signals[0]);
+ }
+
+ SignalHandling(const std::vector<int> &posix_signals = make_default_signals())
+ : _loaded(false) {
+ bool success = true;
+
+ const size_t stack_size = 1024 * 1024 * 8;
+ _stack_content.reset(static_cast<char *>(malloc(stack_size)));
+ if (_stack_content) {
+ stack_t ss;
+ ss.ss_sp = _stack_content.get();
+ ss.ss_size = stack_size;
+ ss.ss_flags = 0;
+ if (sigaltstack(&ss, nullptr) < 0) {
+ success = false;
+ }
+ } else {
+ success = false;
+ }
+
+ for (size_t i = 0; i < posix_signals.size(); ++i) {
+ struct sigaction action;
+ memset(&action, 0, sizeof action);
+ action.sa_flags =
+ static_cast<int>(SA_SIGINFO | SA_ONSTACK | SA_NODEFER | SA_RESETHAND);
+ sigfillset(&action.sa_mask);
+ sigdelset(&action.sa_mask, posix_signals[i]);
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdisabled-macro-expansion"
+#endif
+ action.sa_sigaction = &sig_handler;
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+
+ int r = sigaction(posix_signals[i], &action, nullptr);
+ if (r < 0)
+ success = false;
+ }
+
+ _loaded = success;
+ }
+
+ bool loaded() const { return _loaded; }
+
+ static void handleSignal(int, siginfo_t *info, void *_ctx) {
+ ucontext_t *uctx = static_cast<ucontext_t *>(_ctx);
+
+ StackTrace st;
+ void *error_addr = nullptr;
+#ifdef REG_RIP // x86_64
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_RIP]);
+#elif defined(REG_EIP) // x86_32
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_EIP]);
+#elif defined(__arm__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.arm_pc);
+#elif defined(__aarch64__)
+ #if defined(__APPLE__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext->__ss.__pc);
+ #else
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.pc);
+ #endif
+#elif defined(__loongarch__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.__pc);
+#elif defined(__mips__)
+ error_addr = reinterpret_cast<void *>(
+ reinterpret_cast<struct sigcontext *>(&uctx->uc_mcontext)->sc_pc);
+#elif defined(__ppc__) || defined(__powerpc) || defined(__powerpc__) || \
+ defined(__POWERPC__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.regs->nip);
+#elif defined(__riscv)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.__gregs[REG_PC]);
+#elif defined(__s390x__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.psw.addr);
+#elif defined(__APPLE__) && defined(__x86_64__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext->__ss.__rip);
+#elif defined(__APPLE__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext->__ss.__eip);
+#else
+#warning ":/ sorry, ain't know no nothing none not of your architecture!"
+#endif
+ if (error_addr) {
+ st.load_from(error_addr, 32, reinterpret_cast<void *>(uctx),
+ info->si_addr);
+ } else {
+ st.load_here(32, reinterpret_cast<void *>(uctx), info->si_addr);
+ }
+
+ Printer printer;
+ printer.address = true;
+ printer.print(st, stderr);
+
+#if _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L
+ psiginfo(info, nullptr);
+#else
+ (void)info;
+#endif
+ }
+
+private:
+ details::handle<char *> _stack_content;
+ bool _loaded;
+
+#ifdef __GNUC__
+ __attribute__((noreturn))
+#endif
+ static void
+ sig_handler(int signo, siginfo_t *info, void *_ctx) {
+ handleSignal(signo, info, _ctx);
+
+ // try to forward the signal.
+ raise(info->si_signo);
+
+ // terminate the process immediately.
+ puts("watf? exit");
+ _exit(EXIT_FAILURE);
+ }
+};
+
+#endif // BACKWARD_SYSTEM_LINUX || BACKWARD_SYSTEM_DARWIN
+
+#ifdef BACKWARD_SYSTEM_WINDOWS
+
+class SignalHandling {
+public:
+ SignalHandling(const std::vector<int> & = std::vector<int>())
+ : reporter_thread_([]() {
+ /* We handle crashes in a utility thread:
+ backward structures and some Windows functions called here
+ need stack space, which we do not have when we encounter a
+ stack overflow.
+ To support reporting stack traces during a stack overflow,
+ we create a utility thread at startup, which waits until a
+ crash happens or the program exits normally. */
+
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ cv().wait(lk, [] { return crashed() != crash_status::running; });
+ }
+ if (crashed() == crash_status::crashed) {
+ handle_stacktrace(skip_recs());
+ }
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ crashed() = crash_status::ending;
+ }
+ cv().notify_one();
+ }) {
+ SetUnhandledExceptionFilter(crash_handler);
+
+ signal(SIGABRT, signal_handler);
+ _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+
+ std::set_terminate(&terminator);
+#ifndef BACKWARD_ATLEAST_CXX17
+ std::set_unexpected(&terminator);
+#endif
+ _set_purecall_handler(&terminator);
+ _set_invalid_parameter_handler(&invalid_parameter_handler);
+ }
+ bool loaded() const { return true; }
+
+ ~SignalHandling() {
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ crashed() = crash_status::normal_exit;
+ }
+
+ cv().notify_one();
+
+ reporter_thread_.join();
+ }
+
+private:
+ static CONTEXT *ctx() {
+ static CONTEXT data;
+ return &data;
+ }
+
+ enum class crash_status { running, crashed, normal_exit, ending };
+
+ static crash_status &crashed() {
+ static crash_status data;
+ return data;
+ }
+
+ static std::mutex &mtx() {
+ static std::mutex data;
+ return data;
+ }
+
+ static std::condition_variable &cv() {
+ static std::condition_variable data;
+ return data;
+ }
+
+ static HANDLE &thread_handle() {
+ static HANDLE handle;
+ return handle;
+ }
+
+ std::thread reporter_thread_;
+
+ // TODO: how not to hardcode these?
+ static const constexpr int signal_skip_recs =
+#ifdef __clang__
+ // With clang, RtlCaptureContext also captures the stack frame of the
+ // current function Below that, there ar 3 internal Windows functions
+ 4
+#else
+ // With MSVC cl, RtlCaptureContext misses the stack frame of the current
+ // function The first entries during StackWalk are the 3 internal Windows
+ // functions
+ 3
+#endif
+ ;
+
+ static int &skip_recs() {
+ static int data;
+ return data;
+ }
+
+ static inline void terminator() {
+ crash_handler(signal_skip_recs);
+ abort();
+ }
+
+ static inline void signal_handler(int) {
+ crash_handler(signal_skip_recs);
+ abort();
+ }
+
+ static inline void __cdecl invalid_parameter_handler(const wchar_t *,
+ const wchar_t *,
+ const wchar_t *,
+ unsigned int,
+ uintptr_t) {
+ crash_handler(signal_skip_recs);
+ abort();
+ }
+
+ NOINLINE static LONG WINAPI crash_handler(EXCEPTION_POINTERS *info) {
+ // The exception info supplies a trace from exactly where the issue was,
+ // no need to skip records
+ crash_handler(0, info->ContextRecord);
+ return EXCEPTION_CONTINUE_SEARCH;
+ }
+
+ NOINLINE static void crash_handler(int skip, CONTEXT *ct = nullptr) {
+
+ if (ct == nullptr) {
+ RtlCaptureContext(ctx());
+ } else {
+ memcpy(ctx(), ct, sizeof(CONTEXT));
+ }
+ DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
+ GetCurrentProcess(), &thread_handle(), 0, FALSE,
+ DUPLICATE_SAME_ACCESS);
+
+ skip_recs() = skip;
+
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ crashed() = crash_status::crashed;
+ }
+
+ cv().notify_one();
+
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ cv().wait(lk, [] { return crashed() != crash_status::crashed; });
+ }
+ }
+
+ static void handle_stacktrace(int skip_frames = 0) {
+ // printer creates the TraceResolver, which can supply us a machine type
+ // for stack walking. Without this, StackTrace can only guess using some
+ // macros.
+ // StackTrace also requires that the PDBs are already loaded, which is done
+ // in the constructor of TraceResolver
+ Printer printer;
+
+ StackTrace st;
+ st.set_machine_type(printer.resolver().machine_type());
+ st.set_thread_handle(thread_handle());
+ st.load_here(32 + skip_frames, ctx());
+ st.skip_n_firsts(skip_frames);
+
+ printer.address = true;
+ printer.print(st, std::cerr);
+ }
+};
+
+#endif // BACKWARD_SYSTEM_WINDOWS
+
+#ifdef BACKWARD_SYSTEM_UNKNOWN
+
+class SignalHandling {
+public:
+ SignalHandling(const std::vector<int> & = std::vector<int>()) {}
+ bool init() { return false; }
+ bool loaded() { return false; }
+};
+
+#endif // BACKWARD_SYSTEM_UNKNOWN
+
+} // namespace backward
+
+#endif /* H_GUARD */
diff --git a/contrib/cdb/CMakeLists.txt b/contrib/cdb/CMakeLists.txt
new file mode 100644
index 0000000..6b5b676
--- /dev/null
+++ b/contrib/cdb/CMakeLists.txt
@@ -0,0 +1,8 @@
+# CDB support makefile
+SET(CDBSRC cdb_init.c
+ cdb_find.c
+ cdb_make.c)
+
+ADD_LIBRARY(rspamd-cdb STATIC ${CDBSRC})
+SET_TARGET_PROPERTIES(rspamd-cdb PROPERTIES VERSION ${RSPAMD_VERSION})
+SET_TARGET_PROPERTIES(rspamd-cdb PROPERTIES COMPILE_FLAGS "-DRSPAMD_LIB") \ No newline at end of file
diff --git a/contrib/cdb/cdb.h b/contrib/cdb/cdb.h
new file mode 100644
index 0000000..8774799
--- /dev/null
+++ b/contrib/cdb/cdb.h
@@ -0,0 +1,161 @@
+/* $Id: cdb.h,v 1.10 2009-01-31 17:12:22 mjt Exp $
+ * public cdb include file
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#ifndef TINYCDB_VERSION
+#define TINYCDB_VERSION 0.77
+
+#include "config.h"
+#include "unix-std.h"
+#include "contrib/libev/ev.h"
+
+/*
+ * OpenBSD fix
+ */
+#ifndef EPROTO
+#define EPROTO EPROTONOSUPPORT
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef unsigned int cdbi_t; /* compatibility */
+
+/* common routines */
+unsigned cdb_hash(const void *buf, unsigned len);
+unsigned cdb_unpack(const unsigned char buf[4]);
+void cdb_pack(unsigned num, unsigned char buf[4]);
+
+struct cdb
+{
+ int cdb_fd; /* file descriptor */
+ char *filename; /* file name */
+ time_t mtime; /* mtime of cdb file */
+ struct ev_loop *loop;
+ ev_stat stat_ev; /* event structure for checking cdb for modifications */
+ ev_tstamp check_ts;
+ /* private members */
+ unsigned cdb_fsize; /* datafile size */
+ unsigned cdb_dend; /* end of data ptr */
+ const unsigned char *cdb_mem; /* mmap'ed file memory */
+ unsigned cdb_vpos, cdb_vlen; /* found data */
+ unsigned cdb_kpos, cdb_klen; /* found key */
+};
+
+#define CDB_STATIC_INIT {0,0,0,0,0,0,0,0}
+
+#define cdb_datapos(c) ((c)->cdb_vpos)
+#define cdb_datalen(c) ((c)->cdb_vlen)
+#define cdb_keypos(c) ((c)->cdb_kpos)
+#define cdb_keylen(c) ((c)->cdb_klen)
+#define cdb_fileno(c) ((c)->cdb_fd)
+
+int cdb_init(struct cdb *cdbp, int fd);
+void cdb_add_timer(struct cdb *cdbp, EV_P_ ev_tstamp seconds);
+void cdb_free(struct cdb *cdbp);
+
+int cdb_read(const struct cdb *cdbp, void *buf, unsigned len, unsigned pos);
+#define cdb_readdata(cdbp, buf) \
+ cdb_read((cdbp), (buf), cdb_datalen(cdbp), cdb_datapos(cdbp))
+#define cdb_readkey(cdbp, buf) \
+ cdb_read((cdbp), (buf), cdb_keylen(cdbp), cdb_keypos(cdbp))
+
+const void *cdb_get(const struct cdb *cdbp, unsigned len, unsigned pos);
+#define cdb_getdata(cdbp) \
+ cdb_get((cdbp), cdb_datalen(cdbp), cdb_datapos(cdbp))
+#define cdb_getkey(cdbp) \
+ cdb_get((cdbp), cdb_keylen(cdbp), cdb_keypos(cdbp))
+
+int cdb_find(struct cdb *cdbp, const void *key, unsigned klen);
+
+struct cdb_find
+{
+ struct cdb *cdb_cdbp;
+ unsigned cdb_hval;
+ const unsigned char *cdb_htp, *cdb_htab, *cdb_htend;
+ unsigned cdb_httodo;
+ const void *cdb_key;
+ unsigned cdb_klen;
+};
+
+int cdb_findinit(struct cdb_find *cdbfp, struct cdb *cdbp, const void *key,
+ unsigned klen);
+int cdb_findnext(struct cdb_find *cdbfp);
+
+#define cdb_seqinit(cptr, cdbp) ((*(cptr))=2048)
+int cdb_seqnext(unsigned *cptr, struct cdb *cdbp);
+
+/* old simple interface */
+/* open file using standard routine, then: */
+int cdb_seek(int fd, const void *key, unsigned klen, unsigned *dlenp);
+int cdb_bread(int fd, void *buf, int len);
+
+/* cdb_make */
+
+struct cdb_make
+{
+ int cdb_fd; /* file descriptor */
+
+ /* private */
+ unsigned cdb_dpos; /* data position so far */
+ unsigned cdb_rcnt; /* record count so far */
+ unsigned char cdb_buf[4096]; /* write buffer */
+ unsigned char *cdb_bpos; /* current buf position */
+ struct cdb_rl *cdb_rec[256]; /* list of arrays of record infos */
+};
+
+enum cdb_put_mode
+{
+ CDB_PUT_ADD = 0, /* add unconditionnaly, like cdb_make_add() */
+#define CDB_PUT_ADD CDB_PUT_ADD
+ CDB_FIND = CDB_PUT_ADD, CDB_PUT_REPLACE, /* replace: do not place to index OLD record */
+#define CDB_PUT_REPLACE CDB_PUT_REPLACE
+ CDB_FIND_REMOVE = CDB_PUT_REPLACE, CDB_PUT_INSERT, /* add only if not already exists */
+#define CDB_PUT_INSERT CDB_PUT_INSERT
+ CDB_PUT_WARN, /* add unconditionally but ret. 1 if exists */
+#define CDB_PUT_WARN CDB_PUT_WARN
+ CDB_PUT_REPLACE0, /* if a record exists, fill old one with zeros */
+#define CDB_PUT_REPLACE0 CDB_PUT_REPLACE0
+ CDB_FIND_FILL0 = CDB_PUT_REPLACE0
+};
+
+int cdb_make_start(struct cdb_make *cdbmp, int fd);
+int cdb_make_add(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen);
+int cdb_make_exists(struct cdb_make *cdbmp, const void *key, unsigned klen);
+int cdb_make_find(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ enum cdb_put_mode mode);
+int cdb_make_put(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen, enum cdb_put_mode mode);
+int cdb_make_finish(struct cdb_make *cdbmp);
+
+/** Private API **/
+struct cdb_rec
+{
+ unsigned hval;
+ unsigned rpos;
+};
+
+struct cdb_rl
+{
+ struct cdb_rl *next;
+ unsigned cnt;
+ struct cdb_rec rec[254];
+};
+
+int _cdb_make_write(struct cdb_make *cdbmp, const unsigned char *ptr,
+ unsigned len);
+int _cdb_make_fullwrite(int fd, const unsigned char *buf, unsigned len);
+int _cdb_make_flush(struct cdb_make *cdbmp);
+int _cdb_make_add(struct cdb_make *cdbmp, unsigned hval, const void *key,
+ unsigned klen, const void *val, unsigned vlen);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* include guard */
diff --git a/contrib/cdb/cdb_find.c b/contrib/cdb/cdb_find.c
new file mode 100644
index 0000000..cae0f18
--- /dev/null
+++ b/contrib/cdb/cdb_find.c
@@ -0,0 +1,147 @@
+/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $
+ * cdb_init, cdb_free and cdb_read routines
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#include "cdb.h"
+
+int
+cdb_find(struct cdb *cdbp, const void *key, unsigned klen)
+{
+ const unsigned char *htp; /* hash table pointer */
+ const unsigned char *htab; /* hash table */
+ const unsigned char *htend; /* end of hash table */
+ unsigned httodo; /* ht bytes left to look */
+ unsigned pos, n;
+
+ unsigned hval;
+
+ if (klen >= cdbp->cdb_dend) /* if key size is too large */
+ return 0;
+
+ hval = cdb_hash (key, klen);
+
+ /* find (pos,n) hash table to use */
+ /* first 2048 bytes (toc) are always available */
+ /* (hval % 256) * 8 */
+ htp = cdbp->cdb_mem + ((hval << 3) & 2047); /* index in toc (256x8) */
+ n = cdb_unpack (htp + 4); /* table size */
+ if (!n) /* empty table */
+ return 0; /* not found */
+ httodo = n << 3; /* bytes of htab to lookup */
+ pos = cdb_unpack (htp); /* htab position */
+ if (n > (cdbp->cdb_fsize >> 3) /* overflow of httodo ? */
+ || pos < cdbp->cdb_dend /* is htab inside data section ? */
+ || pos > cdbp->cdb_fsize /* htab start within file ? */
+ || httodo > cdbp->cdb_fsize - pos) /* entrie htab within file ? */
+ return errno = EPROTO, -1;
+
+ htab = cdbp->cdb_mem + pos; /* htab pointer */
+ htend = htab + httodo; /* after end of htab */
+ /* htab starting position: rest of hval modulo htsize, 8bytes per elt */
+ htp = htab + (((hval >> 8) % n) << 3);
+
+ for (;;) {
+ pos = cdb_unpack (htp + 4); /* record position */
+ if (!pos)
+ return 0;
+ if (cdb_unpack (htp) == hval) {
+ if (pos > cdbp->cdb_dend - 8) /* key+val lengths */
+ return errno = EPROTO, -1;
+ if (cdb_unpack (cdbp->cdb_mem + pos) == klen) {
+ if (cdbp->cdb_dend - klen < pos + 8)
+ return errno = EPROTO, -1;
+ if (memcmp (key, cdbp->cdb_mem + pos + 8, klen) == 0) {
+ n = cdb_unpack (cdbp->cdb_mem + pos + 4);
+ pos += 8;
+ if (cdbp->cdb_dend < n || cdbp->cdb_dend - n < pos + klen)
+ return errno = EPROTO, -1;
+ cdbp->cdb_kpos = pos;
+ cdbp->cdb_klen = klen;
+ cdbp->cdb_vpos = pos + klen;
+ cdbp->cdb_vlen = n;
+ return 1;
+ }
+ }
+ }
+ httodo -= 8;
+ if (!httodo)
+ return 0;
+ if ((htp += 8) >= htend)
+ htp = htab;
+ }
+
+}
+
+int
+cdb_findinit(struct cdb_find *cdbfp, struct cdb *cdbp,
+ const void *key, unsigned klen)
+{
+ unsigned n, pos;
+
+ cdbfp->cdb_cdbp = cdbp;
+ cdbfp->cdb_key = key;
+ cdbfp->cdb_klen = klen;
+ cdbfp->cdb_hval = cdb_hash(key, klen);
+
+ cdbfp->cdb_htp = cdbp->cdb_mem + ((cdbfp->cdb_hval << 3) & 2047);
+ n = cdb_unpack(cdbfp->cdb_htp + 4);
+ cdbfp->cdb_httodo = n << 3;
+ if (!n)
+ return 0;
+ pos = cdb_unpack(cdbfp->cdb_htp);
+ if (n > (cdbp->cdb_fsize >> 3)
+ || pos < cdbp->cdb_dend
+ || pos > cdbp->cdb_fsize
+ || cdbfp->cdb_httodo > cdbp->cdb_fsize - pos)
+ return errno = EPROTO, -1;
+
+ cdbfp->cdb_htab = cdbp->cdb_mem + pos;
+ cdbfp->cdb_htend = cdbfp->cdb_htab + cdbfp->cdb_httodo;
+ cdbfp->cdb_htp = cdbfp->cdb_htab + (((cdbfp->cdb_hval >> 8) % n) << 3);
+
+ return 1;
+}
+
+int
+cdb_findnext(struct cdb_find *cdbfp) {
+ struct cdb *cdbp = cdbfp->cdb_cdbp;
+ unsigned pos, n;
+ unsigned klen = cdbfp->cdb_klen;
+
+ while(cdbfp->cdb_httodo) {
+ pos = cdb_unpack(cdbfp->cdb_htp + 4);
+ if (!pos)
+ return 0;
+ n = cdb_unpack(cdbfp->cdb_htp) == cdbfp->cdb_hval;
+ if ((cdbfp->cdb_htp += 8) >= cdbfp->cdb_htend)
+ cdbfp->cdb_htp = cdbfp->cdb_htab;
+ cdbfp->cdb_httodo -= 8;
+ if (n) {
+ if (pos > cdbp->cdb_fsize - 8)
+ return errno = EPROTO, -1;
+ if (cdb_unpack(cdbp->cdb_mem + pos) == klen) {
+ if (cdbp->cdb_fsize - klen < pos + 8)
+ return errno = EPROTO, -1;
+ if (memcmp(cdbfp->cdb_key,
+ cdbp->cdb_mem + pos + 8, klen) == 0) {
+ n = cdb_unpack(cdbp->cdb_mem + pos + 4);
+ pos += 8;
+ if (cdbp->cdb_fsize < n ||
+ cdbp->cdb_fsize - n < pos + klen)
+ return errno = EPROTO, -1;
+ cdbp->cdb_kpos = pos;
+ cdbp->cdb_klen = klen;
+ cdbp->cdb_vpos = pos + klen;
+ cdbp->cdb_vlen = n;
+ return 1;
+ }
+ }
+ }
+ }
+
+ return 0;
+
+}
diff --git a/contrib/cdb/cdb_init.c b/contrib/cdb/cdb_init.c
new file mode 100644
index 0000000..bfc6dd0
--- /dev/null
+++ b/contrib/cdb/cdb_init.c
@@ -0,0 +1,140 @@
+/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $
+ * cdb_init, cdb_free and cdb_read routines
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#include "cdb.h"
+
+unsigned
+cdb_hash(const void *buf, unsigned len)
+{
+ register const unsigned char *p = (const unsigned char *) buf;
+ register const unsigned char *end = p + len;
+ register unsigned hash = 5381; /* start value */
+ while (p < end)
+ hash = (hash + (hash << 5)) ^ *p++;
+ return hash;
+}
+
+int
+cdb_init(struct cdb *cdbp, int fd)
+{
+ struct stat st;
+ unsigned char *mem;
+ unsigned fsize, dend;
+#ifdef _WIN32
+ HANDLE hFile, hMapping;
+#endif
+
+ /* get file size */
+ if (fstat (fd, &st) < 0)
+ return -1;
+ /* trivial sanity check: at least toc should be here */
+ if (st.st_size < 2048)
+ return errno = EPROTO, -1;
+ fsize = (unsigned) (st.st_size & 0xffffffffu);
+ /* memory-map file */
+#ifdef _WIN32
+ hFile = (HANDLE) _get_osfhandle(fd);
+ if (hFile == (HANDLE) -1)
+ return -1;
+ hMapping = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
+ if (!hMapping)
+ return -1;
+ mem = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
+ CloseHandle(hMapping);
+ if (!mem)
+ return -1;
+#else
+ mem = (unsigned char*) mmap (NULL, fsize, PROT_READ, MAP_SHARED, fd, 0);
+ if (mem == MAP_FAILED)
+ return -1;
+#endif /* _WIN32 */
+
+ cdbp->cdb_fd = fd;
+ cdbp->cdb_fsize = fsize;
+ cdbp->cdb_mem = mem;
+ cdbp->mtime = st.st_mtime;
+
+ cdbp->cdb_vpos = cdbp->cdb_vlen = 0;
+ cdbp->cdb_kpos = cdbp->cdb_klen = 0;
+ dend = cdb_unpack (mem);
+ if (dend < 2048)
+ dend = 2048;
+ else if (dend >= fsize)
+ dend = fsize;
+ cdbp->cdb_dend = dend;
+
+ return 0;
+}
+
+void
+cdb_free(struct cdb *cdbp)
+{
+ if (cdbp->cdb_mem) {
+#ifdef _WIN32
+ UnmapViewOfFile((void*) cdbp->cdb_mem);
+#else
+ munmap ((void*) cdbp->cdb_mem, cdbp->cdb_fsize);
+#endif /* _WIN32 */
+ cdbp->cdb_mem = NULL;
+ }
+ cdbp->cdb_fsize = 0;
+
+ if (cdbp->loop) {
+ ev_stat_stop (cdbp->loop, &cdbp->stat_ev);
+ }
+}
+
+const void *
+cdb_get(const struct cdb *cdbp, unsigned len, unsigned pos)
+{
+ if (pos > cdbp->cdb_fsize || cdbp->cdb_fsize - pos < len) {
+ errno = EPROTO;
+ return NULL;
+ }
+ return cdbp->cdb_mem + pos;
+}
+
+int
+cdb_read(const struct cdb *cdbp, void *buf, unsigned len, unsigned pos)
+{
+ const void *data = cdb_get (cdbp, len, pos);
+ if (!data)
+ return -1;
+ memcpy (buf, data, len);
+ return 0;
+}
+
+static void
+cdb_timer_callback (EV_P_ ev_stat *w, int revents)
+{
+ struct cdb *cdbp = w->data;
+ gint nfd;
+
+ /* Check cdb file for modifications */
+ if ((nfd = open (cdbp->filename, O_RDONLY)) != -1) {
+ if (cdbp->cdb_mem) {
+#ifdef _WIN32
+ UnmapViewOfFile((void*) cdbp->cdb_mem);
+#else
+ munmap ((void*) cdbp->cdb_mem, cdbp->cdb_fsize);
+#endif /* _WIN32 */
+ cdbp->cdb_mem = NULL;
+ }
+ (void)close (cdbp->cdb_fd);
+ cdbp->cdb_fsize = 0;
+ (void)cdb_init (cdbp, nfd);
+ }
+}
+
+void
+cdb_add_timer (struct cdb *cdbp, EV_P_ ev_tstamp seconds)
+{
+ cdbp->loop = loop;
+ ev_stat_init (&cdbp->stat_ev, cdb_timer_callback, cdbp->filename, seconds);
+ cdbp->stat_ev.data = cdbp;
+ ev_stat_start (EV_A_ &cdbp->stat_ev);
+}
diff --git a/contrib/cdb/cdb_make.c b/contrib/cdb/cdb_make.c
new file mode 100644
index 0000000..646a7e4
--- /dev/null
+++ b/contrib/cdb/cdb_make.c
@@ -0,0 +1,524 @@
+/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $
+ * cdb_init, cdb_free and cdb_read routines
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#include "cdb.h"
+
+void cdb_pack(unsigned num, unsigned char buf[4])
+{
+ buf[0] = num & 255;
+ num >>= 8;
+ buf[1] = num & 255;
+ num >>= 8;
+ buf[2] = num & 255;
+ buf[3] = num >> 8;
+}
+
+int cdb_make_start(struct cdb_make *cdbmp, int fd)
+{
+ memset (cdbmp, 0, sizeof(*cdbmp));
+ cdbmp->cdb_fd = fd;
+ cdbmp->cdb_dpos = 2048;
+ cdbmp->cdb_bpos = cdbmp->cdb_buf + 2048;
+ return 0;
+}
+
+int
+_cdb_make_fullwrite(int fd, const unsigned char *buf, unsigned len)
+{
+ while (len) {
+ int l = write (fd, buf, len);
+ if (l > 0) {
+ len -= l;
+ buf += l;
+ }
+ else if (l < 0 && errno != EINTR)
+ return -1;
+ }
+ return 0;
+}
+
+int
+_cdb_make_flush(struct cdb_make *cdbmp)
+{
+ unsigned len = cdbmp->cdb_bpos - cdbmp->cdb_buf;
+ if (len) {
+ if (_cdb_make_fullwrite (cdbmp->cdb_fd, cdbmp->cdb_buf, len) < 0)
+ return -1;
+ cdbmp->cdb_bpos = cdbmp->cdb_buf;
+ }
+ return 0;
+}
+
+int
+_cdb_make_write(struct cdb_make *cdbmp, const unsigned char *ptr, unsigned len)
+{
+ unsigned l = sizeof(cdbmp->cdb_buf) - (cdbmp->cdb_bpos - cdbmp->cdb_buf);
+ cdbmp->cdb_dpos += len;
+ if (len > l) {
+ memcpy (cdbmp->cdb_bpos, ptr, l);
+ cdbmp->cdb_bpos += l;
+ if (_cdb_make_flush (cdbmp) < 0)
+ return -1;
+ ptr += l;
+ len -= l;
+ l = len / sizeof(cdbmp->cdb_buf);
+ if (l) {
+ l *= sizeof(cdbmp->cdb_buf);
+ if (_cdb_make_fullwrite (cdbmp->cdb_fd, ptr, l) < 0)
+ return -1;
+ ptr += l;
+ len -= l;
+ }
+ }
+ if (len) {
+ memcpy (cdbmp->cdb_bpos, ptr, len);
+ cdbmp->cdb_bpos += len;
+ }
+ return 0;
+}
+
+static int cdb_make_finish_internal(struct cdb_make *cdbmp)
+{
+ unsigned hcnt[256]; /* hash table counts */
+ unsigned hpos[256]; /* hash table positions */
+ struct cdb_rec *htab;
+ unsigned char *p;
+ struct cdb_rl *rl;
+ unsigned hsize;
+ unsigned t, i;
+
+ if (((0xffffffff - cdbmp->cdb_dpos) >> 3) < cdbmp->cdb_rcnt)
+ return errno = ENOMEM, -1;
+
+ /* count htab sizes and reorder reclists */
+ hsize = 0;
+ for (t = 0; t < 256; ++t) {
+ struct cdb_rl *rlt = NULL;
+ i = 0;
+ rl = cdbmp->cdb_rec[t];
+ while (rl) {
+ struct cdb_rl *rln = rl->next;
+ rl->next = rlt;
+ rlt = rl;
+ i += rl->cnt;
+ rl = rln;
+ }
+ cdbmp->cdb_rec[t] = rlt;
+ if (hsize < (hcnt[t] = i << 1))
+ hsize = hcnt[t];
+ }
+
+ /* allocate memory to hold max htable */
+ htab = (struct cdb_rec*) malloc ((hsize + 2) * sizeof(struct cdb_rec));
+ if (!htab)
+ return errno = ENOENT, -1;
+ p = (unsigned char *) htab;
+ htab += 2;
+
+ /* build hash tables */
+ for (t = 0; t < 256; ++t) {
+ unsigned len, hi;
+ hpos[t] = cdbmp->cdb_dpos;
+ if ((len = hcnt[t]) == 0)
+ continue;
+ for (i = 0; i < len; ++i)
+ htab[i].hval = htab[i].rpos = 0;
+ for (rl = cdbmp->cdb_rec[t]; rl; rl = rl->next)
+ for (i = 0; i < rl->cnt; ++i) {
+ hi = (rl->rec[i].hval >> 8) % len;
+ while (htab[hi].rpos)
+ if (++hi == len)
+ hi = 0;
+ htab[hi] = rl->rec[i];
+ }
+ for (i = 0; i < len; ++i) {
+ cdb_pack (htab[i].hval, p + (i << 3));
+ cdb_pack (htab[i].rpos, p + (i << 3) + 4);
+ }
+ if (_cdb_make_write (cdbmp, p, len << 3) < 0) {
+ free (p);
+ return -1;
+ }
+ }
+ free (p);
+ if (_cdb_make_flush (cdbmp) < 0)
+ return -1;
+ p = cdbmp->cdb_buf;
+ for (t = 0; t < 256; ++t) {
+ cdb_pack (hpos[t], p + (t << 3));
+ cdb_pack (hcnt[t], p + (t << 3) + 4);
+ }
+ if (lseek (cdbmp->cdb_fd, 0, 0) != 0 || _cdb_make_fullwrite (cdbmp->cdb_fd,
+ p, 2048) != 0)
+ return -1;
+
+ return 0;
+}
+
+static void cdb_make_free(struct cdb_make *cdbmp)
+{
+ unsigned t;
+ for (t = 0; t < 256; ++t) {
+ struct cdb_rl *rl = cdbmp->cdb_rec[t];
+ while (rl) {
+ struct cdb_rl *tm = rl;
+ rl = rl->next;
+ free (tm);
+ }
+ }
+}
+
+int cdb_make_finish(struct cdb_make *cdbmp)
+{
+ int r = cdb_make_finish_internal (cdbmp);
+ cdb_make_free (cdbmp);
+ return r;
+}
+
+int
+_cdb_make_add(struct cdb_make *cdbmp, unsigned hval, const void *key,
+ unsigned klen, const void *val, unsigned vlen)
+{
+ unsigned char rlen[8];
+ struct cdb_rl *rl;
+ unsigned i;
+ if (klen > 0xffffffff - (cdbmp->cdb_dpos + 8) || vlen > 0xffffffff
+ - (cdbmp->cdb_dpos + klen + 8))
+ return errno = ENOMEM, -1;
+ i = hval & 255;
+ rl = cdbmp->cdb_rec[i];
+ if (!rl || rl->cnt >= sizeof(rl->rec) / sizeof(rl->rec[0])) {
+ rl = (struct cdb_rl*) malloc (sizeof(struct cdb_rl));
+ if (!rl)
+ return errno = ENOMEM, -1;
+ rl->cnt = 0;
+ rl->next = cdbmp->cdb_rec[i];
+ cdbmp->cdb_rec[i] = rl;
+ }
+ i = rl->cnt++;
+ rl->rec[i].hval = hval;
+ rl->rec[i].rpos = cdbmp->cdb_dpos;
+ ++cdbmp->cdb_rcnt;
+ cdb_pack (klen, rlen);
+ cdb_pack (vlen, rlen + 4);
+ if (_cdb_make_write (cdbmp, rlen, 8) < 0 || _cdb_make_write (cdbmp, key,
+ klen) < 0 || _cdb_make_write (cdbmp, val, vlen) < 0)
+ return -1;
+ return 0;
+}
+
+int cdb_make_add(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen)
+{
+ return _cdb_make_add (cdbmp, cdb_hash (key, klen), key, klen, val, vlen);
+}
+
+static void fixup_rpos(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen)
+{
+ unsigned i;
+ struct cdb_rl *rl;
+ register struct cdb_rec *rp, *rs;
+ for (i = 0; i < 256; ++i) {
+ for (rl = cdbmp->cdb_rec[i]; rl; rl = rl->next)
+ for (rs = rl->rec, rp = rs + rl->cnt; --rp >= rs;)
+ if (rp->rpos <= rpos)
+ goto nexthash;
+ else
+ rp->rpos -= rlen;
+ nexthash: ;
+ }
+}
+
+static int remove_record(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen)
+{
+ unsigned pos, len;
+ int r, fd;
+
+ len = cdbmp->cdb_dpos - rpos - rlen;
+ cdbmp->cdb_dpos -= rlen;
+ if (!len)
+ return 0; /* it was the last record, nothing to do */
+ pos = rpos;
+ fd = cdbmp->cdb_fd;
+ do {
+ r = len > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : len;
+ if (lseek (fd, pos + rlen, SEEK_SET) < 0 || (r = read (fd,
+ cdbmp->cdb_buf, r)) <= 0)
+ return -1;
+ if (lseek (fd, pos, SEEK_SET) < 0 || _cdb_make_fullwrite (fd,
+ cdbmp->cdb_buf, r) < 0)
+ return -1;
+ pos += r;
+ len -= r;
+ } while (len);
+ g_assert (cdbmp->cdb_dpos == pos);
+ fixup_rpos (cdbmp, rpos, rlen);
+ return 0;
+}
+
+static int zerofill_record(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen)
+{
+ if (rpos + rlen == cdbmp->cdb_dpos) {
+ cdbmp->cdb_dpos = rpos;
+ return 0;
+ }
+ if (lseek (cdbmp->cdb_fd, rpos, SEEK_SET) < 0)
+ return -1;
+ memset (cdbmp->cdb_buf, 0, sizeof(cdbmp->cdb_buf));
+ cdb_pack (rlen - 8, cdbmp->cdb_buf + 4);
+ for (;;) {
+ rpos = rlen > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : rlen;
+ if (_cdb_make_fullwrite (cdbmp->cdb_fd, cdbmp->cdb_buf, rpos) < 0)
+ return -1;
+ rlen -= rpos;
+ if (!rlen)
+ return 0;
+ memset (cdbmp->cdb_buf + 4, 0, 4);
+ }
+}
+
+/* return: 0 = not found, 1 = error, or record length */
+static unsigned match(struct cdb_make *cdbmp, unsigned pos, const char *key,
+ unsigned klen)
+{
+ int len;
+ unsigned rlen;
+ if (lseek (cdbmp->cdb_fd, pos, SEEK_SET) < 0)
+ return 1;
+ if (read (cdbmp->cdb_fd, cdbmp->cdb_buf, 8) != 8)
+ return 1;
+ if (cdb_unpack (cdbmp->cdb_buf) != klen)
+ return 0;
+
+ /* record length; check its validity */
+ rlen = cdb_unpack (cdbmp->cdb_buf + 4);
+ if (rlen > cdbmp->cdb_dpos - pos - klen - 8)
+ return errno = EPROTO, 1; /* someone changed our file? */
+ rlen += klen + 8;
+
+ while (klen) {
+ len = klen > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : klen;
+ len = read (cdbmp->cdb_fd, cdbmp->cdb_buf, len);
+ if (len <= 0)
+ return 1;
+ if (memcmp (cdbmp->cdb_buf, key, len) != 0)
+ return 0;
+ key += len;
+ klen -= len;
+ }
+
+ return rlen;
+}
+
+static int findrec(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ unsigned hval, enum cdb_put_mode mode)
+{
+ struct cdb_rl *rl;
+ struct cdb_rec *rp, *rs;
+ unsigned r;
+ int sought = 0;
+ int ret = 0;
+ for (rl = cdbmp->cdb_rec[hval & 255]; rl; rl = rl->next)
+ for (rs = rl->rec, rp = rs + rl->cnt; --rp >= rs;) {
+ if (rp->hval != hval)
+ continue;
+ /*XXX this explicit flush may be unnecessary having
+ * smarter match() that looks into cdb_buf too, but
+ * most of a time here spent in finding hash values
+ * (above), not keys */
+ if (!sought && _cdb_make_flush (cdbmp) < 0)
+ return -1;
+ sought = 1;
+ r = match (cdbmp, rp->rpos, key, klen);
+ if (!r)
+ continue;
+ if (r == 1)
+ return -1;
+ ret = 1;
+ switch (mode)
+ {
+ case CDB_FIND_REMOVE:
+ if (remove_record (cdbmp, rp->rpos, r) < 0)
+ return -1;
+ break;
+ case CDB_FIND_FILL0:
+ if (zerofill_record (cdbmp, rp->rpos, r) < 0)
+ return -1;
+ break;
+ default:
+ goto finish;
+ }
+ memmove (rp, rp + 1, (rs + rl->cnt - 1 - rp) * sizeof(*rp));
+ --rl->cnt;
+ --cdbmp->cdb_rcnt;
+ }
+ finish: if (sought && lseek (cdbmp->cdb_fd, cdbmp->cdb_dpos, SEEK_SET) < 0)
+ return -1;
+ return ret;
+}
+
+int cdb_make_find(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ enum cdb_put_mode mode)
+{
+ return findrec (cdbmp, key, klen, cdb_hash (key, klen), mode);
+}
+
+int cdb_make_exists(struct cdb_make *cdbmp, const void *key, unsigned klen)
+{
+ return cdb_make_find (cdbmp, key, klen, CDB_FIND);
+}
+
+int cdb_make_put(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen, enum cdb_put_mode mode)
+{
+ unsigned hval = cdb_hash (key, klen);
+ int r;
+
+ switch (mode)
+ {
+ case CDB_PUT_REPLACE:
+ case CDB_PUT_INSERT:
+ case CDB_PUT_WARN:
+ case CDB_PUT_REPLACE0:
+ r = findrec (cdbmp, key, klen, hval, mode);
+ if (r < 0)
+ return -1;
+ if (r && mode == CDB_PUT_INSERT)
+ return errno = EEXIST, 1;
+ break;
+
+ case CDB_PUT_ADD:
+ r = 0;
+ break;
+
+ default:
+ return errno = EINVAL, -1;
+ }
+
+ if (_cdb_make_add (cdbmp, hval, key, klen, val, vlen) < 0)
+ return -1;
+
+ return r;
+}
+
+unsigned
+cdb_unpack(const unsigned char buf[4])
+{
+ unsigned n = buf[3];
+ n <<= 8; n |= buf[2];
+ n <<= 8; n |= buf[1];
+ n <<= 8; n |= buf[0];
+ return n;
+}
+
+int
+cdb_seqnext(unsigned *cptr, struct cdb *cdbp) {
+ unsigned klen, vlen;
+ unsigned pos = *cptr;
+ unsigned dend = cdbp->cdb_dend;
+ const unsigned char *mem = cdbp->cdb_mem;
+ if (pos > dend - 8)
+ return 0;
+ klen = cdb_unpack(mem + pos);
+ vlen = cdb_unpack(mem + pos + 4);
+ pos += 8;
+ if (dend - klen < pos || dend - vlen < pos + klen)
+ return errno = EPROTO, -1;
+ cdbp->cdb_kpos = pos;
+ cdbp->cdb_klen = klen;
+ cdbp->cdb_vpos = pos + klen;
+ cdbp->cdb_vlen = vlen;
+ *cptr = pos + klen + vlen;
+ return 1;
+}
+
+/* read a chunk from file, ignoring interrupts (EINTR) */
+
+int
+cdb_bread(int fd, void *buf, int len)
+{
+ int l;
+ while(len > 0) {
+ do l = read(fd, buf, len);
+ while(l < 0 && errno == EINTR);
+ if (l <= 0) {
+ if (!l)
+ errno = EIO;
+ return -1;
+ }
+ buf = (char*)buf + l;
+ len -= l;
+ }
+ return 0;
+}
+
+/* find a given key in cdb file, seek a file pointer to it's value and
+ place data length to *dlenp. */
+
+int
+cdb_seek(int fd, const void *key, unsigned klen, unsigned *dlenp)
+{
+ unsigned htstart; /* hash table start position */
+ unsigned htsize; /* number of elements in a hash table */
+ unsigned httodo; /* hash table elements left to look */
+ unsigned hti; /* hash table index */
+ unsigned pos; /* position in a file */
+ unsigned hval; /* key's hash value */
+ unsigned char rbuf[64]; /* read buffer */
+ int needseek = 1; /* if we should seek to a hash slot */
+
+ hval = cdb_hash(key, klen);
+ pos = (hval & 0xff) << 3; /* position in TOC */
+ /* read the hash table parameters */
+ if (lseek(fd, pos, SEEK_SET) < 0 || cdb_bread(fd, rbuf, 8) < 0)
+ return -1;
+ if ((htsize = cdb_unpack(rbuf + 4)) == 0)
+ return 0;
+ hti = (hval >> 8) % htsize; /* start position in hash table */
+ httodo = htsize;
+ htstart = cdb_unpack(rbuf);
+
+ for(;;) {
+ if (needseek && lseek(fd, htstart + (hti << 3), SEEK_SET) < 0)
+ return -1;
+ if (cdb_bread(fd, rbuf, 8) < 0)
+ return -1;
+ if ((pos = cdb_unpack(rbuf + 4)) == 0) /* not found */
+ return 0;
+
+ if (cdb_unpack(rbuf) != hval) /* hash value not matched */
+ needseek = 0;
+ else { /* hash value matched */
+ if (lseek(fd, pos, SEEK_SET) < 0 || cdb_bread(fd, rbuf, 8) < 0)
+ return -1;
+ if (cdb_unpack(rbuf) == klen) { /* key length matches */
+ /* read the key from file and compare with wanted */
+ unsigned l = klen, c;
+ const char *k = (const char*)key;
+ if (dlenp)
+ *dlenp = cdb_unpack(rbuf + 4); /* save value length */
+ for(;;) {
+ if (!l) /* the whole key read and matches, return */
+ return 1;
+ c = l > sizeof(rbuf) ? sizeof(rbuf) : l;
+ if (cdb_bread(fd, rbuf, c) < 0)
+ return -1;
+ if (memcmp(rbuf, k, c) != 0) /* no, it differs, stop here */
+ break;
+ k += c; l -= c;
+ }
+ }
+ needseek = 1; /* we're looked to other place, should seek back */
+ }
+ if (!--httodo)
+ return 0;
+ if (++hti == htsize) {
+ hti = 0;
+ needseek = 1;
+ }
+ }
+}
diff --git a/contrib/doctest/CMakeLists.txt b/contrib/doctest/CMakeLists.txt
new file mode 100644
index 0000000..4a17708
--- /dev/null
+++ b/contrib/doctest/CMakeLists.txt
@@ -0,0 +1,63 @@
+cmake_minimum_required(VERSION 3.0)
+
+if(POLICY CMP0077)
+ cmake_policy(SET CMP0077 NEW)
+endif()
+
+################################################################################
+## DOCTEST
+################################################################################
+
+project(doctest VERSION 2.4.6 LANGUAGES CXX)
+
+# Determine if doctest is built as a subproject (using add_subdirectory) or if it is the main project.
+set(MAIN_PROJECT OFF)
+if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+ set(MAIN_PROJECT ON)
+endif()
+
+option(DOCTEST_WITH_TESTS "Build tests/examples" ${MAIN_PROJECT})
+option(DOCTEST_WITH_MAIN_IN_STATIC_LIB "Build a static lib (cmake target) with a default main entry point" ON)
+option(DOCTEST_NO_INSTALL "Skip the installation process" OFF)
+option(DOCTEST_USE_STD_HEADERS "Use std headers" OFF)
+
+add_library(${PROJECT_NAME} INTERFACE)
+add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
+
+if(NOT CMAKE_VERSION VERSION_LESS 3.8)
+ target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_11)
+endif()
+
+set(doctest_parts_folder "${CMAKE_CURRENT_SOURCE_DIR}/doctest/parts")
+set(doctest_folder "${CMAKE_CURRENT_SOURCE_DIR}/") # in order to have the mpi extension files, not included into the doctest.h single header
+
+if(MAIN_PROJECT)
+ # use a special hidden version of the header which directly includes the 2 parts - proper reporting of file/line locations during dev
+ target_include_directories(${PROJECT_NAME} INTERFACE
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/scripts/development_only/>
+ $<BUILD_INTERFACE:${doctest_parts_folder}>
+ $<BUILD_INTERFACE:${doctest_folder}>)
+
+ # add a custom target that assembles the single header when any of the parts are touched
+ add_custom_command(
+ OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/doctest/doctest.h
+ DEPENDS
+ ${doctest_parts_folder}/doctest_fwd.h
+ ${doctest_parts_folder}/doctest.cpp
+ COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/assemble_single_header.cmake
+ COMMENT "assembling the single header")
+
+ add_custom_target(assemble_single_header ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/doctest/doctest.h)
+else()
+ target_include_directories(${PROJECT_NAME} INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/>)
+endif()
+
+# hack to support building on XCode 6 and 7 - propagate the definition to everything
+if(DEFINED DOCTEST_THREAD_LOCAL)
+ target_compile_definitions(${PROJECT_NAME} INTERFACE
+ DOCTEST_THREAD_LOCAL=${DOCTEST_THREAD_LOCAL})
+endif()
+
+if(DOCTEST_USE_STD_HEADERS)
+ target_compile_definitions(${PROJECT_NAME} INTERFACE DOCTEST_CONFIG_USE_STD_HEADERS)
+endif()
diff --git a/contrib/doctest/LICENSE.txt b/contrib/doctest/LICENSE.txt
new file mode 100644
index 0000000..d67bb64
--- /dev/null
+++ b/contrib/doctest/LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016-2021 Viktor Kirilov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/doctest/README.md b/contrib/doctest/README.md
new file mode 100644
index 0000000..42148f0
--- /dev/null
+++ b/contrib/doctest/README.md
@@ -0,0 +1,153 @@
+<p align="center"><img src="scripts/data/logo/logo_1.svg"></p>
+<b>
+<table>
+ <tr>
+ <td>
+ master branch
+ </td>
+ <td>
+ Windows <a href="https://ci.appveyor.com/project/onqtam/doctest/branch/master"><img src="https://ci.appveyor.com/api/projects/status/j89qxtahyw1dp4gd/branch/master?svg=true"></a>
+ </td>
+ <td>
+ All <a href="https://github.com/onqtam/doctest/actions?query=branch%3Amaster"><img src="https://github.com/onqtam/doctest/workflows/CI/badge.svg?branch=master"></a>
+ </td>
+ <td>
+ <a href="https://coveralls.io/github/onqtam/doctest?branch=master"><img src="https://coveralls.io/repos/github/onqtam/doctest/badge.svg?branch=master"></a>
+ </td>
+ <!--
+ <td>
+ <a href="https://scan.coverity.com/projects/onqtam-doctest"><img src="https://scan.coverity.com/projects/7865/badge.svg"></a>
+ </td>
+ -->
+ </tr>
+ <tr>
+ <td>
+ dev branch
+ </td>
+ <td>
+ Windows <a href="https://ci.appveyor.com/project/onqtam/doctest/branch/dev"><img src="https://ci.appveyor.com/api/projects/status/j89qxtahyw1dp4gd/branch/dev?svg=true"></a>
+ </td>
+ <td>
+ All <a href="https://github.com/onqtam/doctest/actions?query=branch%3Adev"><img src="https://github.com/onqtam/doctest/workflows/CI/badge.svg?branch=dev"></a>
+ </td>
+ <td>
+ <a href="https://coveralls.io/github/onqtam/doctest?branch=dev"><img src="https://coveralls.io/repos/github/onqtam/doctest/badge.svg?branch=dev"></a>
+ </td>
+ <!--
+ <td>
+ </td>
+ -->
+ </tr>
+</table>
+</b>
+
+**doctest** is a new C++ testing framework but is by far the fastest both in compile times (by [**orders of magnitude**](doc/markdown/benchmarks.md)) and runtime compared to other feature-rich alternatives. It brings the ability of compiled languages such as [**D**](https://dlang.org/spec/unittest.html) / [**Rust**](https://doc.rust-lang.org/book/second-edition/ch11-00-testing.html) / [**Nim**](https://nim-lang.org/docs/unittest.html) to have tests written directly in the production code thanks to a fast, transparent and flexible test runner with a clean interface.
+
+[![Standard](https://img.shields.io/badge/c%2B%2B-11/14/17/20-blue.svg)](https://en.wikipedia.org/wiki/C%2B%2B#Standardization)
+[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Version](https://badge.fury.io/gh/onqtam%2Fdoctest.svg)](https://github.com/onqtam/doctest/releases)
+[![download](https://img.shields.io/badge/download%20%20-link-blue.svg)](https://raw.githubusercontent.com/onqtam/doctest/master/doctest/doctest.h)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/503/badge)](https://bestpractices.coreinfrastructure.org/projects/503)
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/onqtam/doctest.svg)](https://lgtm.com/projects/g/onqtam/doctest/context:cpp)
+[![Join the chat at https://gitter.im/onqtam/doctest](https://badges.gitter.im/onqtam/doctest.svg)](https://gitter.im/onqtam/doctest?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Try it online](https://img.shields.io/badge/try%20it-online-orange.svg)](https://wandbox.org/permlink/nJIibfbivG7BG7r1)
+<!--
+[![Language](https://img.shields.io/badge/language-C++-blue.svg)](https://isocpp.org/)
+[![documentation](https://img.shields.io/badge/documentation%20%20-online-blue.svg)](https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md#reference)
+-->
+
+[<img src="https://cloud.githubusercontent.com/assets/8225057/5990484/70413560-a9ab-11e4-8942-1a63607c0b00.png" align="right">](http://www.patreon.com/onqtam)
+
+The framework is and will stay free but needs your support to sustain its development. There are lots of <a href="doc/markdown/roadmap.md"><b>new features</b></a> and maintenance to do. If you work for a company using **doctest** or have the means to do so, please consider financial support. Monthly donations via Patreon and one-offs via PayPal.
+
+[<img src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif" align="right">](https://www.paypal.me/onqtam/10)
+
+A complete example with a self-registering test that compiles to an executable looks like this:
+
+![cover-example](scripts/data/using_doctest_888px_wide.gif)
+
+There are many C++ testing frameworks - [Catch](https://github.com/catchorg/Catch2), [Boost.Test](http://www.boost.org/doc/libs/1_64_0/libs/test/doc/html/index.html), [UnitTest++](https://github.com/unittest-cpp/unittest-cpp), [cpputest](https://github.com/cpputest/cpputest), [googletest](https://github.com/google/googletest) and many [other](https://en.wikipedia.org/wiki/List_of_unit_testing_frameworks#C.2B.2B).
+
+The **key** differences between it and other testing frameworks are that it is light and unintrusive:
+- Ultra light on compile times both in terms of [**including the header**](doc/markdown/benchmarks.md#cost-of-including-the-header) and writing [**thousands of asserts**](doc/markdown/benchmarks.md#cost-of-an-assertion-macro)
+- Doesn't produce any warnings even on the [**most aggressive**](scripts/cmake/common.cmake#L84) warning levels for **MSVC**/**GCC**/**Clang**
+- Offers a way to remove **everything** testing-related from the binary with the [**```DOCTEST_CONFIG_DISABLE```**](doc/markdown/configuration.md#doctest_config_disable) identifier
+- [**thread-safe**](doc/markdown/faq.md#is-doctest-thread-aware) - asserts (and logging) can be used from multiple threads spawned from a single test case - [**example**](examples/all_features/concurrency.cpp)
+- asserts can be used [**outside of a testing context**](doc/markdown/assertions.md#using-asserts-out-of-a-testing-context) - as a general purpose assert library - [**example**](examples/all_features/asserts_used_outside_of_tests.cpp)
+- Doesn't pollute the global namespace (everything is in namespace ```doctest```) and doesn't drag **any** headers with it
+- Very [**portable**](doc/markdown/features.md#extremely-portable) C++11 (use tag [**1.2.9**](https://github.com/onqtam/doctest/tree/1.2.9) for C++98) with over 180 different CI builds (static analysis, sanitizers...)
+- binaries (exe/dll) can use the test runner of another binary - so tests end up in a single registry - [**example**](examples/executable_dll_and_plugin/)
+
+![cost-of-including-the-framework-header](scripts/data/benchmarks/header.png)
+
+This allows the framework to be used in more ways than any other - tests can be written directly in the production code!
+
+*Tests can be considered a form of documentation and should be able to reside near the production code which they test.*
+
+- This makes the barrier for writing tests **much lower** - you don't have to: **1)** make a separate source file **2)** include a bunch of stuff in it **3)** add it to the build system and **4)** add it to source control - You can just write the tests for a class or a piece of functionality at the bottom of its source file - or even header file!
+- Tests in the production code can be thought of as documentation or up-to-date comments - showing the use of APIs
+- Testing internals that are not exposed through the public API and headers is no longer a mind-bending exercise
+- [**Test-driven development**](https://en.wikipedia.org/wiki/Test-driven_development) in C++ has never been easier!
+
+The framework can be used like any other if you don't want/need to mix production code and tests - check out the [**features**](doc/markdown/features.md).
+
+**doctest** is modeled after [**Catch**](https://github.com/catchorg/Catch2) and some parts of the code have been taken directly - check out [**the differences**](doc/markdown/faq.md#how-is-doctest-different-from-catch).
+
+[This table](https://github.com/martinmoene/catch-lest-other-comparison) compares **doctest** / [**Catch**](https://github.com/catchorg/Catch2) / [**lest**](https://github.com/martinmoene/lest) which are all very similar.
+
+Checkout the [**CppCon 2017 talk**](https://cppcon2017.sched.com/event/BgsI/mix-tests-and-production-code-with-doctest-implementing-and-using-the-fastest-modern-c-testing-framework) on [**YouTube**](https://www.youtube.com/watch?v=eH1CxEC29l8) to get a better understanding of how the framework works and read about how to use it in [**the JetBrains article**](https://blog.jetbrains.com/rscpp/better-ways-testing-with-doctest/) - highlighting the unique aspects of the framework! On a short description on how to use the framework along production code you could refer to [**this GitHub issue**](https://github.com/onqtam/doctest/issues/252). There is also an [**older article**](https://accu.org/var/uploads/journals/Overload137.pdf) in the february edition of ACCU Overload 2017.
+
+[![CppCon 2017 talk about doctest on youtube](scripts/data/youtube-cppcon-talk-thumbnail.png)](https://www.youtube.com/watch?v=eH1CxEC29l8)
+
+Documentation
+-------------
+
+Project:
+
+- [Features and design goals](doc/markdown/features.md) - the complete list of features
+- [Roadmap](doc/markdown/roadmap.md) - upcoming features
+- [Benchmarks](doc/markdown/benchmarks.md) - compile-time and runtime supremacy
+- [Contributing](CONTRIBUTING.md) - how to make a proper pull request
+- [Changelog](CHANGELOG.md) - generated changelog based on closed issues/PRs
+
+Usage:
+
+- [Tutorial](doc/markdown/tutorial.md) - make sure you have read it before the other parts of the documentation
+- [Assertion macros](doc/markdown/assertions.md)
+- [Test cases, subcases and test fixtures](doc/markdown/testcases.md)
+- [Parameterized test cases](doc/markdown/parameterized-tests.md)
+- [Command line](doc/markdown/commandline.md)
+- [Logging macros](doc/markdown/logging.md)
+- [```main()``` entry point](doc/markdown/main.md)
+- [Configuration](doc/markdown/configuration.md)
+- [String conversions](doc/markdown/stringification.md)
+- [Reporters](doc/markdown/reporters.md)
+- [Extensions](doc/markdown/extensions.md)
+- [FAQ](doc/markdown/faq.md)
+- [Build systems](doc/markdown/build-systems.md)
+- [Examples](examples)
+
+Contributing
+------------
+
+[<img src="https://cloud.githubusercontent.com/assets/8225057/5990484/70413560-a9ab-11e4-8942-1a63607c0b00.png" align="right">](http://www.patreon.com/onqtam)
+
+Support the development of the project with donations! There is a list of planned features which are all important and big - see the [**roadmap**](doc/markdown/roadmap.md). I took a break from working in the industry to make open source software so every cent is a big deal.
+
+[<img src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif" align="right">](https://www.paypal.me/onqtam/10)
+
+If you work for a company using **doctest** or have the means to do so, please consider financial support.
+
+Contributions in the form of issues and pull requests are welcome as well - check out the [**Contributing**](CONTRIBUTING.md) page.
+
+Stargazers over time
+------------
+
+[![Stargazers over time](https://starcharts.herokuapp.com/onqtam/doctest.svg)](https://starcharts.herokuapp.com/onqtam/doctest)
+
+Logo
+------------
+
+The [logo](scripts/data/logo) is licensed under a Creative Commons Attribution 4.0 International License. Copyright &copy; 2019 [area55git](https://github.com/area55git) &nbsp; [![License: CC BY 4.0](https://licensebuttons.net/l/by/4.0/80x15.png)](https://creativecommons.org/licenses/by/4.0/)
+
+<p align="center"><img src="scripts/data/logo/icon_2.svg"></p>
diff --git a/contrib/doctest/doctest/doctest.h b/contrib/doctest/doctest/doctest.h
new file mode 100644
index 0000000..42eb039
--- /dev/null
+++ b/contrib/doctest/doctest/doctest.h
@@ -0,0 +1,6580 @@
+// ====================================================================== lgtm [cpp/missing-header-guard]
+// == DO NOT MODIFY THIS FILE BY HAND - IT IS AUTO GENERATED BY CMAKE! ==
+// ======================================================================
+//
+// doctest.h - the lightest feature-rich C++ single-header testing framework for unit tests and TDD
+//
+// Copyright (c) 2016-2021 Viktor Kirilov
+//
+// Distributed under the MIT Software License
+// See accompanying file LICENSE.txt or copy at
+// https://opensource.org/licenses/MIT
+//
+// The documentation can be found at the library's page:
+// https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+//
+// The library is heavily influenced by Catch - https://github.com/catchorg/Catch2
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/catchorg/Catch2/blob/master/LICENSE.txt
+//
+// The concept of subcases (sections in Catch) and expression decomposition are from there.
+// Some parts of the code are taken directly:
+// - stringification - the detection of "ostream& operator<<(ostream&, const T&)" and StringMaker<>
+// - the Approx() helper class for floating point comparison
+// - colors in the console
+// - breaking into a debugger
+// - signal / SEH handling
+// - timer
+// - XmlWriter class - thanks to Phil Nash for allowing the direct reuse (AKA copy/paste)
+//
+// The expression decomposing templates are taken from lest - https://github.com/martinmoene/lest
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/martinmoene/lest/blob/master/LICENSE.txt
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+
+#ifndef DOCTEST_LIBRARY_INCLUDED
+#define DOCTEST_LIBRARY_INCLUDED
+
+// =================================================================================================
+// == VERSION ======================================================================================
+// =================================================================================================
+
+#define DOCTEST_VERSION_MAJOR 2
+#define DOCTEST_VERSION_MINOR 4
+#define DOCTEST_VERSION_PATCH 6
+#define DOCTEST_VERSION_STR "2.4.6"
+
+#define DOCTEST_VERSION \
+ (DOCTEST_VERSION_MAJOR * 10000 + DOCTEST_VERSION_MINOR * 100 + DOCTEST_VERSION_PATCH)
+
+// =================================================================================================
+// == COMPILER VERSION =============================================================================
+// =================================================================================================
+
+// ideas for the version stuff are taken from here: https://github.com/cxxstuff/cxx_detect
+
+#define DOCTEST_COMPILER(MAJOR, MINOR, PATCH) ((MAJOR)*10000000 + (MINOR)*100000 + (PATCH))
+
+// GCC/Clang and GCC/MSVC are mutually exclusive, but Clang/MSVC are not because of clang-cl...
+#if defined(_MSC_VER) && defined(_MSC_FULL_VER)
+#if _MSC_VER == _MSC_FULL_VER / 10000
+#define DOCTEST_MSVC DOCTEST_COMPILER(_MSC_VER / 100, _MSC_VER % 100, _MSC_FULL_VER % 10000)
+#else // MSVC
+#define DOCTEST_MSVC \
+ DOCTEST_COMPILER(_MSC_VER / 100, (_MSC_FULL_VER / 100000) % 100, _MSC_FULL_VER % 100000)
+#endif // MSVC
+#endif // MSVC
+#if defined(__clang__) && defined(__clang_minor__)
+#define DOCTEST_CLANG DOCTEST_COMPILER(__clang_major__, __clang_minor__, __clang_patchlevel__)
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && \
+ !defined(__INTEL_COMPILER)
+#define DOCTEST_GCC DOCTEST_COMPILER(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
+#endif // GCC
+
+#ifndef DOCTEST_MSVC
+#define DOCTEST_MSVC 0
+#endif // DOCTEST_MSVC
+#ifndef DOCTEST_CLANG
+#define DOCTEST_CLANG 0
+#endif // DOCTEST_CLANG
+#ifndef DOCTEST_GCC
+#define DOCTEST_GCC 0
+#endif // DOCTEST_GCC
+
+// =================================================================================================
+// == COMPILER WARNINGS HELPERS ====================================================================
+// =================================================================================================
+
+#if DOCTEST_CLANG
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH _Pragma("clang diagnostic push")
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(clang diagnostic ignored w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP _Pragma("clang diagnostic pop")
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#else // DOCTEST_CLANG
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_CLANG
+
+#if DOCTEST_GCC
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH _Pragma("GCC diagnostic push")
+#define DOCTEST_GCC_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(GCC diagnostic ignored w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP _Pragma("GCC diagnostic pop")
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH DOCTEST_GCC_SUPPRESS_WARNING(w)
+#else // DOCTEST_GCC
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_GCC_SUPPRESS_WARNING(w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_GCC
+
+#if DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH __pragma(warning(push))
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w) __pragma(warning(disable : w))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP __pragma(warning(pop))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#else // DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_MSVC
+
+// =================================================================================================
+// == COMPILER WARNINGS ============================================================================
+// =================================================================================================
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdeprecated")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wctor-dtor-privacy")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-promo")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4623) // default constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtr...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+// 4548 - expression before comma has no effect; expected expression with side - effect
+// 4265 - class has virtual functions, but destructor is not virtual
+// 4986 - exception specification does not match previous declaration
+// 4350 - behavior change: 'member1' called instead of 'member2'
+// 4668 - 'x' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
+// 4365 - conversion from 'int' to 'unsigned long', signed/unsigned mismatch
+// 4774 - format string expected in argument 'x' is not a string literal
+// 4820 - padding in structs
+
+// only 4 should be disabled globally:
+// - 4514 # unreferenced inline function has been removed
+// - 4571 # SEH related
+// - 4710 # function not inlined
+// - 4711 # function 'x' selected for automatic inline expansion
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4548) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4265) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4986) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4350) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4668) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4365) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4774) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4820) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4625) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4626) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5027) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5026) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4623) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5039) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5045) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5105)
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+// =================================================================================================
+// == FEATURE DETECTION ============================================================================
+// =================================================================================================
+
+// general compiler feature support table: https://en.cppreference.com/w/cpp/compiler_support
+// MSVC C++11 feature support table: https://msdn.microsoft.com/en-us/library/hh567368.aspx
+// GCC C++11 feature support table: https://gcc.gnu.org/projects/cxx-status.html
+// MSVC version table:
+// https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering
+// MSVC++ 14.2 (16) _MSC_VER == 1920 (Visual Studio 2019)
+// MSVC++ 14.1 (15) _MSC_VER == 1910 (Visual Studio 2017)
+// MSVC++ 14.0 _MSC_VER == 1900 (Visual Studio 2015)
+// MSVC++ 12.0 _MSC_VER == 1800 (Visual Studio 2013)
+// MSVC++ 11.0 _MSC_VER == 1700 (Visual Studio 2012)
+// MSVC++ 10.0 _MSC_VER == 1600 (Visual Studio 2010)
+// MSVC++ 9.0 _MSC_VER == 1500 (Visual Studio 2008)
+// MSVC++ 8.0 _MSC_VER == 1400 (Visual Studio 2005)
+
+#if DOCTEST_MSVC && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#define DOCTEST_CONFIG_WINDOWS_SEH
+#endif // MSVC
+#if defined(DOCTEST_CONFIG_NO_WINDOWS_SEH) && defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#undef DOCTEST_CONFIG_WINDOWS_SEH
+#endif // DOCTEST_CONFIG_NO_WINDOWS_SEH
+
+#if !defined(_WIN32) && !defined(__QNX__) && !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && \
+ !defined(__EMSCRIPTEN__)
+#define DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // _WIN32
+#if defined(DOCTEST_CONFIG_NO_POSIX_SIGNALS) && defined(DOCTEST_CONFIG_POSIX_SIGNALS)
+#undef DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // DOCTEST_CONFIG_NO_POSIX_SIGNALS
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // no exceptions
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#if defined(DOCTEST_CONFIG_NO_EXCEPTIONS) && !defined(DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS)
+#define DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS && !DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#if defined(DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN) && !defined(DOCTEST_CONFIG_IMPLEMENT)
+#define DOCTEST_CONFIG_IMPLEMENT
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#if DOCTEST_MSVC
+#define DOCTEST_SYMBOL_EXPORT __declspec(dllexport)
+#define DOCTEST_SYMBOL_IMPORT __declspec(dllimport)
+#else // MSVC
+#define DOCTEST_SYMBOL_EXPORT __attribute__((dllexport))
+#define DOCTEST_SYMBOL_IMPORT __attribute__((dllimport))
+#endif // MSVC
+#else // _WIN32
+#define DOCTEST_SYMBOL_EXPORT __attribute__((visibility("default")))
+#define DOCTEST_SYMBOL_IMPORT
+#endif // _WIN32
+
+#ifdef DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#ifdef DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_EXPORT
+#else // DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_IMPORT
+#endif // DOCTEST_CONFIG_IMPLEMENT
+#else // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#define DOCTEST_INTERFACE
+#endif // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+
+#define DOCTEST_EMPTY
+
+#if DOCTEST_MSVC
+#define DOCTEST_NOINLINE __declspec(noinline)
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#elif DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 5, 0)
+#define DOCTEST_NOINLINE
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#else
+#define DOCTEST_NOINLINE __attribute__((noinline))
+#define DOCTEST_UNUSED __attribute__((unused))
+#define DOCTEST_ALIGNMENT(x) __attribute__((aligned(x)))
+#endif
+
+#ifndef DOCTEST_NORETURN
+#define DOCTEST_NORETURN [[noreturn]]
+#endif // DOCTEST_NORETURN
+
+#ifndef DOCTEST_NOEXCEPT
+#define DOCTEST_NOEXCEPT noexcept
+#endif // DOCTEST_NOEXCEPT
+
+// =================================================================================================
+// == FEATURE DETECTION END ========================================================================
+// =================================================================================================
+
+// internal macros for string concatenation and anonymous variable name generation
+#define DOCTEST_CAT_IMPL(s1, s2) s1##s2
+#define DOCTEST_CAT(s1, s2) DOCTEST_CAT_IMPL(s1, s2)
+#ifdef __COUNTER__ // not standard and may be missing for some compilers
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __COUNTER__)
+#else // __COUNTER__
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __LINE__)
+#endif // __COUNTER__
+
+#define DOCTEST_TOSTR(x) #x
+
+#ifndef DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x&
+#else // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x
+#endif // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+
+// not using __APPLE__ because... this is how Catch does it
+#ifdef __MAC_OS_X_VERSION_MIN_REQUIRED
+#define DOCTEST_PLATFORM_MAC
+#elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED)
+#define DOCTEST_PLATFORM_IPHONE
+#elif defined(_WIN32)
+#define DOCTEST_PLATFORM_WINDOWS
+#else // DOCTEST_PLATFORM
+#define DOCTEST_PLATFORM_LINUX
+#endif // DOCTEST_PLATFORM
+
+#define DOCTEST_GLOBAL_NO_WARNINGS(var) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wglobal-constructors") \
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-variable") \
+ static const int var DOCTEST_UNUSED // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp)
+#define DOCTEST_GLOBAL_NO_WARNINGS_END() DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#ifndef DOCTEST_BREAK_INTO_DEBUGGER
+// should probably take a look at https://github.com/scottt/debugbreak
+#ifdef DOCTEST_PLATFORM_LINUX
+#if defined(__GNUC__) && (defined(__i386) || defined(__x86_64))
+// Break at the location of the failing check if possible
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#include <signal.h>
+#define DOCTEST_BREAK_INTO_DEBUGGER() raise(SIGTRAP)
+#endif
+#elif defined(DOCTEST_PLATFORM_MAC)
+#if defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__i386)
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("brk #0"); // NOLINT (hicpp-no-assembler)
+#endif
+#elif DOCTEST_MSVC
+#define DOCTEST_BREAK_INTO_DEBUGGER() __debugbreak()
+#elif defined(__MINGW32__)
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wredundant-decls")
+extern "C" __declspec(dllimport) void __stdcall DebugBreak();
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_BREAK_INTO_DEBUGGER() ::DebugBreak()
+#else // linux
+#define DOCTEST_BREAK_INTO_DEBUGGER() (static_cast<void>(0))
+#endif // linux
+#endif // DOCTEST_BREAK_INTO_DEBUGGER
+
+// this is kept here for backwards compatibility since the config option was changed
+#ifdef DOCTEST_CONFIG_USE_IOSFWD
+#define DOCTEST_CONFIG_USE_STD_HEADERS
+#endif // DOCTEST_CONFIG_USE_IOSFWD
+
+#ifdef DOCTEST_CONFIG_USE_STD_HEADERS
+#ifndef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <iosfwd>
+#include <cstddef>
+#include <ostream>
+#else // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#if DOCTEST_CLANG
+// to detect if libc++ is being used with clang (the _LIBCPP_VERSION identifier)
+#include <ciso646>
+#endif // clang
+
+#ifdef _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN _LIBCPP_BEGIN_NAMESPACE_STD
+#define DOCTEST_STD_NAMESPACE_END _LIBCPP_END_NAMESPACE_STD
+#else // _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN namespace std {
+#define DOCTEST_STD_NAMESPACE_END }
+#endif // _LIBCPP_VERSION
+
+// Forward declaring 'X' in namespace std is not permitted by the C++ Standard.
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4643)
+
+DOCTEST_STD_NAMESPACE_BEGIN // NOLINT (cert-dcl58-cpp)
+typedef decltype(nullptr) nullptr_t;
+template <class charT>
+struct char_traits;
+template <>
+struct char_traits<char>;
+template <class charT, class traits>
+class basic_ostream;
+typedef basic_ostream<char, char_traits<char>> ostream;
+template <class... Types>
+class tuple;
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+template <class _Ty>
+class allocator;
+template <class _Elem, class _Traits, class _Alloc>
+class basic_string;
+using string = basic_string<char, char_traits<char>, allocator<char>>;
+#endif // VS 2019
+DOCTEST_STD_NAMESPACE_END
+
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <type_traits>
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+namespace doctest {
+
+DOCTEST_INTERFACE extern bool is_running_in_test;
+
+// A 24 byte string class (can be as small as 17 for x64 and 13 for x86) that can hold strings with length
+// of up to 23 chars on the stack before going on the heap - the last byte of the buffer is used for:
+// - "is small" bit - the highest bit - if "0" then it is small - otherwise its "1" (128)
+// - if small - capacity left before going on the heap - using the lowest 5 bits
+// - if small - 2 bits are left unused - the second and third highest ones
+// - if small - acts as a null terminator if strlen() is 23 (24 including the null terminator)
+// and the "is small" bit remains "0" ("as well as the capacity left") so its OK
+// Idea taken from this lecture about the string implementation of facebook/folly - fbstring
+// https://www.youtube.com/watch?v=kPR8h4-qZdk
+// TODO:
+// - optimizations - like not deleting memory unnecessarily in operator= and etc.
+// - resize/reserve/clear
+// - substr
+// - replace
+// - back/front
+// - iterator stuff
+// - find & friends
+// - push_back/pop_back
+// - assign/insert/erase
+// - relational operators as free functions - taking const char* as one of the params
+class DOCTEST_INTERFACE String
+{
+ static const unsigned len = 24; //!OCLINT avoid private static members
+ static const unsigned last = len - 1; //!OCLINT avoid private static members
+
+ struct view // len should be more than sizeof(view) - because of the final byte for flags
+ {
+ char* ptr;
+ unsigned size;
+ unsigned capacity;
+ };
+
+ union
+ {
+ char buf[len];
+ view data;
+ };
+
+ bool isOnStack() const { return (buf[last] & 128) == 0; }
+ void setOnHeap();
+ void setLast(unsigned in = last);
+
+ void copy(const String& other);
+
+public:
+ String();
+ ~String();
+
+ // cppcheck-suppress noExplicitConstructor
+ String(const char* in);
+ String(const char* in, unsigned in_size);
+
+ String(const String& other);
+ String& operator=(const String& other);
+
+ String& operator+=(const String& other);
+ String operator+(const String& other) const;
+
+ String(String&& other);
+ String& operator=(String&& other);
+
+ char operator[](unsigned i) const;
+ char& operator[](unsigned i);
+
+ // the only functions I'm willing to leave in the interface - available for inlining
+ const char* c_str() const { return const_cast<String*>(this)->c_str(); } // NOLINT
+ char* c_str() {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf);
+ return data.ptr;
+ }
+
+ unsigned size() const;
+ unsigned capacity() const;
+
+ int compare(const char* other, bool no_case = false) const;
+ int compare(const String& other, bool no_case = false) const;
+};
+
+DOCTEST_INTERFACE bool operator==(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator!=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>=(const String& lhs, const String& rhs);
+
+DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, const String& in);
+
+namespace Color {
+ enum Enum
+ {
+ None = 0,
+ White,
+ Red,
+ Green,
+ Blue,
+ Cyan,
+ Yellow,
+ Grey,
+
+ Bright = 0x10,
+
+ BrightRed = Bright | Red,
+ BrightGreen = Bright | Green,
+ LightGrey = Bright | Grey,
+ BrightWhite = Bright | White
+ };
+
+ DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, Color::Enum code);
+} // namespace Color
+
+namespace assertType {
+ enum Enum
+ {
+ // macro traits
+
+ is_warn = 1,
+ is_check = 2 * is_warn,
+ is_require = 2 * is_check,
+
+ is_normal = 2 * is_require,
+ is_throws = 2 * is_normal,
+ is_throws_as = 2 * is_throws,
+ is_throws_with = 2 * is_throws_as,
+ is_nothrow = 2 * is_throws_with,
+
+ is_false = 2 * is_nothrow,
+ is_unary = 2 * is_false, // not checked anywhere - used just to distinguish the types
+
+ is_eq = 2 * is_unary,
+ is_ne = 2 * is_eq,
+
+ is_lt = 2 * is_ne,
+ is_gt = 2 * is_lt,
+
+ is_ge = 2 * is_gt,
+ is_le = 2 * is_ge,
+
+ // macro types
+
+ DT_WARN = is_normal | is_warn,
+ DT_CHECK = is_normal | is_check,
+ DT_REQUIRE = is_normal | is_require,
+
+ DT_WARN_FALSE = is_normal | is_false | is_warn,
+ DT_CHECK_FALSE = is_normal | is_false | is_check,
+ DT_REQUIRE_FALSE = is_normal | is_false | is_require,
+
+ DT_WARN_THROWS = is_throws | is_warn,
+ DT_CHECK_THROWS = is_throws | is_check,
+ DT_REQUIRE_THROWS = is_throws | is_require,
+
+ DT_WARN_THROWS_AS = is_throws_as | is_warn,
+ DT_CHECK_THROWS_AS = is_throws_as | is_check,
+ DT_REQUIRE_THROWS_AS = is_throws_as | is_require,
+
+ DT_WARN_THROWS_WITH = is_throws_with | is_warn,
+ DT_CHECK_THROWS_WITH = is_throws_with | is_check,
+ DT_REQUIRE_THROWS_WITH = is_throws_with | is_require,
+
+ DT_WARN_THROWS_WITH_AS = is_throws_with | is_throws_as | is_warn,
+ DT_CHECK_THROWS_WITH_AS = is_throws_with | is_throws_as | is_check,
+ DT_REQUIRE_THROWS_WITH_AS = is_throws_with | is_throws_as | is_require,
+
+ DT_WARN_NOTHROW = is_nothrow | is_warn,
+ DT_CHECK_NOTHROW = is_nothrow | is_check,
+ DT_REQUIRE_NOTHROW = is_nothrow | is_require,
+
+ DT_WARN_EQ = is_normal | is_eq | is_warn,
+ DT_CHECK_EQ = is_normal | is_eq | is_check,
+ DT_REQUIRE_EQ = is_normal | is_eq | is_require,
+
+ DT_WARN_NE = is_normal | is_ne | is_warn,
+ DT_CHECK_NE = is_normal | is_ne | is_check,
+ DT_REQUIRE_NE = is_normal | is_ne | is_require,
+
+ DT_WARN_GT = is_normal | is_gt | is_warn,
+ DT_CHECK_GT = is_normal | is_gt | is_check,
+ DT_REQUIRE_GT = is_normal | is_gt | is_require,
+
+ DT_WARN_LT = is_normal | is_lt | is_warn,
+ DT_CHECK_LT = is_normal | is_lt | is_check,
+ DT_REQUIRE_LT = is_normal | is_lt | is_require,
+
+ DT_WARN_GE = is_normal | is_ge | is_warn,
+ DT_CHECK_GE = is_normal | is_ge | is_check,
+ DT_REQUIRE_GE = is_normal | is_ge | is_require,
+
+ DT_WARN_LE = is_normal | is_le | is_warn,
+ DT_CHECK_LE = is_normal | is_le | is_check,
+ DT_REQUIRE_LE = is_normal | is_le | is_require,
+
+ DT_WARN_UNARY = is_normal | is_unary | is_warn,
+ DT_CHECK_UNARY = is_normal | is_unary | is_check,
+ DT_REQUIRE_UNARY = is_normal | is_unary | is_require,
+
+ DT_WARN_UNARY_FALSE = is_normal | is_false | is_unary | is_warn,
+ DT_CHECK_UNARY_FALSE = is_normal | is_false | is_unary | is_check,
+ DT_REQUIRE_UNARY_FALSE = is_normal | is_false | is_unary | is_require,
+ };
+} // namespace assertType
+
+DOCTEST_INTERFACE const char* assertString(assertType::Enum at);
+DOCTEST_INTERFACE const char* failureString(assertType::Enum at);
+DOCTEST_INTERFACE const char* skipPathFromFilename(const char* file);
+
+struct DOCTEST_INTERFACE TestCaseData
+{
+ String m_file; // the file in which the test was registered (using String - see #350)
+ unsigned m_line; // the line where the test was registered
+ const char* m_name; // name of the test case
+ const char* m_test_suite; // the test suite in which the test was added
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+};
+
+struct DOCTEST_INTERFACE AssertData
+{
+ // common - for all asserts
+ const TestCaseData* m_test_case;
+ assertType::Enum m_at;
+ const char* m_file;
+ int m_line;
+ const char* m_expr;
+ bool m_failed;
+
+ // exception-related - for all asserts
+ bool m_threw;
+ String m_exception;
+
+ // for normal asserts
+ String m_decomp;
+
+ // for specific exception-related asserts
+ bool m_threw_as;
+ const char* m_exception_type;
+ const char* m_exception_string;
+};
+
+struct DOCTEST_INTERFACE MessageData
+{
+ String m_string;
+ const char* m_file;
+ int m_line;
+ assertType::Enum m_severity;
+};
+
+struct DOCTEST_INTERFACE SubcaseSignature
+{
+ String m_name;
+ const char* m_file;
+ int m_line;
+
+ bool operator<(const SubcaseSignature& other) const;
+};
+
+struct DOCTEST_INTERFACE IContextScope
+{
+ IContextScope();
+ virtual ~IContextScope();
+ virtual void stringify(std::ostream*) const = 0;
+};
+
+namespace detail {
+ struct DOCTEST_INTERFACE TestCase;
+} // namespace detail
+
+struct ContextOptions //!OCLINT too many fields
+{
+ std::ostream* cout; // stdout stream - std::cout by default
+ std::ostream* cerr; // stderr stream - std::cerr by default
+ String binary_name; // the test binary name
+
+ const detail::TestCase* currentTest = nullptr;
+
+ // == parameters from the command line
+ String out; // output filename
+ String order_by; // how tests should be ordered
+ unsigned rand_seed; // the seed for rand ordering
+
+ unsigned first; // the first (matching) test to be executed
+ unsigned last; // the last (matching) test to be executed
+
+ int abort_after; // stop tests after this many failed assertions
+ int subcase_filter_levels; // apply the subcase filters for the first N levels
+
+ bool success; // include successful assertions in output
+ bool case_sensitive; // if filtering should be case sensitive
+ bool exit; // if the program should be exited after the tests are ran/whatever
+ bool duration; // print the time duration of each test case
+ bool no_throw; // to skip exceptions-related assertion macros
+ bool no_exitcode; // if the framework should return 0 as the exitcode
+ bool no_run; // to not run the tests at all (can be done with an "*" exclude)
+ bool no_version; // to not print the version of the framework
+ bool no_colors; // if output to the console should be colorized
+ bool force_colors; // forces the use of colors even when a tty cannot be detected
+ bool no_breaks; // to not break into the debugger
+ bool no_skip; // don't skip test cases which are marked to be skipped
+ bool gnu_file_line; // if line numbers should be surrounded with :x: and not (x):
+ bool no_path_in_filenames; // if the path to files should be removed from the output
+ bool no_line_numbers; // if source code line numbers should be omitted from the output
+ bool no_debug_output; // no output in the debug console when a debugger is attached
+ bool no_skipped_summary; // don't print "skipped" in the summary !!! UNDOCUMENTED !!!
+ bool no_time_in_output; // omit any time/timestamps from output !!! UNDOCUMENTED !!!
+
+ bool help; // to print the help
+ bool version; // to print the version
+ bool count; // if only the count of matching tests is to be retrieved
+ bool list_test_cases; // to list all tests matching the filters
+ bool list_test_suites; // to list all suites matching the filters
+ bool list_reporters; // lists all registered reporters
+};
+
+namespace detail {
+ template <bool CONDITION, typename TYPE = void>
+ struct enable_if
+ {};
+
+ template <typename TYPE>
+ struct enable_if<true, TYPE>
+ { typedef TYPE type; };
+
+ // clang-format off
+ template<class T> struct remove_reference { typedef T type; };
+ template<class T> struct remove_reference<T&> { typedef T type; };
+ template<class T> struct remove_reference<T&&> { typedef T type; };
+
+ template<typename T, typename U = T&&> U declval(int);
+
+ template<typename T> T declval(long);
+
+ template<typename T> auto declval() DOCTEST_NOEXCEPT -> decltype(declval<T>(0)) ;
+
+ template<class T> struct is_lvalue_reference { const static bool value=false; };
+ template<class T> struct is_lvalue_reference<T&> { const static bool value=true; };
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type& t) DOCTEST_NOEXCEPT
+ {
+ return static_cast<T&&>(t);
+ }
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type&& t) DOCTEST_NOEXCEPT
+ {
+ static_assert(!is_lvalue_reference<T>::value,
+ "Can not forward an rvalue as an lvalue.");
+ return static_cast<T&&>(t);
+ }
+
+ template<class T> struct remove_const { typedef T type; };
+ template<class T> struct remove_const<const T> { typedef T type; };
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template<class T> struct is_enum : public std::is_enum<T> {};
+ template<class T> struct underlying_type : public std::underlying_type<T> {};
+#else
+ // Use compiler intrinsics
+ template<class T> struct is_enum { constexpr static bool value = __is_enum(T); };
+ template<class T> struct underlying_type { typedef __underlying_type(T) type; };
+#endif
+ // clang-format on
+
+ template <typename T>
+ struct deferred_false
+ // cppcheck-suppress unusedStructMember
+ { static const bool value = false; };
+
+ namespace has_insertion_operator_impl {
+ std::ostream &os();
+ template<class T>
+ DOCTEST_REF_WRAP(T) val();
+
+ template<class, class = void>
+ struct check {
+ static constexpr bool value = false;
+ };
+
+ template<class T>
+ struct check<T, decltype(os() << val<T>(), void())> {
+ static constexpr bool value = true;
+ };
+ } // namespace has_insertion_operator_impl
+
+ template<class T>
+ using has_insertion_operator = has_insertion_operator_impl::check<const T>;
+
+ DOCTEST_INTERFACE void my_memcpy(void* dest, const void* src, unsigned num);
+
+ DOCTEST_INTERFACE std::ostream* getTlsOss(); // returns a thread-local ostringstream
+ DOCTEST_INTERFACE String getTlsOssResult();
+
+ template <bool C>
+ struct StringMakerBase
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T)) {
+ return "{?}";
+ }
+ };
+
+ template <>
+ struct StringMakerBase<true>
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T) in) {
+ *getTlsOss() << in;
+ return getTlsOssResult();
+ }
+ };
+
+ DOCTEST_INTERFACE String rawMemoryToString(const void* object, unsigned size);
+
+ template <typename T>
+ String rawMemoryToString(const DOCTEST_REF_WRAP(T) object) {
+ return rawMemoryToString(&object, sizeof(object));
+ }
+
+ template <typename T>
+ const char* type_to_string() {
+ return "<>";
+ }
+} // namespace detail
+
+template <typename T>
+struct StringMaker : public detail::StringMakerBase<detail::has_insertion_operator<T>::value>
+{};
+
+template <typename T>
+struct StringMaker<T*>
+{
+ template <typename U>
+ static String convert(U* p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename R, typename C>
+struct StringMaker<R C::*>
+{
+ static String convert(R C::*p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename T, typename detail::enable_if<!detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ return StringMaker<T>::convert(value);
+}
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(char* in);
+DOCTEST_INTERFACE String toString(const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(bool in);
+DOCTEST_INTERFACE String toString(float in);
+DOCTEST_INTERFACE String toString(double in);
+DOCTEST_INTERFACE String toString(double long in);
+
+DOCTEST_INTERFACE String toString(char in);
+DOCTEST_INTERFACE String toString(char signed in);
+DOCTEST_INTERFACE String toString(char unsigned in);
+DOCTEST_INTERFACE String toString(int short in);
+DOCTEST_INTERFACE String toString(int short unsigned in);
+DOCTEST_INTERFACE String toString(int in);
+DOCTEST_INTERFACE String toString(int unsigned in);
+DOCTEST_INTERFACE String toString(int long in);
+DOCTEST_INTERFACE String toString(int long unsigned in);
+DOCTEST_INTERFACE String toString(int long long in);
+DOCTEST_INTERFACE String toString(int long long unsigned in);
+DOCTEST_INTERFACE String toString(std::nullptr_t in);
+
+template <typename T, typename detail::enable_if<detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ typedef typename detail::underlying_type<T>::type UT;
+ return toString(static_cast<UT>(value));
+}
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+DOCTEST_INTERFACE String toString(const std::string& in);
+#endif // VS 2019
+
+class DOCTEST_INTERFACE Approx
+{
+public:
+ explicit Approx(double value);
+
+ Approx operator()(double value) const;
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ explicit Approx(const T& value,
+ typename detail::enable_if<std::is_constructible<double, T>::value>::type* =
+ static_cast<T*>(nullptr)) {
+ *this = Approx(static_cast<double>(value));
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& epsilon(double newEpsilon);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type epsilon(
+ const T& newEpsilon) {
+ m_epsilon = static_cast<double>(newEpsilon);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& scale(double newScale);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type scale(
+ const T& newScale) {
+ m_scale = static_cast<double>(newScale);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format off
+ DOCTEST_INTERFACE friend bool operator==(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator==(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator!=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator!=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator<=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator<=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator>=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator>=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator< (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator< (const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator> (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator> (const Approx & lhs, double rhs);
+
+ DOCTEST_INTERFACE friend String toString(const Approx& in);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_APPROX_PREFIX \
+ template <typename T> friend typename detail::enable_if<std::is_constructible<double, T>::value, bool>::type
+
+ DOCTEST_APPROX_PREFIX operator==(const T& lhs, const Approx& rhs) { return operator==(double(lhs), rhs); }
+ DOCTEST_APPROX_PREFIX operator==(const Approx& lhs, const T& rhs) { return operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const T& lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const Approx& lhs, const T& rhs) { return !operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator<=(const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator<=(const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) && lhs != rhs; }
+#undef DOCTEST_APPROX_PREFIX
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format on
+
+private:
+ double m_epsilon;
+ double m_scale;
+ double m_value;
+};
+
+DOCTEST_INTERFACE String toString(const Approx& in);
+
+DOCTEST_INTERFACE const ContextOptions* getContextOptions();
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+namespace detail {
+ // clang-format off
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ template<class T> struct decay_array { typedef T type; };
+ template<class T, unsigned N> struct decay_array<T[N]> { typedef T* type; };
+ template<class T> struct decay_array<T[]> { typedef T* type; };
+
+ template<class T> struct not_char_pointer { enum { value = 1 }; };
+ template<> struct not_char_pointer<char*> { enum { value = 0 }; };
+ template<> struct not_char_pointer<const char*> { enum { value = 0 }; };
+
+ template<class T> struct can_use_op : public not_char_pointer<typename decay_array<T>::type> {};
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+ struct DOCTEST_INTERFACE TestFailureException
+ {
+ };
+
+ DOCTEST_INTERFACE bool checkIfShouldThrow(assertType::Enum at);
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_INTERFACE void throwException();
+
+ struct DOCTEST_INTERFACE Subcase
+ {
+ SubcaseSignature m_signature;
+ bool m_entered = false;
+
+ Subcase(const String& name, const char* file, int line);
+ ~Subcase();
+
+ operator bool() const;
+ };
+
+ template <typename L, typename R>
+ String stringifyBinaryExpr(const DOCTEST_REF_WRAP(L) lhs, const char* op,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return toString(lhs) + op + toString(rhs);
+ }
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison")
+#endif
+
+// This will check if there is any way it could find a operator like member or friend and uses it.
+// If not it doesn't find the operator or if the operator at global scope is defined after
+// this template, the template won't be instantiated due to SFINAE. Once the template is not
+// instantiated it can look for global operator using normal conversions.
+#define SFINAE_OP(ret,op) decltype(doctest::detail::declval<L>() op doctest::detail::declval<R>(),static_cast<ret>(0))
+
+#define DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(op, op_str, op_macro) \
+ template <typename R> \
+ DOCTEST_NOINLINE SFINAE_OP(Result,op) operator op(R&& rhs) { \
+ bool res = op_macro(doctest::detail::forward<L>(lhs), doctest::detail::forward<R>(rhs)); \
+ if(m_at & assertType::is_false) \
+ res = !res; \
+ if(!res || doctest::getContextOptions()->success) \
+ return Result(res, stringifyBinaryExpr(lhs, op_str, rhs)); \
+ return Result(res); \
+ }
+
+ // more checks could be added - like in Catch:
+ // https://github.com/catchorg/Catch2/pull/1480/files
+ // https://github.com/catchorg/Catch2/pull/1481/files
+#define DOCTEST_FORBIT_EXPRESSION(rt, op) \
+ template <typename R> \
+ rt& operator op(const R&) { \
+ static_assert(deferred_false<R>::value, \
+ "Expression Too Complex Please Rewrite As Binary Comparison!"); \
+ return *this; \
+ }
+
+ struct DOCTEST_INTERFACE Result
+ {
+ bool m_passed;
+ String m_decomp;
+
+ Result(bool passed, const String& decomposition = String());
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Result, &)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^)
+ DOCTEST_FORBIT_EXPRESSION(Result, |)
+ DOCTEST_FORBIT_EXPRESSION(Result, &&)
+ DOCTEST_FORBIT_EXPRESSION(Result, ||)
+ DOCTEST_FORBIT_EXPRESSION(Result, ==)
+ DOCTEST_FORBIT_EXPRESSION(Result, !=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <)
+ DOCTEST_FORBIT_EXPRESSION(Result, >)
+ DOCTEST_FORBIT_EXPRESSION(Result, <=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >=)
+ DOCTEST_FORBIT_EXPRESSION(Result, =)
+ DOCTEST_FORBIT_EXPRESSION(Result, +=)
+ DOCTEST_FORBIT_EXPRESSION(Result, -=)
+ DOCTEST_FORBIT_EXPRESSION(Result, *=)
+ DOCTEST_FORBIT_EXPRESSION(Result, /=)
+ DOCTEST_FORBIT_EXPRESSION(Result, %=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Result, &=)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Result, |=)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+ // https://stackoverflow.com/questions/39479163 what's the difference between 4018 and 4389
+ DOCTEST_MSVC_SUPPRESS_WARNING(4388) // signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4389) // 'operator' : signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4018) // 'expression' : signed/unsigned mismatch
+ //DOCTEST_MSVC_SUPPRESS_WARNING(4805) // 'operation' : unsafe mix of type 'type' and type 'type' in operation
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ // clang-format off
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE bool
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE typename enable_if<can_use_op<L>::value || can_use_op<R>::value, bool>::type
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ inline bool eq(const char* lhs, const char* rhs) { return String(lhs) == String(rhs); }
+ inline bool ne(const char* lhs, const char* rhs) { return String(lhs) != String(rhs); }
+ inline bool lt(const char* lhs, const char* rhs) { return String(lhs) < String(rhs); }
+ inline bool gt(const char* lhs, const char* rhs) { return String(lhs) > String(rhs); }
+ inline bool le(const char* lhs, const char* rhs) { return String(lhs) <= String(rhs); }
+ inline bool ge(const char* lhs, const char* rhs) { return String(lhs) >= String(rhs); }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+#define DOCTEST_RELATIONAL_OP(name, op) \
+ template <typename L, typename R> \
+ DOCTEST_COMPARISON_RETURN_TYPE name(const DOCTEST_REF_WRAP(L) lhs, \
+ const DOCTEST_REF_WRAP(R) rhs) { \
+ return lhs op rhs; \
+ }
+
+ DOCTEST_RELATIONAL_OP(eq, ==)
+ DOCTEST_RELATIONAL_OP(ne, !=)
+ DOCTEST_RELATIONAL_OP(lt, <)
+ DOCTEST_RELATIONAL_OP(gt, >)
+ DOCTEST_RELATIONAL_OP(le, <=)
+ DOCTEST_RELATIONAL_OP(ge, >=)
+
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) l == r
+#define DOCTEST_CMP_NE(l, r) l != r
+#define DOCTEST_CMP_GT(l, r) l > r
+#define DOCTEST_CMP_LT(l, r) l < r
+#define DOCTEST_CMP_GE(l, r) l >= r
+#define DOCTEST_CMP_LE(l, r) l <= r
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) eq(l, r)
+#define DOCTEST_CMP_NE(l, r) ne(l, r)
+#define DOCTEST_CMP_GT(l, r) gt(l, r)
+#define DOCTEST_CMP_LT(l, r) lt(l, r)
+#define DOCTEST_CMP_GE(l, r) ge(l, r)
+#define DOCTEST_CMP_LE(l, r) le(l, r)
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+
+ template <typename L>
+ // cppcheck-suppress copyCtorAndEqOperator
+ struct Expression_lhs
+ {
+ L lhs;
+ assertType::Enum m_at;
+
+ explicit Expression_lhs(L&& in, assertType::Enum at)
+ : lhs(doctest::detail::forward<L>(in))
+ , m_at(at) {}
+
+ DOCTEST_NOINLINE operator Result() {
+// this is needed only foc MSVC 2015:
+// https://ci.appveyor.com/project/onqtam/doctest/builds/38181202
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4800) // 'int': forcing value to bool
+ bool res = static_cast<bool>(lhs);
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ res = !res;
+
+ if(!res || getContextOptions()->success)
+ return Result(res, toString(lhs));
+ return Result(res);
+ }
+
+ /* This is required for user-defined conversions from Expression_lhs to L */
+ //operator L() const { return lhs; }
+ operator L() const { return lhs; }
+
+ // clang-format off
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(==, " == ", DOCTEST_CMP_EQ) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(!=, " != ", DOCTEST_CMP_NE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>, " > ", DOCTEST_CMP_GT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<, " < ", DOCTEST_CMP_LT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>=, " >= ", DOCTEST_CMP_GE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<=, " <= ", DOCTEST_CMP_LE) //!OCLINT bitwise operator in conditional
+ // clang-format on
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &&)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ||)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, =)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, +=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, -=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, *=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, /=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, %=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |=)
+ // these 2 are unfortunate because they should be allowed - they have higher precedence over the comparisons, but the
+ // ExpressionDecomposer class uses the left shift operator to capture the left operand of the binary expression...
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#endif
+
+ struct DOCTEST_INTERFACE ExpressionDecomposer
+ {
+ assertType::Enum m_at;
+
+ ExpressionDecomposer(assertType::Enum at);
+
+ // The right operator for capturing expressions is "<=" instead of "<<" (based on the operator precedence table)
+ // but then there will be warnings from GCC about "-Wparentheses" and since "_Pragma()" is problematic this will stay for now...
+ // https://github.com/catchorg/Catch2/issues/870
+ // https://github.com/catchorg/Catch2/issues/565
+ template <typename L>
+ Expression_lhs<L> operator<<(L &&operand) {
+ return Expression_lhs<L>(doctest::detail::forward<L>(operand), m_at);
+ }
+ };
+
+ struct DOCTEST_INTERFACE TestSuite
+ {
+ const char* m_test_suite;
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+
+ TestSuite& operator*(const char* in);
+
+ template <typename T>
+ TestSuite& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+ };
+
+ typedef void (*funcType)();
+
+ struct DOCTEST_INTERFACE TestCase : public TestCaseData
+ {
+ funcType m_test; // a function pointer to the test case
+
+ const char* m_type; // for templated test cases - gets appended to the real name
+ int m_template_id; // an ID used to distinguish between the different versions of a templated test case
+ String m_full_name; // contains the name (only for templated test cases!) + the template type
+
+ TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type = "", int template_id = -1);
+
+ TestCase(const TestCase& other);
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ TestCase& operator=(const TestCase& other);
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& operator*(const char* in);
+
+ template <typename T>
+ TestCase& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+
+ bool operator<(const TestCase& other) const;
+ };
+
+ // forward declarations of functions used by the macros
+ DOCTEST_INTERFACE int regTest(const TestCase& tc);
+ DOCTEST_INTERFACE int setTestSuite(const TestSuite& ts);
+ DOCTEST_INTERFACE bool isDebuggerActive();
+
+ template<typename T>
+ int instantiationHelper(const T&) { return 0; }
+
+ namespace binaryAssertComparison {
+ enum Enum
+ {
+ eq = 0,
+ ne,
+ gt,
+ lt,
+ ge,
+ le
+ };
+ } // namespace binaryAssertComparison
+
+ // clang-format off
+ template <int, class L, class R> struct RelationalComparator { bool operator()(const DOCTEST_REF_WRAP(L), const DOCTEST_REF_WRAP(R) ) const { return false; } };
+
+#define DOCTEST_BINARY_RELATIONAL_OP(n, op) \
+ template <class L, class R> struct RelationalComparator<n, L, R> { bool operator()(const DOCTEST_REF_WRAP(L) lhs, const DOCTEST_REF_WRAP(R) rhs) const { return op(lhs, rhs); } };
+ // clang-format on
+
+ DOCTEST_BINARY_RELATIONAL_OP(0, doctest::detail::eq)
+ DOCTEST_BINARY_RELATIONAL_OP(1, doctest::detail::ne)
+ DOCTEST_BINARY_RELATIONAL_OP(2, doctest::detail::gt)
+ DOCTEST_BINARY_RELATIONAL_OP(3, doctest::detail::lt)
+ DOCTEST_BINARY_RELATIONAL_OP(4, doctest::detail::ge)
+ DOCTEST_BINARY_RELATIONAL_OP(5, doctest::detail::le)
+
+ struct DOCTEST_INTERFACE ResultBuilder : public AssertData
+ {
+ ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type = "", const char* exception_string = "");
+
+ void setResult(const Result& res);
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ m_failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+ if(m_failed || getContextOptions()->success)
+ m_decomp = stringifyBinaryExpr(lhs, ", ", rhs);
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(const DOCTEST_REF_WRAP(L) val) {
+ m_failed = !val;
+
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ m_failed = !m_failed;
+
+ if(m_failed || getContextOptions()->success)
+ m_decomp = toString(val);
+ }
+
+ void translateException();
+
+ bool log();
+ void react() const;
+ };
+
+ namespace assertAction {
+ enum Enum
+ {
+ nothing = 0,
+ dbgbreak = 1,
+ shouldthrow = 2
+ };
+ } // namespace assertAction
+
+ DOCTEST_INTERFACE void failed_out_of_a_testing_context(const AssertData& ad);
+
+ DOCTEST_INTERFACE void decomp_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, Result result);
+
+#define DOCTEST_ASSERT_OUT_OF_TESTS(decomp) \
+ do { \
+ if(!is_running_in_test) { \
+ if(failed) { \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ rb.m_decomp = decomp; \
+ failed_out_of_a_testing_context(rb); \
+ if(isDebuggerActive() && !getContextOptions()->no_breaks) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(checkIfShouldThrow(at)) \
+ throwException(); \
+ } \
+ return; \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_IN_TESTS(decomp) \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ if(rb.m_failed || getContextOptions()->success) \
+ rb.m_decomp = decomp; \
+ if(rb.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(rb.m_failed && checkIfShouldThrow(at)) \
+ throwException()
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ bool failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ DOCTEST_ASSERT_IN_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) val) {
+ bool failed = !val;
+
+ if(at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ failed = !failed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(toString(val));
+ DOCTEST_ASSERT_IN_TESTS(toString(val));
+ }
+
+ struct DOCTEST_INTERFACE IExceptionTranslator
+ {
+ IExceptionTranslator();
+ virtual ~IExceptionTranslator();
+ virtual bool translate(String&) const = 0;
+ };
+
+ template <typename T>
+ class ExceptionTranslator : public IExceptionTranslator //!OCLINT destructor of virtual class
+ {
+ public:
+ explicit ExceptionTranslator(String (*translateFunction)(T))
+ : m_translateFunction(translateFunction) {}
+
+ bool translate(String& res) const override {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+ throw; // lgtm [cpp/rethrow-no-exception]
+ // cppcheck-suppress catchExceptionByValue
+ } catch(T ex) { // NOLINT
+ res = m_translateFunction(ex); //!OCLINT parameter reassignment
+ return true;
+ } catch(...) {} //!OCLINT - empty catch statement
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ static_cast<void>(res); // to silence -Wunused-parameter
+ return false;
+ }
+
+ private:
+ String (*m_translateFunction)(T);
+ };
+
+ DOCTEST_INTERFACE void registerExceptionTranslatorImpl(const IExceptionTranslator* et);
+
+ template <bool C>
+ struct StringStreamBase
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << toString(in);
+ }
+
+ // always treat char* as a string in this context - no matter
+ // if DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING is defined
+ static void convert(std::ostream* s, const char* in) { *s << String(in); }
+ };
+
+ template <>
+ struct StringStreamBase<true>
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << in;
+ }
+ };
+
+ template <typename T>
+ struct StringStream : public StringStreamBase<has_insertion_operator<T>::value>
+ {};
+
+ template <typename T>
+ void toStream(std::ostream* s, const T& value) {
+ StringStream<T>::convert(s, value);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char* in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, bool in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, float in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double long in);
+
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char signed in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long unsigned in);
+
+ // ContextScope base class used to allow implementing methods of ContextScope
+ // that don't depend on the template parameter in doctest.cpp.
+ class DOCTEST_INTERFACE ContextScopeBase : public IContextScope {
+ protected:
+ ContextScopeBase();
+
+ void destroy();
+ };
+
+ template <typename L> class ContextScope : public ContextScopeBase
+ {
+ const L lambda_;
+
+ public:
+ explicit ContextScope(const L &lambda) : lambda_(lambda) {}
+
+ ContextScope(ContextScope &&other) : lambda_(other.lambda_) {}
+
+ void stringify(std::ostream* s) const override { lambda_(s); }
+
+ ~ContextScope() override { destroy(); }
+ };
+
+ struct DOCTEST_INTERFACE MessageBuilder : public MessageData
+ {
+ std::ostream* m_stream;
+
+ MessageBuilder(const char* file, int line, assertType::Enum severity);
+ MessageBuilder() = delete;
+ ~MessageBuilder();
+
+ // the preferred way of chaining parameters for stringification
+ template <typename T>
+ MessageBuilder& operator,(const T& in) {
+ toStream(m_stream, in);
+ return *this;
+ }
+
+ // kept here just for backwards-compatibility - the comma operator should be preferred now
+ template <typename T>
+ MessageBuilder& operator<<(const T& in) { return this->operator,(in); }
+
+ // the `,` operator has the lowest operator precedence - if `<<` is used by the user then
+ // the `,` operator will be called last which is not what we want and thus the `*` operator
+ // is used first (has higher operator precedence compared to `<<`) so that we guarantee that
+ // an operator of the MessageBuilder class is called first before the rest of the parameters
+ template <typename T>
+ MessageBuilder& operator*(const T& in) { return this->operator,(in); }
+
+ bool log();
+ void react();
+ };
+
+ template <typename L>
+ ContextScope<L> MakeContextScope(const L &lambda) {
+ return ContextScope<L>(lambda);
+ }
+} // namespace detail
+
+#define DOCTEST_DEFINE_DECORATOR(name, type, def) \
+ struct name \
+ { \
+ type data; \
+ name(type in = def) \
+ : data(in) {} \
+ void fill(detail::TestCase& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ void fill(detail::TestSuite& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ }
+
+DOCTEST_DEFINE_DECORATOR(test_suite, const char*, "");
+DOCTEST_DEFINE_DECORATOR(description, const char*, "");
+DOCTEST_DEFINE_DECORATOR(skip, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_breaks, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_output, bool, true);
+DOCTEST_DEFINE_DECORATOR(timeout, double, 0);
+DOCTEST_DEFINE_DECORATOR(may_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(should_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(expected_failures, int, 0);
+
+template <typename T>
+int registerExceptionTranslator(String (*translateFunction)(T)) {
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors")
+ static detail::ExceptionTranslator<T> exceptionTranslator(translateFunction);
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ detail::registerExceptionTranslatorImpl(&exceptionTranslator);
+ return 0;
+}
+
+} // namespace doctest
+
+// in a separate namespace outside of doctest because the DOCTEST_TEST_SUITE macro
+// introduces an anonymous namespace in which getCurrentTestSuite gets overridden
+namespace doctest_detail_test_suite_ns {
+DOCTEST_INTERFACE doctest::detail::TestSuite& getCurrentTestSuite();
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+#else // DOCTEST_CONFIG_DISABLE
+template <typename T>
+int registerExceptionTranslator(String (*)(T)) {
+ return 0;
+}
+#endif // DOCTEST_CONFIG_DISABLE
+
+namespace detail {
+ typedef void (*assert_handler)(const AssertData&);
+ struct ContextState;
+} // namespace detail
+
+class DOCTEST_INTERFACE Context
+{
+ detail::ContextState* p;
+
+ void parseArgs(int argc, const char* const* argv, bool withDefaults = false);
+
+public:
+ explicit Context(int argc = 0, const char* const* argv = nullptr);
+
+ ~Context();
+
+ void applyCommandLine(int argc, const char* const* argv);
+
+ void addFilter(const char* filter, const char* value);
+ void clearFilters();
+ void setOption(const char* option, int value);
+ void setOption(const char* option, const char* value);
+
+ bool shouldExit();
+
+ void setAsDefaultForAssertsOutOfTestCases();
+
+ void setAssertHandler(detail::assert_handler ah);
+
+ int run();
+};
+
+namespace TestCaseFailureReason {
+ enum Enum
+ {
+ None = 0,
+ AssertFailure = 1, // an assertion has failed in the test case
+ Exception = 2, // test case threw an exception
+ Crash = 4, // a crash...
+ TooManyFailedAsserts = 8, // the abort-after option
+ Timeout = 16, // see the timeout decorator
+ ShouldHaveFailedButDidnt = 32, // see the should_fail decorator
+ ShouldHaveFailedAndDid = 64, // see the should_fail decorator
+ DidntFailExactlyNumTimes = 128, // see the expected_failures decorator
+ FailedExactlyNumTimes = 256, // see the expected_failures decorator
+ CouldHaveFailedAndDid = 512 // see the may_fail decorator
+ };
+} // namespace TestCaseFailureReason
+
+struct DOCTEST_INTERFACE CurrentTestCaseStats
+{
+ int numAssertsCurrentTest;
+ int numAssertsFailedCurrentTest;
+ double seconds;
+ int failure_flags; // use TestCaseFailureReason::Enum
+};
+
+struct DOCTEST_INTERFACE TestCaseException
+{
+ String error_string;
+ bool is_crash;
+};
+
+struct DOCTEST_INTERFACE TestRunStats
+{
+ unsigned numTestCases;
+ unsigned numTestCasesPassingFilters;
+ unsigned numTestSuitesPassingFilters;
+ unsigned numTestCasesFailed;
+ int numAsserts;
+ int numAssertsFailed;
+};
+
+struct QueryData
+{
+ const TestRunStats* run_stats = nullptr;
+ const TestCaseData** data = nullptr;
+ unsigned num_data = 0;
+};
+
+struct DOCTEST_INTERFACE IReporter
+{
+ // The constructor has to accept "const ContextOptions&" as a single argument
+ // which has most of the options for the run + a pointer to the stdout stream
+ // Reporter(const ContextOptions& in)
+
+ // called when a query should be reported (listing test cases, printing the version, etc.)
+ virtual void report_query(const QueryData&) = 0;
+
+ // called when the whole test run starts
+ virtual void test_run_start() = 0;
+ // called when the whole test run ends (caching a pointer to the input doesn't make sense here)
+ virtual void test_run_end(const TestRunStats&) = 0;
+
+ // called when a test case is started (safe to cache a pointer to the input)
+ virtual void test_case_start(const TestCaseData&) = 0;
+ // called when a test case is reentered because of unfinished subcases (safe to cache a pointer to the input)
+ virtual void test_case_reenter(const TestCaseData&) = 0;
+ // called when a test case has ended
+ virtual void test_case_end(const CurrentTestCaseStats&) = 0;
+
+ // called when an exception is thrown from the test case (or it crashes)
+ virtual void test_case_exception(const TestCaseException&) = 0;
+
+ // called whenever a subcase is entered (don't cache pointers to the input)
+ virtual void subcase_start(const SubcaseSignature&) = 0;
+ // called whenever a subcase is exited (don't cache pointers to the input)
+ virtual void subcase_end() = 0;
+
+ // called for each assert (don't cache pointers to the input)
+ virtual void log_assert(const AssertData&) = 0;
+ // called for each message (don't cache pointers to the input)
+ virtual void log_message(const MessageData&) = 0;
+
+ // called when a test case is skipped either because it doesn't pass the filters, has a skip decorator
+ // or isn't in the execution range (between first and last) (safe to cache a pointer to the input)
+ virtual void test_case_skipped(const TestCaseData&) = 0;
+
+ // doctest will not be managing the lifetimes of reporters given to it but this would still be nice to have
+ virtual ~IReporter();
+
+ // can obtain all currently active contexts and stringify them if one wishes to do so
+ static int get_num_active_contexts();
+ static const IContextScope* const* get_active_contexts();
+
+ // can iterate through contexts which have been stringified automatically in their destructors when an exception has been thrown
+ static int get_num_stringified_contexts();
+ static const String* get_stringified_contexts();
+};
+
+namespace detail {
+ typedef IReporter* (*reporterCreatorFunc)(const ContextOptions&);
+
+ DOCTEST_INTERFACE void registerReporterImpl(const char* name, int prio, reporterCreatorFunc c, bool isReporter);
+
+ template <typename Reporter>
+ IReporter* reporterCreator(const ContextOptions& o) {
+ return new Reporter(o);
+ }
+} // namespace detail
+
+template <typename Reporter>
+int registerReporter(const char* name, int priority, bool isReporter) {
+ detail::registerReporterImpl(name, priority, detail::reporterCreator<Reporter>, isReporter);
+ return 0;
+}
+} // namespace doctest
+
+// if registering is not disabled
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// common code in asserts - for convenience
+#define DOCTEST_ASSERT_LOG_AND_REACT(b) \
+ if(b.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ b.react()
+
+#ifdef DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) x;
+#else // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) \
+ try { \
+ x; \
+ } catch(...) { _DOCTEST_RB.translateException(); }
+#endif // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#ifdef DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wuseless-cast") \
+ static_cast<void>(__VA_ARGS__); \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+#else // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) __VA_ARGS__;
+#endif // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+
+// registers the test by initializing a dummy var with a function
+#define DOCTEST_REGISTER_FUNCTION(global_prefix, f, decorators) \
+ global_prefix DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::regTest( \
+ doctest::detail::TestCase( \
+ f, __FILE__, __LINE__, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite()) * \
+ decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, decorators) \
+ namespace { \
+ struct der : public base \
+ { \
+ void f(); \
+ }; \
+ static void func() { \
+ der v; \
+ v.f(); \
+ } \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, func, decorators) \
+ } \
+ inline DOCTEST_NOINLINE void der::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, decorators) \
+ static void f(); \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, f, decorators) \
+ static void f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(f, proxy, decorators) \
+ static doctest::detail::funcType proxy() { return f; } \
+ DOCTEST_REGISTER_FUNCTION(inline const, proxy(), decorators) \
+ static void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for registering tests in classes - requires C++17 for inline variables!
+#if __cplusplus >= 201703L || (DOCTEST_MSVC >= DOCTEST_COMPILER(19, 12, 0) && _MSVC_LANG >= 201703L)
+#define DOCTEST_TEST_CASE_CLASS(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_PROXY_), \
+ decorators)
+#else // DOCTEST_TEST_CASE_CLASS
+#define DOCTEST_TEST_CASE_CLASS(...) \
+ TEST_CASES_CAN_BE_REGISTERED_IN_CLASSES_ONLY_IN_CPP17_MODE_OR_WITH_VS_2017_OR_NEWER
+#endif // DOCTEST_TEST_CASE_CLASS
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(c, decorators) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), c, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING_IMPL(...) \
+ template <> \
+ inline const char* type_to_string<__VA_ARGS__>() { \
+ return "<" #__VA_ARGS__ ">"; \
+ }
+#define DOCTEST_TYPE_TO_STRING(...) \
+ namespace doctest { namespace detail { \
+ DOCTEST_TYPE_TO_STRING_IMPL(__VA_ARGS__) \
+ } \
+ } \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, iter, func) \
+ template <typename T> \
+ static void func(); \
+ namespace { \
+ template <typename Tuple> \
+ struct iter; \
+ template <typename Type, typename... Rest> \
+ struct iter<std::tuple<Type, Rest...>> \
+ { \
+ iter(const char* file, unsigned line, int index) { \
+ doctest::detail::regTest(doctest::detail::TestCase(func<Type>, file, line, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite(), \
+ doctest::detail::type_to_string<Type>(), \
+ int(line) * 1000 + index) \
+ * dec); \
+ iter<std::tuple<Rest...>>(file, line, index + 1); \
+ } \
+ }; \
+ template <> \
+ struct iter<std::tuple<>> \
+ { \
+ iter(const char*, unsigned, int) {} \
+ }; \
+ } \
+ template <typename T> \
+ static void func()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(dec, T, id) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(id, ITERATOR), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_))
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, anon, ...) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_CAT(anon, DUMMY)) = \
+ doctest::detail::instantiationHelper(DOCTEST_CAT(id, ITERATOR)<__VA_ARGS__>(__FILE__, __LINE__, 0));\
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), std::tuple<__VA_ARGS__>) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, anon, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(anon, ITERATOR), anon); \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(anon, anon, std::tuple<__VA_ARGS__>) \
+ template <typename T> \
+ static void anon()
+
+#define DOCTEST_TEST_CASE_TEMPLATE(dec, T, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__)
+
+// for subcases
+#define DOCTEST_SUBCASE(name) \
+ if(const doctest::detail::Subcase & DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUBCASE_) DOCTEST_UNUSED = \
+ doctest::detail::Subcase(name, __FILE__, __LINE__))
+
+// for grouping tests in test suites by using code blocks
+#define DOCTEST_TEST_SUITE_IMPL(decorators, ns_name) \
+ namespace ns_name { namespace doctest_detail_test_suite_ns { \
+ static DOCTEST_NOINLINE doctest::detail::TestSuite& getCurrentTestSuite() { \
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4640) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors") \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmissing-field-initializers") \
+ static doctest::detail::TestSuite data{}; \
+ static bool inited = false; \
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP \
+ if(!inited) { \
+ data* decorators; \
+ inited = true; \
+ } \
+ return data; \
+ } \
+ } \
+ } \
+ namespace ns_name
+
+#define DOCTEST_TEST_SUITE(decorators) \
+ DOCTEST_TEST_SUITE_IMPL(decorators, DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUITE_))
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(decorators) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * ""); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering exception translators
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(translatorName, signature) \
+ inline doctest::String translatorName(signature); \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)) = \
+ doctest::registerExceptionTranslator(translatorName); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ doctest::String translatorName(signature)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_), \
+ signature)
+
+// for registering reporters
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, true); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering listeners
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, false); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for logging
+#define DOCTEST_INFO(...) \
+ DOCTEST_INFO_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), \
+ __VA_ARGS__)
+
+#define DOCTEST_INFO_IMPL(mb_name, s_name, ...) \
+ auto DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_) = doctest::detail::MakeContextScope( \
+ [&](std::ostream* s_name) { \
+ doctest::detail::MessageBuilder mb_name(__FILE__, __LINE__, doctest::assertType::is_warn); \
+ mb_name.m_stream = s_name; \
+ mb_name * __VA_ARGS__; \
+ })
+
+#define DOCTEST_CAPTURE(x) DOCTEST_INFO(#x " := ", x)
+
+#define DOCTEST_ADD_AT_IMPL(type, file, line, mb, ...) \
+ do { \
+ doctest::detail::MessageBuilder mb(file, line, doctest::assertType::type); \
+ mb * __VA_ARGS__; \
+ DOCTEST_ASSERT_LOG_AND_REACT(mb); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+// clang-format on
+
+#define DOCTEST_MESSAGE(...) DOCTEST_ADD_MESSAGE_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL_CHECK(...) DOCTEST_ADD_FAIL_CHECK_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL(...) DOCTEST_ADD_FAIL_AT(__FILE__, __LINE__, __VA_ARGS__)
+
+#define DOCTEST_TO_LVALUE(...) __VA_ARGS__ // Not removed to keep backwards compatibility.
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_ASSERT_IMPLEMENT_2(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.setResult( \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ do { \
+ DOCTEST_ASSERT_IMPLEMENT_2(assert_type, __VA_ARGS__); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+// necessary for <ASSERT>_MESSAGE
+#define DOCTEST_ASSERT_IMPLEMENT_2 DOCTEST_ASSERT_IMPLEMENT_1
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::decomp_assert( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__) DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN, __VA_ARGS__)
+#define DOCTEST_CHECK(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK, __VA_ARGS__)
+#define DOCTEST_REQUIRE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE, __VA_ARGS__)
+#define DOCTEST_WARN_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE_FALSE, __VA_ARGS__)
+
+// clang-format off
+#define DOCTEST_WARN_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } while(false)
+#define DOCTEST_CHECK_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } while(false)
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } while(false)
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } while(false)
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } while(false)
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } while(false)
+// clang-format on
+
+#define DOCTEST_ASSERT_THROWS_AS(expr, assert_type, message, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #expr, #__VA_ARGS__, message); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(const typename doctest::detail::remove_const< \
+ typename doctest::detail::remove_reference<__VA_ARGS__>::type>::type&) { \
+ _DOCTEST_RB.translateException(); \
+ _DOCTEST_RB.m_threw_as = true; \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_THROWS_WITH(expr, expr_str, assert_type, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, expr_str, "", __VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_NOTHROW(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(__VA_ARGS__) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_WARN_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_WARN_THROWS, "")
+#define DOCTEST_CHECK_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_CHECK_THROWS, "")
+#define DOCTEST_REQUIRE_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_REQUIRE_THROWS, "")
+
+#define DOCTEST_WARN_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_AS, "", __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_WARN_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_CHECK_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_REQUIRE_THROWS_WITH, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_WITH_AS, message, __VA_ARGS__)
+
+#define DOCTEST_WARN_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_WARN_NOTHROW, __VA_ARGS__)
+#define DOCTEST_CHECK_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_CHECK_NOTHROW, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_REQUIRE_NOTHROW, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS(expr); } while(false)
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS(expr); } while(false)
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS(expr); } while(false)
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_NOTHROW(expr); } while(false)
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_NOTHROW(expr); } while(false)
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_NOTHROW(expr); } while(false)
+// clang-format on
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comp, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY( \
+ _DOCTEST_RB.binary_assert<doctest::detail::binaryAssertComparison::comp>( \
+ __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.unary_assert(__VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comparison, ...) \
+ doctest::detail::binary_assert<doctest::detail::binaryAssertComparison::comparison>( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, __VA_ARGS__)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ doctest::detail::unary_assert(doctest::assertType::assert_type, __FILE__, __LINE__, \
+ #__VA_ARGS__, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN_EQ(...) DOCTEST_BINARY_ASSERT(DT_WARN_EQ, eq, __VA_ARGS__)
+#define DOCTEST_CHECK_EQ(...) DOCTEST_BINARY_ASSERT(DT_CHECK_EQ, eq, __VA_ARGS__)
+#define DOCTEST_REQUIRE_EQ(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_EQ, eq, __VA_ARGS__)
+#define DOCTEST_WARN_NE(...) DOCTEST_BINARY_ASSERT(DT_WARN_NE, ne, __VA_ARGS__)
+#define DOCTEST_CHECK_NE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_NE, ne, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_NE, ne, __VA_ARGS__)
+#define DOCTEST_WARN_GT(...) DOCTEST_BINARY_ASSERT(DT_WARN_GT, gt, __VA_ARGS__)
+#define DOCTEST_CHECK_GT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GT, gt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GT, gt, __VA_ARGS__)
+#define DOCTEST_WARN_LT(...) DOCTEST_BINARY_ASSERT(DT_WARN_LT, lt, __VA_ARGS__)
+#define DOCTEST_CHECK_LT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LT, lt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LT, lt, __VA_ARGS__)
+#define DOCTEST_WARN_GE(...) DOCTEST_BINARY_ASSERT(DT_WARN_GE, ge, __VA_ARGS__)
+#define DOCTEST_CHECK_GE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GE, ge, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GE, ge, __VA_ARGS__)
+#define DOCTEST_WARN_LE(...) DOCTEST_BINARY_ASSERT(DT_WARN_LE, le, __VA_ARGS__)
+#define DOCTEST_CHECK_LE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LE, le, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LE, le, __VA_ARGS__)
+
+#define DOCTEST_WARN_UNARY(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY, __VA_ARGS__)
+#define DOCTEST_WARN_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY_FALSE, __VA_ARGS__)
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#undef DOCTEST_WARN_THROWS
+#undef DOCTEST_CHECK_THROWS
+#undef DOCTEST_REQUIRE_THROWS
+#undef DOCTEST_WARN_THROWS_AS
+#undef DOCTEST_CHECK_THROWS_AS
+#undef DOCTEST_REQUIRE_THROWS_AS
+#undef DOCTEST_WARN_THROWS_WITH
+#undef DOCTEST_CHECK_THROWS_WITH
+#undef DOCTEST_REQUIRE_THROWS_WITH
+#undef DOCTEST_WARN_THROWS_WITH_AS
+#undef DOCTEST_CHECK_THROWS_WITH_AS
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS
+#undef DOCTEST_WARN_NOTHROW
+#undef DOCTEST_CHECK_NOTHROW
+#undef DOCTEST_REQUIRE_NOTHROW
+
+#undef DOCTEST_WARN_THROWS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_MESSAGE
+#undef DOCTEST_WARN_THROWS_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_AS_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_WARN_NOTHROW_MESSAGE
+#undef DOCTEST_CHECK_NOTHROW_MESSAGE
+#undef DOCTEST_REQUIRE_NOTHROW_MESSAGE
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#undef DOCTEST_REQUIRE
+#undef DOCTEST_REQUIRE_FALSE
+#undef DOCTEST_REQUIRE_MESSAGE
+#undef DOCTEST_REQUIRE_FALSE_MESSAGE
+#undef DOCTEST_REQUIRE_EQ
+#undef DOCTEST_REQUIRE_NE
+#undef DOCTEST_REQUIRE_GT
+#undef DOCTEST_REQUIRE_LT
+#undef DOCTEST_REQUIRE_GE
+#undef DOCTEST_REQUIRE_LE
+#undef DOCTEST_REQUIRE_UNARY
+#undef DOCTEST_REQUIRE_UNARY_FALSE
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+// =================================================================================================
+// == WHAT FOLLOWS IS VERSIONS OF THE MACROS THAT DO NOT DO ANY REGISTERING! ==
+// == THIS CAN BE ENABLED BY DEFINING DOCTEST_CONFIG_DISABLE GLOBALLY! ==
+// =================================================================================================
+#else // DOCTEST_CONFIG_DISABLE
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, name) \
+ namespace { \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ struct der : public base \
+ { void f(); }; \
+ } \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ inline void der<DOCTEST_UNUSED_TEMPLATE_TYPE>::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, name) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests in classes
+#define DOCTEST_TEST_CASE_CLASS(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(x, name) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), x, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING(...) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+#define DOCTEST_TYPE_TO_STRING_IMPL(...)
+
+// for typed tests
+#define DOCTEST_TEST_CASE_TEMPLATE(name, type, ...) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, type, id) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for subcases
+#define DOCTEST_SUBCASE(name)
+
+// for a testsuite block
+#define DOCTEST_TEST_SUITE(name) namespace
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(name) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline doctest::String DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)(signature)
+
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+
+#define DOCTEST_INFO(...) (static_cast<void>(0))
+#define DOCTEST_CAPTURE(x) (static_cast<void>(0))
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_MESSAGE(...) (static_cast<void>(0))
+#define DOCTEST_FAIL_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_FAIL(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN(...) (static_cast<void>(0))
+#define DOCTEST_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_EQ(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_EQ(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_EQ(...) (static_cast<void>(0))
+#define DOCTEST_WARN_NE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_WARN_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) (static_cast<void>(0))
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+// clang-format off
+// KEPT FOR BACKWARDS COMPATIBILITY - FORWARDING TO THE RIGHT MACROS
+#define DOCTEST_FAST_WARN_EQ DOCTEST_WARN_EQ
+#define DOCTEST_FAST_CHECK_EQ DOCTEST_CHECK_EQ
+#define DOCTEST_FAST_REQUIRE_EQ DOCTEST_REQUIRE_EQ
+#define DOCTEST_FAST_WARN_NE DOCTEST_WARN_NE
+#define DOCTEST_FAST_CHECK_NE DOCTEST_CHECK_NE
+#define DOCTEST_FAST_REQUIRE_NE DOCTEST_REQUIRE_NE
+#define DOCTEST_FAST_WARN_GT DOCTEST_WARN_GT
+#define DOCTEST_FAST_CHECK_GT DOCTEST_CHECK_GT
+#define DOCTEST_FAST_REQUIRE_GT DOCTEST_REQUIRE_GT
+#define DOCTEST_FAST_WARN_LT DOCTEST_WARN_LT
+#define DOCTEST_FAST_CHECK_LT DOCTEST_CHECK_LT
+#define DOCTEST_FAST_REQUIRE_LT DOCTEST_REQUIRE_LT
+#define DOCTEST_FAST_WARN_GE DOCTEST_WARN_GE
+#define DOCTEST_FAST_CHECK_GE DOCTEST_CHECK_GE
+#define DOCTEST_FAST_REQUIRE_GE DOCTEST_REQUIRE_GE
+#define DOCTEST_FAST_WARN_LE DOCTEST_WARN_LE
+#define DOCTEST_FAST_CHECK_LE DOCTEST_CHECK_LE
+#define DOCTEST_FAST_REQUIRE_LE DOCTEST_REQUIRE_LE
+
+#define DOCTEST_FAST_WARN_UNARY DOCTEST_WARN_UNARY
+#define DOCTEST_FAST_CHECK_UNARY DOCTEST_CHECK_UNARY
+#define DOCTEST_FAST_REQUIRE_UNARY DOCTEST_REQUIRE_UNARY
+#define DOCTEST_FAST_WARN_UNARY_FALSE DOCTEST_WARN_UNARY_FALSE
+#define DOCTEST_FAST_CHECK_UNARY_FALSE DOCTEST_CHECK_UNARY_FALSE
+#define DOCTEST_FAST_REQUIRE_UNARY_FALSE DOCTEST_REQUIRE_UNARY_FALSE
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id,__VA_ARGS__)
+// clang-format on
+
+// BDD style macros
+// clang-format off
+#define DOCTEST_SCENARIO(name) DOCTEST_TEST_CASE(" Scenario: " name)
+#define DOCTEST_SCENARIO_CLASS(name) DOCTEST_TEST_CASE_CLASS(" Scenario: " name)
+#define DOCTEST_SCENARIO_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(" Scenario: " name, T, __VA_ARGS__)
+#define DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(" Scenario: " name, T, id)
+
+#define DOCTEST_GIVEN(name) DOCTEST_SUBCASE(" Given: " name)
+#define DOCTEST_WHEN(name) DOCTEST_SUBCASE(" When: " name)
+#define DOCTEST_AND_WHEN(name) DOCTEST_SUBCASE("And when: " name)
+#define DOCTEST_THEN(name) DOCTEST_SUBCASE(" Then: " name)
+#define DOCTEST_AND_THEN(name) DOCTEST_SUBCASE(" And: " name)
+// clang-format on
+
+// == SHORT VERSIONS OF THE MACROS
+#if !defined(DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES)
+
+#define TEST_CASE(name) DOCTEST_TEST_CASE(name)
+#define TEST_CASE_CLASS(name) DOCTEST_TEST_CASE_CLASS(name)
+#define TEST_CASE_FIXTURE(x, name) DOCTEST_TEST_CASE_FIXTURE(x, name)
+#define TYPE_TO_STRING(...) DOCTEST_TYPE_TO_STRING(__VA_ARGS__)
+#define TEST_CASE_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(name, T, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, T, id)
+#define TEST_CASE_TEMPLATE_INVOKE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_APPLY(id, ...) DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, __VA_ARGS__)
+#define SUBCASE(name) DOCTEST_SUBCASE(name)
+#define TEST_SUITE(decorators) DOCTEST_TEST_SUITE(decorators)
+#define TEST_SUITE_BEGIN(name) DOCTEST_TEST_SUITE_BEGIN(name)
+#define TEST_SUITE_END DOCTEST_TEST_SUITE_END
+#define REGISTER_EXCEPTION_TRANSLATOR(signature) DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature)
+#define REGISTER_REPORTER(name, priority, reporter) DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define REGISTER_LISTENER(name, priority, reporter) DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+#define INFO(...) DOCTEST_INFO(__VA_ARGS__)
+#define CAPTURE(x) DOCTEST_CAPTURE(x)
+#define ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_MESSAGE_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_FAIL_CHECK_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_FAIL_AT(file, line, __VA_ARGS__)
+#define MESSAGE(...) DOCTEST_MESSAGE(__VA_ARGS__)
+#define FAIL_CHECK(...) DOCTEST_FAIL_CHECK(__VA_ARGS__)
+#define FAIL(...) DOCTEST_FAIL(__VA_ARGS__)
+#define TO_LVALUE(...) DOCTEST_TO_LVALUE(__VA_ARGS__)
+
+#define WARN(...) DOCTEST_WARN(__VA_ARGS__)
+#define WARN_FALSE(...) DOCTEST_WARN_FALSE(__VA_ARGS__)
+#define WARN_THROWS(...) DOCTEST_WARN_THROWS(__VA_ARGS__)
+#define WARN_THROWS_AS(expr, ...) DOCTEST_WARN_THROWS_AS(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH(expr, ...) DOCTEST_WARN_THROWS_WITH(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS(expr, with, ...) DOCTEST_WARN_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define WARN_NOTHROW(...) DOCTEST_WARN_NOTHROW(__VA_ARGS__)
+#define CHECK(...) DOCTEST_CHECK(__VA_ARGS__)
+#define CHECK_FALSE(...) DOCTEST_CHECK_FALSE(__VA_ARGS__)
+#define CHECK_THROWS(...) DOCTEST_CHECK_THROWS(__VA_ARGS__)
+#define CHECK_THROWS_AS(expr, ...) DOCTEST_CHECK_THROWS_AS(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH(expr, ...) DOCTEST_CHECK_THROWS_WITH(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define CHECK_NOTHROW(...) DOCTEST_CHECK_NOTHROW(__VA_ARGS__)
+#define REQUIRE(...) DOCTEST_REQUIRE(__VA_ARGS__)
+#define REQUIRE_FALSE(...) DOCTEST_REQUIRE_FALSE(__VA_ARGS__)
+#define REQUIRE_THROWS(...) DOCTEST_REQUIRE_THROWS(__VA_ARGS__)
+#define REQUIRE_THROWS_AS(expr, ...) DOCTEST_REQUIRE_THROWS_AS(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH(expr, ...) DOCTEST_REQUIRE_THROWS_WITH(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define REQUIRE_NOTHROW(...) DOCTEST_REQUIRE_NOTHROW(__VA_ARGS__)
+
+#define WARN_MESSAGE(cond, ...) DOCTEST_WARN_MESSAGE(cond, __VA_ARGS__)
+#define WARN_FALSE_MESSAGE(cond, ...) DOCTEST_WARN_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define WARN_THROWS_MESSAGE(expr, ...) DOCTEST_WARN_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define WARN_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define WARN_NOTHROW_MESSAGE(expr, ...) DOCTEST_WARN_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_MESSAGE(cond, ...) DOCTEST_CHECK_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_FALSE_MESSAGE(cond, ...) DOCTEST_CHECK_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_THROWS_MESSAGE(expr, ...) DOCTEST_CHECK_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define CHECK_NOTHROW_MESSAGE(expr, ...) DOCTEST_CHECK_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_MESSAGE(cond, ...) DOCTEST_REQUIRE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_FALSE_MESSAGE(cond, ...) DOCTEST_REQUIRE_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_THROWS_MESSAGE(expr, ...) DOCTEST_REQUIRE_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define REQUIRE_NOTHROW_MESSAGE(expr, ...) DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+
+#define SCENARIO(name) DOCTEST_SCENARIO(name)
+#define SCENARIO_CLASS(name) DOCTEST_SCENARIO_CLASS(name)
+#define SCENARIO_TEMPLATE(name, T, ...) DOCTEST_SCENARIO_TEMPLATE(name, T, __VA_ARGS__)
+#define SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id)
+#define GIVEN(name) DOCTEST_GIVEN(name)
+#define WHEN(name) DOCTEST_WHEN(name)
+#define AND_WHEN(name) DOCTEST_AND_WHEN(name)
+#define THEN(name) DOCTEST_THEN(name)
+#define AND_THEN(name) DOCTEST_AND_THEN(name)
+
+#define WARN_EQ(...) DOCTEST_WARN_EQ(__VA_ARGS__)
+#define CHECK_EQ(...) DOCTEST_CHECK_EQ(__VA_ARGS__)
+#define REQUIRE_EQ(...) DOCTEST_REQUIRE_EQ(__VA_ARGS__)
+#define WARN_NE(...) DOCTEST_WARN_NE(__VA_ARGS__)
+#define CHECK_NE(...) DOCTEST_CHECK_NE(__VA_ARGS__)
+#define REQUIRE_NE(...) DOCTEST_REQUIRE_NE(__VA_ARGS__)
+#define WARN_GT(...) DOCTEST_WARN_GT(__VA_ARGS__)
+#define CHECK_GT(...) DOCTEST_CHECK_GT(__VA_ARGS__)
+#define REQUIRE_GT(...) DOCTEST_REQUIRE_GT(__VA_ARGS__)
+#define WARN_LT(...) DOCTEST_WARN_LT(__VA_ARGS__)
+#define CHECK_LT(...) DOCTEST_CHECK_LT(__VA_ARGS__)
+#define REQUIRE_LT(...) DOCTEST_REQUIRE_LT(__VA_ARGS__)
+#define WARN_GE(...) DOCTEST_WARN_GE(__VA_ARGS__)
+#define CHECK_GE(...) DOCTEST_CHECK_GE(__VA_ARGS__)
+#define REQUIRE_GE(...) DOCTEST_REQUIRE_GE(__VA_ARGS__)
+#define WARN_LE(...) DOCTEST_WARN_LE(__VA_ARGS__)
+#define CHECK_LE(...) DOCTEST_CHECK_LE(__VA_ARGS__)
+#define REQUIRE_LE(...) DOCTEST_REQUIRE_LE(__VA_ARGS__)
+#define WARN_UNARY(...) DOCTEST_WARN_UNARY(__VA_ARGS__)
+#define CHECK_UNARY(...) DOCTEST_CHECK_UNARY(__VA_ARGS__)
+#define REQUIRE_UNARY(...) DOCTEST_REQUIRE_UNARY(__VA_ARGS__)
+#define WARN_UNARY_FALSE(...) DOCTEST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define CHECK_UNARY_FALSE(...) DOCTEST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define REQUIRE_UNARY_FALSE(...) DOCTEST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+// KEPT FOR BACKWARDS COMPATIBILITY
+#define FAST_WARN_EQ(...) DOCTEST_FAST_WARN_EQ(__VA_ARGS__)
+#define FAST_CHECK_EQ(...) DOCTEST_FAST_CHECK_EQ(__VA_ARGS__)
+#define FAST_REQUIRE_EQ(...) DOCTEST_FAST_REQUIRE_EQ(__VA_ARGS__)
+#define FAST_WARN_NE(...) DOCTEST_FAST_WARN_NE(__VA_ARGS__)
+#define FAST_CHECK_NE(...) DOCTEST_FAST_CHECK_NE(__VA_ARGS__)
+#define FAST_REQUIRE_NE(...) DOCTEST_FAST_REQUIRE_NE(__VA_ARGS__)
+#define FAST_WARN_GT(...) DOCTEST_FAST_WARN_GT(__VA_ARGS__)
+#define FAST_CHECK_GT(...) DOCTEST_FAST_CHECK_GT(__VA_ARGS__)
+#define FAST_REQUIRE_GT(...) DOCTEST_FAST_REQUIRE_GT(__VA_ARGS__)
+#define FAST_WARN_LT(...) DOCTEST_FAST_WARN_LT(__VA_ARGS__)
+#define FAST_CHECK_LT(...) DOCTEST_FAST_CHECK_LT(__VA_ARGS__)
+#define FAST_REQUIRE_LT(...) DOCTEST_FAST_REQUIRE_LT(__VA_ARGS__)
+#define FAST_WARN_GE(...) DOCTEST_FAST_WARN_GE(__VA_ARGS__)
+#define FAST_CHECK_GE(...) DOCTEST_FAST_CHECK_GE(__VA_ARGS__)
+#define FAST_REQUIRE_GE(...) DOCTEST_FAST_REQUIRE_GE(__VA_ARGS__)
+#define FAST_WARN_LE(...) DOCTEST_FAST_WARN_LE(__VA_ARGS__)
+#define FAST_CHECK_LE(...) DOCTEST_FAST_CHECK_LE(__VA_ARGS__)
+#define FAST_REQUIRE_LE(...) DOCTEST_FAST_REQUIRE_LE(__VA_ARGS__)
+
+#define FAST_WARN_UNARY(...) DOCTEST_FAST_WARN_UNARY(__VA_ARGS__)
+#define FAST_CHECK_UNARY(...) DOCTEST_FAST_CHECK_UNARY(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY(...) DOCTEST_FAST_REQUIRE_UNARY(__VA_ARGS__)
+#define FAST_WARN_UNARY_FALSE(...) DOCTEST_FAST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define FAST_CHECK_UNARY_FALSE(...) DOCTEST_FAST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY_FALSE(...) DOCTEST_FAST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+#define TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// this is here to clear the 'current test suite' for the current translation unit - at the top
+DOCTEST_TEST_SUITE_END();
+
+// add stringification for primitive/fundamental types
+namespace doctest { namespace detail {
+ DOCTEST_TYPE_TO_STRING_IMPL(bool)
+ DOCTEST_TYPE_TO_STRING_IMPL(float)
+ DOCTEST_TYPE_TO_STRING_IMPL(double)
+ DOCTEST_TYPE_TO_STRING_IMPL(long double)
+ DOCTEST_TYPE_TO_STRING_IMPL(char)
+ DOCTEST_TYPE_TO_STRING_IMPL(signed char)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned char)
+#if !DOCTEST_MSVC || defined(_NATIVE_WCHAR_T_DEFINED)
+ DOCTEST_TYPE_TO_STRING_IMPL(wchar_t)
+#endif // not MSVC or wchar_t support enabled
+ DOCTEST_TYPE_TO_STRING_IMPL(short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long long int)
+}} // namespace doctest::detail
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_INCLUDED
+
+#ifndef DOCTEST_SINGLE_HEADER
+#define DOCTEST_SINGLE_HEADER
+#endif // DOCTEST_SINGLE_HEADER
+
+#if defined(DOCTEST_CONFIG_IMPLEMENT) || !defined(DOCTEST_SINGLE_HEADER)
+
+#ifndef DOCTEST_SINGLE_HEADER
+#include "doctest_fwd.h"
+#endif // DOCTEST_SINGLE_HEADER
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-macros")
+
+#ifndef DOCTEST_LIBRARY_IMPLEMENTATION
+#define DOCTEST_LIBRARY_IMPLEMENTATION
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wglobal-constructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wshorten-64-to-32")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-variable-declarations")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wcovered-switch-default")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-noreturn")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdisabled-macro-expansion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-member-function")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnonportable-system-include-path")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-default")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunsafe-loop-optimizations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wold-style-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-function")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmultiple-inheritance")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsuggest-attribute")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4267) // 'var' : conversion from 'x' to 'y', possible loss of data
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4530) // C++ exception handler used, but unwind semantics not enabled
+DOCTEST_MSVC_SUPPRESS_WARNING(4577) // 'noexcept' used with no exception handling mode specified
+DOCTEST_MSVC_SUPPRESS_WARNING(4774) // format string expected in argument is not a string literal
+DOCTEST_MSVC_SUPPRESS_WARNING(4365) // conversion from 'int' to 'unsigned', signed/unsigned mismatch
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding in structs
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+DOCTEST_MSVC_SUPPRESS_WARNING(5039) // pointer to potentially throwing function passed to extern C
+DOCTEST_MSVC_SUPPRESS_WARNING(5045) // Spectre mitigation stuff
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4800) // forcing value to bool 'true' or 'false' (performance warning)
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtor...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN
+
+// required includes - will go only in one translation unit!
+#include <ctime>
+#include <cmath>
+#include <climits>
+// borland (Embarcadero) compiler requires math.h and not cmath - https://github.com/onqtam/doctest/pull/37
+#ifdef __BORLANDC__
+#include <math.h>
+#endif // __BORLANDC__
+#include <new>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <utility>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <algorithm>
+#include <iomanip>
+#include <vector>
+#include <atomic>
+#include <mutex>
+#include <set>
+#include <map>
+#include <exception>
+#include <stdexcept>
+#include <csignal>
+#include <cfloat>
+#include <cctype>
+#include <cstdint>
+
+#ifdef DOCTEST_PLATFORM_MAC
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/sysctl.h>
+#endif // DOCTEST_PLATFORM_MAC
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+// defines for a leaner windows.h
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif // WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif // NOMINMAX
+
+// not sure what AfxWin.h is for - here I do what Catch does
+#ifdef __AFXDLL
+#include <AfxWin.h>
+#else
+#include <windows.h>
+#endif
+#include <io.h>
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+// this is a fix for https://github.com/onqtam/doctest/issues/348
+// https://mail.gnome.org/archives/xml/2012-January/msg00000.html
+#if !defined(HAVE_UNISTD_H) && !defined(STDOUT_FILENO)
+#define STDOUT_FILENO fileno(stdout)
+#endif // HAVE_UNISTD_H
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END
+
+// counts the number of elements in a C array
+#define DOCTEST_COUNTOF(x) (sizeof(x) / sizeof(x[0]))
+
+#ifdef DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_disabled
+#else // DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_not_disabled
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifndef DOCTEST_CONFIG_OPTIONS_PREFIX
+#define DOCTEST_CONFIG_OPTIONS_PREFIX "dt-"
+#endif
+
+#ifndef DOCTEST_THREAD_LOCAL
+#define DOCTEST_THREAD_LOCAL thread_local
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES
+#define DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES 32
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE
+#define DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE 64
+#endif
+
+#ifdef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY DOCTEST_CONFIG_OPTIONS_PREFIX
+#else
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY ""
+#endif
+
+#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
+#define DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+#endif
+
+namespace doctest {
+
+bool is_running_in_test = false;
+
+namespace {
+ using namespace detail;
+ // case insensitive strcmp
+ int stricmp(const char* a, const char* b) {
+ for(;; a++, b++) {
+ const int d = tolower(*a) - tolower(*b);
+ if(d != 0 || !*a)
+ return d;
+ }
+ }
+
+ template <typename T>
+ String fpToString(T value, int precision) {
+ std::ostringstream oss;
+ oss << std::setprecision(precision) << std::fixed << value;
+ std::string d = oss.str();
+ size_t i = d.find_last_not_of('0');
+ if(i != std::string::npos && i != d.size() - 1) {
+ if(d[i] == '.')
+ i++;
+ d = d.substr(0, i + 1);
+ }
+ return d.c_str();
+ }
+
+ struct Endianness
+ {
+ enum Arch
+ {
+ Big,
+ Little
+ };
+
+ static Arch which() {
+ int x = 1;
+ // casting any data pointer to char* is allowed
+ auto ptr = reinterpret_cast<char*>(&x);
+ if(*ptr)
+ return Little;
+ return Big;
+ }
+ };
+} // namespace
+
+namespace detail {
+ void my_memcpy(void* dest, const void* src, unsigned num) { memcpy(dest, src, num); }
+
+ String rawMemoryToString(const void* object, unsigned size) {
+ // Reverse order for little endian architectures
+ int i = 0, end = static_cast<int>(size), inc = 1;
+ if(Endianness::which() == Endianness::Little) {
+ i = end - 1;
+ end = inc = -1;
+ }
+
+ unsigned const char* bytes = static_cast<unsigned const char*>(object);
+ std::ostringstream oss;
+ oss << "0x" << std::setfill('0') << std::hex;
+ for(; i != end; i += inc)
+ oss << std::setw(2) << static_cast<unsigned>(bytes[i]);
+ return oss.str().c_str();
+ }
+
+ DOCTEST_THREAD_LOCAL std::ostringstream g_oss; // NOLINT(cert-err58-cpp)
+
+ std::ostream* getTlsOss() {
+ g_oss.clear(); // there shouldn't be anything worth clearing in the flags
+ g_oss.str(""); // the slow way of resetting a string stream
+ //g_oss.seekp(0); // optimal reset - as seen here: https://stackoverflow.com/a/624291/3162383
+ return &g_oss;
+ }
+
+ String getTlsOssResult() {
+ //g_oss << std::ends; // needed - as shown here: https://stackoverflow.com/a/624291/3162383
+ return g_oss.str().c_str();
+ }
+
+#ifndef DOCTEST_CONFIG_DISABLE
+
+namespace timer_large_integer
+{
+
+#if defined(DOCTEST_PLATFORM_WINDOWS)
+ typedef ULONGLONG type;
+#else // DOCTEST_PLATFORM_WINDOWS
+ using namespace std;
+ typedef uint64_t type;
+#endif // DOCTEST_PLATFORM_WINDOWS
+}
+
+typedef timer_large_integer::type ticks_t;
+
+#ifdef DOCTEST_CONFIG_GETCURRENTTICKS
+ ticks_t getCurrentTicks() { return DOCTEST_CONFIG_GETCURRENTTICKS(); }
+#elif defined(DOCTEST_PLATFORM_WINDOWS)
+ ticks_t getCurrentTicks() {
+ static LARGE_INTEGER hz = {0}, hzo = {0};
+ if(!hz.QuadPart) {
+ QueryPerformanceFrequency(&hz);
+ QueryPerformanceCounter(&hzo);
+ }
+ LARGE_INTEGER t;
+ QueryPerformanceCounter(&t);
+ return ((t.QuadPart - hzo.QuadPart) * LONGLONG(1000000)) / hz.QuadPart;
+ }
+#else // DOCTEST_PLATFORM_WINDOWS
+ ticks_t getCurrentTicks() {
+ timeval t;
+ gettimeofday(&t, nullptr);
+ return static_cast<ticks_t>(t.tv_sec) * 1000000 + static_cast<ticks_t>(t.tv_usec);
+ }
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ struct Timer
+ {
+ void start() { m_ticks = getCurrentTicks(); }
+ unsigned int getElapsedMicroseconds() const {
+ return static_cast<unsigned int>(getCurrentTicks() - m_ticks);
+ }
+ //unsigned int getElapsedMilliseconds() const {
+ // return static_cast<unsigned int>(getElapsedMicroseconds() / 1000);
+ //}
+ double getElapsedSeconds() const { return static_cast<double>(getCurrentTicks() - m_ticks) / 1000000.0; }
+
+ private:
+ ticks_t m_ticks = 0;
+ };
+
+#ifdef DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = std::atomic<T>;
+#else // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ // Provides a multilane implementation of an atomic variable that supports add, sub, load,
+ // store. Instead of using a single atomic variable, this splits up into multiple ones,
+ // each sitting on a separate cache line. The goal is to provide a speedup when most
+ // operations are modifying. It achieves this with two properties:
+ //
+ // * Multiple atomics are used, so chance of congestion from the same atomic is reduced.
+ // * Each atomic sits on a separate cache line, so false sharing is reduced.
+ //
+ // The disadvantage is that there is a small overhead due to the use of TLS, and load/store
+ // is slower because all atomics have to be accessed.
+ template <typename T>
+ class MultiLaneAtomic
+ {
+ struct CacheLineAlignedAtomic
+ {
+ std::atomic<T> atomic{};
+ char padding[DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE - sizeof(std::atomic<T>)];
+ };
+ CacheLineAlignedAtomic m_atomics[DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES];
+
+ static_assert(sizeof(CacheLineAlignedAtomic) == DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE,
+ "guarantee one atomic takes exactly one cache line");
+
+ public:
+ T operator++() DOCTEST_NOEXCEPT { return fetch_add(1) + 1; }
+
+ T operator++(int) DOCTEST_NOEXCEPT { return fetch_add(1); }
+
+ T fetch_add(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_add(arg, order);
+ }
+
+ T fetch_sub(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_sub(arg, order);
+ }
+
+ operator T() const DOCTEST_NOEXCEPT { return load(); }
+
+ T load(std::memory_order order = std::memory_order_seq_cst) const DOCTEST_NOEXCEPT {
+ auto result = T();
+ for(auto const& c : m_atomics) {
+ result += c.atomic.load(order);
+ }
+ return result;
+ }
+
+ T operator=(T desired) DOCTEST_NOEXCEPT {
+ store(desired);
+ return desired;
+ }
+
+ void store(T desired, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ // first value becomes desired", all others become 0.
+ for(auto& c : m_atomics) {
+ c.atomic.store(desired, order);
+ desired = {};
+ }
+ }
+
+ private:
+ // Each thread has a different atomic that it operates on. If more than NumLanes threads
+ // use this, some will use the same atomic. So performance will degrate a bit, but still
+ // everything will work.
+ //
+ // The logic here is a bit tricky. The call should be as fast as possible, so that there
+ // is minimal to no overhead in determining the correct atomic for the current thread.
+ //
+ // 1. A global static counter laneCounter counts continuously up.
+ // 2. Each successive thread will use modulo operation of that counter so it gets an atomic
+ // assigned in a round-robin fashion.
+ // 3. This tlsLaneIdx is stored in the thread local data, so it is directly available with
+ // little overhead.
+ std::atomic<T>& myAtomic() DOCTEST_NOEXCEPT {
+ static std::atomic<size_t> laneCounter;
+ DOCTEST_THREAD_LOCAL size_t tlsLaneIdx =
+ laneCounter++ % DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES;
+
+ return m_atomics[tlsLaneIdx].atomic;
+ }
+ };
+
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = MultiLaneAtomic<T>;
+#endif // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+
+ // this holds both parameters from the command line and runtime data for tests
+ struct ContextState : ContextOptions, TestRunStats, CurrentTestCaseStats
+ {
+ AtomicOrMultiLaneAtomic<int> numAssertsCurrentTest_atomic;
+ AtomicOrMultiLaneAtomic<int> numAssertsFailedCurrentTest_atomic;
+
+ std::vector<std::vector<String>> filters = decltype(filters)(9); // 9 different filters
+
+ std::vector<IReporter*> reporters_currently_used;
+
+ assert_handler ah = nullptr;
+
+ Timer timer;
+
+ std::vector<String> stringifiedContexts; // logging from INFO() due to an exception
+
+ // stuff for subcases
+ std::vector<SubcaseSignature> subcasesStack;
+ std::set<decltype(subcasesStack)> subcasesPassed;
+ int subcasesCurrentMaxLevel;
+ bool should_reenter;
+ std::atomic<bool> shouldLogCurrentException;
+
+ void resetRunData() {
+ numTestCases = 0;
+ numTestCasesPassingFilters = 0;
+ numTestSuitesPassingFilters = 0;
+ numTestCasesFailed = 0;
+ numAsserts = 0;
+ numAssertsFailed = 0;
+ numAssertsCurrentTest = 0;
+ numAssertsFailedCurrentTest = 0;
+ }
+
+ void finalizeTestCaseData() {
+ seconds = timer.getElapsedSeconds();
+
+ // update the non-atomic counters
+ numAsserts += numAssertsCurrentTest_atomic;
+ numAssertsFailed += numAssertsFailedCurrentTest_atomic;
+ numAssertsCurrentTest = numAssertsCurrentTest_atomic;
+ numAssertsFailedCurrentTest = numAssertsFailedCurrentTest_atomic;
+
+ if(numAssertsFailedCurrentTest)
+ failure_flags |= TestCaseFailureReason::AssertFailure;
+
+ if(Approx(currentTest->m_timeout).epsilon(DBL_EPSILON) != 0 &&
+ Approx(seconds).epsilon(DBL_EPSILON) > currentTest->m_timeout)
+ failure_flags |= TestCaseFailureReason::Timeout;
+
+ if(currentTest->m_should_fail) {
+ if(failure_flags) {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedAndDid;
+ } else {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedButDidnt;
+ }
+ } else if(failure_flags && currentTest->m_may_fail) {
+ failure_flags |= TestCaseFailureReason::CouldHaveFailedAndDid;
+ } else if(currentTest->m_expected_failures > 0) {
+ if(numAssertsFailedCurrentTest == currentTest->m_expected_failures) {
+ failure_flags |= TestCaseFailureReason::FailedExactlyNumTimes;
+ } else {
+ failure_flags |= TestCaseFailureReason::DidntFailExactlyNumTimes;
+ }
+ }
+
+ bool ok_to_fail = (TestCaseFailureReason::ShouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::CouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::FailedExactlyNumTimes & failure_flags);
+
+ // if any subcase has failed - the whole test case has failed
+ if(failure_flags && !ok_to_fail)
+ numTestCasesFailed++;
+ }
+ };
+
+ ContextState* g_cs = nullptr;
+
+ // used to avoid locks for the debug output
+ // TODO: figure out if this is indeed necessary/correct - seems like either there still
+ // could be a race or that there wouldn't be a race even if using the context directly
+ DOCTEST_THREAD_LOCAL bool g_no_colors;
+
+#endif // DOCTEST_CONFIG_DISABLE
+} // namespace detail
+
+void String::setOnHeap() { *reinterpret_cast<unsigned char*>(&buf[last]) = 128; }
+void String::setLast(unsigned in) { buf[last] = char(in); }
+
+void String::copy(const String& other) {
+ using namespace std;
+ if(other.isOnStack()) {
+ memcpy(buf, other.buf, len);
+ } else {
+ setOnHeap();
+ data.size = other.data.size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, other.data.ptr, data.size + 1);
+ }
+}
+
+String::String() {
+ buf[0] = '\0';
+ setLast();
+}
+
+String::~String() {
+ if(!isOnStack())
+ delete[] data.ptr;
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+String::String(const char* in)
+ : String(in, strlen(in)) {}
+
+String::String(const char* in, unsigned in_size) {
+ using namespace std;
+ if(in_size <= last) {
+ memcpy(buf, in, in_size);
+ buf[in_size] = '\0';
+ setLast(last - in_size);
+ } else {
+ setOnHeap();
+ data.size = in_size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, in, in_size);
+ data.ptr[in_size] = '\0';
+ }
+}
+
+String::String(const String& other) { copy(other); }
+
+String& String::operator=(const String& other) {
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+
+ copy(other);
+ }
+
+ return *this;
+}
+
+String& String::operator+=(const String& other) {
+ const unsigned my_old_size = size();
+ const unsigned other_size = other.size();
+ const unsigned total_size = my_old_size + other_size;
+ using namespace std;
+ if(isOnStack()) {
+ if(total_size < len) {
+ // append to the current stack space
+ memcpy(buf + my_old_size, other.c_str(), other_size + 1);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ setLast(last - total_size);
+ } else {
+ // alloc new chunk
+ char* temp = new char[total_size + 1];
+ // copy current data to new location before writing in the union
+ memcpy(temp, buf, my_old_size); // skip the +1 ('\0') for speed
+ // update data in union
+ setOnHeap();
+ data.size = total_size;
+ data.capacity = data.size + 1;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ } else {
+ if(data.capacity > total_size) {
+ // append to the current heap block
+ data.size = total_size;
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ } else {
+ // resize
+ data.capacity *= 2;
+ if(data.capacity <= total_size)
+ data.capacity = total_size + 1;
+ // alloc new chunk
+ char* temp = new char[data.capacity];
+ // copy current data to new location before releasing it
+ memcpy(temp, data.ptr, my_old_size); // skip the +1 ('\0') for speed
+ // release old chunk
+ delete[] data.ptr;
+ // update the rest of the union members
+ data.size = total_size;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ }
+
+ return *this;
+}
+
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String String::operator+(const String& other) const { return String(*this) += other; }
+
+String::String(String&& other) {
+ using namespace std;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+}
+
+String& String::operator=(String&& other) {
+ using namespace std;
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+ }
+ return *this;
+}
+
+char String::operator[](unsigned i) const {
+ return const_cast<String*>(this)->operator[](i); // NOLINT
+}
+
+char& String::operator[](unsigned i) {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf)[i];
+ return data.ptr[i];
+}
+
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmaybe-uninitialized")
+unsigned String::size() const {
+ if(isOnStack())
+ return last - (unsigned(buf[last]) & 31); // using "last" would work only if "len" is 32
+ return data.size;
+}
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+unsigned String::capacity() const {
+ if(isOnStack())
+ return len;
+ return data.capacity;
+}
+
+int String::compare(const char* other, bool no_case) const {
+ if(no_case)
+ return doctest::stricmp(c_str(), other);
+ return std::strcmp(c_str(), other);
+}
+
+int String::compare(const String& other, bool no_case) const {
+ return compare(other.c_str(), no_case);
+}
+
+// clang-format off
+bool operator==(const String& lhs, const String& rhs) { return lhs.compare(rhs) == 0; }
+bool operator!=(const String& lhs, const String& rhs) { return lhs.compare(rhs) != 0; }
+bool operator< (const String& lhs, const String& rhs) { return lhs.compare(rhs) < 0; }
+bool operator> (const String& lhs, const String& rhs) { return lhs.compare(rhs) > 0; }
+bool operator<=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) < 0 : true; }
+bool operator>=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) > 0 : true; }
+// clang-format on
+
+std::ostream& operator<<(std::ostream& s, const String& in) { return s << in.c_str(); }
+
+namespace {
+ void color_to_stream(std::ostream&, Color::Enum) DOCTEST_BRANCH_ON_DISABLED({}, ;)
+} // namespace
+
+namespace Color {
+ std::ostream& operator<<(std::ostream& s, Color::Enum code) {
+ color_to_stream(s, code);
+ return s;
+ }
+} // namespace Color
+
+// clang-format off
+const char* assertString(assertType::Enum at) {
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4062) // enum 'x' in switch of enum 'y' is not handled
+ switch(at) { //!OCLINT missing default in switch statements
+ case assertType::DT_WARN : return "WARN";
+ case assertType::DT_CHECK : return "CHECK";
+ case assertType::DT_REQUIRE : return "REQUIRE";
+
+ case assertType::DT_WARN_FALSE : return "WARN_FALSE";
+ case assertType::DT_CHECK_FALSE : return "CHECK_FALSE";
+ case assertType::DT_REQUIRE_FALSE : return "REQUIRE_FALSE";
+
+ case assertType::DT_WARN_THROWS : return "WARN_THROWS";
+ case assertType::DT_CHECK_THROWS : return "CHECK_THROWS";
+ case assertType::DT_REQUIRE_THROWS : return "REQUIRE_THROWS";
+
+ case assertType::DT_WARN_THROWS_AS : return "WARN_THROWS_AS";
+ case assertType::DT_CHECK_THROWS_AS : return "CHECK_THROWS_AS";
+ case assertType::DT_REQUIRE_THROWS_AS : return "REQUIRE_THROWS_AS";
+
+ case assertType::DT_WARN_THROWS_WITH : return "WARN_THROWS_WITH";
+ case assertType::DT_CHECK_THROWS_WITH : return "CHECK_THROWS_WITH";
+ case assertType::DT_REQUIRE_THROWS_WITH : return "REQUIRE_THROWS_WITH";
+
+ case assertType::DT_WARN_THROWS_WITH_AS : return "WARN_THROWS_WITH_AS";
+ case assertType::DT_CHECK_THROWS_WITH_AS : return "CHECK_THROWS_WITH_AS";
+ case assertType::DT_REQUIRE_THROWS_WITH_AS : return "REQUIRE_THROWS_WITH_AS";
+
+ case assertType::DT_WARN_NOTHROW : return "WARN_NOTHROW";
+ case assertType::DT_CHECK_NOTHROW : return "CHECK_NOTHROW";
+ case assertType::DT_REQUIRE_NOTHROW : return "REQUIRE_NOTHROW";
+
+ case assertType::DT_WARN_EQ : return "WARN_EQ";
+ case assertType::DT_CHECK_EQ : return "CHECK_EQ";
+ case assertType::DT_REQUIRE_EQ : return "REQUIRE_EQ";
+ case assertType::DT_WARN_NE : return "WARN_NE";
+ case assertType::DT_CHECK_NE : return "CHECK_NE";
+ case assertType::DT_REQUIRE_NE : return "REQUIRE_NE";
+ case assertType::DT_WARN_GT : return "WARN_GT";
+ case assertType::DT_CHECK_GT : return "CHECK_GT";
+ case assertType::DT_REQUIRE_GT : return "REQUIRE_GT";
+ case assertType::DT_WARN_LT : return "WARN_LT";
+ case assertType::DT_CHECK_LT : return "CHECK_LT";
+ case assertType::DT_REQUIRE_LT : return "REQUIRE_LT";
+ case assertType::DT_WARN_GE : return "WARN_GE";
+ case assertType::DT_CHECK_GE : return "CHECK_GE";
+ case assertType::DT_REQUIRE_GE : return "REQUIRE_GE";
+ case assertType::DT_WARN_LE : return "WARN_LE";
+ case assertType::DT_CHECK_LE : return "CHECK_LE";
+ case assertType::DT_REQUIRE_LE : return "REQUIRE_LE";
+
+ case assertType::DT_WARN_UNARY : return "WARN_UNARY";
+ case assertType::DT_CHECK_UNARY : return "CHECK_UNARY";
+ case assertType::DT_REQUIRE_UNARY : return "REQUIRE_UNARY";
+ case assertType::DT_WARN_UNARY_FALSE : return "WARN_UNARY_FALSE";
+ case assertType::DT_CHECK_UNARY_FALSE : return "CHECK_UNARY_FALSE";
+ case assertType::DT_REQUIRE_UNARY_FALSE : return "REQUIRE_UNARY_FALSE";
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ return "";
+}
+// clang-format on
+
+const char* failureString(assertType::Enum at) {
+ if(at & assertType::is_warn) //!OCLINT bitwise operator in conditional
+ return "WARNING";
+ if(at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ return "ERROR";
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return "FATAL ERROR";
+ return "";
+}
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+// depending on the current options this will remove the path of filenames
+const char* skipPathFromFilename(const char* file) {
+#ifndef DOCTEST_CONFIG_DISABLE
+ if(getContextOptions()->no_path_in_filenames) {
+ auto back = std::strrchr(file, '\\');
+ auto forward = std::strrchr(file, '/');
+ if(back || forward) {
+ if(back > forward)
+ forward = back;
+ return forward + 1;
+ }
+ }
+#endif // DOCTEST_CONFIG_DISABLE
+ return file;
+}
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+bool SubcaseSignature::operator<(const SubcaseSignature& other) const {
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ if(std::strcmp(m_file, other.m_file) != 0)
+ return std::strcmp(m_file, other.m_file) < 0;
+ return m_name.compare(other.m_name) < 0;
+}
+
+IContextScope::IContextScope() = default;
+IContextScope::~IContextScope() = default;
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(char* in) { return toString(static_cast<const char*>(in)); }
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String toString(const char* in) { return String("\"") + (in ? in : "{null string}") + "\""; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(bool in) { return in ? "true" : "false"; }
+String toString(float in) { return fpToString(in, 5) + "f"; }
+String toString(double in) { return fpToString(in, 10); }
+String toString(double long in) { return fpToString(in, 15); }
+
+#define DOCTEST_TO_STRING_OVERLOAD(type, fmt) \
+ String toString(type in) { \
+ char buf[64]; \
+ std::sprintf(buf, fmt, in); \
+ return buf; \
+ }
+
+DOCTEST_TO_STRING_OVERLOAD(char, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char signed, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int short, "%d")
+DOCTEST_TO_STRING_OVERLOAD(int short unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int, "%d")
+DOCTEST_TO_STRING_OVERLOAD(unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int long, "%ld")
+DOCTEST_TO_STRING_OVERLOAD(int long unsigned, "%lu")
+DOCTEST_TO_STRING_OVERLOAD(int long long, "%lld")
+DOCTEST_TO_STRING_OVERLOAD(int long long unsigned, "%llu")
+
+String toString(std::nullptr_t) { return "NULL"; }
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+String toString(const std::string& in) { return in.c_str(); }
+#endif // VS 2019
+
+Approx::Approx(double value)
+ : m_epsilon(static_cast<double>(std::numeric_limits<float>::epsilon()) * 100)
+ , m_scale(1.0)
+ , m_value(value) {}
+
+Approx Approx::operator()(double value) const {
+ Approx approx(value);
+ approx.epsilon(m_epsilon);
+ approx.scale(m_scale);
+ return approx;
+}
+
+Approx& Approx::epsilon(double newEpsilon) {
+ m_epsilon = newEpsilon;
+ return *this;
+}
+Approx& Approx::scale(double newScale) {
+ m_scale = newScale;
+ return *this;
+}
+
+bool operator==(double lhs, const Approx& rhs) {
+ // Thanks to Richard Harris for his help refining this formula
+ return std::fabs(lhs - rhs.m_value) <
+ rhs.m_epsilon * (rhs.m_scale + std::max<double>(std::fabs(lhs), std::fabs(rhs.m_value)));
+}
+bool operator==(const Approx& lhs, double rhs) { return operator==(rhs, lhs); }
+bool operator!=(double lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+bool operator!=(const Approx& lhs, double rhs) { return !operator==(rhs, lhs); }
+bool operator<=(double lhs, const Approx& rhs) { return lhs < rhs.m_value || lhs == rhs; }
+bool operator<=(const Approx& lhs, double rhs) { return lhs.m_value < rhs || lhs == rhs; }
+bool operator>=(double lhs, const Approx& rhs) { return lhs > rhs.m_value || lhs == rhs; }
+bool operator>=(const Approx& lhs, double rhs) { return lhs.m_value > rhs || lhs == rhs; }
+bool operator<(double lhs, const Approx& rhs) { return lhs < rhs.m_value && lhs != rhs; }
+bool operator<(const Approx& lhs, double rhs) { return lhs.m_value < rhs && lhs != rhs; }
+bool operator>(double lhs, const Approx& rhs) { return lhs > rhs.m_value && lhs != rhs; }
+bool operator>(const Approx& lhs, double rhs) { return lhs.m_value > rhs && lhs != rhs; }
+
+String toString(const Approx& in) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return String("Approx( ") + doctest::toString(in.m_value) + " )";
+}
+const ContextOptions* getContextOptions() { return DOCTEST_BRANCH_ON_DISABLED(nullptr, g_cs); }
+
+} // namespace doctest
+
+#ifdef DOCTEST_CONFIG_DISABLE
+namespace doctest {
+Context::Context(int, const char* const*) {}
+Context::~Context() = default;
+void Context::applyCommandLine(int, const char* const*) {}
+void Context::addFilter(const char*, const char*) {}
+void Context::clearFilters() {}
+void Context::setOption(const char*, int) {}
+void Context::setOption(const char*, const char*) {}
+bool Context::shouldExit() { return false; }
+void Context::setAsDefaultForAssertsOutOfTestCases() {}
+void Context::setAssertHandler(detail::assert_handler) {}
+int Context::run() { return 0; }
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return 0; }
+const IContextScope* const* IReporter::get_active_contexts() { return nullptr; }
+int IReporter::get_num_stringified_contexts() { return 0; }
+const String* IReporter::get_stringified_contexts() { return nullptr; }
+
+int registerReporter(const char*, int, IReporter*) { return 0; }
+
+} // namespace doctest
+#else // DOCTEST_CONFIG_DISABLE
+
+#if !defined(DOCTEST_CONFIG_COLORS_NONE)
+#if !defined(DOCTEST_CONFIG_COLORS_WINDOWS) && !defined(DOCTEST_CONFIG_COLORS_ANSI)
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_CONFIG_COLORS_WINDOWS
+#else // linux
+#define DOCTEST_CONFIG_COLORS_ANSI
+#endif // platform
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS && DOCTEST_CONFIG_COLORS_ANSI
+#endif // DOCTEST_CONFIG_COLORS_NONE
+
+namespace doctest_detail_test_suite_ns {
+// holds the current test suite
+doctest::detail::TestSuite& getCurrentTestSuite() {
+ static doctest::detail::TestSuite data{};
+ return data;
+}
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+namespace {
+ // the int (priority) is part of the key for automatic sorting - sadly one can register a
+ // reporter with a duplicate name and a different priority but hopefully that won't happen often :|
+ typedef std::map<std::pair<int, String>, reporterCreatorFunc> reporterMap;
+
+ reporterMap& getReporters() {
+ static reporterMap data;
+ return data;
+ }
+ reporterMap& getListeners() {
+ static reporterMap data;
+ return data;
+ }
+} // namespace
+namespace detail {
+#define DOCTEST_ITERATE_THROUGH_REPORTERS(function, ...) \
+ for(auto& curr_rep : g_cs->reporters_currently_used) \
+ curr_rep->function(__VA_ARGS__)
+
+ bool checkIfShouldThrow(assertType::Enum at) {
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return true;
+
+ if((at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ && getContextOptions()->abort_after > 0 &&
+ (g_cs->numAssertsFailed + g_cs->numAssertsFailedCurrentTest_atomic) >=
+ getContextOptions()->abort_after)
+ return true;
+
+ return false;
+ }
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN void throwException() {
+ g_cs->shouldLogCurrentException = false;
+ throw TestFailureException();
+ } // NOLINT(cert-err60-cpp)
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ void throwException() {}
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+} // namespace detail
+
+namespace {
+ using namespace detail;
+ // matching of a string against a wildcard mask (case sensitivity configurable) taken from
+ // https://www.codeproject.com/Articles/1088/Wildcard-string-compare-globbing
+ int wildcmp(const char* str, const char* wild, bool caseSensitive) {
+ const char* cp = str;
+ const char* mp = wild;
+
+ while((*str) && (*wild != '*')) {
+ if((caseSensitive ? (*wild != *str) : (tolower(*wild) != tolower(*str))) &&
+ (*wild != '?')) {
+ return 0;
+ }
+ wild++;
+ str++;
+ }
+
+ while(*str) {
+ if(*wild == '*') {
+ if(!*++wild) {
+ return 1;
+ }
+ mp = wild;
+ cp = str + 1;
+ } else if((caseSensitive ? (*wild == *str) : (tolower(*wild) == tolower(*str))) ||
+ (*wild == '?')) {
+ wild++;
+ str++;
+ } else {
+ wild = mp; //!OCLINT parameter reassignment
+ str = cp++; //!OCLINT parameter reassignment
+ }
+ }
+
+ while(*wild == '*') {
+ wild++;
+ }
+ return !*wild;
+ }
+
+ //// C string hash function (djb2) - taken from http://www.cse.yorku.ca/~oz/hash.html
+ //unsigned hashStr(unsigned const char* str) {
+ // unsigned long hash = 5381;
+ // char c;
+ // while((c = *str++))
+ // hash = ((hash << 5) + hash) + c; // hash * 33 + c
+ // return hash;
+ //}
+
+ // checks if the name matches any of the filters (and can be configured what to do when empty)
+ bool matchesAny(const char* name, const std::vector<String>& filters, bool matchEmpty,
+ bool caseSensitive) {
+ if(filters.empty() && matchEmpty)
+ return true;
+ for(auto& curr : filters)
+ if(wildcmp(name, curr.c_str(), caseSensitive))
+ return true;
+ return false;
+ }
+} // namespace
+namespace detail {
+
+ Subcase::Subcase(const String& name, const char* file, int line)
+ : m_signature({name, file, line}) {
+ auto* s = g_cs;
+
+ // check subcase filters
+ if(s->subcasesStack.size() < size_t(s->subcase_filter_levels)) {
+ if(!matchesAny(m_signature.m_name.c_str(), s->filters[6], true, s->case_sensitive))
+ return;
+ if(matchesAny(m_signature.m_name.c_str(), s->filters[7], false, s->case_sensitive))
+ return;
+ }
+
+ // if a Subcase on the same level has already been entered
+ if(s->subcasesStack.size() < size_t(s->subcasesCurrentMaxLevel)) {
+ s->should_reenter = true;
+ return;
+ }
+
+ // push the current signature to the stack so we can check if the
+ // current stack + the current new subcase have been traversed
+ s->subcasesStack.push_back(m_signature);
+ if(s->subcasesPassed.count(s->subcasesStack) != 0) {
+ // pop - revert to previous stack since we've already passed this
+ s->subcasesStack.pop_back();
+ return;
+ }
+
+ s->subcasesCurrentMaxLevel = s->subcasesStack.size();
+ m_entered = true;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_start, m_signature);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ Subcase::~Subcase() {
+ if(m_entered) {
+ // only mark the subcase stack as passed if no subcases have been skipped
+ if(g_cs->should_reenter == false)
+ g_cs->subcasesPassed.insert(g_cs->subcasesStack);
+ g_cs->subcasesStack.pop_back();
+
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0
+#else
+ if(std::uncaught_exception()
+#endif
+ && g_cs->shouldLogCurrentException) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(
+ test_case_exception, {"exception thrown in subcase - will translate later "
+ "when the whole test case has been exited (cannot "
+ "translate while there is an active exception)",
+ false});
+ g_cs->shouldLogCurrentException = false;
+ }
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ Subcase::operator bool() const { return m_entered; }
+
+ Result::Result(bool passed, const String& decomposition)
+ : m_passed(passed)
+ , m_decomp(decomposition) {}
+
+ ExpressionDecomposer::ExpressionDecomposer(assertType::Enum at)
+ : m_at(at) {}
+
+ TestSuite& TestSuite::operator*(const char* in) {
+ m_test_suite = in;
+ // clear state
+ m_description = nullptr;
+ m_skip = false;
+ m_no_breaks = false;
+ m_no_output = false;
+ m_may_fail = false;
+ m_should_fail = false;
+ m_expected_failures = 0;
+ m_timeout = 0;
+ return *this;
+ }
+
+ TestCase::TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type, int template_id) {
+ m_file = file;
+ m_line = line;
+ m_name = nullptr; // will be later overridden in operator*
+ m_test_suite = test_suite.m_test_suite;
+ m_description = test_suite.m_description;
+ m_skip = test_suite.m_skip;
+ m_no_breaks = test_suite.m_no_breaks;
+ m_no_output = test_suite.m_no_output;
+ m_may_fail = test_suite.m_may_fail;
+ m_should_fail = test_suite.m_should_fail;
+ m_expected_failures = test_suite.m_expected_failures;
+ m_timeout = test_suite.m_timeout;
+
+ m_test = test;
+ m_type = type;
+ m_template_id = template_id;
+ }
+
+ TestCase::TestCase(const TestCase& other)
+ : TestCaseData() {
+ *this = other;
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ DOCTEST_MSVC_SUPPRESS_WARNING(26437) // Do not slice
+ TestCase& TestCase::operator=(const TestCase& other) {
+ static_cast<TestCaseData&>(*this) = static_cast<const TestCaseData&>(other);
+
+ m_test = other.m_test;
+ m_type = other.m_type;
+ m_template_id = other.m_template_id;
+ m_full_name = other.m_full_name;
+
+ if(m_template_id != -1)
+ m_name = m_full_name.c_str();
+ return *this;
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& TestCase::operator*(const char* in) {
+ m_name = in;
+ // make a new name with an appended type for templated test case
+ if(m_template_id != -1) {
+ m_full_name = String(m_name) + m_type;
+ // redirect the name to point to the newly constructed full name
+ m_name = m_full_name.c_str();
+ }
+ return *this;
+ }
+
+ bool TestCase::operator<(const TestCase& other) const {
+ // this will be used only to differentiate between test cases - not relevant for sorting
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ const int name_cmp = strcmp(m_name, other.m_name);
+ if(name_cmp != 0)
+ return name_cmp < 0;
+ const int file_cmp = m_file.compare(other.m_file);
+ if(file_cmp != 0)
+ return file_cmp < 0;
+ return m_template_id < other.m_template_id;
+ }
+
+ // all the registered tests
+ std::set<TestCase>& getRegisteredTests() {
+ static std::set<TestCase> data;
+ return data;
+ }
+} // namespace detail
+namespace {
+ using namespace detail;
+ // for sorting tests by file/line
+ bool fileOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ // this is needed because MSVC gives different case for drive letters
+ // for __FILE__ when evaluated in a header and a source file
+ const int res = lhs->m_file.compare(rhs->m_file, bool(DOCTEST_MSVC));
+ if(res != 0)
+ return res < 0;
+ if(lhs->m_line != rhs->m_line)
+ return lhs->m_line < rhs->m_line;
+ return lhs->m_template_id < rhs->m_template_id;
+ }
+
+ // for sorting tests by suite/file/line
+ bool suiteOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_test_suite, rhs->m_test_suite);
+ if(res != 0)
+ return res < 0;
+ return fileOrderComparator(lhs, rhs);
+ }
+
+ // for sorting tests by name/suite/file/line
+ bool nameOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_name, rhs->m_name);
+ if(res != 0)
+ return res < 0;
+ return suiteOrderComparator(lhs, rhs);
+ }
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ HANDLE g_stdoutHandle;
+ WORD g_origFgAttrs;
+ WORD g_origBgAttrs;
+ bool g_attrsInitted = false;
+
+ int colors_init() {
+ if(!g_attrsInitted) {
+ g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+ g_attrsInitted = true;
+ CONSOLE_SCREEN_BUFFER_INFO csbiInfo;
+ GetConsoleScreenBufferInfo(g_stdoutHandle, &csbiInfo);
+ g_origFgAttrs = csbiInfo.wAttributes & ~(BACKGROUND_GREEN | BACKGROUND_RED |
+ BACKGROUND_BLUE | BACKGROUND_INTENSITY);
+ g_origBgAttrs = csbiInfo.wAttributes & ~(FOREGROUND_GREEN | FOREGROUND_RED |
+ FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ }
+ return 0;
+ }
+
+ int dumy_init_console_colors = colors_init();
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ void color_to_stream(std::ostream& s, Color::Enum code) {
+ static_cast<void>(s); // for DOCTEST_CONFIG_COLORS_NONE or DOCTEST_CONFIG_COLORS_WINDOWS
+ static_cast<void>(code); // for DOCTEST_CONFIG_COLORS_NONE
+#ifdef DOCTEST_CONFIG_COLORS_ANSI
+ if(g_no_colors ||
+ (isatty(STDOUT_FILENO) == false && getContextOptions()->force_colors == false))
+ return;
+
+ auto col = "";
+ // clang-format off
+ switch(code) { //!OCLINT missing break in switch statement / unnecessary default statement in covered switch statement
+ case Color::Red: col = "[0;31m"; break;
+ case Color::Green: col = "[0;32m"; break;
+ case Color::Blue: col = "[0;34m"; break;
+ case Color::Cyan: col = "[0;36m"; break;
+ case Color::Yellow: col = "[0;33m"; break;
+ case Color::Grey: col = "[1;30m"; break;
+ case Color::LightGrey: col = "[0;37m"; break;
+ case Color::BrightRed: col = "[1;31m"; break;
+ case Color::BrightGreen: col = "[1;32m"; break;
+ case Color::BrightWhite: col = "[1;37m"; break;
+ case Color::Bright: // invalid
+ case Color::None:
+ case Color::White:
+ default: col = "[0m";
+ }
+ // clang-format on
+ s << "\033" << col;
+#endif // DOCTEST_CONFIG_COLORS_ANSI
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ if(g_no_colors ||
+ (isatty(fileno(stdout)) == false && getContextOptions()->force_colors == false))
+ return;
+
+#define DOCTEST_SET_ATTR(x) SetConsoleTextAttribute(g_stdoutHandle, x | g_origBgAttrs)
+
+ // clang-format off
+ switch (code) {
+ case Color::White: DOCTEST_SET_ATTR(FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::Red: DOCTEST_SET_ATTR(FOREGROUND_RED); break;
+ case Color::Green: DOCTEST_SET_ATTR(FOREGROUND_GREEN); break;
+ case Color::Blue: DOCTEST_SET_ATTR(FOREGROUND_BLUE); break;
+ case Color::Cyan: DOCTEST_SET_ATTR(FOREGROUND_BLUE | FOREGROUND_GREEN); break;
+ case Color::Yellow: DOCTEST_SET_ATTR(FOREGROUND_RED | FOREGROUND_GREEN); break;
+ case Color::Grey: DOCTEST_SET_ATTR(0); break;
+ case Color::LightGrey: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY); break;
+ case Color::BrightRed: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_RED); break;
+ case Color::BrightGreen: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN); break;
+ case Color::BrightWhite: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::None:
+ case Color::Bright: // invalid
+ default: DOCTEST_SET_ATTR(g_origFgAttrs);
+ }
+ // clang-format on
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+ std::vector<const IExceptionTranslator*>& getExceptionTranslators() {
+ static std::vector<const IExceptionTranslator*> data;
+ return data;
+ }
+
+ String translateActiveException() {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ String res;
+ auto& translators = getExceptionTranslators();
+ for(auto& curr : translators)
+ if(curr->translate(res))
+ return res;
+ // clang-format off
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wcatch-value")
+ try {
+ throw;
+ } catch(std::exception& ex) {
+ return ex.what();
+ } catch(std::string& msg) {
+ return msg.c_str();
+ } catch(const char* msg) {
+ return msg;
+ } catch(...) {
+ return "unknown exception";
+ }
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+// clang-format on
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ return "";
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+} // namespace
+
+namespace detail {
+ // used by the macros for registering tests
+ int regTest(const TestCase& tc) {
+ getRegisteredTests().insert(tc);
+ return 0;
+ }
+
+ // sets the current test suite
+ int setTestSuite(const TestSuite& ts) {
+ doctest_detail_test_suite_ns::getCurrentTestSuite() = ts;
+ return 0;
+ }
+
+#ifdef DOCTEST_IS_DEBUGGER_ACTIVE
+ bool isDebuggerActive() { return DOCTEST_IS_DEBUGGER_ACTIVE(); }
+#else // DOCTEST_IS_DEBUGGER_ACTIVE
+#ifdef DOCTEST_PLATFORM_LINUX
+ class ErrnoGuard {
+ public:
+ ErrnoGuard() : m_oldErrno(errno) {}
+ ~ErrnoGuard() { errno = m_oldErrno; }
+ private:
+ int m_oldErrno;
+ };
+ // See the comments in Catch2 for the reasoning behind this implementation:
+ // https://github.com/catchorg/Catch2/blob/v2.13.1/include/internal/catch_debugger.cpp#L79-L102
+ bool isDebuggerActive() {
+ ErrnoGuard guard;
+ std::ifstream in("/proc/self/status");
+ for(std::string line; std::getline(in, line);) {
+ static const int PREFIX_LEN = 11;
+ if(line.compare(0, PREFIX_LEN, "TracerPid:\t") == 0) {
+ return line.length() > PREFIX_LEN && line[PREFIX_LEN] != '0';
+ }
+ }
+ return false;
+ }
+#elif defined(DOCTEST_PLATFORM_MAC)
+ // The following function is taken directly from the following technical note:
+ // https://developer.apple.com/library/archive/qa/qa1361/_index.html
+ // Returns true if the current process is being debugged (either
+ // running under the debugger or has a debugger attached post facto).
+ bool isDebuggerActive() {
+ int mib[4];
+ kinfo_proc info;
+ size_t size;
+ // Initialize the flags so that, if sysctl fails for some bizarre
+ // reason, we get a predictable result.
+ info.kp_proc.p_flag = 0;
+ // Initialize mib, which tells sysctl the info we want, in this case
+ // we're looking for information about a specific process ID.
+ mib[0] = CTL_KERN;
+ mib[1] = KERN_PROC;
+ mib[2] = KERN_PROC_PID;
+ mib[3] = getpid();
+ // Call sysctl.
+ size = sizeof(info);
+ if(sysctl(mib, DOCTEST_COUNTOF(mib), &info, &size, 0, 0) != 0) {
+ std::cerr << "\nCall to sysctl failed - unable to determine if debugger is active **\n";
+ return false;
+ }
+ // We're being debugged if the P_TRACED flag is set.
+ return ((info.kp_proc.p_flag & P_TRACED) != 0);
+ }
+#elif DOCTEST_MSVC || defined(__MINGW32__) || defined(__MINGW64__)
+ bool isDebuggerActive() { return ::IsDebuggerPresent() != 0; }
+#else
+ bool isDebuggerActive() { return false; }
+#endif // Platform
+#endif // DOCTEST_IS_DEBUGGER_ACTIVE
+
+ void registerExceptionTranslatorImpl(const IExceptionTranslator* et) {
+ if(std::find(getExceptionTranslators().begin(), getExceptionTranslators().end(), et) ==
+ getExceptionTranslators().end())
+ getExceptionTranslators().push_back(et);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, char* in) { *s << in; }
+ void toStream(std::ostream* s, const char* in) { *s << in; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, bool in) { *s << std::boolalpha << in << std::noboolalpha; }
+ void toStream(std::ostream* s, float in) { *s << in; }
+ void toStream(std::ostream* s, double in) { *s << in; }
+ void toStream(std::ostream* s, double long in) { *s << in; }
+
+ void toStream(std::ostream* s, char in) { *s << in; }
+ void toStream(std::ostream* s, char signed in) { *s << in; }
+ void toStream(std::ostream* s, char unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int short in) { *s << in; }
+ void toStream(std::ostream* s, int short unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int in) { *s << in; }
+ void toStream(std::ostream* s, int unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long in) { *s << in; }
+ void toStream(std::ostream* s, int long unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long long in) { *s << in; }
+ void toStream(std::ostream* s, int long long unsigned in) { *s << in; }
+
+ DOCTEST_THREAD_LOCAL std::vector<IContextScope*> g_infoContexts; // for logging with INFO()
+
+ ContextScopeBase::ContextScopeBase() {
+ g_infoContexts.push_back(this);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ // destroy cannot be inlined into the destructor because that would mean calling stringify after
+ // ContextScope has been destroyed (base class destructors run after derived class destructors).
+ // Instead, ContextScope calls this method directly from its destructor.
+ void ContextScopeBase::destroy() {
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0) {
+#else
+ if(std::uncaught_exception()) {
+#endif
+ std::ostringstream s;
+ this->stringify(&s);
+ g_cs->stringifiedContexts.push_back(s.str().c_str());
+ }
+ g_infoContexts.pop_back();
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+} // namespace detail
+namespace {
+ using namespace detail;
+
+#if !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ struct FatalConditionHandler
+ {
+ static void reset() {}
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+ };
+#else // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+ void reportFatal(const std::string&);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ DWORD id;
+ const char* name;
+ };
+ // There is no 1-1 mapping between signals and windows exceptions.
+ // Windows can easily distinguish between SO and SigSegV,
+ // but SigInt, SigTerm, etc are handled differently.
+ SignalDefs signalDefs[] = {
+ {static_cast<DWORD>(EXCEPTION_ILLEGAL_INSTRUCTION),
+ "SIGILL - Illegal instruction signal"},
+ {static_cast<DWORD>(EXCEPTION_STACK_OVERFLOW), "SIGSEGV - Stack overflow"},
+ {static_cast<DWORD>(EXCEPTION_ACCESS_VIOLATION),
+ "SIGSEGV - Segmentation violation signal"},
+ {static_cast<DWORD>(EXCEPTION_INT_DIVIDE_BY_ZERO), "Divide by zero error"},
+ };
+
+ struct FatalConditionHandler
+ {
+ static LONG CALLBACK handleException(PEXCEPTION_POINTERS ExceptionInfo) {
+ // Multiple threads may enter this filter/handler at once. We want the error message to be printed on the
+ // console just once no matter how many threads have crashed.
+ static std::mutex mutex;
+ static bool execute = true;
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ if(execute) {
+ bool reported = false;
+ for(size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ if(ExceptionInfo->ExceptionRecord->ExceptionCode == signalDefs[i].id) {
+ reportFatal(signalDefs[i].name);
+ reported = true;
+ break;
+ }
+ }
+ if(reported == false)
+ reportFatal("Unhandled SEH exception caught");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ }
+ execute = false;
+ }
+ std::exit(EXIT_FAILURE);
+ }
+
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+
+ FatalConditionHandler() {
+ isSet = true;
+ // 32k seems enough for doctest to handle stack overflow,
+ // but the value was found experimentally, so there is no strong guarantee
+ guaranteeSize = 32 * 1024;
+ // Register an unhandled exception filter
+ previousTop = SetUnhandledExceptionFilter(handleException);
+ // Pass in guarantee size to be filled
+ SetThreadStackGuarantee(&guaranteeSize);
+
+ // On Windows uncaught exceptions from another thread, exceptions from
+ // destructors, or calls to std::terminate are not a SEH exception
+
+ // The terminal handler gets called when:
+ // - std::terminate is called FROM THE TEST RUNNER THREAD
+ // - an exception is thrown from a destructor FROM THE TEST RUNNER THREAD
+ original_terminate_handler = std::get_terminate();
+ std::set_terminate([]() DOCTEST_NOEXCEPT {
+ reportFatal("Terminate handler called");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE); // explicitly exit - otherwise the SIGABRT handler may be called as well
+ });
+
+ // SIGABRT is raised when:
+ // - std::terminate is called FROM A DIFFERENT THREAD
+ // - an exception is thrown from a destructor FROM A DIFFERENT THREAD
+ // - an uncaught exception is thrown FROM A DIFFERENT THREAD
+ prev_sigabrt_handler = std::signal(SIGABRT, [](int signal) DOCTEST_NOEXCEPT {
+ if(signal == SIGABRT) {
+ reportFatal("SIGABRT - Abort (abnormal termination) signal");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE);
+ }
+ });
+
+ // The following settings are taken from google test, and more
+ // specifically from UnitTest::Run() inside of gtest.cc
+
+ // the user does not want to see pop-up dialogs about crashes
+ prev_error_mode_1 = SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT |
+ SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX);
+ // This forces the abort message to go to stderr in all circumstances.
+ prev_error_mode_2 = _set_error_mode(_OUT_TO_STDERR);
+ // In the debug version, Visual Studio pops up a separate dialog
+ // offering a choice to debug the aborted program - we want to disable that.
+ prev_abort_behavior = _set_abort_behavior(0x0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ // In debug mode, the Windows CRT can crash with an assertion over invalid
+ // input (e.g. passing an invalid file descriptor). The default handling
+ // for these assertions is to pop up a dialog and wait for user input.
+ // Instead ask the CRT to dump such assertions to stderr non-interactively.
+ prev_report_mode = _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
+ prev_report_file = _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR);
+ }
+
+ static void reset() {
+ if(isSet) {
+ // Unregister handler and restore the old guarantee
+ SetUnhandledExceptionFilter(previousTop);
+ SetThreadStackGuarantee(&guaranteeSize);
+ std::set_terminate(original_terminate_handler);
+ std::signal(SIGABRT, prev_sigabrt_handler);
+ SetErrorMode(prev_error_mode_1);
+ _set_error_mode(prev_error_mode_2);
+ _set_abort_behavior(prev_abort_behavior, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ static_cast<void>(_CrtSetReportMode(_CRT_ASSERT, prev_report_mode));
+ static_cast<void>(_CrtSetReportFile(_CRT_ASSERT, prev_report_file));
+ isSet = false;
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+
+ private:
+ static UINT prev_error_mode_1;
+ static int prev_error_mode_2;
+ static unsigned int prev_abort_behavior;
+ static int prev_report_mode;
+ static _HFILE prev_report_file;
+ static void (*prev_sigabrt_handler)(int);
+ static std::terminate_handler original_terminate_handler;
+ static bool isSet;
+ static ULONG guaranteeSize;
+ static LPTOP_LEVEL_EXCEPTION_FILTER previousTop;
+ };
+
+ UINT FatalConditionHandler::prev_error_mode_1;
+ int FatalConditionHandler::prev_error_mode_2;
+ unsigned int FatalConditionHandler::prev_abort_behavior;
+ int FatalConditionHandler::prev_report_mode;
+ _HFILE FatalConditionHandler::prev_report_file;
+ void (*FatalConditionHandler::prev_sigabrt_handler)(int);
+ std::terminate_handler FatalConditionHandler::original_terminate_handler;
+ bool FatalConditionHandler::isSet = false;
+ ULONG FatalConditionHandler::guaranteeSize = 0;
+ LPTOP_LEVEL_EXCEPTION_FILTER FatalConditionHandler::previousTop = nullptr;
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ int id;
+ const char* name;
+ };
+ SignalDefs signalDefs[] = {{SIGINT, "SIGINT - Terminal interrupt signal"},
+ {SIGILL, "SIGILL - Illegal instruction signal"},
+ {SIGFPE, "SIGFPE - Floating point error signal"},
+ {SIGSEGV, "SIGSEGV - Segmentation violation signal"},
+ {SIGTERM, "SIGTERM - Termination request signal"},
+ {SIGABRT, "SIGABRT - Abort (abnormal termination) signal"}};
+
+ struct FatalConditionHandler
+ {
+ static bool isSet;
+ static struct sigaction oldSigActions[DOCTEST_COUNTOF(signalDefs)];
+ static stack_t oldSigStack;
+ static size_t altStackSize;
+ static char* altStackMem;
+
+ static void handleSignal(int sig) {
+ const char* name = "<unknown signal>";
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ SignalDefs& def = signalDefs[i];
+ if(sig == def.id) {
+ name = def.name;
+ break;
+ }
+ }
+ reset();
+ reportFatal(name);
+ raise(sig);
+ }
+
+ static void allocateAltStackMem() {
+ altStackMem = new char[altStackSize];
+ }
+
+ static void freeAltStackMem() {
+ delete[] altStackMem;
+ }
+
+ FatalConditionHandler() {
+ isSet = true;
+ stack_t sigStack;
+ sigStack.ss_sp = altStackMem;
+ sigStack.ss_size = altStackSize;
+ sigStack.ss_flags = 0;
+ sigaltstack(&sigStack, &oldSigStack);
+ struct sigaction sa = {};
+ sa.sa_handler = handleSignal; // NOLINT
+ sa.sa_flags = SA_ONSTACK;
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &sa, &oldSigActions[i]);
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+ static void reset() {
+ if(isSet) {
+ // Set signals back to previous values -- hopefully nobody overwrote them in the meantime
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &oldSigActions[i], nullptr);
+ }
+ // Return the old stack
+ sigaltstack(&oldSigStack, nullptr);
+ isSet = false;
+ }
+ }
+ };
+
+ bool FatalConditionHandler::isSet = false;
+ struct sigaction FatalConditionHandler::oldSigActions[DOCTEST_COUNTOF(signalDefs)] = {};
+ stack_t FatalConditionHandler::oldSigStack = {};
+ size_t FatalConditionHandler::altStackSize = 4 * SIGSTKSZ;
+ char* FatalConditionHandler::altStackMem = nullptr;
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+} // namespace
+
+namespace {
+ using namespace detail;
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) ::OutputDebugStringA(text)
+#else
+ // TODO: integration with XCode and other IDEs
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) // NOLINT(clang-diagnostic-unused-macros)
+#endif // Platform
+
+ void addAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsCurrentTest_atomic++;
+ }
+
+ void addFailedAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsFailedCurrentTest_atomic++;
+ }
+
+#if defined(DOCTEST_CONFIG_POSIX_SIGNALS) || defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ void reportFatal(const std::string& message) {
+ g_cs->failure_flags |= TestCaseFailureReason::Crash;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception, {message.c_str(), true});
+
+ while(g_cs->subcasesStack.size()) {
+ g_cs->subcasesStack.pop_back();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+
+ g_cs->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ }
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+} // namespace
+namespace detail {
+
+ ResultBuilder::ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type, const char* exception_string) {
+ m_test_case = g_cs->currentTest;
+ m_at = at;
+ m_file = file;
+ m_line = line;
+ m_expr = expr;
+ m_failed = true;
+ m_threw = false;
+ m_threw_as = false;
+ m_exception_type = exception_type;
+ m_exception_string = exception_string;
+#if DOCTEST_MSVC
+ if(m_expr[0] == ' ') // this happens when variadic macros are disabled under MSVC
+ ++m_expr;
+#endif // MSVC
+ }
+
+ void ResultBuilder::setResult(const Result& res) {
+ m_decomp = res.m_decomp;
+ m_failed = !res.m_passed;
+ }
+
+ void ResultBuilder::translateException() {
+ m_threw = true;
+ m_exception = translateActiveException();
+ }
+
+ bool ResultBuilder::log() {
+ if(m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw;
+ } else if((m_at & assertType::is_throws_as) && (m_at & assertType::is_throws_with)) { //!OCLINT
+ m_failed = !m_threw_as || (m_exception != m_exception_string);
+ } else if(m_at & assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw_as;
+ } else if(m_at & assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ m_failed = m_exception != m_exception_string;
+ } else if(m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ m_failed = m_threw;
+ }
+
+ if(m_exception.size())
+ m_exception = String("\"") + m_exception + "\"";
+
+ if(is_running_in_test) {
+ addAssert(m_at);
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_assert, *this);
+
+ if(m_failed)
+ addFailedAssert(m_at);
+ } else if(m_failed) {
+ failed_out_of_a_testing_context(*this);
+ }
+
+ return m_failed && isDebuggerActive() && !getContextOptions()->no_breaks &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void ResultBuilder::react() const {
+ if(m_failed && checkIfShouldThrow(m_at))
+ throwException();
+ }
+
+ void failed_out_of_a_testing_context(const AssertData& ad) {
+ if(g_cs->ah)
+ g_cs->ah(ad);
+ else
+ std::abort();
+ }
+
+ void decomp_assert(assertType::Enum at, const char* file, int line, const char* expr,
+ Result result) {
+ bool failed = !result.m_passed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(result.m_decomp);
+ DOCTEST_ASSERT_IN_TESTS(result.m_decomp);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ }
+
+ MessageBuilder::MessageBuilder(const char* file, int line, assertType::Enum severity) {
+ m_stream = getTlsOss();
+ m_file = file;
+ m_line = line;
+ m_severity = severity;
+ }
+
+ IExceptionTranslator::IExceptionTranslator() = default;
+ IExceptionTranslator::~IExceptionTranslator() = default;
+
+ bool MessageBuilder::log() {
+ m_string = getTlsOssResult();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_message, *this);
+
+ const bool isWarn = m_severity & assertType::is_warn;
+
+ // warn is just a message in this context so we don't treat it as an assert
+ if(!isWarn) {
+ addAssert(m_severity);
+ addFailedAssert(m_severity);
+ }
+
+ return isDebuggerActive() && !getContextOptions()->no_breaks && !isWarn &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void MessageBuilder::react() {
+ if(m_severity & assertType::is_require) //!OCLINT bitwise operator in conditional
+ throwException();
+ }
+
+ MessageBuilder::~MessageBuilder() = default;
+} // namespace detail
+namespace {
+ using namespace detail;
+
+ template <typename Ex>
+ DOCTEST_NORETURN void throw_exception(Ex const& e) {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ throw e;
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ std::cerr << "doctest will terminate because it needed to throw an exception.\n"
+ << "The message was: " << e.what() << '\n';
+ std::terminate();
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+
+#ifndef DOCTEST_INTERNAL_ERROR
+#define DOCTEST_INTERNAL_ERROR(msg) \
+ throw_exception(std::logic_error( \
+ __FILE__ ":" DOCTEST_TOSTR(__LINE__) ": Internal doctest error: " msg))
+#endif // DOCTEST_INTERNAL_ERROR
+
+ // clang-format off
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+ class XmlEncode {
+ public:
+ enum ForWhat { ForTextNodes, ForAttributes };
+
+ XmlEncode( std::string const& str, ForWhat forWhat = ForTextNodes );
+
+ void encodeTo( std::ostream& os ) const;
+
+ friend std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode );
+
+ private:
+ std::string m_str;
+ ForWhat m_forWhat;
+ };
+
+ class XmlWriter {
+ public:
+
+ class ScopedElement {
+ public:
+ ScopedElement( XmlWriter* writer );
+
+ ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+ ScopedElement& operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+
+ ~ScopedElement();
+
+ ScopedElement& writeText( std::string const& text, bool indent = true );
+
+ template<typename T>
+ ScopedElement& writeAttribute( std::string const& name, T const& attribute ) {
+ m_writer->writeAttribute( name, attribute );
+ return *this;
+ }
+
+ private:
+ mutable XmlWriter* m_writer = nullptr;
+ };
+
+ XmlWriter( std::ostream& os = std::cout );
+ ~XmlWriter();
+
+ XmlWriter( XmlWriter const& ) = delete;
+ XmlWriter& operator=( XmlWriter const& ) = delete;
+
+ XmlWriter& startElement( std::string const& name );
+
+ ScopedElement scopedElement( std::string const& name );
+
+ XmlWriter& endElement();
+
+ XmlWriter& writeAttribute( std::string const& name, std::string const& attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, const char* attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, bool attribute );
+
+ template<typename T>
+ XmlWriter& writeAttribute( std::string const& name, T const& attribute ) {
+ std::stringstream rss;
+ rss << attribute;
+ return writeAttribute( name, rss.str() );
+ }
+
+ XmlWriter& writeText( std::string const& text, bool indent = true );
+
+ //XmlWriter& writeComment( std::string const& text );
+
+ //void writeStylesheetRef( std::string const& url );
+
+ //XmlWriter& writeBlankLine();
+
+ void ensureTagClosed();
+
+ private:
+
+ void writeDeclaration();
+
+ void newlineIfNecessary();
+
+ bool m_tagIsOpen = false;
+ bool m_needsNewline = false;
+ std::vector<std::string> m_tags;
+ std::string m_indent;
+ std::ostream& m_os;
+ };
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+using uchar = unsigned char;
+
+namespace {
+
+ size_t trailingBytes(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return 2;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return 3;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return 4;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ uint32_t headerValue(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return c & 0x1F;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return c & 0x0F;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return c & 0x07;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ void hexEscapeChar(std::ostream& os, unsigned char c) {
+ std::ios_base::fmtflags f(os.flags());
+ os << "\\x"
+ << std::uppercase << std::hex << std::setfill('0') << std::setw(2)
+ << static_cast<int>(c);
+ os.flags(f);
+ }
+
+} // anonymous namespace
+
+ XmlEncode::XmlEncode( std::string const& str, ForWhat forWhat )
+ : m_str( str ),
+ m_forWhat( forWhat )
+ {}
+
+ void XmlEncode::encodeTo( std::ostream& os ) const {
+ // Apostrophe escaping not necessary if we always use " to write attributes
+ // (see: https://www.w3.org/TR/xml/#syntax)
+
+ for( std::size_t idx = 0; idx < m_str.size(); ++ idx ) {
+ uchar c = m_str[idx];
+ switch (c) {
+ case '<': os << "&lt;"; break;
+ case '&': os << "&amp;"; break;
+
+ case '>':
+ // See: https://www.w3.org/TR/xml/#syntax
+ if (idx > 2 && m_str[idx - 1] == ']' && m_str[idx - 2] == ']')
+ os << "&gt;";
+ else
+ os << c;
+ break;
+
+ case '\"':
+ if (m_forWhat == ForAttributes)
+ os << "&quot;";
+ else
+ os << c;
+ break;
+
+ default:
+ // Check for control characters and invalid utf-8
+
+ // Escape control characters in standard ascii
+ // see https://stackoverflow.com/questions/404107/why-are-control-characters-illegal-in-xml-1-0
+ if (c < 0x09 || (c > 0x0D && c < 0x20) || c == 0x7F) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // Plain ASCII: Write it to stream
+ if (c < 0x7F) {
+ os << c;
+ break;
+ }
+
+ // UTF-8 territory
+ // Check if the encoding is valid and if it is not, hex escape bytes.
+ // Important: We do not check the exact decoded values for validity, only the encoding format
+ // First check that this bytes is a valid lead byte:
+ // This means that it is not encoded as 1111 1XXX
+ // Or as 10XX XXXX
+ if (c < 0xC0 ||
+ c >= 0xF8) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ auto encBytes = trailingBytes(c);
+ // Are there enough bytes left to avoid accessing out-of-bounds memory?
+ if (idx + encBytes - 1 >= m_str.size()) {
+ hexEscapeChar(os, c);
+ break;
+ }
+ // The header is valid, check data
+ // The next encBytes bytes must together be a valid utf-8
+ // This means: bitpattern 10XX XXXX and the extracted value is sane (ish)
+ bool valid = true;
+ uint32_t value = headerValue(c);
+ for (std::size_t n = 1; n < encBytes; ++n) {
+ uchar nc = m_str[idx + n];
+ valid &= ((nc & 0xC0) == 0x80);
+ value = (value << 6) | (nc & 0x3F);
+ }
+
+ if (
+ // Wrong bit pattern of following bytes
+ (!valid) ||
+ // Overlong encodings
+ (value < 0x80) ||
+ ( value < 0x800 && encBytes > 2) || // removed "0x80 <= value &&" because redundant
+ (0x800 < value && value < 0x10000 && encBytes > 3) ||
+ // Encoded value out of range
+ (value >= 0x110000)
+ ) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // If we got here, this is in fact a valid(ish) utf-8 sequence
+ for (std::size_t n = 0; n < encBytes; ++n) {
+ os << m_str[idx + n];
+ }
+ idx += encBytes - 1;
+ break;
+ }
+ }
+ }
+
+ std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode ) {
+ xmlEncode.encodeTo( os );
+ return os;
+ }
+
+ XmlWriter::ScopedElement::ScopedElement( XmlWriter* writer )
+ : m_writer( writer )
+ {}
+
+ XmlWriter::ScopedElement::ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT
+ : m_writer( other.m_writer ){
+ other.m_writer = nullptr;
+ }
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT {
+ if ( m_writer ) {
+ m_writer->endElement();
+ }
+ m_writer = other.m_writer;
+ other.m_writer = nullptr;
+ return *this;
+ }
+
+
+ XmlWriter::ScopedElement::~ScopedElement() {
+ if( m_writer )
+ m_writer->endElement();
+ }
+
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::writeText( std::string const& text, bool indent ) {
+ m_writer->writeText( text, indent );
+ return *this;
+ }
+
+ XmlWriter::XmlWriter( std::ostream& os ) : m_os( os )
+ {
+ writeDeclaration();
+ }
+
+ XmlWriter::~XmlWriter() {
+ while( !m_tags.empty() )
+ endElement();
+ }
+
+ XmlWriter& XmlWriter::startElement( std::string const& name ) {
+ ensureTagClosed();
+ newlineIfNecessary();
+ m_os << m_indent << '<' << name;
+ m_tags.push_back( name );
+ m_indent += " ";
+ m_tagIsOpen = true;
+ return *this;
+ }
+
+ XmlWriter::ScopedElement XmlWriter::scopedElement( std::string const& name ) {
+ ScopedElement scoped( this );
+ startElement( name );
+ return scoped;
+ }
+
+ XmlWriter& XmlWriter::endElement() {
+ newlineIfNecessary();
+ m_indent = m_indent.substr( 0, m_indent.size()-2 );
+ if( m_tagIsOpen ) {
+ m_os << "/>";
+ m_tagIsOpen = false;
+ }
+ else {
+ m_os << m_indent << "</" << m_tags.back() << ">";
+ }
+ m_os << std::endl;
+ m_tags.pop_back();
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, std::string const& attribute ) {
+ if( !name.empty() && !attribute.empty() )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, const char* attribute ) {
+ if( !name.empty() && attribute && attribute[0] != '\0' )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, bool attribute ) {
+ m_os << ' ' << name << "=\"" << ( attribute ? "true" : "false" ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeText( std::string const& text, bool indent ) {
+ if( !text.empty() ){
+ bool tagWasOpen = m_tagIsOpen;
+ ensureTagClosed();
+ if( tagWasOpen && indent )
+ m_os << m_indent;
+ m_os << XmlEncode( text );
+ m_needsNewline = true;
+ }
+ return *this;
+ }
+
+ //XmlWriter& XmlWriter::writeComment( std::string const& text ) {
+ // ensureTagClosed();
+ // m_os << m_indent << "<!--" << text << "-->";
+ // m_needsNewline = true;
+ // return *this;
+ //}
+
+ //void XmlWriter::writeStylesheetRef( std::string const& url ) {
+ // m_os << "<?xml-stylesheet type=\"text/xsl\" href=\"" << url << "\"?>\n";
+ //}
+
+ //XmlWriter& XmlWriter::writeBlankLine() {
+ // ensureTagClosed();
+ // m_os << '\n';
+ // return *this;
+ //}
+
+ void XmlWriter::ensureTagClosed() {
+ if( m_tagIsOpen ) {
+ m_os << ">" << std::endl;
+ m_tagIsOpen = false;
+ }
+ }
+
+ void XmlWriter::writeDeclaration() {
+ m_os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+ }
+
+ void XmlWriter::newlineIfNecessary() {
+ if( m_needsNewline ) {
+ m_os << std::endl;
+ m_needsNewline = false;
+ }
+ }
+
+// =================================================================================================
+// End of copy-pasted code from Catch
+// =================================================================================================
+
+ // clang-format on
+
+ struct XmlReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ XmlReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+ std::stringstream ss;
+ for(int i = 0; i < num_contexts; ++i) {
+ contexts[i]->stringify(&ss);
+ xml.scopedElement("Info").writeText(ss.str());
+ ss.str("");
+ }
+ }
+ }
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ void test_case_start_impl(const TestCaseData& in) {
+ bool open_ts_tag = false;
+ if(tc != nullptr) { // we have already opened a test suite
+ if(std::strcmp(tc->m_test_suite, in.m_test_suite) != 0) {
+ xml.endElement();
+ open_ts_tag = true;
+ }
+ }
+ else {
+ open_ts_tag = true; // first test case ==> first test suite
+ }
+
+ if(open_ts_tag) {
+ xml.startElement("TestSuite");
+ xml.writeAttribute("name", in.m_test_suite);
+ }
+
+ tc = &in;
+ xml.startElement("TestCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file.c_str()))
+ .writeAttribute("line", line(in.m_line))
+ .writeAttribute("description", in.m_description);
+
+ if(Approx(in.m_timeout) != 0)
+ xml.writeAttribute("timeout", in.m_timeout);
+ if(in.m_may_fail)
+ xml.writeAttribute("may_fail", true);
+ if(in.m_should_fail)
+ xml.writeAttribute("should_fail", true);
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ test_run_start();
+ if(opt.list_reporters) {
+ for(auto& curr : getListeners())
+ xml.scopedElement("Listener")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ for(auto& curr : getReporters())
+ xml.scopedElement("Reporter")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ } else if(opt.count || opt.list_test_cases) {
+ for(unsigned i = 0; i < in.num_data; ++i) {
+ xml.scopedElement("TestCase").writeAttribute("name", in.data[i]->m_name)
+ .writeAttribute("testsuite", in.data[i]->m_test_suite)
+ .writeAttribute("filename", skipPathFromFilename(in.data[i]->m_file.c_str()))
+ .writeAttribute("line", line(in.data[i]->m_line));
+ }
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ } else if(opt.list_test_suites) {
+ for(unsigned i = 0; i < in.num_data; ++i)
+ xml.scopedElement("TestSuite").writeAttribute("name", in.data[i]->m_test_suite);
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ xml.scopedElement("OverallResultsTestSuites")
+ .writeAttribute("unskipped", in.run_stats->numTestSuitesPassingFilters);
+ }
+ xml.endElement();
+ }
+
+ void test_run_start() override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ xml.startElement("doctest").writeAttribute("binary", binary_name);
+ if(opt.no_version == false)
+ xml.writeAttribute("version", DOCTEST_VERSION_STR);
+
+ // only the consequential ones (TODO: filters)
+ xml.scopedElement("Options")
+ .writeAttribute("order_by", opt.order_by.c_str())
+ .writeAttribute("rand_seed", opt.rand_seed)
+ .writeAttribute("first", opt.first)
+ .writeAttribute("last", opt.last)
+ .writeAttribute("abort_after", opt.abort_after)
+ .writeAttribute("subcase_filter_levels", opt.subcase_filter_levels)
+ .writeAttribute("case_sensitive", opt.case_sensitive)
+ .writeAttribute("no_throw", opt.no_throw)
+ .writeAttribute("no_skip", opt.no_skip);
+ }
+
+ void test_run_end(const TestRunStats& p) override {
+ if(tc) // the TestSuite tag - only if there has been at least 1 test case
+ xml.endElement();
+
+ xml.scopedElement("OverallResultsAsserts")
+ .writeAttribute("successes", p.numAsserts - p.numAssertsFailed)
+ .writeAttribute("failures", p.numAssertsFailed);
+
+ xml.startElement("OverallResultsTestCases")
+ .writeAttribute("successes",
+ p.numTestCasesPassingFilters - p.numTestCasesFailed)
+ .writeAttribute("failures", p.numTestCasesFailed);
+ if(opt.no_skipped_summary == false)
+ xml.writeAttribute("skipped", p.numTestCases - p.numTestCasesPassingFilters);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ test_case_start_impl(in);
+ xml.ensureTagClosed();
+ }
+
+ void test_case_reenter(const TestCaseData&) override {}
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ xml.startElement("OverallResultsAsserts")
+ .writeAttribute("successes",
+ st.numAssertsCurrentTest - st.numAssertsFailedCurrentTest)
+ .writeAttribute("failures", st.numAssertsFailedCurrentTest);
+ if(opt.duration)
+ xml.writeAttribute("duration", st.seconds);
+ if(tc->m_expected_failures)
+ xml.writeAttribute("expected_failures", tc->m_expected_failures);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.scopedElement("Exception")
+ .writeAttribute("crash", e.is_crash)
+ .writeText(e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("SubCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file))
+ .writeAttribute("line", line(in.m_line));
+ xml.ensureTagClosed();
+ }
+
+ void subcase_end() override { xml.endElement(); }
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed && !opt.success)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Expression")
+ .writeAttribute("success", !rb.m_failed)
+ .writeAttribute("type", assertString(rb.m_at))
+ .writeAttribute("filename", skipPathFromFilename(rb.m_file))
+ .writeAttribute("line", line(rb.m_line));
+
+ xml.scopedElement("Original").writeText(rb.m_expr);
+
+ if(rb.m_threw)
+ xml.scopedElement("Exception").writeText(rb.m_exception.c_str());
+
+ if(rb.m_at & assertType::is_throws_as)
+ xml.scopedElement("ExpectedException").writeText(rb.m_exception_type);
+ if(rb.m_at & assertType::is_throws_with)
+ xml.scopedElement("ExpectedExceptionString").writeText(rb.m_exception_string);
+ if((rb.m_at & assertType::is_normal) && !rb.m_threw)
+ xml.scopedElement("Expanded").writeText(rb.m_decomp.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void log_message(const MessageData& mb) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Message")
+ .writeAttribute("type", failureString(mb.m_severity))
+ .writeAttribute("filename", skipPathFromFilename(mb.m_file))
+ .writeAttribute("line", line(mb.m_line));
+
+ xml.scopedElement("Text").writeText(mb.m_string.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void test_case_skipped(const TestCaseData& in) override {
+ if(opt.no_skipped_summary == false) {
+ test_case_start_impl(in);
+ xml.writeAttribute("skipped", "true");
+ xml.endElement();
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("xml", 0, XmlReporter);
+
+ void fulltext_log_assert_to_stream(std::ostream& s, const AssertData& rb) {
+ if((rb.m_at & (assertType::is_throws_as | assertType::is_throws_with)) ==
+ 0) //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << " ) "
+ << Color::None;
+
+ if(rb.m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "threw as expected!" : "did NOT throw at all!") << "\n";
+ } else if((rb.m_at & assertType::is_throws_as) &&
+ (rb.m_at & assertType::is_throws_with)) { //!OCLINT
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\", " << rb.m_exception_type << " ) " << Color::None;
+ if(rb.m_threw) {
+ if(!rb.m_failed) {
+ s << "threw as expected!\n";
+ } else {
+ s << "threw a DIFFERENT exception! (contents: " << rb.m_exception << ")\n";
+ }
+ } else {
+ s << "did NOT throw at all!\n";
+ }
+ } else if(rb.m_at &
+ assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", "
+ << rb.m_exception_type << " ) " << Color::None
+ << (rb.m_threw ? (rb.m_threw_as ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at &
+ assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\" ) " << Color::None
+ << (rb.m_threw ? (!rb.m_failed ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "THREW exception: " : "didn't throw!") << Color::Cyan
+ << rb.m_exception << "\n";
+ } else {
+ s << (rb.m_threw ? "THREW exception: " :
+ (!rb.m_failed ? "is correct!\n" : "is NOT correct!\n"));
+ if(rb.m_threw)
+ s << rb.m_exception << "\n";
+ else
+ s << " values: " << assertString(rb.m_at) << "( " << rb.m_decomp << " )\n";
+ }
+ }
+
+ // TODO:
+ // - log_message()
+ // - respond to queries
+ // - honor remaining options
+ // - more attributes in tags
+ struct JUnitReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+ Timer timer;
+ std::vector<String> deepestSubcaseStackNames;
+
+ struct JUnitTestCaseData
+ {
+ static std::string getCurrentTimestamp() {
+ // Beware, this is not reentrant because of backward compatibility issues
+ // Also, UTC only, again because of backward compatibility (%z is C++11)
+ time_t rawtime;
+ std::time(&rawtime);
+ auto const timeStampSize = sizeof("2017-01-16T17:06:45Z");
+
+ std::tm timeInfo;
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ gmtime_s(&timeInfo, &rawtime);
+#else // DOCTEST_PLATFORM_WINDOWS
+ gmtime_r(&rawtime, &timeInfo);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ char timeStamp[timeStampSize];
+ const char* const fmt = "%Y-%m-%dT%H:%M:%SZ";
+
+ std::strftime(timeStamp, timeStampSize, fmt, &timeInfo);
+ return std::string(timeStamp);
+ }
+
+ struct JUnitTestMessage
+ {
+ JUnitTestMessage(const std::string& _message, const std::string& _type, const std::string& _details)
+ : message(_message), type(_type), details(_details) {}
+
+ JUnitTestMessage(const std::string& _message, const std::string& _details)
+ : message(_message), type(), details(_details) {}
+
+ std::string message, type, details;
+ };
+
+ struct JUnitTestCase
+ {
+ JUnitTestCase(const std::string& _classname, const std::string& _name)
+ : classname(_classname), name(_name), time(0), failures() {}
+
+ std::string classname, name;
+ double time;
+ std::vector<JUnitTestMessage> failures, errors;
+ };
+
+ void add(const std::string& classname, const std::string& name) {
+ testcases.emplace_back(classname, name);
+ }
+
+ void appendSubcaseNamesToLastTestcase(std::vector<String> nameStack) {
+ for(auto& curr: nameStack)
+ if(curr.size())
+ testcases.back().name += std::string("/") + curr.c_str();
+ }
+
+ void addTime(double time) {
+ if(time < 1e-4)
+ time = 0;
+ testcases.back().time = time;
+ totalSeconds += time;
+ }
+
+ void addFailure(const std::string& message, const std::string& type, const std::string& details) {
+ testcases.back().failures.emplace_back(message, type, details);
+ ++totalFailures;
+ }
+
+ void addError(const std::string& message, const std::string& details) {
+ testcases.back().errors.emplace_back(message, details);
+ ++totalErrors;
+ }
+
+ std::vector<JUnitTestCase> testcases;
+ double totalSeconds = 0;
+ int totalErrors = 0, totalFailures = 0;
+ };
+
+ JUnitTestCaseData testCaseData;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ JUnitReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData&) override {}
+
+ void test_run_start() override {}
+
+ void test_run_end(const TestRunStats& p) override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+ xml.startElement("testsuites");
+ xml.startElement("testsuite").writeAttribute("name", binary_name)
+ .writeAttribute("errors", testCaseData.totalErrors)
+ .writeAttribute("failures", testCaseData.totalFailures)
+ .writeAttribute("tests", p.numAsserts);
+ if(opt.no_time_in_output == false) {
+ xml.writeAttribute("time", testCaseData.totalSeconds);
+ xml.writeAttribute("timestamp", JUnitTestCaseData::getCurrentTimestamp());
+ }
+ if(opt.no_version == false)
+ xml.writeAttribute("doctest_version", DOCTEST_VERSION_STR);
+
+ for(const auto& testCase : testCaseData.testcases) {
+ xml.startElement("testcase")
+ .writeAttribute("classname", testCase.classname)
+ .writeAttribute("name", testCase.name);
+ if(opt.no_time_in_output == false)
+ xml.writeAttribute("time", testCase.time);
+ // This is not ideal, but it should be enough to mimic gtest's junit output.
+ xml.writeAttribute("status", "run");
+
+ for(const auto& failure : testCase.failures) {
+ xml.scopedElement("failure")
+ .writeAttribute("message", failure.message)
+ .writeAttribute("type", failure.type)
+ .writeText(failure.details, false);
+ }
+
+ for(const auto& error : testCase.errors) {
+ xml.scopedElement("error")
+ .writeAttribute("message", error.message)
+ .writeText(error.details);
+ }
+
+ xml.endElement();
+ }
+ xml.endElement();
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ timer.start();
+ }
+
+ void test_case_reenter(const TestCaseData& in) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+
+ timer.start();
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ }
+
+ void test_case_end(const CurrentTestCaseStats&) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ testCaseData.addError("exception", e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ deepestSubcaseStackNames.push_back(in.m_name);
+ }
+
+ void subcase_end() override {}
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed) // report only failures & ignore the `success` option
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ std::ostringstream os;
+ os << skipPathFromFilename(rb.m_file) << (opt.gnu_file_line ? ":" : "(")
+ << line(rb.m_line) << (opt.gnu_file_line ? ":" : "):") << std::endl;
+
+ fulltext_log_assert_to_stream(os, rb);
+ log_contexts(os);
+ testCaseData.addFailure(rb.m_decomp.c_str(), assertString(rb.m_at), os.str());
+ }
+
+ void log_message(const MessageData&) override {}
+
+ void test_case_skipped(const TestCaseData&) override {}
+
+ void log_contexts(std::ostringstream& s) {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << std::endl;
+ }
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("junit", 0, JUnitReporter);
+
+ struct Whitespace
+ {
+ int nrSpaces;
+ explicit Whitespace(int nr)
+ : nrSpaces(nr) {}
+ };
+
+ std::ostream& operator<<(std::ostream& out, const Whitespace& ws) {
+ if(ws.nrSpaces != 0)
+ out << std::setw(ws.nrSpaces) << ' ';
+ return out;
+ }
+
+ struct ConsoleReporter : public IReporter
+ {
+ std::ostream& s;
+ bool hasLoggedCurrentTestStart;
+ std::vector<SubcaseSignature> subcasesStack;
+ size_t currentSubcaseLevel;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc;
+
+ ConsoleReporter(const ContextOptions& co)
+ : s(*co.cout)
+ , opt(co) {}
+
+ ConsoleReporter(const ContextOptions& co, std::ostream& ostr)
+ : s(ostr)
+ , opt(co) {}
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE HELPERS USED BY THE OVERRIDES OF THE VIRTUAL METHODS OF THE INTERFACE
+ // =========================================================================================
+
+ void separator_to_stream() {
+ s << Color::Yellow
+ << "==============================================================================="
+ "\n";
+ }
+
+ const char* getSuccessOrFailString(bool success, assertType::Enum at,
+ const char* success_str) {
+ if(success)
+ return success_str;
+ return failureString(at);
+ }
+
+ Color::Enum getSuccessOrFailColor(bool success, assertType::Enum at) {
+ return success ? Color::BrightGreen :
+ (at & assertType::is_warn) ? Color::Yellow : Color::Red;
+ }
+
+ void successOrFailColoredStringToStream(bool success, assertType::Enum at,
+ const char* success_str = "SUCCESS") {
+ s << getSuccessOrFailColor(success, at)
+ << getSuccessOrFailString(success, at, success_str) << ": ";
+ }
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << Color::None << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << "\n";
+ }
+ }
+
+ s << "\n";
+ }
+
+ // this was requested to be made virtual so users could override it
+ virtual void file_line_to_stream(const char* file, int line,
+ const char* tail = "") {
+ s << Color::LightGrey << skipPathFromFilename(file) << (opt.gnu_file_line ? ":" : "(")
+ << (opt.no_line_numbers ? 0 : line) // 0 or the real num depending on the option
+ << (opt.gnu_file_line ? ":" : "):") << tail;
+ }
+
+ void logTestStart() {
+ if(hasLoggedCurrentTestStart)
+ return;
+
+ separator_to_stream();
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, "\n");
+ if(tc->m_description)
+ s << Color::Yellow << "DESCRIPTION: " << Color::None << tc->m_description << "\n";
+ if(tc->m_test_suite && tc->m_test_suite[0] != '\0')
+ s << Color::Yellow << "TEST SUITE: " << Color::None << tc->m_test_suite << "\n";
+ if(strncmp(tc->m_name, " Scenario:", 11) != 0)
+ s << Color::Yellow << "TEST CASE: ";
+ s << Color::None << tc->m_name << "\n";
+
+ for(size_t i = 0; i < currentSubcaseLevel; ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+
+ if(currentSubcaseLevel != subcasesStack.size()) {
+ s << Color::Yellow << "\nDEEPEST SUBCASE STACK REACHED (DIFFERENT FROM THE CURRENT ONE):\n" << Color::None;
+ for(size_t i = 0; i < subcasesStack.size(); ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+ }
+
+ s << "\n";
+
+ hasLoggedCurrentTestStart = true;
+ }
+
+ void printVersion() {
+ if(opt.no_version == false)
+ s << Color::Cyan << "[doctest] " << Color::None << "doctest version is \""
+ << DOCTEST_VERSION_STR << "\"\n";
+ }
+
+ void printIntro() {
+ printVersion();
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "run with \"--" DOCTEST_OPTIONS_PREFIX_DISPLAY "help\" for options\n";
+ }
+
+ void printHelp() {
+ int sizePrefixDisplay = static_cast<int>(strlen(DOCTEST_OPTIONS_PREFIX_DISPLAY));
+ printVersion();
+ // clang-format off
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "boolean values: \"1/on/yes/true\" or \"0/off/no/false\"\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filter values: \"str1,str2,str3\" (comma separated strings)\n";
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filters use wildcards for matching strings\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "something passes a filter if any of the strings in a filter matches\n";
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "ALL FLAGS, OPTIONS AND FILTERS ALSO AVAILABLE WITH A \"" DOCTEST_CONFIG_OPTIONS_PREFIX "\" PREFIX!!!\n";
+#endif
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "Query flags - the program quits after them. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "?, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "help, -" DOCTEST_OPTIONS_PREFIX_DISPLAY "h "
+ << Whitespace(sizePrefixDisplay*0) << "prints this message\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "v, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "version "
+ << Whitespace(sizePrefixDisplay*1) << "prints the version\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "c, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "count "
+ << Whitespace(sizePrefixDisplay*1) << "prints the number of matching tests\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ltc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-cases "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching tests by name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-suites "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching test suites\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-reporters "
+ << Whitespace(sizePrefixDisplay*1) << "lists all registered reporters\n\n";
+ // ================================================================================== << 79
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "The available <int>/<string> options/filters are:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sfe, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tse, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "r, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "reporters=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "reporters to use (console is default)\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "o, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "out=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "output filename\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ob, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "order-by=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "how the tests should be ordered\n";
+ s << Whitespace(sizePrefixDisplay*3) << " <string> - [file/suite/name/rand/none]\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "rs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "rand-seed=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "seed for random ordering\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "f, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "first=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the first test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "l, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "last=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the last test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "aa, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "abort-after=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "stop after <int> failed assertions\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "scfl,--" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-filter-levels=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "apply filters for the first <int> levels\n";
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "Bool options - can be used like flags and true is assumed. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "s, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "success=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "include successful assertions in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "cs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "case-sensitive=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "filters being treated as case sensitive\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "e, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "exit=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "exits after the tests finish\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "d, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "duration=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "prints the time duration of each test\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nt, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-throw=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips exceptions-related assert checks\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ne, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-exitcode=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "returns (or exits) always with success\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-run=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips all runtime doctest operations\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nv, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-version=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "omit the framework version in the output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables colors in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "fc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "force-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "use colors even when not in a tty\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nb, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-breaks=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables breakpoints in debuggers\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ns, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-skip=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "don't skip test cases marked as skip\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "gfl, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "gnu-file-line=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << ":n: vs (n): for line numbers in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "npf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-path-filenames=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "only filenames and no paths in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nln, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-line-numbers=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "0 instead of real line numbers in output\n";
+ // ================================================================================== << 79
+ // clang-format on
+
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "for more information visit the project documentation\n\n";
+ }
+
+ void printRegisteredReporters() {
+ printVersion();
+ auto printReporters = [this] (const reporterMap& reporters, const char* type) {
+ if(reporters.size()) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all registered " << type << "\n";
+ for(auto& curr : reporters)
+ s << "priority: " << std::setw(5) << curr.first.first
+ << " name: " << curr.first.second << "\n";
+ }
+ };
+ printReporters(getListeners(), "listeners");
+ printReporters(getReporters(), "reporters");
+ }
+
+ void list_query_results() {
+ separator_to_stream();
+ if(opt.count || opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ if(opt.version) {
+ printVersion();
+ } else if(opt.help) {
+ printHelp();
+ } else if(opt.list_reporters) {
+ printRegisteredReporters();
+ } else if(opt.count || opt.list_test_cases) {
+ if(opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "listing all test case names\n";
+ separator_to_stream();
+ }
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_name << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all test suites\n";
+ separator_to_stream();
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_test_suite << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ void test_run_start() override { printIntro(); }
+
+ void test_run_end(const TestRunStats& p) override {
+ separator_to_stream();
+ s << std::dec;
+
+ auto totwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters, static_cast<unsigned>(p.numAsserts))) + 1)));
+ auto passwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters - p.numTestCasesFailed, static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) + 1)));
+ auto failwidth = int(std::ceil(log10((std::max(p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) + 1)));
+ const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
+ s << Color::Cyan << "[doctest] " << Color::None << "test cases: " << std::setw(totwidth)
+ << p.numTestCasesPassingFilters << " | "
+ << ((p.numTestCasesPassingFilters == 0 || anythingFailed) ? Color::None :
+ Color::Green)
+ << std::setw(passwidth) << p.numTestCasesPassingFilters - p.numTestCasesFailed << " passed"
+ << Color::None << " | " << (p.numTestCasesFailed > 0 ? Color::Red : Color::None)
+ << std::setw(failwidth) << p.numTestCasesFailed << " failed" << Color::None << " |";
+ if(opt.no_skipped_summary == false) {
+ const int numSkipped = p.numTestCases - p.numTestCasesPassingFilters;
+ s << " " << (numSkipped == 0 ? Color::None : Color::Yellow) << numSkipped
+ << " skipped" << Color::None;
+ }
+ s << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None << "assertions: " << std::setw(totwidth)
+ << p.numAsserts << " | "
+ << ((p.numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
+ << std::setw(passwidth) << (p.numAsserts - p.numAssertsFailed) << " passed" << Color::None
+ << " | " << (p.numAssertsFailed > 0 ? Color::Red : Color::None) << std::setw(failwidth)
+ << p.numAssertsFailed << " failed" << Color::None << " |\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "Status: " << (p.numTestCasesFailed > 0 ? Color::Red : Color::Green)
+ << ((p.numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ hasLoggedCurrentTestStart = false;
+ tc = &in;
+ subcasesStack.clear();
+ currentSubcaseLevel = 0;
+ }
+
+ void test_case_reenter(const TestCaseData&) override {
+ subcasesStack.clear();
+ }
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ if(tc->m_no_output)
+ return;
+
+ // log the preamble of the test case only if there is something
+ // else to print - something other than that an assert has failed
+ if(opt.duration ||
+ (st.failure_flags && st.failure_flags != TestCaseFailureReason::AssertFailure))
+ logTestStart();
+
+ if(opt.duration)
+ s << Color::None << std::setprecision(6) << std::fixed << st.seconds
+ << " s: " << tc->m_name << "\n";
+
+ if(st.failure_flags & TestCaseFailureReason::Timeout)
+ s << Color::Red << "Test case exceeded time limit of " << std::setprecision(6)
+ << std::fixed << tc->m_timeout << "!\n";
+
+ if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedButDidnt) {
+ s << Color::Red << "Should have failed but didn't! Marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedAndDid) {
+ s << Color::Yellow << "Failed as expected so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::CouldHaveFailedAndDid) {
+ s << Color::Yellow << "Allowed to fail so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::DidntFailExactlyNumTimes) {
+ s << Color::Red << "Didn't fail exactly " << tc->m_expected_failures
+ << " times so marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::FailedExactlyNumTimes) {
+ s << Color::Yellow << "Failed exactly " << tc->m_expected_failures
+ << " times as expected so marking it as not failed!\n";
+ }
+ if(st.failure_flags & TestCaseFailureReason::TooManyFailedAsserts) {
+ s << Color::Red << "Aborting - too many failed asserts!\n";
+ }
+ s << Color::None; // lgtm [cpp/useless-expression]
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ if(tc->m_no_output)
+ return;
+
+ logTestStart();
+
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, " ");
+ successOrFailColoredStringToStream(false, e.is_crash ? assertType::is_require :
+ assertType::is_check);
+ s << Color::Red << (e.is_crash ? "test case CRASHED: " : "test case THREW exception: ")
+ << Color::Cyan << e.error_string << "\n";
+
+ int num_stringified_contexts = get_num_stringified_contexts();
+ if(num_stringified_contexts) {
+ auto stringified_contexts = get_stringified_contexts();
+ s << Color::None << " logged: ";
+ for(int i = num_stringified_contexts; i > 0; --i) {
+ s << (i == num_stringified_contexts ? "" : " ")
+ << stringified_contexts[i - 1] << "\n";
+ }
+ }
+ s << "\n" << Color::None;
+ }
+
+ void subcase_start(const SubcaseSignature& subc) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ subcasesStack.push_back(subc);
+ ++currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void subcase_end() override {
+ std::lock_guard<std::mutex> lock(mutex);
+ --currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void log_assert(const AssertData& rb) override {
+ if((!rb.m_failed && !opt.success) || tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(rb.m_file, rb.m_line, " ");
+ successOrFailColoredStringToStream(!rb.m_failed, rb.m_at);
+
+ fulltext_log_assert_to_stream(s, rb);
+
+ log_contexts();
+ }
+
+ void log_message(const MessageData& mb) override {
+ if(tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(mb.m_file, mb.m_line, " ");
+ s << getSuccessOrFailColor(false, mb.m_severity)
+ << getSuccessOrFailString(mb.m_severity & assertType::is_warn, mb.m_severity,
+ "MESSAGE") << ": ";
+ s << Color::None << mb.m_string << "\n";
+ log_contexts();
+ }
+
+ void test_case_skipped(const TestCaseData&) override {}
+ };
+
+ DOCTEST_REGISTER_REPORTER("console", 0, ConsoleReporter);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ struct DebugOutputWindowReporter : public ConsoleReporter
+ {
+ DOCTEST_THREAD_LOCAL static std::ostringstream oss;
+
+ DebugOutputWindowReporter(const ContextOptions& co)
+ : ConsoleReporter(co, oss) {}
+
+#define DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(func, type, arg) \
+ void func(type arg) override { \
+ bool with_col = g_no_colors; \
+ g_no_colors = false; \
+ ConsoleReporter::func(arg); \
+ if(oss.tellp() != std::streampos{}) { \
+ DOCTEST_OUTPUT_DEBUG_STRING(oss.str().c_str()); \
+ oss.str(""); \
+ } \
+ g_no_colors = with_col; \
+ }
+
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_start, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_end, const TestRunStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_start, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_reenter, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_end, const CurrentTestCaseStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_exception, const TestCaseException&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_start, const SubcaseSignature&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_end, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_assert, const AssertData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_message, const MessageData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_skipped, const TestCaseData&, in)
+ };
+
+ DOCTEST_THREAD_LOCAL std::ostringstream DebugOutputWindowReporter::oss;
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // the implementation of parseOption()
+ bool parseOptionImpl(int argc, const char* const* argv, const char* pattern, String* value) {
+ // going from the end to the beginning and stopping on the first occurrence from the end
+ for(int i = argc; i > 0; --i) {
+ auto index = i - 1;
+ auto temp = std::strstr(argv[index], pattern);
+ if(temp && (value || strlen(temp) == strlen(pattern))) { //!OCLINT prefer early exits and continue
+ // eliminate matches in which the chars before the option are not '-'
+ bool noBadCharsFound = true;
+ auto curr = argv[index];
+ while(curr != temp) {
+ if(*curr++ != '-') {
+ noBadCharsFound = false;
+ break;
+ }
+ }
+ if(noBadCharsFound && argv[index][0] == '-') {
+ if(value) {
+ // parsing the value of an option
+ temp += strlen(pattern);
+ const unsigned len = strlen(temp);
+ if(len) {
+ *value = temp;
+ return true;
+ }
+ } else {
+ // just a flag - no value
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ // parses an option and returns the string after the '=' character
+ bool parseOption(int argc, const char* const* argv, const char* pattern, String* value = nullptr,
+ const String& defaultVal = String()) {
+ if(value)
+ *value = defaultVal;
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ // offset (normally 3 for "dt-") to skip prefix
+ if(parseOptionImpl(argc, argv, pattern + strlen(DOCTEST_CONFIG_OPTIONS_PREFIX), value))
+ return true;
+#endif // DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ return parseOptionImpl(argc, argv, pattern, value);
+ }
+
+ // locates a flag on the command line
+ bool parseFlag(int argc, const char* const* argv, const char* pattern) {
+ return parseOption(argc, argv, pattern);
+ }
+
+ // parses a comma separated list of words after a pattern in one of the arguments in argv
+ bool parseCommaSepArgs(int argc, const char* const* argv, const char* pattern,
+ std::vector<String>& res) {
+ String filtersString;
+ if(parseOption(argc, argv, pattern, &filtersString)) {
+ // tokenize with "," as a separator
+ // cppcheck-suppress strtokCalled
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ auto pch = std::strtok(filtersString.c_str(), ","); // modifies the string
+ while(pch != nullptr) {
+ if(strlen(pch))
+ res.push_back(pch);
+ // uses the strtok() internal state to go to the next token
+ // cppcheck-suppress strtokCalled
+ pch = std::strtok(nullptr, ",");
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ return true;
+ }
+ return false;
+ }
+
+ enum optionType
+ {
+ option_bool,
+ option_int
+ };
+
+ // parses an int/bool option from the command line
+ bool parseIntOption(int argc, const char* const* argv, const char* pattern, optionType type,
+ int& res) {
+ String parsedValue;
+ if(!parseOption(argc, argv, pattern, &parsedValue))
+ return false;
+
+ if(type == 0) {
+ // boolean
+ const char positive[][5] = {"1", "true", "on", "yes"}; // 5 - strlen("true") + 1
+ const char negative[][6] = {"0", "false", "off", "no"}; // 6 - strlen("false") + 1
+
+ // if the value matches any of the positive/negative possibilities
+ for(unsigned i = 0; i < 4; i++) {
+ if(parsedValue.compare(positive[i], true) == 0) {
+ res = 1; //!OCLINT parameter reassignment
+ return true;
+ }
+ if(parsedValue.compare(negative[i], true) == 0) {
+ res = 0; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ } else {
+ // integer
+ // TODO: change this to use std::stoi or something else! currently it uses undefined behavior - assumes '0' on failed parse...
+ int theInt = std::atoi(parsedValue.c_str()); // NOLINT
+ if(theInt != 0) {
+ res = theInt; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ return false;
+ }
+} // namespace
+
+Context::Context(int argc, const char* const* argv)
+ : p(new detail::ContextState) {
+ parseArgs(argc, argv, true);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+Context::~Context() {
+ if(g_cs == p)
+ g_cs = nullptr;
+ delete p;
+}
+
+void Context::applyCommandLine(int argc, const char* const* argv) {
+ parseArgs(argc, argv);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+// parses args
+void Context::parseArgs(int argc, const char* const* argv, bool withDefaults) {
+ using namespace detail;
+
+ // clang-format off
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sf=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file-exclude=",p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sfe=", p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ts=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite-exclude=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tse=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tc=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case-exclude=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tce=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sc=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase-exclude=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sce=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "reporters=", p->filters[8]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "r=", p->filters[8]);
+ // clang-format on
+
+ int intRes = 0;
+ String strRes;
+
+#define DOCTEST_PARSE_AS_BOOL_OR_FLAG(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_bool, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_bool, intRes)) \
+ p->var = static_cast<bool>(intRes); \
+ else if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name) || \
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname)) \
+ p->var = true; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_INT_OPTION(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_int, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_int, intRes)) \
+ p->var = intRes; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_STR_OPTION(name, sname, var, default) \
+ if(parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", &strRes, default) || \
+ parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", &strRes, default) || \
+ withDefaults) \
+ p->var = strRes
+
+ // clang-format off
+ DOCTEST_PARSE_STR_OPTION("out", "o", out, "");
+ DOCTEST_PARSE_STR_OPTION("order-by", "ob", order_by, "file");
+ DOCTEST_PARSE_INT_OPTION("rand-seed", "rs", rand_seed, 0);
+
+ DOCTEST_PARSE_INT_OPTION("first", "f", first, 0);
+ DOCTEST_PARSE_INT_OPTION("last", "l", last, UINT_MAX);
+
+ DOCTEST_PARSE_INT_OPTION("abort-after", "aa", abort_after, 0);
+ DOCTEST_PARSE_INT_OPTION("subcase-filter-levels", "scfl", subcase_filter_levels, INT_MAX);
+
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("success", "s", success, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("case-sensitive", "cs", case_sensitive, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("exit", "e", exit, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("duration", "d", duration, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-throw", "nt", no_throw, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-exitcode", "ne", no_exitcode, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-run", "nr", no_run, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-version", "nv", no_version, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-colors", "nc", no_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("force-colors", "fc", force_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-breaks", "nb", no_breaks, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skip", "ns", no_skip, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("gnu-file-line", "gfl", gnu_file_line, !bool(DOCTEST_MSVC));
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-path-filenames", "npf", no_path_in_filenames, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-line-numbers", "nln", no_line_numbers, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-debug-output", "ndo", no_debug_output, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skipped-summary", "nss", no_skipped_summary, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-time-in-output", "ntio", no_time_in_output, false);
+ // clang-format on
+
+ if(withDefaults) {
+ p->help = false;
+ p->version = false;
+ p->count = false;
+ p->list_test_cases = false;
+ p->list_test_suites = false;
+ p->list_reporters = false;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "help") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "h") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "?")) {
+ p->help = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "version") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "v")) {
+ p->version = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "count") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "c")) {
+ p->count = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-cases") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ltc")) {
+ p->list_test_cases = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-suites") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lts")) {
+ p->list_test_suites = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-reporters") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lr")) {
+ p->list_reporters = true;
+ p->exit = true;
+ }
+}
+
+// allows the user to add procedurally to the filters from the command line
+void Context::addFilter(const char* filter, const char* value) { setOption(filter, value); }
+
+// allows the user to clear all filters from the command line
+void Context::clearFilters() {
+ for(auto& curr : p->filters)
+ curr.clear();
+}
+
+// allows the user to override procedurally the int/bool options from the command line
+void Context::setOption(const char* option, int value) {
+ setOption(option, toString(value).c_str());
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+// allows the user to override procedurally the string options from the command line
+void Context::setOption(const char* option, const char* value) {
+ auto argv = String("-") + option + "=" + value;
+ auto lvalue = argv.c_str();
+ parseArgs(1, &lvalue);
+}
+
+// users should query this in their main() and exit the program if true
+bool Context::shouldExit() { return p->exit; }
+
+void Context::setAsDefaultForAssertsOutOfTestCases() { g_cs = p; }
+
+void Context::setAssertHandler(detail::assert_handler ah) { p->ah = ah; }
+
+// the main function that does all the filtering and test running
+int Context::run() {
+ using namespace detail;
+
+ // save the old context state in case such was setup - for using asserts out of a testing context
+ auto old_cs = g_cs;
+ // this is the current contest
+ g_cs = p;
+ is_running_in_test = true;
+
+ g_no_colors = p->no_colors;
+ p->resetRunData();
+
+ // stdout by default
+ p->cout = &std::cout;
+ p->cerr = &std::cerr;
+
+ // or to a file if specified
+ std::fstream fstr;
+ if(p->out.size()) {
+ fstr.open(p->out.c_str(), std::fstream::out);
+ p->cout = &fstr;
+ }
+
+ FatalConditionHandler::allocateAltStackMem();
+
+ auto cleanup_and_return = [&]() {
+ FatalConditionHandler::freeAltStackMem();
+
+ if(fstr.is_open())
+ fstr.close();
+
+ // restore context
+ g_cs = old_cs;
+ is_running_in_test = false;
+
+ // we have to free the reporters which were allocated when the run started
+ for(auto& curr : p->reporters_currently_used)
+ delete curr;
+ p->reporters_currently_used.clear();
+
+ if(p->numTestCasesFailed && !p->no_exitcode)
+ return EXIT_FAILURE;
+ return EXIT_SUCCESS;
+ };
+
+ // setup default reporter if none is given through the command line
+ if(p->filters[8].empty())
+ p->filters[8].push_back("console");
+
+ // check to see if any of the registered reporters has been selected
+ for(auto& curr : getReporters()) {
+ if(matchesAny(curr.first.second.c_str(), p->filters[8], false, p->case_sensitive))
+ p->reporters_currently_used.push_back(curr.second(*g_cs));
+ }
+
+ // TODO: check if there is nothing in reporters_currently_used
+
+ // prepend all listeners
+ for(auto& curr : getListeners())
+ p->reporters_currently_used.insert(p->reporters_currently_used.begin(), curr.second(*g_cs));
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(isDebuggerActive() && p->no_debug_output == false)
+ p->reporters_currently_used.push_back(new DebugOutputWindowReporter(*g_cs));
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // handle version, help and no_run
+ if(p->no_run || p->version || p->help || p->list_reporters) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, QueryData());
+
+ return cleanup_and_return();
+ }
+
+ std::vector<const TestCase*> testArray;
+ for(auto& curr : getRegisteredTests())
+ testArray.push_back(&curr);
+ p->numTestCases = testArray.size();
+
+ // sort the collected records
+ if(!testArray.empty()) {
+ if(p->order_by.compare("file", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), fileOrderComparator);
+ } else if(p->order_by.compare("suite", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), suiteOrderComparator);
+ } else if(p->order_by.compare("name", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), nameOrderComparator);
+ } else if(p->order_by.compare("rand", true) == 0) {
+ std::srand(p->rand_seed);
+
+ // random_shuffle implementation
+ const auto first = &testArray[0];
+ for(size_t i = testArray.size() - 1; i > 0; --i) {
+ int idxToSwap = std::rand() % (i + 1); // NOLINT
+
+ const auto temp = first[i];
+
+ first[i] = first[idxToSwap];
+ first[idxToSwap] = temp;
+ }
+ } else if(p->order_by.compare("none", true) == 0) {
+ // means no sorting - beneficial for death tests which call into the executable
+ // with a specific test case in mind - we don't want to slow down the startup times
+ }
+ }
+
+ std::set<String> testSuitesPassingFilt;
+
+ bool query_mode = p->count || p->list_test_cases || p->list_test_suites;
+ std::vector<const TestCaseData*> queryResults;
+
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_start, DOCTEST_EMPTY);
+
+ // invoke the registered functions if they match the filter criteria (or just count them)
+ for(auto& curr : testArray) {
+ const auto& tc = *curr;
+
+ bool skip_me = false;
+ if(tc.m_skip && !p->no_skip)
+ skip_me = true;
+
+ if(!matchesAny(tc.m_file.c_str(), p->filters[0], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_file.c_str(), p->filters[1], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_test_suite, p->filters[2], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_test_suite, p->filters[3], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_name, p->filters[4], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_name, p->filters[5], false, p->case_sensitive))
+ skip_me = true;
+
+ if(!skip_me)
+ p->numTestCasesPassingFilters++;
+
+ // skip the test if it is not in the execution range
+ if((p->last < p->numTestCasesPassingFilters && p->first <= p->last) ||
+ (p->first > p->numTestCasesPassingFilters))
+ skip_me = true;
+
+ if(skip_me) {
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_skipped, tc);
+ continue;
+ }
+
+ // do not execute the test if we are to only count the number of filter passing tests
+ if(p->count)
+ continue;
+
+ // print the name of the test and don't execute it
+ if(p->list_test_cases) {
+ queryResults.push_back(&tc);
+ continue;
+ }
+
+ // print the name of the test suite if not done already and don't execute it
+ if(p->list_test_suites) {
+ if((testSuitesPassingFilt.count(tc.m_test_suite) == 0) && tc.m_test_suite[0] != '\0') {
+ queryResults.push_back(&tc);
+ testSuitesPassingFilt.insert(tc.m_test_suite);
+ p->numTestSuitesPassingFilters++;
+ }
+ continue;
+ }
+
+ // execute the test if it passes all the filtering
+ {
+ p->currentTest = &tc;
+
+ p->failure_flags = TestCaseFailureReason::None;
+ p->seconds = 0;
+
+ // reset atomic counters
+ p->numAssertsFailedCurrentTest_atomic = 0;
+ p->numAssertsCurrentTest_atomic = 0;
+
+ p->subcasesPassed.clear();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_start, tc);
+
+ p->timer.start();
+
+ bool run_test = true;
+
+ do {
+ // reset some of the fields for subcases (except for the set of fully passed ones)
+ p->should_reenter = false;
+ p->subcasesCurrentMaxLevel = 0;
+ p->subcasesStack.clear();
+
+ p->shouldLogCurrentException = true;
+
+ // reset stuff for logging with INFO()
+ p->stringifiedContexts.clear();
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+// MSVC 2015 diagnoses fatalConditionHandler as unused (because reset() is a static method)
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4101) // unreferenced local variable
+ FatalConditionHandler fatalConditionHandler; // Handle signals
+ // execute the test
+ tc.m_test();
+ fatalConditionHandler.reset();
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ } catch(const TestFailureException&) {
+ p->failure_flags |= TestCaseFailureReason::AssertFailure;
+ } catch(...) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception,
+ {translateActiveException(), false});
+ p->failure_flags |= TestCaseFailureReason::Exception;
+ }
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+ // exit this loop if enough assertions have failed - even if there are more subcases
+ if(p->abort_after > 0 &&
+ p->numAssertsFailed + p->numAssertsFailedCurrentTest_atomic >= p->abort_after) {
+ run_test = false;
+ p->failure_flags |= TestCaseFailureReason::TooManyFailedAsserts;
+ }
+
+ if(p->should_reenter && run_test)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_reenter, tc);
+ if(!p->should_reenter)
+ run_test = false;
+ } while(run_test);
+
+ p->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ p->currentTest = nullptr;
+
+ // stop executing tests if enough assertions have failed
+ if(p->abort_after > 0 && p->numAssertsFailed >= p->abort_after)
+ break;
+ }
+ }
+
+ if(!query_mode) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ } else {
+ QueryData qdata;
+ qdata.run_stats = g_cs;
+ qdata.data = queryResults.data();
+ qdata.num_data = unsigned(queryResults.size());
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, qdata);
+ }
+
+ // see these issues on the reasoning for this:
+ // - https://github.com/onqtam/doctest/issues/143#issuecomment-414418903
+ // - https://github.com/onqtam/doctest/issues/126
+ auto DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS = []() DOCTEST_NOINLINE
+ { std::cout << std::string(); };
+ DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS();
+
+ return cleanup_and_return();
+}
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return detail::g_infoContexts.size(); }
+const IContextScope* const* IReporter::get_active_contexts() {
+ return get_num_active_contexts() ? &detail::g_infoContexts[0] : nullptr;
+}
+
+int IReporter::get_num_stringified_contexts() { return detail::g_cs->stringifiedContexts.size(); }
+const String* IReporter::get_stringified_contexts() {
+ return get_num_stringified_contexts() ? &detail::g_cs->stringifiedContexts[0] : nullptr;
+}
+
+namespace detail {
+ void registerReporterImpl(const char* name, int priority, reporterCreatorFunc c, bool isReporter) {
+ if(isReporter)
+ getReporters().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ else
+ getListeners().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ }
+} // namespace detail
+
+} // namespace doctest
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifdef DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4007) // 'function' : must be 'attribute' - see issue #182
+int main(int argc, char** argv) { return doctest::Context(argc, argv).run(); }
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_IMPLEMENTATION
+#endif // DOCTEST_CONFIG_IMPLEMENT
diff --git a/contrib/doctest/doctest/extensions/doctest_mpi.h b/contrib/doctest/doctest/extensions/doctest_mpi.h
new file mode 100644
index 0000000..45f82dd
--- /dev/null
+++ b/contrib/doctest/doctest/extensions/doctest_mpi.h
@@ -0,0 +1,120 @@
+#pragma once
+
+#ifdef DOCTEST_CONFIG_IMPLEMENT
+
+#include "doctest/doctest.h"
+#include "mpi_reporter.h"
+
+#else
+
+#include "mpi.h"
+#include <numeric>
+#include <vector>
+#include "doctest/doctest.h"
+#include <cassert>
+#include <string>
+
+namespace doctest {
+
+inline
+int mpi_world_nb_procs() {
+ int n;
+ MPI_Comm_size(MPI_COMM_WORLD, &n);
+ return n;
+}
+
+struct mpi_sub_comm {
+ int nb_procs;
+ int rank;
+ MPI_Comm comm;
+
+ mpi_sub_comm( mpi_sub_comm const& ) = delete;
+ mpi_sub_comm& operator=( mpi_sub_comm const& ) = delete;
+
+ mpi_sub_comm(int nb_prcs) noexcept
+ : nb_procs(nb_prcs)
+ , rank(-1)
+ , comm(MPI_COMM_NULL)
+ {
+ int comm_world_rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank);
+ if (nb_procs>mpi_world_nb_procs()) {
+ if (comm_world_rank==0) {
+ MESSAGE(
+ "Unable to run test: need ", std::to_string(nb_procs), " procs",
+ " but program launched with only ", std::to_string(doctest::mpi_world_nb_procs()), "."
+ );
+ CHECK(nb_procs<=mpi_world_nb_procs());
+ }
+ } else {
+ int color = MPI_UNDEFINED;
+ if(comm_world_rank < nb_procs){
+ color = 0;
+ }
+ MPI_Comm_split(MPI_COMM_WORLD, color, comm_world_rank, &comm);
+
+ if(comm != MPI_COMM_NULL){
+ MPI_Comm_rank(comm, &rank);
+ assert(rank==comm_world_rank);
+ }
+ }
+ }
+
+ ~mpi_sub_comm() {
+ if(comm != MPI_COMM_NULL){
+ MPI_Comm_free(&comm);
+ }
+ }
+};
+
+
+template<int nb_procs, class F>
+void execute_mpi_test_case(F func) {
+ mpi_sub_comm sub(nb_procs);
+ if (sub.comm != MPI_COMM_NULL) {
+ func(sub.rank,nb_procs,sub.comm,std::integral_constant<int,nb_procs>{});
+ };
+}
+
+} // doctest
+
+
+#define DOCTEST_MPI_GEN_ASSERTION(rank_to_test, assertion, ...) \
+ static_assert(rank_to_test<test_nb_procs_as_int_constant.value,"Trying to assert on a rank greater than the number of procs of the test!"); \
+ if(rank_to_test == test_rank) assertion(__VA_ARGS__)
+
+#define DOCTEST_MPI_WARN(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_WARN,__VA_ARGS__)
+#define DOCTEST_MPI_CHECK(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_CHECK,__VA_ARGS__)
+#define DOCTEST_MPI_REQUIRE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_REQUIRE,__VA_ARGS__)
+#define DOCTEST_MPI_WARN_FALSE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_WARN_FALSE,__VA_ARGS__)
+#define DOCTEST_MPI_CHECK_FALSE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_CHECK_FALSE,__VA_ARGS__)
+#define DOCTEST_MPI_REQUIRE_FALSE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_REQUIRE_FALSE,__VA_ARGS__)
+
+#define DOCTEST_CREATE_MPI_TEST_CASE(name,nb_procs,func) \
+ static void func(DOCTEST_UNUSED int test_rank, DOCTEST_UNUSED int test_nb_procs, DOCTEST_UNUSED MPI_Comm test_comm, DOCTEST_UNUSED std::integral_constant<int,nb_procs>); \
+ TEST_CASE(name * doctest::description("MPI_TEST_CASE")) { \
+ doctest::execute_mpi_test_case<nb_procs>(func); \
+ } \
+ static void func(DOCTEST_UNUSED int test_rank, DOCTEST_UNUSED int test_nb_procs, DOCTEST_UNUSED MPI_Comm test_comm, DOCTEST_UNUSED std::integral_constant<int,nb_procs> test_nb_procs_as_int_constant)
+ // DOC: test_rank, test_nb_procs, and test_comm are available UNDER THESE SPECIFIC NAMES in the body of the unit test
+ // DOC: test_nb_procs_as_int_constant is equal to test_nb_procs, but as a compile time value
+ // (used in CHECK-like macros to assert the checked rank exists)
+
+#define DOCTEST_MPI_TEST_CASE(name,nb_procs) \
+ DOCTEST_CREATE_MPI_TEST_CASE(name,nb_procs,DOCTEST_ANONYMOUS(DOCTEST_MPI_FUNC))
+
+
+// == SHORT VERSIONS OF THE MACROS
+#if !defined(DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES)
+#define MPI_WARN DOCTEST_MPI_WARN
+#define MPI_CHECK DOCTEST_MPI_CHECK
+#define MPI_REQUIRE DOCTEST_MPI_REQUIRE
+#define MPI_WARN_FALSE DOCTEST_MPI_WARN_FALSE
+#define MPI_CHECK_FALSE DOCTEST_MPI_CHECK_FALSE
+#define MPI_REQUIRE_FALSE DOCTEST_MPI_REQUIRE_FALSE
+
+#define MPI_TEST_CASE DOCTEST_MPI_TEST_CASE
+#endif // DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES
+
+
+#endif // DOCTEST_CONFIG_IMPLEMENT
diff --git a/contrib/doctest/doctest/extensions/doctest_util.h b/contrib/doctest/doctest/extensions/doctest_util.h
new file mode 100644
index 0000000..00e1151
--- /dev/null
+++ b/contrib/doctest/doctest/extensions/doctest_util.h
@@ -0,0 +1,34 @@
+//
+// doctest_util.h - an accompanying extensions header to the main doctest.h header
+//
+// Copyright (c) 2016-2021 Viktor Kirilov
+//
+// Distributed under the MIT Software License
+// See accompanying file LICENSE.txt or copy at
+// https://opensource.org/licenses/MIT
+//
+// The documentation can be found at the library's page:
+// https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md
+//
+
+#pragma once
+
+#ifndef DOCTEST_LIBRARY_INCLUDED
+#include "../doctest.h"
+#endif
+
+#include <memory>
+#include <vector>
+#include <string>
+
+namespace doctest {
+
+ inline void applyCommandLine(doctest::Context& ctx, const std::vector<std::string>& args) {
+ auto doctest_args = std::make_unique<const char*[]>(args.size());
+ for (size_t i = 0; i < args.size(); ++i) {
+ doctest_args[i] = args[i].c_str();
+ }
+ ctx.applyCommandLine(args.size(), doctest_args.get());
+ }
+
+} // namespace doctest
diff --git a/contrib/doctest/doctest/extensions/mpi_reporter.h b/contrib/doctest/doctest/extensions/mpi_reporter.h
new file mode 100644
index 0000000..9cccb5e
--- /dev/null
+++ b/contrib/doctest/doctest/extensions/mpi_reporter.h
@@ -0,0 +1,236 @@
+#pragma once
+
+// #include <doctest/doctest.h>
+#include <fstream>
+#include <string>
+#include "mpi.h"
+
+
+#include <vector>
+#include <mutex>
+
+namespace doctest {
+
+namespace {
+
+// https://stackoverflow.com/a/11826666/1583122
+struct NullBuffer : std::streambuf {
+ int overflow(int c) { return c; }
+};
+class NullStream : public std::ostream {
+ public:
+ NullStream()
+ : std::ostream(&nullBuff)
+ {}
+ private:
+ NullBuffer nullBuff = {};
+};
+static NullStream nullStream;
+
+
+/* \brief Extends the ConsoleReporter of doctest
+ * Each process writes its results to its own file
+ * Intended to be used when a test assertion fails and the user wants to know exactly what happens on which process
+ */
+struct MpiFileReporter : public ConsoleReporter {
+ std::ofstream logfile_stream = {};
+
+ MpiFileReporter(const ContextOptions& co)
+ : ConsoleReporter(co,logfile_stream)
+ {
+ int rank = 0;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+ std::string logfile_name = "doctest_" + std::to_string(rank) + ".log";
+
+ logfile_stream = std::ofstream(logfile_name.c_str(), std::fstream::out);
+ }
+};
+
+
+/* \brief Extends the ConsoleReporter of doctest
+ * Allows to manage the execution of tests in a parallel framework
+ * All results are collected on rank 0
+ */
+struct MpiConsoleReporter : public ConsoleReporter {
+private:
+ static std::ostream& replace_by_null_if_not_rank_0(std::ostream* os) {
+ int rank = 0;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ if (rank==0) {
+ return *os;
+ } else {
+ return nullStream;
+ }
+ }
+public:
+ MpiConsoleReporter(const ContextOptions& co)
+ : ConsoleReporter(co,replace_by_null_if_not_rank_0(co.cout))
+ {}
+
+ std::string file_line_to_string(const char* file, int line,
+ const char* tail = ""){
+ std::stringstream ss;
+ ss << skipPathFromFilename(file)
+ << (opt.gnu_file_line ? ":" : "(")
+ << (opt.no_line_numbers ? 0 : line) // 0 or the real num depending on the option
+ << (opt.gnu_file_line ? ":" : "):") << tail;
+ return ss.str();
+ }
+
+ void test_run_end(const TestRunStats& p) override {
+ ConsoleReporter::test_run_end(p);
+
+ const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
+
+ // -----------------------------------------------------
+ // > Gather information in rank 0
+ int n_rank, rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &n_rank);
+
+ int g_numAsserts = 0;
+ int g_numAssertsFailed = 0;
+ int g_numTestCasesFailed = 0;
+
+ MPI_Reduce(&p.numAsserts , &g_numAsserts , 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&p.numAssertsFailed , &g_numAssertsFailed , 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&p.numTestCasesFailed, &g_numTestCasesFailed, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ std::vector<int> numAssertsFailedByRank;
+ if(rank == 0){
+ numAssertsFailedByRank.resize(static_cast<std::size_t>(n_rank));
+ }
+
+ MPI_Gather(&p.numAssertsFailed, 1, MPI_INT, numAssertsFailedByRank.data(), 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ if(rank == 0) {
+ separator_to_stream();
+ s << Color::Cyan << "[doctest] " << Color::None << "glob assertions: " << std::setw(6)
+ << g_numAsserts << " | "
+ << ((g_numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
+ << std::setw(6) << (g_numAsserts - g_numAssertsFailed) << " passed" << Color::None
+ << " | " << (g_numAssertsFailed > 0 ? Color::Red : Color::None) << std::setw(6)
+ << g_numAssertsFailed << " failed" << Color::None << " |\n";
+
+ separator_to_stream();
+ if(g_numAssertsFailed > 0){
+
+ s << Color::Cyan << "[doctest] " << Color::None << "fail on rank:" << std::setw(6) << "\n";
+ for(std::size_t i = 0; i < numAssertsFailedByRank.size(); ++i){
+ if( numAssertsFailedByRank[i] > 0 ){
+ s << std::setw(16) << " -> On rank [" << i << "] with " << numAssertsFailedByRank[i] << " test failed" << std::endl;
+ }
+ }
+ }
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "Status: " << (g_numTestCasesFailed > 0 ? Color::Red : Color::Green)
+ << ((g_numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
+ }
+ }
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ if (is_mpi_test_case()) {
+ // function called by every rank at the end of a test
+ // if failed assertions happended, they have been sent to rank 0
+ // here rank zero gathers them and prints them all
+
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+ // Compute the number of assert with fail among all procs
+ int nb_fail_asserts_glob = 0;
+ MPI_Reduce(&st.numAssertsFailedCurrentTest, &nb_fail_asserts_glob, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ if(rank == 0) {
+ MPI_Status status;
+ MPI_Status status_recv;
+
+ using id_string = std::pair<int,std::string>;
+ std::vector<id_string> msgs(static_cast<std::size_t>(nb_fail_asserts_glob));
+
+ for (std::size_t i=0; i<static_cast<std::size_t>(nb_fail_asserts_glob); ++i) {
+ MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
+
+ int count;
+ MPI_Get_count(&status, MPI_BYTE, &count);
+
+ std::string recv_msg(static_cast<std::size_t>(count),'\0');
+ void* recv_msg_data = const_cast<char*>(recv_msg.data()); // const_cast needed. Non-const .data() exists in C++11 though...
+ MPI_Recv(recv_msg_data, count, MPI_BYTE, status.MPI_SOURCE,
+ status.MPI_TAG, MPI_COMM_WORLD, &status_recv);
+
+ msgs[i] = {status.MPI_SOURCE,recv_msg};
+ }
+
+ std::sort(begin(msgs),end(msgs),[](const id_string& x, const id_string& y){ return x.first < y.first; });
+
+ // print
+ if (nb_fail_asserts_glob>0) {
+ separator_to_stream();
+ file_line_to_stream(tc->m_file.c_str(), static_cast<int>(tc->m_line), "\n");
+ if(tc->m_test_suite && tc->m_test_suite[0] != '\0')
+ s << Color::Yellow << "TEST SUITE: " << Color::None << tc->m_test_suite << "\n";
+ if(strncmp(tc->m_name, " Scenario:", 11) != 0)
+ s << Color::Yellow << "TEST CASE: ";
+ s << Color::None << tc->m_name << "\n\n";
+ for(const auto& msg : msgs) {
+ s << msg.second;
+ }
+ s << "\n";
+ }
+ }
+ }
+
+ ConsoleReporter::test_case_end(st);
+ }
+
+ bool is_mpi_test_case() const {
+ return tc->m_description != nullptr
+ && std::string(tc->m_description) == std::string("MPI_TEST_CASE");
+ }
+
+ void log_assert(const AssertData& rb) override {
+ if (!is_mpi_test_case()) {
+ ConsoleReporter::log_assert(rb);
+ } else {
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+
+ if(!rb.m_failed && !opt.success)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ std::stringstream failure_msg;
+ failure_msg << Color::Red << "On rank [" << rank << "] : " << Color::None;
+ failure_msg << file_line_to_string(rb.m_file, rb.m_line, " ");
+
+ if((rb.m_at & (assertType::is_throws_as | assertType::is_throws_with)) ==0){
+ failure_msg << Color::Cyan
+ << assertString(rb.m_at)
+ << "( " << rb.m_expr << " ) "
+ << Color::None
+
+ << (!rb.m_failed ? "is correct!\n" : "is NOT correct!\n")
+ << " values: "
+ << assertString(rb.m_at)
+ << "( " << rb.m_decomp.c_str() << " )\n";
+ }
+
+ std::string failure_str = failure_msg.str();
+ int failure_msg_size = static_cast<int>(failure_str.size());
+
+ MPI_Send(failure_str.c_str(), failure_msg_size, MPI_BYTE,
+ 0, rb.m_line, MPI_COMM_WORLD); // Tag = file line
+ }
+ }
+}; // MpiConsoleReporter
+
+// "1" is the priority - used for ordering when multiple reporters/listeners are used
+REGISTER_REPORTER("MpiConsoleReporter", 1, MpiConsoleReporter);
+REGISTER_REPORTER("MpiFileReporter", 1, MpiFileReporter);
+
+} // anonymous
+} // doctest
diff --git a/contrib/doctest/doctest/parts/doctest.cpp b/contrib/doctest/doctest/parts/doctest.cpp
new file mode 100644
index 0000000..5b74420
--- /dev/null
+++ b/contrib/doctest/doctest/parts/doctest.cpp
@@ -0,0 +1,3866 @@
+#if defined(DOCTEST_CONFIG_IMPLEMENT) || !defined(DOCTEST_SINGLE_HEADER)
+
+#ifndef DOCTEST_SINGLE_HEADER
+#include "doctest_fwd.h"
+#endif // DOCTEST_SINGLE_HEADER
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-macros")
+
+#ifndef DOCTEST_LIBRARY_IMPLEMENTATION
+#define DOCTEST_LIBRARY_IMPLEMENTATION
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wglobal-constructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wshorten-64-to-32")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-variable-declarations")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wcovered-switch-default")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-noreturn")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdisabled-macro-expansion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-member-function")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnonportable-system-include-path")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-default")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunsafe-loop-optimizations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wold-style-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-function")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmultiple-inheritance")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsuggest-attribute")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4267) // 'var' : conversion from 'x' to 'y', possible loss of data
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4530) // C++ exception handler used, but unwind semantics not enabled
+DOCTEST_MSVC_SUPPRESS_WARNING(4577) // 'noexcept' used with no exception handling mode specified
+DOCTEST_MSVC_SUPPRESS_WARNING(4774) // format string expected in argument is not a string literal
+DOCTEST_MSVC_SUPPRESS_WARNING(4365) // conversion from 'int' to 'unsigned', signed/unsigned mismatch
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding in structs
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+DOCTEST_MSVC_SUPPRESS_WARNING(5039) // pointer to potentially throwing function passed to extern C
+DOCTEST_MSVC_SUPPRESS_WARNING(5045) // Spectre mitigation stuff
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4800) // forcing value to bool 'true' or 'false' (performance warning)
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtor...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN
+
+// required includes - will go only in one translation unit!
+#include <ctime>
+#include <cmath>
+#include <climits>
+// borland (Embarcadero) compiler requires math.h and not cmath - https://github.com/onqtam/doctest/pull/37
+#ifdef __BORLANDC__
+#include <math.h>
+#endif // __BORLANDC__
+#include <new>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <utility>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <algorithm>
+#include <iomanip>
+#include <vector>
+#include <atomic>
+#include <mutex>
+#include <set>
+#include <map>
+#include <exception>
+#include <stdexcept>
+#include <csignal>
+#include <cfloat>
+#include <cctype>
+#include <cstdint>
+
+#ifdef DOCTEST_PLATFORM_MAC
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/sysctl.h>
+#endif // DOCTEST_PLATFORM_MAC
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+// defines for a leaner windows.h
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif // WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif // NOMINMAX
+
+// not sure what AfxWin.h is for - here I do what Catch does
+#ifdef __AFXDLL
+#include <AfxWin.h>
+#else
+#include <windows.h>
+#endif
+#include <io.h>
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+// this is a fix for https://github.com/onqtam/doctest/issues/348
+// https://mail.gnome.org/archives/xml/2012-January/msg00000.html
+#if !defined(HAVE_UNISTD_H) && !defined(STDOUT_FILENO)
+#define STDOUT_FILENO fileno(stdout)
+#endif // HAVE_UNISTD_H
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END
+
+// counts the number of elements in a C array
+#define DOCTEST_COUNTOF(x) (sizeof(x) / sizeof(x[0]))
+
+#ifdef DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_disabled
+#else // DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_not_disabled
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifndef DOCTEST_CONFIG_OPTIONS_PREFIX
+#define DOCTEST_CONFIG_OPTIONS_PREFIX "dt-"
+#endif
+
+#ifndef DOCTEST_THREAD_LOCAL
+#define DOCTEST_THREAD_LOCAL thread_local
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES
+#define DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES 32
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE
+#define DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE 64
+#endif
+
+#ifdef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY DOCTEST_CONFIG_OPTIONS_PREFIX
+#else
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY ""
+#endif
+
+#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
+#define DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+#endif
+
+namespace doctest {
+
+bool is_running_in_test = false;
+
+namespace {
+ using namespace detail;
+ // case insensitive strcmp
+ int stricmp(const char* a, const char* b) {
+ for(;; a++, b++) {
+ const int d = tolower(*a) - tolower(*b);
+ if(d != 0 || !*a)
+ return d;
+ }
+ }
+
+ template <typename T>
+ String fpToString(T value, int precision) {
+ std::ostringstream oss;
+ oss << std::setprecision(precision) << std::fixed << value;
+ std::string d = oss.str();
+ size_t i = d.find_last_not_of('0');
+ if(i != std::string::npos && i != d.size() - 1) {
+ if(d[i] == '.')
+ i++;
+ d = d.substr(0, i + 1);
+ }
+ return d.c_str();
+ }
+
+ struct Endianness
+ {
+ enum Arch
+ {
+ Big,
+ Little
+ };
+
+ static Arch which() {
+ int x = 1;
+ // casting any data pointer to char* is allowed
+ auto ptr = reinterpret_cast<char*>(&x);
+ if(*ptr)
+ return Little;
+ return Big;
+ }
+ };
+} // namespace
+
+namespace detail {
+ void my_memcpy(void* dest, const void* src, unsigned num) { memcpy(dest, src, num); }
+
+ String rawMemoryToString(const void* object, unsigned size) {
+ // Reverse order for little endian architectures
+ int i = 0, end = static_cast<int>(size), inc = 1;
+ if(Endianness::which() == Endianness::Little) {
+ i = end - 1;
+ end = inc = -1;
+ }
+
+ unsigned const char* bytes = static_cast<unsigned const char*>(object);
+ std::ostringstream oss;
+ oss << "0x" << std::setfill('0') << std::hex;
+ for(; i != end; i += inc)
+ oss << std::setw(2) << static_cast<unsigned>(bytes[i]);
+ return oss.str().c_str();
+ }
+
+ DOCTEST_THREAD_LOCAL std::ostringstream g_oss; // NOLINT(cert-err58-cpp)
+
+ std::ostream* getTlsOss() {
+ g_oss.clear(); // there shouldn't be anything worth clearing in the flags
+ g_oss.str(""); // the slow way of resetting a string stream
+ //g_oss.seekp(0); // optimal reset - as seen here: https://stackoverflow.com/a/624291/3162383
+ return &g_oss;
+ }
+
+ String getTlsOssResult() {
+ //g_oss << std::ends; // needed - as shown here: https://stackoverflow.com/a/624291/3162383
+ return g_oss.str().c_str();
+ }
+
+#ifndef DOCTEST_CONFIG_DISABLE
+
+namespace timer_large_integer
+{
+
+#if defined(DOCTEST_PLATFORM_WINDOWS)
+ typedef ULONGLONG type;
+#else // DOCTEST_PLATFORM_WINDOWS
+ using namespace std;
+ typedef uint64_t type;
+#endif // DOCTEST_PLATFORM_WINDOWS
+}
+
+typedef timer_large_integer::type ticks_t;
+
+#ifdef DOCTEST_CONFIG_GETCURRENTTICKS
+ ticks_t getCurrentTicks() { return DOCTEST_CONFIG_GETCURRENTTICKS(); }
+#elif defined(DOCTEST_PLATFORM_WINDOWS)
+ ticks_t getCurrentTicks() {
+ static LARGE_INTEGER hz = {0}, hzo = {0};
+ if(!hz.QuadPart) {
+ QueryPerformanceFrequency(&hz);
+ QueryPerformanceCounter(&hzo);
+ }
+ LARGE_INTEGER t;
+ QueryPerformanceCounter(&t);
+ return ((t.QuadPart - hzo.QuadPart) * LONGLONG(1000000)) / hz.QuadPart;
+ }
+#else // DOCTEST_PLATFORM_WINDOWS
+ ticks_t getCurrentTicks() {
+ timeval t;
+ gettimeofday(&t, nullptr);
+ return static_cast<ticks_t>(t.tv_sec) * 1000000 + static_cast<ticks_t>(t.tv_usec);
+ }
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ struct Timer
+ {
+ void start() { m_ticks = getCurrentTicks(); }
+ unsigned int getElapsedMicroseconds() const {
+ return static_cast<unsigned int>(getCurrentTicks() - m_ticks);
+ }
+ //unsigned int getElapsedMilliseconds() const {
+ // return static_cast<unsigned int>(getElapsedMicroseconds() / 1000);
+ //}
+ double getElapsedSeconds() const { return static_cast<double>(getCurrentTicks() - m_ticks) / 1000000.0; }
+
+ private:
+ ticks_t m_ticks = 0;
+ };
+
+#ifdef DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = std::atomic<T>;
+#else // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ // Provides a multilane implementation of an atomic variable that supports add, sub, load,
+ // store. Instead of using a single atomic variable, this splits up into multiple ones,
+ // each sitting on a separate cache line. The goal is to provide a speedup when most
+ // operations are modifying. It achieves this with two properties:
+ //
+ // * Multiple atomics are used, so chance of congestion from the same atomic is reduced.
+ // * Each atomic sits on a separate cache line, so false sharing is reduced.
+ //
+ // The disadvantage is that there is a small overhead due to the use of TLS, and load/store
+ // is slower because all atomics have to be accessed.
+ template <typename T>
+ class MultiLaneAtomic
+ {
+ struct CacheLineAlignedAtomic
+ {
+ std::atomic<T> atomic{};
+ char padding[DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE - sizeof(std::atomic<T>)];
+ };
+ CacheLineAlignedAtomic m_atomics[DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES];
+
+ static_assert(sizeof(CacheLineAlignedAtomic) == DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE,
+ "guarantee one atomic takes exactly one cache line");
+
+ public:
+ T operator++() DOCTEST_NOEXCEPT { return fetch_add(1) + 1; }
+
+ T operator++(int) DOCTEST_NOEXCEPT { return fetch_add(1); }
+
+ T fetch_add(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_add(arg, order);
+ }
+
+ T fetch_sub(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_sub(arg, order);
+ }
+
+ operator T() const DOCTEST_NOEXCEPT { return load(); }
+
+ T load(std::memory_order order = std::memory_order_seq_cst) const DOCTEST_NOEXCEPT {
+ auto result = T();
+ for(auto const& c : m_atomics) {
+ result += c.atomic.load(order);
+ }
+ return result;
+ }
+
+ T operator=(T desired) DOCTEST_NOEXCEPT {
+ store(desired);
+ return desired;
+ }
+
+ void store(T desired, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ // first value becomes desired", all others become 0.
+ for(auto& c : m_atomics) {
+ c.atomic.store(desired, order);
+ desired = {};
+ }
+ }
+
+ private:
+ // Each thread has a different atomic that it operates on. If more than NumLanes threads
+ // use this, some will use the same atomic. So performance will degrate a bit, but still
+ // everything will work.
+ //
+ // The logic here is a bit tricky. The call should be as fast as possible, so that there
+ // is minimal to no overhead in determining the correct atomic for the current thread.
+ //
+ // 1. A global static counter laneCounter counts continuously up.
+ // 2. Each successive thread will use modulo operation of that counter so it gets an atomic
+ // assigned in a round-robin fashion.
+ // 3. This tlsLaneIdx is stored in the thread local data, so it is directly available with
+ // little overhead.
+ std::atomic<T>& myAtomic() DOCTEST_NOEXCEPT {
+ static std::atomic<size_t> laneCounter;
+ DOCTEST_THREAD_LOCAL size_t tlsLaneIdx =
+ laneCounter++ % DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES;
+
+ return m_atomics[tlsLaneIdx].atomic;
+ }
+ };
+
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = MultiLaneAtomic<T>;
+#endif // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+
+ // this holds both parameters from the command line and runtime data for tests
+ struct ContextState : ContextOptions, TestRunStats, CurrentTestCaseStats
+ {
+ AtomicOrMultiLaneAtomic<int> numAssertsCurrentTest_atomic;
+ AtomicOrMultiLaneAtomic<int> numAssertsFailedCurrentTest_atomic;
+
+ std::vector<std::vector<String>> filters = decltype(filters)(9); // 9 different filters
+
+ std::vector<IReporter*> reporters_currently_used;
+
+ assert_handler ah = nullptr;
+
+ Timer timer;
+
+ std::vector<String> stringifiedContexts; // logging from INFO() due to an exception
+
+ // stuff for subcases
+ std::vector<SubcaseSignature> subcasesStack;
+ std::set<decltype(subcasesStack)> subcasesPassed;
+ int subcasesCurrentMaxLevel;
+ bool should_reenter;
+ std::atomic<bool> shouldLogCurrentException;
+
+ void resetRunData() {
+ numTestCases = 0;
+ numTestCasesPassingFilters = 0;
+ numTestSuitesPassingFilters = 0;
+ numTestCasesFailed = 0;
+ numAsserts = 0;
+ numAssertsFailed = 0;
+ numAssertsCurrentTest = 0;
+ numAssertsFailedCurrentTest = 0;
+ }
+
+ void finalizeTestCaseData() {
+ seconds = timer.getElapsedSeconds();
+
+ // update the non-atomic counters
+ numAsserts += numAssertsCurrentTest_atomic;
+ numAssertsFailed += numAssertsFailedCurrentTest_atomic;
+ numAssertsCurrentTest = numAssertsCurrentTest_atomic;
+ numAssertsFailedCurrentTest = numAssertsFailedCurrentTest_atomic;
+
+ if(numAssertsFailedCurrentTest)
+ failure_flags |= TestCaseFailureReason::AssertFailure;
+
+ if(Approx(currentTest->m_timeout).epsilon(DBL_EPSILON) != 0 &&
+ Approx(seconds).epsilon(DBL_EPSILON) > currentTest->m_timeout)
+ failure_flags |= TestCaseFailureReason::Timeout;
+
+ if(currentTest->m_should_fail) {
+ if(failure_flags) {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedAndDid;
+ } else {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedButDidnt;
+ }
+ } else if(failure_flags && currentTest->m_may_fail) {
+ failure_flags |= TestCaseFailureReason::CouldHaveFailedAndDid;
+ } else if(currentTest->m_expected_failures > 0) {
+ if(numAssertsFailedCurrentTest == currentTest->m_expected_failures) {
+ failure_flags |= TestCaseFailureReason::FailedExactlyNumTimes;
+ } else {
+ failure_flags |= TestCaseFailureReason::DidntFailExactlyNumTimes;
+ }
+ }
+
+ bool ok_to_fail = (TestCaseFailureReason::ShouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::CouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::FailedExactlyNumTimes & failure_flags);
+
+ // if any subcase has failed - the whole test case has failed
+ if(failure_flags && !ok_to_fail)
+ numTestCasesFailed++;
+ }
+ };
+
+ ContextState* g_cs = nullptr;
+
+ // used to avoid locks for the debug output
+ // TODO: figure out if this is indeed necessary/correct - seems like either there still
+ // could be a race or that there wouldn't be a race even if using the context directly
+ DOCTEST_THREAD_LOCAL bool g_no_colors;
+
+#endif // DOCTEST_CONFIG_DISABLE
+} // namespace detail
+
+void String::setOnHeap() { *reinterpret_cast<unsigned char*>(&buf[last]) = 128; }
+void String::setLast(unsigned in) { buf[last] = char(in); }
+
+void String::copy(const String& other) {
+ using namespace std;
+ if(other.isOnStack()) {
+ memcpy(buf, other.buf, len);
+ } else {
+ setOnHeap();
+ data.size = other.data.size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, other.data.ptr, data.size + 1);
+ }
+}
+
+String::String() {
+ buf[0] = '\0';
+ setLast();
+}
+
+String::~String() {
+ if(!isOnStack())
+ delete[] data.ptr;
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+String::String(const char* in)
+ : String(in, strlen(in)) {}
+
+String::String(const char* in, unsigned in_size) {
+ using namespace std;
+ if(in_size <= last) {
+ memcpy(buf, in, in_size);
+ buf[in_size] = '\0';
+ setLast(last - in_size);
+ } else {
+ setOnHeap();
+ data.size = in_size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, in, in_size);
+ data.ptr[in_size] = '\0';
+ }
+}
+
+String::String(const String& other) { copy(other); }
+
+String& String::operator=(const String& other) {
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+
+ copy(other);
+ }
+
+ return *this;
+}
+
+String& String::operator+=(const String& other) {
+ const unsigned my_old_size = size();
+ const unsigned other_size = other.size();
+ const unsigned total_size = my_old_size + other_size;
+ using namespace std;
+ if(isOnStack()) {
+ if(total_size < len) {
+ // append to the current stack space
+ memcpy(buf + my_old_size, other.c_str(), other_size + 1);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ setLast(last - total_size);
+ } else {
+ // alloc new chunk
+ char* temp = new char[total_size + 1];
+ // copy current data to new location before writing in the union
+ memcpy(temp, buf, my_old_size); // skip the +1 ('\0') for speed
+ // update data in union
+ setOnHeap();
+ data.size = total_size;
+ data.capacity = data.size + 1;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ } else {
+ if(data.capacity > total_size) {
+ // append to the current heap block
+ data.size = total_size;
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ } else {
+ // resize
+ data.capacity *= 2;
+ if(data.capacity <= total_size)
+ data.capacity = total_size + 1;
+ // alloc new chunk
+ char* temp = new char[data.capacity];
+ // copy current data to new location before releasing it
+ memcpy(temp, data.ptr, my_old_size); // skip the +1 ('\0') for speed
+ // release old chunk
+ delete[] data.ptr;
+ // update the rest of the union members
+ data.size = total_size;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ }
+
+ return *this;
+}
+
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String String::operator+(const String& other) const { return String(*this) += other; }
+
+String::String(String&& other) {
+ using namespace std;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+}
+
+String& String::operator=(String&& other) {
+ using namespace std;
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+ }
+ return *this;
+}
+
+char String::operator[](unsigned i) const {
+ return const_cast<String*>(this)->operator[](i); // NOLINT
+}
+
+char& String::operator[](unsigned i) {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf)[i];
+ return data.ptr[i];
+}
+
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmaybe-uninitialized")
+unsigned String::size() const {
+ if(isOnStack())
+ return last - (unsigned(buf[last]) & 31); // using "last" would work only if "len" is 32
+ return data.size;
+}
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+unsigned String::capacity() const {
+ if(isOnStack())
+ return len;
+ return data.capacity;
+}
+
+int String::compare(const char* other, bool no_case) const {
+ if(no_case)
+ return doctest::stricmp(c_str(), other);
+ return std::strcmp(c_str(), other);
+}
+
+int String::compare(const String& other, bool no_case) const {
+ return compare(other.c_str(), no_case);
+}
+
+// clang-format off
+bool operator==(const String& lhs, const String& rhs) { return lhs.compare(rhs) == 0; }
+bool operator!=(const String& lhs, const String& rhs) { return lhs.compare(rhs) != 0; }
+bool operator< (const String& lhs, const String& rhs) { return lhs.compare(rhs) < 0; }
+bool operator> (const String& lhs, const String& rhs) { return lhs.compare(rhs) > 0; }
+bool operator<=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) < 0 : true; }
+bool operator>=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) > 0 : true; }
+// clang-format on
+
+std::ostream& operator<<(std::ostream& s, const String& in) { return s << in.c_str(); }
+
+namespace {
+ void color_to_stream(std::ostream&, Color::Enum) DOCTEST_BRANCH_ON_DISABLED({}, ;)
+} // namespace
+
+namespace Color {
+ std::ostream& operator<<(std::ostream& s, Color::Enum code) {
+ color_to_stream(s, code);
+ return s;
+ }
+} // namespace Color
+
+// clang-format off
+const char* assertString(assertType::Enum at) {
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4062) // enum 'x' in switch of enum 'y' is not handled
+ switch(at) { //!OCLINT missing default in switch statements
+ case assertType::DT_WARN : return "WARN";
+ case assertType::DT_CHECK : return "CHECK";
+ case assertType::DT_REQUIRE : return "REQUIRE";
+
+ case assertType::DT_WARN_FALSE : return "WARN_FALSE";
+ case assertType::DT_CHECK_FALSE : return "CHECK_FALSE";
+ case assertType::DT_REQUIRE_FALSE : return "REQUIRE_FALSE";
+
+ case assertType::DT_WARN_THROWS : return "WARN_THROWS";
+ case assertType::DT_CHECK_THROWS : return "CHECK_THROWS";
+ case assertType::DT_REQUIRE_THROWS : return "REQUIRE_THROWS";
+
+ case assertType::DT_WARN_THROWS_AS : return "WARN_THROWS_AS";
+ case assertType::DT_CHECK_THROWS_AS : return "CHECK_THROWS_AS";
+ case assertType::DT_REQUIRE_THROWS_AS : return "REQUIRE_THROWS_AS";
+
+ case assertType::DT_WARN_THROWS_WITH : return "WARN_THROWS_WITH";
+ case assertType::DT_CHECK_THROWS_WITH : return "CHECK_THROWS_WITH";
+ case assertType::DT_REQUIRE_THROWS_WITH : return "REQUIRE_THROWS_WITH";
+
+ case assertType::DT_WARN_THROWS_WITH_AS : return "WARN_THROWS_WITH_AS";
+ case assertType::DT_CHECK_THROWS_WITH_AS : return "CHECK_THROWS_WITH_AS";
+ case assertType::DT_REQUIRE_THROWS_WITH_AS : return "REQUIRE_THROWS_WITH_AS";
+
+ case assertType::DT_WARN_NOTHROW : return "WARN_NOTHROW";
+ case assertType::DT_CHECK_NOTHROW : return "CHECK_NOTHROW";
+ case assertType::DT_REQUIRE_NOTHROW : return "REQUIRE_NOTHROW";
+
+ case assertType::DT_WARN_EQ : return "WARN_EQ";
+ case assertType::DT_CHECK_EQ : return "CHECK_EQ";
+ case assertType::DT_REQUIRE_EQ : return "REQUIRE_EQ";
+ case assertType::DT_WARN_NE : return "WARN_NE";
+ case assertType::DT_CHECK_NE : return "CHECK_NE";
+ case assertType::DT_REQUIRE_NE : return "REQUIRE_NE";
+ case assertType::DT_WARN_GT : return "WARN_GT";
+ case assertType::DT_CHECK_GT : return "CHECK_GT";
+ case assertType::DT_REQUIRE_GT : return "REQUIRE_GT";
+ case assertType::DT_WARN_LT : return "WARN_LT";
+ case assertType::DT_CHECK_LT : return "CHECK_LT";
+ case assertType::DT_REQUIRE_LT : return "REQUIRE_LT";
+ case assertType::DT_WARN_GE : return "WARN_GE";
+ case assertType::DT_CHECK_GE : return "CHECK_GE";
+ case assertType::DT_REQUIRE_GE : return "REQUIRE_GE";
+ case assertType::DT_WARN_LE : return "WARN_LE";
+ case assertType::DT_CHECK_LE : return "CHECK_LE";
+ case assertType::DT_REQUIRE_LE : return "REQUIRE_LE";
+
+ case assertType::DT_WARN_UNARY : return "WARN_UNARY";
+ case assertType::DT_CHECK_UNARY : return "CHECK_UNARY";
+ case assertType::DT_REQUIRE_UNARY : return "REQUIRE_UNARY";
+ case assertType::DT_WARN_UNARY_FALSE : return "WARN_UNARY_FALSE";
+ case assertType::DT_CHECK_UNARY_FALSE : return "CHECK_UNARY_FALSE";
+ case assertType::DT_REQUIRE_UNARY_FALSE : return "REQUIRE_UNARY_FALSE";
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ return "";
+}
+// clang-format on
+
+const char* failureString(assertType::Enum at) {
+ if(at & assertType::is_warn) //!OCLINT bitwise operator in conditional
+ return "WARNING";
+ if(at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ return "ERROR";
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return "FATAL ERROR";
+ return "";
+}
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+// depending on the current options this will remove the path of filenames
+const char* skipPathFromFilename(const char* file) {
+#ifndef DOCTEST_CONFIG_DISABLE
+ if(getContextOptions()->no_path_in_filenames) {
+ auto back = std::strrchr(file, '\\');
+ auto forward = std::strrchr(file, '/');
+ if(back || forward) {
+ if(back > forward)
+ forward = back;
+ return forward + 1;
+ }
+ }
+#endif // DOCTEST_CONFIG_DISABLE
+ return file;
+}
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+bool SubcaseSignature::operator<(const SubcaseSignature& other) const {
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ if(std::strcmp(m_file, other.m_file) != 0)
+ return std::strcmp(m_file, other.m_file) < 0;
+ return m_name.compare(other.m_name) < 0;
+}
+
+IContextScope::IContextScope() = default;
+IContextScope::~IContextScope() = default;
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(char* in) { return toString(static_cast<const char*>(in)); }
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String toString(const char* in) { return String("\"") + (in ? in : "{null string}") + "\""; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(bool in) { return in ? "true" : "false"; }
+String toString(float in) { return fpToString(in, 5) + "f"; }
+String toString(double in) { return fpToString(in, 10); }
+String toString(double long in) { return fpToString(in, 15); }
+
+#define DOCTEST_TO_STRING_OVERLOAD(type, fmt) \
+ String toString(type in) { \
+ char buf[64]; \
+ std::sprintf(buf, fmt, in); \
+ return buf; \
+ }
+
+DOCTEST_TO_STRING_OVERLOAD(char, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char signed, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int short, "%d")
+DOCTEST_TO_STRING_OVERLOAD(int short unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int, "%d")
+DOCTEST_TO_STRING_OVERLOAD(unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int long, "%ld")
+DOCTEST_TO_STRING_OVERLOAD(int long unsigned, "%lu")
+DOCTEST_TO_STRING_OVERLOAD(int long long, "%lld")
+DOCTEST_TO_STRING_OVERLOAD(int long long unsigned, "%llu")
+
+String toString(std::nullptr_t) { return "NULL"; }
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+String toString(const std::string& in) { return in.c_str(); }
+#endif // VS 2019
+
+Approx::Approx(double value)
+ : m_epsilon(static_cast<double>(std::numeric_limits<float>::epsilon()) * 100)
+ , m_scale(1.0)
+ , m_value(value) {}
+
+Approx Approx::operator()(double value) const {
+ Approx approx(value);
+ approx.epsilon(m_epsilon);
+ approx.scale(m_scale);
+ return approx;
+}
+
+Approx& Approx::epsilon(double newEpsilon) {
+ m_epsilon = newEpsilon;
+ return *this;
+}
+Approx& Approx::scale(double newScale) {
+ m_scale = newScale;
+ return *this;
+}
+
+bool operator==(double lhs, const Approx& rhs) {
+ // Thanks to Richard Harris for his help refining this formula
+ return std::fabs(lhs - rhs.m_value) <
+ rhs.m_epsilon * (rhs.m_scale + std::max<double>(std::fabs(lhs), std::fabs(rhs.m_value)));
+}
+bool operator==(const Approx& lhs, double rhs) { return operator==(rhs, lhs); }
+bool operator!=(double lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+bool operator!=(const Approx& lhs, double rhs) { return !operator==(rhs, lhs); }
+bool operator<=(double lhs, const Approx& rhs) { return lhs < rhs.m_value || lhs == rhs; }
+bool operator<=(const Approx& lhs, double rhs) { return lhs.m_value < rhs || lhs == rhs; }
+bool operator>=(double lhs, const Approx& rhs) { return lhs > rhs.m_value || lhs == rhs; }
+bool operator>=(const Approx& lhs, double rhs) { return lhs.m_value > rhs || lhs == rhs; }
+bool operator<(double lhs, const Approx& rhs) { return lhs < rhs.m_value && lhs != rhs; }
+bool operator<(const Approx& lhs, double rhs) { return lhs.m_value < rhs && lhs != rhs; }
+bool operator>(double lhs, const Approx& rhs) { return lhs > rhs.m_value && lhs != rhs; }
+bool operator>(const Approx& lhs, double rhs) { return lhs.m_value > rhs && lhs != rhs; }
+
+String toString(const Approx& in) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return String("Approx( ") + doctest::toString(in.m_value) + " )";
+}
+const ContextOptions* getContextOptions() { return DOCTEST_BRANCH_ON_DISABLED(nullptr, g_cs); }
+
+} // namespace doctest
+
+#ifdef DOCTEST_CONFIG_DISABLE
+namespace doctest {
+Context::Context(int, const char* const*) {}
+Context::~Context() = default;
+void Context::applyCommandLine(int, const char* const*) {}
+void Context::addFilter(const char*, const char*) {}
+void Context::clearFilters() {}
+void Context::setOption(const char*, int) {}
+void Context::setOption(const char*, const char*) {}
+bool Context::shouldExit() { return false; }
+void Context::setAsDefaultForAssertsOutOfTestCases() {}
+void Context::setAssertHandler(detail::assert_handler) {}
+int Context::run() { return 0; }
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return 0; }
+const IContextScope* const* IReporter::get_active_contexts() { return nullptr; }
+int IReporter::get_num_stringified_contexts() { return 0; }
+const String* IReporter::get_stringified_contexts() { return nullptr; }
+
+int registerReporter(const char*, int, IReporter*) { return 0; }
+
+} // namespace doctest
+#else // DOCTEST_CONFIG_DISABLE
+
+#if !defined(DOCTEST_CONFIG_COLORS_NONE)
+#if !defined(DOCTEST_CONFIG_COLORS_WINDOWS) && !defined(DOCTEST_CONFIG_COLORS_ANSI)
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_CONFIG_COLORS_WINDOWS
+#else // linux
+#define DOCTEST_CONFIG_COLORS_ANSI
+#endif // platform
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS && DOCTEST_CONFIG_COLORS_ANSI
+#endif // DOCTEST_CONFIG_COLORS_NONE
+
+namespace doctest_detail_test_suite_ns {
+// holds the current test suite
+doctest::detail::TestSuite& getCurrentTestSuite() {
+ static doctest::detail::TestSuite data{};
+ return data;
+}
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+namespace {
+ // the int (priority) is part of the key for automatic sorting - sadly one can register a
+ // reporter with a duplicate name and a different priority but hopefully that won't happen often :|
+ typedef std::map<std::pair<int, String>, reporterCreatorFunc> reporterMap;
+
+ reporterMap& getReporters() {
+ static reporterMap data;
+ return data;
+ }
+ reporterMap& getListeners() {
+ static reporterMap data;
+ return data;
+ }
+} // namespace
+namespace detail {
+#define DOCTEST_ITERATE_THROUGH_REPORTERS(function, ...) \
+ for(auto& curr_rep : g_cs->reporters_currently_used) \
+ curr_rep->function(__VA_ARGS__)
+
+ bool checkIfShouldThrow(assertType::Enum at) {
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return true;
+
+ if((at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ && getContextOptions()->abort_after > 0 &&
+ (g_cs->numAssertsFailed + g_cs->numAssertsFailedCurrentTest_atomic) >=
+ getContextOptions()->abort_after)
+ return true;
+
+ return false;
+ }
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN void throwException() {
+ g_cs->shouldLogCurrentException = false;
+ throw TestFailureException();
+ } // NOLINT(cert-err60-cpp)
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ void throwException() {}
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+} // namespace detail
+
+namespace {
+ using namespace detail;
+ // matching of a string against a wildcard mask (case sensitivity configurable) taken from
+ // https://www.codeproject.com/Articles/1088/Wildcard-string-compare-globbing
+ int wildcmp(const char* str, const char* wild, bool caseSensitive) {
+ const char* cp = str;
+ const char* mp = wild;
+
+ while((*str) && (*wild != '*')) {
+ if((caseSensitive ? (*wild != *str) : (tolower(*wild) != tolower(*str))) &&
+ (*wild != '?')) {
+ return 0;
+ }
+ wild++;
+ str++;
+ }
+
+ while(*str) {
+ if(*wild == '*') {
+ if(!*++wild) {
+ return 1;
+ }
+ mp = wild;
+ cp = str + 1;
+ } else if((caseSensitive ? (*wild == *str) : (tolower(*wild) == tolower(*str))) ||
+ (*wild == '?')) {
+ wild++;
+ str++;
+ } else {
+ wild = mp; //!OCLINT parameter reassignment
+ str = cp++; //!OCLINT parameter reassignment
+ }
+ }
+
+ while(*wild == '*') {
+ wild++;
+ }
+ return !*wild;
+ }
+
+ //// C string hash function (djb2) - taken from http://www.cse.yorku.ca/~oz/hash.html
+ //unsigned hashStr(unsigned const char* str) {
+ // unsigned long hash = 5381;
+ // char c;
+ // while((c = *str++))
+ // hash = ((hash << 5) + hash) + c; // hash * 33 + c
+ // return hash;
+ //}
+
+ // checks if the name matches any of the filters (and can be configured what to do when empty)
+ bool matchesAny(const char* name, const std::vector<String>& filters, bool matchEmpty,
+ bool caseSensitive) {
+ if(filters.empty() && matchEmpty)
+ return true;
+ for(auto& curr : filters)
+ if(wildcmp(name, curr.c_str(), caseSensitive))
+ return true;
+ return false;
+ }
+} // namespace
+namespace detail {
+
+ Subcase::Subcase(const String& name, const char* file, int line)
+ : m_signature({name, file, line}) {
+ auto* s = g_cs;
+
+ // check subcase filters
+ if(s->subcasesStack.size() < size_t(s->subcase_filter_levels)) {
+ if(!matchesAny(m_signature.m_name.c_str(), s->filters[6], true, s->case_sensitive))
+ return;
+ if(matchesAny(m_signature.m_name.c_str(), s->filters[7], false, s->case_sensitive))
+ return;
+ }
+
+ // if a Subcase on the same level has already been entered
+ if(s->subcasesStack.size() < size_t(s->subcasesCurrentMaxLevel)) {
+ s->should_reenter = true;
+ return;
+ }
+
+ // push the current signature to the stack so we can check if the
+ // current stack + the current new subcase have been traversed
+ s->subcasesStack.push_back(m_signature);
+ if(s->subcasesPassed.count(s->subcasesStack) != 0) {
+ // pop - revert to previous stack since we've already passed this
+ s->subcasesStack.pop_back();
+ return;
+ }
+
+ s->subcasesCurrentMaxLevel = s->subcasesStack.size();
+ m_entered = true;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_start, m_signature);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ Subcase::~Subcase() {
+ if(m_entered) {
+ // only mark the subcase stack as passed if no subcases have been skipped
+ if(g_cs->should_reenter == false)
+ g_cs->subcasesPassed.insert(g_cs->subcasesStack);
+ g_cs->subcasesStack.pop_back();
+
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0
+#else
+ if(std::uncaught_exception()
+#endif
+ && g_cs->shouldLogCurrentException) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(
+ test_case_exception, {"exception thrown in subcase - will translate later "
+ "when the whole test case has been exited (cannot "
+ "translate while there is an active exception)",
+ false});
+ g_cs->shouldLogCurrentException = false;
+ }
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ Subcase::operator bool() const { return m_entered; }
+
+ Result::Result(bool passed, const String& decomposition)
+ : m_passed(passed)
+ , m_decomp(decomposition) {}
+
+ ExpressionDecomposer::ExpressionDecomposer(assertType::Enum at)
+ : m_at(at) {}
+
+ TestSuite& TestSuite::operator*(const char* in) {
+ m_test_suite = in;
+ // clear state
+ m_description = nullptr;
+ m_skip = false;
+ m_no_breaks = false;
+ m_no_output = false;
+ m_may_fail = false;
+ m_should_fail = false;
+ m_expected_failures = 0;
+ m_timeout = 0;
+ return *this;
+ }
+
+ TestCase::TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type, int template_id) {
+ m_file = file;
+ m_line = line;
+ m_name = nullptr; // will be later overridden in operator*
+ m_test_suite = test_suite.m_test_suite;
+ m_description = test_suite.m_description;
+ m_skip = test_suite.m_skip;
+ m_no_breaks = test_suite.m_no_breaks;
+ m_no_output = test_suite.m_no_output;
+ m_may_fail = test_suite.m_may_fail;
+ m_should_fail = test_suite.m_should_fail;
+ m_expected_failures = test_suite.m_expected_failures;
+ m_timeout = test_suite.m_timeout;
+
+ m_test = test;
+ m_type = type;
+ m_template_id = template_id;
+ }
+
+ TestCase::TestCase(const TestCase& other)
+ : TestCaseData() {
+ *this = other;
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ DOCTEST_MSVC_SUPPRESS_WARNING(26437) // Do not slice
+ TestCase& TestCase::operator=(const TestCase& other) {
+ static_cast<TestCaseData&>(*this) = static_cast<const TestCaseData&>(other);
+
+ m_test = other.m_test;
+ m_type = other.m_type;
+ m_template_id = other.m_template_id;
+ m_full_name = other.m_full_name;
+
+ if(m_template_id != -1)
+ m_name = m_full_name.c_str();
+ return *this;
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& TestCase::operator*(const char* in) {
+ m_name = in;
+ // make a new name with an appended type for templated test case
+ if(m_template_id != -1) {
+ m_full_name = String(m_name) + m_type;
+ // redirect the name to point to the newly constructed full name
+ m_name = m_full_name.c_str();
+ }
+ return *this;
+ }
+
+ bool TestCase::operator<(const TestCase& other) const {
+ // this will be used only to differentiate between test cases - not relevant for sorting
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ const int name_cmp = strcmp(m_name, other.m_name);
+ if(name_cmp != 0)
+ return name_cmp < 0;
+ const int file_cmp = m_file.compare(other.m_file);
+ if(file_cmp != 0)
+ return file_cmp < 0;
+ return m_template_id < other.m_template_id;
+ }
+
+ // all the registered tests
+ std::set<TestCase>& getRegisteredTests() {
+ static std::set<TestCase> data;
+ return data;
+ }
+} // namespace detail
+namespace {
+ using namespace detail;
+ // for sorting tests by file/line
+ bool fileOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ // this is needed because MSVC gives different case for drive letters
+ // for __FILE__ when evaluated in a header and a source file
+ const int res = lhs->m_file.compare(rhs->m_file, bool(DOCTEST_MSVC));
+ if(res != 0)
+ return res < 0;
+ if(lhs->m_line != rhs->m_line)
+ return lhs->m_line < rhs->m_line;
+ return lhs->m_template_id < rhs->m_template_id;
+ }
+
+ // for sorting tests by suite/file/line
+ bool suiteOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_test_suite, rhs->m_test_suite);
+ if(res != 0)
+ return res < 0;
+ return fileOrderComparator(lhs, rhs);
+ }
+
+ // for sorting tests by name/suite/file/line
+ bool nameOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_name, rhs->m_name);
+ if(res != 0)
+ return res < 0;
+ return suiteOrderComparator(lhs, rhs);
+ }
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ HANDLE g_stdoutHandle;
+ WORD g_origFgAttrs;
+ WORD g_origBgAttrs;
+ bool g_attrsInitted = false;
+
+ int colors_init() {
+ if(!g_attrsInitted) {
+ g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+ g_attrsInitted = true;
+ CONSOLE_SCREEN_BUFFER_INFO csbiInfo;
+ GetConsoleScreenBufferInfo(g_stdoutHandle, &csbiInfo);
+ g_origFgAttrs = csbiInfo.wAttributes & ~(BACKGROUND_GREEN | BACKGROUND_RED |
+ BACKGROUND_BLUE | BACKGROUND_INTENSITY);
+ g_origBgAttrs = csbiInfo.wAttributes & ~(FOREGROUND_GREEN | FOREGROUND_RED |
+ FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ }
+ return 0;
+ }
+
+ int dumy_init_console_colors = colors_init();
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ void color_to_stream(std::ostream& s, Color::Enum code) {
+ static_cast<void>(s); // for DOCTEST_CONFIG_COLORS_NONE or DOCTEST_CONFIG_COLORS_WINDOWS
+ static_cast<void>(code); // for DOCTEST_CONFIG_COLORS_NONE
+#ifdef DOCTEST_CONFIG_COLORS_ANSI
+ if(g_no_colors ||
+ (isatty(STDOUT_FILENO) == false && getContextOptions()->force_colors == false))
+ return;
+
+ auto col = "";
+ // clang-format off
+ switch(code) { //!OCLINT missing break in switch statement / unnecessary default statement in covered switch statement
+ case Color::Red: col = "[0;31m"; break;
+ case Color::Green: col = "[0;32m"; break;
+ case Color::Blue: col = "[0;34m"; break;
+ case Color::Cyan: col = "[0;36m"; break;
+ case Color::Yellow: col = "[0;33m"; break;
+ case Color::Grey: col = "[1;30m"; break;
+ case Color::LightGrey: col = "[0;37m"; break;
+ case Color::BrightRed: col = "[1;31m"; break;
+ case Color::BrightGreen: col = "[1;32m"; break;
+ case Color::BrightWhite: col = "[1;37m"; break;
+ case Color::Bright: // invalid
+ case Color::None:
+ case Color::White:
+ default: col = "[0m";
+ }
+ // clang-format on
+ s << "\033" << col;
+#endif // DOCTEST_CONFIG_COLORS_ANSI
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ if(g_no_colors ||
+ (isatty(fileno(stdout)) == false && getContextOptions()->force_colors == false))
+ return;
+
+#define DOCTEST_SET_ATTR(x) SetConsoleTextAttribute(g_stdoutHandle, x | g_origBgAttrs)
+
+ // clang-format off
+ switch (code) {
+ case Color::White: DOCTEST_SET_ATTR(FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::Red: DOCTEST_SET_ATTR(FOREGROUND_RED); break;
+ case Color::Green: DOCTEST_SET_ATTR(FOREGROUND_GREEN); break;
+ case Color::Blue: DOCTEST_SET_ATTR(FOREGROUND_BLUE); break;
+ case Color::Cyan: DOCTEST_SET_ATTR(FOREGROUND_BLUE | FOREGROUND_GREEN); break;
+ case Color::Yellow: DOCTEST_SET_ATTR(FOREGROUND_RED | FOREGROUND_GREEN); break;
+ case Color::Grey: DOCTEST_SET_ATTR(0); break;
+ case Color::LightGrey: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY); break;
+ case Color::BrightRed: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_RED); break;
+ case Color::BrightGreen: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN); break;
+ case Color::BrightWhite: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::None:
+ case Color::Bright: // invalid
+ default: DOCTEST_SET_ATTR(g_origFgAttrs);
+ }
+ // clang-format on
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+ std::vector<const IExceptionTranslator*>& getExceptionTranslators() {
+ static std::vector<const IExceptionTranslator*> data;
+ return data;
+ }
+
+ String translateActiveException() {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ String res;
+ auto& translators = getExceptionTranslators();
+ for(auto& curr : translators)
+ if(curr->translate(res))
+ return res;
+ // clang-format off
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wcatch-value")
+ try {
+ throw;
+ } catch(std::exception& ex) {
+ return ex.what();
+ } catch(std::string& msg) {
+ return msg.c_str();
+ } catch(const char* msg) {
+ return msg;
+ } catch(...) {
+ return "unknown exception";
+ }
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+// clang-format on
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ return "";
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+} // namespace
+
+namespace detail {
+ // used by the macros for registering tests
+ int regTest(const TestCase& tc) {
+ getRegisteredTests().insert(tc);
+ return 0;
+ }
+
+ // sets the current test suite
+ int setTestSuite(const TestSuite& ts) {
+ doctest_detail_test_suite_ns::getCurrentTestSuite() = ts;
+ return 0;
+ }
+
+#ifdef DOCTEST_IS_DEBUGGER_ACTIVE
+ bool isDebuggerActive() { return DOCTEST_IS_DEBUGGER_ACTIVE(); }
+#else // DOCTEST_IS_DEBUGGER_ACTIVE
+#ifdef DOCTEST_PLATFORM_LINUX
+ class ErrnoGuard {
+ public:
+ ErrnoGuard() : m_oldErrno(errno) {}
+ ~ErrnoGuard() { errno = m_oldErrno; }
+ private:
+ int m_oldErrno;
+ };
+ // See the comments in Catch2 for the reasoning behind this implementation:
+ // https://github.com/catchorg/Catch2/blob/v2.13.1/include/internal/catch_debugger.cpp#L79-L102
+ bool isDebuggerActive() {
+ ErrnoGuard guard;
+ std::ifstream in("/proc/self/status");
+ for(std::string line; std::getline(in, line);) {
+ static const int PREFIX_LEN = 11;
+ if(line.compare(0, PREFIX_LEN, "TracerPid:\t") == 0) {
+ return line.length() > PREFIX_LEN && line[PREFIX_LEN] != '0';
+ }
+ }
+ return false;
+ }
+#elif defined(DOCTEST_PLATFORM_MAC)
+ // The following function is taken directly from the following technical note:
+ // https://developer.apple.com/library/archive/qa/qa1361/_index.html
+ // Returns true if the current process is being debugged (either
+ // running under the debugger or has a debugger attached post facto).
+ bool isDebuggerActive() {
+ int mib[4];
+ kinfo_proc info;
+ size_t size;
+ // Initialize the flags so that, if sysctl fails for some bizarre
+ // reason, we get a predictable result.
+ info.kp_proc.p_flag = 0;
+ // Initialize mib, which tells sysctl the info we want, in this case
+ // we're looking for information about a specific process ID.
+ mib[0] = CTL_KERN;
+ mib[1] = KERN_PROC;
+ mib[2] = KERN_PROC_PID;
+ mib[3] = getpid();
+ // Call sysctl.
+ size = sizeof(info);
+ if(sysctl(mib, DOCTEST_COUNTOF(mib), &info, &size, 0, 0) != 0) {
+ std::cerr << "\nCall to sysctl failed - unable to determine if debugger is active **\n";
+ return false;
+ }
+ // We're being debugged if the P_TRACED flag is set.
+ return ((info.kp_proc.p_flag & P_TRACED) != 0);
+ }
+#elif DOCTEST_MSVC || defined(__MINGW32__) || defined(__MINGW64__)
+ bool isDebuggerActive() { return ::IsDebuggerPresent() != 0; }
+#else
+ bool isDebuggerActive() { return false; }
+#endif // Platform
+#endif // DOCTEST_IS_DEBUGGER_ACTIVE
+
+ void registerExceptionTranslatorImpl(const IExceptionTranslator* et) {
+ if(std::find(getExceptionTranslators().begin(), getExceptionTranslators().end(), et) ==
+ getExceptionTranslators().end())
+ getExceptionTranslators().push_back(et);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, char* in) { *s << in; }
+ void toStream(std::ostream* s, const char* in) { *s << in; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, bool in) { *s << std::boolalpha << in << std::noboolalpha; }
+ void toStream(std::ostream* s, float in) { *s << in; }
+ void toStream(std::ostream* s, double in) { *s << in; }
+ void toStream(std::ostream* s, double long in) { *s << in; }
+
+ void toStream(std::ostream* s, char in) { *s << in; }
+ void toStream(std::ostream* s, char signed in) { *s << in; }
+ void toStream(std::ostream* s, char unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int short in) { *s << in; }
+ void toStream(std::ostream* s, int short unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int in) { *s << in; }
+ void toStream(std::ostream* s, int unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long in) { *s << in; }
+ void toStream(std::ostream* s, int long unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long long in) { *s << in; }
+ void toStream(std::ostream* s, int long long unsigned in) { *s << in; }
+
+ DOCTEST_THREAD_LOCAL std::vector<IContextScope*> g_infoContexts; // for logging with INFO()
+
+ ContextScopeBase::ContextScopeBase() {
+ g_infoContexts.push_back(this);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ // destroy cannot be inlined into the destructor because that would mean calling stringify after
+ // ContextScope has been destroyed (base class destructors run after derived class destructors).
+ // Instead, ContextScope calls this method directly from its destructor.
+ void ContextScopeBase::destroy() {
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0) {
+#else
+ if(std::uncaught_exception()) {
+#endif
+ std::ostringstream s;
+ this->stringify(&s);
+ g_cs->stringifiedContexts.push_back(s.str().c_str());
+ }
+ g_infoContexts.pop_back();
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+} // namespace detail
+namespace {
+ using namespace detail;
+
+#if !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ struct FatalConditionHandler
+ {
+ static void reset() {}
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+ };
+#else // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+ void reportFatal(const std::string&);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ DWORD id;
+ const char* name;
+ };
+ // There is no 1-1 mapping between signals and windows exceptions.
+ // Windows can easily distinguish between SO and SigSegV,
+ // but SigInt, SigTerm, etc are handled differently.
+ SignalDefs signalDefs[] = {
+ {static_cast<DWORD>(EXCEPTION_ILLEGAL_INSTRUCTION),
+ "SIGILL - Illegal instruction signal"},
+ {static_cast<DWORD>(EXCEPTION_STACK_OVERFLOW), "SIGSEGV - Stack overflow"},
+ {static_cast<DWORD>(EXCEPTION_ACCESS_VIOLATION),
+ "SIGSEGV - Segmentation violation signal"},
+ {static_cast<DWORD>(EXCEPTION_INT_DIVIDE_BY_ZERO), "Divide by zero error"},
+ };
+
+ struct FatalConditionHandler
+ {
+ static LONG CALLBACK handleException(PEXCEPTION_POINTERS ExceptionInfo) {
+ // Multiple threads may enter this filter/handler at once. We want the error message to be printed on the
+ // console just once no matter how many threads have crashed.
+ static std::mutex mutex;
+ static bool execute = true;
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ if(execute) {
+ bool reported = false;
+ for(size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ if(ExceptionInfo->ExceptionRecord->ExceptionCode == signalDefs[i].id) {
+ reportFatal(signalDefs[i].name);
+ reported = true;
+ break;
+ }
+ }
+ if(reported == false)
+ reportFatal("Unhandled SEH exception caught");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ }
+ execute = false;
+ }
+ std::exit(EXIT_FAILURE);
+ }
+
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+
+ FatalConditionHandler() {
+ isSet = true;
+ // 32k seems enough for doctest to handle stack overflow,
+ // but the value was found experimentally, so there is no strong guarantee
+ guaranteeSize = 32 * 1024;
+ // Register an unhandled exception filter
+ previousTop = SetUnhandledExceptionFilter(handleException);
+ // Pass in guarantee size to be filled
+ SetThreadStackGuarantee(&guaranteeSize);
+
+ // On Windows uncaught exceptions from another thread, exceptions from
+ // destructors, or calls to std::terminate are not a SEH exception
+
+ // The terminal handler gets called when:
+ // - std::terminate is called FROM THE TEST RUNNER THREAD
+ // - an exception is thrown from a destructor FROM THE TEST RUNNER THREAD
+ original_terminate_handler = std::get_terminate();
+ std::set_terminate([]() DOCTEST_NOEXCEPT {
+ reportFatal("Terminate handler called");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE); // explicitly exit - otherwise the SIGABRT handler may be called as well
+ });
+
+ // SIGABRT is raised when:
+ // - std::terminate is called FROM A DIFFERENT THREAD
+ // - an exception is thrown from a destructor FROM A DIFFERENT THREAD
+ // - an uncaught exception is thrown FROM A DIFFERENT THREAD
+ prev_sigabrt_handler = std::signal(SIGABRT, [](int signal) DOCTEST_NOEXCEPT {
+ if(signal == SIGABRT) {
+ reportFatal("SIGABRT - Abort (abnormal termination) signal");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE);
+ }
+ });
+
+ // The following settings are taken from google test, and more
+ // specifically from UnitTest::Run() inside of gtest.cc
+
+ // the user does not want to see pop-up dialogs about crashes
+ prev_error_mode_1 = SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT |
+ SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX);
+ // This forces the abort message to go to stderr in all circumstances.
+ prev_error_mode_2 = _set_error_mode(_OUT_TO_STDERR);
+ // In the debug version, Visual Studio pops up a separate dialog
+ // offering a choice to debug the aborted program - we want to disable that.
+ prev_abort_behavior = _set_abort_behavior(0x0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ // In debug mode, the Windows CRT can crash with an assertion over invalid
+ // input (e.g. passing an invalid file descriptor). The default handling
+ // for these assertions is to pop up a dialog and wait for user input.
+ // Instead ask the CRT to dump such assertions to stderr non-interactively.
+ prev_report_mode = _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
+ prev_report_file = _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR);
+ }
+
+ static void reset() {
+ if(isSet) {
+ // Unregister handler and restore the old guarantee
+ SetUnhandledExceptionFilter(previousTop);
+ SetThreadStackGuarantee(&guaranteeSize);
+ std::set_terminate(original_terminate_handler);
+ std::signal(SIGABRT, prev_sigabrt_handler);
+ SetErrorMode(prev_error_mode_1);
+ _set_error_mode(prev_error_mode_2);
+ _set_abort_behavior(prev_abort_behavior, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ static_cast<void>(_CrtSetReportMode(_CRT_ASSERT, prev_report_mode));
+ static_cast<void>(_CrtSetReportFile(_CRT_ASSERT, prev_report_file));
+ isSet = false;
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+
+ private:
+ static UINT prev_error_mode_1;
+ static int prev_error_mode_2;
+ static unsigned int prev_abort_behavior;
+ static int prev_report_mode;
+ static _HFILE prev_report_file;
+ static void (*prev_sigabrt_handler)(int);
+ static std::terminate_handler original_terminate_handler;
+ static bool isSet;
+ static ULONG guaranteeSize;
+ static LPTOP_LEVEL_EXCEPTION_FILTER previousTop;
+ };
+
+ UINT FatalConditionHandler::prev_error_mode_1;
+ int FatalConditionHandler::prev_error_mode_2;
+ unsigned int FatalConditionHandler::prev_abort_behavior;
+ int FatalConditionHandler::prev_report_mode;
+ _HFILE FatalConditionHandler::prev_report_file;
+ void (*FatalConditionHandler::prev_sigabrt_handler)(int);
+ std::terminate_handler FatalConditionHandler::original_terminate_handler;
+ bool FatalConditionHandler::isSet = false;
+ ULONG FatalConditionHandler::guaranteeSize = 0;
+ LPTOP_LEVEL_EXCEPTION_FILTER FatalConditionHandler::previousTop = nullptr;
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ int id;
+ const char* name;
+ };
+ SignalDefs signalDefs[] = {{SIGINT, "SIGINT - Terminal interrupt signal"},
+ {SIGILL, "SIGILL - Illegal instruction signal"},
+ {SIGFPE, "SIGFPE - Floating point error signal"},
+ {SIGSEGV, "SIGSEGV - Segmentation violation signal"},
+ {SIGTERM, "SIGTERM - Termination request signal"},
+ {SIGABRT, "SIGABRT - Abort (abnormal termination) signal"}};
+
+ struct FatalConditionHandler
+ {
+ static bool isSet;
+ static struct sigaction oldSigActions[DOCTEST_COUNTOF(signalDefs)];
+ static stack_t oldSigStack;
+ static size_t altStackSize;
+ static char* altStackMem;
+
+ static void handleSignal(int sig) {
+ const char* name = "<unknown signal>";
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ SignalDefs& def = signalDefs[i];
+ if(sig == def.id) {
+ name = def.name;
+ break;
+ }
+ }
+ reset();
+ reportFatal(name);
+ raise(sig);
+ }
+
+ static void allocateAltStackMem() {
+ altStackMem = new char[altStackSize];
+ }
+
+ static void freeAltStackMem() {
+ delete[] altStackMem;
+ }
+
+ FatalConditionHandler() {
+ isSet = true;
+ stack_t sigStack;
+ sigStack.ss_sp = altStackMem;
+ sigStack.ss_size = altStackSize;
+ sigStack.ss_flags = 0;
+ sigaltstack(&sigStack, &oldSigStack);
+ struct sigaction sa = {};
+ sa.sa_handler = handleSignal; // NOLINT
+ sa.sa_flags = SA_ONSTACK;
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &sa, &oldSigActions[i]);
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+ static void reset() {
+ if(isSet) {
+ // Set signals back to previous values -- hopefully nobody overwrote them in the meantime
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &oldSigActions[i], nullptr);
+ }
+ // Return the old stack
+ sigaltstack(&oldSigStack, nullptr);
+ isSet = false;
+ }
+ }
+ };
+
+ bool FatalConditionHandler::isSet = false;
+ struct sigaction FatalConditionHandler::oldSigActions[DOCTEST_COUNTOF(signalDefs)] = {};
+ stack_t FatalConditionHandler::oldSigStack = {};
+ size_t FatalConditionHandler::altStackSize = 4 * SIGSTKSZ;
+ char* FatalConditionHandler::altStackMem = nullptr;
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+} // namespace
+
+namespace {
+ using namespace detail;
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) ::OutputDebugStringA(text)
+#else
+ // TODO: integration with XCode and other IDEs
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) // NOLINT(clang-diagnostic-unused-macros)
+#endif // Platform
+
+ void addAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsCurrentTest_atomic++;
+ }
+
+ void addFailedAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsFailedCurrentTest_atomic++;
+ }
+
+#if defined(DOCTEST_CONFIG_POSIX_SIGNALS) || defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ void reportFatal(const std::string& message) {
+ g_cs->failure_flags |= TestCaseFailureReason::Crash;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception, {message.c_str(), true});
+
+ while(g_cs->subcasesStack.size()) {
+ g_cs->subcasesStack.pop_back();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+
+ g_cs->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ }
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+} // namespace
+namespace detail {
+
+ ResultBuilder::ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type, const char* exception_string) {
+ m_test_case = g_cs->currentTest;
+ m_at = at;
+ m_file = file;
+ m_line = line;
+ m_expr = expr;
+ m_failed = true;
+ m_threw = false;
+ m_threw_as = false;
+ m_exception_type = exception_type;
+ m_exception_string = exception_string;
+#if DOCTEST_MSVC
+ if(m_expr[0] == ' ') // this happens when variadic macros are disabled under MSVC
+ ++m_expr;
+#endif // MSVC
+ }
+
+ void ResultBuilder::setResult(const Result& res) {
+ m_decomp = res.m_decomp;
+ m_failed = !res.m_passed;
+ }
+
+ void ResultBuilder::translateException() {
+ m_threw = true;
+ m_exception = translateActiveException();
+ }
+
+ bool ResultBuilder::log() {
+ if(m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw;
+ } else if((m_at & assertType::is_throws_as) && (m_at & assertType::is_throws_with)) { //!OCLINT
+ m_failed = !m_threw_as || (m_exception != m_exception_string);
+ } else if(m_at & assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw_as;
+ } else if(m_at & assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ m_failed = m_exception != m_exception_string;
+ } else if(m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ m_failed = m_threw;
+ }
+
+ if(m_exception.size())
+ m_exception = String("\"") + m_exception + "\"";
+
+ if(is_running_in_test) {
+ addAssert(m_at);
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_assert, *this);
+
+ if(m_failed)
+ addFailedAssert(m_at);
+ } else if(m_failed) {
+ failed_out_of_a_testing_context(*this);
+ }
+
+ return m_failed && isDebuggerActive() && !getContextOptions()->no_breaks &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void ResultBuilder::react() const {
+ if(m_failed && checkIfShouldThrow(m_at))
+ throwException();
+ }
+
+ void failed_out_of_a_testing_context(const AssertData& ad) {
+ if(g_cs->ah)
+ g_cs->ah(ad);
+ else
+ std::abort();
+ }
+
+ void decomp_assert(assertType::Enum at, const char* file, int line, const char* expr,
+ Result result) {
+ bool failed = !result.m_passed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(result.m_decomp);
+ DOCTEST_ASSERT_IN_TESTS(result.m_decomp);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ }
+
+ MessageBuilder::MessageBuilder(const char* file, int line, assertType::Enum severity) {
+ m_stream = getTlsOss();
+ m_file = file;
+ m_line = line;
+ m_severity = severity;
+ }
+
+ IExceptionTranslator::IExceptionTranslator() = default;
+ IExceptionTranslator::~IExceptionTranslator() = default;
+
+ bool MessageBuilder::log() {
+ m_string = getTlsOssResult();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_message, *this);
+
+ const bool isWarn = m_severity & assertType::is_warn;
+
+ // warn is just a message in this context so we don't treat it as an assert
+ if(!isWarn) {
+ addAssert(m_severity);
+ addFailedAssert(m_severity);
+ }
+
+ return isDebuggerActive() && !getContextOptions()->no_breaks && !isWarn &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void MessageBuilder::react() {
+ if(m_severity & assertType::is_require) //!OCLINT bitwise operator in conditional
+ throwException();
+ }
+
+ MessageBuilder::~MessageBuilder() = default;
+} // namespace detail
+namespace {
+ using namespace detail;
+
+ template <typename Ex>
+ DOCTEST_NORETURN void throw_exception(Ex const& e) {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ throw e;
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ std::cerr << "doctest will terminate because it needed to throw an exception.\n"
+ << "The message was: " << e.what() << '\n';
+ std::terminate();
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+
+#ifndef DOCTEST_INTERNAL_ERROR
+#define DOCTEST_INTERNAL_ERROR(msg) \
+ throw_exception(std::logic_error( \
+ __FILE__ ":" DOCTEST_TOSTR(__LINE__) ": Internal doctest error: " msg))
+#endif // DOCTEST_INTERNAL_ERROR
+
+ // clang-format off
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+ class XmlEncode {
+ public:
+ enum ForWhat { ForTextNodes, ForAttributes };
+
+ XmlEncode( std::string const& str, ForWhat forWhat = ForTextNodes );
+
+ void encodeTo( std::ostream& os ) const;
+
+ friend std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode );
+
+ private:
+ std::string m_str;
+ ForWhat m_forWhat;
+ };
+
+ class XmlWriter {
+ public:
+
+ class ScopedElement {
+ public:
+ ScopedElement( XmlWriter* writer );
+
+ ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+ ScopedElement& operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+
+ ~ScopedElement();
+
+ ScopedElement& writeText( std::string const& text, bool indent = true );
+
+ template<typename T>
+ ScopedElement& writeAttribute( std::string const& name, T const& attribute ) {
+ m_writer->writeAttribute( name, attribute );
+ return *this;
+ }
+
+ private:
+ mutable XmlWriter* m_writer = nullptr;
+ };
+
+ XmlWriter( std::ostream& os = std::cout );
+ ~XmlWriter();
+
+ XmlWriter( XmlWriter const& ) = delete;
+ XmlWriter& operator=( XmlWriter const& ) = delete;
+
+ XmlWriter& startElement( std::string const& name );
+
+ ScopedElement scopedElement( std::string const& name );
+
+ XmlWriter& endElement();
+
+ XmlWriter& writeAttribute( std::string const& name, std::string const& attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, const char* attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, bool attribute );
+
+ template<typename T>
+ XmlWriter& writeAttribute( std::string const& name, T const& attribute ) {
+ std::stringstream rss;
+ rss << attribute;
+ return writeAttribute( name, rss.str() );
+ }
+
+ XmlWriter& writeText( std::string const& text, bool indent = true );
+
+ //XmlWriter& writeComment( std::string const& text );
+
+ //void writeStylesheetRef( std::string const& url );
+
+ //XmlWriter& writeBlankLine();
+
+ void ensureTagClosed();
+
+ private:
+
+ void writeDeclaration();
+
+ void newlineIfNecessary();
+
+ bool m_tagIsOpen = false;
+ bool m_needsNewline = false;
+ std::vector<std::string> m_tags;
+ std::string m_indent;
+ std::ostream& m_os;
+ };
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+using uchar = unsigned char;
+
+namespace {
+
+ size_t trailingBytes(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return 2;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return 3;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return 4;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ uint32_t headerValue(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return c & 0x1F;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return c & 0x0F;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return c & 0x07;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ void hexEscapeChar(std::ostream& os, unsigned char c) {
+ std::ios_base::fmtflags f(os.flags());
+ os << "\\x"
+ << std::uppercase << std::hex << std::setfill('0') << std::setw(2)
+ << static_cast<int>(c);
+ os.flags(f);
+ }
+
+} // anonymous namespace
+
+ XmlEncode::XmlEncode( std::string const& str, ForWhat forWhat )
+ : m_str( str ),
+ m_forWhat( forWhat )
+ {}
+
+ void XmlEncode::encodeTo( std::ostream& os ) const {
+ // Apostrophe escaping not necessary if we always use " to write attributes
+ // (see: https://www.w3.org/TR/xml/#syntax)
+
+ for( std::size_t idx = 0; idx < m_str.size(); ++ idx ) {
+ uchar c = m_str[idx];
+ switch (c) {
+ case '<': os << "&lt;"; break;
+ case '&': os << "&amp;"; break;
+
+ case '>':
+ // See: https://www.w3.org/TR/xml/#syntax
+ if (idx > 2 && m_str[idx - 1] == ']' && m_str[idx - 2] == ']')
+ os << "&gt;";
+ else
+ os << c;
+ break;
+
+ case '\"':
+ if (m_forWhat == ForAttributes)
+ os << "&quot;";
+ else
+ os << c;
+ break;
+
+ default:
+ // Check for control characters and invalid utf-8
+
+ // Escape control characters in standard ascii
+ // see https://stackoverflow.com/questions/404107/why-are-control-characters-illegal-in-xml-1-0
+ if (c < 0x09 || (c > 0x0D && c < 0x20) || c == 0x7F) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // Plain ASCII: Write it to stream
+ if (c < 0x7F) {
+ os << c;
+ break;
+ }
+
+ // UTF-8 territory
+ // Check if the encoding is valid and if it is not, hex escape bytes.
+ // Important: We do not check the exact decoded values for validity, only the encoding format
+ // First check that this bytes is a valid lead byte:
+ // This means that it is not encoded as 1111 1XXX
+ // Or as 10XX XXXX
+ if (c < 0xC0 ||
+ c >= 0xF8) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ auto encBytes = trailingBytes(c);
+ // Are there enough bytes left to avoid accessing out-of-bounds memory?
+ if (idx + encBytes - 1 >= m_str.size()) {
+ hexEscapeChar(os, c);
+ break;
+ }
+ // The header is valid, check data
+ // The next encBytes bytes must together be a valid utf-8
+ // This means: bitpattern 10XX XXXX and the extracted value is sane (ish)
+ bool valid = true;
+ uint32_t value = headerValue(c);
+ for (std::size_t n = 1; n < encBytes; ++n) {
+ uchar nc = m_str[idx + n];
+ valid &= ((nc & 0xC0) == 0x80);
+ value = (value << 6) | (nc & 0x3F);
+ }
+
+ if (
+ // Wrong bit pattern of following bytes
+ (!valid) ||
+ // Overlong encodings
+ (value < 0x80) ||
+ ( value < 0x800 && encBytes > 2) || // removed "0x80 <= value &&" because redundant
+ (0x800 < value && value < 0x10000 && encBytes > 3) ||
+ // Encoded value out of range
+ (value >= 0x110000)
+ ) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // If we got here, this is in fact a valid(ish) utf-8 sequence
+ for (std::size_t n = 0; n < encBytes; ++n) {
+ os << m_str[idx + n];
+ }
+ idx += encBytes - 1;
+ break;
+ }
+ }
+ }
+
+ std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode ) {
+ xmlEncode.encodeTo( os );
+ return os;
+ }
+
+ XmlWriter::ScopedElement::ScopedElement( XmlWriter* writer )
+ : m_writer( writer )
+ {}
+
+ XmlWriter::ScopedElement::ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT
+ : m_writer( other.m_writer ){
+ other.m_writer = nullptr;
+ }
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT {
+ if ( m_writer ) {
+ m_writer->endElement();
+ }
+ m_writer = other.m_writer;
+ other.m_writer = nullptr;
+ return *this;
+ }
+
+
+ XmlWriter::ScopedElement::~ScopedElement() {
+ if( m_writer )
+ m_writer->endElement();
+ }
+
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::writeText( std::string const& text, bool indent ) {
+ m_writer->writeText( text, indent );
+ return *this;
+ }
+
+ XmlWriter::XmlWriter( std::ostream& os ) : m_os( os )
+ {
+ writeDeclaration();
+ }
+
+ XmlWriter::~XmlWriter() {
+ while( !m_tags.empty() )
+ endElement();
+ }
+
+ XmlWriter& XmlWriter::startElement( std::string const& name ) {
+ ensureTagClosed();
+ newlineIfNecessary();
+ m_os << m_indent << '<' << name;
+ m_tags.push_back( name );
+ m_indent += " ";
+ m_tagIsOpen = true;
+ return *this;
+ }
+
+ XmlWriter::ScopedElement XmlWriter::scopedElement( std::string const& name ) {
+ ScopedElement scoped( this );
+ startElement( name );
+ return scoped;
+ }
+
+ XmlWriter& XmlWriter::endElement() {
+ newlineIfNecessary();
+ m_indent = m_indent.substr( 0, m_indent.size()-2 );
+ if( m_tagIsOpen ) {
+ m_os << "/>";
+ m_tagIsOpen = false;
+ }
+ else {
+ m_os << m_indent << "</" << m_tags.back() << ">";
+ }
+ m_os << std::endl;
+ m_tags.pop_back();
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, std::string const& attribute ) {
+ if( !name.empty() && !attribute.empty() )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, const char* attribute ) {
+ if( !name.empty() && attribute && attribute[0] != '\0' )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, bool attribute ) {
+ m_os << ' ' << name << "=\"" << ( attribute ? "true" : "false" ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeText( std::string const& text, bool indent ) {
+ if( !text.empty() ){
+ bool tagWasOpen = m_tagIsOpen;
+ ensureTagClosed();
+ if( tagWasOpen && indent )
+ m_os << m_indent;
+ m_os << XmlEncode( text );
+ m_needsNewline = true;
+ }
+ return *this;
+ }
+
+ //XmlWriter& XmlWriter::writeComment( std::string const& text ) {
+ // ensureTagClosed();
+ // m_os << m_indent << "<!--" << text << "-->";
+ // m_needsNewline = true;
+ // return *this;
+ //}
+
+ //void XmlWriter::writeStylesheetRef( std::string const& url ) {
+ // m_os << "<?xml-stylesheet type=\"text/xsl\" href=\"" << url << "\"?>\n";
+ //}
+
+ //XmlWriter& XmlWriter::writeBlankLine() {
+ // ensureTagClosed();
+ // m_os << '\n';
+ // return *this;
+ //}
+
+ void XmlWriter::ensureTagClosed() {
+ if( m_tagIsOpen ) {
+ m_os << ">" << std::endl;
+ m_tagIsOpen = false;
+ }
+ }
+
+ void XmlWriter::writeDeclaration() {
+ m_os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+ }
+
+ void XmlWriter::newlineIfNecessary() {
+ if( m_needsNewline ) {
+ m_os << std::endl;
+ m_needsNewline = false;
+ }
+ }
+
+// =================================================================================================
+// End of copy-pasted code from Catch
+// =================================================================================================
+
+ // clang-format on
+
+ struct XmlReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ XmlReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+ std::stringstream ss;
+ for(int i = 0; i < num_contexts; ++i) {
+ contexts[i]->stringify(&ss);
+ xml.scopedElement("Info").writeText(ss.str());
+ ss.str("");
+ }
+ }
+ }
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ void test_case_start_impl(const TestCaseData& in) {
+ bool open_ts_tag = false;
+ if(tc != nullptr) { // we have already opened a test suite
+ if(std::strcmp(tc->m_test_suite, in.m_test_suite) != 0) {
+ xml.endElement();
+ open_ts_tag = true;
+ }
+ }
+ else {
+ open_ts_tag = true; // first test case ==> first test suite
+ }
+
+ if(open_ts_tag) {
+ xml.startElement("TestSuite");
+ xml.writeAttribute("name", in.m_test_suite);
+ }
+
+ tc = &in;
+ xml.startElement("TestCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file.c_str()))
+ .writeAttribute("line", line(in.m_line))
+ .writeAttribute("description", in.m_description);
+
+ if(Approx(in.m_timeout) != 0)
+ xml.writeAttribute("timeout", in.m_timeout);
+ if(in.m_may_fail)
+ xml.writeAttribute("may_fail", true);
+ if(in.m_should_fail)
+ xml.writeAttribute("should_fail", true);
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ test_run_start();
+ if(opt.list_reporters) {
+ for(auto& curr : getListeners())
+ xml.scopedElement("Listener")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ for(auto& curr : getReporters())
+ xml.scopedElement("Reporter")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ } else if(opt.count || opt.list_test_cases) {
+ for(unsigned i = 0; i < in.num_data; ++i) {
+ xml.scopedElement("TestCase").writeAttribute("name", in.data[i]->m_name)
+ .writeAttribute("testsuite", in.data[i]->m_test_suite)
+ .writeAttribute("filename", skipPathFromFilename(in.data[i]->m_file.c_str()))
+ .writeAttribute("line", line(in.data[i]->m_line));
+ }
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ } else if(opt.list_test_suites) {
+ for(unsigned i = 0; i < in.num_data; ++i)
+ xml.scopedElement("TestSuite").writeAttribute("name", in.data[i]->m_test_suite);
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ xml.scopedElement("OverallResultsTestSuites")
+ .writeAttribute("unskipped", in.run_stats->numTestSuitesPassingFilters);
+ }
+ xml.endElement();
+ }
+
+ void test_run_start() override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ xml.startElement("doctest").writeAttribute("binary", binary_name);
+ if(opt.no_version == false)
+ xml.writeAttribute("version", DOCTEST_VERSION_STR);
+
+ // only the consequential ones (TODO: filters)
+ xml.scopedElement("Options")
+ .writeAttribute("order_by", opt.order_by.c_str())
+ .writeAttribute("rand_seed", opt.rand_seed)
+ .writeAttribute("first", opt.first)
+ .writeAttribute("last", opt.last)
+ .writeAttribute("abort_after", opt.abort_after)
+ .writeAttribute("subcase_filter_levels", opt.subcase_filter_levels)
+ .writeAttribute("case_sensitive", opt.case_sensitive)
+ .writeAttribute("no_throw", opt.no_throw)
+ .writeAttribute("no_skip", opt.no_skip);
+ }
+
+ void test_run_end(const TestRunStats& p) override {
+ if(tc) // the TestSuite tag - only if there has been at least 1 test case
+ xml.endElement();
+
+ xml.scopedElement("OverallResultsAsserts")
+ .writeAttribute("successes", p.numAsserts - p.numAssertsFailed)
+ .writeAttribute("failures", p.numAssertsFailed);
+
+ xml.startElement("OverallResultsTestCases")
+ .writeAttribute("successes",
+ p.numTestCasesPassingFilters - p.numTestCasesFailed)
+ .writeAttribute("failures", p.numTestCasesFailed);
+ if(opt.no_skipped_summary == false)
+ xml.writeAttribute("skipped", p.numTestCases - p.numTestCasesPassingFilters);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ test_case_start_impl(in);
+ xml.ensureTagClosed();
+ }
+
+ void test_case_reenter(const TestCaseData&) override {}
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ xml.startElement("OverallResultsAsserts")
+ .writeAttribute("successes",
+ st.numAssertsCurrentTest - st.numAssertsFailedCurrentTest)
+ .writeAttribute("failures", st.numAssertsFailedCurrentTest);
+ if(opt.duration)
+ xml.writeAttribute("duration", st.seconds);
+ if(tc->m_expected_failures)
+ xml.writeAttribute("expected_failures", tc->m_expected_failures);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.scopedElement("Exception")
+ .writeAttribute("crash", e.is_crash)
+ .writeText(e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("SubCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file))
+ .writeAttribute("line", line(in.m_line));
+ xml.ensureTagClosed();
+ }
+
+ void subcase_end() override { xml.endElement(); }
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed && !opt.success)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Expression")
+ .writeAttribute("success", !rb.m_failed)
+ .writeAttribute("type", assertString(rb.m_at))
+ .writeAttribute("filename", skipPathFromFilename(rb.m_file))
+ .writeAttribute("line", line(rb.m_line));
+
+ xml.scopedElement("Original").writeText(rb.m_expr);
+
+ if(rb.m_threw)
+ xml.scopedElement("Exception").writeText(rb.m_exception.c_str());
+
+ if(rb.m_at & assertType::is_throws_as)
+ xml.scopedElement("ExpectedException").writeText(rb.m_exception_type);
+ if(rb.m_at & assertType::is_throws_with)
+ xml.scopedElement("ExpectedExceptionString").writeText(rb.m_exception_string);
+ if((rb.m_at & assertType::is_normal) && !rb.m_threw)
+ xml.scopedElement("Expanded").writeText(rb.m_decomp.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void log_message(const MessageData& mb) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Message")
+ .writeAttribute("type", failureString(mb.m_severity))
+ .writeAttribute("filename", skipPathFromFilename(mb.m_file))
+ .writeAttribute("line", line(mb.m_line));
+
+ xml.scopedElement("Text").writeText(mb.m_string.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void test_case_skipped(const TestCaseData& in) override {
+ if(opt.no_skipped_summary == false) {
+ test_case_start_impl(in);
+ xml.writeAttribute("skipped", "true");
+ xml.endElement();
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("xml", 0, XmlReporter);
+
+ void fulltext_log_assert_to_stream(std::ostream& s, const AssertData& rb) {
+ if((rb.m_at & (assertType::is_throws_as | assertType::is_throws_with)) ==
+ 0) //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << " ) "
+ << Color::None;
+
+ if(rb.m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "threw as expected!" : "did NOT throw at all!") << "\n";
+ } else if((rb.m_at & assertType::is_throws_as) &&
+ (rb.m_at & assertType::is_throws_with)) { //!OCLINT
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\", " << rb.m_exception_type << " ) " << Color::None;
+ if(rb.m_threw) {
+ if(!rb.m_failed) {
+ s << "threw as expected!\n";
+ } else {
+ s << "threw a DIFFERENT exception! (contents: " << rb.m_exception << ")\n";
+ }
+ } else {
+ s << "did NOT throw at all!\n";
+ }
+ } else if(rb.m_at &
+ assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", "
+ << rb.m_exception_type << " ) " << Color::None
+ << (rb.m_threw ? (rb.m_threw_as ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at &
+ assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\" ) " << Color::None
+ << (rb.m_threw ? (!rb.m_failed ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "THREW exception: " : "didn't throw!") << Color::Cyan
+ << rb.m_exception << "\n";
+ } else {
+ s << (rb.m_threw ? "THREW exception: " :
+ (!rb.m_failed ? "is correct!\n" : "is NOT correct!\n"));
+ if(rb.m_threw)
+ s << rb.m_exception << "\n";
+ else
+ s << " values: " << assertString(rb.m_at) << "( " << rb.m_decomp << " )\n";
+ }
+ }
+
+ // TODO:
+ // - log_message()
+ // - respond to queries
+ // - honor remaining options
+ // - more attributes in tags
+ struct JUnitReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+ Timer timer;
+ std::vector<String> deepestSubcaseStackNames;
+
+ struct JUnitTestCaseData
+ {
+ static std::string getCurrentTimestamp() {
+ // Beware, this is not reentrant because of backward compatibility issues
+ // Also, UTC only, again because of backward compatibility (%z is C++11)
+ time_t rawtime;
+ std::time(&rawtime);
+ auto const timeStampSize = sizeof("2017-01-16T17:06:45Z");
+
+ std::tm timeInfo;
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ gmtime_s(&timeInfo, &rawtime);
+#else // DOCTEST_PLATFORM_WINDOWS
+ gmtime_r(&rawtime, &timeInfo);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ char timeStamp[timeStampSize];
+ const char* const fmt = "%Y-%m-%dT%H:%M:%SZ";
+
+ std::strftime(timeStamp, timeStampSize, fmt, &timeInfo);
+ return std::string(timeStamp);
+ }
+
+ struct JUnitTestMessage
+ {
+ JUnitTestMessage(const std::string& _message, const std::string& _type, const std::string& _details)
+ : message(_message), type(_type), details(_details) {}
+
+ JUnitTestMessage(const std::string& _message, const std::string& _details)
+ : message(_message), type(), details(_details) {}
+
+ std::string message, type, details;
+ };
+
+ struct JUnitTestCase
+ {
+ JUnitTestCase(const std::string& _classname, const std::string& _name)
+ : classname(_classname), name(_name), time(0), failures() {}
+
+ std::string classname, name;
+ double time;
+ std::vector<JUnitTestMessage> failures, errors;
+ };
+
+ void add(const std::string& classname, const std::string& name) {
+ testcases.emplace_back(classname, name);
+ }
+
+ void appendSubcaseNamesToLastTestcase(std::vector<String> nameStack) {
+ for(auto& curr: nameStack)
+ if(curr.size())
+ testcases.back().name += std::string("/") + curr.c_str();
+ }
+
+ void addTime(double time) {
+ if(time < 1e-4)
+ time = 0;
+ testcases.back().time = time;
+ totalSeconds += time;
+ }
+
+ void addFailure(const std::string& message, const std::string& type, const std::string& details) {
+ testcases.back().failures.emplace_back(message, type, details);
+ ++totalFailures;
+ }
+
+ void addError(const std::string& message, const std::string& details) {
+ testcases.back().errors.emplace_back(message, details);
+ ++totalErrors;
+ }
+
+ std::vector<JUnitTestCase> testcases;
+ double totalSeconds = 0;
+ int totalErrors = 0, totalFailures = 0;
+ };
+
+ JUnitTestCaseData testCaseData;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ JUnitReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData&) override {}
+
+ void test_run_start() override {}
+
+ void test_run_end(const TestRunStats& p) override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+ xml.startElement("testsuites");
+ xml.startElement("testsuite").writeAttribute("name", binary_name)
+ .writeAttribute("errors", testCaseData.totalErrors)
+ .writeAttribute("failures", testCaseData.totalFailures)
+ .writeAttribute("tests", p.numAsserts);
+ if(opt.no_time_in_output == false) {
+ xml.writeAttribute("time", testCaseData.totalSeconds);
+ xml.writeAttribute("timestamp", JUnitTestCaseData::getCurrentTimestamp());
+ }
+ if(opt.no_version == false)
+ xml.writeAttribute("doctest_version", DOCTEST_VERSION_STR);
+
+ for(const auto& testCase : testCaseData.testcases) {
+ xml.startElement("testcase")
+ .writeAttribute("classname", testCase.classname)
+ .writeAttribute("name", testCase.name);
+ if(opt.no_time_in_output == false)
+ xml.writeAttribute("time", testCase.time);
+ // This is not ideal, but it should be enough to mimic gtest's junit output.
+ xml.writeAttribute("status", "run");
+
+ for(const auto& failure : testCase.failures) {
+ xml.scopedElement("failure")
+ .writeAttribute("message", failure.message)
+ .writeAttribute("type", failure.type)
+ .writeText(failure.details, false);
+ }
+
+ for(const auto& error : testCase.errors) {
+ xml.scopedElement("error")
+ .writeAttribute("message", error.message)
+ .writeText(error.details);
+ }
+
+ xml.endElement();
+ }
+ xml.endElement();
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ timer.start();
+ }
+
+ void test_case_reenter(const TestCaseData& in) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+
+ timer.start();
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ }
+
+ void test_case_end(const CurrentTestCaseStats&) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ testCaseData.addError("exception", e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ deepestSubcaseStackNames.push_back(in.m_name);
+ }
+
+ void subcase_end() override {}
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed) // report only failures & ignore the `success` option
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ std::ostringstream os;
+ os << skipPathFromFilename(rb.m_file) << (opt.gnu_file_line ? ":" : "(")
+ << line(rb.m_line) << (opt.gnu_file_line ? ":" : "):") << std::endl;
+
+ fulltext_log_assert_to_stream(os, rb);
+ log_contexts(os);
+ testCaseData.addFailure(rb.m_decomp.c_str(), assertString(rb.m_at), os.str());
+ }
+
+ void log_message(const MessageData&) override {}
+
+ void test_case_skipped(const TestCaseData&) override {}
+
+ void log_contexts(std::ostringstream& s) {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << std::endl;
+ }
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("junit", 0, JUnitReporter);
+
+ struct Whitespace
+ {
+ int nrSpaces;
+ explicit Whitespace(int nr)
+ : nrSpaces(nr) {}
+ };
+
+ std::ostream& operator<<(std::ostream& out, const Whitespace& ws) {
+ if(ws.nrSpaces != 0)
+ out << std::setw(ws.nrSpaces) << ' ';
+ return out;
+ }
+
+ struct ConsoleReporter : public IReporter
+ {
+ std::ostream& s;
+ bool hasLoggedCurrentTestStart;
+ std::vector<SubcaseSignature> subcasesStack;
+ size_t currentSubcaseLevel;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc;
+
+ ConsoleReporter(const ContextOptions& co)
+ : s(*co.cout)
+ , opt(co) {}
+
+ ConsoleReporter(const ContextOptions& co, std::ostream& ostr)
+ : s(ostr)
+ , opt(co) {}
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE HELPERS USED BY THE OVERRIDES OF THE VIRTUAL METHODS OF THE INTERFACE
+ // =========================================================================================
+
+ void separator_to_stream() {
+ s << Color::Yellow
+ << "==============================================================================="
+ "\n";
+ }
+
+ const char* getSuccessOrFailString(bool success, assertType::Enum at,
+ const char* success_str) {
+ if(success)
+ return success_str;
+ return failureString(at);
+ }
+
+ Color::Enum getSuccessOrFailColor(bool success, assertType::Enum at) {
+ return success ? Color::BrightGreen :
+ (at & assertType::is_warn) ? Color::Yellow : Color::Red;
+ }
+
+ void successOrFailColoredStringToStream(bool success, assertType::Enum at,
+ const char* success_str = "SUCCESS") {
+ s << getSuccessOrFailColor(success, at)
+ << getSuccessOrFailString(success, at, success_str) << ": ";
+ }
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << Color::None << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << "\n";
+ }
+ }
+
+ s << "\n";
+ }
+
+ // this was requested to be made virtual so users could override it
+ virtual void file_line_to_stream(const char* file, int line,
+ const char* tail = "") {
+ s << Color::LightGrey << skipPathFromFilename(file) << (opt.gnu_file_line ? ":" : "(")
+ << (opt.no_line_numbers ? 0 : line) // 0 or the real num depending on the option
+ << (opt.gnu_file_line ? ":" : "):") << tail;
+ }
+
+ void logTestStart() {
+ if(hasLoggedCurrentTestStart)
+ return;
+
+ separator_to_stream();
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, "\n");
+ if(tc->m_description)
+ s << Color::Yellow << "DESCRIPTION: " << Color::None << tc->m_description << "\n";
+ if(tc->m_test_suite && tc->m_test_suite[0] != '\0')
+ s << Color::Yellow << "TEST SUITE: " << Color::None << tc->m_test_suite << "\n";
+ if(strncmp(tc->m_name, " Scenario:", 11) != 0)
+ s << Color::Yellow << "TEST CASE: ";
+ s << Color::None << tc->m_name << "\n";
+
+ for(size_t i = 0; i < currentSubcaseLevel; ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+
+ if(currentSubcaseLevel != subcasesStack.size()) {
+ s << Color::Yellow << "\nDEEPEST SUBCASE STACK REACHED (DIFFERENT FROM THE CURRENT ONE):\n" << Color::None;
+ for(size_t i = 0; i < subcasesStack.size(); ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+ }
+
+ s << "\n";
+
+ hasLoggedCurrentTestStart = true;
+ }
+
+ void printVersion() {
+ if(opt.no_version == false)
+ s << Color::Cyan << "[doctest] " << Color::None << "doctest version is \""
+ << DOCTEST_VERSION_STR << "\"\n";
+ }
+
+ void printIntro() {
+ printVersion();
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "run with \"--" DOCTEST_OPTIONS_PREFIX_DISPLAY "help\" for options\n";
+ }
+
+ void printHelp() {
+ int sizePrefixDisplay = static_cast<int>(strlen(DOCTEST_OPTIONS_PREFIX_DISPLAY));
+ printVersion();
+ // clang-format off
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "boolean values: \"1/on/yes/true\" or \"0/off/no/false\"\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filter values: \"str1,str2,str3\" (comma separated strings)\n";
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filters use wildcards for matching strings\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "something passes a filter if any of the strings in a filter matches\n";
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "ALL FLAGS, OPTIONS AND FILTERS ALSO AVAILABLE WITH A \"" DOCTEST_CONFIG_OPTIONS_PREFIX "\" PREFIX!!!\n";
+#endif
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "Query flags - the program quits after them. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "?, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "help, -" DOCTEST_OPTIONS_PREFIX_DISPLAY "h "
+ << Whitespace(sizePrefixDisplay*0) << "prints this message\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "v, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "version "
+ << Whitespace(sizePrefixDisplay*1) << "prints the version\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "c, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "count "
+ << Whitespace(sizePrefixDisplay*1) << "prints the number of matching tests\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ltc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-cases "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching tests by name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-suites "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching test suites\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-reporters "
+ << Whitespace(sizePrefixDisplay*1) << "lists all registered reporters\n\n";
+ // ================================================================================== << 79
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "The available <int>/<string> options/filters are:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sfe, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tse, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "r, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "reporters=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "reporters to use (console is default)\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "o, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "out=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "output filename\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ob, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "order-by=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "how the tests should be ordered\n";
+ s << Whitespace(sizePrefixDisplay*3) << " <string> - [file/suite/name/rand/none]\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "rs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "rand-seed=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "seed for random ordering\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "f, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "first=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the first test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "l, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "last=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the last test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "aa, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "abort-after=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "stop after <int> failed assertions\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "scfl,--" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-filter-levels=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "apply filters for the first <int> levels\n";
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "Bool options - can be used like flags and true is assumed. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "s, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "success=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "include successful assertions in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "cs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "case-sensitive=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "filters being treated as case sensitive\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "e, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "exit=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "exits after the tests finish\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "d, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "duration=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "prints the time duration of each test\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nt, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-throw=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips exceptions-related assert checks\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ne, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-exitcode=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "returns (or exits) always with success\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-run=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips all runtime doctest operations\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nv, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-version=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "omit the framework version in the output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables colors in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "fc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "force-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "use colors even when not in a tty\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nb, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-breaks=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables breakpoints in debuggers\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ns, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-skip=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "don't skip test cases marked as skip\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "gfl, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "gnu-file-line=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << ":n: vs (n): for line numbers in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "npf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-path-filenames=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "only filenames and no paths in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nln, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-line-numbers=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "0 instead of real line numbers in output\n";
+ // ================================================================================== << 79
+ // clang-format on
+
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "for more information visit the project documentation\n\n";
+ }
+
+ void printRegisteredReporters() {
+ printVersion();
+ auto printReporters = [this] (const reporterMap& reporters, const char* type) {
+ if(reporters.size()) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all registered " << type << "\n";
+ for(auto& curr : reporters)
+ s << "priority: " << std::setw(5) << curr.first.first
+ << " name: " << curr.first.second << "\n";
+ }
+ };
+ printReporters(getListeners(), "listeners");
+ printReporters(getReporters(), "reporters");
+ }
+
+ void list_query_results() {
+ separator_to_stream();
+ if(opt.count || opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ if(opt.version) {
+ printVersion();
+ } else if(opt.help) {
+ printHelp();
+ } else if(opt.list_reporters) {
+ printRegisteredReporters();
+ } else if(opt.count || opt.list_test_cases) {
+ if(opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "listing all test case names\n";
+ separator_to_stream();
+ }
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_name << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all test suites\n";
+ separator_to_stream();
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_test_suite << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ void test_run_start() override { printIntro(); }
+
+ void test_run_end(const TestRunStats& p) override {
+ separator_to_stream();
+ s << std::dec;
+
+ auto totwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters, static_cast<unsigned>(p.numAsserts))) + 1)));
+ auto passwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters - p.numTestCasesFailed, static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) + 1)));
+ auto failwidth = int(std::ceil(log10((std::max(p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) + 1)));
+ const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
+ s << Color::Cyan << "[doctest] " << Color::None << "test cases: " << std::setw(totwidth)
+ << p.numTestCasesPassingFilters << " | "
+ << ((p.numTestCasesPassingFilters == 0 || anythingFailed) ? Color::None :
+ Color::Green)
+ << std::setw(passwidth) << p.numTestCasesPassingFilters - p.numTestCasesFailed << " passed"
+ << Color::None << " | " << (p.numTestCasesFailed > 0 ? Color::Red : Color::None)
+ << std::setw(failwidth) << p.numTestCasesFailed << " failed" << Color::None << " |";
+ if(opt.no_skipped_summary == false) {
+ const int numSkipped = p.numTestCases - p.numTestCasesPassingFilters;
+ s << " " << (numSkipped == 0 ? Color::None : Color::Yellow) << numSkipped
+ << " skipped" << Color::None;
+ }
+ s << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None << "assertions: " << std::setw(totwidth)
+ << p.numAsserts << " | "
+ << ((p.numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
+ << std::setw(passwidth) << (p.numAsserts - p.numAssertsFailed) << " passed" << Color::None
+ << " | " << (p.numAssertsFailed > 0 ? Color::Red : Color::None) << std::setw(failwidth)
+ << p.numAssertsFailed << " failed" << Color::None << " |\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "Status: " << (p.numTestCasesFailed > 0 ? Color::Red : Color::Green)
+ << ((p.numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ hasLoggedCurrentTestStart = false;
+ tc = &in;
+ subcasesStack.clear();
+ currentSubcaseLevel = 0;
+ }
+
+ void test_case_reenter(const TestCaseData&) override {
+ subcasesStack.clear();
+ }
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ if(tc->m_no_output)
+ return;
+
+ // log the preamble of the test case only if there is something
+ // else to print - something other than that an assert has failed
+ if(opt.duration ||
+ (st.failure_flags && st.failure_flags != TestCaseFailureReason::AssertFailure))
+ logTestStart();
+
+ if(opt.duration)
+ s << Color::None << std::setprecision(6) << std::fixed << st.seconds
+ << " s: " << tc->m_name << "\n";
+
+ if(st.failure_flags & TestCaseFailureReason::Timeout)
+ s << Color::Red << "Test case exceeded time limit of " << std::setprecision(6)
+ << std::fixed << tc->m_timeout << "!\n";
+
+ if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedButDidnt) {
+ s << Color::Red << "Should have failed but didn't! Marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedAndDid) {
+ s << Color::Yellow << "Failed as expected so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::CouldHaveFailedAndDid) {
+ s << Color::Yellow << "Allowed to fail so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::DidntFailExactlyNumTimes) {
+ s << Color::Red << "Didn't fail exactly " << tc->m_expected_failures
+ << " times so marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::FailedExactlyNumTimes) {
+ s << Color::Yellow << "Failed exactly " << tc->m_expected_failures
+ << " times as expected so marking it as not failed!\n";
+ }
+ if(st.failure_flags & TestCaseFailureReason::TooManyFailedAsserts) {
+ s << Color::Red << "Aborting - too many failed asserts!\n";
+ }
+ s << Color::None; // lgtm [cpp/useless-expression]
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ if(tc->m_no_output)
+ return;
+
+ logTestStart();
+
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, " ");
+ successOrFailColoredStringToStream(false, e.is_crash ? assertType::is_require :
+ assertType::is_check);
+ s << Color::Red << (e.is_crash ? "test case CRASHED: " : "test case THREW exception: ")
+ << Color::Cyan << e.error_string << "\n";
+
+ int num_stringified_contexts = get_num_stringified_contexts();
+ if(num_stringified_contexts) {
+ auto stringified_contexts = get_stringified_contexts();
+ s << Color::None << " logged: ";
+ for(int i = num_stringified_contexts; i > 0; --i) {
+ s << (i == num_stringified_contexts ? "" : " ")
+ << stringified_contexts[i - 1] << "\n";
+ }
+ }
+ s << "\n" << Color::None;
+ }
+
+ void subcase_start(const SubcaseSignature& subc) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ subcasesStack.push_back(subc);
+ ++currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void subcase_end() override {
+ std::lock_guard<std::mutex> lock(mutex);
+ --currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void log_assert(const AssertData& rb) override {
+ if((!rb.m_failed && !opt.success) || tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(rb.m_file, rb.m_line, " ");
+ successOrFailColoredStringToStream(!rb.m_failed, rb.m_at);
+
+ fulltext_log_assert_to_stream(s, rb);
+
+ log_contexts();
+ }
+
+ void log_message(const MessageData& mb) override {
+ if(tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(mb.m_file, mb.m_line, " ");
+ s << getSuccessOrFailColor(false, mb.m_severity)
+ << getSuccessOrFailString(mb.m_severity & assertType::is_warn, mb.m_severity,
+ "MESSAGE") << ": ";
+ s << Color::None << mb.m_string << "\n";
+ log_contexts();
+ }
+
+ void test_case_skipped(const TestCaseData&) override {}
+ };
+
+ DOCTEST_REGISTER_REPORTER("console", 0, ConsoleReporter);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ struct DebugOutputWindowReporter : public ConsoleReporter
+ {
+ DOCTEST_THREAD_LOCAL static std::ostringstream oss;
+
+ DebugOutputWindowReporter(const ContextOptions& co)
+ : ConsoleReporter(co, oss) {}
+
+#define DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(func, type, arg) \
+ void func(type arg) override { \
+ bool with_col = g_no_colors; \
+ g_no_colors = false; \
+ ConsoleReporter::func(arg); \
+ if(oss.tellp() != std::streampos{}) { \
+ DOCTEST_OUTPUT_DEBUG_STRING(oss.str().c_str()); \
+ oss.str(""); \
+ } \
+ g_no_colors = with_col; \
+ }
+
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_start, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_end, const TestRunStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_start, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_reenter, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_end, const CurrentTestCaseStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_exception, const TestCaseException&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_start, const SubcaseSignature&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_end, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_assert, const AssertData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_message, const MessageData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_skipped, const TestCaseData&, in)
+ };
+
+ DOCTEST_THREAD_LOCAL std::ostringstream DebugOutputWindowReporter::oss;
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // the implementation of parseOption()
+ bool parseOptionImpl(int argc, const char* const* argv, const char* pattern, String* value) {
+ // going from the end to the beginning and stopping on the first occurrence from the end
+ for(int i = argc; i > 0; --i) {
+ auto index = i - 1;
+ auto temp = std::strstr(argv[index], pattern);
+ if(temp && (value || strlen(temp) == strlen(pattern))) { //!OCLINT prefer early exits and continue
+ // eliminate matches in which the chars before the option are not '-'
+ bool noBadCharsFound = true;
+ auto curr = argv[index];
+ while(curr != temp) {
+ if(*curr++ != '-') {
+ noBadCharsFound = false;
+ break;
+ }
+ }
+ if(noBadCharsFound && argv[index][0] == '-') {
+ if(value) {
+ // parsing the value of an option
+ temp += strlen(pattern);
+ const unsigned len = strlen(temp);
+ if(len) {
+ *value = temp;
+ return true;
+ }
+ } else {
+ // just a flag - no value
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ // parses an option and returns the string after the '=' character
+ bool parseOption(int argc, const char* const* argv, const char* pattern, String* value = nullptr,
+ const String& defaultVal = String()) {
+ if(value)
+ *value = defaultVal;
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ // offset (normally 3 for "dt-") to skip prefix
+ if(parseOptionImpl(argc, argv, pattern + strlen(DOCTEST_CONFIG_OPTIONS_PREFIX), value))
+ return true;
+#endif // DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ return parseOptionImpl(argc, argv, pattern, value);
+ }
+
+ // locates a flag on the command line
+ bool parseFlag(int argc, const char* const* argv, const char* pattern) {
+ return parseOption(argc, argv, pattern);
+ }
+
+ // parses a comma separated list of words after a pattern in one of the arguments in argv
+ bool parseCommaSepArgs(int argc, const char* const* argv, const char* pattern,
+ std::vector<String>& res) {
+ String filtersString;
+ if(parseOption(argc, argv, pattern, &filtersString)) {
+ // tokenize with "," as a separator
+ // cppcheck-suppress strtokCalled
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ auto pch = std::strtok(filtersString.c_str(), ","); // modifies the string
+ while(pch != nullptr) {
+ if(strlen(pch))
+ res.push_back(pch);
+ // uses the strtok() internal state to go to the next token
+ // cppcheck-suppress strtokCalled
+ pch = std::strtok(nullptr, ",");
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ return true;
+ }
+ return false;
+ }
+
+ enum optionType
+ {
+ option_bool,
+ option_int
+ };
+
+ // parses an int/bool option from the command line
+ bool parseIntOption(int argc, const char* const* argv, const char* pattern, optionType type,
+ int& res) {
+ String parsedValue;
+ if(!parseOption(argc, argv, pattern, &parsedValue))
+ return false;
+
+ if(type == 0) {
+ // boolean
+ const char positive[][5] = {"1", "true", "on", "yes"}; // 5 - strlen("true") + 1
+ const char negative[][6] = {"0", "false", "off", "no"}; // 6 - strlen("false") + 1
+
+ // if the value matches any of the positive/negative possibilities
+ for(unsigned i = 0; i < 4; i++) {
+ if(parsedValue.compare(positive[i], true) == 0) {
+ res = 1; //!OCLINT parameter reassignment
+ return true;
+ }
+ if(parsedValue.compare(negative[i], true) == 0) {
+ res = 0; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ } else {
+ // integer
+ // TODO: change this to use std::stoi or something else! currently it uses undefined behavior - assumes '0' on failed parse...
+ int theInt = std::atoi(parsedValue.c_str()); // NOLINT
+ if(theInt != 0) {
+ res = theInt; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ return false;
+ }
+} // namespace
+
+Context::Context(int argc, const char* const* argv)
+ : p(new detail::ContextState) {
+ parseArgs(argc, argv, true);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+Context::~Context() {
+ if(g_cs == p)
+ g_cs = nullptr;
+ delete p;
+}
+
+void Context::applyCommandLine(int argc, const char* const* argv) {
+ parseArgs(argc, argv);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+// parses args
+void Context::parseArgs(int argc, const char* const* argv, bool withDefaults) {
+ using namespace detail;
+
+ // clang-format off
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sf=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file-exclude=",p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sfe=", p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ts=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite-exclude=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tse=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tc=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case-exclude=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tce=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sc=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase-exclude=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sce=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "reporters=", p->filters[8]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "r=", p->filters[8]);
+ // clang-format on
+
+ int intRes = 0;
+ String strRes;
+
+#define DOCTEST_PARSE_AS_BOOL_OR_FLAG(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_bool, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_bool, intRes)) \
+ p->var = static_cast<bool>(intRes); \
+ else if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name) || \
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname)) \
+ p->var = true; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_INT_OPTION(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_int, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_int, intRes)) \
+ p->var = intRes; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_STR_OPTION(name, sname, var, default) \
+ if(parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", &strRes, default) || \
+ parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", &strRes, default) || \
+ withDefaults) \
+ p->var = strRes
+
+ // clang-format off
+ DOCTEST_PARSE_STR_OPTION("out", "o", out, "");
+ DOCTEST_PARSE_STR_OPTION("order-by", "ob", order_by, "file");
+ DOCTEST_PARSE_INT_OPTION("rand-seed", "rs", rand_seed, 0);
+
+ DOCTEST_PARSE_INT_OPTION("first", "f", first, 0);
+ DOCTEST_PARSE_INT_OPTION("last", "l", last, UINT_MAX);
+
+ DOCTEST_PARSE_INT_OPTION("abort-after", "aa", abort_after, 0);
+ DOCTEST_PARSE_INT_OPTION("subcase-filter-levels", "scfl", subcase_filter_levels, INT_MAX);
+
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("success", "s", success, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("case-sensitive", "cs", case_sensitive, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("exit", "e", exit, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("duration", "d", duration, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-throw", "nt", no_throw, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-exitcode", "ne", no_exitcode, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-run", "nr", no_run, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-version", "nv", no_version, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-colors", "nc", no_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("force-colors", "fc", force_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-breaks", "nb", no_breaks, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skip", "ns", no_skip, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("gnu-file-line", "gfl", gnu_file_line, !bool(DOCTEST_MSVC));
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-path-filenames", "npf", no_path_in_filenames, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-line-numbers", "nln", no_line_numbers, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-debug-output", "ndo", no_debug_output, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skipped-summary", "nss", no_skipped_summary, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-time-in-output", "ntio", no_time_in_output, false);
+ // clang-format on
+
+ if(withDefaults) {
+ p->help = false;
+ p->version = false;
+ p->count = false;
+ p->list_test_cases = false;
+ p->list_test_suites = false;
+ p->list_reporters = false;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "help") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "h") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "?")) {
+ p->help = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "version") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "v")) {
+ p->version = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "count") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "c")) {
+ p->count = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-cases") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ltc")) {
+ p->list_test_cases = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-suites") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lts")) {
+ p->list_test_suites = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-reporters") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lr")) {
+ p->list_reporters = true;
+ p->exit = true;
+ }
+}
+
+// allows the user to add procedurally to the filters from the command line
+void Context::addFilter(const char* filter, const char* value) { setOption(filter, value); }
+
+// allows the user to clear all filters from the command line
+void Context::clearFilters() {
+ for(auto& curr : p->filters)
+ curr.clear();
+}
+
+// allows the user to override procedurally the int/bool options from the command line
+void Context::setOption(const char* option, int value) {
+ setOption(option, toString(value).c_str());
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+// allows the user to override procedurally the string options from the command line
+void Context::setOption(const char* option, const char* value) {
+ auto argv = String("-") + option + "=" + value;
+ auto lvalue = argv.c_str();
+ parseArgs(1, &lvalue);
+}
+
+// users should query this in their main() and exit the program if true
+bool Context::shouldExit() { return p->exit; }
+
+void Context::setAsDefaultForAssertsOutOfTestCases() { g_cs = p; }
+
+void Context::setAssertHandler(detail::assert_handler ah) { p->ah = ah; }
+
+// the main function that does all the filtering and test running
+int Context::run() {
+ using namespace detail;
+
+ // save the old context state in case such was setup - for using asserts out of a testing context
+ auto old_cs = g_cs;
+ // this is the current contest
+ g_cs = p;
+ is_running_in_test = true;
+
+ g_no_colors = p->no_colors;
+ p->resetRunData();
+
+ // stdout by default
+ p->cout = &std::cout;
+ p->cerr = &std::cerr;
+
+ // or to a file if specified
+ std::fstream fstr;
+ if(p->out.size()) {
+ fstr.open(p->out.c_str(), std::fstream::out);
+ p->cout = &fstr;
+ }
+
+ FatalConditionHandler::allocateAltStackMem();
+
+ auto cleanup_and_return = [&]() {
+ FatalConditionHandler::freeAltStackMem();
+
+ if(fstr.is_open())
+ fstr.close();
+
+ // restore context
+ g_cs = old_cs;
+ is_running_in_test = false;
+
+ // we have to free the reporters which were allocated when the run started
+ for(auto& curr : p->reporters_currently_used)
+ delete curr;
+ p->reporters_currently_used.clear();
+
+ if(p->numTestCasesFailed && !p->no_exitcode)
+ return EXIT_FAILURE;
+ return EXIT_SUCCESS;
+ };
+
+ // setup default reporter if none is given through the command line
+ if(p->filters[8].empty())
+ p->filters[8].push_back("console");
+
+ // check to see if any of the registered reporters has been selected
+ for(auto& curr : getReporters()) {
+ if(matchesAny(curr.first.second.c_str(), p->filters[8], false, p->case_sensitive))
+ p->reporters_currently_used.push_back(curr.second(*g_cs));
+ }
+
+ // TODO: check if there is nothing in reporters_currently_used
+
+ // prepend all listeners
+ for(auto& curr : getListeners())
+ p->reporters_currently_used.insert(p->reporters_currently_used.begin(), curr.second(*g_cs));
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(isDebuggerActive() && p->no_debug_output == false)
+ p->reporters_currently_used.push_back(new DebugOutputWindowReporter(*g_cs));
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // handle version, help and no_run
+ if(p->no_run || p->version || p->help || p->list_reporters) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, QueryData());
+
+ return cleanup_and_return();
+ }
+
+ std::vector<const TestCase*> testArray;
+ for(auto& curr : getRegisteredTests())
+ testArray.push_back(&curr);
+ p->numTestCases = testArray.size();
+
+ // sort the collected records
+ if(!testArray.empty()) {
+ if(p->order_by.compare("file", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), fileOrderComparator);
+ } else if(p->order_by.compare("suite", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), suiteOrderComparator);
+ } else if(p->order_by.compare("name", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), nameOrderComparator);
+ } else if(p->order_by.compare("rand", true) == 0) {
+ std::srand(p->rand_seed);
+
+ // random_shuffle implementation
+ const auto first = &testArray[0];
+ for(size_t i = testArray.size() - 1; i > 0; --i) {
+ int idxToSwap = std::rand() % (i + 1); // NOLINT
+
+ const auto temp = first[i];
+
+ first[i] = first[idxToSwap];
+ first[idxToSwap] = temp;
+ }
+ } else if(p->order_by.compare("none", true) == 0) {
+ // means no sorting - beneficial for death tests which call into the executable
+ // with a specific test case in mind - we don't want to slow down the startup times
+ }
+ }
+
+ std::set<String> testSuitesPassingFilt;
+
+ bool query_mode = p->count || p->list_test_cases || p->list_test_suites;
+ std::vector<const TestCaseData*> queryResults;
+
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_start, DOCTEST_EMPTY);
+
+ // invoke the registered functions if they match the filter criteria (or just count them)
+ for(auto& curr : testArray) {
+ const auto& tc = *curr;
+
+ bool skip_me = false;
+ if(tc.m_skip && !p->no_skip)
+ skip_me = true;
+
+ if(!matchesAny(tc.m_file.c_str(), p->filters[0], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_file.c_str(), p->filters[1], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_test_suite, p->filters[2], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_test_suite, p->filters[3], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_name, p->filters[4], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_name, p->filters[5], false, p->case_sensitive))
+ skip_me = true;
+
+ if(!skip_me)
+ p->numTestCasesPassingFilters++;
+
+ // skip the test if it is not in the execution range
+ if((p->last < p->numTestCasesPassingFilters && p->first <= p->last) ||
+ (p->first > p->numTestCasesPassingFilters))
+ skip_me = true;
+
+ if(skip_me) {
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_skipped, tc);
+ continue;
+ }
+
+ // do not execute the test if we are to only count the number of filter passing tests
+ if(p->count)
+ continue;
+
+ // print the name of the test and don't execute it
+ if(p->list_test_cases) {
+ queryResults.push_back(&tc);
+ continue;
+ }
+
+ // print the name of the test suite if not done already and don't execute it
+ if(p->list_test_suites) {
+ if((testSuitesPassingFilt.count(tc.m_test_suite) == 0) && tc.m_test_suite[0] != '\0') {
+ queryResults.push_back(&tc);
+ testSuitesPassingFilt.insert(tc.m_test_suite);
+ p->numTestSuitesPassingFilters++;
+ }
+ continue;
+ }
+
+ // execute the test if it passes all the filtering
+ {
+ p->currentTest = &tc;
+
+ p->failure_flags = TestCaseFailureReason::None;
+ p->seconds = 0;
+
+ // reset atomic counters
+ p->numAssertsFailedCurrentTest_atomic = 0;
+ p->numAssertsCurrentTest_atomic = 0;
+
+ p->subcasesPassed.clear();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_start, tc);
+
+ p->timer.start();
+
+ bool run_test = true;
+
+ do {
+ // reset some of the fields for subcases (except for the set of fully passed ones)
+ p->should_reenter = false;
+ p->subcasesCurrentMaxLevel = 0;
+ p->subcasesStack.clear();
+
+ p->shouldLogCurrentException = true;
+
+ // reset stuff for logging with INFO()
+ p->stringifiedContexts.clear();
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+// MSVC 2015 diagnoses fatalConditionHandler as unused (because reset() is a static method)
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4101) // unreferenced local variable
+ FatalConditionHandler fatalConditionHandler; // Handle signals
+ // execute the test
+ tc.m_test();
+ fatalConditionHandler.reset();
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ } catch(const TestFailureException&) {
+ p->failure_flags |= TestCaseFailureReason::AssertFailure;
+ } catch(...) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception,
+ {translateActiveException(), false});
+ p->failure_flags |= TestCaseFailureReason::Exception;
+ }
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+ // exit this loop if enough assertions have failed - even if there are more subcases
+ if(p->abort_after > 0 &&
+ p->numAssertsFailed + p->numAssertsFailedCurrentTest_atomic >= p->abort_after) {
+ run_test = false;
+ p->failure_flags |= TestCaseFailureReason::TooManyFailedAsserts;
+ }
+
+ if(p->should_reenter && run_test)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_reenter, tc);
+ if(!p->should_reenter)
+ run_test = false;
+ } while(run_test);
+
+ p->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ p->currentTest = nullptr;
+
+ // stop executing tests if enough assertions have failed
+ if(p->abort_after > 0 && p->numAssertsFailed >= p->abort_after)
+ break;
+ }
+ }
+
+ if(!query_mode) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ } else {
+ QueryData qdata;
+ qdata.run_stats = g_cs;
+ qdata.data = queryResults.data();
+ qdata.num_data = unsigned(queryResults.size());
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, qdata);
+ }
+
+ // see these issues on the reasoning for this:
+ // - https://github.com/onqtam/doctest/issues/143#issuecomment-414418903
+ // - https://github.com/onqtam/doctest/issues/126
+ auto DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS = []() DOCTEST_NOINLINE
+ { std::cout << std::string(); };
+ DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS();
+
+ return cleanup_and_return();
+}
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return detail::g_infoContexts.size(); }
+const IContextScope* const* IReporter::get_active_contexts() {
+ return get_num_active_contexts() ? &detail::g_infoContexts[0] : nullptr;
+}
+
+int IReporter::get_num_stringified_contexts() { return detail::g_cs->stringifiedContexts.size(); }
+const String* IReporter::get_stringified_contexts() {
+ return get_num_stringified_contexts() ? &detail::g_cs->stringifiedContexts[0] : nullptr;
+}
+
+namespace detail {
+ void registerReporterImpl(const char* name, int priority, reporterCreatorFunc c, bool isReporter) {
+ if(isReporter)
+ getReporters().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ else
+ getListeners().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ }
+} // namespace detail
+
+} // namespace doctest
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifdef DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4007) // 'function' : must be 'attribute' - see issue #182
+int main(int argc, char** argv) { return doctest::Context(argc, argv).run(); }
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_IMPLEMENTATION
+#endif // DOCTEST_CONFIG_IMPLEMENT
diff --git a/contrib/doctest/doctest/parts/doctest_fwd.h b/contrib/doctest/doctest/parts/doctest_fwd.h
new file mode 100644
index 0000000..e83b0ca
--- /dev/null
+++ b/contrib/doctest/doctest/parts/doctest_fwd.h
@@ -0,0 +1,2706 @@
+//
+// doctest.h - the lightest feature-rich C++ single-header testing framework for unit tests and TDD
+//
+// Copyright (c) 2016-2021 Viktor Kirilov
+//
+// Distributed under the MIT Software License
+// See accompanying file LICENSE.txt or copy at
+// https://opensource.org/licenses/MIT
+//
+// The documentation can be found at the library's page:
+// https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+//
+// The library is heavily influenced by Catch - https://github.com/catchorg/Catch2
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/catchorg/Catch2/blob/master/LICENSE.txt
+//
+// The concept of subcases (sections in Catch) and expression decomposition are from there.
+// Some parts of the code are taken directly:
+// - stringification - the detection of "ostream& operator<<(ostream&, const T&)" and StringMaker<>
+// - the Approx() helper class for floating point comparison
+// - colors in the console
+// - breaking into a debugger
+// - signal / SEH handling
+// - timer
+// - XmlWriter class - thanks to Phil Nash for allowing the direct reuse (AKA copy/paste)
+//
+// The expression decomposing templates are taken from lest - https://github.com/martinmoene/lest
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/martinmoene/lest/blob/master/LICENSE.txt
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+
+#ifndef DOCTEST_LIBRARY_INCLUDED
+#define DOCTEST_LIBRARY_INCLUDED
+
+// =================================================================================================
+// == VERSION ======================================================================================
+// =================================================================================================
+
+#define DOCTEST_VERSION_MAJOR 2
+#define DOCTEST_VERSION_MINOR 4
+#define DOCTEST_VERSION_PATCH 6
+#define DOCTEST_VERSION_STR "2.4.6"
+
+#define DOCTEST_VERSION \
+ (DOCTEST_VERSION_MAJOR * 10000 + DOCTEST_VERSION_MINOR * 100 + DOCTEST_VERSION_PATCH)
+
+// =================================================================================================
+// == COMPILER VERSION =============================================================================
+// =================================================================================================
+
+// ideas for the version stuff are taken from here: https://github.com/cxxstuff/cxx_detect
+
+#define DOCTEST_COMPILER(MAJOR, MINOR, PATCH) ((MAJOR)*10000000 + (MINOR)*100000 + (PATCH))
+
+// GCC/Clang and GCC/MSVC are mutually exclusive, but Clang/MSVC are not because of clang-cl...
+#if defined(_MSC_VER) && defined(_MSC_FULL_VER)
+#if _MSC_VER == _MSC_FULL_VER / 10000
+#define DOCTEST_MSVC DOCTEST_COMPILER(_MSC_VER / 100, _MSC_VER % 100, _MSC_FULL_VER % 10000)
+#else // MSVC
+#define DOCTEST_MSVC \
+ DOCTEST_COMPILER(_MSC_VER / 100, (_MSC_FULL_VER / 100000) % 100, _MSC_FULL_VER % 100000)
+#endif // MSVC
+#endif // MSVC
+#if defined(__clang__) && defined(__clang_minor__)
+#define DOCTEST_CLANG DOCTEST_COMPILER(__clang_major__, __clang_minor__, __clang_patchlevel__)
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && \
+ !defined(__INTEL_COMPILER)
+#define DOCTEST_GCC DOCTEST_COMPILER(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
+#endif // GCC
+
+#ifndef DOCTEST_MSVC
+#define DOCTEST_MSVC 0
+#endif // DOCTEST_MSVC
+#ifndef DOCTEST_CLANG
+#define DOCTEST_CLANG 0
+#endif // DOCTEST_CLANG
+#ifndef DOCTEST_GCC
+#define DOCTEST_GCC 0
+#endif // DOCTEST_GCC
+
+// =================================================================================================
+// == COMPILER WARNINGS HELPERS ====================================================================
+// =================================================================================================
+
+#if DOCTEST_CLANG
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH _Pragma("clang diagnostic push")
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(clang diagnostic ignored w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP _Pragma("clang diagnostic pop")
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#else // DOCTEST_CLANG
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_CLANG
+
+#if DOCTEST_GCC
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH _Pragma("GCC diagnostic push")
+#define DOCTEST_GCC_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(GCC diagnostic ignored w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP _Pragma("GCC diagnostic pop")
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH DOCTEST_GCC_SUPPRESS_WARNING(w)
+#else // DOCTEST_GCC
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_GCC_SUPPRESS_WARNING(w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_GCC
+
+#if DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH __pragma(warning(push))
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w) __pragma(warning(disable : w))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP __pragma(warning(pop))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#else // DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_MSVC
+
+// =================================================================================================
+// == COMPILER WARNINGS ============================================================================
+// =================================================================================================
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdeprecated")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wctor-dtor-privacy")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-promo")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4623) // default constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtr...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+// 4548 - expression before comma has no effect; expected expression with side - effect
+// 4265 - class has virtual functions, but destructor is not virtual
+// 4986 - exception specification does not match previous declaration
+// 4350 - behavior change: 'member1' called instead of 'member2'
+// 4668 - 'x' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
+// 4365 - conversion from 'int' to 'unsigned long', signed/unsigned mismatch
+// 4774 - format string expected in argument 'x' is not a string literal
+// 4820 - padding in structs
+
+// only 4 should be disabled globally:
+// - 4514 # unreferenced inline function has been removed
+// - 4571 # SEH related
+// - 4710 # function not inlined
+// - 4711 # function 'x' selected for automatic inline expansion
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4548) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4265) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4986) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4350) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4668) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4365) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4774) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4820) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4625) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4626) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5027) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5026) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4623) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5039) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5045) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5105)
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+// =================================================================================================
+// == FEATURE DETECTION ============================================================================
+// =================================================================================================
+
+// general compiler feature support table: https://en.cppreference.com/w/cpp/compiler_support
+// MSVC C++11 feature support table: https://msdn.microsoft.com/en-us/library/hh567368.aspx
+// GCC C++11 feature support table: https://gcc.gnu.org/projects/cxx-status.html
+// MSVC version table:
+// https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering
+// MSVC++ 14.2 (16) _MSC_VER == 1920 (Visual Studio 2019)
+// MSVC++ 14.1 (15) _MSC_VER == 1910 (Visual Studio 2017)
+// MSVC++ 14.0 _MSC_VER == 1900 (Visual Studio 2015)
+// MSVC++ 12.0 _MSC_VER == 1800 (Visual Studio 2013)
+// MSVC++ 11.0 _MSC_VER == 1700 (Visual Studio 2012)
+// MSVC++ 10.0 _MSC_VER == 1600 (Visual Studio 2010)
+// MSVC++ 9.0 _MSC_VER == 1500 (Visual Studio 2008)
+// MSVC++ 8.0 _MSC_VER == 1400 (Visual Studio 2005)
+
+#if DOCTEST_MSVC && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#define DOCTEST_CONFIG_WINDOWS_SEH
+#endif // MSVC
+#if defined(DOCTEST_CONFIG_NO_WINDOWS_SEH) && defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#undef DOCTEST_CONFIG_WINDOWS_SEH
+#endif // DOCTEST_CONFIG_NO_WINDOWS_SEH
+
+#if !defined(_WIN32) && !defined(__QNX__) && !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && \
+ !defined(__EMSCRIPTEN__)
+#define DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // _WIN32
+#if defined(DOCTEST_CONFIG_NO_POSIX_SIGNALS) && defined(DOCTEST_CONFIG_POSIX_SIGNALS)
+#undef DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // DOCTEST_CONFIG_NO_POSIX_SIGNALS
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // no exceptions
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#if defined(DOCTEST_CONFIG_NO_EXCEPTIONS) && !defined(DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS)
+#define DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS && !DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#if defined(DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN) && !defined(DOCTEST_CONFIG_IMPLEMENT)
+#define DOCTEST_CONFIG_IMPLEMENT
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#if DOCTEST_MSVC
+#define DOCTEST_SYMBOL_EXPORT __declspec(dllexport)
+#define DOCTEST_SYMBOL_IMPORT __declspec(dllimport)
+#else // MSVC
+#define DOCTEST_SYMBOL_EXPORT __attribute__((dllexport))
+#define DOCTEST_SYMBOL_IMPORT __attribute__((dllimport))
+#endif // MSVC
+#else // _WIN32
+#define DOCTEST_SYMBOL_EXPORT __attribute__((visibility("default")))
+#define DOCTEST_SYMBOL_IMPORT
+#endif // _WIN32
+
+#ifdef DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#ifdef DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_EXPORT
+#else // DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_IMPORT
+#endif // DOCTEST_CONFIG_IMPLEMENT
+#else // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#define DOCTEST_INTERFACE
+#endif // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+
+#define DOCTEST_EMPTY
+
+#if DOCTEST_MSVC
+#define DOCTEST_NOINLINE __declspec(noinline)
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#elif DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 5, 0)
+#define DOCTEST_NOINLINE
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#else
+#define DOCTEST_NOINLINE __attribute__((noinline))
+#define DOCTEST_UNUSED __attribute__((unused))
+#define DOCTEST_ALIGNMENT(x) __attribute__((aligned(x)))
+#endif
+
+#ifndef DOCTEST_NORETURN
+#define DOCTEST_NORETURN [[noreturn]]
+#endif // DOCTEST_NORETURN
+
+#ifndef DOCTEST_NOEXCEPT
+#define DOCTEST_NOEXCEPT noexcept
+#endif // DOCTEST_NOEXCEPT
+
+// =================================================================================================
+// == FEATURE DETECTION END ========================================================================
+// =================================================================================================
+
+// internal macros for string concatenation and anonymous variable name generation
+#define DOCTEST_CAT_IMPL(s1, s2) s1##s2
+#define DOCTEST_CAT(s1, s2) DOCTEST_CAT_IMPL(s1, s2)
+#ifdef __COUNTER__ // not standard and may be missing for some compilers
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __COUNTER__)
+#else // __COUNTER__
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __LINE__)
+#endif // __COUNTER__
+
+#define DOCTEST_TOSTR(x) #x
+
+#ifndef DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x&
+#else // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x
+#endif // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+
+// not using __APPLE__ because... this is how Catch does it
+#ifdef __MAC_OS_X_VERSION_MIN_REQUIRED
+#define DOCTEST_PLATFORM_MAC
+#elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED)
+#define DOCTEST_PLATFORM_IPHONE
+#elif defined(_WIN32)
+#define DOCTEST_PLATFORM_WINDOWS
+#else // DOCTEST_PLATFORM
+#define DOCTEST_PLATFORM_LINUX
+#endif // DOCTEST_PLATFORM
+
+#define DOCTEST_GLOBAL_NO_WARNINGS(var) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wglobal-constructors") \
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-variable") \
+ static const int var DOCTEST_UNUSED // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp)
+#define DOCTEST_GLOBAL_NO_WARNINGS_END() DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#ifndef DOCTEST_BREAK_INTO_DEBUGGER
+// should probably take a look at https://github.com/scottt/debugbreak
+#ifdef DOCTEST_PLATFORM_LINUX
+#if defined(__GNUC__) && (defined(__i386) || defined(__x86_64))
+// Break at the location of the failing check if possible
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#include <signal.h>
+#define DOCTEST_BREAK_INTO_DEBUGGER() raise(SIGTRAP)
+#endif
+#elif defined(DOCTEST_PLATFORM_MAC)
+#if defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__i386)
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("brk #0"); // NOLINT (hicpp-no-assembler)
+#endif
+#elif DOCTEST_MSVC
+#define DOCTEST_BREAK_INTO_DEBUGGER() __debugbreak()
+#elif defined(__MINGW32__)
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wredundant-decls")
+extern "C" __declspec(dllimport) void __stdcall DebugBreak();
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_BREAK_INTO_DEBUGGER() ::DebugBreak()
+#else // linux
+#define DOCTEST_BREAK_INTO_DEBUGGER() (static_cast<void>(0))
+#endif // linux
+#endif // DOCTEST_BREAK_INTO_DEBUGGER
+
+// this is kept here for backwards compatibility since the config option was changed
+#ifdef DOCTEST_CONFIG_USE_IOSFWD
+#define DOCTEST_CONFIG_USE_STD_HEADERS
+#endif // DOCTEST_CONFIG_USE_IOSFWD
+
+#ifdef DOCTEST_CONFIG_USE_STD_HEADERS
+#ifndef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <iosfwd>
+#include <cstddef>
+#include <ostream>
+#else // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#if DOCTEST_CLANG
+// to detect if libc++ is being used with clang (the _LIBCPP_VERSION identifier)
+#include <ciso646>
+#endif // clang
+
+#ifdef _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN _LIBCPP_BEGIN_NAMESPACE_STD
+#define DOCTEST_STD_NAMESPACE_END _LIBCPP_END_NAMESPACE_STD
+#else // _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN namespace std {
+#define DOCTEST_STD_NAMESPACE_END }
+#endif // _LIBCPP_VERSION
+
+// Forward declaring 'X' in namespace std is not permitted by the C++ Standard.
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4643)
+
+DOCTEST_STD_NAMESPACE_BEGIN // NOLINT (cert-dcl58-cpp)
+typedef decltype(nullptr) nullptr_t;
+template <class charT>
+struct char_traits;
+template <>
+struct char_traits<char>;
+template <class charT, class traits>
+class basic_ostream;
+typedef basic_ostream<char, char_traits<char>> ostream;
+template <class... Types>
+class tuple;
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+template <class _Ty>
+class allocator;
+template <class _Elem, class _Traits, class _Alloc>
+class basic_string;
+using string = basic_string<char, char_traits<char>, allocator<char>>;
+#endif // VS 2019
+DOCTEST_STD_NAMESPACE_END
+
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <type_traits>
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+namespace doctest {
+
+DOCTEST_INTERFACE extern bool is_running_in_test;
+
+// A 24 byte string class (can be as small as 17 for x64 and 13 for x86) that can hold strings with length
+// of up to 23 chars on the stack before going on the heap - the last byte of the buffer is used for:
+// - "is small" bit - the highest bit - if "0" then it is small - otherwise its "1" (128)
+// - if small - capacity left before going on the heap - using the lowest 5 bits
+// - if small - 2 bits are left unused - the second and third highest ones
+// - if small - acts as a null terminator if strlen() is 23 (24 including the null terminator)
+// and the "is small" bit remains "0" ("as well as the capacity left") so its OK
+// Idea taken from this lecture about the string implementation of facebook/folly - fbstring
+// https://www.youtube.com/watch?v=kPR8h4-qZdk
+// TODO:
+// - optimizations - like not deleting memory unnecessarily in operator= and etc.
+// - resize/reserve/clear
+// - substr
+// - replace
+// - back/front
+// - iterator stuff
+// - find & friends
+// - push_back/pop_back
+// - assign/insert/erase
+// - relational operators as free functions - taking const char* as one of the params
+class DOCTEST_INTERFACE String
+{
+ static const unsigned len = 24; //!OCLINT avoid private static members
+ static const unsigned last = len - 1; //!OCLINT avoid private static members
+
+ struct view // len should be more than sizeof(view) - because of the final byte for flags
+ {
+ char* ptr;
+ unsigned size;
+ unsigned capacity;
+ };
+
+ union
+ {
+ char buf[len];
+ view data;
+ };
+
+ bool isOnStack() const { return (buf[last] & 128) == 0; }
+ void setOnHeap();
+ void setLast(unsigned in = last);
+
+ void copy(const String& other);
+
+public:
+ String();
+ ~String();
+
+ // cppcheck-suppress noExplicitConstructor
+ String(const char* in);
+ String(const char* in, unsigned in_size);
+
+ String(const String& other);
+ String& operator=(const String& other);
+
+ String& operator+=(const String& other);
+ String operator+(const String& other) const;
+
+ String(String&& other);
+ String& operator=(String&& other);
+
+ char operator[](unsigned i) const;
+ char& operator[](unsigned i);
+
+ // the only functions I'm willing to leave in the interface - available for inlining
+ const char* c_str() const { return const_cast<String*>(this)->c_str(); } // NOLINT
+ char* c_str() {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf);
+ return data.ptr;
+ }
+
+ unsigned size() const;
+ unsigned capacity() const;
+
+ int compare(const char* other, bool no_case = false) const;
+ int compare(const String& other, bool no_case = false) const;
+};
+
+DOCTEST_INTERFACE bool operator==(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator!=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>=(const String& lhs, const String& rhs);
+
+DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, const String& in);
+
+namespace Color {
+ enum Enum
+ {
+ None = 0,
+ White,
+ Red,
+ Green,
+ Blue,
+ Cyan,
+ Yellow,
+ Grey,
+
+ Bright = 0x10,
+
+ BrightRed = Bright | Red,
+ BrightGreen = Bright | Green,
+ LightGrey = Bright | Grey,
+ BrightWhite = Bright | White
+ };
+
+ DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, Color::Enum code);
+} // namespace Color
+
+namespace assertType {
+ enum Enum
+ {
+ // macro traits
+
+ is_warn = 1,
+ is_check = 2 * is_warn,
+ is_require = 2 * is_check,
+
+ is_normal = 2 * is_require,
+ is_throws = 2 * is_normal,
+ is_throws_as = 2 * is_throws,
+ is_throws_with = 2 * is_throws_as,
+ is_nothrow = 2 * is_throws_with,
+
+ is_false = 2 * is_nothrow,
+ is_unary = 2 * is_false, // not checked anywhere - used just to distinguish the types
+
+ is_eq = 2 * is_unary,
+ is_ne = 2 * is_eq,
+
+ is_lt = 2 * is_ne,
+ is_gt = 2 * is_lt,
+
+ is_ge = 2 * is_gt,
+ is_le = 2 * is_ge,
+
+ // macro types
+
+ DT_WARN = is_normal | is_warn,
+ DT_CHECK = is_normal | is_check,
+ DT_REQUIRE = is_normal | is_require,
+
+ DT_WARN_FALSE = is_normal | is_false | is_warn,
+ DT_CHECK_FALSE = is_normal | is_false | is_check,
+ DT_REQUIRE_FALSE = is_normal | is_false | is_require,
+
+ DT_WARN_THROWS = is_throws | is_warn,
+ DT_CHECK_THROWS = is_throws | is_check,
+ DT_REQUIRE_THROWS = is_throws | is_require,
+
+ DT_WARN_THROWS_AS = is_throws_as | is_warn,
+ DT_CHECK_THROWS_AS = is_throws_as | is_check,
+ DT_REQUIRE_THROWS_AS = is_throws_as | is_require,
+
+ DT_WARN_THROWS_WITH = is_throws_with | is_warn,
+ DT_CHECK_THROWS_WITH = is_throws_with | is_check,
+ DT_REQUIRE_THROWS_WITH = is_throws_with | is_require,
+
+ DT_WARN_THROWS_WITH_AS = is_throws_with | is_throws_as | is_warn,
+ DT_CHECK_THROWS_WITH_AS = is_throws_with | is_throws_as | is_check,
+ DT_REQUIRE_THROWS_WITH_AS = is_throws_with | is_throws_as | is_require,
+
+ DT_WARN_NOTHROW = is_nothrow | is_warn,
+ DT_CHECK_NOTHROW = is_nothrow | is_check,
+ DT_REQUIRE_NOTHROW = is_nothrow | is_require,
+
+ DT_WARN_EQ = is_normal | is_eq | is_warn,
+ DT_CHECK_EQ = is_normal | is_eq | is_check,
+ DT_REQUIRE_EQ = is_normal | is_eq | is_require,
+
+ DT_WARN_NE = is_normal | is_ne | is_warn,
+ DT_CHECK_NE = is_normal | is_ne | is_check,
+ DT_REQUIRE_NE = is_normal | is_ne | is_require,
+
+ DT_WARN_GT = is_normal | is_gt | is_warn,
+ DT_CHECK_GT = is_normal | is_gt | is_check,
+ DT_REQUIRE_GT = is_normal | is_gt | is_require,
+
+ DT_WARN_LT = is_normal | is_lt | is_warn,
+ DT_CHECK_LT = is_normal | is_lt | is_check,
+ DT_REQUIRE_LT = is_normal | is_lt | is_require,
+
+ DT_WARN_GE = is_normal | is_ge | is_warn,
+ DT_CHECK_GE = is_normal | is_ge | is_check,
+ DT_REQUIRE_GE = is_normal | is_ge | is_require,
+
+ DT_WARN_LE = is_normal | is_le | is_warn,
+ DT_CHECK_LE = is_normal | is_le | is_check,
+ DT_REQUIRE_LE = is_normal | is_le | is_require,
+
+ DT_WARN_UNARY = is_normal | is_unary | is_warn,
+ DT_CHECK_UNARY = is_normal | is_unary | is_check,
+ DT_REQUIRE_UNARY = is_normal | is_unary | is_require,
+
+ DT_WARN_UNARY_FALSE = is_normal | is_false | is_unary | is_warn,
+ DT_CHECK_UNARY_FALSE = is_normal | is_false | is_unary | is_check,
+ DT_REQUIRE_UNARY_FALSE = is_normal | is_false | is_unary | is_require,
+ };
+} // namespace assertType
+
+DOCTEST_INTERFACE const char* assertString(assertType::Enum at);
+DOCTEST_INTERFACE const char* failureString(assertType::Enum at);
+DOCTEST_INTERFACE const char* skipPathFromFilename(const char* file);
+
+struct DOCTEST_INTERFACE TestCaseData
+{
+ String m_file; // the file in which the test was registered (using String - see #350)
+ unsigned m_line; // the line where the test was registered
+ const char* m_name; // name of the test case
+ const char* m_test_suite; // the test suite in which the test was added
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+};
+
+struct DOCTEST_INTERFACE AssertData
+{
+ // common - for all asserts
+ const TestCaseData* m_test_case;
+ assertType::Enum m_at;
+ const char* m_file;
+ int m_line;
+ const char* m_expr;
+ bool m_failed;
+
+ // exception-related - for all asserts
+ bool m_threw;
+ String m_exception;
+
+ // for normal asserts
+ String m_decomp;
+
+ // for specific exception-related asserts
+ bool m_threw_as;
+ const char* m_exception_type;
+ const char* m_exception_string;
+};
+
+struct DOCTEST_INTERFACE MessageData
+{
+ String m_string;
+ const char* m_file;
+ int m_line;
+ assertType::Enum m_severity;
+};
+
+struct DOCTEST_INTERFACE SubcaseSignature
+{
+ String m_name;
+ const char* m_file;
+ int m_line;
+
+ bool operator<(const SubcaseSignature& other) const;
+};
+
+struct DOCTEST_INTERFACE IContextScope
+{
+ IContextScope();
+ virtual ~IContextScope();
+ virtual void stringify(std::ostream*) const = 0;
+};
+
+namespace detail {
+ struct DOCTEST_INTERFACE TestCase;
+} // namespace detail
+
+struct ContextOptions //!OCLINT too many fields
+{
+ std::ostream* cout; // stdout stream - std::cout by default
+ std::ostream* cerr; // stderr stream - std::cerr by default
+ String binary_name; // the test binary name
+
+ const detail::TestCase* currentTest = nullptr;
+
+ // == parameters from the command line
+ String out; // output filename
+ String order_by; // how tests should be ordered
+ unsigned rand_seed; // the seed for rand ordering
+
+ unsigned first; // the first (matching) test to be executed
+ unsigned last; // the last (matching) test to be executed
+
+ int abort_after; // stop tests after this many failed assertions
+ int subcase_filter_levels; // apply the subcase filters for the first N levels
+
+ bool success; // include successful assertions in output
+ bool case_sensitive; // if filtering should be case sensitive
+ bool exit; // if the program should be exited after the tests are ran/whatever
+ bool duration; // print the time duration of each test case
+ bool no_throw; // to skip exceptions-related assertion macros
+ bool no_exitcode; // if the framework should return 0 as the exitcode
+ bool no_run; // to not run the tests at all (can be done with an "*" exclude)
+ bool no_version; // to not print the version of the framework
+ bool no_colors; // if output to the console should be colorized
+ bool force_colors; // forces the use of colors even when a tty cannot be detected
+ bool no_breaks; // to not break into the debugger
+ bool no_skip; // don't skip test cases which are marked to be skipped
+ bool gnu_file_line; // if line numbers should be surrounded with :x: and not (x):
+ bool no_path_in_filenames; // if the path to files should be removed from the output
+ bool no_line_numbers; // if source code line numbers should be omitted from the output
+ bool no_debug_output; // no output in the debug console when a debugger is attached
+ bool no_skipped_summary; // don't print "skipped" in the summary !!! UNDOCUMENTED !!!
+ bool no_time_in_output; // omit any time/timestamps from output !!! UNDOCUMENTED !!!
+
+ bool help; // to print the help
+ bool version; // to print the version
+ bool count; // if only the count of matching tests is to be retrieved
+ bool list_test_cases; // to list all tests matching the filters
+ bool list_test_suites; // to list all suites matching the filters
+ bool list_reporters; // lists all registered reporters
+};
+
+namespace detail {
+ template <bool CONDITION, typename TYPE = void>
+ struct enable_if
+ {};
+
+ template <typename TYPE>
+ struct enable_if<true, TYPE>
+ { typedef TYPE type; };
+
+ // clang-format off
+ template<class T> struct remove_reference { typedef T type; };
+ template<class T> struct remove_reference<T&> { typedef T type; };
+ template<class T> struct remove_reference<T&&> { typedef T type; };
+
+ template<typename T, typename U = T&&> U declval(int);
+
+ template<typename T> T declval(long);
+
+ template<typename T> auto declval() DOCTEST_NOEXCEPT -> decltype(declval<T>(0)) ;
+
+ template<class T> struct is_lvalue_reference { const static bool value=false; };
+ template<class T> struct is_lvalue_reference<T&> { const static bool value=true; };
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type& t) DOCTEST_NOEXCEPT
+ {
+ return static_cast<T&&>(t);
+ }
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type&& t) DOCTEST_NOEXCEPT
+ {
+ static_assert(!is_lvalue_reference<T>::value,
+ "Can not forward an rvalue as an lvalue.");
+ return static_cast<T&&>(t);
+ }
+
+ template<class T> struct remove_const { typedef T type; };
+ template<class T> struct remove_const<const T> { typedef T type; };
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template<class T> struct is_enum : public std::is_enum<T> {};
+ template<class T> struct underlying_type : public std::underlying_type<T> {};
+#else
+ // Use compiler intrinsics
+ template<class T> struct is_enum { constexpr static bool value = __is_enum(T); };
+ template<class T> struct underlying_type { typedef __underlying_type(T) type; };
+#endif
+ // clang-format on
+
+ template <typename T>
+ struct deferred_false
+ // cppcheck-suppress unusedStructMember
+ { static const bool value = false; };
+
+ namespace has_insertion_operator_impl {
+ std::ostream &os();
+ template<class T>
+ DOCTEST_REF_WRAP(T) val();
+
+ template<class, class = void>
+ struct check {
+ static constexpr bool value = false;
+ };
+
+ template<class T>
+ struct check<T, decltype(os() << val<T>(), void())> {
+ static constexpr bool value = true;
+ };
+ } // namespace has_insertion_operator_impl
+
+ template<class T>
+ using has_insertion_operator = has_insertion_operator_impl::check<const T>;
+
+ DOCTEST_INTERFACE void my_memcpy(void* dest, const void* src, unsigned num);
+
+ DOCTEST_INTERFACE std::ostream* getTlsOss(); // returns a thread-local ostringstream
+ DOCTEST_INTERFACE String getTlsOssResult();
+
+ template <bool C>
+ struct StringMakerBase
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T)) {
+ return "{?}";
+ }
+ };
+
+ template <>
+ struct StringMakerBase<true>
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T) in) {
+ *getTlsOss() << in;
+ return getTlsOssResult();
+ }
+ };
+
+ DOCTEST_INTERFACE String rawMemoryToString(const void* object, unsigned size);
+
+ template <typename T>
+ String rawMemoryToString(const DOCTEST_REF_WRAP(T) object) {
+ return rawMemoryToString(&object, sizeof(object));
+ }
+
+ template <typename T>
+ const char* type_to_string() {
+ return "<>";
+ }
+} // namespace detail
+
+template <typename T>
+struct StringMaker : public detail::StringMakerBase<detail::has_insertion_operator<T>::value>
+{};
+
+template <typename T>
+struct StringMaker<T*>
+{
+ template <typename U>
+ static String convert(U* p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename R, typename C>
+struct StringMaker<R C::*>
+{
+ static String convert(R C::*p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename T, typename detail::enable_if<!detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ return StringMaker<T>::convert(value);
+}
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(char* in);
+DOCTEST_INTERFACE String toString(const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(bool in);
+DOCTEST_INTERFACE String toString(float in);
+DOCTEST_INTERFACE String toString(double in);
+DOCTEST_INTERFACE String toString(double long in);
+
+DOCTEST_INTERFACE String toString(char in);
+DOCTEST_INTERFACE String toString(char signed in);
+DOCTEST_INTERFACE String toString(char unsigned in);
+DOCTEST_INTERFACE String toString(int short in);
+DOCTEST_INTERFACE String toString(int short unsigned in);
+DOCTEST_INTERFACE String toString(int in);
+DOCTEST_INTERFACE String toString(int unsigned in);
+DOCTEST_INTERFACE String toString(int long in);
+DOCTEST_INTERFACE String toString(int long unsigned in);
+DOCTEST_INTERFACE String toString(int long long in);
+DOCTEST_INTERFACE String toString(int long long unsigned in);
+DOCTEST_INTERFACE String toString(std::nullptr_t in);
+
+template <typename T, typename detail::enable_if<detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ typedef typename detail::underlying_type<T>::type UT;
+ return toString(static_cast<UT>(value));
+}
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+DOCTEST_INTERFACE String toString(const std::string& in);
+#endif // VS 2019
+
+class DOCTEST_INTERFACE Approx
+{
+public:
+ explicit Approx(double value);
+
+ Approx operator()(double value) const;
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ explicit Approx(const T& value,
+ typename detail::enable_if<std::is_constructible<double, T>::value>::type* =
+ static_cast<T*>(nullptr)) {
+ *this = Approx(static_cast<double>(value));
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& epsilon(double newEpsilon);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type epsilon(
+ const T& newEpsilon) {
+ m_epsilon = static_cast<double>(newEpsilon);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& scale(double newScale);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type scale(
+ const T& newScale) {
+ m_scale = static_cast<double>(newScale);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format off
+ DOCTEST_INTERFACE friend bool operator==(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator==(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator!=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator!=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator<=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator<=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator>=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator>=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator< (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator< (const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator> (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator> (const Approx & lhs, double rhs);
+
+ DOCTEST_INTERFACE friend String toString(const Approx& in);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_APPROX_PREFIX \
+ template <typename T> friend typename detail::enable_if<std::is_constructible<double, T>::value, bool>::type
+
+ DOCTEST_APPROX_PREFIX operator==(const T& lhs, const Approx& rhs) { return operator==(double(lhs), rhs); }
+ DOCTEST_APPROX_PREFIX operator==(const Approx& lhs, const T& rhs) { return operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const T& lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const Approx& lhs, const T& rhs) { return !operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator<=(const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator<=(const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) && lhs != rhs; }
+#undef DOCTEST_APPROX_PREFIX
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format on
+
+private:
+ double m_epsilon;
+ double m_scale;
+ double m_value;
+};
+
+DOCTEST_INTERFACE String toString(const Approx& in);
+
+DOCTEST_INTERFACE const ContextOptions* getContextOptions();
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+namespace detail {
+ // clang-format off
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ template<class T> struct decay_array { typedef T type; };
+ template<class T, unsigned N> struct decay_array<T[N]> { typedef T* type; };
+ template<class T> struct decay_array<T[]> { typedef T* type; };
+
+ template<class T> struct not_char_pointer { enum { value = 1 }; };
+ template<> struct not_char_pointer<char*> { enum { value = 0 }; };
+ template<> struct not_char_pointer<const char*> { enum { value = 0 }; };
+
+ template<class T> struct can_use_op : public not_char_pointer<typename decay_array<T>::type> {};
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+ struct DOCTEST_INTERFACE TestFailureException
+ {
+ };
+
+ DOCTEST_INTERFACE bool checkIfShouldThrow(assertType::Enum at);
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_INTERFACE void throwException();
+
+ struct DOCTEST_INTERFACE Subcase
+ {
+ SubcaseSignature m_signature;
+ bool m_entered = false;
+
+ Subcase(const String& name, const char* file, int line);
+ ~Subcase();
+
+ operator bool() const;
+ };
+
+ template <typename L, typename R>
+ String stringifyBinaryExpr(const DOCTEST_REF_WRAP(L) lhs, const char* op,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return toString(lhs) + op + toString(rhs);
+ }
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison")
+#endif
+
+// This will check if there is any way it could find a operator like member or friend and uses it.
+// If not it doesn't find the operator or if the operator at global scope is defined after
+// this template, the template won't be instantiated due to SFINAE. Once the template is not
+// instantiated it can look for global operator using normal conversions.
+#define SFINAE_OP(ret,op) decltype(doctest::detail::declval<L>() op doctest::detail::declval<R>(),static_cast<ret>(0))
+
+#define DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(op, op_str, op_macro) \
+ template <typename R> \
+ DOCTEST_NOINLINE SFINAE_OP(Result,op) operator op(R&& rhs) { \
+ bool res = op_macro(doctest::detail::forward<L>(lhs), doctest::detail::forward<R>(rhs)); \
+ if(m_at & assertType::is_false) \
+ res = !res; \
+ if(!res || doctest::getContextOptions()->success) \
+ return Result(res, stringifyBinaryExpr(lhs, op_str, rhs)); \
+ return Result(res); \
+ }
+
+ // more checks could be added - like in Catch:
+ // https://github.com/catchorg/Catch2/pull/1480/files
+ // https://github.com/catchorg/Catch2/pull/1481/files
+#define DOCTEST_FORBIT_EXPRESSION(rt, op) \
+ template <typename R> \
+ rt& operator op(const R&) { \
+ static_assert(deferred_false<R>::value, \
+ "Expression Too Complex Please Rewrite As Binary Comparison!"); \
+ return *this; \
+ }
+
+ struct DOCTEST_INTERFACE Result
+ {
+ bool m_passed;
+ String m_decomp;
+
+ Result(bool passed, const String& decomposition = String());
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Result, &)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^)
+ DOCTEST_FORBIT_EXPRESSION(Result, |)
+ DOCTEST_FORBIT_EXPRESSION(Result, &&)
+ DOCTEST_FORBIT_EXPRESSION(Result, ||)
+ DOCTEST_FORBIT_EXPRESSION(Result, ==)
+ DOCTEST_FORBIT_EXPRESSION(Result, !=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <)
+ DOCTEST_FORBIT_EXPRESSION(Result, >)
+ DOCTEST_FORBIT_EXPRESSION(Result, <=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >=)
+ DOCTEST_FORBIT_EXPRESSION(Result, =)
+ DOCTEST_FORBIT_EXPRESSION(Result, +=)
+ DOCTEST_FORBIT_EXPRESSION(Result, -=)
+ DOCTEST_FORBIT_EXPRESSION(Result, *=)
+ DOCTEST_FORBIT_EXPRESSION(Result, /=)
+ DOCTEST_FORBIT_EXPRESSION(Result, %=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Result, &=)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Result, |=)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+ // https://stackoverflow.com/questions/39479163 what's the difference between 4018 and 4389
+ DOCTEST_MSVC_SUPPRESS_WARNING(4388) // signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4389) // 'operator' : signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4018) // 'expression' : signed/unsigned mismatch
+ //DOCTEST_MSVC_SUPPRESS_WARNING(4805) // 'operation' : unsafe mix of type 'type' and type 'type' in operation
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ // clang-format off
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE bool
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE typename enable_if<can_use_op<L>::value || can_use_op<R>::value, bool>::type
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ inline bool eq(const char* lhs, const char* rhs) { return String(lhs) == String(rhs); }
+ inline bool ne(const char* lhs, const char* rhs) { return String(lhs) != String(rhs); }
+ inline bool lt(const char* lhs, const char* rhs) { return String(lhs) < String(rhs); }
+ inline bool gt(const char* lhs, const char* rhs) { return String(lhs) > String(rhs); }
+ inline bool le(const char* lhs, const char* rhs) { return String(lhs) <= String(rhs); }
+ inline bool ge(const char* lhs, const char* rhs) { return String(lhs) >= String(rhs); }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+#define DOCTEST_RELATIONAL_OP(name, op) \
+ template <typename L, typename R> \
+ DOCTEST_COMPARISON_RETURN_TYPE name(const DOCTEST_REF_WRAP(L) lhs, \
+ const DOCTEST_REF_WRAP(R) rhs) { \
+ return lhs op rhs; \
+ }
+
+ DOCTEST_RELATIONAL_OP(eq, ==)
+ DOCTEST_RELATIONAL_OP(ne, !=)
+ DOCTEST_RELATIONAL_OP(lt, <)
+ DOCTEST_RELATIONAL_OP(gt, >)
+ DOCTEST_RELATIONAL_OP(le, <=)
+ DOCTEST_RELATIONAL_OP(ge, >=)
+
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) l == r
+#define DOCTEST_CMP_NE(l, r) l != r
+#define DOCTEST_CMP_GT(l, r) l > r
+#define DOCTEST_CMP_LT(l, r) l < r
+#define DOCTEST_CMP_GE(l, r) l >= r
+#define DOCTEST_CMP_LE(l, r) l <= r
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) eq(l, r)
+#define DOCTEST_CMP_NE(l, r) ne(l, r)
+#define DOCTEST_CMP_GT(l, r) gt(l, r)
+#define DOCTEST_CMP_LT(l, r) lt(l, r)
+#define DOCTEST_CMP_GE(l, r) ge(l, r)
+#define DOCTEST_CMP_LE(l, r) le(l, r)
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+
+ template <typename L>
+ // cppcheck-suppress copyCtorAndEqOperator
+ struct Expression_lhs
+ {
+ L lhs;
+ assertType::Enum m_at;
+
+ explicit Expression_lhs(L&& in, assertType::Enum at)
+ : lhs(doctest::detail::forward<L>(in))
+ , m_at(at) {}
+
+ DOCTEST_NOINLINE operator Result() {
+// this is needed only foc MSVC 2015:
+// https://ci.appveyor.com/project/onqtam/doctest/builds/38181202
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4800) // 'int': forcing value to bool
+ bool res = static_cast<bool>(lhs);
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ res = !res;
+
+ if(!res || getContextOptions()->success)
+ return Result(res, toString(lhs));
+ return Result(res);
+ }
+
+ /* This is required for user-defined conversions from Expression_lhs to L */
+ //operator L() const { return lhs; }
+ operator L() const { return lhs; }
+
+ // clang-format off
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(==, " == ", DOCTEST_CMP_EQ) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(!=, " != ", DOCTEST_CMP_NE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>, " > ", DOCTEST_CMP_GT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<, " < ", DOCTEST_CMP_LT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>=, " >= ", DOCTEST_CMP_GE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<=, " <= ", DOCTEST_CMP_LE) //!OCLINT bitwise operator in conditional
+ // clang-format on
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &&)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ||)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, =)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, +=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, -=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, *=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, /=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, %=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |=)
+ // these 2 are unfortunate because they should be allowed - they have higher precedence over the comparisons, but the
+ // ExpressionDecomposer class uses the left shift operator to capture the left operand of the binary expression...
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#endif
+
+ struct DOCTEST_INTERFACE ExpressionDecomposer
+ {
+ assertType::Enum m_at;
+
+ ExpressionDecomposer(assertType::Enum at);
+
+ // The right operator for capturing expressions is "<=" instead of "<<" (based on the operator precedence table)
+ // but then there will be warnings from GCC about "-Wparentheses" and since "_Pragma()" is problematic this will stay for now...
+ // https://github.com/catchorg/Catch2/issues/870
+ // https://github.com/catchorg/Catch2/issues/565
+ template <typename L>
+ Expression_lhs<L> operator<<(L &&operand) {
+ return Expression_lhs<L>(doctest::detail::forward<L>(operand), m_at);
+ }
+ };
+
+ struct DOCTEST_INTERFACE TestSuite
+ {
+ const char* m_test_suite;
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+
+ TestSuite& operator*(const char* in);
+
+ template <typename T>
+ TestSuite& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+ };
+
+ typedef void (*funcType)();
+
+ struct DOCTEST_INTERFACE TestCase : public TestCaseData
+ {
+ funcType m_test; // a function pointer to the test case
+
+ const char* m_type; // for templated test cases - gets appended to the real name
+ int m_template_id; // an ID used to distinguish between the different versions of a templated test case
+ String m_full_name; // contains the name (only for templated test cases!) + the template type
+
+ TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type = "", int template_id = -1);
+
+ TestCase(const TestCase& other);
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ TestCase& operator=(const TestCase& other);
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& operator*(const char* in);
+
+ template <typename T>
+ TestCase& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+
+ bool operator<(const TestCase& other) const;
+ };
+
+ // forward declarations of functions used by the macros
+ DOCTEST_INTERFACE int regTest(const TestCase& tc);
+ DOCTEST_INTERFACE int setTestSuite(const TestSuite& ts);
+ DOCTEST_INTERFACE bool isDebuggerActive();
+
+ template<typename T>
+ int instantiationHelper(const T&) { return 0; }
+
+ namespace binaryAssertComparison {
+ enum Enum
+ {
+ eq = 0,
+ ne,
+ gt,
+ lt,
+ ge,
+ le
+ };
+ } // namespace binaryAssertComparison
+
+ // clang-format off
+ template <int, class L, class R> struct RelationalComparator { bool operator()(const DOCTEST_REF_WRAP(L), const DOCTEST_REF_WRAP(R) ) const { return false; } };
+
+#define DOCTEST_BINARY_RELATIONAL_OP(n, op) \
+ template <class L, class R> struct RelationalComparator<n, L, R> { bool operator()(const DOCTEST_REF_WRAP(L) lhs, const DOCTEST_REF_WRAP(R) rhs) const { return op(lhs, rhs); } };
+ // clang-format on
+
+ DOCTEST_BINARY_RELATIONAL_OP(0, doctest::detail::eq)
+ DOCTEST_BINARY_RELATIONAL_OP(1, doctest::detail::ne)
+ DOCTEST_BINARY_RELATIONAL_OP(2, doctest::detail::gt)
+ DOCTEST_BINARY_RELATIONAL_OP(3, doctest::detail::lt)
+ DOCTEST_BINARY_RELATIONAL_OP(4, doctest::detail::ge)
+ DOCTEST_BINARY_RELATIONAL_OP(5, doctest::detail::le)
+
+ struct DOCTEST_INTERFACE ResultBuilder : public AssertData
+ {
+ ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type = "", const char* exception_string = "");
+
+ void setResult(const Result& res);
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ m_failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+ if(m_failed || getContextOptions()->success)
+ m_decomp = stringifyBinaryExpr(lhs, ", ", rhs);
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(const DOCTEST_REF_WRAP(L) val) {
+ m_failed = !val;
+
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ m_failed = !m_failed;
+
+ if(m_failed || getContextOptions()->success)
+ m_decomp = toString(val);
+ }
+
+ void translateException();
+
+ bool log();
+ void react() const;
+ };
+
+ namespace assertAction {
+ enum Enum
+ {
+ nothing = 0,
+ dbgbreak = 1,
+ shouldthrow = 2
+ };
+ } // namespace assertAction
+
+ DOCTEST_INTERFACE void failed_out_of_a_testing_context(const AssertData& ad);
+
+ DOCTEST_INTERFACE void decomp_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, Result result);
+
+#define DOCTEST_ASSERT_OUT_OF_TESTS(decomp) \
+ do { \
+ if(!is_running_in_test) { \
+ if(failed) { \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ rb.m_decomp = decomp; \
+ failed_out_of_a_testing_context(rb); \
+ if(isDebuggerActive() && !getContextOptions()->no_breaks) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(checkIfShouldThrow(at)) \
+ throwException(); \
+ } \
+ return; \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_IN_TESTS(decomp) \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ if(rb.m_failed || getContextOptions()->success) \
+ rb.m_decomp = decomp; \
+ if(rb.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(rb.m_failed && checkIfShouldThrow(at)) \
+ throwException()
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ bool failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ DOCTEST_ASSERT_IN_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) val) {
+ bool failed = !val;
+
+ if(at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ failed = !failed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(toString(val));
+ DOCTEST_ASSERT_IN_TESTS(toString(val));
+ }
+
+ struct DOCTEST_INTERFACE IExceptionTranslator
+ {
+ IExceptionTranslator();
+ virtual ~IExceptionTranslator();
+ virtual bool translate(String&) const = 0;
+ };
+
+ template <typename T>
+ class ExceptionTranslator : public IExceptionTranslator //!OCLINT destructor of virtual class
+ {
+ public:
+ explicit ExceptionTranslator(String (*translateFunction)(T))
+ : m_translateFunction(translateFunction) {}
+
+ bool translate(String& res) const override {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+ throw; // lgtm [cpp/rethrow-no-exception]
+ // cppcheck-suppress catchExceptionByValue
+ } catch(T ex) { // NOLINT
+ res = m_translateFunction(ex); //!OCLINT parameter reassignment
+ return true;
+ } catch(...) {} //!OCLINT - empty catch statement
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ static_cast<void>(res); // to silence -Wunused-parameter
+ return false;
+ }
+
+ private:
+ String (*m_translateFunction)(T);
+ };
+
+ DOCTEST_INTERFACE void registerExceptionTranslatorImpl(const IExceptionTranslator* et);
+
+ template <bool C>
+ struct StringStreamBase
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << toString(in);
+ }
+
+ // always treat char* as a string in this context - no matter
+ // if DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING is defined
+ static void convert(std::ostream* s, const char* in) { *s << String(in); }
+ };
+
+ template <>
+ struct StringStreamBase<true>
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << in;
+ }
+ };
+
+ template <typename T>
+ struct StringStream : public StringStreamBase<has_insertion_operator<T>::value>
+ {};
+
+ template <typename T>
+ void toStream(std::ostream* s, const T& value) {
+ StringStream<T>::convert(s, value);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char* in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, bool in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, float in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double long in);
+
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char signed in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long unsigned in);
+
+ // ContextScope base class used to allow implementing methods of ContextScope
+ // that don't depend on the template parameter in doctest.cpp.
+ class DOCTEST_INTERFACE ContextScopeBase : public IContextScope {
+ protected:
+ ContextScopeBase();
+
+ void destroy();
+ };
+
+ template <typename L> class ContextScope : public ContextScopeBase
+ {
+ const L lambda_;
+
+ public:
+ explicit ContextScope(const L &lambda) : lambda_(lambda) {}
+
+ ContextScope(ContextScope &&other) : lambda_(other.lambda_) {}
+
+ void stringify(std::ostream* s) const override { lambda_(s); }
+
+ ~ContextScope() override { destroy(); }
+ };
+
+ struct DOCTEST_INTERFACE MessageBuilder : public MessageData
+ {
+ std::ostream* m_stream;
+
+ MessageBuilder(const char* file, int line, assertType::Enum severity);
+ MessageBuilder() = delete;
+ ~MessageBuilder();
+
+ // the preferred way of chaining parameters for stringification
+ template <typename T>
+ MessageBuilder& operator,(const T& in) {
+ toStream(m_stream, in);
+ return *this;
+ }
+
+ // kept here just for backwards-compatibility - the comma operator should be preferred now
+ template <typename T>
+ MessageBuilder& operator<<(const T& in) { return this->operator,(in); }
+
+ // the `,` operator has the lowest operator precedence - if `<<` is used by the user then
+ // the `,` operator will be called last which is not what we want and thus the `*` operator
+ // is used first (has higher operator precedence compared to `<<`) so that we guarantee that
+ // an operator of the MessageBuilder class is called first before the rest of the parameters
+ template <typename T>
+ MessageBuilder& operator*(const T& in) { return this->operator,(in); }
+
+ bool log();
+ void react();
+ };
+
+ template <typename L>
+ ContextScope<L> MakeContextScope(const L &lambda) {
+ return ContextScope<L>(lambda);
+ }
+} // namespace detail
+
+#define DOCTEST_DEFINE_DECORATOR(name, type, def) \
+ struct name \
+ { \
+ type data; \
+ name(type in = def) \
+ : data(in) {} \
+ void fill(detail::TestCase& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ void fill(detail::TestSuite& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ }
+
+DOCTEST_DEFINE_DECORATOR(test_suite, const char*, "");
+DOCTEST_DEFINE_DECORATOR(description, const char*, "");
+DOCTEST_DEFINE_DECORATOR(skip, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_breaks, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_output, bool, true);
+DOCTEST_DEFINE_DECORATOR(timeout, double, 0);
+DOCTEST_DEFINE_DECORATOR(may_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(should_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(expected_failures, int, 0);
+
+template <typename T>
+int registerExceptionTranslator(String (*translateFunction)(T)) {
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors")
+ static detail::ExceptionTranslator<T> exceptionTranslator(translateFunction);
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ detail::registerExceptionTranslatorImpl(&exceptionTranslator);
+ return 0;
+}
+
+} // namespace doctest
+
+// in a separate namespace outside of doctest because the DOCTEST_TEST_SUITE macro
+// introduces an anonymous namespace in which getCurrentTestSuite gets overridden
+namespace doctest_detail_test_suite_ns {
+DOCTEST_INTERFACE doctest::detail::TestSuite& getCurrentTestSuite();
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+#else // DOCTEST_CONFIG_DISABLE
+template <typename T>
+int registerExceptionTranslator(String (*)(T)) {
+ return 0;
+}
+#endif // DOCTEST_CONFIG_DISABLE
+
+namespace detail {
+ typedef void (*assert_handler)(const AssertData&);
+ struct ContextState;
+} // namespace detail
+
+class DOCTEST_INTERFACE Context
+{
+ detail::ContextState* p;
+
+ void parseArgs(int argc, const char* const* argv, bool withDefaults = false);
+
+public:
+ explicit Context(int argc = 0, const char* const* argv = nullptr);
+
+ ~Context();
+
+ void applyCommandLine(int argc, const char* const* argv);
+
+ void addFilter(const char* filter, const char* value);
+ void clearFilters();
+ void setOption(const char* option, int value);
+ void setOption(const char* option, const char* value);
+
+ bool shouldExit();
+
+ void setAsDefaultForAssertsOutOfTestCases();
+
+ void setAssertHandler(detail::assert_handler ah);
+
+ int run();
+};
+
+namespace TestCaseFailureReason {
+ enum Enum
+ {
+ None = 0,
+ AssertFailure = 1, // an assertion has failed in the test case
+ Exception = 2, // test case threw an exception
+ Crash = 4, // a crash...
+ TooManyFailedAsserts = 8, // the abort-after option
+ Timeout = 16, // see the timeout decorator
+ ShouldHaveFailedButDidnt = 32, // see the should_fail decorator
+ ShouldHaveFailedAndDid = 64, // see the should_fail decorator
+ DidntFailExactlyNumTimes = 128, // see the expected_failures decorator
+ FailedExactlyNumTimes = 256, // see the expected_failures decorator
+ CouldHaveFailedAndDid = 512 // see the may_fail decorator
+ };
+} // namespace TestCaseFailureReason
+
+struct DOCTEST_INTERFACE CurrentTestCaseStats
+{
+ int numAssertsCurrentTest;
+ int numAssertsFailedCurrentTest;
+ double seconds;
+ int failure_flags; // use TestCaseFailureReason::Enum
+};
+
+struct DOCTEST_INTERFACE TestCaseException
+{
+ String error_string;
+ bool is_crash;
+};
+
+struct DOCTEST_INTERFACE TestRunStats
+{
+ unsigned numTestCases;
+ unsigned numTestCasesPassingFilters;
+ unsigned numTestSuitesPassingFilters;
+ unsigned numTestCasesFailed;
+ int numAsserts;
+ int numAssertsFailed;
+};
+
+struct QueryData
+{
+ const TestRunStats* run_stats = nullptr;
+ const TestCaseData** data = nullptr;
+ unsigned num_data = 0;
+};
+
+struct DOCTEST_INTERFACE IReporter
+{
+ // The constructor has to accept "const ContextOptions&" as a single argument
+ // which has most of the options for the run + a pointer to the stdout stream
+ // Reporter(const ContextOptions& in)
+
+ // called when a query should be reported (listing test cases, printing the version, etc.)
+ virtual void report_query(const QueryData&) = 0;
+
+ // called when the whole test run starts
+ virtual void test_run_start() = 0;
+ // called when the whole test run ends (caching a pointer to the input doesn't make sense here)
+ virtual void test_run_end(const TestRunStats&) = 0;
+
+ // called when a test case is started (safe to cache a pointer to the input)
+ virtual void test_case_start(const TestCaseData&) = 0;
+ // called when a test case is reentered because of unfinished subcases (safe to cache a pointer to the input)
+ virtual void test_case_reenter(const TestCaseData&) = 0;
+ // called when a test case has ended
+ virtual void test_case_end(const CurrentTestCaseStats&) = 0;
+
+ // called when an exception is thrown from the test case (or it crashes)
+ virtual void test_case_exception(const TestCaseException&) = 0;
+
+ // called whenever a subcase is entered (don't cache pointers to the input)
+ virtual void subcase_start(const SubcaseSignature&) = 0;
+ // called whenever a subcase is exited (don't cache pointers to the input)
+ virtual void subcase_end() = 0;
+
+ // called for each assert (don't cache pointers to the input)
+ virtual void log_assert(const AssertData&) = 0;
+ // called for each message (don't cache pointers to the input)
+ virtual void log_message(const MessageData&) = 0;
+
+ // called when a test case is skipped either because it doesn't pass the filters, has a skip decorator
+ // or isn't in the execution range (between first and last) (safe to cache a pointer to the input)
+ virtual void test_case_skipped(const TestCaseData&) = 0;
+
+ // doctest will not be managing the lifetimes of reporters given to it but this would still be nice to have
+ virtual ~IReporter();
+
+ // can obtain all currently active contexts and stringify them if one wishes to do so
+ static int get_num_active_contexts();
+ static const IContextScope* const* get_active_contexts();
+
+ // can iterate through contexts which have been stringified automatically in their destructors when an exception has been thrown
+ static int get_num_stringified_contexts();
+ static const String* get_stringified_contexts();
+};
+
+namespace detail {
+ typedef IReporter* (*reporterCreatorFunc)(const ContextOptions&);
+
+ DOCTEST_INTERFACE void registerReporterImpl(const char* name, int prio, reporterCreatorFunc c, bool isReporter);
+
+ template <typename Reporter>
+ IReporter* reporterCreator(const ContextOptions& o) {
+ return new Reporter(o);
+ }
+} // namespace detail
+
+template <typename Reporter>
+int registerReporter(const char* name, int priority, bool isReporter) {
+ detail::registerReporterImpl(name, priority, detail::reporterCreator<Reporter>, isReporter);
+ return 0;
+}
+} // namespace doctest
+
+// if registering is not disabled
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// common code in asserts - for convenience
+#define DOCTEST_ASSERT_LOG_AND_REACT(b) \
+ if(b.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ b.react()
+
+#ifdef DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) x;
+#else // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) \
+ try { \
+ x; \
+ } catch(...) { _DOCTEST_RB.translateException(); }
+#endif // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#ifdef DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wuseless-cast") \
+ static_cast<void>(__VA_ARGS__); \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+#else // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) __VA_ARGS__;
+#endif // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+
+// registers the test by initializing a dummy var with a function
+#define DOCTEST_REGISTER_FUNCTION(global_prefix, f, decorators) \
+ global_prefix DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::regTest( \
+ doctest::detail::TestCase( \
+ f, __FILE__, __LINE__, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite()) * \
+ decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, decorators) \
+ namespace { \
+ struct der : public base \
+ { \
+ void f(); \
+ }; \
+ static void func() { \
+ der v; \
+ v.f(); \
+ } \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, func, decorators) \
+ } \
+ inline DOCTEST_NOINLINE void der::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, decorators) \
+ static void f(); \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, f, decorators) \
+ static void f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(f, proxy, decorators) \
+ static doctest::detail::funcType proxy() { return f; } \
+ DOCTEST_REGISTER_FUNCTION(inline const, proxy(), decorators) \
+ static void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for registering tests in classes - requires C++17 for inline variables!
+#if __cplusplus >= 201703L || (DOCTEST_MSVC >= DOCTEST_COMPILER(19, 12, 0) && _MSVC_LANG >= 201703L)
+#define DOCTEST_TEST_CASE_CLASS(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_PROXY_), \
+ decorators)
+#else // DOCTEST_TEST_CASE_CLASS
+#define DOCTEST_TEST_CASE_CLASS(...) \
+ TEST_CASES_CAN_BE_REGISTERED_IN_CLASSES_ONLY_IN_CPP17_MODE_OR_WITH_VS_2017_OR_NEWER
+#endif // DOCTEST_TEST_CASE_CLASS
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(c, decorators) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), c, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING_IMPL(...) \
+ template <> \
+ inline const char* type_to_string<__VA_ARGS__>() { \
+ return "<" #__VA_ARGS__ ">"; \
+ }
+#define DOCTEST_TYPE_TO_STRING(...) \
+ namespace doctest { namespace detail { \
+ DOCTEST_TYPE_TO_STRING_IMPL(__VA_ARGS__) \
+ } \
+ } \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, iter, func) \
+ template <typename T> \
+ static void func(); \
+ namespace { \
+ template <typename Tuple> \
+ struct iter; \
+ template <typename Type, typename... Rest> \
+ struct iter<std::tuple<Type, Rest...>> \
+ { \
+ iter(const char* file, unsigned line, int index) { \
+ doctest::detail::regTest(doctest::detail::TestCase(func<Type>, file, line, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite(), \
+ doctest::detail::type_to_string<Type>(), \
+ int(line) * 1000 + index) \
+ * dec); \
+ iter<std::tuple<Rest...>>(file, line, index + 1); \
+ } \
+ }; \
+ template <> \
+ struct iter<std::tuple<>> \
+ { \
+ iter(const char*, unsigned, int) {} \
+ }; \
+ } \
+ template <typename T> \
+ static void func()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(dec, T, id) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(id, ITERATOR), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_))
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, anon, ...) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_CAT(anon, DUMMY)) = \
+ doctest::detail::instantiationHelper(DOCTEST_CAT(id, ITERATOR)<__VA_ARGS__>(__FILE__, __LINE__, 0));\
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), std::tuple<__VA_ARGS__>) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, anon, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(anon, ITERATOR), anon); \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(anon, anon, std::tuple<__VA_ARGS__>) \
+ template <typename T> \
+ static void anon()
+
+#define DOCTEST_TEST_CASE_TEMPLATE(dec, T, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__)
+
+// for subcases
+#define DOCTEST_SUBCASE(name) \
+ if(const doctest::detail::Subcase & DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUBCASE_) DOCTEST_UNUSED = \
+ doctest::detail::Subcase(name, __FILE__, __LINE__))
+
+// for grouping tests in test suites by using code blocks
+#define DOCTEST_TEST_SUITE_IMPL(decorators, ns_name) \
+ namespace ns_name { namespace doctest_detail_test_suite_ns { \
+ static DOCTEST_NOINLINE doctest::detail::TestSuite& getCurrentTestSuite() { \
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4640) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors") \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmissing-field-initializers") \
+ static doctest::detail::TestSuite data{}; \
+ static bool inited = false; \
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP \
+ if(!inited) { \
+ data* decorators; \
+ inited = true; \
+ } \
+ return data; \
+ } \
+ } \
+ } \
+ namespace ns_name
+
+#define DOCTEST_TEST_SUITE(decorators) \
+ DOCTEST_TEST_SUITE_IMPL(decorators, DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUITE_))
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(decorators) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * ""); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering exception translators
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(translatorName, signature) \
+ inline doctest::String translatorName(signature); \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)) = \
+ doctest::registerExceptionTranslator(translatorName); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ doctest::String translatorName(signature)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_), \
+ signature)
+
+// for registering reporters
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, true); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering listeners
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, false); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for logging
+#define DOCTEST_INFO(...) \
+ DOCTEST_INFO_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), \
+ __VA_ARGS__)
+
+#define DOCTEST_INFO_IMPL(mb_name, s_name, ...) \
+ auto DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_) = doctest::detail::MakeContextScope( \
+ [&](std::ostream* s_name) { \
+ doctest::detail::MessageBuilder mb_name(__FILE__, __LINE__, doctest::assertType::is_warn); \
+ mb_name.m_stream = s_name; \
+ mb_name * __VA_ARGS__; \
+ })
+
+#define DOCTEST_CAPTURE(x) DOCTEST_INFO(#x " := ", x)
+
+#define DOCTEST_ADD_AT_IMPL(type, file, line, mb, ...) \
+ do { \
+ doctest::detail::MessageBuilder mb(file, line, doctest::assertType::type); \
+ mb * __VA_ARGS__; \
+ DOCTEST_ASSERT_LOG_AND_REACT(mb); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+// clang-format on
+
+#define DOCTEST_MESSAGE(...) DOCTEST_ADD_MESSAGE_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL_CHECK(...) DOCTEST_ADD_FAIL_CHECK_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL(...) DOCTEST_ADD_FAIL_AT(__FILE__, __LINE__, __VA_ARGS__)
+
+#define DOCTEST_TO_LVALUE(...) __VA_ARGS__ // Not removed to keep backwards compatibility.
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_ASSERT_IMPLEMENT_2(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.setResult( \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ do { \
+ DOCTEST_ASSERT_IMPLEMENT_2(assert_type, __VA_ARGS__); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+// necessary for <ASSERT>_MESSAGE
+#define DOCTEST_ASSERT_IMPLEMENT_2 DOCTEST_ASSERT_IMPLEMENT_1
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::decomp_assert( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__) DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN, __VA_ARGS__)
+#define DOCTEST_CHECK(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK, __VA_ARGS__)
+#define DOCTEST_REQUIRE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE, __VA_ARGS__)
+#define DOCTEST_WARN_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE_FALSE, __VA_ARGS__)
+
+// clang-format off
+#define DOCTEST_WARN_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } while(false)
+#define DOCTEST_CHECK_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } while(false)
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } while(false)
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } while(false)
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } while(false)
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } while(false)
+// clang-format on
+
+#define DOCTEST_ASSERT_THROWS_AS(expr, assert_type, message, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #expr, #__VA_ARGS__, message); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(const typename doctest::detail::remove_const< \
+ typename doctest::detail::remove_reference<__VA_ARGS__>::type>::type&) { \
+ _DOCTEST_RB.translateException(); \
+ _DOCTEST_RB.m_threw_as = true; \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_THROWS_WITH(expr, expr_str, assert_type, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, expr_str, "", __VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_NOTHROW(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(__VA_ARGS__) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_WARN_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_WARN_THROWS, "")
+#define DOCTEST_CHECK_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_CHECK_THROWS, "")
+#define DOCTEST_REQUIRE_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_REQUIRE_THROWS, "")
+
+#define DOCTEST_WARN_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_AS, "", __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_WARN_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_CHECK_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_REQUIRE_THROWS_WITH, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_WITH_AS, message, __VA_ARGS__)
+
+#define DOCTEST_WARN_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_WARN_NOTHROW, __VA_ARGS__)
+#define DOCTEST_CHECK_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_CHECK_NOTHROW, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_REQUIRE_NOTHROW, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS(expr); } while(false)
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS(expr); } while(false)
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS(expr); } while(false)
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_NOTHROW(expr); } while(false)
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_NOTHROW(expr); } while(false)
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_NOTHROW(expr); } while(false)
+// clang-format on
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comp, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY( \
+ _DOCTEST_RB.binary_assert<doctest::detail::binaryAssertComparison::comp>( \
+ __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.unary_assert(__VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comparison, ...) \
+ doctest::detail::binary_assert<doctest::detail::binaryAssertComparison::comparison>( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, __VA_ARGS__)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ doctest::detail::unary_assert(doctest::assertType::assert_type, __FILE__, __LINE__, \
+ #__VA_ARGS__, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN_EQ(...) DOCTEST_BINARY_ASSERT(DT_WARN_EQ, eq, __VA_ARGS__)
+#define DOCTEST_CHECK_EQ(...) DOCTEST_BINARY_ASSERT(DT_CHECK_EQ, eq, __VA_ARGS__)
+#define DOCTEST_REQUIRE_EQ(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_EQ, eq, __VA_ARGS__)
+#define DOCTEST_WARN_NE(...) DOCTEST_BINARY_ASSERT(DT_WARN_NE, ne, __VA_ARGS__)
+#define DOCTEST_CHECK_NE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_NE, ne, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_NE, ne, __VA_ARGS__)
+#define DOCTEST_WARN_GT(...) DOCTEST_BINARY_ASSERT(DT_WARN_GT, gt, __VA_ARGS__)
+#define DOCTEST_CHECK_GT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GT, gt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GT, gt, __VA_ARGS__)
+#define DOCTEST_WARN_LT(...) DOCTEST_BINARY_ASSERT(DT_WARN_LT, lt, __VA_ARGS__)
+#define DOCTEST_CHECK_LT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LT, lt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LT, lt, __VA_ARGS__)
+#define DOCTEST_WARN_GE(...) DOCTEST_BINARY_ASSERT(DT_WARN_GE, ge, __VA_ARGS__)
+#define DOCTEST_CHECK_GE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GE, ge, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GE, ge, __VA_ARGS__)
+#define DOCTEST_WARN_LE(...) DOCTEST_BINARY_ASSERT(DT_WARN_LE, le, __VA_ARGS__)
+#define DOCTEST_CHECK_LE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LE, le, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LE, le, __VA_ARGS__)
+
+#define DOCTEST_WARN_UNARY(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY, __VA_ARGS__)
+#define DOCTEST_WARN_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY_FALSE, __VA_ARGS__)
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#undef DOCTEST_WARN_THROWS
+#undef DOCTEST_CHECK_THROWS
+#undef DOCTEST_REQUIRE_THROWS
+#undef DOCTEST_WARN_THROWS_AS
+#undef DOCTEST_CHECK_THROWS_AS
+#undef DOCTEST_REQUIRE_THROWS_AS
+#undef DOCTEST_WARN_THROWS_WITH
+#undef DOCTEST_CHECK_THROWS_WITH
+#undef DOCTEST_REQUIRE_THROWS_WITH
+#undef DOCTEST_WARN_THROWS_WITH_AS
+#undef DOCTEST_CHECK_THROWS_WITH_AS
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS
+#undef DOCTEST_WARN_NOTHROW
+#undef DOCTEST_CHECK_NOTHROW
+#undef DOCTEST_REQUIRE_NOTHROW
+
+#undef DOCTEST_WARN_THROWS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_MESSAGE
+#undef DOCTEST_WARN_THROWS_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_AS_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_WARN_NOTHROW_MESSAGE
+#undef DOCTEST_CHECK_NOTHROW_MESSAGE
+#undef DOCTEST_REQUIRE_NOTHROW_MESSAGE
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#undef DOCTEST_REQUIRE
+#undef DOCTEST_REQUIRE_FALSE
+#undef DOCTEST_REQUIRE_MESSAGE
+#undef DOCTEST_REQUIRE_FALSE_MESSAGE
+#undef DOCTEST_REQUIRE_EQ
+#undef DOCTEST_REQUIRE_NE
+#undef DOCTEST_REQUIRE_GT
+#undef DOCTEST_REQUIRE_LT
+#undef DOCTEST_REQUIRE_GE
+#undef DOCTEST_REQUIRE_LE
+#undef DOCTEST_REQUIRE_UNARY
+#undef DOCTEST_REQUIRE_UNARY_FALSE
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+// =================================================================================================
+// == WHAT FOLLOWS IS VERSIONS OF THE MACROS THAT DO NOT DO ANY REGISTERING! ==
+// == THIS CAN BE ENABLED BY DEFINING DOCTEST_CONFIG_DISABLE GLOBALLY! ==
+// =================================================================================================
+#else // DOCTEST_CONFIG_DISABLE
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, name) \
+ namespace { \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ struct der : public base \
+ { void f(); }; \
+ } \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ inline void der<DOCTEST_UNUSED_TEMPLATE_TYPE>::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, name) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests in classes
+#define DOCTEST_TEST_CASE_CLASS(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(x, name) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), x, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING(...) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+#define DOCTEST_TYPE_TO_STRING_IMPL(...)
+
+// for typed tests
+#define DOCTEST_TEST_CASE_TEMPLATE(name, type, ...) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, type, id) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for subcases
+#define DOCTEST_SUBCASE(name)
+
+// for a testsuite block
+#define DOCTEST_TEST_SUITE(name) namespace
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(name) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline doctest::String DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)(signature)
+
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+
+#define DOCTEST_INFO(...) (static_cast<void>(0))
+#define DOCTEST_CAPTURE(x) (static_cast<void>(0))
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_MESSAGE(...) (static_cast<void>(0))
+#define DOCTEST_FAIL_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_FAIL(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN(...) (static_cast<void>(0))
+#define DOCTEST_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_EQ(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_EQ(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_EQ(...) (static_cast<void>(0))
+#define DOCTEST_WARN_NE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_WARN_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) (static_cast<void>(0))
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+// clang-format off
+// KEPT FOR BACKWARDS COMPATIBILITY - FORWARDING TO THE RIGHT MACROS
+#define DOCTEST_FAST_WARN_EQ DOCTEST_WARN_EQ
+#define DOCTEST_FAST_CHECK_EQ DOCTEST_CHECK_EQ
+#define DOCTEST_FAST_REQUIRE_EQ DOCTEST_REQUIRE_EQ
+#define DOCTEST_FAST_WARN_NE DOCTEST_WARN_NE
+#define DOCTEST_FAST_CHECK_NE DOCTEST_CHECK_NE
+#define DOCTEST_FAST_REQUIRE_NE DOCTEST_REQUIRE_NE
+#define DOCTEST_FAST_WARN_GT DOCTEST_WARN_GT
+#define DOCTEST_FAST_CHECK_GT DOCTEST_CHECK_GT
+#define DOCTEST_FAST_REQUIRE_GT DOCTEST_REQUIRE_GT
+#define DOCTEST_FAST_WARN_LT DOCTEST_WARN_LT
+#define DOCTEST_FAST_CHECK_LT DOCTEST_CHECK_LT
+#define DOCTEST_FAST_REQUIRE_LT DOCTEST_REQUIRE_LT
+#define DOCTEST_FAST_WARN_GE DOCTEST_WARN_GE
+#define DOCTEST_FAST_CHECK_GE DOCTEST_CHECK_GE
+#define DOCTEST_FAST_REQUIRE_GE DOCTEST_REQUIRE_GE
+#define DOCTEST_FAST_WARN_LE DOCTEST_WARN_LE
+#define DOCTEST_FAST_CHECK_LE DOCTEST_CHECK_LE
+#define DOCTEST_FAST_REQUIRE_LE DOCTEST_REQUIRE_LE
+
+#define DOCTEST_FAST_WARN_UNARY DOCTEST_WARN_UNARY
+#define DOCTEST_FAST_CHECK_UNARY DOCTEST_CHECK_UNARY
+#define DOCTEST_FAST_REQUIRE_UNARY DOCTEST_REQUIRE_UNARY
+#define DOCTEST_FAST_WARN_UNARY_FALSE DOCTEST_WARN_UNARY_FALSE
+#define DOCTEST_FAST_CHECK_UNARY_FALSE DOCTEST_CHECK_UNARY_FALSE
+#define DOCTEST_FAST_REQUIRE_UNARY_FALSE DOCTEST_REQUIRE_UNARY_FALSE
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id,__VA_ARGS__)
+// clang-format on
+
+// BDD style macros
+// clang-format off
+#define DOCTEST_SCENARIO(name) DOCTEST_TEST_CASE(" Scenario: " name)
+#define DOCTEST_SCENARIO_CLASS(name) DOCTEST_TEST_CASE_CLASS(" Scenario: " name)
+#define DOCTEST_SCENARIO_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(" Scenario: " name, T, __VA_ARGS__)
+#define DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(" Scenario: " name, T, id)
+
+#define DOCTEST_GIVEN(name) DOCTEST_SUBCASE(" Given: " name)
+#define DOCTEST_WHEN(name) DOCTEST_SUBCASE(" When: " name)
+#define DOCTEST_AND_WHEN(name) DOCTEST_SUBCASE("And when: " name)
+#define DOCTEST_THEN(name) DOCTEST_SUBCASE(" Then: " name)
+#define DOCTEST_AND_THEN(name) DOCTEST_SUBCASE(" And: " name)
+// clang-format on
+
+// == SHORT VERSIONS OF THE MACROS
+#if !defined(DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES)
+
+#define TEST_CASE(name) DOCTEST_TEST_CASE(name)
+#define TEST_CASE_CLASS(name) DOCTEST_TEST_CASE_CLASS(name)
+#define TEST_CASE_FIXTURE(x, name) DOCTEST_TEST_CASE_FIXTURE(x, name)
+#define TYPE_TO_STRING(...) DOCTEST_TYPE_TO_STRING(__VA_ARGS__)
+#define TEST_CASE_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(name, T, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, T, id)
+#define TEST_CASE_TEMPLATE_INVOKE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_APPLY(id, ...) DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, __VA_ARGS__)
+#define SUBCASE(name) DOCTEST_SUBCASE(name)
+#define TEST_SUITE(decorators) DOCTEST_TEST_SUITE(decorators)
+#define TEST_SUITE_BEGIN(name) DOCTEST_TEST_SUITE_BEGIN(name)
+#define TEST_SUITE_END DOCTEST_TEST_SUITE_END
+#define REGISTER_EXCEPTION_TRANSLATOR(signature) DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature)
+#define REGISTER_REPORTER(name, priority, reporter) DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define REGISTER_LISTENER(name, priority, reporter) DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+#define INFO(...) DOCTEST_INFO(__VA_ARGS__)
+#define CAPTURE(x) DOCTEST_CAPTURE(x)
+#define ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_MESSAGE_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_FAIL_CHECK_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_FAIL_AT(file, line, __VA_ARGS__)
+#define MESSAGE(...) DOCTEST_MESSAGE(__VA_ARGS__)
+#define FAIL_CHECK(...) DOCTEST_FAIL_CHECK(__VA_ARGS__)
+#define FAIL(...) DOCTEST_FAIL(__VA_ARGS__)
+#define TO_LVALUE(...) DOCTEST_TO_LVALUE(__VA_ARGS__)
+
+#define WARN(...) DOCTEST_WARN(__VA_ARGS__)
+#define WARN_FALSE(...) DOCTEST_WARN_FALSE(__VA_ARGS__)
+#define WARN_THROWS(...) DOCTEST_WARN_THROWS(__VA_ARGS__)
+#define WARN_THROWS_AS(expr, ...) DOCTEST_WARN_THROWS_AS(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH(expr, ...) DOCTEST_WARN_THROWS_WITH(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS(expr, with, ...) DOCTEST_WARN_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define WARN_NOTHROW(...) DOCTEST_WARN_NOTHROW(__VA_ARGS__)
+#define CHECK(...) DOCTEST_CHECK(__VA_ARGS__)
+#define CHECK_FALSE(...) DOCTEST_CHECK_FALSE(__VA_ARGS__)
+#define CHECK_THROWS(...) DOCTEST_CHECK_THROWS(__VA_ARGS__)
+#define CHECK_THROWS_AS(expr, ...) DOCTEST_CHECK_THROWS_AS(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH(expr, ...) DOCTEST_CHECK_THROWS_WITH(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define CHECK_NOTHROW(...) DOCTEST_CHECK_NOTHROW(__VA_ARGS__)
+#define REQUIRE(...) DOCTEST_REQUIRE(__VA_ARGS__)
+#define REQUIRE_FALSE(...) DOCTEST_REQUIRE_FALSE(__VA_ARGS__)
+#define REQUIRE_THROWS(...) DOCTEST_REQUIRE_THROWS(__VA_ARGS__)
+#define REQUIRE_THROWS_AS(expr, ...) DOCTEST_REQUIRE_THROWS_AS(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH(expr, ...) DOCTEST_REQUIRE_THROWS_WITH(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define REQUIRE_NOTHROW(...) DOCTEST_REQUIRE_NOTHROW(__VA_ARGS__)
+
+#define WARN_MESSAGE(cond, ...) DOCTEST_WARN_MESSAGE(cond, __VA_ARGS__)
+#define WARN_FALSE_MESSAGE(cond, ...) DOCTEST_WARN_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define WARN_THROWS_MESSAGE(expr, ...) DOCTEST_WARN_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define WARN_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define WARN_NOTHROW_MESSAGE(expr, ...) DOCTEST_WARN_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_MESSAGE(cond, ...) DOCTEST_CHECK_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_FALSE_MESSAGE(cond, ...) DOCTEST_CHECK_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_THROWS_MESSAGE(expr, ...) DOCTEST_CHECK_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define CHECK_NOTHROW_MESSAGE(expr, ...) DOCTEST_CHECK_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_MESSAGE(cond, ...) DOCTEST_REQUIRE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_FALSE_MESSAGE(cond, ...) DOCTEST_REQUIRE_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_THROWS_MESSAGE(expr, ...) DOCTEST_REQUIRE_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define REQUIRE_NOTHROW_MESSAGE(expr, ...) DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+
+#define SCENARIO(name) DOCTEST_SCENARIO(name)
+#define SCENARIO_CLASS(name) DOCTEST_SCENARIO_CLASS(name)
+#define SCENARIO_TEMPLATE(name, T, ...) DOCTEST_SCENARIO_TEMPLATE(name, T, __VA_ARGS__)
+#define SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id)
+#define GIVEN(name) DOCTEST_GIVEN(name)
+#define WHEN(name) DOCTEST_WHEN(name)
+#define AND_WHEN(name) DOCTEST_AND_WHEN(name)
+#define THEN(name) DOCTEST_THEN(name)
+#define AND_THEN(name) DOCTEST_AND_THEN(name)
+
+#define WARN_EQ(...) DOCTEST_WARN_EQ(__VA_ARGS__)
+#define CHECK_EQ(...) DOCTEST_CHECK_EQ(__VA_ARGS__)
+#define REQUIRE_EQ(...) DOCTEST_REQUIRE_EQ(__VA_ARGS__)
+#define WARN_NE(...) DOCTEST_WARN_NE(__VA_ARGS__)
+#define CHECK_NE(...) DOCTEST_CHECK_NE(__VA_ARGS__)
+#define REQUIRE_NE(...) DOCTEST_REQUIRE_NE(__VA_ARGS__)
+#define WARN_GT(...) DOCTEST_WARN_GT(__VA_ARGS__)
+#define CHECK_GT(...) DOCTEST_CHECK_GT(__VA_ARGS__)
+#define REQUIRE_GT(...) DOCTEST_REQUIRE_GT(__VA_ARGS__)
+#define WARN_LT(...) DOCTEST_WARN_LT(__VA_ARGS__)
+#define CHECK_LT(...) DOCTEST_CHECK_LT(__VA_ARGS__)
+#define REQUIRE_LT(...) DOCTEST_REQUIRE_LT(__VA_ARGS__)
+#define WARN_GE(...) DOCTEST_WARN_GE(__VA_ARGS__)
+#define CHECK_GE(...) DOCTEST_CHECK_GE(__VA_ARGS__)
+#define REQUIRE_GE(...) DOCTEST_REQUIRE_GE(__VA_ARGS__)
+#define WARN_LE(...) DOCTEST_WARN_LE(__VA_ARGS__)
+#define CHECK_LE(...) DOCTEST_CHECK_LE(__VA_ARGS__)
+#define REQUIRE_LE(...) DOCTEST_REQUIRE_LE(__VA_ARGS__)
+#define WARN_UNARY(...) DOCTEST_WARN_UNARY(__VA_ARGS__)
+#define CHECK_UNARY(...) DOCTEST_CHECK_UNARY(__VA_ARGS__)
+#define REQUIRE_UNARY(...) DOCTEST_REQUIRE_UNARY(__VA_ARGS__)
+#define WARN_UNARY_FALSE(...) DOCTEST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define CHECK_UNARY_FALSE(...) DOCTEST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define REQUIRE_UNARY_FALSE(...) DOCTEST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+// KEPT FOR BACKWARDS COMPATIBILITY
+#define FAST_WARN_EQ(...) DOCTEST_FAST_WARN_EQ(__VA_ARGS__)
+#define FAST_CHECK_EQ(...) DOCTEST_FAST_CHECK_EQ(__VA_ARGS__)
+#define FAST_REQUIRE_EQ(...) DOCTEST_FAST_REQUIRE_EQ(__VA_ARGS__)
+#define FAST_WARN_NE(...) DOCTEST_FAST_WARN_NE(__VA_ARGS__)
+#define FAST_CHECK_NE(...) DOCTEST_FAST_CHECK_NE(__VA_ARGS__)
+#define FAST_REQUIRE_NE(...) DOCTEST_FAST_REQUIRE_NE(__VA_ARGS__)
+#define FAST_WARN_GT(...) DOCTEST_FAST_WARN_GT(__VA_ARGS__)
+#define FAST_CHECK_GT(...) DOCTEST_FAST_CHECK_GT(__VA_ARGS__)
+#define FAST_REQUIRE_GT(...) DOCTEST_FAST_REQUIRE_GT(__VA_ARGS__)
+#define FAST_WARN_LT(...) DOCTEST_FAST_WARN_LT(__VA_ARGS__)
+#define FAST_CHECK_LT(...) DOCTEST_FAST_CHECK_LT(__VA_ARGS__)
+#define FAST_REQUIRE_LT(...) DOCTEST_FAST_REQUIRE_LT(__VA_ARGS__)
+#define FAST_WARN_GE(...) DOCTEST_FAST_WARN_GE(__VA_ARGS__)
+#define FAST_CHECK_GE(...) DOCTEST_FAST_CHECK_GE(__VA_ARGS__)
+#define FAST_REQUIRE_GE(...) DOCTEST_FAST_REQUIRE_GE(__VA_ARGS__)
+#define FAST_WARN_LE(...) DOCTEST_FAST_WARN_LE(__VA_ARGS__)
+#define FAST_CHECK_LE(...) DOCTEST_FAST_CHECK_LE(__VA_ARGS__)
+#define FAST_REQUIRE_LE(...) DOCTEST_FAST_REQUIRE_LE(__VA_ARGS__)
+
+#define FAST_WARN_UNARY(...) DOCTEST_FAST_WARN_UNARY(__VA_ARGS__)
+#define FAST_CHECK_UNARY(...) DOCTEST_FAST_CHECK_UNARY(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY(...) DOCTEST_FAST_REQUIRE_UNARY(__VA_ARGS__)
+#define FAST_WARN_UNARY_FALSE(...) DOCTEST_FAST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define FAST_CHECK_UNARY_FALSE(...) DOCTEST_FAST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY_FALSE(...) DOCTEST_FAST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+#define TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// this is here to clear the 'current test suite' for the current translation unit - at the top
+DOCTEST_TEST_SUITE_END();
+
+// add stringification for primitive/fundamental types
+namespace doctest { namespace detail {
+ DOCTEST_TYPE_TO_STRING_IMPL(bool)
+ DOCTEST_TYPE_TO_STRING_IMPL(float)
+ DOCTEST_TYPE_TO_STRING_IMPL(double)
+ DOCTEST_TYPE_TO_STRING_IMPL(long double)
+ DOCTEST_TYPE_TO_STRING_IMPL(char)
+ DOCTEST_TYPE_TO_STRING_IMPL(signed char)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned char)
+#if !DOCTEST_MSVC || defined(_NATIVE_WCHAR_T_DEFINED)
+ DOCTEST_TYPE_TO_STRING_IMPL(wchar_t)
+#endif // not MSVC or wchar_t support enabled
+ DOCTEST_TYPE_TO_STRING_IMPL(short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long long int)
+}} // namespace doctest::detail
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_INCLUDED
diff --git a/contrib/elastic/kibana.json b/contrib/elastic/kibana.json
new file mode 100644
index 0000000..17b68b6
--- /dev/null
+++ b/contrib/elastic/kibana.json
@@ -0,0 +1,138 @@
+[
+ {
+ "_type": "index-pattern",
+ "_id": "eb48a1c0-23a2-11e8-b222-e710267d9b66",
+ "_score": 1,
+ "_source": {
+ "type": "index-pattern",
+ "index-pattern": {
+ "title": "rspamd-*",
+ "timeFieldName": "@timestamp",
+ "fields": "[{\"name\":\"rspamd_meta.action\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.asn.asn\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.asn.country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.asn.ipnet\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.city_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.continent_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.country_iso_code\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.location.lat\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.location.lon\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.region_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.is_local\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.message_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.qid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.rcpt\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.group\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.options\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.user\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.webmail\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
+ "sourceFilters": "[{\"value\":\"rspamd_meta*\"}]"
+ }
+ }
+ },
+ {
+ "_id": "6c6a2ed0-8660-11e7-85ae-fbc80f1b7844",
+ "_type": "dashboard",
+ "_source": {
+ "type": "dashboard",
+ "dashboard": {
+ "title": "Rspamd Dashboard",
+ "hits": 0,
+ "description": "",
+ "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"6413f870-80f6-11e7-91e6-0986b0b459e7\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"927debf0-8649-11e7-967f-798bfd7ac13a\",\"col\":7,\"row\":1},{\"size_x\":12,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"efa3f7a0-80f6-11e7-91e6-0986b0b459e7\",\"col\":1,\"row\":7},{\"size_x\":12,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"1f7d9210-80f7-11e7-91e6-0986b0b459e7\",\"col\":1,\"row\":10},{\"size_x\":6,\"size_y\":3,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"2be7b6f0-8649-11e7-967f-798bfd7ac13a\",\"col\":7,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":6,\"type\":\"visualization\",\"id\":\"680b6480-826e-11e7-8a20-b7bc68c2e9e7\",\"col\":7,\"row\":13},{\"size_x\":6,\"size_y\":3,\"panelIndex\":7,\"type\":\"visualization\",\"id\":\"158dfc80-864d-11e7-bce7-4532b9d239a0\",\"col\":1,\"row\":4}]",
+ "optionsJSON": "{\"darkTheme\":false}",
+ "uiStateJSON": "{\"P-3\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-4\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-1\":{\"mapZoom\":2,\"mapCenter\":[40.58058466412761,1.7578125]},\"P-6\":{\"vis\":{\"defaultColors\":{\"0 - 0.25\":\"rgb(247,252,245)\",\"0.25 - 0.5\":\"rgb(199,233,192)\",\"0.5 - 0.75\":\"rgb(116,196,118)\",\"0.75 - 1\":\"rgb(35,139,69)\"}}}}",
+ "version": 1,
+ "timeRestore": false,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}],\"highlightAll\":true,\"version\":true}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "927debf0-8649-11e7-967f-798bfd7ac13a",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Actions",
+ "visState": "{\"title\":\"Rspamd Actions\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"rspamd_meta.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"index\":\"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "6413f870-80f6-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Geo Map",
+ "visState": "{\n \"title\": \"Rspamd Geo Map\",\n \"type\": \"tile_map\",\n \"params\": {\n \"mapType\": \"Scaled Circle Markers\",\n \"isDesaturated\": true,\n \"addTooltip\": true,\n \"heatMaxZoom\": 0,\n \"heatMinOpacity\": 0.1,\n \"heatRadius\": 25,\n \"heatBlur\": 15,\n \"legendPosition\": \"bottomright\",\n \"mapZoom\": 2,\n \"mapCenter\": [\n 0,\n 0\n ],\n \"wms\": {\n \"enabled\": false,\n \"url\": \"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\n \"options\": {\n \"version\": \"1.3.0\",\n \"layers\": \"0\",\n \"format\": \"image/png\",\n \"transparent\": true,\n \"attribution\": \"Maps provided by USGS\",\n \"styles\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"geohash_grid\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"rspamd_meta.geoip.location\",\n \"autoPrecision\": true,\n \"useGeocentroid\": true,\n \"precision\": 2\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "92a92c00-80f6-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Spam Map",
+ "visState": "{\n \"title\": \"Rspamd Spam Map\",\n \"type\": \"tile_map\",\n \"params\": {\n \"mapType\": \"Scaled Circle Markers\",\n \"isDesaturated\": true,\n \"addTooltip\": true,\n \"heatMaxZoom\": 0,\n \"heatMinOpacity\": 0.1,\n \"heatRadius\": 25,\n \"heatBlur\": 15,\n \"legendPosition\": \"bottomright\",\n \"mapZoom\": 2,\n \"mapCenter\": [\n 0,\n 0\n ],\n \"wms\": {\n \"enabled\": false,\n \"url\": \"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\n \"options\": {\n \"version\": \"1.3.0\",\n \"layers\": \"0\",\n \"format\": \"image/png\",\n \"transparent\": true,\n \"attribution\": \"Maps provided by USGS\",\n \"styles\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"geohash_grid\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"rspamd_meta.geoip.location\",\n \"autoPrecision\": true,\n \"useGeocentroid\": true,\n \"precision\": 2\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": [\n {\n \"meta\": {\n \"index\": \"rspamd_beat-*\",\n \"negate\": true,\n \"disabled\": false,\n \"alias\": null,\n \"type\": \"phrase\",\n \"key\": \"rspamd_meta.action\",\n \"value\": \"no action\"\n },\n \"query\": {\n \"match\": {\n \"rspamd_meta.action\": {\n \"query\": \"no action\",\n \"type\": \"phrase\"\n }\n }\n },\n \"$state\": {\n \"store\": \"appState\"\n }\n }\n ]\n}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "2be7b6f0-8649-11e7-967f-798bfd7ac13a",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Symbols Cloud",
+ "visState": "{\"title\":\"Rspamd Symbols Cloud\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"linear\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"rspamd_meta.symbols.name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"index\":\"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "1f7d9210-80f7-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Top recipients",
+ "visState": "{\n \"title\": \"Rspamd Top recipients\",\n \"type\": \"metric\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"gauge\",\n \"gauge\": {\n \"verticalSplit\": false,\n \"autoExtend\": false,\n \"percentageMode\": false,\n \"gaugeType\": \"Metric\",\n \"gaugeStyle\": \"Full\",\n \"backStyle\": \"Full\",\n \"orientation\": \"vertical\",\n \"colorSchema\": \"Green to Red\",\n \"gaugeColorMode\": \"None\",\n \"useRange\": false,\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 100\n }\n ],\n \"invertColors\": false,\n \"labels\": {\n \"show\": true,\n \"color\": \"black\"\n },\n \"scale\": {\n \"show\": false,\n \"labels\": false,\n \"color\": \"#333\",\n \"width\": 2\n },\n \"type\": \"simple\",\n \"style\": {\n \"fontSize\": 60,\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"rspamd_meta.rcpt\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{\n \"vis\": {\n \"defaultColors\": {\n \"0 - 100\": \"rgb(0,104,55)\"\n }\n }\n}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "efa3f7a0-80f6-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Top Senders",
+ "visState": "{\n \"title\": \"Rspamd Top Senders\",\n \"type\": \"metric\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"gauge\",\n \"gauge\": {\n \"verticalSplit\": false,\n \"autoExtend\": false,\n \"percentageMode\": false,\n \"gaugeType\": \"Metric\",\n \"gaugeStyle\": \"Full\",\n \"backStyle\": \"Full\",\n \"orientation\": \"vertical\",\n \"colorSchema\": \"Green to Red\",\n \"gaugeColorMode\": \"None\",\n \"useRange\": false,\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 100\n }\n ],\n \"invertColors\": false,\n \"labels\": {\n \"show\": true,\n \"color\": \"black\"\n },\n \"scale\": {\n \"show\": false,\n \"labels\": false,\n \"color\": \"#333\",\n \"width\": 2\n },\n \"type\": \"simple\",\n \"style\": {\n \"fontSize\": 60,\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"rspamd_meta.user\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{\n \"vis\": {\n \"defaultColors\": {\n \"0 - 100\": \"rgb(0,104,55)\"\n }\n }\n}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": [\n {\n \"meta\": {\n \"index\": \"rspamd_beat-*\",\n \"negate\": true,\n \"disabled\": false,\n \"alias\": null,\n \"type\": \"phrase\",\n \"key\": \"rspamd_meta.user\",\n \"value\": \"unknown\"\n },\n \"query\": {\n \"match\": {\n \"rspamd_meta.user\": {\n \"query\": \"unknown\",\n \"type\": \"phrase\"\n }\n }\n },\n \"$state\": {\n \"store\": \"appState\"\n }\n }\n ]\n}"
+ }
+ }
+ }
+ }
+]
diff --git a/contrib/elastic/rspamd_template.json b/contrib/elastic/rspamd_template.json
new file mode 100644
index 0000000..ebd87fa
--- /dev/null
+++ b/contrib/elastic/rspamd_template.json
@@ -0,0 +1,149 @@
+{
+ "mappings": {
+ "_meta": {
+ "version": "5.5.3"
+ },
+ "date_detection": false,
+ "dynamic_templates": [
+ {
+ "strings_as_keyword": {
+ "mapping": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "match_mapping_type": "string"
+ }
+ }
+ ],
+ "properties": {
+ "@timestamp": {
+ "type": "date"
+ },
+ "meta": {
+ "properties": {
+ "cloud": {
+ "properties": {
+ "availability_zone": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "instance_id": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "machine_type": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "project_id": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "provider": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "region": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "rspamd_meta": {
+ "properties": {
+ "action": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "direction": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "asn": {
+ "properties": {
+ "asn": {
+ "type": "long"
+ },
+ "country_code": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "ipnet": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "registrant": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "from": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "is_local": {
+ "type": "boolean"
+ },
+ "webmail": {
+ "type": "boolean"
+ },
+ "sender_ip": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "geoip": {
+ "properties": {
+ "city_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "continent_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "country_iso_code": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "location": {
+ "type": "geo_point"
+ }
+ }
+ },
+ "ip": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "qid": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "hostname": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "score": {
+ "type": "float"
+ },
+ "user": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "tags": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "order": 0,
+ "settings": {
+ "index.mapping.total_fields.limit": 10000,
+ "index.refresh_interval": "5s"
+ },
+ "index_patterns" : ["rspamd-*", "*-rspamd-*"]
+}
diff --git a/contrib/exim/dlfunc-json/README b/contrib/exim/dlfunc-json/README
new file mode 100644
index 0000000..0e52270
--- /dev/null
+++ b/contrib/exim/dlfunc-json/README
@@ -0,0 +1,10 @@
+FEATURES
+1) Support new http-protocol (/checkv2)
+2) Return action, symbols, symbol options, messages, scan time
+
+INSTALL
+
+1) Get cJSON.c, cJSON.h, put to dlfunc src dir (https://github.com/DaveGamble/cJSON)
+2) Compile dlfunc library:
+ cc rspamd.c -fPIC -fpic -shared -I/root/rpmbuild/BUILD/exim-4.89/build-Linux-x86_64/ -o exim-rspamd-http-dlfunc.so
+3) See exim-example.txt for exim configure
diff --git a/contrib/exim/dlfunc-json/exim-example.txt b/contrib/exim/dlfunc-json/exim-example.txt
new file mode 100644
index 0000000..e6da334
--- /dev/null
+++ b/contrib/exim/dlfunc-json/exim-example.txt
@@ -0,0 +1,75 @@
+acl_smtp_data = acl_check_data
+
+.....
+
+acl_check_data:
+
+.....
+
+# RSPAMD: START
+ warn
+ !authenticated = *
+ add_header = X-Spam-Checker-Version: Rspamd
+ add_header = :at_start:Authentication-Results: ip=$sender_host_address:$sender_host_port, host=$sender_host_name, helo=$sender_helo_name, mailfrom=$sender_address
+ warn
+ #spam = nobody:true
+ #set acl_m0_rspamd = $spam_report
+ set acl_m0_rspamd = ${dlfunc{/usr/local/libexec/exim/exim-rspamd-http-dlfunc.so}{rspamd}{/var/run/rspamd/rspamd.sock}{defer_ok}}
+ accept
+ authenticated = *
+ warn
+ condition = ${if eq{$acl_m0_rspamd}{}}
+ logwrite = RSPAMD check failed
+ add_header = X-Spam-Info: Check failed
+ warn
+ condition = ${if match{$acl_m0_rspamd}{\N^rspamd dlfunc:\s*\N}{yes}{no}}
+ logwrite = RSPAMD check defer: ${sg{$acl_m0_rspamd}{\N^rspamd dlfunc:\s*\N}{}}
+ add_header = X-Spam-Info: Check deffered
+
+ warn
+ remove_header = X-Spam-Checker-Version:X-Spam-Status:X-Spam-Info:X-Spam-Result
+ set acl_m1 = No
+ warn
+ condition = ${if !eq{$acl_m0_rspamd}{}}
+ set acl_m1_yesno = ${if match{$acl_m0_rspamd}{\NAction: (.+?)\n\N}{$1}{}}
+ set acl_m2_status = ${if eq{$acl_m1_yesno}{reject}{REJECT}{\
+ ${if eq{$acl_m1_yesno}{add header}{PROBABLY}{\
+ ${if eq{$acl_m1_yesno}{rewrite subject}{PROBABLY}{\
+ ${if eq{$acl_m1_yesno}{soft reject}{SOFT}{\
+ ${if eq{$acl_m1_yesno}{greylist}{GREYLIST}{NO}}\
+ }}\
+ }}\
+ }}\
+ }}
+ set acl_m1_yesno = ${if eq{$acl_m1_yesno}{}{unknown}{\
+ ${if eq{$acl_m1_yesno}{reject}{Yes}{\
+ ${if eq{$acl_m1_yesno}{add header}{Yes}{\
+ ${if eq{$acl_m1_yesno}{rewrite subject}{Yes}{\
+ ${if eq{$acl_m1_yesno}{soft reject}{Probably}{\
+ ${if eq{$acl_m1_yesno}{greylist}{Probably}{No}}\
+ }}\
+ }}\
+ }}\
+ }}\
+ }}
+ #logwrite = RSPAMD: status: $acl_m2_status
+ #logwrite = RSPAMD DEBUG: $acl_m0_rspamd
+ set acl_m0_rspamd = ${sg{$acl_m0_rspamd}{ Action:.+\n}{}}
+ warn
+ condition = ${if !eq{$acl_m0_rspamd}{}}
+ logwrite = RSPAMD: $acl_m2_status, $acl_m0_rspamd
+ add_header = X-Spam-Result: $acl_m0_rspamd
+ add_header = X-Spam-Status: $acl_m1_yesno
+ defer
+ condition = ${if eq{$acl_m2_status}{GREYLIST}}
+ log_message = Rspamd $acl_m2_status
+ message = Try again later. Message greylisted
+ defer
+ condition = ${if eq{$acl_m2_status}{SOFT}}
+ log_message = Rspamd $acl_m2_status
+ message = Try again later. Message previously greylisted
+ deny
+ condition = ${if eq{$acl_m2_status}{REJECT}}
+ log_message = Rspamd $acl_m2_status
+ message = This message detected as SPAM and rejected
+# RSPAMD: END
diff --git a/contrib/exim/dlfunc-json/rspamd.c b/contrib/exim/dlfunc-json/rspamd.c
new file mode 100644
index 0000000..2623c7d
--- /dev/null
+++ b/contrib/exim/dlfunc-json/rspamd.c
@@ -0,0 +1,576 @@
+/*-
+ * Copyright (c) 2013-2015, Alexey Savelyev <info@homeweb.ru>
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
+ * Based on source code from http://www.ols.es/exim/dlext/ by David Saez <david@ols.es>,
+ * source code of exim by Philip Hazel <ph10@cam.ac.uk>
+ * and source code of exiscan by Tom Kistner <tom@duncanthrax.net>
+ * and source code of Victor Ustugov http://mta.org.ua/
+*/
+
+#include "exim.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <errno.h>
+#include "cJSON.h"
+#include "cJSON.c"
+
+#define RSPAMD_TIMEOUT 120
+
+extern uschar *tod_stamp (int);
+//extern BOOL split_spool_directory; /* TRUE to use multiple subdirs */
+//extern uschar *spool_directory; /* Name of spool directory */
+//extern uschar message_subdir[]; /* Subdirectory for messages */
+
+//-------------------------------------------------------------------------
+
+int
+rspamd (uschar **yield, int argc, uschar *argv[]) {
+ char *arg_socket_addr;
+ char *arg_defer_ok;
+ int defer_ok;
+ int rspamd_command;
+ char tcp_addr[15];
+ int tcp_port;
+ FILE *mbox_file = NULL;
+// unsigned long mbox_size;
+ off_t mbox_size;
+ uschar *s, *p;
+ header_line *my_header, *header_new, *header_last, *tmp_headerlist;
+ header_line *last_received = NULL;
+ uschar *address;
+ uschar *helo;
+ uschar *sender_host_name;
+ uschar *authenticated_id;
+ char mbox_path[8192];
+ int max_len, len;
+ int rspamd_sock = 0;
+ struct hostent *he;
+ struct in_addr in;
+ struct sockaddr_un server;
+#ifndef NO_POLL_H
+ int result;
+ struct pollfd pollfd;
+#endif
+ int offset;
+ uschar spamd_buffer[32600];
+ uschar spamd_buffer2[32600];
+ time_t start;
+ size_t read, wrote;
+ int i, j, c;
+
+ arg_socket_addr = argv[0];
+ arg_defer_ok = argv[1];
+
+ if (argc < 2) {
+ defer_ok = 0;
+ } else if (strcmpic (arg_defer_ok, US"1") == 0)
+ || (strcmpic (arg_defer_ok, US
+ "yes") == 0)
+ || (strcmpic (arg_defer_ok, US
+ "true") == 0)
+ || (strcmpic (arg_defer_ok, US
+ "defer_ok") == 0) {
+ defer_ok = 1;
+ } else {
+ defer_ok = 0;
+ }
+
+ if ((arg_socket_addr == NULL) || (arg_socket_addr[0] == 0)) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Socket address expected");
+ *yield = string_sprintf ("rspamd dlfunc: Socket address expected");
+ goto RETURN_DEFER;
+ }
+
+
+ if (split_spool_directory == 0) {
+ snprintf (mbox_path, sizeof (mbox_path), "%s/input/%s-D",
+ spool_directory,
+ message_id);
+ } else {
+ snprintf (mbox_path, sizeof (mbox_path), "%s/input/%s/%s-D",
+ spool_directory, message_subdir, message_id);
+ }
+
+ mbox_file = fopen (mbox_path, "rb");
+
+ if (!mbox_file) {
+ *yield = string_sprintf ("rspamd dlfunc: Unable to spool message '%s'",
+ mbox_path);
+ return (defer_ok ? OK : ERROR);
+ }
+
+ (void) fseek (mbox_file, 0, SEEK_END);
+ mbox_size = ftell (mbox_file);
+//debug_printf(" Total spool file size: %d\n", mbox_size);
+ mbox_size -= SPOOL_DATA_START_OFFSET;
+//debug_printf(" Spool file size: %d\n", mbox_size);
+//debug_printf(" fseek %d, %d\n", SPOOL_DATA_START_OFFSET, SEEK_SET);
+ (void) fseek (mbox_file, SPOOL_DATA_START_OFFSET, SEEK_SET);
+
+ start = time (NULL);
+ /* socket does not start with '/' -> network socket */
+ if (arg_socket_addr[0] != '/') {
+ if (sscanf (CS arg_socket_addr, "%s %u", tcp_addr, &tcp_port) != 2 ) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Invalid rspamd address: '%s'",
+ arg_socket_addr);
+ *yield = string_sprintf (
+ "rspamd dlfunc: Invalid rspamd address: '%s'",
+ arg_socket_addr);
+ goto RETURN_DEFER;
+ }
+
+ /* Lookup the host */
+ if ((he = gethostbyname (CS tcp_addr)) == 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: failed to lookup host '%s'", tcp_addr);
+ *yield = string_sprintf (
+ "rspamd dlfunc: failed to lookup host '%s'", tcp_addr);
+ goto RETURN_DEFER;
+ }
+
+ in = *(struct in_addr *) he->h_addr_list[0];
+
+/* contact a rspamd */
+
+ if ((rspamd_sock = ip_socket (SOCK_STREAM, AF_INET)) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: TCP socket creation failed: %s",
+ strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: TCP socket creation failed: %s",
+ strerror (errno));
+ goto RETURN_DEFER;
+ };
+
+ if (ip_connect (rspamd_sock, AF_INET, (uschar *) inet_ntoa (in),
+ tcp_port, 5, FALSE) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: connection to %s, port %u failed: %s",
+ tcp_addr, tcp_port, strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: connection to %s, port %u failed: %s",
+ tcp_addr, tcp_port, strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+//debug_printf(" Use TCP socket %s:%d\n", tcp_addr, tcp_port);
+ } else {
+ if ((rspamd_sock = socket (AF_UNIX, SOCK_STREAM, 0)) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Unable to acquire socket (%s)",
+ strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: Unable to acquire socket (%s)",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ server.sun_family = AF_UNIX;
+ Ustrcpy (server.sun_path, arg_socket_addr);
+
+ if (connect (rspamd_sock, (struct sockaddr *) &server,
+ sizeof (struct sockaddr_un)) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Unable to connect to UNIX socket %s (%s)",
+ socket, strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: Unable to connect to UNIX socket %s (%s)",
+ socket, strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+//debug_printf(" Use UNIX Domain socket %s\n", arg_socket_addr);
+ }
+
+// now we are connected to rspamd on rspamd_sock
+
+ memset (spamd_buffer2, 0, sizeof (spamd_buffer2));
+ offset = 0;
+
+ // headers list
+ tmp_headerlist = NULL;
+ header_last = NULL;
+ for (my_header = header_list; my_header; my_header = my_header->next) {
+ if ((my_header->type != '*') && (my_header->type != htype_old)) {
+ header_new = store_get (sizeof (header_line));
+ header_new->text = my_header->text;
+
+ header_new->slen = my_header->slen;
+ header_new->type = my_header->type;
+ header_new->next = NULL;
+ //debug_printf(" copy header item: '%s'\n", my_header->text);
+
+ max_len = sizeof (spamd_buffer2) - offset - 1;
+ len = my_header->slen;
+ if (len > max_len) len = max_len;
+ Ustrncpy (spamd_buffer2 + offset, my_header->text, len);
+ offset += len;
+
+ if (header_last != NULL) header_last->next = header_new;
+ header_last = header_new;
+ }
+ }
+
+ s = string_sprintf ("\n");
+ max_len = sizeof (spamd_buffer2) - offset - 1;
+ len = Ustrlen (s);
+ if (len > max_len) len = max_len;
+ Ustrncpy (spamd_buffer2 + offset, s, len);
+ offset += len;
+
+//debug_printf(" Headers size: %d\n", offset);
+ mbox_size += offset;
+//debug_printf(" Total message size: %d\n", mbox_size);
+
+// copy request to buffer
+ memset (spamd_buffer, 0, sizeof (spamd_buffer));
+ string_format (spamd_buffer,
+ sizeof (spamd_buffer),
+ "POST /checkv2 HTTP/1.0\r\nContent-length: "OFF_T_FMT
+ "\r\nQueue-Id: %s\r\nFrom: %s\r\n",
+ mbox_size, message_id, sender_address);
+
+ for (i = 0; i < recipients_count; i++) {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "Rcpt: %s\r\n",
+ recipients_list[i].address);
+ }
+
+ if ((helo = expand_string (US"$sender_helo_name")) != NULL && *helo != '\0') {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "Helo: %s\r\n",
+ helo);
+ }
+
+ if ((sender_host_name = expand_string (US"$sender_host_name")) != NULL &&
+ *sender_host_name != '\0') {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer),
+ "Hostname: %s\r\n",
+ sender_host_name);
+ }
+ //else
+ //string_format(spamd_buffer+Ustrlen(spamd_buffer), sizeof(spamd_buffer)-Ustrlen(spamd_buffer), "Hostname: unknown\r\n");
+
+ if (sender_host_address != NULL) {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "IP: %s\r\n",
+ sender_host_address);
+ }
+
+ //authenticated_id
+ if ((authenticated_id = expand_string (US"$authenticated_id")) != NULL &&
+ *authenticated_id != '\0') {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "User: %s\r\n",
+ authenticated_id);
+ }
+
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "\r\n");
+
+ if (send (rspamd_sock, spamd_buffer, Ustrlen (spamd_buffer), 0) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: rspamd send failed: %s", strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ /*
+ * now send the data buffer and spool file
+ */
+ Ustrcpy (big_buffer, "sending data block");
+
+ wrote = send (rspamd_sock, spamd_buffer2, strlen (spamd_buffer2), 0);
+ if (wrote == -1) {
+ goto WRITE_FAILED;
+ }
+
+ /*
+ * Note: poll() is not supported in OSX 10.2.
+ */
+
+#ifndef NO_POLL_H
+ pollfd.fd = rspamd_sock;
+ pollfd.events = POLLOUT;
+#endif
+// (void)fcntl(rspamd_sock, F_SETFL, O_NONBLOCK);
+ do {
+ read = fread (spamd_buffer, 1, sizeof (spamd_buffer) - 1, mbox_file);
+
+ if (read < sizeof (spamd_buffer)) {
+ spamd_buffer[read] = 0;
+ }
+//debug_printf(" Read from spool file: %s", spamd_buffer);
+ if (read > 0) {
+ offset = 0;
+ again:
+
+#ifndef NO_POLL_H
+ result = poll (&pollfd, 1, 1000);
+ if (result == -1 && errno == EINTR) {
+ continue;
+ }
+ else if (result < 1) {
+ if (result == -1)
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: %s on rspamd socket",
+ strerror (errno));
+ else {
+ if (time (NULL) - start < RSPAMD_TIMEOUT)
+ goto again;
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: timed out writing rspamd socket");
+ *yield = string_sprintf (
+ "rspamd dlfunc: timed out writing rspamd socket");
+ }
+ goto RETURN_DEFER;
+ }
+#endif
+ wrote = send (rspamd_sock, spamd_buffer + offset, read - offset, 0);
+
+ if (wrote == -1) {
+ goto WRITE_FAILED;
+ }
+
+ if (offset + wrote != read) {
+ offset += wrote;
+ goto again;
+ }
+ }
+ } while (!feof (mbox_file) && !ferror (mbox_file));
+
+ if (ferror (mbox_file)) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: error reading spool file: %s",
+ strerror (errno));
+ *yield = string_sprintf ("rspamd dlfunc: error reading spool file: %s",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ /*
+ read rspamd response using what's left of the timeout.
+ */
+
+ memset (spamd_buffer, 0, sizeof (spamd_buffer));
+ offset = 0;
+ while ((i = ip_recv (rspamd_sock,
+ spamd_buffer + offset,
+ sizeof (spamd_buffer) - offset - 1,
+ RSPAMD_TIMEOUT - time (NULL) + start)) > 0
+ ) {
+ //debug_printf(" read %d bytes from socket\n", i);
+ offset += i;
+ }
+//debug_printf(" total read %d bytes from socket\n", offset);
+
+/* error handling */
+ if ((i <= 0) && (errno != 0)) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: error reading from rspamd socket: %s",
+ strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: error reading from rspamd socket: %s",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+//debug_printf("read from socket: %s\n", spamd_buffer);
+
+ if (rspamd_sock > 0) {
+ (void) close (rspamd_sock);
+ rspamd_sock = 0;
+ }
+ if (mbox_file != NULL) {
+ (void) fclose (mbox_file);
+ mbox_file = NULL;
+ }
+
+ //Parse http response code
+ if (strstr (spamd_buffer, "HTTP/1.1 200 OK") == NULL &&
+ strstr (spamd_buffer, "HTTP/1.0 200 OK") == NULL) {
+ *yield = string_sprintf ("rspamd dlfunc: HTTP return code != 200: %s",
+ spamd_buffer);
+ goto RETURN_DEFER;
+ }
+
+ //Parse http response
+ const char *crlf_pos = strstr (spamd_buffer, "\r\n\r\n");
+ if (crlf_pos == NULL) {
+ *yield = string_sprintf ("rspamd dlfunc: HTTP response error: %s",
+ spamd_buffer);
+ goto RETURN_DEFER;
+ }
+
+ char *json_answer = string_sprintf ("%s", crlf_pos + 4);
+
+ //Parse json
+ cJSON *json = NULL;
+ json = cJSON_Parse (json_answer);
+
+ if (!json) {
+ *yield = string_sprintf ("rspamd dlfunc: Json parse error, json: %s",
+ spamd_buffer);
+ goto RETURN_DEFER;
+ }
+
+ //Score
+ cJSON *score = cJSON_GetObjectItem (json, "score");
+ if (!cJSON_IsNumber (score)) {
+ *yield = string_sprintf (
+ "rspamd dlfunc: Json parse error, no found 'score'");
+ goto RETURN_DEFER;
+ }
+ //required_score
+ cJSON *required_score = cJSON_GetObjectItem (json, "required_score");
+ if (!cJSON_IsNumber (required_score)) {
+ *yield = string_sprintf (
+ "rspamd dlfunc: Json parse error, no found 'required_score'");
+ goto RETURN_DEFER;
+ }
+ //Action
+ cJSON *action = cJSON_GetObjectItem (json, "action");
+ if (!cJSON_IsString (action)) {
+ *yield = string_sprintf (
+ "rspamd dlfunc: Json parse error, no found 'action'");
+ goto RETURN_DEFER;
+ }
+ *yield = string_sprintf ("[%.2f / %.2f]", score->valuedouble,
+ required_score->valuedouble);
+
+ //Parse scan time
+ cJSON *time_real = cJSON_GetObjectItem (json, "time_real");
+ cJSON *time_virtual = cJSON_GetObjectItem (json, "time_virtual");
+ if (cJSON_IsNumber (time_real) && cJSON_IsNumber (time_virtual))
+ *yield = string_sprintf ("%s [time: %.6f, %.6f]", *yield,
+ time_real->valuedouble, time_virtual->valuedouble);
+
+ *yield = string_sprintf ("%s\n Action: %s\n", *yield, action->valuestring);
+
+ cJSON *symbol = NULL;
+ cJSON *symbol_name = NULL;
+ cJSON *symbol_score = NULL;
+ cJSON *symbol_options = NULL;
+ cJSON *option = NULL;
+
+ //parse symbols
+ cJSON *symbols = cJSON_GetObjectItem (json, "symbols");
+ for (i = 0; i < cJSON_GetArraySize (symbols); i++) {
+ symbol = cJSON_GetArrayItem (symbols, i);
+ symbol_name = cJSON_GetObjectItem (symbol, "name");
+ symbol_score = cJSON_GetObjectItem (symbol, "score");
+ symbol_options = cJSON_GetObjectItem (symbol, "options");
+
+ if (cJSON_IsString (symbol_name)) {
+ *yield = string_sprintf ("%s %s", *yield, symbol_name->valuestring);
+ }
+ if (cJSON_IsNumber (symbol_score)) {
+ *yield = string_sprintf ("%s(%.2f)", *yield,
+ symbol_score->valuedouble);
+ }
+
+ //parse options
+ c = cJSON_GetArraySize (symbol_options);
+ if (c > 0) {
+ *yield = string_sprintf ("%s[", *yield);
+ }
+
+ for (j = 0; j < c; j++) {
+ option = cJSON_GetArrayItem (symbol_options, j);
+
+ if (cJSON_IsString (option)) {
+ *yield = string_sprintf ("%s%s", *yield, option->valuestring);
+ if (j < c - 1) *yield = string_sprintf ("%s, ", *yield);
+ }
+ }
+ if (c > 0) {
+ *yield = string_sprintf ("%s]", *yield);
+ }
+
+ *yield = string_sprintf ("%s\n", *yield);
+ }
+
+ //Parse messages
+ cJSON *mess = NULL;
+ cJSON *messages = cJSON_GetObjectItem (json, "messages");
+ c = cJSON_GetArraySize (messages);
+
+ for (i = 0; i < c; i++) {
+ mess = cJSON_GetArrayItem (messages, i);
+ if (cJSON_IsString (mess)) {
+ *yield = string_sprintf ("%s %s", *yield, mess->valuestring);
+ }
+
+ if (i < c - 1) {
+ *yield = string_sprintf ("%s\n", *yield);
+ }
+ }
+
+ return OK;
+
+/* Come here if any call to read_response, other than a response after the data
+phase, failed. Analyse the error, and if isn't too bad, send a QUIT
+command. Wait for the response with a short timeout, so we don't wind up this
+process before the far end has had time to read the QUIT. */
+
+ WRITE_FAILED:
+ {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: %s on rspamd socket", strerror (errno));
+ *yield = string_sprintf ("rspamd dlfunc: %s on rspamd socket",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ RESPONSE_FAILED:
+ {
+ int code;
+ int save_errno;
+ int more_errno;
+ uschar message_buffer[256];
+ uschar *message;
+
+ save_errno = errno;
+
+ message = &message_buffer[0];
+
+ log_write (0, LOG_MAIN | LOG_PANIC, "rspamd dlfunc: %s", message);
+ *yield = string_sprintf ("rspamd dlfunc: %s", message);
+
+ goto RETURN_DEFER;
+ }
+
+ RETURN_DEFER:
+ {
+ if (rspamd_sock > 0) {
+ (void) close (rspamd_sock);
+ rspamd_sock = 0;
+ }
+ if (mbox_file != NULL) {
+ (void) fclose (mbox_file);
+ mbox_file = NULL;
+ }
+
+ return (defer_ok ? OK : ERROR);
+ }
+
+ return OK;
+}
diff --git a/contrib/exim/local_scan.c b/contrib/exim/local_scan.c
new file mode 100644
index 0000000..42cd5ad
--- /dev/null
+++ b/contrib/exim/local_scan.c
@@ -0,0 +1,700 @@
+/*
+ This program is RSPAMD agent for use with
+ exim (http://www.exim.org) MTA by its local_scan feature.
+
+ To enable exim local scan please copy this file to exim source tree
+ Local/local_scan.c, edit Local/Makefile to add
+
+ LOCAL_SCAN_SOURCE=Local/local_scan.c
+ LOCAL_SCAN_HAS_OPTIONS=yes
+
+ and compile exim.
+
+ Comment out RSPAM_UNIXSOCKET definition below if you have remote RSPAMD
+ daemon
+
+ AND
+
+ use Exim parameters daemonIP and daemonPort to configure remote
+ RSPAMD daemon.
+
+ For exim compilation with local scan feature details please visit
+ http://www.exim.org/exim-html-4.50/doc/html/spec_toc.html#TOC333
+
+ For RSPAMD details please visit
+ http://rspamd.sourceforge.net
+*/
+
+/* Comment out the row below to use socket type AF_INET
+ to connect RSPAMD daemon */
+//#define RSPAM_UNIXSOCKET
+
+#include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+#include <stdlib.h>
+
+#include "local_scan.h"
+
+extern uschar *sender_helo_name;
+extern int message_size;
+
+#define READ_FAIL(x) ((x) < 0)
+#define RSPAMD_FAILURE_HDR "X-Spam-Flag"
+#define RSPAMD_SCORE_HDR "X-Spam-Status"
+#define REJECT_ON_ERROR 0
+
+static int _OK = 0;
+static int ERR_WRITE = 53;
+static int ERR_READ = 54;
+static int MAX_FAILS_C = 256;
+static int MAX_PATH = 256;
+static int MAX_SIZE_FILE = 64*1024;
+
+static uschar *daemonIP = US"127.0.0.1";
+static int daemonPort = 11333;
+static uschar *temp_dir = US"/var/tmp";
+static uschar *socket_name = US"/var/run/rspamd.sock";
+static int strange = 0;
+
+optionlist local_scan_options[] =
+{
+ {"rspam_ip", opt_stringptr, &daemonIP},
+ {"rspam_port", opt_int, &daemonPort},
+ {"rspam_tmp", opt_stringptr, &temp_dir},
+ {"rspam_sock", opt_stringptr, &socket_name},
+
+};
+
+int local_scan_options_count = sizeof (local_scan_options) / sizeof (optionlist);
+
+typedef int socket_t;
+static socket_t sock = -1;
+
+int iFdInp;
+struct sockaddr_un ssun;
+struct sockaddr_in ssin;
+
+static int mOpenTmp (char *pszDir, char *pszPrefix, char *pszPath)
+{
+ int iLen;
+ int iFd = -1;
+ char *pszSep = "";
+
+ iLen = (int)strlen(pszDir);
+ if (iLen > MAX_PATH)
+ return -1;
+
+ if (pszDir[iLen - 1] != '/')
+ pszSep = "/";
+
+ sprintf (pszPath, "%s%s%sXXXXXX", pszDir, pszSep, pszPrefix);
+ iFd = mkstemp (pszPath);
+
+ if (iFd < 0)
+ log_write (0, LOG_MAIN, "rspam-exim: Temp file create error %d", errno);
+
+ return iFd;
+}
+
+static int ReadFd (int iFdMsg, int fd)
+{
+ char psMsg [MAX_SIZE_FILE]; /* max size SO can swallow */
+ int iLen, result = _OK;
+
+ if ((iLen = read (fd, psMsg, sizeof (psMsg))) > 0)
+ {
+ if (write (iFdMsg, psMsg, (unsigned int) iLen) != iLen)
+ result = ERR_WRITE;
+ }
+ else
+ result = ERR_READ;
+
+ close (iFdMsg);
+
+ return result;
+}
+
+
+void CleanupInp (char *sName)
+{
+ if (sName) unlink (sName);
+
+ close (iFdInp);
+ return;
+}
+
+
+int FakeSMTPCommand (socket_t sock,
+ char *command,
+ char *value,
+ char *sName,
+ int Cleanup,
+ int wa)
+{
+ char sCommand[1024];
+ char answ [3];
+ int Len;
+
+ sprintf (sCommand, "%s %s\r\n", command, value);
+
+ if (send (sock, sCommand, strlen (sCommand), 0) != (int) strlen (sCommand))
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: socket sending '%s' error %d", sCommand, errno);
+ if (Cleanup)
+ CleanupInp (sName);
+ return ERR_WRITE;
+ }
+
+ if(wa) {
+ memset (answ, '\0', sizeof (answ));
+ Len = read (sock, answ, sizeof (answ));
+ if (READ_FAIL (Len))
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: read() error %d, len=%d", errno, Len);
+ if (Cleanup)
+ CleanupInp (sName);
+ return ERR_WRITE;
+ }
+
+ if (strncmp (answ, "OK", 2) != 0)
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: server did not confirm, answ=%s", answ);
+ if (Cleanup)
+ CleanupInp (sName);
+ return ERR_WRITE; /* Cannot read message error code */
+ }
+ }
+
+ return OK;
+}
+
+
+static int written (socket_t fd, const char *vptr, int n)
+{
+ size_t nleft;
+ int nwritten;
+ const char *ptr;
+
+ ptr = vptr;
+ nleft = n;
+ while (nleft > 0)
+ {
+ if ((nwritten = send (fd, ptr, nleft, 0)) <= 0)
+ {
+ if (errno == EINTR)
+ nwritten = 0;
+ else
+ return (-1);
+ }
+
+ nleft -= nwritten;
+ ptr += nwritten;
+ }
+
+ return (n);
+}
+
+
+static int SendEnvelope (char *sFile)
+{
+ int i;
+ char str [256], *rh;
+ void *psBuf;
+ int fd, bytesRead;
+
+ if(message_size > MAX_SIZE_FILE) {
+ log_write (0, LOG_MAIN, "rspam-exim: file %s is great %d bytes", sFile, MAX_SIZE_FILE);
+ return ERR_WRITE;
+ }
+
+ /* send greeting */
+// if(FakeSMTPCommand(sock, "PROCESS", "RSPAMC/1.0", sFile, 1, 0) != _OK)
+// return ERR_WRITE;
+ if(FakeSMTPCommand(sock, "SYMBOLS", "RSPAMC/1.1", sFile, 1, 0) != _OK)
+// if(FakeSMTPCommand(sock, "CHECK", "RSPAMC/1.0", sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+
+
+ /* sender IP */
+ if (FakeSMTPCommand (sock, "IP:", sender_host_address, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* mail from */
+ if (FakeSMTPCommand (sock, "From:",
+ strlen (sender_address) == 0 ? "MAILER-DAEMON" : (char*) sender_address, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* send helo */
+ if (FakeSMTPCommand (sock, "Helo:", sender_helo_name, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* send helo */
+ sprintf(str, "%d", message_size);
+ if (FakeSMTPCommand (sock, "Content-Length:", str, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* number of recipient */
+ sprintf(str, "%d", recipients_count);
+ if (FakeSMTPCommand (sock, "Recipient-Number:", str, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* envelope rcpto */
+ for (i = 0; i < recipients_count; i ++)
+ {
+ if (FakeSMTPCommand (sock, "Rcpt:", recipients_list[i].address, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+ }
+
+ psBuf = store_get (MAX_SIZE_FILE);
+
+ fd = open (sFile, O_RDONLY);
+ if (fd > 0)
+ {
+ bytesRead = read (fd, psBuf, MAX_SIZE_FILE);
+ close (fd);
+
+ if (FakeSMTPCommand (sock, "\r\n", "", sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ if (written (sock, psBuf, bytesRead) != bytesRead)
+ return ERR_WRITE;
+ }
+ else
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: file %s open error %d", sFile, errno);
+ return ERR_WRITE;
+ }
+
+ return _OK;
+}
+
+
+int GetFiles (char *pInpFile, int local_scan_fd)
+{
+ /*
+ Returns OK if no errors, else error code.
+ On successful return, pEnvFile points to Envelope file name and
+ pInpFile points to Message filename
+ */
+ int iStatus;
+ struct header_line *h_line;
+
+ iFdInp = mOpenTmp ((char *)temp_dir, "sp-inp", pInpFile);
+ if (iFdInp == -1)
+ {
+ return ERR_WRITE;
+ }
+
+ /* Emit headers */
+ h_line = header_list;
+ while (h_line != NULL)
+ {
+ if (h_line->type == '*') /* internal header */
+ {
+ h_line = h_line->next;
+ continue;
+ }
+
+ if (write (iFdInp, h_line->text, strlen (h_line->text)) != strlen (h_line->text))
+ {
+ CleanupInp ("");
+ return ERR_WRITE;
+ }
+ h_line = h_line->next;
+ }
+ if (write (iFdInp, "\n", 1) != 1)
+ {
+ CleanupInp ("");
+ return ERR_WRITE;
+ }
+
+ /* Read msg */
+ if ((iStatus = ReadFd (iFdInp, local_scan_fd)))
+ {
+ return iStatus;
+ }
+
+ /* Return success */
+ return _OK;
+}
+
+
+int GetAndTransferMessage (int fd, char *sFile)
+{
+ char answ [4];
+ int iStatus;
+ int Len, ccnt;
+ int test;
+
+ iStatus = GetFiles ((char *)sFile, fd);
+
+ if (iStatus != _OK)
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: Error %d getting message", iStatus);
+ close (sock);
+ return iStatus;
+ }
+
+ for (ccnt = 0; ccnt <= MAX_FAILS_C; ccnt ++)
+ {
+#ifdef RSPAM_UNIXSOCKET
+ test = connect (sock, (struct sockaddr *) &ssun, sizeof (struct sockaddr_un)) < 0;
+#else
+ test = connect (sock, (struct sockaddr *) &ssin, sizeof (struct sockaddr_in)) < 0;
+#endif
+ if (test)
+ {
+ if (ccnt < MAX_FAILS_C)
+ usleep (1000);
+ else
+ {
+ close (sock);
+#ifdef RSPAM_UNIXSOCKET
+ log_write (0, LOG_MAIN, "rspam-exim: socket connect to %s failed", (char *)socket_name);
+#else
+ log_write (0, LOG_MAIN, "rspam-exim: socket connect to %s:%u failed", daemonIP, daemonPort);
+#endif
+ return REJECT_ON_ERROR ? LOCAL_SCAN_TEMPREJECT:LOCAL_SCAN_ACCEPT;
+ }
+ }
+ else
+ break;
+ }
+
+ iStatus = SendEnvelope (sFile);
+ if (iStatus != _OK)
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: error %d sending envelope data", iStatus);
+ close (sock);
+ return iStatus;
+ }
+
+ /* fprintf (stderr, "Transmit OK\n"); */
+ return _OK;
+}
+
+void header_del (uschar *hdr)
+{
+ struct header_line *h_line;
+
+ h_line = header_list;
+ while (h_line != NULL)
+ {
+ if (h_line->type == '*') /* internal header */
+ {
+ h_line = h_line->next;
+ continue;
+ }
+
+ if (strncasecmp (h_line->text, hdr, strlen(hdr)) == 0)
+ {
+ h_line->type = '*';
+ while (h_line->next &&
+ (*h_line->next->text == ' ' || *h_line->next->text == '\t'))
+ {
+ h_line = h_line->next;
+ h_line->type = '*';
+ }
+ }
+ h_line = h_line->next;
+ }
+}
+
+void AlterSubject (char *label)
+{
+ struct header_line *h_line;
+ char *subject, *strP;
+
+ h_line = header_list;
+
+ while (h_line != NULL)
+ {
+ if (h_line->type == '*') /* internal header */
+ {
+ h_line = h_line->next;
+ continue;
+ }
+
+ if (strncasecmp (h_line->text, "Subject", strlen("Subject")) == 0)
+ {
+ strP = strchr (h_line->text, ':');
+ subject = string_copy (++strP);
+ while (h_line->next &&
+ (*h_line->next->text == ' ' || *h_line->next->text == '\t'))
+ {
+ h_line = h_line->next;
+ subject = string_sprintf ("%s\n%s", subject, h_line->text);
+ }
+ header_del (US "Subject");
+ break;
+ }
+
+ h_line = h_line->next;
+ }
+ header_add (' ', "Subject: %s%s", label, subject ? subject : "");
+}
+
+int
+io_read(int fd, char *buf, size_t size)
+{
+ int nfd, next = 0, rcount = 15;
+ size_t len = 0;
+ fd_set fds;
+ struct timeval tv;
+
+ if((sock < 0) || (buf == NULL))
+ return -1;
+
+ FD_ZERO(&fds);
+
+repeat_read:
+
+ tv.tv_sec = 5;
+ tv.tv_usec = 0;
+ FD_SET(fd, &fds);
+
+// log_write(0, LOG_MAIN, "rspam-exim: before select");
+
+ if((nfd=select(fd+1, &fds, NULL, NULL, &tv)) == -1) {
+// log_write(0, LOG_MAIN, "rspam-exim: select error: %s", strerror(errno));
+ return -1;
+ }
+
+// log_write(0, LOG_MAIN, "rspam-exim: select return %d fds, rcount %d, next %d", nfd, rcount, next);
+
+ if((nfd>0) && (FD_ISSET(fd, &fds))) {
+ next += len = read(fd, buf + next, size - next);
+// log_write(0, LOG_MAIN, "rspam-exim: read %d bytes", len);
+// if(next<size)
+// goto repeat_read;
+ }
+ rcount--;
+ if(rcount>0)
+ goto repeat_read;
+
+ return next;
+}
+
+int WaitForScanResult (uschar **resStr)
+{
+ int Len, i;
+ int rej = 0, result = LOCAL_SCAN_ACCEPT, answer_size, spm = 0, code = 0, ns = 0, smb = 0, urf = 0;
+ char *strP, *tok, *tmp;
+ char *hdr = NULL, *hdrv = NULL, *spmStr = NULL, *symbols=NULL, *urls=NULL;
+ char answ [4096], state[6], metric[128], back;
+ float sm=0, smd=0, smr=0;
+
+ memset (answ, '\0', sizeof (answ));
+// log_write(0, LOG_MAIN, "rspam-exim: before read from %d", sock);
+// Len = read (sock, answ, sizeof (answ) - 1);
+ Len = io_read(sock, answ, sizeof (answ) - 1);
+ log_write(0, LOG_MAIN, "rspam-exim: read %d bytes", Len);
+
+ if (strncmp (answ, "RSPAMD/1.1 ", 11) == 0)
+ {
+ strP = (char *)answ;
+ for (tok = strtok (strP, "\n"); tok; tok = strtok (NULL, "\n"))
+ {
+// log_write(0, LOG_MAIN, "rspam-exim: process line '%s'", tok);
+
+ if (strncmp (tok, "RSPAMD/1.1 ", 11) == 0)
+ {
+ if (sscanf (tok, "%*s %d %s", &code, state) == 2)
+ {
+// log_write(0, LOG_MAIN, "rspam-exim: daemon reports code %d %s", code, state);
+ if ((code == 0) && (strcmp(state,"OK")==0)) {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: SKIP\n", RSPAMD_FAILURE_HDR);
+ strange = 1;
+ continue;
+ } else {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: SKIP\n", RSPAMD_FAILURE_HDR);
+ log_write(0, LOG_MAIN, "rspam-exim: daemon reports code %d %s", code, state);
+ return LOCAL_SCAN_ACCEPT;
+ }
+ }
+ continue;
+ }
+
+ /* Metric: default; False; 6.00 / 10.00 */
+ /* Process metric */
+ if (strncmp (tok, "Metric:", 7) == 0)
+ {
+ tmp = tok;
+ while( (*tmp++) &&
+ ((*tmp!='\r') || (*tmp!='\n'))
+ );
+ back = *tmp;
+ *tmp = '\0';
+ if (sscanf (tok, "Metric: %[^';']; %[^';']; %f / %f / %f", metric, state, &sm, &smd, &smr) == 5) {
+ log_write(0, LOG_MAIN, "rspam-exim: metric: %s; %s; %f / %f / %f", metric, state, sm, smd, smr );
+ if(strcasecmp(state,"true")==0) {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: %s\n", RSPAMD_FAILURE_HDR, "Yes");
+ } else if(strcasecmp(state,"skip")==0) {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: %s\n", RSPAMD_FAILURE_HDR, "Skip");
+ } else {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: %s\n", RSPAMD_FAILURE_HDR, "No");
+ }
+ header_del ((uschar *) RSPAMD_SCORE_HDR);
+ header_add (' ', "%s: %.2f / %.2f / %.2f\n", RSPAMD_SCORE_HDR, sm, smd, smr);
+ strange = 0;
+ }
+ *tmp = back;
+ continue;
+ }
+
+ if (strncmp (tok, "Symbol:", 7) == 0)
+ {
+ tmp = tok;
+ while( (*tmp++) &&
+ ((*tmp!='\r') || (*tmp!='\n'))
+ );
+ back = *tmp;
+ *tmp = '\0';
+ if(smb>0) {
+ tok += 7;
+ while(*tok && isspace(*tok)) tok++;
+ if(strlen(tok)>0) {
+ symbols = string_sprintf ("%s\n %s", symbols, tok);
+ }
+ } else {
+ tok += 7;
+ while(*tok && isspace(*tok)) tok++;
+ symbols = string_copy (tok);
+ }
+ smb = 1;
+ *tmp = back;
+ continue;
+ }
+
+ if (strncmp (tok, "Urls:", 5) == 0)
+ {
+ tmp = tok;
+ while( (*tmp++) &&
+ ((*tmp!='\r') || (*tmp!='\n'))
+ );
+ back = *tmp;
+ *tmp = '\0';
+ if(urf>0) {
+ tok[0] = tok[1]= tok[2]= tok[3]= tok[4] = ' ';
+ urls = string_sprintf ("%s\n%s", urls, tok+3);
+ } else {
+ tok += 5;
+ while(*tok && isspace(*tok)) tok++;
+ urls = string_copy (tok);
+ }
+ urf = 1;
+ *tmp = back;
+ continue;
+ }
+ }
+
+
+ /* do not forget the symbols */
+ if (symbols != NULL && strlen(symbols))
+ {
+ i = 0;
+ tmp = tok = string_copy(symbols);
+ header_del ((uschar *) "X-Spam-Sybmols");
+ header_add (' ', "%s: %s\n", "X-Spam-Sybmols", symbols);
+ while(*tmp!='\0') {
+ if(*tmp == '\r')
+ *tmp = ' ';
+ if(*tmp == '\n')
+ *tmp = ',';
+ tmp++;
+ }
+ *tmp = '\0';
+ log_write(0, LOG_MAIN, "rspam-exim: symbols: %s", tok);
+ }
+
+ /* do not forget the urls */
+ if (urls != NULL && strlen(urls))
+ {
+ log_write(0, LOG_MAIN, "rspam-exim: urls: %s", urls);
+ header_del ((uschar *) "X-Spam-Urls");
+ header_add (' ', "%s: %s\n", "X-Spam-Urls", urls);
+ }
+
+ log_write (0, LOG_MAIN, "rspam-exim: For message from %s will return %s, mailfrom: <%s>, rcpto: <%s>", sender_host_address, rej == 2 ? "DISCARD" : rej == 1 ? "REJECT" : "ACCEPT", sender_address, recipients_list[0].address);
+
+ }
+ else
+ {
+ result = LOCAL_SCAN_ACCEPT;
+ log_write(0, LOG_MAIN, "rspam-exim: wrong signature in answer: %s", answ);
+ }
+
+ if((sm>0) && (smr>0) && (sm>=smr)) {
+ result = LOCAL_SCAN_REJECT;
+ }
+ return result;
+}
+
+
+int
+local_scan(int fd, uschar **return_text)
+{
+ int retval = _OK;
+ char sFileInp [MAX_PATH + 81];
+
+ /* Socket stuff */
+
+ strange = 0;
+#ifdef RSPAM_UNIXSOCKET
+ if ((sock = socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
+ {
+ log_write(0, LOG_MAIN, "rspam-exim: socket() failed");
+ exit (EXIT_FAILURE);
+ }
+ memset (&ssun, '\0', sizeof (struct sockaddr_un));
+ ssun.sun_family = AF_UNIX;
+ if (sizeof (socket_name) > sizeof (ssun.sun_path))
+ {
+ close (sock);
+ log_write(0, LOG_MAIN, "rspam-exim: UNIX socket name %s too long", socket_name);
+ exit (EXIT_FAILURE);
+ }
+ strcpy (ssun.sun_path, socket_name);
+#else
+ if ((sock = socket (AF_INET, SOCK_STREAM, 0)) < 0)
+ {
+ log_write(0, LOG_MAIN, "rspam-exim: socket() failed");
+ exit (EXIT_FAILURE);
+ }
+ memset (&ssin, '\0', sizeof (struct sockaddr_in));
+ ssin.sin_family = AF_INET;
+ ssin.sin_addr.s_addr = inet_addr (daemonIP);
+ ssin.sin_port = htons (daemonPort);
+#endif
+
+ if (GetAndTransferMessage (fd, (char *)sFileInp) != _OK)
+ {
+ close (sock);
+ unlink (sFileInp);
+ SPOOL_DATA_START_OFFSET;
+ return REJECT_ON_ERROR ? LOCAL_SCAN_TEMPREJECT:LOCAL_SCAN_ACCEPT;
+ }
+
+ retval = WaitForScanResult (return_text);
+
+ if(!strange)
+ unlink (sFileInp);
+ close (sock);
+ SPOOL_DATA_START_OFFSET;
+
+ return retval;
+}
+
+/* End of local_scan.c */
diff --git a/contrib/exim/local_scan.c.in b/contrib/exim/local_scan.c.in
new file mode 100644
index 0000000..72d2f79
--- /dev/null
+++ b/contrib/exim/local_scan.c.in
@@ -0,0 +1,377 @@
+/*
+ * This program is RSPAMD agent for use with
+ * exim (http://www.exim.org) MTA by its local_scan feature.
+ *
+ * To enable exim local scan please copy this file to exim source tree
+ * Local/local_scan.c, edit Local/Makefile to add
+ *
+ * LOCAL_SCAN_SOURCE=Local/local_scan.c
+ * LOCAL_SCAN_HAS_OPTIONS=yes
+ *
+ * and compile exim.
+ *
+ * For exim compilation with local scan feature details please visit
+ * http://www.exim.org/exim-html-current/doc/html/spec_html/ch42.html
+ *
+ * For RSPAMD details please visit
+ * https://bitbucket.org/vstakhov/rspamd/
+ *
+ * Example configuration:
+ * **********************
+ *
+ * local_scan_timeout = 50s
+ *
+ * begin local_scan
+ * rspam_ip = 127.0.0.1
+ * rspam_port = 11333
+ * rspam_skip_sasl_authenticated = true
+ * # don't reject message if on of recipients from this list
+ * rspam_skip_rcpt = postmaster@example.com : some_user@example.com
+ * rspam_message = "Spam rejected; If this is not spam, please contact <postmaster@example.com>"
+ *
+ *
+ * $Id: local_scan.c 646 2010-08-11 11:49:36Z ayuzhaninov $
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <errno.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "local_scan.h"
+
+#define REQUEST_LINES 64
+#define REPLY_BUF_SIZE 16384
+#define HEADER_STATUS "X-Rspam-Status"
+#define HEADER_METRIC "X-Rspam-Metric"
+#define HEADER_SCORE "X-Rspam-Score"
+
+/* configuration options */
+static uschar *daemon_ip = US"127.0.0.1";
+static int max_scan_size = 4 * 1024 * 1024;
+static uschar *reject_message = US"Spam message rejected";
+static int daemon_port = 11333;
+static BOOL skip_authenticated = TRUE;
+static uschar *want_spam_rcpt_list = US"";
+
+/* the entries must appear in alphabetical order */
+optionlist local_scan_options[] = {
+ { "rspam_ip", opt_stringptr, &daemon_ip },
+ { "rspam_max_scan_size", opt_mkint, &max_scan_size },
+ { "rspam_message", opt_stringptr, &reject_message },
+ { "rspam_port", opt_int, &daemon_port },
+ { "rspam_skip_rcpt", opt_stringptr, &want_spam_rcpt_list },
+ { "rspam_skip_sasl_authenticated", opt_bool, &skip_authenticated },
+};
+
+int local_scan_options_count = sizeof(local_scan_options) / sizeof(optionlist);
+
+/* push formatted line into vector */
+int push_line(struct iovec *iov, int i, const char *fmt, ...);
+
+int
+local_scan(int fd, uschar **return_text)
+{
+ struct stat sb;
+ struct sockaddr_in server_in;
+ int s, i, r, request_p = 0, headers_count = 0, is_spam = 0, is_reject = 0;
+ off_t message_size;
+ struct iovec request_v[REQUEST_LINES], *headers_v;
+#if "@CMAKE_SYSTEM_NAME@" == "FreeBSD"
+ struct sf_hdtr headers_sf;
+#endif
+ uschar *helo, *log_buf;
+ header_line *header_p;
+ char reply_buf[REPLY_BUF_SIZE], io_buf[BUFSIZ];
+ ssize_t size;
+ char *tok_ptr, *str;
+ char mteric[128], result[8];
+ float score, required_score;
+
+ *return_text = reject_message;
+
+ /*
+ * one msaage can be send via exim+rspamd twice
+ * remove header from previous pass
+ */
+ header_remove(0, US HEADER_STATUS);
+ header_remove(0, US HEADER_METRIC);
+
+ /* check message size */
+ fstat(fd,&sb); /* XXX shuld check error */
+ message_size = sb.st_size - SPOOL_DATA_START_OFFSET;
+ if (message_size > max_scan_size) {
+ header_add(' ', HEADER_STATUS ": skip_big\n");
+ log_write (0, LOG_MAIN, "rspam: message larger than rspam_max_scan_size, accept");
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* don't scan mail from authenticated hosts */
+ if (skip_authenticated && sender_host_authenticated != NULL) {
+ header_add(' ', HEADER_STATUS ": skip_authenticated\n");
+ log_write(0, LOG_MAIN, "rspam: from=<%s> ip=%s authenticated (%s), skip check\n",
+ sender_address,
+ sender_host_address == NULL ? US"localhost" : sender_host_address,
+ sender_host_authenticated);
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /*
+ * add status header, which mean, that message was not scanned
+ * if message will be scanned, this header will be replaced
+ */
+ header_add(' ', HEADER_STATUS ": check_error\n");
+
+ /* create socket */
+ memset(&server_in, 0, sizeof(server_in));
+ server_in.sin_family = AF_INET;
+ server_in.sin_port = htons(daemon_port);
+ server_in.sin_addr.s_addr = inet_addr(daemon_ip);
+ if ((s = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
+ log_write(0, LOG_MAIN, "rspam: socket (%d: %s)", errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+ if (connect(s, (struct sockaddr *) &server_in, sizeof(server_in)) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't connect to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* count message headers */
+ for (header_p = header_list; header_p != NULL; header_p = header_p->next) {
+ /* header type '*' is used for replaced or deleted header */
+ if (header_p->type == '*')
+ continue;
+ headers_count++;
+ }
+
+ /* write message headers to vector */
+#if "@CMAKE_SYSTEM_NAME@" == "FreeBSD"
+ memset(&headers_sf, 0, sizeof(headers_sf));
+ if (headers_count > 0) {
+ headers_v = store_get((headers_count + 1)* sizeof(*headers_v));
+ i = 0;
+ for (header_p = header_list; header_p != NULL; header_p = header_p->next) {
+ if (header_p->type == '*')
+ continue;
+ headers_v[i].iov_base = header_p->text;
+ headers_v[i].iov_len = header_p->slen;
+ i++;
+ message_size += header_p->slen;
+ }
+ headers_v[i].iov_base = "\n";
+ headers_v[i].iov_len = strlen("\n");
+ message_size += strlen("\n");
+
+ headers_sf.headers = headers_v;
+ headers_sf.hdr_cnt = headers_count + 1;
+ }
+#else
+ if (headers_count > 0) {
+ headers_v = store_get((headers_count + 1)* sizeof(*headers_v));
+ i = 0;
+ for (header_p = header_list; header_p != NULL; header_p = header_p->next) {
+ if (header_p->type == '*')
+ continue;
+ headers_v[i].iov_base = header_p->text;
+ headers_v[i].iov_len = header_p->slen;
+ i++;
+ message_size += header_p->slen;
+ }
+ headers_v[i].iov_base = "\n";
+ headers_v[i].iov_len = strlen("\n");
+ message_size += strlen("\n");
+#endif
+
+ /* write request to vector */
+ r = 0;
+ r += push_line(request_v, request_p++, "SYMBOLS RSPAMC/1.1\r\n");
+ r += push_line(request_v, request_p++, "Content-length: " OFF_T_FMT "\r\n", message_size);
+ r += push_line(request_v, request_p++, "Queue-Id: %s\r\n", message_id);
+ r += push_line(request_v, request_p++, "From: %s\r\n", sender_address);
+ r += push_line(request_v, request_p++, "Recipient-Number: %d\r\n", recipients_count);
+ for (i = 0; i < recipients_count; i ++)
+ r += push_line(request_v, request_p++, "Rcpt: %s\r\n", recipients_list[i].address);
+ if ((helo = expand_string(US"$sender_helo_name")) != NULL && *helo != '\0')
+ r += push_line(request_v, request_p++, "Helo: %s\r\n", helo);
+ if (sender_host_address != NULL)
+ r += push_line(request_v, request_p++, "IP: %s\r\n", sender_host_address);
+ r += push_line(request_v, request_p++, "\r\n");
+
+ if (r < 0) {
+ close(s);
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* send request */
+ if (writev(s, request_v, request_p) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send request to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+#if "@CMAKE_SYSTEM_NAME@" == "FreeBSD"
+ /* send headers (from iovec) and message body (from file) */
+ if (sendfile(fd, s, SPOOL_DATA_START_OFFSET, 0, &headers_sf, NULL, 0) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send message to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+#else
+ /* send headers */
+ if (writev(s, headers_v, headers_count) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send headers to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* Send message */
+ while ((r = read (fd, io_buf, sizeof (io_buf))) > 0) {
+ if (write (s, io_buf, r) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send message to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+ }
+#endif
+
+ /* read reply from rspamd */
+ reply_buf[0] = '\0';
+ size = 0;
+ while ((r = read(s, reply_buf + size, sizeof(reply_buf) - size - 1)) > 0 && size < sizeof(reply_buf) - 1) {
+ size += r;
+ }
+
+ if (r < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't read from %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+ reply_buf[size] = '\0';
+ close(s);
+
+ if (size >= REPLY_BUF_SIZE - 1) {
+ log_write(0, LOG_MAIN, "rspam: buffer is full, reply may be truncated");
+ }
+
+ /* parse reply */
+ tok_ptr = reply_buf;
+
+ /*
+ * rspamd can use several metrics, logic implemented here:
+ * if any metric more than reject_score - will reject
+ * if any metric true - message will be marked as spam
+ */
+
+ /* First line is: <PROTOCOL>/<VERSION> <ERROR_CODE> <ERROR_REPLY> */
+ str = strsep(&tok_ptr, "\r\n");
+ if (str != NULL && sscanf(str, "%*s %d %*s", &i) == 1) {
+ if (i != 0) {
+ log_write(0, LOG_MAIN, "rspam: server error: %s", str);
+ return LOCAL_SCAN_ACCEPT;
+ }
+ } else {
+ log_write(0, LOG_MAIN, "rspam: bad reply from server: %s", str);
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ while ((str = strsep(&tok_ptr, "\r\n")) != NULL) {
+ /* skip empty tockens */
+ if (*str == '\0')
+ continue;
+ if (strncmp(str, "Metric:", strlen("Metric:")) == 0) {
+ /*
+ * parse line like
+ * Metric: default; False; 27.00 / 30.00
+ */
+ if (sscanf(str, "Metric: %s %s %f / %f",
+ mteric, result, &score, &required_score) == 4) {
+ log_write(0, LOG_MAIN, "rspam: metric %s %s %.2f / %.2f",
+ mteric, result, score, required_score);
+ header_add(' ', HEADER_METRIC ": %s %s %.2f / %.2f\n",
+ mteric, result, score, required_score);
+ /* integers score for use in sieve ascii-numeric comparator */
+ if (strcmp(mteric, "default;") == 0)
+ header_add(' ', HEADER_SCORE ": %d\n",
+ (int)round(score));
+ } else {
+ log_write(0, LOG_MAIN, "rspam: can't parse: %s", str);
+ return LOCAL_SCAN_ACCEPT;
+ }
+ } else if (strncmp(str, "Action:", strlen("Action:")) == 0) {
+ /* line like Action: add header */
+ str += strlen("Action: ");
+ if (strncmp(str, "reject", strlen("reject")) == 0) {
+ is_reject = 1;
+ is_spam = 1;
+ } else if (strncmp(str, "add header", strlen("add header")) == 0) {
+ is_spam = 1;
+ }
+ }
+ }
+
+ /* XXX many allocs by string_sprintf()
+ * better to sprintf() to single buffer allocated by store_get()
+ */
+ log_buf = string_sprintf("message to");
+ for (i = 0; i < recipients_count; i ++) {
+ log_buf = string_sprintf("%s %s", log_buf, recipients_list[i].address);
+ if (is_reject && lss_match_address(recipients_list[i].address, want_spam_rcpt_list, TRUE) == OK) {
+ is_reject = 0;
+ log_write(0, LOG_MAIN, "rspam: %s want spam, don't reject this message", recipients_list[i].address);
+ }
+ }
+
+ if (is_reject) {
+ log_write(0, LOG_MAIN, "rspam: reject %s", log_buf);
+ return LOCAL_SCAN_REJECT;
+ }
+
+ header_remove(0, US HEADER_STATUS);
+ if (is_spam) {
+ header_add(' ', HEADER_STATUS ": spam\n");
+ log_write(0, LOG_MAIN, "rspam: message marked as spam");
+ } else {
+ header_add(' ', HEADER_STATUS ": ham\n");
+ log_write(0, LOG_MAIN, "rspam: message marked as ham");
+ }
+
+ return LOCAL_SCAN_ACCEPT;
+}
+
+int
+push_line(struct iovec *iov, const int i, const char *fmt, ...)
+{
+ va_list ap;
+ size_t len;
+ char buf[512];
+
+ if (i >= REQUEST_LINES) {
+ log_write(0, LOG_MAIN, "rspam: %s: index out of bounds", __FUNCTION__);
+ return (-1);
+ }
+
+ va_start(ap, fmt);
+ len = vsnprintf(buf, sizeof(buf), fmt, ap);
+ va_end(ap);
+
+ iov[i].iov_base = string_copy(US buf);
+ iov[i].iov_len = len;
+
+ if (len >= sizeof(buf)) {
+ log_write(0, LOG_MAIN, "rspam: %s: error, string was longer than %d", __FUNCTION__, sizeof(buf));
+ return (-1);
+ }
+
+ return 0;
+}
+
diff --git a/contrib/exim/patch-exim-src_spam.c.diff b/contrib/exim/patch-exim-src_spam.c.diff
new file mode 100644
index 0000000..f3ac788
--- /dev/null
+++ b/contrib/exim/patch-exim-src_spam.c.diff
@@ -0,0 +1,338 @@
+diff -ru exim-4.70.orig/src/expand.c exim-4.70/src/expand.c
+--- exim-4.70.orig/src/expand.c 2016-04-09 13:42:00.227625074 +0200
++++ exim-4.70/src/expand.c 2016-04-09 13:42:37.183633096 +0200
+@@ -578,6 +578,7 @@
+ { "sn8", vtype_filter_int, &filter_sn[8] },
+ { "sn9", vtype_filter_int, &filter_sn[9] },
+ #ifdef WITH_CONTENT_SCAN
++ { "spam_action", vtype_stringptr, &spam_action },
+ { "spam_bar", vtype_stringptr, &spam_bar },
+ { "spam_report", vtype_stringptr, &spam_report },
+ { "spam_score", vtype_stringptr, &spam_score },
+diff -ru exim-4.70.orig/src/globals.c exim-4.70/src/globals.c
+--- exim-4.70.orig/src/globals.c 2016-04-09 13:42:00.219625073 +0200
++++ exim-4.70/src/globals.c 2016-04-09 13:42:37.187633096 +0200
+@@ -1136,6 +1136,7 @@
+ uschar *spamd_address = US"127.0.0.1 783";
+ uschar *spam_bar = NULL;
+ uschar *spam_report = NULL;
++uschar *spam_action = NULL;
+ uschar *spam_score = NULL;
+ uschar *spam_score_int = NULL;
+ #endif
+diff -ru exim-4.70.orig/src/globals.h exim-4.70/src/globals.h
+--- exim-4.70.orig/src/globals.h 2016-04-09 13:42:00.219625073 +0200
++++ exim-4.70/src/globals.h 2016-04-09 13:42:37.187633096 +0200
+@@ -703,6 +703,7 @@
+ extern uschar *spamd_address; /* address for the spamassassin daemon */
+ extern uschar *spam_bar; /* the spam "bar" (textual representation of spam_score) */
+ extern uschar *spam_report; /* the spamd report (multiline) */
++extern uschar *spam_action; /* the spamd action */
+ extern uschar *spam_score; /* the spam score (float) */
+ extern uschar *spam_score_int; /* spam_score * 10 (int) */
+ #endif
+diff -ru exim-4.70.orig/src/spam.c exim-4.70/src/spam.c
+--- exim-4.70.orig/src/spam.c 2016-04-09 13:42:00.231625075 +0200
++++ exim-4.70/src/spam.c 2016-04-09 13:43:43.927647677 +0200
+@@ -16,6 +16,7 @@
+ uschar spam_score_buffer[16];
+ uschar spam_score_int_buffer[16];
+ uschar spam_bar_buffer[128];
++uschar spam_action_buffer[32];
+ uschar spam_report_buffer[32600];
+ uschar prev_user_name[128] = "";
+ int spam_ok = 0;
+@@ -31,9 +32,11 @@
+ int spamd_sock;
+ uschar spamd_buffer[32600];
+ int i, j, offset, result;
++ BOOL is_rspamd;
+ uschar spamd_version[8];
++ uschar spamd_short_result[8];
+ uschar spamd_score_char;
+- double spamd_threshold, spamd_score;
++ double spamd_threshold, spamd_score, spamd_reject_score;
+ int spamd_report_offset;
+ uschar *p,*q;
+ int override = 0;
+@@ -122,8 +125,15 @@
+ spamd_address_container *this_spamd =
+ (spamd_address_container *)store_get(sizeof(spamd_address_container));
+
++ /* Check for spamd variant */
++ if( Ustrstr(address, "variant=rspamd") != NULL ) {
++ this_spamd->is_rspamd = TRUE;
++ }
++ else {
++ this_spamd->is_rspamd = FALSE;
++ }
+ /* grok spamd address and port */
+- if( sscanf(CS address, "%s %u", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2 ) {
++ if( sscanf(CS address, "%23s %hu", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2 ) {
+ log_write(0, LOG_MAIN,
+ "spam acl condition: warning - invalid spamd address: '%s'", address);
+ continue;
+@@ -165,6 +175,7 @@
+ spamd_address_vector[current_server]->tcp_port,
+ 5 ) > -1) {
+ /* connection OK */
++ is_rspamd = spamd_address_vector[current_server]->is_rspamd;
+ break;
+ };
+
+@@ -197,12 +208,28 @@
+ }
+
+ server.sun_family = AF_UNIX;
+- Ustrcpy(server.sun_path, spamd_address_work);
++ p = Ustrstr(spamd_address_work, "variant=rspamd");
++ if( p != NULL ) {
++ is_rspamd = TRUE;
++ /* strip spaces */
++ p --;
++ while (p > spamd_address_work && isspace (*p)) {
++ p --;
++ }
++ Ustrncpy(server.sun_path, spamd_address_work, p - spamd_address_work + 1);
++ /* zero terminate */
++ server.sun_path[p - spamd_address_work + 1] = 0;
++ }
++ else {
++ is_rspamd = FALSE;
++ Ustrcpy(server.sun_path, spamd_address_work);
++ }
++
+
+ if (connect(spamd_sock, (struct sockaddr *) &server, sizeof(struct sockaddr_un)) < 0) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+ "malware acl condition: spamd: unable to connect to UNIX socket %s (%s)",
+- spamd_address_work, strerror(errno) );
++ server.sun_path, strerror(errno) );
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+@@ -210,22 +237,50 @@
+
+ }
+
++ (void)fcntl(spamd_sock, F_SETFL, O_NONBLOCK);
+ /* now we are connected to spamd on spamd_sock */
+- (void)string_format(spamd_buffer,
+- sizeof(spamd_buffer),
+- "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
+- user_name,
+- mbox_size);
+-
+- /* send our request */
+- if (send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0) < 0) {
++ if (is_rspamd) {
++ /* rspamd variant */
++ const char *helo;
++ const char *fcrdns;
++ const char *authid;
++ uschar *req_str;
++
++ req_str = string_sprintf("CHECK RSPAMC/1.3\r\nContent-length: %lu\r\n"
++ "Queue-Id: %s\r\nFrom: <%s>\r\nRecipient-Number: %d\r\n", mbox_size,
++ message_id, sender_address, recipients_count);
++ for (i = 0; i < recipients_count; i ++)
++ req_str = string_sprintf("%sRcpt: <%s>\r\n", req_str, recipients_list[i].address);
++ if ((helo = expand_string(US"$sender_helo_name")) != NULL && *helo != '\0')
++ req_str = string_sprintf("%sHelo: %s\r\n", req_str, helo);
++ if ((fcrdns = expand_string(US"$sender_host_name")) != NULL && *fcrdns != '\0')
++ req_str = string_sprintf("%sHostname: %s\r\n", req_str, fcrdns);
++ if (sender_host_address != NULL)
++ req_str = string_sprintf("%sIP: %s\r\n", req_str, sender_host_address);
++ if ((authid = expand_string(US"$authenticated_id")) != NULL && *authid != '\0')
++ req_str = string_sprintf("%sUser: %s\r\n", req_str, authid);
++ req_str = string_sprintf("%s\r\n", req_str);
++ wrote = send(spamd_sock, req_str, Ustrlen(req_str), 0);
++ }
++ else {
++ /* spamassassin variant */
++ (void)string_format(spamd_buffer,
++ sizeof(spamd_buffer),
++ "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
++ user_name,
++ mbox_size);
++ /* send our request */
++ wrote = send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0);
++ }
++ if(wrote == -1)
++ {
+ (void)close(spamd_sock);
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: spamd send failed: %s", strerror(errno));
++ "spam acl condition: spamd send failed: %s", strerror(errno));
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+- };
++ }
+
+ /* now send the file */
+ /* spamd sometimes accepts connections but doesn't read data off
+@@ -304,7 +359,9 @@
+ (void)fclose(mbox_file);
+
+ /* we're done sending, close socket for writing */
+- shutdown(spamd_sock,SHUT_WR);
++ if (!is_rspamd) {
++ shutdown(spamd_sock,SHUT_WR);
++ }
+
+ /* read spamd response using what's left of the timeout.
+ */
+@@ -328,60 +385,93 @@
+ /* reading done */
+ (void)close(spamd_sock);
+
+- /* dig in the spamd output and put the report in a multiline header, if requested */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
+-
+- /* try to fall back to pre-2.50 spamd output */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ if (!is_rspamd) {
++ /* dig in the spamd output and put the report in a multiline header, if requested */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++
++ /* try to fall back to pre-2.50 spamd output */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ log_write(0, LOG_MAIN|LOG_PANIC,
++ "spam acl condition: cannot parse spamd output");
++ return DEFER;
++ };
++ };
++
++ if( spamd_score >= spamd_threshold ) {
++ Ustrcpy(spam_action_buffer, "reject");
++ }
++ else {
++ Ustrcpy(spam_action_buffer, "no action");
++ }
++ }
++ else {
++ /* rspamd variant of reply */
++ int r;
++ if( (r = sscanf(CS spamd_buffer,"RSPAMD/%7s 0 EX_OK\r\nMetric: default; %7s %lf / %lf / %lf\r\n%n",
++ spamd_version,spamd_short_result,&spamd_score,&spamd_threshold,&spamd_reject_score,&spamd_report_offset)) != 5 ) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: cannot parse spamd output");
++ "spam acl condition: cannot parse spamd output: %d", r);
+ return DEFER;
+ };
+- };
++ /* now parse action */
++ p = &spamd_buffer[spamd_report_offset];
++
++ if( Ustrncmp(p, "Action: ", sizeof("Action: ") - 1) == 0 ) {
++ p += sizeof("Action: ") - 1;
++ q = &spam_action_buffer[0];
++ while (*p && *p != '\r' && (q - spam_action_buffer) < sizeof(spam_action_buffer) - 1) {
++ *q++ = *p++;
++ }
++ *q = '\0';
++ }
++ }
+
+ /* Create report. Since this is a multiline string,
+ we must hack it into shape first */
+ p = &spamd_buffer[spamd_report_offset];
+ q = spam_report_buffer;
+ while (*p != '\0') {
+- /* skip \r */
+- if (*p == '\r') {
+- p++;
+- continue;
+- };
+- *q = *p;
+- q++;
+- if (*p == '\n') {
+- /* add an extra space after the newline to ensure
+- that it is treated as a header continuation line */
+- *q = ' ';
+- q++;
+- };
+- p++;
++ /* skip \r */
++ if (*p == '\r') {
++ p++;
++ continue;
++ };
++ *q = *p;
++ q++;
++ if (*p == '\n') {
++ /* add an extra space after the newline to ensure
++ that it is treated as a header continuation line */
++ *q = ' ';
++ q++;
++ };
++ p++;
+ };
+ /* NULL-terminate */
+ *q = '\0';
+ q--;
+ /* cut off trailing leftovers */
+ while (*q <= ' ') {
+- *q = '\0';
+- q--;
++ *q = '\0';
++ q--;
+ };
++
++ /* common spamd actions */
+ spam_report = spam_report_buffer;
++ spam_action = spam_action_buffer;
+
+ /* create spam bar */
+ spamd_score_char = spamd_score > 0 ? '+' : '-';
+ j = abs((int)(spamd_score));
+ i = 0;
+ if( j != 0 ) {
+- while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
+- spam_bar_buffer[i++] = spamd_score_char;
++ while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
++ spam_bar_buffer[i++] = spamd_score_char;
+ }
+ else{
+- spam_bar_buffer[0] = '/';
+- i = 1;
++ spam_bar_buffer[0] = '/';
++ i = 1;
+ }
+ spam_bar_buffer[i] = '\0';
+ spam_bar = spam_bar_buffer;
+@@ -397,12 +487,12 @@
+
+ /* compare threshold against score */
+ if (spamd_score >= spamd_threshold) {
+- /* spam as determined by user's threshold */
+- spam_rc = OK;
++ /* spam as determined by user's threshold */
++ spam_rc = OK;
+ }
+ else {
+- /* not spam */
+- spam_rc = FAIL;
++ /* not spam */
++ spam_rc = FAIL;
+ };
+
+ /* remember user name and "been here" for it unless spamd_socket was expanded */
+diff -ru exim-4.70.orig/src/spam.h exim-4.70/src/spam.h
+--- exim-4.70.orig/src/spam.h 2016-04-09 13:42:00.235625076 +0200
++++ exim-4.70/src/spam.h 2016-04-09 13:42:37.187633096 +0200
+@@ -24,7 +24,8 @@
+
+ typedef struct spamd_address_container {
+ uschar tcp_addr[24];
+- unsigned int tcp_port;
++ unsigned short int tcp_port;
++ int is_rspamd:1;
+ } spamd_address_container;
+
+ #endif
diff --git a/contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff b/contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff
new file mode 100644
index 0000000..0389aea
--- /dev/null
+++ b/contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff
@@ -0,0 +1,332 @@
+diff -ru exim-4.85.orig/src/expand.c exim-4.85/src/expand.c
+--- exim-4.85.orig/src/expand.c 2016-04-09 13:47:01.707691638 +0200
++++ exim-4.85/src/expand.c 2016-04-09 13:47:29.771697969 +0200
+@@ -652,6 +652,7 @@
+ { "sn8", vtype_filter_int, &filter_sn[8] },
+ { "sn9", vtype_filter_int, &filter_sn[9] },
+ #ifdef WITH_CONTENT_SCAN
++ { "spam_action", vtype_stringptr, &spam_action },
+ { "spam_bar", vtype_stringptr, &spam_bar },
+ { "spam_report", vtype_stringptr, &spam_report },
+ { "spam_score", vtype_stringptr, &spam_score },
+Only in exim-4.85/src: expand.c.orig
+diff -ru exim-4.85.orig/src/globals.c exim-4.85/src/globals.c
+--- exim-4.85.orig/src/globals.c 2016-04-09 13:47:01.695691635 +0200
++++ exim-4.85/src/globals.c 2016-04-09 13:47:29.771697969 +0200
+@@ -1276,6 +1276,7 @@
+ uschar *spamd_address = US"127.0.0.1 783";
+ uschar *spam_bar = NULL;
+ uschar *spam_report = NULL;
++uschar *spam_action = NULL;
+ uschar *spam_score = NULL;
+ uschar *spam_score_int = NULL;
+ #endif
+Only in exim-4.85/src: globals.c.orig
+diff -ru exim-4.85.orig/src/globals.h exim-4.85/src/globals.h
+--- exim-4.85.orig/src/globals.h 2016-04-09 13:47:01.695691635 +0200
++++ exim-4.85/src/globals.h 2016-04-09 13:47:29.771697969 +0200
+@@ -819,6 +819,7 @@
+ extern uschar *spamd_address; /* address for the spamassassin daemon */
+ extern uschar *spam_bar; /* the spam "bar" (textual representation of spam_score) */
+ extern uschar *spam_report; /* the spamd report (multiline) */
++extern uschar *spam_action; /* the spamd action */
+ extern uschar *spam_score; /* the spam score (float) */
+ extern uschar *spam_score_int; /* spam_score * 10 (int) */
+ #endif
+Only in exim-4.85/src: globals.h.orig
+diff -ru exim-4.85.orig/src/spam.c exim-4.85/src/spam.c
+--- exim-4.85.orig/src/spam.c 2016-04-09 13:47:01.711691638 +0200
++++ exim-4.85/src/spam.c 2016-04-09 13:52:12.611762892 +0200
+@@ -14,6 +14,7 @@
+ uschar spam_score_buffer[16];
+ uschar spam_score_int_buffer[16];
+ uschar spam_bar_buffer[128];
++uschar spam_action_buffer[32];
+ uschar spam_report_buffer[32600];
+ uschar prev_user_name[128] = "";
+ int spam_ok = 0;
+@@ -32,9 +33,11 @@
+ int spamd_sock = -1;
+ uschar spamd_buffer[32600];
+ int i, j, offset, result;
++ BOOL is_rspamd;
+ uschar spamd_version[8];
++ uschar spamd_short_result[8];
+ uschar spamd_score_char;
+- double spamd_threshold, spamd_score;
++ double spamd_threshold, spamd_score, spamd_reject_score;
+ int spamd_report_offset;
+ uschar *p,*q;
+ int override = 0;
+@@ -128,8 +131,15 @@
+ spamd_address_container *this_spamd =
+ (spamd_address_container *)store_get(sizeof(spamd_address_container));
+
++ /* Check for spamd variant */
++ if( Ustrstr(address, "variant=rspamd") != NULL ) {
++ this_spamd->is_rspamd = TRUE;
++ }
++ else {
++ this_spamd->is_rspamd = FALSE;
++ }
+ /* grok spamd address and port */
+- if (sscanf(CS address, "%23s %u", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2)
++ if (sscanf(CS address, "%23s %hu", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2)
+ {
+ log_write(0, LOG_MAIN,
+ "spam acl condition: warning - invalid spamd address: '%s'", address);
+@@ -174,6 +184,7 @@
+ spamd_address_vector[current_server]->tcp_port,
+ 5 ) > -1) {
+ /* connection OK */
++ is_rspamd = spamd_address_vector[current_server]->is_rspamd;
+ break;
+ };
+
+@@ -210,12 +221,28 @@
+ }
+
+ server.sun_family = AF_UNIX;
+- Ustrcpy(server.sun_path, spamd_address_work);
++ p = Ustrstr(spamd_address_work, "variant=rspamd");
++ if( p != NULL ) {
++ is_rspamd = TRUE;
++ /* strip spaces */
++ p --;
++ while (p > spamd_address_work && isspace (*p)) {
++ p --;
++ }
++ Ustrncpy(server.sun_path, spamd_address_work, p - spamd_address_work + 1);
++ /* zero terminate */
++ server.sun_path[p - spamd_address_work + 1] = 0;
++ }
++ else {
++ is_rspamd = FALSE;
++ Ustrcpy(server.sun_path, spamd_address_work);
++ }
++
+
+ if (connect(spamd_sock, (struct sockaddr *) &server, sizeof(struct sockaddr_un)) < 0) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+ "malware acl condition: spamd: unable to connect to UNIX socket %s (%s)",
+- spamd_address_work, strerror(errno) );
++ server.sun_path, strerror(errno) );
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+@@ -231,22 +258,50 @@
+ return DEFER;
+ }
+
++ (void)fcntl(spamd_sock, F_SETFL, O_NONBLOCK);
+ /* now we are connected to spamd on spamd_sock */
+- (void)string_format(spamd_buffer,
+- sizeof(spamd_buffer),
+- "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
+- user_name,
+- mbox_size);
+-
+- /* send our request */
+- if (send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0) < 0) {
++ if (is_rspamd) {
++ /* rspamd variant */
++ const char *helo;
++ const char *fcrdns;
++ const char *authid;
++ uschar *req_str;
++
++ req_str = string_sprintf("CHECK RSPAMC/1.3\r\nContent-length: %lu\r\n"
++ "Queue-Id: %s\r\nFrom: <%s>\r\nRecipient-Number: %d\r\n", mbox_size,
++ message_id, sender_address, recipients_count);
++ for (i = 0; i < recipients_count; i ++)
++ req_str = string_sprintf("%sRcpt: <%s>\r\n", req_str, recipients_list[i].address);
++ if ((helo = expand_string(US"$sender_helo_name")) != NULL && *helo != '\0')
++ req_str = string_sprintf("%sHelo: %s\r\n", req_str, helo);
++ if ((fcrdns = expand_string(US"$sender_host_name")) != NULL && *fcrdns != '\0')
++ req_str = string_sprintf("%sHostname: %s\r\n", req_str, fcrdns);
++ if (sender_host_address != NULL)
++ req_str = string_sprintf("%sIP: %s\r\n", req_str, sender_host_address);
++ if ((authid = expand_string(US"$authenticated_id")) != NULL && *authid != '\0')
++ req_str = string_sprintf("%sUser: %s\r\n", req_str, authid);
++ req_str = string_sprintf("%s\r\n", req_str);
++ wrote = send(spamd_sock, req_str, Ustrlen(req_str), 0);
++ }
++ else {
++ /* spamassassin variant */
++ (void)string_format(spamd_buffer,
++ sizeof(spamd_buffer),
++ "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
++ user_name,
++ mbox_size);
++ /* send our request */
++ wrote = send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0);
++ }
++ if(wrote == -1)
++ {
+ (void)close(spamd_sock);
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: spamd send failed: %s", strerror(errno));
++ "spam acl condition: spamd send failed: %s", strerror(errno));
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+- };
++ }
+
+ /* now send the file */
+ /* spamd sometimes accepts connections but doesn't read data off
+@@ -349,60 +404,93 @@
+ /* reading done */
+ (void)close(spamd_sock);
+
+- /* dig in the spamd output and put the report in a multiline header, if requested */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
+-
+- /* try to fall back to pre-2.50 spamd output */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ if (!is_rspamd) {
++ /* dig in the spamd output and put the report in a multiline header, if requested */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++
++ /* try to fall back to pre-2.50 spamd output */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ log_write(0, LOG_MAIN|LOG_PANIC,
++ "spam acl condition: cannot parse spamd output");
++ return DEFER;
++ };
++ };
++
++ if( spamd_score >= spamd_threshold ) {
++ Ustrcpy(spam_action_buffer, "reject");
++ }
++ else {
++ Ustrcpy(spam_action_buffer, "no action");
++ }
++ }
++ else {
++ /* rspamd variant of reply */
++ int r;
++ if( (r = sscanf(CS spamd_buffer,"RSPAMD/%7s 0 EX_OK\r\nMetric: default; %7s %lf / %lf / %lf\r\n%n",
++ spamd_version,spamd_short_result,&spamd_score,&spamd_threshold,&spamd_reject_score,&spamd_report_offset)) != 5 ) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: cannot parse spamd output");
++ "spam acl condition: cannot parse spamd output: %d", r);
+ return DEFER;
+ };
+- };
++ /* now parse action */
++ p = &spamd_buffer[spamd_report_offset];
++
++ if( Ustrncmp(p, "Action: ", sizeof("Action: ") - 1) == 0 ) {
++ p += sizeof("Action: ") - 1;
++ q = &spam_action_buffer[0];
++ while (*p && *p != '\r' && (q - spam_action_buffer) < sizeof(spam_action_buffer) - 1) {
++ *q++ = *p++;
++ }
++ *q = '\0';
++ }
++ }
+
+ /* Create report. Since this is a multiline string,
+ we must hack it into shape first */
+ p = &spamd_buffer[spamd_report_offset];
+ q = spam_report_buffer;
+ while (*p != '\0') {
+- /* skip \r */
+- if (*p == '\r') {
+- p++;
+- continue;
+- };
+- *q = *p;
+- q++;
+- if (*p == '\n') {
+- /* add an extra space after the newline to ensure
+- that it is treated as a header continuation line */
+- *q = ' ';
+- q++;
+- };
+- p++;
++ /* skip \r */
++ if (*p == '\r') {
++ p++;
++ continue;
++ };
++ *q = *p;
++ q++;
++ if (*p == '\n') {
++ /* add an extra space after the newline to ensure
++ that it is treated as a header continuation line */
++ *q = ' ';
++ q++;
++ };
++ p++;
+ };
+ /* NULL-terminate */
+ *q = '\0';
+ q--;
+ /* cut off trailing leftovers */
+ while (*q <= ' ') {
+- *q = '\0';
+- q--;
++ *q = '\0';
++ q--;
+ };
++
++ /* common spamd actions */
+ spam_report = spam_report_buffer;
++ spam_action = spam_action_buffer;
+
+ /* create spam bar */
+ spamd_score_char = spamd_score > 0 ? '+' : '-';
+ j = abs((int)(spamd_score));
+ i = 0;
+ if( j != 0 ) {
+- while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
+- spam_bar_buffer[i++] = spamd_score_char;
++ while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
++ spam_bar_buffer[i++] = spamd_score_char;
+ }
+ else{
+- spam_bar_buffer[0] = '/';
+- i = 1;
++ spam_bar_buffer[0] = '/';
++ i = 1;
+ }
+ spam_bar_buffer[i] = '\0';
+ spam_bar = spam_bar_buffer;
+@@ -418,12 +506,12 @@
+
+ /* compare threshold against score */
+ if (spamd_score >= spamd_threshold) {
+- /* spam as determined by user's threshold */
+- spam_rc = OK;
++ /* spam as determined by user's threshold */
++ spam_rc = OK;
+ }
+ else {
+- /* not spam */
+- spam_rc = FAIL;
++ /* not spam */
++ spam_rc = FAIL;
+ };
+
+ /* remember expanded spamd_address if needed */
+Only in exim-4.85/src: spam.c.orig
+Only in exim-4.85/src: .spam.c.rej.swp
+diff -ru exim-4.85.orig/src/spam.h exim-4.85/src/spam.h
+--- exim-4.85.orig/src/spam.h 2016-04-09 13:47:01.715691640 +0200
++++ exim-4.85/src/spam.h 2016-04-09 13:47:29.775697969 +0200
+@@ -22,7 +22,8 @@
+
+ typedef struct spamd_address_container {
+ uschar tcp_addr[24];
+- unsigned int tcp_port;
++ unsigned short int tcp_port;
++ int is_rspamd:1;
+ } spamd_address_container;
+
+ #endif
diff --git a/contrib/exim/shutdown.patch b/contrib/exim/shutdown.patch
new file mode 100644
index 0000000..e8bf8a0
--- /dev/null
+++ b/contrib/exim/shutdown.patch
@@ -0,0 +1,14 @@
+--- exim4-4.86.2.orig/src/spam.c
++++ exim4-4.86.2/src/spam.c
+@@ -499,7 +499,10 @@ if (ferror(mbox_file))
+ (void)fclose(mbox_file);
+
+ /* we're done sending, close socket for writing */
+-shutdown(spamd_sock,SHUT_WR);
++if (!sd->is_rspamd)
++ {
++ shutdown(spamd_sock,SHUT_WR);
++ }
+
+ /* read spamd response using what's left of the timeout. */
+ memset(spamd_buffer, 0, sizeof(spamd_buffer));
diff --git a/contrib/expected/COPYING b/contrib/expected/COPYING
new file mode 100644
index 0000000..0e259d4
--- /dev/null
+++ b/contrib/expected/COPYING
@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+ INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+ REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+ PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+ THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+ HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+ i. the right to reproduce, adapt, distribute, perform, display,
+ communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+ likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+ subject to the limitations in paragraph 4(a), below;
+ v. rights protecting the extraction, dissemination, use and reuse of data
+ in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+ European Parliament and of the Council of 11 March 1996 on the legal
+ protection of databases, and under any national implementation
+ thereof, including any amended or successor version of such
+ directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+ world based on applicable law or treaty, and any national
+ implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+ surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+ warranties of any kind concerning the Work, express, implied,
+ statutory or otherwise, including without limitation warranties of
+ title, merchantability, fitness for a particular purpose, non
+ infringement, or the absence of latent or other defects, accuracy, or
+ the present or absence of errors, whether or not discoverable, all to
+ the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+ that may apply to the Work or any use thereof, including without
+ limitation any person's Copyright and Related Rights in the Work.
+ Further, Affirmer disclaims responsibility for obtaining any necessary
+ consents, permissions or other rights required for any use of the
+ Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+ party to this document and has no duty or obligation with respect to
+ this CC0 or use of the Work.
diff --git a/contrib/expected/expected.hpp b/contrib/expected/expected.hpp
new file mode 100644
index 0000000..31a5193
--- /dev/null
+++ b/contrib/expected/expected.hpp
@@ -0,0 +1,2326 @@
+///
+// expected - An implementation of std::expected with extensions
+// Written in 2017 by Simon Brand (simonrbrand@gmail.com, @TartanLlama)
+//
+// Documentation available at http://tl.tartanllama.xyz/
+//
+// To the extent possible under law, the author(s) have dedicated all
+// copyright and related and neighboring rights to this software to the
+// public domain worldwide. This software is distributed without any warranty.
+//
+// You should have received a copy of the CC0 Public Domain Dedication
+// along with this software. If not, see
+// <http://creativecommons.org/publicdomain/zero/1.0/>.
+///
+
+#ifndef TL_EXPECTED_HPP
+#define TL_EXPECTED_HPP
+
+#define TL_EXPECTED_VERSION_MAJOR 1
+#define TL_EXPECTED_VERSION_MINOR 0
+#define TL_EXPECTED_VERSION_PATCH 1
+
+#include <exception>
+#include <functional>
+#include <type_traits>
+#include <utility>
+
+#if defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+#define TL_EXPECTED_EXCEPTIONS_ENABLED
+#endif
+
+#if (defined(_MSC_VER) && _MSC_VER == 1900)
+#define TL_EXPECTED_MSVC2015
+#define TL_EXPECTED_MSVC2015_CONSTEXPR
+#else
+#define TL_EXPECTED_MSVC2015_CONSTEXPR constexpr
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \
+ !defined(__clang__))
+#define TL_EXPECTED_GCC49
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 4 && \
+ !defined(__clang__))
+#define TL_EXPECTED_GCC54
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 5 && \
+ !defined(__clang__))
+#define TL_EXPECTED_GCC55
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \
+ !defined(__clang__))
+// GCC < 5 doesn't support overloading on const&& for member functions
+
+#define TL_EXPECTED_NO_CONSTRR
+// GCC < 5 doesn't support some standard C++11 type traits
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \
+ std::has_trivial_copy_constructor<T>
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \
+ std::has_trivial_copy_assign<T>
+
+// This one will be different for GCC 5.7 if it's ever supported
+#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \
+ std::is_trivially_destructible<T>
+
+// GCC 5 < v < 8 has a bug in is_trivially_copy_constructible which breaks std::vector
+// for non-copyable types
+#elif (defined(__GNUC__) && __GNUC__ < 8 && \
+ !defined(__clang__))
+#ifndef TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX
+#define TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX
+namespace tl {
+ namespace detail {
+ template<class T>
+ struct is_trivially_copy_constructible : std::is_trivially_copy_constructible<T>{};
+#ifdef _GLIBCXX_VECTOR
+ template<class T, class A>
+ struct is_trivially_copy_constructible<std::vector<T,A>>
+ : std::false_type{};
+#endif
+ }
+}
+#endif
+
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \
+ tl::detail::is_trivially_copy_constructible<T>
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \
+ std::is_trivially_copy_assignable<T>
+#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) std::is_trivially_destructible<T>
+#else
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \
+ std::is_trivially_copy_constructible<T>
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \
+ std::is_trivially_copy_assignable<T>
+#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \
+ std::is_trivially_destructible<T>
+#endif
+
+#if __cplusplus > 201103L
+#define TL_EXPECTED_CXX14
+#endif
+
+#ifdef TL_EXPECTED_GCC49
+#define TL_EXPECTED_GCC49_CONSTEXPR
+#else
+#define TL_EXPECTED_GCC49_CONSTEXPR constexpr
+#endif
+
+#if (__cplusplus == 201103L || defined(TL_EXPECTED_MSVC2015) || \
+ defined(TL_EXPECTED_GCC49))
+#define TL_EXPECTED_11_CONSTEXPR
+#else
+#define TL_EXPECTED_11_CONSTEXPR constexpr
+#endif
+
+namespace tl {
+template <class T, class E> class expected;
+
+#ifndef TL_MONOSTATE_INPLACE_MUTEX
+#define TL_MONOSTATE_INPLACE_MUTEX
+class monostate {};
+
+struct in_place_t {
+ explicit in_place_t() = default;
+};
+static constexpr in_place_t in_place{};
+#endif
+
+template <class E> class unexpected {
+public:
+ static_assert(!std::is_same<E, void>::value, "E must not be void");
+
+ unexpected() = delete;
+ constexpr explicit unexpected(const E &e) : m_val(e) {}
+
+ constexpr explicit unexpected(E &&e) : m_val(std::move(e)) {}
+
+ constexpr const E &value() const & { return m_val; }
+ TL_EXPECTED_11_CONSTEXPR E &value() & { return m_val; }
+ TL_EXPECTED_11_CONSTEXPR E &&value() && { return std::move(m_val); }
+ constexpr const E &&value() const && { return std::move(m_val); }
+
+private:
+ E m_val;
+};
+
+template <class E>
+constexpr bool operator==(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() == rhs.value();
+}
+template <class E>
+constexpr bool operator!=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() != rhs.value();
+}
+template <class E>
+constexpr bool operator<(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() < rhs.value();
+}
+template <class E>
+constexpr bool operator<=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() <= rhs.value();
+}
+template <class E>
+constexpr bool operator>(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() > rhs.value();
+}
+template <class E>
+constexpr bool operator>=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() >= rhs.value();
+}
+
+template <class E>
+unexpected<typename std::decay<E>::type> make_unexpected(E &&e) {
+ return unexpected<typename std::decay<E>::type>(std::forward<E>(e));
+}
+
+struct unexpect_t {
+ unexpect_t() = default;
+};
+static constexpr unexpect_t unexpect{};
+
+namespace detail {
+template<typename E>
+[[noreturn]] TL_EXPECTED_11_CONSTEXPR void throw_exception(E &&e) {
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ throw std::forward<E>(e);
+#else
+ #ifdef _MSC_VER
+ __assume(0);
+ #else
+ __builtin_unreachable();
+ #endif
+#endif
+}
+
+#ifndef TL_TRAITS_MUTEX
+#define TL_TRAITS_MUTEX
+// C++14-style aliases for brevity
+template <class T> using remove_const_t = typename std::remove_const<T>::type;
+template <class T>
+using remove_reference_t = typename std::remove_reference<T>::type;
+template <class T> using decay_t = typename std::decay<T>::type;
+template <bool E, class T = void>
+using enable_if_t = typename std::enable_if<E, T>::type;
+template <bool B, class T, class F>
+using conditional_t = typename std::conditional<B, T, F>::type;
+
+// std::conjunction from C++17
+template <class...> struct conjunction : std::true_type {};
+template <class B> struct conjunction<B> : B {};
+template <class B, class... Bs>
+struct conjunction<B, Bs...>
+ : std::conditional<bool(B::value), conjunction<Bs...>, B>::type {};
+
+#if defined(_LIBCPP_VERSION) && __cplusplus == 201103L
+#define TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND
+#endif
+
+// In C++11 mode, there's an issue in libc++'s std::mem_fn
+// which results in a hard-error when using it in a noexcept expression
+// in some cases. This is a check to workaround the common failing case.
+#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND
+template <class T> struct is_pointer_to_non_const_member_func : std::false_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...)> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...)&> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) &&> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) volatile> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) volatile &> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) volatile &&> : std::true_type {};
+
+template <class T> struct is_const_or_const_ref : std::false_type {};
+template <class T> struct is_const_or_const_ref<T const&> : std::true_type {};
+template <class T> struct is_const_or_const_ref<T const> : std::true_type {};
+#endif
+
+// std::invoke from C++17
+// https://stackoverflow.com/questions/38288042/c11-14-invoke-workaround
+template <typename Fn, typename... Args,
+#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND
+ typename = enable_if_t<!(is_pointer_to_non_const_member_func<Fn>::value
+ && is_const_or_const_ref<Args...>::value)>,
+#endif
+ typename = enable_if_t<std::is_member_pointer<decay_t<Fn>>::value>,
+ int = 0>
+ constexpr auto invoke(Fn && f, Args && ... args) noexcept(
+ noexcept(std::mem_fn(f)(std::forward<Args>(args)...)))
+ -> decltype(std::mem_fn(f)(std::forward<Args>(args)...)) {
+ return std::mem_fn(f)(std::forward<Args>(args)...);
+}
+
+template <typename Fn, typename... Args,
+ typename = enable_if_t<!std::is_member_pointer<decay_t<Fn>>::value>>
+ constexpr auto invoke(Fn && f, Args && ... args) noexcept(
+ noexcept(std::forward<Fn>(f)(std::forward<Args>(args)...)))
+ -> decltype(std::forward<Fn>(f)(std::forward<Args>(args)...)) {
+ return std::forward<Fn>(f)(std::forward<Args>(args)...);
+}
+
+// std::invoke_result from C++17
+template <class F, class, class... Us> struct invoke_result_impl;
+
+template <class F, class... Us>
+struct invoke_result_impl<
+ F, decltype(detail::invoke(std::declval<F>(), std::declval<Us>()...), void()),
+ Us...> {
+ using type = decltype(detail::invoke(std::declval<F>(), std::declval<Us>()...));
+};
+
+template <class F, class... Us>
+using invoke_result = invoke_result_impl<F, void, Us...>;
+
+template <class F, class... Us>
+using invoke_result_t = typename invoke_result<F, Us...>::type;
+
+#if defined(_MSC_VER) && _MSC_VER <= 1900
+// TODO make a version which works with MSVC 2015
+template <class T, class U = T> struct is_swappable : std::true_type {};
+
+template <class T, class U = T> struct is_nothrow_swappable : std::true_type {};
+#else
+// https://stackoverflow.com/questions/26744589/what-is-a-proper-way-to-implement-is-swappable-to-test-for-the-swappable-concept
+namespace swap_adl_tests {
+ // if swap ADL finds this then it would call std::swap otherwise (same
+ // signature)
+ struct tag {};
+
+ template <class T> tag swap(T&, T&);
+ template <class T, std::size_t N> tag swap(T(&a)[N], T(&b)[N]);
+
+ // helper functions to test if an unqualified swap is possible, and if it
+ // becomes std::swap
+ template <class, class> std::false_type can_swap(...) noexcept(false);
+ template <class T, class U,
+ class = decltype(swap(std::declval<T&>(), std::declval<U&>()))>
+ std::true_type can_swap(int) noexcept(noexcept(swap(std::declval<T&>(),
+ std::declval<U&>())));
+
+ template <class, class> std::false_type uses_std(...);
+ template <class T, class U>
+ std::is_same<decltype(swap(std::declval<T&>(), std::declval<U&>())), tag>
+ uses_std(int);
+
+ template <class T>
+ struct is_std_swap_noexcept
+ : std::integral_constant<bool,
+ std::is_nothrow_move_constructible<T>::value&&
+ std::is_nothrow_move_assignable<T>::value> {};
+
+ template <class T, std::size_t N>
+ struct is_std_swap_noexcept<T[N]> : is_std_swap_noexcept<T> {};
+
+ template <class T, class U>
+ struct is_adl_swap_noexcept
+ : std::integral_constant<bool, noexcept(can_swap<T, U>(0))> {};
+} // namespace swap_adl_tests
+
+template <class T, class U = T>
+struct is_swappable
+ : std::integral_constant<
+ bool,
+ decltype(detail::swap_adl_tests::can_swap<T, U>(0))::value &&
+ (!decltype(detail::swap_adl_tests::uses_std<T, U>(0))::value ||
+ (std::is_move_assignable<T>::value &&
+ std::is_move_constructible<T>::value))> {};
+
+template <class T, std::size_t N>
+struct is_swappable<T[N], T[N]>
+ : std::integral_constant<
+ bool,
+ decltype(detail::swap_adl_tests::can_swap<T[N], T[N]>(0))::value &&
+ (!decltype(
+ detail::swap_adl_tests::uses_std<T[N], T[N]>(0))::value ||
+ is_swappable<T, T>::value)> {};
+
+template <class T, class U = T>
+struct is_nothrow_swappable
+ : std::integral_constant<
+ bool,
+ is_swappable<T, U>::value &&
+ ((decltype(detail::swap_adl_tests::uses_std<T, U>(0))::value
+ && detail::swap_adl_tests::is_std_swap_noexcept<T>::value) ||
+ (!decltype(detail::swap_adl_tests::uses_std<T, U>(0))::value &&
+ detail::swap_adl_tests::is_adl_swap_noexcept<T,
+ U>::value))> {
+};
+#endif
+#endif
+
+// Trait for checking if a type is a tl::expected
+template <class T> struct is_expected_impl : std::false_type {};
+template <class T, class E>
+struct is_expected_impl<expected<T, E>> : std::true_type {};
+template <class T> using is_expected = is_expected_impl<decay_t<T>>;
+
+template <class T, class E, class U>
+using expected_enable_forward_value = detail::enable_if_t<
+ std::is_constructible<T, U &&>::value &&
+ !std::is_same<detail::decay_t<U>, in_place_t>::value &&
+ !std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
+ !std::is_same<unexpected<E>, detail::decay_t<U>>::value>;
+
+template <class T, class E, class U, class G, class UR, class GR>
+using expected_enable_from_other = detail::enable_if_t<
+ std::is_constructible<T, UR>::value &&
+ std::is_constructible<E, GR>::value &&
+ !std::is_constructible<T, expected<U, G> &>::value &&
+ !std::is_constructible<T, expected<U, G> &&>::value &&
+ !std::is_constructible<T, const expected<U, G> &>::value &&
+ !std::is_constructible<T, const expected<U, G> &&>::value &&
+ !std::is_convertible<expected<U, G> &, T>::value &&
+ !std::is_convertible<expected<U, G> &&, T>::value &&
+ !std::is_convertible<const expected<U, G> &, T>::value &&
+ !std::is_convertible<const expected<U, G> &&, T>::value>;
+
+template <class T, class U>
+using is_void_or = conditional_t<std::is_void<T>::value, std::true_type, U>;
+
+template <class T>
+using is_copy_constructible_or_void =
+ is_void_or<T, std::is_copy_constructible<T>>;
+
+template <class T>
+using is_move_constructible_or_void =
+ is_void_or<T, std::is_move_constructible<T>>;
+
+template <class T>
+using is_copy_assignable_or_void =
+ is_void_or<T, std::is_copy_assignable<T>>;
+
+
+template <class T>
+using is_move_assignable_or_void =
+ is_void_or<T, std::is_move_assignable<T>>;
+
+
+} // namespace detail
+
+namespace detail {
+struct no_init_t {};
+static constexpr no_init_t no_init{};
+
+// Implements the storage of the values, and ensures that the destructor is
+// trivial if it can be.
+//
+// This specialization is for where neither `T` or `E` is trivially
+// destructible, so the destructors must be called on destruction of the
+// `expected`
+template <class T, class E, bool = std::is_trivially_destructible<T>::value,
+ bool = std::is_trivially_destructible<E>::value>
+struct expected_storage_base {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (m_has_val) {
+ m_val.~T();
+ } else {
+ m_unexpect.~unexpected<E>();
+ }
+ }
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// This specialization is for when both `T` and `E` are trivially-destructible,
+// so the destructor of the `expected` can be trivial.
+template <class T, class E> struct expected_storage_base<T, E, true, true> {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() = default;
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// T is trivial, E is not.
+template <class T, class E> struct expected_storage_base<T, E, true, false> {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base(no_init_t)
+ : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (!m_has_val) {
+ m_unexpect.~unexpected<E>();
+ }
+ }
+
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// E is trivial, T is not.
+template <class T, class E> struct expected_storage_base<T, E, false, true> {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (m_has_val) {
+ m_val.~T();
+ }
+ }
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// `T` is `void`, `E` is trivially-destructible
+template <class E> struct expected_storage_base<void, E, false, true> {
+ TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base() : m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_val(), m_has_val(false) {}
+
+ constexpr expected_storage_base(in_place_t) : m_has_val(true) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() = default;
+ struct dummy {};
+ union {
+ unexpected<E> m_unexpect;
+ dummy m_val;
+ };
+ bool m_has_val;
+};
+
+// `T` is `void`, `E` is not trivially-destructible
+template <class E> struct expected_storage_base<void, E, false, false> {
+ constexpr expected_storage_base() : m_dummy(), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_dummy(), m_has_val(false) {}
+
+ constexpr expected_storage_base(in_place_t) : m_dummy(), m_has_val(true) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (!m_has_val) {
+ m_unexpect.~unexpected<E>();
+ }
+ }
+
+ union {
+ unexpected<E> m_unexpect;
+ char m_dummy;
+ };
+ bool m_has_val;
+};
+
+// This base class provides some handy member functions which can be used in
+// further derived classes
+template <class T, class E>
+struct expected_operations_base : expected_storage_base<T, E> {
+ using expected_storage_base<T, E>::expected_storage_base;
+
+ template <class... Args> void construct(Args &&... args) noexcept {
+ new (std::addressof(this->m_val)) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ }
+
+ template <class Rhs> void construct_with(Rhs &&rhs) noexcept {
+ new (std::addressof(this->m_val)) T(std::forward<Rhs>(rhs).get());
+ this->m_has_val = true;
+ }
+
+ template <class... Args> void construct_error(Args &&... args) noexcept {
+ new (std::addressof(this->m_unexpect))
+ unexpected<E>(std::forward<Args>(args)...);
+ this->m_has_val = false;
+ }
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+
+ // These assign overloads ensure that the most efficient assignment
+ // implementation is used while maintaining the strong exception guarantee.
+ // The problematic case is where rhs has a value, but *this does not.
+ //
+ // This overload handles the case where we can just copy-construct `T`
+ // directly into place without throwing.
+ template <class U = T,
+ detail::enable_if_t<std::is_nothrow_copy_constructible<U>::value>
+ * = nullptr>
+ void assign(const expected_operations_base &rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(rhs.get());
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ // This overload handles the case where we can attempt to create a copy of
+ // `T`, then no-throw move it into place if the copy was successful.
+ template <class U = T,
+ detail::enable_if_t<!std::is_nothrow_copy_constructible<U>::value &&
+ std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(const expected_operations_base &rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ T tmp = rhs.get();
+ geterr().~unexpected<E>();
+ construct(std::move(tmp));
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ // This overload is the worst-case, where we have to move-construct the
+ // unexpected value into temporary storage, then try to copy the T into place.
+ // If the construction succeeds, then everything is fine, but if it throws,
+ // then we move the old unexpected value back into place before rethrowing the
+ // exception.
+ template <class U = T,
+ detail::enable_if_t<!std::is_nothrow_copy_constructible<U>::value &&
+ !std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(const expected_operations_base &rhs) {
+ if (!this->m_has_val && rhs.m_has_val) {
+ auto tmp = std::move(geterr());
+ geterr().~unexpected<E>();
+
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ construct(rhs.get());
+ } catch (...) {
+ geterr() = std::move(tmp);
+ throw;
+ }
+#else
+ construct(rhs.get());
+#endif
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ // These overloads do the same as above, but for rvalues
+ template <class U = T,
+ detail::enable_if_t<std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(expected_operations_base &&rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(std::move(rhs).get());
+ } else {
+ assign_common(std::move(rhs));
+ }
+ }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(expected_operations_base &&rhs) {
+ if (!this->m_has_val && rhs.m_has_val) {
+ auto tmp = std::move(geterr());
+ geterr().~unexpected<E>();
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ construct(std::move(rhs).get());
+ } catch (...) {
+ geterr() = std::move(tmp);
+ throw;
+ }
+#else
+ construct(std::move(rhs).get());
+#endif
+ } else {
+ assign_common(std::move(rhs));
+ }
+ }
+
+ #else
+
+ // If exceptions are disabled then we can just copy-construct
+ void assign(const expected_operations_base &rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(rhs.get());
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ void assign(expected_operations_base &&rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(std::move(rhs).get());
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ #endif
+
+ // The common part of move/copy assigning
+ template <class Rhs> void assign_common(Rhs &&rhs) {
+ if (this->m_has_val) {
+ if (rhs.m_has_val) {
+ get() = std::forward<Rhs>(rhs).get();
+ } else {
+ destroy_val();
+ construct_error(std::forward<Rhs>(rhs).geterr());
+ }
+ } else {
+ if (!rhs.m_has_val) {
+ geterr() = std::forward<Rhs>(rhs).geterr();
+ }
+ }
+ }
+
+ bool has_value() const { return this->m_has_val; }
+
+ TL_EXPECTED_11_CONSTEXPR T &get() & { return this->m_val; }
+ constexpr const T &get() const & { return this->m_val; }
+ TL_EXPECTED_11_CONSTEXPR T &&get() && { return std::move(this->m_val); }
+#ifndef TL_EXPECTED_NO_CONSTRR
+ constexpr const T &&get() const && { return std::move(this->m_val); }
+#endif
+
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &geterr() & {
+ return this->m_unexpect;
+ }
+ constexpr const unexpected<E> &geterr() const & { return this->m_unexpect; }
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &&geterr() && {
+ return std::move(this->m_unexpect);
+ }
+#ifndef TL_EXPECTED_NO_CONSTRR
+ constexpr const unexpected<E> &&geterr() const && {
+ return std::move(this->m_unexpect);
+ }
+#endif
+
+ TL_EXPECTED_11_CONSTEXPR void destroy_val() {
+ get().~T();
+ }
+};
+
+// This base class provides some handy member functions which can be used in
+// further derived classes
+template <class E>
+struct expected_operations_base<void, E> : expected_storage_base<void, E> {
+ using expected_storage_base<void, E>::expected_storage_base;
+
+ template <class... Args> void construct() noexcept { this->m_has_val = true; }
+
+ // This function doesn't use its argument, but needs it so that code in
+ // levels above this can work independently of whether T is void
+ template <class Rhs> void construct_with(Rhs &&) noexcept {
+ this->m_has_val = true;
+ }
+
+ template <class... Args> void construct_error(Args &&... args) noexcept {
+ new (std::addressof(this->m_unexpect))
+ unexpected<E>(std::forward<Args>(args)...);
+ this->m_has_val = false;
+ }
+
+ template <class Rhs> void assign(Rhs &&rhs) noexcept {
+ if (!this->m_has_val) {
+ if (rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct();
+ } else {
+ geterr() = std::forward<Rhs>(rhs).geterr();
+ }
+ } else {
+ if (!rhs.m_has_val) {
+ construct_error(std::forward<Rhs>(rhs).geterr());
+ }
+ }
+ }
+
+ bool has_value() const { return this->m_has_val; }
+
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &geterr() & {
+ return this->m_unexpect;
+ }
+ constexpr const unexpected<E> &geterr() const & { return this->m_unexpect; }
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &&geterr() && {
+ return std::move(this->m_unexpect);
+ }
+#ifndef TL_EXPECTED_NO_CONSTRR
+ constexpr const unexpected<E> &&geterr() const && {
+ return std::move(this->m_unexpect);
+ }
+#endif
+
+ TL_EXPECTED_11_CONSTEXPR void destroy_val() {
+ //no-op
+ }
+};
+
+// This class manages conditionally having a trivial copy constructor
+// This specialization is for when T and E are trivially copy constructible
+template <class T, class E,
+ bool = is_void_or<T, TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T)>::
+ value &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value>
+struct expected_copy_base : expected_operations_base<T, E> {
+ using expected_operations_base<T, E>::expected_operations_base;
+};
+
+// This specialization is for when T or E are not trivially copy constructible
+template <class T, class E>
+struct expected_copy_base<T, E, false> : expected_operations_base<T, E> {
+ using expected_operations_base<T, E>::expected_operations_base;
+
+ expected_copy_base() = default;
+ expected_copy_base(const expected_copy_base &rhs)
+ : expected_operations_base<T, E>(no_init) {
+ if (rhs.has_value()) {
+ this->construct_with(rhs);
+ } else {
+ this->construct_error(rhs.geterr());
+ }
+ }
+
+ expected_copy_base(expected_copy_base &&rhs) = default;
+ expected_copy_base &operator=(const expected_copy_base &rhs) = default;
+ expected_copy_base &operator=(expected_copy_base &&rhs) = default;
+};
+
+// This class manages conditionally having a trivial move constructor
+// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it
+// doesn't implement an analogue to std::is_trivially_move_constructible. We
+// have to make do with a non-trivial move constructor even if T is trivially
+// move constructible
+#ifndef TL_EXPECTED_GCC49
+template <class T, class E,
+ bool = is_void_or<T, std::is_trivially_move_constructible<T>>::value
+ &&std::is_trivially_move_constructible<E>::value>
+struct expected_move_base : expected_copy_base<T, E> {
+ using expected_copy_base<T, E>::expected_copy_base;
+};
+#else
+template <class T, class E, bool = false> struct expected_move_base;
+#endif
+template <class T, class E>
+struct expected_move_base<T, E, false> : expected_copy_base<T, E> {
+ using expected_copy_base<T, E>::expected_copy_base;
+
+ expected_move_base() = default;
+ expected_move_base(const expected_move_base &rhs) = default;
+
+ expected_move_base(expected_move_base &&rhs) noexcept(
+ std::is_nothrow_move_constructible<T>::value)
+ : expected_copy_base<T, E>(no_init) {
+ if (rhs.has_value()) {
+ this->construct_with(std::move(rhs));
+ } else {
+ this->construct_error(std::move(rhs.geterr()));
+ }
+ }
+ expected_move_base &operator=(const expected_move_base &rhs) = default;
+ expected_move_base &operator=(expected_move_base &&rhs) = default;
+};
+
+// This class manages conditionally having a trivial copy assignment operator
+template <class T, class E,
+ bool = is_void_or<
+ T, conjunction<TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T),
+ TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T),
+ TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T)>>::value
+ &&TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(E)::value
+ &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value
+ &&TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(E)::value>
+struct expected_copy_assign_base : expected_move_base<T, E> {
+ using expected_move_base<T, E>::expected_move_base;
+};
+
+template <class T, class E>
+struct expected_copy_assign_base<T, E, false> : expected_move_base<T, E> {
+ using expected_move_base<T, E>::expected_move_base;
+
+ expected_copy_assign_base() = default;
+ expected_copy_assign_base(const expected_copy_assign_base &rhs) = default;
+
+ expected_copy_assign_base(expected_copy_assign_base &&rhs) = default;
+ expected_copy_assign_base &operator=(const expected_copy_assign_base &rhs) {
+ this->assign(rhs);
+ return *this;
+ }
+ expected_copy_assign_base &
+ operator=(expected_copy_assign_base &&rhs) = default;
+};
+
+// This class manages conditionally having a trivial move assignment operator
+// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it
+// doesn't implement an analogue to std::is_trivially_move_assignable. We have
+// to make do with a non-trivial move assignment operator even if T is trivially
+// move assignable
+#ifndef TL_EXPECTED_GCC49
+template <class T, class E,
+ bool =
+ is_void_or<T, conjunction<std::is_trivially_destructible<T>,
+ std::is_trivially_move_constructible<T>,
+ std::is_trivially_move_assignable<T>>>::
+ value &&std::is_trivially_destructible<E>::value
+ &&std::is_trivially_move_constructible<E>::value
+ &&std::is_trivially_move_assignable<E>::value>
+struct expected_move_assign_base : expected_copy_assign_base<T, E> {
+ using expected_copy_assign_base<T, E>::expected_copy_assign_base;
+};
+#else
+template <class T, class E, bool = false> struct expected_move_assign_base;
+#endif
+
+template <class T, class E>
+struct expected_move_assign_base<T, E, false>
+ : expected_copy_assign_base<T, E> {
+ using expected_copy_assign_base<T, E>::expected_copy_assign_base;
+
+ expected_move_assign_base() = default;
+ expected_move_assign_base(const expected_move_assign_base &rhs) = default;
+
+ expected_move_assign_base(expected_move_assign_base &&rhs) = default;
+
+ expected_move_assign_base &
+ operator=(const expected_move_assign_base &rhs) = default;
+
+ expected_move_assign_base &
+ operator=(expected_move_assign_base &&rhs) noexcept(
+ std::is_nothrow_move_constructible<T>::value
+ &&std::is_nothrow_move_assignable<T>::value) {
+ this->assign(std::move(rhs));
+ return *this;
+ }
+};
+
+// expected_delete_ctor_base will conditionally delete copy and move
+// constructors depending on whether T is copy/move constructible
+template <class T, class E,
+ bool EnableCopy = (is_copy_constructible_or_void<T>::value &&
+ std::is_copy_constructible<E>::value),
+ bool EnableMove = (is_move_constructible_or_void<T>::value &&
+ std::is_move_constructible<E>::value)>
+struct expected_delete_ctor_base {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_ctor_base<T, E, true, false> {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_ctor_base<T, E, false, true> {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = delete;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_ctor_base<T, E, false, false> {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = delete;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+// expected_delete_assign_base will conditionally delete copy and move
+// constructors depending on whether T and E are copy/move constructible +
+// assignable
+template <class T, class E,
+ bool EnableCopy = (is_copy_constructible_or_void<T>::value &&
+ std::is_copy_constructible<E>::value &&
+ is_copy_assignable_or_void<T>::value &&
+ std::is_copy_assignable<E>::value),
+ bool EnableMove = (is_move_constructible_or_void<T>::value &&
+ std::is_move_constructible<E>::value &&
+ is_move_assignable_or_void<T>::value &&
+ std::is_move_assignable<E>::value)>
+struct expected_delete_assign_base {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_assign_base<T, E, true, false> {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = delete;
+};
+
+template <class T, class E>
+struct expected_delete_assign_base<T, E, false, true> {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = delete;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_assign_base<T, E, false, false> {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = delete;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = delete;
+};
+
+// This is needed to be able to construct the expected_default_ctor_base which
+// follows, while still conditionally deleting the default constructor.
+struct default_constructor_tag {
+ explicit constexpr default_constructor_tag() = default;
+};
+
+// expected_default_ctor_base will ensure that expected has a deleted default
+// consturctor if T is not default constructible.
+// This specialization is for when T is default constructible
+template <class T, class E,
+ bool Enable =
+ std::is_default_constructible<T>::value || std::is_void<T>::value>
+struct expected_default_ctor_base {
+ constexpr expected_default_ctor_base() noexcept = default;
+ constexpr expected_default_ctor_base(
+ expected_default_ctor_base const &) noexcept = default;
+ constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept =
+ default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base const &) noexcept = default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base &&) noexcept = default;
+
+ constexpr explicit expected_default_ctor_base(default_constructor_tag) {}
+};
+
+// This specialization is for when T is not default constructible
+template <class T, class E> struct expected_default_ctor_base<T, E, false> {
+ constexpr expected_default_ctor_base() noexcept = delete;
+ constexpr expected_default_ctor_base(
+ expected_default_ctor_base const &) noexcept = default;
+ constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept =
+ default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base const &) noexcept = default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base &&) noexcept = default;
+
+ constexpr explicit expected_default_ctor_base(default_constructor_tag) {}
+};
+} // namespace detail
+
+template <class E> class bad_expected_access : public std::exception {
+public:
+ explicit bad_expected_access(E e) : m_val(std::move(e)) {}
+
+ virtual const char *what() const noexcept override {
+ return "Bad expected access";
+ }
+
+ const E &error() const & { return m_val; }
+ E &error() & { return m_val; }
+ const E &&error() const && { return std::move(m_val); }
+ E &&error() && { return std::move(m_val); }
+
+private:
+ E m_val;
+};
+
+/// An `expected<T, E>` object is an object that contains the storage for
+/// another object and manages the lifetime of this contained object `T`.
+/// Alternatively it could contain the storage for another unexpected object
+/// `E`. The contained object may not be initialized after the expected object
+/// has been initialized, and may not be destroyed before the expected object
+/// has been destroyed. The initialization state of the contained object is
+/// tracked by the expected object.
+template <class T, class E>
+class expected : private detail::expected_move_assign_base<T, E>,
+ private detail::expected_delete_ctor_base<T, E>,
+ private detail::expected_delete_assign_base<T, E>,
+ private detail::expected_default_ctor_base<T, E> {
+ static_assert(!std::is_reference<T>::value, "T must not be a reference");
+ static_assert(!std::is_same<T, std::remove_cv<in_place_t>>::value,
+ "T must not be in_place_t");
+ static_assert(!std::is_same<T, std::remove_cv<unexpect_t>>::value,
+ "T must not be unexpect_t");
+ static_assert(!std::is_same<T, std::remove_cv<unexpected<E>>>::value,
+ "T must not be unexpected<E>");
+ static_assert(!std::is_reference<E>::value, "E must not be a reference");
+
+ T *valptr() { return std::addressof(this->m_val); }
+ const T *valptr() const { return std::addressof(this->m_val); }
+ unexpected<E> *errptr() { return std::addressof(this->m_unexpect); }
+ const unexpected<E> *errptr() const { return std::addressof(this->m_unexpect); }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &val() {
+ return this->m_val;
+ }
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &err() { return this->m_unexpect; }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ constexpr const U &val() const {
+ return this->m_val;
+ }
+ constexpr const unexpected<E> &err() const { return this->m_unexpect; }
+
+ using impl_base = detail::expected_move_assign_base<T, E>;
+ using ctor_base = detail::expected_default_ctor_base<T, E>;
+
+public:
+ typedef T value_type;
+ typedef E error_type;
+ typedef unexpected<E> unexpected_type;
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) & {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto and_then(F &&f) const & {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F> constexpr auto and_then(F &&f) const && {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR auto
+ and_then(F &&f) & -> decltype(and_then_impl(std::declval<expected&>(), std::forward<F>(f))) {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && -> decltype(
+ and_then_impl(std::declval<expected&&>(), std::forward<F>(f))) {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr auto and_then(F &&f) const & -> decltype(
+ and_then_impl(std::declval<expected const&>(), std::forward<F>(f))) {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr auto and_then(F &&f) const && -> decltype(
+ and_then_impl(std::declval<expected const&&>(), std::forward<F>(f))) {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto map(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> constexpr auto map(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected &>(), std::declval<F &&>()))
+ map(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected>(), std::declval<F &&>()))
+ map(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &>(),
+ std::declval<F &&>()))
+ map(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &&>(),
+ std::declval<F &&>()))
+ map(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto transform(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> constexpr auto transform(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected &>(), std::declval<F &&>()))
+ transform(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected>(), std::declval<F &&>()))
+ transform(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &>(),
+ std::declval<F &&>()))
+ transform(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &&>(),
+ std::declval<F &&>()))
+ transform(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto map_error(F &&f) const & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+ template <class F> constexpr auto map_error(F &&f) const && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval<expected &>(),
+ std::declval<F &&>()))
+ map_error(F &&f) & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval<expected &&>(),
+ std::declval<F &&>()))
+ map_error(F &&f) && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr decltype(map_error_impl(std::declval<const expected &>(),
+ std::declval<F &&>()))
+ map_error(F &&f) const & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr decltype(map_error_impl(std::declval<const expected &&>(),
+ std::declval<F &&>()))
+ map_error(F &&f) const && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+ template <class F> expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) & {
+ return or_else_impl(*this, std::forward<F>(f));
+ }
+
+ template <class F> expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) && {
+ return or_else_impl(std::move(*this), std::forward<F>(f));
+ }
+
+ template <class F> expected constexpr or_else(F &&f) const & {
+ return or_else_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F> expected constexpr or_else(F &&f) const && {
+ return or_else_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+ constexpr expected() = default;
+ constexpr expected(const expected &rhs) = default;
+ constexpr expected(expected &&rhs) = default;
+ expected &operator=(const expected &rhs) = default;
+ expected &operator=(expected &&rhs) = default;
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected(in_place_t, Args &&... args)
+ : impl_base(in_place, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected(in_place_t, std::initializer_list<U> il, Args &&... args)
+ : impl_base(in_place, il, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class G = E,
+ detail::enable_if_t<std::is_constructible<E, const G &>::value> * =
+ nullptr,
+ detail::enable_if_t<!std::is_convertible<const G &, E>::value> * =
+ nullptr>
+ explicit constexpr expected(const unexpected<G> &e)
+ : impl_base(unexpect, e.value()),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <
+ class G = E,
+ detail::enable_if_t<std::is_constructible<E, const G &>::value> * =
+ nullptr,
+ detail::enable_if_t<std::is_convertible<const G &, E>::value> * = nullptr>
+ constexpr expected(unexpected<G> const &e)
+ : impl_base(unexpect, e.value()),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <
+ class G = E,
+ detail::enable_if_t<std::is_constructible<E, G &&>::value> * = nullptr,
+ detail::enable_if_t<!std::is_convertible<G &&, E>::value> * = nullptr>
+ explicit constexpr expected(unexpected<G> &&e) noexcept(
+ std::is_nothrow_constructible<E, G &&>::value)
+ : impl_base(unexpect, std::move(e.value())),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <
+ class G = E,
+ detail::enable_if_t<std::is_constructible<E, G &&>::value> * = nullptr,
+ detail::enable_if_t<std::is_convertible<G &&, E>::value> * = nullptr>
+ constexpr expected(unexpected<G> &&e) noexcept(
+ std::is_nothrow_constructible<E, G &&>::value)
+ : impl_base(unexpect, std::move(e.value())),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected(unexpect_t, Args &&... args)
+ : impl_base(unexpect, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected(unexpect_t, std::initializer_list<U> il,
+ Args &&... args)
+ : impl_base(unexpect, il, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class U, class G,
+ detail::enable_if_t<!(std::is_convertible<U const &, T>::value &&
+ std::is_convertible<G const &, E>::value)> * =
+ nullptr,
+ detail::expected_enable_from_other<T, E, U, G, const U &, const G &>
+ * = nullptr>
+ explicit TL_EXPECTED_11_CONSTEXPR expected(const expected<U, G> &rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(*rhs);
+ } else {
+ this->construct_error(rhs.error());
+ }
+ }
+
+ template <class U, class G,
+ detail::enable_if_t<(std::is_convertible<U const &, T>::value &&
+ std::is_convertible<G const &, E>::value)> * =
+ nullptr,
+ detail::expected_enable_from_other<T, E, U, G, const U &, const G &>
+ * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR expected(const expected<U, G> &rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(*rhs);
+ } else {
+ this->construct_error(rhs.error());
+ }
+ }
+
+ template <
+ class U, class G,
+ detail::enable_if_t<!(std::is_convertible<U &&, T>::value &&
+ std::is_convertible<G &&, E>::value)> * = nullptr,
+ detail::expected_enable_from_other<T, E, U, G, U &&, G &&> * = nullptr>
+ explicit TL_EXPECTED_11_CONSTEXPR expected(expected<U, G> &&rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(std::move(*rhs));
+ } else {
+ this->construct_error(std::move(rhs.error()));
+ }
+ }
+
+ template <
+ class U, class G,
+ detail::enable_if_t<(std::is_convertible<U &&, T>::value &&
+ std::is_convertible<G &&, E>::value)> * = nullptr,
+ detail::expected_enable_from_other<T, E, U, G, U &&, G &&> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR expected(expected<U, G> &&rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(std::move(*rhs));
+ } else {
+ this->construct_error(std::move(rhs.error()));
+ }
+ }
+
+ template <
+ class U = T,
+ detail::enable_if_t<!std::is_convertible<U &&, T>::value> * = nullptr,
+ detail::expected_enable_forward_value<T, E, U> * = nullptr>
+ explicit TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v)
+ : expected(in_place, std::forward<U>(v)) {}
+
+ template <
+ class U = T,
+ detail::enable_if_t<std::is_convertible<U &&, T>::value> * = nullptr,
+ detail::expected_enable_forward_value<T, E, U> * = nullptr>
+ TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v)
+ : expected(in_place, std::forward<U>(v)) {}
+
+ template <
+ class U = T, class G = T,
+ detail::enable_if_t<std::is_nothrow_constructible<T, U &&>::value> * =
+ nullptr,
+ detail::enable_if_t<!std::is_void<G>::value> * = nullptr,
+ detail::enable_if_t<
+ (!std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
+ !detail::conjunction<std::is_scalar<T>,
+ std::is_same<T, detail::decay_t<U>>>::value &&
+ std::is_constructible<T, U>::value &&
+ std::is_assignable<G &, U>::value &&
+ std::is_nothrow_move_constructible<E>::value)> * = nullptr>
+ expected &operator=(U &&v) {
+ if (has_value()) {
+ val() = std::forward<U>(v);
+ } else {
+ err().~unexpected<E>();
+ ::new (valptr()) T(std::forward<U>(v));
+ this->m_has_val = true;
+ }
+
+ return *this;
+ }
+
+ template <
+ class U = T, class G = T,
+ detail::enable_if_t<!std::is_nothrow_constructible<T, U &&>::value> * =
+ nullptr,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr,
+ detail::enable_if_t<
+ (!std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
+ !detail::conjunction<std::is_scalar<T>,
+ std::is_same<T, detail::decay_t<U>>>::value &&
+ std::is_constructible<T, U>::value &&
+ std::is_assignable<G &, U>::value &&
+ std::is_nothrow_move_constructible<E>::value)> * = nullptr>
+ expected &operator=(U &&v) {
+ if (has_value()) {
+ val() = std::forward<U>(v);
+ } else {
+ auto tmp = std::move(err());
+ err().~unexpected<E>();
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (valptr()) T(std::forward<U>(v));
+ this->m_has_val = true;
+ } catch (...) {
+ err() = std::move(tmp);
+ throw;
+ }
+ #else
+ ::new (valptr()) T(std::forward<U>(v));
+ this->m_has_val = true;
+ #endif
+ }
+
+ return *this;
+ }
+
+ template <class G = E,
+ detail::enable_if_t<std::is_nothrow_copy_constructible<G>::value &&
+ std::is_assignable<G &, G>::value> * = nullptr>
+ expected &operator=(const unexpected<G> &rhs) {
+ if (!has_value()) {
+ err() = rhs;
+ } else {
+ this->destroy_val();
+ ::new (errptr()) unexpected<E>(rhs);
+ this->m_has_val = false;
+ }
+
+ return *this;
+ }
+
+ template <class G = E,
+ detail::enable_if_t<std::is_nothrow_move_constructible<G>::value &&
+ std::is_move_assignable<G>::value> * = nullptr>
+ expected &operator=(unexpected<G> &&rhs) noexcept {
+ if (!has_value()) {
+ err() = std::move(rhs);
+ } else {
+ this->destroy_val();
+ ::new (errptr()) unexpected<E>(std::move(rhs));
+ this->m_has_val = false;
+ }
+
+ return *this;
+ }
+
+ template <class... Args, detail::enable_if_t<std::is_nothrow_constructible<
+ T, Args &&...>::value> * = nullptr>
+ void emplace(Args &&... args) {
+ if (has_value()) {
+ val() = T(std::forward<Args>(args)...);
+ } else {
+ err().~unexpected<E>();
+ ::new (valptr()) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ }
+ }
+
+ template <class... Args, detail::enable_if_t<!std::is_nothrow_constructible<
+ T, Args &&...>::value> * = nullptr>
+ void emplace(Args &&... args) {
+ if (has_value()) {
+ val() = T(std::forward<Args>(args)...);
+ } else {
+ auto tmp = std::move(err());
+ err().~unexpected<E>();
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (valptr()) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ } catch (...) {
+ err() = std::move(tmp);
+ throw;
+ }
+ #else
+ ::new (valptr()) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ #endif
+ }
+ }
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_nothrow_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ void emplace(std::initializer_list<U> il, Args &&... args) {
+ if (has_value()) {
+ T t(il, std::forward<Args>(args)...);
+ val() = std::move(t);
+ } else {
+ err().~unexpected<E>();
+ ::new (valptr()) T(il, std::forward<Args>(args)...);
+ this->m_has_val = true;
+ }
+ }
+
+ template <class U, class... Args,
+ detail::enable_if_t<!std::is_nothrow_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ void emplace(std::initializer_list<U> il, Args &&... args) {
+ if (has_value()) {
+ T t(il, std::forward<Args>(args)...);
+ val() = std::move(t);
+ } else {
+ auto tmp = std::move(err());
+ err().~unexpected<E>();
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (valptr()) T(il, std::forward<Args>(args)...);
+ this->m_has_val = true;
+ } catch (...) {
+ err() = std::move(tmp);
+ throw;
+ }
+ #else
+ ::new (valptr()) T(il, std::forward<Args>(args)...);
+ this->m_has_val = true;
+ #endif
+ }
+ }
+
+private:
+ using t_is_void = std::true_type;
+ using t_is_not_void = std::false_type;
+ using t_is_nothrow_move_constructible = std::true_type;
+ using move_constructing_t_can_throw = std::false_type;
+ using e_is_nothrow_move_constructible = std::true_type;
+ using move_constructing_e_can_throw = std::false_type;
+
+ void swap_where_both_have_value(expected &/*rhs*/ , t_is_void) noexcept {
+ // swapping void is a no-op
+ }
+
+ void swap_where_both_have_value(expected &rhs, t_is_not_void) {
+ using std::swap;
+ swap(val(), rhs.val());
+ }
+
+ void swap_where_only_one_has_value(expected &rhs, t_is_void) noexcept(
+ std::is_nothrow_move_constructible<E>::value) {
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ std::swap(this->m_has_val, rhs.m_has_val);
+ }
+
+ void swap_where_only_one_has_value(expected &rhs, t_is_not_void) {
+ swap_where_only_one_has_value_and_t_is_not_void(
+ rhs, typename std::is_nothrow_move_constructible<T>::type{},
+ typename std::is_nothrow_move_constructible<E>::type{});
+ }
+
+ void swap_where_only_one_has_value_and_t_is_not_void(
+ expected &rhs, t_is_nothrow_move_constructible,
+ e_is_nothrow_move_constructible) noexcept {
+ auto temp = std::move(val());
+ val().~T();
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ ::new (rhs.valptr()) T(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+ }
+
+ void swap_where_only_one_has_value_and_t_is_not_void(
+ expected &rhs, t_is_nothrow_move_constructible,
+ move_constructing_e_can_throw) {
+ auto temp = std::move(val());
+ val().~T();
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ ::new (rhs.valptr()) T(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+ } catch (...) {
+ val() = std::move(temp);
+ throw;
+ }
+#else
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ ::new (rhs.valptr()) T(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+#endif
+ }
+
+ void swap_where_only_one_has_value_and_t_is_not_void(
+ expected &rhs, move_constructing_t_can_throw,
+ t_is_nothrow_move_constructible) {
+ auto temp = std::move(rhs.err());
+ rhs.err().~unexpected_type();
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (rhs.valptr()) T(val());
+ val().~T();
+ ::new (errptr()) unexpected_type(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+ } catch (...) {
+ rhs.err() = std::move(temp);
+ throw;
+ }
+#else
+ ::new (rhs.valptr()) T(val());
+ val().~T();
+ ::new (errptr()) unexpected_type(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+#endif
+ }
+
+public:
+ template <class OT = T, class OE = E>
+ detail::enable_if_t<detail::is_swappable<OT>::value &&
+ detail::is_swappable<OE>::value &&
+ (std::is_nothrow_move_constructible<OT>::value ||
+ std::is_nothrow_move_constructible<OE>::value)>
+ swap(expected &rhs) noexcept(
+ std::is_nothrow_move_constructible<T>::value
+ &&detail::is_nothrow_swappable<T>::value
+ &&std::is_nothrow_move_constructible<E>::value
+ &&detail::is_nothrow_swappable<E>::value) {
+ if (has_value() && rhs.has_value()) {
+ swap_where_both_have_value(rhs, typename std::is_void<T>::type{});
+ } else if (!has_value() && rhs.has_value()) {
+ rhs.swap(*this);
+ } else if (has_value()) {
+ swap_where_only_one_has_value(rhs, typename std::is_void<T>::type{});
+ } else {
+ using std::swap;
+ swap(err(), rhs.err());
+ }
+ }
+
+ constexpr const T *operator->() const { return valptr(); }
+ TL_EXPECTED_11_CONSTEXPR T *operator->() { return valptr(); }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ constexpr const U &operator*() const & {
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &operator*() & {
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ constexpr const U &&operator*() const && {
+ return std::move(val());
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &&operator*() && {
+ return std::move(val());
+ }
+
+ constexpr bool has_value() const noexcept { return this->m_has_val; }
+ constexpr explicit operator bool() const noexcept { return this->m_has_val; }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR const U &value() const & {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(err().value()));
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &value() & {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(err().value()));
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR const U &&value() const && {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(std::move(err()).value()));
+ return std::move(val());
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &&value() && {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(std::move(err()).value()));
+ return std::move(val());
+ }
+
+ constexpr const E &error() const & { return err().value(); }
+ TL_EXPECTED_11_CONSTEXPR E &error() & { return err().value(); }
+ constexpr const E &&error() const && { return std::move(err().value()); }
+ TL_EXPECTED_11_CONSTEXPR E &&error() && { return std::move(err().value()); }
+
+ template <class U> constexpr T value_or(U &&v) const & {
+ static_assert(std::is_copy_constructible<T>::value &&
+ std::is_convertible<U &&, T>::value,
+ "T must be copy-constructible and convertible to from U&&");
+ return bool(*this) ? **this : static_cast<T>(std::forward<U>(v));
+ }
+ template <class U> TL_EXPECTED_11_CONSTEXPR T value_or(U &&v) && {
+ static_assert(std::is_move_constructible<T>::value &&
+ std::is_convertible<U &&, T>::value,
+ "T must be move-constructible and convertible to from U&&");
+ return bool(*this) ? std::move(**this) : static_cast<T>(std::forward<U>(v));
+ }
+};
+
+namespace detail {
+template <class Exp> using exp_t = typename detail::decay_t<Exp>::value_type;
+template <class Exp> using err_t = typename detail::decay_t<Exp>::error_type;
+template <class Exp, class Ret> using ret_t = expected<Ret, err_t<Exp>>;
+
+#ifdef TL_EXPECTED_CXX14
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>()))>
+constexpr auto and_then_impl(Exp &&exp, F &&f) {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value()
+ ? detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>()))>
+constexpr auto and_then_impl(Exp &&exp, F &&f) {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value() ? detail::invoke(std::forward<F>(f))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+#else
+template <class> struct TC;
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr>
+auto and_then_impl(Exp &&exp, F &&f) -> Ret {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value()
+ ? detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr>
+constexpr auto and_then_impl(Exp &&exp, F &&f) -> Ret {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value() ? detail::invoke(std::forward<F>(f))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+#endif
+
+#ifdef TL_EXPECTED_CXX14
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f),
+ *std::forward<Exp>(exp)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = expected<void, err_t<Exp>>;
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp));
+ return result();
+ }
+
+ return result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = expected<void, err_t<Exp>>;
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f));
+ return result();
+ }
+
+ return result(unexpect, std::forward<Exp>(exp).error());
+}
+#else
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+
+constexpr auto expected_map_impl(Exp &&exp, F &&f)
+ -> ret_t<Exp, detail::decay_t<Ret>> {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f),
+ *std::forward<Exp>(exp)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+
+auto expected_map_impl(Exp &&exp, F &&f) -> expected<void, err_t<Exp>> {
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp));
+ return {};
+ }
+
+ return unexpected<err_t<Exp>>(std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+
+constexpr auto expected_map_impl(Exp &&exp, F &&f)
+ -> ret_t<Exp, detail::decay_t<Ret>> {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+
+auto expected_map_impl(Exp &&exp, F &&f) -> expected<void, err_t<Exp>> {
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f));
+ return {};
+ }
+
+ return unexpected<err_t<Exp>>(std::forward<Exp>(exp).error());
+}
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+ return exp.has_value()
+ ? result(*std::forward<Exp>(exp))
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result(*std::forward<Exp>(exp));
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+ return exp.has_value()
+ ? result()
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result();
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+#else
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f)
+ -> expected<exp_t<Exp>, detail::decay_t<Ret>> {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+
+ return exp.has_value()
+ ? result(*std::forward<Exp>(exp))
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) -> expected<exp_t<Exp>, monostate> {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result(*std::forward<Exp>(exp));
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f)
+ -> expected<exp_t<Exp>, detail::decay_t<Ret>> {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+
+ return exp.has_value()
+ ? result()
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) -> expected<exp_t<Exp>, monostate> {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result();
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+#endif
+
+#ifdef TL_EXPECTED_CXX14
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto or_else_impl(Exp &&exp, F &&f) {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+detail::decay_t<Exp> or_else_impl(Exp &&exp, F &&f) {
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : (detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error()),
+ std::forward<Exp>(exp));
+}
+#else
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+auto or_else_impl(Exp &&exp, F &&f) -> Ret {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+detail::decay_t<Exp> or_else_impl(Exp &&exp, F &&f) {
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : (detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error()),
+ std::forward<Exp>(exp));
+}
+#endif
+} // namespace detail
+
+template <class T, class E, class U, class F>
+constexpr bool operator==(const expected<T, E> &lhs,
+ const expected<U, F> &rhs) {
+ return (lhs.has_value() != rhs.has_value())
+ ? false
+ : (!lhs.has_value() ? lhs.error() == rhs.error() : *lhs == *rhs);
+}
+template <class T, class E, class U, class F>
+constexpr bool operator!=(const expected<T, E> &lhs,
+ const expected<U, F> &rhs) {
+ return (lhs.has_value() != rhs.has_value())
+ ? true
+ : (!lhs.has_value() ? lhs.error() != rhs.error() : *lhs != *rhs);
+}
+
+template <class T, class E, class U>
+constexpr bool operator==(const expected<T, E> &x, const U &v) {
+ return x.has_value() ? *x == v : false;
+}
+template <class T, class E, class U>
+constexpr bool operator==(const U &v, const expected<T, E> &x) {
+ return x.has_value() ? *x == v : false;
+}
+template <class T, class E, class U>
+constexpr bool operator!=(const expected<T, E> &x, const U &v) {
+ return x.has_value() ? *x != v : true;
+}
+template <class T, class E, class U>
+constexpr bool operator!=(const U &v, const expected<T, E> &x) {
+ return x.has_value() ? *x != v : true;
+}
+
+template <class T, class E>
+constexpr bool operator==(const expected<T, E> &x, const unexpected<E> &e) {
+ return x.has_value() ? false : x.error() == e.value();
+}
+template <class T, class E>
+constexpr bool operator==(const unexpected<E> &e, const expected<T, E> &x) {
+ return x.has_value() ? false : x.error() == e.value();
+}
+template <class T, class E>
+constexpr bool operator!=(const expected<T, E> &x, const unexpected<E> &e) {
+ return x.has_value() ? true : x.error() != e.value();
+}
+template <class T, class E>
+constexpr bool operator!=(const unexpected<E> &e, const expected<T, E> &x) {
+ return x.has_value() ? true : x.error() != e.value();
+}
+
+template <class T, class E,
+ detail::enable_if_t<(std::is_void<T>::value ||
+ std::is_move_constructible<T>::value) &&
+ detail::is_swappable<T>::value &&
+ std::is_move_constructible<E>::value &&
+ detail::is_swappable<E>::value> * = nullptr>
+void swap(expected<T, E> &lhs,
+ expected<T, E> &rhs) noexcept(noexcept(lhs.swap(rhs))) {
+ lhs.swap(rhs);
+}
+} // namespace tl
+
+#endif
diff --git a/contrib/fastutf8/CMakeLists.txt b/contrib/fastutf8/CMakeLists.txt
new file mode 100644
index 0000000..2a98ed8
--- /dev/null
+++ b/contrib/fastutf8/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(UTFSRC ${CMAKE_CURRENT_SOURCE_DIR}/fastutf8.c)
+IF(HAVE_AVX2 AND "${ARCH}" STREQUAL "x86_64")
+ SET(UTFSRC ${UTFSRC} ${CMAKE_CURRENT_SOURCE_DIR}/avx2.c)
+ MESSAGE(STATUS "UTF8: AVX2 support is added")
+ENDIF()
+IF(HAVE_SSE41 AND "${ARCH}" STREQUAL "x86_64")
+ SET(UTFSRC ${UTFSRC} ${CMAKE_CURRENT_SOURCE_DIR}/sse41.c)
+ MESSAGE(STATUS "UTF8: SSE41 support is added")
+ENDIF()
+
+ADD_LIBRARY(rspamd-fastutf8 STATIC ${UTFSRC}) \ No newline at end of file
diff --git a/contrib/fastutf8/LICENSE b/contrib/fastutf8/LICENSE
new file mode 100644
index 0000000..9b5471b
--- /dev/null
+++ b/contrib/fastutf8/LICENSE
@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2019 Yibo Cai
+Copyright (c) 2019 Vsevolod Stakhov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. \ No newline at end of file
diff --git a/contrib/fastutf8/avx2.c b/contrib/fastutf8/avx2.c
new file mode 100644
index 0000000..765c62f
--- /dev/null
+++ b/contrib/fastutf8/avx2.c
@@ -0,0 +1,314 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "config.h"
+#include "fastutf8.h"
+#include "platform_config.h"
+
+
+#ifndef __clang__
+#pragma GCC push_options
+#pragma GCC target("avx2")
+#endif
+
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __SSE4_2__
+#define __SSE4_2__
+#endif
+#ifndef __SSE4_1__
+#define __SSE4_1__
+#endif
+#ifndef __SSEE3__
+#define __SSEE3__
+#endif
+#ifndef __AVX__
+#define __AVX__
+#endif
+#ifndef __AVX2__
+#define __AVX2__
+#endif
+
+#include <immintrin.h>
+
+/*
+ * Map high nibble of "First Byte" to legal character length minus 1
+ * 0x00 ~ 0xBF --> 0
+ * 0xC0 ~ 0xDF --> 1
+ * 0xE0 ~ 0xEF --> 2
+ * 0xF0 ~ 0xFF --> 3
+ */
+static const int8_t _first_len_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
+};
+
+/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */
+static const int8_t _first_range_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
+};
+
+/*
+ * Range table, map range index to min and max values
+ * Index 0 : 00 ~ 7F (First Byte, ascii)
+ * Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte)
+ * Index 4 : A0 ~ BF (Second Byte after E0)
+ * Index 5 : 80 ~ 9F (Second Byte after ED)
+ * Index 6 : 90 ~ BF (Second Byte after F0)
+ * Index 7 : 80 ~ 8F (Second Byte after F4)
+ * Index 8 : C2 ~ F4 (First Byte, non ascii)
+ * Index 9~15 : illegal: i >= 127 && i <= -128
+ */
+static const int8_t _range_min_tbl[] = {
+ 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
+ 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
+ 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
+ 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
+};
+static const int8_t _range_max_tbl[] = {
+ 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
+ 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
+ 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+};
+
+/*
+ * Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after
+ * which the Second Byte are not 80~BF. It contains "range index adjustment".
+ * +------------+---------------+------------------+----------------+
+ * | First Byte | original range| range adjustment | adjusted range |
+ * +------------+---------------+------------------+----------------+
+ * | E0 | 2 | 2 | 4 |
+ * +------------+---------------+------------------+----------------+
+ * | ED | 2 | 3 | 5 |
+ * +------------+---------------+------------------+----------------+
+ * | F0 | 3 | 3 | 6 |
+ * +------------+---------------+------------------+----------------+
+ * | F4 | 4 | 4 | 8 |
+ * +------------+---------------+------------------+----------------+
+ */
+/* index1 -> E0, index14 -> ED */
+static const int8_t _df_ee_tbl[] = {
+ 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
+ 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
+};
+/* index1 -> F0, index5 -> F4 */
+static const int8_t _ef_fe_tbl[] = {
+ 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b)
+ __attribute__((__target__("avx2")));
+static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b)
+{
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15);
+}
+
+static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b)
+ __attribute__((__target__("avx2")));
+static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b)
+{
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 14);
+}
+
+static inline __m256i push_last_3bytes_of_a_to_b(__m256i a, __m256i b)
+ __attribute__((__target__("avx2")));
+static inline __m256i push_last_3bytes_of_a_to_b(__m256i a, __m256i b)
+{
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 13);
+}
+
+off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len)
+ __attribute__((__target__("avx2")));
+
+/* 5x faster than naive method */
+/* Return 0 - success, -1 - error, >0 - first error char(if RET_ERR_IDX = 1) */
+off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len)
+{
+ off_t err_pos = 1;
+
+ if (len >= 32) {
+ __m256i prev_input = _mm256_set1_epi8 (0);
+ __m256i prev_first_len = _mm256_set1_epi8 (0);
+
+ /* Cached tables */
+ const __m256i first_len_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _first_len_tbl);
+ const __m256i first_range_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _first_range_tbl);
+ const __m256i range_min_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _range_min_tbl);
+ const __m256i range_max_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _range_max_tbl);
+ const __m256i df_ee_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _df_ee_tbl);
+ const __m256i ef_fe_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _ef_fe_tbl);
+
+ __m256i error = _mm256_set1_epi8 (0);
+
+ while (len >= 32) {
+ const __m256i input = _mm256_lddqu_si256 ((const __m256i *) data);
+
+ /* high_nibbles = input >> 4 */
+ const __m256i high_nibbles =
+ _mm256_and_si256 (_mm256_srli_epi16 (input, 4), _mm256_set1_epi8 (0x0F));
+
+ /* first_len = legal character length minus 1 */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* first_len = first_len_tbl[high_nibbles] */
+ __m256i first_len = _mm256_shuffle_epi8 (first_len_tbl, high_nibbles);
+
+ /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */
+ /* range = first_range_tbl[high_nibbles] */
+ __m256i range = _mm256_shuffle_epi8 (first_range_tbl, high_nibbles);
+
+ /* Second Byte: set range index to first_len */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* range |= (first_len, prev_first_len) << 1 byte */
+ range = _mm256_or_si256 (
+ range, push_last_byte_of_a_to_b (prev_first_len, first_len));
+
+ /* Third Byte: set range index to saturate_sub(first_len, 1) */
+ /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */
+ __m256i tmp1, tmp2;
+
+ /* tmp1 = saturate_sub(first_len, 1) */
+ tmp1 = _mm256_subs_epu8 (first_len, _mm256_set1_epi8 (1));
+ /* tmp2 = saturate_sub(prev_first_len, 1) */
+ tmp2 = _mm256_subs_epu8 (prev_first_len, _mm256_set1_epi8 (1));
+
+ /* range |= (tmp1, tmp2) << 2 bytes */
+ range = _mm256_or_si256 (range, push_last_2bytes_of_a_to_b (tmp2, tmp1));
+
+ /* Fourth Byte: set range index to saturate_sub(first_len, 2) */
+ /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */
+ /* tmp1 = saturate_sub(first_len, 2) */
+ tmp1 = _mm256_subs_epu8 (first_len, _mm256_set1_epi8 (2));
+ /* tmp2 = saturate_sub(prev_first_len, 2) */
+ tmp2 = _mm256_subs_epu8 (prev_first_len, _mm256_set1_epi8 (2));
+ /* range |= (tmp1, tmp2) << 3 bytes */
+ range = _mm256_or_si256 (range, push_last_3bytes_of_a_to_b (tmp2, tmp1));
+
+ /*
+ * Now we have below range indices caluclated
+ * Correct cases:
+ * - 8 for C0~FF
+ * - 3 for 1st byte after F0~FF
+ * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF
+ * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or
+ * 3rd byte after F0~FF
+ * - 0 for others
+ * Error cases:
+ * 9,10,11 if non ascii First Byte overlaps
+ * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error
+ */
+
+ /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */
+ /* Overlaps lead to index 9~15, which are illegal in range table */
+ __m256i shift1, pos, range2;
+ /* shift1 = (input, prev_input) << 1 byte */
+ shift1 = push_last_byte_of_a_to_b (prev_input, input);
+ pos = _mm256_sub_epi8 (shift1, _mm256_set1_epi8 (0xEF));
+ /*
+ * shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE |
+ * pos: | 0 1 15 | 16 17 239| 240 241 255|
+ * pos-240: | 0 0 0 | 0 0 0 | 0 1 15 |
+ * pos+112: | 112 113 127| >= 128 | >= 128 |
+ */
+ tmp1 = _mm256_subs_epu8 (pos, _mm256_set1_epi8 ((char)240));
+ range2 = _mm256_shuffle_epi8 (df_ee_tbl, tmp1);
+ tmp2 = _mm256_adds_epu8 (pos, _mm256_set1_epi8 (112));
+ range2 = _mm256_add_epi8 (range2, _mm256_shuffle_epi8 (ef_fe_tbl, tmp2));
+
+ range = _mm256_add_epi8 (range, range2);
+
+ /* Load min and max values per calculated range index */
+ __m256i minv = _mm256_shuffle_epi8 (range_min_tbl, range);
+ __m256i maxv = _mm256_shuffle_epi8 (range_max_tbl, range);
+
+ /* Check value range */
+ error = _mm256_cmpgt_epi8(minv, input);
+ error = _mm256_or_si256(error, _mm256_cmpgt_epi8(input, maxv));
+ /* 5% performance drop from this conditional branch */
+ if (!_mm256_testz_si256(error, error)) {
+ break;
+ }
+
+ prev_input = input;
+ prev_first_len = first_len;
+
+ data += 32;
+ len -= 32;
+ err_pos += 32;
+ }
+
+ /* Error in first 16 bytes */
+ if (err_pos == 1) {
+ goto do_naive;
+ }
+
+ /* Find previous token (not 80~BF) */
+ int32_t token4 = _mm256_extract_epi32 (prev_input, 7);
+ const int8_t *token = (const int8_t *) &token4;
+ int lookahead = 0;
+
+ if (token[3] > (int8_t) 0xBF) {
+ lookahead = 1;
+ }
+ else if (token[2] > (int8_t) 0xBF) {
+ lookahead = 2;
+ }
+ else if (token[1] > (int8_t) 0xBF) {
+ lookahead = 3;
+ }
+
+ data -= lookahead;
+ len += lookahead;
+ err_pos -= lookahead;
+ }
+
+ /* Check remaining bytes with naive method */
+do_naive:
+ if (len > 0) {
+ off_t err_pos2 = rspamd_fast_utf8_validate_ref (data, len);
+
+ if (err_pos2) {
+ return err_pos + err_pos2 - 1;
+ }
+ }
+
+ return 0;
+}
+
+#ifndef __clang__
+#pragma GCC pop_options
+#endif
+
diff --git a/contrib/fastutf8/fastutf8.c b/contrib/fastutf8/fastutf8.c
new file mode 100644
index 0000000..89becaf
--- /dev/null
+++ b/contrib/fastutf8/fastutf8.c
@@ -0,0 +1,160 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "fastutf8.h"
+#include "libcryptobox/platform_config.h"
+
+
+/*
+ * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
+ *
+ * Table 3-7. Well-Formed UTF-8 Byte Sequences
+ *
+ * +--------------------+------------+-------------+------------+-------------+
+ * | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+0000..U+007F | 00..7F | | | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+0080..U+07FF | C2..DF | 80..BF | | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+D000..U+D7FF | ED | 80..9F | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
+ * +--------------------+------------+-------------+------------+-------------+
+ */
+
+/* Return 0 - success, >0 - index (1 based) of first error char */
+off_t
+rspamd_fast_utf8_validate_ref (const unsigned char *data, size_t len)
+{
+ off_t err_pos = 1;
+
+ while (len) {
+ int bytes;
+ const unsigned char byte1 = data[0];
+
+ /* 00..7F */
+ if (byte1 <= 0x7F) {
+ bytes = 1;
+ /* C2..DF, 80..BF */
+ }
+ else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF &&
+ (signed char) data[1] <= (signed char) 0xBF) {
+ bytes = 2;
+ }
+ else if (len >= 3) {
+ const unsigned char byte2 = data[1];
+
+ /* Is byte2, byte3 between 0x80 ~ 0xBF */
+ const int byte2_ok = (signed char) byte2 <= (signed char) 0xBF;
+ const int byte3_ok = (signed char) data[2] <= (signed char) 0xBF;
+
+ if (byte2_ok && byte3_ok &&
+ /* E0, A0..BF, 80..BF */
+ ((byte1 == 0xE0 && byte2 >= 0xA0) ||
+ /* E1..EC, 80..BF, 80..BF */
+ (byte1 >= 0xE1 && byte1 <= 0xEC) ||
+ /* ED, 80..9F, 80..BF */
+ (byte1 == 0xED && byte2 <= 0x9F) ||
+ /* EE..EF, 80..BF, 80..BF */
+ (byte1 >= 0xEE && byte1 <= 0xEF))) {
+ bytes = 3;
+ }
+ else if (len >= 4) {
+ /* Is byte4 between 0x80 ~ 0xBF */
+ const int byte4_ok = (signed char) data[3] <= (signed char) 0xBF;
+
+ if (byte2_ok && byte3_ok && byte4_ok &&
+ /* F0, 90..BF, 80..BF, 80..BF */
+ ((byte1 == 0xF0 && byte2 >= 0x90) ||
+ /* F1..F3, 80..BF, 80..BF, 80..BF */
+ (byte1 >= 0xF1 && byte1 <= 0xF3) ||
+ /* F4, 80..8F, 80..BF, 80..BF */
+ (byte1 == 0xF4 && byte2 <= 0x8F))) {
+ bytes = 4;
+ }
+ else {
+ return err_pos;
+ }
+ }
+ else {
+ return err_pos;
+ }
+ }
+ else {
+ return err_pos;
+ }
+
+ len -= bytes;
+ err_pos += bytes;
+ data += bytes;
+ }
+
+ return 0;
+}
+
+/* Prototypes */
+#if defined(HAVE_SSE41) && defined(__x86_64__)
+extern off_t rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len);
+#endif
+#if defined(HAVE_AVX2) && defined(__x86_64__)
+extern off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len);
+#endif
+
+static off_t (*validate_func) (const unsigned char *data, size_t len) =
+ rspamd_fast_utf8_validate_ref;
+
+
+void
+rspamd_fast_utf8_library_init (unsigned flags)
+{
+#if defined(HAVE_SSE41) && defined(__x86_64__)
+ if (flags & RSPAMD_FAST_UTF8_FLAG_SSE41) {
+ validate_func = rspamd_fast_utf8_validate_sse41;
+ }
+#endif
+#if defined(HAVE_AVX2) && defined(__x86_64__)
+ if (flags & RSPAMD_FAST_UTF8_FLAG_AVX2) {
+ validate_func = rspamd_fast_utf8_validate_avx2;
+ }
+#endif
+}
+
+off_t
+rspamd_fast_utf8_validate (const unsigned char *data, size_t len)
+{
+ return len >= 64 ?
+ validate_func (data, len) :
+ rspamd_fast_utf8_validate_ref (data, len);
+} \ No newline at end of file
diff --git a/contrib/fastutf8/fastutf8.h b/contrib/fastutf8/fastutf8.h
new file mode 100644
index 0000000..a1e9cbf
--- /dev/null
+++ b/contrib/fastutf8/fastutf8.h
@@ -0,0 +1,65 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RSPAMD_FASTUTF8_H
+#define RSPAMD_FASTUTF8_H
+
+#include <sys/types.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+enum rspamd_fast_utf8_cpu_flags {
+ RSPAMD_FAST_UTF8_FLAG_SSE41 = 1u << 0u,
+ RSPAMD_FAST_UTF8_FLAG_AVX2 = 1u << 1u,
+};
+
+/**
+ * Called to init codecs
+ * @param flags
+ */
+void rspamd_fast_utf8_library_init(unsigned flags);
+
+/**
+ * Called to validate input using fast codec
+ * @param data
+ * @param len
+ * @return
+ */
+off_t rspamd_fast_utf8_validate(const unsigned char *data, size_t len);
+
+/**
+ * Use plain C implementation
+ * @param data
+ * @param len
+ * @return
+ */
+off_t rspamd_fast_utf8_validate_ref(const unsigned char *data, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/contrib/fastutf8/sse41.c b/contrib/fastutf8/sse41.c
new file mode 100644
index 0000000..df338cf
--- /dev/null
+++ b/contrib/fastutf8/sse41.c
@@ -0,0 +1,272 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "config.h"
+#include "fastutf8.h"
+#include "platform_config.h"
+
+#ifndef __clang__
+#pragma GCC push_options
+#pragma GCC target("sse4.1")
+#endif
+
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __SSEE3__
+#define __SSEE3__
+#endif
+#ifndef __SSE4_1__
+#define __SSE4_1__
+#endif
+
+#include <smmintrin.h>
+
+/*
+ * Map high nibble of "First Byte" to legal character length minus 1
+ * 0x00 ~ 0xBF --> 0
+ * 0xC0 ~ 0xDF --> 1
+ * 0xE0 ~ 0xEF --> 2
+ * 0xF0 ~ 0xFF --> 3
+ */
+static const int8_t _first_len_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
+};
+
+/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */
+static const int8_t _first_range_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
+};
+
+/*
+ * Range table, map range index to min and max values
+ * Index 0 : 00 ~ 7F (First Byte, ascii)
+ * Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte)
+ * Index 4 : A0 ~ BF (Second Byte after E0)
+ * Index 5 : 80 ~ 9F (Second Byte after ED)
+ * Index 6 : 90 ~ BF (Second Byte after F0)
+ * Index 7 : 80 ~ 8F (Second Byte after F4)
+ * Index 8 : C2 ~ F4 (First Byte, non ascii)
+ * Index 9~15 : illegal: i >= 127 && i <= -128
+ */
+static const int8_t _range_min_tbl[] = {
+ 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
+ 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
+};
+static const int8_t _range_max_tbl[] = {
+ 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
+ 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+};
+
+/*
+ * Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after
+ * which the Second Byte are not 80~BF. It contains "range index adjustment".
+ * +------------+---------------+------------------+----------------+
+ * | First Byte | original range| range adjustment | adjusted range |
+ * +------------+---------------+------------------+----------------+
+ * | E0 | 2 | 2 | 4 |
+ * +------------+---------------+------------------+----------------+
+ * | ED | 2 | 3 | 5 |
+ * +------------+---------------+------------------+----------------+
+ * | F0 | 3 | 3 | 6 |
+ * +------------+---------------+------------------+----------------+
+ * | F4 | 4 | 4 | 8 |
+ * +------------+---------------+------------------+----------------+
+ */
+/* index1 -> E0, index14 -> ED */
+static const int8_t _df_ee_tbl[] = {
+ 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
+};
+/* index1 -> F0, index5 -> F4 */
+static const int8_t _ef_fe_tbl[] = {
+ 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+off_t
+rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len)
+ __attribute__((__target__("sse4.1")));
+
+/* Return 0 - success, >0 - first error char(if RET_ERR_IDX = 1) */
+off_t
+rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len)
+{
+ off_t err_pos = 1;
+
+ if (len >= 16) {
+ __m128i prev_input = _mm_set1_epi8 (0);
+ __m128i prev_first_len = _mm_set1_epi8 (0);
+
+ /* Cached tables */
+ const __m128i first_len_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _first_len_tbl);
+ const __m128i first_range_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _first_range_tbl);
+ const __m128i range_min_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _range_min_tbl);
+ const __m128i range_max_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _range_max_tbl);
+ const __m128i df_ee_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _df_ee_tbl);
+ const __m128i ef_fe_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _ef_fe_tbl);
+
+ __m128i error = _mm_set1_epi8 (0);
+
+ while (len >= 16) {
+ const __m128i input = _mm_lddqu_si128 ((const __m128i *) data);
+
+ /* high_nibbles = input >> 4 */
+ const __m128i high_nibbles =
+ _mm_and_si128 (_mm_srli_epi16 (input, 4), _mm_set1_epi8 (0x0F));
+
+ /* first_len = legal character length minus 1 */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* first_len = first_len_tbl[high_nibbles] */
+ __m128i first_len = _mm_shuffle_epi8 (first_len_tbl, high_nibbles);
+
+ /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */
+ /* range = first_range_tbl[high_nibbles] */
+ __m128i range = _mm_shuffle_epi8 (first_range_tbl, high_nibbles);
+
+ /* Second Byte: set range index to first_len */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* range |= (first_len, prev_first_len) << 1 byte */
+ range = _mm_or_si128 (
+ range, _mm_alignr_epi8(first_len, prev_first_len, 15));
+
+ /* Third Byte: set range index to saturate_sub(first_len, 1) */
+ /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */
+ __m128i tmp1, tmp2;
+ /* tmp1 = saturate_sub(first_len, 1) */
+ tmp1 = _mm_subs_epu8 (first_len, _mm_set1_epi8 (1));
+ /* tmp2 = saturate_sub(prev_first_len, 1) */
+ tmp2 = _mm_subs_epu8 (prev_first_len, _mm_set1_epi8 (1));
+ /* range |= (tmp1, tmp2) << 2 bytes */
+ range = _mm_or_si128 (range, _mm_alignr_epi8(tmp1, tmp2, 14));
+
+ /* Fourth Byte: set range index to saturate_sub(first_len, 2) */
+ /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */
+ /* tmp1 = saturate_sub(first_len, 2) */
+ tmp1 = _mm_subs_epu8 (first_len, _mm_set1_epi8 (2));
+ /* tmp2 = saturate_sub(prev_first_len, 2) */
+ tmp2 = _mm_subs_epu8 (prev_first_len, _mm_set1_epi8 (2));
+ /* range |= (tmp1, tmp2) << 3 bytes */
+ range = _mm_or_si128 (range, _mm_alignr_epi8(tmp1, tmp2, 13));
+
+ /*
+ * Now we have below range indices caluclated
+ * Correct cases:
+ * - 8 for C0~FF
+ * - 3 for 1st byte after F0~FF
+ * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF
+ * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or
+ * 3rd byte after F0~FF
+ * - 0 for others
+ * Error cases:
+ * 9,10,11 if non ascii First Byte overlaps
+ * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error
+ */
+
+ /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */
+ /* Overlaps lead to index 9~15, which are illegal in range table */
+ __m128i shift1, pos, range2;
+ /* shift1 = (input, prev_input) << 1 byte */
+ shift1 = _mm_alignr_epi8(input, prev_input, 15);
+ pos = _mm_sub_epi8 (shift1, _mm_set1_epi8 (0xEF));
+ /*
+ * shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE |
+ * pos: | 0 1 15 | 16 17 239| 240 241 255|
+ * pos-240: | 0 0 0 | 0 0 0 | 0 1 15 |
+ * pos+112: | 112 113 127| >= 128 | >= 128 |
+ */
+ tmp1 = _mm_subs_epu8 (pos, _mm_set1_epi8 ((char)240));
+ range2 = _mm_shuffle_epi8 (df_ee_tbl, tmp1);
+ tmp2 = _mm_adds_epu8 (pos, _mm_set1_epi8 (112));
+ range2 = _mm_add_epi8 (range2, _mm_shuffle_epi8 (ef_fe_tbl, tmp2));
+
+ range = _mm_add_epi8 (range, range2);
+
+ /* Load min and max values per calculated range index */
+ __m128i minv = _mm_shuffle_epi8 (range_min_tbl, range);
+ __m128i maxv = _mm_shuffle_epi8 (range_max_tbl, range);
+
+ /* Check value range */
+ error = _mm_cmplt_epi8(input, minv);
+ error = _mm_or_si128(error, _mm_cmpgt_epi8(input, maxv));
+ /* 5% performance drop from this conditional branch */
+ if (!_mm_testz_si128(error, error)) {
+ break;
+ }
+
+ prev_input = input;
+ prev_first_len = first_len;
+
+ data += 16;
+ len -= 16;
+ err_pos += 16;
+ }
+
+ /* Error in first 16 bytes */
+ if (err_pos == 1) {
+ goto do_naive;
+ }
+
+ /* Find previous token (not 80~BF) */
+ int32_t token4 = _mm_extract_epi32 (prev_input, 3);
+ const int8_t *token = (const int8_t *) &token4;
+ int lookahead = 0;
+
+ if (token[3] > (int8_t) 0xBF) {
+ lookahead = 1;
+ }
+ else if (token[2] > (int8_t) 0xBF) {
+ lookahead = 2;
+ }
+ else if (token[1] > (int8_t) 0xBF) {
+ lookahead = 3;
+ }
+
+ data -= lookahead;
+ len += lookahead;
+ err_pos -= lookahead;
+ }
+
+ do_naive:
+ if (len > 0) {
+ off_t err_pos2 = rspamd_fast_utf8_validate_ref (data, len);
+
+ if (err_pos2) {
+ return err_pos + err_pos2 - 1;
+ }
+ }
+
+ return 0;
+}
+
+#ifndef __clang__
+#pragma GCC pop_options
+#endif \ No newline at end of file
diff --git a/contrib/fmt/LICENSE.rst b/contrib/fmt/LICENSE.rst
new file mode 100644
index 0000000..f0ec3db
--- /dev/null
+++ b/contrib/fmt/LICENSE.rst
@@ -0,0 +1,27 @@
+Copyright (c) 2012 - present, Victor Zverovich
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--- Optional exception to the license ---
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into a machine-executable object form of such
+source code, you may redistribute such embedded portions in such object form
+without including the above copyright and permission notices.
diff --git a/contrib/fmt/README.rst b/contrib/fmt/README.rst
new file mode 100644
index 0000000..acddc70
--- /dev/null
+++ b/contrib/fmt/README.rst
@@ -0,0 +1,506 @@
+{fmt}
+=====
+
+.. image:: https://travis-ci.org/fmtlib/fmt.png?branch=master
+ :target: https://travis-ci.org/fmtlib/fmt
+
+.. image:: https://ci.appveyor.com/api/projects/status/ehjkiefde6gucy1v
+ :target: https://ci.appveyor.com/project/vitaut/fmt
+
+.. image:: https://oss-fuzz-build-logs.storage.googleapis.com/badges/fmt.svg
+ :alt: fmt is continuously fuzzed at oss-fuzz
+ :target: https://bugs.chromium.org/p/oss-fuzz/issues/list?\
+ colspec=ID%20Type%20Component%20Status%20Proj%20Reported%20Owner%20\
+ Summary&q=proj%3Dfmt&can=1
+
+.. image:: https://img.shields.io/badge/stackoverflow-fmt-blue.svg
+ :alt: Ask questions at StackOverflow with the tag fmt
+ :target: https://stackoverflow.com/questions/tagged/fmt
+
+**{fmt}** is an open-source formatting library providing a fast and safe
+alternative to C stdio and C++ iostreams.
+
+If you like this project, please consider donating to BYSOL,
+an initiative to help victims of political repressions in Belarus:
+https://www.facebook.com/donate/759400044849707/108388587646909/.
+
+`Documentation <https://fmt.dev>`__
+
+Q&A: ask questions on `StackOverflow with the tag fmt
+<https://stackoverflow.com/questions/tagged/fmt>`_.
+
+Try {fmt} in `Compiler Explorer <https://godbolt.org/z/Eq5763>`_.
+
+Features
+--------
+
+* Simple `format API <https://fmt.dev/latest/api.html>`_ with positional arguments
+ for localization
+* Implementation of `C++20 std::format
+ <https://en.cppreference.com/w/cpp/utility/format>`__
+* `Format string syntax <https://fmt.dev/latest/syntax.html>`_ similar to Python's
+ `format <https://docs.python.org/3/library/stdtypes.html#str.format>`_
+* Fast IEEE 754 floating-point formatter with correct rounding, shortness and
+ round-trip guarantees
+* Safe `printf implementation
+ <https://fmt.dev/latest/api.html#printf-formatting>`_ including the POSIX
+ extension for positional arguments
+* Extensibility: `support for user-defined types
+ <https://fmt.dev/latest/api.html#formatting-user-defined-types>`_
+* High performance: faster than common standard library implementations of
+ ``(s)printf``, iostreams, ``to_string`` and ``to_chars``, see `Speed tests`_
+ and `Converting a hundred million integers to strings per second
+ <http://www.zverovich.net/2020/06/13/fast-int-to-string-revisited.html>`_
+* Small code size both in terms of source code with the minimum configuration
+ consisting of just three files, ``core.h``, ``format.h`` and ``format-inl.h``,
+ and compiled code; see `Compile time and code bloat`_
+* Reliability: the library has an extensive set of `tests
+ <https://github.com/fmtlib/fmt/tree/master/test>`_ and is `continuously fuzzed
+ <https://bugs.chromium.org/p/oss-fuzz/issues/list?colspec=ID%20Type%20
+ Component%20Status%20Proj%20Reported%20Owner%20Summary&q=proj%3Dfmt&can=1>`_
+* Safety: the library is fully type safe, errors in format strings can be
+ reported at compile time, automatic memory management prevents buffer overflow
+ errors
+* Ease of use: small self-contained code base, no external dependencies,
+ permissive MIT `license
+ <https://github.com/fmtlib/fmt/blob/master/LICENSE.rst>`_
+* `Portability <https://fmt.dev/latest/index.html#portability>`_ with
+ consistent output across platforms and support for older compilers
+* Clean warning-free codebase even on high warning levels such as
+ ``-Wall -Wextra -pedantic``
+* Locale-independence by default
+* Optional header-only configuration enabled with the ``FMT_HEADER_ONLY`` macro
+
+See the `documentation <https://fmt.dev>`_ for more details.
+
+Examples
+--------
+
+**Print to stdout** (`run <https://godbolt.org/z/Tevcjh>`_)
+
+.. code:: c++
+
+ #include <fmt/core.h>
+
+ int main() {
+ fmt::print("Hello, world!\n");
+ }
+
+**Format a string** (`run <https://godbolt.org/z/oK8h33>`_)
+
+.. code:: c++
+
+ std::string s = fmt::format("The answer is {}.", 42);
+ // s == "The answer is 42."
+
+**Format a string using positional arguments** (`run <https://godbolt.org/z/Yn7Txe>`_)
+
+.. code:: c++
+
+ std::string s = fmt::format("I'd rather be {1} than {0}.", "right", "happy");
+ // s == "I'd rather be happy than right."
+
+**Print chrono durations** (`run <https://godbolt.org/z/K8s4Mc>`_)
+
+.. code:: c++
+
+ #include <fmt/chrono.h>
+
+ int main() {
+ using namespace std::literals::chrono_literals;
+ fmt::print("Default format: {} {}\n", 42s, 100ms);
+ fmt::print("strftime-like format: {:%H:%M:%S}\n", 3h + 15min + 30s);
+ }
+
+Output::
+
+ Default format: 42s 100ms
+ strftime-like format: 03:15:30
+
+**Print a container** (`run <https://godbolt.org/z/MjsY7c>`_)
+
+.. code:: c++
+
+ #include <vector>
+ #include <fmt/ranges.h>
+
+ int main() {
+ std::vector<int> v = {1, 2, 3};
+ fmt::print("{}\n", v);
+ }
+
+Output::
+
+ {1, 2, 3}
+
+**Check a format string at compile time**
+
+.. code:: c++
+
+ std::string s = fmt::format(FMT_STRING("{:d}"), "don't panic");
+
+This gives a compile-time error because ``d`` is an invalid format specifier for
+a string.
+
+**Write a file from a single thread**
+
+.. code:: c++
+
+ #include <fmt/os.h>
+
+ int main() {
+ auto out = fmt::output_file("guide.txt");
+ out.print("Don't {}", "Panic");
+ }
+
+This can be `5 to 9 times faster than fprintf
+<http://www.zverovich.net/2020/08/04/optimal-file-buffer-size.html>`_.
+
+**Print with colors and text styles**
+
+.. code:: c++
+
+ #include <fmt/color.h>
+
+ int main() {
+ fmt::print(fg(fmt::color::crimson) | fmt::emphasis::bold,
+ "Hello, {}!\n", "world");
+ fmt::print(fg(fmt::color::floral_white) | bg(fmt::color::slate_gray) |
+ fmt::emphasis::underline, "Hello, {}!\n", "мир");
+ fmt::print(fg(fmt::color::steel_blue) | fmt::emphasis::italic,
+ "Hello, {}!\n", "世界");
+ }
+
+Output on a modern terminal:
+
+.. image:: https://user-images.githubusercontent.com/
+ 576385/88485597-d312f600-cf2b-11ea-9cbe-61f535a86e28.png
+
+Benchmarks
+----------
+
+Speed tests
+~~~~~~~~~~~
+
+================= ============= ===========
+Library Method Run Time, s
+================= ============= ===========
+libc printf 1.04
+libc++ std::ostream 3.05
+{fmt} 6.1.1 fmt::print 0.75
+Boost Format 1.67 boost::format 7.24
+Folly Format folly::format 2.23
+================= ============= ===========
+
+{fmt} is the fastest of the benchmarked methods, ~35% faster than ``printf``.
+
+The above results were generated by building ``tinyformat_test.cpp`` on macOS
+10.14.6 with ``clang++ -O3 -DNDEBUG -DSPEED_TEST -DHAVE_FORMAT``, and taking the
+best of three runs. In the test, the format string ``"%0.10f:%04d:%+g:%s:%p:%c:%%\n"``
+or equivalent is filled 2,000,000 times with output sent to ``/dev/null``; for
+further details refer to the `source
+<https://github.com/fmtlib/format-benchmark/blob/master/tinyformat_test.cpp>`_.
+
+{fmt} is up to 20-30x faster than ``std::ostringstream`` and ``sprintf`` on
+floating-point formatting (`dtoa-benchmark <https://github.com/fmtlib/dtoa-benchmark>`_)
+and faster than `double-conversion <https://github.com/google/double-conversion>`_ and
+`ryu <https://github.com/ulfjack/ryu>`_:
+
+.. image:: https://user-images.githubusercontent.com/576385/
+ 95684665-11719600-0ba8-11eb-8e5b-972ff4e49428.png
+ :target: https://fmt.dev/unknown_mac64_clang12.0.html
+
+Compile time and code bloat
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The script `bloat-test.py
+<https://github.com/fmtlib/format-benchmark/blob/master/bloat-test.py>`_
+from `format-benchmark <https://github.com/fmtlib/format-benchmark>`_
+tests compile time and code bloat for nontrivial projects.
+It generates 100 translation units and uses ``printf()`` or its alternative
+five times in each to simulate a medium sized project. The resulting
+executable size and compile time (Apple LLVM version 8.1.0 (clang-802.0.42),
+macOS Sierra, best of three) is shown in the following tables.
+
+**Optimized build (-O3)**
+
+============= =============== ==================== ==================
+Method Compile Time, s Executable size, KiB Stripped size, KiB
+============= =============== ==================== ==================
+printf 2.6 29 26
+printf+string 16.4 29 26
+iostreams 31.1 59 55
+{fmt} 19.0 37 34
+Boost Format 91.9 226 203
+Folly Format 115.7 101 88
+============= =============== ==================== ==================
+
+As you can see, {fmt} has 60% less overhead in terms of resulting binary code
+size compared to iostreams and comes pretty close to ``printf``. Boost Format
+and Folly Format have the largest overheads.
+
+``printf+string`` is the same as ``printf`` but with extra ``<string>``
+include to measure the overhead of the latter.
+
+**Non-optimized build**
+
+============= =============== ==================== ==================
+Method Compile Time, s Executable size, KiB Stripped size, KiB
+============= =============== ==================== ==================
+printf 2.2 33 30
+printf+string 16.0 33 30
+iostreams 28.3 56 52
+{fmt} 18.2 59 50
+Boost Format 54.1 365 303
+Folly Format 79.9 445 430
+============= =============== ==================== ==================
+
+``libc``, ``lib(std)c++`` and ``libfmt`` are all linked as shared libraries to
+compare formatting function overhead only. Boost Format is a
+header-only library so it doesn't provide any linkage options.
+
+Running the tests
+~~~~~~~~~~~~~~~~~
+
+Please refer to `Building the library`__ for the instructions on how to build
+the library and run the unit tests.
+
+__ https://fmt.dev/latest/usage.html#building-the-library
+
+Benchmarks reside in a separate repository,
+`format-benchmarks <https://github.com/fmtlib/format-benchmark>`_,
+so to run the benchmarks you first need to clone this repository and
+generate Makefiles with CMake::
+
+ $ git clone --recursive https://github.com/fmtlib/format-benchmark.git
+ $ cd format-benchmark
+ $ cmake .
+
+Then you can run the speed test::
+
+ $ make speed-test
+
+or the bloat test::
+
+ $ make bloat-test
+
+Projects using this library
+---------------------------
+
+* `0 A.D. <https://play0ad.com/>`_: a free, open-source, cross-platform
+ real-time strategy game
+
+* `AMPL/MP <https://github.com/ampl/mp>`_:
+ an open-source library for mathematical programming
+
+* `Aseprite <https://github.com/aseprite/aseprite>`_:
+ animated sprite editor & pixel art tool
+
+* `AvioBook <https://www.aviobook.aero/en>`_: a comprehensive aircraft
+ operations suite
+
+* `Blizzard Battle.net <https://battle.net/>`_: an online gaming platform
+
+* `Celestia <https://celestia.space/>`_: real-time 3D visualization of space
+
+* `Ceph <https://ceph.com/>`_: a scalable distributed storage system
+
+* `ccache <https://ccache.dev/>`_: a compiler cache
+
+* `ClickHouse <https://github.com/ClickHouse/ClickHouse>`_: analytical database
+ management system
+
+* `CUAUV <http://cuauv.org/>`_: Cornell University's autonomous underwater
+ vehicle
+
+* `Drake <https://drake.mit.edu/>`_: a planning, control, and analysis toolbox
+ for nonlinear dynamical systems (MIT)
+
+* `Envoy <https://lyft.github.io/envoy/>`_: C++ L7 proxy and communication bus
+ (Lyft)
+
+* `FiveM <https://fivem.net/>`_: a modification framework for GTA V
+
+* `Folly <https://github.com/facebook/folly>`_: Facebook open-source library
+
+* `HarpyWar/pvpgn <https://github.com/pvpgn/pvpgn-server>`_:
+ Player vs Player Gaming Network with tweaks
+
+* `KBEngine <https://github.com/kbengine/kbengine>`_: an open-source MMOG server
+ engine
+
+* `Keypirinha <https://keypirinha.com/>`_: a semantic launcher for Windows
+
+* `Kodi <https://kodi.tv/>`_ (formerly xbmc): home theater software
+
+* `Knuth <https://kth.cash/>`_: high-performance Bitcoin full-node
+
+* `Microsoft Verona <https://github.com/microsoft/verona>`_:
+ research programming language for concurrent ownership
+
+* `MongoDB <https://mongodb.com/>`_: distributed document database
+
+* `MongoDB Smasher <https://github.com/duckie/mongo_smasher>`_: a small tool to
+ generate randomized datasets
+
+* `OpenSpace <https://openspaceproject.com/>`_: an open-source
+ astrovisualization framework
+
+* `PenUltima Online (POL) <https://www.polserver.com/>`_:
+ an MMO server, compatible with most Ultima Online clients
+
+* `PyTorch <https://github.com/pytorch/pytorch>`_: an open-source machine
+ learning library
+
+* `quasardb <https://www.quasardb.net/>`_: a distributed, high-performance,
+ associative database
+
+* `Quill <https://github.com/odygrd/quill>`_: asynchronous low-latency logging library
+
+* `QKW <https://github.com/ravijanjam/qkw>`_: generalizing aliasing to simplify
+ navigation, and executing complex multi-line terminal command sequences
+
+* `redis-cerberus <https://github.com/HunanTV/redis-cerberus>`_: a Redis cluster
+ proxy
+
+* `redpanda <https://vectorized.io/redpanda>`_: a 10x faster Kafka® replacement
+ for mission critical systems written in C++
+
+* `rpclib <http://rpclib.net/>`_: a modern C++ msgpack-RPC server and client
+ library
+
+* `Salesforce Analytics Cloud
+ <https://www.salesforce.com/analytics-cloud/overview/>`_:
+ business intelligence software
+
+* `Scylla <https://www.scylladb.com/>`_: a Cassandra-compatible NoSQL data store
+ that can handle 1 million transactions per second on a single server
+
+* `Seastar <http://www.seastar-project.org/>`_: an advanced, open-source C++
+ framework for high-performance server applications on modern hardware
+
+* `spdlog <https://github.com/gabime/spdlog>`_: super fast C++ logging library
+
+* `Stellar <https://www.stellar.org/>`_: financial platform
+
+* `Touch Surgery <https://www.touchsurgery.com/>`_: surgery simulator
+
+* `TrinityCore <https://github.com/TrinityCore/TrinityCore>`_: open-source
+ MMORPG framework
+
+* `Windows Terminal <https://github.com/microsoft/terminal>`_: the new Windows
+ terminal
+
+`More... <https://github.com/search?q=fmtlib&type=Code>`_
+
+If you are aware of other projects using this library, please let me know
+by `email <mailto:victor.zverovich@gmail.com>`_ or by submitting an
+`issue <https://github.com/fmtlib/fmt/issues>`_.
+
+Motivation
+----------
+
+So why yet another formatting library?
+
+There are plenty of methods for doing this task, from standard ones like
+the printf family of function and iostreams to Boost Format and FastFormat
+libraries. The reason for creating a new library is that every existing
+solution that I found either had serious issues or didn't provide
+all the features I needed.
+
+printf
+~~~~~~
+
+The good thing about ``printf`` is that it is pretty fast and readily available
+being a part of the C standard library. The main drawback is that it
+doesn't support user-defined types. ``printf`` also has safety issues although
+they are somewhat mitigated with `__attribute__ ((format (printf, ...))
+<https://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html>`_ in GCC.
+There is a POSIX extension that adds positional arguments required for
+`i18n <https://en.wikipedia.org/wiki/Internationalization_and_localization>`_
+to ``printf`` but it is not a part of C99 and may not be available on some
+platforms.
+
+iostreams
+~~~~~~~~~
+
+The main issue with iostreams is best illustrated with an example:
+
+.. code:: c++
+
+ std::cout << std::setprecision(2) << std::fixed << 1.23456 << "\n";
+
+which is a lot of typing compared to printf:
+
+.. code:: c++
+
+ printf("%.2f\n", 1.23456);
+
+Matthew Wilson, the author of FastFormat, called this "chevron hell". iostreams
+don't support positional arguments by design.
+
+The good part is that iostreams support user-defined types and are safe although
+error handling is awkward.
+
+Boost Format
+~~~~~~~~~~~~
+
+This is a very powerful library which supports both ``printf``-like format
+strings and positional arguments. Its main drawback is performance. According to
+various, benchmarks it is much slower than other methods considered here. Boost
+Format also has excessive build times and severe code bloat issues (see
+`Benchmarks`_).
+
+FastFormat
+~~~~~~~~~~
+
+This is an interesting library which is fast, safe and has positional arguments.
+However, it has significant limitations, citing its author:
+
+ Three features that have no hope of being accommodated within the
+ current design are:
+
+ * Leading zeros (or any other non-space padding)
+ * Octal/hexadecimal encoding
+ * Runtime width/alignment specification
+
+It is also quite big and has a heavy dependency, STLSoft, which might be too
+restrictive for using it in some projects.
+
+Boost Spirit.Karma
+~~~~~~~~~~~~~~~~~~
+
+This is not really a formatting library but I decided to include it here for
+completeness. As iostreams, it suffers from the problem of mixing verbatim text
+with arguments. The library is pretty fast, but slower on integer formatting
+than ``fmt::format_to`` with format string compilation on Karma's own benchmark,
+see `Converting a hundred million integers to strings per second
+<http://www.zverovich.net/2020/06/13/fast-int-to-string-revisited.html>`_.
+
+License
+-------
+
+{fmt} is distributed under the MIT `license
+<https://github.com/fmtlib/fmt/blob/master/LICENSE.rst>`_.
+
+Documentation License
+---------------------
+
+The `Format String Syntax <https://fmt.dev/latest/syntax.html>`_
+section in the documentation is based on the one from Python `string module
+documentation <https://docs.python.org/3/library/string.html#module-string>`_.
+For this reason the documentation is distributed under the Python Software
+Foundation license available in `doc/python-license.txt
+<https://raw.github.com/fmtlib/fmt/master/doc/python-license.txt>`_.
+It only applies if you distribute the documentation of {fmt}.
+
+Maintainers
+-----------
+
+The {fmt} library is maintained by Victor Zverovich (`vitaut
+<https://github.com/vitaut>`_) and Jonathan Müller (`foonathan
+<https://github.com/foonathan>`_) with contributions from many other people.
+See `Contributors <https://github.com/fmtlib/fmt/graphs/contributors>`_ and
+`Releases <https://github.com/fmtlib/fmt/releases>`_ for some of the names.
+Let us know if your contribution is not listed or mentioned incorrectly and
+we'll make it right.
diff --git a/contrib/fmt/include/fmt/args.h b/contrib/fmt/include/fmt/args.h
new file mode 100644
index 0000000..a3966d1
--- /dev/null
+++ b/contrib/fmt/include/fmt/args.h
@@ -0,0 +1,234 @@
+// Formatting library for C++ - dynamic format arguments
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_ARGS_H_
+#define FMT_ARGS_H_
+
+#include <functional> // std::reference_wrapper
+#include <memory> // std::unique_ptr
+#include <vector>
+
+#include "core.h"
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename T> struct is_reference_wrapper : std::false_type {};
+template <typename T>
+struct is_reference_wrapper<std::reference_wrapper<T>> : std::true_type {};
+
+template <typename T> const T& unwrap(const T& v) { return v; }
+template <typename T> const T& unwrap(const std::reference_wrapper<T>& v) {
+ return static_cast<const T&>(v);
+}
+
+class dynamic_arg_list {
+ // Workaround for clang's -Wweak-vtables. Unlike for regular classes, for
+ // templates it doesn't complain about inability to deduce single translation
+ // unit for placing vtable. So storage_node_base is made a fake template.
+ template <typename = void> struct node {
+ virtual ~node() = default;
+ std::unique_ptr<node<>> next;
+ };
+
+ template <typename T> struct typed_node : node<> {
+ T value;
+
+ template <typename Arg>
+ FMT_CONSTEXPR typed_node(const Arg& arg) : value(arg) {}
+
+ template <typename Char>
+ FMT_CONSTEXPR typed_node(const basic_string_view<Char>& arg)
+ : value(arg.data(), arg.size()) {}
+ };
+
+ std::unique_ptr<node<>> head_;
+
+ public:
+ template <typename T, typename Arg> const T& push(const Arg& arg) {
+ auto new_node = std::unique_ptr<typed_node<T>>(new typed_node<T>(arg));
+ auto& value = new_node->value;
+ new_node->next = std::move(head_);
+ head_ = std::move(new_node);
+ return value;
+ }
+};
+} // namespace detail
+
+/**
+ \rst
+ A dynamic version of `fmt::format_arg_store`.
+ It's equipped with a storage to potentially temporary objects which lifetimes
+ could be shorter than the format arguments object.
+
+ It can be implicitly converted into `~fmt::basic_format_args` for passing
+ into type-erased formatting functions such as `~fmt::vformat`.
+ \endrst
+ */
+template <typename Context>
+class dynamic_format_arg_store
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+ // Workaround a GCC template argument substitution bug.
+ : public basic_format_args<Context>
+#endif
+{
+ private:
+ using char_type = typename Context::char_type;
+
+ template <typename T> struct need_copy {
+ static constexpr detail::type mapped_type =
+ detail::mapped_type_constant<T, Context>::value;
+
+ enum {
+ value = !(detail::is_reference_wrapper<T>::value ||
+ std::is_same<T, basic_string_view<char_type>>::value ||
+ std::is_same<T, detail::std_string_view<char_type>>::value ||
+ (mapped_type != detail::type::cstring_type &&
+ mapped_type != detail::type::string_type &&
+ mapped_type != detail::type::custom_type))
+ };
+ };
+
+ template <typename T>
+ using stored_type = conditional_t<
+ std::is_convertible<T, std::basic_string<char_type>>::value &&
+ !detail::is_reference_wrapper<T>::value,
+ std::basic_string<char_type>, T>;
+
+ // Storage of basic_format_arg must be contiguous.
+ std::vector<basic_format_arg<Context>> data_;
+ std::vector<detail::named_arg_info<char_type>> named_info_;
+
+ // Storage of arguments not fitting into basic_format_arg must grow
+ // without relocation because items in data_ refer to it.
+ detail::dynamic_arg_list dynamic_args_;
+
+ friend class basic_format_args<Context>;
+
+ unsigned long long get_types() const {
+ return detail::is_unpacked_bit | data_.size() |
+ (named_info_.empty()
+ ? 0ULL
+ : static_cast<unsigned long long>(detail::has_named_args_bit));
+ }
+
+ const basic_format_arg<Context>* data() const {
+ return named_info_.empty() ? data_.data() : data_.data() + 1;
+ }
+
+ template <typename T> void emplace_arg(const T& arg) {
+ data_.emplace_back(detail::make_arg<Context>(arg));
+ }
+
+ template <typename T>
+ void emplace_arg(const detail::named_arg<char_type, T>& arg) {
+ if (named_info_.empty()) {
+ constexpr const detail::named_arg_info<char_type>* zero_ptr{nullptr};
+ data_.insert(data_.begin(), {zero_ptr, 0});
+ }
+ data_.emplace_back(detail::make_arg<Context>(detail::unwrap(arg.value)));
+ auto pop_one = [](std::vector<basic_format_arg<Context>>* data) {
+ data->pop_back();
+ };
+ std::unique_ptr<std::vector<basic_format_arg<Context>>, decltype(pop_one)>
+ guard{&data_, pop_one};
+ named_info_.push_back({arg.name, static_cast<int>(data_.size() - 2u)});
+ data_[0].value_.named_args = {named_info_.data(), named_info_.size()};
+ guard.release();
+ }
+
+ public:
+ constexpr dynamic_format_arg_store() = default;
+
+ /**
+ \rst
+ Adds an argument into the dynamic store for later passing to a formatting
+ function.
+
+ Note that custom types and string types (but not string views) are copied
+ into the store dynamically allocating memory if necessary.
+
+ **Example**::
+
+ fmt::dynamic_format_arg_store<fmt::format_context> store;
+ store.push_back(42);
+ store.push_back("abc");
+ store.push_back(1.5f);
+ std::string result = fmt::vformat("{} and {} and {}", store);
+ \endrst
+ */
+ template <typename T> void push_back(const T& arg) {
+ if (detail::const_check(need_copy<T>::value))
+ emplace_arg(dynamic_args_.push<stored_type<T>>(arg));
+ else
+ emplace_arg(detail::unwrap(arg));
+ }
+
+ /**
+ \rst
+ Adds a reference to the argument into the dynamic store for later passing to
+ a formatting function.
+
+ **Example**::
+
+ fmt::dynamic_format_arg_store<fmt::format_context> store;
+ char band[] = "Rolling Stones";
+ store.push_back(std::cref(band));
+ band[9] = 'c'; // Changing str affects the output.
+ std::string result = fmt::vformat("{}", store);
+ // result == "Rolling Scones"
+ \endrst
+ */
+ template <typename T> void push_back(std::reference_wrapper<T> arg) {
+ static_assert(
+ need_copy<T>::value,
+ "objects of built-in types and string views are always copied");
+ emplace_arg(arg.get());
+ }
+
+ /**
+ Adds named argument into the dynamic store for later passing to a formatting
+ function. ``std::reference_wrapper`` is supported to avoid copying of the
+ argument. The name is always copied into the store.
+ */
+ template <typename T>
+ void push_back(const detail::named_arg<char_type, T>& arg) {
+ const char_type* arg_name =
+ dynamic_args_.push<std::basic_string<char_type>>(arg.name).c_str();
+ if (detail::const_check(need_copy<T>::value)) {
+ emplace_arg(
+ fmt::arg(arg_name, dynamic_args_.push<stored_type<T>>(arg.value)));
+ } else {
+ emplace_arg(fmt::arg(arg_name, arg.value));
+ }
+ }
+
+ /** Erase all elements from the store */
+ void clear() {
+ data_.clear();
+ named_info_.clear();
+ dynamic_args_ = detail::dynamic_arg_list();
+ }
+
+ /**
+ \rst
+ Reserves space to store at least *new_cap* arguments including
+ *new_cap_named* named arguments.
+ \endrst
+ */
+ void reserve(size_t new_cap, size_t new_cap_named) {
+ FMT_ASSERT(new_cap >= new_cap_named,
+ "Set of arguments includes set of named arguments");
+ data_.reserve(new_cap);
+ named_info_.reserve(new_cap_named);
+ }
+};
+
+FMT_END_NAMESPACE
+
+#endif // FMT_ARGS_H_
diff --git a/contrib/fmt/include/fmt/chrono.h b/contrib/fmt/include/fmt/chrono.h
new file mode 100644
index 0000000..55e8a50
--- /dev/null
+++ b/contrib/fmt/include/fmt/chrono.h
@@ -0,0 +1,2267 @@
+// Formatting library for C++ - chrono support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_CHRONO_H_
+#define FMT_CHRONO_H_
+
+#include <algorithm>
+#include <chrono>
+#include <cmath> // std::isfinite
+#include <cstring> // std::memcpy
+#include <ctime>
+#include <iterator>
+#include <locale>
+#include <ostream>
+#include <type_traits>
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+// Check if std::chrono::local_t is available.
+#ifndef FMT_USE_LOCAL_TIME
+# ifdef __cpp_lib_chrono
+# define FMT_USE_LOCAL_TIME (__cpp_lib_chrono >= 201907L)
+# else
+# define FMT_USE_LOCAL_TIME 0
+# endif
+#endif
+
+// Check if std::chrono::utc_timestamp is available.
+#ifndef FMT_USE_UTC_TIME
+# ifdef __cpp_lib_chrono
+# define FMT_USE_UTC_TIME (__cpp_lib_chrono >= 201907L)
+# else
+# define FMT_USE_UTC_TIME 0
+# endif
+#endif
+
+// Enable tzset.
+#ifndef FMT_USE_TZSET
+// UWP doesn't provide _tzset.
+# if FMT_HAS_INCLUDE("winapifamily.h")
+# include <winapifamily.h>
+# endif
+# if defined(_WIN32) && (!defined(WINAPI_FAMILY) || \
+ (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+# define FMT_USE_TZSET 1
+# else
+# define FMT_USE_TZSET 0
+# endif
+#endif
+
+// Enable safe chrono durations, unless explicitly disabled.
+#ifndef FMT_SAFE_DURATION_CAST
+# define FMT_SAFE_DURATION_CAST 1
+#endif
+#if FMT_SAFE_DURATION_CAST
+
+// For conversion between std::chrono::durations without undefined
+// behaviour or erroneous results.
+// This is a stripped down version of duration_cast, for inclusion in fmt.
+// See https://github.com/pauldreik/safe_duration_cast
+//
+// Copyright Paul Dreik 2019
+namespace safe_duration_cast {
+
+template <typename To, typename From,
+ FMT_ENABLE_IF(!std::is_same<From, To>::value &&
+ std::numeric_limits<From>::is_signed ==
+ std::numeric_limits<To>::is_signed)>
+FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
+ ec = 0;
+ using F = std::numeric_limits<From>;
+ using T = std::numeric_limits<To>;
+ static_assert(F::is_integer, "From must be integral");
+ static_assert(T::is_integer, "To must be integral");
+
+ // A and B are both signed, or both unsigned.
+ if (detail::const_check(F::digits <= T::digits)) {
+ // From fits in To without any problem.
+ } else {
+ // From does not always fit in To, resort to a dynamic check.
+ if (from < (T::min)() || from > (T::max)()) {
+ // outside range.
+ ec = 1;
+ return {};
+ }
+ }
+ return static_cast<To>(from);
+}
+
+/**
+ * converts From to To, without loss. If the dynamic value of from
+ * can't be converted to To without loss, ec is set.
+ */
+template <typename To, typename From,
+ FMT_ENABLE_IF(!std::is_same<From, To>::value &&
+ std::numeric_limits<From>::is_signed !=
+ std::numeric_limits<To>::is_signed)>
+FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
+ ec = 0;
+ using F = std::numeric_limits<From>;
+ using T = std::numeric_limits<To>;
+ static_assert(F::is_integer, "From must be integral");
+ static_assert(T::is_integer, "To must be integral");
+
+ if (detail::const_check(F::is_signed && !T::is_signed)) {
+ // From may be negative, not allowed!
+ if (fmt::detail::is_negative(from)) {
+ ec = 1;
+ return {};
+ }
+ // From is positive. Can it always fit in To?
+ if (detail::const_check(F::digits > T::digits) &&
+ from > static_cast<From>(detail::max_value<To>())) {
+ ec = 1;
+ return {};
+ }
+ }
+
+ if (detail::const_check(!F::is_signed && T::is_signed &&
+ F::digits >= T::digits) &&
+ from > static_cast<From>(detail::max_value<To>())) {
+ ec = 1;
+ return {};
+ }
+ return static_cast<To>(from); // Lossless conversion.
+}
+
+template <typename To, typename From,
+ FMT_ENABLE_IF(std::is_same<From, To>::value)>
+FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
+ ec = 0;
+ return from;
+} // function
+
+// clang-format off
+/**
+ * converts From to To if possible, otherwise ec is set.
+ *
+ * input | output
+ * ---------------------------------|---------------
+ * NaN | NaN
+ * Inf | Inf
+ * normal, fits in output | converted (possibly lossy)
+ * normal, does not fit in output | ec is set
+ * subnormal | best effort
+ * -Inf | -Inf
+ */
+// clang-format on
+template <typename To, typename From,
+ FMT_ENABLE_IF(!std::is_same<From, To>::value)>
+FMT_CONSTEXPR To safe_float_conversion(const From from, int& ec) {
+ ec = 0;
+ using T = std::numeric_limits<To>;
+ static_assert(std::is_floating_point<From>::value, "From must be floating");
+ static_assert(std::is_floating_point<To>::value, "To must be floating");
+
+ // catch the only happy case
+ if (std::isfinite(from)) {
+ if (from >= T::lowest() && from <= (T::max)()) {
+ return static_cast<To>(from);
+ }
+ // not within range.
+ ec = 1;
+ return {};
+ }
+
+ // nan and inf will be preserved
+ return static_cast<To>(from);
+} // function
+
+template <typename To, typename From,
+ FMT_ENABLE_IF(std::is_same<From, To>::value)>
+FMT_CONSTEXPR To safe_float_conversion(const From from, int& ec) {
+ ec = 0;
+ static_assert(std::is_floating_point<From>::value, "From must be floating");
+ return from;
+}
+
+/**
+ * safe duration cast between integral durations
+ */
+template <typename To, typename FromRep, typename FromPeriod,
+ FMT_ENABLE_IF(std::is_integral<FromRep>::value),
+ FMT_ENABLE_IF(std::is_integral<typename To::rep>::value)>
+To safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
+ int& ec) {
+ using From = std::chrono::duration<FromRep, FromPeriod>;
+ ec = 0;
+ // the basic idea is that we need to convert from count() in the from type
+ // to count() in the To type, by multiplying it with this:
+ struct Factor
+ : std::ratio_divide<typename From::period, typename To::period> {};
+
+ static_assert(Factor::num > 0, "num must be positive");
+ static_assert(Factor::den > 0, "den must be positive");
+
+ // the conversion is like this: multiply from.count() with Factor::num
+ // /Factor::den and convert it to To::rep, all this without
+ // overflow/underflow. let's start by finding a suitable type that can hold
+ // both To, From and Factor::num
+ using IntermediateRep =
+ typename std::common_type<typename From::rep, typename To::rep,
+ decltype(Factor::num)>::type;
+
+ // safe conversion to IntermediateRep
+ IntermediateRep count =
+ lossless_integral_conversion<IntermediateRep>(from.count(), ec);
+ if (ec) return {};
+ // multiply with Factor::num without overflow or underflow
+ if (detail::const_check(Factor::num != 1)) {
+ const auto max1 = detail::max_value<IntermediateRep>() / Factor::num;
+ if (count > max1) {
+ ec = 1;
+ return {};
+ }
+ const auto min1 =
+ (std::numeric_limits<IntermediateRep>::min)() / Factor::num;
+ if (detail::const_check(!std::is_unsigned<IntermediateRep>::value) &&
+ count < min1) {
+ ec = 1;
+ return {};
+ }
+ count *= Factor::num;
+ }
+
+ if (detail::const_check(Factor::den != 1)) count /= Factor::den;
+ auto tocount = lossless_integral_conversion<typename To::rep>(count, ec);
+ return ec ? To() : To(tocount);
+}
+
+/**
+ * safe duration_cast between floating point durations
+ */
+template <typename To, typename FromRep, typename FromPeriod,
+ FMT_ENABLE_IF(std::is_floating_point<FromRep>::value),
+ FMT_ENABLE_IF(std::is_floating_point<typename To::rep>::value)>
+To safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
+ int& ec) {
+ using From = std::chrono::duration<FromRep, FromPeriod>;
+ ec = 0;
+ if (std::isnan(from.count())) {
+ // nan in, gives nan out. easy.
+ return To{std::numeric_limits<typename To::rep>::quiet_NaN()};
+ }
+ // maybe we should also check if from is denormal, and decide what to do about
+ // it.
+
+ // +-inf should be preserved.
+ if (std::isinf(from.count())) {
+ return To{from.count()};
+ }
+
+ // the basic idea is that we need to convert from count() in the from type
+ // to count() in the To type, by multiplying it with this:
+ struct Factor
+ : std::ratio_divide<typename From::period, typename To::period> {};
+
+ static_assert(Factor::num > 0, "num must be positive");
+ static_assert(Factor::den > 0, "den must be positive");
+
+ // the conversion is like this: multiply from.count() with Factor::num
+ // /Factor::den and convert it to To::rep, all this without
+ // overflow/underflow. let's start by finding a suitable type that can hold
+ // both To, From and Factor::num
+ using IntermediateRep =
+ typename std::common_type<typename From::rep, typename To::rep,
+ decltype(Factor::num)>::type;
+
+ // force conversion of From::rep -> IntermediateRep to be safe,
+ // even if it will never happen be narrowing in this context.
+ IntermediateRep count =
+ safe_float_conversion<IntermediateRep>(from.count(), ec);
+ if (ec) {
+ return {};
+ }
+
+ // multiply with Factor::num without overflow or underflow
+ if (detail::const_check(Factor::num != 1)) {
+ constexpr auto max1 = detail::max_value<IntermediateRep>() /
+ static_cast<IntermediateRep>(Factor::num);
+ if (count > max1) {
+ ec = 1;
+ return {};
+ }
+ constexpr auto min1 = std::numeric_limits<IntermediateRep>::lowest() /
+ static_cast<IntermediateRep>(Factor::num);
+ if (count < min1) {
+ ec = 1;
+ return {};
+ }
+ count *= static_cast<IntermediateRep>(Factor::num);
+ }
+
+ // this can't go wrong, right? den>0 is checked earlier.
+ if (detail::const_check(Factor::den != 1)) {
+ using common_t = typename std::common_type<IntermediateRep, intmax_t>::type;
+ count /= static_cast<common_t>(Factor::den);
+ }
+
+ // convert to the to type, safely
+ using ToRep = typename To::rep;
+
+ const ToRep tocount = safe_float_conversion<ToRep>(count, ec);
+ if (ec) {
+ return {};
+ }
+ return To{tocount};
+}
+} // namespace safe_duration_cast
+#endif
+
+// Prevents expansion of a preceding token as a function-style macro.
+// Usage: f FMT_NOMACRO()
+#define FMT_NOMACRO
+
+namespace detail {
+template <typename T = void> struct null {};
+inline null<> localtime_r FMT_NOMACRO(...) { return null<>(); }
+inline null<> localtime_s(...) { return null<>(); }
+inline null<> gmtime_r(...) { return null<>(); }
+inline null<> gmtime_s(...) { return null<>(); }
+
+inline const std::locale& get_classic_locale() {
+ static const auto& locale = std::locale::classic();
+ return locale;
+}
+
+template <typename CodeUnit> struct codecvt_result {
+ static constexpr const size_t max_size = 32;
+ CodeUnit buf[max_size];
+ CodeUnit* end;
+};
+template <typename CodeUnit>
+constexpr const size_t codecvt_result<CodeUnit>::max_size;
+
+template <typename CodeUnit>
+void write_codecvt(codecvt_result<CodeUnit>& out, string_view in_buf,
+ const std::locale& loc) {
+#if FMT_CLANG_VERSION
+# pragma clang diagnostic push
+# pragma clang diagnostic ignored "-Wdeprecated"
+ auto& f = std::use_facet<std::codecvt<CodeUnit, char, std::mbstate_t>>(loc);
+# pragma clang diagnostic pop
+#else
+ auto& f = std::use_facet<std::codecvt<CodeUnit, char, std::mbstate_t>>(loc);
+#endif
+ auto mb = std::mbstate_t();
+ const char* from_next = nullptr;
+ auto result = f.in(mb, in_buf.begin(), in_buf.end(), from_next,
+ std::begin(out.buf), std::end(out.buf), out.end);
+ if (result != std::codecvt_base::ok)
+ FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename OutputIt>
+auto write_encoded_tm_str(OutputIt out, string_view in, const std::locale& loc)
+ -> OutputIt {
+ if (detail::is_utf8() && loc != get_classic_locale()) {
+ // char16_t and char32_t codecvts are broken in MSVC (linkage errors) and
+ // gcc-4.
+#if FMT_MSC_VERSION != 0 || \
+ (defined(__GLIBCXX__) && !defined(_GLIBCXX_USE_DUAL_ABI))
+ // The _GLIBCXX_USE_DUAL_ABI macro is always defined in libstdc++ from gcc-5
+ // and newer.
+ using code_unit = wchar_t;
+#else
+ using code_unit = char32_t;
+#endif
+
+ using unit_t = codecvt_result<code_unit>;
+ unit_t unit;
+ write_codecvt(unit, in, loc);
+ // In UTF-8 is used one to four one-byte code units.
+ unicode_to_utf8<code_unit, basic_memory_buffer<char, unit_t::max_size * 4>>
+ u;
+ if (!u.convert({unit.buf, to_unsigned(unit.end - unit.buf)}))
+ FMT_THROW(format_error("failed to format time"));
+ return copy_str<char>(u.c_str(), u.c_str() + u.size(), out);
+ }
+ return copy_str<char>(in.data(), in.data() + in.size(), out);
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+ -> OutputIt {
+ codecvt_result<Char> unit;
+ write_codecvt(unit, sv, loc);
+ return copy_str<Char>(unit.buf, unit.end, out);
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+ -> OutputIt {
+ return write_encoded_tm_str(out, sv, loc);
+}
+
+template <typename Char>
+inline void do_write(buffer<Char>& buf, const std::tm& time,
+ const std::locale& loc, char format, char modifier) {
+ auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+ auto&& os = std::basic_ostream<Char>(&format_buf);
+ os.imbue(loc);
+ using iterator = std::ostreambuf_iterator<Char>;
+ const auto& facet = std::use_facet<std::time_put<Char, iterator>>(loc);
+ auto end = facet.put(os, os, Char(' '), &time, format, modifier);
+ if (end.failed()) FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto write(OutputIt out, const std::tm& time, const std::locale& loc,
+ char format, char modifier = 0) -> OutputIt {
+ auto&& buf = get_buffer<Char>(out);
+ do_write<Char>(buf, time, loc, format, modifier);
+ return get_iterator(buf, out);
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write(OutputIt out, const std::tm& time, const std::locale& loc,
+ char format, char modifier = 0) -> OutputIt {
+ auto&& buf = basic_memory_buffer<Char>();
+ do_write<char>(buf, time, loc, format, modifier);
+ return write_encoded_tm_str(out, string_view(buf.data(), buf.size()), loc);
+}
+
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+/**
+ Converts given time since epoch as ``std::time_t`` value into calendar time,
+ expressed in local time. Unlike ``std::localtime``, this function is
+ thread-safe on most platforms.
+ */
+inline std::tm localtime(std::time_t time) {
+ struct dispatcher {
+ std::time_t time_;
+ std::tm tm_;
+
+ dispatcher(std::time_t t) : time_(t) {}
+
+ bool run() {
+ using namespace fmt::detail;
+ return handle(localtime_r(&time_, &tm_));
+ }
+
+ bool handle(std::tm* tm) { return tm != nullptr; }
+
+ bool handle(detail::null<>) {
+ using namespace fmt::detail;
+ return fallback(localtime_s(&tm_, &time_));
+ }
+
+ bool fallback(int res) { return res == 0; }
+
+#if !FMT_MSC_VERSION
+ bool fallback(detail::null<>) {
+ using namespace fmt::detail;
+ std::tm* tm = std::localtime(&time_);
+ if (tm) tm_ = *tm;
+ return tm != nullptr;
+ }
+#endif
+ };
+ dispatcher lt(time);
+ // Too big time values may be unsupported.
+ if (!lt.run()) FMT_THROW(format_error("time_t value out of range"));
+ return lt.tm_;
+}
+
+#if FMT_USE_LOCAL_TIME
+template <typename Duration>
+inline auto localtime(std::chrono::local_time<Duration> time) -> std::tm {
+ return localtime(std::chrono::system_clock::to_time_t(
+ std::chrono::current_zone()->to_sys(time)));
+}
+#endif
+
+/**
+ Converts given time since epoch as ``std::time_t`` value into calendar time,
+ expressed in Coordinated Universal Time (UTC). Unlike ``std::gmtime``, this
+ function is thread-safe on most platforms.
+ */
+inline std::tm gmtime(std::time_t time) {
+ struct dispatcher {
+ std::time_t time_;
+ std::tm tm_;
+
+ dispatcher(std::time_t t) : time_(t) {}
+
+ bool run() {
+ using namespace fmt::detail;
+ return handle(gmtime_r(&time_, &tm_));
+ }
+
+ bool handle(std::tm* tm) { return tm != nullptr; }
+
+ bool handle(detail::null<>) {
+ using namespace fmt::detail;
+ return fallback(gmtime_s(&tm_, &time_));
+ }
+
+ bool fallback(int res) { return res == 0; }
+
+#if !FMT_MSC_VERSION
+ bool fallback(detail::null<>) {
+ std::tm* tm = std::gmtime(&time_);
+ if (tm) tm_ = *tm;
+ return tm != nullptr;
+ }
+#endif
+ };
+ dispatcher gt(time);
+ // Too big time values may be unsupported.
+ if (!gt.run()) FMT_THROW(format_error("time_t value out of range"));
+ return gt.tm_;
+}
+
+inline std::tm gmtime(
+ std::chrono::time_point<std::chrono::system_clock> time_point) {
+ return gmtime(std::chrono::system_clock::to_time_t(time_point));
+}
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// DEPRECATED!
+template <typename Char>
+FMT_CONSTEXPR auto parse_align(const Char* begin, const Char* end,
+ format_specs<Char>& specs) -> const Char* {
+ FMT_ASSERT(begin != end, "");
+ auto align = align::none;
+ auto p = begin + code_point_length(begin);
+ if (end - p <= 0) p = begin;
+ for (;;) {
+ switch (to_ascii(*p)) {
+ case '<':
+ align = align::left;
+ break;
+ case '>':
+ align = align::right;
+ break;
+ case '^':
+ align = align::center;
+ break;
+ }
+ if (align != align::none) {
+ if (p != begin) {
+ auto c = *begin;
+ if (c == '}') return begin;
+ if (c == '{') {
+ throw_format_error("invalid fill character '{'");
+ return begin;
+ }
+ specs.fill = {begin, to_unsigned(p - begin)};
+ begin = p + 1;
+ } else {
+ ++begin;
+ }
+ break;
+ } else if (p == begin) {
+ break;
+ }
+ p = begin;
+ }
+ specs.align = align;
+ return begin;
+}
+
+// Writes two-digit numbers a, b and c separated by sep to buf.
+// The method by Pavel Novikov based on
+// https://johnnylee-sde.github.io/Fast-unsigned-integer-to-time-string/.
+inline void write_digit2_separated(char* buf, unsigned a, unsigned b,
+ unsigned c, char sep) {
+ unsigned long long digits =
+ a | (b << 24) | (static_cast<unsigned long long>(c) << 48);
+ // Convert each value to BCD.
+ // We have x = a * 10 + b and we want to convert it to BCD y = a * 16 + b.
+ // The difference is
+ // y - x = a * 6
+ // a can be found from x:
+ // a = floor(x / 10)
+ // then
+ // y = x + a * 6 = x + floor(x / 10) * 6
+ // floor(x / 10) is (x * 205) >> 11 (needs 16 bits).
+ digits += (((digits * 205) >> 11) & 0x000f00000f00000f) * 6;
+ // Put low nibbles to high bytes and high nibbles to low bytes.
+ digits = ((digits & 0x00f00000f00000f0) >> 4) |
+ ((digits & 0x000f00000f00000f) << 8);
+ auto usep = static_cast<unsigned long long>(sep);
+ // Add ASCII '0' to each digit byte and insert separators.
+ digits |= 0x3030003030003030 | (usep << 16) | (usep << 40);
+
+ constexpr const size_t len = 8;
+ if (const_check(is_big_endian())) {
+ char tmp[len];
+ std::memcpy(tmp, &digits, len);
+ std::reverse_copy(tmp, tmp + len, buf);
+ } else {
+ std::memcpy(buf, &digits, len);
+ }
+}
+
+template <typename Period> FMT_CONSTEXPR inline const char* get_units() {
+ if (std::is_same<Period, std::atto>::value) return "as";
+ if (std::is_same<Period, std::femto>::value) return "fs";
+ if (std::is_same<Period, std::pico>::value) return "ps";
+ if (std::is_same<Period, std::nano>::value) return "ns";
+ if (std::is_same<Period, std::micro>::value) return "µs";
+ if (std::is_same<Period, std::milli>::value) return "ms";
+ if (std::is_same<Period, std::centi>::value) return "cs";
+ if (std::is_same<Period, std::deci>::value) return "ds";
+ if (std::is_same<Period, std::ratio<1>>::value) return "s";
+ if (std::is_same<Period, std::deca>::value) return "das";
+ if (std::is_same<Period, std::hecto>::value) return "hs";
+ if (std::is_same<Period, std::kilo>::value) return "ks";
+ if (std::is_same<Period, std::mega>::value) return "Ms";
+ if (std::is_same<Period, std::giga>::value) return "Gs";
+ if (std::is_same<Period, std::tera>::value) return "Ts";
+ if (std::is_same<Period, std::peta>::value) return "Ps";
+ if (std::is_same<Period, std::exa>::value) return "Es";
+ if (std::is_same<Period, std::ratio<60>>::value) return "m";
+ if (std::is_same<Period, std::ratio<3600>>::value) return "h";
+ return nullptr;
+}
+
+enum class numeric_system {
+ standard,
+ // Alternative numeric system, e.g. 十二 instead of 12 in ja_JP locale.
+ alternative
+};
+
+// Glibc extensions for formatting numeric values.
+enum class pad_type {
+ unspecified,
+ // Do not pad a numeric result string.
+ none,
+ // Pad a numeric result string with zeros even if the conversion specifier
+ // character uses space-padding by default.
+ zero,
+ // Pad a numeric result string with spaces.
+ space,
+};
+
+template <typename OutputIt>
+auto write_padding(OutputIt out, pad_type pad, int width) -> OutputIt {
+ if (pad == pad_type::none) return out;
+ return std::fill_n(out, width, pad == pad_type::space ? ' ' : '0');
+}
+
+template <typename OutputIt>
+auto write_padding(OutputIt out, pad_type pad) -> OutputIt {
+ if (pad != pad_type::none) *out++ = pad == pad_type::space ? ' ' : '0';
+ return out;
+}
+
+// Parses a put_time-like format string and invokes handler actions.
+template <typename Char, typename Handler>
+FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin,
+ const Char* end,
+ Handler&& handler) {
+ if (begin == end || *begin == '}') return begin;
+ if (*begin != '%') FMT_THROW(format_error("invalid format"));
+ auto ptr = begin;
+ pad_type pad = pad_type::unspecified;
+ while (ptr != end) {
+ auto c = *ptr;
+ if (c == '}') break;
+ if (c != '%') {
+ ++ptr;
+ continue;
+ }
+ if (begin != ptr) handler.on_text(begin, ptr);
+ ++ptr; // consume '%'
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr;
+ switch (c) {
+ case '_':
+ pad = pad_type::space;
+ ++ptr;
+ break;
+ case '-':
+ pad = pad_type::none;
+ ++ptr;
+ break;
+ case '0':
+ pad = pad_type::zero;
+ ++ptr;
+ break;
+ }
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr++;
+ switch (c) {
+ case '%':
+ handler.on_text(ptr - 1, ptr);
+ break;
+ case 'n': {
+ const Char newline[] = {'\n'};
+ handler.on_text(newline, newline + 1);
+ break;
+ }
+ case 't': {
+ const Char tab[] = {'\t'};
+ handler.on_text(tab, tab + 1);
+ break;
+ }
+ // Year:
+ case 'Y':
+ handler.on_year(numeric_system::standard);
+ break;
+ case 'y':
+ handler.on_short_year(numeric_system::standard);
+ break;
+ case 'C':
+ handler.on_century(numeric_system::standard);
+ break;
+ case 'G':
+ handler.on_iso_week_based_year();
+ break;
+ case 'g':
+ handler.on_iso_week_based_short_year();
+ break;
+ // Day of the week:
+ case 'a':
+ handler.on_abbr_weekday();
+ break;
+ case 'A':
+ handler.on_full_weekday();
+ break;
+ case 'w':
+ handler.on_dec0_weekday(numeric_system::standard);
+ break;
+ case 'u':
+ handler.on_dec1_weekday(numeric_system::standard);
+ break;
+ // Month:
+ case 'b':
+ case 'h':
+ handler.on_abbr_month();
+ break;
+ case 'B':
+ handler.on_full_month();
+ break;
+ case 'm':
+ handler.on_dec_month(numeric_system::standard);
+ break;
+ // Day of the year/month:
+ case 'U':
+ handler.on_dec0_week_of_year(numeric_system::standard);
+ break;
+ case 'W':
+ handler.on_dec1_week_of_year(numeric_system::standard);
+ break;
+ case 'V':
+ handler.on_iso_week_of_year(numeric_system::standard);
+ break;
+ case 'j':
+ handler.on_day_of_year();
+ break;
+ case 'd':
+ handler.on_day_of_month(numeric_system::standard);
+ break;
+ case 'e':
+ handler.on_day_of_month_space(numeric_system::standard);
+ break;
+ // Hour, minute, second:
+ case 'H':
+ handler.on_24_hour(numeric_system::standard, pad);
+ break;
+ case 'I':
+ handler.on_12_hour(numeric_system::standard, pad);
+ break;
+ case 'M':
+ handler.on_minute(numeric_system::standard, pad);
+ break;
+ case 'S':
+ handler.on_second(numeric_system::standard, pad);
+ break;
+ // Other:
+ case 'c':
+ handler.on_datetime(numeric_system::standard);
+ break;
+ case 'x':
+ handler.on_loc_date(numeric_system::standard);
+ break;
+ case 'X':
+ handler.on_loc_time(numeric_system::standard);
+ break;
+ case 'D':
+ handler.on_us_date();
+ break;
+ case 'F':
+ handler.on_iso_date();
+ break;
+ case 'r':
+ handler.on_12_hour_time();
+ break;
+ case 'R':
+ handler.on_24_hour_time();
+ break;
+ case 'T':
+ handler.on_iso_time();
+ break;
+ case 'p':
+ handler.on_am_pm();
+ break;
+ case 'Q':
+ handler.on_duration_value();
+ break;
+ case 'q':
+ handler.on_duration_unit();
+ break;
+ case 'z':
+ handler.on_utc_offset(numeric_system::standard);
+ break;
+ case 'Z':
+ handler.on_tz_name();
+ break;
+ // Alternative representation:
+ case 'E': {
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr++;
+ switch (c) {
+ case 'Y':
+ handler.on_year(numeric_system::alternative);
+ break;
+ case 'y':
+ handler.on_offset_year();
+ break;
+ case 'C':
+ handler.on_century(numeric_system::alternative);
+ break;
+ case 'c':
+ handler.on_datetime(numeric_system::alternative);
+ break;
+ case 'x':
+ handler.on_loc_date(numeric_system::alternative);
+ break;
+ case 'X':
+ handler.on_loc_time(numeric_system::alternative);
+ break;
+ case 'z':
+ handler.on_utc_offset(numeric_system::alternative);
+ break;
+ default:
+ FMT_THROW(format_error("invalid format"));
+ }
+ break;
+ }
+ case 'O':
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr++;
+ switch (c) {
+ case 'y':
+ handler.on_short_year(numeric_system::alternative);
+ break;
+ case 'm':
+ handler.on_dec_month(numeric_system::alternative);
+ break;
+ case 'U':
+ handler.on_dec0_week_of_year(numeric_system::alternative);
+ break;
+ case 'W':
+ handler.on_dec1_week_of_year(numeric_system::alternative);
+ break;
+ case 'V':
+ handler.on_iso_week_of_year(numeric_system::alternative);
+ break;
+ case 'd':
+ handler.on_day_of_month(numeric_system::alternative);
+ break;
+ case 'e':
+ handler.on_day_of_month_space(numeric_system::alternative);
+ break;
+ case 'w':
+ handler.on_dec0_weekday(numeric_system::alternative);
+ break;
+ case 'u':
+ handler.on_dec1_weekday(numeric_system::alternative);
+ break;
+ case 'H':
+ handler.on_24_hour(numeric_system::alternative, pad);
+ break;
+ case 'I':
+ handler.on_12_hour(numeric_system::alternative, pad);
+ break;
+ case 'M':
+ handler.on_minute(numeric_system::alternative, pad);
+ break;
+ case 'S':
+ handler.on_second(numeric_system::alternative, pad);
+ break;
+ case 'z':
+ handler.on_utc_offset(numeric_system::alternative);
+ break;
+ default:
+ FMT_THROW(format_error("invalid format"));
+ }
+ break;
+ default:
+ FMT_THROW(format_error("invalid format"));
+ }
+ begin = ptr;
+ }
+ if (begin != ptr) handler.on_text(begin, ptr);
+ return ptr;
+}
+
+template <typename Derived> struct null_chrono_spec_handler {
+ FMT_CONSTEXPR void unsupported() {
+ static_cast<Derived*>(this)->unsupported();
+ }
+ FMT_CONSTEXPR void on_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_short_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_offset_year() { unsupported(); }
+ FMT_CONSTEXPR void on_century(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_iso_week_based_year() { unsupported(); }
+ FMT_CONSTEXPR void on_iso_week_based_short_year() { unsupported(); }
+ FMT_CONSTEXPR void on_abbr_weekday() { unsupported(); }
+ FMT_CONSTEXPR void on_full_weekday() { unsupported(); }
+ FMT_CONSTEXPR void on_dec0_weekday(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_dec1_weekday(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_abbr_month() { unsupported(); }
+ FMT_CONSTEXPR void on_full_month() { unsupported(); }
+ FMT_CONSTEXPR void on_dec_month(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_iso_week_of_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_day_of_year() { unsupported(); }
+ FMT_CONSTEXPR void on_day_of_month(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_day_of_month_space(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_24_hour(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_12_hour(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_minute(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_second(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_datetime(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_loc_date(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_loc_time(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_us_date() { unsupported(); }
+ FMT_CONSTEXPR void on_iso_date() { unsupported(); }
+ FMT_CONSTEXPR void on_12_hour_time() { unsupported(); }
+ FMT_CONSTEXPR void on_24_hour_time() { unsupported(); }
+ FMT_CONSTEXPR void on_iso_time() { unsupported(); }
+ FMT_CONSTEXPR void on_am_pm() { unsupported(); }
+ FMT_CONSTEXPR void on_duration_value() { unsupported(); }
+ FMT_CONSTEXPR void on_duration_unit() { unsupported(); }
+ FMT_CONSTEXPR void on_utc_offset(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_tz_name() { unsupported(); }
+};
+
+struct tm_format_checker : null_chrono_spec_handler<tm_format_checker> {
+ FMT_NORETURN void unsupported() { FMT_THROW(format_error("no format")); }
+
+ template <typename Char>
+ FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+ FMT_CONSTEXPR void on_year(numeric_system) {}
+ FMT_CONSTEXPR void on_short_year(numeric_system) {}
+ FMT_CONSTEXPR void on_offset_year() {}
+ FMT_CONSTEXPR void on_century(numeric_system) {}
+ FMT_CONSTEXPR void on_iso_week_based_year() {}
+ FMT_CONSTEXPR void on_iso_week_based_short_year() {}
+ FMT_CONSTEXPR void on_abbr_weekday() {}
+ FMT_CONSTEXPR void on_full_weekday() {}
+ FMT_CONSTEXPR void on_dec0_weekday(numeric_system) {}
+ FMT_CONSTEXPR void on_dec1_weekday(numeric_system) {}
+ FMT_CONSTEXPR void on_abbr_month() {}
+ FMT_CONSTEXPR void on_full_month() {}
+ FMT_CONSTEXPR void on_dec_month(numeric_system) {}
+ FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system) {}
+ FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system) {}
+ FMT_CONSTEXPR void on_iso_week_of_year(numeric_system) {}
+ FMT_CONSTEXPR void on_day_of_year() {}
+ FMT_CONSTEXPR void on_day_of_month(numeric_system) {}
+ FMT_CONSTEXPR void on_day_of_month_space(numeric_system) {}
+ FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_second(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_datetime(numeric_system) {}
+ FMT_CONSTEXPR void on_loc_date(numeric_system) {}
+ FMT_CONSTEXPR void on_loc_time(numeric_system) {}
+ FMT_CONSTEXPR void on_us_date() {}
+ FMT_CONSTEXPR void on_iso_date() {}
+ FMT_CONSTEXPR void on_12_hour_time() {}
+ FMT_CONSTEXPR void on_24_hour_time() {}
+ FMT_CONSTEXPR void on_iso_time() {}
+ FMT_CONSTEXPR void on_am_pm() {}
+ FMT_CONSTEXPR void on_utc_offset(numeric_system) {}
+ FMT_CONSTEXPR void on_tz_name() {}
+};
+
+inline const char* tm_wday_full_name(int wday) {
+ static constexpr const char* full_name_list[] = {
+ "Sunday", "Monday", "Tuesday", "Wednesday",
+ "Thursday", "Friday", "Saturday"};
+ return wday >= 0 && wday <= 6 ? full_name_list[wday] : "?";
+}
+inline const char* tm_wday_short_name(int wday) {
+ static constexpr const char* short_name_list[] = {"Sun", "Mon", "Tue", "Wed",
+ "Thu", "Fri", "Sat"};
+ return wday >= 0 && wday <= 6 ? short_name_list[wday] : "???";
+}
+
+inline const char* tm_mon_full_name(int mon) {
+ static constexpr const char* full_name_list[] = {
+ "January", "February", "March", "April", "May", "June",
+ "July", "August", "September", "October", "November", "December"};
+ return mon >= 0 && mon <= 11 ? full_name_list[mon] : "?";
+}
+inline const char* tm_mon_short_name(int mon) {
+ static constexpr const char* short_name_list[] = {
+ "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
+ };
+ return mon >= 0 && mon <= 11 ? short_name_list[mon] : "???";
+}
+
+template <typename T, typename = void>
+struct has_member_data_tm_gmtoff : std::false_type {};
+template <typename T>
+struct has_member_data_tm_gmtoff<T, void_t<decltype(T::tm_gmtoff)>>
+ : std::true_type {};
+
+template <typename T, typename = void>
+struct has_member_data_tm_zone : std::false_type {};
+template <typename T>
+struct has_member_data_tm_zone<T, void_t<decltype(T::tm_zone)>>
+ : std::true_type {};
+
+#if FMT_USE_TZSET
+inline void tzset_once() {
+ static bool init = []() -> bool {
+ _tzset();
+ return true;
+ }();
+ ignore_unused(init);
+}
+#endif
+
+// Converts value to Int and checks that it's in the range [0, upper).
+template <typename T, typename Int, FMT_ENABLE_IF(std::is_integral<T>::value)>
+inline Int to_nonnegative_int(T value, Int upper) {
+ FMT_ASSERT(std::is_unsigned<Int>::value ||
+ (value >= 0 && to_unsigned(value) <= to_unsigned(upper)),
+ "invalid value");
+ (void)upper;
+ return static_cast<Int>(value);
+}
+template <typename T, typename Int, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+inline Int to_nonnegative_int(T value, Int upper) {
+ if (value < 0 || value > static_cast<T>(upper))
+ FMT_THROW(format_error("invalid value"));
+ return static_cast<Int>(value);
+}
+
+constexpr long long pow10(std::uint32_t n) {
+ return n == 0 ? 1 : 10 * pow10(n - 1);
+}
+
+// Counts the number of fractional digits in the range [0, 18] according to the
+// C++20 spec. If more than 18 fractional digits are required then returns 6 for
+// microseconds precision.
+template <long long Num, long long Den, int N = 0,
+ bool Enabled = (N < 19) && (Num <= max_value<long long>() / 10)>
+struct count_fractional_digits {
+ static constexpr int value =
+ Num % Den == 0 ? N : count_fractional_digits<Num * 10, Den, N + 1>::value;
+};
+
+// Base case that doesn't instantiate any more templates
+// in order to avoid overflow.
+template <long long Num, long long Den, int N>
+struct count_fractional_digits<Num, Den, N, false> {
+ static constexpr int value = (Num % Den == 0) ? N : 6;
+};
+
+// Format subseconds which are given as an integer type with an appropriate
+// number of digits.
+template <typename Char, typename OutputIt, typename Duration>
+void write_fractional_seconds(OutputIt& out, Duration d, int precision = -1) {
+ constexpr auto num_fractional_digits =
+ count_fractional_digits<Duration::period::num,
+ Duration::period::den>::value;
+
+ using subsecond_precision = std::chrono::duration<
+ typename std::common_type<typename Duration::rep,
+ std::chrono::seconds::rep>::type,
+ std::ratio<1, detail::pow10(num_fractional_digits)>>;
+
+ const auto fractional =
+ d - std::chrono::duration_cast<std::chrono::seconds>(d);
+ const auto subseconds =
+ std::chrono::treat_as_floating_point<
+ typename subsecond_precision::rep>::value
+ ? fractional.count()
+ : std::chrono::duration_cast<subsecond_precision>(fractional).count();
+ auto n = static_cast<uint32_or_64_or_128_t<long long>>(subseconds);
+ const int num_digits = detail::count_digits(n);
+
+ int leading_zeroes = (std::max)(0, num_fractional_digits - num_digits);
+ if (precision < 0) {
+ FMT_ASSERT(!std::is_floating_point<typename Duration::rep>::value, "");
+ if (std::ratio_less<typename subsecond_precision::period,
+ std::chrono::seconds::period>::value) {
+ *out++ = '.';
+ out = std::fill_n(out, leading_zeroes, '0');
+ out = format_decimal<Char>(out, n, num_digits).end;
+ }
+ } else {
+ *out++ = '.';
+ leading_zeroes = (std::min)(leading_zeroes, precision);
+ out = std::fill_n(out, leading_zeroes, '0');
+ int remaining = precision - leading_zeroes;
+ if (remaining != 0 && remaining < num_digits) {
+ n /= to_unsigned(detail::pow10(to_unsigned(num_digits - remaining)));
+ out = format_decimal<Char>(out, n, remaining).end;
+ return;
+ }
+ out = format_decimal<Char>(out, n, num_digits).end;
+ remaining -= num_digits;
+ out = std::fill_n(out, remaining, '0');
+ }
+}
+
+// Format subseconds which are given as a floating point type with an
+// appropriate number of digits. We cannot pass the Duration here, as we
+// explicitly need to pass the Rep value in the chrono_formatter.
+template <typename Duration>
+void write_floating_seconds(memory_buffer& buf, Duration duration,
+ int num_fractional_digits = -1) {
+ using rep = typename Duration::rep;
+ FMT_ASSERT(std::is_floating_point<rep>::value, "");
+
+ auto val = duration.count();
+
+ if (num_fractional_digits < 0) {
+ // For `std::round` with fallback to `round`:
+ // On some toolchains `std::round` is not available (e.g. GCC 6).
+ using namespace std;
+ num_fractional_digits =
+ count_fractional_digits<Duration::period::num,
+ Duration::period::den>::value;
+ if (num_fractional_digits < 6 && static_cast<rep>(round(val)) != val)
+ num_fractional_digits = 6;
+ }
+
+ format_to(std::back_inserter(buf), FMT_STRING("{:.{}f}"),
+ std::fmod(val * static_cast<rep>(Duration::period::num) /
+ static_cast<rep>(Duration::period::den),
+ static_cast<rep>(60)),
+ num_fractional_digits);
+}
+
+template <typename OutputIt, typename Char,
+ typename Duration = std::chrono::seconds>
+class tm_writer {
+ private:
+ static constexpr int days_per_week = 7;
+
+ const std::locale& loc_;
+ const bool is_classic_;
+ OutputIt out_;
+ const Duration* subsecs_;
+ const std::tm& tm_;
+
+ auto tm_sec() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_sec >= 0 && tm_.tm_sec <= 61, "");
+ return tm_.tm_sec;
+ }
+ auto tm_min() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_min >= 0 && tm_.tm_min <= 59, "");
+ return tm_.tm_min;
+ }
+ auto tm_hour() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_hour >= 0 && tm_.tm_hour <= 23, "");
+ return tm_.tm_hour;
+ }
+ auto tm_mday() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_mday >= 1 && tm_.tm_mday <= 31, "");
+ return tm_.tm_mday;
+ }
+ auto tm_mon() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_mon >= 0 && tm_.tm_mon <= 11, "");
+ return tm_.tm_mon;
+ }
+ auto tm_year() const noexcept -> long long { return 1900ll + tm_.tm_year; }
+ auto tm_wday() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_wday >= 0 && tm_.tm_wday <= 6, "");
+ return tm_.tm_wday;
+ }
+ auto tm_yday() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_yday >= 0 && tm_.tm_yday <= 365, "");
+ return tm_.tm_yday;
+ }
+
+ auto tm_hour12() const noexcept -> int {
+ const auto h = tm_hour();
+ const auto z = h < 12 ? h : h - 12;
+ return z == 0 ? 12 : z;
+ }
+
+ // POSIX and the C Standard are unclear or inconsistent about what %C and %y
+ // do if the year is negative or exceeds 9999. Use the convention that %C
+ // concatenated with %y yields the same output as %Y, and that %Y contains at
+ // least 4 characters, with more only if necessary.
+ auto split_year_lower(long long year) const noexcept -> int {
+ auto l = year % 100;
+ if (l < 0) l = -l; // l in [0, 99]
+ return static_cast<int>(l);
+ }
+
+ // Algorithm:
+ // https://en.wikipedia.org/wiki/ISO_week_date#Calculating_the_week_number_from_a_month_and_day_of_the_month_or_ordinal_date
+ auto iso_year_weeks(long long curr_year) const noexcept -> int {
+ const auto prev_year = curr_year - 1;
+ const auto curr_p =
+ (curr_year + curr_year / 4 - curr_year / 100 + curr_year / 400) %
+ days_per_week;
+ const auto prev_p =
+ (prev_year + prev_year / 4 - prev_year / 100 + prev_year / 400) %
+ days_per_week;
+ return 52 + ((curr_p == 4 || prev_p == 3) ? 1 : 0);
+ }
+ auto iso_week_num(int tm_yday, int tm_wday) const noexcept -> int {
+ return (tm_yday + 11 - (tm_wday == 0 ? days_per_week : tm_wday)) /
+ days_per_week;
+ }
+ auto tm_iso_week_year() const noexcept -> long long {
+ const auto year = tm_year();
+ const auto w = iso_week_num(tm_yday(), tm_wday());
+ if (w < 1) return year - 1;
+ if (w > iso_year_weeks(year)) return year + 1;
+ return year;
+ }
+ auto tm_iso_week_of_year() const noexcept -> int {
+ const auto year = tm_year();
+ const auto w = iso_week_num(tm_yday(), tm_wday());
+ if (w < 1) return iso_year_weeks(year - 1);
+ if (w > iso_year_weeks(year)) return 1;
+ return w;
+ }
+
+ void write1(int value) {
+ *out_++ = static_cast<char>('0' + to_unsigned(value) % 10);
+ }
+ void write2(int value) {
+ const char* d = digits2(to_unsigned(value) % 100);
+ *out_++ = *d++;
+ *out_++ = *d;
+ }
+ void write2(int value, pad_type pad) {
+ unsigned int v = to_unsigned(value) % 100;
+ if (v >= 10) {
+ const char* d = digits2(v);
+ *out_++ = *d++;
+ *out_++ = *d;
+ } else {
+ out_ = detail::write_padding(out_, pad);
+ *out_++ = static_cast<char>('0' + v);
+ }
+ }
+
+ void write_year_extended(long long year) {
+ // At least 4 characters.
+ int width = 4;
+ if (year < 0) {
+ *out_++ = '-';
+ year = 0 - year;
+ --width;
+ }
+ uint32_or_64_or_128_t<long long> n = to_unsigned(year);
+ const int num_digits = count_digits(n);
+ if (width > num_digits) out_ = std::fill_n(out_, width - num_digits, '0');
+ out_ = format_decimal<Char>(out_, n, num_digits).end;
+ }
+ void write_year(long long year) {
+ if (year >= 0 && year < 10000) {
+ write2(static_cast<int>(year / 100));
+ write2(static_cast<int>(year % 100));
+ } else {
+ write_year_extended(year);
+ }
+ }
+
+ void write_utc_offset(long offset, numeric_system ns) {
+ if (offset < 0) {
+ *out_++ = '-';
+ offset = -offset;
+ } else {
+ *out_++ = '+';
+ }
+ offset /= 60;
+ write2(static_cast<int>(offset / 60));
+ if (ns != numeric_system::standard) *out_++ = ':';
+ write2(static_cast<int>(offset % 60));
+ }
+ template <typename T, FMT_ENABLE_IF(has_member_data_tm_gmtoff<T>::value)>
+ void format_utc_offset_impl(const T& tm, numeric_system ns) {
+ write_utc_offset(tm.tm_gmtoff, ns);
+ }
+ template <typename T, FMT_ENABLE_IF(!has_member_data_tm_gmtoff<T>::value)>
+ void format_utc_offset_impl(const T& tm, numeric_system ns) {
+#if defined(_WIN32) && defined(_UCRT)
+# if FMT_USE_TZSET
+ tzset_once();
+# endif
+ long offset = 0;
+ _get_timezone(&offset);
+ if (tm.tm_isdst) {
+ long dstbias = 0;
+ _get_dstbias(&dstbias);
+ offset += dstbias;
+ }
+ write_utc_offset(-offset, ns);
+#else
+ if (ns == numeric_system::standard) return format_localized('z');
+
+ // Extract timezone offset from timezone conversion functions.
+ std::tm gtm = tm;
+ std::time_t gt = std::mktime(&gtm);
+ std::tm ltm = gmtime(gt);
+ std::time_t lt = std::mktime(&ltm);
+ long offset = gt - lt;
+ write_utc_offset(offset, ns);
+#endif
+ }
+
+ template <typename T, FMT_ENABLE_IF(has_member_data_tm_zone<T>::value)>
+ void format_tz_name_impl(const T& tm) {
+ if (is_classic_)
+ out_ = write_tm_str<Char>(out_, tm.tm_zone, loc_);
+ else
+ format_localized('Z');
+ }
+ template <typename T, FMT_ENABLE_IF(!has_member_data_tm_zone<T>::value)>
+ void format_tz_name_impl(const T&) {
+ format_localized('Z');
+ }
+
+ void format_localized(char format, char modifier = 0) {
+ out_ = write<Char>(out_, tm_, loc_, format, modifier);
+ }
+
+ public:
+ tm_writer(const std::locale& loc, OutputIt out, const std::tm& tm,
+ const Duration* subsecs = nullptr)
+ : loc_(loc),
+ is_classic_(loc_ == get_classic_locale()),
+ out_(out),
+ subsecs_(subsecs),
+ tm_(tm) {}
+
+ OutputIt out() const { return out_; }
+
+ FMT_CONSTEXPR void on_text(const Char* begin, const Char* end) {
+ out_ = copy_str<Char>(begin, end, out_);
+ }
+
+ void on_abbr_weekday() {
+ if (is_classic_)
+ out_ = write(out_, tm_wday_short_name(tm_wday()));
+ else
+ format_localized('a');
+ }
+ void on_full_weekday() {
+ if (is_classic_)
+ out_ = write(out_, tm_wday_full_name(tm_wday()));
+ else
+ format_localized('A');
+ }
+ void on_dec0_weekday(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) return write1(tm_wday());
+ format_localized('w', 'O');
+ }
+ void on_dec1_weekday(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto wday = tm_wday();
+ write1(wday == 0 ? days_per_week : wday);
+ } else {
+ format_localized('u', 'O');
+ }
+ }
+
+ void on_abbr_month() {
+ if (is_classic_)
+ out_ = write(out_, tm_mon_short_name(tm_mon()));
+ else
+ format_localized('b');
+ }
+ void on_full_month() {
+ if (is_classic_)
+ out_ = write(out_, tm_mon_full_name(tm_mon()));
+ else
+ format_localized('B');
+ }
+
+ void on_datetime(numeric_system ns) {
+ if (is_classic_) {
+ on_abbr_weekday();
+ *out_++ = ' ';
+ on_abbr_month();
+ *out_++ = ' ';
+ on_day_of_month_space(numeric_system::standard);
+ *out_++ = ' ';
+ on_iso_time();
+ *out_++ = ' ';
+ on_year(numeric_system::standard);
+ } else {
+ format_localized('c', ns == numeric_system::standard ? '\0' : 'E');
+ }
+ }
+ void on_loc_date(numeric_system ns) {
+ if (is_classic_)
+ on_us_date();
+ else
+ format_localized('x', ns == numeric_system::standard ? '\0' : 'E');
+ }
+ void on_loc_time(numeric_system ns) {
+ if (is_classic_)
+ on_iso_time();
+ else
+ format_localized('X', ns == numeric_system::standard ? '\0' : 'E');
+ }
+ void on_us_date() {
+ char buf[8];
+ write_digit2_separated(buf, to_unsigned(tm_mon() + 1),
+ to_unsigned(tm_mday()),
+ to_unsigned(split_year_lower(tm_year())), '/');
+ out_ = copy_str<Char>(std::begin(buf), std::end(buf), out_);
+ }
+ void on_iso_date() {
+ auto year = tm_year();
+ char buf[10];
+ size_t offset = 0;
+ if (year >= 0 && year < 10000) {
+ copy2(buf, digits2(static_cast<size_t>(year / 100)));
+ } else {
+ offset = 4;
+ write_year_extended(year);
+ year = 0;
+ }
+ write_digit2_separated(buf + 2, static_cast<unsigned>(year % 100),
+ to_unsigned(tm_mon() + 1), to_unsigned(tm_mday()),
+ '-');
+ out_ = copy_str<Char>(std::begin(buf) + offset, std::end(buf), out_);
+ }
+
+ void on_utc_offset(numeric_system ns) { format_utc_offset_impl(tm_, ns); }
+ void on_tz_name() { format_tz_name_impl(tm_); }
+
+ void on_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write_year(tm_year());
+ format_localized('Y', 'E');
+ }
+ void on_short_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(split_year_lower(tm_year()));
+ format_localized('y', 'O');
+ }
+ void on_offset_year() {
+ if (is_classic_) return write2(split_year_lower(tm_year()));
+ format_localized('y', 'E');
+ }
+
+ void on_century(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto year = tm_year();
+ auto upper = year / 100;
+ if (year >= -99 && year < 0) {
+ // Zero upper on negative year.
+ *out_++ = '-';
+ *out_++ = '0';
+ } else if (upper >= 0 && upper < 100) {
+ write2(static_cast<int>(upper));
+ } else {
+ out_ = write<Char>(out_, upper);
+ }
+ } else {
+ format_localized('C', 'E');
+ }
+ }
+
+ void on_dec_month(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_mon() + 1);
+ format_localized('m', 'O');
+ }
+
+ void on_dec0_week_of_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2((tm_yday() + days_per_week - tm_wday()) / days_per_week);
+ format_localized('U', 'O');
+ }
+ void on_dec1_week_of_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto wday = tm_wday();
+ write2((tm_yday() + days_per_week -
+ (wday == 0 ? (days_per_week - 1) : (wday - 1))) /
+ days_per_week);
+ } else {
+ format_localized('W', 'O');
+ }
+ }
+ void on_iso_week_of_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_iso_week_of_year());
+ format_localized('V', 'O');
+ }
+
+ void on_iso_week_based_year() { write_year(tm_iso_week_year()); }
+ void on_iso_week_based_short_year() {
+ write2(split_year_lower(tm_iso_week_year()));
+ }
+
+ void on_day_of_year() {
+ auto yday = tm_yday() + 1;
+ write1(yday / 100);
+ write2(yday % 100);
+ }
+ void on_day_of_month(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) return write2(tm_mday());
+ format_localized('d', 'O');
+ }
+ void on_day_of_month_space(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto mday = to_unsigned(tm_mday()) % 100;
+ const char* d2 = digits2(mday);
+ *out_++ = mday < 10 ? ' ' : d2[0];
+ *out_++ = d2[1];
+ } else {
+ format_localized('e', 'O');
+ }
+ }
+
+ void on_24_hour(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_hour(), pad);
+ format_localized('H', 'O');
+ }
+ void on_12_hour(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_hour12(), pad);
+ format_localized('I', 'O');
+ }
+ void on_minute(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_min(), pad);
+ format_localized('M', 'O');
+ }
+
+ void on_second(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ write2(tm_sec(), pad);
+ if (subsecs_) {
+ if (std::is_floating_point<typename Duration::rep>::value) {
+ auto buf = memory_buffer();
+ write_floating_seconds(buf, *subsecs_);
+ if (buf.size() > 1) {
+ // Remove the leading "0", write something like ".123".
+ out_ = std::copy(buf.begin() + 1, buf.end(), out_);
+ }
+ } else {
+ write_fractional_seconds<Char>(out_, *subsecs_);
+ }
+ }
+ } else {
+ // Currently no formatting of subseconds when a locale is set.
+ format_localized('S', 'O');
+ }
+ }
+
+ void on_12_hour_time() {
+ if (is_classic_) {
+ char buf[8];
+ write_digit2_separated(buf, to_unsigned(tm_hour12()),
+ to_unsigned(tm_min()), to_unsigned(tm_sec()), ':');
+ out_ = copy_str<Char>(std::begin(buf), std::end(buf), out_);
+ *out_++ = ' ';
+ on_am_pm();
+ } else {
+ format_localized('r');
+ }
+ }
+ void on_24_hour_time() {
+ write2(tm_hour());
+ *out_++ = ':';
+ write2(tm_min());
+ }
+ void on_iso_time() {
+ on_24_hour_time();
+ *out_++ = ':';
+ on_second(numeric_system::standard, pad_type::unspecified);
+ }
+
+ void on_am_pm() {
+ if (is_classic_) {
+ *out_++ = tm_hour() < 12 ? 'A' : 'P';
+ *out_++ = 'M';
+ } else {
+ format_localized('p');
+ }
+ }
+
+ // These apply to chrono durations but not tm.
+ void on_duration_value() {}
+ void on_duration_unit() {}
+};
+
+struct chrono_format_checker : null_chrono_spec_handler<chrono_format_checker> {
+ bool has_precision_integral = false;
+
+ FMT_NORETURN void unsupported() { FMT_THROW(format_error("no date")); }
+
+ template <typename Char>
+ FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+ FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_second(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_12_hour_time() {}
+ FMT_CONSTEXPR void on_24_hour_time() {}
+ FMT_CONSTEXPR void on_iso_time() {}
+ FMT_CONSTEXPR void on_am_pm() {}
+ FMT_CONSTEXPR void on_duration_value() const {
+ if (has_precision_integral) {
+ FMT_THROW(format_error("precision not allowed for this argument type"));
+ }
+ }
+ FMT_CONSTEXPR void on_duration_unit() {}
+};
+
+template <typename T,
+ FMT_ENABLE_IF(std::is_integral<T>::value&& has_isfinite<T>::value)>
+inline bool isfinite(T) {
+ return true;
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+inline T mod(T x, int y) {
+ return x % static_cast<T>(y);
+}
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+inline T mod(T x, int y) {
+ return std::fmod(x, static_cast<T>(y));
+}
+
+// If T is an integral type, maps T to its unsigned counterpart, otherwise
+// leaves it unchanged (unlike std::make_unsigned).
+template <typename T, bool INTEGRAL = std::is_integral<T>::value>
+struct make_unsigned_or_unchanged {
+ using type = T;
+};
+
+template <typename T> struct make_unsigned_or_unchanged<T, true> {
+ using type = typename std::make_unsigned<T>::type;
+};
+
+#if FMT_SAFE_DURATION_CAST
+// throwing version of safe_duration_cast
+template <typename To, typename FromRep, typename FromPeriod>
+To fmt_safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from) {
+ int ec;
+ To to = safe_duration_cast::safe_duration_cast<To>(from, ec);
+ if (ec) FMT_THROW(format_error("cannot format duration"));
+ return to;
+}
+#endif
+
+template <typename Rep, typename Period,
+ FMT_ENABLE_IF(std::is_integral<Rep>::value)>
+inline std::chrono::duration<Rep, std::milli> get_milliseconds(
+ std::chrono::duration<Rep, Period> d) {
+ // this may overflow and/or the result may not fit in the
+ // target type.
+#if FMT_SAFE_DURATION_CAST
+ using CommonSecondsType =
+ typename std::common_type<decltype(d), std::chrono::seconds>::type;
+ const auto d_as_common = fmt_safe_duration_cast<CommonSecondsType>(d);
+ const auto d_as_whole_seconds =
+ fmt_safe_duration_cast<std::chrono::seconds>(d_as_common);
+ // this conversion should be nonproblematic
+ const auto diff = d_as_common - d_as_whole_seconds;
+ const auto ms =
+ fmt_safe_duration_cast<std::chrono::duration<Rep, std::milli>>(diff);
+ return ms;
+#else
+ auto s = std::chrono::duration_cast<std::chrono::seconds>(d);
+ return std::chrono::duration_cast<std::chrono::milliseconds>(d - s);
+#endif
+}
+
+template <typename Char, typename Rep, typename OutputIt,
+ FMT_ENABLE_IF(std::is_integral<Rep>::value)>
+OutputIt format_duration_value(OutputIt out, Rep val, int) {
+ return write<Char>(out, val);
+}
+
+template <typename Char, typename Rep, typename OutputIt,
+ FMT_ENABLE_IF(std::is_floating_point<Rep>::value)>
+OutputIt format_duration_value(OutputIt out, Rep val, int precision) {
+ auto specs = format_specs<Char>();
+ specs.precision = precision;
+ specs.type = precision >= 0 ? presentation_type::fixed_lower
+ : presentation_type::general_lower;
+ return write<Char>(out, val, specs);
+}
+
+template <typename Char, typename OutputIt>
+OutputIt copy_unit(string_view unit, OutputIt out, Char) {
+ return std::copy(unit.begin(), unit.end(), out);
+}
+
+template <typename OutputIt>
+OutputIt copy_unit(string_view unit, OutputIt out, wchar_t) {
+ // This works when wchar_t is UTF-32 because units only contain characters
+ // that have the same representation in UTF-16 and UTF-32.
+ utf8_to_utf16 u(unit);
+ return std::copy(u.c_str(), u.c_str() + u.size(), out);
+}
+
+template <typename Char, typename Period, typename OutputIt>
+OutputIt format_duration_unit(OutputIt out) {
+ if (const char* unit = get_units<Period>())
+ return copy_unit(string_view(unit), out, Char());
+ *out++ = '[';
+ out = write<Char>(out, Period::num);
+ if (const_check(Period::den != 1)) {
+ *out++ = '/';
+ out = write<Char>(out, Period::den);
+ }
+ *out++ = ']';
+ *out++ = 's';
+ return out;
+}
+
+class get_locale {
+ private:
+ union {
+ std::locale locale_;
+ };
+ bool has_locale_ = false;
+
+ public:
+ get_locale(bool localized, locale_ref loc) : has_locale_(localized) {
+ if (localized)
+ ::new (&locale_) std::locale(loc.template get<std::locale>());
+ }
+ ~get_locale() {
+ if (has_locale_) locale_.~locale();
+ }
+ operator const std::locale&() const {
+ return has_locale_ ? locale_ : get_classic_locale();
+ }
+};
+
+template <typename FormatContext, typename OutputIt, typename Rep,
+ typename Period>
+struct chrono_formatter {
+ FormatContext& context;
+ OutputIt out;
+ int precision;
+ bool localized = false;
+ // rep is unsigned to avoid overflow.
+ using rep =
+ conditional_t<std::is_integral<Rep>::value && sizeof(Rep) < sizeof(int),
+ unsigned, typename make_unsigned_or_unchanged<Rep>::type>;
+ rep val;
+ using seconds = std::chrono::duration<rep>;
+ seconds s;
+ using milliseconds = std::chrono::duration<rep, std::milli>;
+ bool negative;
+
+ using char_type = typename FormatContext::char_type;
+ using tm_writer_type = tm_writer<OutputIt, char_type>;
+
+ chrono_formatter(FormatContext& ctx, OutputIt o,
+ std::chrono::duration<Rep, Period> d)
+ : context(ctx),
+ out(o),
+ val(static_cast<rep>(d.count())),
+ negative(false) {
+ if (d.count() < 0) {
+ val = 0 - val;
+ negative = true;
+ }
+
+ // this may overflow and/or the result may not fit in the
+ // target type.
+#if FMT_SAFE_DURATION_CAST
+ // might need checked conversion (rep!=Rep)
+ auto tmpval = std::chrono::duration<rep, Period>(val);
+ s = fmt_safe_duration_cast<seconds>(tmpval);
+#else
+ s = std::chrono::duration_cast<seconds>(
+ std::chrono::duration<rep, Period>(val));
+#endif
+ }
+
+ // returns true if nan or inf, writes to out.
+ bool handle_nan_inf() {
+ if (isfinite(val)) {
+ return false;
+ }
+ if (isnan(val)) {
+ write_nan();
+ return true;
+ }
+ // must be +-inf
+ if (val > 0) {
+ write_pinf();
+ } else {
+ write_ninf();
+ }
+ return true;
+ }
+
+ Rep hour() const { return static_cast<Rep>(mod((s.count() / 3600), 24)); }
+
+ Rep hour12() const {
+ Rep hour = static_cast<Rep>(mod((s.count() / 3600), 12));
+ return hour <= 0 ? 12 : hour;
+ }
+
+ Rep minute() const { return static_cast<Rep>(mod((s.count() / 60), 60)); }
+ Rep second() const { return static_cast<Rep>(mod(s.count(), 60)); }
+
+ std::tm time() const {
+ auto time = std::tm();
+ time.tm_hour = to_nonnegative_int(hour(), 24);
+ time.tm_min = to_nonnegative_int(minute(), 60);
+ time.tm_sec = to_nonnegative_int(second(), 60);
+ return time;
+ }
+
+ void write_sign() {
+ if (negative) {
+ *out++ = '-';
+ negative = false;
+ }
+ }
+
+ void write(Rep value, int width, pad_type pad = pad_type::unspecified) {
+ write_sign();
+ if (isnan(value)) return write_nan();
+ uint32_or_64_or_128_t<int> n =
+ to_unsigned(to_nonnegative_int(value, max_value<int>()));
+ int num_digits = detail::count_digits(n);
+ if (width > num_digits) {
+ out = detail::write_padding(out, pad, width - num_digits);
+ }
+ out = format_decimal<char_type>(out, n, num_digits).end;
+ }
+
+ void write_nan() { std::copy_n("nan", 3, out); }
+ void write_pinf() { std::copy_n("inf", 3, out); }
+ void write_ninf() { std::copy_n("-inf", 4, out); }
+
+ template <typename Callback, typename... Args>
+ void format_tm(const tm& time, Callback cb, Args... args) {
+ if (isnan(val)) return write_nan();
+ get_locale loc(localized, context.locale());
+ auto w = tm_writer_type(loc, out, time);
+ (w.*cb)(args...);
+ out = w.out();
+ }
+
+ void on_text(const char_type* begin, const char_type* end) {
+ std::copy(begin, end, out);
+ }
+
+ // These are not implemented because durations don't have date information.
+ void on_abbr_weekday() {}
+ void on_full_weekday() {}
+ void on_dec0_weekday(numeric_system) {}
+ void on_dec1_weekday(numeric_system) {}
+ void on_abbr_month() {}
+ void on_full_month() {}
+ void on_datetime(numeric_system) {}
+ void on_loc_date(numeric_system) {}
+ void on_loc_time(numeric_system) {}
+ void on_us_date() {}
+ void on_iso_date() {}
+ void on_utc_offset(numeric_system) {}
+ void on_tz_name() {}
+ void on_year(numeric_system) {}
+ void on_short_year(numeric_system) {}
+ void on_offset_year() {}
+ void on_century(numeric_system) {}
+ void on_iso_week_based_year() {}
+ void on_iso_week_based_short_year() {}
+ void on_dec_month(numeric_system) {}
+ void on_dec0_week_of_year(numeric_system) {}
+ void on_dec1_week_of_year(numeric_system) {}
+ void on_iso_week_of_year(numeric_system) {}
+ void on_day_of_year() {}
+ void on_day_of_month(numeric_system) {}
+ void on_day_of_month_space(numeric_system) {}
+
+ void on_24_hour(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) return write(hour(), 2, pad);
+ auto time = tm();
+ time.tm_hour = to_nonnegative_int(hour(), 24);
+ format_tm(time, &tm_writer_type::on_24_hour, ns, pad);
+ }
+
+ void on_12_hour(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) return write(hour12(), 2, pad);
+ auto time = tm();
+ time.tm_hour = to_nonnegative_int(hour12(), 12);
+ format_tm(time, &tm_writer_type::on_12_hour, ns, pad);
+ }
+
+ void on_minute(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) return write(minute(), 2, pad);
+ auto time = tm();
+ time.tm_min = to_nonnegative_int(minute(), 60);
+ format_tm(time, &tm_writer_type::on_minute, ns, pad);
+ }
+
+ void on_second(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) {
+ if (std::is_floating_point<rep>::value) {
+ auto buf = memory_buffer();
+ write_floating_seconds(buf, std::chrono::duration<rep, Period>(val),
+ precision);
+ if (negative) *out++ = '-';
+ if (buf.size() < 2 || buf[1] == '.') {
+ out = detail::write_padding(out, pad);
+ }
+ out = std::copy(buf.begin(), buf.end(), out);
+ } else {
+ write(second(), 2, pad);
+ write_fractional_seconds<char_type>(
+ out, std::chrono::duration<rep, Period>(val), precision);
+ }
+ return;
+ }
+ auto time = tm();
+ time.tm_sec = to_nonnegative_int(second(), 60);
+ format_tm(time, &tm_writer_type::on_second, ns, pad);
+ }
+
+ void on_12_hour_time() {
+ if (handle_nan_inf()) return;
+ format_tm(time(), &tm_writer_type::on_12_hour_time);
+ }
+
+ void on_24_hour_time() {
+ if (handle_nan_inf()) {
+ *out++ = ':';
+ handle_nan_inf();
+ return;
+ }
+
+ write(hour(), 2);
+ *out++ = ':';
+ write(minute(), 2);
+ }
+
+ void on_iso_time() {
+ on_24_hour_time();
+ *out++ = ':';
+ if (handle_nan_inf()) return;
+ on_second(numeric_system::standard, pad_type::unspecified);
+ }
+
+ void on_am_pm() {
+ if (handle_nan_inf()) return;
+ format_tm(time(), &tm_writer_type::on_am_pm);
+ }
+
+ void on_duration_value() {
+ if (handle_nan_inf()) return;
+ write_sign();
+ out = format_duration_value<char_type>(out, val, precision);
+ }
+
+ void on_duration_unit() {
+ out = format_duration_unit<char_type, Period>(out);
+ }
+};
+
+FMT_END_DETAIL_NAMESPACE
+
+#if defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907
+using weekday = std::chrono::weekday;
+#else
+// A fallback version of weekday.
+class weekday {
+ private:
+ unsigned char value;
+
+ public:
+ weekday() = default;
+ explicit constexpr weekday(unsigned wd) noexcept
+ : value(static_cast<unsigned char>(wd != 7 ? wd : 0)) {}
+ constexpr unsigned c_encoding() const noexcept { return value; }
+};
+
+class year_month_day {};
+#endif
+
+// A rudimentary weekday formatter.
+template <typename Char> struct formatter<weekday, Char> {
+ private:
+ bool localized = false;
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto begin = ctx.begin(), end = ctx.end();
+ if (begin != end && *begin == 'L') {
+ ++begin;
+ localized = true;
+ }
+ return begin;
+ }
+
+ template <typename FormatContext>
+ auto format(weekday wd, FormatContext& ctx) const -> decltype(ctx.out()) {
+ auto time = std::tm();
+ time.tm_wday = static_cast<int>(wd.c_encoding());
+ detail::get_locale loc(localized, ctx.locale());
+ auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+ w.on_abbr_weekday();
+ return w.out();
+ }
+};
+
+template <typename Rep, typename Period, typename Char>
+struct formatter<std::chrono::duration<Rep, Period>, Char> {
+ private:
+ format_specs<Char> specs;
+ int precision = -1;
+ using arg_ref_type = detail::arg_ref<Char>;
+ arg_ref_type width_ref;
+ arg_ref_type precision_ref;
+ bool localized = false;
+ basic_string_view<Char> format_str;
+ using duration = std::chrono::duration<Rep, Period>;
+
+ using iterator = typename basic_format_parse_context<Char>::iterator;
+ struct parse_range {
+ iterator begin;
+ iterator end;
+ };
+
+ FMT_CONSTEXPR parse_range do_parse(basic_format_parse_context<Char>& ctx) {
+ auto begin = ctx.begin(), end = ctx.end();
+ if (begin == end || *begin == '}') return {begin, begin};
+
+ begin = detail::parse_align(begin, end, specs);
+ if (begin == end) return {begin, begin};
+
+ begin = detail::parse_dynamic_spec(begin, end, specs.width, width_ref, ctx);
+ if (begin == end) return {begin, begin};
+
+ auto checker = detail::chrono_format_checker();
+ if (*begin == '.') {
+ checker.has_precision_integral = !std::is_floating_point<Rep>::value;
+ begin =
+ detail::parse_precision(begin, end, precision, precision_ref, ctx);
+ }
+ if (begin != end && *begin == 'L') {
+ ++begin;
+ localized = true;
+ }
+ end = detail::parse_chrono_format(begin, end, checker);
+ return {begin, end};
+ }
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto range = do_parse(ctx);
+ format_str = basic_string_view<Char>(
+ &*range.begin, detail::to_unsigned(range.end - range.begin));
+ return range.end;
+ }
+
+ template <typename FormatContext>
+ auto format(const duration& d, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto specs_copy = specs;
+ auto precision_copy = precision;
+ auto begin = format_str.begin(), end = format_str.end();
+ // As a possible future optimization, we could avoid extra copying if width
+ // is not specified.
+ basic_memory_buffer<Char> buf;
+ auto out = std::back_inserter(buf);
+ detail::handle_dynamic_spec<detail::width_checker>(specs_copy.width,
+ width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(precision_copy,
+ precision_ref, ctx);
+ if (begin == end || *begin == '}') {
+ out = detail::format_duration_value<Char>(out, d.count(), precision_copy);
+ detail::format_duration_unit<Char, Period>(out);
+ } else {
+ detail::chrono_formatter<FormatContext, decltype(out), Rep, Period> f(
+ ctx, out, d);
+ f.precision = precision_copy;
+ f.localized = localized;
+ detail::parse_chrono_format(begin, end, f);
+ }
+ return detail::write(
+ ctx.out(), basic_string_view<Char>(buf.data(), buf.size()), specs_copy);
+ }
+};
+
+template <typename Char, typename Duration>
+struct formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+ Char> : formatter<std::tm, Char> {
+ FMT_CONSTEXPR formatter() {
+ this->format_str = detail::string_literal<Char, '%', 'F', ' ', '%', 'T'>{};
+ }
+
+ template <typename FormatContext>
+ auto format(std::chrono::time_point<std::chrono::system_clock, Duration> val,
+ FormatContext& ctx) const -> decltype(ctx.out()) {
+ using period = typename Duration::period;
+ if (period::num != 1 || period::den != 1 ||
+ std::is_floating_point<typename Duration::rep>::value) {
+ const auto epoch = val.time_since_epoch();
+ auto subsecs = std::chrono::duration_cast<Duration>(
+ epoch - std::chrono::duration_cast<std::chrono::seconds>(epoch));
+
+ if (subsecs.count() < 0) {
+ auto second = std::chrono::seconds(1);
+ if (epoch.count() < ((Duration::min)() + second).count())
+ FMT_THROW(format_error("duration is too small"));
+ subsecs += second;
+ val -= second;
+ }
+
+ return formatter<std::tm, Char>::do_format(
+ gmtime(std::chrono::time_point_cast<std::chrono::seconds>(val)), ctx,
+ &subsecs);
+ }
+
+ return formatter<std::tm, Char>::format(
+ gmtime(std::chrono::time_point_cast<std::chrono::seconds>(val)), ctx);
+ }
+};
+
+#if FMT_USE_LOCAL_TIME
+template <typename Char, typename Duration>
+struct formatter<std::chrono::local_time<Duration>, Char>
+ : formatter<std::tm, Char> {
+ FMT_CONSTEXPR formatter() {
+ this->format_str = detail::string_literal<Char, '%', 'F', ' ', '%', 'T'>{};
+ }
+
+ template <typename FormatContext>
+ auto format(std::chrono::local_time<Duration> val, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ using period = typename Duration::period;
+ if (period::num != 1 || period::den != 1 ||
+ std::is_floating_point<typename Duration::rep>::value) {
+ const auto epoch = val.time_since_epoch();
+ const auto subsecs = std::chrono::duration_cast<Duration>(
+ epoch - std::chrono::duration_cast<std::chrono::seconds>(epoch));
+
+ return formatter<std::tm, Char>::do_format(
+ localtime(std::chrono::time_point_cast<std::chrono::seconds>(val)),
+ ctx, &subsecs);
+ }
+
+ return formatter<std::tm, Char>::format(
+ localtime(std::chrono::time_point_cast<std::chrono::seconds>(val)),
+ ctx);
+ }
+};
+#endif
+
+#if FMT_USE_UTC_TIME
+template <typename Char, typename Duration>
+struct formatter<std::chrono::time_point<std::chrono::utc_clock, Duration>,
+ Char>
+ : formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+ Char> {
+ template <typename FormatContext>
+ auto format(std::chrono::time_point<std::chrono::utc_clock, Duration> val,
+ FormatContext& ctx) const -> decltype(ctx.out()) {
+ return formatter<
+ std::chrono::time_point<std::chrono::system_clock, Duration>,
+ Char>::format(std::chrono::utc_clock::to_sys(val), ctx);
+ }
+};
+#endif
+
+template <typename Char> struct formatter<std::tm, Char> {
+ private:
+ format_specs<Char> specs;
+ detail::arg_ref<Char> width_ref;
+
+ protected:
+ basic_string_view<Char> format_str;
+
+ FMT_CONSTEXPR auto do_parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto begin = ctx.begin(), end = ctx.end();
+ if (begin == end || *begin == '}') return begin;
+
+ begin = detail::parse_align(begin, end, specs);
+ if (begin == end) return end;
+
+ begin = detail::parse_dynamic_spec(begin, end, specs.width, width_ref, ctx);
+ if (begin == end) return end;
+
+ end = detail::parse_chrono_format(begin, end, detail::tm_format_checker());
+ // Replace default format_str only if the new spec is not empty.
+ if (end != begin) format_str = {begin, detail::to_unsigned(end - begin)};
+ return end;
+ }
+
+ template <typename FormatContext, typename Duration>
+ auto do_format(const std::tm& tm, FormatContext& ctx,
+ const Duration* subsecs) const -> decltype(ctx.out()) {
+ auto specs_copy = specs;
+ basic_memory_buffer<Char> buf;
+ auto out = std::back_inserter(buf);
+ detail::handle_dynamic_spec<detail::width_checker>(specs_copy.width,
+ width_ref, ctx);
+
+ const auto loc_ref = ctx.locale();
+ detail::get_locale loc(static_cast<bool>(loc_ref), loc_ref);
+ auto w =
+ detail::tm_writer<decltype(out), Char, Duration>(loc, out, tm, subsecs);
+ detail::parse_chrono_format(format_str.begin(), format_str.end(), w);
+ return detail::write(
+ ctx.out(), basic_string_view<Char>(buf.data(), buf.size()), specs_copy);
+ }
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ return this->do_parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(const std::tm& tm, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ return do_format<FormatContext, std::chrono::seconds>(tm, ctx, nullptr);
+ }
+};
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_CHRONO_H_
diff --git a/contrib/fmt/include/fmt/color.h b/contrib/fmt/include/fmt/color.h
new file mode 100644
index 0000000..d175448
--- /dev/null
+++ b/contrib/fmt/include/fmt/color.h
@@ -0,0 +1,633 @@
+// Formatting library for C++ - color support
+//
+// Copyright (c) 2018 - present, Victor Zverovich and fmt contributors
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_COLOR_H_
+#define FMT_COLOR_H_
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+enum class color : uint32_t {
+ alice_blue = 0xF0F8FF, // rgb(240,248,255)
+ antique_white = 0xFAEBD7, // rgb(250,235,215)
+ aqua = 0x00FFFF, // rgb(0,255,255)
+ aquamarine = 0x7FFFD4, // rgb(127,255,212)
+ azure = 0xF0FFFF, // rgb(240,255,255)
+ beige = 0xF5F5DC, // rgb(245,245,220)
+ bisque = 0xFFE4C4, // rgb(255,228,196)
+ black = 0x000000, // rgb(0,0,0)
+ blanched_almond = 0xFFEBCD, // rgb(255,235,205)
+ blue = 0x0000FF, // rgb(0,0,255)
+ blue_violet = 0x8A2BE2, // rgb(138,43,226)
+ brown = 0xA52A2A, // rgb(165,42,42)
+ burly_wood = 0xDEB887, // rgb(222,184,135)
+ cadet_blue = 0x5F9EA0, // rgb(95,158,160)
+ chartreuse = 0x7FFF00, // rgb(127,255,0)
+ chocolate = 0xD2691E, // rgb(210,105,30)
+ coral = 0xFF7F50, // rgb(255,127,80)
+ cornflower_blue = 0x6495ED, // rgb(100,149,237)
+ cornsilk = 0xFFF8DC, // rgb(255,248,220)
+ crimson = 0xDC143C, // rgb(220,20,60)
+ cyan = 0x00FFFF, // rgb(0,255,255)
+ dark_blue = 0x00008B, // rgb(0,0,139)
+ dark_cyan = 0x008B8B, // rgb(0,139,139)
+ dark_golden_rod = 0xB8860B, // rgb(184,134,11)
+ dark_gray = 0xA9A9A9, // rgb(169,169,169)
+ dark_green = 0x006400, // rgb(0,100,0)
+ dark_khaki = 0xBDB76B, // rgb(189,183,107)
+ dark_magenta = 0x8B008B, // rgb(139,0,139)
+ dark_olive_green = 0x556B2F, // rgb(85,107,47)
+ dark_orange = 0xFF8C00, // rgb(255,140,0)
+ dark_orchid = 0x9932CC, // rgb(153,50,204)
+ dark_red = 0x8B0000, // rgb(139,0,0)
+ dark_salmon = 0xE9967A, // rgb(233,150,122)
+ dark_sea_green = 0x8FBC8F, // rgb(143,188,143)
+ dark_slate_blue = 0x483D8B, // rgb(72,61,139)
+ dark_slate_gray = 0x2F4F4F, // rgb(47,79,79)
+ dark_turquoise = 0x00CED1, // rgb(0,206,209)
+ dark_violet = 0x9400D3, // rgb(148,0,211)
+ deep_pink = 0xFF1493, // rgb(255,20,147)
+ deep_sky_blue = 0x00BFFF, // rgb(0,191,255)
+ dim_gray = 0x696969, // rgb(105,105,105)
+ dodger_blue = 0x1E90FF, // rgb(30,144,255)
+ fire_brick = 0xB22222, // rgb(178,34,34)
+ floral_white = 0xFFFAF0, // rgb(255,250,240)
+ forest_green = 0x228B22, // rgb(34,139,34)
+ fuchsia = 0xFF00FF, // rgb(255,0,255)
+ gainsboro = 0xDCDCDC, // rgb(220,220,220)
+ ghost_white = 0xF8F8FF, // rgb(248,248,255)
+ gold = 0xFFD700, // rgb(255,215,0)
+ golden_rod = 0xDAA520, // rgb(218,165,32)
+ gray = 0x808080, // rgb(128,128,128)
+ green = 0x008000, // rgb(0,128,0)
+ green_yellow = 0xADFF2F, // rgb(173,255,47)
+ honey_dew = 0xF0FFF0, // rgb(240,255,240)
+ hot_pink = 0xFF69B4, // rgb(255,105,180)
+ indian_red = 0xCD5C5C, // rgb(205,92,92)
+ indigo = 0x4B0082, // rgb(75,0,130)
+ ivory = 0xFFFFF0, // rgb(255,255,240)
+ khaki = 0xF0E68C, // rgb(240,230,140)
+ lavender = 0xE6E6FA, // rgb(230,230,250)
+ lavender_blush = 0xFFF0F5, // rgb(255,240,245)
+ lawn_green = 0x7CFC00, // rgb(124,252,0)
+ lemon_chiffon = 0xFFFACD, // rgb(255,250,205)
+ light_blue = 0xADD8E6, // rgb(173,216,230)
+ light_coral = 0xF08080, // rgb(240,128,128)
+ light_cyan = 0xE0FFFF, // rgb(224,255,255)
+ light_golden_rod_yellow = 0xFAFAD2, // rgb(250,250,210)
+ light_gray = 0xD3D3D3, // rgb(211,211,211)
+ light_green = 0x90EE90, // rgb(144,238,144)
+ light_pink = 0xFFB6C1, // rgb(255,182,193)
+ light_salmon = 0xFFA07A, // rgb(255,160,122)
+ light_sea_green = 0x20B2AA, // rgb(32,178,170)
+ light_sky_blue = 0x87CEFA, // rgb(135,206,250)
+ light_slate_gray = 0x778899, // rgb(119,136,153)
+ light_steel_blue = 0xB0C4DE, // rgb(176,196,222)
+ light_yellow = 0xFFFFE0, // rgb(255,255,224)
+ lime = 0x00FF00, // rgb(0,255,0)
+ lime_green = 0x32CD32, // rgb(50,205,50)
+ linen = 0xFAF0E6, // rgb(250,240,230)
+ magenta = 0xFF00FF, // rgb(255,0,255)
+ maroon = 0x800000, // rgb(128,0,0)
+ medium_aquamarine = 0x66CDAA, // rgb(102,205,170)
+ medium_blue = 0x0000CD, // rgb(0,0,205)
+ medium_orchid = 0xBA55D3, // rgb(186,85,211)
+ medium_purple = 0x9370DB, // rgb(147,112,219)
+ medium_sea_green = 0x3CB371, // rgb(60,179,113)
+ medium_slate_blue = 0x7B68EE, // rgb(123,104,238)
+ medium_spring_green = 0x00FA9A, // rgb(0,250,154)
+ medium_turquoise = 0x48D1CC, // rgb(72,209,204)
+ medium_violet_red = 0xC71585, // rgb(199,21,133)
+ midnight_blue = 0x191970, // rgb(25,25,112)
+ mint_cream = 0xF5FFFA, // rgb(245,255,250)
+ misty_rose = 0xFFE4E1, // rgb(255,228,225)
+ moccasin = 0xFFE4B5, // rgb(255,228,181)
+ navajo_white = 0xFFDEAD, // rgb(255,222,173)
+ navy = 0x000080, // rgb(0,0,128)
+ old_lace = 0xFDF5E6, // rgb(253,245,230)
+ olive = 0x808000, // rgb(128,128,0)
+ olive_drab = 0x6B8E23, // rgb(107,142,35)
+ orange = 0xFFA500, // rgb(255,165,0)
+ orange_red = 0xFF4500, // rgb(255,69,0)
+ orchid = 0xDA70D6, // rgb(218,112,214)
+ pale_golden_rod = 0xEEE8AA, // rgb(238,232,170)
+ pale_green = 0x98FB98, // rgb(152,251,152)
+ pale_turquoise = 0xAFEEEE, // rgb(175,238,238)
+ pale_violet_red = 0xDB7093, // rgb(219,112,147)
+ papaya_whip = 0xFFEFD5, // rgb(255,239,213)
+ peach_puff = 0xFFDAB9, // rgb(255,218,185)
+ peru = 0xCD853F, // rgb(205,133,63)
+ pink = 0xFFC0CB, // rgb(255,192,203)
+ plum = 0xDDA0DD, // rgb(221,160,221)
+ powder_blue = 0xB0E0E6, // rgb(176,224,230)
+ purple = 0x800080, // rgb(128,0,128)
+ rebecca_purple = 0x663399, // rgb(102,51,153)
+ red = 0xFF0000, // rgb(255,0,0)
+ rosy_brown = 0xBC8F8F, // rgb(188,143,143)
+ royal_blue = 0x4169E1, // rgb(65,105,225)
+ saddle_brown = 0x8B4513, // rgb(139,69,19)
+ salmon = 0xFA8072, // rgb(250,128,114)
+ sandy_brown = 0xF4A460, // rgb(244,164,96)
+ sea_green = 0x2E8B57, // rgb(46,139,87)
+ sea_shell = 0xFFF5EE, // rgb(255,245,238)
+ sienna = 0xA0522D, // rgb(160,82,45)
+ silver = 0xC0C0C0, // rgb(192,192,192)
+ sky_blue = 0x87CEEB, // rgb(135,206,235)
+ slate_blue = 0x6A5ACD, // rgb(106,90,205)
+ slate_gray = 0x708090, // rgb(112,128,144)
+ snow = 0xFFFAFA, // rgb(255,250,250)
+ spring_green = 0x00FF7F, // rgb(0,255,127)
+ steel_blue = 0x4682B4, // rgb(70,130,180)
+ tan = 0xD2B48C, // rgb(210,180,140)
+ teal = 0x008080, // rgb(0,128,128)
+ thistle = 0xD8BFD8, // rgb(216,191,216)
+ tomato = 0xFF6347, // rgb(255,99,71)
+ turquoise = 0x40E0D0, // rgb(64,224,208)
+ violet = 0xEE82EE, // rgb(238,130,238)
+ wheat = 0xF5DEB3, // rgb(245,222,179)
+ white = 0xFFFFFF, // rgb(255,255,255)
+ white_smoke = 0xF5F5F5, // rgb(245,245,245)
+ yellow = 0xFFFF00, // rgb(255,255,0)
+ yellow_green = 0x9ACD32 // rgb(154,205,50)
+}; // enum class color
+
+enum class terminal_color : uint8_t {
+ black = 30,
+ red,
+ green,
+ yellow,
+ blue,
+ magenta,
+ cyan,
+ white,
+ bright_black = 90,
+ bright_red,
+ bright_green,
+ bright_yellow,
+ bright_blue,
+ bright_magenta,
+ bright_cyan,
+ bright_white
+};
+
+enum class emphasis : uint8_t {
+ bold = 1,
+ faint = 1 << 1,
+ italic = 1 << 2,
+ underline = 1 << 3,
+ blink = 1 << 4,
+ reverse = 1 << 5,
+ conceal = 1 << 6,
+ strikethrough = 1 << 7,
+};
+
+// rgb is a struct for red, green and blue colors.
+// Using the name "rgb" makes some editors show the color in a tooltip.
+struct rgb {
+ FMT_CONSTEXPR rgb() : r(0), g(0), b(0) {}
+ FMT_CONSTEXPR rgb(uint8_t r_, uint8_t g_, uint8_t b_) : r(r_), g(g_), b(b_) {}
+ FMT_CONSTEXPR rgb(uint32_t hex)
+ : r((hex >> 16) & 0xFF), g((hex >> 8) & 0xFF), b(hex & 0xFF) {}
+ FMT_CONSTEXPR rgb(color hex)
+ : r((uint32_t(hex) >> 16) & 0xFF),
+ g((uint32_t(hex) >> 8) & 0xFF),
+ b(uint32_t(hex) & 0xFF) {}
+ uint8_t r;
+ uint8_t g;
+ uint8_t b;
+};
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// color is a struct of either a rgb color or a terminal color.
+struct color_type {
+ FMT_CONSTEXPR color_type() noexcept : is_rgb(), value{} {}
+ FMT_CONSTEXPR color_type(color rgb_color) noexcept : is_rgb(true), value{} {
+ value.rgb_color = static_cast<uint32_t>(rgb_color);
+ }
+ FMT_CONSTEXPR color_type(rgb rgb_color) noexcept : is_rgb(true), value{} {
+ value.rgb_color = (static_cast<uint32_t>(rgb_color.r) << 16) |
+ (static_cast<uint32_t>(rgb_color.g) << 8) | rgb_color.b;
+ }
+ FMT_CONSTEXPR color_type(terminal_color term_color) noexcept
+ : is_rgb(), value{} {
+ value.term_color = static_cast<uint8_t>(term_color);
+ }
+ bool is_rgb;
+ union color_union {
+ uint8_t term_color;
+ uint32_t rgb_color;
+ } value;
+};
+
+FMT_END_DETAIL_NAMESPACE
+
+/** A text style consisting of foreground and background colors and emphasis. */
+class text_style {
+ public:
+ FMT_CONSTEXPR text_style(emphasis em = emphasis()) noexcept
+ : set_foreground_color(), set_background_color(), ems(em) {}
+
+ FMT_CONSTEXPR text_style& operator|=(const text_style& rhs) {
+ if (!set_foreground_color) {
+ set_foreground_color = rhs.set_foreground_color;
+ foreground_color = rhs.foreground_color;
+ } else if (rhs.set_foreground_color) {
+ if (!foreground_color.is_rgb || !rhs.foreground_color.is_rgb)
+ FMT_THROW(format_error("can't OR a terminal color"));
+ foreground_color.value.rgb_color |= rhs.foreground_color.value.rgb_color;
+ }
+
+ if (!set_background_color) {
+ set_background_color = rhs.set_background_color;
+ background_color = rhs.background_color;
+ } else if (rhs.set_background_color) {
+ if (!background_color.is_rgb || !rhs.background_color.is_rgb)
+ FMT_THROW(format_error("can't OR a terminal color"));
+ background_color.value.rgb_color |= rhs.background_color.value.rgb_color;
+ }
+
+ ems = static_cast<emphasis>(static_cast<uint8_t>(ems) |
+ static_cast<uint8_t>(rhs.ems));
+ return *this;
+ }
+
+ friend FMT_CONSTEXPR text_style operator|(text_style lhs,
+ const text_style& rhs) {
+ return lhs |= rhs;
+ }
+
+ FMT_CONSTEXPR bool has_foreground() const noexcept {
+ return set_foreground_color;
+ }
+ FMT_CONSTEXPR bool has_background() const noexcept {
+ return set_background_color;
+ }
+ FMT_CONSTEXPR bool has_emphasis() const noexcept {
+ return static_cast<uint8_t>(ems) != 0;
+ }
+ FMT_CONSTEXPR detail::color_type get_foreground() const noexcept {
+ FMT_ASSERT(has_foreground(), "no foreground specified for this style");
+ return foreground_color;
+ }
+ FMT_CONSTEXPR detail::color_type get_background() const noexcept {
+ FMT_ASSERT(has_background(), "no background specified for this style");
+ return background_color;
+ }
+ FMT_CONSTEXPR emphasis get_emphasis() const noexcept {
+ FMT_ASSERT(has_emphasis(), "no emphasis specified for this style");
+ return ems;
+ }
+
+ private:
+ FMT_CONSTEXPR text_style(bool is_foreground,
+ detail::color_type text_color) noexcept
+ : set_foreground_color(), set_background_color(), ems() {
+ if (is_foreground) {
+ foreground_color = text_color;
+ set_foreground_color = true;
+ } else {
+ background_color = text_color;
+ set_background_color = true;
+ }
+ }
+
+ friend FMT_CONSTEXPR text_style fg(detail::color_type foreground) noexcept;
+
+ friend FMT_CONSTEXPR text_style bg(detail::color_type background) noexcept;
+
+ detail::color_type foreground_color;
+ detail::color_type background_color;
+ bool set_foreground_color;
+ bool set_background_color;
+ emphasis ems;
+};
+
+/** Creates a text style from the foreground (text) color. */
+FMT_CONSTEXPR inline text_style fg(detail::color_type foreground) noexcept {
+ return text_style(true, foreground);
+}
+
+/** Creates a text style from the background color. */
+FMT_CONSTEXPR inline text_style bg(detail::color_type background) noexcept {
+ return text_style(false, background);
+}
+
+FMT_CONSTEXPR inline text_style operator|(emphasis lhs, emphasis rhs) noexcept {
+ return text_style(lhs) | rhs;
+}
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+template <typename Char> struct ansi_color_escape {
+ FMT_CONSTEXPR ansi_color_escape(detail::color_type text_color,
+ const char* esc) noexcept {
+ // If we have a terminal color, we need to output another escape code
+ // sequence.
+ if (!text_color.is_rgb) {
+ bool is_background = esc == string_view("\x1b[48;2;");
+ uint32_t value = text_color.value.term_color;
+ // Background ASCII codes are the same as the foreground ones but with
+ // 10 more.
+ if (is_background) value += 10u;
+
+ size_t index = 0;
+ buffer[index++] = static_cast<Char>('\x1b');
+ buffer[index++] = static_cast<Char>('[');
+
+ if (value >= 100u) {
+ buffer[index++] = static_cast<Char>('1');
+ value %= 100u;
+ }
+ buffer[index++] = static_cast<Char>('0' + value / 10u);
+ buffer[index++] = static_cast<Char>('0' + value % 10u);
+
+ buffer[index++] = static_cast<Char>('m');
+ buffer[index++] = static_cast<Char>('\0');
+ return;
+ }
+
+ for (int i = 0; i < 7; i++) {
+ buffer[i] = static_cast<Char>(esc[i]);
+ }
+ rgb color(text_color.value.rgb_color);
+ to_esc(color.r, buffer + 7, ';');
+ to_esc(color.g, buffer + 11, ';');
+ to_esc(color.b, buffer + 15, 'm');
+ buffer[19] = static_cast<Char>(0);
+ }
+ FMT_CONSTEXPR ansi_color_escape(emphasis em) noexcept {
+ uint8_t em_codes[num_emphases] = {};
+ if (has_emphasis(em, emphasis::bold)) em_codes[0] = 1;
+ if (has_emphasis(em, emphasis::faint)) em_codes[1] = 2;
+ if (has_emphasis(em, emphasis::italic)) em_codes[2] = 3;
+ if (has_emphasis(em, emphasis::underline)) em_codes[3] = 4;
+ if (has_emphasis(em, emphasis::blink)) em_codes[4] = 5;
+ if (has_emphasis(em, emphasis::reverse)) em_codes[5] = 7;
+ if (has_emphasis(em, emphasis::conceal)) em_codes[6] = 8;
+ if (has_emphasis(em, emphasis::strikethrough)) em_codes[7] = 9;
+
+ size_t index = 0;
+ for (size_t i = 0; i < num_emphases; ++i) {
+ if (!em_codes[i]) continue;
+ buffer[index++] = static_cast<Char>('\x1b');
+ buffer[index++] = static_cast<Char>('[');
+ buffer[index++] = static_cast<Char>('0' + em_codes[i]);
+ buffer[index++] = static_cast<Char>('m');
+ }
+ buffer[index++] = static_cast<Char>(0);
+ }
+ FMT_CONSTEXPR operator const Char*() const noexcept { return buffer; }
+
+ FMT_CONSTEXPR const Char* begin() const noexcept { return buffer; }
+ FMT_CONSTEXPR_CHAR_TRAITS const Char* end() const noexcept {
+ return buffer + std::char_traits<Char>::length(buffer);
+ }
+
+ private:
+ static constexpr size_t num_emphases = 8;
+ Char buffer[7u + 3u * num_emphases + 1u];
+
+ static FMT_CONSTEXPR void to_esc(uint8_t c, Char* out,
+ char delimiter) noexcept {
+ out[0] = static_cast<Char>('0' + c / 100);
+ out[1] = static_cast<Char>('0' + c / 10 % 10);
+ out[2] = static_cast<Char>('0' + c % 10);
+ out[3] = static_cast<Char>(delimiter);
+ }
+ static FMT_CONSTEXPR bool has_emphasis(emphasis em, emphasis mask) noexcept {
+ return static_cast<uint8_t>(em) & static_cast<uint8_t>(mask);
+ }
+};
+
+template <typename Char>
+FMT_CONSTEXPR ansi_color_escape<Char> make_foreground_color(
+ detail::color_type foreground) noexcept {
+ return ansi_color_escape<Char>(foreground, "\x1b[38;2;");
+}
+
+template <typename Char>
+FMT_CONSTEXPR ansi_color_escape<Char> make_background_color(
+ detail::color_type background) noexcept {
+ return ansi_color_escape<Char>(background, "\x1b[48;2;");
+}
+
+template <typename Char>
+FMT_CONSTEXPR ansi_color_escape<Char> make_emphasis(emphasis em) noexcept {
+ return ansi_color_escape<Char>(em);
+}
+
+template <typename Char> inline void reset_color(buffer<Char>& buffer) {
+ auto reset_color = string_view("\x1b[0m");
+ buffer.append(reset_color.begin(), reset_color.end());
+}
+
+template <typename T> struct styled_arg {
+ const T& value;
+ text_style style;
+};
+
+template <typename Char>
+void vformat_to(buffer<Char>& buf, const text_style& ts,
+ basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ bool has_style = false;
+ if (ts.has_emphasis()) {
+ has_style = true;
+ auto emphasis = detail::make_emphasis<Char>(ts.get_emphasis());
+ buf.append(emphasis.begin(), emphasis.end());
+ }
+ if (ts.has_foreground()) {
+ has_style = true;
+ auto foreground = detail::make_foreground_color<Char>(ts.get_foreground());
+ buf.append(foreground.begin(), foreground.end());
+ }
+ if (ts.has_background()) {
+ has_style = true;
+ auto background = detail::make_background_color<Char>(ts.get_background());
+ buf.append(background.begin(), background.end());
+ }
+ detail::vformat_to(buf, format_str, args, {});
+ if (has_style) detail::reset_color<Char>(buf);
+}
+
+FMT_END_DETAIL_NAMESPACE
+
+inline void vprint(std::FILE* f, const text_style& ts, string_view fmt,
+ format_args args) {
+ // Legacy wide streams are not supported.
+ auto buf = memory_buffer();
+ detail::vformat_to(buf, ts, fmt, args);
+ if (detail::is_utf8()) {
+ detail::print(f, string_view(buf.begin(), buf.size()));
+ return;
+ }
+ buf.push_back('\0');
+ int result = std::fputs(buf.data(), f);
+ if (result < 0)
+ FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+/**
+ \rst
+ Formats a string and prints it to the specified file stream using ANSI
+ escape sequences to specify text formatting.
+
+ **Example**::
+
+ fmt::print(fmt::emphasis::bold | fg(fmt::color::red),
+ "Elapsed time: {0:.2f} seconds", 1.23);
+ \endrst
+ */
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_string<S>::value)>
+void print(std::FILE* f, const text_style& ts, const S& format_str,
+ const Args&... args) {
+ vprint(f, ts, format_str,
+ fmt::make_format_args<buffer_context<char_t<S>>>(args...));
+}
+
+/**
+ \rst
+ Formats a string and prints it to stdout using ANSI escape sequences to
+ specify text formatting.
+
+ **Example**::
+
+ fmt::print(fmt::emphasis::bold | fg(fmt::color::red),
+ "Elapsed time: {0:.2f} seconds", 1.23);
+ \endrst
+ */
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_string<S>::value)>
+void print(const text_style& ts, const S& format_str, const Args&... args) {
+ return print(stdout, ts, format_str, args...);
+}
+
+template <typename S, typename Char = char_t<S>>
+inline std::basic_string<Char> vformat(
+ const text_style& ts, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ basic_memory_buffer<Char> buf;
+ detail::vformat_to(buf, ts, detail::to_string_view(format_str), args);
+ return fmt::to_string(buf);
+}
+
+/**
+ \rst
+ Formats arguments and returns the result as a string using ANSI
+ escape sequences to specify text formatting.
+
+ **Example**::
+
+ #include <fmt/color.h>
+ std::string message = fmt::format(fmt::emphasis::bold | fg(fmt::color::red),
+ "The answer is {}", 42);
+ \endrst
+*/
+template <typename S, typename... Args, typename Char = char_t<S>>
+inline std::basic_string<Char> format(const text_style& ts, const S& format_str,
+ const Args&... args) {
+ return fmt::vformat(ts, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+/**
+ Formats a string with the given text_style and writes the output to ``out``.
+ */
+template <typename OutputIt, typename Char,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value)>
+OutputIt vformat_to(
+ OutputIt out, const text_style& ts, basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ auto&& buf = detail::get_buffer<Char>(out);
+ detail::vformat_to(buf, ts, format_str, args);
+ return detail::get_iterator(buf, out);
+}
+
+/**
+ \rst
+ Formats arguments with the given text_style, writes the result to the output
+ iterator ``out`` and returns the iterator past the end of the output range.
+
+ **Example**::
+
+ std::vector<char> out;
+ fmt::format_to(std::back_inserter(out),
+ fmt::emphasis::bold | fg(fmt::color::red), "{}", 42);
+ \endrst
+*/
+template <typename OutputIt, typename S, typename... Args,
+ bool enable = detail::is_output_iterator<OutputIt, char_t<S>>::value&&
+ detail::is_string<S>::value>
+inline auto format_to(OutputIt out, const text_style& ts, const S& format_str,
+ Args&&... args) ->
+ typename std::enable_if<enable, OutputIt>::type {
+ return vformat_to(out, ts, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<char_t<S>>>(args...));
+}
+
+template <typename T, typename Char>
+struct formatter<detail::styled_arg<T>, Char> : formatter<T, Char> {
+ template <typename FormatContext>
+ auto format(const detail::styled_arg<T>& arg, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ const auto& ts = arg.style;
+ const auto& value = arg.value;
+ auto out = ctx.out();
+
+ bool has_style = false;
+ if (ts.has_emphasis()) {
+ has_style = true;
+ auto emphasis = detail::make_emphasis<Char>(ts.get_emphasis());
+ out = std::copy(emphasis.begin(), emphasis.end(), out);
+ }
+ if (ts.has_foreground()) {
+ has_style = true;
+ auto foreground =
+ detail::make_foreground_color<Char>(ts.get_foreground());
+ out = std::copy(foreground.begin(), foreground.end(), out);
+ }
+ if (ts.has_background()) {
+ has_style = true;
+ auto background =
+ detail::make_background_color<Char>(ts.get_background());
+ out = std::copy(background.begin(), background.end(), out);
+ }
+ out = formatter<T, Char>::format(value, ctx);
+ if (has_style) {
+ auto reset_color = string_view("\x1b[0m");
+ out = std::copy(reset_color.begin(), reset_color.end(), out);
+ }
+ return out;
+ }
+};
+
+/**
+ \rst
+ Returns an argument that will be formatted using ANSI escape sequences,
+ to be used in a formatting function.
+
+ **Example**::
+
+ fmt::print("Elapsed time: {0:.2f} seconds",
+ fmt::styled(1.23, fmt::fg(fmt::color::green) |
+ fmt::bg(fmt::color::blue)));
+ \endrst
+ */
+template <typename T>
+FMT_CONSTEXPR auto styled(const T& value, text_style ts)
+ -> detail::styled_arg<remove_cvref_t<T>> {
+ return detail::styled_arg<remove_cvref_t<T>>{value, ts};
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_COLOR_H_
diff --git a/contrib/fmt/include/fmt/compile.h b/contrib/fmt/include/fmt/compile.h
new file mode 100644
index 0000000..94e13c0
--- /dev/null
+++ b/contrib/fmt/include/fmt/compile.h
@@ -0,0 +1,607 @@
+// Formatting library for C++ - experimental format string compilation
+//
+// Copyright (c) 2012 - present, Victor Zverovich and fmt contributors
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_COMPILE_H_
+#define FMT_COMPILE_H_
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template <typename Char, typename InputIt>
+FMT_CONSTEXPR inline counting_iterator copy_str(InputIt begin, InputIt end,
+ counting_iterator it) {
+ return it + (end - begin);
+}
+
+template <typename OutputIt> class truncating_iterator_base {
+ protected:
+ OutputIt out_;
+ size_t limit_;
+ size_t count_ = 0;
+
+ truncating_iterator_base() : out_(), limit_(0) {}
+
+ truncating_iterator_base(OutputIt out, size_t limit)
+ : out_(out), limit_(limit) {}
+
+ public:
+ using iterator_category = std::output_iterator_tag;
+ using value_type = typename std::iterator_traits<OutputIt>::value_type;
+ using difference_type = std::ptrdiff_t;
+ using pointer = void;
+ using reference = void;
+ FMT_UNCHECKED_ITERATOR(truncating_iterator_base);
+
+ OutputIt base() const { return out_; }
+ size_t count() const { return count_; }
+};
+
+// An output iterator that truncates the output and counts the number of objects
+// written to it.
+template <typename OutputIt,
+ typename Enable = typename std::is_void<
+ typename std::iterator_traits<OutputIt>::value_type>::type>
+class truncating_iterator;
+
+template <typename OutputIt>
+class truncating_iterator<OutputIt, std::false_type>
+ : public truncating_iterator_base<OutputIt> {
+ mutable typename truncating_iterator_base<OutputIt>::value_type blackhole_;
+
+ public:
+ using value_type = typename truncating_iterator_base<OutputIt>::value_type;
+
+ truncating_iterator() = default;
+
+ truncating_iterator(OutputIt out, size_t limit)
+ : truncating_iterator_base<OutputIt>(out, limit) {}
+
+ truncating_iterator& operator++() {
+ if (this->count_++ < this->limit_) ++this->out_;
+ return *this;
+ }
+
+ truncating_iterator operator++(int) {
+ auto it = *this;
+ ++*this;
+ return it;
+ }
+
+ value_type& operator*() const {
+ return this->count_ < this->limit_ ? *this->out_ : blackhole_;
+ }
+};
+
+template <typename OutputIt>
+class truncating_iterator<OutputIt, std::true_type>
+ : public truncating_iterator_base<OutputIt> {
+ public:
+ truncating_iterator() = default;
+
+ truncating_iterator(OutputIt out, size_t limit)
+ : truncating_iterator_base<OutputIt>(out, limit) {}
+
+ template <typename T> truncating_iterator& operator=(T val) {
+ if (this->count_++ < this->limit_) *this->out_++ = val;
+ return *this;
+ }
+
+ truncating_iterator& operator++() { return *this; }
+ truncating_iterator& operator++(int) { return *this; }
+ truncating_iterator& operator*() { return *this; }
+};
+
+// A compile-time string which is compiled into fast formatting code.
+class compiled_string {};
+
+template <typename S>
+struct is_compiled_string : std::is_base_of<compiled_string, S> {};
+
+/**
+ \rst
+ Converts a string literal *s* into a format string that will be parsed at
+ compile time and converted into efficient formatting code. Requires C++17
+ ``constexpr if`` compiler support.
+
+ **Example**::
+
+ // Converts 42 into std::string using the most efficient method and no
+ // runtime format string processing.
+ std::string s = fmt::format(FMT_COMPILE("{}"), 42);
+ \endrst
+ */
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+# define FMT_COMPILE(s) \
+ FMT_STRING_IMPL(s, fmt::detail::compiled_string, explicit)
+#else
+# define FMT_COMPILE(s) FMT_STRING(s)
+#endif
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct udl_compiled_string : compiled_string {
+ using char_type = Char;
+ explicit constexpr operator basic_string_view<char_type>() const {
+ return {Str.data, N - 1};
+ }
+};
+#endif
+
+template <typename T, typename... Tail>
+const T& first(const T& value, const Tail&...) {
+ return value;
+}
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+template <typename... Args> struct type_list {};
+
+// Returns a reference to the argument at index N from [first, rest...].
+template <int N, typename T, typename... Args>
+constexpr const auto& get([[maybe_unused]] const T& first,
+ [[maybe_unused]] const Args&... rest) {
+ static_assert(N < 1 + sizeof...(Args), "index is out of bounds");
+ if constexpr (N == 0)
+ return first;
+ else
+ return detail::get<N - 1>(rest...);
+}
+
+template <typename Char, typename... Args>
+constexpr int get_arg_index_by_name(basic_string_view<Char> name,
+ type_list<Args...>) {
+ return get_arg_index_by_name<Args...>(name);
+}
+
+template <int N, typename> struct get_type_impl;
+
+template <int N, typename... Args> struct get_type_impl<N, type_list<Args...>> {
+ using type =
+ remove_cvref_t<decltype(detail::get<N>(std::declval<Args>()...))>;
+};
+
+template <int N, typename T>
+using get_type = typename get_type_impl<N, T>::type;
+
+template <typename T> struct is_compiled_format : std::false_type {};
+
+template <typename Char> struct text {
+ basic_string_view<Char> data;
+ using char_type = Char;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&...) const {
+ return write<Char>(out, data);
+ }
+};
+
+template <typename Char>
+struct is_compiled_format<text<Char>> : std::true_type {};
+
+template <typename Char>
+constexpr text<Char> make_text(basic_string_view<Char> s, size_t pos,
+ size_t size) {
+ return {{&s[pos], size}};
+}
+
+template <typename Char> struct code_unit {
+ Char value;
+ using char_type = Char;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&...) const {
+ return write<Char>(out, value);
+ }
+};
+
+// This ensures that the argument type is convertible to `const T&`.
+template <typename T, int N, typename... Args>
+constexpr const T& get_arg_checked(const Args&... args) {
+ const auto& arg = detail::get<N>(args...);
+ if constexpr (detail::is_named_arg<remove_cvref_t<decltype(arg)>>()) {
+ return arg.value;
+ } else {
+ return arg;
+ }
+}
+
+template <typename Char>
+struct is_compiled_format<code_unit<Char>> : std::true_type {};
+
+// A replacement field that refers to argument N.
+template <typename Char, typename T, int N> struct field {
+ using char_type = Char;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&... args) const {
+ return write<Char>(out, get_arg_checked<T, N>(args...));
+ }
+};
+
+template <typename Char, typename T, int N>
+struct is_compiled_format<field<Char, T, N>> : std::true_type {};
+
+// A replacement field that refers to argument with name.
+template <typename Char> struct runtime_named_field {
+ using char_type = Char;
+ basic_string_view<Char> name;
+
+ template <typename OutputIt, typename T>
+ constexpr static bool try_format_argument(
+ OutputIt& out,
+ // [[maybe_unused]] due to unused-but-set-parameter warning in GCC 7,8,9
+ [[maybe_unused]] basic_string_view<Char> arg_name, const T& arg) {
+ if constexpr (is_named_arg<typename std::remove_cv<T>::type>::value) {
+ if (arg_name == arg.name) {
+ out = write<Char>(out, arg.value);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&... args) const {
+ bool found = (try_format_argument(out, name, args) || ...);
+ if (!found) {
+ FMT_THROW(format_error("argument with specified name is not found"));
+ }
+ return out;
+ }
+};
+
+template <typename Char>
+struct is_compiled_format<runtime_named_field<Char>> : std::true_type {};
+
+// A replacement field that refers to argument N and has format specifiers.
+template <typename Char, typename T, int N> struct spec_field {
+ using char_type = Char;
+ formatter<T, Char> fmt;
+
+ template <typename OutputIt, typename... Args>
+ constexpr FMT_INLINE OutputIt format(OutputIt out,
+ const Args&... args) const {
+ const auto& vargs =
+ fmt::make_format_args<basic_format_context<OutputIt, Char>>(args...);
+ basic_format_context<OutputIt, Char> ctx(out, vargs);
+ return fmt.format(get_arg_checked<T, N>(args...), ctx);
+ }
+};
+
+template <typename Char, typename T, int N>
+struct is_compiled_format<spec_field<Char, T, N>> : std::true_type {};
+
+template <typename L, typename R> struct concat {
+ L lhs;
+ R rhs;
+ using char_type = typename L::char_type;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&... args) const {
+ out = lhs.format(out, args...);
+ return rhs.format(out, args...);
+ }
+};
+
+template <typename L, typename R>
+struct is_compiled_format<concat<L, R>> : std::true_type {};
+
+template <typename L, typename R>
+constexpr concat<L, R> make_concat(L lhs, R rhs) {
+ return {lhs, rhs};
+}
+
+struct unknown_format {};
+
+template <typename Char>
+constexpr size_t parse_text(basic_string_view<Char> str, size_t pos) {
+ for (size_t size = str.size(); pos != size; ++pos) {
+ if (str[pos] == '{' || str[pos] == '}') break;
+ }
+ return pos;
+}
+
+template <typename Args, size_t POS, int ID, typename S>
+constexpr auto compile_format_string(S format_str);
+
+template <typename Args, size_t POS, int ID, typename T, typename S>
+constexpr auto parse_tail(T head, S format_str) {
+ if constexpr (POS !=
+ basic_string_view<typename S::char_type>(format_str).size()) {
+ constexpr auto tail = compile_format_string<Args, POS, ID>(format_str);
+ if constexpr (std::is_same<remove_cvref_t<decltype(tail)>,
+ unknown_format>())
+ return tail;
+ else
+ return make_concat(head, tail);
+ } else {
+ return head;
+ }
+}
+
+template <typename T, typename Char> struct parse_specs_result {
+ formatter<T, Char> fmt;
+ size_t end;
+ int next_arg_id;
+};
+
+enum { manual_indexing_id = -1 };
+
+template <typename T, typename Char>
+constexpr parse_specs_result<T, Char> parse_specs(basic_string_view<Char> str,
+ size_t pos, int next_arg_id) {
+ str.remove_prefix(pos);
+ auto ctx =
+ compile_parse_context<Char>(str, max_value<int>(), nullptr, next_arg_id);
+ auto f = formatter<T, Char>();
+ auto end = f.parse(ctx);
+ return {f, pos + fmt::detail::to_unsigned(end - str.data()),
+ next_arg_id == 0 ? manual_indexing_id : ctx.next_arg_id()};
+}
+
+template <typename Char> struct arg_id_handler {
+ arg_ref<Char> arg_id;
+
+ constexpr int on_auto() {
+ FMT_ASSERT(false, "handler cannot be used with automatic indexing");
+ return 0;
+ }
+ constexpr int on_index(int id) {
+ arg_id = arg_ref<Char>(id);
+ return 0;
+ }
+ constexpr int on_name(basic_string_view<Char> id) {
+ arg_id = arg_ref<Char>(id);
+ return 0;
+ }
+};
+
+template <typename Char> struct parse_arg_id_result {
+ arg_ref<Char> arg_id;
+ const Char* arg_id_end;
+};
+
+template <int ID, typename Char>
+constexpr auto parse_arg_id(const Char* begin, const Char* end) {
+ auto handler = arg_id_handler<Char>{arg_ref<Char>{}};
+ auto arg_id_end = parse_arg_id(begin, end, handler);
+ return parse_arg_id_result<Char>{handler.arg_id, arg_id_end};
+}
+
+template <typename T, typename Enable = void> struct field_type {
+ using type = remove_cvref_t<T>;
+};
+
+template <typename T>
+struct field_type<T, enable_if_t<detail::is_named_arg<T>::value>> {
+ using type = remove_cvref_t<decltype(T::value)>;
+};
+
+template <typename T, typename Args, size_t END_POS, int ARG_INDEX, int NEXT_ID,
+ typename S>
+constexpr auto parse_replacement_field_then_tail(S format_str) {
+ using char_type = typename S::char_type;
+ constexpr auto str = basic_string_view<char_type>(format_str);
+ constexpr char_type c = END_POS != str.size() ? str[END_POS] : char_type();
+ if constexpr (c == '}') {
+ return parse_tail<Args, END_POS + 1, NEXT_ID>(
+ field<char_type, typename field_type<T>::type, ARG_INDEX>(),
+ format_str);
+ } else if constexpr (c != ':') {
+ FMT_THROW(format_error("expected ':'"));
+ } else {
+ constexpr auto result = parse_specs<typename field_type<T>::type>(
+ str, END_POS + 1, NEXT_ID == manual_indexing_id ? 0 : NEXT_ID);
+ if constexpr (result.end >= str.size() || str[result.end] != '}') {
+ FMT_THROW(format_error("expected '}'"));
+ return 0;
+ } else {
+ return parse_tail<Args, result.end + 1, result.next_arg_id>(
+ spec_field<char_type, typename field_type<T>::type, ARG_INDEX>{
+ result.fmt},
+ format_str);
+ }
+ }
+}
+
+// Compiles a non-empty format string and returns the compiled representation
+// or unknown_format() on unrecognized input.
+template <typename Args, size_t POS, int ID, typename S>
+constexpr auto compile_format_string(S format_str) {
+ using char_type = typename S::char_type;
+ constexpr auto str = basic_string_view<char_type>(format_str);
+ if constexpr (str[POS] == '{') {
+ if constexpr (POS + 1 == str.size())
+ FMT_THROW(format_error("unmatched '{' in format string"));
+ if constexpr (str[POS + 1] == '{') {
+ return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), format_str);
+ } else if constexpr (str[POS + 1] == '}' || str[POS + 1] == ':') {
+ static_assert(ID != manual_indexing_id,
+ "cannot switch from manual to automatic argument indexing");
+ constexpr auto next_id =
+ ID != manual_indexing_id ? ID + 1 : manual_indexing_id;
+ return parse_replacement_field_then_tail<get_type<ID, Args>, Args,
+ POS + 1, ID, next_id>(
+ format_str);
+ } else {
+ constexpr auto arg_id_result =
+ parse_arg_id<ID>(str.data() + POS + 1, str.data() + str.size());
+ constexpr auto arg_id_end_pos = arg_id_result.arg_id_end - str.data();
+ constexpr char_type c =
+ arg_id_end_pos != str.size() ? str[arg_id_end_pos] : char_type();
+ static_assert(c == '}' || c == ':', "missing '}' in format string");
+ if constexpr (arg_id_result.arg_id.kind == arg_id_kind::index) {
+ static_assert(
+ ID == manual_indexing_id || ID == 0,
+ "cannot switch from automatic to manual argument indexing");
+ constexpr auto arg_index = arg_id_result.arg_id.val.index;
+ return parse_replacement_field_then_tail<get_type<arg_index, Args>,
+ Args, arg_id_end_pos,
+ arg_index, manual_indexing_id>(
+ format_str);
+ } else if constexpr (arg_id_result.arg_id.kind == arg_id_kind::name) {
+ constexpr auto arg_index =
+ get_arg_index_by_name(arg_id_result.arg_id.val.name, Args{});
+ if constexpr (arg_index != invalid_arg_index) {
+ constexpr auto next_id =
+ ID != manual_indexing_id ? ID + 1 : manual_indexing_id;
+ return parse_replacement_field_then_tail<
+ decltype(get_type<arg_index, Args>::value), Args, arg_id_end_pos,
+ arg_index, next_id>(format_str);
+ } else {
+ if constexpr (c == '}') {
+ return parse_tail<Args, arg_id_end_pos + 1, ID>(
+ runtime_named_field<char_type>{arg_id_result.arg_id.val.name},
+ format_str);
+ } else if constexpr (c == ':') {
+ return unknown_format(); // no type info for specs parsing
+ }
+ }
+ }
+ }
+ } else if constexpr (str[POS] == '}') {
+ if constexpr (POS + 1 == str.size())
+ FMT_THROW(format_error("unmatched '}' in format string"));
+ return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), format_str);
+ } else {
+ constexpr auto end = parse_text(str, POS + 1);
+ if constexpr (end - POS > 1) {
+ return parse_tail<Args, end, ID>(make_text(str, POS, end - POS),
+ format_str);
+ } else {
+ return parse_tail<Args, end, ID>(code_unit<char_type>{str[POS]},
+ format_str);
+ }
+ }
+}
+
+template <typename... Args, typename S,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+constexpr auto compile(S format_str) {
+ constexpr auto str = basic_string_view<typename S::char_type>(format_str);
+ if constexpr (str.size() == 0) {
+ return detail::make_text(str, 0, 0);
+ } else {
+ constexpr auto result =
+ detail::compile_format_string<detail::type_list<Args...>, 0, 0>(
+ format_str);
+ return result;
+ }
+}
+#endif // defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+
+template <typename CompiledFormat, typename... Args,
+ typename Char = typename CompiledFormat::char_type,
+ FMT_ENABLE_IF(detail::is_compiled_format<CompiledFormat>::value)>
+FMT_INLINE std::basic_string<Char> format(const CompiledFormat& cf,
+ const Args&... args) {
+ auto s = std::basic_string<Char>();
+ cf.format(std::back_inserter(s), args...);
+ return s;
+}
+
+template <typename OutputIt, typename CompiledFormat, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_format<CompiledFormat>::value)>
+constexpr FMT_INLINE OutputIt format_to(OutputIt out, const CompiledFormat& cf,
+ const Args&... args) {
+ return cf.format(out, args...);
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_INLINE std::basic_string<typename S::char_type> format(const S&,
+ Args&&... args) {
+ if constexpr (std::is_same<typename S::char_type, char>::value) {
+ constexpr auto str = basic_string_view<typename S::char_type>(S());
+ if constexpr (str.size() == 2 && str[0] == '{' && str[1] == '}') {
+ const auto& first = detail::first(args...);
+ if constexpr (detail::is_named_arg<
+ remove_cvref_t<decltype(first)>>::value) {
+ return fmt::to_string(first.value);
+ } else {
+ return fmt::to_string(first);
+ }
+ }
+ }
+ constexpr auto compiled = detail::compile<Args...>(S());
+ if constexpr (std::is_same<remove_cvref_t<decltype(compiled)>,
+ detail::unknown_format>()) {
+ return fmt::format(
+ static_cast<basic_string_view<typename S::char_type>>(S()),
+ std::forward<Args>(args)...);
+ } else {
+ return fmt::format(compiled, std::forward<Args>(args)...);
+ }
+}
+
+template <typename OutputIt, typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_CONSTEXPR OutputIt format_to(OutputIt out, const S&, Args&&... args) {
+ constexpr auto compiled = detail::compile<Args...>(S());
+ if constexpr (std::is_same<remove_cvref_t<decltype(compiled)>,
+ detail::unknown_format>()) {
+ return fmt::format_to(
+ out, static_cast<basic_string_view<typename S::char_type>>(S()),
+ std::forward<Args>(args)...);
+ } else {
+ return fmt::format_to(out, compiled, std::forward<Args>(args)...);
+ }
+}
+#endif
+
+template <typename OutputIt, typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+format_to_n_result<OutputIt> format_to_n(OutputIt out, size_t n,
+ const S& format_str, Args&&... args) {
+ auto it = fmt::format_to(detail::truncating_iterator<OutputIt>(out, n),
+ format_str, std::forward<Args>(args)...);
+ return {it.base(), it.count()};
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_CONSTEXPR20 size_t formatted_size(const S& format_str,
+ const Args&... args) {
+ return fmt::format_to(detail::counting_iterator(), format_str, args...)
+ .count();
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+void print(std::FILE* f, const S& format_str, const Args&... args) {
+ memory_buffer buffer;
+ fmt::format_to(std::back_inserter(buffer), format_str, args...);
+ detail::print(f, {buffer.data(), buffer.size()});
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+void print(const S& format_str, const Args&... args) {
+ print(stdout, format_str, args...);
+}
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+inline namespace literals {
+template <detail_exported::fixed_string Str> constexpr auto operator""_cf() {
+ using char_t = remove_cvref_t<decltype(Str.data[0])>;
+ return detail::udl_compiled_string<char_t, sizeof(Str.data) / sizeof(char_t),
+ Str>();
+}
+} // namespace literals
+#endif
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_COMPILE_H_
diff --git a/contrib/fmt/include/fmt/core.h b/contrib/fmt/include/fmt/core.h
new file mode 100644
index 0000000..46723d5
--- /dev/null
+++ b/contrib/fmt/include/fmt/core.h
@@ -0,0 +1,2951 @@
+// Formatting library for C++ - the core API for char/UTF-8
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_CORE_H_
+#define FMT_CORE_H_
+
+#include <cstddef> // std::byte
+#include <cstdio> // std::FILE
+#include <cstring> // std::strlen
+#include <iterator>
+#include <limits>
+#include <string>
+#include <type_traits>
+
+// The fmt library version in the form major * 10000 + minor * 100 + patch.
+#define FMT_VERSION 100000
+
+#if defined(__clang__) && !defined(__ibmxl__)
+# define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__)
+#else
+# define FMT_CLANG_VERSION 0
+#endif
+
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && \
+ !defined(__NVCOMPILER)
+# define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#else
+# define FMT_GCC_VERSION 0
+#endif
+
+#ifndef FMT_GCC_PRAGMA
+// Workaround _Pragma bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59884.
+# if FMT_GCC_VERSION >= 504
+# define FMT_GCC_PRAGMA(arg) _Pragma(arg)
+# else
+# define FMT_GCC_PRAGMA(arg)
+# endif
+#endif
+
+#ifdef __ICL
+# define FMT_ICC_VERSION __ICL
+#elif defined(__INTEL_COMPILER)
+# define FMT_ICC_VERSION __INTEL_COMPILER
+#else
+# define FMT_ICC_VERSION 0
+#endif
+
+#ifdef _MSC_VER
+# define FMT_MSC_VERSION _MSC_VER
+# define FMT_MSC_WARNING(...) __pragma(warning(__VA_ARGS__))
+#else
+# define FMT_MSC_VERSION 0
+# define FMT_MSC_WARNING(...)
+#endif
+
+#ifdef _MSVC_LANG
+# define FMT_CPLUSPLUS _MSVC_LANG
+#else
+# define FMT_CPLUSPLUS __cplusplus
+#endif
+
+#ifdef __has_feature
+# define FMT_HAS_FEATURE(x) __has_feature(x)
+#else
+# define FMT_HAS_FEATURE(x) 0
+#endif
+
+#if defined(__has_include) || FMT_ICC_VERSION >= 1600 || FMT_MSC_VERSION > 1900
+# define FMT_HAS_INCLUDE(x) __has_include(x)
+#else
+# define FMT_HAS_INCLUDE(x) 0
+#endif
+
+#ifdef __has_cpp_attribute
+# define FMT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define FMT_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+#define FMT_HAS_CPP14_ATTRIBUTE(attribute) \
+ (FMT_CPLUSPLUS >= 201402L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+
+#define FMT_HAS_CPP17_ATTRIBUTE(attribute) \
+ (FMT_CPLUSPLUS >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+
+// Check if relaxed C++14 constexpr is supported.
+// GCC doesn't allow throw in constexpr until version 6 (bug 67371).
+#ifndef FMT_USE_CONSTEXPR
+# if (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VERSION >= 1912 || \
+ (FMT_GCC_VERSION >= 600 && FMT_CPLUSPLUS >= 201402L)) && \
+ !FMT_ICC_VERSION && !defined(__NVCC__)
+# define FMT_USE_CONSTEXPR 1
+# else
+# define FMT_USE_CONSTEXPR 0
+# endif
+#endif
+#if FMT_USE_CONSTEXPR
+# define FMT_CONSTEXPR constexpr
+#else
+# define FMT_CONSTEXPR
+#endif
+
+#if ((FMT_CPLUSPLUS >= 202002L) && \
+ (!defined(_GLIBCXX_RELEASE) || _GLIBCXX_RELEASE > 9)) || \
+ (FMT_CPLUSPLUS >= 201709L && FMT_GCC_VERSION >= 1002)
+# define FMT_CONSTEXPR20 constexpr
+#else
+# define FMT_CONSTEXPR20
+#endif
+
+// Check if constexpr std::char_traits<>::{compare,length} are supported.
+#if defined(__GLIBCXX__)
+# if FMT_CPLUSPLUS >= 201703L && defined(_GLIBCXX_RELEASE) && \
+ _GLIBCXX_RELEASE >= 7 // GCC 7+ libstdc++ has _GLIBCXX_RELEASE.
+# define FMT_CONSTEXPR_CHAR_TRAITS constexpr
+# endif
+#elif defined(_LIBCPP_VERSION) && FMT_CPLUSPLUS >= 201703L && \
+ _LIBCPP_VERSION >= 4000
+# define FMT_CONSTEXPR_CHAR_TRAITS constexpr
+#elif FMT_MSC_VERSION >= 1914 && FMT_CPLUSPLUS >= 201703L
+# define FMT_CONSTEXPR_CHAR_TRAITS constexpr
+#endif
+#ifndef FMT_CONSTEXPR_CHAR_TRAITS
+# define FMT_CONSTEXPR_CHAR_TRAITS
+#endif
+
+// Check if exceptions are disabled.
+#ifndef FMT_EXCEPTIONS
+# if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || \
+ (FMT_MSC_VERSION && !_HAS_EXCEPTIONS)
+# define FMT_EXCEPTIONS 0
+# else
+# define FMT_EXCEPTIONS 1
+# endif
+#endif
+
+// Disable [[noreturn]] on MSVC/NVCC because of bogus unreachable code warnings.
+#if FMT_EXCEPTIONS && FMT_HAS_CPP_ATTRIBUTE(noreturn) && !FMT_MSC_VERSION && \
+ !defined(__NVCC__)
+# define FMT_NORETURN [[noreturn]]
+#else
+# define FMT_NORETURN
+#endif
+
+#ifndef FMT_NODISCARD
+# if FMT_HAS_CPP17_ATTRIBUTE(nodiscard)
+# define FMT_NODISCARD [[nodiscard]]
+# else
+# define FMT_NODISCARD
+# endif
+#endif
+
+#ifndef FMT_INLINE
+# if FMT_GCC_VERSION || FMT_CLANG_VERSION
+# define FMT_INLINE inline __attribute__((always_inline))
+# else
+# define FMT_INLINE inline
+# endif
+#endif
+
+// An inline std::forward replacement.
+#define FMT_FORWARD(...) static_cast<decltype(__VA_ARGS__)&&>(__VA_ARGS__)
+
+#ifdef _MSC_VER
+# define FMT_UNCHECKED_ITERATOR(It) \
+ using _Unchecked_type = It // Mark iterator as checked.
+#else
+# define FMT_UNCHECKED_ITERATOR(It) using unchecked_type = It
+#endif
+
+#ifndef FMT_BEGIN_NAMESPACE
+# define FMT_BEGIN_NAMESPACE \
+ namespace fmt { \
+ inline namespace v10 {
+# define FMT_END_NAMESPACE \
+ } \
+ }
+#endif
+
+#ifndef FMT_MODULE_EXPORT
+# define FMT_MODULE_EXPORT
+# define FMT_BEGIN_EXPORT
+# define FMT_END_EXPORT
+#endif
+
+#if !defined(FMT_HEADER_ONLY) && defined(_WIN32)
+# ifdef FMT_LIB_EXPORT
+# define FMT_API __declspec(dllexport)
+# elif defined(FMT_SHARED)
+# define FMT_API __declspec(dllimport)
+# endif
+#else
+# if defined(FMT_LIB_EXPORT) || defined(FMT_SHARED)
+# if defined(__GNUC__) || defined(__clang__)
+# define FMT_API __attribute__((visibility("default")))
+# endif
+# endif
+#endif
+#ifndef FMT_API
+# define FMT_API
+#endif
+
+// libc++ supports string_view in pre-c++17.
+#if FMT_HAS_INCLUDE(<string_view>) && \
+ (FMT_CPLUSPLUS >= 201703L || defined(_LIBCPP_VERSION))
+# include <string_view>
+# define FMT_USE_STRING_VIEW
+#elif FMT_HAS_INCLUDE("experimental/string_view") && FMT_CPLUSPLUS >= 201402L
+# include <experimental/string_view>
+# define FMT_USE_EXPERIMENTAL_STRING_VIEW
+#endif
+
+#ifndef FMT_UNICODE
+# define FMT_UNICODE !FMT_MSC_VERSION
+#endif
+
+#ifndef FMT_CONSTEVAL
+# if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) && \
+ (!defined(__apple_build_version__) || \
+ __apple_build_version__ >= 14000029L) && \
+ FMT_CPLUSPLUS >= 202002L) || \
+ (defined(__cpp_consteval) && \
+ (!FMT_MSC_VERSION || _MSC_FULL_VER >= 193030704))
+// consteval is broken in MSVC before VS2022 and Apple clang before 14.
+# define FMT_CONSTEVAL consteval
+# define FMT_HAS_CONSTEVAL
+# else
+# define FMT_CONSTEVAL
+# endif
+#endif
+
+#ifndef FMT_USE_NONTYPE_TEMPLATE_ARGS
+# if defined(__cpp_nontype_template_args) && \
+ ((FMT_GCC_VERSION >= 903 && FMT_CPLUSPLUS >= 201709L) || \
+ __cpp_nontype_template_args >= 201911L) && \
+ !defined(__NVCOMPILER) && !defined(__LCC__)
+# define FMT_USE_NONTYPE_TEMPLATE_ARGS 1
+# else
+# define FMT_USE_NONTYPE_TEMPLATE_ARGS 0
+# endif
+#endif
+
+#if defined __cpp_inline_variables && __cpp_inline_variables >= 201606L
+# define FMT_INLINE_VARIABLE inline
+#else
+# define FMT_INLINE_VARIABLE
+#endif
+
+// Enable minimal optimizations for more compact code in debug mode.
+FMT_GCC_PRAGMA("GCC push_options")
+#if !defined(__OPTIMIZE__) && !defined(__NVCOMPILER) && !defined(__LCC__) && \
+ !defined(__CUDACC__)
+FMT_GCC_PRAGMA("GCC optimize(\"Og\")")
+#endif
+
+FMT_BEGIN_NAMESPACE
+
+// Implementations of enable_if_t and other metafunctions for older systems.
+template <bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+template <bool B, typename T, typename F>
+using conditional_t = typename std::conditional<B, T, F>::type;
+template <bool B> using bool_constant = std::integral_constant<bool, B>;
+template <typename T>
+using remove_reference_t = typename std::remove_reference<T>::type;
+template <typename T>
+using remove_const_t = typename std::remove_const<T>::type;
+template <typename T>
+using remove_cvref_t = typename std::remove_cv<remove_reference_t<T>>::type;
+template <typename T> struct type_identity { using type = T; };
+template <typename T> using type_identity_t = typename type_identity<T>::type;
+template <typename T>
+using underlying_t = typename std::underlying_type<T>::type;
+
+struct monostate {
+ constexpr monostate() {}
+};
+
+// An enable_if helper to be used in template parameters which results in much
+// shorter symbols: https://godbolt.org/z/sWw4vP. Extra parentheses are needed
+// to workaround a bug in MSVC 2019 (see #1140 and #1186).
+#ifdef FMT_DOC
+# define FMT_ENABLE_IF(...)
+#else
+# define FMT_ENABLE_IF(...) fmt::enable_if_t<(__VA_ARGS__), int> = 0
+#endif
+
+#ifdef __cpp_lib_byte
+inline auto format_as(std::byte b) -> unsigned char {
+ return static_cast<unsigned char>(b);
+}
+#endif
+
+namespace detail {
+// Suppresses "unused variable" warnings with the method described in
+// https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/.
+// (void)var does not work on many Intel compilers.
+template <typename... T> FMT_CONSTEXPR void ignore_unused(const T&...) {}
+
+constexpr FMT_INLINE auto is_constant_evaluated(
+ bool default_value = false) noexcept -> bool {
+// Workaround for incompatibility between libstdc++ consteval-based
+// std::is_constant_evaluated() implementation and clang-14.
+// https://github.com/fmtlib/fmt/issues/3247
+#if FMT_CPLUSPLUS >= 202002L && defined(_GLIBCXX_RELEASE) && \
+ _GLIBCXX_RELEASE >= 12 && \
+ (FMT_CLANG_VERSION >= 1400 && FMT_CLANG_VERSION < 1500)
+ ignore_unused(default_value);
+ return __builtin_is_constant_evaluated();
+#elif defined(__cpp_lib_is_constant_evaluated)
+ ignore_unused(default_value);
+ return std::is_constant_evaluated();
+#else
+ return default_value;
+#endif
+}
+
+// Suppresses "conditional expression is constant" warnings.
+template <typename T> constexpr FMT_INLINE auto const_check(T value) -> T {
+ return value;
+}
+
+FMT_NORETURN FMT_API void assert_fail(const char* file, int line,
+ const char* message);
+
+#ifndef FMT_ASSERT
+# ifdef NDEBUG
+// FMT_ASSERT is not empty to avoid -Wempty-body.
+# define FMT_ASSERT(condition, message) \
+ fmt::detail::ignore_unused((condition), (message))
+# else
+# define FMT_ASSERT(condition, message) \
+ ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \
+ ? (void)0 \
+ : fmt::detail::assert_fail(__FILE__, __LINE__, (message)))
+# endif
+#endif
+
+#if defined(FMT_USE_STRING_VIEW)
+template <typename Char> using std_string_view = std::basic_string_view<Char>;
+#elif defined(FMT_USE_EXPERIMENTAL_STRING_VIEW)
+template <typename Char>
+using std_string_view = std::experimental::basic_string_view<Char>;
+#else
+template <typename T> struct std_string_view {};
+#endif
+
+#ifdef FMT_USE_INT128
+// Do nothing.
+#elif defined(__SIZEOF_INT128__) && !defined(__NVCC__) && \
+ !(FMT_CLANG_VERSION && FMT_MSC_VERSION)
+# define FMT_USE_INT128 1
+using int128_opt = __int128_t; // An optional native 128-bit integer.
+using uint128_opt = __uint128_t;
+template <typename T> inline auto convert_for_visit(T value) -> T {
+ return value;
+}
+#else
+# define FMT_USE_INT128 0
+#endif
+#if !FMT_USE_INT128
+enum class int128_opt {};
+enum class uint128_opt {};
+// Reduce template instantiations.
+template <typename T> auto convert_for_visit(T) -> monostate { return {}; }
+#endif
+
+// Casts a nonnegative integer to unsigned.
+template <typename Int>
+FMT_CONSTEXPR auto to_unsigned(Int value) ->
+ typename std::make_unsigned<Int>::type {
+ FMT_ASSERT(std::is_unsigned<Int>::value || value >= 0, "negative value");
+ return static_cast<typename std::make_unsigned<Int>::type>(value);
+}
+
+FMT_CONSTEXPR inline auto is_utf8() -> bool {
+ FMT_MSC_WARNING(suppress : 4566) constexpr unsigned char section[] = "\u00A7";
+
+ // Avoid buggy sign extensions in MSVC's constant evaluation mode (#2297).
+ using uchar = unsigned char;
+ return FMT_UNICODE || (sizeof(section) == 3 && uchar(section[0]) == 0xC2 &&
+ uchar(section[1]) == 0xA7);
+}
+} // namespace detail
+
+/**
+ An implementation of ``std::basic_string_view`` for pre-C++17. It provides a
+ subset of the API. ``fmt::basic_string_view`` is used for format strings even
+ if ``std::string_view`` is available to prevent issues when a library is
+ compiled with a different ``-std`` option than the client code (which is not
+ recommended).
+ */
+FMT_MODULE_EXPORT
+template <typename Char> class basic_string_view {
+ private:
+ const Char* data_;
+ size_t size_;
+
+ public:
+ using value_type = Char;
+ using iterator = const Char*;
+
+ constexpr basic_string_view() noexcept : data_(nullptr), size_(0) {}
+
+ /** Constructs a string reference object from a C string and a size. */
+ constexpr basic_string_view(const Char* s, size_t count) noexcept
+ : data_(s), size_(count) {}
+
+ /**
+ \rst
+ Constructs a string reference object from a C string computing
+ the size with ``std::char_traits<Char>::length``.
+ \endrst
+ */
+ FMT_CONSTEXPR_CHAR_TRAITS
+ FMT_INLINE
+ basic_string_view(const Char* s)
+ : data_(s),
+ size_(detail::const_check(std::is_same<Char, char>::value &&
+ !detail::is_constant_evaluated(true))
+ ? std::strlen(reinterpret_cast<const char*>(s))
+ : std::char_traits<Char>::length(s)) {}
+
+ /** Constructs a string reference from a ``std::basic_string`` object. */
+ template <typename Traits, typename Alloc>
+ FMT_CONSTEXPR basic_string_view(
+ const std::basic_string<Char, Traits, Alloc>& s) noexcept
+ : data_(s.data()), size_(s.size()) {}
+
+ template <typename S, FMT_ENABLE_IF(std::is_same<
+ S, detail::std_string_view<Char>>::value)>
+ FMT_CONSTEXPR basic_string_view(S s) noexcept
+ : data_(s.data()), size_(s.size()) {}
+
+ /** Returns a pointer to the string data. */
+ constexpr auto data() const noexcept -> const Char* { return data_; }
+
+ /** Returns the string size. */
+ constexpr auto size() const noexcept -> size_t { return size_; }
+
+ constexpr auto begin() const noexcept -> iterator { return data_; }
+ constexpr auto end() const noexcept -> iterator { return data_ + size_; }
+
+ constexpr auto operator[](size_t pos) const noexcept -> const Char& {
+ return data_[pos];
+ }
+
+ FMT_CONSTEXPR void remove_prefix(size_t n) noexcept {
+ data_ += n;
+ size_ -= n;
+ }
+
+ FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(
+ basic_string_view<Char> sv) const noexcept {
+ return size_ >= sv.size_ &&
+ std::char_traits<Char>::compare(data_, sv.data_, sv.size_) == 0;
+ }
+ FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(Char c) const noexcept {
+ return size_ >= 1 && std::char_traits<Char>::eq(*data_, c);
+ }
+ FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(const Char* s) const {
+ return starts_with(basic_string_view<Char>(s));
+ }
+
+ // Lexicographically compare this string reference to other.
+ FMT_CONSTEXPR_CHAR_TRAITS auto compare(basic_string_view other) const -> int {
+ size_t str_size = size_ < other.size_ ? size_ : other.size_;
+ int result = std::char_traits<Char>::compare(data_, other.data_, str_size);
+ if (result == 0)
+ result = size_ == other.size_ ? 0 : (size_ < other.size_ ? -1 : 1);
+ return result;
+ }
+
+ FMT_CONSTEXPR_CHAR_TRAITS friend auto operator==(basic_string_view lhs,
+ basic_string_view rhs)
+ -> bool {
+ return lhs.compare(rhs) == 0;
+ }
+ friend auto operator!=(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) != 0;
+ }
+ friend auto operator<(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) < 0;
+ }
+ friend auto operator<=(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) <= 0;
+ }
+ friend auto operator>(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) > 0;
+ }
+ friend auto operator>=(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) >= 0;
+ }
+};
+
+FMT_MODULE_EXPORT
+using string_view = basic_string_view<char>;
+
+/** Specifies if ``T`` is a character type. Can be specialized by users. */
+FMT_MODULE_EXPORT
+template <typename T> struct is_char : std::false_type {};
+template <> struct is_char<char> : std::true_type {};
+
+namespace detail {
+
+// A base class for compile-time strings.
+struct compile_string {};
+
+template <typename S>
+struct is_compile_string : std::is_base_of<compile_string, S> {};
+
+template <typename Char, FMT_ENABLE_IF(is_char<Char>::value)>
+FMT_INLINE auto to_string_view(const Char* s) -> basic_string_view<Char> {
+ return s;
+}
+template <typename Char, typename Traits, typename Alloc>
+inline auto to_string_view(const std::basic_string<Char, Traits, Alloc>& s)
+ -> basic_string_view<Char> {
+ return s;
+}
+template <typename Char>
+constexpr auto to_string_view(basic_string_view<Char> s)
+ -> basic_string_view<Char> {
+ return s;
+}
+template <typename Char,
+ FMT_ENABLE_IF(!std::is_empty<std_string_view<Char>>::value)>
+inline auto to_string_view(std_string_view<Char> s) -> basic_string_view<Char> {
+ return s;
+}
+template <typename S, FMT_ENABLE_IF(is_compile_string<S>::value)>
+constexpr auto to_string_view(const S& s)
+ -> basic_string_view<typename S::char_type> {
+ return basic_string_view<typename S::char_type>(s);
+}
+void to_string_view(...);
+
+// Specifies whether S is a string type convertible to fmt::basic_string_view.
+// It should be a constexpr function but MSVC 2017 fails to compile it in
+// enable_if and MSVC 2015 fails to compile it as an alias template.
+// ADL is intentionally disabled as to_string_view is not an extension point.
+template <typename S>
+struct is_string
+ : std::is_class<decltype(detail::to_string_view(std::declval<S>()))> {};
+
+template <typename S, typename = void> struct char_t_impl {};
+template <typename S> struct char_t_impl<S, enable_if_t<is_string<S>::value>> {
+ using result = decltype(to_string_view(std::declval<S>()));
+ using type = typename result::value_type;
+};
+
+enum class type {
+ none_type,
+ // Integer types should go first,
+ int_type,
+ uint_type,
+ long_long_type,
+ ulong_long_type,
+ int128_type,
+ uint128_type,
+ bool_type,
+ char_type,
+ last_integer_type = char_type,
+ // followed by floating-point types.
+ float_type,
+ double_type,
+ long_double_type,
+ last_numeric_type = long_double_type,
+ cstring_type,
+ string_type,
+ pointer_type,
+ custom_type
+};
+
+// Maps core type T to the corresponding type enum constant.
+template <typename T, typename Char>
+struct type_constant : std::integral_constant<type, type::custom_type> {};
+
+#define FMT_TYPE_CONSTANT(Type, constant) \
+ template <typename Char> \
+ struct type_constant<Type, Char> \
+ : std::integral_constant<type, type::constant> {}
+
+FMT_TYPE_CONSTANT(int, int_type);
+FMT_TYPE_CONSTANT(unsigned, uint_type);
+FMT_TYPE_CONSTANT(long long, long_long_type);
+FMT_TYPE_CONSTANT(unsigned long long, ulong_long_type);
+FMT_TYPE_CONSTANT(int128_opt, int128_type);
+FMT_TYPE_CONSTANT(uint128_opt, uint128_type);
+FMT_TYPE_CONSTANT(bool, bool_type);
+FMT_TYPE_CONSTANT(Char, char_type);
+FMT_TYPE_CONSTANT(float, float_type);
+FMT_TYPE_CONSTANT(double, double_type);
+FMT_TYPE_CONSTANT(long double, long_double_type);
+FMT_TYPE_CONSTANT(const Char*, cstring_type);
+FMT_TYPE_CONSTANT(basic_string_view<Char>, string_type);
+FMT_TYPE_CONSTANT(const void*, pointer_type);
+
+constexpr bool is_integral_type(type t) {
+ return t > type::none_type && t <= type::last_integer_type;
+}
+constexpr bool is_arithmetic_type(type t) {
+ return t > type::none_type && t <= type::last_numeric_type;
+}
+
+constexpr auto set(type rhs) -> int { return 1 << static_cast<int>(rhs); }
+constexpr auto in(type t, int set) -> bool {
+ return ((set >> static_cast<int>(t)) & 1) != 0;
+}
+
+// Bitsets of types.
+enum {
+ sint_set =
+ set(type::int_type) | set(type::long_long_type) | set(type::int128_type),
+ uint_set = set(type::uint_type) | set(type::ulong_long_type) |
+ set(type::uint128_type),
+ bool_set = set(type::bool_type),
+ char_set = set(type::char_type),
+ float_set = set(type::float_type) | set(type::double_type) |
+ set(type::long_double_type),
+ string_set = set(type::string_type),
+ cstring_set = set(type::cstring_type),
+ pointer_set = set(type::pointer_type)
+};
+
+FMT_NORETURN FMT_API void throw_format_error(const char* message);
+
+struct error_handler {
+ constexpr error_handler() = default;
+
+ // This function is intentionally not constexpr to give a compile-time error.
+ FMT_NORETURN void on_error(const char* message) {
+ throw_format_error(message);
+ }
+};
+} // namespace detail
+
+/** String's character type. */
+template <typename S> using char_t = typename detail::char_t_impl<S>::type;
+
+/**
+ \rst
+ Parsing context consisting of a format string range being parsed and an
+ argument counter for automatic indexing.
+ You can use the ``format_parse_context`` type alias for ``char`` instead.
+ \endrst
+ */
+FMT_MODULE_EXPORT
+template <typename Char> class basic_format_parse_context {
+ private:
+ basic_string_view<Char> format_str_;
+ int next_arg_id_;
+
+ FMT_CONSTEXPR void do_check_arg_id(int id);
+
+ public:
+ using char_type = Char;
+ using iterator = const Char*;
+
+ explicit constexpr basic_format_parse_context(
+ basic_string_view<Char> format_str, int next_arg_id = 0)
+ : format_str_(format_str), next_arg_id_(next_arg_id) {}
+
+ /**
+ Returns an iterator to the beginning of the format string range being
+ parsed.
+ */
+ constexpr auto begin() const noexcept -> iterator {
+ return format_str_.begin();
+ }
+
+ /**
+ Returns an iterator past the end of the format string range being parsed.
+ */
+ constexpr auto end() const noexcept -> iterator { return format_str_.end(); }
+
+ /** Advances the begin iterator to ``it``. */
+ FMT_CONSTEXPR void advance_to(iterator it) {
+ format_str_.remove_prefix(detail::to_unsigned(it - begin()));
+ }
+
+ /**
+ Reports an error if using the manual argument indexing; otherwise returns
+ the next argument index and switches to the automatic indexing.
+ */
+ FMT_CONSTEXPR auto next_arg_id() -> int {
+ if (next_arg_id_ < 0) {
+ detail::throw_format_error(
+ "cannot switch from manual to automatic argument indexing");
+ return 0;
+ }
+ int id = next_arg_id_++;
+ do_check_arg_id(id);
+ return id;
+ }
+
+ /**
+ Reports an error if using the automatic argument indexing; otherwise
+ switches to the manual indexing.
+ */
+ FMT_CONSTEXPR void check_arg_id(int id) {
+ if (next_arg_id_ > 0) {
+ detail::throw_format_error(
+ "cannot switch from automatic to manual argument indexing");
+ return;
+ }
+ next_arg_id_ = -1;
+ do_check_arg_id(id);
+ }
+ FMT_CONSTEXPR void check_arg_id(basic_string_view<Char>) {}
+ FMT_CONSTEXPR void check_dynamic_spec(int arg_id);
+};
+
+FMT_MODULE_EXPORT
+using format_parse_context = basic_format_parse_context<char>;
+
+namespace detail {
+// A parse context with extra data used only in compile-time checks.
+template <typename Char>
+class compile_parse_context : public basic_format_parse_context<Char> {
+ private:
+ int num_args_;
+ const type* types_;
+ using base = basic_format_parse_context<Char>;
+
+ public:
+ explicit FMT_CONSTEXPR compile_parse_context(
+ basic_string_view<Char> format_str, int num_args, const type* types,
+ int next_arg_id = 0)
+ : base(format_str, next_arg_id), num_args_(num_args), types_(types) {}
+
+ constexpr auto num_args() const -> int { return num_args_; }
+ constexpr auto arg_type(int id) const -> type { return types_[id]; }
+
+ FMT_CONSTEXPR auto next_arg_id() -> int {
+ int id = base::next_arg_id();
+ if (id >= num_args_) throw_format_error("argument not found");
+ return id;
+ }
+
+ FMT_CONSTEXPR void check_arg_id(int id) {
+ base::check_arg_id(id);
+ if (id >= num_args_) throw_format_error("argument not found");
+ }
+ using base::check_arg_id;
+
+ FMT_CONSTEXPR void check_dynamic_spec(int arg_id) {
+ detail::ignore_unused(arg_id);
+#if !defined(__LCC__)
+ if (arg_id < num_args_ && types_ && !is_integral_type(types_[arg_id]))
+ throw_format_error("width/precision is not integer");
+#endif
+ }
+};
+} // namespace detail
+
+template <typename Char>
+FMT_CONSTEXPR void basic_format_parse_context<Char>::do_check_arg_id(int id) {
+ // Argument id is only checked at compile-time during parsing because
+ // formatting has its own validation.
+ if (detail::is_constant_evaluated() &&
+ (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) {
+ using context = detail::compile_parse_context<Char>;
+ if (id >= static_cast<context*>(this)->num_args())
+ detail::throw_format_error("argument not found");
+ }
+}
+
+template <typename Char>
+FMT_CONSTEXPR void basic_format_parse_context<Char>::check_dynamic_spec(
+ int arg_id) {
+ if (detail::is_constant_evaluated() &&
+ (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) {
+ using context = detail::compile_parse_context<Char>;
+ static_cast<context*>(this)->check_dynamic_spec(arg_id);
+ }
+}
+
+FMT_MODULE_EXPORT template <typename Context> class basic_format_arg;
+FMT_MODULE_EXPORT template <typename Context> class basic_format_args;
+FMT_MODULE_EXPORT template <typename Context> class dynamic_format_arg_store;
+
+// A formatter for objects of type T.
+FMT_MODULE_EXPORT
+template <typename T, typename Char = char, typename Enable = void>
+struct formatter {
+ // A deleted default constructor indicates a disabled formatter.
+ formatter() = delete;
+};
+
+// Specifies if T has an enabled formatter specialization. A type can be
+// formattable even if it doesn't have a formatter e.g. via a conversion.
+template <typename T, typename Context>
+using has_formatter =
+ std::is_constructible<typename Context::template formatter_type<T>>;
+
+// Checks whether T is a container with contiguous storage.
+template <typename T> struct is_contiguous : std::false_type {};
+template <typename Char>
+struct is_contiguous<std::basic_string<Char>> : std::true_type {};
+
+class appender;
+
+namespace detail {
+
+template <typename Context, typename T>
+constexpr auto has_const_formatter_impl(T*)
+ -> decltype(typename Context::template formatter_type<T>().format(
+ std::declval<const T&>(), std::declval<Context&>()),
+ true) {
+ return true;
+}
+template <typename Context>
+constexpr auto has_const_formatter_impl(...) -> bool {
+ return false;
+}
+template <typename T, typename Context>
+constexpr auto has_const_formatter() -> bool {
+ return has_const_formatter_impl<Context>(static_cast<T*>(nullptr));
+}
+
+// Extracts a reference to the container from back_insert_iterator.
+template <typename Container>
+inline auto get_container(std::back_insert_iterator<Container> it)
+ -> Container& {
+ using base = std::back_insert_iterator<Container>;
+ struct accessor : base {
+ accessor(base b) : base(b) {}
+ using base::container;
+ };
+ return *accessor(it).container;
+}
+
+template <typename Char, typename InputIt, typename OutputIt>
+FMT_CONSTEXPR auto copy_str(InputIt begin, InputIt end, OutputIt out)
+ -> OutputIt {
+ while (begin != end) *out++ = static_cast<Char>(*begin++);
+ return out;
+}
+
+template <typename Char, typename T, typename U,
+ FMT_ENABLE_IF(
+ std::is_same<remove_const_t<T>, U>::value&& is_char<U>::value)>
+FMT_CONSTEXPR auto copy_str(T* begin, T* end, U* out) -> U* {
+ if (is_constant_evaluated()) return copy_str<Char, T*, U*>(begin, end, out);
+ auto size = to_unsigned(end - begin);
+ if (size > 0) memcpy(out, begin, size * sizeof(U));
+ return out + size;
+}
+
+/**
+ \rst
+ A contiguous memory buffer with an optional growing ability. It is an internal
+ class and shouldn't be used directly, only via `~fmt::basic_memory_buffer`.
+ \endrst
+ */
+template <typename T> class buffer {
+ private:
+ T* ptr_;
+ size_t size_;
+ size_t capacity_;
+
+ protected:
+ // Don't initialize ptr_ since it is not accessed to save a few cycles.
+ FMT_MSC_WARNING(suppress : 26495)
+ buffer(size_t sz) noexcept : size_(sz), capacity_(sz) {}
+
+ FMT_CONSTEXPR20 buffer(T* p = nullptr, size_t sz = 0, size_t cap = 0) noexcept
+ : ptr_(p), size_(sz), capacity_(cap) {}
+
+ FMT_CONSTEXPR20 ~buffer() = default;
+ buffer(buffer&&) = default;
+
+ /** Sets the buffer data and capacity. */
+ FMT_CONSTEXPR void set(T* buf_data, size_t buf_capacity) noexcept {
+ ptr_ = buf_data;
+ capacity_ = buf_capacity;
+ }
+
+ /** Increases the buffer capacity to hold at least *capacity* elements. */
+ virtual FMT_CONSTEXPR20 void grow(size_t capacity) = 0;
+
+ public:
+ using value_type = T;
+ using const_reference = const T&;
+
+ buffer(const buffer&) = delete;
+ void operator=(const buffer&) = delete;
+
+ FMT_INLINE auto begin() noexcept -> T* { return ptr_; }
+ FMT_INLINE auto end() noexcept -> T* { return ptr_ + size_; }
+
+ FMT_INLINE auto begin() const noexcept -> const T* { return ptr_; }
+ FMT_INLINE auto end() const noexcept -> const T* { return ptr_ + size_; }
+
+ /** Returns the size of this buffer. */
+ constexpr auto size() const noexcept -> size_t { return size_; }
+
+ /** Returns the capacity of this buffer. */
+ constexpr auto capacity() const noexcept -> size_t { return capacity_; }
+
+ /** Returns a pointer to the buffer data. */
+ FMT_CONSTEXPR auto data() noexcept -> T* { return ptr_; }
+
+ /** Returns a pointer to the buffer data. */
+ FMT_CONSTEXPR auto data() const noexcept -> const T* { return ptr_; }
+
+ /** Clears this buffer. */
+ void clear() { size_ = 0; }
+
+ // Tries resizing the buffer to contain *count* elements. If T is a POD type
+ // the new elements may not be initialized.
+ FMT_CONSTEXPR20 void try_resize(size_t count) {
+ try_reserve(count);
+ size_ = count <= capacity_ ? count : capacity_;
+ }
+
+ // Tries increasing the buffer capacity to *new_capacity*. It can increase the
+ // capacity by a smaller amount than requested but guarantees there is space
+ // for at least one additional element either by increasing the capacity or by
+ // flushing the buffer if it is full.
+ FMT_CONSTEXPR20 void try_reserve(size_t new_capacity) {
+ if (new_capacity > capacity_) grow(new_capacity);
+ }
+
+ FMT_CONSTEXPR20 void push_back(const T& value) {
+ try_reserve(size_ + 1);
+ ptr_[size_++] = value;
+ }
+
+ /** Appends data to the end of the buffer. */
+ template <typename U> void append(const U* begin, const U* end);
+
+ template <typename Idx> FMT_CONSTEXPR auto operator[](Idx index) -> T& {
+ return ptr_[index];
+ }
+ template <typename Idx>
+ FMT_CONSTEXPR auto operator[](Idx index) const -> const T& {
+ return ptr_[index];
+ }
+};
+
+struct buffer_traits {
+ explicit buffer_traits(size_t) {}
+ auto count() const -> size_t { return 0; }
+ auto limit(size_t size) -> size_t { return size; }
+};
+
+class fixed_buffer_traits {
+ private:
+ size_t count_ = 0;
+ size_t limit_;
+
+ public:
+ explicit fixed_buffer_traits(size_t limit) : limit_(limit) {}
+ auto count() const -> size_t { return count_; }
+ auto limit(size_t size) -> size_t {
+ size_t n = limit_ > count_ ? limit_ - count_ : 0;
+ count_ += size;
+ return size < n ? size : n;
+ }
+};
+
+// A buffer that writes to an output iterator when flushed.
+template <typename OutputIt, typename T, typename Traits = buffer_traits>
+class iterator_buffer final : public Traits, public buffer<T> {
+ private:
+ OutputIt out_;
+ enum { buffer_size = 256 };
+ T data_[buffer_size];
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {
+ if (this->size() == buffer_size) flush();
+ }
+
+ void flush() {
+ auto size = this->size();
+ this->clear();
+ out_ = copy_str<T>(data_, data_ + this->limit(size), out_);
+ }
+
+ public:
+ explicit iterator_buffer(OutputIt out, size_t n = buffer_size)
+ : Traits(n), buffer<T>(data_, 0, buffer_size), out_(out) {}
+ iterator_buffer(iterator_buffer&& other)
+ : Traits(other), buffer<T>(data_, 0, buffer_size), out_(other.out_) {}
+ ~iterator_buffer() { flush(); }
+
+ auto out() -> OutputIt {
+ flush();
+ return out_;
+ }
+ auto count() const -> size_t { return Traits::count() + this->size(); }
+};
+
+template <typename T>
+class iterator_buffer<T*, T, fixed_buffer_traits> final
+ : public fixed_buffer_traits,
+ public buffer<T> {
+ private:
+ T* out_;
+ enum { buffer_size = 256 };
+ T data_[buffer_size];
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {
+ if (this->size() == this->capacity()) flush();
+ }
+
+ void flush() {
+ size_t n = this->limit(this->size());
+ if (this->data() == out_) {
+ out_ += n;
+ this->set(data_, buffer_size);
+ }
+ this->clear();
+ }
+
+ public:
+ explicit iterator_buffer(T* out, size_t n = buffer_size)
+ : fixed_buffer_traits(n), buffer<T>(out, 0, n), out_(out) {}
+ iterator_buffer(iterator_buffer&& other)
+ : fixed_buffer_traits(other),
+ buffer<T>(std::move(other)),
+ out_(other.out_) {
+ if (this->data() != out_) {
+ this->set(data_, buffer_size);
+ this->clear();
+ }
+ }
+ ~iterator_buffer() { flush(); }
+
+ auto out() -> T* {
+ flush();
+ return out_;
+ }
+ auto count() const -> size_t {
+ return fixed_buffer_traits::count() + this->size();
+ }
+};
+
+template <typename T> class iterator_buffer<T*, T> final : public buffer<T> {
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {}
+
+ public:
+ explicit iterator_buffer(T* out, size_t = 0) : buffer<T>(out, 0, ~size_t()) {}
+
+ auto out() -> T* { return &*this->end(); }
+};
+
+// A buffer that writes to a container with the contiguous storage.
+template <typename Container>
+class iterator_buffer<std::back_insert_iterator<Container>,
+ enable_if_t<is_contiguous<Container>::value,
+ typename Container::value_type>>
+ final : public buffer<typename Container::value_type> {
+ private:
+ Container& container_;
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t capacity) override {
+ container_.resize(capacity);
+ this->set(&container_[0], capacity);
+ }
+
+ public:
+ explicit iterator_buffer(Container& c)
+ : buffer<typename Container::value_type>(c.size()), container_(c) {}
+ explicit iterator_buffer(std::back_insert_iterator<Container> out, size_t = 0)
+ : iterator_buffer(get_container(out)) {}
+
+ auto out() -> std::back_insert_iterator<Container> {
+ return std::back_inserter(container_);
+ }
+};
+
+// A buffer that counts the number of code units written discarding the output.
+template <typename T = char> class counting_buffer final : public buffer<T> {
+ private:
+ enum { buffer_size = 256 };
+ T data_[buffer_size];
+ size_t count_ = 0;
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {
+ if (this->size() != buffer_size) return;
+ count_ += this->size();
+ this->clear();
+ }
+
+ public:
+ counting_buffer() : buffer<T>(data_, 0, buffer_size) {}
+
+ auto count() -> size_t { return count_ + this->size(); }
+};
+
+template <typename T>
+using buffer_appender = conditional_t<std::is_same<T, char>::value, appender,
+ std::back_insert_iterator<buffer<T>>>;
+
+// Maps an output iterator to a buffer.
+template <typename T, typename OutputIt>
+auto get_buffer(OutputIt out) -> iterator_buffer<OutputIt, T> {
+ return iterator_buffer<OutputIt, T>(out);
+}
+template <typename T, typename Buf,
+ FMT_ENABLE_IF(std::is_base_of<buffer<char>, Buf>::value)>
+auto get_buffer(std::back_insert_iterator<Buf> out) -> buffer<char>& {
+ return get_container(out);
+}
+
+template <typename Buf, typename OutputIt>
+FMT_INLINE auto get_iterator(Buf& buf, OutputIt) -> decltype(buf.out()) {
+ return buf.out();
+}
+template <typename T, typename OutputIt>
+auto get_iterator(buffer<T>&, OutputIt out) -> OutputIt {
+ return out;
+}
+
+struct view {};
+
+template <typename Char, typename T> struct named_arg : view {
+ const Char* name;
+ const T& value;
+ named_arg(const Char* n, const T& v) : name(n), value(v) {}
+};
+
+template <typename Char> struct named_arg_info {
+ const Char* name;
+ int id;
+};
+
+template <typename T, typename Char, size_t NUM_ARGS, size_t NUM_NAMED_ARGS>
+struct arg_data {
+ // args_[0].named_args points to named_args_ to avoid bloating format_args.
+ // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning.
+ T args_[1 + (NUM_ARGS != 0 ? NUM_ARGS : +1)];
+ named_arg_info<Char> named_args_[NUM_NAMED_ARGS];
+
+ template <typename... U>
+ arg_data(const U&... init) : args_{T(named_args_, NUM_NAMED_ARGS), init...} {}
+ arg_data(const arg_data& other) = delete;
+ auto args() const -> const T* { return args_ + 1; }
+ auto named_args() -> named_arg_info<Char>* { return named_args_; }
+};
+
+template <typename T, typename Char, size_t NUM_ARGS>
+struct arg_data<T, Char, NUM_ARGS, 0> {
+ // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning.
+ T args_[NUM_ARGS != 0 ? NUM_ARGS : +1];
+
+ template <typename... U>
+ FMT_CONSTEXPR FMT_INLINE arg_data(const U&... init) : args_{init...} {}
+ FMT_CONSTEXPR FMT_INLINE auto args() const -> const T* { return args_; }
+ FMT_CONSTEXPR FMT_INLINE auto named_args() -> std::nullptr_t {
+ return nullptr;
+ }
+};
+
+template <typename Char>
+inline void init_named_args(named_arg_info<Char>*, int, int) {}
+
+template <typename T> struct is_named_arg : std::false_type {};
+template <typename T> struct is_statically_named_arg : std::false_type {};
+
+template <typename T, typename Char>
+struct is_named_arg<named_arg<Char, T>> : std::true_type {};
+
+template <typename Char, typename T, typename... Tail,
+ FMT_ENABLE_IF(!is_named_arg<T>::value)>
+void init_named_args(named_arg_info<Char>* named_args, int arg_count,
+ int named_arg_count, const T&, const Tail&... args) {
+ init_named_args(named_args, arg_count + 1, named_arg_count, args...);
+}
+
+template <typename Char, typename T, typename... Tail,
+ FMT_ENABLE_IF(is_named_arg<T>::value)>
+void init_named_args(named_arg_info<Char>* named_args, int arg_count,
+ int named_arg_count, const T& arg, const Tail&... args) {
+ named_args[named_arg_count++] = {arg.name, arg_count};
+ init_named_args(named_args, arg_count + 1, named_arg_count, args...);
+}
+
+template <typename... Args>
+FMT_CONSTEXPR FMT_INLINE void init_named_args(std::nullptr_t, int, int,
+ const Args&...) {}
+
+template <bool B = false> constexpr auto count() -> size_t { return B ? 1 : 0; }
+template <bool B1, bool B2, bool... Tail> constexpr auto count() -> size_t {
+ return (B1 ? 1 : 0) + count<B2, Tail...>();
+}
+
+template <typename... Args> constexpr auto count_named_args() -> size_t {
+ return count<is_named_arg<Args>::value...>();
+}
+
+template <typename... Args>
+constexpr auto count_statically_named_args() -> size_t {
+ return count<is_statically_named_arg<Args>::value...>();
+}
+
+struct unformattable {};
+struct unformattable_char : unformattable {};
+struct unformattable_pointer : unformattable {};
+
+template <typename Char> struct string_value {
+ const Char* data;
+ size_t size;
+};
+
+template <typename Char> struct named_arg_value {
+ const named_arg_info<Char>* data;
+ size_t size;
+};
+
+template <typename Context> struct custom_value {
+ using parse_context = typename Context::parse_context_type;
+ void* value;
+ void (*format)(void* arg, parse_context& parse_ctx, Context& ctx);
+};
+
+// A formatting argument value.
+template <typename Context> class value {
+ public:
+ using char_type = typename Context::char_type;
+
+ union {
+ monostate no_value;
+ int int_value;
+ unsigned uint_value;
+ long long long_long_value;
+ unsigned long long ulong_long_value;
+ int128_opt int128_value;
+ uint128_opt uint128_value;
+ bool bool_value;
+ char_type char_value;
+ float float_value;
+ double double_value;
+ long double long_double_value;
+ const void* pointer;
+ string_value<char_type> string;
+ custom_value<Context> custom;
+ named_arg_value<char_type> named_args;
+ };
+
+ constexpr FMT_INLINE value() : no_value() {}
+ constexpr FMT_INLINE value(int val) : int_value(val) {}
+ constexpr FMT_INLINE value(unsigned val) : uint_value(val) {}
+ constexpr FMT_INLINE value(long long val) : long_long_value(val) {}
+ constexpr FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {}
+ FMT_INLINE value(int128_opt val) : int128_value(val) {}
+ FMT_INLINE value(uint128_opt val) : uint128_value(val) {}
+ constexpr FMT_INLINE value(float val) : float_value(val) {}
+ constexpr FMT_INLINE value(double val) : double_value(val) {}
+ FMT_INLINE value(long double val) : long_double_value(val) {}
+ constexpr FMT_INLINE value(bool val) : bool_value(val) {}
+ constexpr FMT_INLINE value(char_type val) : char_value(val) {}
+ FMT_CONSTEXPR FMT_INLINE value(const char_type* val) {
+ string.data = val;
+ if (is_constant_evaluated()) string.size = {};
+ }
+ FMT_CONSTEXPR FMT_INLINE value(basic_string_view<char_type> val) {
+ string.data = val.data();
+ string.size = val.size();
+ }
+ FMT_INLINE value(const void* val) : pointer(val) {}
+ FMT_INLINE value(const named_arg_info<char_type>* args, size_t size)
+ : named_args{args, size} {}
+
+ template <typename T> FMT_CONSTEXPR FMT_INLINE value(T& val) {
+ using value_type = remove_cvref_t<T>;
+ custom.value = const_cast<value_type*>(&val);
+ // Get the formatter type through the context to allow different contexts
+ // have different extension points, e.g. `formatter<T>` for `format` and
+ // `printf_formatter<T>` for `printf`.
+ custom.format = format_custom_arg<
+ value_type, typename Context::template formatter_type<value_type>>;
+ }
+ value(unformattable);
+ value(unformattable_char);
+ value(unformattable_pointer);
+
+ private:
+ // Formats an argument of a custom type, such as a user-defined class.
+ template <typename T, typename Formatter>
+ static void format_custom_arg(void* arg,
+ typename Context::parse_context_type& parse_ctx,
+ Context& ctx) {
+ auto f = Formatter();
+ parse_ctx.advance_to(f.parse(parse_ctx));
+ using qualified_type =
+ conditional_t<has_const_formatter<T, Context>(), const T, T>;
+ ctx.advance_to(f.format(*static_cast<qualified_type*>(arg), ctx));
+ }
+};
+
+template <typename Context, typename T>
+FMT_CONSTEXPR auto make_arg(T&& value) -> basic_format_arg<Context>;
+
+// To minimize the number of types we need to deal with, long is translated
+// either to int or to long long depending on its size.
+enum { long_short = sizeof(long) == sizeof(int) };
+using long_type = conditional_t<long_short, int, long long>;
+using ulong_type = conditional_t<long_short, unsigned, unsigned long long>;
+
+template <typename T> struct format_as_result {
+ template <typename U,
+ FMT_ENABLE_IF(std::is_enum<U>::value || std::is_class<U>::value)>
+ static auto map(U*) -> decltype(format_as(std::declval<U>()));
+ static auto map(...) -> void;
+
+ using type = decltype(map(static_cast<T*>(nullptr)));
+};
+template <typename T> using format_as_t = typename format_as_result<T>::type;
+
+template <typename T>
+struct has_format_as
+ : bool_constant<!std::is_same<format_as_t<T>, void>::value> {};
+
+// Maps formatting arguments to core types.
+// arg_mapper reports errors by returning unformattable instead of using
+// static_assert because it's used in the is_formattable trait.
+template <typename Context> struct arg_mapper {
+ using char_type = typename Context::char_type;
+
+ FMT_CONSTEXPR FMT_INLINE auto map(signed char val) -> int { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned char val) -> unsigned {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(short val) -> int { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned short val) -> unsigned {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(int val) -> int { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned val) -> unsigned { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(long val) -> long_type { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned long val) -> ulong_type {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(long long val) -> long long { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned long long val)
+ -> unsigned long long {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(int128_opt val) -> int128_opt {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(uint128_opt val) -> uint128_opt {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(bool val) -> bool { return val; }
+
+ template <typename T, FMT_ENABLE_IF(std::is_same<T, char>::value ||
+ std::is_same<T, char_type>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(T val) -> char_type {
+ return val;
+ }
+ template <typename T, enable_if_t<(std::is_same<T, wchar_t>::value ||
+#ifdef __cpp_char8_t
+ std::is_same<T, char8_t>::value ||
+#endif
+ std::is_same<T, char16_t>::value ||
+ std::is_same<T, char32_t>::value) &&
+ !std::is_same<T, char_type>::value,
+ int> = 0>
+ FMT_CONSTEXPR FMT_INLINE auto map(T) -> unformattable_char {
+ return {};
+ }
+
+ FMT_CONSTEXPR FMT_INLINE auto map(float val) -> float { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(double val) -> double { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(long double val) -> long double {
+ return val;
+ }
+
+ FMT_CONSTEXPR FMT_INLINE auto map(char_type* val) -> const char_type* {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(const char_type* val) -> const char_type* {
+ return val;
+ }
+ template <typename T,
+ FMT_ENABLE_IF(is_string<T>::value && !std::is_pointer<T>::value &&
+ std::is_same<char_type, char_t<T>>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T& val)
+ -> basic_string_view<char_type> {
+ return to_string_view(val);
+ }
+ template <typename T,
+ FMT_ENABLE_IF(is_string<T>::value && !std::is_pointer<T>::value &&
+ !std::is_same<char_type, char_t<T>>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T&) -> unformattable_char {
+ return {};
+ }
+
+ FMT_CONSTEXPR FMT_INLINE auto map(void* val) -> const void* { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(const void* val) -> const void* {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(std::nullptr_t val) -> const void* {
+ return val;
+ }
+
+ // Use SFINAE instead of a const T* parameter to avoid a conflict with the
+ // array overload.
+ template <
+ typename T,
+ FMT_ENABLE_IF(
+ std::is_pointer<T>::value || std::is_member_pointer<T>::value ||
+ std::is_function<typename std::remove_pointer<T>::type>::value ||
+ (std::is_convertible<const T&, const void*>::value &&
+ !std::is_convertible<const T&, const char_type*>::value &&
+ !has_formatter<T, Context>::value))>
+ FMT_CONSTEXPR auto map(const T&) -> unformattable_pointer {
+ return {};
+ }
+
+ template <typename T, std::size_t N,
+ FMT_ENABLE_IF(!std::is_same<T, wchar_t>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T (&values)[N]) -> const T (&)[N] {
+ return values;
+ }
+
+ // Only map owning types because mapping views can be unsafe.
+ template <typename T, typename U = format_as_t<T>,
+ FMT_ENABLE_IF(std::is_arithmetic<U>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T& val) -> decltype(this->map(U())) {
+ return map(format_as(val));
+ }
+
+ template <typename T, typename U = remove_cvref_t<T>>
+ struct formattable
+ : bool_constant<has_const_formatter<U, Context>() ||
+ (has_formatter<U, Context>::value &&
+ !std::is_const<remove_reference_t<T>>::value)> {};
+
+ template <typename T, FMT_ENABLE_IF(formattable<T>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& {
+ return val;
+ }
+ template <typename T, FMT_ENABLE_IF(!formattable<T>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto do_map(T&&) -> unformattable {
+ return {};
+ }
+
+ template <typename T, typename U = remove_cvref_t<T>,
+ FMT_ENABLE_IF((std::is_class<U>::value || std::is_enum<U>::value ||
+ std::is_union<U>::value) &&
+ !is_string<U>::value && !is_char<U>::value &&
+ !is_named_arg<U>::value &&
+ !std::is_arithmetic<format_as_t<U>>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(T&& val)
+ -> decltype(this->do_map(std::forward<T>(val))) {
+ return do_map(std::forward<T>(val));
+ }
+
+ template <typename T, FMT_ENABLE_IF(is_named_arg<T>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T& named_arg)
+ -> decltype(this->map(named_arg.value)) {
+ return map(named_arg.value);
+ }
+
+ auto map(...) -> unformattable { return {}; }
+};
+
+// A type constant after applying arg_mapper<Context>.
+template <typename T, typename Context>
+using mapped_type_constant =
+ type_constant<decltype(arg_mapper<Context>().map(std::declval<const T&>())),
+ typename Context::char_type>;
+
+enum { packed_arg_bits = 4 };
+// Maximum number of arguments with packed types.
+enum { max_packed_args = 62 / packed_arg_bits };
+enum : unsigned long long { is_unpacked_bit = 1ULL << 63 };
+enum : unsigned long long { has_named_args_bit = 1ULL << 62 };
+} // namespace detail
+
+// An output iterator that appends to a buffer.
+// It is used to reduce symbol sizes for the common case.
+class appender : public std::back_insert_iterator<detail::buffer<char>> {
+ using base = std::back_insert_iterator<detail::buffer<char>>;
+
+ public:
+ using std::back_insert_iterator<detail::buffer<char>>::back_insert_iterator;
+ appender(base it) noexcept : base(it) {}
+ FMT_UNCHECKED_ITERATOR(appender);
+
+ auto operator++() noexcept -> appender& { return *this; }
+ auto operator++(int) noexcept -> appender { return *this; }
+};
+
+// A formatting argument. It is a trivially copyable/constructible type to
+// allow storage in basic_memory_buffer.
+template <typename Context> class basic_format_arg {
+ private:
+ detail::value<Context> value_;
+ detail::type type_;
+
+ template <typename ContextType, typename T>
+ friend FMT_CONSTEXPR auto detail::make_arg(T&& value)
+ -> basic_format_arg<ContextType>;
+
+ template <typename Visitor, typename Ctx>
+ friend FMT_CONSTEXPR auto visit_format_arg(Visitor&& vis,
+ const basic_format_arg<Ctx>& arg)
+ -> decltype(vis(0));
+
+ friend class basic_format_args<Context>;
+ friend class dynamic_format_arg_store<Context>;
+
+ using char_type = typename Context::char_type;
+
+ template <typename T, typename Char, size_t NUM_ARGS, size_t NUM_NAMED_ARGS>
+ friend struct detail::arg_data;
+
+ basic_format_arg(const detail::named_arg_info<char_type>* args, size_t size)
+ : value_(args, size) {}
+
+ public:
+ class handle {
+ public:
+ explicit handle(detail::custom_value<Context> custom) : custom_(custom) {}
+
+ void format(typename Context::parse_context_type& parse_ctx,
+ Context& ctx) const {
+ custom_.format(custom_.value, parse_ctx, ctx);
+ }
+
+ private:
+ detail::custom_value<Context> custom_;
+ };
+
+ constexpr basic_format_arg() : type_(detail::type::none_type) {}
+
+ constexpr explicit operator bool() const noexcept {
+ return type_ != detail::type::none_type;
+ }
+
+ auto type() const -> detail::type { return type_; }
+
+ auto is_integral() const -> bool { return detail::is_integral_type(type_); }
+ auto is_arithmetic() const -> bool {
+ return detail::is_arithmetic_type(type_);
+ }
+};
+
+/**
+ \rst
+ Visits an argument dispatching to the appropriate visit method based on
+ the argument type. For example, if the argument type is ``double`` then
+ ``vis(value)`` will be called with the value of type ``double``.
+ \endrst
+ */
+FMT_MODULE_EXPORT
+template <typename Visitor, typename Context>
+FMT_CONSTEXPR FMT_INLINE auto visit_format_arg(
+ Visitor&& vis, const basic_format_arg<Context>& arg) -> decltype(vis(0)) {
+ switch (arg.type_) {
+ case detail::type::none_type:
+ break;
+ case detail::type::int_type:
+ return vis(arg.value_.int_value);
+ case detail::type::uint_type:
+ return vis(arg.value_.uint_value);
+ case detail::type::long_long_type:
+ return vis(arg.value_.long_long_value);
+ case detail::type::ulong_long_type:
+ return vis(arg.value_.ulong_long_value);
+ case detail::type::int128_type:
+ return vis(detail::convert_for_visit(arg.value_.int128_value));
+ case detail::type::uint128_type:
+ return vis(detail::convert_for_visit(arg.value_.uint128_value));
+ case detail::type::bool_type:
+ return vis(arg.value_.bool_value);
+ case detail::type::char_type:
+ return vis(arg.value_.char_value);
+ case detail::type::float_type:
+ return vis(arg.value_.float_value);
+ case detail::type::double_type:
+ return vis(arg.value_.double_value);
+ case detail::type::long_double_type:
+ return vis(arg.value_.long_double_value);
+ case detail::type::cstring_type:
+ return vis(arg.value_.string.data);
+ case detail::type::string_type:
+ using sv = basic_string_view<typename Context::char_type>;
+ return vis(sv(arg.value_.string.data, arg.value_.string.size));
+ case detail::type::pointer_type:
+ return vis(arg.value_.pointer);
+ case detail::type::custom_type:
+ return vis(typename basic_format_arg<Context>::handle(arg.value_.custom));
+ }
+ return vis(monostate());
+}
+
+namespace detail {
+
+template <typename Char, typename InputIt>
+auto copy_str(InputIt begin, InputIt end, appender out) -> appender {
+ get_container(out).append(begin, end);
+ return out;
+}
+
+template <typename Char, typename R, typename OutputIt>
+FMT_CONSTEXPR auto copy_str(R&& rng, OutputIt out) -> OutputIt {
+ return detail::copy_str<Char>(rng.begin(), rng.end(), out);
+}
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500
+// A workaround for gcc 4.8 to make void_t work in a SFINAE context.
+template <typename...> struct void_t_impl { using type = void; };
+template <typename... T> using void_t = typename void_t_impl<T...>::type;
+#else
+template <typename...> using void_t = void;
+#endif
+
+template <typename It, typename T, typename Enable = void>
+struct is_output_iterator : std::false_type {};
+
+template <typename It, typename T>
+struct is_output_iterator<
+ It, T,
+ void_t<typename std::iterator_traits<It>::iterator_category,
+ decltype(*std::declval<It>() = std::declval<T>())>>
+ : std::true_type {};
+
+template <typename It> struct is_back_insert_iterator : std::false_type {};
+template <typename Container>
+struct is_back_insert_iterator<std::back_insert_iterator<Container>>
+ : std::true_type {};
+
+template <typename It>
+struct is_contiguous_back_insert_iterator : std::false_type {};
+template <typename Container>
+struct is_contiguous_back_insert_iterator<std::back_insert_iterator<Container>>
+ : is_contiguous<Container> {};
+template <>
+struct is_contiguous_back_insert_iterator<appender> : std::true_type {};
+
+// A type-erased reference to an std::locale to avoid a heavy <locale> include.
+class locale_ref {
+ private:
+ const void* locale_; // A type-erased pointer to std::locale.
+
+ public:
+ constexpr FMT_INLINE locale_ref() : locale_(nullptr) {}
+ template <typename Locale> explicit locale_ref(const Locale& loc);
+
+ explicit operator bool() const noexcept { return locale_ != nullptr; }
+
+ template <typename Locale> auto get() const -> Locale;
+};
+
+template <typename> constexpr auto encode_types() -> unsigned long long {
+ return 0;
+}
+
+template <typename Context, typename Arg, typename... Args>
+constexpr auto encode_types() -> unsigned long long {
+ return static_cast<unsigned>(mapped_type_constant<Arg, Context>::value) |
+ (encode_types<Context, Args...>() << packed_arg_bits);
+}
+
+template <typename Context, typename T>
+FMT_CONSTEXPR FMT_INLINE auto make_value(T&& val) -> value<Context> {
+ auto&& arg = arg_mapper<Context>().map(FMT_FORWARD(val));
+ using arg_type = remove_cvref_t<decltype(arg)>;
+
+ constexpr bool formattable_char =
+ !std::is_same<arg_type, unformattable_char>::value;
+ static_assert(formattable_char, "Mixing character types is disallowed.");
+
+ // Formatting of arbitrary pointers is disallowed. If you want to format a
+ // pointer cast it to `void*` or `const void*`. In particular, this forbids
+ // formatting of `[const] volatile char*` printed as bool by iostreams.
+ constexpr bool formattable_pointer =
+ !std::is_same<arg_type, unformattable_pointer>::value;
+ static_assert(formattable_pointer,
+ "Formatting of non-void pointers is disallowed.");
+
+ constexpr bool formattable = !std::is_same<arg_type, unformattable>::value;
+ static_assert(
+ formattable,
+ "Cannot format an argument. To make type T formattable provide a "
+ "formatter<T> specialization: https://fmt.dev/latest/api.html#udt");
+ return {arg};
+}
+
+template <typename Context, typename T>
+FMT_CONSTEXPR auto make_arg(T&& value) -> basic_format_arg<Context> {
+ auto arg = basic_format_arg<Context>();
+ arg.type_ = mapped_type_constant<T, Context>::value;
+ arg.value_ = make_value<Context>(value);
+ return arg;
+}
+
+// The DEPRECATED type template parameter is there to avoid an ODR violation
+// when using a fallback formatter in one translation unit and an implicit
+// conversion in another (not recommended).
+template <bool IS_PACKED, typename Context, type, typename T,
+ FMT_ENABLE_IF(IS_PACKED)>
+FMT_CONSTEXPR FMT_INLINE auto make_arg(T&& val) -> value<Context> {
+ return make_value<Context>(val);
+}
+
+template <bool IS_PACKED, typename Context, type, typename T,
+ FMT_ENABLE_IF(!IS_PACKED)>
+FMT_CONSTEXPR inline auto make_arg(T&& value) -> basic_format_arg<Context> {
+ return make_arg<Context>(value);
+}
+} // namespace detail
+FMT_BEGIN_EXPORT
+
+// Formatting context.
+template <typename OutputIt, typename Char> class basic_format_context {
+ private:
+ OutputIt out_;
+ basic_format_args<basic_format_context> args_;
+ detail::locale_ref loc_;
+
+ public:
+ using iterator = OutputIt;
+ using format_arg = basic_format_arg<basic_format_context>;
+ using format_args = basic_format_args<basic_format_context>;
+ using parse_context_type = basic_format_parse_context<Char>;
+ template <typename T> using formatter_type = formatter<T, Char>;
+
+ /** The character type for the output. */
+ using char_type = Char;
+
+ basic_format_context(basic_format_context&&) = default;
+ basic_format_context(const basic_format_context&) = delete;
+ void operator=(const basic_format_context&) = delete;
+ /**
+ Constructs a ``basic_format_context`` object. References to the arguments
+ are stored in the object so make sure they have appropriate lifetimes.
+ */
+ constexpr basic_format_context(OutputIt out, format_args ctx_args,
+ detail::locale_ref loc = {})
+ : out_(out), args_(ctx_args), loc_(loc) {}
+
+ constexpr auto arg(int id) const -> format_arg { return args_.get(id); }
+ FMT_CONSTEXPR auto arg(basic_string_view<Char> name) -> format_arg {
+ return args_.get(name);
+ }
+ FMT_CONSTEXPR auto arg_id(basic_string_view<Char> name) -> int {
+ return args_.get_id(name);
+ }
+ auto args() const -> const format_args& { return args_; }
+
+ FMT_CONSTEXPR auto error_handler() -> detail::error_handler { return {}; }
+ void on_error(const char* message) { error_handler().on_error(message); }
+
+ // Returns an iterator to the beginning of the output range.
+ FMT_CONSTEXPR auto out() -> iterator { return out_; }
+
+ // Advances the begin iterator to ``it``.
+ void advance_to(iterator it) {
+ if (!detail::is_back_insert_iterator<iterator>()) out_ = it;
+ }
+
+ FMT_CONSTEXPR auto locale() -> detail::locale_ref { return loc_; }
+};
+
+template <typename Char>
+using buffer_context =
+ basic_format_context<detail::buffer_appender<Char>, Char>;
+using format_context = buffer_context<char>;
+
+template <typename T, typename Char = char>
+using is_formattable = bool_constant<!std::is_base_of<
+ detail::unformattable, decltype(detail::arg_mapper<buffer_context<Char>>()
+ .map(std::declval<T>()))>::value>;
+
+/**
+ \rst
+ An array of references to arguments. It can be implicitly converted into
+ `~fmt::basic_format_args` for passing into type-erased formatting functions
+ such as `~fmt::vformat`.
+ \endrst
+ */
+template <typename Context, typename... Args>
+class format_arg_store
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+ // Workaround a GCC template argument substitution bug.
+ : public basic_format_args<Context>
+#endif
+{
+ private:
+ static const size_t num_args = sizeof...(Args);
+ static const size_t num_named_args = detail::count_named_args<Args...>();
+ static const bool is_packed = num_args <= detail::max_packed_args;
+
+ using value_type = conditional_t<is_packed, detail::value<Context>,
+ basic_format_arg<Context>>;
+
+ detail::arg_data<value_type, typename Context::char_type, num_args,
+ num_named_args>
+ data_;
+
+ friend class basic_format_args<Context>;
+
+ static constexpr unsigned long long desc =
+ (is_packed ? detail::encode_types<Context, Args...>()
+ : detail::is_unpacked_bit | num_args) |
+ (num_named_args != 0
+ ? static_cast<unsigned long long>(detail::has_named_args_bit)
+ : 0);
+
+ public:
+ template <typename... T>
+ FMT_CONSTEXPR FMT_INLINE format_arg_store(T&&... args)
+ :
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+ basic_format_args<Context>(*this),
+#endif
+ data_{detail::make_arg<
+ is_packed, Context,
+ detail::mapped_type_constant<remove_cvref_t<T>, Context>::value>(
+ FMT_FORWARD(args))...} {
+ detail::init_named_args(data_.named_args(), 0, 0, args...);
+ }
+};
+
+/**
+ \rst
+ Constructs a `~fmt::format_arg_store` object that contains references to
+ arguments and can be implicitly converted to `~fmt::format_args`. `Context`
+ can be omitted in which case it defaults to `~fmt::context`.
+ See `~fmt::arg` for lifetime considerations.
+ \endrst
+ */
+template <typename Context = format_context, typename... T>
+constexpr auto make_format_args(T&&... args)
+ -> format_arg_store<Context, remove_cvref_t<T>...> {
+ return {FMT_FORWARD(args)...};
+}
+
+/**
+ \rst
+ Returns a named argument to be used in a formatting function.
+ It should only be used in a call to a formatting function or
+ `dynamic_format_arg_store::push_back`.
+
+ **Example**::
+
+ fmt::print("Elapsed time: {s:.2f} seconds", fmt::arg("s", 1.23));
+ \endrst
+ */
+template <typename Char, typename T>
+inline auto arg(const Char* name, const T& arg) -> detail::named_arg<Char, T> {
+ static_assert(!detail::is_named_arg<T>(), "nested named arguments");
+ return {name, arg};
+}
+FMT_END_EXPORT
+
+/**
+ \rst
+ A view of a collection of formatting arguments. To avoid lifetime issues it
+ should only be used as a parameter type in type-erased functions such as
+ ``vformat``::
+
+ void vlog(string_view format_str, format_args args); // OK
+ format_args args = make_format_args(42); // Error: dangling reference
+ \endrst
+ */
+template <typename Context> class basic_format_args {
+ public:
+ using size_type = int;
+ using format_arg = basic_format_arg<Context>;
+
+ private:
+ // A descriptor that contains information about formatting arguments.
+ // If the number of arguments is less or equal to max_packed_args then
+ // argument types are passed in the descriptor. This reduces binary code size
+ // per formatting function call.
+ unsigned long long desc_;
+ union {
+ // If is_packed() returns true then argument values are stored in values_;
+ // otherwise they are stored in args_. This is done to improve cache
+ // locality and reduce compiled code size since storing larger objects
+ // may require more code (at least on x86-64) even if the same amount of
+ // data is actually copied to stack. It saves ~10% on the bloat test.
+ const detail::value<Context>* values_;
+ const format_arg* args_;
+ };
+
+ constexpr auto is_packed() const -> bool {
+ return (desc_ & detail::is_unpacked_bit) == 0;
+ }
+ auto has_named_args() const -> bool {
+ return (desc_ & detail::has_named_args_bit) != 0;
+ }
+
+ FMT_CONSTEXPR auto type(int index) const -> detail::type {
+ int shift = index * detail::packed_arg_bits;
+ unsigned int mask = (1 << detail::packed_arg_bits) - 1;
+ return static_cast<detail::type>((desc_ >> shift) & mask);
+ }
+
+ constexpr FMT_INLINE basic_format_args(unsigned long long desc,
+ const detail::value<Context>* values)
+ : desc_(desc), values_(values) {}
+ constexpr basic_format_args(unsigned long long desc, const format_arg* args)
+ : desc_(desc), args_(args) {}
+
+ public:
+ constexpr basic_format_args() : desc_(0), args_(nullptr) {}
+
+ /**
+ \rst
+ Constructs a `basic_format_args` object from `~fmt::format_arg_store`.
+ \endrst
+ */
+ template <typename... Args>
+ constexpr FMT_INLINE basic_format_args(
+ const format_arg_store<Context, Args...>& store)
+ : basic_format_args(format_arg_store<Context, Args...>::desc,
+ store.data_.args()) {}
+
+ /**
+ \rst
+ Constructs a `basic_format_args` object from
+ `~fmt::dynamic_format_arg_store`.
+ \endrst
+ */
+ constexpr FMT_INLINE basic_format_args(
+ const dynamic_format_arg_store<Context>& store)
+ : basic_format_args(store.get_types(), store.data()) {}
+
+ /**
+ \rst
+ Constructs a `basic_format_args` object from a dynamic set of arguments.
+ \endrst
+ */
+ constexpr basic_format_args(const format_arg* args, int count)
+ : basic_format_args(detail::is_unpacked_bit | detail::to_unsigned(count),
+ args) {}
+
+ /** Returns the argument with the specified id. */
+ FMT_CONSTEXPR auto get(int id) const -> format_arg {
+ format_arg arg;
+ if (!is_packed()) {
+ if (id < max_size()) arg = args_[id];
+ return arg;
+ }
+ if (id >= detail::max_packed_args) return arg;
+ arg.type_ = type(id);
+ if (arg.type_ == detail::type::none_type) return arg;
+ arg.value_ = values_[id];
+ return arg;
+ }
+
+ template <typename Char>
+ auto get(basic_string_view<Char> name) const -> format_arg {
+ int id = get_id(name);
+ return id >= 0 ? get(id) : format_arg();
+ }
+
+ template <typename Char>
+ auto get_id(basic_string_view<Char> name) const -> int {
+ if (!has_named_args()) return -1;
+ const auto& named_args =
+ (is_packed() ? values_[-1] : args_[-1].value_).named_args;
+ for (size_t i = 0; i < named_args.size; ++i) {
+ if (named_args.data[i].name == name) return named_args.data[i].id;
+ }
+ return -1;
+ }
+
+ auto max_size() const -> int {
+ unsigned long long max_packed = detail::max_packed_args;
+ return static_cast<int>(is_packed() ? max_packed
+ : desc_ & ~detail::is_unpacked_bit);
+ }
+};
+
+/** An alias to ``basic_format_args<format_context>``. */
+// A separate type would result in shorter symbols but break ABI compatibility
+// between clang and gcc on ARM (#1919).
+FMT_MODULE_EXPORT using format_args = basic_format_args<format_context>;
+
+// We cannot use enum classes as bit fields because of a gcc bug, so we put them
+// in namespaces instead (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414).
+// Additionally, if an underlying type is specified, older gcc incorrectly warns
+// that the type is too small. Both bugs are fixed in gcc 9.3.
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 903
+# define FMT_ENUM_UNDERLYING_TYPE(type)
+#else
+# define FMT_ENUM_UNDERLYING_TYPE(type) : type
+#endif
+namespace align {
+enum type FMT_ENUM_UNDERLYING_TYPE(unsigned char){none, left, right, center,
+ numeric};
+}
+using align_t = align::type;
+namespace sign {
+enum type FMT_ENUM_UNDERLYING_TYPE(unsigned char){none, minus, plus, space};
+}
+using sign_t = sign::type;
+
+namespace detail {
+
+// Workaround an array initialization issue in gcc 4.8.
+template <typename Char> struct fill_t {
+ private:
+ enum { max_size = 4 };
+ Char data_[max_size] = {Char(' '), Char(0), Char(0), Char(0)};
+ unsigned char size_ = 1;
+
+ public:
+ FMT_CONSTEXPR void operator=(basic_string_view<Char> s) {
+ auto size = s.size();
+ FMT_ASSERT(size <= max_size, "invalid fill");
+ for (size_t i = 0; i < size; ++i) data_[i] = s[i];
+ size_ = static_cast<unsigned char>(size);
+ }
+
+ constexpr auto size() const -> size_t { return size_; }
+ constexpr auto data() const -> const Char* { return data_; }
+
+ FMT_CONSTEXPR auto operator[](size_t index) -> Char& { return data_[index]; }
+ FMT_CONSTEXPR auto operator[](size_t index) const -> const Char& {
+ return data_[index];
+ }
+};
+} // namespace detail
+
+enum class presentation_type : unsigned char {
+ none,
+ dec, // 'd'
+ oct, // 'o'
+ hex_lower, // 'x'
+ hex_upper, // 'X'
+ bin_lower, // 'b'
+ bin_upper, // 'B'
+ hexfloat_lower, // 'a'
+ hexfloat_upper, // 'A'
+ exp_lower, // 'e'
+ exp_upper, // 'E'
+ fixed_lower, // 'f'
+ fixed_upper, // 'F'
+ general_lower, // 'g'
+ general_upper, // 'G'
+ chr, // 'c'
+ string, // 's'
+ pointer, // 'p'
+ debug // '?'
+};
+
+// Format specifiers for built-in and string types.
+template <typename Char = char> struct format_specs {
+ int width;
+ int precision;
+ presentation_type type;
+ align_t align : 4;
+ sign_t sign : 3;
+ bool alt : 1; // Alternate form ('#').
+ bool localized : 1;
+ detail::fill_t<Char> fill;
+
+ constexpr format_specs()
+ : width(0),
+ precision(-1),
+ type(presentation_type::none),
+ align(align::none),
+ sign(sign::none),
+ alt(false),
+ localized(false) {}
+};
+
+namespace detail {
+
+enum class arg_id_kind { none, index, name };
+
+// An argument reference.
+template <typename Char> struct arg_ref {
+ FMT_CONSTEXPR arg_ref() : kind(arg_id_kind::none), val() {}
+
+ FMT_CONSTEXPR explicit arg_ref(int index)
+ : kind(arg_id_kind::index), val(index) {}
+ FMT_CONSTEXPR explicit arg_ref(basic_string_view<Char> name)
+ : kind(arg_id_kind::name), val(name) {}
+
+ FMT_CONSTEXPR auto operator=(int idx) -> arg_ref& {
+ kind = arg_id_kind::index;
+ val.index = idx;
+ return *this;
+ }
+
+ arg_id_kind kind;
+ union value {
+ FMT_CONSTEXPR value(int idx = 0) : index(idx) {}
+ FMT_CONSTEXPR value(basic_string_view<Char> n) : name(n) {}
+
+ int index;
+ basic_string_view<Char> name;
+ } val;
+};
+
+// Format specifiers with width and precision resolved at formatting rather
+// than parsing time to allow reusing the same parsed specifiers with
+// different sets of arguments (precompilation of format strings).
+template <typename Char = char>
+struct dynamic_format_specs : format_specs<Char> {
+ arg_ref<Char> width_ref;
+ arg_ref<Char> precision_ref;
+};
+
+// Converts a character to ASCII. Returns '\0' on conversion failure.
+template <typename Char, FMT_ENABLE_IF(std::is_integral<Char>::value)>
+constexpr auto to_ascii(Char c) -> char {
+ return c <= 0xff ? static_cast<char>(c) : '\0';
+}
+template <typename Char, FMT_ENABLE_IF(std::is_enum<Char>::value)>
+constexpr auto to_ascii(Char c) -> char {
+ return c <= 0xff ? static_cast<char>(c) : '\0';
+}
+
+// Returns the number of code units in a code point or 1 on error.
+template <typename Char>
+FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int {
+ if (const_check(sizeof(Char) != 1)) return 1;
+ auto c = static_cast<unsigned char>(*begin);
+ return static_cast<int>((0x3a55000000000000ull >> (2 * (c >> 3))) & 0x3) + 1;
+}
+
+// Return the result via the out param to workaround gcc bug 77539.
+template <bool IS_CONSTEXPR, typename T, typename Ptr = const T*>
+FMT_CONSTEXPR auto find(Ptr first, Ptr last, T value, Ptr& out) -> bool {
+ for (out = first; out != last; ++out) {
+ if (*out == value) return true;
+ }
+ return false;
+}
+
+template <>
+inline auto find<false, char>(const char* first, const char* last, char value,
+ const char*& out) -> bool {
+ out = static_cast<const char*>(
+ std::memchr(first, value, to_unsigned(last - first)));
+ return out != nullptr;
+}
+
+// Parses the range [begin, end) as an unsigned integer. This function assumes
+// that the range is non-empty and the first character is a digit.
+template <typename Char>
+FMT_CONSTEXPR auto parse_nonnegative_int(const Char*& begin, const Char* end,
+ int error_value) noexcept -> int {
+ FMT_ASSERT(begin != end && '0' <= *begin && *begin <= '9', "");
+ unsigned value = 0, prev = 0;
+ auto p = begin;
+ do {
+ prev = value;
+ value = value * 10 + unsigned(*p - '0');
+ ++p;
+ } while (p != end && '0' <= *p && *p <= '9');
+ auto num_digits = p - begin;
+ begin = p;
+ if (num_digits <= std::numeric_limits<int>::digits10)
+ return static_cast<int>(value);
+ // Check for overflow.
+ const unsigned max = to_unsigned((std::numeric_limits<int>::max)());
+ return num_digits == std::numeric_limits<int>::digits10 + 1 &&
+ prev * 10ull + unsigned(p[-1] - '0') <= max
+ ? static_cast<int>(value)
+ : error_value;
+}
+
+FMT_CONSTEXPR inline auto parse_align(char c) -> align_t {
+ switch (c) {
+ case '<':
+ return align::left;
+ case '>':
+ return align::right;
+ case '^':
+ return align::center;
+ }
+ return align::none;
+}
+
+template <typename Char> constexpr auto is_name_start(Char c) -> bool {
+ return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_';
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto do_parse_arg_id(const Char* begin, const Char* end,
+ Handler&& handler) -> const Char* {
+ Char c = *begin;
+ if (c >= '0' && c <= '9') {
+ int index = 0;
+ constexpr int max = (std::numeric_limits<int>::max)();
+ if (c != '0')
+ index = parse_nonnegative_int(begin, end, max);
+ else
+ ++begin;
+ if (begin == end || (*begin != '}' && *begin != ':'))
+ throw_format_error("invalid format string");
+ else
+ handler.on_index(index);
+ return begin;
+ }
+ if (!is_name_start(c)) {
+ throw_format_error("invalid format string");
+ return begin;
+ }
+ auto it = begin;
+ do {
+ ++it;
+ } while (it != end && (is_name_start(*it) || ('0' <= *it && *it <= '9')));
+ handler.on_name({begin, to_unsigned(it - begin)});
+ return it;
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR FMT_INLINE auto parse_arg_id(const Char* begin, const Char* end,
+ Handler&& handler) -> const Char* {
+ FMT_ASSERT(begin != end, "");
+ Char c = *begin;
+ if (c != '}' && c != ':') return do_parse_arg_id(begin, end, handler);
+ handler.on_auto();
+ return begin;
+}
+
+template <typename Char> struct dynamic_spec_id_handler {
+ basic_format_parse_context<Char>& ctx;
+ arg_ref<Char>& ref;
+
+ FMT_CONSTEXPR void on_auto() {
+ int id = ctx.next_arg_id();
+ ref = arg_ref<Char>(id);
+ ctx.check_dynamic_spec(id);
+ }
+ FMT_CONSTEXPR void on_index(int id) {
+ ref = arg_ref<Char>(id);
+ ctx.check_arg_id(id);
+ ctx.check_dynamic_spec(id);
+ }
+ FMT_CONSTEXPR void on_name(basic_string_view<Char> id) {
+ ref = arg_ref<Char>(id);
+ ctx.check_arg_id(id);
+ }
+};
+
+// Parses [integer | "{" [arg_id] "}"].
+template <typename Char>
+FMT_CONSTEXPR auto parse_dynamic_spec(const Char* begin, const Char* end,
+ int& value, arg_ref<Char>& ref,
+ basic_format_parse_context<Char>& ctx)
+ -> const Char* {
+ FMT_ASSERT(begin != end, "");
+ if ('0' <= *begin && *begin <= '9') {
+ int val = parse_nonnegative_int(begin, end, -1);
+ if (val != -1)
+ value = val;
+ else
+ throw_format_error("number is too big");
+ } else if (*begin == '{') {
+ ++begin;
+ auto handler = dynamic_spec_id_handler<Char>{ctx, ref};
+ if (begin != end) begin = parse_arg_id(begin, end, handler);
+ if (begin != end && *begin == '}') return ++begin;
+ throw_format_error("invalid format string");
+ }
+ return begin;
+}
+
+template <typename Char>
+FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end,
+ int& value, arg_ref<Char>& ref,
+ basic_format_parse_context<Char>& ctx)
+ -> const Char* {
+ ++begin;
+ if (begin == end || *begin == '}') {
+ throw_format_error("invalid precision");
+ return begin;
+ }
+ return parse_dynamic_spec(begin, end, value, ref, ctx);
+}
+
+enum class state { start, align, sign, hash, zero, width, precision, locale };
+
+// Parses standard format specifiers.
+template <typename Char>
+FMT_CONSTEXPR FMT_INLINE auto parse_format_specs(
+ const Char* begin, const Char* end, dynamic_format_specs<Char>& specs,
+ basic_format_parse_context<Char>& ctx, type arg_type) -> const Char* {
+ auto c = '\0';
+ if (end - begin > 1) {
+ auto next = to_ascii(begin[1]);
+ c = parse_align(next) == align::none ? to_ascii(*begin) : '\0';
+ } else {
+ if (begin == end) return begin;
+ c = to_ascii(*begin);
+ }
+
+ struct {
+ state current_state = state::start;
+ FMT_CONSTEXPR void operator()(state s, bool valid = true) {
+ if (current_state >= s || !valid)
+ throw_format_error("invalid format specifier");
+ current_state = s;
+ }
+ } enter_state;
+
+ using pres = presentation_type;
+ constexpr auto integral_set = sint_set | uint_set | bool_set | char_set;
+ struct {
+ const Char*& begin;
+ dynamic_format_specs<Char>& specs;
+ type arg_type;
+
+ FMT_CONSTEXPR auto operator()(pres type, int set) -> const Char* {
+ if (!in(arg_type, set)) throw_format_error("invalid format specifier");
+ specs.type = type;
+ return begin + 1;
+ }
+ } parse_presentation_type{begin, specs, arg_type};
+
+ for (;;) {
+ switch (c) {
+ case '<':
+ case '>':
+ case '^':
+ enter_state(state::align);
+ specs.align = parse_align(c);
+ ++begin;
+ break;
+ case '+':
+ case '-':
+ case ' ':
+ enter_state(state::sign, in(arg_type, sint_set | float_set));
+ switch (c) {
+ case '+':
+ specs.sign = sign::plus;
+ break;
+ case '-':
+ specs.sign = sign::minus;
+ break;
+ case ' ':
+ specs.sign = sign::space;
+ break;
+ }
+ ++begin;
+ break;
+ case '#':
+ enter_state(state::hash, is_arithmetic_type(arg_type));
+ specs.alt = true;
+ ++begin;
+ break;
+ case '0':
+ enter_state(state::zero);
+ if (!is_arithmetic_type(arg_type))
+ throw_format_error("format specifier requires numeric argument");
+ if (specs.align == align::none) {
+ // Ignore 0 if align is specified for compatibility with std::format.
+ specs.align = align::numeric;
+ specs.fill[0] = Char('0');
+ }
+ ++begin;
+ break;
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ case '{':
+ enter_state(state::width);
+ begin = parse_dynamic_spec(begin, end, specs.width, specs.width_ref, ctx);
+ break;
+ case '.':
+ enter_state(state::precision,
+ in(arg_type, float_set | string_set | cstring_set));
+ begin = parse_precision(begin, end, specs.precision, specs.precision_ref,
+ ctx);
+ break;
+ case 'L':
+ enter_state(state::locale, is_arithmetic_type(arg_type));
+ specs.localized = true;
+ ++begin;
+ break;
+ case 'd':
+ return parse_presentation_type(pres::dec, integral_set);
+ case 'o':
+ return parse_presentation_type(pres::oct, integral_set);
+ case 'x':
+ return parse_presentation_type(pres::hex_lower, integral_set);
+ case 'X':
+ return parse_presentation_type(pres::hex_upper, integral_set);
+ case 'b':
+ return parse_presentation_type(pres::bin_lower, integral_set);
+ case 'B':
+ return parse_presentation_type(pres::bin_upper, integral_set);
+ case 'a':
+ return parse_presentation_type(pres::hexfloat_lower, float_set);
+ case 'A':
+ return parse_presentation_type(pres::hexfloat_upper, float_set);
+ case 'e':
+ return parse_presentation_type(pres::exp_lower, float_set);
+ case 'E':
+ return parse_presentation_type(pres::exp_upper, float_set);
+ case 'f':
+ return parse_presentation_type(pres::fixed_lower, float_set);
+ case 'F':
+ return parse_presentation_type(pres::fixed_upper, float_set);
+ case 'g':
+ return parse_presentation_type(pres::general_lower, float_set);
+ case 'G':
+ return parse_presentation_type(pres::general_upper, float_set);
+ case 'c':
+ return parse_presentation_type(pres::chr, integral_set);
+ case 's':
+ return parse_presentation_type(pres::string,
+ bool_set | string_set | cstring_set);
+ case 'p':
+ return parse_presentation_type(pres::pointer, pointer_set | cstring_set);
+ case '?':
+ return parse_presentation_type(pres::debug,
+ char_set | string_set | cstring_set);
+ case '}':
+ return begin;
+ default: {
+ if (*begin == '}') return begin;
+ // Parse fill and alignment.
+ auto fill_end = begin + code_point_length(begin);
+ if (end - fill_end <= 0) {
+ throw_format_error("invalid format specifier");
+ return begin;
+ }
+ if (*begin == '{') {
+ throw_format_error("invalid fill character '{'");
+ return begin;
+ }
+ auto align = parse_align(to_ascii(*fill_end));
+ enter_state(state::align, align != align::none);
+ specs.fill = {begin, to_unsigned(fill_end - begin)};
+ specs.align = align;
+ begin = fill_end + 1;
+ }
+ }
+ if (begin == end) return begin;
+ c = to_ascii(*begin);
+ }
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end,
+ Handler&& handler) -> const Char* {
+ struct id_adapter {
+ Handler& handler;
+ int arg_id;
+
+ FMT_CONSTEXPR void on_auto() { arg_id = handler.on_arg_id(); }
+ FMT_CONSTEXPR void on_index(int id) { arg_id = handler.on_arg_id(id); }
+ FMT_CONSTEXPR void on_name(basic_string_view<Char> id) {
+ arg_id = handler.on_arg_id(id);
+ }
+ };
+
+ ++begin;
+ if (begin == end) return handler.on_error("invalid format string"), end;
+ if (*begin == '}') {
+ handler.on_replacement_field(handler.on_arg_id(), begin);
+ } else if (*begin == '{') {
+ handler.on_text(begin, begin + 1);
+ } else {
+ auto adapter = id_adapter{handler, 0};
+ begin = parse_arg_id(begin, end, adapter);
+ Char c = begin != end ? *begin : Char();
+ if (c == '}') {
+ handler.on_replacement_field(adapter.arg_id, begin);
+ } else if (c == ':') {
+ begin = handler.on_format_specs(adapter.arg_id, begin + 1, end);
+ if (begin == end || *begin != '}')
+ return handler.on_error("unknown format specifier"), end;
+ } else {
+ return handler.on_error("missing '}' in format string"), end;
+ }
+ }
+ return begin + 1;
+}
+
+template <bool IS_CONSTEXPR, typename Char, typename Handler>
+FMT_CONSTEXPR FMT_INLINE void parse_format_string(
+ basic_string_view<Char> format_str, Handler&& handler) {
+ auto begin = format_str.data();
+ auto end = begin + format_str.size();
+ if (end - begin < 32) {
+ // Use a simple loop instead of memchr for small strings.
+ const Char* p = begin;
+ while (p != end) {
+ auto c = *p++;
+ if (c == '{') {
+ handler.on_text(begin, p - 1);
+ begin = p = parse_replacement_field(p - 1, end, handler);
+ } else if (c == '}') {
+ if (p == end || *p != '}')
+ return handler.on_error("unmatched '}' in format string");
+ handler.on_text(begin, p);
+ begin = ++p;
+ }
+ }
+ handler.on_text(begin, end);
+ return;
+ }
+ struct writer {
+ FMT_CONSTEXPR void operator()(const Char* from, const Char* to) {
+ if (from == to) return;
+ for (;;) {
+ const Char* p = nullptr;
+ if (!find<IS_CONSTEXPR>(from, to, Char('}'), p))
+ return handler_.on_text(from, to);
+ ++p;
+ if (p == to || *p != '}')
+ return handler_.on_error("unmatched '}' in format string");
+ handler_.on_text(from, p);
+ from = p + 1;
+ }
+ }
+ Handler& handler_;
+ } write = {handler};
+ while (begin != end) {
+ // Doing two passes with memchr (one for '{' and another for '}') is up to
+ // 2.5x faster than the naive one-pass implementation on big format strings.
+ const Char* p = begin;
+ if (*begin != '{' && !find<IS_CONSTEXPR>(begin + 1, end, Char('{'), p))
+ return write(begin, end);
+ write(begin, p);
+ begin = parse_replacement_field(p, end, handler);
+ }
+}
+
+template <typename T, bool = is_named_arg<T>::value> struct strip_named_arg {
+ using type = T;
+};
+template <typename T> struct strip_named_arg<T, true> {
+ using type = remove_cvref_t<decltype(T::value)>;
+};
+
+template <typename T, typename ParseContext>
+FMT_CONSTEXPR auto parse_format_specs(ParseContext& ctx)
+ -> decltype(ctx.begin()) {
+ using char_type = typename ParseContext::char_type;
+ using context = buffer_context<char_type>;
+ using mapped_type = conditional_t<
+ mapped_type_constant<T, context>::value != type::custom_type,
+ decltype(arg_mapper<context>().map(std::declval<const T&>())),
+ typename strip_named_arg<T>::type>;
+ return formatter<mapped_type, char_type>().parse(ctx);
+}
+
+// Checks char specs and returns true iff the presentation type is char-like.
+template <typename Char>
+FMT_CONSTEXPR auto check_char_specs(const format_specs<Char>& specs) -> bool {
+ if (specs.type != presentation_type::none &&
+ specs.type != presentation_type::chr &&
+ specs.type != presentation_type::debug) {
+ return false;
+ }
+ if (specs.align == align::numeric || specs.sign != sign::none || specs.alt)
+ throw_format_error("invalid format specifier for char");
+ return true;
+}
+
+constexpr FMT_INLINE_VARIABLE int invalid_arg_index = -1;
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <int N, typename T, typename... Args, typename Char>
+constexpr auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
+ if constexpr (is_statically_named_arg<T>()) {
+ if (name == T::name) return N;
+ }
+ if constexpr (sizeof...(Args) > 0)
+ return get_arg_index_by_name<N + 1, Args...>(name);
+ (void)name; // Workaround an MSVC bug about "unused" parameter.
+ return invalid_arg_index;
+}
+#endif
+
+template <typename... Args, typename Char>
+FMT_CONSTEXPR auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+ if constexpr (sizeof...(Args) > 0)
+ return get_arg_index_by_name<0, Args...>(name);
+#endif
+ (void)name;
+ return invalid_arg_index;
+}
+
+template <typename Char, typename... Args> class format_string_checker {
+ private:
+ using parse_context_type = compile_parse_context<Char>;
+ static constexpr int num_args = sizeof...(Args);
+
+ // Format specifier parsing function.
+ // In the future basic_format_parse_context will replace compile_parse_context
+ // here and will use is_constant_evaluated and downcasting to access the data
+ // needed for compile-time checks: https://godbolt.org/z/GvWzcTjh1.
+ using parse_func = const Char* (*)(parse_context_type&);
+
+ parse_context_type context_;
+ parse_func parse_funcs_[num_args > 0 ? static_cast<size_t>(num_args) : 1];
+ type types_[num_args > 0 ? static_cast<size_t>(num_args) : 1];
+
+ public:
+ explicit FMT_CONSTEXPR format_string_checker(basic_string_view<Char> fmt)
+ : context_(fmt, num_args, types_),
+ parse_funcs_{&parse_format_specs<Args, parse_context_type>...},
+ types_{mapped_type_constant<Args, buffer_context<Char>>::value...} {}
+
+ FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+
+ FMT_CONSTEXPR auto on_arg_id() -> int { return context_.next_arg_id(); }
+ FMT_CONSTEXPR auto on_arg_id(int id) -> int {
+ return context_.check_arg_id(id), id;
+ }
+ FMT_CONSTEXPR auto on_arg_id(basic_string_view<Char> id) -> int {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+ auto index = get_arg_index_by_name<Args...>(id);
+ if (index == invalid_arg_index) on_error("named argument is not found");
+ return index;
+#else
+ (void)id;
+ on_error("compile-time checks for named arguments require C++20 support");
+ return 0;
+#endif
+ }
+
+ FMT_CONSTEXPR void on_replacement_field(int, const Char*) {}
+
+ FMT_CONSTEXPR auto on_format_specs(int id, const Char* begin, const Char*)
+ -> const Char* {
+ context_.advance_to(begin);
+ // id >= 0 check is a workaround for gcc 10 bug (#2065).
+ return id >= 0 && id < num_args ? parse_funcs_[id](context_) : begin;
+ }
+
+ FMT_CONSTEXPR void on_error(const char* message) {
+ throw_format_error(message);
+ }
+};
+
+// Reports a compile-time error if S is not a valid format string.
+template <typename..., typename S, FMT_ENABLE_IF(!is_compile_string<S>::value)>
+FMT_INLINE void check_format_string(const S&) {
+#ifdef FMT_ENFORCE_COMPILE_STRING
+ static_assert(is_compile_string<S>::value,
+ "FMT_ENFORCE_COMPILE_STRING requires all format strings to use "
+ "FMT_STRING.");
+#endif
+}
+template <typename... Args, typename S,
+ FMT_ENABLE_IF(is_compile_string<S>::value)>
+void check_format_string(S format_str) {
+ using char_t = typename S::char_type;
+ FMT_CONSTEXPR auto s = basic_string_view<char_t>(format_str);
+ using checker = format_string_checker<char_t, remove_cvref_t<Args>...>;
+ FMT_CONSTEXPR bool error = (parse_format_string<true>(s, checker(s)), true);
+ ignore_unused(error);
+}
+
+template <typename Char = char> struct vformat_args {
+ using type = basic_format_args<
+ basic_format_context<std::back_insert_iterator<buffer<Char>>, Char>>;
+};
+template <> struct vformat_args<char> { using type = format_args; };
+
+// Use vformat_args and avoid type_identity to keep symbols short.
+template <typename Char>
+void vformat_to(buffer<Char>& buf, basic_string_view<Char> fmt,
+ typename vformat_args<Char>::type args, locale_ref loc = {});
+
+FMT_API void vprint_mojibake(std::FILE*, string_view, format_args);
+#ifndef _WIN32
+inline void vprint_mojibake(std::FILE*, string_view, format_args) {}
+#endif
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+// A formatter specialization for natively supported types.
+template <typename T, typename Char>
+struct formatter<T, Char,
+ enable_if_t<detail::type_constant<T, Char>::value !=
+ detail::type::custom_type>> {
+ private:
+ detail::dynamic_format_specs<Char> specs_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const Char* {
+ auto type = detail::type_constant<T, Char>::value;
+ auto end =
+ detail::parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx, type);
+ if (type == detail::type::char_type) detail::check_char_specs(specs_);
+ return end;
+ }
+
+ template <detail::type U = detail::type_constant<T, Char>::value,
+ FMT_ENABLE_IF(U == detail::type::string_type ||
+ U == detail::type::cstring_type ||
+ U == detail::type::char_type)>
+ FMT_CONSTEXPR void set_debug_format(bool set = true) {
+ specs_.type = set ? presentation_type::debug : presentation_type::none;
+ }
+
+ template <typename FormatContext>
+ FMT_CONSTEXPR auto format(const T& val, FormatContext& ctx) const
+ -> decltype(ctx.out());
+};
+
+#define FMT_FORMAT_AS(Type, Base) \
+ template <typename Char> \
+ struct formatter<Type, Char> : formatter<Base, Char> { \
+ template <typename FormatContext> \
+ auto format(const Type& val, FormatContext& ctx) const \
+ -> decltype(ctx.out()) { \
+ return formatter<Base, Char>::format(static_cast<Base>(val), ctx); \
+ } \
+ }
+
+FMT_FORMAT_AS(signed char, int);
+FMT_FORMAT_AS(unsigned char, unsigned);
+FMT_FORMAT_AS(short, int);
+FMT_FORMAT_AS(unsigned short, unsigned);
+FMT_FORMAT_AS(long, long long);
+FMT_FORMAT_AS(unsigned long, unsigned long long);
+FMT_FORMAT_AS(Char*, const Char*);
+FMT_FORMAT_AS(std::basic_string<Char>, basic_string_view<Char>);
+FMT_FORMAT_AS(std::nullptr_t, const void*);
+FMT_FORMAT_AS(detail::std_string_view<Char>, basic_string_view<Char>);
+
+template <typename Char = char> struct runtime_format_string {
+ basic_string_view<Char> str;
+};
+
+/** A compile-time format string. */
+template <typename Char, typename... Args> class basic_format_string {
+ private:
+ basic_string_view<Char> str_;
+
+ public:
+ template <typename S,
+ FMT_ENABLE_IF(
+ std::is_convertible<const S&, basic_string_view<Char>>::value)>
+ FMT_CONSTEVAL FMT_INLINE basic_format_string(const S& s) : str_(s) {
+ static_assert(
+ detail::count<
+ (std::is_base_of<detail::view, remove_reference_t<Args>>::value &&
+ std::is_reference<Args>::value)...>() == 0,
+ "passing views as lvalues is disallowed");
+#ifdef FMT_HAS_CONSTEVAL
+ if constexpr (detail::count_named_args<Args...>() ==
+ detail::count_statically_named_args<Args...>()) {
+ using checker =
+ detail::format_string_checker<Char, remove_cvref_t<Args>...>;
+ detail::parse_format_string<true>(str_, checker(s));
+ }
+#else
+ detail::check_format_string<Args...>(s);
+#endif
+ }
+ basic_format_string(runtime_format_string<Char> fmt) : str_(fmt.str) {}
+
+ FMT_INLINE operator basic_string_view<Char>() const { return str_; }
+ FMT_INLINE auto get() const -> basic_string_view<Char> { return str_; }
+};
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+// Workaround broken conversion on older gcc.
+template <typename...> using format_string = string_view;
+inline auto runtime(string_view s) -> string_view { return s; }
+#else
+template <typename... Args>
+using format_string = basic_format_string<char, type_identity_t<Args>...>;
+/**
+ \rst
+ Creates a runtime format string.
+
+ **Example**::
+
+ // Check format string at runtime instead of compile-time.
+ fmt::print(fmt::runtime("{:d}"), "I am not a number");
+ \endrst
+ */
+inline auto runtime(string_view s) -> runtime_format_string<> { return {{s}}; }
+#endif
+
+FMT_API auto vformat(string_view fmt, format_args args) -> std::string;
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt`` and returns the result
+ as a string.
+
+ **Example**::
+
+ #include <fmt/core.h>
+ std::string message = fmt::format("The answer is {}.", 42);
+ \endrst
+*/
+template <typename... T>
+FMT_NODISCARD FMT_INLINE auto format(format_string<T...> fmt, T&&... args)
+ -> std::string {
+ return vformat(fmt, fmt::make_format_args(args...));
+}
+
+/** Formats a string and writes the output to ``out``. */
+template <typename OutputIt,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+auto vformat_to(OutputIt out, string_view fmt, format_args args) -> OutputIt {
+ auto&& buf = detail::get_buffer<char>(out);
+ detail::vformat_to(buf, fmt, args, {});
+ return detail::get_iterator(buf, out);
+}
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt``, writes the result to
+ the output iterator ``out`` and returns the iterator past the end of the output
+ range. `format_to` does not append a terminating null character.
+
+ **Example**::
+
+ auto out = std::vector<char>();
+ fmt::format_to(std::back_inserter(out), "{}", 42);
+ \endrst
+ */
+template <typename OutputIt, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+FMT_INLINE auto format_to(OutputIt out, format_string<T...> fmt, T&&... args)
+ -> OutputIt {
+ return vformat_to(out, fmt, fmt::make_format_args(args...));
+}
+
+template <typename OutputIt> struct format_to_n_result {
+ /** Iterator past the end of the output range. */
+ OutputIt out;
+ /** Total (not truncated) output size. */
+ size_t size;
+};
+
+template <typename OutputIt, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+auto vformat_to_n(OutputIt out, size_t n, string_view fmt, format_args args)
+ -> format_to_n_result<OutputIt> {
+ using traits = detail::fixed_buffer_traits;
+ auto buf = detail::iterator_buffer<OutputIt, char, traits>(out, n);
+ detail::vformat_to(buf, fmt, args, {});
+ return {buf.out(), buf.count()};
+}
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt``, writes up to ``n``
+ characters of the result to the output iterator ``out`` and returns the total
+ (not truncated) output size and the iterator past the end of the output range.
+ `format_to_n` does not append a terminating null character.
+ \endrst
+ */
+template <typename OutputIt, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+FMT_INLINE auto format_to_n(OutputIt out, size_t n, format_string<T...> fmt,
+ T&&... args) -> format_to_n_result<OutputIt> {
+ return vformat_to_n(out, n, fmt, fmt::make_format_args(args...));
+}
+
+/** Returns the number of chars in the output of ``format(fmt, args...)``. */
+template <typename... T>
+FMT_NODISCARD FMT_INLINE auto formatted_size(format_string<T...> fmt,
+ T&&... args) -> size_t {
+ auto buf = detail::counting_buffer<>();
+ detail::vformat_to<char>(buf, fmt, fmt::make_format_args(args...), {});
+ return buf.count();
+}
+
+FMT_API void vprint(string_view fmt, format_args args);
+FMT_API void vprint(std::FILE* f, string_view fmt, format_args args);
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt`` and writes the output
+ to ``stdout``.
+
+ **Example**::
+
+ fmt::print("Elapsed time: {0:.2f} seconds", 1.23);
+ \endrst
+ */
+template <typename... T>
+FMT_INLINE void print(format_string<T...> fmt, T&&... args) {
+ const auto& vargs = fmt::make_format_args(args...);
+ return detail::is_utf8() ? vprint(fmt, vargs)
+ : detail::vprint_mojibake(stdout, fmt, vargs);
+}
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt`` and writes the
+ output to the file ``f``.
+
+ **Example**::
+
+ fmt::print(stderr, "Don't {}!", "panic");
+ \endrst
+ */
+template <typename... T>
+FMT_INLINE void print(std::FILE* f, format_string<T...> fmt, T&&... args) {
+ const auto& vargs = fmt::make_format_args(args...);
+ return detail::is_utf8() ? vprint(f, fmt, vargs)
+ : detail::vprint_mojibake(f, fmt, vargs);
+}
+
+/**
+ Formats ``args`` according to specifications in ``fmt`` and writes the
+ output to the file ``f`` followed by a newline.
+ */
+template <typename... T>
+FMT_INLINE void println(std::FILE* f, format_string<T...> fmt, T&&... args) {
+ return fmt::print(f, "{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+/**
+ Formats ``args`` according to specifications in ``fmt`` and writes the output
+ to ``stdout`` followed by a newline.
+ */
+template <typename... T>
+FMT_INLINE void println(format_string<T...> fmt, T&&... args) {
+ return fmt::println(stdout, fmt, std::forward<T>(args)...);
+}
+
+FMT_END_EXPORT
+FMT_GCC_PRAGMA("GCC pop_options")
+FMT_END_NAMESPACE
+
+#ifdef FMT_HEADER_ONLY
+# include "format.h"
+#endif
+#endif // FMT_CORE_H_
diff --git a/contrib/fmt/include/fmt/format-inl.h b/contrib/fmt/include/fmt/format-inl.h
new file mode 100644
index 0000000..5bae3c7
--- /dev/null
+++ b/contrib/fmt/include/fmt/format-inl.h
@@ -0,0 +1,1681 @@
+// Formatting library for C++ - implementation
+//
+// Copyright (c) 2012 - 2016, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_FORMAT_INL_H_
+#define FMT_FORMAT_INL_H_
+
+#include <algorithm>
+#include <cerrno> // errno
+#include <climits>
+#include <cmath>
+#include <exception>
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+# include <locale>
+#endif
+
+#ifdef _WIN32
+# include <io.h> // _isatty
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+FMT_FUNC void assert_fail(const char* file, int line, const char* message) {
+ // Use unchecked std::fprintf to avoid triggering another assertion when
+ // writing to stderr fails
+ std::fprintf(stderr, "%s:%d: assertion failed: %s", file, line, message);
+ // Chosen instead of std::abort to satisfy Clang in CUDA mode during device
+ // code pass.
+ std::terminate();
+}
+
+FMT_FUNC void throw_format_error(const char* message) {
+ FMT_THROW(format_error(message));
+}
+
+FMT_FUNC void format_error_code(detail::buffer<char>& out, int error_code,
+ string_view message) noexcept {
+ // Report error code making sure that the output fits into
+ // inline_buffer_size to avoid dynamic memory allocation and potential
+ // bad_alloc.
+ out.try_resize(0);
+ static const char SEP[] = ": ";
+ static const char ERROR_STR[] = "error ";
+ // Subtract 2 to account for terminating null characters in SEP and ERROR_STR.
+ size_t error_code_size = sizeof(SEP) + sizeof(ERROR_STR) - 2;
+ auto abs_value = static_cast<uint32_or_64_or_128_t<int>>(error_code);
+ if (detail::is_negative(error_code)) {
+ abs_value = 0 - abs_value;
+ ++error_code_size;
+ }
+ error_code_size += detail::to_unsigned(detail::count_digits(abs_value));
+ auto it = buffer_appender<char>(out);
+ if (message.size() <= inline_buffer_size - error_code_size)
+ format_to(it, FMT_STRING("{}{}"), message, SEP);
+ format_to(it, FMT_STRING("{}{}"), ERROR_STR, error_code);
+ FMT_ASSERT(out.size() <= inline_buffer_size, "");
+}
+
+FMT_FUNC void report_error(format_func func, int error_code,
+ const char* message) noexcept {
+ memory_buffer full_message;
+ func(full_message, error_code, message);
+ // Don't use fwrite_fully because the latter may throw.
+ if (std::fwrite(full_message.data(), full_message.size(), 1, stderr) > 0)
+ std::fputc('\n', stderr);
+}
+
+// A wrapper around fwrite that throws on error.
+inline void fwrite_fully(const void* ptr, size_t size, size_t count,
+ FILE* stream) {
+ size_t written = std::fwrite(ptr, size, count, stream);
+ if (written < count)
+ FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+template <typename Locale>
+locale_ref::locale_ref(const Locale& loc) : locale_(&loc) {
+ static_assert(std::is_same<Locale, std::locale>::value, "");
+}
+
+template <typename Locale> Locale locale_ref::get() const {
+ static_assert(std::is_same<Locale, std::locale>::value, "");
+ return locale_ ? *static_cast<const std::locale*>(locale_) : std::locale();
+}
+
+template <typename Char>
+FMT_FUNC auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result<Char> {
+ auto& facet = std::use_facet<std::numpunct<Char>>(loc.get<std::locale>());
+ auto grouping = facet.grouping();
+ auto thousands_sep = grouping.empty() ? Char() : facet.thousands_sep();
+ return {std::move(grouping), thousands_sep};
+}
+template <typename Char> FMT_FUNC Char decimal_point_impl(locale_ref loc) {
+ return std::use_facet<std::numpunct<Char>>(loc.get<std::locale>())
+ .decimal_point();
+}
+#else
+template <typename Char>
+FMT_FUNC auto thousands_sep_impl(locale_ref) -> thousands_sep_result<Char> {
+ return {"\03", FMT_STATIC_THOUSANDS_SEPARATOR};
+}
+template <typename Char> FMT_FUNC Char decimal_point_impl(locale_ref) {
+ return '.';
+}
+#endif
+
+FMT_FUNC auto write_loc(appender out, loc_value value,
+ const format_specs<>& specs, locale_ref loc) -> bool {
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+ auto locale = loc.get<std::locale>();
+ // We cannot use the num_put<char> facet because it may produce output in
+ // a wrong encoding.
+ using facet = format_facet<std::locale>;
+ if (std::has_facet<facet>(locale))
+ return std::use_facet<facet>(locale).put(out, value, specs);
+ return facet(locale).put(out, value, specs);
+#endif
+ return false;
+}
+} // namespace detail
+
+template <typename Locale> typename Locale::id format_facet<Locale>::id;
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+template <typename Locale> format_facet<Locale>::format_facet(Locale& loc) {
+ auto& numpunct = std::use_facet<std::numpunct<char>>(loc);
+ grouping_ = numpunct.grouping();
+ if (!grouping_.empty()) separator_ = std::string(1, numpunct.thousands_sep());
+}
+
+template <>
+FMT_API FMT_FUNC auto format_facet<std::locale>::do_put(
+ appender out, loc_value val, const format_specs<>& specs) const -> bool {
+ return val.visit(
+ detail::loc_writer<>{out, specs, separator_, grouping_, decimal_point_});
+}
+#endif
+
+FMT_FUNC std::system_error vsystem_error(int error_code, string_view fmt,
+ format_args args) {
+ auto ec = std::error_code(error_code, std::generic_category());
+ return std::system_error(ec, vformat(fmt, args));
+}
+
+namespace detail {
+
+template <typename F> inline bool operator==(basic_fp<F> x, basic_fp<F> y) {
+ return x.f == y.f && x.e == y.e;
+}
+
+// Compilers should be able to optimize this into the ror instruction.
+FMT_CONSTEXPR inline uint32_t rotr(uint32_t n, uint32_t r) noexcept {
+ r &= 31;
+ return (n >> r) | (n << (32 - r));
+}
+FMT_CONSTEXPR inline uint64_t rotr(uint64_t n, uint32_t r) noexcept {
+ r &= 63;
+ return (n >> r) | (n << (64 - r));
+}
+
+// Implementation of Dragonbox algorithm: https://github.com/jk-jeon/dragonbox.
+namespace dragonbox {
+// Computes upper 64 bits of multiplication of a 32-bit unsigned integer and a
+// 64-bit unsigned integer.
+inline uint64_t umul96_upper64(uint32_t x, uint64_t y) noexcept {
+ return umul128_upper64(static_cast<uint64_t>(x) << 32, y);
+}
+
+// Computes lower 128 bits of multiplication of a 64-bit unsigned integer and a
+// 128-bit unsigned integer.
+inline uint128_fallback umul192_lower128(uint64_t x,
+ uint128_fallback y) noexcept {
+ uint64_t high = x * y.high();
+ uint128_fallback high_low = umul128(x, y.low());
+ return {high + high_low.high(), high_low.low()};
+}
+
+// Computes lower 64 bits of multiplication of a 32-bit unsigned integer and a
+// 64-bit unsigned integer.
+inline uint64_t umul96_lower64(uint32_t x, uint64_t y) noexcept {
+ return x * y;
+}
+
+// Various fast log computations.
+inline int floor_log10_pow2_minus_log10_4_over_3(int e) noexcept {
+ FMT_ASSERT(e <= 2936 && e >= -2985, "too large exponent");
+ return (e * 631305 - 261663) >> 21;
+}
+
+FMT_INLINE_VARIABLE constexpr struct {
+ uint32_t divisor;
+ int shift_amount;
+} div_small_pow10_infos[] = {{10, 16}, {100, 16}};
+
+// Replaces n by floor(n / pow(10, N)) returning true if and only if n is
+// divisible by pow(10, N).
+// Precondition: n <= pow(10, N + 1).
+template <int N>
+bool check_divisibility_and_divide_by_pow10(uint32_t& n) noexcept {
+ // The numbers below are chosen such that:
+ // 1. floor(n/d) = floor(nm / 2^k) where d=10 or d=100,
+ // 2. nm mod 2^k < m if and only if n is divisible by d,
+ // where m is magic_number, k is shift_amount
+ // and d is divisor.
+ //
+ // Item 1 is a common technique of replacing division by a constant with
+ // multiplication, see e.g. "Division by Invariant Integers Using
+ // Multiplication" by Granlund and Montgomery (1994). magic_number (m) is set
+ // to ceil(2^k/d) for large enough k.
+ // The idea for item 2 originates from Schubfach.
+ constexpr auto info = div_small_pow10_infos[N - 1];
+ FMT_ASSERT(n <= info.divisor * 10, "n is too large");
+ constexpr uint32_t magic_number =
+ (1u << info.shift_amount) / info.divisor + 1;
+ n *= magic_number;
+ const uint32_t comparison_mask = (1u << info.shift_amount) - 1;
+ bool result = (n & comparison_mask) < magic_number;
+ n >>= info.shift_amount;
+ return result;
+}
+
+// Computes floor(n / pow(10, N)) for small n and N.
+// Precondition: n <= pow(10, N + 1).
+template <int N> uint32_t small_division_by_pow10(uint32_t n) noexcept {
+ constexpr auto info = div_small_pow10_infos[N - 1];
+ FMT_ASSERT(n <= info.divisor * 10, "n is too large");
+ constexpr uint32_t magic_number =
+ (1u << info.shift_amount) / info.divisor + 1;
+ return (n * magic_number) >> info.shift_amount;
+}
+
+// Computes floor(n / 10^(kappa + 1)) (float)
+inline uint32_t divide_by_10_to_kappa_plus_1(uint32_t n) noexcept {
+ // 1374389535 = ceil(2^37/100)
+ return static_cast<uint32_t>((static_cast<uint64_t>(n) * 1374389535) >> 37);
+}
+// Computes floor(n / 10^(kappa + 1)) (double)
+inline uint64_t divide_by_10_to_kappa_plus_1(uint64_t n) noexcept {
+ // 2361183241434822607 = ceil(2^(64+7)/1000)
+ return umul128_upper64(n, 2361183241434822607ull) >> 7;
+}
+
+// Various subroutines using pow10 cache
+template <typename T> struct cache_accessor;
+
+template <> struct cache_accessor<float> {
+ using carrier_uint = float_info<float>::carrier_uint;
+ using cache_entry_type = uint64_t;
+
+ static uint64_t get_cached_power(int k) noexcept {
+ FMT_ASSERT(k >= float_info<float>::min_k && k <= float_info<float>::max_k,
+ "k is out of range");
+ static constexpr const uint64_t pow10_significands[] = {
+ 0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3f,
+ 0xfd87b5f28300ca0e, 0x9e74d1b791e07e49, 0xc612062576589ddb,
+ 0xf79687aed3eec552, 0x9abe14cd44753b53, 0xc16d9a0095928a28,
+ 0xf1c90080baf72cb2, 0x971da05074da7bef, 0xbce5086492111aeb,
+ 0xec1e4a7db69561a6, 0x9392ee8e921d5d08, 0xb877aa3236a4b44a,
+ 0xe69594bec44de15c, 0x901d7cf73ab0acda, 0xb424dc35095cd810,
+ 0xe12e13424bb40e14, 0x8cbccc096f5088cc, 0xafebff0bcb24aaff,
+ 0xdbe6fecebdedd5bf, 0x89705f4136b4a598, 0xabcc77118461cefd,
+ 0xd6bf94d5e57a42bd, 0x8637bd05af6c69b6, 0xa7c5ac471b478424,
+ 0xd1b71758e219652c, 0x83126e978d4fdf3c, 0xa3d70a3d70a3d70b,
+ 0xcccccccccccccccd, 0x8000000000000000, 0xa000000000000000,
+ 0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000,
+ 0xc350000000000000, 0xf424000000000000, 0x9896800000000000,
+ 0xbebc200000000000, 0xee6b280000000000, 0x9502f90000000000,
+ 0xba43b74000000000, 0xe8d4a51000000000, 0x9184e72a00000000,
+ 0xb5e620f480000000, 0xe35fa931a0000000, 0x8e1bc9bf04000000,
+ 0xb1a2bc2ec5000000, 0xde0b6b3a76400000, 0x8ac7230489e80000,
+ 0xad78ebc5ac620000, 0xd8d726b7177a8000, 0x878678326eac9000,
+ 0xa968163f0a57b400, 0xd3c21bcecceda100, 0x84595161401484a0,
+ 0xa56fa5b99019a5c8, 0xcecb8f27f4200f3a, 0x813f3978f8940985,
+ 0xa18f07d736b90be6, 0xc9f2c9cd04674edf, 0xfc6f7c4045812297,
+ 0x9dc5ada82b70b59e, 0xc5371912364ce306, 0xf684df56c3e01bc7,
+ 0x9a130b963a6c115d, 0xc097ce7bc90715b4, 0xf0bdc21abb48db21,
+ 0x96769950b50d88f5, 0xbc143fa4e250eb32, 0xeb194f8e1ae525fe,
+ 0x92efd1b8d0cf37bf, 0xb7abc627050305ae, 0xe596b7b0c643c71a,
+ 0x8f7e32ce7bea5c70, 0xb35dbf821ae4f38c, 0xe0352f62a19e306f};
+ return pow10_significands[k - float_info<float>::min_k];
+ }
+
+ struct compute_mul_result {
+ carrier_uint result;
+ bool is_integer;
+ };
+ struct compute_mul_parity_result {
+ bool parity;
+ bool is_integer;
+ };
+
+ static compute_mul_result compute_mul(
+ carrier_uint u, const cache_entry_type& cache) noexcept {
+ auto r = umul96_upper64(u, cache);
+ return {static_cast<carrier_uint>(r >> 32),
+ static_cast<carrier_uint>(r) == 0};
+ }
+
+ static uint32_t compute_delta(const cache_entry_type& cache,
+ int beta) noexcept {
+ return static_cast<uint32_t>(cache >> (64 - 1 - beta));
+ }
+
+ static compute_mul_parity_result compute_mul_parity(
+ carrier_uint two_f, const cache_entry_type& cache, int beta) noexcept {
+ FMT_ASSERT(beta >= 1, "");
+ FMT_ASSERT(beta < 64, "");
+
+ auto r = umul96_lower64(two_f, cache);
+ return {((r >> (64 - beta)) & 1) != 0,
+ static_cast<uint32_t>(r >> (32 - beta)) == 0};
+ }
+
+ static carrier_uint compute_left_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return static_cast<carrier_uint>(
+ (cache - (cache >> (num_significand_bits<float>() + 2))) >>
+ (64 - num_significand_bits<float>() - 1 - beta));
+ }
+
+ static carrier_uint compute_right_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return static_cast<carrier_uint>(
+ (cache + (cache >> (num_significand_bits<float>() + 1))) >>
+ (64 - num_significand_bits<float>() - 1 - beta));
+ }
+
+ static carrier_uint compute_round_up_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return (static_cast<carrier_uint>(
+ cache >> (64 - num_significand_bits<float>() - 2 - beta)) +
+ 1) /
+ 2;
+ }
+};
+
+template <> struct cache_accessor<double> {
+ using carrier_uint = float_info<double>::carrier_uint;
+ using cache_entry_type = uint128_fallback;
+
+ static uint128_fallback get_cached_power(int k) noexcept {
+ FMT_ASSERT(k >= float_info<double>::min_k && k <= float_info<double>::max_k,
+ "k is out of range");
+
+ static constexpr const uint128_fallback pow10_significands[] = {
+#if FMT_USE_FULL_CACHE_DRAGONBOX
+ {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b},
+ {0x9faacf3df73609b1, 0x77b191618c54e9ad},
+ {0xc795830d75038c1d, 0xd59df5b9ef6a2418},
+ {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e},
+ {0x9becce62836ac577, 0x4ee367f9430aec33},
+ {0xc2e801fb244576d5, 0x229c41f793cda740},
+ {0xf3a20279ed56d48a, 0x6b43527578c11110},
+ {0x9845418c345644d6, 0x830a13896b78aaaa},
+ {0xbe5691ef416bd60c, 0x23cc986bc656d554},
+ {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9},
+ {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa},
+ {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54},
+ {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69},
+ {0x91376c36d99995be, 0x23100809b9c21fa2},
+ {0xb58547448ffffb2d, 0xabd40a0c2832a78b},
+ {0xe2e69915b3fff9f9, 0x16c90c8f323f516d},
+ {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4},
+ {0xb1442798f49ffb4a, 0x99cd11cfdf41779d},
+ {0xdd95317f31c7fa1d, 0x40405643d711d584},
+ {0x8a7d3eef7f1cfc52, 0x482835ea666b2573},
+ {0xad1c8eab5ee43b66, 0xda3243650005eed0},
+ {0xd863b256369d4a40, 0x90bed43e40076a83},
+ {0x873e4f75e2224e68, 0x5a7744a6e804a292},
+ {0xa90de3535aaae202, 0x711515d0a205cb37},
+ {0xd3515c2831559a83, 0x0d5a5b44ca873e04},
+ {0x8412d9991ed58091, 0xe858790afe9486c3},
+ {0xa5178fff668ae0b6, 0x626e974dbe39a873},
+ {0xce5d73ff402d98e3, 0xfb0a3d212dc81290},
+ {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a},
+ {0xa139029f6a239f72, 0x1c1fffc1ebc44e81},
+ {0xc987434744ac874e, 0xa327ffb266b56221},
+ {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9},
+ {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa},
+ {0xc4ce17b399107c22, 0xcb550fb4384d21d4},
+ {0xf6019da07f549b2b, 0x7e2a53a146606a49},
+ {0x99c102844f94e0fb, 0x2eda7444cbfc426e},
+ {0xc0314325637a1939, 0xfa911155fefb5309},
+ {0xf03d93eebc589f88, 0x793555ab7eba27cb},
+ {0x96267c7535b763b5, 0x4bc1558b2f3458df},
+ {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17},
+ {0xea9c227723ee8bcb, 0x465e15a979c1cadd},
+ {0x92a1958a7675175f, 0x0bfacd89ec191eca},
+ {0xb749faed14125d36, 0xcef980ec671f667c},
+ {0xe51c79a85916f484, 0x82b7e12780e7401b},
+ {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811},
+ {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16},
+ {0xdfbdcece67006ac9, 0x67a791e093e1d49b},
+ {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1},
+ {0xaecc49914078536d, 0x58fae9f773886e19},
+ {0xda7f5bf590966848, 0xaf39a475506a899f},
+ {0x888f99797a5e012d, 0x6d8406c952429604},
+ {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84},
+ {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65},
+ {0x855c3be0a17fcd26, 0x5cf2eea09a550680},
+ {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f},
+ {0xd0601d8efc57b08b, 0xf13b94daf124da27},
+ {0x823c12795db6ce57, 0x76c53d08d6b70859},
+ {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f},
+ {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a},
+ {0xfe5d54150b090b02, 0xd3f93b35435d7c4d},
+ {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0},
+ {0xc6b8e9b0709f109a, 0x359ab6419ca1091c},
+ {0xf867241c8cc6d4c0, 0xc30163d203c94b63},
+ {0x9b407691d7fc44f8, 0x79e0de63425dcf1e},
+ {0xc21094364dfb5636, 0x985915fc12f542e5},
+ {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e},
+ {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43},
+ {0xbd8430bd08277231, 0x50c6ff782a838354},
+ {0xece53cec4a314ebd, 0xa4f8bf5635246429},
+ {0x940f4613ae5ed136, 0x871b7795e136be9a},
+ {0xb913179899f68584, 0x28e2557b59846e40},
+ {0xe757dd7ec07426e5, 0x331aeada2fe589d0},
+ {0x9096ea6f3848984f, 0x3ff0d2c85def7622},
+ {0xb4bca50b065abe63, 0x0fed077a756b53aa},
+ {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895},
+ {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d},
+ {0xb080392cc4349dec, 0xbd8d794d96aacfb4},
+ {0xdca04777f541c567, 0xecf0d7a0fc5583a1},
+ {0x89e42caaf9491b60, 0xf41686c49db57245},
+ {0xac5d37d5b79b6239, 0x311c2875c522ced6},
+ {0xd77485cb25823ac7, 0x7d633293366b828c},
+ {0x86a8d39ef77164bc, 0xae5dff9c02033198},
+ {0xa8530886b54dbdeb, 0xd9f57f830283fdfd},
+ {0xd267caa862a12d66, 0xd072df63c324fd7c},
+ {0x8380dea93da4bc60, 0x4247cb9e59f71e6e},
+ {0xa46116538d0deb78, 0x52d9be85f074e609},
+ {0xcd795be870516656, 0x67902e276c921f8c},
+ {0x806bd9714632dff6, 0x00ba1cd8a3db53b7},
+ {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5},
+ {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce},
+ {0xfad2a4b13d1b5d6c, 0x796b805720085f82},
+ {0x9cc3a6eec6311a63, 0xcbe3303674053bb1},
+ {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d},
+ {0xf4f1b4d515acb93b, 0xee92fb5515482d45},
+ {0x991711052d8bf3c5, 0x751bdd152d4d1c4b},
+ {0xbf5cd54678eef0b6, 0xd262d45a78a0635e},
+ {0xef340a98172aace4, 0x86fb897116c87c35},
+ {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1},
+ {0xbae0a846d2195712, 0x8974836059cca10a},
+ {0xe998d258869facd7, 0x2bd1a438703fc94c},
+ {0x91ff83775423cc06, 0x7b6306a34627ddd0},
+ {0xb67f6455292cbf08, 0x1a3bc84c17b1d543},
+ {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94},
+ {0x8e938662882af53e, 0x547eb47b7282ee9d},
+ {0xb23867fb2a35b28d, 0xe99e619a4f23aa44},
+ {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5},
+ {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05},
+ {0xae0b158b4738705e, 0x9624ab50b148d446},
+ {0xd98ddaee19068c76, 0x3badd624dd9b0958},
+ {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7},
+ {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d},
+ {0xd47487cc8470652b, 0x7647c32000696720},
+ {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074},
+ {0xa5fb0a17c777cf09, 0xf468107100525891},
+ {0xcf79cc9db955c2cc, 0x7182148d4066eeb5},
+ {0x81ac1fe293d599bf, 0xc6f14cd848405531},
+ {0xa21727db38cb002f, 0xb8ada00e5a506a7d},
+ {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d},
+ {0xfd442e4688bd304a, 0x908f4a166d1da664},
+ {0x9e4a9cec15763e2e, 0x9a598e4e043287ff},
+ {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe},
+ {0xf7549530e188c128, 0xd12bee59e68ef47d},
+ {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf},
+ {0xc13a148e3032d6e7, 0xe36a52363c1faf02},
+ {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2},
+ {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba},
+ {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8},
+ {0xebdf661791d60f56, 0x111b495b3464ad22},
+ {0x936b9fcebb25c995, 0xcab10dd900beec35},
+ {0xb84687c269ef3bfb, 0x3d5d514f40eea743},
+ {0xe65829b3046b0afa, 0x0cb4a5a3112a5113},
+ {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac},
+ {0xb3f4e093db73a093, 0x59ed216765690f57},
+ {0xe0f218b8d25088b8, 0x306869c13ec3532d},
+ {0x8c974f7383725573, 0x1e414218c73a13fc},
+ {0xafbd2350644eeacf, 0xe5d1929ef90898fb},
+ {0xdbac6c247d62a583, 0xdf45f746b74abf3a},
+ {0x894bc396ce5da772, 0x6b8bba8c328eb784},
+ {0xab9eb47c81f5114f, 0x066ea92f3f326565},
+ {0xd686619ba27255a2, 0xc80a537b0efefebe},
+ {0x8613fd0145877585, 0xbd06742ce95f5f37},
+ {0xa798fc4196e952e7, 0x2c48113823b73705},
+ {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6},
+ {0x82ef85133de648c4, 0x9a984d73dbe722fc},
+ {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb},
+ {0xcc963fee10b7d1b3, 0x318df905079926a9},
+ {0xffbbcfe994e5c61f, 0xfdf17746497f7053},
+ {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634},
+ {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1},
+ {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1},
+ {0x9c1661a651213e2d, 0x06bea10ca65c084f},
+ {0xc31bfa0fe5698db8, 0x486e494fcff30a63},
+ {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb},
+ {0x986ddb5c6b3a76b7, 0xf89629465a75e01d},
+ {0xbe89523386091465, 0xf6bbb397f1135824},
+ {0xee2ba6c0678b597f, 0x746aa07ded582e2d},
+ {0x94db483840b717ef, 0xa8c2a44eb4571cdd},
+ {0xba121a4650e4ddeb, 0x92f34d62616ce414},
+ {0xe896a0d7e51e1566, 0x77b020baf9c81d18},
+ {0x915e2486ef32cd60, 0x0ace1474dc1d122f},
+ {0xb5b5ada8aaff80b8, 0x0d819992132456bb},
+ {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a},
+ {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2},
+ {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3},
+ {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf},
+ {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c},
+ {0xad4ab7112eb3929d, 0x86c16c98d2c953c7},
+ {0xd89d64d57a607744, 0xe871c7bf077ba8b8},
+ {0x87625f056c7c4a8b, 0x11471cd764ad4973},
+ {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0},
+ {0xd389b47879823479, 0x4aff1d108d4ec2c4},
+ {0x843610cb4bf160cb, 0xcedf722a585139bb},
+ {0xa54394fe1eedb8fe, 0xc2974eb4ee658829},
+ {0xce947a3da6a9273e, 0x733d226229feea33},
+ {0x811ccc668829b887, 0x0806357d5a3f5260},
+ {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8},
+ {0xc9bcff6034c13052, 0xfc89b393dd02f0b6},
+ {0xfc2c3f3841f17c67, 0xbbac2078d443ace3},
+ {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e},
+ {0xc5029163f384a931, 0x0a9e795e65d4df12},
+ {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6},
+ {0x99ea0196163fa42e, 0x504bced1bf8e4e46},
+ {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7},
+ {0xf07da27a82c37088, 0x5d767327bb4e5a4d},
+ {0x964e858c91ba2655, 0x3a6a07f8d510f870},
+ {0xbbe226efb628afea, 0x890489f70a55368c},
+ {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f},
+ {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e},
+ {0xb77ada0617e3bbcb, 0x09ce6ebb40173745},
+ {0xe55990879ddcaabd, 0xcc420a6a101d0516},
+ {0x8f57fa54c2a9eab6, 0x9fa946824a12232e},
+ {0xb32df8e9f3546564, 0x47939822dc96abfa},
+ {0xdff9772470297ebd, 0x59787e2b93bc56f8},
+ {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b},
+ {0xaefae51477a06b03, 0xede622920b6b23f2},
+ {0xdab99e59958885c4, 0xe95fab368e45ecee},
+ {0x88b402f7fd75539b, 0x11dbcb0218ebb415},
+ {0xaae103b5fcd2a881, 0xd652bdc29f26a11a},
+ {0xd59944a37c0752a2, 0x4be76d3346f04960},
+ {0x857fcae62d8493a5, 0x6f70a4400c562ddc},
+ {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953},
+ {0xd097ad07a71f26b2, 0x7e2000a41346a7a8},
+ {0x825ecc24c873782f, 0x8ed400668c0c28c9},
+ {0xa2f67f2dfa90563b, 0x728900802f0f32fb},
+ {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba},
+ {0xfea126b7d78186bc, 0xe2f610c84987bfa9},
+ {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca},
+ {0xc6ede63fa05d3143, 0x91503d1c79720dbc},
+ {0xf8a95fcf88747d94, 0x75a44c6397ce912b},
+ {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb},
+ {0xc24452da229b021b, 0xfbe85badce996169},
+ {0xf2d56790ab41c2a2, 0xfae27299423fb9c4},
+ {0x97c560ba6b0919a5, 0xdccd879fc967d41b},
+ {0xbdb6b8e905cb600f, 0x5400e987bbc1c921},
+ {0xed246723473e3813, 0x290123e9aab23b69},
+ {0x9436c0760c86e30b, 0xf9a0b6720aaf6522},
+ {0xb94470938fa89bce, 0xf808e40e8d5b3e6a},
+ {0xe7958cb87392c2c2, 0xb60b1d1230b20e05},
+ {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3},
+ {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4},
+ {0xe2280b6c20dd5232, 0x25c6da63c38de1b1},
+ {0x8d590723948a535f, 0x579c487e5a38ad0f},
+ {0xb0af48ec79ace837, 0x2d835a9df0c6d852},
+ {0xdcdb1b2798182244, 0xf8e431456cf88e66},
+ {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900},
+ {0xac8b2d36eed2dac5, 0xe272467e3d222f40},
+ {0xd7adf884aa879177, 0x5b0ed81dcc6abb10},
+ {0x86ccbb52ea94baea, 0x98e947129fc2b4ea},
+ {0xa87fea27a539e9a5, 0x3f2398d747b36225},
+ {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae},
+ {0x83a3eeeef9153e89, 0x1953cf68300424ad},
+ {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8},
+ {0xcdb02555653131b6, 0x3792f412cb06794e},
+ {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1},
+ {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5},
+ {0xc8de047564d20a8b, 0xf245825a5a445276},
+ {0xfb158592be068d2e, 0xeed6e2f0f0d56713},
+ {0x9ced737bb6c4183d, 0x55464dd69685606c},
+ {0xc428d05aa4751e4c, 0xaa97e14c3c26b887},
+ {0xf53304714d9265df, 0xd53dd99f4b3066a9},
+ {0x993fe2c6d07b7fab, 0xe546a8038efe402a},
+ {0xbf8fdb78849a5f96, 0xde98520472bdd034},
+ {0xef73d256a5c0f77c, 0x963e66858f6d4441},
+ {0x95a8637627989aad, 0xdde7001379a44aa9},
+ {0xbb127c53b17ec159, 0x5560c018580d5d53},
+ {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7},
+ {0x9226712162ab070d, 0xcab3961304ca70e9},
+ {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23},
+ {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b},
+ {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243},
+ {0xb267ed1940f1c61c, 0x55f038b237591ed4},
+ {0xdf01e85f912e37a3, 0x6b6c46dec52f6689},
+ {0x8b61313bbabce2c6, 0x2323ac4b3b3da016},
+ {0xae397d8aa96c1b77, 0xabec975e0a0d081b},
+ {0xd9c7dced53c72255, 0x96e7bd358c904a22},
+ {0x881cea14545c7575, 0x7e50d64177da2e55},
+ {0xaa242499697392d2, 0xdde50bd1d5d0b9ea},
+ {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865},
+ {0x84ec3c97da624ab4, 0xbd5af13bef0b113f},
+ {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f},
+ {0xcfb11ead453994ba, 0x67de18eda5814af3},
+ {0x81ceb32c4b43fcf4, 0x80eacf948770ced8},
+ {0xa2425ff75e14fc31, 0xa1258379a94d028e},
+ {0xcad2f7f5359a3b3e, 0x096ee45813a04331},
+ {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd},
+ {0x9e74d1b791e07e48, 0x775ea264cf55347e},
+ {0xc612062576589dda, 0x95364afe032a819e},
+ {0xf79687aed3eec551, 0x3a83ddbd83f52205},
+ {0x9abe14cd44753b52, 0xc4926a9672793543},
+ {0xc16d9a0095928a27, 0x75b7053c0f178294},
+ {0xf1c90080baf72cb1, 0x5324c68b12dd6339},
+ {0x971da05074da7bee, 0xd3f6fc16ebca5e04},
+ {0xbce5086492111aea, 0x88f4bb1ca6bcf585},
+ {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6},
+ {0x9392ee8e921d5d07, 0x3aff322e62439fd0},
+ {0xb877aa3236a4b449, 0x09befeb9fad487c3},
+ {0xe69594bec44de15b, 0x4c2ebe687989a9b4},
+ {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11},
+ {0xb424dc35095cd80f, 0x538484c19ef38c95},
+ {0xe12e13424bb40e13, 0x2865a5f206b06fba},
+ {0x8cbccc096f5088cb, 0xf93f87b7442e45d4},
+ {0xafebff0bcb24aafe, 0xf78f69a51539d749},
+ {0xdbe6fecebdedd5be, 0xb573440e5a884d1c},
+ {0x89705f4136b4a597, 0x31680a88f8953031},
+ {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e},
+ {0xd6bf94d5e57a42bc, 0x3d32907604691b4d},
+ {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110},
+ {0xa7c5ac471b478423, 0x0fcf80dc33721d54},
+ {0xd1b71758e219652b, 0xd3c36113404ea4a9},
+ {0x83126e978d4fdf3b, 0x645a1cac083126ea},
+ {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4},
+ {0xcccccccccccccccc, 0xcccccccccccccccd},
+ {0x8000000000000000, 0x0000000000000000},
+ {0xa000000000000000, 0x0000000000000000},
+ {0xc800000000000000, 0x0000000000000000},
+ {0xfa00000000000000, 0x0000000000000000},
+ {0x9c40000000000000, 0x0000000000000000},
+ {0xc350000000000000, 0x0000000000000000},
+ {0xf424000000000000, 0x0000000000000000},
+ {0x9896800000000000, 0x0000000000000000},
+ {0xbebc200000000000, 0x0000000000000000},
+ {0xee6b280000000000, 0x0000000000000000},
+ {0x9502f90000000000, 0x0000000000000000},
+ {0xba43b74000000000, 0x0000000000000000},
+ {0xe8d4a51000000000, 0x0000000000000000},
+ {0x9184e72a00000000, 0x0000000000000000},
+ {0xb5e620f480000000, 0x0000000000000000},
+ {0xe35fa931a0000000, 0x0000000000000000},
+ {0x8e1bc9bf04000000, 0x0000000000000000},
+ {0xb1a2bc2ec5000000, 0x0000000000000000},
+ {0xde0b6b3a76400000, 0x0000000000000000},
+ {0x8ac7230489e80000, 0x0000000000000000},
+ {0xad78ebc5ac620000, 0x0000000000000000},
+ {0xd8d726b7177a8000, 0x0000000000000000},
+ {0x878678326eac9000, 0x0000000000000000},
+ {0xa968163f0a57b400, 0x0000000000000000},
+ {0xd3c21bcecceda100, 0x0000000000000000},
+ {0x84595161401484a0, 0x0000000000000000},
+ {0xa56fa5b99019a5c8, 0x0000000000000000},
+ {0xcecb8f27f4200f3a, 0x0000000000000000},
+ {0x813f3978f8940984, 0x4000000000000000},
+ {0xa18f07d736b90be5, 0x5000000000000000},
+ {0xc9f2c9cd04674ede, 0xa400000000000000},
+ {0xfc6f7c4045812296, 0x4d00000000000000},
+ {0x9dc5ada82b70b59d, 0xf020000000000000},
+ {0xc5371912364ce305, 0x6c28000000000000},
+ {0xf684df56c3e01bc6, 0xc732000000000000},
+ {0x9a130b963a6c115c, 0x3c7f400000000000},
+ {0xc097ce7bc90715b3, 0x4b9f100000000000},
+ {0xf0bdc21abb48db20, 0x1e86d40000000000},
+ {0x96769950b50d88f4, 0x1314448000000000},
+ {0xbc143fa4e250eb31, 0x17d955a000000000},
+ {0xeb194f8e1ae525fd, 0x5dcfab0800000000},
+ {0x92efd1b8d0cf37be, 0x5aa1cae500000000},
+ {0xb7abc627050305ad, 0xf14a3d9e40000000},
+ {0xe596b7b0c643c719, 0x6d9ccd05d0000000},
+ {0x8f7e32ce7bea5c6f, 0xe4820023a2000000},
+ {0xb35dbf821ae4f38b, 0xdda2802c8a800000},
+ {0xe0352f62a19e306e, 0xd50b2037ad200000},
+ {0x8c213d9da502de45, 0x4526f422cc340000},
+ {0xaf298d050e4395d6, 0x9670b12b7f410000},
+ {0xdaf3f04651d47b4c, 0x3c0cdd765f114000},
+ {0x88d8762bf324cd0f, 0xa5880a69fb6ac800},
+ {0xab0e93b6efee0053, 0x8eea0d047a457a00},
+ {0xd5d238a4abe98068, 0x72a4904598d6d880},
+ {0x85a36366eb71f041, 0x47a6da2b7f864750},
+ {0xa70c3c40a64e6c51, 0x999090b65f67d924},
+ {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d},
+ {0x82818f1281ed449f, 0xbff8f10e7a8921a5},
+ {0xa321f2d7226895c7, 0xaff72d52192b6a0e},
+ {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764491},
+ {0xfee50b7025c36a08, 0x02f236d04753d5b5},
+ {0x9f4f2726179a2245, 0x01d762422c946591},
+ {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef6},
+ {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb3},
+ {0x9b934c3b330c8577, 0x63cc55f49f88eb30},
+ {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fc},
+ {0xf316271c7fc3908a, 0x8bef464e3945ef7b},
+ {0x97edd871cfda3a56, 0x97758bf0e3cbb5ad},
+ {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea318},
+ {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bde},
+ {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6b},
+ {0xb975d6b6ee39e436, 0xb3e2fd538e122b45},
+ {0xe7d34c64a9c85d44, 0x60dbbca87196b617},
+ {0x90e40fbeea1d3a4a, 0xbc8955e946fe31ce},
+ {0xb51d13aea4a488dd, 0x6babab6398bdbe42},
+ {0xe264589a4dcdab14, 0xc696963c7eed2dd2},
+ {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca3},
+ {0xb0de65388cc8ada8, 0x3b25a55f43294bcc},
+ {0xdd15fe86affad912, 0x49ef0eb713f39ebf},
+ {0x8a2dbf142dfcc7ab, 0x6e3569326c784338},
+ {0xacb92ed9397bf996, 0x49c2c37f07965405},
+ {0xd7e77a8f87daf7fb, 0xdc33745ec97be907},
+ {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a4},
+ {0xa8acd7c0222311bc, 0xc40832ea0d68ce0d},
+ {0xd2d80db02aabd62b, 0xf50a3fa490c30191},
+ {0x83c7088e1aab65db, 0x792667c6da79e0fb},
+ {0xa4b8cab1a1563f52, 0x577001b891185939},
+ {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87},
+ {0x80b05e5ac60b6178, 0x544f8158315b05b5},
+ {0xa0dc75f1778e39d6, 0x696361ae3db1c722},
+ {0xc913936dd571c84c, 0x03bc3a19cd1e38ea},
+ {0xfb5878494ace3a5f, 0x04ab48a04065c724},
+ {0x9d174b2dcec0e47b, 0x62eb0d64283f9c77},
+ {0xc45d1df942711d9a, 0x3ba5d0bd324f8395},
+ {0xf5746577930d6500, 0xca8f44ec7ee3647a},
+ {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecc},
+ {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67f},
+ {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101f},
+ {0x95d04aee3b80ece5, 0xbba1f1d158724a13},
+ {0xbb445da9ca61281f, 0x2a8a6e45ae8edc98},
+ {0xea1575143cf97226, 0xf52d09d71a3293be},
+ {0x924d692ca61be758, 0x593c2626705f9c57},
+ {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836d},
+ {0xe498f455c38b997a, 0x0b6dfb9c0f956448},
+ {0x8edf98b59a373fec, 0x4724bd4189bd5ead},
+ {0xb2977ee300c50fe7, 0x58edec91ec2cb658},
+ {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ee},
+ {0x8b865b215899f46c, 0xbd79e0d20082ee75},
+ {0xae67f1e9aec07187, 0xecd8590680a3aa12},
+ {0xda01ee641a708de9, 0xe80e6f4820cc9496},
+ {0x884134fe908658b2, 0x3109058d147fdcde},
+ {0xaa51823e34a7eede, 0xbd4b46f0599fd416},
+ {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91b},
+ {0x850fadc09923329e, 0x03e2cf6bc604ddb1},
+ {0xa6539930bf6bff45, 0x84db8346b786151d},
+ {0xcfe87f7cef46ff16, 0xe612641865679a64},
+ {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07f},
+ {0xa26da3999aef7749, 0xe3be5e330f38f09e},
+ {0xcb090c8001ab551c, 0x5cadf5bfd3072cc6},
+ {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f7},
+ {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afb},
+ {0xc646d63501a1511d, 0xb281e1fd541501b9},
+ {0xf7d88bc24209a565, 0x1f225a7ca91a4227},
+ {0x9ae757596946075f, 0x3375788de9b06959},
+ {0xc1a12d2fc3978937, 0x0052d6b1641c83af},
+ {0xf209787bb47d6b84, 0xc0678c5dbd23a49b},
+ {0x9745eb4d50ce6332, 0xf840b7ba963646e1},
+ {0xbd176620a501fbff, 0xb650e5a93bc3d899},
+ {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebf},
+ {0x93ba47c980e98cdf, 0xc66f336c36b10138},
+ {0xb8a8d9bbe123f017, 0xb80b0047445d4185},
+ {0xe6d3102ad96cec1d, 0xa60dc059157491e6},
+ {0x9043ea1ac7e41392, 0x87c89837ad68db30},
+ {0xb454e4a179dd1877, 0x29babe4598c311fc},
+ {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67b},
+ {0x8ce2529e2734bb1d, 0x1899e4a65f58660d},
+ {0xb01ae745b101e9e4, 0x5ec05dcff72e7f90},
+ {0xdc21a1171d42645d, 0x76707543f4fa1f74},
+ {0x899504ae72497eba, 0x6a06494a791c53a9},
+ {0xabfa45da0edbde69, 0x0487db9d17636893},
+ {0xd6f8d7509292d603, 0x45a9d2845d3c42b7},
+ {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3},
+ {0xa7f26836f282b732, 0x8e6cac7768d7141f},
+ {0xd1ef0244af2364ff, 0x3207d795430cd927},
+ {0x8335616aed761f1f, 0x7f44e6bd49e807b9},
+ {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a7},
+ {0xcd036837130890a1, 0x36dba887c37a8c10},
+ {0x802221226be55a64, 0xc2494954da2c978a},
+ {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6d},
+ {0xc83553c5c8965d3d, 0x6f92829494e5acc8},
+ {0xfa42a8b73abbf48c, 0xcb772339ba1f17fa},
+ {0x9c69a97284b578d7, 0xff2a760414536efc},
+ {0xc38413cf25e2d70d, 0xfef5138519684abb},
+ {0xf46518c2ef5b8cd1, 0x7eb258665fc25d6a},
+ {0x98bf2f79d5993802, 0xef2f773ffbd97a62},
+ {0xbeeefb584aff8603, 0xaafb550ffacfd8fb},
+ {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf39},
+ {0x952ab45cfa97a0b2, 0xdd945a747bf26184},
+ {0xba756174393d88df, 0x94f971119aeef9e5},
+ {0xe912b9d1478ceb17, 0x7a37cd5601aab85e},
+ {0x91abb422ccb812ee, 0xac62e055c10ab33b},
+ {0xb616a12b7fe617aa, 0x577b986b314d600a},
+ {0xe39c49765fdf9d94, 0xed5a7e85fda0b80c},
+ {0x8e41ade9fbebc27d, 0x14588f13be847308},
+ {0xb1d219647ae6b31c, 0x596eb2d8ae258fc9},
+ {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bc},
+ {0x8aec23d680043bee, 0x25de7bb9480d5855},
+ {0xada72ccc20054ae9, 0xaf561aa79a10ae6b},
+ {0xd910f7ff28069da4, 0x1b2ba1518094da05},
+ {0x87aa9aff79042286, 0x90fb44d2f05d0843},
+ {0xa99541bf57452b28, 0x353a1607ac744a54},
+ {0xd3fa922f2d1675f2, 0x42889b8997915ce9},
+ {0x847c9b5d7c2e09b7, 0x69956135febada12},
+ {0xa59bc234db398c25, 0x43fab9837e699096},
+ {0xcf02b2c21207ef2e, 0x94f967e45e03f4bc},
+ {0x8161afb94b44f57d, 0x1d1be0eebac278f6},
+ {0xa1ba1ba79e1632dc, 0x6462d92a69731733},
+ {0xca28a291859bbf93, 0x7d7b8f7503cfdcff},
+ {0xfcb2cb35e702af78, 0x5cda735244c3d43f},
+ {0x9defbf01b061adab, 0x3a0888136afa64a8},
+ {0xc56baec21c7a1916, 0x088aaa1845b8fdd1},
+ {0xf6c69a72a3989f5b, 0x8aad549e57273d46},
+ {0x9a3c2087a63f6399, 0x36ac54e2f678864c},
+ {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7de},
+ {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d6},
+ {0x969eb7c47859e743, 0x9f644ae5a4b1b326},
+ {0xbc4665b596706114, 0x873d5d9f0dde1fef},
+ {0xeb57ff22fc0c7959, 0xa90cb506d155a7eb},
+ {0x9316ff75dd87cbd8, 0x09a7f12442d588f3},
+ {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb30},
+ {0xe5d3ef282a242e81, 0x8f1668c8a86da5fb},
+ {0x8fa475791a569d10, 0xf96e017d694487bd},
+ {0xb38d92d760ec4455, 0x37c981dcc395a9ad},
+ {0xe070f78d3927556a, 0x85bbe253f47b1418},
+ {0x8c469ab843b89562, 0x93956d7478ccec8f},
+ {0xaf58416654a6babb, 0x387ac8d1970027b3},
+ {0xdb2e51bfe9d0696a, 0x06997b05fcc0319f},
+ {0x88fcf317f22241e2, 0x441fece3bdf81f04},
+ {0xab3c2fddeeaad25a, 0xd527e81cad7626c4},
+ {0xd60b3bd56a5586f1, 0x8a71e223d8d3b075},
+ {0x85c7056562757456, 0xf6872d5667844e4a},
+ {0xa738c6bebb12d16c, 0xb428f8ac016561dc},
+ {0xd106f86e69d785c7, 0xe13336d701beba53},
+ {0x82a45b450226b39c, 0xecc0024661173474},
+ {0xa34d721642b06084, 0x27f002d7f95d0191},
+ {0xcc20ce9bd35c78a5, 0x31ec038df7b441f5},
+ {0xff290242c83396ce, 0x7e67047175a15272},
+ {0x9f79a169bd203e41, 0x0f0062c6e984d387},
+ {0xc75809c42c684dd1, 0x52c07b78a3e60869},
+ {0xf92e0c3537826145, 0xa7709a56ccdf8a83},
+ {0x9bbcc7a142b17ccb, 0x88a66076400bb692},
+ {0xc2abf989935ddbfe, 0x6acff893d00ea436},
+ {0xf356f7ebf83552fe, 0x0583f6b8c4124d44},
+ {0x98165af37b2153de, 0xc3727a337a8b704b},
+ {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5d},
+ {0xeda2ee1c7064130c, 0x1162def06f79df74},
+ {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba9},
+ {0xb9a74a0637ce2ee1, 0x6d953e2bd7173693},
+ {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0438},
+ {0x910ab1d4db9914a0, 0x1d9c9892400a22a3},
+ {0xb54d5e4a127f59c8, 0x2503beb6d00cab4c},
+ {0xe2a0b5dc971f303a, 0x2e44ae64840fd61e},
+ {0x8da471a9de737e24, 0x5ceaecfed289e5d3},
+ {0xb10d8e1456105dad, 0x7425a83e872c5f48},
+ {0xdd50f1996b947518, 0xd12f124e28f7771a},
+ {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa70},
+ {0xace73cbfdc0bfb7b, 0x636cc64d1001550c},
+ {0xd8210befd30efa5a, 0x3c47f7e05401aa4f},
+ {0x8714a775e3e95c78, 0x65acfaec34810a72},
+ {0xa8d9d1535ce3b396, 0x7f1839a741a14d0e},
+ {0xd31045a8341ca07c, 0x1ede48111209a051},
+ {0x83ea2b892091e44d, 0x934aed0aab460433},
+ {0xa4e4b66b68b65d60, 0xf81da84d56178540},
+ {0xce1de40642e3f4b9, 0x36251260ab9d668f},
+ {0x80d2ae83e9ce78f3, 0xc1d72b7c6b42601a},
+ {0xa1075a24e4421730, 0xb24cf65b8612f820},
+ {0xc94930ae1d529cfc, 0xdee033f26797b628},
+ {0xfb9b7cd9a4a7443c, 0x169840ef017da3b2},
+ {0x9d412e0806e88aa5, 0x8e1f289560ee864f},
+ {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e3},
+ {0xf5b5d7ec8acb58a2, 0xae10af696774b1dc},
+ {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef2a},
+ {0xbff610b0cc6edd3f, 0x17fd090a58d32af4},
+ {0xeff394dcff8a948e, 0xddfc4b4cef07f5b1},
+ {0x95f83d0a1fb69cd9, 0x4abdaf101564f98f},
+ {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f2},
+ {0xea53df5fd18d5513, 0x84c86189216dc5ee},
+ {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb5},
+ {0xb7118682dbb66a77, 0x3fbc8c33221dc2a2},
+ {0xe4d5e82392a40515, 0x0fabaf3feaa5334b},
+ {0x8f05b1163ba6832d, 0x29cb4d87f2a7400f},
+ {0xb2c71d5bca9023f8, 0x743e20e9ef511013},
+ {0xdf78e4b2bd342cf6, 0x914da9246b255417},
+ {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548f},
+ {0xae9672aba3d0c320, 0xa184ac2473b529b2},
+ {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741f},
+ {0x8865899617fb1871, 0x7e2fa67c7a658893},
+ {0xaa7eebfb9df9de8d, 0xddbb901b98feeab8},
+ {0xd51ea6fa85785631, 0x552a74227f3ea566},
+ {0x8533285c936b35de, 0xd53a88958f872760},
+ {0xa67ff273b8460356, 0x8a892abaf368f138},
+ {0xd01fef10a657842c, 0x2d2b7569b0432d86},
+ {0x8213f56a67f6b29b, 0x9c3b29620e29fc74},
+ {0xa298f2c501f45f42, 0x8349f3ba91b47b90},
+ {0xcb3f2f7642717713, 0x241c70a936219a74},
+ {0xfe0efb53d30dd4d7, 0xed238cd383aa0111},
+ {0x9ec95d1463e8a506, 0xf4363804324a40ab},
+ {0xc67bb4597ce2ce48, 0xb143c6053edcd0d6},
+ {0xf81aa16fdc1b81da, 0xdd94b7868e94050b},
+ {0x9b10a4e5e9913128, 0xca7cf2b4191c8327},
+ {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f1},
+ {0xf24a01a73cf2dccf, 0xbc633b39673c8ced},
+ {0x976e41088617ca01, 0xd5be0503e085d814},
+ {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e19},
+ {0xec9c459d51852ba2, 0xddf8e7d60ed1219f},
+ {0x93e1ab8252f33b45, 0xcabb90e5c942b504},
+ {0xb8da1662e7b00a17, 0x3d6a751f3b936244},
+ {0xe7109bfba19c0c9d, 0x0cc512670a783ad5},
+ {0x906a617d450187e2, 0x27fb2b80668b24c6},
+ {0xb484f9dc9641e9da, 0xb1f9f660802dedf7},
+ {0xe1a63853bbd26451, 0x5e7873f8a0396974},
+ {0x8d07e33455637eb2, 0xdb0b487b6423e1e9},
+ {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda63},
+ {0xdc5c5301c56b75f7, 0x7641a140cc7810fc},
+ {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9e},
+ {0xac2820d9623bf429, 0x546345fa9fbdcd45},
+ {0xd732290fbacaf133, 0xa97c177947ad4096},
+ {0x867f59a9d4bed6c0, 0x49ed8eabcccc485e},
+ {0xa81f301449ee8c70, 0x5c68f256bfff5a75},
+ {0xd226fc195c6a2f8c, 0x73832eec6fff3112},
+ {0x83585d8fd9c25db7, 0xc831fd53c5ff7eac},
+ {0xa42e74f3d032f525, 0xba3e7ca8b77f5e56},
+ {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35ec},
+ {0x80444b5e7aa7cf85, 0x7980d163cf5b81b4},
+ {0xa0555e361951c366, 0xd7e105bcc3326220},
+ {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa8},
+ {0xfa856334878fc150, 0xb14f98f6f0feb952},
+ {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d4},
+ {0xc3b8358109e84f07, 0x0a862f80ec4700c9},
+ {0xf4a642e14c6262c8, 0xcd27bb612758c0fb},
+ {0x98e7e9cccfbd7dbd, 0x8038d51cb897789d},
+ {0xbf21e44003acdd2c, 0xe0470a63e6bd56c4},
+ {0xeeea5d5004981478, 0x1858ccfce06cac75},
+ {0x95527a5202df0ccb, 0x0f37801e0c43ebc9},
+ {0xbaa718e68396cffd, 0xd30560258f54e6bb},
+ {0xe950df20247c83fd, 0x47c6b82ef32a206a},
+ {0x91d28b7416cdd27e, 0x4cdc331d57fa5442},
+ {0xb6472e511c81471d, 0xe0133fe4adf8e953},
+ {0xe3d8f9e563a198e5, 0x58180fddd97723a7},
+ {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7649},
+ {0xb201833b35d63f73, 0x2cd2cc6551e513db},
+ {0xde81e40a034bcf4f, 0xf8077f7ea65e58d2},
+ {0x8b112e86420f6191, 0xfb04afaf27faf783},
+ {0xadd57a27d29339f6, 0x79c5db9af1f9b564},
+ {0xd94ad8b1c7380874, 0x18375281ae7822bd},
+ {0x87cec76f1c830548, 0x8f2293910d0b15b6},
+ {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb23},
+ {0xd433179d9c8cb841, 0x5fa60692a46151ec},
+ {0x849feec281d7f328, 0xdbc7c41ba6bcd334},
+ {0xa5c7ea73224deff3, 0x12b9b522906c0801},
+ {0xcf39e50feae16bef, 0xd768226b34870a01},
+ {0x81842f29f2cce375, 0xe6a1158300d46641},
+ {0xa1e53af46f801c53, 0x60495ae3c1097fd1},
+ {0xca5e89b18b602368, 0x385bb19cb14bdfc5},
+ {0xfcf62c1dee382c42, 0x46729e03dd9ed7b6},
+ {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d2},
+ {0xc5a05277621be293, 0xc7098b7305241886},
+ {0xf70867153aa2db38, 0xb8cbee4fc66d1ea8},
+ {0x9a65406d44a5c903, 0x737f74f1dc043329},
+ {0xc0fe908895cf3b44, 0x505f522e53053ff3},
+ {0xf13e34aabb430a15, 0x647726b9e7c68ff0},
+ {0x96c6e0eab509e64d, 0x5eca783430dc19f6},
+ {0xbc789925624c5fe0, 0xb67d16413d132073},
+ {0xeb96bf6ebadf77d8, 0xe41c5bd18c57e890},
+ {0x933e37a534cbaae7, 0x8e91b962f7b6f15a},
+ {0xb80dc58e81fe95a1, 0x723627bbb5a4adb1},
+ {0xe61136f2227e3b09, 0xcec3b1aaa30dd91d},
+ {0x8fcac257558ee4e6, 0x213a4f0aa5e8a7b2},
+ {0xb3bd72ed2af29e1f, 0xa988e2cd4f62d19e},
+ {0xe0accfa875af45a7, 0x93eb1b80a33b8606},
+ {0x8c6c01c9498d8b88, 0xbc72f130660533c4},
+ {0xaf87023b9bf0ee6a, 0xeb8fad7c7f8680b5},
+ { 0xdb68c2ca82ed2a05,
+ 0xa67398db9f6820e2 }
+#else
+ {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b},
+ {0xce5d73ff402d98e3, 0xfb0a3d212dc81290},
+ {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f},
+ {0x86a8d39ef77164bc, 0xae5dff9c02033198},
+ {0xd98ddaee19068c76, 0x3badd624dd9b0958},
+ {0xafbd2350644eeacf, 0xe5d1929ef90898fb},
+ {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2},
+ {0xe55990879ddcaabd, 0xcc420a6a101d0516},
+ {0xb94470938fa89bce, 0xf808e40e8d5b3e6a},
+ {0x95a8637627989aad, 0xdde7001379a44aa9},
+ {0xf1c90080baf72cb1, 0x5324c68b12dd6339},
+ {0xc350000000000000, 0x0000000000000000},
+ {0x9dc5ada82b70b59d, 0xf020000000000000},
+ {0xfee50b7025c36a08, 0x02f236d04753d5b5},
+ {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87},
+ {0xa6539930bf6bff45, 0x84db8346b786151d},
+ {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3},
+ {0xd910f7ff28069da4, 0x1b2ba1518094da05},
+ {0xaf58416654a6babb, 0x387ac8d1970027b3},
+ {0x8da471a9de737e24, 0x5ceaecfed289e5d3},
+ {0xe4d5e82392a40515, 0x0fabaf3feaa5334b},
+ {0xb8da1662e7b00a17, 0x3d6a751f3b936244},
+ {0x95527a5202df0ccb, 0x0f37801e0c43ebc9},
+ {0xf13e34aabb430a15, 0x647726b9e7c68ff0}
+#endif
+ };
+
+#if FMT_USE_FULL_CACHE_DRAGONBOX
+ return pow10_significands[k - float_info<double>::min_k];
+#else
+ static constexpr const uint64_t powers_of_5_64[] = {
+ 0x0000000000000001, 0x0000000000000005, 0x0000000000000019,
+ 0x000000000000007d, 0x0000000000000271, 0x0000000000000c35,
+ 0x0000000000003d09, 0x000000000001312d, 0x000000000005f5e1,
+ 0x00000000001dcd65, 0x00000000009502f9, 0x0000000002e90edd,
+ 0x000000000e8d4a51, 0x0000000048c27395, 0x000000016bcc41e9,
+ 0x000000071afd498d, 0x0000002386f26fc1, 0x000000b1a2bc2ec5,
+ 0x000003782dace9d9, 0x00001158e460913d, 0x000056bc75e2d631,
+ 0x0001b1ae4d6e2ef5, 0x000878678326eac9, 0x002a5a058fc295ed,
+ 0x00d3c21bcecceda1, 0x0422ca8b0a00a425, 0x14adf4b7320334b9};
+
+ static const int compression_ratio = 27;
+
+ // Compute base index.
+ int cache_index = (k - float_info<double>::min_k) / compression_ratio;
+ int kb = cache_index * compression_ratio + float_info<double>::min_k;
+ int offset = k - kb;
+
+ // Get base cache.
+ uint128_fallback base_cache = pow10_significands[cache_index];
+ if (offset == 0) return base_cache;
+
+ // Compute the required amount of bit-shift.
+ int alpha = floor_log2_pow10(kb + offset) - floor_log2_pow10(kb) - offset;
+ FMT_ASSERT(alpha > 0 && alpha < 64, "shifting error detected");
+
+ // Try to recover the real cache.
+ uint64_t pow5 = powers_of_5_64[offset];
+ uint128_fallback recovered_cache = umul128(base_cache.high(), pow5);
+ uint128_fallback middle_low = umul128(base_cache.low(), pow5);
+
+ recovered_cache += middle_low.high();
+
+ uint64_t high_to_middle = recovered_cache.high() << (64 - alpha);
+ uint64_t middle_to_low = recovered_cache.low() << (64 - alpha);
+
+ recovered_cache =
+ uint128_fallback{(recovered_cache.low() >> alpha) | high_to_middle,
+ ((middle_low.low() >> alpha) | middle_to_low)};
+ FMT_ASSERT(recovered_cache.low() + 1 != 0, "");
+ return {recovered_cache.high(), recovered_cache.low() + 1};
+#endif
+ }
+
+ struct compute_mul_result {
+ carrier_uint result;
+ bool is_integer;
+ };
+ struct compute_mul_parity_result {
+ bool parity;
+ bool is_integer;
+ };
+
+ static compute_mul_result compute_mul(
+ carrier_uint u, const cache_entry_type& cache) noexcept {
+ auto r = umul192_upper128(u, cache);
+ return {r.high(), r.low() == 0};
+ }
+
+ static uint32_t compute_delta(cache_entry_type const& cache,
+ int beta) noexcept {
+ return static_cast<uint32_t>(cache.high() >> (64 - 1 - beta));
+ }
+
+ static compute_mul_parity_result compute_mul_parity(
+ carrier_uint two_f, const cache_entry_type& cache, int beta) noexcept {
+ FMT_ASSERT(beta >= 1, "");
+ FMT_ASSERT(beta < 64, "");
+
+ auto r = umul192_lower128(two_f, cache);
+ return {((r.high() >> (64 - beta)) & 1) != 0,
+ ((r.high() << beta) | (r.low() >> (64 - beta))) == 0};
+ }
+
+ static carrier_uint compute_left_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return (cache.high() -
+ (cache.high() >> (num_significand_bits<double>() + 2))) >>
+ (64 - num_significand_bits<double>() - 1 - beta);
+ }
+
+ static carrier_uint compute_right_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return (cache.high() +
+ (cache.high() >> (num_significand_bits<double>() + 1))) >>
+ (64 - num_significand_bits<double>() - 1 - beta);
+ }
+
+ static carrier_uint compute_round_up_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return ((cache.high() >> (64 - num_significand_bits<double>() - 2 - beta)) +
+ 1) /
+ 2;
+ }
+};
+
+FMT_FUNC uint128_fallback get_cached_power(int k) noexcept {
+ return cache_accessor<double>::get_cached_power(k);
+}
+
+// Various integer checks
+template <typename T>
+bool is_left_endpoint_integer_shorter_interval(int exponent) noexcept {
+ const int case_shorter_interval_left_endpoint_lower_threshold = 2;
+ const int case_shorter_interval_left_endpoint_upper_threshold = 3;
+ return exponent >= case_shorter_interval_left_endpoint_lower_threshold &&
+ exponent <= case_shorter_interval_left_endpoint_upper_threshold;
+}
+
+// Remove trailing zeros from n and return the number of zeros removed (float)
+FMT_INLINE int remove_trailing_zeros(uint32_t& n) noexcept {
+ FMT_ASSERT(n != 0, "");
+ // Modular inverse of 5 (mod 2^32): (mod_inv_5 * 5) mod 2^32 = 1.
+ // See https://github.com/fmtlib/fmt/issues/3163 for more details.
+ const uint32_t mod_inv_5 = 0xcccccccd;
+ // Casts are needed to workaround a bug in MSVC 19.22 and older.
+ const uint32_t mod_inv_25 =
+ static_cast<uint32_t>(uint64_t(mod_inv_5) * mod_inv_5);
+
+ int s = 0;
+ while (true) {
+ auto q = rotr(n * mod_inv_25, 2);
+ if (q > max_value<uint32_t>() / 100) break;
+ n = q;
+ s += 2;
+ }
+ auto q = rotr(n * mod_inv_5, 1);
+ if (q <= max_value<uint32_t>() / 10) {
+ n = q;
+ s |= 1;
+ }
+ return s;
+}
+
+// Removes trailing zeros and returns the number of zeros removed (double)
+FMT_INLINE int remove_trailing_zeros(uint64_t& n) noexcept {
+ FMT_ASSERT(n != 0, "");
+
+ // This magic number is ceil(2^90 / 10^8).
+ constexpr uint64_t magic_number = 12379400392853802749ull;
+ auto nm = umul128(n, magic_number);
+
+ // Is n is divisible by 10^8?
+ if ((nm.high() & ((1ull << (90 - 64)) - 1)) == 0 && nm.low() < magic_number) {
+ // If yes, work with the quotient.
+ auto n32 = static_cast<uint32_t>(nm.high() >> (90 - 64));
+
+ const uint32_t mod_inv_5 = 0xcccccccd;
+ const uint32_t mod_inv_25 = mod_inv_5 * mod_inv_5;
+
+ int s = 8;
+ while (true) {
+ auto q = rotr(n32 * mod_inv_25, 2);
+ if (q > max_value<uint32_t>() / 100) break;
+ n32 = q;
+ s += 2;
+ }
+ auto q = rotr(n32 * mod_inv_5, 1);
+ if (q <= max_value<uint32_t>() / 10) {
+ n32 = q;
+ s |= 1;
+ }
+
+ n = n32;
+ return s;
+ }
+
+ // If n is not divisible by 10^8, work with n itself.
+ const uint64_t mod_inv_5 = 0xcccccccccccccccd;
+ const uint64_t mod_inv_25 = mod_inv_5 * mod_inv_5;
+
+ int s = 0;
+ while (true) {
+ auto q = rotr(n * mod_inv_25, 2);
+ if (q > max_value<uint64_t>() / 100) break;
+ n = q;
+ s += 2;
+ }
+ auto q = rotr(n * mod_inv_5, 1);
+ if (q <= max_value<uint64_t>() / 10) {
+ n = q;
+ s |= 1;
+ }
+
+ return s;
+}
+
+// The main algorithm for shorter interval case
+template <typename T>
+FMT_INLINE decimal_fp<T> shorter_interval_case(int exponent) noexcept {
+ decimal_fp<T> ret_value;
+ // Compute k and beta
+ const int minus_k = floor_log10_pow2_minus_log10_4_over_3(exponent);
+ const int beta = exponent + floor_log2_pow10(-minus_k);
+
+ // Compute xi and zi
+ using cache_entry_type = typename cache_accessor<T>::cache_entry_type;
+ const cache_entry_type cache = cache_accessor<T>::get_cached_power(-minus_k);
+
+ auto xi = cache_accessor<T>::compute_left_endpoint_for_shorter_interval_case(
+ cache, beta);
+ auto zi = cache_accessor<T>::compute_right_endpoint_for_shorter_interval_case(
+ cache, beta);
+
+ // If the left endpoint is not an integer, increase it
+ if (!is_left_endpoint_integer_shorter_interval<T>(exponent)) ++xi;
+
+ // Try bigger divisor
+ ret_value.significand = zi / 10;
+
+ // If succeed, remove trailing zeros if necessary and return
+ if (ret_value.significand * 10 >= xi) {
+ ret_value.exponent = minus_k + 1;
+ ret_value.exponent += remove_trailing_zeros(ret_value.significand);
+ return ret_value;
+ }
+
+ // Otherwise, compute the round-up of y
+ ret_value.significand =
+ cache_accessor<T>::compute_round_up_for_shorter_interval_case(cache,
+ beta);
+ ret_value.exponent = minus_k;
+
+ // When tie occurs, choose one of them according to the rule
+ if (exponent >= float_info<T>::shorter_interval_tie_lower_threshold &&
+ exponent <= float_info<T>::shorter_interval_tie_upper_threshold) {
+ ret_value.significand = ret_value.significand % 2 == 0
+ ? ret_value.significand
+ : ret_value.significand - 1;
+ } else if (ret_value.significand < xi) {
+ ++ret_value.significand;
+ }
+ return ret_value;
+}
+
+template <typename T> decimal_fp<T> to_decimal(T x) noexcept {
+ // Step 1: integer promotion & Schubfach multiplier calculation.
+
+ using carrier_uint = typename float_info<T>::carrier_uint;
+ using cache_entry_type = typename cache_accessor<T>::cache_entry_type;
+ auto br = bit_cast<carrier_uint>(x);
+
+ // Extract significand bits and exponent bits.
+ const carrier_uint significand_mask =
+ (static_cast<carrier_uint>(1) << num_significand_bits<T>()) - 1;
+ carrier_uint significand = (br & significand_mask);
+ int exponent =
+ static_cast<int>((br & exponent_mask<T>()) >> num_significand_bits<T>());
+
+ if (exponent != 0) { // Check if normal.
+ exponent -= exponent_bias<T>() + num_significand_bits<T>();
+
+ // Shorter interval case; proceed like Schubfach.
+ // In fact, when exponent == 1 and significand == 0, the interval is
+ // regular. However, it can be shown that the end-results are anyway same.
+ if (significand == 0) return shorter_interval_case<T>(exponent);
+
+ significand |= (static_cast<carrier_uint>(1) << num_significand_bits<T>());
+ } else {
+ // Subnormal case; the interval is always regular.
+ if (significand == 0) return {0, 0};
+ exponent =
+ std::numeric_limits<T>::min_exponent - num_significand_bits<T>() - 1;
+ }
+
+ const bool include_left_endpoint = (significand % 2 == 0);
+ const bool include_right_endpoint = include_left_endpoint;
+
+ // Compute k and beta.
+ const int minus_k = floor_log10_pow2(exponent) - float_info<T>::kappa;
+ const cache_entry_type cache = cache_accessor<T>::get_cached_power(-minus_k);
+ const int beta = exponent + floor_log2_pow10(-minus_k);
+
+ // Compute zi and deltai.
+ // 10^kappa <= deltai < 10^(kappa + 1)
+ const uint32_t deltai = cache_accessor<T>::compute_delta(cache, beta);
+ const carrier_uint two_fc = significand << 1;
+
+ // For the case of binary32, the result of integer check is not correct for
+ // 29711844 * 2^-82
+ // = 6.1442653300000000008655037797566933477355632930994033813476... * 10^-18
+ // and 29711844 * 2^-81
+ // = 1.2288530660000000001731007559513386695471126586198806762695... * 10^-17,
+ // and they are the unique counterexamples. However, since 29711844 is even,
+ // this does not cause any problem for the endpoints calculations; it can only
+ // cause a problem when we need to perform integer check for the center.
+ // Fortunately, with these inputs, that branch is never executed, so we are
+ // fine.
+ const typename cache_accessor<T>::compute_mul_result z_mul =
+ cache_accessor<T>::compute_mul((two_fc | 1) << beta, cache);
+
+ // Step 2: Try larger divisor; remove trailing zeros if necessary.
+
+ // Using an upper bound on zi, we might be able to optimize the division
+ // better than the compiler; we are computing zi / big_divisor here.
+ decimal_fp<T> ret_value;
+ ret_value.significand = divide_by_10_to_kappa_plus_1(z_mul.result);
+ uint32_t r = static_cast<uint32_t>(z_mul.result - float_info<T>::big_divisor *
+ ret_value.significand);
+
+ if (r < deltai) {
+ // Exclude the right endpoint if necessary.
+ if (r == 0 && (z_mul.is_integer & !include_right_endpoint)) {
+ --ret_value.significand;
+ r = float_info<T>::big_divisor;
+ goto small_divisor_case_label;
+ }
+ } else if (r > deltai) {
+ goto small_divisor_case_label;
+ } else {
+ // r == deltai; compare fractional parts.
+ const typename cache_accessor<T>::compute_mul_parity_result x_mul =
+ cache_accessor<T>::compute_mul_parity(two_fc - 1, cache, beta);
+
+ if (!(x_mul.parity | (x_mul.is_integer & include_left_endpoint)))
+ goto small_divisor_case_label;
+ }
+ ret_value.exponent = minus_k + float_info<T>::kappa + 1;
+
+ // We may need to remove trailing zeros.
+ ret_value.exponent += remove_trailing_zeros(ret_value.significand);
+ return ret_value;
+
+ // Step 3: Find the significand with the smaller divisor.
+
+small_divisor_case_label:
+ ret_value.significand *= 10;
+ ret_value.exponent = minus_k + float_info<T>::kappa;
+
+ uint32_t dist = r - (deltai / 2) + (float_info<T>::small_divisor / 2);
+ const bool approx_y_parity =
+ ((dist ^ (float_info<T>::small_divisor / 2)) & 1) != 0;
+
+ // Is dist divisible by 10^kappa?
+ const bool divisible_by_small_divisor =
+ check_divisibility_and_divide_by_pow10<float_info<T>::kappa>(dist);
+
+ // Add dist / 10^kappa to the significand.
+ ret_value.significand += dist;
+
+ if (!divisible_by_small_divisor) return ret_value;
+
+ // Check z^(f) >= epsilon^(f).
+ // We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1,
+ // where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f).
+ // Since there are only 2 possibilities, we only need to care about the
+ // parity. Also, zi and r should have the same parity since the divisor
+ // is an even number.
+ const auto y_mul = cache_accessor<T>::compute_mul_parity(two_fc, cache, beta);
+
+ // If z^(f) >= epsilon^(f), we might have a tie when z^(f) == epsilon^(f),
+ // or equivalently, when y is an integer.
+ if (y_mul.parity != approx_y_parity)
+ --ret_value.significand;
+ else if (y_mul.is_integer & (ret_value.significand % 2 != 0))
+ --ret_value.significand;
+ return ret_value;
+}
+} // namespace dragonbox
+} // namespace detail
+
+template <> struct formatter<detail::bigint> {
+ FMT_CONSTEXPR auto parse(format_parse_context& ctx)
+ -> format_parse_context::iterator {
+ return ctx.begin();
+ }
+
+ auto format(const detail::bigint& n, format_context& ctx) const
+ -> format_context::iterator {
+ auto out = ctx.out();
+ bool first = true;
+ for (auto i = n.bigits_.size(); i > 0; --i) {
+ auto value = n.bigits_[i - 1u];
+ if (first) {
+ out = format_to(out, FMT_STRING("{:x}"), value);
+ first = false;
+ continue;
+ }
+ out = format_to(out, FMT_STRING("{:08x}"), value);
+ }
+ if (n.exp_ > 0)
+ out = format_to(out, FMT_STRING("p{}"),
+ n.exp_ * detail::bigint::bigit_bits);
+ return out;
+ }
+};
+
+FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) {
+ for_each_codepoint(s, [this](uint32_t cp, string_view) {
+ if (cp == invalid_code_point) FMT_THROW(std::runtime_error("invalid utf8"));
+ if (cp <= 0xFFFF) {
+ buffer_.push_back(static_cast<wchar_t>(cp));
+ } else {
+ cp -= 0x10000;
+ buffer_.push_back(static_cast<wchar_t>(0xD800 + (cp >> 10)));
+ buffer_.push_back(static_cast<wchar_t>(0xDC00 + (cp & 0x3FF)));
+ }
+ return true;
+ });
+ buffer_.push_back(0);
+}
+
+FMT_FUNC void format_system_error(detail::buffer<char>& out, int error_code,
+ const char* message) noexcept {
+ FMT_TRY {
+ auto ec = std::error_code(error_code, std::generic_category());
+ write(std::back_inserter(out), std::system_error(ec, message).what());
+ return;
+ }
+ FMT_CATCH(...) {}
+ format_error_code(out, error_code, message);
+}
+
+FMT_FUNC void report_system_error(int error_code,
+ const char* message) noexcept {
+ report_error(format_system_error, error_code, message);
+}
+
+FMT_FUNC std::string vformat(string_view fmt, format_args args) {
+ // Don't optimize the "{}" case to keep the binary size small and because it
+ // can be better optimized in fmt::format anyway.
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, fmt, args);
+ return to_string(buffer);
+}
+
+namespace detail {
+#ifndef _WIN32
+FMT_FUNC bool write_console(std::FILE*, string_view) { return false; }
+#else
+using dword = conditional_t<sizeof(long) == 4, unsigned long, unsigned>;
+extern "C" __declspec(dllimport) int __stdcall WriteConsoleW( //
+ void*, const void*, dword, dword*, void*);
+
+FMT_FUNC bool write_console(std::FILE* f, string_view text) {
+ auto fd = _fileno(f);
+ if (!_isatty(fd)) return false;
+ auto u16 = utf8_to_utf16(text);
+ auto written = dword();
+ return WriteConsoleW(reinterpret_cast<void*>(_get_osfhandle(fd)), u16.c_str(),
+ static_cast<uint32_t>(u16.size()), &written, nullptr);
+}
+
+// Print assuming legacy (non-Unicode) encoding.
+FMT_FUNC void vprint_mojibake(std::FILE* f, string_view fmt, format_args args) {
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, fmt,
+ basic_format_args<buffer_context<char>>(args));
+ fwrite_fully(buffer.data(), 1, buffer.size(), f);
+}
+#endif
+
+FMT_FUNC void print(std::FILE* f, string_view text) {
+ if (!write_console(f, text)) fwrite_fully(text.data(), 1, text.size(), f);
+}
+} // namespace detail
+
+FMT_FUNC void vprint(std::FILE* f, string_view fmt, format_args args) {
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, fmt, args);
+ detail::print(f, {buffer.data(), buffer.size()});
+}
+
+FMT_FUNC void vprint(string_view fmt, format_args args) {
+ vprint(stdout, fmt, args);
+}
+
+namespace detail {
+
+struct singleton {
+ unsigned char upper;
+ unsigned char lower_count;
+};
+
+inline auto is_printable(uint16_t x, const singleton* singletons,
+ size_t singletons_size,
+ const unsigned char* singleton_lowers,
+ const unsigned char* normal, size_t normal_size)
+ -> bool {
+ auto upper = x >> 8;
+ auto lower_start = 0;
+ for (size_t i = 0; i < singletons_size; ++i) {
+ auto s = singletons[i];
+ auto lower_end = lower_start + s.lower_count;
+ if (upper < s.upper) break;
+ if (upper == s.upper) {
+ for (auto j = lower_start; j < lower_end; ++j) {
+ if (singleton_lowers[j] == (x & 0xff)) return false;
+ }
+ }
+ lower_start = lower_end;
+ }
+
+ auto xsigned = static_cast<int>(x);
+ auto current = true;
+ for (size_t i = 0; i < normal_size; ++i) {
+ auto v = static_cast<int>(normal[i]);
+ auto len = (v & 0x80) != 0 ? (v & 0x7f) << 8 | normal[++i] : v;
+ xsigned -= len;
+ if (xsigned < 0) break;
+ current = !current;
+ }
+ return current;
+}
+
+// This code is generated by support/printable.py.
+FMT_FUNC auto is_printable(uint32_t cp) -> bool {
+ static constexpr singleton singletons0[] = {
+ {0x00, 1}, {0x03, 5}, {0x05, 6}, {0x06, 3}, {0x07, 6}, {0x08, 8},
+ {0x09, 17}, {0x0a, 28}, {0x0b, 25}, {0x0c, 20}, {0x0d, 16}, {0x0e, 13},
+ {0x0f, 4}, {0x10, 3}, {0x12, 18}, {0x13, 9}, {0x16, 1}, {0x17, 5},
+ {0x18, 2}, {0x19, 3}, {0x1a, 7}, {0x1c, 2}, {0x1d, 1}, {0x1f, 22},
+ {0x20, 3}, {0x2b, 3}, {0x2c, 2}, {0x2d, 11}, {0x2e, 1}, {0x30, 3},
+ {0x31, 2}, {0x32, 1}, {0xa7, 2}, {0xa9, 2}, {0xaa, 4}, {0xab, 8},
+ {0xfa, 2}, {0xfb, 5}, {0xfd, 4}, {0xfe, 3}, {0xff, 9},
+ };
+ static constexpr unsigned char singletons0_lower[] = {
+ 0xad, 0x78, 0x79, 0x8b, 0x8d, 0xa2, 0x30, 0x57, 0x58, 0x8b, 0x8c, 0x90,
+ 0x1c, 0x1d, 0xdd, 0x0e, 0x0f, 0x4b, 0x4c, 0xfb, 0xfc, 0x2e, 0x2f, 0x3f,
+ 0x5c, 0x5d, 0x5f, 0xb5, 0xe2, 0x84, 0x8d, 0x8e, 0x91, 0x92, 0xa9, 0xb1,
+ 0xba, 0xbb, 0xc5, 0xc6, 0xc9, 0xca, 0xde, 0xe4, 0xe5, 0xff, 0x00, 0x04,
+ 0x11, 0x12, 0x29, 0x31, 0x34, 0x37, 0x3a, 0x3b, 0x3d, 0x49, 0x4a, 0x5d,
+ 0x84, 0x8e, 0x92, 0xa9, 0xb1, 0xb4, 0xba, 0xbb, 0xc6, 0xca, 0xce, 0xcf,
+ 0xe4, 0xe5, 0x00, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+ 0x3b, 0x45, 0x46, 0x49, 0x4a, 0x5e, 0x64, 0x65, 0x84, 0x91, 0x9b, 0x9d,
+ 0xc9, 0xce, 0xcf, 0x0d, 0x11, 0x29, 0x45, 0x49, 0x57, 0x64, 0x65, 0x8d,
+ 0x91, 0xa9, 0xb4, 0xba, 0xbb, 0xc5, 0xc9, 0xdf, 0xe4, 0xe5, 0xf0, 0x0d,
+ 0x11, 0x45, 0x49, 0x64, 0x65, 0x80, 0x84, 0xb2, 0xbc, 0xbe, 0xbf, 0xd5,
+ 0xd7, 0xf0, 0xf1, 0x83, 0x85, 0x8b, 0xa4, 0xa6, 0xbe, 0xbf, 0xc5, 0xc7,
+ 0xce, 0xcf, 0xda, 0xdb, 0x48, 0x98, 0xbd, 0xcd, 0xc6, 0xce, 0xcf, 0x49,
+ 0x4e, 0x4f, 0x57, 0x59, 0x5e, 0x5f, 0x89, 0x8e, 0x8f, 0xb1, 0xb6, 0xb7,
+ 0xbf, 0xc1, 0xc6, 0xc7, 0xd7, 0x11, 0x16, 0x17, 0x5b, 0x5c, 0xf6, 0xf7,
+ 0xfe, 0xff, 0x80, 0x0d, 0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x0f, 0x1f, 0x6e,
+ 0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e, 0xae, 0xaf, 0xbb, 0xbc, 0xfa, 0x16,
+ 0x17, 0x1e, 0x1f, 0x46, 0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c, 0x5e, 0x7e,
+ 0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc, 0xf0, 0xf1, 0xf5, 0x72, 0x73, 0x8f,
+ 0x74, 0x75, 0x96, 0x2f, 0x5f, 0x26, 0x2e, 0x2f, 0xa7, 0xaf, 0xb7, 0xbf,
+ 0xc7, 0xcf, 0xd7, 0xdf, 0x9a, 0x40, 0x97, 0x98, 0x30, 0x8f, 0x1f, 0xc0,
+ 0xc1, 0xce, 0xff, 0x4e, 0x4f, 0x5a, 0x5b, 0x07, 0x08, 0x0f, 0x10, 0x27,
+ 0x2f, 0xee, 0xef, 0x6e, 0x6f, 0x37, 0x3d, 0x3f, 0x42, 0x45, 0x90, 0x91,
+ 0xfe, 0xff, 0x53, 0x67, 0x75, 0xc8, 0xc9, 0xd0, 0xd1, 0xd8, 0xd9, 0xe7,
+ 0xfe, 0xff,
+ };
+ static constexpr singleton singletons1[] = {
+ {0x00, 6}, {0x01, 1}, {0x03, 1}, {0x04, 2}, {0x08, 8}, {0x09, 2},
+ {0x0a, 5}, {0x0b, 2}, {0x0e, 4}, {0x10, 1}, {0x11, 2}, {0x12, 5},
+ {0x13, 17}, {0x14, 1}, {0x15, 2}, {0x17, 2}, {0x19, 13}, {0x1c, 5},
+ {0x1d, 8}, {0x24, 1}, {0x6a, 3}, {0x6b, 2}, {0xbc, 2}, {0xd1, 2},
+ {0xd4, 12}, {0xd5, 9}, {0xd6, 2}, {0xd7, 2}, {0xda, 1}, {0xe0, 5},
+ {0xe1, 2}, {0xe8, 2}, {0xee, 32}, {0xf0, 4}, {0xf8, 2}, {0xf9, 2},
+ {0xfa, 2}, {0xfb, 1},
+ };
+ static constexpr unsigned char singletons1_lower[] = {
+ 0x0c, 0x27, 0x3b, 0x3e, 0x4e, 0x4f, 0x8f, 0x9e, 0x9e, 0x9f, 0x06, 0x07,
+ 0x09, 0x36, 0x3d, 0x3e, 0x56, 0xf3, 0xd0, 0xd1, 0x04, 0x14, 0x18, 0x36,
+ 0x37, 0x56, 0x57, 0x7f, 0xaa, 0xae, 0xaf, 0xbd, 0x35, 0xe0, 0x12, 0x87,
+ 0x89, 0x8e, 0x9e, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+ 0x45, 0x46, 0x49, 0x4a, 0x4e, 0x4f, 0x64, 0x65, 0x5c, 0xb6, 0xb7, 0x1b,
+ 0x1c, 0x07, 0x08, 0x0a, 0x0b, 0x14, 0x17, 0x36, 0x39, 0x3a, 0xa8, 0xa9,
+ 0xd8, 0xd9, 0x09, 0x37, 0x90, 0x91, 0xa8, 0x07, 0x0a, 0x3b, 0x3e, 0x66,
+ 0x69, 0x8f, 0x92, 0x6f, 0x5f, 0xee, 0xef, 0x5a, 0x62, 0x9a, 0x9b, 0x27,
+ 0x28, 0x55, 0x9d, 0xa0, 0xa1, 0xa3, 0xa4, 0xa7, 0xa8, 0xad, 0xba, 0xbc,
+ 0xc4, 0x06, 0x0b, 0x0c, 0x15, 0x1d, 0x3a, 0x3f, 0x45, 0x51, 0xa6, 0xa7,
+ 0xcc, 0xcd, 0xa0, 0x07, 0x19, 0x1a, 0x22, 0x25, 0x3e, 0x3f, 0xc5, 0xc6,
+ 0x04, 0x20, 0x23, 0x25, 0x26, 0x28, 0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c,
+ 0x50, 0x53, 0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e, 0x60, 0x63, 0x65, 0x66,
+ 0x6b, 0x73, 0x78, 0x7d, 0x7f, 0x8a, 0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0,
+ 0xae, 0xaf, 0x79, 0xcc, 0x6e, 0x6f, 0x93,
+ };
+ static constexpr unsigned char normal0[] = {
+ 0x00, 0x20, 0x5f, 0x22, 0x82, 0xdf, 0x04, 0x82, 0x44, 0x08, 0x1b, 0x04,
+ 0x06, 0x11, 0x81, 0xac, 0x0e, 0x80, 0xab, 0x35, 0x28, 0x0b, 0x80, 0xe0,
+ 0x03, 0x19, 0x08, 0x01, 0x04, 0x2f, 0x04, 0x34, 0x04, 0x07, 0x03, 0x01,
+ 0x07, 0x06, 0x07, 0x11, 0x0a, 0x50, 0x0f, 0x12, 0x07, 0x55, 0x07, 0x03,
+ 0x04, 0x1c, 0x0a, 0x09, 0x03, 0x08, 0x03, 0x07, 0x03, 0x02, 0x03, 0x03,
+ 0x03, 0x0c, 0x04, 0x05, 0x03, 0x0b, 0x06, 0x01, 0x0e, 0x15, 0x05, 0x3a,
+ 0x03, 0x11, 0x07, 0x06, 0x05, 0x10, 0x07, 0x57, 0x07, 0x02, 0x07, 0x15,
+ 0x0d, 0x50, 0x04, 0x43, 0x03, 0x2d, 0x03, 0x01, 0x04, 0x11, 0x06, 0x0f,
+ 0x0c, 0x3a, 0x04, 0x1d, 0x25, 0x5f, 0x20, 0x6d, 0x04, 0x6a, 0x25, 0x80,
+ 0xc8, 0x05, 0x82, 0xb0, 0x03, 0x1a, 0x06, 0x82, 0xfd, 0x03, 0x59, 0x07,
+ 0x15, 0x0b, 0x17, 0x09, 0x14, 0x0c, 0x14, 0x0c, 0x6a, 0x06, 0x0a, 0x06,
+ 0x1a, 0x06, 0x59, 0x07, 0x2b, 0x05, 0x46, 0x0a, 0x2c, 0x04, 0x0c, 0x04,
+ 0x01, 0x03, 0x31, 0x0b, 0x2c, 0x04, 0x1a, 0x06, 0x0b, 0x03, 0x80, 0xac,
+ 0x06, 0x0a, 0x06, 0x21, 0x3f, 0x4c, 0x04, 0x2d, 0x03, 0x74, 0x08, 0x3c,
+ 0x03, 0x0f, 0x03, 0x3c, 0x07, 0x38, 0x08, 0x2b, 0x05, 0x82, 0xff, 0x11,
+ 0x18, 0x08, 0x2f, 0x11, 0x2d, 0x03, 0x20, 0x10, 0x21, 0x0f, 0x80, 0x8c,
+ 0x04, 0x82, 0x97, 0x19, 0x0b, 0x15, 0x88, 0x94, 0x05, 0x2f, 0x05, 0x3b,
+ 0x07, 0x02, 0x0e, 0x18, 0x09, 0x80, 0xb3, 0x2d, 0x74, 0x0c, 0x80, 0xd6,
+ 0x1a, 0x0c, 0x05, 0x80, 0xff, 0x05, 0x80, 0xdf, 0x0c, 0xee, 0x0d, 0x03,
+ 0x84, 0x8d, 0x03, 0x37, 0x09, 0x81, 0x5c, 0x14, 0x80, 0xb8, 0x08, 0x80,
+ 0xcb, 0x2a, 0x38, 0x03, 0x0a, 0x06, 0x38, 0x08, 0x46, 0x08, 0x0c, 0x06,
+ 0x74, 0x0b, 0x1e, 0x03, 0x5a, 0x04, 0x59, 0x09, 0x80, 0x83, 0x18, 0x1c,
+ 0x0a, 0x16, 0x09, 0x4c, 0x04, 0x80, 0x8a, 0x06, 0xab, 0xa4, 0x0c, 0x17,
+ 0x04, 0x31, 0xa1, 0x04, 0x81, 0xda, 0x26, 0x07, 0x0c, 0x05, 0x05, 0x80,
+ 0xa5, 0x11, 0x81, 0x6d, 0x10, 0x78, 0x28, 0x2a, 0x06, 0x4c, 0x04, 0x80,
+ 0x8d, 0x04, 0x80, 0xbe, 0x03, 0x1b, 0x03, 0x0f, 0x0d,
+ };
+ static constexpr unsigned char normal1[] = {
+ 0x5e, 0x22, 0x7b, 0x05, 0x03, 0x04, 0x2d, 0x03, 0x66, 0x03, 0x01, 0x2f,
+ 0x2e, 0x80, 0x82, 0x1d, 0x03, 0x31, 0x0f, 0x1c, 0x04, 0x24, 0x09, 0x1e,
+ 0x05, 0x2b, 0x05, 0x44, 0x04, 0x0e, 0x2a, 0x80, 0xaa, 0x06, 0x24, 0x04,
+ 0x24, 0x04, 0x28, 0x08, 0x34, 0x0b, 0x01, 0x80, 0x90, 0x81, 0x37, 0x09,
+ 0x16, 0x0a, 0x08, 0x80, 0x98, 0x39, 0x03, 0x63, 0x08, 0x09, 0x30, 0x16,
+ 0x05, 0x21, 0x03, 0x1b, 0x05, 0x01, 0x40, 0x38, 0x04, 0x4b, 0x05, 0x2f,
+ 0x04, 0x0a, 0x07, 0x09, 0x07, 0x40, 0x20, 0x27, 0x04, 0x0c, 0x09, 0x36,
+ 0x03, 0x3a, 0x05, 0x1a, 0x07, 0x04, 0x0c, 0x07, 0x50, 0x49, 0x37, 0x33,
+ 0x0d, 0x33, 0x07, 0x2e, 0x08, 0x0a, 0x81, 0x26, 0x52, 0x4e, 0x28, 0x08,
+ 0x2a, 0x56, 0x1c, 0x14, 0x17, 0x09, 0x4e, 0x04, 0x1e, 0x0f, 0x43, 0x0e,
+ 0x19, 0x07, 0x0a, 0x06, 0x48, 0x08, 0x27, 0x09, 0x75, 0x0b, 0x3f, 0x41,
+ 0x2a, 0x06, 0x3b, 0x05, 0x0a, 0x06, 0x51, 0x06, 0x01, 0x05, 0x10, 0x03,
+ 0x05, 0x80, 0x8b, 0x62, 0x1e, 0x48, 0x08, 0x0a, 0x80, 0xa6, 0x5e, 0x22,
+ 0x45, 0x0b, 0x0a, 0x06, 0x0d, 0x13, 0x39, 0x07, 0x0a, 0x36, 0x2c, 0x04,
+ 0x10, 0x80, 0xc0, 0x3c, 0x64, 0x53, 0x0c, 0x48, 0x09, 0x0a, 0x46, 0x45,
+ 0x1b, 0x48, 0x08, 0x53, 0x1d, 0x39, 0x81, 0x07, 0x46, 0x0a, 0x1d, 0x03,
+ 0x47, 0x49, 0x37, 0x03, 0x0e, 0x08, 0x0a, 0x06, 0x39, 0x07, 0x0a, 0x81,
+ 0x36, 0x19, 0x80, 0xb7, 0x01, 0x0f, 0x32, 0x0d, 0x83, 0x9b, 0x66, 0x75,
+ 0x0b, 0x80, 0xc4, 0x8a, 0xbc, 0x84, 0x2f, 0x8f, 0xd1, 0x82, 0x47, 0xa1,
+ 0xb9, 0x82, 0x39, 0x07, 0x2a, 0x04, 0x02, 0x60, 0x26, 0x0a, 0x46, 0x0a,
+ 0x28, 0x05, 0x13, 0x82, 0xb0, 0x5b, 0x65, 0x4b, 0x04, 0x39, 0x07, 0x11,
+ 0x40, 0x05, 0x0b, 0x02, 0x0e, 0x97, 0xf8, 0x08, 0x84, 0xd6, 0x2a, 0x09,
+ 0xa2, 0xf7, 0x81, 0x1f, 0x31, 0x03, 0x11, 0x04, 0x08, 0x81, 0x8c, 0x89,
+ 0x04, 0x6b, 0x05, 0x0d, 0x03, 0x09, 0x07, 0x10, 0x93, 0x60, 0x80, 0xf6,
+ 0x0a, 0x73, 0x08, 0x6e, 0x17, 0x46, 0x80, 0x9a, 0x14, 0x0c, 0x57, 0x09,
+ 0x19, 0x80, 0x87, 0x81, 0x47, 0x03, 0x85, 0x42, 0x0f, 0x15, 0x85, 0x50,
+ 0x2b, 0x80, 0xd5, 0x2d, 0x03, 0x1a, 0x04, 0x02, 0x81, 0x70, 0x3a, 0x05,
+ 0x01, 0x85, 0x00, 0x80, 0xd7, 0x29, 0x4c, 0x04, 0x0a, 0x04, 0x02, 0x83,
+ 0x11, 0x44, 0x4c, 0x3d, 0x80, 0xc2, 0x3c, 0x06, 0x01, 0x04, 0x55, 0x05,
+ 0x1b, 0x34, 0x02, 0x81, 0x0e, 0x2c, 0x04, 0x64, 0x0c, 0x56, 0x0a, 0x80,
+ 0xae, 0x38, 0x1d, 0x0d, 0x2c, 0x04, 0x09, 0x07, 0x02, 0x0e, 0x06, 0x80,
+ 0x9a, 0x83, 0xd8, 0x08, 0x0d, 0x03, 0x0d, 0x03, 0x74, 0x0c, 0x59, 0x07,
+ 0x0c, 0x14, 0x0c, 0x04, 0x38, 0x08, 0x0a, 0x06, 0x28, 0x08, 0x22, 0x4e,
+ 0x81, 0x54, 0x0c, 0x15, 0x03, 0x03, 0x05, 0x07, 0x09, 0x19, 0x07, 0x07,
+ 0x09, 0x03, 0x0d, 0x07, 0x29, 0x80, 0xcb, 0x25, 0x0a, 0x84, 0x06,
+ };
+ auto lower = static_cast<uint16_t>(cp);
+ if (cp < 0x10000) {
+ return is_printable(lower, singletons0,
+ sizeof(singletons0) / sizeof(*singletons0),
+ singletons0_lower, normal0, sizeof(normal0));
+ }
+ if (cp < 0x20000) {
+ return is_printable(lower, singletons1,
+ sizeof(singletons1) / sizeof(*singletons1),
+ singletons1_lower, normal1, sizeof(normal1));
+ }
+ if (0x2a6de <= cp && cp < 0x2a700) return false;
+ if (0x2b735 <= cp && cp < 0x2b740) return false;
+ if (0x2b81e <= cp && cp < 0x2b820) return false;
+ if (0x2cea2 <= cp && cp < 0x2ceb0) return false;
+ if (0x2ebe1 <= cp && cp < 0x2f800) return false;
+ if (0x2fa1e <= cp && cp < 0x30000) return false;
+ if (0x3134b <= cp && cp < 0xe0100) return false;
+ if (0xe01f0 <= cp && cp < 0x110000) return false;
+ return cp < 0x110000;
+}
+
+} // namespace detail
+
+FMT_END_NAMESPACE
+
+#endif // FMT_FORMAT_INL_H_
diff --git a/contrib/fmt/include/fmt/format.h b/contrib/fmt/include/fmt/format.h
new file mode 100644
index 0000000..ed8b29e
--- /dev/null
+++ b/contrib/fmt/include/fmt/format.h
@@ -0,0 +1,4735 @@
+/*
+ Formatting library for C++
+
+ Copyright (c) 2012 - present, Victor Zverovich
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ --- Optional exception to the license ---
+
+ As an exception, if, as a result of your compiling your source code, portions
+ of this Software are embedded into a machine-executable object form of such
+ source code, you may redistribute such embedded portions in such object form
+ without including the above copyright and permission notices.
+ */
+
+#ifndef FMT_FORMAT_H_
+#define FMT_FORMAT_H_
+
+#include <cmath> // std::signbit
+#include <cstdint> // uint32_t
+#include <cstring> // std::memcpy
+#include <initializer_list> // std::initializer_list
+#include <limits> // std::numeric_limits
+#include <memory> // std::uninitialized_copy
+#include <stdexcept> // std::runtime_error
+#include <system_error> // std::system_error
+
+#ifdef __cpp_lib_bit_cast
+# include <bit> // std::bitcast
+#endif
+
+#include "core.h"
+
+#ifndef FMT_BEGIN_DETAIL_NAMESPACE
+# define FMT_BEGIN_DETAIL_NAMESPACE namespace detail {
+# define FMT_END_DETAIL_NAMESPACE }
+#endif
+
+#if FMT_HAS_CPP17_ATTRIBUTE(fallthrough)
+# define FMT_FALLTHROUGH [[fallthrough]]
+#elif defined(__clang__)
+# define FMT_FALLTHROUGH [[clang::fallthrough]]
+#elif FMT_GCC_VERSION >= 700 && \
+ (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520)
+# define FMT_FALLTHROUGH [[gnu::fallthrough]]
+#else
+# define FMT_FALLTHROUGH
+#endif
+
+#ifndef FMT_DEPRECATED
+# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VERSION >= 1900
+# define FMT_DEPRECATED [[deprecated]]
+# else
+# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__)
+# define FMT_DEPRECATED __attribute__((deprecated))
+# elif FMT_MSC_VERSION
+# define FMT_DEPRECATED __declspec(deprecated)
+# else
+# define FMT_DEPRECATED /* deprecated */
+# endif
+# endif
+#endif
+
+#if FMT_GCC_VERSION
+# define FMT_GCC_VISIBILITY_HIDDEN __attribute__((visibility("hidden")))
+#else
+# define FMT_GCC_VISIBILITY_HIDDEN
+#endif
+
+#ifdef __NVCC__
+# define FMT_CUDA_VERSION (__CUDACC_VER_MAJOR__ * 100 + __CUDACC_VER_MINOR__)
+#else
+# define FMT_CUDA_VERSION 0
+#endif
+
+#ifdef __has_builtin
+# define FMT_HAS_BUILTIN(x) __has_builtin(x)
+#else
+# define FMT_HAS_BUILTIN(x) 0
+#endif
+
+#if FMT_GCC_VERSION || FMT_CLANG_VERSION
+# define FMT_NOINLINE __attribute__((noinline))
+#else
+# define FMT_NOINLINE
+#endif
+
+#ifndef FMT_THROW
+# if FMT_EXCEPTIONS
+# if FMT_MSC_VERSION || defined(__NVCC__)
+FMT_BEGIN_NAMESPACE
+namespace detail {
+template <typename Exception> inline void do_throw(const Exception& x) {
+ // Silence unreachable code warnings in MSVC and NVCC because these
+ // are nearly impossible to fix in a generic code.
+ volatile bool b = true;
+ if (b) throw x;
+}
+} // namespace detail
+FMT_END_NAMESPACE
+# define FMT_THROW(x) detail::do_throw(x)
+# else
+# define FMT_THROW(x) throw x
+# endif
+# else
+# define FMT_THROW(x) \
+ do { \
+ FMT_ASSERT(false, (x).what()); \
+ } while (false)
+# endif
+#endif
+
+#if FMT_EXCEPTIONS
+# define FMT_TRY try
+# define FMT_CATCH(x) catch (x)
+#else
+# define FMT_TRY if (true)
+# define FMT_CATCH(x) if (false)
+#endif
+
+#ifndef FMT_MAYBE_UNUSED
+# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused)
+# define FMT_MAYBE_UNUSED [[maybe_unused]]
+# else
+# define FMT_MAYBE_UNUSED
+# endif
+#endif
+
+#ifndef FMT_USE_USER_DEFINED_LITERALS
+// EDG based compilers (Intel, NVIDIA, Elbrus, etc), GCC and MSVC support UDLs.
+# if (FMT_HAS_FEATURE(cxx_user_literals) || FMT_GCC_VERSION >= 407 || \
+ FMT_MSC_VERSION >= 1900) && \
+ (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= /* UDL feature */ 480)
+# define FMT_USE_USER_DEFINED_LITERALS 1
+# else
+# define FMT_USE_USER_DEFINED_LITERALS 0
+# endif
+#endif
+
+// Defining FMT_REDUCE_INT_INSTANTIATIONS to 1, will reduce the number of
+// integer formatter template instantiations to just one by only using the
+// largest integer type. This results in a reduction in binary size but will
+// cause a decrease in integer formatting performance.
+#if !defined(FMT_REDUCE_INT_INSTANTIATIONS)
+# define FMT_REDUCE_INT_INSTANTIATIONS 0
+#endif
+
+// __builtin_clz is broken in clang with Microsoft CodeGen:
+// https://github.com/fmtlib/fmt/issues/519.
+#if !FMT_MSC_VERSION
+# if FMT_HAS_BUILTIN(__builtin_clz) || FMT_GCC_VERSION || FMT_ICC_VERSION
+# define FMT_BUILTIN_CLZ(n) __builtin_clz(n)
+# endif
+# if FMT_HAS_BUILTIN(__builtin_clzll) || FMT_GCC_VERSION || FMT_ICC_VERSION
+# define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n)
+# endif
+#endif
+
+// __builtin_ctz is broken in Intel Compiler Classic on Windows:
+// https://github.com/fmtlib/fmt/issues/2510.
+#ifndef __ICL
+# if FMT_HAS_BUILTIN(__builtin_ctz) || FMT_GCC_VERSION || FMT_ICC_VERSION || \
+ defined(__NVCOMPILER)
+# define FMT_BUILTIN_CTZ(n) __builtin_ctz(n)
+# endif
+# if FMT_HAS_BUILTIN(__builtin_ctzll) || FMT_GCC_VERSION || \
+ FMT_ICC_VERSION || defined(__NVCOMPILER)
+# define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n)
+# endif
+#endif
+
+#if FMT_MSC_VERSION
+# include <intrin.h> // _BitScanReverse[64], _BitScanForward[64], _umul128
+#endif
+
+// Some compilers masquerade as both MSVC and GCC-likes or otherwise support
+// __builtin_clz and __builtin_clzll, so only define FMT_BUILTIN_CLZ using the
+// MSVC intrinsics if the clz and clzll builtins are not available.
+#if FMT_MSC_VERSION && !defined(FMT_BUILTIN_CLZLL) && \
+ !defined(FMT_BUILTIN_CTZLL)
+FMT_BEGIN_NAMESPACE
+namespace detail {
+// Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning.
+# if !defined(__clang__)
+# pragma intrinsic(_BitScanForward)
+# pragma intrinsic(_BitScanReverse)
+# if defined(_WIN64)
+# pragma intrinsic(_BitScanForward64)
+# pragma intrinsic(_BitScanReverse64)
+# endif
+# endif
+
+inline auto clz(uint32_t x) -> int {
+ unsigned long r = 0;
+ _BitScanReverse(&r, x);
+ FMT_ASSERT(x != 0, "");
+ // Static analysis complains about using uninitialized data
+ // "r", but the only way that can happen is if "x" is 0,
+ // which the callers guarantee to not happen.
+ FMT_MSC_WARNING(suppress : 6102)
+ return 31 ^ static_cast<int>(r);
+}
+# define FMT_BUILTIN_CLZ(n) detail::clz(n)
+
+inline auto clzll(uint64_t x) -> int {
+ unsigned long r = 0;
+# ifdef _WIN64
+ _BitScanReverse64(&r, x);
+# else
+ // Scan the high 32 bits.
+ if (_BitScanReverse(&r, static_cast<uint32_t>(x >> 32)))
+ return 63 ^ static_cast<int>(r + 32);
+ // Scan the low 32 bits.
+ _BitScanReverse(&r, static_cast<uint32_t>(x));
+# endif
+ FMT_ASSERT(x != 0, "");
+ FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning.
+ return 63 ^ static_cast<int>(r);
+}
+# define FMT_BUILTIN_CLZLL(n) detail::clzll(n)
+
+inline auto ctz(uint32_t x) -> int {
+ unsigned long r = 0;
+ _BitScanForward(&r, x);
+ FMT_ASSERT(x != 0, "");
+ FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning.
+ return static_cast<int>(r);
+}
+# define FMT_BUILTIN_CTZ(n) detail::ctz(n)
+
+inline auto ctzll(uint64_t x) -> int {
+ unsigned long r = 0;
+ FMT_ASSERT(x != 0, "");
+ FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning.
+# ifdef _WIN64
+ _BitScanForward64(&r, x);
+# else
+ // Scan the low 32 bits.
+ if (_BitScanForward(&r, static_cast<uint32_t>(x))) return static_cast<int>(r);
+ // Scan the high 32 bits.
+ _BitScanForward(&r, static_cast<uint32_t>(x >> 32));
+ r += 32;
+# endif
+ return static_cast<int>(r);
+}
+# define FMT_BUILTIN_CTZLL(n) detail::ctzll(n)
+} // namespace detail
+FMT_END_NAMESPACE
+#endif
+
+FMT_BEGIN_NAMESPACE
+
+template <typename...> struct disjunction : std::false_type {};
+template <typename P> struct disjunction<P> : P {};
+template <typename P1, typename... Pn>
+struct disjunction<P1, Pn...>
+ : conditional_t<bool(P1::value), P1, disjunction<Pn...>> {};
+
+template <typename...> struct conjunction : std::true_type {};
+template <typename P> struct conjunction<P> : P {};
+template <typename P1, typename... Pn>
+struct conjunction<P1, Pn...>
+ : conditional_t<bool(P1::value), conjunction<Pn...>, P1> {};
+
+namespace detail {
+
+FMT_CONSTEXPR inline void abort_fuzzing_if(bool condition) {
+ ignore_unused(condition);
+#ifdef FMT_FUZZ
+ if (condition) throw std::runtime_error("fuzzing limit reached");
+#endif
+}
+
+template <typename CharT, CharT... C> struct string_literal {
+ static constexpr CharT value[sizeof...(C)] = {C...};
+ constexpr operator basic_string_view<CharT>() const {
+ return {value, sizeof...(C)};
+ }
+};
+
+#if FMT_CPLUSPLUS < 201703L
+template <typename CharT, CharT... C>
+constexpr CharT string_literal<CharT, C...>::value[sizeof...(C)];
+#endif
+
+template <typename Streambuf> class formatbuf : public Streambuf {
+ private:
+ using char_type = typename Streambuf::char_type;
+ using streamsize = decltype(std::declval<Streambuf>().sputn(nullptr, 0));
+ using int_type = typename Streambuf::int_type;
+ using traits_type = typename Streambuf::traits_type;
+
+ buffer<char_type>& buffer_;
+
+ public:
+ explicit formatbuf(buffer<char_type>& buf) : buffer_(buf) {}
+
+ protected:
+ // The put area is always empty. This makes the implementation simpler and has
+ // the advantage that the streambuf and the buffer are always in sync and
+ // sputc never writes into uninitialized memory. A disadvantage is that each
+ // call to sputc always results in a (virtual) call to overflow. There is no
+ // disadvantage here for sputn since this always results in a call to xsputn.
+
+ auto overflow(int_type ch) -> int_type override {
+ if (!traits_type::eq_int_type(ch, traits_type::eof()))
+ buffer_.push_back(static_cast<char_type>(ch));
+ return ch;
+ }
+
+ auto xsputn(const char_type* s, streamsize count) -> streamsize override {
+ buffer_.append(s, s + count);
+ return count;
+ }
+};
+
+// Implementation of std::bit_cast for pre-C++20.
+template <typename To, typename From, FMT_ENABLE_IF(sizeof(To) == sizeof(From))>
+FMT_CONSTEXPR20 auto bit_cast(const From& from) -> To {
+#ifdef __cpp_lib_bit_cast
+ if (is_constant_evaluated()) return std::bit_cast<To>(from);
+#endif
+ auto to = To();
+ // The cast suppresses a bogus -Wclass-memaccess on GCC.
+ std::memcpy(static_cast<void*>(&to), &from, sizeof(to));
+ return to;
+}
+
+inline auto is_big_endian() -> bool {
+#ifdef _WIN32
+ return false;
+#elif defined(__BIG_ENDIAN__)
+ return true;
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__)
+ return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__;
+#else
+ struct bytes {
+ char data[sizeof(int)];
+ };
+ return bit_cast<bytes>(1).data[0] == 0;
+#endif
+}
+
+class uint128_fallback {
+ private:
+ uint64_t lo_, hi_;
+
+ friend uint128_fallback umul128(uint64_t x, uint64_t y) noexcept;
+
+ public:
+ constexpr uint128_fallback(uint64_t hi, uint64_t lo) : lo_(lo), hi_(hi) {}
+ constexpr uint128_fallback(uint64_t value = 0) : lo_(value), hi_(0) {}
+
+ constexpr uint64_t high() const noexcept { return hi_; }
+ constexpr uint64_t low() const noexcept { return lo_; }
+
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ constexpr explicit operator T() const {
+ return static_cast<T>(lo_);
+ }
+
+ friend constexpr auto operator==(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> bool {
+ return lhs.hi_ == rhs.hi_ && lhs.lo_ == rhs.lo_;
+ }
+ friend constexpr auto operator!=(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> bool {
+ return !(lhs == rhs);
+ }
+ friend constexpr auto operator>(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> bool {
+ return lhs.hi_ != rhs.hi_ ? lhs.hi_ > rhs.hi_ : lhs.lo_ > rhs.lo_;
+ }
+ friend constexpr auto operator|(const uint128_fallback& lhs,
+ const uint128_fallback& rhs)
+ -> uint128_fallback {
+ return {lhs.hi_ | rhs.hi_, lhs.lo_ | rhs.lo_};
+ }
+ friend constexpr auto operator&(const uint128_fallback& lhs,
+ const uint128_fallback& rhs)
+ -> uint128_fallback {
+ return {lhs.hi_ & rhs.hi_, lhs.lo_ & rhs.lo_};
+ }
+ friend constexpr auto operator~(const uint128_fallback& n)
+ -> uint128_fallback {
+ return {~n.hi_, ~n.lo_};
+ }
+ friend auto operator+(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> uint128_fallback {
+ auto result = uint128_fallback(lhs);
+ result += rhs;
+ return result;
+ }
+ friend auto operator*(const uint128_fallback& lhs, uint32_t rhs)
+ -> uint128_fallback {
+ FMT_ASSERT(lhs.hi_ == 0, "");
+ uint64_t hi = (lhs.lo_ >> 32) * rhs;
+ uint64_t lo = (lhs.lo_ & ~uint32_t()) * rhs;
+ uint64_t new_lo = (hi << 32) + lo;
+ return {(hi >> 32) + (new_lo < lo ? 1 : 0), new_lo};
+ }
+ friend auto operator-(const uint128_fallback& lhs, uint64_t rhs)
+ -> uint128_fallback {
+ return {lhs.hi_ - (lhs.lo_ < rhs ? 1 : 0), lhs.lo_ - rhs};
+ }
+ FMT_CONSTEXPR auto operator>>(int shift) const -> uint128_fallback {
+ if (shift == 64) return {0, hi_};
+ if (shift > 64) return uint128_fallback(0, hi_) >> (shift - 64);
+ return {hi_ >> shift, (hi_ << (64 - shift)) | (lo_ >> shift)};
+ }
+ FMT_CONSTEXPR auto operator<<(int shift) const -> uint128_fallback {
+ if (shift == 64) return {lo_, 0};
+ if (shift > 64) return uint128_fallback(lo_, 0) << (shift - 64);
+ return {hi_ << shift | (lo_ >> (64 - shift)), (lo_ << shift)};
+ }
+ FMT_CONSTEXPR auto operator>>=(int shift) -> uint128_fallback& {
+ return *this = *this >> shift;
+ }
+ FMT_CONSTEXPR void operator+=(uint128_fallback n) {
+ uint64_t new_lo = lo_ + n.lo_;
+ uint64_t new_hi = hi_ + n.hi_ + (new_lo < lo_ ? 1 : 0);
+ FMT_ASSERT(new_hi >= hi_, "");
+ lo_ = new_lo;
+ hi_ = new_hi;
+ }
+ FMT_CONSTEXPR void operator&=(uint128_fallback n) {
+ lo_ &= n.lo_;
+ hi_ &= n.hi_;
+ }
+
+ FMT_CONSTEXPR20 uint128_fallback& operator+=(uint64_t n) noexcept {
+ if (is_constant_evaluated()) {
+ lo_ += n;
+ hi_ += (lo_ < n ? 1 : 0);
+ return *this;
+ }
+#if FMT_HAS_BUILTIN(__builtin_addcll) && !defined(__ibmxl__)
+ unsigned long long carry;
+ lo_ = __builtin_addcll(lo_, n, 0, &carry);
+ hi_ += carry;
+#elif FMT_HAS_BUILTIN(__builtin_ia32_addcarryx_u64) && !defined(__ibmxl__)
+ unsigned long long result;
+ auto carry = __builtin_ia32_addcarryx_u64(0, lo_, n, &result);
+ lo_ = result;
+ hi_ += carry;
+#elif defined(_MSC_VER) && defined(_M_X64)
+ auto carry = _addcarry_u64(0, lo_, n, &lo_);
+ _addcarry_u64(carry, hi_, 0, &hi_);
+#else
+ lo_ += n;
+ hi_ += (lo_ < n ? 1 : 0);
+#endif
+ return *this;
+ }
+};
+
+using uint128_t = conditional_t<FMT_USE_INT128, uint128_opt, uint128_fallback>;
+
+#ifdef UINTPTR_MAX
+using uintptr_t = ::uintptr_t;
+#else
+using uintptr_t = uint128_t;
+#endif
+
+// Returns the largest possible value for type T. Same as
+// std::numeric_limits<T>::max() but shorter and not affected by the max macro.
+template <typename T> constexpr auto max_value() -> T {
+ return (std::numeric_limits<T>::max)();
+}
+template <typename T> constexpr auto num_bits() -> int {
+ return std::numeric_limits<T>::digits;
+}
+// std::numeric_limits<T>::digits may return 0 for 128-bit ints.
+template <> constexpr auto num_bits<int128_opt>() -> int { return 128; }
+template <> constexpr auto num_bits<uint128_t>() -> int { return 128; }
+
+// A heterogeneous bit_cast used for converting 96-bit long double to uint128_t
+// and 128-bit pointers to uint128_fallback.
+template <typename To, typename From, FMT_ENABLE_IF(sizeof(To) > sizeof(From))>
+inline auto bit_cast(const From& from) -> To {
+ constexpr auto size = static_cast<int>(sizeof(From) / sizeof(unsigned));
+ struct data_t {
+ unsigned value[static_cast<unsigned>(size)];
+ } data = bit_cast<data_t>(from);
+ auto result = To();
+ if (const_check(is_big_endian())) {
+ for (int i = 0; i < size; ++i)
+ result = (result << num_bits<unsigned>()) | data.value[i];
+ } else {
+ for (int i = size - 1; i >= 0; --i)
+ result = (result << num_bits<unsigned>()) | data.value[i];
+ }
+ return result;
+}
+
+template <typename UInt>
+FMT_CONSTEXPR20 inline auto countl_zero_fallback(UInt n) -> int {
+ int lz = 0;
+ constexpr UInt msb_mask = static_cast<UInt>(1) << (num_bits<UInt>() - 1);
+ for (; (n & msb_mask) == 0; n <<= 1) lz++;
+ return lz;
+}
+
+FMT_CONSTEXPR20 inline auto countl_zero(uint32_t n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+ if (!is_constant_evaluated()) return FMT_BUILTIN_CLZ(n);
+#endif
+ return countl_zero_fallback(n);
+}
+
+FMT_CONSTEXPR20 inline auto countl_zero(uint64_t n) -> int {
+#ifdef FMT_BUILTIN_CLZLL
+ if (!is_constant_evaluated()) return FMT_BUILTIN_CLZLL(n);
+#endif
+ return countl_zero_fallback(n);
+}
+
+FMT_INLINE void assume(bool condition) {
+ (void)condition;
+#if FMT_HAS_BUILTIN(__builtin_assume) && !FMT_ICC_VERSION
+ __builtin_assume(condition);
+#endif
+}
+
+// An approximation of iterator_t for pre-C++20 systems.
+template <typename T>
+using iterator_t = decltype(std::begin(std::declval<T&>()));
+template <typename T> using sentinel_t = decltype(std::end(std::declval<T&>()));
+
+// A workaround for std::string not having mutable data() until C++17.
+template <typename Char>
+inline auto get_data(std::basic_string<Char>& s) -> Char* {
+ return &s[0];
+}
+template <typename Container>
+inline auto get_data(Container& c) -> typename Container::value_type* {
+ return c.data();
+}
+
+#if defined(_SECURE_SCL) && _SECURE_SCL
+// Make a checked iterator to avoid MSVC warnings.
+template <typename T> using checked_ptr = stdext::checked_array_iterator<T*>;
+template <typename T>
+constexpr auto make_checked(T* p, size_t size) -> checked_ptr<T> {
+ return {p, size};
+}
+#else
+template <typename T> using checked_ptr = T*;
+template <typename T> constexpr auto make_checked(T* p, size_t) -> T* {
+ return p;
+}
+#endif
+
+// Attempts to reserve space for n extra characters in the output range.
+// Returns a pointer to the reserved range or a reference to it.
+template <typename Container, FMT_ENABLE_IF(is_contiguous<Container>::value)>
+#if FMT_CLANG_VERSION >= 307 && !FMT_ICC_VERSION
+__attribute__((no_sanitize("undefined")))
+#endif
+inline auto
+reserve(std::back_insert_iterator<Container> it, size_t n)
+ -> checked_ptr<typename Container::value_type> {
+ Container& c = get_container(it);
+ size_t size = c.size();
+ c.resize(size + n);
+ return make_checked(get_data(c) + size, n);
+}
+
+template <typename T>
+inline auto reserve(buffer_appender<T> it, size_t n) -> buffer_appender<T> {
+ buffer<T>& buf = get_container(it);
+ buf.try_reserve(buf.size() + n);
+ return it;
+}
+
+template <typename Iterator>
+constexpr auto reserve(Iterator& it, size_t) -> Iterator& {
+ return it;
+}
+
+template <typename OutputIt>
+using reserve_iterator =
+ remove_reference_t<decltype(reserve(std::declval<OutputIt&>(), 0))>;
+
+template <typename T, typename OutputIt>
+constexpr auto to_pointer(OutputIt, size_t) -> T* {
+ return nullptr;
+}
+template <typename T> auto to_pointer(buffer_appender<T> it, size_t n) -> T* {
+ buffer<T>& buf = get_container(it);
+ auto size = buf.size();
+ if (buf.capacity() < size + n) return nullptr;
+ buf.try_resize(size + n);
+ return buf.data() + size;
+}
+
+template <typename Container, FMT_ENABLE_IF(is_contiguous<Container>::value)>
+inline auto base_iterator(std::back_insert_iterator<Container>& it,
+ checked_ptr<typename Container::value_type>)
+ -> std::back_insert_iterator<Container> {
+ return it;
+}
+
+template <typename Iterator>
+constexpr auto base_iterator(Iterator, Iterator it) -> Iterator {
+ return it;
+}
+
+// <algorithm> is spectacularly slow to compile in C++20 so use a simple fill_n
+// instead (#1998).
+template <typename OutputIt, typename Size, typename T>
+FMT_CONSTEXPR auto fill_n(OutputIt out, Size count, const T& value)
+ -> OutputIt {
+ for (Size i = 0; i < count; ++i) *out++ = value;
+ return out;
+}
+template <typename T, typename Size>
+FMT_CONSTEXPR20 auto fill_n(T* out, Size count, char value) -> T* {
+ if (is_constant_evaluated()) {
+ return fill_n<T*, Size, T>(out, count, value);
+ }
+ std::memset(out, value, to_unsigned(count));
+ return out + count;
+}
+
+#ifdef __cpp_char8_t
+using char8_type = char8_t;
+#else
+enum char8_type : unsigned char {};
+#endif
+
+template <typename OutChar, typename InputIt, typename OutputIt>
+FMT_CONSTEXPR FMT_NOINLINE auto copy_str_noinline(InputIt begin, InputIt end,
+ OutputIt out) -> OutputIt {
+ return copy_str<OutChar>(begin, end, out);
+}
+
+// A public domain branchless UTF-8 decoder by Christopher Wellons:
+// https://github.com/skeeto/branchless-utf8
+/* Decode the next character, c, from s, reporting errors in e.
+ *
+ * Since this is a branchless decoder, four bytes will be read from the
+ * buffer regardless of the actual length of the next character. This
+ * means the buffer _must_ have at least three bytes of zero padding
+ * following the end of the data stream.
+ *
+ * Errors are reported in e, which will be non-zero if the parsed
+ * character was somehow invalid: invalid byte sequence, non-canonical
+ * encoding, or a surrogate half.
+ *
+ * The function returns a pointer to the next character. When an error
+ * occurs, this pointer will be a guess that depends on the particular
+ * error, but it will always advance at least one byte.
+ */
+FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e)
+ -> const char* {
+ constexpr const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07};
+ constexpr const uint32_t mins[] = {4194304, 0, 128, 2048, 65536};
+ constexpr const int shiftc[] = {0, 18, 12, 6, 0};
+ constexpr const int shifte[] = {0, 6, 4, 2, 0};
+
+ int len = "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4"
+ [static_cast<unsigned char>(*s) >> 3];
+ // Compute the pointer to the next character early so that the next
+ // iteration can start working on the next character. Neither Clang
+ // nor GCC figure out this reordering on their own.
+ const char* next = s + len + !len;
+
+ using uchar = unsigned char;
+
+ // Assume a four-byte character and load four bytes. Unused bits are
+ // shifted out.
+ *c = uint32_t(uchar(s[0]) & masks[len]) << 18;
+ *c |= uint32_t(uchar(s[1]) & 0x3f) << 12;
+ *c |= uint32_t(uchar(s[2]) & 0x3f) << 6;
+ *c |= uint32_t(uchar(s[3]) & 0x3f) << 0;
+ *c >>= shiftc[len];
+
+ // Accumulate the various error conditions.
+ *e = (*c < mins[len]) << 6; // non-canonical encoding
+ *e |= ((*c >> 11) == 0x1b) << 7; // surrogate half?
+ *e |= (*c > 0x10FFFF) << 8; // out of range?
+ *e |= (uchar(s[1]) & 0xc0) >> 2;
+ *e |= (uchar(s[2]) & 0xc0) >> 4;
+ *e |= uchar(s[3]) >> 6;
+ *e ^= 0x2a; // top two bits of each tail byte correct?
+ *e >>= shifte[len];
+
+ return next;
+}
+
+constexpr FMT_INLINE_VARIABLE uint32_t invalid_code_point = ~uint32_t();
+
+// Invokes f(cp, sv) for every code point cp in s with sv being the string view
+// corresponding to the code point. cp is invalid_code_point on error.
+template <typename F>
+FMT_CONSTEXPR void for_each_codepoint(string_view s, F f) {
+ auto decode = [f](const char* buf_ptr, const char* ptr) {
+ auto cp = uint32_t();
+ auto error = 0;
+ auto end = utf8_decode(buf_ptr, &cp, &error);
+ bool result = f(error ? invalid_code_point : cp,
+ string_view(ptr, error ? 1 : to_unsigned(end - buf_ptr)));
+ return result ? (error ? buf_ptr + 1 : end) : nullptr;
+ };
+ auto p = s.data();
+ const size_t block_size = 4; // utf8_decode always reads blocks of 4 chars.
+ if (s.size() >= block_size) {
+ for (auto end = p + s.size() - block_size + 1; p < end;) {
+ p = decode(p, p);
+ if (!p) return;
+ }
+ }
+ if (auto num_chars_left = s.data() + s.size() - p) {
+ char buf[2 * block_size - 1] = {};
+ copy_str<char>(p, p + num_chars_left, buf);
+ const char* buf_ptr = buf;
+ do {
+ auto end = decode(buf_ptr, p);
+ if (!end) return;
+ p += end - buf_ptr;
+ buf_ptr = end;
+ } while (buf_ptr - buf < num_chars_left);
+ }
+}
+
+template <typename Char>
+inline auto compute_width(basic_string_view<Char> s) -> size_t {
+ return s.size();
+}
+
+// Computes approximate display width of a UTF-8 string.
+FMT_CONSTEXPR inline size_t compute_width(string_view s) {
+ size_t num_code_points = 0;
+ // It is not a lambda for compatibility with C++14.
+ struct count_code_points {
+ size_t* count;
+ FMT_CONSTEXPR auto operator()(uint32_t cp, string_view) const -> bool {
+ *count += detail::to_unsigned(
+ 1 +
+ (cp >= 0x1100 &&
+ (cp <= 0x115f || // Hangul Jamo init. consonants
+ cp == 0x2329 || // LEFT-POINTING ANGLE BRACKET
+ cp == 0x232a || // RIGHT-POINTING ANGLE BRACKET
+ // CJK ... Yi except IDEOGRAPHIC HALF FILL SPACE:
+ (cp >= 0x2e80 && cp <= 0xa4cf && cp != 0x303f) ||
+ (cp >= 0xac00 && cp <= 0xd7a3) || // Hangul Syllables
+ (cp >= 0xf900 && cp <= 0xfaff) || // CJK Compatibility Ideographs
+ (cp >= 0xfe10 && cp <= 0xfe19) || // Vertical Forms
+ (cp >= 0xfe30 && cp <= 0xfe6f) || // CJK Compatibility Forms
+ (cp >= 0xff00 && cp <= 0xff60) || // Fullwidth Forms
+ (cp >= 0xffe0 && cp <= 0xffe6) || // Fullwidth Forms
+ (cp >= 0x20000 && cp <= 0x2fffd) || // CJK
+ (cp >= 0x30000 && cp <= 0x3fffd) ||
+ // Miscellaneous Symbols and Pictographs + Emoticons:
+ (cp >= 0x1f300 && cp <= 0x1f64f) ||
+ // Supplemental Symbols and Pictographs:
+ (cp >= 0x1f900 && cp <= 0x1f9ff))));
+ return true;
+ }
+ };
+ // We could avoid branches by using utf8_decode directly.
+ for_each_codepoint(s, count_code_points{&num_code_points});
+ return num_code_points;
+}
+
+inline auto compute_width(basic_string_view<char8_type> s) -> size_t {
+ return compute_width(
+ string_view(reinterpret_cast<const char*>(s.data()), s.size()));
+}
+
+template <typename Char>
+inline auto code_point_index(basic_string_view<Char> s, size_t n) -> size_t {
+ size_t size = s.size();
+ return n < size ? n : size;
+}
+
+// Calculates the index of the nth code point in a UTF-8 string.
+inline auto code_point_index(string_view s, size_t n) -> size_t {
+ const char* data = s.data();
+ size_t num_code_points = 0;
+ for (size_t i = 0, size = s.size(); i != size; ++i) {
+ if ((data[i] & 0xc0) != 0x80 && ++num_code_points > n) return i;
+ }
+ return s.size();
+}
+
+inline auto code_point_index(basic_string_view<char8_type> s, size_t n)
+ -> size_t {
+ return code_point_index(
+ string_view(reinterpret_cast<const char*>(s.data()), s.size()), n);
+}
+
+template <typename T> struct is_integral : std::is_integral<T> {};
+template <> struct is_integral<int128_opt> : std::true_type {};
+template <> struct is_integral<uint128_t> : std::true_type {};
+
+template <typename T>
+using is_signed =
+ std::integral_constant<bool, std::numeric_limits<T>::is_signed ||
+ std::is_same<T, int128_opt>::value>;
+
+template <typename T>
+using is_integer =
+ bool_constant<is_integral<T>::value && !std::is_same<T, bool>::value &&
+ !std::is_same<T, char>::value &&
+ !std::is_same<T, wchar_t>::value>;
+
+#ifndef FMT_USE_FLOAT
+# define FMT_USE_FLOAT 1
+#endif
+#ifndef FMT_USE_DOUBLE
+# define FMT_USE_DOUBLE 1
+#endif
+#ifndef FMT_USE_LONG_DOUBLE
+# define FMT_USE_LONG_DOUBLE 1
+#endif
+
+#ifndef FMT_USE_FLOAT128
+# ifdef __clang__
+// Clang emulates GCC, so it has to appear early.
+# if FMT_HAS_INCLUDE(<quadmath.h>)
+# define FMT_USE_FLOAT128 1
+# endif
+# elif defined(__GNUC__)
+// GNU C++:
+# if defined(_GLIBCXX_USE_FLOAT128) && !defined(__STRICT_ANSI__)
+# define FMT_USE_FLOAT128 1
+# endif
+# endif
+# ifndef FMT_USE_FLOAT128
+# define FMT_USE_FLOAT128 0
+# endif
+#endif
+
+#if FMT_USE_FLOAT128
+using float128 = __float128;
+#else
+using float128 = void;
+#endif
+template <typename T> using is_float128 = std::is_same<T, float128>;
+
+template <typename T>
+using is_floating_point =
+ bool_constant<std::is_floating_point<T>::value || is_float128<T>::value>;
+
+template <typename T, bool = std::is_floating_point<T>::value>
+struct is_fast_float : bool_constant<std::numeric_limits<T>::is_iec559 &&
+ sizeof(T) <= sizeof(double)> {};
+template <typename T> struct is_fast_float<T, false> : std::false_type {};
+
+template <typename T>
+using is_double_double = bool_constant<std::numeric_limits<T>::digits == 106>;
+
+#ifndef FMT_USE_FULL_CACHE_DRAGONBOX
+# define FMT_USE_FULL_CACHE_DRAGONBOX 0
+#endif
+
+template <typename T>
+template <typename U>
+void buffer<T>::append(const U* begin, const U* end) {
+ while (begin != end) {
+ auto count = to_unsigned(end - begin);
+ try_reserve(size_ + count);
+ auto free_cap = capacity_ - size_;
+ if (free_cap < count) count = free_cap;
+ std::uninitialized_copy_n(begin, count, make_checked(ptr_ + size_, count));
+ size_ += count;
+ begin += count;
+ }
+}
+
+template <typename T, typename Enable = void>
+struct is_locale : std::false_type {};
+template <typename T>
+struct is_locale<T, void_t<decltype(T::classic())>> : std::true_type {};
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+// The number of characters to store in the basic_memory_buffer object itself
+// to avoid dynamic memory allocation.
+enum { inline_buffer_size = 500 };
+
+/**
+ \rst
+ A dynamically growing memory buffer for trivially copyable/constructible types
+ with the first ``SIZE`` elements stored in the object itself.
+
+ You can use the ``memory_buffer`` type alias for ``char`` instead.
+
+ **Example**::
+
+ auto out = fmt::memory_buffer();
+ format_to(std::back_inserter(out), "The answer is {}.", 42);
+
+ This will append the following output to the ``out`` object:
+
+ .. code-block:: none
+
+ The answer is 42.
+
+ The output can be converted to an ``std::string`` with ``to_string(out)``.
+ \endrst
+ */
+template <typename T, size_t SIZE = inline_buffer_size,
+ typename Allocator = std::allocator<T>>
+class basic_memory_buffer final : public detail::buffer<T> {
+ private:
+ T store_[SIZE];
+
+ // Don't inherit from Allocator avoid generating type_info for it.
+ Allocator alloc_;
+
+ // Deallocate memory allocated by the buffer.
+ FMT_CONSTEXPR20 void deallocate() {
+ T* data = this->data();
+ if (data != store_) alloc_.deallocate(data, this->capacity());
+ }
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t size) override {
+ detail::abort_fuzzing_if(size > 5000);
+ const size_t max_size = std::allocator_traits<Allocator>::max_size(alloc_);
+ size_t old_capacity = this->capacity();
+ size_t new_capacity = old_capacity + old_capacity / 2;
+ if (size > new_capacity)
+ new_capacity = size;
+ else if (new_capacity > max_size)
+ new_capacity = size > max_size ? size : max_size;
+ T* old_data = this->data();
+ T* new_data =
+ std::allocator_traits<Allocator>::allocate(alloc_, new_capacity);
+ // The following code doesn't throw, so the raw pointer above doesn't leak.
+ std::uninitialized_copy(old_data, old_data + this->size(),
+ detail::make_checked(new_data, new_capacity));
+ this->set(new_data, new_capacity);
+ // deallocate must not throw according to the standard, but even if it does,
+ // the buffer already uses the new storage and will deallocate it in
+ // destructor.
+ if (old_data != store_) alloc_.deallocate(old_data, old_capacity);
+ }
+
+ public:
+ using value_type = T;
+ using const_reference = const T&;
+
+ FMT_CONSTEXPR20 explicit basic_memory_buffer(
+ const Allocator& alloc = Allocator())
+ : alloc_(alloc) {
+ this->set(store_, SIZE);
+ if (detail::is_constant_evaluated()) detail::fill_n(store_, SIZE, T());
+ }
+ FMT_CONSTEXPR20 ~basic_memory_buffer() { deallocate(); }
+
+ private:
+ // Move data from other to this buffer.
+ FMT_CONSTEXPR20 void move(basic_memory_buffer& other) {
+ alloc_ = std::move(other.alloc_);
+ T* data = other.data();
+ size_t size = other.size(), capacity = other.capacity();
+ if (data == other.store_) {
+ this->set(store_, capacity);
+ detail::copy_str<T>(other.store_, other.store_ + size,
+ detail::make_checked(store_, capacity));
+ } else {
+ this->set(data, capacity);
+ // Set pointer to the inline array so that delete is not called
+ // when deallocating.
+ other.set(other.store_, 0);
+ other.clear();
+ }
+ this->resize(size);
+ }
+
+ public:
+ /**
+ \rst
+ Constructs a :class:`fmt::basic_memory_buffer` object moving the content
+ of the other object to it.
+ \endrst
+ */
+ FMT_CONSTEXPR20 basic_memory_buffer(basic_memory_buffer&& other) noexcept {
+ move(other);
+ }
+
+ /**
+ \rst
+ Moves the content of the other ``basic_memory_buffer`` object to this one.
+ \endrst
+ */
+ auto operator=(basic_memory_buffer&& other) noexcept -> basic_memory_buffer& {
+ FMT_ASSERT(this != &other, "");
+ deallocate();
+ move(other);
+ return *this;
+ }
+
+ // Returns a copy of the allocator associated with this buffer.
+ auto get_allocator() const -> Allocator { return alloc_; }
+
+ /**
+ Resizes the buffer to contain *count* elements. If T is a POD type new
+ elements may not be initialized.
+ */
+ FMT_CONSTEXPR20 void resize(size_t count) { this->try_resize(count); }
+
+ /** Increases the buffer capacity to *new_capacity*. */
+ void reserve(size_t new_capacity) { this->try_reserve(new_capacity); }
+
+ // Directly append data into the buffer
+ using detail::buffer<T>::append;
+ template <typename ContiguousRange>
+ void append(const ContiguousRange& range) {
+ append(range.data(), range.data() + range.size());
+ }
+};
+
+using memory_buffer = basic_memory_buffer<char>;
+
+template <typename T, size_t SIZE, typename Allocator>
+struct is_contiguous<basic_memory_buffer<T, SIZE, Allocator>> : std::true_type {
+};
+
+FMT_END_EXPORT
+namespace detail {
+FMT_API bool write_console(std::FILE* f, string_view text);
+FMT_API void print(std::FILE*, string_view);
+} // namespace detail
+FMT_BEGIN_EXPORT
+
+// Suppress a misleading warning in older versions of clang.
+#if FMT_CLANG_VERSION
+# pragma clang diagnostic ignored "-Wweak-vtables"
+#endif
+
+/** An error reported from a formatting function. */
+class FMT_API format_error : public std::runtime_error {
+ public:
+ using std::runtime_error::runtime_error;
+};
+
+namespace detail_exported {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename Char, size_t N> struct fixed_string {
+ constexpr fixed_string(const Char (&str)[N]) {
+ detail::copy_str<Char, const Char*, Char*>(static_cast<const Char*>(str),
+ str + N, data);
+ }
+ Char data[N] = {};
+};
+#endif
+
+// Converts a compile-time string to basic_string_view.
+template <typename Char, size_t N>
+constexpr auto compile_string_to_view(const Char (&s)[N])
+ -> basic_string_view<Char> {
+ // Remove trailing NUL character if needed. Won't be present if this is used
+ // with a raw character array (i.e. not defined as a string).
+ return {s, N - (std::char_traits<Char>::to_int_type(s[N - 1]) == 0 ? 1 : 0)};
+}
+template <typename Char>
+constexpr auto compile_string_to_view(detail::std_string_view<Char> s)
+ -> basic_string_view<Char> {
+ return {s.data(), s.size()};
+}
+} // namespace detail_exported
+
+class loc_value {
+ private:
+ basic_format_arg<format_context> value_;
+
+ public:
+ template <typename T, FMT_ENABLE_IF(!detail::is_float128<T>::value)>
+ loc_value(T value) : value_(detail::make_arg<format_context>(value)) {}
+
+ template <typename T, FMT_ENABLE_IF(detail::is_float128<T>::value)>
+ loc_value(T) {}
+
+ template <typename Visitor> auto visit(Visitor&& vis) -> decltype(vis(0)) {
+ return visit_format_arg(vis, value_);
+ }
+};
+
+// A locale facet that formats values in UTF-8.
+// It is parameterized on the locale to avoid the heavy <locale> include.
+template <typename Locale> class format_facet : public Locale::facet {
+ private:
+ std::string separator_;
+ std::string grouping_;
+ std::string decimal_point_;
+
+ protected:
+ virtual auto do_put(appender out, loc_value val,
+ const format_specs<>& specs) const -> bool;
+
+ public:
+ static FMT_API typename Locale::id id;
+
+ explicit format_facet(Locale& loc);
+ explicit format_facet(string_view sep = "",
+ std::initializer_list<unsigned char> g = {3},
+ std::string decimal_point = ".")
+ : separator_(sep.data(), sep.size()),
+ grouping_(g.begin(), g.end()),
+ decimal_point_(decimal_point) {}
+
+ auto put(appender out, loc_value val, const format_specs<>& specs) const
+ -> bool {
+ return do_put(out, val, specs);
+ }
+};
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// Returns true if value is negative, false otherwise.
+// Same as `value < 0` but doesn't produce warnings if T is an unsigned type.
+template <typename T, FMT_ENABLE_IF(is_signed<T>::value)>
+constexpr auto is_negative(T value) -> bool {
+ return value < 0;
+}
+template <typename T, FMT_ENABLE_IF(!is_signed<T>::value)>
+constexpr auto is_negative(T) -> bool {
+ return false;
+}
+
+template <typename T>
+FMT_CONSTEXPR auto is_supported_floating_point(T) -> bool {
+ if (std::is_same<T, float>()) return FMT_USE_FLOAT;
+ if (std::is_same<T, double>()) return FMT_USE_DOUBLE;
+ if (std::is_same<T, long double>()) return FMT_USE_LONG_DOUBLE;
+ return true;
+}
+
+// Smallest of uint32_t, uint64_t, uint128_t that is large enough to
+// represent all values of an integral type T.
+template <typename T>
+using uint32_or_64_or_128_t =
+ conditional_t<num_bits<T>() <= 32 && !FMT_REDUCE_INT_INSTANTIATIONS,
+ uint32_t,
+ conditional_t<num_bits<T>() <= 64, uint64_t, uint128_t>>;
+template <typename T>
+using uint64_or_128_t = conditional_t<num_bits<T>() <= 64, uint64_t, uint128_t>;
+
+#define FMT_POWERS_OF_10(factor) \
+ factor * 10, (factor)*100, (factor)*1000, (factor)*10000, (factor)*100000, \
+ (factor)*1000000, (factor)*10000000, (factor)*100000000, \
+ (factor)*1000000000
+
+// Converts value in the range [0, 100) to a string.
+constexpr const char* digits2(size_t value) {
+ // GCC generates slightly better code when value is pointer-size.
+ return &"0001020304050607080910111213141516171819"
+ "2021222324252627282930313233343536373839"
+ "4041424344454647484950515253545556575859"
+ "6061626364656667686970717273747576777879"
+ "8081828384858687888990919293949596979899"[value * 2];
+}
+
+// Sign is a template parameter to workaround a bug in gcc 4.8.
+template <typename Char, typename Sign> constexpr Char sign(Sign s) {
+#if !FMT_GCC_VERSION || FMT_GCC_VERSION >= 604
+ static_assert(std::is_same<Sign, sign_t>::value, "");
+#endif
+ return static_cast<Char>("\0-+ "[s]);
+}
+
+template <typename T> FMT_CONSTEXPR auto count_digits_fallback(T n) -> int {
+ int count = 1;
+ for (;;) {
+ // Integer division is slow so do it for a group of four digits instead
+ // of for every digit. The idea comes from the talk by Alexandrescu
+ // "Three Optimization Tips for C++". See speed-test for a comparison.
+ if (n < 10) return count;
+ if (n < 100) return count + 1;
+ if (n < 1000) return count + 2;
+ if (n < 10000) return count + 3;
+ n /= 10000u;
+ count += 4;
+ }
+}
+#if FMT_USE_INT128
+FMT_CONSTEXPR inline auto count_digits(uint128_opt n) -> int {
+ return count_digits_fallback(n);
+}
+#endif
+
+#ifdef FMT_BUILTIN_CLZLL
+// It is a separate function rather than a part of count_digits to workaround
+// the lack of static constexpr in constexpr functions.
+inline auto do_count_digits(uint64_t n) -> int {
+ // This has comparable performance to the version by Kendall Willets
+ // (https://github.com/fmtlib/format-benchmark/blob/master/digits10)
+ // but uses smaller tables.
+ // Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)).
+ static constexpr uint8_t bsr2log10[] = {
+ 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5,
+ 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10,
+ 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15,
+ 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20};
+ auto t = bsr2log10[FMT_BUILTIN_CLZLL(n | 1) ^ 63];
+ static constexpr const uint64_t zero_or_powers_of_10[] = {
+ 0, 0, FMT_POWERS_OF_10(1U), FMT_POWERS_OF_10(1000000000ULL),
+ 10000000000000000000ULL};
+ return t - (n < zero_or_powers_of_10[t]);
+}
+#endif
+
+// Returns the number of decimal digits in n. Leading zeros are not counted
+// except for n == 0 in which case count_digits returns 1.
+FMT_CONSTEXPR20 inline auto count_digits(uint64_t n) -> int {
+#ifdef FMT_BUILTIN_CLZLL
+ if (!is_constant_evaluated()) {
+ return do_count_digits(n);
+ }
+#endif
+ return count_digits_fallback(n);
+}
+
+// Counts the number of digits in n. BITS = log2(radix).
+template <int BITS, typename UInt>
+FMT_CONSTEXPR auto count_digits(UInt n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+ if (!is_constant_evaluated() && num_bits<UInt>() == 32)
+ return (FMT_BUILTIN_CLZ(static_cast<uint32_t>(n) | 1) ^ 31) / BITS + 1;
+#endif
+ // Lambda avoids unreachable code warnings from NVHPC.
+ return [](UInt m) {
+ int num_digits = 0;
+ do {
+ ++num_digits;
+ } while ((m >>= BITS) != 0);
+ return num_digits;
+ }(n);
+}
+
+#ifdef FMT_BUILTIN_CLZ
+// It is a separate function rather than a part of count_digits to workaround
+// the lack of static constexpr in constexpr functions.
+FMT_INLINE auto do_count_digits(uint32_t n) -> int {
+// An optimization by Kendall Willets from https://bit.ly/3uOIQrB.
+// This increments the upper 32 bits (log10(T) - 1) when >= T is added.
+# define FMT_INC(T) (((sizeof(# T) - 1ull) << 32) - T)
+ static constexpr uint64_t table[] = {
+ FMT_INC(0), FMT_INC(0), FMT_INC(0), // 8
+ FMT_INC(10), FMT_INC(10), FMT_INC(10), // 64
+ FMT_INC(100), FMT_INC(100), FMT_INC(100), // 512
+ FMT_INC(1000), FMT_INC(1000), FMT_INC(1000), // 4096
+ FMT_INC(10000), FMT_INC(10000), FMT_INC(10000), // 32k
+ FMT_INC(100000), FMT_INC(100000), FMT_INC(100000), // 256k
+ FMT_INC(1000000), FMT_INC(1000000), FMT_INC(1000000), // 2048k
+ FMT_INC(10000000), FMT_INC(10000000), FMT_INC(10000000), // 16M
+ FMT_INC(100000000), FMT_INC(100000000), FMT_INC(100000000), // 128M
+ FMT_INC(1000000000), FMT_INC(1000000000), FMT_INC(1000000000), // 1024M
+ FMT_INC(1000000000), FMT_INC(1000000000) // 4B
+ };
+ auto inc = table[FMT_BUILTIN_CLZ(n | 1) ^ 31];
+ return static_cast<int>((n + inc) >> 32);
+}
+#endif
+
+// Optional version of count_digits for better performance on 32-bit platforms.
+FMT_CONSTEXPR20 inline auto count_digits(uint32_t n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+ if (!is_constant_evaluated()) {
+ return do_count_digits(n);
+ }
+#endif
+ return count_digits_fallback(n);
+}
+
+template <typename Int> constexpr auto digits10() noexcept -> int {
+ return std::numeric_limits<Int>::digits10;
+}
+template <> constexpr auto digits10<int128_opt>() noexcept -> int { return 38; }
+template <> constexpr auto digits10<uint128_t>() noexcept -> int { return 38; }
+
+template <typename Char> struct thousands_sep_result {
+ std::string grouping;
+ Char thousands_sep;
+};
+
+template <typename Char>
+FMT_API auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result<Char>;
+template <typename Char>
+inline auto thousands_sep(locale_ref loc) -> thousands_sep_result<Char> {
+ auto result = thousands_sep_impl<char>(loc);
+ return {result.grouping, Char(result.thousands_sep)};
+}
+template <>
+inline auto thousands_sep(locale_ref loc) -> thousands_sep_result<wchar_t> {
+ return thousands_sep_impl<wchar_t>(loc);
+}
+
+template <typename Char>
+FMT_API auto decimal_point_impl(locale_ref loc) -> Char;
+template <typename Char> inline auto decimal_point(locale_ref loc) -> Char {
+ return Char(decimal_point_impl<char>(loc));
+}
+template <> inline auto decimal_point(locale_ref loc) -> wchar_t {
+ return decimal_point_impl<wchar_t>(loc);
+}
+
+// Compares two characters for equality.
+template <typename Char> auto equal2(const Char* lhs, const char* rhs) -> bool {
+ return lhs[0] == Char(rhs[0]) && lhs[1] == Char(rhs[1]);
+}
+inline auto equal2(const char* lhs, const char* rhs) -> bool {
+ return memcmp(lhs, rhs, 2) == 0;
+}
+
+// Copies two characters from src to dst.
+template <typename Char>
+FMT_CONSTEXPR20 FMT_INLINE void copy2(Char* dst, const char* src) {
+ if (!is_constant_evaluated() && sizeof(Char) == sizeof(char)) {
+ memcpy(dst, src, 2);
+ return;
+ }
+ *dst++ = static_cast<Char>(*src++);
+ *dst = static_cast<Char>(*src);
+}
+
+template <typename Iterator> struct format_decimal_result {
+ Iterator begin;
+ Iterator end;
+};
+
+// Formats a decimal unsigned integer value writing into out pointing to a
+// buffer of specified size. The caller must ensure that the buffer is large
+// enough.
+template <typename Char, typename UInt>
+FMT_CONSTEXPR20 auto format_decimal(Char* out, UInt value, int size)
+ -> format_decimal_result<Char*> {
+ FMT_ASSERT(size >= count_digits(value), "invalid digit count");
+ out += size;
+ Char* end = out;
+ while (value >= 100) {
+ // Integer division is slow so do it for a group of two digits instead
+ // of for every digit. The idea comes from the talk by Alexandrescu
+ // "Three Optimization Tips for C++". See speed-test for a comparison.
+ out -= 2;
+ copy2(out, digits2(static_cast<size_t>(value % 100)));
+ value /= 100;
+ }
+ if (value < 10) {
+ *--out = static_cast<Char>('0' + value);
+ return {out, end};
+ }
+ out -= 2;
+ copy2(out, digits2(static_cast<size_t>(value)));
+ return {out, end};
+}
+
+template <typename Char, typename UInt, typename Iterator,
+ FMT_ENABLE_IF(!std::is_pointer<remove_cvref_t<Iterator>>::value)>
+FMT_CONSTEXPR inline auto format_decimal(Iterator out, UInt value, int size)
+ -> format_decimal_result<Iterator> {
+ // Buffer is large enough to hold all digits (digits10 + 1).
+ Char buffer[digits10<UInt>() + 1] = {};
+ auto end = format_decimal(buffer, value, size).end;
+ return {out, detail::copy_str_noinline<Char>(buffer, end, out)};
+}
+
+template <unsigned BASE_BITS, typename Char, typename UInt>
+FMT_CONSTEXPR auto format_uint(Char* buffer, UInt value, int num_digits,
+ bool upper = false) -> Char* {
+ buffer += num_digits;
+ Char* end = buffer;
+ do {
+ const char* digits = upper ? "0123456789ABCDEF" : "0123456789abcdef";
+ unsigned digit = static_cast<unsigned>(value & ((1 << BASE_BITS) - 1));
+ *--buffer = static_cast<Char>(BASE_BITS < 4 ? static_cast<char>('0' + digit)
+ : digits[digit]);
+ } while ((value >>= BASE_BITS) != 0);
+ return end;
+}
+
+template <unsigned BASE_BITS, typename Char, typename It, typename UInt>
+inline auto format_uint(It out, UInt value, int num_digits, bool upper = false)
+ -> It {
+ if (auto ptr = to_pointer<Char>(out, to_unsigned(num_digits))) {
+ format_uint<BASE_BITS>(ptr, value, num_digits, upper);
+ return out;
+ }
+ // Buffer should be large enough to hold all digits (digits / BASE_BITS + 1).
+ char buffer[num_bits<UInt>() / BASE_BITS + 1];
+ format_uint<BASE_BITS>(buffer, value, num_digits, upper);
+ return detail::copy_str_noinline<Char>(buffer, buffer + num_digits, out);
+}
+
+// A converter from UTF-8 to UTF-16.
+class utf8_to_utf16 {
+ private:
+ basic_memory_buffer<wchar_t> buffer_;
+
+ public:
+ FMT_API explicit utf8_to_utf16(string_view s);
+ operator basic_string_view<wchar_t>() const { return {&buffer_[0], size()}; }
+ auto size() const -> size_t { return buffer_.size() - 1; }
+ auto c_str() const -> const wchar_t* { return &buffer_[0]; }
+ auto str() const -> std::wstring { return {&buffer_[0], size()}; }
+};
+
+// A converter from UTF-16/UTF-32 (host endian) to UTF-8.
+template <typename WChar, typename Buffer = memory_buffer>
+class unicode_to_utf8 {
+ private:
+ Buffer buffer_;
+
+ public:
+ unicode_to_utf8() {}
+ explicit unicode_to_utf8(basic_string_view<WChar> s) {
+ static_assert(sizeof(WChar) == 2 || sizeof(WChar) == 4,
+ "Expect utf16 or utf32");
+
+ if (!convert(s))
+ FMT_THROW(std::runtime_error(sizeof(WChar) == 2 ? "invalid utf16"
+ : "invalid utf32"));
+ }
+ operator string_view() const { return string_view(&buffer_[0], size()); }
+ size_t size() const { return buffer_.size() - 1; }
+ const char* c_str() const { return &buffer_[0]; }
+ std::string str() const { return std::string(&buffer_[0], size()); }
+
+ // Performs conversion returning a bool instead of throwing exception on
+ // conversion error. This method may still throw in case of memory allocation
+ // error.
+ bool convert(basic_string_view<WChar> s) {
+ if (!convert(buffer_, s)) return false;
+ buffer_.push_back(0);
+ return true;
+ }
+ static bool convert(Buffer& buf, basic_string_view<WChar> s) {
+ for (auto p = s.begin(); p != s.end(); ++p) {
+ uint32_t c = static_cast<uint32_t>(*p);
+ if (sizeof(WChar) == 2 && c >= 0xd800 && c <= 0xdfff) {
+ // surrogate pair
+ ++p;
+ if (p == s.end() || (c & 0xfc00) != 0xd800 || (*p & 0xfc00) != 0xdc00) {
+ return false;
+ }
+ c = (c << 10) + static_cast<uint32_t>(*p) - 0x35fdc00;
+ }
+ if (c < 0x80) {
+ buf.push_back(static_cast<char>(c));
+ } else if (c < 0x800) {
+ buf.push_back(static_cast<char>(0xc0 | (c >> 6)));
+ buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+ } else if ((c >= 0x800 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xffff)) {
+ buf.push_back(static_cast<char>(0xe0 | (c >> 12)));
+ buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+ buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+ } else if (c >= 0x10000 && c <= 0x10ffff) {
+ buf.push_back(static_cast<char>(0xf0 | (c >> 18)));
+ buf.push_back(static_cast<char>(0x80 | ((c & 0x3ffff) >> 12)));
+ buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+ buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+};
+
+// Computes 128-bit result of multiplication of two 64-bit unsigned integers.
+inline uint128_fallback umul128(uint64_t x, uint64_t y) noexcept {
+#if FMT_USE_INT128
+ auto p = static_cast<uint128_opt>(x) * static_cast<uint128_opt>(y);
+ return {static_cast<uint64_t>(p >> 64), static_cast<uint64_t>(p)};
+#elif defined(_MSC_VER) && defined(_M_X64)
+ auto result = uint128_fallback();
+ result.lo_ = _umul128(x, y, &result.hi_);
+ return result;
+#else
+ const uint64_t mask = static_cast<uint64_t>(max_value<uint32_t>());
+
+ uint64_t a = x >> 32;
+ uint64_t b = x & mask;
+ uint64_t c = y >> 32;
+ uint64_t d = y & mask;
+
+ uint64_t ac = a * c;
+ uint64_t bc = b * c;
+ uint64_t ad = a * d;
+ uint64_t bd = b * d;
+
+ uint64_t intermediate = (bd >> 32) + (ad & mask) + (bc & mask);
+
+ return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32),
+ (intermediate << 32) + (bd & mask)};
+#endif
+}
+
+namespace dragonbox {
+// Computes floor(log10(pow(2, e))) for e in [-2620, 2620] using the method from
+// https://fmt.dev/papers/Dragonbox.pdf#page=28, section 6.1.
+inline int floor_log10_pow2(int e) noexcept {
+ FMT_ASSERT(e <= 2620 && e >= -2620, "too large exponent");
+ static_assert((-1 >> 1) == -1, "right shift is not arithmetic");
+ return (e * 315653) >> 20;
+}
+
+inline int floor_log2_pow10(int e) noexcept {
+ FMT_ASSERT(e <= 1233 && e >= -1233, "too large exponent");
+ return (e * 1741647) >> 19;
+}
+
+// Computes upper 64 bits of multiplication of two 64-bit unsigned integers.
+inline uint64_t umul128_upper64(uint64_t x, uint64_t y) noexcept {
+#if FMT_USE_INT128
+ auto p = static_cast<uint128_opt>(x) * static_cast<uint128_opt>(y);
+ return static_cast<uint64_t>(p >> 64);
+#elif defined(_MSC_VER) && defined(_M_X64)
+ return __umulh(x, y);
+#else
+ return umul128(x, y).high();
+#endif
+}
+
+// Computes upper 128 bits of multiplication of a 64-bit unsigned integer and a
+// 128-bit unsigned integer.
+inline uint128_fallback umul192_upper128(uint64_t x,
+ uint128_fallback y) noexcept {
+ uint128_fallback r = umul128(x, y.high());
+ r += umul128_upper64(x, y.low());
+ return r;
+}
+
+FMT_API uint128_fallback get_cached_power(int k) noexcept;
+
+// Type-specific information that Dragonbox uses.
+template <typename T, typename Enable = void> struct float_info;
+
+template <> struct float_info<float> {
+ using carrier_uint = uint32_t;
+ static const int exponent_bits = 8;
+ static const int kappa = 1;
+ static const int big_divisor = 100;
+ static const int small_divisor = 10;
+ static const int min_k = -31;
+ static const int max_k = 46;
+ static const int shorter_interval_tie_lower_threshold = -35;
+ static const int shorter_interval_tie_upper_threshold = -35;
+};
+
+template <> struct float_info<double> {
+ using carrier_uint = uint64_t;
+ static const int exponent_bits = 11;
+ static const int kappa = 2;
+ static const int big_divisor = 1000;
+ static const int small_divisor = 100;
+ static const int min_k = -292;
+ static const int max_k = 341;
+ static const int shorter_interval_tie_lower_threshold = -77;
+ static const int shorter_interval_tie_upper_threshold = -77;
+};
+
+// An 80- or 128-bit floating point number.
+template <typename T>
+struct float_info<T, enable_if_t<std::numeric_limits<T>::digits == 64 ||
+ std::numeric_limits<T>::digits == 113 ||
+ is_float128<T>::value>> {
+ using carrier_uint = detail::uint128_t;
+ static const int exponent_bits = 15;
+};
+
+// A double-double floating point number.
+template <typename T>
+struct float_info<T, enable_if_t<is_double_double<T>::value>> {
+ using carrier_uint = detail::uint128_t;
+};
+
+template <typename T> struct decimal_fp {
+ using significand_type = typename float_info<T>::carrier_uint;
+ significand_type significand;
+ int exponent;
+};
+
+template <typename T> FMT_API auto to_decimal(T x) noexcept -> decimal_fp<T>;
+} // namespace dragonbox
+
+// Returns true iff Float has the implicit bit which is not stored.
+template <typename Float> constexpr bool has_implicit_bit() {
+ // An 80-bit FP number has a 64-bit significand an no implicit bit.
+ return std::numeric_limits<Float>::digits != 64;
+}
+
+// Returns the number of significand bits stored in Float. The implicit bit is
+// not counted since it is not stored.
+template <typename Float> constexpr int num_significand_bits() {
+ // std::numeric_limits may not support __float128.
+ return is_float128<Float>() ? 112
+ : (std::numeric_limits<Float>::digits -
+ (has_implicit_bit<Float>() ? 1 : 0));
+}
+
+template <typename Float>
+constexpr auto exponent_mask() ->
+ typename dragonbox::float_info<Float>::carrier_uint {
+ using float_uint = typename dragonbox::float_info<Float>::carrier_uint;
+ return ((float_uint(1) << dragonbox::float_info<Float>::exponent_bits) - 1)
+ << num_significand_bits<Float>();
+}
+template <typename Float> constexpr auto exponent_bias() -> int {
+ // std::numeric_limits may not support __float128.
+ return is_float128<Float>() ? 16383
+ : std::numeric_limits<Float>::max_exponent - 1;
+}
+
+// Writes the exponent exp in the form "[+-]d{2,3}" to buffer.
+template <typename Char, typename It>
+FMT_CONSTEXPR auto write_exponent(int exp, It it) -> It {
+ FMT_ASSERT(-10000 < exp && exp < 10000, "exponent out of range");
+ if (exp < 0) {
+ *it++ = static_cast<Char>('-');
+ exp = -exp;
+ } else {
+ *it++ = static_cast<Char>('+');
+ }
+ if (exp >= 100) {
+ const char* top = digits2(to_unsigned(exp / 100));
+ if (exp >= 1000) *it++ = static_cast<Char>(top[0]);
+ *it++ = static_cast<Char>(top[1]);
+ exp %= 100;
+ }
+ const char* d = digits2(to_unsigned(exp));
+ *it++ = static_cast<Char>(d[0]);
+ *it++ = static_cast<Char>(d[1]);
+ return it;
+}
+
+// A floating-point number f * pow(2, e) where F is an unsigned type.
+template <typename F> struct basic_fp {
+ F f;
+ int e;
+
+ static constexpr const int num_significand_bits =
+ static_cast<int>(sizeof(F) * num_bits<unsigned char>());
+
+ constexpr basic_fp() : f(0), e(0) {}
+ constexpr basic_fp(uint64_t f_val, int e_val) : f(f_val), e(e_val) {}
+
+ // Constructs fp from an IEEE754 floating-point number.
+ template <typename Float> FMT_CONSTEXPR basic_fp(Float n) { assign(n); }
+
+ // Assigns n to this and return true iff predecessor is closer than successor.
+ template <typename Float, FMT_ENABLE_IF(!is_double_double<Float>::value)>
+ FMT_CONSTEXPR auto assign(Float n) -> bool {
+ static_assert(std::numeric_limits<Float>::digits <= 113, "unsupported FP");
+ // Assume Float is in the format [sign][exponent][significand].
+ using carrier_uint = typename dragonbox::float_info<Float>::carrier_uint;
+ const auto num_float_significand_bits =
+ detail::num_significand_bits<Float>();
+ const auto implicit_bit = carrier_uint(1) << num_float_significand_bits;
+ const auto significand_mask = implicit_bit - 1;
+ auto u = bit_cast<carrier_uint>(n);
+ f = static_cast<F>(u & significand_mask);
+ auto biased_e = static_cast<int>((u & exponent_mask<Float>()) >>
+ num_float_significand_bits);
+ // The predecessor is closer if n is a normalized power of 2 (f == 0)
+ // other than the smallest normalized number (biased_e > 1).
+ auto is_predecessor_closer = f == 0 && biased_e > 1;
+ if (biased_e == 0)
+ biased_e = 1; // Subnormals use biased exponent 1 (min exponent).
+ else if (has_implicit_bit<Float>())
+ f += static_cast<F>(implicit_bit);
+ e = biased_e - exponent_bias<Float>() - num_float_significand_bits;
+ if (!has_implicit_bit<Float>()) ++e;
+ return is_predecessor_closer;
+ }
+
+ template <typename Float, FMT_ENABLE_IF(is_double_double<Float>::value)>
+ FMT_CONSTEXPR auto assign(Float n) -> bool {
+ static_assert(std::numeric_limits<double>::is_iec559, "unsupported FP");
+ return assign(static_cast<double>(n));
+ }
+};
+
+using fp = basic_fp<unsigned long long>;
+
+// Normalizes the value converted from double and multiplied by (1 << SHIFT).
+template <int SHIFT = 0, typename F>
+FMT_CONSTEXPR basic_fp<F> normalize(basic_fp<F> value) {
+ // Handle subnormals.
+ const auto implicit_bit = F(1) << num_significand_bits<double>();
+ const auto shifted_implicit_bit = implicit_bit << SHIFT;
+ while ((value.f & shifted_implicit_bit) == 0) {
+ value.f <<= 1;
+ --value.e;
+ }
+ // Subtract 1 to account for hidden bit.
+ const auto offset = basic_fp<F>::num_significand_bits -
+ num_significand_bits<double>() - SHIFT - 1;
+ value.f <<= offset;
+ value.e -= offset;
+ return value;
+}
+
+// Computes lhs * rhs / pow(2, 64) rounded to nearest with half-up tie breaking.
+FMT_CONSTEXPR inline uint64_t multiply(uint64_t lhs, uint64_t rhs) {
+#if FMT_USE_INT128
+ auto product = static_cast<__uint128_t>(lhs) * rhs;
+ auto f = static_cast<uint64_t>(product >> 64);
+ return (static_cast<uint64_t>(product) & (1ULL << 63)) != 0 ? f + 1 : f;
+#else
+ // Multiply 32-bit parts of significands.
+ uint64_t mask = (1ULL << 32) - 1;
+ uint64_t a = lhs >> 32, b = lhs & mask;
+ uint64_t c = rhs >> 32, d = rhs & mask;
+ uint64_t ac = a * c, bc = b * c, ad = a * d, bd = b * d;
+ // Compute mid 64-bit of result and round.
+ uint64_t mid = (bd >> 32) + (ad & mask) + (bc & mask) + (1U << 31);
+ return ac + (ad >> 32) + (bc >> 32) + (mid >> 32);
+#endif
+}
+
+FMT_CONSTEXPR inline fp operator*(fp x, fp y) {
+ return {multiply(x.f, y.f), x.e + y.e + 64};
+}
+
+template <typename T = void> struct basic_data {
+ // Normalized 64-bit significands of pow(10, k), for k = -348, -340, ..., 340.
+ // These are generated by support/compute-powers.py.
+ static constexpr uint64_t pow10_significands[87] = {
+ 0xfa8fd5a0081c0288, 0xbaaee17fa23ebf76, 0x8b16fb203055ac76,
+ 0xcf42894a5dce35ea, 0x9a6bb0aa55653b2d, 0xe61acf033d1a45df,
+ 0xab70fe17c79ac6ca, 0xff77b1fcbebcdc4f, 0xbe5691ef416bd60c,
+ 0x8dd01fad907ffc3c, 0xd3515c2831559a83, 0x9d71ac8fada6c9b5,
+ 0xea9c227723ee8bcb, 0xaecc49914078536d, 0x823c12795db6ce57,
+ 0xc21094364dfb5637, 0x9096ea6f3848984f, 0xd77485cb25823ac7,
+ 0xa086cfcd97bf97f4, 0xef340a98172aace5, 0xb23867fb2a35b28e,
+ 0x84c8d4dfd2c63f3b, 0xc5dd44271ad3cdba, 0x936b9fcebb25c996,
+ 0xdbac6c247d62a584, 0xa3ab66580d5fdaf6, 0xf3e2f893dec3f126,
+ 0xb5b5ada8aaff80b8, 0x87625f056c7c4a8b, 0xc9bcff6034c13053,
+ 0x964e858c91ba2655, 0xdff9772470297ebd, 0xa6dfbd9fb8e5b88f,
+ 0xf8a95fcf88747d94, 0xb94470938fa89bcf, 0x8a08f0f8bf0f156b,
+ 0xcdb02555653131b6, 0x993fe2c6d07b7fac, 0xe45c10c42a2b3b06,
+ 0xaa242499697392d3, 0xfd87b5f28300ca0e, 0xbce5086492111aeb,
+ 0x8cbccc096f5088cc, 0xd1b71758e219652c, 0x9c40000000000000,
+ 0xe8d4a51000000000, 0xad78ebc5ac620000, 0x813f3978f8940984,
+ 0xc097ce7bc90715b3, 0x8f7e32ce7bea5c70, 0xd5d238a4abe98068,
+ 0x9f4f2726179a2245, 0xed63a231d4c4fb27, 0xb0de65388cc8ada8,
+ 0x83c7088e1aab65db, 0xc45d1df942711d9a, 0x924d692ca61be758,
+ 0xda01ee641a708dea, 0xa26da3999aef774a, 0xf209787bb47d6b85,
+ 0xb454e4a179dd1877, 0x865b86925b9bc5c2, 0xc83553c5c8965d3d,
+ 0x952ab45cfa97a0b3, 0xde469fbd99a05fe3, 0xa59bc234db398c25,
+ 0xf6c69a72a3989f5c, 0xb7dcbf5354e9bece, 0x88fcf317f22241e2,
+ 0xcc20ce9bd35c78a5, 0x98165af37b2153df, 0xe2a0b5dc971f303a,
+ 0xa8d9d1535ce3b396, 0xfb9b7cd9a4a7443c, 0xbb764c4ca7a44410,
+ 0x8bab8eefb6409c1a, 0xd01fef10a657842c, 0x9b10a4e5e9913129,
+ 0xe7109bfba19c0c9d, 0xac2820d9623bf429, 0x80444b5e7aa7cf85,
+ 0xbf21e44003acdd2d, 0x8e679c2f5e44ff8f, 0xd433179d9c8cb841,
+ 0x9e19db92b4e31ba9, 0xeb96bf6ebadf77d9, 0xaf87023b9bf0ee6b,
+ };
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wnarrowing"
+#endif
+ // Binary exponents of pow(10, k), for k = -348, -340, ..., 340, corresponding
+ // to significands above.
+ static constexpr int16_t pow10_exponents[87] = {
+ -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954,
+ -927, -901, -874, -847, -821, -794, -768, -741, -715, -688, -661,
+ -635, -608, -582, -555, -529, -502, -475, -449, -422, -396, -369,
+ -343, -316, -289, -263, -236, -210, -183, -157, -130, -103, -77,
+ -50, -24, 3, 30, 56, 83, 109, 136, 162, 189, 216,
+ 242, 269, 295, 322, 348, 375, 402, 428, 455, 481, 508,
+ 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800,
+ 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066};
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+# pragma GCC diagnostic pop
+#endif
+
+ static constexpr uint64_t power_of_10_64[20] = {
+ 1, FMT_POWERS_OF_10(1ULL), FMT_POWERS_OF_10(1000000000ULL),
+ 10000000000000000000ULL};
+
+ // For checking rounding thresholds.
+ // The kth entry is chosen to be the smallest integer such that the
+ // upper 32-bits of 10^(k+1) times it is strictly bigger than 5 * 10^k.
+ static constexpr uint32_t fractional_part_rounding_thresholds[8] = {
+ 2576980378, // ceil(2^31 + 2^32/10^1)
+ 2190433321, // ceil(2^31 + 2^32/10^2)
+ 2151778616, // ceil(2^31 + 2^32/10^3)
+ 2147913145, // ceil(2^31 + 2^32/10^4)
+ 2147526598, // ceil(2^31 + 2^32/10^5)
+ 2147487943, // ceil(2^31 + 2^32/10^6)
+ 2147484078, // ceil(2^31 + 2^32/10^7)
+ 2147483691 // ceil(2^31 + 2^32/10^8)
+ };
+};
+
+#if FMT_CPLUSPLUS < 201703L
+template <typename T> constexpr uint64_t basic_data<T>::pow10_significands[];
+template <typename T> constexpr int16_t basic_data<T>::pow10_exponents[];
+template <typename T> constexpr uint64_t basic_data<T>::power_of_10_64[];
+template <typename T>
+constexpr uint32_t basic_data<T>::fractional_part_rounding_thresholds[];
+#endif
+
+// This is a struct rather than an alias to avoid shadowing warnings in gcc.
+struct data : basic_data<> {};
+
+// Returns a cached power of 10 `c_k = c_k.f * pow(2, c_k.e)` such that its
+// (binary) exponent satisfies `min_exponent <= c_k.e <= min_exponent + 28`.
+FMT_CONSTEXPR inline fp get_cached_power(int min_exponent,
+ int& pow10_exponent) {
+ const int shift = 32;
+ // log10(2) = 0x0.4d104d427de7fbcc...
+ const int64_t significand = 0x4d104d427de7fbcc;
+ int index = static_cast<int>(
+ ((min_exponent + fp::num_significand_bits - 1) * (significand >> shift) +
+ ((int64_t(1) << shift) - 1)) // ceil
+ >> 32 // arithmetic shift
+ );
+ // Decimal exponent of the first (smallest) cached power of 10.
+ const int first_dec_exp = -348;
+ // Difference between 2 consecutive decimal exponents in cached powers of 10.
+ const int dec_exp_step = 8;
+ index = (index - first_dec_exp - 1) / dec_exp_step + 1;
+ pow10_exponent = first_dec_exp + index * dec_exp_step;
+ // Using *(x + index) instead of x[index] avoids an issue with some compilers
+ // using the EDG frontend (e.g. nvhpc/22.3 in C++17 mode).
+ return {*(data::pow10_significands + index),
+ *(data::pow10_exponents + index)};
+}
+
+template <typename T>
+using convert_float_result =
+ conditional_t<std::is_same<T, float>::value ||
+ std::numeric_limits<T>::digits ==
+ std::numeric_limits<double>::digits,
+ double, T>;
+
+template <typename T>
+constexpr auto convert_float(T value) -> convert_float_result<T> {
+ return static_cast<convert_float_result<T>>(value);
+}
+
+template <typename OutputIt, typename Char>
+FMT_NOINLINE FMT_CONSTEXPR auto fill(OutputIt it, size_t n,
+ const fill_t<Char>& fill) -> OutputIt {
+ auto fill_size = fill.size();
+ if (fill_size == 1) return detail::fill_n(it, n, fill[0]);
+ auto data = fill.data();
+ for (size_t i = 0; i < n; ++i)
+ it = copy_str<Char>(data, data + fill_size, it);
+ return it;
+}
+
+// Writes the output of f, padded according to format specifications in specs.
+// size: output size in code units.
+// width: output display width in (terminal) column positions.
+template <align::type align = align::left, typename OutputIt, typename Char,
+ typename F>
+FMT_CONSTEXPR auto write_padded(OutputIt out, const format_specs<Char>& specs,
+ size_t size, size_t width, F&& f) -> OutputIt {
+ static_assert(align == align::left || align == align::right, "");
+ unsigned spec_width = to_unsigned(specs.width);
+ size_t padding = spec_width > width ? spec_width - width : 0;
+ // Shifts are encoded as string literals because static constexpr is not
+ // supported in constexpr functions.
+ auto* shifts = align == align::left ? "\x1f\x1f\x00\x01" : "\x00\x1f\x00\x01";
+ size_t left_padding = padding >> shifts[specs.align];
+ size_t right_padding = padding - left_padding;
+ auto it = reserve(out, size + padding * specs.fill.size());
+ if (left_padding != 0) it = fill(it, left_padding, specs.fill);
+ it = f(it);
+ if (right_padding != 0) it = fill(it, right_padding, specs.fill);
+ return base_iterator(out, it);
+}
+
+template <align::type align = align::left, typename OutputIt, typename Char,
+ typename F>
+constexpr auto write_padded(OutputIt out, const format_specs<Char>& specs,
+ size_t size, F&& f) -> OutputIt {
+ return write_padded<align>(out, specs, size, size, f);
+}
+
+template <align::type align = align::left, typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write_bytes(OutputIt out, string_view bytes,
+ const format_specs<Char>& specs) -> OutputIt {
+ return write_padded<align>(
+ out, specs, bytes.size(), [bytes](reserve_iterator<OutputIt> it) {
+ const char* data = bytes.data();
+ return copy_str<Char>(data, data + bytes.size(), it);
+ });
+}
+
+template <typename Char, typename OutputIt, typename UIntPtr>
+auto write_ptr(OutputIt out, UIntPtr value, const format_specs<Char>* specs)
+ -> OutputIt {
+ int num_digits = count_digits<4>(value);
+ auto size = to_unsigned(num_digits) + size_t(2);
+ auto write = [=](reserve_iterator<OutputIt> it) {
+ *it++ = static_cast<Char>('0');
+ *it++ = static_cast<Char>('x');
+ return format_uint<4, Char>(it, value, num_digits);
+ };
+ return specs ? write_padded<align::right>(out, *specs, size, write)
+ : base_iterator(out, write(reserve(out, size)));
+}
+
+// Returns true iff the code point cp is printable.
+FMT_API auto is_printable(uint32_t cp) -> bool;
+
+inline auto needs_escape(uint32_t cp) -> bool {
+ return cp < 0x20 || cp == 0x7f || cp == '"' || cp == '\\' ||
+ !is_printable(cp);
+}
+
+template <typename Char> struct find_escape_result {
+ const Char* begin;
+ const Char* end;
+ uint32_t cp;
+};
+
+template <typename Char>
+using make_unsigned_char =
+ typename conditional_t<std::is_integral<Char>::value,
+ std::make_unsigned<Char>,
+ type_identity<uint32_t>>::type;
+
+template <typename Char>
+auto find_escape(const Char* begin, const Char* end)
+ -> find_escape_result<Char> {
+ for (; begin != end; ++begin) {
+ uint32_t cp = static_cast<make_unsigned_char<Char>>(*begin);
+ if (const_check(sizeof(Char) == 1) && cp >= 0x80) continue;
+ if (needs_escape(cp)) return {begin, begin + 1, cp};
+ }
+ return {begin, nullptr, 0};
+}
+
+inline auto find_escape(const char* begin, const char* end)
+ -> find_escape_result<char> {
+ if (!is_utf8()) return find_escape<char>(begin, end);
+ auto result = find_escape_result<char>{end, nullptr, 0};
+ for_each_codepoint(string_view(begin, to_unsigned(end - begin)),
+ [&](uint32_t cp, string_view sv) {
+ if (needs_escape(cp)) {
+ result = {sv.begin(), sv.end(), cp};
+ return false;
+ }
+ return true;
+ });
+ return result;
+}
+
+#define FMT_STRING_IMPL(s, base, explicit) \
+ [] { \
+ /* Use the hidden visibility as a workaround for a GCC bug (#1973). */ \
+ /* Use a macro-like name to avoid shadowing warnings. */ \
+ struct FMT_GCC_VISIBILITY_HIDDEN FMT_COMPILE_STRING : base { \
+ using char_type FMT_MAYBE_UNUSED = fmt::remove_cvref_t<decltype(s[0])>; \
+ FMT_MAYBE_UNUSED FMT_CONSTEXPR explicit \
+ operator fmt::basic_string_view<char_type>() const { \
+ return fmt::detail_exported::compile_string_to_view<char_type>(s); \
+ } \
+ }; \
+ return FMT_COMPILE_STRING(); \
+ }()
+
+/**
+ \rst
+ Constructs a compile-time format string from a string literal *s*.
+
+ **Example**::
+
+ // A compile-time error because 'd' is an invalid specifier for strings.
+ std::string s = fmt::format(FMT_STRING("{:d}"), "foo");
+ \endrst
+ */
+#define FMT_STRING(s) FMT_STRING_IMPL(s, fmt::detail::compile_string, )
+
+template <size_t width, typename Char, typename OutputIt>
+auto write_codepoint(OutputIt out, char prefix, uint32_t cp) -> OutputIt {
+ *out++ = static_cast<Char>('\\');
+ *out++ = static_cast<Char>(prefix);
+ Char buf[width];
+ fill_n(buf, width, static_cast<Char>('0'));
+ format_uint<4>(buf, cp, width);
+ return copy_str<Char>(buf, buf + width, out);
+}
+
+template <typename OutputIt, typename Char>
+auto write_escaped_cp(OutputIt out, const find_escape_result<Char>& escape)
+ -> OutputIt {
+ auto c = static_cast<Char>(escape.cp);
+ switch (escape.cp) {
+ case '\n':
+ *out++ = static_cast<Char>('\\');
+ c = static_cast<Char>('n');
+ break;
+ case '\r':
+ *out++ = static_cast<Char>('\\');
+ c = static_cast<Char>('r');
+ break;
+ case '\t':
+ *out++ = static_cast<Char>('\\');
+ c = static_cast<Char>('t');
+ break;
+ case '"':
+ FMT_FALLTHROUGH;
+ case '\'':
+ FMT_FALLTHROUGH;
+ case '\\':
+ *out++ = static_cast<Char>('\\');
+ break;
+ default:
+ if (escape.cp < 0x100) {
+ return write_codepoint<2, Char>(out, 'x', escape.cp);
+ }
+ if (escape.cp < 0x10000) {
+ return write_codepoint<4, Char>(out, 'u', escape.cp);
+ }
+ if (escape.cp < 0x110000) {
+ return write_codepoint<8, Char>(out, 'U', escape.cp);
+ }
+ for (Char escape_char : basic_string_view<Char>(
+ escape.begin, to_unsigned(escape.end - escape.begin))) {
+ out = write_codepoint<2, Char>(out, 'x',
+ static_cast<uint32_t>(escape_char) & 0xFF);
+ }
+ return out;
+ }
+ *out++ = c;
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+auto write_escaped_string(OutputIt out, basic_string_view<Char> str)
+ -> OutputIt {
+ *out++ = static_cast<Char>('"');
+ auto begin = str.begin(), end = str.end();
+ do {
+ auto escape = find_escape(begin, end);
+ out = copy_str<Char>(begin, escape.begin, out);
+ begin = escape.end;
+ if (!begin) break;
+ out = write_escaped_cp<OutputIt, Char>(out, escape);
+ } while (begin != end);
+ *out++ = static_cast<Char>('"');
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+auto write_escaped_char(OutputIt out, Char v) -> OutputIt {
+ *out++ = static_cast<Char>('\'');
+ if ((needs_escape(static_cast<uint32_t>(v)) && v != static_cast<Char>('"')) ||
+ v == static_cast<Char>('\'')) {
+ out = write_escaped_cp(
+ out, find_escape_result<Char>{&v, &v + 1, static_cast<uint32_t>(v)});
+ } else {
+ *out++ = v;
+ }
+ *out++ = static_cast<Char>('\'');
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write_char(OutputIt out, Char value,
+ const format_specs<Char>& specs) -> OutputIt {
+ bool is_debug = specs.type == presentation_type::debug;
+ return write_padded(out, specs, 1, [=](reserve_iterator<OutputIt> it) {
+ if (is_debug) return write_escaped_char(it, value);
+ *it++ = value;
+ return it;
+ });
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, Char value,
+ const format_specs<Char>& specs, locale_ref loc = {})
+ -> OutputIt {
+ // char is formatted as unsigned char for consistency across platforms.
+ using unsigned_type =
+ conditional_t<std::is_same<Char, char>::value, unsigned char, unsigned>;
+ return check_char_specs(specs)
+ ? write_char(out, value, specs)
+ : write(out, static_cast<unsigned_type>(value), specs, loc);
+}
+
+// Data for write_int that doesn't depend on output iterator type. It is used to
+// avoid template code bloat.
+template <typename Char> struct write_int_data {
+ size_t size;
+ size_t padding;
+
+ FMT_CONSTEXPR write_int_data(int num_digits, unsigned prefix,
+ const format_specs<Char>& specs)
+ : size((prefix >> 24) + to_unsigned(num_digits)), padding(0) {
+ if (specs.align == align::numeric) {
+ auto width = to_unsigned(specs.width);
+ if (width > size) {
+ padding = width - size;
+ size = width;
+ }
+ } else if (specs.precision > num_digits) {
+ size = (prefix >> 24) + to_unsigned(specs.precision);
+ padding = to_unsigned(specs.precision - num_digits);
+ }
+ }
+};
+
+// Writes an integer in the format
+// <left-padding><prefix><numeric-padding><digits><right-padding>
+// where <digits> are written by write_digits(it).
+// prefix contains chars in three lower bytes and the size in the fourth byte.
+template <typename OutputIt, typename Char, typename W>
+FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits,
+ unsigned prefix,
+ const format_specs<Char>& specs,
+ W write_digits) -> OutputIt {
+ // Slightly faster check for specs.width == 0 && specs.precision == -1.
+ if ((specs.width | (specs.precision + 1)) == 0) {
+ auto it = reserve(out, to_unsigned(num_digits) + (prefix >> 24));
+ if (prefix != 0) {
+ for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8)
+ *it++ = static_cast<Char>(p & 0xff);
+ }
+ return base_iterator(out, write_digits(it));
+ }
+ auto data = write_int_data<Char>(num_digits, prefix, specs);
+ return write_padded<align::right>(
+ out, specs, data.size, [=](reserve_iterator<OutputIt> it) {
+ for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8)
+ *it++ = static_cast<Char>(p & 0xff);
+ it = detail::fill_n(it, data.padding, static_cast<Char>('0'));
+ return write_digits(it);
+ });
+}
+
+template <typename Char> class digit_grouping {
+ private:
+ std::string grouping_;
+ std::basic_string<Char> thousands_sep_;
+
+ struct next_state {
+ std::string::const_iterator group;
+ int pos;
+ };
+ next_state initial_state() const { return {grouping_.begin(), 0}; }
+
+ // Returns the next digit group separator position.
+ int next(next_state& state) const {
+ if (thousands_sep_.empty()) return max_value<int>();
+ if (state.group == grouping_.end()) return state.pos += grouping_.back();
+ if (*state.group <= 0 || *state.group == max_value<char>())
+ return max_value<int>();
+ state.pos += *state.group++;
+ return state.pos;
+ }
+
+ public:
+ explicit digit_grouping(locale_ref loc, bool localized = true) {
+ if (!localized) return;
+ auto sep = thousands_sep<Char>(loc);
+ grouping_ = sep.grouping;
+ if (sep.thousands_sep) thousands_sep_.assign(1, sep.thousands_sep);
+ }
+ digit_grouping(std::string grouping, std::basic_string<Char> sep)
+ : grouping_(std::move(grouping)), thousands_sep_(std::move(sep)) {}
+
+ bool has_separator() const { return !thousands_sep_.empty(); }
+
+ int count_separators(int num_digits) const {
+ int count = 0;
+ auto state = initial_state();
+ while (num_digits > next(state)) ++count;
+ return count;
+ }
+
+ // Applies grouping to digits and write the output to out.
+ template <typename Out, typename C>
+ Out apply(Out out, basic_string_view<C> digits) const {
+ auto num_digits = static_cast<int>(digits.size());
+ auto separators = basic_memory_buffer<int>();
+ separators.push_back(0);
+ auto state = initial_state();
+ while (int i = next(state)) {
+ if (i >= num_digits) break;
+ separators.push_back(i);
+ }
+ for (int i = 0, sep_index = static_cast<int>(separators.size() - 1);
+ i < num_digits; ++i) {
+ if (num_digits - i == separators[sep_index]) {
+ out =
+ copy_str<Char>(thousands_sep_.data(),
+ thousands_sep_.data() + thousands_sep_.size(), out);
+ --sep_index;
+ }
+ *out++ = static_cast<Char>(digits[to_unsigned(i)]);
+ }
+ return out;
+ }
+};
+
+// Writes a decimal integer with digit grouping.
+template <typename OutputIt, typename UInt, typename Char>
+auto write_int(OutputIt out, UInt value, unsigned prefix,
+ const format_specs<Char>& specs,
+ const digit_grouping<Char>& grouping) -> OutputIt {
+ static_assert(std::is_same<uint64_or_128_t<UInt>, UInt>::value, "");
+ int num_digits = count_digits(value);
+ char digits[40];
+ format_decimal(digits, value, num_digits);
+ unsigned size = to_unsigned((prefix != 0 ? 1 : 0) + num_digits +
+ grouping.count_separators(num_digits));
+ return write_padded<align::right>(
+ out, specs, size, size, [&](reserve_iterator<OutputIt> it) {
+ if (prefix != 0) {
+ char sign = static_cast<char>(prefix);
+ *it++ = static_cast<Char>(sign);
+ }
+ return grouping.apply(it, string_view(digits, to_unsigned(num_digits)));
+ });
+}
+
+// Writes a localized value.
+FMT_API auto write_loc(appender out, loc_value value,
+ const format_specs<>& specs, locale_ref loc) -> bool;
+template <typename OutputIt, typename Char>
+inline auto write_loc(OutputIt, loc_value, const format_specs<Char>&,
+ locale_ref) -> bool {
+ return false;
+}
+
+FMT_CONSTEXPR inline void prefix_append(unsigned& prefix, unsigned value) {
+ prefix |= prefix != 0 ? value << 8 : value;
+ prefix += (1u + (value > 0xff ? 1 : 0)) << 24;
+}
+
+template <typename UInt> struct write_int_arg {
+ UInt abs_value;
+ unsigned prefix;
+};
+
+template <typename T>
+FMT_CONSTEXPR auto make_write_int_arg(T value, sign_t sign)
+ -> write_int_arg<uint32_or_64_or_128_t<T>> {
+ auto prefix = 0u;
+ auto abs_value = static_cast<uint32_or_64_or_128_t<T>>(value);
+ if (is_negative(value)) {
+ prefix = 0x01000000 | '-';
+ abs_value = 0 - abs_value;
+ } else {
+ constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+',
+ 0x1000000u | ' '};
+ prefix = prefixes[sign];
+ }
+ return {abs_value, prefix};
+}
+
+template <typename Char = char> struct loc_writer {
+ buffer_appender<Char> out;
+ const format_specs<Char>& specs;
+ std::basic_string<Char> sep;
+ std::string grouping;
+ std::basic_string<Char> decimal_point;
+
+ template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+ auto operator()(T value) -> bool {
+ auto arg = make_write_int_arg(value, specs.sign);
+ write_int(out, static_cast<uint64_or_128_t<T>>(arg.abs_value), arg.prefix,
+ specs, digit_grouping<Char>(grouping, sep));
+ return true;
+ }
+
+ template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+ auto operator()(T) -> bool {
+ return false;
+ }
+};
+
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg<T> arg,
+ const format_specs<Char>& specs,
+ locale_ref) -> OutputIt {
+ static_assert(std::is_same<T, uint32_or_64_or_128_t<T>>::value, "");
+ auto abs_value = arg.abs_value;
+ auto prefix = arg.prefix;
+ switch (specs.type) {
+ case presentation_type::none:
+ case presentation_type::dec: {
+ auto num_digits = count_digits(abs_value);
+ return write_int(
+ out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+ return format_decimal<Char>(it, abs_value, num_digits).end;
+ });
+ }
+ case presentation_type::hex_lower:
+ case presentation_type::hex_upper: {
+ bool upper = specs.type == presentation_type::hex_upper;
+ if (specs.alt)
+ prefix_append(prefix, unsigned(upper ? 'X' : 'x') << 8 | '0');
+ int num_digits = count_digits<4>(abs_value);
+ return write_int(
+ out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+ return format_uint<4, Char>(it, abs_value, num_digits, upper);
+ });
+ }
+ case presentation_type::bin_lower:
+ case presentation_type::bin_upper: {
+ bool upper = specs.type == presentation_type::bin_upper;
+ if (specs.alt)
+ prefix_append(prefix, unsigned(upper ? 'B' : 'b') << 8 | '0');
+ int num_digits = count_digits<1>(abs_value);
+ return write_int(out, num_digits, prefix, specs,
+ [=](reserve_iterator<OutputIt> it) {
+ return format_uint<1, Char>(it, abs_value, num_digits);
+ });
+ }
+ case presentation_type::oct: {
+ int num_digits = count_digits<3>(abs_value);
+ // Octal prefix '0' is counted as a digit, so only add it if precision
+ // is not greater than the number of digits.
+ if (specs.alt && specs.precision <= num_digits && abs_value != 0)
+ prefix_append(prefix, '0');
+ return write_int(out, num_digits, prefix, specs,
+ [=](reserve_iterator<OutputIt> it) {
+ return format_uint<3, Char>(it, abs_value, num_digits);
+ });
+ }
+ case presentation_type::chr:
+ return write_char(out, static_cast<Char>(abs_value), specs);
+ default:
+ throw_format_error("invalid format specifier");
+ }
+ return out;
+}
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR FMT_NOINLINE auto write_int_noinline(
+ OutputIt out, write_int_arg<T> arg, const format_specs<Char>& specs,
+ locale_ref loc) -> OutputIt {
+ return write_int(out, arg, specs, loc);
+}
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_integral<T>::value &&
+ !std::is_same<T, bool>::value &&
+ std::is_same<OutputIt, buffer_appender<Char>>::value)>
+FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value,
+ const format_specs<Char>& specs,
+ locale_ref loc) -> OutputIt {
+ if (specs.localized && write_loc(out, value, specs, loc)) return out;
+ return write_int_noinline(out, make_write_int_arg(value, specs.sign), specs,
+ loc);
+}
+// An inlined version of write used in format string compilation.
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_integral<T>::value &&
+ !std::is_same<T, bool>::value &&
+ !std::is_same<OutputIt, buffer_appender<Char>>::value)>
+FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value,
+ const format_specs<Char>& specs,
+ locale_ref loc) -> OutputIt {
+ if (specs.localized && write_loc(out, value, specs, loc)) return out;
+ return write_int(out, make_write_int_arg(value, specs.sign), specs, loc);
+}
+
+// An output iterator that counts the number of objects written to it and
+// discards them.
+class counting_iterator {
+ private:
+ size_t count_;
+
+ public:
+ using iterator_category = std::output_iterator_tag;
+ using difference_type = std::ptrdiff_t;
+ using pointer = void;
+ using reference = void;
+ FMT_UNCHECKED_ITERATOR(counting_iterator);
+
+ struct value_type {
+ template <typename T> FMT_CONSTEXPR void operator=(const T&) {}
+ };
+
+ FMT_CONSTEXPR counting_iterator() : count_(0) {}
+
+ FMT_CONSTEXPR size_t count() const { return count_; }
+
+ FMT_CONSTEXPR counting_iterator& operator++() {
+ ++count_;
+ return *this;
+ }
+ FMT_CONSTEXPR counting_iterator operator++(int) {
+ auto it = *this;
+ ++*this;
+ return it;
+ }
+
+ FMT_CONSTEXPR friend counting_iterator operator+(counting_iterator it,
+ difference_type n) {
+ it.count_ += static_cast<size_t>(n);
+ return it;
+ }
+
+ FMT_CONSTEXPR value_type operator*() const { return {}; }
+};
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, basic_string_view<Char> s,
+ const format_specs<Char>& specs) -> OutputIt {
+ auto data = s.data();
+ auto size = s.size();
+ if (specs.precision >= 0 && to_unsigned(specs.precision) < size)
+ size = code_point_index(s, to_unsigned(specs.precision));
+ bool is_debug = specs.type == presentation_type::debug;
+ size_t width = 0;
+ if (specs.width != 0) {
+ if (is_debug)
+ width = write_escaped_string(counting_iterator{}, s).count();
+ else
+ width = compute_width(basic_string_view<Char>(data, size));
+ }
+ return write_padded(out, specs, size, width,
+ [=](reserve_iterator<OutputIt> it) {
+ if (is_debug) return write_escaped_string(it, s);
+ return copy_str<Char>(data, data + size, it);
+ });
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out,
+ basic_string_view<type_identity_t<Char>> s,
+ const format_specs<Char>& specs, locale_ref)
+ -> OutputIt {
+ return write(out, s, specs);
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, const Char* s,
+ const format_specs<Char>& specs, locale_ref)
+ -> OutputIt {
+ return specs.type != presentation_type::pointer
+ ? write(out, basic_string_view<Char>(s), specs, {})
+ : write_ptr<Char>(out, bit_cast<uintptr_t>(s), &specs);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_integral<T>::value &&
+ !std::is_same<T, bool>::value &&
+ !std::is_same<T, Char>::value)>
+FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt {
+ auto abs_value = static_cast<uint32_or_64_or_128_t<T>>(value);
+ bool negative = is_negative(value);
+ // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer.
+ if (negative) abs_value = ~abs_value + 1;
+ int num_digits = count_digits(abs_value);
+ auto size = (negative ? 1 : 0) + static_cast<size_t>(num_digits);
+ auto it = reserve(out, size);
+ if (auto ptr = to_pointer<Char>(it, size)) {
+ if (negative) *ptr++ = static_cast<Char>('-');
+ format_decimal<Char>(ptr, abs_value, num_digits);
+ return out;
+ }
+ if (negative) *it++ = static_cast<Char>('-');
+ it = format_decimal<Char>(it, abs_value, num_digits).end;
+ return base_iterator(out, it);
+}
+
+// A floating-point presentation format.
+enum class float_format : unsigned char {
+ general, // General: exponent notation or fixed point based on magnitude.
+ exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3.
+ fixed, // Fixed point with the default precision of 6, e.g. 0.0012.
+ hex
+};
+
+struct float_specs {
+ int precision;
+ float_format format : 8;
+ sign_t sign : 8;
+ bool upper : 1;
+ bool locale : 1;
+ bool binary32 : 1;
+ bool showpoint : 1;
+};
+
+template <typename ErrorHandler = error_handler, typename Char>
+FMT_CONSTEXPR auto parse_float_type_spec(const format_specs<Char>& specs,
+ ErrorHandler&& eh = {})
+ -> float_specs {
+ auto result = float_specs();
+ result.showpoint = specs.alt;
+ result.locale = specs.localized;
+ switch (specs.type) {
+ case presentation_type::none:
+ result.format = float_format::general;
+ break;
+ case presentation_type::general_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::general_lower:
+ result.format = float_format::general;
+ break;
+ case presentation_type::exp_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::exp_lower:
+ result.format = float_format::exp;
+ result.showpoint |= specs.precision != 0;
+ break;
+ case presentation_type::fixed_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::fixed_lower:
+ result.format = float_format::fixed;
+ result.showpoint |= specs.precision != 0;
+ break;
+ case presentation_type::hexfloat_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::hexfloat_lower:
+ result.format = float_format::hex;
+ break;
+ default:
+ eh.on_error("invalid format specifier");
+ break;
+ }
+ return result;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR20 auto write_nonfinite(OutputIt out, bool isnan,
+ format_specs<Char> specs,
+ const float_specs& fspecs) -> OutputIt {
+ auto str =
+ isnan ? (fspecs.upper ? "NAN" : "nan") : (fspecs.upper ? "INF" : "inf");
+ constexpr size_t str_size = 3;
+ auto sign = fspecs.sign;
+ auto size = str_size + (sign ? 1 : 0);
+ // Replace '0'-padding with space for non-finite values.
+ const bool is_zero_fill =
+ specs.fill.size() == 1 && *specs.fill.data() == static_cast<Char>('0');
+ if (is_zero_fill) specs.fill[0] = static_cast<Char>(' ');
+ return write_padded(out, specs, size, [=](reserve_iterator<OutputIt> it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ return copy_str<Char>(str, str + str_size, it);
+ });
+}
+
+// A decimal floating-point number significand * pow(10, exp).
+struct big_decimal_fp {
+ const char* significand;
+ int significand_size;
+ int exponent;
+};
+
+constexpr auto get_significand_size(const big_decimal_fp& f) -> int {
+ return f.significand_size;
+}
+template <typename T>
+inline auto get_significand_size(const dragonbox::decimal_fp<T>& f) -> int {
+ return count_digits(f.significand);
+}
+
+template <typename Char, typename OutputIt>
+constexpr auto write_significand(OutputIt out, const char* significand,
+ int significand_size) -> OutputIt {
+ return copy_str<Char>(significand, significand + significand_size, out);
+}
+template <typename Char, typename OutputIt, typename UInt>
+inline auto write_significand(OutputIt out, UInt significand,
+ int significand_size) -> OutputIt {
+ return format_decimal<Char>(out, significand, significand_size).end;
+}
+template <typename Char, typename OutputIt, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+ int significand_size, int exponent,
+ const Grouping& grouping) -> OutputIt {
+ if (!grouping.has_separator()) {
+ out = write_significand<Char>(out, significand, significand_size);
+ return detail::fill_n(out, exponent, static_cast<Char>('0'));
+ }
+ auto buffer = memory_buffer();
+ write_significand<char>(appender(buffer), significand, significand_size);
+ detail::fill_n(appender(buffer), exponent, '0');
+ return grouping.apply(out, string_view(buffer.data(), buffer.size()));
+}
+
+template <typename Char, typename UInt,
+ FMT_ENABLE_IF(std::is_integral<UInt>::value)>
+inline auto write_significand(Char* out, UInt significand, int significand_size,
+ int integral_size, Char decimal_point) -> Char* {
+ if (!decimal_point)
+ return format_decimal(out, significand, significand_size).end;
+ out += significand_size + 1;
+ Char* end = out;
+ int floating_size = significand_size - integral_size;
+ for (int i = floating_size / 2; i > 0; --i) {
+ out -= 2;
+ copy2(out, digits2(static_cast<std::size_t>(significand % 100)));
+ significand /= 100;
+ }
+ if (floating_size % 2 != 0) {
+ *--out = static_cast<Char>('0' + significand % 10);
+ significand /= 10;
+ }
+ *--out = decimal_point;
+ format_decimal(out - integral_size, significand, integral_size);
+ return end;
+}
+
+template <typename OutputIt, typename UInt, typename Char,
+ FMT_ENABLE_IF(!std::is_pointer<remove_cvref_t<OutputIt>>::value)>
+inline auto write_significand(OutputIt out, UInt significand,
+ int significand_size, int integral_size,
+ Char decimal_point) -> OutputIt {
+ // Buffer is large enough to hold digits (digits10 + 1) and a decimal point.
+ Char buffer[digits10<UInt>() + 2];
+ auto end = write_significand(buffer, significand, significand_size,
+ integral_size, decimal_point);
+ return detail::copy_str_noinline<Char>(buffer, end, out);
+}
+
+template <typename OutputIt, typename Char>
+FMT_CONSTEXPR auto write_significand(OutputIt out, const char* significand,
+ int significand_size, int integral_size,
+ Char decimal_point) -> OutputIt {
+ out = detail::copy_str_noinline<Char>(significand,
+ significand + integral_size, out);
+ if (!decimal_point) return out;
+ *out++ = decimal_point;
+ return detail::copy_str_noinline<Char>(significand + integral_size,
+ significand + significand_size, out);
+}
+
+template <typename OutputIt, typename Char, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+ int significand_size, int integral_size,
+ Char decimal_point,
+ const Grouping& grouping) -> OutputIt {
+ if (!grouping.has_separator()) {
+ return write_significand(out, significand, significand_size, integral_size,
+ decimal_point);
+ }
+ auto buffer = basic_memory_buffer<Char>();
+ write_significand(buffer_appender<Char>(buffer), significand,
+ significand_size, integral_size, decimal_point);
+ grouping.apply(
+ out, basic_string_view<Char>(buffer.data(), to_unsigned(integral_size)));
+ return detail::copy_str_noinline<Char>(buffer.data() + integral_size,
+ buffer.end(), out);
+}
+
+template <typename OutputIt, typename DecimalFP, typename Char,
+ typename Grouping = digit_grouping<Char>>
+FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& f,
+ const format_specs<Char>& specs,
+ float_specs fspecs, locale_ref loc)
+ -> OutputIt {
+ auto significand = f.significand;
+ int significand_size = get_significand_size(f);
+ const Char zero = static_cast<Char>('0');
+ auto sign = fspecs.sign;
+ size_t size = to_unsigned(significand_size) + (sign ? 1 : 0);
+ using iterator = reserve_iterator<OutputIt>;
+
+ Char decimal_point =
+ fspecs.locale ? detail::decimal_point<Char>(loc) : static_cast<Char>('.');
+
+ int output_exp = f.exponent + significand_size - 1;
+ auto use_exp_format = [=]() {
+ if (fspecs.format == float_format::exp) return true;
+ if (fspecs.format != float_format::general) return false;
+ // Use the fixed notation if the exponent is in [exp_lower, exp_upper),
+ // e.g. 0.0001 instead of 1e-04. Otherwise use the exponent notation.
+ const int exp_lower = -4, exp_upper = 16;
+ return output_exp < exp_lower ||
+ output_exp >= (fspecs.precision > 0 ? fspecs.precision : exp_upper);
+ };
+ if (use_exp_format()) {
+ int num_zeros = 0;
+ if (fspecs.showpoint) {
+ num_zeros = fspecs.precision - significand_size;
+ if (num_zeros < 0) num_zeros = 0;
+ size += to_unsigned(num_zeros);
+ } else if (significand_size == 1) {
+ decimal_point = Char();
+ }
+ auto abs_output_exp = output_exp >= 0 ? output_exp : -output_exp;
+ int exp_digits = 2;
+ if (abs_output_exp >= 100) exp_digits = abs_output_exp >= 1000 ? 4 : 3;
+
+ size += to_unsigned((decimal_point ? 1 : 0) + 2 + exp_digits);
+ char exp_char = fspecs.upper ? 'E' : 'e';
+ auto write = [=](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ // Insert a decimal point after the first digit and add an exponent.
+ it = write_significand(it, significand, significand_size, 1,
+ decimal_point);
+ if (num_zeros > 0) it = detail::fill_n(it, num_zeros, zero);
+ *it++ = static_cast<Char>(exp_char);
+ return write_exponent<Char>(output_exp, it);
+ };
+ return specs.width > 0 ? write_padded<align::right>(out, specs, size, write)
+ : base_iterator(out, write(reserve(out, size)));
+ }
+
+ int exp = f.exponent + significand_size;
+ if (f.exponent >= 0) {
+ // 1234e5 -> 123400000[.0+]
+ size += to_unsigned(f.exponent);
+ int num_zeros = fspecs.precision - exp;
+ abort_fuzzing_if(num_zeros > 5000);
+ if (fspecs.showpoint) {
+ ++size;
+ if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 0;
+ if (num_zeros > 0) size += to_unsigned(num_zeros);
+ }
+ auto grouping = Grouping(loc, fspecs.locale);
+ size += to_unsigned(grouping.count_separators(exp));
+ return write_padded<align::right>(out, specs, size, [&](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ it = write_significand<Char>(it, significand, significand_size,
+ f.exponent, grouping);
+ if (!fspecs.showpoint) return it;
+ *it++ = decimal_point;
+ return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
+ });
+ } else if (exp > 0) {
+ // 1234e-2 -> 12.34[0+]
+ int num_zeros = fspecs.showpoint ? fspecs.precision - significand_size : 0;
+ size += 1 + to_unsigned(num_zeros > 0 ? num_zeros : 0);
+ auto grouping = Grouping(loc, fspecs.locale);
+ size += to_unsigned(grouping.count_separators(exp));
+ return write_padded<align::right>(out, specs, size, [&](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ it = write_significand(it, significand, significand_size, exp,
+ decimal_point, grouping);
+ return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
+ });
+ }
+ // 1234e-6 -> 0.001234
+ int num_zeros = -exp;
+ if (significand_size == 0 && fspecs.precision >= 0 &&
+ fspecs.precision < num_zeros) {
+ num_zeros = fspecs.precision;
+ }
+ bool pointy = num_zeros != 0 || significand_size != 0 || fspecs.showpoint;
+ size += 1 + (pointy ? 1 : 0) + to_unsigned(num_zeros);
+ return write_padded<align::right>(out, specs, size, [&](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ *it++ = zero;
+ if (!pointy) return it;
+ *it++ = decimal_point;
+ it = detail::fill_n(it, num_zeros, zero);
+ return write_significand<Char>(it, significand, significand_size);
+ });
+}
+
+template <typename Char> class fallback_digit_grouping {
+ public:
+ constexpr fallback_digit_grouping(locale_ref, bool) {}
+
+ constexpr bool has_separator() const { return false; }
+
+ constexpr int count_separators(int) const { return 0; }
+
+ template <typename Out, typename C>
+ constexpr Out apply(Out out, basic_string_view<C>) const {
+ return out;
+ }
+};
+
+template <typename OutputIt, typename DecimalFP, typename Char>
+FMT_CONSTEXPR20 auto write_float(OutputIt out, const DecimalFP& f,
+ const format_specs<Char>& specs,
+ float_specs fspecs, locale_ref loc)
+ -> OutputIt {
+ if (is_constant_evaluated()) {
+ return do_write_float<OutputIt, DecimalFP, Char,
+ fallback_digit_grouping<Char>>(out, f, specs, fspecs,
+ loc);
+ } else {
+ return do_write_float(out, f, specs, fspecs, loc);
+ }
+}
+
+template <typename T> constexpr bool isnan(T value) {
+ return !(value >= value); // std::isnan doesn't support __float128.
+}
+
+template <typename T, typename Enable = void>
+struct has_isfinite : std::false_type {};
+
+template <typename T>
+struct has_isfinite<T, enable_if_t<sizeof(std::isfinite(T())) != 0>>
+ : std::true_type {};
+
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value&&
+ has_isfinite<T>::value)>
+FMT_CONSTEXPR20 bool isfinite(T value) {
+ constexpr T inf = T(std::numeric_limits<double>::infinity());
+ if (is_constant_evaluated())
+ return !detail::isnan(value) && value < inf && value > -inf;
+ return std::isfinite(value);
+}
+template <typename T, FMT_ENABLE_IF(!has_isfinite<T>::value)>
+FMT_CONSTEXPR bool isfinite(T value) {
+ T inf = T(std::numeric_limits<double>::infinity());
+ // std::isfinite doesn't support __float128.
+ return !detail::isnan(value) && value < inf && value > -inf;
+}
+
+template <typename T, FMT_ENABLE_IF(is_floating_point<T>::value)>
+FMT_INLINE FMT_CONSTEXPR bool signbit(T value) {
+ if (is_constant_evaluated()) {
+#ifdef __cpp_if_constexpr
+ if constexpr (std::numeric_limits<double>::is_iec559) {
+ auto bits = detail::bit_cast<uint64_t>(static_cast<double>(value));
+ return (bits >> (num_bits<uint64_t>() - 1)) != 0;
+ }
+#endif
+ }
+ return std::signbit(static_cast<double>(value));
+}
+
+enum class round_direction { unknown, up, down };
+
+// Given the divisor (normally a power of 10), the remainder = v % divisor for
+// some number v and the error, returns whether v should be rounded up, down, or
+// whether the rounding direction can't be determined due to error.
+// error should be less than divisor / 2.
+FMT_CONSTEXPR inline round_direction get_round_direction(uint64_t divisor,
+ uint64_t remainder,
+ uint64_t error) {
+ FMT_ASSERT(remainder < divisor, ""); // divisor - remainder won't overflow.
+ FMT_ASSERT(error < divisor, ""); // divisor - error won't overflow.
+ FMT_ASSERT(error < divisor - error, ""); // error * 2 won't overflow.
+ // Round down if (remainder + error) * 2 <= divisor.
+ if (remainder <= divisor - remainder && error * 2 <= divisor - remainder * 2)
+ return round_direction::down;
+ // Round up if (remainder - error) * 2 >= divisor.
+ if (remainder >= error &&
+ remainder - error >= divisor - (remainder - error)) {
+ return round_direction::up;
+ }
+ return round_direction::unknown;
+}
+
+namespace digits {
+enum result {
+ more, // Generate more digits.
+ done, // Done generating digits.
+ error // Digit generation cancelled due to an error.
+};
+}
+
+struct gen_digits_handler {
+ char* buf;
+ int size;
+ int precision;
+ int exp10;
+ bool fixed;
+
+ FMT_CONSTEXPR digits::result on_digit(char digit, uint64_t divisor,
+ uint64_t remainder, uint64_t error,
+ bool integral) {
+ FMT_ASSERT(remainder < divisor, "");
+ buf[size++] = digit;
+ if (!integral && error >= remainder) return digits::error;
+ if (size < precision) return digits::more;
+ if (!integral) {
+ // Check if error * 2 < divisor with overflow prevention.
+ // The check is not needed for the integral part because error = 1
+ // and divisor > (1 << 32) there.
+ if (error >= divisor || error >= divisor - error) return digits::error;
+ } else {
+ FMT_ASSERT(error == 1 && divisor > 2, "");
+ }
+ auto dir = get_round_direction(divisor, remainder, error);
+ if (dir != round_direction::up)
+ return dir == round_direction::down ? digits::done : digits::error;
+ ++buf[size - 1];
+ for (int i = size - 1; i > 0 && buf[i] > '9'; --i) {
+ buf[i] = '0';
+ ++buf[i - 1];
+ }
+ if (buf[0] > '9') {
+ buf[0] = '1';
+ if (fixed)
+ buf[size++] = '0';
+ else
+ ++exp10;
+ }
+ return digits::done;
+ }
+};
+
+inline FMT_CONSTEXPR20 void adjust_precision(int& precision, int exp10) {
+ // Adjust fixed precision by exponent because it is relative to decimal
+ // point.
+ if (exp10 > 0 && precision > max_value<int>() - exp10)
+ FMT_THROW(format_error("number is too big"));
+ precision += exp10;
+}
+
+// Generates output using the Grisu digit-gen algorithm.
+// error: the size of the region (lower, upper) outside of which numbers
+// definitely do not round to value (Delta in Grisu3).
+FMT_INLINE FMT_CONSTEXPR20 auto grisu_gen_digits(fp value, uint64_t error,
+ int& exp,
+ gen_digits_handler& handler)
+ -> digits::result {
+ const fp one(1ULL << -value.e, value.e);
+ // The integral part of scaled value (p1 in Grisu) = value / one. It cannot be
+ // zero because it contains a product of two 64-bit numbers with MSB set (due
+ // to normalization) - 1, shifted right by at most 60 bits.
+ auto integral = static_cast<uint32_t>(value.f >> -one.e);
+ FMT_ASSERT(integral != 0, "");
+ FMT_ASSERT(integral == value.f >> -one.e, "");
+ // The fractional part of scaled value (p2 in Grisu) c = value % one.
+ uint64_t fractional = value.f & (one.f - 1);
+ exp = count_digits(integral); // kappa in Grisu.
+ // Non-fixed formats require at least one digit and no precision adjustment.
+ if (handler.fixed) {
+ adjust_precision(handler.precision, exp + handler.exp10);
+ // Check if precision is satisfied just by leading zeros, e.g.
+ // format("{:.2f}", 0.001) gives "0.00" without generating any digits.
+ if (handler.precision <= 0) {
+ if (handler.precision < 0) return digits::done;
+ // Divide by 10 to prevent overflow.
+ uint64_t divisor = data::power_of_10_64[exp - 1] << -one.e;
+ auto dir = get_round_direction(divisor, value.f / 10, error * 10);
+ if (dir == round_direction::unknown) return digits::error;
+ handler.buf[handler.size++] = dir == round_direction::up ? '1' : '0';
+ return digits::done;
+ }
+ }
+ // Generate digits for the integral part. This can produce up to 10 digits.
+ do {
+ uint32_t digit = 0;
+ auto divmod_integral = [&](uint32_t divisor) {
+ digit = integral / divisor;
+ integral %= divisor;
+ };
+ // This optimization by Milo Yip reduces the number of integer divisions by
+ // one per iteration.
+ switch (exp) {
+ case 10:
+ divmod_integral(1000000000);
+ break;
+ case 9:
+ divmod_integral(100000000);
+ break;
+ case 8:
+ divmod_integral(10000000);
+ break;
+ case 7:
+ divmod_integral(1000000);
+ break;
+ case 6:
+ divmod_integral(100000);
+ break;
+ case 5:
+ divmod_integral(10000);
+ break;
+ case 4:
+ divmod_integral(1000);
+ break;
+ case 3:
+ divmod_integral(100);
+ break;
+ case 2:
+ divmod_integral(10);
+ break;
+ case 1:
+ digit = integral;
+ integral = 0;
+ break;
+ default:
+ FMT_ASSERT(false, "invalid number of digits");
+ }
+ --exp;
+ auto remainder = (static_cast<uint64_t>(integral) << -one.e) + fractional;
+ auto result = handler.on_digit(static_cast<char>('0' + digit),
+ data::power_of_10_64[exp] << -one.e,
+ remainder, error, true);
+ if (result != digits::more) return result;
+ } while (exp > 0);
+ // Generate digits for the fractional part.
+ for (;;) {
+ fractional *= 10;
+ error *= 10;
+ char digit = static_cast<char>('0' + (fractional >> -one.e));
+ fractional &= one.f - 1;
+ --exp;
+ auto result = handler.on_digit(digit, one.f, fractional, error, false);
+ if (result != digits::more) return result;
+ }
+}
+
+class bigint {
+ private:
+ // A bigint is stored as an array of bigits (big digits), with bigit at index
+ // 0 being the least significant one.
+ using bigit = uint32_t;
+ using double_bigit = uint64_t;
+ enum { bigits_capacity = 32 };
+ basic_memory_buffer<bigit, bigits_capacity> bigits_;
+ int exp_;
+
+ FMT_CONSTEXPR20 bigit operator[](int index) const {
+ return bigits_[to_unsigned(index)];
+ }
+ FMT_CONSTEXPR20 bigit& operator[](int index) {
+ return bigits_[to_unsigned(index)];
+ }
+
+ static constexpr const int bigit_bits = num_bits<bigit>();
+
+ friend struct formatter<bigint>;
+
+ FMT_CONSTEXPR20 void subtract_bigits(int index, bigit other, bigit& borrow) {
+ auto result = static_cast<double_bigit>((*this)[index]) - other - borrow;
+ (*this)[index] = static_cast<bigit>(result);
+ borrow = static_cast<bigit>(result >> (bigit_bits * 2 - 1));
+ }
+
+ FMT_CONSTEXPR20 void remove_leading_zeros() {
+ int num_bigits = static_cast<int>(bigits_.size()) - 1;
+ while (num_bigits > 0 && (*this)[num_bigits] == 0) --num_bigits;
+ bigits_.resize(to_unsigned(num_bigits + 1));
+ }
+
+ // Computes *this -= other assuming aligned bigints and *this >= other.
+ FMT_CONSTEXPR20 void subtract_aligned(const bigint& other) {
+ FMT_ASSERT(other.exp_ >= exp_, "unaligned bigints");
+ FMT_ASSERT(compare(*this, other) >= 0, "");
+ bigit borrow = 0;
+ int i = other.exp_ - exp_;
+ for (size_t j = 0, n = other.bigits_.size(); j != n; ++i, ++j)
+ subtract_bigits(i, other.bigits_[j], borrow);
+ while (borrow > 0) subtract_bigits(i, 0, borrow);
+ remove_leading_zeros();
+ }
+
+ FMT_CONSTEXPR20 void multiply(uint32_t value) {
+ const double_bigit wide_value = value;
+ bigit carry = 0;
+ for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+ double_bigit result = bigits_[i] * wide_value + carry;
+ bigits_[i] = static_cast<bigit>(result);
+ carry = static_cast<bigit>(result >> bigit_bits);
+ }
+ if (carry != 0) bigits_.push_back(carry);
+ }
+
+ template <typename UInt, FMT_ENABLE_IF(std::is_same<UInt, uint64_t>::value ||
+ std::is_same<UInt, uint128_t>::value)>
+ FMT_CONSTEXPR20 void multiply(UInt value) {
+ using half_uint =
+ conditional_t<std::is_same<UInt, uint128_t>::value, uint64_t, uint32_t>;
+ const int shift = num_bits<half_uint>() - bigit_bits;
+ const UInt lower = static_cast<half_uint>(value);
+ const UInt upper = value >> num_bits<half_uint>();
+ UInt carry = 0;
+ for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+ UInt result = lower * bigits_[i] + static_cast<bigit>(carry);
+ carry = (upper * bigits_[i] << shift) + (result >> bigit_bits) +
+ (carry >> bigit_bits);
+ bigits_[i] = static_cast<bigit>(result);
+ }
+ while (carry != 0) {
+ bigits_.push_back(static_cast<bigit>(carry));
+ carry >>= bigit_bits;
+ }
+ }
+
+ template <typename UInt, FMT_ENABLE_IF(std::is_same<UInt, uint64_t>::value ||
+ std::is_same<UInt, uint128_t>::value)>
+ FMT_CONSTEXPR20 void assign(UInt n) {
+ size_t num_bigits = 0;
+ do {
+ bigits_[num_bigits++] = static_cast<bigit>(n);
+ n >>= bigit_bits;
+ } while (n != 0);
+ bigits_.resize(num_bigits);
+ exp_ = 0;
+ }
+
+ public:
+ FMT_CONSTEXPR20 bigint() : exp_(0) {}
+ explicit bigint(uint64_t n) { assign(n); }
+
+ bigint(const bigint&) = delete;
+ void operator=(const bigint&) = delete;
+
+ FMT_CONSTEXPR20 void assign(const bigint& other) {
+ auto size = other.bigits_.size();
+ bigits_.resize(size);
+ auto data = other.bigits_.data();
+ std::copy(data, data + size, make_checked(bigits_.data(), size));
+ exp_ = other.exp_;
+ }
+
+ template <typename Int> FMT_CONSTEXPR20 void operator=(Int n) {
+ FMT_ASSERT(n > 0, "");
+ assign(uint64_or_128_t<Int>(n));
+ }
+
+ FMT_CONSTEXPR20 int num_bigits() const {
+ return static_cast<int>(bigits_.size()) + exp_;
+ }
+
+ FMT_NOINLINE FMT_CONSTEXPR20 bigint& operator<<=(int shift) {
+ FMT_ASSERT(shift >= 0, "");
+ exp_ += shift / bigit_bits;
+ shift %= bigit_bits;
+ if (shift == 0) return *this;
+ bigit carry = 0;
+ for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+ bigit c = bigits_[i] >> (bigit_bits - shift);
+ bigits_[i] = (bigits_[i] << shift) + carry;
+ carry = c;
+ }
+ if (carry != 0) bigits_.push_back(carry);
+ return *this;
+ }
+
+ template <typename Int> FMT_CONSTEXPR20 bigint& operator*=(Int value) {
+ FMT_ASSERT(value > 0, "");
+ multiply(uint32_or_64_or_128_t<Int>(value));
+ return *this;
+ }
+
+ friend FMT_CONSTEXPR20 int compare(const bigint& lhs, const bigint& rhs) {
+ int num_lhs_bigits = lhs.num_bigits(), num_rhs_bigits = rhs.num_bigits();
+ if (num_lhs_bigits != num_rhs_bigits)
+ return num_lhs_bigits > num_rhs_bigits ? 1 : -1;
+ int i = static_cast<int>(lhs.bigits_.size()) - 1;
+ int j = static_cast<int>(rhs.bigits_.size()) - 1;
+ int end = i - j;
+ if (end < 0) end = 0;
+ for (; i >= end; --i, --j) {
+ bigit lhs_bigit = lhs[i], rhs_bigit = rhs[j];
+ if (lhs_bigit != rhs_bigit) return lhs_bigit > rhs_bigit ? 1 : -1;
+ }
+ if (i != j) return i > j ? 1 : -1;
+ return 0;
+ }
+
+ // Returns compare(lhs1 + lhs2, rhs).
+ friend FMT_CONSTEXPR20 int add_compare(const bigint& lhs1, const bigint& lhs2,
+ const bigint& rhs) {
+ auto minimum = [](int a, int b) { return a < b ? a : b; };
+ auto maximum = [](int a, int b) { return a > b ? a : b; };
+ int max_lhs_bigits = maximum(lhs1.num_bigits(), lhs2.num_bigits());
+ int num_rhs_bigits = rhs.num_bigits();
+ if (max_lhs_bigits + 1 < num_rhs_bigits) return -1;
+ if (max_lhs_bigits > num_rhs_bigits) return 1;
+ auto get_bigit = [](const bigint& n, int i) -> bigit {
+ return i >= n.exp_ && i < n.num_bigits() ? n[i - n.exp_] : 0;
+ };
+ double_bigit borrow = 0;
+ int min_exp = minimum(minimum(lhs1.exp_, lhs2.exp_), rhs.exp_);
+ for (int i = num_rhs_bigits - 1; i >= min_exp; --i) {
+ double_bigit sum =
+ static_cast<double_bigit>(get_bigit(lhs1, i)) + get_bigit(lhs2, i);
+ bigit rhs_bigit = get_bigit(rhs, i);
+ if (sum > rhs_bigit + borrow) return 1;
+ borrow = rhs_bigit + borrow - sum;
+ if (borrow > 1) return -1;
+ borrow <<= bigit_bits;
+ }
+ return borrow != 0 ? -1 : 0;
+ }
+
+ // Assigns pow(10, exp) to this bigint.
+ FMT_CONSTEXPR20 void assign_pow10(int exp) {
+ FMT_ASSERT(exp >= 0, "");
+ if (exp == 0) return *this = 1;
+ // Find the top bit.
+ int bitmask = 1;
+ while (exp >= bitmask) bitmask <<= 1;
+ bitmask >>= 1;
+ // pow(10, exp) = pow(5, exp) * pow(2, exp). First compute pow(5, exp) by
+ // repeated squaring and multiplication.
+ *this = 5;
+ bitmask >>= 1;
+ while (bitmask != 0) {
+ square();
+ if ((exp & bitmask) != 0) *this *= 5;
+ bitmask >>= 1;
+ }
+ *this <<= exp; // Multiply by pow(2, exp) by shifting.
+ }
+
+ FMT_CONSTEXPR20 void square() {
+ int num_bigits = static_cast<int>(bigits_.size());
+ int num_result_bigits = 2 * num_bigits;
+ basic_memory_buffer<bigit, bigits_capacity> n(std::move(bigits_));
+ bigits_.resize(to_unsigned(num_result_bigits));
+ auto sum = uint128_t();
+ for (int bigit_index = 0; bigit_index < num_bigits; ++bigit_index) {
+ // Compute bigit at position bigit_index of the result by adding
+ // cross-product terms n[i] * n[j] such that i + j == bigit_index.
+ for (int i = 0, j = bigit_index; j >= 0; ++i, --j) {
+ // Most terms are multiplied twice which can be optimized in the future.
+ sum += static_cast<double_bigit>(n[i]) * n[j];
+ }
+ (*this)[bigit_index] = static_cast<bigit>(sum);
+ sum >>= num_bits<bigit>(); // Compute the carry.
+ }
+ // Do the same for the top half.
+ for (int bigit_index = num_bigits; bigit_index < num_result_bigits;
+ ++bigit_index) {
+ for (int j = num_bigits - 1, i = bigit_index - j; i < num_bigits;)
+ sum += static_cast<double_bigit>(n[i++]) * n[j--];
+ (*this)[bigit_index] = static_cast<bigit>(sum);
+ sum >>= num_bits<bigit>();
+ }
+ remove_leading_zeros();
+ exp_ *= 2;
+ }
+
+ // If this bigint has a bigger exponent than other, adds trailing zero to make
+ // exponents equal. This simplifies some operations such as subtraction.
+ FMT_CONSTEXPR20 void align(const bigint& other) {
+ int exp_difference = exp_ - other.exp_;
+ if (exp_difference <= 0) return;
+ int num_bigits = static_cast<int>(bigits_.size());
+ bigits_.resize(to_unsigned(num_bigits + exp_difference));
+ for (int i = num_bigits - 1, j = i + exp_difference; i >= 0; --i, --j)
+ bigits_[j] = bigits_[i];
+ std::uninitialized_fill_n(bigits_.data(), exp_difference, 0);
+ exp_ -= exp_difference;
+ }
+
+ // Divides this bignum by divisor, assigning the remainder to this and
+ // returning the quotient.
+ FMT_CONSTEXPR20 int divmod_assign(const bigint& divisor) {
+ FMT_ASSERT(this != &divisor, "");
+ if (compare(*this, divisor) < 0) return 0;
+ FMT_ASSERT(divisor.bigits_[divisor.bigits_.size() - 1u] != 0, "");
+ align(divisor);
+ int quotient = 0;
+ do {
+ subtract_aligned(divisor);
+ ++quotient;
+ } while (compare(*this, divisor) >= 0);
+ return quotient;
+ }
+};
+
+// format_dragon flags.
+enum dragon {
+ predecessor_closer = 1,
+ fixup = 2, // Run fixup to correct exp10 which can be off by one.
+ fixed = 4,
+};
+
+// Formats a floating-point number using a variation of the Fixed-Precision
+// Positive Floating-Point Printout ((FPP)^2) algorithm by Steele & White:
+// https://fmt.dev/papers/p372-steele.pdf.
+FMT_CONSTEXPR20 inline void format_dragon(basic_fp<uint128_t> value,
+ unsigned flags, int num_digits,
+ buffer<char>& buf, int& exp10) {
+ bigint numerator; // 2 * R in (FPP)^2.
+ bigint denominator; // 2 * S in (FPP)^2.
+ // lower and upper are differences between value and corresponding boundaries.
+ bigint lower; // (M^- in (FPP)^2).
+ bigint upper_store; // upper's value if different from lower.
+ bigint* upper = nullptr; // (M^+ in (FPP)^2).
+ // Shift numerator and denominator by an extra bit or two (if lower boundary
+ // is closer) to make lower and upper integers. This eliminates multiplication
+ // by 2 during later computations.
+ bool is_predecessor_closer = (flags & dragon::predecessor_closer) != 0;
+ int shift = is_predecessor_closer ? 2 : 1;
+ if (value.e >= 0) {
+ numerator = value.f;
+ numerator <<= value.e + shift;
+ lower = 1;
+ lower <<= value.e;
+ if (is_predecessor_closer) {
+ upper_store = 1;
+ upper_store <<= value.e + 1;
+ upper = &upper_store;
+ }
+ denominator.assign_pow10(exp10);
+ denominator <<= shift;
+ } else if (exp10 < 0) {
+ numerator.assign_pow10(-exp10);
+ lower.assign(numerator);
+ if (is_predecessor_closer) {
+ upper_store.assign(numerator);
+ upper_store <<= 1;
+ upper = &upper_store;
+ }
+ numerator *= value.f;
+ numerator <<= shift;
+ denominator = 1;
+ denominator <<= shift - value.e;
+ } else {
+ numerator = value.f;
+ numerator <<= shift;
+ denominator.assign_pow10(exp10);
+ denominator <<= shift - value.e;
+ lower = 1;
+ if (is_predecessor_closer) {
+ upper_store = 1ULL << 1;
+ upper = &upper_store;
+ }
+ }
+ int even = static_cast<int>((value.f & 1) == 0);
+ if (!upper) upper = &lower;
+ if ((flags & dragon::fixup) != 0) {
+ if (add_compare(numerator, *upper, denominator) + even <= 0) {
+ --exp10;
+ numerator *= 10;
+ if (num_digits < 0) {
+ lower *= 10;
+ if (upper != &lower) *upper *= 10;
+ }
+ }
+ if ((flags & dragon::fixed) != 0) adjust_precision(num_digits, exp10 + 1);
+ }
+ // Invariant: value == (numerator / denominator) * pow(10, exp10).
+ if (num_digits < 0) {
+ // Generate the shortest representation.
+ num_digits = 0;
+ char* data = buf.data();
+ for (;;) {
+ int digit = numerator.divmod_assign(denominator);
+ bool low = compare(numerator, lower) - even < 0; // numerator <[=] lower.
+ // numerator + upper >[=] pow10:
+ bool high = add_compare(numerator, *upper, denominator) + even > 0;
+ data[num_digits++] = static_cast<char>('0' + digit);
+ if (low || high) {
+ if (!low) {
+ ++data[num_digits - 1];
+ } else if (high) {
+ int result = add_compare(numerator, numerator, denominator);
+ // Round half to even.
+ if (result > 0 || (result == 0 && (digit % 2) != 0))
+ ++data[num_digits - 1];
+ }
+ buf.try_resize(to_unsigned(num_digits));
+ exp10 -= num_digits - 1;
+ return;
+ }
+ numerator *= 10;
+ lower *= 10;
+ if (upper != &lower) *upper *= 10;
+ }
+ }
+ // Generate the given number of digits.
+ exp10 -= num_digits - 1;
+ if (num_digits == 0) {
+ denominator *= 10;
+ auto digit = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0';
+ buf.push_back(digit);
+ return;
+ }
+ buf.try_resize(to_unsigned(num_digits));
+ for (int i = 0; i < num_digits - 1; ++i) {
+ int digit = numerator.divmod_assign(denominator);
+ buf[i] = static_cast<char>('0' + digit);
+ numerator *= 10;
+ }
+ int digit = numerator.divmod_assign(denominator);
+ auto result = add_compare(numerator, numerator, denominator);
+ if (result > 0 || (result == 0 && (digit % 2) != 0)) {
+ if (digit == 9) {
+ const auto overflow = '0' + 10;
+ buf[num_digits - 1] = overflow;
+ // Propagate the carry.
+ for (int i = num_digits - 1; i > 0 && buf[i] == overflow; --i) {
+ buf[i] = '0';
+ ++buf[i - 1];
+ }
+ if (buf[0] == overflow) {
+ buf[0] = '1';
+ ++exp10;
+ }
+ return;
+ }
+ ++digit;
+ }
+ buf[num_digits - 1] = static_cast<char>('0' + digit);
+}
+
+// Formats a floating-point number using the hexfloat format.
+template <typename Float, FMT_ENABLE_IF(!is_double_double<Float>::value)>
+FMT_CONSTEXPR20 void format_hexfloat(Float value, int precision,
+ float_specs specs, buffer<char>& buf) {
+ // float is passed as double to reduce the number of instantiations and to
+ // simplify implementation.
+ static_assert(!std::is_same<Float, float>::value, "");
+
+ using info = dragonbox::float_info<Float>;
+
+ // Assume Float is in the format [sign][exponent][significand].
+ using carrier_uint = typename info::carrier_uint;
+
+ constexpr auto num_float_significand_bits =
+ detail::num_significand_bits<Float>();
+
+ basic_fp<carrier_uint> f(value);
+ f.e += num_float_significand_bits;
+ if (!has_implicit_bit<Float>()) --f.e;
+
+ constexpr auto num_fraction_bits =
+ num_float_significand_bits + (has_implicit_bit<Float>() ? 1 : 0);
+ constexpr auto num_xdigits = (num_fraction_bits + 3) / 4;
+
+ constexpr auto leading_shift = ((num_xdigits - 1) * 4);
+ const auto leading_mask = carrier_uint(0xF) << leading_shift;
+ const auto leading_xdigit =
+ static_cast<uint32_t>((f.f & leading_mask) >> leading_shift);
+ if (leading_xdigit > 1) f.e -= (32 - countl_zero(leading_xdigit) - 1);
+
+ int print_xdigits = num_xdigits - 1;
+ if (precision >= 0 && print_xdigits > precision) {
+ const int shift = ((print_xdigits - precision - 1) * 4);
+ const auto mask = carrier_uint(0xF) << shift;
+ const auto v = static_cast<uint32_t>((f.f & mask) >> shift);
+
+ if (v >= 8) {
+ const auto inc = carrier_uint(1) << (shift + 4);
+ f.f += inc;
+ f.f &= ~(inc - 1);
+ }
+
+ // Check long double overflow
+ if (!has_implicit_bit<Float>()) {
+ const auto implicit_bit = carrier_uint(1) << num_float_significand_bits;
+ if ((f.f & implicit_bit) == implicit_bit) {
+ f.f >>= 4;
+ f.e += 4;
+ }
+ }
+
+ print_xdigits = precision;
+ }
+
+ char xdigits[num_bits<carrier_uint>() / 4];
+ detail::fill_n(xdigits, sizeof(xdigits), '0');
+ format_uint<4>(xdigits, f.f, num_xdigits, specs.upper);
+
+ // Remove zero tail
+ while (print_xdigits > 0 && xdigits[print_xdigits] == '0') --print_xdigits;
+
+ buf.push_back('0');
+ buf.push_back(specs.upper ? 'X' : 'x');
+ buf.push_back(xdigits[0]);
+ if (specs.showpoint || print_xdigits > 0 || print_xdigits < precision)
+ buf.push_back('.');
+ buf.append(xdigits + 1, xdigits + 1 + print_xdigits);
+ for (; print_xdigits < precision; ++print_xdigits) buf.push_back('0');
+
+ buf.push_back(specs.upper ? 'P' : 'p');
+
+ uint32_t abs_e;
+ if (f.e < 0) {
+ buf.push_back('-');
+ abs_e = static_cast<uint32_t>(-f.e);
+ } else {
+ buf.push_back('+');
+ abs_e = static_cast<uint32_t>(f.e);
+ }
+ format_decimal<char>(appender(buf), abs_e, detail::count_digits(abs_e));
+}
+
+template <typename Float, FMT_ENABLE_IF(is_double_double<Float>::value)>
+FMT_CONSTEXPR20 void format_hexfloat(Float value, int precision,
+ float_specs specs, buffer<char>& buf) {
+ format_hexfloat(static_cast<double>(value), precision, specs, buf);
+}
+
+template <typename Float>
+FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs,
+ buffer<char>& buf) -> int {
+ // float is passed as double to reduce the number of instantiations.
+ static_assert(!std::is_same<Float, float>::value, "");
+ FMT_ASSERT(value >= 0, "value is negative");
+ auto converted_value = convert_float(value);
+
+ const bool fixed = specs.format == float_format::fixed;
+ if (value <= 0) { // <= instead of == to silence a warning.
+ if (precision <= 0 || !fixed) {
+ buf.push_back('0');
+ return 0;
+ }
+ buf.try_resize(to_unsigned(precision));
+ fill_n(buf.data(), precision, '0');
+ return -precision;
+ }
+
+ int exp = 0;
+ bool use_dragon = true;
+ unsigned dragon_flags = 0;
+ if (!is_fast_float<Float>()) {
+ const auto inv_log2_10 = 0.3010299956639812; // 1 / log2(10)
+ using info = dragonbox::float_info<decltype(converted_value)>;
+ const auto f = basic_fp<typename info::carrier_uint>(converted_value);
+ // Compute exp, an approximate power of 10, such that
+ // 10^(exp - 1) <= value < 10^exp or 10^exp <= value < 10^(exp + 1).
+ // This is based on log10(value) == log2(value) / log2(10) and approximation
+ // of log2(value) by e + num_fraction_bits idea from double-conversion.
+ exp = static_cast<int>(
+ std::ceil((f.e + count_digits<1>(f.f) - 1) * inv_log2_10 - 1e-10));
+ dragon_flags = dragon::fixup;
+ } else if (!is_constant_evaluated() && precision < 0) {
+ // Use Dragonbox for the shortest format.
+ if (specs.binary32) {
+ auto dec = dragonbox::to_decimal(static_cast<float>(value));
+ write<char>(buffer_appender<char>(buf), dec.significand);
+ return dec.exponent;
+ }
+ auto dec = dragonbox::to_decimal(static_cast<double>(value));
+ write<char>(buffer_appender<char>(buf), dec.significand);
+ return dec.exponent;
+ } else if (is_constant_evaluated()) {
+ // Use Grisu + Dragon4 for the given precision:
+ // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf.
+ const int min_exp = -60; // alpha in Grisu.
+ int cached_exp10 = 0; // K in Grisu.
+ fp normalized = normalize(fp(converted_value));
+ const auto cached_pow = get_cached_power(
+ min_exp - (normalized.e + fp::num_significand_bits), cached_exp10);
+ normalized = normalized * cached_pow;
+ gen_digits_handler handler{buf.data(), 0, precision, -cached_exp10, fixed};
+ if (grisu_gen_digits(normalized, 1, exp, handler) != digits::error &&
+ !is_constant_evaluated()) {
+ exp += handler.exp10;
+ buf.try_resize(to_unsigned(handler.size));
+ use_dragon = false;
+ } else {
+ exp += handler.size - cached_exp10 - 1;
+ precision = handler.precision;
+ }
+ } else {
+ // Extract significand bits and exponent bits.
+ using info = dragonbox::float_info<double>;
+ auto br = bit_cast<uint64_t>(static_cast<double>(value));
+
+ const uint64_t significand_mask =
+ (static_cast<uint64_t>(1) << num_significand_bits<double>()) - 1;
+ uint64_t significand = (br & significand_mask);
+ int exponent = static_cast<int>((br & exponent_mask<double>()) >>
+ num_significand_bits<double>());
+
+ if (exponent != 0) { // Check if normal.
+ exponent -= exponent_bias<double>() + num_significand_bits<double>();
+ significand |=
+ (static_cast<uint64_t>(1) << num_significand_bits<double>());
+ significand <<= 1;
+ } else {
+ // Normalize subnormal inputs.
+ FMT_ASSERT(significand != 0, "zeros should not appear hear");
+ int shift = countl_zero(significand);
+ FMT_ASSERT(shift >= num_bits<uint64_t>() - num_significand_bits<double>(),
+ "");
+ shift -= (num_bits<uint64_t>() - num_significand_bits<double>() - 2);
+ exponent = (std::numeric_limits<double>::min_exponent -
+ num_significand_bits<double>()) -
+ shift;
+ significand <<= shift;
+ }
+
+ // Compute the first several nonzero decimal significand digits.
+ // We call the number we get the first segment.
+ const int k = info::kappa - dragonbox::floor_log10_pow2(exponent);
+ exp = -k;
+ const int beta = exponent + dragonbox::floor_log2_pow10(k);
+ uint64_t first_segment;
+ bool has_more_segments;
+ int digits_in_the_first_segment;
+ {
+ const auto r = dragonbox::umul192_upper128(
+ significand << beta, dragonbox::get_cached_power(k));
+ first_segment = r.high();
+ has_more_segments = r.low() != 0;
+
+ // The first segment can have 18 ~ 19 digits.
+ if (first_segment >= 1000000000000000000ULL) {
+ digits_in_the_first_segment = 19;
+ } else {
+ // When it is of 18-digits, we align it to 19-digits by adding a bogus
+ // zero at the end.
+ digits_in_the_first_segment = 18;
+ first_segment *= 10;
+ }
+ }
+
+ // Compute the actual number of decimal digits to print.
+ if (fixed) {
+ adjust_precision(precision, exp + digits_in_the_first_segment);
+ }
+
+ // Use Dragon4 only when there might be not enough digits in the first
+ // segment.
+ if (digits_in_the_first_segment > precision) {
+ use_dragon = false;
+
+ if (precision <= 0) {
+ exp += digits_in_the_first_segment;
+
+ if (precision < 0) {
+ // Nothing to do, since all we have are just leading zeros.
+ buf.try_resize(0);
+ } else {
+ // We may need to round-up.
+ buf.try_resize(1);
+ if ((first_segment | static_cast<uint64_t>(has_more_segments)) >
+ 5000000000000000000ULL) {
+ buf[0] = '1';
+ } else {
+ buf[0] = '0';
+ }
+ }
+ } // precision <= 0
+ else {
+ exp += digits_in_the_first_segment - precision;
+
+ // When precision > 0, we divide the first segment into three
+ // subsegments, each with 9, 9, and 0 ~ 1 digits so that each fits
+ // in 32-bits which usually allows faster calculation than in
+ // 64-bits. Since some compiler (e.g. MSVC) doesn't know how to optimize
+ // division-by-constant for large 64-bit divisors, we do it here
+ // manually. The magic number 7922816251426433760 below is equal to
+ // ceil(2^(64+32) / 10^10).
+ const uint32_t first_subsegment = static_cast<uint32_t>(
+ dragonbox::umul128_upper64(first_segment, 7922816251426433760ULL) >>
+ 32);
+ const uint64_t second_third_subsegments =
+ first_segment - first_subsegment * 10000000000ULL;
+
+ uint64_t prod;
+ uint32_t digits;
+ bool should_round_up;
+ int number_of_digits_to_print = precision > 9 ? 9 : precision;
+
+ // Print a 9-digits subsegment, either the first or the second.
+ auto print_subsegment = [&](uint32_t subsegment, char* buffer) {
+ int number_of_digits_printed = 0;
+
+ // If we want to print an odd number of digits from the subsegment,
+ if ((number_of_digits_to_print & 1) != 0) {
+ // Convert to 64-bit fixed-point fractional form with 1-digit
+ // integer part. The magic number 720575941 is a good enough
+ // approximation of 2^(32 + 24) / 10^8; see
+ // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case
+ // for details.
+ prod = ((subsegment * static_cast<uint64_t>(720575941)) >> 24) + 1;
+ digits = static_cast<uint32_t>(prod >> 32);
+ *buffer = static_cast<char>('0' + digits);
+ number_of_digits_printed++;
+ }
+ // If we want to print an even number of digits from the
+ // first_subsegment,
+ else {
+ // Convert to 64-bit fixed-point fractional form with 2-digits
+ // integer part. The magic number 450359963 is a good enough
+ // approximation of 2^(32 + 20) / 10^7; see
+ // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case
+ // for details.
+ prod = ((subsegment * static_cast<uint64_t>(450359963)) >> 20) + 1;
+ digits = static_cast<uint32_t>(prod >> 32);
+ copy2(buffer, digits2(digits));
+ number_of_digits_printed += 2;
+ }
+
+ // Print all digit pairs.
+ while (number_of_digits_printed < number_of_digits_to_print) {
+ prod = static_cast<uint32_t>(prod) * static_cast<uint64_t>(100);
+ digits = static_cast<uint32_t>(prod >> 32);
+ copy2(buffer + number_of_digits_printed, digits2(digits));
+ number_of_digits_printed += 2;
+ }
+ };
+
+ // Print first subsegment.
+ print_subsegment(first_subsegment, buf.data());
+
+ // Perform rounding if the first subsegment is the last subsegment to
+ // print.
+ if (precision <= 9) {
+ // Rounding inside the subsegment.
+ // We round-up if:
+ // - either the fractional part is strictly larger than 1/2, or
+ // - the fractional part is exactly 1/2 and the last digit is odd.
+ // We rely on the following observations:
+ // - If fractional_part >= threshold, then the fractional part is
+ // strictly larger than 1/2.
+ // - If the MSB of fractional_part is set, then the fractional part
+ // must be at least 1/2.
+ // - When the MSB of fractional_part is set, either
+ // second_third_subsegments being nonzero or has_more_segments
+ // being true means there are further digits not printed, so the
+ // fractional part is strictly larger than 1/2.
+ if (precision < 9) {
+ uint32_t fractional_part = static_cast<uint32_t>(prod);
+ should_round_up = fractional_part >=
+ data::fractional_part_rounding_thresholds
+ [8 - number_of_digits_to_print] ||
+ ((fractional_part >> 31) &
+ ((digits & 1) | (second_third_subsegments != 0) |
+ has_more_segments)) != 0;
+ }
+ // Rounding at the subsegment boundary.
+ // In this case, the fractional part is at least 1/2 if and only if
+ // second_third_subsegments >= 5000000000ULL, and is strictly larger
+ // than 1/2 if we further have either second_third_subsegments >
+ // 5000000000ULL or has_more_segments == true.
+ else {
+ should_round_up = second_third_subsegments > 5000000000ULL ||
+ (second_third_subsegments == 5000000000ULL &&
+ ((digits & 1) != 0 || has_more_segments));
+ }
+ }
+ // Otherwise, print the second subsegment.
+ else {
+ // Compilers are not aware of how to leverage the maximum value of
+ // second_third_subsegments to find out a better magic number which
+ // allows us to eliminate an additional shift. 1844674407370955162 =
+ // ceil(2^64/10) < ceil(2^64*(10^9/(10^10 - 1))).
+ const uint32_t second_subsegment =
+ static_cast<uint32_t>(dragonbox::umul128_upper64(
+ second_third_subsegments, 1844674407370955162ULL));
+ const uint32_t third_subsegment =
+ static_cast<uint32_t>(second_third_subsegments) -
+ second_subsegment * 10;
+
+ number_of_digits_to_print = precision - 9;
+ print_subsegment(second_subsegment, buf.data() + 9);
+
+ // Rounding inside the subsegment.
+ if (precision < 18) {
+ // The condition third_subsegment != 0 implies that the segment was
+ // of 19 digits, so in this case the third segment should be
+ // consisting of a genuine digit from the input.
+ uint32_t fractional_part = static_cast<uint32_t>(prod);
+ should_round_up = fractional_part >=
+ data::fractional_part_rounding_thresholds
+ [8 - number_of_digits_to_print] ||
+ ((fractional_part >> 31) &
+ ((digits & 1) | (third_subsegment != 0) |
+ has_more_segments)) != 0;
+ }
+ // Rounding at the subsegment boundary.
+ else {
+ // In this case, the segment must be of 19 digits, thus
+ // the third subsegment should be consisting of a genuine digit from
+ // the input.
+ should_round_up = third_subsegment > 5 ||
+ (third_subsegment == 5 &&
+ ((digits & 1) != 0 || has_more_segments));
+ }
+ }
+
+ // Round-up if necessary.
+ if (should_round_up) {
+ ++buf[precision - 1];
+ for (int i = precision - 1; i > 0 && buf[i] > '9'; --i) {
+ buf[i] = '0';
+ ++buf[i - 1];
+ }
+ if (buf[0] > '9') {
+ buf[0] = '1';
+ if (fixed)
+ buf[precision++] = '0';
+ else
+ ++exp;
+ }
+ }
+ buf.try_resize(to_unsigned(precision));
+ }
+ } // if (digits_in_the_first_segment > precision)
+ else {
+ // Adjust the exponent for its use in Dragon4.
+ exp += digits_in_the_first_segment - 1;
+ }
+ }
+ if (use_dragon) {
+ auto f = basic_fp<uint128_t>();
+ bool is_predecessor_closer = specs.binary32
+ ? f.assign(static_cast<float>(value))
+ : f.assign(converted_value);
+ if (is_predecessor_closer) dragon_flags |= dragon::predecessor_closer;
+ if (fixed) dragon_flags |= dragon::fixed;
+ // Limit precision to the maximum possible number of significant digits in
+ // an IEEE754 double because we don't need to generate zeros.
+ const int max_double_digits = 767;
+ if (precision > max_double_digits) precision = max_double_digits;
+ format_dragon(f, dragon_flags, precision, buf, exp);
+ }
+ if (!fixed && !specs.showpoint) {
+ // Remove trailing zeros.
+ auto num_digits = buf.size();
+ while (num_digits > 0 && buf[num_digits - 1] == '0') {
+ --num_digits;
+ ++exp;
+ }
+ buf.try_resize(num_digits);
+ }
+ return exp;
+}
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR20 auto write_float(OutputIt out, T value,
+ format_specs<Char> specs, locale_ref loc)
+ -> OutputIt {
+ float_specs fspecs = parse_float_type_spec(specs);
+ fspecs.sign = specs.sign;
+ if (detail::signbit(value)) { // value < 0 is false for NaN so use signbit.
+ fspecs.sign = sign::minus;
+ value = -value;
+ } else if (fspecs.sign == sign::minus) {
+ fspecs.sign = sign::none;
+ }
+
+ if (!detail::isfinite(value))
+ return write_nonfinite(out, detail::isnan(value), specs, fspecs);
+
+ if (specs.align == align::numeric && fspecs.sign) {
+ auto it = reserve(out, 1);
+ *it++ = detail::sign<Char>(fspecs.sign);
+ out = base_iterator(out, it);
+ fspecs.sign = sign::none;
+ if (specs.width != 0) --specs.width;
+ }
+
+ memory_buffer buffer;
+ if (fspecs.format == float_format::hex) {
+ if (fspecs.sign) buffer.push_back(detail::sign<char>(fspecs.sign));
+ format_hexfloat(convert_float(value), specs.precision, fspecs, buffer);
+ return write_bytes<align::right>(out, {buffer.data(), buffer.size()},
+ specs);
+ }
+ int precision = specs.precision >= 0 || specs.type == presentation_type::none
+ ? specs.precision
+ : 6;
+ if (fspecs.format == float_format::exp) {
+ if (precision == max_value<int>())
+ throw_format_error("number is too big");
+ else
+ ++precision;
+ } else if (fspecs.format != float_format::fixed && precision == 0) {
+ precision = 1;
+ }
+ if (const_check(std::is_same<T, float>())) fspecs.binary32 = true;
+ int exp = format_float(convert_float(value), precision, fspecs, buffer);
+ fspecs.precision = precision;
+ auto f = big_decimal_fp{buffer.data(), static_cast<int>(buffer.size()), exp};
+ return write_float(out, f, specs, fspecs, loc);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_floating_point<T>::value)>
+FMT_CONSTEXPR20 auto write(OutputIt out, T value, format_specs<Char> specs,
+ locale_ref loc = {}) -> OutputIt {
+ if (const_check(!is_supported_floating_point(value))) return out;
+ return specs.localized && write_loc(out, value, specs, loc)
+ ? out
+ : write_float(out, value, specs, loc);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_fast_float<T>::value)>
+FMT_CONSTEXPR20 auto write(OutputIt out, T value) -> OutputIt {
+ if (is_constant_evaluated()) return write(out, value, format_specs<Char>());
+ if (const_check(!is_supported_floating_point(value))) return out;
+
+ auto fspecs = float_specs();
+ if (detail::signbit(value)) {
+ fspecs.sign = sign::minus;
+ value = -value;
+ }
+
+ constexpr auto specs = format_specs<Char>();
+ using floaty = conditional_t<std::is_same<T, long double>::value, double, T>;
+ using floaty_uint = typename dragonbox::float_info<floaty>::carrier_uint;
+ floaty_uint mask = exponent_mask<floaty>();
+ if ((bit_cast<floaty_uint>(value) & mask) == mask)
+ return write_nonfinite(out, std::isnan(value), specs, fspecs);
+
+ auto dec = dragonbox::to_decimal(static_cast<floaty>(value));
+ return write_float(out, dec, specs, fspecs, {});
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_floating_point<T>::value &&
+ !is_fast_float<T>::value)>
+inline auto write(OutputIt out, T value) -> OutputIt {
+ return write(out, value, format_specs<Char>());
+}
+
+template <typename Char, typename OutputIt>
+auto write(OutputIt out, monostate, format_specs<Char> = {}, locale_ref = {})
+ -> OutputIt {
+ FMT_ASSERT(false, "");
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, basic_string_view<Char> value)
+ -> OutputIt {
+ auto it = reserve(out, value.size());
+ it = copy_str_noinline<Char>(value.begin(), value.end(), it);
+ return base_iterator(out, it);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_string<T>::value)>
+constexpr auto write(OutputIt out, const T& value) -> OutputIt {
+ return write<Char>(out, to_string_view(value));
+}
+
+// FMT_ENABLE_IF() condition separated to workaround an MSVC bug.
+template <
+ typename Char, typename OutputIt, typename T,
+ bool check =
+ std::is_enum<T>::value && !std::is_same<T, Char>::value &&
+ mapped_type_constant<T, basic_format_context<OutputIt, Char>>::value !=
+ type::custom_type,
+ FMT_ENABLE_IF(check)>
+FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt {
+ return write<Char>(out, static_cast<underlying_t<T>>(value));
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(std::is_same<T, bool>::value)>
+FMT_CONSTEXPR auto write(OutputIt out, T value,
+ const format_specs<Char>& specs = {}, locale_ref = {})
+ -> OutputIt {
+ return specs.type != presentation_type::none &&
+ specs.type != presentation_type::string
+ ? write(out, value ? 1 : 0, specs, {})
+ : write_bytes(out, value ? "true" : "false", specs);
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, Char value) -> OutputIt {
+ auto it = reserve(out, 1);
+ *it++ = value;
+ return base_iterator(out, it);
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR_CHAR_TRAITS auto write(OutputIt out, const Char* value)
+ -> OutputIt {
+ if (value) return write(out, basic_string_view<Char>(value));
+ throw_format_error("string pointer is null");
+ return out;
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(std::is_same<T, void>::value)>
+auto write(OutputIt out, const T* value, const format_specs<Char>& specs = {},
+ locale_ref = {}) -> OutputIt {
+ return write_ptr<Char>(out, bit_cast<uintptr_t>(value), &specs);
+}
+
+// A write overload that handles implicit conversions.
+template <typename Char, typename OutputIt, typename T,
+ typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value) -> enable_if_t<
+ std::is_class<T>::value && !is_string<T>::value &&
+ !is_floating_point<T>::value && !std::is_same<T, Char>::value &&
+ !std::is_same<T, remove_cvref_t<decltype(arg_mapper<Context>().map(
+ value))>>::value,
+ OutputIt> {
+ return write<Char>(out, arg_mapper<Context>().map(value));
+}
+
+template <typename Char, typename OutputIt, typename T,
+ typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value)
+ -> enable_if_t<mapped_type_constant<T, Context>::value == type::custom_type,
+ OutputIt> {
+ auto ctx = Context(out, {}, {});
+ return typename Context::template formatter_type<T>().format(value, ctx);
+}
+
+// An argument visitor that formats the argument and writes it via the output
+// iterator. It's a class and not a generic lambda for compatibility with C++11.
+template <typename Char> struct default_arg_formatter {
+ using iterator = buffer_appender<Char>;
+ using context = buffer_context<Char>;
+
+ iterator out;
+ basic_format_args<context> args;
+ locale_ref loc;
+
+ template <typename T> auto operator()(T value) -> iterator {
+ return write<Char>(out, value);
+ }
+ auto operator()(typename basic_format_arg<context>::handle h) -> iterator {
+ basic_format_parse_context<Char> parse_ctx({});
+ context format_ctx(out, args, loc);
+ h.format(parse_ctx, format_ctx);
+ return format_ctx.out();
+ }
+};
+
+template <typename Char> struct arg_formatter {
+ using iterator = buffer_appender<Char>;
+ using context = buffer_context<Char>;
+
+ iterator out;
+ const format_specs<Char>& specs;
+ locale_ref locale;
+
+ template <typename T>
+ FMT_CONSTEXPR FMT_INLINE auto operator()(T value) -> iterator {
+ return detail::write(out, value, specs, locale);
+ }
+ auto operator()(typename basic_format_arg<context>::handle) -> iterator {
+ // User-defined types are handled separately because they require access
+ // to the parse context.
+ return out;
+ }
+};
+
+template <typename Char> struct custom_formatter {
+ basic_format_parse_context<Char>& parse_ctx;
+ buffer_context<Char>& ctx;
+
+ void operator()(
+ typename basic_format_arg<buffer_context<Char>>::handle h) const {
+ h.format(parse_ctx, ctx);
+ }
+ template <typename T> void operator()(T) const {}
+};
+
+template <typename ErrorHandler> class width_checker {
+ public:
+ explicit FMT_CONSTEXPR width_checker(ErrorHandler& eh) : handler_(eh) {}
+
+ template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T value) -> unsigned long long {
+ if (is_negative(value)) handler_.on_error("negative width");
+ return static_cast<unsigned long long>(value);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T) -> unsigned long long {
+ handler_.on_error("width is not integer");
+ return 0;
+ }
+
+ private:
+ ErrorHandler& handler_;
+};
+
+template <typename ErrorHandler> class precision_checker {
+ public:
+ explicit FMT_CONSTEXPR precision_checker(ErrorHandler& eh) : handler_(eh) {}
+
+ template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T value) -> unsigned long long {
+ if (is_negative(value)) handler_.on_error("negative precision");
+ return static_cast<unsigned long long>(value);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T) -> unsigned long long {
+ handler_.on_error("precision is not integer");
+ return 0;
+ }
+
+ private:
+ ErrorHandler& handler_;
+};
+
+template <template <typename> class Handler, typename FormatArg,
+ typename ErrorHandler>
+FMT_CONSTEXPR auto get_dynamic_spec(FormatArg arg, ErrorHandler eh) -> int {
+ unsigned long long value = visit_format_arg(Handler<ErrorHandler>(eh), arg);
+ if (value > to_unsigned(max_value<int>())) eh.on_error("number is too big");
+ return static_cast<int>(value);
+}
+
+template <typename Context, typename ID>
+FMT_CONSTEXPR auto get_arg(Context& ctx, ID id) ->
+ typename Context::format_arg {
+ auto arg = ctx.arg(id);
+ if (!arg) ctx.on_error("argument not found");
+ return arg;
+}
+
+template <template <typename> class Handler, typename Context>
+FMT_CONSTEXPR void handle_dynamic_spec(int& value,
+ arg_ref<typename Context::char_type> ref,
+ Context& ctx) {
+ switch (ref.kind) {
+ case arg_id_kind::none:
+ break;
+ case arg_id_kind::index:
+ value = detail::get_dynamic_spec<Handler>(get_arg(ctx, ref.val.index),
+ ctx.error_handler());
+ break;
+ case arg_id_kind::name:
+ value = detail::get_dynamic_spec<Handler>(get_arg(ctx, ref.val.name),
+ ctx.error_handler());
+ break;
+ }
+}
+
+#if FMT_USE_USER_DEFINED_LITERALS
+template <typename Char> struct udl_formatter {
+ basic_string_view<Char> str;
+
+ template <typename... T>
+ auto operator()(T&&... args) const -> std::basic_string<Char> {
+ return vformat(str, fmt::make_format_args<buffer_context<Char>>(args...));
+ }
+};
+
+# if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename T, typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct statically_named_arg : view {
+ static constexpr auto name = Str.data;
+
+ const T& value;
+ statically_named_arg(const T& v) : value(v) {}
+};
+
+template <typename T, typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct is_named_arg<statically_named_arg<T, Char, N, Str>> : std::true_type {};
+
+template <typename T, typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct is_statically_named_arg<statically_named_arg<T, Char, N, Str>>
+ : std::true_type {};
+
+template <typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct udl_arg {
+ template <typename T> auto operator=(T&& value) const {
+ return statically_named_arg<T, Char, N, Str>(std::forward<T>(value));
+ }
+};
+# else
+template <typename Char> struct udl_arg {
+ const Char* str;
+
+ template <typename T> auto operator=(T&& value) const -> named_arg<Char, T> {
+ return {str, std::forward<T>(value)};
+ }
+};
+# endif
+#endif // FMT_USE_USER_DEFINED_LITERALS
+
+template <typename Locale, typename Char>
+auto vformat(const Locale& loc, basic_string_view<Char> fmt,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ auto buf = basic_memory_buffer<Char>();
+ detail::vformat_to(buf, fmt, args, detail::locale_ref(loc));
+ return {buf.data(), buf.size()};
+}
+
+using format_func = void (*)(detail::buffer<char>&, int, const char*);
+
+FMT_API void format_error_code(buffer<char>& out, int error_code,
+ string_view message) noexcept;
+
+FMT_API void report_error(format_func func, int error_code,
+ const char* message) noexcept;
+FMT_END_DETAIL_NAMESPACE
+
+FMT_API auto vsystem_error(int error_code, string_view format_str,
+ format_args args) -> std::system_error;
+
+/**
+ \rst
+ Constructs :class:`std::system_error` with a message formatted with
+ ``fmt::format(fmt, args...)``.
+ *error_code* is a system error code as given by ``errno``.
+
+ **Example**::
+
+ // This throws std::system_error with the description
+ // cannot open file 'madeup': No such file or directory
+ // or similar (system message may vary).
+ const char* filename = "madeup";
+ std::FILE* file = std::fopen(filename, "r");
+ if (!file)
+ throw fmt::system_error(errno, "cannot open file '{}'", filename);
+ \endrst
+*/
+template <typename... T>
+auto system_error(int error_code, format_string<T...> fmt, T&&... args)
+ -> std::system_error {
+ return vsystem_error(error_code, fmt, fmt::make_format_args(args...));
+}
+
+/**
+ \rst
+ Formats an error message for an error returned by an operating system or a
+ language runtime, for example a file opening error, and writes it to *out*.
+ The format is the same as the one used by ``std::system_error(ec, message)``
+ where ``ec`` is ``std::error_code(error_code, std::generic_category()})``.
+ It is implementation-defined but normally looks like:
+
+ .. parsed-literal::
+ *<message>*: *<system-message>*
+
+ where *<message>* is the passed message and *<system-message>* is the system
+ message corresponding to the error code.
+ *error_code* is a system error code as given by ``errno``.
+ \endrst
+ */
+FMT_API void format_system_error(detail::buffer<char>& out, int error_code,
+ const char* message) noexcept;
+
+// Reports a system error without throwing an exception.
+// Can be used to report errors from destructors.
+FMT_API void report_system_error(int error_code, const char* message) noexcept;
+
+/** Fast integer formatter. */
+class format_int {
+ private:
+ // Buffer should be large enough to hold all digits (digits10 + 1),
+ // a sign and a null character.
+ enum { buffer_size = std::numeric_limits<unsigned long long>::digits10 + 3 };
+ mutable char buffer_[buffer_size];
+ char* str_;
+
+ template <typename UInt> auto format_unsigned(UInt value) -> char* {
+ auto n = static_cast<detail::uint32_or_64_or_128_t<UInt>>(value);
+ return detail::format_decimal(buffer_, n, buffer_size - 1).begin;
+ }
+
+ template <typename Int> auto format_signed(Int value) -> char* {
+ auto abs_value = static_cast<detail::uint32_or_64_or_128_t<Int>>(value);
+ bool negative = value < 0;
+ if (negative) abs_value = 0 - abs_value;
+ auto begin = format_unsigned(abs_value);
+ if (negative) *--begin = '-';
+ return begin;
+ }
+
+ public:
+ explicit format_int(int value) : str_(format_signed(value)) {}
+ explicit format_int(long value) : str_(format_signed(value)) {}
+ explicit format_int(long long value) : str_(format_signed(value)) {}
+ explicit format_int(unsigned value) : str_(format_unsigned(value)) {}
+ explicit format_int(unsigned long value) : str_(format_unsigned(value)) {}
+ explicit format_int(unsigned long long value)
+ : str_(format_unsigned(value)) {}
+
+ /** Returns the number of characters written to the output buffer. */
+ auto size() const -> size_t {
+ return detail::to_unsigned(buffer_ - str_ + buffer_size - 1);
+ }
+
+ /**
+ Returns a pointer to the output buffer content. No terminating null
+ character is appended.
+ */
+ auto data() const -> const char* { return str_; }
+
+ /**
+ Returns a pointer to the output buffer content with terminating null
+ character appended.
+ */
+ auto c_str() const -> const char* {
+ buffer_[buffer_size - 1] = '\0';
+ return str_;
+ }
+
+ /**
+ \rst
+ Returns the content of the output buffer as an ``std::string``.
+ \endrst
+ */
+ auto str() const -> std::string { return std::string(str_, size()); }
+};
+
+template <typename T, typename Char>
+struct formatter<T, Char, enable_if_t<detail::has_format_as<T>::value>>
+ : private formatter<detail::format_as_t<T>> {
+ using base = formatter<detail::format_as_t<T>>;
+ using base::parse;
+
+ template <typename FormatContext>
+ auto format(const T& value, FormatContext& ctx) const -> decltype(ctx.out()) {
+ return base::format(format_as(value), ctx);
+ }
+};
+
+template <typename Char>
+struct formatter<void*, Char> : formatter<const void*, Char> {
+ template <typename FormatContext>
+ auto format(void* val, FormatContext& ctx) const -> decltype(ctx.out()) {
+ return formatter<const void*, Char>::format(val, ctx);
+ }
+};
+
+template <typename Char, size_t N>
+struct formatter<Char[N], Char> : formatter<basic_string_view<Char>, Char> {
+ template <typename FormatContext>
+ FMT_CONSTEXPR auto format(const Char* val, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ return formatter<basic_string_view<Char>, Char>::format(val, ctx);
+ }
+};
+
+/**
+ \rst
+ Converts ``p`` to ``const void*`` for pointer formatting.
+
+ **Example**::
+
+ auto s = fmt::format("{}", fmt::ptr(p));
+ \endrst
+ */
+template <typename T> auto ptr(T p) -> const void* {
+ static_assert(std::is_pointer<T>::value, "");
+ return detail::bit_cast<const void*>(p);
+}
+template <typename T, typename Deleter>
+auto ptr(const std::unique_ptr<T, Deleter>& p) -> const void* {
+ return p.get();
+}
+template <typename T> auto ptr(const std::shared_ptr<T>& p) -> const void* {
+ return p.get();
+}
+
+/**
+ \rst
+ Converts ``e`` to the underlying type.
+
+ **Example**::
+
+ enum class color { red, green, blue };
+ auto s = fmt::format("{}", fmt::underlying(color::red));
+ \endrst
+ */
+template <typename Enum>
+constexpr auto underlying(Enum e) noexcept -> underlying_t<Enum> {
+ return static_cast<underlying_t<Enum>>(e);
+}
+
+namespace enums {
+template <typename Enum, FMT_ENABLE_IF(std::is_enum<Enum>::value)>
+constexpr auto format_as(Enum e) noexcept -> underlying_t<Enum> {
+ return static_cast<underlying_t<Enum>>(e);
+}
+} // namespace enums
+
+class bytes {
+ private:
+ string_view data_;
+ friend struct formatter<bytes>;
+
+ public:
+ explicit bytes(string_view data) : data_(data) {}
+};
+
+template <> struct formatter<bytes> {
+ private:
+ detail::dynamic_format_specs<> specs_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const char* {
+ return parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx,
+ detail::type::string_type);
+ }
+
+ template <typename FormatContext>
+ auto format(bytes b, FormatContext& ctx) -> decltype(ctx.out()) {
+ detail::handle_dynamic_spec<detail::width_checker>(specs_.width,
+ specs_.width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs_.precision, specs_.precision_ref, ctx);
+ return detail::write_bytes(ctx.out(), b.data_, specs_);
+ }
+};
+
+// group_digits_view is not derived from view because it copies the argument.
+template <typename T> struct group_digits_view { T value; };
+
+/**
+ \rst
+ Returns a view that formats an integer value using ',' as a locale-independent
+ thousands separator.
+
+ **Example**::
+
+ fmt::print("{}", fmt::group_digits(12345));
+ // Output: "12,345"
+ \endrst
+ */
+template <typename T> auto group_digits(T value) -> group_digits_view<T> {
+ return {value};
+}
+
+template <typename T> struct formatter<group_digits_view<T>> : formatter<T> {
+ private:
+ detail::dynamic_format_specs<> specs_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const char* {
+ return parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx,
+ detail::type::int_type);
+ }
+
+ template <typename FormatContext>
+ auto format(group_digits_view<T> t, FormatContext& ctx)
+ -> decltype(ctx.out()) {
+ detail::handle_dynamic_spec<detail::width_checker>(specs_.width,
+ specs_.width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs_.precision, specs_.precision_ref, ctx);
+ return detail::write_int(
+ ctx.out(), static_cast<detail::uint64_or_128_t<T>>(t.value), 0, specs_,
+ detail::digit_grouping<char>("\3", ","));
+ }
+};
+
+// DEPRECATED! join_view will be moved to ranges.h.
+template <typename It, typename Sentinel, typename Char = char>
+struct join_view : detail::view {
+ It begin;
+ Sentinel end;
+ basic_string_view<Char> sep;
+
+ join_view(It b, Sentinel e, basic_string_view<Char> s)
+ : begin(b), end(e), sep(s) {}
+};
+
+template <typename It, typename Sentinel, typename Char>
+struct formatter<join_view<It, Sentinel, Char>, Char> {
+ private:
+ using value_type =
+#ifdef __cpp_lib_ranges
+ std::iter_value_t<It>;
+#else
+ typename std::iterator_traits<It>::value_type;
+#endif
+ formatter<remove_cvref_t<value_type>, Char> value_formatter_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const Char* {
+ return value_formatter_.parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(const join_view<It, Sentinel, Char>& value,
+ FormatContext& ctx) const -> decltype(ctx.out()) {
+ auto it = value.begin;
+ auto out = ctx.out();
+ if (it != value.end) {
+ out = value_formatter_.format(*it, ctx);
+ ++it;
+ while (it != value.end) {
+ out = detail::copy_str<Char>(value.sep.begin(), value.sep.end(), out);
+ ctx.advance_to(out);
+ out = value_formatter_.format(*it, ctx);
+ ++it;
+ }
+ }
+ return out;
+ }
+};
+
+/**
+ Returns a view that formats the iterator range `[begin, end)` with elements
+ separated by `sep`.
+ */
+template <typename It, typename Sentinel>
+auto join(It begin, Sentinel end, string_view sep) -> join_view<It, Sentinel> {
+ return {begin, end, sep};
+}
+
+/**
+ \rst
+ Returns a view that formats `range` with elements separated by `sep`.
+
+ **Example**::
+
+ std::vector<int> v = {1, 2, 3};
+ fmt::print("{}", fmt::join(v, ", "));
+ // Output: "1, 2, 3"
+
+ ``fmt::join`` applies passed format specifiers to the range elements::
+
+ fmt::print("{:02}", fmt::join(v, ", "));
+ // Output: "01, 02, 03"
+ \endrst
+ */
+template <typename Range>
+auto join(Range&& range, string_view sep)
+ -> join_view<detail::iterator_t<Range>, detail::sentinel_t<Range>> {
+ return join(std::begin(range), std::end(range), sep);
+}
+
+/**
+ \rst
+ Converts *value* to ``std::string`` using the default format for type *T*.
+
+ **Example**::
+
+ #include <fmt/format.h>
+
+ std::string answer = fmt::to_string(42);
+ \endrst
+ */
+template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+inline auto to_string(const T& value) -> std::string {
+ auto buffer = memory_buffer();
+ detail::write<char>(appender(buffer), value);
+ return {buffer.data(), buffer.size()};
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+FMT_NODISCARD inline auto to_string(T value) -> std::string {
+ // The buffer should be large enough to store the number including the sign
+ // or "false" for bool.
+ constexpr int max_size = detail::digits10<T>() + 2;
+ char buffer[max_size > 5 ? static_cast<unsigned>(max_size) : 5];
+ char* begin = buffer;
+ return std::string(begin, detail::write<char>(begin, value));
+}
+
+template <typename Char, size_t SIZE>
+FMT_NODISCARD auto to_string(const basic_memory_buffer<Char, SIZE>& buf)
+ -> std::basic_string<Char> {
+ auto size = buf.size();
+ detail::assume(size < std::basic_string<Char>().max_size());
+ return std::basic_string<Char>(buf.data(), size);
+}
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+template <typename Char>
+void vformat_to(buffer<Char>& buf, basic_string_view<Char> fmt,
+ typename vformat_args<Char>::type args, locale_ref loc) {
+ auto out = buffer_appender<Char>(buf);
+ if (fmt.size() == 2 && equal2(fmt.data(), "{}")) {
+ auto arg = args.get(0);
+ if (!arg) error_handler().on_error("argument not found");
+ visit_format_arg(default_arg_formatter<Char>{out, args, loc}, arg);
+ return;
+ }
+
+ struct format_handler : error_handler {
+ basic_format_parse_context<Char> parse_context;
+ buffer_context<Char> context;
+
+ format_handler(buffer_appender<Char> p_out, basic_string_view<Char> str,
+ basic_format_args<buffer_context<Char>> p_args,
+ locale_ref p_loc)
+ : parse_context(str), context(p_out, p_args, p_loc) {}
+
+ void on_text(const Char* begin, const Char* end) {
+ auto text = basic_string_view<Char>(begin, to_unsigned(end - begin));
+ context.advance_to(write<Char>(context.out(), text));
+ }
+
+ FMT_CONSTEXPR auto on_arg_id() -> int {
+ return parse_context.next_arg_id();
+ }
+ FMT_CONSTEXPR auto on_arg_id(int id) -> int {
+ return parse_context.check_arg_id(id), id;
+ }
+ FMT_CONSTEXPR auto on_arg_id(basic_string_view<Char> id) -> int {
+ int arg_id = context.arg_id(id);
+ if (arg_id < 0) on_error("argument not found");
+ return arg_id;
+ }
+
+ FMT_INLINE void on_replacement_field(int id, const Char*) {
+ auto arg = get_arg(context, id);
+ context.advance_to(visit_format_arg(
+ default_arg_formatter<Char>{context.out(), context.args(),
+ context.locale()},
+ arg));
+ }
+
+ auto on_format_specs(int id, const Char* begin, const Char* end)
+ -> const Char* {
+ auto arg = get_arg(context, id);
+ if (arg.type() == type::custom_type) {
+ parse_context.advance_to(begin);
+ visit_format_arg(custom_formatter<Char>{parse_context, context}, arg);
+ return parse_context.begin();
+ }
+ auto specs = detail::dynamic_format_specs<Char>();
+ begin = parse_format_specs(begin, end, specs, parse_context, arg.type());
+ detail::handle_dynamic_spec<detail::width_checker>(
+ specs.width, specs.width_ref, context);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs.precision, specs.precision_ref, context);
+ if (begin == end || *begin != '}')
+ on_error("missing '}' in format string");
+ auto f = arg_formatter<Char>{context.out(), specs, context.locale()};
+ context.advance_to(visit_format_arg(f, arg));
+ return begin;
+ }
+ };
+ detail::parse_format_string<false>(fmt, format_handler(out, fmt, args, loc));
+}
+
+#ifndef FMT_HEADER_ONLY
+extern template FMT_API void vformat_to(buffer<char>&, string_view,
+ typename vformat_args<>::type,
+ locale_ref);
+extern template FMT_API auto thousands_sep_impl<char>(locale_ref)
+ -> thousands_sep_result<char>;
+extern template FMT_API auto thousands_sep_impl<wchar_t>(locale_ref)
+ -> thousands_sep_result<wchar_t>;
+extern template FMT_API auto decimal_point_impl(locale_ref) -> char;
+extern template FMT_API auto decimal_point_impl(locale_ref) -> wchar_t;
+#endif // FMT_HEADER_ONLY
+
+FMT_END_DETAIL_NAMESPACE
+
+#if FMT_USE_USER_DEFINED_LITERALS
+inline namespace literals {
+/**
+ \rst
+ User-defined literal equivalent of :func:`fmt::arg`.
+
+ **Example**::
+
+ using namespace fmt::literals;
+ fmt::print("Elapsed time: {s:.2f} seconds", "s"_a=1.23);
+ \endrst
+ */
+# if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <detail_exported::fixed_string Str> constexpr auto operator""_a() {
+ using char_t = remove_cvref_t<decltype(Str.data[0])>;
+ return detail::udl_arg<char_t, sizeof(Str.data) / sizeof(char_t), Str>();
+}
+# else
+constexpr auto operator"" _a(const char* s, size_t) -> detail::udl_arg<char> {
+ return {s};
+}
+# endif
+} // namespace literals
+#endif // FMT_USE_USER_DEFINED_LITERALS
+
+template <typename Locale, FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+inline auto vformat(const Locale& loc, string_view fmt, format_args args)
+ -> std::string {
+ return detail::vformat(loc, fmt, args);
+}
+
+template <typename Locale, typename... T,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+inline auto format(const Locale& loc, format_string<T...> fmt, T&&... args)
+ -> std::string {
+ return fmt::vformat(loc, string_view(fmt), fmt::make_format_args(args...));
+}
+
+template <typename OutputIt, typename Locale,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value&&
+ detail::is_locale<Locale>::value)>
+auto vformat_to(OutputIt out, const Locale& loc, string_view fmt,
+ format_args args) -> OutputIt {
+ using detail::get_buffer;
+ auto&& buf = get_buffer<char>(out);
+ detail::vformat_to(buf, fmt, args, detail::locale_ref(loc));
+ return detail::get_iterator(buf, out);
+}
+
+template <typename OutputIt, typename Locale, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value&&
+ detail::is_locale<Locale>::value)>
+FMT_INLINE auto format_to(OutputIt out, const Locale& loc,
+ format_string<T...> fmt, T&&... args) -> OutputIt {
+ return vformat_to(out, loc, fmt, fmt::make_format_args(args...));
+}
+
+template <typename Locale, typename... T,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+FMT_NODISCARD FMT_INLINE auto formatted_size(const Locale& loc,
+ format_string<T...> fmt,
+ T&&... args) -> size_t {
+ auto buf = detail::counting_buffer<>();
+ detail::vformat_to<char>(buf, fmt, fmt::make_format_args(args...),
+ detail::locale_ref(loc));
+ return buf.count();
+}
+
+FMT_END_EXPORT
+
+template <typename T, typename Char>
+template <typename FormatContext>
+FMT_CONSTEXPR FMT_INLINE auto
+formatter<T, Char,
+ enable_if_t<detail::type_constant<T, Char>::value !=
+ detail::type::custom_type>>::format(const T& val,
+ FormatContext& ctx)
+ const -> decltype(ctx.out()) {
+ if (specs_.width_ref.kind != detail::arg_id_kind::none ||
+ specs_.precision_ref.kind != detail::arg_id_kind::none) {
+ auto specs = specs_;
+ detail::handle_dynamic_spec<detail::width_checker>(specs.width,
+ specs.width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs.precision, specs.precision_ref, ctx);
+ return detail::write<Char>(ctx.out(), val, specs, ctx.locale());
+ }
+ return detail::write<Char>(ctx.out(), val, specs_, ctx.locale());
+}
+
+FMT_END_NAMESPACE
+
+#ifdef FMT_HEADER_ONLY
+# define FMT_FUNC inline
+# include "format-inl.h"
+#else
+# define FMT_FUNC
+#endif
+
+#endif // FMT_FORMAT_H_
diff --git a/contrib/fmt/include/fmt/locale.h b/contrib/fmt/include/fmt/locale.h
new file mode 100644
index 0000000..7571b52
--- /dev/null
+++ b/contrib/fmt/include/fmt/locale.h
@@ -0,0 +1,2 @@
+#include "xchar.h"
+#warning fmt/locale.h is deprecated, include fmt/format.h or fmt/xchar.h instead
diff --git a/contrib/fmt/include/fmt/os.h b/contrib/fmt/include/fmt/os.h
new file mode 100644
index 0000000..ec29040
--- /dev/null
+++ b/contrib/fmt/include/fmt/os.h
@@ -0,0 +1,451 @@
+// Formatting library for C++ - optional OS-specific functionality
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_OS_H_
+#define FMT_OS_H_
+
+#include <cerrno>
+#include <cstddef>
+#include <cstdio>
+#include <system_error> // std::system_error
+
+#if defined __APPLE__ || defined(__FreeBSD__)
+# include <xlocale.h> // for LC_NUMERIC_MASK on OS X
+#endif
+
+#include "format.h"
+
+#ifndef FMT_USE_FCNTL
+// UWP doesn't provide _pipe.
+# if FMT_HAS_INCLUDE("winapifamily.h")
+# include <winapifamily.h>
+# endif
+# if (FMT_HAS_INCLUDE(<fcntl.h>) || defined(__APPLE__) || \
+ defined(__linux__)) && \
+ (!defined(WINAPI_FAMILY) || \
+ (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+# include <fcntl.h> // for O_RDONLY
+# define FMT_USE_FCNTL 1
+# else
+# define FMT_USE_FCNTL 0
+# endif
+#endif
+
+#ifndef FMT_POSIX
+# if defined(_WIN32) && !defined(__MINGW32__)
+// Fix warnings about deprecated symbols.
+# define FMT_POSIX(call) _##call
+# else
+# define FMT_POSIX(call) call
+# endif
+#endif
+
+// Calls to system functions are wrapped in FMT_SYSTEM for testability.
+#ifdef FMT_SYSTEM
+# define FMT_POSIX_CALL(call) FMT_SYSTEM(call)
+#else
+# define FMT_SYSTEM(call) ::call
+# ifdef _WIN32
+// Fix warnings about deprecated symbols.
+# define FMT_POSIX_CALL(call) ::_##call
+# else
+# define FMT_POSIX_CALL(call) ::call
+# endif
+#endif
+
+// Retries the expression while it evaluates to error_result and errno
+// equals to EINTR.
+#ifndef _WIN32
+# define FMT_RETRY_VAL(result, expression, error_result) \
+ do { \
+ (result) = (expression); \
+ } while ((result) == (error_result) && errno == EINTR)
+#else
+# define FMT_RETRY_VAL(result, expression, error_result) result = (expression)
+#endif
+
+#define FMT_RETRY(result, expression) FMT_RETRY_VAL(result, expression, -1)
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+/**
+ \rst
+ A reference to a null-terminated string. It can be constructed from a C
+ string or ``std::string``.
+
+ You can use one of the following type aliases for common character types:
+
+ +---------------+-----------------------------+
+ | Type | Definition |
+ +===============+=============================+
+ | cstring_view | basic_cstring_view<char> |
+ +---------------+-----------------------------+
+ | wcstring_view | basic_cstring_view<wchar_t> |
+ +---------------+-----------------------------+
+
+ This class is most useful as a parameter type to allow passing
+ different types of strings to a function, for example::
+
+ template <typename... Args>
+ std::string format(cstring_view format_str, const Args & ... args);
+
+ format("{}", 42);
+ format(std::string("{}"), 42);
+ \endrst
+ */
+template <typename Char> class basic_cstring_view {
+ private:
+ const Char* data_;
+
+ public:
+ /** Constructs a string reference object from a C string. */
+ basic_cstring_view(const Char* s) : data_(s) {}
+
+ /**
+ \rst
+ Constructs a string reference from an ``std::string`` object.
+ \endrst
+ */
+ basic_cstring_view(const std::basic_string<Char>& s) : data_(s.c_str()) {}
+
+ /** Returns the pointer to a C string. */
+ const Char* c_str() const { return data_; }
+};
+
+using cstring_view = basic_cstring_view<char>;
+using wcstring_view = basic_cstring_view<wchar_t>;
+
+#ifdef _WIN32
+FMT_API const std::error_category& system_category() noexcept;
+
+FMT_BEGIN_DETAIL_NAMESPACE
+FMT_API void format_windows_error(buffer<char>& out, int error_code,
+ const char* message) noexcept;
+FMT_END_DETAIL_NAMESPACE
+
+FMT_API std::system_error vwindows_error(int error_code, string_view format_str,
+ format_args args);
+
+/**
+ \rst
+ Constructs a :class:`std::system_error` object with the description
+ of the form
+
+ .. parsed-literal::
+ *<message>*: *<system-message>*
+
+ where *<message>* is the formatted message and *<system-message>* is the
+ system message corresponding to the error code.
+ *error_code* is a Windows error code as given by ``GetLastError``.
+ If *error_code* is not a valid error code such as -1, the system message
+ will look like "error -1".
+
+ **Example**::
+
+ // This throws a system_error with the description
+ // cannot open file 'madeup': The system cannot find the file specified.
+ // or similar (system message may vary).
+ const char *filename = "madeup";
+ LPOFSTRUCT of = LPOFSTRUCT();
+ HFILE file = OpenFile(filename, &of, OF_READ);
+ if (file == HFILE_ERROR) {
+ throw fmt::windows_error(GetLastError(),
+ "cannot open file '{}'", filename);
+ }
+ \endrst
+*/
+template <typename... Args>
+std::system_error windows_error(int error_code, string_view message,
+ const Args&... args) {
+ return vwindows_error(error_code, message, fmt::make_format_args(args...));
+}
+
+// Reports a Windows error without throwing an exception.
+// Can be used to report errors from destructors.
+FMT_API void report_windows_error(int error_code, const char* message) noexcept;
+#else
+inline const std::error_category& system_category() noexcept {
+ return std::system_category();
+}
+#endif // _WIN32
+
+// std::system is not available on some platforms such as iOS (#2248).
+#ifdef __OSX__
+template <typename S, typename... Args, typename Char = char_t<S>>
+void say(const S& format_str, Args&&... args) {
+ std::system(format("say \"{}\"", format(format_str, args...)).c_str());
+}
+#endif
+
+// A buffered file.
+class buffered_file {
+ private:
+ FILE* file_;
+
+ friend class file;
+
+ explicit buffered_file(FILE* f) : file_(f) {}
+
+ public:
+ buffered_file(const buffered_file&) = delete;
+ void operator=(const buffered_file&) = delete;
+
+ // Constructs a buffered_file object which doesn't represent any file.
+ buffered_file() noexcept : file_(nullptr) {}
+
+ // Destroys the object closing the file it represents if any.
+ FMT_API ~buffered_file() noexcept;
+
+ public:
+ buffered_file(buffered_file&& other) noexcept : file_(other.file_) {
+ other.file_ = nullptr;
+ }
+
+ buffered_file& operator=(buffered_file&& other) {
+ close();
+ file_ = other.file_;
+ other.file_ = nullptr;
+ return *this;
+ }
+
+ // Opens a file.
+ FMT_API buffered_file(cstring_view filename, cstring_view mode);
+
+ // Closes the file.
+ FMT_API void close();
+
+ // Returns the pointer to a FILE object representing this file.
+ FILE* get() const noexcept { return file_; }
+
+ FMT_API int descriptor() const;
+
+ void vprint(string_view format_str, format_args args) {
+ fmt::vprint(file_, format_str, args);
+ }
+
+ template <typename... Args>
+ inline void print(string_view format_str, const Args&... args) {
+ vprint(format_str, fmt::make_format_args(args...));
+ }
+};
+
+#if FMT_USE_FCNTL
+// A file. Closed file is represented by a file object with descriptor -1.
+// Methods that are not declared with noexcept may throw
+// fmt::system_error in case of failure. Note that some errors such as
+// closing the file multiple times will cause a crash on Windows rather
+// than an exception. You can get standard behavior by overriding the
+// invalid parameter handler with _set_invalid_parameter_handler.
+class FMT_API file {
+ private:
+ int fd_; // File descriptor.
+
+ // Constructs a file object with a given descriptor.
+ explicit file(int fd) : fd_(fd) {}
+
+ public:
+ // Possible values for the oflag argument to the constructor.
+ enum {
+ RDONLY = FMT_POSIX(O_RDONLY), // Open for reading only.
+ WRONLY = FMT_POSIX(O_WRONLY), // Open for writing only.
+ RDWR = FMT_POSIX(O_RDWR), // Open for reading and writing.
+ CREATE = FMT_POSIX(O_CREAT), // Create if the file doesn't exist.
+ APPEND = FMT_POSIX(O_APPEND), // Open in append mode.
+ TRUNC = FMT_POSIX(O_TRUNC) // Truncate the content of the file.
+ };
+
+ // Constructs a file object which doesn't represent any file.
+ file() noexcept : fd_(-1) {}
+
+ // Opens a file and constructs a file object representing this file.
+ file(cstring_view path, int oflag);
+
+ public:
+ file(const file&) = delete;
+ void operator=(const file&) = delete;
+
+ file(file&& other) noexcept : fd_(other.fd_) { other.fd_ = -1; }
+
+ // Move assignment is not noexcept because close may throw.
+ file& operator=(file&& other) {
+ close();
+ fd_ = other.fd_;
+ other.fd_ = -1;
+ return *this;
+ }
+
+ // Destroys the object closing the file it represents if any.
+ ~file() noexcept;
+
+ // Returns the file descriptor.
+ int descriptor() const noexcept { return fd_; }
+
+ // Closes the file.
+ void close();
+
+ // Returns the file size. The size has signed type for consistency with
+ // stat::st_size.
+ long long size() const;
+
+ // Attempts to read count bytes from the file into the specified buffer.
+ size_t read(void* buffer, size_t count);
+
+ // Attempts to write count bytes from the specified buffer to the file.
+ size_t write(const void* buffer, size_t count);
+
+ // Duplicates a file descriptor with the dup function and returns
+ // the duplicate as a file object.
+ static file dup(int fd);
+
+ // Makes fd be the copy of this file descriptor, closing fd first if
+ // necessary.
+ void dup2(int fd);
+
+ // Makes fd be the copy of this file descriptor, closing fd first if
+ // necessary.
+ void dup2(int fd, std::error_code& ec) noexcept;
+
+ // Creates a pipe setting up read_end and write_end file objects for reading
+ // and writing respectively.
+ static void pipe(file& read_end, file& write_end);
+
+ // Creates a buffered_file object associated with this file and detaches
+ // this file object from the file.
+ buffered_file fdopen(const char* mode);
+
+# if defined(_WIN32) && !defined(__MINGW32__)
+ // Opens a file and constructs a file object representing this file by
+ // wcstring_view filename. Windows only.
+ static file open_windows_file(wcstring_view path, int oflag);
+# endif
+};
+
+// Returns the memory page size.
+long getpagesize();
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+struct buffer_size {
+ buffer_size() = default;
+ size_t value = 0;
+ buffer_size operator=(size_t val) const {
+ auto bs = buffer_size();
+ bs.value = val;
+ return bs;
+ }
+};
+
+struct ostream_params {
+ int oflag = file::WRONLY | file::CREATE | file::TRUNC;
+ size_t buffer_size = BUFSIZ > 32768 ? BUFSIZ : 32768;
+
+ ostream_params() {}
+
+ template <typename... T>
+ ostream_params(T... params, int new_oflag) : ostream_params(params...) {
+ oflag = new_oflag;
+ }
+
+ template <typename... T>
+ ostream_params(T... params, detail::buffer_size bs)
+ : ostream_params(params...) {
+ this->buffer_size = bs.value;
+ }
+
+// Intel has a bug that results in failure to deduce a constructor
+// for empty parameter packs.
+# if defined(__INTEL_COMPILER) && __INTEL_COMPILER < 2000
+ ostream_params(int new_oflag) : oflag(new_oflag) {}
+ ostream_params(detail::buffer_size bs) : buffer_size(bs.value) {}
+# endif
+};
+
+class file_buffer final : public buffer<char> {
+ file file_;
+
+ FMT_API void grow(size_t) override;
+
+ public:
+ FMT_API file_buffer(cstring_view path, const ostream_params& params);
+ FMT_API file_buffer(file_buffer&& other);
+ FMT_API ~file_buffer();
+
+ void flush() {
+ if (size() == 0) return;
+ file_.write(data(), size() * sizeof(data()[0]));
+ clear();
+ }
+
+ void close() {
+ flush();
+ file_.close();
+ }
+};
+
+FMT_END_DETAIL_NAMESPACE
+
+// Added {} below to work around default constructor error known to
+// occur in Xcode versions 7.2.1 and 8.2.1.
+constexpr detail::buffer_size buffer_size{};
+
+/** A fast output stream which is not thread-safe. */
+class FMT_API ostream {
+ private:
+ FMT_MSC_WARNING(suppress : 4251)
+ detail::file_buffer buffer_;
+
+ ostream(cstring_view path, const detail::ostream_params& params)
+ : buffer_(path, params) {}
+
+ public:
+ ostream(ostream&& other) : buffer_(std::move(other.buffer_)) {}
+
+ ~ostream();
+
+ void flush() { buffer_.flush(); }
+
+ template <typename... T>
+ friend ostream output_file(cstring_view path, T... params);
+
+ void close() { buffer_.close(); }
+
+ /**
+ Formats ``args`` according to specifications in ``fmt`` and writes the
+ output to the file.
+ */
+ template <typename... T> void print(format_string<T...> fmt, T&&... args) {
+ vformat_to(detail::buffer_appender<char>(buffer_), fmt,
+ fmt::make_format_args(args...));
+ }
+};
+
+/**
+ \rst
+ Opens a file for writing. Supported parameters passed in *params*:
+
+ * ``<integer>``: Flags passed to `open
+ <https://pubs.opengroup.org/onlinepubs/007904875/functions/open.html>`_
+ (``file::WRONLY | file::CREATE | file::TRUNC`` by default)
+ * ``buffer_size=<integer>``: Output buffer size
+
+ **Example**::
+
+ auto out = fmt::output_file("guide.txt");
+ out.print("Don't {}", "Panic");
+ \endrst
+ */
+template <typename... T>
+inline ostream output_file(cstring_view path, T... params) {
+ return {path, detail::ostream_params(params...)};
+}
+#endif // FMT_USE_FCNTL
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_OS_H_
diff --git a/contrib/fmt/include/fmt/ostream.h b/contrib/fmt/include/fmt/ostream.h
new file mode 100644
index 0000000..ce65909
--- /dev/null
+++ b/contrib/fmt/include/fmt/ostream.h
@@ -0,0 +1,209 @@
+// Formatting library for C++ - std::ostream support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_OSTREAM_H_
+#define FMT_OSTREAM_H_
+
+#include <fstream> // std::filebuf
+
+#if defined(_WIN32) && defined(__GLIBCXX__)
+# include <ext/stdio_filebuf.h>
+# include <ext/stdio_sync_filebuf.h>
+#elif defined(_WIN32) && defined(_LIBCPP_VERSION)
+# include <__std_stream>
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+// Generate a unique explicit instantion in every translation unit using a tag
+// type in an anonymous namespace.
+namespace {
+struct file_access_tag {};
+} // namespace
+template <typename Tag, typename BufType, FILE* BufType::*FileMemberPtr>
+class file_access {
+ friend auto get_file(BufType& obj) -> FILE* { return obj.*FileMemberPtr; }
+};
+
+#if FMT_MSC_VERSION
+template class file_access<file_access_tag, std::filebuf,
+ &std::filebuf::_Myfile>;
+auto get_file(std::filebuf&) -> FILE*;
+#elif defined(_WIN32) && defined(_LIBCPP_VERSION)
+template class file_access<file_access_tag, std::__stdoutbuf<char>,
+ &std::__stdoutbuf<char>::__file_>;
+auto get_file(std::__stdoutbuf<char>&) -> FILE*;
+#endif
+
+inline bool write_ostream_unicode(std::ostream& os, fmt::string_view data) {
+#if FMT_MSC_VERSION
+ if (auto* buf = dynamic_cast<std::filebuf*>(os.rdbuf()))
+ if (FILE* f = get_file(*buf)) return write_console(f, data);
+#elif defined(_WIN32) && defined(__GLIBCXX__)
+ auto* rdbuf = os.rdbuf();
+ FILE* c_file;
+ if (auto* sfbuf = dynamic_cast<__gnu_cxx::stdio_sync_filebuf<char>*>(rdbuf))
+ c_file = sfbuf->file();
+ else if (auto* fbuf = dynamic_cast<__gnu_cxx::stdio_filebuf<char>*>(rdbuf))
+ c_file = fbuf->file();
+ else
+ return false;
+ if (c_file) return write_console(c_file, data);
+#elif defined(_WIN32) && defined(_LIBCPP_VERSION)
+ if (auto* buf = dynamic_cast<std::__stdoutbuf<char>*>(os.rdbuf()))
+ if (FILE* f = get_file(*buf)) return write_console(f, data);
+#else
+ ignore_unused(os, data);
+#endif
+ return false;
+}
+inline bool write_ostream_unicode(std::wostream&,
+ fmt::basic_string_view<wchar_t>) {
+ return false;
+}
+
+// Write the content of buf to os.
+// It is a separate function rather than a part of vprint to simplify testing.
+template <typename Char>
+void write_buffer(std::basic_ostream<Char>& os, buffer<Char>& buf) {
+ const Char* buf_data = buf.data();
+ using unsigned_streamsize = std::make_unsigned<std::streamsize>::type;
+ unsigned_streamsize size = buf.size();
+ unsigned_streamsize max_size = to_unsigned(max_value<std::streamsize>());
+ do {
+ unsigned_streamsize n = size <= max_size ? size : max_size;
+ os.write(buf_data, static_cast<std::streamsize>(n));
+ buf_data += n;
+ size -= n;
+ } while (size != 0);
+}
+
+template <typename Char, typename T>
+void format_value(buffer<Char>& buf, const T& value,
+ locale_ref loc = locale_ref()) {
+ auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+ auto&& output = std::basic_ostream<Char>(&format_buf);
+#if !defined(FMT_STATIC_THOUSANDS_SEPARATOR)
+ if (loc) output.imbue(loc.get<std::locale>());
+#endif
+ output << value;
+ output.exceptions(std::ios_base::failbit | std::ios_base::badbit);
+}
+
+template <typename T> struct streamed_view { const T& value; };
+
+} // namespace detail
+
+// Formats an object of type T that has an overloaded ostream operator<<.
+template <typename Char>
+struct basic_ostream_formatter : formatter<basic_string_view<Char>, Char> {
+ void set_debug_format() = delete;
+
+ template <typename T, typename OutputIt>
+ auto format(const T& value, basic_format_context<OutputIt, Char>& ctx) const
+ -> OutputIt {
+ auto buffer = basic_memory_buffer<Char>();
+ detail::format_value(buffer, value, ctx.locale());
+ return formatter<basic_string_view<Char>, Char>::format(
+ {buffer.data(), buffer.size()}, ctx);
+ }
+};
+
+using ostream_formatter = basic_ostream_formatter<char>;
+
+template <typename T, typename Char>
+struct formatter<detail::streamed_view<T>, Char>
+ : basic_ostream_formatter<Char> {
+ template <typename OutputIt>
+ auto format(detail::streamed_view<T> view,
+ basic_format_context<OutputIt, Char>& ctx) const -> OutputIt {
+ return basic_ostream_formatter<Char>::format(view.value, ctx);
+ }
+};
+
+/**
+ \rst
+ Returns a view that formats `value` via an ostream ``operator<<``.
+
+ **Example**::
+
+ fmt::print("Current thread id: {}\n",
+ fmt::streamed(std::this_thread::get_id()));
+ \endrst
+ */
+template <typename T>
+auto streamed(const T& value) -> detail::streamed_view<T> {
+ return {value};
+}
+
+namespace detail {
+
+inline void vprint_directly(std::ostream& os, string_view format_str,
+ format_args args) {
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, format_str, args);
+ detail::write_buffer(os, buffer);
+}
+
+} // namespace detail
+
+FMT_MODULE_EXPORT template <typename Char>
+void vprint(std::basic_ostream<Char>& os,
+ basic_string_view<type_identity_t<Char>> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ auto buffer = basic_memory_buffer<Char>();
+ detail::vformat_to(buffer, format_str, args);
+ if (detail::write_ostream_unicode(os, {buffer.data(), buffer.size()})) return;
+ detail::write_buffer(os, buffer);
+}
+
+/**
+ \rst
+ Prints formatted data to the stream *os*.
+
+ **Example**::
+
+ fmt::print(cerr, "Don't {}!", "panic");
+ \endrst
+ */
+FMT_MODULE_EXPORT template <typename... T>
+void print(std::ostream& os, format_string<T...> fmt, T&&... args) {
+ const auto& vargs = fmt::make_format_args(args...);
+ if (detail::is_utf8())
+ vprint(os, fmt, vargs);
+ else
+ detail::vprint_directly(os, fmt, vargs);
+}
+
+FMT_MODULE_EXPORT
+template <typename... Args>
+void print(std::wostream& os,
+ basic_format_string<wchar_t, type_identity_t<Args>...> fmt,
+ Args&&... args) {
+ vprint(os, fmt, fmt::make_format_args<buffer_context<wchar_t>>(args...));
+}
+
+FMT_MODULE_EXPORT template <typename... T>
+void println(std::ostream& os, format_string<T...> fmt, T&&... args) {
+ fmt::print(os, "{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+FMT_MODULE_EXPORT
+template <typename... Args>
+void println(std::wostream& os,
+ basic_format_string<wchar_t, type_identity_t<Args>...> fmt,
+ Args&&... args) {
+ print(os, L"{}\n", fmt::format(fmt, std::forward<Args>(args)...));
+}
+
+FMT_END_NAMESPACE
+
+#endif // FMT_OSTREAM_H_
diff --git a/contrib/fmt/include/fmt/posix.h b/contrib/fmt/include/fmt/posix.h
new file mode 100644
index 0000000..da19e9d
--- /dev/null
+++ b/contrib/fmt/include/fmt/posix.h
@@ -0,0 +1,2 @@
+#include "os.h"
+#warning "fmt/posix.h is deprecated; use fmt/os.h instead"
diff --git a/contrib/fmt/include/fmt/printf.h b/contrib/fmt/include/fmt/printf.h
new file mode 100644
index 0000000..554715e
--- /dev/null
+++ b/contrib/fmt/include/fmt/printf.h
@@ -0,0 +1,679 @@
+// Formatting library for C++ - legacy printf implementation
+//
+// Copyright (c) 2012 - 2016, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_PRINTF_H_
+#define FMT_PRINTF_H_
+
+#include <algorithm> // std::max
+#include <limits> // std::numeric_limits
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+template <typename T> struct printf_formatter { printf_formatter() = delete; };
+
+template <typename Char>
+class basic_printf_parse_context : public basic_format_parse_context<Char> {
+ using basic_format_parse_context<Char>::basic_format_parse_context;
+};
+
+template <typename OutputIt, typename Char> class basic_printf_context {
+ private:
+ OutputIt out_;
+ basic_format_args<basic_printf_context> args_;
+
+ public:
+ using char_type = Char;
+ using format_arg = basic_format_arg<basic_printf_context>;
+ using parse_context_type = basic_printf_parse_context<Char>;
+ template <typename T> using formatter_type = printf_formatter<T>;
+
+ /**
+ \rst
+ Constructs a ``printf_context`` object. References to the arguments are
+ stored in the context object so make sure they have appropriate lifetimes.
+ \endrst
+ */
+ basic_printf_context(OutputIt out,
+ basic_format_args<basic_printf_context> args)
+ : out_(out), args_(args) {}
+
+ OutputIt out() { return out_; }
+ void advance_to(OutputIt it) { out_ = it; }
+
+ detail::locale_ref locale() { return {}; }
+
+ format_arg arg(int id) const { return args_.get(id); }
+
+ FMT_CONSTEXPR void on_error(const char* message) {
+ detail::error_handler().on_error(message);
+ }
+};
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// Checks if a value fits in int - used to avoid warnings about comparing
+// signed and unsigned integers.
+template <bool IsSigned> struct int_checker {
+ template <typename T> static bool fits_in_int(T value) {
+ unsigned max = max_value<int>();
+ return value <= max;
+ }
+ static bool fits_in_int(bool) { return true; }
+};
+
+template <> struct int_checker<true> {
+ template <typename T> static bool fits_in_int(T value) {
+ return value >= (std::numeric_limits<int>::min)() &&
+ value <= max_value<int>();
+ }
+ static bool fits_in_int(int) { return true; }
+};
+
+class printf_precision_handler {
+ public:
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ int operator()(T value) {
+ if (!int_checker<std::numeric_limits<T>::is_signed>::fits_in_int(value))
+ throw_format_error("number is too big");
+ return (std::max)(static_cast<int>(value), 0);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ int operator()(T) {
+ throw_format_error("precision is not integer");
+ return 0;
+ }
+};
+
+// An argument visitor that returns true iff arg is a zero integer.
+class is_zero_int {
+ public:
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ bool operator()(T value) {
+ return value == 0;
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ bool operator()(T) {
+ return false;
+ }
+};
+
+template <typename T> struct make_unsigned_or_bool : std::make_unsigned<T> {};
+
+template <> struct make_unsigned_or_bool<bool> { using type = bool; };
+
+template <typename T, typename Context> class arg_converter {
+ private:
+ using char_type = typename Context::char_type;
+
+ basic_format_arg<Context>& arg_;
+ char_type type_;
+
+ public:
+ arg_converter(basic_format_arg<Context>& arg, char_type type)
+ : arg_(arg), type_(type) {}
+
+ void operator()(bool value) {
+ if (type_ != 's') operator()<bool>(value);
+ }
+
+ template <typename U, FMT_ENABLE_IF(std::is_integral<U>::value)>
+ void operator()(U value) {
+ bool is_signed = type_ == 'd' || type_ == 'i';
+ using target_type = conditional_t<std::is_same<T, void>::value, U, T>;
+ if (const_check(sizeof(target_type) <= sizeof(int))) {
+ // Extra casts are used to silence warnings.
+ if (is_signed) {
+ arg_ = detail::make_arg<Context>(
+ static_cast<int>(static_cast<target_type>(value)));
+ } else {
+ using unsigned_type = typename make_unsigned_or_bool<target_type>::type;
+ arg_ = detail::make_arg<Context>(
+ static_cast<unsigned>(static_cast<unsigned_type>(value)));
+ }
+ } else {
+ if (is_signed) {
+ // glibc's printf doesn't sign extend arguments of smaller types:
+ // std::printf("%lld", -42); // prints "4294967254"
+ // but we don't have to do the same because it's a UB.
+ arg_ = detail::make_arg<Context>(static_cast<long long>(value));
+ } else {
+ arg_ = detail::make_arg<Context>(
+ static_cast<typename make_unsigned_or_bool<U>::type>(value));
+ }
+ }
+ }
+
+ template <typename U, FMT_ENABLE_IF(!std::is_integral<U>::value)>
+ void operator()(U) {} // No conversion needed for non-integral types.
+};
+
+// Converts an integer argument to T for printf, if T is an integral type.
+// If T is void, the argument is converted to corresponding signed or unsigned
+// type depending on the type specifier: 'd' and 'i' - signed, other -
+// unsigned).
+template <typename T, typename Context, typename Char>
+void convert_arg(basic_format_arg<Context>& arg, Char type) {
+ visit_format_arg(arg_converter<T, Context>(arg, type), arg);
+}
+
+// Converts an integer argument to char for printf.
+template <typename Context> class char_converter {
+ private:
+ basic_format_arg<Context>& arg_;
+
+ public:
+ explicit char_converter(basic_format_arg<Context>& arg) : arg_(arg) {}
+
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ void operator()(T value) {
+ arg_ = detail::make_arg<Context>(
+ static_cast<typename Context::char_type>(value));
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ void operator()(T) {} // No conversion needed for non-integral types.
+};
+
+// An argument visitor that return a pointer to a C string if argument is a
+// string or null otherwise.
+template <typename Char> struct get_cstring {
+ template <typename T> const Char* operator()(T) { return nullptr; }
+ const Char* operator()(const Char* s) { return s; }
+};
+
+// Checks if an argument is a valid printf width specifier and sets
+// left alignment if it is negative.
+template <typename Char> class printf_width_handler {
+ private:
+ format_specs<Char>& specs_;
+
+ public:
+ explicit printf_width_handler(format_specs<Char>& specs) : specs_(specs) {}
+
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ unsigned operator()(T value) {
+ auto width = static_cast<uint32_or_64_or_128_t<T>>(value);
+ if (detail::is_negative(value)) {
+ specs_.align = align::left;
+ width = 0 - width;
+ }
+ unsigned int_max = max_value<int>();
+ if (width > int_max) throw_format_error("number is too big");
+ return static_cast<unsigned>(width);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ unsigned operator()(T) {
+ throw_format_error("width is not integer");
+ return 0;
+ }
+};
+
+// Workaround for a bug with the XL compiler when initializing
+// printf_arg_formatter's base class.
+template <typename Char>
+auto make_arg_formatter(buffer_appender<Char> iter, format_specs<Char>& s)
+ -> arg_formatter<Char> {
+ return {iter, s, locale_ref()};
+}
+
+// The ``printf`` argument formatter.
+template <typename OutputIt, typename Char>
+class printf_arg_formatter : public arg_formatter<Char> {
+ private:
+ using base = arg_formatter<Char>;
+ using context_type = basic_printf_context<OutputIt, Char>;
+
+ context_type& context_;
+
+ OutputIt write_null_pointer(bool is_string = false) {
+ auto s = this->specs;
+ s.type = presentation_type::none;
+ return write_bytes(this->out, is_string ? "(null)" : "(nil)", s);
+ }
+
+ public:
+ printf_arg_formatter(OutputIt iter, format_specs<Char>& s, context_type& ctx)
+ : base(make_arg_formatter(iter, s)), context_(ctx) {}
+
+ OutputIt operator()(monostate value) { return base::operator()(value); }
+
+ template <typename T, FMT_ENABLE_IF(detail::is_integral<T>::value)>
+ OutputIt operator()(T value) {
+ // MSVC2013 fails to compile separate overloads for bool and Char so use
+ // std::is_same instead.
+ if (std::is_same<T, Char>::value) {
+ format_specs<Char> fmt_specs = this->specs;
+ if (fmt_specs.type != presentation_type::none &&
+ fmt_specs.type != presentation_type::chr) {
+ return (*this)(static_cast<int>(value));
+ }
+ fmt_specs.sign = sign::none;
+ fmt_specs.alt = false;
+ fmt_specs.fill[0] = ' '; // Ignore '0' flag for char types.
+ // align::numeric needs to be overwritten here since the '0' flag is
+ // ignored for non-numeric types
+ if (fmt_specs.align == align::none || fmt_specs.align == align::numeric)
+ fmt_specs.align = align::right;
+ return write<Char>(this->out, static_cast<Char>(value), fmt_specs);
+ }
+ return base::operator()(value);
+ }
+
+ template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+ OutputIt operator()(T value) {
+ return base::operator()(value);
+ }
+
+ /** Formats a null-terminated C string. */
+ OutputIt operator()(const char* value) {
+ if (value) return base::operator()(value);
+ return write_null_pointer(this->specs.type != presentation_type::pointer);
+ }
+
+ /** Formats a null-terminated wide C string. */
+ OutputIt operator()(const wchar_t* value) {
+ if (value) return base::operator()(value);
+ return write_null_pointer(this->specs.type != presentation_type::pointer);
+ }
+
+ OutputIt operator()(basic_string_view<Char> value) {
+ return base::operator()(value);
+ }
+
+ /** Formats a pointer. */
+ OutputIt operator()(const void* value) {
+ return value ? base::operator()(value) : write_null_pointer();
+ }
+
+ /** Formats an argument of a custom (user-defined) type. */
+ OutputIt operator()(typename basic_format_arg<context_type>::handle handle) {
+ auto parse_ctx =
+ basic_printf_parse_context<Char>(basic_string_view<Char>());
+ handle.format(parse_ctx, context_);
+ return this->out;
+ }
+};
+
+template <typename Char>
+void parse_flags(format_specs<Char>& specs, const Char*& it, const Char* end) {
+ for (; it != end; ++it) {
+ switch (*it) {
+ case '-':
+ specs.align = align::left;
+ break;
+ case '+':
+ specs.sign = sign::plus;
+ break;
+ case '0':
+ specs.fill[0] = '0';
+ break;
+ case ' ':
+ if (specs.sign != sign::plus) {
+ specs.sign = sign::space;
+ }
+ break;
+ case '#':
+ specs.alt = true;
+ break;
+ default:
+ return;
+ }
+ }
+}
+
+template <typename Char, typename GetArg>
+int parse_header(const Char*& it, const Char* end, format_specs<Char>& specs,
+ GetArg get_arg) {
+ int arg_index = -1;
+ Char c = *it;
+ if (c >= '0' && c <= '9') {
+ // Parse an argument index (if followed by '$') or a width possibly
+ // preceded with '0' flag(s).
+ int value = parse_nonnegative_int(it, end, -1);
+ if (it != end && *it == '$') { // value is an argument index
+ ++it;
+ arg_index = value != -1 ? value : max_value<int>();
+ } else {
+ if (c == '0') specs.fill[0] = '0';
+ if (value != 0) {
+ // Nonzero value means that we parsed width and don't need to
+ // parse it or flags again, so return now.
+ if (value == -1) throw_format_error("number is too big");
+ specs.width = value;
+ return arg_index;
+ }
+ }
+ }
+ parse_flags(specs, it, end);
+ // Parse width.
+ if (it != end) {
+ if (*it >= '0' && *it <= '9') {
+ specs.width = parse_nonnegative_int(it, end, -1);
+ if (specs.width == -1) throw_format_error("number is too big");
+ } else if (*it == '*') {
+ ++it;
+ specs.width = static_cast<int>(visit_format_arg(
+ detail::printf_width_handler<Char>(specs), get_arg(-1)));
+ }
+ }
+ return arg_index;
+}
+
+inline auto parse_printf_presentation_type(char c, type t)
+ -> presentation_type {
+ using pt = presentation_type;
+ constexpr auto integral_set = sint_set | uint_set | bool_set | char_set;
+ switch (c) {
+ case 'd':
+ return in(t, integral_set) ? pt::dec : pt::none;
+ case 'o':
+ return in(t, integral_set) ? pt::oct : pt::none;
+ case 'x':
+ return in(t, integral_set) ? pt::hex_lower : pt::none;
+ case 'X':
+ return in(t, integral_set) ? pt::hex_upper : pt::none;
+ case 'a':
+ return in(t, float_set) ? pt::hexfloat_lower : pt::none;
+ case 'A':
+ return in(t, float_set) ? pt::hexfloat_upper : pt::none;
+ case 'e':
+ return in(t, float_set) ? pt::exp_lower : pt::none;
+ case 'E':
+ return in(t, float_set) ? pt::exp_upper : pt::none;
+ case 'f':
+ return in(t, float_set) ? pt::fixed_lower : pt::none;
+ case 'F':
+ return in(t, float_set) ? pt::fixed_upper : pt::none;
+ case 'g':
+ return in(t, float_set) ? pt::general_lower : pt::none;
+ case 'G':
+ return in(t, float_set) ? pt::general_upper : pt::none;
+ case 'c':
+ return in(t, integral_set) ? pt::chr : pt::none;
+ case 's':
+ return in(t, string_set | cstring_set) ? pt::string : pt::none;
+ case 'p':
+ return in(t, pointer_set | cstring_set) ? pt::pointer : pt::none;
+ default:
+ return pt::none;
+ }
+}
+
+template <typename Char, typename Context>
+void vprintf(buffer<Char>& buf, basic_string_view<Char> format,
+ basic_format_args<Context> args) {
+ using iterator = buffer_appender<Char>;
+ auto out = iterator(buf);
+ auto context = basic_printf_context<iterator, Char>(out, args);
+ auto parse_ctx = basic_printf_parse_context<Char>(format);
+
+ // Returns the argument with specified index or, if arg_index is -1, the next
+ // argument.
+ auto get_arg = [&](int arg_index) {
+ if (arg_index < 0)
+ arg_index = parse_ctx.next_arg_id();
+ else
+ parse_ctx.check_arg_id(--arg_index);
+ return detail::get_arg(context, arg_index);
+ };
+
+ const Char* start = parse_ctx.begin();
+ const Char* end = parse_ctx.end();
+ auto it = start;
+ while (it != end) {
+ if (!find<false, Char>(it, end, '%', it)) {
+ it = end; // find leaves it == nullptr if it doesn't find '%'.
+ break;
+ }
+ Char c = *it++;
+ if (it != end && *it == c) {
+ out = write(out, basic_string_view<Char>(start, to_unsigned(it - start)));
+ start = ++it;
+ continue;
+ }
+ out =
+ write(out, basic_string_view<Char>(start, to_unsigned(it - 1 - start)));
+
+ auto specs = format_specs<Char>();
+ specs.align = align::right;
+
+ // Parse argument index, flags and width.
+ int arg_index = parse_header(it, end, specs, get_arg);
+ if (arg_index == 0) throw_format_error("argument not found");
+
+ // Parse precision.
+ if (it != end && *it == '.') {
+ ++it;
+ c = it != end ? *it : 0;
+ if ('0' <= c && c <= '9') {
+ specs.precision = parse_nonnegative_int(it, end, 0);
+ } else if (c == '*') {
+ ++it;
+ specs.precision = static_cast<int>(
+ visit_format_arg(printf_precision_handler(), get_arg(-1)));
+ } else {
+ specs.precision = 0;
+ }
+ }
+
+ auto arg = get_arg(arg_index);
+ // For d, i, o, u, x, and X conversion specifiers, if a precision is
+ // specified, the '0' flag is ignored
+ if (specs.precision >= 0 && arg.is_integral())
+ specs.fill[0] =
+ ' '; // Ignore '0' flag for non-numeric types or if '-' present.
+ if (specs.precision >= 0 && arg.type() == type::cstring_type) {
+ auto str = visit_format_arg(get_cstring<Char>(), arg);
+ auto str_end = str + specs.precision;
+ auto nul = std::find(str, str_end, Char());
+ arg = make_arg<basic_printf_context<iterator, Char>>(
+ basic_string_view<Char>(
+ str, to_unsigned(nul != str_end ? nul - str : specs.precision)));
+ }
+ if (specs.alt && visit_format_arg(is_zero_int(), arg)) specs.alt = false;
+ if (specs.fill[0] == '0') {
+ if (arg.is_arithmetic() && specs.align != align::left)
+ specs.align = align::numeric;
+ else
+ specs.fill[0] = ' '; // Ignore '0' flag for non-numeric types or if '-'
+ // flag is also present.
+ }
+
+ // Parse length and convert the argument to the required type.
+ c = it != end ? *it++ : 0;
+ Char t = it != end ? *it : 0;
+ switch (c) {
+ case 'h':
+ if (t == 'h') {
+ ++it;
+ t = it != end ? *it : 0;
+ convert_arg<signed char>(arg, t);
+ } else {
+ convert_arg<short>(arg, t);
+ }
+ break;
+ case 'l':
+ if (t == 'l') {
+ ++it;
+ t = it != end ? *it : 0;
+ convert_arg<long long>(arg, t);
+ } else {
+ convert_arg<long>(arg, t);
+ }
+ break;
+ case 'j':
+ convert_arg<intmax_t>(arg, t);
+ break;
+ case 'z':
+ convert_arg<size_t>(arg, t);
+ break;
+ case 't':
+ convert_arg<std::ptrdiff_t>(arg, t);
+ break;
+ case 'L':
+ // printf produces garbage when 'L' is omitted for long double, no
+ // need to do the same.
+ break;
+ default:
+ --it;
+ convert_arg<void>(arg, c);
+ }
+
+ // Parse type.
+ if (it == end) throw_format_error("invalid format string");
+ char type = static_cast<char>(*it++);
+ if (arg.is_integral()) {
+ // Normalize type.
+ switch (type) {
+ case 'i':
+ case 'u':
+ type = 'd';
+ break;
+ case 'c':
+ visit_format_arg(
+ char_converter<basic_printf_context<iterator, Char>>(arg), arg);
+ break;
+ }
+ }
+ specs.type = parse_printf_presentation_type(type, arg.type());
+ if (specs.type == presentation_type::none)
+ throw_format_error("invalid format specifier");
+
+ start = it;
+
+ // Format argument.
+ out = visit_format_arg(
+ printf_arg_formatter<iterator, Char>(out, specs, context), arg);
+ }
+ write(out, basic_string_view<Char>(start, to_unsigned(it - start)));
+}
+FMT_END_DETAIL_NAMESPACE
+
+template <typename Char>
+using basic_printf_context_t =
+ basic_printf_context<detail::buffer_appender<Char>, Char>;
+
+using printf_context = basic_printf_context_t<char>;
+using wprintf_context = basic_printf_context_t<wchar_t>;
+
+using printf_args = basic_format_args<printf_context>;
+using wprintf_args = basic_format_args<wprintf_context>;
+
+/**
+ \rst
+ Constructs an `~fmt::format_arg_store` object that contains references to
+ arguments and can be implicitly converted to `~fmt::printf_args`.
+ \endrst
+ */
+template <typename... T>
+inline auto make_printf_args(const T&... args)
+ -> format_arg_store<printf_context, T...> {
+ return {args...};
+}
+
+/**
+ \rst
+ Constructs an `~fmt::format_arg_store` object that contains references to
+ arguments and can be implicitly converted to `~fmt::wprintf_args`.
+ \endrst
+ */
+template <typename... T>
+inline auto make_wprintf_args(const T&... args)
+ -> format_arg_store<wprintf_context, T...> {
+ return {args...};
+}
+
+template <typename S, typename Char = char_t<S>>
+inline auto vsprintf(
+ const S& fmt,
+ basic_format_args<basic_printf_context_t<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ auto buf = basic_memory_buffer<Char>();
+ detail::vprintf(buf, detail::to_string_view(fmt), args);
+ return to_string(buf);
+}
+
+/**
+ \rst
+ Formats arguments and returns the result as a string.
+
+ **Example**::
+
+ std::string message = fmt::sprintf("The answer is %d", 42);
+ \endrst
+*/
+template <typename S, typename... T,
+ typename Char = enable_if_t<detail::is_string<S>::value, char_t<S>>>
+inline auto sprintf(const S& fmt, const T&... args) -> std::basic_string<Char> {
+ using context = basic_printf_context_t<Char>;
+ return vsprintf(detail::to_string_view(fmt),
+ fmt::make_format_args<context>(args...));
+}
+
+template <typename S, typename Char = char_t<S>>
+inline auto vfprintf(
+ std::FILE* f, const S& fmt,
+ basic_format_args<basic_printf_context_t<type_identity_t<Char>>> args)
+ -> int {
+ auto buf = basic_memory_buffer<Char>();
+ detail::vprintf(buf, detail::to_string_view(fmt), args);
+ size_t size = buf.size();
+ return std::fwrite(buf.data(), sizeof(Char), size, f) < size
+ ? -1
+ : static_cast<int>(size);
+}
+
+/**
+ \rst
+ Prints formatted data to the file *f*.
+
+ **Example**::
+
+ fmt::fprintf(stderr, "Don't %s!", "panic");
+ \endrst
+ */
+template <typename S, typename... T, typename Char = char_t<S>>
+inline auto fprintf(std::FILE* f, const S& fmt, const T&... args) -> int {
+ using context = basic_printf_context_t<Char>;
+ return vfprintf(f, detail::to_string_view(fmt),
+ fmt::make_format_args<context>(args...));
+}
+
+template <typename S, typename Char = char_t<S>>
+inline auto vprintf(
+ const S& fmt,
+ basic_format_args<basic_printf_context_t<type_identity_t<Char>>> args)
+ -> int {
+ return vfprintf(stdout, detail::to_string_view(fmt), args);
+}
+
+/**
+ \rst
+ Prints formatted data to ``stdout``.
+
+ **Example**::
+
+ fmt::printf("Elapsed time: %.2f seconds", 1.23);
+ \endrst
+ */
+template <typename S, typename... T, FMT_ENABLE_IF(detail::is_string<S>::value)>
+inline auto printf(const S& fmt, const T&... args) -> int {
+ return vprintf(
+ detail::to_string_view(fmt),
+ fmt::make_format_args<basic_printf_context_t<char_t<S>>>(args...));
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_PRINTF_H_
diff --git a/contrib/fmt/include/fmt/ranges.h b/contrib/fmt/include/fmt/ranges.h
new file mode 100644
index 0000000..266b9e1
--- /dev/null
+++ b/contrib/fmt/include/fmt/ranges.h
@@ -0,0 +1,732 @@
+// Formatting library for C++ - experimental range support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+//
+// Copyright (c) 2018 - present, Remotion (Igor Schulz)
+// All Rights Reserved
+// {fmt} support for ranges, containers and types tuple interface.
+
+#ifndef FMT_RANGES_H_
+#define FMT_RANGES_H_
+
+#include <initializer_list>
+#include <tuple>
+#include <type_traits>
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename Range, typename OutputIt>
+auto copy(const Range& range, OutputIt out) -> OutputIt {
+ for (auto it = range.begin(), end = range.end(); it != end; ++it)
+ *out++ = *it;
+ return out;
+}
+
+template <typename OutputIt>
+auto copy(const char* str, OutputIt out) -> OutputIt {
+ while (*str) *out++ = *str++;
+ return out;
+}
+
+template <typename OutputIt> auto copy(char ch, OutputIt out) -> OutputIt {
+ *out++ = ch;
+ return out;
+}
+
+template <typename OutputIt> auto copy(wchar_t ch, OutputIt out) -> OutputIt {
+ *out++ = ch;
+ return out;
+}
+
+// Returns true if T has a std::string-like interface, like std::string_view.
+template <typename T> class is_std_string_like {
+ template <typename U>
+ static auto check(U* p)
+ -> decltype((void)p->find('a'), p->length(), (void)p->data(), int());
+ template <typename> static void check(...);
+
+ public:
+ static constexpr const bool value =
+ is_string<T>::value ||
+ std::is_convertible<T, std_string_view<char>>::value ||
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+template <typename Char>
+struct is_std_string_like<fmt::basic_string_view<Char>> : std::true_type {};
+
+template <typename T> class is_map {
+ template <typename U> static auto check(U*) -> typename U::mapped_type;
+ template <typename> static void check(...);
+
+ public:
+#ifdef FMT_FORMAT_MAP_AS_LIST // DEPRECATED!
+ static constexpr const bool value = false;
+#else
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+#endif
+};
+
+template <typename T> class is_set {
+ template <typename U> static auto check(U*) -> typename U::key_type;
+ template <typename> static void check(...);
+
+ public:
+#ifdef FMT_FORMAT_SET_AS_LIST // DEPRECATED!
+ static constexpr const bool value = false;
+#else
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value && !is_map<T>::value;
+#endif
+};
+
+template <typename... Ts> struct conditional_helper {};
+
+template <typename T, typename _ = void> struct is_range_ : std::false_type {};
+
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION > 1800
+
+# define FMT_DECLTYPE_RETURN(val) \
+ ->decltype(val) { return val; } \
+ static_assert( \
+ true, "") // This makes it so that a semicolon is required after the
+ // macro, which helps clang-format handle the formatting.
+
+// C array overload
+template <typename T, std::size_t N>
+auto range_begin(const T (&arr)[N]) -> const T* {
+ return arr;
+}
+template <typename T, std::size_t N>
+auto range_end(const T (&arr)[N]) -> const T* {
+ return arr + N;
+}
+
+template <typename T, typename Enable = void>
+struct has_member_fn_begin_end_t : std::false_type {};
+
+template <typename T>
+struct has_member_fn_begin_end_t<T, void_t<decltype(std::declval<T>().begin()),
+ decltype(std::declval<T>().end())>>
+ : std::true_type {};
+
+// Member function overload
+template <typename T>
+auto range_begin(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).begin());
+template <typename T>
+auto range_end(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).end());
+
+// ADL overload. Only participates in overload resolution if member functions
+// are not found.
+template <typename T>
+auto range_begin(T&& rng)
+ -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+ decltype(begin(static_cast<T&&>(rng)))> {
+ return begin(static_cast<T&&>(rng));
+}
+template <typename T>
+auto range_end(T&& rng) -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+ decltype(end(static_cast<T&&>(rng)))> {
+ return end(static_cast<T&&>(rng));
+}
+
+template <typename T, typename Enable = void>
+struct has_const_begin_end : std::false_type {};
+template <typename T, typename Enable = void>
+struct has_mutable_begin_end : std::false_type {};
+
+template <typename T>
+struct has_const_begin_end<
+ T,
+ void_t<
+ decltype(detail::range_begin(std::declval<const remove_cvref_t<T>&>())),
+ decltype(detail::range_end(std::declval<const remove_cvref_t<T>&>()))>>
+ : std::true_type {};
+
+template <typename T>
+struct has_mutable_begin_end<
+ T, void_t<decltype(detail::range_begin(std::declval<T>())),
+ decltype(detail::range_end(std::declval<T>())),
+ // the extra int here is because older versions of MSVC don't
+ // SFINAE properly unless there are distinct types
+ int>> : std::true_type {};
+
+template <typename T>
+struct is_range_<T, void>
+ : std::integral_constant<bool, (has_const_begin_end<T>::value ||
+ has_mutable_begin_end<T>::value)> {};
+# undef FMT_DECLTYPE_RETURN
+#endif
+
+// tuple_size and tuple_element check.
+template <typename T> class is_tuple_like_ {
+ template <typename U>
+ static auto check(U* p) -> decltype(std::tuple_size<U>::value, int());
+ template <typename> static void check(...);
+
+ public:
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+// Check for integer_sequence
+#if defined(__cpp_lib_integer_sequence) || FMT_MSC_VERSION >= 1900
+template <typename T, T... N>
+using integer_sequence = std::integer_sequence<T, N...>;
+template <size_t... N> using index_sequence = std::index_sequence<N...>;
+template <size_t N> using make_index_sequence = std::make_index_sequence<N>;
+#else
+template <typename T, T... N> struct integer_sequence {
+ using value_type = T;
+
+ static FMT_CONSTEXPR size_t size() { return sizeof...(N); }
+};
+
+template <size_t... N> using index_sequence = integer_sequence<size_t, N...>;
+
+template <typename T, size_t N, T... Ns>
+struct make_integer_sequence : make_integer_sequence<T, N - 1, N - 1, Ns...> {};
+template <typename T, T... Ns>
+struct make_integer_sequence<T, 0, Ns...> : integer_sequence<T, Ns...> {};
+
+template <size_t N>
+using make_index_sequence = make_integer_sequence<size_t, N>;
+#endif
+
+template <typename T>
+using tuple_index_sequence = make_index_sequence<std::tuple_size<T>::value>;
+
+template <typename T, typename C, bool = is_tuple_like_<T>::value>
+class is_tuple_formattable_ {
+ public:
+ static constexpr const bool value = false;
+};
+template <typename T, typename C> class is_tuple_formattable_<T, C, true> {
+ template <std::size_t... Is>
+ static std::true_type check2(index_sequence<Is...>,
+ integer_sequence<bool, (Is == Is)...>);
+ static std::false_type check2(...);
+ template <std::size_t... Is>
+ static decltype(check2(
+ index_sequence<Is...>{},
+ integer_sequence<
+ bool, (is_formattable<typename std::tuple_element<Is, T>::type,
+ C>::value)...>{})) check(index_sequence<Is...>);
+
+ public:
+ static constexpr const bool value =
+ decltype(check(tuple_index_sequence<T>{}))::value;
+};
+
+template <typename Tuple, typename F, size_t... Is>
+FMT_CONSTEXPR void for_each(index_sequence<Is...>, Tuple&& t, F&& f) {
+ using std::get;
+ // Using a free function get<Is>(Tuple) now.
+ const int unused[] = {0, ((void)f(get<Is>(t)), 0)...};
+ ignore_unused(unused);
+}
+
+template <typename Tuple, typename F>
+FMT_CONSTEXPR void for_each(Tuple&& t, F&& f) {
+ for_each(tuple_index_sequence<remove_cvref_t<Tuple>>(),
+ std::forward<Tuple>(t), std::forward<F>(f));
+}
+
+template <typename Tuple1, typename Tuple2, typename F, size_t... Is>
+void for_each2(index_sequence<Is...>, Tuple1&& t1, Tuple2&& t2, F&& f) {
+ using std::get;
+ const int unused[] = {0, ((void)f(get<Is>(t1), get<Is>(t2)), 0)...};
+ ignore_unused(unused);
+}
+
+template <typename Tuple1, typename Tuple2, typename F>
+void for_each2(Tuple1&& t1, Tuple2&& t2, F&& f) {
+ for_each2(tuple_index_sequence<remove_cvref_t<Tuple1>>(),
+ std::forward<Tuple1>(t1), std::forward<Tuple2>(t2),
+ std::forward<F>(f));
+}
+
+namespace tuple {
+// Workaround a bug in MSVC 2019 (v140).
+template <typename Char, typename... T>
+using result_t = std::tuple<formatter<remove_cvref_t<T>, Char>...>;
+
+using std::get;
+template <typename Tuple, typename Char, std::size_t... Is>
+auto get_formatters(index_sequence<Is...>)
+ -> result_t<Char, decltype(get<Is>(std::declval<Tuple>()))...>;
+} // namespace tuple
+
+#if FMT_MSC_VERSION && FMT_MSC_VERSION < 1920
+// Older MSVC doesn't get the reference type correctly for arrays.
+template <typename R> struct range_reference_type_impl {
+ using type = decltype(*detail::range_begin(std::declval<R&>()));
+};
+
+template <typename T, std::size_t N> struct range_reference_type_impl<T[N]> {
+ using type = T&;
+};
+
+template <typename T>
+using range_reference_type = typename range_reference_type_impl<T>::type;
+#else
+template <typename Range>
+using range_reference_type =
+ decltype(*detail::range_begin(std::declval<Range&>()));
+#endif
+
+// We don't use the Range's value_type for anything, but we do need the Range's
+// reference type, with cv-ref stripped.
+template <typename Range>
+using uncvref_type = remove_cvref_t<range_reference_type<Range>>;
+
+template <typename Formatter>
+FMT_CONSTEXPR auto maybe_set_debug_format(Formatter& f, bool set)
+ -> decltype(f.set_debug_format(set)) {
+ f.set_debug_format(set);
+}
+template <typename Formatter>
+FMT_CONSTEXPR void maybe_set_debug_format(Formatter&, ...) {}
+
+// These are not generic lambdas for compatibility with C++11.
+template <typename ParseContext> struct parse_empty_specs {
+ template <typename Formatter> FMT_CONSTEXPR void operator()(Formatter& f) {
+ f.parse(ctx);
+ detail::maybe_set_debug_format(f, true);
+ }
+ ParseContext& ctx;
+};
+template <typename FormatContext> struct format_tuple_element {
+ using char_type = typename FormatContext::char_type;
+
+ template <typename T>
+ void operator()(const formatter<T, char_type>& f, const T& v) {
+ if (i > 0)
+ ctx.advance_to(detail::copy_str<char_type>(separator, ctx.out()));
+ ctx.advance_to(f.format(v, ctx));
+ ++i;
+ }
+
+ int i;
+ FormatContext& ctx;
+ basic_string_view<char_type> separator;
+};
+
+} // namespace detail
+
+template <typename T> struct is_tuple_like {
+ static constexpr const bool value =
+ detail::is_tuple_like_<T>::value && !detail::is_range_<T>::value;
+};
+
+template <typename T, typename C> struct is_tuple_formattable {
+ static constexpr const bool value =
+ detail::is_tuple_formattable_<T, C>::value;
+};
+
+template <typename Tuple, typename Char>
+struct formatter<Tuple, Char,
+ enable_if_t<fmt::is_tuple_like<Tuple>::value &&
+ fmt::is_tuple_formattable<Tuple, Char>::value>> {
+ private:
+ decltype(detail::tuple::get_formatters<Tuple, Char>(
+ detail::tuple_index_sequence<Tuple>())) formatters_;
+
+ basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
+ basic_string_view<Char> opening_bracket_ =
+ detail::string_literal<Char, '('>{};
+ basic_string_view<Char> closing_bracket_ =
+ detail::string_literal<Char, ')'>{};
+
+ public:
+ FMT_CONSTEXPR formatter() {}
+
+ FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
+ separator_ = sep;
+ }
+
+ FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
+ basic_string_view<Char> close) {
+ opening_bracket_ = open;
+ closing_bracket_ = close;
+ }
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ auto it = ctx.begin();
+ if (it != ctx.end() && *it != '}')
+ FMT_THROW(format_error("invalid format specifier"));
+ detail::for_each(formatters_, detail::parse_empty_specs<ParseContext>{ctx});
+ return it;
+ }
+
+ template <typename FormatContext>
+ auto format(const Tuple& value, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ ctx.advance_to(detail::copy_str<Char>(opening_bracket_, ctx.out()));
+ detail::for_each2(
+ formatters_, value,
+ detail::format_tuple_element<FormatContext>{0, ctx, separator_});
+ return detail::copy_str<Char>(closing_bracket_, ctx.out());
+ }
+};
+
+template <typename T, typename Char> struct is_range {
+ static constexpr const bool value =
+ detail::is_range_<T>::value && !detail::is_std_string_like<T>::value &&
+ !std::is_convertible<T, std::basic_string<Char>>::value &&
+ !std::is_convertible<T, detail::std_string_view<Char>>::value;
+};
+
+namespace detail {
+template <typename Context> struct range_mapper {
+ using mapper = arg_mapper<Context>;
+
+ template <typename T,
+ FMT_ENABLE_IF(has_formatter<remove_cvref_t<T>, Context>::value)>
+ static auto map(T&& value) -> T&& {
+ return static_cast<T&&>(value);
+ }
+ template <typename T,
+ FMT_ENABLE_IF(!has_formatter<remove_cvref_t<T>, Context>::value)>
+ static auto map(T&& value)
+ -> decltype(mapper().map(static_cast<T&&>(value))) {
+ return mapper().map(static_cast<T&&>(value));
+ }
+};
+
+template <typename Char, typename Element>
+using range_formatter_type =
+ formatter<remove_cvref_t<decltype(range_mapper<buffer_context<Char>>{}.map(
+ std::declval<Element>()))>,
+ Char>;
+
+template <typename R>
+using maybe_const_range =
+ conditional_t<has_const_begin_end<R>::value, const R, R>;
+
+// Workaround a bug in MSVC 2015 and earlier.
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
+template <typename R, typename Char>
+struct is_formattable_delayed
+ : is_formattable<uncvref_type<maybe_const_range<R>>, Char> {};
+#endif
+} // namespace detail
+
+template <typename T, typename Char, typename Enable = void>
+struct range_formatter;
+
+template <typename T, typename Char>
+struct range_formatter<
+ T, Char,
+ enable_if_t<conjunction<std::is_same<T, remove_cvref_t<T>>,
+ is_formattable<T, Char>>::value>> {
+ private:
+ detail::range_formatter_type<Char, T> underlying_;
+ basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
+ basic_string_view<Char> opening_bracket_ =
+ detail::string_literal<Char, '['>{};
+ basic_string_view<Char> closing_bracket_ =
+ detail::string_literal<Char, ']'>{};
+
+ public:
+ FMT_CONSTEXPR range_formatter() {}
+
+ FMT_CONSTEXPR auto underlying() -> detail::range_formatter_type<Char, T>& {
+ return underlying_;
+ }
+
+ FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
+ separator_ = sep;
+ }
+
+ FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
+ basic_string_view<Char> close) {
+ opening_bracket_ = open;
+ closing_bracket_ = close;
+ }
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ auto it = ctx.begin();
+ auto end = ctx.end();
+
+ if (it != end && *it == 'n') {
+ set_brackets({}, {});
+ ++it;
+ }
+
+ if (it != end && *it != '}') {
+ if (*it != ':') FMT_THROW(format_error("invalid format specifier"));
+ ++it;
+ } else {
+ detail::maybe_set_debug_format(underlying_, true);
+ }
+
+ ctx.advance_to(it);
+ return underlying_.parse(ctx);
+ }
+
+ template <typename R, typename FormatContext>
+ auto format(R&& range, FormatContext& ctx) const -> decltype(ctx.out()) {
+ detail::range_mapper<buffer_context<Char>> mapper;
+ auto out = ctx.out();
+ out = detail::copy_str<Char>(opening_bracket_, out);
+ int i = 0;
+ auto it = detail::range_begin(range);
+ auto end = detail::range_end(range);
+ for (; it != end; ++it) {
+ if (i > 0) out = detail::copy_str<Char>(separator_, out);
+ ctx.advance_to(out);
+ out = underlying_.format(mapper.map(*it), ctx);
+ ++i;
+ }
+ out = detail::copy_str<Char>(closing_bracket_, out);
+ return out;
+ }
+};
+
+enum class range_format { disabled, map, set, sequence, string, debug_string };
+
+namespace detail {
+template <typename T>
+struct range_format_kind_
+ : std::integral_constant<range_format,
+ std::is_same<uncvref_type<T>, T>::value
+ ? range_format::disabled
+ : is_map<T>::value ? range_format::map
+ : is_set<T>::value ? range_format::set
+ : range_format::sequence> {};
+
+template <range_format K, typename R, typename Char, typename Enable = void>
+struct range_default_formatter;
+
+template <range_format K>
+using range_format_constant = std::integral_constant<range_format, K>;
+
+template <range_format K, typename R, typename Char>
+struct range_default_formatter<
+ K, R, Char,
+ enable_if_t<(K == range_format::sequence || K == range_format::map ||
+ K == range_format::set)>> {
+ using range_type = detail::maybe_const_range<R>;
+ range_formatter<detail::uncvref_type<range_type>, Char> underlying_;
+
+ FMT_CONSTEXPR range_default_formatter() { init(range_format_constant<K>()); }
+
+ FMT_CONSTEXPR void init(range_format_constant<range_format::set>) {
+ underlying_.set_brackets(detail::string_literal<Char, '{'>{},
+ detail::string_literal<Char, '}'>{});
+ }
+
+ FMT_CONSTEXPR void init(range_format_constant<range_format::map>) {
+ underlying_.set_brackets(detail::string_literal<Char, '{'>{},
+ detail::string_literal<Char, '}'>{});
+ underlying_.underlying().set_brackets({}, {});
+ underlying_.underlying().set_separator(
+ detail::string_literal<Char, ':', ' '>{});
+ }
+
+ FMT_CONSTEXPR void init(range_format_constant<range_format::sequence>) {}
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return underlying_.parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(range_type& range, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ return underlying_.format(range, ctx);
+ }
+};
+} // namespace detail
+
+template <typename T, typename Char, typename Enable = void>
+struct range_format_kind
+ : conditional_t<
+ is_range<T, Char>::value, detail::range_format_kind_<T>,
+ std::integral_constant<range_format, range_format::disabled>> {};
+
+template <typename R, typename Char>
+struct formatter<
+ R, Char,
+ enable_if_t<conjunction<bool_constant<range_format_kind<R, Char>::value !=
+ range_format::disabled>
+// Workaround a bug in MSVC 2015 and earlier.
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
+ ,
+ detail::is_formattable_delayed<R, Char>
+#endif
+ >::value>>
+ : detail::range_default_formatter<range_format_kind<R, Char>::value, R,
+ Char> {
+};
+
+template <typename Char, typename... T> struct tuple_join_view : detail::view {
+ const std::tuple<T...>& tuple;
+ basic_string_view<Char> sep;
+
+ tuple_join_view(const std::tuple<T...>& t, basic_string_view<Char> s)
+ : tuple(t), sep{s} {}
+};
+
+// Define FMT_TUPLE_JOIN_SPECIFIERS to enable experimental format specifiers
+// support in tuple_join. It is disabled by default because of issues with
+// the dynamic width and precision.
+#ifndef FMT_TUPLE_JOIN_SPECIFIERS
+# define FMT_TUPLE_JOIN_SPECIFIERS 0
+#endif
+
+template <typename Char, typename... T>
+struct formatter<tuple_join_view<Char, T...>, Char> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return do_parse(ctx, std::integral_constant<size_t, sizeof...(T)>());
+ }
+
+ template <typename FormatContext>
+ auto format(const tuple_join_view<Char, T...>& value,
+ FormatContext& ctx) const -> typename FormatContext::iterator {
+ return do_format(value, ctx,
+ std::integral_constant<size_t, sizeof...(T)>());
+ }
+
+ private:
+ std::tuple<formatter<typename std::decay<T>::type, Char>...> formatters_;
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+ std::integral_constant<size_t, 0>)
+ -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename ParseContext, size_t N>
+ FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+ std::integral_constant<size_t, N>)
+ -> decltype(ctx.begin()) {
+ auto end = ctx.begin();
+#if FMT_TUPLE_JOIN_SPECIFIERS
+ end = std::get<sizeof...(T) - N>(formatters_).parse(ctx);
+ if (N > 1) {
+ auto end1 = do_parse(ctx, std::integral_constant<size_t, N - 1>());
+ if (end != end1)
+ FMT_THROW(format_error("incompatible format specs for tuple elements"));
+ }
+#endif
+ return end;
+ }
+
+ template <typename FormatContext>
+ auto do_format(const tuple_join_view<Char, T...>&, FormatContext& ctx,
+ std::integral_constant<size_t, 0>) const ->
+ typename FormatContext::iterator {
+ return ctx.out();
+ }
+
+ template <typename FormatContext, size_t N>
+ auto do_format(const tuple_join_view<Char, T...>& value, FormatContext& ctx,
+ std::integral_constant<size_t, N>) const ->
+ typename FormatContext::iterator {
+ auto out = std::get<sizeof...(T) - N>(formatters_)
+ .format(std::get<sizeof...(T) - N>(value.tuple), ctx);
+ if (N > 1) {
+ out = std::copy(value.sep.begin(), value.sep.end(), out);
+ ctx.advance_to(out);
+ return do_format(value, ctx, std::integral_constant<size_t, N - 1>());
+ }
+ return out;
+ }
+};
+
+namespace detail {
+// Check if T has an interface like a container adaptor (e.g. std::stack,
+// std::queue, std::priority_queue).
+template <typename T> class is_container_adaptor_like {
+ template <typename U> static auto check(U* p) -> typename U::container_type;
+ template <typename> static void check(...);
+
+ public:
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+template <typename Container> struct all {
+ const Container& c;
+ auto begin() const -> typename Container::const_iterator { return c.begin(); }
+ auto end() const -> typename Container::const_iterator { return c.end(); }
+};
+} // namespace detail
+
+template <typename T, typename Char>
+struct formatter<T, Char,
+ enable_if_t<detail::is_container_adaptor_like<T>::value>>
+ : formatter<detail::all<typename T::container_type>, Char> {
+ using all = detail::all<typename T::container_type>;
+ template <typename FormatContext>
+ auto format(const T& t, FormatContext& ctx) const -> decltype(ctx.out()) {
+ struct getter : T {
+ static auto get(const T& t) -> all {
+ return {t.*(&getter::c)}; // Access c through the derived class.
+ }
+ };
+ return formatter<all>::format(getter::get(t), ctx);
+ }
+};
+
+FMT_BEGIN_EXPORT
+
+/**
+ \rst
+ Returns an object that formats `tuple` with elements separated by `sep`.
+
+ **Example**::
+
+ std::tuple<int, char> t = {1, 'a'};
+ fmt::print("{}", fmt::join(t, ", "));
+ // Output: "1, a"
+ \endrst
+ */
+template <typename... T>
+FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple, string_view sep)
+ -> tuple_join_view<char, T...> {
+ return {tuple, sep};
+}
+
+template <typename... T>
+FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple,
+ basic_string_view<wchar_t> sep)
+ -> tuple_join_view<wchar_t, T...> {
+ return {tuple, sep};
+}
+
+/**
+ \rst
+ Returns an object that formats `initializer_list` with elements separated by
+ `sep`.
+
+ **Example**::
+
+ fmt::print("{}", fmt::join({1, 2, 3}, ", "));
+ // Output: "1, 2, 3"
+ \endrst
+ */
+template <typename T>
+auto join(std::initializer_list<T> list, string_view sep)
+ -> join_view<const T*, const T*> {
+ return join(std::begin(list), std::end(list), sep);
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_RANGES_H_
diff --git a/contrib/fmt/include/fmt/std.h b/contrib/fmt/include/fmt/std.h
new file mode 100644
index 0000000..4c2a28c
--- /dev/null
+++ b/contrib/fmt/include/fmt/std.h
@@ -0,0 +1,349 @@
+// Formatting library for C++ - formatters for standard library types
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_STD_H_
+#define FMT_STD_H_
+
+#include <cstdlib>
+#include <exception>
+#include <memory>
+#include <thread>
+#include <type_traits>
+#include <typeinfo>
+#include <utility>
+
+#include "ostream.h"
+
+#if FMT_HAS_INCLUDE(<version>)
+# include <version>
+#endif
+// Checking FMT_CPLUSPLUS for warning suppression in MSVC.
+#if FMT_CPLUSPLUS >= 201703L
+# if FMT_HAS_INCLUDE(<filesystem>)
+# include <filesystem>
+# endif
+# if FMT_HAS_INCLUDE(<variant>)
+# include <variant>
+# endif
+# if FMT_HAS_INCLUDE(<optional>)
+# include <optional>
+# endif
+#endif
+
+// GCC 4 does not support FMT_HAS_INCLUDE.
+#if FMT_HAS_INCLUDE(<cxxabi.h>) || defined(__GLIBCXX__)
+# include <cxxabi.h>
+// Android NDK with gabi++ library on some architectures does not implement
+// abi::__cxa_demangle().
+# ifndef __GABIXX_CXXABI_H__
+# define FMT_HAS_ABI_CXA_DEMANGLE
+# endif
+#endif
+
+#ifdef __cpp_lib_filesystem
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename Char>
+void write_escaped_path(basic_memory_buffer<Char>& quoted,
+ const std::filesystem::path& p) {
+ write_escaped_string<Char>(std::back_inserter(quoted), p.string<Char>());
+}
+# ifdef _WIN32
+template <>
+inline void write_escaped_path<char>(memory_buffer& quoted,
+ const std::filesystem::path& p) {
+ auto buf = basic_memory_buffer<wchar_t>();
+ write_escaped_string<wchar_t>(std::back_inserter(buf), p.native());
+ // Convert UTF-16 to UTF-8.
+ if (!unicode_to_utf8<wchar_t>::convert(quoted, {buf.data(), buf.size()}))
+ FMT_THROW(std::runtime_error("invalid utf16"));
+}
+# endif
+template <>
+inline void write_escaped_path<std::filesystem::path::value_type>(
+ basic_memory_buffer<std::filesystem::path::value_type>& quoted,
+ const std::filesystem::path& p) {
+ write_escaped_string<std::filesystem::path::value_type>(
+ std::back_inserter(quoted), p.native());
+}
+
+} // namespace detail
+
+FMT_MODULE_EXPORT
+template <typename Char>
+struct formatter<std::filesystem::path, Char>
+ : formatter<basic_string_view<Char>> {
+ template <typename ParseContext> FMT_CONSTEXPR auto parse(ParseContext& ctx) {
+ auto out = formatter<basic_string_view<Char>>::parse(ctx);
+ this->set_debug_format(false);
+ return out;
+ }
+ template <typename FormatContext>
+ auto format(const std::filesystem::path& p, FormatContext& ctx) const ->
+ typename FormatContext::iterator {
+ auto quoted = basic_memory_buffer<Char>();
+ detail::write_escaped_path(quoted, p);
+ return formatter<basic_string_view<Char>>::format(
+ basic_string_view<Char>(quoted.data(), quoted.size()), ctx);
+ }
+};
+FMT_END_NAMESPACE
+#endif
+
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename Char>
+struct formatter<std::thread::id, Char> : basic_ostream_formatter<Char> {};
+FMT_END_NAMESPACE
+
+#ifdef __cpp_lib_optional
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename T, typename Char>
+struct formatter<std::optional<T>, Char,
+ std::enable_if_t<is_formattable<T, Char>::value>> {
+ private:
+ formatter<T, Char> underlying_;
+ static constexpr basic_string_view<Char> optional =
+ detail::string_literal<Char, 'o', 'p', 't', 'i', 'o', 'n', 'a', 'l',
+ '('>{};
+ static constexpr basic_string_view<Char> none =
+ detail::string_literal<Char, 'n', 'o', 'n', 'e'>{};
+
+ template <class U>
+ FMT_CONSTEXPR static auto maybe_set_debug_format(U& u, bool set)
+ -> decltype(u.set_debug_format(set)) {
+ u.set_debug_format(set);
+ }
+
+ template <class U>
+ FMT_CONSTEXPR static void maybe_set_debug_format(U&, ...) {}
+
+ public:
+ template <typename ParseContext> FMT_CONSTEXPR auto parse(ParseContext& ctx) {
+ maybe_set_debug_format(underlying_, true);
+ return underlying_.parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(std::optional<T> const& opt, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ if (!opt) return detail::write<Char>(ctx.out(), none);
+
+ auto out = ctx.out();
+ out = detail::write<Char>(out, optional);
+ ctx.advance_to(out);
+ out = underlying_.format(*opt, ctx);
+ return detail::write(out, ')');
+ }
+};
+FMT_END_NAMESPACE
+#endif // __cpp_lib_optional
+
+#ifdef __cpp_lib_variant
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename Char> struct formatter<std::monostate, Char> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename FormatContext>
+ auto format(const std::monostate&, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto out = ctx.out();
+ out = detail::write<Char>(out, "monostate");
+ return out;
+ }
+};
+
+namespace detail {
+
+template <typename T>
+using variant_index_sequence =
+ std::make_index_sequence<std::variant_size<T>::value>;
+
+template <typename> struct is_variant_like_ : std::false_type {};
+template <typename... Types>
+struct is_variant_like_<std::variant<Types...>> : std::true_type {};
+
+// formattable element check.
+template <typename T, typename C> class is_variant_formattable_ {
+ template <std::size_t... Is>
+ static std::conjunction<
+ is_formattable<std::variant_alternative_t<Is, T>, C>...>
+ check(std::index_sequence<Is...>);
+
+ public:
+ static constexpr const bool value =
+ decltype(check(variant_index_sequence<T>{}))::value;
+};
+
+template <typename Char, typename OutputIt, typename T>
+auto write_variant_alternative(OutputIt out, const T& v) -> OutputIt {
+ if constexpr (is_string<T>::value)
+ return write_escaped_string<Char>(out, detail::to_string_view(v));
+ else if constexpr (std::is_same_v<T, Char>)
+ return write_escaped_char(out, v);
+ else
+ return write<Char>(out, v);
+}
+
+} // namespace detail
+template <typename T> struct is_variant_like {
+ static constexpr const bool value = detail::is_variant_like_<T>::value;
+};
+
+template <typename T, typename C> struct is_variant_formattable {
+ static constexpr const bool value =
+ detail::is_variant_formattable_<T, C>::value;
+};
+
+FMT_MODULE_EXPORT
+template <typename Variant, typename Char>
+struct formatter<
+ Variant, Char,
+ std::enable_if_t<std::conjunction_v<
+ is_variant_like<Variant>, is_variant_formattable<Variant, Char>>>> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename FormatContext>
+ auto format(const Variant& value, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto out = ctx.out();
+
+ out = detail::write<Char>(out, "variant(");
+ try {
+ std::visit(
+ [&](const auto& v) {
+ out = detail::write_variant_alternative<Char>(out, v);
+ },
+ value);
+ } catch (const std::bad_variant_access&) {
+ detail::write<Char>(out, "valueless by exception");
+ }
+ *out++ = ')';
+ return out;
+ }
+};
+FMT_END_NAMESPACE
+#endif // __cpp_lib_variant
+
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename Char> struct formatter<std::error_code, Char> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename FormatContext>
+ FMT_CONSTEXPR auto format(const std::error_code& ec, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto out = ctx.out();
+ out = detail::write_bytes(out, ec.category().name(), format_specs<Char>());
+ out = detail::write<Char>(out, Char(':'));
+ out = detail::write<Char>(out, ec.value());
+ return out;
+ }
+};
+
+FMT_MODULE_EXPORT
+template <typename T, typename Char>
+struct formatter<
+ T, Char,
+ typename std::enable_if<std::is_base_of<std::exception, T>::value>::type> {
+ private:
+ bool with_typename_ = false;
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto it = ctx.begin();
+ auto end = ctx.end();
+ if (it == end || *it == '}') return it;
+ if (*it == 't') {
+ ++it;
+ with_typename_ = true;
+ }
+ return it;
+ }
+
+ template <typename OutputIt>
+ auto format(const std::exception& ex,
+ basic_format_context<OutputIt, Char>& ctx) const -> OutputIt {
+ format_specs<Char> spec;
+ auto out = ctx.out();
+ if (!with_typename_)
+ return detail::write_bytes(out, string_view(ex.what()), spec);
+
+ const std::type_info& ti = typeid(ex);
+#ifdef FMT_HAS_ABI_CXA_DEMANGLE
+ int status = 0;
+ std::size_t size = 0;
+ std::unique_ptr<char, decltype(&std::free)> demangled_name_ptr(
+ abi::__cxa_demangle(ti.name(), nullptr, &size, &status), &std::free);
+
+ string_view demangled_name_view;
+ if (demangled_name_ptr) {
+ demangled_name_view = demangled_name_ptr.get();
+
+ // Normalization of stdlib inline namespace names.
+ // libc++ inline namespaces.
+ // std::__1::* -> std::*
+ // std::__1::__fs::* -> std::*
+ // libstdc++ inline namespaces.
+ // std::__cxx11::* -> std::*
+ // std::filesystem::__cxx11::* -> std::filesystem::*
+ if (demangled_name_view.starts_with("std::")) {
+ char* begin = demangled_name_ptr.get();
+ char* to = begin + 5; // std::
+ for (char *from = to, *end = begin + demangled_name_view.size();
+ from < end;) {
+ // This is safe, because demangled_name is NUL-terminated.
+ if (from[0] == '_' && from[1] == '_') {
+ char* next = from + 1;
+ while (next < end && *next != ':') next++;
+ if (next[0] == ':' && next[1] == ':') {
+ from = next + 2;
+ continue;
+ }
+ }
+ *to++ = *from++;
+ }
+ demangled_name_view = {begin, detail::to_unsigned(to - begin)};
+ }
+ } else {
+ demangled_name_view = string_view(ti.name());
+ }
+ out = detail::write_bytes(out, demangled_name_view, spec);
+#elif FMT_MSC_VERSION
+ string_view demangled_name_view(ti.name());
+ if (demangled_name_view.starts_with("class "))
+ demangled_name_view.remove_prefix(6);
+ else if (demangled_name_view.starts_with("struct "))
+ demangled_name_view.remove_prefix(7);
+ out = detail::write_bytes(out, demangled_name_view, spec);
+#else
+ out = detail::write_bytes(out, string_view(ti.name()), spec);
+#endif
+ out = detail::write<Char>(out, Char(':'));
+ out = detail::write<Char>(out, Char(' '));
+ out = detail::write_bytes(out, string_view(ex.what()), spec);
+
+ return out;
+ }
+};
+FMT_END_NAMESPACE
+
+#endif // FMT_STD_H_
diff --git a/contrib/fmt/include/fmt/xchar.h b/contrib/fmt/include/fmt/xchar.h
new file mode 100644
index 0000000..4b87f8d
--- /dev/null
+++ b/contrib/fmt/include/fmt/xchar.h
@@ -0,0 +1,259 @@
+// Formatting library for C++ - optional wchar_t and exotic character support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_XCHAR_H_
+#define FMT_XCHAR_H_
+
+#include <cwchar>
+
+#include "format.h"
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+# include <locale>
+#endif
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template <typename T>
+using is_exotic_char = bool_constant<!std::is_same<T, char>::value>;
+
+inline auto write_loc(std::back_insert_iterator<detail::buffer<wchar_t>> out,
+ loc_value value, const format_specs<wchar_t>& specs,
+ locale_ref loc) -> bool {
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+ auto& numpunct =
+ std::use_facet<std::numpunct<wchar_t>>(loc.get<std::locale>());
+ auto separator = std::wstring();
+ auto grouping = numpunct.grouping();
+ if (!grouping.empty()) separator = std::wstring(1, numpunct.thousands_sep());
+ return value.visit(loc_writer<wchar_t>{out, specs, separator, grouping, {}});
+#endif
+ return false;
+}
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+using wstring_view = basic_string_view<wchar_t>;
+using wformat_parse_context = basic_format_parse_context<wchar_t>;
+using wformat_context = buffer_context<wchar_t>;
+using wformat_args = basic_format_args<wformat_context>;
+using wmemory_buffer = basic_memory_buffer<wchar_t>;
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+// Workaround broken conversion on older gcc.
+template <typename... Args> using wformat_string = wstring_view;
+inline auto runtime(wstring_view s) -> wstring_view { return s; }
+#else
+template <typename... Args>
+using wformat_string = basic_format_string<wchar_t, type_identity_t<Args>...>;
+inline auto runtime(wstring_view s) -> runtime_format_string<wchar_t> {
+ return {{s}};
+}
+#endif
+
+template <> struct is_char<wchar_t> : std::true_type {};
+template <> struct is_char<detail::char8_type> : std::true_type {};
+template <> struct is_char<char16_t> : std::true_type {};
+template <> struct is_char<char32_t> : std::true_type {};
+
+template <typename... Args>
+constexpr format_arg_store<wformat_context, Args...> make_wformat_args(
+ const Args&... args) {
+ return {args...};
+}
+
+inline namespace literals {
+#if FMT_USE_USER_DEFINED_LITERALS && !FMT_USE_NONTYPE_TEMPLATE_ARGS
+constexpr detail::udl_arg<wchar_t> operator"" _a(const wchar_t* s, size_t) {
+ return {s};
+}
+#endif
+} // namespace literals
+
+template <typename It, typename Sentinel>
+auto join(It begin, Sentinel end, wstring_view sep)
+ -> join_view<It, Sentinel, wchar_t> {
+ return {begin, end, sep};
+}
+
+template <typename Range>
+auto join(Range&& range, wstring_view sep)
+ -> join_view<detail::iterator_t<Range>, detail::sentinel_t<Range>,
+ wchar_t> {
+ return join(std::begin(range), std::end(range), sep);
+}
+
+template <typename T>
+auto join(std::initializer_list<T> list, wstring_view sep)
+ -> join_view<const T*, const T*, wchar_t> {
+ return join(std::begin(list), std::end(list), sep);
+}
+
+template <typename Char, FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto vformat(basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ basic_memory_buffer<Char> buffer;
+ detail::vformat_to(buffer, format_str, args);
+ return to_string(buffer);
+}
+
+template <typename... T>
+auto format(wformat_string<T...> fmt, T&&... args) -> std::wstring {
+ return vformat(fmt::wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+// Pass char_t as a default template parameter instead of using
+// std::basic_string<char_t<S>> to reduce the symbol size.
+template <typename S, typename... Args, typename Char = char_t<S>,
+ FMT_ENABLE_IF(!std::is_same<Char, char>::value &&
+ !std::is_same<Char, wchar_t>::value)>
+auto format(const S& format_str, Args&&... args) -> std::basic_string<Char> {
+ return vformat(detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename Locale, typename S, typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto vformat(
+ const Locale& loc, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ return detail::vformat(loc, detail::to_string_view(format_str), args);
+}
+
+template <typename Locale, typename S, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto format(const Locale& loc, const S& format_str, Args&&... args)
+ -> std::basic_string<Char> {
+ return detail::vformat(loc, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename OutputIt, typename S, typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+auto vformat_to(OutputIt out, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> OutputIt {
+ auto&& buf = detail::get_buffer<Char>(out);
+ detail::vformat_to(buf, detail::to_string_view(format_str), args);
+ return detail::get_iterator(buf, out);
+}
+
+template <typename OutputIt, typename S, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto format_to(OutputIt out, const S& fmt, Args&&... args) -> OutputIt {
+ return vformat_to(out, detail::to_string_view(fmt),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename Locale, typename S, typename OutputIt, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_locale<Locale>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto vformat_to(
+ OutputIt out, const Locale& loc, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) -> OutputIt {
+ auto&& buf = detail::get_buffer<Char>(out);
+ vformat_to(buf, detail::to_string_view(format_str), args,
+ detail::locale_ref(loc));
+ return detail::get_iterator(buf, out);
+}
+
+template <
+ typename OutputIt, typename Locale, typename S, typename... Args,
+ typename Char = char_t<S>,
+ bool enable = detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_locale<Locale>::value&& detail::is_exotic_char<Char>::value>
+inline auto format_to(OutputIt out, const Locale& loc, const S& format_str,
+ Args&&... args) ->
+ typename std::enable_if<enable, OutputIt>::type {
+ return vformat_to(out, loc, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename OutputIt, typename Char, typename... Args,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto vformat_to_n(
+ OutputIt out, size_t n, basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> format_to_n_result<OutputIt> {
+ detail::iterator_buffer<OutputIt, Char, detail::fixed_buffer_traits> buf(out,
+ n);
+ detail::vformat_to(buf, format_str, args);
+ return {buf.out(), buf.count()};
+}
+
+template <typename OutputIt, typename S, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto format_to_n(OutputIt out, size_t n, const S& fmt,
+ const Args&... args) -> format_to_n_result<OutputIt> {
+ return vformat_to_n(out, n, detail::to_string_view(fmt),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename S, typename... Args, typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_exotic_char<Char>::value)>
+inline auto formatted_size(const S& fmt, Args&&... args) -> size_t {
+ detail::counting_buffer<Char> buf;
+ detail::vformat_to(buf, detail::to_string_view(fmt),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+ return buf.count();
+}
+
+inline void vprint(std::FILE* f, wstring_view fmt, wformat_args args) {
+ wmemory_buffer buffer;
+ detail::vformat_to(buffer, fmt, args);
+ buffer.push_back(L'\0');
+ if (std::fputws(buffer.data(), f) == -1)
+ FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+inline void vprint(wstring_view fmt, wformat_args args) {
+ vprint(stdout, fmt, args);
+}
+
+template <typename... T>
+void print(std::FILE* f, wformat_string<T...> fmt, T&&... args) {
+ return vprint(f, wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+template <typename... T> void print(wformat_string<T...> fmt, T&&... args) {
+ return vprint(wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+template <typename... T>
+void println(std::FILE* f, wformat_string<T...> fmt, T&&... args) {
+ return print(f, L"{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+template <typename... T> void println(wformat_string<T...> fmt, T&&... args) {
+ return print(L"{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+/**
+ Converts *value* to ``std::wstring`` using the default format for type *T*.
+ */
+template <typename T> inline auto to_wstring(const T& value) -> std::wstring {
+ return format(FMT_STRING(L"{}"), value);
+}
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_XCHAR_H_
diff --git a/contrib/fpconv/CMakeLists.txt b/contrib/fpconv/CMakeLists.txt
new file mode 100644
index 0000000..b330525
--- /dev/null
+++ b/contrib/fpconv/CMakeLists.txt
@@ -0,0 +1,7 @@
+SET(FPCONVSRC fpconv.c)
+
+SET(FTPCONV_COMPILE_FLAGS "-DRSPAMD_LIB")
+
+ADD_LIBRARY(rspamd-fpconv STATIC ${FPCONVSRC})
+SET_TARGET_PROPERTIES(rspamd-fpconv PROPERTIES VERSION ${RSPAMD_VERSION})
+SET_TARGET_PROPERTIES(rspamd-fpconv PROPERTIES COMPILE_FLAGS "${FTPCONV_COMPILE_FLAGS}") \ No newline at end of file
diff --git a/contrib/fpconv/LICENSE b/contrib/fpconv/LICENSE
new file mode 100644
index 0000000..36b7cd9
--- /dev/null
+++ b/contrib/fpconv/LICENSE
@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/contrib/fpconv/fpconv.c b/contrib/fpconv/fpconv.c
new file mode 100644
index 0000000..f8b601a
--- /dev/null
+++ b/contrib/fpconv/fpconv.c
@@ -0,0 +1,480 @@
+#include <stdbool.h>
+#include <string.h>
+#include <sys/param.h>
+
+#include "fpconv.h"
+#include "powers.h"
+
+#define fracmask 0x000FFFFFFFFFFFFFU
+#define expmask 0x7FF0000000000000U
+#define hiddenbit 0x0010000000000000U
+#define signmask 0x8000000000000000U
+#define expbias (1023 + 52)
+
+#define absv(n) ((n) < 0 ? -(n) : (n))
+#define minv(a, b) ((a) < (b) ? (a) : (b))
+
+static uint64_t tens[] = {
+ 10000000000000000000U, 1000000000000000000U, 100000000000000000U,
+ 10000000000000000U, 1000000000000000U, 100000000000000U,
+ 10000000000000U, 1000000000000U, 100000000000U,
+ 10000000000U, 1000000000U, 100000000U,
+ 10000000U, 1000000U, 100000U,
+ 10000U, 1000U, 100U,
+ 10U, 1U
+};
+
+static inline uint64_t get_dbits (double d) {
+ union {
+ double dbl;
+ uint64_t i;
+ } dbl_bits = {d};
+
+ return dbl_bits.i;
+}
+
+static Fp build_fp (double d) {
+ uint64_t bits = get_dbits (d);
+
+ Fp fp;
+ fp.frac = bits & fracmask;
+ fp.exp = (bits & expmask) >> 52u;
+
+ if (fp.exp) {
+ fp.frac += hiddenbit;
+ fp.exp -= expbias;
+
+ }
+ else {
+ fp.exp = -expbias + 1;
+ }
+
+ return fp;
+}
+
+static void normalize (Fp *fp) {
+ while ((fp->frac & hiddenbit) == 0) {
+ fp->frac <<= 1u;
+ fp->exp--;
+ }
+
+ const unsigned int shift = 64 - 52 - 1;
+ fp->frac <<= shift;
+ fp->exp -= shift;
+}
+
+static void get_normalized_boundaries (Fp *fp, Fp *lower, Fp *upper) {
+ upper->frac = (fp->frac << 1u) + 1u;
+ upper->exp = fp->exp - 1u;
+
+ while ((upper->frac & (hiddenbit << 1u)) == 0) {
+ upper->frac <<= 1u;
+ upper->exp--;
+ }
+
+ const unsigned int u_shift = 64 - 52 - 2;
+
+ upper->frac <<= u_shift;
+ upper->exp = upper->exp - u_shift;
+
+
+ unsigned int l_shift = fp->frac == hiddenbit ? 2u : 1u;
+
+ lower->frac = (fp->frac << l_shift) - 1;
+ lower->exp = fp->exp - l_shift;
+
+
+ lower->frac <<= lower->exp - upper->exp;
+ lower->exp = upper->exp;
+}
+
+static Fp multiply (Fp *a, Fp *b) {
+ const uint64_t lomask = 0x00000000FFFFFFFFu;
+
+ uint64_t ah_bl = (a->frac >> 32u) * (b->frac & lomask);
+ uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32u);
+ uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask);
+ uint64_t ah_bh = (a->frac >> 32u) * (b->frac >> 32u);
+
+ uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32u);
+ /* round up */
+ tmp += 1U << 31u;
+
+ Fp fp = {
+ ah_bh + (ah_bl >> 32u) + (al_bh >> 32u) + (tmp >> 32u),
+ a->exp + b->exp + 64u
+ };
+
+ return fp;
+}
+
+static void round_digit (char *digits, int ndigits, uint64_t delta, uint64_t rem, uint64_t kappa, uint64_t frac) {
+ while (rem < frac && delta - rem >= kappa &&
+ (rem + kappa < frac || frac - rem > rem + kappa - frac)) {
+
+ digits[ndigits - 1]--;
+ rem += kappa;
+ }
+}
+
+static int generate_digits (Fp *fp, Fp *upper, Fp *lower, char *digits, int *K) {
+ uint64_t wfrac = upper->frac - fp->frac;
+ uint64_t delta = upper->frac - lower->frac;
+
+ Fp one;
+ one.frac = 1ULL << -upper->exp;
+ one.exp = upper->exp;
+
+ uint64_t part1 = upper->frac >> -one.exp;
+ uint64_t part2 = upper->frac & (one.frac - 1);
+
+ int idx = 0, kappa = 10;
+ uint64_t *divp;
+ /* 1000000000 */
+ for (divp = tens + 10; kappa > 0; divp++) {
+
+ uint64_t div = *divp;
+ unsigned digit = part1 / div;
+
+ if (digit || idx) {
+ digits[idx++] = digit + '0';
+ }
+
+ part1 -= digit * div;
+ kappa--;
+
+ uint64_t tmp = (part1 << -one.exp) + part2;
+ if (tmp <= delta) {
+ *K += kappa;
+ round_digit (digits, idx, delta, tmp, div << -one.exp, wfrac);
+
+ return idx;
+ }
+ }
+
+ /* 10 */
+ uint64_t *unit = tens + 18;
+
+ while (true) {
+ part2 *= 10;
+ delta *= 10;
+ kappa--;
+
+ unsigned digit = part2 >> -one.exp;
+ if (digit || idx) {
+ digits[idx++] = digit + '0';
+ }
+
+ part2 &= one.frac - 1;
+ if (part2 < delta) {
+ *K += kappa;
+ round_digit (digits, idx, delta, part2, one.frac, wfrac * *unit);
+
+ return idx;
+ }
+
+ unit--;
+ }
+}
+
+static int grisu2 (double d, char *digits, int *K) {
+ Fp w = build_fp (d);
+
+ Fp lower, upper;
+ get_normalized_boundaries (&w, &lower, &upper);
+
+ normalize (&w);
+
+ int k;
+ Fp cp = find_cachedpow10 (upper.exp, &k);
+
+ w = multiply (&w, &cp);
+ upper = multiply (&upper, &cp);
+ lower = multiply (&lower, &cp);
+
+ lower.frac++;
+ upper.frac--;
+
+ *K = -k;
+
+ return generate_digits (&w, &upper, &lower, digits, K);
+}
+
+static inline int emit_integer (char *digits, int ndigits,
+ char *dest, int K, bool neg,
+ unsigned precision)
+{
+ char *d = dest;
+
+ memcpy (d, digits, ndigits);
+ d += ndigits;
+ memset (d, '0', K);
+ d += K;
+
+ precision = MIN(precision, FPCONV_BUFLEN - (ndigits + K + 1));
+
+ if (precision) {
+ *d++ = '.';
+ memset (d, '0', precision);
+ d += precision;
+ }
+
+ return d - dest;
+}
+
+static inline int emit_scientific_digits (char *digits, int ndigits,
+ char *dest, int K, bool neg,
+ unsigned precision, int exp)
+{
+ /* write decimal w/ scientific notation */
+ ndigits = minv(ndigits, 18 - neg);
+
+ int idx = 0;
+ dest[idx++] = digits[0];
+
+ if (ndigits > 1) {
+ dest[idx++] = '.';
+ memcpy(dest + idx, digits + 1, ndigits - 1);
+ idx += ndigits - 1;
+ }
+
+ dest[idx++] = 'e';
+
+ char sign = K + ndigits - 1 < 0 ? '-' : '+';
+ dest[idx++] = sign;
+
+ int cent = 0;
+
+ if (exp > 99) {
+ cent = exp / 100;
+ dest[idx++] = cent + '0';
+ exp -= cent * 100;
+ }
+ if (exp > 9) {
+ int dec = exp / 10;
+ dest[idx++] = dec + '0';
+ exp -= dec * 10;
+
+ }
+ else if (cent) {
+ dest[idx++] = '0';
+ }
+
+ dest[idx++] = exp % 10 + '0';
+
+ return idx;
+}
+
+static inline int emit_fixed_digits (char *digits, int ndigits,
+ char *dest, int K, bool neg,
+ unsigned precision, int exp)
+{
+ int offset = ndigits - absv(K), to_print;
+ /* fp < 1.0 -> write leading zero */
+ if (K < 0) {
+ if (offset <= 0) {
+ if (precision) {
+ if (-offset >= precision) {
+ /* Just print 0.[0]{precision} */
+ dest[0] = '0';
+ dest[1] = '.';
+ memset(dest + 2, '0', precision);
+
+ return precision + 2;
+ }
+
+ to_print = MAX(ndigits - offset, precision);
+ }
+ else {
+ to_print = ndigits - offset;
+ }
+
+ if (to_print <= FPCONV_BUFLEN - 3) {
+ offset = -offset;
+ dest[0] = '0';
+ dest[1] = '.';
+ memset(dest + 2, '0', offset);
+
+ if (precision) {
+ /* The case where offset > precision is covered previously */
+ precision -= offset;
+
+ if (precision <= ndigits) {
+ /* Truncate or leave as is */
+ memcpy(dest + offset + 2, digits, precision);
+
+ return precision + 2 + offset;
+ }
+ else {
+ /* Expand */
+ memcpy(dest + offset + 2, digits, ndigits);
+ precision -= ndigits;
+ memset(dest + offset + 2 + ndigits, '0', precision);
+
+ return ndigits + 2 + offset + precision;
+ }
+ }
+ else {
+ memcpy(dest + offset + 2, digits, ndigits);
+ }
+
+ return ndigits + 2 + offset;
+ }
+ else {
+ return emit_scientific_digits (digits, ndigits, dest, K, neg, precision, exp);
+ }
+ }
+ else {
+ /*
+ * fp > 1.0, if offset > 0 then we have less digits than
+ * fp exponent, so we need to switch to scientific notation to
+ * display number at least more or less precisely
+ */
+ if (offset > 0 && ndigits <= FPCONV_BUFLEN - 3) {
+ char *d = dest;
+ memcpy(d, digits, offset);
+ d += offset;
+ *d++ = '.';
+
+ ndigits -= offset;
+
+ if (precision) {
+ if (ndigits >= precision) {
+ /* Truncate or leave as is */
+ memcpy(d, digits + offset, precision);
+ d += precision;
+ }
+ else {
+ /* Expand */
+ memcpy(d, digits + offset, ndigits);
+ precision -= ndigits;
+ d += ndigits;
+
+ /* Check if we have enough bufspace */
+ if ((d - dest) + precision <= FPCONV_BUFLEN) {
+ memset (d, '0', precision);
+ d += precision;
+ }
+ else {
+ memset (d, '0', FPCONV_BUFLEN - (d - dest));
+ d += FPCONV_BUFLEN - (d - dest);
+ }
+ }
+ }
+ else {
+ memcpy(d, digits + offset, ndigits);
+ d += ndigits;
+ }
+
+ return d - dest;
+ }
+ }
+ }
+
+ return emit_scientific_digits (digits, ndigits, dest, K, neg, precision, exp);
+}
+
+static int emit_digits (char *digits, int ndigits, char *dest, int K, bool neg,
+ unsigned precision, bool scientific)
+{
+ int exp = absv(K + ndigits - 1);
+
+ /* write plain integer */
+ if (K >= 0 && (exp < (ndigits + 7))) {
+ return emit_integer (digits, ndigits, dest, K, neg, precision);
+ }
+
+ /* write decimal w/o scientific notation */
+ if (!scientific || (K < 0 && (K > -7 || exp < 4))) {
+ return emit_fixed_digits (digits, ndigits, dest, K, neg, precision, exp);
+ }
+
+ return emit_scientific_digits (digits, ndigits, dest, K, neg, precision, exp);
+}
+
+static int filter_special (double fp, char *dest, unsigned precision)
+{
+ int nchars = 3;
+ char *d = dest;
+
+ if (fp == 0.0) {
+ if (get_dbits (fp) & signmask) {
+ *d++ = '-';
+ *d++ = '0';
+ }
+ else {
+ *d++ = '0';
+ }
+
+ if (precision) {
+ *d ++ = '.';
+ memset (d, '0', precision);
+ }
+
+ return d - dest + precision;
+ }
+
+ uint64_t bits = get_dbits (fp);
+
+ bool nan = (bits & expmask) == expmask;
+
+ if (!nan) {
+ return 0;
+ }
+
+ if (bits & fracmask) {
+ dest[0] = 'n';
+ dest[1] = 'a';
+ dest[2] = 'n';
+ }
+ else {
+ if (get_dbits (fp) & signmask) {
+ dest[0] = '-';
+ dest[1] = 'i';
+ dest[2] = 'n';
+ dest[3] = 'f';
+ nchars = 4;
+ }
+ else {
+ dest[0] = 'i';
+ dest[1] = 'n';
+ dest[2] = 'f';
+ }
+ }
+
+ return nchars;
+}
+
+int
+fpconv_dtoa (double d, char dest[FPCONV_BUFLEN],
+ unsigned precision, bool scientific)
+{
+ char digits[18];
+
+ int str_len = 0;
+ bool neg = false;
+
+ if (precision > FPCONV_BUFLEN - 5) {
+ precision = FPCONV_BUFLEN - 5;
+ }
+
+ int spec = filter_special (d, dest, precision);
+
+ if (spec) {
+ return spec;
+ }
+
+ if (get_dbits (d) & signmask) {
+ dest[0] = '-';
+ str_len++;
+ neg = true;
+ }
+
+ int K = 0;
+ int ndigits = grisu2 (d, digits, &K);
+
+ str_len += emit_digits (digits, ndigits, dest + str_len, K, neg, precision,
+ scientific);
+
+ return str_len;
+}
diff --git a/contrib/fpconv/fpconv.h b/contrib/fpconv/fpconv.h
new file mode 100644
index 0000000..8c07c13
--- /dev/null
+++ b/contrib/fpconv/fpconv.h
@@ -0,0 +1,35 @@
+#ifndef FPCONV_H
+#define FPCONV_H
+
+#define FPCONV_BUFLEN 32
+/* Fast and accurate double to string conversion based on Florian Loitsch's
+ * Grisu-algorithm[1].
+ *
+ * Input:
+ * fp -> the double to convert, dest -> destination buffer.
+ * The generated string will never be longer than 24 characters.
+ * Make sure to pass a pointer to at least 24 bytes of memory.
+ * The emitted string will not be null terminated.
+ *
+ * Output:
+ * The number of written characters.
+ *
+ * Exemplary usage:
+ *
+ * void print(double d)
+ * {
+ * char buf[24 + 1] // plus null terminator
+ * int str_len = fpconv_dtoa(d, buf);
+ *
+ * buf[str_len] = '\0';
+ * printf("%s", buf);
+ * }
+ *
+ */
+
+int fpconv_dtoa(double fp, char dest[FPCONV_BUFLEN], unsigned precision,
+ bool scientific);
+
+#endif
+
+/* [1] http://florian.loitsch.com/publications/dtoa-pldi2010.pdf */
diff --git a/contrib/fpconv/powers.h b/contrib/fpconv/powers.h
new file mode 100644
index 0000000..c707eed
--- /dev/null
+++ b/contrib/fpconv/powers.h
@@ -0,0 +1,87 @@
+#include <stdint.h>
+
+#define npowers 87
+#define steppowers 8
+#define firstpower -348 /* 10 ^ -348 */
+
+#define expmax -32
+#define expmin -60
+
+
+typedef struct Fp {
+ uint64_t frac;
+ int exp;
+} Fp;
+
+static Fp powers_ten[] = {
+ { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 },
+ { 10022474136428063862U, -1166 }, { 14934650266808366570U, -1140 },
+ { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 },
+ { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 },
+ { 13715310171984221708U, -1007 }, { 10218702384817765436U, -980 },
+ { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 },
+ { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 },
+ { 9384396036005875287U, -847 }, { 13983839803942852151U, -821 },
+ { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 },
+ { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 },
+ { 12842128665889583758U, -688 }, { 9568131466127621947U, -661 },
+ { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 },
+ { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 },
+ { 17573882009934360870U, -529 }, { 13093562431584567480U, -502 },
+ { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 },
+ { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 },
+ { 12024538023802026127U, -369 }, { 17917957937422433684U, -343 },
+ { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 },
+ { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 },
+ { 16455045573212060422U, -210 }, { 12259964326927110867U, -183 },
+ { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 },
+ { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 },
+ { 11258999068426240000U, -50 }, { 16777216000000000000U, -24 },
+ { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 },
+ { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 },
+ { 15407439555097886824U, 109 }, { 11479437019748901445U, 136 },
+ { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 },
+ { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 },
+ { 10542197943230523224U, 269 }, { 15709099088952724970U, 295 },
+ { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 },
+ { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 },
+ { 14426529090290212157U, 428 }, { 10748601772107342003U, 455 },
+ { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 },
+ { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 },
+ { 9871031767461413346U, 588 }, { 14708983551653345445U, 614 },
+ { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 },
+ { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 },
+ { 13508068024458167312U, 747 }, { 10064294952495520794U, 774 },
+ { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 },
+ { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 },
+ { 9242595204427927429U, 907 }, { 13772540099066387757U, 933 },
+ { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 },
+ { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 },
+ { 12648080533535911531U, 1066 }
+};
+
+static Fp find_cachedpow10(int exp, int* k)
+{
+ const double one_log_ten = 0.30102999566398114;
+
+ int approx = -(exp + npowers) * one_log_ten;
+ int idx = (approx - firstpower) / steppowers;
+
+ while(1) {
+ int current = exp + powers_ten[idx].exp + 64;
+
+ if(current < expmin) {
+ idx++;
+ continue;
+ }
+
+ if(current > expmax) {
+ idx--;
+ continue;
+ }
+
+ *k = (firstpower + idx * steppowers);
+
+ return powers_ten[idx];
+ }
+}
diff --git a/contrib/frozen/AUTHORS b/contrib/frozen/AUTHORS
new file mode 100644
index 0000000..d83d0f8
--- /dev/null
+++ b/contrib/frozen/AUTHORS
@@ -0,0 +1,3 @@
+serge-sans-paille <sguelton@quarkslab.com>
+Jérôme Dumesnil <jerome.dumesnil@gmail.com>
+Chris Beck <chbeck@tesla.com>
diff --git a/contrib/frozen/CMakeLists.txt b/contrib/frozen/CMakeLists.txt
new file mode 100644
index 0000000..185378d
--- /dev/null
+++ b/contrib/frozen/CMakeLists.txt
@@ -0,0 +1,12 @@
+target_sources(frozen-headers INTERFACE
+ "${prefix}/frozen/algorithm.h"
+ "${prefix}/frozen/map.h"
+ "${prefix}/frozen/random.h"
+ "${prefix}/frozen/set.h"
+ "${prefix}/frozen/string.h"
+ "${prefix}/frozen/unordered_map.h"
+ "${prefix}/frozen/unordered_set.h"
+ "${prefix}/frozen/bits/algorithms.h"
+ "${prefix}/frozen/bits/basic_types.h"
+ "${prefix}/frozen/bits/elsa.h"
+ "${prefix}/frozen/bits/pmh.h")
diff --git a/contrib/frozen/LICENSE b/contrib/frozen/LICENSE
new file mode 100644
index 0000000..5b4b9bd
--- /dev/null
+++ b/contrib/frozen/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2017 Quarkslab
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/contrib/frozen/include/frozen/algorithm.h b/contrib/frozen/include/frozen/algorithm.h
new file mode 100644
index 0000000..a543eb3
--- /dev/null
+++ b/contrib/frozen/include/frozen/algorithm.h
@@ -0,0 +1,197 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_ALGORITHM_H
+#define FROZEN_LETITGO_ALGORITHM_H
+
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/version.h"
+#include "frozen/string.h"
+
+namespace frozen {
+
+// 'search' implementation if C++17 is not available
+// https://en.cppreference.com/w/cpp/algorithm/search
+template<class ForwardIterator, class Searcher>
+ForwardIterator search(ForwardIterator first, ForwardIterator last, const Searcher & searcher)
+{
+ return searcher(first, last).first;
+}
+
+// text book implementation from
+// https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm
+
+template <std::size_t size> class knuth_morris_pratt_searcher {
+ bits::carray<std::ptrdiff_t, size> step_;
+ bits::carray<char, size> needle_;
+
+ static constexpr bits::carray<std::ptrdiff_t, size>
+ build_kmp_cache(char const (&needle)[size + 1]) {
+ std::ptrdiff_t cnd = 0;
+ bits::carray<std::ptrdiff_t, size> cache;
+
+ cache.fill(-1);
+ for (std::size_t pos = 1; pos < size; ++pos) {
+ if (needle[pos] == needle[cnd]) {
+ cache[pos] = cache[cnd];
+ cnd += 1;
+ } else {
+ cache[pos] = cnd;
+ cnd = cache[cnd];
+ while (cnd >= 0 && needle[pos] != needle[cnd])
+ cnd = cache[cnd];
+ cnd += 1;
+ }
+ }
+ return cache;
+ }
+
+public:
+ constexpr knuth_morris_pratt_searcher(char const (&needle)[size + 1])
+ : step_{build_kmp_cache(needle)}, needle_(needle) {}
+
+ template <class ForwardIterator>
+ constexpr std::pair<ForwardIterator, ForwardIterator> operator()(ForwardIterator first, ForwardIterator last) const {
+ std::size_t i = 0;
+ ForwardIterator iter = first;
+ while (iter != last) {
+ if (needle_[i] == *iter) {
+ if (i == (size - 1))
+ return { iter - i, iter - i + size };
+ ++i;
+ ++iter;
+ } else {
+ if (step_[i] > -1) {
+ i = step_[i];
+ } else {
+ ++iter;
+ i = 0;
+ }
+ }
+ }
+ return { last, last };
+ }
+};
+
+template <std::size_t N>
+constexpr knuth_morris_pratt_searcher<N - 1> make_knuth_morris_pratt_searcher(char const (&needle)[N]) {
+ return {needle};
+}
+
+// text book implementation from
+// https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm
+
+template <std::size_t size> class boyer_moore_searcher {
+ using skip_table_type = bits::carray<std::ptrdiff_t, sizeof(char) << 8>;
+ using suffix_table_type = bits::carray<std::ptrdiff_t, size>;
+
+ skip_table_type skip_table_;
+ suffix_table_type suffix_table_;
+ bits::carray<char, size> needle_;
+
+ constexpr auto build_skip_table(char const (&needle)[size + 1]) {
+ skip_table_type skip_table;
+
+ skip_table.fill(size);
+ for (std::size_t i = 0; i < size - 1; ++i)
+ skip_table[needle[i]] -= i + 1;
+ return skip_table;
+ }
+
+ constexpr bool is_prefix(char const (&needle)[size + 1], std::size_t pos) {
+ std::size_t suffixlen = size - pos;
+
+ for (std::size_t i = 0; i < suffixlen; i++) {
+ if (needle[i] != needle[pos + i])
+ return false;
+ }
+ return true;
+ }
+
+ constexpr std::size_t suffix_length(char const (&needle)[size + 1],
+ std::size_t pos) {
+ // increment suffix length slen to the first mismatch or beginning
+ // of the word
+ for (std::size_t slen = 0; slen < pos ; slen++)
+ if (needle[pos - slen] != needle[size - 1 - slen])
+ return slen;
+
+ return pos;
+ }
+
+ constexpr auto build_suffix_table(char const (&needle)[size + 1]) {
+ suffix_table_type suffix;
+ std::ptrdiff_t last_prefix_index = size - 1;
+
+ // first loop
+ for (std::ptrdiff_t p = size - 1; p >= 0; p--) {
+ if (is_prefix(needle, p + 1))
+ last_prefix_index = p + 1;
+
+ suffix[p] = last_prefix_index + (size - 1 - p);
+ }
+
+ // second loop
+ for (std::size_t p = 0; p < size - 1; p++) {
+ auto slen = suffix_length(needle, p);
+ if (needle[p - slen] != needle[size - 1 - slen])
+ suffix[size - 1 - slen] = size - 1 - p + slen;
+
+ }
+ return suffix;
+ }
+
+public:
+ constexpr boyer_moore_searcher(char const (&needle)[size + 1])
+ : skip_table_{build_skip_table(needle)},
+ suffix_table_{build_suffix_table(needle)},
+ needle_(needle) {}
+
+ template <class ForwardIterator>
+ constexpr std::pair<ForwardIterator, ForwardIterator> operator()(ForwardIterator first, ForwardIterator last) const {
+ if (size == 0)
+ return { first, first + size };
+
+ ForwardIterator iter = first + size - 1;
+ while (iter < last) {
+ std::ptrdiff_t j = size - 1;
+ while (j > 0 && (*iter == needle_[j])) {
+ --iter;
+ --j;
+ }
+ if (*iter == needle_[0])
+ return { iter, iter + size};
+
+ iter += std::max(skip_table_[*iter], suffix_table_[j]);
+ }
+ return { last, last + size};
+ }
+};
+
+template <std::size_t N>
+constexpr boyer_moore_searcher<N - 1> make_boyer_moore_searcher(char const (&needle)[N]) {
+ return {needle};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/algorithms.h b/contrib/frozen/include/frozen/bits/algorithms.h
new file mode 100644
index 0000000..8d1ffbc
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/algorithms.h
@@ -0,0 +1,229 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_BITS_ALGORITHMS_H
+#define FROZEN_LETITGO_BITS_ALGORITHMS_H
+
+#include "frozen/bits/basic_types.h"
+
+#include <limits>
+#include <tuple>
+
+namespace frozen {
+
+namespace bits {
+
+auto constexpr next_highest_power_of_two(std::size_t v) {
+ // https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+ constexpr auto trip_count = std::numeric_limits<decltype(v)>::digits;
+ v--;
+ for(std::size_t i = 1; i < trip_count; i <<= 1)
+ v |= v >> i;
+ v++;
+ return v;
+}
+
+template<class T>
+auto constexpr log(T v) {
+ std::size_t n = 0;
+ while (v > 1) {
+ n += 1;
+ v >>= 1;
+ }
+ return n;
+}
+
+constexpr std::size_t bit_weight(std::size_t n) {
+ return (n <= 8*sizeof(unsigned int))
+ + (n <= 8*sizeof(unsigned long))
+ + (n <= 8*sizeof(unsigned long long))
+ + (n <= 128);
+}
+
+unsigned int select_uint_least(std::integral_constant<std::size_t, 4>);
+unsigned long select_uint_least(std::integral_constant<std::size_t, 3>);
+unsigned long long select_uint_least(std::integral_constant<std::size_t, 2>);
+template<std::size_t N>
+unsigned long long select_uint_least(std::integral_constant<std::size_t, N>) {
+ static_assert(N < 2, "unsupported type size");
+ return {};
+}
+
+
+template<std::size_t N>
+using select_uint_least_t = decltype(select_uint_least(std::integral_constant<std::size_t, bit_weight(N)>()));
+
+template <typename Iter, typename Compare>
+constexpr auto min_element(Iter begin, const Iter end,
+ Compare const &compare) {
+ auto result = begin;
+ while (begin != end) {
+ if (compare(*begin, *result)) {
+ result = begin;
+ }
+ ++begin;
+ }
+ return result;
+}
+
+template <class T>
+constexpr void cswap(T &a, T &b) {
+ auto tmp = a;
+ a = b;
+ b = tmp;
+}
+
+template <class T, class U>
+constexpr void cswap(std::pair<T, U> & a, std::pair<T, U> & b) {
+ cswap(a.first, b.first);
+ cswap(a.second, b.second);
+}
+
+template <class... Tys, std::size_t... Is>
+constexpr void cswap(std::tuple<Tys...> &a, std::tuple<Tys...> &b, std::index_sequence<Is...>) {
+ using swallow = int[];
+ (void) swallow{(cswap(std::get<Is>(a), std::get<Is>(b)), 0)...};
+}
+
+template <class... Tys>
+constexpr void cswap(std::tuple<Tys...> &a, std::tuple<Tys...> &b) {
+ cswap(a, b, std::make_index_sequence<sizeof...(Tys)>());
+}
+
+template <typename Iterator, class Compare>
+constexpr Iterator partition(Iterator left, Iterator right, Compare const &compare) {
+ auto pivot = left + (right - left) / 2;
+ auto value = *pivot;
+ cswap(*right, *pivot);
+ for (auto it = left; 0 < right - it; ++it) {
+ if (compare(*it, value)) {
+ cswap(*it, *left);
+ left++;
+ }
+ }
+ cswap(*right, *left);
+ return left;
+}
+
+template <typename Iterator, class Compare>
+constexpr void quicksort(Iterator left, Iterator right, Compare const &compare) {
+ while (0 < right - left) {
+ auto new_pivot = bits::partition(left, right, compare);
+ quicksort(left, new_pivot, compare);
+ left = new_pivot + 1;
+ }
+}
+
+template <typename T, std::size_t N, class Compare>
+constexpr bits::carray<T, N> quicksort(bits::carray<T, N> const &array,
+ Compare const &compare) {
+ bits::carray<T, N> res = array;
+ quicksort(res.begin(), res.end() - 1, compare);
+ return res;
+}
+
+template <class T, class Compare> struct LowerBound {
+ T const &value_;
+ Compare const &compare_;
+ constexpr LowerBound(T const &value, Compare const &compare)
+ : value_(value), compare_(compare) {}
+
+ template <class ForwardIt>
+ inline constexpr ForwardIt doit_fast(ForwardIt first,
+ std::integral_constant<std::size_t, 0>) {
+ return first;
+ }
+
+ template <class ForwardIt, std::size_t N>
+ inline constexpr ForwardIt doit_fast(ForwardIt first,
+ std::integral_constant<std::size_t, N>) {
+ auto constexpr step = N / 2;
+ static_assert(N/2 == N - N / 2 - 1, "power of two minus 1");
+ auto it = first + step;
+ auto next_it = compare_(*it, value_) ? it + 1 : first;
+ return doit_fast(next_it, std::integral_constant<std::size_t, N / 2>{});
+ }
+
+ template <class ForwardIt, std::size_t N>
+ inline constexpr ForwardIt doitfirst(ForwardIt first, std::integral_constant<std::size_t, N>, std::integral_constant<bool, true>) {
+ return doit_fast(first, std::integral_constant<std::size_t, N>{});
+ }
+
+ template <class ForwardIt, std::size_t N>
+ inline constexpr ForwardIt doitfirst(ForwardIt first, std::integral_constant<std::size_t, N>, std::integral_constant<bool, false>) {
+ auto constexpr next_power = next_highest_power_of_two(N);
+ auto constexpr next_start = next_power / 2 - 1;
+ auto it = first + next_start;
+ if (compare_(*it, value_)) {
+ auto constexpr next = N - next_start - 1;
+ return doitfirst(it + 1, std::integral_constant<std::size_t, next>{}, std::integral_constant<bool, next_highest_power_of_two(next) - 1 == next>{});
+ }
+ else
+ return doit_fast(first, std::integral_constant<std::size_t, next_start>{});
+ }
+
+ template <class ForwardIt>
+ inline constexpr ForwardIt doitfirst(ForwardIt first, std::integral_constant<std::size_t, 1>, std::integral_constant<bool, false>) {
+ return doit_fast(first, std::integral_constant<std::size_t, 1>{});
+ }
+};
+
+template <std::size_t N, class ForwardIt, class T, class Compare>
+constexpr ForwardIt lower_bound(ForwardIt first, const T &value, Compare const &compare) {
+ return LowerBound<T, Compare>{value, compare}.doitfirst(first, std::integral_constant<std::size_t, N>{}, std::integral_constant<bool, next_highest_power_of_two(N) - 1 == N>{});
+}
+
+template <std::size_t N, class Compare, class ForwardIt, class T>
+constexpr bool binary_search(ForwardIt first, const T &value,
+ Compare const &compare) {
+ ForwardIt where = lower_bound<N>(first, value, compare);
+ return (!(where == first + N) && !(compare(value, *where)));
+}
+
+
+template<class InputIt1, class InputIt2>
+constexpr bool equal(InputIt1 first1, InputIt1 last1, InputIt2 first2)
+{
+ for (; first1 != last1; ++first1, ++first2) {
+ if (!(*first1 == *first2)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+template<class InputIt1, class InputIt2>
+constexpr bool lexicographical_compare(InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2)
+{
+ for (; (first1 != last1) && (first2 != last2); ++first1, ++first2) {
+ if (*first1 < *first2)
+ return true;
+ if (*first2 < *first1)
+ return false;
+ }
+ return (first1 == last1) && (first2 != last2);
+}
+
+} // namespace bits
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/basic_types.h b/contrib/frozen/include/frozen/bits/basic_types.h
new file mode 100644
index 0000000..9814bac
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/basic_types.h
@@ -0,0 +1,200 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_BASIC_TYPES_H
+#define FROZEN_LETITGO_BASIC_TYPES_H
+
+#include "frozen/bits/exceptions.h"
+
+#include <utility>
+#include <iterator>
+#include <string>
+
+namespace frozen {
+
+namespace bits {
+
+// used as a fake argument for frozen::make_set and frozen::make_map in the case of N=0
+struct ignored_arg {};
+
+template <class T, std::size_t N>
+class cvector {
+ T data [N] = {}; // zero-initialization for scalar type T, default-initialized otherwise
+ std::size_t dsize = 0;
+
+public:
+ // Container typdefs
+ using value_type = T;
+ using reference = value_type &;
+ using const_reference = const value_type &;
+ using pointer = value_type *;
+ using const_pointer = const value_type *;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using size_type = std::size_t;
+ using difference_type = std::ptrdiff_t;
+
+ // Constructors
+ constexpr cvector(void) = default;
+ constexpr cvector(size_type count, const T& value) : dsize(count) {
+ for (std::size_t i = 0; i < N; ++i)
+ data[i] = value;
+ }
+
+ // Iterators
+ constexpr iterator begin() noexcept { return data; }
+ constexpr iterator end() noexcept { return data + dsize; }
+
+ // Capacity
+ constexpr size_type size() const { return dsize; }
+
+ // Element access
+ constexpr reference operator[](std::size_t index) { return data[index]; }
+ constexpr const_reference operator[](std::size_t index) const { return data[index]; }
+
+ constexpr reference back() { return data[dsize - 1]; }
+ constexpr const_reference back() const { return data[dsize - 1]; }
+
+ // Modifiers
+ constexpr void push_back(const T & a) { data[dsize++] = a; }
+ constexpr void push_back(T && a) { data[dsize++] = std::move(a); }
+ constexpr void pop_back() { --dsize; }
+
+ constexpr void clear() { dsize = 0; }
+};
+
+template <class T, std::size_t N>
+class carray {
+ T data_ [N] = {}; // zero-initialization for scalar type T, default-initialized otherwise
+
+ template <std::size_t M, std::size_t... I>
+ constexpr carray(T const (&init)[M], std::index_sequence<I...>)
+ : data_{init[I]...} {}
+ template <class Iter, std::size_t... I>
+ constexpr carray(Iter iter, std::index_sequence<I...>)
+ : data_{((void)I, *iter++)...} {}
+
+public:
+ // Container typdefs
+ using value_type = T;
+ using reference = value_type &;
+ using const_reference = const value_type &;
+ using pointer = value_type *;
+ using const_pointer = const value_type *;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+ using size_type = std::size_t;
+ using difference_type = std::ptrdiff_t;
+
+ // Constructors
+ constexpr carray(void) = default;
+ template <std::size_t M>
+ constexpr carray(T const (&init)[M])
+ : carray(init, std::make_index_sequence<N>())
+ {
+ static_assert(M >= N, "Cannot initialize a carray with an smaller array");
+ }
+ constexpr carray(std::initializer_list<T> init)
+ : carray(init.begin(), std::make_index_sequence<N>())
+ {
+ // clang & gcc doesn't recognize init.size() as a constexpr
+ // static_assert(init.size() >= N, "Cannot initialize a carray with an smaller initializer list");
+ }
+
+ // Iterators
+ constexpr iterator begin() noexcept { return data_; }
+ constexpr const_iterator begin() const noexcept { return data_; }
+ constexpr const_iterator cbegin() const noexcept { return data_; }
+ constexpr iterator end() noexcept { return data_ + N; }
+ constexpr const_iterator end() const noexcept { return data_ + N; }
+ constexpr const_iterator cend() const noexcept { return data_ + N; }
+
+ constexpr reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
+ constexpr const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
+ constexpr const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
+ constexpr reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
+ constexpr const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
+ constexpr const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
+
+ // Capacity
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ // Element access
+ constexpr reference operator[](std::size_t index) { return data_[index]; }
+ constexpr const_reference operator[](std::size_t index) const { return data_[index]; }
+
+ constexpr reference at(std::size_t index) {
+ if (index > N)
+ FROZEN_THROW_OR_ABORT(std::out_of_range("Index (" + std::to_string(index) + ") out of bound (" + std::to_string(N) + ')'));
+ return data_[index];
+ }
+ constexpr const_reference at(std::size_t index) const {
+ if (index > N)
+ FROZEN_THROW_OR_ABORT(std::out_of_range("Index (" + std::to_string(index) + ") out of bound (" + std::to_string(N) + ')'));
+ return data_[index];
+ }
+
+ constexpr reference front() { return data_[0]; }
+ constexpr const_reference front() const { return data_[0]; }
+
+ constexpr reference back() { return data_[N - 1]; }
+ constexpr const_reference back() const { return data_[N - 1]; }
+
+ constexpr value_type* data() noexcept { return data_; }
+ constexpr const value_type* data() const noexcept { return data_; }
+
+ // Modifiers
+ constexpr void fill(const value_type& val) {
+ for (std::size_t i = 0; i < N; ++i)
+ data_[i] = val;
+ }
+};
+template <class T>
+class carray<T, 0> {
+
+public:
+ // Container typdefs
+ using value_type = T;
+ using reference = value_type &;
+ using const_reference = const value_type &;
+ using pointer = value_type *;
+ using const_pointer = const value_type *;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+ using size_type = std::size_t;
+ using difference_type = std::ptrdiff_t;
+
+ // Constructors
+ constexpr carray(void) = default;
+
+};
+
+} // namespace bits
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/constexpr_assert.h b/contrib/frozen/include/frozen/bits/constexpr_assert.h
new file mode 100644
index 0000000..912210d
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/constexpr_assert.h
@@ -0,0 +1,40 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_CONSTEXPR_ASSERT_H
+#define FROZEN_LETITGO_CONSTEXPR_ASSERT_H
+
+#include <cassert>
+
+#ifdef _MSC_VER
+
+// FIXME: find a way to implement that correctly for msvc
+#define constexpr_assert(cond, msg)
+
+#else
+
+#define constexpr_assert(cond, msg)\
+ assert(cond && msg);
+#endif
+
+#endif
+
diff --git a/contrib/frozen/include/frozen/bits/defines.h b/contrib/frozen/include/frozen/bits/defines.h
new file mode 100644
index 0000000..0a1663d
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/defines.h
@@ -0,0 +1,58 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_DEFINES_H
+#define FROZEN_LETITGO_DEFINES_H
+
+#if defined(_MSVC_LANG) && !(defined(__EDG__) && defined(__clang__)) // TRANSITION, VSO#273681
+ #define FROZEN_LETITGO_IS_MSVC
+#endif
+
+// Code taken from https://stackoverflow.com/questions/43639122/which-values-can-msvc-lang-have
+#if defined(FROZEN_LETITGO_IS_MSVC)
+ #if _MSVC_LANG > 201402
+ #define FROZEN_LETITGO_HAS_CXX17 1
+ #else /* _MSVC_LANG > 201402 */
+ #define FROZEN_LETITGO_HAS_CXX17 0
+ #endif /* _MSVC_LANG > 201402 */
+#else /* _MSVC_LANG etc. */
+ #if __cplusplus > 201402
+ #define FROZEN_LETITGO_HAS_CXX17 1
+ #else /* __cplusplus > 201402 */
+ #define FROZEN_LETITGO_HAS_CXX17 0
+ #endif /* __cplusplus > 201402 */
+#endif /* _MSVC_LANG etc. */
+// End if taken code
+
+#if FROZEN_LETITGO_HAS_CXX17 == 1 && defined(FROZEN_LETITGO_IS_MSVC)
+ #define FROZEN_LETITGO_HAS_STRING_VIEW // We assume Visual Studio always has string_view in C++17
+#else
+ #if FROZEN_LETITGO_HAS_CXX17 == 1 && __has_include(<string_view>)
+ #define FROZEN_LETITGO_HAS_STRING_VIEW
+ #endif
+#endif
+
+#ifdef __cpp_char8_t
+ #define FROZEN_LETITGO_HAS_CHAR8T
+#endif
+
+#endif // FROZEN_LETITGO_DEFINES_H
diff --git a/contrib/frozen/include/frozen/bits/elsa.h b/contrib/frozen/include/frozen/bits/elsa.h
new file mode 100644
index 0000000..d7388be
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/elsa.h
@@ -0,0 +1,50 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_ELSA_H
+#define FROZEN_LETITGO_ELSA_H
+
+#include <type_traits>
+
+namespace frozen {
+
+template <class T> struct elsa {
+ static_assert(std::is_integral<T>::value || std::is_enum<T>::value,
+ "only supports integral types, specialize for other types");
+
+ constexpr std::size_t operator()(T const &value, std::size_t seed) const {
+ std::size_t key = seed ^ static_cast<std::size_t>(value);
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >> 28);
+ key = key + (key << 31);
+ return key;
+ }
+};
+
+template <class T> using anna = elsa<T>;
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/exceptions.h b/contrib/frozen/include/frozen/bits/exceptions.h
new file mode 100644
index 0000000..b43e3e6
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/exceptions.h
@@ -0,0 +1,39 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_EXCEPTIONS_H
+#define FROZEN_LETITGO_EXCEPTIONS_H
+
+#if defined(FROZEN_NO_EXCEPTIONS) || (defined(_MSC_VER) && !defined(_CPPUNWIND)) || (!defined(_MSC_VER) && !defined(__cpp_exceptions))
+
+#include <cstdlib>
+#define FROZEN_THROW_OR_ABORT(_) std::abort()
+
+#else
+
+#include <stdexcept>
+#define FROZEN_THROW_OR_ABORT(err) throw err
+
+
+#endif
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/pmh.h b/contrib/frozen/include/frozen/bits/pmh.h
new file mode 100644
index 0000000..76e7ebe
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/pmh.h
@@ -0,0 +1,240 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// inspired from http://stevehanov.ca/blog/index.php?id=119
+#ifndef FROZEN_LETITGO_PMH_H
+#define FROZEN_LETITGO_PMH_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/basic_types.h"
+
+#include <array>
+#include <limits>
+
+namespace frozen {
+
+namespace bits {
+
+// Function object for sorting buckets in decreasing order of size
+struct bucket_size_compare {
+ template <typename B>
+ bool constexpr operator()(B const &b0,
+ B const &b1) const {
+ return b0.size() > b1.size();
+ }
+};
+
+// Step One in pmh routine is to take all items and hash them into buckets,
+// with some collisions. Then process those buckets further to build a perfect
+// hash function.
+// pmh_buckets represents the initial placement into buckets.
+
+template <size_t M>
+struct pmh_buckets {
+ // Step 0: Bucket max is 2 * sqrt M
+ // TODO: Come up with justification for this, should it not be O(log M)?
+ static constexpr auto bucket_max = 2 * (1u << (log(M) / 2));
+
+ using bucket_t = cvector<std::size_t, bucket_max>;
+ carray<bucket_t, M> buckets;
+ uint64_t seed;
+
+ // Represents a reference to a bucket. This is used because the buckets
+ // have to be sorted, but buckets are big, making it slower than sorting refs
+ struct bucket_ref {
+ unsigned hash;
+ const bucket_t * ptr;
+
+ // Forward some interface of bucket
+ using value_type = typename bucket_t::value_type;
+ using const_iterator = typename bucket_t::const_iterator;
+
+ constexpr auto size() const { return ptr->size(); }
+ constexpr const auto & operator[](std::size_t idx) const { return (*ptr)[idx]; }
+ constexpr auto begin() const { return ptr->begin(); }
+ constexpr auto end() const { return ptr->end(); }
+ };
+
+ // Make a bucket_ref for each bucket
+ template <std::size_t... Is>
+ carray<bucket_ref, M> constexpr make_bucket_refs(std::index_sequence<Is...>) const {
+ return {{ bucket_ref{Is, &buckets[Is]}... }};
+ }
+
+ // Makes a bucket_ref for each bucket and sorts them by size
+ carray<bucket_ref, M> constexpr get_sorted_buckets() const {
+ carray<bucket_ref, M> result{this->make_bucket_refs(std::make_index_sequence<M>())};
+ bits::quicksort(result.begin(), result.end() - 1, bucket_size_compare{});
+ return result;
+ }
+};
+
+template <size_t M, class Item, size_t N, class Hash, class Key, class PRG>
+pmh_buckets<M> constexpr make_pmh_buckets(const carray<Item, N> & items,
+ Hash const & hash,
+ Key const & key,
+ PRG & prg) {
+ using result_t = pmh_buckets<M>;
+ result_t result{};
+ bool rejected = false;
+ // Continue until all items are placed without exceeding bucket_max
+ while (1) {
+ for (auto & b : result.buckets) {
+ b.clear();
+ }
+ result.seed = prg();
+ rejected = false;
+ for (std::size_t i = 0; i < N; ++i) {
+ auto & bucket = result.buckets[hash(key(items[i]), static_cast<size_t>(result.seed)) % M];
+ if (bucket.size() >= result_t::bucket_max) {
+ rejected = true;
+ break;
+ }
+ bucket.push_back(i);
+ }
+ if (!rejected) { return result; }
+ }
+}
+
+// Check if an item appears in a cvector
+template<class T, size_t N>
+constexpr bool all_different_from(cvector<T, N> & data, T & a) {
+ for (std::size_t i = 0; i < data.size(); ++i)
+ if (data[i] == a)
+ return false;
+
+ return true;
+}
+
+// Represents either an index to a data item array, or a seed to be used with
+// a hasher. Seed must have high bit of 1, value has high bit of zero.
+struct seed_or_index {
+ using value_type = uint64_t;
+
+private:
+ static constexpr value_type MINUS_ONE = std::numeric_limits<value_type>::max();
+ static constexpr value_type HIGH_BIT = ~(MINUS_ONE >> 1);
+
+ value_type value_ = 0;
+
+public:
+ constexpr value_type value() const { return value_; }
+ constexpr bool is_seed() const { return value_ & HIGH_BIT; }
+
+ constexpr seed_or_index(bool is_seed, value_type value)
+ : value_(is_seed ? (value | HIGH_BIT) : (value & ~HIGH_BIT)) {}
+
+ constexpr seed_or_index() = default;
+ constexpr seed_or_index(const seed_or_index &) = default;
+ constexpr seed_or_index & operator =(const seed_or_index &) = default;
+};
+
+// Represents the perfect hash function created by pmh algorithm
+template <std::size_t M, class Hasher>
+struct pmh_tables {
+ uint64_t first_seed_;
+ carray<seed_or_index, M> first_table_;
+ carray<std::size_t, M> second_table_;
+ Hasher hash_;
+
+ // Looks up a given key, to find its expected index in carray<Item, N>
+ // Always returns a valid index, must use KeyEqual test after to confirm.
+ template <typename KeyType>
+ constexpr std::size_t lookup(const KeyType & key) const {
+ auto const d = first_table_[hash_(key, static_cast<size_t>(first_seed_)) % M];
+ if (!d.is_seed()) { return static_cast<std::size_t>(d.value()); } // this is narrowing uint64 -> size_t but should be fine
+ else { return second_table_[hash_(key, static_cast<std::size_t>(d.value())) % M]; }
+ }
+};
+
+// Make pmh tables for given items, hash function, prg, etc.
+template <std::size_t M, class Item, std::size_t N, class Hash, class Key, class PRG>
+pmh_tables<M, Hash> constexpr make_pmh_tables(const carray<Item, N> &
+ items,
+ Hash const &hash,
+ Key const &key,
+ PRG prg) {
+ // Step 1: Place all of the keys into buckets
+ auto step_one = make_pmh_buckets<M>(items, hash, key, prg);
+
+ // Step 2: Sort the buckets to process the ones with the most items first.
+ auto buckets = step_one.get_sorted_buckets();
+
+ // G becomes the first hash table in the resulting pmh function
+ carray<seed_or_index, M> G; // Default constructed to "index 0"
+
+ // H becomes the second hash table in the resulting pmh function
+ constexpr std::size_t UNUSED = std::numeric_limits<std::size_t>::max();
+ carray<std::size_t, M> H;
+ H.fill(UNUSED);
+
+ // Step 3: Map the items in buckets into hash tables.
+ for (const auto & bucket : buckets) {
+ auto const bsize = bucket.size();
+
+ if (bsize == 1) {
+ // Store index to the (single) item in G
+ // assert(bucket.hash == hash(key(items[bucket[0]]), step_one.seed) % M);
+ G[bucket.hash] = {false, static_cast<uint64_t>(bucket[0])};
+ } else if (bsize > 1) {
+
+ // Repeatedly try different H of d until we find a hash function
+ // that places all items in the bucket into free slots
+ seed_or_index d{true, prg()};
+ cvector<std::size_t, decltype(step_one)::bucket_max> bucket_slots;
+
+ while (bucket_slots.size() < bsize) {
+ auto slot = hash(key(items[bucket[bucket_slots.size()]]), static_cast<size_t>(d.value())) % M;
+
+ if (H[slot] != UNUSED || !all_different_from(bucket_slots, slot)) {
+ bucket_slots.clear();
+ d = {true, prg()};
+ continue;
+ }
+
+ bucket_slots.push_back(slot);
+ }
+
+ // Put successful seed in G, and put indices to items in their slots
+ // assert(bucket.hash == hash(key(items[bucket[0]]), step_one.seed) % M);
+ G[bucket.hash] = d;
+ for (std::size_t i = 0; i < bsize; ++i)
+ H[bucket_slots[i]] = bucket[i];
+ }
+ }
+
+ // Any unused entries in the H table have to get changed to zero.
+ // This is because hashing should not fail or return an out-of-bounds entry.
+ // A lookup fails after we apply user-supplied KeyEqual to the query and the
+ // key found by hashing. Sending such queries to zero cannot hurt.
+ for (std::size_t i = 0; i < M; ++i)
+ if (H[i] == UNUSED)
+ H[i] = 0;
+
+ return {step_one.seed, G, H, hash};
+}
+
+} // namespace bits
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/version.h b/contrib/frozen/include/frozen/bits/version.h
new file mode 100644
index 0000000..51804d2
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/version.h
@@ -0,0 +1,30 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_VERSION_H
+#define FROZEN_LETITGO_VERSION_H
+
+#define FROZEN_MAJOR_VERSION 1
+#define FROZEN_MINOR_VERSION 0
+#define FROZEN_PATCH_VERSION 1
+
+#endif
diff --git a/contrib/frozen/include/frozen/map.h b/contrib/frozen/include/frozen/map.h
new file mode 100644
index 0000000..107179c
--- /dev/null
+++ b/contrib/frozen/include/frozen/map.h
@@ -0,0 +1,323 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_MAP_H
+#define FROZEN_LETITGO_MAP_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/exceptions.h"
+#include "frozen/bits/version.h"
+
+#include <utility>
+
+namespace frozen {
+
+namespace impl {
+
+template <class Comparator> class CompareKey {
+
+ Comparator const comparator_;
+
+public:
+ constexpr CompareKey(Comparator const &comparator)
+ : comparator_(comparator) {}
+
+ template <class Key, class Value>
+ constexpr int operator()(std::pair<Key, Value> const &self,
+ std::pair<Key, Value> const &other) const {
+ return comparator_(std::get<0>(self), std::get<0>(other));
+ }
+
+ template <class Key, class Value>
+ constexpr int operator()(Key const &self_key,
+ std::pair<Key, Value> const &other) const {
+ return comparator_(self_key, std::get<0>(other));
+ }
+
+ template <class Key, class Value>
+ constexpr int operator()(std::pair<Key, Value> const &self,
+ Key const &other_key) const {
+ return comparator_(std::get<0>(self), other_key);
+ }
+
+ template <class Key>
+ constexpr int operator()(Key const &self_key, Key const &other_key) const {
+ return comparator_(self_key, other_key);
+ }
+};
+
+} // namespace impl
+
+template <class Key, class Value, std::size_t N, class Compare = std::less<Key>>
+class map {
+ using container_type = bits::carray<std::pair<Key, Value>, N>;
+ impl::CompareKey<Compare> less_than_;
+ container_type items_;
+
+public:
+ using key_type = Key;
+ using mapped_type = Value;
+ using value_type = typename container_type::value_type;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using key_compare = decltype(less_than_);
+ using reference = typename container_type::reference;
+ using const_reference = typename container_type::const_reference;
+ using pointer = typename container_type::pointer;
+ using const_pointer = typename container_type::const_pointer;
+ using iterator = typename container_type::iterator;
+ using const_iterator = typename container_type::const_iterator;
+ using reverse_iterator = typename container_type::reverse_iterator;
+ using const_reverse_iterator =
+ typename container_type::const_reverse_iterator;
+
+public:
+ /* constructors */
+ constexpr map(container_type items, Compare const &compare)
+ : less_than_{compare}
+ , items_{bits::quicksort(items, less_than_)} {}
+
+ explicit constexpr map(container_type items)
+ : map{items, Compare{}} {}
+
+ constexpr map(std::initializer_list<value_type> items, Compare const &compare)
+ : map{container_type {items}, compare} {
+ constexpr_assert(items.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ constexpr map(std::initializer_list<value_type> items)
+ : map{items, Compare{}} {}
+
+ /* element access */
+ constexpr Value const& at(Key const &key) const {
+ return at_impl(*this, key);
+ }
+ constexpr Value& at(Key const &key) {
+ return at_impl(*this, key);
+ }
+
+ /* iterators */
+ constexpr iterator begin() { return items_.begin(); }
+ constexpr const_iterator begin() const { return items_.begin(); }
+ constexpr const_iterator cbegin() const { return items_.cbegin(); }
+ constexpr iterator end() { return items_.end(); }
+ constexpr const_iterator end() const { return items_.end(); }
+ constexpr const_iterator cend() const { return items_.cend(); }
+
+ constexpr reverse_iterator rbegin() { return items_.rbegin(); }
+ constexpr const_reverse_iterator rbegin() const { return items_.rbegin(); }
+ constexpr const_reverse_iterator crbegin() const { return items_.crbegin(); }
+ constexpr reverse_iterator rend() { return items_.rend(); }
+ constexpr const_reverse_iterator rend() const { return items_.rend(); }
+ constexpr const_reverse_iterator crend() const { return items_.crend(); }
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+
+ constexpr std::size_t count(Key const &key) const {
+ return bits::binary_search<N>(items_.begin(), key, less_than_);
+ }
+
+ constexpr const_iterator find(Key const &key) const {
+ return find_impl(*this, key);
+ }
+ constexpr iterator find(Key const &key) {
+ return find_impl(*this, key);
+ }
+
+ constexpr std::pair<const_iterator, const_iterator>
+ equal_range(Key const &key) const {
+ return equal_range_impl(*this, key);
+ }
+ constexpr std::pair<iterator, iterator> equal_range(Key const &key) {
+ return equal_range_impl(*this, key);
+ }
+
+ constexpr const_iterator lower_bound(Key const &key) const {
+ return lower_bound_impl(*this, key);
+ }
+ constexpr iterator lower_bound(Key const &key) {
+ return lower_bound_impl(*this, key);
+ }
+
+ constexpr const_iterator upper_bound(Key const &key) const {
+ return upper_bound_impl(*this, key);
+ }
+ constexpr iterator upper_bound(Key const &key) {
+ return upper_bound_impl(*this, key);
+ }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+
+ private:
+ template <class This>
+ static inline constexpr auto& at_impl(This&& self, Key const &key) {
+ auto where = self.lower_bound(key);
+ if (where != self.end())
+ return where->second;
+ else
+ FROZEN_THROW_OR_ABORT(std::out_of_range("unknown key"));
+ }
+
+ template <class This>
+ static inline constexpr auto find_impl(This&& self, Key const &key) {
+ auto where = self.lower_bound(key);
+ if ((where != self.end()) && !self.less_than_(key, *where))
+ return where;
+ else
+ return self.end();
+ }
+
+ template <class This>
+ static inline constexpr auto equal_range_impl(This&& self, Key const &key) {
+ auto lower = self.lower_bound(key);
+ using lower_t = decltype(lower);
+ if (lower == self.end())
+ return std::pair<lower_t, lower_t>{lower, lower};
+ else
+ return std::pair<lower_t, lower_t>{lower, lower + 1};
+ }
+
+ template <class This>
+ static inline constexpr auto lower_bound_impl(This&& self, Key const &key) -> decltype(self.end()) {
+ auto where = bits::lower_bound<N>(self.items_.begin(), key, self.less_than_);
+ if ((where != self.end()) && !self.less_than_(key, *where))
+ return where;
+ else
+ return self.end();
+ }
+
+ template <class This>
+ static inline constexpr auto upper_bound_impl(This&& self, Key const &key) -> decltype(self.end()) {
+ auto where = bits::lower_bound<N>(self.items_.begin(), key, self.less_than_);
+ if ((where != self.end()) && !self.less_than_(key, *where))
+ return where + 1;
+ else
+ return self.end();
+ }
+};
+
+template <class Key, class Value, class Compare>
+class map<Key, Value, 0, Compare> {
+ using container_type = bits::carray<std::pair<Key, Value>, 0>;
+ impl::CompareKey<Compare> less_than_;
+
+public:
+ using key_type = Key;
+ using mapped_type = Value;
+ using value_type = typename container_type::value_type;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using key_compare = decltype(less_than_);
+ using reference = typename container_type::reference;
+ using const_reference = typename container_type::const_reference;
+ using pointer = typename container_type::pointer;
+ using const_pointer = typename container_type::const_pointer;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using reverse_iterator = pointer;
+ using const_reverse_iterator = const_pointer;
+
+public:
+ /* constructors */
+ constexpr map(const map &other) = default;
+ constexpr map(std::initializer_list<value_type>, Compare const &compare)
+ : less_than_{compare} {}
+ constexpr map(std::initializer_list<value_type> items)
+ : map{items, Compare{}} {}
+
+ /* element access */
+ constexpr mapped_type at(Key const &) const {
+ FROZEN_THROW_OR_ABORT(std::out_of_range("invalid key"));
+ }
+ constexpr mapped_type at(Key const &) {
+ FROZEN_THROW_OR_ABORT(std::out_of_range("invalid key"));
+ }
+
+ /* iterators */
+ constexpr iterator begin() { return nullptr; }
+ constexpr const_iterator begin() const { return nullptr; }
+ constexpr const_iterator cbegin() const { return nullptr; }
+ constexpr iterator end() { return nullptr; }
+ constexpr const_iterator end() const { return nullptr; }
+ constexpr const_iterator cend() const { return nullptr; }
+
+ constexpr reverse_iterator rbegin() { return nullptr; }
+ constexpr const_reverse_iterator rbegin() const { return nullptr; }
+ constexpr const_reverse_iterator crbegin() const { return nullptr; }
+ constexpr reverse_iterator rend() { return nullptr; }
+ constexpr const_reverse_iterator rend() const { return nullptr; }
+ constexpr const_reverse_iterator crend() const { return nullptr; }
+
+ /* capacity */
+ constexpr bool empty() const { return true; }
+ constexpr size_type size() const { return 0; }
+ constexpr size_type max_size() const { return 0; }
+
+ /* lookup */
+
+ constexpr std::size_t count(Key const &) const { return 0; }
+
+ constexpr const_iterator find(Key const &) const { return end(); }
+ constexpr iterator find(Key const &) { return end(); }
+
+ constexpr std::pair<const_iterator, const_iterator>
+ equal_range(Key const &) const {
+ return {end(), end()};
+ }
+ constexpr std::pair<iterator, iterator>
+ equal_range(Key const &) {
+ return {end(), end()};
+ }
+
+ constexpr const_iterator lower_bound(Key const &) const { return end(); }
+ constexpr iterator lower_bound(Key const &) { return end(); }
+
+ constexpr const_iterator upper_bound(Key const &) const { return end(); }
+ constexpr iterator upper_bound(Key const &) { return end(); }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+};
+
+template <typename T, typename U>
+constexpr auto make_map(bits::ignored_arg = {}/* for consistency with the initializer below for N = 0*/) {
+ return map<T, U, 0>{};
+}
+
+template <typename T, typename U, std::size_t N>
+constexpr auto make_map(std::pair<T, U> const (&items)[N]) {
+ return map<T, U, N>{items};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/random.h b/contrib/frozen/include/frozen/random.h
new file mode 100644
index 0000000..ea494dc
--- /dev/null
+++ b/contrib/frozen/include/frozen/random.h
@@ -0,0 +1,90 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_RANDOM_H
+#define FROZEN_LETITGO_RANDOM_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/version.h"
+
+#include <cstdint>
+#include <type_traits>
+
+namespace frozen {
+template <class UIntType, UIntType a, UIntType c, UIntType m>
+class linear_congruential_engine {
+
+ static_assert(std::is_unsigned<UIntType>::value,
+ "UIntType must be an unsigned integral type");
+
+public:
+ using result_type = UIntType;
+ static constexpr result_type multiplier = a;
+ static constexpr result_type increment = c;
+ static constexpr result_type modulus = m;
+ static constexpr result_type default_seed = 1u;
+
+ linear_congruential_engine() = default;
+ constexpr linear_congruential_engine(result_type s) { seed(s); }
+
+ void seed(result_type s = default_seed) { state_ = s; }
+ constexpr result_type operator()() {
+ using uint_least_t = bits::select_uint_least_t<bits::log(a) + bits::log(m) + 4>;
+ uint_least_t tmp = static_cast<uint_least_t>(multiplier) * state_ + increment;
+
+ // the static cast below may end up doing a truncation
+ if(modulus != 0)
+ state_ = static_cast<result_type>(tmp % modulus);
+ else
+ state_ = static_cast<result_type>(tmp);
+ return state_;
+ }
+ constexpr void discard(unsigned long long n) {
+ while (n--)
+ operator()();
+ }
+ static constexpr result_type min() { return increment == 0u ? 1u : 0u; };
+ static constexpr result_type max() { return modulus - 1u; };
+ friend constexpr bool operator==(linear_congruential_engine const &self,
+ linear_congruential_engine const &other) {
+ return self.state_ == other.state_;
+ }
+ friend constexpr bool operator!=(linear_congruential_engine const &self,
+ linear_congruential_engine const &other) {
+ return !(self == other);
+ }
+
+private:
+ result_type state_ = default_seed;
+};
+
+using minstd_rand0 =
+ linear_congruential_engine<std::uint_fast32_t, 16807, 0, 2147483647>;
+using minstd_rand =
+ linear_congruential_engine<std::uint_fast32_t, 48271, 0, 2147483647>;
+
+// This generator is used by default in unordered frozen containers
+using default_prg_t = minstd_rand;
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/set.h b/contrib/frozen/include/frozen/set.h
new file mode 100644
index 0000000..d86d814
--- /dev/null
+++ b/contrib/frozen/include/frozen/set.h
@@ -0,0 +1,220 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_SET_H
+#define FROZEN_SET_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/version.h"
+
+#include <utility>
+
+namespace frozen {
+
+template <class Key, std::size_t N, class Compare = std::less<Key>> class set {
+ using container_type = bits::carray<Key, N>;
+ Compare less_than_;
+ container_type keys_;
+
+public:
+ /* container typedefs*/
+ using key_type = Key;
+ using value_type = Key;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::size_type;
+ using key_compare = Compare;
+ using value_compare = Compare;
+ using reference = typename container_type::const_reference;
+ using const_reference = reference;
+ using pointer = typename container_type::const_pointer;
+ using const_pointer = pointer;
+ using iterator = typename container_type::const_iterator;
+ using reverse_iterator = typename container_type::const_reverse_iterator;
+ using const_iterator = iterator;
+ using const_reverse_iterator = reverse_iterator;
+
+public:
+ /* constructors */
+ constexpr set(const set &other) = default;
+
+ constexpr set(container_type keys, Compare const & comp)
+ : less_than_{comp}
+ , keys_(bits::quicksort(keys, less_than_)) {
+ }
+
+ explicit constexpr set(container_type keys)
+ : set{keys, Compare{}} {}
+
+ constexpr set(std::initializer_list<Key> keys, Compare const & comp)
+ : set{container_type{keys}, comp} {
+ constexpr_assert(keys.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ constexpr set(std::initializer_list<Key> keys)
+ : set{keys, Compare{}} {}
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &key) const {
+ return bits::binary_search<N>(keys_.begin(), key, less_than_);
+ }
+
+ constexpr const_iterator find(Key const &key) const {
+ const_iterator where = lower_bound(key);
+ if ((where != end()) && !less_than_(key, *where))
+ return where;
+ else
+ return end();
+ }
+
+ constexpr std::pair<const_iterator, const_iterator> equal_range(Key const &key) const {
+ auto const lower = lower_bound(key);
+ if (lower == end())
+ return {lower, lower};
+ else
+ return {lower, lower + 1};
+ }
+
+ constexpr const_iterator lower_bound(Key const &key) const {
+ auto const where = bits::lower_bound<N>(keys_.begin(), key, less_than_);
+ if ((where != end()) && !less_than_(key, *where))
+ return where;
+ else
+ return end();
+ }
+
+ constexpr const_iterator upper_bound(Key const &key) const {
+ auto const where = bits::lower_bound<N>(keys_.begin(), key, less_than_);
+ if ((where != end()) && !less_than_(key, *where))
+ return where + 1;
+ else
+ return end();
+ }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+
+ /* iterators */
+ constexpr const_iterator begin() const { return keys_.begin(); }
+ constexpr const_iterator cbegin() const { return keys_.cbegin(); }
+ constexpr const_iterator end() const { return keys_.end(); }
+ constexpr const_iterator cend() const { return keys_.cend(); }
+
+ constexpr const_reverse_iterator rbegin() const { return keys_.rbegin(); }
+ constexpr const_reverse_iterator crbegin() const { return keys_.crbegin(); }
+ constexpr const_reverse_iterator rend() const { return keys_.rend(); }
+ constexpr const_reverse_iterator crend() const { return keys_.crend(); }
+
+ /* comparison */
+ constexpr bool operator==(set const& rhs) const { return bits::equal(begin(), end(), rhs.begin()); }
+ constexpr bool operator!=(set const& rhs) const { return !(*this == rhs); }
+ constexpr bool operator<(set const& rhs) const { return bits::lexicographical_compare(begin(), end(), rhs.begin(), rhs.end()); }
+ constexpr bool operator<=(set const& rhs) const { return (*this < rhs) || (*this == rhs); }
+ constexpr bool operator>(set const& rhs) const { return bits::lexicographical_compare(rhs.begin(), rhs.end(), begin(), end()); }
+ constexpr bool operator>=(set const& rhs) const { return (*this > rhs) || (*this == rhs); }
+};
+
+template <class Key, class Compare> class set<Key, 0, Compare> {
+ using container_type = bits::carray<Key, 0>; // just for the type definitions
+ Compare less_than_;
+
+public:
+ /* container typedefs*/
+ using key_type = Key;
+ using value_type = Key;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::size_type;
+ using key_compare = Compare;
+ using value_compare = Compare;
+ using reference = typename container_type::const_reference;
+ using const_reference = reference;
+ using pointer = typename container_type::const_pointer;
+ using const_pointer = pointer;
+ using iterator = pointer;
+ using reverse_iterator = pointer;
+ using const_iterator = const_pointer;
+ using const_reverse_iterator = const_pointer;
+
+public:
+ /* constructors */
+ constexpr set(const set &other) = default;
+ constexpr set(bits::carray<Key, 0>, Compare const &) {}
+ explicit constexpr set(bits::carray<Key, 0>) {}
+
+ constexpr set(std::initializer_list<Key>, Compare const &comp)
+ : less_than_{comp} {}
+ constexpr set(std::initializer_list<Key> keys) : set{keys, Compare{}} {}
+
+ /* capacity */
+ constexpr bool empty() const { return true; }
+ constexpr size_type size() const { return 0; }
+ constexpr size_type max_size() const { return 0; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &) const { return 0; }
+
+ constexpr const_iterator find(Key const &) const { return end(); }
+
+ constexpr std::pair<const_iterator, const_iterator>
+ equal_range(Key const &) const { return {end(), end()}; }
+
+ constexpr const_iterator lower_bound(Key const &) const { return end(); }
+
+ constexpr const_iterator upper_bound(Key const &) const { return end(); }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+
+ /* iterators */
+ constexpr const_iterator begin() const { return nullptr; }
+ constexpr const_iterator cbegin() const { return nullptr; }
+ constexpr const_iterator end() const { return nullptr; }
+ constexpr const_iterator cend() const { return nullptr; }
+
+ constexpr const_reverse_iterator rbegin() const { return nullptr; }
+ constexpr const_reverse_iterator crbegin() const { return nullptr; }
+ constexpr const_reverse_iterator rend() const { return nullptr; }
+ constexpr const_reverse_iterator crend() const { return nullptr; }
+};
+
+template <typename T>
+constexpr auto make_set(bits::ignored_arg = {}/* for consistency with the initializer below for N = 0*/) {
+ return set<T, 0>{};
+}
+
+template <typename T, std::size_t N>
+constexpr auto make_set(const T (&args)[N]) {
+ return set<T, N>(args);
+}
+
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/string.h b/contrib/frozen/include/frozen/string.h
new file mode 100644
index 0000000..4505bbf
--- /dev/null
+++ b/contrib/frozen/include/frozen/string.h
@@ -0,0 +1,152 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_STRING_H
+#define FROZEN_LETITGO_STRING_H
+
+#include "frozen/bits/elsa.h"
+#include "frozen/bits/version.h"
+#include "frozen/bits/defines.h"
+
+#include <functional>
+
+#ifdef FROZEN_LETITGO_HAS_STRING_VIEW
+#include <string_view>
+#endif
+
+namespace frozen {
+
+template <typename _CharT>
+class basic_string {
+ using chr_t = _CharT;
+
+ chr_t const *data_;
+ std::size_t size_;
+
+public:
+ template <std::size_t N>
+ constexpr basic_string(chr_t const (&data)[N])
+ : data_(data), size_(N - 1) {}
+ constexpr basic_string(chr_t const *data, std::size_t size)
+ : data_(data), size_(size) {}
+
+#ifdef FROZEN_LETITGO_HAS_STRING_VIEW
+ constexpr basic_string(std::basic_string_view<chr_t> data)
+ : data_(data.data()), size_(data.size()) {}
+#endif
+
+ constexpr basic_string(const basic_string &) noexcept = default;
+ constexpr basic_string &operator=(const basic_string &) noexcept = default;
+
+ constexpr std::size_t size() const { return size_; }
+
+ constexpr chr_t operator[](std::size_t i) const { return data_[i]; }
+
+ constexpr bool operator==(basic_string other) const {
+ if (size_ != other.size_)
+ return false;
+ for (std::size_t i = 0; i < size_; ++i)
+ if (data_[i] != other.data_[i])
+ return false;
+ return true;
+ }
+
+ constexpr bool operator<(const basic_string &other) const {
+ unsigned i = 0;
+ while (i < size() && i < other.size()) {
+ if ((*this)[i] < other[i]) {
+ return true;
+ }
+ if ((*this)[i] > other[i]) {
+ return false;
+ }
+ ++i;
+ }
+ return size() < other.size();
+ }
+
+ constexpr const chr_t *data() const { return data_; }
+};
+
+template <typename _CharT> struct elsa<basic_string<_CharT>> {
+ constexpr std::size_t operator()(basic_string<_CharT> value) const {
+ std::size_t d = 5381;
+ for (std::size_t i = 0; i < value.size(); ++i)
+ d = d * 33 + value[i];
+ return d;
+ }
+ // https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
+ // With the lowest bits removed, based on experimental setup.
+ constexpr std::size_t operator()(basic_string<_CharT> value, std::size_t seed) const {
+ std::size_t d = (0x811c9dc5 ^ seed) * 0x01000193;
+ for (std::size_t i = 0; i < value.size(); ++i)
+ d = (d ^ value[i]) * 0x01000193;
+ return d >> 8 ;
+ }
+};
+
+using string = basic_string<char>;
+using wstring = basic_string<wchar_t>;
+using u16string = basic_string<char16_t>;
+using u32string = basic_string<char32_t>;
+
+#ifdef FROZEN_LETITGO_HAS_CHAR8T
+using u8string = basic_string<char8_t>;
+#endif
+
+namespace string_literals {
+
+constexpr string operator"" _s(const char *data, std::size_t size) {
+ return {data, size};
+}
+
+constexpr wstring operator"" _s(const wchar_t *data, std::size_t size) {
+ return {data, size};
+}
+
+constexpr u16string operator"" _s(const char16_t *data, std::size_t size) {
+ return {data, size};
+}
+
+constexpr u32string operator"" _s(const char32_t *data, std::size_t size) {
+ return {data, size};
+}
+
+#ifdef FROZEN_LETITGO_HAS_CHAR8T
+constexpr u8string operator"" _s(const char8_t *data, std::size_t size) {
+ return {data, size};
+}
+#endif
+
+} // namespace string_literals
+
+} // namespace frozen
+
+namespace std {
+template <typename _CharT> struct hash<frozen::basic_string<_CharT>> {
+ size_t operator()(frozen::basic_string<_CharT> s) const {
+ return frozen::elsa<frozen::basic_string<_CharT>>{}(s);
+ }
+};
+} // namespace std
+
+#endif
diff --git a/contrib/frozen/include/frozen/unordered_map.h b/contrib/frozen/include/frozen/unordered_map.h
new file mode 100644
index 0000000..5e1e399
--- /dev/null
+++ b/contrib/frozen/include/frozen/unordered_map.h
@@ -0,0 +1,197 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_UNORDERED_MAP_H
+#define FROZEN_LETITGO_UNORDERED_MAP_H
+
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/elsa.h"
+#include "frozen/bits/exceptions.h"
+#include "frozen/bits/pmh.h"
+#include "frozen/bits/version.h"
+#include "frozen/random.h"
+
+#include <tuple>
+#include <functional>
+
+namespace frozen {
+
+namespace bits {
+
+struct GetKey {
+ template <class KV> constexpr auto const &operator()(KV const &kv) const {
+ return kv.first;
+ }
+};
+
+} // namespace bits
+
+template <class Key, class Value, std::size_t N, typename Hash = anna<Key>,
+ class KeyEqual = std::equal_to<Key>>
+class unordered_map {
+ static constexpr std::size_t storage_size =
+ bits::next_highest_power_of_two(N) * (N < 32 ? 2 : 1); // size adjustment to prevent high collision rate for small sets
+ using container_type = bits::carray<std::pair<Key, Value>, N>;
+ using tables_type = bits::pmh_tables<storage_size, Hash>;
+
+ KeyEqual const equal_;
+ container_type items_;
+ tables_type tables_;
+
+public:
+ /* typedefs */
+ using Self = unordered_map<Key, Value, N, Hash, KeyEqual>;
+ using key_type = Key;
+ using mapped_type = Value;
+ using value_type = typename container_type::value_type;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using reference = typename container_type::reference;
+ using const_reference = typename container_type::const_reference;
+ using pointer = typename container_type::pointer;
+ using const_pointer = typename container_type::const_pointer;
+ using iterator = typename container_type::iterator;
+ using const_iterator = typename container_type::const_iterator;
+
+public:
+ /* constructors */
+ unordered_map(unordered_map const &) = default;
+ constexpr unordered_map(container_type items,
+ Hash const &hash, KeyEqual const &equal)
+ : equal_{equal}
+ , items_{items}
+ , tables_{
+ bits::make_pmh_tables<storage_size>(
+ items_, hash, bits::GetKey{}, default_prg_t{})} {}
+ explicit constexpr unordered_map(container_type items)
+ : unordered_map{items, Hash{}, KeyEqual{}} {}
+
+ constexpr unordered_map(std::initializer_list<value_type> items,
+ Hash const & hash, KeyEqual const & equal)
+ : unordered_map{container_type{items}, hash, equal} {
+ constexpr_assert(items.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ constexpr unordered_map(std::initializer_list<value_type> items)
+ : unordered_map{items, Hash{}, KeyEqual{}} {}
+
+ /* iterators */
+ constexpr iterator begin() { return items_.begin(); }
+ constexpr iterator end() { return items_.end(); }
+ constexpr const_iterator begin() const { return items_.begin(); }
+ constexpr const_iterator end() const { return items_.end(); }
+ constexpr const_iterator cbegin() const { return items_.cbegin(); }
+ constexpr const_iterator cend() const { return items_.cend(); }
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &key) const {
+ auto const &kv = lookup(key);
+ return equal_(kv.first, key);
+ }
+
+ constexpr Value const &at(Key const &key) const {
+ return at_impl(*this, key);
+ }
+ constexpr Value &at(Key const &key) {
+ return at_impl(*this, key);
+ }
+
+ constexpr const_iterator find(Key const &key) const {
+ return find_impl(*this, key);
+ }
+ constexpr iterator find(Key const &key) {
+ return find_impl(*this, key);
+ }
+
+ constexpr std::pair<const_iterator, const_iterator> equal_range(Key const &key) const {
+ return equal_range_impl(*this, key);
+ }
+ constexpr std::pair<iterator, iterator> equal_range(Key const &key) {
+ return equal_range_impl(*this, key);
+ }
+
+ /* bucket interface */
+ constexpr std::size_t bucket_count() const { return storage_size; }
+ constexpr std::size_t max_bucket_count() const { return storage_size; }
+
+ /* observers*/
+ constexpr hasher hash_function() const { return tables_.hash_; }
+ constexpr key_equal key_eq() const { return equal_; }
+
+private:
+ template <class This>
+ static inline constexpr auto& at_impl(This&& self, Key const &key) {
+ auto& kv = self.lookup(key);
+ if (self.equal_(kv.first, key))
+ return kv.second;
+ else
+ FROZEN_THROW_OR_ABORT(std::out_of_range("unknown key"));
+ }
+
+ template <class This>
+ static inline constexpr auto find_impl(This&& self, Key const &key) {
+ auto& kv = self.lookup(key);
+ if (self.equal_(kv.first, key))
+ return &kv;
+ else
+ return self.items_.end();
+ }
+
+ template <class This>
+ static inline constexpr auto equal_range_impl(This&& self, Key const &key) {
+ auto& kv = self.lookup(key);
+ using kv_ptr = decltype(&kv);
+ if (self.equal_(kv.first, key))
+ return std::pair<kv_ptr, kv_ptr>{&kv, &kv + 1};
+ else
+ return std::pair<kv_ptr, kv_ptr>{self.items_.end(), self.items_.end()};
+ }
+
+ template <class This>
+ static inline constexpr auto& lookup_impl(This&& self, Key const &key) {
+ return self.items_[self.tables_.lookup(key)];
+ }
+
+ constexpr auto const& lookup(Key const &key) const {
+ return lookup_impl(*this, key);
+ }
+ constexpr auto& lookup(Key const &key) {
+ return lookup_impl(*this, key);
+ }
+};
+
+template <typename T, typename U, std::size_t N>
+constexpr auto make_unordered_map(std::pair<T, U> const (&items)[N]) {
+ return unordered_map<T, U, N>{items};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/unordered_set.h b/contrib/frozen/include/frozen/unordered_set.h
new file mode 100644
index 0000000..0fca292
--- /dev/null
+++ b/contrib/frozen/include/frozen/unordered_set.h
@@ -0,0 +1,147 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_UNORDERED_SET_H
+#define FROZEN_LETITGO_UNORDERED_SET_H
+
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/elsa.h"
+#include "frozen/bits/pmh.h"
+#include "frozen/bits/version.h"
+#include "frozen/random.h"
+
+#include <utility>
+
+namespace frozen {
+
+namespace bits {
+
+struct Get {
+ template <class T> constexpr T const &operator()(T const &key) const {
+ return key;
+ }
+};
+
+} // namespace bits
+
+template <class Key, std::size_t N, typename Hash = elsa<Key>,
+ class KeyEqual = std::equal_to<Key>>
+class unordered_set {
+ static constexpr std::size_t storage_size =
+ bits::next_highest_power_of_two(N) * (N < 32 ? 2 : 1); // size adjustment to prevent high collision rate for small sets
+ using container_type = bits::carray<Key, N>;
+ using tables_type = bits::pmh_tables<storage_size, Hash>;
+
+ KeyEqual const equal_;
+ container_type keys_;
+ tables_type tables_;
+
+public:
+ /* typedefs */
+ using key_type = Key;
+ using value_type = Key;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using const_reference = typename container_type::const_reference;
+ using reference = const_reference;
+ using const_pointer = typename container_type::const_pointer;
+ using pointer = const_pointer;
+ using const_iterator = const_pointer;
+ using iterator = const_iterator;
+
+public:
+ /* constructors */
+ unordered_set(unordered_set const &) = default;
+ constexpr unordered_set(container_type keys, Hash const &hash,
+ KeyEqual const &equal)
+ : equal_{equal}
+ , keys_{keys}
+ , tables_{bits::make_pmh_tables<storage_size>(
+ keys_, hash, bits::Get{}, default_prg_t{})} {}
+ explicit constexpr unordered_set(container_type keys)
+ : unordered_set{keys, Hash{}, KeyEqual{}} {}
+
+ constexpr unordered_set(std::initializer_list<Key> keys)
+ : unordered_set{keys, Hash{}, KeyEqual{}} {}
+
+ constexpr unordered_set(std::initializer_list<Key> keys, Hash const & hash, KeyEqual const & equal)
+ : unordered_set{container_type{keys}, hash, equal} {
+ constexpr_assert(keys.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ /* iterators */
+ constexpr const_iterator begin() const { return keys_.begin(); }
+ constexpr const_iterator end() const { return keys_.end(); }
+ constexpr const_iterator cbegin() const { return keys_.cbegin(); }
+ constexpr const_iterator cend() const { return keys_.cend(); }
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &key) const {
+ auto const k = lookup(key);
+ return equal_(k, key);
+ }
+ constexpr const_iterator find(Key const &key) const {
+ auto const &k = lookup(key);
+ if (equal_(k, key))
+ return &k;
+ else
+ return keys_.end();
+ }
+
+ constexpr std::pair<const_iterator, const_iterator> equal_range(Key const &key) const {
+ auto const &k = lookup(key);
+ if (equal_(k, key))
+ return {&k, &k + 1};
+ else
+ return {keys_.end(), keys_.end()};
+ }
+
+ /* bucket interface */
+ constexpr std::size_t bucket_count() const { return storage_size; }
+ constexpr std::size_t max_bucket_count() const { return storage_size; }
+
+ /* observers*/
+ constexpr hasher hash_function() const { return tables_.hash_; }
+ constexpr key_equal key_eq() const { return equal_; }
+
+private:
+ constexpr auto const &lookup(Key const &key) const {
+ return keys_[tables_.lookup(key)];
+ }
+};
+
+template <typename T, std::size_t N>
+constexpr auto make_unordered_set(T const (&keys)[N]) {
+ return unordered_set<T, N>{keys};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/fu2/LICENSE.txt b/contrib/fu2/LICENSE.txt
new file mode 100644
index 0000000..36b7cd9
--- /dev/null
+++ b/contrib/fu2/LICENSE.txt
@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/contrib/fu2/include/function2/function2.hpp b/contrib/fu2/include/function2/function2.hpp
new file mode 100644
index 0000000..a45cb58
--- /dev/null
+++ b/contrib/fu2/include/function2/function2.hpp
@@ -0,0 +1,1792 @@
+
+// Copyright 2015-2020 Denis Blank <denis.blank at outlook dot com>
+// Distributed under the Boost Software License, Version 1.0
+// (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef FU2_INCLUDED_FUNCTION2_HPP_
+#define FU2_INCLUDED_FUNCTION2_HPP_
+
+#include <cassert>
+#include <cstddef>
+#include <cstdlib>
+#include <memory>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+// Defines:
+// - FU2_HAS_DISABLED_EXCEPTIONS
+#if defined(FU2_WITH_DISABLED_EXCEPTIONS) || \
+ defined(FU2_MACRO_DISABLE_EXCEPTIONS)
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#else // FU2_WITH_DISABLED_EXCEPTIONS
+#if defined(_MSC_VER)
+#if !defined(_HAS_EXCEPTIONS) || (_HAS_EXCEPTIONS == 0)
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#endif
+#elif defined(__clang__)
+#if !(__EXCEPTIONS && __has_feature(cxx_exceptions))
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#endif
+#elif defined(__GNUC__)
+#if !__EXCEPTIONS
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#endif
+#endif
+#endif // FU2_WITH_DISABLED_EXCEPTIONS
+// - FU2_HAS_NO_FUNCTIONAL_HEADER
+#if !defined(FU2_WITH_NO_FUNCTIONAL_HEADER) && \
+ !defined(FU2_NO_FUNCTIONAL_HEADER) && \
+ !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+#include <functional>
+#else
+#define FU2_HAS_NO_FUNCTIONAL_HEADER
+#endif
+// - FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#if defined(FU2_WITH_CXX17_NOEXCEPT_FUNCTION_TYPE)
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#else // FU2_WITH_CXX17_NOEXCEPT_FUNCTION_TYPE
+#if defined(_MSC_VER)
+#if defined(_HAS_CXX17) && _HAS_CXX17
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#endif
+#elif defined(__cpp_noexcept_function_type)
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#elif defined(__cplusplus) && (__cplusplus >= 201703L)
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#endif
+#endif // FU2_WITH_CXX17_NOEXCEPT_FUNCTION_TYPE
+
+// - FU2_HAS_NO_EMPTY_PROPAGATION
+#if defined(FU2_WITH_NO_EMPTY_PROPAGATION)
+#define FU2_HAS_NO_EMPTY_PROPAGATION
+#endif // FU2_WITH_NO_EMPTY_PROPAGATION
+
+#if !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+#include <exception>
+#endif
+
+#if defined(__cpp_constexpr) && (__cpp_constexpr >= 201304)
+#define FU2_DETAIL_CXX14_CONSTEXPR constexpr
+#elif defined(__clang__) && defined(__has_feature)
+#if __has_feature(__cxx_generic_lambdas__) && \
+ __has_feature(__cxx_relaxed_constexpr__)
+#define FU2_DETAIL_CXX14_CONSTEXPR constexpr
+#endif
+#elif defined(_MSC_VER) && (_MSC_VER >= 1915) && (_MSVC_LANG >= 201402)
+#define FU2_DETAIL_CXX14_CONSTEXPR constexpr
+#endif
+#ifndef FU2_DETAIL_CXX14_CONSTEXPR
+#define FU2_DETAIL_CXX14_CONSTEXPR
+#endif
+
+/// Hint for the compiler that this point should be unreachable
+#if defined(_MSC_VER)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() __assume(false)
+#elif defined(__GNUC__)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() __builtin_unreachable()
+#elif defined(__has_builtin)
+#if __has_builtin(__builtin_unreachable)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() __builtin_unreachable()
+#endif
+#endif
+#ifndef FU2_DETAIL_UNREACHABLE_INTRINSIC
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() abort()
+#endif
+
+/// Causes the application to exit abnormally
+#if defined(_MSC_VER)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() __debugbreak()
+#elif defined(__GNUC__)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() __builtin_trap()
+#elif defined(__has_builtin)
+#if __has_builtin(__builtin_trap)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() __builtin_trap()
+#endif
+#endif
+#ifndef FU2_DETAIL_TRAP
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() *(volatile int*)0x11 = 0
+#endif
+
+#ifndef NDEBUG
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE() ::fu2::detail::unreachable_debug()
+#else
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE() FU2_DETAIL_UNREACHABLE_INTRINSIC()
+#endif
+
+namespace fu2 {
+inline namespace abi_400 {
+namespace detail {
+template <typename Config, typename Property>
+class function;
+
+template <typename...>
+struct identity {};
+
+// Equivalent to C++17's std::void_t which targets a bug in GCC,
+// that prevents correct SFINAE behavior.
+// See http://stackoverflow.com/questions/35753920 for details.
+template <typename...>
+struct deduce_to_void : std::common_type<void> {};
+
+template <typename... T>
+using void_t = typename deduce_to_void<T...>::type;
+
+template <typename T>
+using unrefcv_t = std::remove_cv_t<std::remove_reference_t<T>>;
+
+// Copy enabler helper class
+template <bool /*Copyable*/>
+struct copyable {};
+template <>
+struct copyable<false> {
+ copyable() = default;
+ ~copyable() = default;
+ copyable(copyable const&) = delete;
+ copyable(copyable&&) = default;
+ copyable& operator=(copyable const&) = delete;
+ copyable& operator=(copyable&&) = default;
+};
+
+/// Configuration trait to configure the function_base class.
+template <bool Owning, bool Copyable, typename Capacity>
+struct config {
+ // Is true if the function is owning.
+ static constexpr auto const is_owning = Owning;
+
+ // Is true if the function is copyable.
+ static constexpr auto const is_copyable = Copyable;
+
+ // The internal capacity of the function
+ // used in small functor optimization.
+ // The object shall expose the real capacity through Capacity::capacity
+ // and the intended alignment through Capacity::alignment.
+ using capacity = Capacity;
+};
+
+/// A config which isn't compatible to other configs
+template <bool Throws, bool HasStrongExceptGuarantee, typename... Args>
+struct property {
+ // Is true when the function throws an exception on empty invocation.
+ static constexpr auto const is_throwing = Throws;
+
+ // Is true when the function throws an exception on empty invocation.
+ static constexpr auto const is_strong_exception_guaranteed =
+ HasStrongExceptGuarantee;
+};
+
+#ifndef NDEBUG
+[[noreturn]] inline void unreachable_debug() {
+ FU2_DETAIL_TRAP();
+ std::abort();
+}
+#endif
+
+/// Provides utilities for invocing callable objects
+namespace invocation {
+/// Invokes the given callable object with the given arguments
+template <typename Callable, typename... Args>
+constexpr auto invoke(Callable&& callable, Args&&... args) noexcept(
+ noexcept(std::forward<Callable>(callable)(std::forward<Args>(args)...)))
+ -> decltype(std::forward<Callable>(callable)(std::forward<Args>(args)...)) {
+
+ return std::forward<Callable>(callable)(std::forward<Args>(args)...);
+}
+/// Invokes the given member function pointer by reference
+template <typename T, typename Type, typename Self, typename... Args>
+constexpr auto invoke(Type T::*member, Self&& self, Args&&... args) noexcept(
+ noexcept((std::forward<Self>(self).*member)(std::forward<Args>(args)...)))
+ -> decltype((std::forward<Self>(self).*
+ member)(std::forward<Args>(args)...)) {
+ return (std::forward<Self>(self).*member)(std::forward<Args>(args)...);
+}
+/// Invokes the given member function pointer by pointer
+template <typename T, typename Type, typename Self, typename... Args>
+constexpr auto invoke(Type T::*member, Self&& self, Args&&... args) noexcept(
+ noexcept((std::forward<Self>(self)->*member)(std::forward<Args>(args)...)))
+ -> decltype(
+ (std::forward<Self>(self)->*member)(std::forward<Args>(args)...)) {
+ return (std::forward<Self>(self)->*member)(std::forward<Args>(args)...);
+}
+/// Invokes the given pointer to a scalar member by reference
+template <typename T, typename Type, typename Self>
+constexpr auto
+invoke(Type T::*member,
+ Self&& self) noexcept(noexcept(std::forward<Self>(self).*member))
+ -> decltype(std::forward<Self>(self).*member) {
+ return (std::forward<Self>(self).*member);
+}
+/// Invokes the given pointer to a scalar member by pointer
+template <typename T, typename Type, typename Self>
+constexpr auto
+invoke(Type T::*member,
+ Self&& self) noexcept(noexcept(std::forward<Self>(self)->*member))
+ -> decltype(std::forward<Self>(self)->*member) {
+ return std::forward<Self>(self)->*member;
+}
+
+/// Deduces to a true type if the callable object can be invoked with
+/// the given arguments.
+/// We don't use invoke here because MSVC can't evaluate the nested expression
+/// SFINAE here.
+template <typename T, typename Args, typename = void>
+struct can_invoke : std::false_type {};
+template <typename T, typename... Args>
+struct can_invoke<T, identity<Args...>,
+ decltype((void)std::declval<T>()(std::declval<Args>()...))>
+ : std::true_type {};
+template <typename Pointer, typename T, typename... Args>
+struct can_invoke<Pointer, identity<T&, Args...>,
+ decltype((void)((std::declval<T&>().*std::declval<Pointer>())(
+ std::declval<Args>()...)))> : std::true_type {};
+template <typename Pointer, typename T, typename... Args>
+struct can_invoke<Pointer, identity<T&&, Args...>,
+ decltype(
+ (void)((std::declval<T&&>().*std::declval<Pointer>())(
+ std::declval<Args>()...)))> : std::true_type {};
+template <typename Pointer, typename T, typename... Args>
+struct can_invoke<Pointer, identity<T*, Args...>,
+ decltype(
+ (void)((std::declval<T*>()->*std::declval<Pointer>())(
+ std::declval<Args>()...)))> : std::true_type {};
+template <typename Pointer, typename T>
+struct can_invoke<Pointer, identity<T&>,
+ decltype((void)(std::declval<T&>().*std::declval<Pointer>()))>
+ : std::true_type {};
+template <typename Pointer, typename T>
+struct can_invoke<Pointer, identity<T&&>,
+ decltype(
+ (void)(std::declval<T&&>().*std::declval<Pointer>()))>
+ : std::true_type {};
+template <typename Pointer, typename T>
+struct can_invoke<Pointer, identity<T*>,
+ decltype(
+ (void)(std::declval<T*>()->*std::declval<Pointer>()))>
+ : std::true_type {};
+
+template <bool RequiresNoexcept, typename T, typename Args>
+struct is_noexcept_correct : std::true_type {};
+template <typename T, typename... Args>
+struct is_noexcept_correct<true, T, identity<Args...>>
+ : std::integral_constant<bool,
+ noexcept(::fu2::detail::invocation::invoke(
+ std::declval<T>(), std::declval<Args>()...))> {
+};
+} // end namespace invocation
+
+namespace overloading {
+template <typename... Args>
+struct overload_impl;
+template <typename Current, typename Next, typename... Rest>
+struct overload_impl<Current, Next, Rest...> : Current,
+ overload_impl<Next, Rest...> {
+ explicit overload_impl(Current current, Next next, Rest... rest)
+ : Current(std::move(current)), overload_impl<Next, Rest...>(
+ std::move(next), std::move(rest)...) {
+ }
+
+ using Current::operator();
+ using overload_impl<Next, Rest...>::operator();
+};
+template <typename Current>
+struct overload_impl<Current> : Current {
+ explicit overload_impl(Current current) : Current(std::move(current)) {
+ }
+
+ using Current::operator();
+};
+
+template <typename... T>
+constexpr auto overload(T&&... callables) {
+ return overload_impl<std::decay_t<T>...>{std::forward<T>(callables)...};
+}
+} // namespace overloading
+
+/// Declares the namespace which provides the functionality to work with a
+/// type-erased object.
+namespace type_erasure {
+/// Specialization to work with addresses of callable objects
+template <typename T, typename = void>
+struct address_taker {
+ template <typename O>
+ static void* take(O&& obj) {
+ return std::addressof(obj);
+ }
+ static T& restore(void* ptr) {
+ return *static_cast<T*>(ptr);
+ }
+ static T const& restore(void const* ptr) {
+ return *static_cast<T const*>(ptr);
+ }
+ static T volatile& restore(void volatile* ptr) {
+ return *static_cast<T volatile*>(ptr);
+ }
+ static T const volatile& restore(void const volatile* ptr) {
+ return *static_cast<T const volatile*>(ptr);
+ }
+};
+/// Specialization to work with addresses of raw function pointers
+template <typename T>
+struct address_taker<T, std::enable_if_t<std::is_pointer<T>::value>> {
+ template <typename O>
+ static void* take(O&& obj) {
+ return reinterpret_cast<void*>(obj);
+ }
+ template <typename O>
+ static T restore(O ptr) {
+ return reinterpret_cast<T>(const_cast<void*>(ptr));
+ }
+};
+
+template <typename Box>
+struct box_factory;
+/// Store the allocator inside the box
+template <bool IsCopyable, typename T, typename Allocator>
+struct box : private Allocator {
+ friend box_factory<box>;
+
+ T value_;
+
+ explicit box(T value, Allocator allocator)
+ : Allocator(std::move(allocator)), value_(std::move(value)) {
+ }
+
+ box(box&&) = default;
+ box(box const&) = default;
+ box& operator=(box&&) = default;
+ box& operator=(box const&) = default;
+ ~box() = default;
+};
+template <typename T, typename Allocator>
+struct box<false, T, Allocator> : private Allocator {
+ friend box_factory<box>;
+
+ T value_;
+
+ explicit box(T value, Allocator allocator)
+ : Allocator(std::move(allocator)), value_(std::move(value)) {
+ }
+
+ box(box&&) = default;
+ box(box const&) = delete;
+ box& operator=(box&&) = default;
+ box& operator=(box const&) = delete;
+ ~box() = default;
+};
+
+template <bool IsCopyable, typename T, typename Allocator>
+struct box_factory<box<IsCopyable, T, Allocator>> {
+ using real_allocator =
+ typename std::allocator_traits<std::decay_t<Allocator>>::
+ template rebind_alloc<box<IsCopyable, T, Allocator>>;
+
+ /// Allocates space through the boxed allocator
+ static box<IsCopyable, T, Allocator>*
+ box_allocate(box<IsCopyable, T, Allocator> const* me) {
+ real_allocator allocator(*static_cast<Allocator const*>(me));
+
+ return static_cast<box<IsCopyable, T, Allocator>*>(
+ std::allocator_traits<real_allocator>::allocate(allocator, 1U));
+ }
+
+ /// Destroys the box through the given allocator
+ static void box_deallocate(box<IsCopyable, T, Allocator>* me) {
+ real_allocator allocator(*static_cast<Allocator const*>(me));
+
+ me->~box();
+ std::allocator_traits<real_allocator>::deallocate(allocator, me, 1U);
+ }
+};
+
+/// Creates a box containing the given value and allocator
+template <bool IsCopyable, typename T, typename Allocator>
+auto make_box(std::integral_constant<bool, IsCopyable>, T&& value,
+ Allocator&& allocator) {
+ return box<IsCopyable, std::decay_t<T>, std::decay_t<Allocator>>(
+ std::forward<T>(value), std::forward<Allocator>(allocator));
+}
+
+template <typename T>
+struct is_box : std::false_type {};
+template <bool IsCopyable, typename T, typename Allocator>
+struct is_box<box<IsCopyable, T, Allocator>> : std::true_type {};
+
+/// Provides access to the pointer to a heal allocated erased object
+/// as well to the inplace storage.
+union data_accessor {
+ data_accessor() = default;
+ explicit constexpr data_accessor(std::nullptr_t) noexcept : ptr_(nullptr) {
+ }
+ explicit constexpr data_accessor(void* ptr) noexcept : ptr_(ptr) {
+ }
+
+ /// The pointer we use if the object is on the heap
+ void* ptr_;
+ /// The first field of the inplace storage
+ std::size_t inplace_storage_;
+};
+
+/// See opcode::op_fetch_empty
+static FU2_DETAIL_CXX14_CONSTEXPR void write_empty(data_accessor* accessor,
+ bool empty) noexcept {
+ accessor->inplace_storage_ = std::size_t(empty);
+}
+
+template <typename From, typename To>
+using transfer_const_t =
+ std::conditional_t<std::is_const<std::remove_pointer_t<From>>::value,
+ std::add_const_t<To>, To>;
+template <typename From, typename To>
+using transfer_volatile_t =
+ std::conditional_t<std::is_volatile<std::remove_pointer_t<From>>::value,
+ std::add_volatile_t<To>, To>;
+
+/// The retriever when the object is allocated inplace
+template <typename T, typename Accessor>
+FU2_DETAIL_CXX14_CONSTEXPR auto retrieve(std::true_type /*is_inplace*/,
+ Accessor from,
+ std::size_t from_capacity) {
+ using type = transfer_const_t<Accessor, transfer_volatile_t<Accessor, void>>*;
+
+ /// Process the command by using the data inside the internal capacity
+ auto storage = &(from->inplace_storage_);
+ auto inplace = const_cast<void*>(static_cast<type>(storage));
+ return type(std::align(alignof(T), sizeof(T), inplace, from_capacity));
+}
+
+/// The retriever which is used when the object is allocated
+/// through the allocator
+template <typename T, typename Accessor>
+constexpr auto retrieve(std::false_type /*is_inplace*/, Accessor from,
+ std::size_t /*from_capacity*/) {
+
+ return from->ptr_;
+}
+
+namespace invocation_table {
+#if !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+#if defined(FU2_HAS_NO_FUNCTIONAL_HEADER)
+struct bad_function_call : std::exception {
+ bad_function_call() noexcept {
+ }
+
+ char const* what() const noexcept override {
+ return "bad function call";
+ }
+};
+#else
+using std::bad_function_call;
+#endif
+#endif
+
+#ifdef FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#define FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT(F) \
+ F(, , noexcept, , &) \
+ F(const, , noexcept, , &) \
+ F(, volatile, noexcept, , &) \
+ F(const, volatile, noexcept, , &) \
+ F(, , noexcept, &, &) \
+ F(const, , noexcept, &, &) \
+ F(, volatile, noexcept, &, &) \
+ F(const, volatile, noexcept, &, &) \
+ F(, , noexcept, &&, &&) \
+ F(const, , noexcept, &&, &&) \
+ F(, volatile, noexcept, &&, &&) \
+ F(const, volatile, noexcept, &&, &&)
+#define FU2_DETAIL_EXPAND_CV_NOEXCEPT(F) \
+ F(, , noexcept) \
+ F(const, , noexcept) \
+ F(, volatile, noexcept) \
+ F(const, volatile, noexcept)
+#else // FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#define FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT(F)
+#define FU2_DETAIL_EXPAND_CV_NOEXCEPT(F)
+#endif // FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+
+#define FU2_DETAIL_EXPAND_QUALIFIERS(F) \
+ F(, , , , &) \
+ F(const, , , , &) \
+ F(, volatile, , , &) \
+ F(const, volatile, , , &) \
+ F(, , , &, &) \
+ F(const, , , &, &) \
+ F(, volatile, , &, &) \
+ F(const, volatile, , &, &) \
+ F(, , , &&, &&) \
+ F(const, , , &&, &&) \
+ F(, volatile, , &&, &&) \
+ F(const, volatile, , &&, &&) \
+ FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT(F)
+#define FU2_DETAIL_EXPAND_CV(F) \
+ F(, , ) \
+ F(const, , ) \
+ F(, volatile, ) \
+ F(const, volatile, ) \
+ FU2_DETAIL_EXPAND_CV_NOEXCEPT(F)
+
+/// If the function is qualified as noexcept, the call will never throw
+template <bool IsNoexcept>
+[[noreturn]] void throw_or_abortnoexcept(
+ std::integral_constant<bool, IsNoexcept> /*is_throwing*/) noexcept {
+ std::abort();
+}
+/// Calls std::abort on empty function calls
+[[noreturn]] inline void
+throw_or_abort(std::false_type /*is_throwing*/) noexcept {
+ std::abort();
+}
+/// Throws bad_function_call on empty funciton calls
+[[noreturn]] inline void throw_or_abort(std::true_type /*is_throwing*/) {
+#ifdef FU2_HAS_DISABLED_EXCEPTIONS
+ throw_or_abort(std::false_type{});
+#else
+ throw bad_function_call{};
+#endif
+}
+
+template <typename T>
+struct function_trait;
+
+using is_noexcept_ = std::false_type;
+using is_noexcept_noexcept = std::true_type;
+
+#define FU2_DEFINE_FUNCTION_TRAIT(CONST, VOLATILE, NOEXCEPT, OVL_REF, REF) \
+ template <typename Ret, typename... Args> \
+ struct function_trait<Ret(Args...) CONST VOLATILE OVL_REF NOEXCEPT> { \
+ using pointer_type = Ret (*)(data_accessor CONST VOLATILE*, \
+ std::size_t capacity, Args...); \
+ template <typename T, bool IsInplace> \
+ struct internal_invoker { \
+ static Ret invoke(data_accessor CONST VOLATILE* data, \
+ std::size_t capacity, Args... args) NOEXCEPT { \
+ auto obj = retrieve<T>(std::integral_constant<bool, IsInplace>{}, \
+ data, capacity); \
+ auto box = static_cast<T CONST VOLATILE*>(obj); \
+ return invocation::invoke( \
+ static_cast<std::decay_t<decltype(box->value_)> CONST VOLATILE \
+ REF>(box->value_), \
+ std::forward<Args>(args)...); \
+ } \
+ }; \
+ \
+ template <typename T> \
+ struct view_invoker { \
+ static Ret invoke(data_accessor CONST VOLATILE* data, std::size_t, \
+ Args... args) NOEXCEPT { \
+ \
+ auto ptr = static_cast<void CONST VOLATILE*>(data->ptr_); \
+ return invocation::invoke(address_taker<T>::restore(ptr), \
+ std::forward<Args>(args)...); \
+ } \
+ }; \
+ \
+ template <typename T> \
+ using callable = T CONST VOLATILE REF; \
+ \
+ using arguments = identity<Args...>; \
+ \
+ using is_noexcept = is_noexcept_##NOEXCEPT; \
+ \
+ template <bool Throws> \
+ struct empty_invoker { \
+ static Ret invoke(data_accessor CONST VOLATILE* /*data*/, \
+ std::size_t /*capacity*/, Args... /*args*/) NOEXCEPT { \
+ throw_or_abort##NOEXCEPT(std::integral_constant<bool, Throws>{}); \
+ } \
+ }; \
+ };
+
+FU2_DETAIL_EXPAND_QUALIFIERS(FU2_DEFINE_FUNCTION_TRAIT)
+#undef FU2_DEFINE_FUNCTION_TRAIT
+
+/// Deduces to the function pointer to the given signature
+template <typename Signature>
+using function_pointer_of = typename function_trait<Signature>::pointer_type;
+
+template <typename... Args>
+struct invoke_table;
+
+/// We optimize the vtable_t in case there is a single function overload
+template <typename First>
+struct invoke_table<First> {
+ using type = function_pointer_of<First>;
+
+ /// Return the function pointer itself
+ template <std::size_t Index>
+ static constexpr auto fetch(type pointer) noexcept {
+ static_assert(Index == 0U, "The index should be 0 here!");
+ return pointer;
+ }
+
+ /// Returns the thunk of an single overloaded callable
+ template <typename T, bool IsInplace>
+ static constexpr type get_invocation_table_of() noexcept {
+ return &function_trait<First>::template internal_invoker<T,
+ IsInplace>::invoke;
+ }
+ /// Returns the thunk of an single overloaded callable
+ template <typename T>
+ static constexpr type get_invocation_view_table_of() noexcept {
+ return &function_trait<First>::template view_invoker<T>::invoke;
+ }
+ /// Returns the thunk of an empty single overloaded callable
+ template <bool IsThrowing>
+ static constexpr type get_empty_invocation_table() noexcept {
+ return &function_trait<First>::template empty_invoker<IsThrowing>::invoke;
+ }
+};
+/// We generate a table in case of multiple function overloads
+template <typename First, typename Second, typename... Args>
+struct invoke_table<First, Second, Args...> {
+ using type =
+ std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...> const*;
+
+ /// Return the function pointer at the particular index
+ template <std::size_t Index>
+ static constexpr auto fetch(type table) noexcept {
+ return std::get<Index>(*table);
+ }
+
+ /// The invocation vtable for a present object
+ template <typename T, bool IsInplace>
+ struct invocation_vtable : public std::tuple<function_pointer_of<First>,
+ function_pointer_of<Second>,
+ function_pointer_of<Args>...> {
+ constexpr invocation_vtable() noexcept
+ : std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...>(std::make_tuple(
+ &function_trait<First>::template internal_invoker<
+ T, IsInplace>::invoke,
+ &function_trait<Second>::template internal_invoker<
+ T, IsInplace>::invoke,
+ &function_trait<Args>::template internal_invoker<
+ T, IsInplace>::invoke...)) {
+ }
+ };
+
+ /// Returns the thunk of an multi overloaded callable
+ template <typename T, bool IsInplace>
+ static type get_invocation_table_of() noexcept {
+ static invocation_vtable<T, IsInplace> const table;
+ return &table;
+ }
+
+ /// The invocation vtable for a present object
+ template <typename T>
+ struct invocation_view_vtable
+ : public std::tuple<function_pointer_of<First>,
+ function_pointer_of<Second>,
+ function_pointer_of<Args>...> {
+ constexpr invocation_view_vtable() noexcept
+ : std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...>(std::make_tuple(
+ &function_trait<First>::template view_invoker<T>::invoke,
+ &function_trait<Second>::template view_invoker<T>::invoke,
+ &function_trait<Args>::template view_invoker<T>::invoke...)) {
+ }
+ };
+
+ /// Returns the thunk of an multi overloaded callable
+ template <typename T>
+ static type get_invocation_view_table_of() noexcept {
+ static invocation_view_vtable<T> const table;
+ return &table;
+ }
+
+ /// The invocation table for an empty wrapper
+ template <bool IsThrowing>
+ struct empty_vtable : public std::tuple<function_pointer_of<First>,
+ function_pointer_of<Second>,
+ function_pointer_of<Args>...> {
+ constexpr empty_vtable() noexcept
+ : std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...>(
+ std::make_tuple(&function_trait<First>::template empty_invoker<
+ IsThrowing>::invoke,
+ &function_trait<Second>::template empty_invoker<
+ IsThrowing>::invoke,
+ &function_trait<Args>::template empty_invoker<
+ IsThrowing>::invoke...)) {
+ }
+ };
+
+ /// Returns the thunk of an multi single overloaded callable
+ template <bool IsThrowing>
+ static type get_empty_invocation_table() noexcept {
+ static empty_vtable<IsThrowing> const table;
+ return &table;
+ }
+};
+
+template <std::size_t Index, typename Function, typename... Signatures>
+class operator_impl;
+
+#define FU2_DEFINE_FUNCTION_TRAIT(CONST, VOLATILE, NOEXCEPT, OVL_REF, REF) \
+ template <std::size_t Index, typename Function, typename Ret, \
+ typename... Args, typename Next, typename... Signatures> \
+ class operator_impl<Index, Function, \
+ Ret(Args...) CONST VOLATILE OVL_REF NOEXCEPT, Next, \
+ Signatures...> \
+ : operator_impl<Index + 1, Function, Next, Signatures...> { \
+ \
+ template <std::size_t, typename, typename...> \
+ friend class operator_impl; \
+ \
+ protected: \
+ operator_impl() = default; \
+ ~operator_impl() = default; \
+ operator_impl(operator_impl const&) = default; \
+ operator_impl(operator_impl&&) = default; \
+ operator_impl& operator=(operator_impl const&) = default; \
+ operator_impl& operator=(operator_impl&&) = default; \
+ \
+ using operator_impl<Index + 1, Function, Next, Signatures...>::operator(); \
+ \
+ Ret operator()(Args... args) CONST VOLATILE OVL_REF NOEXCEPT { \
+ auto parent = static_cast<Function CONST VOLATILE*>(this); \
+ using erasure_t = std::decay_t<decltype(parent->erasure_)>; \
+ \
+ /* `std::decay_t<decltype(parent->erasure_)>` is a workaround for a */ \
+ /* compiler regression of MSVC 16.3.1, see #29 for details. */ \
+ return std::decay_t<decltype(parent->erasure_)>::template invoke<Index>( \
+ static_cast<erasure_t CONST VOLATILE REF>(parent->erasure_), \
+ std::forward<Args>(args)...); \
+ } \
+ }; \
+ template <std::size_t Index, typename Config, typename Property, \
+ typename Ret, typename... Args> \
+ class operator_impl<Index, function<Config, Property>, \
+ Ret(Args...) CONST VOLATILE OVL_REF NOEXCEPT> \
+ : copyable<!Config::is_owning || Config::is_copyable> { \
+ \
+ template <std::size_t, typename, typename...> \
+ friend class operator_impl; \
+ \
+ protected: \
+ operator_impl() = default; \
+ ~operator_impl() = default; \
+ operator_impl(operator_impl const&) = default; \
+ operator_impl(operator_impl&&) = default; \
+ operator_impl& operator=(operator_impl const&) = default; \
+ operator_impl& operator=(operator_impl&&) = default; \
+ \
+ Ret operator()(Args... args) CONST VOLATILE OVL_REF NOEXCEPT { \
+ auto parent = \
+ static_cast<function<Config, Property> CONST VOLATILE*>(this); \
+ using erasure_t = std::decay_t<decltype(parent->erasure_)>; \
+ \
+ /* `std::decay_t<decltype(parent->erasure_)>` is a workaround for a */ \
+ /* compiler regression of MSVC 16.3.1, see #29 for details. */ \
+ return std::decay_t<decltype(parent->erasure_)>::template invoke<Index>( \
+ static_cast<erasure_t CONST VOLATILE REF>(parent->erasure_), \
+ std::forward<Args>(args)...); \
+ } \
+ };
+
+FU2_DETAIL_EXPAND_QUALIFIERS(FU2_DEFINE_FUNCTION_TRAIT)
+#undef FU2_DEFINE_FUNCTION_TRAIT
+} // namespace invocation_table
+
+namespace tables {
+/// Identifies the action which is dispatched on the erased object
+enum class opcode {
+ op_move, ///< Move the object and set the vtable
+ op_copy, ///< Copy the object and set the vtable
+ op_destroy, ///< Destroy the object and reset the vtable
+ op_weak_destroy, ///< Destroy the object without resetting the vtable
+ op_fetch_empty, ///< Stores true or false into the to storage
+ ///< to indicate emptiness
+};
+
+/// Abstraction for a vtable together with a command table
+/// TODO Add optimization for a single formal argument
+/// TODO Add optimization to merge both tables if the function is size
+/// optimized
+template <typename Property>
+class vtable;
+template <bool IsThrowing, bool HasStrongExceptGuarantee,
+ typename... FormalArgs>
+class vtable<property<IsThrowing, HasStrongExceptGuarantee, FormalArgs...>> {
+ using command_function_t = void (*)(vtable* /*this*/, opcode /*op*/,
+ data_accessor* /*from*/,
+ std::size_t /*from_capacity*/,
+ data_accessor* /*to*/,
+ std::size_t /*to_capacity*/);
+
+ using invoke_table_t = invocation_table::invoke_table<FormalArgs...>;
+
+ command_function_t cmd_;
+ typename invoke_table_t::type vtable_;
+
+ template <typename T>
+ struct trait {
+ static_assert(is_box<T>::value,
+ "The trait must be specialized with a box!");
+
+ /// The command table
+ template <bool IsInplace>
+ static void process_cmd(vtable* to_table, opcode op, data_accessor* from,
+ std::size_t from_capacity, data_accessor* to,
+ std::size_t to_capacity) {
+
+ switch (op) {
+ case opcode::op_move: {
+ /// Retrieve the pointer to the object
+ auto box = static_cast<T*>(retrieve<T>(
+ std::integral_constant<bool, IsInplace>{}, from, from_capacity));
+ assert(box && "The object must not be over aligned or null!");
+
+ if (!IsInplace) {
+ // Just swap both pointers if we allocated on the heap
+ to->ptr_ = from->ptr_;
+
+#ifndef NDEBUG
+ // We don't need to null the pointer since we know that
+ // we don't own the data anymore through the vtable
+ // which is set to empty.
+ from->ptr_ = nullptr;
+#endif
+
+ to_table->template set_allocated<T>();
+
+ }
+ // The object is allocated inplace
+ else {
+ construct(std::true_type{}, std::move(*box), to_table, to,
+ to_capacity);
+ box->~T();
+ }
+ return;
+ }
+ case opcode::op_copy: {
+ auto box = static_cast<T const*>(retrieve<T>(
+ std::integral_constant<bool, IsInplace>{}, from, from_capacity));
+ assert(box && "The object must not be over aligned or null!");
+
+ assert(std::is_copy_constructible<T>::value &&
+ "The box is required to be copyable here!");
+
+ // Try to allocate the object inplace
+ construct(std::is_copy_constructible<T>{}, *box, to_table, to,
+ to_capacity);
+ return;
+ }
+ case opcode::op_destroy:
+ case opcode::op_weak_destroy: {
+
+ assert(!to && !to_capacity && "Arg overflow!");
+ auto box = static_cast<T*>(retrieve<T>(
+ std::integral_constant<bool, IsInplace>{}, from, from_capacity));
+
+ if (IsInplace) {
+ box->~T();
+ } else {
+ box_factory<T>::box_deallocate(box);
+ }
+
+ if (op == opcode::op_destroy) {
+ to_table->set_empty();
+ }
+ return;
+ }
+ case opcode::op_fetch_empty: {
+ write_empty(to, false);
+ return;
+ }
+ }
+
+ FU2_DETAIL_UNREACHABLE();
+ }
+
+ template <typename Box>
+ static void
+ construct(std::true_type /*apply*/, Box&& box, vtable* to_table,
+ data_accessor* to,
+ std::size_t to_capacity) noexcept(HasStrongExceptGuarantee) {
+ // Try to allocate the object inplace
+ void* storage = retrieve<T>(std::true_type{}, to, to_capacity);
+ if (storage) {
+ to_table->template set_inplace<T>();
+ } else {
+ // Allocate the object through the allocator
+ to->ptr_ = storage =
+ box_factory<std::decay_t<Box>>::box_allocate(std::addressof(box));
+ to_table->template set_allocated<T>();
+ }
+ new (storage) T(std::forward<Box>(box));
+ }
+
+ template <typename Box>
+ static void
+ construct(std::false_type /*apply*/, Box&& /*box*/, vtable* /*to_table*/,
+ data_accessor* /*to*/,
+ std::size_t /*to_capacity*/) noexcept(HasStrongExceptGuarantee) {
+ }
+ };
+
+ /// The command table
+ static void empty_cmd(vtable* to_table, opcode op, data_accessor* /*from*/,
+ std::size_t /*from_capacity*/, data_accessor* to,
+ std::size_t /*to_capacity*/) {
+
+ switch (op) {
+ case opcode::op_move:
+ case opcode::op_copy: {
+ to_table->set_empty();
+ break;
+ }
+ case opcode::op_destroy:
+ case opcode::op_weak_destroy: {
+ // Do nothing
+ break;
+ }
+ case opcode::op_fetch_empty: {
+ write_empty(to, true);
+ break;
+ }
+ default: {
+ FU2_DETAIL_UNREACHABLE();
+ }
+ }
+ }
+
+public:
+ vtable() noexcept = default;
+
+ /// Initialize an object at the given position
+ template <typename T>
+ static void init(vtable& table, T&& object, data_accessor* to,
+ std::size_t to_capacity) {
+
+ trait<std::decay_t<T>>::construct(std::true_type{}, std::forward<T>(object),
+ &table, to, to_capacity);
+ }
+
+ /// Moves the object at the given position
+ void move(vtable& to_table, data_accessor* from, std::size_t from_capacity,
+ data_accessor* to,
+ std::size_t to_capacity) noexcept(HasStrongExceptGuarantee) {
+ cmd_(&to_table, opcode::op_move, from, from_capacity, to, to_capacity);
+ set_empty();
+ }
+
+ /// Destroys the object at the given position
+ void copy(vtable& to_table, data_accessor const* from,
+ std::size_t from_capacity, data_accessor* to,
+ std::size_t to_capacity) const {
+ cmd_(&to_table, opcode::op_copy, const_cast<data_accessor*>(from),
+ from_capacity, to, to_capacity);
+ }
+
+ /// Destroys the object at the given position
+ void destroy(data_accessor* from,
+ std::size_t from_capacity) noexcept(HasStrongExceptGuarantee) {
+ cmd_(this, opcode::op_destroy, from, from_capacity, nullptr, 0U);
+ }
+
+ /// Destroys the object at the given position without invalidating the
+ /// vtable
+ void
+ weak_destroy(data_accessor* from,
+ std::size_t from_capacity) noexcept(HasStrongExceptGuarantee) {
+ cmd_(this, opcode::op_weak_destroy, from, from_capacity, nullptr, 0U);
+ }
+
+ /// Returns true when the vtable doesn't hold any erased object
+ bool empty() const noexcept {
+ data_accessor data;
+ cmd_(nullptr, opcode::op_fetch_empty, nullptr, 0U, &data, 0U);
+ return bool(data.inplace_storage_);
+ }
+
+ /// Invoke the function at the given index
+ template <std::size_t Index, typename... Args>
+ constexpr decltype(auto) invoke(Args&&... args) const {
+ auto thunk = invoke_table_t::template fetch<Index>(vtable_);
+ return thunk(std::forward<Args>(args)...);
+ }
+ /// Invoke the function at the given index
+ template <std::size_t Index, typename... Args>
+ constexpr decltype(auto) invoke(Args&&... args) const volatile {
+ auto thunk = invoke_table_t::template fetch<Index>(vtable_);
+ return thunk(std::forward<Args>(args)...);
+ }
+
+ template <typename T>
+ void set_inplace() noexcept {
+ using type = std::decay_t<T>;
+ vtable_ = invoke_table_t::template get_invocation_table_of<type, true>();
+ cmd_ = &trait<type>::template process_cmd<true>;
+ }
+
+ template <typename T>
+ void set_allocated() noexcept {
+ using type = std::decay_t<T>;
+ vtable_ = invoke_table_t::template get_invocation_table_of<type, false>();
+ cmd_ = &trait<type>::template process_cmd<false>;
+ }
+
+ void set_empty() noexcept {
+ vtable_ = invoke_table_t::template get_empty_invocation_table<IsThrowing>();
+ cmd_ = &empty_cmd;
+ }
+};
+} // namespace tables
+
+/// A union which makes the pointer to the heap object share the
+/// same space with the internal capacity.
+/// The storage type is distinguished by multiple versions of the
+/// control and vtable.
+template <typename Capacity, typename = void>
+struct internal_capacity {
+ /// We extend the union through a technique similar to the tail object hack
+ typedef union {
+ /// Tag to access the structure in a type-safe way
+ data_accessor accessor_;
+ /// The internal capacity we use to allocate in-place
+ std::aligned_storage_t<Capacity::capacity, Capacity::alignment> capacity_;
+ } type;
+};
+template <typename Capacity>
+struct internal_capacity<
+ Capacity, std::enable_if_t<(Capacity::capacity < sizeof(void*))>> {
+ typedef struct {
+ /// Tag to access the structure in a type-safe way
+ data_accessor accessor_;
+ } type;
+};
+
+template <typename Capacity>
+class internal_capacity_holder {
+ // Tag to access the structure in a type-safe way
+ typename internal_capacity<Capacity>::type storage_;
+
+public:
+ constexpr internal_capacity_holder() = default;
+
+ FU2_DETAIL_CXX14_CONSTEXPR data_accessor* opaque_ptr() noexcept {
+ return &storage_.accessor_;
+ }
+ constexpr data_accessor const* opaque_ptr() const noexcept {
+ return &storage_.accessor_;
+ }
+ FU2_DETAIL_CXX14_CONSTEXPR data_accessor volatile*
+ opaque_ptr() volatile noexcept {
+ return &storage_.accessor_;
+ }
+ constexpr data_accessor const volatile* opaque_ptr() const volatile noexcept {
+ return &storage_.accessor_;
+ }
+
+ static constexpr std::size_t capacity() noexcept {
+ return sizeof(storage_);
+ }
+};
+
+/// An owning erasure
+template <bool IsOwning /* = true*/, typename Config, typename Property>
+class erasure : internal_capacity_holder<typename Config::capacity> {
+ template <bool, typename, typename>
+ friend class erasure;
+ template <std::size_t, typename, typename...>
+ friend class operator_impl;
+
+ using vtable_t = tables::vtable<Property>;
+
+ vtable_t vtable_;
+
+public:
+ /// Returns the capacity of this erasure
+ static constexpr std::size_t capacity() noexcept {
+ return internal_capacity_holder<typename Config::capacity>::capacity();
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure() noexcept {
+ vtable_.set_empty();
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::nullptr_t) noexcept {
+ vtable_.set_empty();
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR
+ erasure(erasure&& right) noexcept(Property::is_strong_exception_guaranteed) {
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(erasure const& right) {
+ right.vtable_.copy(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ }
+
+ template <typename OtherConfig>
+ FU2_DETAIL_CXX14_CONSTEXPR
+ erasure(erasure<true, OtherConfig, Property> right) noexcept(
+ Property::is_strong_exception_guaranteed) {
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ }
+
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::false_type /*use_bool_op*/,
+ T&& callable,
+ Allocator&& allocator = Allocator{}) {
+ vtable_t::init(vtable_,
+ type_erasure::make_box(
+ std::integral_constant<bool, Config::is_copyable>{},
+ std::forward<T>(callable),
+ std::forward<Allocator>(allocator)),
+ this->opaque_ptr(), capacity());
+ }
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::true_type /*use_bool_op*/,
+ T&& callable,
+ Allocator&& allocator = Allocator{}) {
+ if (bool(callable)) {
+ vtable_t::init(vtable_,
+ type_erasure::make_box(
+ std::integral_constant<bool, Config::is_copyable>{},
+ std::forward<T>(callable),
+ std::forward<Allocator>(allocator)),
+ this->opaque_ptr(), capacity());
+ } else {
+ vtable_.set_empty();
+ }
+ }
+
+ ~erasure() {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure&
+ operator=(std::nullptr_t) noexcept(Property::is_strong_exception_guaranteed) {
+ vtable_.destroy(this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure& operator=(erasure&& right) noexcept(
+ Property::is_strong_exception_guaranteed) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure& operator=(erasure const& right) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ right.vtable_.copy(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ template <typename OtherConfig>
+ FU2_DETAIL_CXX14_CONSTEXPR erasure&
+ operator=(erasure<true, OtherConfig, Property> right) noexcept(
+ Property::is_strong_exception_guaranteed) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ void assign(std::false_type /*use_bool_op*/, T&& callable,
+ Allocator&& allocator = {}) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ vtable_t::init(vtable_,
+ type_erasure::make_box(
+ std::integral_constant<bool, Config::is_copyable>{},
+ std::forward<T>(callable),
+ std::forward<Allocator>(allocator)),
+ this->opaque_ptr(), capacity());
+ }
+
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ void assign(std::true_type /*use_bool_op*/, T&& callable,
+ Allocator&& allocator = {}) {
+ if (bool(callable)) {
+ assign(std::false_type{}, std::forward<T>(callable),
+ std::forward<Allocator>(allocator));
+ } else {
+ operator=(nullptr);
+ }
+ }
+
+ /// Returns true when the erasure doesn't hold any erased object
+ constexpr bool empty() const noexcept {
+ return vtable_.empty();
+ }
+
+ /// Invoke the function of the erasure at the given index
+ ///
+ /// We define this out of class to be able to forward the qualified
+ /// erasure correctly.
+ template <std::size_t Index, typename Erasure, typename... Args>
+ static constexpr decltype(auto) invoke(Erasure&& erasure, Args&&... args) {
+ auto const capacity = erasure.capacity();
+ return erasure.vtable_.template invoke<Index>(
+ std::forward<Erasure>(erasure).opaque_ptr(), capacity,
+ std::forward<Args>(args)...);
+ }
+};
+
+// A non owning erasure
+template </*bool IsOwning = false, */ typename Config, bool IsThrowing,
+ bool HasStrongExceptGuarantee, typename... Args>
+class erasure<false, Config,
+ property<IsThrowing, HasStrongExceptGuarantee, Args...>> {
+ template <bool, typename, typename>
+ friend class erasure;
+ template <std::size_t, typename, typename...>
+ friend class operator_impl;
+
+ using property_t = property<IsThrowing, HasStrongExceptGuarantee, Args...>;
+
+ using invoke_table_t = invocation_table::invoke_table<Args...>;
+ typename invoke_table_t::type invoke_table_;
+
+ /// The internal pointer to the non owned object
+ data_accessor view_;
+
+public:
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure() noexcept
+ : invoke_table_(
+ invoke_table_t::template get_empty_invocation_table<IsThrowing>()),
+ view_(nullptr) {
+ }
+
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(std::nullptr_t) noexcept
+ : invoke_table_(
+ invoke_table_t::template get_empty_invocation_table<IsThrowing>()),
+ view_(nullptr) {
+ }
+
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(erasure&& right) noexcept
+ : invoke_table_(right.invoke_table_), view_(right.view_) {
+ }
+
+ constexpr erasure(erasure const& /*right*/) = default;
+
+ template <typename OtherConfig>
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(erasure<false, OtherConfig, property_t> right) noexcept
+ : invoke_table_(right.invoke_table_), view_(right.view_) {
+ }
+
+ template <typename T>
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(std::false_type /*use_bool_op*/, T&& object)
+ : invoke_table_(invoke_table_t::template get_invocation_view_table_of<
+ std::decay_t<T>>()),
+ view_(address_taker<std::decay_t<T>>::take(std::forward<T>(object))) {
+ }
+ template <typename T>
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::true_type use_bool_op, T&& object) {
+ this->assign(use_bool_op, std::forward<T>(object));
+ }
+
+ ~erasure() = default;
+
+ constexpr erasure&
+ operator=(std::nullptr_t) noexcept(HasStrongExceptGuarantee) {
+ invoke_table_ =
+ invoke_table_t::template get_empty_invocation_table<IsThrowing>();
+ view_.ptr_ = nullptr;
+ return *this;
+ }
+
+ constexpr erasure& operator=(erasure&& right) noexcept {
+ invoke_table_ = right.invoke_table_;
+ view_ = right.view_;
+ right = nullptr;
+ return *this;
+ }
+
+ constexpr erasure& operator=(erasure const& /*right*/) = default;
+
+ template <typename OtherConfig>
+ constexpr erasure&
+ operator=(erasure<true, OtherConfig, property_t> right) noexcept {
+ invoke_table_ = right.invoke_table_;
+ view_ = right.view_;
+ return *this;
+ }
+
+ template <typename T>
+ constexpr void assign(std::false_type /*use_bool_op*/, T&& callable) {
+ invoke_table_ = invoke_table_t::template get_invocation_view_table_of<
+ std::decay_t<T>>();
+ view_.ptr_ =
+ address_taker<std::decay_t<T>>::take(std::forward<T>(callable));
+ }
+ template <typename T>
+ constexpr void assign(std::true_type /*use_bool_op*/, T&& callable) {
+ if (bool(callable)) {
+ assign(std::false_type{}, std::forward<T>(callable));
+ } else {
+ operator=(nullptr);
+ }
+ }
+
+ /// Returns true when the erasure doesn't hold any erased object
+ constexpr bool empty() const noexcept {
+ return view_.ptr_ == nullptr;
+ }
+
+ template <std::size_t Index, typename Erasure, typename... T>
+ static constexpr decltype(auto) invoke(Erasure&& erasure, T&&... args) {
+ auto thunk = invoke_table_t::template fetch<Index>(erasure.invoke_table_);
+ return thunk(&(erasure.view_), 0UL, std::forward<T>(args)...);
+ }
+};
+} // namespace type_erasure
+
+/// Deduces to a true_type if the type T provides the given signature and the
+/// signature is noexcept correct callable.
+template <typename T, typename Signature,
+ typename Trait =
+ type_erasure::invocation_table::function_trait<Signature>>
+struct accepts_one
+ : std::integral_constant<
+ bool, invocation::can_invoke<typename Trait::template callable<T>,
+ typename Trait::arguments>::value &&
+ invocation::is_noexcept_correct<
+ Trait::is_noexcept::value,
+ typename Trait::template callable<T>,
+ typename Trait::arguments>::value> {};
+
+/// Deduces to a true_type if the type T provides all signatures
+template <typename T, typename Signatures, typename = void>
+struct accepts_all : std::false_type {};
+template <typename T, typename... Signatures>
+struct accepts_all<
+ T, identity<Signatures...>,
+ void_t<std::enable_if_t<accepts_one<T, Signatures>::value>...>>
+ : std::true_type {};
+
+/// Deduces to a true_type if the type T is implementing operator bool()
+/// or if the type is convertible to bool directly, this also implements an
+/// optimizations for function references `void(&)()` which are can never
+/// be null and for such a conversion to bool would never return false.
+#if defined(FU2_HAS_NO_EMPTY_PROPAGATION)
+template <typename T>
+struct use_bool_op : std::false_type {};
+#else
+template <typename T, typename = void>
+struct has_bool_op : std::false_type {};
+template <typename T>
+struct has_bool_op<T, void_t<decltype(bool(std::declval<T>()))>>
+ : std::true_type {
+#ifndef NDEBUG
+ static_assert(!std::is_pointer<T>::value,
+ "Missing deduction for function pointer!");
+#endif
+};
+
+template <typename T>
+struct use_bool_op : has_bool_op<T> {};
+
+#define FU2_DEFINE_USE_OP_TRAIT(CONST, VOLATILE, NOEXCEPT) \
+ template <typename Ret, typename... Args> \
+ struct use_bool_op<Ret (*CONST VOLATILE)(Args...) NOEXCEPT> \
+ : std::true_type {};
+
+FU2_DETAIL_EXPAND_CV(FU2_DEFINE_USE_OP_TRAIT)
+#undef FU2_DEFINE_USE_OP_TRAIT
+
+template <typename Ret, typename... Args>
+struct use_bool_op<Ret(Args...)> : std::false_type {};
+
+#if defined(FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE)
+template <typename Ret, typename... Args>
+struct use_bool_op<Ret(Args...) noexcept> : std::false_type {};
+#endif
+#endif // FU2_HAS_NO_EMPTY_PROPAGATION
+
+template <typename Config, typename T>
+struct assert_wrong_copy_assign {
+ static_assert(!Config::is_owning || !Config::is_copyable ||
+ std::is_copy_constructible<std::decay_t<T>>::value,
+ "Can't wrap a non copyable object into a unique function!");
+
+ using type = void;
+};
+
+template <bool IsStrongExceptGuaranteed, typename T>
+struct assert_no_strong_except_guarantee {
+ static_assert(
+ !IsStrongExceptGuaranteed ||
+ (std::is_nothrow_move_constructible<T>::value &&
+ std::is_nothrow_destructible<T>::value),
+ "Can't wrap a object an object that has no strong exception guarantees "
+ "if this is required by the wrapper!");
+
+ using type = void;
+};
+
+/// SFINAES out if the given callable is not copyable correct to the left one.
+template <typename LeftConfig, typename RightConfig>
+using enable_if_copyable_correct_t =
+ std::enable_if_t<(!LeftConfig::is_copyable || RightConfig::is_copyable)>;
+
+template <typename LeftConfig, typename RightConfig>
+using is_owning_correct =
+ std::integral_constant<bool,
+ (LeftConfig::is_owning == RightConfig::is_owning)>;
+
+/// SFINAES out if the given function2 is not owning correct to this one
+template <typename LeftConfig, typename RightConfig>
+using enable_if_owning_correct_t =
+ std::enable_if_t<is_owning_correct<LeftConfig, RightConfig>::value>;
+
+template <typename Config, bool IsThrowing, bool HasStrongExceptGuarantee,
+ typename... Args>
+class function<Config, property<IsThrowing, HasStrongExceptGuarantee, Args...>>
+ : type_erasure::invocation_table::operator_impl<
+ 0U,
+ function<Config,
+ property<IsThrowing, HasStrongExceptGuarantee, Args...>>,
+ Args...> {
+
+ template <typename, typename>
+ friend class function;
+
+ template <std::size_t, typename, typename...>
+ friend class type_erasure::invocation_table::operator_impl;
+
+ using property_t = property<IsThrowing, HasStrongExceptGuarantee, Args...>;
+ using erasure_t =
+ type_erasure::erasure<Config::is_owning, Config, property_t>;
+
+ template <typename T>
+ using enable_if_can_accept_all_t =
+ std::enable_if_t<accepts_all<std::decay_t<T>, identity<Args...>>::value>;
+
+ template <typename Function, typename = void>
+ struct is_convertible_to_this : std::false_type {};
+ template <typename RightConfig>
+ struct is_convertible_to_this<
+ function<RightConfig, property_t>,
+ void_t<enable_if_copyable_correct_t<Config, RightConfig>,
+ enable_if_owning_correct_t<Config, RightConfig>>>
+ : std::true_type {};
+
+ template <typename T>
+ using enable_if_not_convertible_to_this =
+ std::enable_if_t<!is_convertible_to_this<std::decay_t<T>>::value>;
+
+ template <typename T>
+ using enable_if_owning_t =
+ std::enable_if_t<std::is_same<T, T>::value && Config::is_owning>;
+
+ template <typename T>
+ using assert_wrong_copy_assign_t =
+ typename assert_wrong_copy_assign<Config, std::decay_t<T>>::type;
+
+ template <typename T>
+ using assert_no_strong_except_guarantee_t =
+ typename assert_no_strong_except_guarantee<HasStrongExceptGuarantee,
+ std::decay_t<T>>::type;
+
+ erasure_t erasure_;
+
+public:
+ /// Default constructor which empty constructs the function
+ function() = default;
+ ~function() = default;
+
+ explicit FU2_DETAIL_CXX14_CONSTEXPR
+ function(function const& /*right*/) = default;
+ explicit FU2_DETAIL_CXX14_CONSTEXPR function(function&& /*right*/) = default;
+
+ /// Copy construction from another copyable function
+ template <typename RightConfig,
+ std::enable_if_t<RightConfig::is_copyable>* = nullptr,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR
+ function(function<RightConfig, property_t> const& right)
+ : erasure_(right.erasure_) {
+ }
+
+ /// Move construction from another function
+ template <typename RightConfig,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR function(function<RightConfig, property_t>&& right)
+ : erasure_(std::move(right.erasure_)) {
+ }
+
+ /// Construction from a callable object which overloads the `()` operator
+ template <typename T, //
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR function(T&& callable)
+ : erasure_(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable)) {
+ }
+ template <typename T, typename Allocator, //
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ enable_if_owning_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR function(T&& callable, Allocator&& allocator)
+ : erasure_(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable),
+ std::forward<Allocator>(allocator)) {
+ }
+
+ /// Empty constructs the function
+ FU2_DETAIL_CXX14_CONSTEXPR function(std::nullptr_t np) : erasure_(np) {
+ }
+
+ function& operator=(function const& /*right*/) = default;
+ function& operator=(function&& /*right*/) = default;
+
+ /// Copy assigning from another copyable function
+ template <typename RightConfig,
+ std::enable_if_t<RightConfig::is_copyable>* = nullptr,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ function& operator=(function<RightConfig, property_t> const& right) {
+ erasure_ = right.erasure_;
+ return *this;
+ }
+
+ /// Move assigning from another function
+ template <typename RightConfig,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ function& operator=(function<RightConfig, property_t>&& right) {
+ erasure_ = std::move(right.erasure_);
+ return *this;
+ }
+
+ /// Move assigning from a callable object
+ template <typename T, // ...
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ function& operator=(T&& callable) {
+ erasure_.assign(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable));
+ return *this;
+ }
+
+ /// Clears the function
+ function& operator=(std::nullptr_t np) {
+ erasure_ = np;
+ return *this;
+ }
+
+ /// Returns true when the function is empty
+ bool empty() const noexcept {
+ return erasure_.empty();
+ }
+
+ /// Returns true when the function isn't empty
+ explicit operator bool() const noexcept {
+ return !empty();
+ }
+
+ /// Assigns a new target with an optional allocator
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>,
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ void assign(T&& callable, Allocator&& allocator = Allocator{}) {
+ erasure_.assign(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable),
+ std::forward<Allocator>(allocator));
+ }
+
+ /// Swaps this function with the given function
+ void swap(function& other) noexcept(HasStrongExceptGuarantee) {
+ if (&other == this) {
+ return;
+ }
+
+ function cache = std::move(other);
+ other = std::move(*this);
+ *this = std::move(cache);
+ }
+
+ /// Swaps the left function with the right one
+ friend void swap(function& left,
+ function& right) noexcept(HasStrongExceptGuarantee) {
+ left.swap(right);
+ }
+
+ /// Calls the wrapped callable object
+ using type_erasure::invocation_table::operator_impl<
+ 0U, function<Config, property_t>, Args...>::operator();
+};
+
+template <typename Config, typename Property>
+bool operator==(function<Config, Property> const& f, std::nullptr_t) {
+ return !bool(f);
+}
+
+template <typename Config, typename Property>
+bool operator!=(function<Config, Property> const& f, std::nullptr_t) {
+ return bool(f);
+}
+
+template <typename Config, typename Property>
+bool operator==(std::nullptr_t, function<Config, Property> const& f) {
+ return !bool(f);
+}
+
+template <typename Config, typename Property>
+bool operator!=(std::nullptr_t, function<Config, Property> const& f) {
+ return bool(f);
+}
+
+// Default intended object size of the function
+using object_size = std::integral_constant<std::size_t, 32U>;
+} // namespace detail
+} // namespace abi_400
+
+/// Can be passed to function_base as template argument which causes
+/// the internal small buffer to be sized according to the given size,
+/// and aligned with the given alignment.
+template <std::size_t Capacity,
+ std::size_t Alignment = alignof(std::max_align_t)>
+struct capacity_fixed {
+ static constexpr std::size_t capacity = Capacity;
+ static constexpr std::size_t alignment = Alignment;
+};
+
+/// Default capacity for small functor optimization
+struct capacity_default
+ : capacity_fixed<detail::object_size::value - (2 * sizeof(void*))> {};
+
+/// Can be passed to function_base as template argument which causes
+/// the internal small buffer to be removed from the callable wrapper.
+/// The owning function_base will then allocate memory for every object
+/// it applies a type erasure on.
+struct capacity_none : capacity_fixed<0UL> {};
+
+/// Can be passed to function_base as template argument which causes
+/// the internal small buffer to be sized such that it can hold
+/// the given object without allocating memory for an applied type erasure.
+template <typename T>
+struct capacity_can_hold {
+ static constexpr std::size_t capacity = sizeof(T);
+ static constexpr std::size_t alignment = alignof(T);
+};
+
+/// An adaptable function wrapper base for arbitrary functional types.
+///
+/// \tparam IsOwning Is true when the type erasure shall be owning the object.
+///
+/// \tparam IsCopyable Defines whether the function is copyable or not
+///
+/// \tparam Capacity Defines the internal capacity of the function
+/// for small functor optimization.
+/// The size of the whole function object will be the capacity
+/// plus the size of two pointers. If the capacity is zero,
+/// the size will increase through one additional pointer
+/// so the whole object has the size of 3 * sizeof(void*).
+/// The type which is passed to the Capacity template parameter
+/// shall provide a capacity and alignment member which
+/// looks like the following example:
+/// ```cpp
+/// struct my_capacity {
+/// static constexpr std::size_t capacity = sizeof(my_type);
+/// static constexpr std::size_t alignment = alignof(my_type);
+/// };
+/// ```
+///
+/// \tparam IsThrowing Defines whether the function throws an exception on
+/// empty function call, `std::abort` is called otherwise.
+///
+/// \tparam HasStrongExceptGuarantee Defines whether all objects satisfy the
+/// strong exception guarantees,
+/// which means the function type will satisfy
+/// the strong exception guarantees too.
+///
+/// \tparam Signatures Defines the signature of the callable wrapper
+///
+template <bool IsOwning, bool IsCopyable, typename Capacity, bool IsThrowing,
+ bool HasStrongExceptGuarantee, typename... Signatures>
+using function_base = detail::function<
+ detail::config<IsOwning, IsCopyable, Capacity>,
+ detail::property<IsThrowing, HasStrongExceptGuarantee, Signatures...>>;
+
+/// An owning copyable function wrapper for arbitrary callable types.
+template <typename... Signatures>
+using function = function_base<true, true, capacity_default, //
+ true, false, Signatures...>;
+
+/// An owning non copyable function wrapper for arbitrary callable types.
+template <typename... Signatures>
+using unique_function = function_base<true, false, capacity_default, //
+ true, false, Signatures...>;
+
+/// A non owning copyable function wrapper for arbitrary callable types.
+template <typename... Signatures>
+using function_view = function_base<false, true, capacity_default, //
+ true, false, Signatures...>;
+
+#if !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+/// Exception type that is thrown when invoking empty function objects
+/// and exception support isn't disabled.
+///
+/// Exception support is enabled if
+/// the template parameter 'Throwing' is set to true (default).
+///
+/// This type will default to std::bad_function_call if the
+/// functional header is used, otherwise the library provides its own type.
+///
+/// You may disable the inclusion of the functional header
+/// through defining `FU2_WITH_NO_FUNCTIONAL_HEADER`.
+///
+using detail::type_erasure::invocation_table::bad_function_call;
+#endif
+
+/// Returns a callable object, which unifies all callable objects
+/// that were passed to this function.
+///
+/// ```cpp
+/// auto overloaded = fu2::overload([](std::true_type) { return true; },
+/// [](std::false_type) { return false; });
+/// ```
+///
+/// \param callables A pack of callable objects with arbitrary signatures.
+///
+/// \returns A callable object which exposes the
+///
+template <typename... T>
+constexpr auto overload(T&&... callables) {
+ return detail::overloading::overload(std::forward<T>(callables)...);
+}
+} // namespace fu2
+
+#undef FU2_DETAIL_EXPAND_QUALIFIERS
+#undef FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT
+#undef FU2_DETAIL_EXPAND_CV
+#undef FU2_DETAIL_EXPAND_CV_NOEXCEPT
+#undef FU2_DETAIL_UNREACHABLE_INTRINSIC
+#undef FU2_DETAIL_TRAP
+#undef FU2_DETAIL_CXX14_CONSTEXPR
+
+#endif // FU2_INCLUDED_FUNCTION2_HPP_
diff --git a/contrib/google-ced/CMakeLists.txt b/contrib/google-ced/CMakeLists.txt
new file mode 100644
index 0000000..668c635
--- /dev/null
+++ b/contrib/google-ced/CMakeLists.txt
@@ -0,0 +1,26 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# This library is derived from https://github.com/google/compact_enc_det
+# git id: 37529e628fbac2e4c0d4d8520be9db789f316c9e
+
+project(CED CXX)
+set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1 CACHE INTERNAL "No dev warnings")
+
+option(BUILD_SHARED_LIBS "Build shared libraries" OFF)
+
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wno-narrowing")
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+set(CED_LIBRARY_SOURCES
+ compact_enc_det.cc
+ compact_enc_det_hint_code.cc
+ util/encodings/encodings.cc
+ util/languages/languages.cc
+ ced_c.cc
+ )
+
+add_library(rspamd-ced STATIC ${CED_LIBRARY_SOURCES})
diff --git a/contrib/google-ced/LICENSE b/contrib/google-ced/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/contrib/google-ced/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/contrib/google-ced/ced_c.cc b/contrib/google-ced/ced_c.cc
new file mode 100644
index 0000000..5167121
--- /dev/null
+++ b/contrib/google-ced/ced_c.cc
@@ -0,0 +1,25 @@
+#include "ced_c.h"
+#include "compact_enc_det.h"
+
+const char* ced_encoding_detect(const char* text, int text_length,
+ const char* url_hint,
+ const char* http_charset_hint,
+ const char* meta_charset_hint,
+ const int encoding_hint,
+ CedTextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+ int* bytes_consumed, bool* is_reliable)
+{
+ CompactEncDet::TextCorpusType ct = CompactEncDet::NUM_CORPA;
+
+ ct = static_cast<CompactEncDet::TextCorpusType>(corpus_type);
+
+ auto enc = CompactEncDet::DetectEncoding(text, text_length, url_hint,
+ http_charset_hint, meta_charset_hint, encoding_hint, default_language(),
+ ct, ignore_7bit_mail_encodings, bytes_consumed, is_reliable);
+
+ if (IsValidEncoding(enc)) {
+ return MimeEncodingName(enc);
+ }
+
+ return nullptr;
+}
diff --git a/contrib/google-ced/ced_c.h b/contrib/google-ced/ced_c.h
new file mode 100644
index 0000000..c086f0c
--- /dev/null
+++ b/contrib/google-ced/ced_c.h
@@ -0,0 +1,32 @@
+#ifndef RSPAMD_CED_C_H
+#define RSPAMD_CED_C_H
+
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+enum CedTextCorpusType {
+ CED_WEB_CORPUS,
+ CED_XML_CORPUS,
+ CED_QUERY_CORPUS,
+ CED_EMAIL_CORPUS,
+ CED_NUM_CORPA,
+};
+
+/*
+ * XXX: Rspamd addition: it actually returns Mime format of the encoding
+ */
+const char *ced_encoding_detect (const char *text, int text_length,
+ const char *url_hint,
+ const char *http_charset_hint,
+ const char *meta_charset_hint,
+ const int encoding_hint,
+ enum CedTextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings,
+ int *bytes_consumed, bool *is_reliable);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/contrib/google-ced/compact_enc_det.cc b/contrib/google-ced/compact_enc_det.cc
new file mode 100644
index 0000000..c962b43
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det.cc
@@ -0,0 +1,5719 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "compact_enc_det.h"
+
+#include <math.h> // for sqrt
+#include <stddef.h> // for size_t
+#include <stdio.h> // for printf, fprintf, NULL, etc
+#include <stdlib.h> // for qsort
+#include <string.h> // for memset, memcpy, memcmp, etc
+#include <memory>
+#include <string> // for string, operator==, etc
+
+#include "compact_enc_det_hint_code.h"
+#include "util/string_util.h"
+#include "util/basictypes.h"
+#include "util/commandlineflags.h"
+#include "util/logging.h"
+
+using std::string;
+
+// TODO as of 2007.10.09:
+//
+// Consider font=TT-BHxxx as user-defined => binary
+// Demote GB18030 if no 8x3x pair
+// Map byte2 ascii punct to 0x60, digits to 0x7e, gets them into hires
+// Consider removing/ignoring bytes 01-1F to avoid crap pollution
+// Possibly boost declared encoding in robust scan
+// googlebot tiny files
+// look for ranges of encodings
+// consider tags just as > < within aligned block of 32
+// flag too few characters in postproc (Latin 6 problem)
+// Remove slow scan beyond 16KB
+// Consider removing kMostLikelyEncoding or cut it in half
+
+
+// A note on mixed encodings
+//
+// The most common encoding error on the web is a page containing a mixture of
+// CP-1252 and UTF-8. A less common encoding error is a third-party feed that
+// has been converted from CP-1252 to UTF-8 and then those bytes converted a
+// second time to UTF-8. CED originally attempted to detect these error cases
+// by using two synthetic encodings, UTF8CP1252 and UTF8UTF8. The intended
+// implementation was to start these just below CP1252 and UTF8 respectively in
+// overall liklihood, and allow 1252 and UTF8 to fall behind if mixtures are
+// found.
+//
+// The UTF8UTF8 encoding is a possible outcome from CED, but unfortunately the
+// UTF8CP1252 internal encoding was added late and not put into encodings.proto,
+// so at the final step it is mapped to UTF8UTF8 also. This was a bad idea and
+// is removed in this November 2011 CL.
+//
+// Mixed encoding detection never worked out as well as envisioned, so the
+// ced_allow_utf8utf8 flag normally disables all this.
+//
+// The effect is that CP-1252 and UTF-8 mixtures will usually be detected as
+// UTF8, and the inputconverter code for UTF8 normally will convert bare
+// CP-1252 bytes to UTF-8, instead of the less-helpful FFFD substitution. UTF-8
+// and double-UTF-8 mixtures will be detected as UTF-8, and the double
+// conversion will stand.
+//
+// However, it is occasionally useful to use CED to detect double-converted
+// UTF-8 coming from third-party data feeds, so they can be fixed at the source.
+// For this purpose, the UTF8UTF8 encoding remains available under the
+// ced_allow_utf8utf8 flag.
+//
+// When UTF8UTF8 is detected, the inputconverter code will undo the double
+// conversion, giving good text.
+
+// Norbert Runge has noted these words in CP1252 that are mistakenly identified
+// as UTF-8 because of the last pair of characters:
+// NESTLÉ® 0xC9 0xAE U+00C9 U+00AE C9AE = U+026E;SMALL LEZH
+// drauß\u2019 0xDF 0x92 U+00DF U+2019 DF92 = U+07D2;NKO LETTER N
+// Mutterschoß\u201c 0xDF 0x93 U+00DF U+201C DF93 = U+07D3;NKO LETTER BA
+// Schoß\u201c 0xDF 0x93 U+00DF U+201C
+// weiß\u201c 0xDF 0x93 U+00DF U+00AB
+// Schnellfuß\u201c 0xDF 0x93 U+00DF U+201C
+// süß« 0xDF 0xAB U+00DF U+00AB DFAB = U+07EB;NKO HIGH TONE
+// These four byte combinations now explicitly boost Latin1/CP1252.
+
+// And for reference, here are a couple of Portuguese spellings
+// that may be mistaken as double-byte encodings.
+// informações 0xE7 0xF5
+// traição 0xE7 0xE3
+
+
+static const char* kVersion = "2.2";
+
+DEFINE_bool(ced_allow_utf8utf8, false, "Allow the UTF8UTF8 encoding, "
+ "to handle mixtures of CP1252 "
+ "converted to UTF-8 zero, one, "
+ "or two times");
+DEFINE_int32(enc_detect_slow_max_kb, 16,
+ "Maximum number of Kbytes to examine for "
+ "7-bit-only (2022, Hz, UTF7) encoding detect. "
+ "You are unlikely to want to change this.");
+DEFINE_int32(enc_detect_fast_max_kb, 256,
+ "Maximum number of Kbytes to examine for encoding detect. "
+ "You are unlikely to want to change this.");
+
+DEFINE_int32(ced_reliable_difference, 300, "30 * Bits of minimum probablility "
+ "difference 1st - 2nd to be considered reliable \n"
+ " 2 corresponds to min 4x difference\n"
+ " 4 corresponds to min 16x difference\n"
+ " 8 corresponds to min 256x difference\n"
+ " 10 corresponds to min 1024x difference\n"
+ " 20 corresponds to min 1Mx difference.");
+
+// Text debug output options
+DEFINE_bool(enc_detect_summary, false,
+ "Print first 16 interesting pairs at exit.");
+DEFINE_bool(counts, false, "Count major-section usage");
+
+// PostScript debug output options
+DEFINE_bool(enc_detect_detail, false,
+ "Print PostScript of every update, to stderr.");
+DEFINE_bool(enc_detect_detail2, false,
+ "More PostScript detail of every update, to stderr.");
+DEFINE_bool(enc_detect_source, false, "Include source text in detail");
+// Encoding name must exactly match FIRST column of kI18NInfoByEncoding in
+// lang_enc.cc
+
+// Following flags are not in use. Replace them with constants to
+// avoid static initialization.
+
+//DEFINE_string(enc_detect_watch1, "", "Do detail2 about this encoding name.");
+//DEFINE_string(enc_detect_watch2, "", "Do detail2 about this encoding name.");
+
+static const char* const FLAGS_enc_detect_watch1 = "";
+static const char* const FLAGS_enc_detect_watch2 = "";
+
+// Only for experiments. Delete soon.
+DEFINE_bool(force127, false, "Force Latin1, Latin2, Latin7 based on trigrams");
+
+// Demo-mode/debugging experiment
+DEFINE_bool(demo_nodefault, false,
+ "Default to all equal; no boost for declared encoding.");
+DEFINE_bool(dirtsimple, false, "Just scan and count for all encodings");
+DEFINE_bool(ced_echo_input, false, "Echo ced input to stderr");
+
+
+static const int XDECILOG2 = 3; // Multiplier for log base 2 ** n/10
+static const int XLOG2 = 30; // Multiplier for log base 2 ** n
+
+static const int kFinalPruneDifference = 10 * XLOG2;
+ // Final bits of minimum
+ // probability difference 1st-nth
+ // to be pruned
+
+static const int kInititalPruneDifference = kFinalPruneDifference * 4;
+ // Initial bits of minimum
+ // probability difference 1st-nth
+ // to be pruned
+ //
+static const int kPruneDiffDecrement = kFinalPruneDifference;
+ // Decrements bits of minimum
+ // probability difference 1st-nth
+ // to be pruned
+
+static const int kSmallInitDiff = 2 * XLOG2; // bits of minimum
+ // probability difference, base to
+ // superset encodings
+
+static const int kBoostInitial = 20 * XLOG2; // bits of boost for
+ // initial byte patterns (BOM, 00)
+
+static const int kBadPairWhack = 20 * XLOG2; // bits of whack for
+ // one bad pair
+
+static const int kBoostOnePair = 20 * XLOG2; // bits of boost for
+ // one good pair in Hz, etc.
+
+static const int kGentleOnePair = 4 * XLOG2; // bits of boost for
+ // one good sequence
+ //
+static const int kGentlePairWhack = 2 * XLOG2; // bits of whack
+ // for ill-formed sequence
+
+static const int kGentlePairBoost = 2 * XLOG2; // bits of boost
+ // for well-formed sequence
+
+static const int kDeclaredEncBoost = 5 * XDECILOG2; // bits/10 of boost for
+ // best declared encoding per bigram
+
+static const int kBestEncBoost = 5 * XDECILOG2; // bits/10 of boost for
+ // best encoding per bigram
+
+static const int kTrigramBoost = 2 * XLOG2; // bits of boost for Latin127 tri
+
+static const int kMaxPairs = 48; // Max interesting pairs to look at
+ // If you change this,
+ // adjust *PruneDiff*
+
+static const int kPruneMask = 0x07; // Prune every 8 interesting pairs
+
+
+static const int kBestPairsCount = 16; // For first N pairs, do extra boost
+ // based on most likely encoding
+ // of pair over entire web
+
+static const int kDerateHintsBelow = 12; // If we have fewer than N bigrams,
+ // weaken the hints enough that
+ // unhinted encodings have a hope of
+ // rising to the top
+
+static const int kMinRescanLength = 800; // Don't bother rescanning for
+ // unreliable encoding if fewer
+ // than this many bytes unscanned.
+ // We will rescan at most last half
+ // of this.
+
+static const int kStrongBinary = 12; // Make F_BINARY the only encoding
+static const int kWeakerBinary = 4; // Make F_BINARY likely encoding
+
+// These are byte counts from front of file
+static const int kBinaryHardAsciiLimit = 6 * 1024; // Not binary if all ASCII
+static const int kBinarySoftAsciiLimit = 8 * 1024; // " if mostly ASCII
+
+// We try here to avoid having title text dominate the encoding detection,
+// for the not-infrequent error case of title in encoding1, body in encoding2:
+// we want to bias toward encoding2 winning.
+//
+// kMaxBigramsTagTitleText should be a multiple of 2, 3, and 4, so that we
+// rarely cut off mid-character in the original (not-yet-detected) encoding.
+// This matters most for UTF-8 two- and three-byte codes and for
+// Shift-JIS three-byte codes.
+static const int kMaxBigramsTagTitleText = 12; // Keep only some tag text
+static const int kWeightshiftForTagTitleText = 4; // Give text in tags, etc.
+ // 1/16 normal weight
+
+static const int kStrongPairs = 6; // Let reliable enc with this many
+ // pairs overcome missing hint
+
+enum CEDInternalFlags {
+ kCEDNone = 0, // The empty flag
+ kCEDRescanning = 1, // Do not further recurse
+ kCEDSlowscore = 2, // Do extra scoring
+ kCEDForceTags = 4, // Always examine text inside tags
+};
+
+// Forward declaration
+Encoding InternalDetectEncoding(
+ CEDInternalFlags flags, const char* text, int text_length,
+ const char* url_hint, const char* http_charset_hint,
+ const char* meta_charset_hint, const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const CompactEncDet::TextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings, int* bytes_consumed, bool* is_reliable,
+ Encoding* second_best_enc);
+
+typedef struct {
+ const uint8* hires[4]; // Pointers to possible high-resolution bigram deltas
+ uint8 x_bar; // Average byte2 value
+ uint8 y_bar; // Average byte1 value
+ uint8 x_stddev; // Standard deviation of byte2 value
+ uint8 y_stddev; // Standard deviation of byte1 value
+ int so; // Scaling offset -- add to probabilities below
+ uint8 b1[256]; // Unigram probability for first byte of aligned bigram
+ uint8 b2[256]; // Unigram probability for second byte of aligned bigram
+ uint8 b12[256]; // Unigram probability for cross bytes of aligned bigram
+} UnigramEntry;
+
+//typedef struct {
+// uint8 b12[256*256]; // Bigram probability for aligned bigram
+//} FullBigramEntry;
+
+
+// Include all the postproc-generated tables here:
+// RankedEncoding
+// kMapToEncoding
+// unigram_table
+// kMostLIkelyEncoding
+// kTLDHintProbs
+// kCharsetHintProbs
+// HintEntry, kMaxTldKey kMaxTldVector, etc.
+// =============================================================================
+
+#include "compact_enc_det_generated_tables.h"
+
+
+#define F_ASCII F_Latin1 // "ASCII" is a misnomer, so this code uses "Latin1"
+
+#define F_BINARY F_X_BINARYENC // We are mid-update for name change
+#define F_UTF8UTF8 F_X_UTF8UTF8 // We are mid-update for name change
+#define F_BIG5_CP950 F_BIG5 // We are mid-update for name change
+#define F_Unicode F_UTF_16LE // We are mid-update for name change
+// =============================================================================
+
+// 7-bit encodings have at least one "interesting" byte value < 0x80
+// (00 0E 1B + ~)
+// JIS 2022-cn 2022-kr hz utf7
+// Unicode UTF-16 UTF-32
+// 8-bit encodings have no interesting byte values < 0x80
+static const uint32 kSevenBitActive = 0x00000001; // needs <80 to detect
+static const uint32 kUTF7Active = 0x00000002; // <80 and +
+static const uint32 kHzActive = 0x00000004; // <80 and ~
+static const uint32 kIso2022Active = 0x00000008; // <80 and 1B 0E 0F
+static const uint32 kUTF8Active = 0x00000010;
+static const uint32 kUTF8UTF8Active = 0x00000020;
+static const uint32 kUTF1632Active = 0x00000040; // <80 and 00
+static const uint32 kBinaryActive = 0x00000080; // <80 and 00
+static const uint32 kTwobyteCode = 0x00000100; // Needs 8xxx
+static const uint32 kIsIndicCode = 0x00000200; //
+static const uint32 kHighAlphaCode = 0x00000400; // full alphabet in 8x-Fx
+static const uint32 kHighAccentCode = 0x00000800; // accents in 8x-Fx
+static const uint32 kEUCJPActive = 0x00001000; // Have to mess with phase
+
+
+// Debug only. not thread safe
+static int encdet_used = 0;
+static int rescore_used = 0;
+static int rescan_used = 0;
+static int robust_used = 0;
+static int looking_used = 0;
+static int doing_used = 0;
+
+
+// For debugging only -- about 256B/entry times about 500 = 128KB
+// TODO: only allocate this if being used
+typedef struct {
+ int offset;
+ int best_enc; // Best ranked encoding for this bigram, or
+ // -1 for overhead entries
+ string label;
+ int detail_enc_prob[NUM_RANKEDENCODING];
+} DetailEntry;
+
+static int watch1_rankedenc = -1; // Debug. not threadsafe
+static int watch2_rankedenc = -1; // Debug. not threadsafe
+////static int next_detail_entry = 0; // Debug. not threadsafe
+////static DetailEntry details[kMaxPairs * 10]; // Allow 10 details per bigram
+// End For debugging only
+
+// Must match kTestPrintableAsciiTildePlus exit codes, minus one
+enum PairSet {AsciiPair = 0, OtherPair = 1, NUM_PAIR_SETS = 2};
+
+// The reasons for pruning
+enum PruneReason {PRUNE_NORMAL, PRUNE_SLOWEND, PRUNE_FINAL};
+
+static const char* kWhatSetName[] = {"Ascii", "Other"};
+
+
+// State for encodings that do shift-out/shift-in between one- and two-byte
+// regions (ISO-2022-xx, HZ)
+enum StateSoSi {SOSI_NONE, SOSI_ERROR, SOSI_ONEBYTE, SOSI_TWOBYTE};
+
+typedef struct {
+ const uint8* initial_src; // For calculating byte offsets
+ const uint8* limit_src; // Range of input source
+ const uint8* prior_src; // Source consumed by prior call to BoostPrune
+ const uint8* last_pair; // Last pair inserted into interesting_pairs
+
+ DetailEntry* debug_data; // Normally NULL. Ptr to debug data for
+ // FLAGS_enc_detect_detail PostScript data
+ int next_detail_entry; // Debug
+
+ bool done;
+ bool reliable;
+ bool hints_derated;
+ int declared_enc_1; // From http/meta hint
+ int declared_enc_2; // from http/meta hint
+ int prune_count; // Number of times we have pruned
+
+ int trigram_highwater_mark; // Byte offset of last trigram processing
+ bool looking_for_latin_trigrams; // True if we should test for doing
+ // Latin1/2/7 trigram processing
+ bool do_latin_trigrams; // True if we actually are scoring trigrams
+
+ // Miscellaneous state variables for difficult encodings
+ int binary_quadrants_count; // Number of four bigram quadrants seen:
+ // 0xxxxxxx0xxxxxxx 0xxxxxxx1xxxxxx
+ // 1xxxxxxx0xxxxxxx 1xxxxxxx1xxxxxx
+ int binary_8x4_count; // Number of 8x4 buckets seen:
+ uint32 binary_quadrants_seen; // Bit[i] set if bigram i.......i....... seen
+ uint32 binary_8x4_seen; // Bit[i] set if bigram iii.....ii...... seen
+ int utf7_starts; // Count of possible UTF-7 beginnings seen
+ int prior_utf7_offset; // Source consumed by prior UTF-7 string
+ int next_utf8_ministate; // Mini state for UTF-8 sequences
+ int utf8_minicount[6]; // Number of correct 2- 3- 4-byte seq, errors
+ int next_utf8utf8_ministate; // Mini state for UTF8UTF8 sequences
+ int utf8utf8_odd_byte; // UTF8UTF8 seq has odd number of bytes
+ int utf8utf8_minicount[6]; // Number of correct 2- 3- 4-byte seq, errors
+ StateSoSi next_2022_state; // Mini state for 2022 sequences
+ StateSoSi next_hz_state; // Mini state for HZ sequences
+ bool next_eucjp_oddphase; // Mini state for EUC-JP sequences
+ int byte32_count[8]; // Count of top 3 bits of byte1 of bigram
+ // 0x1x 2x3x 4x5x 6x7x 8x9x AxBx CxDx ExFx
+ uint32 active_special; // Bits showing which special cases are active
+
+ Encoding tld_hint; // Top TLD encoding or UNKNOWN
+ Encoding http_hint; // What the document says about itself or
+ Encoding meta_hint; // UNKNOWN_ENCODING. BOM is initial byte
+ Encoding bom_hint; // order mark for UTF-xx
+
+ // small cache of previous interesting bigrams
+ int next_prior_bigram;
+ int prior_bigram[4];
+ int prior_binary[1];
+
+ int top_rankedencoding; // Top two probabilities and families
+ int second_top_rankedencoding;
+ int top_prob;
+ int second_top_prob;
+ int prune_difference; // Prune things this much below the top prob
+ int rankedencoding_list_len; // Number of active encodings
+ int rankedencoding_list[NUM_RANKEDENCODING]; // List of active encodings
+ //
+ int enc_prob[NUM_RANKEDENCODING]; // Cumulative probability per enc
+ // This is where all the action is
+ int hint_prob[NUM_RANKEDENCODING]; // Initial hint probabilities
+ int hint_weight[NUM_RANKEDENCODING]; // Number of hints for this enc
+
+ // Two sets -- one for printable ASCII, one for the rest
+ int prior_interesting_pair[NUM_PAIR_SETS]; // Pairs consumed by prior call
+ int next_interesting_pair[NUM_PAIR_SETS]; // Next pair to write
+ char interesting_pairs[NUM_PAIR_SETS][kMaxPairs * 2]; // Two bytes per pair
+ int interesting_offsets[NUM_PAIR_SETS][kMaxPairs]; // Src offset of pair
+ int interesting_weightshift[NUM_PAIR_SETS][kMaxPairs]; // weightshift of pair
+} DetectEncodingState;
+
+
+// Record a debug event that changes probabilities
+void SetDetailsEncProb(DetectEncodingState* destatep,
+ int offset, int best_enc, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = offset;
+ destatep->debug_data[next].best_enc = best_enc;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+// Record a debug event that changes probabilities, copy offset
+void SetDetailsEncProbCopyOffset(DetectEncodingState* destatep,
+ int best_enc, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
+ destatep->debug_data[next].best_enc = best_enc;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+// Record a debug event that changes probs and has simple text label
+void SetDetailsEncLabel(DetectEncodingState* destatep, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
+ destatep->debug_data[next].best_enc = -1;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+// Record a debug event that is just a text label, no change in probs
+void SetDetailsLabel(DetectEncodingState* destatep, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
+ destatep->debug_data[next].best_enc = -1;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->debug_data[next - 1].detail_enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+
+// Maps superset encodings to base, to see if 2 encodings are compatible
+// (Non-identity mappings are marked "-->" below.)
+static const Encoding kMapEncToBaseEncoding[] = {
+ ISO_8859_1, // 0: Teragram ASCII
+ ISO_8859_2, // 1: Teragram Latin2
+ ISO_8859_3, // 2: in BasisTech but not in Teragram
+ ISO_8859_4, // 3: Teragram Latin4
+ ISO_8859_5, // 4: Teragram ISO-8859-5
+ ISO_8859_6, // 5: Teragram Arabic
+ ISO_8859_7, // 6: Teragram Greek
+ MSFT_CP1255, // 7: Teragram Hebrew --> 36
+ ISO_8859_9, // 8: in BasisTech but not in Teragram
+ ISO_8859_10, // 9: in BasisTech but not in Teragram
+ JAPANESE_EUC_JP, // 10: Teragram EUC_JP
+ JAPANESE_SHIFT_JIS, // 11: Teragram SJS
+ JAPANESE_JIS, // 12: Teragram JIS
+ CHINESE_BIG5, // 13: Teragram BIG5
+ CHINESE_GB, // 14: Teragram GB
+ CHINESE_EUC_CN, // 15: Teragram EUC-CN
+ KOREAN_EUC_KR, // 16: Teragram KSC
+ UNICODE, // 17: Teragram Unicode
+ CHINESE_EUC_CN, // 18: Teragram EUC --> 15
+ CHINESE_EUC_CN, // 19: Teragram CNS --> 15
+ CHINESE_BIG5, // 20: Teragram BIG5_CP950 --> 13
+ JAPANESE_SHIFT_JIS, // 21: Teragram CP932 --> 11
+ UTF8, // 22
+ UNKNOWN_ENCODING, // 23
+ ISO_8859_1, // 24: ISO_8859_1 with all characters <= 127 --> 0
+ RUSSIAN_KOI8_R, // 25: Teragram KOI8R
+ RUSSIAN_CP1251, // 26: Teragram CP1251
+ ISO_8859_1, // 27: CP1252 aka MSFT euro ascii --> 0
+ RUSSIAN_KOI8_RU, // 28: CP21866 aka KOI8_RU, used for Ukrainian
+ MSFT_CP1250, // 29: CP1250 aka MSFT eastern european
+ ISO_8859_1, // 30: aka ISO_8859_0 aka ISO_8859_1 euroized --> 0
+ ISO_8859_9, // 31: used for Turkish
+ ISO_8859_13, // 32: used in Baltic countries --> 43
+ ISO_8859_11, // 33: aka TIS-620, used for Thai
+ ISO_8859_11, // 34: used for Thai --> 33
+ MSFT_CP1256, // 35: used for Arabic
+ MSFT_CP1255, // 36: Logical Hebrew Microsoft
+ MSFT_CP1255, // 37: Iso Hebrew Logical --> 36
+ MSFT_CP1255, // 38: Iso Hebrew Visual --> 36
+ CZECH_CP852, // 39
+ ISO_8859_2, // 40: aka ISO_IR_139 aka KOI8_CS --> 1
+ MSFT_CP1253, // 41: used for Greek, but NOT a superset of 8859-7
+ RUSSIAN_CP866, // 42
+ ISO_8859_13, // 43
+ ISO_2022_KR, // 44
+ CHINESE_GB, // 45 GBK --> 14
+ CHINESE_GB, // 46 GB18030 --> 14
+ CHINESE_BIG5, // 47 BIG5_HKSCS --> 13
+ ISO_2022_KR, // 48 ISO_2022_CN --> 44
+ TSCII, // 49 Indic encoding
+ TAMIL_MONO, // 50 Indic encoding - Tamil
+ TAMIL_BI, // 51 Indic encoding - Tamil
+ JAGRAN, // 52 Indic encoding - Devanagari
+ MACINTOSH_ROMAN, // 53
+ UTF7, // 54
+ BHASKAR, // 55 Indic encoding - Devanagari
+ HTCHANAKYA, // 56 Indic encoding - Devanagari
+ UTF16BE, // 57
+ UTF16LE, // 58
+ UTF32BE, // 59
+ UTF32LE, // 60
+ BINARYENC, // 61
+ HZ_GB_2312, // 62
+ UTF8UTF8, // 63
+ TAM_ELANGO, // 64 Elango - Tamil
+ TAM_LTTMBARANI, // 65 Barani - Tamil
+ TAM_SHREE, // 66 Shree - Tamil
+ TAM_TBOOMIS, // 67 TBoomis - Tamil
+ TAM_TMNEWS, // 68 TMNews - Tamil
+ TAM_WEBTAMIL, // 69 Webtamil - Tamil
+ KDDI_SHIFT_JIS, // 70 KDDI Shift_JIS
+ DOCOMO_SHIFT_JIS, // 71 DoCoMo Shift_JIS
+ SOFTBANK_SHIFT_JIS, // 72 SoftBank Shift_JIS
+ KDDI_ISO_2022_JP, // 73 KDDI ISO-2022-JP
+ SOFTBANK_ISO_2022_JP, // 74 SOFTBANK ISO-2022-JP
+};
+
+COMPILE_ASSERT(arraysize(kMapEncToBaseEncoding) == NUM_ENCODINGS,
+ kMapEncToBaseEncoding_has_incorrect_size);
+
+// Maps base encodings to 0, supersets to 1+, undesired to -1
+// (Non-identity mappings are marked "-->" below.)
+static const int kMapEncToSuperLevel[] = {
+ 0, // 0: Teragram ASCII
+ 0, // 1: Teragram Latin2
+ 0, // 2: in BasisTech but not in Teragram
+ 0, // 3: Teragram Latin4
+ 0, // 4: Teragram ISO-8859-5
+ 0, // 5: Teragram Arabic
+ 0, // 6: Teragram Greek
+ 0, // 7: Teragram Hebrew
+ 0, // 8: in BasisTech but not in Teragram
+ 0, // 9: in BasisTech but not in Teragram
+ 0, // 10: Teragram EUC_JP
+ 0, // 11: Teragram SJS
+ 0, // 12: Teragram JIS
+ 0, // 13: Teragram BIG5
+ 0, // 14: Teragram GB
+ 0, // 15: Teragram EUC-CN
+ 0, // 16: Teragram KSC
+ 0, // 17: Teragram Unicode
+ -1, // 18: Teragram EUC --> 15
+ -1, // 19: Teragram CNS --> 15
+ 1, // 20: Teragram BIG5_CP950 --> 13
+ 1, // 21: Teragram CP932 --> 11
+ 0, // 22
+ -1, // 23
+ -1, // 24: ISO_8859_1 with all characters <= 127 --> 0
+ 0, // 25: Teragram KOI8R
+ 0, // 26: Teragram CP1251
+ 1, // 27: CP1252 aka MSFT euro ascii --> 0
+ 0, // 28: CP21866 aka KOI8_RU, used for Ukrainian
+ 0, // 29: CP1250 aka MSFT eastern european
+ 1, // 30: aka ISO_8859_0 aka ISO_8859_1 euroized --> 0
+ 0, // 31: used for Turkish
+ 1, // 32: used in Baltic countries --> 43
+ 0, // 33: aka TIS-620, used for Thai
+ 1, // 34: used for Thai --> 33
+ 0, // 35: used for Arabic
+ 0, // 36: Logical Hebrew Microsoft
+ -1, // 37: Iso Hebrew Logical --> 36
+ -1, // 38: Iso Hebrew Visual --> 7
+ 0, // 39
+ 1, // 40: aka ISO_IR_139 aka KOI8_CS --> 1
+ 0, // 41: used for Greek, NOT superset of 8859-7
+ 0, // 42
+ 0, // 43
+ 0, // 44
+ 1, // 45 GBK --> 14
+ 1, // 46 GB18030 --> 14
+ 1, // 47 BIG5_HKSCS --> 13
+ 1, // 48 ISO_2022_CN --> 44
+ 0, // 49 Indic encoding
+ 0, // 50 Indic encoding - Tamil
+ 0, // 51 Indic encoding - Tamil
+ 0, // 52 Indic encoding - Devanagari
+ 0, // 53
+ 0, // 54
+ 0, // 55 Indic encoding - Devanagari
+ 0, // 56 Indic encoding - Devanagari
+ 0, // 57
+ 0, // 58
+ 0, // 59
+ 0, // 60
+ 0, // 61
+ 0, // 62
+ 2, // 63
+ 0, 0, 0, 0, 0, 0, // add six more Tamil
+ 0, 0, 0, 0, 0, // add five encodings with emoji
+};
+
+COMPILE_ASSERT(arraysize(kMapEncToSuperLevel) == NUM_ENCODINGS,
+ kMapEncToSuperLevel_has_incorrect_size);
+
+
+
+// Subscripted by Encoding enum value
+static const uint32 kSpecialMask[] = {
+ kHighAccentCode, // 0
+ kHighAccentCode,
+ kHighAccentCode,
+ kHighAccentCode,
+ kHighAlphaCode, // 4
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAccentCode,
+ kHighAccentCode,
+
+ kTwobyteCode + kEUCJPActive, // 10 euc-jp
+ kTwobyteCode,
+ kSevenBitActive + kIso2022Active, // jis
+ kTwobyteCode,
+ kTwobyteCode,
+ kTwobyteCode,
+ kTwobyteCode,
+ kSevenBitActive + kUTF1632Active, // Unicode
+ kTwobyteCode,
+ kTwobyteCode,
+
+ kTwobyteCode, // 20
+ kTwobyteCode,
+ kUTF8Active, // UTF-8
+ 0,
+ 0,
+ kHighAlphaCode, // 25
+ kHighAlphaCode,
+ kHighAccentCode,
+ kHighAlphaCode,
+ kHighAccentCode,
+
+ kHighAccentCode, // 30
+ kHighAccentCode,
+ kHighAccentCode,
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAlphaCode, // 35
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAlphaCode,
+ 0,
+
+ 0, // 40
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAccentCode,
+ kSevenBitActive + kIso2022Active, // 2022-kr
+ kTwobyteCode,
+ kTwobyteCode,
+ kTwobyteCode,
+ kSevenBitActive + kIso2022Active, // 2022-cn
+ kHighAlphaCode + kIsIndicCode, // 49 TSCII
+
+ kHighAlphaCode + kIsIndicCode, // 50 TAMIL_MONO
+ kHighAlphaCode + kIsIndicCode, // 51 TAMIL_BI
+ kHighAlphaCode + kIsIndicCode, // 52 JAGRAN
+ kHighAccentCode, // 53 MACINTOSH_ROMAN
+ kSevenBitActive + kUTF7Active, // 54 UTF-7
+ kHighAlphaCode + kIsIndicCode, // 55 BHASKAR Indic encoding - Devanagari
+ kHighAlphaCode + kIsIndicCode, // 56 HTCHANAKYA Indic encoding - Devanagari
+ kSevenBitActive + kUTF1632Active, // 57 UTF16BE
+ kSevenBitActive + kUTF1632Active, // 58 UTF16LE
+ kSevenBitActive + kUTF1632Active, // 59 UTF32BE
+ kSevenBitActive + kUTF1632Active, // 60 UTF32LE
+
+ kSevenBitActive + kBinaryActive, // 61 BINARYENC
+ kSevenBitActive + kHzActive, // 62 HZ_GB_2312
+ kHighAccentCode + kUTF8Active + kUTF8UTF8Active, // 63 UTF8UTF8
+ kHighAlphaCode + kIsIndicCode, // 64 Elango - Tamil
+ kHighAlphaCode + kIsIndicCode, // 65 Barani - Tamil
+ kHighAlphaCode + kIsIndicCode, // 66 Shree - Tamil
+ kHighAlphaCode + kIsIndicCode, // 67 TBoomis - Tamil
+ kHighAlphaCode + kIsIndicCode, // 68 TMNews - Tamil
+ kHighAlphaCode + kIsIndicCode, // 69 Webtamil - Tamil
+ kTwobyteCode, // 70 KDDI Shift_JIS
+ kTwobyteCode, // 71 DoCoMo Shift_JIS
+ kTwobyteCode, // 72 SoftBank Shift_JIS
+ kSevenBitActive + kIso2022Active, // 73 KDDI-ISO-2022-JP
+ kSevenBitActive + kIso2022Active, // 74 SOFTBANK-ISO-2022-JP
+};
+
+COMPILE_ASSERT(arraysize(kSpecialMask) == NUM_ENCODINGS,
+ kSpecialMask_has_incorrect_size);
+
+
+/***
+ kHighAlphaCode -- full alphabet in 8x-Fx range, not just accents
+
+ ISO_8859_5, // 4: Teragram ISO-8859-5 Cyrl UL bd
+ RUSSIAN_CP1251, // 26: Teragram CP1251 UL cdef
+ RUSSIAN_KOI8_R, // 25: Teragram KOI8R LU cdef
+ RUSSIAN_KOI8_RU, // 28: CP21866 aka KOI8_RU, LU cdef
+ RUSSIAN_CP866, // 42 89ae
+
+ ISO_8859_6, // 5: Teragram Arabic nocase cde
+ MSFT_CP1256, // 35: used for Arabic nocase cde
+
+ ISO_8859_7, // 6: Teragram Greek UL cdef
+ MSFT_CP1253, // 41: used for Greek UL cdef
+
+ ISO_8859_8, // 7: Teragram Hebrew nocase ef
+ MSFT_CP1255, // 36: Logical Hebrew Microsoft nocase ef
+ ISO_8859_8_I, // 37: Iso Hebrew Logical nocase ef
+ HEBREW_VISUAL, // 38: Iso Hebrew Visual nocase ef
+
+ ISO_8859_11, // 33: aka TIS-620, used for Thai nocase abcde
+ MSFT_CP874, // 34: used for Thai nocase abcde
+
+ TSCII, // 49 8-f
+ TAMIL_MONO, // 50
+ TAMIL_BI, // 51
+ JAGRAN, // 52
+ BHASKAR, // 55 Indic encoding - Devanagari
+ HTCHANAKYA, // 56 Indic encoding - Devanagari
+***/
+
+// We can scan bytes using this at about 500 MB/sec 2.8GHz P4
+// Slow scan uses this, stopping on NUL ESC SO SI bad C0 and + ~
+// We allow FF, 0x0C, here because it gives a better result for old
+// Ascii text formatted for a TTY
+// non-zero exits scan loop -- 1 for printable ASCII, 2 otherwise
+static const char kTestPrintableAsciiTildePlus[256] = {
+ 2,2,2,2,2,2,2,2, 2,0,0,2,0,0,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,1,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,1,2,
+
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+};
+
+// We can scan bytes using this at about 550 MB/sec 2.8GHz P4
+// Slow scan uses this, stopping on NUL ESC SO SI and bad C0
+// after Hz and UTF7 are pruned away
+// We allow Form Feed, 0x0C, here
+static const char kTestPrintableAscii[256] = {
+ 2,2,2,2,2,2,2,2, 2,0,0,2,0,0,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,2,
+
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+};
+
+// Used in first-four-byte testing
+static const char kIsPrintableAscii[256] = {
+ 0,0,0,0,0,0,0,0, 0,1,1,0,0,1,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,0,
+
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+
+static const signed char kBase64Value[256] = {
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,62,-1,-1,-1,63,
+ 52,53,54,55,56,57,58,59, 60,61,-1,-1,-1,-1,-1,-1,
+
+ -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,
+ 15,16,17,18,19,20,21,22, 23,24,25,-1,-1,-1,-1,-1,
+ -1,26,27,28,29,30,31,32, 33,34,35,36,37,38,39,40,
+ 41,42,43,44,45,46,47,48, 49,50,51,-1,-1,-1,-1,-1,
+
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+};
+
+
+// Subscripted by <state, byte/16>
+// Accepts Cx->8x Dx->8x Ex->8x->8x Fx->8x->8x->8x
+//
+// Fixed Problem: GB has sequences like B2DB B8D6 BDE1 B9B9
+// which we can mis-parse as an error byte followed by good UTF-8:
+// B2 DBB8 D6BD E1B9B9
+// To counteract this, we now require an ASCII7 byte to resync out
+// of the error state
+// Next problem: good UTF-8 with bad byte
+// efbc a012 eea4 bee7 b280 c2b7
+// efbca0 12 eea4be e7b280 c2b7
+// ^^ bad byte
+// fix: change state0 byte 1x to be don't-care
+//
+// Short UTF-8 ending in ASCII7 byte should resync immediately:
+// E0 20 E0 A6 AA should give one error and resync at 2nd E0
+//
+static const char kMiniUTF8State[8][16] = {
+ {0,0,0,0,0,0,0,0, 7,7,7,7,1,1,2,4,}, // [0] start char (allow cr/lf/ht)
+ {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [1] continue 1 of 2
+ {0,7,0,0,0,0,0,0, 3,3,3,3,7,7,7,7,}, // [2] continue 1 of 3
+ {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [3] continue 2 of 3
+ {0,7,0,0,0,0,0,0, 5,5,5,5,7,7,7,7,}, // [4] continue 1 of 4
+ {0,7,0,0,0,0,0,0, 6,6,6,6,7,7,7,7,}, // [5] continue 2 of 4
+ {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [6] continue 3 of 4
+ {0,7,0,0,0,0,0,0, 7,7,7,7,7,7,7,7,}, // [7] error, soak up continues,
+ // ONLY resync after Ascii char
+ // then restart
+};
+// Counter to increment: 0-don'tcare 1-error 2-good_2B 3-good_3B 4-good_4B
+static const char kMiniUTF8Count[8][16] = {
+ {0,0,0,0,0,0,0,0, 1,1,1,1,0,0,0,0,}, // [0] start char (allow cr/lf/ht)
+ {1,1,1,1,1,1,1,1, 2,2,2,2,1,1,1,1,}, // [1] continue 1 of 2
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [2] continue 1 of 3
+ {1,1,1,1,1,1,1,1, 3,3,3,3,1,1,1,1,}, // [3] continue 2 of 3
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [4] continue 1 of 4
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [5] continue 2 of 4
+ {1,1,1,1,1,1,1,1, 4,4,4,4,1,1,1,1,}, // [6] continue 3 of 4
+ {0,1,0,0,0,0,0,0, 1,1,1,1,1,1,1,1,}, // [7] error, soak up continues,
+ // then restart
+};
+
+// Subscripted by <state, f(byte1) + g(byte2)>
+// where f(x)= E2->4, Cx->8 and C3->12 and 0 otherwise
+// and g(x) = (x >> 4) & 3 8x->0 9x->1 Ax->2 Bx->3 Cx->0, etc.
+// (no checking for illegal bytes)
+// Here are example patterns of CP1252 converted to UTF-8 0/1/2 times. We want
+// to detect two, so we can back-convert to one.
+// zero one two pattern
+// ---- ------ ---------------- -----------------
+// 81 C281 C382C281 C3->8x->C2->xx
+// 98 CB9C C38BC593 C3->8x->C5->xx
+// C3 C383 C383C692 C3->8x->C6->xx
+// C8 C388 C383CB86 C3->8x->CB->xx
+// 83 C692 C386E28099 C3->8x->E2->xx->8x
+// 80 E282AC C3A2E2809AC2AC C3->A2->E2->xx->xx->Cx->xx
+// 92 E28099 C3A2E282ACE284A2 C3->A2->E2->xx->xx->E2->xx->xx
+//
+// We also want to detect bare-byte extra UTF-8 conversions:
+// zero one two pattern
+// ---- ------ ---------------- -----------------
+// C3 C3 C383 C3->8x->C2->xx
+// D3 D3 C393 C3->9x->C2->xx->C2->xx
+// E3 E3 C3A3 C3->Ax->C2->xx->C2->xx->C2->xx
+// F3 F3 C3B2 C3->Bx->C2->xx->C2->xx->C2->xx->C2->xx
+//
+
+/**
+CP1252 => UTF8 => UTF8UTF8
+80 => E282AC => C3A2E2809AC2AC
+81 => C281 => C382C281
+82 => E2809A => C3A2E282ACC5A1
+83 => C692 => C386E28099
+84 => E2809E => C3A2E282ACC5BE
+85 => E280A6 => C3A2E282ACC2A6
+86 => E280A0 => C3A2E282ACC2A0
+87 => E280A1 => C3A2E282ACC2A1
+88 => CB86 => C38BE280A0
+89 => E280B0 => C3A2E282ACC2B0
+8A => C5A0 => C385C2A0
+8B => E280B9 => C3A2E282ACC2B9
+8C => C592 => C385E28099
+8D => C28D => C382C28D
+8E => C5BD => C385C2BD
+8F => C28F => C382C28F
+90 => C290 => C382C290
+91 => E28098 => C3A2E282ACCB9C
+92 => E28099 => C3A2E282ACE284A2
+93 => E2809C => C3A2E282ACC593
+94 => E2809D => C3A2E282ACC29D
+95 => E280A2 => C3A2E282ACC2A2
+96 => E28093 => C3A2E282ACE2809C
+97 => E28094 => C3A2E282ACE2809D
+98 => CB9C => C38BC593
+99 => E284A2 => C3A2E2809EC2A2
+9A => C5A1 => C385C2A1
+9B => E280BA => C3A2E282ACC2BA
+9C => C593 => C385E2809C
+9D => C29D => C382C29D
+9E => C5BE => C385C2BE
+9F => C5B8 => C385C2B8
+A0 => C2A0 => C382C2A0
+A1 => C2A1 => C382C2A1
+A2 => C2A2 => C382C2A2
+A3 => C2A3 => C382C2A3
+A4 => C2A4 => C382C2A4
+A5 => C2A5 => C382C2A5
+A6 => C2A6 => C382C2A6
+A7 => C2A7 => C382C2A7
+A8 => C2A8 => C382C2A8
+A9 => C2A9 => C382C2A9
+AA => C2AA => C382C2AA
+AB => C2AB => C382C2AB
+AC => C2AC => C382C2AC
+AD => C2AD => C382C2AD
+AE => C2AE => C382C2AE
+AF => C2AF => C382C2AF
+B0 => C2B0 => C382C2B0
+B1 => C2B1 => C382C2B1
+B2 => C2B2 => C382C2B2
+B3 => C2B3 => C382C2B3
+B4 => C2B4 => C382C2B4
+B5 => C2B5 => C382C2B5
+B6 => C2B6 => C382C2B6
+B7 => C2B7 => C382C2B7
+B8 => C2B8 => C382C2B8
+B9 => C2B9 => C382C2B9
+BA => C2BA => C382C2BA
+BB => C2BB => C382C2BB
+BC => C2BC => C382C2BC
+BD => C2BD => C382C2BD
+BE => C2BE => C382C2BE
+BF => C2BF => C382C2BF
+C0 => C380 => C383E282AC
+C1 => C381 => C383C281
+C2 => C382 => C383E2809A
+C3 => C383 => C383C692
+C4 => C384 => C383E2809E
+C5 => C385 => C383E280A6
+C6 => C386 => C383E280A0
+C7 => C387 => C383E280A1
+C8 => C388 => C383CB86
+C9 => C389 => C383E280B0
+CA => C38A => C383C5A0
+CB => C38B => C383E280B9
+CC => C38C => C383C592
+CD => C38D => C383C28D
+CE => C38E => C383C5BD
+CF => C38F => C383C28F
+D0 => C390 => C383C290
+D1 => C391 => C383E28098
+D2 => C392 => C383E28099
+D3 => C393 => C383E2809C
+D4 => C394 => C383E2809D
+D5 => C395 => C383E280A2
+D6 => C396 => C383E28093
+D7 => C397 => C383E28094
+D8 => C398 => C383CB9C
+D9 => C399 => C383E284A2
+DA => C39A => C383C5A1
+DB => C39B => C383E280BA
+DC => C39C => C383C593
+DD => C39D => C383C29D
+DE => C39E => C383C5BE
+DF => C39F => C383C5B8
+E0 => C3A0 => C383C2A0
+E1 => C3A1 => C383C2A1
+E2 => C3A2 => C383C2A2
+E3 => C3A3 => C383C2A3
+E4 => C3A4 => C383C2A4
+E5 => C3A5 => C383C2A5
+E6 => C3A6 => C383C2A6
+E7 => C3A7 => C383C2A7
+E8 => C3A8 => C383C2A8
+E9 => C3A9 => C383C2A9
+EA => C3AA => C383C2AA
+EB => C3AB => C383C2AB
+EC => C3AC => C383C2AC
+ED => C3AD => C383C2AD
+EE => C3AE => C383C2AE
+EF => C3AF => C383C2AF
+F0 => C3B0 => C383C2B0
+F1 => C3B1 => C383C2B1
+F2 => C3B2 => C383C2B2
+F3 => C3B3 => C383C2B3
+F4 => C3B4 => C383C2B4
+F5 => C3B5 => C383C2B5
+F6 => C3B6 => C383C2B6
+F7 => C3B7 => C383C2B7
+F8 => C3B8 => C383C2B8
+F9 => C3B9 => C383C2B9
+FA => C3BA => C383C2BA
+FB => C3BB => C383C2BB
+FC => C3BC => C383C2BC
+FD => C3BD => C383C2BD
+FE => C3BE => C383C2BE
+FF => C3BF => C383C2BF
+**/
+
+// Subscripted by <state, f(byte1) + g(byte2)>
+// where f(x)= E2->4, C2/5/6/B->8 and C3->12 and 0 otherwise
+// and g(x) = (x >> 4) & 3 8x->0 9x->1 Ax->2 Bx->3 Cx->0, etc.
+
+// 81 C281 C382C281 C3->8x->C2->xx
+// 98 CB9C C38BC593 C3->8x->C5->xx
+// C3 C383 C383C692 C3->8x->C6->xx
+// C8 C388 C383CB86 C3->8x->CB->xx
+// [0] [2] [0]
+// 83 C692 C386E28099 C3->8x->E2->xx->xx
+// odd_byte=0 [0] [2] [0+] odd_byte flipped
+// odd_byte=1 [0+] [2] [0] [0] odd_byte unflipped
+// 80 E282AC C3A2E2809AC2AC C3->A2->E2->xx->xx->Cx->xx
+// odd_byte=0 [0] [3] [4] [0+]
+// odd_byte=1 [0+] [3] [4] [4] [0]
+// 92 E28099 C3A2E282ACE284A2 C3->A2->E2->xx->xx->E2->xx->xx
+// odd_byte=0 [0] [3] [4] [0] [0]
+// odd_byte=1 [0+] [3] [4] [4] [0+]
+//
+// When an E2xxxx sequence is encountered, we absorb the two bytes E2xx and flip
+// the odd_byte state. If that goes from 0 to 1, the next pair is offset up
+// by one byte, picking up the two bytes just after E2xxxx. If odd_byte goes
+// from 1 to 0, the next two bytes picked up are the two bytes xxxx of E2xxxx.
+// These are absorbed with no error in state 0 or state 4
+//
+// C3 C3 C383 C3->8x->C2->xx
+// D3 D3 C393 C3->9x->C2->xx->C2->xx
+// E3 E3 C3A3 C3->Ax->C2->xx->C2->xx->C2->xx
+// F3 F3 C3B2 C3->Bx->C2->xx->C2->xx->C2->xx->C2->xx
+// Counter3 for Fx Ex sequences is incremented at last C2
+
+static const char kMiniUTF8UTF8State[8][16] = {
+ // xxxx E2xx CXxx C3xx
+ // 8 9 a b 8 9 a b 8 9 a b
+ {0,0,0,0,1,1,1,1, 1,1,1,1,2,2,3,5,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
+ {0,0,0,0,1,1,1,1, 1,1,1,1,2,2,3,5,}, // [1] error, back to looking
+ {1,1,1,1,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [2] C38x looking for CXxx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,4,4,4,4, 7,7,7,7,1,1,1,1,}, // [3] C3Ax looking for E2xx or C2xxC2xx
+ // + + + + // E2xxxx flips odd_byte
+ {4,4,4,4,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,1,1,1,1, 6,6,6,6,1,1,1,1,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
+ {1,1,1,1,1,1,1,1, 7,7,7,7,1,1,1,1,}, // [6] C3Bx -- looking for C2xxC2xx
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [7] C3Bx -- looking for C2xx
+};
+// Counter to increment: 0-don'tcare 1-error 2-good_2B 3-good_3B 4-good_4B
+static const char kMiniUTF8UTF8Count[8][16] = {
+ // xxxx E2xx C2Xx C3xx
+ // 8 9 a b 8 9 a b 8 9 a b
+ {0,0,0,0,1,1,1,1, 1,1,1,1,0,0,0,0,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
+ {0,0,0,0,1,1,1,1, 1,1,1,1,0,0,0,0,}, // [1] error, back to looking
+ {1,1,1,1,3,3,3,3, 2,2,2,2,1,1,1,1,}, // [2] C38x looking for CXxx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [3] C3Ax looking for E2xx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,4,4,4,4, 4,4,4,4,1,1,1,1,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [6] C3Bx -- looking for C2xxC2xx
+ {1,1,1,1,1,1,1,1, 3,3,3,3,1,1,1,1,}, // [7] C3Bx -- looking for C2xx
+};
+
+static const char kMiniUTF8UTF8Odd[8][16] = {
+ // xxxx E2xx C2Xx C3xx
+ // 8 9 a b 8 9 a b 8 9 a b
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [1] error, back to looking
+ {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [2] C38x looking for CXxx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [3] C3Ax looking for E2xx
+ // + + + + // E2xxxx flips odd_byte
+ {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [6] C3Bx -- looking for C2xxC2xx
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [7] C3Bx -- looking for C2xx
+};
+
+// Turn a pair of bytes into the subscript for UTF8UTF8 tables above
+int UTF88Sub(char s0, char s1) {
+ int sub = (s1 >> 4) & 0x03;
+ uint8 u0 = static_cast<uint8>(s0);
+ if (u0 == 0xc3) {
+ sub += 12;
+ } else if ((u0 & 0xf0) == 0xc0) {
+ if ((u0 == 0xc2) || (u0 == 0xc5) || (u0 == 0xc6) || (u0 == 0xcb)) {
+ sub += 8;
+ }
+ } else if (u0 == 0xe2) {
+ sub += 4;
+ }
+ return sub;
+}
+
+
+
+
+
+// Default probability for an encoding rankedencoding
+// Based on a scan of 55M web pages
+// These values are 255 - log base 2**1/10 (occurrences / total)
+// Large values are most likely. This the reverse of some Google code
+// 255 = 1.0, 245 = 1/2, 235 = 1/4, 15 = 1/2**24, 0 = 0 (< 1/50M)
+//
+// TODO change this to be per encoding, not permuted
+//
+
+
+// Support function for unit test program
+// Return ranked encoding corresponding to enc
+// (also exported to compact_enc_det_text.cc)
+int CompactEncDet::BackmapEncodingToRankedEncoding(Encoding enc) {
+ for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
+ if (kMapToEncoding[i] == enc) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+
+string DecodeActive(uint32 active) {
+ string temp("");
+ if (active & kBinaryActive) {
+ temp.append("Binary ");
+ }
+ if (active & kUTF1632Active) {
+ temp.append("UTF1632 ");
+ }
+ if (active & kUTF8UTF8Active) {
+ temp.append("UTF8UTF8 ");
+ }
+ if (active & kUTF8Active) {
+ temp.append("UTF8 ");
+ }
+ if (active & kIso2022Active) {
+ temp.append("Iso2022 ");
+ }
+ if (active & kHzActive) {
+ temp.append("Hz ");
+ }
+ if (active & kUTF7Active) {
+ temp.append("UTF7A ");
+ }
+ if (active & kSevenBitActive) {
+ temp.append("SevenBit ");
+ }
+ if (active & kIsIndicCode) {
+ temp.append("Indic ");
+ }
+ if (active & kHighAlphaCode) {
+ temp.append("HighAlpha ");
+ }
+ if (active & kHighAccentCode) {
+ temp.append("HighAccent ");
+ }
+ if (active & kEUCJPActive) {
+ temp.append("EUCJP ");
+ }
+ return temp;
+}
+
+static inline bool SevenBitEncoding(int enc) {
+ return ((kSpecialMask[enc] & kSevenBitActive) != 0);
+}
+static inline bool TwoByteEncoding(int enc) {
+ return ((kSpecialMask[enc] & kTwobyteCode) != 0);
+}
+static inline bool IndicEncoding(int enc) {
+ return ((kSpecialMask[enc] & kIsIndicCode) != 0);
+}
+static inline bool HighAlphaEncoding(int enc) {
+ return ((kSpecialMask[enc] & kHighAlphaCode) != 0);
+}
+static inline bool HighAccentEncoding(int enc) {
+ return ((kSpecialMask[enc] & kHighAccentCode) != 0);
+}
+
+
+static inline bool AnyActive(DetectEncodingState* destatep) {
+ return (destatep->active_special != 0);
+}
+static inline bool SevenBitActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & kSevenBitActive) != 0;
+}
+static inline bool HzActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & kHzActive) != 0;
+}
+static inline bool Iso2022Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kIso2022Active) != 0;
+}
+static inline bool UTF8Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kUTF8Active) != 0;
+}
+static inline bool UTF8UTF8Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kUTF8UTF8Active) != 0;
+}
+static inline bool UTF1632Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kUTF1632Active) != 0;
+}
+static inline bool BinaryActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & kBinaryActive) != 0;
+}
+static inline bool UTF7OrHzActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & (kHzActive + kUTF7Active)) != 0;
+}
+static inline bool EUCJPActive(DetectEncodingState* destatep) {
+ return ((destatep->active_special & kEUCJPActive) != 0);
+}
+static inline bool OtherActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & (kIso2022Active + kBinaryActive +
+ kUTF8Active + kUTF8UTF8Active +
+ kUTF1632Active + kEUCJPActive)) != 0;
+}
+
+
+static inline bool CEDFlagRescanning(CEDInternalFlags flags) {
+ return (flags & kCEDRescanning) != 0;
+}
+
+static inline bool CEDFlagForceTags(CEDInternalFlags flags) {
+ return (flags & kCEDForceTags) != 0;
+}
+
+
+static inline int maxint(int a, int b) {return (a > b) ? a : b;}
+static inline int minint(int a, int b) {return (a < b) ? a : b;}
+
+static inline const char* MyRankedEncName(int r_enc) {
+ return MyEncodingName(kMapToEncoding[r_enc]);
+}
+
+
+// Only for debugging. not thread safe
+static const int kPsSourceWidth = 32;
+static int pssourcenext = 0; // debug only. not threadsafe. dump only >= this
+static int pssourcewidth = 0; // debug only.
+static char* pssource_mark_buffer = NULL;
+int next_do_src_line;
+int do_src_offset[16];
+
+
+void PsSourceInit(int len) {
+ pssourcenext = 0;
+ pssourcewidth = len;
+ delete[] pssource_mark_buffer;
+ // Allocate 2 Ascii characters per input byte
+ pssource_mark_buffer = new char[(pssourcewidth * 2) + 8]; // 8 = overscan
+ memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
+ memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
+
+ next_do_src_line = 0;
+ memset(do_src_offset, 0, sizeof(do_src_offset));
+}
+
+void PsSourceFinish() {
+ // Print preceding mark buffer
+ int j = (pssourcewidth * 2) - 1;
+ while ((0 <= j) && (pssource_mark_buffer[j] == ' ')) {--j;} // trim
+ pssource_mark_buffer[j + 1] = '\0';
+ fprintf(stderr, "( %s) do-src\n", pssource_mark_buffer);
+ memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
+ memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
+
+ delete[] pssource_mark_buffer;
+ pssource_mark_buffer = NULL;
+}
+
+// Dump aligned len bytes src... if not already dumped
+void PsSource(const uint8* src, const uint8* isrc, const uint8* srclimit) {
+ int offset = src - isrc;
+ offset -= (offset % pssourcewidth); // round down to multiple of len bytes
+ if (offset < pssourcenext) {
+ return;
+ }
+ pssourcenext = offset + pssourcewidth; // Min offset for next dump
+
+ // Print preceding mark buffer
+ int j = (pssourcewidth * 2) - 1;
+ while ((0 <= j) && (pssource_mark_buffer[j] == ' ')) {--j;} // trim
+ pssource_mark_buffer[j + 1] = '\0';
+ fprintf(stderr, "( %s) do-src\n", pssource_mark_buffer);
+ memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
+ memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
+
+ // Print source bytes
+ const uint8* src_aligned = isrc + offset;
+ int length = srclimit - src_aligned;
+ length = minint(pssourcewidth, length);
+
+ fprintf(stderr, "(%05x ", offset);
+ for (int i = 0; i < length; ++i) {
+ char c = src_aligned[i];
+ if (c == '\n') {c = ' ';}
+ if (c == '\r') {c = ' ';}
+ if (c == '\t') {c = ' ';}
+ if (c == '(') {
+ fprintf(stderr, "%s", "\\( ");
+ } else if (c == ')') {
+ fprintf(stderr, "%s", "\\) ");
+ } else if (c == '\\') {
+ fprintf(stderr, "%s", "\\\\ ");
+ } else if ((0x20 <= c) && (c <= 0x7e)) {
+ fprintf(stderr, "%c ", c);
+ } else {
+ fprintf(stderr, "%02x", c);
+ }
+ }
+ fprintf(stderr, ") do-src\n");
+ // Remember which source offsets are where, mod 16
+ do_src_offset[next_do_src_line & 0x0f] = offset;
+ ++next_do_src_line;
+}
+
+// Mark bytes in just-previous source bytes
+void PsMark(const uint8* src, int len, const uint8* isrc, int weightshift) {
+ int offset = src - isrc;
+ offset = (offset % pssourcewidth); // mod len bytes
+ char mark = (weightshift == 0) ? '-' : 'x';
+
+ pssource_mark_buffer[(offset * 2)] = '=';
+ pssource_mark_buffer[(offset * 2) + 1] = '=';
+ for (int i = 1; i < len; ++i) {
+ pssource_mark_buffer[(offset + i) * 2] = mark;
+ pssource_mark_buffer[((offset + i) * 2) + 1] = mark;
+ }
+}
+
+
+// Highlight trigram bytes in just-previous source bytes
+// Unfortunately, we have to skip back N lines since source was printed for
+// up to 8 bigrams before we get here. Match on src+1 to handle 0/31 better
+void PsHighlight(const uint8* src, const uint8* isrc, int trigram_val, int n) {
+ int offset = (src + 1) - isrc;
+ int offset32 = (offset % pssourcewidth); // mod len bytes
+ offset -= offset32; // round down to multiple of len bytes
+
+ for (int i = 1; i <= 16; ++i) {
+ if (do_src_offset[(next_do_src_line - i) & 0x0f] == offset) {
+ fprintf(stderr, "%d %d %d do-highlight%d\n",
+ i, offset32 - 1, trigram_val, n);
+ break;
+ }
+ }
+}
+
+
+void InitDetectEncodingState(DetectEncodingState* destatep) {
+ destatep->initial_src = NULL; // Filled in by caller
+ destatep->limit_src = NULL;
+ destatep->prior_src = NULL;
+ destatep->last_pair = NULL;
+
+ destatep->debug_data = NULL;
+ destatep->next_detail_entry = 0;
+
+ destatep->done = false;
+ destatep->reliable = false;
+ destatep->hints_derated = false;
+ //destatep->declared_enc_1 init in ApplyHints
+ //destatep->declared_enc_2 init in ApplyHints
+ destatep->prune_count = 0;
+
+ destatep->trigram_highwater_mark = 0;
+ destatep->looking_for_latin_trigrams = false;
+ destatep->do_latin_trigrams = false;
+
+ // Miscellaneous state variables for difficult encodings
+ destatep->binary_quadrants_count = 0;
+ destatep->binary_8x4_count = 0;
+ destatep->binary_quadrants_seen = 0;
+ destatep->binary_8x4_seen = 0;
+ destatep->utf7_starts = 0;
+ destatep->prior_utf7_offset = 0;
+ destatep->next_utf8_ministate = 0;
+ for (int i = 0; i < 6; i++) {destatep->utf8_minicount[i] = 0;}
+ destatep->next_utf8utf8_ministate = 0;
+ destatep->utf8utf8_odd_byte = 0;
+ for (int i = 0; i < 6; i++) {destatep->utf8utf8_minicount[i] = 0;}
+ destatep->next_2022_state = SOSI_NONE;
+ destatep->next_hz_state = SOSI_NONE;
+ destatep->next_eucjp_oddphase = false;
+ for (int i = 0; i < 8; i++) {destatep->byte32_count[i] = 0;}
+ destatep->active_special = 0xffffffff;
+ destatep->tld_hint = UNKNOWN_ENCODING;
+ destatep->http_hint = UNKNOWN_ENCODING;
+ destatep->meta_hint = UNKNOWN_ENCODING;
+ destatep->bom_hint = UNKNOWN_ENCODING;
+ destatep->top_rankedencoding = 0; // ASCII [seven-bit] is the default
+ destatep->second_top_rankedencoding = 0; // ASCII [seven-bit] is the default
+ destatep->top_prob = -1;
+ destatep->second_top_prob = -1;
+ // This is wide for first pruning, shrinks for 2nd and later
+ destatep->prune_difference = kInititalPruneDifference;
+
+ destatep->next_prior_bigram = 0;
+ destatep->prior_bigram[0] = -1;
+ destatep->prior_bigram[1] = -1;
+ destatep->prior_bigram[2] = -1;
+ destatep->prior_bigram[3] = -1;
+
+ destatep->prior_binary[0] = -1;
+
+ // Initialize with all but Indic encodings, which we never detect
+ int k = 0;
+ for (int rankedencoding = 0;
+ rankedencoding < NUM_RANKEDENCODING;
+ rankedencoding++) {
+ Encoding enc = kMapToEncoding[rankedencoding];
+ if (!IndicEncoding(enc)) {
+ destatep->rankedencoding_list[k++] = rankedencoding;
+ }
+ }
+ destatep->rankedencoding_list_len = k;
+
+ // This is where all the action is
+ memset(destatep->enc_prob, 0, sizeof(destatep->enc_prob));
+
+ memset(destatep->hint_prob, 0, sizeof(destatep->hint_prob));
+ memset(destatep->hint_weight, 0, sizeof(destatep->hint_weight));
+
+ destatep->prior_interesting_pair[AsciiPair] = 0;
+ destatep->prior_interesting_pair[OtherPair] = 0;
+ destatep->next_interesting_pair[AsciiPair] = 0;
+ destatep->next_interesting_pair[OtherPair] = 0;
+ // interesting_pairs/offsets/weightshifts not initialized; no need
+}
+
+// Probability strings are uint8, with zeros removed via simple run-length:
+// (<skip-take byte> <data bytes>)*
+// skip-take:
+// 00 end
+// x0 skip 16 x locations, take 0 data values
+// xy skip x locations, take y data values
+// Multiply all the incoming values by 3 to account for 3x unigram sums
+//
+// {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x35,
+// 0x01,0xc2,0x10,0x41,0xfe,0x71,0xba,0x00,}}, // "wind1255"
+//
+// Weight is 0..100 percent
+//
+// Returns subscript of largest (most probable) value
+//
+
+
+// {{0x6e,0x6c,0x5f,0x5f, 0x05,0xb2,0xae,0xa0,0x32,0xa1,0x36,0x31,0x42,0x39,0x3b,0x33,0x45,0x11,0x6f,0x00,}}, // "nl__"
+// // ASCII-7-bit=178 Latin1=174 UTF8=160 GB=50 CP1252=161 BIG5=49 Latin2=66 CP1251=57 CP1256=59 CP1250=51 Latin5=69 ISO-8859-15=111 [top ASCII-7-bit]
+int ApplyCompressedProb(const char* iprob, int len,
+ int weight, DetectEncodingState* destatep) {
+ int* dst = &destatep->enc_prob[0];
+ int* dst2 = &destatep->hint_weight[0];
+ const uint8* prob = reinterpret_cast<const uint8*>(iprob);
+ const uint8* problimit = prob + len;
+
+ int largest = -1;
+ int subscript_of_largest = 0;
+
+ // Continue with first byte and subsequent ones
+ while (prob < problimit) {
+ int skiptake = *prob++;
+ int skip = (skiptake & 0xf0) >> 4;
+ int take = skiptake & 0x0f;
+ if (skiptake == 00) {
+ break;
+ } else if (take == 0) {
+ dst += (skip << 4);
+ dst2 += (skip << 4);
+ } else {
+ dst += skip; // Normal case
+ dst2 += skip; // Normal case
+ for (int i = 0; i < take; i++) {
+ int enc = static_cast<int>(dst - &destatep->enc_prob[0]) + i;
+ if (largest < prob[i]) {
+ largest = prob[i];
+ subscript_of_largest = enc;
+ }
+
+ int increment = prob[i] * 3; // The actual increment
+
+ // Do maximum of previous hints plus this new one
+ if (weight > 0) {
+ increment = (increment * weight) / 100;
+ dst[i] = maxint(dst[i], increment);
+ dst2[i] = 1; // New total weight
+ }
+ }
+ prob += take;
+ dst += take;
+ dst2 += take;
+ }
+ }
+ return subscript_of_largest;
+}
+
+
+// Returns subscript of largest (most probable) value [for unit test]
+int TopCompressedProb(const char* iprob, int len) {
+ const uint8* prob = reinterpret_cast<const uint8*>(iprob);
+ const uint8* problimit = prob + len;
+ int next_prob_sub = 0;
+ int topprob = 0;
+ int toprankenc = 0;
+
+ while (prob < problimit) {
+ int skiptake = *prob++;
+ int skip = (skiptake & 0xf0) >> 4;
+ int take = skiptake & 0x0f;
+ if (skiptake == 0) {
+ break;
+ } else if (take == 0) {
+ next_prob_sub += (skip << 4);
+ } else {
+ next_prob_sub += skip; // Normal case
+ for (int i = 0; i < take; i++) {
+ if (topprob < prob[i]) {
+ topprob = prob[i];
+ toprankenc = next_prob_sub + i;
+ }
+ }
+ prob += take;
+ next_prob_sub += take;
+ }
+ }
+ return toprankenc;
+}
+
+
+// Find subscript of matching key in first 8 bytes of sorted hint array, or -1
+int HintBinaryLookup8(const HintEntry* hintprobs, int hintprobssize,
+ const char* norm_key) {
+ // Key is always in range [lo..hi)
+ int lo = 0;
+ int hi = hintprobssize;
+ while (lo < hi) {
+ int mid = (lo + hi) >> 1;
+ int comp = memcmp(&hintprobs[mid].key_prob[0], norm_key, 8);
+ if (comp < 0) {
+ lo = mid + 1;
+ } else if (comp > 0) {
+ hi = mid;
+ } else {
+ return mid;
+ }
+ }
+ return -1;
+}
+
+// Find subscript of matching key in first 4 bytes of sorted hint array, or -1
+int HintBinaryLookup4(const HintEntry* hintprobs, int hintprobssize,
+ const char* norm_key) {
+ // Key is always in range [lo..hi)
+ int lo = 0;
+ int hi = hintprobssize;
+ while (lo < hi) {
+ int mid = (lo + hi) >> 1;
+ int comp = memcmp(&hintprobs[mid].key_prob[0], norm_key, 4);
+ if (comp < 0) {
+ lo = mid + 1;
+ } else if (comp > 0) {
+ hi = mid;
+ } else {
+ return mid;
+ }
+ }
+ return -1;
+}
+
+static inline void Boost(DetectEncodingState* destatep, int r_enc, int boost) {
+ destatep->enc_prob[r_enc] += boost;
+}
+
+static inline void Whack(DetectEncodingState* destatep, int r_enc, int whack) {
+ destatep->enc_prob[r_enc] -= whack;
+}
+
+// Apply initial probability hint based on top level domain name
+// Weight is 0..100 percent
+// Return 1 if name match found
+int ApplyTldHint(const char* url_tld_hint, int weight,
+ DetectEncodingState* destatep) {
+ if (url_tld_hint[0] == '~') {
+ return 0;
+ }
+ string normalized_tld = MakeChar4(string(url_tld_hint));
+ int n = HintBinaryLookup4(kTLDHintProbs, kTLDHintProbsSize,
+ normalized_tld.c_str());
+ if (n >= 0) {
+ // TLD is four bytes, probability table is ~12 bytes
+ int best_sub = ApplyCompressedProb((const char *)&kTLDHintProbs[n].key_prob[kMaxTldKey],
+ kMaxTldVector, weight, destatep);
+ // Never boost ASCII7; do CP1252 instead
+ if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
+ destatep->declared_enc_1 = best_sub;
+ if (destatep->debug_data != NULL) {
+ // Show TLD hint
+ SetDetailsEncProb(destatep, 0, best_sub, url_tld_hint);
+ }
+ return 1;
+ }
+ return 0;
+}
+
+// Apply initial probability hint based on charset= name
+// Weight is 0..100 percent
+// Return 1 if name match found
+int ApplyCharsetHint(const char* charset_hint, int weight,
+ DetectEncodingState* destatep) {
+ if (charset_hint[0] == '~') {
+ return 0;
+ }
+ string normalized_charset = MakeChar44(string(charset_hint));
+ int n = HintBinaryLookup8(kCharsetHintProbs, kCharsetHintProbsSize,
+ normalized_charset.c_str());
+ if (n >= 0) {
+ // Charset is eight bytes, probability table is ~eight bytes
+ int best_sub = ApplyCompressedProb((const char *)&kCharsetHintProbs[n].key_prob[kMaxCharsetKey],
+ kMaxCharsetVector, weight, destatep);
+ // Never boost ASCII7; do CP1252 instead
+ if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
+ destatep->declared_enc_1 = best_sub;
+
+ // If first explicitly declared charset is confusable with Latin1/1252, put
+ // both declared forms in declared_enc_*, displacing Latin1/1252.
+ // This avoids a bit of Latin1 creep.
+ // Also boost the declared encoding and its pair
+ // TODO: This should all be folded into postproc-enc-detect.cc
+ if ((destatep->http_hint == UNKNOWN_ENCODING) &&
+ (destatep->meta_hint == UNKNOWN_ENCODING)) {
+ // This is the first charset=hint
+ switch (best_sub) {
+ case F_Latin2: // 8859-2 Latin2, east euro
+ destatep->declared_enc_2 = F_CP1250;
+ Boost(destatep, F_Latin2, kGentleOnePair);
+ Boost(destatep, F_CP1250, kGentleOnePair);
+ break;
+ case F_CP1250:
+ destatep->declared_enc_2 = F_Latin2;
+ Boost(destatep, F_Latin2, kGentleOnePair);
+ Boost(destatep, F_CP1250, kGentleOnePair);
+ break;
+
+ case F_Latin3: // 8859-3 Latin3, south euro, Esperanto
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_Latin3, kGentleOnePair);
+ break;
+
+ case F_Latin4: // 8859-4 Latin4, north euro
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_Latin4, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_5: // 8859-5 Cyrillic
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost 1251
+ Boost(destatep, F_ISO_8859_5, kGentleOnePair); // (too different)
+ break;
+ case F_CP1251:
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost -5
+ Boost(destatep, F_CP1251, kGentleOnePair); // (too different)
+ break;
+
+ case F_Arabic: // 8859-6 Arabic
+ destatep->declared_enc_2 = F_CP1256;
+ Boost(destatep, F_Arabic, kGentleOnePair);
+ Boost(destatep, F_CP1256, kGentleOnePair);
+ break;
+ case F_CP1256:
+ destatep->declared_enc_2 = F_Arabic;
+ Boost(destatep, F_Arabic, kGentleOnePair);
+ Boost(destatep, F_CP1256, kGentleOnePair);
+ break;
+
+ case F_Greek: // 8859-7 Greek
+ destatep->declared_enc_2 = F_CP1253;
+ Boost(destatep, F_Greek, kGentleOnePair);
+ Boost(destatep, F_CP1253, kGentleOnePair);
+ break;
+ case F_CP1253:
+ destatep->declared_enc_2 = F_Greek;
+ Boost(destatep, F_Greek, kGentleOnePair);
+ Boost(destatep, F_CP1253, kGentleOnePair);
+ break;
+
+ case F_Hebrew: // 8859-8 Hebrew
+ destatep->declared_enc_2 = F_CP1255;
+ Boost(destatep, F_Hebrew, kGentleOnePair);
+ Boost(destatep, F_CP1255, kGentleOnePair);
+ break;
+ case F_CP1255:
+ destatep->declared_enc_2 = F_Hebrew;
+ Boost(destatep, F_Hebrew, kGentleOnePair);
+ Boost(destatep, F_CP1255, kGentleOnePair);
+ break;
+
+ case F_Latin5: // 8859-9 Latin5, Turkish
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost 1254
+ Boost(destatep, F_Latin5, kGentleOnePair); // (too different)
+ break;
+ case F_CP1254:
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost Latin5
+ Boost(destatep, F_CP1254, kGentleOnePair); // (too different)
+ break;
+
+ case F_Latin6: // 8859-10 Latin6, Nordic
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_Latin6, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_11: // 8859-11 Thai,
+ destatep->declared_enc_2 = F_CP874;
+ Boost(destatep, F_ISO_8859_11, kGentleOnePair);
+ Boost(destatep, F_CP874, kGentleOnePair);
+ break;
+ case F_CP874:
+ destatep->declared_enc_2 = F_ISO_8859_11;
+ Boost(destatep, F_ISO_8859_11, kGentleOnePair);
+ Boost(destatep, F_CP874, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_13: // 8859-13 Latin7, Baltic
+ destatep->declared_enc_2 = F_CP1257;
+ Boost(destatep, F_ISO_8859_13, kGentleOnePair);
+ Boost(destatep, F_CP1257, kGentleOnePair);
+ break;
+ case F_CP1257:
+ destatep->declared_enc_2 = F_ISO_8859_13;
+ Boost(destatep, F_ISO_8859_13, kGentleOnePair);
+ Boost(destatep, F_CP1257, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_15: // 8859-15 Latin9, Latin0, Euro-ized Latin1
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_ISO_8859_15, kGentleOnePair);
+ break;
+
+
+ // Greek all-caps is confusable with KOI8x all-lower and Hebrew.
+ // This turns some Greek documents into Cyrillic, etc. by mistake.
+ // Greek and Hebrew are boosted explicitly above; do KOI8x here.
+ // Boosting the declared encodingmakes it harder for the wrong one to
+ // creep up.
+ case F_KOI8R:
+ Boost(destatep, F_KOI8R, kGentleOnePair);
+ break;
+ case F_KOI8U:
+ Boost(destatep, F_KOI8U, kGentleOnePair);
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ // Show charset hint
+ SetDetailsEncProb(destatep, 0, best_sub, charset_hint);
+ }
+
+ //
+ // Some fix-ups for the declared encodings
+ //
+
+ // If non-UTF8, non-Latin1/1252 encoding declared, disable UTF8 combos
+ // TODO: This should all be folded into postproc-enc-detect.cc
+ if ((best_sub != F_UTF8) &&
+ (best_sub != F_Latin1) &&
+ (best_sub != F_CP1252)) {
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 4); // demote
+ }
+
+ // Latin2 and CP1250 differ in the overlap part, such as B1 or B9
+ // The initial probabilites for charset=Latin2 explicitly put CP1250
+ // down twice as far as normal, and vice versa. This is done in
+ // postproc-enc-detect.cc
+
+ // If charset=user-defined, treat as Binary --
+ // we can safely only do low ASCII, might be Indic
+ if (normalized_charset.substr(0,4) == "user") {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ }
+
+ return 1;
+ }
+ return 0;
+}
+
+// Apply initial probability hint based on caller-supplied encoding
+// Negative hint whacks ~encoding, non-negative boosts encoding
+//
+// Negative hints are an experiment to see if they might be useful.
+// Not operator used instead of unary minus to allow specifying not-zero
+int ApplyEncodingHint(const int encoding_hint, int weight,
+ DetectEncodingState* destatep) {
+ Encoding enc_hint = static_cast<Encoding>((encoding_hint < 0) ?
+ ~encoding_hint : encoding_hint);
+ // Map to the right internal subscript
+ int rankedenc_hint = CompactEncDet::BackmapEncodingToRankedEncoding(enc_hint);
+
+ // I'm not sure how strong this hint should be. Weight 100% = 1 bigram
+ int increment = (kBoostOnePair * weight) / 100;
+
+ if (encoding_hint < 0) {
+ destatep->enc_prob[rankedenc_hint] -= increment;
+ } else {
+ destatep->enc_prob[rankedenc_hint] += increment;
+ }
+
+ if (destatep->debug_data != NULL) {
+ // Show encoding hint
+ SetDetailsEncProb(destatep, 0, -1, MyEncodingName(enc_hint));
+ }
+ return 1;
+}
+
+// Apply initial probability hint based on user interface language
+// Weight is 0..100 percent
+// Return 1 if name match found
+int ApplyUILanguageHint(const Language language_hint,
+ int weight, DetectEncodingState* destatep) {
+ if (language_hint == UNKNOWN_LANGUAGE) {
+ return 0;
+ }
+ string normalized_lang = MakeChar8(LanguageName(language_hint));
+ int n = HintBinaryLookup8(kLangHintProbs, kLangHintProbsSize,
+ normalized_lang.c_str());
+ if (n >= 0) {
+ // Language is eight bytes, probability table is ~eight bytes
+ int best_sub = ApplyCompressedProb((const char *)&kLangHintProbs[n].key_prob[kMaxLangKey],
+ kMaxLangVector, weight, destatep);
+ // Never boost ASCII7; do CP1252 instead
+ if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
+ destatep->declared_enc_1 = best_sub;
+ if (destatep->debug_data != NULL) {
+ // Show language hint
+ SetDetailsEncProb(destatep, 0, best_sub, normalized_lang.c_str());
+ }
+ return 1;
+ }
+ return 0;
+}
+
+// Apply initial probability hint based on corpus type (web, email, etc)
+// Return 1 if name match found
+int ApplyDefaultHint(const CompactEncDet::TextCorpusType corpus_type,
+ DetectEncodingState* destatep) {
+
+ for (int i = 0; i < NUM_RANKEDENCODING; i++) {
+ // Set the default probability
+ destatep->enc_prob[i] = kDefaultProb[i] * 3;
+ // Deliberately set 2022 seven-bit encodings to zero,
+ // so we can look for actual use
+ // TODO: This should all be folded into postproc-enc-detect.cc
+ if (SevenBitEncoding(kMapToEncoding[i])) {
+ destatep->enc_prob[i] = 0;
+ }
+ }
+
+ // A little corpus distinction
+ switch (corpus_type) {
+ case CompactEncDet::WEB_CORPUS:
+ case CompactEncDet::XML_CORPUS:
+ // Allow double-converted UTF-8 to start nearly equal to normal UTF-8
+ destatep->enc_prob[F_UTF8UTF8] =
+ destatep->enc_prob[F_UTF8] - kSmallInitDiff;
+ break;
+ case CompactEncDet::QUERY_CORPUS:
+ case CompactEncDet::EMAIL_CORPUS:
+ default:
+ break;
+ }
+
+ if (FLAGS_demo_nodefault) {
+ // Demo, make initial probs all zero
+ for (int i = 0; i < NUM_RANKEDENCODING; i++) {
+ destatep->enc_prob[i] = 0;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ // Show default hint
+ SetDetailsEncProb(destatep, 0, -1, "Default");
+ }
+ return 1;
+}
+
+
+
+// Do reverse search for c in [str..str+len)
+// Note: initial pointer is to FRONT of string, not back
+const char* MyMemrchr(const char* str, char c, size_t len) {
+ const char* ret = str + len;
+ while (str <= --ret) {
+ if (*ret == c) {return ret;}
+ }
+ return NULL;
+}
+
+
+// Minimum real URL is 11 bytes: "http://a.bc" -- shorter is assumed to be TLD
+// Now that we are no longer trying to do Indic font-based encodigns, we
+// don't need the full URL and can go back to simple TLD. This test remains for
+// backwards compatility with any caller using full URL.
+static const int kMinURLLength = 11;
+
+// Extract TLD from a full URL or just a TLD
+// Return hostname and length if a full URL
+void ExtractTLD(const char* url_hint, char* tld_hint, int tld_hint_len,
+ const char** ret_host_start, int* ret_host_len) {
+ // url_hint can either be a full URL (preferred) or just top-level domain name
+ // Extract the TLD from a full URL and use it for
+ // a normal TLD hint
+
+ strncpy(tld_hint, "~", tld_hint_len);
+ tld_hint[tld_hint_len - 1] = '\0';
+ *ret_host_start = NULL;
+ *ret_host_len = 0;
+
+ int url_len = (url_hint != NULL) ? strlen(url_hint) : 0;
+ if (url_len == 0) {
+ // Empty TLD
+ return;
+ }
+
+ // Minimum real URL is 11 bytes: "http://a.bc" -- shorter is assumed to be TLD
+ if (kMinURLLength <= url_len) {
+ // See if it really is a URL
+ const char* first_slash = strchr(url_hint, '/');
+ if ((first_slash != NULL) && (first_slash != url_hint) &&
+ (first_slash[-1] == ':') && (first_slash[1] == '/') &&
+ (memrchr(url_hint, '.', first_slash - url_hint) == NULL)) {
+ // We found :// and no dot in front of it, so declare a real URL
+
+ const char* hostname_start = first_slash + 2;
+ const char* hostname_end = strchr(hostname_start, '/');
+ if (hostname_end == NULL) {
+ // No slash; end is first byte off end of the URL string
+ hostname_end = url_hint + url_len;
+ }
+ size_t hostname_len = hostname_end - hostname_start;
+ const char* port_start =
+ (const char*)memchr(hostname_start, ':', hostname_len);
+ if (port_start != NULL) {
+ // Port; shorten hostname
+ hostname_end = port_start;
+ hostname_len = hostname_end - hostname_start;
+ }
+
+ const char* tld_start = MyMemrchr(hostname_start, '.', hostname_len);
+ if (tld_start != NULL) {
+ // Remember the TLD we just found
+ int tld_len = hostname_start + hostname_len - tld_start - 1;
+ if (tld_len > (tld_hint_len - 1)) {
+ tld_len = tld_hint_len - 1;
+ }
+ memcpy(tld_hint, tld_start + 1, tld_len);
+ tld_hint[tld_len] = '\0';
+ }
+ *ret_host_start = hostname_start;
+ *ret_host_len = hostname_len;
+ return;
+ }
+ } else {
+ strncpy(tld_hint, url_hint, tld_hint_len);
+ tld_hint[tld_hint_len - 1] = '\0';
+ }
+}
+
+// Apply hints, if any, to probabilities
+// NOTE: Encoding probabilites are all zero at this point
+void ApplyHints(const char* url_hint,
+ const char* http_charset_hint,
+ const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint,
+ const CompactEncDet::TextCorpusType corpus_type,
+ DetectEncodingState* destatep) {
+ int hint_count = 0;
+ // url_hint can either be a full URL (preferred) or just top-level domain name
+ // Extract the TLD from a full URL and use it for
+ // a normal TLD hint
+
+ char tld_hint[16];
+ const char* hostname_start = NULL;
+ int hostname_len = 0;
+ ExtractTLD(url_hint, tld_hint, sizeof(tld_hint),
+ &hostname_start, &hostname_len);
+
+
+ // Initial hints give slight boost to Ascii-7-bit and code page 1252
+ // ApplyXxx routines copy enc_1 to enc_2 then update declared_enc_1
+ // This gives a boost to 1252 if one of HTTP/META is specified,
+ // but this could be the wrong thing to do if Latin2/3/4/etc. is specified
+ destatep->declared_enc_1 = F_CP1252;
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+
+ // Applying various hints takes max of new hint and any old hint.
+ // This does better on multiple hints that a weighted average
+
+ // Weight is 0..100 percent
+ if ((http_charset_hint != NULL) && (http_charset_hint[0] != '~')) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyCharsetHint(http_charset_hint, 100, destatep);
+ destatep->http_hint = kMapToEncoding[destatep->declared_enc_1];
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true;
+ }
+ }
+ if ((meta_charset_hint != NULL) && (meta_charset_hint[0] != '~')) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyCharsetHint(meta_charset_hint, 100, destatep);
+ destatep->meta_hint = kMapToEncoding[destatep->declared_enc_1];
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true;
+ }
+ }
+ if (encoding_hint != UNKNOWN_ENCODING) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyEncodingHint(encoding_hint, 50, destatep);
+ }
+ if (language_hint != UNKNOWN_LANGUAGE) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyUILanguageHint(language_hint, 50, destatep);
+ }
+ // Use top level domain if not .com and <=1 other hint was available
+ if (url_hint != NULL) {
+ destatep->tld_hint = CompactEncDet::TopEncodingOfTLDHint(tld_hint);
+ if (hint_count == 0) {
+ // Apply with weight 100%
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyTldHint(tld_hint, 100, destatep);
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true;
+ }
+ if (strcmp("hu", tld_hint) == 0) {
+ // Hungarian is particularly difficult to separate Latin2 from Latin1,
+ // so always look for trigram scanning if bare TLD=hu hint
+ destatep->looking_for_latin_trigrams = true;
+ }
+ // Treat .com as no TLD hint at all
+ } else if ((hint_count == 1) && (strcmp("com", tld_hint) != 0)) {
+ // Either shift weighting or consider doing no TLD here -- seems to
+ // distract from correct charset= hints. Or perhaps apply only if
+ // charset = Latin1/1252...
+ // Apply with weight 50%
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyTldHint(tld_hint, 50, destatep);
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true; // These need trigrams
+ }
+ }
+ // Else ignore TLD hint entirely
+ }
+
+ // Use all-web default distribution if not even a TLD hint
+ if (hint_count == 0) {
+ destatep->looking_for_latin_trigrams = true; // Default needs trigrams
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyDefaultHint(corpus_type, destatep);
+ }
+
+
+// ISO-Microsoft Pairs
+// F_Latin1, F_CP1252,
+// F_Latin2, F_CP1250, NOT really strict subset/superset pairs
+// F_Latin3,
+// F_Latin4,
+// F_ISO_8859_5, F_CP1251,
+// F_Arabic, F_CP1256, NOT
+// F_Greek, F_CP1253, NOT really pairs
+// (or upgrade incvt to make Greek use CP)
+// F_Hebrew, F_CP1255, NOT really pairs
+// F_Latin5, F_CP1254,
+// F_Latin6,
+// F_ISO_8859_11,
+// F_ISO_8859_13, F_CP1257,
+// F_ISO_8859_15,
+// ISO-Microsoft Pairs
+
+ // Get important families started together
+ // // This should fall out of the initializatoin vectors for charset,
+ // but we need to get rid of families alltogetrher
+ //
+ // TODO make this more graceful
+
+ // Add small bias for subsets
+
+ // Subtract small bias for supersets
+ destatep->enc_prob[F_CP932] = destatep->enc_prob[F_SJS] - kSmallInitDiff;
+
+ destatep->enc_prob[F_GBK] = destatep->enc_prob[F_GB] - kSmallInitDiff;
+ destatep->enc_prob[F_GB18030] = destatep->enc_prob[F_GB] - kSmallInitDiff;
+
+ destatep->enc_prob[F_BIG5_CP950] = destatep->enc_prob[F_BIG5] -
+ kSmallInitDiff;
+ destatep->enc_prob[F_BIG5_HKSCS] = destatep->enc_prob[F_BIG5] -
+ kSmallInitDiff;
+
+ // Deliberate over-bias Ascii7 and underbias Binary [unneeded]
+ // destatep->enc_prob[F_ASCII_7_bit] = destatep->enc_prob[F_ASCII_7_bit] + kSmallInitDiff;
+ // destatep->enc_prob[F_BINARY] = destatep->enc_prob[F_BINARY] - (kBoostInitial / 2);
+
+ if (destatep->debug_data != NULL) {
+ // Show state at end of hints
+ SetDetailsEncProb(destatep, 0, -1, "Endhints");
+ if(FLAGS_enc_detect_detail2) {
+ // Add a line showing the watched encoding(s)
+ if (watch1_rankedenc >= 0) {
+ SetDetailsEncProb(destatep, 0,
+ watch1_rankedenc, FLAGS_enc_detect_watch1);
+ }
+ if (watch2_rankedenc >= 0) {
+ SetDetailsEncProb(destatep, 0,
+ watch2_rankedenc, FLAGS_enc_detect_watch2);
+ }
+ } // End detail2
+ }
+
+ // If duplicate hints, set second one to ASCII_7BIT to prevent double-boost
+ if (destatep->declared_enc_1 == destatep->declared_enc_2) {
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ }
+
+ if (FLAGS_force127) {
+ destatep->do_latin_trigrams = true;
+ if (FLAGS_enc_detect_source) {
+ PsHighlight(0, destatep->initial_src, 0, 2);
+ }
+ }
+
+
+ if (FLAGS_counts && destatep->looking_for_latin_trigrams) {++looking_used;}
+ if (FLAGS_counts && destatep->do_latin_trigrams) {++doing_used;}
+
+ //
+ // At this point, destatep->enc_prob[] is an initial probability vector based
+ // on the given hints/default. In general, it spreads out least-likely
+ // encodings to be about 2**-25 below the most-likely encoding.
+ // For input text with lots of bigrams, an unlikely encoding can rise to
+ // the top at a rate of about 2**6 per bigram, and more commonly 2**2 per
+ // bigram. So more than 4 bigrams and commonly more than 12 are
+ // needed to overcome the initial hints when the least-likely encoding
+ // is in fact the correct answer. So if the entire text has very few bigrams
+ // (as a two-word query might), it can be impossible for the correct
+ // encoding to win.
+ //
+ // To compensate for this, we take the initial hint vector and effectively
+ // apply it at the rate of 1/16 every bigram for the first 16 bigrams. The
+ // actual mechanism is done just before the last prune.
+ //
+
+ // Remember Initial hint probabilities
+ memcpy(destatep->hint_prob, destatep->enc_prob, sizeof(destatep->enc_prob));
+}
+
+// Look for specific high-value patterns in the first 4 bytes
+// Byte order marks (BOM)
+// EFBBBF UTF-8
+// FEFF UTF-16 BE
+// FFFE UTF-16 LE
+// FFFE0000 UTF-32 BE
+// 0000FEFF UTF-32 LE
+//
+// Likely UTF-x of seven-bit ASCII
+// 00xx UTF-16 BE xx printable ASCII
+// xx00 UTF-16 LE
+// 000000xx UTF-32 BE
+// xx000000 UTF-32 LE
+//
+void InitialBytesBoost(const uint8* src,
+ int text_length,
+ DetectEncodingState* destatep) {
+ if (text_length < 4) {return;}
+
+ uint32 pair01 = (src[0] << 8) | src[1];
+ uint32 pair23 = (src[2] << 8) | src[3];
+ uint32 quad0123 = (pair01 << 16) | pair23;
+
+ bool utf_16_indication = false;
+ bool utf_32_indication = false;
+ int best_enc = -1;
+
+ // Byte order marks
+ // UTF-8
+ if ((quad0123 & 0xffffff00) == 0xEFBBBF00) {
+ destatep->bom_hint = UTF8;
+ Boost(destatep, F_UTF8, kBoostInitial * 2);
+ Boost(destatep, F_UTF8UTF8, kBoostInitial * 2);
+ best_enc = F_UTF8;
+ // UTF-32 (test before UTF-16)
+ } else if (quad0123 == 0x0000FEFF) {
+ destatep->bom_hint = UTF32BE;
+ Boost(destatep, F_UTF_32BE, kBoostInitial * 2);
+ best_enc = F_UTF_32BE;
+ } else if (quad0123 == 0xFFFE0000) {
+ destatep->bom_hint = UTF32LE;
+ Boost(destatep, F_UTF_32LE, kBoostInitial * 2);
+ best_enc = F_UTF_32LE;
+ // UTF-16
+ } else if (pair01 == 0xFEFF) {
+ destatep->bom_hint = UTF16BE;
+ Boost(destatep, F_UTF_16BE, kBoostInitial * 3);
+ best_enc = F_UTF_16BE;
+ } else if (pair01 == 0xFFFE) {
+ destatep->bom_hint = UTF16LE;
+ Boost(destatep, F_UTF_16LE, kBoostInitial * 3);
+ best_enc = F_UTF_16LE;
+
+ // Possible seven-bit ASCII encoded as UTF-16/32
+ // UTF-32 (test before UTF-16)
+ } else if (((quad0123 & 0xffffff00) == 0) &&
+ (kIsPrintableAscii[src[3]] != 0)) {
+ Boost(destatep, F_UTF_32BE, kBoostInitial);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal char
+ best_enc = F_UTF_32BE;
+ } else if (((quad0123 & 0x00ffffff) == 0) &&
+ (kIsPrintableAscii[src[0]] != 0)) {
+ Boost(destatep, F_UTF_32LE, kBoostInitial);
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
+ best_enc = F_UTF_32LE;
+ } else if ((src[0] == 0x00) && (kIsPrintableAscii[src[1]] != 0)) {
+ Boost(destatep, F_UTF_16BE, kBoostInitial);
+ best_enc = F_UTF_16BE;
+ } else if ((src[1] == 0x00) && (kIsPrintableAscii[src[0]] != 0)) {
+ Boost(destatep, F_UTF_16LE, kBoostInitial);
+ best_enc = F_UTF_16LE;
+
+ // Whack if 0000 or FFFF
+ // UTF-32 (test before UTF-16)
+ } else if (quad0123 == 0x00000000) {
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_32LE, kBadPairWhack);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+ } else if (quad0123 == 0xffffffff) {
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_32LE, kBadPairWhack);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+ } else if (pair01 == 0x0000) {
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+ } else if (pair01 == 0xffff) {
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+
+
+ // These are the first four bytes of some known binary file formats
+
+ // Boost BINARY bigtime if JPEG FFD8FFxx
+ // Boost BINARY bigtime if png 89504E47 (.PNG)
+ // Boost BINARY bigtime if gif 47494638 (GIF8)
+ // Boost BINARY bigtime if zip 504B0304 (PK..)
+ // Boost BINARY bigtime if gzip 1F8B08xx
+ // Boost BINARY bigtime if gzip 78DAxxxx
+ // Boost BINARY if PDF 25504446 (%PDF)
+ // Boost BINARY if SWF (FWSx or CWSx where x <= 0x1f)
+ } else if ((quad0123 & 0xffffff00) == 0xFFD8FF00) { // JPEG FFD8FFxx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x89504E47) { // Hex 89 P N G
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x47494638) { // Hex GIF8
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x504B0304) { // Hex P K 03 04
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((quad0123 & 0xffffff00) == 0x1F8B0800) { // gzip 1F8B08xx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (pair01 == 0x78DA) { // gzip 78DAxxxx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x25504446) { // Hex %PDF
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((quad0123 & 0xffffff1f) == 0x66535700) { // Hex FWSx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((quad0123 & 0xffffff1f) == 0x63535700) { // Hex CWSx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+
+ // More binary detect prefixes
+ // 7F E L F Executable and linking format
+ // M M 00 * TIFF (little-endian)
+ // * 00 M M TIFF (big-endian)
+ // 01 f c p Final cut pro
+ } else if (quad0123 == 0x7F454C46) { // Hex 7F E L F
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x4D4D002A) { // Hex M M 00 *
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x2A004D4D) { // Hex * 00 M M
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x01666370) { // Hex 01 f c p
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+
+ // More binary detect prefixes; all-ASCII names; heavy weight to avoid ASCII
+ // prefix overcoming binary
+ // C C S D USGS ISIS 3-D cube files
+ // S I M P FITS image header "SIMPLE "
+ } else if (quad0123 == 0x43435344) { // Hex C C S D
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x53494D50) { // Hex S I M P
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+
+ // More binary detect prefixes; all-ASCII names; lighter weight
+ // H W P Hangul word processor
+ // 8 B P S Photoshop
+ // P D S _ xx "PDS_VERSION_ID "
+ } else if (quad0123 == 0x48575020) { // Hex H W P
+ if ((19 <= text_length) &&
+ (memcmp(src, "HWP.Document.File.V", 19) == 0)) {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((19 <= text_length) &&
+ (memcmp(src, "HWP Document File V", 19) == 0)) {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else {
+ Boost(destatep, F_BINARY, kBoostInitial * kWeakerBinary);
+ }
+ } else if (quad0123 == 0x38425053) { // Hex 8 B P S
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x5044535F) { // Hex P D S _
+ if ((14 <= text_length) && (memcmp(src, "PDS_VERSION_ID", 14) == 0)) {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else {
+ Boost(destatep, F_BINARY, kBoostInitial * kWeakerBinary);
+ }
+ }
+
+ // There are several main Windows EXE file formats.
+ // Not examined here (prefix too short; never see them in Google pipeline)
+ // M Z DOS .exe Mark Zbikowski
+ // N E DOS 4.0 16-bit
+ // L E OS/2 VxD drivers
+ // L X OS/2
+ // P E Windows NT
+
+
+ // More user-defined
+ // http://www.freenet.am/armscii/ Armenian
+
+ // If any hints or BOM, etc. keep UTF 16/32 around
+ if ((destatep->enc_prob[F_UTF_16BE] > 0) ||
+ (destatep->enc_prob[F_UTF_16LE] > 0)) {
+ utf_16_indication = true;
+ }
+ if ((destatep->enc_prob[F_UTF_32BE] > 0) ||
+ (destatep->enc_prob[F_UTF_32LE] > 0)) {
+ utf_32_indication = true;
+ }
+
+
+ // Kill UTF16/32 right now if no positive indication of them
+ // Otherwise, they tend to rise to the top in 7-bit files with an
+ // occasional 0x02 byte in some comment or javascript
+ if (!utf_16_indication) {
+ Whack(destatep, F_UTF_16BE, kBadPairWhack * 8);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack * 8);
+ Whack(destatep, F_Unicode, kBadPairWhack * 8);
+ }
+ if (!utf_32_indication) {
+ Whack(destatep, F_UTF_32BE, kBadPairWhack * 8);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack * 8);
+ }
+
+ // Usually kill mixed encodings
+ if (!FLAGS_ced_allow_utf8utf8) {
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8);
+ }
+ // 2011.11.07 never use UTF8CP1252 -- answer will be UTF8 instead
+ Whack(destatep, F_UTF8CP1252, kBadPairWhack * 8);
+
+ if (destatep->debug_data != NULL) {
+ // Show first four bytes of the input
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%04x%04x", pair01, pair23);
+ SetDetailsEncProb(destatep, 0, best_enc, buff);
+ }
+}
+
+
+
+// Descending order
+int IntCompare(const void* v1, const void* v2) {
+ const int* p1 = reinterpret_cast<const int*>(v1);
+ const int* p2 = reinterpret_cast<const int*>(v2);
+ if (*p1 < *p2) {return 1;}
+ if (*p1 > *p2) {return -1;}
+ return 0;
+}
+
+bool Base64Char(uint8 c) {
+ if (('A' <= c) && (c <= 'Z')) {return true;}
+ if (('a' <= c) && (c <= 'z')) {return true;}
+ if (('0' <= c) && (c <= '9')) {return true;}
+ if ('+' == c) {return true;}
+ if ('/' == c) {return true;}
+ return false;
+}
+
+int Base64ScanLen(const uint8* start, const uint8* limit) {
+ // We have a plausible beginning; scan entire base64 string
+ const uint8* ib64str = start;
+ const uint8* b64str = ib64str;
+ const uint8* b64strlimit = limit;
+ // if starts with + +++, assume it is drawing, so bogus
+ if (((limit - start) > 3) && (start[0] == '+') &&
+ (start[1] == '+') && (start[2] == '+')) {
+ return 81;
+ }
+ // Scan over base64
+ while ((b64str < b64strlimit) && (kBase64Value[*b64str++] >= 0)) {
+ }
+ b64str--; // We overshot by 1
+ return b64str - ib64str;
+}
+
+// Input is at least 8-character legal base64 string after +.
+// But might be say + "Presse+Termine"
+bool GoodUnicodeFromBase64(const uint8* start, const uint8* limit) {
+ // Reject base64 string len N if density of '+' is > 1 + N/16 (expect 1/64)
+ // Reject base64 string len N if density of A-Z is < 1 + N/16 (expect 26/64)
+ // Reject base64 string len N if density of a-z is < 1 + N/16 (expect 26/64)
+ // Reject base64 string len N if density of 0-9 is < 1 + N/32 (expect 10/64)
+ // NOTE: this requires at least one lower AND one upper AND one digit to pass
+ //
+ int plus_count = 0;
+ int lower_count = 0;
+ int upper_count = 0;
+ int digit_count = 0;
+ int len = limit - start;
+ for (const uint8* src = start; src < limit; ++src) {
+ uint8 c = *src;
+ if (('a' <= c) && (c <= 'z')) {
+ ++lower_count;
+ } else if (('A' <= c) && (c <= 'Z')) {
+ ++upper_count;
+ } else if (('0' <= c) && (c <= '0')) {
+ ++digit_count;
+ } else if (*src == '+') {
+ ++plus_count;
+ }
+ }
+
+ if (plus_count > (1 + (len >> 4))) {return false;}
+ if (lower_count < (1 + (len >> 4))) {return false;}
+ if (upper_count < (1 + (len >> 4))) {return false;}
+ if (digit_count < (1 + (len >> 5))) {return false;}
+
+ // checking the last character to reduce false positive
+ // since the last character may be padded to 0 bits at the end.
+ // refer to http://en.wikipedia.org/wiki/UTF-7
+ int nmod8 = len & 7;
+ const uint8 last = *(start+len-1);
+ // When UTF-7 string length%8=3, the last two bits must be padded as 0
+ if ((nmod8 == 3) && (kBase64Value[last] & 3)) {return false;}
+ // When UTF-7 string length%8=6, the last four bits must be padded as 0
+ if ((nmod8 == 6) && (kBase64Value[last] & 15)) {return false;}
+ return true;
+}
+
+// Prune here after N bytes
+// Boost here for seven-bit sequences (at every prune)
+// if (sevenbitrankedencoding)
+// + UTF7 scan and boost/demote len mod 8 = 0 3 6
+// ~ Hz scan and boost/demote len mod 8 = 0 2 4 6
+// 1B 2022 scan and boost/demote len mod 8 = 0 2 4 6
+// 0E 2022 scan and boost/demote len mod 8 = 0 2 4 6
+// [0F 2022 boost/demote]
+// 00 UTF16/32 scan and boost/demote offset = even/odd
+//
+// If still some seven-bit possibilities > pure ASCII,
+// scan each possibility for clearer prob, s.t. about
+// two good sequences is a clear win
+// A-Z 00-19 00xx-64xx (B = 04xx)
+// a-z 1A-33 68xx-CCxx (f = 7Cxx)
+// 0-9 34-3D D0xx-F4xx (1 = D4xx)
+// + 3E F8xx
+// / 3F FCxx
+// do another chunk with slow scan
+
+
+// Boost, whack, or leave alone UTF-7 probablilty
+void UTF7BoostWhack(DetectEncodingState* destatep, int next_pair, uint8 byte2) {
+ int off = destatep->interesting_offsets[AsciiPair][next_pair];
+ if (off >= destatep->prior_utf7_offset) {
+ // Not part of a previous successful UTF-7 string
+ ++destatep->utf7_starts;
+
+ if (byte2 == '-') {
+ // +- encoding for '+' neutral
+ } else if (!Base64Char(byte2)) {
+ // Not base64 -- not UTF-7, whack
+ Whack(destatep, F_UTF7, kBadPairWhack); // Illegal pair
+ } else {
+ // Starts with base64 byte, might be a good UTF7 sequence
+ const uint8* start = destatep->initial_src + off + 1; // over the +
+ int n = Base64ScanLen(start, destatep->limit_src);
+ int nmod8 = n & 7;
+ if ((n == 3) || (n == 6)) {
+ // short but legal -- treat as neutral
+ } else if ((nmod8 == 0) | (nmod8 == 3) | (nmod8 == 6)) {
+ // Good length. Check for good Unicode.
+ if (GoodUnicodeFromBase64(start, start + n)) {
+ // Good length and Unicode, boost
+ Boost(destatep, F_UTF7, kBoostOnePair); // Found good
+ destatep->prior_utf7_offset = off + n + 1;
+ } else {
+ // Bad Unicode. Whack
+ Whack(destatep, F_UTF7, kBadPairWhack); // Illegal length
+ }
+ } else {
+ // Bad length. Whack
+ Whack(destatep, F_UTF7, kBadPairWhack); // Illegal length
+ }
+ }
+ }
+}
+
+// Boost, whack, or leave alone HZ probablilty
+void HzBoostWhack(DetectEncodingState* destatep, uint8 byte2) {
+ if ((byte2 == '{') || (byte2 == '}')) {
+ Boost(destatep, F_HZ_GB_2312, kBoostOnePair); // Found ~{ or ~}
+ } else if ((byte2 == '~') || (byte2 == '\n')) {
+ destatep->enc_prob[F_HZ_GB_2312] += 0; // neutral
+ } else {
+ Whack(destatep, F_HZ_GB_2312, kBadPairWhack); // Illegal pair
+ }
+}
+
+// Boost, whack, or leave alone BINARY probablilty
+void BinaryBoostWhack(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
+ int quadrant = ((byte1 & 0x80) >> 6) | ((byte2 & 0x80) >> 7);
+ int bucket8x4 = ((byte1 & 0xe0) >> 3) | ((byte2 & 0xc0) >> 6);
+ uint32 quad_mask = 1 << quadrant;
+ uint32 bucket8x4_mask = 1 << bucket8x4;
+ if ((destatep->binary_quadrants_seen & quad_mask) == 0) {
+ destatep->binary_quadrants_seen |= quad_mask;
+ destatep->binary_quadrants_count += 1;
+ if (destatep->binary_quadrants_count == 4) {
+ Boost(destatep, F_BINARY, kBoostOnePair * 2); // Found all 4 quadrants,
+ // boost 2 pairs
+ }
+ }
+ if ((destatep->binary_8x4_seen & bucket8x4_mask) == 0) {
+ destatep->binary_8x4_seen |= bucket8x4_mask;
+ destatep->binary_8x4_count += 1;
+ if (destatep->binary_8x4_count >= 11) {
+ Boost(destatep, F_BINARY, kBoostOnePair * 4); // Found 11+/20 buckets,
+ // boost 4 pairs each time
+ }
+ }
+}
+
+
+// Demote UTF-16/32 on 0000 or FFFF, favoring Binary
+void UTF1632BoostWhack(DetectEncodingState* destatep, int offset, uint8 byte1) {
+ if (byte1 == 0) { // We have 0000
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_16LE, kBadPairWhack); // Illegal pair
+ switch (offset & 3) {
+ case 0: // We get called with 0 4 8, etc. for ASCII/BMP as UTF-32BE
+ Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal pair
+ Boost(destatep, F_UTF_32BE, kSmallInitDiff); // Good pair
+ break;
+ case 1: // We get called with 1 5 9, etc. for ASCII as UTF-32LE
+ case 2: // We get called with 2 6 10, etc. for BMP as UTF-32LE
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal pair
+ Boost(destatep, F_UTF_32LE, kSmallInitDiff); // Good pair
+ break;
+ case 3: // ambiguous
+ break;
+ }
+ } else { // We have ffff
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_16LE, kBadPairWhack); // Illegal pair
+ }
+}
+
+// Make even offset
+void UTF16MakeEven(DetectEncodingState* destatep, int next_pair) {
+ destatep->interesting_offsets[OtherPair][next_pair] &= ~1;
+}
+
+bool ConsecutivePair(DetectEncodingState* destatep, int i) {
+ if (i <= 0) {
+ return false;
+ }
+ return destatep->interesting_offsets[OtherPair][i] ==
+ (destatep->interesting_offsets[OtherPair][i - 1] + 2);
+}
+
+// boost, whack, or leave alone UTF-8 probablilty
+// Any whacks are also applied to UTF8UTF8; CheckUTF8UTF8Seq assumes good UTF8
+// Returns total boost
+int CheckUTF8Seq(DetectEncodingState* destatep, int weightshift) {
+ int startcount = destatep->prior_interesting_pair[OtherPair];
+ int endcount = destatep->next_interesting_pair[OtherPair];
+
+ int demotion_count = 0;
+ for (int i = startcount; i < endcount; ++i) {
+ int sub;
+ char* s = &destatep->interesting_pairs[OtherPair][i * 2];
+ // Demote four byte patterns that are more likely Latin1 than UTF-8
+ // C9AE, DF92, DF93, DFAB. See note at top.
+ // Demotion also boosts Latin1 and CP1252
+ uint8 s0 = static_cast<uint8>(s[0]);
+ uint8 s1 = static_cast<uint8>(s[1]);
+ if ((s0 == 0xc9) && (s1 == 0xae)) {++demotion_count;}
+ if ((s0 == 0xdf) && (s1 == 0x92)) {++demotion_count;}
+ if ((s0 == 0xdf) && (s1 == 0x93)) {++demotion_count;}
+ if ((s0 == 0xdf) && (s1 == 0xab)) {++demotion_count;}
+
+ if (!ConsecutivePair(destatep, i)) {
+ // Insert a blank into the sequence; avoid wrong splices
+ sub = (' ' >> 4) & 0x0f;
+ ++destatep->utf8_minicount[
+ static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
+ destatep->next_utf8_ministate =
+ kMiniUTF8State[destatep->next_utf8_ministate][sub];
+ }
+ // Byte 0
+ sub = (s0 >> 4) & 0x0f;
+ ++destatep->utf8_minicount[
+ static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
+ destatep->next_utf8_ministate =
+ kMiniUTF8State[destatep->next_utf8_ministate][sub];
+ // Byte 1
+ sub = (s1 >> 4) & 0x0f;
+ ++destatep->utf8_minicount[
+ static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
+ destatep->next_utf8_ministate =
+ kMiniUTF8State[destatep->next_utf8_ministate][sub];
+ DCHECK((0 <= destatep->next_utf8_ministate) &&
+ (destatep->next_utf8_ministate < 8));
+ }
+
+
+ // For the four specific byte combinations above, Latin1/CP1252 is more likely
+ if (demotion_count > 0) {
+ Boost(destatep, F_Latin1, kGentleOnePair * demotion_count);
+ Boost(destatep, F_CP1252, kGentleOnePair * demotion_count);
+ }
+
+ // Boost UTF8 for completed good sequences
+ int total_boost = 2 * destatep->utf8_minicount[2] +
+ 3 * destatep->utf8_minicount[3] +
+ 4 * destatep->utf8_minicount[4];
+ // But not so much for demoted bytes
+ total_boost -= (3 * demotion_count);
+
+ total_boost *= kGentleOnePair;
+ total_boost >>= weightshift;
+ // Design: boost both UTF8 and UTF8UTF8 for each good sequence
+ Boost(destatep, F_UTF8, total_boost);
+ Boost(destatep, F_UTF8UTF8, total_boost);
+
+ destatep->utf8_minicount[5] += destatep->utf8_minicount[2]; // total chars
+ destatep->utf8_minicount[5] += destatep->utf8_minicount[3]; // total chars
+ destatep->utf8_minicount[5] += destatep->utf8_minicount[4]; // total chars
+ destatep->utf8_minicount[2] = 0;
+ destatep->utf8_minicount[3] = 0;
+ destatep->utf8_minicount[4] = 0;
+
+ // Whack (2 bytes) for errors
+ int error_whack = 2 * destatep->utf8_minicount[1];
+ error_whack *= kGentlePairWhack;
+ error_whack >>= weightshift;
+ Whack(destatep, F_UTF8, error_whack);
+ Whack(destatep, F_UTF8UTF8, error_whack);
+ destatep->utf8_minicount[1] = 0;
+
+ return total_boost - error_whack;
+}
+
+
+// Boost, whack, or leave alone UTF8UTF8 probablilty
+//
+// We are looking for
+// (1) chars ONLY in set UTF8(0080)..UTF8(00FF), including for 80..9F the
+// MS CP1252 mappings, and
+// (2) sequences of 2 or more such characters
+//
+// If so, we could be looking at some non-7-bit encoding extra-converted
+// to UTF-8. The most common observed is CP1252->UTF8 twice,
+// 1252=>UTF8 : 1252=>UTF8
+// where the colon means "take those bytes and pretend that they are 1252".
+// We have a couple of examples of BIG5 bytes converted as though
+// they were 1252,
+// BIG5 : 1252=>UTF8
+//
+// Of course, we don't want correctly converted 1252 to be flagged here
+// 1252=>UTF8
+// So we want the input high bytes to be in pairs or longer, hence the
+// output UTF8 in groups of four bytes or more
+//
+// Good chars: C2xx, C3xx,
+// Good chars: C592, C593, C5A0, C5A1, C5B8, C5BD, C5BE, C692, CB86, CB9C
+// Good chars: E280xx E282AC E284A2
+// C2xx 1100001x 10xxxxxx (128/128)
+// C5xx 11000101 10xx00xx (16/4)
+// C5xx 11000101 10111xxx (8/3)
+// C692 11000110 10010010 (1/1)
+// CBxx 11001011 100xx1x0 (8/2)
+// E28x 11100010 10000xx0 (4/3)
+//
+// Returns total boost
+int CheckUTF8UTF8Seq(DetectEncodingState* destatep, int weightshift) {
+ int this_pair = destatep->prior_interesting_pair[OtherPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
+
+ int pair_number = this_pair;
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ int next = destatep->next_utf8utf8_ministate;
+ if (!ConsecutivePair(destatep, pair_number)) {
+ // Insert two blanks into the sequence to avoid wrong splices
+ // go back to no odd-byte offset
+ destatep->utf8utf8_odd_byte = 0;
+ int sub = UTF88Sub(' ', ' ');
+ ++destatep->utf8utf8_minicount[static_cast<int>(kMiniUTF8UTF8Count[next][sub])];
+ next = kMiniUTF8UTF8State[next][sub];
+ }
+
+ int odd = destatep->utf8utf8_odd_byte;
+ if (s + 1 + odd >= endbyte) continue;
+ int sub = UTF88Sub(s[0 + odd], s[1 + odd]);
+ destatep->utf8utf8_odd_byte ^= kMiniUTF8UTF8Odd[next][sub];
+ ++destatep->utf8utf8_minicount[
+ static_cast<int>(kMiniUTF8UTF8Count[next][sub])];
+ destatep->next_utf8utf8_ministate = kMiniUTF8UTF8State[next][sub];
+ ++pair_number;
+ }
+
+ // Boost for completed good sequences; each count covers two chars.
+ // Design: boost UTF8UTF8 above UTF8 for each good sequence
+ int total_boost = (2) * destatep->utf8utf8_minicount[2] +
+ (2) * destatep->utf8utf8_minicount[3] +
+ (2) * destatep->utf8utf8_minicount[4];
+ total_boost *= kGentleOnePair;
+ total_boost >>= weightshift;
+ Boost(destatep, F_UTF8UTF8, total_boost);
+
+ // Track total characters
+ destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[2];
+ destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[3];
+ destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[4];
+ destatep->utf8utf8_minicount[2] = 0;
+ destatep->utf8utf8_minicount[3] = 0;
+ destatep->utf8utf8_minicount[4] = 0;
+
+ // Design: Do not whack UTF8UTF8 below UTF8 for each bad sequence
+
+ destatep->utf8utf8_minicount[1] = 0;
+ return total_boost;
+}
+
+
+// We give a gentle boost for each paired SO ... SI, whack others
+void CheckIso2022ActiveSeq(DetectEncodingState* destatep) {
+ int this_pair = destatep->prior_interesting_pair[OtherPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
+
+ // Initial <esc> char must precede SO/SI
+ // HZ_GB_2312 has no alternation constraint on 1- and 2-byte segments
+ // ISO-2022-JP (JIS) has no alternation constraint on 1- and 2-byte segments
+ // ISO-2022-CN has no alternation constraint on 1- and 2-byte segments
+ // ISO-2022-KR requires alternation between 1- and 2-byte segments
+ // JIS:
+ // <esc> ( B ISO-2022-JP [1b 28 42] SI to ASCII
+ // <esc> ( J ISO-2022-JP [1b 28 4a] SI to X0201
+ // <esc> $ @ ISO-2022-JP [1b 24 40] SO to X0208-78 twobyte
+ // <esc> $ B ISO-2022-JP [1b 24 42] SO to X0208-83 twobyte
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ if (s[0] == 0x1b) {
+ if (s[1] == 0x24) {
+ // <esc> $ is SO
+ destatep->next_2022_state = SOSI_TWOBYTE; // SO to two-byte
+ } else if (s[1] == 0x28) {
+ if (destatep->next_2022_state == SOSI_TWOBYTE) {
+ Boost(destatep, F_JIS, kGentlePairBoost);
+ } else if (destatep->next_2022_state == SOSI_ONEBYTE) {
+ Whack(destatep, F_JIS, kGentlePairWhack);
+ }
+ destatep->next_2022_state = SOSI_ONEBYTE; // JIS SI to one-byte
+ } else {
+ Whack(destatep, F_JIS, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ destatep->next_2022_state = SOSI_ERROR; // not 2022
+ }
+ } else if (s[0] == 0x0e) {
+ // <so>
+ Whack(destatep, F_JIS, kBadPairWhack);
+ if (destatep->next_2022_state != SOSI_NONE) {
+ destatep->next_2022_state = SOSI_TWOBYTE; // SO to two-byte
+ } else {
+ // ESC required before SO/SI
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack * 4);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack * 4);
+ destatep->next_2022_state = SOSI_ERROR; // SO not after SI
+ }
+ } else if (s[0] == 0x0f) {
+ // <si>
+ Whack(destatep, F_JIS, kBadPairWhack);
+ if (destatep->next_2022_state != SOSI_NONE) {
+ if (destatep->next_2022_state == SOSI_TWOBYTE) {
+ Boost(destatep, F_ISO_2022_CN, kGentlePairBoost);
+ Boost(destatep, F_ISO_2022_KR, kGentlePairBoost);
+ } else if (destatep->next_2022_state == SOSI_ONEBYTE) {
+ Whack(destatep, F_ISO_2022_CN, kGentlePairWhack);
+ Whack(destatep, F_ISO_2022_KR, kGentlePairWhack);
+ }
+ destatep->next_2022_state = SOSI_ONEBYTE; // SI to one-byte
+ } else {
+ // ESC required before SO/SI
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack * 4);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack * 4);
+ destatep->next_2022_state = SOSI_ERROR; // SI not after SO
+ }
+ } else if (s[0] <= 0x1f) {
+ // Some other control code. Allow ht lf [ff] cr
+ if ((s[0] != 0x09) && (s[0] != 0x0a) &&
+ (s[0] != 0x0c) && (s[0] != 0x0d)) {
+ // Otherwise these can float to the top on bad bytes
+ Whack(destatep, F_JIS, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ }
+ }
+ }
+
+ // If no start, keep the probability pinned at zero (or below)
+ if (destatep->next_2022_state == SOSI_NONE) {
+ destatep->enc_prob[F_ISO_2022_CN] =
+ minint(0, destatep->enc_prob[F_ISO_2022_CN]);
+ destatep->enc_prob[F_ISO_2022_KR] =
+ minint(0, destatep->enc_prob[F_ISO_2022_KR]);
+ destatep->enc_prob[F_JIS] =
+ minint(0, destatep->enc_prob[F_JIS]);
+ }
+}
+
+// We give a gentle boost for each paired ~{ ... ~}, whack others
+void CheckHzActiveSeq(DetectEncodingState* destatep) {
+ int this_pair = destatep->prior_interesting_pair[AsciiPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[AsciiPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[AsciiPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[AsciiPair][endbyteoffset];
+
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ // Look for initial ~{ pair
+ if ((s[0] == '~') && (s[1] == '{')) {
+ destatep->next_hz_state = SOSI_TWOBYTE; // SO to two-byte
+ }
+ // Also look for closing ~} pair
+ if ((s[0] == '~') && (s[1] == '}')) {
+ if (destatep->next_hz_state == SOSI_TWOBYTE) {
+ Boost(destatep, F_HZ_GB_2312, kGentlePairBoost);
+ } else if (destatep->next_hz_state == SOSI_ONEBYTE) {
+ Whack(destatep, F_HZ_GB_2312, kGentlePairWhack);
+ }
+ destatep->next_hz_state = SOSI_ONEBYTE; // SI to one-byte
+ }
+ }
+
+ // If no start, keep the probability pinned at zero (or below)
+ if (destatep->next_hz_state == SOSI_NONE) {
+ destatep->enc_prob[F_HZ_GB_2312] =
+ minint(0, destatep->enc_prob[F_HZ_GB_2312]);
+ }
+}
+
+// We give a gentle boost after an odd number of 8Fxxxx triples, which
+// put subsequent bigrams out of phase until a low byte or another 8Fxxxx
+void CheckEucJpSeq(DetectEncodingState* destatep) {
+ int this_pair = destatep->prior_interesting_pair[OtherPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
+
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ // Boost if out of phase (otherwise, EUC-JP will score badly after 8Fxxxx)
+ if (destatep->next_eucjp_oddphase) {
+ //printf(" EucJp boost[%02x%02x]\n", s[0], s[1]); // TEMP
+ Boost(destatep, F_EUC_JP, kGentlePairBoost * 2);
+ }
+
+ uint8 s0 = static_cast<uint8>(s[0]);
+ uint8 s1 = static_cast<uint8>(s[1]);
+ // Look for phase flip at 8F
+ if ((s0 & 0x80) == 0x00) {
+ destatep->next_eucjp_oddphase = false;
+ } else if (s0 == 0x8f) {
+ destatep->next_eucjp_oddphase = !destatep->next_eucjp_oddphase;
+ }
+ if ((s1 & 0x80) == 0x00) {
+ destatep->next_eucjp_oddphase = false;
+ } else if (s1 == 0x8f) {
+ destatep->next_eucjp_oddphase = !destatep->next_eucjp_oddphase;
+ }
+ }
+}
+
+// Boost, whack, or leave alone BINARY probablilty
+// Also called if UTF 16/32 active
+void CheckBinaryDensity(const uint8* src, DetectEncodingState* destatep,
+ int delta_otherpairs) {
+ // No change if not much gathered information
+ if (delta_otherpairs == 0) {
+ // Only ASCII pairs this call
+ return;
+ }
+ int next_pair = destatep->next_interesting_pair[OtherPair];
+
+ // Look at density of interesting pairs [0..src)
+ int delta_offset = static_cast<int>(src - destatep->initial_src); // actual
+
+ // Look at density of interesting pairs [0..next_interesting)
+ int low_byte = destatep->interesting_offsets[OtherPair][0];
+ //int high_byte = destatep->interesting_offsets[OtherPair][next_pair - 1] + 2;
+ //int byte_span = high_byte - low_byte;
+ int byte_span = delta_offset - low_byte;
+
+ // If all ASCII for the first 4KB, reject
+ // If mostly ASCII in the first 5KB, reject
+ if ((low_byte >= kBinaryHardAsciiLimit) || (delta_offset >= kBinarySoftAsciiLimit)) {
+ // Not binary early enough in text
+ Whack(destatep, F_BINARY, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack * 4);
+ return;
+ }
+
+ // Density 1.0 for N pairs takes 2*N bytes
+ // Whack if < 1/16 after first non_ASCII pair
+ if ((next_pair * 2 * 16) < byte_span) {
+ // Not dense enough
+ Whack(destatep, F_BINARY, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack * 4);
+ }
+
+ if (next_pair < 8) {
+ // Fewer than 8 non-ASCII total; too soon to boost
+ return;
+ }
+
+ // Density 1.0 for N pairs takes 2*N bytes
+ // Boost if density >= 1/4, whack if < 1/16
+ if ((next_pair * 2 * 4) >= byte_span) {
+ // Very dense
+ // Only boost if at least 2 quadrants seen
+ if (destatep->binary_quadrants_count >= 2) {
+ Boost(destatep, F_BINARY, kSmallInitDiff);
+ Boost(destatep, F_UTF_32BE, kSmallInitDiff);
+ Boost(destatep, F_UTF_32LE, kSmallInitDiff);
+ Boost(destatep, F_UTF_16BE, kSmallInitDiff);
+ Boost(destatep, F_UTF_16LE, kSmallInitDiff);
+ }
+ }
+}
+
+
+// Look at a number of special-case encodings whose reliable detection depends
+// on sequencing or other properties
+// AsciiPair probibilities (UTF7 and HZ) are all done here
+void ActiveSpecialBoostWhack(const uint8* src, DetectEncodingState* destatep) {
+ int delta_asciipairs = destatep->next_interesting_pair[AsciiPair] -
+ destatep->prior_interesting_pair[AsciiPair];
+ int delta_otherpairs = destatep->next_interesting_pair[OtherPair] -
+ destatep->prior_interesting_pair[OtherPair];
+
+ // The two pure ASCII encodings
+ if (UTF7OrHzActive(destatep) && (delta_asciipairs > 0)) {
+ // Adjust per pair
+ for (int i = 0; i < delta_asciipairs; ++i) {
+ int next_pair = destatep->prior_interesting_pair[AsciiPair] + i;
+ uint8 byte1 = destatep->interesting_pairs[AsciiPair][next_pair * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[AsciiPair][next_pair * 2 + 1];
+ if (byte1 == '+') {
+ // Boost, whack, or leave alone UTF-7 probablilty
+ UTF7BoostWhack(destatep, next_pair, byte2);
+ if (destatep->debug_data != NULL) {
+ // Show UTF7 entry
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02x+", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[AsciiPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ } else if (byte1 == '~') {
+ // Boost, whack, or leave alone HZ probablilty
+ HzBoostWhack(destatep, byte2);
+ if (destatep->debug_data != NULL) {
+ // Show Hz entry
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02x~", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[AsciiPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ }
+ }
+
+ // Kill UTF-7 now if at least 8 + pairs and not confirmed valid UTF-7
+ if ((destatep->utf7_starts >= 8) && (destatep->prior_utf7_offset == 0)) {
+ Whack(destatep, F_UTF7, kBadPairWhack * 8); // flush
+ }
+ }
+
+
+
+ // All the other encodings
+ if (OtherActive(destatep) && (delta_otherpairs > 0)) {
+ // Adjust per pair
+ int biggest_weightshift = 0;
+ for (int i = 0; i < delta_otherpairs; ++i) {
+ int next_pair = destatep->prior_interesting_pair[OtherPair] + i;
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 1];
+ int off = destatep->interesting_offsets[OtherPair][next_pair];
+ int weightshift = destatep->interesting_weightshift[OtherPair][next_pair];
+ biggest_weightshift = maxint(biggest_weightshift, weightshift);
+
+ if (byte1 == 0x00) {
+ if (byte2 == 0x00) {
+ UTF1632BoostWhack(destatep, off, byte1);
+ } else if ((kIsPrintableAscii[byte2] != 0) && ((off & 1) != 0)) {
+ // We have 00xx at an odd offset. Turn into preceding even offset
+ // for possible Ascii text in UTF-16LE or UTF-32LE (vs BE)
+ // This will cascade into caller's probability update
+ // 00 is illegal for all other encodings, so it doesn't matter to them
+ UTF16MakeEven(destatep, next_pair);
+ }
+ if (destatep->debug_data != NULL) {
+ // Show 0000 detail entry for this bigram
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02xZ", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[OtherPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ }
+ if (byte1 == 0xff) {
+ if (byte2 == 0xff) {
+ UTF1632BoostWhack(destatep, off, byte1);
+ }
+ if (destatep->debug_data != NULL) {
+ // Show FFFF detail entry for this bigram
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02xF", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[OtherPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ }
+ if (BinaryActive(destatep)) {
+ BinaryBoostWhack(destatep, byte1, byte2);
+ }
+ } // End for i
+
+ // Adjust per entire-pair-span
+ if (UTF8Active(destatep)) {
+ CheckUTF8Seq(destatep, biggest_weightshift);
+ }
+
+ if (UTF8UTF8Active(destatep)) {
+ CheckUTF8UTF8Seq(destatep, biggest_weightshift);
+ }
+
+ if (Iso2022Active(destatep)) {
+ CheckIso2022ActiveSeq(destatep);
+ }
+
+ if (HzActive(destatep)) {
+ CheckHzActiveSeq(destatep);
+ }
+
+ if (EUCJPActive(destatep)) {
+ CheckEucJpSeq(destatep);
+ }
+
+ if (BinaryActive(destatep) || UTF1632Active(destatep)) {
+ CheckBinaryDensity(src, destatep, delta_otherpairs);
+ }
+ }
+ // ISO-2022 do OK on their own, using stright probabilities? Not on bad bytes
+
+ if (destatep->debug_data != NULL) {
+ // Show sequencing result
+ SetDetailsEncLabel(destatep, "seq");
+ }
+}
+
+
+void PrintTopEnc(DetectEncodingState* destatep, int n) {
+ // Print top n or fewer
+ int temp_sort[NUM_RANKEDENCODING];
+ for (int j = 0; j < destatep->rankedencoding_list_len; ++j) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ temp_sort[j] = destatep->enc_prob[rankedencoding];
+ }
+
+ qsort(temp_sort, destatep->rankedencoding_list_len,
+ sizeof(temp_sort[0]), IntCompare);
+
+ int top_n = minint(n, destatep->rankedencoding_list_len);
+ int showme = temp_sort[top_n - 1]; // Print this value and above
+
+ printf("rankedencodingList top %d: ", top_n);
+ for (int j = 0; j < destatep->rankedencoding_list_len; ++j) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (showme <= destatep->enc_prob[rankedencoding]) {
+ printf("%s=%d ",
+ MyEncodingName(kMapToEncoding[rankedencoding]),
+ destatep->enc_prob[rankedencoding]);
+ }
+ }
+ printf("\n\n");
+}
+
+// If the same bigram repeats, don't boost its best encoding too much
+bool RepeatedBigram(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
+ int this_bigram = (byte1 << 8) | byte2;
+ // If 00xx 01xx 02xx ... 1fxx, take out bottom 4 bits of xx.
+ // This ignores parts of Yahoo 0255 0254 0243 0247 0245 0243 0250 0255 ...
+ // It may screw up UTF-16BE
+ // It may screw up ISO-2022 (1b24 suppresses 1b28)
+ if (byte1 < 0x20) {
+ this_bigram &= 0xfff0;
+ }
+ if (this_bigram == destatep->prior_bigram[0]) {return true;}
+ if (this_bigram == destatep->prior_bigram[1]) {return true;}
+ if (this_bigram == destatep->prior_bigram[2]) {return true;}
+ if (this_bigram == destatep->prior_bigram[3]) {return true;}
+ // Round-robin replacement
+ destatep->prior_bigram[destatep->next_prior_bigram] = this_bigram;
+ destatep->next_prior_bigram = (destatep->next_prior_bigram + 1) & 3;
+ return false;
+}
+
+// Sometimes illegal bytes are used as markers between text that Javascript
+// is going to decode. Don't overboost the Binary encoding for markers 01-FF.
+// Just count first pair per 8x4 bucket
+bool RepeatedBinary(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
+ int bucket8x4 = ((byte1 & 0xe0) >> 3) | ((byte2 & 0xc0) >> 6);
+ uint32 bucket8x4_mask = 1 << bucket8x4;
+ if ((destatep->binary_8x4_seen & bucket8x4_mask) == 0) {
+ destatep->binary_8x4_seen |= bucket8x4_mask;
+ destatep->binary_8x4_count += 1;
+ return false;
+ }
+ return true;
+}
+
+
+
+
+// Find current top two rankedencoding probabilities
+void ReRank(DetectEncodingState* destatep) {
+ destatep->top_prob = -1;
+ destatep->second_top_prob = -1;
+ // Leave unchanged
+ //destatep->top_rankedencoding =
+ // destatep->rankedencoding_list[0]; // Just to make well-defined
+ //destatep->second_top_rankedencoding =
+ // destatep->rankedencoding_list[1]; // Just to make well-defined
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (destatep->top_prob < destatep->enc_prob[rankedencoding]) {
+ // Make sure top 2 are in different superset groups
+ if (kMapEncToBaseEncoding[kMapToEncoding[destatep->top_rankedencoding]] !=
+ kMapEncToBaseEncoding[kMapToEncoding[rankedencoding]]) {
+ destatep->second_top_prob =
+ destatep->top_prob; // old top to second
+ destatep->second_top_rankedencoding =
+ destatep->top_rankedencoding; // old top to second
+ }
+ destatep->top_prob = destatep->enc_prob[rankedencoding];
+ destatep->top_rankedencoding = rankedencoding;
+ } else if (destatep->second_top_prob < destatep->enc_prob[rankedencoding]) {
+ if (kMapEncToBaseEncoding[kMapToEncoding[destatep->top_rankedencoding]] !=
+ kMapEncToBaseEncoding[kMapToEncoding[rankedencoding]]) {
+ destatep->second_top_prob = destatep->enc_prob[rankedencoding];
+ destatep->second_top_rankedencoding = rankedencoding;
+ }
+ }
+ }
+}
+
+void SimplePrune(DetectEncodingState* destatep, int prune_diff) {
+ // Prune the list of active encoding families
+ int keep_prob = destatep->top_prob - prune_diff;
+
+ destatep->active_special = 0;
+ int k = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ bool keep = true;
+ int rankedencoding = destatep->rankedencoding_list[j];
+
+ // If count is too low, ditch it
+ if (destatep->enc_prob[rankedencoding] < keep_prob) {keep = false;}
+
+ // Keep it. This will always keep at least top_prob rankedencoding
+ if (keep) {
+ destatep->active_special |= kSpecialMask[kMapToEncoding[rankedencoding]];
+ destatep->rankedencoding_list[k++] = rankedencoding;
+ }
+ }
+
+ destatep->rankedencoding_list_len = k;
+}
+
+// Recalculate reliable
+void CalcReliable(DetectEncodingState* destatep) {
+ // Encoding result is reliable if big difference in top two, or if
+ // only Ascii7 ever encountered
+ // Also reliable if exactly one OtherPair and it's best encoding matches top
+ destatep->reliable = false;
+ if (destatep->next_interesting_pair[OtherPair] == 0) {
+ // Only 7-bit ASCII
+ destatep->reliable = true;
+ return;
+ }
+ if ((destatep->top_prob - destatep->second_top_prob) >=
+ FLAGS_ced_reliable_difference) {
+ destatep->reliable = true;
+ return;
+ }
+ if (destatep->next_interesting_pair[OtherPair] == 1) {
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][1];
+ int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
+ if (best_enc == destatep->top_rankedencoding) {
+ destatep->reliable = true;
+ return;
+ }
+ }
+
+ // If we pruned to one encoding, we are done
+ if (destatep->rankedencoding_list_len == 1) {
+ destatep->reliable = true;
+ destatep->done = true;
+ return;
+ }
+
+ // If we pruned to two or three encodings in the same *superset/subset
+ // rankedencoding* and enough pairs, we are done. Else keep going
+ if (destatep->rankedencoding_list_len == 2) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ if (kMapEncToBaseEncoding[enc0] == kMapEncToBaseEncoding[enc1]) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ return;
+ }
+ }
+ } else if (destatep->rankedencoding_list_len == 3) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ Encoding enc2 = kMapToEncoding[destatep->rankedencoding_list[2]];
+ Encoding base0 = kMapEncToBaseEncoding[enc0];
+ Encoding base1 = kMapEncToBaseEncoding[enc1];
+ Encoding base2 = kMapEncToBaseEncoding[enc2];
+
+ if ((base0 == base1) && (base0 == base2)) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ return;
+ }
+ }
+ }
+
+}
+
+
+// Find current top two rankedencoding probabilities
+void FindTop2(DetectEncodingState* destatep,
+ int* first_renc, int* second_renc,
+ int* first_prob, int* second_prob) {
+ *first_prob = -1;
+ *second_prob = -1;
+ *first_renc = 0;
+ *second_renc = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (*first_prob < destatep->enc_prob[rankedencoding]) {
+ *second_prob = *first_prob; // old top to second
+ *second_renc = *first_renc; // old top to second
+ *first_prob = destatep->enc_prob[rankedencoding];
+ *first_renc = rankedencoding;
+ } else if (*second_prob < destatep->enc_prob[rankedencoding]) {
+ *second_prob = destatep->enc_prob[rankedencoding];
+ *second_renc = rankedencoding;
+ }
+ }
+}
+
+
+void PrintRankedEncodingList(DetectEncodingState* destatep, const char* str) {
+ printf("Current ranked encoding list %s\n", str);
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if ((rankedencoding < 0) || (rankedencoding > NUM_RANKEDENCODING)) {
+ printf(" [%d] BOGUS rankedencoding = %d\n", j, rankedencoding);
+ } else {
+ printf(" [%d] rankedencoding = %d %-12.12s enc_prob = %d\n",
+ j, rankedencoding, MyRankedEncName(rankedencoding),
+ destatep->enc_prob[rankedencoding]);
+ }
+ }
+ printf("End current ranked encoding list\n\n");
+}
+
+
+
+
+// Map unencoded bytes down to five bits, largely preserving letters
+// This design struggles to put 33 values into 5 bits.
+#define XX 0 // Punctuation (00-7F range)
+#define HA 27 // High vowel a in Latin1/2/sometimes7
+#define HE 28 // High vowel e
+#define HI 29 // High vowel i
+#define HO 30 // High vowel o
+#define HU 30 // High vowel u on top of HO
+#define Hc 31 // High consonant (80-FF range)
+static const char kMapToFiveBits[256] = {
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+
+ XX, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,
+ 16,17,18,19,20,21,22,23, 24,25,26,XX,XX,XX,XX,XX,
+ XX, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,
+ 16,17,18,19,20,21,22,23, 24,25,26,XX,XX,XX,XX,XX,
+
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+
+ Hc,HA,HA,HA,HA,Hc,Hc,Hc, Hc,HE,HE,HE,HI,HI,HI,Hc,
+ Hc,Hc,Hc,HO,HO,HO,HO,Hc, Hc,HU,HU,HU,HU,Hc,Hc,Hc,
+ Hc,HA,HA,HA,HA,Hc,Hc,Hc, Hc,HE,HE,HE,HI,HI,HI,Hc,
+ Hc,Hc,Hc,HO,HO,HO,HO,Hc, Hc,HU,HU,HU,HU,Hc,Hc,Hc,
+
+};
+#undef XX
+#undef HA
+#undef HE
+#undef HI
+#undef HO
+#undef HU
+#undef Hc
+
+static const int kTriLatin1Likely = 1;
+static const int kTriLatin2Likely = 2;
+static const int kTriLatin7Likely = 3;
+
+// Each table entry has 32 times two bits, selected by byte[2]
+// Entry subscript is selected by byte[0] and byte[1]
+// Latin1/2/7 boost vector, generated 2007.09.26 by postproc-enc-detect-short.cc
+static const uint64 kLatin127Trigrams[1024] = {
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x304080c0402c3330ULL, 0x0008400004000000ULL, 0x082800000c200000ULL,
+0x23a0000420800030ULL, 0x00000000000ccc00ULL, 0x0500100100100000ULL, 0x0388400000200010ULL,
+0x0000000000000c00ULL, 0xd0f0300740f0cf00ULL, 0x2aa0a2a22882a2acULL, 0x081d800000000080ULL,
+0x0c82000020000000ULL, 0x200a03c000a00000ULL, 0x0008400400290000ULL, 0x0400870000000000ULL,
+0x00f040c00000c080ULL, 0x0008004000000410ULL, 0x0020300000000030ULL, 0x00a030002c300000ULL,
+0x0c8030c020a00000ULL, 0x15410030f0f4c000ULL, 0x3000000300a00000ULL, 0xa2880980a0880a88ULL,
+0x0900300000000000ULL, 0x0000040100300000ULL, 0x0888820020a00000ULL, 0xc044002242010000ULL,
+0x000000121d300040ULL, 0x40100040440c0d54ULL, 0x00008423102f8144ULL, 0x0b40808400000280ULL,
+0x0000000000000000ULL, 0x0680a000000c0000ULL, 0x0880008020aa0000ULL, 0x2aaa0141010a4940ULL,
+0xcb80000000010000ULL, 0x2280000000000000ULL, 0x5248000001800000ULL, 0x8000401004040010ULL,
+0x1540010201001010ULL, 0x0080080400000000ULL, 0x5a00044040000108ULL, 0x0288000282080008ULL,
+0x4800008002200000ULL, 0x4a00000000010100ULL, 0x8a88040080000800ULL, 0x0140800000000400ULL,
+0x40010050000c0000ULL, 0x0000008000000000ULL, 0x0028000020140040ULL, 0x8620401401005308ULL,
+0xc082000000000400ULL, 0x05c0b004c0240600ULL, 0x0288000080000000ULL, 0x0000014000000000ULL,
+0x00000000040000c0ULL, 0x8001861008004280ULL, 0x0200000000000300ULL, 0x0000240242288620ULL,
+0x801000c05434c200ULL, 0x9020162040a2d2b4ULL, 0x0021840000240704ULL, 0x2a80280080084908ULL,
+0x0000000000000000ULL, 0x0500004000000040ULL, 0x0080000000040000ULL, 0x0108058104440000ULL,
+0x0900000000040000ULL, 0x00c0000000208008ULL, 0x2000005000000000ULL, 0x0080000000050000ULL,
+0x0808000000001080ULL, 0x9880810100308000ULL, 0x2285480080081a08ULL, 0x8a80000080080000ULL,
+0x1450000000600010ULL, 0x2210000100000000ULL, 0x8a88000100011000ULL, 0x1541804000000010ULL,
+0xc084011140040100ULL, 0x0000000000000800ULL, 0x0400000000000030ULL, 0x2a800000a0890128ULL,
+0x1140a00054000104ULL, 0x1440000101200404ULL, 0x028800400400d800ULL, 0x0000000000000000ULL,
+0x0000000000002330ULL, 0x0020820228a02280ULL, 0xa2888a02aa8008a8ULL, 0xd0040a0044202500ULL,
+0x8000044104a29424ULL, 0xc000100178b2c5b4ULL, 0x0000810100241504ULL, 0xd040030000380008ULL,
+0x0000000000000000ULL, 0x26c08c0000200130ULL, 0x4a08000110080000ULL, 0x2aa0004001080800ULL,
+0x0aac000000004000ULL, 0x2000000000200000ULL, 0x4240000100020000ULL, 0x4100000080000000ULL,
+0x4900040000000000ULL, 0x0800000400300040ULL, 0x6a80000000040800ULL, 0x2a08182000588008ULL,
+0x0a00000c81000008ULL, 0x0a000c0010000000ULL, 0x8a88001080280808ULL, 0x0020000200300600ULL,
+0xaac00000900a0000ULL, 0x0000100004000000ULL, 0x0020081020000000ULL, 0x8220105010084110ULL,
+0x4a80800000004000ULL, 0x050000c0c0200000ULL, 0x288c000084000000ULL, 0xa048082280000000ULL,
+0x0000000000000000ULL, 0x8000900000032080ULL, 0xee889e81b8880820ULL, 0xc2200a8142800424ULL,
+0xc020141543361010ULL, 0x10a000204a801634ULL, 0x3a808800802a00a0ULL, 0x28808b00803d0800ULL,
+0x0000000000000000ULL, 0x0020000000000030ULL, 0x0808400121010040ULL, 0x0c28240100200040ULL,
+0x2008200028800000ULL, 0xc10004c80f30c030ULL, 0x0400440114100000ULL, 0x2208200280a22220ULL,
+0x0600000030c01000ULL, 0x1201001040c00000ULL, 0x0aa02ea22aa22aa0ULL, 0x30008000000200a0ULL,
+0x20c8400400800000ULL, 0x08280b0420800000ULL, 0x0800100000210000ULL, 0x10000300c0100400ULL,
+0xc8c0000420000000ULL, 0x1000000010000000ULL, 0x0420000400000000ULL, 0x0220000500204000ULL,
+0x2200000420000000ULL, 0x0000540400000000ULL, 0x0000000020000000ULL, 0x00080c00a0810080ULL,
+0x1540000000043000ULL, 0x0000000000100000ULL, 0x2e88a22220200a20ULL, 0xc06030e34ea503a0ULL,
+0x0001100204048500ULL, 0x000000e0000c0d54ULL, 0x3000820310a31400ULL, 0x13088c0320e00280ULL,
+0x0000000000000000ULL, 0x0480000000200000ULL, 0x4000200100000000ULL, 0x0000300040040000ULL,
+0x4400000000000000ULL, 0x0401000002240000ULL, 0x0540000000040000ULL, 0x4004010000000000ULL,
+0x4001111001100000ULL, 0x2880000000300040ULL, 0x4040004040002404ULL, 0x0200000000000000ULL,
+0x0140040000100000ULL, 0x4040010040040080ULL, 0x0a00140000041004ULL, 0x0000a00400808000ULL,
+0x1010200000430040ULL, 0x0010000000000000ULL, 0x0540000000104000ULL, 0x1400114005000000ULL,
+0x0000204000440010ULL, 0x0500000000004400ULL, 0x4500000018000400ULL, 0x0000400000000000ULL,
+0x000000300000cc00ULL, 0x0100001011300000ULL, 0x0040000000000000ULL, 0xc0e0000248a00444ULL,
+0x0000040020340144ULL, 0x0000046445105454ULL, 0x32a0a80280880128ULL, 0x0880040000100100ULL,
+0x0000000000000000ULL, 0x14003000030c0004ULL, 0x4a04001100000000ULL, 0x0a00108010000000ULL,
+0x28a8004000200248ULL, 0x0100040000b00000ULL, 0x42000000000008c0ULL, 0x6008044010550010ULL,
+0x0800401000010400ULL, 0x080080040cf80000ULL, 0x5080000001001010ULL, 0x2a80100000000000ULL,
+0xcc8010010d401100ULL, 0x0200000001001000ULL, 0x0480001004001000ULL, 0x8d00800040b40210ULL,
+0x6200800000300000ULL, 0x0000010000000000ULL, 0x0428004100010000ULL, 0x4320105141501100ULL,
+0xe28c0000000c1000ULL, 0xd5c000c3c0e00300ULL, 0x0001000000100200ULL, 0x1004010202400008ULL,
+0x0000000000003000ULL, 0x2aa038a0800aab08ULL, 0x2a88038000000000ULL, 0xc220040242f09720ULL,
+0x8020200200ba0420ULL, 0x0020106105101004ULL, 0x0480800000220400ULL, 0x2280100080000008ULL,
+0x0000000000000000ULL, 0x9000000000200000ULL, 0x0001000000100000ULL, 0x2aa40c0000080800ULL,
+0x0040000040010000ULL, 0x0040000000c01000ULL, 0x4000000040000400ULL, 0x0000001000200000ULL,
+0x0000010000000000ULL, 0x05808004000c0000ULL, 0x50400c0000000400ULL, 0x020040008f000040ULL,
+0x0800000000100000ULL, 0x0000000000000000ULL, 0x0a08440000004000ULL, 0x0064000400008200ULL,
+0x0010010010034170ULL, 0x0000000010000000ULL, 0x0100204021000000ULL, 0x022000d000010100ULL,
+0x0840300000c00000ULL, 0x1400000040204400ULL, 0x09800c0040000000ULL, 0x0209708000000000ULL,
+0x000000000000c040ULL, 0x90000c50204040a0ULL, 0x0000000000000000ULL, 0x00e1500040200004ULL,
+0x8020260540204494ULL, 0x0020026150201054ULL, 0x0281800380105634ULL, 0x0884900481105000ULL,
+0x0000000000000000ULL, 0x84203c00002c0200ULL, 0xc089040000000000ULL, 0xc2a8100040200004ULL,
+0xe00c1c0000000000ULL, 0x0ce1330080200080ULL, 0x0000000000200000ULL, 0xc400110000404010ULL,
+0x0088400000000000ULL, 0x00083cc00c00c00cULL, 0xcac01c00c000580cULL, 0xe300b0f000100000ULL,
+0x0300000000000000ULL, 0xc0000f0000000000ULL, 0xc3c01c0400000000ULL, 0x81008004c0f40000ULL,
+0xc3d8003000000440ULL, 0x0000000000000000ULL, 0xc430000000000000ULL, 0x0060000000001000ULL,
+0x0800000000000000ULL, 0x00c03300f0fc0008ULL, 0x3000000400200010ULL, 0xa2a80892a0880a28ULL,
+0x0500000040000004ULL, 0x0000000000000000ULL, 0xc80032070c200020ULL, 0x0220820060a296a0ULL,
+0x802084021db486a0ULL, 0x00000d60080c0080ULL, 0xb281803313a32428ULL, 0x1808300320300000ULL,
+0x0000000000000000ULL, 0x85208cc0ccac1f20ULL, 0x2081000186100808ULL, 0x22a80880000a0808ULL,
+0xaaa8086880000000ULL, 0x802084800a2e9200ULL, 0xa280000000002008ULL, 0xa000000080080400ULL,
+0x2080010000000008ULL, 0x802020c00c028c80ULL, 0x2080000000140810ULL, 0x2a80086080080008ULL,
+0x2a800000a8000800ULL, 0xaa881800a2080800ULL, 0xaa98004080280808ULL, 0x004483d0c0300000ULL,
+0xa280002080080000ULL, 0x0000000000300000ULL, 0x22a1030000000008ULL, 0xa8a0301088880880ULL,
+0xaa80002080222808ULL, 0x85400c03fc030400ULL, 0x8a88000000000008ULL, 0xa008008010080008ULL,
+0x0000000000010000ULL, 0x0040100000301040ULL, 0x28800000a0002008ULL, 0x122482306cbc0eacULL,
+0x8020224222b8c6a0ULL, 0x802002004a82c284ULL, 0x0aa08fc440a41c80ULL, 0x888080d181385098ULL,
+0x0000000000000000ULL, 0x00c0b000000c0080ULL, 0x2208001000000800ULL, 0x0a28000000200000ULL,
+0x0000000300000000ULL, 0x00c1040000200000ULL, 0x0203020000000000ULL, 0x0248000000020000ULL,
+0x0000840000100000ULL, 0x0a808c00c000008cULL, 0x5200040040000004ULL, 0x02000c00000080a0ULL,
+0x0b0c000020000000ULL, 0x0b04000001000000ULL, 0x088c0010002000c0ULL, 0x80e08b00c0030c20ULL,
+0x0280000200014040ULL, 0x0000000000000000ULL, 0x0e20a0a008000020ULL, 0x0e280fd03f00111cULL,
+0x200080c020001000ULL, 0x8cc00c02c02f0400ULL, 0x480c0001000c404cULL, 0x0208014281080808ULL,
+0x000000000000fcfcULL, 0x004403300cf00030ULL, 0x2200000000004400ULL, 0x02202000c08c0c20ULL,
+0x02202022683a80a0ULL, 0x4020228028008c00ULL, 0x32208cc0002c0200ULL, 0x3ec00c0080304008ULL,
+0x0000000000000000ULL, 0x34000c00002c0000ULL, 0x0b00000100100030ULL, 0x0823018000000000ULL,
+0x0e8c001c01e00000ULL, 0x1200800600330000ULL, 0x4000110000000000ULL, 0x0080000300000000ULL,
+0x0800000000000000ULL, 0x08c08c04000c0000ULL, 0x0080400000880000ULL, 0x0a08000080c00008ULL,
+0x0800000304400000ULL, 0x0208000000c00000ULL, 0x2888300080400800ULL, 0x8dc0204400000000ULL,
+0xc0000000c0800000ULL, 0x0000c10000000000ULL, 0x24000c4010c00000ULL, 0x272000541d811000ULL,
+0x0200400000001000ULL, 0x0400000400001004ULL, 0xc08c007004001000ULL, 0x2048004000000000ULL,
+0x000000000003fcfcULL, 0x2aa030000cf8c800ULL, 0xe280000000000000ULL, 0x0a21008142000340ULL,
+0x0021002000b61040ULL, 0x800004064006d444ULL, 0x3aa0800300230008ULL, 0x0b00030000300000ULL,
+0x0000000000000000ULL, 0x01c080000000040cULL, 0x0100000000004000ULL, 0x0aa8018010001000ULL,
+0x0800000000100000ULL, 0x3000000000008c00ULL, 0x5400000013000000ULL, 0x02c0c00004004010ULL,
+0x5241100010000c00ULL, 0x0e00080000000808ULL, 0x5281000000000800ULL, 0x0a08108020000800ULL,
+0x0a80000000005210ULL, 0x0100000041000000ULL, 0x2a88000002080110ULL, 0x8520800000c00080ULL,
+0x01000010108c0100ULL, 0x0000000000000000ULL, 0x42a0420080000000ULL, 0x0020001004010010ULL,
+0xc4000000000c0000ULL, 0x01000c00c0200400ULL, 0x4600000100000000ULL, 0x0000000000000000ULL,
+0x0010001000000010ULL, 0x910400900820d030ULL, 0x2280000000000000ULL, 0xc2212004400040e4ULL,
+0x8001000000b61420ULL, 0xa00002a248e810b4ULL, 0x32008000002c0008ULL, 0x0c010034803c5010ULL,
+0x0000000000000000ULL, 0x85008002002c0000ULL, 0x0204001000004010ULL, 0x0120008000200000ULL,
+0x000010000c2000c0ULL, 0xccc0000000200000ULL, 0x0400000c00100040ULL, 0x0003300100004100ULL,
+0x4000551040000004ULL, 0x0e0080000c820808ULL, 0xc000000000080800ULL, 0xc803000000000000ULL,
+0x0a4000c000200000ULL, 0x0040000000c00000ULL, 0x0918145000405000ULL, 0x81400000c0300400ULL,
+0x0050000000000000ULL, 0xd000045000000000ULL, 0x0400004000400000ULL, 0x0420104010000110ULL,
+0x0700000000203000ULL, 0x34800300c0e00704ULL, 0x4440100044000400ULL, 0x0040000040000000ULL,
+0x0030000044000000ULL, 0xeaaca0008808c880ULL, 0x0a01000000200000ULL, 0x1220a300403ccf20ULL,
+0x002024c200b61044ULL, 0x802014346aa2d434ULL, 0x30008c00c0820c44ULL, 0x0a000000000c4800ULL,
+0x0000000000000000ULL, 0x0000404000340c90ULL, 0x08a8a10820800280ULL, 0x8128009022201000ULL,
+0x0020808228a000a0ULL, 0x0020400100410000ULL, 0x0400000110000000ULL, 0xa609000000200000ULL,
+0x8008330000d00000ULL, 0x8060100040404010ULL, 0xeaa00ea0ea00808cULL, 0x200c8020a0000020ULL,
+0x0408800020200000ULL, 0x0189001403200000ULL, 0xc00800000000c000ULL, 0x200430c00c300000ULL,
+0x0100300100004000ULL, 0x0000040000000000ULL, 0x2420000400001000ULL, 0x89a1200400000000ULL,
+0x20c8a000208c0000ULL, 0x8080000000000000ULL, 0x28a0108020210080ULL, 0xa2a84800a0880988ULL,
+0x258008000400c000ULL, 0x0140000000100000ULL, 0xa028a222a0aa0228ULL, 0xc060012054044040ULL,
+0x0010010400000000ULL, 0x00000050150c0114ULL, 0x0000008010c20010ULL, 0xaa088000a0200880ULL,
+0x0000000000000000ULL, 0x0700b0c0000c0000ULL, 0x2200040000080030ULL, 0x2aa8808040240800ULL,
+0x08b0500000000100ULL, 0x1000830400200000ULL, 0x4204000010000000ULL, 0x40c2200050040050ULL,
+0x0104404001010000ULL, 0x1a808c8103c00030ULL, 0x30900010c0000b00ULL, 0x200812b283000008ULL,
+0x000c000020e00000ULL, 0x2140000000400000ULL, 0x0288000080200000ULL, 0x8060a200c8a20280ULL,
+0x0400114010215000ULL, 0x0000000000000000ULL, 0x082b200002000010ULL, 0x22a0030000031000ULL,
+0x008100001000000cULL, 0x05400c00c0230400ULL, 0xca3000003c080100ULL, 0x0000000020000004ULL,
+0x0000000100000000ULL, 0x8004320813f5c000ULL, 0xa280080200000800ULL, 0xc22000044e334c20ULL,
+0x000004146e361024ULL, 0x800126806aa0d584ULL, 0xb000a0040023c41cULL, 0x0a083000803053d8ULL,
+0x0000000000000000ULL, 0x0000100000020000ULL, 0x0000000010000010ULL, 0x0000000045040004ULL,
+0x0000000000100000ULL, 0x0000020400000010ULL, 0x0003015000000000ULL, 0x0400000000000000ULL,
+0x0000000400000000ULL, 0x0100000000000800ULL, 0x0000001000000000ULL, 0x0000000000000000ULL,
+0x0000000040000000ULL, 0x0000000000000000ULL, 0x0004001000000000ULL, 0x0008001000000000ULL,
+0x0010000000000004ULL, 0x0000010100001000ULL, 0x0004000000000004ULL, 0x0000014040050014ULL,
+0x0014000000000040ULL, 0x5540000000041000ULL, 0x0000000000000000ULL, 0x0000040000000d00ULL,
+0x0000000000000000ULL, 0x0000000000100000ULL, 0x0001000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x4500000000040400ULL, 0x0000800000000400ULL,
+0x0000000000000000ULL, 0x13e080000020000cULL, 0xcf00001005100000ULL, 0x04a8008000200300ULL,
+0x00280100100000c0ULL, 0x1c8c000040200000ULL, 0x0600005000100000ULL, 0x050800000c104000ULL,
+0x4c10101000110000ULL, 0x0c00000000300000ULL, 0x22040c00100000c0ULL, 0x0800700010100000ULL,
+0x0000000000001000ULL, 0x0a08000010000040ULL, 0x0800034004210010ULL, 0x04e0000400000000ULL,
+0x0800030020000000ULL, 0x0000005000000000ULL, 0x0400110101304110ULL, 0x0428000010a01000ULL,
+0x060b000000800010ULL, 0x35810c00c020c000ULL, 0x00800c4321800000ULL, 0x4208088020000080ULL,
+0x040000111003ff00ULL, 0x0020900020202080ULL, 0x22888180a8000888ULL, 0x0225200542005420ULL,
+0x2020040400340020ULL, 0x10300424500cc444ULL, 0x3081a00400e00200ULL, 0x33001300c0300000ULL,
+0x0000000000000000ULL, 0x04003c0000000000ULL, 0x0a04001000100100ULL, 0x1408000001000000ULL,
+0x1800000044100000ULL, 0x3400040400000300ULL, 0x5000040801000040ULL, 0x4088401040000040ULL,
+0x1010110130100000ULL, 0xca800c3000300000ULL, 0x5a01000000080100ULL, 0x020280000cd01300ULL,
+0x0302000410200010ULL, 0x0000102000300000ULL, 0x0b09000000000000ULL, 0x20008004c4800004ULL,
+0x28c0410010000000ULL, 0x0004015041000050ULL, 0x0a01006000200200ULL, 0x0020d00000100040ULL,
+0x0010a00100900000ULL, 0x3500bf00c0030300ULL, 0x080c010000200d00ULL, 0x2248000004020010ULL,
+0x0000c00000000000ULL, 0x8044b00200e08000ULL, 0xaaa82aa2aa8a2aa8ULL, 0x0220002241c08604ULL,
+0x4200260440328444ULL, 0x68001226103008b4ULL, 0x3a0080c0b0000400ULL, 0x2a804804803c4008ULL,
+0x0000000000000000ULL, 0x04008c0300000400ULL, 0x008000c0000c0000ULL, 0x088001000000001cULL,
+0x0840000001000010ULL, 0x0400000000200c00ULL, 0x4244000101040000ULL, 0x4238007011100000ULL,
+0x1000d00100000010ULL, 0x1d00800400300000ULL, 0x4204080c00000000ULL, 0x2a88080080000008ULL,
+0x08001c0200001000ULL, 0x0a00000400000000ULL, 0x8a88003080080000ULL, 0x0521800400300000ULL,
+0x3200051000201000ULL, 0x0000000000000000ULL, 0x0020801404000000ULL, 0x322010401c0c101cULL,
+0x0c01100013000000ULL, 0x04003000c0204000ULL, 0x088c0020a0cc0000ULL, 0x2200000080000018ULL,
+0x0404000044000000ULL, 0x82a0b000008820b0ULL, 0x0000040020440000ULL, 0xc2650004403f1420ULL,
+0x0021340241b64464ULL, 0x8020040242c2d474ULL, 0x32018c0480288000ULL, 0x00800b0080300000ULL,
+0x0000000000000000ULL, 0x05008c0000040130ULL, 0xc0d8000000800000ULL, 0x0020000020200200ULL,
+0x23a2000120204000ULL, 0x5052100550104150ULL, 0x1000101100040000ULL, 0xc40001c301000000ULL,
+0x8288000000c00000ULL, 0x5150040144d01404ULL, 0xea8c0ea028ae088cULL, 0xc31010c000000c80ULL,
+0x0002000060000000ULL, 0xc80800f030000000ULL, 0x0000000400300000ULL, 0xc00080c00ff0c344ULL,
+0x00080001200c0000ULL, 0x0000050080000000ULL, 0x0328000300300000ULL, 0x082030000cc01040ULL,
+0xeb08800100004000ULL, 0x8030003300c80f00ULL, 0xfb0d0000e4ac0000ULL, 0x0020006080000008ULL,
+0x0500100100040000ULL, 0x1140000000000000ULL, 0xcb883330a0e00000ULL, 0xc000010050000080ULL,
+0x0010104005b54150ULL, 0x40111d5155001554ULL, 0x80000070140f0004ULL, 0x0b0830c3a0003380ULL,
+0x0000000000000000ULL, 0x04c13000000f830cULL, 0x2808000000000000ULL, 0x2810000000000800ULL,
+0x08c0080004400000ULL, 0x04c0240300801c20ULL, 0x4040000080000004ULL, 0x0000400100100010ULL,
+0x020001008000c0c0ULL, 0x1d008c000c3c0000ULL, 0x0080003000000800ULL, 0x2288080080000008ULL,
+0x0a84004020220000ULL, 0x0800080000100000ULL, 0xaa80004080400008ULL, 0x8024000400c01660ULL,
+0x80841c2001000104ULL, 0x0001000000000000ULL, 0x0020028020020280ULL, 0x0860404011900100ULL,
+0xec80080200000000ULL, 0x010103c100200400ULL, 0x0200004000000000ULL, 0x0000000000400400ULL,
+0x000010000003fcfcULL, 0x8040083238c20000ULL, 0x08800220a0920a00ULL, 0x08210004483c0c24ULL,
+0xc020240740b0a200ULL, 0x802006014a201494ULL, 0x3201233070ac0e00ULL, 0x08002806033a48a0ULL,
+0x0000000000000000ULL, 0x8020820028a00680ULL, 0x2000002000000104ULL, 0x22a80801100a0808ULL,
+0xa2a8002080000000ULL, 0xa000800008a08000ULL, 0x0000100000400000ULL, 0x8000002100000000ULL,
+0x0000010000004404ULL, 0xa2a0088080000888ULL, 0x0000000010400800ULL, 0xa280082080080008ULL,
+0x2280000080010008ULL, 0x2000000000000000ULL, 0x228800008c080808ULL, 0x8021828002a98200ULL,
+0xa200002000080000ULL, 0x0000040000000000ULL, 0x22a0000080000000ULL, 0x202882c200800080ULL,
+0xa000000001004000ULL, 0x000000c808a00600ULL, 0x0000000010000000ULL, 0x000001000000040cULL,
+0x0000000000000000ULL, 0x802002a2a8aa82a0ULL, 0x20000024a8088228ULL, 0x8020820001000000ULL,
+0x8020000000808280ULL, 0x8000000000000000ULL, 0x0020800000200280ULL, 0x2080082280a00888ULL,
+0x0000000000000000ULL, 0x0000015000000040ULL, 0x0000040000040000ULL, 0x0100010010001000ULL,
+0x0000003210008000ULL, 0x0000000404000000ULL, 0x0000000000000400ULL, 0x0200000000000000ULL,
+0x0000000000000100ULL, 0x5180014400004050ULL, 0x1000000014000000ULL, 0x4200000000000000ULL,
+0x0040200000000000ULL, 0x0201004000000000ULL, 0x0a00000000000010ULL, 0x0040200000800000ULL,
+0x0040051000000500ULL, 0x0000000100800400ULL, 0x6000000000000000ULL, 0x0000000000000000ULL,
+0x280000c1400040ccULL, 0x4180001000000000ULL, 0x00000000c1000104ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0080000000c00000ULL, 0x0004006066004000ULL,
+0x0000005000040440ULL, 0x0000106005804044ULL, 0x0000a10511004440ULL, 0x0000000000000110ULL,
+0x0000000000000000ULL, 0x0000000000080000ULL, 0xeb0808a020800080ULL, 0x29a80081002a1800ULL,
+0x0b2c000202100100ULL, 0x0001000000888000ULL, 0x2280102010000000ULL, 0x020000602a004110ULL,
+0x8a800160a6108100ULL, 0x0280000000000020ULL, 0x8a8000a0a8808208ULL, 0x0280882080500308ULL,
+0x0b18010020804100ULL, 0xeb080000c0080080ULL, 0x2b08000000810130ULL, 0x0000000008040020ULL,
+0xaa0a08e082894140ULL, 0x0000000000000000ULL, 0x202081409010001cULL, 0x8aa8805082806000ULL,
+0xeb082900289c0000ULL, 0x0000000000008000ULL, 0xf80c2e20002e0000ULL, 0xa288080420880888ULL,
+0x0000010000000000ULL, 0x0000000000102000ULL, 0x22880000a8a80808ULL, 0x022022a22aa880a0ULL,
+0x0000222222aa0620ULL, 0x0000022002800000ULL, 0x208080004028a000ULL, 0x2b888800801c0828ULL,
+0x0000000000000000ULL, 0x22e0828280a08028ULL, 0xaa88002082080308ULL, 0x0ea80080410a0040ULL,
+0x2a28222000a00000ULL, 0x8aa2808028a0a2a0ULL, 0x0200001000000000ULL, 0x82080000a0000000ULL,
+0x8800000082000808ULL, 0x2a008a0000300888ULL, 0x0a80080080080808ULL, 0xaa882800840b0808ULL,
+0x0a80000080000040ULL, 0xea080820a0000000ULL, 0xaa88080080080808ULL, 0x8040a2800a8024a0ULL,
+0xaa800020a0080808ULL, 0x0000040000000000ULL, 0x2a280a0080080880ULL, 0x2a20081080008a00ULL,
+0x2a88882088aa0008ULL, 0x81800202c0a01480ULL, 0xea88082082200000ULL, 0xaa88002080080008ULL,
+0x0000100000000000ULL, 0x802082a22aa0a2a0ULL, 0x2e80000000000000ULL, 0x0220a2a26aa0a2a8ULL,
+0x800022a2228a22a0ULL, 0x880002212e82c0b0ULL, 0x02a0aa0002a82228ULL, 0x2d808b0080380008ULL,
+0x0000000000000000ULL, 0x000407551c154244ULL, 0x2a00208088a02228ULL, 0x12a82182a2402a88ULL,
+0xe32821e020826d00ULL, 0x801130100ccc1330ULL, 0x028010c000841008ULL, 0x88a08002a0a664a0ULL,
+0x0048270080000100ULL, 0x00001f010cd10f30ULL, 0xe2242ce22aaea2a0ULL, 0xc2c00cc20ae22460ULL,
+0xe208003128021c10ULL, 0x2a2021c010821080ULL, 0x2a88202082202020ULL, 0x4010111104941410ULL,
+0xc80c02c182b00080ULL, 0x0000040000000000ULL, 0xe28030068002c300ULL, 0x2aa02024a2a22228ULL,
+0xe20889328aa22080ULL, 0x0000000000210100ULL, 0xaa0028e0a9b221a0ULL, 0x2000008080400000ULL,
+0x0000010041150404ULL, 0x0000105114410100ULL, 0xeaa82aa6aaaaaaa8ULL, 0x000000f44300c434ULL,
+0x0000222222b00020ULL, 0x0000002000000000ULL, 0x0000004014000000ULL, 0x0039b3f73fbcd3fcULL,
+0x0000000000000000ULL, 0x0000104015045040ULL, 0x20a80490a08800a0ULL, 0x40a8258410a909a0ULL,
+0xe0a8a2022aa2e2a0ULL, 0xc111010014000500ULL, 0x2080044041840004ULL, 0x28a8200220a2aba0ULL,
+0x008400a0a2840800ULL, 0x0101015451009464ULL, 0x20000ea0e02c2c2cULL, 0xe2a828a2aca2aaa8ULL,
+0x682020a228a222a0ULL, 0xe8882ae22aa2a2a0ULL, 0xe9a80e6022a24140ULL, 0x0011055005001040ULL,
+0x2aa8208229a0aaa4ULL, 0x0000040000000000ULL, 0x28a0228026a62260ULL, 0xe2a020a422a2a020ULL,
+0xe808a0022aa1a220ULL, 0x0000010014000100ULL, 0x28ac22802aa2a020ULL, 0x0020000000000000ULL,
+0x0100010100040000ULL, 0x0000000000000000ULL, 0x22a822a22a8aaaa0ULL, 0x0000000000000000ULL,
+0x0000102410800100ULL, 0x0000000000000000ULL, 0x0000000002000000ULL, 0x00000fb2a08c0aa8ULL,
+0x0000000000000000ULL, 0x4010005015440140ULL, 0x18c81c00b180001cULL, 0x2800048021820800ULL,
+0x8ab820c06a802580ULL, 0x00100170f4040000ULL, 0x4000144041041404ULL, 0x0ac800d0002e440cULL,
+0x20880820a2000808ULL, 0x400000f03f300c00ULL, 0xaa000ea22aa22aa0ULL, 0xa2880ac0a8942a20ULL,
+0xaa880a81a1804188ULL, 0xeea022a0aaa02080ULL, 0xaaa820a2aaa66120ULL, 0x0000005115800150ULL,
+0x2a880920a0840040ULL, 0x0000040000000000ULL, 0xaea82222aaa22a28ULL, 0x8a28041260055150ULL,
+0xa28824008aa28880ULL, 0x0000025014019000ULL, 0xea882ae02aa200a0ULL, 0x0000000000000000ULL,
+0x0000000040000400ULL, 0x0000000000000000ULL, 0xaaa82aa22aaaaaa0ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x002003003c80c000ULL, 0x0000020014000000ULL, 0x00200010a0980a20ULL,
+0x0000000000000000ULL, 0x0020001200801240ULL, 0x0a88000089800020ULL, 0xcaa00080a1000000ULL,
+0x0a200c0020a04080ULL, 0x4002034003840880ULL, 0x4690500190000050ULL, 0x2228004000601000ULL,
+0x0a803f00803f400cULL, 0x400033e24dd0cf34ULL, 0xaa80a2a229a220a0ULL, 0x0a224000002c0000ULL,
+0x028000202000008cULL, 0x0a08000070000030ULL, 0x00800c040020000cULL, 0x0000000002850000ULL,
+0x02881cc310200000ULL, 0x0000040004000000ULL, 0xcba8000400000080ULL, 0xcaa02c0680000000ULL,
+0xcc880002008c4080ULL, 0x300000f007f0cf0cULL, 0x0a80001080a00000ULL, 0x820880802a880a80ULL,
+0x0000050001040004ULL, 0x0000011000000000ULL, 0x0a8020a2a0202000ULL, 0x0000022202008000ULL,
+0x0000222212808000ULL, 0x0020226010000000ULL, 0x000033f33ff3c33cULL, 0x00288002a08c02a8ULL,
+0x0000000000000000ULL, 0x04408e0000008200ULL, 0x0808004000900000ULL, 0x0aa8200010ca00c0ULL,
+0x0ba80101005d4010ULL, 0x00018604802c8288ULL, 0x00049400101c0000ULL, 0x000c101110505010ULL,
+0x0000000000100000ULL, 0x30000c00c022000cULL, 0xd0c00dd0d51d431cULL, 0x0008000010100000ULL,
+0x000c1001a0280000ULL, 0x0bc80000c0000000ULL, 0x0a00000080280000ULL, 0x8000a00220308420ULL,
+0x0808000010301000ULL, 0x0000040000000000ULL, 0x0d00031480100000ULL, 0x07200000108c0300ULL,
+0x0bc0a0c000004000ULL, 0x8000b002c0208480ULL, 0x340c0100118c111cULL, 0x8008008020890000ULL,
+0x0000000000040010ULL, 0x0020b00320c1d0b0ULL, 0x00002000000c0000ULL, 0x0020be226e2008a0ULL,
+0x002010c03fb0a6a0ULL, 0x00202e222aaec284ULL, 0x00008f0000208400ULL, 0x0000000000300000ULL,
+};
+// Latin1 6%, Latin2 11%, Latin7 3%
+
+
+
+// Just for debugging. not thread-safe
+static char tri_string[4];
+char* Latin127Str(int trisub) {
+ tri_string[0] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 10) & 0x1f];
+ tri_string[1] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 5) & 0x1f];
+ tri_string[2] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 0) & 0x1f];
+ tri_string[3] = '\0';
+ return tri_string;
+}
+
+// Returns two bits per three-byte trigram, indicating
+// dont-care, Latin1 likely, Latin2 likely, and Latin7 (ISO-8859-13) likely
+int TrigramValue(const uint8* trisrc) {
+ int byte0_p = kMapToFiveBits[trisrc[0]];
+ int byte1_p = kMapToFiveBits[trisrc[1]];
+ int byte2_p = kMapToFiveBits[trisrc[2]];
+ int subscr = ((byte0_p) << 5) | byte1_p;
+ int temp = static_cast<int>((kLatin127Trigrams[subscr] >> (byte2_p * 2)));
+ //printf("%s=%d ", Latin127Str((subscr << 5) | byte2_p), temp & 3);
+ return temp & 3;
+}
+
+
+// Put out trigrams for surrounding 32 bytes for Latin encodings
+// Return true if more Latin2 & 7 than Latin1
+bool BoostLatin127Trigrams(int tri_block_offset,
+ DetectEncodingState* destatep) {
+ //printf("BoostLatin127Trigrams[%06x]\n", tri_block_offset);
+ int excess_latin27 = 0;
+ int srclen = destatep->limit_src - destatep->initial_src;
+ int hi_limit = minint(tri_block_offset + 32, srclen - 2);
+ const uint8* trisrc = &destatep->initial_src[tri_block_offset];
+ const uint8* trisrclimit = &destatep->initial_src[hi_limit];
+ while (trisrc < trisrclimit) {
+ // Selectively boost Latin1, Latin2, or Latin7 and friends
+ int trigram_val = TrigramValue(trisrc);
+ if (trigram_val != 0) {
+ if (FLAGS_enc_detect_source) {
+ PsHighlight(trisrc, destatep->initial_src, trigram_val, 1);
+ }
+ if (trigram_val == kTriLatin1Likely) {
+ Boost(destatep, F_Latin1, kTrigramBoost);
+ Boost(destatep, F_CP1252, kTrigramBoost);
+ // We don't want to upset the relative rank of a declared 8859-15
+ Boost(destatep, F_ISO_8859_15, kTrigramBoost);
+ --excess_latin27;
+ } else if (trigram_val == kTriLatin2Likely) {
+ Boost(destatep, F_Latin2, kTrigramBoost);
+ Boost(destatep, F_CP1250, kTrigramBoost);
+ ++excess_latin27;
+ } else if (trigram_val == kTriLatin7Likely) {
+ Boost(destatep, F_ISO_8859_13, kTrigramBoost);
+ Boost(destatep, F_CP1257, kTrigramBoost);
+ // We don't want to upset the relative rank of a declared 8859-4 or -6
+ // for Estonian
+ Boost(destatep, F_Latin4, kTrigramBoost);
+ Boost(destatep, F_Latin6, kTrigramBoost);
+ ++excess_latin27;
+ }
+ }
+
+ ++trisrc;
+ }
+ //printf("\n");
+
+ return (0 < excess_latin27);
+}
+
+
+
+// Boost any encodings that need extra detection help, then prune
+// src is first unscanned byte
+// slowend means extra pruning when dropping out of initial slow scan
+// final means last call -- no bigram at src
+void BoostPrune(const uint8* src, DetectEncodingState* destatep,
+ int prunereason) {
+ int delta_asciipairs = destatep->next_interesting_pair[AsciiPair] -
+ destatep->prior_interesting_pair[AsciiPair];
+ int delta_otherpairs = destatep->next_interesting_pair[OtherPair] -
+ destatep->prior_interesting_pair[OtherPair];
+
+ if (prunereason == PRUNE_FINAL) {
+ // We are about done
+ // If we get here with very little accumulated data, the initial hints
+ // were too strong, so we derate them to n+1 / 12 for n bigrams
+ if (!destatep->hints_derated &&
+ (destatep->next_interesting_pair[OtherPair] < kDerateHintsBelow)) {
+ int n = destatep->next_interesting_pair[OtherPair];
+
+ // Map N pairs to (N+1)/12 portions of the initial hints, etc.
+ // Floor of 3/12 -- 1/12 and 2/12 are too easy to overcome
+ int m = maxint(3, (n + 1));
+ for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
+ int original_delta = destatep->hint_prob[i];
+ int scaled_delta = (original_delta * m) / kDerateHintsBelow;
+ destatep->enc_prob[i] -= original_delta;
+ destatep->enc_prob[i] += scaled_delta;
+ }
+ destatep->hints_derated = true;
+ if (destatep->debug_data != NULL) {
+ // Show derated-hint result
+ char buff[32];
+ snprintf(buff, sizeof(buff), "Hints %d/%d", m, kDerateHintsBelow);
+ SetDetailsEncLabel(destatep, buff);
+ }
+ }
+ }
+
+
+ ++destatep->prune_count;
+
+ if (prunereason != PRUNE_FINAL) {
+ // Early outs
+ if (destatep->rankedencoding_list_len <= 1) { // nothing to prune
+ destatep->done = true;
+ return;
+ }
+
+ if ((destatep->prune_count > 0) &&
+ (delta_asciipairs + delta_otherpairs) == 0) {
+ // Nothing to do; must have just been called earlier
+ return;
+ }
+ }
+
+
+
+ // INCREMENT
+ // ====================
+ // Accumulate OtherPair probibilities over all active families
+ // AsciiPair probibilities are all done in ActiveSpecialBoostWhack
+ uint8 prior_bad_byte1 = ' '; // won't match first bad pair
+ uint8 prior_bad_byte2 = ' '; // won't match first bad pair
+ uint8 or_byte1 = 0; // Track if any current pair has a high bit
+ int counted_otherpairs = 0;
+ uint8 prior_byte1x2x = 0;
+ for (int i = 0; i < delta_otherpairs; ++i) {
+ int watch1_incr = 0;
+ int watch2_incr = 0;
+ int next_pair = destatep->prior_interesting_pair[OtherPair] + i;
+
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 1];
+ uint8 byte1x2x = (byte1 & 0xf0) | ((byte2 >> 4) & 0x0f);
+ int weightshift = destatep->interesting_weightshift[OtherPair][next_pair];
+
+ int offset_byte12 = destatep->interesting_offsets[OtherPair][next_pair];
+
+ // To help distinguish some Cyrillic, Arabic, Greek, Hebrew, Thai
+ // Remember if this is a CDEF pair immediately following the previous pair
+ // 8xxx CxCx or CxCx 8xxx
+ bool next_pair_consec_hi = false;
+ if (ConsecutivePair(destatep, next_pair)) {
+ if ((byte1x2x & 0xcc) == 0xcc) { // 8xxx CxCx
+ next_pair_consec_hi = true;
+ } else if ((prior_byte1x2x & 0xcc) == 0xcc) { // CxCx 8xxx
+ next_pair_consec_hi = true;
+ }
+ }
+ //printf("prior/cur/consec %02x %02x %d\n",
+ // prior_byte1x2x, byte1x2x, next_pair_consec_hi);
+ prior_byte1x2x = byte1x2x;
+
+ or_byte1 |= byte1;
+ uint8 byte1f = byte1;
+ // Flip top bit of subscript to better separate quadrant 4 (esp. for Hebrew)
+ byte1f ^= (byte2 & 0x80);
+
+ // If the same bigram occurred recently, don't increment again
+ bool pair_used = false;
+ if (!RepeatedBigram(destatep, byte1, byte2)) {
+ ++counted_otherpairs;
+ pair_used = true;
+ // Boost both charset= declared encodings, so
+ // Nearly-same probability nearby encoding doesn't drift to the top
+ if (!FLAGS_demo_nodefault) {
+ destatep->enc_prob[destatep->declared_enc_1] += kDeclaredEncBoost >> weightshift;
+ destatep->enc_prob[destatep->declared_enc_2] += kDeclaredEncBoost >> weightshift;
+ }
+ bool was_bad_pair = false;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int incr_shift = 0;
+ int rankedencoding = destatep->rankedencoding_list[j];
+ Encoding enc = kMapToEncoding[rankedencoding];
+
+ // For binary, Skip over repeated marker bytes, such as 02, FF, etc.
+ if ((rankedencoding == F_BINARY) &&
+ RepeatedBinary(destatep, byte1, byte2)) {
+ incr_shift = 2; // count 1/4 as much if repeated
+ }
+
+ // If byte 1x2x for this encoding is exactly zero, illegal byte pair
+ // Don't increment, but instead penalize
+ const UnigramEntry* ue = &unigram_table[rankedencoding];
+ if (ue->b12[byte1x2x] == 0) {
+ // Don't whack consecutive duplicate bad pairs -- overkill
+ if ((byte1 != prior_bad_byte1) || (byte2 != prior_bad_byte2)) {
+ // Extra whack for illegal pair in this encoding
+ Whack(destatep, rankedencoding, kBadPairWhack >> weightshift);
+ was_bad_pair = true;
+ }
+ } else {
+ // OK to do the real increment
+ int incr = ue->b1[byte1f] + ue->b2[byte2] + ue->b12[byte1x2x];
+ if ((ue->b12[byte1x2x] & 0x01) != 0) {
+ // Use a more-precise table
+ int byte32x32 = ((byte1 & 0x1f) << 5) | (byte2 & 0x1f);
+ int hiressub = (byte2 & 0x60) >> 5; // select w/bits 5&6 of byte 2
+ DCHECK(ue->hires[hiressub] != NULL);
+ incr += ue->hires[hiressub][byte32x32];
+ } else {
+ // Default final offset
+ incr += ue->so;
+ }
+ incr >>= incr_shift;
+
+ incr >>= weightshift;
+ destatep->enc_prob[rankedencoding] += incr; // The actual increment
+
+ if (FLAGS_enc_detect_detail2) {
+ if (watch1_rankedenc == rankedencoding) {watch1_incr = incr;}
+ if (watch2_rankedenc == rankedencoding) {watch2_incr = incr;}
+ }
+ }
+
+
+ // If consecutive pair of high bytes, give slight boost to one-byte
+ // encodings that have a full alphabet in the high bytes
+ if (next_pair_consec_hi && HighAlphaEncoding(enc)) {
+ Boost(destatep, rankedencoding, kDeclaredEncBoost >> weightshift);
+ }
+ } // End for j < rankedencoding_list_len
+
+ if (was_bad_pair) {
+ prior_bad_byte1 = byte1;
+ prior_bad_byte2 = byte2;
+ }
+
+ // Fold in per-bigram most likely encoding for first N bigrams
+ if (next_pair < kBestPairsCount) {
+ int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
+ Boost(destatep, best_enc, kBestEncBoost >> weightshift);
+ }
+
+ // Possibly score 32 trigrams around a bigram to better separate
+ // Latin1 from Latin2 and Latin7. Especially helpful for detecting
+ // mis-labelled Hungarian latin2.
+ // If looking and at bigram 0,8,16,... do full scoring, else just 1 tri
+ if (destatep->do_latin_trigrams ||
+ destatep->looking_for_latin_trigrams) {
+ // If just looking, do full scan every 8 times
+ // Just look up one trigram the other 7 and do full scan if Latin2,7
+ bool scan32 = false;
+ const uint8* trisrc = &destatep->initial_src[offset_byte12 - 1];
+ if (!destatep->do_latin_trigrams) {
+ if ((i & 7) == 0 || trisrc + 3 > destatep->limit_src) {
+ scan32 = true;
+ } else {
+ scan32 = (kTriLatin1Likely < TrigramValue(trisrc));
+ }
+ }
+ if (destatep->do_latin_trigrams || scan32) {
+ // Just score each block of 32 bytes once
+ int tri_block_offset = offset_byte12 & ~0x1f;
+ if (destatep->trigram_highwater_mark <= tri_block_offset) {
+ bool turnon = BoostLatin127Trigrams(tri_block_offset, destatep);
+ if (FLAGS_counts && !destatep->do_latin_trigrams && turnon) {
+ ++doing_used; // First time
+ }
+ if (FLAGS_enc_detect_source) {
+ if (!destatep->do_latin_trigrams && turnon) {
+ // First time
+ PsHighlight(trisrc, destatep->initial_src, 0, 2);
+ }
+ }
+ destatep->do_latin_trigrams |= turnon;
+ destatep->trigram_highwater_mark = tri_block_offset + 32;
+ }
+ }
+ }
+
+ } // end if RepeatedBigram()
+
+ // Keep track of initial byte high 3 bits
+ ++destatep->byte32_count[byte1 >> 5];
+
+
+ // TODO: boost subset/superset also
+ // Boost(destatep, kRelatedEncoding[best_enc], kBestEncBoost);
+
+ if (destatep->debug_data != NULL) {
+ // Show detail entry for this bigram
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%c%02x%02x%c%c",
+ pair_used ? ' ' : '[',
+ byte1,
+ byte2,
+ pair_used ? ' ' : ']',
+ (weightshift == 0) ? ' ' : '-');
+
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[OtherPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ if (FLAGS_enc_detect_detail2) {
+ if ((watch1_incr != 0) || (watch2_incr != 0)) {
+ // Show increment detail for this encoding
+ char buff[32];
+ snprintf(buff, sizeof(buff), "%c%d %c%d",
+ (watch1_incr < 0) ? '-' : '+', watch1_incr,
+ (watch2_incr < 0) ? '-' : '+', watch2_incr);
+ SetDetailsEncLabel(destatep, buff);
+ }
+ }
+ } // End for i
+
+
+ // If no high bit on, demote all the two-byte codes
+ // WAS BUG. This was inside the loop above and should be outside
+ if ((counted_otherpairs > 0) && ((or_byte1 & 0x80) == 0)) {
+ // No high bit in this group (just 02xx, etc.). Whack 2-byte codes
+ // This keeps SJS from creeping past Latin1 on illegal C0 bytes
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ Encoding enc = kMapToEncoding[rankedencoding];
+ if (TwoByteEncoding(enc)) {
+ Whack(destatep, rankedencoding, kGentlePairWhack * counted_otherpairs);
+ }
+ }
+ }
+
+
+ // BOOST
+ // ====================
+ if (AnyActive(destatep)) {
+ ActiveSpecialBoostWhack(src, destatep);
+ }
+
+ // Update for next time
+ destatep->prior_src = src;
+ destatep->prior_interesting_pair[AsciiPair] =
+ destatep->next_interesting_pair[AsciiPair];
+ destatep->prior_interesting_pair[OtherPair] =
+ destatep->next_interesting_pair[OtherPair];
+
+
+ // Do any pre-prune final adjustments
+ // ====================
+ if (prunereason == PRUNE_FINAL) {
+ // If UTF8 not in base state, whack
+ if (destatep->next_utf8_ministate != 0) {
+ Whack(destatep, F_UTF8, kGentlePairWhack * 2 * 1);
+ }
+ // If UTF8UTF8 not in base state, whack
+ if (destatep->next_utf8utf8_ministate != 0) {
+ Whack(destatep, F_UTF8UTF8, kGentlePairWhack * 2 * 1);
+ }
+
+ // If no valid UTF-8 char ever seen, whack
+ if (destatep->utf8_minicount[5] == 0) {
+ Whack(destatep, F_UTF8, kBadPairWhack * 8); // No sequence
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8); // No sequence
+ }
+
+ // If no valid UTF8UTF8 char ever seen, whack
+ if (destatep->utf8utf8_minicount[5] == 0) {
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8); // No sequence
+ }
+
+ // If not all four binary quadrants, whack BINARY;
+ // worth 2 pair if 3 quads, 4 pair if 1 or 2 quads
+ if (destatep->binary_quadrants_count < 4) {
+ if (destatep->binary_quadrants_count == 3) {
+ Whack(destatep, F_BINARY, kBadPairWhack * 2);
+ } else {
+ Whack(destatep, F_BINARY, kBadPairWhack * 4);
+ }
+ }
+
+ // If 1st pair is 1b24, choose between ISO-2022-xx
+ // <esc> $ ) C ISO-2022-KR [1b 24 29 43]
+ // <esc> $ ) A ISO-2022-CN [1b 24 29 41]
+ // <esc> $ ) G ISO-2022-CN [1b 24 29 47]
+ // <esc> $ * H ISO-2022-CN [1b 24 2a 48]
+ // <esc> ( B ISO-2022-JP [1b 28 42] to ASCII
+ // <esc> ( J ISO-2022-JP [1b 28 4a] to X0201
+ // <esc> $ @ ISO-2022-JP [1b 24 40] to X0208-78 twobyte
+ // <esc> $ B ISO-2022-JP [1b 24 42] to X0208-83 twobyte
+ if ((destatep->next_interesting_pair[OtherPair] >= 1) &&
+ Iso2022Active(destatep)) {
+ if ((destatep->interesting_pairs[OtherPair][0] == 0x1b) &&
+ (destatep->interesting_pairs[OtherPair][1] == 0x24)) {
+ int offset = destatep->interesting_offsets[OtherPair][0];
+ const uint8* esc_src = destatep->initial_src + offset;
+ if ((destatep->initial_src + offset) < (destatep->limit_src - 3)) {
+ if ((esc_src[2] == ')') && (esc_src[3] == 'C')) {
+ Boost(destatep, F_ISO_2022_KR, kBoostOnePair);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_JIS, kBadPairWhack);
+ } else if ((esc_src[2] == ')') && ((esc_src[3] == 'A') ||
+ (esc_src[3] == 'G'))) {
+ Boost(destatep, F_ISO_2022_CN, kBoostOnePair);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ Whack(destatep, F_JIS, kBadPairWhack);
+ } else if ((esc_src[2] == '@') || (esc_src[2] == 'B')) {
+ Boost(destatep, F_JIS, kBoostOnePair);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ }
+ } else {
+ // Incomplete escape sequence. Whack them all
+ Whack(destatep, F_JIS, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ }
+ }
+ }
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, "pre-final");
+ }
+ }
+
+ // PRUNE
+ // ====================
+ // Find current top two rankedencoding probabilities
+ ReRank(destatep);
+
+ if (prunereason == PRUNE_SLOWEND) {
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, "slow-end");
+ }
+ }
+
+ // Keep every rankedencoding with probablity >= top_prob - prune_difference
+ int prune_diff = destatep->prune_difference;
+ // If the top encoding is BINARY, it might be overstated, and we might
+ // therefore prune away the real encoding. Make the pruning delta
+ // twice as big.
+ if (destatep->top_rankedencoding == F_BINARY) {
+ prune_diff *= 2;
+ }
+ int keep_prob = destatep->top_prob - prune_diff;
+
+ // Tighten pruning difference (we start wide) for next time
+ if (destatep->prune_difference > kFinalPruneDifference) {
+ int decrement = kPruneDiffDecrement;
+ // If only ASCII pairs, small tighten; if some non-ASCII, full tighten
+ if (counted_otherpairs == 0) {
+ decrement >>= 1;
+ }
+ destatep->prune_difference -= decrement;
+ }
+
+ // Prune the list of active encoding families
+ destatep->active_special = 0;
+ int k = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ bool keep = true;
+ int rankedencoding = destatep->rankedencoding_list[j];
+
+ // If count is too low, ditch it
+ if (destatep->enc_prob[rankedencoding] < keep_prob) {
+ keep = false;
+ }
+
+ // If at end of slow section, ditch any 7-bit with zero evidence so far
+ if ((prunereason == PRUNE_SLOWEND) &&
+ SevenBitEncoding(kMapToEncoding[rankedencoding]) &&
+ (destatep->enc_prob[rankedencoding] <= 0) &&
+ (rankedencoding != destatep->top_rankedencoding)) {
+ keep = false;
+ }
+
+ // Keep it. This will always keep at least top_prob rankedencoding
+ if (keep) {
+ destatep->active_special |= kSpecialMask[kMapToEncoding[rankedencoding]];
+ destatep->rankedencoding_list[k++] = rankedencoding;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "%d prune", prune_diff / XLOG2);
+ SetDetailsEncLabel(destatep, buff);
+ }
+ destatep->rankedencoding_list_len = k;
+
+
+
+ // Force final result in some cases
+ // Do any post-prune final adjustments
+ if (prunereason == PRUNE_FINAL) {
+ // If no high-byte pairs, result is ASCII7, BINARY, UTF7, 2022, or HZ
+ if (destatep->next_interesting_pair[OtherPair] == 0) {
+ if ((destatep->top_rankedencoding != F_BINARY) &&
+ (destatep->top_rankedencoding != F_UTF7) &&
+ (destatep->top_rankedencoding != F_ISO_2022_CN) &&
+ (destatep->top_rankedencoding != F_ISO_2022_KR) &&
+ (destatep->top_rankedencoding != F_JIS) &&
+ (destatep->top_rankedencoding != F_HZ_GB_2312)) {
+ destatep->top_rankedencoding = F_ASCII_7_bit;
+ Boost(destatep, F_ASCII_7_bit, kBoostOnePair * 2);
+ }
+ }
+
+ // If some 89 pairs, not ISO_8859_x and vice versa
+ if (destatep->byte32_count[4] > 0) {
+ switch (destatep->top_rankedencoding) {
+ case F_ASCII: // ISO-8859-1
+ destatep->top_rankedencoding = F_CP1252;
+ // Better: destatep->enc_prob[F_ASCII] <==> destatep->enc_prob[F_CP1252]
+ Boost(destatep, F_CP1252, kBoostOnePair * 2);
+ break;
+ case F_Latin2: // ISO-8859-2
+ // Don't swap back; not superset
+ //destatep->top_rankedencoding = F_CP1250;
+ //Boost(destatep, F_CP1250, kBoostOnePair * 2);
+ break;
+ case F_Arabic: // ISO-8859-6
+ destatep->top_rankedencoding = F_CP1256;
+ Boost(destatep, F_CP1256, kBoostOnePair * 2);
+ break;
+ case F_Greek: // ISO-8859-7
+ // Don't swap -- not proper superset
+ // Capital Alpha tonos at 0xB6 in ISO-8859-7, 0xA2 in CP1253
+ //destatep->top_rankedencoding = F_CP1253;
+ //Boost(destatep, F_CP1253, kBoostOnePair * 2);
+ break;
+ case F_Hebrew: // ISO-8859-8
+ // Don't swap -- visual vs. logical
+ //destatep->top_rankedencoding = F_CP1255;
+ //Boost(destatep, F_CP1255, kBoostOnePair * 2);
+ break;
+ case F_Latin5: // ISO-8859-9
+ destatep->top_rankedencoding = F_CP1254;
+ Boost(destatep, F_CP1254, kBoostOnePair * 2);
+ break;
+ case F_ISO_8859_11: // ISO-8859-11
+ destatep->top_rankedencoding = F_CP874;
+ Boost(destatep, F_CP874, kBoostOnePair * 2);
+ break;
+ }
+ } else {
+ switch (destatep->top_rankedencoding) {
+ case F_CP1252: // ISO-8859-1
+ destatep->top_rankedencoding = F_ASCII;
+ Boost(destatep, F_ASCII, kBoostOnePair * 2);
+ break;
+ case F_CP1250: // ISO-8859-2
+ // Don't swap back; not superset
+ //destatep->top_rankedencoding = F_Latin2;
+ //Boost(destatep, F_Latin2, kBoostOnePair * 2);
+ break;
+ case F_CP1256: // ISO-8859-6
+ // Don't swap back -- not proper superset
+ //destatep->top_rankedencoding = F_Arabic;
+ //Boost(destatep, F_Arabic, kBoostOnePair * 2);
+ break;
+ case F_CP1253: // ISO-8859-7
+ // Don't swap back -- not proper superset
+ //destatep->top_rankedencoding = F_Greek;
+ //Boost(destatep, F_Greek, kBoostOnePair * 2);
+ break;
+ case F_CP1255: // ISO-8859-8
+ // Don't swap back -- not proper superset
+ //destatep->top_rankedencoding = F_Hebrew;
+ //Boost(destatep, F_Hebrew, kBoostOnePair * 2);
+ break;
+ case F_CP1254: // ISO-8859-9
+ destatep->top_rankedencoding = F_Latin5;
+ Boost(destatep, F_Latin5, kBoostOnePair * 2);
+ break;
+ case F_CP874: // ISO-8859-11
+ destatep->top_rankedencoding = F_ISO_8859_11;
+ Boost(destatep, F_ISO_8859_11, kBoostOnePair * 2);
+ break;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "final %d",
+ static_cast<int>(src - destatep->initial_src));
+ SetDetailsEncLabel(destatep, buff);
+
+ // Show winning encoding and its delta log base2 from 2nd-best
+ // Divide delta by XLOG2 to get log base 2
+ int delta = destatep->top_prob - destatep->second_top_prob;
+ if (delta < (2 * XLOG2)) {
+ delta /= XDECILOG2;
+ snprintf(buff, sizeof(buff), "+%d.%d %s ",
+ delta / 10, delta % 10,
+ MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
+ } else if (delta < (50 * XLOG2)) {
+ delta /= XLOG2;
+ snprintf(buff, sizeof(buff), "+%d %s",
+ delta,
+ MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
+ } else {
+ snprintf(buff, sizeof(buff), "%s",
+ MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
+ }
+ SetDetailsEncProbCopyOffset(destatep, destatep->top_rankedencoding, buff);
+ }
+ }
+
+
+ // FINISH
+ // ====================
+ // Eventual encoding result is reliable if big difference in top two, or if
+ // only Ascii7 ever encountered
+ // Also reliable if exactly one OtherPair and it's best encoding matches top
+ destatep->reliable = false;
+ if (destatep->next_interesting_pair[OtherPair] == 0) {
+ // Only 7-bit ASCII
+ destatep->reliable = true;
+ }
+ if ((destatep->top_prob - destatep->second_top_prob) >=
+ FLAGS_ced_reliable_difference) {
+ destatep->reliable = true;
+ }
+ if (destatep->next_interesting_pair[OtherPair] == 1) {
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][1];
+ int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
+ if (best_enc == destatep->top_rankedencoding) {
+ destatep->reliable = true;
+ }
+ }
+
+ // If we pruned to one encoding, we are done
+ if (destatep->rankedencoding_list_len == 1) {
+ destatep->reliable = true;
+ destatep->done = true;
+ }
+
+ // If we pruned to two or three encodings in the same *superset/subset
+ // rankedencoding* and enough pairs, we are done. Else keep going
+ if (destatep->rankedencoding_list_len == 2) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ if (kMapEncToBaseEncoding[enc0] == kMapEncToBaseEncoding[enc1]) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ }
+ }
+ } else if (destatep->rankedencoding_list_len == 3) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ Encoding enc2 = kMapToEncoding[destatep->rankedencoding_list[2]];
+ Encoding base0 = kMapEncToBaseEncoding[enc0];
+ Encoding base1 = kMapEncToBaseEncoding[enc1];
+ Encoding base2 = kMapEncToBaseEncoding[enc2];
+
+ if ((base0 == base1) && (base0 == base2)) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ }
+ }
+ }
+}
+
+
+// Accumulate aligned byte-pair at src
+// Occasionally, calc boost for some encodings and then prune the active list
+// weightshift is used to give low weight some text, such as inside tags
+// Returns true if pruning occurred
+bool IncrementAndBoostPrune(const uint8* src,
+ int remaining_length,
+ DetectEncodingState* destatep,
+ int weightshift,
+ int exit_reason) {
+ destatep->last_pair = src;
+ // Pick up byte pair, or very last byte plus 0x20
+ uint8 byte1 = src[0];
+ uint8 byte2 = 0x20;
+ if (1 < remaining_length) {byte2 = src[1];}
+
+ // whatset=0 for Ascii + ~, 1 for all others; see kTestPrintableAsciiTildePlus
+ int whatset = exit_reason - 1;
+ int next_pair = destatep->next_interesting_pair[whatset];
+
+ if (next_pair > 16) {
+ // If not clear by 16 bigrams, stop accumulating + ~ 00
+ if (byte1 == '+') {return false;}
+ if (byte1 == '~') {return false;}
+ if (byte1 == 0x00) {return false;}
+ }
+
+ // Remember pair in appropriate list
+ if (next_pair >= kMaxPairs) {
+ // We have filled up our alloted space for interesting pairs with no
+ // decision. If ASCII pairs full, just skip until end of slow loop; if
+ // non-Ascii pairs full, force done
+ if (whatset == OtherPair) {
+ destatep->done = true;
+ }
+ } else {
+ int offset = static_cast<int>(src - destatep->initial_src);
+ destatep->interesting_pairs[whatset][next_pair * 2 + 0] = byte1;
+ destatep->interesting_pairs[whatset][next_pair * 2 + 1] = byte2;
+ destatep->interesting_offsets[whatset][next_pair] = offset;
+ destatep->interesting_weightshift[whatset][next_pair] = weightshift;
+ ++destatep->next_interesting_pair[whatset];
+ ++next_pair;
+ }
+
+ // Prune now and then , but always if forced to be done
+ if (destatep->done || ((next_pair & kPruneMask) == 0)) { // Prune every M
+ BoostPrune(src + 2, destatep, PRUNE_NORMAL); // src+2 first unscanned byte
+ // may be off end of input
+ return true;
+ }
+ return false;
+}
+
+void DumpSummary(DetectEncodingState* destatep, int whatset, int n) {
+ printf(" %sSummary[%2d]: ", kWhatSetName[whatset],
+ destatep->next_interesting_pair[whatset]);
+ int limit = minint(n, destatep->next_interesting_pair[whatset]);
+ for (int i = 0; i < limit; ++i) {
+ printf("%02x%02x ",
+ destatep->interesting_pairs[whatset][i * 2 + 0],
+ destatep->interesting_pairs[whatset][i * 2 + 1]);
+ if ((i & 7) == 7) {printf(" ");}
+ }
+ printf("\n");
+}
+
+void BeginDetail(DetectEncodingState* destatep) {
+ fprintf(stderr, "%d [", NUM_RANKEDENCODING);
+ for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
+ fprintf(stderr, "(%s)", MyRankedEncName(e));
+ if ((e % 10) == 9) {fprintf(stderr, "\n ");}
+ }
+ fprintf(stderr, "] size-detail\n");
+ destatep->next_detail_entry = 0;
+}
+
+// Single character to represent (printable ASCII) gap between bigrams
+char DetailOffsetChar(int delta) {
+ if (delta == 0) {return ' ';}
+ if (delta <= 2) {return '=';}
+ if (delta <= 15) {return '_';}
+ if (delta <= 31) {return '+';}
+ {return ' ';}
+}
+
+void DumpDetail(DetectEncodingState* destatep) {
+ // Turn all counts into delta from previous entry
+ fprintf(stderr, "%d count-detail\n", destatep->next_detail_entry);
+ // Rewrite, recording deltas
+ for (int z = destatep->next_detail_entry - 1; z > 0; --z) {
+ destatep->debug_data[z].offset -= destatep->debug_data[z - 1].offset;
+ for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
+ destatep->debug_data[z].detail_enc_prob[e] -=
+ destatep->debug_data[z - 1].detail_enc_prob[e];
+ }
+ }
+ // Now print
+ for (int z = 0; z < destatep->next_detail_entry; ++z) {
+ // Highlight some entries ending in '!' with light red underbar
+ int len = destatep->debug_data[z].label.size();
+ if (destatep->debug_data[z].label[len - 1] == '!') {
+ fprintf(stderr, "1 0.9 0.9 do-flag\n");
+ }
+ fprintf(stderr, "(%c%s) %d [",
+ DetailOffsetChar(destatep->debug_data[z].offset),
+ destatep->debug_data[z].label.c_str(),
+ destatep->debug_data[z].best_enc);
+ for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
+ fprintf(stderr, "%d ", destatep->debug_data[z].detail_enc_prob[e]);
+ if ((e % 10) == 9) {fprintf(stderr, " ");}
+ }
+ fprintf(stderr, "] do-detail-e\n");
+ }
+ // Get ready for next time,if any
+ destatep->next_detail_entry = 0;
+}
+
+void PsRecurse(const char* buff) {
+ fprintf(stderr, "() end-detail (%s) start-detail\n\n", buff);
+}
+
+void DumpReliable(DetectEncodingState* destatep) {
+ printf("Not reliable: ");
+
+ // Find center of gravity of OtherPair list
+ int x_sum = 0;
+ int y_sum = 0;
+ int count = destatep->next_interesting_pair[OtherPair];
+ for (int i = 0; i < count; ++i) {
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][i * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][i * 2 + 1];
+ x_sum += byte2;
+ y_sum += byte1;
+ }
+ if (count == 0) {count = 1;} // adoid zdiv
+ int x_bar = x_sum / count;
+ int y_bar = y_sum / count;
+ printf("center %02X,%02X\n", x_bar, y_bar);
+
+ double closest_dist = 999.0;
+ int closest = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ const UnigramEntry* ue = &unigram_table[rankedencoding];
+ printf(" %8s = %4d at %02x,%02x +/- %02X,%02X ",
+ MyEncodingName(kMapToEncoding[rankedencoding]),
+ destatep->enc_prob[rankedencoding],
+ ue->x_bar, ue->y_bar,
+ ue->x_stddev, ue->y_stddev);
+ double x_diff = x_bar - ue->x_bar;
+ double y_diff = y_bar - ue->y_bar;
+ double dist = sqrt((x_diff * x_diff) + (y_diff * y_diff));
+ printf("(%3.1f)\n", dist);
+
+ if (closest_dist > dist) {
+ closest_dist = dist;
+ closest = rankedencoding;
+ }
+ }
+ printf("Closest=%s (%3.1f)\n",
+ MyEncodingName(kMapToEncoding[closest]), closest_dist);
+
+ for (int i = 0; i < 8; ++i) {
+ // Demote by distance to CG and see if that helps, or just quit
+ }
+}
+
+// Scan short single lines quickly for all printable ASCII
+// Return true if all bytes are in [20..7F], false otherwise
+bool QuickPrintableAsciiScan(const char* text, int text_length) {
+ const uint8* src = reinterpret_cast<const uint8*>(text);
+ const uint8* srclimit = src + text_length;
+ const uint8* srclimit8 = srclimit - 7;
+ while (src < srclimit8) {
+ // Exits on any byte outside [0x20..0x7E] range (HT LF CR exit)
+ uint8 mask = 0;
+ for (int i = 0; i < 8; ++i) mask |= (src[i]-0x20)|(src[i]+0x01);
+ if ((mask & 0x80) != 0) break;
+ src += 8;
+ }
+ while (src < srclimit) {
+ uint8 uc = *src++;
+ if (kIsPrintableAscii[uc] == 0) {return false;}
+ }
+ return true;
+}
+
+static const int kMaxScanBack = 192;
+
+// Return true if text is inside a tag or JS comment
+bool TextInsideTag(const uint8* isrc, const uint8* src, const uint8* srclimit) {
+ const uint8* srcbacklimit = src - kMaxScanBack;
+ if (srcbacklimit < isrc) {
+ srcbacklimit = isrc;
+ }
+ const uint8* ss = src - 1;
+ while (srcbacklimit <= ss) {
+ uint8 c = *ss--;
+ if ((c & ~0x02) == '<') {
+ // We found preceding < 3C or > 3E nearby
+ // Even cheaper: if inside a tag, we don't care what tag; return true
+ if (c == '<') {
+ return true;
+ }
+ // See if we are just after <title>...
+ if ((c == '>') && (isrc <= (ss - 5)) &&
+ (ss[-5] == '<') &&
+ ((ss[-4] | 0x20) == 't') &&
+ ((ss[-3] | 0x20) == 'i') &&
+ ((ss[-2] | 0x20) == 't') &&
+ ((ss[-1] | 0x20) == 'l') &&
+ ((ss[-0] | 0x20) == 'e')) {
+ return true;
+ }
+ // See if we are just after <SCRIPT language=javascript>...
+ if ((c == '>') && (isrc <= (ss - 5)) &&
+ (ss[-5] == 's') &&
+ ((ss[-4] | 0x20) == 'c') &&
+ ((ss[-3] | 0x20) == 'r') &&
+ ((ss[-2] | 0x20) == 'i') &&
+ ((ss[-1] | 0x20) == 'p') &&
+ ((ss[-0] | 0x20) == 't')) {
+ return true;
+ }
+ // Not in a tag
+ return false;
+ // See if we are just after JavaScript comment /* ...
+ } else if (c == '/') {
+ if (((ss + 2) < srclimit) && (ss[2] == '*')) {
+ // We backscanned to /*
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+const uint8* SkipToTagEnd(const uint8* src, const uint8* srclimit) {
+ const uint8* ss = src + 1;
+ while (ss <= srclimit) {
+ uint8 c = *ss++;
+ if ((c == '<') || (c == '>')) {
+ return ss;
+ }
+ }
+ return src + 2; // Always make progress, Otherwise we get an infinite loop
+}
+
+
+// Take a watch string and map to a ranked encoding. If no match, return -1
+int LookupWatchEnc(const string& watch_str) {
+ int watchval = -1;
+ // Mixed encoding maps to enc=UTF8UTF8
+ if (watch_str == "UTF8UTF8") {
+ watchval = F_UTF8UTF8;
+ } else {
+ Encoding enc;
+ if (EncodingFromName(watch_str.c_str(), &enc)) {
+ watchval = CompactEncDet::BackmapEncodingToRankedEncoding(enc);
+ }
+ }
+ return watchval;
+}
+
+// Return true if enc and enc2 are equal or one is a subset of the other
+// or either is UNKNOWN
+// also UTF8UTF8 is compatible with both Latin1 and UTF8
+bool CompatibleEnc(Encoding enc, Encoding enc2) {
+ if (enc < 0) {return false;}
+ if (NUM_ENCODINGS <= enc) {return false;}
+ if (enc2 < 0) {return false;}
+ if (NUM_ENCODINGS <= enc2) {return false;}
+ if (enc == enc2) {return true;}
+ if (kMapEncToBaseEncoding[enc] == kMapEncToBaseEncoding[enc2]) {return true;}
+
+ if (enc == ASCII_7BIT) {return true;}
+ if (enc2 == ASCII_7BIT) {return true;}
+ if (enc == UNKNOWN_ENCODING) {return true;}
+ if (enc2 == UNKNOWN_ENCODING) {return true;}
+ if (enc == UTF8UTF8) {
+ if (enc2 == UTF8) {return true;}
+ if (kMapEncToBaseEncoding[enc2] == ISO_8859_1) {return true;}
+ }
+ if (enc2 == UTF8UTF8) {
+ if (enc == UTF8) {return true;}
+ if (kMapEncToBaseEncoding[enc] == ISO_8859_1) {return true;}
+ }
+
+ return false;
+}
+
+// Return superset of enc and enc2, which must be compatible
+Encoding SupersetEnc(Encoding enc, Encoding enc2) {
+ //printf(" SupersetEnc (%s, ", MyEncodingName(enc)); // TEMP
+ //printf("%s) ", MyEncodingName(enc2));
+ //printf("= %s\n",
+ // MyEncodingName(kMapEncToSuperLevel[enc] >= kMapEncToSuperLevel[enc2] ?
+ // enc :enc2));
+ if (kMapEncToSuperLevel[enc] >= kMapEncToSuperLevel[enc2]) {
+ return enc;
+ }
+ return enc2;
+}
+
+
+// If unreliable, try rescoring to separate some encodings
+Encoding Rescore(Encoding enc, const uint8* isrc,
+ const uint8* srctextlimit, DetectEncodingState* destatep) {
+ if (FLAGS_counts) {++rescore_used;}
+ Encoding new_enc = enc;
+
+ bool rescore_change = false;
+
+ int count = destatep->next_interesting_pair[OtherPair];
+ int text_length = srctextlimit - isrc;
+ for (int i = 0; i < count; ++i) {
+ int bigram_offset = destatep->interesting_offsets[OtherPair][i];
+ uint8 byte0 = (0 < bigram_offset) ?
+ isrc[bigram_offset - 1] : 0x20;
+ uint8 byte1 = isrc[bigram_offset + 0]; // Known to have high bit on
+ uint8 byte2 = ((bigram_offset + 1) < text_length) ?
+ isrc[bigram_offset + 1] : 0x20;
+ uint8 byte3 = ((bigram_offset + 2) < text_length) ?
+ isrc[bigram_offset + 2] : 0x20;
+ int high_hash = ((byte0 & 0xc0) >> 0) |
+ ((byte1 & 0xc0) >> 1) |
+ ((byte2 & 0xc0) >> 4) |
+ ((byte3 & 0xc0) >> 6); // 00112233
+
+ // Boost HighAccent encodings for Ascii bit patterns
+ // 0x1x 0x0x
+ // 1010 1010
+ // 0010 0000
+ //
+ if ((high_hash & 0xaa) == 0x20) {
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (HighAccentEncoding(kMapToEncoding[rankedencoding])) {
+ // TODO: also want to boost Shift-JIS here if byte1 is Ax..Dx
+ // TEMP
+ //printf(" Rescore[%02x] %s +%d\n",
+ // high_hash, MyRankedEncName(rankedencoding), kGentlePairBoost);
+ Boost(destatep, rankedencoding, kGentlePairBoost);
+ rescore_change = true;
+ }
+ }
+ }
+
+ // Whack HighAccent encodings for high bit patterns
+ // 1x1x 1x1x
+ // 1010 1010
+ // 1010 1010
+ //
+ if ((high_hash & 0xaa) == 0xaa) {
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (HighAccentEncoding(kMapToEncoding[rankedencoding])) {
+ // TEMP
+ //printf(" Rescore[%02x] %s -%d\n",
+ // high_hash, MyRankedEncName(rankedencoding), kGentlePairBoost);
+ Whack(destatep, rankedencoding, kGentlePairBoost);
+ rescore_change = true;
+ }
+ }
+ }
+
+ }
+
+ if (rescore_change) {
+ ReRank(destatep);
+ new_enc = kMapToEncoding[destatep->top_rankedencoding];
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "=Rescore %s", MyEncodingName(new_enc));
+ SetDetailsEncProb(destatep,
+ 0,
+ CompactEncDet::BackmapEncodingToRankedEncoding(new_enc),
+ buff);
+ //// DumpDetail(destatep);
+ }
+
+ SimplePrune(destatep, kFinalPruneDifference);
+ CalcReliable(destatep);
+ }
+
+ //if (new_enc != enc) {
+ // // TEMP
+ // printf(" Rescore new top encoding = %s\n",
+ // MyRankedEncName(destatep->top_rankedencoding));
+ //}
+
+ return new_enc;
+}
+
+
+// Given an encoding, add its corresponding ranked encoding to the set
+void AddToSet(Encoding enc, int* list_len, int* list) {
+ // TEMP print
+ int item = CompactEncDet::BackmapEncodingToRankedEncoding(enc);
+ for (int i = 0; i < *list_len; ++i) {
+ if (list[i] == item) {
+ return; // Already in the set; don't add again
+ }
+ }
+ list[(*list_len)++] = item;
+}
+
+
+static const int kMinRobustBigramCount = 1000;
+static const int kMinKBToRobustScan = 64;
+static const int kMaxKBToRobustScan = 256;
+
+// Scan the first 64K or so, just doing raw bigram increments on given
+// probability list.
+// No fancy duplicate filtering or anything else here.
+// Returns number of bigrams counted
+int RobustScan(const char* text,
+ int text_length,
+ int robust_renc_list_len,
+ int* robust_renc_list,
+ int* robust_renc_probs) {
+ if (FLAGS_counts) {++robust_used;}
+ // Zero all the result probabilities
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ robust_renc_probs[i] = 0;
+ }
+ int max_fast_len = minint(text_length, (kMaxKBToRobustScan << 10));
+ const uint8* isrc = reinterpret_cast<const uint8*>(text);
+ const uint8* src = isrc;
+ const uint8* srclimitfast2 = isrc + max_fast_len - 1;
+ const uint8* srclimitfast4 = isrc + max_fast_len - 3;
+
+ int min_fast_len = minint(text_length, (kMinKBToRobustScan << 10));
+ const uint8* srclimitmin = isrc + min_fast_len - 1;
+
+ int bigram_count = 0;
+
+ if (FLAGS_enc_detect_source) {
+ PsSourceInit(kPsSourceWidth);
+ fprintf(stderr, "(RobustScan) do-src\n");
+ }
+
+ // Sum over a big chunk of the input
+ // Faster loop, no 7-bit-encodings possible, approx 3000 GB/sec
+ //====================================
+ while (src < srclimitfast2) {
+ // Skip to next interesting bigram
+
+ while (src < srclimitfast4) {
+ if (((src[0] | src[1] | src[2] | src[3]) & 0x80) != 0) break;
+ src += 4;
+ }
+
+ while (src < srclimitfast2) {
+ if ((src[0] & 0x80) != 0) break;
+ src++;
+ }
+
+ if (src < srclimitfast2) {
+ // We found a bigram with high bit on
+ // Next 5 lines commented out so we don't show all the source.
+ //const uint8* srctextlimit = isrc + text_length;
+ //if (FLAGS_enc_detect_source) {
+ // PsSource(src, isrc, srctextlimit);
+ // PsMark(src, 2, isrc, 0);
+ //}
+
+ uint8 byte1 = src[0];
+ uint8 byte2 = src[1];
+ uint8 byte1x2x = (byte1 & 0xf0) | ((byte2 >> 4) & 0x0f);
+ uint8 byte1f = byte1;
+ // Flip top bit of subscript to better separate quadrant 4 (esp. for Hebrew)
+ byte1f ^= (byte2 & 0x80);
+
+ // The real increments
+ for (int j = 0; j < robust_renc_list_len; ++j) {
+ int rankedencoding = robust_renc_list[j];
+ const UnigramEntry* ue = &unigram_table[rankedencoding];
+ int incr = ue->b1[byte1f] + ue->b2[byte2] + ue->b12[byte1x2x];
+ if ((ue->b12[byte1x2x] & 0x01) != 0) {
+ // Use a more-precise table
+ int byte32x32 = ((byte1 & 0x1f) << 5) | (byte2 & 0x1f);
+ int hiressub = (byte2 & 0x60) >> 5; // select w/bits 5&6 of byte 2
+ DCHECK(ue->hires[hiressub] != NULL);
+ incr += ue->hires[hiressub][byte32x32];
+ } else {
+ // Default final offset
+ incr += ue->so;
+ }
+ robust_renc_probs[j] += incr;
+ }
+
+ src += 2; // Continue after this bigram
+ ++bigram_count;
+
+ // Stop after 1000 bigrams reached, if at least 64KB scanned
+ if ((bigram_count > kMinRobustBigramCount) && (src > srclimitmin)) {
+ break;
+ }
+
+ }
+ }
+
+ if (FLAGS_enc_detect_source) {
+ fprintf(stderr, "( bigram_count = %d) do-src\n", bigram_count);
+ if (bigram_count == 0) {bigram_count = 1;} // zdiv
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ fprintf(stderr, "( enc[%-12.12s] = %7d (avg %d)) do-src\n",
+ MyRankedEncName(robust_renc_list[i]), robust_renc_probs[i],
+ robust_renc_probs[i] / bigram_count);
+ }
+ PsSourceFinish();
+ }
+
+ return bigram_count;
+}
+
+// If unreliable, rescan middle of document to see if we can get a better
+// answer. Rescan is only worthwhile if there are ~200 bytes or more left,
+// since the detector takes as much as 96 bytes of bigrams to decide.
+Encoding Rescan(Encoding enc,
+ const uint8* isrc,
+ const uint8* src,
+ const uint8* srctextlimit,
+ const char* url_hint,
+ const char* http_charset_hint,
+ const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint,
+ const CompactEncDet::TextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings,
+ DetectEncodingState* destatep) {
+ bool enc_is_reliable = destatep->reliable;
+ Encoding new_enc = enc;
+ Encoding second_best_enc =
+ kMapToEncoding[destatep->second_top_rankedencoding];
+
+ if (FLAGS_counts) {++rescan_used;}
+
+ int scanned_bytes = src - isrc;
+ int unscanned_bytes = srctextlimit - src;
+ int text_length = srctextlimit - isrc;
+ bool empty_rescan = true;
+
+ // See if enough bytes left to bother doing rescan
+ if (kMinRescanLength < unscanned_bytes) {
+ const char* text = reinterpret_cast<const char*>(isrc);
+
+ Encoding one_hint = destatep->http_hint;
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destatep->meta_hint != UNKNOWN_ENCODING)) {
+ one_hint = destatep->meta_hint;
+ }
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destatep->bom_hint != UNKNOWN_ENCODING)) {
+ one_hint = destatep->bom_hint;
+ }
+
+ // Go to an even offset to keep UTF-16 in synch
+ int middle_offset = (scanned_bytes + (unscanned_bytes / 2)) & ~1;
+ CHECK(middle_offset <= text_length);
+
+ // Look back a bit for a low byte to synchronize, else hope for the best.
+ const uint8* srcbacklimit = isrc + middle_offset - kMaxScanBack;
+ if (srcbacklimit < src) {
+ srcbacklimit = src;
+ }
+ const uint8* ss = isrc + middle_offset - 1;
+ while (srcbacklimit <= ss) {
+ if ((*ss & 0x80) == 0) {break;}
+ --ss;
+ }
+ // Leave middle offset unchanged unless we found a low byte
+ if (srcbacklimit <= ss) {
+ // Align to low byte or high byte just after it, whichever is even
+ middle_offset = (ss - isrc + 1) & ~1; // Even to keep UTF-16 in sync
+ }
+ CHECK(middle_offset <= text_length);
+
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, ">> Rescan");
+ // Print the current chart before recursive call
+ DumpDetail(destatep);
+
+ char buff[32];
+ snprintf(buff, sizeof(buff), ">> Rescan[%d..%d]",
+ middle_offset, text_length);
+ PsRecurse(buff);
+ }
+
+ int mid_bytes_consumed;
+ bool mid_is_reliable;
+ Encoding mid_second_best_enc;
+ CEDInternalFlags newflags = static_cast<CEDInternalFlags>(
+ kCEDRescanning + kCEDForceTags);
+ // Recursive call for rescan of half of remaining
+ Encoding mid_enc = InternalDetectEncoding(
+ newflags,
+ text + middle_offset,
+ text_length - middle_offset,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint, // User interface lang
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ &mid_bytes_consumed,
+ &mid_is_reliable,
+ &mid_second_best_enc);
+ destatep->reliable = mid_is_reliable;
+
+ empty_rescan = (mid_enc == ASCII_7BIT);
+
+ // Not the right decision if, e.g. enc=Greek, mid=ASCII7, one=KSC
+ // hence the !empty_rescan term
+ if (!empty_rescan && CompatibleEnc(one_hint, mid_enc)) {
+ // Encoding we just found is compatible with the
+ // single hint (if any); return superset
+ new_enc = SupersetEnc(one_hint, mid_enc);
+ }
+
+ // If original and mid are compatible, and both reliable,
+ // return new_enc = SupersetEnc(enc, mid_enc)
+ //
+ // This avoids too much weight on a bogus hint causing a RobustScan
+ // that gets the wrong answer
+ if (!empty_rescan && mid_is_reliable && enc_is_reliable &&
+ CompatibleEnc(enc, mid_enc)) {
+ new_enc = SupersetEnc(enc, mid_enc);
+ return new_enc;
+ }
+
+ // if mid unreliable, robustscan
+ // if mid empty, robustscan
+ // if original and mid not compatible, robustscan
+ // if mid and one_hint not compatible, robustscan
+
+ // If we found conflicting data, drop back and do a robust scan of a big
+ // chunk of the input over a set of candidate encodings
+ //
+ if (!mid_is_reliable ||
+ empty_rescan ||
+ !CompatibleEnc(enc, mid_enc) ||
+ !CompatibleEnc(one_hint, mid_enc)) {
+ int robust_renc_list_len; // Number of active encodings
+ int robust_renc_list[NUM_RANKEDENCODING]; // List of ranked encodings
+ int robust_renc_probs[NUM_RANKEDENCODING]; // List of matching probs
+
+ robust_renc_list_len = 0;
+ AddToSet(enc, &robust_renc_list_len, robust_renc_list);
+ AddToSet(second_best_enc, &robust_renc_list_len, robust_renc_list);
+ AddToSet(mid_enc, &robust_renc_list_len, robust_renc_list);
+ AddToSet(mid_second_best_enc, &robust_renc_list_len, robust_renc_list);
+ if (destatep->http_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->http_hint, &robust_renc_list_len, robust_renc_list);
+ }
+ if (destatep->meta_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->meta_hint, &robust_renc_list_len, robust_renc_list);
+ }
+ if (destatep->bom_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->bom_hint, &robust_renc_list_len, robust_renc_list);
+ }
+ if (destatep->tld_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->tld_hint, &robust_renc_list_len, robust_renc_list);
+ }
+
+ // Separate simple scan
+ // =====================
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, ">> RobustScan");
+ // Print the current chart before recursive call
+ DumpDetail(destatep);
+
+ char buff[32];
+ snprintf(buff, sizeof(buff), ">> RobustScan[0..%d]", text_length);
+ PsRecurse(buff);
+ }
+
+ int bigram_count = RobustScan(text, text_length,
+ robust_renc_list_len, robust_renc_list, robust_renc_probs);
+
+ // Default to new_enc and update if something better was found
+ int best_prob = -1;
+ // TEMP print
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ if (best_prob < robust_renc_probs[i]) {
+ best_prob = robust_renc_probs[i];
+ new_enc = kMapToEncoding[robust_renc_list[i]];
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "=Robust[%d] %s",
+ bigram_count, MyEncodingName(new_enc));
+ SetDetailsEncProb(destatep,
+ 0,
+ CompactEncDet::BackmapEncodingToRankedEncoding(new_enc),
+ buff);
+ }
+ }
+ } // End if enough bytes
+
+ return new_enc;
+}
+
+// With no hints at all, and perhaps on rescan, we relax our pickiness
+// and go ahead and accept the top multibyte encodings, even though
+// strictly their web pages should have declared an explicit encoding to
+// avoid the HTML standard's default ISO-8859-1.
+bool NoHintsCloseEnoughCompatible(Encoding top_enc) {
+ // First test accepts degenerate cases plus UTF8 and UTF8UTF8
+ if (CompatibleEnc(UTF8, top_enc)) {return true;}
+
+ // The rest look for exact match of base encoding
+ Encoding base_enc = kMapEncToBaseEncoding[top_enc];
+ if (base_enc == JAPANESE_EUC_JP) {return true;}
+ if (base_enc == JAPANESE_SHIFT_JIS) {return true;}
+ if (base_enc == CHINESE_BIG5) {return true;}
+ if (base_enc == CHINESE_GB) {return true;}
+ if (base_enc == KOREAN_EUC_KR) {return true;}
+ return false;
+}
+
+
+
+// Scan raw bytes and detect most likely encoding
+// Design goals:
+// Skip over big initial stretches of seven-bit ASCII bytes very quickly
+// Thread safe
+// Works equally well on
+// 50-byte queries,
+// 5000-byte email and
+// 50000-byte web pages
+// Length 0 input returns ISO_8859_1 (ASCII) encoding
+// Setting ignore_7bit_mail_encodings effectively turns off detection of
+// UTF-7, HZ, and ISO-2022-xx
+Encoding InternalDetectEncoding(
+ CEDInternalFlags flags, const char* text, int text_length,
+ const char* url_hint, const char* http_charset_hint,
+ const char* meta_charset_hint, const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const CompactEncDet::TextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings, int* bytes_consumed, bool* is_reliable,
+ Encoding* second_best_enc) {
+ *bytes_consumed = 0;
+ *is_reliable = false;
+ *second_best_enc = ASCII_7BIT;
+
+ if (text_length == 0) {
+ // Follow the spec. Text might be NULL.
+ *is_reliable = true;
+ return ISO_8859_1;
+ }
+
+ // For very short (20-50 byte) input strings that are highly likely to be
+ // all printable ASCII, our startup overhead might dominate. We have to do the
+ // full detection if the ISO-2022-xx, HZ, or UTF-7 encodings are possible.
+ // Otherwise, we can do a quick scan for printable ASCII.
+ if ((text_length <= 500) && ignore_7bit_mail_encodings &&
+ QuickPrintableAsciiScan(text, text_length)) {
+ *is_reliable = true;
+ return ASCII_7BIT;
+ }
+
+ // Go for the full boat detection
+ DetectEncodingState destate;
+ InitDetectEncodingState(&destate);
+
+ std::unique_ptr<DetailEntry[]> scoped_debug_data;
+ if (FLAGS_enc_detect_detail) {
+ // Allocate max 10 details per bigram
+ scoped_debug_data.reset(new DetailEntry[kMaxPairs * 10]);
+ destate.debug_data = scoped_debug_data.get();
+ // NOTE: destate and scoped_debug_data have exactly the same scope
+ // All other FLAGS_enc_detect_detail tests use destate.debug_data != NULL
+ }
+
+ // Get text length limits
+ // Typically, we scan the first 16KB looking for all encodings, then
+ // scan the rest (up to 256KB) a bit faster by no longer looking for
+ // interesting bytes below 0x80. This allows us to skip over runs of
+ // 7-bit-ASCII much more quickly.
+ int slow_len = minint(text_length, (FLAGS_enc_detect_slow_max_kb << 10));
+ int fast_len = minint(text_length, (FLAGS_enc_detect_fast_max_kb << 10));
+
+ // Initialize pointers.
+ // In general, we do not look at last 3 bytes of input in the fast scan
+ // We do, however want to look at the last byte or so in the slow scan,
+ // especilly in the case of a very short text whose only interesting
+ // information is a 3-byte UTF-8 character in the last three bytes.
+ // If necessary, we fake a last bigram with 0x20 space as a pad byte.
+ const uint8* isrc = reinterpret_cast<const uint8*>(text);
+ const uint8* src = isrc;
+ const uint8* srctextlimit = isrc + text_length;
+ const uint8* srclimitslow2 = isrc + slow_len - 1;
+ const uint8* srclimitfast2 = isrc + fast_len - 1;
+ const uint8* srclimitfast4 = isrc + fast_len - 3;
+ if (srclimitslow2 > srclimitfast2) {
+ srclimitslow2 = srclimitfast2;
+ }
+ destate.initial_src = isrc;
+ destate.limit_src = srclimitfast2 + 1; // May include last byte
+ destate.prior_src = isrc;
+ destate.last_pair = isrc - 2;
+
+ const char* scan_table = kTestPrintableAsciiTildePlus;
+ if (ignore_7bit_mail_encodings) {
+ // Caller wants to ignore UTF-7, HZ, ISO-2022-xx
+ // Don't stop on + (for UTF-7), nor on ~ (for HZ)
+ scan_table = kTestPrintableAscii;
+ }
+ int exit_reason = 0;
+
+ if (destate.debug_data != NULL) {
+ BeginDetail(&destate);
+ // Take any incoming watch encoding name and backmap to the corresponding
+ // ranked enum value
+ watch1_rankedenc = LookupWatchEnc(FLAGS_enc_detect_watch1);
+ if (watch1_rankedenc >= 0) {
+ fprintf(stderr, "/track-me %d def\n", watch1_rankedenc);
+ }
+
+ watch2_rankedenc = LookupWatchEnc(FLAGS_enc_detect_watch2);
+ if (watch2_rankedenc >= 0) {
+ fprintf(stderr, "/track-me2 %d def\n", watch2_rankedenc);
+ }
+
+ fprintf(stderr, "%% kDerateHintsBelow = %d\n", kDerateHintsBelow);
+ }
+ if (FLAGS_enc_detect_source) {
+ PsSourceInit(kPsSourceWidth);
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 4, isrc, 0);
+ }
+
+ // Apply hints, if any, to probabilities
+ // NOTE: Encoding probabilites are all zero at this point
+ ApplyHints(url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint,
+ corpus_type,
+ &destate);
+
+ // NOTE: probabilities up to this point are subject to derating for
+ // small numbers of bigrams.
+ // Probability changes after this point are not derated.
+
+ // Do first 4 bytes to pick off strong markers
+ InitialBytesBoost(isrc, text_length, &destate);
+
+ bool ignored_some_tag_text = false;
+ int tag_text_bigram_count = 0;
+
+ // Slower loop, approx 500 MB/sec (2.8 GHz P4)
+ // ASSERT(srclimitslow2 <= srclimitfast2);
+ //====================================
+ DoMoreSlowLoop:
+ while (src < srclimitslow2) {
+ // Skip to next interesting byte (this is the slower part)
+ while (src < srclimitslow2) {
+ uint8 uc = *src++;
+ if (scan_table[uc] != 0) {exit_reason = scan_table[uc]; src--; break;}
+ }
+
+ if (src < srclimitslow2) {
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit); // don't mark yet
+ }
+
+ int weightshift = 0;
+ // In the first 16KB, derate new text run inside <title>...</title> and
+ // inside <!-- ... -->
+ if (////((destate.last_pair + 6) <= src) && // if beyond last one
+ ////(tag_text_bigram_count < kMaxBigramsTagTitleText) &&
+ (corpus_type == CompactEncDet::WEB_CORPUS) && // and web page
+ !CEDFlagForceTags(flags)) { // and OK to skip
+ ////if (TextInsideTag(destate.last_pair + 2, src, srclimitslow2)) {
+ if (TextInsideTag(isrc, src, srclimitslow2)) {
+ if (tag_text_bigram_count >= kMaxBigramsTagTitleText) {
+ ignored_some_tag_text = true;
+ src = SkipToTagEnd(src, srclimitslow2);
+ continue;
+ } else {
+ weightshift = kWeightshiftForTagTitleText;
+ ++tag_text_bigram_count;
+ }
+ }
+ }
+ if (FLAGS_enc_detect_source) {
+ PsMark(src, 2, isrc, weightshift);
+ }
+ // Saves byte pair and offset
+ bool pruned = IncrementAndBoostPrune(src, srctextlimit - src,
+ &destate, weightshift, exit_reason);
+ // Advance; if inside tag, advance to end of tag
+ if (weightshift == 0) {
+ src += exit_reason; // 1 Ascii, 2 other
+ } else {
+ src += exit_reason; // 1 Ascii, 2 other
+ //// src = SkipToTagEnd(src, srclimitslow2);
+ }
+
+ if (pruned) {
+ // Scoring and active encodings have been updated
+ if (destate.done) {break;}
+ // Check if all the reasons for the slow loop have been pruned
+ // If so, go to fast loop
+ if (!SevenBitActive(&destate)) {break;}
+ }
+ }
+ }
+ //====================================
+
+ // We reached the end of a slow scan, possibly because no more SevenBitActive,
+ // or possibly are at end of source.
+ // If we are exactly at the end of the source, make sure we look at the very
+ // last byte.
+ bool very_last_byte_incremented = false;
+ if (src == (srctextlimit - 1)) {
+ exit_reason = scan_table[*src];
+ if (exit_reason != 0) {
+ // The very last byte is an interesting byte
+ // Saves byte pair and offset
+ //printf("Interesting very last slow byte = 0x%02x\n", *src);
+ IncrementAndBoostPrune(src, srctextlimit - src, &destate, 0, exit_reason);
+ very_last_byte_incremented = true;
+ }
+ }
+
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 2, isrc, 0);
+ }
+ // Force a pruning based on whatever we have
+ // Delete the seven-bit encodings if there is no evidence of them so far
+ BoostPrune(src, &destate, PRUNE_SLOWEND);
+
+ if (!destate.done) {
+ // If not clear yet on 7-bit-encodings and more bytes, do more slow
+ if (SevenBitActive(&destate) && (src < srclimitfast2)) {
+ // Increment limit by another xxxK
+ slow_len += (FLAGS_enc_detect_slow_max_kb << 10);
+ srclimitslow2 = isrc + slow_len - 1;
+ if (srclimitslow2 > srclimitfast2) {
+ srclimitslow2 = srclimitfast2;
+ }
+ if (!UTF7OrHzActive(&destate)) {
+ // We can switch to table that does not stop on + ~
+ scan_table = kTestPrintableAscii;
+ }
+ goto DoMoreSlowLoop;
+ }
+
+
+ exit_reason = 2;
+ // Faster loop, no 7-bit-encodings possible, approx 3000 GB/sec
+ //====================================
+ while (src < srclimitfast2) {
+ // Skip to next interesting byte (this is the faster part)
+ while (src < srclimitfast4) {
+ if (((src[0] | src[1] | src[2] | src[3]) & 0x80) != 0) break;
+ src += 4;
+ }
+
+ while (src < srclimitfast2) {
+ if ((src[0] & 0x80) != 0) break;
+ src++;
+ }
+
+ if (src < srclimitfast2) {
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 2, isrc, 0);
+ }
+ // saves byte pair and offset
+ bool pruned = IncrementAndBoostPrune(src, srctextlimit - src,
+ &destate, 0, exit_reason);
+ src += exit_reason; // 1 Ascii, 2 other
+ if (pruned) {
+ // Scoring and active encodings have been updated
+ if (destate.done) {break;}
+ }
+ }
+ }
+ //====================================
+ // We reached the end of fast scan
+
+ // If we are exactly at the end of the source, make sure we look at the very
+ // last byte.
+ if (src == (srctextlimit - 1) && !very_last_byte_incremented) {
+ exit_reason = scan_table[*src];
+ if (exit_reason != 0) {
+ // The very last byte is an interesting byte
+ // Saves byte pair and offset
+ //printf("Interesting very last fast byte = 0x%02x\n", *src);
+ IncrementAndBoostPrune(src, srctextlimit - src, &destate, 0, exit_reason);
+ very_last_byte_incremented = true;
+ }
+ }
+
+ } // End if !done
+
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 2, isrc, 0);
+ }
+ // Force a pruning based on whatever we have
+ BoostPrune(src, &destate, PRUNE_FINAL);
+
+ if (FLAGS_enc_detect_summary) {
+ DumpSummary(&destate, AsciiPair, 32);
+ DumpSummary(&destate, OtherPair, 32);
+ }
+ if (FLAGS_enc_detect_source) {
+ PsSourceFinish();
+ }
+ if (destate.debug_data != NULL) {
+ //// DumpDetail(&destate);
+ }
+
+
+ if (ignored_some_tag_text &&
+ (kMapToEncoding[destate.top_rankedencoding] == ASCII_7BIT)) {
+ // There were some interesting bytes, but only in tag text.
+ // Recursive call to reprocess looking at the tags this time.
+
+ if (destate.debug_data != NULL) {
+ SetDetailsEncLabel(&destate, ">> Recurse/tags");
+ // Print the current chart before recursive call
+ DumpDetail(&destate);
+
+ char buff[32];
+ snprintf(buff, sizeof(buff), ">> Recurse for tags");
+ PsRecurse(buff);
+ }
+
+ // Recursive call for high bytes in tags [no longer used, 1/16 tag score]
+ Encoding enc2 = InternalDetectEncoding(
+ kCEDForceTags, // force
+ text,
+ text_length,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint,
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ bytes_consumed,
+ is_reliable,
+ second_best_enc);
+
+ if (destate.debug_data != NULL) {
+ // Show winning encoding and dump PostScript
+ char buff[32];
+ snprintf(buff, sizeof(buff), "=2 %s", MyEncodingName(enc2));
+ SetDetailsEncProb(&destate,
+ 0,
+ CompactEncDet::BackmapEncodingToRankedEncoding(enc2),
+ buff);
+ DumpDetail(&destate);
+ }
+
+ return enc2;
+ }
+
+
+ // If the detected encoding does not match default/hints, or if the hints
+ // conflict with each other, mark as unreliable. This can be used to trigger
+ // further scoring.
+ // Three buckets of input documents;
+ // ~19% of the web no hints, and top == 7bit, Latin1, or CP1252
+ // ~79% of the web one or more hints, all same encoding X and top == X
+ // ~ 2% of the web one or more hints that are inconsistent
+
+ Encoding top_enc = kMapToEncoding[destate.top_rankedencoding];
+ Encoding one_hint = destate.http_hint;
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destate.meta_hint != UNKNOWN_ENCODING)) {
+ one_hint = destate.meta_hint;
+ }
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destate.bom_hint != UNKNOWN_ENCODING)) {
+ one_hint = destate.bom_hint;
+ }
+
+ bool found_compatible_encoding = true;
+ if (one_hint == UNKNOWN_ENCODING) {
+ // [~14% of the web] No hints, and top == 7bit, Latin1, or CP1252
+ if (!CompatibleEnc(ISO_8859_1, top_enc)) {
+ found_compatible_encoding = false;
+ // If there is nothing but a TLD hint and its top encoding matches, OK
+ if ((destate.tld_hint != UNKNOWN_ENCODING) &&
+ CompatibleEnc(destate.tld_hint, top_enc)) {
+ found_compatible_encoding = true;
+ }
+ }
+ } else if (CompatibleEnc(one_hint, destate.http_hint) &&
+ CompatibleEnc(one_hint, destate.meta_hint) &&
+ CompatibleEnc(one_hint, destate.bom_hint)) {
+ // [~83% of the web] One or more hints, all same encoding X and top == X
+ if (!CompatibleEnc(one_hint, top_enc)) {
+ // [~ 2% of the web] Oops, not the declared encoding
+ found_compatible_encoding = false;
+ }
+ } else {
+ // [~ 3% of the web] Two or more hints that are inconsistent
+ one_hint = UNKNOWN_ENCODING;
+ found_compatible_encoding = false;
+ }
+
+ // If we turned Latin1 into Latin2 or 7 via trigrams, don't fail it here
+ if (destate.do_latin_trigrams) {
+ if (CompatibleEnc(kMapToEncoding[F_Latin1], top_enc) ||
+ CompatibleEnc(kMapToEncoding[F_Latin2], top_enc) ||
+ CompatibleEnc(kMapToEncoding[F_CP1250], top_enc) ||
+ CompatibleEnc(kMapToEncoding[F_ISO_8859_13], top_enc)) {
+ found_compatible_encoding = true;
+ destate.reliable = true;
+ }
+ }
+
+ // If top encoding is not compatible with the hints, but it is reliably
+ // UTF-8, accept it anyway.
+ // This will perform badly with mixed UTF-8 prefix plus another encoding in
+ // the body if done too early, so we want to be rescanning.
+ if (!found_compatible_encoding &&
+ destate.reliable &&
+ NoHintsCloseEnoughCompatible(top_enc) &&
+ (destate.next_interesting_pair[OtherPair] >= kStrongPairs) &&
+ CEDFlagRescanning(flags)) {
+ found_compatible_encoding = true;
+ }
+
+ // Hold off on this so Rescan() can see if the original encoding was reliable
+ //if (!found_compatible_encoding) {
+ // destate.reliable = false;
+ //}
+
+ // If unreliable, try rescoring to separate some encodings
+ if (!destate.reliable || !found_compatible_encoding) {
+ top_enc = Rescore(top_enc, isrc, srctextlimit, &destate);
+ }
+
+ *second_best_enc = kMapToEncoding[destate.second_top_rankedencoding];
+
+ // If unreliable, and not already rescanning,
+ // rescan middle of document to see if we can get a better
+ // answer. Rescan is only worthwhile if there are ~200 bytes or more left,
+ // since the detector takes as much as 96 bytes of bigrams to decide.
+ //
+ // CANNOT retry ISO-2022-xx HZ etc. because no declaration escape at the front
+ // or we may land in the middle of some partial state. Skip them all.
+ //
+ if ((!destate.reliable || !found_compatible_encoding) &&
+ !CEDFlagRescanning(flags) &&
+ !SevenBitEncoding(top_enc)) {
+ top_enc = Rescan(top_enc,
+ isrc,
+ src,
+ srctextlimit,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint,
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ &destate);
+ } else {
+ if (!found_compatible_encoding) {
+ destate.reliable = false;
+ }
+ }
+
+ if (destate.debug_data != NULL) {
+ // Dump PostScript
+ DumpDetail(&destate);
+ }
+
+ *bytes_consumed = src - isrc + 1; // We looked 1 byte beyond src
+ *is_reliable = destate.reliable;
+ return top_enc;
+}
+
+Encoding CompactEncDet::DetectEncoding(
+ const char* text, int text_length, const char* url_hint,
+ const char* http_charset_hint, const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+ int* bytes_consumed, bool* is_reliable) {
+ if (FLAGS_ced_echo_input) {
+ string temp(text, text_length);
+ fprintf(stderr, "CompactEncDet::DetectEncoding()\n%s\n\n", temp.c_str());
+ }
+
+ if (FLAGS_counts) {
+ encdet_used = 0;
+ rescore_used = 0;
+ rescan_used = 0;
+ robust_used = 0;
+ looking_used = 0;
+ doing_used = 0;
+ ++encdet_used;
+ }
+ if (FLAGS_dirtsimple) {
+ // Just count first 64KB bigram encoding probabilities for each encoding
+ int robust_renc_list_len; // Number of active encodings
+ int robust_renc_list[NUM_RANKEDENCODING]; // List of ranked encodings
+ int robust_renc_probs[NUM_RANKEDENCODING]; // List of matching probs
+
+ for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
+ robust_renc_list[i] = i;
+ }
+ robust_renc_list_len = NUM_RANKEDENCODING;
+
+ RobustScan(text, text_length,
+ robust_renc_list_len, robust_renc_list, robust_renc_probs);
+
+ // Pick off best encoding
+ int best_prob = -1;
+ Encoding enc = UNKNOWN_ENCODING;
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ if (best_prob < robust_renc_probs[i]) {
+ best_prob = robust_renc_probs[i];
+ enc = kMapToEncoding[robust_renc_list[i]];
+ }
+ }
+
+ *bytes_consumed = minint(text_length, (kMaxKBToRobustScan << 10));
+ *is_reliable = true;
+ if (FLAGS_counts) {
+ printf("CEDcounts ");
+ while (encdet_used--) {printf("encdet ");}
+ while (rescore_used--) {printf("rescore ");}
+ while (rescan_used--) {printf("rescan ");}
+ while (robust_used--) {printf("robust ");}
+ while (looking_used--) {printf("looking ");}
+ while (doing_used--) {printf("doing ");}
+ printf("\n");
+ }
+
+ return enc;
+ }
+
+ Encoding second_best_enc;
+ Encoding enc = InternalDetectEncoding(kCEDNone,
+ text,
+ text_length,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint, // User interface lang
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ bytes_consumed,
+ is_reliable,
+ &second_best_enc);
+ if (FLAGS_counts) {
+ printf("CEDcounts ");
+ while (encdet_used--) {printf("encdet ");}
+ while (rescore_used--) {printf("rescore ");}
+ while (rescan_used--) {printf("rescan ");}
+ while (robust_used--) {printf("robust ");}
+ while (looking_used--) {printf("looking ");}
+ while (doing_used--) {printf("doing ");}
+ printf("\n");
+ }
+
+#if defined(HTML5_MODE)
+ // Map all the Shift-JIS variants to Shift-JIS when used in Japanese locale.
+ if (language_hint == JAPANESE && IsShiftJisOrVariant(enc)) {
+ enc = JAPANESE_SHIFT_JIS;
+ }
+
+ // 7-bit encodings (except ISO-2022-JP), and some obscure encodings not
+ // supported in WHATWG encoding standard are marked as ASCII to keep the raw
+ // bytes intact.
+ switch (enc) {
+ case ISO_2022_KR:
+ case ISO_2022_CN:
+ case HZ_GB_2312:
+ case UTF7:
+ case UTF16LE:
+ case UTF16BE:
+
+ case CHINESE_EUC_DEC:
+ case CHINESE_CNS:
+ case CHINESE_BIG5_CP950:
+ case JAPANESE_CP932:
+ case MSFT_CP874:
+ case TSCII:
+ case TAMIL_MONO:
+ case TAMIL_BI:
+ case JAGRAN:
+ case BHASKAR:
+ case HTCHANAKYA:
+ case BINARYENC:
+ case UTF8UTF8:
+ case TAM_ELANGO:
+ case TAM_LTTMBARANI:
+ case TAM_SHREE:
+ case TAM_TBOOMIS:
+ case TAM_TMNEWS:
+ case TAM_WEBTAMIL:
+ case KDDI_SHIFT_JIS:
+ case DOCOMO_SHIFT_JIS:
+ case SOFTBANK_SHIFT_JIS:
+ case KDDI_ISO_2022_JP:
+ case SOFTBANK_ISO_2022_JP:
+ enc = ASCII_7BIT;
+ break;
+ default:
+ break;
+ }
+#endif
+
+ return enc;
+}
+
+
+// Return top encoding hint for given string
+Encoding CompactEncDet::TopEncodingOfLangHint(const char* name) {
+ string normalized_lang = MakeChar8(string(name));
+ int n = HintBinaryLookup8(kLangHintProbs, kLangHintProbsSize,
+ normalized_lang.c_str());
+ if (n < 0) {return UNKNOWN_ENCODING;}
+
+ // Charset is eight bytes, probability table is eight bytes
+ int toprankenc =
+ TopCompressedProb((const char *)&kLangHintProbs[n].key_prob[kMaxLangKey],
+ kMaxLangVector);
+ return kMapToEncoding[toprankenc];
+}
+
+// Return top encoding hint for given string
+Encoding CompactEncDet::TopEncodingOfTLDHint(const char* name) {
+ string normalized_tld = MakeChar4(string(name));
+ int n = HintBinaryLookup4(kTLDHintProbs, kTLDHintProbsSize,
+ normalized_tld.c_str());
+ if (n < 0) {return UNKNOWN_ENCODING;}
+
+ // TLD is four bytes, probability table is 12 bytes
+ int toprankenc =
+ TopCompressedProb((const char *)&kTLDHintProbs[n].key_prob[kMaxTldKey],
+ kMaxTldVector);
+ return kMapToEncoding[toprankenc];
+}
+
+// Return top encoding hint for given string
+Encoding CompactEncDet::TopEncodingOfCharsetHint(const char* name) {
+ string normalized_charset = MakeChar44(string(name));
+ int n = HintBinaryLookup8(kCharsetHintProbs, kCharsetHintProbsSize,
+ normalized_charset.c_str());
+ if (n < 0) {return UNKNOWN_ENCODING;}
+
+ // Charset is eight bytes, probability table is eight bytes
+ int toprankenc =
+ TopCompressedProb((const char *)&kCharsetHintProbs[n].key_prob[kMaxCharsetKey],
+ kMaxCharsetVector);
+ return kMapToEncoding[toprankenc];
+}
+
+const char* CompactEncDet::Version(void) {
+ return kVersion;
+}
diff --git a/contrib/google-ced/compact_enc_det.h b/contrib/google-ced/compact_enc_det.h
new file mode 100644
index 0000000..01adf45
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det.h
@@ -0,0 +1,83 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+
+#include "util/encodings/encodings.h" // for Encoding
+#include "util/languages/languages.h" // for Language
+
+#include <string.h>
+
+namespace CompactEncDet {
+ // We may want different statistics, depending on whether the text being
+ // identfied is from the web, from email, etc. This is currently ignored,
+ // except WEB_CORPUS enables ignoring chars inside tags.
+ enum TextCorpusType {
+ WEB_CORPUS,
+ XML_CORPUS,
+ QUERY_CORPUS, // Use this for vanilla plaintext
+ EMAIL_CORPUS,
+ NUM_CORPA, // always last
+ };
+
+ // Scan raw bytes and detect most likely encoding
+ // Design goals:
+ // Skip over big initial stretches of seven-bit ASCII bytes very quickly
+ // Thread safe
+ // Works equally well on
+ // 50-byte queries,
+ // 5000-byte email and
+ // 50000-byte web pages
+ // Length 0 input returns ASCII (aka ISO-8859-1 or Latin1)
+ //
+ // Inputs: text and text_length
+ // web page's url (preferred) or just
+ // top-level domain name (e.g. "com") or NULL as a hint
+ // web page's HTTPheader charset= string (e.g. "Latin1") or NULL as a hint
+ // web page's <meta> tag charset= string (e.g. "utf-8") or NULL as a hint
+ // an Encoding or UNKNOWN_ENCODING as a hint
+ // a Language or UNKNOWN_LANGUAGE as a hint
+ // corpus type from the list above. Currently ignored; may select
+ // different probability tables in the future
+ // ignore_7bit if true says to NOT return the pure seven-bit encodings
+ // ISO-2022-JP (aka JIS), ISO-2022-CN, ISO-2022-KR, HZ, and UTF-7.
+ // This may save a little scoring time on pure printable ASCII input text
+ // Outputs: bytes_consumed says how much of text_length was actually examined
+ // is_reliable set true if the returned encoding is at least 2**10 time more
+ // probable then the second-best encoding
+ // Return value: the most likely encoding for the input text
+ //
+ // Setting ignore_7bit_mail_encodings effectively turns off detection of
+ // UTF-7, HZ, and ISO-2022-xx. It is recommended that this flag be true
+ // when corpus_type is QUERY_CORPUS.
+ Encoding DetectEncoding(
+ const char* text, int text_length, const char* url_hint,
+ const char* http_charset_hint, const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+ int* bytes_consumed, bool* is_reliable);
+
+ // Support functions for unit test program
+ int BackmapEncodingToRankedEncoding(Encoding enc);
+ Encoding TopEncodingOfLangHint(const char* name);
+ Encoding TopEncodingOfTLDHint(const char* name);
+ Encoding TopEncodingOfCharsetHint(const char* name);
+ const char* Version(void);
+} // End namespace CompactEncDet
+
+#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_H_
diff --git a/contrib/google-ced/compact_enc_det_generated_tables.h b/contrib/google-ced/compact_enc_det_generated_tables.h
new file mode 100644
index 0000000..d2174a1
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_generated_tables.h
@@ -0,0 +1,6326 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_GENERATED_TABLES_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_GENERATED_TABLES_H_
+
+#include "compact_enc_det.h"
+#include "compact_enc_det_generated_tables2.h"
+#include "util/basictypes.h"
+#include "util/encodings/encodings.pb.h"
+
+enum RankedEncoding {
+ F_ASCII_7_bit, // [0] encoding 24
+ F_Latin1, // [1] encoding 0
+ F_UTF8, // [2] encoding 22
+ F_GB, // [3] encoding 14
+ F_CP1252, // [4] encoding 27
+ F_KSC, // [5] encoding 16
+ F_SJS, // [6] encoding 11
+ F_EUC_JP, // [7] encoding 10
+ F_BIG5, // [8] encoding 13
+ F_Latin2, // [9] encoding 1
+ F_CP1251, // [10] encoding 26
+ F_CP1256, // [11] encoding 35
+ F_CP1250, // [12] encoding 29
+ F_Latin5, // [13] encoding 8
+ F_ISO_8859_11, // [14] encoding 33
+ F_ISO_8859_15, // [15] encoding 30
+ F_CP1257, // [16] encoding 32
+ F_CP1255, // [17] encoding 36
+ F_KOI8R, // [18] encoding 25
+ F_GBK, // [19] encoding 45
+ F_Greek, // [20] encoding 6
+ F_JIS, // [21] encoding 12
+ F_CP1254, // [22] encoding 31
+ F_CP1253, // [23] encoding 41
+ F_CP932, // [24] encoding 21
+ F_Hebrew, // [25] encoding 7
+ F_KOI8U, // [26] encoding 28
+ F_ISO_8859_5, // [27] encoding 4
+ F_CP874, // [28] encoding 34
+ F_ISO_8859_13, // [29] encoding 43
+ F_Latin4, // [30] encoding 3
+ F_MACINTOSH, // [31] encoding 53
+ F_GB18030, // [32] encoding 46
+ F_CP852, // [33] encoding 39
+ F_Arabic, // [34] encoding 5
+ F_BIG5_HKSCS, // [35] encoding 47
+ F_CP866, // [36] encoding 42
+ F_UTF_16BE, // [37] encoding 57
+ F_Latin3, // [38] encoding 2
+ F_UTF_16LE, // [39] encoding 58
+ F_HZ_GB_2312, // [40] encoding 62
+ F_CSN_369103, // [41] encoding 40
+ F_ISO_2022_KR, // [42] encoding 44
+ F_Latin6, // [43] encoding 9
+ F_UTF7, // [44] encoding 54
+ F_ISO_2022_CN, // [45] encoding 48
+ F_BIG5_CP950, // [46] encoding 20
+ F_JAGRAN, // [47] encoding 52
+ F_BHASKAR, // [48] encoding 55
+ F_HTCHANAKYA, // [49] encoding 56
+ F_TSCII, // [50] encoding 49
+ F_TAM, // [51] encoding 50
+ F_TAB, // [52] encoding 51
+ F_EUC_CN, // [53] encoding 15
+ F_EUC, // [54] encoding 18
+ F_CNS, // [55] encoding 19
+ F_UTF_32BE, // [56] encoding 59
+ F_UTF_32LE, // [57] encoding 60
+ F_X_BINARYENC, // [58] encoding 61
+ F_X_UTF8UTF8, // [59] encoding 63
+ F_X_TAM_ELANGO, // [60] encoding 64
+ F_X_TAM_LTTMBARANI, // [61] encoding 65
+ F_X_TAM_SHREE, // [62] encoding 66
+ F_X_TAM_TBOOMIS, // [63] encoding 67
+ F_X_TAM_TMNEWS, // [64] encoding 68
+ F_X_TAM_WEBTAMIL, // [65] encoding 69
+ F_UTF8CP1252, // [66] encoding 63
+ NUM_RANKEDENCODING
+};
+
+static const Encoding kMapToEncoding[NUM_RANKEDENCODING] = {
+ ASCII_7BIT, // encoding 24
+ ISO_8859_1, // encoding 0
+ UTF8, // encoding 22
+ CHINESE_GB, // encoding 14
+ MSFT_CP1252, // encoding 27
+ KOREAN_EUC_KR, // encoding 16
+ JAPANESE_SHIFT_JIS, // encoding 11
+ JAPANESE_EUC_JP, // encoding 10
+ CHINESE_BIG5, // encoding 13
+ ISO_8859_2, // encoding 1
+ RUSSIAN_CP1251, // encoding 26
+ MSFT_CP1256, // encoding 35
+ MSFT_CP1250, // encoding 29
+ ISO_8859_9, // encoding 8
+ ISO_8859_11, // encoding 33
+ ISO_8859_15, // encoding 30
+ MSFT_CP1257, // encoding 32
+ MSFT_CP1255, // encoding 36
+ RUSSIAN_KOI8_R, // encoding 25
+ GBK, // encoding 45
+ ISO_8859_7, // encoding 6
+ JAPANESE_JIS, // encoding 12
+ MSFT_CP1254, // encoding 31
+ MSFT_CP1253, // encoding 41
+ JAPANESE_CP932, // encoding 21
+ ISO_8859_8, // encoding 7
+ RUSSIAN_KOI8_RU, // encoding 28
+ ISO_8859_5, // encoding 4
+ MSFT_CP874, // encoding 34
+ ISO_8859_13, // encoding 43
+ ISO_8859_4, // encoding 3
+ MACINTOSH_ROMAN, // encoding 53
+ GB18030, // encoding 46
+ CZECH_CP852, // encoding 39
+ ISO_8859_6, // encoding 5
+ BIG5_HKSCS, // encoding 47
+ RUSSIAN_CP866, // encoding 42
+ UTF16BE, // encoding 57
+ ISO_8859_3, // encoding 2
+ UTF16LE, // encoding 58
+ HZ_GB_2312, // encoding 62
+ CZECH_CSN_369103, // encoding 40
+ ISO_2022_KR, // encoding 44
+ ISO_8859_10, // encoding 9
+ UTF7, // encoding 54
+ ISO_2022_CN, // encoding 48
+ CHINESE_BIG5_CP950, // encoding 20
+ JAGRAN, // encoding 52
+ BHASKAR, // encoding 55
+ HTCHANAKYA, // encoding 56
+ TSCII, // encoding 49
+ TAMIL_MONO, // encoding 50
+ TAMIL_BI, // encoding 51
+ CHINESE_EUC_CN, // encoding 15
+ CHINESE_EUC_DEC, // encoding 18
+ CHINESE_CNS, // encoding 19
+ UTF32BE, // encoding 59
+ UTF32LE, // encoding 60
+ BINARYENC, // encoding 61
+ UTF8UTF8, // encoding 63
+ TAM_ELANGO, // encoding 64
+ TAM_LTTMBARANI, // encoding 65
+ TAM_SHREE, // encoding 66
+ TAM_TBOOMIS, // encoding 67
+ TAM_TMNEWS, // encoding 68
+ TAM_WEBTAMIL, // encoding 69
+ UTF8UTF8, // encoding 63
+};
+
+// Massaged TLD or charset, followed by packed encoding probs
+typedef struct {
+ unsigned char key_prob[20];
+} HintEntry;
+
+static const HintEntry kLangHintProbs[] = { // MaxRange 192
+ {{0x61,0x62,0x6b,0x68,0x61,0x7a,0x69,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "abkhazia"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x66,0x61,0x72,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "afar____"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x66,0x72,0x69,0x6b,0x61,0x61,0x6e, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "afrikaan"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x61,0x6c,0x62,0x61,0x6e,0x69,0x61,0x6e, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "albanian"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x61,0x6d,0x68,0x61,0x72,0x69,0x63,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "amharic_"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x72,0x61,0x62,0x69,0x63,0x5f,0x5f, 0x03,0x84,0x53,0xa2,0x11,0x3b,0x62,0xbc,0x34,0x10,0x51,0x83,}}, // "arabic__"
+ // ASCII-7-bit=132 Latin1=83 UTF8=162 CP1252=59 CP1256=188 CP1250=52 Arabic=131 [top CP1256]
+ {{0x61,0x72,0x6d,0x65,0x6e,0x69,0x61,0x6e, 0x01,0x5f,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "armenian"
+ // ASCII-7-bit=95 UTF8=190 [top UTF8]
+ {{0x61,0x73,0x73,0x61,0x6d,0x65,0x73,0x65, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "assamese"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x79,0x6d,0x61,0x72,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "aymara__"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x7a,0x65,0x72,0x62,0x61,0x69,0x6a, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "azerbaij"
+ // UTF8=191 [top UTF8]
+ {{0x62,0x61,0x73,0x68,0x6b,0x69,0x72,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bashkir_"
+ // UTF8=191 [top UTF8]
+ {{0x62,0x61,0x73,0x71,0x75,0x65,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "basque__"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x62,0x65,0x6c,0x61,0x72,0x75,0x73,0x69, 0xa1,0xb5,0x71,0xa1,0x72,0x97,0xab,0x81,0x8d,0x00,0x00,0x00,}}, // "belarusi"
+ // CP1251=181 KOI8R=161 KOI8U=151 ISO-8859-5=171 CP866=141 [top CP1251]
+ {{0x62,0x65,0x6e,0x67,0x61,0x6c,0x69,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bengali_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x62,0x69,0x68,0x61,0x72,0x69,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bihari__"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x62,0x69,0x73,0x6c,0x61,0x6d,0x61,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bislama_"
+ // UTF8=191 [top UTF8]
+ {{0x62,0x6f,0x73,0x6e,0x69,0x61,0x6e,0x5f, 0x91,0xaf,0x21,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bosnian_"
+ // Latin2=175 CP1250=185 [top CP1250]
+ {{0x62,0x72,0x65,0x74,0x6f,0x6e,0x5f,0x5f, 0x11,0xb5,0x21,0x97,0x81,0xab,0x11,0xa1,0x00,0x00,0x00,0x00,}}, // "breton__"
+ // Latin1=181 CP1252=151 Latin5=171 ISO-8859-15=161 [top Latin1]
+ {{0x62,0x75,0x6c,0x67,0x61,0x72,0x69,0x61, 0x03,0x70,0x47,0xad,0x11,0x45,0x51,0xb5,0x71,0x95,0x81,0x9f,}}, // "bulgaria"
+ // ASCII-7-bit=112 Latin1=71 UTF8=173 CP1252=69 CP1251=181 KOI8R=149 ISO-8859-5=159 [top CP1251]
+ {{0x62,0x75,0x72,0x6d,0x65,0x73,0x65,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "burmese_"
+ // UTF8=191 [top UTF8]
+ {{0x63,0x61,0x74,0x61,0x6c,0x61,0x6e,0x5f, 0x03,0x8b,0xb8,0xa0,0x11,0xa4,0xa1,0x96,0x10,0x61,0x31,0x00,}}, // "catalan_"
+ // ASCII-7-bit=139 Latin1=184 UTF8=160 CP1252=164 ISO-8859-15=150 Latin3=49 [top Latin1]
+ {{0x63,0x68,0x65,0x72,0x6f,0x6b,0x65,0x65, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cherokee"
+ // UTF8=191 [top UTF8]
+ {{0x63,0x68,0x69,0x6e,0x65,0x73,0x65,0x5f, 0x01,0x5c,0x12,0xa8,0xbb,0x11,0x74,0x21,0x6d,0xa1,0x7d,0x00,}}, // "chinese_"
+ // ASCII-7-bit=92 UTF8=168 GB=187 KSC=116 BIG5=109 GBK=125 [top GB]
+ {{0x63,0x68,0x69,0x6e,0x65,0x73,0x65,0x74, 0x06,0x73,0x5f,0xad,0x59,0x43,0x36,0x21,0xb9,0x10,0xa1,0x38,}}, // "chineset"
+ // ASCII-7-bit=115 Latin1=95 UTF8=173 GB=89 CP1252=67 KSC=54 BIG5=185 BIG5_HKSCS=56 [top BIG5]
+ {{0x63,0x6f,0x72,0x73,0x69,0x63,0x61,0x6e, 0x12,0xaf,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "corsican"
+ // Latin1=175 UTF8=185 [top UTF8]
+ {{0x63,0x72,0x65,0x6f,0x6c,0x65,0x73,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "creolesa"
+ // UTF8=191 [top UTF8]
+ {{0x63,0x72,0x6f,0x61,0x74,0x69,0x61,0x6e, 0x03,0x91,0x7b,0xa6,0x11,0x86,0x41,0xac,0x21,0xb4,0x31,0x4d,}}, // "croatian"
+ // ASCII-7-bit=145 Latin1=123 UTF8=166 CP1252=134 Latin2=172 CP1250=180 CP1257=77 [top CP1250]
+ {{0x63,0x7a,0x65,0x63,0x68,0x5f,0x5f,0x5f, 0x01,0x89,0x11,0xb1,0x61,0x98,0x21,0xb5,0x10,0x41,0x7d,0x00,}}, // "czech___"
+ // ASCII-7-bit=137 UTF8=177 Latin2=152 CP1250=181 CP852=125 [top CP1250]
+ {{0x64,0x61,0x6e,0x69,0x73,0x68,0x5f,0x5f, 0x03,0x99,0xb8,0xa6,0x11,0x9a,0x41,0x38,0x21,0x32,0x21,0x84,}}, // "danish__"
+ // ASCII-7-bit=153 Latin1=184 UTF8=166 CP1252=154 Latin2=56 CP1250=50 ISO-8859-15=132 [top Latin1]
+ {{0x64,0x68,0x69,0x76,0x65,0x68,0x69,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dhivehi_"
+ // UTF8=191 [top UTF8]
+ {{0x64,0x75,0x74,0x63,0x68,0x5f,0x5f,0x5f, 0x03,0xb1,0xae,0xa3,0x11,0xa1,0x41,0x41,0x21,0x44,0x21,0x7f,}}, // "dutch___"
+ // ASCII-7-bit=177 Latin1=174 UTF8=163 CP1252=161 Latin2=65 CP1250=68 ISO-8859-15=127 [top ASCII-7-bit]
+ {{0x64,0x7a,0x6f,0x6e,0x67,0x6b,0x68,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dzongkha"
+ // UTF8=191 [top UTF8]
+ {{0x65,0x6e,0x67,0x6c,0x69,0x73,0x68,0x5f, 0x06,0xb9,0xa0,0xa2,0x5d,0x94,0x55,0x21,0x56,0x61,0x69,0x00,}}, // "english_"
+ // ASCII-7-bit=185 Latin1=160 UTF8=162 GB=93 CP1252=148 KSC=85 BIG5=86 ISO-8859-15=105 [top ASCII-7-bit]
+ {{0x65,0x73,0x70,0x65,0x72,0x61,0x6e,0x74, 0x03,0x89,0xb4,0xa2,0x12,0xaa,0x45,0x61,0x4c,0x21,0xa0,0x00,}}, // "esperant"
+ // ASCII-7-bit=137 Latin1=180 UTF8=162 CP1252=170 KSC=69 CP1250=76 ISO-8859-15=160 [top Latin1]
+ {{0x65,0x73,0x74,0x6f,0x6e,0x69,0x61,0x6e, 0x03,0x90,0xab,0xb1,0x11,0x91,0xa2,0x7e,0xa3,0xc2,0x8e,0x98,}}, // "estonian"
+ // ASCII-7-bit=144 Latin1=171 UTF8=177 CP1252=145 ISO-8859-15=126 CP1257=163 ISO-8859-13=142 Latin4=152 [top UTF8]
+ {{0x66,0x61,0x72,0x6f,0x65,0x73,0x65,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "faroese_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x66,0x69,0x6a,0x69,0x61,0x6e,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fijian__"
+ // UTF8=191 [top UTF8]
+ {{0x66,0x69,0x6e,0x6e,0x69,0x73,0x68,0x5f, 0x03,0x96,0xb7,0xa9,0x11,0x9c,0x71,0x42,0x22,0x8b,0x39,0x00,}}, // "finnish_"
+ // ASCII-7-bit=150 Latin1=183 UTF8=169 CP1252=156 CP1250=66 ISO-8859-15=139 CP1257=57 [top Latin1]
+ {{0x66,0x72,0x65,0x6e,0x63,0x68,0x5f,0x5f, 0x03,0x99,0xb6,0xaa,0x11,0xa0,0x62,0x4f,0x46,0x21,0x86,0x00,}}, // "french__"
+ // ASCII-7-bit=153 Latin1=182 UTF8=170 CP1252=160 CP1256=79 CP1250=70 ISO-8859-15=134 [top Latin1]
+ {{0x66,0x72,0x69,0x73,0x69,0x61,0x6e,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frisian_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x67,0x61,0x6c,0x69,0x63,0x69,0x61,0x6e, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "galician"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x67,0x61,0x6e,0x64,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ganda___"
+ // UTF8=191 [top UTF8]
+ {{0x67,0x65,0x6f,0x72,0x67,0x69,0x61,0x6e, 0x01,0x6c,0x11,0xbe,0x11,0x1c,0x10,0x21,0x1c,0x00,0x00,0x00,}}, // "georgian"
+ // ASCII-7-bit=108 UTF8=190 CP1252=28 CP1253=28 [top UTF8]
+ {{0x67,0x65,0x72,0x6d,0x61,0x6e,0x5f,0x5f, 0x03,0xa2,0xb7,0xa6,0x11,0x9b,0x41,0x56,0x21,0x5d,0x21,0x7c,}}, // "german__"
+ // ASCII-7-bit=162 Latin1=183 UTF8=166 CP1252=155 Latin2=86 CP1250=93 ISO-8859-15=124 [top Latin1]
+ {{0x67,0x72,0x65,0x65,0x6b,0x5f,0x5f,0x5f, 0x03,0x81,0x54,0xad,0x11,0x52,0xd1,0x31,0x11,0xb4,0x21,0xa6,}}, // "greek___"
+ // ASCII-7-bit=129 Latin1=84 UTF8=173 CP1252=82 KOI8R=49 Greek=180 CP1253=166 [top Greek]
+ {{0x67,0x72,0x65,0x65,0x6e,0x6c,0x61,0x6e, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "greenlan"
+ // UTF8=191 [top UTF8]
+ {{0x67,0x75,0x61,0x72,0x61,0x6e,0x69,0x5f, 0x11,0xb9,0x20,0x91,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "guarani_"
+ // Latin1=185 Latin6=175 [top Latin1]
+ {{0x67,0x75,0x6a,0x61,0x72,0x61,0x74,0x69, 0x03,0x79,0xb6,0x76,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "gujarati"
+ // ASCII-7-bit=121 Latin1=182 UTF8=118 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x68,0x61,0x69,0x74,0x69,0x61,0x6e,0x63, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "haitianc"
+ // UTF8=191 [top UTF8]
+ {{0x68,0x61,0x75,0x73,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hausa___"
+ // UTF8=191 [top UTF8]
+ {{0x68,0x65,0x62,0x72,0x65,0x77,0x5f,0x5f, 0x03,0x76,0x46,0xab,0x11,0x3b,0x51,0x32,0x61,0xb8,0x71,0x9f,}}, // "hebrew__"
+ // ASCII-7-bit=118 Latin1=70 UTF8=171 CP1252=59 CP1251=50 CP1255=184 Hebrew=159 [top CP1255]
+ {{0x68,0x69,0x6e,0x64,0x69,0x5f,0x5f,0x5f, 0x11,0xb5,0x21,0xab,0xa1,0xa1,0x10,0xf3,0x97,0x8d,0x83,0x00,}}, // "hindi___"
+ // Latin1=181 CP1252=171 ISO-8859-15=161 JAGRAN=151 BHASKAR=141 HTCHANAKYA=131 [top Latin1]
+ {{0x68,0x75,0x6e,0x67,0x61,0x72,0x69,0x61, 0x03,0x93,0x9f,0xad,0x11,0x6f,0x41,0xae,0x21,0xa9,0x21,0x40,}}, // "hungaria"
+ // ASCII-7-bit=147 Latin1=159 UTF8=173 CP1252=111 Latin2=174 CP1250=169 ISO-8859-15=64 [top Latin2]
+ {{0x69,0x63,0x65,0x6c,0x61,0x6e,0x64,0x69, 0x03,0x7f,0xb8,0x9c,0x11,0xa4,0x11,0x1d,0x51,0x2f,0x21,0x99,}}, // "icelandi"
+ // ASCII-7-bit=127 Latin1=184 UTF8=156 CP1252=164 SJS=29 CP1250=47 ISO-8859-15=153 [top Latin1]
+ {{0x69,0x6e,0x64,0x6f,0x6e,0x65,0x73,0x69, 0x03,0xb2,0xae,0x99,0x11,0xa2,0x11,0x5b,0x41,0x70,0x31,0x91,}}, // "indonesi"
+ // ASCII-7-bit=178 Latin1=174 UTF8=153 CP1252=162 SJS=91 CP1256=112 ISO-8859-15=145 [top ASCII-7-bit]
+ {{0x69,0x6e,0x74,0x65,0x72,0x6c,0x69,0x6e, 0x12,0xb0,0xb0,0x11,0xa6,0xa1,0x9c,0x00,0x00,0x00,0x00,0x00,}}, // "interlin"
+ // Latin1=176 UTF8=176 CP1252=166 ISO-8859-15=156 [top Latin1]
+ {{0x69,0x6e,0x75,0x6b,0x74,0x69,0x74,0x75, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "inuktitu"
+ // UTF8=191 [top UTF8]
+ {{0x69,0x6e,0x75,0x70,0x69,0x61,0x6b,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "inupiak_"
+ // UTF8=191 [top UTF8]
+ {{0x69,0x72,0x69,0x73,0x68,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "irish___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x69,0x74,0x61,0x6c,0x69,0x61,0x6e,0x5f, 0x03,0xa7,0xb4,0xa4,0x11,0xa4,0x41,0x4d,0x21,0x55,0x21,0x78,}}, // "italian_"
+ // ASCII-7-bit=167 Latin1=180 UTF8=164 CP1252=164 Latin2=77 CP1250=85 ISO-8859-15=120 [top Latin1]
+ {{0x6a,0x61,0x70,0x61,0x6e,0x65,0x73,0x65, 0x01,0x68,0x11,0xa7,0x32,0xb4,0xad,0xd1,0x78,0x21,0x62,0x00,}}, // "japanese"
+ // ASCII-7-bit=104 UTF8=167 SJS=180 EUC-JP=173 JIS=120 CP932=98 [top SJS]
+ {{0x6a,0x61,0x76,0x61,0x6e,0x65,0x73,0x65, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "javanese"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6b,0x61,0x6e,0x6e,0x61,0x64,0x61,0x5f, 0x03,0x65,0xb6,0x81,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "kannada_"
+ // ASCII-7-bit=101 Latin1=182 UTF8=129 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6b,0x61,0x73,0x68,0x6d,0x69,0x72,0x69, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kashmiri"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x61,0x7a,0x61,0x6b,0x68,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kazakh__"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x68,0x61,0x73,0x69,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "khasi___"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x68,0x6d,0x65,0x72,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "khmer___"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x69,0x6e,0x79,0x61,0x72,0x77,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kinyarwa"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x6f,0x72,0x65,0x61,0x6e,0x5f,0x5f, 0x06,0x5d,0x34,0x9d,0x20,0x1a,0xbd,0x11,0x0c,0x20,0x21,0x76,}}, // "korean__"
+ // ASCII-7-bit=93 Latin1=52 UTF8=157 GB=32 CP1252=26 KSC=189 EUC-JP=12 ISO-2022-KR=118 [top KSC]
+ {{0x6b,0x75,0x72,0x64,0x69,0x73,0x68,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kurdish_"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x6b,0x79,0x72,0x67,0x79,0x7a,0x5f,0x5f, 0x10,0x61,0xaf,0x41,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kyrgyz__"
+ // CP1254=175 ISO-8859-5=185 [top ISO-8859-5]
+ {{0x6c,0x61,0x6f,0x74,0x68,0x69,0x61,0x6e, 0x01,0x40,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "laothian"
+ // ASCII-7-bit=64 UTF8=190 [top UTF8]
+ {{0x6c,0x61,0x74,0x69,0x6e,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "latin___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6c,0x61,0x74,0x76,0x69,0x61,0x6e,0x5f, 0x03,0x80,0x55,0xac,0x11,0x64,0xb1,0xb4,0xc2,0x99,0xa3,0x00,}}, // "latvian_"
+ // ASCII-7-bit=128 Latin1=85 UTF8=172 CP1252=100 CP1257=180 ISO-8859-13=153 Latin4=163 [top CP1257]
+ {{0x6c,0x69,0x6d,0x62,0x75,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "limbu___"
+ // UTF8=191 [top UTF8]
+ {{0x6c,0x69,0x6e,0x67,0x61,0x6c,0x61,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lingala_"
+ // UTF8=191 [top UTF8]
+ {{0x6c,0x69,0x74,0x68,0x75,0x61,0x6e,0x69, 0x03,0x7c,0x5d,0xaa,0x11,0x73,0xb1,0xb7,0xc2,0x94,0x9d,0x00,}}, // "lithuani"
+ // ASCII-7-bit=124 Latin1=93 UTF8=170 CP1252=115 CP1257=183 ISO-8859-13=148 Latin4=157 [top CP1257]
+ {{0x6c,0x75,0x78,0x65,0x6d,0x62,0x6f,0x75, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "luxembou"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x63,0x65,0x64,0x6f,0x6e,0x69, 0x03,0x7a,0x54,0xa9,0x11,0x4b,0x51,0xb3,0x71,0x9e,0x81,0xa8,}}, // "macedoni"
+ // ASCII-7-bit=122 Latin1=84 UTF8=169 CP1252=75 CP1251=179 KOI8R=158 ISO-8859-5=168 [top CP1251]
+ {{0x6d,0x61,0x6c,0x61,0x67,0x61,0x73,0x79, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "malagasy"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x61,0x6c,0x61,0x79,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "malay___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x6c,0x61,0x79,0x61,0x6c,0x61, 0x03,0x48,0xb6,0x81,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "malayala"
+ // ASCII-7-bit=72 Latin1=182 UTF8=129 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x6c,0x74,0x65,0x73,0x65,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "maltese_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x6e,0x78,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "manx____"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x61,0x6f,0x72,0x69,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "maori___"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x61,0x72,0x61,0x74,0x68,0x69,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "marathi_"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x6f,0x6c,0x64,0x61,0x76,0x69,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "moldavia"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x6f,0x6e,0x67,0x6f,0x6c,0x69,0x61, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mongolia"
+ // CP1251=191 [top CP1251]
+ {{0x6e,0x61,0x75,0x72,0x75,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nauru___"
+ // UTF8=191 [top UTF8]
+ {{0x6e,0x65,0x70,0x61,0x6c,0x69,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nepali__"
+ // UTF8=191 [top UTF8]
+ {{0x6e,0x6f,0x72,0x77,0x65,0x67,0x69,0x61, 0x03,0x92,0xb8,0xa8,0x11,0x9c,0x41,0x30,0x31,0x24,0x11,0x8e,}}, // "norwegia"
+ // ASCII-7-bit=146 Latin1=184 UTF8=168 CP1252=156 Latin2=48 Latin5=36 ISO-8859-15=142 [top Latin1]
+ {{0x6f,0x63,0x63,0x69,0x74,0x61,0x6e,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "occitan_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6f,0x72,0x69,0x79,0x61,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "oriya___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6f,0x72,0x6f,0x6d,0x6f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "oromo___"
+ // UTF8=191 [top UTF8]
+ {{0x70,0x61,0x73,0x68,0x74,0x6f,0x5f,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pashto__"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x70,0x65,0x72,0x73,0x69,0x61,0x6e,0x5f, 0x12,0x44,0xb6,0x11,0x33,0x62,0xae,0x19,0x10,0x51,0x9f,0x00,}}, // "persian_"
+ // Latin1=68 UTF8=182 CP1252=51 CP1256=174 CP1250=25 Arabic=159 [top UTF8]
+ {{0x70,0x6f,0x6c,0x69,0x73,0x68,0x5f,0x5f, 0x05,0x85,0x6c,0xa8,0x26,0x57,0x41,0xb9,0x21,0x99,0x31,0x23,}}, // "polish__"
+ // ASCII-7-bit=133 Latin1=108 UTF8=168 GB=38 CP1252=87 Latin2=185 CP1250=153 CP1257=35 [top Latin2]
+ {{0x70,0x6f,0x72,0x74,0x75,0x67,0x75,0x65, 0x03,0x96,0xb9,0xa6,0x11,0x9a,0x11,0x30,0x51,0x36,0x21,0x86,}}, // "portugue"
+ // ASCII-7-bit=150 Latin1=185 UTF8=166 CP1252=154 SJS=48 CP1250=54 ISO-8859-15=134 [top Latin1]
+ {{0x70,0x75,0x6e,0x6a,0x61,0x62,0x69,0x5f, 0x03,0x42,0xb6,0x7b,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "punjabi_"
+ // ASCII-7-bit=66 Latin1=182 UTF8=123 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x71,0x75,0x65,0x63,0x68,0x75,0x61,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "quechua_"
+ // UTF8=191 [top UTF8]
+ {{0x72,0x68,0x61,0x65,0x74,0x6f,0x72,0x6f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rhaetoro"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x72,0x6f,0x6d,0x61,0x6e,0x69,0x61,0x6e, 0x03,0xb2,0x9d,0xa5,0x11,0x92,0x42,0xa7,0x51,0x11,0x99,0x00,}}, // "romanian"
+ // ASCII-7-bit=178 Latin1=157 UTF8=165 CP1252=146 Latin2=167 CP1251=81 CP1250=153 [top ASCII-7-bit]
+ {{0x72,0x75,0x6e,0x64,0x69,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rundi___"
+ // UTF8=191 [top UTF8]
+ {{0x72,0x75,0x73,0x73,0x69,0x61,0x6e,0x5f, 0x01,0x74,0x11,0xa9,0x71,0xb9,0x71,0x99,0x81,0x82,0x81,0x6d,}}, // "russian_"
+ // ASCII-7-bit=116 UTF8=169 CP1251=185 KOI8R=153 ISO-8859-5=130 CP866=109 [top CP1251]
+ {{0x73,0x61,0x6d,0x6f,0x61,0x6e,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "samoan__"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x61,0x6e,0x67,0x6f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sango___"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x61,0x6e,0x73,0x6b,0x72,0x69,0x74, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sanskrit"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x63,0x6f,0x74,0x73,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "scots___"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x63,0x6f,0x74,0x73,0x67,0x61,0x65, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "scotsgae"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x65,0x72,0x62,0x69,0x61,0x6e,0x5f, 0x03,0x93,0x77,0xad,0x11,0x85,0x42,0xad,0x52,0x12,0xae,0x4a,}}, // "serbian_"
+ // ASCII-7-bit=147 Latin1=119 UTF8=173 CP1252=133 Latin2=173 CP1251=82 CP1250=174 Latin5=74 [top CP1250]
+ {{0x73,0x65,0x72,0x62,0x6f,0x63,0x72,0x6f, 0x91,0xaf,0x21,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "serbocro"
+ // Latin2=175 CP1250=185 [top CP1250]
+ {{0x73,0x65,0x73,0x6f,0x74,0x68,0x6f,0x5f, 0x11,0xb9,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sesotho_"
+ // Latin1=185 CP1252=175 [top Latin1]
+ {{0x73,0x68,0x6f,0x6e,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "shona___"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x69,0x6e,0x64,0x68,0x69,0x5f,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sindhi__"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x73,0x69,0x6e,0x68,0x61,0x6c,0x65,0x73, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sinhales"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x69,0x73,0x77,0x61,0x6e,0x74,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "siswant_"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x6c,0x6f,0x76,0x61,0x6b,0x5f,0x5f, 0x03,0x88,0x6e,0xaf,0x11,0x67,0x41,0xa5,0x21,0xb3,0x00,0x00,}}, // "slovak__"
+ // ASCII-7-bit=136 Latin1=110 UTF8=175 CP1252=103 Latin2=165 CP1250=179 [top CP1250]
+ {{0x73,0x6c,0x6f,0x76,0x65,0x6e,0x69,0x61, 0x03,0x8e,0x71,0xb2,0x11,0x80,0x42,0xaa,0x39,0x11,0xad,0x00,}}, // "slovenia"
+ // ASCII-7-bit=142 Latin1=113 UTF8=178 CP1252=128 Latin2=170 CP1251=57 CP1250=173 [top UTF8]
+ {{0x73,0x6f,0x6d,0x61,0x6c,0x69,0x5f,0x5f, 0x11,0xb9,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "somali__"
+ // Latin1=185 CP1252=175 [top Latin1]
+ {{0x73,0x70,0x61,0x6e,0x69,0x73,0x68,0x5f, 0x03,0x9b,0xb8,0xa7,0x11,0x98,0x41,0x45,0x21,0x45,0x21,0x77,}}, // "spanish_"
+ // ASCII-7-bit=155 Latin1=184 UTF8=167 CP1252=152 Latin2=69 CP1250=69 ISO-8859-15=119 [top Latin1]
+ {{0x73,0x75,0x6e,0x64,0x61,0x6e,0x65,0x73, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sundanes"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x77,0x61,0x68,0x69,0x6c,0x69,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "swahili_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x77,0x65,0x64,0x69,0x73,0x68,0x5f, 0x03,0x90,0xba,0xa4,0x11,0x8d,0x41,0x2c,0x21,0x2c,0x21,0x7a,}}, // "swedish_"
+ // ASCII-7-bit=144 Latin1=186 UTF8=164 CP1252=141 Latin2=44 CP1250=44 ISO-8859-15=122 [top Latin1]
+ {{0x73,0x79,0x72,0x69,0x61,0x63,0x5f,0x5f, 0x01,0x6a,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "syriac__"
+ // ASCII-7-bit=106 UTF8=190 [top UTF8]
+ {{0x74,0x61,0x67,0x61,0x6c,0x6f,0x67,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tagalog_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x74,0x61,0x6a,0x69,0x6b,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tajik___"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x61,0x6d,0x69,0x6c,0x5f,0x5f,0x5f, 0x12,0xb4,0x8e,0x11,0xaa,0xa1,0xa0,0x20,0x23,0x96,0x8c,0x82,}}, // "tamil___"
+ // Latin1=180 UTF8=142 CP1252=170 ISO-8859-15=160 TSCII=150 TAM=140 TAB=130 [top Latin1]
+ {{0x74,0x61,0x74,0x61,0x72,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tatar___"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x65,0x6c,0x75,0x67,0x75,0x5f,0x5f, 0x03,0x66,0xb6,0x90,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "telugu__"
+ // ASCII-7-bit=102 Latin1=182 UTF8=144 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x74,0x68,0x61,0x69,0x5f,0x5f,0x5f,0x5f, 0x05,0x7a,0x53,0xa2,0x24,0x46,0x91,0xba,0xd1,0x9e,0x21,0x29,}}, // "thai____"
+ // ASCII-7-bit=122 Latin1=83 UTF8=162 GB=36 CP1252=70 ISO-8859-11=186 CP874=158 MACINTOSH=41 [top ISO-8859-11]
+ {{0x74,0x69,0x62,0x65,0x74,0x61,0x6e,0x5f, 0x01,0x42,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tibetan_"
+ // ASCII-7-bit=66 UTF8=190 [top UTF8]
+ {{0x74,0x69,0x67,0x72,0x69,0x6e,0x79,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tigrinya"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x6f,0x6e,0x67,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tonga___"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x73,0x6f,0x6e,0x67,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tsonga__"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x73,0x77,0x61,0x6e,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tswana__"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x75,0x72,0x6b,0x69,0x73,0x68,0x5f, 0x03,0x81,0x7f,0xa5,0x11,0x6e,0x81,0xba,0x11,0x3d,0x61,0x95,}}, // "turkish_"
+ // ASCII-7-bit=129 Latin1=127 UTF8=165 CP1252=110 Latin5=186 ISO-8859-15=61 CP1254=149 [top Latin5]
+ {{0x74,0x75,0x72,0x6b,0x6d,0x65,0x6e,0x5f, 0x91,0xb9,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "turkmen_"
+ // Latin2=185 CP1250=175 [top Latin2]
+ {{0x74,0x77,0x69,0x5f,0x5f,0x5f,0x5f,0x5f, 0x11,0xac,0x21,0xb6,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "twi_____"
+ // Latin1=172 CP1252=182 ISO-8859-15=162 [top CP1252]
+ {{0x75,0x69,0x67,0x68,0x75,0x72,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uighur__"
+ // UTF8=191 [top UTF8]
+ {{0x75,0x6b,0x72,0x61,0x69,0x6e,0x69,0x61, 0x21,0xa0,0x71,0xb7,0x71,0x91,0x72,0x98,0xa2,0x81,0x84,0x00,}}, // "ukrainia"
+ // UTF8=160 CP1251=183 KOI8R=145 KOI8U=152 ISO-8859-5=162 CP866=132 [top CP1251]
+ {{0x75,0x72,0x64,0x75,0x5f,0x5f,0x5f,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "urdu____"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x75,0x7a,0x62,0x65,0x6b,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uzbek___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x76,0x69,0x65,0x74,0x6e,0x61,0x6d,0x65, 0x03,0x81,0xa8,0xb7,0x11,0x9e,0xa1,0x94,0x00,0x00,0x00,0x00,}}, // "vietname"
+ // ASCII-7-bit=129 Latin1=168 UTF8=183 CP1252=158 ISO-8859-15=148 [top UTF8]
+ {{0x76,0x6f,0x6c,0x61,0x70,0x75,0x6b,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "volapuk_"
+ // UTF8=191 [top UTF8]
+ {{0x77,0x65,0x6c,0x73,0x68,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "welsh___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x77,0x6f,0x6c,0x6f,0x66,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wolof___"
+ // UTF8=191 [top UTF8]
+ {{0x78,0x68,0x6f,0x73,0x61,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "xhosa___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x79,0x69,0x64,0x64,0x69,0x73,0x68,0x5f, 0x10,0x11,0xb9,0x71,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yiddish_"
+ // CP1255=185 Hebrew=175 [top CP1255]
+ {{0x79,0x6f,0x72,0x75,0x62,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yoruba__"
+ // UTF8=191 [top UTF8]
+ {{0x7a,0x68,0x75,0x61,0x6e,0x67,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zhuang__"
+ // UTF8=191 [top UTF8]
+ {{0x7a,0x75,0x6c,0x75,0x5f,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zulu____"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+};
+
+static const int kLangHintProbsSize = 151;
+
+static const HintEntry kTLDHintProbs[] = { // MaxRange 192
+ {{0x5f,0x5f,0x5f,0x5f, 0x0f,0xa8,0xa1,0xa3,0xa0,0x8e,0x8e,0x8a,0x7e,0xa8,0x77,0x7b,0x8b,0x75,0x79,0x7e,}}, // "____"
+ // ASCII-7-bit=168 Latin1=161 UTF8=163 GB=160 CP1252=142 KSC=142 SJS=138 EUC-JP=126 BIG5=168 Latin2=119 CP1251=123 CP1256=139 CP1250=117 Latin5=121 ISO-8859-11=126 [top ASCII-7-bit]
+ {{0x61,0x63,0x5f,0x5f, 0x08,0xa0,0x9a,0xa1,0x65,0x92,0x8f,0xb1,0xa2,0x22,0x56,0x8a,0x21,0x56,0x61,0x87,}}, // "ac__"
+ // ASCII-7-bit=160 Latin1=154 UTF8=161 GB=101 CP1252=146 KSC=143 SJS=177 EUC-JP=162 CP1251=86 CP1256=138 ISO-8859-11=86 JIS=135 [top SJS]
+ {{0x61,0x64,0x5f,0x5f, 0x03,0xa6,0xb6,0x93,0x11,0xa8,0x11,0x74,0x81,0x5d,0x81,0x5d,0x00,0x00,0x00,0x00,}}, // "ad__"
+ // ASCII-7-bit=166 Latin1=182 UTF8=147 CP1252=168 SJS=116 ISO-8859-15=93 CP932=93 [top Latin1]
+ {{0x61,0x65,0x5f,0x5f, 0x05,0xa4,0x81,0xac,0x42,0x86,0x11,0x5b,0x25,0x4f,0x4a,0xb5,0x3b,0x52,0x00,0x00,}}, // "ae__"
+ // ASCII-7-bit=164 Latin1=129 UTF8=172 GB=66 CP1252=134 SJS=91 Latin2=79 CP1251=74 CP1256=181 CP1250=59 Latin5=82 [top CP1256]
+ {{0x61,0x65,0x72,0x6f, 0x03,0xaf,0xab,0xab,0x12,0x98,0x6a,0x11,0x6a,0x21,0x96,0x21,0x6a,0x00,0x00,0x00,}}, // "aero"
+ // ASCII-7-bit=175 Latin1=171 UTF8=171 CP1252=152 KSC=106 EUC-JP=106 CP1251=150 Latin5=106 [top ASCII-7-bit]
+ {{0x61,0x66,0x5f,0x5f, 0x03,0xb6,0x95,0xaf,0x11,0x8c,0x61,0x80,0x11,0x62,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "af__"
+ // ASCII-7-bit=182 Latin1=149 UTF8=175 CP1252=140 CP1256=128 Latin5=98 [top ASCII-7-bit]
+ {{0x61,0x67,0x5f,0x5f, 0x03,0xa8,0xb4,0xa2,0x11,0x9a,0x12,0x95,0x86,0x21,0x60,0x41,0x7a,0x00,0x00,0x00,}}, // "ag__"
+ // ASCII-7-bit=168 Latin1=180 UTF8=162 CP1252=154 SJS=149 EUC-JP=134 CP1251=96 ISO-8859-15=122 [top Latin1]
+ {{0x61,0x69,0x5f,0x5f, 0x03,0xb8,0x8f,0x9b,0x11,0x9d,0x12,0x8c,0x97,0x11,0x90,0xb1,0x67,0x00,0x00,0x00,}}, // "ai__"
+ // ASCII-7-bit=184 Latin1=143 UTF8=155 CP1252=157 SJS=140 EUC-JP=151 Latin2=144 JIS=103 [top ASCII-7-bit]
+ {{0x61,0x6c,0x5f,0x5f, 0x03,0xac,0x99,0xae,0x11,0xa1,0x31,0x57,0x41,0x57,0x21,0xa7,0x31,0x57,0x00,0x00,}}, // "al__"
+ // ASCII-7-bit=172 Latin1=153 UTF8=174 CP1252=161 BIG5=87 Latin5=87 CP1257=167 Greek=87 [top UTF8]
+ {{0x61,0x6d,0x5f,0x5f, 0x08,0xac,0x9a,0xab,0x68,0x9d,0x58,0x82,0x56,0x22,0xac,0x5a,0x00,0x00,0x00,0x00,}}, // "am__"
+ // ASCII-7-bit=172 Latin1=154 UTF8=171 GB=104 CP1252=157 KSC=88 SJS=130 EUC-JP=86 CP1251=172 CP1256=90 [top ASCII-7-bit]
+ {{0x61,0x6e,0x5f,0x5f, 0x03,0xb6,0xad,0x94,0x11,0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "an__"
+ // ASCII-7-bit=182 Latin1=173 UTF8=148 CP1252=153 [top ASCII-7-bit]
+ {{0x61,0x6f,0x5f,0x5f, 0x03,0x9f,0xb5,0xab,0x11,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ao__"
+ // ASCII-7-bit=159 Latin1=181 UTF8=171 CP1252=159 [top Latin1]
+ {{0x61,0x71,0x5f,0x5f, 0x03,0xb7,0xa9,0x9c,0x11,0x8a,0x51,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "aq__"
+ // ASCII-7-bit=183 Latin1=169 UTF8=156 CP1252=138 CP1251=151 [top ASCII-7-bit]
+ {{0x61,0x72,0x5f,0x5f, 0x03,0xa0,0xb9,0x9e,0x13,0x98,0x55,0x2c,0x13,0x28,0x26,0x27,0x11,0x2e,0x21,0x42,}}, // "ar__"
+ // ASCII-7-bit=160 Latin1=185 UTF8=158 CP1252=152 KSC=85 SJS=44 BIG5=40 Latin2=38 CP1251=39 CP1250=46 ISO-8859-15=66 [top Latin1]
+ {{0x61,0x73,0x5f,0x5f, 0x03,0xa9,0xb7,0x9f,0x11,0x94,0x11,0x52,0x22,0x64,0x52,0x12,0x7d,0x74,0x21,0x52,}}, // "as__"
+ // ASCII-7-bit=169 Latin1=183 UTF8=159 CP1252=148 SJS=82 Latin2=100 CP1251=82 CP1250=125 Latin5=116 CP1257=82 [top Latin1]
+ {{0x61,0x74,0x5f,0x5f, 0x03,0xa1,0xb8,0xa5,0x11,0x9a,0x11,0x48,0x21,0x51,0x13,0x45,0x53,0x4a,0x11,0x62,}}, // "at__"
+ // ASCII-7-bit=161 Latin1=184 UTF8=165 CP1252=154 SJS=72 Latin2=81 CP1256=69 CP1250=83 Latin5=74 ISO-8859-15=98 [top Latin1]
+ {{0x61,0x75,0x5f,0x5f, 0x09,0xb8,0xa3,0x9f,0x4e,0x9a,0x55,0x54,0x3e,0x5e,0x22,0x30,0x3d,0x21,0x36,0x00,}}, // "au__"
+ // ASCII-7-bit=184 Latin1=163 UTF8=159 GB=78 CP1252=154 KSC=85 SJS=84 EUC-JP=62 BIG5=94 CP1256=48 CP1250=61 ISO-8859-15=54 [top ASCII-7-bit]
+ {{0x61,0x77,0x5f,0x5f, 0x03,0xb6,0xa2,0xaa,0x11,0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "aw__"
+ // ASCII-7-bit=182 Latin1=162 UTF8=170 CP1252=153 [top ASCII-7-bit]
+ {{0x61,0x78,0x5f,0x5f, 0x03,0x9d,0xba,0xa2,0x11,0x90,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ax__"
+ // ASCII-7-bit=157 Latin1=186 UTF8=162 CP1252=144 [top Latin1]
+ {{0x61,0x7a,0x5f,0x5f, 0x03,0x9a,0x7d,0xb8,0x11,0x86,0x54,0xa8,0x54,0x54,0x91,0x41,0x4c,0x31,0x6c,0x00,}}, // "az__"
+ // ASCII-7-bit=154 Latin1=125 UTF8=184 CP1252=134 CP1251=168 CP1256=84 CP1250=84 Latin5=145 KOI8R=76 CP1254=108 [top UTF8]
+ {{0x62,0x61,0x5f,0x5f, 0x03,0xa0,0x7e,0xb2,0x11,0x78,0x44,0x89,0x66,0x49,0xb1,0x00,0x00,0x00,0x00,0x00,}}, // "ba__"
+ // ASCII-7-bit=160 Latin1=126 UTF8=178 CP1252=120 Latin2=137 CP1251=102 CP1256=73 CP1250=177 [top UTF8]
+ {{0x62,0x62,0x5f,0x5f, 0x03,0xba,0xa0,0x7f,0x11,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bb__"
+ // ASCII-7-bit=186 Latin1=160 UTF8=127 CP1252=160 [top ASCII-7-bit]
+ {{0x62,0x64,0x5f,0x5f, 0x03,0xbd,0x94,0x8c,0x11,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bd__"
+ // ASCII-7-bit=189 Latin1=148 UTF8=140 CP1252=138 [top ASCII-7-bit]
+ {{0x62,0x65,0x5f,0x5f, 0x03,0xb1,0xb0,0xa1,0x11,0x9d,0x11,0x5f,0x22,0x4e,0x50,0x12,0x4d,0x59,0x11,0x5f,}}, // "be__"
+ // ASCII-7-bit=177 Latin1=176 UTF8=161 CP1252=157 SJS=95 Latin2=78 CP1251=80 CP1250=77 Latin5=89 ISO-8859-15=95 [top ASCII-7-bit]
+ {{0x62,0x66,0x5f,0x5f, 0x03,0x9f,0xb9,0x63,0x11,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bf__"
+ // ASCII-7-bit=159 Latin1=185 UTF8=99 CP1252=166 [top Latin1]
+ {{0x62,0x67,0x5f,0x5f, 0x05,0x96,0x70,0xab,0x4a,0x74,0x51,0xb9,0x11,0x4f,0x51,0x44,0x31,0x45,0x41,0x54,}}, // "bg__"
+ // ASCII-7-bit=150 Latin1=112 UTF8=171 GB=74 CP1252=116 CP1251=185 CP1250=79 KOI8R=68 CP1254=69 ISO-8859-5=84 [top CP1251]
+ {{0x62,0x68,0x5f,0x5f, 0x03,0x9f,0x94,0xa5,0x11,0x84,0x11,0x53,0x41,0xb8,0x10,0x61,0x70,0x00,0x00,0x00,}}, // "bh__"
+ // ASCII-7-bit=159 Latin1=148 UTF8=165 CP1252=132 SJS=83 CP1256=184 Arabic=112 [top CP1256]
+ {{0x62,0x69,0x5f,0x5f, 0x03,0xa4,0xa5,0xb8,0x12,0x82,0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bi__"
+ // ASCII-7-bit=164 Latin1=165 UTF8=184 CP1252=130 KSC=101 [top UTF8]
+ {{0x62,0x69,0x7a,0x5f, 0x0e,0xae,0xa5,0xa1,0x77,0x96,0x7f,0x95,0x9c,0x7a,0x8e,0x8b,0x80,0x80,0x92,0x00,}}, // "biz_"
+ // ASCII-7-bit=174 Latin1=165 UTF8=161 GB=119 CP1252=150 KSC=127 SJS=149 EUC-JP=156 BIG5=122 Latin2=142 CP1251=139 CP1256=128 CP1250=128 Latin5=146 [top ASCII-7-bit]
+ {{0x62,0x6a,0x5f,0x5f, 0x03,0x9b,0xb6,0x8a,0x11,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bj__"
+ // ASCII-7-bit=155 Latin1=182 UTF8=138 CP1252=175 [top Latin1]
+ {{0x62,0x6d,0x5f,0x5f, 0x05,0xbb,0x95,0xa0,0x5a,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bm__"
+ // ASCII-7-bit=187 Latin1=149 UTF8=160 GB=90 CP1252=149 [top ASCII-7-bit]
+ {{0x62,0x6e,0x5f,0x5f, 0x05,0xb8,0x98,0xa6,0x6d,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bn__"
+ // ASCII-7-bit=184 Latin1=152 UTF8=166 GB=109 CP1252=160 [top ASCII-7-bit]
+ {{0x62,0x6f,0x5f,0x5f, 0x03,0x9a,0xba,0x9f,0x11,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bo__"
+ // ASCII-7-bit=154 Latin1=186 UTF8=159 CP1252=156 [top Latin1]
+ {{0x62,0x72,0x5f,0x5f, 0x07,0x9c,0xba,0x9c,0x1f,0x95,0x21,0x43,0x15,0x1c,0x20,0x17,0x0e,0x2b,0x21,0x5a,}}, // "br__"
+ // ASCII-7-bit=156 Latin1=186 UTF8=156 GB=31 CP1252=149 KSC=33 SJS=67 BIG5=28 Latin2=32 CP1251=23 CP1256=14 CP1250=43 ISO-8859-15=90 [top Latin1]
+ {{0x62,0x73,0x5f,0x5f, 0x03,0xb2,0xb4,0x9c,0x11,0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bs__"
+ // ASCII-7-bit=178 Latin1=180 UTF8=156 CP1252=118 [top Latin1]
+ {{0x62,0x74,0x5f,0x5f, 0x03,0xb9,0x96,0xa7,0x11,0x94,0x11,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bt__"
+ // ASCII-7-bit=185 Latin1=150 UTF8=167 CP1252=148 SJS=111 [top ASCII-7-bit]
+ {{0x62,0x77,0x5f,0x5f, 0x03,0xbb,0x9b,0x88,0x11,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bw__"
+ // ASCII-7-bit=187 Latin1=155 UTF8=136 CP1252=157 [top ASCII-7-bit]
+ {{0x62,0x79,0x5f,0x5f, 0x03,0x8a,0x7b,0xa4,0x11,0x74,0x42,0x5d,0xb6,0x11,0x4b,0x21,0x5f,0x21,0xa9,0x00,}}, // "by__"
+ // ASCII-7-bit=138 Latin1=123 UTF8=164 CP1252=116 Latin2=93 CP1251=182 CP1250=75 ISO-8859-15=95 KOI8R=169 [top CP1251]
+ {{0x62,0x7a,0x5f,0x5f, 0x03,0xaf,0x9f,0xa1,0x19,0x90,0x89,0xa4,0x9e,0x8c,0x65,0x8d,0x64,0x70,0x51,0x7e,}}, // "bz__"
+ // ASCII-7-bit=175 Latin1=159 UTF8=161 CP1252=144 KSC=137 SJS=164 EUC-JP=158 BIG5=140 Latin2=101 CP1251=141 CP1256=100 CP1250=112 KOI8R=126 [top ASCII-7-bit]
+ {{0x63,0x61,0x5f,0x5f, 0x07,0xb3,0xac,0xa0,0x5b,0x9b,0x5f,0x49,0x15,0x56,0x3c,0x5d,0x48,0x42,0x21,0x94,}}, // "ca__"
+ // ASCII-7-bit=179 Latin1=172 UTF8=160 GB=91 CP1252=155 KSC=95 SJS=73 BIG5=86 Latin2=60 CP1251=93 CP1256=72 CP1250=66 ISO-8859-15=148 [top ASCII-7-bit]
+ {{0x63,0x61,0x74,0x5f, 0x03,0x9a,0xb4,0xad,0x11,0x9f,0x11,0x30,0x31,0x30,0x32,0x30,0x6e,0x00,0x00,0x00,}}, // "cat_"
+ // ASCII-7-bit=154 Latin1=180 UTF8=173 CP1252=159 SJS=48 CP1251=48 ISO-8859-11=48 ISO-8859-15=110 [top Latin1]
+ {{0x63,0x63,0x5f,0x5f, 0x09,0x9d,0xab,0xad,0x9b,0x86,0x80,0x90,0x9e,0x92,0x21,0x8a,0x11,0x7a,0x51,0x75,}}, // "cc__"
+ // ASCII-7-bit=157 Latin1=171 UTF8=173 GB=155 CP1252=134 KSC=128 SJS=144 EUC-JP=158 BIG5=146 CP1256=138 Latin5=122 GBK=117 [top UTF8]
+ {{0x63,0x64,0x5f,0x5f, 0x09,0xae,0xa2,0xb2,0x5a,0x95,0x5a,0x8f,0x64,0x5a,0x11,0x7d,0x11,0x74,0x11,0x5a,}}, // "cd__"
+ // ASCII-7-bit=174 Latin1=162 UTF8=178 GB=90 CP1252=149 KSC=90 SJS=143 EUC-JP=100 BIG5=90 CP1251=125 CP1250=116 ISO-8859-11=90 [top UTF8]
+ {{0x63,0x67,0x5f,0x5f, 0x03,0x83,0x8d,0xbe,0x11,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cg__"
+ // ASCII-7-bit=131 Latin1=141 UTF8=190 CP1252=131 [top UTF8]
+ {{0x63,0x68,0x5f,0x5f, 0x05,0xaa,0xb6,0xa1,0x4c,0x9a,0x11,0x46,0x25,0x49,0x3e,0x41,0x44,0x43,0x11,0x66,}}, // "ch__"
+ // ASCII-7-bit=170 Latin1=182 UTF8=161 GB=76 CP1252=154 SJS=70 Latin2=73 CP1251=62 CP1256=65 CP1250=68 Latin5=67 ISO-8859-15=102 [top Latin1]
+ {{0x63,0x69,0x5f,0x5f, 0x03,0x9c,0xae,0xb3,0x11,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ci__"
+ // ASCII-7-bit=156 Latin1=174 UTF8=179 CP1252=161 [top UTF8]
+ {{0x63,0x6b,0x5f,0x5f, 0x03,0xba,0x9c,0x9e,0x11,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ck__"
+ // ASCII-7-bit=186 Latin1=156 UTF8=158 CP1252=154 [top ASCII-7-bit]
+ {{0x63,0x6c,0x5f,0x5f, 0x03,0xa4,0xb9,0x9c,0x11,0x97,0x11,0x3e,0x27,0x1b,0x1b,0x2d,0x34,0x2b,0x21,0x3b,}}, // "cl__"
+ // ASCII-7-bit=164 Latin1=185 UTF8=156 CP1252=151 SJS=62 Latin2=27 CP1251=27 CP1256=45 CP1250=52 Latin5=43 ISO-8859-11=33 ISO-8859-15=59 [top Latin1]
+ {{0x63,0x6d,0x5f,0x5f, 0x03,0x93,0xbd,0x64,0x11,0x97,0xa1,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cm__"
+ // ASCII-7-bit=147 Latin1=189 UTF8=100 CP1252=151 ISO-8859-15=108 [top Latin1]
+ {{0x63,0x6e,0x5f,0x5f, 0x09,0x8c,0x5c,0xa7,0xba,0x4f,0x48,0x57,0x3c,0x8d,0x12,0x4e,0x4f,0x71,0x64,0x00,}}, // "cn__"
+ // ASCII-7-bit=140 Latin1=92 UTF8=167 GB=186 CP1252=79 KSC=72 SJS=87 EUC-JP=60 BIG5=141 CP1251=78 CP1256=79 GBK=100 [top GB]
+ {{0x63,0x6f,0x5f,0x5f, 0x03,0xa8,0xb7,0xa3,0x12,0x91,0x27,0x31,0x3f,0x13,0x2f,0x2a,0x2a,0x61,0x27,0x00,}}, // "co__"
+ // ASCII-7-bit=168 Latin1=183 UTF8=163 CP1252=145 KSC=39 Latin2=63 CP1256=47 CP1250=42 Latin5=42 Greek=39 [top Latin1]
+ {{0x63,0x6f,0x6d,0x5f, 0x09,0xb2,0xa5,0xa7,0x94,0x94,0x87,0x87,0x7d,0x82,0x12,0x6e,0x89,0x12,0x7f,0x70,}}, // "com_"
+ // ASCII-7-bit=178 Latin1=165 UTF8=167 GB=148 CP1252=148 KSC=135 SJS=135 EUC-JP=125 BIG5=130 CP1251=110 CP1256=137 Latin5=127 ISO-8859-11=112 [top ASCII-7-bit]
+ {{0x63,0x6f,0x6f,0x70, 0x03,0xaf,0xa8,0xa0,0x14,0x9c,0x75,0xa7,0x86,0x71,0x78,0x00,0x00,0x00,0x00,0x00,}}, // "coop"
+ // ASCII-7-bit=175 Latin1=168 UTF8=160 CP1252=156 KSC=117 SJS=167 EUC-JP=134 ISO-8859-15=120 [top ASCII-7-bit]
+ {{0x63,0x72,0x5f,0x5f, 0x03,0x99,0xb7,0xad,0x11,0x84,0x81,0x28,0x11,0x28,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cr__"
+ // ASCII-7-bit=153 Latin1=183 UTF8=173 CP1252=132 Latin5=40 ISO-8859-15=40 [top Latin1]
+ {{0x63,0x75,0x5f,0x5f, 0x03,0xa0,0xb7,0x9f,0x11,0xa6,0x53,0x31,0x45,0x45,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cu__"
+ // ASCII-7-bit=160 Latin1=183 UTF8=159 CP1252=166 CP1251=49 CP1256=69 CP1250=69 [top Latin1]
+ {{0x63,0x76,0x5f,0x5f, 0x03,0x90,0xbc,0x8f,0x11,0x98,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cv__"
+ // ASCII-7-bit=144 Latin1=188 UTF8=143 CP1252=152 [top Latin1]
+ {{0x63,0x78,0x5f,0x5f, 0x03,0xae,0xa2,0xa5,0x15,0x87,0x76,0x9f,0x9a,0x83,0x41,0x85,0x11,0x7e,0x51,0x99,}}, // "cx__"
+ // ASCII-7-bit=174 Latin1=162 UTF8=165 CP1252=135 KSC=118 SJS=159 EUC-JP=154 BIG5=131 Latin5=133 ISO-8859-15=126 JIS=153 [top ASCII-7-bit]
+ {{0x63,0x79,0x5f,0x5f, 0x03,0xaa,0x88,0xac,0x11,0x86,0x51,0x63,0x21,0x5c,0x61,0x9e,0x12,0x52,0xae,0x00,}}, // "cy__"
+ // ASCII-7-bit=170 Latin1=136 UTF8=172 CP1252=134 CP1251=99 Latin5=92 Greek=158 CP1254=82 CP1253=174 [top CP1253]
+ {{0x63,0x7a,0x5f,0x5f, 0x03,0x8f,0x74,0xb2,0x11,0x56,0x42,0x8c,0x4a,0x11,0xb5,0x10,0x21,0x3f,0x11,0x41,}}, // "cz__"
+ // ASCII-7-bit=143 Latin1=116 UTF8=178 CP1252=86 Latin2=140 CP1251=74 CP1250=181 MACINTOSH=63 CP852=65 [top CP1250]
+ {{0x64,0x65,0x5f,0x5f, 0x06,0xa4,0xb7,0xa4,0x40,0x9a,0x36,0x35,0x4b,0x4d,0x43,0x4d,0x4f,0x11,0x79,0x00,}}, // "de__"
+ // ASCII-7-bit=164 Latin1=183 UTF8=164 GB=64 CP1252=154 KSC=54 Latin2=75 CP1251=77 CP1256=67 CP1250=77 Latin5=79 ISO-8859-15=121 [top Latin1]
+ {{0x64,0x6a,0x5f,0x5f, 0x08,0xa3,0xad,0xa9,0x90,0xa2,0x7d,0x7a,0x68,0x21,0xa0,0x11,0x5e,0xb1,0x5e,0x00,}}, // "dj__"
+ // ASCII-7-bit=163 Latin1=173 UTF8=169 GB=144 CP1252=162 KSC=125 SJS=122 EUC-JP=104 CP1251=160 CP1250=94 CP932=94 [top Latin1]
+ {{0x64,0x6b,0x5f,0x5f, 0x03,0x9d,0xb8,0xa7,0x11,0x93,0x11,0x39,0x25,0x38,0x34,0x57,0x43,0x3d,0x11,0x54,}}, // "dk__"
+ // ASCII-7-bit=157 Latin1=184 UTF8=167 CP1252=147 SJS=57 Latin2=56 CP1251=52 CP1256=87 CP1250=67 Latin5=61 ISO-8859-15=84 [top Latin1]
+ {{0x64,0x6d,0x5f,0x5f, 0x03,0xbc,0x76,0xa3,0x11,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dm__"
+ // ASCII-7-bit=188 Latin1=118 UTF8=163 CP1252=131 [top ASCII-7-bit]
+ {{0x64,0x6f,0x5f,0x5f, 0x05,0xa4,0xb6,0xa9,0x6b,0x93,0x31,0x43,0x61,0x57,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "do__"
+ // ASCII-7-bit=164 Latin1=182 UTF8=169 GB=107 CP1252=147 BIG5=67 ISO-8859-15=87 [top Latin1]
+ {{0x64,0x7a,0x5f,0x5f, 0x03,0x9e,0xb6,0x8d,0x11,0xa1,0x62,0xa6,0x4e,0x10,0x51,0x58,0x00,0x00,0x00,0x00,}}, // "dz__"
+ // ASCII-7-bit=158 Latin1=182 UTF8=141 CP1252=161 CP1256=166 CP1250=78 Arabic=88 [top Latin1]
+ {{0x65,0x63,0x5f,0x5f, 0x03,0xa2,0xba,0x9c,0x11,0x96,0x35,0x3c,0x32,0x5a,0x32,0x3c,0x00,0x00,0x00,0x00,}}, // "ec__"
+ // ASCII-7-bit=162 Latin1=186 UTF8=156 CP1252=150 BIG5=60 Latin2=50 CP1251=90 CP1256=50 CP1250=60 [top Latin1]
+ {{0x65,0x64,0x75,0x5f, 0x07,0xbb,0x97,0x99,0x51,0x94,0x6b,0x49,0x11,0x4e,0x21,0x4f,0x13,0x4c,0x41,0x44,}}, // "edu_"
+ // ASCII-7-bit=187 Latin1=151 UTF8=153 GB=81 CP1252=148 KSC=107 SJS=73 BIG5=78 CP1256=79 Latin5=76 ISO-8859-11=65 ISO-8859-15=68 [top ASCII-7-bit]
+ {{0x65,0x65,0x5f,0x5f, 0x03,0x97,0xaf,0xb4,0x11,0x95,0x42,0x6f,0x78,0x42,0x82,0x87,0xc2,0x5e,0x65,0x00,}}, // "ee__"
+ // ASCII-7-bit=151 Latin1=175 UTF8=180 CP1252=149 Latin2=111 CP1251=120 ISO-8859-15=130 CP1257=135 ISO-8859-13=94 Latin4=101 [top UTF8]
+ {{0x65,0x67,0x5f,0x5f, 0x05,0x9f,0x7a,0xa7,0x55,0x7d,0x61,0xb9,0x32,0x28,0x28,0x61,0x28,0x00,0x00,0x00,}}, // "eg__"
+ // ASCII-7-bit=159 Latin1=122 UTF8=167 GB=85 CP1252=125 CP1256=185 ISO-8859-15=40 CP1257=40 CP1253=40 [top CP1256]
+ {{0x65,0x73,0x5f,0x5f, 0x05,0x9f,0xb8,0xa8,0x22,0x91,0x11,0x2b,0x15,0x18,0x33,0x4d,0x31,0x23,0x21,0x6c,}}, // "es__"
+ // ASCII-7-bit=159 Latin1=184 UTF8=168 GB=34 CP1252=145 SJS=43 BIG5=24 Latin2=51 CP1251=77 CP1256=49 CP1250=35 ISO-8859-15=108 [top Latin1]
+ {{0x65,0x74,0x5f,0x5f, 0x03,0xb5,0x9a,0xa8,0x11,0xa6,0x11,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "et__"
+ // ASCII-7-bit=181 Latin1=154 UTF8=168 CP1252=166 SJS=126 [top ASCII-7-bit]
+ {{0x65,0x75,0x5f,0x5f, 0x03,0xa8,0xa4,0xb5,0x11,0x91,0x42,0x8d,0x69,0x12,0x8b,0x6a,0x11,0x78,0x41,0x78,}}, // "eu__"
+ // ASCII-7-bit=168 Latin1=164 UTF8=181 CP1252=145 Latin2=141 CP1251=105 CP1250=139 Latin5=106 ISO-8859-15=120 Greek=120 [top UTF8]
+ {{0x66,0x69,0x5f,0x5f, 0x03,0x9e,0xb7,0xaa,0x11,0x96,0x51,0x46,0x11,0x31,0x22,0x69,0x31,0x00,0x00,0x00,}}, // "fi__"
+ // ASCII-7-bit=158 Latin1=183 UTF8=170 CP1252=150 CP1251=70 CP1250=49 ISO-8859-15=105 CP1257=49 [top Latin1]
+ {{0x66,0x6a,0x5f,0x5f, 0x05,0xba,0x8b,0x9f,0x59,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fj__"
+ // ASCII-7-bit=186 Latin1=139 UTF8=159 GB=89 CP1252=160 [top ASCII-7-bit]
+ {{0x66,0x6b,0x5f,0x5f, 0x02,0xba,0xa6,0x21,0x96,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fk__"
+ // ASCII-7-bit=186 Latin1=166 CP1252=150 [top ASCII-7-bit]
+ {{0x66,0x6d,0x5f,0x5f, 0x03,0x91,0x92,0xbc,0x11,0x82,0x12,0x7d,0x6d,0x12,0x86,0x6a,0x51,0x67,0x00,0x00,}}, // "fm__"
+ // ASCII-7-bit=145 Latin1=146 UTF8=188 CP1252=130 SJS=125 EUC-JP=109 Latin2=134 CP1251=106 CP1257=103 [top UTF8]
+ {{0x66,0x6f,0x5f,0x5f, 0x03,0x93,0xbc,0x9c,0x11,0x8c,0x71,0x57,0x21,0x51,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fo__"
+ // ASCII-7-bit=147 Latin1=188 UTF8=156 CP1252=140 CP1250=87 ISO-8859-15=81 [top Latin1]
+ {{0x66,0x72,0x5f,0x5f, 0x03,0x9f,0xb3,0xaf,0x11,0x99,0x17,0x37,0x32,0x32,0x35,0x41,0x4f,0x35,0x21,0x77,}}, // "fr__"
+ // ASCII-7-bit=159 Latin1=179 UTF8=175 CP1252=153 SJS=55 EUC-JP=50 BIG5=50 Latin2=53 CP1251=65 CP1256=79 CP1250=53 ISO-8859-15=119 [top Latin1]
+ {{0x67,0x61,0x5f,0x5f, 0x03,0xa8,0xb6,0xa5,0x11,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ga__"
+ // ASCII-7-bit=168 Latin1=182 UTF8=165 CP1252=149 [top Latin1]
+ {{0x67,0x64,0x5f,0x5f, 0x05,0xb5,0xac,0x9a,0x80,0x8a,0x81,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gd__"
+ // ASCII-7-bit=181 Latin1=172 UTF8=154 GB=128 CP1252=138 Latin5=151 [top ASCII-7-bit]
+ {{0x67,0x65,0x5f,0x5f, 0x03,0xa5,0x7d,0xba,0x11,0x8d,0x21,0x58,0x21,0x8b,0x11,0x71,0x21,0x4f,0xb1,0x4c,}}, // "ge__"
+ // ASCII-7-bit=165 Latin1=125 UTF8=186 CP1252=141 EUC-JP=88 CP1251=139 CP1250=113 ISO-8859-15=79 ISO-8859-5=76 [top UTF8]
+ {{0x67,0x67,0x5f,0x5f, 0x03,0xad,0xa9,0xb0,0x11,0x95,0x11,0x60,0x81,0x93,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gg__"
+ // ASCII-7-bit=173 Latin1=169 UTF8=176 CP1252=149 SJS=96 ISO-8859-15=147 [top UTF8]
+ {{0x67,0x68,0x5f,0x5f, 0x03,0xac,0xb3,0x99,0x11,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gh__"
+ // ASCII-7-bit=172 Latin1=179 UTF8=153 CP1252=166 [top Latin1]
+ {{0x67,0x69,0x5f,0x5f, 0x03,0xb3,0xa1,0xa1,0x11,0x9c,0xa1,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gi__"
+ // ASCII-7-bit=179 Latin1=161 UTF8=161 CP1252=156 ISO-8859-15=168 [top ASCII-7-bit]
+ {{0x67,0x6c,0x5f,0x5f, 0x03,0xa7,0xb2,0xaa,0x11,0xa1,0x11,0x43,0x11,0x4d,0x61,0x70,0x00,0x00,0x00,0x00,}}, // "gl__"
+ // ASCII-7-bit=167 Latin1=178 UTF8=170 CP1252=161 SJS=67 BIG5=77 ISO-8859-15=112 [top Latin1]
+ {{0x67,0x6d,0x5f,0x5f, 0x03,0xb8,0x89,0xa0,0x11,0xa7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gm__"
+ // ASCII-7-bit=184 Latin1=137 UTF8=160 CP1252=167 [top ASCII-7-bit]
+ {{0x67,0x6e,0x5f,0x5f, 0x03,0xaa,0xb8,0x89,0x11,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gn__"
+ // ASCII-7-bit=170 Latin1=184 UTF8=137 CP1252=157 [top Latin1]
+ {{0x67,0x6f,0x76,0x5f, 0x05,0xbd,0x92,0x94,0x22,0x87,0x11,0x2e,0x33,0x36,0x15,0x2e,0x13,0x14,0x1f,0x0e,}}, // "gov_"
+ // ASCII-7-bit=189 Latin1=146 UTF8=148 GB=34 CP1252=135 SJS=46 CP1251=54 CP1256=21 CP1250=46 ISO-8859-11=20 ISO-8859-15=31 CP1257=14 [top ASCII-7-bit]
+ {{0x67,0x70,0x5f,0x5f, 0x03,0x98,0x9f,0xbb,0x11,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gp__"
+ // ASCII-7-bit=152 Latin1=159 UTF8=187 CP1252=131 [top UTF8]
+ {{0x67,0x72,0x5f,0x5f, 0x05,0xa1,0x7f,0xa8,0x45,0x80,0x42,0x49,0x54,0x12,0x44,0x45,0x61,0xb5,0x21,0x9e,}}, // "gr__"
+ // ASCII-7-bit=161 Latin1=127 UTF8=168 GB=69 CP1252=128 Latin2=73 CP1251=84 CP1250=68 Latin5=69 Greek=181 CP1253=158 [top Greek]
+ {{0x67,0x73,0x5f,0x5f, 0x05,0xb0,0x93,0x98,0x68,0x88,0x13,0xa2,0xa7,0x75,0x42,0x99,0x98,0x91,0x66,0x00,}}, // "gs__"
+ // ASCII-7-bit=176 Latin1=147 UTF8=152 GB=104 CP1252=136 SJS=162 EUC-JP=167 BIG5=117 Latin5=153 ISO-8859-11=152 CP932=102 [top ASCII-7-bit]
+ {{0x67,0x74,0x5f,0x5f, 0x03,0xa3,0xb6,0xa9,0x12,0x95,0x3d,0x91,0x3d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gt__"
+ // ASCII-7-bit=163 Latin1=182 UTF8=169 CP1252=149 KSC=61 ISO-8859-15=61 [top Latin1]
+ {{0x67,0x75,0x5f,0x5f, 0x03,0xb9,0xa0,0x7d,0x11,0xa5,0x11,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gu__"
+ // ASCII-7-bit=185 Latin1=160 UTF8=125 CP1252=165 SJS=125 [top ASCII-7-bit]
+ {{0x67,0x79,0x5f,0x5f, 0x03,0xbc,0x82,0xa2,0x11,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gy__"
+ // ASCII-7-bit=188 Latin1=130 UTF8=162 CP1252=135 [top ASCII-7-bit]
+ {{0x68,0x6b,0x5f,0x5f, 0x07,0xa6,0x6c,0xa3,0xa0,0x6b,0x38,0x53,0x11,0xb5,0xa1,0x73,0xc1,0x3d,0x21,0x49,}}, // "hk__"
+ // ASCII-7-bit=166 Latin1=108 UTF8=163 GB=160 CP1252=107 KSC=56 SJS=83 BIG5=181 GBK=115 GB18030=61 BIG5_HKSCS=73 [top BIG5]
+ {{0x68,0x6d,0x5f,0x5f, 0x05,0xa8,0x9e,0xaa,0x67,0x67,0x13,0xac,0x9f,0x95,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hm__"
+ // ASCII-7-bit=168 Latin1=158 UTF8=170 GB=103 CP1252=103 SJS=172 EUC-JP=159 BIG5=149 [top SJS]
+ {{0x68,0x6e,0x5f,0x5f, 0x03,0xa1,0xb7,0xa6,0x11,0x9c,0x51,0x4d,0x31,0x4d,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hn__"
+ // ASCII-7-bit=161 Latin1=183 UTF8=166 CP1252=156 CP1251=77 ISO-8859-11=77 [top Latin1]
+ {{0x68,0x72,0x5f,0x5f, 0x03,0x99,0x67,0xa7,0x11,0x67,0x42,0x9d,0x20,0x11,0xb8,0x10,0x21,0x26,0x11,0x32,}}, // "hr__"
+ // ASCII-7-bit=153 Latin1=103 UTF8=167 CP1252=103 Latin2=157 CP1251=32 CP1250=184 MACINTOSH=38 CP852=50 [top CP1250]
+ {{0x68,0x74,0x5f,0x5f, 0x03,0x95,0x97,0xbc,0x11,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ht__"
+ // ASCII-7-bit=149 Latin1=151 UTF8=188 CP1252=146 [top UTF8]
+ {{0x68,0x75,0x5f,0x5f, 0x05,0xa2,0xa4,0xad,0x33,0x76,0x11,0x3d,0x25,0xa7,0x4a,0x40,0xa7,0x35,0x11,0x3a,}}, // "hu__"
+ // ASCII-7-bit=162 Latin1=164 UTF8=173 GB=51 CP1252=118 SJS=61 Latin2=167 CP1251=74 CP1256=64 CP1250=167 Latin5=53 ISO-8859-15=58 [top UTF8]
+ {{0x69,0x64,0x5f,0x5f, 0x07,0xb8,0xab,0x8c,0x24,0x97,0x33,0x46,0x14,0x2c,0x1f,0x1f,0x7b,0x22,0x29,0x4b,}}, // "id__"
+ // ASCII-7-bit=184 Latin1=171 UTF8=140 GB=36 CP1252=151 KSC=51 SJS=70 BIG5=44 Latin2=31 CP1251=31 CP1256=123 ISO-8859-11=41 ISO-8859-15=75 [top ASCII-7-bit]
+ {{0x69,0x65,0x5f,0x5f, 0x05,0xb3,0xa6,0xa0,0x45,0x9d,0x11,0x46,0x21,0x67,0x12,0x32,0x4a,0x22,0xa3,0x36,}}, // "ie__"
+ // ASCII-7-bit=179 Latin1=166 UTF8=160 GB=69 CP1252=157 SJS=70 Latin2=103 CP1256=50 CP1250=74 ISO-8859-15=163 CP1257=54 [top ASCII-7-bit]
+ {{0x69,0x6c,0x5f,0x5f, 0x03,0x94,0x74,0xa9,0x11,0x6e,0x21,0x3b,0x23,0x79,0x83,0x3d,0x41,0xb8,0x71,0x90,}}, // "il__"
+ // ASCII-7-bit=148 Latin1=116 UTF8=169 CP1252=110 EUC-JP=59 CP1251=121 CP1256=131 CP1250=61 CP1255=184 Hebrew=144 [top CP1255]
+ {{0x69,0x6d,0x5f,0x5f, 0x05,0xb7,0x8e,0xaf,0x89,0x7e,0x51,0x48,0x21,0x71,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "im__"
+ // ASCII-7-bit=183 Latin1=142 UTF8=175 GB=137 CP1252=126 CP1251=72 Latin5=113 [top ASCII-7-bit]
+ {{0x69,0x6e,0x5f,0x5f, 0x08,0xbb,0x95,0x9c,0x5e,0x99,0x61,0x6f,0x67,0x21,0x60,0x22,0x69,0x4d,0x11,0x4c,}}, // "in__"
+ // ASCII-7-bit=187 Latin1=149 UTF8=156 GB=94 CP1252=153 KSC=97 SJS=111 EUC-JP=103 CP1251=96 Latin5=105 ISO-8859-11=77 CP1257=76 [top ASCII-7-bit]
+ {{0x69,0x6e,0x66,0x6f, 0x05,0xac,0xa7,0xa7,0x76,0x94,0x18,0x93,0x82,0x70,0x90,0x91,0x82,0x9c,0x78,0x00,}}, // "info"
+ // ASCII-7-bit=172 Latin1=167 UTF8=167 GB=118 CP1252=148 SJS=147 EUC-JP=130 BIG5=112 Latin2=144 CP1251=145 CP1256=130 CP1250=156 Latin5=120 [top ASCII-7-bit]
+ {{0x69,0x6e,0x74,0x5f, 0x05,0xb2,0xa7,0xb0,0x3a,0x84,0x53,0x62,0x55,0x54,0x71,0x45,0x21,0x39,0x31,0x50,}}, // "int_"
+ // ASCII-7-bit=178 Latin1=167 UTF8=176 GB=58 CP1252=132 CP1251=98 CP1256=85 CP1250=84 Greek=69 CP1253=57 ISO-8859-5=80 [top ASCII-7-bit]
+ {{0x69,0x6f,0x5f,0x5f, 0x09,0xaf,0x98,0xac,0xa8,0x85,0x77,0x83,0x7b,0x67,0x12,0x88,0x71,0x22,0x77,0x67,}}, // "io__"
+ // ASCII-7-bit=175 Latin1=152 UTF8=172 GB=168 CP1252=133 KSC=119 SJS=131 EUC-JP=123 BIG5=103 CP1251=136 CP1256=113 ISO-8859-11=119 ISO-8859-15=103 [top ASCII-7-bit]
+ {{0x69,0x72,0x5f,0x5f, 0x07,0x9e,0x8c,0xb5,0x55,0x7e,0x22,0x3c,0x25,0x32,0x32,0xae,0x36,0x3e,0x81,0x52,}}, // "ir__"
+ // ASCII-7-bit=158 Latin1=140 UTF8=181 GB=85 CP1252=126 KSC=34 SJS=60 Latin2=50 CP1251=50 CP1256=174 CP1250=54 Latin5=62 CP1254=82 [top UTF8]
+ {{0x69,0x73,0x5f,0x5f, 0x05,0x8f,0xbc,0xa0,0x1b,0x84,0x15,0x3b,0x1b,0x2f,0x1b,0x25,0x11,0x32,0x21,0x1b,}}, // "is__"
+ // ASCII-7-bit=143 Latin1=188 UTF8=160 GB=27 CP1252=132 SJS=59 EUC-JP=27 BIG5=47 Latin2=27 CP1251=37 CP1250=50 ISO-8859-15=27 [top Latin1]
+ {{0x69,0x74,0x5f,0x5f, 0x03,0xac,0xb4,0x9e,0x11,0xa2,0x11,0x35,0x21,0x44,0x21,0x3e,0x21,0x60,0x51,0x55,}}, // "it__"
+ // ASCII-7-bit=172 Latin1=180 UTF8=158 CP1252=162 SJS=53 Latin2=68 CP1250=62 ISO-8859-15=96 JIS=85 [top Latin1]
+ {{0x6a,0x65,0x5f,0x5f, 0x03,0xb7,0x8e,0xab,0x11,0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "je__"
+ // ASCII-7-bit=183 Latin1=142 UTF8=171 CP1252=158 [top ASCII-7-bit]
+ {{0x6a,0x6d,0x5f,0x5f, 0x03,0xb8,0xa6,0x93,0x11,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jm__"
+ // ASCII-7-bit=184 Latin1=166 UTF8=147 CP1252=160 [top ASCII-7-bit]
+ {{0x6a,0x6f,0x5f,0x5f, 0x03,0xa5,0x73,0xb0,0x11,0x82,0x52,0x3c,0xb2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jo__"
+ // ASCII-7-bit=165 Latin1=115 UTF8=176 CP1252=130 CP1251=60 CP1256=178 [top CP1256]
+ {{0x6a,0x6f,0x62,0x73, 0x03,0xb7,0xa1,0xa5,0x11,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jobs"
+ // ASCII-7-bit=183 Latin1=161 UTF8=165 CP1252=160 [top ASCII-7-bit]
+ {{0x6a,0x70,0x5f,0x5f, 0x09,0x9b,0x48,0xa4,0x45,0x42,0x4b,0xb3,0xad,0x48,0x51,0x1e,0x61,0x73,0x21,0x63,}}, // "jp__"
+ // ASCII-7-bit=155 Latin1=72 UTF8=164 GB=69 CP1252=66 KSC=75 SJS=179 EUC-JP=173 BIG5=72 ISO-8859-11=30 JIS=115 CP932=99 [top SJS]
+ {{0x6b,0x65,0x5f,0x5f, 0x03,0xb9,0x9a,0xa1,0x13,0x9e,0x4b,0x5b,0x11,0x4b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ke__"
+ // ASCII-7-bit=185 Latin1=154 UTF8=161 CP1252=158 KSC=75 SJS=91 BIG5=75 [top ASCII-7-bit]
+ {{0x6b,0x67,0x5f,0x5f, 0x05,0x94,0x71,0x9f,0x67,0x83,0x11,0xa8,0x31,0xb7,0x12,0x57,0x57,0x41,0x6f,0x00,}}, // "kg__"
+ // ASCII-7-bit=148 Latin1=113 UTF8=159 GB=103 CP1252=131 SJS=168 CP1251=183 CP1250=87 Latin5=87 KOI8R=111 [top CP1251]
+ {{0x6b,0x68,0x5f,0x5f, 0x05,0xb6,0xa1,0xa1,0x53,0xa5,0x12,0x74,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kh__"
+ // ASCII-7-bit=182 Latin1=161 UTF8=161 GB=83 CP1252=165 SJS=116 EUC-JP=115 [top ASCII-7-bit]
+ {{0x6b,0x69,0x5f,0x5f, 0x03,0xb8,0xac,0x98,0x11,0x82,0x81,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ki__"
+ // ASCII-7-bit=184 Latin1=172 UTF8=152 CP1252=130 Latin5=97 [top ASCII-7-bit]
+ {{0x6b,0x6e,0x5f,0x5f, 0x01,0xba,0x11,0x89,0x11,0xaa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kn__"
+ // ASCII-7-bit=186 UTF8=137 CP1252=170 [top ASCII-7-bit]
+ {{0x6b,0x72,0x5f,0x5f, 0x09,0x80,0x39,0x92,0x43,0x2e,0xbe,0x5f,0x3a,0x3d,0x31,0x0c,0x00,0x00,0x00,0x00,}}, // "kr__"
+ // ASCII-7-bit=128 Latin1=57 UTF8=146 GB=67 CP1252=46 KSC=190 SJS=95 EUC-JP=58 BIG5=61 CP1250=12 [top KSC]
+ {{0x6b,0x77,0x5f,0x5f, 0x03,0x91,0x69,0xb2,0x11,0x71,0x61,0xb5,0x11,0x40,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kw__"
+ // ASCII-7-bit=145 Latin1=105 UTF8=178 CP1252=113 CP1256=181 Latin5=64 [top CP1256]
+ {{0x6b,0x79,0x5f,0x5f, 0x03,0xab,0x9d,0xb6,0x11,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ky__"
+ // ASCII-7-bit=171 Latin1=157 UTF8=182 CP1252=154 [top UTF8]
+ {{0x6b,0x7a,0x5f,0x5f, 0x03,0x8d,0x7b,0xb1,0x12,0x74,0x3d,0x41,0xb5,0x21,0x4a,0x41,0x8b,0x81,0x4a,0x00,}}, // "kz__"
+ // ASCII-7-bit=141 Latin1=123 UTF8=177 CP1252=116 KSC=61 CP1251=181 Latin5=74 KOI8R=139 ISO-8859-5=74 [top CP1251]
+ {{0x6c,0x61,0x5f,0x5f, 0x05,0xa8,0x8e,0xab,0xab,0x71,0x13,0x9f,0x6a,0x99,0x41,0x77,0x51,0x64,0x41,0x7e,}}, // "la__"
+ // ASCII-7-bit=168 Latin1=142 UTF8=171 GB=171 CP1252=113 SJS=159 EUC-JP=106 BIG5=153 Latin5=119 GBK=100 CP932=126 [top UTF8]
+ {{0x6c,0x62,0x5f,0x5f, 0x03,0xb3,0x95,0xaf,0x11,0x90,0x11,0x35,0x41,0x9f,0x10,0x61,0x3f,0x00,0x00,0x00,}}, // "lb__"
+ // ASCII-7-bit=179 Latin1=149 UTF8=175 CP1252=144 SJS=53 CP1256=159 Arabic=63 [top ASCII-7-bit]
+ {{0x6c,0x63,0x5f,0x5f, 0x02,0xab,0xa8,0x21,0xb5,0x11,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lc__"
+ // ASCII-7-bit=171 Latin1=168 CP1252=181 SJS=127 [top CP1252]
+ {{0x6c,0x69,0x5f,0x5f, 0x05,0xad,0xb5,0x9d,0x72,0x93,0x12,0x5f,0x53,0x22,0x89,0x64,0x11,0x5f,0x51,0x53,}}, // "li__"
+ // ASCII-7-bit=173 Latin1=181 UTF8=157 GB=114 CP1252=147 SJS=95 EUC-JP=83 CP1251=137 CP1256=100 Latin5=95 GBK=83 [top Latin1]
+ {{0x6c,0x6b,0x5f,0x5f, 0x03,0xb9,0x9e,0x9d,0x11,0xa1,0x12,0x47,0x47,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lk__"
+ // ASCII-7-bit=185 Latin1=158 UTF8=157 CP1252=161 SJS=71 EUC-JP=71 [top ASCII-7-bit]
+ {{0x6c,0x72,0x5f,0x5f, 0x03,0xad,0xac,0x8a,0x11,0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lr__"
+ // ASCII-7-bit=173 Latin1=172 UTF8=138 CP1252=177 [top CP1252]
+ {{0x6c,0x73,0x5f,0x5f, 0x03,0xb7,0xa3,0x97,0x11,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ls__"
+ // ASCII-7-bit=183 Latin1=163 UTF8=151 CP1252=168 [top ASCII-7-bit]
+ {{0x6c,0x74,0x5f,0x5f, 0x06,0x8c,0x64,0xae,0x3b,0x6d,0x5d,0x32,0x48,0x71,0x11,0x55,0x31,0xb8,0xc1,0x6e,}}, // "lt__"
+ // ASCII-7-bit=140 Latin1=100 UTF8=174 GB=59 CP1252=109 KSC=93 Latin2=72 CP1251=113 CP1250=85 CP1257=184 ISO-8859-13=110 [top CP1257]
+ {{0x6c,0x75,0x5f,0x5f, 0x05,0xa6,0xb2,0xad,0x47,0x9f,0x41,0x42,0x22,0x54,0x3f,0x11,0x53,0x10,0x51,0x3b,}}, // "lu__"
+ // ASCII-7-bit=166 Latin1=178 UTF8=173 GB=71 CP1252=159 Latin2=66 CP1250=84 Latin5=63 ISO-8859-15=83 UTF-16BE=59 [top Latin1]
+ {{0x6c,0x76,0x5f,0x5f, 0x03,0x97,0x6d,0xb5,0x11,0x6b,0x51,0x92,0x51,0xb0,0x11,0x4e,0xa2,0x5f,0x52,0x00,}}, // "lv__"
+ // ASCII-7-bit=151 Latin1=109 UTF8=181 CP1252=107 CP1251=146 CP1257=176 KOI8R=78 ISO-8859-13=95 Latin4=82 [top UTF8]
+ {{0x6c,0x79,0x5f,0x5f, 0x07,0xa4,0x77,0xb0,0x59,0x8c,0x92,0x59,0x32,0x59,0xb1,0x00,0x00,0x00,0x00,0x00,}}, // "ly__"
+ // ASCII-7-bit=164 Latin1=119 UTF8=176 GB=89 CP1252=140 KSC=146 SJS=89 CP1251=89 CP1256=177 [top CP1256]
+ {{0x6d,0x61,0x5f,0x5f, 0x03,0xa2,0xb3,0xa6,0x11,0xa0,0x62,0x9e,0x5d,0x22,0x7d,0x4b,0x10,0x11,0x47,0x00,}}, // "ma__"
+ // ASCII-7-bit=162 Latin1=179 UTF8=166 CP1252=160 CP1256=158 CP1250=93 ISO-8859-15=125 CP1257=75 Arabic=71 [top Latin1]
+ {{0x6d,0x63,0x5f,0x5f, 0x03,0x87,0xbe,0x73,0x11,0x89,0xa1,0x47,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mc__"
+ // ASCII-7-bit=135 Latin1=190 UTF8=115 CP1252=137 ISO-8859-15=71 [top Latin1]
+ {{0x6d,0x64,0x5f,0x5f, 0x03,0xa4,0x82,0xad,0x11,0x86,0x11,0x61,0x15,0x61,0x74,0xae,0x6e,0x99,0x51,0x9d,}}, // "md__"
+ // ASCII-7-bit=164 Latin1=130 UTF8=173 CP1252=134 SJS=97 BIG5=97 Latin2=116 CP1251=174 CP1256=110 CP1250=153 KOI8R=157 [top CP1251]
+ {{0x6d,0x67,0x5f,0x5f, 0x03,0x99,0xba,0x85,0x11,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mg__"
+ // ASCII-7-bit=153 Latin1=186 UTF8=133 CP1252=166 [top Latin1]
+ {{0x6d,0x69,0x6c,0x5f, 0x03,0xba,0x9a,0x9a,0x13,0x9a,0x44,0x50,0x11,0x24,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mil_"
+ // ASCII-7-bit=186 Latin1=154 UTF8=154 CP1252=154 KSC=68 SJS=80 BIG5=36 [top ASCII-7-bit]
+ {{0x6d,0x6b,0x5f,0x5f, 0x03,0x95,0x72,0xab,0x11,0x73,0x42,0x3f,0xa0,0x11,0x66,0x51,0xb6,0x81,0x56,0x00,}}, // "mk__"
+ // ASCII-7-bit=149 Latin1=114 UTF8=171 CP1252=115 Latin2=63 CP1251=160 CP1250=102 KOI8R=182 ISO-8859-5=86 [top KOI8R]
+ {{0x6d,0x6c,0x5f,0x5f, 0x03,0x85,0xbd,0x85,0x11,0x96,0x81,0x59,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ml__"
+ // ASCII-7-bit=133 Latin1=189 UTF8=133 CP1252=150 Latin5=89 [top Latin1]
+ {{0x6d,0x6d,0x5f,0x5f, 0x03,0xb5,0xa1,0x7d,0x11,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mm__"
+ // ASCII-7-bit=181 Latin1=161 UTF8=125 CP1252=175 [top ASCII-7-bit]
+ {{0x6d,0x6e,0x5f,0x5f, 0x07,0x9f,0x7f,0xb8,0x79,0x7d,0x4f,0x5f,0x11,0x56,0x11,0xa7,0x00,0x00,0x00,0x00,}}, // "mn__"
+ // ASCII-7-bit=159 Latin1=127 UTF8=184 GB=121 CP1252=125 KSC=79 SJS=95 BIG5=86 CP1251=167 [top UTF8]
+ {{0x6d,0x6f,0x5f,0x5f, 0x05,0xa0,0x9e,0xaa,0x9e,0x86,0x11,0x4a,0x11,0xb2,0xa1,0x59,0xc1,0x36,0x00,0x00,}}, // "mo__"
+ // ASCII-7-bit=160 Latin1=158 UTF8=170 GB=158 CP1252=134 SJS=74 BIG5=178 GBK=89 GB18030=54 [top BIG5]
+ {{0x6d,0x6f,0x62,0x69, 0x08,0xb6,0x95,0xaa,0x7f,0x7d,0x53,0xa0,0x71,0x13,0x65,0x6d,0x78,0xc1,0x73,0x00,}}, // "mobi"
+ // ASCII-7-bit=182 Latin1=149 UTF8=170 GB=127 CP1252=125 KSC=83 SJS=160 EUC-JP=113 Latin2=101 CP1251=109 CP1256=120 CP932=115 [top ASCII-7-bit]
+ {{0x6d,0x70,0x5f,0x5f, 0x03,0xbe,0x5a,0x54,0x11,0x5c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mp__"
+ // ASCII-7-bit=190 Latin1=90 UTF8=84 CP1252=92 [top ASCII-7-bit]
+ {{0x6d,0x71,0x5f,0x5f, 0x03,0xa3,0xb8,0x95,0x11,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mq__"
+ // ASCII-7-bit=163 Latin1=184 UTF8=149 CP1252=165 [top Latin1]
+ {{0x6d,0x72,0x5f,0x5f, 0x03,0x9b,0xab,0x8d,0x11,0x96,0x61,0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mr__"
+ // ASCII-7-bit=155 Latin1=171 UTF8=141 CP1252=150 CP1256=182 [top CP1256]
+ {{0x6d,0x73,0x5f,0x5f, 0x03,0xb4,0xae,0x9c,0x18,0x8a,0x73,0x86,0x6f,0x53,0x56,0x6c,0x75,0x11,0x85,0x00,}}, // "ms__"
+ // ASCII-7-bit=180 Latin1=174 UTF8=156 CP1252=138 KSC=115 SJS=134 EUC-JP=111 BIG5=83 Latin2=86 CP1251=108 CP1256=117 Latin5=133 [top ASCII-7-bit]
+ {{0x6d,0x74,0x5f,0x5f, 0x05,0xbc,0x87,0x9b,0x5c,0x8b,0x42,0x2c,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mt__"
+ // ASCII-7-bit=188 Latin1=135 UTF8=155 GB=92 CP1252=139 Latin2=44 CP1251=64 [top ASCII-7-bit]
+ {{0x6d,0x75,0x5f,0x5f, 0x09,0xb4,0xa4,0xa9,0x57,0x9f,0x4d,0x7e,0x70,0x4d,0x11,0x4d,0x21,0x57,0x11,0x4d,}}, // "mu__"
+ // ASCII-7-bit=180 Latin1=164 UTF8=169 GB=87 CP1252=159 KSC=77 SJS=126 EUC-JP=112 BIG5=77 CP1251=77 Latin5=87 ISO-8859-15=77 [top ASCII-7-bit]
+ {{0x6d,0x75,0x73,0x65, 0x07,0xb3,0xa9,0xa6,0x76,0x90,0x56,0x88,0x13,0x8e,0x6a,0x7a,0x00,0x00,0x00,0x00,}}, // "muse"
+ // ASCII-7-bit=179 Latin1=169 UTF8=166 GB=118 CP1252=144 KSC=86 SJS=136 BIG5=142 Latin2=106 CP1251=122 [top ASCII-7-bit]
+ {{0x6d,0x76,0x5f,0x5f, 0x03,0xb6,0x98,0xab,0x11,0x9f,0x61,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mv__"
+ // ASCII-7-bit=182 Latin1=152 UTF8=171 CP1252=159 CP1256=83 [top ASCII-7-bit]
+ {{0x6d,0x77,0x5f,0x5f, 0x05,0xb7,0xa3,0xa8,0x7f,0x8d,0x11,0x7f,0xc1,0x6b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mw__"
+ // ASCII-7-bit=183 Latin1=163 UTF8=168 GB=127 CP1252=141 SJS=127 GBK=107 [top ASCII-7-bit]
+ {{0x6d,0x78,0x5f,0x5f, 0x05,0xa7,0xba,0x94,0x45,0x93,0x11,0x1d,0x12,0x25,0x29,0x21,0x27,0x21,0x3c,0x00,}}, // "mx__"
+ // ASCII-7-bit=167 Latin1=186 UTF8=148 GB=69 CP1252=147 SJS=29 BIG5=37 Latin2=41 CP1250=39 ISO-8859-15=60 [top Latin1]
+ {{0x6d,0x79,0x5f,0x5f, 0x05,0xb3,0xb3,0x8d,0x6b,0x8f,0x13,0x4d,0x50,0x7f,0x12,0x3f,0x6b,0x12,0x4b,0x3e,}}, // "my__"
+ // ASCII-7-bit=179 Latin1=179 UTF8=141 GB=107 CP1252=143 SJS=77 EUC-JP=80 BIG5=127 CP1251=63 CP1256=107 Latin5=75 ISO-8859-11=62 [top ASCII-7-bit]
+ {{0x6d,0x7a,0x5f,0x5f, 0x03,0x8e,0xaf,0xb7,0x11,0x8f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mz__"
+ // ASCII-7-bit=142 Latin1=175 UTF8=183 CP1252=143 [top UTF8]
+ {{0x6e,0x61,0x5f,0x5f, 0x03,0xba,0xa1,0x97,0x11,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "na__"
+ // ASCII-7-bit=186 Latin1=161 UTF8=151 CP1252=156 [top ASCII-7-bit]
+ {{0x6e,0x61,0x6d,0x65, 0x08,0xb2,0xa4,0xa9,0x8a,0x90,0x7d,0x7c,0x6b,0x15,0x80,0x94,0x6b,0x8b,0x77,0x00,}}, // "name"
+ // ASCII-7-bit=178 Latin1=164 UTF8=169 GB=138 CP1252=144 KSC=125 SJS=124 EUC-JP=107 Latin2=128 CP1251=148 CP1256=107 CP1250=139 Latin5=119 [top ASCII-7-bit]
+ {{0x6e,0x63,0x5f,0x5f, 0x03,0xa7,0xb9,0x8c,0x11,0x9e,0x11,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nc__"
+ // ASCII-7-bit=167 Latin1=185 UTF8=140 CP1252=158 SJS=114 [top Latin1]
+ {{0x6e,0x65,0x5f,0x5f, 0x03,0xad,0xb9,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ne__"
+ // ASCII-7-bit=173 Latin1=185 UTF8=128 [top Latin1]
+ {{0x6e,0x65,0x74,0x5f, 0x0f,0xac,0xa6,0xa4,0x93,0x93,0x94,0x97,0x83,0x86,0x7e,0x80,0x96,0x7a,0x89,0x73,}}, // "net_"
+ // ASCII-7-bit=172 Latin1=166 UTF8=164 GB=147 CP1252=147 KSC=148 SJS=151 EUC-JP=131 BIG5=134 Latin2=126 CP1251=128 CP1256=150 CP1250=122 Latin5=137 ISO-8859-11=115 [top ASCII-7-bit]
+ {{0x6e,0x66,0x5f,0x5f, 0x03,0xb5,0xa9,0xa8,0x11,0x87,0xa1,0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nf__"
+ // ASCII-7-bit=181 Latin1=169 UTF8=168 CP1252=135 ISO-8859-15=119 [top ASCII-7-bit]
+ {{0x6e,0x67,0x5f,0x5f, 0x03,0xbd,0x95,0x89,0x11,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ng__"
+ // ASCII-7-bit=189 Latin1=149 UTF8=137 CP1252=146 [top ASCII-7-bit]
+ {{0x6e,0x69,0x5f,0x5f, 0x03,0x9d,0xb3,0xa9,0x11,0xa8,0x81,0x40,0x11,0x36,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ni__"
+ // ASCII-7-bit=157 Latin1=179 UTF8=169 CP1252=168 Latin5=64 ISO-8859-15=54 [top Latin1]
+ {{0x6e,0x6c,0x5f,0x5f, 0x05,0xb2,0xae,0xa0,0x32,0xa1,0x36,0x31,0x42,0x39,0x3b,0x33,0x45,0x11,0x6f,0x00,}}, // "nl__"
+ // ASCII-7-bit=178 Latin1=174 UTF8=160 GB=50 CP1252=161 BIG5=49 Latin2=66 CP1251=57 CP1256=59 CP1250=51 Latin5=69 ISO-8859-15=111 [top ASCII-7-bit]
+ {{0x6e,0x6f,0x5f,0x5f, 0x05,0x99,0xb8,0xaa,0x47,0x8d,0x11,0x31,0x22,0x34,0x3e,0x42,0x70,0x33,0x00,0x00,}}, // "no__"
+ // ASCII-7-bit=153 Latin1=184 UTF8=170 GB=71 CP1252=141 SJS=49 Latin2=52 CP1251=62 ISO-8859-15=112 CP1257=51 [top Latin1]
+ {{0x6e,0x70,0x5f,0x5f, 0x03,0xb2,0x9f,0xa5,0x11,0xac,0x11,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "np__"
+ // ASCII-7-bit=178 Latin1=159 UTF8=165 CP1252=172 SJS=97 [top ASCII-7-bit]
+ {{0x6e,0x72,0x5f,0x5f, 0x03,0xbe,0x77,0x7a,0x11,0x62,0x71,0x44,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nr__"
+ // ASCII-7-bit=190 Latin1=119 UTF8=122 CP1252=98 CP1250=68 [top ASCII-7-bit]
+ {{0x6e,0x75,0x5f,0x5f, 0x03,0xae,0xb4,0x9e,0x11,0x96,0x12,0x7c,0x76,0x22,0x75,0x62,0xa1,0x65,0x00,0x00,}}, // "nu__"
+ // ASCII-7-bit=174 Latin1=180 UTF8=158 CP1252=150 SJS=124 EUC-JP=118 CP1251=117 CP1256=98 CP1254=101 [top Latin1]
+ {{0x6e,0x7a,0x5f,0x5f, 0x0d,0xba,0x97,0xa2,0x5f,0x97,0x5c,0x5a,0x4b,0x4a,0x30,0x47,0x36,0x34,0x21,0x2b,}}, // "nz__"
+ // ASCII-7-bit=186 Latin1=151 UTF8=162 GB=95 CP1252=151 KSC=92 SJS=90 EUC-JP=75 BIG5=74 Latin2=48 CP1251=71 CP1256=54 CP1250=52 ISO-8859-15=43 [top ASCII-7-bit]
+ {{0x6f,0x6d,0x5f,0x5f, 0x03,0x9a,0x8a,0x89,0x11,0x87,0x61,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "om__"
+ // ASCII-7-bit=154 Latin1=138 UTF8=137 CP1252=135 CP1256=188 [top CP1256]
+ {{0x6f,0x72,0x67,0x5f, 0x0e,0xb5,0x9f,0xac,0x76,0x90,0x7e,0x6e,0x69,0x71,0x6d,0x6a,0x78,0x60,0x73,0x00,}}, // "org_"
+ // ASCII-7-bit=181 Latin1=159 UTF8=172 GB=118 CP1252=144 KSC=126 SJS=110 EUC-JP=105 BIG5=113 Latin2=109 CP1251=106 CP1256=120 CP1250=96 Latin5=115 [top ASCII-7-bit]
+ {{0x70,0x61,0x5f,0x5f, 0x03,0xa3,0xb8,0xa7,0x11,0x87,0x71,0x31,0x21,0x41,0x21,0x31,0x00,0x00,0x00,0x00,}}, // "pa__"
+ // ASCII-7-bit=163 Latin1=184 UTF8=167 CP1252=135 CP1250=49 ISO-8859-15=65 KOI8R=49 [top Latin1]
+ {{0x70,0x65,0x5f,0x5f, 0x05,0xa8,0xb8,0x9d,0x3a,0x99,0x11,0x2e,0x13,0x2e,0x1e,0x1e,0x11,0x32,0x00,0x00,}}, // "pe__"
+ // ASCII-7-bit=168 Latin1=184 UTF8=157 GB=58 CP1252=153 SJS=46 BIG5=46 Latin2=30 CP1251=30 CP1250=50 [top Latin1]
+ {{0x70,0x66,0x5f,0x5f, 0x03,0xa2,0xb5,0xa3,0x11,0xa7,0x11,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pf__"
+ // ASCII-7-bit=162 Latin1=181 UTF8=163 CP1252=167 SJS=99 [top Latin1]
+ {{0x70,0x67,0x5f,0x5f, 0x05,0xb8,0xa6,0x81,0x6a,0xa5,0x11,0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pg__"
+ // ASCII-7-bit=184 Latin1=166 UTF8=129 GB=106 CP1252=165 SJS=112 [top ASCII-7-bit]
+ {{0x70,0x68,0x5f,0x5f, 0x03,0xb5,0x9e,0x99,0x15,0xac,0x57,0x65,0x4f,0x51,0x31,0x31,0x11,0x4b,0xd1,0x29,}}, // "ph__"
+ // ASCII-7-bit=181 Latin1=158 UTF8=153 CP1252=172 KSC=87 SJS=101 EUC-JP=79 BIG5=81 CP1250=49 ISO-8859-11=75 CP874=41 [top ASCII-7-bit]
+ {{0x70,0x6b,0x5f,0x5f, 0x03,0xb9,0x9b,0x8c,0x11,0xa3,0x11,0x33,0x11,0x99,0x21,0x29,0x00,0x00,0x00,0x00,}}, // "pk__"
+ // ASCII-7-bit=185 Latin1=155 UTF8=140 CP1252=163 SJS=51 BIG5=153 CP1256=41 [top ASCII-7-bit]
+ {{0x70,0x6c,0x5f,0x5f, 0x03,0x89,0x62,0xa8,0x11,0x4b,0x42,0xba,0x40,0x11,0x92,0xe1,0x3e,0x00,0x00,0x00,}}, // "pl__"
+ // ASCII-7-bit=137 Latin1=98 UTF8=168 CP1252=75 Latin2=186 CP1251=64 CP1250=146 ISO-8859-5=62 [top Latin2]
+ {{0x70,0x6e,0x5f,0x5f, 0x06,0xbb,0xa1,0x99,0x66,0x66,0x66,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pn__"
+ // ASCII-7-bit=187 Latin1=161 UTF8=153 GB=102 CP1252=102 KSC=102 [top ASCII-7-bit]
+ {{0x70,0x72,0x5f,0x5f, 0x03,0x9b,0xb5,0xaf,0x11,0x94,0x41,0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pr__"
+ // ASCII-7-bit=155 Latin1=181 UTF8=175 CP1252=148 Latin2=101 [top Latin1]
+ {{0x70,0x72,0x6f,0x5f, 0x03,0xb3,0x9e,0xad,0x11,0x87,0x13,0x65,0x6f,0x65,0x11,0x9f,0x21,0x79,0x00,0x00,}}, // "pro_"
+ // ASCII-7-bit=179 Latin1=158 UTF8=173 CP1252=135 SJS=101 EUC-JP=111 BIG5=101 CP1251=159 Latin5=121 [top ASCII-7-bit]
+ {{0x70,0x73,0x5f,0x5f, 0x03,0x99,0x8f,0x9d,0x11,0x9f,0x61,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ps__"
+ // ASCII-7-bit=153 Latin1=143 UTF8=157 CP1252=159 CP1256=185 [top CP1256]
+ {{0x70,0x74,0x5f,0x5f, 0x05,0x99,0xb5,0xad,0x1d,0x95,0x11,0x21,0x52,0x36,0x40,0x12,0x90,0x23,0x51,0x1d,}}, // "pt__"
+ // ASCII-7-bit=153 Latin1=181 UTF8=173 GB=29 CP1252=149 SJS=33 CP1250=54 Latin5=64 ISO-8859-15=144 CP1257=35 CP1254=29 [top Latin1]
+ {{0x70,0x79,0x5f,0x5f, 0x03,0x9f,0xbb,0x90,0x13,0x97,0x49,0x35,0x21,0x35,0x51,0x5a,0x00,0x00,0x00,0x00,}}, // "py__"
+ // ASCII-7-bit=159 Latin1=187 UTF8=144 CP1252=151 KSC=73 SJS=53 Latin2=53 ISO-8859-15=90 [top Latin1]
+ {{0x71,0x61,0x5f,0x5f, 0x03,0x9a,0x89,0xb0,0x11,0x82,0x61,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "qa__"
+ // ASCII-7-bit=154 Latin1=137 UTF8=176 CP1252=130 CP1256=181 [top CP1256]
+ {{0x72,0x65,0x5f,0x5f, 0x03,0x8a,0xb4,0xb1,0x11,0x97,0xa1,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "re__"
+ // ASCII-7-bit=138 Latin1=180 UTF8=177 CP1252=151 ISO-8859-15=138 [top Latin1]
+ {{0x72,0x6f,0x5f,0x5f, 0x08,0xb4,0xa0,0xa4,0x3c,0x94,0x57,0x2e,0x29,0x14,0xa2,0x3a,0x33,0x8b,0x21,0x37,}}, // "ro__"
+ // ASCII-7-bit=180 Latin1=160 UTF8=164 GB=60 CP1252=148 KSC=87 SJS=46 EUC-JP=41 Latin2=162 CP1251=58 CP1256=51 CP1250=139 ISO-8859-15=55 [top ASCII-7-bit]
+ {{0x72,0x75,0x5f,0x5f, 0x05,0x8d,0x6d,0xa1,0x56,0x67,0x31,0x43,0x12,0xba,0x46,0x61,0x9b,0x72,0x46,0x48,}}, // "ru__"
+ // ASCII-7-bit=141 Latin1=109 UTF8=161 GB=86 CP1252=103 BIG5=67 CP1251=186 CP1256=70 KOI8R=155 KOI8U=70 ISO-8859-5=72 [top CP1251]
+ {{0x72,0x77,0x5f,0x5f, 0x03,0xb7,0xa2,0xa4,0x11,0xa0,0x61,0x5e,0x31,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rw__"
+ // ASCII-7-bit=183 Latin1=162 UTF8=164 CP1252=160 CP1256=94 ISO-8859-15=110 [top ASCII-7-bit]
+ {{0x73,0x61,0x5f,0x5f, 0x05,0x91,0x5b,0xac,0x3f,0x69,0x11,0x1f,0x41,0xb9,0x11,0x1f,0x00,0x00,0x00,0x00,}}, // "sa__"
+ // ASCII-7-bit=145 Latin1=91 UTF8=172 GB=63 CP1252=105 SJS=31 CP1256=185 Latin5=31 [top CP1256]
+ {{0x73,0x62,0x5f,0x5f, 0x03,0xb8,0x8a,0xad,0x11,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sb__"
+ // ASCII-7-bit=184 Latin1=138 UTF8=173 CP1252=139 [top ASCII-7-bit]
+ {{0x73,0x63,0x5f,0x5f, 0x03,0xb5,0x9e,0xad,0x11,0x85,0x12,0x8d,0x88,0x23,0x57,0x4d,0x5c,0x00,0x00,0x00,}}, // "sc__"
+ // ASCII-7-bit=181 Latin1=158 UTF8=173 CP1252=133 SJS=141 EUC-JP=136 CP1251=87 CP1256=77 CP1250=92 [top ASCII-7-bit]
+ {{0x73,0x64,0x5f,0x5f, 0x03,0x9b,0x77,0x8a,0x11,0x6d,0x61,0xbd,0x10,0x61,0x73,0x00,0x00,0x00,0x00,0x00,}}, // "sd__"
+ // ASCII-7-bit=155 Latin1=119 UTF8=138 CP1252=109 CP1256=189 Arabic=115 [top CP1256]
+ {{0x73,0x65,0x5f,0x5f, 0x05,0x96,0xbb,0xa0,0x23,0x82,0x36,0x23,0x2e,0x25,0x3b,0x43,0x22,0x11,0x4c,0x00,}}, // "se__"
+ // ASCII-7-bit=150 Latin1=187 UTF8=160 GB=35 CP1252=130 BIG5=35 Latin2=46 CP1251=37 CP1256=59 CP1250=67 Latin5=34 ISO-8859-15=76 [top Latin1]
+ {{0x73,0x67,0x5f,0x5f, 0x09,0xb8,0x9c,0xa6,0x84,0x94,0x4c,0x6e,0x50,0x71,0x31,0x41,0x11,0x48,0xd1,0x3a,}}, // "sg__"
+ // ASCII-7-bit=184 Latin1=156 UTF8=166 GB=132 CP1252=148 KSC=76 SJS=110 EUC-JP=80 BIG5=113 CP1250=65 ISO-8859-11=72 CP874=58 [top ASCII-7-bit]
+ {{0x73,0x68,0x5f,0x5f, 0x0a,0xaa,0x9b,0xa1,0xa9,0x84,0x77,0xa1,0x98,0x9b,0x5d,0x51,0x6d,0x31,0x6f,0x00,}}, // "sh__"
+ // ASCII-7-bit=170 Latin1=155 UTF8=161 GB=169 CP1252=132 KSC=119 SJS=161 EUC-JP=152 BIG5=155 Latin2=93 ISO-8859-15=109 GBK=111 [top ASCII-7-bit]
+ {{0x73,0x69,0x5f,0x5f, 0x03,0x95,0x6b,0xb7,0x11,0x6e,0x42,0x9f,0x3d,0x11,0xa9,0x21,0x17,0x10,0x11,0x24,}}, // "si__"
+ // ASCII-7-bit=149 Latin1=107 UTF8=183 CP1252=110 Latin2=159 CP1251=61 CP1250=169 ISO-8859-15=23 CP852=36 [top UTF8]
+ {{0x73,0x6b,0x5f,0x5f, 0x03,0x95,0x74,0xb0,0x11,0x60,0x36,0x53,0x92,0x55,0x47,0xb5,0x3f,0x10,0x11,0x3a,}}, // "sk__"
+ // ASCII-7-bit=149 Latin1=116 UTF8=176 CP1252=96 BIG5=83 Latin2=146 CP1251=85 CP1256=71 CP1250=181 Latin5=63 MACINTOSH=58 [top CP1250]
+ {{0x73,0x6c,0x5f,0x5f, 0x03,0xac,0x85,0x8f,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sl__"
+ // ASCII-7-bit=172 Latin1=133 UTF8=143 CP1252=185 [top CP1252]
+ {{0x73,0x6d,0x5f,0x5f, 0x03,0xa8,0xa7,0xb1,0x11,0xa8,0x91,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sm__"
+ // ASCII-7-bit=168 Latin1=167 UTF8=177 CP1252=168 ISO-8859-11=111 [top UTF8]
+ {{0x73,0x6e,0x5f,0x5f, 0x03,0x9d,0xb8,0x9f,0x11,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sn__"
+ // ASCII-7-bit=157 Latin1=184 UTF8=159 CP1252=162 [top Latin1]
+ {{0x73,0x72,0x5f,0x5f, 0x03,0xa6,0xad,0xb2,0x12,0x9d,0x6a,0x61,0x84,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sr__"
+ // ASCII-7-bit=166 Latin1=173 UTF8=178 CP1252=157 KSC=106 CP1250=132 [top UTF8]
+ {{0x73,0x74,0x5f,0x5f, 0x03,0xb4,0xab,0x9d,0x15,0x88,0x79,0x99,0x94,0x7e,0x31,0x61,0x81,0x4e,0x21,0x6d,}}, // "st__"
+ // ASCII-7-bit=180 Latin1=171 UTF8=157 CP1252=136 KSC=121 SJS=153 EUC-JP=148 BIG5=126 CP1250=97 JIS=78 CP932=109 [top ASCII-7-bit]
+ {{0x73,0x75,0x5f,0x5f, 0x03,0xa4,0x6f,0xa0,0x11,0x70,0x51,0xb9,0x71,0x94,0x71,0x44,0x00,0x00,0x00,0x00,}}, // "su__"
+ // ASCII-7-bit=164 Latin1=111 UTF8=160 CP1252=112 CP1251=185 KOI8R=148 KOI8U=68 [top CP1251]
+ {{0x73,0x76,0x5f,0x5f, 0x03,0x9d,0xb9,0xa5,0x11,0x8f,0x41,0x3c,0x51,0x3c,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sv__"
+ // ASCII-7-bit=157 Latin1=185 UTF8=165 CP1252=143 Latin2=60 ISO-8859-15=60 [top Latin1]
+ {{0x73,0x79,0x5f,0x5f, 0x03,0x82,0x5e,0x90,0x11,0x5d,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sy__"
+ // ASCII-7-bit=130 Latin1=94 UTF8=144 CP1252=93 CP1256=190 [top CP1256]
+ {{0x73,0x7a,0x5f,0x5f, 0x03,0xb7,0xac,0x6c,0x11,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sz__"
+ // ASCII-7-bit=183 Latin1=172 UTF8=108 CP1252=161 [top ASCII-7-bit]
+ {{0x74,0x63,0x5f,0x5f, 0x08,0xa9,0xaa,0x9b,0x62,0x74,0x8d,0x8f,0x7a,0x21,0x67,0x22,0xab,0x9b,0x41,0x70,}}, // "tc__"
+ // ASCII-7-bit=169 Latin1=170 UTF8=155 GB=98 CP1252=116 KSC=141 SJS=143 EUC-JP=122 CP1251=103 Latin5=171 ISO-8859-11=155 GBK=112 [top Latin5]
+ {{0x74,0x66,0x5f,0x5f, 0x02,0xa3,0xbc,0x21,0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tf__"
+ // ASCII-7-bit=163 Latin1=188 CP1252=113 [top Latin1]
+ {{0x74,0x67,0x5f,0x5f, 0x03,0xb0,0xb3,0x7e,0x11,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tg__"
+ // ASCII-7-bit=176 Latin1=179 UTF8=126 CP1252=165 [top Latin1]
+ {{0x74,0x68,0x5f,0x5f, 0x09,0x96,0x61,0x93,0x3e,0x67,0x2f,0x52,0x38,0x23,0x11,0x35,0x31,0xbd,0xd1,0x76,}}, // "th__"
+ // ASCII-7-bit=150 Latin1=97 UTF8=147 GB=62 CP1252=103 KSC=47 SJS=82 EUC-JP=56 BIG5=35 CP1251=53 ISO-8859-11=189 CP874=118 [top ISO-8859-11]
+ {{0x74,0x6a,0x5f,0x5f, 0x03,0xab,0x74,0xb1,0x11,0x67,0x51,0xae,0x71,0x84,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tj__"
+ // ASCII-7-bit=171 Latin1=116 UTF8=177 CP1252=103 CP1251=174 KOI8R=132 [top UTF8]
+ {{0x74,0x6b,0x5f,0x5f, 0x03,0xbc,0x94,0x9d,0x11,0x6b,0x12,0x74,0x6b,0x12,0x53,0x46,0x12,0x74,0x6d,0x00,}}, // "tk__"
+ // ASCII-7-bit=188 Latin1=148 UTF8=157 CP1252=107 SJS=116 EUC-JP=107 Latin2=83 CP1251=70 CP1250=116 Latin5=109 [top ASCII-7-bit]
+ {{0x74,0x6c,0x5f,0x5f, 0x05,0xb1,0xb0,0x88,0x61,0x83,0xa1,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tl__"
+ // ASCII-7-bit=177 Latin1=176 UTF8=136 GB=97 CP1252=131 ISO-8859-15=168 [top ASCII-7-bit]
+ {{0x74,0x6d,0x5f,0x5f, 0x03,0xb4,0x84,0xad,0x11,0x82,0x42,0x5a,0xa4,0x12,0x5a,0x5a,0x00,0x00,0x00,0x00,}}, // "tm__"
+ // ASCII-7-bit=180 Latin1=132 UTF8=173 CP1252=130 Latin2=90 CP1251=164 CP1250=90 Latin5=90 [top ASCII-7-bit]
+ {{0x74,0x6e,0x5f,0x5f, 0x03,0x9f,0xa2,0xac,0x11,0x9d,0x11,0x4b,0x21,0x3b,0x11,0xb0,0x10,0x61,0x3b,0x00,}}, // "tn__"
+ // ASCII-7-bit=159 Latin1=162 UTF8=172 CP1252=157 SJS=75 Latin2=59 CP1256=176 Arabic=59 [top CP1256]
+ {{0x74,0x6f,0x5f,0x5f, 0x03,0xa5,0xa2,0xa5,0x15,0x89,0x8a,0xac,0x99,0x9b,0x31,0x70,0x21,0x87,0x51,0x6c,}}, // "to__"
+ // ASCII-7-bit=165 Latin1=162 UTF8=165 CP1252=137 KSC=138 SJS=172 EUC-JP=153 BIG5=155 CP1250=112 ISO-8859-15=135 JIS=108 [top SJS]
+ {{0x74,0x70,0x5f,0x5f, 0x03,0x95,0x9e,0xad,0x11,0x67,0x12,0xb3,0x99,0xd1,0x67,0x21,0x67,0x00,0x00,0x00,}}, // "tp__"
+ // ASCII-7-bit=149 Latin1=158 UTF8=173 CP1252=103 SJS=179 EUC-JP=153 JIS=103 CP932=103 [top SJS]
+ {{0x74,0x72,0x5f,0x5f, 0x03,0x8d,0x6c,0xa6,0x11,0x61,0x11,0x3c,0x32,0x3c,0x4a,0x11,0xba,0x81,0x91,0x00,}}, // "tr__"
+ // ASCII-7-bit=141 Latin1=108 UTF8=166 CP1252=97 SJS=60 CP1251=60 CP1256=74 Latin5=186 CP1254=145 [top Latin5]
+ {{0x74,0x72,0x61,0x76, 0x05,0xa9,0xa3,0xa7,0x97,0x95,0x11,0xac,0x13,0x74,0x7f,0x6b,0x11,0x63,0x00,0x00,}}, // "trav"
+ // ASCII-7-bit=169 Latin1=163 UTF8=167 GB=151 CP1252=149 SJS=172 BIG5=116 Latin2=127 CP1251=107 CP1250=99 [top SJS]
+ {{0x74,0x74,0x5f,0x5f, 0x07,0xb5,0xaf,0x9a,0x49,0x92,0x59,0x49,0x61,0x75,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tt__"
+ // ASCII-7-bit=181 Latin1=175 UTF8=154 GB=73 CP1252=146 KSC=89 SJS=73 Latin5=117 [top ASCII-7-bit]
+ {{0x74,0x76,0x5f,0x5f, 0x0e,0xa7,0xa6,0xad,0x89,0x94,0x85,0x9e,0x8d,0x7b,0x74,0x7c,0x88,0x7b,0x81,0x00,}}, // "tv__"
+ // ASCII-7-bit=167 Latin1=166 UTF8=173 GB=137 CP1252=148 KSC=133 SJS=158 EUC-JP=141 BIG5=123 Latin2=116 CP1251=124 CP1256=136 CP1250=123 Latin5=129 [top UTF8]
+ {{0x74,0x77,0x5f,0x5f, 0x05,0x85,0x52,0xab,0x5d,0x57,0x13,0x50,0x2e,0xba,0x31,0x23,0x61,0x2e,0xf1,0x21,}}, // "tw__"
+ // ASCII-7-bit=133 Latin1=82 UTF8=171 GB=93 CP1252=87 SJS=80 EUC-JP=46 BIG5=186 CP1250=35 GBK=46 BIG5_HKSCS=33 [top BIG5]
+ {{0x74,0x7a,0x5f,0x5f, 0x03,0xae,0xb5,0x8b,0x13,0xa0,0x4c,0x4c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tz__"
+ // ASCII-7-bit=174 Latin1=181 UTF8=139 CP1252=160 KSC=76 SJS=76 [top Latin1]
+ {{0x75,0x61,0x5f,0x5f, 0x03,0x87,0x61,0xa2,0x11,0x57,0x44,0x4d,0xbb,0x35,0x48,0x51,0x80,0x72,0x8c,0x3d,}}, // "ua__"
+ // ASCII-7-bit=135 Latin1=97 UTF8=162 CP1252=87 Latin2=77 CP1251=187 CP1256=53 CP1250=72 KOI8R=128 KOI8U=140 ISO-8859-5=61 [top CP1251]
+ {{0x75,0x67,0x5f,0x5f, 0x03,0xb8,0x9a,0xa0,0x11,0xa6,0x11,0x68,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ug__"
+ // ASCII-7-bit=184 Latin1=154 UTF8=160 CP1252=166 SJS=104 [top ASCII-7-bit]
+ {{0x75,0x6b,0x5f,0x5f, 0x05,0xb7,0xa7,0x9e,0x62,0x97,0x27,0x58,0x62,0x51,0x53,0x62,0x64,0x54,0x11,0x6f,}}, // "uk__"
+ // ASCII-7-bit=183 Latin1=167 UTF8=158 GB=98 CP1252=151 EUC-JP=88 BIG5=98 Latin2=81 CP1251=83 CP1256=98 CP1250=100 Latin5=84 ISO-8859-15=111 [top ASCII-7-bit]
+ {{0x75,0x73,0x5f,0x5f, 0x06,0xba,0x94,0xa5,0x45,0x92,0x48,0x24,0x54,0x47,0x5a,0x56,0x11,0x66,0x11,0x40,}}, // "us__"
+ // ASCII-7-bit=186 Latin1=148 UTF8=165 GB=69 CP1252=146 KSC=72 BIG5=84 Latin2=71 CP1251=90 CP1256=86 Latin5=102 ISO-8859-15=64 [top ASCII-7-bit]
+ {{0x75,0x79,0x5f,0x5f, 0x03,0xa2,0xba,0x96,0x11,0x9a,0x72,0x2c,0x2c,0x11,0x52,0x00,0x00,0x00,0x00,0x00,}}, // "uy__"
+ // ASCII-7-bit=162 Latin1=186 UTF8=150 CP1252=154 CP1250=44 Latin5=44 ISO-8859-15=82 [top Latin1]
+ {{0x75,0x7a,0x5f,0x5f, 0x05,0x91,0x68,0xb1,0x39,0x88,0x54,0xb5,0x43,0x39,0x6e,0x41,0x7c,0x00,0x00,0x00,}}, // "uz__"
+ // ASCII-7-bit=145 Latin1=104 UTF8=177 GB=57 CP1252=136 CP1251=181 CP1256=67 CP1250=57 Latin5=110 KOI8R=124 [top CP1251]
+ {{0x76,0x61,0x5f,0x5f, 0x03,0xae,0xb3,0x8f,0x11,0xa7,0xb1,0x4a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "va__"
+ // ASCII-7-bit=174 Latin1=179 UTF8=143 CP1252=167 CP1257=74 [top Latin1]
+ {{0x76,0x63,0x5f,0x5f, 0x08,0xba,0x7a,0x9a,0x5e,0x79,0x4a,0x9a,0x95,0x12,0x54,0x7f,0x81,0x4a,0x00,0x00,}}, // "vc__"
+ // ASCII-7-bit=186 Latin1=122 UTF8=154 GB=94 CP1252=121 KSC=74 SJS=154 EUC-JP=149 Latin2=84 CP1251=127 GBK=74 [top ASCII-7-bit]
+ {{0x76,0x65,0x5f,0x5f, 0x05,0x97,0xbc,0x8c,0x14,0x9b,0x11,0x14,0x11,0x1e,0x31,0x42,0x21,0x55,0x31,0x24,}}, // "ve__"
+ // ASCII-7-bit=151 Latin1=188 UTF8=140 GB=20 CP1252=155 SJS=20 BIG5=30 CP1250=66 ISO-8859-15=85 GBK=36 [top Latin1]
+ {{0x76,0x67,0x5f,0x5f, 0x05,0xac,0xb2,0x9e,0x6d,0x97,0x13,0x95,0x81,0x95,0x11,0x69,0x21,0x63,0x61,0x59,}}, // "vg__"
+ // ASCII-7-bit=172 Latin1=178 UTF8=158 GB=109 CP1252=151 SJS=149 EUC-JP=129 BIG5=149 CP1251=105 Latin5=99 Greek=89 [top Latin1]
+ {{0x76,0x69,0x5f,0x5f, 0x03,0xb9,0x90,0xaa,0x11,0x93,0x11,0x68,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "vi__"
+ // ASCII-7-bit=185 Latin1=144 UTF8=170 CP1252=147 SJS=104 [top ASCII-7-bit]
+ {{0x76,0x6e,0x5f,0x5f, 0x03,0x94,0x92,0xbd,0x12,0x83,0x22,0x21,0x2c,0x32,0x26,0x2e,0x00,0x00,0x00,0x00,}}, // "vn__"
+ // ASCII-7-bit=148 Latin1=146 UTF8=189 CP1252=131 KSC=34 BIG5=44 CP1250=38 Latin5=46 [top UTF8]
+ {{0x76,0x75,0x5f,0x5f, 0x03,0xae,0xb6,0x97,0x11,0x7b,0x11,0x5b,0x31,0x6a,0x41,0x4e,0x71,0x90,0x00,0x00,}}, // "vu__"
+ // ASCII-7-bit=174 Latin1=182 UTF8=151 CP1252=123 SJS=91 CP1251=106 ISO-8859-15=78 CP1253=144 [top Latin1]
+ {{0x77,0x73,0x5f,0x5f, 0x05,0xae,0xaa,0xa6,0x6c,0x91,0x18,0x76,0x69,0x7c,0x78,0x99,0x9d,0x7d,0x70,0x00,}}, // "ws__"
+ // ASCII-7-bit=174 Latin1=170 UTF8=166 GB=108 CP1252=145 SJS=118 EUC-JP=105 BIG5=124 Latin2=120 CP1251=153 CP1256=157 CP1250=125 Latin5=112 [top ASCII-7-bit]
+ {{0x79,0x65,0x5f,0x5f, 0x03,0x9e,0x94,0xab,0x11,0x8b,0x61,0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ye__"
+ // ASCII-7-bit=158 Latin1=148 UTF8=171 CP1252=139 CP1256=182 [top CP1256]
+ {{0x79,0x75,0x5f,0x5f, 0x03,0xa4,0x7f,0xaf,0x11,0x78,0x42,0x87,0x80,0x12,0xb3,0x2c,0xd1,0x60,0x00,0x00,}}, // "yu__"
+ // ASCII-7-bit=164 Latin1=127 UTF8=175 CP1252=120 Latin2=135 CP1251=128 CP1250=179 Latin5=44 ISO-8859-5=96 [top CP1250]
+ {{0x7a,0x61,0x5f,0x5f, 0x05,0xb8,0xa3,0x97,0x42,0xa1,0x11,0x30,0x11,0x4e,0x13,0x3a,0x4f,0x41,0x21,0x42,}}, // "za__"
+ // ASCII-7-bit=184 Latin1=163 UTF8=151 GB=66 CP1252=161 SJS=48 BIG5=78 CP1251=58 CP1256=79 CP1250=65 ISO-8859-15=66 [top ASCII-7-bit]
+ {{0x7a,0x6d,0x5f,0x5f, 0x03,0xb8,0x8e,0x9c,0x11,0xa9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zm__"
+ // ASCII-7-bit=184 Latin1=142 UTF8=156 CP1252=169 [top ASCII-7-bit]
+ {{0x7a,0x77,0x5f,0x5f, 0x05,0xbb,0x95,0x9b,0x59,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zw__"
+ // ASCII-7-bit=187 Latin1=149 UTF8=155 GB=89 CP1252=154 [top ASCII-7-bit]
+};
+
+static const int kTLDHintProbsSize = 247;
+
+static const HintEntry kCharsetHintProbs[] = { // MaxRange 192
+ {{0x5f,0x5f,0x5f,0x5f,0x30,0x36,0x34,0x36, 0x02,0xbd,0x7f,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____0646"
+ // ASCII-7-bit=189 Latin1=127 CP1252=149 [top ASCII-7-bit]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x30, 0x01,0x96,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1250"
+ // ASCII-7-bit=150 CP1250=190 [top CP1250]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x7a,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1251"
+ // ASCII-7-bit=122 CP1251=190 [top CP1251]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x32, 0x02,0x99,0x9d,0x21,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1252"
+ // ASCII-7-bit=153 Latin1=157 CP1252=188 [top CP1252]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x33, 0x01,0x79,0x10,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1253"
+ // ASCII-7-bit=121 CP1253=190 [top CP1253]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x34, 0x01,0x71,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1254"
+ // ASCII-7-bit=113 Latin5=175 CP1254=185 [top CP1254]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x35, 0x01,0x86,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1255"
+ // ASCII-7-bit=134 CP1255=190 [top CP1255]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x36, 0x01,0x78,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1256"
+ // ASCII-7-bit=120 CP1256=190 [top CP1256]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x37, 0x01,0x79,0xf1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1257"
+ // ASCII-7-bit=121 CP1257=190 [top CP1257]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x38,0x30,0x30, 0x10,0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1800"
+ // KOI8R=191 [top KOI8R]
+ {{0x5f,0x5f,0x5f,0x5f,0x33,0x36,0x30,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____3600"
+ // CP1250=191 [top CP1250]
+ {{0x5f,0x5f,0x5f,0x5f,0x33,0x36,0x39,0x39, 0x01,0xad,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____3699"
+ // ASCII-7-bit=173 UTF8=185 [top UTF8]
+ {{0x5f,0x5f,0x5f,0x5f,0x34,0x34,0x30,0x30, 0x02,0xbc,0x87,0x21,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____4400"
+ // ASCII-7-bit=188 Latin1=135 CP1252=164 [top ASCII-7-bit]
+ {{0x5f,0x5f,0x5f,0x5f,0x35,0x30,0x30,0x31, 0x01,0x9a,0x11,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____5001"
+ // ASCII-7-bit=154 UTF8=189 [top UTF8]
+ {{0x5f,0x5f,0x5f,0x5f,0x35,0x39,0x31,0x35, 0x02,0xa8,0xa6,0x21,0xa7,0xa1,0xb2,0x00,0x00,0x00,0x00,0x00,}}, // "____5915"
+ // ASCII-7-bit=168 Latin1=166 CP1252=167 ISO-8859-15=178 [top ISO-8859-15]
+ {{0x5f,0x5f,0x5f,0x5f,0x36,0x34,0x36,0x5f, 0x02,0xbe,0x8e,0x21,0x81,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____646_"
+ // ASCII-7-bit=190 Latin1=142 CP1252=129 [top ASCII-7-bit]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0xae,0xb8,0x21,0x94,0xa1,0x2f,0x00,0x00,0x00,0x00,0x00,}}, // "____8591"
+ // ASCII-7-bit=174 Latin1=184 CP1252=148 ISO-8859-15=47 [top Latin1]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x32, 0x01,0x8c,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8592"
+ // ASCII-7-bit=140 Latin2=190 [top Latin2]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x34, 0x10,0xe1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8594"
+ // Latin4=191 [top Latin4]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x35, 0x01,0x6f,0x10,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8595"
+ // ASCII-7-bit=111 ISO-8859-5=190 [top ISO-8859-5]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x37, 0x10,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8597"
+ // Greek=191 [top Greek]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x39, 0x01,0x72,0xc1,0xbe,0x81,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8599"
+ // ASCII-7-bit=114 Latin5=190 CP1254=123 [top Latin5]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x38,0x36,0x31, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8861"
+ // Latin2=191 [top Latin2]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x5f,0x5f,0x5f, 0x03,0x98,0x5d,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8___"
+ // ASCII-7-bit=152 Latin1=93 UTF8=189 [top UTF8]
+ {{0x5f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x07,0xb1,0xaa,0x9c,0x95,0x96,0x8e,0x8c,0x11,0x82,0x00,0x00,}}, // "________"
+ // ASCII-7-bit=177 Latin1=170 UTF8=156 GB=149 CP1252=150 KSC=142 SJS=140 BIG5=130 [top ASCII-7-bit]
+ {{0x61,0x6e,0x73,0x69,0x33,0x34,0x5f,0x5f, 0x02,0xbe,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ansi34__"
+ // ASCII-7-bit=190 Latin1=110 [top ASCII-7-bit]
+ {{0x61,0x6e,0x73,0x69,0x5f,0x5f,0x5f,0x5f, 0x02,0xa2,0xb9,0x21,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ansi____"
+ // ASCII-7-bit=162 Latin1=185 CP1252=164 [top Latin1]
+ {{0x61,0x72,0x72,0x61,0x5f,0x5f,0x5f,0x5f, 0x01,0x90,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "arra____"
+ // ASCII-7-bit=144 CP1256=190 [top CP1256]
+ {{0x61,0x73,0x63,0x69,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x72,0x21,0x71,0xa1,0x53,0x00,0x00,0x00,0x00,0x00,}}, // "asci____"
+ // ASCII-7-bit=190 Latin1=114 CP1252=113 ISO-8859-15=83 [top ASCII-7-bit]
+ {{0x61,0x75,0x74,0x6f,0x5f,0x5f,0x5f,0x5f, 0x01,0x9b,0x51,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "auto____"
+ // ASCII-7-bit=155 SJS=189 [top SJS]
+ {{0x62,0x67,0x5f,0x5f,0x32,0x33,0x31,0x32, 0x01,0x93,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bg__2312"
+ // ASCII-7-bit=147 GB=190 [top GB]
+ {{0x62,0x68,0x61,0x73,0x5f,0x5f,0x5f,0x5f, 0x30,0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bhas____"
+ // BHASKAR=191 [top BHASKAR]
+ {{0x62,0x69,0x67,0x5f,0x35,0x5f,0x5f,0x5f, 0x01,0x84,0x71,0xbe,0x10,0xa1,0x2f,0x00,0x00,0x00,0x00,0x00,}}, // "big_5___"
+ // ASCII-7-bit=132 BIG5=190 BIG5_HKSCS=47 [top BIG5]
+ {{0x62,0x69,0x67,0x5f,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "big_8591"
+ // Latin1=191 [top Latin1]
+ {{0x62,0x69,0x67,0x68,0x35,0x5f,0x5f,0x5f, 0x01,0x88,0x71,0xae,0x10,0xa1,0xb8,0x00,0x00,0x00,0x00,0x00,}}, // "bigh5___"
+ // ASCII-7-bit=136 BIG5=174 BIG5_HKSCS=184 [top BIG5_HKSCS]
+ {{0x62,0x69,0x6e,0x61,0x5f,0x5f,0x5f,0x5f, 0x30,0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bina____"
+ // X-BINARYENC=191 [top X-BINARYENC]
+ {{0x62,0x6f,0x74,0x5f,0x5f,0x5f,0x5f,0x5f, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bot_____"
+ // Latin5=191 [top Latin5]
+ {{0x62,0x73,0x5f,0x5f,0x34,0x37,0x33,0x30, 0x02,0xb8,0xa8,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bs__4730"
+ // ASCII-7-bit=184 Latin1=168 CP1252=163 [top ASCII-7-bit]
+ {{0x63,0x68,0x61,0x72,0x5f,0x5f,0x5f,0x5f, 0x02,0xa5,0xbb,0x21,0x91,0xa1,0x28,0x00,0x00,0x00,0x00,0x00,}}, // "char____"
+ // ASCII-7-bit=165 Latin1=187 CP1252=145 ISO-8859-15=40 [top Latin1]
+ {{0x63,0x6e,0x73,0x5f,0x5f,0x5f,0x5f,0x5f, 0x30,0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cns_____"
+ // CNS=191 [top CNS]
+ {{0x63,0x6f,0x6e,0x66,0x5f,0x5f,0x5f,0x5f, 0x01,0x9f,0x11,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "conf____"
+ // ASCII-7-bit=159 UTF8=189 [top UTF8]
+ {{0x63,0x6f,0x6e,0x74,0x5f,0x5f,0x5f,0x5f, 0x01,0xa4,0x11,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cont____"
+ // ASCII-7-bit=164 UTF8=188 [top UTF8]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x30, 0x01,0x97,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1250"
+ // ASCII-7-bit=151 CP1250=190 [top CP1250]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x7c,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1251"
+ // ASCII-7-bit=124 CP1251=190 [top CP1251]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x32, 0x02,0xab,0xa9,0x21,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1252"
+ // ASCII-7-bit=171 Latin1=169 CP1252=181 [top CP1252]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x33, 0x01,0x79,0x10,0x31,0x7f,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1253"
+ // ASCII-7-bit=121 Greek=127 CP1253=190 [top CP1253]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x34, 0x01,0x5b,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1254"
+ // ASCII-7-bit=91 Latin5=175 CP1254=185 [top CP1254]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x35, 0x01,0x86,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1255"
+ // ASCII-7-bit=134 CP1255=190 [top CP1255]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x36, 0x01,0x5e,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1256"
+ // ASCII-7-bit=94 CP1256=190 [top CP1256]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x37, 0x01,0xa8,0xf1,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1257"
+ // ASCII-7-bit=168 CP1257=187 [top CP1257]
+ {{0x63,0x70,0x5f,0x5f,0x38,0x35,0x30,0x5f, 0x02,0x97,0x98,0x21,0x8c,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,}}, // "cp__850_"
+ // ASCII-7-bit=151 Latin1=152 CP1252=140 ISO-8859-15=188 [top ISO-8859-15]
+ {{0x63,0x70,0x5f,0x5f,0x38,0x35,0x32,0x5f, 0x01,0x8f,0x20,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__852_"
+ // ASCII-7-bit=143 CP852=190 [top CP852]
+ {{0x63,0x70,0x5f,0x5f,0x38,0x36,0x36,0x5f, 0x01,0xa2,0x20,0x31,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__866_"
+ // ASCII-7-bit=162 CP866=188 [top CP866]
+ {{0x63,0x70,0x63,0x5f,0x39,0x34,0x33,0x5f, 0x01,0x26,0x51,0xbe,0x10,0x11,0x68,0x00,0x00,0x00,0x00,0x00,}}, // "cpc_943_"
+ // ASCII-7-bit=38 SJS=190 CP932=104 [top SJS]
+ {{0x63,0x70,0x63,0x7a,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cpcz1250"
+ // CP1250=191 [top CP1250]
+ {{0x63,0x73,0x69,0x73,0x5f,0x5f,0x5f,0x5f, 0x01,0x9c,0x10,0x01,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "csis____"
+ // ASCII-7-bit=156 CP1255=189 [top CP1255]
+ {{0x63,0x73,0x6e,0x5f,0x39,0x31,0x30,0x33, 0x20,0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "csn_9103"
+ // CSN_369103=191 [top CSN_369103]
+ {{0x63,0x73,0x73,0x68,0x5f,0x5f,0x5f,0x5f, 0x01,0x7f,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cssh____"
+ // ASCII-7-bit=127 SJS=190 [top SJS]
+ {{0x63,0x73,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cswi1250"
+ // CP1250=191 [top CP1250]
+ {{0x63,0x73,0x77,0x69,0x33,0x31,0x5f,0x5f, 0x61,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cswi31__"
+ // SJS=191 [top SJS]
+ {{0x63,0x7a,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "czwi1250"
+ // CP1250=191 [top CP1250]
+ {{0x64,0x61,0x64,0x6b,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dadk8591"
+ // Latin1=190 CP1252=125 [top Latin1]
+ {{0x64,0x61,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dais8591"
+ // ASCII-7-bit=111 Latin1=190 CP1252=111 [top Latin1]
+ {{0x64,0x65,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0x9d,0xbc,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "de______"
+ // ASCII-7-bit=157 Latin1=188 CP1252=149 [top Latin1]
+ {{0x64,0x65,0x61,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0x8f,0xbd,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deas____"
+ // ASCII-7-bit=143 Latin1=189 CP1252=146 [top Latin1]
+ {{0x64,0x65,0x64,0x65,0x38,0x35,0x39,0x31, 0x02,0x92,0xbe,0x21,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dede8591"
+ // ASCII-7-bit=146 Latin1=190 CP1252=135 [top Latin1]
+ {{0x64,0x65,0x66,0x61,0x5f,0x5f,0x5f,0x5f, 0x02,0xbc,0x9f,0x21,0x89,0xa1,0x6b,0x00,0x00,0x00,0x00,0x00,}}, // "defa____"
+ // ASCII-7-bit=188 Latin1=159 CP1252=137 ISO-8859-15=107 [top ASCII-7-bit]
+ {{0x64,0x65,0x69,0x73,0x35,0x39,0x31,0x35, 0x11,0x83,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deis5915"
+ // Latin1=131 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x64,0x65,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x92,0xbd,0x21,0x89,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deis8591"
+ // ASCII-7-bit=146 Latin1=189 CP1252=137 [top Latin1]
+ {{0x64,0x65,0x6c,0x65,0x5f,0x5f,0x5f,0x5f, 0x02,0xa9,0xba,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dele____"
+ // ASCII-7-bit=169 Latin1=186 CP1252=146 [top Latin1]
+ {{0x64,0x65,0x75,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0x74,0xb8,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deut____"
+ // ASCII-7-bit=116 Latin1=184 CP1252=175 [top Latin1]
+ {{0x64,0x6f,0x6f,0x72,0x31,0x32,0x35,0x32, 0x11,0x79,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "door1252"
+ // Latin1=121 CP1252=190 [top CP1252]
+ {{0x65,0x63,0x75,0x6a,0x5f,0x5f,0x5f,0x5f, 0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ecuj____"
+ // EUC-JP=191 [top EUC-JP]
+ {{0x65,0x63,0x75,0x6b,0x5f,0x5f,0x5f,0x5f, 0x01,0x71,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ecuk____"
+ // ASCII-7-bit=113 KSC=190 [top KSC]
+ {{0x65,0x65,0x6d,0x73,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eems1250"
+ // CP1250=191 [top CP1250]
+ {{0x65,0x6e,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "en__8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x65,0x6e,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x92,0x21,0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "en______"
+ // ASCII-7-bit=190 Latin1=146 CP1252=130 [top ASCII-7-bit]
+ {{0x65,0x6e,0x63,0x6f,0x5f,0x5f,0x5f,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enco____"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x65,0x6e,0x67,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x8b,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eng_____"
+ // ASCII-7-bit=139 BIG5=190 [top BIG5]
+ {{0x65,0x6e,0x67,0x62,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "engb____"
+ // ASCII-7-bit=190 Latin1=125 [top ASCII-7-bit]
+ {{0x65,0x6e,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x96,0xbc,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enis8591"
+ // ASCII-7-bit=150 Latin1=188 CP1252=154 [top Latin1]
+ {{0x65,0x6e,0x75,0x6b,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enuk8591"
+ // Latin1=191 [top Latin1]
+ {{0x65,0x6e,0x75,0x6b,0x5f,0x5f,0x5f,0x5f, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enuk____"
+ // KSC=191 [top KSC]
+ {{0x65,0x6e,0x75,0x73,0x35,0x39,0x31,0x35, 0x02,0x6f,0x7f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enus5915"
+ // ASCII-7-bit=111 Latin1=127 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x65,0x6e,0x75,0x73,0x38,0x35,0x39,0x31, 0x02,0x9c,0xbc,0x21,0x9b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enus8591"
+ // ASCII-7-bit=156 Latin1=188 CP1252=155 [top Latin1]
+ {{0x65,0x6e,0x75,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0xbb,0xa1,0x21,0x9e,0xa1,0x68,0x00,0x00,0x00,0x00,0x00,}}, // "enus____"
+ // ASCII-7-bit=187 Latin1=161 CP1252=158 ISO-8859-15=104 [top ASCII-7-bit]
+ {{0x65,0x6e,0x75,0x74,0x38,0x5f,0x5f,0x5f, 0x01,0x81,0xf1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enut8___"
+ // ASCII-7-bit=129 CP1257=190 [top CP1257]
+ {{0x65,0x73,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xb4,0xb3,0x21,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "es______"
+ // ASCII-7-bit=180 Latin1=179 CP1252=157 [top ASCII-7-bit]
+ {{0x65,0x73,0x65,0x73,0x38,0x35,0x39,0x31, 0x02,0x82,0xbe,0x21,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eses8591"
+ // ASCII-7-bit=130 Latin1=190 CP1252=110 [top Latin1]
+ {{0x65,0x73,0x65,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0xa6,0xba,0x21,0x96,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eses____"
+ // ASCII-7-bit=166 Latin1=186 CP1252=150 [top Latin1]
+ {{0x65,0x73,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "esis8591"
+ // Latin1=190 CP1252=135 [top Latin1]
+ {{0x65,0x74,0x65,0x65,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "etee8591"
+ // Latin1=191 [top Latin1]
+ {{0x65,0x74,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "etis8591"
+ // Latin1=191 [top Latin1]
+ {{0x65,0x75,0x63,0x5f,0x32,0x5f,0x5f,0x5f, 0x01,0xbe,0x31,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "euc_2___"
+ // ASCII-7-bit=190 CP1252=114 [top ASCII-7-bit]
+ {{0x65,0x75,0x63,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x7d,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "euc_____"
+ // ASCII-7-bit=125 EUC-JP=190 [top EUC-JP]
+ {{0x65,0x75,0x63,0x63,0x5f,0x5f,0x5f,0x5f, 0x01,0x6f,0x30,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucc____"
+ // ASCII-7-bit=111 EUC-CN=190 [top EUC-CN]
+ {{0x65,0x75,0x63,0x64,0x5f,0x5f,0x5f,0x5f, 0x30,0x61,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucd____"
+ // EUC=191 [top EUC]
+ {{0x65,0x75,0x63,0x6a,0x5f,0x5f,0x5f,0x5f, 0x01,0x68,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucj____"
+ // ASCII-7-bit=104 EUC-JP=190 [top EUC-JP]
+ {{0x65,0x75,0x63,0x6b,0x5f,0x5f,0x5f,0x5f, 0x01,0x6d,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "euck____"
+ // ASCII-7-bit=109 KSC=190 [top KSC]
+ {{0x65,0x75,0x63,0x75,0x5f,0x5f,0x5f,0x5f, 0x01,0x6d,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucu____"
+ // ASCII-7-bit=109 KSC=190 [top KSC]
+ {{0x65,0x75,0x6b,0x6b,0x5f,0x5f,0x5f,0x5f, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eukk____"
+ // KSC=191 [top KSC]
+ {{0x65,0x75,0x72,0x6b,0x5f,0x5f,0x5f,0x5f, 0x01,0x71,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eurk____"
+ // ASCII-7-bit=113 KSC=190 [top KSC]
+ {{0x66,0x65,0x61,0x74,0x5f,0x5f,0x5f,0x5f, 0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "feat____"
+ // CP1252=191 [top CP1252]
+ {{0x66,0x66,0x5f,0x5f,0x30,0x5f,0x5f,0x5f, 0x02,0x9e,0xba,0x21,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ff__0___"
+ // ASCII-7-bit=158 Latin1=186 CP1252=165 [top Latin1]
+ {{0x66,0x69,0x66,0x69,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fifi8591"
+ // Latin1=191 [top Latin1]
+ {{0x66,0x72,0x66,0x72,0x38,0x35,0x39,0x31, 0x02,0x79,0xbc,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frfr8591"
+ // ASCII-7-bit=121 Latin1=188 CP1252=163 [top Latin1]
+ {{0x66,0x72,0x66,0x72,0x38,0x5f,0x5f,0x5f, 0x02,0xa6,0xad,0x21,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frfr8___"
+ // ASCII-7-bit=166 Latin1=173 CP1252=181 [top CP1252]
+ {{0x66,0x72,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x80,0xbd,0x21,0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fris8591"
+ // ASCII-7-bit=128 Latin1=189 CP1252=158 [top Latin1]
+ {{0x66,0x72,0x75,0x74,0x38,0x5f,0x5f,0x5f, 0x02,0x8c,0xb3,0x21,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frut8___"
+ // ASCII-7-bit=140 Latin1=179 CP1252=181 [top CP1252]
+ {{0x67,0x62,0x5f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x6f,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__1251"
+ // ASCII-7-bit=111 CP1251=190 [top CP1251]
+ {{0x67,0x62,0x5f,0x5f,0x32,0x31,0x33,0x32, 0x01,0x91,0x21,0xbe,0xf1,0x70,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__2132"
+ // ASCII-7-bit=145 GB=190 GBK=112 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x32,0x33,0x31,0x32, 0x01,0x7a,0x21,0xbe,0xf1,0x5c,0xc1,0x37,0x00,0x00,0x00,0x00,}}, // "gb__2312"
+ // ASCII-7-bit=122 GB=190 GBK=92 GB18030=55 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x32,0x33,0x32,0x31, 0x01,0x7d,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__2321"
+ // ASCII-7-bit=125 GB=190 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x33,0x32,0x31,0x32, 0x01,0x92,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__3212"
+ // ASCII-7-bit=146 GB=190 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x38,0x30,0x33,0x30, 0x01,0x73,0x21,0xaf,0xf1,0x59,0xc1,0xb9,0x00,0x00,0x00,0x00,}}, // "gb__8030"
+ // ASCII-7-bit=115 GB=175 GBK=89 GB18030=185 [top GB18030]
+ {{0x67,0x62,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x7f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__8591"
+ // ASCII-7-bit=127 Latin1=190 [top Latin1]
+ {{0x67,0x62,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x71,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb______"
+ // ASCII-7-bit=113 GB=190 [top GB]
+ {{0x67,0x62,0x6b,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x76,0x21,0xaf,0xf1,0xb9,0xc1,0x13,0x00,0x00,0x00,0x00,}}, // "gbk_____"
+ // ASCII-7-bit=118 GB=175 GBK=185 GB18030=19 [top GBK]
+ {{0x67,0x64,0x5f,0x5f,0x32,0x33,0x31,0x32, 0x01,0x56,0x21,0xbe,0xf1,0x72,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gd__2312"
+ // ASCII-7-bit=86 GB=190 GBK=114 [top GB]
+ {{0x67,0x65,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "geis8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x67,0x65,0x6e,0x65,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gene1251"
+ // CP1251=191 [top CP1251]
+ {{0x67,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "giso8591"
+ // Latin1=190 CP1252=111 [top Latin1]
+ {{0x67,0x72,0x65,0x65,0x5f,0x5f,0x5f,0x5f, 0x01,0x90,0x10,0x31,0xbe,0x21,0x86,0x00,0x00,0x00,0x00,0x00,}}, // "gree____"
+ // ASCII-7-bit=144 Greek=190 CP1253=134 [top Greek]
+ {{0x68,0x72,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hrwi1250"
+ // CP1250=191 [top CP1250]
+ {{0x68,0x74,0x63,0x68,0x5f,0x5f,0x5f,0x5f, 0x30,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "htch____"
+ // HTCHANAKYA=191 [top HTCHANAKYA]
+ {{0x68,0x74,0x6d,0x6c,0x5f,0x5f,0x5f,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "html____"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x68,0x74,0x74,0x70,0x5f,0x5f,0x5f,0x5f, 0x02,0xbb,0xa4,0x21,0x8d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "http____"
+ // ASCII-7-bit=187 Latin1=164 CP1252=141 [top ASCII-7-bit]
+ {{0x68,0x7a,0x67,0x62,0x32,0x33,0x31,0x32, 0x01,0x85,0x20,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hzgb2312"
+ // ASCII-7-bit=133 HZ-GB-2312=190 [top HZ-GB-2312]
+ {{0x69,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "i___8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x61,0x6e,0x6f,0x35,0x5f,0x5f,0x5f, 0x02,0xbe,0x61,0x21,0x54,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iano5___"
+ // ASCII-7-bit=190 Latin1=97 CP1252=84 [top ASCII-7-bit]
+ {{0x69,0x62,0x6d,0x5f,0x38,0x35,0x32,0x5f, 0x01,0xac,0x20,0x01,0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ibm_852_"
+ // ASCII-7-bit=172 CP852=186 [top CP852]
+ {{0x69,0x62,0x6d,0x5f,0x38,0x36,0x36,0x5f, 0x01,0x84,0x20,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ibm_866_"
+ // ASCII-7-bit=132 CP866=190 [top CP866]
+ {{0x69,0x62,0x6d,0x5f,0x39,0x34,0x32,0x5f, 0x61,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ibm_942_"
+ // SJS=191 [top SJS]
+ {{0x69,0x63,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbb,0x21,0xa9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ico_8591"
+ // ASCII-7-bit=121 Latin1=187 CP1252=169 [top Latin1]
+ {{0x69,0x6e,0x64,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "indo1251"
+ // CP1251=191 [top CP1251]
+ {{0x69,0x6e,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "inso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x6f,0x73,0x5f,0x38,0x35,0x39,0x31, 0x02,0x97,0xbd,0x21,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ios_8591"
+ // ASCII-7-bit=151 Latin1=189 CP1252=110 [top Latin1]
+ {{0x69,0x6f,0x73,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ioso8591"
+ // Latin1=190 CP1252=121 [top Latin1]
+ {{0x69,0x73,0x5f,0x5f,0x35,0x39,0x31,0x35, 0x11,0x7f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__5915"
+ // Latin1=127 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0xad,0xb7,0x21,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8591"
+ // ASCII-7-bit=173 Latin1=183 CP1252=159 [top Latin1]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x32, 0x01,0x78,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8592"
+ // ASCII-7-bit=120 Latin2=190 [top Latin2]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x37, 0x10,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8597"
+ // Greek=191 [top Greek]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x38, 0x01,0x6f,0x10,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8598"
+ // ASCII-7-bit=111 Hebrew=190 [top Hebrew]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x39, 0xd1,0xbe,0x81,0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8599"
+ // Latin5=190 CP1254=136 [top Latin5]
+ {{0x69,0x73,0x61,0x5f,0x35,0x39,0x31,0x35, 0x02,0x86,0x89,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isa_5915"
+ // ASCII-7-bit=134 Latin1=137 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x64,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isd_8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x64,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isdo8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6e,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isn_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x30,0x36,0x34,0x36, 0x02,0xb8,0xaa,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_0646"
+ // ASCII-7-bit=184 Latin1=170 CP1252=163 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x30,0x34,0x30, 0x02,0x98,0xb2,0x21,0xb4,0xa1,0x5e,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1040"
+ // ASCII-7-bit=152 Latin1=178 CP1252=180 ISO-8859-15=94 [top CP1252]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x30, 0x01,0x90,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1250"
+ // ASCII-7-bit=144 CP1250=190 [top CP1250]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x78,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1251"
+ // ASCII-7-bit=120 CP1251=190 [top CP1251]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x32, 0x02,0xad,0x9e,0x21,0xb7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1252"
+ // ASCII-7-bit=173 Latin1=158 CP1252=183 [top CP1252]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x33, 0x10,0x41,0x83,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1253"
+ // Greek=131 CP1253=190 [top CP1253]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x34, 0xd1,0x9b,0x81,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1254"
+ // Latin5=155 CP1254=189 [top CP1254]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x35, 0x01,0x79,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1255"
+ // ASCII-7-bit=121 CP1255=190 [top CP1255]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x36, 0x01,0x6f,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1256"
+ // ASCII-7-bit=111 CP1256=190 [top CP1256]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x37, 0x01,0x7f,0xf1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1257"
+ // ASCII-7-bit=127 CP1257=190 [top CP1257]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x5f,0x5f,0x5f, 0x02,0x85,0xb5,0x21,0xb3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1___"
+ // ASCII-7-bit=133 Latin1=181 CP1252=179 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x32,0x30,0x32,0x32, 0x01,0x97,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_2022"
+ // ASCII-7-bit=151 SJS=190 [top SJS]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x35,0x39,0x31, 0x02,0xa9,0xb8,0x21,0xa4,0xa1,0x3c,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5591"
+ // ASCII-7-bit=169 Latin1=184 CP1252=164 ISO-8859-15=60 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x35,0x39,0x32, 0x02,0x9a,0xbd,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5592"
+ // ASCII-7-bit=154 Latin1=189 CP1252=146 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x38,0x39,0x31, 0x02,0xa1,0xbc,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5891"
+ // ASCII-7-bit=161 Latin1=188 CP1252=139 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x30, 0x01,0xaa,0x20,0xa1,0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5910"
+ // ASCII-7-bit=170 Latin6=186 [top Latin6]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x31, 0x01,0x86,0xd1,0xbe,0xd1,0x66,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5911"
+ // ASCII-7-bit=134 ISO-8859-11=190 CP874=102 [top ISO-8859-11]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x33, 0x01,0x9c,0xf1,0xa1,0xc1,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5913"
+ // ASCII-7-bit=156 CP1257=161 ISO-8859-13=187 [top ISO-8859-13]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x34, 0x02,0x93,0xbd,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5914"
+ // ASCII-7-bit=147 Latin1=189 CP1252=149 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x35, 0x02,0x98,0xad,0x21,0x81,0xa1,0xb7,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5915"
+ // ASCII-7-bit=152 Latin1=173 CP1252=129 ISO-8859-15=183 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x36, 0x01,0xae,0xb1,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5916"
+ // ASCII-7-bit=174 CP1250=185 [top CP1250]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x32,0x32, 0x01,0xa7,0x81,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5922"
+ // ASCII-7-bit=167 Latin2=187 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x36,0x33,0x39,0x32, 0x02,0x7e,0xbe,0x21,0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_6392"
+ // ASCII-7-bit=126 Latin1=190 CP1252=130 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x36,0x33,0x39,0x5f, 0x01,0xa6,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_639_"
+ // ASCII-7-bit=166 CP1256=188 [top CP1256]
+ {{0x69,0x73,0x6f,0x5f,0x36,0x34,0x36,0x31, 0x02,0x7d,0xbe,0x21,0x68,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_6461"
+ // ASCII-7-bit=125 Latin1=190 CP1252=104 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x31,0x31, 0x02,0xb0,0xb7,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8511"
+ // ASCII-7-bit=176 Latin1=183 CP1252=146 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x36,0x31, 0x02,0x9f,0xba,0x21,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8561"
+ // ASCII-7-bit=159 Latin1=186 CP1252=165 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x38,0x31, 0x01,0x8d,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8581"
+ // ASCII-7-bit=141 SJS=190 [top SJS]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x30, 0x02,0x99,0xbc,0x21,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8590"
+ // ASCII-7-bit=153 Latin1=188 CP1252=156 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0xae,0xb8,0x21,0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8591"
+ // ASCII-7-bit=174 Latin1=184 CP1252=153 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x32, 0x01,0x95,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8592"
+ // ASCII-7-bit=149 Latin2=190 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x33, 0x01,0x9f,0x20,0x51,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8593"
+ // ASCII-7-bit=159 Latin3=189 [top Latin3]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x34, 0x01,0xac,0x10,0xd1,0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8594"
+ // ASCII-7-bit=172 Latin4=186 [top Latin4]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x35, 0x01,0xa6,0x10,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8595"
+ // ASCII-7-bit=166 ISO-8859-5=188 [top ISO-8859-5]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x36, 0x01,0xae,0x20,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8596"
+ // ASCII-7-bit=174 Arabic=185 [top Arabic]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x37, 0x01,0x96,0x10,0x31,0xbd,0x21,0x8f,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8597"
+ // ASCII-7-bit=150 Greek=189 CP1253=143 [top Greek]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x38, 0x01,0x9b,0x10,0x81,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8598"
+ // ASCII-7-bit=155 Hebrew=189 [top Hebrew]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x39, 0x01,0x7a,0xc1,0xbe,0x81,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8599"
+ // ASCII-7-bit=122 Latin5=190 CP1254=126 [top Latin5]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x36,0x30,0x31, 0x02,0xba,0x94,0x21,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8601"
+ // ASCII-7-bit=186 Latin1=148 CP1252=166 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x36,0x39,0x31, 0x02,0xad,0xb9,0x21,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8691"
+ // ASCII-7-bit=173 Latin1=185 CP1252=131 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x36,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8692"
+ // Latin2=191 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x38,0x35,0x31, 0x02,0xac,0xb7,0x21,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8851"
+ // ASCII-7-bit=172 Latin1=183 CP1252=159 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x38,0x35,0x39, 0x02,0xaa,0xba,0x21,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8859"
+ // ASCII-7-bit=170 Latin1=186 CP1252=128 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x38,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8899"
+ // Latin5=191 [top Latin5]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x39,0x31,0x31, 0x02,0x8c,0xbd,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8911"
+ // ASCII-7-bit=140 Latin1=189 CP1252=154 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x39,0x31,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_891_"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x39,0x35,0x31, 0x02,0xa3,0xbc,0x21,0x91,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8951"
+ // ASCII-7-bit=163 Latin1=188 CP1252=145 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x30,0x30,0x31, 0x02,0x75,0xa7,0x21,0x85,0xa1,0xbb,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9001"
+ // ASCII-7-bit=117 Latin1=167 CP1252=133 ISO-8859-15=187 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x35,0x31, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9551"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x39,0x31, 0x02,0x72,0xbe,0x21,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9591"
+ // ASCII-7-bit=114 Latin1=190 CP1252=123 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x39,0x32, 0x01,0x7f,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9592"
+ // ASCII-7-bit=127 Latin2=190 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x39,0x39, 0x01,0x84,0xc1,0xbb,0x81,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9599"
+ // ASCII-7-bit=132 Latin5=187 CP1254=166 [top Latin5]
+ {{0x69,0x73,0x6f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0x99,0xbc,0x21,0x96,0xa1,0x2e,0x00,0x00,0x00,0x00,0x00,}}, // "iso_____"
+ // ASCII-7-bit=153 Latin1=188 CP1252=150 ISO-8859-15=46 [top Latin1]
+ {{0x69,0x73,0x6f,0x61,0x38,0x35,0x39,0x31, 0x02,0x9a,0xbd,0x21,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoa8591"
+ // ASCII-7-bit=154 Latin1=189 CP1252=138 [top Latin1]
+ {{0x69,0x73,0x6f,0x62,0x38,0x35,0x39,0x31, 0x02,0x86,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isob8591"
+ // ASCII-7-bit=134 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x63,0x32,0x30,0x32,0x32, 0x20,0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoc2022"
+ // ISO_2022_CN=191 [top ISO_2022_CN]
+ {{0x69,0x73,0x6f,0x63,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoc8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x63,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoc8592"
+ // Latin2=191 [top Latin2]
+ {{0x69,0x73,0x6f,0x64,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isod8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x65,0x38,0x35,0x39,0x31, 0x02,0x93,0xbd,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoe8591"
+ // ASCII-7-bit=147 Latin1=189 CP1252=139 [top Latin1]
+ {{0x69,0x73,0x6f,0x66,0x35,0x39,0x31,0x35, 0x11,0x6f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isof5915"
+ // Latin1=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x68,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoh8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x69,0x36,0x5f,0x5f,0x5f, 0x02,0xbe,0x92,0x21,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi6___"
+ // ASCII-7-bit=190 Latin1=146 CP1252=127 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x31, 0x02,0xa2,0xbc,0x21,0x8c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8591"
+ // ASCII-7-bit=162 Latin1=188 CP1252=140 [top Latin1]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8592"
+ // Latin2=191 [top Latin2]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x35, 0x01,0xa4,0x10,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8595"
+ // ASCII-7-bit=164 ISO-8859-5=188 [top ISO-8859-5]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x36, 0x01,0x79,0x20,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8596"
+ // ASCII-7-bit=121 Arabic=190 [top Arabic]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x38, 0x01,0x83,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8598"
+ // ASCII-7-bit=131 CP1255=190 [top CP1255]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8599"
+ // Latin5=191 [top Latin5]
+ {{0x69,0x73,0x6f,0x69,0x38,0x38,0x35,0x39, 0x02,0xae,0xb7,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8859"
+ // ASCII-7-bit=174 Latin1=183 CP1252=154 [top Latin1]
+ {{0x69,0x73,0x6f,0x69,0x38,0x39,0x5f,0x5f, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi89__"
+ // CP1256=191 [top CP1256]
+ {{0x69,0x73,0x6f,0x6a,0x32,0x30,0x30,0x32, 0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoj2002"
+ // EUC-JP=191 [top EUC-JP]
+ {{0x69,0x73,0x6f,0x6a,0x32,0x30,0x32,0x32, 0x01,0x44,0x10,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoj2022"
+ // ASCII-7-bit=68 JIS=190 [top JIS]
+ {{0x69,0x73,0x6f,0x6a,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoj8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6b,0x32,0x30,0x30,0x32, 0x01,0x7a,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isok2002"
+ // ASCII-7-bit=122 KSC=190 [top KSC]
+ {{0x69,0x73,0x6f,0x6b,0x32,0x30,0x32,0x32, 0x20,0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isok2022"
+ // ISO-2022-KR=191 [top ISO-2022-KR]
+ {{0x69,0x73,0x6f,0x6c,0x31,0x5f,0x5f,0x5f, 0x01,0xa6,0x11,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol1___"
+ // ASCII-7-bit=166 UTF8=188 [top UTF8]
+ {{0x69,0x73,0x6f,0x6c,0x35,0x39,0x31,0x31, 0x01,0x83,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol5911"
+ // ASCII-7-bit=131 ISO-8859-11=190 [top ISO-8859-11]
+ {{0x69,0x73,0x6f,0x6c,0x37,0x5f,0x5f,0x5f, 0x02,0xa4,0xb8,0x21,0xa7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol7___"
+ // ASCII-7-bit=164 Latin1=184 CP1252=167 [top Latin1]
+ {{0x69,0x73,0x6f,0x6c,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6d,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isom8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6e,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ison8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6e,0xbe,0x21,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoo8591"
+ // ASCII-7-bit=110 Latin1=190 CP1252=110 [top Latin1]
+ {{0x69,0x73,0x6f,0x70,0x35,0x39,0x31,0x35, 0xf1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isop5915"
+ // ISO-8859-15=191 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x70,0x38,0x35,0x39,0x31, 0x02,0x91,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isop8591"
+ // ASCII-7-bit=145 Latin1=190 CP1252=111 [top Latin1]
+ {{0x69,0x73,0x6f,0x73,0x38,0x35,0x39,0x31, 0x02,0x84,0xbe,0x21,0x8d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isos8591"
+ // ASCII-7-bit=132 Latin1=190 CP1252=141 [top Latin1]
+ {{0x69,0x73,0x6f,0x75,0x36,0x34,0x36,0x31, 0x02,0xa6,0xb9,0x21,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou6461"
+ // ASCII-7-bit=166 Latin1=185 CP1252=161 [top Latin1]
+ {{0x69,0x73,0x6f,0x75,0x36,0x34,0x36,0x5f, 0x01,0xbe,0x31,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou646_"
+ // ASCII-7-bit=190 CP1252=142 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x75,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x75,0x38,0x5f,0x5f,0x5f, 0x02,0xa2,0xbc,0x21,0x8c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou8___"
+ // ASCII-7-bit=162 Latin1=188 CP1252=140 [top Latin1]
+ {{0x69,0x73,0x6f,0x77,0x31,0x32,0x35,0x30, 0x01,0x6e,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow1250"
+ // ASCII-7-bit=110 CP1250=190 [top CP1250]
+ {{0x69,0x73,0x6f,0x77,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow1251"
+ // CP1251=191 [top CP1251]
+ {{0x69,0x73,0x6f,0x77,0x31,0x32,0x35,0x33, 0x01,0x6f,0x10,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow1253"
+ // ASCII-7-bit=111 CP1253=190 [top CP1253]
+ {{0x69,0x73,0x6f,0x77,0x38,0x35,0x39,0x31, 0x02,0x89,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow8591"
+ // ASCII-7-bit=137 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x78,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isox8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x7a,0x38,0x35,0x39,0x31, 0x02,0x8b,0xbe,0x21,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoz8591"
+ // ASCII-7-bit=139 Latin1=190 CP1252=121 [top Latin1]
+ {{0x69,0x73,0x70,0x5f,0x38,0x35,0x39,0x31, 0x02,0x86,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isp_8591"
+ // ASCII-7-bit=134 Latin1=190 CP1252=111 [top Latin1]
+ {{0x69,0x73,0x73,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iss_8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x74,0x5f,0x35,0x39,0x31,0x35, 0x01,0x79,0xe1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ist_5915"
+ // ASCII-7-bit=121 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x74,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isto8591"
+ // Latin1=190 CP1252=111 [top Latin1]
+ {{0x69,0x74,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x21,0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itis8591"
+ // ASCII-7-bit=111 Latin1=190 CP1252=134 [top Latin1]
+ {{0x69,0x74,0x69,0x74,0x35,0x39,0x31,0x35, 0x41,0x79,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itit5915"
+ // CP1252=121 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x74,0x69,0x74,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x8f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itit8591"
+ // Latin1=190 CP1252=143 [top Latin1]
+ {{0x69,0x74,0x69,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xb7,0xab,0x21,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itit____"
+ // ASCII-7-bit=183 Latin1=171 CP1252=164 [top ASCII-7-bit]
+ {{0x69,0x75,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iu__8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x77,0x69,0x6e,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iwin1250"
+ // CP1250=191 [top CP1250]
+ {{0x69,0x77,0x69,0x6e,0x31,0x32,0x35,0x37, 0x10,0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iwin1257"
+ // CP1257=191 [top CP1257]
+ {{0x69,0x79,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iyso8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x6a,0x61,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x78,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ja______"
+ // ASCII-7-bit=120 SJS=190 [top SJS]
+ {{0x6a,0x61,0x67,0x72,0x5f,0x5f,0x5f,0x5f, 0x20,0xf1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jagr____"
+ // JAGRAN=191 [top JAGRAN]
+ {{0x6a,0x69,0x73,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x81,0x10,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jis_____"
+ // ASCII-7-bit=129 JIS=190 [top JIS]
+ {{0x6b,0x61,0x6d,0x63,0x5f,0x5f,0x5f,0x5f, 0x20,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kamc____"
+ // CP852=191 [top CP852]
+ {{0x6b,0x6f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x7c,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ko______"
+ // ASCII-7-bit=124 KSC=190 [top KSC]
+ {{0x6b,0x6f,0x69,0x5f,0x37,0x5f,0x5f,0x5f, 0x01,0xbe,0x31,0x6b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koi_7___"
+ // ASCII-7-bit=190 CP1252=107 [top ASCII-7-bit]
+ {{0x6b,0x6f,0x69,0x72,0x38,0x5f,0x5f,0x5f, 0x01,0x8b,0x10,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koir8___"
+ // ASCII-7-bit=139 KOI8R=190 [top KOI8R]
+ {{0x6b,0x6f,0x69,0x75,0x38,0x5f,0x5f,0x5f, 0x01,0x77,0x10,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koiu8___"
+ // ASCII-7-bit=119 KOI8U=190 [top KOI8U]
+ {{0x6b,0x6f,0x6b,0x72,0x5f,0x5f,0x5f,0x5f, 0x01,0x4b,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kokr____"
+ // ASCII-7-bit=75 KSC=190 [top KSC]
+ {{0x6b,0x6f,0x6b,0x73,0x35,0x36,0x30,0x31, 0x01,0x75,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koks5601"
+ // ASCII-7-bit=117 KSC=190 [top KSC]
+ {{0x6b,0x6f,0x72,0x65,0x5f,0x5f,0x5f,0x5f, 0x01,0x4e,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kore____"
+ // ASCII-7-bit=78 KSC=190 [top KSC]
+ {{0x6b,0x72,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x74,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kr______"
+ // ASCII-7-bit=116 KSC=190 [top KSC]
+ {{0x6b,0x72,0x63,0x5f,0x35,0x36,0x30,0x31, 0x01,0x74,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "krc_5601"
+ // ASCII-7-bit=116 KSC=190 [top KSC]
+ {{0x6b,0x73,0x63,0x5f,0x35,0x35,0x30,0x31, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ksc_5501"
+ // KSC=191 [top KSC]
+ {{0x6b,0x73,0x63,0x5f,0x35,0x36,0x30,0x31, 0x01,0x62,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ksc_5601"
+ // ASCII-7-bit=98 KSC=190 [top KSC]
+ {{0x6b,0x73,0x63,0x5f,0x36,0x30,0x30,0x31, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ksc_6001"
+ // KSC=191 [top KSC]
+ {{0x6c,0x61,0x73,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xb7,0xaf,0x21,0x90,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "last____"
+ // ASCII-7-bit=183 Latin1=175 CP1252=144 [top ASCII-7-bit]
+ {{0x6c,0x61,0x74,0x69,0x31,0x5f,0x5f,0x5f, 0x02,0xa3,0xbb,0x21,0x9b,0xa1,0x73,0x00,0x00,0x00,0x00,0x00,}}, // "lati1___"
+ // ASCII-7-bit=163 Latin1=187 CP1252=155 ISO-8859-15=115 [top Latin1]
+ {{0x6c,0x61,0x74,0x69,0x32,0x5f,0x5f,0x5f, 0x01,0x94,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati2___"
+ // ASCII-7-bit=148 Latin2=190 [top Latin2]
+ {{0x6c,0x61,0x74,0x69,0x35,0x5f,0x5f,0x5f, 0x01,0x7c,0xc1,0xbe,0x81,0x87,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati5___"
+ // ASCII-7-bit=124 Latin5=190 CP1254=135 [top Latin5]
+ {{0x6c,0x61,0x74,0x69,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati8591"
+ // Latin1=191 [top Latin1]
+ {{0x6c,0x61,0x74,0x69,0x38,0x38,0x35,0x39, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati8859"
+ // Latin2=191 [top Latin2]
+ {{0x6c,0x69,0x6e,0x75,0x31,0x32,0x35,0x32, 0x11,0x79,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "linu1252"
+ // Latin1=121 CP1252=190 [top CP1252]
+ {{0x6c,0x6f,0x67,0x69,0x5f,0x5f,0x5f,0x5f, 0x01,0x88,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "logi____"
+ // ASCII-7-bit=136 UTF8=190 [top UTF8]
+ {{0x6c,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lso_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6c,0x74,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lto_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6c,0x74,0x77,0x69,0x31,0x32,0x35,0x37, 0x10,0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ltwi1257"
+ // CP1257=191 [top CP1257]
+ {{0x6d,0x61,0x63,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x82,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mac_____"
+ // ASCII-7-bit=130 CP1251=190 [top CP1251]
+ {{0x6d,0x61,0x63,0x63,0x5f,0x5f,0x5f,0x5f, 0x01,0x94,0x10,0xe1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "macc____"
+ // ASCII-7-bit=148 MACINTOSH=190 [top MACINTOSH]
+ {{0x6d,0x61,0x63,0x69,0x5f,0x5f,0x5f,0x5f, 0x02,0xbd,0x93,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "maci____"
+ // ASCII-7-bit=189 Latin1=147 CP1252=139 [top ASCII-7-bit]
+ {{0x6d,0x61,0x63,0x72,0x5f,0x5f,0x5f,0x5f, 0x01,0xaf,0x10,0xe1,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "macr____"
+ // ASCII-7-bit=175 MACINTOSH=185 [top MACINTOSH]
+ {{0x6d,0x73,0x5f,0x5f,0x38,0x37,0x34,0x5f, 0x01,0x80,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ms__874_"
+ // ASCII-7-bit=128 ISO-8859-11=190 [top ISO-8859-11]
+ {{0x6d,0x73,0x5f,0x5f,0x39,0x33,0x32,0x5f, 0x01,0x91,0x51,0xbe,0x10,0x11,0x82,0x00,0x00,0x00,0x00,0x00,}}, // "ms__932_"
+ // ASCII-7-bit=145 SJS=190 CP932=130 [top SJS]
+ {{0x6d,0x73,0x5f,0x5f,0x39,0x34,0x39,0x5f, 0x01,0x49,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ms__949_"
+ // ASCII-7-bit=73 KSC=190 [top KSC]
+ {{0x6d,0x73,0x5f,0x5f,0x39,0x35,0x30,0x5f, 0x01,0x75,0x71,0xbe,0x10,0xa1,0x43,0x00,0x00,0x00,0x00,0x00,}}, // "ms__950_"
+ // ASCII-7-bit=117 BIG5=190 BIG5_HKSCS=67 [top BIG5]
+ {{0x6d,0x73,0x63,0x70,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mscp1250"
+ // CP1250=191 [top CP1250]
+ {{0x6d,0x73,0x68,0x6b,0x39,0x35,0x30,0x5f, 0x01,0x82,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mshk950_"
+ // ASCII-7-bit=130 BIG5=190 [top BIG5]
+ {{0x6d,0x73,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mswi1250"
+ // CP1250=191 [top CP1250]
+ {{0x6d,0x73,0x77,0x69,0x31,0x32,0x35,0x33, 0x10,0x41,0x8f,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mswi1253"
+ // Greek=143 CP1253=190 [top CP1253]
+ {{0x6d,0x78,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mx______"
+ // UTF8=191 [top UTF8]
+ {{0x6e,0x65,0x77,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xab,0xb2,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "new_____"
+ // ASCII-7-bit=171 Latin1=178 CP1252=175 [top Latin1]
+ {{0x6e,0x66,0x7a,0x5f,0x32,0x30,0x31,0x30, 0x02,0x80,0xbc,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nfz_2010"
+ // ASCII-7-bit=128 Latin1=188 CP1252=163 [top Latin1]
+ {{0x6e,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "niso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6e,0x6c,0x61,0x69,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nlai8591"
+ // Latin1=191 [top Latin1]
+ {{0x6e,0x6c,0x6e,0x6c,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nlnl8591"
+ // Latin1=190 CP1252=111 [top Latin1]
+ {{0x6e,0x6f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0xa4,0x71,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "no______"
+ // ASCII-7-bit=164 BIG5=188 [top BIG5]
+ {{0x6e,0x6f,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nois8591"
+ // Latin1=191 [top Latin1]
+ {{0x6e,0x6f,0x6e,0x65,0x5f,0x5f,0x5f,0x5f, 0x01,0x9b,0x51,0xbd,0x10,0x11,0x70,0x00,0x00,0x00,0x00,0x00,}}, // "none____"
+ // ASCII-7-bit=155 SJS=189 CP932=112 [top SJS]
+ {{0x6e,0x75,0x6c,0x6c,0x5f,0x5f,0x5f,0x5f, 0x01,0x92,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "null____"
+ // ASCII-7-bit=146 BIG5=190 [top BIG5]
+ {{0x6f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "o___8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6f,0x6e,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "on______"
+ // UTF8=191 [top UTF8]
+ {{0x6f,0x73,0x69,0x5f,0x35,0x39,0x31,0x35, 0x01,0x6f,0xe1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "osi_5915"
+ // ASCII-7-bit=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x6f,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "oso_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6f,0x73,0x70,0x5f,0x38,0x35,0x39,0x38, 0x01,0x6f,0x10,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "osp_8598"
+ // ASCII-7-bit=111 Hebrew=190 [top Hebrew]
+ {{0x6f,0x77,0x69,0x6e,0x31,0x32,0x35,0x36, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "owin1256"
+ // CP1256=191 [top CP1256]
+ {{0x70,0x61,0x72,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0x6e,0xb8,0x21,0xaf,0xa1,0x64,0x00,0x00,0x00,0x00,0x00,}}, // "par_____"
+ // ASCII-7-bit=110 Latin1=184 CP1252=175 ISO-8859-15=100 [top Latin1]
+ {{0x70,0x63,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pc______"
+ // UTF8=191 [top UTF8]
+ {{0x70,0x6c,0x69,0x73,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "plis8592"
+ // Latin2=191 [top Latin2]
+ {{0x70,0x6c,0x70,0x6c,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "plpl8592"
+ // Latin2=191 [top Latin2]
+ {{0x70,0x72,0x65,0x64,0x5f,0x5f,0x5f,0x5f, 0x02,0xb4,0xa3,0x21,0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pred____"
+ // ASCII-7-bit=180 Latin1=163 CP1252=177 [top ASCII-7-bit]
+ {{0x70,0x74,0x62,0x72,0x38,0x35,0x39,0x31, 0x02,0x6e,0xbd,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptbr8591"
+ // ASCII-7-bit=110 Latin1=189 CP1252=154 [top Latin1]
+ {{0x70,0x74,0x62,0x72,0x5f,0x5f,0x5f,0x5f, 0x01,0x79,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptbr____"
+ // ASCII-7-bit=121 UTF8=190 [top UTF8]
+ {{0x70,0x74,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptis8591"
+ // Latin1=190 CP1252=126 [top Latin1]
+ {{0x70,0x74,0x70,0x74,0x35,0x39,0x31,0x35, 0x11,0x89,0x21,0x6f,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptpt5915"
+ // Latin1=137 CP1252=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x72,0x66,0x63,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x87,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rfc_____"
+ // ASCII-7-bit=135 UTF8=190 [top UTF8]
+ {{0x72,0x6f,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x83,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rois8591"
+ // ASCII-7-bit=131 Latin1=190 [top Latin1]
+ {{0x72,0x6f,0x72,0x6f,0x38,0x35,0x39,0x32, 0x01,0x99,0x81,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "roro8592"
+ // ASCII-7-bit=153 Latin2=189 [top Latin2]
+ {{0x72,0x75,0x72,0x75,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ruru1251"
+ // CP1251=191 [top CP1251]
+ {{0x72,0x75,0x77,0x69,0x31,0x32,0x35,0x31, 0x01,0x6f,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ruwi1251"
+ // ASCII-7-bit=111 CP1251=190 [top CP1251]
+ {{0x73,0x65,0x65,0x6d,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "seem8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x73,0x65,0x74,0x63,0x5f,0x5f,0x5f,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "setc____"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x73,0x68,0x69,0x66,0x31,0x32,0x35,0x32, 0x02,0x86,0x6f,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "shif1252"
+ // ASCII-7-bit=134 Latin1=111 CP1252=190 [top CP1252]
+ {{0x73,0x68,0x69,0x66,0x5f,0x5f,0x5f,0x5f, 0x01,0x6e,0x51,0xbe,0x10,0x11,0x6b,0x00,0x00,0x00,0x00,0x00,}}, // "shif____"
+ // ASCII-7-bit=110 SJS=190 CP932=107 [top SJS]
+ {{0x73,0x69,0x66,0x74,0x5f,0x5f,0x5f,0x5f, 0x01,0x72,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sift____"
+ // ASCII-7-bit=114 SJS=190 [top SJS]
+ {{0x73,0x6a,0x69,0x73,0x5f,0x5f,0x5f,0x5f, 0x01,0x79,0x51,0xbe,0x10,0x11,0x5d,0x00,0x00,0x00,0x00,0x00,}}, // "sjis____"
+ // ASCII-7-bit=121 SJS=190 CP932=93 [top SJS]
+ {{0x73,0x6b,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "skwi1250"
+ // CP1250=191 [top CP1250]
+ {{0x73,0x6f,0x5f,0x5f,0x35,0x39,0x31,0x35, 0x02,0x86,0x6f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "so__5915"
+ // ASCII-7-bit=134 Latin1=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x73,0x6f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x9a,0xbd,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "so__8591"
+ // ASCII-7-bit=154 Latin1=189 CP1252=139 [top Latin1]
+ {{0x73,0x6f,0x5f,0x5f,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "so__8592"
+ // Latin2=191 [top Latin2]
+ {{0x73,0x76,0x73,0x65,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "svse8591"
+ // Latin1=191 [top Latin1]
+ {{0x74,0x61,0x62,0x5f,0x5f,0x5f,0x5f,0x5f, 0x30,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tab_____"
+ // TAB=191 [top TAB]
+ {{0x74,0x61,0x6d,0x5f,0x5f,0x5f,0x5f,0x5f, 0x30,0x31,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tam_____"
+ // TAM=191 [top TAM]
+ {{0x74,0x65,0x78,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xac,0xb7,0x21,0xa0,0xa1,0x49,0x00,0x00,0x00,0x00,0x00,}}, // "text____"
+ // ASCII-7-bit=172 Latin1=183 CP1252=160 ISO-8859-15=73 [top Latin1]
+ {{0x74,0x69,0x73,0x5f,0x36,0x31,0x38,0x5f, 0x01,0x75,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tis_618_"
+ // ASCII-7-bit=117 ISO-8859-11=190 [top ISO-8859-11]
+ {{0x74,0x69,0x73,0x5f,0x36,0x32,0x30,0x5f, 0x01,0x82,0xd1,0xbe,0xd1,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tis_620_"
+ // ASCII-7-bit=130 ISO-8859-11=190 CP874=123 [top ISO-8859-11]
+ {{0x74,0x72,0x5f,0x5f,0x38,0x35,0x39,0x39, 0xd1,0xbe,0x81,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tr__8599"
+ // Latin5=190 CP1254=111 [top Latin5]
+ {{0x74,0x72,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0xd1,0xbe,0x81,0x5f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tr______"
+ // Latin5=190 CP1254=95 [top Latin5]
+ {{0x74,0x72,0x69,0x73,0x38,0x35,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tris8599"
+ // Latin5=191 [top Latin5]
+ {{0x74,0x73,0x63,0x69,0x5f,0x5f,0x5f,0x5f, 0x30,0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tsci____"
+ // TSCII=191 [top TSCII]
+ {{0x75,0x63,0x73,0x5f,0x32,0x5f,0x5f,0x5f, 0x02,0xb8,0xa7,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ucs_2___"
+ // ASCII-7-bit=184 Latin1=167 CP1252=163 [top ASCII-7-bit]
+ {{0x75,0x66,0x74,0x5f,0x38,0x5f,0x5f,0x5f, 0x01,0xb0,0x11,0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uft_8___"
+ // ASCII-7-bit=176 UTF8=184 [top UTF8]
+ {{0x75,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uiso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x75,0x6e,0x69,0x63,0x31,0x31,0x5f,0x5f, 0x01,0xa7,0x11,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "unic11__"
+ // ASCII-7-bit=167 UTF8=187 [top UTF8]
+ {{0x75,0x6e,0x69,0x63,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x90,0x21,0x85,0xa1,0x45,0x00,0x00,0x00,0x00,0x00,}}, // "unic____"
+ // ASCII-7-bit=190 Latin1=144 CP1252=133 ISO-8859-15=69 [top ASCII-7-bit]
+ {{0x75,0x6e,0x6b,0x6e,0x38,0x5f,0x5f,0x5f, 0x02,0xa2,0xbb,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "unkn8___"
+ // ASCII-7-bit=162 Latin1=187 CP1252=149 [top Latin1]
+ {{0x75,0x6e,0x6b,0x6e,0x5f,0x5f,0x5f,0x5f, 0x01,0x9c,0x51,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "unkn____"
+ // ASCII-7-bit=156 SJS=189 [top SJS]
+ {{0x75,0x70,0x66,0x5f,0x38,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "upf_8___"
+ // UTF8=191 [top UTF8]
+ {{0x75,0x73,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x7d,0x21,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "us______"
+ // ASCII-7-bit=190 Latin1=125 CP1252=125 [top ASCII-7-bit]
+ {{0x75,0x73,0x61,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x83,0x21,0x6a,0xa1,0x38,0x00,0x00,0x00,0x00,0x00,}}, // "usas____"
+ // ASCII-7-bit=190 Latin1=131 CP1252=106 ISO-8859-15=56 [top ASCII-7-bit]
+ {{0x75,0x73,0x65,0x6e,0x5f,0x5f,0x5f,0x5f, 0x02,0xb8,0x94,0x21,0xad,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "usen____"
+ // ASCII-7-bit=184 Latin1=148 CP1252=173 [top ASCII-7-bit]
+ {{0x75,0x73,0x65,0x72,0x5f,0x5f,0x5f,0x5f, 0x02,0xb9,0x9e,0x21,0xa7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "user____"
+ // ASCII-7-bit=185 Latin1=158 CP1252=167 [top ASCII-7-bit]
+ {{0x75,0x73,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x78,0xbe,0x21,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "usis8591"
+ // ASCII-7-bit=120 Latin1=190 CP1252=120 [top Latin1]
+ {{0x75,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbc,0x21,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uso_8591"
+ // ASCII-7-bit=121 Latin1=188 CP1252=165 [top Latin1]
+ {{0x75,0x74,0x66,0x5f,0x31,0x36,0x5f,0x5f, 0x01,0xb0,0x11,0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_16__"
+ // ASCII-7-bit=176 UTF8=184 [top UTF8]
+ {{0x75,0x74,0x66,0x5f,0x33,0x32,0x5f,0x5f, 0x02,0xb5,0xa9,0x21,0x9f,0xa1,0xa1,0x00,0x00,0x00,0x00,0x00,}}, // "utf_32__"
+ // ASCII-7-bit=181 Latin1=169 CP1252=159 ISO-8859-15=161 [top ASCII-7-bit]
+ {{0x75,0x74,0x66,0x5f,0x35,0x39,0x31,0x35, 0x11,0x90,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_5915"
+ // Latin1=144 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x75,0x74,0x66,0x5f,0x37,0x5f,0x5f,0x5f, 0x01,0x88,0x20,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_7___"
+ // ASCII-7-bit=136 UTF7=190 [top UTF7]
+ {{0x75,0x74,0x66,0x5f,0x38,0x35,0x39,0x31, 0x02,0x95,0xbd,0x21,0x8c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_8591"
+ // ASCII-7-bit=149 Latin1=189 CP1252=140 [top Latin1]
+ {{0x75,0x74,0x66,0x5f,0x38,0x35,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_8599"
+ // Latin5=191 [top Latin5]
+ {{0x75,0x74,0x66,0x5f,0x38,0x5f,0x5f,0x5f, 0x01,0xae,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_8___"
+ // ASCII-7-bit=174 UTF8=185 [top UTF8]
+ {{0x75,0x74,0x66,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x8a,0x21,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_____"
+ // ASCII-7-bit=190 Latin1=138 CP1252=116 [top ASCII-7-bit]
+ {{0x75,0x74,0x66,0x62,0x31,0x36,0x5f,0x5f, 0x01,0xa5,0x20,0x41,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfb16__"
+ // ASCII-7-bit=165 UTF-16BE=188 [top UTF-16BE]
+ {{0x75,0x74,0x66,0x62,0x33,0x32,0x5f,0x5f, 0x30,0x81,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfb32__"
+ // UTF-32BE=191 [top UTF-32BE]
+ {{0x75,0x74,0x66,0x69,0x38,0x35,0x39,0x31, 0x02,0x99,0xbd,0x21,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfi8591"
+ // ASCII-7-bit=153 Latin1=189 CP1252=135 [top Latin1]
+ {{0x75,0x74,0x66,0x6c,0x31,0x36,0x5f,0x5f, 0x20,0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfl16__"
+ // UTF-16LE=191 [top UTF-16LE]
+ {{0x75,0x74,0x66,0x6c,0x33,0x32,0x5f,0x5f, 0x30,0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfl32__"
+ // UTF-32LE=191 [top UTF-32LE]
+ {{0x75,0x74,0x66,0x75,0x38,0x38,0x5f,0x5f, 0x30,0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfu88__"
+ // X-UTF8UTF8=191 [top X-UTF8UTF8]
+ {{0x76,0x61,0x6c,0x75,0x5f,0x5f,0x5f,0x5f, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "valu____"
+ // Latin1=191 [top Latin1]
+ {{0x76,0x69,0x73,0x75,0x5f,0x5f,0x5f,0x5f, 0x01,0x84,0x10,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "visu____"
+ // ASCII-7-bit=132 Hebrew=190 [top Hebrew]
+ {{0x77,0x61,0x69,0x6e,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wain1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x65,0x69,0x73,0x35,0x39,0x31,0x35, 0x02,0x9f,0x7d,0x21,0x84,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,}}, // "weis5915"
+ // ASCII-7-bit=159 Latin1=125 CP1252=132 ISO-8859-15=188 [top ISO-8859-15]
+ {{0x77,0x65,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "weis8591"
+ // Latin1=190 CP1252=126 [top Latin1]
+ {{0x77,0x65,0x73,0x74,0x31,0x32,0x35,0x32, 0x01,0x6f,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "west1252"
+ // ASCII-7-bit=111 CP1252=190 [top CP1252]
+ {{0x77,0x65,0x73,0x74,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "west8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x77,0x65,0x73,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xa9,0x9d,0x21,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "west____"
+ // ASCII-7-bit=169 Latin1=157 CP1252=185 [top CP1252]
+ {{0x77,0x69,0x64,0x6e,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "widn1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x30, 0x01,0x7c,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1250"
+ // ASCII-7-bit=124 CP1250=190 [top CP1250]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x32, 0x11,0xa9,0x21,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1252"
+ // Latin1=169 CP1252=187 [top CP1252]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x36, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1256"
+ // CP1256=191 [top CP1256]
+ {{0x77,0x69,0x6d,0x64,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wimd1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x30, 0x01,0x8d,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1250"
+ // ASCII-7-bit=141 CP1250=190 [top CP1250]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x31, 0x01,0x8f,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1251"
+ // ASCII-7-bit=143 CP1251=190 [top CP1251]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x32, 0x02,0xac,0xa4,0x21,0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1252"
+ // ASCII-7-bit=172 Latin1=164 CP1252=182 [top CP1252]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x33, 0x10,0x41,0x85,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1253"
+ // Greek=133 CP1253=190 [top CP1253]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x34, 0x01,0x6f,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1254"
+ // ASCII-7-bit=111 Latin5=175 CP1254=185 [top CP1254]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x35, 0x10,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1255"
+ // CP1255=191 [top CP1255]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x36, 0x01,0x7f,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1256"
+ // ASCII-7-bit=127 CP1256=190 [top CP1256]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x37, 0x01,0x8c,0xf1,0xbe,0xc1,0x77,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1257"
+ // ASCII-7-bit=140 CP1257=190 ISO-8859-13=119 [top CP1257]
+ {{0x77,0x69,0x6e,0x5f,0x38,0x37,0x34,0x5f, 0x01,0x56,0xd1,0xaf,0xd1,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_874_"
+ // ASCII-7-bit=86 ISO-8859-11=175 CP874=185 [top CP874]
+ {{0x77,0x69,0x6e,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x9a,0x91,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_____"
+ // ASCII-7-bit=154 CP1251=189 [top CP1251]
+ {{0x77,0x69,0x6e,0x63,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "winc1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x69,0x6e,0x63,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "winc1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x33,0x34, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1234"
+ // CP1256=191 [top CP1256]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x30, 0x01,0x88,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1250"
+ // ASCII-7-bit=136 CP1250=190 [top CP1250]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x31, 0x01,0x8b,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1251"
+ // ASCII-7-bit=139 CP1251=190 [top CP1251]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x32, 0x02,0xa5,0xac,0x21,0xb6,0xa1,0x4f,0x00,0x00,0x00,0x00,0x00,}}, // "wind1252"
+ // ASCII-7-bit=165 Latin1=172 CP1252=182 ISO-8859-15=79 [top CP1252]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x33, 0x01,0x94,0x10,0x31,0xae,0x21,0xb8,0x00,0x00,0x00,0x00,0x00,}}, // "wind1253"
+ // ASCII-7-bit=148 Greek=174 CP1253=184 [top CP1253]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x34, 0x01,0x73,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1254"
+ // ASCII-7-bit=115 Latin5=175 CP1254=185 [top CP1254]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x35, 0x01,0x86,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1255"
+ // ASCII-7-bit=134 CP1255=190 [top CP1255]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x36, 0x01,0x74,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1256"
+ // ASCII-7-bit=116 CP1256=190 [top CP1256]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x37, 0x01,0x87,0xf1,0xbe,0xc1,0x52,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1257"
+ // ASCII-7-bit=135 CP1257=190 ISO-8859-13=82 [top CP1257]
+ {{0x77,0x69,0x6e,0x64,0x33,0x31,0x5f,0x5f, 0x01,0x62,0x51,0xbe,0x10,0x11,0x5e,0x00,0x00,0x00,0x00,0x00,}}, // "wind31__"
+ // ASCII-7-bit=98 SJS=190 CP932=94 [top SJS]
+ {{0x77,0x69,0x6e,0x64,0x38,0x34,0x37,0x5f, 0xe1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind847_"
+ // ISO-8859-11=191 [top ISO-8859-11]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x32,0x5f, 0x01,0x79,0x20,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind852_"
+ // ASCII-7-bit=121 CP852=190 [top CP852]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x31, 0x02,0x9a,0xbd,0x21,0x89,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8591"
+ // ASCII-7-bit=154 Latin1=189 CP1252=137 [top Latin1]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x32, 0x01,0x83,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8592"
+ // ASCII-7-bit=131 Latin2=190 [top Latin2]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x36, 0x01,0x6f,0x20,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8596"
+ // ASCII-7-bit=111 Arabic=190 [top Arabic]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x37, 0x01,0x6f,0x10,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8597"
+ // ASCII-7-bit=111 Greek=190 [top Greek]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x39, 0x01,0x6c,0xc1,0xbe,0x81,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8599"
+ // ASCII-7-bit=108 Latin5=190 CP1254=108 [top Latin5]
+ {{0x77,0x69,0x6e,0x64,0x38,0x36,0x36,0x5f, 0x20,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind866_"
+ // CP866=191 [top CP866]
+ {{0x77,0x69,0x6e,0x64,0x38,0x37,0x34,0x5f, 0x01,0x8a,0xd1,0xbe,0xd1,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind874_"
+ // ASCII-7-bit=138 ISO-8859-11=190 CP874=125 [top ISO-8859-11]
+ {{0x77,0x69,0x6e,0x64,0x38,0x38,0x35,0x39, 0x02,0x97,0xb6,0x21,0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8859"
+ // ASCII-7-bit=151 Latin1=182 CP1252=177 [top Latin1]
+ {{0x77,0x69,0x6e,0x64,0x38,0x5f,0x5f,0x5f, 0x01,0x93,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8___"
+ // ASCII-7-bit=147 UTF8=190 [top UTF8]
+ {{0x77,0x69,0x6e,0x64,0x39,0x33,0x32,0x5f, 0x01,0x7d,0x51,0xa4,0x10,0x11,0xbc,0x00,0x00,0x00,0x00,0x00,}}, // "wind932_"
+ // ASCII-7-bit=125 SJS=164 CP932=188 [top CP932]
+ {{0x77,0x69,0x6e,0x64,0x39,0x34,0x39,0x5f, 0x01,0x7b,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind949_"
+ // ASCII-7-bit=123 KSC=190 [top KSC]
+ {{0x77,0x69,0x6e,0x64,0x39,0x35,0x30,0x5f, 0x01,0x6f,0x71,0x7f,0x20,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,}}, // "wind950_"
+ // ASCII-7-bit=111 BIG5=127 BIG5-CP950=190 [top BIG5-CP950]
+ {{0x77,0x69,0x6e,0x64,0x5f,0x5f,0x5f,0x5f, 0x01,0xb5,0x11,0xb4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind____"
+ // ASCII-7-bit=181 UTF8=180 [top ASCII-7-bit]
+ {{0x77,0x69,0x6e,0x65,0x31,0x32,0x35,0x32, 0x01,0x6f,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wine1252"
+ // ASCII-7-bit=111 CP1252=190 [top CP1252]
+ {{0x77,0x69,0x6e,0x6f,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wino1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x69,0x6e,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wino1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x6e,0x73,0x31,0x32,0x35,0x35, 0x10,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wins1255"
+ // CP1255=191 [top CP1255]
+ {{0x77,0x69,0x72,0x64,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wird1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wiso8591"
+ // Latin1=190 CP1252=127 [top Latin1]
+ {{0x77,0x6e,0x64,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wndo1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x6e,0x64,0x6f,0x31,0x32,0x35,0x36, 0x01,0x6e,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wndo1256"
+ // ASCII-7-bit=110 CP1256=190 [top CP1256]
+ {{0x77,0x6f,0x6e,0x64,0x31,0x32,0x35,0x32, 0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wond1252"
+ // CP1252=191 [top CP1252]
+ {{0x77,0x6f,0x72,0x67,0x31,0x32,0x35,0x32, 0x01,0x83,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "worg1252"
+ // ASCII-7-bit=131 CP1252=190 [top CP1252]
+ {{0x79,0x65,0x73,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x81,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yes_____"
+ // ASCII-7-bit=190 Latin1=129 CP1252=139 [top ASCII-7-bit]
+ {{0x79,0x6b,0x74,0x63,0x5f,0x5f,0x5f,0x5f, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yktc____"
+ // KSC=191 [top KSC]
+ {{0x7a,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zso_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+};
+
+static const int kCharsetHintProbsSize = 438;
+
+static const uint8 kDefaultProb[NUM_RANKEDENCODING] = { // MaxRange 192
+177, 170, 156, 149, 150, 142, 140, 124, 130, 127, 124, 118, 127, 118, 109, 104, 98, 93, 96, 82, 84, 81, 80, 64, 61, 57, 53, 47, 42, 28, 24, 22,
+ 17, 0, 5, 1, 5, 12, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, };
+
+static const int kMaxTldKey = 4;
+static const int kMaxTldVector = 16;
+static const int kMaxCharsetKey = 8;
+static const int kMaxCharsetVector = 12;
+static const int kMaxLangKey = 8;
+static const int kMaxLangVector = 12;
+// Smoothing percentage across encodings with same UTF-8 result: 100%
+
+static const UnigramEntry unigram_table[NUM_RANKEDENCODING] = {
+{ // ASCII-7-bit (788.373M chars) [0]
+ {NULL, NULL, NULL, NULL},
+ 77, 207, 29, 27, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,189,189,0,0,189,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // Latin1 (1792.786M chars) [1]
+ {NULL, NULL, ced_hires_13, ced_hires_13, },
+ 87, 217, 37, 20, 128,
+ {1,0,1,1,0,0,1,0, 0,9,9,0,1,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 186,137,105,112,140,106,126,145, 132,113,128,124,123,101,126,119, 141,113,107,116,129,113,113,143, 130,105,127,128,103,116,106,128,
+ 122,138,132,155,161,129,133,190, 124,152,124,119,117,144,120,121, 127,136,122,132,112,139,144,116, 113,101,117,117,145,120,135,114,
+ 121,138,107,125,163,129,131,190, 120,141,112,110,114,143,115,133, 124,135,126,131,118,141,160,110, 114,104,119,99,148,119,129,117,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 195,159,115,166,155,125,133,156, 130,168,156,159,121,135,150,133, 169,125,135,118,150,113,114,173, 126,107,176,178,97,131,103,170,
+ 165,209,166,183,211,198,182,194, 193,219,182,153,130,205,161,148, 160,187,142,208,169,167,210,131, 192,122,189,152,212,181,158,185,
+ 195,214,155,184,212,208,182,181, 183,213,171,154,161,215,150,137, 175,187,162,210,158,168,208,131, 192,156,189,142,199,193,155,120,
+ },
+ {43,13,19,71,0,0,0,0, 0,145,153,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,142,175,92,109,120,168,161, 150,151,135,110,183,159,177,142, 146,163,159,149,146,144,137,136, 133,134,166,128,200,102,117,143,
+ 102,197,188,202,195,190,180,196, 185,179,172,189,202,198,215,195, 185,164,216,206,204,172,190,182, 158,170,181,119,107,136,114,132,
+ 134,194,189,198,192,189,178,194, 182,176,171,189,201,194,213,194, 181,151,213,203,201,171,189,182, 155,176,182,106,136,85,122,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 187,138,112,113,139,107,128,136, 133,137,118,120,118,129,135,123, 139,125,101,126,128,108,123,140, 131,107,125,133,123,119,104,138,
+ 133,128,130,187,160,126,123,145, 123,153,122,122,114,136,125,117, 153,141,127,129,107,167,144,114, 113,105,118,118,134,117,122,164,
+ 130,132,122,187,161,131,122,139, 122,137,119,120,113,138,123,110, 153,141,127,130,133,167,146,104, 117,105,130,109,131,117,133,119,
+ },
+ {128,0,110,142,142,142,142,140, 0,0,142,142,142,142,142,142, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 22,0,2,2,2,2,2,2, 0,0,2,14,2,2,2,2, 46,0,2,14,2,2,2,2, 0,0,2,18,2,4,2,6,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 42,0,2,10,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 18,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 38,0,2,6,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 164,0,136,164,110,108,108,106, 0,0,190,164,136,134,122,152, 158,0,144,156,112,116,106,106, 0,0,166,212,158,164,146,162,
+ 90,0,92,96,139,139,94,92, 0,0,148,166,184,176,134,146, 98,0,96,104,139,139,110,110, 0,0,138,172,174,190,154,164,
+ 130,0,138,126,38,48,135,135, 0,0,130,156,128,148,186,180, 120,0,126,120,48,56,139,141, 0,0,142,156,140,186,166,188,
+ },
+},
+
+{ // UTF8 (16713.069M chars) [2]
+ {NULL, NULL, NULL, NULL},
+ 169, 203, 42, 24, 131,
+ {197,207,201,202,188,181,180,180, 188,183,184,187,188,182,178,183, 181,179,172,178,182,182,183,183, 181,181,183,180,189,181,175,180,
+ 182,183,175,177,184,183,183,184, 187,176,181,181,178,186,187,184, 187,177,175,183,184,176,174,177, 196,183,188,186,194,184,180,181,
+ 0,0,183,211,189,187,147,104, 65,123,87,102,121,13,187,176, 216,203,118,108,88,134,124,193, 206,202,155,161,58,0,119,0,
+ 181,176,180,213,192,206,202,197, 192,187,165,177,183,170,90,182, 71,0,0,13,1,0,0,0, 0,0,0,0,0,0,0,0,
+ 119,115,120,96,114,98,104,106, 121,111,96,110,113,105,109,100, 109,107,92,112,112,97,91,102, 110,119,116,99,117,110,104,106,
+ 133,132,135,130,132,139,129,131, 133,127,121,126,132,128,121,138, 136,128,120,126,145,114,120,123, 132,124,128,122,138,120,117,132,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,64,
+ },
+ {0,0,0,0,0,0,0,0, 0,109,143,0,0,144,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,125,169,73,68,107,151,137, 146,147,111,102,152,138,154,129, 142,152,148,137,131,132,127,127, 123,121,139,118,189,98,96,127,
+ 87,124,119,124,117,115,112,111, 114,127,109,102,114,120,120,114, 120,94,118,122,122,113,106,113, 95,99,88,117,109,125,97,107,
+ 80,139,113,146,112,126,103,99, 115,147,96,97,118,131,156,123, 132,80,111,140,146,136,107,102, 86,116,85,83,110,84,118,0,
+ 204,210,207,204,196,192,190,188, 194,187,191,189,194,188,179,187, 184,183,175,181,186,187,183,187, 184,189,185,186,191,184,178,186,
+ 192,196,179,183,192,185,183,198, 192,196,189,182,182,193,189,192, 199,194,188,195,190,194,185,185, 203,191,196,195,200,196,197,188,
+ 0,0,121,113,72,69,49,4, 0,0,0,71,0,0,113,107, 113,102,0,0,0,0,19,86, 134,132,91,105,0,0,0,0,
+ 174,140,150,207,185,199,194,190, 187,181,160,174,177,157,75,176, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 6,0,2,2,2,6,2,2, 0,0,0,0,50,38,2,128, 4,0,2,2,2,4,2,2, 0,0,0,0,54,40,2,128,
+ 4,0,2,2,2,6,2,2, 0,0,0,0,128,128,128,128, 2,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128, 128,0,118,118,128,128,128,128, 0,0,0,0,128,128,64,128,
+ 176,0,168,172,172,172,162,164, 120,126,126,116,126,140,134,128, 168,0,170,166,172,170,174,176, 118,122,118,116,142,116,138,128,
+ 168,0,170,168,168,172,176,174, 110,120,114,112,136,142,140,128, 168,0,172,170,160,166,160,164, 120,126,118,116,140,134,136,128,
+ 0,0,0,0,0,0,0,0, 116,122,140,128,0,0,0,0, 0,0,0,0,0,0,0,0, 128,120,126,134,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 134,132,120,126,0,0,0,0, 0,0,0,0,0,0,0,0, 116,152,138,116,0,0,0,0,
+ },
+},
+
+{ // GB (9061.562M chars) [3]
+ {NULL, ced_hires_3, ced_hires_4, ced_hires_5, },
+ 204, 189, 27, 16, 128,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,66,73, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 70,204,125,202,219,216,120,114, 118,156,113,85,96,78,81,79, 190,196,199,195,195,208,196,199, 196,203,195,200,201,197,193,195,
+ 193,195,190,195,195,183,196,191, 201,197,207,197,197,198,201,199, 201,191,203,201,198,196,202,200, 134,116,114,132,122,124,118,115,
+ 115,120,116,120,141,121,124,123, 121,120,121,113,125,116,117,110, 117,114,112,113,119,112,124,123, 84,74,83,78,69,83,77,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,93,66,68,84,3,51,0, 116,102,41,61,83,57,66,12, 70,85,73,66,54,64,44,68, 0,3,3,3,54,73,70,54,
+ 95,66,54,44,66,78,26,37, 57,41,3,57,13,2,0,0, 54,51,0,41,68,70,70,59, 41,70,41,13,0,64,59,80,
+ 12,3,51,12,13,26,41,3, 0,79,47,61,2,19,66,64, 37,37,13,93,54,13,54,51, 51,12,59,73,89,54,54,94,
+ },
+ {101,14,0,0,0,0,0,0, 16,16,0,0,68,49,44,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 81,56,73,0,0,0,64,71, 49,49,0,84,53,69,60,32, 84,60,49,35,15,19,18,1, 2,0,15,0,92,29,0,0,
+ 83,69,66,85,103,49,69,51, 56,71,53,83,70,73,65,91, 71,59,61,71,80,77,72,79, 61,69,67,64,66,73,77,73,
+ 70,75,71,70,86,62,64,69, 71,66,39,69,73,64,74,75, 79,69,71,70,83,80,53,92, 69,66,59,69,69,85,90,3,
+ 94,72,77,73,64,67,62,71, 80,76,76,72,65,74,80,73, 77,93,79,99,84,69,81,94, 51,35,67,74,49,67,84,78,
+ 126,196,197,192,196,176,185,184, 195,185,196,194,198,187,184,191, 190,190,180,189,189,194,181,191, 182,191,188,196,189,191,187,192,
+ 186,174,192,193,203,190,193,194, 190,188,189,195,181,187,196,191, 200,178,184,186,189,186,192,180, 193,176,195,182,185,179,182,186,
+ 185,187,190,185,183,187,185,186, 177,190,189,192,187,187,181,180, 178,185,188,196,172,177,186,192, 192,182,197,186,178,187,185,0,
+ },
+ {128,0,128,128,128,128,128,128, 128,128,104,128,114,124,128,118, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 46,0,46,50,32,40,40,34, 128,128,26,28,26,30,32,32, 42,0,42,44,26,34,34,28, 128,128,46,50,46,52,54,54,
+ 44,0,44,46,28,36,36,30, 128,128,48,50,46,52,56,54, 50,0,50,54,34,42,42,38, 128,128,50,52,50,56,58,58,
+ 128,0,128,128,128,128,128,128, 128,128,58,62,58,64,70,66, 128,0,128,128,128,128,128,128, 128,128,40,42,40,44,48,46,
+ 0,0,0,0,154,180,164,168, 188,176,128,106,122,124,134,92, 0,0,0,0,128,128,128,136, 128,128,132,130,130,132,130,130,
+ 0,0,0,0,242,218,218,232, 138,134,137,131,129,117,127,113, 0,0,0,0,202,206,218,216, 110,112,129,127,129,127,127,131,
+ 0,0,0,0,202,218,218,206, 114,106,122,126,126,128,130,132, 0,0,0,0,210,214,206,206, 120,114,124,128,126,136,122,126,
+ 0,0,0,0,206,208,200,202, 218,236,136,124,124,122,126,126, 230,194,200,214,190,198,194,216, 192,186,128,122,132,128,130,120,
+ },
+},
+
+{ // CP1252 (408.280M chars) [4]
+ {NULL, NULL, ced_hires_13, ced_hires_13, },
+ 89, 209, 40, 30, 128,
+ {116,114,130,121,123,133,113,118, 108,96,180,101,111,113,172,102, 63,68,105,82,59,104,73,70, 61,79,141,65,74,59,133,92,
+ 184,136,106,111,139,104,125,143, 132,111,126,123,121,100,125,118, 143,111,106,114,127,111,112,142, 128,104,127,127,102,114,106,126,
+ 120,172,132,158,159,131,136,188, 123,153,123,119,120,159,118,120, 127,136,121,131,136,137,143,117, 112,109,131,116,143,151,134,114,
+ 120,172,130,128,161,130,131,188, 120,143,112,109,119,159,114,131, 122,133,125,130,137,139,158,109, 114,108,131,98,146,151,128,116,
+ 169,126,98,99,143,148,118,105, 94,83,203,88,113,99,199,98, 95,134,176,160,158,170,169,143, 114,141,201,118,111,110,201,130,
+ 193,157,113,163,153,123,131,154, 127,166,153,157,119,132,148,131, 167,123,133,115,148,111,112,171, 124,105,174,176,96,128,101,168,
+ 163,207,164,180,208,196,180,192, 191,217,180,151,128,203,159,146, 158,185,140,206,167,165,207,128, 190,119,187,150,210,179,156,183,
+ 193,212,152,182,209,206,180,179, 181,211,168,151,159,213,148,135, 173,185,160,208,156,166,206,129, 190,153,187,139,197,191,153,118,
+ },
+ {128,43,4,69,19,0,0,0, 0,146,153,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,141,174,92,108,119,170,160, 148,152,134,110,182,158,177,142, 147,162,157,147,145,143,136,135, 132,133,165,129,200,100,121,143,
+ 118,197,187,200,193,200,178,194, 183,191,170,191,201,196,214,194, 184,162,214,205,205,174,188,180, 157,168,179,122,110,135,115,131,
+ 133,196,189,196,191,201,176,192, 180,192,169,191,199,192,212,193, 182,149,211,202,203,175,187,180, 153,174,180,105,136,86,122,30,
+ 113,107,124,132,114,134,112,118, 113,93,171,97,104,105,166,96, 99,108,124,118,117,133,112,109, 102,129,171,117,112,107,166,135,
+ 183,137,111,110,137,105,126,133, 132,134,125,119,116,126,133,122, 144,123,99,123,127,108,120,139, 128,105,124,131,120,117,103,138,
+ 130,152,132,184,157,126,125,142, 124,153,120,120,118,183,123,115, 150,138,125,127,107,164,145,111, 111,103,118,117,132,116,120,161,
+ 130,157,121,184,158,129,123,136, 124,145,121,118,111,183,121,110, 150,139,124,127,130,164,143,101, 116,102,132,106,129,115,130,117,
+ },
+ {32,0,2,24,2,2,2,2, 22,20,16,36,6,24,6,26, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 24,0,2,2,2,2,2,2, 2,2,2,16,2,2,2,4, 46,0,2,16,2,2,2,2, 6,6,2,20,2,6,2,10,
+ 14,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 38,0,2,6,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 14,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 34,0,2,4,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 130,0,106,124,140,126,112,112, 160,134,134,140,180,136,142,134, 134,0,132,126,86,84,140,128, 118,162,132,166,130,138,180,136,
+ 162,0,136,166,110,110,108,106, 118,138,192,162,130,136,116,152, 158,0,144,154,112,118,106,106, 148,164,166,210,152,164,138,164,
+ 88,0,92,96,139,141,94,92, 180,136,146,162,174,174,124,144, 98,0,96,104,139,139,110,112, 184,150,134,164,162,186,142,158,
+ 128,0,138,128,38,48,135,137, 130,178,128,152,118,146,176,178, 120,0,126,120,48,56,139,141, 136,180,140,150,130,184,156,184,
+ },
+},
+
+{ // KSC (5258.976M chars) [5]
+ {NULL, ced_hires_6, ced_hires_7, ced_hires_8, },
+ 203, 186, 27, 9, 128,
+ {71,109,117,106,108,109,104,107, 110,108,108,112,103,104,106,105, 102,108,101,103,104,107,99,103, 104,99,107,103,102,103,107,98,
+ 106,206,164,204,164,121,141,122, 146,119,136,130,120,90,91,81, 216,212,164,201,215,207,187,206, 213,208,207,204,210,209,205,216,
+ 225,214,178,201,188,198,199,213, 199,93,107,104,116,113,115,113, 108,107,109,113,110,97,99,104, 101,113,115,107,110,108,111,117,
+ 110,108,103,116,115,142,146,136, 118,134,125,131,144,112,112,126, 112,117,107,103,107,103,102,104, 101,114,109,104,105,93,0,0,
+ 52,0,0,60,0,1,0,0, 0,12,0,0,28,0,18,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,78,41,83,69,52,75,65, 1,50,60,76,0,62,88,58, 79,72,60,68,0,65,54,44, 72,58,78,86,46,82,73,48,
+ 57,67,58,67,72,57,60,86, 18,31,65,50,69,65,85,79, 52,57,54,34,54,0,41,38, 0,78,82,58,50,78,1,46,
+ 0,57,68,0,72,67,48,65, 48,41,31,107,18,67,63,0, 12,48,0,52,0,28,85,46, 82,67,44,48,48,1,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,44,71,0,0,61,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 117,80,77,0,0,0,66,63, 78,84,0,97,84,72,82,66, 4,74,65,78,51,54,50,9, 24,6,52,5,117,40,0,44,
+ 63,9,0,52,36,0,9,0, 0,0,0,24,48,0,51,51, 60,1,55,58,40,0,7,6, 59,8,0,81,54,73,75,73,
+ 57,2,57,35,48,28,1,2, 0,42,0,6,50,1,20,8, 38,0,0,32,57,0,4,2, 35,32,36,56,57,50,89,38,
+ 66,99,111,102,98,106,96,97, 95,99,98,96,98,104,96,95, 98,97,92,103,92,97,93,96, 95,96,104,97,94,93,92,91,
+ 157,208,188,195,195,184,194,180, 174,186,184,181,197,195,200,187, 193,181,186,192,167,192,186,169, 205,187,203,199,192,172,174,185,
+ 186,179,196,199,181,187,182,201, 179,175,184,177,204,183,204,207, 190,189,185,176,191,179,192,189, 182,200,194,190,178,181,182,187,
+ 177,176,199,166,189,187,186,198, 184,182,174,196,175,194,193,169, 178,186,174,191,161,176,204,191, 198,180,187,178,193,181,149,9,
+ },
+ {128,0,128,128,128,128,128,128, 128,128,80,86,78,94,92,92, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 92,0,42,48,90,68,96,74, 128,128,44,44,42,48,48,50, 58,0,10,16,56,36,58,40, 128,128,42,44,40,46,48,48,
+ 66,0,18,26,66,44,66,50, 128,128,42,44,42,48,48,50, 128,0,128,128,128,128,128,128, 128,128,50,52,48,58,58,58,
+ 128,0,108,108,128,128,128,128, 128,128,40,40,38,44,44,46, 128,0,128,128,128,128,128,128, 128,128,54,56,50,60,58,60,
+ 0,0,0,0,186,204,190,182, 244,244,124,124,72,82,142,64, 0,0,0,0,164,206,174,184, 240,238,122,124,78,80,144,52,
+ 0,0,0,0,186,200,180,188, 142,140,147,131,99,107,101,91, 0,0,0,0,174,196,184,194, 112,114,125,127,127,125,133,129,
+ 0,0,0,0,196,204,194,204, 120,108,124,126,130,132,116,128, 0,0,0,0,0,0,0,0, 0,0,124,130,122,130,126,126,
+ 0,0,0,0,0,0,0,0, 0,0,144,130,102,108,110,110, 0,0,0,0,0,0,0,0, 0,0,126,130,124,124,130,126,
+ },
+},
+
+{ // SJS (6339.756M chars) [6]
+ {NULL, NULL, NULL, NULL},
+ 151, 136, 55, 16, 129,
+ {50,188,238,215,160,59,66,108, 192,200,192,194,200,199,202,200, 190,190,186,192,182,192,188,186, 101,96,109,95,94,95,102,113,
+ 84,100,117,108,117,135,81,77, 99,97,99,87,94,104,96,118, 120,115,111,101,96,105,108,108, 111,98,126,107,129,116,102,97,
+ 121,104,84,106,118,104,105,80, 97,98,110,102,121,108,114,99, 91,90,109,89,86,81,89,103, 106,103,118,100,101,114,121,115,
+ 114,112,106,111,106,104,109,112, 108,105,167,179,185,80,87,184, 83,79,20,45,22,55,39,0, 21,0,65,55,62,0,0,0,
+ 0,223,195,229,105,32,59,131, 61,180,189,185,180,185,191,185, 177,167,169,174,171,174,169,172, 163,80,84,78,90,80,80,85,
+ 59,108,81,89,100,121,105,17, 50,24,63,0,25,17,1,53, 95,61,66,50,0,31,31,81, 122,2,32,1,95,88,0,56,
+ 61,50,56,0,94,59,44,0, 38,84,50,0,106,63,0,18, 44,63,28,0,24,0,80,50, 56,88,38,24,1,93,98,91,
+ 113,91,95,105,97,102,91,93, 93,99,91,0,23,0,0,44, 0,76,0,0,7,42,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,15,79,0,0,65,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 113,69,98,19,0,8,69,67, 80,86,51,58,73,87,70,67, 54,73,78,62,54,54,44,46, 38,34,67,70,119,41,36,55,
+ 200,198,192,189,158,190,180,168, 171,181,178,171,176,161,191,184, 172,169,180,178,178,160,185,181, 191,169,175,200,180,158,184,172,
+ 177,159,190,176,163,179,174,191, 178,187,183,159,184,168,174,182, 179,167,178,179,178,180,188,176, 172,175,172,178,169,173,171,37,
+ 175,177,171,172,170,176,176,180, 168,182,188,188,179,179,172,172, 155,178,155,196,173,164,176,173, 162,162,164,156,168,174,177,175,
+ 179,175,195,159,185,163,180,172, 181,189,191,180,175,184,173,177, 166,185,174,185,165,192,178,192, 167,176,168,178,189,191,180,179,
+ 167,183,176,162,191,193,192,176, 191,193,177,160,203,189,170,166, 173,161,169,158,174,146,173,170, 162,160,175,166,189,175,165,174,
+ 186,163,175,177,160,176,185,183, 187,191,188,159,174,173,175,176, 193,187,161,164,151,172,158,144, 152,140,181,168,175,37,20,2,
+ },
+ {46,0,6,6,2,2,2,2, 2,2,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 128,0,126,120,8,12,14,18, 34,38,30,30,26,42,32,42, 128,0,128,126,6,10,10,14, 38,42,34,34,30,46,36,46,
+ 128,0,128,128,10,14,16,20, 52,56,48,46,42,60,50,60, 128,0,128,128,12,16,18,22, 52,56,48,46,42,60,50,60,
+ 94,0,52,54,2,2,2,2, 32,38,28,28,24,40,30,40, 128,0,128,128,46,50,52,58, 82,88,78,78,72,94,80,90,
+ 0,0,0,0,140,140,138,138, 132,130,132,130,134,132,132,128, 0,0,0,0,134,142,142,142, 130,136,134,134,124,134,132,142,
+ 214,0,248,242,110,114,102,98, 152,140,134,122,120,120,88,72, 182,0,226,234,144,134,80,86, 136,114,134,134,126,154,96,86,
+ 178,0,234,228,102,112,100,108, 140,126,126,128,112,160,72,74, 180,0,232,230,100,114,108,110, 138,130,120,146,128,142,50,56,
+ 0,0,0,0,134,146,130,142, 138,144,128,148,64,80,66,78, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // EUC-JP (4368.914M chars) [7]
+ {NULL, ced_hires_0, ced_hires_1, ced_hires_2, },
+ 202, 178, 27, 15, 128,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,173,100, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,228,186,180,229,225,122,123, 168,7,2,28,62,133,86,74, 195,190,202,195,193,195,191,198, 201,199,199,204,200,197,198,200,
+ 200,193,197,196,192,199,202,195, 189,196,201,190,197,197,191,183, 120,107,102,109,112,111,101,113, 107,110,96,107,105,115,111,124,
+ 116,109,107,105,105,141,147,137, 110,140,100,120,146,102,103,102, 105,112,105,133,84,0,0,0, 0,54,7,43,73,0,9,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {146,59,0,0,61,22,13,0, 60,71,0,0,80,26,61,28, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 83,27,41,0,46,51,35,0, 49,37,24,68,35,38,32,13, 131,42,28,35,34,12,0,0, 63,21,33,0,74,25,11,68,
+ 61,0,0,0,68,11,0,0, 31,1,1,0,39,0,101,78, 61,83,87,94,79,39,85,70, 76,64,19,86,75,87,77,75,
+ 72,46,73,78,49,96,75,69, 60,49,27,69,85,65,60,49, 74,36,61,72,76,65,80,67, 67,77,74,64,73,63,88,36,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 164,202,199,199,199,177,202,190, 185,187,195,193,192,191,183,196, 192,181,174,190,182,189,177,195, 190,197,177,183,211,189,184,194,
+ 186,192,177,195,187,176,192,194, 198,186,201,196,186,188,198,191, 190,186,182,176,189,182,193,190, 188,181,187,183,189,177,189,189,
+ 180,185,188,180,184,187,184,191, 191,188,197,192,194,179,192,186, 182,187,188,200,180,176,184,190, 174,170,179,160,193,182,193,31,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,44,8,34,20,28,28, 0,0,0,0,0,0,0,0, 2,0,56,14,40,28,34,34, 0,0,0,0,0,0,0,0,
+ 2,0,60,16,42,30,36,38, 0,0,0,0,0,0,0,0, 92,0,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 70,0,128,88,128,128,128,128, 0,0,0,0,0,0,0,0, 102,0,128,120,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,146,186,190,162,182,180, 178,180,134,130,124,136,18,30, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 246,0,174,232,204,218,212,204, 0,0,131,131,129,123,123,117, 250,0,190,230,210,212,216,214, 0,0,125,121,125,131,131,133,
+ 250,0,192,232,204,216,202,214, 0,0,120,126,124,128,128,132, 244,0,172,236,206,224,218,216, 0,0,126,128,124,130,124,128,
+ 246,0,202,232,208,214,212,210, 0,0,144,110,104,110,108,108, 238,0,210,224,196,224,210,208, 0,0,118,114,104,116,104,152,
+ },
+},
+
+{ // BIG5 (2431.102M chars) [8]
+ {NULL, NULL, NULL, NULL},
+ 157, 174, 62, 10, 129,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,70,70, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,182,164,130,214,207,204,195, 202,198,207,192,194,201,196,187, 202,189,195,196,188,194,190,181, 194,188,192,185,188,188,191,185,
+ 184,179,185,186,180,189,110,108, 94,123,111,121,108,113,105,112, 116,103,101,106,110,102,114,99, 88,91,120,98,95,107,108,94,
+ 113,93,95,93,94,131,139,129, 121,130,93,111,138,87,98,87, 90,89,97,92,97,92,92,96, 102,147,84,82,84,87,88,107,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,216,162,127,212,204,205,192, 184,187,189,191,193,185,187,179, 188,190,183,192,177,189,189,198, 181,189,172,184,183,180,173,165,
+ 177,175,172,174,167,176,167,100, 42,127,129,92,89,80,107,94, 90,84,93,108,107,102,93,84, 105,100,83,90,94,79,92,95,
+ 84,84,90,92,77,89,103,113, 89,87,81,74,82,87,92,56, 77,96,70,88,72,107,102,81, 90,70,90,80,91,56,42,19,
+ },
+ {90,0,0,0,0,0,0,0, 0,0,0,0,0,4,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 77,44,72,0,0,0,44,0, 53,54,4,69,21,39,51,1, 78,36,31,5,0,4,0,0, 0,0,5,0,85,0,0,0,
+ 200,201,188,193,189,168,184,189, 192,178,176,174,183,184,182,191, 186,180,173,170,180,184,162,188, 182,171,166,172,168,185,178,176,
+ 173,183,178,161,180,180,169,177, 177,186,185,178,177,170,187,186, 184,188,165,193,177,186,185,182, 179,178,178,175,185,176,192,2,
+ 51,0,0,0,0,0,0,0, 0,0,0,19,0,0,0,14, 52,0,0,0,3,17,19,1, 0,0,0,0,0,0,0,0,
+ 157,190,169,191,185,175,165,184, 179,175,174,184,175,175,167,166, 181,177,178,189,170,179,183,169, 182,186,200,181,167,183,169,167,
+ 184,179,171,169,182,173,187,181, 182,184,182,175,183,177,188,186, 186,187,182,194,163,176,175,182, 180,172,184,181,179,176,173,180,
+ 175,175,178,172,178,181,187,170, 177,191,189,178,186,179,167,176, 165,169,173,182,190,175,181,181, 187,183,185,177,183,178,177,107,
+ },
+ {128,0,128,128,72,86,86,80, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 48,0,42,44,2,2,2,2, 80,76,2,2,2,2,2,2, 54,0,52,52,2,2,2,2, 96,96,2,2,2,2,2,2,
+ 74,0,76,72,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,8,18,18,14, 128,128,28,26,26,26,26,28,
+ 128,0,128,128,2,4,4,2, 128,128,34,30,30,30,32,32, 128,0,128,128,4,14,12,10, 128,128,38,34,34,34,36,36,
+ 0,0,0,0,160,160,160,166, 0,0,160,122,152,116,140,114, 0,0,0,0,150,150,150,148, 0,0,148,150,150,150,150,148,
+ 0,0,0,0,142,138,138,134, 0,0,136,138,134,134,136,134, 0,0,0,0,132,138,142,146, 0,0,136,130,138,138,134,136,
+ 0,0,0,0,136,146,134,142, 0,0,126,134,134,130,144,140, 0,0,0,0,136,138,138,144, 0,0,142,144,128,136,130,124,
+ 0,0,0,0,144,140,134,134, 0,0,160,126,108,102,108,106, 224,128,202,216,130,148,132,136, 204,204,126,118,114,144,122,150,
+ },
+},
+
+{ // Latin2 (315.882M chars) [9]
+ {NULL, NULL, ced_hires_14, ced_hires_14, },
+ 90, 204, 45, 27, 127,
+ {0,0,0,0,0,0,0,0, 0,13,13,0,0,13,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,178,79,184,121,96,188,129, 119,153,115,86,132,84,143,152, 125,178,73,182,114,95,188,79, 117,152,115,86,133,93,141,152,
+ 89,170,119,130,151,109,85,137, 171,147,170,100,153,155,102,118, 87,92,116,171,110,99,128,101, 181,128,158,80,133,126,117,98,
+ 90,170,94,124,153,103,95,127, 169,141,170,94,155,155,99,117, 81,92,116,171,113,99,148,82, 180,129,154,79,137,126,117,81,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 185,202,72,207,144,140,210,145, 119,178,153,122,170,124,172,196, 158,217,79,211,140,141,210,120, 115,176,151,152,187,124,173,200,
+ 97,198,155,156,200,127,165,183, 202,208,195,143,194,195,151,143, 149,174,157,197,159,157,199,120, 192,171,178,136,201,170,155,174,
+ 102,203,144,173,201,101,197,171, 203,203,208,143,202,204,139,150, 148,188,166,199,147,166,197,121, 192,191,178,144,189,182,155,121,
+ },
+ {65,0,0,0,0,0,0,0, 0,129,154,0,0,162,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,154,181,85,87,110,164,158, 144,155,125,102,191,153,189,148, 131,145,140,130,127,125,117,115, 113,114,177,135,201,88,123,155,
+ 81,200,180,210,197,203,168,188, 175,191,170,190,200,190,208,194, 182,141,204,199,200,181,182,186, 146,188,181,102,112,126,100,120,
+ 110,200,181,209,196,205,167,186, 172,191,169,190,200,189,207,194, 180,135,203,198,199,182,181,191, 143,189,181,91,125,72,102,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,176,79,167,122,89,163,118, 119,144,100,124,140,114,142,179, 121,178,71,168,109,88,163,67, 117,144,103,124,140,102,142,179,
+ 91,160,116,126,149,84,190,132, 166,138,143,91,83,185,105,126, 81,118,127,168,91,58,130,99, 171,124,126,82,122,95,120,153,
+ 93,161,107,127,150,87,190,124, 167,128,145,91,82,185,102,129, 83,119,127,170,121,59,134,85, 172,125,112,80,118,93,120,85,
+ },
+ {132,0,128,156,156,156,156,154, 0,0,156,156,156,156,156,156, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 10,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 14,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 132,0,98,130,142,136,102,114, 0,0,172,130,168,162,108,142, 130,0,136,132,52,54,136,130, 0,0,108,174,82,90,170,166,
+ 98,0,90,98,141,143,100,90, 0,0,164,118,170,178,130,128, 100,0,92,100,139,143,108,110, 0,0,166,92,174,156,128,142,
+ 134,0,136,134,36,54,135,135, 0,0,114,168,90,144,168,180, 130,0,130,134,40,56,137,141, 0,0,102,170,80,158,172,156,
+ },
+},
+
+{ // CP1251 (2609.249M chars) [10]
+ {NULL, NULL, ced_hires_10, ced_hires_10, },
+ 190, 219, 69, 19, 128,
+ {61,54,71,50,88,0,0,0, 0,0,73,0,84,68,72,51, 56,82,97,103,74,0,0,0, 0,0,69,0,81,63,70,32,
+ 175,85,84,108,0,54,0,0, 121,94,119,0,0,89,99,124, 0,0,150,148,47,70,0,0, 121,0,117,0,105,2,0,124,
+ 203,190,201,189,197,200,175,191, 205,172,201,198,196,207,207,203, 205,207,204,187,179,172,176,183, 172,169,147,176,170,168,160,171,
+ 201,185,197,186,194,200,174,188, 203,172,197,197,192,205,205,197, 203,204,203,185,172,167,174,181, 170,169,147,176,170,161,157,169,
+ 31,0,88,0,134,0,0,0, 85,0,28,0,0,0,5,0, 0,100,141,125,124,0,0,0, 1,0,16,0,11,11,33,0,
+ 184,2,95,21,0,6,0,0, 87,157,63,0,0,123,138,72, 0,0,102,143,7,102,0,0, 126,0,110,0,84,0,0,130,
+ 157,125,161,130,131,135,111,123, 148,122,140,123,137,137,153,133, 131,147,134,135,118,122,118,114, 110,103,83,118,114,116,116,143,
+ 193,147,189,166,165,195,131,156, 199,187,178,167,181,169,190,141, 170,175,180,173,120,175,145,137, 140,111,88,180,182,106,164,193,
+ },
+ {0,0,0,0,0,0,0,0, 0,123,152,0,0,158,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,147,180,67,83,115,163,151, 135,154,99,111,188,166,187,138, 128,140,138,123,122,123,115,111, 108,109,171,133,198,101,111,143,
+ 134,133,114,124,118,126,111,112, 113,129,111,108,118,121,114,117, 124,83,116,126,121,108,110,114, 76,103,91,97,102,121,55,113,
+ 81,143,106,111,119,137,102,102, 116,138,96,102,116,118,119,128, 122,71,118,150,131,126,116,96, 72,98,99,47,123,70,85,0,
+ 48,44,53,47,0,85,0,0, 0,0,66,0,81,17,64,0, 50,75,98,66,98,71,62,54, 0,60,67,96,81,29,65,16,
+ 175,84,89,98,74,0,54,89, 125,68,109,58,57,85,81,121, 74,34,155,158,0,74,29,81, 128,29,110,103,98,0,0,121,
+ 210,181,193,178,187,209,169,174, 206,177,192,194,187,202,211,177, 199,195,203,188,160,171,169,173, 162,156,142,188,185,132,168,185,
+ 213,184,195,179,188,212,171,176, 208,178,193,197,188,203,215,181, 203,197,205,192,162,172,170,174, 162,157,150,190,185,141,170,186,
+ },
+ {128,0,52,78,128,128,128,128, 128,128,74,82,12,26,10,22, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 42,0,2,6,48,58,46,48, 106,84,22,28,2,2,2,2, 56,0,2,20,64,76,62,64, 128,128,56,64,2,8,2,6,
+ 2,0,2,2,2,2,2,2, 128,110,26,34,2,2,2,2, 2,0,2,2,2,6,2,2, 128,128,42,50,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 66,44,2,2,2,2,2,2, 2,0,2,2,2,8,2,2, 80,56,2,4,2,2,2,2,
+ 130,0,112,138,218,228,196,188, 184,166,182,184,138,132,110,112, 144,0,134,138,190,188,202,204, 196,226,158,170,124,124,124,126,
+ 174,0,136,170,182,184,172,174, 168,194,204,146,106,102,84,84, 154,0,152,148,150,166,162,138, 158,178,156,166,124,128,130,130,
+ 146,0,152,154,166,162,144,134, 134,86,98,126,139,139,112,110, 148,0,150,162,176,170,154,134, 130,104,96,128,139,141,112,108,
+ 150,0,154,148,132,128,120,110, 122,130,92,130,6,4,139,139, 150,0,154,152,96,132,122,106, 124,124,82,130,6,2,139,141,
+ },
+},
+
+{ // CP1256 (4291.965M chars) [11]
+ {NULL, NULL, NULL, NULL},
+ 175, 213, 75, 16, 129,
+ {89,120,82,85,70,119,81,87, 78,57,0,61,52,107,87,0, 120,88,122,83,73,125,94,89, 114,96,1,85,34,116,73,0,
+ 173,142,76,85,128,71,110,132, 117,107,30,127,106,73,102,99, 133,84,73,90,108,82,96,130, 115,70,110,116,69,78,76,146,
+ 37,143,167,196,136,188,168,229, 204,148,208,168,191,193,183,197, 170,203,175,195,189,190,178,101, 181,157,204,175,195,197,193,197,
+ 97,209,116,216,198,191,209,110, 85,125,90,85,128,210,82,85, 126,99,116,154,99,137,142,85, 146,71,143,75,117,89,124,44,
+ 157,89,85,86,132,136,106,93, 83,69,0,77,102,72,53,0, 66,103,144,128,127,138,137,111, 67,109,0,87,78,86,35,10,
+ 182,172,102,152,141,111,120,142, 116,155,25,145,107,121,136,120, 155,111,121,104,137,100,101,159, 112,93,117,165,83,117,88,152,
+ 26,167,114,129,77,84,119,189, 178,200,192,163,167,160,159,182, 136,190,152,168,143,156,162,117, 154,133,178,121,152,172,165,172,
+ 181,188,141,190,186,184,180,168, 169,200,157,140,190,187,136,123, 149,117,121,139,145,135,137,118, 122,142,125,128,186,100,134,77,
+ },
+ {116,3,1,0,0,0,0,0, 0,134,143,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,134,177,78,100,111,160,151, 140,149,130,106,178,144,169,141, 137,152,147,136,134,132,126,124, 121,123,169,107,200,102,108,114,
+ 99,132,117,131,120,127,115,113, 116,121,112,109,121,123,118,119, 123,94,122,129,129,117,115,119, 115,108,101,110,97,133,114,122,
+ 120,160,152,167,156,164,144,156, 150,149,124,140,162,163,170,153, 151,130,177,173,168,136,148,104, 122,130,139,80,126,96,114,0,
+ 97,97,88,86,83,119,83,85, 98,64,79,60,61,80,88,47, 115,91,101,80,93,122,92,91, 105,117,46,102,80,115,80,37,
+ 173,163,81,85,125,72,114,122, 119,118,106,104,102,81,117,107, 133,87,73,88,113,82,96,129, 115,78,124,125,69,76,76,152,
+ 36,168,139,171,153,157,172,218, 198,200,201,168,187,192,186,204, 174,209,179,200,190,181,185,101, 182,157,199,169,196,190,195,192,
+ 104,226,105,204,209,198,208,118, 101,130,101,80,174,214,92,83, 163,120,130,171,118,155,158,79, 156,70,154,80,108,87,119,81,
+ },
+ {90,0,18,44,128,128,54,56, 102,84,40,60,2,2,2,34, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 42,0,2,2,48,48,10,14, 62,48,10,28,2,2,2,2, 62,0,2,22,72,74,30,34, 80,62,22,42,2,2,2,16,
+ 2,0,2,2,2,2,2,2, 36,24,2,6,2,2,2,2, 2,0,2,2,2,2,2,2, 48,34,2,16,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 30,18,2,2,2,2,2,2, 48,0,2,8,54,56,16,20, 64,52,12,30,2,2,2,4,
+ 178,0,138,160,188,202,138,136, 246,218,168,188,116,124,126,114, 160,0,142,144,184,174,160,162, 212,248,162,204,118,126,126,120,
+ 168,0,140,164,178,172,140,138, 162,176,200,170,106,100,92,136, 172,0,144,162,174,166,138,138, 212,212,170,220,92,90,90,122,
+ 142,0,150,150,92,128,44,72, 114,114,114,108,128,130,134,130, 142,0,150,148,114,142,74,80, 126,116,112,110,134,132,130,132,
+ 140,0,148,142,112,124,154,154, 120,132,116,116,134,132,128,132, 140,0,132,132,126,136,172,174, 158,152,148,140,132,130,128,164,
+ },
+},
+
+{ // CP1250 (456.295M chars) [12]
+ {NULL, NULL, ced_hires_15, ced_hires_15, },
+ 90, 207, 44, 30, 128,
+ {106,94,109,3,114,124,102,101, 0,71,177,95,143,118,167,98, 48,59,96,73,40,99,64,61, 0,67,141,59,108,83,130,60,
+ 178,82,81,175,131,134,115,136, 125,101,117,114,114,90,115,134, 133,104,75,173,117,99,105,135, 120,134,117,119,129,95,129,131,
+ 94,177,121,140,156,127,86,139, 173,151,161,106,170,163,105,122, 92,104,118,172,129,101,131,105, 183,159,162,83,135,145,120,103,
+ 95,177,123,127,157,116,103,130, 171,145,162,96,170,163,102,121, 83,93,119,172,131,103,150,86, 182,159,158,82,139,145,119,83,
+ 162,120,91,35,137,141,111,98, 13,75,196,81,168,151,193,131, 88,128,169,153,152,163,162,136, 10,134,194,112,166,184,194,139,
+ 187,92,75,209,146,163,125,147, 121,160,155,150,112,126,141,198, 160,116,81,213,142,104,106,164, 117,176,153,170,170,126,171,202,
+ 99,200,157,158,202,129,167,185, 204,210,197,145,196,197,153,145, 151,176,159,199,161,159,201,122, 194,173,180,138,203,172,157,176,
+ 104,205,146,175,203,103,199,173, 205,205,211,145,204,206,141,152, 150,191,168,201,150,168,199,123, 194,193,180,146,191,184,157,123,
+ },
+ {122,0,0,0,0,0,0,0, 0,143,154,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,153,180,86,97,112,169,159, 147,157,128,106,190,155,189,149, 138,150,147,134,133,129,124,123, 120,121,178,134,203,92,128,154,
+ 110,204,183,200,193,208,171,187, 178,196,172,193,197,190,210,197, 183,143,205,201,201,184,184,176, 150,190,178,114,114,132,107,122,
+ 115,204,184,198,191,209,169,185, 175,197,172,193,196,187,209,197, 181,137,202,200,201,186,184,176, 145,192,178,94,130,77,114,0,
+ 105,89,105,77,114,127,87,104, 65,77,167,87,121,154,174,103, 85,93,106,115,112,126,100,99, 71,123,167,110,121,155,174,104,
+ 177,116,81,166,127,131,117,124, 123,122,102,105,107,115,124,164, 136,109,72,166,119,93,103,132, 119,133,104,122,116,103,122,164,
+ 93,165,121,130,150,102,158,134, 169,143,144,96,85,190,109,129, 88,98,136,169,98,62,135,101, 172,130,135,83,124,100,121,154,
+ 95,167,111,128,151,90,158,126, 170,139,146,91,84,191,105,135, 89,99,137,172,123,63,135,88, 173,130,131,81,120,97,121,86,
+ },
+ {28,0,2,18,2,2,2,2, 14,14,14,22,4,16,2,16, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 22,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 32,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 18,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 18,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 130,0,104,120,140,130,110,116, 162,132,114,116,180,146,140,112, 134,0,136,128,84,84,138,128, 108,162,124,148,120,118,180,146,
+ 140,0,112,138,140,134,98,104, 108,128,184,142,144,178,120,158, 136,0,132,138,94,84,138,132, 116,142,142,178,124,144,154,188,
+ 94,0,90,96,141,143,100,90, 180,114,154,138,170,172,130,124, 96,0,94,98,139,143,108,112, 170,106,170,116,174,152,128,136,
+ 130,0,138,132,36,52,135,135, 114,180,122,162,92,140,170,174, 128,0,130,132,40,56,137,141, 94,170,112,180,80,154,172,150,
+ },
+},
+
+{ // Latin5 (322.539M chars) [13]
+ {NULL, NULL, ced_hires_18, ced_hires_18, },
+ 96, 232, 51, 21, 128,
+ {20,0,20,20,20,20,20,20, 20,37,37,20,20,16,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,125,94,100,128,93,114,133, 120,100,115,113,111,97,114,106, 129,100,95,102,119,100,101,130, 117,91,117,117,87,95,93,116,
+ 109,114,123,142,149,117,104,183, 111,140,112,106,104,116,108,107, 147,123,109,109,99,127,169,104, 100,88,100,105,175,189,165,102,
+ 108,113,104,109,151,116,97,192, 108,128,102,97,101,120,103,120, 178,123,114,107,105,128,165,98, 101,92,102,87,175,201,186,105,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 183,147,103,153,143,113,121,144, 117,156,143,147,109,122,138,121, 157,113,123,105,138,101,102,161, 114,95,164,166,85,118,91,158,
+ 153,197,154,170,198,186,170,182, 181,207,170,141,118,193,149,136, 201,175,130,196,157,155,197,118, 180,109,177,140,200,218,211,173,
+ 183,202,142,172,199,196,170,169, 171,201,158,141,149,203,138,125, 202,175,150,198,146,156,196,119, 180,143,177,129,187,238,211,108,
+ },
+ {50,6,11,57,0,0,0,0, 0,144,149,0,0,164,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,146,185,81,100,113,168,166, 142,154,126,112,190,161,183,145, 134,151,147,137,134,132,126,124, 121,122,176,137,206,105,128,149,
+ 92,194,178,190,185,190,170,185, 176,197,160,189,202,193,206,184, 174,152,207,197,198,181,179,170, 146,175,180,111,108,128,102,138,
+ 127,196,180,195,187,192,172,183, 173,199,158,205,212,204,220,183, 177,140,211,202,199,186,178,170, 143,189,200,94,140,80,111,5,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,126,100,100,127,94,116,124, 121,125,106,108,106,120,123,111, 127,113,88,113,118,96,111,128, 119,95,113,121,111,107,92,126,
+ 118,110,119,175,148,113,108,168, 110,141,110,109,101,124,114,104, 171,129,114,115,94,154,141,101, 100,92,102,106,132,171,177,152,
+ 114,115,112,174,149,118,107,171, 109,124,108,108,100,125,114,97, 190,129,114,113,120,155,143,91, 105,92,113,100,137,200,199,107,
+ },
+ {130,0,102,144,144,142,108,138, 0,0,154,154,154,154,154,154, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,2,2,2,2, 0,0,14,38,8,2,8,2, 62,0,2,22,2,10,2,4, 0,0,20,44,12,6,12,2,
+ 26,0,2,2,2,2,2,2, 0,0,2,4,2,2,2,2, 14,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 20,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 164,0,136,160,116,116,104,106, 0,0,202,178,144,126,130,118, 160,0,142,150,118,124,102,106, 0,0,180,246,168,156,154,124,
+ 90,0,92,92,145,147,90,92, 0,0,156,176,188,168,138,134, 112,0,112,116,141,141,114,116, 0,0,110,138,166,180,166,148,
+ 132,0,138,122,44,54,131,135, 0,0,130,156,124,128,182,162, 128,0,132,132,50,60,135,133, 0,0,112,120,100,130,142,168,
+ },
+},
+
+{ // ISO-8859-11 (489.481M chars) [14]
+ {NULL, NULL, NULL, NULL},
+ 184, 198, 48, 20, 127,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,209,195,88,202,83,128,203, 192,152,191,169,88,166,132,138, 156,134,149,172,200,200,176,200, 176,211,199,194,179,150,192,164,
+ 175,205,201,213,136,199,75,202, 176,173,203,200,140,209,144,123, 197,203,215,179,198,198,169,189, 190,187,93,39,18,43,4,72,
+ 210,196,186,186,192,106,151,183, 207,206,151,138,179,120,113,126, 122,127,132,111,120,111,102,99, 96,94,33,88,83,68,64,77,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 184,175,143,82,159,80,91,182, 155,114,141,116,87,135,98,96, 126,88,93,153,172,160,139,156, 134,188,176,156,101,92,163,122,
+ 113,183,179,173,84,159,74,169, 155,135,151,108,92,170,98,140, 166,90,180,144,149,166,80,79, 133,137,84,1,4,52,40,119,
+ 116,99,97,96,100,92,161,101, 174,168,108,110,179,87,92,109, 117,117,114,109,105,106,99,96, 94,95,20,89,93,21,54,98,
+ },
+ {70,0,0,46,0,0,0,0, 0,136,148,0,0,160,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 207,129,181,82,82,93,164,157, 152,152,114,111,166,150,188,146, 128,141,136,126,124,123,115,111, 109,109,165,100,202,96,118,126,
+ 79,118,108,118,109,116,106,106, 104,104,108,101,111,116,105,100, 115,76,111,119,110,92,99,104, 73,98,81,125,114,120,90,106,
+ 61,105,100,99,109,106,92,82, 92,104,93,95,95,106,105,107, 114,58,104,111,104,93,100,89, 80,71,96,55,132,63,99,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,206,190,91,199,78,140,206, 190,148,188,166,92,168,152,144, 148,139,143,179,199,196,177,200, 172,213,199,192,172,143,191,162,
+ 175,208,201,213,153,202,62,203, 176,170,200,196,121,209,145,145, 194,206,218,188,201,202,172,189, 193,191,94,64,59,63,57,74,
+ 203,186,178,183,187,119,161,181, 208,206,150,139,187,118,121,123, 127,115,118,111,108,125,108,109, 106,112,71,99,90,69,77,91,
+ },
+ {128,0,148,148,148,148,148,148, 0,0,148,148,148,148,148,148, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,8,8,16,14, 0,0,2,2,2,2,2,36, 2,0,2,2,4,4,14,10, 0,0,2,2,2,2,2,34,
+ 2,0,2,2,2,2,8,4, 0,0,2,2,2,2,2,32, 2,0,2,2,4,4,14,10, 0,0,2,2,2,2,2,44,
+ 2,0,2,2,6,6,14,10, 0,0,2,2,2,2,2,42, 68,0,20,30,128,126,128,112, 0,0,16,12,6,8,14,128,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 166,0,158,164,182,174,180,178, 0,0,126,118,128,132,130,78, 158,0,162,160,132,152,142,144, 0,0,120,122,126,134,130,92,
+ 158,0,162,160,128,160,132,146, 0,0,126,126,124,134,128,90, 166,0,162,160,142,146,146,156, 0,0,134,134,128,68,132,84,
+ 162,0,162,160,138,154,148,150, 0,0,130,134,132,114,112,128, 158,0,164,148,196,208,204,216, 0,0,84,76,64,52,134,254,
+ },
+},
+
+{ // ISO-8859-15 (27.581M chars) [15]
+ {NULL, NULL, ced_hires_21, ced_hires_21, },
+ 86, 217, 37, 21, 127,
+ {0,0,0,0,0,0,0,0, 0,85,85,0,0,85,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 187,137,107,112,127,105,104,142, 106,111,127,124,123,101,126,118, 133,112,107,116,117,113,113,142, 111,105,127,127,92,89,91,126,
+ 121,138,130,159,161,129,133,190, 123,152,123,118,116,144,119,121, 126,135,122,132,111,139,144,115, 113,103,117,117,145,120,135,114,
+ 121,137,113,131,163,128,131,190, 121,141,112,111,114,143,115,133, 123,135,125,131,110,140,160,110, 114,105,118,101,148,119,129,118,
+ 0,0,0,0,0,0,0,0, 0,34,33,0,0,33,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 195,159,116,166,180,125,125,156, 145,168,155,159,121,135,150,134, 169,125,135,118,145,114,115,173, 125,108,176,178,138,141,110,170,
+ 165,209,166,183,211,198,182,194, 193,219,182,153,130,205,161,148, 160,187,142,208,169,167,209,131, 192,122,189,152,212,181,158,185,
+ 195,214,155,184,212,208,182,181, 183,213,170,154,161,215,150,137, 175,187,162,210,158,168,208,131, 192,156,189,142,199,193,155,120,
+ },
+ {23,0,0,70,0,0,0,0, 0,145,155,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,142,175,96,115,116,169,162, 150,154,125,112,184,159,177,144, 147,165,159,150,148,149,139,138, 134,136,166,128,201,103,116,143,
+ 104,197,188,202,195,190,180,196, 185,179,172,189,202,198,215,195, 185,164,216,206,204,173,190,182, 158,170,181,116,107,135,114,133,
+ 130,194,189,197,192,189,178,194, 182,176,171,189,201,194,213,194, 181,151,213,203,201,172,189,182, 155,176,182,107,136,90,117,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 187,138,113,113,149,107,108,134, 123,135,117,120,118,129,131,122, 135,125,101,126,120,109,123,140, 113,107,124,133,118,119,101,138,
+ 132,128,129,187,160,126,122,145, 123,152,121,121,113,136,125,116, 152,141,127,129,106,167,144,114, 113,104,118,117,134,117,122,164,
+ 130,132,121,187,161,130,119,138, 122,137,119,120,112,137,123,109, 153,141,126,130,124,167,145,104, 116,104,130,109,131,116,130,120,
+ },
+ {178,0,128,156,122,126,124,128, 0,0,168,202,168,202,168,202, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 22,0,2,2,2,2,2,2, 0,0,2,14,2,2,2,2, 50,0,2,18,2,2,2,2, 0,0,2,20,2,2,2,6,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 42,0,2,8,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 38,0,2,4,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 160,0,138,164,108,108,102,104, 0,0,190,160,132,134,120,146, 158,0,144,154,112,118,106,110, 0,0,160,218,160,164,148,160,
+ 88,0,92,94,139,139,94,92, 0,0,152,166,184,176,134,148, 98,0,96,102,139,139,110,110, 0,0,138,172,174,190,154,164,
+ 128,0,138,126,40,48,135,135, 0,0,130,160,128,148,186,180, 120,0,126,120,48,54,139,141, 0,0,136,160,142,188,166,188,
+ },
+},
+
+{ // CP1257 (41.264M chars) [16]
+ {NULL, NULL, ced_hires_20, ced_hires_20, },
+ 84, 222, 39, 20, 128,
+ {112,108,100,0,132,119,103,89, 0,78,64,75,0,77,68,67, 11,35,79,71,21,82,53,35, 0,31,45,33,0,0,49,0,
+ 173,72,85,94,121,69,110,128, 74,94,81,110,109,101,104,75, 124,97,89,88,110,94,97,129, 74,80,81,113,82,88,85,80,
+ 132,139,163,137,149,121,106,164, 124,141,114,153,127,139,154,153, 181,121,165,109,89,101,125,103, 97,82,78,145,112,85,162,98,
+ 138,140,163,106,150,118,115,164, 124,135,112,155,125,133,155,153, 179,115,165,109,88,104,145,100, 130,82,79,145,129,84,161,94,
+ 157,114,85,13,131,136,105,92, 1,81,72,77,10,83,100,70, 81,122,164,148,146,158,157,131, 0,129,85,106,48,81,114,1,
+ 181,85,101,152,141,118,119,142, 101,154,98,145,107,120,136,92, 155,111,121,103,136,99,100,159, 111,92,101,164,84,116,89,89,
+ 171,189,211,72,196,184,158,199, 199,205,85,210,168,152,203,176, 211,170,179,194,91,153,195,116, 172,97,91,199,198,89,204,171,
+ 205,199,217,92,198,194,183,201, 200,199,104,216,166,147,205,177, 211,185,178,196,93,154,194,117, 220,101,91,199,185,90,202,118,
+ },
+ {116,0,0,0,0,0,0,0, 0,139,155,0,0,164,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,153,188,77,92,107,206,155, 138,165,122,102,192,146,189,135, 135,150,145,135,132,131,124,123, 120,121,178,143,202,85,105,154,
+ 107,193,189,191,196,191,164,188, 169,204,198,197,197,192,203,181, 177,134,209,211,206,182,187,168, 142,176,171,109,99,129,101,112,
+ 100,195,190,187,196,195,162,186, 165,207,198,197,197,192,202,181, 176,126,208,211,206,184,192,168, 136,178,172,83,124,77,109,0,
+ 117,90,127,119,115,124,85,99, 57,71,0,89,81,86,68,85, 80,86,101,138,134,121,102,96, 36,92,81,109,21,74,91,19,
+ 173,101,111,88,123,70,112,120, 89,109,84,101,109,115,119,115, 133,107,98,98,116,107,100,128, 113,93,107,121,96,107,110,117,
+ 136,143,168,99,146,122,112,156, 159,136,103,157,139,159,155,146, 175,101,144,111,83,103,128,97, 149,82,75,140,115,76,159,150,
+ 138,144,169,93,147,125,112,157, 159,131,104,158,139,160,159,147, 175,102,144,111,84,103,122,91, 149,83,76,143,108,75,158,101,
+ },
+ {100,0,60,88,62,66,62,66, 128,122,100,126,86,90,84,92, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 38,0,2,2,2,2,2,2, 52,46,20,44,6,12,6,14, 70,0,6,34,6,12,6,12, 64,56,32,54,18,22,16,24,
+ 26,0,2,2,2,2,2,2, 4,2,2,2,2,2,2,2, 22,0,2,2,2,2,2,2, 8,4,2,2,2,2,2,2,
+ 26,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 4,2,2,2,2,2,2,2,
+ 168,0,132,156,116,128,102,104, 234,200,166,204,168,184,164,158, 152,0,138,142,110,104,124,126, 210,216,166,230,168,168,172,154,
+ 164,0,130,166,110,108,106,106, 148,164,204,170,130,130,130,128, 164,0,140,158,102,98,106,104, 204,214,178,244,158,162,146,152,
+ 102,0,88,106,141,141,94,98, 190,154,154,162,178,188,140,126, 102,0,90,98,141,139,114,106, 152,140,118,176,188,174,148,132,
+ 132,0,134,134,26,40,135,135, 176,186,144,164,96,138,176,190, 126,0,136,122,28,38,135,133, 140,172,130,130,114,154,186,176,
+ },
+},
+
+{ // CP1255 (313.575M chars) [17]
+ {NULL, NULL, NULL, ced_hires_12, },
+ 192, 233, 81, 15, 127,
+ {98,94,106,111,72,121,83,91, 81,65,26,68,0,71,0,69, 81,85,102,93,99,127,102,95, 75,97,17,87,61,82,71,67,
+ 175,124,97,90,58,73,104,127, 119,93,86,110,111,82,101,112, 129,87,94,110,112,86,97,132, 116,73,60,114,71,123,75,120,
+ 119,65,141,119,118,109,112,113, 121,112,61,74,117,67,73,62, 61,98,78,56,68,54,124,148, 38,49,42,32,35,40,36,27,
+ 211,211,193,200,214,221,187,200, 193,220,109,200,214,125,214,131, 205,198,202,108,201,91,192,199, 210,210,202,32,36,81,108,32,
+ 159,117,88,89,134,138,108,95, 85,73,57,78,26,89,0,88, 63,104,145,129,128,139,138,112, 83,110,64,88,33,78,35,6,
+ 184,147,104,154,150,113,122,144, 118,156,77,147,109,123,138,122, 157,113,123,106,139,101,103,161, 114,95,71,167,86,119,91,159,
+ 81,40,33,27,57,50,53,76, 89,84,40,25,97,25,54,51, 36,79,53,49,25,25,27,56, 26,26,27,24,26,39,25,39,
+ 181,180,170,178,203,186,155,176, 171,197,173,144,189,203,159,191, 133,169,170,166,142,163,146,172, 192,178,205,35,59,102,136,39,
+ },
+ {118,0,0,0,0,0,0,0, 0,137,146,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,155,187,100,97,112,172,178, 136,158,128,132,188,170,183,153, 137,155,149,140,137,135,129,128, 124,127,173,136,203,112,110,154,
+ 108,138,122,146,128,136,120,118, 124,127,114,112,126,128,125,124, 133,137,129,135,139,118,124,121, 117,118,103,111,143,128,106,131,
+ 142,144,112,120,123,140,110,110, 119,135,106,107,129,123,126,129, 124,108,122,153,134,129,118,103, 84,104,101,80,145,91,116,0,
+ 106,72,108,107,84,120,79,80, 98,59,0,68,0,76,0,66, 101,102,115,98,109,128,101,107, 92,118,68,108,103,102,97,114,
+ 175,126,108,86,95,75,110,121, 124,112,102,101,106,84,113,116, 133,82,132,131,119,84,97,130, 116,74,64,119,72,123,75,135,
+ 128,83,129,119,124,114,117,127, 127,114,43,82,134,59,75,65, 44,112,90,59,50,58,80,109, 50,44,39,43,41,44,40,43,
+ 205,205,192,202,209,223,186,199, 192,223,176,195,211,204,206,190, 204,195,200,171,199,162,190,198, 213,203,213,43,41,82,94,46,
+ },
+ {86,0,44,70,130,132,130,122, 110,94,68,88,90,132,2,10, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,40,40,40,40, 60,46,16,34,34,72,2,2, 62,0,2,26,74,72,72,70, 70,54,24,42,44,82,2,2,
+ 60,0,2,22,70,70,68,68, 128,128,100,128,128,128,26,36, 62,0,2,26,66,66,66,68, 128,128,128,128,128,128,48,62,
+ 2,0,2,2,2,2,2,2, 24,10,2,2,2,34,2,2, 2,0,2,2,2,2,2,2, 34,20,2,10,10,46,2,2,
+ 170,0,136,156,182,190,170,166, 248,224,180,222,232,172,106,66, 154,0,140,142,176,164,188,188, 214,246,176,224,176,218,104,106,
+ 166,0,132,164,178,170,168,166, 160,170,204,170,154,182,76,76, 162,0,140,154,182,180,172,168, 224,230,176,228,214,218,94,74,
+ 172,0,148,146,180,182,188,180, 236,188,168,220,190,202,120,120, 158,0,146,158,180,180,180,178, 162,232,182,208,174,184,68,66,
+ 146,0,150,148,106,132,122,126, 110,90,74,98,128,90,131,133, 142,0,150,144,92,132,120,128, 78,90,74,72,124,138,133,127,
+ },
+},
+
+{ // KOI8R (315.553M chars) [18]
+ {NULL, NULL, ced_hires_9, ced_hires_9, },
+ 189, 220, 69, 17, 128,
+ {17,17,17,17,17,17,17,17, 17,39,39,17,17,39,17,17, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 17,17,17,117,17,17,17,17, 17,17,17,17,17,17,17,17, 17,17,17,118,17,25,17,17, 17,17,17,25,17,17,17,47,
+ 155,199,183,172,193,200,169,185, 168,203,166,197,197,191,205,205, 196,169,201,203,201,186,173,196, 170,177,187,171,168,166,181,137,
+ 158,201,188,174,195,200,175,189, 171,204,166,200,198,196,207,207, 201,170,204,206,202,187,175,199, 170,177,189,174,172,166,183,137,
+ 7,7,7,7,7,7,7,7, 7,39,39,7,7,39,7,7, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,
+ 7,7,7,123,7,7,7,7, 7,7,7,7,7,7,7,7, 7,7,7,74,7,7,7,7, 7,7,7,7,7,7,7,100,
+ 164,191,135,141,166,194,118,163, 175,198,187,177,169,182,165,191, 138,192,164,174,180,171,125,189, 179,178,152,139,98,108,139,73,
+ 131,162,130,110,132,145,129,147, 126,158,123,145,139,148,148,152, 142,148,128,154,139,149,108,167, 114,119,123,105,122,90,109,85,
+ },
+ {0,0,0,0,0,0,0,0, 0,111,171,0,0,149,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,146,172,30,25,90,164,138, 116,148,90,87,187,167,194,124, 72,106,101,90,90,85,93,77, 75,75,165,131,188,82,114,147,
+ 125,92,36,71,49,93,30,0, 64,104,0,30,49,47,61,97, 110,0,62,58,56,11,54,71, 45,83,11,70,85,112,54,106,
+ 92,94,57,74,51,94,36,30, 66,106,0,34,57,53,67,98, 113,0,67,61,51,49,56,17, 49,84,41,0,111,57,57,0,
+ 47,49,56,30,0,36,0,0, 0,0,0,25,17,0,0,11, 0,36,17,0,0,0,0,0, 0,0,49,0,0,11,0,0,
+ 25,0,0,123,0,0,0,0, 0,0,0,0,0,0,0,0, 57,17,45,121,30,57,11,36, 66,11,36,41,36,49,58,53,
+ 169,212,182,169,188,212,160,181, 172,207,177,191,196,188,203,214, 181,185,201,197,203,191,170,194, 184,189,174,162,139,157,176,118,
+ 167,209,180,168,186,208,159,179, 171,206,177,190,193,187,201,211, 178,185,198,195,202,188,168,192, 184,187,173,162,133,156,176,117,
+ },
+ {130,0,102,154,154,154,154,154, 154,154,154,154,112,154,116,154, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 92,0,38,74,128,128,128,128, 128,128,128,128,22,34,24,36, 92,0,38,74,128,128,128,128, 128,128,128,128,44,54,46,56,
+ 2,0,2,2,18,22,26,26, 110,128,40,40,2,2,2,2, 2,0,2,2,22,28,32,30, 128,128,48,50,2,2,2,2,
+ 2,0,2,2,16,20,24,24, 128,128,76,80,2,2,2,2, 2,0,2,2,20,24,30,28, 128,128,82,84,2,2,2,2,
+ 154,0,128,178,178,178,178,178, 178,178,178,178,128,178,140,178, 154,0,132,178,178,178,178,178, 178,178,178,178,138,178,178,178,
+ 146,0,154,136,178,182,180,178, 178,178,180,178,134,146,82,92, 148,0,152,168,180,184,180,178, 178,180,180,180,90,106,136,148,
+ 154,0,154,152,154,146,152,150, 134,154,138,86,141,141,6,4, 154,0,154,156,114,150,158,160, 166,172,142,122,139,141,2,12,
+ 136,0,154,156,180,184,184,158, 124,140,110,136,112,106,141,139, 140,0,152,158,180,186,178,172, 130,144,112,138,114,110,139,141,
+ },
+},
+
+{ // GBK (106.219M chars) [19]
+ {NULL, NULL, NULL, NULL},
+ 203, 189, 27, 17, 128,
+ {80,136,138,128,141,121,130,133, 125,121,114,108,141,130,70,75, 125,127,117,120,119,135,127,109, 133,129,119,130,122,125,128,114,
+ 74,204,125,202,219,215,119,114, 118,156,113,86,97,79,82,81, 189,196,199,195,195,208,196,199, 196,203,194,200,201,197,193,195,
+ 192,195,189,194,195,183,196,191, 201,197,207,197,197,197,201,199, 201,191,203,201,198,196,202,200, 134,116,114,132,122,124,118,115,
+ 115,120,116,120,141,121,124,123, 121,120,121,113,125,116,117,110, 117,114,112,113,119,112,124,123, 85,77,84,80,72,84,79,0,
+ 135,112,99,113,112,124,117,103, 124,95,92,89,122,95,0,51, 0,0,0,0,0,0,0,0, 25,1,0,0,0,0,0,0,
+ 1,113,69,80,84,69,59,13, 116,104,61,70,84,69,73,47, 74,88,74,72,72,73,61,73, 51,54,54,59,70,77,75,75,
+ 96,68,68,66,84,83,47,68, 64,64,44,78,54,61,26,26, 64,64,47,59,70,74,70,66, 64,80,61,44,44,69,66,82,
+ 51,57,66,57,64,47,64,54, 18,82,68,66,40,66,70,69, 51,70,40,88,37,37,40,59, 59,40,102,70,73,70,68,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,1,60,0,0,64,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 94,78,82,0,0,0,93,107, 64,60,46,107,76,56,82,44, 29,64,73,39,27,27,18,14, 14,15,18,0,111,60,0,46,
+ 104,95,102,121,118,73,105,75, 96,131,74,109,89,100,103,111, 93,108,98,107,126,103,101,125, 113,99,98,90,89,109,120,88,
+ 77,85,85,92,91,104,101,109, 101,83,74,90,111,98,98,117, 96,93,127,97,106,100,91,99, 91,102,105,84,114,93,91,17,
+ 124,109,99,116,90,100,105,89, 96,95,103,108,93,117,111,110, 98,115,123,116,94,82,112,111, 83,97,106,96,74,105,115,121,
+ 126,196,197,192,196,176,185,184, 195,185,196,194,198,187,184,191, 190,190,180,189,189,194,181,191, 181,191,188,196,189,191,187,192,
+ 186,174,192,193,202,189,193,194, 190,188,189,195,181,187,196,191, 200,178,184,186,189,186,192,180, 193,176,195,182,185,179,182,186,
+ 185,187,190,185,183,187,185,186, 177,190,189,191,187,187,181,180, 178,185,188,196,173,177,186,192, 192,182,197,186,177,187,185,0,
+ },
+ {144,0,168,168,120,122,152,128, 124,116,2,2,2,2,4,4, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 78,0,26,38,2,2,12,8, 128,128,22,22,20,24,26,26, 72,0,20,32,2,2,6,2, 128,128,40,42,40,44,46,46,
+ 74,0,22,34,2,2,6,4, 128,128,40,42,38,44,46,46, 80,0,28,40,6,4,14,10, 128,128,44,46,42,50,52,50,
+ 128,0,128,128,98,102,124,110, 128,128,46,50,46,54,56,54, 128,0,128,128,128,128,128,128, 128,128,38,40,38,44,46,44,
+ 0,0,0,0,236,252,250,242, 230,226,122,106,114,124,130,128, 0,0,0,0,252,238,254,254, 240,234,112,116,106,116,144,130,
+ 0,0,0,0,242,220,220,232, 106,108,136,130,128,116,126,114, 0,0,0,0,206,208,218,216, 82,88,128,126,128,126,126,132,
+ 0,0,0,0,204,218,220,208, 84,82,122,128,128,128,132,132, 0,0,0,0,212,214,210,210, 92,88,124,128,126,136,120,126,
+ 0,0,0,0,208,210,204,206, 200,208,136,124,124,122,126,126, 0,0,0,0,198,202,200,218, 200,196,128,122,132,128,130,120,
+ },
+},
+
+{ // Greek (109.816M chars) [20]
+ {NULL, NULL, ced_hires_11, ced_hires_11, },
+ 186, 219, 72, 19, 128,
+ {0,0,0,0,0,0,0,0, 0,24,24,0,0,24,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,64,123,80,13,49,101,123, 114,89,0,152,107,83,76,53, 122,81,71,65,105,84,180,126, 183,174,186,111,183,56,165,168,
+ 117,206,178,191,194,206,163,186, 183,198,202,195,198,199,165,203, 201,196,56,205,210,185,180,183, 154,177,136,87,180,183,177,186,
+ 157,202,171,187,188,204,165,188, 183,199,199,194,197,198,161,202, 194,196,154,201,208,185,175,180, 147,179,136,79,183,166,168,0,
+ 0,0,0,0,0,0,0,0, 0,34,34,0,0,34,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 183,58,94,153,91,63,120,143, 117,155,71,146,108,122,78,71, 156,112,122,105,105,84,83,160, 96,97,84,166,98,118,76,75,
+ 70,162,129,130,139,152,103,165, 133,144,142,123,130,144,98,158, 139,113,63,159,134,140,122,126, 98,117,81,46,175,143,180,165,
+ 64,199,125,122,122,176,111,189, 115,196,146,142,141,197,113,188, 137,142,206,133,145,191,115,124, 78,148,115,74,181,161,152,93,
+ },
+ {98,0,0,0,0,0,0,0, 2,135,160,0,67,161,42,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 225,142,174,72,92,107,173,153, 133,152,109,98,183,147,185,142, 132,150,145,135,133,131,124,123, 119,121,167,138,195,117,108,120,
+ 78,122,108,129,112,120,111,107, 109,113,108,105,115,117,111,121, 118,81,119,121,112,95,108,105, 80,100,85,102,97,121,98,109,
+ 109,116,101,104,112,116,102,95, 104,115,100,100,121,112,116,124, 116,72,112,117,112,104,110,87, 82,90,98,64,123,66,101,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,42,84,78,172,159,104,116, 116,102,153,94,101,82,171,60, 122,76,68,71,118,78,185,127, 180,182,189,143,179,67,177,172,
+ 111,207,161,179,175,201,159,197, 167,203,190,188,186,197,160,205, 189,198,0,199,200,190,174,173, 145,183,137,110,188,186,184,191,
+ 66,211,163,182,177,206,160,199, 172,206,192,193,189,200,165,208, 196,203,194,200,201,194,176,176, 146,185,138,110,182,181,174,0,
+ },
+ {146,0,130,170,170,170,170,168, 0,0,166,170,136,168,130,154, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 34,0,2,6,52,58,56,62, 0,0,4,2,2,2,2,2, 4,0,2,2,22,28,24,32, 0,0,14,2,2,2,2,2,
+ 2,0,2,2,2,2,2,4, 0,0,6,2,2,2,2,2, 2,0,2,2,2,6,2,8, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,6, 0,0,2,2,2,2,2,2, 2,0,2,2,2,10,6,12, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 166,0,136,172,188,188,186,188, 0,0,186,116,110,106,106,106, 164,0,146,164,180,178,180,174, 0,0,98,100,140,144,98,104,
+ 146,0,152,154,164,164,162,156, 0,0,52,142,139,137,118,112, 148,0,152,154,150,144,142,134, 0,0,48,136,139,131,120,126,
+ 150,0,152,146,92,128,132,126, 0,0,144,72,30,124,139,139, 150,0,152,150,88,124,136,128, 0,0,66,76,4,118,141,139,
+ },
+},
+
+{ // JIS (138.804M chars) [21]
+ {NULL, NULL, NULL, NULL},
+ 37, 154, 2, 1, 129,
+ {0,0,0,0,0,0,0,0, 0,99,99,0,0,99,107,197, 0,0,0,0,0,0,0,0, 0,0,0,254,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {119,0,0,0,0,0,0,0, 0,53,49,0,0,49,0,0, 0,0,0,0,0,0,0,0, 0,0,0,197,0,0,0,0,
+ 73,56,65,75,246,65,49,49, 243,61,49,49,49,49,49,49, 49,56,61,49,49,49,49,49, 49,49,56,56,49,49,49,49,
+ 49,49,49,49,49,49,49,49, 49,49,49,49,49,56,143,49, 49,49,97,49,75,49,59,49, 49,49,61,49,49,53,56,53,
+ 49,98,49,53,49,49,49,94, 49,49,49,49,63,69,74,49, 49,49,53,65,49,49,49,49, 49,49,49,49,49,56,53,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {130,184,30,216,132,172,164,208, 0,0,0,0,0,0,0,0, 128,2,128,38,128,122,124,104, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // CP1254 (20.130M chars) [22]
+ {NULL, NULL, ced_hires_18, ced_hires_18, },
+ 97, 228, 51, 26, 128,
+ {105,102,117,110,111,122,102,106, 97,93,168,92,91,102,30,86, 53,83,126,96,72,96,70,66, 56,74,134,61,61,50,82,84,
+ 173,124,96,100,127,93,113,131, 120,100,114,112,110,97,113,106, 131,100,95,101,117,100,100,131, 116,92,116,116,90,96,94,114,
+ 108,154,124,147,148,125,114,181, 110,139,111,106,104,133,107,106, 146,124,109,108,100,126,167,105, 99,95,111,104,173,187,163,102,
+ 108,154,121,114,150,117,103,190, 108,128,103,98,103,136,103,119, 177,122,113,108,105,127,163,98, 101,94,113,91,173,200,184,105,
+ 157,114,89,88,132,136,106,94, 88,90,191,83,103,92,21,83, 80,123,164,148,146,158,157,131, 101,129,189,106,99,97,60,118,
+ 182,145,102,152,141,111,120,142, 116,154,142,145,107,121,136,120, 155,111,121,104,137,100,101,159, 112,95,162,165,88,117,91,157,
+ 152,195,152,169,197,184,168,180, 179,205,169,140,116,192,148,134, 200,173,128,194,156,153,196,117, 179,108,175,138,198,216,209,171,
+ 181,200,141,171,198,195,168,167, 169,200,157,140,147,201,136,123, 200,173,148,196,144,154,194,118, 178,142,175,128,185,237,209,106,
+ },
+ {116,4,4,52,4,0,0,0, 0,144,148,0,0,163,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 225,145,184,86,101,112,168,165, 141,153,125,111,189,160,182,143, 136,151,146,136,133,132,125,124, 121,122,175,136,205,105,128,148,
+ 108,194,176,188,184,192,169,184, 174,197,159,189,200,191,205,183, 174,151,205,195,198,180,178,169, 146,174,178,113,109,127,105,136,
+ 126,195,178,194,186,194,171,181, 172,199,157,204,211,202,219,182, 177,139,210,201,200,185,177,168, 141,187,198,94,139,85,111,0,
+ 103,94,114,122,104,131,103,108, 103,87,157,89,94,91,51,79, 84,99,153,107,139,122,104,101, 94,118,157,108,103,94,118,122,
+ 173,126,101,99,126,94,115,123, 121,124,114,109,106,119,122,111, 133,112,90,112,118,97,110,129, 118,94,113,120,110,107,93,127,
+ 117,118,123,173,147,114,112,166, 112,142,110,109,107,167,114,104, 170,127,113,114,95,153,141,101, 99,92,103,107,131,170,176,150,
+ 116,123,113,173,147,118,110,169, 112,133,111,107,99,168,113,98, 188,128,113,112,119,153,142,91, 104,92,117,99,135,199,197,107,
+ },
+ {84,0,50,70,50,58,42,52, 90,82,80,100,68,64,68,44, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,2,2,2,2, 32,24,16,38,6,2,6,2, 62,0,2,22,2,10,2,4, 36,28,22,44,12,8,10,2,
+ 26,0,2,2,2,2,2,2, 2,2,2,4,2,2,2,2, 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 20,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 140,0,112,126,144,140,106,118, 180,152,152,160,192,130,148,104, 142,0,134,130,100,98,134,132, 138,168,142,178,148,152,184,152,
+ 164,0,136,162,116,118,104,106, 140,152,204,176,140,126,126,118, 160,0,144,152,118,124,102,106, 174,182,180,238,162,154,150,126,
+ 92,0,94,92,145,149,90,92, 190,154,156,172,184,168,132,134, 112,0,114,118,143,143,116,118, 140,146,112,138,166,182,162,148,
+ 132,0,138,124,44,56,131,135, 142,176,130,156,120,128,176,162, 128,0,132,132,50,60,137,135, 130,160,114,122,98,130,138,168,
+ },
+},
+
+{ // CP1253 (37.682M chars) [23]
+ {NULL, NULL, ced_hires_11, ced_hires_11, },
+ 186, 218, 73, 21, 128,
+ {98,79,82,91,78,122,85,90, 0,82,68,76,0,75,0,65, 79,105,101,115,101,127,107,94, 0,98,0,91,0,78,48,0,
+ 174,70,182,88,122,76,110,130, 119,95,70,153,108,88,101,71, 130,87,79,77,106,197,107,129, 184,175,187,114,184,75,166,169,
+ 118,206,179,192,195,207,164,187, 184,199,203,195,198,200,165,203, 202,197,70,205,210,185,180,184, 155,178,137,90,181,184,178,187,
+ 86,203,172,188,189,203,161,188, 181,199,199,194,198,198,162,203, 195,196,156,202,208,186,176,181, 148,179,137,85,184,167,169,70,
+ 158,116,88,89,133,138,107,95, 0,84,72,80,9,91,0,87, 58,101,143,127,125,137,136,110, 0,108,60,84,28,75,32,6,
+ 183,68,95,153,143,112,121,144, 117,156,66,147,109,122,138,78, 157,112,123,105,106,101,102,161, 98,99,87,166,100,118,81,80,
+ 77,163,130,131,140,152,105,166, 133,145,142,124,131,145,100,159, 140,114,66,160,135,141,123,127, 100,118,86,68,176,144,181,166,
+ 74,200,125,123,122,177,112,190, 115,197,147,142,142,198,114,189, 137,143,206,134,146,192,116,125, 82,149,116,80,182,162,153,66,
+ },
+ {118,0,0,0,0,0,0,0, 0,138,161,0,0,163,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,143,175,80,96,111,174,154, 134,154,123,101,184,149,186,143, 136,152,147,137,135,133,126,125, 122,123,168,139,197,118,114,121,
+ 107,133,118,132,121,129,117,114, 119,125,113,112,122,124,119,128, 125,97,124,130,131,115,120,120, 116,111,103,108,101,126,102,112,
+ 111,143,112,125,126,141,110,108, 120,135,105,107,128,122,124,134, 122,86,123,151,133,128,120,103, 90,103,102,77,126,78,111,0,
+ 91,79,78,82,79,127,85,84, 0,77,0,78,58,76,0,63, 76,87,132,84,113,123,101,97, 0,101,76,105,55,86,59,12,
+ 175,74,185,85,123,84,113,121, 119,111,74,101,104,87,118,74, 133,87,79,85,119,144,95,130, 181,183,190,144,180,79,178,172,
+ 112,208,162,180,176,202,160,198, 168,204,190,189,187,198,160,206, 190,199,70,200,200,190,175,174, 145,183,138,111,190,189,185,193,
+ 80,212,167,183,177,208,161,200, 173,208,193,194,190,201,165,209, 197,204,194,200,202,195,177,177, 147,186,139,111,184,182,175,72,
+ },
+ {120,0,80,110,184,184,160,166, 142,128,86,68,44,50,40,48, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 20,0,2,2,32,36,28,32, 64,50,2,2,2,2,2,2, 2,0,2,2,10,14,6,10, 78,62,14,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 70,56,6,2,2,2,2,2, 2,0,2,2,2,2,2,2, 58,46,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 32,20,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 38,26,2,2,2,2,2,2,
+ 168,0,138,162,188,202,172,168, 248,222,164,132,100,104,104,108, 150,0,142,148,180,174,188,192, 212,242,158,156,116,114,118,120,
+ 162,0,136,170,180,178,174,172, 148,160,176,108,132,134,106,104, 162,0,146,164,172,168,166,156, 118,142,100,98,134,138,126,114,
+ 144,0,152,152,156,152,144,136, 116,120,132,142,139,135,116,112, 146,0,150,152,142,134,126,118, 106,104,132,134,139,131,120,126,
+ 148,0,152,144,90,118,116,110, 122,122,68,86,30,122,137,139, 150,0,152,148,90,116,122,112, 122,132,66,88,12,118,139,137,
+ },
+},
+
+{ // CP932 (5.390M chars) [24]
+ {NULL, NULL, NULL, NULL},
+ 151, 140, 55, 26, 129,
+ {73,188,237,214,159,83,84,109, 191,199,191,194,200,199,202,199, 190,190,186,192,182,191,188,186, 105,102,112,102,102,102,107,115,
+ 101,107,119,112,119,135,100,99, 106,106,107,101,103,109,105,120, 121,117,114,106,104,110,111,112, 114,105,127,110,129,118,108,105,
+ 123,109,100,111,120,109,109,99, 105,106,114,108,122,112,116,106, 102,102,113,101,100,99,102,108, 110,108,119,107,108,116,122,117,
+ 117,116,111,115,112,109,113,115, 113,111,167,178,184,104,106,183, 160,142,115,156,149,130,163,163, 184,191,154,133,75,97,94,93,
+ 0,223,195,229,107,12,47,131, 44,180,189,184,179,185,191,185, 177,166,168,174,170,173,169,172, 162,86,89,86,93,87,87,91,
+ 99,113,100,104,106,122,111,98, 99,99,99,99,99,99,99,99, 105,99,100,99,98,99,99,102, 124,99,99,99,106,103,99,99,
+ 100,99,99,98,105,99,99,99, 99,101,99,99,112,99,98,99, 99,100,99,99,99,98,102,99, 99,103,99,99,99,104,106,104,
+ 115,99,101,108,103,106,99,99, 99,103,97,0,23,91,96,0, 105,107,95,165,148,101,154,130, 96,175,132,116,100,109,98,101,
+ },
+ {87,0,0,0,0,0,0,0, 0,84,91,0,0,87,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 114,86,99,84,84,84,86,86, 90,93,85,89,87,94,87,86, 91,88,89,86,85,85,84,84, 84,84,85,87,119,84,84,84,
+ 199,198,192,189,158,189,180,168, 171,180,178,171,176,161,191,183, 172,169,179,177,178,160,185,181, 190,169,175,200,180,158,185,172,
+ 177,159,190,175,162,179,173,191, 177,187,182,158,184,167,174,182, 179,167,177,179,178,179,187,176, 174,176,173,178,169,174,176,0,
+ 175,177,175,172,170,176,176,180, 169,182,188,188,178,179,172,172, 159,180,158,195,174,169,176,176, 162,162,166,158,168,174,176,177,
+ 179,175,195,161,185,163,180,172, 181,189,190,180,175,184,173,177, 166,185,175,184,166,192,178,192, 166,176,169,178,190,190,180,180,
+ 168,183,176,163,192,193,192,176, 191,192,177,162,203,189,171,167, 173,162,169,159,174,148,173,170, 163,160,175,166,188,175,165,174,
+ 186,163,176,177,162,176,185,182, 187,190,188,161,175,174,179,176, 192,187,162,164,156,172,160,145, 155,145,181,167,175,93,93,93,
+ },
+ {84,0,82,82,2,2,4,6, 2,2,8,8,4,18,2,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 128,0,126,120,6,10,10,14, 22,24,18,18,14,28,20,30, 128,0,128,126,2,8,8,12, 22,26,18,18,14,30,20,30,
+ 128,0,128,128,6,10,12,14, 26,28,22,22,18,32,24,34, 128,0,128,128,8,12,14,16, 26,30,22,22,18,32,24,34,
+ 68,0,44,44,2,2,2,2, 26,30,24,22,18,34,24,34, 66,0,40,42,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 0,0,0,0,140,138,140,138, 132,130,132,132,136,132,134,128, 0,0,0,0,134,140,142,142, 130,134,132,134,124,134,132,142,
+ 222,128,252,248,124,128,128,130, 148,136,134,126,124,132,86,120, 214,128,240,244,136,132,130,132, 136,118,136,132,126,152,84,118,
+ 212,128,244,242,126,130,132,134, 136,128,130,128,118,156,82,122, 212,128,242,242,126,130,132,134, 136,130,128,144,130,142,66,122,
+ 0,0,0,0,134,144,136,142, 138,144,128,150,72,88,76,86, 184,180,164,164,126,140,126,154, 134,144,126,132,124,128,136,134,
+ },
+},
+
+{ // Hebrew (10.753M chars) [25]
+ {NULL, NULL, NULL, NULL},
+ 196, 235, 78, 9, 128,
+ {0,0,0,0,0,0,0,0, 0,63,63,0,0,63,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 165,0,80,80,80,80,80,80, 80,82,80,80,82,89,80,80, 85,80,80,80,86,80,80,82, 80,80,83,80,80,80,80,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,80,
+ 205,203,189,201,216,218,185,196, 190,220,185,190,212,212,202,200, 199,194,199,175,193,168,187,196, 214,202,217,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,77,77,0,0,77,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 145,0,87,87,105,87,87,87, 88,138,88,88,88,89,105,87, 89,88,88,87,91,87,87,118, 87,87,87,88,88,89,87,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,87,
+ 194,196,170,172,206,189,164,179, 163,185,130,182,196,152,197,145, 174,174,179,130,174,124,162,174, 179,192,183,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,136,147,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,138,183,80,80,95,170,157, 104,163,103,95,150,163,151,133, 86,103,112,90,86,90,87,88, 85,86,126,96,204,95,106,111,
+ 82,84,81,84,82,82,81,81, 82,84,81,81,81,81,82,82, 82,80,85,94,81,81,80,82, 81,80,82,83,104,130,83,93,
+ 114,87,82,84,83,83,82,81, 82,88,80,81,82,82,82,86, 83,81,84,85,88,82,81,85, 80,82,80,81,92,108,97,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 165,0,81,80,80,80,80,81, 80,82,80,80,81,81,80,80, 86,80,81,80,85,80,80,83, 80,80,81,80,80,80,80,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,80,
+ 209,210,192,200,210,223,187,198, 189,223,131,200,211,161,212,142, 207,193,203,135,198,112,191,196, 208,208,204,0,0,0,0,0,
+ },
+ {180,0,128,152,204,204,204,202, 0,0,202,202,0,164,118,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 54,0,2,14,128,84,104,102, 0,0,66,128,0,128,2,2, 128,0,36,58,128,128,128,128, 0,0,98,128,0,128,4,16,
+ 128,0,128,128,128,128,128,128, 0,0,128,128,0,128,128,128, 128,0,76,98,128,128,128,128, 0,0,128,128,0,128,50,62,
+ 2,0,2,2,16,2,8,10, 0,0,2,38,0,78,2,2, 2,0,2,2,26,10,18,20, 0,0,6,56,0,98,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 154,0,146,158,230,204,230,232, 0,0,186,226,0,186,128,130, 186,0,128,164,228,226,226,226, 0,0,226,226,0,186,118,124,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 164,0,128,152,188,188,188,186, 0,0,186,186,0,148,116,124,
+ 150,0,150,150,136,142,142,136, 0,0,130,118,0,118,130,132, 154,0,150,154,146,158,148,158, 0,0,124,124,0,122,132,128,
+ },
+},
+
+{ // KOI8U (14.358M chars) [26]
+ {NULL, NULL, ced_hires_9, ced_hires_9, },
+ 189, 220, 69, 18, 129,
+ {87,87,87,87,87,87,87,87, 87,95,95,87,87,95,87,87, 84,84,84,84,84,84,84,84, 84,84,84,84,84,84,84,84,
+ 87,87,87,119,129,87,162,137, 87,87,87,87,87,90,87,87, 87,87,87,119,131,87,163,137, 87,87,87,87,87,92,87,87,
+ 156,198,183,173,193,200,169,185, 168,203,166,197,197,191,205,205, 196,169,201,203,201,185,173,196, 170,177,187,171,168,165,181,137,
+ 158,200,188,175,195,200,175,189, 171,204,166,200,198,196,207,207, 201,170,204,206,202,187,175,199, 170,177,189,174,172,166,183,137,
+ 82,82,82,82,82,82,82,82, 82,95,95,82,82,95,82,82, 56,56,56,56,56,56,56,56, 56,56,56,56,56,56,56,56,
+ 82,82,82,124,131,82,157,142, 82,82,82,82,82,83,82,82, 82,82,82,87,89,82,115,97, 82,82,82,82,82,82,82,103,
+ 164,191,135,141,166,194,119,163, 175,198,187,177,169,181,165,191, 138,192,163,174,180,171,125,189, 179,178,152,139,101,110,139,87,
+ 131,161,130,111,132,145,129,147, 126,157,123,145,139,148,148,152, 142,148,128,154,139,149,110,167, 115,120,124,107,123,95,110,92,
+ },
+ {0,0,0,0,0,0,0,0, 0,112,170,0,0,149,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,146,172,86,86,96,164,138, 117,148,96,95,187,167,194,124, 88,108,104,96,97,94,99,91, 91,91,165,131,188,92,116,147,
+ 126,98,86,90,87,99,86,86, 87,107,86,86,86,87,87,101, 112,86,87,87,86,86,87,91, 86,93,86,87,94,113,86,107,
+ 97,99,87,90,86,99,86,86, 87,108,86,86,87,87,88,102, 114,86,88,87,86,86,87,86, 86,94,86,86,113,86,87,0,
+ 87,87,87,86,86,86,86,86, 86,86,86,86,86,86,86,86, 86,86,86,86,86,86,86,86, 86,86,86,86,86,86,86,86,
+ 86,86,86,123,128,86,170,136, 86,86,86,86,86,87,86,86, 88,86,87,121,127,88,168,135, 88,86,86,87,87,87,86,87,
+ 169,212,182,169,188,211,160,181, 172,207,177,191,196,188,203,214, 181,185,201,197,203,191,170,194, 184,189,174,162,139,157,176,119,
+ 167,209,179,168,186,208,159,179, 171,205,177,190,193,187,202,210, 178,185,198,195,202,188,168,192, 184,187,173,162,133,156,176,118,
+ },
+ {132,0,98,138,200,200,200,198, 200,200,174,176,92,102,94,104, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 44,0,2,24,84,92,96,96, 128,128,46,48,2,2,2,2, 42,0,2,24,84,90,96,94, 128,128,88,90,14,24,16,26,
+ 2,0,2,2,10,14,16,16, 36,36,2,2,2,2,2,2, 2,0,2,2,16,20,22,22, 46,46,2,2,2,2,2,2,
+ 2,0,2,2,8,12,14,14, 128,128,26,28,2,2,2,2, 2,0,2,2,14,16,20,18, 112,128,30,32,2,2,2,2,
+ 174,0,126,164,224,224,224,222, 224,224,180,182,112,124,116,126, 200,0,128,168,224,224,224,222, 224,224,182,184,114,126,118,128,
+ 146,0,154,150,184,194,200,194, 224,224,170,142,134,140,78,88, 158,0,142,170,224,224,224,222, 224,224,140,172,104,106,136,142,
+ 156,0,154,152,148,140,142,142, 118,120,138,74,139,139,16,24, 154,0,154,156,130,146,148,152, 126,124,138,84,139,141,18,30,
+ 136,0,154,156,180,186,186,174, 116,118,108,138,112,106,141,139, 140,0,152,158,182,184,182,178, 122,122,112,138,114,110,139,141,
+ },
+},
+
+{ // ISO-8859-5 (4.566M chars) [27]
+ {NULL, NULL, NULL, NULL},
+ 178, 203, 63, 18, 128,
+ {0,0,0,0,0,0,0,0, 0,73,73,0,0,73,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,139,153,105,121,99,146,124, 153,123,130,127,103,100,117,115, 203,190,202,193,196,200,173,190, 205,165,202,200,198,206,204,202,
+ 204,206,200,185,176,172,175,186, 177,166,129,174,167,168,175,170, 198,179,195,186,191,199,170,186, 202,165,197,197,190,203,202,194,
+ 202,202,199,184,167,167,172,183, 170,164,142,174,169,160,162,170, 145,139,152,104,117,99,146,124, 151,121,129,126,103,100,117,115,
+ 0,0,0,0,0,0,0,0, 0,96,96,0,0,96,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 183,107,100,119,101,102,114,101, 108,107,102,105,110,123,100,105, 155,132,162,147,135,132,126,125, 144,118,136,128,135,134,143,136,
+ 129,148,131,134,127,123,119,121, 119,113,118,126,116,118,119,138, 196,133,191,156,166,192,126,148, 197,189,175,160,180,171,185,142,
+ 170,174,180,171,119,174,145,137, 136,107,110,179,181,109,162,193, 112,129,101,118,119,103,147,123, 126,105,102,131,111,99,125,102,
+ },
+ {0,0,0,0,0,0,0,0, 0,114,146,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 222,138,169,99,103,106,165,146, 132,152,105,105,185,162,182,129, 128,138,133,123,122,122,115,115, 112,113,166,133,204,103,107,138,
+ 107,127,114,123,114,133,112,117, 112,112,116,112,118,118,113,127, 126,101,118,122,116,107,107,118, 101,108,104,104,106,112,100,115,
+ 103,127,109,118,116,128,106,107, 105,117,110,111,115,113,118,136, 126,100,117,119,115,110,109,117, 100,113,108,99,124,100,99,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,147,138,136,115,100,153,120, 139,118,125,128,109,100,117,138, 209,174,192,178,187,205,163,176, 205,178,190,191,185,200,208,172,
+ 198,193,199,188,157,168,168,173, 166,150,130,185,182,133,166,183, 215,178,196,181,189,211,165,178, 208,178,192,200,187,202,213,176,
+ 204,195,202,197,156,173,167,173, 161,154,133,189,185,135,171,185, 146,148,138,136,116,99,155,119, 139,120,128,128,109,103,118,137,
+ },
+ {176,0,130,152,216,216,216,214, 0,0,182,122,134,118,130,216, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,42,50,44,50, 0,0,16,2,2,2,2,24, 2,0,2,2,2,2,2,2, 0,0,18,2,2,2,2,26,
+ 2,0,2,2,2,4,2,4, 0,0,32,2,2,2,2,42, 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,8,2,8, 0,0,2,2,2,2,2,2, 46,0,2,4,54,64,54,64, 0,0,38,2,2,2,2,48,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 178,0,132,168,188,188,180,186, 0,0,192,126,130,94,98,138, 140,0,150,154,182,178,186,182, 0,0,122,138,138,120,116,102,
+ 146,0,146,160,198,200,184,190, 0,0,126,138,140,118,114,104, 148,0,154,154,116,130,140,130, 0,0,78,28,40,138,138,134,
+ 156,0,154,148,128,138,132,134, 0,0,88,38,50,138,140,144, 150,0,152,140,186,198,198,204, 0,0,142,88,100,130,142,182,
+ },
+},
+
+{ // CP874 (90.889M chars) [28]
+ {NULL, NULL, NULL, NULL},
+ 183, 197, 49, 21, 127,
+ {105,102,97,58,82,121,0,91, 87,94,89,61,69,81,0,58, 84,99,96,131,98,112,94,97, 77,90,89,80,56,80,84,0,
+ 175,209,195,89,202,83,128,203, 192,151,191,169,87,166,132,138, 156,134,149,172,200,200,176,200, 176,211,199,193,179,149,192,164,
+ 175,205,201,213,136,199,77,202, 175,173,203,200,140,209,144,122, 197,203,215,179,198,198,169,189, 190,187,93,58,56,58,53,74,
+ 209,196,186,186,191,106,151,183, 207,205,151,138,179,120,113,126, 122,127,132,111,120,111,102,99, 97,95,58,89,84,72,69,79,
+ 159,116,43,30,27,138,0,65, 1,58,62,0,19,89,0,87, 48,92,134,118,116,128,127,100, 0,39,50,0,17,66,20,0,
+ 184,175,143,83,159,81,91,181, 155,114,141,116,87,135,98,96, 126,89,93,153,172,160,139,156, 133,188,176,156,101,93,162,122,
+ 113,182,178,173,85,158,76,169, 155,135,151,108,93,170,98,140, 166,90,179,144,149,166,81,80, 132,137,85,52,53,62,58,119,
+ 116,99,98,96,101,93,161,101, 173,168,107,110,179,86,93,109, 117,116,114,109,105,106,99,96, 94,95,54,89,94,56,64,98,
+ },
+ {119,0,0,44,0,0,0,0, 0,138,149,0,0,161,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 209,129,181,84,86,94,166,157, 152,153,115,111,167,151,188,147, 133,142,138,128,126,125,117,113, 111,112,165,104,202,96,121,126,
+ 106,131,115,123,116,126,111,111, 114,120,111,105,118,121,113,118, 121,94,117,127,130,114,114,118, 116,109,100,126,115,121,92,108,
+ 78,142,110,112,119,136,104,98, 116,133,97,101,116,118,116,127, 119,79,119,150,131,126,114,102, 84,97,98,64,134,70,101,0,
+ 99,94,75,63,80,121,0,92, 94,84,92,59,78,81,49,49, 77,82,103,84,123,111,96,101, 80,100,87,78,55,86,81,0,
+ 175,206,190,92,199,79,139,206, 190,148,188,166,94,168,152,144, 148,139,143,179,199,195,177,200, 172,213,199,192,172,143,191,161,
+ 174,207,201,213,153,202,68,203, 176,170,200,196,121,209,144,145, 194,206,217,188,201,202,172,189, 193,191,94,68,66,68,64,77,
+ 203,186,177,183,187,119,161,181, 208,206,149,139,187,118,121,123, 127,115,118,111,108,125,108,109, 106,112,74,99,91,72,79,91,
+ },
+ {110,0,78,92,172,172,158,156, 114,122,40,36,30,34,38,118, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 54,48,2,2,2,2,2,36, 2,0,2,2,2,2,2,2, 54,46,2,2,2,2,2,34,
+ 2,0,2,2,2,2,2,2, 54,46,2,2,2,2,2,34, 2,0,2,2,2,2,2,2, 64,58,2,2,2,2,2,44,
+ 2,0,2,2,2,2,2,2, 62,56,2,2,2,2,2,42, 66,0,18,30,108,104,86,80, 128,128,16,12,6,8,14,128,
+ 174,0,152,160,182,202,168,162, 246,218,114,92,92,80,144,174, 156,0,156,144,186,176,196,194, 204,246,128,124,120,72,138,184,
+ 164,0,158,164,172,166,158,160, 106,118,126,118,128,132,130,82, 156,0,162,160,124,144,118,126, 140,136,120,122,126,134,130,94,
+ 156,0,162,160,120,152,110,128, 92,114,126,124,126,134,128,90, 164,0,160,162,132,138,124,138, 88,116,134,134,128,66,132,86,
+ 160,0,160,160,130,146,126,132, 98,116,132,134,134,114,112,128, 156,0,164,148,202,212,190,188, 168,192,88,80,70,70,134,254,
+ },
+},
+
+{ // ISO-8859-13 (0.207M chars) [29]
+ {NULL, NULL, ced_hires_20, ced_hires_20, },
+ 87, 221, 44, 20, 128,
+ {0,0,0,0,0,0,0,0, 0,140,140,0,0,140,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 172,141,141,141,142,142,141,144, 141,141,141,141,141,141,141,141, 142,141,141,141,141,141,141,144, 141,141,141,142,141,141,141,141,
+ 145,148,162,142,153,142,141,165, 141,149,141,156,144,147,157,156, 180,141,166,141,141,141,143,141, 141,141,141,151,142,141,162,141,
+ 145,149,162,141,154,141,141,165, 142,147,141,156,144,146,157,156, 178,141,165,141,141,141,151,141, 142,141,141,151,145,141,162,141,
+ 0,0,0,0,0,0,0,0, 0,103,103,0,0,103,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 179,147,141,155,148,163,142,148, 141,157,141,149,141,141,147,141, 156,141,143,141,164,141,141,160, 142,141,141,164,141,141,141,141,
+ 171,188,209,141,195,182,160,197, 197,203,141,208,168,155,202,175, 209,170,178,192,141,156,194,142, 172,141,141,197,196,141,202,170,
+ 203,197,216,141,196,193,181,199, 198,198,141,215,167,153,203,176, 209,184,177,194,141,156,192,141, 218,141,141,197,184,141,201,147,
+ },
+ {0,0,0,0,0,0,0,0, 0,147,156,0,0,163,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,155,187,141,141,141,205,156, 147,165,143,141,191,150,188,145, 145,153,151,146,146,145,143,143, 142,143,177,149,200,141,141,157,
+ 141,192,188,189,194,190,165,186, 169,203,196,195,196,191,201,180, 177,147,207,210,204,181,186,168, 149,175,171,141,141,143,141,141,
+ 141,193,189,186,194,193,163,185, 166,205,196,195,195,191,200,180, 175,144,206,209,204,182,190,168, 147,177,171,141,143,141,141,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 172,142,142,141,143,141,142,142, 141,142,141,141,142,141,142,143, 144,141,141,141,144,142,141,144, 142,141,142,143,141,142,142,143,
+ 146,151,168,141,152,142,142,159, 160,146,141,160,148,161,157,152, 174,141,150,141,141,141,143,141, 154,141,141,149,142,141,160,154,
+ 147,151,168,141,152,142,142,159, 161,143,141,160,148,161,160,152, 174,141,150,141,141,141,143,141, 154,141,141,150,142,141,160,141,
+ },
+ {158,0,124,150,126,130,126,128, 0,0,170,174,162,166,162,166, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 24,0,2,2,2,2,2,2, 0,0,2,6,2,2,2,2, 30,0,2,2,2,2,2,2, 0,0,6,12,2,2,2,4,
+ 18,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 18,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 162,0,130,160,124,122,116,118, 0,0,184,168,156,160,156,160, 160,0,136,152,118,120,118,120, 0,0,168,176,162,164,162,166,
+ 112,0,92,110,143,143,98,102, 0,0,160,164,168,178,152,154, 116,0,94,110,143,141,116,110, 0,0,156,162,178,166,152,152,
+ 130,0,134,134,68,72,135,137, 0,0,158,164,152,156,168,178, 126,0,136,124,74,78,137,135, 0,0,156,162,150,158,178,166,
+ },
+},
+
+{ // Latin4 (1.274M chars) [30]
+ {NULL, NULL, ced_hires_17, ced_hires_17, },
+ 82, 215, 39, 25, 128,
+ {0,0,0,0,0,0,0,0, 0,115,115,0,0,115,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 173,117,116,117,126,117,148,132, 125,177,162,132,116,116,164,120, 128,118,116,117,124,117,148,116, 124,175,162,130,116,116,163,116,
+ 157,149,123,151,148,122,117,138, 167,143,123,118,151,148,118,153, 121,162,122,139,120,119,129,118, 117,116,148,118,124,121,142,117,
+ 157,149,118,127,151,122,117,137, 166,138,123,117,150,149,118,153, 120,162,122,131,121,119,146,117, 118,128,146,116,134,116,142,117,
+ 0,0,0,0,0,0,0,0, 0,70,70,0,0,70,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 181,199,116,118,142,116,174,142, 124,175,196,171,116,126,169,127, 155,213,117,118,137,116,173,125, 122,173,197,168,117,142,170,118,
+ 207,195,152,169,196,183,168,186, 199,205,192,140,210,191,148,201, 147,178,118,155,156,154,195,123, 178,174,175,140,198,117,198,171,
+ 213,200,142,170,197,194,168,195, 200,199,205,141,216,201,138,202, 146,177,121,146,145,154,194,125, 178,220,175,131,185,119,198,126,
+ },
+ {57,0,0,38,0,0,0,0, 0,131,151,0,0,166,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,152,180,117,117,121,211,153, 140,167,128,117,189,148,187,138, 132,143,139,132,130,129,124,123, 122,123,176,138,202,117,118,154,
+ 118,186,190,201,200,189,167,191, 171,184,194,192,197,190,201,180, 181,140,208,211,206,171,184,129, 140,156,180,118,122,126,119,124,
+ 127,186,190,200,200,192,165,190, 168,185,194,192,197,189,200,179, 179,135,207,211,206,172,186,130, 136,162,180,118,128,117,119,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 173,117,117,117,128,123,146,126, 126,173,155,140,117,124,156,121, 126,119,117,117,121,134,147,117, 125,173,155,141,136,134,156,135,
+ 164,151,124,120,147,123,118,143, 164,136,118,120,153,166,122,151, 121,140,124,152,118,120,129,119, 117,154,119,117,123,117,140,151,
+ 164,153,120,117,148,124,119,144, 165,128,118,120,153,167,121,156, 125,142,122,153,127,119,128,121, 118,154,121,117,120,117,144,118,
+ },
+ {162,0,122,152,126,130,126,128, 0,0,168,174,170,184,168,186, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 20,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 36,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 36,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 142,0,104,138,144,136,100,96, 0,0,180,140,178,180,148,160, 132,0,136,126,68,68,138,130, 0,0,140,176,140,152,182,186,
+ 108,0,90,112,143,143,98,98, 0,0,176,164,182,164,150,154, 118,0,100,112,139,145,112,108, 0,0,180,154,176,174,158,170,
+ 134,0,132,134,46,52,137,137, 0,0,142,182,140,160,182,166, 120,0,140,126,56,60,127,133, 0,0,152,184,148,180,174,174,
+ },
+},
+
+{ // MACINTOSH (7.890M chars) [31]
+ {NULL, NULL, NULL, NULL},
+ 93, 176, 57, 40, 128,
+ {157,149,165,157,149,138,123,160, 152,165,162,148,156,168,162,155, 132,145,140,141,135,131,120,139, 145,132,114,87,138,116,126,107,
+ 122,121,109,120,124,153,124,137, 115,116,118,128,124,104,106,113, 104,117,104,104,109,107,139,107, 126,104,109,117,116,104,105,110,
+ 111,159,167,120,108,142,122,128, 139,154,202,125,151,121,112,145, 122,135,125,128,127,169,111,105, 105,105,106,131,106,115,157,151,
+ 178,165,177,166,176,175,150,161, 175,138,169,172,167,178,180,169, 181,177,176,161,145,151,149,162, 150,150,134,154,149,146,136,108,
+ 187,160,176,222,183,180,186,200, 209,171,190,175,171,176,227,198, 178,155,199,157,162,155,181,198, 159,170,176,133,176,169,177,184,
+ 153,174,115,162,149,204,131,171, 181,164,183,160,143,100,144,155, 100,123,132,132,125,113,123,102, 131,114,100,143,166,100,142,153,
+ 167,156,133,122,124,144,123,171, 183,183,202,158,173,141,144,148, 197,182,196,197,174,218,115,99, 113,111,102,198,116,150,173,165,
+ 163,179,162,158,169,174,179,196, 171,195,195,169,176,142,198,171, 137,149,180,180,122,144,118,130, 126,177,118,147,152,115,134,107,
+ },
+ {42,0,13,0,0,0,0,0, 0,162,166,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 232,145,182,109,125,120,185,162, 148,164,123,123,190,165,187,146, 152,167,161,152,152,149,142,140, 139,140,164,140,212,106,144,148,
+ 114,197,179,197,187,193,175,187, 174,172,164,164,193,190,204,189, 182,167,202,202,200,165,181,159, 152,152,154,137,122,135,112,138,
+ 124,201,182,197,188,200,175,187, 178,184,163,166,195,192,204,192, 182,163,203,213,204,179,184,151, 152,150,156,105,146,106,124,0,
+ 154,118,136,147,125,135,127,149, 150,171,149,158,123,138,165,150, 145,167,173,164,158,153,119,174, 161,141,125,139,166,124,149,128,
+ 121,135,106,117,127,149,123,154, 119,135,121,131,129,103,106,107, 103,121,103,103,108,106,141,104, 125,103,103,109,119,103,106,107,
+ 121,146,166,124,112,147,123,119, 142,156,201,137,158,141,109,143, 126,135,114,134,124,136,109,103, 105,103,103,132,103,152,150,164,
+ 189,156,172,160,169,185,143,151, 179,154,167,169,166,174,187,145, 178,169,174,164,137,148,147,154, 144,138,124,165,164,121,147,108,
+ },
+ {64,0,40,60,48,54,44,48, 40,32,58,74,24,52,16,30, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 30,0,2,2,2,2,2,2, 2,2,2,14,2,2,2,2, 40,0,2,14,2,8,2,2, 14,8,34,52,2,28,2,6,
+ 2,0,2,2,2,2,2,2, 2,2,2,14,2,2,2,2, 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,4,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,14,30,2,8,2,2,
+ 110,0,128,112,130,132,130,130, 172,160,146,144,140,156,152,144, 116,0,126,118,72,74,142,134, 166,176,152,142,114,142,138,132,
+ 150,0,140,144,110,108,104,102, 150,144,198,192,158,160,124,142, 118,0,138,130,110,122,126,128, 152,146,184,206,166,166,128,142,
+ 154,0,132,150,124,122,106,110, 110,102,160,166,174,156,114,104, 126,0,130,136,116,114,128,136, 178,160,156,162,138,174,134,142,
+ 118,0,122,114,142,144,108,96, 120,128,142,140,104,154,162,162, 124,0,132,118,132,136,124,118, 114,100,130,140,112,150,162,164,
+ },
+},
+
+{ // GB18030 (0.640M chars) [32]
+ {NULL, NULL, NULL, NULL},
+ 202, 189, 30, 18, 127,
+ {107,143,144,138,146,135,139,141, 137,141,140,132,146,144,130,130, 137,138,134,135,134,142,138,131, 141,139,135,139,136,137,138,133,
+ 130,205,136,202,219,216,134,132, 133,158,133,130,130,130,130,130, 190,196,199,196,196,209,197,199, 197,204,195,200,202,198,193,195,
+ 193,196,190,195,196,183,196,192, 202,197,207,198,197,198,202,200, 202,191,204,201,199,197,203,200, 140,132,132,140,134,135,133,131,
+ 132,133,132,133,146,134,134,134, 133,133,134,132,135,132,132,131, 133,132,131,131,133,132,135,135, 130,130,130,130,130,130,130,0,
+ 150,126,124,127,127,133,129,124, 132,124,123,123,132,122,120,121, 35,63,39,52,35,68,64,64, 75,69,63,67,52,52,39,65,
+ 121,121,121,121,122,120,121,121, 126,122,121,120,121,120,121,121, 121,121,121,120,121,121,120,121, 121,121,120,121,122,121,121,121,
+ 121,121,121,121,121,120,120,120, 121,120,120,121,121,121,121,120, 120,120,121,120,120,120,120,121, 121,121,121,121,121,121,121,121,
+ 121,120,121,120,121,120,120,120, 121,121,121,121,120,121,120,120, 120,120,120,122,120,120,120,121, 121,121,120,120,121,121,120,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,79,79,0,0,79,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 79,85,79,79,79,79,95,79, 83,83,79,93,93,79,87,79, 126,115,111,106,110,106,103,106, 108,108,79,79,140,83,79,110,
+ 132,131,132,136,135,131,132,131, 131,141,131,133,131,132,132,133, 131,133,131,132,138,132,132,137, 134,131,131,131,131,133,136,131,
+ 131,131,131,131,131,132,132,133, 131,131,131,131,134,131,131,135, 131,131,139,131,132,131,131,132, 131,132,132,131,134,131,131,0,
+ 138,132,131,134,130,131,132,130, 130,130,131,132,131,134,133,133, 131,134,137,134,131,130,133,133, 130,131,132,131,130,132,134,136,
+ 138,196,198,193,197,176,186,185, 196,185,197,195,199,187,185,192, 190,191,180,190,190,195,182,192, 182,191,188,196,190,191,188,193,
+ 187,175,193,194,203,190,193,195, 191,189,190,196,182,188,196,192, 200,179,185,187,190,187,193,181, 193,177,196,183,186,180,183,186,
+ 186,188,190,186,184,188,186,187, 178,191,190,192,188,188,182,181, 179,186,189,196,174,178,187,193, 193,183,197,187,178,188,186,0,
+ },
+ {218,0,240,172,146,146,148,146, 120,44,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 56,0,26,2,2,2,2,2, 74,70,2,2,2,2,2,2, 50,0,20,2,2,2,2,2, 84,76,2,2,2,2,2,2,
+ 50,0,22,2,2,2,2,2, 86,76,2,2,2,2,2,2, 58,0,28,2,2,2,2,2, 86,76,2,2,2,2,2,2,
+ 128,0,128,64,36,36,38,38, 86,76,2,2,2,2,2,2, 128,0,128,68,40,40,42,42, 84,76,2,2,2,2,2,2,
+ 0,0,0,212,188,194,188,186, 180,180,120,114,116,122,126,126, 0,0,0,192,194,188,194,194, 176,176,116,120,116,122,136,128,
+ 0,0,0,194,206,204,216,214, 118,118,136,130,128,116,126,112, 0,0,0,198,208,208,228,220, 112,112,126,126,128,126,126,130,
+ 0,0,0,188,210,208,232,222, 112,112,120,126,126,126,130,130, 0,0,0,192,210,208,232,222, 120,120,124,128,126,134,120,124,
+ 0,0,0,188,210,208,236,222, 176,176,128,122,120,122,124,124, 0,0,0,180,210,208,236,222, 178,178,122,122,122,124,126,124,
+ },
+},
+
+{ // CP852 (9.112M chars) [33]
+ {NULL, NULL, NULL, NULL},
+ 85, 183, 47, 36, 128,
+ {147,99,153,98,113,162,97,141, 139,113,105,98,99,105,112,106, 117,111,78,82,68,70,92,81, 93,63,63,89,87,89,89,145,
+ 187,171,114,163,117,105,174,173, 163,149,96,97,180,97,98,99, 96,96,96,96,96,186,101,174, 97,96,96,96,96,99,100,96,
+ 96,96,96,96,96,96,112,102, 96,96,96,96,96,96,96,96, 97,99,112,109,110,120,170,99, 174,96,96,96,96,98,162,96,
+ 115,98,115,99,99,120,184,183, 99,167,99,97,153,153,98,101, 97,97,96,97,96,158,99,96, 100,97,97,100,191,191,96,158,
+ 178,142,216,118,134,202,143,174, 177,140,130,157,125,109,134,147, 194,166,135,125,142,148,164,161, 172,142,141,143,163,165,129,208,
+ 222,229,177,175,143,129,196,201, 175,165,92,102,208,119,112,129, 92,92,92,92,92,217,114,205, 120,92,92,92,92,119,130,92,
+ 92,92,92,92,92,92,117,134, 95,95,92,92,92,92,92,99, 100,101,140,134,151,163,210,124, 213,92,92,92,92,116,181,94,
+ 176,147,127,120,122,171,198,197, 98,182,97,109,208,197,117,134, 100,102,93,126,93,173,129,92, 111,100,112,145,203,203,92,168,
+ },
+ {0,0,0,0,0,0,0,0, 0,124,152,0,0,168,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 234,148,183,99,97,113,164,166, 117,160,105,108,196,160,193,147, 123,151,151,145,144,134,140,137, 123,116,177,135,205,109,140,149,
+ 99,185,176,199,192,207,123,147, 188,196,177,198,197,199,208,171, 173,98,190,193,202,160,196,129, 101,116,185,101,102,119,97,112,
+ 97,188,176,199,192,210,124,151, 188,197,178,199,198,199,208,173, 176,98,191,194,203,165,197,129, 100,117,186,97,109,97,97,0,
+ 127,96,131,106,97,136,102,129, 109,96,96,129,100,96,97,102, 122,147,109,96,96,114,120,101, 101,96,96,149,150,109,105,174,
+ 176,201,108,134,125,115,179,179, 153,141,96,96,173,96,98,99, 96,96,96,96,96,174,106,99, 96,96,96,96,96,110,110,96,
+ 96,96,96,96,96,96,98,98, 96,96,96,96,96,96,96,98, 97,97,137,96,141,138,199,98, 99,96,96,96,96,100,136,96,
+ 108,105,96,97,97,138,174,174, 99,131,100,96,100,100,100,102, 116,100,96,107,96,115,97,96, 100,97,97,99,180,181,96,164,
+ },
+ {92,0,50,80,56,70,54,68, 82,58,30,64,104,40,54,46, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 8,0,2,2,2,2,2,2, 2,2,2,2,6,2,2,2, 20,0,2,2,2,2,2,2, 4,2,2,2,24,2,2,2,
+ 78,0,10,38,14,28,12,26, 96,60,28,64,128,38,54,46, 26,0,2,2,2,2,2,2, 2,2,2,2,24,2,2,2,
+ 14,0,2,2,2,2,2,2, 2,2,2,2,24,2,2,2, 8,0,2,2,2,2,2,2, 10,2,2,2,32,2,2,2,
+ 138,0,138,136,100,100,126,124, 180,178,162,150,190,144,146,142, 122,0,120,134,128,124,134,128, 182,146,170,148,172,110,140,132,
+ 132,0,132,130,120,106,130,132, 158,170,152,172,148,158,164,162, 86,0,86,90,142,152,50,60, 150,144,154,132,162,132,174,174,
+ 148,0,132,140,116,136,116,134, 232,176,140,174,232,152,168,164, 128,0,124,124,134,132,122,132, 156,160,164,142,166,136,162,164,
+ 124,0,124,120,130,132,128,132, 156,156,160,132,156,166,160,146, 118,0,96,134,136,98,136,108, 136,130,160,158,150,168,118,150,
+ },
+},
+
+{ // Arabic (0.205M chars) [34]
+ {NULL, NULL, NULL, NULL},
+ 180, 214, 71, 14, 128,
+ {0,0,0,0,0,0,0,0, 0,121,121,0,0,121,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 136,0,0,0,136,0,0,0, 0,0,0,0,138,136,0,0, 0,0,0,0,0,0,0,0, 0,0,0,136,0,0,0,152,
+ 0,151,167,198,147,188,171,230, 205,155,210,171,192,196,187,199, 176,204,173,195,191,195,175,181, 170,203,174,0,0,0,0,0,
+ 182,201,194,197,213,218,204,199, 211,169,210,171,161,136,141,183, 141,152,136,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,133,133,0,0,133,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 145,0,0,0,145,0,0,0, 0,0,0,0,184,145,0,0, 0,0,0,0,0,0,0,0, 0,0,0,148,0,0,0,159,
+ 0,171,146,149,145,145,147,191, 180,203,194,168,171,165,164,185, 152,192,159,171,155,163,167,168, 152,175,156,0,0,0,0,0,
+ 152,177,173,172,190,189,191,184, 173,192,184,156,145,145,146,145, 145,149,145,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,137,171,0,0,150,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,142,169,137,137,137,163,144, 137,152,138,137,171,141,178,147, 138,139,137,137,137,138,137,137, 137,137,167,137,194,137,137,137,
+ 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,138,137,137,142,
+ 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 137,164,151,154,137,159,165,165, 160,155,154,156,173,137,172,165, 170,158,160,163,159,161,154,161, 189,173,179,140,185,172,164,157,
+ 0,171,144,171,160,163,176,218, 199,202,201,175,192,195,189,204, 182,210,177,202,184,184,183,185, 163,200,179,0,0,0,0,0,
+ 160,196,197,191,226,205,211,195, 209,178,216,158,137,138,147,151, 146,154,138,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {158,0,126,158,180,178,180,180, 0,0,144,124,130,132,124,178, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 52,0,2,28,50,50,50,52, 0,0,2,2,2,2,2,34, 52,0,2,28,50,50,50,52, 0,0,20,10,2,2,2,56,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,16,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 48,0,2,24,46,46,46,48, 0,0,22,12,2,2,2,58,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 166,0,150,140,158,156,158,158, 0,0,146,124,130,132,122,178, 154,0,138,166,170,170,170,170, 0,0,134,162,120,124,114,168,
+ 134,0,150,152,144,144,144,144, 0,0,106,68,128,132,132,124, 144,0,150,152,154,154,154,154, 0,0,110,76,134,124,130,130,
+ 156,0,150,144,146,146,146,146, 0,0,140,142,130,128,126,130, 156,0,134,156,178,178,178,178, 0,0,138,116,132,130,128,174,
+ },
+},
+
+{ // BIG5_HKSCS (0.271M chars) [35]
+ {NULL, NULL, NULL, NULL},
+ 157, 175, 62, 14, 128,
+ {0,0,0,0,0,0,0,0, 108,149,150,140,0,150,152,140, 132,132,131,132,132,132,132,132, 132,132,132,131,131,132,131,130,
+ 139,181,164,132,214,207,204,195, 203,198,208,192,195,201,196,187, 202,189,195,196,188,194,190,182, 195,188,192,186,188,188,191,185,
+ 184,180,185,187,180,189,169,181, 135,142,140,142,141,141,140,141, 141,140,140,140,141,140,141,140, 140,140,142,140,140,141,141,140,
+ 140,140,140,140,140,146,149,145, 143,146,140,141,149,140,140,140, 140,140,140,140,140,140,140,140, 140,151,141,140,141,139,140,0,
+ 0,0,0,0,0,0,0,0, 134,133,133,134,0,124,134,133, 122,122,122,122,122,122,122,122, 122,122,122,121,121,122,122,121,
+ 132,216,163,137,212,204,205,192, 184,187,190,192,193,185,187,179, 188,190,183,193,178,189,189,198, 181,189,172,184,183,181,173,166,
+ 178,176,172,174,168,176,168,176, 163,140,141,134,134,134,135,134, 134,134,135,135,135,135,134,134, 135,135,134,134,134,134,134,134,
+ 134,134,134,134,134,134,135,136, 134,134,134,134,134,134,135,134, 134,134,134,134,134,136,135,134, 134,134,144,134,134,134,134,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,81,81,0,0,81,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,81,81,81,81,81,81,81, 81,81,81,81,81,81,81,81, 81,81,81,81,81,81,81,81, 81,81,81,81,85,81,81,81,
+ 200,202,189,193,189,173,185,189, 193,179,177,176,184,185,182,192, 186,181,175,172,181,185,166,189, 183,172,168,173,170,186,179,178,
+ 174,184,179,166,182,181,171,178, 178,187,186,179,178,172,187,186, 185,189,169,194,178,186,186,183, 180,180,179,177,186,177,193,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 157,191,170,191,186,176,168,184, 181,177,175,185,177,176,169,168, 182,178,179,190,172,180,184,171, 183,186,201,182,169,184,171,170,
+ 185,180,173,171,183,175,187,182, 183,185,183,177,184,178,189,186, 187,188,183,195,166,177,177,182, 181,174,185,182,180,178,175,181,
+ 176,176,179,175,179,183,187,172, 179,192,190,179,187,180,169,178, 168,171,175,183,191,176,182,182, 188,183,186,178,183,179,177,0,
+ },
+ {232,0,254,254,110,120,118,114, 128,128,4,2,2,2,2,4, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 40,0,16,16,2,2,2,2, 128,128,2,2,2,2,2,2, 46,0,24,22,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 66,0,42,40,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 128,0,128,128,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 0,0,0,0,132,142,144,140, 0,0,140,124,126,126,126,126, 0,0,0,0,134,142,142,138, 0,0,136,134,134,134,134,134,
+ 0,0,0,0,142,138,138,134, 0,0,136,138,132,132,134,134, 0,0,0,0,132,138,140,144, 0,0,134,128,138,138,132,136,
+ 0,0,0,0,134,144,136,140, 0,0,126,134,132,132,142,138, 0,0,0,0,134,144,142,138, 0,0,136,134,134,134,136,136,
+ 0,0,0,0,134,144,142,138, 0,0,144,132,132,132,132,132, 0,0,0,0,136,142,140,138, 0,0,136,132,134,136,134,138,
+ },
+},
+
+{ // CP866 (75.238M chars) [36]
+ {NULL, NULL, NULL, NULL},
+ 144, 168, 54, 34, 129,
+ {202,189,201,189,198,201,176,191, 205,171,201,198,196,207,207,203, 202,204,202,184,177,168,173,180, 170,166,142,175,168,167,156,168,
+ 200,185,197,186,194,201,174,188, 204,171,197,197,192,205,205,197, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 203,204,203,185,172,167,174,181, 171,168,145,178,171,163,156,169, 122,121,61,61,64,61,61,61, 82,60,60,60,60,60,60,149,
+ 155,129,160,132,130,136,112,122, 155,123,142,126,140,137,156,133, 109,124,109,108,94,99,94,88, 84,75,58,94,87,91,94,119,
+ 193,150,189,167,164,195,129,155, 199,188,180,165,183,167,190,143, 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55,
+ 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55,
+ 172,176,181,172,118,175,146,138, 137,99,105,181,182,104,163,194, 90,129,56,55,55,62,55,57, 106,70,89,55,55,55,55,167,
+ },
+ {0,0,0,0,0,0,0,0, 0,118,150,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 221,147,180,74,65,105,160,149, 120,156,94,110,186,165,186,136, 108,128,124,115,112,121,107,104, 100,97,169,132,195,113,104,144,
+ 134,102,74,101,82,91,78,71, 88,102,80,94,92,80,75,94, 112,60,73,125,77,72,89,82, 78,91,81,80,102,111,73,107,
+ 93,109,104,99,117,99,90,77, 101,105,109,107,97,101,119,121, 134,59,107,112,113,105,117,82, 72,94,116,60,90,75,74,0,
+ 210,180,193,179,186,209,168,174, 206,179,191,195,187,202,211,177, 198,196,203,188,161,170,168,174, 161,156,124,189,187,127,169,186,
+ 213,183,195,180,188,212,170,177, 208,179,193,198,188,204,215,181, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 202,198,205,193,164,172,168,175, 162,157,125,191,187,137,171,187, 127,130,61,61,67,67,60,61, 82,60,61,60,66,61,60,147,
+ },
+ {2,0,2,2,50,52,44,42, 12,26,10,176,176,176,22,96, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,4,6,2,2, 2,2,2,68,68,68,2,8, 128,0,58,84,128,128,128,128, 52,68,48,128,128,128,66,128,
+ 128,0,58,84,128,128,128,128, 52,68,48,128,128,128,66,128, 128,0,58,84,128,128,128,128, 52,68,48,128,128,128,66,128,
+ 2,0,2,2,16,18,10,10, 2,2,2,86,86,86,2,18, 64,0,4,30,96,98,92,86, 2,2,2,128,128,128,2,62,
+ 146,0,152,154,174,180,156,152, 138,138,114,116,116,116,110,112, 148,0,150,162,184,184,176,158, 138,140,112,130,130,130,110,118,
+ 154,0,152,150,150,138,126,128, 10,12,138,118,118,118,138,116, 176,0,132,158,200,200,200,198, 118,136,116,200,200,200,132,200,
+ 176,0,132,158,200,200,200,198, 118,136,116,200,200,200,132,200, 176,0,132,158,200,200,200,198, 118,136,116,200,200,200,132,200,
+ 152,0,152,154,122,140,132,122, 10,14,136,132,132,132,138,120, 166,0,126,164,192,204,210,212, 118,118,110,200,200,200,116,224,
+ },
+},
+
+{ // UTF-16BE (921.761M chars) [37]
+ {NULL, NULL, NULL, NULL},
+ 58, 128, 46, 5, 128,
+ {255,148,128,107,171,104,157,100, 118,108,101,120,124,90,133,117, 110,115,104,101,109,109,114,104, 109,116,118,96,107,107,124,118,
+ 70,74,71,58,71,66,66,63, 77,54,49,59,103,87,65,64, 97,64,107,90,85,62,70,84, 68,102,96,78,93,84,65,62,
+ 98,86,102,72,68,106,98,98, 86,95,76,75,78,83,75,75, 86,85,82,77,76,85,76,77, 73,80,63,66,87,93,89,101,
+ 71,70,54,77,57,100,56,43, 81,50,69,75,59,63,75,80, 86,62,64,76,70,35,84,71, 72,68,83,83,81,83,96,112,
+ 158,116,117,132,124,138,121,108, 116,113,119,126,121,103,105,108, 120,97,117,112,89,96,106,109, 108,102,107,100,99,84,118,104,
+ 97,103,109,97,115,85,102,101, 103,94,96,102,108,105,104,111, 112,95,96,109,94,101,100,108, 109,120,113,105,98,89,95,106,
+ 95,111,93,114,92,107,109,118, 118,105,105,108,86,103,97,102, 101,110,88,98,94,117,109,105, 78,77,73,73,66,78,86,78,
+ 104,107,89,105,83,121,89,89, 97,107,111,111,101,103,99,100, 88,81,89,122,99,70,98,85, 99,87,106,114,125,103,97,136,
+ },
+ {236,123,89,99,100,109,89,100, 108,198,193,73,119,181,104,83, 99,103,97,99,99,97,82,94, 101,109,110,107,104,93,93,106,
+ 216,151,203,161,118,166,169,173, 164,164,144,152,168,182,186,198, 189,182,179,173,169,172,169,168, 170,168,177,180,198,196,198,153,
+ 139,171,167,169,169,168,165,158, 160,167,150,145,164,164,166,162, 165,146,165,168,170,153,155,156, 147,151,149,145,145,143,117,176,
+ 111,206,186,196,198,208,187,191, 193,205,165,175,201,192,202,201, 195,149,202,203,208,188,183,182, 175,180,162,150,135,150,118,103,
+ 112,108,111,92,98,93,88,76, 111,111,111,115,106,106,97,112, 115,111,81,95,101,109,110,98, 111,93,99,76,83,70,93,76,
+ 118,92,84,86,91,83,65,85, 86,99,71,87,101,87,83,95, 96,99,90,82,97,95,62,97, 88,93,93,96,90,93,79,99,
+ 100,101,86,92,104,90,86,96, 95,94,77,82,107,97,85,85, 105,105,86,90,102,108,87,87, 97,112,82,93,109,99,100,96,
+ 112,124,97,99,120,103,92,104, 114,131,110,75,88,121,83,84, 84,109,72,116,98,89,110,72, 99,78,96,85,114,106,69,129,
+ },
+ {126,120,126,128,126,126,126,126, 206,210,222,240,224,222,220,216, 122,164,100,126,150,150,112,122, 232,224,222,208,220,232,222,228,
+ 28,128,42,46,76,82,30,42, 128,128,128,128,128,128,128,128, 18,128,28,34,58,64,18,28, 128,128,128,128,128,128,128,128,
+ 16,128,26,32,54,62,16,26, 128,128,128,128,128,128,128,128, 22,128,32,38,70,74,22,32, 128,128,128,128,128,128,128,128,
+ 30,128,42,48,82,88,32,42, 128,128,128,128,128,128,128,128, 90,190,74,128,122,156,100,110, 212,182,194,168,162,186,118,178,
+ 96,180,78,128,126,140,100,104, 98,96,94,100,98,90,80,92, 108,222,102,132,140,156,114,122, 236,234,220,202,226,226,226,230,
+ 118,206,78,150,134,154,108,118, 228,218,192,196,192,176,210,224, 112,218,96,148,132,158,110,122, 232,232,220,208,226,212,224,228,
+ 112,224,102,146,142,156,108,124, 234,232,226,222,226,226,216,222, 114,196,84,148,140,164,110,118, 242,232,196,208,200,198,192,206,
+ 110,160,74,150,140,156,114,118, 230,224,160,160,162,160,160,220, 124,240,102,144,132,152,106,116, 230,226,200,198,162,174,190,230,
+ },
+},
+
+{ // Latin3 (0.294M chars) [38]
+ {NULL, NULL, ced_hires_16, ced_hires_16, },
+ 99, 202, 43, 23, 128,
+ {0,0,0,0,0,0,0,0, 0,142,142,0,0,142,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,143,143,143,145,0,143,148, 145,143,143,143,143,143,0,146, 147,143,143,143,144,143,143,148, 145,206,195,143,143,143,0,145,
+ 143,143,143,0,158,143,143,150, 143,151,143,143,143,143,143,143, 0,145,143,162,143,143,147,143, 143,143,143,143,172,143,143,143,
+ 154,143,143,0,165,177,154,182, 165,150,143,148,143,144,143,145, 0,145,143,162,143,143,157,143, 143,143,143,143,172,144,143,143,
+ 0,0,0,0,0,0,0,0, 0,112,112,0,0,112,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 188,148,144,162,154,0,145,154, 146,209,160,192,146,145,0,200, 163,148,147,145,151,144,144,167, 145,236,159,192,146,145,0,204,
+ 161,202,162,87,203,144,144,187, 186,211,176,153,145,198,159,150, 76,180,149,201,165,145,202,146, 144,144,182,153,204,144,144,178,
+ 188,206,154,0,204,144,144,175, 177,206,166,153,158,208,152,146, 0,180,159,203,157,145,200,146, 144,155,182,149,192,144,144,147,
+ },
+ {86,0,0,0,0,0,0,0, 0,149,151,0,0,159,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,146,167,143,143,144,158,154, 152,149,146,143,193,154,169,146, 150,160,164,151,150,149,147,146, 146,146,160,144,200,143,144,146,
+ 143,194,180,194,186,193,173,187, 178,180,164,179,196,190,207,184, 177,157,206,202,196,184,182,176, 156,184,174,143,143,145,144,143,
+ 144,192,181,189,184,193,172,184, 175,179,163,204,214,197,221,182, 173,152,217,199,214,184,180,176, 154,186,193,144,147,143,144,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,143,143,143,146,164,143,146, 146,145,143,171,143,144,177,162, 146,143,143,143,145,143,143,148, 146,199,202,192,143,143,169,162,
+ 145,144,145,0,158,143,143,149, 143,151,144,143,143,146,144,143, 0,144,144,146,143,143,148,144, 143,143,143,143,146,143,143,161,
+ 144,144,144,0,159,143,143,146, 143,147,144,143,143,146,143,143, 0,144,144,147,146,143,150,143, 143,143,144,143,145,143,143,143,
+ },
+ {162,0,130,150,128,132,120,124, 0,0,156,140,170,170,172,172, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 30,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 144,0,124,142,134,134,124,120, 0,0,174,132,162,162,162,166, 118,0,132,130,74,76,134,134, 0,0,126,164,140,140,140,142,
+ 118,0,100,110,147,145,94,90, 0,0,152,136,178,170,170,170, 124,0,102,116,143,145,108,108, 0,0,170,132,166,166,164,166,
+ 134,0,140,130,82,86,131,131, 0,0,168,152,154,156,164,156, 130,0,124,124,92,96,137,137, 0,0,148,152,164,170,166,166,
+ },
+},
+
+{ // UTF-16LE (912.138M chars) [39]
+ {NULL, NULL, NULL, NULL},
+ 58, 128, 46, 4, 127,
+ {255,146,126,120,177,123,159,99, 110,78,94,116,115,99,130,112, 110,112,82,97,100,94,113,102, 100,94,112,92,103,102,120,113,
+ 60,84,76,72,67,43,63,72, 75,64,74,96,95,88,83,52, 78,81,96,84,76,57,31,82, 83,85,86,82,92,78,88,68,
+ 84,91,96,52,84,96,99,105, 89,83,72,65,83,88,84,86, 81,94,75,70,97,95,78,57, 109,82,33,91,80,77,77,75,
+ 25,59,64,81,47,69,60,19, 96,31,92,30,20,68,72,81, 58,66,54,80,88,55,60,66, 55,40,45,78,55,99,113,116,
+ 161,117,113,130,127,131,118,108, 119,110,115,123,129,107,105,108, 116,99,114,110,93,99,108,111, 109,103,113,103,97,84,101,103,
+ 91,100,101,94,110,85,98,97, 98,88,86,93,108,101,99,107, 111,91,107,105,91,95,97,104, 105,118,111,98,97,92,87,101,
+ 101,106,99,111,87,110,107,116, 114,105,99,102,87,99,93,98, 100,103,88,98,89,111,105,101, 111,92,89,98,104,108,110,105,
+ 102,102,83,99,77,115,78,86, 85,100,104,107,98,100,92,97, 86,77,86,117,94,64,98,84, 97,89,103,109,121,97,98,126,
+ },
+ {236,123,100,94,96,100,91,97, 108,196,193,90,112,182,101,92, 101,105,95,98,102,96,89,96, 98,112,104,103,108,101,95,106,
+ 216,149,203,169,119,166,174,176, 165,165,142,152,168,181,187,198, 189,182,180,175,173,177,172,171, 173,176,177,184,198,197,198,156,
+ 139,172,168,169,171,168,166,159, 159,167,150,145,165,165,166,163, 165,145,164,168,170,153,155,156, 147,150,149,144,147,141,118,176,
+ 125,206,186,196,197,208,187,190, 192,205,164,175,200,191,201,201, 195,151,202,202,207,188,182,182, 175,179,161,149,134,149,121,100,
+ 115,103,100,89,101,94,90,87, 101,114,108,116,108,98,92,113, 113,107,90,81,101,110,105,94, 107,95,107,74,84,72,92,81,
+ 118,95,89,86,95,86,81,84, 92,99,79,88,102,91,89,97, 104,100,92,91,99,99,79,96, 89,102,98,99,90,95,85,101,
+ 101,103,87,94,104,94,85,94, 97,94,83,90,106,96,82,86, 105,103,85,93,102,108,92,91, 91,103,79,87,101,91,92,87,
+ 112,124,97,99,119,112,90,105, 108,128,107,81,91,121,89,85, 88,97,79,113,98,90,111,79, 102,83,100,94,112,102,86,130,
+ },
+ {128,124,128,126,126,126,126,126, 204,202,224,222,224,224,220,218, 120,164,116,98,140,156,124,122, 220,224,168,162,168,220,206,238,
+ 82,158,96,72,110,162,88,92, 170,160,158,158,158,176,158,164, 54,158,66,68,104,108,56,66, 158,158,158,158,158,148,158,158,
+ 48,158,58,62,88,96,50,60, 158,158,158,158,158,148,158,158, 50,158,60,64,90,98,52,62, 158,158,158,158,158,148,158,158,
+ 62,158,74,80,110,116,66,76, 158,158,158,158,158,148,158,158, 112,192,96,138,138,132,104,106, 206,210,184,198,188,196,186,186,
+ 106,174,86,120,124,134,92,112, 96,94,82,82,82,78,66,76, 122,214,110,132,136,160,112,112, 228,230,210,212,214,210,216,212,
+ 90,200,118,146,140,162,114,120, 234,222,202,200,206,188,196,196, 96,206,128,142,144,156,116,126, 238,216,210,206,214,206,218,208,
+ 94,216,130,142,136,160,108,126, 238,236,218,214,222,212,214,212, 94,214,134,138,140,158,114,124, 238,238,212,208,212,204,214,208,
+ 62,162,54,148,148,164,116,124, 232,226,164,164,162,150,164,174, 94,210,110,146,134,162,112,118, 232,228,192,198,194,188,198,208,
+ },
+},
+
+{ // HZ-GB-2312 (87.400M chars) [40]
+ {NULL, NULL, NULL, NULL},
+ 92, 0, 37, 0, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {126,0,164,0,0,0,126,0, 178,139,148,0,0,132,154,146, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 161,195,176,211,162,158,148,144, 213,209,179,164,154,148,168,174, 182,184,172,182,184,188,187,175, 180,183,176,176,182,177,179,172,
+ 170,193,196,199,167,165,195,168, 183,178,178,180,168,178,186,182, 183,158,180,182,181,183,186,180, 155,132,136,132,146,136,132,132,
+ 132,132,136,139,173,183,178,171, 162,161,132,142,144,132,139,139, 132,136,139,144,132,132,149,139, 132,132,136,152,139,132,245,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // CSN_369103 (8.850M chars) [41]
+ {NULL, NULL, NULL, NULL},
+ 90, 204, 46, 27, 127,
+ {0,0,0,0,0,0,0,0, 0,90,90,0,0,90,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,178,96,184,92,102,188,127, 120,153,117,98,133,96,143,153, 122,178,93,182,115,102,188,96, 119,152,117,98,133,101,141,152,
+ 98,169,119,130,151,111,96,137, 171,147,170,103,153,155,105,120, 96,98,118,171,112,104,128,105, 180,129,158,96,133,127,119,102,
+ 99,169,99,125,153,107,102,128, 169,142,170,99,155,155,103,119, 95,99,118,171,112,105,148,95, 180,130,154,95,137,127,119,96,
+ 0,0,0,0,0,0,0,0, 0,55,55,0,0,55,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 185,202,93,207,98,140,210,145, 119,178,153,123,170,124,172,196, 158,217,94,211,139,141,210,121, 116,176,151,152,187,124,173,200,
+ 103,198,155,156,200,128,165,183, 202,208,195,143,194,194,151,144, 149,174,157,197,159,157,199,120, 192,171,178,137,201,170,155,174,
+ 106,203,144,173,201,105,197,170, 203,203,208,143,202,204,139,150, 148,188,166,199,147,166,197,121, 192,191,178,144,188,182,155,122,
+ },
+ {62,0,0,0,0,0,0,0, 0,129,153,0,0,162,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,154,181,95,91,108,164,158, 144,155,114,105,191,153,189,148, 132,145,139,130,128,126,119,117, 115,116,177,135,201,95,124,155,
+ 93,200,180,210,197,203,168,188, 175,191,170,190,200,190,208,194, 182,141,204,199,200,181,182,186, 146,188,181,105,114,126,105,120,
+ 112,201,181,209,196,205,167,186, 172,191,169,190,200,189,207,194, 180,135,203,198,199,182,181,191, 143,189,181,100,125,92,99,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,176,95,167,93,99,163,117, 120,144,106,125,140,115,142,179, 121,178,93,168,111,97,163,92, 119,144,107,125,141,107,143,179,
+ 99,160,117,127,149,96,190,132, 166,138,143,97,95,185,108,127, 94,120,128,168,99,91,130,104, 171,126,127,95,123,101,122,153,
+ 100,161,110,128,150,96,190,125, 167,128,145,96,95,185,105,130, 94,120,128,170,114,91,134,95, 172,126,113,95,119,100,122,97,
+ },
+ {182,0,124,152,122,132,122,132, 0,0,162,164,156,176,156,176, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 12,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 14,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 132,0,96,130,142,136,100,114, 0,0,172,130,168,162,110,142, 130,0,136,132,52,54,136,130, 0,0,108,174,92,106,170,166,
+ 98,0,90,100,140,142,100,90, 0,0,164,122,170,178,132,132, 100,0,92,102,138,142,108,110, 0,0,166,106,174,156,130,144,
+ 134,0,136,134,38,54,134,134, 0,0,118,168,102,146,168,180, 132,0,130,134,44,58,136,140, 0,0,108,170,98,158,172,156,
+ },
+},
+
+{ // ISO-2022-KR (85.145M chars) [42]
+ {NULL, NULL, NULL, NULL},
+ 44, 144, 15, 3, 129,
+ {0,0,130,0,0,0,0,0, 0,66,66,0,0,66,213,252, 0,0,0,0,0,0,0,0, 0,0,0,224,0,0,91,91,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {80,0,0,0,0,0,0,0, 0,155,178,0,0,191,94,16, 0,0,0,0,0,0,0,0, 0,0,0,159,0,0,115,80,
+ 237,178,211,150,215,142,197,197, 215,186,147,141,202,191,204,184, 182,193,189,179,176,177,168,168, 173,174,186,191,233,164,175,184,
+ 177,172,153,166,162,158,159,169, 164,164,152,159,147,160,152,147, 160,143,152,154,156,148,153,153, 128,113,118,160,143,179,134,168,
+ 120,133,133,128,134,139,123,121, 142,133,113,107,120,125,168,114, 123,102,122,140,136,109,112,132, 116,115,107,101,157,124,155,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {130,130,126,130,130,130,118,128, 0,0,0,0,0,0,0,0, 42,2,134,2,62,40,70,60, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // Latin6 (0.061M chars) [43]
+ {NULL, NULL, ced_hires_19, ced_hires_19, },
+ 93, 214, 54, 26, 129,
+ {0,0,0,0,0,0,0,0, 0,156,156,0,0,156,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,158,158,158,158,158,158,159, 158,158,161,158,159,158,158,158, 158,158,158,158,159,158,158,159, 158,158,161,158,159,158,158,158,
+ 159,164,158,159,162,158,158,158, 171,161,158,158,159,162,158,158, 158,160,158,158,158,158,159,158, 158,158,162,158,158,158,158,158,
+ 159,164,158,158,163,158,158,158, 171,160,158,158,159,162,158,158, 158,160,158,158,158,158,162,158, 158,159,162,158,159,158,158,158,
+ 0,0,0,0,0,0,0,0, 0,130,130,0,0,130,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,198,158,158,160,158,158,160, 158,163,190,158,168,158,159,158, 163,212,158,158,162,158,158,166, 159,158,180,158,168,158,158,158,
+ 206,194,163,172,195,183,171,185, 198,203,191,160,209,190,162,159, 161,179,158,193,165,163,194,158, 179,176,175,160,196,171,160,172,
+ 211,198,160,173,196,193,171,194, 199,198,204,160,214,199,160,158, 167,178,159,195,161,163,193,158, 178,219,176,159,185,179,160,158,
+ },
+ {0,0,0,0,0,0,0,0, 0,160,164,0,0,168,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,163,177,159,159,159,210,162, 160,170,159,159,189,161,185,159, 162,161,160,159,159,159,159,159, 159,159,178,159,200,159,159,164,
+ 159,184,177,200,197,187,171,187, 174,180,189,189,194,188,202,180, 180,160,204,208,200,171,181,173, 162,166,178,159,159,159,159,159,
+ 159,184,179,199,197,189,170,185, 172,181,189,189,194,186,201,179, 179,159,203,208,200,171,185,173, 162,169,178,159,159,159,159,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,159,159,159,159,159,159,159, 159,159,163,159,161,159,159,159, 159,159,159,159,159,160,159,159, 159,159,163,159,161,159,159,159,
+ 161,164,159,159,163,159,159,159, 168,160,159,159,159,168,159,159, 160,159,159,159,159,159,159,159, 159,159,159,159,159,159,159,164,
+ 161,164,159,159,163,159,159,159, 169,159,159,159,159,169,159,159, 160,159,159,159,159,159,159,159, 159,159,159,159,159,159,159,159,
+ },
+ {152,0,122,146,128,132,128,130, 0,0,154,156,154,156,154,158, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 144,0,114,140,138,132,120,128, 0,0,162,154,152,154,152,154, 142,0,138,134,108,112,134,126, 0,0,154,158,154,156,154,156,
+ 118,0,98,118,143,145,106,106, 0,0,154,154,162,156,154,154, 132,0,108,126,143,143,118,118, 0,0,154,156,158,158,154,156,
+ 126,0,132,132,90,92,139,139, 0,0,152,156,152,156,162,156, 126,0,140,126,98,102,131,131, 0,0,154,156,154,158,156,158,
+ },
+},
+
+{ // UTF7 (0.037M chars) [44]
+ {NULL, NULL, NULL, NULL},
+ 77, 207, 29, 27, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,184,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,189,189,0,0,189,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,189,189,189,189,189,189,189, 189,189,189,0,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // ISO_2022_CN (63.392M chars) [45]
+ {NULL, NULL, NULL, NULL},
+ 43, 144, 14, 3, 129,
+ {0,0,0,0,0,0,0,0, 0,70,70,0,0,70,213,252, 0,0,0,0,0,0,0,0, 0,0,0,222,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {79,0,0,0,0,0,0,0, 0,155,177,0,0,190,19,19, 0,0,0,0,0,0,0,0, 0,0,0,158,0,0,114,79,
+ 240,177,210,148,212,141,197,195, 214,185,147,140,202,190,203,183, 182,192,188,179,175,177,167,168, 173,174,186,190,231,164,175,183,
+ 177,172,152,166,161,157,158,168, 164,163,151,158,146,160,151,146, 159,143,151,153,155,147,152,152, 127,112,117,160,142,178,133,168,
+ 119,132,130,126,124,138,122,120, 141,131,112,106,119,124,122,112, 122,101,108,135,135,101,111,130, 112,115,106,96,156,123,154,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {130,130,126,128,130,128,128,130, 0,0,0,0,0,0,0,0, 44,2,134,2,62,26,62,10, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // BIG5-CP950 (0.029M chars) [46]
+ {NULL, NULL, NULL, NULL},
+ 158, 182, 61, 26, 128,
+ {0,166,166,166,166,166,166,166, 166,177,177,166,166,177,166,166, 158,158,158,158,158,158,158,158, 158,158,158,158,158,158,158,158,
+ 166,179,169,151,208,201,198,190, 197,193,202,187,190,196,191,184, 197,185,191,191,184,189,185,179, 190,185,188,182,184,184,187,182,
+ 181,178,182,183,178,186,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166,
+ 166,166,166,166,166,167,168,167, 166,167,166,166,168,166,166,166, 166,166,166,166,166,166,166,166, 166,168,166,166,166,166,166,101,
+ 0,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 149,149,149,149,149,149,149,149, 149,149,149,149,149,149,149,149,
+ 161,211,166,161,206,199,200,187, 180,182,185,186,188,181,182,176, 183,186,179,188,175,185,185,193, 178,185,170,180,179,177,170,167,
+ 175,174,171,172,168,174,169,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161,
+ 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,167,
+ },
+ {0,0,0,0,0,0,0,0, 0,113,113,0,0,113,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,113,113,113,113,113,113,113, 113,113,113,113,113,113,113,113, 113,113,113,113,113,113,113,113, 113,113,113,113,113,113,113,113,
+ 197,198,187,191,188,176,184,188, 190,180,179,179,184,184,183,189, 185,182,178,177,182,184,174,187, 183,177,175,178,176,185,181,180,
+ 178,183,181,174,182,182,176,180, 180,186,185,181,180,177,186,185, 184,187,175,191,180,185,185,183, 181,180,181,179,185,179,190,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 150,188,176,189,185,179,174,184, 181,179,178,184,179,179,175,174, 182,179,180,187,176,180,183,176, 183,185,197,182,175,183,176,175,
+ 184,181,177,176,183,178,186,182, 182,184,183,179,184,180,187,185, 186,186,182,191,174,179,179,182, 182,177,184,182,180,179,178,181,
+ 179,179,180,177,180,182,186,176, 180,190,188,180,185,181,175,179, 174,176,178,183,188,178,182,182, 186,183,185,180,183,180,180,121,
+ },
+ {170,0,170,170,104,110,110,108, 128,128,2,2,2,2,2,78, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 14,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 20,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 34,0,10,10,2,2,2,2, 128,128,2,2,2,2,2,2, 42,0,18,18,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 40,0,18,16,2,2,2,2, 128,128,2,2,2,2,2,2, 42,0,18,18,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 0,0,0,0,134,142,140,138, 0,0,132,132,132,132,132,132, 0,0,0,0,134,142,140,138, 0,0,136,134,134,136,136,136,
+ 0,0,0,0,142,136,138,134, 0,0,134,138,132,134,134,134, 0,0,0,0,134,136,140,144, 0,0,134,130,138,136,134,136,
+ 0,0,0,0,136,142,138,140, 0,0,130,134,134,132,140,138, 0,0,0,0,134,142,140,138, 0,0,136,134,134,136,136,136,
+ 0,0,0,0,134,142,140,138, 0,0,138,134,134,134,134,134, 160,128,160,160,134,140,140,138, 128,128,136,134,134,134,136,136,
+ },
+},
+
+{ // JAGRAN (0.046M chars) [47]
+ {NULL, NULL, NULL, NULL},
+ 142, 199, 66, 34, 133,
+ {174,174,174,174,174,174,174,174, 174,182,182,174,174,182,174,174, 174,174,174,174,174,174,174,174, 174,174,174,0,0,0,174,174,
+ 0,182,182,182,182,182,182,182, 182,182,182,182,182,0,182,182, 182,182,0,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 166,166,166,166,166,166,166,166, 166,178,178,166,166,178,166,166, 166,166,166,166,166,166,166,166, 166,166,166,0,0,0,166,166,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,178,0,178,178, 166,166,0,166,166,166,166,166, 166,166,166,166,166,166,166,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {0,0,0,0,0,0,0,0, 0,180,180,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,180,127,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,127,180, 180,180,180,180,180,180,180,180, 180,180,127,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,127,180,180,0,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,0,0,0,176,176,
+ 0,176,176,176,176,176,176,176, 176,176,176,176,176,0,176,176, 176,176,0,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 136,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130,
+ },
+},
+
+{ // BHASKAR (0.047M chars) [48]
+ {NULL, NULL, NULL, NULL},
+ 141, 199, 66, 34, 132,
+ {174,174,174,174,174,174,174,174, 174,182,182,174,174,182,174,174, 174,174,174,174,174,174,174,174, 174,174,174,0,0,0,174,174,
+ 0,182,182,182,181,182,182,182, 182,182,182,182,182,0,182,182, 182,182,0,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,181, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 166,166,166,166,166,166,166,166, 166,178,178,166,166,178,166,166, 166,166,166,166,166,166,166,166, 166,166,166,0,0,0,166,166,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,178,0,178,178, 166,166,0,166,166,166,166,166, 166,166,166,166,166,166,166,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {0,0,0,0,0,0,0,0, 0,180,180,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,126,180, 180,180,180,180,180,180,180,180, 180,180,126,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,0,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,0,0,0,176,176,
+ 0,176,176,176,176,176,176,176, 176,176,176,176,176,0,176,176, 176,176,0,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 130,130,130,130,130,130,130,130, 134,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 134,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 134,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130,
+ },
+},
+
+{ // HTCHANAKYA (0.041M chars) [49]
+ {NULL, NULL, NULL, NULL},
+ 142, 202, 68, 32, 133,
+ {173,0,0,0,173,173,173,173, 173,182,171,173,105,171,0,0, 0,173,173,0,173,173,173,173, 0,0,173,0,0,0,0,0,
+ 181,182,182,182,181,182,182,182, 182,182,182,182,182,0,182,182, 182,182,182,182,182,182,182,0, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,181, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 169,0,0,0,169,169,169,169, 169,180,170,169,0,170,0,0, 0,169,169,0,169,169,169,169, 0,0,169,0,0,0,0,0,
+ 0,180,180,180,179,180,180,180, 180,180,180,180,180,0,180,180, 169,169,169,169,169,169,169,0, 169,169,169,169,169,169,169,169,
+ 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,169,169,
+ 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,0,168, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ },
+ {0,0,0,0,0,0,0,0, 0,181,181,0,0,181,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,181,129,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,129,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,0,
+ 176,0,0,0,176,176,176,176, 176,176,0,176,0,0,0,0, 0,176,176,0,176,176,176,176, 0,0,176,0,0,0,0,0,
+ 175,176,176,176,176,176,176,176, 176,176,176,176,176,0,176,176, 176,176,176,176,176,176,176,0, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 130,0,128,130,128,130,130,128, 128,128,128,128,128,128,128,128, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ },
+},
+
+{ // TSCII (0.047M chars) [50]
+ {NULL, NULL, NULL, NULL},
+ 141, 199, 66, 33, 134,
+ {173,173,173,173,173,173,173,173, 173,182,182,173,173,182,173,173, 173,173,173,173,173,173,173,173, 173,173,173,173,173,173,0,173,
+ 0,181,181,181,181,181,157,157, 157,181,0,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,0,
+ 167,167,167,167,167,167,167,167, 167,179,179,167,167,167,167,167, 167,167,175,168,170,167,167,167, 167,167,167,167,167,167,0,167,
+ 0,178,178,178,178,178,0,0, 0,183,0,178,178,178,178,178, 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167,
+ 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167,
+ 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,125,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 170,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,180,179,179,179,180,179,179, 179,180,179,179,179,179,179,179, 179,179,179,182,179,179,179,179, 179,179,179,179,179,179,179,0,
+ 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,0,175,
+ 0,175,175,175,175,175,0,0, 0,175,0,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175,
+ 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175,
+ 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 128,128,130,128,128,128,128,128, 132,0,134,132,132,132,134,136, 130,130,132,130,130,130,130,130,
+ 134,0,136,134,134,134,134,134, 132,130,130,132,132,130,130,130, 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130,
+ 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130, 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130,
+ 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130, 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130,
+ },
+},
+
+{ // TAM (0.036M chars) [51]
+ {NULL, NULL, NULL, NULL},
+ 140, 203, 70, 33, 133,
+ {0,0,174,174,174,174,174,174, 174,184,184,174,174,174,0,0, 0,0,0,0,0,0,0,0, 174,174,174,174,174,0,0,174,
+ 0,183,183,183,183,0,183,183, 183,0,161,161,161,0,183,183, 183,183,183,183,183,183,183,0, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,0,183,183,183,183,183, 0,0,0,0,0,0,183,183, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 0,0,170,170,170,170,170,170, 170,181,181,170,170,0,0,0, 0,0,0,0,0,0,0,0, 170,170,170,170,170,0,0,170,
+ 0,181,181,181,181,0,181,181, 181,0,0,0,0,0,181,181, 170,170,170,170,170,170,170,0, 170,170,170,170,170,170,170,170,
+ 170,170,170,170,170,170,170,170, 170,170,0,170,170,170,170,170, 0,0,0,0,0,0,170,170, 170,170,170,170,170,170,170,170,
+ 170,170,170,170,170,170,170,170, 170,170,170,170,170,170,170,170, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ },
+ {0,0,0,0,0,0,0,0, 0,182,182,0,0,132,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,132,132,132,132,132,
+ 132,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,132,132,132,132,0,
+ 0,0,177,177,177,177,177,177, 177,177,177,177,177,0,0,0, 0,0,0,0,0,0,0,0, 177,177,177,177,177,0,0,177,
+ 0,177,177,177,177,0,177,177, 177,0,0,0,0,0,177,177, 177,177,177,177,177,177,177,0, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,0,177,177,177,177,177, 0,0,0,0,0,0,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,177,177,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 130,130,130,128,128,128,128,128, 134,0,136,136,136,134,136,134, 132,132,132,134,134,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,134,134, 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,134,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132,
+ },
+},
+
+{ // TAB (0.030M chars) [52]
+ {NULL, NULL, NULL, NULL},
+ 137, 210, 72, 28, 132,
+ {0,0,0,0,0,0,0,0, 0,174,174,0,0,174,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,183,183,183,0,183,183, 183,0,165,165,165,0,183,183, 183,183,183,183,183,183,183,0, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,0,183,183,183,183,183, 0,0,0,0,0,0,183,0, 0,0,0,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 0,0,0,0,0,0,0,0, 0,174,174,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,186,186,186,0,186,186, 186,0,0,0,0,0,186,186, 177,177,177,177,177,177,177,0, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,0,177,177,177,177,177, 0,0,0,0,0,0,177,0, 0,0,0,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 186,186,186,186,186,186,186,186, 186,186,186,186,186,186,186,186,
+ },
+ {0,0,0,0,0,0,0,0, 0,183,183,0,0,136,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,176,176,176,0,176,176, 176,0,0,0,0,0,176,176, 176,176,176,176,176,176,176,0, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,0,176,176,176,176,176, 0,0,0,0,0,0,176,0, 0,0,0,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 132,0,132,134,134,134,134,132, 128,128,134,134,134,134,138,138, 132,0,132,132,132,132,132,132, 128,128,136,136,136,136,136,136,
+ 132,0,132,132,132,132,132,132, 128,128,136,136,136,136,136,136, 132,0,134,134,134,134,134,134, 128,128,136,136,136,136,136,136,
+ 132,0,132,134,134,134,134,132, 128,128,136,136,136,136,136,136, 132,0,132,134,134,134,134,132, 128,128,136,136,136,136,136,136,
+ },
+},
+
+{ // EUC-CN (0.035M chars) [53]
+ {NULL, NULL, NULL, NULL},
+ 197, 192, 37, 32, 128,
+ {0,0,0,0,0,0,0,0, 0,169,169,0,0,169,119,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,240,216,176,191,186,168,185, 184,166,0,0,118,0,0,0, 0,103,0,0,0,0,29,0, 0,0,0,0,0,0,0,0,
+ 0,0,169,0,197,197,190,192, 190,212,188,187,188,186,190,188, 187,186,186,185,184,186,187,187, 185,185,186,185,193,186,186,190,
+ 185,185,185,187,189,185,186,191, 185,185,185,184,186,185,184,184, 185,184,184,184,184,184,184,184, 186,184,184,184,184,173,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,147,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,134,134,0,0,134,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,134,
+ 134,134,134,134,138,134,138,134, 134,138,134,134,138,134,134,134, 134,134,134,134,138,138,134,134, 134,134,134,134,134,134,134,134,
+ 134,134,134,134,134,134,134,134, 134,134,134,134,138,134,134,134, 134,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 170,230,193,197,184,184,184,184, 210,184,185,182,188,181,181,185, 183,207,189,182,181,182,181,180, 181,181,182,185,181,179,187,187,
+ 183,180,180,181,182,180,186,180, 180,185,195,191,183,180,182,180, 179,182,189,183,182,180,192,195, 179,179,181,182,181,179,183,183,
+ 180,180,178,184,182,198,195,192, 180,183,186,180,180,183,181,181, 183,179,180,189,179,179,178,178, 182,179,212,180,186,180,211,0,
+ },
+ {182,0,182,182,180,182,182,182, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,2,10,14,14,12,8, 76,0,52,52,50,52,52,52, 128,128,128,128,128,128,128,128,
+ 4,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 8,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 8,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 12,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 0,0,0,0,0,0,0,0, 0,0,76,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,134,124,114,118,122,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,116,126,130,128,126,136, 0,0,0,0,0,0,0,0, 0,0,116,128,136,132,134,124,
+ 0,0,0,0,0,0,0,0, 0,0,118,130,132,134,130,124, 0,0,0,0,0,0,0,0, 0,0,118,130,132,134,130,126,
+ },
+},
+
+{ // EUC (15478 chars) [54]
+ {NULL, NULL, NULL, NULL},
+ 197, 200, 38, 33, 129,
+ {0,0,0,0,0,0,0,0, 0,173,173,0,0,173,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,234,211,180,190,188,173,189, 189,170,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,174,0,193,195,190,191, 191,208,190,189,190,189,191,190, 190,189,190,189,189,190,189,190, 189,189,190,189,193,189,190,191,
+ 189,189,189,190,191,190,190,193, 189,190,189,189,190,189,189,189, 189,189,189,189,189,189,189,189, 190,189,189,189,189,178,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,140,140,0,0,140,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 162,226,192,195,186,186,185,186, 205,186,187,185,188,185,185,186, 184,202,187,184,184,184,183,183, 184,184,184,185,183,183,187,187,
+ 183,183,183,183,184,183,186,183, 183,185,192,189,184,183,184,183, 182,184,187,184,183,182,190,192, 182,182,182,183,184,182,184,184,
+ 182,182,182,185,184,194,191,190, 183,184,185,183,183,184,183,183, 184,182,182,188,182,182,182,182, 184,182,206,183,185,182,206,0,
+ },
+ {176,0,176,176,176,176,176,176, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 0,0,0,0,0,0,0,0, 0,0,2,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,136,124,116,118,122,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,118,126,128,128,128,136, 0,0,0,0,0,0,0,0, 0,0,118,128,132,130,130,126,
+ 0,0,0,0,0,0,0,0, 0,0,120,128,130,132,130,126, 0,0,0,0,0,0,0,0, 0,0,120,130,130,130,130,126,
+ },
+},
+
+{ // CNS (15478 chars) [55]
+ {NULL, NULL, NULL, NULL},
+ 197, 200, 38, 33, 129,
+ {0,0,0,0,0,0,0,0, 0,173,173,0,0,173,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,234,211,180,190,188,173,189, 189,170,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,174,0,193,195,190,191, 191,208,190,189,190,189,191,190, 190,189,190,189,189,190,189,190, 189,189,190,189,193,189,190,191,
+ 189,189,189,190,191,190,190,193, 189,190,189,189,190,189,189,189, 189,189,189,189,189,189,189,189, 190,189,189,189,189,178,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,140,140,0,0,140,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 162,226,192,195,186,186,185,186, 205,186,187,185,188,185,185,186, 184,202,187,184,184,184,183,183, 184,184,184,185,183,183,187,187,
+ 183,183,183,183,184,183,186,183, 183,185,192,189,184,183,184,183, 182,184,187,184,183,182,190,192, 182,182,182,183,184,182,184,184,
+ 182,182,182,185,184,194,191,190, 183,184,185,183,183,184,183,183, 184,182,182,188,182,182,182,182, 184,182,206,183,185,182,206,0,
+ },
+ {176,0,176,176,176,176,176,176, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 0,0,0,0,0,0,0,0, 0,0,2,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,136,124,116,118,122,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,118,126,128,128,128,136, 0,0,0,0,0,0,0,0, 0,0,118,128,132,130,130,126,
+ 0,0,0,0,0,0,0,0, 0,0,120,128,130,132,130,126, 0,0,0,0,0,0,0,0, 0,0,120,130,130,130,130,126,
+ },
+},
+
+{ // UTF-32BE (1032.458M chars) [56]
+ {NULL, NULL, NULL, NULL},
+ 77, 151, 56, 41, 127,
+ {250,137,119,116,158,116,151,92, 103,89,82,111,111,75,204,151, 102,106,71,86,94,92,104,91, 88,92,106,215,97,93,99,106,
+ 79,97,116,106,115,126,78,66, 89,96,94,80,79,92,90,90, 93,101,90,90,72,96,79,91, 82,89,78,88,80,99,84,86,
+ 86,92,83,59,101,100,101,79, 89,93,71,46,102,89,64,68, 78,74,72,65,71,56,71,64, 76,84,65,68,92,98,94,106,
+ 182,177,181,215,194,207,203,198, 193,188,166,179,184,171,90,183, 67,2,0,27,20,10,16,20, 7,0,3,46,24,71,43,104,
+ 160,104,104,120,120,123,106,99, 106,101,107,114,119,100,91,96, 110,88,107,102,79,91,100,101, 99,92,100,94,88,67,104,92,
+ 68,80,83,73,92,62,75,75, 79,64,69,73,88,82,78,88, 92,73,82,84,71,76,74,82, 84,98,91,76,78,66,71,80,
+ 78,89,78,91,66,88,86,94, 97,83,76,79,62,78,71,75, 80,86,62,78,68,93,85,80, 83,80,76,75,69,83,90,83,
+ 82,82,61,78,58,98,56,64, 68,80,85,85,74,85,73,74, 60,46,58,83,68,31,72,64, 71,62,78,78,88,72,62,103,
+ },
+ {231,96,83,86,82,95,79,87, 91,190,189,71,101,175,87,75, 88,93,82,83,89,84,75,84, 89,98,94,150,95,84,82,97,
+ 211,164,198,158,206,163,163,167, 204,160,139,149,163,176,181,194, 186,180,173,172,169,172,164,166, 171,171,175,177,193,193,193,164,
+ 169,173,164,169,169,167,165,166, 164,164,153,153,160,162,163,160, 163,145,162,165,167,153,156,157, 144,147,146,140,138,138,110,171,
+ 103,201,181,192,192,203,181,185, 187,200,161,171,196,187,196,196, 190,145,197,197,203,183,178,178, 170,174,157,147,130,147,113,85,
+ 194,202,196,197,161,172,160,164, 174,168,169,165,163,162,161,172, 167,152,151,154,172,173,173,172, 170,167,172,172,181,172,158,158,
+ 158,165,152,155,174,160,165,166, 161,156,155,157,154,167,173,166, 170,159,161,164,160,162,155,162, 190,174,180,181,186,173,165,168,
+ 96,98,83,90,99,87,83,90, 91,88,75,80,102,90,78,81, 99,97,81,86,97,102,86,85, 95,107,84,92,106,96,96,92,
+ 105,120,91,98,114,104,86,102, 104,125,102,72,80,118,81,79, 79,91,70,110,91,83,108,69, 93,74,94,86,107,97,65,124,
+ },
+ {132,132,126,132,132,132,132,132, 122,128,140,130,230,224,222,224, 60,24,152,14,68,74,40,40, 104,122,122,98,204,218,206,224,
+ 8,96,12,22,38,48,10,20, 44,64,80,56,128,128,128,128, 20,128,26,34,56,70,22,32, 40,58,72,50,128,128,128,128,
+ 20,128,24,34,54,66,22,32, 38,56,70,50,128,128,128,128, 26,128,30,40,62,74,28,38, 40,60,76,52,128,128,128,128,
+ 2,2,2,2,2,2,2,2, 42,62,78,54,128,128,128,128, 134,180,74,128,144,158,114,92, 116,128,122,78,150,192,132,164,
+ 118,142,64,114,106,120,82,106, 14,12,26,10,92,80,68,72, 138,192,94,128,132,154,112,108, 34,52,40,14,124,118,102,106,
+ 142,188,72,144,148,176,108,110, 156,152,90,112,202,162,200,174, 142,200,86,138,146,162,110,118, 156,146,132,106,208,200,212,192,
+ 142,208,88,138,138,162,96,118, 154,152,138,122,208,212,182,178, 138,180,68,144,148,178,114,122, 154,162,112,112,184,192,158,168,
+ 140,128,102,142,150,172,108,116, 148,148,148,148,20,14,4,12, 136,196,112,138,136,176,110,118, 132,148,136,112,148,160,162,150,
+ },
+},
+
+{ // UTF-32LE (1032.461M chars) [57]
+ {NULL, NULL, NULL, NULL},
+ 77, 152, 56, 41, 127,
+ {250,137,119,116,158,116,151,92, 103,89,82,111,111,75,204,150, 102,106,71,86,94,92,104,91, 88,92,106,215,97,93,99,106,
+ 79,97,116,106,114,126,78,66, 89,96,94,144,79,92,90,90, 93,101,90,90,72,96,79,91, 80,89,78,88,80,99,84,86,
+ 86,92,83,58,101,100,101,79, 89,93,71,46,102,89,64,68, 78,74,72,65,71,54,71,64, 76,84,65,68,92,98,94,106,
+ 182,177,181,215,193,207,203,198, 193,188,166,178,184,171,90,183, 67,3,0,27,20,11,17,20, 8,0,4,46,24,71,162,104,
+ 160,104,104,120,120,123,106,99, 106,101,107,114,118,100,91,94, 110,88,107,102,79,91,100,101, 99,92,100,94,88,67,104,92,
+ 68,80,83,73,92,62,75,75, 79,64,71,124,88,82,78,88, 92,73,82,84,71,76,74,82, 84,98,91,76,78,66,71,80,
+ 78,89,78,91,66,88,86,94, 97,83,76,79,62,78,71,75, 80,86,62,78,68,93,85,80, 83,80,76,75,69,83,90,83,
+ 82,82,61,78,57,98,57,64, 68,80,85,85,74,85,73,74, 60,46,57,83,68,31,72,64, 71,62,78,78,88,72,122,103,
+ },
+ {231,96,83,86,82,95,79,87, 91,190,189,71,101,175,87,75, 88,93,82,83,89,84,75,84, 88,98,94,158,95,84,82,97,
+ 211,164,198,159,206,163,163,167, 204,160,139,149,163,176,181,194, 186,180,174,172,169,172,165,166, 171,171,175,177,193,193,193,164,
+ 169,173,164,169,169,167,165,166, 164,164,155,154,161,163,165,160, 163,145,162,165,167,154,157,157, 144,147,146,140,138,138,110,171,
+ 103,201,181,192,192,203,181,185, 187,200,161,171,196,187,196,196, 190,145,197,197,203,183,178,177, 170,174,157,147,130,147,131,85,
+ 193,202,196,197,161,172,160,164, 174,168,169,165,163,162,161,172, 167,152,151,154,172,173,173,172, 170,167,172,172,181,172,158,158,
+ 158,165,152,155,174,160,165,166, 161,155,155,157,154,167,173,166, 170,159,161,164,160,162,155,162, 190,174,180,181,186,173,165,168,
+ 99,99,87,92,100,90,85,93, 93,89,79,83,102,91,81,82, 101,97,82,87,97,102,87,86, 95,107,84,92,106,96,96,92,
+ 105,120,93,102,115,107,92,103, 105,125,103,77,82,118,81,84, 79,91,70,110,91,83,107,69, 93,74,94,86,107,97,65,124,
+ },
+ {132,124,126,132,132,132,132,132, 122,128,140,130,228,224,222,224, 60,14,152,16,66,74,40,40, 104,122,122,98,204,218,206,224,
+ 44,212,2,12,34,104,44,134, 116,112,144,150,220,208,220,130, 20,116,26,34,56,70,22,32, 40,58,72,50,128,128,128,128,
+ 20,104,24,34,54,66,22,32, 38,56,70,50,128,128,128,128, 26,114,30,40,60,74,28,38, 42,60,76,52,128,128,128,128,
+ 2,2,2,2,2,2,2,2, 44,62,78,54,128,128,128,128, 84,190,114,148,160,156,70,62, 116,118,160,144,212,200,222,166,
+ 118,134,64,114,106,120,82,106, 14,12,26,8,90,80,66,72, 138,168,94,128,130,154,112,108, 34,52,40,14,122,116,100,106,
+ 130,162,60,130,122,146,98,100, 146,140,78,102,202,162,172,162, 142,200,86,138,144,162,110,118, 158,146,132,106,208,200,212,192,
+ 142,208,88,138,136,162,96,118, 154,152,138,122,208,212,182,176, 138,180,68,144,148,178,114,122, 156,162,112,112,182,192,156,168,
+ 140,128,102,142,150,172,108,116, 148,148,148,148,18,14,4,12, 124,168,100,124,114,154,98,106, 76,90,78,54,122,120,108,108,
+ },
+},
+
+{ // X-BINARYENC (0 chars) [58]
+ {NULL, NULL, NULL, NULL},
+ 0, 0, 0, 0, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ },
+},
+
+{ // X-UTF8UTF8 (43.271M chars) [59]
+ {NULL, NULL, NULL, NULL},
+ 150, 194, 32, 13, 129,
+ {191,104,167,173,203,123,113,0, 0,69,69,80,0,107,0,105, 95,0,125,146,116,0,0,0, 104,129,184,0,125,122,159,0,
+ 139,144,162,111,112,105,148,117, 115,101,122,104,217,112,130,114, 151,116,107,120,116,116,98,89, 124,119,136,113,115,116,126,105,
+ 0,0,239,243,62,181,145,62, 62,62,62,147,62,62,62,62, 62,62,62,62,62,62,62,62, 62,62,62,62,62,62,62,62,
+ 0,0,220,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,58,0,0,0,0,5,0, 0,0,36,0,102,23,86,28, 9,0,75,142,74,0,0,0, 89,85,64,0,119,139,99,0,
+ 121,119,159,71,88,75,139,88, 67,69,78,82,156,71,75,61, 134,70,55,54,61,63,56,60, 80,102,119,63,76,71,134,73,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,24,140,0,0,147,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 195,107,138,80,100,83,158,123, 107,139,106,91,157,127,159,122, 111,135,127,120,115,117,108,109, 106,109,120,113,182,78,121,111,
+ 84,149,138,143,143,145,133,136, 138,145,127,129,143,142,149,143, 141,113,143,148,151,137,137,140, 124,119,115,112,96,117,97,106,
+ 108,137,132,140,136,131,138,130, 131,133,119,128,140,133,137,128, 135,113,137,152,143,136,130,123, 116,108,106,112,84,77,91,0,
+ 196,150,226,235,119,165,141,52, 52,52,52,115,52,142,52,124, 121,52,147,165,107,52,52,52, 97,124,126,52,187,166,117,52,
+ 216,191,222,181,200,177,168,187, 191,218,177,170,169,189,169,153, 169,175,160,194,171,165,189,171, 182,163,176,175,198,160,155,150,
+ 0,0,205,182,0,189,133,0, 0,0,0,163,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,209,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 12,0,2,2,2,2,2,2, 0,0,0,0,2,128,2,128, 66,0,20,36,40,48,50,54, 0,0,0,0,2,128,2,128,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128, 128,0,84,128,128,128,128,128, 0,0,0,0,128,128,128,128,
+ 10,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128, 128,0,128,128,128,128,128,128, 0,0,0,0,128,128,128,128,
+ 120,0,130,112,192,198,136,140, 2,168,134,98,134,128,124,128, 160,0,168,162,178,176,178,178, 2,2,2,2,170,128,2,128,
+ 178,0,174,178,150,154,158,156, 2,2,2,2,158,128,164,128, 156,0,162,152,182,182,182,184, 2,26,2,2,170,128,10,128,
+ 0,0,128,128,128,128,128,128, 128,108,130,130,2,128,2,128, 0,0,128,128,128,128,128,128, 120,210,124,154,74,128,70,128,
+ 0,0,128,128,128,128,128,128, 140,2,2,2,2,128,2,128, 0,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ },
+},
+
+{ // X-TAM-ELANGO (0.036M chars) [60]
+ {NULL, NULL, NULL, NULL},
+ 126, 180, 58, 30, 129,
+ {0,180,180,180,180,180,180,180, 180,191,191,180,180,191,0,180, 170,170,170,170,170,170,170,170, 170,170,170,170,170,170,0,170,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,0, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,0,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,180,
+ 180,0,180,180,0,0,0,0, 0,0,0,0,180,0,180,180, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,181,181,181,181,181,181,181, 181,191,191,181,181,191,0,181, 171,171,171,171,171,171,171,171, 171,171,171,171,171,171,0,171,
+ 0,181,181,181,181,181,181,181, 181,181,181,181,0,0,181,0, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,0,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,0,0,181,181,
+ 181,0,181,181,0,0,0,0, 0,0,0,0,181,0,181,181, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,180,180,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,0,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,180,180,0,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,0,180,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,0, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,0,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,180,
+ 180,0,180,180,0,0,0,0, 0,0,0,0,180,0,180,180, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {110,0,110,110,110,110,110,110, 110,110,110,110,110,110,110,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 132,0,132,132,132,132,132,132, 132,132,132,132,132,132,132,128, 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128,
+ 136,0,136,136,136,136,136,136, 136,136,136,136,136,136,136,128, 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128,
+ 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128, 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128,
+ 136,0,136,136,136,136,136,136, 136,136,136,136,136,136,136,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ },
+},
+
+{ // X-TAM-LTTMBARANI (0.043M chars) [61]
+ {NULL, NULL, NULL, NULL},
+ 141, 199, 69, 34, 128,
+ {0,178,178,0,178,0,178,178, 0,187,187,178,178,176,0,0, 0,0,0,0,0,168,0,0, 0,0,168,168,168,0,0,168,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 0,176,176,0,176,0,176,176, 0,187,187,176,176,178,0,0, 0,0,0,0,0,165,0,0, 0,0,165,165,165,0,0,165,
+ 0,176,176,176,176,176,177,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,182, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,177,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 181,176,176,176,176,176,176,176, 176,176,176,176,176,189,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,178,177,0,0,177,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 185,177,177,177,177,177,178,177, 177,177,177,177,178,177,178,177, 177,177,177,177,177,177,177,177, 177,177,177,177,180,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,0,
+ 0,178,178,0,178,0,178,178, 0,178,178,178,178,0,0,0, 0,0,0,0,0,178,0,0, 0,0,178,178,178,0,0,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,179, 179,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {116,0,114,116,116,116,116,116, 118,118,116,118,118,118,118,118, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 134,0,132,134,134,134,134,134, 132,132,132,132,132,132,132,132, 138,0,136,138,138,138,138,138, 136,136,136,136,136,136,136,136,
+ 138,0,138,138,138,138,138,138, 136,136,136,136,136,136,136,136, 138,0,138,138,138,138,138,138, 136,136,136,136,136,136,136,136,
+ 138,0,136,138,138,138,138,138, 136,136,136,136,136,136,136,136, 138,0,138,138,138,138,138,138, 136,136,136,136,136,136,136,136,
+ 138,0,136,138,138,138,138,138, 136,136,136,136,136,136,136,136, 136,0,142,138,136,136,136,136, 136,136,136,136,136,136,136,136,
+ },
+},
+
+{ // X-TAM-SHREE (0.037M chars) [62]
+ {NULL, NULL, NULL, NULL},
+ 140, 204, 70, 30, 129,
+ {0,0,0,0,0,0,0,0, 0,188,179,0,0,179,0,0, 0,168,0,0,0,168,0,0, 0,0,0,168,0,0,0,168,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,0,0,178,178, 178,178,178,178,178,178,0,0, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,0,178,
+ 0,0,0,0,0,0,0,0, 0,188,178,0,0,178,0,0, 0,169,0,0,0,169,0,0, 0,0,0,169,0,0,0,169,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,0,0,179,179, 179,179,179,179,179,179,0,0, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,0,179,
+ },
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,0,
+ 0,0,0,0,0,0,0,0, 0,179,0,0,0,0,0,0, 0,179,0,0,0,179,0,0, 0,0,0,179,0,0,0,179,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,0,0,179,179, 179,179,179,179,179,179,0,0, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,0,179,
+ },
+ {132,0,132,132,132,132,132,132, 134,134,132,132,132,132,132,132, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 118,0,118,118,118,118,118,118, 118,118,118,118,118,118,118,118, 136,0,136,138,138,138,138,136, 138,138,138,138,138,138,138,138,
+ 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138, 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138,
+ 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138, 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138,
+ 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138, 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138,
+ },
+},
+
+{ // X-TAM-TBOOMIS (0.038M chars) [63]
+ {NULL, NULL, NULL, NULL},
+ 139, 205, 71, 31, 129,
+ {178,0,0,0,0,0,0,0, 0,178,178,0,0,178,0,0, 0,0,0,168,168,0,0,0, 0,0,0,0,0,0,0,0,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,0,0,0,0,0,0,0, 0,178,178,0,0,178,0,0, 0,0,0,168,168,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,200,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 179,178,178,178,181,190,178,188,
+ },
+ {0,0,0,0,0,0,0,0, 0,178,178,0,0,178,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 200,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,180,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,177,
+ 178,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,178,178,0,0,0, 0,0,0,0,0,0,0,0,
+ 177,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {134,0,130,132,132,132,132,132, 132,132,132,132,132,132,132,132, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 94,94,94,94,94,94,94,94,
+ 118,0,114,116,116,116,116,118, 116,116,118,116,116,116,116,116, 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136,
+ 138,0,134,136,136,136,136,136, 136,136,138,136,136,136,136,136, 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136,
+ 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136, 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136,
+ 134,0,144,134,134,134,134,134, 136,136,138,136,136,136,136,136, 134,0,138,136,134,134,134,134, 136,136,138,136,136,136,136,136,
+ },
+},
+
+{ // X-TAM-TMNEWS (0.037M chars) [64]
+ {NULL, NULL, NULL, NULL},
+ 141, 205, 71, 29, 128,
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,190,179,180,181,179, 180,179,179,179,179,179,179,179,
+ },
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,179,180,179,179,179,179,179, 179,179,179,179,180,179,180,179, 179,179,179,179,179,179,179,179, 179,179,180,179,181,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 179,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ },
+ {138,0,136,136,138,138,138,138, 128,128,136,136,136,136,136,136, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 138,0,136,136,136,136,136,136, 128,128,136,136,136,136,136,136, 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136,
+ 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136, 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136,
+ 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136, 136,0,140,136,136,136,136,136, 128,128,136,136,136,136,136,136,
+ },
+},
+
+{ // X-TAM-WEBTAMIL (0.050M chars) [65]
+ {NULL, NULL, NULL, NULL},
+ 142, 193, 66, 37, 129,
+ {178,178,178,178,178,178,178,178, 178,186,186,178,178,186,178,178, 169,169,169,169,169,169,0,169, 169,169,169,169,169,169,169,169,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,179, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 174,174,174,174,174,174,174,174, 174,186,186,174,174,186,174,174, 162,162,162,162,162,162,0,162, 162,162,162,162,162,162,162,162,
+ 0,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,181, 174,174,174,174,174,174,174,174,
+ 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174,
+ 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174,
+ },
+ {0,0,0,0,0,0,0,0, 0,178,177,0,0,177,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 158,177,177,177,177,177,178,177, 177,177,177,177,177,177,178,177, 177,177,177,177,177,177,177,177, 177,177,177,177,179,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,178,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,0,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,133,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,178, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ },
+ {108,0,108,108,108,108,108,108, 110,110,110,110,110,110,110,110, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ },
+},
+
+{ // UTF8CP1252 (178.156M chars) [66]
+ {NULL, NULL, NULL, NULL},
+ 127, 200, 59, 31, 133,
+ {181,189,183,184,176,167,163,162, 171,165,167,170,170,164,161,165, 163,161,165,161,164,166,165,166, 163,164,168,163,171,163,158,162,
+ 206,174,166,168,175,173,174,175, 177,166,173,171,187,176,178,175, 178,168,166,174,175,166,165,168, 186,173,179,177,184,174,171,171,
+ 119,121,206,216,183,183,147,141, 116,157,118,125,127,125,181,171, 210,198,125,120,113,136,145,187, 200,196,150,157,110,121,122,116,
+ 177,173,191,209,190,201,198,198, 188,185,166,178,183,167,123,183, 120,134,134,122,116,132,155,115, 122,120,125,97,146,127,124,122,
+ 186,143,146,128,162,163,140,136, 145,136,144,136,144,130,143,126, 134,159,199,183,181,182,183,161, 136,161,149,139,147,147,140,133,
+ 194,145,152,160,146,142,140,152, 139,166,140,162,150,144,159,142, 165,141,142,133,158,130,129,165, 138,130,156,172,141,139,132,152,
+ 146,177,150,157,188,159,153,165, 178,205,165,144,125,177,148,136, 125,164,126,181,156,148,180,118, 161,109,162,137,189,137,127,170,
+ 196,183,151,159,189,172,153,166, 185,210,166,152,154,180,147,138, 133,164,153,186,156,147,178,152, 160,162,164,140,188,144,126,109,
+ },
+ {148,50,56,78,0,0,0,0, 0,146,153,0,0,166,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 225,137,174,103,110,121,177,155, 149,159,126,115,183,160,179,142, 162,172,165,154,151,151,144,143, 141,143,157,138,205,108,138,142,
+ 124,187,176,189,179,184,170,182, 175,169,154,158,186,182,198,181, 175,156,199,194,193,169,173,155, 158,150,147,134,124,138,112,134,
+ 122,193,181,189,179,191,172,183, 177,178,152,159,187,182,199,184, 175,153,200,203,195,176,174,143, 147,148,143,122,135,97,118,45,
+ 189,193,197,200,179,177,173,171, 177,169,174,172,176,171,162,170, 167,165,159,165,169,171,167,170, 167,172,168,170,176,168,162,169,
+ 201,180,183,168,179,169,167,182, 176,187,173,166,167,178,173,174, 182,177,171,180,173,177,171,169, 186,175,179,179,186,179,179,171,
+ 129,129,169,152,124,152,130,133, 107,158,114,130,116,114,119,120, 120,118,126,120,107,124,147,116, 137,132,118,122,118,119,119,156,
+ 174,145,176,207,186,198,193,190, 186,184,161,173,176,161,128,175, 133,123,125,123,107,155,137,103, 119,110,131,110,136,122,127,123,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,2,4,2,6, 2,0,2,2,2,2,2,2, 0,0,0,0,4,14,2,16,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,2,2,2,2, 32,0,2,4,2,2,2,2, 0,0,0,0,2,2,2,2,
+ 156,0,136,152,124,138,112,108, 126,134,130,124,120,126,140,96, 138,0,138,136,122,116,134,138, 124,130,118,122,140,122,146,112,
+ 150,0,140,156,114,112,112,112, 108,122,140,112,150,128,142,122, 144,0,142,152,118,120,114,114, 124,134,120,126,132,136,144,128,
+ 0,0,90,80,150,152,98,98, 134,120,142,130,128,128,70,112, 0,0,102,98,150,148,124,110, 134,128,128,144,104,128,66,114,
+ 0,0,138,118,64,72,142,142, 138,138,122,134,110,120,110,140, 0,0,126,122,70,82,148,144, 88,108,108,110,162,194,128,196,
+ },
+},
+
+}; // End unigram_table
+
+static const uint8 kMostLikelyEncoding[] = {
+// 00xx
+ 37,39,39,39,39,39,39,39, 39,37,37,39,39,39,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 37,37,37,39,39,37,39,39, 39,39,37,39,37,37,39,37, 39,39,39,39,39,39,39,39, 39,39,37,39,37,39,37,39,
+ 37,39,39,39,39,37,39,39, 37,37,39,37,39,39,39,39, 37,39,37,37,37,37,37,37, 39,39,39,37,39,37,56,39,
+ 39,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 39,37,37,37,39,37,39,56,
+ 56,39,56,56,39,39,39,39, 56,56,56,56,56,56,39,56, 56,56,39,39,57,56,56,39, 56,56,56,57,39,39,56,39,
+ 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,57, 39,56,39,39,39,56,39,39, 39,39,39,39,39,57,39,39,
+ 39,39,57,39,39,39,39,39, 39,39,39,39,56,39,39,39, 39,39,39,39,39,57,39,39, 57,57,57,57,56,57,56,57,
+ 39,37,39,39,37,39,39,39, 39,37,39,39,39,37,39,39, 39,37,39,37,39,39,39,39, 39,39,39,39,37,39,39,57,
+ // 01xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 39,39,39,37,37,37,39,39, 39,39,37,37,39,39,39,37, 39,37,39,37,37,37,37,37, 37,37,39,39,39,37,37,39,
+ 39,37,37,37,37,39,37,39, 37,39,39,39,39,39,37,39, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,39,39,39,37,37, 37,39,39,37,39,39,39,39, 39,37,39,39,39,39,39,39, 39,39,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 39,37,37,39,39,39,39,39, 37,39,37,39,39,39,37,37, 37,39,37,39,39,37,39,39, 37,39,37,37,37,37,37,39,
+ 37,39,39,39,37,39,39,39, 39,39,39,39,39,37,37,37, 37,37,37,39,37,39,37,39, 37,56,56,56,37,56,39,56,
+ 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,37, 39,39,39,39,39,37,39,37, 39,39,39,37,39,39,37,39,
+ // 02xx
+ 37,37,37,39,37,37,37,39, 39,39,39,37,37,39,37,37, 37,8,37,37,37,37,37,37, 37,37,37,37,39,39,37,39,
+ 37,37,39,40,37,39,37,37, 39,39,39,39,39,39,39,39, 39,37,37,37,37,37,40,37, 40,40,39,39,39,39,37,37,
+ 6,39,39,40,39,37,39,39, 39,39,37,37,39,39,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,39,37,
+ 37,37,37,37,39,37,37,37, 37,37,39,37,37,37,37,39, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,39,37,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,39,37,37,39, 39,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,39,37,37,37,37,37,39, 37,39,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,37,37,
+ // 03xx
+ 56,37,37,37,56,37,37,37, 37,37,39,56,37,39,39,37, 39,37,37,39,37,39,37,37, 37,39,56,39,37,37,39,39,
+ 39,39,39,37,37,37,39,39, 39,39,39,39,39,39,39,39, 37,39,39,39,39,39,39,39, 39,56,39,39,39,39,39,56,
+ 39,39,39,56,56,39,39,39, 39,39,39,39,39,39,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,39,37,
+ 37,37,37,39,37,37,37,39, 37,39,37,37,37,39,39,39, 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,40,37,
+ 39,37,37,39,39,37,37,37, 37,37,37,37,39,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,39,37,39,37,37, 37,39,39,37,39,37,37,37, 39,37,39,37,39,37,37,37, 39,39,39,39,39,39,37,37,
+ // 04xx
+ 56,39,37,39,37,39,37,37, 37,39,39,37,39,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,39,37,37,37,39,39, 37,39,39,37,39,39,39,37, 39,39,37,39,39,39,39,37, 39,39,39,39,39,39,39,37,
+ 39,39,39,39,39,37,39,39, 39,39,39,39,39,37,39,39, 37,39,37,37,39,37,37,37, 37,39,37,37,39,39,37,39,
+ 37,39,37,39,39,37,37,37, 39,39,37,39,37,37,39,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,39,37,
+ 37,37,37,39,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,39,37,37,37,39, 37,37,37,37,39,37,37,39,
+ 39,37,39,39,37,37,39,39, 37,39,37,37,39,39,37,37, 37,39,37,37,39,39,37,39, 37,39,37,39,39,39,39,37,
+ 37,37,37,39,37,37,39,37, 39,37,39,37,37,39,39,37, 39,39,37,37,37,39,39,39, 37,37,37,37,37,37,37,37,
+ 39,37,39,39,39,39,39,39, 37,37,39,37,39,37,37,37, 37,37,37,37,39,39,39,37, 39,39,39,39,37,39,37,37,
+ // 05xx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,37,39,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,39,39,37,37,37,39,39, 39,39,39,39,39,39,39,39, 37,39,39,39,37,37,40,37, 37,39,39,39,39,39,39,39,
+ 37,39,40,39,37,37,37,39, 37,37,37,37,40,37,37,37, 37,37,37,39,37,37,39,37, 37,37,37,37,37,37,39,39,
+ 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,39,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,39,37, 39,39,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,39,37,37,37,
+ // 06xx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,39,39,39,39, 37,37,37,39,37,37,37,37, 39,39,37,37,37,37,37,37,
+ 39,39,39,37,37,37,39,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,39,39,39,39,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,39,37,37,37,37,37, 37,37,37,39,37,37,37,37,
+ 39,37,37,39,37,37,37,39, 37,37,37,39,39,39,39,37, 37,37,37,37,37,37,37,39, 37,37,37,39,37,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,39, 37,37,37,39,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 39,39,39,39,39,37,39,37, 39,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 07xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 39,37,39,40,37,37,37,37, 37,39,37,37,39,37,39,37, 37,39,37,40,37,40,37,37, 37,37,40,37,39,37,37,37,
+ 37,37,39,37,37,37,40,40, 37,37,37,40,37,37,39,37, 37,37,39,37,37,39,37,37, 37,39,39,37,39,39,37,37,
+ 37,37,37,39,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,37,37,37,39,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 08xx
+ 37,37,37,37,37,37,37,37, 3,39,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,40,40,37,37,37, 40,40,37,37,37,37,37,37, 37,39,40,40,40,40,40,40, 40,40,40,40,40,40,40,37,
+ 40,40,40,40,37,37,37,37, 40,40,37,37,40,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,39,37,39,37,
+ 37,39,37,37,37,37,37,39, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,37, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,39,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37, 37,39,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 37,37,37,39,37,37,39,37, 39,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,39,
+ // 09xx
+ 37,37,37,37,37,37,37,37, 37,0,0,37,37,0,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 0,10,0,0,0,0,0,0, 0,0,0,0,0,10,0,10, 10,0,0,0,0,0,0,0, 0,0,0,0,0,10,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ // 0Axx
+ 37,37,37,37,37,37,37,37, 37,0,0,37,37,0,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 0,10,0,0,0,0,0,0, 0,0,0,0,0,10,0,10, 10,0,0,0,0,0,0,0, 0,0,0,0,0,10,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ // 0Bxx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,40, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 39,40,39,40,37,37,37,37, 37,39,37,37,39,39,39,40, 37,39,39,39,39,39,39,40, 39,39,39,37,39,37,37,37,
+ 37,37,40,40,37,37,40,37, 39,37,37,37,39,37,39,37, 39,37,37,37,37,37,37,39, 37,37,37,37,39,37,37,37,
+ 37,37,37,37,37,37,37,39, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,39, 37,37,39,37,37,37,40,37,
+ 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 0Cxx
+ 57,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,39,37,37,37,37,37, 37,39,37,37,39,37,37,37, 37,39,37,37,37,37,37,37, 37,37,39,37,39,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 0Dxx
+ 37,37,37,37,37,37,37,37, 37,0,0,37,37,0,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 0,10,0,0,0,0,0,0, 0,0,0,0,0,10,0,10, 10,0,0,0,0,0,0,0, 0,0,0,0,0,10,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ // 0Exx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,57,56,57,56,37,56,37, 56,56,37,56,56,37,39,39, 57,57,57,57,57,57,57,57, 57,57,57,57,57,57,57,57,
+ 57,57,56,57,57,57,57,57, 57,57,56,57,57,57,56,56, 57,56,57,56,56,57,56,57, 56,56,56,56,57,56,56,56,
+ 56,56,56,56,56,56,39,56, 56,56,56,56,56,56,56,56, 56,56,56,56,56,37,56,56, 37,56,39,56,56,37,37,37,
+ 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,39,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 6,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 0Fxx
+ 37,37,40,37,37,37,37,37, 40,42,42,37,37,42,40,37, 37,37,37,37,37,37,37,37, 37,40,37,57,37,37,37,37,
+ 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42,
+ 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42,
+ 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,37,
+ 37,39,42,37,37,37,37,37, 37,37,37,37,37,37,39,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 10xx
+ 56,39,37,37,39,37,39,37, 37,37,39,37,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,39,39,37,39,
+ 39,40,39,37,37,37,37,37, 40,39,40,37,39,39,39,39, 37,39,37,39,40,40,37,37, 40,40,39,39,39,37,37,39,
+ 37,40,40,40,37,40,37,37, 37,40,40,37,40,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,39,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,39,37,39,37, 39,37,37,39,37,37,37,37, 39,39,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 3,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37,
+ // 11xx
+ 56,39,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,40,39,40,37,37,37,37, 37,37,37,37,37,37,39,37, 37,40,40,40,40,40,40,37, 40,40,40,40,40,40,40,40,
+ 37,40,40,40,40,37,40,40, 40,40,40,40,37,40,37,39, 39,37,37,37,39,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,39,39,37,37,37,37, 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,40,37,
+ 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,3,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37,
+ // 12xx
+ 37,37,37,37,39,39,37,37, 37,37,37,39,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,40,39,39,37,37,37,37, 37,39,37,39,37,37,39,39, 37,39,37,40,37,37,37,37, 39,40,37,39,39,37,37,37,
+ 40,37,37,37,37,39,37,37, 37,37,39,37,37,37,37,37, 39,37,39,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 39,37,39,37,37,37,37,37, 37,39,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 13xx
+ 37,39,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 39,37,37,40,37,37,37,37, 37,37,39,37,39,39,37,37, 37,39,40,37,40,37,37,37, 37,37,40,37,39,37,37,37,
+ 37,37,37,40,37,37,37,37, 37,37,37,37,40,37,39,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,39,37,39,37, 39,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,39,
+ 37,37,37,37,37,39,37,37, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,39, 37,39,39,37,39,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,6,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 39,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,39,37,39,37,37, 37,37,37,37,37,37,37,39,
+ // 14xx
+ 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,39,39,39,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,40,37,39,37,37, 39,37,37,39,39,39,39,37, 37,37,37,39,40,40,40,40, 40,37,40,40,39,37,40,40,
+ 40,40,40,40,40,37,39,39, 40,40,40,40,37,40,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,39,37,39,40,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,39,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 15xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,40,39,40,39,37,39,37, 37,37,37,37,39,37,37,37, 37,37,39,37,40,40,37,37, 37,40,37,37,39,37,40,40,
+ 37,40,40,37,37,37,37,37, 37,37,40,37,40,37,37,37, 37,37,37,37,37,37,39,37, 39,39,37,37,37,37,37,39,
+ 39,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,40,37,
+ 39,37,37,37,37,37,37,39, 39,37,37,37,37,37,37,39, 37,39,39,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 37,39,37,37,37,39,37,39, 37,37,37,39,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,39,37,37,37,37,37, 39,37,37,37,37,37,37,37, 39,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 16xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,39,37,37,37,37,37,37, 37,40,37,37,37,37,37,40, 37,39,37,40,37,37,40,37, 40,40,37,40,37,37,40,37,
+ 37,40,37,37,40,37,40,37, 40,37,37,40,37,37,39,37, 37,37,37,39,37,37,37,39, 37,39,37,37,37,37,37,37,
+ 39,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 17xx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,39,37,37, 40,37,37,37,37,37,40,37, 37,39,40,37,37,37,37,37, 37,37,37,37,37,40,40,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,39,39,39,37,39,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 39,37,37,37,39,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 39,37,39,37,37,39,37,39, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 18xx
+ 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,39,37,40,37,37,37,37, 37,37,37,37,40,37,37,40,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 39,37,37,37,37,37,37,39, 37,37,39,37,39,37,37,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ 37,37,37,37,37,37,37,39, 37,37,39,37,37,37,37,37, 37,37,37,37,39,37,37,37, 39,39,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 19xx
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,37,40,37,37,37, 37,37,37,37,37,37,40,37, 37,37,37,37,37,37,37,37, 37,37,40,37,37,40,37,37,
+ 37,37,37,40,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,39,37,37,37,37, 37,37,39,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,39,37,37,37,39,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 1Axx
+ 37,37,37,37,37,37,37,37, 37,37,4,37,37,37,37,40, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,40,39,40,37,37,37,37, 37,37,37,37,37,37,37,37, 37,39,37,37,40,40,37,37, 37,37,37,37,37,37,40,6,
+ 40,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,39, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 1Bxx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,57,37,37,37, 57,37,37,37,37,37,37,37, 37,39,37,37,40,40,37,37, 40,37,37,37,37,40,37,37,
+ 37,37,37,37,37,40,37,37, 37,37,37,37,37,37,56,37, 37,37,39,37,39,37,37,37, 37,37,39,37,37,39,37,37,
+ 37,39,37,37,37,37,37,39, 37,37,37,37,39,39,39,37, 37,37,39,39,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 1Cxx
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,40,37,39,37,37, 37,37,40,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,40,37,37,37,37, 37,40,40,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,39,39,37,37,37,37,37,
+ 37,37,37,37,39,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,39,37,39,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 1Dxx
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,40,40,37,39,37,37, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 40,37,37,37,37,40,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 39,37,37,37,37,37,37,39, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,39,37,37,37, 37,37,37,39,37,39,39,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 1Exx
+ 57,37,37,37,37,37,37,37, 37,39,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 39,40,39,40,37,37,39,39, 37,39,37,37,39,39,39,39, 37,40,42,40,40,40,40,37, 40,40,39,39,39,40,37,39,
+ 37,39,42,37,37,37,42,37, 42,39,40,40,40,39,39,39, 39,37,37,37,39,39,37,37, 37,37,42,37,37,37,37,37,
+ 37,39,37,39,39,39,37,37, 39,39,42,39,37,37,39,39, 39,37,37,37,39,39,37,39, 37,39,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,39,37,37, 39,37,39,37,37,37,37,39, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 1Fxx
+ 37,37,37,37,39,37,39,37, 37,39,37,37,37,39,40,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,40,42,40,37,37,37,37, 42,37,37,42,39,37,37,42, 37,39,40,40,40,40,40,40, 40,40,40,40,40,40,42,40,
+ 40,40,42,40,40,40,40,42, 40,40,40,40,42,42,37,37, 39,37,37,37,37,37,37,39, 37,37,39,37,37,37,37,37,
+ 37,39,37,37,37,37,39,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,39, 37,37,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,39,37,37, 39,37,37,37,37,37,39,37, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,39,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 20xx
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 21xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 22xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 23xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 24xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 25xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 26xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 27xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 28xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 29xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Bxx
+ 2,39,1,37,37,37,37,5, 3,2,2,37,3,2,42,42, 37,40,37,37,37,40,37,37, 40,37,40,57,37,37,37,21,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,3,3,3,11,11,11,11, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,11,12,11,12,36,12,4,
+ 11,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,
+ 2,2,2,2,2,2,2,11, 2,1,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,11,2,1,11,13,6,
+ 14,10,2,2,2,2,2,2, 2,2,2,2,2,10,17,2, 10,10,10,10,10,10,1,17, 17,17,17,2,11,10,13,37,
+ // 2Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 30xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 31xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 32xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 33xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 34xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 35xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 36xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 37xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 38xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 39xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 40xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 41xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 42xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 43xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 44xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 45xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 46xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 47xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 48xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 49xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 50xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 51xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 52xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 53xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 54xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 55xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 56xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 57xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 58xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 59xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 60xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 61xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 62xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 63xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 64xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 65xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 66xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 67xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 68xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 69xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 70xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 71xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 72xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 73xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,31, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,33,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 74xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 75xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 76xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 77xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 78xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 79xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Exx
+ 2,39,39,39,39,39,39,39, 39,2,2,39,39,2,42,42, 39,39,39,39,39,39,39,39, 39,39,39,56,39,39,39,39,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,3,
+ 3,3,3,3,39,39,11,11, 11,3,3,3,3,3,3,3, 3,3,3,3,3,11,3,3, 39,11,39,3,39,39,39,37,
+ 11,3,3,3,11,3,3,3, 3,3,3,3,3,3,11,3, 11,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,
+ 2,2,2,2,2,2,2,11, 2,2,2,2,11,11,2,2, 2,11,2,11,11,2,2,2, 2,37,11,11,11,11,11,11,
+ 17,11,2,2,2,2,2,2, 2,2,2,2,2,2,17,2, 14,17,17,11,37,1,11,17, 16,17,11,39,37,11,39,37,
+ // 7Fxx
+ 56,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39, 39,39,39,37,39,37,37,39, 37,39,39,39,39,37,39,39,
+ 39,39,39,39,39,39,39,39, 39,37,39,39,39,39,39,39, 37,39,39,39,39,39,37,39, 39,39,37,39,39,39,39,39,
+ 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,37,39, 37,37,37,37,37,39,37,37, 39,37,39,37,37,37,39,37,
+ 37,39,39,37,37,37,37,39, 39,37,37,37,37,37,37,39, 37,37,37,39,37,37,37,39, 39,37,39,37,39,37,37,39,
+ 37,39,37,37,39,37,39,39, 37,39,37,37,39,37,37,37, 39,39,39,39,37,39,37,39, 39,37,37,39,39,39,39,39,
+ 39,39,39,39,37,39,39,39, 37,37,39,39,39,39,39,39, 39,39,39,39,37,39,39,39, 39,39,39,39,39,37,39,39,
+ 39,37,39,39,39,37,39,39, 39,39,39,39,37,39,39,39, 39,39,37,39,37,39,39,39, 56,56,39,56,56,56,56,56,
+ 37,39,39,39,39,39,39,39, 39,37,39,39,39,39,39,39, 37,37,39,39,39,39,39,39, 39,39,39,37,37,39,39,37,
+ // 80xx
+ 11,37,39,37,39,37,39,39, 39,2,2,39,37,2,39,39, 37,39,39,39,39,37,39,37, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,11, 2,11,11,2,11,11,11,11, 11,11,11,2,11,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,11,2,2, 2,2,2,2,2,2,2,39,
+ 2,2,2,2,36,2,2,36, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 59,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,39,2,2,2,10,37,2, 10,2,39,2,37,39,2,2, 2,2,2,10,39,2,33,11, 2,2,2,2,37,5,37,5,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 36,31,39,39,39,31,37,39, 37,39,37,39,11,37,4,37,
+ // 81xx
+ 56,39,37,39,39,37,39,37, 37,2,2,39,39,2,39,39, 37,37,39,39,39,39,39,39, 39,37,37,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,6,2,6,6,6,2,6, 6,6,6,2,2,6,6,2, 6,2,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 36,6,2,6,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,2, 2,2,6,2,6,2,2,2,
+ 2,6,6,2,2,6,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,39,2,2,2,10,4,11, 6,6,6,6,6,6,2,2, 2,11,11,11,11,39,12,19, 2,2,6,2,6,6,6,6,
+ 2,6,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,6,6,6,6,6,6,6, 4,4,37,39,6,19,39,37,
+ // 82xx
+ 56,39,39,39,39,39,39,37, 37,2,2,39,39,2,42,39, 39,39,37,39,39,39,39,39, 39,39,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,6, 6,6,6,6,6,6,6,6, 6,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,2,2,2,2,2,37,
+ 2,2,2,2,2,6,2,2, 2,2,2,2,2,2,6,2, 6,36,2,2,6,6,6,6, 2,2,2,36,2,2,2,6,
+ 6,2,6,2,6,2,6,2, 6,6,6,6,2,6,6,2, 2,6,6,6,2,6,6,6, 2,2,2,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,2,2,6,6, 6,6,6,6,6,6,6,2, 6,6,31,12,11,31,39,19, 5,19,39,19,11,33,39,37,
+ // 83xx
+ 56,39,37,37,37,37,39,39, 39,2,2,39,39,2,37,39, 39,39,39,39,39,39,39,37, 39,39,39,39,37,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 6,6,6,2,6,6,2,6, 2,6,6,6,6,6,2,6, 2,2,2,6,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,6,
+ 6,6,59,6,6,59,59,6, 19,6,6,6,6,6,2,2, 2,2,6,6,19,6,6,11, 2,2,2,2,37,6,4,4,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 31,4,11,10,39,19,19,11, 37,39,39,39,39,39,39,37,
+ // 84xx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,11,2,2,2, 2,2,2,11,2,2,2,2, 2,2,2,11,2,2,11,11, 2,2,11,2,2,2,2,2,
+ 37,11,2,2,11,2,2,11, 2,2,2,11,2,2,11,2, 2,6,2,2,6,6,39,2, 6,6,11,6,2,6,2,37,
+ 36,2,2,2,2,36,2,2, 2,2,2,36,2,36,36,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,2,59,6,2,6,2,2, 36,6,6,6,6,2,36,6, 2,2,2,2,2,6,2,2, 2,2,6,5,2,5,2,2,
+ 10,10,2,2,2,10,10,10, 12,12,10,2,10,10,2,2, 2,2,10,19,10,19,12,2, 2,2,2,2,12,19,16,10,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 36,10,10,10,11,12,12,10, 12,12,12,39,11,39,16,37,
+ // 85xx
+ 56,11,39,37,39,39,39,39, 39,2,2,39,39,2,39,11, 39,39,39,37,37,39,39,37, 37,37,37,39,39,39,39,39,
+ 2,2,2,2,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,39,2,2,2,2,11,11, 2,2,2,2,2,2,19,2,
+ 2,11,11,2,11,2,2,2, 2,11,33,2,2,2,2,11, 11,11,2,11,11,2,33,11, 11,11,33,2,2,2,2,39,
+ 2,2,36,2,2,2,2,2, 2,2,36,2,2,2,2,36, 2,36,2,2,2,2,36,36, 2,2,2,5,2,2,2,36,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 39,5,2,2,19,11,5,11, 11,12,5,2,2,37,2,2, 2,2,6,11,11,19,12,39, 2,2,2,2,11,11,11,11,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 36,36,4,23,23,4,19,11, 19,11,12,37,11,33,11,37,
+ // 86xx
+ 56,39,39,11,39,39,39,37, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,37,
+ 2,6,2,2,36,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,19,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,36,2, 2,2,2,2,2,5,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 39,19,2,2,2,2,2,2, 11,2,19,2,19,37,2,2, 2,2,4,12,12,19,12,11, 2,2,2,2,11,12,4,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 36,36,39,31,39,4,39,11, 4,39,39,39,11,12,19,37,
+ // 87xx
+ 56,39,37,39,39,39,39,39, 39,2,2,39,39,2,39,37, 39,37,37,37,39,37,39,37, 37,39,39,39,37,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,2,6,6,2,2,2,2, 2,11,2,2,6,6,6,6, 6,6,6,2,2,2,2,2,
+ 6,2,2,2,6,39,2,31, 2,11,11,2,2,2,2,6, 2,6,31,31,2,2,31,2, 11,2,37,2,2,2,2,37,
+ 36,6,2,2,2,36,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 2,2,2,36,2,2,5,2,
+ 2,2,36,2,2,2,2,2, 2,5,2,36,2,2,11,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 19,12,2,2,2,2,4,11, 4,12,4,2,19,39,2,2, 2,2,4,12,12,4,19,11, 2,2,2,2,11,11,31,19,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 36,4,37,11,11,11,4,11, 19,37,19,11,19,4,4,37,
+ // 88xx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,39,
+ 2,36,36,36,2,36,2,2, 2,2,2,36,36,36,2,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,2,2,36,
+ 2,2,6,6,6,2,2,36, 2,2,2,2,2,2,6,2, 2,2,2,6,2,6,2,2, 2,2,2,2,6,2,2,2,
+ 6,6,2,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6,
+ 2,6,6,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,39,37,
+ // 89xx
+ 56,39,39,39,39,39,39,37, 39,2,2,39,39,2,39,39, 37,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,2,2,6,6,6, 6,2,6,6,6,2,6,2, 6,6,6,6,2,2,6,2, 6,6,6,2,6,2,6,6,
+ 6,6,6,6,2,6,6,6, 6,6,6,6,6,2,2,2, 6,6,6,6,6,6,6,6, 6,37,6,6,6,6,6,37,
+ 2,2,2,6,6,6,2,2, 2,6,2,2,2,2,2,6, 6,6,36,2,6,2,6,6, 6,2,6,6,6,6,6,6,
+ 6,6,6,2,6,2,2,2, 6,2,6,6,2,6,6,2, 2,2,2,6,2,2,2,6, 6,6,6,6,6,6,2,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,2, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,6,2,2,2,2,6,2, 2,2,6,2,2,2,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,39,37,
+ // 8Axx
+ 56,39,37,39,39,39,39,39, 37,2,2,39,39,2,37,39, 39,39,39,37,39,39,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,12,6,6,6,6,6,6, 2,12,6,6,12,2,12,6, 12,6,2,2,6,6,12,6, 6,6,6,6,2,2,6,6,
+ 6,12,2,2,6,6,37,6, 2,6,6,12,6,6,6,6, 6,37,6,6,6,6,6,6, 6,6,6,6,6,6,6,39,
+ 2,6,2,6,6,36,36,6, 6,2,2,6,6,12,36,6, 2,2,36,2,2,2,2,2, 2,6,6,2,2,2,2,2,
+ 2,2,2,6,2,2,6,6, 2,2,6,2,2,2,6,6, 6,2,6,6,6,2,2,6, 2,2,6,6,2,6,6,2,
+ 10,12,6,6,6,6,12,6, 6,6,6,6,6,12,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,6,6,2,2,2,2,2, 2,2,2,6,2,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,4,37,
+ // 8Bxx
+ 56,39,37,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,37,39,37,39,37,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,2,6,6,6, 6,6,2,6,6,6,6,6, 6,6,6,6,6,6,6,2, 6,6,6,2,6,2,6,6,
+ 6,2,2,6,2,6,2,6, 2,6,6,6,6,2,2,31, 6,6,6,6,6,6,6,2, 6,6,6,6,6,6,6,37,
+ 2,6,2,6,6,6,6,2, 2,2,36,6,6,2,6,6, 6,2,6,6,2,2,6,2, 2,2,5,6,2,2,6,6,
+ 2,2,2,6,2,2,6,2, 2,2,2,6,2,6,2,2, 2,2,6,6,6,6,6,6, 6,2,6,6,2,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,37,37,
+ // 8Cxx
+ 56,39,39,37,39,39,39,39, 37,2,2,37,39,2,39,39, 37,39,39,37,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,2,2,12,2,2,6,2, 2,6,6,6,12,2,6,2, 6,6,6,6,2,11,2,6, 6,6,6,6,6,2,6,6,
+ 6,6,6,6,2,6,6,6, 6,6,6,6,2,2,6,6, 6,6,2,2,2,6,6,12, 6,6,2,6,6,6,2,37,
+ 2,2,6,2,2,2,6,2, 6,2,6,6,6,36,6,6, 2,2,6,6,6,6,2,2, 6,6,6,6,6,6,6,6,
+ 6,2,6,6,6,2,2,6, 2,6,2,2,6,6,2,6, 6,6,2,6,6,6,2,6, 6,6,2,6,6,2,6,2,
+ 6,6,6,6,6,6,12,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 2,2,2,2,6,6,6,6,
+ 2,6,2,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,4,4,37,
+ // 8Dxx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,37,37,
+ 2,2,2,37,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,12,2,6,6,6,6,6, 6,6,6,6,6,2,6,6, 6,6,6,6,6,6,2,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,6,6,6,6, 6,6,6,6,6,6,2,6, 2,6,6,6,2,6,6,6, 6,6,6,2,6,6,6,39,
+ 6,2,6,2,6,36,6,6, 2,2,2,6,6,36,2,6, 6,6,2,2,2,2,2,2, 2,36,2,36,6,6,6,2,
+ 6,6,6,2,2,6,2,2, 2,6,6,2,6,6,2,2, 2,2,6,2,2,2,6,2, 6,6,2,6,2,6,2,2,
+ 6,12,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,11,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,4,6,6,39,39,37,
+ // 8Exx
+ 56,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,37, 39,39,39,37,37,37,39,39, 37,39,39,39,39,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,12,6,2,6,12,6,6, 2,12,2,12,2,6,12,6, 2,6,6,12,2,6,2,2, 6,6,6,2,6,6,6,6,
+ 6,12,6,6,6,12,6,6, 6,6,6,6,6,6,6,6, 6,6,31,6,6,6,6,6, 6,2,2,6,6,6,6,39,
+ 6,6,2,6,6,2,6,2, 2,2,6,2,6,6,6,6, 36,6,2,2,6,6,6,6, 2,6,6,6,6,6,6,6,
+ 6,6,2,7,7,2,6,2, 2,6,2,6,7,6,6,6, 2,7,2,7,7,6,7,2, 6,7,6,2,7,7,6,6,
+ 6,12,6,7,7,6,7,11, 7,6,6,6,7,12,6,6, 6,7,6,6,6,6,6,7, 6,7,7,7,7,7,7,7,
+ 2,6,2,2,2,2,2,2, 6,2,6,6,2,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,8,37,
+ // 8Fxx
+ 57,39,39,37,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,37,37,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,2,6,6,6, 6,6,6,6,6,6,12,6, 6,6,6,2,6,2,6,6, 6,6,6,6,6,6,2,6,
+ 6,6,6,6,6,6,6,2, 6,2,6,6,2,6,6,6, 6,6,6,2,6,6,6,2, 6,6,6,6,2,6,6,39,
+ 6,6,2,6,2,2,6,6, 6,6,6,2,2,2,36,6, 2,6,2,2,6,6,2,6, 2,2,2,2,6,6,6,6,
+ 2,2,6,2,6,2,2,6, 36,2,2,6,6,6,36,2, 2,2,2,6,2,6,2,2, 2,6,6,6,6,2,2,5,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,6,6,6,
+ 2,6,2,2,2,2,2,2, 2,2,6,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,8,37,
+ // 90xx
+ 57,37,37,37,39,39,37,39, 39,2,2,39,39,2,39,37, 39,39,37,39,37,39,39,39, 39,37,37,39,39,39,39,37,
+ 2,2,2,37,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,2,6,6,6,6, 6,2,6,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,2,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 36,6,6,2,2,6,2,6, 2,2,2,6,2,2,2,6, 2,2,36,36,6,2,2,2, 2,6,6,2,2,6,2,6,
+ 36,6,6,6,6,2,2,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,2,6,6, 6,6,6,6,6,2,2,6,
+ 6,6,6,6,6,6,6,11, 6,6,6,6,6,6,6,6, 6,6,11,6,6,6,6,2, 6,6,6,6,6,6,6,6,
+ 6,2,6,2,2,2,2,2, 2,2,6,2,2,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,39,37,
+ // 91xx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,37,39,39, 39,39,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,2,6,2,6,6,6,2, 6,6,6,6,2,2,6,6, 6,6,6,6,6,6,6,11, 6,6,11,6,6,6,6,6,
+ 6,6,6,2,6,6,6,6, 6,2,6,6,2,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,2,39,
+ 6,6,2,6,2,36,2,6, 6,2,2,6,36,6,36,36, 6,6,36,36,6,6,6,6, 2,2,2,6,6,6,2,6,
+ 6,6,6,6,6,6,6,6, 2,2,6,6,6,6,6,36, 6,6,6,2,6,6,6,6, 6,6,6,2,6,6,6,6,
+ 10,6,2,6,6,6,2,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,2,2,2,2,6,2,2, 2,2,2,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,22,22,37,
+ // 92xx
+ 56,39,39,37,39,39,37,39, 39,2,2,39,39,2,39,39, 37,39,37,39,37,37,39,37, 37,39,39,39,37,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,11,2,2,2, 39,2,2,2,2,2,2,2, 2,2,2,11,2,2,2,2,
+ 6,11,6,6,6,6,2,2, 11,6,6,6,2,6,6,6, 6,6,2,6,6,6,2,6, 2,6,6,6,2,6,6,6,
+ 6,11,6,6,11,11,6,6, 11,6,6,6,6,6,6,11, 6,6,6,11,11,6,6,6, 6,11,6,6,6,6,6,37,
+ 36,6,36,6,2,6,6,6, 2,6,6,6,2,6,36,2, 36,6,2,6,2,2,2,2, 6,2,6,36,36,6,2,2,
+ 6,6,2,6,6,6,6,6, 2,6,6,2,6,2,6,2, 2,2,6,2,6,6,6,6, 6,6,6,2,6,2,6,6,
+ 6,6,6,6,6,6,2,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,2,2,2,2, 6,2,6,6,2,6,11,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,22,10,10,
+ // 93xx
+ 56,39,39,39,37,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,37, 37,39,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,11,11,11,11,6,11,6, 11,6,6,6,11,11,6,6, 11,11,11,6,6,11,6,6, 6,6,6,2,6,6,6,6,
+ 6,6,2,6,6,11,6,11, 6,2,11,6,6,2,2,6, 11,6,6,6,11,2,11,6, 6,6,6,6,6,6,6,37,
+ 6,2,6,6,2,2,36,6, 2,2,6,6,6,2,2,6, 6,36,6,2,6,6,6,36, 36,6,6,6,2,6,6,6,
+ 2,2,6,2,2,2,2,6, 2,6,6,6,6,6,6,6, 6,6,2,5,6,6,6,2, 2,6,2,6,2,6,6,6,
+ 10,6,6,59,6,6,6,6, 6,12,6,6,10,6,2,6, 6,2,6,10,6,6,6,2, 6,2,6,6,6,6,6,6,
+ 6,2,2,2,2,2,2,2, 2,2,2,2,6,6,6,2, 6,6,6,6,19,6,6,6, 6,6,6,6,6,10,19,37,
+ // 94xx
+ 56,39,39,39,39,39,39,39, 39,2,2,11,39,2,39,39, 39,39,39,39,39,39,39,37, 37,39,39,39,37,39,39,39,
+ 2,2,2,11,2,2,2,2, 2,2,2,2,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 6,6,2,6,6,6,6,6, 2,2,6,6,6,6,6,6, 2,2,6,6,6,2,6,2, 6,6,6,6,6,6,2,6,
+ 6,2,2,6,6,6,2,6, 6,2,6,2,2,2,6,6, 6,6,6,6,6,2,6,6, 6,6,6,6,6,6,6,39,
+ 2,2,2,6,6,2,2,2, 2,2,2,2,2,6,6,6, 2,6,6,6,2,6,6,6, 2,2,6,6,2,2,6,2,
+ 6,2,2,2,2,36,6,6, 2,2,6,6,2,6,2,2, 2,2,6,2,2,2,2,2, 2,2,2,2,6,6,2,2,
+ 6,6,6,2,6,6,6,6, 6,6,6,6,6,6,2,6, 2,6,6,6,6,6,6,2, 6,2,6,6,6,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,22,37,37,
+ // 95xx
+ 56,39,39,39,39,37,39,39, 37,11,2,39,39,11,39,39, 37,39,39,39,39,39,39,39, 39,37,39,39,39,39,39,39,
+ 11,2,2,39,11,2,11,2, 2,2,2,2,2,2,11,2, 2,2,2,2,2,2,2,2, 2,2,11,2,2,2,2,2,
+ 6,2,2,6,6,6,6,6, 6,6,2,6,11,6,11,6, 6,6,6,6,11,6,11,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 2,6,6,6,2,2,2,2, 2,6,2,6,2,6,6,2, 6,6,6,2,6,6,2,6, 2,2,6,6,2,6,6,6,
+ 6,6,2,2,2,6,2,6, 6,2,6,2,2,2,2,2, 2,6,6,6,2,6,6,6, 2,6,6,6,2,6,6,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,2,6,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,19,37,
+ // 96xx
+ 57,39,39,39,39,37,39,39, 39,2,2,39,39,11,39,39, 37,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,2,2,2,11,11,2,2, 2,2,2,2,2,2,2,2, 11,11,11,2,11,2,11,11, 11,11,2,2,2,2,11,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,2,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,11,6,6, 6,6,6,11,11,6,6,6, 6,6,6,6,6,6,6,39,
+ 2,6,6,6,2,6,2,2, 6,2,2,2,6,6,2,2, 2,2,2,2,6,6,6,2, 6,2,2,6,6,6,6,6,
+ 2,6,6,6,2,2,2,6, 6,2,2,2,2,2,2,6, 2,6,2,6,2,2,2,2, 2,2,6,2,6,2,6,6,
+ 6,6,6,6,6,6,6,11, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,2,6,2,2,2,2,2, 2,2,2,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,17,6,6,39,37,37,
+ // 97xx
+ 56,39,39,39,39,39,39,37, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,37,39,37,39,
+ 2,2,2,11,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,11,6,6,6,6,6, 6,2,6,6,6,6,6,11, 6,6,6,11,6,6,6,6, 6,6,6,6,2,6,6,39,
+ 36,6,6,6,2,2,2,2, 6,2,6,2,2,6,6,2, 2,6,6,6,2,2,6,6, 6,6,6,2,2,6,6,6,
+ 2,2,6,6,6,2,6,6, 2,2,6,6,6,2,2,6, 2,6,6,6,2,6,2,6, 2,6,2,2,6,2,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,6,6,2,2,2,2,2, 2,2,6,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,39,37,
+ // 98xx
+ 57,37,37,37,39,37,37,39, 37,2,2,39,39,2,39,39, 37,37,37,37,39,39,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,2,2,6,2, 6,6,6,2,6,2,6,6, 2,6,6,2,2,6,6,2, 6,6,6,6,6,2,6,6,
+ 6,6,6,2,2,6,6,6, 6,2,6,2,6,2,2,2, 6,6,6,2,2,2,2,2, 2,2,2,2,2,2,2,39,
+ 2,2,2,2,2,2,2,2, 2,2,36,2,2,2,2,2, 2,2,36,2,2,2,2,2, 2,5,2,2,2,11,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,5,2,2,
+ 11,39,2,2,2,6,39,11, 6,11,11,19,6,39,2,11, 2,11,6,11,11,39,6,2, 2,2,2,2,11,11,5,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 6,6,37,6,6,4,39,6, 6,5,6,39,6,37,37,37,
+ // 99xx
+ 56,39,39,37,39,37,39,39, 39,2,2,39,39,2,39,39, 37,39,39,37,39,39,39,39, 37,37,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,2,2,2,11,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,6,2,2,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,37,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,5,2,2,2,2,2, 2,2,2,2,2,2,2,5,
+ 6,6,2,2,2,2,2,11, 6,6,6,2,6,6,2,2, 2,2,6,6,6,6,6,2, 2,2,19,2,11,6,11,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 6,36,6,6,6,6,6,6, 6,6,6,6,6,39,4,37,
+ // 9Axx
+ 56,39,39,39,39,39,39,39, 39,2,2,39,39,2,37,39, 39,39,37,37,39,39,39,39, 39,37,39,37,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,12,2,2,2,12,2,2, 2,12,2,12,12,2,12,12, 12,6,2,2,12,12,12,2, 2,2,2,2,2,2,2,37,
+ 2,2,2,5,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,2,2,2,2,2, 2,2,12,2,2,12,6,36,
+ 6,6,2,2,2,5,2,2, 2,2,2,2,6,6,2,5, 2,2,5,5,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 5,6,59,2,2,2,6,12, 6,2,6,2,4,6,2,2, 2,2,6,6,37,39,6,2, 37,2,6,6,6,19,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,5,2,12,6,2, 36,6,12,10,12,6,6,6, 6,12,6,6,12,12,39,37,
+ // 9Bxx
+ 56,39,39,39,39,39,37,39, 39,2,2,39,39,2,39,39, 39,39,39,37,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,6,2,2,2,2,2,2, 2,2,39,2,2,2,2,2, 5,2,6,2,2,2,2,2,
+ 2,2,2,2,2,2,2,6, 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,6,2,2,2,37,
+ 2,2,36,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,6,2,36,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,5,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,2,2,6,2,6,2, 4,6,6,2,2,12,2,2, 2,2,6,6,6,39,2,2, 2,2,6,6,6,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,2, 6,6,6,6,6,6,6,11, 4,6,6,6,11,11,22,37,
+ // 9Cxx
+ 56,39,39,11,37,37,39,39, 39,2,2,39,39,2,39,39, 37,37,39,37,39,37,39,39, 39,39,39,39,39,39,39,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,6,2,2,2,
+ 2,2,2,12,2,2,2,2, 2,2,2,2,12,2,2,2, 2,2,12,2,2,11,2,12, 2,2,2,2,2,2,2,37,
+ 2,2,2,2,2,36,2,2, 2,2,2,2,2,2,2,2, 2,36,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,4,2,2,2,2,6,2, 6,6,19,2,6,4,2,2, 2,2,2,2,6,2,2,2, 2,2,2,6,6,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,12,6,10,37,4,6,6, 6,6,6,6,11,4,19,37,
+ // 9Dxx
+ 56,39,12,4,37,39,37,39, 37,2,2,39,39,2,39,39, 37,39,37,39,37,37,39,37, 37,37,37,39,37,37,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,2,2,2,2,2,2,2,
+ 2,12,2,2,2,2,6,2, 2,2,2,2,2,2,2,12, 2,2,2,2,12,12,2,2, 2,2,2,2,6,2,2,37,
+ 2,2,2,2,2,5,2,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,5, 2,2,2,36,2,2,2,2, 2,2,5,2,2,2,2,2, 2,2,6,2,2,2,2,2,
+ 5,6,2,2,2,2,11,11, 11,6,11,2,11,11,11,11, 2,11,11,11,11,11,4,6, 2,2,11,6,6,11,11,11,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,19,2, 6,19,6,6,6,6,6,19, 6,6,12,10,4,19,11,37,
+ // 9Exx
+ 56,39,39,39,37,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,37,37,
+ 12,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,12,12,2,12,12,2,2, 2,12,2,12,2,2,12,12, 2,2,2,12,2,12,2,2, 2,2,2,2,2,2,2,37,
+ 2,2,2,2,2,2,6,2, 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,5,6,2,2,2,2,2, 2,6,2,2,2,6,2,2,
+ 6,12,59,2,6,2,2,2, 6,6,6,6,2,5,2,6, 2,2,6,11,6,6,6,2, 11,6,6,2,6,19,37,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,12,5,2, 12,4,12,10,6,19,39,6, 6,12,12,6,6,12,11,37,
+ // 9Fxx
+ 56,39,39,39,4,39,39,39, 39,2,2,39,39,2,37,39, 39,39,39,37,39,37,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,6, 2,2,2,2,6,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,33,2,31,12,33,2,31, 31,2,2,33,33,2,2,2, 33,6,31,2,33,33,2,12, 2,2,2,6,2,2,2,37,
+ 2,2,36,2,2,36,6,36, 2,2,36,2,36,36,2,2, 2,2,2,2,2,6,36,2, 2,36,2,2,2,2,36,2,
+ 2,33,2,2,2,2,2,2, 6,2,36,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,4,2,2,6,2,6,6, 6,6,6,2,6,6,2,2, 2,2,1,6,6,6,4,2, 2,2,6,6,4,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,12,6,6,19,6,6,6, 6,6,6,6,6,19,4,37,
+ // A0xx
+ 7,11,37,11,37,1,37,1, 1,11,11,37,1,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,2,2,11,11,11,2,2, 11,2,2,11,2,11,2,2, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,2,
+ 2,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,2,2,2,
+ 2,11,11,11,11,11,11,11, 11,11,11,11,33,11,33,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,2,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,11,2,2,12,2,
+ 11,2,36,36,2,2,36,36, 2,36,2,36,36,2,36,2, 2,2,2,5,2,2,2,2, 2,2,2,2,2,2,2,12,
+ 10,10,2,2,10,11,10,11, 11,1,10,14,10,10,10,10, 2,11,11,11,11,11,1,2, 2,2,11,11,1,10,11,11,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,36,2, 10,10,10,10,10,10,10,10, 12,17,1,10,11,33,1,10,
+ // A1xx
+ 7,37,37,1,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 11,2,2,2,2,2,11,2, 2,2,2,2,2,2,14,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2,
+ 8,8,8,8,9,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,9,9,8,8,8, 8,1,9,2,8,8,8,2,
+ 8,8,33,2,33,8,8,1, 33,8,8,33,33,8,8,2, 2,8,8,33,2,8,8,8, 8,8,8,39,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,39,2,2,2,2, 2,2,2,2,2,2,6,2, 2,2,2,2,2,2,2,33,
+ 2,3,3,3,3,7,7,7, 2,7,7,3,2,3,3,3, 3,3,7,8,8,2,3,3, 8,7,3,3,7,7,3,3,
+ 3,7,7,7,7,14,9,14, 7,7,7,7,7,14,7,7, 8,14,14,14,14,14,7,7, 7,7,7,7,7,7,5,7,
+ 2,5,5,2,2,2,2,2, 2,2,2,7,2,2,3,3, 3,3,3,7,3,7,7,7, 3,7,7,7,7,7,7,37,
+ // A2xx
+ 7,37,37,1,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,2,2,8,2,2,8, 8,8,8,2,2,2,2,2, 2,2,2,2,2,2,2,2, 8,2,2,2,2,2,2,2,
+ 2,2,8,8,33,8,8,2, 8,8,2,8,2,2,2,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,6,2,2,2,2,2, 2,6,2,2,2,2,2,6, 6,2,2,2,6,6,2,2, 2,2,2,2,2,2,2,2,
+ 36,7,7,7,7,7,7,7, 7,7,7,2,7,7,36,8, 2,8,8,8,8,8,8,14, 5,5,5,14,2,5,5,5,
+ 5,5,5,5,5,14,5,14, 5,5,14,23,5,14,5,5, 8,5,14,8,8,14,14,8, 14,3,3,3,3,5,8,5,
+ 2,2,2,2,2,2,2,2, 2,2,8,36,8,8,8,2, 8,3,3,3,8,8,7,8, 8,3,8,8,8,8,8,37,
+ // A3xx
+ 7,37,37,39,37,37,37,37, 37,2,2,37,4,2,9,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,11,11,2,2, 2,2,2,2,2,2,2,2, 11,11,11,11,11,11,11,11, 11,11,2,2,2,2,2,2,
+ 36,12,12,12,12,12,2,12, 2,2,2,12,2,12,12,12, 12,2,2,12,12,12,11,12, 8,12,12,2,8,2,11,2,
+ 8,12,2,2,33,12,2,8, 11,2,8,12,2,2,2,2, 2,11,2,8,2,12,8,2, 8,2,8,8,8,8,8,37,
+ 2,2,2,6,2,2,2,2, 3,2,2,2,2,2,2,2, 2,2,2,2,3,2,5,2, 2,2,2,2,2,2,2,2,
+ 3,3,3,3,3,3,3,3, 3,3,3,2,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,2,2,2,3,
+ 10,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,5,3,3,3,
+ 2,2,2,2,2,2,2,2, 2,2,2,3,3,3,3,2, 3,3,3,12,3,3,3,3, 3,3,3,3,3,3,3,37,
+ // A4xx
+ 7,37,37,11,1,37,11,39, 11,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,7,37,37,37,37,37,
+ 2,2,2,11,2,11,2,2, 2,2,11,2,2,2,2,2, 37,2,11,2,2,2,2,17, 2,17,2,2,2,2,2,2,
+ 8,8,8,8,8,8,8,8, 8,2,8,8,8,8,2,8, 2,8,8,8,8,8,8,8, 8,8,8,8,8,8,2,8,
+ 8,8,8,11,8,11,8,8, 8,8,8,8,8,2,2,8, 8,8,8,8,8,8,8,8, 8,8,8,2,2,8,8,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,6,2,2,2,2, 2,2,2,6,6,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,3,3,8,3,36,3,2, 3,2,3,3,3,3,3,3, 3,3,3,3,3,3,8,3, 3,3,8,3,3,3,2,3,
+ 3,3,14,3,3,14,3,3, 3,3,3,3,3,3,3,3, 3,8,3,3,14,3,8,3, 3,3,8,3,3,3,3,3,
+ 3,3,3,2,3,8,2,2, 3,3,3,3,3,3,8,3, 8,8,3,3,8,8,8,8, 11,8,8,8,8,5,8,1,
+ // A5xx
+ 7,6,37,39,37,37,6,37, 1,2,2,37,1,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,11, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,9,8,12,8,2,8,8, 8,8,8,8,8,2,8,8, 8,2,8,8,12,8,8,2, 8,8,8,8,8,8,8,8,
+ 8,8,8,2,8,8,8,2, 8,8,8,8,8,2,2,2, 2,8,2,8,2,8,8,2, 8,2,8,8,8,8,8,37,
+ 2,2,2,6,2,2,2,2, 2,2,2,2,2,2,6,2, 2,2,6,6,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,3,3,3,6,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,8,2,2,3,3,
+ 3,3,2,3,3,8,3,3, 3,3,3,3,3,8,8,3, 3,3,3,3,3,3,3,3, 8,3,3,3,3,3,3,3,
+ 3,3,3,2,2,2,2,2, 2,3,3,3,3,3,8,3, 8,8,8,3,8,8,3,8, 8,8,8,8,8,8,8,37,
+ // A6xx
+ 7,37,37,39,37,37,37,1, 12,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,6,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,8,9,8,33,2,2, 8,33,8,9,9,9,9,8, 8,8,9,8,8,8,8,8, 8,8,8,2,2,8,8,8,
+ 8,8,8,9,2,2,2,2, 8,8,8,8,9,9,8,8, 8,2,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,8,5,8,2,36,5,2, 8,8,2,8,2,8,2,2, 2,8,2,8,2,8,8,8, 8,8,8,2,2,2,2,8,
+ 8,3,3,8,8,3,9,8, 8,14,8,8,8,3,2,8, 8,8,8,8,3,8,33,8, 3,8,8,8,8,8,8,8,
+ 2,2,8,2,2,2,8,2, 2,2,8,8,8,8,8,2, 3,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // A7xx
+ 7,1,1,37,37,37,37,37, 37,14,2,14,37,14,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,11,2,2,14, 2,2,11,2,2,2,2,2, 11,11,11,11,11,11,11,11, 11,11,2,2,2,2,11,2,
+ 8,8,8,8,8,8,2,8, 8,8,8,8,8,8,8,8, 8,8,8,2,8,8,8,2, 8,8,8,11,2,11,8,8,
+ 8,2,8,8,8,8,8,8, 8,8,8,8,8,2,2,8, 8,8,8,8,8,2,8,8, 8,2,2,8,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,5, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,14,14,2,14,3,2,11, 8,8,14,2,2,36,8,2, 2,2,8,5,14,14,14,14, 2,8,8,2,14,8,14,2,
+ 8,14,14,14,14,14,8,8, 14,8,14,14,8,14,8,8, 8,8,14,8,14,3,8,14, 2,2,8,2,8,8,8,8,
+ 2,14,8,2,2,2,2,2, 2,2,2,8,8,8,3,2, 8,3,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // A8xx
+ 7,11,1,39,1,37,37,37, 4,2,2,14,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,11,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,8,8,8,2,8,2, 8,8,2,8,8,8,8,3, 2,2,8,8,8,8,2,2, 2,2,8,2,2,2,8,2,
+ 11,8,8,8,8,8,8,8, 2,2,8,8,8,8,8,8, 8,8,3,8,8,8,8,3, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,5,2,2,2,2,2,3, 3,3,3,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,7,8,8,8,7,7, 14,8,2,8,7,8,8,8, 11,2,7,2,2,8,2,14, 2,2,8,2,2,8,8,2,
+ 8,14,2,8,8,8,8,8, 8,8,8,14,8,14,8,8, 14,8,14,8,14,5,8,8, 14,2,8,5,5,6,5,8,
+ 2,2,8,2,2,2,2,2, 2,2,2,2,8,2,8,2, 5,5,8,8,8,8,5,8, 8,5,8,8,8,8,8,37,
+ // A9xx
+ 7,1,37,39,37,37,39,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,2,2,2,11,2,11,2, 2,2,2,11,2,2,2,2, 37,11,11,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,9,8,8,9,9,11,8, 8,8,8,9,9,8,9,8, 8,8,8,8,8,8,9,8, 2,8,8,8,2,8,8,8,
+ 8,2,8,2,8,8,8,8, 8,8,8,9,8,8,2,3, 8,2,2,8,2,8,8,8, 8,8,8,8,8,8,8,39,
+ 2,3,6,3,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,6,2,2,3,2, 2,2,2,2,2,2,2,2,
+ 2,8,2,8,3,3,3,3, 8,2,3,8,2,3,3,8, 2,8,8,8,2,8,2,2, 3,8,2,2,8,3,8,2,
+ 3,3,2,2,3,14,8,8, 8,8,8,2,3,9,8,3, 8,14,8,8,8,14,8,2, 2,2,8,8,8,8,8,8,
+ 2,2,8,2,2,2,2,2, 2,2,2,2,2,2,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // AAxx
+ 7,37,37,37,37,1,37,37, 4,2,2,1,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,8,8,8,8,8,8, 2,12,2,8,8,8,2,8, 8,8,8,2,12,2,2,2, 2,8,8,8,2,2,8,8,
+ 8,8,2,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,2,2,8,8,8, 8,8,8,8,2,2,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,5,2,2,2,2,2,2,
+ 36,14,8,8,2,3,8,8, 2,8,2,8,8,2,36,8, 2,8,2,5,8,5,2,5, 14,14,8,2,8,8,8,2,
+ 8,14,14,14,10,8,5,14, 8,1,5,5,8,14,8,8, 14,8,14,8,14,14,8,14, 14,2,8,2,8,8,8,5,
+ 2,8,2,2,2,2,2,2, 2,2,8,5,8,8,8,2, 8,8,8,5,8,10,8,8, 8,8,8,8,8,8,8,37,
+ // ABxx
+ 7,1,1,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,6,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,11,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,11,8,8,8,11,8,8, 8,8,8,8,8,11,8,8, 8,8,2,11,8,8,11,8, 8,8,2,2,8,2,8,8,
+ 8,8,8,11,8,8,11,8, 8,8,8,8,8,2,8,8, 8,8,8,8,8,2,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,5,2, 2,2,2,2,2,2,39,2, 5,2,2,2,2,11,2,2, 2,2,5,2,2,5,2,2,
+ 2,8,8,8,5,2,2,8, 36,2,2,8,8,2,36,2, 8,8,3,8,8,8,2,5, 2,2,8,2,8,8,2,2,
+ 8,8,8,8,8,8,5,8, 8,8,8,8,8,14,8,11, 14,8,8,8,14,14,14,8, 8,14,11,8,8,8,11,8,
+ 2,8,2,2,2,2,2,2, 2,2,2,2,36,8,36,2, 20,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // ACxx
+ 7,37,37,39,37,39,39,37, 37,2,2,37,39,2,37,39, 37,37,37,37,37,39,37,37, 39,37,37,39,37,37,37,37,
+ 2,2,2,11,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,8,9,8,33,8,8, 2,8,8,8,9,8,9,8, 8,8,9,2,2,8,8,9, 8,8,8,8,8,2,2,8,
+ 8,8,2,8,8,33,8,11, 8,2,2,2,2,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,2,8,8,8,37,
+ 2,6,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,5,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,8,2,8,2,36,2,2, 36,6,8,36,2,36,36,8, 8,2,2,8,2,8,8,3, 2,8,2,2,2,8,2,3,
+ 8,8,59,8,8,59,8,3, 8,8,3,59,8,8,3,2, 8,5,8,8,8,8,8,8, 2,8,5,8,8,8,8,8,
+ 2,2,59,2,2,2,2,2, 2,2,2,2,2,2,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // ADxx
+ 7,39,37,37,39,39,37,39, 39,2,2,37,39,2,37,39, 39,37,39,39,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,2,8,8,2,8,8, 2,8,8,2,8,8,2,2, 8,2,2,8,8,8,8,8, 8,8,8,8,2,8,8,2,
+ 8,2,8,2,11,8,8,11, 8,2,11,11,8,8,8,11, 8,8,11,8,2,8,11,8, 8,2,8,8,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,39,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2,
+ 36,2,2,2,2,2,2,8, 36,2,14,8,2,8,36,2, 2,8,8,2,2,8,8,8, 2,8,8,8,8,2,2,8,
+ 8,14,8,14,8,8,8,8, 8,8,8,8,8,8,2,2, 7,8,14,8,8,14,7,8, 2,8,8,8,8,8,8,8,
+ 2,8,2,2,2,2,2,2, 2,2,2,2,8,2,36,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // AExx
+ 7,39,1,39,39,39,37,37, 37,2,2,37,39,2,37,39, 39,39,39,39,39,39,37,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,5,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,9,8,2,2,9,2,2, 8,9,8,9,8,8,9,9, 2,8,2,8,8,2,8,2, 2,2,8,2,2,8,2,8,
+ 8,8,8,8,8,8,2,8, 8,8,2,8,8,8,2,8, 8,8,8,2,8,8,8,8, 8,8,8,8,8,8,11,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,36,8,8, 2,2,2,2,36,2,2,2, 2,2,2,2,2,2,2,8, 2,2,2,8,8,2,2,2,
+ 8,8,8,14,8,6,8,8, 8,8,8,14,8,9,2,2, 8,8,8,8,8,8,8,8, 8,2,8,8,8,6,8,8,
+ 2,36,36,2,2,2,2,2, 2,2,2,2,2,8,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // AFxx
+ 7,9,1,1,39,39,39,37, 37,2,2,37,39,2,37,37, 37,37,37,37,37,39,37,37, 39,39,37,37,39,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,11,2,2,2,2,2,2, 39,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,12,8,2,12,12,8,2, 2,8,2,12,12,8,12,12, 8,8,8,8,8,2,2,2, 2,12,8,8,2,8,2,2,
+ 11,2,8,2,8,8,8,8, 2,2,2,8,8,2,2,12, 2,8,2,2,2,8,8,8, 8,12,8,8,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,9,2,6,2,2,2,2, 2,2,8,8,8,2,36,2, 11,2,8,8,2,8,2,2, 8,2,8,2,2,8,2,2,
+ 8,8,8,8,6,8,8,8, 8,8,8,6,6,10,8,8, 8,8,8,8,14,10,1,8, 2,2,2,2,8,8,8,8,
+ 8,2,2,2,2,2,2,2, 2,2,2,8,2,2,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // B0xx
+ 7,39,37,37,39,4,37,37, 39,11,2,37,37,2,37,37, 39,39,37,37,39,39,37,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,11,2,2,2, 2,2,11,2,2,2,2,2, 11,11,11,11,11,11,11,11, 11,11,2,2,2,2,2,2,
+ 8,8,2,11,8,11,11,2, 2,8,8,8,2,2,2,8, 2,8,2,2,8,8,8,2, 2,2,8,2,8,8,8,8,
+ 11,8,2,2,8,8,8,8, 8,2,8,8,8,11,2,11, 8,8,2,8,8,8,8,8, 8,8,2,8,8,8,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 11,5,5,5,2,5,7,2, 5,5,8,8,3,5,3,2, 5,2,3,5,3,8,5,8, 3,5,3,8,8,2,2,2,
+ 6,3,7,2,3,5,7,5, 8,3,8,5,7,5,3,8, 2,3,14,8,5,7,3,3, 3,3,5,7,5,5,7,5,
+ 2,5,8,2,2,2,5,2, 5,2,8,2,3,5,5,3, 5,5,8,8,3,8,8,5, 5,3,5,7,5,5,3,37,
+ // B1xx
+ 7,11,11,39,39,37,39,37, 37,2,2,37,37,9,39,39, 37,39,37,37,39,39,37,37, 37,39,37,37,39,39,37,37,
+ 9,9,2,2,37,2,2,2, 2,2,2,5,9,2,9,2, 2,2,2,2,2,2,2,2, 2,2,9,9,2,2,2,9,
+ 8,2,8,8,2,8,8,8, 8,8,8,8,8,8,8,8, 2,2,11,2,8,8,8,8, 8,8,8,8,2,2,8,8,
+ 8,8,8,9,9,8,8,9, 8,8,8,9,8,8,2,8, 9,8,8,9,9,2,8,8, 8,8,8,8,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 2,2,2,2,2,2,2,2,
+ 2,8,7,3,5,3,3,3, 3,3,2,7,3,3,3,3, 2,3,3,5,3,8,3,8, 5,5,5,3,5,2,3,7,
+ 8,5,8,5,7,5,8,5, 3,7,3,3,3,5,3,3, 8,7,3,8,5,3,5,5, 3,5,8,5,3,5,5,7,
+ 3,8,5,2,3,2,2,2, 5,2,3,2,2,3,5,2, 3,8,5,7,5,3,3,7, 8,3,5,7,7,8,7,37,
+ // B2xx
+ 7,39,37,39,37,37,39,1, 37,2,2,39,37,2,39,39, 39,39,37,37,39,39,37,39, 39,39,37,37,39,39,37,39,
+ 2,2,2,2,2,39,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,8,2,8,8,2,8, 8,2,8,8,8,8,8,8, 8,8,8,2,2,8,8,8, 8,8,8,8,8,8,8,8,
+ 8,8,8,2,8,2,2,8, 2,8,8,8,8,8,8,8, 8,8,8,2,3,2,8,8, 8,8,8,8,2,39,8,37,
+ 2,2,2,2,2,2,5,5, 2,2,5,2,2,2,5,5, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,3,8,8,3,7,8, 3,3,3,7,7,7,3,7, 7,8,5,8,2,3,3,2, 2,3,3,3,3,3,2,3,
+ 5,7,3,7,8,3,3,7, 7,3,3,3,7,3,3,7, 7,10,3,8,7,8,7,8, 3,3,7,7,3,3,5,3,
+ 2,3,3,2,2,2,2,2, 7,3,7,5,3,10,3,2, 7,7,7,7,5,7,5,7, 8,7,3,5,8,3,7,37,
+ // B3xx
+ 7,39,1,39,39,37,37,37, 39,12,2,37,39,2,37,39, 39,39,37,37,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 12,12,2,2,11,39,2,2, 2,2,2,2,12,2,12,12, 37,2,2,2,2,2,2,2, 2,2,12,12,2,2,2,12,
+ 8,2,8,8,8,2,2,2, 2,2,8,2,2,2,8,12, 2,8,2,8,8,8,8,8, 8,8,8,2,8,8,8,2,
+ 2,12,12,12,8,12,8,8, 8,2,8,12,2,12,8,8, 8,8,8,8,8,12,8,8, 8,12,8,8,2,8,8,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 5,2,2,2,2,2,2,2,
+ 2,3,2,3,3,3,7,3, 2,3,5,7,3,5,8,5, 7,9,5,5,2,3,2,3, 2,8,8,5,2,8,3,3,
+ 7,3,3,5,5,3,3,3, 7,3,5,3,3,7,7,3, 3,5,8,7,3,8,3,5, 7,3,7,7,8,5,3,8,
+ 5,8,5,2,2,2,2,2, 2,2,3,5,5,5,5,2, 5,10,8,5,7,8,3,3, 8,8,5,3,8,3,3,10,
+ // B4xx
+ 7,39,1,39,39,37,37,4, 39,14,2,14,39,2,37,39, 37,39,37,37,39,39,37,37, 37,39,37,39,37,39,39,37,
+ 2,2,2,11,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,11,8,8,8,11,2,2, 8,8,8,8,8,8,8,8, 8,8,11,11,8,8,8,2, 8,8,8,8,2,2,8,8,
+ 8,11,8,8,8,11,8,8, 11,11,8,8,2,11,11,8, 11,8,8,11,11,11,11,8, 8,8,8,2,2,8,2,37,
+ 2,2,2,6,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,2,8,8,3,3,14, 3,5,2,3,3,2,3,2, 2,7,3,7,3,8,7,14, 2,2,5,2,8,3,2,3,
+ 5,8,5,5,7,3,7,3, 3,5,3,3,3,14,3,5, 5,7,14,3,5,14,7,5, 7,5,8,7,5,5,5,3,
+ 2,14,2,2,5,2,2,5, 2,2,2,5,2,2,3,3, 3,5,3,3,7,5,5,3, 3,7,3,3,7,3,8,37,
+ // B5xx
+ 7,39,37,39,39,37,37,37, 39,2,2,37,39,2,37,37, 37,39,39,37,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 2,2,2,2,39,2,2,2, 2,2,2,2,2,2,14,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,33,33,33,2,11,8, 8,8,8,33,8,8,33,8, 8,39,33,8,33,8,33,8, 8,8,33,8,8,2,8,2,
+ 8,9,8,8,8,8,8,11, 8,8,8,8,8,11,8,8, 8,8,11,8,3,8,8,8, 8,8,8,8,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,5, 2,2,2,2,2,2,2,5,
+ 2,7,7,3,7,3,8,8, 5,3,8,3,7,7,3,3, 2,3,8,3,8,5,5,2, 5,8,3,7,3,3,3,5,
+ 3,3,3,3,3,5,3,5, 3,5,5,5,3,3,5,3, 3,14,7,14,14,7,3,3, 3,7,3,8,7,3,7,7,
+ 2,7,5,3,3,5,2,3, 2,5,3,2,2,2,5,2, 5,5,5,8,3,3,7,3, 8,8,3,8,8,8,7,37,
+ // B6xx
+ 7,1,4,39,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,39,37,37,37, 37,39,37,37,37,37,37,37,
+ 9,2,2,2,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,9,2,2,2,9,
+ 8,2,8,2,8,8,8,2, 8,2,8,8,8,2,2,8, 8,8,8,8,8,8,8,8, 8,3,8,2,2,8,8,2,
+ 8,30,8,9,8,8,2,8, 8,8,2,9,9,8,9,8, 9,8,9,8,8,8,8,9, 8,2,8,8,8,8,8,37,
+ 2,2,2,5,2,2,5,5, 2,2,2,2,2,5,2,2, 2,11,5,2,2,2,5,5, 2,2,2,2,2,2,2,2,
+ 2,8,8,2,8,3,7,5, 3,3,3,3,3,3,8,3, 8,5,2,2,3,7,7,3, 7,3,8,5,3,3,3,7,
+ 3,3,8,8,3,3,3,5, 3,7,7,3,3,3,3,3, 7,7,14,3,3,8,3,8, 7,3,7,7,8,7,6,5,
+ 3,7,7,2,2,2,2,2, 2,2,8,8,7,3,3,2, 8,7,5,5,5,5,5,5, 3,3,3,3,3,8,3,37,
+ // B7xx
+ 7,39,37,14,39,37,39,37, 39,11,11,37,39,2,37,39, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 11,2,2,11,11,2,11,2, 2,2,11,2,2,2,11,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 8,8,8,33,33,11,8,8, 8,2,8,33,8,8,8,8, 8,8,8,8,33,8,8,8, 8,8,33,2,8,2,2,2,
+ 11,11,8,2,8,11,8,11, 8,8,8,8,11,8,8,8, 2,8,8,8,2,8,8,8, 8,8,8,2,8,8,8,37,
+ 2,6,6,2,2,2,2,2, 2,6,2,2,2,2,2,2, 2,2,2,2,2,11,2,2, 2,2,2,5,2,2,2,2,
+ 2,5,3,5,7,2,5,8, 3,5,7,5,3,3,5,5, 5,5,2,5,3,3,3,11, 2,5,3,5,2,3,3,3,
+ 3,5,5,5,5,3,3,3, 7,3,7,8,3,14,5,5, 5,3,14,8,5,14,3,7, 14,14,7,3,3,3,7,3,
+ 2,5,3,2,2,2,2,3, 2,2,7,7,5,8,7,7, 3,3,3,7,3,7,3,7, 3,5,7,3,5,5,3,37,
+ // B8xx
+ 7,39,37,39,39,39,37,37, 37,2,2,37,37,2,37,1, 39,39,37,39,39,39,39,37, 39,37,37,37,39,39,39,37,
+ 2,2,2,2,37,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,2,8,2,6,2,8, 8,8,8,8,8,6,8,8, 2,2,8,8,8,8,8,8, 8,8,8,2,37,2,2,8,
+ 8,8,2,11,8,8,8,8, 8,8,8,8,2,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,7,2,5,2,5,5,5, 8,7,2,7,2,2,5,2, 2,2,5,5,3,5,5,5, 5,8,2,5,5,3,3,5,
+ 7,5,5,3,3,5,5,5, 8,3,3,8,8,7,3,3, 3,8,3,5,5,5,3,3, 14,3,3,3,8,7,5,3,
+ 2,7,3,2,2,2,3,5, 3,5,8,2,2,5,7,3, 5,5,7,5,8,8,3,3, 3,3,3,8,3,7,7,37,
+ // B9xx
+ 7,39,4,37,39,39,39,37, 39,14,2,14,39,14,4,37, 39,39,39,39,39,39,37,37, 39,39,37,39,39,39,39,37,
+ 14,2,2,2,2,2,2,14, 2,2,2,2,2,2,14,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2,
+ 8,8,8,8,8,8,8,8, 8,2,8,8,8,8,2,8, 2,8,2,8,8,2,8,2, 8,8,2,2,8,8,2,8,
+ 8,8,12,12,12,9,2,12, 8,9,8,9,9,8,9,9, 8,8,38,8,9,8,8,8, 8,8,12,8,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,14,8,2,3,7,3,14, 14,3,8,3,3,7,5,7, 5,3,3,2,2,14,2,14, 2,3,3,3,2,7,2,2,
+ 3,7,5,14,3,3,7,3, 3,3,3,3,5,7,5,8, 5,14,14,14,7,14,7,5, 3,5,5,3,3,5,5,5,
+ 7,7,3,3,2,2,5,7, 5,5,8,2,2,3,8,8, 7,7,3,3,8,7,5,5, 5,3,3,3,5,3,7,37,
+ // BAxx
+ 7,39,37,39,39,39,37,37, 39,2,2,37,39,2,37,39, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 1,2,2,2,2,1,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,14,2,2,2,2,2,
+ 8,8,2,8,8,2,2,8, 2,8,8,8,8,8,8,8, 8,2,8,2,2,8,8,1, 8,2,8,8,2,8,8,1,
+ 2,8,8,8,2,12,2,2, 8,12,8,8,2,2,8,12, 8,8,2,2,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,8,3,3,2,5,2,2, 14,8,2,2,2,2,5,5, 5,3,2,2,5,3,2,2, 5,5,2,5,3,3,5,2,
+ 5,3,7,3,3,3,7,7, 3,3,8,3,7,3,5,3, 5,7,5,3,14,3,8,7, 8,7,3,3,3,7,8,7,
+ 7,3,8,2,2,2,2,2, 2,2,5,8,3,5,7,2, 3,5,3,3,8,3,7,7, 3,5,3,8,5,3,3,37,
+ // BBxx
+ 7,39,1,39,39,37,14,39, 39,2,2,1,39,11,37,11, 39,39,37,37,39,39,39,37, 39,39,37,37,39,39,37,37,
+ 11,2,2,2,2,11,11,11, 2,2,2,2,11,2,11,2, 37,2,2,11,11,2,11,11, 11,11,2,11,11,2,14,2,
+ 8,8,8,2,8,8,8,8, 2,8,2,11,8,2,2,8, 8,8,8,2,8,2,11,8, 8,8,8,8,8,8,2,8,
+ 8,8,2,2,11,8,11,2, 8,2,11,8,2,2,8,2, 8,8,8,8,2,8,37,8, 8,8,8,8,8,8,8,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,8,3,2,3,2,3,3, 3,5,3,8,2,3,3,3, 3,7,7,7,2,3,2,3, 2,3,2,3,3,2,2,2,
+ 7,14,7,14,7,7,3,5, 7,7,3,7,5,7,7,7, 7,14,7,3,7,3,3,7, 3,7,8,3,7,3,7,7,
+ 7,3,8,2,2,2,2,5, 2,2,5,7,5,2,3,7, 3,3,3,5,8,5,5,3, 3,3,3,7,3,5,7,37,
+ // BCxx
+ 7,39,39,39,39,39,37,37, 39,2,2,37,39,2,37,37, 39,39,39,39,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,12,8,8,2,2,8,12, 8,2,8,12,8,12,12,12, 8,8,2,12,2,12,12,8, 8,8,8,8,2,2,8,2,
+ 2,8,8,9,9,8,8,8, 8,8,2,9,9,9,9,8, 8,8,9,8,8,8,8,8, 8,8,8,8,2,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,7,2,7,3,5,3,2, 7,2,3,7,2,5,5,3, 3,5,3,2,3,2,3,2, 3,2,5,2,5,8,5,5,
+ 3,7,7,3,3,5,3,3, 3,8,3,8,7,3,3,8, 8,3,3,3,7,5,8,3, 3,3,8,3,3,3,3,8,
+ 3,3,5,2,3,2,2,2, 7,2,7,2,3,2,5,7, 7,7,3,7,3,3,5,5, 5,3,5,3,3,7,3,37,
+ // BDxx
+ 7,39,37,11,37,37,37,37, 39,2,2,37,39,2,6,37, 37,39,37,37,39,39,37,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,2,2,2,6,8,8, 8,2,2,8,8,8,2,2, 2,8,8,8,8,8,8,8, 8,8,8,2,8,8,2,8,
+ 8,8,8,8,8,2,8,8, 2,2,2,8,8,8,8,8, 2,8,2,8,8,8,8,8, 2,3,2,8,8,8,8,39,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,2,3,7,2,2,8, 3,2,7,3,2,3,2,2, 2,2,3,2,5,3,3,3, 7,3,5,3,5,2,7,2,
+ 5,5,5,5,5,5,8,5, 5,5,5,7,3,5,5,3, 7,7,8,3,3,8,3,8, 3,3,3,3,3,3,8,8,
+ 3,3,3,2,2,2,2,3, 2,3,7,2,3,2,3,2, 3,7,3,3,3,3,3,7, 3,7,3,3,3,7,3,37,
+ // BExx
+ 7,39,37,14,39,1,37,37, 39,2,2,37,37,2,37,39, 37,39,39,37,37,39,37,37, 39,39,37,39,39,37,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2,
+ 8,8,8,2,8,8,8,8, 8,2,8,8,8,2,2,2, 8,8,8,59,8,8,2,8, 8,8,8,2,8,2,8,8,
+ 8,12,9,8,8,9,2,12, 8,8,8,12,8,12,8,12, 8,8,8,12,12,12,8,8, 8,8,8,8,8,2,8,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,3,3,2,8,7,3, 8,3,3,3,3,3,7,3, 3,3,5,3,5,3,3,2, 5,8,3,3,2,2,5,3,
+ 3,14,8,3,14,3,5,8, 5,8,5,5,8,3,7,5, 7,14,5,3,8,5,3,5, 5,3,7,3,7,3,3,5,
+ 5,8,2,2,2,2,2,5, 2,2,2,7,7,3,5,5, 7,3,7,5,8,3,3,5, 5,3,5,5,3,3,7,37,
+ // BFxx
+ 7,11,37,37,37,12,37,1, 37,2,2,37,39,2,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,39,37,37,37,
+ 12,11,2,2,2,2,2,11, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,12,2,2,2,2,1,
+ 8,8,1,1,8,8,1,2, 1,1,2,8,1,1,8,8, 1,1,1,1,1,8,1,8, 8,1,8,2,8,2,8,2,
+ 8,12,12,8,12,12,8,2, 1,8,8,12,12,2,2,12, 2,8,1,12,2,2,8,12, 2,12,8,2,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,6,2,2,2,2, 2,2,2,5,2,2,2,2, 2,2,5,2,2,2,2,2,
+ 2,5,5,5,8,5,7,7, 3,5,3,2,5,5,7,3, 5,5,7,8,3,5,7,7, 3,5,2,2,3,2,5,7,
+ 5,5,5,5,8,3,3,7, 7,3,7,3,3,7,3,5, 5,14,3,3,5,3,3,3, 3,3,3,3,5,3,7,8,
+ 2,3,3,2,5,2,2,2, 2,8,3,5,5,3,5,5, 8,3,5,3,7,5,3,3, 5,5,8,8,5,8,5,37,
+ // C0xx
+ 7,39,37,1,39,39,37,1, 39,10,18,1,39,10,37,39, 39,39,37,37,39,39,39,37, 39,39,7,37,39,39,37,39,
+ 10,10,10,10,1,10,10,10, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10,
+ 8,8,1,1,1,8,1,1, 8,8,8,30,1,1,1,1, 1,1,8,1,1,3,8,8, 1,8,8,8,8,8,3,3,
+ 8,3,8,1,8,8,8,8, 39,3,8,3,3,8,8,39, 8,8,1,8,8,8,8,8, 8,8,8,8,8,37,8,37,
+ 10,6,6,6,6,39,3,3, 39,6,12,6,10,6,6,6, 6,6,6,6,6,6,6,6, 6,3,37,3,3,5,37,1,
+ 10,3,3,8,7,5,8,5, 3,3,7,3,3,3,7,5, 5,5,5,8,3,7,3,14, 5,7,5,5,3,5,7,3,
+ 3,10,10,10,10,10,10,5, 3,8,10,10,5,10,5,5, 10,10,10,5,5,10,5,3, 10,8,5,5,7,10,10,7,
+ 3,5,10,7,3,5,3,5, 7,7,10,3,7,3,3,3, 10,3,8,10,8,7,3,3, 3,8,5,5,5,3,7,10,
+ // C1xx
+ 7,39,1,39,39,39,37,37, 39,14,18,37,39,11,37,39, 39,39,37,39,39,39,37,39, 39,39,39,39,37,37,37,37,
+ 11,18,14,14,37,11,18,14, 14,11,11,14,18,18,14,11, 37,20,11,14,14,14,14,11, 37,14,11,18,11,10,18,18,
+ 8,8,1,1,1,1,1,1, 1,1,1,1,1,1,1,8, 1,1,1,1,1,1,1,8, 1,8,1,14,8,11,8,8,
+ 8,8,1,8,16,16,1,1, 1,10,8,1,1,16,8,8, 8,8,1,16,1,8,16,8, 8,8,1,8,8,8,8,37,
+ 39,6,6,4,39,11,3,6, 3,39,12,39,37,12,12,6, 37,6,6,6,39,3,6,6, 37,3,3,3,4,11,12,1,
+ 14,5,3,3,5,3,5,3, 5,5,3,3,3,3,5,5, 7,7,10,7,3,3,5,5, 5,5,3,5,3,3,5,3,
+ 10,5,5,14,3,10,3,7, 10,11,8,3,18,20,10,3, 3,14,14,10,18,14,5,14, 5,3,3,10,3,5,3,5,
+ 10,7,3,3,8,3,3,8, 14,3,7,10,3,3,5,5, 5,5,5,10,3,5,5,3, 5,3,5,10,7,5,5,10,
+ // C2xx
+ 7,39,37,37,39,39,39,37, 39,14,14,1,39,14,37,1, 39,37,39,39,39,39,39,37, 37,37,37,39,37,39,37,39,
+ 10,14,14,14,39,10,14,14, 14,14,14,10,14,10,10,14, 37,10,10,14,14,10,10,10, 39,14,14,10,14,10,14,14,
+ 8,8,1,16,16,8,8,1, 8,8,16,16,16,1,1,8, 16,8,16,16,1,1,16,8, 8,1,8,8,8,14,8,8,
+ 8,8,8,8,8,8,8,1, 1,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,1, 8,8,8,8,14,8,8,37,
+ 37,59,2,11,3,39,11,11, 2,39,10,2,2,59,11,59, 11,11,12,3,2,3,2,2, 11,11,3,12,4,59,17,11,
+ 2,2,7,2,2,3,7,2, 59,2,2,2,3,7,2,3, 2,2,8,7,2,2,3,3, 7,14,2,2,3,3,7,2,
+ 10,3,8,3,3,10,3,3, 10,3,3,10,3,10,10,18, 7,10,14,7,3,18,3,3, 7,14,10,3,8,3,3,7,
+ 8,11,10,11,3,7,7,7, 7,3,7,3,14,3,10,5, 3,10,3,3,3,10,3,5, 5,5,3,5,5,3,3,10,
+ // C3xx
+ 7,1,4,39,39,37,37,1, 37,14,12,3,39,14,1,1, 39,37,39,37,37,39,37,37, 39,39,37,37,39,37,37,37,
+ 14,14,14,1,1,1,14,14, 14,11,11,5,14,11,14,11, 37,14,10,14,14,10,14,14, 10,10,11,14,14,11,14,1,
+ 8,10,8,8,8,1,8,12, 8,12,8,8,12,8,12,1, 8,8,12,1,12,12,10,10, 8,8,8,8,8,8,8,39,
+ 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 1,8,1,8,8,8,8,8, 8,8,8,8,14,8,8,37,
+ 2,2,59,59,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,5,2,5, 2,2,2,5,2,2,2,3,
+ 3,14,7,14,10,3,7,3, 11,3,5,3,11,11,10,7, 14,11,14,11,11,11,5,3, 3,14,11,11,3,11,11,5,
+ 5,11,5,11,11,7,3,3, 3,14,7,5,14,11,10,3, 10,3,5,11,3,11,8,3, 5,3,7,3,3,8,8,10,
+ // C4xx
+ 7,6,1,39,37,37,37,37, 39,10,18,1,39,18,1,37, 37,37,37,37,39,37,37,37, 39,37,37,37,39,37,37,37,
+ 18,18,10,1,1,1,10,1, 6,10,6,3,18,10,18,10, 37,10,10,10,10,5,5,5, 5,5,18,18,10,6,10,1,
+ 8,8,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,8,1,1,1,1,1,8, 1,8,1,8,8,8,8,8,
+ 3,8,8,8,8,1,8,1, 1,8,8,1,8,8,1,8, 1,8,1,1,1,1,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,5,7,3,3,7,3,5, 5,3,3,5,3,5,7,2, 2,2,8,8,7,8,3,3, 3,7,7,3,2,3,3,3,
+ 10,5,10,3,5,10,10,3, 10,3,10,10,7,10,10,3, 3,3,8,10,3,18,3,18, 3,18,3,10,3,8,5,10,
+ 10,3,3,3,3,10,8,20, 10,20,3,10,7,5,10,3, 10,3,8,7,3,8,5,5, 7,7,3,5,5,8,3,10,
+ // C5xx
+ 7,39,37,14,39,39,37,37, 39,14,18,39,39,18,39,39, 39,39,37,37,39,39,37,39, 39,39,39,39,39,39,37,37,
+ 18,18,18,14,39,18,18,1, 14,18,14,3,18,18,18,14, 37,10,10,10,10,6,10,10, 39,20,18,18,18,1,18,18,
+ 8,8,1,3,1,1,8,1, 1,1,8,1,1,1,1,1, 1,8,1,1,1,8,1,8, 8,8,1,8,8,8,8,8,
+ 39,8,1,3,1,8,8,1, 1,8,12,1,1,1,8,8, 1,8,1,1,1,8,8,8, 8,8,12,3,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,37,5,2,2,2,12,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,3,2,3,14, 3,5,7,2,5,5,3,2, 5,2,5,2,7,3,3,7, 5,7,2,5,2,2,2,5,
+ 7,5,5,5,10,3,3,10, 11,10,10,10,10,5,18,11, 10,10,10,11,7,10,11,5, 5,7,11,5,8,3,11,7,
+ 3,11,3,8,5,8,5,7, 14,8,7,5,7,11,3,3, 5,10,7,3,3,5,7,5, 5,7,3,3,7,8,7,37,
+ // C6xx
+ 7,37,39,1,39,39,37,37, 37,10,12,37,39,11,37,37, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,37,39,
+ 11,12,10,1,37,39,11,1, 37,11,11,10,12,18,10,10, 37,3,11,10,3,3,3,3, 39,3,12,39,11,6,10,12,
+ 37,12,1,8,1,12,8,1, 1,12,8,1,1,1,1,8, 8,8,1,1,1,8,1,8, 8,8,8,8,8,8,8,8,
+ 8,8,1,1,1,8,8,8, 8,12,8,8,8,12,1,12, 8,8,8,8,1,12,8,12, 8,1,8,8,8,8,8,37,
+ 37,6,6,6,4,3,2,4, 6,39,6,39,6,6,6,2, 6,6,2,6,37,6,39,6, 39,2,4,2,39,3,37,4,
+ 2,2,3,7,3,3,7,7, 7,7,3,3,3,3,5,5, 2,3,5,7,3,3,3,3, 3,3,3,7,5,3,3,3,
+ 3,3,3,7,5,10,3,5, 10,7,10,5,3,10,10,7, 5,11,5,10,5,3,3,3, 3,3,3,5,11,5,3,3,
+ 7,11,7,11,3,10,3,3, 10,7,5,3,3,5,7,3, 3,7,5,3,3,3,11,5, 3,5,5,3,7,7,7,37,
+ // C7xx
+ 7,39,39,39,37,39,39,39, 39,11,11,14,39,11,37,39, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,14,11,11, 11,11,11,11,11,11,1,11,
+ 18,1,1,16,16,1,8,16, 1,1,16,16,1,1,16,1, 16,8,16,16,16,1,16,1, 37,1,16,11,39,11,11,11,
+ 39,1,1,31,16,1,8,1, 1,1,8,1,1,1,1,1, 8,8,1,1,37,1,1,3, 37,8,8,39,37,11,8,37,
+ 2,11,6,2,37,37,39,12, 37,6,39,3,39,2,2,39, 11,2,2,2,2,11,2,6, 11,11,2,2,2,2,11,2,
+ 11,11,3,3,7,5,3,3, 3,3,5,3,3,7,3,7, 5,3,5,5,14,7,3,3, 3,14,7,5,7,7,7,3,
+ 10,11,10,1,10,3,11,11, 11,11,11,11,11,11,11,5, 5,5,5,11,11,5,11,5, 5,5,11,11,7,11,11,11,
+ 5,11,5,11,11,11,11,3, 5,3,3,3,3,11,7,3, 11,7,3,3,5,5,5,3, 3,5,3,5,5,5,3,10,
+ // C8xx
+ 7,37,37,14,37,37,39,39, 37,11,11,37,39,11,37,39, 37,37,37,37,39,37,37,39, 39,39,37,39,37,37,37,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,18,11, 18,11,11,18,11,11,10,18,
+ 11,12,1,1,1,12,1,1, 12,12,12,12,12,1,12,12, 12,1,1,1,1,12,1,12, 1,1,1,11,11,11,11,11,
+ 11,12,1,1,1,12,3,1, 12,12,12,12,12,1,1,12, 10,37,12,12,12,12,12,11, 39,12,12,11,11,11,3,37,
+ 39,11,6,6,3,37,2,3, 39,6,12,37,39,11,10,6, 11,6,6,6,11,11,6,6, 11,2,2,2,2,11,11,1,
+ 11,3,7,5,3,3,3,3, 3,7,3,3,7,5,5,5, 5,3,5,7,3,3,3,3, 5,5,3,3,3,7,3,5,
+ 10,10,10,3,5,10,5,11, 3,11,10,3,10,11,3,3, 10,11,10,11,11,3,7,10, 11,3,11,11,11,3,11,10,
+ 7,11,3,11,11,11,11,3, 7,3,5,3,10,11,10,5, 10,5,7,7,3,3,11,5, 3,5,11,5,3,3,7,37,
+ // C9xx
+ 7,39,11,39,37,39,37,10, 39,11,18,39,37,11,37,39, 39,37,39,37,39,39,37,37, 39,39,37,39,39,39,37,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,11,11, 11,11,11,18,11,11,11,18,
+ 8,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,11,11,11,11,11,
+ 18,8,1,1,1,8,1,1, 1,1,1,1,1,1,1,8, 1,1,1,1,1,8,1,3, 1,1,1,1,11,11,11,37,
+ 39,6,6,6,6,11,37,12, 6,6,12,6,6,6,12,6, 2,2,2,6,2,11,6,2, 6,2,2,2,2,11,11,2,
+ 11,11,3,3,7,3,7,7, 3,7,3,3,7,3,7,3, 3,3,7,3,7,3,3,3, 7,3,3,11,7,3,7,11,
+ 14,3,7,7,10,18,3,3, 18,18,7,3,3,10,18,3, 3,10,10,18,7,7,18,7, 7,3,18,7,3,7,18,7,
+ 3,7,7,3,3,7,3,3, 3,7,3,7,3,3,3,20, 7,3,3,3,7,3,3,7, 3,3,3,7,7,3,7,37,
+ // CAxx
+ 7,6,39,37,39,37,37,37, 37,11,11,37,37,11,37,37, 37,37,37,37,39,37,37,37, 39,37,37,37,39,37,37,4,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,11,11, 14,11,11,18,11,11,10,18,
+ 11,10,12,12,12,1,1,12, 1,8,1,12,1,1,1,10, 12,1,1,1,1,1,1,12, 1,37,12,11,8,11,8,11,
+ 37,10,1,8,12,8,37,37, 1,1,12,12,12,12,1,8, 10,1,1,12,1,1,39,1, 1,8,12,1,11,11,11,37,
+ 2,2,6,2,2,11,2,2, 2,2,2,2,12,2,39,6, 11,39,2,6,2,11,6,6, 11,11,37,2,10,11,2,2,
+ 11,3,3,7,3,3,3,3, 3,3,7,3,7,3,3,3, 3,3,3,3,14,3,3,3, 7,3,14,3,3,3,3,3,
+ 10,14,3,11,7,10,3,3, 11,7,3,10,7,11,10,11, 3,11,3,11,3,3,3,3, 11,3,11,3,3,11,11,11,
+ 10,11,7,11,3,11,11,7, 14,3,3,10,3,11,10,3, 10,7,10,7,3,3,3,3, 3,20,7,3,7,3,7,11,
+ // CBxx
+ 7,1,39,37,37,39,37,37, 1,11,18,37,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,7,37,37,37,37,37,
+ 11,18,11,10,1,39,18,11, 11,11,11,3,18,18,10,11, 37,10,10,3,10,3,10,10, 3,11,11,18,11,3,11,18,
+ 8,10,1,1,16,1,1,16, 1,1,16,16,1,16,1,10, 16,1,1,16,16,1,16,1, 1,39,1,37,11,10,11,11,
+ 37,1,1,1,1,37,1,1, 1,10,1,1,1,8,1,1, 1,1,1,1,1,37,37,37, 8,39,39,3,8,37,3,37,
+ 2,6,6,4,39,2,2,2, 2,2,2,2,2,2,2,37, 2,2,6,16,6,11,37,6, 2,2,2,2,2,2,2,1,
+ 11,7,3,3,3,3,7,3, 7,3,3,3,7,3,3,3, 3,7,10,3,7,3,3,7, 3,3,7,7,3,3,3,3,
+ 10,14,3,14,3,10,3,11, 10,3,10,10,7,3,3,18, 3,3,14,10,18,3,3,3, 3,3,7,10,7,8,10,3,
+ 3,11,7,3,3,10,3,14, 7,14,3,3,11,11,10,3, 3,3,8,10,7,3,7,3, 3,3,3,3,3,3,7,10,
+ // CCxx
+ 7,39,39,37,39,39,39,37, 39,11,18,37,37,11,37,39, 39,39,39,37,39,39,39,37, 39,39,37,37,39,39,37,37,
+ 11,10,11,11,11,11,11,11, 18,11,11,10,11,10,10,11, 37,11,11,11,11,11,11,11, 11,11,11,6,11,37,6,18,
+ 8,1,12,12,12,16,1,1, 12,16,12,12,12,12,12,31, 12,1,12,12,12,8,12,6, 8,10,12,1,11,11,1,11,
+ 10,1,1,1,8,16,1,1, 1,10,1,1,1,3,1,1, 1,8,18,1,1,10,37,11, 8,10,1,8,8,11,3,39,
+ 2,2,6,2,2,2,2,2, 2,2,12,6,6,12,12,6, 6,6,6,6,6,6,3,6, 6,37,2,3,37,11,2,1,
+ 11,7,7,7,7,3,3,3, 3,3,7,3,3,3,7,3, 3,7,3,3,7,7,3,3, 3,3,3,3,3,3,7,7,
+ 10,18,7,3,7,10,3,11, 10,18,11,10,7,10,10,11, 11,11,11,10,3,3,3,3, 3,3,7,10,7,3,11,10,
+ 10,3,3,11,11,3,11,7, 10,7,3,7,3,11,10,3, 3,7,7,7,3,3,7,7, 3,3,3,3,3,3,7,10,
+ // CDxx
+ 7,39,37,14,39,37,37,37, 37,14,18,14,37,11,14,11, 39,39,37,39,39,39,37,37, 39,37,37,37,39,37,37,37,
+ 11,18,11,11,11,11,14,14, 14,11,11,11,18,18,14,11, 37,14,11,14,14,14,14,14, 14,11,14,18,11,11,10,18,
+ 18,1,1,1,1,1,1,1, 1,10,1,1,1,1,1,1, 1,1,1,1,1,1,1,8, 1,10,1,37,37,11,37,11,
+ 6,1,1,1,1,16,37,1, 8,10,8,39,1,1,1,1, 37,8,1,1,1,37,10,8, 1,39,8,37,14,11,11,37,
+ 37,6,6,6,37,39,37,2, 6,37,12,6,6,12,12,6, 6,39,6,6,37,6,6,6, 11,37,37,3,37,11,11,1,
+ 1,14,7,3,14,3,3,14, 3,3,3,7,3,7,3,3, 3,3,10,3,3,3,3,3, 3,14,14,3,3,7,3,3,
+ 10,3,14,14,10,10,3,11, 10,11,11,3,11,10,10,11, 3,7,10,11,14,7,11,7, 3,11,3,10,11,11,11,11,
+ 10,11,3,11,11,10,11,3, 7,14,3,3,11,11,10,20, 7,7,3,11,3,3,3,7, 3,3,3,3,3,7,3,10,
+ // CExx
+ 7,39,37,37,39,39,37,37, 39,11,18,37,39,11,37,37, 39,39,37,37,39,39,37,37, 39,39,3,37,39,39,37,37,
+ 11,10,11,11,39,10,10,10, 11,10,11,5,11,18,10,11, 37,11,10,10,11,10,10,11, 11,11,11,18,11,11,10,18,
+ 5,8,16,16,16,10,1,16, 1,1,16,16,1,16,1,37, 16,37,16,16,1,31,16,39, 37,1,16,37,37,8,37,10,
+ 8,18,37,1,8,37,10,8, 37,18,1,39,1,1,1,1, 16,8,16,16,37,37,16,37, 8,18,1,37,39,8,3,37,
+ 37,6,6,6,2,2,2,2, 2,2,2,6,2,6,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,3,2,2,3,2,3, 7,7,3,3,3,2,2,3, 3,2,2,2,3,2,3,2, 2,2,2,3,2,2,2,2,
+ 3,10,10,10,3,3,10,11, 3,10,3,10,10,10,11,7, 10,10,3,7,10,3,10,10, 11,18,3,3,11,3,3,10,
+ 7,11,3,7,3,3,11,3, 3,3,10,10,10,11,7,3, 10,3,10,3,10,3,3,3, 7,10,3,3,3,3,3,37,
+ // CFxx
+ 7,11,37,37,37,39,37,37, 37,11,18,11,39,11,37,37, 39,37,37,37,39,39,37,37, 39,39,37,39,37,39,37,37,
+ 11,11,11,6,11,11,11,11, 11,11,11,11,11,18,11,11, 37,11,11,11,11,11,11,11, 11,11,11,18,11,11,11,18,
+ 11,12,1,1,1,1,1,30, 1,16,30,1,1,1,1,16, 10,1,1,1,1,16,1,1, 1,1,1,11,11,11,11,11,
+ 8,12,37,39,12,39,8,37, 8,8,1,39,1,8,1,10, 10,8,12,8,1,12,8,8, 8,8,37,37,18,11,39,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,37, 11,39,11,6,11,2,2,6, 11,11,37,2,2,11,2,1,
+ 11,11,3,3,3,3,3,3, 3,7,3,7,3,7,3,3, 3,3,3,7,3,3,3,3, 3,3,7,7,3,3,20,7,
+ 10,3,3,7,3,10,3,11, 3,11,3,10,18,18,10,11, 10,11,18,10,3,3,3,3, 3,3,11,10,3,11,3,3,
+ 3,11,3,11,3,11,11,3, 3,3,3,3,3,11,10,3, 10,3,3,3,3,11,11,3, 11,3,3,3,3,3,3,10,
+ // D0xx
+ 7,39,1,37,39,1,39,9, 39,14,14,1,39,14,1,39, 39,39,37,37,39,39,37,37, 39,39,37,39,39,37,37,37,
+ 11,14,14,11,1,10,14,14, 14,14,14,14,10,10,10,11, 37,10,10,10,10,14,10,31, 10,10,11,10,14,10,10,14,
+ 1,13,1,13,13,13,1,1, 1,13,1,16,13,13,16,16, 16,37,13,1,16,13,16,1, 39,16,13,14,3,14,14,10,
+ 8,16,1,39,1,16,31,3, 37,16,1,16,16,16,16,16, 18,8,16,8,16,16,16,8, 39,16,8,8,14,8,3,37,
+ 10,2,6,2,2,2,2,2, 2,2,2,2,2,3,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,2,3,2,2,3,3, 2,3,14,2,3,3,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,3,3,10,3,3,10,3, 10,11,10,3,10,3,10,18, 3,10,10,10,3,3,3,3, 3,18,10,10,10,3,3,11,
+ 10,11,3,3,11,10,11,3, 3,3,3,3,3,3,10,20, 3,20,3,10,3,3,11,3, 3,3,11,3,3,3,3,10,
+ // D1xx
+ 7,37,37,37,39,39,11,37, 39,11,18,37,39,11,37,39, 39,39,37,37,39,39,37,37, 39,39,37,39,39,37,37,37,
+ 11,11,11,10,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,11,11, 11,11,11,18,11,11,11,18,
+ 11,1,12,12,10,1,10,1, 10,1,1,12,8,12,37,1, 10,37,37,12,10,1,10,31, 11,3,1,37,11,11,11,11,
+ 39,1,3,10,31,1,37,37, 10,10,1,11,11,37,7,1, 31,11,10,37,31,1,37,31, 11,39,37,8,11,11,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 11,2,2,2,2,2,2,2, 2,2,2,2,2,11,2,2,
+ 11,3,3,3,14,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,14,14,3,3,14, 3,3,14,3,3,3,14,3,
+ 10,20,10,11,11,10,11,11, 11,11,10,10,11,11,10,11, 3,10,10,11,3,3,11,3, 3,20,11,3,10,11,11,11,
+ 10,10,10,11,11,11,11,3, 10,3,10,3,11,11,10,10, 10,10,10,3,3,3,11,10, 3,3,11,3,3,3,3,10,
+ // D2xx
+ 7,39,39,37,39,39,37,12, 39,14,14,39,39,14,37,37, 39,39,37,39,39,39,39,37, 39,39,37,37,39,37,37,37,
+ 11,10,14,14,11,11,14,14, 14,14,11,11,11,10,10,14, 37,10,14,14,10,14,11,11, 14,11,14,18,14,14,14,18,
+ 11,16,1,1,1,16,8,1, 31,16,31,12,1,1,1,12, 1,1,1,12,12,12,1,31, 1,31,1,11,8,14,11,11,
+ 10,31,31,31,31,12,31,31, 31,12,8,39,31,31,31,1, 31,31,1,31,31,16,10,31, 1,31,8,1,14,39,14,39,
+ 37,6,6,2,2,39,3,4, 39,39,3,6,6,12,3,6, 11,2,2,2,28,3,2,2, 11,2,2,2,2,2,39,1,
+ 14,14,14,3,14,3,3,14, 14,3,3,3,3,14,3,2, 3,3,3,3,14,3,3,14, 14,14,3,3,14,3,14,14,
+ 10,14,10,14,3,10,3,11, 10,3,14,3,11,10,10,18, 10,3,3,10,3,3,3,3, 18,18,3,10,10,3,11,3,
+ 10,11,3,11,3,3,3,3, 10,3,3,3,3,11,10,3, 10,3,3,10,3,3,3,3, 3,3,10,3,3,3,3,10,
+ // D3xx
+ 7,37,37,1,39,37,37,37, 39,11,18,39,37,11,37,18, 37,39,37,39,37,39,37,37, 39,39,37,39,39,37,37,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,18,18,11, 11,11,11,11,11,11,11,11, 11,11,11,18,11,11,31,18,
+ 11,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,3,8,11,3,11,
+ 37,1,1,1,1,1,1,37, 1,8,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,8,1,11,11,8,39,37,
+ 10,11,6,6,2,11,3,12, 39,6,6,3,6,3,6,12, 11,2,11,3,11,6,6,2, 2,2,3,3,37,11,3,2,
+ 11,3,3,12,3,3,3,14, 3,2,3,3,9,3,3,3, 3,3,3,3,3,3,3,14, 20,14,14,14,14,3,3,3,
+ 3,10,3,3,10,3,10,11, 11,3,3,18,11,3,3,10, 3,3,3,11,18,3,3,10, 11,10,3,18,11,11,10,11,
+ 3,11,10,11,11,11,11,10, 3,3,3,3,11,11,3,3, 10,3,3,11,3,20,3,10, 11,3,3,3,3,3,3,10,
+ // D4xx
+ 7,39,37,1,39,39,39,37, 39,14,18,37,39,11,37,37, 39,39,39,37,39,39,37,37, 39,39,37,37,39,37,37,37,
+ 11,18,18,1,11,11,11,1, 18,11,11,3,18,18,18,14, 37,11,11,10,11,11,10,11, 11,11,18,18,11,11,11,18,
+ 1,1,1,1,1,8,1,8, 1,1,1,1,1,1,1,1, 1,1,1,1,1,8,1,11, 1,39,1,37,37,3,11,18,
+ 18,33,14,31,39,20,31,1, 11,10,1,39,1,1,1,20, 18,8,8,31,1,3,31,31, 1,18,1,8,11,11,3,39,
+ 37,6,6,3,3,11,12,3, 37,6,3,6,6,6,12,37, 11,6,3,3,3,6,6,6, 11,3,39,3,3,11,3,4,
+ 11,14,3,3,3,3,3,14, 14,3,3,3,3,3,3,3, 3,3,3,5,3,3,3,14, 3,14,3,14,3,3,3,3,
+ 10,18,3,3,3,18,3,11, 11,18,3,3,11,11,11,18, 3,11,18,3,3,18,3,18, 3,3,3,11,11,3,11,11,
+ 10,11,3,11,11,11,11,3, 10,14,14,3,3,11,10,20, 3,3,3,3,3,3,3,3, 3,3,3,3,3,10,3,37,
+ // D5xx
+ 7,39,37,39,37,39,37,37, 39,14,18,14,37,11,37,37, 39,39,37,39,39,39,37,37, 39,39,37,39,39,39,39,39,
+ 11,14,11,14,39,11,14,14, 14,11,11,14,18,10,18,11, 37,11,11,11,11,11,10,11, 11,11,11,18,11,11,10,18,
+ 1,31,1,12,12,1,12,1, 1,1,1,12,1,1,1,12, 1,37,1,12,1,1,12,37, 37,37,12,37,37,18,8,11,
+ 37,31,1,1,31,31,3,37, 1,1,1,1,39,31,1,31, 1,39,31,31,31,8,31,1, 1,31,37,3,14,37,8,37,
+ 2,2,2,2,2,2,2,2, 2,37,2,39,2,2,2,2, 2,2,2,3,2,2,2,6, 39,11,2,2,2,2,2,4,
+ 14,2,3,2,14,2,3,2, 3,3,3,2,3,3,3,3, 3,3,2,3,2,2,2,14, 2,3,2,14,3,3,3,3,
+ 10,14,3,3,3,3,3,11, 11,11,3,3,18,11,10,11, 3,11,3,18,3,3,3,3, 3,3,11,3,11,11,18,3,
+ 10,11,3,3,11,11,11,3, 14,14,3,3,3,11,10,3, 3,3,3,3,3,3,11,3, 11,3,11,3,3,3,3,37,
+ // D6xx
+ 7,39,37,37,37,39,37,37, 37,11,11,37,39,11,39,39, 39,37,37,37,37,37,37,37, 39,39,39,39,39,39,37,37,
+ 11,11,11,1,39,11,11,1, 11,6,11,3,11,1,11,1, 37,1,11,1,11,11,11,11, 37,1,11,1,11,3,1,1,
+ 39,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,37,37,11,3,11,
+ 8,1,1,1,1,37,1,1, 1,1,1,1,1,1,1,1, 1,8,1,1,1,1,1,1, 1,1,1,1,39,8,3,37,
+ 2,2,2,2,2,2,2,2, 6,2,39,12,6,4,6,37, 6,2,2,6,2,12,2,2, 6,2,2,2,2,2,2,4,
+ 2,14,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,2,3, 3,3,20,3,3,3,3,3,
+ 10,3,3,3,18,10,3,11, 10,11,3,18,3,11,3,11, 3,11,3,3,3,3,3,3, 3,3,3,10,3,3,3,8,
+ 3,11,10,11,11,10,11,3, 10,3,3,3,3,11,3,3, 3,3,3,3,3,3,11,3, 3,3,3,3,3,3,3,10,
+ // D7xx
+ 7,39,37,37,39,37,37,6, 39,18,18,37,39,18,37,4, 37,37,37,39,39,39,37,37, 37,39,37,37,39,39,37,37,
+ 18,18,18,11,11,11,18,10, 11,18,11,3,18,10,18,10, 37,11,11,11,11,11,11,11, 11,11,18,18,18,11,10,11,
+ 11,10,11,3,11,10,11,11, 8,10,11,11,11,11,11,37, 11,8,3,39,11,8,11,11, 11,10,11,11,11,11,8,39,
+ 11,11,11,11,11,37,11,3, 11,18,11,11,11,11,11,3, 18,11,11,11,11,3,37,11, 3,18,11,11,11,11,11,37,
+ 2,2,2,2,2,3,11,6, 2,6,37,6,3,4,6,6, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,3,3,2,3,2,2, 2,2,2,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,
+ 10,18,10,3,7,3,3,3, 10,3,3,3,3,14,18,18, 3,20,10,3,3,18,3,11, 10,18,3,3,3,3,3,3,
+ 10,20,3,3,3,3,3,3, 14,3,3,10,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,11,10,7,8,20,37,
+ // D8xx
+ 7,39,39,39,39,39,39,39, 39,1,18,39,39,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,39,39,39,11,11, 11,11,1,3,11,1,11,1, 7,1,1,1,1,1,1,1, 1,1,11,18,11,1,18,18,
+ 1,12,1,12,1,12,1,1, 1,12,1,1,1,1,1,12, 1,39,1,1,1,1,1,1, 1,1,1,11,11,11,39,11,
+ 39,12,1,33,33,12,1,1, 33,1,33,1,33,33,1,1, 8,39,1,1,33,1,1,1, 1,1,33,8,18,11,3,39,
+ 39,6,6,6,39,39,39,39, 3,39,12,6,2,6,6,39, 3,6,39,3,3,6,6,6, 39,39,39,2,33,11,39,2,
+ 11,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,14,14,3,14,5,
+ 10,14,14,14,3,10,11,11, 11,11,10,10,10,12,18,10, 10,11,10,10,3,27,10,27, 11,27,11,11,11,11,11,11,
+ 10,11,10,11,11,11,11,20, 14,14,10,10,11,11,10,10, 10,11,10,11,3,11,11,3, 11,3,11,10,3,3,3,3,
+ // D9xx
+ 7,12,39,1,4,39,4,39, 39,18,18,39,39,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,18,18,14,1,18,18,11, 6,18,14,5,18,18,18,18, 7,6,6,6,6,6,6,6, 6,6,18,18,18,18,14,18,
+ 8,1,12,12,12,1,3,1, 12,1,12,12,12,12,12,1, 12,3,12,12,12,1,12,7, 1,3,12,7,7,6,8,39,
+ 18,8,12,12,12,1,7,8, 12,1,1,12,1,12,1,8, 3,8,12,12,12,3,12,6, 1,8,39,7,3,14,5,3,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,3,6,6, 6,6,3,3,3,3,39,3,
+ 2,14,2,5,3,5,2,14, 14,3,14,2,2,14,9,3, 6,2,3,3,14,14,3,14, 3,2,14,14,2,5,2,3,
+ 10,14,7,14,18,10,3,11, 10,11,18,18,18,20,10,6, 20,11,18,20,20,5,20,18, 12,3,11,18,10,11,18,11,
+ 14,11,3,11,11,11,11,8, 14,14,14,5,11,11,10,7, 3,3,3,11,3,11,11,3, 11,5,11,7,5,5,5,10,
+ // DAxx
+ 7,1,39,1,39,39,39,39, 11,11,11,39,39,11,39,11, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,5,11,11,11,11, 7,11,11,5,5,3,5,11, 11,11,11,18,11,11,11,18,
+ 11,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,3, 1,1,1,11,11,11,11,11,
+ 8,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,8,1,1,1,1,1,3, 1,1,1,8,8,39,11,3,
+ 2,2,6,6,2,2,2,2, 2,2,12,6,2,12,12,2, 6,2,6,2,11,2,2,6, 2,2,2,3,2,11,12,1,
+ 11,11,5,3,3,8,5,3, 10,2,5,3,8,2,3,2, 6,5,6,2,3,2,3,5, 5,3,2,8,3,6,2,11,
+ 18,18,11,3,18,10,10,11, 11,11,11,11,11,18,18,11, 11,11,11,11,11,11,11,18, 11,11,11,3,11,11,11,10,
+ 27,11,27,11,11,11,11,7, 12,3,3,5,11,11,3,3, 11,11,11,11,3,11,11,3, 12,3,11,3,3,5,3,11,
+ // DBxx
+ 7,39,39,39,39,39,39,39, 39,10,10,39,39,10,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,10,10,10,39,10,10,1, 3,11,11,5,11,10,10,31, 7,31,31,31,31,31,31,31, 31,31,10,18,10,10,10,10,
+ 18,12,1,16,16,12,12,16, 12,1,12,16,16,16,16,39, 16,7,16,16,1,1,12,7, 10,1,16,7,7,8,7,8,
+ 8,1,12,39,16,1,39,16, 31,8,8,16,8,8,1,10, 8,39,1,1,1,8,39,1, 1,8,7,39,3,8,5,39,
+ 2,2,6,2,39,39,2,2, 2,6,6,2,2,2,2,39, 2,6,2,2,2,2,6,6, 39,3,2,39,2,2,2,1,
+ 11,3,3,8,3,8,8,3, 10,2,3,3,3,3,5,3, 2,2,2,2,2,2,2,2, 2,2,3,3,3,6,5,3,
+ 3,18,10,10,10,10,10,11, 11,10,10,10,10,10,18,11, 11,11,10,11,18,10,11,10, 11,7,3,1,11,11,27,10,
+ 3,11,3,11,11,11,11,3, 3,1,7,3,11,11,27,27, 7,3,11,11,5,11,5,3, 11,7,11,3,3,3,3,3,
+ // DCxx
+ 7,39,1,11,4,39,39,39, 39,1,11,39,39,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,11,11,11,11,1, 11,1,11,11,11,11,11,11, 7,11,11,11,11,11,11,11, 11,11,11,20,11,11,1,1,
+ 11,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,11,1,1,1,11,1,1, 1,1,1,11,11,11,11,11,
+ 8,11,1,1,1,1,1,8, 1,11,11,1,1,1,1,11, 11,11,1,1,1,11,1,11, 1,1,1,11,11,3,11,3,
+ 39,6,6,39,6,11,11,3, 39,3,3,6,6,6,3,39, 2,11,11,6,3,11,6,6, 2,11,2,3,3,2,3,2,
+ 2,11,3,3,3,7,11,11, 5,11,2,3,5,3,11,6, 3,5,3,3,7,3,6,3, 3,5,11,5,3,3,3,3,
+ 3,10,11,11,11,10,18,11, 11,11,11,18,11,10,11,11, 13,11,11,11,18,11,11,11, 11,11,11,11,11,11,13,11,
+ 3,11,5,11,11,11,11,3, 20,20,20,20,11,11,3,3, 11,20,11,20,20,11,3,20, 11,3,11,3,1,5,3,11,
+ // DDxx
+ 7,1,4,39,39,39,39,39, 39,13,11,13,39,11,4,39, 39,3,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,13,11,11,11,11,13,13, 13,11,13,5,11,13,13,11, 7,11,11,11,11,11,11,13, 11,11,11,13,11,11,13,13,
+ 11,13,1,1,13,13,13,13, 13,1,1,13,13,13,13,13, 1,39,13,13,13,1,1,13, 13,13,13,7,13,11,3,11,
+ 13,13,13,13,13,13,13,13, 13,13,13,13,13,13,13,13, 13,13,13,13,13,7,13,7, 13,13,13,11,13,11,5,3,
+ 56,6,6,6,3,3,3,6, 39,6,12,6,6,12,12,6, 6,6,6,6,6,6,6,6, 11,6,6,3,3,11,3,3,
+ 11,11,8,5,5,3,7,8, 7,9,3,3,3,3,3,8, 3,3,6,1,3,7,3,3, 3,3,3,5,3,6,5,11,
+ 6,5,10,11,11,11,11,11, 11,11,11,10,11,11,11,11, 11,11,10,11,11,11,11,5, 11,11,11,11,11,11,11,11,
+ 3,11,10,11,11,11,11,13, 20,10,10,10,11,11,20,10, 10,20,10,11,20,11,11,20, 11,20,11,8,3,10,13,3,
+ // DExx
+ 7,1,39,39,4,39,4,39, 39,11,11,39,6,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 6,11,11,11,11,11,13,11, 7,11,11,11,11,11,11,6, 11,11,11,6,11,11,13,18,
+ 11,13,13,13,16,13,13,13, 13,13,1,13,13,13,16,16, 16,8,1,16,13,13,13,13, 1,13,33,11,11,11,11,1,
+ 13,13,13,1,1,13,13,1, 13,13,1,3,13,16,13,13, 1,3,1,8,13,13,16,1, 7,16,39,7,6,11,11,3,
+ 2,6,2,2,2,2,2,2, 2,2,2,2,2,2,2,6, 2,2,31,2,2,2,2,2, 6,2,2,4,2,2,2,4,
+ 11,11,3,3,3,3,2,2, 2,2,2,2,2,2,2,7, 2,3,6,8,5,6,6,3, 6,3,3,3,7,6,6,11,
+ 5,18,10,10,10,18,10,11, 11,11,11,18,10,11,18,11, 10,11,10,11,18,11,11,10, 11,10,11,18,11,11,11,11,
+ 27,11,10,11,11,11,11,10, 20,5,20,10,11,11,13,10, 10,20,20,11,20,11,11,7, 11,10,11,7,13,13,11,11,
+ // DFxx
+ 7,1,39,39,7,10,39,39, 39,11,11,7,39,11,39,11, 39,39,39,39,39,39,39,39, 39,39,37,37,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,6,11,1,11,11, 7,11,11,11,11,14,14,11, 14,14,11,6,11,11,11,1,
+ 3,11,1,1,1,1,1,1, 1,1,1,8,1,3,8,1, 1,1,1,1,1,1,1,39, 11,8,1,3,3,11,3,11,
+ 3,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,39,1,1,1,1,1,1, 3,1,1,7,11,11,39,3,
+ 3,6,6,6,6,3,12,6, 6,6,6,6,6,6,6,6, 6,6,6,12,6,6,6,3, 39,6,6,6,6,11,3,4,
+ 11,11,3,5,5,6,3,5, 11,5,10,5,3,6,3,6, 6,3,5,3,3,3,3,7, 3,3,3,3,6,3,5,11,
+ 7,10,10,11,10,10,10,11, 11,11,11,11,10,10,11,11, 11,11,10,11,11,10,10,10, 10,10,11,11,11,11,10,11,
+ 27,11,10,11,11,11,11,10, 20,3,10,20,11,11,20,20, 10,10,3,11,20,11,11,3, 11,20,11,3,1,7,11,11,
+ // E0xx
+ 7,37,37,14,1,37,37,37, 37,10,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,10,10,10,11,11,10,10, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10,
+ 10,3,11,11,33,11,6,33, 8,11,11,6,33,11,33,3, 11,39,33,8,8,11,6,33, 11,6,8,10,10,10,10,6,
+ 17,11,11,11,11,39,11,11, 6,11,16,30,11,11,11,11, 11,11,11,11,11,6,11,11, 11,11,39,6,17,37,3,37,
+ 6,6,6,6,6,11,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,10,6,6,6, 37,6,12,3,10,6,10,10,
+ 36,14,14,36,2,2,2,2, 14,14,14,14,36,36,2,2, 2,2,2,2,2,14,2,14, 2,2,14,14,14,2,14,14,
+ 14,14,14,14,17,14,17,14, 14,1,14,14,18,14,14,8, 27,3,18,27,5,27,3,3, 27,3,3,6,27,7,3,6,
+ 3,10,10,10,10,17,10,10, 17,17,10,10,10,10,17,10, 10,10,10,10,10,10,10,10, 17,17,17,18,5,10,10,10,
+ // E1xx
+ 7,1,1,37,37,39,4,37, 12,1,11,20,37,11,3,11, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,1,10,11, 37,11,11,11,17,11,11,11, 11,11,11,1,11,11,11,1,
+ 11,11,1,1,1,17,1,17, 17,8,1,1,1,8,1,39, 6,39,1,6,39,1,8,11, 6,8,1,11,17,11,11,11,
+ 37,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,6, 1,6,1,6,11,11,11,37,
+ 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,6, 11,2,12,6,12,12,12,2,
+ 36,14,14,3,14,36,5,14, 14,6,36,36,36,36,36,36, 3,7,3,10,14,14,14,14, 2,14,2,2,2,2,14,14,
+ 3,14,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 10,11,36,11,11,11,11,17, 10,17,20,10,11,11,10,17, 10,18,17,10,18,11,11,18, 17,17,17,10,10,18,18,10,
+ // E2xx
+ 7,7,37,37,37,39,37,37, 37,10,10,37,4,10,1,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,10,17, 10,10,10,3,10,10,10,10, 37,10,17,10,10,10,10,10, 10,10,10,10,10,10,10,10,
+ 10,8,8,11,3,6,11,6, 6,6,6,6,6,6,11,6, 6,6,11,6,11,6,6,6, 6,37,6,10,17,10,37,10,
+ 17,10,11,16,16,10,16,11, 11,10,16,16,16,11,11,10, 16,8,16,16,11,11,16,6, 8,11,16,3,6,6,3,37,
+ 2,2,2,6,2,2,2,2, 2,2,2,6,2,6,2,6, 6,2,2,2,2,2,2,2, 2,2,10,6,2,2,6,6,
+ 36,14,36,3,14,36,14,3, 36,5,14,14,3,36,36,3, 3,3,3,10,14,14,3,14, 10,14,14,14,14,3,14,3,
+ 14,18,14,14,6,18,3,14, 3,18,14,14,18,14,14,18, 27,3,18,3,7,18,3,3, 27,18,27,7,20,20,27,3,
+ 10,18,10,17,10,10,17,10, 10,17,10,10,17,10,10,18, 10,10,10,10,17,18,10,10, 17,18,10,10,10,3,16,10,
+ // E3xx
+ 7,37,37,37,37,37,37,1, 3,11,11,37,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,10,11, 37,11,11,11,11,11,11,11, 11,11,11,10,11,11,11,17,
+ 10,12,1,6,6,12,6,1, 31,18,6,6,1,31,39,6, 6,37,6,39,12,1,6,6, 1,6,8,11,11,10,11,11,
+ 6,10,12,12,12,1,37,12, 11,12,6,6,12,12,8,1, 12,8,12,1,12,12,10,37, 37,3,12,11,17,11,11,37,
+ 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 11,11,6,6,6,6,6,6, 11,11,6,4,6,11,6,4,
+ 36,11,36,36,14,36,36,36, 14,6,14,36,36,36,7,36, 3,3,3,10,14,14,3,7, 8,14,14,11,5,3,3,11,
+ 5,5,11,11,11,18,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 10,11,17,11,11,11,11,36, 10,17,10,10,11,11,10,20, 10,20,17,11,17,11,11,17, 17,17,11,3,3,5,3,1,
+ // E4xx
+ 7,11,1,11,37,37,37,37, 1,11,11,4,4,11,3,11, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,17,11,11,11,11,11,17, 11,11,11,17,11,17,11,11, 11,11,11,11,11,11,11,11, 11,11,11,17,11,11,11,17,
+ 11,1,6,6,6,1,6,1, 1,17,1,1,1,1,6,11, 6,37,1,6,1,1,1,8, 6,1,1,11,11,11,11,11,
+ 17,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,17,11,11,1,
+ 2,11,6,6,12,11,2,6, 6,6,6,6,6,6,11,6, 11,6,31,3,11,6,6,6, 11,11,6,3,12,12,2,6,
+ 36,11,14,7,14,36,8,14, 36,14,14,14,5,3,36,3, 3,3,5,10,14,14,6,14, 2,2,2,2,2,2,2,2,
+ 3,14,3,14,11,18,11,11, 11,11,11,14,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 10,17,10,11,1,10,11,17, 10,17,10,10,11,11,10,18, 10,17,17,10,17,18,17,17, 17,17,17,10,10,20,11,10,
+ // E5xx
+ 7,10,37,37,37,39,37,4, 39,10,10,1,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,11,11,10,10,10, 11,11,11,10,10,10,10,10, 37,11,11,11,36,11,10,11, 11,11,10,10,11,11,10,10,
+ 11,6,31,31,6,31,8,1, 6,6,6,6,1,6,6,37, 8,6,1,37,31,6,1,37, 3,6,6,11,10,11,11,11,
+ 17,1,1,10,1,1,1,1, 1,1,6,1,1,1,1,33, 1,39,1,1,1,33,1,39, 8,6,1,11,11,11,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 17,7,11,11,11,18,18,11, 11,11,11,3,11,18,11,11, 11,11,11,18,11,5,11,18, 11,11,11,3,11,11,11,20,
+ 17,11,10,11,10,11,11,10, 17,10,10,10,10,10,18,17, 10,10,10,18,17,11,10,17, 17,17,17,18,5,20,10,10,
+ // E6xx
+ 7,11,1,9,1,39,1,12, 1,12,12,11,37,11,37,11, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,12,11,11,11,11,11,11, 11,11,11,5,12,11,12,11, 37,11,11,11,11,11,11,11, 11,11,11,12,11,11,11,12,
+ 6,33,11,8,11,33,6,11, 11,33,11,33,33,31,33,33, 33,6,1,6,33,33,6,11, 8,37,11,39,17,6,11,11,
+ 17,12,1,1,1,12,1,1, 1,12,1,1,1,1,1,12, 1,1,1,1,1,12,1,12, 6,8,1,11,6,11,6,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 5,11,11,11,3,11,7,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,3, 11,11,11,11,11,11,11,11,
+ 10,11,10,11,11,11,11,20, 10,17,10,17,11,11,17,18, 1,10,11,11,18,11,11,17, 11,20,11,10,10,3,11,10,
+ // E7xx
+ 7,10,37,37,4,39,37,37, 37,11,10,20,4,10,10,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 10,10,10,11,11,10,11,11, 10,10,10,11,10,10,10,17, 37,10,10,11,11,11,10,17, 11,10,11,20,10,11,11,10,
+ 11,11,11,11,31,8,31,31, 6,11,11,11,31,8,31,6, 31,3,31,31,31,11,31,6, 11,11,31,11,17,11,6,11,
+ 11,11,11,16,16,11,11,16, 11,11,16,16,11,11,16,11, 16,6,16,16,16,11,16,11, 8,11,16,11,17,11,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,14,2,2,2,2,2,2,
+ 1,14,14,14,18,18,1,14, 5,18,14,14,18,3,6,18, 27,3,18,7,3,18,6,3, 27,18,27,3,8,5,3,3,
+ 10,17,10,1,10,17,17,17, 10,17,10,10,10,10,10,18, 10,17,18,10,17,1,1,17, 17,17,17,10,11,13,10,10,
+ // E8xx
+ 7,1,37,1,12,39,1,37, 37,10,10,10,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,17,11,10,10,10, 10,10,10,10,10,10,10,10, 37,14,14,14,14,14,14,14, 14,14,10,10,10,10,10,10,
+ 10,6,6,11,6,12,3,11, 14,6,6,12,31,6,12,6, 18,37,11,11,11,12,6,6, 8,37,6,10,10,10,10,10,
+ 10,12,11,11,11,12,11,11, 12,12,12,12,12,11,12,12, 12,11,11,11,12,12,11,6, 11,11,11,6,10,39,10,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,14,2,2,2,2,2,2,
+ 14,14,14,14,7,14,3,14, 14,3,14,14,18,14,14,18, 14,3,14,14,6,18,3,3, 5,3,27,3,20,20,20,7,
+ 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,18, 10,10,10,10,10,10,10,10, 17,10,17,8,20,20,10,10,
+ // E9xx
+ 7,1,11,4,1,37,39,37, 37,11,11,11,37,11,37,1, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,39,37,
+ 11,10,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,17,11,17,11,11,11,11, 11,11,10,10,11,10,11,11,
+ 11,11,11,11,11,11,3,8, 33,11,6,33,6,31,11,11, 33,6,31,31,3,11,11,11, 11,11,33,6,6,11,11,11,
+ 6,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,8, 11,11,11,6,17,6,17,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,14, 2,2,2,2,2,2,2,2, 2,2,2,2,14,14,2,14, 2,14,2,2,2,2,2,3,
+ 18,14,14,14,3,14,3,14, 18,3,14,14,18,14,18,3, 14,3,14,14,18,27,18,18, 3,3,18,18,20,20,20,3,
+ 14,17,17,17,17,17,17,17, 17,17,20,17,17,17,18,17, 17,10,10,18,18,17,17,17, 17,17,17,18,20,18,18,10,
+ // EAxx
+ 7,37,9,37,37,39,37,9, 37,10,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,12,10,12,11,10,10,10, 10,10,10,10,10,10,10,10, 37,10,6,10,10,10,10,10, 10,10,10,10,10,10,10,12,
+ 10,31,31,31,31,31,6,12, 31,6,6,31,31,8,31,31, 3,31,37,31,31,37,31,8, 11,11,31,10,12,10,12,10,
+ 10,10,12,12,12,11,37,12, 11,10,11,12,11,11,11,10, 12,11,11,11,11,11,11,12, 11,6,12,8,17,6,3,37,
+ 6,39,6,6,2,3,2,3, 6,6,6,2,39,6,6,6, 6,3,6,6,6,6,6,6, 6,11,6,11,12,6,6,6,
+ 10,14,6,5,14,36,8,14, 3,3,5,5,5,5,5,3, 2,2,2,2,2,2,9,2, 2,2,2,2,2,2,2,12,
+ 3,3,14,3,10,5,3,3, 3,3,3,3,3,3,5,3, 14,3,14,3,3,3,5,3, 3,34,3,7,20,20,20,20,
+ 10,20,10,10,18,10,10,20, 10,20,10,10,10,10,10,20, 10,10,10,10,18,20,10,10, 10,20,10,18,20,20,18,10,
+ // EBxx
+ 7,10,37,37,4,37,37,37, 37,10,10,10,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,16,10, 10,11,10,5,10,10,10,11, 37,10,17,17,17,10,10,17, 17,10,10,10,10,36,10,10,
+ 10,3,8,8,8,3,11,11, 3,18,14,11,8,3,31,18, 18,37,8,3,31,3,8,37, 39,18,37,3,10,10,37,39,
+ 17,10,11,11,16,11,11,16, 11,11,16,16,11,16,11,10, 16,11,11,16,16,11,16,11, 3,3,11,37,8,37,5,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,36,2,5,2,2,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,5,2,2, 2,2,5,2,5,2,7,2,
+ 5,18,14,5,3,18,3,14, 3,18,3,3,18,14,18,18, 3,3,18,18,18,18,3,3, 3,3,3,8,20,20,20,20,
+ 10,18,10,10,10,10,10,20, 10,18,10,10,17,10,10,18, 17,10,18,10,18,18,18,10, 17,17,17,10,10,20,10,10,
+ // ECxx
+ 7,11,37,1,37,37,37,37, 37,14,10,14,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,10,11,11,11,10,10,17, 14,10,11,11,10,10,10,10, 37,11,10,10,17,11,17,11, 17,17,11,10,11,10,14,10,
+ 11,1,17,8,8,18,8,8, 8,8,1,8,31,17,39,11, 17,31,17,17,1,1,17,17, 17,18,3,14,17,11,11,11,
+ 3,1,12,12,12,16,1,1, 12,16,12,12,12,12,12,1, 12,1,12,12,12,16,12,1, 11,10,12,11,17,11,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,10,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 14,2,2,3,36,36,3,2, 2,2,2,10,2,2,2,14, 2,2,2,2,2,2,14,11,
+ 18,14,14,14,17,3,17,14, 14,18,14,14,3,14,14,18, 3,5,3,11,7,18,3,5, 18,3,12,5,20,20,20,20,
+ 10,18,17,17,17,10,17,17, 10,17,10,17,10,10,10,10, 17,17,17,10,17,18,17,17, 18,17,17,10,10,10,11,10,
+ // EDxx
+ 7,11,37,37,37,37,37,37, 4,1,1,1,37,1,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,1,11,17,11,11,1,1, 11,11,11,17,1,10,1,1, 37,11,11,11,11,11,11,11, 11,11,10,1,11,11,10,1,
+ 11,1,33,33,33,18,1,1, 33,18,1,33,8,33,1,1, 33,39,33,33,33,33,33,37, 11,37,33,11,10,11,11,11,
+ 17,1,1,1,1,1,1,1, 1,10,1,1,1,1,1,1, 1,1,1,1,1,1,1,8, 1,10,1,11,17,10,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 11,2,2,3,2,2,2,2, 2,2,2,2,2,2,2,10,
+ 1,11,36,3,11,3,3,3, 11,7,36,36,3,5,5,7, 7,5,3,10,7,3,3,3, 10,9,5,5,3,3,9,11,
+ 5,18,11,11,11,18,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11,
+ 10,11,36,11,11,10,11,10, 10,18,10,10,11,10,10,18, 18,10,10,10,10,11,10,10, 11,18,11,10,10,20,10,10,
+ // EExx
+ 7,10,37,37,37,37,4,37, 37,10,10,1,37,10,10,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,10,10, 10,10,10,10,10,10,10,10, 37,17,17,17,10,17,10,10, 10,17,10,10,10,10,10,10,
+ 10,3,3,31,31,18,31,31, 3,3,31,8,31,31,31,39, 8,37,31,31,31,37,31,31, 31,18,8,10,10,10,3,10,
+ 17,11,16,16,16,37,11,16, 8,11,16,16,11,16,11,11, 16,11,16,16,11,8,16,3, 11,11,16,37,10,3,3,37,
+ 2,2,6,2,2,10,3,2, 2,2,3,2,6,2,6,2, 3,6,11,2,10,6,2,2, 37,2,2,2,2,3,2,2,
+ 10,3,5,3,5,3,8,3, 3,3,3,3,36,10,3,5, 5,7,3,5,3,7,3,3, 10,2,10,5,10,3,3,10,
+ 17,18,3,10,17,18,3,8, 3,18,5,3,17,5,3,18, 3,3,3,3,3,18,3,27, 18,3,3,3,5,20,20,20,
+ 17,10,10,10,10,10,10,17, 10,10,10,10,10,10,18,10, 10,10,10,18,10,10,10,10, 17,17,17,18,18,10,10,10,
+ // EFxx
+ 7,37,37,37,37,37,37,10, 37,17,20,17,37,17,37,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 17,17,17,17,17,10,18,17, 17,17,17,17,17,10,10,17, 37,10,10,17,10,10,10,17, 17,10,17,17,17,17,17,17,
+ 3,11,11,11,11,11,3,3, 8,18,3,11,3,31,37,37, 18,37,37,39,31,11,37,8, 8,37,11,37,17,17,37,17,
+ 17,12,11,11,11,11,11,30, 3,16,30,11,11,11,11,16, 10,11,11,11,11,16,11,11, 11,39,39,8,17,37,3,37,
+ 2,2,2,2,2,2,2,2, 2,37,2,2,2,39,2,2, 2,2,2,2,2,2,2,3, 39,3,2,2,2,2,12,2,
+ 17,36,36,3,2,5,5,5, 2,36,7,7,36,2,3,2, 5,3,3,10,2,3,3,5, 2,2,2,2,2,2,2,2,
+ 3,5,18,18,18,7,18,18, 18,3,18,18,18,18,18,3, 18,3,18,18,18,7,18,5, 3,5,3,3,20,20,18,20,
+ 10,5,18,20,18,10,18,18, 10,20,18,10,18,18,10,10, 10,20,18,10,18,20,18,18, 10,20,18,10,10,20,18,10,
+ // F0xx
+ 3,37,1,37,10,37,39,39, 37,10,10,37,37,11,4,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,10,10, 10,10,11,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,1,
+ 10,12,3,14,1,1,1,13, 1,13,8,8,13,13,37,18, 18,37,13,39,37,13,39,11, 39,37,1,11,10,10,3,10,
+ 17,13,1,13,13,13,1,1, 1,13,1,16,13,13,16,16, 16,37,13,1,16,13,16,1, 1,16,13,11,10,11,14,39,
+ 39,6,6,6,36,11,4,36, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,4,3,3,10,2,10,10,
+ 2,11,3,3,3,3,2,3, 5,3,3,5,5,5,3,5, 3,3,5,10,5,3,5,5, 7,5,11,11,5,3,5,11,
+ 3,18,5,8,3,18,3,11, 5,18,3,3,18,3,18,18, 3,18,18,18,18,18,3,7, 3,18,5,3,20,20,20,20,
+ 10,10,10,10,17,10,10,10, 10,17,10,17,10,10,10,18, 10,10,10,10,10,10,10,17, 10,17,17,10,10,13,10,10,
+ // F1xx
+ 3,9,37,9,1,10,39,37, 1,12,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,10,10,10,10, 10,10,17,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,12,10,10,10,10,
+ 1,1,1,12,8,1,8,1, 1,1,1,8,1,1,37,1, 3,8,3,12,8,1,37,12, 8,37,8,10,1,12,8,10,
+ 17,1,12,12,10,1,10,1, 10,1,8,12,10,12,1,1, 10,1,10,12,10,1,37,1, 10,10,1,3,17,11,5,37,
+ 39,6,6,6,4,10,4,4, 6,6,37,6,6,6,6,6, 6,6,6,6,10,6,6,6, 6,37,10,3,4,3,4,4,
+ 10,11,36,3,3,7,5,5, 3,8,5,3,5,7,3,8, 3,3,3,10,3,3,5,3, 10,5,3,3,3,7,3,3,
+ 17,5,11,3,3,3,3,3, 11,3,3,18,3,18,18,11, 18,3,18,5,3,5,3,18, 3,7,18,18,20,20,20,20,
+ 10,20,10,17,10,10,10,20, 10,17,10,10,10,10,10,10, 10,10,10,10,17,10,10,10, 10,20,10,10,10,18,10,10,
+ // F2xx
+ 3,1,37,1,37,37,39,37, 37,10,10,10,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,1,10,10,10, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,20,10,10,
+ 10,31,31,17,31,12,10,1, 1,18,31,1,31,31,31,1, 31,37,8,31,31,8,39,37, 39,3,31,10,10,10,8,10,
+ 17,16,1,1,1,16,39,1, 12,16,1,12,1,1,1,12, 1,1,1,12,12,12,1,1, 1,10,1,1,17,11,37,37,
+ 37,6,6,39,4,10,3,4, 37,6,6,6,6,6,6,37, 6,6,6,6,10,6,39,6, 39,4,12,11,3,12,3,4,
+ 10,11,5,3,5,5,3,3, 5,5,5,3,5,8,1,3, 3,5,7,10,7,5,26,5, 10,3,3,20,3,3,3,7,
+ 3,18,11,3,5,18,3,11, 11,18,11,3,3,11,3,18, 3,3,1,11,3,18,11,3, 5,18,3,3,11,11,11,3,
+ 10,18,10,17,17,10,17,18, 10,17,10,10,17,10,10,18, 10,10,10,10,18,18,17,10, 17,18,17,10,10,10,10,10,
+ // F3xx
+ 3,1,1,37,4,4,39,37, 1,10,10,9,37,10,37,1, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,17,10,10,10, 10,10,11,17,10,10,10,1, 37,18,1,10,10,18,10,18, 10,18,1,10,10,10,1,10,
+ 3,11,1,1,1,1,1,1, 18,18,31,31,31,1,1,8, 1,11,1,39,31,37,39,1, 37,37,31,37,1,10,37,1,
+ 39,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,11,17,11,24,37,
+ 37,39,6,6,37,10,3,12, 37,37,6,6,6,6,6,6, 6,11,23,6,12,6,6,39, 37,24,10,3,12,3,10,12,
+ 10,11,5,3,8,8,3,3, 5,3,8,3,3,5,5,3, 3,3,3,12,3,3,9,5, 3,3,10,3,9,2,3,12,
+ 5,18,18,11,18,18,3,11, 11,11,11,18,18,11,11,18, 18,11,18,11,18,18,11,18, 11,11,11,11,11,11,11,11,
+ 10,10,10,10,10,18,10,10, 10,18,10,18,18,10,18,10, 10,10,10,18,18,10,10,10, 10,10,18,18,20,20,10,10,
+ // F4xx
+ 3,4,37,37,37,37,39,37, 37,11,10,37,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,11,17,11,11,10,20,11, 11,10,11,3,10,10,18,20, 37,10,25,17,10,10,10,11, 10,10,10,11,11,17,10,10,
+ 17,18,11,8,11,18,11,11, 11,18,24,11,11,11,37,18, 18,37,11,37,11,11,39,11, 11,37,11,11,17,11,37,11,
+ 17,11,11,11,11,37,39,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,39,3,37,
+ 2,39,6,6,3,24,3,3, 11,37,37,39,11,6,6,6, 37,6,23,6,3,39,6,37, 37,11,12,3,39,12,12,4,
+ 25,5,3,7,11,3,11,11, 8,3,8,3,3,8,3,8, 8,8,3,10,3,3,5,3, 5,5,3,5,3,3,9,8,
+ 18,18,3,3,3,18,3,3, 5,3,3,3,3,3,8,18, 3,5,18,3,3,18,3,18, 3,18,1,3,20,20,20,3,
+ 10,18,17,17,17,17,3,20, 10,17,10,10,17,10,10,18, 10,17,18,10,10,18,17,18, 17,17,17,10,20,20,20,37,
+ // F5xx
+ 3,37,1,4,37,39,39,37, 37,10,10,37,37,10,37,39, 37,39,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,10,10,10,20,10,10,
+ 10,10,11,1,3,8,8,1, 10,1,1,1,1,1,37,8, 1,1,1,1,1,1,1,8, 8,1,1,1,17,10,37,39,
+ 17,12,1,12,12,1,12,1, 1,1,1,12,1,1,1,12, 1,3,1,12,1,1,12,1, 1,1,12,1,8,11,3,37,
+ 37,6,6,6,4,10,4,4, 6,6,6,6,6,39,6,6, 6,11,6,6,10,6,6,6, 4,4,4,4,3,3,3,4,
+ 10,11,5,3,3,3,8,8, 5,1,3,3,8,8,5,1, 8,5,3,10,3,3,5,3, 3,3,10,3,3,5,3,3,
+ 3,5,18,5,18,5,18,1, 11,18,11,18,3,11,3,11, 18,11,18,18,11,11,11,18, 11,11,11,11,11,11,18,11,
+ 10,11,18,11,18,11,11,18, 10,25,20,18,18,10,10,10, 10,1,18,18,18,10,18,18, 20,3,18,18,3,18,18,33,
+ // F6xx
+ 3,1,37,37,37,37,39,37, 37,1,1,1,37,1,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,10,1,1,1,10,1,17, 3,10,11,5,10,10,10,1, 37,5,1,1,5,10,10,3, 1,1,10,10,1,11,1,1,
+ 5,11,1,1,1,1,1,1, 1,24,1,1,1,1,39,37, 11,37,1,1,1,37,1,37, 1,1,1,37,17,1,39,1,
+ 17,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,8,1,11,5,37,
+ 39,6,6,4,24,24,3,3, 37,6,6,6,37,37,6,37, 6,17,17,6,39,37,39,6, 37,37,3,11,4,3,3,4,
+ 11,11,5,3,3,8,3,3, 3,3,3,3,3,3,3,8, 3,3,3,10,3,3,3,5, 3,5,3,3,3,5,5,5,
+ 3,3,3,3,3,18,11,11, 11,18,11,11,11,11,3,11, 11,11,11,11,11,18,11,3, 11,11,11,11,11,20,11,1,
+ 10,17,10,11,17,10,11,17, 10,17,10,18,17,11,10,20, 13,20,17,10,17,18,1,17, 17,20,17,10,10,20,13,10,
+ // F7xx
+ 3,1,37,37,37,37,39,37, 37,10,18,6,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,7,37,37,37,37,39,
+ 17,17,17,17,11,10,18,17, 11,10,17,3,17,10,18,10, 11,11,11,11,11,11,11,11, 11,11,17,17,10,17,10,17,
+ 8,18,11,8,3,18,8,8, 3,18,11,3,11,11,39,18, 11,37,11,39,37,3,11,39, 11,37,3,11,11,8,17,17,
+ 17,10,37,11,11,10,11,11, 37,8,11,11,11,39,11,20, 11,11,11,11,11,11,11,11, 39,10,11,37,17,17,8,37,
+ 3,6,6,3,3,3,3,3, 37,6,6,39,6,37,6,39, 6,37,11,6,10,6,6,6, 4,37,3,3,10,4,12,4,
+ 17,3,3,3,11,3,26,3, 3,3,8,8,3,3,3,3, 3,3,3,10,8,3,26,3, 10,5,3,3,3,5,3,3,
+ 3,18,3,3,3,18,3,5, 3,18,3,18,18,18,18,18, 18,3,18,18,18,18,3,18, 18,18,18,3,20,20,20,20,
+ 10,18,10,17,17,10,17,17, 10,17,10,10,17,10,18,18, 17,17,10,18,17,18,17,18, 17,18,17,18,10,20,20,10,
+ // F8xx
+ 3,1,37,37,37,37,39,37, 1,17,17,37,37,17,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 17,17,17,17,1,17,16,17, 17,16,17,5,17,17,17,17, 37,5,1,17,1,1,1,1, 1,10,17,17,17,17,17,17,
+ 10,8,3,1,1,1,12,1, 1,1,8,1,1,1,37,37, 1,37,1,39,37,8,1,8, 8,37,1,37,17,17,37,1,
+ 17,12,1,12,1,12,1,1, 1,12,1,1,1,1,1,12, 1,8,1,1,1,1,1,1, 1,1,1,3,37,37,5,37,
+ 39,6,6,39,4,16,3,17, 39,6,37,39,6,37,6,37, 37,11,6,16,16,37,16,6, 37,37,12,3,10,3,12,24,
+ 17,11,5,3,11,8,8,8, 5,1,8,5,8,8,1,8, 8,5,3,10,3,5,3,3, 10,5,5,3,5,5,3,5,
+ 5,5,3,11,24,17,17,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,3, 11,3,11,11,11,11,11,11,
+ 17,17,17,17,17,17,17,17, 10,17,10,17,17,12,18,17, 17,17,10,11,17,17,11,17, 17,17,17,18,10,18,10,37,
+ // F9xx
+ 3,1,37,37,37,37,39,39, 37,12,12,1,37,12,37,39, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 12,17,12,17,11,11,30,12, 12,12,17,5,12,12,12,17, 37,17,17,17,17,17,10,17, 3,17,12,12,12,17,12,12,
+ 8,8,11,8,12,11,24,24, 24,24,11,17,11,11,11,8, 39,37,11,8,11,11,11,39, 11,11,3,39,17,12,24,11,
+ 17,11,12,12,12,11,12,11, 12,11,12,12,12,12,12,11, 12,37,12,12,12,11,12,24, 11,11,12,37,17,24,24,37,
+ 37,24,6,11,11,12,37,24, 24,24,24,37,37,24,24,24, 24,24,24,12,24,24,37,24, 24,24,12,24,24,24,12,24,
+ 12,5,24,3,8,5,5,24, 5,24,1,5,5,5,8,8, 5,5,3,8,3,3,5,8, 10,5,8,5,24,3,9,24,
+ 1,5,24,5,8,5,3,5, 3,1,5,24,17,5,5,5, 8,17,17,5,3,8,8,8, 8,8,8,5,8,8,8,8,
+ 10,17,17,17,17,10,17,17, 10,17,18,17,17,17,17,17, 17,20,17,20,17,7,17,17, 17,17,17,18,10,8,8,10,
+ // FAxx
+ 3,10,1,37,37,37,37,37, 37,1,17,1,37,17,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 17,17,17,17,1,17,17,17, 17,17,17,17,17,17,17,17, 37,17,17,17,17,17,17,17, 1,17,17,17,17,17,1,17,
+ 8,8,1,8,8,1,1,8, 8,1,1,24,1,1,37,37, 8,39,1,37,1,1,39,39, 11,1,1,37,17,11,39,17,
+ 17,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,37, 1,37,1,1,17,11,11,37,
+ 39,6,6,6,12,17,37,12, 37,6,37,37,6,6,6,6, 6,6,6,6,12,6,6,24, 6,37,12,12,24,12,12,4,
+ 17,5,3,5,5,6,8,8, 8,5,5,11,8,5,1,5, 5,3,3,3,5,3,3,5, 10,5,3,9,5,3,5,11,
+ 5,18,11,11,18,18,3,11, 11,18,11,11,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 17,18,17,11,17,17,11,17, 12,17,17,17,17,18,17,18, 17,18,17,17,17,18,17,17, 17,17,17,1,5,5,5,10,
+ // FBxx
+ 3,37,37,37,37,37,37,14, 37,10,10,10,39,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 10,10,10,10,11,10,10,10, 10,10,10,10,10,10,10,10, 39,10,10,10,10,10,10,11, 10,10,10,10,10,10,10,10,
+ 10,8,11,11,8,18,8,3, 11,18,8,11,11,10,39,37, 11,39,37,37,37,37,37,37, 11,37,11,37,10,10,3,10,
+ 10,12,11,16,16,12,12,16, 12,8,12,16,16,16,16,11, 16,3,16,16,11,37,12,37, 10,39,16,37,39,37,37,37,
+ 37,6,6,6,37,10,11,39, 37,6,37,39,6,6,6,6, 3,37,37,6,6,39,6,6, 6,37,37,11,3,10,37,3,
+ 10,5,10,3,8,8,5,8, 8,3,5,5,5,16,11,8, 3,8,8,5,5,3,3,3, 5,3,5,3,5,3,5,5,
+ 5,18,5,3,3,18,3,5, 3,18,3,18,18,18,18,18, 18,3,18,3,18,18,3,18, 3,18,7,5,5,5,5,5,
+ 5,18,10,10,10,10,10,10, 16,10,10,10,10,10,18,18, 10,10,10,5,18,10,10,10, 10,5,5,11,11,5,5,10,
+ // FCxx
+ 3,10,37,37,37,37,37,37, 1,11,10,37,4,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,11,11,10,10,11, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,11,10,10,
+ 10,33,11,11,11,33,11,11, 11,33,11,11,11,11,11,33, 11,37,11,11,11,33,11,11, 11,11,11,39,10,10,37,11,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,37,11,37,39,39,
+ 37,6,6,6,37,11,11,11, 6,6,6,6,39,6,6,6, 6,6,11,6,10,6,6,6, 39,11,3,11,11,4,39,4,
+ 10,5,5,5,5,5,8,5, 8,8,8,8,5,11,8,5, 5,5,3,5,5,33,3,5, 5,5,5,5,3,5,5,5,
+ 5,5,18,5,18,3,18,18, 18,3,18,18,18,18,18,5, 18,3,18,18,18,5,33,18, 5,3,18,18,5,1,5,1,
+ 10,10,20,10,10,10,18,11, 10,18,10,18,20,10,10,10, 13,10,10,20,18,10,10,20, 10,10,18,18,11,1,13,10,
+ // FDxx
+ 3,37,1,37,37,37,37,37, 37,13,13,37,37,13,4,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 13,13,13,13,13,13,13,13, 13,13,13,13,13,13,13,13, 11,11,11,11,11,11,11,11, 11,11,13,13,13,13,13,13,
+ 13,13,13,13,13,13,13,13, 13,13,1,13,13,13,13,37, 13,37,13,13,13,13,39,37, 1,13,13,13,13,13,37,13,
+ 13,33,1,13,13,33,13,13, 1,33,1,13,13,13,13,13, 13,13,13,13,13,13,1,13, 13,13,13,37,13,13,3,37,
+ 39,37,37,37,37,22,3,3, 3,37,37,37,39,37,39,39, 39,37,22,12,22,11,37,37, 3,37,12,3,3,12,12,4,
+ 33,33,8,3,8,8,5,5, 5,8,8,8,8,5,8,1, 3,5,3,5,13,3,3,3, 3,9,5,3,3,3,9,3,
+ 5,3,3,3,3,18,3,11, 3,18,3,3,5,3,3,3, 13,3,3,3,3,3,3,3, 5,5,11,5,11,1,13,11,
+ 10,18,10,11,10,18,11,13, 12,18,10,10,10,10,18,10, 13,10,10,20,10,10,1,20, 18,20,5,5,5,13,13,10,
+ // FExx
+ 3,39,39,39,37,37,39,39, 37,13,10,37,39,11,39,39, 39,39,39,39,39,37,39,37, 39,39,37,39,37,39,37,39,
+ 10,10,10,1,1,10,13,13, 11,10,11,13,10,10,10,13, 11,11,11,11,11,11,11,11, 11,11,13,13,13,13,13,10,
+ 13,13,1,11,13,13,13,13, 11,13,11,13,13,11,37,37, 11,37,37,11,13,13,39,37, 37,13,11,11,13,11,3,13,
+ 13,13,13,13,16,13,13,13, 13,13,1,13,13,13,16,16, 16,13,1,16,13,13,13,11, 11,13,13,11,13,11,3,37,
+ 3,11,37,37,4,4,4,4, 37,37,39,39,37,11,37,37, 11,37,22,16,39,11,39,37, 39,4,3,37,4,11,37,3,
+ 3,11,1,3,8,8,8,8, 8,8,8,11,8,13,1,1, 8,1,3,18,3,3,3,8, 3,3,10,3,3,3,3,11,
+ 5,18,11,11,3,18,3,11, 11,18,11,11,11,11,11,11, 11,11,11,11,18,18,18,3, 11,11,11,11,11,11,11,11,
+ 10,18,10,10,10,18,10,10, 10,18,10,18,10,20,18,18, 10,10,10,20,18,18,13,10, 18,10,1,18,13,13,11,37,
+ // FFxx
+ 3,37,39,39,39,37,37,39, 37,37,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,37,37,10,10,10, 10,10,10,10,10,10,10,10, 37,36,36,37,36,36,39,39, 39,39,10,10,10,1,10,10,
+ 10,39,39,39,39,37,39,39, 1,39,39,36,39,39,37,39, 39,39,37,36,39,39,36,37, 36,39,36,10,10,10,37,10,
+ 10,2,2,2,36,2,2,2, 2,2,2,36,2,2,36,36, 36,2,2,2,2,2,36,2, 2,2,36,39,10,39,39,39,
+ 36,36,39,37,39,10,39,39, 4,37,36,39,36,36,36,36, 36,37,39,39,10,11,39,39, 37,39,39,39,39,37,39,33,
+ 10,39,10,33,39,39,36,33, 36,1,36,39,33,10,36,36, 39,39,39,18,39,39,39,39, 39,39,10,11,39,39,39,1,
+ 10,1,1,11,39,39,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39, 3,3,3,3,11,3,3,1,
+ 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,39,39,10,37,10,10,
+
+}; // End kMostLikelyEncoding
+
+// End of generated tables
+#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_GENERATED_TABLES_H_
diff --git a/contrib/google-ced/compact_enc_det_generated_tables2.h b/contrib/google-ced/compact_enc_det_generated_tables2.h
new file mode 100644
index 0000000..4ecf966
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_generated_tables2.h
@@ -0,0 +1,856 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/basictypes.h"
+
+static const uint8 ced_hires_0[1024] = {
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 0,135,134,135,89,111,136,129, 81,121,130,40,64,74,78,13, 90,87,103,56,51,37,63,44, 0,97,60,55,143,97,70,100,
+ 29,126,116,149,131,156,108,153, 147,119,135,111,133,135,115,0, 2,13,20,4,12,5,17,5, 4,0,58,50,2,30,80,64,
+ 35,0,0,0,0,22,0,9, 14,12,4,6,7,8,16,3, 147,160,163,141,144,139,146,127, 131,124,23,17,0,11,16,6,
+ 0,68,108,52,129,66,109,65, 113,58,116,124,126,117,88,114, 85,122,102,126,116,122,96,131, 102,126,107,117,49,113,80,128,
+ 0,85,119,107,127,75,102,105, 118,101,108,118,101,117,98,128, 125,114,115,125,115,121,116,121, 125,130,122,119,64,102,78,124,
+ 93,87,64,93,105,92,76,80, 113,82,75,88,89,68,88,97, 87,102,145,90,90,130,106,116, 129,61,81,75,47,69,74,64,
+ 92,98,81,95,81,161,75,81, 101,95,93,75,100,87,108,91, 113,110,118,114,105,105,123,94, 89,92,104,90,62,85,81,84,
+
+ 47,141,155,88,85,108,121,141, 94,73,82,81,163,125,111,89, 106,133,156,95,80,96,88,61, 65,47,74,55,33,54,59,19,
+ 128,129,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,131,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,135,128,128,128, 128,128,161,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 141,115,118,118,118,128,115,127, 136,136,122,124,125,126,128,121, 147,128,128,128,128,128,128,123, 128,121,128,128,107,154,128,124,
+ 82,141,145,136,128,144,115,123, 124,117,107,104,103,99,107,90, 92,98,102,82,88,134,161,126, 115,104,114,106,73,91,111,84,
+ 128,93,94,94,94,116,91,103, 108,106,98,100,101,102,110,97, 137,133,120,132,146,114,136,131, 144,145,155,131,107,151,144,133,
+ 128,103,106,106,106,128,103,115, 120,118,110,112,113,114,122,109, 157,138,128,117,136,123,136,128, 137,123,152,126,104,136,137,134,
+
+ 32,94,46,28,98,102,130,97, 59,91,81,73,26,125,102,48, 100,66,105,57,87,121,71,120, 86,64,64,83,68,105,91,70,
+ 32,130,98,103,64,30,123,122, 97,123,58,127,70,74,55,59, 53,99,95,99,105,52,48,73, 52,68,138,89,29,104,97,142,
+ 20,99,54,114,87,86,108,55, 46,27,42,99,130,118,83,95, 138,115,69,67,96,95,125,86, 82,121,92,145,131,142,115,134,
+ 27,53,57,104,139,102,124,124, 135,81,79,148,118,90,71,25, 151,83,146,71,74,129,109,35, 84,115,121,77,48,24,58,63,
+ 29,55,85,51,70,123,21,95, 116,128,90,107,124,74,98,80, 132,125,112,114,118,109,167,103, 57,127,95,84,13,65,84,109,
+ 27,137,123,69,151,81,84,95, 124,80,115,38,140,164,121,122, 86,116,83,102,117,110,111,92, 72,94,114,144,62,90,84,109,
+ 31,128,63,37,14,138,126,79, 132,63,54,95,60,132,89,132, 25,77,134,94,93,154,147,125, 104,98,113,119,70,130,98,111,
+ 24,78,61,30,100,41,65,115, 80,28,93,97,86,38,74,112, 85,105,135,113,132,45,65,30, 126,91,113,114,73,89,65,130,
+
+ 21,150,114,47,106,125,121,37, 79,158,90,155,73,92,94,107, 87,137,97,135,91,139,152,102, 98,85,126,121,80,140,42,25,
+ 23,47,75,42,64,150,67,93, 100,140,70,62,117,140,54,122, 100,98,95,105,88,100,132,41, 81,137,76,144,60,131,128,66,
+ 23,59,107,144,98,44,20,119, 70,80,58,85,119,93,113,59, 83,28,117,67,135,65,77,60, 130,114,138,111,31,109,92,87,
+ 18,93,79,109,57,128,94,69, 135,54,72,38,53,55,94,56, 133,98,153,147,82,58,124,55, 48,50,153,128,24,54,80,89,
+ 29,130,95,117,49,47,46,91, 164,58,99,148,64,113,86,110, 144,134,49,60,102,62,103,107, 69,89,149,71,113,31,107,71,
+ 25,90,106,113,119,82,75,91, 111,128,139,79,130,101,90,31, 97,103,87,84,53,138,43,87, 155,57,75,150,93,128,120,43,
+ 24,118,82,52,79,56,143,92, 47,103,22,46,70,106,155,129, 93,104,125,42,96,124,137,93, 73,43,134,98,100,111,139,56,
+ 22,65,103,76,38,122,121,155, 111,147,47,76,96,103,153,78, 71,103,137,104,150,89,139,159, 77,120,71,98,98,125,78,143,
+};
+
+static const uint8 ced_hires_1[1024] = {
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 62,124,71,116,117,77,46,71, 78,98,127,132,69,67,72,79, 34,37,88,95,81,88,135,138, 116,123,132,136,97,128,45,101,
+ 68,71,36,0,24,19,3,1, 0,9,60,40,64,146,70,113, 20,16,20,26,13,20,9,12, 14,21,15,19,55,50,99,61,
+ 17,127,133,119,128,134,113,109, 106,124,100,107,119,131,111,122, 138,113,136,154,132,125,120,114, 115,119,101,25,19,31,19,19,
+ 122,112,48,123,115,89,133,132, 124,114,123,130,73,103,139,130, 106,90,103,114,71,97,92,74, 115,112,59,105,83,90,140,121,
+ 118,113,51,133,112,43,123,112, 133,130,102,109,71,110,89,102, 128,125,104,139,115,139,129,134, 112,120,123,117,111,132,129,118,
+ 73,151,165,130,127,146,100,107, 108,121,103,114,156,112,107,111, 125,129,150,135,107,155,118,116, 168,85,79,83,77,89,77,77,
+ 98,101,81,63,71,82,66,64, 60,72,57,62,72,70,60,67, 75,153,138,154,129,161,146,125, 125,139,151,134,139,151,135,146,
+
+ 59,21,36,18,26,37,21,19, 15,27,12,17,27,25,15,22, 30,34,38,44,31,38,27,30, 32,39,33,37,31,43,31,31,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,127,128,124,144,128,127,126, 121,128,118,123,128,128,121,128, 128,128,128,128,128,128,128,128, 128,128,128,128,150,128,128,128,
+ 90,112,109,86,83,124,57,54, 84,70,53,84,62,78,53,75, 124,134,116,95,118,85,129,65, 67,74,68,72,66,78,66,66,
+ 141,134,118,107,135,127,130,134, 118,109,94,99,109,107,97,104, 112,116,120,126,113,120,109,112, 114,121,115,119,113,125,113,113,
+ 161,134,128,112,120,139,115,128, 122,121,106,111,121,119,109,116, 124,128,128,128,125,128,121,124, 126,128,127,128,125,128,125,125,
+
+ 76,29,158,87,47,122,133,89, 63,101,139,116,147,111,79,122, 60,112,93,68,42,145,73,100, 89,123,46,122,124,112,99,90,
+ 118,56,99,61,145,53,131,144, 66,122,110,97,113,82,19,27, 33,134,120,84,79,124,78,108, 138,87,26,133,114,84,47,165,
+ 62,148,104,137,139,97,125,86, 134,47,127,93,141,85,119,111, 126,56,42,26,102,89,122,48, 59,116,110,106,46,115,64,113,
+ 107,81,49,29,28,64,130,22, 111,53,142,89,89,127,131,86, 122,122,37,143,66,96,82,105, 155,96,154,124,63,121,90,60,
+ 115,107,42,30,134,130,119,108, 66,145,122,85,105,19,88,92, 36,134,34,101,97,112,152,73, 150,89,116,144,71,130,131,103,
+ 88,123,77,30,132,53,98,51, 116,61,95,57,84,109,31,74, 20,98,135,114,78,114,95,123, 45,136,116,111,130,94,120,99,
+ 114,121,64,107,94,104,29,77, 152,133,124,119,135,98,0,78, 118,115,88,102,57,61,51,89, 120,51,115,109,64,111,61,66,
+ 110,135,106,110,98,74,86,137, 124,103,134,80,86,43,76,131, 134,105,91,84,73,34,71,133, 60,111,117,53,90,133,106,61,
+
+ 147,38,145,28,137,149,110,112, 52,83,99,128,56,129,105,74, 93,60,45,62,73,86,30,82, 80,44,19,97,85,87,108,103,
+ 99,114,94,104,97,104,140,51, 98,62,119,74,85,141,51,23, 71,58,119,110,151,65,116,74, 130,81,86,90,7,99,38,116,
+ 61,59,140,66,91,133,123,152, 80,82,98,67,112,110,110,89, 121,132,120,62,43,74,45,110, 88,146,113,101,134,135,104,131,
+ 110,47,96,67,123,147,69,70, 131,106,98,109,71,118,116,128, 92,100,149,53,144,117,99,136, 127,128,3,43,114,100,85,117,
+ 92,134,156,8,88,62,89,98, 6,19,46,96,145,108,88,69, 78,94,152,96,146,108,133,66, 75,102,101,10,79,44,57,61,
+ 102,96,123,91,109,149,84,63, 39,134,53,110,111,64,97,103, 155,133,105,105,34,125,93,56, 28,84,65,95,103,58,41,83,
+ 48,25,116,120,91,104,124,84, 111,119,105,64,63,97,116,114, 128,103,136,74,59,33,47,93, 54,80,130,39,139,130,119,83,
+ 140,68,94,85,77,79,118,109, 120,106,114,103,115,165,94,79, 60,52,84,77,82,78,49,74, 88,49,48,76,102,99,138,71,
+};
+
+static const uint8 ced_hires_2[1024] = {
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 44,106,18,108,117,63,65,73, 40,57,43,81,77,63,68,90, 54,16,20,95,85,131,112,100, 58,144,140,135,114,96,110,128,
+ 111,93,68,129,135,62,95,32, 82,42,46,10,8,23,10,16, 26,21,70,26,93,84,168,105, 79,71,29,48,15,26,68,128,
+ 28,126,105,123,109,126,102,109, 105,122,72,104,108,132,113,126, 118,82,122,110,129,125,114,134, 113,133,103,54,21,32,21,128,
+ 113,122,134,118,123,77,97,101, 121,135,128,138,125,120,35,120, 31,24,148,129,0,0,0,0, 0,0,0,5,0,0,0,128,
+ 137,137,112,130,105,127,110,121, 86,140,134,139,128,145,17,121, 19,27,60,150,106,62,95,0, 0,0,0,9,0,0,0,128,
+ 86,81,78,87,82,79,82,75, 75,78,69,74,72,87,74,80, 90,85,84,72,92,96,88,82, 98,102,93,112,79,90,79,128,
+ 162,135,144,153,151,137,127,118, 113,125,105,106,91,139,129,109, 124,140,83,71,91,95,87,81, 97,101,92,111,78,89,78,128,
+
+ 40,35,34,43,36,33,36,29, 29,32,23,28,26,41,28,34, 44,39,38,26,46,50,42,36, 52,56,47,66,33,44,33,128,
+ 128,128,128,131,128,128,130,128, 147,152,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,138,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,143,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 166,128,128,130,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,146,128,128, 128,128,128,128,148,128,128,128,
+ 139,134,133,75,121,105,114,103, 64,68,129,100,62,76,63,69, 79,74,73,114,137,85,77,71, 174,140,82,101,68,79,68,128,
+ 122,117,114,122,118,115,118,113, 112,117,105,110,108,123,110,116, 126,121,120,108,128,128,124,118, 128,128,128,128,115,126,115,128,
+ 128,128,126,154,128,127,132,123, 137,126,117,122,120,128,122,128, 128,128,128,120,128,128,128,128, 128,128,128,128,127,128,127,128,
+
+ 67,115,51,135,110,136,135,45, 132,133,58,84,156,90,77,88, 106,108,86,29,51,132,78,133, 122,67,145,151,77,36,57,128,
+ 142,57,81,110,113,64,18,123, 101,136,98,45,102,79,83,102, 85,86,54,106,94,38,117,74, 115,80,89,151,109,98,137,128,
+ 57,126,70,74,47,54,112,60, 146,46,72,36,108,101,48,40, 135,151,135,128,78,111,70,101, 107,100,25,118,65,57,106,128,
+ 67,74,25,123,123,65,42,95, 135,62,82,86,45,70,104,59, 92,30,63,55,143,71,26,83, 97,107,87,111,25,86,45,128,
+ 65,102,90,116,124,16,45,51, 110,129,118,128,73,120,108,139, 139,102,109,110,120,134,85,66, 104,100,96,129,134,94,84,128,
+ 68,135,60,114,78,126,135,56, 22,121,35,117,101,118,119,120, 109,82,99,96,29,91,118,96, 63,104,18,139,47,66,138,128,
+ 78,148,152,94,129,120,107,96, 141,71,52,46,111,50,97,36, 101,133,87,23,91,150,92,38, 110,103,97,84,79,68,85,128,
+ 118,16,110,124,126,74,99,94, 132,79,88,135,103,84,161,153, 35,41,125,78,122,107,74,90, 93,122,134,117,97,91,38,128,
+
+ 56,121,75,93,68,139,117,78, 23,63,43,75,128,101,116,59, 57,65,121,13,48,108,74,121, 144,120,128,78,95,135,119,128,
+ 141,128,143,68,91,46,137,151, 29,50,32,92,55,73,81,107, 139,145,72,67,54,129,86,35, 105,40,40,99,98,61,142,128,
+ 136,98,109,69,113,126,85,56, 56,103,117,104,17,70,148,128, 55,61,117,55,58,53,121,155, 86,126,75,104,27,105,100,128,
+ 117,110,66,101,135,87,114,88, 69,93,78,106,98,98,117,132, 43,125,54,39,97,119,157,98, 79,131,147,134,59,130,143,128,
+ 60,89,40,119,87,99,73,123, 138,107,135,72,78,97,83,134, 85,110,119,93,43,141,77,98, 119,124,28,104,55,131,118,128,
+ 130,92,71,117,97,53,50,134, 114,136,136,98,51,71,62,115, 96,155,36,1,104,124,59,137, 117,93,26,47,129,107,82,128,
+ 25,44,71,115,56,152,109,40, 117,70,60,119,146,80,83,130, 162,18,126,52,102,137,91,39, 105,113,91,105,37,46,95,128,
+ 68,104,78,51,112,141,87,82, 79,68,59,77,84,49,21,91, 81,68,66,30,149,62,30,70, 81,114,25,107,6,89,62,128,
+};
+
+static const uint8 ced_hires_3[1024] = {
+ 145,108,107,112,108,128,119,120, 123,130,108,110,133,138,120,117, 120,120,128,121,121,116,128,119, 128,119,122,114,121,119,123,124,
+ 77,160,155,161,143,37,29,56, 47,35,127,103,75,130,115,110, 149,149,88,80,89,85,145,135, 102,93,107,99,91,89,140,135,
+ 123,90,70,105,58,80,64,65, 54,64,75,114,51,62,65,58, 91,121,130,110,109,105,111,102, 107,100,93,91,94,94,90,92,
+ 173,130,70,57,104,116,65,65, 135,145,70,76,167,129,117,91, 127,129,135,120,115,112,120,109, 117,108,170,131,84,86,91,132,
+ 62,79,115,64,137,72,131,76, 108,65,120,128,125,126,92,124, 97,123,106,137,119,127,102,145, 120,143,106,114,81,121,87,140,
+ 32,95,125,118,134,80,123,115, 112,107,111,122,99,125,101,137, 136,114,118,135,117,125,121,134, 142,145,120,115,95,109,84,135,
+ 149,90,62,97,105,90,90,83, 100,81,70,84,80,69,84,99, 91,95,141,93,85,127,104,122, 139,91,91,84,114,85,87,82,
+ 128,108,87,106,88,166,96,91, 95,101,96,78,98,95,111,100, 124,110,121,124,107,109,128,107, 106,107,102,86,93,92,87,95,
+
+ 128,128,121,112,126,121,150,106, 125,124,120,96,136,130,135,110, 131,109,148,108,124,113,113,94, 107,139,145,102,114,94,96,95,
+ 92,96,25,73,155,144,172,124, 82,104,144,100,98,115,154,121, 106,60,79,85,106,68,75,81, 150,81,74,87,105,46,52,81,
+ 128,115,75,109,150,190,102,84, 108,126,118,117,98,102,82,70, 101,160,145,120,104,121,112,110, 124,103,124,102,101,99,97,98,
+ 128,121,92,125,93,113,104,105, 94,104,93,99,93,102,105,98, 129,127,166,129,127,135,139,136, 142,143,142,131,131,127,123,126,
+ 128,115,81,140,82,102,107,94, 83,93,83,84,80,91,94,87, 131,132,152,131,139,139,150,144, 142,131,134,133,139,135,137,136,
+ 128,144,99,131,100,120,124,112, 101,111,100,102,98,109,112,105, 126,133,137,137,132,146,148,131, 144,153,139,142,147,123,115,120,
+ 128,132,96,135,97,117,108,109, 98,108,97,99,95,106,109,108, 133,126,134,126,133,142,142,127, 140,133,144,121,134,129,135,117,
+ 128,137,98,138,99,119,110,111, 100,110,99,102,97,112,111,111, 133,122,147,135,132,142,126,127, 144,133,138,140,129,124,135,131,
+
+ 66,120,131,103,77,97,82,96, 49,122,50,79,108,111,157,61, 103,104,175,93,150,114,129,97, 163,62,107,74,85,83,86,65,
+ 60,104,55,155,93,106,145,111, 154,124,81,79,114,119,104,98, 78,156,105,125,122,58,126,51, 158,67,68,140,110,97,170,83,
+ 57,133,137,102,68,149,108,61, 123,146,85,82,59,65,109,45, 93,71,73,77,93,78,111,93, 84,127,90,164,148,138,91,161,
+ 61,151,94,146,151,131,111,141, 77,115,111,98,134,96,86,120, 72,120,108,84,108,154,91,107, 59,119,71,81,97,99,110,111,
+ 61,123,117,54,44,130,154,70, 125,120,23,146,110,91,116,88, 138,83,134,93,150,98,76,53, 90,99,140,60,102,90,84,118,
+ 48,50,50,114,97,154,78,31, 88,101,72,126,31,104,99,105, 108,142,96,135,96,117,112,62, 90,103,116,59,144,151,100,63,
+ 60,111,72,74,81,149,104,72, 154,140,90,149,106,117,107,164, 94,68,40,103,113,75,95,119, 78,119,86,84,153,133,135,61,
+ 57,55,162,113,44,109,106,106, 156,45,51,86,91,122,91,49, 59,115,122,101,140,123,148,88, 128,112,113,99,106,160,94,146,
+
+ 60,103,65,134,55,108,82,107, 109,74,62,66,51,60,154,112, 107,132,118,98,153,90,145,96, 127,103,138,131,32,128,125,71,
+ 83,37,62,74,156,128,140,85, 67,149,49,158,98,78,96,58, 74,105,154,86,85,104,70,106, 71,139,144,112,70,87,62,57,
+ 61,52,121,154,38,91,135,69, 50,69,82,123,119,105,113,111, 94,87,99,91,73,100,97,82, 110,100,136,69,126,142,55,51,
+ 74,69,108,46,132,141,110,157, 133,69,143,44,100,133,144,154, 148,71,78,118,106,110,142,141, 77,145,112,124,118,87,69,36,
+ 55,94,65,106,116,57,117,82, 111,82,114,127,58,134,115,141, 151,122,124,60,133,57,158,91, 139,85,137,75,157,84,128,56,
+ 59,132,91,110,40,123,69,72, 156,84,95,148,88,151,121,95, 59,132,143,80,88,128,94,95, 76,120,121,149,98,86,94,91,
+ 63,124,105,95,61,107,107,129, 65,159,114,147,62,170,118,134, 140,104,135,145,113,123,129,67, 107,117,135,114,64,66,85,147,
+ 65,101,84,63,70,103,113,106, 138,80,159,36,114,114,86,133, 93,102,95,90,159,137,73,65, 75,126,75,50,154,87,101,119,
+};
+
+static const uint8 ced_hires_4[1024] = {
+ 125,128,119,118,134,121,118,117, 121,123,122,116,128,124,115,120, 123,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,133,
+ 89,125,49,76,12,21,28,18, 21,36,39,85,85,24,34,27, 45,91,39,57,98,41,58,61, 34,65,57,69,88,91,63,51,
+ 88,107,83,89,76,130,128,123, 123,120,114,104,121,106,102,107, 111,122,121,124,104,124,105,123, 75,193,173,183,166,161,151,140,
+ 57,123,96,99,90,98,90,87, 92,100,90,86,102,110,91,100, 111,104,117,127,115,104,104,107, 93,107,76,126,75,129,66,99,
+ 132,140,43,135,109,85,142,143, 142,123,145,142,88,114,151,140, 111,113,116,119,86,108,108,99, 125,132,66,121,102,103,162,139,
+ 127,140,45,144,105,38,131,121, 150,137,124,119,85,120,100,111, 132,147,116,143,129,149,144,158, 121,139,129,132,129,144,150,135,
+ 118,171,152,134,113,134,101,109, 118,121,117,117,163,115,111,113, 122,144,155,132,114,158,126,133, 170,108,78,105,109,94,91,87,
+ 107,128,102,85,100,86,92,85, 99,107,95,79,99,102,94,112, 103,175,150,158,143,171,161,149, 134,158,157,149,157,163,156,163,
+
+ 109,134,92,99,79,88,99,96, 113,116,109,96,117,109,94,107, 109,117,140,123,112,109,118,129, 82,99,110,93,90,120,93,105,
+ 174,130,49,65,96,52,55,98, 100,54,66,43,94,53,51,88, 117,70,89,96,67,74,69,96, 111,61,42,98,52,58,55,82,
+ 104,127,95,100,95,101,95,103, 111,102,126,112,114,120,101,120, 111,128,128,120,116,125,119,128, 87,104,85,98,95,101,98,94,
+ 127,145,120,137,126,116,135,123, 128,134,149,145,134,137,137,135, 138,172,148,150,144,142,146,155, 116,128,113,126,123,128,126,122,
+ 127,149,116,132,127,126,132,129, 142,132,146,137,136,137,141,133, 138,152,166,158,154,155,150,158, 104,121,102,120,112,119,115,111,
+ 147,147,113,127,121,115,115,109, 131,129,132,124,135,125,120,129, 126,150,153,163,145,131,145,157, 122,128,120,128,128,128,128,128,
+ 132,146,128,118,123,118,136,128, 140,142,131,133,128,133,123,134, 135,144,155,144,129,141,146,152, 119,128,117,129,127,128,128,126,
+ 136,161,124,119,112,127,127,117, 125,135,136,116,159,139,127,131, 135,144,151,151,142,140,146,158, 121,128,119,128,128,128,128,128,
+
+ 56,117,131,52,109,89,49,75, 81,137,58,134,87,143,104,64, 67,158,68,113,116,101,108,155, 107,161,107,92,127,123,72,117,
+ 86,78,40,98,54,51,93,94, 146,77,132,88,105,47,86,121, 62,41,132,68,70,140,41,104, 135,103,110,99,123,62,88,138,
+ 101,108,91,108,121,139,137,42, 81,139,135,123,107,124,135,85, 101,65,99,90,94,85,106,115, 139,139,64,101,103,134,95,143,
+ 70,126,127,75,75,101,140,147, 87,164,107,104,165,99,81,128, 127,71,73,92,129,93,149,99, 116,117,63,116,99,109,92,116,
+ 76,80,61,56,75,111,73,103, 101,107,139,145,119,80,146,97, 80,85,92,153,99,90,106,97, 63,145,46,83,77,87,100,104,
+ 146,115,124,137,168,42,104,127, 143,56,58,87,91,127,81,104, 84,80,65,78,61,37,98,132, 153,100,149,107,103,115,86,77,
+ 130,158,89,75,83,106,85,85, 154,103,67,119,146,96,127,133, 57,111,108,142,164,78,117,83, 96,129,56,73,114,66,45,67,
+ 144,99,131,129,98,141,102,131, 94,130,123,75,85,40,91,109, 73,161,102,81,53,102,161,124, 69,88,72,127,111,144,48,90,
+
+ 63,71,74,140,135,125,89,112, 80,139,117,97,84,70,104,105, 134,91,113,99,91,128,126,100, 80,119,115,145,82,39,81,166,
+ 114,92,92,84,94,122,49,104, 105,137,124,110,127,85,72,101, 21,83,117,52,78,79,102,69, 159,137,104,143,160,134,95,104,
+ 121,128,79,153,100,154,100,103, 107,108,78,130,91,172,136,158, 103,65,53,145,51,100,86,107, 108,103,136,103,149,108,105,77,
+ 79,64,42,108,78,77,129,65, 55,77,109,88,67,93,68,52, 53,94,114,121,109,121,106,60, 146,113,83,116,96,136,58,100,
+ 84,132,50,139,86,86,148,141, 107,91,140,68,130,131,107,42, 102,135,166,160,54,62,91,126, 79,136,52,161,124,123,100,71,
+ 70,99,52,78,44,116,54,121, 67,112,82,59,164,83,92,133, 108,81,120,156,91,84,129,131, 107,110,139,91,122,128,82,92,
+ 118,77,72,119,66,133,131,40, 125,123,54,65,72,159,54,65, 84,86,68,142,68,101,155,79, 102,155,56,134,117,163,131,155,
+ 66,82,99,78,48,100,154,96, 81,163,89,132,131,155,126,110, 54,98,82,94,51,152,112,120, 138,77,149,127,74,99,109,80,
+};
+
+static const uint8 ced_hires_5[1024] = {
+ 128,127,124,128,128,147,138,139, 162,124,125,126,127,127,152,184, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 59,56,52,84,74,62,105,50, 75,46,54,59,89,56,113,130, 119,132,110,115,136,127,123,76, 85,121,121,94,112,104,106,128,
+ 126,117,109,74,77,123,124,119, 122,105,96,86,86,78,87,79, 94,169,180,158,166,149,129,116, 113,117,109,100,100,85,105,128,
+ 59,98,77,92,84,100,75,88, 93,94,54,78,89,98,98,106, 103,65,103,95,118,105,93,113, 76,102,66,54,135,52,60,128,
+ 114,126,138,119,130,83,102,112, 141,139,142,144,138,118,52,132, 48,39,161,146,26,1,0,0, 0,6,0,43,34,8,25,128,
+ 137,140,115,130,111,132,114,131, 105,143,147,144,140,142,33,132, 35,41,72,166,126,73,105,0, 0,0,0,53,3,50,23,128,
+ 112,128,104,79,120,147,178,100, 105,85,90,74,80,77,120,87, 130,112,180,83,120,100,91,85, 85,95,84,102,99,90,92,128,
+ 162,138,147,153,157,142,131,128, 132,128,118,111,103,136,145,120, 140,154,95,87,111,106,97,91, 91,101,86,97,105,96,98,128,
+
+ 96,79,91,123,95,99,109,128, 124,80,77,74,81,79,85,86, 101,94,91,83,107,102,93,87, 87,97,91,93,101,92,94,128,
+ 130,76,66,68,64,65,77,111, 81,61,51,63,45,76,57,136, 63,56,53,46,77,64,62,49, 49,59,123,97,63,54,56,128,
+ 86,86,81,86,91,131,96,85, 96,81,82,79,84,117,90,91, 106,99,96,88,112,107,98,92, 95,102,87,98,113,97,99,128,
+ 114,112,109,114,116,119,125,113, 127,111,110,121,112,112,118,119, 128,127,124,116,128,128,126,120, 120,128,115,126,140,125,127,128,
+ 103,101,98,103,105,118,105,102, 113,98,99,96,101,101,107,108, 147,116,113,105,128,124,115,109, 109,119,104,115,136,114,116,128,
+ 121,119,116,121,123,142,128,128, 128,116,117,114,119,119,125,126, 128,128,128,123,128,128,128,127, 127,128,122,128,128,128,128,128,
+ 118,116,113,118,120,127,150,122, 139,118,114,111,116,116,122,123, 128,128,128,120,128,128,128,124, 124,128,119,128,128,128,128,128,
+ 127,118,115,120,139,136,123,124, 134,127,116,113,118,118,124,125, 128,128,128,122,128,128,128,126, 126,128,121,128,128,128,128,128,
+
+ 144,113,79,134,119,150,168,110, 109,123,87,135,164,70,123,153, 60,128,94,93,119,96,90,82, 80,91,52,120,157,65,95,128,
+ 152,77,95,138,152,63,122,108, 72,111,161,80,53,163,71,71, 154,47,85,88,86,124,118,33, 115,126,72,98,91,95,82,128,
+ 128,146,140,137,59,123,98,75, 133,154,41,39,133,67,133,64, 115,98,30,37,84,76,58,26, 78,73,153,84,82,120,63,128,
+ 106,93,89,71,142,126,122,109, 127,113,101,74,47,71,102,125, 99,87,54,87,115,144,165,81, 99,49,47,81,86,139,114,128,
+ 95,47,91,59,105,149,149,123, 89,57,69,124,88,133,119,156, 141,64,149,165,107,86,69,108, 135,57,146,122,119,128,82,128,
+ 44,75,63,157,128,38,92,158, 41,77,129,36,81,96,88,54, 44,93,58,46,115,99,80,131, 96,65,76,87,70,72,83,128,
+ 165,111,57,90,104,46,83,56, 72,75,68,75,96,118,139,55, 81,118,70,56,99,104,83,106, 143,142,105,139,81,83,155,128,
+ 86,127,119,93,103,124,116,150, 118,69,88,93,98,72,109,119, 122,130,119,64,126,86,111,57, 87,131,74,125,103,69,156,128,
+
+ 98,68,93,126,62,132,161,123, 149,72,93,84,65,65,112,144, 101,154,60,90,129,83,167,144, 140,140,113,105,164,79,61,128,
+ 112,113,139,157,96,91,153,68, 106,113,83,60,100,103,74,71, 111,102,68,110,41,73,115,67, 93,107,158,143,83,149,120,128,
+ 95,117,116,117,99,81,87,114, 115,118,121,91,142,90,110,101, 82,114,128,148,135,125,105,70, 80,91,110,89,129,99,140,128,
+ 85,169,69,133,73,49,110,60, 96,123,99,77,114,72,157,121, 140,133,145,94,106,142,90,139, 96,154,153,76,95,136,38,128,
+ 144,125,98,54,162,77,113,95, 108,88,42,67,143,69,97,30, 68,71,136,64,127,135,138,72, 107,115,59,137,148,97,158,128,
+ 122,156,149,122,108,71,79,147, 128,144,38,68,131,98,99,121, 164,143,123,52,140,126,130,91, 154,67,95,124,146,43,91,128,
+ 124,62,79,123,127,91,91,135, 121,69,95,76,76,128,74,47, 52,71,100,47,105,141,145,88, 123,140,106,86,152,110,91,128,
+ 123,121,138,108,94,76,62,115, 57,128,70,49,149,129,153,78, 71,120,117,115,74,80,143,95, 64,35,73,80,92,79,49,128,
+};
+
+static const uint8 ced_hires_6[1024] = {
+ 131,89,116,77,83,97,102,136, 133,105,111,105,87,87,87,105, 108,118,117,124,140,105,110,104, 68,96,93,87,81,101,115,102,
+ 17,136,152,147,132,68,100,48, 56,48,81,53,20,72,87,102, 143,155,79,74,108,84,137,154, 76,94,89,93,83,102,92,87,
+ 66,79,73,41,49,115,104,62, 46,89,89,70,100,49,89,121, 91,76,85,75,125,76,65,130, 119,156,144,96,123,158,156,118,
+ 20,106,67,42,24,96,44,57, 144,132,70,77,156,109,89,83, 121,135,126,114,134,111,112,128, 91,109,152,125,78,102,101,136,
+ 43,96,88,68,104,16,25,125, 72,107,35,59,22,13,0,14, 89,145,128,73,85,131,89,165, 108,108,98,155,106,124,167,120,
+ 128,81,99,82,73,82,85,63, 69,94,110,62,46,48,43,56, 155,167,150,139,160,127,125,138, 97,126,85,77,73,94,89,89,
+ 67,136,152,100,83,91,124,114, 106,89,92,124,146,131,103,110, 120,137,118,86,122,92,116,123, 90,103,95,79,104,103,104,97,
+ 122,74,119,104,124,104,96,98, 90,118,93,81,76,100,118,130, 143,123,120,167,152,126,138,151, 125,120,83,101,102,118,116,108,
+
+ 79,88,103,83,60,34,70,38, 92,77,94,49,116,95,82,88, 43,144,139,131,133,98,98,121, 86,94,95,101,96,80,109,97,
+ 88,98,105,102,82,110,51,104, 115,100,128,93,119,80,46,69, 110,117,74,90,120,121,93,128, 119,85,83,79,94,103,104,110,
+ 89,75,115,84,121,92,114,102, 122,90,122,128,106,108,84,118, 107,133,113,126,141,123,115,158, 103,135,99,112,107,138,121,141,
+ 94,79,115,97,115,104,113,108, 122,102,110,119,95,109,90,121, 119,127,118,122,154,126,117,158, 112,137,104,112,113,135,125,129,
+ 121,83,109,77,84,83,62,98, 104,84,113,116,93,81,73,85, 118,113,99,96,123,96,108,131, 95,111,79,75,101,113,113,103,
+ 140,66,86,79,79,90,80,94, 100,88,90,93,77,79,74,87, 118,152,103,106,142,108,110,143, 111,131,122,106,115,147,126,135,
+ 166,67,85,78,78,89,79,93, 99,87,89,92,76,78,73,86, 130,127,102,124,155,105,121,151, 123,149,123,109,120,162,148,136,
+ 154,75,95,88,88,99,89,103, 109,97,99,102,86,88,83,96, 149,130,112,108,140,112,120,145, 114,125,119,102,115,145,139,135,
+
+ 31,150,134,141,48,120,36,27, 151,116,122,82,89,136,91,28, 119,68,44,150,145,40,107,55, 36,92,23,66,46,41,24,59,
+ 41,45,12,0,141,58,83,30, 9,9,61,106,0,45,9,16, 14,23,66,152,39,5,42,56, 144,155,123,64,151,84,75,33,
+ 107,62,98,81,89,126,87,70, 167,111,120,112,104,100,108,129, 72,124,176,74,96,72,84,77, 115,63,29,72,108,97,111,154,
+ 85,144,123,65,80,95,99,66, 65,55,176,121,104,135,27,146, 62,44,158,114,101,106,105,126, 60,80,72,160,55,126,113,101,
+ 36,0,87,20,72,44,8,0, 67,127,87,121,10,84,77,51, 34,21,68,32,57,64,38,24, 9,14,132,0,17,87,36,41,
+ 67,104,95,43,73,154,82,100, 126,72,76,68,26,71,58,32, 24,45,36,21,55,166,135,135, 134,126,7,36,81,101,84,170,
+ 75,82,59,49,43,129,38,169, 67,116,63,94,69,64,52,101, 133,117,129,120,66,52,28,69, 66,82,50,119,104,83,74,71,
+ 47,131,113,126,57,142,116,109, 99,107,76,126,9,6,123,152, 96,136,103,126,142,67,90,109, 102,148,97,101,76,87,83,95,
+
+ 49,8,75,129,53,138,158,150, 98,83,101,27,3,66,159,125, 128,114,158,134,119,126,153,139, 137,128,35,126,84,61,108,84,
+ 34,0,18,21,24,22,93,33, 38,1,18,168,88,79,156,105, 149,47,24,56,50,69,76,64, 20,102,71,68,46,62,48,58,
+ 30,53,84,128,83,135,48,120, 32,36,94,29,44,92,100,149, 138,79,46,24,153,61,30,49, 158,143,56,138,115,118,110,93,
+ 74,93,20,31,73,44,42,111, 52,105,68,74,46,43,29,66, 45,67,37,7,76,88,82,101, 52,70,30,45,51,78,66,100,
+ 54,32,43,64,72,117,82,71, 69,29,37,29,4,167,126,75, 26,161,51,143,53,30,104,121, 74,79,144,27,157,129,153,117,
+ 55,53,53,87,24,19,80,75, 68,63,40,42,111,20,59,68, 72,114,50,26,138,29,70,86, 43,51,156,34,99,130,66,103,
+ 53,0,46,31,71,79,87,57, 59,51,34,39,24,0,25,66, 1,47,140,45,144,92,34,50, 90,75,50,2,24,40,147,114,
+ 63,150,105,113,95,108,52,72, 62,157,153,73,141,125,32,38, 104,104,25,64,137,147,31,105, 16,139,27,46,1,90,101,23,
+};
+
+static const uint8 ced_hires_7[1024] = {
+ 120,127,110,107,125,119,124,105, 127,128,122,128,102,123,102,99, 129,154,113,122,107,119,106,109, 116,98,104,108,120,117,116,111,
+ 91,83,50,46,97,86,122,78, 99,143,61,80,78,80,62,52, 17,15,56,62,51,91,64,93, 138,118,150,133,136,119,114,121,
+ 173,162,166,159,148,141,205,123, 204,166,143,146,120,144,133,153, 134,173,138,140,103,106,108,117, 108,108,71,103,148,151,114,164,
+ 86,147,121,122,141,130,130,109, 132,142,124,133,108,143,112,113, 130,102,125,146,122,120,113,107, 113,92,86,127,123,136,75,107,
+ 137,141,85,124,139,117,115,121, 96,81,80,124,153,77,61,54, 173,171,86,199,83,61,48,51, 58,40,46,50,62,59,58,53,
+ 145,149,144,119,151,127,147,125, 167,161,116,130,99,120,128,128, 128,103,131,112,106,147,108,130, 151,83,89,93,105,102,101,96,
+ 129,134,99,103,101,147,132,118, 111,96,87,94,100,88,67,64, 93,74,78,87,72,84,86,74, 81,63,69,73,85,82,81,76,
+ 125,123,130,99,124,109,144,109, 111,115,106,113,93,107,93,83, 92,101,134,122,117,122,132,121, 100,122,88,92,104,101,112,95,
+
+ 117,129,112,84,103,101,127,90, 87,91,82,100,75,179,112,185, 110,132,107,115,106,185,103,103, 136,103,135,134,129,86,137,125,
+ 139,147,101,95,125,132,138,134, 114,118,109,116,89,113,94,123, 98,96,100,109,94,106,93,96, 103,85,91,95,107,104,103,98,
+ 159,165,129,148,166,141,172,150, 179,163,171,181,122,149,165,145, 138,128,145,149,124,146,132,127, 143,122,121,133,144,135,162,148,
+ 171,165,133,151,163,146,162,148, 172,168,156,165,126,154,137,132, 143,140,143,155,135,156,140,142, 142,127,130,137,149,145,153,146,
+ 132,133,96,93,127,105,110,92, 113,117,108,115,88,109,88,85, 94,169,171,176,140,166,165,125, 134,143,158,156,158,166,150,158,
+ 169,154,126,147,134,128,175,148, 138,128,128,128,118,128,118,115, 124,125,128,128,123,128,122,125, 128,114,120,124,128,128,128,127,
+ 160,159,125,129,155,136,156,153, 149,128,128,128,117,128,117,114, 123,124,128,128,122,128,121,124, 128,113,119,123,128,128,128,126,
+ 168,147,128,128,128,139,128,143, 141,128,128,128,127,128,127,124, 128,128,128,128,128,128,128,128, 128,123,128,128,128,128,128,128,
+
+ 26,71,23,0,18,145,93,129, 88,129,0,160,78,145,20,29, 21,64,41,28,155,90,68,73, 46,65,128,9,130,142,83,131,
+ 59,84,81,102,49,55,8,132, 69,48,0,80,0,123,0,36, 37,33,0,62,129,121,64,162, 123,122,0,154,69,160,143,89,
+ 155,85,24,119,133,127,86,100, 126,180,98,119,16,37,81,98, 82,71,113,46,31,50,30,33, 70,148,98,135,141,102,162,99,
+ 94,74,90,98,122,40,42,20, 72,125,152,93,42,97,112,5, 103,130,75,82,21,65,120,149, 107,58,80,48,61,150,41,63,
+ 118,82,161,115,70,0,55,0, 13,151,100,56,59,0,25,151, 102,102,94,0,133,95,68,98, 35,166,95,80,163,108,138,85,
+ 47,27,46,3,18,108,113,138, 156,140,109,142,22,49,123,83, 98,107,78,74,32,109,58,60, 83,0,109,0,1,0,105,55,
+ 71,49,44,31,49,30,38,141, 120,37,96,114,14,14,14,21, 22,111,109,31,70,103,7,100, 17,109,62,65,72,49,18,128,
+ 40,154,151,137,126,105,104,61, 119,144,127,42,0,0,164,141, 135,126,95,119,107,112,29,0, 0,0,99,11,15,13,9,0,
+
+ 108,133,116,95,53,149,107,100, 56,66,115,79,15,100,65,0, 1,0,7,140,111,132,101,0, 95,4,92,92,46,22,161,72,
+ 58,38,113,14,88,36,11,97, 67,63,27,39,143,108,133,82, 135,0,52,60,72,93,90,149, 109,147,143,103,45,159,148,153,
+ 132,115,1,88,45,51,72,56, 44,62,5,83,23,28,153,121, 167,37,147,101,0,82,26,76, 97,99,17,1,1,0,40,86,
+ 83,46,30,15,41,21,26,83, 78,66,41,84,84,78,0,41, 50,103,63,122,73,69,0,61, 48,0,97,105,70,96,62,90,
+ 85,84,93,6,54,125,46,123, 102,62,7,71,96,44,54,0, 54,4,166,155,31,144,107,0, 87,55,65,143,79,47,0,5,
+ 163,95,121,171,161,163,63,142, 100,152,141,64,95,125,105,56, 0,89,99,83,64,43,95,89, 83,50,23,0,68,0,0,76,
+ 92,65,42,71,88,88,176,120, 171,105,154,154,18,44,54,106, 112,92,134,139,0,134,133,130, 116,101,84,51,64,46,90,155,
+ 152,122,117,113,19,86,30,51, 85,122,99,90,18,146,45,102, 58,13,43,75,104,119,101,0, 77,0,0,0,146,0,119,0,
+};
+
+static const uint8 ced_hires_8[1024] = {
+ 130,127,155,168,146,193,173,178, 138,176,189,169,199,152,146,172, 128,127,128,122,128,128,109,122, 115,128,126,128,120,128,128,128,
+ 142,160,113,159,112,138,145,87, 133,125,100,110,109,101,70,134, 52,81,111,42,63,48,9,22, 34,44,34,64,45,60,106,128,
+ 98,91,146,105,104,134,59,53, 74,82,71,49,70,51,52,76, 77,69,81,64,94,79,51,64, 57,75,68,77,62,74,106,128,
+ 90,132,91,134,101,123,97,99, 109,125,92,97,124,114,109,140, 123,84,137,120,149,126,95,134, 90,124,96,82,140,78,116,128,
+ 68,69,88,133,99,128,124,111, 101,105,137,125,134,111,84,138, 137,75,81,64,111,89,60,64, 57,75,68,77,62,215,110,128,
+ 111,181,158,179,148,178,161,155, 155,149,184,172,200,153,139,187, 156,138,159,137,153,154,108,115, 129,118,111,120,105,117,128,128,
+ 91,101,111,146,104,81,82,70, 84,86,94,72,93,74,75,99, 100,92,104,87,117,102,74,87, 80,98,91,100,85,97,128,128,
+ 137,111,137,155,136,166,170,156, 138,140,168,155,174,149,139,169, 119,111,123,106,128,121,93,106, 99,117,110,119,104,116,128,128,
+
+ 127,121,128,160,166,147,149,173, 185,181,184,160,174,149,143,166, 164,138,146,126,151,136,178,115, 117,182,167,95,80,92,124,128,
+ 113,114,134,159,120,170,172,178, 160,151,180,174,185,158,134,167, 122,114,126,109,128,124,181,188, 193,130,119,145,158,155,152,128,
+ 155,164,146,164,148,177,155,147, 159,166,174,167,174,147,135,174, 105,97,186,168,122,107,79,92, 85,103,96,105,90,102,128,128,
+ 161,164,146,176,146,180,170,159, 156,165,177,172,179,155,145,171, 111,103,162,180,176,113,104,98, 91,109,102,111,96,108,128,128,
+ 187,159,158,190,172,180,181,159, 155,156,183,160,177,165,170,171, 148,172,125,108,128,141,95,108, 101,119,112,121,126,118,128,128,
+ 128,128,120,133,128,128,128,121, 128,128,128,123,128,125,126,128, 128,128,128,128,128,128,125,128, 166,128,128,128,131,128,128,128,
+ 128,128,119,128,128,128,128,122, 129,131,128,122,128,124,125,128, 128,128,128,128,128,128,124,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,154,128,128,133,128, 142,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+
+ 20,141,64,92,31,78,151,53, 145,20,53,48,65,154,102,114, 93,115,27,0,49,93,58,115, 144,65,153,88,149,88,17,128,
+ 84,40,161,60,108,42,125,33, 139,68,83,54,43,81,132,82, 69,84,108,0,122,85,6,2, 53,70,102,0,36,34,41,128,
+ 100,104,66,145,69,110,103,89, 76,65,98,98,84,63,121,133, 106,95,152,44,168,45,122,127, 129,38,117,149,53,137,128,128,
+ 134,104,149,56,114,62,74,104, 71,45,50,137,132,122,110,44, 112,56,54,136,150,128,59,0, 0,81,103,3,0,0,33,128,
+ 81,17,22,151,126,101,49,126, 22,36,76,149,81,76,40,83, 64,111,53,68,65,145,96,0, 110,0,87,0,39,98,99,128,
+ 104,20,78,59,26,158,114,116, 105,158,62,77,97,99,149,60, 158,104,111,51,111,77,41,70, 35,124,61,140,87,83,139,128,
+ 89,103,71,128,99,85,122,108, 62,50,67,61,120,64,44,79, 43,9,107,179,176,163,107,154, 141,87,110,163,2,37,115,128,
+ 0,162,20,49,20,47,97,121, 95,83,96,88,115,58,47,97, 61,11,73,19,29,17,0,0, 0,148,82,94,111,89,40,128,
+
+ 120,92,69,58,56,38,49,120, 38,153,95,56,69,140,92,44, 157,155,88,91,145,0,89,50, 109,97,0,66,0,18,97,128,
+ 114,13,57,120,93,77,153,77, 152,135,99,80,88,24,69,60, 107,53,43,2,13,0,131,78, 141,49,121,0,119,153,135,128,
+ 0,4,6,55,124,56,65,29, 21,40,149,41,100,137,17,39, 3,163,111,111,140,0,52,59, 72,113,76,118,116,81,92,128,
+ 47,32,26,62,0,51,45,166, 142,38,168,49,140,20,96,145, 124,74,90,169,73,151,143,89, 64,116,26,75,20,162,143,128,
+ 0,0,91,31,9,39,39,26, 90,29,42,35,53,27,112,56, 58,67,4,76,79,51,153,115, 122,50,137,120,19,114,123,128,
+ 78,120,61,100,65,59,85,52, 72,57,93,54,61,10,72,78, 56,75,77,0,33,34,43,38, 0,0,14,44,0,44,24,128,
+ 151,95,67,85,59,72,70,135, 53,60,105,54,54,25,160,139, 142,61,116,126,90,1,104,159, 139,115,141,100,41,62,100,128,
+ 0,0,16,46,152,103,55,43, 56,47,60,148,157,92,133,148, 0,0,158,50,138,103,111,67, 150,157,47,12,78,21,146,128,
+};
+
+static const uint8 ced_hires_9[1024] = {
+ 130,71,129,141,137,60,133,127, 113,78,80,127,94,127,117,51, 103,62,119,129,148,68,133,101, 38,42,158,125,112,190,163,124,
+ 139,45,127,155,132,105,145,125, 136,78,127,128,145,136,141,61, 122,142,128,129,137,97,140,143, 1,3,145,120,131,129,135,62,
+ 124,125,104,88,80,128,69,72, 68,127,36,103,135,92,101,131, 77,126,132,112,51,143,88,87, 94,151,72,95,120,110,80,161,
+ 81,123,101,98,60,133,79,54, 52,155,28,125,46,72,67,101, 90,70,86,76,45,128,37,119, 86,129,81,86,102,49,41,88,
+ 96,132,92,99,114,136,90,108, 121,130,15,114,132,93,122,129, 135,112,129,111,82,133,131,124, 119,129,105,108,112,39,105,147,
+ 115,68,124,132,149,103,109,139, 127,72,144,126,137,143,146,76, 122,99,138,142,133,69,134,126, 32,11,131,145,107,148,126,61,
+ 77,126,72,78,49,124,127,85, 75,135,43,82,121,79,83,143, 74,43,129,98,111,130,44,45, 87,106,101,92,138,51,52,89,
+ 95,126,67,51,135,116,72,112, 51,131,24,106,133,97,111,143, 71,33,136,82,68,135,62,78, 22,49,63,84,114,36,104,82,
+
+ 46,129,63,71,69,93,73,121, 119,140,58,72,114,107,136,142, 96,57,126,112,97,126,48,117, 55,54,76,101,134,53,81,140,
+ 139,93,114,144,122,123,128,130, 146,120,146,137,129,138,134,101, 109,151,120,136,133,78,129,131, 0,6,154,131,126,122,151,58,
+ 103,57,111,129,142,74,122,99, 101,56,40,136,131,124,138,107, 93,87,91,150,153,48,74,96, 29,52,125,155,90,119,156,95,
+ 71,137,72,129,58,106,76,87, 65,135,10,79,127,82,98,139, 95,29,127,121,126,138,126,104, 58,30,84,95,111,27,55,91,
+ 148,128,84,57,100,136,84,113, 74,138,28,114,116,81,111,129, 86,138,17,107,80,134,130,64, 160,116,80,94,104,93,101,84,
+ 76,131,112,98,55,135,114,120, 76,132,40,106,110,135,121,131, 146,114,77,99,59,136,66,90, 106,140,89,78,129,75,105,92,
+ 119,136,75,128,125,129,141,113, 81,139,14,124,57,64,124,132, 74,124,75,119,117,122,73,70, 119,145,101,93,142,120,105,136,
+ 111,62,156,130,141,107,139,152, 124,103,148,125,129,141,131,71, 138,116,130,137,129,82,139,148, 0,0,125,133,142,126,134,55,
+
+ 72,116,48,77,48,122,93,65, 57,115,22,108,127,51,83,142, 87,100,151,86,87,131,39,43, 86,113,70,90,98,53,85,72,
+ 149,20,138,146,143,112,62,118, 158,65,125,130,127,150,129,33, 114,116,113,124,140,58,141,141, 24,20,165,129,84,163,122,98,
+ 106,140,115,104,111,132,132,124, 123,125,23,112,87,134,120,128, 118,121,93,124,114,141,143,116, 117,135,106,137,107,90,103,86,
+ 104,114,115,105,103,116,131,80, 117,119,30,147,132,120,108,122, 147,136,106,132,147,127,81,123, 134,114,71,127,123,66,120,147,
+ 106,131,95,77,65,135,100,63, 69,133,8,113,86,81,116,133, 74,114,131,122,67,128,53,135, 151,132,60,61,111,92,110,88,
+ 161,88,152,116,143,105,115,146, 136,61,114,137,128,126,120,44, 151,90,132,138,125,54,155,126, 8,28,138,144,121,160,160,75,
+ 87,124,98,82,154,145,88,92, 46,138,32,132,81,107,138,90, 81,34,84,103,27,129,104,81, 109,61,73,49,116,47,121,91,
+ 72,136,72,95,97,132,49,95, 111,125,35,117,129,103,124,133, 112,95,117,134,95,115,67,97, 102,148,126,135,108,97,99,123,
+
+ 165,12,134,132,98,115,126,113, 92,122,36,138,34,110,157,63, 107,139,40,148,113,22,79,71, 31,22,127,133,78,127,95,90,
+ 38,12,112,64,133,138,43,108, 175,16,168,125,147,148,91,8, 100,54,87,106,121,17,106,130, 17,47,101,161,65,95,131,83,
+ 119,144,110,98,143,110,64,120, 43,121,32,102,111,138,133,123, 86,119,120,80,52,126,109,143, 98,130,90,84,85,38,91,148,
+ 95,124,58,99,38,144,76,50, 77,139,32,143,125,91,120,110, 106,28,108,77,132,126,36,92, 136,57,38,62,101,62,40,90,
+ 47,23,65,73,87,20,162,94, 103,127,100,162,133,118,116,37, 117,60,94,98,151,51,49,98, 24,20,116,121,117,51,42,90,
+ 54,123,33,45,31,151,56,33, 42,145,36,29,25,35,119,81, 32,43,48,22,12,118,44,43, 149,32,38,50,76,65,37,95,
+ 46,130,41,34,22,145,59,49, 59,134,20,125,111,79,130,70, 69,37,81,47,140,127,73,75, 123,43,61,147,80,46,72,78,
+ 115,32,81,74,58,149,102,67, 71,46,65,60,49,57,61,30, 68,174,54,51,45,52,72,53, 57,53,71,79,105,84,67,123,
+};
+
+static const uint8 ced_hires_10[1024] = {
+ 58,122,143,128,136,100,142,138, 75,129,136,141,139,140,49,124, 126,129,133,99,142,134,152,137, 120,138,45,14,25,143,136,140,
+ 125,108,87,90,85,127,88,79, 128,11,95,136,93,104,132,81, 132,106,62,140,86,72,90,83, 121,114,149,146,89,121,124,118,
+ 133,75,100,96,92,129,79,127, 127,44,117,130,100,119,132,110, 114,131,93,115,52,110,92,107, 134,108,137,144,103,119,70,100,
+ 129,70,93,99,140,110,67,56, 133,40,103,132,85,100,142,72, 134,84,77,132,70,53,62,94, 79,31,106,60,43,131,68,32,
+ 132,100,125,106,118,135,136,106, 130,30,113,134,97,123,128,124, 125,111,84,130,80,106,89,109, 111,56,133,127,111,123,95,111,
+ 67,126,128,140,149,99,132,133, 69,144,128,139,143,147,68,121, 136,137,133,67,119,131,136,128, 136,149,79,10,23,96,114,96,
+ 125,102,73,98,152,143,106,80, 137,19,128,84,119,136,89,83, 74,107,89,132,74,58,81,126, 52,49,110,73,108,116,88,42,
+ 142,113,137,119,142,109,104,86, 122,38,105,107,137,130,124,108, 116,84,69,126,79,79,97,100, 101,51,133,133,97,103,115,109,
+
+ 96,116,125,136,124,122,129,150, 119,148,137,131,138,134,97,111, 120,136,131,74,132,148,147,147, 126,125,57,11,16,101,139,150,
+ 64,113,99,105,134,74,84,122, 61,63,130,131,122,139,112,93, 90,147,154,53,127,109,126,145, 147,109,83,59,31,99,68,83,
+ 137,90,110,94,59,103,126,88, 132,21,84,127,93,105,137,97, 124,122,120,138,85,68,128,65, 88,44,132,42,58,106,79,34,
+ 124,85,81,119,108,133,128,91, 138,36,114,116,86,113,128,87, 26,102,87,132,98,77,76,98, 95,93,108,117,161,118,149,141,
+ 131,125,92,102,68,135,80,85, 131,41,104,110,136,120,132,145, 74,100,62,135,119,73,95,106, 84,72,119,134,102,130,74,113,
+ 136,71,72,117,124,128,73,106, 138,23,122,64,52,124,130,76, 75,120,120,120,127,85,126,105, 103,120,93,146,119,110,119,124,
+ 59,156,145,150,141,110,137,123, 104,147,127,129,138,125,77,138, 130,135,130,87,148,123,129,136, 129,131,48,15,26,124,110,114,
+ 115,51,73,57,54,120,36,57, 115,11,101,125,70,76,140,93, 150,94,84,126,85,49,83,103, 85,56,128,111,78,103,71,102,
+
+ 137,112,116,124,112,131,132,116, 126,33,113,86,134,122,127,115, 94,124,117,137,127,116,110,91, 128,86,117,135,114,119,99,120,
+ 116,104,122,83,102,118,81,68, 114,12,145,131,124,108,121,145, 103,127,145,127,115,112,108,119, 140,58,140,108,123,116,98,134,
+ 131,98,133,63,58,136,35,61, 131,29,113,81,78,117,131,74, 128,121,76,127,101,71,79,105, 70,83,126,131,150,127,103,111,
+ 95,145,124,149,145,102,158,144, 73,115,137,128,126,118,39,154, 127,136,124,85,112,136,102,157, 149,155,63,39,21,131,157,85,
+ 121,75,41,91,52,123,59,122, 133,55,87,113,80,75,144,80, 121,95,110,126,123,72,70,79, 101,39,94,99,84,136,68,51,
+ 127,76,124,126,75,95,60,87, 130,36,76,111,110,138,142,109, 125,113,92,127,77,120,87,88, 92,44,116,59,67,146,52,55,
+ 124,94,127,65,59,135,31,83, 154,42,119,47,70,56,99,85, 70,79,69,127,80,57,107,42, 77,38,108,135,113,116,94,95,
+ 134,58,90,56,42,142,76,51, 134,21,125,102,78,135,80,81, 78,60,137,122,61,53,43,84, 147,45,82,70,120,107,56,52,
+
+ 125,58,101,70,50,142,31,51, 142,44,142,128,93,121,116,115, 115,88,119,127,79,67,100,81, 78,68,109,74,131,115,109,34,
+ 128,20,41,24,44,151,46,58, 143,31,53,37,38,115,104,35, 49,13,50,110,40,29,34,32, 52,77,75,38,140,67,53,62,
+ 30,102,98,113,102,138,130,139, 39,47,119,112,103,104,37,128, 128,103,110,33,80,102,106,125, 86,137,98,43,38,91,95,175,
+ 21,105,132,102,123,141,110,109, 31,168,125,139,150,92,18,94, 86,105,122,38,63,172,77,132, 155,93,48,68,35,60,65,79,
+ 28,139,76,117,98,118,79,120, 117,45,140,50,110,154,88,110, 45,147,116,36,127,99,132,98, 137,124,54,34,56,82,164,143,
+ 35,74,103,109,98,33,86,125, 55,113,149,141,121,121,50,122, 113,105,152,60,154,113,84,61, 128,49,72,16,20,144,58,61,
+ 82,126,110,148,135,61,143,155, 65,78,116,108,135,109,46,91, 122,125,145,67,103,116,124,162, 127,185,84,36,46,110,131,52,
+ 42,137,146,121,136,110,144,167, 67,120,128,125,146,130,34,118, 111,122,138,47,72,152,133,119, 127,160,56,29,36,97,143,120,
+};
+
+static const uint8 ced_hires_11[1024] = {
+ 129,46,102,93,87,61,93,55, 96,48,77,76,65,67,91,46, 64,58,113,75,57,62,82,91, 106,70,112,123,68,70,75,108,
+ 113,55,138,144,134,87,155,40, 138,123,135,133,127,142,137,53, 147,126,126,136,132,133,154,124, 123,43,142,128,28,86,28,104,
+ 122,134,145,124,106,123,46,119, 48,128,47,146,45,45,47,127, 41,140,89,43,38,100,46,40, 62,104,61,104,126,110,126,124,
+ 91,125,26,136,96,129,23,115, 19,144,133,118,113,115,63,132, 59,137,94,51,29,117,36,127, 39,136,46,74,118,107,102,110,
+ 89,128,72,40,74,135,21,135, 34,144,57,55,66,55,51,129, 53,127,93,50,32,126,63,29, 37,128,44,72,113,117,128,118,
+ 77,69,127,126,130,77,123,31, 130,136,133,130,123,130,153,82, 138,132,127,134,127,137,125,128, 123,115,114,116,75,59,43,117,
+ 113,116,62,47,39,142,82,137, 41,119,39,88,88,58,47,136, 28,38,72,11,27,108,35,44, 65,155,73,100,130,105,132,111,
+ 93,34,108,139,100,55,67,50, 146,50,129,131,153,145,111,31, 105,138,137,151,137,51,116,127, 146,30,74,76,15,37,10,59,
+
+ 98,133,33,28,17,143,30,136, 41,115,34,100,131,73,38,123, 12,134,69,24,4,134,39,29, 52,118,53,81,110,140,124,88,
+ 82,128,133,117,145,106,126,85, 134,68,159,123,131,130,125,127, 111,117,130,146,125,44,111,141, 127,118,87,70,96,9,66,50,
+ 92,145,86,40,68,123,37,120, 71,114,90,126,74,74,28,128, 73,120,89,54,119,118,68,72, 51,112,35,63,137,116,114,104,
+ 98,123,104,91,72,135,54,139, 112,130,98,141,114,67,63,136, 86,42,87,61,112,117,101,58, 106,134,41,69,123,123,123,117,
+ 85,137,150,37,54,142,42,126, 36,124,60,30,130,98,12,132, 135,24,89,53,49,98,125,57, 119,105,38,66,134,116,118,107,
+ 98,141,79,89,141,133,98,129, 133,130,72,83,48,92,48,131, 38,31,95,107,139,113,59,69, 40,143,37,65,112,132,107,111,
+ 112,118,69,50,46,149,49,152, 39,130,45,42,40,30,66,118, 54,26,75,14,107,112,49,54, 65,116,73,100,115,143,78,91,
+ 91,69,145,144,140,88,104,98, 127,131,122,136,136,132,115,76, 134,134,127,139,121,152,149,142, 144,83,166,167,31,77,39,99,
+
+ 82,130,53,59,64,129,54,110, 44,118,48,133,87,91,27,140, 84,148,91,51,122,101,61,86, 50,134,46,84,126,106,112,112,
+ 84,135,121,143,124,127,87,121, 113,142,112,75,123,109,90,133, 90,98,88,84,123,123,125,129, 92,141,44,70,118,108,114,104,
+ 134,108,127,101,121,121,98,125, 121,115,125,94,128,88,98,112, 124,82,126,112,130,134,126,138, 101,122,105,119,92,85,86,126,
+ 77,117,116,48,78,138,27,142, 122,120,126,63,134,48,25,112, 112,39,86,120,154,143,120,143, 28,123,36,63,109,110,108,101,
+ 71,131,58,45,30,128,122,146, 62,130,49,95,90,25,18,143, 48,127,97,99,74,119,33,43, 22,146,30,57,118,112,106,106,
+ 95,104,131,129,128,72,131,81, 132,75,120,142,143,138,133,92, 155,132,138,149,139,63,128,144, 160,88,119,81,75,7,47,66,
+ 102,128,48,47,57,119,70,107, 140,130,80,106,65,90,32,136, 116,123,82,61,117,132,66,43, 55,154,61,89,127,128,105,122,
+ 110,129,58,13,38,138,30,112, 136,126,30,98,89,135,39,126, 55,147,70,32,117,122,68,87, 58,145,57,84,130,114,103,117,
+
+ 118,115,58,40,45,141,61,156, 51,109,30,30,37,33,59,110, 34,25,81,42,99,147,71,54, 97,128,88,115,120,107,128,103,
+ 102,62,107,133,115,63,64,42, 135,82,88,103,135,156,108,60, 147,137,139,149,144,30,125,110, 68,80,128,101,53,14,38,71,
+ 127,100,145,84,145,70,134,37, 72,39,175,114,120,112,75,120, 76,99,111,119,117,48,91,106, 110,125,102,128,84,56,63,90,
+ 129,77,130,151,138,88,126,88, 117,82,96,136,127,108,124,80, 155,101,115,124,94,99,175,159, 129,107,129,129,106,107,112,114,
+ 98,9,84,76,73,44,76,33, 73,22,61,73,76,66,62,27, 84,63,137,71,74,28,80,84, 106,47,83,88,29,18,15,66,
+ 102,23,80,68,70,48,77,40, 82,24,57,72,77,67,80,46, 70,66,135,70,62,10,75,87, 108,64,76,91,16,23,22,74,
+ 105,19,61,52,70,26,63,47, 80,27,40,55,73,79,73,33, 71,69,127,73,72,12,72,69, 65,55,77,93,18,43,34,72,
+ 104,44,77,65,69,48,79,52, 74,47,58,58,60,52,78,46, 61,55,123,55,55,61,74,74, 89,67,93,106,71,67,69,73,
+};
+
+static const uint8 ced_hires_12[1024] = {
+ 79,129,118,114,118,130,135,136, 122,129,124,105,133,117,122,111, 134,119,65,123,122,86,117,114, 121,123,136,124,126,111,73,121,
+ 134,116,126,128,129,122,119,131, 131,128,98,130,123,67,131,120, 126,125,137,42,120,129,124,126, 132,124,121,124,126,120,89,121,
+ 112,133,109,133,111,132,130,71, 99,133,71,58,135,135,120,135, 124,96,123,110,95,72,54,65, 130,119,105,127,127,103,105,127,
+ 118,122,128,106,129,130,78,100, 80,135,92,115,113,114,117,119, 117,117,148,147,127,77,100,118, 129,121,108,127,127,101,91,127,
+ 135,132,136,125,118,124,139,134, 122,120,34,135,116,109,141,104, 122,131,129,5,132,82,135,130, 120,130,119,128,123,119,103,118,
+ 126,133,130,135,119,105,128,126, 122,109,135,125,129,124,128,144, 130,124,127,128,125,138,129,123, 129,123,141,114,116,100,75,111,
+ 126,110,99,100,152,132,105,74, 73,128,77,140,108,97,135,114, 115,76,102,86,74,55,93,128, 128,65,82,127,127,109,97,127,
+ 83,137,114,142,118,131,144,118, 104,132,68,106,127,101,119,118, 124,123,62,94,123,122,117,132, 123,130,118,127,145,96,99,127,
+
+ 115,129,121,73,117,134,107,125, 107,139,57,129,133,75,111,122, 115,108,124,112,115,65,89,120, 136,89,100,127,127,107,91,162,
+ 116,124,116,129,125,122,126,120, 126,115,136,128,123,153,120,132, 126,124,119,124,121,125,124,127, 124,129,124,115,117,98,87,112,
+ 110,93,120,88,101,100,92,100, 101,103,193,110,109,83,98,97, 81,85,93,129,84,120,83,96, 93,94,86,127,127,127,127,127,
+ 130,129,102,125,118,128,139,116, 89,126,144,112,143,114,130,139, 137,125,99,129,122,105,77,70, 122,122,127,127,144,123,84,127,
+ 140,124,123,126,138,124,114,136, 125,129,136,130,115,106,135,110, 119,120,124,115,130,112,127,128, 111,122,116,121,123,113,80,118,
+ 109,106,107,107,110,111,111,99, 109,127,112,107,106,162,102,138, 105,111,114,99,121,103,86,90, 93,107,100,127,127,170,152,127,
+ 129,122,126,129,130,127,131,138, 125,126,108,134,116,77,116,126, 127,138,131,43,123,110,137,129, 121,138,123,121,123,110,70,118,
+ 108,114,111,100,106,124,104,100, 108,110,114,106,106,96,119,162, 90,141,96,108,93,105,112,107, 100,92,93,127,127,144,144,127,
+
+ 119,102,134,125,132,132,111,116, 139,135,118,123,93,104,109,107, 109,137,126,109,126,113,120,127, 111,127,122,127,127,96,84,127,
+ 108,121,133,121,111,125,73,107, 148,128,119,114,117,108,124,100, 119,106,116,170,155,38,84,144, 123,58,113,127,127,105,99,154,
+ 57,128,109,135,124,126,134,42, 110,124,72,117,140,133,125,114, 124,133,85,97,99,143,140,113, 129,122,120,127,127,99,87,127,
+ 93,88,77,107,97,101,91,104, 98,96,155,103,106,69,77,79, 81,137,97,206,81,184,134,112, 92,116,64,127,127,127,127,127,
+ 118,85,115,94,120,133,99,119, 127,125,107,92,120,53,74,109, 133,143,137,92,89,114,118,130, 145,133,124,127,127,100,88,127,
+ 93,97,96,121,91,124,100,109, 94,125,112,118,104,82,100,96, 123,97,112,138,108,200,107,142, 135,117,94,127,127,127,127,127,
+ 130,129,119,115,125,130,53,116, 116,133,51,40,120,97,122,101, 108,62,141,108,137,124,103,105, 138,40,104,127,127,109,97,127,
+ 111,131,74,122,129,132,85,105, 144,128,63,62,121,83,101,114, 133,139,115,123,125,105,139,99, 136,139,110,127,127,106,92,127,
+
+ 132,119,131,119,128,131,116,120, 132,131,135,136,87,90,112,113, 116,132,115,124,125,154,127,132, 94,130,122,127,127,108,106,127,
+ 127,129,109,98,125,122,112,115, 115,128,120,126,142,114,126,103, 132,96,127,105,124,69,102,122, 128,103,120,127,127,98,112,127,
+ 121,120,145,97,119,126,115,132, 89,127,134,127,112,113,133,129, 130,102,128,127,135,64,118,140, 138,124,104,127,127,129,118,127,
+ 127,127,127,127,127,122,127,127, 127,122,127,127,127,127,127,127, 127,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
+ 127,127,127,127,127,118,127,127, 127,118,127,127,127,127,127,127, 145,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
+ 121,124,104,109,109,81,110,97, 104,77,120,119,111,92,119,106, 100,107,107,127,117,127,112,104, 89,124,89,127,127,197,184,127,
+ 115,99,101,103,110,78,92,110, 84,142,93,113,91,65,122,79, 105,120,109,104,120,113,97,90, 86,115,97,127,127,182,217,127,
+ 127,127,127,127,127,122,127,127, 127,124,127,127,127,127,127,127, 127,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
+};
+
+static const uint8 ced_hires_13[1024] = {
+ 79,36,84,88,70,26,101,101, 57,93,76,44,82,95,88,77, 79,102,93,112,76,86,62,31, 126,102,50,102,97,110,99,131,
+ 52,43,113,117,112,51,119,123, 107,86,124,118,123,119,119,49, 114,113,112,125,118,105,132,18, 125,60,129,75,77,71,51,90,
+ 99,65,124,123,98,65,96,121, 107,100,76,93,112,140,131,73, 92,127,90,86,132,109,81,54, 57,124,94,100,98,115,106,112,
+ 81,31,42,63,61,107,26,43, 32,76,44,28,57,75,56,165, 61,40,75,79,77,82,47,21, 39,68,76,80,81,88,78,107,
+ 53,55,83,115,118,88,124,129, 135,116,109,113,126,117,121,71, 103,69,120,116,124,140,113,25, 116,121,78,69,55,61,52,95,
+ 75,63,96,27,126,105,92,121, 78,64,55,123,122,101,117,34, 107,33,121,105,112,47,100,18, 60,36,113,61,64,87,64,99,
+ 70,75,100,78,116,77,119,135, 81,90,103,129,128,106,113,57, 91,57,132,124,117,87,117,49, 60,46,37,98,75,96,77,107,
+ 104,148,95,50,41,136,45,51, 72,150,86,83,112,103,30,141, 59,39,43,55,90,129,56,44, 57,84,26,104,80,110,75,126,
+
+ 72,54,96,117,102,69,79,123, 44,79,60,58,105,135,103,47, 69,152,129,129,112,72,124,24, 118,59,89,72,68,75,69,97,
+ 49,104,110,125,122,126,125,119, 116,94,112,92,115,120,104,110, 129,129,113,124,117,103,117,22, 126,38,90,48,80,82,62,97,
+ 76,70,64,110,64,64,46,62, 48,63,80,67,84,137,129,30, 93,115,59,121,142,110,109,61, 115,61,70,84,90,95,95,95,
+ 100,63,73,65,82,99,97,66, 73,125,80,83,138,86,127,52, 78,109,102,98,105,78,103,62, 112,76,108,111,109,108,113,111,
+ 111,114,86,93,91,76,81,78, 85,82,114,98,92,112,111,94, 98,94,75,90,95,93,90,72, 91,82,103,151,113,118,141,119,
+ 49,138,104,130,121,72,113,92, 114,38,101,119,105,122,105,109, 107,112,87,116,119,68,115,8, 83,24,113,61,68,71,56,94,
+ 114,52,76,101,85,55,85,54, 85,94,75,85,117,114,135,41, 81,88,76,85,140,64,70,51, 76,101,82,105,98,101,102,103,
+ 114,111,126,117,134,117,113,90, 68,92,84,110,117,110,122,91, 91,164,90,122,124,100,108,120, 102,94,88,118,111,108,113,111,
+
+ 101,137,103,32,71,95,111,96, 97,145,123,85,103,96,79,61, 52,54,86,108,68,157,102,41, 69,71,61,101,99,107,96,119,
+ 73,150,20,28,25,114,26,26, 37,112,46,33,31,22,13,156, 23,49,1,30,13,101,28,20, 32,47,30,71,90,77,68,96,
+ 105,79,95,84,96,64,96,113, 63,97,73,68,104,114,113,65, 121,104,111,119,80,88,85,63, 121,61,65,125,108,129,103,127,
+ 42,53,109,91,120,40,90,107, 68,92,130,92,111,108,134,29, 112,80,111,94,96,54,105,150, 132,48,101,55,61,61,49,94,
+ 106,68,93,88,93,69,69,61, 73,98,117,78,128,133,126,94, 114,78,82,97,141,98,102,65, 75,66,104,102,89,122,99,112,
+ 107,83,114,50,97,149,84,90, 128,149,119,100,113,91,105,79, 132,52,98,96,108,142,94,37, 91,66,96,111,86,94,85,107,
+ 46,66,107,104,113,66,126,117, 117,92,123,126,115,109,116,65, 119,42,129,113,104,53,118,79, 78,127,128,50,61,72,51,99,
+ 121,71,63,62,64,61,69,65, 74,70,74,66,65,54,63,58, 71,84,44,54,50,87,74,66, 113,81,60,152,130,138,134,138,
+
+ 84,83,129,39,123,95,106,128, 82,84,142,138,113,108,107,57, 118,38,127,106,107,90,120,60, 59,142,45,75,98,85,68,107,
+ 105,90,94,88,71,76,71,67, 72,101,88,64,81,91,95,66, 85,76,67,85,79,82,82,58, 109,96,81,127,113,121,114,123,
+ 73,100,141,107,128,71,80,84, 100,85,121,103,123,126,115,69, 119,72,94,128,109,59,112,26, 83,59,116,115,83,89,73,94,
+ 104,73,90,91,87,88,89,78, 90,76,102,101,111,102,89,43, 82,81,118,96,146,75,100,50, 87,96,96,113,113,106,109,114,
+ 66,58,131,124,106,86,118,114, 132,74,68,119,117,117,117,35, 102,36,125,106,106,24,98,35, 71,128,128,81,81,80,47,96,
+ 73,41,118,138,109,28,78,48, 115,45,110,113,95,133,88,35, 121,38,106,109,103,103,117,18, 44,86,119,71,77,86,76,105,
+ 94,137,71,54,52,142,72,61, 72,154,145,98,92,97,38,93, 58,58,95,69,85,132,127,42, 91,123,55,108,102,106,97,140,
+ 113,65,76,58,62,92,85,62, 80,69,57,64,67,66,51,59, 80,72,50,69,62,68,58,73, 71,60,58,101,115,108,109,118,
+};
+
+static const uint8 ced_hires_14[1024] = {
+ 127,73,147,96,119,88,105,109, 120,113,103,118,88,140,101,79, 110,127,68,112,151,94,90,83, 125,83,125,127,127,127,127,127,
+ 88,49,130,116,118,52,139,140, 125,89,134,128,134,134,134,59, 124,140,131,139,131,105,148,19, 145,52,137,100,89,91,77,99,
+ 131,71,141,122,104,66,116,138, 125,103,87,103,122,155,146,83, 102,154,109,100,144,109,97,55, 77,116,102,118,110,122,130,121,
+ 134,25,93,100,106,94,68,101, 54,113,93,18,111,141,103,68, 142,68,136,117,127,146,99,18, 62,18,121,110,88,106,105,109,
+ 87,60,100,113,123,88,143,145, 153,119,119,122,136,131,136,80, 113,95,138,130,136,140,128,26, 136,113,85,93,67,79,77,103,
+ 127,99,154,57,97,71,91,93, 125,69,89,116,117,76,110,81, 149,113,85,77,84,90,75,68, 110,72,120,127,127,120,127,123,
+ 107,127,106,67,34,132,96,32, 46,138,56,96,38,110,100,88, 74,63,26,39,49,130,29,120, 45,34,49,93,90,113,96,118,
+ 140,153,112,49,46,136,65,67, 90,152,96,92,122,118,45,150, 69,64,61,68,102,129,71,44, 77,75,33,128,92,127,99,134,
+
+ 74,133,89,64,55,143,71,52, 72,150,122,145,125,84,137,124, 106,53,105,97,125,131,90,27, 24,49,71,72,81,88,67,109,
+ 83,109,127,124,128,126,144,135, 134,97,122,101,125,134,119,119, 139,155,131,138,129,102,132,22, 146,29,97,71,91,102,87,105,
+ 88,4,133,137,142,3,28,140, 12,31,52,141,80,31,31,20, 149,46,21,114,133,20,2,90, 69,0,136,73,62,95,79,102,
+ 126,68,90,64,87,99,117,82, 91,127,90,92,147,101,142,60, 88,136,120,111,116,77,118,62, 130,67,116,128,121,114,137,119,
+ 79,15,89,109,132,0,34,27, 136,25,159,134,135,122,132,40, 105,35,121,133,141,23,146,2, 28,0,125,76,68,108,72,91,
+ 84,143,121,128,126,72,133,108, 132,41,111,128,115,136,120,118, 117,138,105,130,131,68,130,9, 103,16,120,86,80,91,80,101,
+ 144,57,93,100,91,55,105,70, 103,96,85,94,127,129,150,50, 91,114,94,98,152,63,85,51, 96,92,89,120,109,108,123,111,
+ 121,154,45,60,63,64,68,39, 59,85,56,120,31,120,40,125, 43,86,21,83,136,120,56,36, 80,36,43,124,106,113,123,115,
+
+ 121,150,74,14,27,151,56,36, 50,150,69,55,34,102,58,106, 39,82,28,56,34,160,40,31, 75,45,39,131,101,109,119,112,
+ 104,54,77,132,55,34,31,18, 17,42,46,106,0,82,44,80, 36,49,0,155,40,40,6,33, 43,19,25,91,101,99,89,114,
+ 112,141,38,57,37,72,43,25, 101,67,64,150,25,96,40,152, 26,69,15,132,116,149,27,18, 62,32,26,112,97,105,106,108,
+ 78,60,127,91,126,42,111,125, 87,96,141,103,122,124,150,40, 122,106,129,108,108,54,120,151, 152,40,108,78,73,81,74,102,
+ 140,74,110,87,99,70,89,78, 91,102,128,88,139,149,142,104, 123,104,100,110,153,97,117,66, 94,57,111,124,100,129,123,119,
+ 112,104,136,78,126,105,142,117, 127,127,143,142,137,115,105,119, 116,69,120,135,123,76,130,29, 62,18,144,114,88,105,106,108,
+ 82,72,125,103,119,67,146,134, 136,96,134,136,126,124,132,75, 129,68,147,127,116,53,133,80, 97,119,135,74,72,91,76,106,
+ 131,76,81,61,69,61,88,82, 92,73,84,76,75,69,78,66, 81,110,61,67,61,85,87,66, 132,71,67,154,139,145,139,146,
+
+ 84,133,98,52,42,158,37,23, 89,160,53,107,63,86,87,104, 70,38,30,103,87,98,78,0, 32,26,51,80,58,80,75,91,
+ 95,14,130,98,109,14,63,29, 116,13,156,109,122,149,89,39, 70,49,106,128,105,19,149,25, 43,4,137,91,82,95,86,110,
+ 109,107,159,107,134,73,101,102, 119,89,132,114,134,142,131,80, 129,98,112,142,121,59,127,27, 103,51,123,138,94,108,99,102,
+ 123,108,129,80,86,116,133,102, 135,85,156,146,124,112,121,59, 80,90,122,135,124,64,156,40, 93,40,143,127,110,116,125,119,
+ 102,64,148,123,112,87,139,131, 152,77,79,129,128,132,133,45, 111,62,143,119,118,22,113,35, 90,119,135,104,92,99,72,104,
+ 109,48,136,138,115,29,99,66, 134,49,121,124,106,149,104,46, 131,63,124,123,115,103,132,19, 63,78,126,96,88,105,101,112,
+ 117,136,38,70,20,133,49,30, 43,174,47,36,24,27,9,70, 34,75,10,15,28,140,32,25, 69,25,32,117,95,106,112,109,
+ 136,54,77,34,51,77,76,61, 86,49,60,57,54,66,64,51, 69,93,44,77,50,54,57,55, 81,42,72,112,121,115,118,126,
+};
+
+static const uint8 ced_hires_15[1024] = {
+ 128,67,142,105,122,81,101,108, 115,105,99,113,90,139,98,75, 108,127,69,108,148,90,86,94, 122,79,127,128,128,128,128,128,
+ 69,43,125,125,121,45,135,139, 120,81,130,123,135,133,130,54, 122,137,129,135,128,100,144,30, 140,48,139,91,84,84,66,98,
+ 115,65,136,131,106,59,112,137, 120,95,82,98,123,154,142,78, 100,151,107,96,141,104,93,66, 72,112,103,112,105,119,121,120,
+ 117,20,88,109,109,87,64,100, 48,105,89,13,112,140,99,63, 140,65,134,113,124,141,95,29, 57,14,123,101,83,100,94,108,
+ 68,54,95,122,126,81,139,144, 148,111,115,117,137,130,132,75, 111,92,136,126,133,135,124,37, 131,109,87,85,62,73,66,102,
+ 128,93,149,66,99,65,86,92, 121,64,84,111,119,76,106,76, 146,112,85,74,81,85,71,79, 108,70,121,126,126,117,128,122,
+ 88,121,101,75,36,126,91,31, 42,131,51,91,40,109,96,83, 71,59,25,36,46,125,25,131, 41,30,50,84,85,105,85,117,
+ 121,147,107,57,49,130,60,66, 85,145,91,87,124,117,41,145, 66,61,60,65,99,124,67,55, 72,71,35,119,87,122,89,133,
+
+ 57,127,84,73,58,136,67,51, 67,142,118,140,126,83,133,119, 104,49,103,93,122,126,86,38, 19,45,73,63,77,81,56,108,
+ 65,103,122,132,130,120,140,134, 129,89,118,96,126,133,115,114, 137,152,130,134,126,97,128,34, 141,25,99,63,86,95,76,104,
+ 69,0,128,145,144,0,23,138, 6,23,47,135,81,30,27,14, 146,42,19,110,129,14,0,100, 64,0,137,63,57,87,67,101,
+ 115,62,85,72,89,93,112,81, 86,120,86,87,149,100,138,56, 85,132,119,108,113,72,114,73, 126,63,117,122,116,111,127,118,
+ 60,9,84,117,134,0,29,27, 131,18,154,129,137,121,128,35, 102,31,120,130,138,19,142,14, 24,0,126,67,63,101,61,91,
+ 65,137,116,137,129,65,129,107, 127,33,107,123,116,135,116,113, 115,135,103,126,128,63,126,20, 98,12,122,77,75,84,70,100,
+ 130,51,88,108,93,49,100,69, 98,89,80,89,129,128,146,44, 88,110,92,95,149,58,81,63, 91,88,91,114,104,105,116,110,
+ 111,148,40,69,66,57,65,38, 55,77,52,115,34,119,36,120, 41,83,19,79,133,115,53,47, 75,32,45,118,101,111,112,114,
+
+ 109,144,70,22,29,145,51,35, 45,143,64,50,37,101,54,101, 36,78,28,54,31,155,37,42, 71,41,40,124,96,106,108,111,
+ 84,47,72,141,57,27,25,15, 12,34,40,100,1,81,40,74, 33,45,0,151,36,34,1,42, 38,14,26,81,95,91,77,113,
+ 96,135,34,65,40,66,38,23, 96,60,59,145,27,95,36,147, 23,65,15,129,113,144,24,29, 58,28,27,104,92,101,95,107,
+ 59,54,122,100,129,35,107,124, 82,88,137,98,123,123,146,35, 120,103,127,104,105,49,116,162, 147,36,110,70,68,74,63,101,
+ 123,67,105,95,101,63,84,76, 86,94,123,82,140,147,138,98, 120,100,98,106,149,91,112,76, 89,52,112,117,95,126,112,118,
+ 96,98,131,86,128,99,137,116, 122,120,138,137,139,114,101,114, 113,65,119,132,120,71,126,41, 58,14,145,105,83,101,95,107,
+ 63,66,120,112,122,60,142,133, 131,88,130,131,127,123,128,70, 127,65,145,123,113,48,129,91, 92,115,137,65,68,84,65,105,
+ 132,70,76,69,72,54,84,81, 87,66,80,71,77,68,74,62, 78,106,61,63,58,81,84,77, 127,67,68,155,136,142,140,145,
+
+ 65,127,93,61,45,151,33,22, 84,152,49,102,64,85,83,99, 68,35,28,99,84,93,74,2, 27,22,53,71,53,74,64,90,
+ 76,8,125,107,112,8,59,28, 111,6,152,104,123,148,85,34, 68,46,104,124,102,13,145,36, 38,2,139,82,77,88,75,110,
+ 90,101,154,116,137,66,97,101, 114,81,128,109,135,141,127,75, 127,95,110,138,118,54,123,38, 98,47,125,130,89,101,88,102,
+ 117,102,124,88,89,109,129,101, 130,77,152,141,125,111,117,55, 79,87,120,131,121,58,152,51, 90,36,145,122,105,113,116,118,
+ 83,58,143,132,115,81,134,130, 147,70,74,124,129,131,129,40, 109,59,141,116,115,18,109,46, 85,115,137,95,87,92,61,103,
+ 90,42,131,147,118,23,95,65, 129,41,117,119,107,148,100,41, 129,60,122,119,112,98,128,30, 59,74,128,87,83,99,90,112,
+ 102,130,34,79,23,126,45,29, 38,166,43,32,26,27,5,65, 30,72,8,11,24,135,28,36, 64,21,34,108,90,103,101,108,
+ 121,48,73,42,53,70,71,61, 81,41,56,52,55,65,60,46, 67,89,42,73,48,49,53,67, 76,38,74,106,116,112,115,125,
+};
+
+static const uint8 ced_hires_16[1024] = {
+ 126,77,94,99,85,77,105,111, 93,98,106,78,81,99,92,92, 100,116,99,119,82,86,89,94, 132,90,87,127,125,126,126,131,
+ 97,47,126,130,125,63,130,137, 117,95,136,122,124,126,123,57, 128,120,120,134,123,99,146,65, 132,56,132,97,96,97,97,110,
+ 143,93,132,131,106,93,114,132, 110,112,123,95,107,141,130,103, 112,132,92,86,131,103,106,111, 132,107,103,143,142,143,143,130,
+ 128,105,112,105,109,106,116,109, 113,112,120,113,104,107,99,110, 114,124,100,102,105,111,112,115, 125,111,116,128,128,128,128,128,
+ 97,58,96,127,131,95,134,142, 146,126,121,116,127,123,126,86, 117,87,128,126,128,135,126,65, 123,114,81,98,96,97,97,110,
+ 157,107,120,109,115,107,128,115, 124,121,137,109,95,107,86,117, 126,146,89,100,96,117,120,125, 146,116,117,157,156,157,157,139,
+ 157,107,120,109,115,107,128,115, 124,121,137,109,95,107,86,117, 126,146,89,100,96,117,120,125, 146,116,117,157,156,157,157,139,
+ 120,155,109,72,78,142,91,78, 87,159,100,85,112,108,49,156, 89,109,52,63,93,123,83,88, 109,79,80,120,119,120,120,118,
+
+ 119,70,107,128,115,75,90,135, 86,88,99,71,105,140,107,80, 93,159,136,138,115,79,136,87, 118,78,88,120,118,119,119,118,
+ 92,112,123,137,135,133,135,132, 127,104,124,95,116,126,108,126, 143,137,121,134,121,97,130,60, 133,51,92,93,91,92,92,106,
+ 130,80,93,121,88,80,101,88, 97,94,110,82,83,142,131,90, 104,119,62,128,144,100,121,98, 119,89,90,130,129,130,130,123,
+ 148,98,111,100,106,98,119,106, 115,127,128,100,131,98,124,108, 117,137,102,96,97,108,111,116, 137,107,108,148,147,148,148,135,
+ 149,100,112,101,108,100,120,107, 116,113,129,101,88,99,99,110, 118,139,82,93,88,109,112,117, 138,108,110,150,148,149,149,139,
+ 98,145,117,142,133,78,123,105, 125,62,112,122,105,128,109,124, 120,119,94,125,123,58,128,66, 97,57,115,98,97,98,98,112,
+ 145,96,108,110,104,96,116,103, 112,109,125,97,110,112,132,106, 114,135,78,89,137,105,108,113, 134,104,106,146,144,145,145,132,
+ 153,111,124,118,134,108,124,111, 120,117,133,105,101,103,113,113, 122,159,85,115,115,113,116,121, 142,112,113,153,152,153,153,136,
+
+ 128,112,119,112,116,113,123,116, 120,119,127,120,111,114,106,117, 120,128,106,108,111,117,118,121, 128,117,122,128,128,128,128,128,
+ 120,158,83,72,78,121,91,78, 87,121,100,72,58,70,49,172, 88,108,51,62,58,95,82,87, 108,78,79,119,118,119,119,122,
+ 146,96,109,98,104,96,117,120, 113,110,126,98,94,116,108,106, 130,134,109,118,84,105,108,113, 134,104,105,145,144,145,145,137,
+ 98,58,122,104,132,48,101,121, 75,101,142,96,111,115,138,58, 126,86,118,103,100,57,118,161, 139,56,102,97,96,97,97,111,
+ 139,89,102,96,102,89,110,97, 106,103,124,91,124,135,126,104, 122,127,80,99,141,98,106,106, 127,97,98,138,137,138,138,129,
+ 155,105,128,107,113,105,126,113, 122,119,135,107,93,105,84,115, 133,143,86,97,93,114,117,122, 143,113,114,154,153,154,154,139,
+ 99,72,121,117,126,72,137,131, 129,103,136,130,116,116,121,79, 134,87,137,123,109,58,132,89, 87,121,131,98,97,98,98,111,
+ 154,104,117,106,112,104,125,112, 121,118,134,106,92,104,83,114, 122,142,85,96,92,113,116,121, 142,112,113,153,152,153,153,139,
+
+ 156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
+ 150,101,113,102,109,101,121,108, 117,114,130,102,89,100,80,111, 118,139,82,93,88,109,112,117, 138,108,110,150,148,149,149,140,
+ 118,107,154,120,140,68,89,96, 111,92,132,106,123,133,119,78, 133,106,101,137,113,77,125,85, 106,76,118,122,116,117,117,121,
+ 149,99,112,101,107,99,120,107, 116,113,129,101,102,99,83,109, 117,137,115,96,139,108,111,116, 137,107,108,148,147,148,148,135,
+ 102,62,144,137,119,94,129,128, 144,82,82,123,118,124,122,62, 116,90,133,116,111,61,112,69, 90,121,131,101,100,101,101,110,
+ 156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
+ 156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
+ 137,88,100,89,96,96,108,95, 104,101,117,89,76,87,67,98, 105,126,69,80,75,96,99,104, 125,95,97,137,135,136,136,123,
+};
+
+static const uint8 ced_hires_17[1024] = {
+ 89,91,92,133,130,37,125,74, 42,95,144,140,137,137,121,42, 123,75,132,124,133,87,128,83, 74,53,119,94,87,91,93,108,
+ 102,61,121,126,115,65,141,137, 130,96,110,126,137,135,142,73, 128,143,129,129,127,116,147,95, 154,82,141,107,100,103,106,114,
+ 152,89,132,131,102,84,114,134, 128,107,81,104,125,155,153,95, 103,154,108,87,139,120,95,146, 137,146,100,157,150,154,156,136,
+ 130,66,62,72,62,122,86,62, 83,84,58,60,65,91,78,190, 73,115,92,83,85,91,68,123, 115,94,83,135,128,131,134,127,
+ 103,75,93,124,121,103,146,143, 159,127,96,121,140,133,145,96, 118,98,137,121,133,153,128,96, 146,146,90,108,101,104,107,114,
+ 111,83,105,33,129,120,114,135, 102,75,39,131,136,117,141,54, 122,96,138,110,121,62,115,104, 96,75,125,116,109,112,115,120,
+ 131,94,108,84,119,91,141,148, 104,99,90,137,142,121,136,74, 104,117,149,128,125,99,131,125, 116,95,74,136,129,133,135,128,
+ 109,71,69,85,106,85,106,131, 62,57,93,110,101,126,73,52, 116,94,127,130,102,60,143,102, 94,73,52,114,107,118,113,119,
+
+ 100,147,79,74,50,157,72,48, 76,157,98,143,128,85,145,139, 110,85,103,87,121,143,89,93, 85,84,74,105,98,101,104,112,
+ 97,124,119,134,125,141,147,132, 139,104,99,100,129,135,127,134, 143,159,130,128,125,115,131,91, 155,61,101,102,95,99,101,109,
+ 101,37,124,147,139,33,57,137, 54,39,29,139,83,34,39,44, 153,86,16,104,129,52,39,151, 86,65,140,106,99,102,105,116,
+ 159,95,91,81,81,113,115,91, 112,133,87,89,150,97,149,102, 102,144,118,100,111,110,113,152, 144,123,117,164,157,160,163,142,
+ 86,23,75,47,132,18,43,105, 39,54,141,105,128,124,110,29, 89,72,105,147,131,38,115,80, 71,50,78,91,84,88,90,107,
+ 103,158,113,138,124,86,135,105, 137,41,88,127,119,137,128,133, 120,142,104,120,127,81,129,97, 111,67,124,108,101,105,107,116,
+ 156,93,94,108,84,88,113,88, 109,99,85,92,130,128,156,99, 99,142,88,85,147,108,94,150, 141,120,99,161,154,158,160,138,
+ 98,50,162,115,131,30,54,149, 51,46,135,121,106,128,119,41, 135,83,109,116,138,49,147,91, 83,62,136,103,96,99,102,111,
+
+ 160,171,92,82,82,172,116,92, 113,164,88,90,85,109,82,129, 97,139,69,66,71,175,92,147, 139,118,97,159,152,155,158,137,
+ 129,176,77,84,94,159,85,102, 82,160,57,105,70,85,71,176, 89,108,48,73,73,173,81,116, 108,87,76,128,121,124,127,122,
+ 180,139,119,109,109,119,143,119, 140,135,125,117,112,120,109,130, 124,166,112,93,98,160,119,174, 166,145,124,180,179,177,180,152,
+ 156,139,88,78,116,166,138,88, 109,177,94,86,81,89,111,115, 93,135,95,88,77,162,98,171, 135,114,93,155,148,151,154,133,
+ 156,92,106,103,101,88,112,88, 114,119,110,91,149,156,157,125, 129,135,101,102,152,114,117,143, 135,114,116,155,148,151,154,133,
+ 152,109,131,74,109,171,109,112, 159,167,115,117,135,115,136,115, 148,132,118,103,119,157,112,140, 131,110,109,151,144,148,150,134,
+ 112,94,124,121,124,90,156,139, 149,111,118,142,137,133,148,98, 137,91,149,121,116,67,136,145, 107,155,143,111,104,107,110,113,
+ 180,119,115,104,105,114,139,114, 135,120,111,113,108,115,104,125, 119,162,92,88,93,128,114,170, 161,140,119,178,174,174,178,149,
+
+ 128,111,146,50,134,118,137,150, 112,103,137,155,135,132,138,87, 135,108,147,113,118,105,137,126, 107,169,65,127,120,124,126,122,
+ 109,46,52,31,32,41,66,41, 62,47,127,60,65,52,41,52, 46,89,19,130,30,55,41,97, 88,67,66,108,101,105,107,132,
+ 131,129,158,124,139,93,111,106, 131,102,116,120,145,150,146,102, 136,111,114,135,120,77,129,119, 110,89,130,146,123,127,129,123,
+ 171,107,103,101,93,108,127,103, 124,109,99,114,130,122,116,114, 108,150,135,102,155,116,111,158, 150,129,108,170,163,166,169,141,
+ 115,87,147,141,117,109,149,136, 164,92,64,135,139,141,148,71, 119,94,145,113,118,60,116,102, 99,155,143,114,107,110,113,112,
+ 180,125,121,120,111,120,145,120, 141,126,117,119,130,121,126,131, 125,168,98,94,99,134,120,176, 167,146,125,180,179,177,180,152,
+ 108,45,99,103,139,40,96,117, 61,87,85,145,139,127,138,51, 127,88,152,133,136,64,131,96, 87,66,135,107,100,104,106,112,
+ 157,94,95,80,80,109,114,90, 119,96,86,88,83,91,80,101, 95,137,67,64,69,103,90,145, 137,116,95,154,150,150,154,125,
+};
+
+static const uint8 ced_hires_18[1024] = {
+ 100,47,105,106,88,37,119,122, 76,90,99,47,87,103,100,98, 95,121,106,126,89,84,80,50, 146,101,49,119,100,126,121,130,
+ 74,55,134,135,131,65,138,145, 126,83,147,122,129,128,131,71, 130,133,126,138,131,103,151,36, 145,59,129,97,84,83,73,90,
+ 123,78,145,142,117,80,115,143, 127,97,99,98,117,149,143,95, 108,146,104,100,145,107,101,72, 83,123,94,119,107,123,126,111,
+ 102,42,63,81,78,121,44,64, 50,73,66,32,62,84,68,187, 77,62,89,92,89,80,66,40, 63,66,76,102,89,103,100,106,
+ 72,67,105,133,136,102,142,150, 155,114,132,117,131,125,133,93, 119,88,133,130,137,139,132,46, 137,120,78,89,64,76,73,94,
+ 94,75,117,44,144,118,110,142, 98,61,78,127,127,109,129,55, 123,48,134,119,125,45,118,38, 80,35,112,83,72,101,83,96,
+ 95,87,122,95,134,91,137,156, 101,88,126,133,133,114,125,79, 107,76,145,138,130,85,136,69, 79,46,38,117,85,112,100,104,
+ 124,160,116,69,60,150,64,73, 92,147,109,87,118,112,42,163, 75,62,57,68,103,128,75,63, 80,83,28,120,89,124,96,125,
+
+ 94,66,118,135,120,83,97,144, 63,76,82,62,110,143,115,69, 85,171,142,143,125,71,143,43, 138,58,89,97,78,95,93,94,
+ 67,116,131,143,140,140,143,140, 136,91,135,96,120,128,115,131, 145,148,127,138,130,101,135,42, 146,37,89,70,86,94,79,96,
+ 101,82,86,128,82,78,65,84, 68,60,103,72,89,145,141,49, 109,135,73,134,155,108,128,81, 135,61,70,108,97,107,117,96,
+ 123,75,94,83,100,113,116,87, 92,122,102,87,143,94,139,71, 93,129,116,111,117,75,121,80, 131,72,108,126,118,118,133,111,
+ 125,126,107,111,110,90,98,100, 104,78,136,102,98,121,122,116, 113,113,88,103,108,91,109,90, 113,82,103,166,118,129,154,121,
+ 71,150,125,148,139,85,131,113, 134,35,124,123,110,130,117,130, 123,131,100,130,132,66,133,26, 103,24,112,82,78,81,75,92,
+ 134,66,98,119,103,70,104,77, 104,91,98,89,123,122,146,64, 97,108,89,98,153,65,88,72, 98,100,82,121,109,113,126,105,
+ 134,123,148,135,153,131,131,112, 86,89,109,114,122,118,134,113, 106,183,104,135,137,97,127,140, 120,93,88,129,122,120,132,113,
+
+ 95,138,100,94,128,146,84,59, 64,158,26,72,140,108,64,96, 48,37,129,66,66,162,79,20, 36,60,93,75,70,90,70,98,
+ 95,163,41,47,45,129,41,47, 56,110,67,38,37,31,25,179, 45,69,19,50,32,105,52,43, 62,51,33,101,100,101,98,98,
+ 122,92,117,103,114,78,115,136, 82,95,96,73,110,123,125,87, 143,128,130,139,99,92,109,87, 147,65,68,148,119,143,129,127,
+ 68,66,131,110,138,55,109,130, 88,90,153,97,117,117,146,52, 134,105,130,114,115,58,129,176, 158,53,106,82,71,82,78,96,
+ 129,80,114,106,112,83,88,82, 92,96,140,82,134,142,139,116, 135,103,101,116,160,101,127,91, 102,70,110,124,105,134,125,113,
+ 123,95,135,69,116,163,102,112, 149,147,143,105,119,100,118,101, 153,80,118,115,127,146,119,62, 116,70,102,133,102,113,117,109,
+ 70,78,129,122,132,80,145,139, 137,90,147,130,121,118,129,87, 141,67,148,133,123,58,143,105, 104,132,134,80,74,95,79,100,
+ 136,82,86,79,83,76,90,87, 92,67,102,69,70,65,74,80, 92,115,66,71,68,91,96,94, 140,85,71,156,136,150,142,140,
+
+ 104,96,151,58,141,110,125,150, 102,82,165,143,118,117,119,80, 140,61,146,125,125,94,144,85, 85,147,50,100,110,107,95,109,
+ 122,103,115,106,89,90,88,87, 92,99,110,70,86,100,108,86, 107,107,86,104,98,85,106,87, 134,101,84,138,122,134,133,128,
+ 93,113,163,126,146,86,99,107, 120,83,144,108,129,135,127,92, 141,96,113,148,128,63,136,50, 109,64,121,140,95,106,100,97,
+ 128,86,112,109,106,102,108,101, 111,75,126,105,117,111,101,66, 104,108,137,116,165,81,125,82, 114,102,102,129,126,121,134,116,
+ 89,71,152,143,125,101,137,136, 153,72,92,124,123,126,129,57, 123,62,145,126,125,29,123,61, 97,133,134,107,94,103,79,98,
+ 99,71,108,117,119,66,122,99, 108,25,83,132,126,131,130,88, 120,83,124,123,119,71,105,40, 88,140,139,89,119,109,90,120,
+ 109,146,119,71,82,143,99,110, 107,147,49,135,131,127,53,105, 80,80,69,92,140,135,125,42, 60,114,37,69,92,91,80,109,
+ 129,78,95,77,81,106,104,83, 99,66,82,66,73,73,63,80, 101,97,69,88,80,74,80,98, 98,65,64,114,130,119,121,120,
+};
+
+static const uint8 ced_hires_19[1024] = {
+ 98,93,104,135,134,69,120,71, 84,99,150,144,141,141,121,77, 123,96,135,127,139,85,130,83, 94,88,120,97,97,97,97,113,
+ 110,85,134,128,119,81,138,142, 127,104,115,130,141,138,142,90, 128,119,133,131,132,117,148,95, 129,101,142,109,109,109,109,119,
+ 145,120,126,112,107,116,133,123, 131,123,115,115,110,142,136,124, 123,143,99,95,128,132,120,130, 141,135,125,144,144,144,144,134,
+ 134,109,115,93,96,120,122,107, 120,112,104,104,99,106,91,186, 112,132,88,84,92,121,109,119, 130,124,114,133,133,133,133,130,
+ 111,86,102,124,125,107,142,148, 156,131,104,126,144,136,144,90, 119,109,140,123,138,153,129,96, 122,136,91,110,110,110,110,118,
+ 118,93,119,78,133,123,107,139, 104,97,88,134,139,118,140,98, 122,117,141,111,125,105,113,103, 114,109,123,117,117,117,117,124,
+ 135,110,116,95,117,106,124,148, 121,114,105,136,141,117,132,115, 114,134,148,125,126,122,125,120, 131,126,115,134,134,134,134,130,
+ 117,92,98,86,109,98,105,136, 103,95,97,112,105,129,74,96, 110,115,130,131,106,104,144,102, 113,107,97,116,116,116,116,123,
+
+ 108,150,89,67,70,160,96,81, 94,161,103,147,132,80,144,139, 109,106,105,88,126,143,83,93, 104,98,88,107,107,107,107,117,
+ 106,127,132,135,129,144,143,138, 136,107,104,103,133,139,126,134, 143,136,133,131,131,114,133,91, 130,96,102,105,105,105,105,114,
+ 109,84,137,148,143,80,97,142, 95,87,79,143,84,81,66,88, 152,107,63,106,134,96,84,104, 105,99,141,108,108,108,108,120,
+ 146,121,127,106,108,117,135,119, 132,125,116,116,133,118,126,126, 125,145,101,96,104,133,121,131, 142,137,126,145,145,145,145,136,
+ 95,70,86,54,136,66,83,111, 81,73,147,109,132,128,110,74, 88,93,108,151,137,82,117,80, 91,85,75,94,94,94,94,111,
+ 112,161,124,140,127,83,131,110, 135,90,92,132,123,141,128,133, 120,110,106,123,133,99,130,97, 108,102,125,111,111,111,111,121,
+ 145,120,126,105,107,116,134,118, 131,124,115,115,115,117,137,125, 124,144,100,95,132,132,120,130, 141,136,125,144,144,144,144,135,
+ 148,123,129,107,115,119,136,121, 134,126,118,118,113,120,115,127, 126,146,102,103,111,135,123,133, 144,138,128,147,147,147,147,136,
+
+ 147,153,128,106,109,118,135,120, 133,153,117,117,112,119,104,126, 126,146,102,108,106,165,123,133, 144,138,128,147,147,147,147,136,
+ 132,174,113,92,94,156,121,105, 118,160,102,102,97,104,90,172, 112,132,88,83,91,175,108,118, 129,124,113,132,132,132,132,127,
+ 152,127,133,112,114,123,141,125, 138,131,122,122,117,124,110,132, 132,152,108,103,111,140,128,138, 149,144,133,152,152,152,152,138,
+ 117,92,136,107,132,88,105,132, 103,110,127,110,134,133,162,96, 133,116,138,108,116,105,127,176, 144,108,118,117,117,117,117,120,
+ 148,123,129,107,110,119,136,121, 134,126,118,118,135,141,139,127, 127,147,103,99,147,136,124,134, 145,139,129,148,148,148,148,134,
+ 148,123,129,107,110,162,136,121, 144,159,118,118,123,120,120,127, 142,147,113,99,117,151,124,134, 145,139,129,148,148,148,148,135,
+ 117,92,133,120,125,98,149,141, 144,116,122,143,138,132,145,97, 142,117,157,127,125,105,141,103, 114,150,148,117,117,117,117,120,
+ 153,128,134,112,115,124,141,126, 139,131,123,123,118,125,110,132, 132,152,108,104,112,141,129,139, 150,144,134,153,153,153,153,138,
+
+ 132,107,153,92,134,113,121,151, 118,111,137,154,135,129,133,112, 140,132,153,117,125,120,141,118, 129,163,113,132,132,132,132,127,
+ 113,88,94,73,75,84,102,86, 99,92,127,83,78,85,71,93, 93,113,69,135,72,101,89,99, 110,105,94,113,113,113,113,129,
+ 135,125,166,120,137,106,124,108, 121,114,115,115,143,149,142,115, 138,135,119,140,128,123,131,121, 132,127,131,135,135,135,135,129,
+ 151,126,132,111,113,122,140,124, 137,130,121,121,116,123,109,131, 131,151,114,102,138,139,127,137, 148,143,132,151,151,151,151,137,
+ 120,95,157,140,118,108,142,138, 158,99,90,136,140,141,145,100, 121,120,153,120,127,108,119,106, 117,149,148,120,120,120,120,119,
+ 136,111,140,151,118,107,124,109, 137,114,106,126,111,155,113,115, 138,135,129,118,120,124,137,122, 133,127,132,136,136,136,136,131,
+ 151,146,132,110,113,145,139,124, 137,154,121,121,116,123,108,130, 130,150,106,102,110,139,127,137, 148,142,132,151,151,151,151,137,
+ 146,121,127,105,108,117,134,119, 132,124,116,116,111,118,103,125, 125,145,101,97,105,134,122,132, 143,137,127,146,146,146,146,131,
+};
+
+static const uint8 ced_hires_20[1024] = {
+ 111,51,50,66,67,37,61,60, 62,52,151,76,117,107,89,44, 64,85,123,112,43,48,109,55, 80,47,73,105,103,120,107,126,
+ 99,78,66,40,113,81,103,136, 51,33,85,110,79,133,68,53, 121,80,125,133,107,49,144,49, 79,46,58,104,94,114,94,119,
+ 82,89,103,144,141,45,125,85, 60,80,142,139,139,140,118,46, 133,65,134,127,134,77,126,29, 65,28,130,79,73,84,74,99,
+ 148,112,111,112,105,115,135,114, 133,109,103,112,104,109,99,132, 123,145,97,101,94,117,110,138, 144,123,128,148,147,147,148,137,
+ 97,70,96,138,128,103,151,149, 163,109,95,119,143,133,146,96, 122,107,138,122,134,143,125,67, 141,129,98,104,95,92,94,114,
+ 112,73,108,62,136,120,118,141, 105,57,53,129,139,116,142,70, 126,92,139,111,122,64,111,67, 100,63,134,104,102,110,102,119,
+ 126,53,74,82,77,64,78,76, 74,51,58,72,68,60,68,61, 85,102,50,148,65,58,59,77, 97,64,90,122,116,120,117,129,
+ 94,50,95,141,134,56,69,127, 60,37,151,133,141,146,122,58, 115,74,138,121,145,52,105,44, 78,35,128,96,85,89,89,105,
+
+ 93,143,83,89,63,158,83,60, 83,139,98,142,132,86,147,141, 115,84,105,89,123,134,85,67, 69,69,82,94,100,99,89,116,
+ 92,119,122,149,133,142,152,139, 144,87,98,98,132,136,128,136, 148,160,131,130,127,105,129,63, 151,58,111,91,104,103,95,113,
+ 148,106,116,121,106,102,130,108, 127,89,97,117,103,117,112,113, 133,145,97,94,107,111,109,132, 144,121,140,148,147,147,148,137,
+ 96,26,70,54,136,23,56,118, 57,39,143,109,136,129,114,24, 102,75,104,149,132,26,115,40, 62,32,99,95,91,85,89,104,
+ 122,116,63,61,63,156,88,58, 77,162,52,66,68,77,80,112, 84,106,43,53,46,123,55,75, 100,67,77,125,117,120,120,120,
+ 137,128,86,79,104,158,111,95, 93,154,68,90,84,84,82,113, 113,121,63,81,93,155,94,91, 115,82,99,136,132,131,136,128,
+ 91,40,166,130,139,40,74,156, 53,33,137,119,108,132,120,47, 139,70,109,117,140,49,148,47, 74,31,142,89,83,85,84,103,
+ 113,165,84,84,91,133,107,85, 101,142,80,94,94,82,76,158, 92,97,86,89,67,171,112,67, 91,63,89,116,108,115,112,117,
+
+ 84,151,102,50,102,126,72,103, 59,149,74,148,115,116,104,148, 125,78,106,107,133,146,137,43, 60,158,87,85,79,92,80,104,
+ 115,72,78,158,69,65,71,53, 67,50,47,103,38,82,62,99, 72,98,29,152,56,69,47,84, 93,67,72,118,125,119,113,127,
+ 116,167,96,89,95,156,80,88, 85,135,64,95,66,79,70,161, 97,98,65,84,85,166,84,67, 92,59,94,117,117,113,112,116,
+ 95,66,122,116,132,60,119,129, 93,85,117,101,129,126,160,66, 136,118,134,105,111,69,122,180, 161,71,127,101,101,101,96,115,
+ 148,107,121,119,120,108,132,120, 135,112,126,122,123,123,121,115, 140,147,122,115,120,131,127,129, 146,120,138,149,149,149,149,138,
+ 145,100,125,89,108,164,120,114, 156,141,106,107,130,110,130,109, 154,126,119,105,123,147,113,96, 136,98,123,146,132,131,136,131,
+ 96,81,121,129,125,83,154,139, 147,86,111,134,134,126,142,92, 143,98,152,124,120,68,135,108, 111,140,154,101,101,106,99,117,
+ 148,101,97,103,91,93,119,102, 119,82,86,90,96,95,97,105, 112,145,87,80,83,109,104,119, 153,110,113,160,153,158,154,146,
+
+ 96,28,42,33,46,35,53,45, 49,31,126,40,50,53,52,43, 68,80,39,131,35,47,41,50, 74,51,47,99,104,105,97,134,
+ 148,137,108,111,112,125,128,118, 139,87,95,116,95,112,110,149, 117,147,86,88,103,140,105,130, 146,146,127,149,149,149,149,138,
+ 148,129,121,142,101,121,132,114, 128,114,99,114,126,128,120,128, 126,147,104,99,97,132,116,156, 146,121,126,149,149,149,149,138,
+ 94,39,97,117,138,45,88,117, 67,38,75,139,136,117,133,55, 139,78,153,138,143,54,128,55, 79,53,151,97,89,100,92,107,
+ 108,73,144,149,117,103,147,136, 162,65,59,127,136,135,143,70, 126,97,148,117,122,55,115,79, 109,140,154,117,114,112,100,115,
+ 148,128,120,108,115,137,133,110, 129,119,99,112,110,115,116,137, 127,147,98,107,93,131,122,130, 146,149,127,149,149,149,149,138,
+ 90,141,92,62,123,145,98,94, 71,157,54,115,84,131,112,151, 121,73,99,126,103,146,129,51, 68,144,73,93,87,100,88,107,
+ 137,80,87,78,75,97,105,85, 110,75,63,75,82,82,81,90, 99,126,66,86,72,87,81,105, 118,85,106,131,142,134,137,133,
+};
+
+static const uint8 ced_hires_21[1024] = {
+ 83,35,83,86,68,31,99,99, 55,99,74,45,80,93,87,75, 79,100,91,111,76,86,61,33, 125,101,49,100,98,111,105,129,
+ 59,43,112,116,111,56,117,121, 105,92,122,118,122,117,117,48, 113,112,111,124,119,105,131,14, 124,58,128,80,75,64,53,89,
+ 102,65,123,122,96,71,94,119, 105,106,73,93,110,138,129,71, 91,126,89,85,132,109,80,55, 63,122,92,108,98,117,104,108,
+ 87,29,40,62,58,112,27,40, 29,82,41,27,55,73,54,164, 60,39,74,78,77,82,45,14, 40,65,75,85,83,89,81,104,
+ 56,54,82,113,116,92,121,126, 133,122,107,112,124,114,119,69, 102,67,118,115,124,140,111,23, 115,119,76,70,57,61,58,92,
+ 78,62,95,25,124,109,90,119, 76,70,53,123,120,99,116,31, 107,29,119,104,112,46,98,15, 58,34,111,65,65,82,61,97,
+ 81,74,99,75,114,81,117,133, 79,96,101,129,126,104,112,55, 91,50,130,123,117,86,115,48, 57,41,32,98,76,95,82,105,
+ 107,147,94,49,39,141,43,49, 70,156,84,83,111,101,28,140, 58,35,42,54,90,129,55,41, 53,82,24,102,79,105,77,124,
+
+ 75,53,95,115,100,73,77,121, 43,85,59,58,103,133,102,46, 69,150,127,128,112,72,122,25, 117,58,87,80,75,79,76,97,
+ 52,103,109,123,121,131,123,117, 114,100,110,92,113,118,102,108, 129,127,111,123,117,103,115,21, 125,36,88,52,79,81,65,95,
+ 87,70,63,109,62,69,46,61, 46,69,77,67,83,135,128,27, 93,114,58,120,143,110,108,59, 114,61,69,92,90,96,100,95,
+ 109,61,71,65,81,104,96,64, 70,131,79,83,136,84,125,49, 77,107,101,97,105,80,101,61, 111,75,106,115,115,112,116,109,
+ 112,114,84,92,90,82,79,75, 83,89,111,97,91,110,108,93, 97,93,73,90,96,92,89,68, 88,77,101,154,114,122,143,121,
+ 53,137,103,128,119,76,111,90, 112,44,99,119,103,120,104,107, 107,110,85,115,119,68,113,4, 82,21,111,63,72,62,54,91,
+ 115,54,75,100,84,61,84,54, 83,100,70,85,116,112,133,43, 80,88,74,84,141,60,68,52, 81,99,81,113,103,108,109,105,
+ 120,111,126,115,133,122,111,89, 67,99,81,110,116,108,120,90, 90,163,89,121,125,100,107,119, 103,94,88,124,116,114,122,112,
+
+ 102,136,101,31,69,99,109,94, 95,151,120,85,100,94,78,59, 51,53,85,107,69,157,101,38, 64,67,59,99,103,108,95,116,
+ 79,150,19,27,24,119,22,24, 34,118,44,32,28,19,11,155, 18,43,0,29,14,101,25,9, 34,44,25,79,93,78,75,98,
+ 109,79,94,83,94,67,94,111, 63,103,71,67,102,112,111,62, 121,102,109,118,80,89,83,60, 119,63,60,126,109,130,110,127,
+ 52,53,108,90,118,45,88,105, 66,98,128,92,109,106,132,29, 112,78,109,93,96,54,103,149, 131,47,99,57,55,61,53,92,
+ 113,67,91,86,91,74,66,59, 71,104,115,78,126,131,125,92, 113,78,81,96,142,98,101,64, 76,66,103,108,98,123,99,112,
+ 107,81,112,47,95,153,81,88, 126,155,117,100,111,89,104,77, 131,53,97,95,109,142,93,38, 90,63,95,109,89,98,95,108,
+ 53,65,106,102,111,70,124,115, 115,98,121,126,113,107,115,63, 119,40,128,113,105,54,117,78, 77,126,127,58,62,74,59,98,
+ 130,70,67,61,62,63,68,65, 77,75,76,63,59,56,60,58, 74,89,44,52,52,89,72,70, 112,74,65,152,122,141,139,137,
+
+ 87,83,128,38,121,100,104,126, 80,90,140,138,111,106,105,56, 118,38,125,105,107,90,118,59, 57,141,42,74,97,86,75,106,
+ 113,89,93,86,69,80,73,66, 73,107,89,65,79,88,93,67, 86,81,65,83,79,83,81,62, 108,93,80,129,114,126,119,125,
+ 72,100,140,106,126,76,78,82, 98,91,119,103,121,124,113,68, 119,69,92,127,109,58,110,23, 82,57,114,116,85,81,78,92,
+ 114,71,89,89,85,92,85,77, 88,84,100,100,109,99,87,42, 81,78,116,95,146,76,98,54, 89,93,94,119,114,113,115,110,
+ 73,57,129,122,104,91,116,112, 130,80,66,119,115,115,115,33, 101,33,124,105,107,25,97,34, 70,127,127,84,81,81,57,94,
+ 74,41,117,137,107,32,76,46, 113,50,108,113,93,131,86,33, 121,33,104,108,103,103,115,9, 34,85,117,79,74,86,80,101,
+ 105,136,69,50,52,146,69,59, 68,160,143,98,90,95,36,91, 54,64,94,68,86,132,126,39, 89,121,50,110,105,109,106,138,
+ 119,64,72,56,60,96,83,58, 74,75,59,61,65,63,48,59, 78,73,49,68,62,65,54,71, 69,57,57,111,119,111,115,117,
+};
+
+
+
diff --git a/contrib/google-ced/compact_enc_det_hint_code.cc b/contrib/google-ced/compact_enc_det_hint_code.cc
new file mode 100644
index 0000000..7e7b77a
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_hint_code.cc
@@ -0,0 +1,169 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "compact_enc_det_hint_code.h"
+
+#include <ctype.h> // for isalpha
+#include <string.h> // for NULL, memchr, strlen, etc
+
+#include "util/basictypes.h" // for uint8, uint32
+#include "util/string_util.h"
+
+// Upper to lower, keep digits, everything else to minus '-' (2d)
+static const char kCharsetToLowerTbl[256] = {
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37, 0x38,0x39,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+ 0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+ 0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+ 0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+};
+
+
+static const char kIsAlpha[256] = {
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+ 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char kIsDigit[256] = {
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 1,1,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char* kFakeEncodingName[] = {
+ "FakeEnc100", "FakeEnc101", "FakeEnc102", "FakeEnc103", "FakeEnc104",
+ "FakeEnc105", "FakeEnc106", "FakeEnc107", "FakeEnc108", "FakeEnc109",
+ "FakeEnc110", "FakeEnc111", "FakeEnc112", "FakeEnc113", "FakeEnc114",
+ "FakeEnc115", "FakeEnc116", "FakeEnc117", "FakeEnc118", "FakeEnc119",
+};
+static const char* kFakeEncodingName2[] = {
+ "FakeEnc_0", "FakeEnc_1", "FakeEnc_2", "FakeEnc_3", "FakeEnc_4",
+};
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc) {
+ if (enc < 0) {
+ return "~";
+ }
+ if (enc == ISO_8859_1) {
+ return "Latin1"; // I can't stand "ASCII" for this
+ }
+ if (enc < NUM_ENCODINGS) {
+ return EncodingName(enc);
+ }
+ // allow fake names, for exploration
+ if ((NUM_ENCODINGS <= enc) && (enc < (NUM_ENCODINGS + 4))) {
+ return kFakeEncodingName2[enc - NUM_ENCODINGS];
+ }
+ if ((100 <= enc) && (enc < 120)) {
+ return kFakeEncodingName[enc - 100];
+ }
+ return "~";
+}
+
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str) {
+ string res("________"); // eight underscores
+ int l_ptr = 0;
+ int d_ptr = 0;
+ for (uint32 i = 0; i < str.size(); ++i) {
+ uint8 uc = static_cast<uint8>(str[i]);
+ if (kIsAlpha[uc]) {
+ if (l_ptr < 4) { // Else ignore
+ res[l_ptr] = kCharsetToLowerTbl[uc];
+ l_ptr++;
+ }
+ } else if (kIsDigit[uc]) {
+ if (d_ptr < 4) {
+ res[4 + d_ptr] = kCharsetToLowerTbl[uc];
+ } else {
+ // Keep last 4 digits by shifting left
+ res[4] = res[5];
+ res[5] = res[6];
+ res[6] = res[7];
+ res[7] = kCharsetToLowerTbl[uc];
+ }
+ d_ptr++;
+ } // If neither letter nor digit, drop entirely
+ }
+ return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str) {
+ string res("____"); // four underscores
+ int l_ptr = 0;
+ for (uint32 i = 0; i < str.size(); ++i) {
+ uint8 uc = static_cast<uint8>(str[i]);
+ if (kIsAlpha[uc] | kIsDigit[uc]) {
+ if (l_ptr < 4) { // Else ignore
+ res[l_ptr] = kCharsetToLowerTbl[uc];
+ l_ptr++;
+ }
+ }
+ }
+ return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar8(const string& str) {
+ string res("________"); // eight dots
+ int l_ptr = 0;
+ for (uint32 i = 0; i < str.size(); ++i) {
+ uint8 uc = static_cast<uint8>(str[i]);
+ if (kIsAlpha[uc] | kIsDigit[uc]) {
+ if (l_ptr < 8) { // Else ignore
+ res[l_ptr] = kCharsetToLowerTbl[uc];
+ l_ptr++;
+ }
+ }
+ }
+ return res;
+}
diff --git a/contrib/google-ced/compact_enc_det_hint_code.h b/contrib/google-ced/compact_enc_det_hint_code.h
new file mode 100644
index 0000000..53fe67a
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_hint_code.h
@@ -0,0 +1,45 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+
+#include <string> // for string
+
+#include "util/basictypes.h" // for uint32
+#include "util/encodings/encodings.h" // for Encoding
+
+using std::string;
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc);
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str);
+
+// Normalize ASCII string to first 4 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str);
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize other values
+string MakeChar8(const string& str);
+
+#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
diff --git a/contrib/google-ced/detail_head_string.inc b/contrib/google-ced/detail_head_string.inc
new file mode 100644
index 0000000..e70e5ff
--- /dev/null
+++ b/contrib/google-ced/detail_head_string.inc
@@ -0,0 +1,152 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+// Produced by stringify.cc on 2007-09-28 09:13 from file i18n/encodings/compact_enc_det/tools/detail_head.ps
+ "%!PS-Adobe-2.0\n\n/inch {72 mul} def\n/cshow {dup stringwidth pop -2 div"
+ " 0 rmoveto show} def\n\n/lmargin 0.5 inch def\n/rmargin 8.5 inch def\n/t"
+ "margin 10.5 inch def\n/bmargin 0.5 inch def\n\n\n% set to N>=0 to track "
+ "ranked encoding N\n/track-me -1 def\n/track-me2 -1 def\n\n/columns 2 def"
+ "\n\n/lpi 18 def % lines per inch\n/lpc lpi 10 mul def "
+ " % lines per column\n/lpp lpc columns mul def % lines per page\n"
+ "/probw 3.0 inch def % probability width\n/probr 50 def "
+ " % probability range\n/widowlines lpi 2 idiv def % 1/2 inch widow a"
+ "t bottom\n/widowlines lpi def % 1 inch widow at bottom\n\n/lpg l"
+ "pi 2 idiv def % 1/2 inch spacing between groups\n\n/delc 4 inch "
+ "def\n/dell -1 inch lpi div def\n\n/next-line 0 def % 24 lines per i"
+ "nch, 240 per column\n\n/Cfont /Courier findfont 6 scalefont def\n/Hfont "
+ "/Helvetica findfont 6 scalefont def\nHfont setfont\n\n\n% simple string "
+ "hash -- sum the characters\n/strhash {\n /hstr exch def\n /h 0 def\n "
+ "0 1 hstr length 1 sub {/i exch def /h h hstr i get add def} for\n h\n}"
+ " def\n\n% convert pro at 30 per 2x to 0-2.5 inches spanning -50 to 0\n/p"
+ "rob2x {\n 30 idiv probr div probw mul neg probw add\n}def\n\n\n/cliptoc"
+ "olumn {\n % ====== MUST MATCH ME ======\n gsave\n lmargin tmargin mov"
+ "eto\n next-line lpc idiv delc mul 0 rmoveto\n -1 18 rmoveto 0 -10.5 in"
+ "ch rlineto delc 2 add 0 rlineto 0 10.5 inch rlineto closepath\n clip\n"
+ " newpath\n} def\n\n/endcliptocolumn {\n grestore\n % ====== MUST MATC"
+ "H ME ======\n} def\n\n\n\n/column-box {\n lmargin tmargin moveto\n nex"
+ "t-line 1 sub lpc idiv delc mul next-line 1 sub lpc mod 1 add dell mul r"
+ "moveto\n % box\n gsave -1.5 0 rmoveto 0 detail-count dell mul neg rmov"
+ "eto probw 3 add 0 rlineto 0.25 setlinewidth stroke grestore\n gsave -1"
+ ".5 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth strok"
+ "e grestore\n gsave probw .8 mul 0 rmoveto 0 detail-count dell mul neg r"
+ "lineto 0.25 setlinewidth 0.8 setgray stroke grestore\n gsave probw .6 m"
+ "ul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth 0.8 s"
+ "etgray stroke grestore\n gsave probw .4 mul 0 rmoveto 0 detail-count de"
+ "ll mul neg rlineto 0.25 setlinewidth 0.8 setgray stroke grestore\n gsav"
+ "e probw .2 mul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlin"
+ "ewidth 0.8 setgray stroke grestore\n gsave probw 1.5 add 0 rmoveto 0 de"
+ "tail-count dell mul neg rlineto 0.25 setlinewidth stroke grestore\n} def"
+ "\n\n\n/IncrementLine {\n /incr exch def\n /next-line next-line incr ad"
+ "d def\n next-line lpc ge next-line incr sub lpc lt and {\n % We "
+ "just went to the top of column 2; redo clip\n endcliptocolumn % M"
+ "UST match\n column-box\n /next-line lpc def\n cliptocolumn "
+ " % MUST match\n } if\n next-line lpp ge {\n % We just went to the"
+ " top of column 3; start new page column 1\n endcliptocolumn % MUS"
+ "T match\n column-box\n showpage\n Hfont setfont\n /next-line"
+ " 0 def\n show-pageno\n cliptocolumn % MUST match\n } if\n}"
+ " def\n\n/IncrementLineOutside {\n /incr exch def\n /next-line next-lin"
+ "e incr add def\n next-line lpc ge next-line incr sub lpc lt and {\n"
+ " % We just went to the top of column 2\n /next-line lpc def\n } i"
+ "f\n next-line lpp ge {\n % We just went to the top of column 3; star"
+ "t new page column 1\n showpage\n Hfont setfont\n /next-line 0 d"
+ "ef\n show-pageno\n } if\n} def\n\n/NextColumn {\n lpc 1 sub Increme"
+ "ntLine\n} def\n\n/NextPage {\n lpp 1 sub IncrementLine\n} def\n\n% Up"
+ "on entry, we are OUTSIDE the clip\n/start-detail {\n /d-title exch def\n"
+ "\n % align >= 1 inch at bottom of column, and/or start new column\n lp"
+ "c next-line lpc mod sub widowlines lt {\n % Start at top of a column\n"
+ " next-line lpc ge {\n % Start new page\n showpage\n Hf"
+ "ont setfont\n /next-line 0 def\n show-pageno\n } {\n %"
+ " Start new column\n /next-line lpc def\n } ifelse\n } if\n\n l"
+ "margin tmargin moveto\n next-line lpc idiv delc mul next-line lpc mod "
+ "dell mul rmoveto\n gsave d-title show grestore\n 0 dell rmoveto\n 1 1"
+ " 4 {/j exch def gsave probw j mul 5 div -2 rmoveto 50 j 10 mul sub 20 st"
+ "ring cvs cshow grestore} for\n 2 IncrementLineOutside\n /detail-count "
+ "1 def\n cliptocolumn % MUST match\n /d-array [] def\n} def\n"
+ "\n/size-detail {\n /d-names exch def\n /d-size exch def\n % zero sums"
+ "\n /sums d-size array def\n 0 1 d-size 1 sub {/i exch def sums i 0 pu"
+ "t} for\n /old-d-max 0 def\n /colors d-size array def\n 0 1 d-size 1 s"
+ "ub {/i exch def colors i i 3 mul 17 mod 17 div put} for\n %0 1 d-size "
+ "1 sub {/i exch def colors i d-names i get strhash 3 mul 17 mod 17 div p"
+ "ut} for\n %0 1 d-size 1 sub {/i exch def ( ) show colors i get 20 stri"
+ "ng cvs show} for\n} def\n\n/count-detail {\n /detail-total-count exch d"
+ "ef\n % if total-count >= one column, start at top of a column\n detail"
+ "-total-count lpp ge {\n % Start new page\n NextPage\n } {\n de"
+ "tail-total-count lpc ge {\n % Start new column\n NextColumn\n "
+ " } if\n } ifelse\n} def\n\n% highlight next entry with underbar\n/do-"
+ "flag {\ngsave\n setrgbcolor\n lmargin tmargin moveto\n next-line lpc "
+ "idiv delc mul next-line lpc mod dell mul rmoveto\n 0 -2 rmoveto\n pro"
+ "bw 0 rlineto\n 0 dell neg rlineto\n probw neg 0 rlineto\n closepath\n"
+ " fill\ngrestore\n} def\n\n/do-detail-e {\n /d-array exch def\n /d-enc"
+ " exch def\n /d-label exch def\n /d-max -999999 def\n\n lmargin tmargi"
+ "n moveto\n next-line lpc idiv delc mul next-line lpc mod dell mul rmov"
+ "eto\n 0.25 setlinewidth\n\n % show label, using encoding color\n gsav"
+ "e\n probw 2 add -2 rmoveto\n detail-count 1 sub 2 mod 0.25 inch mu"
+ "l 0 rmoveto\n % ([) show detail-count 20 string cvs show (] ) show\n "
+ " d-enc 0 lt {\n 0 setgray\n }{\n colors d-enc get 1 .8 se"
+ "thsbcolor\n } ifelse\n d-label show\n grestore\n % For -prune- d"
+ "raw horizontal line\n d-label length 8 gt {d-label 4 get (p) 0 get eq d"
+ "-label 5 get (r) 0 get eq and {\n /prune-val d-label cvi def\n /ne"
+ "wx prune-val 30 mul prob2x def\n gsave newx 6 rmoveto 0 -12 rlineto 1"
+ ".5 setlinewidth 0.8 setgray stroke grestore\n gsave probw 0 add 0 rli"
+ "neto 0.25 setlinewidth 0.8 setgray stroke grestore\n } if } if\n\n % t"
+ "rack max per new row\n 0 1 d-array length 1 sub {\n /i exch def\n "
+ " /sum sums i get d-array i get add def\n d-max sum lt {/d-max sum def"
+ "} if\n } for\n\n % draw line increments\n 0 1 d-array length 1 sub {\n"
+ " /i exch def\n detail-count 1 gt {\n /oldx old-d-max sums i g"
+ "et sub prob2x def\n } {\n /oldx 600 prob2x def\n } ifelse\n "
+ " /oldy dell neg def\n /newx d-max sums i get d-array i get add sub"
+ " prob2x def\n /newy 0 def\n gsave\n oldx oldy rmoveto\n newx"
+ " oldx sub newy oldy sub rlineto\n % if encoding is being tracked, ma"
+ "ke bold line\n i track-me eq\n {2 setlinewidth}\n {i track-"
+ "me2 eq {1.25 setlinewidth} {0.25 setlinewidth} ifelse}\n ifelse\n "
+ " colors i get 1 .8 sethsbcolor stroke\n grestore\n } for\n /detail"
+ "-count detail-count 1 add def\n\n % increment running total in sums, tr"
+ "ack max per row\n 0 1 d-array length 1 sub {\n /i exch def\n sums"
+ " i sums i get d-array i get add put\n } for\n /old-d-max d-max def\n"
+ "\n 1 IncrementLine\n} def\n\n\n/do-detail {\n /d-array exch def\n /d-"
+ "label exch def\n d-label -1 d-array do-detail-e\n} def\n\n% Upon exit, "
+ "we are outside the clip\n/end-detail {\n pop\n endcliptocolumn "
+ " % MUST match\n column-box\n\n % text labels\n 0 1 d-array length 1 s"
+ "ub {\n /i exch def\n gsave\n /newx old-d-max sums i get sub pro"
+ "b2x def\n newx 0 ge {\n newx 0 rmoveto\n currentpoint trans"
+ "late\n colors i get 1 .8 sethsbcolor\n gsave 0 dell neg rline"
+ "to 0.25 setlinewidth stroke grestore\n -60 rotate\n 0 -2 movet"
+ "o d-names i get show\n } if\n grestore\n } for\n d-array length "
+ "0 gt {\n lpg IncrementLineOutside\n } {\n lpg 4 idiv IncrementLin"
+ "eOutside\n } ifelse\n} def\n\n/do-src {\n/src exch def\n lmargin tmarg"
+ "in moveto\n next-line lpc idiv delc mul next-line lpc mod dell mul rmo"
+ "veto\n Cfont setfont\n src show\n Hfont setfont\n 1 IncrementLine\n}"
+ " def\n\n% Underline trigram in source text\n/do-highlight1 {\n /hl-colo"
+ "r exch def\n /hl-offset exch def\n /hl-line exch 1 sub 2 mul def\n gs"
+ "ave\n lmargin tmargin moveto\n next-line hl-line sub lpc idiv delc mul"
+ "\n next-line hl-line sub lpc mod dell mul rmoveto\n % Assume text is 6"
+ " chars in and 3.6 pts per char, but 2 chars per offset\n hl-offset 2 mu"
+ "l 6 add 3.6 mul 4 rmoveto\n\n 0 setgray 0.5 setlinewidth\n hl-color 1"
+ " eq {0 0 1 setrgbcolor} if % Latin1 blue\n hl-color 2 eq {1 0 1 setrgb"
+ "color} if % Latin2 magenta\n hl-color 3 eq {1 0.67 0 setrgbcolor} if "
+ "% Latin7 orange\n 18 -2 rlineto stroke\n grestore\n} def\n\n% Box trig"
+ "ram in source text\n/do-highlight2 {\n /hl-color exch def\n /hl-offset"
+ " exch def\n /hl-line exch 1 sub 2 mul def\n gsave\n lmargin tmargin m"
+ "oveto\n next-line hl-line sub lpc idiv delc mul\n next-line hl-line su"
+ "b lpc mod dell mul rmoveto\n % Assume text is 6 chars in and 3.6 pts pe"
+ "r char, but 2 chars per offset\n hl-offset 2 mul 6 add 3.6 mul 4 rmove"
+ "to\n\n 0 setgray 0.25 setlinewidth\n hl-color 1 eq {0 0 1 setrgbcolor}"
+ " if % Latin1 blue\n hl-color 2 eq {1 0 1 setrgbcolor} if % Latin2 mag"
+ "enta\n hl-color 3 eq {1 0.67 0 setrgbcolor} if % Latin7 orange\n -0.5"
+ " -0.5 rmoveto\n 22 0 rlineto\n 0 4 rlineto\n -11 2 rlineto\n -11 -2 "
+ "rlineto\n closepath\n stroke\n grestore\n} def\n\n/show-pageno {\ngsa"
+ "ve\nlmargin bmargin moveto 0 -12 rmoveto\n(Page ) show pageno 20 string "
+ "cvs show\ngrestore\n/pageno pageno 1 add def\n} def\n\n/pageno 1 def\nsh"
+ "ow-pageno\n%=============================\n\n\n"
diff --git a/contrib/google-ced/util/basictypes.h b/contrib/google-ced/util/basictypes.h
new file mode 100644
index 0000000..af391c7
--- /dev/null
+++ b/contrib/google-ced/util/basictypes.h
@@ -0,0 +1,331 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_BASICTYPES_H_
+#define UTIL_BASICTYPES_H_
+
+#include <limits.h> // So we can set the bounds of our types
+#include <stddef.h> // For size_t
+#include <string.h> // for memcpy
+
+#include "util/port.h" // Types that only need exist on certain systems
+
+#ifndef COMPILER_MSVC
+// stdint.h is part of C99 but MSVC doesn't have it.
+#include <stdint.h> // For intptr_t.
+#endif
+
+typedef signed char schar;
+typedef signed char int8;
+typedef short int16;
+// TODO(mbelshe) Remove these type guards. These are
+// temporary to avoid conflicts with npapi.h.
+#ifndef _INT32
+#define _INT32
+typedef int int32;
+#endif
+
+// The NSPR system headers define 64-bit as |long| when possible. In order to
+// not have typedef mismatches, we do the same on LP64.
+#if __LP64__
+typedef long int64;
+#else
+typedef long long int64;
+#endif
+
+// NOTE: unsigned types are DANGEROUS in loops and other arithmetical
+// places. Use the signed types unless your variable represents a bit
+// pattern (eg a hash value) or you really need the extra bit. Do NOT
+// use 'unsigned' to express "this value should always be positive";
+// use assertions for this.
+
+typedef unsigned char uint8;
+typedef unsigned short uint16;
+// TODO(mbelshe) Remove these type guards. These are
+// temporary to avoid conflicts with npapi.h.
+#ifndef _UINT32
+#define _UINT32
+typedef unsigned int uint32;
+#endif
+
+// See the comment above about NSPR and 64-bit.
+#if __LP64__
+typedef unsigned long uint64;
+#else
+typedef unsigned long long uint64;
+#endif
+
+// A type to represent a Unicode code-point value. As of Unicode 4.0,
+// such values require up to 21 bits.
+// (For type-checking on pointers, make this explicitly signed,
+// and it should always be the signed version of whatever int32 is.)
+typedef signed int char32;
+
+const uint8 kuint8max = (( uint8) 0xFF);
+const uint16 kuint16max = ((uint16) 0xFFFF);
+const uint32 kuint32max = ((uint32) 0xFFFFFFFF);
+const uint64 kuint64max = ((uint64) GG_LONGLONG(0xFFFFFFFFFFFFFFFF));
+const int8 kint8min = (( int8) 0x80);
+const int8 kint8max = (( int8) 0x7F);
+const int16 kint16min = (( int16) 0x8000);
+const int16 kint16max = (( int16) 0x7FFF);
+const int32 kint32min = (( int32) 0x80000000);
+const int32 kint32max = (( int32) 0x7FFFFFFF);
+const int64 kint64min = (( int64) GG_LONGLONG(0x8000000000000000));
+const int64 kint64max = (( int64) GG_LONGLONG(0x7FFFFFFFFFFFFFFF));
+
+// A macro to disallow the copy constructor and operator= functions
+// This should be used in the private: declarations for a class
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+ TypeName(const TypeName&); \
+ void operator=(const TypeName&)
+
+// An older, deprecated, politically incorrect name for the above.
+#define DISALLOW_EVIL_CONSTRUCTORS(TypeName) DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// A macro to disallow all the implicit constructors, namely the
+// default constructor, copy constructor and operator= functions.
+//
+// This should be used in the private: declarations for a class
+// that wants to prevent anyone from instantiating it. This is
+// especially useful for classes containing only static methods.
+#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \
+ TypeName(); \
+ DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// The arraysize(arr) macro returns the # of elements in an array arr.
+// The expression is a compile-time constant, and therefore can be
+// used in defining new arrays, for example. If you use arraysize on
+// a pointer by mistake, you will get a compile-time error.
+
+// This template function declaration is used in defining arraysize.
+// Note that the function doesn't need an implementation, as we only
+// use its type.
+template <typename T, size_t N>
+char (&ArraySizeHelper(T (&array)[N]))[N];
+
+// That gcc wants both of these prototypes seems mysterious. VC, for
+// its part, can't decide which to use (another mystery). Matching of
+// template overloads: the final frontier.
+#ifndef _MSC_VER
+template <typename T, size_t N>
+char (&ArraySizeHelper(const T (&array)[N]))[N];
+#endif
+
+#define arraysize(array) (sizeof(ArraySizeHelper(array)))
+
+
+// Use implicit_cast as a safe version of static_cast or const_cast
+// for upcasting in the type hierarchy (i.e. casting a pointer to Foo
+// to a pointer to SuperclassOfFoo or casting a pointer to Foo to
+// a const pointer to Foo).
+// When you use implicit_cast, the compiler checks that the cast is safe.
+// Such explicit implicit_casts are necessary in surprisingly many
+// situations where C++ demands an exact type match instead of an
+// argument type convertable to a target type.
+//
+// The From type can be inferred, so the preferred syntax for using
+// implicit_cast is the same as for static_cast etc.:
+//
+// implicit_cast<ToType>(expr)
+//
+// implicit_cast would have been part of the C++ standard library,
+// but the proposal was submitted too late. It will probably make
+// its way into the language in the future.
+template<typename To, typename From>
+inline To implicit_cast(From const &f) {
+ return f;
+}
+
+// The COMPILE_ASSERT macro can be used to verify that a compile time
+// expression is true. For example, you could use it to verify the
+// size of a static array:
+//
+// COMPILE_ASSERT(arraysize(content_type_names) == CONTENT_NUM_TYPES,
+// content_type_names_incorrect_size);
+//
+// or to make sure a struct is smaller than a certain size:
+//
+// COMPILE_ASSERT(sizeof(foo) < 128, foo_too_large);
+//
+// The second argument to the macro is the name of the variable. If
+// the expression is false, most compilers will issue a warning/error
+// containing the name of the variable.
+
+template <bool>
+struct CompileAssert {
+};
+
+#undef COMPILE_ASSERT
+#define COMPILE_ASSERT(expr, msg) \
+ typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1]
+
+// Implementation details of COMPILE_ASSERT:
+//
+// - COMPILE_ASSERT works by defining an array type that has -1
+// elements (and thus is invalid) when the expression is false.
+//
+// - The simpler definition
+//
+// #define COMPILE_ASSERT(expr, msg) typedef char msg[(expr) ? 1 : -1]
+//
+// does not work, as gcc supports variable-length arrays whose sizes
+// are determined at run-time (this is gcc's extension and not part
+// of the C++ standard). As a result, gcc fails to reject the
+// following code with the simple definition:
+//
+// int foo;
+// COMPILE_ASSERT(foo, msg); // not supposed to compile as foo is
+// // not a compile-time constant.
+//
+// - By using the type CompileAssert<(bool(expr))>, we ensures that
+// expr is a compile-time constant. (Template arguments must be
+// determined at compile-time.)
+//
+// - The outter parentheses in CompileAssert<(bool(expr))> are necessary
+// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written
+//
+// CompileAssert<bool(expr)>
+//
+// instead, these compilers will refuse to compile
+//
+// COMPILE_ASSERT(5 > 0, some_message);
+//
+// (They seem to think the ">" in "5 > 0" marks the end of the
+// template argument list.)
+//
+// - The array size is (bool(expr) ? 1 : -1), instead of simply
+//
+// ((expr) ? 1 : -1).
+//
+// This is to avoid running into a bug in MS VC 7.1, which
+// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1.
+
+
+// MetatagId refers to metatag-id that we assign to
+// each metatag <name, value> pair..
+typedef uint32 MetatagId;
+
+// Argument type used in interfaces that can optionally take ownership
+// of a passed in argument. If TAKE_OWNERSHIP is passed, the called
+// object takes ownership of the argument. Otherwise it does not.
+enum Ownership {
+ DO_NOT_TAKE_OWNERSHIP,
+ TAKE_OWNERSHIP
+};
+
+// bit_cast<Dest,Source> is a template function that implements the
+// equivalent of "*reinterpret_cast<Dest*>(&source)". We need this in
+// very low-level functions like the protobuf library and fast math
+// support.
+//
+// float f = 3.14159265358979;
+// int i = bit_cast<int32>(f);
+// // i = 0x40490fdb
+//
+// The classical address-casting method is:
+//
+// // WRONG
+// float f = 3.14159265358979; // WRONG
+// int i = * reinterpret_cast<int*>(&f); // WRONG
+//
+// The address-casting method actually produces undefined behavior
+// according to ISO C++ specification section 3.10 -15 -. Roughly, this
+// section says: if an object in memory has one type, and a program
+// accesses it with a different type, then the result is undefined
+// behavior for most values of "different type".
+//
+// This is true for any cast syntax, either *(int*)&f or
+// *reinterpret_cast<int*>(&f). And it is particularly true for
+// conversions betweeen integral lvalues and floating-point lvalues.
+//
+// The purpose of 3.10 -15- is to allow optimizing compilers to assume
+// that expressions with different types refer to different memory. gcc
+// 4.0.1 has an optimizer that takes advantage of this. So a
+// non-conforming program quietly produces wildly incorrect output.
+//
+// The problem is not the use of reinterpret_cast. The problem is type
+// punning: holding an object in memory of one type and reading its bits
+// back using a different type.
+//
+// The C++ standard is more subtle and complex than this, but that
+// is the basic idea.
+//
+// Anyways ...
+//
+// bit_cast<> calls memcpy() which is blessed by the standard,
+// especially by the example in section 3.9 . Also, of course,
+// bit_cast<> wraps up the nasty logic in one place.
+//
+// Fortunately memcpy() is very fast. In optimized mode, with a
+// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
+// code with the minimal amount of data movement. On a 32-bit system,
+// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
+// compiles to two loads and two stores.
+//
+// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
+//
+// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
+// is likely to surprise you.
+
+template <class Dest, class Source>
+inline Dest bit_cast(const Source& source) {
+ // Compile time assertion: sizeof(Dest) == sizeof(Source)
+ // A compile error here means your Dest and Source have different sizes.
+ // typedef char VerifySizesAreEqual [sizeof(Dest) == sizeof(Source) ? 1 : -1];
+
+ Dest dest;
+ memcpy(&dest, &source, sizeof(dest));
+ return dest;
+}
+
+// The following enum should be used only as a constructor argument to indicate
+// that the variable has static storage class, and that the constructor should
+// do nothing to its state. It indicates to the reader that it is legal to
+// declare a static instance of the class, provided the constructor is given
+// the base::LINKER_INITIALIZED argument. Normally, it is unsafe to declare a
+// static variable that has a constructor or a destructor because invocation
+// order is undefined. However, IF the type can be initialized by filling with
+// zeroes (which the loader does for static variables), AND the destructor also
+// does nothing to the storage, AND there are no virtual methods, then a
+// constructor declared as
+// explicit MyClass(base::LinkerInitialized x) {}
+// and invoked as
+// static MyClass my_variable_name(base::LINKER_INITIALIZED);
+namespace base {
+enum LinkerInitialized { LINKER_INITIALIZED };
+} // base
+
+// UnaligndLoad32 is put here instead of util/port.h to
+// avoid the circular dependency between port.h and basictypes.h
+// ARM does not support unaligned memory access.
+#if defined(ARCH_CPU_X86_FAMILY)
+// x86 and x86-64 can perform unaligned loads/stores directly;
+inline uint32 UnalignedLoad32(const void* p) {
+ return *reinterpret_cast<const uint32*>(p);
+}
+#else
+#define NEED_ALIGNED_LOADS
+// If target architecture does not support unaligned loads and stores,
+// use memcpy version of UNALIGNED_LOAD32.
+inline uint32 UnalignedLoad32(const void* p) {
+ uint32 t;
+ memcpy(&t, reinterpret_cast<const uint8*>(p), sizeof(t));
+ return t;
+}
+
+#endif
+#endif // UTIL_BASICTYPES_H_
diff --git a/contrib/google-ced/util/case_insensitive_hash.h b/contrib/google-ced/util/case_insensitive_hash.h
new file mode 100644
index 0000000..7b0c9db
--- /dev/null
+++ b/contrib/google-ced/util/case_insensitive_hash.h
@@ -0,0 +1,88 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_CASE_INSENSITIVE_HASH_H_
+#define UTIL_CASE_INSENSITIVE_HASH_H_
+
+#include <ctype.h>
+#include <stddef.h>
+#ifndef _MSC_VER
+#include <strings.h>
+#endif
+
+#include <string>
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+
+// Functors for hashing c-strings with case-insensitive semantics.
+struct CStringCaseHash {
+ size_t operator()(const char *str) const {
+ unsigned long hash_val = 0;
+ while (*str) {
+ hash_val = 5*hash_val + tolower(*str);
+ str++;
+ }
+ return (size_t)hash_val;
+ }
+};
+
+struct CStringCaseEqual {
+ bool operator()(const char *str1, const char *str2) const {
+ return !base::strcasecmp(str1, str2);
+ }
+};
+
+// These functors, in addition to being case-insensitive, ignore all
+// non-alphanumeric characters. This is useful when we want all variants of
+// a string -- where variants can differ in puncutation and whitespace -- to
+// map to the same value.
+struct CStringAlnumCaseHash {
+ size_t operator()(const char *str) const {
+ unsigned long hash_val = 0;
+ while (*str) {
+ if (isalnum(*str)) {
+ hash_val = 5*hash_val + tolower(*str);
+ }
+ str++;
+ }
+ return (size_t)hash_val;
+ }
+};
+
+struct CStringAlnumCaseEqual {
+ bool operator()(const char *str1, const char *str2) const {
+ while (true) {
+ // Skip until each pointer is pointing to an alphanumeric char or '\0'
+ while (!isalnum(*str1) && (*str1 != '\0')) {
+ str1++;
+ }
+ while (!isalnum(*str2) && (*str2 != '\0')) {
+ str2++;
+ }
+ if (tolower(*str1) != tolower(*str2)) {
+ return false; // mismatch on alphanumeric char or '\0'
+ }
+ if (*str1 == '\0') { // in which case *str2 must be '\0' as well
+ return true; // reached '\0' in both strings without mismatch
+ }
+ str1++;
+ str2++;
+ }
+ }
+};
+
+#endif // UTIL_CASE_INSENSITIVE_HASH_H_
diff --git a/contrib/google-ced/util/commandlineflags.h b/contrib/google-ced/util/commandlineflags.h
new file mode 100644
index 0000000..341a659
--- /dev/null
+++ b/contrib/google-ced/util/commandlineflags.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_COMMANDLINEFLAGS_H_
+#define UTIL_COMMANDLINEFLAGS_H_
+
+
+#undef DEFINE_bool
+#define DEFINE_bool(name, default_value, comment) \
+ bool FLAGS_##name = default_value
+#undef DEFINE_int32
+#define DEFINE_int32(name, default_value, comment) \
+ int32 FLAGS_##name = default_value
+#undef DEFINE_string
+#define DEFINE_string(name, default_value, comment) \
+ string FLAGS_##name = default_value
+
+#undef DECLARE_bool
+#define DECLARE_bool(name) extern bool FLAGS_##name
+#undef DECLARE_int32
+#define DECLARE_int32(name) extern int32 FLAGS_##name
+#undef DECLARE_string
+#define DECLARE_string(name) extern string FLAGS_##name
+
+
+#endif // UTIL_COMMANDLINEFLAGS_H_
diff --git a/contrib/google-ced/util/encodings/encodings.cc b/contrib/google-ced/util/encodings/encodings.cc
new file mode 100644
index 0000000..b5f8dc5
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings.cc
@@ -0,0 +1,891 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/encodings/encodings.h"
+
+#include <string.h> // for strcasecmp
+#include <unordered_map>
+#include <utility> // for pair
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+#include "util/case_insensitive_hash.h"
+
+struct EncodingInfo {
+ // The standard name for this encoding.
+ //
+ const char* encoding_name_;
+
+ // The "preferred MIME name" of an encoding as specified by the IANA at:
+ // http://www.iana.org/assignments/character-sets
+ //
+ // Note that the preferred MIME name may differ slightly from the
+ // official IANA name: i.e. ISO-8859-1 vs. ISO_8859-1:1987
+ //
+ const char* mime_encoding_name_;
+
+ // It is an internal policy that if an encoding has an IANA name,
+ // then encoding_name_ and mime_encoding_name_ must be the same string.
+ //
+ // However, there can be exceptions if there are compelling reasons.
+ // For example, Japanese mobile handsets require the name
+ // "Shift_JIS" in charset=... parameter in Content-Type headers to
+ // process emoji (emoticons) in their private encodings. In that
+ // case, mime_encoding_name_ should be "Shift_JIS", despite
+ // encoding_name_ actually is "X-KDDI-Shift_JIS".
+
+ // Some multi-byte encodings use byte values that coincide with the
+ // ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
+ // can misinterpret these, as indicated in an external XSS report from
+ // 2007-02-15. Here, we map these dangerous encodings to safer ones. We
+ // also use UTF8 instead of encodings that we don't support in our
+ // output, and we generally try to be conservative in what we send out.
+ // Where the client asks for single- or double-byte encodings that are
+ // not as common, we substitute a more common single- or double-byte
+ // encoding, if there is one, thereby preserving the client's intent
+ // to use less space than UTF-8. This also means that characters
+ // outside the destination set will be converted to HTML NCRs (&#NNN;)
+ // if requested.
+
+ Encoding preferred_web_output_encoding_;
+};
+
+static const EncodingInfo kEncodingInfoTable[] = {
+ { "ASCII", "ISO-8859-1", ISO_8859_1},
+ { "Latin2", "ISO-8859-2", ISO_8859_2},
+ { "Latin3", "ISO-8859-3", UTF8},
+ // MSIE 6 does not support ISO-8859-3 (XSS issue)
+ { "Latin4", "ISO-8859-4", ISO_8859_4},
+ { "ISO-8859-5", "ISO-8859-5", ISO_8859_5},
+ { "Arabic", "ISO-8859-6", ISO_8859_6},
+ { "Greek", "ISO-8859-7", ISO_8859_7},
+ { "Hebrew", "ISO-8859-8", MSFT_CP1255},
+ // we do not endorse the visual order
+ { "Latin5", "ISO-8859-9", ISO_8859_9},
+ { "Latin6", "ISO-8859-10", UTF8},
+ // MSIE does not support ISO-8859-10 (XSS issue)
+ { "EUC-JP", "EUC-JP", JAPANESE_EUC_JP},
+ { "SJS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ { "JIS", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+ // due to potential confusion with HTML syntax chars
+ { "BIG5", "Big5", CHINESE_BIG5},
+ { "GB", "GB2312", CHINESE_GB},
+ { "EUC-CN",
+ "EUC-CN",
+ // Misnamed. Should be EUC-TW.
+ CHINESE_BIG5},
+ // MSIE treats "EUC-CN" like GB2312, which is not EUC-TW,
+ // and EUC-TW is rare, so we prefer Big5 for output.
+ { "KSC", "EUC-KR", KOREAN_EUC_KR},
+ { "Unicode",
+ "UTF-16LE",
+ // Internet Explorer doesn't recognize "ISO-10646-UCS-2"
+ UTF8
+ // due to potential confusion with HTML syntax chars
+ },
+ { "EUC",
+ "EUC", // Misnamed. Should be EUC-TW.
+ CHINESE_BIG5
+ // MSIE does not recognize "EUC" (XSS issue),
+ // and EUC-TW is rare, so we prefer Big5 for output.
+ },
+ { "CNS",
+ "CNS", // Misnamed. Should be EUC-TW.
+ CHINESE_BIG5},
+ // MSIE does not recognize "CNS" (XSS issue),
+ // and EUC-TW is rare, so we prefer Big5 for output.
+ { "BIG5-CP950",
+ "BIG5-CP950", // Not an IANA name
+ CHINESE_BIG5
+ // MSIE does not recognize "BIG5-CP950" (XSS issue)
+ },
+ { "CP932", "CP932", // Not an IANA name
+ JAPANESE_SHIFT_JIS}, // MSIE does not recognize "CP932" (XSS issue)
+ { "UTF8", "UTF-8", UTF8},
+ { "Unknown",
+ "x-unknown", // Not an IANA name
+ UTF8}, // UTF-8 is our default output encoding
+ { "ASCII-7-bit", "US-ASCII", ASCII_7BIT},
+ { "KOI8R", "KOI8-R", RUSSIAN_KOI8_R},
+ { "CP1251", "windows-1251", RUSSIAN_CP1251},
+ { "CP1252", "windows-1252", MSFT_CP1252},
+ { "KOI8U",
+ "KOI8-U",
+ ISO_8859_5}, // because koi8-u is not as common
+ { "CP1250", "windows-1250", MSFT_CP1250},
+ { "ISO-8859-15", "ISO-8859-15", ISO_8859_15},
+ { "CP1254", "windows-1254", MSFT_CP1254},
+ { "CP1257", "windows-1257", MSFT_CP1257},
+ { "ISO-8859-11", "ISO-8859-11", ISO_8859_11},
+ { "CP874", "windows-874", MSFT_CP874},
+ { "CP1256", "windows-1256", MSFT_CP1256},
+ { "CP1255", "windows-1255", MSFT_CP1255},
+ { "ISO-8859-8-I", "ISO-8859-8-I", MSFT_CP1255},
+ // Java does not support iso-8859-8-i
+ { "VISUAL", "ISO-8859-8", MSFT_CP1255},
+ // we do not endorse the visual order
+ { "CP852", "cp852", MSFT_CP1250},
+ // because cp852 is not as common
+ { "CSN_369103", "csn_369103", MSFT_CP1250},
+ // MSIE does not recognize "csn_369103" (XSS issue)
+ { "CP1253", "windows-1253", MSFT_CP1253},
+ { "CP866", "IBM866", RUSSIAN_CP1251},
+ // because cp866 is not as common
+ { "ISO-8859-13", "ISO-8859-13", UTF8},
+ // because iso-8859-13 is not widely supported
+ { "ISO-2022-KR", "ISO-2022-KR", KOREAN_EUC_KR},
+ // due to potential confusion with HTML syntax chars
+ { "GBK", "GBK", GBK},
+ { "GB18030", "GB18030", GBK},
+ // because gb18030 is not widely supported
+ { "BIG5_HKSCS", "BIG5-HKSCS", CHINESE_BIG5},
+ // because Big5-HKSCS is not widely supported
+ { "ISO_2022_CN", "ISO-2022-CN", CHINESE_GB},
+ // due to potential confusion with HTML syntax chars
+ { "TSCII", "tscii", UTF8},
+ // we do not have an output converter for this font encoding
+ { "TAM", "tam", UTF8},
+ // we do not have an output converter for this font encoding
+ { "TAB", "tab", UTF8},
+ // we do not have an output converter for this font encoding
+ { "JAGRAN", "jagran", UTF8},
+ // we do not have an output converter for this font encoding
+ { "MACINTOSH", "MACINTOSH", ISO_8859_1},
+ // because macintosh is relatively uncommon
+ { "UTF7", "UTF-7",
+ UTF8}, // UTF-7 has been the subject of XSS attacks and is deprecated
+ { "BHASKAR", "bhaskar",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "HTCHANAKYA", "htchanakya", // not an IANA charset name.
+ UTF8}, // we do not have an output converter for this font encoding
+ { "UTF-16BE", "UTF-16BE",
+ UTF8}, // due to potential confusion with HTML syntax chars
+ { "UTF-16LE", "UTF-16LE",
+ UTF8}, // due to potential confusion with HTML syntax chars
+ { "UTF-32BE", "UTF-32BE",
+ UTF8}, // unlikely to cause XSS bugs, but very uncommon on Web
+ { "UTF-32LE", "UTF-32LE",
+ UTF8}, // unlikely to cause XSS bugs, but very uncommon on Web
+ { "X-BINARYENC", "x-binaryenc", // Not an IANA name
+ UTF8}, // because this one is not intended for output (just input)
+ { "HZ-GB-2312", "HZ-GB-2312",
+ CHINESE_GB}, // due to potential confusion with HTML syntax chars
+ { "X-UTF8UTF8", "x-utf8utf8", // Not an IANA name
+ UTF8}, // because this one is not intended for output (just input)
+ { "X-TAM-ELANGO", "x-tam-elango",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-LTTMBARANI", "x-tam-lttmbarani",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-SHREE", "x-tam-shree",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-TBOOMIS", "x-tam-tboomis",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-TMNEWS", "x-tam-tmnews",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-WEBTAMIL", "x-tam-webtamil",
+ UTF8}, // we do not have an output converter for this font encoding
+
+ { "X-KDDI-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ // KDDI version of Shift_JIS with Google Emoji PUA mappings.
+ // Note that MimeEncodingName() returns "Shift_JIS", since KDDI uses
+ // "Shift_JIS" in HTTP headers and email messages.
+
+ { "X-DoCoMo-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ // DoCoMo version of Shift_JIS with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+
+ { "X-SoftBank-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ // SoftBank version of Shift_JIS with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+
+ { "X-KDDI-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+ // KDDI version of ISO-2022-JP with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+ // The preferred Web encoding is due to potential confusion with
+ // HTML syntax chars.
+
+ { "X-SoftBank-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+ // SoftBank version of ISO-2022-JP with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+ // The preferred Web encoding is due to potential confusion with
+ // HTML syntax chars.
+
+ // Please refer to NOTE: section in the comments in the definition
+ // of "struct I18NInfoByEncoding", before adding new encodings.
+
+};
+
+
+
+COMPILE_ASSERT(arraysize(kEncodingInfoTable) == NUM_ENCODINGS,
+ kEncodingInfoTable_has_incorrect_size);
+
+Encoding default_encoding() {return LATIN1;}
+
+// *************************************************************
+// Encoding predicates
+// IsValidEncoding()
+// IsEncEncCompatible
+// IsEncodingWithSupportedLanguage
+// IsSupersetOfAscii7Bit
+// Is8BitEncoding
+// IsCJKEncoding
+// IsHebrewEncoding
+// IsRightToLeftEncoding
+// IsLogicalRightToLeftEncoding
+// IsVisualRightToLeftEncoding
+// IsIso2022Encoding
+// IsIso2022JpOrVariant
+// IsShiftJisOrVariant
+// IsJapaneseCellPhoneCarrierSpecificEncoding
+// *************************************************************
+
+bool IsValidEncoding(Encoding enc) {
+ return ((enc >= 0) && (enc < kNumEncodings));
+}
+
+bool IsEncEncCompatible(const Encoding from, const Encoding to) {
+ // Tests compatibility between the "from" and "to" encodings; in
+ // the typical case -- when both are valid known encodings -- this
+ // returns true iff converting from first to second is a no-op.
+ if (!IsValidEncoding(from) || !IsValidEncoding(to)) {
+ return false; // we only work with valid encodings...
+ } else if (to == from) {
+ return true; // the trivial common case
+ }
+
+ if (to == UNKNOWN_ENCODING) {
+ return true; // all valid encodings are compatible with the unknown
+ }
+
+ if (from == UNKNOWN_ENCODING) {
+ return false; // no unknown encoding is compatible with one that is
+ }
+
+ if (from == ASCII_7BIT) {
+ return IsSupersetOfAscii7Bit(to);
+ }
+
+ return (from == ISO_8859_1 && to == MSFT_CP1252) ||
+ (from == ISO_8859_8 && to == HEBREW_VISUAL) ||
+ (from == HEBREW_VISUAL && to == ISO_8859_8) ||
+ (from == ISO_8859_9 && to == MSFT_CP1254) ||
+ (from == ISO_8859_11 && to == MSFT_CP874) ||
+ (from == JAPANESE_SHIFT_JIS && to == JAPANESE_CP932) ||
+ (from == CHINESE_BIG5 && to == CHINESE_BIG5_CP950) ||
+ (from == CHINESE_GB && to == GBK) ||
+ (from == CHINESE_GB && to == GB18030) ||
+ (from == CHINESE_EUC_CN && to == CHINESE_EUC_DEC) ||
+ (from == CHINESE_EUC_CN && to == CHINESE_CNS) ||
+ (from == CHINESE_EUC_DEC && to == CHINESE_EUC_CN) ||
+ (from == CHINESE_EUC_DEC && to == CHINESE_CNS) ||
+ (from == CHINESE_CNS && to == CHINESE_EUC_CN) ||
+ (from == CHINESE_CNS && to == CHINESE_EUC_DEC);
+}
+
+// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
+// encoding represent the same characters as they do in ISO_8859_1.
+
+// TODO: This list could be expanded. Many other encodings are supersets
+// of 7-bit Ascii. In fact, Japanese JIS and Unicode are the only two
+// encodings that I know for a fact should *not* be in this list.
+bool IsSupersetOfAscii7Bit(Encoding e) {
+ switch (e) {
+ case ISO_8859_1:
+ case ISO_8859_2:
+ case ISO_8859_3:
+ case ISO_8859_4:
+ case ISO_8859_5:
+ case ISO_8859_6:
+ case ISO_8859_7:
+ case ISO_8859_8:
+ case ISO_8859_9:
+ case ISO_8859_10:
+ case JAPANESE_EUC_JP:
+ case JAPANESE_SHIFT_JIS:
+ case CHINESE_BIG5:
+ case CHINESE_GB:
+ case CHINESE_EUC_CN:
+ case KOREAN_EUC_KR:
+ case CHINESE_EUC_DEC:
+ case CHINESE_CNS:
+ case CHINESE_BIG5_CP950:
+ case JAPANESE_CP932:
+ case UTF8:
+ case UNKNOWN_ENCODING:
+ case ASCII_7BIT:
+ case RUSSIAN_KOI8_R:
+ case RUSSIAN_CP1251:
+ case MSFT_CP1252:
+ case RUSSIAN_KOI8_RU:
+ case MSFT_CP1250:
+ case ISO_8859_15:
+ case MSFT_CP1254:
+ case MSFT_CP1257:
+ case ISO_8859_11:
+ case MSFT_CP874:
+ case MSFT_CP1256:
+ case MSFT_CP1255:
+ case ISO_8859_8_I:
+ case HEBREW_VISUAL:
+ case CZECH_CP852:
+ case MSFT_CP1253:
+ case RUSSIAN_CP866:
+ case ISO_8859_13:
+ case GBK:
+ case GB18030:
+ case BIG5_HKSCS:
+ case MACINTOSH_ROMAN:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// To be an 8-bit encoding means that there are fewer than 256 symbols.
+// Each byte determines a new character; there are no multi-byte sequences.
+
+// TODO: This list could maybe be expanded. Other encodings may be 8-bit.
+bool Is8BitEncoding(Encoding e) {
+ switch (e) {
+ case ASCII_7BIT:
+ case ISO_8859_1:
+ case ISO_8859_2:
+ case ISO_8859_3:
+ case ISO_8859_4:
+ case ISO_8859_5:
+ case ISO_8859_6:
+ case ISO_8859_7:
+ case ISO_8859_8:
+ case ISO_8859_8_I:
+ case ISO_8859_9:
+ case ISO_8859_10:
+ case ISO_8859_11:
+ case ISO_8859_13:
+ case ISO_8859_15:
+ case MSFT_CP1252:
+ case MSFT_CP1253:
+ case MSFT_CP1254:
+ case MSFT_CP1255:
+ case MSFT_CP1256:
+ case MSFT_CP1257:
+ case RUSSIAN_KOI8_R:
+ case RUSSIAN_KOI8_RU:
+ case RUSSIAN_CP866:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool IsCJKEncoding(Encoding e) {
+ switch (e) {
+ case JAPANESE_EUC_JP:
+ case JAPANESE_SHIFT_JIS:
+ case JAPANESE_JIS:
+ case CHINESE_BIG5:
+ case CHINESE_GB:
+ case CHINESE_EUC_CN:
+ case KOREAN_EUC_KR:
+ case CHINESE_EUC_DEC:
+ case CHINESE_CNS:
+ case CHINESE_BIG5_CP950:
+ case JAPANESE_CP932:
+ case ISO_2022_KR:
+ case GBK:
+ case GB18030:
+ case BIG5_HKSCS:
+ case ISO_2022_CN:
+ case HZ_GB_2312:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool IsHebrewEncoding(Encoding e) {
+ return (e == ISO_8859_8 ||
+ e == ISO_8859_8_I ||
+ e == MSFT_CP1255 ||
+ e == HEBREW_VISUAL);
+}
+
+
+
+bool IsRightToLeftEncoding(Encoding enc) {
+ switch (enc) {
+ case MSFT_CP1255:
+ case MSFT_CP1256:
+ case ARABIC_ENCODING:
+ case HEBREW_ENCODING:
+ case ISO_8859_8_I:
+ case HEBREW_VISUAL:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool IsLogicalRightToLeftEncoding(Encoding enc) {
+ return IsRightToLeftEncoding(enc) && !IsVisualRightToLeftEncoding(enc);
+}
+
+// Note that despite an RFC to the contrary, ARABIC_ENCODING (ISO-8859-6)
+// is NOT visual.
+bool IsVisualRightToLeftEncoding(Encoding enc) {
+ switch (enc) {
+ case HEBREW_ENCODING:
+ case HEBREW_VISUAL:
+ return true;
+ default:
+ return false;
+ }
+}
+
+
+
+
+
+bool IsIso2022Encoding(Encoding enc) {
+ return (IsIso2022JpOrVariant(enc) ||
+ enc == ISO_2022_KR ||
+ enc == ISO_2022_CN);
+}
+
+bool IsIso2022JpOrVariant(Encoding enc) {
+ return (enc == JAPANESE_JIS ||
+ enc == KDDI_ISO_2022_JP ||
+ enc == SOFTBANK_ISO_2022_JP);
+}
+
+bool IsShiftJisOrVariant(Encoding enc) {
+ return (enc == JAPANESE_SHIFT_JIS ||
+ enc == JAPANESE_CP932 ||
+ enc == KDDI_SHIFT_JIS ||
+ enc == DOCOMO_SHIFT_JIS ||
+ enc == SOFTBANK_SHIFT_JIS);
+}
+
+bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc) {
+ return (enc == KDDI_ISO_2022_JP ||
+ enc == KDDI_SHIFT_JIS ||
+ enc == DOCOMO_SHIFT_JIS ||
+ enc == SOFTBANK_SHIFT_JIS ||
+ enc == SOFTBANK_ISO_2022_JP);
+}
+
+
+// *************************************************************
+// ENCODING NAMES
+// EncodingName() [Encoding to name]
+// MimeEncodingName() [Encoding to name]
+// EncodingFromName() [name to Encoding]
+// EncodingNameAliasToEncoding() [name to Encoding]
+// default_encoding_name()
+// invalid_encoding_name()
+// *************************************************************
+
+const char * EncodingName(const Encoding enc) {
+ if ( (enc < 0) || (enc >= kNumEncodings) )
+ return invalid_encoding_name();
+ return kEncodingInfoTable[enc].encoding_name_;
+}
+
+// TODO: Unify MimeEncodingName and EncodingName, or determine why
+// such a unification is not possible.
+
+const char * MimeEncodingName(Encoding enc) {
+ if ( (enc < 0) || (enc >= kNumEncodings) )
+ return ""; // TODO: Should this be invalid_encoding_name()?
+ return kEncodingInfoTable[enc].mime_encoding_name_;
+}
+
+bool EncodingFromName(const char* enc_name, Encoding *encoding) {
+ *encoding = UNKNOWN_ENCODING;
+ if ( enc_name == NULL ) return false;
+
+ for ( int i = 0; i < kNumEncodings; i++ ) {
+ if (!base::strcasecmp(enc_name, kEncodingInfoTable[i].encoding_name_) ) {
+ *encoding = static_cast<Encoding>(i);
+ return true;
+ }
+ }
+ return false;
+}
+
+// The encoding_map maps standard and non-standard encoding-names
+// (strings) to Encoding enums. It is used only by
+// EncodingNameAliasToEncoding. Note that the map uses
+// case-insensitive hash and comparison functions.
+
+typedef std::unordered_map<const char *, Encoding,
+ CStringAlnumCaseHash,
+ CStringAlnumCaseEqual> EncodingMap;
+
+static const EncodingMap& GetEncodingMap() {
+ static EncodingMap encoding_map;
+ if (!encoding_map.empty()) {
+ // Already initialized
+ return encoding_map;
+ }
+
+ // Initialize the map with all the "standard" encoding names,
+ // i.e., the ones returned by EncodingName and MimeEncodingName.
+ //
+ // First, add internal encoding names returned by EncodingName().
+ for (int i = 0; i < NUM_ENCODINGS; ++i) {
+ Encoding e = static_cast<Encoding>(i);
+ // Internal encoding names must be unique.
+ // The internal names are guaranteed to be unique by the CHECK_EQ.
+ const char *encoding_name = EncodingName(e);
+ // CHECK_EQ(0, encoding_map.count(encoding_name))
+ // << "Duplicate found for " << encoding_name;
+ encoding_map[encoding_name] = e;
+ }
+ // Then, add mime encoding names returned by MimeEncodingName().
+ // We don't override existing entries, to give precedence to entries
+ // added earlier.
+ for (int i = 0; i < NUM_ENCODINGS; ++i) {
+ Encoding e = static_cast<Encoding>(i);
+ // Note that MimeEncodingName() can return the same mime encoding
+ // name for different encoding enums like JAPANESE_SHIFT_JIS and
+ // KDDI_SHIFT_JIS. In that case, the encoding enum first seen
+ // will be the value for the encoding name in the map.
+ const char *mime_encoding_name = MimeEncodingName(e);
+ if (encoding_map.count(mime_encoding_name) == 0) {
+ encoding_map[mime_encoding_name] = e;
+ }
+ }
+
+ // Add some non-standard names: alternate spellings, common typos,
+ // etc. (It does no harm to add names already in the map.) Note
+ // that although the map is case-insensitive, by convention the
+ // keys are written here in lower case. For ease of maintenance,
+ // they are listed in alphabetical order.
+ encoding_map["5601"] = KOREAN_EUC_KR;
+ encoding_map["646"] = ASCII_7BIT;
+ encoding_map["852"] = CZECH_CP852;
+ encoding_map["866"] = RUSSIAN_CP866;
+ encoding_map["8859-1"] = ISO_8859_1;
+ encoding_map["ansi-1251"] = RUSSIAN_CP1251;
+ encoding_map["ansi_x3.4-1968"] = ASCII_7BIT;
+ encoding_map["arabic"] = ISO_8859_6;
+ encoding_map["ascii"] = ISO_8859_1;
+ encoding_map["ascii-7-bit"] = ASCII_7BIT; // not iana standard
+ encoding_map["asmo-708"] = ISO_8859_6;
+ encoding_map["bhaskar"] = BHASKAR;
+ encoding_map["big5"] = CHINESE_BIG5;
+ encoding_map["big5-cp950"] = CHINESE_BIG5_CP950; // not iana standard
+ encoding_map["big5-hkscs"] = BIG5_HKSCS;
+ encoding_map["chinese"] = CHINESE_GB;
+ encoding_map["cns"] = CHINESE_CNS; // not iana standard
+ encoding_map["cns11643"] = CHINESE_CNS;
+ encoding_map["cp1250"] = MSFT_CP1250; // not iana standard
+ encoding_map["cp1251"] = RUSSIAN_CP1251; // not iana standard
+ encoding_map["cp1252"] = MSFT_CP1252; // not iana standard
+ encoding_map["cp1253"] = MSFT_CP1253; // not iana standard
+ encoding_map["cp1254"] = MSFT_CP1254; // not iana standard
+ encoding_map["cp1255"] = MSFT_CP1255;
+ encoding_map["cp1256"] = MSFT_CP1256;
+ encoding_map["cp1257"] = MSFT_CP1257; // not iana standard
+ encoding_map["cp819"] = ISO_8859_1;
+ encoding_map["cp852"] = CZECH_CP852;
+ encoding_map["cp866"] = RUSSIAN_CP866;
+ encoding_map["cp-866"] = RUSSIAN_CP866;
+ encoding_map["cp874"] = MSFT_CP874;
+ encoding_map["cp932"] = JAPANESE_CP932; // not iana standard
+ encoding_map["cp950"] = CHINESE_BIG5_CP950; // not iana standard
+ encoding_map["csbig5"] = CHINESE_BIG5;
+ encoding_map["cseucjpkdfmtjapanese"] = JAPANESE_EUC_JP;
+ encoding_map["cseuckr"] = KOREAN_EUC_KR;
+ encoding_map["csgb2312"] = CHINESE_GB;
+ encoding_map["csibm852"] = CZECH_CP852;
+ encoding_map["csibm866"] = RUSSIAN_CP866;
+ encoding_map["csiso2022jp"] = JAPANESE_JIS;
+ encoding_map["csiso2022kr"] = ISO_2022_KR;
+ encoding_map["csiso58gb231280"] = CHINESE_GB;
+ encoding_map["csiso88598i"] = ISO_8859_8_I;
+ encoding_map["csisolatin1"] = ISO_8859_1;
+ encoding_map["csisolatin2"] = ISO_8859_2;
+ encoding_map["csisolatin3"] = ISO_8859_3;
+ encoding_map["csisolatin4"] = ISO_8859_4;
+ encoding_map["csisolatin5"] = ISO_8859_9;
+ encoding_map["csisolatin6"] = ISO_8859_10;
+ encoding_map["csisolatinarabic"] = ISO_8859_6;
+ encoding_map["csisolatincyrillic"] = ISO_8859_5;
+ encoding_map["csisolatingreek"] = ISO_8859_7;
+ encoding_map["csisolatinhebrew"] = ISO_8859_8;
+ encoding_map["csksc56011987"] = KOREAN_EUC_KR;
+ encoding_map["csmacintosh"] = MACINTOSH_ROMAN;
+ encoding_map["csn-369103"] = CZECH_CSN_369103;
+ encoding_map["csshiftjis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["csunicode"] = UTF16BE;
+ encoding_map["csunicode11"] = UTF16BE;
+ encoding_map["csunicode11utf7"] = UTF7;
+ encoding_map["csunicodeascii"] = UTF16BE;
+ encoding_map["csunicodelatin1"] = UTF16BE;
+ encoding_map["cyrillic"] = ISO_8859_5;
+ encoding_map["ecma-114"] = ISO_8859_6;
+ encoding_map["ecma-118"] = ISO_8859_7;
+ encoding_map["elot_928"] = ISO_8859_7;
+ encoding_map["euc"] = CHINESE_EUC_DEC; // not iana standard
+ encoding_map["euc-cn"] = CHINESE_EUC_CN; // not iana standard
+ encoding_map["euc-dec"] = CHINESE_EUC_DEC; // not iana standard
+ encoding_map["euc-jp"] = JAPANESE_EUC_JP;
+ encoding_map["euc-kr"] = KOREAN_EUC_KR;
+ encoding_map["eucgb2312_cn"] = CHINESE_GB;
+ encoding_map["gb"] = CHINESE_GB; // not iana standard
+ encoding_map["gb18030"] = GB18030;
+ encoding_map["gb2132"] = CHINESE_GB; // common typo
+ encoding_map["gb2312"] = CHINESE_GB;
+ encoding_map["gb_2312-80"] = CHINESE_GB;
+ encoding_map["gbk"] = GBK;
+ encoding_map["greek"] = ISO_8859_7;
+ encoding_map["greek8"] = ISO_8859_7;
+ encoding_map["hebrew"] = ISO_8859_8;
+ encoding_map["htchanakya"] = HTCHANAKYA;
+ encoding_map["hz-gb-2312"] = HZ_GB_2312;
+ encoding_map["ibm819"] = ISO_8859_1;
+ encoding_map["ibm852"] = CZECH_CP852;
+ encoding_map["ibm874"] = MSFT_CP874;
+ encoding_map["iso-10646"] = UTF16BE;
+ encoding_map["iso-10646-j-1"] = UTF16BE;
+ encoding_map["iso-10646-ucs-2"] = UNICODE;
+ encoding_map["iso-10646-ucs-4"] = UTF32BE;
+ encoding_map["iso-10646-ucs-basic"] = UTF16BE;
+ encoding_map["iso-10646-unicode-latin1"] = UTF16BE;
+ encoding_map["iso-2022-cn"] = ISO_2022_CN;
+ encoding_map["iso-2022-jp"] = JAPANESE_JIS;
+ encoding_map["iso-2022-kr"] = ISO_2022_KR;
+ encoding_map["iso-8559-1"] = ISO_8859_1; // common typo
+ encoding_map["iso-874"] = MSFT_CP874;
+ encoding_map["iso-8858-1"] = ISO_8859_1; // common typo
+ // iso-8859-0 was a temporary name, eventually renamed iso-8859-15
+ encoding_map["iso-8859-0"] = ISO_8859_15;
+ encoding_map["iso-8859-1"] = ISO_8859_1;
+ encoding_map["iso-8859-10"] = ISO_8859_10;
+ encoding_map["iso-8859-11"] = ISO_8859_11;
+ encoding_map["iso-8859-13"] = ISO_8859_13;
+ encoding_map["iso-8859-15"] = ISO_8859_15;
+ encoding_map["iso-8859-2"] = ISO_8859_2;
+ encoding_map["iso-8859-3"] = ISO_8859_3;
+ encoding_map["iso-8859-4"] = ISO_8859_4;
+ encoding_map["iso-8859-5"] = ISO_8859_5;
+ encoding_map["iso-8859-6"] = ISO_8859_6;
+ encoding_map["iso-8859-7"] = ISO_8859_7;
+ encoding_map["iso-8859-8"] = ISO_8859_8;
+ encoding_map["iso-8859-8-i"] = ISO_8859_8_I;
+ encoding_map["iso-8859-9"] = ISO_8859_9;
+ encoding_map["iso-9959-1"] = ISO_8859_1; // common typo
+ encoding_map["iso-ir-100"] = ISO_8859_1;
+ encoding_map["iso-ir-101"] = ISO_8859_2;
+ encoding_map["iso-ir-109"] = ISO_8859_3;
+ encoding_map["iso-ir-110"] = ISO_8859_4;
+ encoding_map["iso-ir-126"] = ISO_8859_7;
+ encoding_map["iso-ir-127"] = ISO_8859_6;
+ encoding_map["iso-ir-138"] = ISO_8859_8;
+ encoding_map["iso-ir-144"] = ISO_8859_5;
+ encoding_map["iso-ir-148"] = ISO_8859_9;
+ encoding_map["iso-ir-149"] = KOREAN_EUC_KR;
+ encoding_map["iso-ir-157"] = ISO_8859_10;
+ encoding_map["iso-ir-58"] = CHINESE_GB;
+ encoding_map["iso-latin-1"] = ISO_8859_1;
+ encoding_map["iso_2022-cn"] = ISO_2022_CN;
+ encoding_map["iso_2022-kr"] = ISO_2022_KR;
+ encoding_map["iso_8859-1"] = ISO_8859_1;
+ encoding_map["iso_8859-10:1992"] = ISO_8859_10;
+ encoding_map["iso_8859-11"] = ISO_8859_11;
+ encoding_map["iso_8859-13"] = ISO_8859_13;
+ encoding_map["iso_8859-15"] = ISO_8859_15;
+ encoding_map["iso_8859-1:1987"] = ISO_8859_1;
+ encoding_map["iso_8859-2"] = ISO_8859_2;
+ encoding_map["iso_8859-2:1987"] = ISO_8859_2;
+ encoding_map["iso_8859-3"] = ISO_8859_3;
+ encoding_map["iso_8859-3:1988"] = ISO_8859_3;
+ encoding_map["iso_8859-4"] = ISO_8859_4;
+ encoding_map["iso_8859-4:1988"] = ISO_8859_4;
+ encoding_map["iso_8859-5"] = ISO_8859_5;
+ encoding_map["iso_8859-5:1988"] = ISO_8859_5;
+ encoding_map["iso_8859-6"] = ISO_8859_6;
+ encoding_map["iso_8859-6:1987"] = ISO_8859_6;
+ encoding_map["iso_8859-7"] = ISO_8859_7;
+ encoding_map["iso_8859-7:1987"] = ISO_8859_7;
+ encoding_map["iso_8859-8"] = ISO_8859_8;
+ encoding_map["iso_8859-8:1988:"] = ISO_8859_8;
+ encoding_map["iso_8859-9"] = ISO_8859_9;
+ encoding_map["iso_8859-9:1989"] = ISO_8859_9;
+ encoding_map["jagran"] = JAGRAN;
+ encoding_map["jis"] = JAPANESE_JIS; // not iana standard
+ encoding_map["koi8-cs"] = CZECH_CSN_369103;
+ encoding_map["koi8-r"] = RUSSIAN_KOI8_R;
+ encoding_map["koi8-ru"] = RUSSIAN_KOI8_RU; // not iana standard
+ encoding_map["koi8-u"] = RUSSIAN_KOI8_RU;
+ encoding_map["koi8r"] = RUSSIAN_KOI8_R; // not iana standard
+ encoding_map["koi8u"] = RUSSIAN_KOI8_RU; // not iana standard
+ encoding_map["korean"] = KOREAN_EUC_KR; // i assume this is what is meant
+ encoding_map["ks-c-5601"] = KOREAN_EUC_KR; // not iana standard
+ encoding_map["ks-c-5601-1987"] = KOREAN_EUC_KR; // not iana standard
+ encoding_map["ks_c_5601-1989"] = KOREAN_EUC_KR;
+ encoding_map["ksc"] = KOREAN_EUC_KR; // not iana standard
+ encoding_map["l1"] = ISO_8859_1;
+ encoding_map["l2"] = ISO_8859_2;
+ encoding_map["l3"] = ISO_8859_3;
+ encoding_map["l4"] = ISO_8859_4;
+ encoding_map["l5"] = ISO_8859_9;
+ encoding_map["l6"] = ISO_8859_10;
+ encoding_map["latin-1"] = ISO_8859_1; // not iana standard
+ encoding_map["latin1"] = ISO_8859_1;
+ encoding_map["latin2"] = ISO_8859_2;
+ encoding_map["latin3"] = ISO_8859_3;
+ encoding_map["latin4"] = ISO_8859_4;
+ encoding_map["latin5"] = ISO_8859_9;
+ encoding_map["latin6"] = ISO_8859_10;
+ encoding_map["mac"] = MACINTOSH_ROMAN;
+ encoding_map["macintosh"] = MACINTOSH_ROMAN;
+ encoding_map["macintosh-roman"] = MACINTOSH_ROMAN;
+ encoding_map["ms932"] = JAPANESE_CP932; // not iana standard
+ encoding_map["ms_kanji"] = JAPANESE_CP932;
+ encoding_map["shift-jis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["shift_jis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["sjis"] = JAPANESE_SHIFT_JIS; // not iana standard
+ encoding_map["sjs"] = JAPANESE_SHIFT_JIS; // not iana standard
+ encoding_map["sun_eu_greek"] = ISO_8859_7;
+ encoding_map["tab"] = TAMIL_BI;
+ encoding_map["tam"] = TAMIL_MONO;
+ encoding_map["tis-620"] = ISO_8859_11;
+ encoding_map["tscii"] = TSCII;
+ encoding_map["un"] = UNKNOWN_ENCODING; // not iana standard
+ encoding_map["unicode"] = UNICODE; // not iana standard
+ encoding_map["unicode-1-1-utf-7"] = UTF7;
+ encoding_map["unicode-1-1-utf-8"] = UTF8;
+ encoding_map["unicode-2-0-utf-7"] = UTF7;
+ encoding_map["unknown"] = UNKNOWN_ENCODING; // not iana standard
+ encoding_map["us"] = ISO_8859_1;
+ encoding_map["us-ascii"] = ISO_8859_1;
+ encoding_map["utf-16be"] = UTF16BE;
+ encoding_map["utf-16le"] = UTF16LE;
+ encoding_map["utf-32be"] = UTF32BE;
+ encoding_map["utf-32le"] = UTF32LE;
+ encoding_map["utf-7"] = UTF7;
+ encoding_map["utf-8"] = UTF8;
+ encoding_map["utf7"] = UTF7;
+ encoding_map["utf8"] = UTF8; // not iana standard
+ encoding_map["visual"] = HEBREW_VISUAL;
+ encoding_map["win-1250"] = MSFT_CP1250; // not iana standard
+ encoding_map["win-1251"] = RUSSIAN_CP1251; // not iana standard
+ encoding_map["window-874"] = MSFT_CP874;
+ encoding_map["windows-1250"] = MSFT_CP1250;
+ encoding_map["windows-1251"] = RUSSIAN_CP1251;
+ encoding_map["windows-1252"] = MSFT_CP1252;
+ encoding_map["windows-1253"] = MSFT_CP1253;
+ encoding_map["windows-1254"] = MSFT_CP1254;
+ encoding_map["windows-1255"] = MSFT_CP1255;
+ encoding_map["windows-1256"] = MSFT_CP1256;
+ encoding_map["windows-1257"] = MSFT_CP1257;
+ encoding_map["windows-31j"] = JAPANESE_CP932;
+ encoding_map["windows-874"] = MSFT_CP874;
+ encoding_map["windows-936"] = GBK;
+ encoding_map["x-big5"] = CHINESE_BIG5;
+ encoding_map["x-binaryenc"] = BINARYENC; // not iana standard
+ encoding_map["x-cp1250"] = MSFT_CP1250;
+ encoding_map["x-cp1251"] = RUSSIAN_CP1251;
+ encoding_map["x-cp1252"] = MSFT_CP1252;
+ encoding_map["x-cp1253"] = MSFT_CP1253;
+ encoding_map["x-cp1254"] = MSFT_CP1254;
+ encoding_map["x-cp1255"] = MSFT_CP1255;
+ encoding_map["x-cp1256"] = MSFT_CP1256;
+ encoding_map["x-cp1257"] = MSFT_CP1257;
+ encoding_map["x-euc-jp"] = JAPANESE_EUC_JP;
+ encoding_map["x-euc-tw"] = CHINESE_CNS;
+ encoding_map["x-gbk"] = GBK;
+ encoding_map["x-iso-10646-ucs-2-be"] = UTF16BE;
+ encoding_map["x-iso-10646-ucs-2-le"] = UTF16LE;
+ encoding_map["x-iso-10646-ucs-4-be"] = UTF32BE;
+ encoding_map["x-iso-10646-ucs-4-le"] = UTF32LE;
+ encoding_map["x-jis"] = JAPANESE_JIS; // not iana standard
+ encoding_map["x-mac-roman"] = MACINTOSH_ROMAN;
+ encoding_map["x-shift_jis"] = JAPANESE_SHIFT_JIS; // not iana standard
+ encoding_map["x-sjis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["x-unicode-2-0-utf-7"] = UTF7;
+ encoding_map["x-utf8utf8"] = UTF8UTF8; // not iana standard
+ encoding_map["x-x-big5"] = CHINESE_BIG5;
+ encoding_map["zh_cn.euc"] = CHINESE_GB;
+ encoding_map["zh_tw-big5"] = CHINESE_BIG5;
+ encoding_map["zh_tw-euc"] = CHINESE_CNS;
+
+ // Remove they entry for the empty string, if any.
+ encoding_map.erase("");
+
+ return encoding_map;
+}
+
+// ----------------------------------------------------------------------
+// EncodingNameAliasToEncoding()
+//
+// This function takes an encoding name/alias and returns the Encoding
+// enum. The input is case insensitive. It is the union of the common
+// IANA standard names, the charset names used in Netscape Navigator,
+// and some common names we have been using.
+// See: http://www.iana.org/assignments/character-sets
+// http://physics.hallym.ac.kr/resource/relnotes/windows-2.0.html
+//
+// UNKNOWN_ENCODING is returned if none matches.
+//
+// TODO: Check if it is possible to remove the non-standard,
+// non-netscape-use names. It is because this routine is used for
+// encoding detections from html meta info. Non-standard names may
+// introduce noise on encoding detection.
+//
+// TODO: Unify EncodingNameAliasToEncoding and EncodingFromName,
+// or determine why such a unification is not possible.
+// ----------------------------------------------------------------------
+Encoding EncodingNameAliasToEncoding(const char *encoding_name) {
+ if (!encoding_name) {
+ return UNKNOWN_ENCODING;
+ }
+
+ const EncodingMap& encoding_map = GetEncodingMap();
+
+ EncodingMap::const_iterator emi = encoding_map.find(encoding_name);
+ if (emi != encoding_map.end()) {
+ return emi->second;
+ } else {
+ return UNKNOWN_ENCODING;
+ }
+}
+
+const char* default_encoding_name() {
+ return kEncodingInfoTable[LATIN1].encoding_name_;
+}
+
+static const char* const kInvalidEncodingName = "invalid_encoding";
+
+const char *invalid_encoding_name() {
+ return kInvalidEncodingName;
+}
+
+
+
+// *************************************************************
+// Miscellany
+// *************************************************************
+
+
+Encoding PreferredWebOutputEncoding(Encoding enc) {
+ return IsValidEncoding(enc)
+ ? kEncodingInfoTable[enc].preferred_web_output_encoding_
+ : UTF8;
+}
diff --git a/contrib/google-ced/util/encodings/encodings.h b/contrib/google-ced/util/encodings/encodings.h
new file mode 100644
index 0000000..6477974
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings.h
@@ -0,0 +1,299 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_ENCODINGS_ENCODINGS_H_
+#define UTIL_ENCODINGS_ENCODINGS_H_
+
+// This interface defines the Encoding enum and various functions that
+// depend only on Encoding values.
+
+// A hash-function for Encoding, hash<Encoding>, is defined in
+// i18n/encodings/public/encodings-hash.h
+
+// On some Windows projects, UNICODE may be defined, which would prevent the
+// Encoding enum below from compiling. Note that this is a quick fix that does
+// not break any existing projects. The UNICODE enum may someday be changed
+// to something more specific and non-colliding, but this involves careful
+// testing of changes in many other projects.
+#undef UNICODE
+
+// NOTE: The Encoding enum must always start at 0. This assumption has
+// been made and used.
+
+#ifndef SWIG
+
+#include "util/encodings/encodings.pb.h"
+
+#else
+
+// TODO: Include a SWIG workaround header file.
+
+#endif
+
+const int kNumEncodings = NUM_ENCODINGS;
+
+// some of the popular encoding aliases
+// TODO: Make these static const Encoding values instead of macros.
+#define LATIN1 ISO_8859_1
+#define LATIN2 ISO_8859_2
+#define LATIN3 ISO_8859_3
+#define LATIN4 ISO_8859_4
+#define CYRILLIC ISO_8859_5
+#define ARABIC_ENCODING ISO_8859_6 // avoiding the same name as language
+#define GREEK_ENCODING ISO_8859_7 // avoiding the same name as language
+#define HEBREW_ENCODING ISO_8859_8 // avoiding the same name as language
+#define LATIN5 ISO_8859_9
+#define LATIN6 ISO_8859_10
+#define KOREAN_HANGUL KOREAN_EUC_KR
+
+// The default Encoding (LATIN1).
+Encoding default_encoding();
+
+
+
+// *************************************************************
+// Encoding predicates
+// IsValidEncoding()
+// IsEncEncCompatible
+// IsSupersetOfAscii7Bit
+// Is8BitEncoding
+// IsCJKEncoding
+// IsHebrewEncoding
+// IsRightToLeftEncoding
+// IsLogicalRightToLeftEncoding
+// IsVisualRightToLeftEncoding
+// IsIso2022Encoding
+// IsIso2022JpOrVariant
+// IsShiftJisOrVariant
+// IsJapaneseCellPhoneCarrierSpecificEncoding
+// *************************************************************
+
+// IsValidEncoding
+// ===================================
+//
+// Function to check if the input language enum is within range.
+//
+
+bool IsValidEncoding(Encoding enc);
+
+//
+// IsEncEncCompatible
+// ------------------
+//
+// This function is to determine whether or not converting from the
+// first encoding to the second requires any changes to the underlying
+// text (e.g. ASCII_7BIT is a subset of UTF8).
+//
+// TODO: the current implementation is likely incomplete. It would be
+// good to consider the full matrix of all pairs of encodings and to fish out
+// all compatible pairs.
+//
+bool IsEncEncCompatible(const Encoding from, const Encoding to);
+
+// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
+// encoding represent the same characters as they do in ISO_8859_1.
+
+// WARNING: This function does not currently return true for all encodings that
+// are supersets of Ascii 7-bit.
+bool IsSupersetOfAscii7Bit(Encoding e);
+
+// To be an 8-bit encoding means that there are fewer than 256 symbols.
+// Each byte determines a new character; there are no multi-byte sequences.
+
+// WARNING: This function does not currently return true for all encodings that
+// are 8-bit encodings.
+bool Is8BitEncoding(Encoding e);
+
+// IsCJKEncoding
+// -------------
+//
+// This function returns true if the encoding is either Chinese
+// (simplified or traditional), Japanese, or Korean. Note: UTF8 is not
+// considered a CJK encoding.
+bool IsCJKEncoding(Encoding e);
+
+// IsHebrewEncoding
+// -------------
+//
+// This function returns true if the encoding is a Hebrew specific
+// encoding (not UTF8, etc).
+bool IsHebrewEncoding(Encoding e);
+
+// IsRightToLeftEncoding
+// ---------------------
+//
+// Returns true if the encoding is a right-to-left encoding.
+//
+// Note that the name of this function is somewhat misleading. There is nothing
+// "right to left" about these encodings. They merely contain code points for
+// characters in RTL languages such as Hebrew and Arabic. But this is also
+// true for UTF-8.
+//
+// TODO: Get rid of this function. The only special-case we
+// should need to worry about are visual encodings. Anything we
+// need to do for all 'RTL' encodings we need to do for UTF-8 as well.
+bool IsRightToLeftEncoding(Encoding enc);
+
+// IsLogicalRightToLeftEncoding
+// ----------------------------
+//
+// Returns true if the encoding is a logical right-to-left encoding.
+// Logical right-to-left encodings are those that the browser renders
+// right-to-left and applies the BiDi algorithm to. Therefore the characters
+// appear in reading order in the file, and indexing, snippet generation etc.
+// should all just work with no special processing.
+//
+// TODO: Get rid of this function. The only special-case we
+// should need to worry about are visual encodings.
+bool IsLogicalRightToLeftEncoding(Encoding enc);
+
+// IsVisualRightToLeftEncoding
+// ---------------------------
+//
+// Returns true if the encoding is a visual right-to-left encoding.
+// Visual right-to-left encodings are those that the browser renders
+// left-to-right and does not apply the BiDi algorithm to. Therefore each
+// line appears in reverse order in the file, lines are manually wrapped
+// by abusing <br> or <p> tags, etc. Visual RTL encoding is a relic of
+// the prehistoric days when browsers couldn't render right-to-left, but
+// unfortunately some visual pages persist to this day. These documents require
+// special processing so that we don't index or snippet them with each line
+// reversed.
+bool IsVisualRightToLeftEncoding(Encoding enc);
+
+// IsIso2022Encoding
+// -----------------
+//
+// Returns true if the encoding is a kind of ISO 2022 such as
+// ISO-2022-JP.
+bool IsIso2022Encoding(Encoding enc);
+
+// IsIso2022JpOrVariant
+// --------------------
+//
+// Returns true if the encoding is ISO-2022-JP or a variant such as
+// KDDI's ISO-2022-JP.
+bool IsIso2022JpOrVariant(Encoding enc);
+
+// IsShiftJisOrVariant
+// --------------------
+//
+// Returns true if the encoding is Shift_JIS or a variant such as
+// KDDI's Shift_JIS.
+bool IsShiftJisOrVariant(Encoding enc);
+
+// IsJapanesCellPhoneCarrierSpecificEncoding
+// -----------------------------------------
+//
+// Returns true if it's Japanese cell phone carrier specific encoding
+// such as KDDI_SHIFT_JIS.
+bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc);
+
+
+
+// *************************************************************
+// ENCODING NAMES
+//
+// This interface defines a standard name for each valid encoding, and
+// a standard name for invalid encodings. (Some names use all upper
+// case, but others use mixed case.)
+//
+// EncodingName() [Encoding to name]
+// MimeEncodingName() [Encoding to name]
+// EncodingFromName() [name to Encoding]
+// EncodingNameAliasToEncoding() [name to Encoding]
+// default_encoding_name()
+// invalid_encoding_name()
+// *************************************************************
+
+// EncodingName
+// ------------
+//
+// Given the encoding, returns its standard name.
+// Return invalid_encoding_name() if the encoding is invalid.
+//
+const char* EncodingName(Encoding enc);
+
+//
+// MimeEncodingName
+// ----------------
+//
+// Return the "preferred MIME name" of an encoding.
+//
+// This name is suitable for using in HTTP headers, HTML tags,
+// and as the "charset" parameter of a MIME Content-Type.
+const char* MimeEncodingName(Encoding enc);
+
+
+// The maximum length of an encoding name
+const int kMaxEncodingNameSize = 50;
+
+// The standard name of the default encoding.
+const char* default_encoding_name();
+
+// The name used for an invalid encoding.
+const char* invalid_encoding_name();
+
+// EncodingFromName
+// ----------------
+//
+// If enc_name matches the standard name of an Encoding, using a
+// case-insensitive comparison, set *encoding to that Encoding and
+// return true. Otherwise set *encoding to UNKNOWN_ENCODING and
+// return false.
+//
+// REQUIRES: encoding must not be NULL.
+//
+bool EncodingFromName(const char* enc_name, Encoding *encoding);
+
+//
+// EncodingNameAliasToEncoding
+// ---------------------------
+//
+// If enc_name matches the standard name or an alias of an Encoding,
+// using a case-insensitive comparison, return that
+// Encoding. Otherwise, return UNKNOWN_ENCODING.
+//
+// Aliases include most mime-encoding names (e.g., "ISO-8859-7" for
+// GREEK), alternate names (e.g., "cyrillic" for ISO_8859_5) and
+// common variations with hyphens and underscores (e.g., "koi8-u" and
+// "koi8u" for RUSSIAN_KOI8_R).
+
+Encoding EncodingNameAliasToEncoding(const char *enc_name);
+
+// *************************************************************
+// Miscellany
+// *************************************************************
+
+// PreferredWebOutputEncoding
+// --------------------------
+//
+// Some multi-byte encodings use byte values that coincide with the
+// ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
+// can misinterpret these, as indicated in an external XSS report from
+// 2007-02-15. Here, we map these dangerous encodings to safer ones. We
+// also use UTF8 instead of encodings that we don't support in our
+// output, and we generally try to be conservative in what we send out.
+// Where the client asks for single- or double-byte encodings that are
+// not as common, we substitute a more common single- or double-byte
+// encoding, if there is one, thereby preserving the client's intent
+// to use less space than UTF-8. This also means that characters
+// outside the destination set will be converted to HTML NCRs (&#NNN;)
+// if requested.
+Encoding PreferredWebOutputEncoding(Encoding enc);
+
+
+#endif // UTIL_ENCODINGS_ENCODINGS_H_
diff --git a/contrib/google-ced/util/encodings/encodings.pb.h b/contrib/google-ced/util/encodings/encodings.pb.h
new file mode 100644
index 0000000..ffbd716
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings.pb.h
@@ -0,0 +1,181 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_ENCODINGS_ENCODINGS_PB_H_
+#define UTIL_ENCODINGS_ENCODINGS_PB_H_
+
+enum Encoding {
+ ISO_8859_1 = 0, // Teragram ASCII
+ ISO_8859_2 = 1, // Teragram Latin2
+ ISO_8859_3 = 2, // in BasisTech but not in Teragram
+ ISO_8859_4 = 3, // Teragram Latin4
+ ISO_8859_5 = 4, // Teragram ISO-8859-5
+ ISO_8859_6 = 5, // Teragram Arabic
+ ISO_8859_7 = 6, // Teragram Greek
+ ISO_8859_8 = 7, // Teragram Hebrew
+ ISO_8859_9 = 8, // in BasisTech but not in Teragram
+ ISO_8859_10 = 9, // in BasisTech but not in Teragram
+ JAPANESE_EUC_JP = 10, // Teragram EUC_JP
+ JAPANESE_SHIFT_JIS = 11, // Teragram SJS
+ JAPANESE_JIS = 12, // Teragram JIS
+ CHINESE_BIG5 = 13, // Teragram BIG5
+ CHINESE_GB = 14, // Teragram GB
+ CHINESE_EUC_CN = 15, // Misnamed. Should be EUC_TW. Was Basis Tech
+ // CNS11643EUC, before that Teragram EUC-CN(!)
+ // See //i18n/basistech/basistech_encodings.h
+ KOREAN_EUC_KR = 16, // Teragram KSC
+ UNICODE = 17, // Teragram Unicode
+ CHINESE_EUC_DEC = 18, // Misnamed. Should be EUC_TW. Was Basis Tech
+ // CNS11643EUC, before that Teragram EUC.
+ CHINESE_CNS = 19, // Misnamed. Should be EUC_TW. Was Basis Tech
+ // CNS11643EUC, before that Teragram CNS.
+ CHINESE_BIG5_CP950 = 20, // Teragram BIG5_CP950
+ JAPANESE_CP932 = 21, // Teragram CP932
+ UTF8 = 22,
+ UNKNOWN_ENCODING = 23,
+ ASCII_7BIT = 24, // ISO_8859_1 with all characters <= 127.
+ // Should be present only in the crawler
+ // and in the repository,
+ // *never* as a result of Document::encoding().
+ RUSSIAN_KOI8_R = 25, // Teragram KOI8R
+ RUSSIAN_CP1251 = 26, // Teragram CP1251
+
+ //----------------------------------------------------------
+ // These are _not_ output from teragram. Instead, they are as
+ // detected in the headers of usenet articles.
+ MSFT_CP1252 = 27, // 27: CP1252 aka MSFT euro ascii
+ RUSSIAN_KOI8_RU = 28, // CP21866 aka KOI8-U, used for Ukrainian.
+ // Misnamed, this is _not_ KOI8-RU but KOI8-U.
+ // KOI8-U is used much more often than KOI8-RU.
+ MSFT_CP1250 = 29, // CP1250 aka MSFT eastern european
+ ISO_8859_15 = 30, // aka ISO_8859_0 aka ISO_8859_1 euroized
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ // These are in BasisTech but not in Teragram. They are
+ // needed for new interface languages. Now detected by
+ // research langid
+ MSFT_CP1254 = 31, // used for Turkish
+ MSFT_CP1257 = 32, // used in Baltic countries
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ //----------------------------------------------------------
+ // New encodings detected by Teragram
+ ISO_8859_11 = 33, // aka TIS-620, used for Thai
+ MSFT_CP874 = 34, // used for Thai
+ MSFT_CP1256 = 35, // used for Arabic
+
+ //----------------------------------------------------------
+ // Detected as ISO_8859_8 by Teragram, but can be found in META tags
+ MSFT_CP1255 = 36, // Logical Hebrew Microsoft
+ ISO_8859_8_I = 37, // Iso Hebrew Logical
+ HEBREW_VISUAL = 38, // Iso Hebrew Visual
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ // Detected by research langid
+ CZECH_CP852 = 39,
+ CZECH_CSN_369103 = 40, // aka ISO_IR_139 aka KOI8_CS
+ MSFT_CP1253 = 41, // used for Greek
+ RUSSIAN_CP866 = 42,
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ // Handled by iconv in glibc
+ ISO_8859_13 = 43,
+ ISO_2022_KR = 44,
+ GBK = 45,
+ GB18030 = 46,
+ BIG5_HKSCS = 47,
+ ISO_2022_CN = 48,
+
+ //-----------------------------------------------------------
+ // Detected by xin liu's detector
+ // Handled by transcoder
+ // (Indic encodings)
+
+ TSCII = 49,
+ TAMIL_MONO = 50,
+ TAMIL_BI = 51,
+ JAGRAN = 52,
+
+
+ MACINTOSH_ROMAN = 53,
+ UTF7 = 54,
+ BHASKAR = 55, // Indic encoding - Devanagari
+ HTCHANAKYA = 56, // 56 Indic encoding - Devanagari
+
+ //-----------------------------------------------------------
+ // These allow a single place (inputconverter and outputconverter)
+ // to do UTF-16 <==> UTF-8 bulk conversions and UTF-32 <==> UTF-8
+ // bulk conversions, with interchange-valid checking on input and
+ // fallback if needed on ouput.
+ UTF16BE = 57, // big-endian UTF-16
+ UTF16LE = 58, // little-endian UTF-16
+ UTF32BE = 59, // big-endian UTF-32
+ UTF32LE = 60, // little-endian UTF-32
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // An encoding that means "This is not text, but it may have some
+ // simple ASCII text embedded". Intended input conversion (not yet
+ // implemented) is to keep strings of >=4 seven-bit ASCII characters
+ // (follow each kept string with an ASCII space), delete the rest of
+ // the bytes. This will pick up and allow indexing of e.g. captions
+ // in JPEGs. No output conversion needed.
+ BINARYENC = 61,
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Some Web pages allow a mixture of HZ-GB and GB-2312 by using
+ // ~{ ... ~} for 2-byte pairs, and the browsers support this.
+ HZ_GB_2312 = 62,
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Some external vendors make the common input error of
+ // converting MSFT_CP1252 to UTF8 *twice*. No output conversion needed.
+ UTF8UTF8 = 63,
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Handled by transcoder for tamil language specific font
+ // encodings without the support for detection at present.
+ TAM_ELANGO = 64, // Elango - Tamil
+ TAM_LTTMBARANI = 65, // Barani - Tamil
+ TAM_SHREE = 66, // Shree - Tamil
+ TAM_TBOOMIS = 67, // TBoomis - Tamil
+ TAM_TMNEWS = 68, // TMNews - Tamil
+ TAM_WEBTAMIL = 69, // Webtamil - Tamil
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Shift_JIS variants used by Japanese cell phone carriers.
+ KDDI_SHIFT_JIS = 70,
+ DOCOMO_SHIFT_JIS = 71,
+ SOFTBANK_SHIFT_JIS = 72,
+ // ISO-2022-JP variants used by KDDI and SoftBank.
+ KDDI_ISO_2022_JP = 73,
+ SOFTBANK_ISO_2022_JP = 74,
+ //-----------------------------------------------------------
+
+ NUM_ENCODINGS = 75, // Always keep this at the end. It is not a
+ // valid Encoding enum, it is only used to
+ // indicate the total number of Encodings.
+};
+
+#endif // UTIL_ENCODINGS_ENCODINGS_PB_H_
diff --git a/contrib/google-ced/util/encodings/encodings_unittest.cc b/contrib/google-ced/util/encodings/encodings_unittest.cc
new file mode 100644
index 0000000..223e3e4
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings_unittest.cc
@@ -0,0 +1,34 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/encodings/encodings.h"
+
+#include "gtest/gtest.h"
+
+TEST(EncodingsTest, EncodingNameAliasToEncoding) {
+ // Test that cases, non-alpha-numeric chars are ignored.
+ EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso_8859_1"));
+ EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso-8859-1"));
+
+ // Test that spaces are ignored.
+ EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF8"));
+ EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF 8"));
+ EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF-8"));
+
+ // Test alphanumeric differences are counted.
+ EXPECT_NE(UTF8, EncodingNameAliasToEncoding("UTF-7"));
+ EXPECT_NE(KOREAN_EUC_KR, EncodingNameAliasToEncoding("euc-jp"));
+}
diff --git a/contrib/google-ced/util/languages/languages.cc b/contrib/google-ced/util/languages/languages.cc
new file mode 100644
index 0000000..852351f
--- /dev/null
+++ b/contrib/google-ced/util/languages/languages.cc
@@ -0,0 +1,349 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/languages/languages.h"
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+
+
+Language default_language() {return ENGLISH;}
+
+
+// Language names and codes
+
+struct LanguageInfo {
+ const char * language_name_;
+ const char * language_code_639_1_; // the ISO-639-1 code for the language
+ const char * language_code_639_2_; // the ISO-639-2 code for the language
+ const char * language_code_other_; // some nonstandard code for the language
+};
+
+static const LanguageInfo kLanguageInfoTable[] = {
+ { "ENGLISH", "en", "eng", NULL},
+ { "DANISH", "da", "dan", NULL},
+ { "DUTCH", "nl", "dut", NULL},
+ { "FINNISH", "fi", "fin", NULL},
+ { "FRENCH", "fr", "fre", NULL},
+ { "GERMAN", "de", "ger", NULL},
+ { "HEBREW", "he", "heb", NULL},
+ { "ITALIAN", "it", "ita", NULL},
+ { "Japanese", "ja", "jpn", NULL},
+ { "Korean", "ko", "kor", NULL},
+ { "NORWEGIAN", "nb", "nor", NULL},
+ { "POLISH", "pl", "pol", NULL},
+ { "PORTUGUESE", "pt", "por", NULL},
+ { "RUSSIAN", "ru", "rus", NULL},
+ { "SPANISH", "es", "spa", NULL},
+ { "SWEDISH", "sv", "swe", NULL},
+ { "Chinese", "zh", "chi", "zh-CN"},
+ { "CZECH", "cs", "cze", NULL},
+ { "GREEK", "el", "gre", NULL},
+ { "ICELANDIC", "is", "ice", NULL},
+ { "LATVIAN", "lv", "lav", NULL},
+ { "LITHUANIAN", "lt", "lit", NULL},
+ { "ROMANIAN", "ro", "rum", NULL},
+ { "HUNGARIAN", "hu", "hun", NULL},
+ { "ESTONIAN", "et", "est", NULL},
+ // TODO: Although Teragram has two output names "TG_UNKNOWN_LANGUAGE"
+ // and "Unknown", they are essentially the same. Need to unify them.
+ // "un" and "ut" are invented by us, not from ISO-639.
+ //
+ { "TG_UNKNOWN_LANGUAGE", NULL, NULL, "ut"},
+ { "Unknown", NULL, NULL, "un"},
+ { "BULGARIAN", "bg", "bul", NULL},
+ { "CROATIAN", "hr", "scr", NULL},
+ { "SERBIAN", "sr", "scc", NULL},
+ { "IRISH", "ga", "gle", NULL},
+ { "GALICIAN", "gl", "glg", NULL},
+ // Impossible to tell Tagalog from Filipino at the moment.
+ // Use ISO 639-2 code for Filipino here.
+ { "TAGALOG", NULL, "fil", NULL},
+ { "TURKISH", "tr", "tur", NULL},
+ { "UKRAINIAN", "uk", "ukr", NULL},
+ { "HINDI", "hi", "hin", NULL},
+ { "MACEDONIAN", "mk", "mac", NULL},
+ { "BENGALI", "bn", "ben", NULL},
+ { "INDONESIAN", "id", "ind", NULL},
+ { "LATIN", "la", "lat", NULL},
+ { "MALAY", "ms", "may", NULL},
+ { "MALAYALAM", "ml", "mal", NULL},
+ { "WELSH", "cy", "wel", NULL},
+ { "NEPALI", "ne", "nep", NULL},
+ { "TELUGU", "te", "tel", NULL},
+ { "ALBANIAN", "sq", "alb", NULL},
+ { "TAMIL", "ta", "tam", NULL},
+ { "BELARUSIAN", "be", "bel", NULL},
+ { "JAVANESE", "jw", "jav", NULL},
+ { "OCCITAN", "oc", "oci", NULL},
+ { "URDU", "ur", "urd", NULL},
+ { "BIHARI", "bh", "bih", NULL},
+ { "GUJARATI", "gu", "guj", NULL},
+ { "THAI", "th", "tha", NULL},
+ { "ARABIC", "ar", "ara", NULL},
+ { "CATALAN", "ca", "cat", NULL},
+ { "ESPERANTO", "eo", "epo", NULL},
+ { "BASQUE", "eu", "baq", NULL},
+ { "INTERLINGUA", "ia", "ina", NULL},
+ { "KANNADA", "kn", "kan", NULL},
+ { "PUNJABI", "pa", "pan", NULL},
+ { "SCOTS_GAELIC", "gd", "gla", NULL},
+ { "SWAHILI", "sw", "swa", NULL},
+ { "SLOVENIAN", "sl", "slv", NULL},
+ { "MARATHI", "mr", "mar", NULL},
+ { "MALTESE", "mt", "mlt", NULL},
+ { "VIETNAMESE", "vi", "vie", NULL},
+ { "FRISIAN", "fy", "fry", NULL},
+ { "SLOVAK", "sk", "slo", NULL},
+ { "ChineseT",
+ NULL, NULL, // We intentionally set these 2 fields to NULL to avoid
+ // confusion between CHINESE_T and CHINESE.
+ "zh-TW"},
+ { "FAROESE", "fo", "fao", NULL},
+ { "SUNDANESE", "su", "sun", NULL},
+ { "UZBEK", "uz", "uzb", NULL},
+ { "AMHARIC", "am", "amh", NULL},
+ { "AZERBAIJANI", "az", "aze", NULL},
+ { "GEORGIAN", "ka", "geo", NULL},
+ { "TIGRINYA", "ti", "tir", NULL},
+ { "PERSIAN", "fa", "per", NULL},
+ { "BOSNIAN", "bs", "bos", NULL},
+ { "SINHALESE", "si", "sin", NULL},
+ { "NORWEGIAN_N", "nn", "nno", NULL},
+ { "PORTUGUESE_P", NULL, NULL, "pt-PT"},
+ { "PORTUGUESE_B", NULL, NULL, "pt-BR"},
+ { "XHOSA", "xh", "xho", NULL},
+ { "ZULU", "zu", "zul", NULL},
+ { "GUARANI", "gn", "grn", NULL},
+ { "SESOTHO", "st", "sot", NULL},
+ { "TURKMEN", "tk", "tuk", NULL},
+ { "KYRGYZ", "ky", "kir", NULL},
+ { "BRETON", "br", "bre", NULL},
+ { "TWI", "tw", "twi", NULL},
+ { "YIDDISH", "yi", "yid", NULL},
+ { "SERBO_CROATIAN", "sh", NULL, NULL},
+ { "SOMALI", "so", "som", NULL},
+ { "UIGHUR", "ug", "uig", NULL},
+ { "KURDISH", "ku", "kur", NULL},
+ { "MONGOLIAN", "mn", "mon", NULL},
+ { "ARMENIAN", "hy", "arm", NULL},
+ { "LAOTHIAN", "lo", "lao", NULL},
+ { "SINDHI", "sd", "snd", NULL},
+ { "RHAETO_ROMANCE", "rm", "roh", NULL},
+ { "AFRIKAANS", "af", "afr", NULL},
+ { "LUXEMBOURGISH", "lb", "ltz", NULL},
+ { "BURMESE", "my", "bur", NULL},
+ // KHMER is known as Cambodian for Google user interfaces.
+ { "KHMER", "km", "khm", NULL},
+ { "TIBETAN", "bo", "tib", NULL},
+ { "DHIVEHI", "dv", "div", NULL},
+ { "CHEROKEE", NULL, "chr", NULL},
+ { "SYRIAC", NULL, "syr", NULL},
+ { "LIMBU", NULL, NULL, "sit-NP"},
+ { "ORIYA", "or", "ori", NULL},
+ { "ASSAMESE", "as", "asm", NULL},
+ { "CORSICAN", "co", "cos", NULL},
+ { "INTERLINGUE", "ie", "ine", NULL},
+ { "KAZAKH", "kk", "kaz", NULL},
+ { "LINGALA", "ln", "lin", NULL},
+ { "MOLDAVIAN", "mo", "mol", NULL},
+ { "PASHTO", "ps", "pus", NULL},
+ { "QUECHUA", "qu", "que", NULL},
+ { "SHONA", "sn", "sna", NULL},
+ { "TAJIK", "tg", "tgk", NULL},
+ { "TATAR", "tt", "tat", NULL},
+ { "TONGA", "to", "tog", NULL},
+ { "YORUBA", "yo", "yor", NULL},
+ { "CREOLES_AND_PIDGINS_ENGLISH_BASED", NULL, "cpe", NULL},
+ { "CREOLES_AND_PIDGINS_FRENCH_BASED", NULL, "cpf", NULL},
+ { "CREOLES_AND_PIDGINS_PORTUGUESE_BASED", NULL, "cpp", NULL},
+ { "CREOLES_AND_PIDGINS_OTHER", NULL, "crp", NULL},
+ { "MAORI", "mi", "mao", NULL},
+ { "WOLOF", "wo", "wol", NULL},
+ { "ABKHAZIAN", "ab", "abk", NULL},
+ { "AFAR", "aa", "aar", NULL},
+ { "AYMARA", "ay", "aym", NULL},
+ { "BASHKIR", "ba", "bak", NULL},
+ { "BISLAMA", "bi", "bis", NULL},
+ { "DZONGKHA", "dz", "dzo", NULL},
+ { "FIJIAN", "fj", "fij", NULL},
+ { "GREENLANDIC", "kl", "kal", NULL},
+ { "HAUSA", "ha", "hau", NULL},
+ { "HAITIAN_CREOLE", "ht", NULL, NULL},
+ { "INUPIAK", "ik", "ipk", NULL},
+ { "INUKTITUT", "iu", "iku", NULL},
+ { "KASHMIRI", "ks", "kas", NULL},
+ { "KINYARWANDA", "rw", "kin", NULL},
+ { "MALAGASY", "mg", "mlg", NULL},
+ { "NAURU", "na", "nau", NULL},
+ { "OROMO", "om", "orm", NULL},
+ { "RUNDI", "rn", "run", NULL},
+ { "SAMOAN", "sm", "smo", NULL},
+ { "SANGO", "sg", "sag", NULL},
+ { "SANSKRIT", "sa", "san", NULL},
+ { "SISWANT", "ss", "ssw", NULL},
+ { "TSONGA", "ts", "tso", NULL},
+ { "TSWANA", "tn", "tsn", NULL},
+ { "VOLAPUK", "vo", "vol", NULL},
+ { "ZHUANG", "za", "zha", NULL},
+ { "KHASI", NULL, "kha", NULL},
+ { "SCOTS", NULL, "sco", NULL},
+ { "GANDA", "lg", "lug", NULL},
+ { "MANX", "gv", "glv", NULL},
+ { "MONTENEGRIN", NULL, NULL, "sr-ME"},
+ { "XX", NULL, NULL, "XX"},
+};
+
+COMPILE_ASSERT(arraysize(kLanguageInfoTable) == NUM_LANGUAGES + 1,
+ kLanguageInfoTable_has_incorrect_length);
+
+
+// LANGUAGE NAMES
+
+const char* default_language_name() {
+ return kLanguageInfoTable[ENGLISH].language_name_;
+}
+
+static const char* const kInvalidLanguageName = "invalid_language";
+
+const char *invalid_language_name() {
+ return kInvalidLanguageName;
+}
+
+const char* LanguageName(Language lang) {
+ return IsValidLanguage(lang)
+ ? kLanguageInfoTable[lang].language_name_
+ : kInvalidLanguageName;
+}
+
+
+
+// LANGUAGE CODES
+
+
+// The space before invalid_language_code is intentional. It is used
+// to prevent it matching any two letter language code.
+//
+static const char* const kInvalidLanguageCode = " invalid_language_code";
+
+const char *invalid_language_code() {
+ return kInvalidLanguageCode;
+}
+
+const char * LanguageCode(Language lang) {
+ if (! IsValidLanguage(lang))
+ return kInvalidLanguageCode;
+ const LanguageInfo& info = kLanguageInfoTable[lang];
+ if (info.language_code_639_1_) {
+ return info.language_code_639_1_;
+ } else if (info.language_code_639_2_) {
+ return info.language_code_639_2_;
+ } else if (info.language_code_other_) {
+ return info.language_code_other_;
+ } else {
+ return kInvalidLanguageCode;
+ }
+}
+
+const char* default_language_code() {
+ return kLanguageInfoTable[ENGLISH].language_code_639_1_;
+}
+
+const char* LanguageCodeISO639_1(Language lang) {
+ if (! IsValidLanguage(lang))
+ return kInvalidLanguageCode;
+ if (const char* code = kLanguageInfoTable[lang].language_code_639_1_)
+ return code;
+ return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeISO639_2(Language lang) {
+ if (! IsValidLanguage(lang))
+ return kInvalidLanguageCode;
+ if (const char* code = kLanguageInfoTable[lang].language_code_639_2_)
+ return code;
+ return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeWithDialects(Language lang) {
+ if (lang == CHINESE)
+ return "zh-CN";
+ return LanguageCode(lang);
+}
+
+
+
+bool LanguageFromCode(const char* lang_code, Language *language) {
+ *language = UNKNOWN_LANGUAGE;
+ if ( lang_code == NULL ) return false;
+
+ for ( int i = 0 ; i < kNumLanguages ; i++ ) {
+ const LanguageInfo& info = kLanguageInfoTable[i];
+ if ((info.language_code_639_1_ &&
+ !base::strcasecmp(lang_code, info.language_code_639_1_)) ||
+ (info.language_code_639_2_ &&
+ !base::strcasecmp(lang_code, info.language_code_639_2_)) ||
+ (info.language_code_other_ &&
+ !base::strcasecmp(lang_code, info.language_code_other_))) {
+ *language = static_cast<Language>(i);
+ return true;
+ }
+ }
+
+ // For convenience, this function can also parse the non-standard
+ // five-letter language codes "zh-cn" and "zh-tw" which are used by
+ // front-ends such as GWS to distinguish Simplified from Traditional
+ // Chinese.
+ if (!base::strcasecmp(lang_code, "zh-cn") ||
+ !base::strcasecmp(lang_code, "zh_cn")) {
+ *language = CHINESE;
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "zh-tw") ||
+ !base::strcasecmp(lang_code, "zh_tw")) {
+ *language = CHINESE_T;
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "sr-me") ||
+ !base::strcasecmp(lang_code, "sr_me")) {
+ *language = MONTENEGRIN;
+ return true;
+ }
+
+ // Process language-code synonyms.
+ if (!base::strcasecmp(lang_code, "he")) {
+ *language = HEBREW; // Use "iw".
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "in")) {
+ *language = INDONESIAN; // Use "id".
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "ji")) {
+ *language = YIDDISH; // Use "yi".
+ return true;
+ }
+
+ // Process language-detection synonyms.
+ // These distinct languages cannot be differentiated by our current
+ // language-detection algorithms.
+ if (!base::strcasecmp(lang_code, "fil")) {
+ *language = TAGALOG;
+ return true;
+ }
+
+ return false;
+}
diff --git a/contrib/google-ced/util/languages/languages.h b/contrib/google-ced/util/languages/languages.h
new file mode 100644
index 0000000..4237961
--- /dev/null
+++ b/contrib/google-ced/util/languages/languages.h
@@ -0,0 +1,381 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LANGUAGES_LANGUAGES_H_
+#define UTIL_LANGUAGES_LANGUAGES_H_
+
+// This interface defines the Language enum and functions that depend
+// only on Language values.
+
+// A hash-function for Language, hash<Language>, is defined in
+// i18n/languages/public/languages-hash.h
+
+#ifndef SWIG
+// Language enum defined in languages.proto
+// Also description on how to add languages.
+#include "util/languages/languages.pb.h"
+
+#else
+
+// TODO: Include a header containing swig-compatible enum.
+
+#endif
+
+const int kNumLanguages = NUM_LANGUAGES;
+
+// Return the default language (ENGLISH).
+Language default_language();
+
+
+// *******************************************
+// Language predicates
+// IsValidLanguage()
+// IS_LANGUAGE_UNKNOWN()
+// IsCJKLanguage()
+// IsChineseLanguage()
+// IsNorwegianLanguage()
+// IsPortugueseLanguage()
+// IsRightToLeftLanguage()
+// IsMaybeRightToLeftLanguage()
+// IsSameLanguage()
+// IsScriptRequiringLongerSnippets()
+// *******************************************
+
+// IsValidLanguage
+// ===============
+//
+// Function to check if the input is within range of the Language enum. If
+// IsValidLanguage(lang) returns true, it is safe to call
+// static_cast<Language>(lang).
+//
+inline bool IsValidLanguage(int lang) {
+ return ((lang >= 0) && (lang < kNumLanguages));
+}
+
+// Return true if the language is "unknown". (This function was
+// previously a macro, hence the spelling in all caps.)
+//
+inline bool IS_LANGUAGE_UNKNOWN(Language lang) {
+ return lang == TG_UNKNOWN_LANGUAGE || lang == UNKNOWN_LANGUAGE;
+}
+
+// IsCJKLanguage
+// -------------
+//
+// This function returns true if the language is either Chinese
+// (simplified or traditional), Japanese, or Korean.
+bool IsCJKLanguage(Language lang);
+
+// IsChineseLanguage
+// -----------------
+//
+// This function returns true if the language is either Chinese
+// (simplified or traditional)
+bool IsChineseLanguage(Language lang);
+
+// IsNorwegianLanguage
+// --------------------
+//
+// This function returns true if the language is any of the Norwegian
+// (regular or Nynorsk).
+bool IsNorwegianLanguage(Language lang);
+
+// IsPortugueseLanguage
+// --------------------
+//
+// This function returns true if the language is any of the Portuguese
+// languages (regular, Portugal or Brazil)
+bool IsPortugueseLanguage(Language lang);
+
+// IsSameLanguage
+// --------------
+//
+// WARNING: This function provides only a simple test on the values of
+// the two Language arguments. It returns false if either language is
+// invalid. It returns true if the language arguments are equal, or
+// if they are both Chinese languages, both Norwegian languages, or
+// both Portuguese languages, as defined by IsChineseLanguage,
+// IsNorwegianLanguage, and IsPortugueseLanguage. Otherwise it returns
+// false.
+bool IsSameLanguage(Language lang1, Language lang2);
+
+
+// IsRightToLeftLanguage
+// ---------------------
+//
+// This function returns true if the language is only written right-to-left
+// (E.g., Hebrew, Arabic, Persian etc.)
+//
+// IMPORTANT NOTE: Technically we're talking about scripts, not languages.
+// There are languages that can be written in more than one script.
+// Examples:
+// - Kurdish and Azeri ('AZERBAIJANI') can be written left-to-right in
+// Latin or Cyrillic script, and right-to-left in Arabic script.
+// - Sindhi and Punjabi are written in different scripts, depending on
+// region and dialect.
+// - Turkmen used an Arabic script historically, but not any more.
+// - Pashto and Uyghur can use Arabic script, but use a Roman script
+// on the Internet.
+// - Kashmiri and Urdu are written either with Arabic or Devanagari script.
+//
+// This function only returns true for languages that are always, unequivocally
+// written in right-to-left script.
+//
+// TODO: If we want to do anything special with multi-script languages
+// we should create new 'languages' for each language+script, as we do for
+// traditional vs. simplified Chinese. However most such languages are rare in
+// use and even rarer on the web, so this is unlikely to be something we'll
+// be concerned with for a while.
+bool IsRightToLeftLanguage(Language lang);
+
+// IsMaybeRightToLeftLanguage
+// --------------------------
+//
+// This function returns true if the language may appear on the web in a
+// right-to-left script (E.g., Hebrew, Arabic, Persian, Urdu, Kurdish, etc.)
+//
+// NOTE: See important notes under IsRightToLeftLanguage(...).
+//
+// This function returns true for languages that *may* appear on the web in a
+// right-to-left script, even if they may also appear in a left-to-right
+// script.
+//
+// This function should typically be used in cases where doing some work on
+// left-to-right text would be OK (usually a no-op), and this function is used
+// just to cut down on unnecessary work on regular, LTR text.
+bool IsMaybeRightToLeftLanguage(Language lang);
+
+// IsScriptRequiringLongerSnippets
+// --------------------
+//
+// This function returns true if the script chracteristics require longer
+// snippet length (Devanagari, Bengali, Gurmukhi,
+// Gujarati, Oriya, Tamil, Telugu, Kannada, Malayalam).
+// COMMENTED OUT TO REDUCE DEPENDENCIES ON GOOGLE3 CODE
+// bool IsScriptRequiringLongerSnippets(UnicodeScript script);
+
+
+// *******************************************
+// LANGUAGE NAMES
+//
+// This interface defines a standard name for each valid Language,
+// and a standard name for invalid languages. Some language names use all
+// uppercase letters, but others use mixed case.
+// LanguageName() [Language to name]
+// LanguageEnumName() [language to enum name]
+// LanguageFromName() [name to Language]
+// default_language_name()
+// invalid_language_name()
+// *******************************************
+
+// Given a Language, returns its standard name.
+// Return invalid_language_name() if the language is invalid.
+const char* LanguageName(Language lang);
+
+// Given a Language, return the name of the enum constant for that
+// language. In all but a few cases, this is the same as its standard
+// name. For example, LanguageName(CHINESE) returns "Chinese", but
+// LanguageEnumName(CHINESE) returns "CHINESE". This is intended for
+// code that is generating C++ code, where the enum constant is more
+// useful than its integer value. Return "NUM_LANGUAGES" if
+// the language is invalid.
+const char* LanguageEnumName(Language lang);
+
+// The maximum length of a standard language name.
+const int kMaxLanguageNameSize = 50;
+
+// The standard name for the default language.
+const char* default_language_name();
+
+// The standard name for all invalid languages.
+const char* invalid_language_name();
+
+// If lang_name matches the standard name of a Language, using a
+// case-insensitive comparison, set *language to that Language and
+// return true.
+// Otherwise, set *language to UNKNOWN_LANGUAGE and return false.
+//
+// For backwards compatibility, "HATIAN_CREOLE" is allowed as a name
+// for HAITIAN_CREOLE, and "QUECHAU" is allowed as a name for QUECHUA.
+// For compatibility with LanguageEnumName, "UNKNOWN_LANGUAGE" is allowed
+// as a name for UNKNOWN_LANGUAGE (the return value is true in this case,
+// as it is for "Unknown"), and "CHINESE_T" is allowed as a name for
+// CHINESE_T (i.e., a synonym for "ChineseT").
+//
+// REQUIRES: language must not be NULL.
+//
+bool LanguageFromName(const char* lang_name, Language *language);
+
+
+
+// *******************************************
+// LANGUAGE CODES
+//
+// This interface defines a standard code for each valid language, and
+// a standard code for invalid languages. These are derived from ISO codes,
+// with some Google additions.
+// LanguageCode()
+// default_language_code()
+// invalid_language_code()
+// LanguageCodeWithDialects()
+// LanguageCodeISO639_1()
+// LanguageCodeISO639_2()
+// *******************************************
+
+// Given a Language, return its standard code. There are Google-specific codes:
+// For CHINESE_T, return "zh-TW".
+// For TG_UNKNOWN_LANGUAGE, return "ut".
+// For UNKNOWN_LANGUAGE, return "un".
+// For PORTUGUESE_P, return "pt-PT".
+// For PORTUGUESE_B, return "pt-BR".
+// For LIMBU, return "sit-NP".
+// For CHEROKEE, return "chr".
+// For SYRIAC, return "syr".
+// Otherwise return the ISO 639-1 two-letter language code for lang.
+// If lang is invalid, return invalid_language_code().
+//
+// NOTE: See the note below about the codes for Chinese languages.
+//
+const char* LanguageCode(Language lang);
+
+// The maximum length of a language code.
+const int kMaxLanguageCodeSize = 50;
+
+// The standard code for the default language.
+const char* default_language_code();
+
+// The standard code for all invalid languages.
+const char* invalid_language_code();
+
+
+// --------------------------------------------
+// NOTE: CHINESE LANGUAGE CODES
+//
+// There are three functions that return codes for Chinese languages.
+// LanguageCode(lang) and LanguageCodeWithDialects(lang) are defined here.
+// LanguageCode(lang, encoding) is defined in i18n/encodings.lang_enc.h.
+// The following list shows the different results.
+//
+// LanguageCode(CHINESE) returns "zh"
+// LanguageCode(CHINESE_T) returns "zh-TW".
+//
+// LanguageCodeWithDialects(CHINESE) returns "zh-CN".
+// LanguageCodeWithDialects(CHINESE_T) returns "zh-TW".
+//
+// LanguageCode(CHINESE_T, <any encoding>) returns "zh-TW".
+// LanguageCode(CHINESE, CHINESE_BIG5) returns "zh-TW".
+// LanguageCode(CHINESE, <any other encoding>) returns "zh-CN".
+//
+// --------------------------------------------
+
+// LanguageCodeWithDialects
+// ------------------------
+//
+// If lang is CHINESE, return "zh-CN". Otherwise return LanguageCode(lang).
+const char* LanguageCodeWithDialects(Language lang);
+
+// LanguageCodeISO639_1
+// --------------------
+//
+// Return the ISO 639-1 two-letter language code for lang.
+// Return invalid_language_code() if lang is invalid or does not have
+// an ISO 639-1 two-letter language code.
+const char* LanguageCodeISO639_1(Language lang);
+
+// LanguageCodeISO639_2
+// --------------------
+//
+// Return the ISO 639-2 three-letter language for lang.
+// Return invalid_language_code() if lang is invalid or does not have
+// an ISO 639-2 three-letter language code.
+const char* LanguageCodeISO639_2(Language lang);
+
+// LanguageFromCode
+// ----------------
+//
+// If lang_code matches the code for a Language, using a case-insensitive
+// comparison, set *lang to that Language and return true.
+// Otherwise, set *lang to UNKNOWN_LANGUAGE and return false.
+//
+// lang_code can be an ISO 639-1 (two-letter) code, an ISO 639-2
+// (three-letter) code, or a Google-specific code (see LanguageCode).
+//
+// Certain language-code aliases are also allowed:
+// For "zh-cn" and "zh_cn", set *lang to CHINESE.
+// For "zh-tw" and "zh_tw", set *lang to CHINESE_T.
+// For "he", set *lang to HEBREW.
+// For "in", set *lang to INDONESIAN.
+// For "ji", set *lang to YIDDISH.
+// For "fil", set *lang to TAGALOG.
+//
+// REQUIRES: 'lang' must not be NULL.
+bool LanguageFromCode(const char* lang_code, Language *language);
+
+
+// LanguageFromCodeOrName
+// ----------------------
+//
+// If lang_code_or_name is a language code or a language name.
+// set *language to the corresponding Language and return true.
+// Otherwise set *language to UNKNOWN_LANGUAGE and return false.
+//
+bool LanguageFromCodeOrName(const char* lang_code_or_name,
+ Language* language);
+
+// LanguageNameFromCode
+// --------------------
+//
+// If language_code is the code for a Language (see LanguageFromCode),
+// return the standard name of that language (see LanguageName).
+// Otherwise return invalid_language_name().
+//
+const char* LanguageNameFromCode(const char* language_code);
+
+
+// Miscellany
+
+// LanguageCodeToUnderscoreForm
+// ----------------------------
+//
+// Given a language code, convert the dash "-" to underscore "_".
+//
+// Specifically, if result_length <= strlen(lang_code), set result[0]
+// to '\0' and return false. Otherwise, copy lang_code to result,
+// converting every dash to an underscore, converting every character
+// before the first dash or underscore to lower case, and converting
+// every character after the first dash or underscore to upper
+// case. If there is no dash or underscore, convert the entire string
+// to lower case.
+//
+// REQUIRES: 'lang_code' must not be NULL. 'result' must not be NULL.
+
+bool LanguageCodeToUnderscoreForm(const char* lang_code,
+ char* result,
+ int result_length);
+
+//
+// AlwaysPutInExpectedRestrict
+// ---------------------------
+//
+// For Web pages in certain top-level domains, Web Search always
+// applies a "country restrict". If 'tld' matches one of those, using
+// a case-SENSITIVE comparison, set *expected_language to the Language
+// most commonly found in that top-level domain and return true.
+// Otherwise, set *expected_language to UNKNOWN_LANGUAGE and return false.
+bool AlwaysPutInExpectedRestrict(const char *tld, Language *expected_language);
+
+
+#endif // UTIL_LANGUAGES_LANGUAGES_H_
diff --git a/contrib/google-ced/util/languages/languages.pb.h b/contrib/google-ced/util/languages/languages.pb.h
new file mode 100644
index 0000000..84f1d6a
--- /dev/null
+++ b/contrib/google-ced/util/languages/languages.pb.h
@@ -0,0 +1,191 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LANGUAGES_LANGUAGES_PB_H_
+#define UTIL_LANGUAGES_LANGUAGES_PB_H_
+
+enum Language {
+ ENGLISH = 0,
+ DANISH = 1,
+ DUTCH = 2,
+ FINNISH = 3,
+ FRENCH = 4,
+ GERMAN = 5,
+ HEBREW = 6,
+ ITALIAN = 7,
+ JAPANESE = 8,
+ KOREAN = 9,
+ NORWEGIAN = 10,
+ POLISH = 11,
+ PORTUGUESE = 12,
+ RUSSIAN = 13,
+ SPANISH = 14,
+ SWEDISH = 15,
+ CHINESE = 16,
+ CZECH = 17,
+ GREEK = 18,
+ ICELANDIC = 19,
+ LATVIAN = 20,
+ LITHUANIAN = 21,
+ ROMANIAN = 22,
+ HUNGARIAN = 23,
+ ESTONIAN = 24,
+ TG_UNKNOWN_LANGUAGE = 25,
+ UNKNOWN_LANGUAGE = 26,
+ BULGARIAN = 27,
+ CROATIAN = 28,
+ SERBIAN = 29,
+ IRISH = 30, // UI only.
+ GALICIAN = 31,
+ TAGALOG = 32, // Tagalog (tl) + Filipino (fil),
+ TURKISH = 33,
+ UKRAINIAN = 34,
+ HINDI = 35,
+ MACEDONIAN = 36,
+ BENGALI = 37,
+ INDONESIAN = 38,
+ LATIN = 39, // UI only.
+ MALAY = 40,
+ MALAYALAM = 41,
+ WELSH = 42, // UI only.
+ NEPALI = 43,
+ TELUGU = 44,
+ ALBANIAN = 45,
+ TAMIL = 46,
+ BELARUSIAN = 47,
+ JAVANESE = 48, // UI only.
+ OCCITAN = 49, // UI only.
+ URDU = 50,
+ BIHARI = 51,
+ GUJARATI = 52,
+ THAI = 53,
+ ARABIC = 54,
+ CATALAN = 55,
+ ESPERANTO = 56,
+ BASQUE = 57,
+ INTERLINGUA = 58, // UI only.
+ KANNADA = 59,
+ PUNJABI = 60,
+ SCOTS_GAELIC = 61, // UI only.
+ SWAHILI = 62,
+ SLOVENIAN = 63,
+ MARATHI = 64,
+ MALTESE = 65,
+ VIETNAMESE = 66,
+ FRISIAN = 67, // UI only.
+ SLOVAK = 68,
+ CHINESE_T = 69, // This is added to solve the problem of
+ // distinguishing Traditional and Simplified
+ // Chinese when the encoding is UTF8.
+ FAROESE = 70, // UI only.
+ SUNDANESE = 71, // UI only.
+ UZBEK = 72,
+ AMHARIC = 73,
+ AZERBAIJANI = 74,
+ GEORGIAN = 75,
+ TIGRINYA = 76, // UI only.
+ PERSIAN = 77,
+ BOSNIAN = 78, // UI only. LangId language: CROATIAN (28)
+ SINHALESE = 79,
+ NORWEGIAN_N = 80, // UI only. LangId language: NORWEGIAN (10)
+ PORTUGUESE_P = 81, // UI only. LangId language: PORTUGUESE (12)
+ PORTUGUESE_B = 82, // UI only. LangId language: PORTUGUESE (12)
+ XHOSA = 83, // UI only.
+ ZULU = 84, // UI only.
+ GUARANI = 85,
+ SESOTHO = 86, // UI only.
+ TURKMEN = 87, // UI only.
+ KYRGYZ = 88,
+ BRETON = 89, // UI only.
+ TWI = 90, // UI only.
+ YIDDISH = 91, // UI only.
+ SERBO_CROATIAN= 92, // UI only. LangId language: SERBIAN (29)
+ SOMALI = 93, // UI only.
+ UIGHUR = 94,
+ KURDISH = 95,
+ MONGOLIAN = 96,
+ ARMENIAN = 97,
+ LAOTHIAN = 98,
+ SINDHI = 99,
+ RHAETO_ROMANCE= 100, // UI only.
+ AFRIKAANS = 101,
+ LUXEMBOURGISH = 102, // UI only.
+ BURMESE = 103,
+ KHMER = 104,
+ TIBETAN = 105,
+ DHIVEHI = 106, // sometimes spelled Divehi, lang of Maldives
+ CHEROKEE = 107,
+ SYRIAC = 108, // UI only.
+ LIMBU = 109, // UI only.
+ ORIYA = 110,
+ ASSAMESE = 111, // UI only.
+ CORSICAN = 112, // UI only.
+ INTERLINGUE = 113, // UI only.
+ KAZAKH = 114,
+ LINGALA = 115, // UI only.
+ MOLDAVIAN = 116, // UI only. LangId language: ROMANIAN (22)
+ PASHTO = 117,
+ QUECHUA = 118, // UI only.
+ SHONA = 119, // UI only.
+ TAJIK = 120,
+ TATAR = 121, // UI only.
+ TONGA = 122, // UI only.
+ YORUBA = 123, // UI only.
+ CREOLES_AND_PIDGINS_ENGLISH_BASED = 124, // UI only.
+ CREOLES_AND_PIDGINS_FRENCH_BASED = 125, // UI only.
+ CREOLES_AND_PIDGINS_PORTUGUESE_BASED = 126, // UI only.
+ CREOLES_AND_PIDGINS_OTHER = 127, // UI only.
+ MAORI = 128, // UI only.
+ WOLOF = 129, // UI only.
+ ABKHAZIAN = 130, // UI only.
+ AFAR = 131, // UI only.
+ AYMARA = 132, // UI only.
+ BASHKIR = 133, // UI only.
+ BISLAMA = 134, // UI only.
+ DZONGKHA = 135, // UI only.
+ FIJIAN = 136, // UI only.
+ GREENLANDIC = 137, // UI only.
+ HAUSA = 138, // UI only.
+ HAITIAN_CREOLE= 139, // UI only.
+ INUPIAK = 140, // UI only.
+ INUKTITUT = 141,
+ KASHMIRI = 142, // UI only.
+ KINYARWANDA = 143, // UI only.
+ MALAGASY = 144, // UI only.
+ NAURU = 145, // UI only.
+ OROMO = 146, // UI only.
+ RUNDI = 147, // UI only.
+ SAMOAN = 148, // UI only.
+ SANGO = 149, // UI only.
+ SANSKRIT = 150,
+ SISWANT = 151, // UI only.
+ TSONGA = 152, // UI only.
+ TSWANA = 153, // UI only.
+ VOLAPUK = 154, // UI only.
+ ZHUANG = 155, // UI only.
+ KHASI = 156, // UI only.
+ SCOTS = 157, // UI only.
+ GANDA = 158, // UI only.
+ MANX = 159, // UI only.
+ MONTENEGRIN = 160, // UI only. LangId language: SERBIAN (29)
+ NUM_LANGUAGES = 161, // Always keep this at the end. It is not a
+ // valid Language enum. It is only used to
+ // indicate the total number of Languages.
+ // NOTE: If you add a language, you will break a unittest. See the note
+ // at the top of this enum.
+};
+
+#endif // UTIL_LANGUAGES_LANGUAGES_PB_H_
diff --git a/contrib/google-ced/util/logging.h b/contrib/google-ced/util/logging.h
new file mode 100644
index 0000000..16e50f2
--- /dev/null
+++ b/contrib/google-ced/util/logging.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LOGGING_H_
+#define UTIL_LOGGING_H_
+
+#undef CHECK
+#define CHECK(expr)
+#undef DCHECK
+#define DCHECK(expr)
+
+#endif // UTIL_LOGGING_H_
diff --git a/contrib/google-ced/util/port.h b/contrib/google-ced/util/port.h
new file mode 100644
index 0000000..3799b16
--- /dev/null
+++ b/contrib/google-ced/util/port.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_PORT_H_
+#define UTIL_PORT_H_
+
+#include <stdarg.h>
+
+#if defined(_MSC_VER)
+#define GG_LONGLONG(x) x##I64
+#define GG_ULONGLONG(x) x##UI64
+#else
+#define GG_LONGLONG(x) x##LL
+#define GG_ULONGLONG(x) x##ULL
+#endif
+
+// Per C99 7.8.14, define __STDC_CONSTANT_MACROS before including <stdint.h>
+// to get the INTn_C and UINTn_C macros for integer constants. It's difficult
+// to guarantee any specific ordering of header includes, so it's difficult to
+// guarantee that the INTn_C macros can be defined by including <stdint.h> at
+// any specific point. Provide GG_INTn_C macros instead.
+
+#define GG_INT8_C(x) (x)
+#define GG_INT16_C(x) (x)
+#define GG_INT32_C(x) (x)
+#define GG_INT64_C(x) GG_LONGLONG(x)
+
+#define GG_UINT8_C(x) (x ## U)
+#define GG_UINT16_C(x) (x ## U)
+#define GG_UINT32_C(x) (x ## U)
+#define GG_UINT64_C(x) GG_ULONGLONG(x)
+
+// Define an OS-neutral wrapper for shared library entry points
+#if defined(_WIN32)
+#define API_CALL __stdcall
+#else
+#define API_CALL
+#endif
+
+#endif // UTIL_PORT_H_
diff --git a/contrib/google-ced/util/string_util.h b/contrib/google-ced/util/string_util.h
new file mode 100644
index 0000000..5977f4f
--- /dev/null
+++ b/contrib/google-ced/util/string_util.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_STRING_UTIL_H_
+#define UTIL_STRING_UTIL_H_
+
+#include <string.h>
+
+namespace base {
+
+#if defined(_WIN32)
+// Compare the two strings s1 and s2 without regard to case using
+// the current locale; returns 0 if they are equal, 1 if s1 > s2, and -1 if
+// s2 > s1 according to a lexicographic comparison.
+inline int strcasecmp(const char* s1, const char* s2) {
+ return _stricmp(s1, s2);
+}
+inline int strncasecmp(const char* s1, const char* s2, size_t n) {
+ return _strnicmp(s1, s2, n);
+}
+#else
+inline int strcasecmp(const char* s1, const char* s2) {
+ return ::strcasecmp(s1, s2);
+}
+inline int strncasecmp(const char* s1, const char* s2, size_t n) {
+ return ::strncasecmp(s1, s2, n);
+}
+#endif
+}
+
+#ifndef HAVE_MEMRCHR
+#if defined(__GLIBC__) && ((__GLIBC__ > 2) || ((__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 2)))
+#define HAVE_MEMRCHR
+#endif
+#endif
+
+#ifndef HAVE_MEMRCHR
+inline void* memrchr(const void* s, int c, size_t n) {
+ const unsigned char* p = (const unsigned char*) s;
+ for (p += n; n > 0; n--) {
+ if (*--p == c)
+ return (void*) p;
+ }
+ return NULL;
+}
+#endif
+
+#endif // UTIL_STRING_UTIL_H_
diff --git a/contrib/google-ced/util/varsetter.h b/contrib/google-ced/util/varsetter.h
new file mode 100644
index 0000000..8e8cbf2
--- /dev/null
+++ b/contrib/google-ced/util/varsetter.h
@@ -0,0 +1,66 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_VARSETTER_H_
+#define UTIL_VARSETTER_H_
+
+//
+// Use a VarSetter object to temporarily set an object of some sort to
+// a particular value. When the VarSetter object is destructed, the
+// underlying object will revert to its former value.
+//
+// Sample code:
+//
+#if 0
+{
+ bool b = true;
+ {
+ VarSetter<bool> bool_setter(&b, false);
+ // Now b == false.
+ }
+ // Now b == true again.
+}
+#endif
+
+template <class C>
+class VarSetter {
+public:
+
+ // Constructor that just sets the object to a fixed value
+ VarSetter(C* object, const C& value) : object_(object), old_value_(*object) {
+ *object = value;
+ }
+
+ ~VarSetter() { *object_ = old_value_; }
+
+private:
+
+ C*const object_;
+ C old_value_;
+
+ // Disallow
+ VarSetter(const VarSetter&);
+ VarSetter& operator=(const VarSetter&);
+
+ // VarSetters always live on the stack
+ static void* operator new (size_t);
+ static void* operator new[](size_t); // Redundant, no default ctor
+
+ static void operator delete (void*);
+ static void operator delete[](void*);
+};
+
+#endif // UTIL_VARSETTER_H_
diff --git a/contrib/hiredis/CMakeLists.txt b/contrib/hiredis/CMakeLists.txt
new file mode 100644
index 0000000..1e05631
--- /dev/null
+++ b/contrib/hiredis/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(HIREDISSRC async.c
+ dict.c
+ hiredis.c
+ net.c
+ read.c
+ sds.c)
+
+SET(HIREDIS_CFLAGS "")
+ADD_LIBRARY(rspamd-hiredis STATIC ${HIREDISSRC})
+
+SET_TARGET_PROPERTIES(rspamd-hiredis PROPERTIES COMPILE_FLAGS "${HIREDIS_CFLAGS}") \ No newline at end of file
diff --git a/contrib/hiredis/COPYING b/contrib/hiredis/COPYING
new file mode 100644
index 0000000..a5fc973
--- /dev/null
+++ b/contrib/hiredis/COPYING
@@ -0,0 +1,29 @@
+Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+* Neither the name of Redis nor the names of its contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/hiredis/README.md b/contrib/hiredis/README.md
new file mode 100644
index 0000000..4f1a58d
--- /dev/null
+++ b/contrib/hiredis/README.md
@@ -0,0 +1,392 @@
+[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis)
+
+# HIREDIS
+
+Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database.
+
+It is minimalistic because it just adds minimal support for the protocol, but
+at the same time it uses a high level printf-alike API in order to make it
+much higher level than otherwise suggested by its minimal code base and the
+lack of explicit bindings for every Redis command.
+
+Apart from supporting sending commands and receiving replies, it comes with
+a reply parser that is decoupled from the I/O layer. It
+is a stream parser designed for easy reusability, which can for instance be used
+in higher level language bindings for efficient reply parsing.
+
+Hiredis only supports the binary-safe Redis protocol, so you can use it with any
+Redis version >= 1.2.0.
+
+The library comes with multiple APIs. There is the
+*synchronous API*, the *asynchronous API* and the *reply parsing API*.
+
+## UPGRADING
+
+Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing
+code using hiredis should not be a big pain. The key thing to keep in mind when
+upgrading is that hiredis >= 0.9.0 uses a `redisContext*` to keep state, in contrast to
+the stateless 0.0.1 that only has a file descriptor to work with.
+
+## Synchronous API
+
+To consume the synchronous API, there are only a few function calls that need to be introduced:
+
+```c
+redisContext *redisConnect(const char *ip, int port);
+void *redisCommand(redisContext *c, const char *format, ...);
+void freeReplyObject(void *reply);
+```
+
+### Connecting
+
+The function `redisConnect` is used to create a so-called `redisContext`. The
+context is where Hiredis holds state for a connection. The `redisContext`
+struct has an integer `err` field that is non-zero when the connection is in
+an error state. The field `errstr` will contain a string with a description of
+the error. More information on errors can be found in the **Errors** section.
+After trying to connect to Redis using `redisConnect` you should
+check the `err` field to see if establishing the connection was successful:
+```c
+redisContext *c = redisConnect("127.0.0.1", 6379);
+if (c != NULL && c->err) {
+ printf("Error: %s\n", c->errstr);
+ // handle error
+}
+```
+
+### Sending commands
+
+There are several ways to issue commands to Redis. The first that will be introduced is
+`redisCommand`. This function takes a format similar to printf. In the simplest form,
+it is used like this:
+```c
+reply = redisCommand(context, "SET foo bar");
+```
+
+The specifier `%s` interpolates a string in the command, and uses `strlen` to
+determine the length of the string:
+```c
+reply = redisCommand(context, "SET foo %s", value);
+```
+When you need to pass binary safe strings in a command, the `%b` specifier can be
+used. Together with a pointer to the string, it requires a `size_t` length argument
+of the string:
+```c
+reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen);
+```
+Internally, Hiredis splits the command in different arguments and will
+convert it to the protocol used to communicate with Redis.
+One or more spaces separates arguments, so you can use the specifiers
+anywhere in an argument:
+```c
+reply = redisCommand(context, "SET key:%s %s", myid, value);
+```
+
+### Using replies
+
+The return value of `redisCommand` holds a reply when the command was
+successfully executed. When an error occurs, the return value is `NULL` and
+the `err` field in the context will be set (see section on **Errors**).
+Once an error is returned the context cannot be reused and you should set up
+a new connection.
+
+The standard replies that `redisCommand` are of the type `redisReply`. The
+`type` field in the `redisReply` should be used to test what kind of reply
+was received:
+
+* **`REDIS_REPLY_STATUS`**:
+ * The command replied with a status reply. The status string can be accessed using `reply->str`.
+ The length of this string can be accessed using `reply->len`.
+
+* **`REDIS_REPLY_ERROR`**:
+ * The command replied with an error. The error string can be accessed identical to `REDIS_REPLY_STATUS`.
+
+* **`REDIS_REPLY_INTEGER`**:
+ * The command replied with an integer. The integer value can be accessed using the
+ `reply->integer` field of type `long long`.
+
+* **`REDIS_REPLY_NIL`**:
+ * The command replied with a **nil** object. There is no data to access.
+
+* **`REDIS_REPLY_STRING`**:
+ * A bulk (string) reply. The value of the reply can be accessed using `reply->str`.
+ The length of this string can be accessed using `reply->len`.
+
+* **`REDIS_REPLY_ARRAY`**:
+ * A multi bulk reply. The number of elements in the multi bulk reply is stored in
+ `reply->elements`. Every element in the multi bulk reply is a `redisReply` object as well
+ and can be accessed via `reply->element[..index..]`.
+ Redis may reply with nested arrays but this is fully supported.
+
+Replies should be freed using the `freeReplyObject()` function.
+Note that this function will take care of freeing sub-reply objects
+contained in arrays and nested arrays, so there is no need for the user to
+free the sub replies (it is actually harmful and will corrupt the memory).
+
+**Important:** the current version of hiredis (0.10.0) frees replies when the
+asynchronous API is used. This means you should not call `freeReplyObject` when
+you use this API. The reply is cleaned up by hiredis _after_ the callback
+returns. This behavior will probably change in future releases, so make sure to
+keep an eye on the changelog when upgrading (see issue #39).
+
+### Cleaning up
+
+To disconnect and free the context the following function can be used:
+```c
+void redisFree(redisContext *c);
+```
+This function immediately closes the socket and then frees the allocations done in
+creating the context.
+
+### Sending commands (cont'd)
+
+Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands.
+It has the following prototype:
+```c
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+```
+It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the
+arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will
+use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments
+need to be binary safe, the entire array of lengths `argvlen` should be provided.
+
+The return value has the same semantic as `redisCommand`.
+
+### Pipelining
+
+To explain how Hiredis supports pipelining in a blocking connection, there needs to be
+understanding of the internal execution flow.
+
+When any of the functions in the `redisCommand` family is called, Hiredis first formats the
+command according to the Redis protocol. The formatted command is then put in the output buffer
+of the context. This output buffer is dynamic, so it can hold any number of commands.
+After the command is put in the output buffer, `redisGetReply` is called. This function has the
+following two execution paths:
+
+1. The input buffer is non-empty:
+ * Try to parse a single reply from the input buffer and return it
+ * If no reply could be parsed, continue at *2*
+2. The input buffer is empty:
+ * Write the **entire** output buffer to the socket
+ * Read from the socket until a single reply could be parsed
+
+The function `redisGetReply` is exported as part of the Hiredis API and can be used when a reply
+is expected on the socket. To pipeline commands, the only things that needs to be done is
+filling up the output buffer. For this cause, two commands can be used that are identical
+to the `redisCommand` family, apart from not returning a reply:
+```c
+void redisAppendCommand(redisContext *c, const char *format, ...);
+void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+```
+After calling either function one or more times, `redisGetReply` can be used to receive the
+subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where
+the latter means an error occurred while reading a reply. Just as with the other commands,
+the `err` field in the context can be used to find out what the cause of this error is.
+
+The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and
+a single call to `read(2)`):
+```c
+redisReply *reply;
+redisAppendCommand(context,"SET foo bar");
+redisAppendCommand(context,"GET foo");
+redisGetReply(context,&reply); // reply for SET
+freeReplyObject(reply);
+redisGetReply(context,&reply); // reply for GET
+freeReplyObject(reply);
+```
+This API can also be used to implement a blocking subscriber:
+```c
+reply = redisCommand(context,"SUBSCRIBE foo");
+freeReplyObject(reply);
+while(redisGetReply(context,&reply) == REDIS_OK) {
+ // consume message
+ freeReplyObject(reply);
+}
+```
+### Errors
+
+When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is
+returned. The `err` field inside the context will be non-zero and set to one of the
+following constants:
+
+* **`REDIS_ERR_IO`**:
+ There was an I/O error while creating the connection, trying to write
+ to the socket or read from the socket. If you included `errno.h` in your
+ application, you can use the global `errno` variable to find out what is
+ wrong.
+
+* **`REDIS_ERR_EOF`**:
+ The server closed the connection which resulted in an empty read.
+
+* **`REDIS_ERR_PROTOCOL`**:
+ There was an error while parsing the protocol.
+
+* **`REDIS_ERR_OTHER`**:
+ Any other error. Currently, it is only used when a specified hostname to connect
+ to cannot be resolved.
+
+In every case, the `errstr` field in the context will be set to hold a string representation
+of the error.
+
+## Asynchronous API
+
+Hiredis comes with an asynchronous API that works easily with any event library.
+Examples are bundled that show using Hiredis with [libev](http://software.schmorp.de/pkg/libev.html)
+and [libevent](http://monkey.org/~provos/libevent/).
+
+### Connecting
+
+The function `redisAsyncConnect` can be used to establish a non-blocking connection to
+Redis. It returns a pointer to the newly created `redisAsyncContext` struct. The `err` field
+should be checked after creation to see if there were errors creating the connection.
+Because the connection that will be created is non-blocking, the kernel is not able to
+instantly return if the specified host and port is able to accept a connection.
+```c
+redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+if (c->err) {
+ printf("Error: %s\n", c->errstr);
+ // handle error
+}
+```
+
+The asynchronous context can hold a disconnect callback function that is called when the
+connection is disconnected (either because of an error or per user request). This function should
+have the following prototype:
+```c
+void(const redisAsyncContext *c, int status);
+```
+On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the
+user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err`
+field in the context can be accessed to find out the cause of the error.
+
+The context object is always freed after the disconnect callback fired. When a reconnect is needed,
+the disconnect callback is a good point to do so.
+
+Setting the disconnect callback can only be done once per context. For subsequent calls it will
+return `REDIS_ERR`. The function to set the disconnect callback has the following prototype:
+```c
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
+```
+### Sending commands and their callbacks
+
+In an asynchronous context, commands are automatically pipelined due to the nature of an event loop.
+Therefore, unlike the synchronous API, there is only a single way to send commands.
+Because commands are sent to Redis asynchronously, issuing a command requires a callback function
+that is called when the reply is received. Reply callbacks should have the following prototype:
+```c
+void(redisAsyncContext *c, void *reply, void *privdata);
+```
+The `privdata` argument can be used to curry arbitrary data to the callback from the point where
+the command is initially queued for execution.
+
+The functions that can be used to issue commands in an asynchronous context are:
+```c
+int redisAsyncCommand(
+ redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
+ const char *format, ...);
+int redisAsyncCommandArgv(
+ redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
+ int argc, const char **argv, const size_t *argvlen);
+```
+Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command
+was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection
+is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is
+returned on calls to the `redisAsyncCommand` family.
+
+If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback
+for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only
+valid for the duration of the callback.
+
+All pending callbacks are called with a `NULL` reply when the context encountered an error.
+
+### Disconnecting
+
+An asynchronous connection can be terminated using:
+```c
+void redisAsyncDisconnect(redisAsyncContext *ac);
+```
+When this function is called, the connection is **not** immediately terminated. Instead, new
+commands are no longer accepted and the connection is only terminated when all pending commands
+have been written to the socket, their respective replies have been read and their respective
+callbacks have been executed. After this, the disconnection callback is executed with the
+`REDIS_OK` status and the context object is freed.
+
+### Hooking it up to event library *X*
+
+There are a few hooks that need to be set on the context object after it is created.
+See the `adapters/` directory for bindings to *libev* and *libevent*.
+
+## Reply parsing API
+
+Hiredis comes with a reply parsing API that makes it easy for writing higher
+level language bindings.
+
+The reply parsing API consists of the following functions:
+```c
+redisReader *redisReaderCreate(void);
+void redisReaderFree(redisReader *reader);
+int redisReaderFeed(redisReader *reader, const char *buf, size_t len);
+int redisReaderGetReply(redisReader *reader, void **reply);
+```
+The same set of functions are used internally by hiredis when creating a
+normal Redis context, the above API just exposes it to the user for a direct
+usage.
+
+### Usage
+
+The function `redisReaderCreate` creates a `redisReader` structure that holds a
+buffer with unparsed data and state for the protocol parser.
+
+Incoming data -- most likely from a socket -- can be placed in the internal
+buffer of the `redisReader` using `redisReaderFeed`. This function will make a
+copy of the buffer pointed to by `buf` for `len` bytes. This data is parsed
+when `redisReaderGetReply` is called. This function returns an integer status
+and a reply object (as described above) via `void **reply`. The returned status
+can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went
+wrong (either a protocol error, or an out of memory error).
+
+The parser limits the level of nesting for multi bulk payloads to 7. If the
+multi bulk nesting level is higher than this, the parser returns an error.
+
+### Customizing replies
+
+The function `redisReaderGetReply` creates `redisReply` and makes the function
+argument `reply` point to the created `redisReply` variable. For instance, if
+the response of type `REDIS_REPLY_STATUS` then the `str` field of `redisReply`
+will hold the status as a vanilla C string. However, the functions that are
+responsible for creating instances of the `redisReply` can be customized by
+setting the `fn` field on the `redisReader` struct. This should be done
+immediately after creating the `redisReader`.
+
+For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c)
+uses customized reply object functions to create Ruby objects.
+
+### Reader max buffer
+
+Both when using the Reader API directly or when using it indirectly via a
+normal Redis context, the redisReader structure uses a buffer in order to
+accumulate data from the server.
+Usually this buffer is destroyed when it is empty and is larger than 16
+KiB in order to avoid wasting memory in unused buffers
+
+However when working with very big payloads destroying the buffer may slow
+down performances considerably, so it is possible to modify the max size of
+an idle buffer changing the value of the `maxbuf` field of the reader structure
+to the desired value. The special value of 0 means that there is no maximum
+value for an idle buffer, so the buffer will never get freed.
+
+For instance if you have a normal Redis context you can set the maximum idle
+buffer to zero (unlimited) just with:
+```c
+context->reader->maxbuf = 0;
+```
+This should be done only in order to maximize performances when working with
+large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again
+as soon as possible in order to prevent allocation of useless memory.
+
+## AUTHORS
+
+Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and
+Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license.
+Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and
+Jan-Erik Rediger (janerik at fnordig dot com)
diff --git a/contrib/hiredis/adapters/libev.h b/contrib/hiredis/adapters/libev.h
new file mode 100644
index 0000000..9cf00df
--- /dev/null
+++ b/contrib/hiredis/adapters/libev.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_LIBEV_H__
+#define __HIREDIS_LIBEV_H__
+#include <stdlib.h>
+#include <sys/types.h>
+#include "contrib/libev/ev.h"
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct redisLibevEvents {
+ redisAsyncContext *context;
+ struct ev_loop *loop;
+ int reading, writing;
+ ev_io rev, wev;
+} redisLibevEvents;
+
+static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
+#if EV_MULTIPLICITY
+ ((void)loop);
+#endif
+ ((void)revents);
+
+ redisLibevEvents *e = (redisLibevEvents*)watcher->data;
+ redisAsyncHandleRead(e->context);
+}
+
+static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
+#if EV_MULTIPLICITY
+ ((void)loop);
+#endif
+ ((void)revents);
+
+ redisLibevEvents *e = (redisLibevEvents*)watcher->data;
+ redisAsyncHandleWrite(e->context);
+}
+
+static void redisLibevAddRead(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (!e->reading) {
+ e->reading = 1;
+ ev_io_start(EV_A_ &e->rev);
+ }
+}
+
+static void redisLibevDelRead(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (e->reading) {
+ e->reading = 0;
+ ev_io_stop(EV_A_ &e->rev);
+ }
+}
+
+static void redisLibevAddWrite(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (!e->writing) {
+ e->writing = 1;
+ ev_io_start(EV_A_ &e->wev);
+ }
+}
+
+static void redisLibevDelWrite(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (e->writing) {
+ e->writing = 0;
+ ev_io_stop(EV_A_ &e->wev);
+ }
+}
+
+static void redisLibevCleanup(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ redisLibevDelRead(privdata);
+ redisLibevDelWrite(privdata);
+ free(e);
+}
+
+static int redisLibevAttach(EV_P_ redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ redisLibevEvents *e;
+
+ /* Nothing should be attached when something is already attached */
+ if (ac->ev.data != NULL)
+ return REDIS_ERR;
+
+ /* Create container for context and r/w events */
+ e = (redisLibevEvents*)malloc(sizeof(*e));
+ e->context = ac;
+#if EV_MULTIPLICITY
+ e->loop = loop;
+#else
+ e->loop = NULL;
+#endif
+ e->reading = e->writing = 0;
+ e->rev.data = e;
+ e->wev.data = e;
+
+ /* Register functions to start/stop listening for events */
+ ac->ev.addRead = redisLibevAddRead;
+ ac->ev.delRead = redisLibevDelRead;
+ ac->ev.addWrite = redisLibevAddWrite;
+ ac->ev.delWrite = redisLibevDelWrite;
+ ac->ev.cleanup = redisLibevCleanup;
+ ac->ev.data = e;
+
+ /* Initialize read/write events */
+ ev_io_init(&e->rev,redisLibevReadEvent,c->fd,EV_READ);
+ ev_io_init(&e->wev,redisLibevWriteEvent,c->fd,EV_WRITE);
+ return REDIS_OK;
+}
+
+#endif
diff --git a/contrib/hiredis/adapters/libevent.h b/contrib/hiredis/adapters/libevent.h
new file mode 100644
index 0000000..1c2b271
--- /dev/null
+++ b/contrib/hiredis/adapters/libevent.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_LIBEVENT_H__
+#define __HIREDIS_LIBEVENT_H__
+#include <event.h>
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct redisLibeventEvents {
+ redisAsyncContext *context;
+ struct event rev, wev;
+} redisLibeventEvents;
+
+static void redisLibeventReadEvent(int fd, short event, void *arg) {
+ ((void)fd); ((void)event);
+ redisLibeventEvents *e = (redisLibeventEvents*)arg;
+ redisAsyncHandleRead(e->context);
+}
+
+static void redisLibeventWriteEvent(int fd, short event, void *arg) {
+ ((void)fd); ((void)event);
+ redisLibeventEvents *e = (redisLibeventEvents*)arg;
+ redisAsyncHandleWrite(e->context);
+}
+
+static void redisLibeventAddRead(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_add(&e->rev,NULL);
+}
+
+static void redisLibeventDelRead(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_del(&e->rev);
+}
+
+static void redisLibeventAddWrite(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_add(&e->wev,NULL);
+}
+
+static void redisLibeventDelWrite(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_del(&e->wev);
+}
+
+static void redisLibeventCleanup(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_del(&e->rev);
+ event_del(&e->wev);
+ free(e);
+}
+
+static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) {
+ redisContext *c = &(ac->c);
+ redisLibeventEvents *e;
+
+ /* Nothing should be attached when something is already attached */
+ if (ac->ev.data != NULL)
+ return REDIS_ERR;
+
+ /* Create container for context and r/w events */
+ e = (redisLibeventEvents*)malloc(sizeof(*e));
+ e->context = ac;
+
+ /* Register functions to start/stop listening for events */
+ ac->ev.addRead = redisLibeventAddRead;
+ ac->ev.delRead = redisLibeventDelRead;
+ ac->ev.addWrite = redisLibeventAddWrite;
+ ac->ev.delWrite = redisLibeventDelWrite;
+ ac->ev.cleanup = redisLibeventCleanup;
+ ac->ev.data = e;
+
+ /* Initialize and install read/write events */
+ event_set(&e->rev,c->fd,EV_READ,redisLibeventReadEvent,e);
+ event_set(&e->wev,c->fd,EV_WRITE,redisLibeventWriteEvent,e);
+ event_base_set(base,&e->rev);
+ event_base_set(base,&e->wev);
+ return REDIS_OK;
+}
+#endif
diff --git a/contrib/hiredis/async.c b/contrib/hiredis/async.c
new file mode 100644
index 0000000..afa5413
--- /dev/null
+++ b/contrib/hiredis/async.c
@@ -0,0 +1,693 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <assert.h>
+#include <ctype.h>
+#include <errno.h>
+#include "async.h"
+#include "net.h"
+#include "dict.c"
+#include "sds.h"
+
+#define _EL_ADD_READ(ctx) do { \
+ if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \
+ } while(0)
+#define _EL_DEL_READ(ctx) do { \
+ if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \
+ } while(0)
+#define _EL_ADD_WRITE(ctx) do { \
+ if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \
+ } while(0)
+#define _EL_DEL_WRITE(ctx) do { \
+ if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \
+ } while(0)
+#define _EL_CLEANUP(ctx) do { \
+ if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \
+ } while(0);
+
+/* Forward declaration of function in hiredis.c */
+int __redisAppendCommand(redisContext *c, const char *cmd, size_t len);
+
+/* Functions managing dictionary of callbacks for pub/sub. */
+static unsigned int callbackHash(const void *key) {
+ return dictGenHashFunction((const unsigned char *)key,
+ sdslen((const sds)key));
+}
+
+static void *callbackValDup(void *privdata, const void *src) {
+ ((void) privdata);
+ redisCallback *dup = malloc(sizeof(*dup));
+ memcpy(dup,src,sizeof(*dup));
+ return dup;
+}
+
+static int callbackKeyCompare(void *privdata, const void *key1, const void *key2) {
+ int l1, l2;
+ ((void) privdata);
+
+ l1 = sdslen((const sds)key1);
+ l2 = sdslen((const sds)key2);
+ if (l1 != l2) return 0;
+ return memcmp(key1,key2,l1) == 0;
+}
+
+static void callbackKeyDestructor(void *privdata, void *key) {
+ ((void) privdata);
+ sdsfree((sds)key);
+}
+
+static void callbackValDestructor(void *privdata, void *val) {
+ ((void) privdata);
+ free(val);
+}
+
+static dictType callbackDict = {
+ callbackHash,
+ NULL,
+ callbackValDup,
+ callbackKeyCompare,
+ callbackKeyDestructor,
+ callbackValDestructor
+};
+
+static redisAsyncContext *redisAsyncInitialize(redisContext *c) {
+ redisAsyncContext *ac;
+
+ ac = realloc(c,sizeof(redisAsyncContext));
+ if (ac == NULL)
+ return NULL;
+
+ c = &(ac->c);
+
+ /* The regular connect functions will always set the flag REDIS_CONNECTED.
+ * For the async API, we want to wait until the first write event is
+ * received up before setting this flag, so reset it here. */
+ c->flags &= ~REDIS_CONNECTED;
+
+ ac->err = 0;
+ ac->errstr = NULL;
+ ac->data = NULL;
+
+ ac->ev.data = NULL;
+ ac->ev.addRead = NULL;
+ ac->ev.delRead = NULL;
+ ac->ev.addWrite = NULL;
+ ac->ev.delWrite = NULL;
+ ac->ev.cleanup = NULL;
+
+ ac->onConnect = NULL;
+ ac->onDisconnect = NULL;
+
+ ac->replies.head = NULL;
+ ac->replies.tail = NULL;
+ ac->sub.invalid.head = NULL;
+ ac->sub.invalid.tail = NULL;
+ ac->sub.channels = dictCreate(&callbackDict,NULL);
+ ac->sub.patterns = dictCreate(&callbackDict,NULL);
+ return ac;
+}
+
+/* We want the error field to be accessible directly instead of requiring
+ * an indirection to the redisContext struct. */
+static void __redisAsyncCopyError(redisAsyncContext *ac) {
+ if (!ac)
+ return;
+
+ redisContext *c = &(ac->c);
+ ac->err = c->err;
+ ac->errstr = c->errstr;
+}
+
+redisAsyncContext *redisAsyncConnect(const char *ip, int port) {
+ redisContext *c;
+ redisAsyncContext *ac;
+
+ c = redisConnectNonBlock(ip,port);
+ if (c == NULL)
+ return NULL;
+
+ ac = redisAsyncInitialize(c);
+ if (ac == NULL) {
+ redisFree(c);
+ return NULL;
+ }
+
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+redisAsyncContext *redisAsyncConnectBind(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisConnectBindNonBlock(ip,port,source_addr);
+ redisAsyncContext *ac = redisAsyncInitialize(c);
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr);
+ redisAsyncContext *ac = redisAsyncInitialize(c);
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+redisAsyncContext *redisAsyncConnectUnix(const char *path) {
+ redisContext *c;
+ redisAsyncContext *ac;
+
+ c = redisConnectUnixNonBlock(path);
+ if (c == NULL)
+ return NULL;
+
+ ac = redisAsyncInitialize(c);
+ if (ac == NULL) {
+ redisFree(c);
+ return NULL;
+ }
+
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) {
+ if (ac->onConnect == NULL) {
+ ac->onConnect = fn;
+
+ /* The common way to detect an established connection is to wait for
+ * the first write event to be fired. This assumes the related event
+ * library functions are already set. */
+ _EL_ADD_WRITE(ac);
+ return REDIS_OK;
+ }
+ return REDIS_ERR;
+}
+
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn) {
+ if (ac->onDisconnect == NULL) {
+ ac->onDisconnect = fn;
+ return REDIS_OK;
+ }
+ return REDIS_ERR;
+}
+
+/* Helper functions to push/shift callbacks */
+static int __redisPushCallback(redisCallbackList *list, redisCallback *source) {
+ redisCallback *cb;
+
+ /* Copy callback from stack to heap */
+ cb = malloc(sizeof(*cb));
+ if (cb == NULL)
+ return REDIS_ERR_OOM;
+
+ if (source != NULL) {
+ memcpy(cb,source,sizeof(*cb));
+ cb->next = NULL;
+ }
+
+ /* Store callback in list */
+ if (list->head == NULL)
+ list->head = cb;
+ if (list->tail != NULL)
+ list->tail->next = cb;
+ list->tail = cb;
+ return REDIS_OK;
+}
+
+static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) {
+ redisCallback *cb = list->head;
+ if (cb != NULL) {
+ list->head = cb->next;
+ if (cb == list->tail)
+ list->tail = NULL;
+
+ /* Copy callback from heap to stack */
+ if (target != NULL)
+ memcpy(target,cb,sizeof(*cb));
+ free(cb);
+ return REDIS_OK;
+ }
+ return REDIS_ERR;
+}
+
+static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisReply *reply) {
+ redisContext *c = &(ac->c);
+ if (cb->fn != NULL) {
+ c->flags |= REDIS_IN_CALLBACK;
+ cb->fn(ac,reply,cb->privdata);
+ c->flags &= ~REDIS_IN_CALLBACK;
+ }
+}
+
+/* Helper function to free the context. */
+static void __redisAsyncFree(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ redisCallback cb;
+ dictIterator *it;
+ dictEntry *de;
+
+ /* Execute pending callbacks with NULL reply. */
+ while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK)
+ __redisRunCallback(ac,&cb,NULL);
+
+ /* Execute callbacks for invalid commands */
+ while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK)
+ __redisRunCallback(ac,&cb,NULL);
+
+ /* Run subscription callbacks callbacks with NULL reply */
+ it = dictGetIterator(ac->sub.channels);
+ while ((de = dictNext(it)) != NULL)
+ __redisRunCallback(ac,dictGetEntryVal(de),NULL);
+ dictReleaseIterator(it);
+ dictRelease(ac->sub.channels);
+
+ it = dictGetIterator(ac->sub.patterns);
+ while ((de = dictNext(it)) != NULL)
+ __redisRunCallback(ac,dictGetEntryVal(de),NULL);
+ dictReleaseIterator(it);
+ dictRelease(ac->sub.patterns);
+
+ /* Signal event lib to clean up */
+ _EL_CLEANUP(ac);
+
+ /* Execute disconnect callback. When redisAsyncFree() initiated destroying
+ * this context, the status will always be REDIS_OK. */
+ if (ac->onDisconnect && (c->flags & REDIS_CONNECTED)) {
+ if (c->flags & REDIS_FREEING) {
+ ac->onDisconnect(ac,REDIS_OK);
+ } else {
+ c->flags |= REDIS_FREEING;
+ ac->onDisconnect(ac,(ac->err == 0) ? REDIS_OK : REDIS_ERR);
+ }
+ }
+
+ /* Cleanup self */
+ redisFree(c);
+}
+
+/* Free the async context. When this function is called from a callback,
+ * control needs to be returned to redisProcessCallbacks() before actual
+ * free'ing. To do so, a flag is set on the context which is picked up by
+ * redisProcessCallbacks(). Otherwise, the context is immediately free'd. */
+void redisAsyncFree(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ c->flags |= REDIS_FREEING;
+ if (!(c->flags & REDIS_IN_CALLBACK))
+ __redisAsyncFree(ac);
+}
+
+/* Helper function to make the disconnect happen and clean up. */
+static void __redisAsyncDisconnect(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+
+ /* Make sure error is accessible if there is any */
+ __redisAsyncCopyError(ac);
+
+ if (ac->err == 0) {
+ /* For clean disconnects, there should be no pending callbacks. */
+ assert(__redisShiftCallback(&ac->replies,NULL) == REDIS_ERR);
+ } else {
+ /* Disconnection is caused by an error, make sure that pending
+ * callbacks cannot call new commands. */
+ c->flags |= REDIS_DISCONNECTING;
+ }
+
+ /* For non-clean disconnects, __redisAsyncFree() will execute pending
+ * callbacks with a NULL-reply. */
+ __redisAsyncFree(ac);
+}
+
+/* Tries to do a clean disconnect from Redis, meaning it stops new commands
+ * from being issued, but tries to flush the output buffer and execute
+ * callbacks for all remaining replies. When this function is called from a
+ * callback, there might be more replies and we can safely defer disconnecting
+ * to redisProcessCallbacks(). Otherwise, we can only disconnect immediately
+ * when there are no pending callbacks. */
+void redisAsyncDisconnect(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ c->flags |= REDIS_DISCONNECTING;
+ if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL)
+ __redisAsyncDisconnect(ac);
+}
+
+static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) {
+ redisContext *c = &(ac->c);
+ dict *callbacks;
+ dictEntry *de;
+ int pvariant;
+ char *stype;
+ sds sname;
+
+ /* Custom reply functions are not supported for pub/sub. This will fail
+ * very hard when they are used... */
+ if (reply->type == REDIS_REPLY_ARRAY) {
+ assert(reply->elements >= 2);
+ assert(reply->element[0]->type == REDIS_REPLY_STRING);
+ stype = reply->element[0]->str;
+ pvariant = (tolower(stype[0]) == 'p') ? 1 : 0;
+
+ if (pvariant)
+ callbacks = ac->sub.patterns;
+ else
+ callbacks = ac->sub.channels;
+
+ /* Locate the right callback */
+ assert(reply->element[1]->type == REDIS_REPLY_STRING);
+ sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len);
+ de = dictFind(callbacks,sname);
+ if (de != NULL) {
+ memcpy(dstcb,dictGetEntryVal(de),sizeof(*dstcb));
+
+ /* If this is an unsubscribe message, remove it. */
+ if (strcasecmp(stype+pvariant,"unsubscribe") == 0) {
+ dictDelete(callbacks,sname);
+
+ /* If this was the last unsubscribe message, revert to
+ * non-subscribe mode. */
+ assert(reply->element[2]->type == REDIS_REPLY_INTEGER);
+ if (reply->element[2]->integer == 0)
+ c->flags &= ~REDIS_SUBSCRIBED;
+ }
+ }
+ sdsfree(sname);
+ } else {
+ /* Shift callback for invalid commands. */
+ __redisShiftCallback(&ac->sub.invalid,dstcb);
+ }
+ return REDIS_OK;
+}
+
+void redisProcessCallbacks(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ redisCallback cb = {NULL, NULL, NULL};
+ void *reply = NULL;
+ int status;
+
+ while((status = redisGetReply(c,&reply)) == REDIS_OK) {
+ if (reply == NULL) {
+ /* When the connection is being disconnected and there are
+ * no more replies, this is the cue to really disconnect. */
+ if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0
+ && ac->replies.head == NULL) {
+ __redisAsyncDisconnect(ac);
+ return;
+ }
+
+ /* If monitor mode, repush callback */
+ if(c->flags & REDIS_MONITORING) {
+ __redisPushCallback(&ac->replies,&cb);
+ }
+
+ /* When the connection is not being disconnected, simply stop
+ * trying to get replies and wait for the next loop tick. */
+ break;
+ }
+
+ /* Even if the context is subscribed, pending regular callbacks will
+ * get a reply before pub/sub messages arrive. */
+ if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) {
+ /*
+ * A spontaneous reply in a not-subscribed context can be the error
+ * reply that is sent when a new connection exceeds the maximum
+ * number of allowed connections on the server side.
+ *
+ * This is seen as an error instead of a regular reply because the
+ * server closes the connection after sending it.
+ *
+ * To prevent the error from being overwritten by an EOF error the
+ * connection is closed here. See issue #43.
+ *
+ * Another possibility is that the server is loading its dataset.
+ * In this case we also want to close the connection, and have the
+ * user wait until the server is ready to take our request.
+ */
+ if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) {
+ c->err = REDIS_ERR_OTHER;
+ snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str);
+ c->reader->fn->freeObject(reply);
+ __redisAsyncDisconnect(ac);
+ return;
+ }
+ /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */
+ assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING));
+ if(c->flags & REDIS_SUBSCRIBED)
+ __redisGetSubscribeCallback(ac,reply,&cb);
+ }
+
+ if (cb.fn != NULL) {
+ __redisRunCallback(ac,&cb,reply);
+ c->reader->fn->freeObject(reply);
+
+ /* Proceed with free'ing when redisAsyncFree() was called. */
+ if (c->flags & REDIS_FREEING) {
+ __redisAsyncFree(ac);
+ return;
+ }
+ } else {
+ /* No callback for this reply. This can either be a NULL callback,
+ * or there were no callbacks to begin with. Either way, don't
+ * abort with an error, but simply ignore it because the client
+ * doesn't know what the server will spit out over the wire. */
+ c->reader->fn->freeObject(reply);
+ /* Proceed with free'ing when redisAsyncFree() was called. */
+ if (c->flags & REDIS_FREEING) {
+ __redisAsyncFree(ac);
+ return;
+ }
+ }
+ }
+
+ /* Disconnect when there was an error reading the reply */
+ if (status != REDIS_OK)
+ __redisAsyncDisconnect(ac);
+}
+
+/* Internal helper function to detect socket status the first time a read or
+ * write event fires. When connecting was not successful, the connect callback
+ * is called with a REDIS_ERR status and the context is free'd. */
+static int __redisAsyncHandleConnect(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+
+ if (redisCheckSocketError(c) == REDIS_ERR) {
+ /* Try again later when connect(2) is still in progress. */
+ if (errno == EINPROGRESS)
+ return REDIS_OK;
+
+ if (ac->onConnect) ac->onConnect(ac,REDIS_ERR);
+ __redisAsyncDisconnect(ac);
+ return REDIS_ERR;
+ }
+
+ /* Mark context as connected. */
+ c->flags |= REDIS_CONNECTED;
+ if (ac->onConnect) ac->onConnect(ac,REDIS_OK);
+ return REDIS_OK;
+}
+
+/* This function should be called when the socket is readable.
+ * It processes all replies that can be read and executes their callbacks.
+ */
+void redisAsyncHandleRead(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+
+ if (!(c->flags & REDIS_CONNECTED)) {
+ /* Abort connect was not successful. */
+ if (__redisAsyncHandleConnect(ac) != REDIS_OK)
+ return;
+ /* Try again later when the context is still not connected. */
+ if (!(c->flags & REDIS_CONNECTED))
+ return;
+ }
+
+ if (redisBufferRead(c) == REDIS_ERR) {
+ __redisAsyncDisconnect(ac);
+ } else {
+ /* Always re-schedule reads */
+ _EL_ADD_READ(ac);
+ redisProcessCallbacks(ac);
+ }
+}
+
+void redisAsyncHandleWrite(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ int done = 0;
+
+ if (!(c->flags & REDIS_CONNECTED)) {
+ /* Abort connect was not successful. */
+ if (__redisAsyncHandleConnect(ac) != REDIS_OK)
+ return;
+ /* Try again later when the context is still not connected. */
+ if (!(c->flags & REDIS_CONNECTED))
+ return;
+ }
+
+ if (redisBufferWrite(c,&done) == REDIS_ERR) {
+ __redisAsyncDisconnect(ac);
+ } else {
+ /* Continue writing when not done, stop writing otherwise */
+ if (!done)
+ _EL_ADD_WRITE(ac);
+ else
+ _EL_DEL_WRITE(ac);
+
+ /* Always schedule reads after writes */
+ _EL_ADD_READ(ac);
+ }
+}
+
+/* Sets a pointer to the first argument and its length starting at p. Returns
+ * the number of bytes to skip to get to the following argument. */
+static const char *nextArgument(const char *start, const char **str, size_t *len) {
+ const char *p = start;
+ if (p[0] != '$') {
+ p = strchr(p,'$');
+ if (p == NULL) return NULL;
+ }
+
+ *len = (int)strtol(p+1,NULL,10);
+ p = strchr(p,'\r');
+ assert(p);
+ *str = p+2;
+ return p+2+(*len)+2;
+}
+
+/* Helper function for the redisAsyncCommand* family of functions. Writes a
+ * formatted command to the output buffer and registers the provided callback
+ * function with the context. */
+static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
+ redisContext *c = &(ac->c);
+ redisCallback cb;
+ int pvariant, hasnext;
+ const char *cstr, *astr;
+ size_t clen, alen;
+ const char *p;
+ sds sname;
+ int ret;
+
+ /* Don't accept new commands when the connection is about to be closed. */
+ if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR;
+
+ /* Setup callback */
+ cb.fn = fn;
+ cb.privdata = privdata;
+
+ /* Find out which command will be appended. */
+ p = nextArgument(cmd,&cstr,&clen);
+ assert(p != NULL);
+ hasnext = (p[0] == '$');
+ pvariant = (tolower(cstr[0]) == 'p') ? 1 : 0;
+ cstr += pvariant;
+ clen -= pvariant;
+
+ if (hasnext && clen >= 9 && strncasecmp(cstr,"subscribe\r\n",9) == 0) {
+ c->flags |= REDIS_SUBSCRIBED;
+
+ /* Add every channel/pattern to the list of subscription callbacks. */
+ while ((p = nextArgument(p,&astr,&alen)) != NULL) {
+ sname = sdsnewlen(astr,alen);
+ if (pvariant)
+ ret = dictReplace(ac->sub.patterns,sname,&cb);
+ else
+ ret = dictReplace(ac->sub.channels,sname,&cb);
+
+ if (ret == 0) sdsfree(sname);
+ }
+ } else if (clen >= 11 && strncasecmp(cstr,"unsubscribe\r\n",11) == 0) {
+ /* It is only useful to call (P)UNSUBSCRIBE when the context is
+ * subscribed to one or more channels or patterns. */
+ if (!(c->flags & REDIS_SUBSCRIBED)) return REDIS_ERR;
+
+ /* (P)UNSUBSCRIBE does not have its own response: every channel or
+ * pattern that is unsubscribed will receive a message. This means we
+ * should not append a callback function for this command. */
+ } else if(clen >= 7 && strncasecmp(cstr,"monitor\r\n",7) == 0) {
+ /* Set monitor flag and push callback */
+ c->flags |= REDIS_MONITORING;
+ __redisPushCallback(&ac->replies,&cb);
+ } else {
+ if (c->flags & REDIS_SUBSCRIBED)
+ /* This will likely result in an error reply, but it needs to be
+ * received and passed to the callback. */
+ __redisPushCallback(&ac->sub.invalid,&cb);
+ else
+ __redisPushCallback(&ac->replies,&cb);
+ }
+
+ __redisAppendCommand(c,cmd,len);
+
+ /* Always schedule a write when the write buffer is non-empty */
+ _EL_ADD_WRITE(ac);
+
+ return REDIS_OK;
+}
+
+int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) {
+ char *cmd;
+ int len;
+ int status;
+ len = redisvFormatCommand(&cmd,format,ap);
+
+ /* We don't want to pass -1 or -2 to future functions as a length. */
+ if (len < 0)
+ return REDIS_ERR;
+
+ status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+ free(cmd);
+ return status;
+}
+
+int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...) {
+ va_list ap;
+ int status;
+ va_start(ap,format);
+ status = redisvAsyncCommand(ac,fn,privdata,format,ap);
+ va_end(ap);
+ return status;
+}
+
+int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) {
+ sds cmd;
+ int len;
+ int status;
+ len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
+ status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+ sdsfree(cmd);
+ return status;
+}
+
+int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
+ int status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+ return status;
+}
diff --git a/contrib/hiredis/async.h b/contrib/hiredis/async.h
new file mode 100644
index 0000000..f19139c
--- /dev/null
+++ b/contrib/hiredis/async.h
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_ASYNC_H
+#define __HIREDIS_ASYNC_H
+#include "hiredis.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct redisAsyncContext; /* need forward declaration of redisAsyncContext */
+struct dict; /* dictionary header is included in async.c */
+
+/* Reply callback prototype and container */
+typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*);
+typedef struct redisCallback {
+ struct redisCallback *next; /* simple singly linked list */
+ redisCallbackFn *fn;
+ void *privdata;
+} redisCallback;
+
+/* List of callbacks for either regular replies or pub/sub */
+typedef struct redisCallbackList {
+ redisCallback *head, *tail;
+} redisCallbackList;
+
+/* Connection callback prototypes */
+typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status);
+typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status);
+
+/* Context for an async connection to Redis */
+typedef struct redisAsyncContext {
+ /* Hold the regular context, so it can be realloc'ed. */
+ redisContext c;
+
+ /* Setup error flags so they can be used directly. */
+ int err;
+ char *errstr;
+
+ /* Not used by hiredis */
+ void *data;
+
+ /* Event library data and hooks */
+ struct {
+ void *data;
+
+ /* Hooks that are called when the library expects to start
+ * reading/writing. These functions should be idempotent. */
+ void (*addRead)(void *privdata);
+ void (*delRead)(void *privdata);
+ void (*addWrite)(void *privdata);
+ void (*delWrite)(void *privdata);
+ void (*cleanup)(void *privdata);
+ } ev;
+
+ /* Called when either the connection is terminated due to an error or per
+ * user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */
+ redisDisconnectCallback *onDisconnect;
+
+ /* Called when the first write event was received. */
+ redisConnectCallback *onConnect;
+
+ /* Regular command callbacks */
+ redisCallbackList replies;
+
+ /* Subscription callbacks */
+ struct {
+ redisCallbackList invalid;
+ struct dict *channels;
+ struct dict *patterns;
+ } sub;
+} redisAsyncContext;
+
+/* Functions that proxy to hiredis */
+redisAsyncContext *redisAsyncConnect(const char *ip, int port);
+redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr);
+redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
+ const char *source_addr);
+redisAsyncContext *redisAsyncConnectUnix(const char *path);
+int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
+void redisAsyncDisconnect(redisAsyncContext *ac);
+void redisAsyncFree(redisAsyncContext *ac);
+
+/* Handle read/write events */
+void redisAsyncHandleRead(redisAsyncContext *ac);
+void redisAsyncHandleWrite(redisAsyncContext *ac);
+
+/* Command functions for an async context. Write the command to the
+ * output buffer and register the provided callback. */
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 0)))
+#endif
+int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 5)))
+#endif
+int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...);
+int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
+int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/hiredis/dict.c b/contrib/hiredis/dict.c
new file mode 100644
index 0000000..0fbc1b4
--- /dev/null
+++ b/contrib/hiredis/dict.c
@@ -0,0 +1,340 @@
+/* Hash table implementation.
+ *
+ * This file implements in memory hash tables with insert/del/replace/find/
+ * get-random-element operations. Hash tables will auto resize if needed
+ * tables of power of two in size are used, collisions are handled by
+ * chaining. See the source code for more information... :)
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <stdlib.h>
+#include <assert.h>
+#include <limits.h>
+#include "dict.h"
+
+/* -------------------------- private prototypes ---------------------------- */
+
+static int _dictExpandIfNeeded(dict *ht);
+static unsigned long _dictNextPower(unsigned long size);
+static int _dictKeyIndex(dict *ht, const void *key);
+static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
+
+/* -------------------------- hash functions -------------------------------- */
+
+/* Generic hash function (a popular one from Bernstein).
+ * I tested a few and this was the best. */
+static unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
+ unsigned int hash = 5381;
+
+ while (len--)
+ hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */
+ return hash;
+}
+
+/* ----------------------------- API implementation ------------------------- */
+
+/* Reset an hashtable already initialized with ht_init().
+ * NOTE: This function should only called by ht_destroy(). */
+static void _dictReset(dict *ht) {
+ ht->table = NULL;
+ ht->size = 0;
+ ht->sizemask = 0;
+ ht->used = 0;
+}
+
+/* Create a new hash table */
+static dict *dictCreate(dictType *type, void *privDataPtr) {
+ dict *ht = malloc(sizeof(*ht));
+ _dictInit(ht,type,privDataPtr);
+ return ht;
+}
+
+/* Initialize the hash table */
+static int _dictInit(dict *ht, dictType *type, void *privDataPtr) {
+ _dictReset(ht);
+ ht->type = type;
+ ht->privdata = privDataPtr;
+ return DICT_OK;
+}
+
+/* Expand or create the hashtable */
+static int dictExpand(dict *ht, unsigned long size) {
+ dict n; /* the new hashtable */
+ unsigned long realsize = _dictNextPower(size), i;
+
+ /* the size is invalid if it is smaller than the number of
+ * elements already inside the hashtable */
+ if (ht->used > size)
+ return DICT_ERR;
+
+ _dictInit(&n, ht->type, ht->privdata);
+ n.size = realsize;
+ n.sizemask = realsize-1;
+ n.table = calloc(realsize,sizeof(dictEntry*));
+
+ /* Copy all the elements from the old to the new table:
+ * note that if the old hash table is empty ht->size is zero,
+ * so dictExpand just creates an hash table. */
+ n.used = ht->used;
+ for (i = 0; i < ht->size && ht->used > 0; i++) {
+ dictEntry *he, *nextHe;
+
+ if (ht->table[i] == NULL) continue;
+
+ /* For each hash entry on this slot... */
+ he = ht->table[i];
+ while(he) {
+ unsigned int h;
+
+ nextHe = he->next;
+ /* Get the new element index */
+ h = dictHashKey(ht, he->key) & n.sizemask;
+ he->next = n.table[h];
+ n.table[h] = he;
+ ht->used--;
+ /* Pass to the next element */
+ he = nextHe;
+ }
+ }
+ assert(ht->used == 0);
+ free(ht->table);
+
+ /* Remap the new hashtable in the old */
+ *ht = n;
+ return DICT_OK;
+}
+
+/* Add an element to the target hash table */
+static int dictAdd(dict *ht, void *key, void *val) {
+ int index;
+ dictEntry *entry;
+
+ /* Get the index of the new element, or -1 if
+ * the element already exists. */
+ if ((index = _dictKeyIndex(ht, key)) == -1)
+ return DICT_ERR;
+
+ /* Allocates the memory and stores key */
+ entry = malloc(sizeof(*entry));
+ entry->next = ht->table[index];
+ ht->table[index] = entry;
+
+ /* Set the hash entry fields. */
+ dictSetHashKey(ht, entry, key);
+ dictSetHashVal(ht, entry, val);
+ ht->used++;
+ return DICT_OK;
+}
+
+/* Add an element, discarding the old if the key already exists.
+ * Return 1 if the key was added from scratch, 0 if there was already an
+ * element with such key and dictReplace() just performed a value update
+ * operation. */
+static int dictReplace(dict *ht, void *key, void *val) {
+ dictEntry *entry, auxentry;
+
+ /* Try to add the element. If the key
+ * does not exists dictAdd will succeed. */
+ if (dictAdd(ht, key, val) == DICT_OK)
+ return 1;
+ /* It already exists, get the entry */
+ entry = dictFind(ht, key);
+ /* Free the old value and set the new one */
+ /* Set the new value and free the old one. Note that it is important
+ * to do that in this order, as the value may just be exactly the same
+ * as the previous one. In this context, think to reference counting,
+ * you want to increment (set), and then decrement (free), and not the
+ * reverse. */
+ if (entry) {
+ auxentry = *entry;
+ dictSetHashVal(ht, entry, val);
+ dictFreeEntryVal(ht, &auxentry);
+ }
+ return 0;
+}
+
+/* Search and remove an element */
+static int dictDelete(dict *ht, const void *key) {
+ unsigned int h;
+ dictEntry *de, *prevde;
+
+ if (ht->size == 0)
+ return DICT_ERR;
+ h = dictHashKey(ht, key) & ht->sizemask;
+ de = ht->table[h];
+
+ prevde = NULL;
+ while(de) {
+ if (dictCompareHashKeys(ht,key,de->key)) {
+ /* Unlink the element from the list */
+ if (prevde)
+ prevde->next = de->next;
+ else
+ ht->table[h] = de->next;
+
+ dictFreeEntryKey(ht,de);
+ dictFreeEntryVal(ht,de);
+ free(de);
+ ht->used--;
+ return DICT_OK;
+ }
+ prevde = de;
+ de = de->next;
+ }
+ return DICT_ERR; /* not found */
+}
+
+/* Destroy an entire hash table */
+static int _dictClear(dict *ht) {
+ unsigned long i;
+
+ /* Free all the elements */
+ for (i = 0; i < ht->size && ht->used > 0; i++) {
+ dictEntry *he, *nextHe;
+
+ if ((he = ht->table[i]) == NULL) continue;
+ while(he) {
+ nextHe = he->next;
+ dictFreeEntryKey(ht, he);
+ dictFreeEntryVal(ht, he);
+ free(he);
+ ht->used--;
+ he = nextHe;
+ }
+ }
+ /* Free the table and the allocated cache structure */
+ free(ht->table);
+ /* Re-initialize the table */
+ _dictReset(ht);
+ return DICT_OK; /* never fails */
+}
+
+/* Clear & Release the hash table */
+static void dictRelease(dict *ht) {
+ _dictClear(ht);
+ free(ht);
+}
+
+static dictEntry *dictFind(dict *ht, const void *key) {
+ dictEntry *he;
+ unsigned int h;
+
+ if (ht->size == 0) return NULL;
+ h = dictHashKey(ht, key) & ht->sizemask;
+ he = ht->table[h];
+ while(he) {
+ if (dictCompareHashKeys(ht, key, he->key))
+ return he;
+ he = he->next;
+ }
+ return NULL;
+}
+
+static dictIterator *dictGetIterator(dict *ht) {
+ dictIterator *iter = malloc(sizeof(*iter));
+
+ iter->ht = ht;
+ iter->index = -1;
+ iter->entry = NULL;
+ iter->nextEntry = NULL;
+ return iter;
+}
+
+static dictEntry *dictNext(dictIterator *iter) {
+ while (1) {
+ if (iter->entry == NULL) {
+ iter->index++;
+ if (iter->index >=
+ (signed)iter->ht->size) break;
+ iter->entry = iter->ht->table[iter->index];
+ } else {
+ iter->entry = iter->nextEntry;
+ }
+ if (iter->entry) {
+ /* We need to save the 'next' here, the iterator user
+ * may delete the entry we are returning. */
+ iter->nextEntry = iter->entry->next;
+ return iter->entry;
+ }
+ }
+ return NULL;
+}
+
+static void dictReleaseIterator(dictIterator *iter) {
+ free(iter);
+}
+
+/* ------------------------- private functions ------------------------------ */
+
+/* Expand the hash table if needed */
+static int _dictExpandIfNeeded(dict *ht) {
+ /* If the hash table is empty expand it to the initial size,
+ * if the table is "full" dobule its size. */
+ if (ht->size == 0)
+ return dictExpand(ht, DICT_HT_INITIAL_SIZE);
+ if (ht->used == ht->size)
+ return dictExpand(ht, ht->size*2);
+ return DICT_OK;
+}
+
+/* Our hash table capability is a power of two */
+static unsigned long _dictNextPower(unsigned long size) {
+ unsigned long i = DICT_HT_INITIAL_SIZE;
+
+ if (size >= LONG_MAX) return LONG_MAX;
+ while(1) {
+ if (i >= size)
+ return i;
+ i *= 2;
+ }
+}
+
+/* Returns the index of a free slot that can be populated with
+ * an hash entry for the given 'key'.
+ * If the key already exists, -1 is returned. */
+static int _dictKeyIndex(dict *ht, const void *key) {
+ unsigned int h;
+ dictEntry *he;
+
+ /* Expand the hashtable if needed */
+ if (_dictExpandIfNeeded(ht) == DICT_ERR)
+ return -1;
+ /* Compute the key hash value */
+ h = dictHashKey(ht, key) & ht->sizemask;
+ /* Search if this slot does not already contain the given key */
+ he = ht->table[h];
+ while(he) {
+ if (dictCompareHashKeys(ht, key, he->key))
+ return -1;
+ he = he->next;
+ }
+ return h;
+}
+
diff --git a/contrib/hiredis/dict.h b/contrib/hiredis/dict.h
new file mode 100644
index 0000000..95fcd28
--- /dev/null
+++ b/contrib/hiredis/dict.h
@@ -0,0 +1,126 @@
+/* Hash table implementation.
+ *
+ * This file implements in memory hash tables with insert/del/replace/find/
+ * get-random-element operations. Hash tables will auto resize if needed
+ * tables of power of two in size are used, collisions are handled by
+ * chaining. See the source code for more information... :)
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __DICT_H
+#define __DICT_H
+
+#define DICT_OK 0
+#define DICT_ERR 1
+
+/* Unused arguments generate annoying warnings... */
+#define DICT_NOTUSED(V) ((void) V)
+
+typedef struct dictEntry {
+ void *key;
+ void *val;
+ struct dictEntry *next;
+} dictEntry;
+
+typedef struct dictType {
+ unsigned int (*hashFunction)(const void *key);
+ void *(*keyDup)(void *privdata, const void *key);
+ void *(*valDup)(void *privdata, const void *obj);
+ int (*keyCompare)(void *privdata, const void *key1, const void *key2);
+ void (*keyDestructor)(void *privdata, void *key);
+ void (*valDestructor)(void *privdata, void *obj);
+} dictType;
+
+typedef struct dict {
+ dictEntry **table;
+ dictType *type;
+ unsigned long size;
+ unsigned long sizemask;
+ unsigned long used;
+ void *privdata;
+} dict;
+
+typedef struct dictIterator {
+ dict *ht;
+ int index;
+ dictEntry *entry, *nextEntry;
+} dictIterator;
+
+/* This is the initial size of every hash table */
+#define DICT_HT_INITIAL_SIZE 4
+
+/* ------------------------------- Macros ------------------------------------*/
+#define dictFreeEntryVal(ht, entry) \
+ if ((ht)->type->valDestructor) \
+ (ht)->type->valDestructor((ht)->privdata, (entry)->val)
+
+#define dictSetHashVal(ht, entry, _val_) do { \
+ if ((ht)->type->valDup) \
+ entry->val = (ht)->type->valDup((ht)->privdata, _val_); \
+ else \
+ entry->val = (_val_); \
+} while(0)
+
+#define dictFreeEntryKey(ht, entry) \
+ if ((ht)->type->keyDestructor) \
+ (ht)->type->keyDestructor((ht)->privdata, (entry)->key)
+
+#define dictSetHashKey(ht, entry, _key_) do { \
+ if ((ht)->type->keyDup) \
+ entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \
+ else \
+ entry->key = (_key_); \
+} while(0)
+
+#define dictCompareHashKeys(ht, key1, key2) \
+ (((ht)->type->keyCompare) ? \
+ (ht)->type->keyCompare((ht)->privdata, key1, key2) : \
+ (key1) == (key2))
+
+#define dictHashKey(ht, key) (ht)->type->hashFunction(key)
+
+#define dictGetEntryKey(he) ((he)->key)
+#define dictGetEntryVal(he) ((he)->val)
+#define dictSlots(ht) ((ht)->size)
+#define dictSize(ht) ((ht)->used)
+
+/* API */
+static unsigned int dictGenHashFunction(const unsigned char *buf, int len);
+static dict *dictCreate(dictType *type, void *privDataPtr);
+static int dictExpand(dict *ht, unsigned long size);
+static int dictAdd(dict *ht, void *key, void *val);
+static int dictReplace(dict *ht, void *key, void *val);
+static int dictDelete(dict *ht, const void *key);
+static void dictRelease(dict *ht);
+static dictEntry * dictFind(dict *ht, const void *key);
+static dictIterator *dictGetIterator(dict *ht);
+static dictEntry *dictNext(dictIterator *iter);
+static void dictReleaseIterator(dictIterator *iter);
+
+#endif /* __DICT_H */
diff --git a/contrib/hiredis/fmacros.h b/contrib/hiredis/fmacros.h
new file mode 100644
index 0000000..19d7b21
--- /dev/null
+++ b/contrib/hiredis/fmacros.h
@@ -0,0 +1,21 @@
+#ifndef __HIREDIS_FMACRO_H
+#define __HIREDIS_FMACRO_H
+
+#if defined(__linux__)
+#define _BSD_SOURCE
+#define _DEFAULT_SOURCE
+#endif
+
+#if defined(__sun__)
+#define _POSIX_C_SOURCE 200112L
+#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__)
+#define _XOPEN_SOURCE 600
+#else
+#define _XOPEN_SOURCE
+#endif
+
+#if __APPLE__ && __MACH__
+#define _OSX
+#endif
+
+#endif
diff --git a/contrib/hiredis/hiredis.c b/contrib/hiredis/hiredis.c
new file mode 100644
index 0000000..0f87bc3
--- /dev/null
+++ b/contrib/hiredis/hiredis.c
@@ -0,0 +1,1025 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+
+#include "hiredis.h"
+#include "net.h"
+#include "sds.h"
+
+static redisReply *createReplyObject(int type);
+static void *createStringObject(const redisReadTask *task, char *str, size_t len);
+static void *createArrayObject(const redisReadTask *task, int elements);
+static void *createIntegerObject(const redisReadTask *task, long long value);
+static void *createNilObject(const redisReadTask *task);
+
+/* Default set of functions to build the reply. Keep in mind that such a
+ * function returning NULL is interpreted as OOM. */
+static redisReplyObjectFunctions defaultFunctions = {
+ createStringObject,
+ createArrayObject,
+ createIntegerObject,
+ createNilObject,
+ freeReplyObject
+};
+
+/* Create a reply object */
+static redisReply *createReplyObject(int type) {
+ redisReply *r = calloc(1,sizeof(*r));
+
+ if (r == NULL)
+ return NULL;
+
+ r->type = type;
+ return r;
+}
+
+/* Free a reply object */
+void freeReplyObject(void *reply) {
+ redisReply *r = reply;
+ size_t j;
+
+ if (r == NULL)
+ return;
+
+ switch(r->type) {
+ case REDIS_REPLY_INTEGER:
+ break; /* Nothing to free */
+ case REDIS_REPLY_ARRAY:
+ if (r->element != NULL) {
+ for (j = 0; j < r->elements; j++)
+ if (r->element[j] != NULL)
+ freeReplyObject(r->element[j]);
+ free(r->element);
+ }
+ break;
+ case REDIS_REPLY_ERROR:
+ case REDIS_REPLY_STATUS:
+ case REDIS_REPLY_STRING:
+ if (r->str != NULL)
+ free(r->str);
+ break;
+ }
+ free(r);
+}
+
+static void *createStringObject(const redisReadTask *task, char *str, size_t len) {
+ redisReply *r, *parent;
+ char *buf;
+
+ r = createReplyObject(task->type);
+ if (r == NULL)
+ return NULL;
+
+ buf = malloc(len+1);
+ if (buf == NULL) {
+ freeReplyObject(r);
+ return NULL;
+ }
+
+ assert(task->type == REDIS_REPLY_ERROR ||
+ task->type == REDIS_REPLY_STATUS ||
+ task->type == REDIS_REPLY_STRING);
+
+ /* Copy string value */
+ memcpy(buf,str,len);
+ buf[len] = '\0';
+ r->str = buf;
+ r->len = len;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+static void *createArrayObject(const redisReadTask *task, int elements) {
+ redisReply *r, *parent;
+
+ r = createReplyObject(REDIS_REPLY_ARRAY);
+ if (r == NULL)
+ return NULL;
+
+ if (elements > 0) {
+ r->element = calloc(elements,sizeof(redisReply*));
+ if (r->element == NULL) {
+ freeReplyObject(r);
+ return NULL;
+ }
+ }
+
+ r->elements = elements;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+static void *createIntegerObject(const redisReadTask *task, long long value) {
+ redisReply *r, *parent;
+
+ r = createReplyObject(REDIS_REPLY_INTEGER);
+ if (r == NULL)
+ return NULL;
+
+ r->integer = value;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+static void *createNilObject(const redisReadTask *task) {
+ redisReply *r, *parent;
+
+ r = createReplyObject(REDIS_REPLY_NIL);
+ if (r == NULL)
+ return NULL;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+/* Return the number of digits of 'v' when converted to string in radix 10.
+ * Implementation borrowed from link in redis/src/util.c:string2ll(). */
+static uint32_t countDigits(uint64_t v) {
+ uint32_t result = 1;
+ for (;;) {
+ if (v < 10) return result;
+ if (v < 100) return result + 1;
+ if (v < 1000) return result + 2;
+ if (v < 10000) return result + 3;
+ v /= 10000U;
+ result += 4;
+ }
+}
+
+/* Helper that calculates the bulk length given a certain string length. */
+static size_t bulklen(size_t len) {
+ return 1+countDigits(len)+2+len+2;
+}
+
+int redisvFormatCommand(char **target, const char *format, va_list ap) {
+ const char *c = format;
+ char *cmd = NULL; /* final command */
+ int pos; /* position in final command */
+ sds curarg, newarg; /* current argument */
+ int touched = 0; /* was the current argument touched? */
+ char **curargv = NULL, **newargv = NULL;
+ int argc = 0;
+ int totlen = 0;
+ int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */
+ int j;
+
+ /* Abort if there is not target to set */
+ if (target == NULL)
+ return -1;
+
+ /* Build the command string accordingly to protocol */
+ curarg = sdsempty();
+ if (curarg == NULL)
+ return -1;
+
+ while(*c != '\0') {
+ if (*c != '%' || c[1] == '\0') {
+ if (*c == ' ') {
+ if (touched) {
+ newargv = realloc(curargv,sizeof(char*)*(argc+1));
+ if (newargv == NULL) goto memory_err;
+ curargv = newargv;
+ curargv[argc++] = curarg;
+ totlen += bulklen(sdslen(curarg));
+
+ /* curarg is put in argv so it can be overwritten. */
+ curarg = sdsempty();
+ if (curarg == NULL) goto memory_err;
+ touched = 0;
+ }
+ } else {
+ newarg = sdscatlen(curarg,c,1);
+ if (newarg == NULL) goto memory_err;
+ curarg = newarg;
+ touched = 1;
+ }
+ } else {
+ char *arg;
+ size_t size;
+
+ /* Set newarg so it can be checked even if it is not touched. */
+ newarg = curarg;
+
+ switch(c[1]) {
+ case 's':
+ arg = va_arg(ap,char*);
+ size = strlen(arg);
+ if (size > 0)
+ newarg = sdscatlen(curarg,arg,size);
+ break;
+ case 'b':
+ arg = va_arg(ap,char*);
+ size = va_arg(ap,size_t);
+ if (size > 0)
+ newarg = sdscatlen(curarg,arg,size);
+ break;
+ case '%':
+ newarg = sdscat(curarg,"%");
+ break;
+ default:
+ /* Try to detect printf format */
+ {
+ static const char intfmts[] = "diouxX";
+ static const char flags[] = "#0-+ ";
+ char _format[16];
+ const char *_p = c+1;
+ size_t _l = 0;
+ va_list _cpy;
+
+ /* Flags */
+ while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++;
+
+ /* Field width */
+ while (*_p != '\0' && isdigit(*_p)) _p++;
+
+ /* Precision */
+ if (*_p == '.') {
+ _p++;
+ while (*_p != '\0' && isdigit(*_p)) _p++;
+ }
+
+ /* Copy va_list before consuming with va_arg */
+ va_copy(_cpy,ap);
+
+ /* Integer conversion (without modifiers) */
+ if (strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,int);
+ goto fmt_valid;
+ }
+
+ /* Double conversion (without modifiers) */
+ if (strchr("eEfFgGaA",*_p) != NULL) {
+ va_arg(ap,double);
+ goto fmt_valid;
+ }
+
+ /* Size: char */
+ if (_p[0] == 'h' && _p[1] == 'h') {
+ _p += 2;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,int); /* char gets promoted to int */
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ /* Size: short */
+ if (_p[0] == 'h') {
+ _p += 1;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,int); /* short gets promoted to int */
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ /* Size: long long */
+ if (_p[0] == 'l' && _p[1] == 'l') {
+ _p += 2;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,long long);
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ /* Size: long */
+ if (_p[0] == 'l') {
+ _p += 1;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,long);
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ fmt_invalid:
+ va_end(_cpy);
+ goto format_err;
+
+ fmt_valid:
+ _l = (_p+1)-c;
+ if (_l < sizeof(_format)-2) {
+ memcpy(_format,c,_l);
+ _format[_l] = '\0';
+ newarg = sdscatvprintf(curarg,_format,_cpy);
+
+ /* Update current position (note: outer blocks
+ * increment c twice so compensate here) */
+ c = _p-1;
+ }
+
+ va_end(_cpy);
+ break;
+ }
+ }
+
+ if (newarg == NULL) goto memory_err;
+ curarg = newarg;
+
+ touched = 1;
+ c++;
+ }
+ c++;
+ }
+
+ /* Add the last argument if needed */
+ if (touched) {
+ newargv = realloc(curargv,sizeof(char*)*(argc+1));
+ if (newargv == NULL) goto memory_err;
+ curargv = newargv;
+ curargv[argc++] = curarg;
+ totlen += bulklen(sdslen(curarg));
+ } else {
+ sdsfree(curarg);
+ }
+
+ /* Clear curarg because it was put in curargv or was free'd. */
+ curarg = NULL;
+
+ /* Add bytes needed to hold multi bulk count */
+ totlen += 1+countDigits(argc)+2;
+
+ /* Build the command at protocol level */
+ cmd = malloc(totlen+1);
+ if (cmd == NULL) goto memory_err;
+
+ pos = sprintf(cmd,"*%d\r\n",argc);
+ for (j = 0; j < argc; j++) {
+ pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j]));
+ memcpy(cmd+pos,curargv[j],sdslen(curargv[j]));
+ pos += sdslen(curargv[j]);
+ sdsfree(curargv[j]);
+ cmd[pos++] = '\r';
+ cmd[pos++] = '\n';
+ }
+ assert(pos == totlen);
+ cmd[pos] = '\0';
+
+ free(curargv);
+ *target = cmd;
+ return totlen;
+
+format_err:
+ error_type = -2;
+ goto cleanup;
+
+memory_err:
+ error_type = -1;
+ goto cleanup;
+
+cleanup:
+ if (curargv) {
+ while(argc--)
+ sdsfree(curargv[argc]);
+ free(curargv);
+ }
+
+ sdsfree(curarg);
+
+ /* No need to check cmd since it is the last statement that can fail,
+ * but do it anyway to be as defensive as possible. */
+ if (cmd != NULL)
+ free(cmd);
+
+ return error_type;
+}
+
+/* Format a command according to the Redis protocol. This function
+ * takes a format similar to printf:
+ *
+ * %s represents a C null terminated string you want to interpolate
+ * %b represents a binary safe string
+ *
+ * When using %b you need to provide both the pointer to the string
+ * and the length in bytes as a size_t. Examples:
+ *
+ * len = redisFormatCommand(target, "GET %s", mykey);
+ * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen);
+ */
+int redisFormatCommand(char **target, const char *format, ...) {
+ va_list ap;
+ int len;
+ va_start(ap,format);
+ len = redisvFormatCommand(target,format,ap);
+ va_end(ap);
+
+ /* The API says "-1" means bad result, but we now also return "-2" in some
+ * cases. Force the return value to always be -1. */
+ if (len < 0)
+ len = -1;
+
+ return len;
+}
+
+/* Format a command according to the Redis protocol using an sds string and
+ * sdscatfmt for the processing of arguments. This function takes the
+ * number of arguments, an array with arguments and an array with their
+ * lengths. If the latter is set to NULL, strlen will be used to compute the
+ * argument lengths.
+ */
+int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv,
+ const size_t *argvlen)
+{
+ sds cmd;
+ unsigned long long totlen;
+ int j;
+ size_t len;
+
+ /* Abort on a NULL target */
+ if (target == NULL)
+ return -1;
+
+ /* Calculate our total size */
+ totlen = 1+countDigits(argc)+2;
+ for (j = 0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ totlen += bulklen(len);
+ }
+
+ /* Use an SDS string for command construction */
+ cmd = sdsempty();
+ if (cmd == NULL)
+ return -1;
+
+ /* We already know how much storage we need */
+ cmd = sdsMakeRoomFor(cmd, totlen);
+ if (cmd == NULL)
+ return -1;
+
+ /* Construct command */
+ cmd = sdscatfmt(cmd, "*%i\r\n", argc);
+ for (j=0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ cmd = sdscatfmt(cmd, "$%T\r\n", len);
+ cmd = sdscatlen(cmd, argv[j], len);
+ cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
+ }
+
+ assert(sdslen(cmd)==totlen);
+
+ *target = cmd;
+ return totlen;
+}
+
+void redisFreeSdsCommand(sds cmd) {
+ sdsfree(cmd);
+}
+
+/* Format a command according to the Redis protocol. This function takes the
+ * number of arguments, an array with arguments and an array with their
+ * lengths. If the latter is set to NULL, strlen will be used to compute the
+ * argument lengths.
+ */
+int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) {
+ char *cmd = NULL; /* final command */
+ int pos; /* position in final command */
+ size_t len;
+ int totlen, j;
+
+ /* Abort on a NULL target */
+ if (target == NULL)
+ return -1;
+
+ /* Calculate number of bytes needed for the command */
+ totlen = 1+countDigits(argc)+2;
+ for (j = 0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ totlen += bulklen(len);
+ }
+
+ /* Build the command at protocol level */
+ cmd = malloc(totlen+1);
+ if (cmd == NULL)
+ return -1;
+
+ pos = sprintf(cmd,"*%d\r\n",argc);
+ for (j = 0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ pos += sprintf(cmd+pos,"$%zu\r\n",len);
+ memcpy(cmd+pos,argv[j],len);
+ pos += len;
+ cmd[pos++] = '\r';
+ cmd[pos++] = '\n';
+ }
+ assert(pos == totlen);
+ cmd[pos] = '\0';
+
+ *target = cmd;
+ return totlen;
+}
+
+void redisFreeCommand(char *cmd) {
+ free(cmd);
+}
+
+void __redisSetError(redisContext *c, int type, const char *str) {
+ size_t len;
+
+ c->err = type;
+ if (str != NULL) {
+ len = strlen(str);
+ len = len < (sizeof(c->errstr)-1) ? len : (sizeof(c->errstr)-1);
+ memcpy(c->errstr,str,len);
+ c->errstr[len] = '\0';
+ } else {
+ /* Only REDIS_ERR_IO may lack a description! */
+ assert(type == REDIS_ERR_IO);
+ __redis_strerror_r(errno, c->errstr, sizeof(c->errstr));
+ }
+}
+
+redisReader *redisReaderCreate(void) {
+ return redisReaderCreateWithFunctions(&defaultFunctions);
+}
+
+static redisContext *redisContextInit(void) {
+ redisContext *c;
+
+ c = calloc(1,sizeof(redisContext));
+ if (c == NULL)
+ return NULL;
+
+ c->err = 0;
+ c->errstr[0] = '\0';
+ c->obuf = sdsempty();
+ c->reader = redisReaderCreate();
+ c->tcp.host = NULL;
+ c->tcp.source_addr = NULL;
+ c->unix_sock.path = NULL;
+ c->timeout = NULL;
+
+ if (c->obuf == NULL || c->reader == NULL) {
+ redisFree(c);
+ return NULL;
+ }
+
+ return c;
+}
+
+void redisFree(redisContext *c) {
+ if (c == NULL)
+ return;
+ if (c->fd > 0)
+ close(c->fd);
+ if (c->obuf != NULL)
+ sdsfree(c->obuf);
+ if (c->reader != NULL)
+ redisReaderFree(c->reader);
+ if (c->tcp.host)
+ free(c->tcp.host);
+ if (c->tcp.source_addr)
+ free(c->tcp.source_addr);
+ if (c->unix_sock.path)
+ free(c->unix_sock.path);
+ if (c->timeout)
+ free(c->timeout);
+ free(c);
+}
+
+int redisFreeKeepFd(redisContext *c) {
+ int fd = c->fd;
+ c->fd = -1;
+ redisFree(c);
+ return fd;
+}
+
+int redisReconnect(redisContext *c) {
+ c->err = 0;
+ memset(c->errstr, '\0', strlen(c->errstr));
+
+ if (c->fd > 0) {
+ close(c->fd);
+ }
+
+ sdsfree(c->obuf);
+ redisReaderFree(c->reader);
+
+ c->obuf = sdsempty();
+ c->reader = redisReaderCreate();
+
+ if (c->connection_type == REDIS_CONN_TCP) {
+ return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port,
+ c->timeout, c->tcp.source_addr);
+ } else if (c->connection_type == REDIS_CONN_UNIX) {
+ return redisContextConnectUnix(c, c->unix_sock.path, c->timeout);
+ } else {
+ /* Something bad happened here and shouldn't have. There isn't
+ enough information in the context to reconnect. */
+ __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect");
+ }
+
+ return REDIS_ERR;
+}
+
+/* Connect to a Redis instance. On error the field error in the returned
+ * context will be set to the return value of the error function.
+ * When no set of reply functions is given, the default set will be used. */
+redisContext *redisConnect(const char *ip, int port) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectTcp(c,ip,port,NULL);
+ return c;
+}
+
+redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectTcp(c,ip,port,&tv);
+ return c;
+}
+
+redisContext *redisConnectNonBlock(const char *ip, int port) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags &= ~REDIS_BLOCK;
+ redisContextConnectTcp(c,ip,port,NULL);
+ return c;
+}
+
+redisContext *redisConnectBindNonBlock(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+ c->flags &= ~REDIS_BLOCK;
+ redisContextConnectBindTcp(c,ip,port,NULL,source_addr);
+ return c;
+}
+
+redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+ c->flags &= ~REDIS_BLOCK;
+ c->flags |= REDIS_REUSEADDR;
+ redisContextConnectBindTcp(c,ip,port,NULL,source_addr);
+ return c;
+}
+
+redisContext *redisConnectUnix(const char *path) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectUnix(c,path,NULL);
+ return c;
+}
+
+redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectUnix(c,path,&tv);
+ return c;
+}
+
+redisContext *redisConnectUnixNonBlock(const char *path) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags &= ~REDIS_BLOCK;
+ redisContextConnectUnix(c,path,NULL);
+ return c;
+}
+
+redisContext *redisConnectFd(int fd) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->fd = fd;
+ c->flags |= REDIS_BLOCK | REDIS_CONNECTED;
+ return c;
+}
+
+/* Set read/write timeout on a blocking socket. */
+int redisSetTimeout(redisContext *c, const struct timeval tv) {
+ if (c->flags & REDIS_BLOCK)
+ return redisContextSetTimeout(c,tv);
+ return REDIS_ERR;
+}
+
+/* Enable connection KeepAlive. */
+int redisEnableKeepAlive(redisContext *c) {
+ if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK)
+ return REDIS_ERR;
+ return REDIS_OK;
+}
+
+/* Use this function to handle a read event on the descriptor. It will try
+ * and read some bytes from the socket and feed them to the reply parser.
+ *
+ * After this function is called, you may use redisContextReadReply to
+ * see if there is a reply available. */
+int redisBufferRead(redisContext *c) {
+ char buf[1024*16];
+ int nread;
+
+ /* Return early when the context has seen an error. */
+ if (c->err)
+ return REDIS_ERR;
+
+ nread = read(c->fd,buf,sizeof(buf));
+ if (nread == -1) {
+ if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
+ /* Try again later */
+ } else {
+ __redisSetError(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+ } else if (nread == 0) {
+ __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection");
+ return REDIS_ERR;
+ } else {
+ if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) {
+ __redisSetError(c,c->reader->err,c->reader->errstr);
+ return REDIS_ERR;
+ }
+ }
+ return REDIS_OK;
+}
+
+/* Write the output buffer to the socket.
+ *
+ * Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was
+ * successfully written to the socket. When the buffer is empty after the
+ * write operation, "done" is set to 1 (if given).
+ *
+ * Returns REDIS_ERR if an error occurred trying to write and sets
+ * c->errstr to hold the appropriate error string.
+ */
+int redisBufferWrite(redisContext *c, int *done) {
+ int nwritten;
+
+ /* Return early when the context has seen an error. */
+ if (c->err)
+ return REDIS_ERR;
+
+ if (sdslen(c->obuf) > 0) {
+ nwritten = write(c->fd,c->obuf,sdslen(c->obuf));
+ if (nwritten == -1) {
+ if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
+ /* Try again later */
+ } else {
+ __redisSetError(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+ } else if (nwritten > 0) {
+ if (nwritten == (signed)sdslen(c->obuf)) {
+ sdsfree(c->obuf);
+ c->obuf = sdsempty();
+ } else {
+ sdsrange(c->obuf,nwritten,-1);
+ }
+ }
+ }
+ if (done != NULL) *done = (sdslen(c->obuf) == 0);
+ return REDIS_OK;
+}
+
+/* Internal helper function to try and get a reply from the reader,
+ * or set an error in the context otherwise. */
+int redisGetReplyFromReader(redisContext *c, void **reply) {
+ if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) {
+ __redisSetError(c,c->reader->err,c->reader->errstr);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+int redisGetReply(redisContext *c, void **reply) {
+ int wdone = 0;
+ void *aux = NULL;
+
+ /* Try to read pending replies */
+ if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
+ return REDIS_ERR;
+
+ /* For the blocking context, flush output buffer and read reply */
+ if (aux == NULL && c->flags & REDIS_BLOCK) {
+ /* Write until done */
+ do {
+ if (redisBufferWrite(c,&wdone) == REDIS_ERR)
+ return REDIS_ERR;
+ } while (!wdone);
+
+ /* Read until there is a reply */
+ do {
+ if (redisBufferRead(c) == REDIS_ERR)
+ return REDIS_ERR;
+ if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
+ return REDIS_ERR;
+ } while (aux == NULL);
+ }
+
+ /* Set reply object */
+ if (reply != NULL) *reply = aux;
+ return REDIS_OK;
+}
+
+
+/* Helper function for the redisAppendCommand* family of functions.
+ *
+ * Write a formatted command to the output buffer. When this family
+ * is used, you need to call redisGetReply yourself to retrieve
+ * the reply (or replies in pub/sub).
+ */
+int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) {
+ sds newbuf;
+
+ newbuf = sdscatlen(c->obuf,cmd,len);
+ if (newbuf == NULL) {
+ __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+ return REDIS_ERR;
+ }
+
+ c->obuf = newbuf;
+ return REDIS_OK;
+}
+
+int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) {
+
+ if (__redisAppendCommand(c, cmd, len) != REDIS_OK) {
+ return REDIS_ERR;
+ }
+
+ return REDIS_OK;
+}
+
+int redisvAppendCommand(redisContext *c, const char *format, va_list ap) {
+ char *cmd;
+ int len;
+
+ len = redisvFormatCommand(&cmd,format,ap);
+ if (len == -1) {
+ __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+ return REDIS_ERR;
+ } else if (len == -2) {
+ __redisSetError(c,REDIS_ERR_OTHER,"Invalid format string");
+ return REDIS_ERR;
+ }
+
+ if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
+ free(cmd);
+ return REDIS_ERR;
+ }
+
+ free(cmd);
+ return REDIS_OK;
+}
+
+int redisAppendCommand(redisContext *c, const char *format, ...) {
+ va_list ap;
+ int ret;
+
+ va_start(ap,format);
+ ret = redisvAppendCommand(c,format,ap);
+ va_end(ap);
+ return ret;
+}
+
+int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
+ sds cmd;
+ int len;
+
+ len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
+ if (len == -1) {
+ __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+ return REDIS_ERR;
+ }
+
+ if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
+ sdsfree(cmd);
+ return REDIS_ERR;
+ }
+
+ sdsfree(cmd);
+ return REDIS_OK;
+}
+
+/* Helper function for the redisCommand* family of functions.
+ *
+ * Write a formatted command to the output buffer. If the given context is
+ * blocking, immediately read the reply into the "reply" pointer. When the
+ * context is non-blocking, the "reply" pointer will not be used and the
+ * command is simply appended to the write buffer.
+ *
+ * Returns the reply when a reply was successfully retrieved. Returns NULL
+ * otherwise. When NULL is returned in a blocking context, the error field
+ * in the context will be set.
+ */
+static void *__redisBlockForReply(redisContext *c) {
+ void *reply;
+
+ if (c->flags & REDIS_BLOCK) {
+ if (redisGetReply(c,&reply) != REDIS_OK)
+ return NULL;
+ return reply;
+ }
+ return NULL;
+}
+
+void *redisvCommand(redisContext *c, const char *format, va_list ap) {
+ if (redisvAppendCommand(c,format,ap) != REDIS_OK)
+ return NULL;
+ return __redisBlockForReply(c);
+}
+
+void *redisCommand(redisContext *c, const char *format, ...) {
+ va_list ap;
+ void *reply = NULL;
+ va_start(ap,format);
+ reply = redisvCommand(c,format,ap);
+ va_end(ap);
+ return reply;
+}
+
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
+ if (redisAppendCommandArgv(c,argc,argv,argvlen) != REDIS_OK)
+ return NULL;
+ return __redisBlockForReply(c);
+}
diff --git a/contrib/hiredis/hiredis.h b/contrib/hiredis/hiredis.h
new file mode 100644
index 0000000..6b531b9
--- /dev/null
+++ b/contrib/hiredis/hiredis.h
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_H
+#define __HIREDIS_H
+#include "read.h"
+#include <stdarg.h> /* for va_list */
+#include <sys/time.h> /* for struct timeval */
+#include <stdint.h> /* uintXX_t, etc */
+#include <string.h> /* strerror_r, etc */
+#include "sds.h" /* for sds */
+
+#define HIREDIS_MAJOR 0
+#define HIREDIS_MINOR 13
+#define HIREDIS_PATCH 3
+#define HIREDIS_SONAME 0.13
+
+/* Connection type can be blocking or non-blocking and is set in the
+ * least significant bit of the flags field in redisContext. */
+#define REDIS_BLOCK 0x1
+
+/* Connection may be disconnected before being free'd. The second bit
+ * in the flags field is set when the context is connected. */
+#define REDIS_CONNECTED 0x2
+
+/* The async API might try to disconnect cleanly and flush the output
+ * buffer and read all subsequent replies before disconnecting.
+ * This flag means no new commands can come in and the connection
+ * should be terminated once all replies have been read. */
+#define REDIS_DISCONNECTING 0x4
+
+/* Flag specific to the async API which means that the context should be clean
+ * up as soon as possible. */
+#define REDIS_FREEING 0x8
+
+/* Flag that is set when an async callback is executed. */
+#define REDIS_IN_CALLBACK 0x10
+
+/* Flag that is set when the async context has one or more subscriptions. */
+#define REDIS_SUBSCRIBED 0x20
+
+/* Flag that is set when monitor mode is active */
+#define REDIS_MONITORING 0x40
+
+/* Flag that is set when we should set SO_REUSEADDR before calling bind() */
+#define REDIS_REUSEADDR 0x80
+
+#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */
+
+/* number of times we retry to connect in the case of EADDRNOTAVAIL and
+ * SO_REUSEADDR is being used. */
+#define REDIS_CONNECT_RETRIES 10
+
+/* strerror_r has two completely different prototypes and behaviors
+ * depending on system issues, so we need to operate on the error buffer
+ * differently depending on which strerror_r we're using. */
+#ifndef _GNU_SOURCE
+/* "regular" POSIX strerror_r that does the right thing. */
+#define __redis_strerror_r(errno, buf, len) \
+ do { \
+ strerror_r((errno), (buf), (len)); \
+ } while (0)
+#else
+/* "bad" GNU strerror_r we need to clean up after. */
+#define __redis_strerror_r(errno, buf, len) \
+ do { \
+ char *err_str = strerror((errno)); \
+ /* If return value _isn't_ the start of the buffer we passed in, \
+ * then GNU strerror_r returned an internal static buffer and we \
+ * need to copy the result into our private buffer. */ \
+ if (err_str != (buf)) { \
+ buf[(len)-1] = '\0'; \
+ strncat((buf), err_str, ((len) - 1)); \
+ } \
+ } while (0)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* This is the reply object returned by redisCommand() */
+typedef struct redisReply {
+ int type; /* REDIS_REPLY_* */
+ long long integer; /* The integer when type is REDIS_REPLY_INTEGER */
+ int len; /* Length of string */
+ char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */
+ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */
+ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */
+} redisReply;
+
+redisReader *redisReaderCreate(void);
+
+/* Function to free the reply objects hiredis returns by default. */
+void freeReplyObject(void *reply);
+
+/* Functions to format a command according to the protocol. */
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+int redisvFormatCommand(char **target, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 3)))
+#endif
+int redisFormatCommand(char **target, const char *format, ...);
+int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
+int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen);
+void redisFreeCommand(char *cmd);
+void redisFreeSdsCommand(sds cmd);
+
+enum redisConnectionType {
+ REDIS_CONN_TCP,
+ REDIS_CONN_UNIX,
+};
+
+/* Context for a connection to Redis */
+typedef struct redisContext {
+ int err; /* Error flags, 0 when there is no error */
+ char errstr[128]; /* String representation of error when applicable */
+ int fd;
+ int flags;
+ char *obuf; /* Write buffer */
+ redisReader *reader; /* Protocol reader */
+
+ enum redisConnectionType connection_type;
+ struct timeval *timeout;
+
+ struct {
+ char *host;
+ char *source_addr;
+ int port;
+ } tcp;
+
+ struct {
+ char *path;
+ } unix_sock;
+
+} redisContext;
+
+redisContext *redisConnect(const char *ip, int port);
+redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv);
+redisContext *redisConnectNonBlock(const char *ip, int port);
+redisContext *redisConnectBindNonBlock(const char *ip, int port,
+ const char *source_addr);
+redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
+ const char *source_addr);
+redisContext *redisConnectUnix(const char *path);
+redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv);
+redisContext *redisConnectUnixNonBlock(const char *path);
+redisContext *redisConnectFd(int fd);
+
+/**
+ * Reconnect the given context using the saved information.
+ *
+ * This re-uses the exact same connect options as in the initial connection.
+ * host, ip (or path), timeout and bind address are reused,
+ * flags are used unmodified from the existing context.
+ *
+ * Returns REDIS_OK on successful connect or REDIS_ERR otherwise.
+ */
+int redisReconnect(redisContext *c);
+
+int redisSetTimeout(redisContext *c, const struct timeval tv);
+int redisEnableKeepAlive(redisContext *c);
+void redisFree(redisContext *c);
+int redisFreeKeepFd(redisContext *c);
+int redisBufferRead(redisContext *c);
+int redisBufferWrite(redisContext *c, int *done);
+
+/* In a blocking context, this function first checks if there are unconsumed
+ * replies to return and returns one if so. Otherwise, it flushes the output
+ * buffer to the socket and reads until it has a reply. In a non-blocking
+ * context, it will return unconsumed replies until there are no more. */
+int redisGetReply(redisContext *c, void **reply);
+int redisGetReplyFromReader(redisContext *c, void **reply);
+
+/* Write a formatted command to the output buffer. Use these functions in blocking mode
+ * to get a pipeline of commands. */
+int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len);
+
+/* Write a command to the output buffer. Use these functions in blocking mode
+ * to get a pipeline of commands. */
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+int redisvAppendCommand(redisContext *c, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 3)))
+#endif
+int redisAppendCommand(redisContext *c, const char *format, ...);
+int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+
+/* Issue a command to Redis. In a blocking context, it is identical to calling
+ * redisAppendCommand, followed by redisGetReply. The function will return
+ * NULL if there was an error in performing the request, otherwise it will
+ * return the reply. In a non-blocking context, it is identical to calling
+ * only redisAppendCommand and will always return NULL. */
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+void *redisvCommand(redisContext *c, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 3)))
+#endif
+void *redisCommand(redisContext *c, const char *format, ...);
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/hiredis/net.c b/contrib/hiredis/net.c
new file mode 100644
index 0000000..97fd42c
--- /dev/null
+++ b/contrib/hiredis/net.c
@@ -0,0 +1,464 @@
+/* Extracted from anet.c to work properly with Hiredis error reporting.
+ *
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/select.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <netdb.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <poll.h>
+#include <limits.h>
+#include <stdlib.h>
+
+#include "net.h"
+#include "sds.h"
+
+/* Defined in hiredis.c */
+void __redisSetError(redisContext *c, int type, const char *str);
+
+static void redisContextCloseFd(redisContext *c) {
+ if (c && c->fd >= 0) {
+ close(c->fd);
+ c->fd = -1;
+ }
+}
+
+static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) {
+ char buf[128] = { 0 };
+ char *p;
+ size_t len = 0;
+
+ if (prefix != NULL)
+ len = snprintf(buf,sizeof(buf),"%s: ",prefix);
+ p = buf + len;
+ __redis_strerror_r(errno, p, sizeof(buf) - len);
+ __redisSetError(c,type,buf);
+}
+
+static int redisSetReuseAddr(redisContext *c) {
+ int on = 1;
+ if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+static int redisCreateSocket(redisContext *c, int type) {
+ int s;
+ if ((s = socket(type, SOCK_STREAM, 0)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+ c->fd = s;
+ if (type == AF_INET) {
+ if (redisSetReuseAddr(c) == REDIS_ERR) {
+ return REDIS_ERR;
+ }
+ }
+ return REDIS_OK;
+}
+
+static int redisSetBlocking(redisContext *c, int blocking) {
+ int flags;
+
+ /* Set the socket nonblocking.
+ * Note that fcntl(2) for F_GETFL and F_SETFL can't be
+ * interrupted by a signal. */
+ if ((flags = fcntl(c->fd, F_GETFL)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+
+ if (blocking)
+ flags &= ~O_NONBLOCK;
+ else
+ flags |= O_NONBLOCK;
+
+ if (fcntl(c->fd, F_SETFL, flags) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+int redisKeepAlive(redisContext *c, int interval) {
+ int val = 1;
+ int fd = c->fd;
+
+ if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+
+ val = interval;
+
+#ifdef _OSX
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+#else
+#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__)
+ val = interval;
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+
+ val = interval/3;
+ if (val == 0) val = 1;
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+
+ val = 3;
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+#endif
+#endif
+
+ return REDIS_OK;
+}
+
+static int redisSetTcpNoDelay(redisContext *c) {
+ int yes = 1;
+ if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+#define __MAX_MSEC (((LONG_MAX) - 999) / 1000)
+
+static int redisContextWaitReady(redisContext *c, const struct timeval *timeout) {
+ struct pollfd wfd[1];
+ long msec;
+
+ msec = -1;
+ wfd[0].fd = c->fd;
+ wfd[0].events = POLLOUT;
+
+ /* Only use timeout when not NULL. */
+ if (timeout != NULL) {
+ if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) {
+ __redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+
+ msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000);
+
+ if (msec < 0 || msec > INT_MAX) {
+ msec = INT_MAX;
+ }
+ }
+
+ if (errno == EINPROGRESS) {
+ int res;
+
+ if ((res = poll(wfd, 1, msec)) == -1) {
+ __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ } else if (res == 0) {
+ errno = ETIMEDOUT;
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+
+ if (redisCheckSocketError(c) != REDIS_OK)
+ return REDIS_ERR;
+
+ return REDIS_OK;
+ }
+
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+}
+
+int redisCheckSocketError(redisContext *c) {
+ int err = 0;
+ socklen_t errlen = sizeof(err);
+
+ if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)");
+ return REDIS_ERR;
+ }
+
+ if (err) {
+ errno = err;
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+
+ return REDIS_OK;
+}
+
+int redisContextSetTimeout(redisContext *c, const struct timeval tv) {
+ if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)");
+ return REDIS_ERR;
+ }
+ if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)");
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout,
+ const char *source_addr) {
+ int s, rv, n;
+ char _port[6]; /* strlen("65535"); */
+ struct addrinfo hints, *servinfo, *bservinfo, *p, *b;
+ int blocking = (c->flags & REDIS_BLOCK);
+ int reuseaddr = (c->flags & REDIS_REUSEADDR);
+ int reuses = 0;
+
+ c->connection_type = REDIS_CONN_TCP;
+ c->tcp.port = port;
+
+ /* We need to take possession of the passed parameters
+ * to make them reusable for a reconnect.
+ * We also carefully check we don't free data we already own,
+ * as in the case of the reconnect method.
+ *
+ * This is a bit ugly, but atleast it works and doesn't leak memory.
+ **/
+ if (c->tcp.host != addr) {
+ if (c->tcp.host)
+ free(c->tcp.host);
+
+ c->tcp.host = strdup(addr);
+ }
+
+ if (timeout) {
+ if (c->timeout != timeout) {
+ if (c->timeout == NULL)
+ c->timeout = malloc(sizeof(struct timeval));
+
+ memcpy(c->timeout, timeout, sizeof(struct timeval));
+ }
+ } else {
+ if (c->timeout)
+ free(c->timeout);
+ c->timeout = NULL;
+ }
+
+ if (source_addr == NULL) {
+ free(c->tcp.source_addr);
+ c->tcp.source_addr = NULL;
+ } else if (c->tcp.source_addr != source_addr) {
+ free(c->tcp.source_addr);
+ c->tcp.source_addr = strdup(source_addr);
+ }
+
+ snprintf(_port, 6, "%d", port);
+ memset(&hints,0,sizeof(hints));
+ hints.ai_family = AF_INET;
+ hints.ai_socktype = SOCK_STREAM;
+ if (!blocking) {
+ /* Rspamd specific: never try to resolve on non-blocking conn requests */
+ hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV;
+ }
+
+ /* Try with IPv6 if no IPv4 address was found. We do it in this order since
+ * in a Redis client you can't afford to test if you have IPv6 connectivity
+ * as this would add latency to every connect. Otherwise a more sensible
+ * route could be: Use IPv6 if both addresses are available and there is IPv6
+ * connectivity. */
+ if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) {
+ hints.ai_family = AF_INET6;
+ if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv));
+ return REDIS_ERR;
+ }
+ }
+ for (p = servinfo; p != NULL; p = p->ai_next) {
+addrretry:
+ if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1)
+ continue;
+
+ c->fd = s;
+ if (redisSetBlocking(c,0) != REDIS_OK)
+ goto error;
+ if (c->tcp.source_addr) {
+ int bound = 0;
+ /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */
+ if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) {
+ char buf[128];
+ snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv));
+ __redisSetError(c,REDIS_ERR_OTHER,buf);
+ goto error;
+ }
+
+ if (reuseaddr) {
+ n = 1;
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n,
+ sizeof(n)) < 0) {
+ goto error;
+ }
+ }
+
+ for (b = bservinfo; b != NULL; b = b->ai_next) {
+ if (bind(s,b->ai_addr,b->ai_addrlen) != -1) {
+ bound = 1;
+ break;
+ }
+ }
+ freeaddrinfo(bservinfo);
+ if (!bound) {
+ char buf[128];
+ snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno));
+ __redisSetError(c,REDIS_ERR_OTHER,buf);
+ goto error;
+ }
+ }
+ if (redisSetTcpNoDelay(c) != REDIS_OK)
+ goto error;
+ if (connect(s,p->ai_addr,p->ai_addrlen) == -1) {
+ if (errno == EHOSTUNREACH) {
+ redisContextCloseFd(c);
+ continue;
+ } else if (errno == EINPROGRESS && !blocking) {
+ /* This is ok. */
+ } else if (errno == EADDRNOTAVAIL && reuseaddr) {
+ if (++reuses >= REDIS_CONNECT_RETRIES) {
+ goto error;
+ } else {
+ goto addrretry;
+ }
+ } else {
+ if (redisContextWaitReady(c,c->timeout) != REDIS_OK)
+ goto error;
+ }
+ }
+
+ c->flags |= REDIS_CONNECTED;
+ rv = REDIS_OK;
+ goto end;
+ }
+ if (blocking && redisSetBlocking(c,1) != REDIS_OK)
+ goto error;
+ if (p == NULL) {
+ char buf[128];
+ snprintf(buf,sizeof(buf),"Can't create socket: %s",strerror(errno));
+ __redisSetError(c,REDIS_ERR_OTHER,buf);
+ goto error;
+ }
+
+error:
+ rv = REDIS_ERR;
+end:
+ freeaddrinfo(servinfo);
+ return rv; // Need to return REDIS_OK if alright
+}
+
+int redisContextConnectTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout) {
+ return _redisContextConnectTcp(c, addr, port, timeout, NULL);
+}
+
+int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout,
+ const char *source_addr) {
+ return _redisContextConnectTcp(c, addr, port, timeout, source_addr);
+}
+
+int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) {
+ int blocking = (c->flags & REDIS_BLOCK);
+ struct sockaddr_un sa;
+
+ if (redisCreateSocket(c,AF_LOCAL) < 0)
+ return REDIS_ERR;
+ if (redisSetBlocking(c,0) != REDIS_OK)
+ return REDIS_ERR;
+
+ c->connection_type = REDIS_CONN_UNIX;
+ if (c->unix_sock.path != path)
+ c->unix_sock.path = strdup(path);
+
+ if (timeout) {
+ if (c->timeout != timeout) {
+ if (c->timeout == NULL)
+ c->timeout = malloc(sizeof(struct timeval));
+
+ memcpy(c->timeout, timeout, sizeof(struct timeval));
+ }
+ } else {
+ if (c->timeout)
+ free(c->timeout);
+ c->timeout = NULL;
+ }
+
+ sa.sun_family = AF_LOCAL;
+ strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1);
+ if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) {
+ if (errno == EINPROGRESS && !blocking) {
+ /* This is ok. */
+ } else {
+ if (redisContextWaitReady(c,c->timeout) != REDIS_OK)
+ return REDIS_ERR;
+ }
+ }
+
+ /* Reset socket to be blocking after connect(2). */
+ if (blocking && redisSetBlocking(c,1) != REDIS_OK)
+ return REDIS_ERR;
+
+ c->flags |= REDIS_CONNECTED;
+ return REDIS_OK;
+}
diff --git a/contrib/hiredis/net.h b/contrib/hiredis/net.h
new file mode 100644
index 0000000..2f1a0bf
--- /dev/null
+++ b/contrib/hiredis/net.h
@@ -0,0 +1,53 @@
+/* Extracted from anet.c to work properly with Hiredis error reporting.
+ *
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NET_H
+#define __NET_H
+
+#include "hiredis.h"
+
+#if defined(__sun)
+#define AF_LOCAL AF_UNIX
+#endif
+
+int redisCheckSocketError(redisContext *c);
+int redisContextSetTimeout(redisContext *c, const struct timeval tv);
+int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout);
+int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout,
+ const char *source_addr);
+int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout);
+int redisKeepAlive(redisContext *c, int interval);
+
+#endif
diff --git a/contrib/hiredis/read.c b/contrib/hiredis/read.c
new file mode 100644
index 0000000..df1a467
--- /dev/null
+++ b/contrib/hiredis/read.c
@@ -0,0 +1,525 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "fmacros.h"
+#include <string.h>
+#include <stdlib.h>
+#ifndef _MSC_VER
+#include <unistd.h>
+#endif
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+
+#include "read.h"
+#include "sds.h"
+
+static void __redisReaderSetError(redisReader *r, int type, const char *str) {
+ size_t len;
+
+ if (r->reply != NULL && r->fn && r->fn->freeObject) {
+ r->fn->freeObject(r->reply);
+ r->reply = NULL;
+ }
+
+ /* Clear input buffer on errors. */
+ if (r->buf != NULL) {
+ sdsfree(r->buf);
+ r->buf = NULL;
+ r->pos = r->len = 0;
+ }
+
+ /* Reset task stack. */
+ r->ridx = -1;
+
+ /* Set error. */
+ r->err = type;
+ len = strlen(str);
+ len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
+ memcpy(r->errstr,str,len);
+ r->errstr[len] = '\0';
+}
+
+static size_t chrtos(char *buf, size_t size, char byte) {
+ size_t len = 0;
+
+ switch(byte) {
+ case '\\':
+ case '"':
+ len = snprintf(buf,size,"\"\\%c\"",byte);
+ break;
+ case '\n': len = snprintf(buf,size,"\"\\n\""); break;
+ case '\r': len = snprintf(buf,size,"\"\\r\""); break;
+ case '\t': len = snprintf(buf,size,"\"\\t\""); break;
+ case '\a': len = snprintf(buf,size,"\"\\a\""); break;
+ case '\b': len = snprintf(buf,size,"\"\\b\""); break;
+ default:
+ if (isprint(byte))
+ len = snprintf(buf,size,"\"%c\"",byte);
+ else
+ len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
+ break;
+ }
+
+ return len;
+}
+
+static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
+ char cbuf[8], sbuf[128];
+
+ chrtos(cbuf,sizeof(cbuf),byte);
+ snprintf(sbuf,sizeof(sbuf),
+ "Protocol error, got %s as reply type byte", cbuf);
+ __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
+}
+
+static void __redisReaderSetErrorOOM(redisReader *r) {
+ __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
+}
+
+static char *readBytes(redisReader *r, unsigned int bytes) {
+ char *p;
+ if (r->len-r->pos >= bytes) {
+ p = r->buf+r->pos;
+ r->pos += bytes;
+ return p;
+ }
+ return NULL;
+}
+
+/* Find pointer to \r\n. */
+static char *seekNewline(char *s, size_t len) {
+ int pos = 0;
+ int _len = len-1;
+
+ /* Position should be < len-1 because the character at "pos" should be
+ * followed by a \n. Note that strchr cannot be used because it doesn't
+ * allow to search a limited length and the buffer that is being searched
+ * might not have a trailing NULL character. */
+ while (pos < _len) {
+ while(pos < _len && s[pos] != '\r') pos++;
+ if (s[pos] != '\r') {
+ /* Not found. */
+ return NULL;
+ } else {
+ if (s[pos+1] == '\n') {
+ /* Found. */
+ return s+pos;
+ } else {
+ /* Continue searching. */
+ pos++;
+ }
+ }
+ }
+ return NULL;
+}
+
+/* Read a long long value starting at *s, under the assumption that it will be
+ * terminated by \r\n. Ambiguously returns -1 for unexpected input. */
+static long long readLongLong(char *s) {
+ long long v = 0;
+ int dec, mult = 1;
+ char c;
+
+ if (*s == '-') {
+ mult = -1;
+ s++;
+ } else if (*s == '+') {
+ mult = 1;
+ s++;
+ }
+
+ while ((c = *(s++)) != '\r') {
+ dec = c - '0';
+ if (dec >= 0 && dec < 10) {
+ v *= 10;
+ v += dec;
+ } else {
+ /* Should not happen... */
+ return -1;
+ }
+ }
+
+ return mult*v;
+}
+
+static char *readLine(redisReader *r, int *_len) {
+ char *p, *s;
+ int len;
+
+ p = r->buf+r->pos;
+ s = seekNewline(p,(r->len-r->pos));
+ if (s != NULL) {
+ len = s-(r->buf+r->pos);
+ r->pos += len+2; /* skip \r\n */
+ if (_len) *_len = len;
+ return p;
+ }
+ return NULL;
+}
+
+static void moveToNextTask(redisReader *r) {
+ redisReadTask *cur, *prv;
+ while (r->ridx >= 0) {
+ /* Return a.s.a.p. when the stack is now empty. */
+ if (r->ridx == 0) {
+ r->ridx--;
+ return;
+ }
+
+ cur = &(r->rstack[r->ridx]);
+ prv = &(r->rstack[r->ridx-1]);
+ assert(prv->type == REDIS_REPLY_ARRAY);
+ if (cur->idx == prv->elements-1) {
+ r->ridx--;
+ } else {
+ /* Reset the type because the next item can be anything */
+ assert(cur->idx < prv->elements);
+ cur->type = -1;
+ cur->elements = -1;
+ cur->idx++;
+ return;
+ }
+ }
+}
+
+static int processLineItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ void *obj;
+ char *p;
+ int len;
+
+ if ((p = readLine(r,&len)) != NULL) {
+ if (cur->type == REDIS_REPLY_INTEGER) {
+ if (r->fn && r->fn->createInteger)
+ obj = r->fn->createInteger(cur,readLongLong(p));
+ else
+ obj = (void*)REDIS_REPLY_INTEGER;
+ } else {
+ /* Type will be error or status. */
+ if (r->fn && r->fn->createString)
+ obj = r->fn->createString(cur,p,len);
+ else
+ obj = (void*)(size_t)(cur->type);
+ }
+
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ /* Set reply if this is the root object. */
+ if (r->ridx == 0) r->reply = obj;
+ moveToNextTask(r);
+ return REDIS_OK;
+ }
+
+ return REDIS_ERR;
+}
+
+static int processBulkItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ void *obj = NULL;
+ char *p, *s;
+ long len;
+ unsigned long bytelen;
+ int success = 0;
+
+ p = r->buf+r->pos;
+ s = seekNewline(p,r->len-r->pos);
+ if (s != NULL) {
+ p = r->buf+r->pos;
+ bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
+ len = readLongLong(p);
+
+ if (len < 0) {
+ /* The nil object can always be created. */
+ if (r->fn && r->fn->createNil)
+ obj = r->fn->createNil(cur);
+ else
+ obj = (void*)REDIS_REPLY_NIL;
+ success = 1;
+ } else {
+ /* Only continue when the buffer contains the entire bulk item. */
+ bytelen += len+2; /* include \r\n */
+ if (r->pos+bytelen <= r->len) {
+ if (r->fn && r->fn->createString)
+ obj = r->fn->createString(cur,s+2,len);
+ else
+ obj = (void*)REDIS_REPLY_STRING;
+ success = 1;
+ }
+ }
+
+ /* Proceed when obj was created. */
+ if (success) {
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ r->pos += bytelen;
+
+ /* Set reply if this is the root object. */
+ if (r->ridx == 0) r->reply = obj;
+ moveToNextTask(r);
+ return REDIS_OK;
+ }
+ }
+
+ return REDIS_ERR;
+}
+
+static int processMultiBulkItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ void *obj;
+ char *p;
+ long elements;
+ int root = 0;
+
+ /* Set error for nested multi bulks with depth > 7 */
+ if (r->ridx == 8) {
+ __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+ "No support for nested multi bulk replies with depth > 7");
+ return REDIS_ERR;
+ }
+
+ if ((p = readLine(r,NULL)) != NULL) {
+ elements = readLongLong(p);
+ root = (r->ridx == 0);
+
+ if (elements == -1) {
+ if (r->fn && r->fn->createNil)
+ obj = r->fn->createNil(cur);
+ else
+ obj = (void*)REDIS_REPLY_NIL;
+
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ moveToNextTask(r);
+ } else {
+ if (r->fn && r->fn->createArray)
+ obj = r->fn->createArray(cur,elements);
+ else
+ obj = (void*)REDIS_REPLY_ARRAY;
+
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ /* Modify task stack when there are more than 0 elements. */
+ if (elements > 0) {
+ cur->elements = elements;
+ cur->obj = obj;
+ r->ridx++;
+ r->rstack[r->ridx].type = -1;
+ r->rstack[r->ridx].elements = -1;
+ r->rstack[r->ridx].idx = 0;
+ r->rstack[r->ridx].obj = NULL;
+ r->rstack[r->ridx].parent = cur;
+ r->rstack[r->ridx].privdata = r->privdata;
+ } else {
+ moveToNextTask(r);
+ }
+ }
+
+ /* Set reply if this is the root object. */
+ if (root) r->reply = obj;
+ return REDIS_OK;
+ }
+
+ return REDIS_ERR;
+}
+
+static int processItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ char *p;
+
+ /* check if we need to read type */
+ if (cur->type < 0) {
+ if ((p = readBytes(r,1)) != NULL) {
+ switch (p[0]) {
+ case '-':
+ cur->type = REDIS_REPLY_ERROR;
+ break;
+ case '+':
+ cur->type = REDIS_REPLY_STATUS;
+ break;
+ case ':':
+ cur->type = REDIS_REPLY_INTEGER;
+ break;
+ case '$':
+ cur->type = REDIS_REPLY_STRING;
+ break;
+ case '*':
+ cur->type = REDIS_REPLY_ARRAY;
+ break;
+ default:
+ __redisReaderSetErrorProtocolByte(r,*p);
+ return REDIS_ERR;
+ }
+ } else {
+ /* could not consume 1 byte */
+ return REDIS_ERR;
+ }
+ }
+
+ /* process typed item */
+ switch(cur->type) {
+ case REDIS_REPLY_ERROR:
+ case REDIS_REPLY_STATUS:
+ case REDIS_REPLY_INTEGER:
+ return processLineItem(r);
+ case REDIS_REPLY_STRING:
+ return processBulkItem(r);
+ case REDIS_REPLY_ARRAY:
+ return processMultiBulkItem(r);
+ default:
+ assert(NULL);
+ return REDIS_ERR; /* Avoid warning. */
+ }
+}
+
+redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
+ redisReader *r;
+
+ r = calloc(sizeof(redisReader),1);
+ if (r == NULL)
+ return NULL;
+
+ r->err = 0;
+ r->errstr[0] = '\0';
+ r->fn = fn;
+ r->buf = sdsempty();
+ r->maxbuf = REDIS_READER_MAX_BUF;
+ if (r->buf == NULL) {
+ free(r);
+ return NULL;
+ }
+
+ r->ridx = -1;
+ return r;
+}
+
+void redisReaderFree(redisReader *r) {
+ if (r->reply != NULL && r->fn && r->fn->freeObject)
+ r->fn->freeObject(r->reply);
+ if (r->buf != NULL)
+ sdsfree(r->buf);
+ free(r);
+}
+
+int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
+ sds newbuf;
+
+ /* Return early when this reader is in an erroneous state. */
+ if (r->err)
+ return REDIS_ERR;
+
+ /* Copy the provided buffer. */
+ if (buf != NULL && len >= 1) {
+ /* Destroy internal buffer when it is empty and is quite large. */
+ if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
+ sdsfree(r->buf);
+ r->buf = sdsempty();
+ r->pos = 0;
+
+ /* r->buf should not be NULL since we just free'd a larger one. */
+ assert(r->buf != NULL);
+ }
+
+ newbuf = sdscatlen(r->buf,buf,len);
+ if (newbuf == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ r->buf = newbuf;
+ r->len = sdslen(r->buf);
+ }
+
+ return REDIS_OK;
+}
+
+int redisReaderGetReply(redisReader *r, void **reply) {
+ /* Default target pointer to NULL. */
+ if (reply != NULL)
+ *reply = NULL;
+
+ /* Return early when this reader is in an erroneous state. */
+ if (r->err)
+ return REDIS_ERR;
+
+ /* When the buffer is empty, there will never be a reply. */
+ if (r->len == 0)
+ return REDIS_OK;
+
+ /* Set first item to process when the stack is empty. */
+ if (r->ridx == -1) {
+ r->rstack[0].type = -1;
+ r->rstack[0].elements = -1;
+ r->rstack[0].idx = -1;
+ r->rstack[0].obj = NULL;
+ r->rstack[0].parent = NULL;
+ r->rstack[0].privdata = r->privdata;
+ r->ridx = 0;
+ }
+
+ /* Process items in reply. */
+ while (r->ridx >= 0)
+ if (processItem(r) != REDIS_OK)
+ break;
+
+ /* Return ASAP when an error occurred. */
+ if (r->err)
+ return REDIS_ERR;
+
+ /* Discard part of the buffer when we've consumed at least 1k, to avoid
+ * doing unnecessary calls to memmove() in sds.c. */
+ if (r->pos >= 1024) {
+ sdsrange(r->buf,r->pos,-1);
+ r->pos = 0;
+ r->len = sdslen(r->buf);
+ }
+
+ /* Emit a reply when there is one. */
+ if (r->ridx == -1) {
+ if (reply != NULL)
+ *reply = r->reply;
+ r->reply = NULL;
+ }
+ return REDIS_OK;
+}
diff --git a/contrib/hiredis/read.h b/contrib/hiredis/read.h
new file mode 100644
index 0000000..180e6c6
--- /dev/null
+++ b/contrib/hiredis/read.h
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef __HIREDIS_READ_H
+#define __HIREDIS_READ_H
+#include <stdio.h> /* for size_t */
+
+#define REDIS_ERR -1
+#define REDIS_OK 0
+
+/* When an error occurs, the err flag in a context is set to hold the type of
+ * error that occurred. REDIS_ERR_IO means there was an I/O error and you
+ * should use the "errno" variable to find out what is wrong.
+ * For other values, the "errstr" field will hold a description. */
+#define REDIS_ERR_IO 1 /* Error in read or write */
+#define REDIS_ERR_EOF 3 /* End of file */
+#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
+#define REDIS_ERR_OOM 5 /* Out of memory */
+#define REDIS_ERR_OTHER 2 /* Everything else... */
+
+#define REDIS_REPLY_STRING 1
+#define REDIS_REPLY_ARRAY 2
+#define REDIS_REPLY_INTEGER 3
+#define REDIS_REPLY_NIL 4
+#define REDIS_REPLY_STATUS 5
+#define REDIS_REPLY_ERROR 6
+
+#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct redisReadTask {
+ int type;
+ int elements; /* number of elements in multibulk container */
+ int idx; /* index in parent (array) object */
+ void *obj; /* holds user-generated value for a read task */
+ struct redisReadTask *parent; /* parent task */
+ void *privdata; /* user-settable arbitrary field */
+} redisReadTask;
+
+typedef struct redisReplyObjectFunctions {
+ void *(*createString)(const redisReadTask*, char*, size_t);
+ void *(*createArray)(const redisReadTask*, int);
+ void *(*createInteger)(const redisReadTask*, long long);
+ void *(*createNil)(const redisReadTask*);
+ void (*freeObject)(void*);
+} redisReplyObjectFunctions;
+
+typedef struct redisReader {
+ int err; /* Error flags, 0 when there is no error */
+ char errstr[128]; /* String representation of error when applicable */
+
+ char *buf; /* Read buffer */
+ size_t pos; /* Buffer cursor */
+ size_t len; /* Buffer length */
+ size_t maxbuf; /* Max length of unused buffer */
+
+ redisReadTask rstack[9];
+ int ridx; /* Index of current read task */
+ void *reply; /* Temporary reply pointer */
+
+ redisReplyObjectFunctions *fn;
+ void *privdata;
+} redisReader;
+
+/* Public API for the protocol parser. */
+redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn);
+void redisReaderFree(redisReader *r);
+int redisReaderFeed(redisReader *r, const char *buf, size_t len);
+int redisReaderGetReply(redisReader *r, void **reply);
+
+/* Backwards compatibility, can be removed on big version bump. */
+#define redisReplyReaderCreate redisReaderCreate
+#define redisReplyReaderFree redisReaderFree
+#define redisReplyReaderFeed redisReaderFeed
+#define redisReplyReaderGetReply redisReaderGetReply
+#define redisReplyReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
+#define redisReplyReaderGetObject(_r) (((redisReader*)(_r))->reply)
+#define redisReplyReaderGetError(_r) (((redisReader*)(_r))->errstr)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/hiredis/sds.c b/contrib/hiredis/sds.c
new file mode 100644
index 0000000..5e75516
--- /dev/null
+++ b/contrib/hiredis/sds.c
@@ -0,0 +1,1095 @@
+/* SDS (Simple Dynamic Strings), A C dynamic strings library.
+ *
+ * Copyright (c) 2006-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+
+#include "sds.h"
+
+/* Create a new sds string with the content specified by the 'init' pointer
+ * and 'initlen'.
+ * If NULL is used for 'init' the string is initialized with zero bytes.
+ *
+ * The string is always null-termined (all the sds strings are, always) so
+ * even if you create an sds string with:
+ *
+ * mystring = sdsnewlen("abc",3");
+ *
+ * You can print the string with printf() as there is an implicit \0 at the
+ * end of the string. However the string is binary safe and can contain
+ * \0 characters in the middle, as the length is stored in the sds header. */
+sds sdsnewlen(const void *init, size_t initlen) {
+ struct sdshdr *sh;
+
+ if (init) {
+ sh = malloc(sizeof *sh+initlen+1);
+ } else {
+ sh = calloc(sizeof *sh+initlen+1,1);
+ }
+ if (sh == NULL) return NULL;
+ sh->len = initlen;
+ sh->free = 0;
+ if (initlen && init)
+ memcpy(sh->buf, init, initlen);
+ sh->buf[initlen] = '\0';
+ return (char*)sh->buf;
+}
+
+/* Create an empty (zero length) sds string. Even in this case the string
+ * always has an implicit null term. */
+sds sdsempty(void) {
+ return sdsnewlen("",0);
+}
+
+/* Create a new sds string starting from a null termined C string. */
+sds sdsnew(const char *init) {
+ size_t initlen = (init == NULL) ? 0 : strlen(init);
+ return sdsnewlen(init, initlen);
+}
+
+/* Duplicate an sds string. */
+sds sdsdup(const sds s) {
+ return sdsnewlen(s, sdslen(s));
+}
+
+/* Free an sds string. No operation is performed if 's' is NULL. */
+void sdsfree(sds s) {
+ if (s == NULL) return;
+ free(s-sizeof(struct sdshdr));
+}
+
+/* Set the sds string length to the length as obtained with strlen(), so
+ * considering as content only up to the first null term character.
+ *
+ * This function is useful when the sds string is hacked manually in some
+ * way, like in the following example:
+ *
+ * s = sdsnew("foobar");
+ * s[2] = '\0';
+ * sdsupdatelen(s);
+ * printf("%d\n", sdslen(s));
+ *
+ * The output will be "2", but if we comment out the call to sdsupdatelen()
+ * the output will be "6" as the string was modified but the logical length
+ * remains 6 bytes. */
+void sdsupdatelen(sds s) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ int reallen = strlen(s);
+ sh->free += (sh->len-reallen);
+ sh->len = reallen;
+}
+
+/* Modify an sds string on-place to make it empty (zero length).
+ * However all the existing buffer is not discarded but set as free space
+ * so that next append operations will not require allocations up to the
+ * number of bytes previously available. */
+void sdsclear(sds s) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ sh->free += sh->len;
+ sh->len = 0;
+ sh->buf[0] = '\0';
+}
+
+/* Enlarge the free space at the end of the sds string so that the caller
+ * is sure that after calling this function can overwrite up to addlen
+ * bytes after the end of the string, plus one more byte for nul term.
+ *
+ * Note: this does not change the *length* of the sds string as returned
+ * by sdslen(), but only the free buffer space we have. */
+sds sdsMakeRoomFor(sds s, size_t addlen) {
+ struct sdshdr *sh, *newsh;
+ size_t free = sdsavail(s);
+ size_t len, newlen;
+
+ if (free >= addlen) return s;
+ len = sdslen(s);
+ sh = (void*) (s-sizeof *sh);
+ newlen = (len+addlen);
+ if (newlen < SDS_MAX_PREALLOC)
+ newlen *= 2;
+ else
+ newlen += SDS_MAX_PREALLOC;
+ newsh = realloc(sh, sizeof *newsh+newlen+1);
+ if (newsh == NULL) return NULL;
+
+ newsh->free = newlen - len;
+ return newsh->buf;
+}
+
+/* Reallocate the sds string so that it has no free space at the end. The
+ * contained string remains not altered, but next concatenation operations
+ * will require a reallocation.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdsRemoveFreeSpace(sds s) {
+ struct sdshdr *sh;
+
+ sh = (void*) (s-sizeof *sh);
+ sh = realloc(sh, sizeof *sh+sh->len+1);
+ sh->free = 0;
+ return sh->buf;
+}
+
+/* Return the total size of the allocation of the specified sds string,
+ * including:
+ * 1) The sds header before the pointer.
+ * 2) The string.
+ * 3) The free buffer at the end if any.
+ * 4) The implicit null term.
+ */
+size_t sdsAllocSize(sds s) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+
+ return sizeof(*sh)+sh->len+sh->free+1;
+}
+
+/* Increment the sds length and decrements the left free space at the
+ * end of the string according to 'incr'. Also set the null term
+ * in the new end of the string.
+ *
+ * This function is used in order to fix the string length after the
+ * user calls sdsMakeRoomFor(), writes something after the end of
+ * the current string, and finally needs to set the new length.
+ *
+ * Note: it is possible to use a negative increment in order to
+ * right-trim the string.
+ *
+ * Usage example:
+ *
+ * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the
+ * following schema, to cat bytes coming from the kernel to the end of an
+ * sds string without copying into an intermediate buffer:
+ *
+ * oldlen = sdslen(s);
+ * s = sdsMakeRoomFor(s, BUFFER_SIZE);
+ * nread = read(fd, s+oldlen, BUFFER_SIZE);
+ * ... check for nread <= 0 and handle it ...
+ * sdsIncrLen(s, nread);
+ */
+void sdsIncrLen(sds s, int incr) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+
+ assert(sh->free >= incr);
+ sh->len += incr;
+ sh->free -= incr;
+ assert(sh->free >= 0);
+ s[sh->len] = '\0';
+}
+
+/* Grow the sds to have the specified length. Bytes that were not part of
+ * the original length of the sds will be set to zero.
+ *
+ * if the specified length is smaller than the current length, no operation
+ * is performed. */
+sds sdsgrowzero(sds s, size_t len) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ size_t totlen, curlen = sh->len;
+
+ if (len <= curlen) return s;
+ s = sdsMakeRoomFor(s,len-curlen);
+ if (s == NULL) return NULL;
+
+ /* Make sure added region doesn't contain garbage */
+ sh = (void*)(s-sizeof *sh);
+ memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */
+ totlen = sh->len+sh->free;
+ sh->len = len;
+ sh->free = totlen-sh->len;
+ return s;
+}
+
+/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the
+ * end of the specified sds string 's'.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatlen(sds s, const void *t, size_t len) {
+ struct sdshdr *sh;
+ size_t curlen = sdslen(s);
+
+ s = sdsMakeRoomFor(s,len);
+ if (s == NULL) return NULL;
+ sh = (void*) (s-sizeof *sh);
+ memcpy(s+curlen, t, len);
+ sh->len = curlen+len;
+ sh->free = sh->free-len;
+ s[curlen+len] = '\0';
+ return s;
+}
+
+/* Append the specified null termianted C string to the sds string 's'.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscat(sds s, const char *t) {
+ return sdscatlen(s, t, strlen(t));
+}
+
+/* Append the specified sds 't' to the existing sds 's'.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatsds(sds s, const sds t) {
+ return sdscatlen(s, t, sdslen(t));
+}
+
+/* Destructively modify the sds string 's' to hold the specified binary
+ * safe string pointed by 't' of length 'len' bytes. */
+sds sdscpylen(sds s, const char *t, size_t len) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ size_t totlen = sh->free+sh->len;
+
+ if (totlen < len) {
+ s = sdsMakeRoomFor(s,len-sh->len);
+ if (s == NULL) return NULL;
+ sh = (void*) (s-sizeof *sh);
+ totlen = sh->free+sh->len;
+ }
+ memcpy(s, t, len);
+ s[len] = '\0';
+ sh->len = len;
+ sh->free = totlen-len;
+ return s;
+}
+
+/* Like sdscpylen() but 't' must be a null-termined string so that the length
+ * of the string is obtained with strlen(). */
+sds sdscpy(sds s, const char *t) {
+ return sdscpylen(s, t, strlen(t));
+}
+
+/* Helper for sdscatlonglong() doing the actual number -> string
+ * conversion. 's' must point to a string with room for at least
+ * SDS_LLSTR_SIZE bytes.
+ *
+ * The function returns the length of the null-terminated string
+ * representation stored at 's'. */
+#define SDS_LLSTR_SIZE 21
+int sdsll2str(char *s, long long value) {
+ char *p, aux;
+ unsigned long long v;
+ size_t l;
+
+ /* Generate the string representation, this method produces
+ * an reversed string. */
+ v = (value < 0) ? -value : value;
+ p = s;
+ do {
+ *p++ = '0'+(v%10);
+ v /= 10;
+ } while(v);
+ if (value < 0) *p++ = '-';
+
+ /* Compute length and add null term. */
+ l = p-s;
+ *p = '\0';
+
+ /* Reverse the string. */
+ p--;
+ while(s < p) {
+ aux = *s;
+ *s = *p;
+ *p = aux;
+ s++;
+ p--;
+ }
+ return l;
+}
+
+/* Identical sdsll2str(), but for unsigned long long type. */
+int sdsull2str(char *s, unsigned long long v) {
+ char *p, aux;
+ size_t l;
+
+ /* Generate the string representation, this method produces
+ * an reversed string. */
+ p = s;
+ do {
+ *p++ = '0'+(v%10);
+ v /= 10;
+ } while(v);
+
+ /* Compute length and add null term. */
+ l = p-s;
+ *p = '\0';
+
+ /* Reverse the string. */
+ p--;
+ while(s < p) {
+ aux = *s;
+ *s = *p;
+ *p = aux;
+ s++;
+ p--;
+ }
+ return l;
+}
+
+/* Like sdscatpritf() but gets va_list instead of being variadic. */
+sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
+ va_list cpy;
+ char *buf, *t;
+ size_t buflen = 16;
+
+ while(1) {
+ buf = malloc(buflen);
+ if (buf == NULL) return NULL;
+ buf[buflen-2] = '\0';
+ va_copy(cpy,ap);
+ vsnprintf(buf, buflen, fmt, cpy);
+ if (buf[buflen-2] != '\0') {
+ free(buf);
+ buflen *= 2;
+ continue;
+ }
+ break;
+ }
+ t = sdscat(s, buf);
+ free(buf);
+ return t;
+}
+
+/* Append to the sds string 's' a string obtained using printf-alike format
+ * specifier.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call.
+ *
+ * Example:
+ *
+ * s = sdsnew("Sum is: ");
+ * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b);
+ *
+ * Often you need to create a string from scratch with the printf-alike
+ * format. When this is the need, just use sdsempty() as the target string:
+ *
+ * s = sdscatprintf(sdsempty(), "... your format ...", args);
+ */
+sds sdscatprintf(sds s, const char *fmt, ...) {
+ va_list ap;
+ char *t;
+ va_start(ap, fmt);
+ t = sdscatvprintf(s,fmt,ap);
+ va_end(ap);
+ return t;
+}
+
+/* This function is similar to sdscatprintf, but much faster as it does
+ * not rely on sprintf() family functions implemented by the libc that
+ * are often very slow. Moreover directly handling the sds string as
+ * new data is concatenated provides a performance improvement.
+ *
+ * However this function only handles an incompatible subset of printf-alike
+ * format specifiers:
+ *
+ * %s - C String
+ * %S - SDS string
+ * %i - signed int
+ * %I - 64 bit signed integer (long long, int64_t)
+ * %u - unsigned int
+ * %U - 64 bit unsigned integer (unsigned long long, uint64_t)
+ * %T - A size_t variable.
+ * %% - Verbatim "%" character.
+ */
+sds sdscatfmt(sds s, char const *fmt, ...) {
+ struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr)));
+ size_t initlen = sdslen(s);
+ const char *f = fmt;
+ int i;
+ va_list ap;
+
+ va_start(ap,fmt);
+ f = fmt; /* Next format specifier byte to process. */
+ i = initlen; /* Position of the next byte to write to dest str. */
+ while(*f) {
+ char next, *str;
+ int l;
+ long long num;
+ unsigned long long unum;
+
+ /* Make sure there is always space for at least 1 char. */
+ if (sh->free == 0) {
+ s = sdsMakeRoomFor(s,1);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+
+ switch(*f) {
+ case '%':
+ next = *(f+1);
+ f++;
+ switch(next) {
+ case 's':
+ case 'S':
+ str = va_arg(ap,char*);
+ l = (next == 's') ? strlen(str) : sdslen(str);
+ if (sh->free < l) {
+ s = sdsMakeRoomFor(s,l);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+ memcpy(s+i,str,l);
+ sh->len += l;
+ sh->free -= l;
+ i += l;
+ break;
+ case 'i':
+ case 'I':
+ if (next == 'i')
+ num = va_arg(ap,int);
+ else
+ num = va_arg(ap,long long);
+ {
+ char buf[SDS_LLSTR_SIZE];
+ l = sdsll2str(buf,num);
+ if (sh->free < l) {
+ s = sdsMakeRoomFor(s,l);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+ memcpy(s+i,buf,l);
+ sh->len += l;
+ sh->free -= l;
+ i += l;
+ }
+ break;
+ case 'u':
+ case 'U':
+ case 'T':
+ if (next == 'u')
+ unum = va_arg(ap,unsigned int);
+ else if(next == 'U')
+ unum = va_arg(ap,unsigned long long);
+ else
+ unum = (unsigned long long)va_arg(ap,size_t);
+ {
+ char buf[SDS_LLSTR_SIZE];
+ l = sdsull2str(buf,unum);
+ if (sh->free < l) {
+ s = sdsMakeRoomFor(s,l);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+ memcpy(s+i,buf,l);
+ sh->len += l;
+ sh->free -= l;
+ i += l;
+ }
+ break;
+ default: /* Handle %% and generally %<unknown>. */
+ s[i++] = next;
+ sh->len += 1;
+ sh->free -= 1;
+ break;
+ }
+ break;
+ default:
+ s[i++] = *f;
+ sh->len += 1;
+ sh->free -= 1;
+ break;
+ }
+ f++;
+ }
+ va_end(ap);
+
+ /* Add null-term */
+ s[i] = '\0';
+ return s;
+}
+
+
+/* Remove the part of the string from left and from right composed just of
+ * contiguous characters found in 'cset', that is a null terminted C string.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call.
+ *
+ * Example:
+ *
+ * s = sdsnew("AA...AA.a.aa.aHelloWorld :::");
+ * s = sdstrim(s,"A. :");
+ * printf("%s\n", s);
+ *
+ * Output will be just "Hello World".
+ */
+void sdstrim(sds s, const char *cset) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ char *start, *end, *sp, *ep;
+ size_t len;
+
+ sp = start = s;
+ ep = end = s+sdslen(s)-1;
+ while(sp <= end && strchr(cset, *sp)) sp++;
+ while(ep > start && strchr(cset, *ep)) ep--;
+ len = (sp > ep) ? 0 : ((ep-sp)+1);
+ if (sh->buf != sp) memmove(sh->buf, sp, len);
+ sh->buf[len] = '\0';
+ sh->free = sh->free+(sh->len-len);
+ sh->len = len;
+}
+
+/* Turn the string into a smaller (or equal) string containing only the
+ * substring specified by the 'start' and 'end' indexes.
+ *
+ * start and end can be negative, where -1 means the last character of the
+ * string, -2 the penultimate character, and so forth.
+ *
+ * The interval is inclusive, so the start and end characters will be part
+ * of the resulting string.
+ *
+ * The string is modified in-place.
+ *
+ * Example:
+ *
+ * s = sdsnew("Hello World");
+ * sdsrange(s,1,-1); => "ello World"
+ */
+void sdsrange(sds s, int start, int end) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ size_t newlen, len = sdslen(s);
+
+ if (len == 0) return;
+ if (start < 0) {
+ start = len+start;
+ if (start < 0) start = 0;
+ }
+ if (end < 0) {
+ end = len+end;
+ if (end < 0) end = 0;
+ }
+ newlen = (start > end) ? 0 : (end-start)+1;
+ if (newlen != 0) {
+ if (start >= (signed)len) {
+ newlen = 0;
+ } else if (end >= (signed)len) {
+ end = len-1;
+ newlen = (start > end) ? 0 : (end-start)+1;
+ }
+ } else {
+ start = 0;
+ }
+ if (start && newlen) memmove(sh->buf, sh->buf+start, newlen);
+ sh->buf[newlen] = 0;
+ sh->free = sh->free+(sh->len-newlen);
+ sh->len = newlen;
+}
+
+/* Apply tolower() to every character of the sds string 's'. */
+void sdstolower(sds s) {
+ int len = sdslen(s), j;
+
+ for (j = 0; j < len; j++) s[j] = tolower(s[j]);
+}
+
+/* Apply toupper() to every character of the sds string 's'. */
+void sdstoupper(sds s) {
+ int len = sdslen(s), j;
+
+ for (j = 0; j < len; j++) s[j] = toupper(s[j]);
+}
+
+/* Compare two sds strings s1 and s2 with memcmp().
+ *
+ * Return value:
+ *
+ * 1 if s1 > s2.
+ * -1 if s1 < s2.
+ * 0 if s1 and s2 are exactly the same binary string.
+ *
+ * If two strings share exactly the same prefix, but one of the two has
+ * additional characters, the longer string is considered to be greater than
+ * the smaller one. */
+int sdscmp(const sds s1, const sds s2) {
+ size_t l1, l2, minlen;
+ int cmp;
+
+ l1 = sdslen(s1);
+ l2 = sdslen(s2);
+ minlen = (l1 < l2) ? l1 : l2;
+ cmp = memcmp(s1,s2,minlen);
+ if (cmp == 0) return l1-l2;
+ return cmp;
+}
+
+/* Split 's' with separator in 'sep'. An array
+ * of sds strings is returned. *count will be set
+ * by reference to the number of tokens returned.
+ *
+ * On out of memory, zero length string, zero length
+ * separator, NULL is returned.
+ *
+ * Note that 'sep' is able to split a string using
+ * a multi-character separator. For example
+ * sdssplit("foo_-_bar","_-_"); will return two
+ * elements "foo" and "bar".
+ *
+ * This version of the function is binary-safe but
+ * requires length arguments. sdssplit() is just the
+ * same function but for zero-terminated strings.
+ */
+sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
+ int elements = 0, slots = 5, start = 0, j;
+ sds *tokens;
+
+ if (seplen < 1 || len < 0) return NULL;
+
+ tokens = malloc(sizeof(sds)*slots);
+ if (tokens == NULL) return NULL;
+
+ if (len == 0) {
+ *count = 0;
+ return tokens;
+ }
+ for (j = 0; j < (len-(seplen-1)); j++) {
+ /* make sure there is room for the next element and the final one */
+ if (slots < elements+2) {
+ sds *newtokens;
+
+ slots *= 2;
+ newtokens = realloc(tokens,sizeof(sds)*slots);
+ if (newtokens == NULL) goto cleanup;
+ tokens = newtokens;
+ }
+ /* search the separator */
+ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) {
+ tokens[elements] = sdsnewlen(s+start,j-start);
+ if (tokens[elements] == NULL) goto cleanup;
+ elements++;
+ start = j+seplen;
+ j = j+seplen-1; /* skip the separator */
+ }
+ }
+ /* Add the final element. We are sure there is room in the tokens array. */
+ tokens[elements] = sdsnewlen(s+start,len-start);
+ if (tokens[elements] == NULL) goto cleanup;
+ elements++;
+ *count = elements;
+ return tokens;
+
+cleanup:
+ {
+ int i;
+ for (i = 0; i < elements; i++) sdsfree(tokens[i]);
+ free(tokens);
+ *count = 0;
+ return NULL;
+ }
+}
+
+/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */
+void sdsfreesplitres(sds *tokens, int count) {
+ if (!tokens) return;
+ while(count--)
+ sdsfree(tokens[count]);
+ free(tokens);
+}
+
+/* Create an sds string from a long long value. It is much faster than:
+ *
+ * sdscatprintf(sdsempty(),"%lld\n", value);
+ */
+sds sdsfromlonglong(long long value) {
+ char buf[32], *p;
+ unsigned long long v;
+
+ v = (value < 0) ? -value : value;
+ p = buf+31; /* point to the last character */
+ do {
+ *p-- = '0'+(v%10);
+ v /= 10;
+ } while(v);
+ if (value < 0) *p-- = '-';
+ p++;
+ return sdsnewlen(p,32-(p-buf));
+}
+
+/* Append to the sds string "s" an escaped string representation where
+ * all the non-printable characters (tested with isprint()) are turned into
+ * escapes in the form "\n\r\a...." or "\x<hex-number>".
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatrepr(sds s, const char *p, size_t len) {
+ s = sdscatlen(s,"\"",1);
+ while(len--) {
+ switch(*p) {
+ case '\\':
+ case '"':
+ s = sdscatprintf(s,"\\%c",*p);
+ break;
+ case '\n': s = sdscatlen(s,"\\n",2); break;
+ case '\r': s = sdscatlen(s,"\\r",2); break;
+ case '\t': s = sdscatlen(s,"\\t",2); break;
+ case '\a': s = sdscatlen(s,"\\a",2); break;
+ case '\b': s = sdscatlen(s,"\\b",2); break;
+ default:
+ if (isprint(*p))
+ s = sdscatprintf(s,"%c",*p);
+ else
+ s = sdscatprintf(s,"\\x%02x",(unsigned char)*p);
+ break;
+ }
+ p++;
+ }
+ return sdscatlen(s,"\"",1);
+}
+
+/* Helper function for sdssplitargs() that returns non zero if 'c'
+ * is a valid hex digit. */
+int is_hex_digit(char c) {
+ return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
+ (c >= 'A' && c <= 'F');
+}
+
+/* Helper function for sdssplitargs() that converts a hex digit into an
+ * integer from 0 to 15 */
+int hex_digit_to_int(char c) {
+ switch(c) {
+ case '0': return 0;
+ case '1': return 1;
+ case '2': return 2;
+ case '3': return 3;
+ case '4': return 4;
+ case '5': return 5;
+ case '6': return 6;
+ case '7': return 7;
+ case '8': return 8;
+ case '9': return 9;
+ case 'a': case 'A': return 10;
+ case 'b': case 'B': return 11;
+ case 'c': case 'C': return 12;
+ case 'd': case 'D': return 13;
+ case 'e': case 'E': return 14;
+ case 'f': case 'F': return 15;
+ default: return 0;
+ }
+}
+
+/* Split a line into arguments, where every argument can be in the
+ * following programming-language REPL-alike form:
+ *
+ * foo bar "newline are supported\n" and "\xff\x00otherstuff"
+ *
+ * The number of arguments is stored into *argc, and an array
+ * of sds is returned.
+ *
+ * The caller should free the resulting array of sds strings with
+ * sdsfreesplitres().
+ *
+ * Note that sdscatrepr() is able to convert back a string into
+ * a quoted string in the same format sdssplitargs() is able to parse.
+ *
+ * The function returns the allocated tokens on success, even when the
+ * input string is empty, or NULL if the input contains unbalanced
+ * quotes or closed quotes followed by non space characters
+ * as in: "foo"bar or "foo'
+ */
+sds *sdssplitargs(const char *line, int *argc) {
+ const char *p = line;
+ char *current = NULL;
+ char **vector = NULL;
+
+ *argc = 0;
+ while(1) {
+ /* skip blanks */
+ while(*p && isspace(*p)) p++;
+ if (*p) {
+ /* get a token */
+ int inq=0; /* set to 1 if we are in "quotes" */
+ int insq=0; /* set to 1 if we are in 'single quotes' */
+ int done=0;
+
+ if (current == NULL) current = sdsempty();
+ while(!done) {
+ if (inq) {
+ if (*p == '\\' && *(p+1) == 'x' &&
+ is_hex_digit(*(p+2)) &&
+ is_hex_digit(*(p+3)))
+ {
+ unsigned char byte;
+
+ byte = (hex_digit_to_int(*(p+2))*16)+
+ hex_digit_to_int(*(p+3));
+ current = sdscatlen(current,(char*)&byte,1);
+ p += 3;
+ } else if (*p == '\\' && *(p+1)) {
+ char c;
+
+ p++;
+ switch(*p) {
+ case 'n': c = '\n'; break;
+ case 'r': c = '\r'; break;
+ case 't': c = '\t'; break;
+ case 'b': c = '\b'; break;
+ case 'a': c = '\a'; break;
+ default: c = *p; break;
+ }
+ current = sdscatlen(current,&c,1);
+ } else if (*p == '"') {
+ /* closing quote must be followed by a space or
+ * nothing at all. */
+ if (*(p+1) && !isspace(*(p+1))) goto err;
+ done=1;
+ } else if (!*p) {
+ /* unterminated quotes */
+ goto err;
+ } else {
+ current = sdscatlen(current,p,1);
+ }
+ } else if (insq) {
+ if (*p == '\\' && *(p+1) == '\'') {
+ p++;
+ current = sdscatlen(current,"'",1);
+ } else if (*p == '\'') {
+ /* closing quote must be followed by a space or
+ * nothing at all. */
+ if (*(p+1) && !isspace(*(p+1))) goto err;
+ done=1;
+ } else if (!*p) {
+ /* unterminated quotes */
+ goto err;
+ } else {
+ current = sdscatlen(current,p,1);
+ }
+ } else {
+ switch(*p) {
+ case ' ':
+ case '\n':
+ case '\r':
+ case '\t':
+ case '\0':
+ done=1;
+ break;
+ case '"':
+ inq=1;
+ break;
+ case '\'':
+ insq=1;
+ break;
+ default:
+ current = sdscatlen(current,p,1);
+ break;
+ }
+ }
+ if (*p) p++;
+ }
+ /* add the token to the vector */
+ vector = realloc(vector,((*argc)+1)*sizeof(char*));
+ vector[*argc] = current;
+ (*argc)++;
+ current = NULL;
+ } else {
+ /* Even on empty input string return something not NULL. */
+ if (vector == NULL) vector = malloc(sizeof(void*));
+ return vector;
+ }
+ }
+
+err:
+ while((*argc)--)
+ sdsfree(vector[*argc]);
+ free(vector);
+ if (current) sdsfree(current);
+ *argc = 0;
+ return NULL;
+}
+
+/* Modify the string substituting all the occurrences of the set of
+ * characters specified in the 'from' string to the corresponding character
+ * in the 'to' array.
+ *
+ * For instance: sdsmapchars(mystring, "ho", "01", 2)
+ * will have the effect of turning the string "hello" into "0ell1".
+ *
+ * The function returns the sds string pointer, that is always the same
+ * as the input pointer since no resize is needed. */
+sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
+ size_t j, i, l = sdslen(s);
+
+ for (j = 0; j < l; j++) {
+ for (i = 0; i < setlen; i++) {
+ if (s[j] == from[i]) {
+ s[j] = to[i];
+ break;
+ }
+ }
+ }
+ return s;
+}
+
+/* Join an array of C strings using the specified separator (also a C string).
+ * Returns the result as an sds string. */
+sds sdsjoin(char **argv, int argc, char *sep, size_t seplen) {
+ sds join = sdsempty();
+ int j;
+
+ for (j = 0; j < argc; j++) {
+ join = sdscat(join, argv[j]);
+ if (j != argc-1) join = sdscatlen(join,sep,seplen);
+ }
+ return join;
+}
+
+/* Like sdsjoin, but joins an array of SDS strings. */
+sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
+ sds join = sdsempty();
+ int j;
+
+ for (j = 0; j < argc; j++) {
+ join = sdscatsds(join, argv[j]);
+ if (j != argc-1) join = sdscatlen(join,sep,seplen);
+ }
+ return join;
+}
+
+#ifdef SDS_TEST_MAIN
+#include <stdio.h>
+#include "testhelp.h"
+
+int main(void) {
+ {
+ struct sdshdr *sh;
+ sds x = sdsnew("foo"), y;
+
+ test_cond("Create a string and obtain the length",
+ sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
+
+ sdsfree(x);
+ x = sdsnewlen("foo",2);
+ test_cond("Create a string with specified length",
+ sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
+
+ x = sdscat(x,"bar");
+ test_cond("Strings concatenation",
+ sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
+
+ x = sdscpy(x,"a");
+ test_cond("sdscpy() against an originally longer string",
+ sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
+
+ x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
+ test_cond("sdscpy() against an originally shorter string",
+ sdslen(x) == 33 &&
+ memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
+
+ sdsfree(x);
+ x = sdscatprintf(sdsempty(),"%d",123);
+ test_cond("sdscatprintf() seems working in the base case",
+ sdslen(x) == 3 && memcmp(x,"123\0",4) ==0)
+
+ sdsfree(x);
+ x = sdsnew("xxciaoyyy");
+ sdstrim(x,"xy");
+ test_cond("sdstrim() correctly trims characters",
+ sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
+
+ y = sdsdup(x);
+ sdsrange(y,1,1);
+ test_cond("sdsrange(...,1,1)",
+ sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,1,-1);
+ test_cond("sdsrange(...,1,-1)",
+ sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,-2,-1);
+ test_cond("sdsrange(...,-2,-1)",
+ sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,2,1);
+ test_cond("sdsrange(...,2,1)",
+ sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,1,100);
+ test_cond("sdsrange(...,1,100)",
+ sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,100,100);
+ test_cond("sdsrange(...,100,100)",
+ sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnew("foo");
+ y = sdsnew("foa");
+ test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnew("bar");
+ y = sdsnew("bar");
+ test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnew("aar");
+ y = sdsnew("bar");
+ test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnewlen("\a\n\0foo\r",7);
+ y = sdscatrepr(sdsempty(),x,sdslen(x));
+ test_cond("sdscatrepr(...data...)",
+ memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
+
+ {
+ int oldfree;
+
+ sdsfree(x);
+ x = sdsnew("0");
+ sh = (void*) (x-(sizeof(struct sdshdr)));
+ test_cond("sdsnew() free/len buffers", sh->len == 1 && sh->free == 0);
+ x = sdsMakeRoomFor(x,1);
+ sh = (void*) (x-(sizeof(struct sdshdr)));
+ test_cond("sdsMakeRoomFor()", sh->len == 1 && sh->free > 0);
+ oldfree = sh->free;
+ x[1] = '1';
+ sdsIncrLen(x,1);
+ test_cond("sdsIncrLen() -- content", x[0] == '0' && x[1] == '1');
+ test_cond("sdsIncrLen() -- len", sh->len == 2);
+ test_cond("sdsIncrLen() -- free", sh->free == oldfree-1);
+ }
+ }
+ test_report()
+ return 0;
+}
+#endif
diff --git a/contrib/hiredis/sds.h b/contrib/hiredis/sds.h
new file mode 100644
index 0000000..a494f2e
--- /dev/null
+++ b/contrib/hiredis/sds.h
@@ -0,0 +1,108 @@
+/* SDS (Simple Dynamic Strings), A C dynamic strings library.
+ *
+ * Copyright (c) 2006-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __SDS_H
+#define __SDS_H
+
+#define SDS_MAX_PREALLOC (1024*1024)
+
+#include <sys/types.h>
+#include <stdarg.h>
+#ifdef _MSC_VER
+#include "win32.h"
+#endif
+
+typedef char *sds;
+
+struct sdshdr {
+ int len;
+ int free;
+ char buf[];
+};
+
+static inline size_t sdslen(const sds s) {
+ struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh);
+ return sh->len;
+}
+
+static inline size_t sdsavail(const sds s) {
+ struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh);
+ return sh->free;
+}
+
+sds sdsnewlen(const void *init, size_t initlen);
+sds sdsnew(const char *init);
+sds sdsempty(void);
+size_t sdslen(const sds s);
+sds sdsdup(const sds s);
+void sdsfree(sds s);
+size_t sdsavail(const sds s);
+sds sdsgrowzero(sds s, size_t len);
+sds sdscatlen(sds s, const void *t, size_t len);
+sds sdscat(sds s, const char *t);
+sds sdscatsds(sds s, const sds t);
+sds sdscpylen(sds s, const char *t, size_t len);
+sds sdscpy(sds s, const char *t);
+
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+sds sdscatvprintf(sds s, const char *fmt, va_list ap);
+#ifdef __GNUC__
+sds sdscatprintf(sds s, const char *fmt, ...)
+ __attribute__((format(printf, 2, 3)));
+#else
+sds sdscatprintf(sds s, const char *fmt, ...);
+#endif
+
+sds sdscatfmt(sds s, char const *fmt, ...);
+void sdstrim(sds s, const char *cset);
+void sdsrange(sds s, int start, int end);
+void sdsupdatelen(sds s);
+void sdsclear(sds s);
+int sdscmp(const sds s1, const sds s2);
+sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
+void sdsfreesplitres(sds *tokens, int count);
+void sdstolower(sds s);
+void sdstoupper(sds s);
+sds sdsfromlonglong(long long value);
+sds sdscatrepr(sds s, const char *p, size_t len);
+sds *sdssplitargs(const char *line, int *argc);
+sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
+sds sdsjoin(char **argv, int argc, char *sep, size_t seplen);
+sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
+
+/* Low level functions exposed to the user API */
+sds sdsMakeRoomFor(sds s, size_t addlen);
+void sdsIncrLen(sds s, int incr);
+sds sdsRemoveFreeSpace(sds s);
+size_t sdsAllocSize(sds s);
+
+#endif
diff --git a/contrib/http-parser/CMakeLists.txt b/contrib/http-parser/CMakeLists.txt
new file mode 100644
index 0000000..a5da701
--- /dev/null
+++ b/contrib/http-parser/CMakeLists.txt
@@ -0,0 +1,8 @@
+SET(HTTPSRC http_parser.c)
+
+
+SET(HTTP_COMPILE_FLAGS "-DRSPAMD_LIB")
+
+ADD_LIBRARY(rspamd-http-parser STATIC ${HTTPSRC})
+SET_TARGET_PROPERTIES(rspamd-http-parser PROPERTIES VERSION ${RSPAMD_VERSION})
+SET_TARGET_PROPERTIES(rspamd-http-parser PROPERTIES COMPILE_FLAGS "${HTTP_COMPILE_FLAGS}") \ No newline at end of file
diff --git a/contrib/http-parser/LICENSE-MIT b/contrib/http-parser/LICENSE-MIT
new file mode 100644
index 0000000..58010b3
--- /dev/null
+++ b/contrib/http-parser/LICENSE-MIT
@@ -0,0 +1,23 @@
+http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright
+Igor Sysoev.
+
+Additional changes are licensed under the same terms as NGINX and
+copyright Joyent, Inc. and other Node contributors. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.
diff --git a/contrib/http-parser/http_parser.c b/contrib/http-parser/http_parser.c
new file mode 100644
index 0000000..c14ecc0
--- /dev/null
+++ b/contrib/http-parser/http_parser.c
@@ -0,0 +1,2268 @@
+/* Based on src/http/ngx_http_parse.c from NGINX copyright Igor Sysoev
+ *
+ * Additional changes are licensed under the same terms as NGINX and
+ * copyright Joyent, Inc. and other Node contributors. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#include "config.h"
+#include "http_parser.h"
+#include <assert.h>
+
+#ifndef ULLONG_MAX
+# define ULLONG_MAX ((uint64_t) -1) /* 2^64-1 */
+#endif
+
+#ifndef MIN
+# define MIN(a,b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef ARRAY_SIZE
+# define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
+#endif
+
+#ifndef BIT_AT
+# define BIT_AT(a, i) \
+ (!!((unsigned int) (a)[(unsigned int) (i) >> 3] & \
+ (1 << ((unsigned int) (i) & 7))))
+#endif
+
+#ifndef ELEM_AT
+# define ELEM_AT(a, i, v) ((unsigned int) (i) < ARRAY_SIZE(a) ? (a)[(i)] : (v))
+#endif
+
+#define SET_ERRNO(e) \
+do { \
+ parser->http_errno = (e); \
+} while(0)
+
+
+/* Run the notify callback FOR, returning ER if it fails */
+#define CALLBACK_NOTIFY_(FOR, ER) \
+do { \
+ assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
+ \
+ if (settings->on_##FOR) { \
+ if (0 != settings->on_##FOR(parser)) { \
+ SET_ERRNO(HPE_CB_##FOR); \
+ } \
+ \
+ /* We either errored above or got paused; get out */ \
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
+ return (ER); \
+ } \
+ } \
+} while (0)
+
+/* Run the notify callback FOR and consume the current byte */
+#define CALLBACK_NOTIFY(FOR) CALLBACK_NOTIFY_(FOR, p - data + 1)
+
+/* Run the notify callback FOR and don't consume the current byte */
+#define CALLBACK_NOTIFY_NOADVANCE(FOR) CALLBACK_NOTIFY_(FOR, p - data)
+
+/* Run data callback FOR with LEN bytes, returning ER if it fails */
+#define CALLBACK_DATA_(FOR, LEN, ER) \
+do { \
+ assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
+ \
+ if (FOR##_mark) { \
+ if (settings->on_##FOR) { \
+ if (0 != settings->on_##FOR(parser, FOR##_mark, (LEN))) { \
+ SET_ERRNO(HPE_CB_##FOR); \
+ } \
+ \
+ /* We either errored above or got paused; get out */ \
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
+ return (ER); \
+ } \
+ } \
+ FOR##_mark = NULL; \
+ } \
+} while (0)
+
+/* Run the data callback FOR and consume the current byte */
+#define CALLBACK_DATA(FOR) \
+ CALLBACK_DATA_(FOR, p - FOR##_mark, p - data + 1)
+
+/* Run the data callback FOR and don't consume the current byte */
+#define CALLBACK_DATA_NOADVANCE(FOR) \
+ CALLBACK_DATA_(FOR, p - FOR##_mark, p - data)
+
+/* Set the mark FOR; non-destructive if mark is already set */
+#define MARK(FOR) \
+do { \
+ if (!FOR##_mark) { \
+ FOR##_mark = p; \
+ } \
+} while (0)
+
+
+#define PROXY_CONNECTION "proxy-connection"
+#define CONNECTION "connection"
+#define CONTENT_LENGTH "content-length"
+#define TRANSFER_ENCODING "transfer-encoding"
+#define UPGRADE "upgrade"
+#define CHUNKED "chunked"
+#define KEEP_ALIVE "keep-alive"
+#define CLOSE "close"
+
+enum rspamd_http_message_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
+
+static const char *method_strings[] =
+ {
+#define XX(num, name, string) #string,
+ HTTP_METHOD_MAP(XX)
+#undef XX
+ };
+
+
+/* Tokens as defined by rfc 2616. Also lowercases them.
+ * token = 1*<any CHAR except CTLs or separators>
+ * separators = "(" | ")" | "<" | ">" | "@"
+ * | "," | ";" | ":" | "\" | <">
+ * | "/" | "[" | "]" | "?" | "="
+ * | "{" | "}" | SP | HT
+ */
+static const char tokens[256] = {
+/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
+ 0, '!', 0, '#', '$', '%', '&', '\'',
+/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
+ 0, 0, '*', '+', 0, '-', '.', 0,
+/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
+ '0', '1', '2', '3', '4', '5', '6', '7',
+/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
+ '8', '9', 0, 0, 0, 0, 0, 0,
+/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
+ 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
+ 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
+ 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
+/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
+ 'x', 'y', 'z', 0, 0, 0, '^', '_',
+/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
+ '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
+ 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
+ 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
+/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
+ 'x', 'y', 'z', 0, '|', 0, '~', 0 };
+
+
+static const int8_t unhex[256] =
+ {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
+ ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ };
+
+
+#if HTTP_PARSER_STRICT
+# define T(v) 0
+#else
+# define T(v) v
+#endif
+
+
+static const uint8_t normal_url_char[32] = {
+/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
+ 0 | T(2) | 0 | 0 | T(16) | 0 | 0 | 0,
+/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
+ 0 | 2 | 4 | 0 | 16 | 32 | 64 | 128,
+/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0,
+/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, };
+
+#undef T
+
+enum state
+ { s_dead = 1 /* important that this is > 0 */
+
+ , s_start_req_or_res
+ , s_res_or_resp_H
+ , s_start_res
+ , s_res_H
+ , s_res_HT
+ , s_res_HTT
+ , s_res_HTTP
+ , s_res_first_http_major
+ , s_res_http_major
+ , s_res_first_http_minor
+ , s_res_http_minor
+ , s_res_first_status_code
+ , s_res_status_code
+ , s_res_status_start
+ , s_res_status
+ , s_res_line_almost_done
+
+ , s_start_req
+
+ , s_req_method
+ , s_req_spaces_before_url
+ , s_req_schema
+ , s_req_schema_slash
+ , s_req_schema_slash_slash
+ , s_req_server_start
+ , s_req_server
+ , s_req_server_with_at
+ , s_req_path
+ , s_req_query_string_start
+ , s_req_query_string
+ , s_req_fragment_start
+ , s_req_fragment
+ , s_req_http_start
+ , s_req_http_H
+ , s_req_http_HT
+ , s_req_http_HTT
+ , s_req_http_HTTP
+ , s_req_first_http_major
+ , s_req_http_major
+ , s_req_first_http_minor
+ , s_req_http_minor
+ , s_req_spamc_start
+ , s_req_spamc
+ , s_req_line_almost_done
+
+ , s_header_field_start
+ , s_header_field
+ , s_header_value_start
+ , s_header_value
+ , s_header_value_lws
+
+ , s_header_almost_done
+
+ , s_chunk_size_start
+ , s_chunk_size
+ , s_chunk_parameters
+ , s_chunk_size_almost_done
+
+ , s_headers_almost_done
+ , s_headers_done
+
+ /* Important: 's_headers_done' must be the last 'header' state. All
+ * states beyond this must be 'body' states. It is used for overflow
+ * checking. See the PARSING_HEADER() macro.
+ */
+
+ , s_chunk_data
+ , s_chunk_data_almost_done
+ , s_chunk_data_done
+
+ , s_body_identity
+ , s_body_identity_eof
+
+ , s_message_done
+ };
+
+
+#define PARSING_HEADER(state) (state <= s_headers_done)
+
+
+enum header_states
+ { h_general = 0
+ , h_C
+ , h_CO
+ , h_CON
+
+ , h_matching_connection
+ , h_matching_proxy_connection
+ , h_matching_content_length
+ , h_matching_transfer_encoding
+ , h_matching_upgrade
+
+ , h_connection
+ , h_content_length
+ , h_transfer_encoding
+ , h_upgrade
+
+ , h_matching_transfer_encoding_chunked
+ , h_matching_connection_keep_alive
+ , h_matching_connection_close
+
+ , h_transfer_encoding_chunked
+ , h_connection_keep_alive
+ , h_connection_close
+ };
+
+enum http_host_state
+ {
+ s_http_host_dead = 1
+ , s_http_userinfo_start
+ , s_http_userinfo
+ , s_http_host_start
+ , s_http_host_v6_start
+ , s_http_host
+ , s_http_host_v6
+ , s_http_host_v6_end
+ , s_http_host_port_start
+ , s_http_host_port
+};
+
+/* Macros for character classes; depends on strict-mode */
+#define CR '\r'
+#define LF '\n'
+#define LOWER(c) (unsigned char)(c | 0x20)
+#define IS_ALPHA(c) (LOWER(c) >= 'a' && LOWER(c) <= 'z')
+#define IS_NUM(c) ((c) >= '0' && (c) <= '9')
+#define IS_ALPHANUM(c) (IS_ALPHA(c) || IS_NUM(c))
+#define IS_HEX(c) (IS_NUM(c) || (LOWER(c) >= 'a' && LOWER(c) <= 'f'))
+#define IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \
+ (c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \
+ (c) == ')')
+#define IS_USERINFO_CHAR(c) (IS_ALPHANUM(c) || IS_MARK(c) || (c) == '%' || \
+ (c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \
+ (c) == '$' || (c) == ',')
+
+#if HTTP_PARSER_STRICT
+#define TOKEN(c) (tokens[(unsigned char)c])
+#define IS_URL_CHAR(c) (BIT_AT(normal_url_char, (unsigned char)c))
+#define IS_HOST_CHAR(c) (IS_ALPHANUM(c) || (c) == '.' || (c) == '-')
+#else
+#define TOKEN(c) ((c == ' ') ? ' ' : tokens[(unsigned char)c])
+#define IS_URL_CHAR(c) \
+ (BIT_AT(normal_url_char, (unsigned char)c) || ((c) & 0x80))
+#define IS_HOST_CHAR(c) \
+ (IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_')
+#endif
+
+
+#define start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res)
+
+
+#if HTTP_PARSER_STRICT
+# define STRICT_CHECK(cond) \
+do { \
+ if (cond) { \
+ SET_ERRNO(HPE_STRICT); \
+ goto error; \
+ } \
+} while (0)
+# define NEW_MESSAGE() (http_should_keep_alive(parser) ? start_state : s_dead)
+#else
+# define STRICT_CHECK(cond)
+# define NEW_MESSAGE() start_state
+#endif
+
+
+/* Map errno values to strings for human-readable output */
+#define HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s },
+static struct {
+ const char *name;
+ const char *description;
+} http_strerror_tab[] = {
+ HTTP_ERRNO_MAP(HTTP_STRERROR_GEN)
+};
+#undef HTTP_STRERROR_GEN
+
+int http_message_needs_eof(const http_parser *parser);
+
+/* Our URL parser.
+ *
+ * This is designed to be shared by http_parser_execute() for URL validation,
+ * hence it has a state transition + byte-for-byte interface. In addition, it
+ * is meant to be embedded in http_parser_parse_url(), which does the dirty
+ * work of turning state transitions URL components for its API.
+ *
+ * This function should only be invoked with non-space characters. It is
+ * assumed that the caller cares about (and can detect) the transition between
+ * URL and non-URL states by looking for these.
+ */
+static enum state
+parse_url_char(enum state s, const char ch)
+{
+ if (ch == ' ' || ch == '\r' || ch == '\n') {
+ return s_dead;
+ }
+
+#if HTTP_PARSER_STRICT
+ if (ch == '\t' || ch == '\f') {
+ return s_dead;
+ }
+#endif
+
+ switch (s) {
+ case s_req_spaces_before_url:
+ /* Proxied requests are followed by scheme of an absolute URI (alpha).
+ * All methods except CONNECT are followed by '/' or '*'.
+ */
+
+ if (ch == '/' || ch == '*') {
+ return s_req_path;
+ }
+
+ if (IS_ALPHA(ch)) {
+ return s_req_schema;
+ }
+
+ break;
+
+ case s_req_schema:
+ if (IS_ALPHA(ch)) {
+ return s;
+ }
+
+ if (ch == ':') {
+ return s_req_schema_slash;
+ }
+
+ break;
+
+ case s_req_schema_slash:
+ if (ch == '/') {
+ return s_req_schema_slash_slash;
+ }
+
+ break;
+
+ case s_req_schema_slash_slash:
+ if (ch == '/') {
+ return s_req_server_start;
+ }
+
+ break;
+
+ case s_req_server_with_at:
+ if (ch == '@') {
+ return s_dead;
+ }
+
+ /* FALLTHROUGH */
+ case s_req_server_start:
+ case s_req_server:
+ if (ch == '/') {
+ return s_req_path;
+ }
+
+ if (ch == '?') {
+ return s_req_query_string_start;
+ }
+
+ if (ch == '@') {
+ return s_req_server_with_at;
+ }
+
+ if (IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') {
+ return s_req_server;
+ }
+
+ break;
+
+ case s_req_path:
+ if (IS_URL_CHAR(ch)) {
+ return s;
+ }
+
+ switch (ch) {
+ case '?':
+ return s_req_query_string_start;
+
+ case '#':
+ return s_req_fragment_start;
+ }
+
+ break;
+
+ case s_req_query_string_start:
+ case s_req_query_string:
+ if (IS_URL_CHAR(ch)) {
+ return s_req_query_string;
+ }
+
+ switch (ch) {
+ case '?':
+ /* allow extra '?' in query string */
+ return s_req_query_string;
+
+ case '#':
+ return s_req_fragment_start;
+ }
+
+ break;
+
+ case s_req_fragment_start:
+ if (IS_URL_CHAR(ch)) {
+ return s_req_fragment;
+ }
+
+ switch (ch) {
+ case '?':
+ return s_req_fragment;
+
+ case '#':
+ return s;
+ }
+
+ break;
+
+ case s_req_fragment:
+ if (IS_URL_CHAR(ch)) {
+ return s;
+ }
+
+ switch (ch) {
+ case '?':
+ case '#':
+ return s;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+
+ /* We should never fall out of the switch above unless there's an error */
+ return s_dead;
+}
+
+size_t http_parser_execute (http_parser *parser,
+ const http_parser_settings *settings,
+ const char *data,
+ size_t len)
+{
+ char c, ch;
+ int8_t unhex_val;
+ const char *p = data;
+ const char *header_field_mark = 0;
+ const char *header_value_mark = 0;
+ const char *url_mark = 0;
+ const char *body_mark = 0;
+ const char *status_mark = 0;
+
+ /* We're in an error state. Don't bother doing anything. */
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
+ return 0;
+ }
+
+ if (len == 0) {
+ switch (parser->state) {
+ case s_body_identity_eof:
+ /* Use of CALLBACK_NOTIFY() here would erroneously return 1 byte read if
+ * we got paused.
+ */
+ CALLBACK_NOTIFY_NOADVANCE(message_complete);
+ return 0;
+
+ case s_dead:
+ case s_start_req_or_res:
+ case s_start_res:
+ case s_start_req:
+ return 0;
+
+ default:
+ SET_ERRNO(HPE_INVALID_EOF_STATE);
+ return 1;
+ }
+ }
+
+
+ if (parser->state == s_header_field)
+ header_field_mark = data;
+ if (parser->state == s_header_value)
+ header_value_mark = data;
+ switch (parser->state) {
+ case s_req_path:
+ case s_req_schema:
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ case s_req_server:
+ case s_req_server_with_at:
+ case s_req_query_string_start:
+ case s_req_query_string:
+ case s_req_fragment_start:
+ case s_req_fragment:
+ url_mark = data;
+ break;
+ case s_res_status:
+ status_mark = data;
+ break;
+ }
+
+ for (p=data; p != data + len; p++) {
+ ch = *p;
+
+ if (PARSING_HEADER(parser->state)) {
+ ++parser->nread;
+ /* Don't allow the total size of the HTTP headers (including the status
+ * line) to exceed HTTP_MAX_HEADER_SIZE. This check is here to protect
+ * embedders against denial-of-service attacks where the attacker feeds
+ * us a never-ending header that the embedder keeps buffering.
+ *
+ * This check is arguably the responsibility of embedders but we're doing
+ * it on the embedder's behalf because most won't bother and this way we
+ * make the web a little safer. HTTP_MAX_HEADER_SIZE is still far bigger
+ * than any reasonable request or response so this should never affect
+ * day-to-day operation.
+ */
+ if (parser->nread > HTTP_MAX_HEADER_SIZE) {
+ SET_ERRNO(HPE_HEADER_OVERFLOW);
+ goto error;
+ }
+ }
+
+ reexecute_byte:
+ switch (parser->state) {
+
+ case s_dead:
+ /* this state is used after a 'Connection: close' message
+ * the parser will error out if it reads another message
+ */
+ if (ch == CR || ch == LF)
+ break;
+
+ SET_ERRNO(HPE_CLOSED_CONNECTION);
+ goto error;
+
+ case s_start_req_or_res:
+ {
+ if (ch == CR || ch == LF)
+ break;
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ if (ch == 'H') {
+ parser->state = s_res_or_resp_H;
+
+ CALLBACK_NOTIFY(message_begin);
+ } else {
+ parser->type = HTTP_REQUEST;
+ parser->state = s_start_req;
+ goto reexecute_byte;
+ }
+
+ break;
+ }
+
+ case s_res_or_resp_H:
+ if (ch == 'T') {
+ parser->type = HTTP_RESPONSE;
+ parser->state = s_res_HT;
+ } else {
+ if (ch != 'E') {
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ parser->type = HTTP_REQUEST;
+ parser->method = HTTP_HEAD;
+ parser->index = 2;
+ parser->state = s_req_method;
+ }
+ break;
+
+ case s_start_res:
+ {
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ switch (ch) {
+ case 'H':
+ parser->state = s_res_H;
+ break;
+
+ case CR:
+ case LF:
+ break;
+
+ default:
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ CALLBACK_NOTIFY(message_begin);
+ break;
+ }
+
+ case s_res_H:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_res_HT;
+ break;
+
+ case s_res_HT:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_res_HTT;
+ break;
+
+ case s_res_HTT:
+ STRICT_CHECK(ch != 'P');
+ parser->state = s_res_HTTP;
+ break;
+
+ case s_res_HTTP:
+ STRICT_CHECK(ch != '/');
+ parser->state = s_res_first_http_major;
+ break;
+
+ case s_res_first_http_major:
+ if (ch < '0' || ch > '9') {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major = ch - '0';
+ parser->state = s_res_http_major;
+ break;
+
+ /* major HTTP version or dot */
+ case s_res_http_major:
+ {
+ if (ch == '.') {
+ parser->state = s_res_first_http_minor;
+ break;
+ }
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major *= 10;
+ parser->http_major += ch - '0';
+
+ if (parser->http_major > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+
+ /* first digit of minor HTTP version */
+ case s_res_first_http_minor:
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor = ch - '0';
+ parser->state = s_res_http_minor;
+ break;
+
+ /* minor HTTP version or end of request line */
+ case s_res_http_minor:
+ {
+ if (ch == ' ') {
+ parser->state = s_res_first_status_code;
+ break;
+ }
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor *= 10;
+ parser->http_minor += ch - '0';
+
+ if (parser->http_minor > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_res_first_status_code:
+ {
+ if (!IS_NUM(ch)) {
+ if (ch == ' ') {
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+ parser->status_code = ch - '0';
+ parser->state = s_res_status_code;
+ break;
+ }
+
+ case s_res_status_code:
+ {
+ if (!IS_NUM(ch)) {
+ switch (ch) {
+ case ' ':
+ parser->state = s_res_status_start;
+ break;
+ case CR:
+ parser->state = s_res_line_almost_done;
+ break;
+ case LF:
+ parser->state = s_header_field_start;
+ break;
+ default:
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+ break;
+ }
+
+ parser->status_code *= 10;
+ parser->status_code += ch - '0';
+
+ if (parser->status_code > 999) {
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_res_status_start:
+ {
+ if (ch == CR) {
+ parser->state = s_res_line_almost_done;
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ break;
+ }
+
+ MARK(status);
+ parser->state = s_res_status;
+ parser->index = 0;
+ break;
+ }
+
+ case s_res_status:
+ if (ch == CR) {
+ parser->state = s_res_line_almost_done;
+ CALLBACK_DATA(status);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ CALLBACK_DATA(status);
+ break;
+ }
+
+ break;
+
+ case s_res_line_almost_done:
+ STRICT_CHECK(ch != LF);
+ parser->state = s_header_field_start;
+ break;
+
+ case s_start_req:
+ {
+ if (ch == CR || ch == LF)
+ break;
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ if (!IS_ALPHA(ch)) {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ parser->method = (enum http_method) 0;
+ parser->index = 1;
+ switch (ch) {
+ case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break;
+ case 'D': parser->method = HTTP_DELETE; break;
+ case 'G': parser->method = HTTP_GET; break;
+ case 'H': parser->method = HTTP_HEAD; break;
+ case 'L': parser->method = HTTP_LOCK; break;
+ case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH */ break;
+ case 'N': parser->method = HTTP_NOTIFY; break;
+ case 'O': parser->method = HTTP_OPTIONS; break;
+ case 'P': parser->method = HTTP_POST;
+ /* or PROPFIND|PROPPATCH|PUT|PATCH|PURGE */
+ break;
+ case 'R': parser->method = HTTP_REPORT; break;
+ case 'S': parser->method = HTTP_SUBSCRIBE; /* or SEARCH or SYMBOLS */ break;
+ case 'T': parser->method = HTTP_TRACE; break;
+ case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE */ break;
+ default:
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ parser->state = s_req_method;
+
+ CALLBACK_NOTIFY(message_begin);
+
+ break;
+ }
+
+ case s_req_method:
+ {
+ const char *matcher;
+ if (ch == '\0') {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ matcher = method_strings[parser->method];
+ if (ch == ' ' && matcher[parser->index] == '\0') {
+ if (parser->method != HTTP_SYMBOLS && parser->method != HTTP_CHECK) {
+ parser->state = s_req_spaces_before_url;
+ }
+ else {
+ parser->state = s_req_spamc_start;
+ }
+ } else if (ch == matcher[parser->index]) {
+ ; /* nada */
+ } else if (parser->method == HTTP_CONNECT) {
+ if (parser->index == 1 && ch == 'H') {
+ /* XXX: CHECKOUT has been removed */
+ parser->method = HTTP_CHECK;
+ } else if (parser->index == 2 && ch == 'P') {
+ parser->method = HTTP_COPY;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->method == HTTP_MKCOL) {
+ if (parser->index == 1 && ch == 'O') {
+ parser->method = HTTP_MOVE;
+ } else if (parser->index == 1 && ch == 'E') {
+ parser->method = HTTP_MERGE;
+ } else if (parser->index == 1 && ch == '-') {
+ parser->method = HTTP_MSEARCH;
+ } else if (parser->index == 2 && ch == 'A') {
+ parser->method = HTTP_MKACTIVITY;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->method == HTTP_SUBSCRIBE) {
+ if (parser->index == 1 && ch == 'E') {
+ parser->method = HTTP_SEARCH;
+ } else if (ch == 'Y') {
+ parser->method = HTTP_SYMBOLS;
+ }
+ else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->index == 1 && parser->method == HTTP_POST) {
+ if (ch == 'R') {
+ parser->method = HTTP_PROPFIND; /* or HTTP_PROPPATCH */
+ } else if (ch == 'U') {
+ parser->method = HTTP_PUT; /* or HTTP_PURGE */
+ } else if (ch == 'A') {
+ parser->method = HTTP_PATCH;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->index == 2) {
+ if (parser->method == HTTP_PUT) {
+ if (ch == 'R') {
+ parser->method = HTTP_PURGE;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->method == HTTP_UNLOCK) {
+ if (ch == 'S') {
+ parser->method = HTTP_UNSUBSCRIBE;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->index == 4 && parser->method == HTTP_PROPFIND && ch == 'P') {
+ parser->method = HTTP_PROPPATCH;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ ++parser->index;
+ break;
+ }
+
+ case s_req_spaces_before_url:
+ {
+ if (ch == ' ') break;
+
+ MARK(url);
+ if (parser->method == HTTP_CONNECT) {
+ parser->state = s_req_server_start;
+ }
+
+ parser->state = parse_url_char((enum state)parser->state, ch);
+ if (parser->state == s_dead) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_req_schema:
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ {
+ switch (ch) {
+ /* No whitespace allowed here */
+ case ' ':
+ case CR:
+ case LF:
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ default:
+ parser->state = parse_url_char((enum state)parser->state, ch);
+ if (parser->state == s_dead) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+ }
+
+ break;
+ }
+
+ case s_req_server:
+ case s_req_server_with_at:
+ case s_req_path:
+ case s_req_query_string_start:
+ case s_req_query_string:
+ case s_req_fragment_start:
+ case s_req_fragment:
+ {
+ switch (ch) {
+ case ' ':
+ parser->state = s_req_http_start;
+ CALLBACK_DATA(url);
+ break;
+ case CR:
+ case LF:
+ parser->http_major = 0;
+ parser->http_minor = 9;
+ parser->state = (ch == CR) ?
+ s_req_line_almost_done :
+ s_header_field_start;
+ CALLBACK_DATA(url);
+ break;
+ default:
+ parser->state = parse_url_char((enum state)parser->state, ch);
+ if (parser->state == s_dead) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+ }
+ break;
+ }
+
+ case s_req_http_start:
+ switch (ch) {
+ case 'H':
+ parser->state = s_req_http_H;
+ break;
+ case ' ':
+ break;
+ default:
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+ break;
+
+ case s_req_http_H:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_req_http_HT;
+ break;
+
+ case s_req_http_HT:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_req_http_HTT;
+ break;
+
+ case s_req_http_HTT:
+ STRICT_CHECK(ch != 'P');
+ parser->state = s_req_http_HTTP;
+ break;
+
+ case s_req_http_HTTP:
+ STRICT_CHECK(ch != '/');
+ parser->state = s_req_first_http_major;
+ break;
+
+ /* first digit of major HTTP version */
+ case s_req_first_http_major:
+ if (ch < '1' || ch > '9') {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major = ch - '0';
+ parser->state = s_req_http_major;
+ break;
+
+ /* major HTTP version or dot */
+ case s_req_http_major:
+ {
+ if (ch == '.') {
+ parser->state = s_req_first_http_minor;
+ break;
+ }
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major *= 10;
+ parser->http_major += ch - '0';
+
+ if (parser->http_major > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+
+ /* first digit of minor HTTP version */
+ case s_req_first_http_minor:
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor = ch - '0';
+ parser->state = s_req_http_minor;
+ break;
+
+ /* minor HTTP version or end of request line */
+ case s_req_http_minor:
+ {
+ if (ch == CR) {
+ parser->state = s_req_line_almost_done;
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ break;
+ }
+
+ /* XXX allow spaces after digit? */
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor *= 10;
+ parser->http_minor += ch - '0';
+
+ if (parser->http_minor > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+ case s_req_spamc_start: {
+ if (ch == 'S') {
+ parser->flags |= F_SPAMC;
+ parser->state = s_req_spamc;
+ }
+ else if (ch == 'R') {
+ parser->state = s_req_spamc;
+ }
+ else if (ch != ' ') {
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_req_spamc:
+ {
+ if (ch == CR) {
+ parser->state = s_req_line_almost_done;
+ }
+ else if (ch == LF) {
+ parser->state = s_header_field_start;
+ }
+ break;
+ }
+
+ /* end of request line */
+ case s_req_line_almost_done:
+ {
+ if (ch != LF) {
+ SET_ERRNO(HPE_LF_EXPECTED);
+ goto error;
+ }
+
+ parser->state = s_header_field_start;
+ break;
+ }
+
+ case s_header_field_start:
+ {
+ if (ch == CR) {
+ parser->state = s_headers_almost_done;
+ break;
+ }
+
+ if (ch == LF) {
+ /* they might be just sending \n instead of \r\n so this would be
+ * the second \n to denote the end of headers*/
+ parser->state = s_headers_almost_done;
+ goto reexecute_byte;
+ }
+
+ c = TOKEN(ch);
+
+ if (!c) {
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+
+ MARK(header_field);
+
+ parser->index = 0;
+ parser->state = s_header_field;
+
+ switch (c) {
+ case 'c':
+ parser->header_state = h_C;
+ break;
+
+ case 'p':
+ parser->header_state = h_matching_proxy_connection;
+ break;
+
+ case 't':
+ parser->header_state = h_matching_transfer_encoding;
+ break;
+
+ case 'u':
+ parser->header_state = h_matching_upgrade;
+ break;
+
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_field:
+ {
+ c = TOKEN(ch);
+
+ if (c) {
+ switch (parser->header_state) {
+ case h_general:
+ break;
+
+ case h_C:
+ parser->index++;
+ parser->header_state = (c == 'o' ? h_CO : h_general);
+ break;
+
+ case h_CO:
+ parser->index++;
+ parser->header_state = (c == 'n' ? h_CON : h_general);
+ break;
+
+ case h_CON:
+ parser->index++;
+ switch (c) {
+ case 'n':
+ parser->header_state = h_matching_connection;
+ break;
+ case 't':
+ parser->header_state = h_matching_content_length;
+ break;
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+
+ /* connection */
+
+ case h_matching_connection:
+ parser->index++;
+ if (parser->index > sizeof(CONNECTION)-1
+ || c != CONNECTION[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CONNECTION)-2) {
+ parser->header_state = h_connection;
+ }
+ break;
+
+ /* proxy-connection */
+
+ case h_matching_proxy_connection:
+ parser->index++;
+ if (parser->index > sizeof(PROXY_CONNECTION)-1
+ || c != PROXY_CONNECTION[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(PROXY_CONNECTION)-2) {
+ parser->header_state = h_connection;
+ }
+ break;
+
+ /* content-length */
+
+ case h_matching_content_length:
+ parser->index++;
+ if (parser->index > sizeof(CONTENT_LENGTH)-1
+ || c != CONTENT_LENGTH[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CONTENT_LENGTH)-2) {
+ parser->header_state = h_content_length;
+ }
+ break;
+
+ /* transfer-encoding */
+
+ case h_matching_transfer_encoding:
+ parser->index++;
+ if (parser->index > sizeof(TRANSFER_ENCODING)-1
+ || c != TRANSFER_ENCODING[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) {
+ parser->header_state = h_transfer_encoding;
+ }
+ break;
+
+ /* upgrade */
+
+ case h_matching_upgrade:
+ parser->index++;
+ if (parser->index > sizeof(UPGRADE)-1
+ || c != UPGRADE[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(UPGRADE)-2) {
+ parser->header_state = h_upgrade;
+ }
+ break;
+
+ case h_connection:
+ case h_content_length:
+ case h_transfer_encoding:
+ case h_upgrade:
+ if (ch != ' ') parser->header_state = h_general;
+ break;
+
+ default:
+ assert(0 && "Unknown header_state");
+ break;
+ }
+ break;
+ }
+
+ if (ch == ':') {
+ parser->state = s_header_value_start;
+ CALLBACK_DATA(header_field);
+ break;
+ }
+
+ if (ch == CR) {
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA(header_field);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ CALLBACK_DATA(header_field);
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+
+ case s_header_value_start:
+ {
+ if (ch == ' ' || ch == '\t') break;
+
+ MARK(header_value);
+
+ parser->state = s_header_value;
+ parser->index = 0;
+
+ if (ch == CR) {
+ parser->header_state = h_general;
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA(header_value);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ CALLBACK_DATA(header_value);
+ break;
+ }
+
+ c = LOWER(ch);
+
+ switch (parser->header_state) {
+ case h_upgrade:
+ parser->flags |= F_UPGRADE;
+ parser->header_state = h_general;
+ break;
+
+ case h_transfer_encoding:
+ /* looking for 'Transfer-Encoding: chunked' */
+ if ('c' == c) {
+ parser->header_state = h_matching_transfer_encoding_chunked;
+ } else {
+ parser->header_state = h_general;
+ }
+ break;
+
+ case h_content_length:
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->content_length = ch - '0';
+ break;
+
+ case h_connection:
+ /* looking for 'Connection: keep-alive' */
+ if (c == 'k') {
+ parser->header_state = h_matching_connection_keep_alive;
+ /* looking for 'Connection: close' */
+ } else if (c == 'c') {
+ parser->header_state = h_matching_connection_close;
+ } else {
+ parser->header_state = h_general;
+ }
+ break;
+
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_value:
+ {
+
+ if (ch == CR) {
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA(header_value);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA_NOADVANCE(header_value);
+ goto reexecute_byte;
+ }
+
+ c = LOWER(ch);
+
+ switch (parser->header_state) {
+ case h_general:
+ break;
+
+ case h_connection:
+ case h_transfer_encoding:
+ assert(0 && "Shouldn't get here.");
+ break;
+
+ case h_content_length:
+ {
+ uint64_t t;
+
+ if (ch == ' ') break;
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ t = parser->content_length;
+ t *= 10;
+ t += ch - '0';
+
+ /* Overflow? */
+ if (t < parser->content_length || t == ULLONG_MAX) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->content_length = t;
+ break;
+ }
+
+ /* Transfer-Encoding: chunked */
+ case h_matching_transfer_encoding_chunked:
+ parser->index++;
+ if (parser->index > sizeof(CHUNKED)-1
+ || c != CHUNKED[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CHUNKED)-2) {
+ parser->header_state = h_transfer_encoding_chunked;
+ }
+ break;
+
+ /* looking for 'Connection: keep-alive' */
+ case h_matching_connection_keep_alive:
+ parser->index++;
+ if (parser->index > sizeof(KEEP_ALIVE)-1
+ || c != KEEP_ALIVE[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(KEEP_ALIVE)-2) {
+ parser->header_state = h_connection_keep_alive;
+ }
+ break;
+
+ /* looking for 'Connection: close' */
+ case h_matching_connection_close:
+ parser->index++;
+ if (parser->index > sizeof(CLOSE)-1 || c != CLOSE[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CLOSE)-2) {
+ parser->header_state = h_connection_close;
+ }
+ break;
+
+ case h_transfer_encoding_chunked:
+ case h_connection_keep_alive:
+ case h_connection_close:
+ if (ch != ' ') parser->header_state = h_general;
+ break;
+
+ default:
+ parser->state = s_header_value;
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_almost_done:
+ {
+ STRICT_CHECK(ch != LF);
+
+ parser->state = s_header_value_lws;
+
+ switch (parser->header_state) {
+ case h_connection_keep_alive:
+ parser->flags |= F_CONNECTION_KEEP_ALIVE;
+ break;
+ case h_connection_close:
+ /* XXX: not needed for rspamd parser->flags |= F_CONNECTION_CLOSE; */
+ break;
+ case h_transfer_encoding_chunked:
+ parser->flags |= F_CHUNKED;
+ break;
+ default:
+ break;
+ }
+
+ break;
+ }
+
+ case s_header_value_lws:
+ {
+ if (ch == ' ' || ch == '\t')
+ parser->state = s_header_value_start;
+ else
+ {
+ parser->state = s_header_field_start;
+ goto reexecute_byte;
+ }
+ break;
+ }
+
+ case s_headers_almost_done:
+ {
+ STRICT_CHECK(ch != LF);
+
+ if (parser->flags & F_TRAILING) {
+ /* End of a chunked request */
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ break;
+ }
+
+ parser->state = s_headers_done;
+
+ /* Set this here so that on_headers_complete() callbacks can see it */
+ parser->upgrade =
+ (parser->flags & F_UPGRADE || parser->method == HTTP_CONNECT);
+
+ /* Here we call the headers_complete callback. This is somewhat
+ * different than other callbacks because if the user returns 1, we
+ * will interpret that as saying that this message has no body. This
+ * is needed for the annoying case of receiving a response to a HEAD
+ * request.
+ *
+ * We'd like to use CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so
+ * we have to simulate it by handling a change in errno below.
+ */
+ if (settings->on_headers_complete) {
+ switch (settings->on_headers_complete(parser)) {
+ case 0:
+ break;
+
+ case 1:
+ parser->flags |= F_SKIPBODY;
+ break;
+
+ default:
+ SET_ERRNO(HPE_CB_headers_complete);
+ return p - data; /* Error */
+ }
+ }
+
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
+ return p - data;
+ }
+
+ goto reexecute_byte;
+ }
+
+ case s_headers_done:
+ {
+ STRICT_CHECK(ch != LF);
+
+ parser->nread = 0;
+
+ /* Exit, the rest of the connect is in a different protocol. */
+ if (parser->upgrade) {
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ return (p - data) + 1;
+ }
+
+ if (parser->flags & F_SKIPBODY) {
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ } else if (parser->flags & F_CHUNKED) {
+ /* chunked encoding - ignore Content-Length header */
+ parser->state = s_chunk_size_start;
+ } else {
+ if (parser->content_length == 0) {
+ /* Content-Length header given but zero: Content-Length: 0\r\n */
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ } else if (parser->content_length != ULLONG_MAX) {
+ /* Content-Length header given and non-zero */
+ parser->state = s_body_identity;
+ } else {
+ if (parser->type == HTTP_REQUEST ||
+ !http_message_needs_eof(parser)) {
+ /* Assume content-length 0 - read the next */
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ } else {
+ /* Read body until EOF */
+ parser->state = s_body_identity_eof;
+ }
+ }
+ }
+
+ break;
+ }
+
+ case s_body_identity:
+ {
+ uint64_t to_read = MIN(parser->content_length,
+ (uint64_t) ((data + len) - p));
+
+ assert(parser->content_length != 0
+ && parser->content_length != ULLONG_MAX);
+
+ /* The difference between advancing content_length and p is because
+ * the latter will automatically advance on the next loop iteration.
+ * Further, if content_length ends up at 0, we want to see the last
+ * byte again for our message complete callback.
+ */
+ MARK(body);
+ parser->content_length -= to_read;
+ p += to_read - 1;
+
+ if (parser->content_length == 0) {
+ parser->state = s_message_done;
+
+ /* Mimic CALLBACK_DATA_NOADVANCE() but with one extra byte.
+ *
+ * The alternative to doing this is to wait for the next byte to
+ * trigger the data callback, just as in every other case. The
+ * problem with this is that this makes it difficult for the test
+ * harness to distinguish between complete-on-EOF and
+ * complete-on-length. It's not clear that this distinction is
+ * important for applications, but let's keep it for now.
+ */
+ CALLBACK_DATA_(body, p - body_mark + 1, p - data);
+ goto reexecute_byte;
+ }
+
+ break;
+ }
+
+ /* read until EOF */
+ case s_body_identity_eof:
+ MARK(body);
+ p = data + len - 1;
+
+ break;
+
+ case s_message_done:
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ break;
+
+ case s_chunk_size_start:
+ {
+ assert(parser->nread == 1);
+ assert(parser->flags & F_CHUNKED);
+
+ unhex_val = unhex[(unsigned char)ch];
+ if (unhex_val == -1) {
+ SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
+ goto error;
+ }
+
+ parser->content_length = unhex_val;
+ parser->state = s_chunk_size;
+ break;
+ }
+
+ case s_chunk_size:
+ {
+ uint64_t t;
+
+ assert(parser->flags & F_CHUNKED);
+
+ if (ch == CR) {
+ parser->state = s_chunk_size_almost_done;
+ break;
+ }
+
+ unhex_val = unhex[(unsigned char)ch];
+
+ if (unhex_val == -1) {
+ if (ch == ';' || ch == ' ') {
+ parser->state = s_chunk_parameters;
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
+ goto error;
+ }
+
+ t = parser->content_length;
+ t *= 16;
+ t += unhex_val;
+
+ /* Overflow? */
+ if (t < parser->content_length || t == ULLONG_MAX) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->content_length = t;
+ break;
+ }
+
+ case s_chunk_parameters:
+ {
+ assert(parser->flags & F_CHUNKED);
+ /* just ignore this shit. TODO check for overflow */
+ if (ch == CR) {
+ parser->state = s_chunk_size_almost_done;
+ break;
+ }
+ break;
+ }
+
+ case s_chunk_size_almost_done:
+ {
+ assert(parser->flags & F_CHUNKED);
+ STRICT_CHECK(ch != LF);
+
+ parser->nread = 0;
+
+ if (parser->content_length == 0) {
+ parser->flags |= F_TRAILING;
+ parser->state = s_header_field_start;
+ } else {
+ parser->state = s_chunk_data;
+ }
+ break;
+ }
+
+ case s_chunk_data:
+ {
+ uint64_t to_read = MIN(parser->content_length,
+ (uint64_t) ((data + len) - p));
+
+ assert(parser->flags & F_CHUNKED);
+ assert(parser->content_length != 0
+ && parser->content_length != ULLONG_MAX);
+
+ /* See the explanation in s_body_identity for why the content
+ * length and data pointers are managed this way.
+ */
+ MARK(body);
+ parser->content_length -= to_read;
+ p += to_read - 1;
+
+ if (parser->content_length == 0) {
+ parser->state = s_chunk_data_almost_done;
+ }
+
+ break;
+ }
+
+ case s_chunk_data_almost_done:
+ assert(parser->flags & F_CHUNKED);
+ assert(parser->content_length == 0);
+ STRICT_CHECK(ch != CR);
+ parser->state = s_chunk_data_done;
+ CALLBACK_DATA(body);
+ break;
+
+ case s_chunk_data_done:
+ assert(parser->flags & F_CHUNKED);
+ STRICT_CHECK(ch != LF);
+ parser->nread = 0;
+ parser->state = s_chunk_size_start;
+ break;
+
+ default:
+ assert(0 && "unhandled state");
+ SET_ERRNO(HPE_INVALID_INTERNAL_STATE);
+ goto error;
+ }
+ }
+
+ /* Run callbacks for any marks that we have leftover after we ran our of
+ * bytes. There should be at most one of these set, so it's OK to invoke
+ * them in series (unset marks will not result in callbacks).
+ *
+ * We use the NOADVANCE() variety of callbacks here because 'p' has already
+ * overflowed 'data' and this allows us to correct for the off-by-one that
+ * we'd otherwise have (since CALLBACK_DATA() is meant to be run with a 'p'
+ * value that's in-bounds).
+ */
+
+ assert(((header_field_mark ? 1 : 0) +
+ (header_value_mark ? 1 : 0) +
+ (url_mark ? 1 : 0) +
+ (body_mark ? 1 : 0) +
+ (status_mark ? 1 : 0)) <= 1);
+
+ CALLBACK_DATA_NOADVANCE(header_field);
+ CALLBACK_DATA_NOADVANCE(header_value);
+ CALLBACK_DATA_NOADVANCE(url);
+ CALLBACK_DATA_NOADVANCE(body);
+ CALLBACK_DATA_NOADVANCE(status);
+
+ return len;
+
+error:
+ if (HTTP_PARSER_ERRNO(parser) == HPE_OK) {
+ SET_ERRNO(HPE_UNKNOWN);
+ }
+
+ return (p - data);
+}
+
+
+/* Does the parser need to see an EOF to find the end of the message? */
+int
+http_message_needs_eof (const http_parser *parser)
+{
+ if (parser->type == HTTP_REQUEST) {
+ return 0;
+ }
+
+ /* See RFC 2616 section 4.4 */
+ if (parser->status_code / 100 == 1 || /* 1xx e.g. Continue */
+ parser->status_code == 204 || /* No Content */
+ parser->status_code == 304 || /* Not Modified */
+ parser->flags & F_SKIPBODY) { /* response to a HEAD request */
+ return 0;
+ }
+
+ if ((parser->flags & F_CHUNKED) || parser->content_length != ULLONG_MAX) {
+ return 0;
+ }
+
+ return 1;
+}
+
+
+int
+http_should_keep_alive (const http_parser *parser)
+{
+ if (parser->http_major > 0 && parser->http_minor > 0) {
+ /* HTTP/1.1 */
+ if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) {
+ return 0;
+ }
+ } else {
+ /* HTTP/1.0 or earlier */
+ if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) {
+ return 0;
+ }
+ }
+
+ return !http_message_needs_eof(parser);
+}
+
+
+const char *
+http_method_str (enum http_method m)
+{
+ return ELEM_AT(method_strings, m, "<unknown>");
+}
+
+
+void
+http_parser_init (http_parser *parser, int t)
+{
+ void *data = parser->data; /* preserve application data */
+ memset(parser, 0, sizeof(*parser));
+ parser->data = data;
+ parser->type = t;
+ parser->state = (t == HTTP_REQUEST ? s_start_req : (t == HTTP_RESPONSE ? s_start_res : s_start_req_or_res));
+ parser->http_errno = HPE_OK;
+}
+
+const char *
+http_errno_name(enum http_errno err) {
+ assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
+ return http_strerror_tab[err].name;
+}
+
+const char *
+http_errno_description(enum http_errno err) {
+ assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
+ return http_strerror_tab[err].description;
+}
+
+static enum http_host_state
+http_parse_host_char(enum http_host_state s, const char ch) {
+ switch(s) {
+ case s_http_userinfo:
+ case s_http_userinfo_start:
+ if (ch == '@') {
+ return s_http_host_start;
+ }
+
+ if (IS_USERINFO_CHAR(ch)) {
+ return s_http_userinfo;
+ }
+ break;
+
+ case s_http_host_start:
+ if (ch == '[') {
+ return s_http_host_v6_start;
+ }
+
+ if (IS_HOST_CHAR(ch)) {
+ return s_http_host;
+ }
+
+ break;
+
+ case s_http_host:
+ if (IS_HOST_CHAR(ch)) {
+ return s_http_host;
+ }
+
+ /* FALLTHROUGH */
+ case s_http_host_v6_end:
+ if (ch == ':') {
+ return s_http_host_port_start;
+ }
+
+ break;
+
+ case s_http_host_v6:
+ if (ch == ']') {
+ return s_http_host_v6_end;
+ }
+
+ /* FALLTHROUGH */
+ case s_http_host_v6_start:
+ if (IS_HEX(ch) || ch == ':' || ch == '.') {
+ return s_http_host_v6;
+ }
+
+ break;
+
+ case s_http_host_port:
+ case s_http_host_port_start:
+ if (IS_NUM(ch)) {
+ return s_http_host_port;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+ return s_http_host_dead;
+}
+
+static int
+http_parse_host(const char * buf, struct http_parser_url *u, int found_at) {
+ enum http_host_state s;
+
+ const char *p;
+ size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len;
+
+ u->field_data[UF_HOST].len = 0;
+
+ s = found_at ? s_http_userinfo_start : s_http_host_start;
+
+ for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) {
+ enum http_host_state new_s = http_parse_host_char(s, *p);
+
+ if (new_s == s_http_host_dead) {
+ return 1;
+ }
+
+ switch(new_s) {
+ case s_http_host:
+ if (s != s_http_host) {
+ u->field_data[UF_HOST].off = p - buf;
+ }
+ u->field_data[UF_HOST].len++;
+ break;
+
+ case s_http_host_v6:
+ if (s != s_http_host_v6) {
+ u->field_data[UF_HOST].off = p - buf;
+ }
+ u->field_data[UF_HOST].len++;
+ break;
+
+ case s_http_host_port:
+ if (s != s_http_host_port) {
+ u->field_data[UF_PORT].off = p - buf;
+ u->field_data[UF_PORT].len = 0;
+ u->field_set |= (1 << UF_PORT);
+ }
+ u->field_data[UF_PORT].len++;
+ break;
+
+ case s_http_userinfo:
+ if (s != s_http_userinfo) {
+ u->field_data[UF_USERINFO].off = p - buf ;
+ u->field_data[UF_USERINFO].len = 0;
+ u->field_set |= (1 << UF_USERINFO);
+ }
+ u->field_data[UF_USERINFO].len++;
+ break;
+
+ default:
+ break;
+ }
+ s = new_s;
+ }
+
+ /* Make sure we don't end somewhere unexpected */
+ switch (s) {
+ case s_http_host_start:
+ case s_http_host_v6_start:
+ case s_http_host_v6:
+ case s_http_host_port_start:
+ case s_http_userinfo:
+ case s_http_userinfo_start:
+ return 1;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+int
+http_parser_parse_url(const char *buf, size_t buflen, int is_connect,
+ struct http_parser_url *u)
+{
+ enum state s;
+ const char *p;
+ enum http_parser_url_fields uf, old_uf;
+ int found_at = 0;
+
+ u->port = u->field_set = 0;
+ s = is_connect ? s_req_server_start : s_req_spaces_before_url;
+ uf = old_uf = UF_MAX;
+
+ for (p = buf; p < buf + buflen; p++) {
+ s = parse_url_char(s, *p);
+
+ /* Figure out the next field that we're operating on */
+ switch (s) {
+ case s_dead:
+ return 1;
+
+ /* Skip delimiters */
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ case s_req_query_string_start:
+ case s_req_fragment_start:
+ continue;
+
+ case s_req_schema:
+ uf = UF_SCHEMA;
+ break;
+
+ case s_req_server_with_at:
+ found_at = 1;
+
+ /* FALLTROUGH */
+ case s_req_server:
+ uf = UF_HOST;
+ break;
+
+ case s_req_path:
+ uf = UF_PATH;
+ break;
+
+ case s_req_query_string:
+ uf = UF_QUERY;
+ break;
+
+ case s_req_fragment:
+ uf = UF_FRAGMENT;
+ break;
+
+ default:
+ assert(!"Unexpected state");
+ return 1;
+ }
+
+ /* Nothing's changed; soldier on */
+ if (uf == old_uf) {
+ u->field_data[uf].len++;
+ continue;
+ }
+
+ u->field_data[uf].off = p - buf;
+ u->field_data[uf].len = 1;
+
+ u->field_set |= (1 << uf);
+ old_uf = uf;
+ }
+
+ /* host must be present if there is a schema */
+ /* parsing http:///toto will fail */
+ if ((u->field_set & ((1 << UF_SCHEMA) | (1 << UF_HOST))) != 0) {
+ if (http_parse_host(buf, u, found_at) != 0) {
+ return 1;
+ }
+ }
+
+ /* CONNECT requests can only contain "hostname:port" */
+ if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) {
+ return 1;
+ }
+
+ if (u->field_set & (1 << UF_PORT)) {
+ /* Don't bother with endp; we've already validated the string */
+ unsigned long v = strtoul(buf + u->field_data[UF_PORT].off, NULL, 10);
+
+ /* Ports have a max value of 2^16 */
+ if (v > 0xffff) {
+ return 1;
+ }
+
+ u->port = (uint16_t) v;
+ }
+
+ return 0;
+}
+
+void
+http_parser_pause(http_parser *parser, int paused) {
+ /* Users should only be pausing/unpausing a parser that is not in an error
+ * state. In non-debug builds, there's not much that we can do about this
+ * other than ignore it.
+ */
+ if (HTTP_PARSER_ERRNO(parser) == HPE_OK ||
+ HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) {
+ SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK);
+ } else {
+ assert(0 && "Attempting to pause parser in error state");
+ }
+}
+
+int
+http_body_is_final(const struct http_parser *parser) {
+ return parser->state == s_message_done;
+}
+
+unsigned long
+http_parser_version(void) {
+ return HTTP_PARSER_VERSION_MAJOR * 0x10000 |
+ HTTP_PARSER_VERSION_MINOR * 0x00100 |
+ HTTP_PARSER_VERSION_PATCH * 0x00001;
+}
diff --git a/contrib/http-parser/http_parser.h b/contrib/http-parser/http_parser.h
new file mode 100644
index 0000000..7b4ac49
--- /dev/null
+++ b/contrib/http-parser/http_parser.h
@@ -0,0 +1,321 @@
+/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifndef http_parser_h
+#define http_parser_h
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Also update SONAME in the Makefile whenever you change these. */
+#define HTTP_PARSER_VERSION_MAJOR 2
+#define HTTP_PARSER_VERSION_MINOR 2
+#define HTTP_PARSER_VERSION_PATCH 0
+
+#include <sys/types.h>
+#if defined(_WIN32) && !defined(__MINGW32__) && (!defined(_MSC_VER) || _MSC_VER<1600)
+#include <BaseTsd.h>
+#include <stddef.h>
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run
+ * faster
+ */
+#ifndef HTTP_PARSER_STRICT
+# define HTTP_PARSER_STRICT 1
+#endif
+
+/* Maximium header size allowed */
+#define HTTP_MAX_HEADER_SIZE (80*1024)
+
+
+typedef struct http_parser http_parser;
+typedef struct http_parser_settings http_parser_settings;
+
+
+/* Callbacks should return non-zero to indicate an error. The parser will
+ * then halt execution.
+ *
+ * The one exception is on_headers_complete. In a HTTP_RESPONSE parser
+ * returning '1' from on_headers_complete will tell the parser that it
+ * should not expect a body. This is used when receiving a response to a
+ * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding:
+ * chunked' headers that indicate the presence of a body.
+ *
+ * http_data_cb does not return data chunks. It will be call arbitrarally
+ * many times for each string. E.G. you might get 10 callbacks for "on_url"
+ * each providing just a few characters more data.
+ */
+typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
+typedef int (*http_cb) (http_parser*);
+
+
+/* Request Methods */
+#define HTTP_METHOD_MAP(XX) \
+ XX(0, DELETE, DELETE) \
+ XX(1, GET, GET) \
+ XX(2, HEAD, HEAD) \
+ XX(3, POST, POST) \
+ XX(4, PUT, PUT) \
+ /* pathological */ \
+ XX(5, CONNECT, CONNECT) \
+ XX(6, OPTIONS, OPTIONS) \
+ XX(7, TRACE, TRACE) \
+ /* webdav */ \
+ XX(8, COPY, COPY) \
+ XX(9, LOCK, LOCK) \
+ XX(10, MKCOL, MKCOL) \
+ XX(11, MOVE, MOVE) \
+ XX(12, PROPFIND, PROPFIND) \
+ XX(13, PROPPATCH, PROPPATCH) \
+ XX(14, SEARCH, SEARCH) \
+ XX(15, UNLOCK, UNLOCK) \
+ /* subversion */ \
+ XX(16, REPORT, REPORT) \
+ XX(17, MKACTIVITY, MKACTIVITY) \
+ XX(18, CHECKOUT, CHECKOUT) \
+ XX(19, MERGE, MERGE) \
+ /* upnp */ \
+ XX(20, MSEARCH, M-SEARCH) \
+ XX(21, NOTIFY, NOTIFY) \
+ XX(22, SUBSCRIBE, SUBSCRIBE) \
+ XX(23, UNSUBSCRIBE, UNSUBSCRIBE) \
+ /* RFC-5789 */ \
+ XX(24, PATCH, PATCH) \
+ XX(25, PURGE, PURGE) \
+ /* SPAMC compatibility */ \
+ XX(26, SYMBOLS, SYMBOLS) \
+ XX(27, CHECK, CHECK) \
+ XX(28, METHOD_MAX, METHOD_MAX) \
+ XX(-1, INVALID, INVALID) \
+
+enum http_method
+ {
+#define XX(num, name, string) HTTP_##name = num,
+ HTTP_METHOD_MAP(XX)
+#undef XX
+ };
+
+
+
+/* Flag values for http_parser.flags field */
+enum flags
+ { F_CHUNKED = 1 << 0
+ , F_CONNECTION_KEEP_ALIVE = 1 << 1
+ , F_SPAMC = 1 << 2
+ , F_TRAILING = 1 << 3
+ , F_UPGRADE = 1 << 4
+ , F_SKIPBODY = 1 << 5
+ };
+
+
+/* Map for errno-related constants
+ *
+ * The provided argument should be a macro that takes 2 arguments.
+ */
+#define HTTP_ERRNO_MAP(XX) \
+ /* No error */ \
+ XX(OK, "success") \
+ \
+ /* Callback-related errors */ \
+ XX(CB_message_begin, "the on_message_begin callback failed") \
+ XX(CB_url, "the on_url callback failed") \
+ XX(CB_header_field, "the on_header_field callback failed") \
+ XX(CB_header_value, "the on_header_value callback failed") \
+ XX(CB_headers_complete, "the on_headers_complete callback failed") \
+ XX(CB_body, "the on_body callback failed") \
+ XX(CB_message_complete, "the on_message_complete callback failed") \
+ XX(CB_status, "the on_status callback failed") \
+ \
+ /* Parsing-related errors */ \
+ XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \
+ XX(HEADER_OVERFLOW, \
+ "too many header bytes seen; overflow detected") \
+ XX(CLOSED_CONNECTION, \
+ "data received after completed connection: close message") \
+ XX(INVALID_VERSION, "invalid HTTP version") \
+ XX(INVALID_STATUS, "invalid HTTP status code") \
+ XX(INVALID_METHOD, "invalid HTTP method") \
+ XX(INVALID_URL, "invalid URL") \
+ XX(INVALID_HOST, "invalid host") \
+ XX(INVALID_PORT, "invalid port") \
+ XX(INVALID_PATH, "invalid path") \
+ XX(INVALID_QUERY_STRING, "invalid query string") \
+ XX(INVALID_FRAGMENT, "invalid fragment") \
+ XX(LF_EXPECTED, "LF character expected") \
+ XX(INVALID_HEADER_TOKEN, "invalid character in header") \
+ XX(INVALID_CONTENT_LENGTH, \
+ "invalid character in content-length header") \
+ XX(INVALID_CHUNK_SIZE, \
+ "invalid character in chunk size header") \
+ XX(INVALID_CONSTANT, "invalid constant string") \
+ XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
+ XX(STRICT, "strict mode assertion failed") \
+ XX(PAUSED, "parser is paused") \
+ XX(UNKNOWN, "an unknown error occurred")
+
+
+/* Define HPE_* values for each errno value above */
+#define HTTP_ERRNO_GEN(n, s) HPE_##n,
+enum http_errno {
+ HTTP_ERRNO_MAP(HTTP_ERRNO_GEN)
+};
+#undef HTTP_ERRNO_GEN
+
+
+/* Get an http_errno value from an http_parser */
+#define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno)
+
+
+struct http_parser {
+ /** PRIVATE **/
+ unsigned int type : 2; /* enum http_parser_type */
+ unsigned int flags : 6; /* F_* values from 'flags' enum; semi-public */
+ unsigned int state : 8; /* enum state from http_parser.c */
+ unsigned int header_state : 8; /* enum header_state from http_parser.c */
+ unsigned int index : 8; /* index into current matcher */
+
+ uint32_t nread; /* # bytes read in various scenarios */
+ uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */
+
+ /** READ-ONLY **/
+ unsigned short http_major;
+ unsigned short http_minor;
+ unsigned int status_code : 16; /* responses only */
+ unsigned int method : 8; /* requests only */
+ unsigned int http_errno : 7;
+
+ /* 1 = Upgrade header was present and the parser has exited because of that.
+ * 0 = No upgrade header present.
+ * Should be checked when http_parser_execute() returns in addition to
+ * error checking.
+ */
+ unsigned int upgrade : 1;
+
+ /** PUBLIC **/
+ void *data; /* A pointer to get hook to the "connection" or "socket" object */
+};
+
+
+struct http_parser_settings {
+ http_cb on_message_begin;
+ http_data_cb on_url;
+ http_data_cb on_status;
+ http_data_cb on_header_field;
+ http_data_cb on_header_value;
+ http_cb on_headers_complete;
+ http_data_cb on_body;
+ http_cb on_message_complete;
+};
+
+
+enum http_parser_url_fields
+ { UF_SCHEMA = 0
+ , UF_HOST = 1
+ , UF_PORT = 2
+ , UF_PATH = 3
+ , UF_QUERY = 4
+ , UF_FRAGMENT = 5
+ , UF_USERINFO = 6
+ , UF_MAX = 7
+ };
+
+
+/* Result structure for http_parser_parse_url().
+ *
+ * Callers should index into field_data[] with UF_* values iff field_set
+ * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and
+ * because we probably have padding left over), we convert any port to
+ * a uint16_t.
+ */
+struct http_parser_url {
+ uint16_t field_set; /* Bitmask of (1 << UF_*) values */
+ uint16_t port; /* Converted UF_PORT string */
+
+ struct {
+ uint16_t off; /* Offset into buffer in which field starts */
+ uint16_t len; /* Length of run in buffer */
+ } field_data[UF_MAX];
+};
+
+
+/* Returns the library version. Bits 16-23 contain the major version number,
+ * bits 8-15 the minor version number and bits 0-7 the patch level.
+ * Usage example:
+ *
+ * unsigned long version = http_parser_version();
+ * unsigned major = (version >> 16) & 255;
+ * unsigned minor = (version >> 8) & 255;
+ * unsigned patch = version & 255;
+ * printf("http_parser v%u.%u.%u\n", major, minor, version);
+ */
+unsigned long http_parser_version(void);
+
+void http_parser_init(http_parser *parser, int type);
+
+
+size_t http_parser_execute(http_parser *parser,
+ const http_parser_settings *settings,
+ const char *data,
+ size_t len);
+
+
+/* If http_should_keep_alive() in the on_headers_complete or
+ * on_message_complete callback returns 0, then this should be
+ * the last message on the connection.
+ * If you are the server, respond with the "Connection: close" header.
+ * If you are the client, close the connection.
+ */
+int http_should_keep_alive(const http_parser *parser);
+
+/* Returns a string version of the HTTP method. */
+const char *http_method_str(enum http_method m);
+
+/* Return a string name of the given error */
+const char *http_errno_name(enum http_errno err);
+
+/* Return a string description of the given error */
+const char *http_errno_description(enum http_errno err);
+
+/* Parse a URL; return nonzero on failure */
+int http_parser_parse_url(const char *buf, size_t buflen,
+ int is_connect,
+ struct http_parser_url *u);
+
+/* Pause or un-pause the parser; a nonzero value pauses */
+void http_parser_pause(http_parser *parser, int paused);
+
+/* Checks if this is the final chunk of the body. */
+int http_body_is_final(const http_parser *parser);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/contrib/kann/CMakeLists.txt b/contrib/kann/CMakeLists.txt
new file mode 100644
index 0000000..5f1b17a
--- /dev/null
+++ b/contrib/kann/CMakeLists.txt
@@ -0,0 +1,16 @@
+SET(LIBKANNSRC kautodiff.c kann.c)
+
+IF(ENABLE_STATIC MATCHES "ON")
+ ADD_LIBRARY(rspamd-kann STATIC ${LIBKANNSRC})
+ELSE()
+ ADD_LIBRARY(rspamd-kann SHARED ${LIBKANNSRC})
+ENDIF()
+
+target_link_libraries(rspamd-kann "${RSPAMD_REQUIRED_LIBRARIES}")
+target_link_libraries(rspamd-kann "m")
+IF(WITH_BLAS)
+ MESSAGE(STATUS "Use openblas to accelerate kann")
+ TARGET_LINK_LIBRARIES(rspamd-kann ${BLAS_REQUIRED_LIBRARIES})
+ENDIF(WITH_BLAS)
+
+INSTALL(TARGETS rspamd-kann LIBRARY DESTINATION ${RSPAMD_LIBDIR}) \ No newline at end of file
diff --git a/contrib/kann/LICENSE.txt b/contrib/kann/LICENSE.txt
new file mode 100644
index 0000000..8b2cf11
--- /dev/null
+++ b/contrib/kann/LICENSE.txt
@@ -0,0 +1,24 @@
+The MIT License
+
+Copyright (c) 2018-2019 Dana-Farber Cancer Institute
+ 2016-2018 Broad Institute
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/kann/kann.c b/contrib/kann/kann.c
new file mode 100644
index 0000000..70d1f02
--- /dev/null
+++ b/contrib/kann/kann.c
@@ -0,0 +1,992 @@
+#include "config.h"
+
+#include <math.h>
+#include <float.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include "kann.h"
+
+int kann_verbose = 3;
+
+/******************************************
+ *** @@BASIC: fundamental KANN routines ***
+ ******************************************/
+
+static void kad_ext_collate(int n, kad_node_t **a, float **_x, float **_g, float **_c)
+{
+ int i, j, k, l, n_var;
+ float *x, *g, *c;
+ n_var = kad_size_var(n, a);
+ x = *_x = (float*)realloc(*_x, n_var * sizeof(float));
+ g = *_g = (float*)realloc(*_g, n_var * sizeof(float));
+ c = *_c = (float*)realloc(*_c, kad_size_const(n, a) * sizeof(float));
+ memset(g, 0, n_var * sizeof(float));
+ for (i = j = k = 0; i < n; ++i) {
+ kad_node_t *v = a[i];
+ if (kad_is_var(v)) {
+ l = kad_len(v);
+ memcpy(&x[j], v->x, l * sizeof(float));
+ free(v->x);
+ v->x = &x[j];
+ v->g = &g[j];
+ j += l;
+ } else if (kad_is_const(v)) {
+ l = kad_len(v);
+ memcpy(&c[k], v->x, l * sizeof(float));
+ free(v->x);
+ v->x = &c[k];
+ k += l;
+ }
+ }
+}
+
+static void kad_ext_sync(int n, kad_node_t **a, float *x, float *g, float *c)
+{
+ int i, j, k;
+ for (i = j = k = 0; i < n; ++i) {
+ kad_node_t *v = a[i];
+ if (kad_is_var(v)) {
+ v->x = &x[j];
+ v->g = &g[j];
+ j += kad_len(v);
+ } else if (kad_is_const(v)) {
+ v->x = &c[k];
+ k += kad_len(v);
+ }
+ }
+}
+
+kann_t *kann_new(kad_node_t *cost, int n_rest, ...)
+{
+ kann_t *a;
+ int i, n_roots = 1 + n_rest, has_pivot = 0, has_recur = 0;
+ kad_node_t **roots;
+ va_list ap;
+
+ if (cost->n_d != 0) return 0;
+
+ va_start(ap, n_rest);
+ roots = (kad_node_t**)malloc((n_roots + 1) * sizeof(kad_node_t*));
+ for (i = 0; i < n_rest; ++i)
+ roots[i] = va_arg(ap, kad_node_t*);
+ roots[i++] = cost;
+ va_end(ap);
+
+ cost->ext_flag |= KANN_F_COST;
+ a = (kann_t*)calloc(1, sizeof(kann_t));
+ a->v = kad_compile_array(&a->n, n_roots, roots);
+
+ for (i = 0; i < a->n; ++i) {
+ if (a->v[i]->pre) has_recur = 1;
+ if (kad_is_pivot(a->v[i])) has_pivot = 1;
+ }
+ if (has_recur && !has_pivot) { /* an RNN that doesn't have a pivot; then add a pivot on top of cost and recompile */
+ cost->ext_flag &= ~KANN_F_COST;
+ roots[n_roots-1] = cost = kad_avg(1, &cost), cost->ext_flag |= KANN_F_COST;
+ free(a->v);
+ a->v = kad_compile_array(&a->n, n_roots, roots);
+ }
+ kad_ext_collate(a->n, a->v, &a->x, &a->g, &a->c);
+ free(roots);
+ return a;
+}
+
+kann_t *kann_clone(kann_t *a, int batch_size)
+{
+ kann_t *b;
+ b = (kann_t*)calloc(1, sizeof(kann_t));
+ b->n = a->n;
+ b->v = kad_clone(a->n, a->v, batch_size);
+ kad_ext_collate(b->n, b->v, &b->x, &b->g, &b->c);
+ return b;
+}
+
+kann_t *kann_unroll_array(kann_t *a, int *len)
+{
+ kann_t *b;
+ b = (kann_t*)calloc(1, sizeof(kann_t));
+ b->x = a->x, b->g = a->g, b->c = a->c; /* these arrays are shared */
+ b->v = kad_unroll(a->n, a->v, &b->n, len);
+ return b;
+}
+
+kann_t *kann_unroll(kann_t *a, ...)
+{
+ kann_t *b;
+ va_list ap;
+ int i, n_pivots, *len;
+ n_pivots = kad_n_pivots(a->n, a->v);
+ len = (int*)calloc(n_pivots, sizeof(int));
+ va_start(ap, a);
+ for (i = 0; i < n_pivots; ++i) len[i] = va_arg(ap, int);
+ va_end(ap);
+ b = kann_unroll_array(a, len);
+ free(len);
+ return b;
+}
+
+void kann_delete_unrolled(kann_t *a)
+{
+ if (a && a->mt) kann_mt(a, 0, 0);
+ if (a && a->v) kad_delete(a->n, a->v);
+ free(a);
+}
+
+void kann_delete(kann_t *a)
+{
+ if (a == 0) return;
+ free(a->x); free(a->g); free(a->c);
+ kann_delete_unrolled(a);
+}
+
+static void kann_switch_core(kann_t *a, int is_train)
+{
+ int i;
+ for (i = 0; i < a->n; ++i)
+ if (a->v[i]->op == 12 && a->v[i]->n_child == 2)
+ *(int32_t*)a->v[i]->ptr = !!is_train;
+}
+
+#define chk_flg(flag, mask) ((mask) == 0 || ((flag) & (mask)))
+#define chk_lbl(label, query) ((query) == 0 || (label) == (query))
+
+int kann_find(const kann_t *a, uint32_t ext_flag, int32_t ext_label)
+{
+ int i, k, r = -1;
+ for (i = k = 0; i < a->n; ++i)
+ if (chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ ++k, r = i;
+ return k == 1? r : k == 0? -1 : -2;
+}
+
+int kann_feed_bind(kann_t *a, uint32_t ext_flag, int32_t ext_label, float **x)
+{
+ int i, k;
+ if (x == 0) return 0;
+ for (i = k = 0; i < a->n; ++i)
+ if (kad_is_feed(a->v[i]) && chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ a->v[i]->x = x[k++];
+ return k;
+}
+
+int kann_feed_dim(const kann_t *a, uint32_t ext_flag, int32_t ext_label)
+{
+ int i, k, n = 0;
+ for (i = k = 0; i < a->n; ++i)
+ if (kad_is_feed(a->v[i]) && chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ ++k, n = a->v[i]->n_d > 1? kad_len(a->v[i]) / a->v[i]->d[0] : a->v[i]->n_d == 1? a->v[i]->d[0] : 1;
+ return k == 1? n : k == 0? -1 : -2;
+}
+
+static float kann_cost_core(kann_t *a, int cost_label, int cal_grad)
+{
+ int i_cost;
+ float cost;
+ i_cost = kann_find(a, KANN_F_COST, cost_label);
+ assert(i_cost >= 0);
+ cost = *kad_eval_at(a->n, a->v, i_cost);
+ if (cal_grad) kad_grad(a->n, a->v, i_cost);
+ return cost;
+}
+
+int kann_eval(kann_t *a, uint32_t ext_flag, int ext_label)
+{
+ int i, k;
+ for (i = k = 0; i < a->n; ++i)
+ if (chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ ++k, a->v[i]->tmp = 1;
+ kad_eval_marked(a->n, a->v);
+ return k;
+}
+
+void kann_rnn_start(kann_t *a)
+{
+ int i;
+ kann_set_batch_size(a, 1);
+ for (i = 0; i < a->n; ++i) {
+ kad_node_t *p = a->v[i];
+ if (p->pre) { /* NB: BE CAREFUL of the interaction between kann_rnn_start() and kann_set_batch_size() */
+ kad_node_t *q = p->pre;
+ if (q->x) memcpy(p->x, q->x, kad_len(p) * sizeof(float));
+ else memset(p->x, 0, kad_len(p) * sizeof(float));
+ if (q->n_child > 0) free(q->x);
+ q->x = p->x;
+ }
+ }
+}
+
+void kann_rnn_end(kann_t *a)
+{
+ int i;
+ kad_ext_sync(a->n, a->v, a->x, a->g, a->c);
+ for (i = 0; i < a->n; ++i)
+ if (a->v[i]->pre && a->v[i]->pre->n_child > 0)
+ a->v[i]->pre->x = (float*)calloc(kad_len(a->v[i]->pre), sizeof(float));
+}
+
+static int kann_class_error_core(const kann_t *ann, int *base)
+{
+ int i, j, k, m, n, off, n_err = 0;
+ for (i = 0, *base = 0; i < ann->n; ++i) {
+ kad_node_t *p = ann->v[i];
+ if (((p->op == 13 && (p->n_child == 2 || p->n_child == 3)) || (p->op == 22 && p->n_child == 2)) && p->n_d == 0) { /* ce_bin or ce_multi */
+ kad_node_t *x = p->child[0], *t = p->child[1];
+ n = t->d[t->n_d - 1], m = kad_len(t) / n;
+ for (j = off = 0; j < m; ++j, off += n) {
+ float t_sum = 0.0f, t_min = 1.0f, t_max = 0.0f, x_max = 0.0f, x_min = 1.0f;
+ int x_max_k = -1, t_max_k = -1;
+ for (k = 0; k < n; ++k) {
+ float xk = x->x[off+k], tk = t->x[off+k];
+ t_sum += tk;
+ t_min = t_min < tk? t_min : tk;
+ x_min = x_min < xk? x_min : xk;
+ if (t_max < tk) t_max = tk, t_max_k = k;
+ if (x_max < xk) x_max = xk, x_max_k = k;
+ }
+ if (t_sum - 1.0f == 0 && t_min >= 0.0f && x_min >= 0.0f && x_max <= 1.0f) {
+ ++(*base);
+ n_err += (x_max_k != t_max_k);
+ }
+ }
+ }
+ }
+ return n_err;
+}
+
+/*************************
+ * @@MT: multi-threading *
+ *************************/
+
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+
+struct mtaux_t;
+
+typedef struct { /* per-worker data */
+ kann_t *a;
+ float cost;
+ int action;
+ pthread_t tid;
+ struct mtaux_t *g;
+} mtaux1_t;
+
+typedef struct mtaux_t { /* cross-worker data */
+ int n_threads, max_batch_size;
+ int cal_grad, cost_label, eval_out;
+ volatile int n_idle; /* we will be busy waiting on this, so volatile necessary */
+ pthread_mutex_t mtx;
+ pthread_cond_t cv;
+ mtaux1_t *mt;
+} mtaux_t;
+
+static void *mt_worker(void *data) /* pthread worker */
+{
+ mtaux1_t *mt1 = (mtaux1_t*)data;
+ mtaux_t *mt = mt1->g;
+ for (;;) {
+ int action;
+ pthread_mutex_lock(&mt->mtx);
+ mt1->action = 0;
+ ++mt->n_idle;
+ while (mt1->action == 0)
+ pthread_cond_wait(&mt->cv, &mt->mtx);
+ action = mt1->action;
+ pthread_mutex_unlock(&mt->mtx);
+ if (action == -1) break;
+
+ if (mt->eval_out) kann_eval(mt1->a, KANN_F_OUT, 0);
+ else mt1->cost = kann_cost_core(mt1->a, mt->cost_label, mt->cal_grad);
+ }
+ pthread_exit(0);
+}
+
+static void mt_destroy(mtaux_t *mt) /* de-allocate an entire mtaux_t struct */
+{
+ int i;
+ pthread_mutex_lock(&mt->mtx);
+ mt->n_idle = 0;
+ for (i = 1; i < mt->n_threads; ++i) mt->mt[i].action = -1;
+ pthread_cond_broadcast(&mt->cv);
+ pthread_mutex_unlock(&mt->mtx);
+ for (i = 1; i < mt->n_threads; ++i) pthread_join(mt->mt[i].tid, 0);
+ for (i = 0; i < mt->n_threads; ++i) kann_delete(mt->mt[i].a);
+ free(mt->mt);
+ pthread_cond_destroy(&mt->cv);
+ pthread_mutex_destroy(&mt->mtx);
+ free(mt);
+}
+
+void kann_mt(kann_t *ann, int n_threads, int max_batch_size)
+{
+ mtaux_t *mt;
+ int i, k;
+
+ if (n_threads <= 1) {
+ if (ann->mt) mt_destroy((mtaux_t*)ann->mt);
+ ann->mt = 0;
+ return;
+ }
+ if (n_threads > max_batch_size) n_threads = max_batch_size;
+ if (n_threads <= 1) return;
+
+ mt = (mtaux_t*)calloc(1, sizeof(mtaux_t));
+ mt->n_threads = n_threads, mt->max_batch_size = max_batch_size;
+ pthread_mutex_init(&mt->mtx, 0);
+ pthread_cond_init(&mt->cv, 0);
+ mt->mt = (mtaux1_t*)calloc(n_threads, sizeof(mtaux1_t));
+ for (i = k = 0; i < n_threads; ++i) {
+ int size = (max_batch_size - k) / (n_threads - i);
+ mt->mt[i].a = kann_clone(ann, size);
+ mt->mt[i].g = mt;
+ k += size;
+ }
+ for (i = 1; i < n_threads; ++i)
+ pthread_create(&mt->mt[i].tid, 0, mt_worker, &mt->mt[i]);
+ while (mt->n_idle < n_threads - 1); /* busy waiting until all threads in sync */
+ ann->mt = mt;
+}
+
+static void mt_kickoff(kann_t *a, int cost_label, int cal_grad, int eval_out)
+{
+ mtaux_t *mt = (mtaux_t*)a->mt;
+ int i, j, k, B, n_var;
+
+ B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */
+ assert(B <= mt->max_batch_size); /* TODO: can be relaxed */
+ n_var = kann_size_var(a);
+
+ pthread_mutex_lock(&mt->mtx);
+ mt->cost_label = cost_label, mt->cal_grad = cal_grad, mt->eval_out = eval_out;
+ for (i = k = 0; i < mt->n_threads; ++i) {
+ int size = (B - k) / (mt->n_threads - i);
+ for (j = 0; j < a->n; ++j)
+ if (kad_is_feed(a->v[j]))
+ mt->mt[i].a->v[j]->x = &a->v[j]->x[k * kad_len(a->v[j]) / a->v[j]->d[0]];
+ kad_sync_dim(mt->mt[i].a->n, mt->mt[i].a->v, size); /* TODO: we can point ->x to internal nodes, too */
+ k += size;
+ memcpy(mt->mt[i].a->x, a->x, n_var * sizeof(float));
+ mt->mt[i].action = 1;
+ }
+ mt->n_idle = 0;
+ pthread_cond_broadcast(&mt->cv);
+ pthread_mutex_unlock(&mt->mtx);
+}
+
+float kann_cost(kann_t *a, int cost_label, int cal_grad)
+{
+ mtaux_t *mt = (mtaux_t*)a->mt;
+ int i, j, B, k, n_var;
+ float cost;
+
+ if (mt == 0) return kann_cost_core(a, cost_label, cal_grad);
+ B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */
+ n_var = kann_size_var(a);
+
+ mt_kickoff(a, cost_label, cal_grad, 0);
+ mt->mt[0].cost = kann_cost_core(mt->mt[0].a, cost_label, cal_grad);
+ while (mt->n_idle < mt->n_threads - 1); /* busy waiting until all threads in sync */
+
+ memset(a->g, 0, n_var * sizeof(float)); /* TODO: check if this is necessary when cal_grad is false */
+ for (i = k = 0, cost = 0.0f; i < mt->n_threads; ++i) {
+ int size = (B - k) / (mt->n_threads - i);
+ cost += mt->mt[i].cost * size / B;
+ kad_saxpy(n_var, (float)size / B, mt->mt[i].a->g, a->g);
+ k += size;
+ }
+ for (j = 0; j < a->n; ++j) { /* copy values back at recurrent nodes (needed by textgen; TODO: temporary solution) */
+ kad_node_t *p = a->v[j];
+ if (p->pre && p->n_d >= 2 && p->d[0] == B) {
+ for (i = k = 0; i < mt->n_threads; ++i) {
+ kad_node_t *q = mt->mt[i].a->v[j];
+ memcpy(&p->x[k], q->x, kad_len(q) * sizeof(float));
+ k += kad_len(q);
+ }
+ }
+ }
+ return cost;
+}
+
+int kann_eval_out(kann_t *a)
+{
+ mtaux_t *mt = (mtaux_t*)a->mt;
+ int j, B, n_eval;
+ if (mt == 0) return kann_eval(a, KANN_F_OUT, 0);
+ B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */
+ mt_kickoff(a, 0, 0, 1);
+ n_eval = kann_eval(mt->mt[0].a, KANN_F_OUT, 0);
+ while (mt->n_idle < mt->n_threads - 1); /* busy waiting until all threads in sync */
+ for (j = 0; j < a->n; ++j) { /* copy output values back */
+ kad_node_t *p = a->v[j];
+ if (p->ext_flag & KANN_F_OUT) {
+ int i, t, k, d0 = p->d[0] / B, d1 = 1; /* for RNN, p->d[0] may equal unroll_len * batch_size */
+ assert(p->d[0] % B == 0);
+ for (i = 1; i < p->n_d; ++i) d1 *= p->d[i];
+ for (i = 0; i < d0; ++i) {
+ for (t = k = 0; t < mt->n_threads; ++t) { /* similar to the forward pass of kad_op_concat() */
+ kad_node_t *q = mt->mt[t].a->v[j];
+ int size = q->d[0] / d0;
+ memcpy(&p->x[(i * B + k) * d1], &q->x[i * size * d1], size * d1 * sizeof(float));
+ k += size;
+ }
+ }
+ }
+ }
+ return n_eval;
+}
+
+int kann_class_error(const kann_t *ann, int *base)
+{
+ mtaux_t *mt = (mtaux_t*)ann->mt;
+ int i, n_err = 0, b = 0;
+ if (mt == 0) return kann_class_error_core(ann, base);
+ for (i = 0; i < mt->n_threads; ++i) {
+ n_err += kann_class_error_core(mt->mt[i].a, &b);
+ *base += b;
+ }
+ return n_err;
+}
+
+void kann_switch(kann_t *ann, int is_train)
+{
+ mtaux_t *mt = (mtaux_t*)ann->mt;
+ int i;
+ if (mt == 0) {
+ kann_switch_core(ann, is_train);
+ return;
+ }
+ for (i = 0; i < mt->n_threads; ++i)
+ kann_switch_core(mt->mt[i].a, is_train);
+}
+#else
+void kann_mt(kann_t *ann, int n_threads, int max_batch_size) {}
+float kann_cost(kann_t *a, int cost_label, int cal_grad) { return kann_cost_core(a, cost_label, cal_grad); }
+int kann_eval_out(kann_t *a) { return kann_eval(a, KANN_F_OUT, 0); }
+int kann_class_error(const kann_t *a, int *base) { return kann_class_error_core(a, base); }
+void kann_switch(kann_t *ann, int is_train) { return kann_switch_core(ann, is_train); }
+#endif
+
+/***********************
+ *** @@IO: model I/O ***
+ ***********************/
+
+#define KANN_MAGIC "KAN\1"
+
+void kann_save_fp(FILE *fp, kann_t *ann)
+{
+ kann_set_batch_size(ann, 1);
+ fwrite(KANN_MAGIC, 1, 4, fp);
+ kad_save(fp, ann->n, ann->v);
+ fwrite(ann->x, sizeof(float), kann_size_var(ann), fp);
+ fwrite(ann->c, sizeof(float), kann_size_const(ann), fp);
+}
+
+void kann_save(const char *fn, kann_t *ann)
+{
+ FILE *fp;
+ fp = fn && strcmp(fn, "-")? fopen(fn, "wb") : stdout;
+ kann_save_fp(fp, ann);
+ fclose(fp);
+}
+
+kann_t *kann_load_fp(FILE *fp)
+{
+ char magic[4];
+ kann_t *ann;
+ int n_var, n_const;
+
+ (void) !fread(magic, 1, 4, fp);
+ if (strncmp(magic, KANN_MAGIC, 4) != 0) {
+ return 0;
+ }
+ ann = (kann_t*)calloc(1, sizeof(kann_t));
+ ann->v = kad_load(fp, &ann->n);
+ n_var = kad_size_var(ann->n, ann->v);
+ n_const = kad_size_const(ann->n, ann->v);
+ ann->x = (float*)malloc(n_var * sizeof(float));
+ ann->g = (float*)calloc(n_var, sizeof(float));
+ ann->c = (float*)malloc(n_const * sizeof(float));
+ (void) !fread(ann->x, sizeof(float), n_var, fp);
+ (void) !fread(ann->c, sizeof(float), n_const, fp);
+ kad_ext_sync(ann->n, ann->v, ann->x, ann->g, ann->c);
+ return ann;
+}
+
+kann_t *kann_load(const char *fn)
+{
+ FILE *fp;
+ kann_t *ann;
+ fp = fn && strcmp(fn, "-")? fopen(fn, "rb") : stdin;
+ ann = kann_load_fp(fp);
+ fclose(fp);
+ return ann;
+}
+
+/**********************************************
+ *** @@LAYER: layers and model generation ***
+ **********************************************/
+
+/********** General but more complex APIs **********/
+
+kad_node_t *kann_new_leaf_array(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, int32_t d[KAD_MAX_DIM])
+{
+ int i, len, off = offset && par? *offset : -1;
+ kad_node_t *p;
+
+ if (off >= 0 && par[off]) return par[(*offset)++];
+ p = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ p->n_d = n_d, p->flag = flag;
+ memcpy(p->d, d, n_d * sizeof(int32_t));
+ len = kad_len(p);
+ p->x = (float*)calloc(len, sizeof(float));
+ if (p->n_d <= 1) {
+ for (i = 0; i < len; ++i)
+ p->x[i] = x0_01;
+ } else {
+ double sdev_inv;
+ sdev_inv = 1.0 / sqrt((double)len / p->d[0]);
+ for (i = 0; i < len; ++i)
+ p->x[i] = (float)(kad_drand_normal(0) * sdev_inv);
+ }
+ if (off >= 0) par[off] = p, ++(*offset);
+ return p;
+}
+
+kad_node_t *kann_new_leaf2(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, ...)
+{
+ int32_t i, d[KAD_MAX_DIM];
+ va_list ap;
+ va_start(ap, n_d); for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); va_end(ap);
+ return kann_new_leaf_array(offset, par, flag, x0_01, n_d, d);
+}
+
+kad_node_t *kann_layer_dense2(int *offset, kad_node_p *par, kad_node_t *in, int n1)
+{
+ int n0;
+ kad_node_t *w, *b;
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ return kad_add(kad_cmul(in, w), b);
+}
+
+kad_node_t *kann_layer_dropout2(int *offset, kad_node_p *par, kad_node_t *t, float r)
+{
+ kad_node_t *x[2], *cr;
+ cr = kann_new_leaf2(offset, par, KAD_CONST, r, 0);
+ x[0] = t, x[1] = kad_dropout(t, cr);
+ return kad_switch(2, x);
+}
+
+kad_node_t *kann_layer_layernorm2(int *offset, kad_node_t **par, kad_node_t *in)
+{
+ int n0;
+ kad_node_t *alpha, *beta;
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ alpha = kann_new_leaf2(offset, par, KAD_VAR, 1.0f, 1, n0);
+ beta = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n0);
+ return kad_add(kad_mul(kad_stdnorm(in), alpha), beta);
+}
+
+static inline kad_node_t *cmul_norm2(int *offset, kad_node_t **par, kad_node_t *x, kad_node_t *w, int use_norm)
+{
+ return use_norm? kann_layer_layernorm2(offset, par, kad_cmul(x, w)) : kad_cmul(x, w);
+}
+
+kad_node_t *kann_layer_rnn2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag)
+{
+ int n0, n1 = h0->d[h0->n_d-1], use_norm = !!(rnn_flag & KANN_RNN_NORM);
+ kad_node_t *t, *w, *u, *b, *out;
+
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, h0, u, use_norm);
+ if (in) {
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ out = kad_tanh(kad_add(t, b));
+ out->pre = h0;
+ return out;
+}
+
+kad_node_t *kann_layer_gru2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag)
+{
+ int n0 = 0, n1 = h0->d[h0->n_d-1], use_norm = !!(rnn_flag & KANN_RNN_NORM);
+ kad_node_t *t, *r, *z, *w, *u, *b, *s, *out;
+
+ if (in) n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ /* z = sigm(x_t * W_z + h_{t-1} * U_z + b_z) */
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, h0, u, use_norm);
+ if (in) {
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ z = kad_sigm(kad_add(t, b));
+ /* r = sigm(x_t * W_r + h_{t-1} * U_r + b_r) */
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, h0, u, use_norm);
+ if (in) {
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ r = kad_sigm(kad_add(t, b));
+ /* s = tanh(x_t * W_s + (h_{t-1} # r) * U_s + b_s) */
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, kad_mul(r, h0), u, use_norm);
+ if (in) {
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ s = kad_tanh(kad_add(t, b));
+ /* h_t = z # h_{t-1} + (1 - z) # s */
+ out = kad_add(kad_mul(kad_1minus(z), s), kad_mul(z, h0));
+ out->pre = h0;
+ return out;
+}
+
+/********** APIs without offset & par **********/
+
+kad_node_t *kann_new_leaf(uint8_t flag, float x0_01, int n_d, ...)
+{
+ int32_t i, d[KAD_MAX_DIM];
+ va_list ap;
+ va_start(ap, n_d); for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); va_end(ap);
+ return kann_new_leaf_array(0, 0, flag, x0_01, n_d, d);
+}
+
+kad_node_t *kann_new_scalar(uint8_t flag, float x) { return kann_new_leaf(flag, x, 0); }
+kad_node_t *kann_new_weight(int n_row, int n_col) { return kann_new_leaf(KAD_VAR, 0.0f, 2, n_row, n_col); }
+kad_node_t *kann_new_vec(int n, float x) { return kann_new_leaf(KAD_VAR, x, 1, n); }
+kad_node_t *kann_new_bias(int n) { return kann_new_vec(n, 0.0f); }
+kad_node_t *kann_new_weight_conv2d(int n_out, int n_in, int k_row, int k_col) { return kann_new_leaf(KAD_VAR, 0.0f, 4, n_out, n_in, k_row, k_col); }
+kad_node_t *kann_new_weight_conv1d(int n_out, int n_in, int kernel_len) { return kann_new_leaf(KAD_VAR, 0.0f, 3, n_out, n_in, kernel_len); }
+
+kad_node_t *kann_layer_input(int n1)
+{
+ kad_node_t *t;
+ t = kad_feed(2, 1, n1);
+ t->ext_flag |= KANN_F_IN;
+ return t;
+}
+
+kad_node_t *kann_layer_dense(kad_node_t *in, int n1) { return kann_layer_dense2(0, 0, in, n1); }
+kad_node_t *kann_layer_dropout(kad_node_t *t, float r) { return kann_layer_dropout2(0, 0, t, r); }
+kad_node_t *kann_layer_layernorm(kad_node_t *in) { return kann_layer_layernorm2(0, 0, in); }
+
+kad_node_t *kann_layer_rnn(kad_node_t *in, int n1, int rnn_flag)
+{
+ kad_node_t *h0;
+ h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ h0->x = (float*)calloc(n1, sizeof(float));
+ return kann_layer_rnn2(0, 0, in, h0, rnn_flag);
+}
+
+kad_node_t *kann_layer_gru(kad_node_t *in, int n1, int rnn_flag)
+{
+ kad_node_t *h0;
+ h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ h0->x = (float*)calloc(n1, sizeof(float));
+ return kann_layer_gru2(0, 0, in, h0, rnn_flag);
+}
+
+static kad_node_t *kann_cmul_norm(kad_node_t *x, kad_node_t *w)
+{
+ return kann_layer_layernorm(kad_cmul(x, w));
+}
+
+kad_node_t *kann_layer_lstm(kad_node_t *in, int n1, int rnn_flag)
+{
+ int n0;
+ kad_node_t *i, *f, *o, *g, *w, *u, *b, *h0, *c0, *c, *out;
+ kad_node_t *(*cmul)(kad_node_t*, kad_node_t*) = (rnn_flag & KANN_RNN_NORM)? kann_cmul_norm : kad_cmul;
+
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ h0->x = (float*)calloc(n1, sizeof(float));
+ c0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ c0->x = (float*)calloc(n1, sizeof(float));
+
+ /* i = sigm(x_t * W_i + h_{t-1} * U_i + b_i) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_bias(n1);
+ i = kad_sigm(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* f = sigm(x_t * W_f + h_{t-1} * U_f + b_f) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_vec(n1, 1.0f); /* see Jozefowicz et al on using a large bias */
+ f = kad_sigm(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* o = sigm(x_t * W_o + h_{t-1} * U_o + b_o) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_bias(n1);
+ o = kad_sigm(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* g = tanh(x_t * W_g + h_{t-1} * U_g + b_g) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_bias(n1);
+ g = kad_tanh(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* c_t = c_{t-1} # f + g # i */
+ c = kad_add(kad_mul(f, c0), kad_mul(g, i)); /* can't be kad_mul(c0, f)!!! */
+ c->pre = c0;
+ /* h_t = tanh(c_t) # o */
+ if (rnn_flag & KANN_RNN_NORM) c = kann_layer_layernorm(c); /* see Ba et al (2016) about how to apply layer normalization to LSTM */
+ out = kad_mul(kad_tanh(c), o);
+ out->pre = h0;
+ return out;
+}
+
+kad_node_t *kann_layer_conv2d(kad_node_t *in, int n_flt, int k_rows, int k_cols, int stride_r, int stride_c, int pad_r, int pad_c)
+{
+ kad_node_t *w;
+ w = kann_new_weight_conv2d(n_flt, in->d[1], k_rows, k_cols);
+ return kad_conv2d(in, w, stride_r, stride_c, pad_r, pad_c);
+}
+
+kad_node_t *kann_layer_conv1d(kad_node_t *in, int n_flt, int k_size, int stride, int pad)
+{
+ kad_node_t *w;
+ w = kann_new_weight_conv1d(n_flt, in->d[1], k_size);
+ return kad_conv1d(in, w, stride, pad);
+}
+
+kad_node_t *kann_layer_cost(kad_node_t *t, int n_out, int cost_type)
+{
+ kad_node_t *cost = 0, *truth = 0;
+ assert(cost_type == KANN_C_CEB || cost_type == KANN_C_CEM || cost_type == KANN_C_CEB_NEG || cost_type == KANN_C_MSE);
+ t = kann_layer_dense(t, n_out);
+ truth = kad_feed(2, 1, n_out), truth->ext_flag |= KANN_F_TRUTH;
+
+ if (cost_type == KANN_C_MSE) {
+ cost = kad_mse(t, truth);
+ } else if (cost_type == KANN_C_CEB) {
+ t = kad_sigm(t);
+ cost = kad_ce_bin(t, truth);
+ } else if (cost_type == KANN_C_CEB_NEG) {
+ t = kad_tanh(t);
+ cost = kad_ce_bin_neg(t, truth);
+ } else if (cost_type == KANN_C_CEM) {
+ t = kad_softmax(t);
+ cost = kad_ce_multi(t, truth);
+ }
+ else {
+ assert (0);
+ }
+
+ t->ext_flag |= KANN_F_OUT;
+ cost->ext_flag |= KANN_F_COST;
+
+ return cost;
+}
+
+void kann_shuffle(int n, int *s)
+{
+ int i, j, t;
+ for (i = 0; i < n; ++i) s[i] = i;
+ for (i = n; i > 0; --i) {
+ j = (int)(i * kad_drand(0));
+ t = s[j], s[j] = s[i-1], s[i-1] = t;
+ }
+}
+
+/***************************
+ *** @@MIN: minimization ***
+ ***************************/
+
+#ifdef __SSE__
+#include <xmmintrin.h>
+
+void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, float *t, float *r)
+{
+ int i, n4 = n>>2<<2;
+ __m128 vh, vg, vr, vt, vd, vd1, tmp, vtiny;
+ vh = _mm_set1_ps(h0);
+ vd = _mm_set1_ps(decay);
+ vd1 = _mm_set1_ps(1.0f - decay);
+ vtiny = _mm_set1_ps(1e-6f);
+ for (i = 0; i < n4; i += 4) {
+ vt = _mm_loadu_ps(&t[i]);
+ vr = _mm_loadu_ps(&r[i]);
+ vg = _mm_loadu_ps(&g[i]);
+ if (h) vh = _mm_loadu_ps(&h[i]);
+ vr = _mm_add_ps(_mm_mul_ps(vd1, _mm_mul_ps(vg, vg)), _mm_mul_ps(vd, vr));
+ _mm_storeu_ps(&r[i], vr);
+ tmp = _mm_sub_ps(vt, _mm_mul_ps(_mm_mul_ps(vh, _mm_rsqrt_ps(_mm_add_ps(vtiny, vr))), vg));
+ _mm_storeu_ps(&t[i], tmp);
+ }
+ for (; i < n; ++i) {
+ r[i] = (1. - decay) * g[i] * g[i] + decay * r[i];
+ t[i] -= (h? h[i] : h0) / sqrtf(1e-6f + r[i]) * g[i];
+ }
+}
+#else
+void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, float *t, float *r)
+{
+ int i;
+ for (i = 0; i < n; ++i) {
+ float lr = h? h[i] : h0;
+ r[i] = (1.0f - decay) * g[i] * g[i] + decay * r[i];
+ t[i] -= lr / sqrtf(1e-6f + r[i]) * g[i];
+ }
+}
+#endif
+
+float kann_grad_clip(float thres, int n, float *g)
+{
+ int i;
+ double s2 = 0.0;
+ for (i = 0; i < n; ++i)
+ s2 += g[i] * g[i];
+ s2 = sqrt(s2);
+ if (s2 > thres)
+ for (i = 0, s2 = 1.0 / s2; i < n; ++i)
+ g[i] *= (float)s2;
+ return (float)s2 / thres;
+}
+
+/****************************************************************
+ *** @@XY: simpler API for network with a single input/output ***
+ ****************************************************************/
+
+int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch,
+ int max_drop_streak, float frac_val, int n,
+ float **_x, float **_y,
+ kann_train_cb cb, void *ud)
+{
+ int i, j, *shuf, n_train, n_val, n_in, n_out, n_var, n_const, drop_streak = 0, min_set = 0;
+ float **x, **y, *x1, *y1, *r, min_val_cost = FLT_MAX, *min_x, *min_c;
+
+ n_in = kann_dim_in(ann);
+ n_out = kann_dim_out(ann);
+ if (n_in < 0 || n_out < 0) return -1;
+ n_var = kann_size_var(ann);
+ n_const = kann_size_const(ann);
+ r = (float*)calloc(n_var, sizeof(float));
+ shuf = (int*)malloc(n * sizeof(int));
+ x = (float**)malloc(n * sizeof(float*));
+ y = (float**)malloc(n * sizeof(float*));
+ kann_shuffle(n, shuf);
+ for (j = 0; j < n; ++j)
+ x[j] = _x[shuf[j]], y[j] = _y[shuf[j]];
+ n_val = (int)(n * frac_val);
+ n_train = n - n_val;
+ min_x = (float*)malloc(n_var * sizeof(float));
+ min_c = (float*)malloc(n_const * sizeof(float));
+
+ x1 = (float*)malloc(n_in * mini_size * sizeof(float));
+ y1 = (float*)malloc(n_out * mini_size * sizeof(float));
+ kann_feed_bind(ann, KANN_F_IN, 0, &x1);
+ kann_feed_bind(ann, KANN_F_TRUTH, 0, &y1);
+
+ for (i = 0; i < max_epoch; ++i) {
+ int n_proc = 0, n_train_err = 0, n_val_err = 0, n_train_base = 0, n_val_base = 0;
+ double train_cost = 0.0, val_cost = 0.0;
+ kann_shuffle(n_train, shuf);
+ kann_switch(ann, 1);
+ while (n_proc < n_train) {
+ int b, c, ms = n_train - n_proc < mini_size? n_train - n_proc : mini_size;
+ for (b = 0; b < ms; ++b) {
+ memcpy(&x1[b*n_in], x[shuf[n_proc+b]], n_in * sizeof(float));
+ memcpy(&y1[b*n_out], y[shuf[n_proc+b]], n_out * sizeof(float));
+ }
+ kann_set_batch_size(ann, ms);
+ train_cost += kann_cost(ann, 0, 1) * ms;
+ c = kann_class_error(ann, &b);
+ n_train_err += c, n_train_base += b;
+ kann_RMSprop(n_var, lr, 0, 0.9f, ann->g, ann->x, r);
+ n_proc += ms;
+ }
+ train_cost /= n_train;
+ kann_switch(ann, 0);
+ n_proc = 0;
+ while (n_proc < n_val) {
+ int b, c, ms = n_val - n_proc < mini_size? n_val - n_proc : mini_size;
+ for (b = 0; b < ms; ++b) {
+ memcpy(&x1[b*n_in], x[n_train+n_proc+b], n_in * sizeof(float));
+ memcpy(&y1[b*n_out], y[n_train+n_proc+b], n_out * sizeof(float));
+ }
+ kann_set_batch_size(ann, ms);
+ val_cost += kann_cost(ann, 0, 0) * ms;
+ c = kann_class_error(ann, &b);
+ n_val_err += c, n_val_base += b;
+ n_proc += ms;
+ }
+ if (n_val > 0) val_cost /= n_val;
+ if (cb) {
+ cb(i + 1, train_cost, val_cost, ud);
+#if 0
+ fprintf(stderr, "epoch: %d; training cost: %g", i+1, train_cost);
+ if (n_train_base) fprintf(stderr, " (class error: %.2f%%)", 100.0f * n_train_err / n_train);
+ if (n_val > 0) {
+ fprintf(stderr, "; validation cost: %g", val_cost);
+ if (n_val_base) fprintf(stderr, " (class error: %.2f%%)", 100.0f * n_val_err / n_val);
+ }
+ fputc('\n', stderr);
+#endif
+ }
+ if (i >= max_drop_streak && n_val > 0) {
+ if (val_cost < min_val_cost) {
+ min_set = 1;
+ memcpy(min_x, ann->x, n_var * sizeof(float));
+ memcpy(min_c, ann->c, n_const * sizeof(float));
+ drop_streak = 0;
+ min_val_cost = (float)val_cost;
+ } else if (++drop_streak >= max_drop_streak)
+ break;
+ }
+ }
+ if (min_set) {
+ memcpy(ann->x, min_x, n_var * sizeof(float));
+ memcpy(ann->c, min_c, n_const * sizeof(float));
+ }
+
+ free(min_c); free(min_x); free(y1); free(x1); free(y); free(x); free(shuf); free(r);
+ return i;
+}
+
+float kann_cost_fnn1(kann_t *ann, int n, float **x, float **y)
+{
+ int n_in, n_out, n_proc = 0, mini_size = 64 < n? 64 : n;
+ float *x1, *y1;
+ double cost = 0.0;
+
+ n_in = kann_dim_in(ann);
+ n_out = kann_dim_out(ann);
+ if (n <= 0 || n_in < 0 || n_out < 0) return 0.0;
+
+ x1 = (float*)malloc(n_in * mini_size * sizeof(float));
+ y1 = (float*)malloc(n_out * mini_size * sizeof(float));
+ kann_feed_bind(ann, KANN_F_IN, 0, &x1);
+ kann_feed_bind(ann, KANN_F_TRUTH, 0, &y1);
+ kann_switch(ann, 0);
+ while (n_proc < n) {
+ int b, ms = n - n_proc < mini_size? n - n_proc : mini_size;
+ for (b = 0; b < ms; ++b) {
+ memcpy(&x1[b*n_in], x[n_proc+b], n_in * sizeof(float));
+ memcpy(&y1[b*n_out], y[n_proc+b], n_out * sizeof(float));
+ }
+ kann_set_batch_size(ann, ms);
+ cost += kann_cost(ann, 0, 0) * ms;
+ n_proc += ms;
+ }
+ free(y1); free(x1);
+ return (float)(cost / n);
+}
+
+const float *kann_apply1(kann_t *a, float *x)
+{
+ int i_out;
+ i_out = kann_find(a, KANN_F_OUT, 0);
+ if (i_out < 0) return 0;
+ kann_set_batch_size(a, 1);
+ kann_feed_bind(a, KANN_F_IN, 0, &x);
+ kad_eval_at(a->n, a->v, i_out);
+ return a->v[i_out]->x;
+}
diff --git a/contrib/kann/kann.h b/contrib/kann/kann.h
new file mode 100644
index 0000000..af0de5f
--- /dev/null
+++ b/contrib/kann/kann.h
@@ -0,0 +1,240 @@
+/*
+ The MIT License
+
+ Copyright (c) 2018-2019 Dana-Farber Cancer Institute
+ 2016-2018 Broad Institute
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+#ifndef KANN_H
+#define KANN_H
+
+#define KANN_VERSION "r536"
+
+#define KANN_F_IN 0x1 /* input */
+#define KANN_F_OUT 0x2 /* output */
+#define KANN_F_TRUTH 0x4 /* truth output */
+#define KANN_F_COST 0x8 /* final cost */
+
+#define KANN_C_CEB 1 /* binary cross-entropy cost, used with sigmoid */
+#define KANN_C_CEM 2 /* multi-class cross-entropy cost, used with softmax */
+#define KANN_C_CEB_NEG 3 /* binary cross-enytopy-like cost, used with tanh */
+#define KANN_C_MSE 4 /* mean square error */
+
+#define KANN_RNN_VAR_H0 0x1 /* take the initial hidden values as variables */
+#define KANN_RNN_NORM 0x2 /* apply layer normalization */
+
+#include "kautodiff.h"
+
+typedef struct {
+ int n; /* number of nodes in the computational graph */
+ kad_node_t **v; /* list of nodes */
+ float *x, *g, *c; /* collated variable values, gradients and constant values */
+ void *mt; /* auxiliary data for multi-threading; NULL if multi-threading disabled */
+} kann_t;
+
+extern int kann_verbose;
+
+#define kann_size_var(a) kad_size_var((a)->n, (a)->v)
+#define kann_size_const(a) kad_size_const((a)->n, (a)->v)
+#define kann_dim_in(a) kann_feed_dim((a), KANN_F_IN, 0)
+#define kann_dim_out(a) kann_feed_dim((a), KANN_F_TRUTH, 0)
+#define kann_srand(seed) kad_srand(0, (seed))
+#define kann_drand() kad_drand(0)
+#define kann_set_batch_size(ann, B) kad_sync_dim((ann)->n, (ann)->v, (B))
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Generate a network from a computational graph
+ *
+ * A network must have at least one scalar cost node (i.e. whose n_d==0). It
+ * may optionally contain other cost nodes or output nodes not leading to the
+ * primary cost node.
+ *
+ * @param cost cost node (must be a scalar, i.e. cost->n_d==0)
+ * @param n_rest number of other nodes without predecessors
+ * @param ... other nodes (of type kad_node_t*) without predecessors
+ *
+ * @return network on success, or NULL otherwise
+ */
+kann_t *kann_new(kad_node_t *cost, int n_rest, ...);
+
+/**
+ * Unroll an RNN
+ *
+ * @param a network
+ * @param len number of unrolls
+ *
+ * @return an unrolled network, or NULL if the network is not an RNN
+ */
+kann_t *kann_unroll(kann_t *a, ...);
+
+kann_t *kann_unroll_array(kann_t *a, int *len);
+kann_t *kann_clone(kann_t *a, int batch_size);
+void kann_delete(kann_t *a); /* delete a network generated by kann_new() or kann_layer_final() */
+void kann_delete_unrolled(kann_t *a); /* delete a network generated by kann_unroll() */
+
+/**
+ * Enable/disable multi-threading (requiring pthread)
+ *
+ * KANN splits a mini-batch to $n_threads mini-mini-batches and puts each of
+ * them on one thread. So far, only kann_cost() takes the advantage of
+ * multi-threading.
+ *
+ * @param ann network
+ * @param n_threads number of threads; <=1 to completely disable multi-threading
+ * @param max_batch_size max mini-batch size; shall no smaller than n_threads
+ */
+void kann_mt(kann_t *ann, int n_threads, int max_batch_size);
+
+/**
+ * Bind float arrays to feed nodes
+ *
+ * @param a network
+ * @param ext_flag required external flags
+ * @param ext_label required external label
+ * @param x pointers (size equal to the number of matching feed nodes)
+ *
+ * @return number of matching feed nodes
+ */
+int kann_feed_bind(kann_t *a, uint32_t ext_flag, int32_t ext_label, float **x);
+
+/**
+ * Compute the cost and optionally gradients
+ *
+ * @param a network
+ * @param cost_label required external label
+ * @param cal_grad whether to compute gradients
+ *
+ * @return cost
+ */
+float kann_cost(kann_t *a, int cost_label, int cal_grad);
+
+int kann_eval(kann_t *a, uint32_t ext_flag, int ext_label);
+int kann_eval_out(kann_t *a);
+int kann_class_error(const kann_t *ann, int *base);
+
+/**
+ * Find a node
+ *
+ * @param a network
+ * @param ext_flag required external flags; set to 0 to match all flags
+ * @param ext_label required external label
+ *
+ * @return >=0 if found; -1 if not found; -2 if found multiple
+ */
+int kann_find(const kann_t *a, uint32_t ext_flag, int32_t ext_label);
+
+/**
+ * Get the size of a feed node, assuming mini-batch size 1
+ *
+ * @param a network
+ * @param ext_flag required external flags
+ * @param ext_label required external label
+ *
+ * @return size>=0; -1 if not found; -2 if found multiple
+ */
+int kann_feed_dim(const kann_t *a, uint32_t ext_flag, int32_t ext_label);
+
+/**
+ * Get an RNN ready for continuous feeding
+ *
+ * @param a network
+ */
+void kann_rnn_start(kann_t *a);
+
+void kann_rnn_end(kann_t *a);
+
+/**
+ * Switch between training and prediction networks (effective only when there are switch nodes)
+ *
+ * @param a network
+ * @param is_train 0 for prediction network and non-zero for training net
+ */
+void kann_switch(kann_t *a, int is_train);
+
+/**
+ * RMSprop update
+ *
+ * @param n number of variables
+ * @param h0 learning rate
+ * @param h per-variable learning rate; NULL if not applicable
+ * @param decay RMSprop decay; use 0.9 if unsure
+ * @param g gradient, of size n
+ * @param t variables to change
+ * @param r memory, of size n
+ */
+void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, float *t, float *r);
+
+void kann_shuffle(int n, int *s);
+float kann_grad_clip(float thres, int n, float *g);
+
+/* common layers */
+kad_node_t *kann_layer_input(int n1);
+kad_node_t *kann_layer_dense(kad_node_t *in, int n1);
+kad_node_t *kann_layer_dropout(kad_node_t *t, float r);
+kad_node_t *kann_layer_layernorm(kad_node_t *in);
+kad_node_t *kann_layer_rnn(kad_node_t *in, int n1, int rnn_flag);
+kad_node_t *kann_layer_lstm(kad_node_t *in, int n1, int rnn_flag);
+kad_node_t *kann_layer_gru(kad_node_t *in, int n1, int rnn_flag);
+kad_node_t *kann_layer_conv2d(kad_node_t *in, int n_flt, int k_rows, int k_cols, int stride_r, int stride_c, int pad_r, int pad_c);
+kad_node_t *kann_layer_conv1d(kad_node_t *in, int n_flt, int k_size, int stride, int pad);
+kad_node_t *kann_layer_cost(kad_node_t *t, int n_out, int cost_type);
+
+kad_node_t *kann_new_leaf(uint8_t flag, float x0_01, int n_d, ...); /* flag can be KAD_CONST or KAD_VAR */
+kad_node_t *kann_new_scalar(uint8_t flag, float x);
+kad_node_t *kann_new_weight(int n_row, int n_col);
+kad_node_t *kann_new_bias(int n);
+kad_node_t *kann_new_weight_conv2d(int n_out, int n_in, int k_row, int k_col);
+kad_node_t *kann_new_weight_conv1d(int n_out, int n_in, int kernel_len);
+
+kad_node_t *kann_new_leaf_array(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, int32_t d[KAD_MAX_DIM]);
+
+kad_node_t *kann_new_leaf2(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, ...);
+kad_node_t *kann_layer_dense2(int *offset, kad_node_p *par, kad_node_t *in, int n1);
+kad_node_t *kann_layer_dropout2(int *offset, kad_node_p *par, kad_node_t *t, float r);
+kad_node_t *kann_layer_layernorm2(int *offset, kad_node_t **par, kad_node_t *in);
+kad_node_t *kann_layer_rnn2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag);
+kad_node_t *kann_layer_gru2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag);
+
+/* operations on network with a single input node and a single output node */
+typedef void (*kann_train_cb)(int iter, float train_cost, float val_cost, void *ud);
+int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch,
+ int max_drop_streak, float frac_val, int n,
+ float **_x, float **_y, kann_train_cb cb, void *ud);
+float kann_cost_fnn1(kann_t *a, int n, float **x, float **y);
+const float *kann_apply1(kann_t *a, float *x);
+
+/* model I/O */
+void kann_save_fp(FILE *fp, kann_t *ann);
+void kann_save(const char *fn, kann_t *ann);
+kann_t *kann_load_fp(FILE *fp);
+kann_t *kann_load(const char *fn);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/kann/kautodiff.c b/contrib/kann/kautodiff.c
new file mode 100644
index 0000000..d05cc00
--- /dev/null
+++ b/contrib/kann/kautodiff.c
@@ -0,0 +1,2460 @@
+#include "config.h"
+
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+#include <math.h>
+#include "kautodiff.h"
+#include "blas-config.h"
+
+typedef struct {
+ uint64_t s[2];
+ double n_gset;
+ int n_iset;
+ volatile int lock;
+} kad_rng_t;
+
+/**********************
+ * Graph construction *
+ **********************/
+
+static inline kad_node_t *kad_new_core(int n_d, int op, int n_child)
+{
+ kad_node_t *s;
+ if (n_d >= KAD_MAX_DIM) return 0;
+ s = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ s->n_d = n_d, s->op = op, s->n_child = n_child;
+ if (s->n_child) s->child = (kad_node_t**)calloc(s->n_child, sizeof(kad_node_t*));
+ return s;
+}
+
+static inline kad_node_t *kad_vleaf(uint8_t flag, float *x, float *g, int n_d, va_list ap)
+{
+ int i;
+ kad_node_t *p;
+ if (n_d > KAD_MAX_DIM) return 0;
+ p = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ p->n_d = n_d;
+ for (i = 0; i < n_d; ++i)
+ p->d[i] = va_arg(ap, int32_t);
+ p->x = x, p->g = g, p->flag = flag;
+ return p;
+}
+
+kad_node_t *kad_const(float *x, int n_d, ...)
+{
+ kad_node_t *p;
+ va_list ap;
+ va_start(ap, n_d); p = kad_vleaf(KAD_CONST, x, 0, n_d, ap); va_end(ap);
+ return p;
+}
+
+kad_node_t *kad_feed(int n_d, ...)
+{
+ kad_node_t *p;
+ va_list ap;
+ va_start(ap, n_d); p = kad_vleaf(0, 0, 0, n_d, ap); va_end(ap);
+ return p;
+}
+
+kad_node_t *kad_var(float *x, float *g, int n_d, ...)
+{
+ kad_node_t *p;
+ va_list ap;
+ va_start(ap, n_d); p = kad_vleaf(KAD_VAR, x, g, n_d, ap); va_end(ap);
+ return p;
+}
+
+static inline kad_node_t *kad_finalize_node(kad_node_t *s) /* a helper function */
+{
+ int i;
+ if (kad_op_list[s->op](s, KAD_SYNC_DIM) < 0) { /* check dimension */
+ if (s->ptr) free(s->ptr);
+ free(s->child); free(s);
+ return 0;
+ }
+ for (i = 0; i < s->n_child; ++i)
+ if (kad_is_back(s->child[i]))
+ break;
+ if (i < s->n_child) s->flag |= KAD_VAR;
+ return s;
+}
+
+/********** Simple arithmetic **********/
+
+static inline kad_node_t *kad_op2_core(int op, kad_node_t *x, kad_node_t *y)
+{
+ kad_node_t *s;
+ s = kad_new_core(0, op, 2);
+ s->child[0] = x, s->child[1] = y;
+ return kad_finalize_node(s);
+}
+
+static inline kad_node_t *kad_op1_core(int op, kad_node_t *x)
+{
+ kad_node_t *s;
+ s = kad_new_core(0, op, 1);
+ s->child[0] = x;
+ return kad_finalize_node(s);
+}
+
+#define KAD_FUNC_OP2(fname, op) kad_node_t *fname(kad_node_t *x, kad_node_t *y) { return kad_op2_core((op), x, y); }
+
+KAD_FUNC_OP2(kad_add, 1)
+KAD_FUNC_OP2(kad_sub, 23)
+KAD_FUNC_OP2(kad_mul, 2)
+KAD_FUNC_OP2(kad_cmul, 3)
+KAD_FUNC_OP2(kad_matmul, 9)
+KAD_FUNC_OP2(kad_ce_multi, 13)
+KAD_FUNC_OP2(kad_ce_bin, 22)
+KAD_FUNC_OP2(kad_ce_bin_neg, 4)
+KAD_FUNC_OP2(kad_mse, 29)
+
+#define KAD_FUNC_OP1(fname, op) kad_node_t *fname(kad_node_t *x) { return kad_op1_core((op), x); }
+
+KAD_FUNC_OP1(kad_log, 27)
+KAD_FUNC_OP1(kad_exp, 33)
+KAD_FUNC_OP1(kad_sin, 34)
+KAD_FUNC_OP1(kad_square, 5)
+KAD_FUNC_OP1(kad_sigm, 6)
+KAD_FUNC_OP1(kad_tanh, 7)
+KAD_FUNC_OP1(kad_relu, 8)
+KAD_FUNC_OP1(kad_1minus, 11)
+KAD_FUNC_OP1(kad_softmax, 14)
+KAD_FUNC_OP1(kad_stdnorm, 32)
+
+kad_node_t *kad_ce_multi_weighted(kad_node_t *pred, kad_node_t *truth, kad_node_t *weight)
+{
+ kad_node_t *s;
+ s = kad_new_core(0, 13, 3);
+ s->child[0] = pred, s->child[1] = truth, s->child[2] = weight;
+ return kad_finalize_node(s);
+}
+
+/********** Convolution **********/
+
+/* compute output dimension and padding sizes on both sides */
+static inline int conv_find_par(int in_size, int kernel_size, int stride, int pad0, int *new_pad0, int *new_pad1)
+{
+ int out_size, pad_both;
+ /* key equation: out_size = (in_size - kernel_size + pad_both) / stride + 1 */
+ if (pad0 == KAD_PAD_SAME && stride == 1) out_size = in_size;
+ else out_size = (in_size - kernel_size + (pad0 > 0? pad0 : 0) + stride - 1) / stride + 1;
+ pad_both = (out_size - 1) * stride + kernel_size - in_size;
+ *new_pad0 = pad_both / 2;
+ *new_pad1 = pad_both - *new_pad0;
+ return out_size;
+}
+
+typedef struct {
+ int kernel_size, stride, pad[2];
+} conv_conf_t;
+
+static inline conv_conf_t *conv2d_gen_aux(int in_row, int in_col, int kernel_r, int kernel_c, int stride_r, int stride_c, int top_pad, int left_pad)
+{
+ conv_conf_t *cnn;
+ cnn = (conv_conf_t*)calloc(2, sizeof(conv_conf_t));
+ cnn[0].kernel_size = kernel_r, cnn[0].stride = stride_r;
+ cnn[1].kernel_size = kernel_c, cnn[1].stride = stride_c;
+ conv_find_par(in_row, kernel_r, stride_r, top_pad, &cnn[0].pad[0], &cnn[0].pad[1]);
+ conv_find_par(in_col, kernel_c, stride_c, left_pad, &cnn[1].pad[0], &cnn[1].pad[1]);
+ return cnn;
+}
+
+kad_node_t *kad_conv2d(kad_node_t *x, kad_node_t *w, int stride_r, int stride_c, int top_pad, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 4 || w->n_d != 4) return 0;
+ s = kad_new_core(0, 16, 2);
+ s->child[0] = x, s->child[1] = w;
+ s->ptr = conv2d_gen_aux(x->d[2], x->d[3], w->d[2], w->d[3], stride_r, stride_c, top_pad, left_pad);
+ s->ptr_size = sizeof(conv_conf_t) * 2;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_max2d(kad_node_t *x, int kernel_r, int kernel_c, int stride_r, int stride_c, int top_pad, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 4) return 0;
+ s = kad_new_core(0, 17, 1);
+ s->child[0] = x;
+ s->ptr = conv2d_gen_aux(x->d[2], x->d[3], kernel_r, kernel_c, stride_r, stride_c, top_pad, left_pad);
+ s->ptr_size = sizeof(conv_conf_t) * 2;
+ return kad_finalize_node(s);
+}
+
+static inline conv_conf_t *conv1d_gen_aux(int in_col, int kernel_c, int stride_c, int left_pad)
+{
+ conv_conf_t *cnn;
+ cnn = (conv_conf_t*)calloc(1, sizeof(conv_conf_t));
+ cnn->kernel_size = kernel_c, cnn->stride = stride_c;
+ conv_find_par(in_col, kernel_c, stride_c, left_pad, &cnn->pad[0], &cnn->pad[1]);
+ return cnn;
+}
+
+kad_node_t *kad_conv1d(kad_node_t *x, kad_node_t *w, int stride, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 3 || w->n_d != 3) return 0;
+ s = kad_new_core(0, 18, 2);
+ s->child[0] = x, s->child[1] = w;
+ s->ptr = conv1d_gen_aux(x->d[2], w->d[2], stride, left_pad);
+ s->ptr_size = sizeof(conv_conf_t);
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_max1d(kad_node_t *x, int kernel_size, int stride, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 3) return 0;
+ s = kad_new_core(0, 19, 1);
+ s->child[0] = x;
+ s->ptr = conv1d_gen_aux(x->d[2], kernel_size, stride, left_pad);
+ s->ptr_size = sizeof(conv_conf_t);
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_avg1d(kad_node_t *x, int kernel_size, int stride, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 3) return 0;
+ s = kad_new_core(0, 28, 1);
+ s->child[0] = x;
+ s->ptr = conv1d_gen_aux(x->d[2], kernel_size, stride, left_pad);
+ s->ptr_size = sizeof(conv_conf_t);
+ return kad_finalize_node(s);
+}
+
+/********** Multi-node pooling **********/
+
+static kad_node_t *kad_pooling_general(int op, int n, kad_node_t **x)
+{
+ int i;
+ kad_node_t *s;
+ s = kad_new_core(0, op, n);
+ s->flag |= KAD_POOL;
+ for (i = 0; i < n; ++i)
+ s->child[i] = x[i];
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_avg(int n, kad_node_t **x) { return kad_pooling_general(10, n, x); }
+kad_node_t *kad_max(int n, kad_node_t **x) { return kad_pooling_general(21, n, x); }
+kad_node_t *kad_stack(int n, kad_node_t **x) { return kad_pooling_general(35, n, x); }
+
+kad_node_t *kad_select(int n, kad_node_t **x, int which)
+{
+ kad_node_t *s;
+ int32_t i, *aux;
+ aux = (int32_t*)calloc(1, 4);
+ *aux = which;
+ s = kad_new_core(0, 12, n);
+ for (i = 0; i < n; ++i) s->child[i] = x[i];
+ s->flag |= KAD_POOL, s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+/********** Dimension reduction **********/
+
+static kad_node_t *kad_reduce_general(int op, kad_node_t *x, int axis)
+{
+ kad_node_t *s;
+ int32_t *aux;
+ aux = (int32_t*)malloc(4);
+ aux[0] = axis;
+ s = kad_new_core(0, op, 1);
+ s->child[0] = x;
+ s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_reduce_sum(kad_node_t *x, int axis) { return kad_reduce_general(25, x, axis); }
+kad_node_t *kad_reduce_mean(kad_node_t *x, int axis) { return kad_reduce_general(26, x, axis); }
+
+/********** Sampling related **********/
+
+kad_node_t *kad_dropout(kad_node_t *x, kad_node_t *y)
+{
+ kad_node_t *z;
+ z = kad_op2_core(15, x, y);
+ z->ptr = kad_rng(), z->ptr_size = sizeof(kad_rng_t);
+ return z;
+}
+
+kad_node_t *kad_sample_normal(kad_node_t *x)
+{
+ kad_node_t *z;
+ z = kad_op1_core(24, x);
+ z->ptr = kad_rng(), z->ptr_size = sizeof(kad_rng_t);
+ return z;
+}
+
+/********** Miscellaneous **********/
+
+kad_node_t *kad_slice(kad_node_t *x, int axis, int start, int end)
+{
+ kad_node_t *s;
+ int32_t *aux;
+ if (end < start || start < 0) return 0;
+ aux = (int32_t*)malloc(3 * 4);
+ aux[0] = axis, aux[1] = start, aux[2] = end;
+ s = kad_new_core(0, 20, 1);
+ s->child[0] = x;
+ s->ptr = aux, s->ptr_size = 3 * 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_concat_array(int axis, int n, kad_node_t **p)
+{
+ kad_node_t *s;
+ int32_t i, *aux;
+ aux = (int32_t*)malloc(4);
+ aux[0] = axis;
+ s = kad_new_core(0, 31, n);
+ for (i = 0; i < n; ++i)
+ s->child[i] = p[i];
+ s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_concat(int axis, int n, ...)
+{
+ int i;
+ kad_node_t **p, *s;
+ va_list ap;
+ p = (kad_node_t**)malloc(n * sizeof(kad_node_t*));
+ va_start(ap, n);
+ for (i = 0; i < n; ++i) p[i] = va_arg(ap, kad_node_p);
+ va_end(ap);
+ s = kad_concat_array(axis, n, p);
+ free(p);
+ return s;
+}
+
+kad_node_t *kad_reshape(kad_node_t *x, int n_d, int *d)
+{
+ kad_node_t *s;
+ int32_t i, *aux = 0;
+ if (n_d > 0) {
+ aux = (int32_t*)malloc(n_d * 4);
+ for (i = 0; i < n_d; ++i) aux[i] = d? d[i] : -1;
+ }
+ s = kad_new_core(0, 30, 1);
+ s->child[0] = x, s->ptr = aux, s->ptr_size = n_d * 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_reverse(kad_node_t *x, int axis)
+{
+ kad_node_t *s;
+ int32_t *aux;
+ aux = (int32_t*)malloc(4);
+ *aux = axis;
+ s = kad_new_core(0, 36, 1);
+ s->child[0] = x, s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_switch(int n, kad_node_t **p)
+{
+ kad_node_t *s;
+ int32_t i, *aux;
+ aux = (int32_t*)calloc(1, 4);
+ s = kad_new_core(0, 12, n);
+ for (i = 0; i < n; ++i)
+ s->child[i] = p[i];
+ s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+/***********************
+ * Graph linearization *
+ ***********************/
+
+static void kad_mark_back(int n, kad_node_t **v)
+{
+ int i, j;
+ for (i = 0; i < n; ++i) {
+ if (v[i]->n_child == 0) continue;
+ for (j = 0; j < v[i]->n_child; ++j)
+ if (kad_is_back(v[i]->child[j]))
+ break;
+ if (j < v[i]->n_child) v[i]->flag |= KAD_VAR;
+ else v[i]->flag &= ~KAD_VAR;
+ }
+}
+
+static void kad_allocate_internal(int n, kad_node_t **v)
+{
+ int i;
+ kad_mark_back(n, v);
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = v[i];
+ if (p->n_child == 0) continue;
+ p->x = (float*)realloc(p->x, kad_len(p) * sizeof(float));
+ if (kad_is_back(p)) {
+ p->g = (float*)realloc(p->g, kad_len(p) * sizeof(float));
+ kad_op_list[p->op](p, KAD_ALLOC);
+ }
+ }
+}
+
+int kad_sync_dim(int n, kad_node_t **v, int batch_size)
+{
+ int i, req_alloc = 0, req_sync = 0, old_size = 0;
+ for (i = 0; i < n; ++i) {
+ if (kad_is_feed(v[i])) {
+ old_size = v[i]->d[0]; /* TODO: check if all feeds have the same batch size */
+ if (batch_size > 0 && v[i]->d[0] != batch_size)
+ v[i]->d[0] = batch_size, req_sync = 1;
+ } else if (v[i]->n_child > 0 && req_sync)
+ kad_op_list[v[i]->op](v[i], KAD_SYNC_DIM);
+ }
+ if (old_size < batch_size) req_alloc = 1;
+ for (i = 0; i < n; ++i)
+ if (v[i]->n_child > 0 && v[i]->x == 0) req_alloc = 1;
+ if (req_alloc) kad_allocate_internal(n, v);
+ return batch_size > 0? batch_size : old_size;
+}
+
+#define kvec_t(type) struct { size_t n, m; type *a; }
+
+#define kv_pop(v) ((v).a[--(v).n])
+
+#define kv_push(type, v, x) do { \
+ if ((v).n == (v).m) { \
+ (v).m = (v).m? (v).m<<1 : 2; \
+ (v).a = (type*)realloc((v).a, sizeof(type) * (v).m); \
+ } \
+ (v).a[(v).n++] = (x); \
+ } while (0)
+
+/* IMPORTANT: kad_node_t::tmp MUST BE set to zero before calling this function */
+kad_node_t **kad_compile_array(int *n_node, int n_roots, kad_node_t **roots)
+{
+ int i;
+ kvec_t(kad_node_p) stack = {0,0,0}, a = {0,0,0};
+
+ /* generate kad_node_t::tmp, the count of the parent nodes; shifted by 1; lowest bit to detect fake roots */
+ for (i = 0; i < n_roots; ++i) {
+ roots[i]->tmp = 1; /* mark the root */
+ kv_push(kad_node_p, stack, roots[i]);
+ }
+ while (stack.n) {
+ kad_node_t *p = kv_pop(stack);
+ for (i = 0; i < p->n_child; ++i) {
+ kad_node_t *q = p->child[i];
+ if (q->tmp == 0) kv_push(kad_node_p, stack, q);
+ q->tmp += 1<<1;
+ }
+ }
+
+ /* topological sorting (Kahn's algorithm) */
+ for (i = 0; i < n_roots; ++i)
+ if (roots[i]->tmp>>1 == 0) /* if roots[i]->tmp>>1 != 0, it is not a real root */
+ kv_push(kad_node_p, stack, roots[i]);
+ while (stack.n) {
+ kad_node_t *p = kv_pop(stack);
+ kv_push(kad_node_p, a, p);
+ for (i = 0; i < p->n_child; ++i) {
+ p->child[i]->tmp -= 1<<1;
+ if (p->child[i]->tmp>>1 == 0)
+ kv_push(kad_node_p, stack, p->child[i]);
+ }
+ }
+ free(stack.a);
+ for (i = 0; i < (int)a.n; ++i) { /* check cycles; no cycles if constructed with kad_add() etc */
+ assert(a.a[i]->tmp>>1 == 0);
+ a.a[i]->tmp = 0;
+ }
+
+ /* reverse */
+ for (i = 0; i < (int)a.n>>1; ++i) { /* reverse a.a[] */
+ kad_node_p t;
+ t = a.a[i], a.a[i] = a.a[a.n-1-i], a.a[a.n-1-i] = t;
+ }
+ kad_allocate_internal(a.n, a.a);
+
+ *n_node = a.n;
+ return a.a;
+}
+
+kad_node_t **kad_compile(int *n_node, int n_roots, ...)
+{
+ int i;
+ kad_node_t **roots, **ret;
+ va_list ap;
+
+ roots = (kad_node_t**)malloc(n_roots * sizeof(kad_node_t*));
+ va_start(ap, n_roots);
+ for (i = 0; i < n_roots; ++i) roots[i] = va_arg(ap, kad_node_p);
+ va_end(ap);
+ ret = kad_compile_array(n_node, n_roots, roots);
+ free(roots);
+ return ret;
+}
+
+/************************************
+ * Miscellaneous on compiled graphs *
+ ************************************/
+
+void kad_delete(int n, kad_node_t **a)
+{
+ int i;
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = a[i];
+ if (p->n_child) {
+ free(p->x); free(p->g);
+ }
+ free(p->child); free(p->ptr); free(p->gtmp); free(p);
+ }
+ free(a);
+}
+
+int kad_size_var(int n, kad_node_t *const* v)
+{
+ int c, i;
+ for (i = c = 0; i < n; ++i)
+ if (kad_is_var(v[i]))
+ c += kad_len(v[i]);
+ return c;
+}
+
+int kad_size_const(int n, kad_node_t *const* v)
+{
+ int c, i;
+ for (i = c = 0; i < n; ++i)
+ if (kad_is_const(v[i]))
+ c += kad_len(v[i]);
+ return c;
+}
+
+/**********************************
+ * Computate values and gradients *
+ **********************************/
+
+static void kad_propagate_marks(int n, kad_node_t **a)
+{
+ int i, j;
+ for (i = n - 1; i >= 0; --i) {
+ kad_node_t *p = a[i];
+ if (p->tmp > 0) {
+ if (kad_is_switch(p)) {
+ int32_t *aux = (int32_t*)p->ptr;
+ if (p->child[*aux]->tmp == 0)
+ p->child[*aux]->tmp = 1;
+ } else {
+ for (j = 0; j < p->n_child; ++j)
+ if (p->child[j]->tmp == 0)
+ p->child[j]->tmp = 1;
+ }
+ }
+ }
+}
+
+void kad_eval_marked(int n, kad_node_t **a)
+{
+ int i;
+ kad_propagate_marks(n, a);
+ for (i = 0; i < n; ++i)
+ if (a[i]->n_child && a[i]->tmp > 0)
+ kad_op_list[a[i]->op](a[i], KAD_FORWARD);
+ for (i = 0; i < n; ++i) a[i]->tmp = 0;
+}
+
+const float *kad_eval_at(int n, kad_node_t **a, int from)
+{
+ int i;
+ if (from < 0 || from >= n) from = n - 1;
+ for (i = 0; i < n; ++i) a[i]->tmp = (i == from);
+ kad_eval_marked(n, a);
+ return a[from]->x;
+}
+
+void kad_grad(int n, kad_node_t **a, int from)
+{
+ int i;
+ if (from < 0 || from >= n) from = n - 1;
+ assert(a[from]->n_d == 0);
+ for (i = 0; i < n; ++i) a[i]->tmp = (i == from);
+ kad_propagate_marks(n, a);
+ for (i = 0; i <= from; ++i) /* set all grandients to zero */
+ if (a[i]->g && a[i]->tmp > 0)
+ memset(a[i]->g, 0, kad_len(a[i]) * sizeof(float));
+ for (i = from, a[i]->g[0] = 1.0f; i >= 0; --i) /* backprop */
+ if (a[i]->n_child && a[i]->tmp > 0)
+ kad_op_list[a[i]->op](a[i], KAD_BACKWARD);
+ for (i = 0; i <= from; ++i) a[i]->tmp = 0;
+}
+
+/***********************
+ * Load and save graph *
+ ***********************/
+
+static void kad_save1(FILE *fp, const kad_node_t *p)
+{
+ fwrite(&p->ext_label, 4, 1, fp);
+ fwrite(&p->ext_flag, 4, 1, fp);
+ fwrite(&p->flag, 1, 1, fp);
+ fwrite(&p->n_child, 4, 1, fp);
+ if (p->n_child) {
+ int32_t j, pre = p->pre? p->pre->tmp : -1;
+ fwrite(&p->op, 2, 1, fp);
+ for (j = 0; j < p->n_child; ++j)
+ fwrite(&p->child[j]->tmp, 4, 1, fp);
+ fwrite(&pre, 4, 1, fp);
+ fwrite(&p->ptr_size, 4, 1, fp);
+ if (p->ptr_size > 0 && p->ptr)
+ fwrite(p->ptr, p->ptr_size, 1, fp);
+ } else {
+ fwrite(&p->n_d, 1, 1, fp);
+ if (p->n_d) fwrite(p->d, 4, p->n_d, fp);
+ }
+}
+
+static kad_node_t *kad_load1(FILE *fp, kad_node_t **node)
+{
+ kad_node_t *p;
+ p = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ (void) !fread(&p->ext_label, 4, 1, fp);
+ (void) !fread(&p->ext_flag, 4, 1, fp);
+ (void) !fread(&p->flag, 1, 1, fp);
+ (void) !fread(&p->n_child, 4, 1, fp);
+ if (p->n_child) {
+ int32_t j, k;
+ p->child = (kad_node_t**)calloc(p->n_child, sizeof(kad_node_t*));
+ (void) !fread(&p->op, 2, 1, fp);
+ for (j = 0; j < p->n_child; ++j) {
+ (void) !fread(&k, 4, 1, fp);
+ p->child[j] = node? node[k] : 0;
+ }
+ (void) !fread(&k, 4, 1, fp);
+ if (k >= 0) p->pre = node[k];
+ (void) !fread(&p->ptr_size, 4, 1, fp);
+ if (p->ptr_size > 0) {
+ p->ptr = malloc(p->ptr_size);
+ (void) !fread(p->ptr, p->ptr_size, 1, fp);
+ }
+ } else {
+ (void) !fread(&p->n_d, 1, 1, fp);
+ if (p->n_d) (void) !fread(p->d, 4, p->n_d, fp);
+ }
+ return p;
+}
+
+int kad_save(FILE *fp, int n_node, kad_node_t **node)
+{
+ int32_t i, k = n_node;
+ fwrite(&k, 4, 1, fp);
+ for (i = 0; i < n_node; ++i) node[i]->tmp = i;
+ for (i = 0; i < n_node; ++i) kad_save1(fp, node[i]);
+ for (i = 0; i < n_node; ++i) node[i]->tmp = 0;
+ return 0;
+}
+
+kad_node_t **kad_load(FILE *fp, int *_n_node)
+{
+ int32_t i, n_node;
+ kad_node_t **node;
+ (void) !fread(&n_node, 4, 1, fp);
+ node = (kad_node_t**)malloc(n_node * sizeof(kad_node_t*));
+ for (i = 0; i < n_node; ++i) {
+ kad_node_t *p;
+ p = node[i] = kad_load1(fp, node);
+ if (p->n_child) {
+ kad_op_list[p->op](p, KAD_ALLOC);
+ kad_op_list[p->op](p, KAD_SYNC_DIM);
+ }
+ }
+ *_n_node = n_node;
+ kad_mark_back(n_node, node);
+ return node;
+}
+
+/***************
+ * Graph clone *
+ ***************/
+
+static inline kad_node_t *kad_dup1(const kad_node_t *p)
+{
+ kad_node_t *q;
+ q = (kad_node_t*)malloc(sizeof(kad_node_t));
+ memcpy(q, p, sizeof(kad_node_t));
+ q->pre = 0, q->tmp = 0, q->gtmp = 0;
+ if (p->ptr && p->ptr_size > 0) {
+ if (kad_use_rng(p) && !(p->flag & KAD_SHARE_RNG) && p->ptr_size == sizeof(kad_rng_t)) {
+ q->ptr = kad_rng(); /* each time step uses a different RNG */
+ } else {
+ q->ptr = malloc(p->ptr_size);
+ memcpy(q->ptr, p->ptr, p->ptr_size);
+ }
+ }
+ if (q->n_child) {
+ q->x = q->g = 0;
+ q->child = (kad_node_t**)calloc(q->n_child, sizeof(kad_node_t*));
+ }
+ return q;
+}
+
+kad_node_t **kad_clone(int n, kad_node_t **v, int batch_size)
+{
+ int i, j;
+ kad_node_t **u;
+ u = (kad_node_t**)calloc(n, sizeof(kad_node_t*));
+ for (i = 0; i < n; ++i) v[i]->tmp = i;
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = v[i], *q;
+ q = u[i] = kad_dup1(p);
+ if (p->pre) q->pre = u[p->pre->tmp];
+ if (p->n_child) {
+ for (j = 0; j < p->n_child; ++j)
+ q->child[j] = u[p->child[j]->tmp];
+ } else if (!kad_is_feed(p)) {
+ q->x = (float*)malloc(kad_len(p) * sizeof(float));
+ memcpy(q->x, p->x, kad_len(p) * sizeof(float));
+ q->g = 0;
+ }
+ }
+ for (i = 0; i < n; ++i) v[i]->tmp = 0;
+ kad_sync_dim(n, u, batch_size); /* this will allocate x[] and g[] at internal nodes */
+ return u;
+}
+
+/**************
+ * Unroll RNN *
+ **************/
+
+typedef struct {
+ int32_t n, m;
+ kad_node_t **v;
+} nodes_t;
+
+static inline void push_nodes(nodes_t *w, kad_node_t *p)
+{
+ if (w->n == w->m) {
+ w->m = w->m? w->m<<1 : 16;
+ w->v = (kad_node_t**)realloc(w->v, w->m * sizeof(kad_node_t*));
+ }
+ w->v[w->n++] = p;
+}
+
+static void kad_unroll_helper(int n_v, kad_node_t **v, int i_pivot, kad_node_t **t, int len, nodes_t *w)
+{
+ int i, j, l;
+ uint8_t *flag;
+ kad_node_t **aux;
+
+ assert(kad_is_pivot(v[i_pivot]) && t[i_pivot] == 0);
+ t[i_pivot] = kad_dup1(v[i_pivot]);
+ t[i_pivot]->n_child = len;
+ t[i_pivot]->child = (kad_node_t**)realloc(t[i_pivot]->child, len * sizeof(kad_node_t*));
+
+ flag = (uint8_t*)calloc(n_v, 1);
+ for (i = i_pivot, flag[i] = 16; i >= 0; --i) {
+ if (i < i_pivot && kad_is_pivot(v[i])) continue; /* don't trespass other pivots */
+ if (flag[i]&16) /* flag 16: nodes to unroll */
+ for (j = 0; j < v[i]->n_child; ++j)
+ flag[v[i]->child[j]->tmp] = 16;
+ }
+ for (i = 0; i < i_pivot; ++i) {
+ if (!(flag[i]&16)) continue;
+ if (kad_is_var(v[i]) || kad_is_const(v[i]) || kad_is_pivot(v[i])) flag[i] |= 1; /* external nodes that should not be duplicated */
+ if (v[i]->pre) flag[v[i]->pre->tmp] |= 2;
+ }
+ flag[v[i_pivot]->child[0]->tmp] |= 4;
+ aux = (kad_node_t**)calloc(n_v, sizeof(kad_node_t*));
+ for (l = 0; l < len; ++l) {
+ for (i = 0; i < i_pivot; ++i) {
+ if (!(flag[i]&16) || ((flag[i]&3) && t[i])) continue;
+ t[i] = kad_dup1(v[i]);
+ if (v[i]->n_child)
+ for (j = 0; j < v[i]->n_child; ++j)
+ t[i]->child[j] = t[v[i]->child[j]->tmp];
+ if (flag[i]&4) t[i_pivot]->child[l] = t[i];
+ if (l == 0 && (flag[i]&2)) aux[i] = t[i];
+ if (v[i]->pre) {
+ t[v[i]->pre->tmp] = t[i];
+ if (l == len - 1) t[i]->pre = aux[v[i]->pre->tmp]; /* this forms a cycle! */
+ }
+ push_nodes(w, t[i]);
+ }
+ }
+ push_nodes(w, t[i_pivot]);
+ free(aux); free(flag);
+}
+
+int kad_n_pivots(int n_v, kad_node_t **v)
+{
+ int i, n_pivots = 0;
+ for (i = 0; i < n_v; ++i)
+ if (kad_is_pivot(v[i])) ++n_pivots;
+ return n_pivots;
+}
+
+kad_node_t **kad_unroll(int n_v, kad_node_t **v, int *new_n, int *len)
+{
+ int i, j, n_pivots = 0;
+ kad_node_t **t;
+ nodes_t w = {0,0,0};
+
+ t = (kad_node_t**)calloc(n_v, sizeof(kad_node_t*));
+ n_pivots = kad_n_pivots(n_v, v);
+ for (i = 0; i < n_v; ++i) v[i]->tmp = i;
+ if (n_pivots) {
+ int k, *i_pivots;
+ i_pivots = (int*)calloc(n_pivots, sizeof(int));
+ for (i = k = 0; i < n_v; ++i) /* collect pivots */
+ if (kad_is_pivot(v[i])) i_pivots[k++] = i;
+ for (i = 0; i < n_pivots; ++i) /* unroll each pivot, from the lowest to the highest */
+ kad_unroll_helper(n_v, v, i_pivots[i], t, len[i], &w);
+ free(i_pivots);
+ }
+ for (i = 0; i < n_v; ++i) { /* copy over the rest of nodes */
+ if (t[i]) continue;
+ t[i] = kad_dup1(v[i]);
+ if (v[i]->n_child)
+ for (j = 0; j < v[i]->n_child; ++j)
+ t[i]->child[j] = t[v[i]->child[j]->tmp];
+ push_nodes(&w, t[i]);
+ }
+ free(t);
+ for (i = 0; i < n_v; ++i) v[i]->tmp = 0;
+ for (i = 0; i < w.n; ++i) /* stack may change the output dimension */
+ if (w.v[i]->n_child > 0)
+ kad_op_list[w.v[i]->op](w.v[i], KAD_SYNC_DIM);
+ kad_allocate_internal(w.n, w.v);
+ *new_n = w.n;
+ return w.v;
+}
+
+/********************************
+ * Vector and matrix operations *
+ ********************************/
+
+#ifdef __SSE__
+#include <xmmintrin.h>
+
+static inline float kad_sdot(int n, const float *x, const float *y) /* BLAS sdot using SSE */
+{
+ int i, n8 = n>>3<<3;
+ __m128 vs1, vs2;
+ float s, t[4];
+ vs1 = _mm_setzero_ps();
+ vs2 = _mm_setzero_ps();
+ for (i = 0; i < n8; i += 8) {
+ __m128 vx1, vx2, vy1, vy2;
+ vx1 = _mm_loadu_ps(&x[i]);
+ vx2 = _mm_loadu_ps(&x[i+4]);
+ vy1 = _mm_loadu_ps(&y[i]);
+ vy2 = _mm_loadu_ps(&y[i+4]);
+ vs1 = _mm_add_ps(vs1, _mm_mul_ps(vx1, vy1));
+ vs2 = _mm_add_ps(vs2, _mm_mul_ps(vx2, vy2));
+ }
+ for (s = 0.; i < n; ++i) s += x[i] * y[i];
+ _mm_storeu_ps(t, vs1);
+ s += t[0] + t[1] + t[2] + t[3];
+ _mm_storeu_ps(t, vs2);
+ s += t[0] + t[1] + t[2] + t[3];
+ return s;
+}
+static inline void kad_saxpy_inlined(int n, float a, const float *x, float *y) /* BLAS saxpy using SSE */
+{
+ int i, n8 = n>>3<<3;
+ __m128 va;
+ va = _mm_set1_ps(a);
+ for (i = 0; i < n8; i += 8) {
+ __m128 vx1, vx2, vy1, vy2, vt1, vt2;
+ vx1 = _mm_loadu_ps(&x[i]);
+ vx2 = _mm_loadu_ps(&x[i+4]);
+ vy1 = _mm_loadu_ps(&y[i]);
+ vy2 = _mm_loadu_ps(&y[i+4]);
+ vt1 = _mm_add_ps(_mm_mul_ps(va, vx1), vy1);
+ vt2 = _mm_add_ps(_mm_mul_ps(va, vx2), vy2);
+ _mm_storeu_ps(&y[i], vt1);
+ _mm_storeu_ps(&y[i+4], vt2);
+ }
+ for (; i < n; ++i) y[i] += a * x[i];
+}
+#else
+static inline float kad_sdot(int n, const float *x, const float *y) /* BLAS sdot */
+{
+ int i;
+ float s = 0.;
+ for (i = 0; i < n; ++i) s += x[i] * y[i];
+ return s;
+}
+static inline void kad_saxpy_inlined(int n, float a, const float *x, float *y) // BLAS saxpy
+{
+ int i;
+ for (i = 0; i < n; ++i) y[i] += a * x[i];
+}
+#endif
+
+void kad_vec_mul_sum(int n, float *a, const float *b, const float *c)
+{
+ int i;
+ for (i = 0; i < n; ++i) a[i] += b[i] * c[i];
+}
+
+/* This is actually lapack not cblas, but this definition is used */
+#ifdef HAVE_CBLAS
+#ifndef __APPLE__
+/* As gfortran mangles names */
+#define ssyev ssyev_
+#endif
+extern void ssyev(const char* jobz, const char* uplo, int* n, float* a, int* lda, float* w, float* work, int* lwork, int* info);
+#endif
+
+#ifdef HAVE_CBLAS_SGEMM
+
+#ifdef HAVE_CBLAS_H
+#include "cblas.h"
+#else
+/* Poor man approach, thanks for that Apple */
+enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102 };
+enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112 };
+extern void cblas_sgemm(const enum CBLAS_ORDER Order,
+ const enum CBLAS_TRANSPOSE TA,
+ const enum CBLAS_TRANSPOSE TB,
+ const int M, const int N, const int K,
+ const float alpha, const float *A, const int lda,
+ const float *B, const int ldb, const float beta,
+ float *C, const int ldc);
+#endif
+
+void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C)
+{
+ cblas_sgemm(CblasRowMajor, trans_A? CblasTrans : CblasNoTrans, trans_B? CblasTrans : CblasNoTrans, M, N, K, 1.0f, A, trans_A? M : K, B, trans_B? K : N, 1.0f, C, N);
+}
+#else
+void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C) /* simplified BLAS sgemm */
+{
+ static const int x = 16;
+ int i, j, k;
+ if (!trans_A && trans_B) {
+ for (i = 0; i < M; i += x)
+ for (j = 0; j < N; j += x) {
+ int ii, ie = M < i + x? M : i + x;
+ int jj, je = N < j + x? N : j + x;
+ for (ii = i; ii < ie; ++ii) { /* loop tiling */
+ const float *aii = A + ii * K, *bjj;
+ float *cii = C + ii * N;
+ for (jj = j, bjj = B + j * K; jj < je; ++jj, bjj += K)
+ cii[jj] += kad_sdot(K, aii, bjj);
+ }
+ }
+ } else if (!trans_A && !trans_B) {
+ for (i = 0; i < M; ++i)
+ for (k = 0; k < K; ++k)
+ kad_saxpy_inlined(N, A[i*K+k], &B[k*N], &C[i*N]);
+ } else if (trans_A && !trans_B) {
+ for (k = 0; k < K; ++k)
+ for (i = 0; i < M; ++i)
+ kad_saxpy_inlined(N, A[k*M+i], &B[k*N], &C[i*N]);
+ } else abort(); /* not implemented for (trans_A && trans_B) */
+}
+#endif
+
+#ifdef HAVE_CBLAS_SAXPY
+#ifndef HAVE_CBLAS_H
+extern void cblas_saxpy(const int __N,
+ const float __alpha, const float *__X, const int __incX, float *__Y, const int __incY);
+#endif
+
+void kad_saxpy(int n, float a, const float *x, float *y) { cblas_saxpy(n, a, x, 1, y, 1); }
+#else
+void kad_saxpy(int n, float a, const float *x, float *y) { kad_saxpy_inlined(n, a, x, y); }
+#endif
+
+bool kad_ssyev_simple(int N, float *A, float *eigenvals)
+{
+#ifndef HAVE_CBLAS
+ return false;
+#else
+ int n = N, lda = N, info, lwork;
+ float wkopt;
+ float *work;
+
+ /* Query and allocate the optimal workspace */
+ lwork = -1;
+ ssyev ("Vectors", "Upper", &n, A, &lda, eigenvals, &wkopt, &lwork, &info);
+ lwork = wkopt;
+ work = (float*) g_malloc(lwork * sizeof(double));
+ ssyev ("Vectors", "Upper", &n, A, &lda, eigenvals, work, &lwork, &info);
+ /* Check for convergence */
+ if (info > 0) {
+ g_free (work);
+
+ return false;
+ }
+
+ g_free (work);
+
+ return true;
+#endif
+}
+
+/***************************
+ * Random number generator *
+ ***************************/
+
+static kad_rng_t kad_rng_dat = { {0x50f5647d2380309dULL, 0x91ffa96fc4c62cceULL}, 0.0, 0, 0 };
+
+static inline uint64_t kad_splitmix64(uint64_t x)
+{
+ uint64_t z = (x += 0x9E3779B97F4A7C15ULL);
+ z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
+ z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
+ return z ^ (z >> 31);
+}
+
+static inline uint64_t kad_xoroshiro128plus_next(kad_rng_t *r)
+{
+ const uint64_t s0 = r->s[0];
+ uint64_t s1 = r->s[1];
+ const uint64_t result = s0 + s1;
+ s1 ^= s0;
+ r->s[0] = (s0 << 55 | s0 >> 9) ^ s1 ^ (s1 << 14);
+ r->s[1] = s0 << 36 | s0 >> 28;
+ return result;
+}
+
+static inline void kad_xoroshiro128plus_jump(kad_rng_t *r)
+{
+ static const uint64_t JUMP[] = { 0xbeac0467eba5facbULL, 0xd86b048b86aa9922ULL };
+ uint64_t s0 = 0, s1 = 0;
+ int i, b;
+ for (i = 0; i < 2; ++i)
+ for (b = 0; b < 64; b++) {
+ if (JUMP[i] & 1ULL << b)
+ s0 ^= r->s[0], s1 ^= r->s[1];
+ kad_xoroshiro128plus_next(r);
+ }
+ r->s[0] = s0, r->s[1] = s1;
+}
+
+void kad_srand(void *d, uint64_t seed)
+{
+ kad_rng_t *r = d? (kad_rng_t*)d : &kad_rng_dat;
+ r->n_gset = 0.0, r->n_iset = 0;
+ r->s[0] = kad_splitmix64(seed);
+ r->s[1] = kad_splitmix64(r->s[0]);
+}
+
+void *kad_rng(void)
+{
+ kad_rng_t *r;
+ r = (kad_rng_t*)calloc(1, sizeof(kad_rng_t));
+ kad_xoroshiro128plus_jump(&kad_rng_dat);
+ r->s[0] = kad_rng_dat.s[0], r->s[1] = kad_rng_dat.s[1];
+ return r;
+}
+
+uint64_t kad_rand(void *d) { return kad_xoroshiro128plus_next(d? (kad_rng_t*)d : &kad_rng_dat); }
+
+double kad_drand(void *d)
+{
+ union { uint64_t i; double d; } u;
+ u.i = 0x3FFULL << 52 | kad_xoroshiro128plus_next(d? (kad_rng_t*)d : &kad_rng_dat) >> 12;
+ return u.d - 1.0;
+}
+
+double kad_drand_normal(void *d)
+{
+ kad_rng_t *r = d? (kad_rng_t*)d : &kad_rng_dat;
+ if (r->n_iset == 0) {
+ double fac, rsq, v1, v2;
+ do {
+ v1 = 2.0 * kad_drand(d) - 1.0;
+ v2 = 2.0 * kad_drand(d) - 1.0;
+ rsq = v1 * v1 + v2 * v2;
+ } while (rsq >= 1.0 || rsq == 0.0);
+ fac = sqrt(-2.0 * log(rsq) / rsq);
+ r->n_gset = v1 * fac;
+ r->n_iset = 1;
+ return v2 * fac;
+ } else {
+ r->n_iset = 0;
+ return r->n_gset;
+ }
+}
+
+/*************
+ * Operators *
+ *************/
+
+static inline void kad_copy_dim1(kad_node_t *dst, const kad_node_t *src) /* set the dimension/shape of dst to src */
+{
+ dst->n_d = src->n_d;
+ if (src->n_d) memcpy(dst->d, src->d, src->n_d * sizeof(int));
+}
+
+/********** Arithmetic operations **********/
+
+int kad_op_add(kad_node_t *p, int action)
+{
+ int i, n0, n1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], n0 = kad_len(q[0]);
+ q[1] = p->child[1], n1 = kad_len(q[1]);
+ if (action == KAD_SYNC_DIM) {
+ if (n0 % n1 != 0) return -1;
+ kad_copy_dim1(p, q[0]);
+ } else if (action == KAD_FORWARD) {
+ assert(n0 >= n1);
+ memcpy(p->x, q[0]->x, n0 * sizeof(float));
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, 1.0f, q[1]->x, p->x + i);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0])) kad_saxpy(n0, 1.0f, p->g, q[0]->g);
+ if (kad_is_back(q[1]))
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, 1.0f, p->g + i, q[1]->g);
+ }
+ return 0;
+}
+
+int kad_op_sub(kad_node_t *p, int action)
+{
+ int i, n0, n1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], n0 = kad_len(q[0]);
+ q[1] = p->child[1], n1 = kad_len(q[1]);
+ if (action == KAD_SYNC_DIM) {
+ if (n0 % n1 != 0) return -1;
+ kad_copy_dim1(p, q[0]);
+ } else if (action == KAD_FORWARD) {
+ assert(n0 >= n1);
+ memcpy(p->x, q[0]->x, n0 * sizeof(float));
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, -1.0f, q[1]->x, p->x + i);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0])) kad_saxpy(n0, 1.0f, p->g, q[0]->g);
+ if (kad_is_back(q[1]))
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, -1.0f, p->g + i, q[1]->g);
+ }
+ return 0;
+}
+
+int kad_op_mul(kad_node_t *p, int action)
+{
+ int i, n0, n1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], n0 = kad_len(q[0]);
+ q[1] = p->child[1], n1 = kad_len(q[1]);
+ if (action == KAD_SYNC_DIM) {
+ if (n0 % n1 != 0) return -1;
+ kad_copy_dim1(p, q[0]);
+ } else if (action == KAD_FORWARD) {
+ assert(n0 >= n1);
+ memset(p->x, 0, n0 * sizeof(float));
+ if (q[0]->x != 0 && q[1]->x != 0)
+ for (i = 0; i < n0; i += n1) /* TODO: optimize when n1==1 */
+ kad_vec_mul_sum(n1, p->x + i, q[0]->x + i, q[1]->x);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0]) && q[1]->x)
+ for (i = 0; i < n0; i += n1)
+ kad_vec_mul_sum(n1, q[0]->g + i, p->g + i, q[1]->x);
+ if (kad_is_back(q[1]) && q[0]->x)
+ for (i = 0; i < n0; i += n1)
+ kad_vec_mul_sum(n1, q[1]->g, p->g + i, q[0]->x + i);
+ }
+ return 0;
+}
+
+int kad_op_cmul(kad_node_t *p, int action)
+{
+ int i, n_a_row, n_b_row, n_col, n_a_col = 1, n_b_col = 1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], q[1] = p->child[1];
+ n_col = q[0]->d[q[0]->n_d - 1] > q[1]->d[q[1]->n_d - 1]? q[0]->d[q[0]->n_d - 1] : q[1]->d[q[1]->n_d - 1];
+ for (i = q[0]->n_d - 1; i >= 0; --i) if (n_a_col < n_col) n_a_col *= q[0]->d[i];
+ for (i = q[1]->n_d - 1; i >= 0; --i) if (n_b_col < n_col) n_b_col *= q[1]->d[i];
+ n_a_row = kad_len(q[0]) / n_a_col, n_b_row = kad_len(q[1]) / n_b_col;
+ if (action == KAD_SYNC_DIM) {
+ if (n_a_col != n_b_col) return -1;
+ p->n_d = 2, p->d[0] = n_a_row, p->d[1] = n_b_row;
+ } else if (action == KAD_FORWARD) {
+ memset(p->x, 0, n_a_row * n_b_row * sizeof(float));
+ if (q[0]->x && q[1]->x)
+ kad_sgemm_simple(0, 1, n_a_row, n_b_row, n_col, q[0]->x, q[1]->x, p->x); /* Y = X * trans(W) */
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0]) && q[1]->x)
+ kad_sgemm_simple(0, 0, n_a_row, n_col, n_b_row, p->g, q[1]->x, q[0]->g); /* G_x <- G_y * W */
+ if (kad_is_back(q[1]) && q[0]->x)
+ kad_sgemm_simple(1, 0, n_b_row, n_col, n_a_row, p->g, q[0]->x, q[1]->g); /* G_w <- trans(G_y) * X */
+ }
+ return 0;
+}
+
+int kad_op_matmul(kad_node_t *p, int action) /* TODO: matmul and cmul have different broadcasting rules */
+{
+ int n_a_row, n_b_row, n_a_col, n_b_col;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0];
+ q[1] = p->child[1];
+ n_a_row = q[0]->n_d == 1? 1 : q[0]->d[0];
+ n_b_row = q[1]->n_d == 1? 1 : q[1]->d[0];
+ n_a_col = kad_len(q[0]) / n_a_row;
+ n_b_col = kad_len(q[1]) / n_b_row;
+ if (action == KAD_SYNC_DIM) {
+ if (n_a_col != n_b_row) return -1;
+ p->n_d = 2, p->d[0] = n_a_row, p->d[1] = n_b_col;
+ } else if (action == KAD_FORWARD) {
+ memset(p->x, 0, n_a_row * n_b_col * sizeof(float));
+ if (q[0]->x && q[1]->x)
+ kad_sgemm_simple(0, 0, n_a_row, n_b_col, n_a_col, q[0]->x, q[1]->x, p->x); /* Y = X * W */
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0]) && q[1]->x)
+ kad_sgemm_simple(0, 1, n_a_row, n_a_col, n_b_col, p->g, q[1]->x, q[0]->g); /* G_x <- G_y * trans(W) */
+ if (kad_is_back(q[1]) && q[0]->x)
+ kad_sgemm_simple(1, 0, n_b_row, n_b_col, n_a_row, q[0]->x, p->g, q[1]->g); /* G_y <- trans(A) * G_y */
+ }
+ return 0;
+}
+
+int kad_op_square(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i)
+ p->x[i] = q->x[i] * q->x[i];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * (q->x[i] + q->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_1minus(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = 1.0f - q->x[i];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ kad_saxpy(n, -1.0f, p->g, q->g);
+ }
+ return 0;
+}
+
+int kad_op_exp(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = expf(q->x[i]);
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * p->x[i];
+ }
+ return 0;
+}
+
+int kad_op_log(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = logf(q->x[i]);
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] / q->x[i];
+ }
+ return 0;
+}
+
+int kad_op_reduce_sum(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int i, j, k, axis, d0, d1;
+
+ assert(p->ptr);
+ axis = *(int32_t*)p->ptr;
+ if (axis < 0 || axis >= q->n_d) return -1;
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ p->n_d = q->n_d - 1;
+ for (i = j = 0; i < q->n_d; ++i)
+ if (i != axis) p->d[j++] = q->d[i];
+ } else if (action == KAD_FORWARD) {
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ p->x[i * d1 + k] += q->x[(i * q->d[axis] + j) * d1 + k];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ q->g[(i * q->d[axis] + j) * d1 + k] += p->g[i * d1 + k];
+ }
+ return 0;
+}
+
+int kad_op_reduce_mean(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int i, j, k, axis, d0, d1;
+
+ assert(p->ptr);
+ axis = *(int32_t*)p->ptr;
+ if (axis < 0 || axis >= q->n_d) return -1;
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ p->n_d = q->n_d - 1;
+ for (i = j = 0; i < q->n_d; ++i)
+ if (i != axis) p->d[j++] = q->d[i];
+ } else if (action == KAD_FORWARD) {
+ float t = 1.0f / q->d[axis];
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ p->x[i * d1 + k] += t * q->x[(i * q->d[axis] + j) * d1 + k];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ float t = 1.0f / q->d[axis];
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ q->g[(i * q->d[axis] + j) * d1 + k] += t * p->g[i * d1 + k];
+ }
+ return 0;
+}
+
+/********** Miscellaneous **********/
+
+int kad_op_dropout(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ assert(p->child[1]->n_d == 0);
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_ALLOC) {
+ if (kad_is_back(p->child[0]))
+ p->gtmp = realloc(p->gtmp, n);
+ } else if (action == KAD_FORWARD) {
+ float r = kad_is_const(q) || kad_is_var(q)? 0.0f : *p->child[1]->x, z = 1.0f / (1.0f - r);
+ uint8_t *flag = (uint8_t*)p->gtmp;
+ for (i = 0; i < n; ++i) {
+ int kept = (kad_drand(p->ptr) >= r);
+ p->x[i] = kept? q->x[i] * z : 0.0f;
+ if (flag) flag[i] = kept;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(p->child[0])) {
+ float r = kad_is_const(q) || kad_is_var(q)? 0.0f : *p->child[1]->x, z = 1.0f / (1.0f - r);
+ uint8_t *flag = (uint8_t*)p->gtmp;
+ for (i = 0; i < n; ++i)
+ if (flag[i]) q->g[i] += z * p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_sample_normal(kad_node_t *p, int action) /* not tested */
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_ALLOC) {
+ if (kad_is_back(p->child[0]))
+ p->gtmp = realloc(p->gtmp, n * sizeof(float));
+ } else if (action == KAD_FORWARD) {
+ float *r = (float*)p->gtmp;
+ for (i = 0; i < n; ++i) {
+ float z;
+ z = (float)kad_drand_normal(p->ptr);
+ p->x[i] = q->x[i] * z;
+ if (r) r[i] = z;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(p->child[0])) {
+ float *r = (float*)p->gtmp;
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * r[i];
+ }
+ return 0;
+}
+
+int kad_op_slice(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int32_t *aux, *range;
+ int i, axis, d0, d1;
+
+ assert(p->ptr);
+ aux = (int32_t*)p->ptr, axis = aux[0], range = aux + 1;
+ if (axis < 0 || axis >= q->n_d) return -1;
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ if (range[0] >= range[1] || range[0] < 0 || range[1] > q->d[axis]) return -1;
+ kad_copy_dim1(p, q);
+ p->d[axis] = range[1] - range[0];
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < d0; ++i)
+ memcpy(&p->x[i * p->d[axis] * d1], &q->x[(i * q->d[axis] + range[0]) * d1], (range[1] - range[0]) * d1 * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < d0; ++i)
+ kad_saxpy((range[1] - range[0]) * d1, 1.0f, &p->g[i * p->d[axis] * d1], &q->g[(i * q->d[axis] + range[0]) * d1]);
+ }
+ return 0;
+}
+
+int kad_op_concat(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int32_t *aux;
+ int i, j, k, axis, d0, d1;
+
+ assert(p->ptr);
+ aux = (int32_t*)p->ptr, axis = aux[0];
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ for (i = 1; i < p->n_child; ++i) {
+ if (p->child[i]->n_d != q->n_d) return -1;
+ for (j = 0; j < q->n_d; ++j)
+ if (j != axis && q->d[j] != p->child[i]->d[j]) return -1;
+ }
+ kad_copy_dim1(p, q);
+ for (i = 1; i < p->n_child; ++i)
+ p->d[axis] += p->child[i]->d[axis];
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < d0; ++i)
+ for (j = k = 0; j < p->n_child; ++j) {
+ q = p->child[j];
+ memcpy(&p->x[(i * p->d[axis] + k) * d1], &q->x[i * q->d[axis] * d1], q->d[axis] * d1 * sizeof(float));
+ k += q->d[axis];
+ }
+ } else if (action == KAD_BACKWARD) {
+ for (i = 0; i < d0; ++i)
+ for (j = k = 0; j < p->n_child; ++j) {
+ q = p->child[j];
+ if (!kad_is_back(q)) continue;
+ kad_saxpy(q->d[axis] * d1, 1.0f, &p->g[(i * p->d[axis] + k) * d1], &q->g[i * q->d[axis] * d1]);
+ k += q->d[axis];
+ }
+ }
+ return 0;
+}
+
+int kad_op_reshape(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+
+ if (action == KAD_SYNC_DIM) {
+ if (p->ptr) {
+ int32_t *aux = (int32_t*)p->ptr;
+ int i, len = 1, n_missing = 0;
+ p->n_d = p->ptr_size / 4;
+ for (i = 0; i < p->n_d; ++i) p->d[i] = aux[i];
+ for (i = 0; i < p->n_d; ++i)
+ if (p->d[i] <= 0) ++n_missing;
+ else len *= p->d[i];
+ if (n_missing == 0 && len != kad_len(q)) return -1;
+ if (n_missing > 1) { /* attempt to infer missing dimensions except the last one */
+ for (i = 0; i < p->n_d; ++i)
+ if (p->d[i] <= 0 && i < q->n_d) {
+ p->d[i] = q->d[i], len *= p->d[i];
+ if (--n_missing == 1) break;
+ }
+ if (n_missing > 1) return -1;
+ }
+ if (n_missing == 1) { /* infer the last missing dimension */
+ if (kad_len(q) % len != 0) return -1;
+ for (i = 0; i < p->n_d; ++i)
+ if (p->d[i] <= 0) p->d[i] = kad_len(q) / len;
+ }
+ } else kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ memcpy(p->x, q->x, kad_len(p) * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ kad_saxpy(kad_len(p), 1.0f, p->g, q->g);
+ }
+ return 0;
+}
+
+int kad_op_reverse(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int axis, i, j, n, d0, d1;
+
+ axis = p->ptr? *(int32_t*)p->ptr : 0;
+ if (axis < 0) axis += q->n_d;
+ assert(axis >= 0 && axis < q->n_d);
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ n = q->d[axis];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < n; ++j)
+ memcpy(&p->x[(i * n + n - 1 - j) * d1], &q->x[(i * n + j) * d1], d1 * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < n; ++j)
+ kad_saxpy(d1, 1.0f, &p->g[(i * n + n - 1 - j) * d1], &q->g[(i * n + j) * d1]);
+ }
+ return 0;
+}
+
+/********** Cost functions **********/
+
+int kad_op_mse(kad_node_t *p, int action)
+{
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ int i, n;
+
+ n = kad_len(y0);
+ if (action == KAD_SYNC_DIM) {
+ if (n != kad_len(y1)) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ for (i = 0; i < n; ++i)
+ cost += (y1->x[i] - y0->x[i]) * (y1->x[i] - y0->x[i]);
+ p->x[0] = (float)(cost / n);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = 2.0f * p->g[0] / n;
+ for (i = 0; i < n; ++i)
+ y1->g[i] += t * (y1->x[i] - y0->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_ce_bin(kad_node_t *p, int action)
+{
+ static const float tiny = 1e-9f;
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ int i, n;
+
+ n = kad_len(y0);
+ if (action == KAD_SYNC_DIM) {
+ if (n != kad_len(y1)) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ for (i = 0; i < n; ++i) {
+ if (y0->x[i] > 0.0f)
+ cost += y0->x[i] * log(y0->x[i] / (y1->x[i] > tiny? y1->x[i] : tiny));
+ if (1.0f - y0->x[i] > 0.0f)
+ cost += (1.0f - y0->x[i]) * log((1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny));
+ }
+ p->x[0] = (float)(cost / n);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = p->g[0] / n;
+ for (i = 0; i < n; ++i) {
+ if (y0->x[i] > 0.0f)
+ y1->g[i] -= t * y0->x[i] / (y1->x[i] > tiny? y1->x[i] : tiny);
+ if (1.0f - y0->x[i] > 0.0f)
+ y1->g[i] += t * (1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny);
+ }
+ }
+ return 0;
+}
+
+int kad_op_ce_bin_neg(kad_node_t *p, int action)
+{
+ static const float tiny = 1e-9f;
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ int i, n;
+
+ n = kad_len(y0);
+ if (action == KAD_SYNC_DIM) {
+ if (n != kad_len(y1)) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ for (i = 0; i < n; ++i) {
+ if (1.0f + y0->x[i] > 0.0f)
+ cost += .5f * (1.0f + y0->x[i]) * log((1.0f + y0->x[i]) / (1.0f + y1->x[i] > tiny? 1.0f + y1->x[i] : tiny));
+ if (1.0f - y0->x[i] > 0.0f)
+ cost += .5f * (1.0f - y0->x[i]) * log((1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny));
+ }
+ p->x[0] = (float)(cost / n);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = p->g[0] / n;
+ for (i = 0; i < n; ++i) {
+ if (1.0f + y0->x[i] > 0.0f)
+ y1->g[i] -= .5f * t * (1.0f + y0->x[i]) / (1.0f + y1->x[i] > tiny? 1.0f + y1->x[i] : tiny);
+ if (1.0f - y0->x[i] > 0.0f)
+ y1->g[i] += .5f * t * (1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny);
+ }
+ }
+ return 0;
+}
+
+int kad_op_ce_multi(kad_node_t *p, int action)
+{
+ static const float tiny = 1e-9f;
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ kad_node_t *c = 0;
+ int i, j, n1, d0;
+
+ n1 = y0->d[y0->n_d - 1];
+ d0 = kad_len(y0) / n1;
+ if (p->n_child == 3) {
+ c = p->child[2];
+ assert(c->n_d == 1 && c->d[0] == n1);
+ }
+ if (action == KAD_SYNC_DIM) {
+ if (kad_len(y0) != kad_len(y1) || y0->d[y0->n_d - 1] != y1->d[y1->n_d - 1]) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ if (c == 0) {
+ for (j = 0; j < d0; ++j) {
+ float *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ if (x0[i] > 0.0f)
+ cost += x0[i] * log(x0[i] / (x1[i] > tiny? x1[i] : tiny));
+ }
+ } else {
+ for (j = 0; j < d0; ++j) {
+ float *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ if (x0[i] > 0.0f)
+ cost += c->x[i] * x0[i] * log(x0[i] / (x1[i] > tiny? x1[i] : tiny));
+ }
+ }
+ p->x[0] = (float)(cost / d0);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = p->g[0] / d0;
+ if (c == 0) {
+ for (j = 0; j < d0; ++j) {
+ float *g = &y1->g[j * n1], *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ g[i] -= t * x0[i] / (x1[i] > tiny? x1[i] : tiny);
+ }
+ } else {
+ for (j = 0; j < d0; ++j) {
+ float *g = &y1->g[j * n1], *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ g[i] -= t * c->x[i] * x0[i] / (x1[i] > tiny? x1[i] : tiny);
+ }
+ }
+ }
+ return 0;
+}
+
+/********** Normalization **********/
+
+int kad_op_stdnorm(kad_node_t *p, int action)
+{
+ int i, j, n, m;
+ kad_node_t *q = p->child[0];
+ assert(q->n_d > 0);
+ n = q->d[q->n_d - 1];
+ m = kad_len(q) / n;
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, m * sizeof(float));
+ } else if (action == KAD_FORWARD) {
+ float *si = (float*)p->gtmp;
+ for (j = 0; j < m; ++j) {
+ float *px = &p->x[j * n], *qx = &q->x[j * n];
+ float avg, std_inv;
+ double s;
+ for (i = 0, s = 0.0; i < n; ++i) s += qx[i];
+ avg = (float)(s / n);
+ for (i = 0; i < n; ++i) px[i] = qx[i] - avg;
+ for (i = 0, s = 0.0; i < n; ++i) s += px[i] * px[i];
+ std_inv = s == 0.0? 1.0f : (float)(1.0 / sqrt(s / n));
+ for (i = 0; i < n; ++i) px[i] *= std_inv;
+ si[j] = std_inv;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ float *si = (float*)p->gtmp;
+ for (j = 0; j < m; ++j) {
+ float *pg = &p->g[j * n], *qg = &q->g[j * n], *px = &p->x[j * n], std_inv = si[j];
+ double s, t;
+ for (i = 0, s = t = 0.0; i < n; ++i)
+ s += pg[i], t += px[i] * pg[i];
+ s /= n, t /= n;
+ for (i = 0; i < n; ++i)
+ qg[i] += std_inv * (pg[i] - s - px[i] * t);
+ }
+ }
+ return 0;
+}
+
+/********** Activation functions **********/
+
+int kad_op_sigm(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i)
+ p->x[i] = 1.0f / (1.0f + expf(-q->x[i]));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * (p->x[i] * (1.0f - p->x[i]));
+ }
+ return 0;
+}
+
+int kad_op_tanh(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) {
+ if (q->x[i] < -20.0f) p->x[i] = -1.0f;
+ else {
+ float y;
+ y = expf(-2.0f * q->x[i]);
+ p->x[i] = (1.0f - y) / (1.0f + y);
+ }
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * (1.0f - p->x[i] * p->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_relu(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i)
+ p->x[i] = q->x[i] > 0.0f? q->x[i] : 0.0f;
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ if (q->x[i] > 0.0f)
+ q->g[i] += p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_sin(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = sinf(q->x[i]);
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * cosf(q->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_softmax(kad_node_t *p, int action)
+{
+ int i, j, n1, d0;
+ kad_node_t *q = p->child[0];
+
+ n1 = q->d[q->n_d - 1];
+ d0 = kad_len(q) / n1;
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (j = 0; j < d0; ++j) {
+ float s, max, *x = &q->x[j * n1], *y = &p->x[j * n1];
+ for (i = 0, max = -FLT_MAX; i < n1; ++i)
+ max = max > x[i]? max : x[i];
+ for (i = 0, s = 0.0f; i < n1; ++i) {
+ y[i] = expf(x[i] - max);
+ s += y[i];
+ }
+ for (i = 0, s = 1.0f / s; i < n1; ++i) y[i] *= s;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (j = 0; j < d0; ++j) {
+ float s, *g = &p->g[j * n1], *y = &p->x[j * n1], *h = &q->g[j * n1];
+ for (i = 0, s = 0.0f; i < n1; ++i)
+ s += g[i] * y[i];
+ for (i = 0; i < n1; ++i)
+ h[i] += y[i] * (g[i] - s);
+ }
+ }
+ return 0;
+}
+
+/********** Multi-node pooling **********/
+
+int kad_op_avg(kad_node_t *p, int action)
+{
+ int i, n;
+ float tmp;
+ kad_node_t *q;
+
+ assert(p->n_child > 0);
+ tmp = 1.0f / p->n_child;
+ q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ for (i = 1; i < p->n_child; ++i)
+ if (kad_len(p->child[i]) != n) return -1;
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ memcpy(p->x, q->x, n * sizeof(float));
+ for (i = 1; i < p->n_child; ++i)
+ kad_saxpy(n, 1.0f, p->child[i]->x, p->x);
+ for (i = 0; i < n; ++i) p->x[i] *= tmp;
+ } else if (action == KAD_BACKWARD) {
+ for (i = 0; i < p->n_child; ++i)
+ if (kad_is_back(p->child[i]))
+ kad_saxpy(n, tmp, p->g, p->child[i]->g);
+ }
+ return 0;
+}
+
+int kad_op_max(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ int *max_j;
+ for (i = 1; i < p->n_child; ++i)
+ if (kad_len(p->child[i]) != n) return -1;
+ kad_copy_dim1(p, q);
+ max_j = (int*)calloc(n, sizeof(int));
+ p->gtmp = max_j;
+ } else if (action == KAD_FORWARD) {
+ int j, *max_j = (int*)p->gtmp;
+ memset(max_j, 0, n * sizeof(int));
+ memcpy(p->x, q->x, n * sizeof(float));
+ for (j = 1; j < p->n_child; ++j)
+ for (i = 0, q = p->child[j]; i < n; ++i)
+ if (q->x[i] > p->x[i]) p->x[i] = q->x[i], max_j[i] = j;
+ } else if (action == KAD_BACKWARD) {
+ int *max_j = (int*)p->gtmp;
+ for (i = 0; i < n; ++i)
+ p->child[max_j[i]]->g[i] += p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_stack(kad_node_t *p, int action) /* TODO: allow axis, as in TensorFlow */
+{
+ int i, n, axis = 0;
+ kad_node_t *q;
+
+ assert(p->n_child > 0);
+ q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ for (i = 1; i < p->n_child; ++i)
+ if (kad_len(p->child[i]) != n) return -1;
+ p->n_d = q->n_d + 1;
+ for (i = 0; i < axis; ++i) p->d[i] = q->d[i];
+ p->d[axis] = p->n_child;
+ for (; i < q->n_d; ++i) p->d[i+1] = q->d[i];
+ } else if (action == KAD_FORWARD) { /* TODO: doesn't work when axis != 0 */
+ for (i = 0; i < p->n_child; ++i)
+ memcpy(&p->x[i * n], p->child[i]->x, n * sizeof(float));
+ } else if (action == KAD_BACKWARD) {
+ for (i = 0; i < p->n_child; ++i)
+ if (kad_is_back(p->child[i]))
+ kad_saxpy(n, 1.0f, &p->g[i * n], p->child[i]->g);
+ }
+ return 0;
+}
+
+int kad_op_select(kad_node_t *p, int action)
+{
+ kad_node_t *q;
+ int i, n, which;
+
+ which = *(int32_t*)p->ptr;
+ if (which < 0) which += p->n_child;
+ assert(which >= 0 && which < p->n_child);
+ q = p->child[which];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ for (i = 0; i < p->n_child; ++i)
+ if (p->child[i]->n_d != q->n_d || kad_len(p->child[i]) != n)
+ break;
+ if (i < p->n_child) return -1;
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ memcpy(p->x, q->x, n * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ kad_saxpy(n, 1.0f, p->g, q->g);
+ }
+ return 0;
+}
+
+/********** 2D convolution **********/
+
+static void conv_rot180(int d0, int d1, float *x) /* rotate/reverse a weight martix */
+{
+ int i, j;
+ for (i = 0; i < d0; ++i) {
+ float tmp, *xi = &x[i * d1];
+ for (j = 0; j < d1>>1; ++j)
+ tmp = xi[j], xi[j] = xi[d1-1-j], xi[d1-1-j] = tmp;
+ }
+}
+
+static void conv2d_move_1to3(int d[4], const float *x, float *y) /* convert the NCHW shape to the NHWC shape */
+{
+ int i, j, k, l;
+ for (i = 0; i < d[0]; ++i)
+ for (j = 0; j < d[1]; ++j)
+ for (k = 0; k < d[2]; ++k) {
+ int ik = (i * d[2] + k) * d[3], ijk = ((i * d[1] + j) * d[2] + k) * d[3];
+ for (l = 0; l < d[3]; ++l)
+ y[(ik + l) * d[1] + j] = x[ijk + l];
+ }
+}
+
+static void conv2d_add_3to1(int d[4], const float *y, float *x) /* convert the NHWC shape back to NCHW and add to another NCHW-shaped array */
+{
+ int i, j, k, l;
+ for (i = 0; i < d[0]; ++i)
+ for (j = 0; j < d[1]; ++j)
+ for (k = 0; k < d[2]; ++k) {
+ int ik = (i * d[2] + k) * d[3], ijk = ((i * d[1] + j) * d[2] + k) * d[3];
+ for (l = 0; l < d[3]; ++l)
+ x[ijk + l] += y[(ik + l) * d[1] + j];
+ }
+}
+
+#define conv_out_size(in_size, aux) (((in_size) - (aux)->kernel_size + (aux)->pad[0] + (aux)->pad[1]) / (aux)->stride + 1)
+
+#define process_row_for(_xx, _ww, _yy, _wn, _pn, _stride, _pad, _t) do { \
+ int j, l; \
+ if (_stride > 1) { \
+ for (l = 0; l < _wn; ++l) { \
+ const float *xl = &_xx[l - _pad]; \
+ for (j = 0; j < _pn; ++j, xl += _stride) _t[j] = *xl; \
+ kad_saxpy(_pn, _ww[l], _t, _yy); \
+ } \
+ } else for (l = 0; l < _wn; ++l) kad_saxpy(_pn, _ww[l], &_xx[l - _pad], _yy); \
+} while (0)
+
+#define process_row_back_x(_xx, _ww, _yy, _wn, _pn, _stride, _pad, _t) do { \
+ int j, l; \
+ if (_stride > 1) { \
+ for (l = 0; l < _wn; ++l) { \
+ float *xl = &_xx[l - _pad]; \
+ memset(_t, 0, _pn * sizeof(float)); \
+ kad_saxpy(_pn, _ww[l], _yy, _t); \
+ for (j = 0; j < _pn; ++j, xl += _stride) *xl += _t[j]; \
+ } \
+ } else for (l = 0; l < _wn; ++l) kad_saxpy(_pn, _ww[l], _yy, &_xx[l - _pad]); \
+} while (0)
+
+#define process_row_back_w(_xx, _ww, _yy, _wn, _pn, _stride, _pad, _t) do { \
+ int j, l; \
+ if (_stride > 1) { \
+ for (l = 0; l < _wn; ++l) { \
+ const float *xl = &_xx[l - _pad]; \
+ for (j = 0; j < _pn; ++j, xl += _stride) _t[j] = *xl; \
+ _ww[l] += kad_sdot(_pn, _yy, _t); \
+ } \
+ } else for (l = 0; l < _wn; ++l) _ww[l] += kad_sdot(_pn, _yy, &_xx[l - _pad]); \
+} while (0)
+
+/* Forward and backward passes are implemented with two different algorithms.
+ * The first is faster for small kernels with few input channels; otherwise the
+ * second algorithm is faster. Both algorithms should produce identical
+ * results, up to the precision of "float".
+ */
+int kad_op_conv2d(kad_node_t *p, int action) /* in the number-channel-height-width (NCHW) shape */
+{
+#define conv2d_loop1(_x, _w, _y, _tmp, _row_func) do { /* for the NCHW shape */ \
+ int n, c1, c0, i, k, ii; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) /* output channel */ \
+ for (c0 = 0; c0 < w->d[1]; ++c0) /* input channel */ \
+ for (k = 0; k < w->d[2]; ++k) { /* kernel row */ \
+ float *_ww = &(_w)[((c1 * w->d[1] + c0) * w->d[2] + k) * w->d[3]]; \
+ for (i = 0, ii = k - aux[0].pad[0]; i < p->d[2] && ii >= 0 && ii < q->d[2]; ++i, ii += aux[0].stride) { /* output row */ \
+ float *_xx = &(_x)[((n * q->d[1] + c0) * q->d[2] + ii) * q->d[3]]; \
+ float *_yy = &(_y)[((n * p->d[1] + c1) * p->d[2] + i) * p->d[3]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux[1].pad[0], _xx, q->d[3] * sizeof(float)); \
+ _xx = x_padded + aux[1].pad[0]; \
+ } \
+ _row_func(_xx, _ww, _yy, w->d[3], p->d[3], aux[1].stride, aux[1].pad[0], (_tmp)); \
+ } /* ~i */ \
+ } /* ~k, c0, c1, n */ \
+ } while (0)
+
+#define conv2d_loop2(_x, _w, _y, _code) do { /* for the NHWC shape */ \
+ int n, c1, i, j, k, ii, j_skip = aux[1].stride * q->d[1], m = w->d[3] * w->d[1]; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) /* output channel */ \
+ for (k = 0; k < w->d[2]; ++k) { /* kernel row */ \
+ float *_ww = &(_w)[(c1 * w->d[2] + k) * m]; \
+ for (i = 0, ii = k - aux[0].pad[0]; i < p->d[2] && ii >= 0 && ii < q->d[2]; ++i, ii += aux[0].stride) { /* output and input row */ \
+ float *_xx = &(_x)[(n * q->d[2] + ii) * q->d[3] * q->d[1]]; \
+ float *_yy = &(_y)[((n * p->d[1] + c1) * p->d[2] + i) * p->d[3]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux[1].pad[0] * q->d[1], _xx, q->d[3] * q->d[1] * sizeof(float)); \
+ _xx = x_padded; \
+ } \
+ for (j = 0; j < p->d[3]; ++j, _xx += j_skip, ++_yy) _code; /* output and input column */ \
+ } /* ~i */ \
+ } /* ~k, c1, n */ \
+ } while (0)
+
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0], *w = p->child[1];
+ float *t = 0, *q1 = 0, *w1 = 0, *x_padded = 0;
+ int algo_switch = 0;
+
+ if (action == KAD_FORWARD || action == KAD_BACKWARD) { /* allocate working space */
+ if (w->d[3] * w->d[1] < 16) {
+ t = (float*)malloc(p->d[3] * sizeof(float));
+ x_padded = aux[1].pad[0] + aux[1].pad[1] > 0? (float*)calloc(q->d[3] + aux[1].pad[0] + aux[1].pad[1], sizeof(float)) : 0;
+ } else {
+ q1 = (float*)malloc(kad_len(q) * sizeof(float));
+ w1 = (float*)malloc(kad_len(w) * sizeof(float));
+ x_padded = aux[1].pad[0] + aux[1].pad[1] > 0? (float*)calloc((q->d[3] + aux[1].pad[0] + aux[1].pad[1]) * q->d[1], sizeof(float)) : 0;
+ algo_switch = 1;
+ }
+ }
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 4 || w->n_d != 4) return -1;
+ if (q->d[1] != w->d[1]) return -1; /* unmatched input channels */
+ p->n_d = 4;
+ p->d[0] = q->d[0], p->d[1] = w->d[0], p->d[2] = conv_out_size(q->d[2], &aux[0]), p->d[3] = conv_out_size(q->d[3], &aux[1]);
+ } else if (action == KAD_FORWARD) {
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ if (!algo_switch) { /* this is the first algorithm */
+ conv2d_loop1(q->x, w->x, p->x, t, process_row_for);
+ } else { /* this is the second algorithm */
+ conv2d_move_1to3(q->d, q->x, q1);
+ conv2d_move_1to3(w->d, w->x, w1);
+ conv2d_loop2(q1, w1, p->x, (*_yy += kad_sdot(m, _ww, _xx)));
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(p->child[0])) { /* backprop to the input array */
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ if (!algo_switch) {
+ conv2d_loop1(q->g, w->x, p->g, t, process_row_back_x);
+ } else {
+ memset(q1, 0, kad_len(q) * sizeof(float));
+ conv2d_move_1to3(w->d, w->x, w1);
+ conv2d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _ww, _xx));
+ conv2d_add_3to1(q->d, q1, q->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ }
+ if (kad_is_back(p->child[1])) { /* backprop to the weight matrix */
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->g);
+ if (!algo_switch) {
+ conv2d_loop1(q->x, w->g, p->g, t, process_row_back_w);
+ } else {
+ conv2d_move_1to3(q->d, q->x, q1);
+ memset(w1, 0, kad_len(w) * sizeof(float));
+ conv2d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _xx, _ww));
+ conv2d_add_3to1(w->d, w1, w->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->g);
+ }
+ }
+ free(t); free(q1); free(w1); free(x_padded);
+ return 0;
+}
+
+int kad_op_max2d(kad_node_t *p, int action)
+{
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0];
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 4) return -1;
+ p->n_d = 4;
+ p->d[0] = q->d[0], p->d[1] = q->d[1], p->d[2] = conv_out_size(q->d[2], &aux[0]), p->d[3] = conv_out_size(q->d[3], &aux[1]);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, kad_len(p) * sizeof(int));
+ } else if (action == KAD_FORWARD) {
+ int rest = 1, len, t, i;
+ int *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) p->x[i] = -FLT_MAX;
+ for (i = 0; i < p->n_d - 2; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int i, j, k, l, p_row = p->d[p->n_d - 2], p_col = p->d[p->n_d - 1];
+ for (i = 0; i < p_row; ++i) {
+ int u = (t * p_row + i) * p_col;
+ for (k = 0; k < aux[0].kernel_size; ++k) {
+ int v, v0, v_end, ii = i * aux[0].stride + k - aux[0].pad[0];
+ if (ii < 0 || ii >= q->d[p->n_d - 2]) continue;
+ v0 = (t * q->d[p->n_d - 2] + ii) * q->d[p->n_d - 1];
+ v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux[1].kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux[1].pad[0]? l - aux[1].pad[0] : 0); j < p_col && v < v_end; ++j, v += aux[1].stride)
+ if (p->x[u + j] < q->x[v])
+ p->x[u + j] = q->x[v], f[u + j] = v;
+ } /* ~k */
+ } /* ~i */
+ }
+ } else if (action == KAD_BACKWARD) {
+ int i, len, *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) q->g[f[i]] += p->g[i];
+ }
+ return 0;
+}
+
+/********** 1D convolution **********/
+
+static void conv1d_move_1to2(int d[3], const float *x, float *y)
+{
+ int i, j, k;
+ for (k = 0; k < d[0]; ++k)
+ for (j = 0; j < d[1]; ++j)
+ for (i = 0; i < d[2]; ++i)
+ y[(k * d[2] + i) * d[1] + j] = x[(k * d[1] + j) * d[2] + i];
+}
+
+static void conv1d_add_2to1(int d[3], const float *y, float *x)
+{
+ int i, j, k;
+ for (k = 0; k < d[0]; ++k)
+ for (j = 0; j < d[1]; ++j)
+ for (i = 0; i < d[2]; ++i)
+ x[(k * d[1] + j) * d[2] + i] += y[(k * d[2] + i) * d[1] + j];
+}
+
+int kad_op_conv1d(kad_node_t *p, int action) /* in the number-channel-width (NCW) shape */
+{
+#define conv1d_loop1(_x, _w, _y, _tmp, _row_func) do { /* for the NCW shape */ \
+ int n, c1, c0; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) /* output channel */ \
+ for (c0 = 0; c0 < w->d[1]; ++c0) { /* input channel */ \
+ float *_ww = &(_w)[(c1 * w->d[1] + c0) * w->d[2]]; \
+ float *_xx = &(_x)[(n * q->d[1] + c0) * q->d[2]]; \
+ float *_yy = &(_y)[(n * p->d[1] + c1) * p->d[2]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux->pad[0], _xx, q->d[2] * sizeof(float)); \
+ _xx = x_padded + aux->pad[0]; \
+ } \
+ _row_func(_xx, _ww, _yy, w->d[2], p->d[2], aux->stride, aux->pad[0], (_tmp)); \
+ } /* ~c0, c1, n */ \
+ } while (0)
+
+#define conv1d_loop2(_x, _w, _y, _code) do { /* for the NWC shape */ \
+ int n, c1, j, j_skip = aux->stride * q->d[1], m = w->d[2] * w->d[1]; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) { /* output channel */ \
+ float *_ww = &(_w)[c1 * m]; \
+ float *_xx = &(_x)[n * q->d[1] * q->d[2]]; \
+ float *_yy = &(_y)[(n * p->d[1] + c1) * p->d[2]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux->pad[0] * q->d[1], _xx, q->d[2] * q->d[1] * sizeof(float)); \
+ _xx = x_padded; \
+ } \
+ for (j = 0; j < p->d[2]; ++j, _xx += j_skip, ++_yy) _code; \
+ } /* ~c1, n */ \
+ } while (0)
+
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0], *w = p->child[1];
+ float *t = 0, *q1 = 0, *w1 = 0, *x_padded = 0;
+ int algo_switch = 0;
+
+ if (action == KAD_FORWARD || action == KAD_BACKWARD) { /* allocate working space */
+ if (w->d[2] * w->d[1] < 32) {
+ t = (float*)malloc(p->d[2] * sizeof(float));
+ x_padded = aux->pad[0] + aux->pad[1] > 0? (float*)calloc(q->d[2] + aux->pad[0] + aux->pad[1], sizeof(float)) : 0;
+ } else {
+ q1 = (float*)malloc(kad_len(q) * sizeof(float));
+ w1 = (float*)malloc(kad_len(w) * sizeof(float));
+ x_padded = aux->pad[0] + aux->pad[1] > 0? (float*)calloc((q->d[2] + aux->pad[0] + aux->pad[1]) * q->d[1], sizeof(float)) : 0;
+ algo_switch = 1;
+ }
+ }
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 3 || w->n_d != 3) return -1;
+ if (q->d[1] != w->d[1]) return -1; /* unmatched input channels */
+ p->n_d = 3;
+ p->d[0] = q->d[0], p->d[1] = w->d[0], p->d[2] = conv_out_size(q->d[2], aux);
+ } else if (action == KAD_FORWARD) {
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ if (!algo_switch) { /* this is the first algorithm */
+ conv1d_loop1(q->x, w->x, p->x, t, process_row_for);
+ } else { /* this is the second algorithm */
+ conv1d_move_1to2(q->d, q->x, q1);
+ conv1d_move_1to2(w->d, w->x, w1);
+ conv1d_loop2(q1, w1, p->x, (*_yy += kad_sdot(m, _ww, _xx)));
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(p->child[0])) { /* backprop to the input array */
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ if (!algo_switch) {
+ conv1d_loop1(q->g, w->x, p->g, t, process_row_back_x);
+ } else {
+ memset(q1, 0, kad_len(q) * sizeof(float));
+ conv1d_move_1to2(w->d, w->x, w1);
+ conv1d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _ww, _xx));
+ conv1d_add_2to1(q->d, q1, q->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ }
+ if (kad_is_back(p->child[1])) { /* backprop to the weight matrix */
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->g);
+ if (!algo_switch) {
+ conv1d_loop1(q->x, w->g, p->g, t, process_row_back_w);
+ } else {
+ conv1d_move_1to2(q->d, q->x, q1);
+ memset(w1, 0, kad_len(w) * sizeof(float));
+ conv1d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _xx, _ww));
+ conv1d_add_2to1(w->d, w1, w->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->g);
+ }
+ }
+ free(t); free(q1); free(w1); free(x_padded);
+ return 0;
+}
+
+int kad_op_max1d(kad_node_t *p, int action)
+{
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0];
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 3) return -1;
+ p->n_d = 3;
+ p->d[0] = q->d[0], p->d[1] = q->d[1], p->d[2] = conv_out_size(q->d[2], aux);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, kad_len(p) * sizeof(int));
+ } else if (action == KAD_FORWARD) {
+ int rest = 1, len, t, i;
+ int *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) p->x[i] = -FLT_MAX;
+ for (i = 0; i < p->n_d - 1; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int j, l, p_width = p->d[p->n_d - 1];
+ int u = t * p_width, v, v0 = t * q->d[p->n_d - 1], v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux->kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux->pad[0]? l - aux->pad[0] : 0); j < p_width && v < v_end; ++j, v += aux->stride)
+ if (p->x[u + j] < q->x[v])
+ p->x[u + j] = q->x[v], f[u + j] = v;
+ }
+ } else if (action == KAD_BACKWARD) {
+ int i, len, *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) q->g[f[i]] += p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_avg1d(kad_node_t *p, int action)
+{
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0];
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 3) return -1;
+ p->n_d = 3;
+ p->d[0] = q->d[0], p->d[1] = q->d[1], p->d[2] = conv_out_size(q->d[2], aux);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, kad_len(p) * sizeof(int));
+ } else if (action == KAD_FORWARD) {
+ int rest = 1, len, t, i;
+ int *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) p->x[i] = 0.0f, f[i] = 0;
+ for (i = 0; i < p->n_d - 1; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int j, l, p_width = p->d[p->n_d - 1];
+ int u = t * p_width, v, v0 = t * q->d[p->n_d - 1], v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux->kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux->pad[0]? l - aux->pad[0] : 0); j < p_width && v < v_end; ++j, v += aux->stride)
+ p->x[u + j] += q->x[v], ++f[u + j];
+ }
+ for (i = 0; i < len; ++i) p->x[i] /= f[i];
+ } else if (action == KAD_BACKWARD) {
+ int rest = 1, t, i;
+ int *f = (int*)p->gtmp;
+ for (i = 0; i < p->n_d - 1; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int j, l, p_width = p->d[p->n_d - 1];
+ int u = t * p_width, v, v0 = t * q->d[p->n_d - 1], v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux->kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux->pad[0]? l - aux->pad[0] : 0); j < p_width && v < v_end; ++j, v += aux->stride)
+ q->g[v] += p->g[u + j] / f[u + j];
+ }
+ }
+ return 0;
+}
+
+/********** List of operators **********/
+
+kad_op_f kad_op_list[KAD_MAX_OP] = {
+ 0,
+ kad_op_add, /* 1: element-wise addition */
+ kad_op_mul, /* 2: element-wise multiplication */
+ kad_op_cmul, /* 3: column multiplication */
+ kad_op_ce_bin_neg, /* 4: binary cross-entropy for (-1,1) */
+ kad_op_square, /* 5: square */
+ kad_op_sigm, /* 6: sigmoid */
+ kad_op_tanh, /* 7: tanh */
+ kad_op_relu, /* 8: ReLU */
+ kad_op_matmul, /* 9: matrix multiplication */
+ kad_op_avg, /* 10: general average pooling (not for ConvNet) */
+ kad_op_1minus, /* 11: 1-x */
+ kad_op_select, /* 12: choose between one of the children */
+ kad_op_ce_multi, /* 13: multi-class cross-entropy */
+ kad_op_softmax, /* 14: softmax */
+ kad_op_dropout, /* 15: dropout */
+ kad_op_conv2d, /* 16: 2D convolution */
+ kad_op_max2d, /* 17: 2D max pooling (for 2D ConvNet) */
+ kad_op_conv1d, /* 18: 1D convolution */
+ kad_op_max1d, /* 19: 1D max pooling (for 1D ConvNet) */
+ kad_op_slice, /* 20: slice data at a dimension */
+ kad_op_max, /* 21: general max pooling */
+ kad_op_ce_bin, /* 22: binary cross-entropy for (0,1) */
+ kad_op_sub, /* 23: element-wise subtraction */
+ kad_op_sample_normal, /* 24: sample from a normal distribution */
+ kad_op_reduce_sum, /* 25 */
+ kad_op_reduce_mean, /* 26 */
+ kad_op_log, /* 27: log() */
+ kad_op_avg1d, /* 28: 1D average pooling (for 1D ConvNet) */
+ kad_op_mse, /* 29: mean square error */
+ kad_op_reshape, /* 30 */
+ kad_op_concat, /* 31 */
+ kad_op_stdnorm, /* 32: layer normalization */
+ kad_op_exp, /* 33: exp() */
+ kad_op_sin, /* 34: sin() */
+ kad_op_stack, /* 35: tf.stack, but on the first axis only */
+ kad_op_reverse /* 36: tf.reverse, but on one axis only */
+};
+
+char *kad_op_name[KAD_MAX_OP] = {
+ 0, "add", "mul", "cmul", "ce_bin_neg", "square", "sigm", "tanh", "relu", "matmul", "avg", "1minus", "select", "ce_multi", "softmax",
+ "dropout", "conv2d", "max2d", "conv1d", "max1d", "slice", "max", "ce_bin", "sub", "sample_normal", "reduce_sum", "reduce_mean", "log",
+ "avg1d", "mse", "reshape", "concat", "stdnorm", "exp", "sin", "stack", "reverse"
+};
+
+/**************************
+ *** Debugging routines ***
+ **************************/
+
+void kad_trap_fe(void)
+{
+#ifdef __SSE__
+ _MM_SET_EXCEPTION_MASK(_MM_GET_EXCEPTION_MASK() & ~(_MM_MASK_INVALID | _MM_MASK_DIV_ZERO));
+#endif
+}
+
+void kad_print_graph(FILE *fp, int n, kad_node_t **v)
+{
+ int i, j;
+ for (i = 0; i < n; ++i) v[i]->tmp = i;
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = v[i];
+ fprintf(fp, "%d\t%x:%x\t%d\t", i, p->flag, p->ext_flag, p->ext_label);
+ if (p->pre) fprintf(fp, "%d\t", p->pre->tmp);
+ else fprintf(fp, ".\t");
+ fputs("[", fp);
+ for (j = 0; j < p->n_d; ++j) {
+ if (j) fputc(',', fp);
+ fprintf(fp, "%d", p->d[j]);
+ }
+ fprintf(fp, "]\t");
+ if (p->n_child) {
+ fprintf(fp, "%s(", kad_op_name[p->op]);
+ for (j = 0; j < p->n_child; ++j) {
+ if (j) fputc(',', fp);
+ fprintf(fp, "$%d", p->child[j]->tmp);
+ }
+ fprintf(fp, ")");
+ } else fprintf(fp, "%s", kad_is_feed(p)? "feed" : kad_is_var(p)? "var" : kad_is_const(p)? "const" : "N/A");
+ fputc('\n', fp);
+ }
+ for (i = 0; i < n; ++i) v[i]->tmp = 0;
+}
+
+static void kad_add_delta(int n, kad_node_t **a, float c, float *delta)
+{
+ int i, k;
+ for (i = k = 0; i < n; ++i)
+ if (kad_is_var(a[i])) {
+ kad_saxpy(kad_len(a[i]), c, &delta[k], a[i]->x);
+ k += kad_len(a[i]);
+ }
+}
+
+void kad_check_grad(int n, kad_node_t **a, int from)
+{
+ const float eps = 1e-5f, rel = 1e-7f / eps;
+ int i, k, n_var;
+ float *g0, *delta, f0, f_minus, f_plus, s0, s1, rel_err, p_m_err;
+ n_var = kad_size_var(n, a);
+ g0 = (float*)calloc(n_var, sizeof(float));
+ f0 = *kad_eval_at(n, a, from);
+ kad_grad(n, a, from);
+ for (i = k = 0; i < n; ++i)
+ if (kad_is_var(a[i])) {
+ memcpy(&g0[k], a[i]->g, kad_len(a[i]) * sizeof(float));
+ k += kad_len(a[i]);
+ }
+ delta = (float*)calloc(n_var, sizeof(float));
+ for (k = 0; k < n_var; ++k) delta[k] = (float)kad_drand(0) * eps;
+ kad_add_delta(n, a, 1.0f, delta);
+ f_plus = *kad_eval_at(n, a, from);
+ kad_add_delta(n, a, -2.0f, delta);
+ f_minus = *kad_eval_at(n, a, from);
+ kad_add_delta(n, a, 1.0f, delta);
+ s0 = kad_sdot(n_var, g0, delta);
+ s1 = .5f * (f_plus - f_minus);
+ fprintf(stderr, "Gradient check -- %g <=> %g @ %g -- ", s0/eps, s1/eps, f0);
+ if (fabs(s1) >= rel * eps) {
+ rel_err = fabsf(fabsf(s0) - fabsf(s1)) / (fabsf(s0) + fabsf(s1));
+ p_m_err = fabsf(f_plus + f_minus - 2.0f * f0) / fabsf(f_plus - f_minus);
+ fprintf(stderr, "rel_err:%g p_m_err:%g -- ", rel_err, p_m_err);
+ if (rel_err >= rel && rel_err > p_m_err) fprintf(stderr, "failed\n");
+ else fprintf(stderr, "passed\n");
+ } else fprintf(stderr, "skipped\n");
+ free(delta); free(g0);
+}
diff --git a/contrib/kann/kautodiff.h b/contrib/kann/kautodiff.h
new file mode 100644
index 0000000..d7e7133
--- /dev/null
+++ b/contrib/kann/kautodiff.h
@@ -0,0 +1,256 @@
+/*
+ The MIT License
+
+ Copyright (c) 2018-2019 Dana-Farber Cancer Institute
+ 2016-2018 Broad Institute
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+#ifndef KANN_AUTODIFF_H
+#define KANN_AUTODIFF_H
+
+#define KAD_VERSION "r544"
+
+#include <stdio.h>
+#include <stdint.h>
+
+#ifdef __STRICT_ANSI__
+#define inline
+#endif
+
+#define KAD_MAX_DIM 4 /* max dimension */
+#define KAD_MAX_OP 64 /* max number of operators */
+
+/* A computational graph is a directed acyclic graph. In the graph, an external
+ * node represents a variable, a constant or a feed; an internal node
+ * represents an operator; an edge from node v to w indicates v is an operand
+ * of w.
+ */
+
+#define KAD_VAR 0x1
+#define KAD_CONST 0x2
+#define KAD_POOL 0x4
+#define KAD_SHARE_RNG 0x10 /* with this flag on, different time step shares the same RNG status after unroll */
+
+#define kad_is_back(p) ((p)->flag & KAD_VAR)
+#define kad_is_ext(p) ((p)->n_child == 0)
+#define kad_is_var(p) (kad_is_ext(p) && kad_is_back(p))
+#define kad_is_const(p) (kad_is_ext(p) && ((p)->flag & KAD_CONST))
+#define kad_is_feed(p) (kad_is_ext(p) && !kad_is_back(p) && !((p)->flag & KAD_CONST))
+#define kad_is_pivot(p) ((p)->n_child == 1 && ((p)->flag & KAD_POOL))
+#define kad_is_switch(p) ((p)->op == 12 && !((p)->flag & KAD_POOL))
+#define kad_use_rng(p) ((p)->op == 15 || (p)->op == 24)
+
+#define kad_eval_enable(p) ((p)->tmp = 1)
+#define kad_eval_disable(p) ((p)->tmp = -1)
+
+/* a node in the computational graph */
+typedef struct kad_node_t {
+ uint8_t n_d; /* number of dimensions; no larger than KAD_MAX_DIM */
+ uint8_t flag; /* type of the node; see KAD_F_* for valid flags */
+ uint16_t op; /* operator; kad_op_list[op] is the actual function */
+ int32_t n_child; /* number of operands/child nodes */
+ int32_t tmp; /* temporary field; MUST BE zero before calling kad_compile() */
+ int32_t ptr_size; /* size of ptr below */
+ int32_t d[KAD_MAX_DIM]; /* dimensions */
+ int32_t ext_label; /* labels for external uses (not modified by the kad_* APIs) */
+ uint32_t ext_flag; /* flags for external uses (not modified by the kad_* APIs) */
+ float *x; /* value; allocated for internal nodes */
+ float *g; /* gradient; allocated for internal nodes */
+ void *ptr; /* for special operators that need additional parameters (e.g. conv2d) */
+ void *gtmp; /* temporary data generated at the forward pass but used at the backward pass */
+ struct kad_node_t **child; /* operands/child nodes */
+ struct kad_node_t *pre; /* usually NULL; only used for RNN */
+} kad_node_t, *kad_node_p;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Compile/linearize a computational graph
+ *
+ * @param n_node number of nodes (out)
+ * @param n_roots number of nodes without predecessors
+ * @param roots list of nodes without predecessors
+ *
+ * @return list of nodes, of size *n_node
+ */
+kad_node_t **kad_compile_array(int *n_node, int n_roots, kad_node_t **roots);
+
+kad_node_t **kad_compile(int *n_node, int n_roots, ...); /* an alternative API to above */
+void kad_delete(int n, kad_node_t **a); /* deallocate a compiled/linearized graph */
+
+/**
+ * Compute the value at a node
+ *
+ * @param n number of nodes
+ * @param a list of nodes
+ * @param from compute the value at this node, 0<=from<n
+ *
+ * @return a pointer to the value (pointing to kad_node_t::x, so don't call
+ * free() on it!)
+ */
+const float *kad_eval_at(int n, kad_node_t **a, int from);
+
+void kad_eval_marked(int n, kad_node_t **a);
+int kad_sync_dim(int n, kad_node_t **v, int batch_size);
+
+/**
+ * Compute gradient
+ *
+ * @param n number of nodes
+ * @param a list of nodes
+ * @param from the function node; must be a scalar (compute \nabla a[from])
+ */
+void kad_grad(int n, kad_node_t **a, int from);
+
+/**
+ * Unroll a recurrent computation graph
+ *
+ * @param n_v number of nodes
+ * @param v list of nodes
+ * @param new_n number of nodes in the unrolled graph (out)
+ * @param len how many times to unroll, one for each pivot
+ *
+ * @return list of nodes in the unrolled graph
+ */
+kad_node_t **kad_unroll(int n_v, kad_node_t **v, int *new_n, int *len);
+int kad_n_pivots(int n_v, kad_node_t **v);
+
+kad_node_t **kad_clone(int n, kad_node_t **v, int batch_size);
+
+/* define a variable, a constant or a feed (placeholder in TensorFlow) */
+kad_node_t *kad_var(float *x, float *g, int n_d, ...); /* a variable; gradients to be computed; not unrolled */
+kad_node_t *kad_const(float *x, int n_d, ...); /* a constant; no gradients computed; not unrolled */
+kad_node_t *kad_feed(int n_d, ...); /* an input/output; no gradients computed; unrolled */
+
+/* operators taking two operands */
+kad_node_t *kad_add(kad_node_t *x, kad_node_t *y); /* f(x,y) = x + y (generalized element-wise addition; f[i*n+j]=x[i*n+j]+y[j], n=kad_len(y), 0<j<n, 0<i<kad_len(x)/n) */
+kad_node_t *kad_sub(kad_node_t *x, kad_node_t *y); /* f(x,y) = x - y (generalized element-wise subtraction) */
+kad_node_t *kad_mul(kad_node_t *x, kad_node_t *y); /* f(x,y) = x * y (generalized element-wise product) */
+
+kad_node_t *kad_matmul(kad_node_t *x, kad_node_t *y); /* f(x,y) = x * y (general matrix product) */
+kad_node_t *kad_cmul(kad_node_t *x, kad_node_t *y); /* f(x,y) = x * y^T (column-wise matrix product; i.e. y is transposed) */
+
+/* loss functions; output scalar */
+kad_node_t *kad_mse(kad_node_t *x, kad_node_t *y); /* mean square error */
+kad_node_t *kad_ce_multi(kad_node_t *x, kad_node_t *y); /* multi-class cross-entropy; x is the preidction and y is the truth */
+kad_node_t *kad_ce_bin(kad_node_t *x, kad_node_t *y); /* binary cross-entropy for (0,1) */
+kad_node_t *kad_ce_bin_neg(kad_node_t *x, kad_node_t *y); /* binary cross-entropy for (-1,1) */
+kad_node_t *kad_ce_multi_weighted(kad_node_t *pred, kad_node_t *truth, kad_node_t *weight);
+
+#define KAD_PAD_NONE 0 /* use the smallest zero-padding */
+#define KAD_PAD_SAME (-2) /* output to have the same dimension as input */
+
+kad_node_t *kad_conv2d(kad_node_t *x, kad_node_t *w, int r_stride, int c_stride, int r_pad, int c_pad); /* 2D convolution with weight matrix flipped */
+kad_node_t *kad_max2d(kad_node_t *x, int kernel_h, int kernel_w, int r_stride, int c_stride, int r_pad, int c_pad); /* 2D max pooling */
+kad_node_t *kad_conv1d(kad_node_t *x, kad_node_t *w, int stride, int pad); /* 1D convolution with weight flipped */
+kad_node_t *kad_max1d(kad_node_t *x, int kernel_size, int stride, int pad); /* 1D max pooling */
+kad_node_t *kad_avg1d(kad_node_t *x, int kernel_size, int stride, int pad); /* 1D average pooling */
+
+kad_node_t *kad_dropout(kad_node_t *x, kad_node_t *r); /* dropout at rate r */
+kad_node_t *kad_sample_normal(kad_node_t *x); /* f(x) = x * r, where r is drawn from a standard normal distribution */
+
+/* operators taking one operand */
+kad_node_t *kad_square(kad_node_t *x); /* f(x) = x^2 (element-wise square) */
+kad_node_t *kad_sigm(kad_node_t *x); /* f(x) = 1/(1+exp(-x)) (element-wise sigmoid) */
+kad_node_t *kad_tanh(kad_node_t *x); /* f(x) = (1-exp(-2x)) / (1+exp(-2x)) (element-wise tanh) */
+kad_node_t *kad_relu(kad_node_t *x); /* f(x) = max{0,x} (element-wise rectifier, aka ReLU) */
+kad_node_t *kad_softmax(kad_node_t *x);/* f_i(x_1,...,x_n) = exp(x_i) / \sum_j exp(x_j) (softmax: tf.nn.softmax(x,dim=-1)) */
+kad_node_t *kad_1minus(kad_node_t *x); /* f(x) = 1 - x */
+kad_node_t *kad_exp(kad_node_t *x); /* f(x) = exp(x) */
+kad_node_t *kad_log(kad_node_t *x); /* f(x) = log(x) */
+kad_node_t *kad_sin(kad_node_t *x); /* f(x) = sin(x) */
+
+kad_node_t *kad_stdnorm(kad_node_t *x); /* layer normalization; applied to the last dimension */
+
+/* operators taking an indefinite number of operands (e.g. pooling) */
+kad_node_t *kad_avg(int n, kad_node_t **x); /* f(x_1,...,x_n) = \sum_i x_i/n (mean pooling) */
+kad_node_t *kad_max(int n, kad_node_t **x); /* f(x_1,...,x_n) = max{x_1,...,x_n} (max pooling) */
+kad_node_t *kad_stack(int n, kad_node_t **x); /* f(x_1,...,x_n) = [x_1,...,x_n] (stack pooling) */
+kad_node_t *kad_select(int n, kad_node_t **x, int which); /* f(x_1,...,x_n;i) = x_i (select pooling; -1 for the last) */
+
+/* dimension reduction */
+kad_node_t *kad_reduce_sum(kad_node_t *x, int axis); /* tf.reduce_sum(x, axis) */
+kad_node_t *kad_reduce_mean(kad_node_t *x, int axis); /* tf.reduce_mean(x, axis) */
+
+/* special operators */
+kad_node_t *kad_slice(kad_node_t *x, int axis, int start, int end); /* take a slice on the axis-th dimension */
+kad_node_t *kad_concat(int axis, int n, ...); /* concatenate on the axis-th dimension */
+kad_node_t *kad_concat_array(int axis, int n, kad_node_t **p); /* the array version of concat */
+kad_node_t *kad_reshape(kad_node_t *x, int n_d, int *d); /* reshape; similar behavior to TensorFlow's reshape() */
+kad_node_t *kad_reverse(kad_node_t *x, int axis);
+kad_node_t *kad_switch(int n, kad_node_t **p); /* manually (as a hyperparameter) choose one input, default to 0 */
+
+/* miscellaneous operations on a compiled graph */
+int kad_size_var(int n, kad_node_t *const* v); /* total size of all variables */
+int kad_size_const(int n, kad_node_t *const* v); /* total size of all constants */
+
+/* graph I/O */
+int kad_save(FILE *fp, int n_node, kad_node_t **node);
+kad_node_t **kad_load(FILE *fp, int *_n_node);
+
+/* random number generator */
+void *kad_rng(void);
+void kad_srand(void *d, uint64_t seed);
+uint64_t kad_rand(void *d);
+double kad_drand(void *d);
+double kad_drand_normal(void *d);
+void kad_saxpy(int n, float a, const float *x, float *y);
+
+/* debugging routines */
+void kad_trap_fe(void); /* abort on divide-by-zero and NaN */
+void kad_print_graph(FILE *fp, int n, kad_node_t **v);
+void kad_check_grad(int n, kad_node_t **a, int from);
+
+#ifdef __cplusplus
+}
+#endif
+
+#define KAD_ALLOC 1
+#define KAD_FORWARD 2
+#define KAD_BACKWARD 3
+#define KAD_SYNC_DIM 4
+
+typedef int (*kad_op_f)(kad_node_t*, int);
+extern kad_op_f kad_op_list[KAD_MAX_OP];
+extern char *kad_op_name[KAD_MAX_OP];
+
+static inline int kad_len(const kad_node_t *p) /* calculate the size of p->x */
+{
+ int n = 1, i;
+ for (i = 0; i < p->n_d; ++i) n *= p->d[i];
+ return n;
+}
+
+/* Additions by Rspamd */
+void kad_sgemm_simple (int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C);
+/**
+ * Calculate eigenvectors and eigenvalues
+ * @param N dimensions of A (must be NxN)
+ * @param A input matrix (part of it will be destroyed, so copy if needed), on finish the first `nwork` columns will have eigenvectors
+ * @param eigenvals eigenvalues, must be N elements vector
+ */
+bool kad_ssyev_simple (int N, float *A, float *eigenvals);
+
+#endif
diff --git a/contrib/languages-data/af.json b/contrib/languages-data/af.json
new file mode 100644
index 0000000..136228f
--- /dev/null
+++ b/contrib/languages-data/af.json
@@ -0,0 +1 @@
+{"freq":{"D":9246,"E":2445,"F":2510,"G":3299,"A":6930,"B":3706,"C":2451,"L":2519,"M":3951,"N":3334,"O":2514,"H":3034,"I":2837,"J":2196,"K":3663,"U":687,"T":2336,"W":2258,"V":2714,"Q":182,"P":3097,"S":8234,"R":3039,"Y":252,"X":214,"Z":422,"f":13583,"g":42805,"d":77385,"Feb":207,"e":240974,"b":21626,"c":4896,"a":128566,"n":127153,"o":86673,"l":57433,"m":31352,"j":4048,"k":45378,"h":17527,"i":140621,"w":24930,"v":32618,"u":35166,"t":82606,"s":102389,"r":98861,"q":199,"p":23331,"z":1187,"y":11757,"x":1123,"ï":264,"ë":2903,"ê":1053,"é":765,"á":212,"ü":233,"ö":184,"ó":216,"Eur":318,"Eng":637," l":3565," m":7731," n":16000," o":12065," h":7358," i":23795," j":1325," k":6363," d":33601," e":13358," f":1200," g":11018,"р":242,"с":306," a":8747,"т":161," b":8379," c":434," u":1931," t":8537," w":13128," v":24617," p":4859," s":15482," r":3617," J":2155," K":3559," H":2961," I":2185," N":3120," O":2318," L":2396," M":3803," B":3554," C":2109," A":6365," F":2371," G":3138," D":8986," E":2271,"л":219,"к":266," Z":368," Y":241,"и":371,"о":333,"н":199," S":7708,"Ger":200," R":2881,"в":199," Q":162," P":2912,"а":481," W":2205," V":2322," U":571,"е":266," T":2130,"Fra":1006,"A ":345,"Da":804,"Co":478,"Ch":621,"Du":1025,"Do":201,"De":763,"Di":5828,"Fe":367,"Eu":354,"En":721,"El":212,"Ge":659,"Ga":319,"I ":452,"Fr":1217,"Fo":165,"Fi":216,"II ":246,"C ":278,"Au":486,"Ar":425,"At":187,"As":201,"D ":158,"Ba":648,"Af":2087,"Am":566,"An":491,"Ap":353,"Al":628,"Bu":243,"Br":778,"Ca":399,"Bi":180,"Be":880,"Bo":481,"Bl":161,"Kr":224,"Ko":657,"Le":490,"Li":504,"La":658,"Lu":245,"Lo":347,"Me":800,"Mi":548,"Ma":1360,"Mu":186,"Mo":627,"Ni":257,"Ne":763,"Na":666,"No":1092,"Ok":339,"Ol":206,"Her":157,"Gr":1326,"Go":356,"Ha":534,"He":680,"II":369,"Hi":301,"Ho":503,"Hu":294,"Hy":550,"In":919,"Is":158,"It":218,"Ja":713,"Je":157,"Jo":565,"Ju":623,"Ka":1489,"Ki":194,"Ke":447,"Un":253,"Tu":248,"Tr":236,"To":272,"Th":313,"Te":262,"Ta":276,"V ":280,"Sw":402,"Sy":292,"St":964,"Su":1701,"Wo":181,"Wi":534,"Wa":412,"We":720,"Vo":315,"Vr":251,"Vi":374,"Va":314,"Ve":689,"Pr":551,"S ":157,"Pe":310,"Pa":727,"Po":681,"Pi":230,"Os":236,"Oo":423,"Or":191,"Se":814,"Sc":197,"Si":387,"Sl":222,"Sk":201,"Sp":443,"So":680,"Ru":645,"Ry":194,"Sa":728,"Re":621,"Ri":222,"Ro":746,"SA":233,"Ra":223,"Gre":501,"Gri":383,"Gra":158,"b ":1179,"Gro":254,"a ":7054,"i ":2513,"gd":570,"ge":16432,"ga":1621,"gb":319,"fk":224,"fl":183,"fg":323,"ff":351,"fi":1111,"fh":169,"fs":1224,"fr":2334,"fu":174,"ft":300,"fo":725,"Int":180,"he":6229,"ha":2610,"gn":360,"gl":334,"gi":2135,"gh":921,"gg":418,"gu":592,"gt":1512,"gs":1974,"gr":3459,"go":1385,"dt":211,"du":998,"dw":506,"g ":10256,"ea":936,"eb":3497,"ec":406,"ed":5721,"de":18394,"dd":606,"dg":161,"di":29432,"dh":249,"dj":173,"dm":299,"do":2521,"ds":2062,"dr":1453,"ew":3034,"eu":3603,"ev":2016,"ey":309,"fa":570,"h ":864,"Ind":251,"fd":469,"fe":948,"eh":993,"eg":3187,"ef":995,"ee":12296,"el":15653,"ek":7920,"ei":5726,"ep":2393,"eo":692,"en":27638,"em":4686,"et":10282,"es":15156,"er":33393,"ca":479,"e ":78745,"by":1025,"br":1953,"bu":1057,"bo":2123,"bl":1117,"bi":1966,"bb":156,"be":8513,"db":222,"In ":319,"da":3617,"f ":4067,"ct":207,"co":446,"ck":502,"ci":340,"ch":1526,"ce":547,"c ":311,"az":190,"ay":279,"ba":2057,"d ":15502,"at":11369,"as":9342,"ar":11432,"aw":597,"av":407,"au":883,"ak":2797,"al":9554,"ai":1291,"aj":155,"ap":2087,"am":3989,"an":36357,"ac":615,"ad":4564,"aa":18307,"ab":1064,"ag":2729,"ah":292,"ae":907,"af":1901,"nu":917,"nt":6760,"ns":9243,"nr":212,"no":2885,"nn":1621,"ny":191,"nw":666,"nv":455,"oe":6026,"of":3797,"oc":387,"od":1636,"oa":178,"ob":729,"om":5480,"on":10533,"ok":2525,"ol":5346,"oi":587,"og":2271,"oh":382,"ot":3827,"os":3306,"ov":1152,"ou":2993,"op":4558,"oo":12667,"or":14221,"r ":19504,"ow":1144,"pe":3683,"pg":229,"pa":2371,"pl":1195,"lê":351,"po":1932,"ph":223,"pi":1008,"lo":3369,"lm":315,"ll":2990,"ls":2634,"lp":392,"lw":311,"lv":239,"lu":1548,"lt":993,"ly":716,"o ":2083,"md":261,"ma":3853,"mb":2182,"mg":224,"me":9151,"mi":2940,"mm":802,"mp":1223,"mo":1485,"ië":1437,"mt":249,"ms":966,"mu":1085,"p ":4720,"na":6444,"nb":510,"nc":507,"nd":12581,"ne":5737,"nf":203,"ng":9804,"nh":460,"ni":6127,"nj":300,"nk":2057,"nl":616,"nm":203,"jo":532,"ki":2683,"kh":210,"kg":239,"ke":8584,"ka":6722,"m ":5913,"kw":457,"ky":282,"ks":2318,"kt":2084,"ku":1443,"ko":3908,"kr":2375,"kk":1579,"kl":2200,"km":469,"li":9515,"lh":279,"lk":1158,"lj":705,"le":10290,"ld":1944,"lg":1526,"lf":717,"la":8341,"lb":446,"n ":58065,"hr":313,"ht":702,"hu":1684,"hi":1067,"ho":3048,"dé":160,"id":5034,"ic":1058,"ib":451,"ia":2568,"ig":5540,"if":581,"ie":47836,"hy":348,"k ":9212,"ir":2359,"is":17403,"it":9361,"iu":405,"iv":1008,"iw":219,"ik":8953,"il":3774,"im":1386,"in":25004,"io":1984,"eë":1032,"ip":899,"je":609,"ji":572,"iz":156,"l ":8172,"ja":1960,"wy":994,"z ":242,"wi":1800,"wo":4179,"vy":166,"y ":4684,"wa":9856,"we":6959,"vl":1196,"vi":4040,"vu":178,"vr":662,"vo":4078,"uw":282,"uu":992,"ve":5906,"va":16173,"x ":845,"ui":7822,"uk":678,"ul":2052,"ue":905,"ug":1045,"ur":5410,"us":5098,"ut":907,"um":1711,"un":2596,"up":170,"ty":1434,"tu":2643,"tt":1277,"tw":1177,"tv":217,"ub":1182,"ua":728,"ud":950,"uc":160,"w ":232,"to":5433,"tm":201,"tl":667,"ts":3814,"tr":4026,"tg":532,"te":20430,"tk":279,"tj":177,"ti":5658,"th":1701,"tb":213,"ta":9118,"su":1177,"sv":424,"ss":2799,"st":17122,"sy":1309,"sw":531,"sl":1811,"sk":5006,"sn":242,"sm":693,"sp":2566,"oë":412,"so":3731,"sr":312,"sd":385,"sc":448,"sf":208,"se":15556,"sh":473,"sg":396,"sj":338,"si":8436,"u ":1834,"sa":2367,"sb":577,"rr":652,"rs":6262,"rt":4139,"ru":2543,"rv":1198,"rw":1199,"ry":2450,"rp":1265,"ro":8165,"rn":1586,"rm":2087,"rl":1734,"rk":2996,"ri":11752,"rh":614,"rg":2653,"rf":378,"re":10923,"rd":7372,"rc":234,"rb":955,"ra":7710,"t ":22731,"qu":168,"s ":35284,"px":614,"Hy ":529,"py":231,"pt":765,"pu":844,"pp":1058,"pr":3258,"ps":659,"wê":320,"zi":170,"ze":169,"za":209,"yg":162,"ye":406,"yf":643,"yd":927,"yw":439,"ys":1141,"yn":1041,"yl":288,"yk":1145,"Apr":247,"Aug":272,"Afr":2048,"Ame":464,"Ber":218,"Bel":171,"Bre":163,"Bra":191,"Bri":282,"Des":273,"Daa":460,"Chr":224,"Cha":171,"ër":307,"ël":325,"êr":697,"ë ":1979,"ê ":310,"é ":228,"Dit":1028,"Die":4537,"Dui":918,"Ned":417,"Nas":187,"Nov":238,"Noo":595,"Okt":256,"Oli":158,"Oos":361,"Par":313,"Pro":177,"Pre":186,"SA ":161,"Ita":207,"Jan":348,"Joh":290,"Jul":297,"Jun":245,"Kaa":543,"Kan":220,"Kat":191,"Kar":171,"Ker":270,"Kon":276,"Lat":181,"Lit":162,"Mei":281,"Mar":370,"Maa":286,"Mon":210,"Mid":157,"Wil":165,"Wes":439,"Vry":192,"Vol":161,"êre":674,"Swe":193,"Sy ":252,"Sui":1515,"Sta":443,"Ste":208,"Sep":228,"Spa":253,"Rus":560,"Sch":162,"Rep":214,"Rom":176,"Ver":555,"Uni":236,"The":196,"Tur":159,"bin":400,"blo":205,"bli":525,"bla":215,"boe":246,"boo":276,"bor":587,"bou":330,"ban":283,"bal":289,"bai":191,"baa":372,"bas":270,"bar":272,"beh":366,"beg":372,"bee":325,"bed":285,"ber":1916,"bel":540,"bek":1148,"bew":349,"bev":630,"bes":1308,"bet":510,"bie":1052,"ce ":276,"bri":159,"bro":237,"bra":211,"bre":258,"bru":1062,"bur":584,"by ":693,"am ":1182,"ake":292,"al ":2759,"ain":204,"ak ":856,"aie":241,"agt":446,"anu":467,"ann":632,"ant":1705,"ans":3841,"ane":404,"ang":1856,"ani":742,"anj":191,"ank":961,"ap ":635,"ana":788,"anc":195,"and":5528,"amm":186,"amp":480,"ami":512,"ame":657,"amb":236,"ama":204,"alt":231,"als":160,"all":667,"alk":171,"alg":320,"ali":1276,"ald":217,"ale":2352,"alf":209,"ala":367,"an ":18298,"aks":261,"akt":740,"akl":166,"abe":229,"abi":201,"aby":216,"ae ":624,"aag":175,"aad":172,"aak":679,"aai":350,"aan":6190,"aal":1515,"aam":1083,"aas":579,"aar":5293,"aap":567,"aat":1563,"ad ":2565,"afg":266,"ai ":311,"age":184,"afd":268,"adm":206,"adi":436,"ade":539,"ag ":1304,"ads":176,"ach":166,"ada":249,"af ":494,"at ":6755,"arg":256,"are":965,"ard":1124,"ara":390,"aro":332,"arn":185,"arm":157,"arl":301,"ark":397,"ari":1177,"arv":249,"ars":463,"art":1494,"ary":171,"asi":1669,"ase":210,"aso":169,"ar ":3216,"apa":189,"app":418,"aps":269,"as ":5230,"awe":308,"awi":169,"ata":346,"ast":673,"ass":518,"ato":426,"ate":1382,"ati":871,"ats":404,"atu":409,"aty":167,"aus":156,"jaa":1087,"jar":470,"je ":175,"joe":306,"jin":161,"jie":306,"ito":170,"itt":191,"its":1623,"isk":182,"ism":266,"iss":374,"ist":1582,"ita":608,"ite":1331,"itg":386,"iti":469,"ius":176,"ium":203,"ivi":590,"ive":294,"is ":12546,"ion":1252,"eër":158,"ipa":265,"ir ":1648,"isi":1018,"ise":601,"isa":220,"ire":181,"it ":3772,"kil":644,"kie":536,"kin":914,"km ":266,"kgr":173,"kee":210,"kei":339,"kel":962,"ken":2090,"kep":166,"ker":1342,"ke ":3014,"kra":345,"kse":472,"kry":1085,"kri":662,"kou":249,"kor":369,"kop":214,"koo":391,"kon":866,"kom":903,"kol":246,"koe":157,"ks ":710,"kke":1272,"kki":178,"klu":430,"kle":511,"kla":387,"kli":749,"kat":157,"kar":183,"kas":204,"kap":818,"kan":1256,"kal":611,"kaa":1596,"ka ":1388," Ga":319," Ge":658," Fo":161," Fr":1217," Fi":213," Ha":534," He":680," Go":354," Gr":1318," Hy":549," Hu":294," Ho":502," II":202," Hi":301," Ja":710," Is":157," It":218," In":916,"han":779," Ka":1486,"hal":311," Ke":447,"haw":164," Ki":192,"har":356," Jo":563," Ju":622,"haa":238,"had":164," La":657," Le":488," Li":502," Ko":657," Kr":224," Ma":1348," Mi":547," Me":799,"he ":399," Lo":346," Lu":244," Ne":762," Na":662," Ni":257," Mo":624," Mu":186,"hel":273,"hei":994,"hee":465,"hed":169,"het":2911,"her":350,"hem":255," Ap":349," Am":563," An":491," Al":626," Af":2082," Ba":645," Au":486," At":187," As":200," Ar":422," Be":877,"hie":290," Bi":179," Bl":161," Bo":479," Br":777," Bu":243,"his":173," Ca":384," Ch":612," Co":473," Da":803," Di":5802," De":761," Do":196," Du":1024," El":212," En":720," Eu":354," Fe":367," Wo":179," Wi":530," We":720," Wa":412,"god":193,"gs ":887,"gor":522,"gro":2150,"gra":537,"gri":320,"gre":401," Os":236," Or":191," Oo":422," Po":674," Pi":229," Pe":309," Pa":725,"gst":406," No":1092," Ol":205," Ok":339,"gte":962,"gti":391," Ra":221," Ro":743," Re":620," Ri":222," Pr":547,"gus":284," Sy":292," Sw":400," Su":1700," St":953," Ta":273," Th":307," Te":261," Tr":236," To":270," Ry":194," Ru":645," Sa":724," Si":385," Sc":196," Se":811," So":678," Sp":441," Sk":201," Sl":222," Va":313," Ve":669," Vi":371," Vo":314," Vr":251," Tu":243," Un":253," ja":1102,"ial":357,"ian":256," in":12303,"iaa":736," is":11238," ka":1533," ki":531," ke":481,"id ":2425," ha":612," he":3438," gr":2075," go":365,"ia ":794," hy":292," hi":477," ho":1750," hu":727,"iet":320,"ieu":180,"iew":413," ni":722,"iel":277," ne":437,"ien":998," na":2339,"ier":2228,"ies":4471,"ied":1248,"ief":177,"iek":2103," mu":691,"ig ":1346," mo":667," om":1497," on":2106," of":1952,"ifi":218," no":1205," le":910," li":598," n ":10980," la":1290," ku":387,"ich":258,"ie ":34696," km":407," kl":879,"ica":209," kr":319," ko":1672," me":4100," mi":830,"ids":257," ma":1329," lu":186,"idi":291,"ide":993,"idd":457,"ida":156," lo":197," af":820," aa":2320," ad":269," am":322," an":759," ak":286," al":829," ar":263," at":229," as":2284," ba":599,"il ":459," bi":320," be":5430," bo":565," bl":263," by":612," bu":213," br":340,"ika":2950,"igd":381,"ige":1604,"igh":698,"igi":270,"igg":185,"igt":498,"igs":156,"ik ":2305," en":9738,"imp":231," ei":517," el":502,"ime":187," ek":223," ee":1730,"ind":1030,"ina":506," fa":191,"inn":302," fo":227,"int":638,"ins":1349,"ine":545,"ing":6095," fi":368,"ini":615,"ink":417," ge":8191," ga":169,"inw":455,"ikk":629," ch":185,"ike":1814,"ila":498," da":1923,"in ":12178,"iku":209,"iks":287," do":1111,"ilo":514,"ill":662," dr":523," de":3947,"ilj":228,"ili":684,"ild":294," di":25510,"imb":245,"eë ":693,"io ":196," du":309," wê":298,"hom":166,"hou":360,"hoo":1325,"hoe":410," wy":201,"hul":552,"hui":260,"hri":224,"ht ":578," ru":233," sa":888," se":2315," si":590," sl":329," sk":1250," sp":887," so":2211," ra":237," re":1576," ri":825," ro":614," pr":1589," s ":207," px":614,"hy ":302," ou":447,"hum":674," oo":2639," op":2809," or":325," pe":402," pa":556," pl":641," po":737," lê":242," wa":7840," we":1395," wo":2888," wi":454," va":14670," ve":4043," vo":2359," vr":575," vi":2068," vl":594," ty":439," tw":582," tu":692," ui":1746," ta":895," sw":227," sy":1183," st":4293," su":859," tr":387," to":1857," th":729," ti":190," te":2715,"ffe":165,"fer":157,"fel":155,"fha":158,"fge":290,"fam":176,"fde":429,"eta":359,"ete":1299,"eti":372,"esp":358,"eso":210,"est":2951,"ess":405,"eun":234,"eto":320,"etr":438,"ets":217,"ett":493,"eve":456,"eva":262,"evo":907,"evi":274,"eur":2292,"eus":242,"ewi":337,"ewe":1704,"ewo":449,"ey ":181,"ewa":222,"epe":254,"er ":10617,"epa":228,"eor":221,"es ":4626,"ept":277,"epu":400,"epr":184,"erk":2067,"erl":875,"eri":1765,"erg":1022,"erh":416,"ere":1861,"erf":286,"erd":1514,"era":1470,"erb":529,"et ":6083,"esk":1018,"esl":228,"esi":976,"ese":3607,"eu ":338,"erv":860,"erw":949,"err":349,"ert":1101,"ers":4583,"ern":1142,"erm":861,"erp":342,"ero":382,"ekg":155,"ekk":206,"eko":474,"eks":950,"ekt":701,"en ":13492,"ela":904,"eld":1199,"elf":322,"ele":2593,"eli":1906,"elj":427,"elg":226,"elk":209,"ell":778,"elo":234,"els":1983,"elt":333,"ely":255,"emb":839,"ema":484,"eme":1266,"emo":181,"emi":456,"ep ":699,"ene":1142,"enh":254,"eng":314,"enb":269,"ena":610,"end":3112,"eno":500,"enn":400,"enk":275,"eni":1151,"ens":2864,"ent":2318,"ego":497,"ege":690,"egi":516,"eha":370,"egr":238,"egs":217,"egt":193,"eho":266,"ehe":259,"ek ":1799,"eis":330,"eil":544,"ein":1010,"eie":633,"eid":1307,"el ":3516,"eit":680,"eke":2739,"eka":220,"em ":967,"gin":784,"gie":714,"ght":548,"gep":249,"gen":1564,"get":297,"ger":1248,"ges":2014,"gev":788,"gew":944,"gee":448,"ged":475,"geb":2499,"geh":356,"geg":181,"gem":756,"gel":1995,"gek":350,"gde":427,"ge ":1916,"gaa":266,"gan":539,"ga ":157,"fst":852,"fri":2089,"for":371,"fie":369,"fil":208,"fin":174,"fis":177,"da ":327,"de ":6409,"daa":645,"dag":700,"dae":480,"dat":659,"dan":233,"dam":165,"dde":490,"ch ":316,"cha":160,"ck ":233,"che":490,"ed ":1090,"eba":159,"ebe":354,"ebi":752,"ebo":768,"ebr":1168,"ei ":821,"ega":168,"eek":631,"een":2520,"eel":2072,"eem":410,"eed":587,"ees":884,"eer":3295,"eeu":449,"eet":195,"edi":638,"ede":2561,"eda":161,"eg ":316,"eds":321,"edr":340,"ee ":892,"ef ":280,"dwe":310,"dus":171,"dor":875,"doo":416,"don":160,"dom":227,"ds ":353,"dmi":211,"doe":283,"dst":428,"dui":309,"dri":421,"dra":423,"dry":204,"dsk":181,"dse":527,"dia":294,"der":4829,"des":476,"deu":1676,"dee":1279,"del":1695,"dek":186,"den":1206,"do ":172,"din":875,"dio":177,"dis":425,"dit":656,"die":24964,"dig":1168,"dik":198,"rhe":301,"rga":496,"rgi":335,"rge":595,"ret":312,"res":944,"rg ":777,"rea":245,"ree":1091,"ref":257,"red":294,"rei":545,"reg":1039,"ren":1300,"rek":765,"rel":674,"rep":191,"rf ":180,"rdo":215,"rdi":841,"rde":1873,"re ":2607,"rd ":3667,"ras":532,"rat":587,"rbi":190,"rba":160,"rbe":287,"rag":291,"ran":2011,"ram":317,"ral":832,"rak":247,"raa":1046,"raf":284,"rad":331,"rs ":1922,"ros":273,"rot":330,"rom":305,"ron":1072,"roo":1778,"rop":575,"rou":212,"rov":708,"rod":199,"rol":315,"roe":1277,"rog":195,"rno":196,"rp ":728,"rna":508,"rne":469,"rmo":164,"rma":539,"rme":324,"rmi":175,"rlo":320,"rli":409,"rle":270,"rla":508,"rks":184,"rko":248,"rki":199,"rkl":203,"rke":440,"rka":271,"rm ":692,"rio":174,"rit":493,"ris":571,"riv":501,"rig":863,"ril":278,"rik":3384,"rin":1384,"ria":924,"ric":236,"rie":2029,"rk ":1040,"rwe":410,"rwy":498,"ryf":393,"rui":1143,"rug":256,"rum":244,"ruk":231,"rus":225,"rva":502,"rvl":353,"rvo":192,"rwa":171,"ry ":383,"rsk":872,"rsi":432,"rso":249,"rsp":591,"rsa":225,"rse":478,"rta":186,"rst":1083,"rtk":160,"rto":274,"rte":620,"rti":334,"rua":209,"rty":351,"rt ":1413,"rre":272,"saa":540,"sal":170,"sam":303,"san":408,"sas":204,"sa ":155,"ryw":338,"rys":282,"ryk":576,"sge":305,"sie":4039,"sid":185,"sia":299,"sit":436,"sis":296,"sip":279,"sin":541,"sio":799,"sil":194,"sim":173,"sik":231,"sif":160,"sig":289,"sbu":231,"se ":9840,"sch":268,"ser":501,"ses":400,"set":250,"seu":239,"sea":162,"see":618,"sed":264,"sen":1323,"sem":298,"sel":1093,"sek":186,"spo":405,"spr":756,"spe":934,"spa":260,"sow":508,"som":247,"son":545,"soo":954,"soe":195,"sok":377,"st ":267,"sli":202,"slu":297,"sky":183,"sla":1006,"sle":205,"ski":804,"sko":594,"skr":1152,"sku":244,"ska":1212,"ske":665,"sië":283,"sma":173,"sme":382,"sse":1275,"ssa":198,"ssi":922,"ste":6829,"sta":5065,"sto":805,"sti":1396,"stu":693,"str":1673,"sty":226,"sui":596,"sve":167,"sy ":1199,"swa":313,"tal":1301,"taa":2499,"tad":2323,"tau":165,"tat":456,"tas":164,"tan":1021,"te ":8469,"ta ":339,"pa ":202,"pe ":459,"par":608,"pas":176,"paa":333,"pal":324,"pan":428,"pge":207,"pen":295,"per":1379,"pes":438,"pee":201,"pel":568,"pla":660,"lê ":268,"pli":169,"ple":241,"pie":480,"por":394,"poo":160,"pos":197,"pol":518,"ppy":184,"ppe":636,"pst":229,"pub":435,"pte":575,"pra":251,"pri":484,"pre":726,"pro":1677,"pun":246,"px ":614,"py ":166,"ra ":424,"ngo":161,"ngr":289,"ngs":1292,"nge":2327,"nhe":276,"nel":314,"nen":189,"nem":225,"ner":1014,"net":468,"nes":533,"ng ":4906,"nee":762,"nce":206,"ne ":1530,"ndr":216,"nds":657,"ndo":326,"ndi":878,"nde":5081,"nda":453,"nal":790,"nam":291,"nad":316,"naf":372,"nab":229,"naa":1198,"nd ":4245,"nat":282,"nas":677,"na ":1572,"nwo":542,"nus":209,"nua":266,"ntw":393,"nto":201,"nts":300,"ntr":543,"nti":571,"ntl":164,"nta":457,"nte":1815,"nst":787,"nse":3345,"nsi":1079,"nsl":207,"nsk":498,"nt ":1757,"ns ":2476,"nog":456,"noe":477,"noo":659,"nom":368,"nne":904,"nni":442,"nië":246,"nli":373,"nke":345,"nkl":391,"nks":179,"nkr":453,"nje":156,"nig":640,"nie":1831,"nk ":274,"niv":210,"nis":1512,"nin":804,"ogr":272,"ogi":423,"oi ":216,"oha":228,"oeë":178,"ok ":1432,"ol ":554,"ock":164,"oe ":303,"ode":551,"odi":176,"ods":177,"of ":2323,"oek":499,"oel":276,"oem":563,"oeg":231,"oei":336,"oer":752,"oes":295,"oet":302,"oen":602,"oep":714,"odu":188,"oed":477,"og ":895,"ofs":803,"oew":261,"od ":254,"obe":382,"oud":510,"oue":197,"ote":350,"ott":175,"ots":913,"oto":266,"ost":637,"osi":266,"ose":346,"oss":176,"oso":190,"owa":484,"owe":208,"ovi":678,"ove":370,"ous":302,"our":167,"out":306,"opo":205,"opp":449,"ope":438,"opg":213,"opa":195,"os ":1171,"oon":731,"ool":561,"oom":393,"ook":1376,"ooi":288,"oof":1146,"oog":389,"ood":288,"or ":1152,"oot":1351,"oos":958,"oor":4776,"oop":341,"ork":260,"orl":386,"orm":964,"orp":858,"ord":4583,"ore":773,"org":587,"ori":1212,"ou ":999,"ort":1219,"ors":871,"orw":195,"ot ":1528,"orb":186,"ora":235,"ola":171,"on ":1522,"oli":772,"oll":288,"olk":702,"ole":263,"olg":904,"ols":270,"olo":636,"om ":1870,"okk":553,"ona":980,"ond":1915,"one":1178,"ong":620,"oni":1012,"onl":220,"onk":232,"onn":184,"ono":391,"ons":511,"ont":1339,"oma":425,"ome":845,"omi":324,"omm":454,"omp":297,"oms":595,"op ":2264,"la ":334,"le ":3834,"lf ":175,"lde":601,"laa":982,"lad":180,"lag":434,"lak":490,"lan":4154,"lar":155,"lat":361,"las":433,"ld ":695,"kus":410,"kun":548,"kul":242,"kwe":204,"kwa":191,"kte":822,"kst":257,"ksi":463,"ktr":342,"ktu":210,"kti":247,"kto":369,"ls ":1008,"lon":293,"lom":430,"loo":382,"loe":423,"log":655,"los":274,"lië":349,"lti":157,"lub":411,"lug":221,"lst":643,"lte":252,"lse":623,"lge":754,"lew":250,"leu":193,"les":329,"let":347,"ler":415,"lem":358,"len":1056,"lek":605,"lei":1010,"leg":257,"lee":477,"led":218,"lg ":483,"lo ":169,"lla":325,"lle":1578,"lli":615,"lke":200,"lki":447,"ljo":223,"ll ":176,"lja":430,"lit":831,"lis":504,"leë":449,"lin":1208,"lim":201,"lid":165,"lia":364,"lik":2917,"lig":818,"lie":1618,"ma ":226,"mb ":655,"maa":1244,"mag":221,"mar":331,"mas":207,"mal":270,"man":726,"mat":394,"mba":172,"mbi":179,"mbe":814,"mbo":161,"me ":936,"mde":163,"med":223,"mee":1533,"met":2981,"mes":247,"mer":991,"mel":330,"men":1550,"lui":390,"lus":194,"lwe":213,"lyk":221,"lyn":187,"mpi":220,"mpe":208,"mpo":176,"mpt":267,"ms ":488,"moe":196,"mod":233,"mon":329,"mst":248,"mus":488,"mun":417,"ër ":180,"mge":191,"min":806,"mil":465,"mit":231,"mig":184,"mie":523,"mid":310,"ië ":1136,"mme":353,"wêr":319,"yst":183,"ys ":680,"ywe":370,"ye ":306,"yf ":380,"yde":281,"yds":165,"yd ":230,"yn ":461,"yns":175,"yk ":810,"wys":531,"wor":2620,"woo":760,"won":526,"we ":1260,"wes":799,"wer":1583,"wet":305,"wen":427,"wel":545,"weg":270,"wee":1257,"wis":166,"wit":342,"wie":194,"win":417,"wil":177,"wik":231,"wan":300,"wat":5174,"war":532,"was":2236,"waa":1031,"vry":194,"vro":313,"vir":1570,"vin":921,"vie":880,"vis":289,"vla":709,"vlo":280,"voe":444,"vol":1592,"voo":1083,"vor":625,"ver":4566,"ven":170,"vem":236,"vel":250,"vee":302,"val":319,"van":14723,"vat":155,"vaa":414,"uwe":229,"uur":863,"usl":180,"usi":606,"use":380,"ust":585,"uss":1129,"ute":176,"uto":171,"us ":1998,"ure":395,"urg":669,"uri":191,"urk":167,"uro":352,"urs":211,"urt":189,"ur ":2547,"umb":689,"ume":172,"unt":325,"uns":289,"uni":820,"und":530,"um ":614,"ult":270,"ull":459,"uli":358,"un ":219,"uid":2285,"uik":850,"uim":162,"uis":508,"uk ":200,"uit":3378,"ul ":272,"ugb":161,"ugu":278,"ude":184,"udi":240,"ue ":322,"ug ":159,"ub ":406,"uar":522,"ubl":464,"ud ":181,"tyn":228,"ty ":384,"tur":232,"tus":988,"tuu":617,"tui":232,"tud":171,"tyd":628,"twi":269,"twe":751,"ts ":533,"tre":1022,"tra":1128,"tri":607,"tru":366,"tro":780,"tse":746,"tsk":298,"tsl":425,"tst":993,"tte":641,"tti":226,"to ":272,"tof":244,"toe":713,"tob":268,"tot":1108,"tom":182,"ton":586,"tol":317,"tor":808,"too":280,"til":187,"tik":334,"tie":1846,"tig":1053,"tis":241,"tin":826,"tio":267,"thu":695,"tkl":165,"tli":191,"tla":301,"tem":732,"ten":1059,"tei":844,"tek":528,"tel":2135,"tee":779,"teg":166,"ted":237,"th ":270,"teu":212,"tes":357,"ter":4231,"tge":442,"the":380},"n_words":[1541130,1808182,1328687],"name":"af","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/an.json b/contrib/languages-data/an.json
new file mode 100644
index 0000000..299ca71
--- /dev/null
+++ b/contrib/languages-data/an.json
@@ -0,0 +1 @@
+{"freq":{"D":4007,"E":9481,"F":6264,"G":7528,"A":24874,"B":11416,"C":19854,"L":12511,"M":14032,"N":5019,"O":7507,"H":3089,"I":5567,"J":2444,"K":1244,"U":4208,"T":7124,"W":1232,"V":6098,"Q":391,"P":13954,"S":13768,"R":7862,"Y":2048,"X":966,"Z":2105,"f":30904,"g":39756,"d":198344,"e":418034,"b":46837,"Fed":103,"c":184692,"a":500037,"n":332450,"o":309189,"l":153654,"m":96885,"j":1745,"Fel":132,"k":9548,"h":38996,"Fen":49,"i":297238,"w":1593,"v":28687,"Fer":367,"u":160116,"t":216624,"s":206494,"r":234141,"q":14602,"p":79744,"z":13501,"y":82928,"x":11341,"ª":69,"º":77,"²":4897,"Í":100,"É":262,"Á":131,"À":43,"Fil":103,"ï":110,"í":10894,"Fin":74,"ì":53,"ë":76,"ê":61,"é":16262,"è":1639,"ç":714,"ä":54,"Fie":45,"ã":102,"â":100,"Fig":108,"á":13401,"à":575,"ü":700,"ú":1261,"ø":41,"ù":89,"Fis":88,"ö":123,"ô":70,"ò":886,"ó":30243,"ñ":789,"ā":118,"ī":53,"ş":47,"ł":55,"š":45,"ţ":70,"ū":44,"Fa ":481,"Fan":52,"Fal":65,"Far":69,"Fab":67,"Aín":47,"Era":60,"Eri":53,"Ess":225,"Est":1829,"Esp":2346,"Ern":54,"Esc":294,"ə":50,"Esl":79,"Eus":72,"Eur":458,"Exe":241,"ˈ":99,"El ":405,"́":61,"En ":645,"Ele":55,"Enc":143,"μ":51,"ν":114,"ο":175,"Eng":50,"θ":57,"ι":93,"κ":64,"λ":98,"ε":64,"η":68,"α":169,"γ":43,"ά":43,"ί":50,"Emp":168,"Emi":77,"Eli":66,"Ent":56,"ό":42,"σ":63,"ς":149,"ρ":123,"τ":88," l":24121," m":23977," n":9906," o":46236," h":9312," i":7953," j":137," k":5618," d":142714," e":62754," f":15333," g":3703,"ч":80,"р":160," a":64432,"с":155," b":4777,"т":77," c":53853,"у":58," y":53553," x":86," z":803," u":35543," t":11133," w":158," v":4587," q":10572," p":39916," s":31762," r":12559," J":2437," K":1235,"Cál":77," H":3071," I":5555," N":5003," O":7480," L":12489," M":14016," B":11404," C":19641," A":24795,"С":41," F":6210," G":7491," D":3992," E":9455,"л":143," Z":2101,"к":142," Y":2045,"й":58," X":966,"и":262,"п":41,"о":215,"н":140,"м":41," S":13749,"Ges":43,"Ger":648," R":7849,"в":147," Q":385," P":13942,"а":282,"Geo":165," W":1227," V":6094,"Gen":180," U":4207,"е":199,"д":65," T":7101," á":138," ª":48," º":71," Á":127," À":43,"Gio":49," É":259," Í":100,"Gil":43,"Gir":65,"Giu":47,"י":48,"Gan":52,"Gal":553,"Gav":41,"Gau":302,"Gar":1065,"Gas":177,"Gab":72,"و":72,"ي":130,"ل":158,"م":103,"ن":111,"Fun":74,"Fue":512,"د":70,"ج":41,"ح":44,"ب":113,"ة":63,"ا":224,"ع":41,"س":66,"ر":91,"Fro":85," ˈ":61,"Flo":152,"Fla":78,"Fra":1775,"Fri":136,"A ":6623,"Fre":184,"Fon":159,"For":357,"F ":41,"Da":680,"Cu":873,"Cy":45,"Cl":510,"Co":3422,"Cr":538,"Ce":1517,"Ch":3175,"Ci":1041,"Ec":152,"Ed":409,"Ea":51,"Eb":284,"Du":470,"Do":659,"Dr":118,"De":985,"Di":657,"Fe":747,"H ":44,"Fa":903,"Ez":49,"Eu":622,"Ev":68,"Ex":340,"Er":278,"Aí":55,"Et":88,"Es":4887,"En":989,"Em":290,"Ei":68,"El":695,"Ge":1116,"Cá":129,"Ga":2449,"I ":1360,"Fu":667,"Fr":2222,"Fo":713,"Bé":72,"Fl":289,"Fi":498,"B ":58," С":41,"II ":724,"C ":328,"Av":152,"Au":1363,"Ar":3874,"Aq":804,"At":949,"As":1097,"D ":277,"Ba":3644,"Az":169,"Ay":82,"Af":232,"Ag":240,"Ab":293,"Ac":412,"Ad":285,"Am":430,"An":2149,"Ap":98,"Ai":212,"Al":5289,"Hit":59,"His":162,"Bu":1318,"Br":1101,"Ca":8068,"E ":158,"Bi":1877,"Be":1779,"Hil":81,"Bo":1120,"Bl":233,"Ku":88,"Kl":47,"Kr":75,"Ko":157,"Le":3160,"Li":1190,"N ":108,"Gú":51,"La":4114,"Lu":751,"Ly":43,"Ll":306,"Lo":1960,"Me":3060,"Mi":1407,"O ":3098,"Ma":5263,"Mc":66,"Mu":1274,"Mo":2562,"Ni":338,"Ne":437,"Na":1458,"P ":71,"Hel":71,"Hei":41,"Nu":796,"No":1516,"Ol":435,"Om":55,"On":170,"Oh":48,"Oc":530,"Od":84,"Hen":171,"Hes":57,"Ob":102,"Her":278,"Gi":355,"Cé":64,"Gl":100,"Gr":1086,"Go":468,"Gu":1597,"J ":56,"Ha":782,"He":720,"Hi":448,"Ho":594,"Hu":367,"Ib":231,"Id":48,"Ig":114,"Ip":108,"Im":754,"In":710,"Il":297,"Is":1033,"It":588,"Ir":296,"Ja":569,"L ":794,"Ji":54,"Je":311,"Jo":919,"Had":44,"Jr":59,"Ju":422,"Fú":72,"Ka":309,"Han":91,"M ":73,"Ham":67,"Kh":65,"Har":170,"Ki":176,"Hau":83,"Ke":203,"Us":67,"Ut":63,"Ur":312,"Un":1997,"Ul":142,"Ue":1325,"Uc":50,"Tu":387,"Tr":784,"Pé":62,"To":1748,"Th":491,"Ti":748,"Te":1399,"Ta":1245,"V ":289,"Sy":56,"St":450,"Su":913,"Wo":126,"Wu":48,"Wi":414,"Sã":53,"Wh":41,"Sé":45,"Wa":266,"We":251,"Vo":163,"Rí":105,"Vi":2398,"X ":302,"Va":2416,"Ve":691,"Uz":41,"Mé":58,"Pu":602,"Pr":1737,"S ":375,"Pe":1603,"Gua":636,"Pa":3522,"Gue":449,"Gub":94,"Lé":44,"Gui":272,"Pl":536,"Po":1463,"Pi":4009,"Ph":120,"Os":1276,"Ot":107,"Ou":63,"Ov":54," ا":100,"Op":71,"Or":1187,"R ":115," ب":40,"Oz":48,"Se":3318,"Sc":255,"Si":1208,"Sh":202,"Sp":125,"So":1654,"Ru":549,"U ":53,"Nù":59,"Sa":4961,"Ná":254,"Re":2929,"Ri":1399,"Ro":1931,"Qu":333,"T ":55,"Mú":79,"Ra":687,"Gre":227,"Gri":72,"Gra":645,"b ":2959,"Gru":82,"Gro":44,"a ":189570,"Ye":1235,"Ya":59,"Yo":603,"Yu":72,"Xa":108,"Tè":61,"Xi":132,"Gon":55,"Gol":63,"Gor":90,"Za":1741,"Ze":94,"Zi":59,"Zo":51,"Zu":79,"God":44,"i ":5556,"bó":122,"gd":66,"ge":1559,"cá":136,"ga":9445,"Inf":101,"bé":284,"fl":536,"bè":98,"fg":41,"ff":208,"fi":10107,"fr":7629,"fu":1368,"ft":72,"Int":142,"fo":3130,"Ins":100,"bí":66,"j ":55,"cú":47,"có":298,"Ipa":95,"dá":285,"he":5671,"ha":11432,"gn":724,"cé":6120,"gm":75,"gl":1957,"gi":469,"gh":348,"gg":120,"gu":5563,"gt":79,"gs":124,"gr":4212,"cí":202,"go":11703,"dt":51,"du":1967,"dv":108,"dw":105,"dy":121,"g ":1073,"ea":2408,"eb":3508,"ec":11322,"ed":4670,"de":103587,"dd":182,"dg":113,"di":13701,"Ile":92,"dh":96,"dm":483,"dl":77,"do":9223,"dn":56,"ds":221,"Ill":119,"dr":2386,"ew":230,"ex":1637,"añ":291,"eu":4920,"ev":2707,"ey":6445,"ez":1432,"fa":2850,"h ":1778,"Ind":230,"bá":102,"fe":3740,"eh":152,"eg":5665,"ef":994,"ee":482,"el":19178,"aç":51,"ek":155,"ej":370,"ei":5762,"ep":5798,"Imp":731,"eo":1144,"en":93499,"em":9197,"et":7416,"es":36862,"aï":66,"er":42074,"aí":1342,"eq":445,"ca":36727,"e ":149038,"by":52,"bs":272,"br":9258,"bu":1135,"bt":83,"bn":53,"bo":2387,"bp":52,"bl":8980,"bf":48,"bi":6912,"bb":79,"bc":225,"bd":79,"be":4754,"dc":59,"da":17903,"f ":486,"cy":72,"cu":2858,"ct":6432,"cs":190,"cq":106,"cr":2764,"co":36494,"cn":93,"ck":597,"cl":2381,"ci":55866,"ch":19957,"ce":5873,"cc":5812,"c ":1440,"aC":163,"az":1496,"ay":2308,"ba":8653,"d ":44350,"at":36951,"as":32668,"ar":48825,"aq":326,"ax":419,"aw":197,"av":3185,"au":6954,"ak":399,"al":37899,"ai":7870,"aj":395,"ao":175,"ap":3319,"am":13303,"an":60138,"ac":18793,"ad":8656,"aa":113,"ab":11063,"ag":9213,"ah":422,"ae":821,"af":2379,"nu":1742,"nt":42111,"ns":10765,"nr":248,"nq":367,"no":15371,"nn":1413,"q ":75,"nz":961,"ny":8884,"nv":500,"oe":616,"of":4127,"oc":10213,"od":4405,"oa":667,"ob":10136,"om":23722,"on":36685,"ok":161,"ol":10871,"oi":2459,"oj":139,"og":2423,"oh":450,"ot":2800,"m²":4897,"os":31113,"ov":10545,"ou":3052,"op":2703,"oo":407,"or":29437,"oq":218,"r ":21128,"ox":224,"ow":337,"oz":1792,"oy":492,"là":56,"lá":4259,"pe":9762,"Igu":42,"pa":17607,"pc":112,"pl":2368,"lè":221,"lé":756,"po":17316,"ph":314,"pi":11513,"lo":14735,"ln":95,"lm":4777,"hé":116,"hè":52,"ll":16721,"ls":3809,"lr":42,"hí":504,"lq":133,"lp":382,"hó":68,"lv":726,"lu":5364,"lt":4369,"lz":119,"ly":296,"o ":118703,"ià":102,"ma":22094,"mb":4820,"iá":1740,"me":16458,"mf":70,"eţ":45,"iè":238,"mi":9268,"mn":143,"mm":270,"ié":1837,"mp":6474,"mo":7848,"ms":136,"mu":18950,"iñ":101,"ió":20432,"my":57,"p ":402,"na":42520,"nb":139,"nc":27078,"nd":12460,"ne":13823,"nf":817,"ng":5331,"nh":555,"ni":23728,"nj":111,"nk":294,"nl":294,"nm":99,"Ibe":153,"ju":130,"fí":317,"jo":403,"ki":580,"kh":127,"ke":399,"ka":550,"fú":249,"m ":1910,"gó":1532,"ky":77,"ks":168,"ku":119,"gò":79,"ko":322,"kr":45,"kl":108,"km":5396,"li":16018,"lh":504,"lk":130,"há":74,"le":12141,"ld":1220,"lg":344,"lf":641,"hâ":41,"gü":326,"la":35610,"lc":1027,"lb":1209,"gú":46,"n ":117507,"hr":216,"hs":61,"dí":2145,"dò":202,"dó":142,"ht":207,"hu":6167,"hi":10022,"hn":341,"ho":1603,"hl":125,"dè":107,"dé":369,"hm":76,"id":11407,"ic":35920,"ib":3636,"ia":33192,"ih":91,"ig":5438,"if":2242,"ie":19241,"hy":121,"k ":1459,"eí":56,"iq":449,"ir":8944,"is":13378,"it":41046,"eñ":206,"iu":2673,"iv":4272,"eó":80,"ix":7972,"ii":54,"ij":127,"ik":367,"il":9035,"im":5807,"in":28928,"io":20463,"ip":9686,"je":160,"ji":76,"iz":2558,"iy":76,"l ":26412,"ja":752,"xi":5714,"xo":1208,"tè":189,"té":212,"xm":56,"xp":240,"tí":1477,"tó":3270,"xt":359,"xu":93,"só":139,"sú":82,"z ":3529,"xc":69,"xa":1261,"tá":2483,"xe":1372,"wi":289,"sè":69,"sé":315,"wn":65,"wo":96,"sí":199,"ws":51,"sò":136,"ró":536,"rò":79,"y ":30607,"rú":70,"wa":371,"sá":61,"we":271,"rè":89,"vl":53,"ré":394,"vi":14449,"vu":203,"vr":185,"rí":2661,"vo":2258,"uz":858,"uy":8987,"ux":238,"uv":360,"rá":410,"ve":4086,"rà":73,"va":6899,"qü":104,"x ":772,"ui":5582,"uj":105,"uk":95,"ul":4535,"ue":24260,"uf":231,"ug":3258,"uh":51,"uq":149,"ur":7842,"us":8828,"ut":4110,"um":1883,"un":51530,"uo":167,"up":3752,"ty":165,"tx":92,"tz":1335,"tu":10787,"tt":792,"tw":45,"pó":230,"ub":3284,"ua":11407,"ud":6151,"uc":2944,"w ":229,"to":49861,"tn":174,"tm":126,"tl":1081,"pè":62,"ts":4585,"tr":16094,"pí":43,"tp":51,"tg":118,"tf":100,"te":23226,"ti":24986,"th":1241,"v ":145,"tb":466,"tc":162,"ta":49236,"su":14097,"sv":101,"ss":1947,"st":36036,"sy":54,"sz":61,"sw":69,"sl":1702,"sk":471,"sn":175,"sm":1643,"sp":5348,"so":6479,"sr":79,"oí":58,"sq":228,"sd":259,"sc":6321,"sf":297,"se":12003,"sh":816,"sg":92,"si":17580,"rz":1385,"u ":7771,"sa":8339,"sb":292,"rr":10518,"rs":4137,"rt":18219,"ru":3976,"rv":867,"nò":54,"rw":51,"nó":211,"rx":65,"ry":499,"rq":661,"ní":360,"rp":829,"ro":26141,"rn":3248,"né":3667,"rm":3699,"nè":64,"rl":1733,"rk":850,"nç":561,"ri":26132,"rh":94,"rg":3076,"rf":2952,"re":36472,"ná":154,"rd":4758,"rc":8506,"rb":2086,"ra":47500,"t ":23983,"mú":101,"mó":149,"qu":14376,"mí":198,"mé":389,"má":2404,"s ":90855,"px":43,"pt":487,"pu":3631,"lò":43,"ló":828,"pp":201,"lí":371,"pr":15141,"ps":257,"Hum":43,"IX ":132,"Hue":159,"zá":80,"yú":116,"zí":110,"zó":127,"yá":156,"IV ":156,"yó":1722,"xó":86,"zz":103,"Hor":78,"zh":42,"zi":554,"zb":53,"zc":370,"Hos":42,"ze":662,"vá":44,"za":5335,"Hon":180,"Hol":97,"zu":406,"zt":78,"zo":1662,"zq":48,"ví":116,"zk":61,"vé":50,"zm":43,"vè":48,"ye":26389,"uá":93,"yd":68,"ya":13178,"tú":55,"uñ":102,"yu":88,"yt":84,"ys":168,"yr":129,"uí":262,"yo":9039,"yn":174,"ué":1015,"ym":83,"uè":92,"yl":123,"yi":439,"Arg":208,"Are":82,"Arc":243,"Ard":75,"Ara":1956,"Arb":109,"Arm":172,"Arn":49,"Arl":72,"Ari":148,"Aqu":797,"As ":436,"Ate":44,"Ath":42,"Atl":768,"Ast":163,"Asi":176,"Aso":50,"Asp":82,"Arr":261,"Arq":53,"Art":241,"Ars":45,"Avi":56,"Ave":58,"Aut":173,"Aus":575,"Aur":142,"Aul":197,"Aug":97,"Azu":58,"Bai":742,"Bal":491,"Ban":328,"Bac":68,"Bad":97,"Bay":89,"Bar":1038,"Bat":90,"Bas":270,"Bav":98,"Bau":42,"Abr":40,"Aca":110,"Abe":43,"Aba":96,"Act":184,"Ado":91,"Adr":43,"Afr":187,"Agu":108,"Agr":41,"Ain":41,"Air":59,"Al ":59,"Ala":196,"Alb":564,"Alg":48,"Ali":368,"Alc":398,"Ald":69,"Ale":590,"Alf":219,"Alt":2147,"Alm":205,"All":75,"Alp":176,"Ame":169,"Ama":66,"Amo":43,"Anh":44,"Ang":673,"Ani":69,"Ana":102,"And":336,"Anc":46,"Ans":48,"Ant":494,"Ano":61,"Ann":99,"Bus":74,"Bul":86,"Bun":61,"Bur":806,"Bud":62,"Bue":80,"Bru":135,"² ":4882,"Cab":319,"Cad":58,"Cal":895,"Cam":1021,"Cai":58,"Cas":3464,"Car":802,"Cau":98,"Cat":426,"ª ":69,"Can":590,"Cap":118,"Caz":55,"Bea":178,"Bes":62,"Baï":53,"Bet":64,"Ber":553,"Ben":365,"Bel":385,"Bid":83,"Bie":159,"Bia":400,"Án":71,"Biz":81,"Bil":130,"Big":731,"Bis":62,"Bla":163,"Bre":219,"Bra":269,"Bro":165,"Bri":276,"Boc":41,"Boh":42,"Bol":188,"º ":75,"Bon":127,"Bor":282,"Bos":103,"Bou":109,"Cur":50,"Cub":64,"Cue":508,"Cul":58,"Íx":93,"De ":52,"Des":142,"Del":71,"Dem":144,"Den":147,"Dep":124,"Deb":61,"Év":65,"Ét":123,"Dan":127,"Dar":133,"Dav":113,"Dac":115,"Dal":45,"Cho":110,"Chr":98,"Che":497,"Chi":767,"Chu":642,"às":90,"án":10781,"Cin":435,"áp":259,"Cit":50,"Ciu":227,"Cir":43,"ál":141,"Cis":56,"áb":47,"ác":66,"Civ":71,"ár":205,"át":58,"ás":1227,"ão":95,"Cle":56,"Cla":205,"Cel":82,"Cen":268,"Cer":1047,"à ":342,"á ":406,"Cha":975,"Cri":147,"Cra":64,"Cre":79,"Cru":110,"Cro":121,"Clu":178,"Cob":41,"Cof":46,"Cop":81,"Cos":287,"Cor":738,"Com":767,"Col":268,"Con":936,"Cou":52,"ós":186,"òr":179,"òs":256,"ór":60,"óp":44,"ón":23204,"óm":55,"òc":49,"òm":188,"òn":43,"ó ":6509,"ña":383,"ñe":71,"ño":229,"ò ":54,"ïs":55,"ín":1828,"ío":885,"ít":375,"ís":1863,"íc":52,"íd":62,"ía":5276,"í ":325,"él":127,"ém":47,"én":2272,"és":12234,"ét":78,"ér":268,"éu":47,"év":65,"Edu":48,"éa":65,"éc":86,"ée":59,"Edw":43,"ég":58,"èi":57,"Eda":108,"èl":161,"èm":44,"èr":649,"ès":299,"Edi":90,"èt":179,"èd":50,"Ech":107,"ço":75,"ça":209,"é ":635,"Ebr":275,"Ed ":70,"ç ":366,"Dia":43,"Dic":108,"Dio":78,"Din":113,"Die":65,"Diz":56,"Div":60,"Due":176,"Duc":86,"ür":63,"Dur":53,"üe":263,"üi":139,"ùg":59,"ún":299,"úl":52,"út":328,"ús":134,"úd":56,"Dre":57,"ú ":245,"ER ":56,"Dos":138,"Dou":55,"Don":139,"Dom":136,"Dor":53,"Neg":61,"Neu":41,"Ner":42,"Nat":109,"Nav":689,"Nic":135,"New":87,"Nap":52,"Nar":43,"Nai":88,"Nac":229,"Nue":683,"No ":53,"Nov":50,"Nor":638,"Nom":360,"Nog":123,"Nob":162,"Odr":44,"Ohi":45,"Oci":122,"Oca":40,"Occ":315,"Ovi":40,"Oli":252,"Ons":56,"Íxa":93,"Os ":984,"Ope":55,"Ort":62,"Ord":119,"Ori":680,"Org":56,"Oro":63,"Oso":45,"Ple":45,"Pla":459,"Pin":194,"Pil":51,"Peñ":79,"Pit":44,"Pis":83,"Pir":3266,"Pie":204,"Pic":78,"Phi":78,"Ped":168,"Paí":579,"Per":591,"Pet":150,"Pen":382,"Pel":40,"Pat":94,"Pas":141,"Par":783,"Pau":504,"Pab":49,"Pac":95,"Pan":179,"Pam":292,"Pap":92,"Pal":487,"Pue":323,"Pui":43,"Pro":378,"Pri":295,"Pre":652,"Pru":131,"Pra":244,"Pob":65,"Pol":283,"Pom":61,"Pon":189,"Poi":57,"Pot":41,"Pop":70,"Por":454,"Poz":58," ال":82,"Mún":76,"Rab":66,"Raf":117,"Rad":50,"Ram":97,"Ran":121,"Qua":92,"Qui":124,"Que":100,"Isa":135,"Irl":118,"Ita":545,"Isl":555,"Isr":43,"Ist":200,"Ira":43,"Irá":41,"Jac":136,"Jar":51,"Jan":70,"Jam":131,"Jer":62,"Jes":44,"Jea":137,"Jos":335,"Jor":54,"Jon":53,"Joh":274,"Jr ":59,"Joa":95,"Jud":52,"Jua":119,"Jus":45,"Jul":90,"LA ":58,"Fút":72,"Kan":52,"Kat":41,"Kar":100,"Ken":92,"Kin":58,"Kha":40,"Éta":96,"Kon":42,"Évr":60,"Let":41,"Les":137,"Ler":40,"Leo":104,"Len":373,"Lem":47,"Lei":300,"Leg":47,"Lec":40,"Lea":60,"Lau":180,"Laz":44,"Le ":98,"Lag":94,"Las":229,"Lat":44,"Lar":146,"Lam":54,"Lan":389,"Lac":69,"Lab":132,"Gúd":51,"La ":2467,"Llo":122,"Lla":76,"Lo ":323,"Lib":230,"Lig":108,"Lim":47,"Lin":136,"Lis":91,"Lit":320,"Ley":1614,"Lus":44,"Lui":166,"Lun":61,"Luc":146,"Lue":59,"Lug":56,"Loí":45,"Lou":130,"Los":419,"Loi":111,"Log":143,"Lor":170,"Lon":283,"Lom":95,"Lob":53,"Men":238,"Mel":100,"Mes":113,"Mer":200,"Met":54,"Med":234,"Mex":142,"Mey":1833,"Man":1211,"Mal":289,"Mam":40,"Mar":1797,"Mas":242,"Mag":110,"Mad":503,"Mai":84,"Mac":135,"May":164,"Max":51,"Mau":231,"Mat":250,"Mol":213,"Mon":1596,"Mos":99,"Mor":353,"Mot":51,"Mie":115,"Mig":96,"Mic":206,"Mir":340,"Mis":69,"Mil":244,"Min":117,"Mun":416,"Mur":389,"Mus":93,"Mue":57,"Mug":163,"Xil":74,"çoi":41,"Xal":52,"XX ":114,"Wur":43,"Wil":218,"Win":54,"ère":155,"èra":204,"èrr":55,"èrs":123,"Wei":53,"Wes":94,"XI ":57,"èth":158,"São":53,"èr ":80,"War":42,"Wal":86,"ès ":220,"èla":91,"éch":44,"Vol":63,"Río":103,"Viz":145,"Vis":59,"Vit":75,"Viv":66,"Viè":41,"évi":40,"éti":41,"Zar":1183,"Zam":440,"én ":2163,"éli":52,"ére":56,"éri":41,"éu ":44,"és ":12144,"之":56,"三":85,"Yor":546,"Ye ":991,"Yer":168,"Sur":84,"Sus":52,"Sup":40,"Sue":132,"Sud":292,"Sui":113,"Str":75,"Sto":48,"Sta":135,"Ste":129,"Ten":108,"Teo":41,"Tel":72,"ão ":95,"Tam":196,"Tan":48,"Tar":597,"Tai":54,"Taj":45,"Tal":63,"She":58,"Sha":58,"Sim":93,"Sil":128,"Sig":48,"Sis":99,"Sir":77,"Sin":151,"Sie":418,"Ser":298,"Seu":68,"Set":46,"Sev":235,"Sep":51,"Sen":840,"Sel":119,"Sem":182,"Sei":53,"Seg":496,"Sec":243,"Soc":173,"Sob":421,"Sot":54,"Sou":61,"Sov":80,"Sol":223,"Som":57,"Son":105,"Sor":351,"Rus":221,"Rum":113,"Rub":41,"Sag":50,"Saf":57,"Sai":418,"Sam":249,"Sal":930,"Sac":112,"Sab":95,"Se ":453,"Sco":41,"Sci":60,"Sch":126,"Sax":87,"Sav":109,"Sau":148,"Sar":312,"Sas":117,"San":2098,"Nùg":59,"Rey":105,"Res":57,"Rev":100,"Rio":251,"Riv":46,"Riu":60,"Ris":43,"Rin":49,"Rib":614,"Ric":155,"Rec":256,"Red":59,"Rei":1060,"Reg":57,"Rem":46,"Ren":132,"Rel":50,"Náp":249,"Rep":613,"Rea":232,"Roi":95,"Rob":204,"Roc":80,"Rod":121,"Roy":73,"Rou":41,"Ros":411,"Ron":93,"Rom":609,"Vel":47,"Ven":264,"Veg":102,"Vas":237,"Van":48,"Val":1946,"Var":71,"Vig":45,"Vid":63,"Vic":256,"Vie":268,"Vir":86,"Vil":1006,"Vin":160,"Ver":197,"Una":265,"Uni":1303,"Un ":378,"Ult":91,"Uru":44,"Urc":58,"Ucr":41,"VI ":101,"Ues":1284,"Tex":48,"Ter":931,"Tes":47,"The":314,"Tho":94,"Tie":516,"Tim":56,"Tir":42,"ça ":138,"Tor":554,"Tol":692,"Tom":76,"Tos":86,"Tou":103,"US ":58,"Tro":100,"Tri":127,"Tre":169,"Tra":328,"Tur":210,"Tud":41,"aC ":158,"вич":54,"às ":80,"biz":69,"bis":197,"bit":3960,"bio":344,"bir":182,"bil":199,"bin":181,"bo ":313,"blu":41,"blo":388,"ble":668,"bli":1408,"bn ":44,"bla":6430,"boc":153,"bol":805,"bió":291,"bpa":44,"bon":258,"bor":360,"bot":129,"bos":126,"bou":51,"áll":83,"be ":906,"ban":1137,"bal":979,"bai":592,"baj":40,"bag":336,"bac":233,"bad":107,"bab":70,"án ":10527,"bau":76,"bat":496,"bas":778,"bar":826,"bch":196,"ánd":139,"ánc":42,"ás ":1198,"ápo":250,"bi ":255,"bei":51,"bec":106,"ber":1735,"ben":218,"bel":1025,"bey":41,"bez":71,"bes":144,"bet":294,"bfa":45,"bia":312,"bib":60,"bic":70,"bid":52,"bie":575,"árb":69,"ca ":14794,"car":2739,"cas":4851,"cat":3216,"cau":248,"can":4948,"cap":1502,"caz":64,"cay":268,"cac":340,"cab":314,"cad":562,"cam":688,"cal":1804,"cag":56,"cai":191,"ce ":491,"bri":1569,"bro":804,"bra":1561,"bre":5163,"bru":86,"bso":41,"bse":48,"bst":107,"bte":55,"bur":411,"bul":93,"bun":55,"bui":89,"buc":81,"bue":92,"bus":122,"buy":48,"aka":66,"am ":354,"ake":58,"aki":53,"akh":42,"ajo":179,"afí":261,"aix":4540,"al ":11240,"aja":109,"aje":50,"ail":226,"aim":120,"ain":870,"air":581,"ais":411,"ait":77,"ak ":65,"aig":94,"aif":44,"aid":83,"aic":166,"aia":53,"adí":134,"aho":59,"acó":65,"adá":67,"aha":105,"agr":176,"agu":831,"agn":198,"ago":5671,"anu":472,"anz":412,"any":6533,"ano":1852,"ann":351,"ant":15027,"ans":1324,"anq":163,"ane":625,"ang":1039,"anh":151,"ani":3885,"anj":43,"ank":148,"anl":160,"ana":6098,"anc":10977,"and":4107,"amu":134,"amm":73,"amo":865,"amp":1732,"ami":2709,"amf":47,"ame":4595,"amb":576,"ama":1728,"ao ":68,"alz":65,"alv":228,"alu":654,"alt":1006,"als":2354,"alq":109,"alp":109,"alo":709,"alm":4203,"all":2888,"alg":57,"alh":70,"ali":4784,"alc":339,"ald":658,"ale":2737,"alf":222,"ala":3055,"alb":415,"an ":6113,"agó":1389,"ako":58,"aba":1272,"abe":873,"abi":3680,"abl":933,"abo":367,"abr":1214,"abs":89,"abu":101,"ae ":319,"aca":1187,"ad ":418,"ac ":612,"ab ":2350,"afo":146,"afr":204,"aff":59,"afe":128,"afi":1228,"afl":109,"ai ":461,"aga":473,"age":155,"ael":246,"aes":53,"aer":81,"ah ":76,"afa":137,"ado":2009,"adr":581,"adm":420,"adh":47,"adi":1004,"adc":41,"ade":908,"adu":331,"adv":85,"acq":78,"aco":595,"acl":40,"ack":101,"aci":10341,"ach":1441,"ace":581,"acc":247,"ada":2435,"act":3007,"acu":155,"acr":135,"acs":115,"azo":174,"azi":123,"azu":65,"azt":42,"aze":141,"aza":531,"azz":58,"axi":147,"axo":137,"atí":551,"asò":111,"az ":147,"ayo":1230,"ató":98,"aya":299,"aye":245,"azí":106,"ba ":2806,"ayú":116,"amó":65,"aqu":292,"amí":58,"at ":8853,"arg":457,"are":871,"ard":1884,"arc":6367,"arb":1022,"ara":7157,"arp":123,"aro":1482,"arn":763,"ané":98,"arm":309,"arl":816,"anç":88,"ark":146,"ari":3225,"aru":81,"arx":55,"aní":220,"arq":324,"arr":4340,"ars":729,"art":10745,"au ":1514,"asa":1087,"ary":98,"arz":796,"asi":1104,"ash":77,"asc":1409,"ase":427,"aso":509,"asq":49,"asp":165,"ask":71,"asm":49,"ar ":6113,"apa":550,"alá":1638,"ape":296,"api":1469,"alé":48,"apl":89,"apo":393,"alí":68,"apr":117,"apt":99,"apu":137,"aló":105,"as ":19306,"amá":261,"ava":1125,"ax ":55,"aux":83,"auv":126,"aut":2694,"arí":594,"avo":154,"avi":1192,"ave":551,"ará":49,"ay ":211,"awa":67,"aró":66,"ata":7591,"asu":67,"ast":7538,"ass":494,"atl":127,"atr":1247,"ato":13605,"ate":1325,"ati":1701,"ath":153,"auc":108,"aub":88,"att":124,"ats":80,"atu":842,"apó":59,"atz":500,"aul":310,"aum":84,"aun":85,"aur":378,"aus":598,"aud":445,"aug":298,"ος":66,"ος ":66,"ς ":149,"α ":52,"ич ":54,"jas":71,"jau":48,"jar":113,"jan":59,"je ":44,"jos":60,"jor":51,"fía":291,"jo ":179,"itr":81,"ües":75,"ito":14065,"itu":6260,"üen":106,"itt":116,"itz":160,"ity":56,"ipó":66,"eña":141,"iud":1817,"ism":873,"isl":671,"iso":251,"isp":308,"iss":346,"isu":85,"ist":6571,"ita":13238,"itc":43,"ite":2513,"ith":88,"iti":1102,"ivo":1136,"iró":43,"ius":271,"iur":45,"ium":78,"eño":50,"iva":1324,"ix ":335,"ivi":1220,"irá":76,"ive":525,"ipr":52,"ipo":232,"ipp":56,"ipu":138,"ipt":82,"ipi":7426,"ipl":106,"is ":1549,"ion":3766,"iop":54,"ior":777,"ios":2033,"iot":81,"iog":52,"iol":276,"iom":261,"ipa":1088,"ipc":44,"ilá":43,"ipe":312,"iov":42,"ir ":1069,"inó":91,"iru":67,"irv":40,"irr":87,"iro":527,"irm":204,"iné":146,"irl":117,"iri":575,"isi":1148,"ish":214,"ise":307,"isc":524,"isb":61,"isa":284,"iu ":332,"iqu":443,"ire":4699,"irg":44,"ira":1068,"irc":161,"it ":973,"ixó":72,"ja ":289,"iya":43,"üis":117,"ixo":921,"ixi":4792,"ixa":896,"ixe":850,"itá":2129,"iz ":822,"eón":60,"izo":170,"izi":124,"ize":43,"izc":159,"iza":1143,"kil":72,"kin":118,"kis":56,"útb":317,"km ":480,"ús ":123,"ki ":146,"kha":50,"kel":50,"ken":42,"ker":97,"ket":62,"ke ":87,"úne":45,"úni":50," ª ":48,"km²":4894,"ks ":44,"ko ":90," Án":69,"kar":110,"kan":40,"kal":59," º ":69,"fút":245,"ka ":210," Ga":2446," Ge":1114," Cá":129," I ":417," Fo":711," Fu":665,"cú ":41," Fr":2182," Fi":496," Fl":289," Bé":72," Ha":781," He":719," J ":52," Go":465," Gr":1065," Gu":1594," Gi":352," Gl":100," Cé":64," Ig":114," Id":48," Ib":231," Hu":365," Ho":591," Hi":444,"ha ":1929," Ji":54," Je":310," L ":757," Ja":568," Ir":296," Is":1032," It":588," Im":754," In":704," Ip":108," Il":295,"ham":219,"han":579,"hap":158," Ka":309,"hai":129," Fú":72,"hal":167,"hau":82," Ke":199," Ki":173,"har":1290,"has":147," Kh":65,"hat":40," Jo":916," Ju":421,"hae":98," Jr":59,"hab":5867,"had":57,"hac":508," Gú":51," La":4106," Le":3156," Li":1187," Kl":47," Ko":156," Kr":75," Ku":88," Mc":66," Ma":5257," O ":3000," Mi":1402," Me":3058,"he ":928,"dá ":99," Lo":1959," Ll":306," Ly":43," Lu":751," Ne":437,"а ":67," Na":1457," Ni":336," Mo":2562," Mu":1272,"hel":358,"hei":72,"heg":254,"hef":53,"hec":353,"hea":69,"heb":242,"hez":41," A ":6341,"het":220,"hes":362,"her":745,"heo":251,"hen":1459,"dán":167,"hem":117,"hi ":92," C ":63," Ap":96," Am":424," An":2142," Al":5280," Ai":211," Ag":239," Af":231," Ac":390," Ad":284," Ab":290," Ba":3640," D ":255," Az":169," Ay":81," Av":152," Au":1361," At":949," As":1091," Ar":3860," Aq":804,"hig":61," Be":1778,"hie":94,"hid":163,"hic":622," Bi":1877,"hib":145,"hia":201,"hip":217,"hio":409,"hin":1554,"him":149," Bl":233," Bo":1120,"hil":241," Br":1100," Bu":1316,"his":710,"hit":396,"hir":613," Ca":8053," Ce":1515," Ci":1040," Ch":3168," Cl":508," Cr":536," Co":3415," Cu":873," Cy":45," Da":678," Di":655," De":984," Dr":118," Do":649," Du":469,"hn ":206," Ea":51," Eb":283," Ec":152," Ed":408," El":689,"hle":43," Ei":68," Et":87," Es":4884," Er":277," Aí":54," En":982," Em":289,"dèr":52," Ez":47," Ex":340," Eu":620," Ev":67,"ho ":186," Fe":743," Fa":900,"go ":3253," Tè":61," Xi":132,"glo":720," Xa":108,"gle":161,"gli":83," Wu":48,"gla":523," Wo":125," Sé":45," Wi":413," Wh":40," Sã":53," We":250," Wa":265,"й ":42," Zu":77,"god":95,"gob":53," Zo":51," Ze":94,"gno":114," Zi":59,"gni":197," Za":1740,"gne":136,"gna":214," Yu":72," Yo":603,"cés":6099," Ya":59," Ye":1232,"cía":153,"gs ":50,"glé":443,"goz":1189,"gol":195,"gon":3334,"gos":1875,"gor":1373,"got":66,"gov":132,"gu ":63,"úda":51," a ":33903,"gro":300,"gru":390,"gra":2556,"gri":582,"gre":209," Oz":48," Ou":62," Ov":54," Os":1274,"gto":64," Ot":107," Or":1187," Op":71," Po":1463," Lé":44,"gui":929," Pl":536," Pi":4008," Ph":119,"gul":273,"có ":168,"gua":1007," Pe":1600,"gub":134," Pa":3520,"gue":1452," Nu":796," No":1504," Ol":435," On":167," Om":55," Oh":48," Od":84," Oc":530," Ob":95," Ra":686," Mú":79,"grí":111," Qu":327,"cón":94," Ro":1930," Ná":254," Re":2927," Ri":1399," S ":115,"gut":43," Pr":1734,"gur":246,"gus":162," Pu":601,"gun":881," Mé":58," Sy":56," Su":912," St":445," Ta":1241," V ":80," Th":489," Ti":744," Te":1398," Tr":784,"gué":212," To":1738," Pé":62," Ru":549," Sa":4961," Nù":59," Sh":196," Si":1206," Sc":255," Se":3315," So":1654," Sp":125,"ún ":198," Uz":41," Va":2415," X ":52," Ve":691," Vi":2397," Rí":105," Vo":162," Tu":387," Uc":50," Ue":1325," Ul":142," Un":1997," Ur":312," Us":67," Ut":63," ja":45," l ":7346,"iam":215,"ial":6627,"ian":2111,"ias":1288,"iar":803,"iau":82,"iat":539," im":548," in":2520," il":210,"ic ":192," ix":548,"iac":505," is":1346,"iad":189," it":1473," ir":178,"iag":113,"ibl":190," fú":245,"ibi":1047," ka":43,"ibo":144," m ":86,"ibn":40,"ibr":375," ki":82,"ibu":235,"id ":208,"iba":752,"ibe":709," ju":44," ha":6493," he":482," gl":81," gr":1637," go":384,"ia ":20555," gu":873," ib":130," id":377," ie":162," ig":40," hi":763," ho":443," dí":123," hu":1100,"iet":455,"ieu":89,"iev":42," ni":192,"iez":72,"iel":3483," ne":482,"iem":1602,"ien":5235," na":4148,"ier":2261,"ies":690,"ied":470,"ieg":1134,"iej":52," mu":11537,"ig ":130," mo":1373,"ibó":41," ol":51,"ifu":46," on":583," oc":5124,"ifo":507," of":3427," ob":670,"ife":480,"ifi":975," nu":622," no":4290,"ifa":124," le":1142,"icr":45,"ict":263," li":1110,"icu":187," n ":127,"ico":6536,"ick":140,"icl":297," la":7106,"ici":16857,"ich":1956,"icc":197,"ice":399,"ie ":3544," km":5375,"ica":8644," me":3304,"idu":108," mi":1891,"idr":120," o ":28468,"ido":565," ma":4705," lu":2504,"idi":602,"idg":67,"ide":1774,"idd":66,"ida":7766," lo":4749," ae":70," af":235," ag":650," ab":1016," ac":2837," ad":973," am":802," an":6430," ap":406," ai":177," al":2449," av":614," au":2876," ar":5285," aq":61," at":946," as":4436," d ":41259," ba":1763," az":45,"il ":1291,"ija":46," bi":684," be":950," bo":292," bl":117," bu":158," br":776," ca":13929," e ":354,"im ":145,"ika":72,"ige":58,"icá":71,"iga":1659,"igl":91,"igh":138,"igi":46,"igu":556,"igr":84,"igo":2086,"ign":388,"icó":83," aC":158,"ik ":59," er":161,"imo":645,"imn":76," eq":225," et":307," es":12368," en":45737," em":791," ep":103,"imp":1430," ei":79,"ime":1567," el":836," ef":58,"imi":832," fe":1998,"inc":9264,"ind":1169,"ina":3383," fa":1854,"imu":52," eu":203," ev":123," ex":831," fu":1130,"inn":131," fr":6946,"ino":2528,"inq":49," fo":1674,"int":2294,"ins":841,"inf":351,"ine":3588," fl":166,"inh":136,"ing":916," fi":1256,"ini":1221,"inl":57," ge":55," ga":614,"ioc":379,"iod":531,"inu":133," i ":370,"inv":192,"iny":434," cl":828,"iko":59," co":25285," cr":940,"iki":75," ce":943," ch":6428,"ike":47," ci":4380,"igü":76,"ila":709,"ilb":85," da":436,"in ":1797," cu":1036,"igò":47," do":1088,"ilo":709,"ill":3569," dr":358,"ilm":50,"ilh":186," de":93040,"ili":1413,"ild":72," di":6008,"ile":486,"ima":777,"imb":156,"ч ":61," ec":252," ed":270,"io ":12068," du":304,"ils":93,"ilv":161,"hs ":41,"día":1990," ví":46," zo":210,"hol":113,"hom":232,"hon":397," za":502,"hos":55," ze":43,"hov":67,"hor":197,"ка":41,"hod":105," ye":24958," ya":136,"hió":4220,"hna":61," tí":352,"ич":60,"dés":276,"hul":634,"hug":321,"hue":1138,"hud":1823,"hua":226,"dó ":56,"ов":67,"hre":40,"hri":107,"ht ":118,"dís":58,"dín":52," ru":519," u ":3972," sa":1314," se":5716," sc":327," si":7787," so":2523," qu":10550,"ви":64," t ":73," ra":569," re":9633," ri":229," ro":846," pu":1468," pr":13974," ps":89," lí":44," s ":862," px":42,"hy ":54," má":765," mé":265,"dón":67," os":5952,"hum":282,"hun":1148,"hus":252," op":121,"hur":185," or":1663," pe":2382," pa":7968," pl":740," po":12254," pi":915,"dòm":187," we":61," y ":28425," sí":53," va":1856," ve":970," vo":365," rí":719," vu":86," vi":1199," ub":66,"ùgo":59," tu":267," us":125," ur":107," un":30714," ue":407," ta":2947," su":13017," tr":2669," to":940," th":178," ti":1270," te":2399,"ffe":52,"fes":224,"fer":1148,"fec":203,"fed":134,"feb":1076,"fem":90,"bán":64,"fen":310,"fel":119,"fei":204,"fib":68,"fga":41,"fas":57,"fat":64,"far":136,"fam":559,"fan":320,"fal":168,"fai":59,"fae":49,"fad":52,"fac":309,"fab":307,"ff ":44,"fe ":106,"eyó":1631,"fa ":651,"eya":1305,"etó":68,"ext":325,"eyo":2607,"eye":329,"esú":73,"exa":137,"ez ":631,"exo":63,"exp":204,"exm":46,"exi":568,"exc":56,"exe":134,"ezu":123,"eza":368,"ezo":65,"ezi":98,"eta":1993,"ete":572,"etc":51,"eti":1075,"eth":102,"etn":131,"esp":2252,"esq":111,"esn":47,"eso":805,"est":15719,"esu":238,"ess":304,"aïs":55,"esv":63,"aña":108,"eud":108,"eui":40,"año":118,"eto":512,"etr":713,"ets":74,"ett":317,"etu":65,"etz":165,"ew ":79,"erá":64,"eve":388,"eva":1219,"eqü":84,"evo":337,"evi":643,"euv":62,"eut":86,"eur":262,"eus":3359,"ex ":75,"ewi":52,"esí":70,"eró":42,"erí":642,"ey ":440,"erú":45,"epe":349,"epc":40,"epi":88,"eph":122,"er ":4019,"epa":3520,"eor":324,"eol":192,"eop":46,"eon":126,"emá":600,"es ":7690,"ept":204,"epu":741,"elè":120,"epp":42,"epo":219,"epr":370,"erk":43,"enç":430,"erl":400,"eri":3958,"erg":428,"ere":1282,"erf":2781,"erc":992,"erd":1295,"era":7654,"erb":545,"et ":1474,"equ":358,"aís":1262,"aín":66,"esl":255,"esm":548,"esf":145,"esh":54,"esi":1456,"esb":43,"esc":3180,"esd":183,"ese":1276,"eu ":796,"esa":2233,"erz":160,"erv":658,"eru":753,"err":3654,"ení":57,"ert":3943,"ers":2076,"ern":1404,"erm":1179,"ené":47,"erp":493,"ero":3393,"egó":100,"en ":45647,"elb":64,"ela":1464,"eld":142,"elc":143,"ele":1251,"eli":1398,"elg":80,"elh":102,"elm":81,"ell":8850,"elo":1006,"elu":165,"elv":192,"els":263,"elt":264,"eo ":161,"emb":1646,"ema":2624,"eme":667,"emo":579,"emi":1471,"emp":1335,"enf":65,"ene":5939,"enh":83,"eng":1616,"enb":80,"ena":1793,"end":3462,"enc":4753,"eno":895,"enn":286,"eni":1492,"enu":197,"env":137,"ens":4894,"ent":19699,"enr":137,"enz":322,"eny":1221,"eog":94,"eod":44,"egl":633,"ego":1470,"ege":41,"egi":69,"egr":354,"egu":1397,"edé":50,"ek ":44,"eic":131,"eis":231,"eir":354,"eim":83,"eil":138,"ein":1974,"eid":281,"eig":91,"eja":198,"el ":3419,"eix":920,"eit":740,"ejo":101,"em ":132,"gin":62,"gio":83,"gia":75,"ght":102,"ghe":49,"gha":41,"ggi":46,"gi ":48,"gen":256,"cán":112,"ger":176,"ges":165,"gh ":89,"gel":473,"ge ":357,"gab":64,"gac":180,"gad":214,"gai":80,"gas":720,"gar":2153,"gau":48,"gat":585,"gam":80,"gal":634,"gan":993,"ga ":3535,"fur":43,"fuo":41,"fus":124,"fut":63,"bón":87,"fue":546,"fun":451,"fra":6871,"fre":244,"fri":349,"fro":68,"fru":71,"for":2092,"fos":84,"fot":124,"fon":296,"fol":141,"foz":70,"bén":225,"fle":62,"fla":55,"flu":281,"bèr":62,"flo":94,"fo ":248,"fic":8082,"fie":199,"fig":50,"fil":651,"fin":464,"fir":81,"fis":211,"fit":61,"fix":143,"da ":4544,"de ":80790,"dac":110,"dad":509,"dal":1070,"dai":51,"dag":63,"dae":153,"daf":41,"dat":9256,"das":725,"dar":409,"dap":72,"dan":479,"dam":162,"dau":66,"dda":65," Év":65," Ét":123,"dch":41,"cup":186,"cun":111,"cul":744,"cum":166,"cui":42,"cue":498,"cub":164,"cuc":89,"cua":224,"ctu":1573,"ctr":587,"cto":2549,"cti":1087,"cte":242,"cta":350,"cy ":62," Íx":93,"cus":151,"cur":292,"cut":113,"cla":931,"chá":42,"cle":173,"chí":487,"clu":743,"chó":41,"cli":257,"clo":250,"ciá":588,"co ":7064,"ció":9852,"cni":55,"coi":64,"cod":67,"cof":52,"coa":55,"cob":54,"coc":226,"cq ":43,"con":9719,"col":848,"com":14762,"cor":808,"cos":2167,"cop":107,"cov":40,"cot":281,"cou":62,"cs ":170,"cqu":62,"cre":669,"cra":303,"cri":1318,"cru":146,"cro":284,"cci":5499,"cca":49,"cce":203,"cea":126,"ch ":604,"cer":583,"ces":1530,"cet":349,"cen":946,"cep":171,"cel":1113,"ceg":45,"ceb":79,"ced":326,"ci ":68," á ":52,"cha":2607,"chu":4144,"cia":18130,"ck ":381,"cie":3721,"cid":704,"cic":176,"cib":852,"che":3543,"chl":46,"chi":7691,"cho":415,"cht":85,"ciz":58," ár":62,"civ":67,"cil":415,"cim":107,"cif":135,"cir":339,"cis":357,"cit":4565,"ciu":1650,"cin":2607,"cio":3004,"cip":8439,"cke":58,"ed ":199,"eba":1588,"ebe":146,"ebi":129,"ebl":394,"ebo":83,"ebr":993,"ebu":74,"ec ":41,"eac":66,"eag":52,"eae":70,"ean":276,"eal":403,"ear":257,"eas":101,"eat":219,"eau":208,"eb ":59,"ea ":612,"efi":288,"efo":67,"efa":54,"efe":402,"ei ":675,"ega":1451,"efu":50,"een":85,"eer":148,"edi":1178,"edd":40,"ede":794,"eda":712,"edu":87,"edo":984,"edr":427,"ecl":113,"eck":59,"ech":4825,"eci":2528,"ece":250,"ecc":313,"eca":350,"ee ":110,"ef ":40,"ecu":265,"ect":1719,"ecr":84,"eco":638,"ecn":84,"dwi":49,"dwa":45,"dy ":91,"dve":42,"dvo":55,"drí":82,"dré":42,"dur":262,"duq":92,"dus":68,"duy":49,"dor":1504,"dop":45,"don":410,"dom":223,"dol":760,"dou":58,"dot":86,"dos":1016,"ds ":102,"deţ":45,"dmi":426,"dió":145,"dob":178,"doc":435,"doe":44,"duo":46,"dul":73,"due":204,"dua":107,"duc":808,"dri":604,"dra":531,"dt ":44,"dre":695,"du ":102,"dro":341,"dge":81,"dic":4356,"did":81,"dia":1190,"dib":46,"dhi":40,"der":1900,"des":5613,"det":240,"deu":41,"dev":209,"dey":218,"dez":184,"deb":265,"dea":184,"ded":106,"dec":377,"def":199,"dei":236,"del":1335,"den":7502,"dem":337,"dep":3598,"deo":89,"di ":271,"do ":4312,"div":679,"diz":245,"dim":719,"din":616,"dio":972,"dip":45,"dir":869,"dis":954,"dit":1042,"die":409,"dif":510,"dig":188,"dil":238,"rgu":256,"rdá":123,"rdè":52,"rdí":75,"rga":811,"ri ":510,"rgi":58,"rge":464,"rcí":64,"rgo":1114,"ret":1001,"res":4668,"rev":232,"reu":243,"rez":147,"rey":634,"rfe":48,"rfi":2743,"rfo":56,"rdu":65,"rds":65,"rg ":290,"reb":488,"rea":344,"ree":87,"ref":461,"rec":6366,"red":815,"rei":1976,"rej":70,"reg":565,"rem":1469,"nán":92,"ren":6329,"rel":727,"raí":55,"req":52,"rer":1624,"rep":629,"rda":881,"rcu":194,"rct":43,"rdo":870,"rdi":439,"rde":1540,"ós ":153,"re ":7388,"rbu":85,"rco":266,"rci":629,"rch":786,"rce":963,"rca":5430,"ray":93,"raz":455,"rd ":524,"rap":218,"raq":58,"rar":802,"ras":2221,"rat":2401,"rau":304,"rav":344,"rbi":229,"rbo":341,"rba":625,"rbe":641,"rai":190,"rah":88,"rag":6073,"ran":12090,"ram":788,"ral":2056,"rab":769,"raf":1535,"rae":83,"rad":1513,"rac":1579,"rpr":167,"rlí":92,"rpo":272,"rs ":2827,"rpe":55,"rpa":54,"rpi":201,"roq":99,"ror":130,"ros":1359,"rot":408,"rom":669,"ron":3295,"rop":1326,"roy":164,"roz":64,"rou":179,"rov":8451,"row":58,"rob":921,"roa":141,"rod":803,"roc":838,"roi":133,"rol":525,"rof":212,"roe":57,"rog":138,"rno":450,"rns":42,"rnu":49,"rna":910,"rne":366,"rni":615,"rmo":194,"nés":3547,"rmu":56,"rió":226,"ro ":6031,"rma":2089,"riá":116,"rme":392,"rmi":794,"rly":44,"rlo":224,"rli":134,"rle":220,"rla":839,"rn ":546,"rks":54,"nço":60,"rki":43,"rke":44,"nça":132,"né ":50,"riz":654,"rl ":72,"rip":80,"rio":3419,"rir":118,"riq":78,"rit":5633,"ris":1199,"riv":286,"riu":101,"rig":256,"ril":1019,"rik":79,"rin":1137,"rim":1306,"ria":3749,"rib":574,"ric":2717,"rid":579,"rie":2036,"rif":80,"rk ":612,"nç ":341,"rté":55,"rtè":42,"rtí":184,"rui":184,"rug":86,"rue":1064,"rud":70,"ruc":376,"rur":59,"rup":477,"run":160,"rum":242,"rul":41,"ruy":43,"ruz":231,"rus":648,"rut":44,"rva":257,"rvi":411,"rve":120,"rvo":53,"ry ":412,"rsi":295,"rso":324,"rsa":318,"rse":122,"rta":4462,"rst":64,"rto":2506,"rte":4811,"rth":164,"rti":4275,"rtz":99,"nó ":165,"rub":49,"rts":44,"rtr":69,"rtu":523,"nín":54,"rmá":65,"nía":237,"rt ":839,"rqu":659,"rné":74,"rro":1014,"rri":1235,"rná":86,"rre":2401,"rra":5371,"ru ":71,"rry":81,"rru":212,"sab":254,"sac":303,"sad":68,"sag":89,"sai":44,"sal":425,"sam":123,"óni":49,"sap":102,"san":614,"sau":128,"sat":386,"sas":492,"sar":1166,"sa ":3938,"ón ":23122,"rze":61,"rza":479,"rzo":759,"sha":111,"òrt":45,"sho":45,"òrr":49,"she":41,"shi":211,"si ":476,"siv":107,"sie":897,"sid":3345,"sic":1265,"sib":103,"sia":1021,"sk ":46,"sit":6032,"òst":117,"sir":64,"sis":499,"siq":107,"sin":887,"sio":389,"sil":374,"sim":339,"sif":156,"sig":443,"scr":1111,"scu":413,"sde":183,"sbo":69,"sbu":127,"se ":5048,"sca":2190,"sce":133,"sci":365,"sch":197,"sco":1662,"scl":134,"sex":85,"sey":101,"ser":1152,"ses":640,"set":627,"seu":313,"sh ":334,"òs ":104,"sfe":195,"sfo":54,"sea":92,"sei":105,"seg":655,"sef":47,"sed":71,"sec":228,"seq":43,"sep":285,"sen":1175,"sem":568,"sel":661,"spu":61,"spo":693,"spr":52,"spe":1130,"spl":78,"spi":211,"spa":3018,"sot":81,"sou":70,"sov":57,"sol":429,"son":2287,"sor":319,"sos":346,"sof":205,"soc":286,"sob":802,"su ":157,"sra":56,"st ":507,"oís":51,"squ":227,"ss ":107,"sli":62,"slo":145,"sla":1276,"sle":207,"ski":109,"sko":57,"ska":167,"ske":47,"sno":42,"sni":58,"smo":1140,"sió":969,"so ":1404,"sma":361,"smi":54,"sme":59,"swi":41,"stí":156,"sté":58,"stè":110,"stá":292,"stó":58,"sse":691,"ssa":416,"sso":402,"ssi":161,"ssu":66,"ste":7013,"stf":50,"sta":9411,"sto":2359,"sti":8938,"stu":608,"str":6362,"sua":62,"sud":745,"sue":210,"sub":387,"suc":192,"sui":69,"suf":112,"sul":509,"sum":63,"sup":2871,"sun":122,"sus":166,"sur":324,"suy":8064,"svi":51,"tai":444,"tal":7927,"taf":42,"tag":121,"tab":863,"tac":1213,"tad":424,"tc ":40,"tba":42,"tay":123,"tax":48,"tav":96,"tau":237,"tat":3988,"tas":2612,"tar":2116,"tap":45,"tan":9479,"tam":5022,"tch":72,"òm ":188,"te ":7101,"tbo":377,"가":45,"ta ":14271,"ño ":46,"pa ":846,"pci":102,"pe ":499,"lá ":57,"par":10175,"pat":307,"pas":460,"là ":40,"pac":352,"pad":63,"pag":64,"pal":1063,"ñor":41,"pai":206,"ños":42,"pap":112,"pan":3073,"ñol":77,"phe":58,"phi":59,"ph ":101,"pañ":78,"peu":90,"pez":143,"pea":157,"pec":832,"ped":144,"lán":4159,"pen":661,"per":6227,"paí":658,"pet":128,"pes":448,"pei":72,"pel":266,"pla":716,"pli":171,"ple":898,"lèr":124,"lès":68,"plo":489,"plu":67,"pia":402,"pid":90,"pic":437,"pie":425,"pil":124,"pin":588,"pio":7267,"pir":221,"pis":101,"pit":1534,"por":5544,"pop":238,"pot":83,"pos":1131,"pon":437,"pol":1645,"pob":6027,"poe":320,"poc":176,"pod":154,"ps ":76,"ppe":91,"pió":148,"lés":609,"lén":62,"po ":1377,"lí ":42,"pta":96,"pse":75,"psi":41,"ló ":140,"pub":839,"pte":85,"pti":114,"pto":155,"ñas":48,"pra":148,"ña ":201,"líd":41,"lín":186,"pri":1181,"pre":3562,"pro":10201,"lón":649,"pur":65,"put":63,"pun":204,"pul":337,"pue":1281,"px ":43,"puz":122,"pué":579,"más":1055,"mán":1305,"més":297,"mía":98,"mín":65,"qua":2094,"mó ":46,"que":9584,"qui":2366,"món":89,"qué":50,"quí":216,"ra ":13591,"mún":92,"ncé":6029,"ngo":337,"ngi":53,"ngl":1038,"ngu":596,"ngr":238,"ngt":66,"ngs":81,"ni ":358,"nge":459,"ngh":51,"nga":1589,"ndí":93,"nho":77,"ndé":233,"ndó":55,"ndò":191,"nha":307,"neg":266,"nei":429,"nel":171,"nen":562,"nem":1044,"neo":91,"ner":1944,"net":586,"nes":1419,"nev":50,"neu":3333,"ng ":522,"nea":100,"neb":239,"nec":186,"ned":220,"nee":129,"nfi":62,"nfo":139,"nfl":201,"nfr":79,"ney":90,"nez":217,"nex":100,"nh ":93,"nfa":138,"nfe":159,"ncr":95,"ncs":41,"nco":784,"nci":13450,"ncl":579,"nce":1594,"nch":1989,"nca":2343,"ne ":2575,"nbu":46,"ndu":163,"ndr":650,"nds":75,"ndo":2567,"ndi":1523,"ïsa":48,"nde":3342,"nda":2934,"nal":2141,"nam":297,"nan":568,"nap":162,"nar":1025,"nac":947,"nad":422,"nae":55,"nag":81,"nai":3351,"nc ":42,"nab":73,"nbe":48,"nd ":533,"nav":217,"nau":204,"nat":1290,"nas":1528,"naz":58,"na ":29914,"ión":15324,"nya":4504,"nye":371,"nyi":95,"ntí":175,"ntó":2943,"nz ":101,"ny ":185,"nvi":113,"nvu":57,"nve":230,"nva":59,"num":304,"nun":319,"nui":47,"nus":137,"nut":82,"nur":49,"nua":50,"nub":51,"nue":339,"nuc":179,"ntz":90,"nto":7609,"ntu":217,"nts":4132,"ntr":3775,"nti":4508,"nth":93,"ntg":68,"ntl":40,"nta":5325,"ntb":47,"nte":2859,"nsu":361,"nsp":80,"nso":356,"nst":724,"nsf":49,"nse":1949,"nsi":3321,"nsk":43,"nsc":53,"nsa":388,"nu ":68,"nry":46,"nri":113,"nre":41,"nt ":9999,"nqu":365,"ns ":3297,"noc":64,"noa":44,"nob":131,"nol":187,"noi":1471,"nop":65,"nom":4566,"non":142,"not":89,"nos":736,"nor":1456,"nov":997,"nou":46,"noy":53,"nne":666,"nna":170,"nno":92,"nni":155,"nns":57,"nió":453,"nli":101,"nn ":180,"nla":104,"ngü":186,"nle":54,"no ":5180,"nke":42,"nki":41,"nja":54,"nig":161,"nif":282,"nie":336,"nid":2791,"nic":8881,"nia":2959,"nk ":109,"niz":323,"niu":41,"niv":269,"nis":1543,"nit":2772,"nir":59,"nio":1015,"nim":685,"nin":419,"nil":164,"ogr":1569,"ogu":112,"océ":67,"ogo":312,"oga":220,"oge":46,"oi ":141,"odí":1685,"ohn":220,"oha":82,"ohe":67,"oix":1425,"ois":211,"oir":170,"oit":63,"oin":160,"oid":74,"ofí":41,"oja":86,"ol ":1565,"oce":431,"och":1133,"oci":755,"ock":200,"ocl":96,"oco":258,"ocr":193,"obt":71,"obs":51,"oe ":44,"oca":1477,"occ":4327,"ode":444,"odi":479,"odo":671,"odr":74,"ocu":240,"oct":667,"of ":232,"oda":219,"oes":72,"oet":294,"oeu":48,"odu":636,"obé":220,"ofi":3286,"ofr":53,"ofo":167,"ofe":194,"ofa":81,"oa ":142,"ob ":41,"nzó":57,"oc ":329,"oan":85,"oal":56,"oac":115,"oba":1120,"od ":84,"oar":74,"obo":70,"obr":1405,"obl":6299,"obi":203,"obc":126,"obe":460,"nyo":3435,"ة ":63,"nza":531,"nze":53,"nzi":60,"nzo":99,"nyá":130,"oya":232,"oxi":91,"oxe":41,"oz ":271,"osé":216,"own":58,"ozo":77,"oza":1310,"oyo":79,"otz":224,"opó":65,"oue":43,"oub":45,"ow ":78,"oti":169,"oth":60,"ote":332,"ott":123,"ots":45,"otr":79,"oto":449,"ost":1452,"ota":773,"ov ":53,"osi":694,"ose":613,"osf":51,"osp":46,"oss":155,"osl":56,"oso":499,"osn":62,"oró":198,"oy ":78,"owa":53,"owe":59,"ovi":9219,"orí":319,"ovo":42,"ova":354,"ove":772,"oug":65,"oui":114,"oul":124,"oun":1514,"ous":206,"our":395,"out":72,"opo":466,"opi":489,"ope":454,"oph":55,"opa":516,"os ":25441,"opu":320,"oló":41,"opr":46,"olí":46,"opt":64,"ops":40,"ool":53,"ood":102,"or ":9087,"ork":566,"orl":115,"oné":3235,"orm":1749,"orn":778,"oro":1408,"orp":161,"orq":101,"oní":41,"orr":1698,"orc":390,"ord":1194,"ore":783,"orf":120,"org":706,"ori":3242,"ou ":222,"osa":1430,"osc":233,"ort":2810,"ors":909,"orv":50,"oru":324,"orz":356,"ory":64,"omé":50,"omí":105,"omá":496,"ot ":423,"m² ":4882,"orb":171,"ora":2008,"omú":90,"oqu":218,"íto":354,"ola":1622,"old":202,"olc":47,"on ":9278,"oli":2360,"oll":594,"olk":73,"olf":145,"ole":736,"ols":519,"olt":132,"olm":67,"olo":2074,"olp":41,"olu":385,"olv":43,"íst":73,"om ":111,"ona":5291,"ond":1478,"onc":1207,"onf":246,"one":1454,"onh":79,"ong":765,"oni":1854,"onn":399,"onm":40,"ono":4250,"onq":106,"ons":3093,"ont":2981,"onu":172,"onv":115,"ony":278,"onz":129,"oma":8598,"ome":737,"omb":2217,"omi":687,"omm":82,"omp":969,"omo":2541,"omu":6923,"op ":77,"la ":13225,"gún":43,"ína":74,"íns":58,"ín ":1619,"ío ":786,"le ":1482,"lce":50,"lca":313,"lch":255,"lcu":105,"lco":195,"lf ":51,"lde":545,"lda":117,"ldo":91,"íse":239,"ldi":63,"ldr":41,"lab":279,"lac":7068,"lad":759,"laf":116,"güe":154,"lah":42,"lag":299,"güi":134,"lai":286,"lal":104,"lan":4366,"lam":1159,"lap":58,"lar":1828,"íos":92,"lat":2045,"las":2771,"lau":337,"lav":397,"lay":70,"laz":237,"lba":848,"ld ":212,"lbe":146,"ís ":1479,"lbo":81,"ky ":62,"gòr":69,"ksh":40,"gó ":119,"íde":44,"gón":1401,"llé":182,"llè":44,"llá":2506,"lpe":216,"lph":49,"ls ":3268,"hía":487,"lló":632,"lol":93,"lon":1637,"lom":283,"lop":119,"lor":1072,"lod":42,"loc":1507,"log":342,"loi":46,"lpa":53,"los":2434,"lot":169,"lou":71,"lov":143,"lob":150,"lmo":183,"lme":4078,"liá":885,"lma":308,"lmu":74,"lió":99,"lti":157,"lto":1581,"ltr":243,"ltu":284,"luc":305,"lub":390,"lug":1219,"lue":1296,"lsi":170,"lso":93,"ías":665,"lta":1746,"lte":171,"lu ":50,"lse":50,"lsa":112,"ía ":4569,"lqu":129,"lt ":112,"lho":61,"lhe":92,"lha":188,"lgo":41,"li ":326,"lga":211,"lfr":59,"lfo":195,"lh ":50,"lfa":274,"lez":140,"ley":747,"lex":169,"leu":75,"lev":257,"les":1591,"let":670,"ler":899,"leo":90,"lem":1561,"hán":48,"len":1629,"lel":48,"lei":286,"lej":60,"leg":883,"led":346,"lec":686,"leb":181,"lea":191,"lls":428,"llu":110,"lly":119,"lo ":6451,"lla":7059,"lle":1785,"lli":819,"lhé":44,"llo":2287,"lm ":61,"ll ":601,"lit":2372,"lis":1111,"lir":71,"liq":95,"lip":156,"lio":1048,"lin":1102,"lim":460,"liz":384,"lix":43,"liv":118,"liu":92,"lic":2196,"lid":1116,"lia":2670,"lib":283,"lk ":57,"lil":59,"lig":423,"lie":227,"lif":574,"ma ":4262,"mb ":46,"mac":390,"mai":134,"mad":213,"mag":134,"mar":6853,"mas":1463,"mal":552,"mam":100,"man":3868,"maz":73,"may":1010,"max":40,"mau":44,"mat":2795,"mba":215,"ià ":88,"mbl":71,"mbi":310,"mbe":233,"mbr":3381,"mbo":389,"me ":312,"mbu":159,"med":417,"mea":52,"mec":61,"meu":62,"met":732,"mes":1084,"mer":2230,"mel":507,"ián":1702,"men":10149,"mez":72,"mex":49,"mey":594,"mfi":49,"luy":521,"luz":282,"lva":423,"lve":168,"lvi":79,"lul":85,"lui":166,"lun":337,"lum":164,"lut":90,"lus":184,"lur":85,"luv":44,"ly ":179,"lza":60,"mpi":508,"mpe":1346,"mpr":198,"mpo":1935,"mpl":1206,"mpu":673,"mps":47,"ms ":48,"mog":40,"moc":196,"mod":219,"mon":1273,"mol":200,"mov":182,"mor":989,"mos":648,"mot":101,"mou":58,"mpa":409,"mug":267,"mue":2339,"mud":113,"iña":49,"ió ":5096,"mur":109,"mus":767,"mui":583,"mul":270,"mun":14364,"mi ":66,"min":1797,"mio":915,"mil":953,"mir":128,"mis":230,"mit":490,"mic":569,"mia":389,"mig":100,"mif":80,"mie":1796,"mid":134,"mo ":3815,"ièr":107,"ièl":73,"mno":74,"mp ":61,"iés":139,"ién":1647,"mié":1515,"mma":94,"mme":71,"vía":77,"zqu":48,"zu ":46,"zue":177,"zul":76,"zz ":43,"zi ":55,"zeg":44,"zet":109,"zen":76,"zer":95,"ze ":198,"zca":158,"zcu":130,"zab":60,"zad":59,"zac":289,"zag":449,"zam":70,"zan":466,"zal":125,"zar":348,"zau":49,"zas":154,"zat":450,"zos":77,"zor":59,"zon":343,"zol":64,"ال":87,"zo ":978,"zia":62,"zie":76,"zin":46,"zio":145,"Áng":58,"uña":55,"uín":41,"ys ":57,"uía":139,"yol":662,"yos":1713,"yor":588,"yon":591,"yod":1669,"uí ":43,"za ":2695,"ye ":23794,"yes":179,"yer":1805,"uán":43,"yen":347,"yev":162,"ya ":11135,"yad":248,"yag":43,"yac":58,"yat":477,"yar":166,"yas":560,"yan":160,"yal":259,"ué ":86,"yo ":3771,"ués":858,"yin":59,"yit":255,"tín":895,"tít":343,"tía":161,"xto":93,"xte":148,"xió":155,"tés":77,"xma":49,"xo ":835,"tèt":120,"xpr":86,"xpo":40,"xpl":51,"xor":48,"xos":144,"xon":140,"tí ":44,"ن ":61,"tó ":198,"tón":3024,"tán":2438,"xen":186,"xem":129,"xer":290,"xey":127,"xec":40,"xed":66,"xeb":52,"té ":56,"xis":225,"xit":4235,"xiu":87,"xil":66,"xim":208,"xid":80,"xic":268,"xie":229,"sús":77,"xa ":634,"xe ":402,"xas":176,"xat":41,"xar":156,"xan":151,"só ":81,"sòs":122,"són":50,"sé ":258,"wn ":47,"sèr":48,"sía":111,"woo":45,"wer":45,"sán":43,"wel":61,"wis":40,"wig":45,"wic":75,"win":65,"rú ":40,"wa ":59,"war":116,"rís":312,"río":740,"rín":122,"ría":1416,"vry":61,"vri":44,"vre":45,"ró ":128,"vue":107,"vul":60,"rón":372,"via":506,"vio":76,"vir":277,"vil":743,"vim":180,"vin":8416,"vic":421,"vid":633,"vie":1569,"viv":217,"vit":110,"vis":844,"ré ":45,"rès":47,"vo ":1023,"réc":41,"viè":54,"rén":65,"vié":152,"rés":144,"vió":41,"voc":147,"vog":50,"vol":331,"von":103,"vor":132,"vos":269,"voz":67,"vey":52,"vez":50,"ver":1295,"ves":245,"vet":87,"vei":63,"veg":260,"rán":277,"ven":911,"vel":555,"ved":70,"vec":112,"ve ":292,"rá ":52,"val":1231,"van":360,"vam":114,"var":1013,"vat":345,"vas":972,"qüe":96,"vac":122,"vad":166,"vai":52,"vag":42,"va ":2367,"uza":179,"uzc":121,"uyi":284,"uye":389,"uyo":1635,"uya":6536,"uxe":40,"uz ":427,"ux ":137,"uvi":103,"uva":115,"uve":116,"uy ":68,"urí":43,"uró":53,"usk":97,"ush":233,"usi":1046,"use":320,"usc":110,"usa":376,"usu":133,"ust":1133,"uss":242,"uso":426,"uth":76,"uti":206,"ute":121,"uta":240,"utb":55,"utz":40,"uts":45,"uto":2752,"utr":63,"us ":4557,"umá":123,"uqu":145,"ut ":319,"urb":94,"ura":2237,"urd":85,"urc":256,"ure":412,"urg":1130,"uri":877,"url":50,"urn":118,"uro":825,"urq":156,"urr":231,"urs":199,"urt":257,"uru":86,"urv":46,"uny":294,"uor":52,"uos":46,"upa":191,"ur ":505,"upi":40,"upe":2913,"upo":429,"ump":44,"umu":43,"uió":159,"umi":95,"umo":103,"uma":425,"umb":157,"ume":577,"unt":1291,"uns":190,"unk":72,"uni":12278,"uno":513,"unn":65,"unc":549,"und":1442,"una":19774,"ung":103,"une":139,"um ":208,"ulu":41,"ult":655,"uls":64,"ulo":359,"ulm":116,"ull":355,"uli":893,"ulg":112,"ule":147,"ulc":55,"ula":1293,"un ":14687,"uid":133,"uie":269,"uig":53,"uil":266,"uim":168,"uin":360,"uio":269,"uip":268,"uir":116,"uis":370,"uia":191,"uic":106,"uiv":83,"uiu":45,"uit":1942,"uiz":200,"uix":148,"ul ":264,"ugh":63,"uge":55,"ugo":115,"ucí":66,"ui ":377,"uga":2225,"ugu":646,"uco":62,"ucr":81,"uct":498,"ucu":40,"uda":1999,"ude":796,"udi":2062,"ubo":95,"ubp":52,"ubs":95,"ubr":706,"uca":364,"ue ":10127,"uce":207,"ucc":373,"uci":673,"uch":247,"ucl":167,"uck":53,"uet":490,"uev":956,"uer":4945,"ues":1507,"uey":113,"uez":111,"ufi":110,"udu":147,"udo":192,"ueg":394,"ued":332,"uec":249,"ueb":404,"uen":2417,"uem":50,"uel":1803,"uei":224,"ub ":368,"ua ":575,"uay":40,"uau":59,"uat":5774,"uas":205,"uar":331,"ual":1415,"uan":1770,"ubi":467,"ubl":925,"ube":202,"ubf":47,"uba":115,"ubc":72,"ud ":775,"uai":86,"uad":981,"uac":71,"uc ":85,"tzi":52,"tza":114,"tze":111,"ty ":139,"tró":52,"tuy":78,"tur":1993,"tus":72,"tut":233,"tui":52,"tul":64,"tun":41,"tum":52,"tub":619,"tua":6572,"tud":253,"tuc":165,"tug":422,"tz ":970,"pón":114,"ts ":4329,"tre":3631,"tt ":108,"tra":3520,"tri":4717,"tru":524,"tro":3498,"tu ":83,"tta":116,"tte":197,"tti":104,"tto":79,"tts":75,"pó ":91,"tiá":58,"to ":29963,"tió":3631,"tmo":48,"tié":42,"tni":61,"tna":54,"tod":133,"toc":171,"toi":42,"tog":1133,"tou":1509,"tov":69,"tos":6512,"tot":518,"toz":49,"tom":251,"ton":3336,"tol":876,"tor":4909,"top":212,"til":588,"tif":310,"tie":4123,"tig":1952,"tir":249,"teí":49,"tit":2499,"tis":1361,"tin":1092,"tim":251,"tip":273,"tio":464,"thu":114,"tia":471,"tib":196,"tic":4224,"tid":328,"tiz":57,"tiu":93,"tiv":2043,"tla":799,"tle":198,"tem":761,"ten":3384,"teo":124,"tep":75,"tei":151,"tel":3865,"teg":218,"tea":206,"teb":98,"tec":464,"ted":70,"tfa":44,"th ":346,"tex":101,"tev":102,"teu":43,"tet":109,"tes":1306,"ter":4859,"ti ":579,"thn":65,"tho":92,"the":306,"tha":124,"AS ":76,"zán":49,"yú ":114,"yón":1632,"zón":80,"zín":104,"yán":143,"xón":76,"yó ":84},"n_words":[3992558,4845689,3895070],"name":"an","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/ar.json b/contrib/languages-data/ar.json
new file mode 100644
index 0000000..d2cd2b1
--- /dev/null
+++ b/contrib/languages-data/ar.json
@@ -0,0 +1 @@
+{"freq":{"و":674395,"ى":83925,"ي":1050070,"ً":13534,"َ":5694,"ُ":4812,"ـ":6044,"ف":291773,"ق":234289,"ك":238048,"ل":1258387,"م":769173,"ن":600182,"ه":275861,"ّ":6098,"ِ":3286,"ْ":2044,"خ":81779,"د":374335,"ج":159763,"ح":207504,"ت":462068,"ث":62775,"ب":456733,"ة":436596,"ئ":43113,"ا":1630465,"ؤ":8533,"إ":87017,"آ":11409,"أ":206598,"ء":28935,"غ":62643,"ع":367154,"ظ":26879,"ط":114141,"ض":60872,"ص":101344,"ش":119185,"س":320648,"ز":83586,"ر":577132,"ذ":48426,"،":89553,"؛":1398," ،":20335," ن":38705," ه":63492," ل":88748," م":255388," ق":40529," ك":61242," ف":145932," ي":66494," و":214375," ص":12486," ش":28169," ط":11510," ض":8567," ر":24040," ذ":6579," س":49888," ز":6199," ع":110158," ظ":1667," غ":11703," إ":56701," ا":619492," آ":6461," أ":122164," ج":33981," ح":46703," خ":15646," د":26005," ب":145150," ت":96353," ث":8150,"ア":1366,"ا، ":7320,"ئة ":2129,"ؤسس":2077,"إن ":1277,"أيض":3023,"أور":2981,"أهم":1974,"أهل":1229,"أول":9618,"أنو":1568,"أنه":3955,"أمي":2439,"ألم":3407,"أما":1795,"ألف":1545,"أمر":6318,"أكب":2802,"أكت":1724,"أكث":2574,"أفر":1800,"أعل":1292,"أعم":1551,"أغس":1326,"أصل":1634,"اث ":2044,"ئي ":3037,"ب، ":1611,"ات ":57401,"إضا":1407,"اح ":3081,"اج ":4358,"إسب":3960,"ئل ":2536,"اة ":4792,"إسل":3234,"اب ":9098,"إسر":1395,"إدا":2059,"اء ":22733,"إحد":6124,"إذا":1265,"ئر ":2350,"ة، ":15452,"اي ":2212,"بت ":1435,"بب ":1489,"اه ":2688,"بة ":12759,"ان ":40858,"با ":4030,"بد ":6780,"بح ":1410,"اً ":10489,"اف ":3597,"إيط":2527,"إير":1277,"ام ":31875,"ال ":23428,"ئرة":2003,"اك ":2756,"اق ":4807,"اع ":5640,"إلي":1606,"إلى":18043,"إما":1946,"اض ":1731,"إقل":1614,"اط ":2017,"إنج":4004,"إنت":2754,"إنس":1518,"ار ":15484,"اد ":13875,"اص ":1586,"از ":3651,"اس ":4999,"بي ":11247,"ت، ":2665,"اضي":4506,"اصر":1618,"بو ":3109,"اصم":3245,"اعب":4589,"اعة":3428,"اعت":1731,"اعد":2039,"اعر":1432,"اطع":4959,"اطق":1642,"اطي":1824,"اعي":4248,"ادا":1408,"اخل":1728,"احي":1991,"اخت":2071,"احل":1721,"احت":2360,"احة":4653,"احد":2974,"ارب":3471,"ارة":7735,"ارا":5132,"ادي":11501,"ادة":5295,"بق ":1932,"ادر":1555,"ازي":2304,"است":9208,"اسة":1966,"اسا":1699,"اري":13333,"اره":1487,"ارو":1690,"ارك":3428,"ارع":1412,"ارس":4342,"بل ":7375,"ارد":1984,"ارت":2585,"ارج":1460,"اصة":1686,"اشم":2112,"به ":4466,"اسم":7414,"اشت":1547,"اسي":6201,"اسع":1320,"بن ":14722,"ئلة":1510,"بع ":5351,"ائد":1719,"ائر":5727,"ائز":1678,"ائم":1524,"ائل":4622,"ائي":8565,"ابا":4420,"ابت":1320,"ابة":2266,"ابر":1286,"ابع":5010,"ابي":3084,"ات،":1833,"ابل":1965,"ابق":3015,"ابن":2545,"اتح":2012,"اتب":1961,"ئيس":4458,"اتي":3444,"اته":3712,"ئية":4071,"اجت":1267,"ئيل":1637,"بر ":15051,"بط ":1447,"آخر":1604,"أن ":6708,"آن ":1439,"أس ":2019,"أبر":1929,"أبي":2141,"أبو":3598,"أحد":5818,"أحم":2690,"أخر":2780,"أحي":1348,"أرا":1757,"أرب":1317,"أرض":1852,"أرد":4510,"أست":1490,"أسا":2106,"أسس":2383,"أشه":1227,"أصب":1379,"أو ":17977,"أي ":2409,"جزي":2701,"جدي":2169,"جزء":1922,"حل ":2135,"جرا":1466,"جري":2220,"جزا":2275,"جتم":2280,"ثير":2870,"جات":1361,"ثنا":1548,"جار":2741,"جال":2447,"جان":2465,"جبا":1237,"جام":4767,"جبل":1832,"جلي":4317,"جما":2589,"جلس":2643,"حاف":7429,"حال":5921,"حاك":1299,"جهة":1426,"جنو":6846,"جها":2454,"جمي":2428,"حاد":3278,"حار":1507,"جنس":1263,"حاس":1289,"جمع":2692,"جمه":2396,"جمو":3506,"خط ":1642,"خر ":2126,"حي ":2379,"د، ":2106,"دث ":1521,"حصل":1369,"دة ":22913,"دت ":1894,"حسب":1473,"دا ":3896,"حسي":1367,"حسن":1628,"حرك":2813,"حرا":2117,"حرب":2616,"خل ":1874,"حزب":1786,"حري":2754,"دأ ":1419,"حدث":1823,"حدة":6151,"حدا":1890,"حدي":3344,"حدو":1601,"حدى":6092,"حتى":2850,"حتو":1235,"جية":1498,"جيا":1555,"جيد":1284,"جين":1406,"جود":2059,"ختص":1274,"حول":2075,"حوض":3972,"حوا":3753,"خاص":2779,"خار":1757,"حمل":1584,"خاب":1378,"حلي":1951,"حمد":7071,"حما":1348,"حكو":1587,"حكم":2998,"حقي":1342,"دس ":1564,"در ":3643,"دد ":5353,"ذا ":6858,"دن ":3532,"ده ":2855,"دو ":1586,"دى ":7927,"خصي":1891,"ر، ":3918,"دي ":16032,"حيا":3483,"ختل":2576,"حيث":5743,"حية":2840,"حيو":1435,"خدا":1713,"خدم":4681,"خرا":1406,"دل ":1527,"خرج":2035,"خرى":2288,"دم ":12380,"خلي":2944,"خلا":4049,"دان":2670,"دام":2603,"دال":1357,"داي":1553,"داء":1232,"دائ":2836,"دار":5259,"دات":2179,"داخ":1695,"داد":3415,"ذي ":7019,"رج ":3503,"رة ":42156,"رت ":3799,"رد ":2816,"را ":5024,"رب ":12795,"ذه ":5431,"درس":2590,"درج":3597,"درا":3796,"دري":2750,"دول":7090,"دون":2547,"دوا":2452,"دود":1701,"دور":4858,"دها":2546,"دني":4787,"ذات":1939,"رف ":5073,"زء ":1646,"ديو":1641,"ديم":4297,"دين":17630,"ديا":3615,"دية":10224,"ديس":2111,"ديد":5637,"دير":2378,"ديث":1831,"رس ":4802,"رز ":1453,"رع ":2500,"دما":2598,"رض ":4034,"اقي":2703,"اقت":1926,"اقة":1778,"افي":2956,"اقع":1373,"الق":27005,"الف":20852,"الل":14610,"الك":19511,"الي":32249,"ان،":1862,"امت":1485,"امج":2075,"اما":2969,"الن":19094,"الم":106832,"امة":2785,"الو":15207,"اله":10510,"امر":2004,"امع":4584,"الأ":53765,"الآ":3087,"الج":24268,"اكي":1240,"الث":10222,"الت":44136,"الة":4500,"الب":27575,"الا":23279,"اكم":1385,"الإ":23108,"الس":25474,"الش":23985,"الر":18584,"الز":4628,"الد":22592,"الذ":9992,"الح":28530,"الخ":10845,"الع":45326,"الغ":7597,"الط":9353,"الص":10951,"الض":2239,"افة":3763,"افظ":6821,"اين":1399,"ايو":2112,"بتم":1591,"ايا":4981,"اية":7953,"اير":3864,"بدأ":2040,"، ":87650,"بحي":1333,"بحر":3662,"اهر":2224,"اني":27995,"انه":3927,"انو":4021,"بان":8783,"بال":24841,"باد":1374,"اند":2371,"بار":7325,"انس":1452,"باس":3805,"انا":3741,"باب":2616,"انب":1315,"انة":1432,"بات":4730,"انت":10216,"امي":6789,"باح":1289,"امل":2591,"بائ":1535,"اوي":2785,"اول":1939,"اهي":1800,"تا ":1767,"تب ":3208,"بشك":1596,"بدا":2243,"برا":7053,"برت":1316,"برل":1347,"برو":1928,"برن":1554,"بري":6364,"تر ":3568,"بطو":2790,"بعد":8590,"بعة":3652,"بغد":1205,"بعض":3329,"بني":2271,"بها":4540,"بنا":4249,"ئ ":1205,"ا ":148281,"بلي":1501,"بلا":3530,"ب ":70841,"بلغ":5588,"بلد":6651,"ء ":25878,"بقا":1527,"أ ":2761,"ح ":16042,"خ ":7746,"د ":111902,"ذ ":7044,"بين":10021,"ة ":417779,"بيل":2585,"بيض":1538,"بيع":2501,"ت ":98281,"بيا":2983,"بية":13845,"بير":5823,"بيت":1659,"بون":1363,"ث ":14734,"بول":3156,"بور":2404,"ج ":16315,"بوا":1540,"تي ":19035,"ثة ":2508,"تو ":1236,"تى ":3327,"ته ":7416,"تل ":1533,"تم ":5813,"ثر ":3741,"؛ ":1372,"تأس":2342,"تبر":4383,"تبع":2953,"تال":2664,"تان":2849,"تبا":1684,"تبة":1215,"تار":5301,"تاب":6518,"تاج":2425,"تبل":2737,"تجا":3016,"تخد":5000,"تخب":1329,"تخا":1370,"تحد":6290,"تحر":1572,"تحا":3110,"تحت":3198,"تدا":1332,"ثم ":2995,"تري":1455,"ترو":2478,"ثل ":4130,"ترا":5683,"ترة":2099,"ترك":2885,"تشا":2599,"تسم":2030,"تست":1924,"تصا":3295,"تشر":1532,"جة ":6594,"تشي":1720,"تضم":1238,"تطو":1569,"تعا":1531,"تعت":2311,"تعم":2414,"تعل":2476,"تعر":2059,"تعد":2614,"جد ":3856,"جر ":1340,"تفا":2566,"تقا":2028,"تقد":2088,"تقر":1287,"تقس":2440,"تقع":8338,"تقل":2844,"تهر":1315,"ثال":2031,"تها":6374,"ثان":4499,"تنظ":2616,"تمي":2924,"تنا":1728,"تمد":1486,"تمر":1589,"تلف":3439,"تلك":1400,"تما":2848,"تمب":1576,"تكو":3404,"تلا":1846,"تين":4590,"تية":1561,"تيا":2052,"تون":4528,"توي":1644,"توف":2858,"تول":1447,"تور":2920,"توس":1688,"توب":2148,"توا":1906,"تهم":1291,"جه ":1478,"جم ":2117,"جل ":2059,"جي ":1843,"حة ":7424,"حت ":2989,"حر ":2424,"حد ":6903,"ثلا":2751,"ثما":1889,"ثقا":1374,"وي":38534,"ي،":8021,"وو":3027,"وى":2435,"ية":149201,"يب":15166,"يا":76978,"يئ":2097,"يض":5937,"يط":9332,"يع":16206,"يز":13642,"يس":24541,"يش":6638,"يص":2709,"يخ":8388,"يد":33231,"ير":50995,"يت":21919,"يث":8777,"يج":6934,"يح":6752,"يه":17510,"ين":82989,"ًا":1622,"يو":34779,"يك":18257,"يق":23953,"يم":33457,"يل":36829,"يف":12030,"يغ":2332,"يي":7762,"فع":3459,"فض":3349,"فص":2010,"فظ":7666,"فز":1405,"فر":20396,"فس":4079,"فة":10560,"فت":8107,"ق،":1414,"فا":19253,"فب":1563,"قف":1276,"قع":16814,"قط":5495,"قض":1727,"قص":4820,"قش":1443,"قس":4834,"قر":18930,"قد":24585,"فى":1203,"قت":5868,"في":137714,"فه":4197,"قب":10150,"قة":17223,"فو":7133,"فم":1876,"فن":5345,"قا":33043,"فك":2212,"فل":7982,"فق":4801,"لأ":58367,"لآ":3323,"كف":1496,"كس":5921,"قي":21786,"كث":5852,"ل،":2207,"كذ":1298,"كر":19872,"كز":7004,"كأ":1420,"قل":10832,"قم":3276,"كا":41128,"قن":3553,"كب":9528,"قه":2295,"كة":14080,"قو":10571,"كت":14016,"قى":1403,"لك":43877,"مؤ":5145,"لق":35996,"لف":31832,"لط":12463,"لض":2442,"لص":14130,"لش":25578,"لغ":17940,"لع":56838,"لظ":1228,"لخ":11729,"لد":39289,"لج":27745,"لح":34548,"لز":5989,"لس":38336,"لذ":10795,"لر":20100,"كم":13188,"لا":89050,"كن":10747,"كل":17531,"لإ":25479,"لت":54464,"م،":3972,"كي":22561,"لث":10985,"لب":37291,"كه":2654,"كو":22542,"لة":26342,"مع":31975,"مغ":4324,"مص":12239,"مض":1909,"مط":2885,"مف":2713,"مق":12733,"مك":8220,"مل":28998,"مت":25250,"لى":49564,"مة":21124,"لو":37077,"مج":15409,"ن،":5643,"لي":91040,"مث":7965,"لم":137485,"لل":37112,"مب":12595,"له":28033,"ما":84509,"لن":25730,"مز":2658,"مر":34445,"مش":6760,"مس":26092,"مخ":5223,"مح":22151,"مذ":1262,"مد":31562,"نغ":3178,"نظ":9512,"نع":2681,"نط":11265,"نل":1207,"نم":4711,"نق":5444,"نك":3963,"نف":6969,"نح":3012,"نج":9690,"مي":49166,"ه،":2071,"مى":3142,"نت":26741,"نة":26561,"مو":33352,"مه":13330,"نب":6671,"نا":51288,"من":110993,"مم":11520,"نص":4888,"نش":6378,"نس":20514,"نز":3094,"نر":1454,"نذ":4480,"ند":18475,"هـ":2446,"وأ":7843,"وإ":2408,"هل":3624,"هم":13508,"هن":5865,"وا":96512,"هب":2043,"نه":22166,"ها":63427,"هت":1254,"نى":2312,"نو":28243,"هة":2492,"هج":2434,"و،":1267,"ني":68053,"هذ":11164,"هد":6895,"هز":1306,"هر":13839,"وغ":3162,"وظ":1284,"وع":15945,"وق":15305,"وك":13977,"وف":17742,"ون":47917,"وه":18574,"ول":64949,"وم":36461,"هي":33444,"وث":2062,"وت":24045,"هو":37979,"وة":2112,"وب":25979,"ود":18341,"وخ":2267,"وح":8074,"وج":13010,"وس":25063,"وز":7609,"ور":46060,"وذ":2568,"وط":5229,"وض":6816,"وص":4846,"وش":3688,"دة":23918,"خو":3104,"دت":2861,"دا":34518,"دب":3013,"دخ":1822,"دث":2213,"خي":4889,"خل":11579,"خم":2109,"دأ":2148,"خط":4248,"خر":10258,"خد":6707,"خص":5211,"ده":6608,"دو":24057,"دى":7961,"دي":71508,"ر،":3948,"دف":3789,"دق":1886,"دك":1306,"دل":4421,"دم":20032,"ذا":11430,"دن":9851,"دع":2356,"دد":6837,"در":20942,"دس":4645,"حث":1327,"جي":13871,"حج":2300,"حة":7643,"جو":9984,"حت":11398,"جن":11985,"حا":28730,"حب":2668,"جه":7398,"جل":12168,"جم":17274,"جع":2397,"جس":2540,"جز":8412,"جر":7944,"جد":8657,"خت":6160,"حي":21851,"د،":2117,"حم":14977,"حن":1639,"خا":10036,"خب":2705,"حو":13676,"حف":3106,"حق":4632,"حك":5593,"حل":8199,"حض":1296,"حص":3737,"حز":2507,"حر":15992,"حس":5733,"حد":30065,"تغ":2721,"تف":6519,"تم":21550,"تل":11923,"تك":7075,"تق":23507,"تى":3340,"ثة":2642,"تو":27372,"ته":18459,"ثا":9771,"تن":10701,"تج":7596,"تح":21120,"تت":7159,"تر":25584,"تخ":11500,"تد":5897,"تش":11506,"تص":9178,"تز":2287,"تس":9386,"تع":17161,"تض":2140,"تط":3895,"ثق":2026,"ثل":9196,"ثن":2113,"جا":22121,"ثم":5275,"جة":6738,"ثو":2447,"جب":5048,"ثي":5910,"جت":3434,"تي":34720,"ثر":5818,"بغ":2025,"بع":25200,"به":11285,"بن":26227,"بم":5940,"بل":28735,"بك":4252,"بق":7310,"بد":14809,"بج":2196,"اً":11293,"بح":10218,"بت":8320,"اي":29388,"ة،":15567,"بب":2982,"اه":11841,"او":11068,"بة":13305,"بط":8037,"بش":3786,"بص":1566,"بس":4381,"بر":41886,"تا":29084,"تب":19287,"تأ":5378,"بو":18997,"ت،":2681,"بي":60818,"ئد":1726,"ئر":5978,"ئز":1688,"إن":13110,"ئا":1293,"إم":3605,"ئة":2198,"ا،":7371,"إي":6297,"اء":25293,"ئل":4704,"ائ":31094,"ئم":1653,"از":9640,"ار":69575,"اذ":2153,"اد":41536,"اض":8516,"اص":11589,"اش":9130,"اس":39158,"ات":77501,"اة":4919,"ئه":1298,"اب":38588,"اخ":6254,"اح":20482,"اج":12583,"اث":6287,"ئي":14445,"ب،":1623,"اف":23866,"بأ":2984,"اق":16552,"ام":64051,"با":69767,"ان":106270,"اك":11934,"ال":726452,"بإ":1475,"اع":27489,"اغ":2741,"اط":14678,"ؤس":2217,"أو":36459,"أه":4128,"أي":8325,"أل":9391,"أك":9086,"أن":19914,"أم":14875,"إس":11365,"إر":1668,"إذ":1752,"إد":2895,"إخ":1619,"إح":7179,"إب":1901,"إل":22468,"إق":2002,"إع":1925,"إض":1479,"آخ":1667,"آل":2561,"أت":2542,"آن":1925,"أب":10779,"أح":11512,"أخ":6156,"أث":2807,"أج":3225,"أر":12567,"أد":4137,"أص":5345,"أش":3739,"أس":15627,"أغ":3162,"أع":5434,"أط":1989,"أق":3836,"أف":5069,"غي":8633,"غو":4024,"غن":2949,"غل":3357,"عي":17935,"غد":1582,"غر":13200,"غس":1789,"غز":1267,"عق":2845,"عل":49520,"عم":19388,"غا":8614,"عن":19968,"عه":6016,"غة":3737,"عو":7644,"ظي":3474,"عث":1594,"عت":10284,"عة":23777,"عد":27555,"عش":4458,"عس":1709,"عز":2669,"عر":25424,"عظ":2315,"عط":1205,"عض":5918,"عص":2498,"عا":50253,"ظه":2422,"عب":25388,"ظم":4301,"ظا":4120,"طن":4817,"طل":7662,"طق":11023,"طف":1950,"طع":6327,"ظر":2587,"طي":9898,"طو":11014,"ظة":6299,"ضم":8353,"ضل":2264,"ضع":1853,"طس":1527,"طر":9793,"ضي":7171,"طح":1428,"طا":16584,"طب":7274,"طة":4986,"ضو":3284,"صل":10145,"صف":5067,"صط":2224,"صص":1416,"صغ":2398,"ضر":2826,"ضة":1642,"صو":8178,"صم":4802,"صن":4515,"ضا":11319,"صي":8440,"شف":1822,"شك":5750,"شق":1763,"شع":5311,"شغ":1513,"صح":4260,"صد":4269,"صر":13415,"شم":11602,"صا":13513,"صب":5260,"شه":5994,"صة":3495,"شو":3361,"شي":12032,"سع":7475,"سط":12291,"سس":4815,"سف":3591,"شأ":1378,"سي":47561,"شت":5440,"سو":18175,"سه":3635,"شب":4456,"شا":16192,"سن":12898,"سم":24549,"سل":21272,"سك":11128,"شر":23735,"شد":1209,"شخ":3092,"زع":1204,"سب":17052,"زه":1918,"زن":1607,"سا":35091,"ست":30958,"سة":7853,"زو":4517,"زم":3202,"زل":2051,"سد":1627,"سر":8485,"سج":2382,"زي":20894,"رس":13875,"رش":2594,"رز":2717,"رط":1498,"رض":6362,"رع":5610,"رغ":2804,"زء":1924,"رل":3156,"رك":24686,"رق":13754,"رف":9661,"رو":32705,"زة":4009,"زب":2789,"ره":9008,"زا":10186,"رن":13036,"رم":7236,"س،":1548,"ري":92759,"رى":6117,"زر":4095,"ذر":1628,"ذك":2116,"رأ":2244,"رئ":5538,"ذل":5170,"رب":36257,"ذه":6871,"را":67798,"رت":12252,"رة":43634,"ذو":1454,"رج":14223,"ذي":9511,"رخ":1356,"رح":4450,"رد":11552,"ف ":28941,"ـ ":3661,"ع ":53161,"غ ":7968,"ص ":6276,"ض ":14917,"ط ":15243,"ظ ":1554,"ر ":119691,"ز ":20518,"س ":42343,"ش ":6531,"ً ":11009,"ي ":305236,"ه ":68493,"ن ":236663,"ى ":82765,"و ":86344,"ك ":22299,"ق ":30798,"م ":131428,"ل ":111126," ، ":18980," و ":12423," م ":5345," جن":4512," حا":7431," جه":1218," جم":4339," جي":2378," جو":3451," حت":2592," جد":1981," جز":3197," جر":1610," ثل":1523," ثم":3265," جا":6114," جب":2225," تي":1533," خل":4736," دا":4684," خط":2109," حق":1280," حك":2014," حل":1546," حي":8173," حم":2037," خا":3436," حو":8126," حر":3258," حس":2733," حد":2242," بك":1952," بق":1818," به":5365," بن":17676," بم":5621," بل":8899," بغ":1392," بع":9614," بس":2145," بر":8567," بط":3259," بش":2647," بت":3125," بد":4624," بج":1734," بح":3333," بأ":2828," ال":581886," بإ":1446," با":31729," ان":6245," ام":2207," اع":1488," ار":1569," اس":7954," اب":3029," ات":1745," اح":1418," اخ":1766," تو":6684," ثا":1511," تن":5058," تم":5949," تل":2911," تك":2690," تق":13534," تع":7393," تط":1471," تش":4009," تص":2525," تس":4420," تر":4248," تخ":1540," تد":1553," تج":1794," تح":6360," تت":4407," تا":4988," تب":4366," تأ":3604," بو":6771," بي":13768," أل":4956," أك":6683," أن":14293," أم":6327," أو":27007," أه":2795," أي":6316," آل":1841," أب":8228," أخ":2816," أح":8158," أج":2000," أث":1423," أر":3743," أد":2035," أص":3105," أس":5597," أش":2245," أع":3015," أغ":2285," أف":3376," أق":2564," إي":4306," إن":5007," إم":1413," إب":1312," إس":5794," إر":1268," إذ":1294," إد":1530," إح":6465," إل":21224," إق":1208," طو":2165," عش":2714," عر":3955," عد":6773," عا":21951," عب":8841," عي":1567," غر":3853," عل":33952," عم":7142," عن":15313," غا":2188," غي":2774," سع":2415," سي":7136," شب":1886," سو":5683," سم":1931," سن":7850," شا":3992," سك":3794," سل":4129," شر":6157," شخ":1437," شع":1651," شك":1235," شي":2028," شم":4082," صا":1583," شه":2018," صح":1413," ضم":5470," طا":1844," طب":1958," طر":3408," در":2995," دي":6094," دو":6652," ذا":1885," رئ":2370," ذل":2035," را":2673," رس":1784," ري":3022," رق":1403," رو":5273," زي":1372," سب":3177," سا":6895," ست":1885," لك":4551," مؤ":2595," لق":1950," لع":2571," لغ":1253," لص":2532," لج":1545," لح":1350," لد":1501," لب":2260," كو":5523," لت":3577," كي":3645," كل":6741," لإ":1374," كم":7185," كن":1321," لا":10830," مل":4355," مك":2868," مق":7894," مغ":1598," مع":14545," مص":6997," مس":11150," مش":2698," مر":11437," مد":14412," مح":14231," مخ":2723," لي":4897," مث":3231," مج":7634," لو":4068," مت":7729," لن":2133," ما":17956," مب":2224," له":5386," لل":21688," لم":6816," نف":1935," نق":1569," نظ":2491," نس":4267," نش":1472," مم":3100," نا":8059," من":85288," مه":1676," مو":12716," مي":4669," نج":1432," وأ":7664," هـ":2402," هن":1965," وا":50314," وإ":2381," هذ":9093," نو":6152," ها":3011," نه":2387," ني":2637," فر":6896," فب":1504," فا":3547," فت":2193," قص":1901," قط":1391," فق":1797," فل":2626," فن":1697," قا":5741," فو":2197," فه":1391," قب":5253," في":115948," قد":7793," قر":6492," لأ":2928," قل":1628," كت":3116," قو":2125," كب":2590," كا":17231," قي":2400," كث":1288," كر":7439," ود":1531," وخ":1367," وح":3870," وج":3378," هي":20610," وت":16290," هو":22238," وب":6445," وص":1974," وش":2083," وس":5388," وز":1947," ور":2782," وذ":1466," وغ":1624," وع":5254," ون":2492," وه":15877," ول":15982," وم":13412," وق":7666," وك":6757," وف":4804," وو":1985," وي":13320," يا":1484," يب":2139," يح":3014," يج":1805," يت":5470," ير":1696," يد":1787," يص":1234," يش":1764," يس":4063," يع":6186," يل":3428," يم":2984," يق":8446," يك":1977," يو":8148," ين":4302,"فس ":1508,"فر ":2554,"فع ":1454,"فا ":1471,"ق، ":1401,"فة ":10112,"قع ":14708,"فار":1789,"فات":2286,"فال":1824,"فاع":2545,"فان":1282,"فبر":1355,"فتر":2588,"قب ":1566,"قا ":1491,"قت ":1589,"قة ":16768,"في ":111805,"قد ":7827,"عظم":1719,"عضو":1448,"عشر":3482,"عسك":1369,"عزي":1296,"عري":1226,"عرو":2224,"عرف":3788,"عرض":1945,"عرب":8060,"عرا":4113,"عدة":2689,"عدا":2204,"عدد":5430,"عدي":2807,"عتم":1554,"ظيم":2842,"ظهر":1451,"عبر":1705,"عبد":6249,"عبي":2224,"عتب":4439,"عات":3583,"عائ":1805,"عاب":1251,"عال":9007,"عام":18844,"عبا":2776,"عاص":3770,"عار":1586,"عاد":3689,"ظمة":1530,"غسط":1434,"غدا":1272,"غرا":1470,"غرب":8757,"عية":4635,"عيد":1672,"عين":3620,"عمل":7530,"عمو":1379,"عمر":2346,"عها":1818,"عني":1581,"غال":2120,"عند":3134,"عهد":2118,"عود":4342,"علا":3567,"علي":8023,"على":25792,"علو":2893,"عما":5586,"علم":6076,"غني":1336,"غير":5788,"شعر":1240,"شعب":2093,"شما":7339,"شكل":4349,"صال":3820,"شهر":2525,"صبح":1804,"صاد":2357,"صار":2022,"شمي":2083,"صدر":2012,"شير":1659,"شيخ":2626,"طة ":4836,"ضي ":2799,"صطل":1544,"صري":4147,"ضم ":1876,"طب ":1314,"طس ":1466,"صغي":1584,"صول":1823,"صور":2575,"ضاف":1670,"ضاء":2485,"صنا":1592,"صمة":2695,"طق ":1799,"صية":1437,"صين":1293,"طي ":1477,"ظة ":6257,"طان":3004,"طال":3847,"طاق":1625,"طار":1511,"طائ":1561,"ضمن":5751,"طبي":3127,"ضية":1599,"طري":4168,"ظم ":1728,"عب ":10724,"عة ":23265,"طعة":4116,"عد ":10757,"عر ":1910,"طلح":1521,"طلق":2117,"طلا":1304,"عض ":2738,"طقة":8653,"طول":4093,"طوي":1533,"طور":2637,"ظام":2445,"طني":2154,"طين":2907,"عل ":1213,"غة ":3658,"عن ":10475,"عه ":1714,"عي ":3208,"س، ":1536,"ري ":14515,"رن ":3048,"زب ":1762,"ره ":3780,"زة ":3835,"رو ":2228,"رى ":5751,"رق ":5370,"رك ":3561,"رجي":1396,"ردن":4580,"ردي":1386,"رتب":1460,"ربع":2964,"ربي":13069,"رجة":3401,"رجا":1686,"ذين":1312,"رتف":1376,"رته":1217,"رتي":1328,"ذلك":5018,"ران":5308,"ربا":2177,"راه":1718,"راي":2650,"رة،":1284,"راً":1437,"راط":1969,"راض":2661,"راع":2044,"راف":2275,"راك":1677,"راق":4338,"رام":2614,"رال":1755,"راب":2845,"رائ":2827,"راج":1725,"رات":7689,"رئي":4628,"رار":2322,"راد":1959,"راز":1251,"راس":3033,"راء":3735,"ذكر":1364,"رأس":1309,"سس ":1682,"سر ":1409,"زي ":2367,"سة ":7637,"ست ":2491,"رسة":1580,"رسا":1400,"سا ":2325,"رسو":1660,"رسم":1498,"رسي":1211,"سب ":2371,"سم ":8379,"زرا":1870,"سل ":2019,"ريو":1361,"ريم":1723,"رين":5069,"ريك":7429,"ريل":2022,"ريف":1889,"ريق":6975,"ريا":9368,"ريب":2334,"ريخ":4312,"ريد":2596,"ريت":1569,"رية":19287,"ريط":2009,"رير":1548,"ريس":2387,"روي":1263,"روف":2873,"رون":3533,"روم":2673,"روع":1294,"روس":3818,"روت":1414,"روا":3408,"روب":3276,"رها":3942,"رنس":4822,"زار":1503,"رنا":1951,"زائ":1892,"سع ":1228,"ركي":2730,"ركز":6519,"ركا":1845,"رقم":1561,"ركة":6574,"سط ":4117,"رقي":3306,"رفي":1260,"رقة":1238,"شر ":3663,"سن ":1880,"سي ":9223,"سري":1260,"سرا":2022,"صب ":1600,"ساع":1424,"سام":1889,"سال":1558,"سبب":1717,"سان":4418,"سبا":5421,"سبت":1759,"سبة":1761,"ساح":5795,"ساس":2185,"سائ":1433,"سات":2066,"ساب":4056,"ستخ":5235,"ستا":3297,"ستر":2386,"زوج":1305,"زيو":1415,"ستع":2199,"ستق":2335,"زيا":2152,"ستي":2146,"زية":3984,"ستو":3126,"زيز":1328,"زير":3447,"صر ":6441,"سلة":1569,"سكن":1586,"سلا":5866,"سكر":1892,"سلي":1241,"سمب":1534,"سمة":2358,"سلم":2588,"سما":2312,"سلط":1809,"سلس":3040,"سكا":4595,"سطس":1421,"سطي":2518,"صة ":3360,"سعو":3150,"شرك":4217,"شرق":6331,"شرو":1260,"شري":3368,"صل ":5101,"ضة ":1538,"ضا ":2191,"سوف":1368,"سون":1558,"شتا":1442,"سوي":1779,"سوا":1477,"سود":1542,"سور":3979,"شاع":1427,"سمى":2128,"سنة":6767,"سمه":2089,"شار":4714,"سمي":3653,"شخص":2650,"صف ":1872,"سين":3825,"سيم":3305,"سيق":1533,"سيس":1471,"سير":1927,"سيد":1957,"سية":7660,"سيا":7442,"شته":1322,"يين":4613,"يقو":1288,"يقي":3960,"يقع":4212,"يكا":2890,"يكي":7123,"يلع":3028,"يلة":3508,"يكو":2936,"يلا":3946,"يلي":4907,"ين،":2158,"يلم":2210,"يما":3516,"يمة":2518,"يلو":2309,"يمت":1285,"ينا":6385,"يمن":1503,"يمك":1768,"ينت":2412,"يمي":5784,"ينة":14065,"ينو":1382,"يني":7771,"ينم":1347,"ينه":1443,"يها":5753,"يوس":2044,"يور":1429,"يوج":1408,"يوا":1512,"يون":8407,"يوي":1456,"يوم":3633,"يول":2410,"يقا":3504,"يقة":2474,"يفي":1837,"يفة":1491,"يضا":3644,"يسي":3184,"يسم":2811,"يزي":6603,"يست":3754,"يسا":1431,"يره":2025,"يرو":2802,"يري":3325,"يعي":2030,"يعر":1330,"يعة":1447,"يعت":2839,"يطا":4786,"يجي":2143,"يتا":1503,"يتو":1517,"يتي":2848,"يتم":2214,"يته":1496,"يدي":4565,"يرا":5132,"يرة":7852,"يحي":1325,"يدة":3171,"يدا":2277,"يال":1789,"يبا":1460,"يان":5624,"يام":1663,"ية،":7809,"ياً":2041,"يبل":2267,"يبي":2606,"يئة":1233,"يا،":2810,"ياس":4902,"يار":3005,"ياد":1919,"ياض":3099,"ياء":2370,"يات":14201,"ياب":2680,"يائ":1532,"وز ":1740,"ور ":10877,"ود ":7936,"وض ":4335,"نفس":2468,"وس ":4788,"هاد":1315,"هاج":1381,"هاز":1295,"هار":1592,"هاش":2115,"نما":2137,"ها،":1369,"وع ":4025,"نقل":1354,"وط ":1225,"نيس":1670,"نيا":8848,"نية":21320,"نوي":1623,"نون":2797,"نور":1366,"نوب":7417,"نهم":1521,"نوا":4198,"نوف":1932,"نوع":3104,"هاي":1376,"هام":1659,"نها":10954,"نهر":1868,"وف ":3389,"هذا":5637,"وق ":2861,"نين":2937,"نيو":3670,"وم ":10050,"ون ":24085,"هرة":1975,"هذه":5272,"هرب":1206,"وك ":1613,"ول ":15598,"وي ":5541,"ي، ":7950,"وى ":2336,"مغر":2269,"معي":2389,"معل":1517,"معه":1532,"معر":3470,"معت":1206,"معة":3514,"معا":4652,"هد ":3199,"هر ":6421,"مقا":7017,"مقر":1875,"مقد":1322,"ملة":1472,"ملا":2140,"مكن":2428,"مكا":1474,"مكت":1317,"منا":3879,"ممل":5220,"نائ":1835,"ناء":3405,"ناد":6151,"منذ":4141,"منت":3699,"نات":4769,"منظ":1521,"منط":9032,"ناط":2245,"ناص":1498,"منص":1619,"ناس":1610,"ناع":1893,"ملك":8686,"مما":1241,"ملي":3442,"ممث":1944,"مهو":2239,"موا":8484,"نبي":1323,"موج":1832,"مور":1727,"مود":1263,"موس":3401,"موع":3576,"موق":1778,"ناك":1264,"نام":2198,"نان":5318,"نبا":1540,"مها":3875,"منه":3913,"ناي":2308,"مني":1297,"نتق":1216,"نتش":1566,"نتخ":2464,"نتج":1837,"نتر":1240,"نتا":3081,"مون":2835,"موم":1267,"مول":1330,"ميل":3548,"ميع":1518,"ميد":1868,"ميز":2165,"مير":3381,"نتم":1247,"ميا":3833,"ميت":1549,"مية":10968,"نتي":2558,"نجل":4522,"مين":4370,"هـ ":2122,"ندا":2054,"ندم":1329,"ندو":1295,"ندر":1700,"ندس":1559,"ندي":3768,"هل ":1310,"هم ":9416,"نسا":3625,"نسب":2400,"وا ":3570,"نسم":2218,"نشا":1687,"نسي":6788,"وب ":7482,"هو ":28172,"وة ":2028,"وت ":2509,"هي ":25821,"نطق":9203,"نظا":2438,"نظر":1914,"نظم":2245,"نظي":2652,"ومن":4650,"ونا":4025,"ومي":3551,"ونس":2979,"وما":5525,"وله":2128,"ولي":9104,"ومة":1640,"ولو":3124,"ولى":4041,"ومت":1425,"ولك":1651,"يع ":3808,"ولة":5953,"وكي":1778,"ولا":11130,"ولد":6407,"وكا":4860,"يط ":1772,"وقا":1242,"وفم":1445,"وفي":6567,"يض ":1235,"وقد":4338,"وقع":2237,"وفا":1769,"يش ":2562,"يس ":7953,"يق ":5733,"يف ":4553,"وين":2166,"ويو":1325,"ويق":1310,"ويل":2767,"ويس":2024,"ويع":1988,"وية":5123,"ويت":3787,"وير":2162,"ويد":1280,"وهي":6413,"وهو":7384,"وني":7306,"يه ":8938,"يو ":7129,"يم ":12246,"ين ":40630,"ًا ":1445,"يك ":2151,"يل ":12694,"وتو":2723,"وتق":1399,"هير":1949,"وتع":2163,"هول":1332,"وتر":1472,"وتت":1213,"وجه":1246,"وجي":2191,"وجو":2155,"وحد":1804,"هيم":1268,"وجد":2346,"واع":2005,"واس":3039,"وار":3337,"هند":2766,"واد":3067,"واح":3498,"واج":1496,"وات":4032,"واب":1388,"هنا":1458,"وائ":3044,"واء":1341,"هما":1800,"هور":3656,"وبي":4664,"وبل":1243,"وبر":3000,"واي":2060,"واق":2063,"وال":49239,"وان":5702,"وبا":3324,"وري":12364,"وسا":1602,"وزي":2144,"يا ":24320,"وسط":5112,"وسي":5438,"يب ":4787,"ية ":139658,"ودا":1655,"ودة":1389,"ودي":4370,"ورا":3830,"ورد":1319,"ورت":1259,"ورة":4176,"وزا":1263,"ورو":3091,"يد ":18420,"ير ":22585,"يز ":3857,"يث ":7060,"يت ":4897,"يج ":1671,"وطن":2615,"يح ":1259,"وعة":3516,"وعا":1992,"يخ ":5401,"وعي":1203,"لد ":7149,"لس ":3640,"لة ":25605,"كو ":1397,"لت ":2572,"لا ":8651,"كن ":4742,"قسم":1469,"لب ":3341,"قسي":2428,"لح ":4377,"لث ":1339,"م، ":3933,"كي ":5663,"لق ":2574,"لف ":3842,"كثي":2170,"له ":10668,"كسي":1295,"ما ":21645,"كرو":1414,"لم ":13383,"كري":3468,"لك ":9653,"كرا":1233,"كرة":8153,"كات":3795,"قنا":1376,"كار":2777,"لغ ":5250,"قلي":3507,"قيم":1916,"قيق":2120,"كثر":3184,"قية":4744,"كتو":3421,"قيا":3458,"كتب":2651,"كتا":3440,"قوم":2204,"قوا":2182,"كبي":3060,"كبر":3750,"كان":20675,"كام":2151,"كال":2865,"قني":1349,"كر ":2225,"كز ":6044,"كس ":1605,"قل ":2905,"فرا":2274,"قم ":1363,"فري":3990,"فرن":4775,"فرق":1262,"كا ":2472,"كة ":13804,"قى ":1356,"ل، ":2187,"قي ":5360,"فضل":1264,"فظة":6038,"فير":1961,"فيز":1412,"فية":2824,"فيد":1310,"فيل":3788,"فيه":4815,"فين":1921,"قتص":1591,"فيا":1360,"قدر":1289,"قدم":9632,"قدي":3437,"قرن":2599,"كم ":5038,"قرى":1679,"قري":5990,"كل ":7728,"قرا":2563,"قرب":1568,"فلس":2794,"فلا":1746,"قاط":5379,"قاف":1502,"قال":3148,"قائ":2080,"قاب":1505,"فنا":1210,"قات":2176,"قاد":1517,"قار":3777,"فمب":1413,"فور":1725,"قبل":4422,"قان":1522,"قام":2893,"لمع":7155,"لمغ":2396,"ماء":2699,"لمر":5941,"لمس":9884,"لمش":2559,"لمص":4216,"لمط":1236,"لمت":9057,"للو":1687,"لمة":1580,"لمج":4154,"للي":1280,"لمخ":2131,"لمح":5998,"لمد":5967,"لمؤ":2072,"للم":3559,"لمب":2156,"لله":4901,"لما":10778,"لنف":1218,"لنس":1662,"ماس":1244,"لنظ":1668,"ماع":3269,"لمي":7408,"مات":4917,"مار":7604,"ماد":2511,"لند":3683,"لمل":4323,"لمك":1995,"لمق":3111,"لمو":6905,"لمه":2311,"لنب":1670,"لنا":3789,"لمن":8707,"لمم":6914,"مائ":1529,"مؤس":2067,"لكر":4170,"لقي":1879,"لكب":1845,"لكا":2896,"لكت":3059,"لكة":5304,"لقو":2652,"للغ":3104,"للع":2547,"للح":1276,"لكي":2900,"للت":1711,"لكو":3675,"للب":1703,"لكه":1307,"للا":2747,"لكن":3420,"لكل":2395,"للأ":1511,"لفر":5137,"لفا":3058,"لفة":1423,"لفت":1915,"لقر":6914,"لقد":5675,"لقص":1290,"لقط":1312,"لفل":2420,"لقب":2460,"لقا":4848,"لفن":2512,"لفي":3665,"نس ":2245,"ند ":3871,"نذ ":4066,"لعب":7815,"لعا":12423,"لعد":3105,"لعش":1359,"لعز":1344,"لعر":10614,"لعص":1325,"لعل":5280,"لغا":2170,"لعم":4185,"لغة":3148,"لغر":4050,"لطب":2889,"لطا":3030,"لطر":1829,"ه، ":2068,"مي ":10289,"لطي":1270,"و، ":1265,"ني ":19921,"نى ":2242,"مصر":6502,"مصط":1846,"نو ":1687,"هة ":2433,"مصا":1239,"مسل":3430,"نه ":6502,"مسي":2261,"مشا":1795,"ها ":46135,"مست":5627,"مسا":7688,"مرك":7198,"مري":7170,"مرو":1205,"مرا":3794,"مرة":1768,"مرب":1566,"مرت":1375,"مدي":14448,"مدر":2755,"مدن":2068,"مخت":2389,"مدا":1536,"محل":1510,"محم":5603,"ليو":6519,"ليه":5228,"مجا":2341,"لين":3186,"مجم":3939,"محا":8346,"مجل":3322,"ليا":10564,"ليب":2394,"لية":13788,"متو":1712,"ليز":4850,"ليس":2021,"ليد":6582,"ليم":5806,"ليل":2383,"مثل":6304,"ليف":2446,"لوي":1597,"لون":3102,"متا":1234,"لول":3949,"لوم":4251,"متر":3819,"متد":1293,"متح":5418,"متع":1688,"لهن":1850,"لوا":3317,"لهو":1322,"مبي":1451,"لوج":1868,"لوح":1460,"لور":2028,"لوس":2838,"لوط":2044,"لوك":1356,"مال":13395,"مام":2587,"لنق":1670,"لنو":1865,"مان":13056,"مبا":2715,"لها":8071,"ماي":3281,"لهج":1315,"مبر":6074,"لأص":1592,"لأس":4356,"لأر":8270,"لأد":1635,"لأع":1940,"لأخ":2355,"لأح":2438,"لأب":1984,"لأف":1387,"لأل":2944,"لأك":1560,"لأن":3516,"لأم":7460,"لأو":8534,"لأي":1221,"لإس":5236,"كلا":1532,"مر ":4953,"مس ":1733,"مد ":7895,"مة ":20092,"لو ":1775,"مت ":2546,"لى ":49394,"لي ":18230,"ن، ":5582,"مج ":2119,"لسف":1264,"لشا":2194,"لسن":1963,"لسك":1528,"لسل":6280,"لسي":5235,"لشب":1219,"لسو":4213,"نب ":1230,"مه ":3791,"لسع":2105,"لسط":2488,"لسا":4332,"من ":76224,"نا ":5184,"لصو":1503,"مى ":3080,"نت ":7260,"لصي":1707,"لشم":4027,"لصا":2983,"لشه":1224,"لصح":2065,"نة ":25901,"لشي":3442,"لشر":6885,"لشع":2442,"لدي":8697,"لدو":6858,"لدر":3930,"لخل":2460,"لدا":2727,"لدة":1345,"لحي":2699,"لري":2867,"لرو":3390,"لرس":1728,"مل ":8293,"لذي":7896,"لرئ":2291,"لرا":2519,"لته":1222,"كيا":1952,"لتن":1720,"لثا":5157,"كية":5479,"لتو":2940,"لتي":15294,"كيل":2229,"كيم":1808,"لجا":2810,"لتا":4876,"كون":7678,"كوم":2797,"لتح":2710,"لتج":1719,"كوي":2650,"لتر":3418,"لتص":1218,"لتش":1260,"لتع":2665,"لتل":1255,"لتق":2650,"لحد":2696,"لجو":1720,"لجي":2140,"لحر":4857,"لحس":1519,"لحم":1993,"لخا":2598,"لحق":1482,"لحك":2052,"لجد":1516,"لجب":1382,"لجز":3847,"لحا":4240,"لجن":5221,"لجه":1784,"لجم":3177,"لاث":2511,"لاج":1537,"لاح":3135,"لاد":4259,"كند":1265,"لار":1311,"لاب":1932,"لات":8075,"لاق":3425,"لاف":2118,"لاس":6196,"لاع":5890,"كلي":3267,"لإي":1693,"مع ":8709,"كلم":2454,"لإم":2003,"لإن":6523,"كما":4559,"كور":1677,"لبو":1874,"لبي":3816,"لبل":3617,"لبن":3363,"كول":1392,"لبر":5552,"كهر":1331,"لبح":3898,"لاي":9010,"كنه":1287,"لام":9049,"لان":6278,"لبا":3771,"لال":6356,"لبط":1368},"n_words":[11749565,13990834,9440598],"name":"ar","type":"arab"} \ No newline at end of file
diff --git a/contrib/languages-data/bg.json b/contrib/languages-data/bg.json
new file mode 100644
index 0000000..38d1b9f
--- /dev/null
+++ b/contrib/languages-data/bg.json
@@ -0,0 +1 @@
+{"freq":{"D":2636,"E":1936,"F":2232,"G":2334,"A":4351,"B":3121,"C":4592,"L":2480,"M":3950,"N":1781,"O":1368,"H":1934,"I":6368,"J":1261,"K":1010,"T":2986,"W":1604,"V":1965,"P":3771,"S":5211,"R":2316,"X":1029,"f":3146,"g":5865,"d":9193,"e":32549,"b":4459,"c":9844,"a":30637,"n":21831,"o":21963,"l":16413,"m":12336,"k":7480,"h":8702,"i":28615,"w":2350,"v":3143,"u":15394,"t":17966,"s":19762,"r":22456,"p":5894,"z":1439,"y":4095,"x":1681,"²":3527,"̀":1150,"μ":1054,"ν":2280,"ο":2756,"ι":1833,"κ":1014,"λ":1144,"ε":1298,"α":2356,"ί":816,"σ":1479,"ς":1823,"ρ":1221,"τ":1963,"ь":4950,"ю":16520," o":1042,"я":119927,"ш":24527,"щ":37947,"ъ":118638," k":4461," d":1024,"ф":35061,"х":26992," e":2141,"ц":57885,"ч":69969,"р":442208,"с":363493," a":954,"т":513431,"у":110117,"ѝ":1026," t":1519," p":1297," s":806,"Й":1921,"И":10722,"Л":9341,"К":22064,"Н":13530,"М":19622,"П":22329,"О":13337,"Б":18556,"А":21177,"Г":11598,"В":16284,"Е":7594,"Д":15403,"З":5627,"Ж":1534," J":1196," K":898,"Ш":3131," H":1769,"Щ":2903," I":3444," N":1367,"Ю":3236,"Я":1596," O":956," L":2204," M":3517," B":2694,"Т":16259,"У":4389," C":3884,"Р":16110," A":3726,"С":33637," F":2034,"Ц":2988," G":2045,"Ч":3270,"Ф":8267," D":2259," E":1564,"Х":7715,"л":282008,"к":273267,"й":56719,"и":734943,"п":177542,"о":637033,"н":589099,"м":161532,"г":129585," S":4264," R":2038,"в":294348,"б":87024," P":3303,"а":881417," W":1432,"з":132692,"ж":46599," V":1365,"е":647345,"д":212987," T":2518," А":17337," Б":18206," В":15997," Г":11364," Д":15029," Е":7181," Ж":1503," З":5448," И":10217," Й":1919," К":20624," Л":9135," М":19107," Н":12710," О":12505," П":21699,"Co":972,"I ":2499," б":25135," а":29615," г":41568," в":81473," е":87884," д":46722," з":26978," ж":5999," и":93394," л":8242," к":53137," н":137556," м":36472," п":102853," о":88041," Р":15097," С":31643," Т":15680," У":4153," Ф":7812," Х":7558," Ц":2895," Ч":3246," Ш":3096," Ю":3200," Я":1567," т":27331," у":9960," р":36272," с":111437," ц":6298," ч":13239," ф":16514," х":6966," ш":3296," щ":6810," ю":3698," я":2381,"Ca":930,"Ma":1469,"II":1888,"Th":962,"b ":891,"a ":4950,"i ":1873,"ge":970,"he":2112,"ha":1204,"g ":1075,"ea":1397,"ec":826,"ed":916,"de":1775,"di":1045,"do":824,"h ":1014,"el":2062,"en":3274,"et":1200,"es":2494,"er":5287,"ca":1035,"e ":9458,"da":1108,"f ":974,"co":1092,"ci":1241,"ch":1482,"ce":1332,"d ":2383,"at":2587,"as":1457,"ar":3743,"al":2739,"ai":928,"am":1115,"an":4726,"ac":1396,"ad":830,"ae":1227,"nu":977,"nt":2097,"ns":1121,"no":824,"of":927,"om":1215,"on":3740,"ol":1525,"m²":3381,"ot":815,"os":1234,"ou":906,"or":3049,"r ":3015,"pe":805,"lo":1036,"ll":2061,"o ":2327,"ma":1173,"mb":856,"me":1262,"mi":900,"na":1825,"nd":1832,"ne":1694,"ng":1442,"ni":2114,"m ":2526,"km":4286,"li":2736,"le":2314,"la":2281,"n ":4895,"hu":824,"hi":896,"id":1203,"ic":2468,"ia":2261,"ig":1082,"ie":1280,"k ":904,"ir":844,"is":2366,"it":1525,"iu":2316,"il":1887,"in":4071,"io":1821,"l ":2342,"y ":2050,"vi":1029,"ve":1027,"x ":1094,"ul":1045,"ur":1295,"us":5560,"um":1527,"un":918,"tu":1122,"to":1372,"tr":962,"te":2234,"ti":2705,"th":2095,"ta":1616,"ss":891,"st":2061,"se":1041,"si":1173,"rt":1054,"ro":1958,"rn":867,"ri":3533,"re":2281,"rd":988,"ra":2624,"t ":3334,"s ":10393,"² ":3527,"ς ":1819,"ν ":1021,"К ":979,"В ":2290,"юг":1081,"юз":1052,"юж":980,"юл":1213,"юн":1123,"юр":1010,"ют":1002,"юц":3295,"юч":2002,"яд":1101,"яг":923,"яв":7999,"ян":5481,"ям":2556,"ял":2107,"як":3070,"ях":803,"яс":1766,"ят":23106,"яр":1490,"ящ":989,"щи":5926,"ще":7866,"ща":13149,"щт":2297,"що":3015,"щн":816,"ъв":5386,"ъг":8535,"ъд":2915,"ъе":2628,"ъж":1821,"ъз":5682,"ъб":2128,"ът":9515,"ъч":1319,"ъц":2116,"ъщ":3958,"ъл":18163,"ък":3158,"ън":6470,"ъм":3577,"ъп":2106,"ъо":998,"ъс":7274,"ър":28553,"ьо":4687,"хе":1611,"хи":3810,"хн":1728,"хо":7132,"хр":1667,"ху":1707,"ха":4667,"ци":32071,"цк":2149,"ца":7811,"це":9248,"чл":917,"чн":11748,"чо":1988,"чи":7731,"чк":2263,"чу":949,"чр":804,"цъ":803,"че":23255,"ча":14633,"чв":2213,"шн":3596,"шк":2046,"ши":6517,"ше":4003,"шв":1228,"ша":3103,"ск":74214,"см":3598,"сл":11817,"со":8891,"сн":7769,"ср":4324,"сп":8021,"св":7943,"се":41908,"си":23052,"рш":1570,"ръ":16028,"ря":3303,"са":17003,"рс":15501,"рт":9867,"ру":13758,"рх":2778,"рц":2298,"тн":10904,"тл":2176,"тк":3861,"тс":3598,"тр":27985,"то":91021,"те":76446,"тд":804,"тв":20249,"ти":47326,"сь":1346,"съ":18957,"ся":1659,"та":97852,"тб":3928,"су":2617,"ст":96170,"сц":1590,"ур":10561,"уп":6474,"ут":4765,"ус":9980,"ум":6401,"ул":6590,"ун":6478,"уи":1236,"уз":3299,"ук":5239,"уд":4037,"уг":3988,"уж":1616,"уе":1334,"уа":3124,"тя":2319,"уб":6162,"ув":3514,"тъ":10928,"тт":3628,"ту":8218,"фу":2995,"фс":1030,"фр":3456,"фо":5112,"фи":11161,"фе":4259,"фа":3044,"уч":6066,"уш":4842,"ух":1233,"уц":1227,"Щ ":2591," I ":819," II":1042," Ma":1457,"а ":358572,"С ":937," Ca":925," Co":962,"Ис":1007,"Им":977,"Ин":1158,"к ":21804,"Из":2286,"Ив":1372,"й ":18686,"Ле":1710,"Ли":1795,"Ла":1638,"Ку":1413,"Кл":1286,"Ко":5445,"м ":14788,"Кр":2175,"Ке":836,"Ки":1609,"Ка":6073,"л ":25621,"Йо":1544,"На":5278,"Не":1875,"Ни":2119,"Мо":2461,"о ":136170,"Ма":9260,"Ми":2960,"Ме":2505,"Ло":1497,"н ":69303,"Лу":1452,"Па":3246,"Пе":2744,"Пи":1272,"Пл":3225,"с ":29451,"По":4526,"Оп":1050,"р ":30654,"Ос":2133,"Ор":1247,"От":1087,"Об":2207,"Ок":2446,"Но":2049,"п ":2647,"в ":63211,"Ам":1040,"Ан":3455,"Ал":2956,"Ав":1300,"Ба":2769,"Ат":831,"Ар":1719,"б ":1391,"АЩ":2595,"Во":969,"д ":24331,"Ве":2399,"Ви":2290,"Въ":2366,"Га":1581,"Бо":2717,"г ":21217,"Бр":1925,"Бе":2785,"Би":1506,"Бъ":3741,"Ва":2968,"Бу":1134,"Ди":2316,"Дж":3197,"Де":2501,"Др":1035,"До":1935,"ж ":1730," Th":951,"Ед":825,"Ев":1551,"Ге":2579,"Гр":2290,"е ":164103,"Го":1786,"Гъ":1180,"Да":1394,"и ":202644,"За":3607,"Ел":966,"з ":12874,"Ша":824,"Юж":1162,"ю ":929," km":4266,"я ":66327,"Ст":4040,"Та":1626,"Съ":4599,"Ти":1166,"Те":2600,"ф ":1914,"То":5455,"Тр":1786,"Ту":994,"Тя":1180," e ":1798,"х ":2350,"Пр":4433,"Пъ":1179,"СА":2694,"Ра":3306,"Ре":3655,"Ри":2528,"т ":98842,"Ро":3022,"Ру":2132,"СС":822,"Са":2940,"Св":1625,"Си":2245,"Се":4020,"Сл":1252,"Ск":946,"Ср":881,"Сп":1322,"Со":2387,"у ":10195,"Це":1510,"ш ":1936,"Че":1438,"щ ":4203,"ц ":3623,"Фо":984,"Фр":2381,"Фе":819,"Фи":1127,"Фл":885,"Ха":2310,"Хр":971,"Хо":1081,"ч ":2829,"Хе":1295,"лю":6411,"мб":1834,"ля":9658,"ма":34017,"мв":3536,"ме":34130,"ми":25035,"лм":3442,"лн":16796,"ло":33848,"лс":4866,"лт":3196,"лу":5370,"лъ":2856,"къ":5642,"лв":914,"лб":2941,"ла":34165,"лж":1120,"ле":40750,"лд":1625,"лг":12737,"лк":4727,"ли":64840,"лз":2343,"км":1179,"кн":2476,"кл":6420,"кр":18141,"кс":6288,"ко":52495,"кт":11901,"ку":5862,"кц":1433,"ка":68697,"ки":58354,"кв":3364,"ке":7674,"йн":7255,"йо":1892,"йк":1995,"йл":1354,"йм":901,"йс":13136,"йт":4652,"ия":73064,"ищ":2902,"иш":2457,"йд":1453,"йв":1615,"ио":15275,"ип":3934,"им":23532,"ин":58356,"ик":36656,"ил":29439,"ии":4086,"ий":12560,"иц":13361,"ич":23215,"иф":2446,"их":3360,"ит":63592,"ир":17875,"ис":34276,"ри":74729,"рк":5043,"рл":2965,"рм":7553,"рн":12220,"ро":46663,"рп":941,"ра":98873,"рб":2434,"рв":7403,"рг":7600,"рд":5085,"ре":68474,"рж":3084,"рз":1336,"пъ":7097,"пр":53247,"пт":2156,"пс":2576,"пу":5784,"пи":12418,"по":49161,"пл":7108,"оя":7068,"па":18259,"пе":14376,"ощ":6422,"ош":1648,"оч":6409,"оц":2874,"ос":31400,"ор":52184,"оп":12711,"оо":918,"ох":1705,"оф":5618,"оу":1691,"от":71852,"ок":20081,"ол":46187,"ом":18413,"он":40545,"ож":8402,"оз":8385,"ои":10610,"ой":13778,"ов":49786,"ог":14314,"од":35825,"ое":7078,"оа":1786,"ня":5762,"об":22311,"нъ":1330,"нц":6022,"нч":1314,"нт":20098,"нс":29338,"нф":1197,"ну":2379,"но":61747,"нн":7922,"нр":827,"нк":4229,"нз":932,"ни":106631,"не":37699,"нг":9593,"нд":12141,"мя":2035,"на":206302,"мъ":3046,"му":8961,"мс":4731,"мп":7095,"мо":14552,"мн":4561,"ге":6356,"ги":12185,"гн":1340,"го":24434,"гл":8226,"гр":23857,"гу":2534,"гъ":1359,"дв":5622,"да":30493,"вг":1251,"вд":1142,"ве":44922,"ви":31125,"вк":2396,"вл":5218,"вн":10484,"во":35579,"вр":11007,"вс":5682,"ву":1966,"вт":3431,"вш":985,"въ":12004,"га":25651,"вя":3485,"би":11046,"бе":7947,"бр":9346,"бн":1672,"бо":12574,"бл":10831,"бу":4434,"бс":2154,"бя":950,"ва":58204,"бъ":10139,"бщ":5858,"ад":29187,"ае":3072,"аж":2734,"аз":20177,"аб":6725,"ав":32263,"аг":6621,"ам":19866,"ан":92124,"ап":10532,"аи":1848,"ай":16768,"ак":18833,"ал":43351,"ах":2622,"аф":3137,"ач":7356,"ац":9711,"ас":30917,"ар":52528,"ау":3593,"ат":95702,"ая":1940,"ба":6691,"ащ":6812,"аш":2543,"зт":3682,"зс":1637,"зр":2676,"зп":8092,"зх":1144,"зу":1891,"зк":2737,"зи":16355,"зо":4852,"зн":6991,"зм":2937,"зл":3437,"ив":14438,"иг":8987,"иа":9057,"иб":2301,"иж":2121,"из":36345,"ид":10074,"ие":26794,"зъ":1412,"жо":1971,"жу":937,"жи":10091,"жк":1307,"жн":4179,"за":35424,"зб":1209,"зв":12292,"зг":1227,"зд":6172,"зе":4307,"еф":2109,"ет":45213,"ес":32066,"ер":49860,"еп":8513,"ео":5548,"ен":105327,"ем":23126,"ел":47225,"ек":18670,"ей":11615,"еи":962,"ез":20414,"еж":9844,"ее":2077,"жд":7425,"же":11581,"жа":5863,"ея":1281,"ещ":3642,"еч":4774,"еш":4262,"ех":2366,"ец":5302,"дс":7551,"др":9809,"ду":10303,"дн":14843,"дм":2251,"дп":805,"до":23013,"ди":35808,"дл":987,"дк":815,"де":31362,"дж":3808,"еб":3744,"ев":22477,"ег":6758,"ед":43311,"еа":2059,"дя":1235," th":1043,"дъ":7215," ар":2430," ас":835," ба":2316," ав":2928," ад":1436," ал":2238," ак":2281," ан":5186," ам":5117," ап":1113," бу":1045," ва":1142," бъ":8445," би":3599," бе":1826," бр":3218," бо":2578," бл":1708," вт":1082," въ":7192," ви":5408," ве":5179," во":6415," вс":1918," вр":3197," вл":1868," вк":1271," дв":3064," да":7239," го":11993," гл":1931," гр":14818," ге":2801," ев":904," ед":7019," дъ":3603," дн":1385," до":9466," др":4782," ду":4688," де":7755," ди":3107," же":950," еп":830," ел":1747," ек":959," ез":2706," зе":1022," за":22793," зв":852," жи":3991," зн":1109," иг":2469," ид":823," из":17689," ил":7883," ин":3652," им":7956," ит":1193," ис":3053," ка":12665," ки":1971," кр":5742," ко":21829," кн":1333," км":1031," кл":2272," ку":1841," ла":1472," къ":3355," ли":3157," ле":2135," ме":7169," ми":3625," ма":9333," мо":4895," мн":1906," му":6862," ни":1121," не":9125," на":119378," но":5035," ок":9717," оз":981," од":1340," об":13021," ня":1648," ощ":1554," оф":1095," от":45416," ор":4392," ос":5533," оп":3022," по":33250," пл":5058," пи":3055," пе":4542," па":4088," Ре":3648," Ра":3274," Ро":3019," Ри":2525," Пр":4415," СА":2616," Пъ":1174," Пе":2739," Па":3238," с ":12955," По":4518," Пл":3222," Пи":1267," От":1080," Ос":2127," Ор":1245," Оп":1048," Те":2595," Ти":1162," То":5392," Тр":1782," Ст":4014," Съ":4598," Та":1620," Св":1624," Си":2198," Се":4013," Сл":1251," Ск":944," Сп":1315," Ср":879," Со":2383," Ру":2131," Са":2935," Фр":2376," Фо":984," Фи":1126," Фл":885," Фе":819," Тя":1180," Ту":993," Це":1509," Хр":970," Хо":1079," Хе":1290," Ха":2307," Ша":822," Че":1435," Юж":1162," Ба":2734," Ат":830," Ар":1714," в ":45152," Ан":3448," Ам":1038," Ал":2947," Ав":1297," Ва":2965," Бъ":3733," Бу":1133," Бо":2714," г ":8207," Бр":1924," Бе":2781," Би":1492," а ":4115," Ед":825," Ев":1550," Ди":2315," Дж":3188," Де":2497," Др":1035," До":1933," Ел":962," Въ":2363," Га":1572," Ве":2397," Ви":2282," Во":966," Гъ":1180," Да":1391," Ге":2574," е ":70638," Го":1784," Гр":2287," Ис":1003," Ин":1152," Им":970," Йо":1543," Ки":1607," Ке":833," Ка":6065," и ":47012," За":3604," Ив":1368," Из":2283," Мо":2455," На":5243," Не":1867," Ни":2110," Но":2047," Об":2206," Ок":2446," Кл":1282," Ко":5424," Кр":2158," Ку":1412," Ла":1629," Ле":1707," Ли":1793," Ло":1496," н ":881," Лу":1447," Ма":9223," Ме":2502," Ми":2950," В ":2161,"Зап":1336,"Ива":1277,"II ":1339,"Кар":1717,"Кал":1241,"Кон":1216,"Кол":892," ра":14338," ре":11569," ри":1504," ро":5884," пр":44389," пс":1741," пу":1035," пъ":4915," св":5747," си":8451," се":28117," сл":4935," см":1742," ск":1388," сп":4517," ср":3911," со":1964," ру":1874," са":8475," ти":1471," те":8827," то":4709," тр":4417," сц":1244," ст":9556," су":971," та":2605," съ":15734," ус":1789," уп":1428," ун":1177," ту":1481," тя":1501," тъ":1070," фо":2263," фр":2272," фу":2793," фе":1675," фи":5383," фа":1634," уч":2779," хр":1022," хо":1370," ху":840," хи":1498," ха":1277," це":3524," чо":1060," чл":866," чи":1314," че":4520," ча":4234,"Мак":3324,"Мар":2452," ша":881," ща":6448," юж":900," юг":1007,"Южн":1054,"Нам":821,"Нас":1183,"Ник":1456,"ад ":8166,"ав ":839,"ам ":1105,"ан ":12026,"ак ":1047,"ал ":5489,"ай ":7880,"Окр":2063,"авт":1723,"ага":1670,"авя":1374,"агр":1076,"аго":1396,"ада":3119,"ади":3451,"аде":5585,"адо":1154,"адм":1400,"адн":2853,"аем":863,"би ":1015,"ажд":973,"Опе":874,"аба":833,"або":2518,"ава":11168,"авн":2861,"авл":2542,"аво":1409,"аве":2231,"ави":4818,"алс":906,"алн":8729,"ало":2628,"алб":1650,"ала":2998,"алк":1807,"али":10040,"але":5565,"амо":2012,"амп":965,"ама":2784,"ами":3947,"аме":7196,"анн":912,"ано":4066,"анс":13953,"ант":4813,"анц":1702,"ана":8635,"анд":5902,"анг":4584,"ани":19618,"ане":11702,"анк":1502,"азр":1036,"азп":4124,"азо":828,"азн":1081,"азл":1632,"ази":2620,"азв":2060,"аза":1484,"айс":963,"айк":1056,"айо":1501,"айн":2031,"акт":4848,"ако":1531,"акс":811,"аке":4784,"аки":1127,"ака":2207,"Пар":949,"ас ":2329,"ар ":3643,"ат ":10948,"Пло":2474,"ба ":1349,"ая ":1362,"ащ ":1410,"Пет":1186,"Пър":1058,"САЩ":2595,"Пол":1310,"При":1038,"Пре":1768,"Про":1122,"Рим":1524,"АЩ ":2589,"Род":1271,"Раз":1500,"Реп":2237,"Але":1070,"Сан":921,"Ант":866,"Рус":1292,"Соф":1291,"Сев":1817,"Све":1167,"Бра":846,"Съе":2317,"Тя ":1123,"Ста":1292,"Сто":900,"Бел":1301,"Тов":901,"Той":2267,"Вел":1059,"Бъл":3319,"Гра":1071,"Вът":1318,"Фра":1531,"Гер":981,"Гео":1043,"Дим":1088,"Хри":815,"Гър":1058,"Джо":1453,"Цен":1121,"Евр":1146,"лбу":1567,"лам":805,"лан":5659,"лас":5904,"лат":3266,"лги":963,"ме ":3219,"лга":11038,"ма ":9888,"ля ":1353,"лав":4235,"лаг":1602,"лад":2109,"къс":900,"към":1928,"кус":995,"кул":1639,"кци":1405,"коя":2700,"кре":815,"кра":5743,"кри":2482,"кръ":8385,"кса":1220,"кси":1227,"кте":804,"кти":2426,"кто":3328,"ктр":1676,"кла":1490,"ло ":7485,"кло":1049,"клю":1993,"кни":998,"ког":1332,"ков":4810,"ком":4727,"кон":4363,"коп":1296,"кор":1875,"кос":1575,"кот":3860,"кое":2107,"кои":3390,"кой":3594,"кол":6405,"кин":1233,"кия":7501,"лм ":1747,"кит":4721,"ле ":963,"кед":4374,"ли ":17289,"ква":2572,"кат":20763,"кар":2222,"кан":8113,"как":1566,"кал":2533,"каз":1398,"ла ":6130,"йто":3630,"йск":8916,"йст":3543,"кт ":1337,"йна":2642,"йно":1113,"йни":1595,"кс ":1165,"йон":1165,"ко ":8475,"км ":856,"ки ":42277,"йво":1025,"ият":14767,"од ":4250,"нац":1700,"нау":1528,"нач":4474,"ог ":1478,"нан":1145,"нам":2383,"нал":6581,"мят":820,"нат":17022,"нас":8028,"нар":6159,"нап":1914,"над":1993,"наг":991,"най":5108,"наз":881,"нде":899,"нда":1986,"нгл":4272,"нга":993,"нем":1875,"нен":7192,"ои ":1291,"нер":3755,"нес":1561,"нет":4998,"нег":1689,"нев":1009,"нди":2598,"ндо":877,"ндс":1234,"ндр":1171,"нив":1213,"низ":3481,"ник":6697,"ниг":977,"ние":16641,"ок ":2212,"ой ":4351,"ня ":922,"ов ":8975,"нав":1281,"нт ":3764,"мпи":1340,"мпе":2702,"мпа":913,"мот":885,"мск":3992,"мун":847,"муз":1721,"мик":1065,"мил":1270,"мич":1772,"мин":5435,"мис":1421,"мир":3990,"мит":3385,"мия":2187,"но ":26769,"мна":831,"мно":2749,"мод":977,"мов":951,"мож":1224,"мон":1361,"мол":819,"мос":1285,"мор":2165,"нс ":993,"нд ":1174,"мац":839,"мал":2220,"мак":2180,"май":1485,"лят":839,"мат":5742,"мас":872,"мар":1751,"ляр":941,"нг ":1106,"ман":6016,"лян":866,"лям":2082,"люц":3280,"люч":1984,"ляв":2485,"маг":843,"мес":1879,"мет":5373,"мен":8396,"ни ":29732,"мер":7582,"мей":1790,"меж":2783,"мед":1422,"мвр":3060,"не ":9731,"на ":142528,"лощ":4100,"му ":4914,"лни":5607,"лно":5548,"лна":4870,"лог":4324,"лож":4131,"лор":940,"лос":1248,"лот":2604,"лом":832,"лон":1406,"лов":4045,"луч":1122,"лст":1405,"лск":2670,"лта":884,"лзв":2068,"лиа":1575,"лиг":889,"лив":1397,"лиз":3791,"лим":1095,"лий":4336,"лик":5518,"леж":946,"лев":1784,"лед":5662,"лер":878,"ми ":2984,"лен":16713,"лем":2731,"лек":3978,"лет":2054,"мо ":1651,"лищ":1562,"лиц":2347,"лич":3301,"лис":3220,"лит":6583,"лиф":806,"лин":3617,"лия":3676,"лко":1751,"лка":1568,"оят":3638,"пат":1787,"ояв":887,"пад":4016,"рг ":1045,"оян":1222,"пан":2587,"пар":3257,"рд ":1208,"ре ":1433,"ра ":12349,"пит":1282,"пис":5195,"пла":2637,"пле":1081,"ро ":1581,"пло":2643,"ри ":13979,"пер":6644,"пет":985,"пей":864,"пен":871,"пец":914,"рк ":1201,"ори":9957,"орд":1685,"оре":5253,"орг":4291,"орс":1471,"оро":2299,"орм":3070,"орн":2306,"опу":959,"ора":5790,"опе":1825,"опи":2343,"опо":1901,"опр":1742,"опа":1705,"оте":1711,"отк":1219,"отл":842,"оти":2542,"ото":15284,"отн":1748,"отр":1056,"отв":1153,"отб":1366,"ота":1480,"осе":1009,"оси":1734,"осл":2267,"осм":1054,"осн":3443,"осо":2118,"ост":14287,"орт":2090,"оръ":1155,"осв":865,"оми":2906,"оме":3032,"оля":2830,"ома":3616,"олю":3412,"олу":2069,"олс":1164,"олн":1271,"по ":9531,"оло":11644,"олк":1549,"оле":5004,"оли":8455,"олз":2123,"ола":2641,"окр":6747,"окт":1051,"око":5161,"оня":990,"онс":3193,"онт":1752,"они":6360,"оно":4114,"онн":3311,"она":6297,"онд":803,"оне":3282,"омо":1980,"омп":2499,"оше":916,"очи":922,"очн":3093,"още":1644,"ощт":2295,"офе":923,"офи":2989,"оце":1219,"оци":1428,"няк":1962,"няв":1202,"ова":9654,"общ":5055,"обр":3456,"обо":1542,"обн":833,"обл":3221,"оби":2313,"обе":2463,"па ":3482,"оит":3156,"ойв":1029,"ойт":3479,"ойс":1115,"ойн":2834,"ока":2144,"ожн":1229,"ози":2118,"оза":2042,"озн":2520,"оиз":4705,"одн":3018,"оди":10451,"оду":932,"одр":1677,"одс":1049,"одо":2573,"оем":891,"оен":1806,"оет":2833,"оже":5313,"ове":9903,"ови":7840,"ово":5297,"овн":3952,"овс":1386,"ога":2254,"оги":3492,"ого":3019,"огр":2846,"ода":3875,"оде":5129,"от ":40884,"нот":4382,"нос":7389,"нош":880,"нор":1012,"ос ":1874,"ном":2595,"ное":844,"ног":2289,"нов":10487,"ор ":8928,"нно":1572,"нни":2936,"нна":3374,"нко":832,"он ":7439,"нка":817,"ом ":1452,"ния":18624,"нир":1151,"нис":3663,"нит":11345,"ним":1508,"нин":2416,"ол ":1852,"нич":2172,"ниц":3898,"нце":866,"нци":4501,"ощ ":1985,"нтъ":2337,"нуа":804,"нта":3339,"нте":1868,"нти":4317,"нто":1031,"нтр":2358,"нск":22095,"нст":3816,"сам":1827,"сан":2444,"сат":1871,"сва":1135,"сво":1889,"те ":30876,"све":3239,"свъ":838,"сев":2490,"сед":1178,"сел":9974,"сек":1165,"сеп":870,"ти ":11022,"сен":2491,"сем":2208,"сет":920,"сер":1205,"сия":1303,"сич":1328,"сис":1833,"сит":2033,"сих":1495,"син":1858,"сил":2220,"сим":1511,"ски":43899,"ска":20500,"сле":5451,"сла":2258,"ско":7743,"сми":843,"слу":1333,"то ":53757,"сло":1460,"сна":1239,"сни":1224,"соб":1154,"сов":1181,"сок":1395,"сно":4759,"спе":2031,"спа":1513,"спи":822,"соф":1092,"соц":1039,"сре":3295,"спо":2745,"роц":854,"рот":1758,"роф":1212,"роп":2362,"рос":2878,"ст ":15103,"роя":1120,"рта":1478,"рти":2728,"рск":11358,"рси":1583,"руг":2898,"руп":3391,"рус":2190,"рум":975,"рци":1577,"рхи":885,"ръц":1917,"ръс":815,"ръг":7752,"ръж":849,"ръб":829,"та ":68838,"рад":11688,"рае":904,"раж":1539,"раз":12192,"раб":3055,"рав":5786,"рам":2735,"ран":13900,"рай":2815,"рак":2465,"рал":6906,"раф":2143,"рац":1472,"рас":2637,"рат":10837,"рая":1251,"ращ":1083,"рби":1070,"рва":2274,"рди":900,"рдж":839,"реб":1455,"рев":6384,"рег":1970,"ред":14324,"рет":1976,"рес":2290,"реп":1732,"си ":5183,"рен":6538,"рем":3987,"рел":1209,"рек":2832,"рей":892,"рез":11468,"реж":2438,"ржа":2590,"рещ":1470,"реч":1092,"реш":2153,"се ":16807,"рво":979,"рве":1684,"рви":2226,"рга":3761,"рги":1489,"рда":805,"рия":9828,"рио":1833,"рим":2306,"рин":4333,"рик":7682,"рил":2477,"рий":2243,"рич":4087,"рит":6765,"рир":851,"рис":4939,"рка":815,"риа":2344,"риг":1450,"рив":950,"рие":1786,"рид":1515,"риз":1180,"рни":3773,"рна":4116,"рок":1893,"рол":1507,"ром":2389,"рон":2340,"роз":1160,"рои":4466,"рой":1470,"ров":7381,"рог":1151,"род":7855,"рое":938,"рно":2617,"рла":1189,"рми":2339,"рма":3665,"пра":5058,"при":9598,"пре":21619,"про":14907,"поп":1206,"пор":4301,"пос":3943,"пот":1094,"поч":1394,"рт ":2255,"пое":904,"под":4233,"пов":2082,"пон":1889,"пом":919,"пол":13153,"пок":1115,"поз":1787,"пуб":3689,"пул":1018,"пте":935,"пси":1481,"са ":8477,"пър":3460,"път":1769,"пъл":1566,"вар":1920,"ват":9299,"ващ":3095,"ван":13950,"вал":2194,"га ":4136,"бщи":2449,"бъд":803,"бър":1419,"бъл":7641,"бща":1581,"бще":1011,"бум":1554,"бук":823,"бск":1045,"вто":2970,"вси":896,"вск":2447,"вст":1194,"вре":3504,"ври":3322,"вро":1874,"връ":1065,"вол":4504,"вой":3658,"вое":1398,"вод":4759,"вот":3890,"вор":2694,"вни":3836,"вна":3375,"вно":2879,"вля":1602,"вле":1219,"вли":1041,"вла":1152,"го ":2882,"вкл":1196,"виц":847,"вич":1491,"вия":2762,"виз":1513,"виж":1141,"вил":1146,"вин":3278,"вис":2864,"вит":3903,"вид":3578,"вие":1422,"веч":1347,"вес":4369,"вет":7499,"вер":5871,"вен":9985,"ги ":2787,"вел":1221,"век":3636,"веж":1619,"вед":2219,"вгу":856,"ва ":24595,"бан":1118,"ачи":1132,"аши":924,"аща":2195,"ащи":2202,"ащо":823,"афс":811,"афи":915,"ача":3148,"аче":1418,"аци":9279,"апр":1718,"апа":4649,"апо":1282,"апи":1535,"арх":1193,"арс":9548,"арт":4677,"аса":1041,"аре":2062,"ард":1683,"ара":4896,"арн":1173,"арм":1084,"аро":3343,"ари":12961,"арл":1170,"арк":1896,"аст":14401,"ася":976,"ата":47391,"аси":2434,"асе":6138,"асо":1029,"ату":1909,"ате":8423,"ати":9387,"ато":12657,"атр":1084,"бол":3321,"бор":2442,"бот":2082,"бро":1543,"бри":1900,"бре":1262,"бра":3810,"бла":3919,"бли":5668,"во ":9926,"ви ":4285,"бен":998,"бер":1182,"без":1026,"бед":1013,"бел":1391,"бек":854,"бит":1770,"бил":1742,"бик":912,"бив":1415,"ве ":3400,"дан":2923,"дар":1829,"дат":3453,"дви":1371,"два":2719,"две":833,"ед ":5333,"дал":1095,"дад":3266,"дав":2214,"ев ":3676,"дее":1168,"дек":992,"дей":1639,"дем":1659,"дел":3837,"ден":12663,"дер":1762,"джи":1277,"ей ":1537,"дес":1272,"дет":1508,"ез ":10672,"дст":4125,"дск":2604,"дро":826,"дру":2364,"дре":1806,"дри":2347,"дра":1827,"ет ":4265,"душ":3708,"ец ":2918,"ен ":33180,"дия":1914,"диц":1640,"ем ":1311,"див":1005,"дим":1128,"дин":15737,"дио":936,"ел ":8445,"дис":974,"дит":1804,"дие":842,"ек ":3487,"доб":1818,"дов":2389,"ес ":1992,"дос":1220,"дор":1353,"док":903,"дон":5500,"дна":5206,"дни":3855,"дне":986,"ер ":6034,"дно":4679,"дми":1636,"да ":13229,"вяв":1024,"гал":815,"вят":853,"гат":2134,"ган":4672,"гар":12015,"де ":1425,"and":809,"an ":963,"във":2376,"въз":2174,"вър":4758,"гол":3089,"гор":2288,"гов":3764,"год":7938,"гру":3152,"ду ":2227,"гръ":1949,"гра":16257,"гри":1115,"гус":909,"ген":2209,"гер":1710,"ди ":4057,"гио":1272,"гия":3508,"гич":1145,"ati":807,"гле":897,"гла":2323,"до ":6155,"гли":4051,"жан":924,"жав":2301,"за ":15694,"еще":814,"жит":2528,"жив":1957,"жис":1541,"жес":1221,"жду":3122,"зи ":3101,"жен":7044,"жда":3366,"жно":1097,"жни":1433,"жна":1550,"ея ":816,"жа ":1065,"дъл":1733,"дър":3663,"еец":1133,"ежи":2367,"ежд":5041,"едс":4153,"жи ":1063,"еза":1907,"езо":914,"ези":4043,"ева":1249,"еви":2758,"еве":5028,"его":2858,"едв":1395,"еда":2616,"еде":4183,"еди":11058,"едо":5545,"едн":6376,"евн":1170,"ево":4583,"же ":1283,"евр":1969,"ега":870,"еги":1678,"ент":10211,"енс":4280,"енц":1053,"ени":27128,"ено":7268,"енн":2947,"ена":10075,"ене":4173,"енд":1760,"еор":1928,"епт":1100,"епу":3069,"епо":1006,"ерс":2306,"ерт":1057,"ерм":2802,"ерн":4552,"еро":3866,"ери":14051,"ерг":986,"ере":1937,"ера":7049,"ерв":1379,"ейн":1799,"ейс":4529,"еке":826,"еки":1118,"еко":1156,"ект":4178,"екс":3530,"ека":2557,"елн":4078,"ели":9768,"елс":1847,"ело":4047,"еле":11585,"елг":908,"ела":1732,"емо":966,"еми":3799,"емс":1747,"еме":6592,"еля":2163,"ема":3994,"емв":2337,"ехн":1011,"еци":1425,"ече":2969,"ешн":2173,"еща":1146,"еса":875,"есе":2292,"еси":1554,"еск":7153,"есн":1000,"ест":15331,"ета":4044,"ети":4225,"ете":3470,"етр":1876,"ето":20630,"етн":1476,"етс":1612,"етъ":1538,"иве":2646,"иви":1509,"ива":2727,"иал":3813,"иан":3297,"иен":1203,"иет":8077,"иже":812,"иев":1368,"игр":2806,"иго":824,"ида":1273,"иди":1105,"иде":2406,"иво":2091,"ивн":2071,"ивш":976,"ига":2136,"иги":1169,"икн":877,"ико":3994,"йн ":1098,"ики":1147,"ика":16650,"ийс":6452,"иит":965,"изъ":865,"изх":1032,"изс":1113,"изт":2743,"изп":3253,"изм":1954,"изл":1205,"изо":1731,"изн":1188,"изи":3858,"изк":1474,"изд":1669,"иза":5067,"изв":5752,"ион":10338,"инц":2399,"иня":874,"иод":1232,"ине":4216,"ини":7734,"ино":2511,"инс":5959,"инт":1527,"ина":17378,"инд":1369,"инг":2575,"ими":3098,"име":5126,"имс":2034,"имо":1892,"имп":3052,"имн":924,"има":3880,"или":12587,"иле":1355,"илм":2481,"илн":880,"ило":3073,"ила":2122,"иси":1489,"иса":3563,"ист":15576,"исп":938,"исо":1673,"исл":1753,"иск":1625,"ити":5468,"ите":38859,"ита":6132,"исъ":1087,"ись":1263,"иту":932,"ито":6371,"итн":995,"ира":10008,"ири":1911,"иро":1628,"ихо":1462,"ице":1072,"ица":5886,"ици":6007,"ифо":843,"ище":1958,"ичи":1220,"ичк":1339,"ичн":6798,"ича":3197,"иче":8842,"ка ":28086,"ив ":1131,"зав":1648,"зае":830,"ид ":2167,"зви":1188,"зве":5171,"зва":3840,"зац":2538,"зат":1310,"зар":1064,"зап":3505,"зан":2665,"защ":954,"зда":4154,"зво":872,"ие ":14021,"ий ":4437,"ии ":2933,"зем":1420,"зик":4455,"зир":1742,"ил ":4244,"ик ":10314,"ин ":8393,"им ":2180,"зия":1529,"зит":1037,"ип ":824,"зма":848,"ио ":855,"зли":2352,"зна":4787,"зни":1000,"зно":822,"ир ":1662,"зов":1006,"ис ":2255,"зон":1141,"зпо":5089,"ит ":1689,"зпр":1401,"зра":1726,"зпъ":815,"зсл":944,"зто":3205,"ич ":1609,"зхо":1090,"ия ":57202,"ius":2045,"is ":934,"ion":1052,"ьор":2769,"km ":885,"южн":872,"km²":3379,"ъцк":1859,"ътр":1736,"ъще":1730,"ъщо":1266,"he ":1146,"ъде":1407,"ъед":2623,"ъве":1335,"ъзд":2638,"ълн":1684,"ълг":11342,"ълж":1010,"ърн":1183,"ърк":1004,"ърж":2771,"ърз":1179,"ърх":1303,"ърт":1540,"ърс":856,"ърш":1020,"ърц":1189,"ъст":3892,"ъпр":1168,"ърв":4886,"ърд":852,"ia ":930,"er ":1669,"es ":1352,"яко":2322,"яло":937,"ява":6515,"юци":3278,"ючв":950,"ят ":9602,"яне":821,"яни":1186,"яма":1227,"ята":8869,"ято":2957,"уци":861,"уча":2759,"учи":956,"уче":1377,"уши":3537,"фес":938,"фер":834,"фан":1085,"фин":933,"физ":1170,"фил":3622,"фия":1796,"фиц":1324,"фут":2220,"фре":1952,"фор":3324,"фон":806,"ца ":4811,"ци ":3879,"хан":805,"хар":1263,"хол":1095,"хор":974,"хов":933,"ход":1906,"хим":863,"стн":3691,"сто":10615,"стр":11666,"ств":16841,"сте":8055,"сти":11129,"ста":11874,"стъ":2246,"сту":903,"стт":2922,"сце":1326,"съд":995,"съв":1667,"съз":2361,"със":4442,"сък":852,"сън":863,"същ":2787,"сьо":1331,"тав":5484,"так":1485,"тал":3945,"тан":6731,"тай":879,"тат":3225,"тар":3238,"тбо":3512,"твъ":860,"тво":9227,"тви":1621,"тве":3786,"тва":3607,"тех":952,"тем":3541,"тел":16601,"тео":1008,"тен":6993,"тер":6816,"тет":1731,"тес":853,"тез":1494,"тек":1131,"тив":4139,"тие":1898,"тка":1702,"ум ":1519,"тич":5203,"тиц":909,"тия":3555,"тин":3775,"тик":4630,"тил":1307,"тир":2210,"тис":902,"тит":2981,"тла":1006,"тно":3117,"ток":1533,"тол":2748,"той":1268,"тов":5201,"тни":5121,"тна":2467,"тре":4236,"тра":10350,"три":4952,"тор":11467,"тот":821,"том":2089,"тон":2179,"ус ":1180,"топ":1014,"точ":3383,"тоя":1817,"тта":3294,"тро":5709,"тру":1929,"тск":2626,"тур":4751,"тър":5898,"тът":942,"тън":1062,"ува":2742,"уги":1696,"уга":1360,"уар":1716,"убл":3878,"узи":1937,"уди":922,"удо":1145,"уме":1194,"ума":1453,"улт":1584,"ули":1004,"ула":1122,"укт":810,"ука":1012,"упр":1211,"ура":2417,"ург":1080,"ури":1177,"упа":3307,"уна":1368,"уни":1783,"уст":2872,"утб":2290,"урс":982,"урн":1905,"уск":2598,"уси":1038,"що ":2387,"шни":1037,"шна":1940,"шин":804,"ще ":3484,"шен":1940,"щи ":1591,"ъв ":2817,"ът ":6352,"ъс ":1987,"ър ":7548,"ън ":3472,"ъм ":3109,"щит":1320,"щин":2379,"ъл ":2252,"ък ":1412,"щен":865,"щес":2508,"ъг ":7573,"щат":7215,"m² ":3381,"on ":1530,"щта":2294,"че ":2916,"цен":4575,"чи ":992,"цел":1258,"цес":947,"цер":828,"циа":2597,"ции":1846,"цио":6596,"цит":1550,"ция":11555,"ча ":1350,"цар":1123,"цат":1708,"цки":1353,"чев":1376,"чен":5933,"чес":8800,"чер":977,"ши ":3970,"чет":1965,"чле":872,"чки":1198,"чин":1845,"чит":1626,"ша ":1093,"чва":2149,"час":5618,"чал":1786,"чан":1529,"чав":2550,"ща ":4581,"чре":802,"чна":4150,"чов":1436,"чни":4370,"чно":3174,"us ":4874,"ter":901},"n_words":[7994134,9177756,6462334],"name":"bg","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/bn.json b/contrib/languages-data/bn.json
new file mode 100644
index 0000000..45b1298
--- /dev/null
+++ b/contrib/languages-data/bn.json
@@ -0,0 +1 @@
+{"freq":{"D":455,"E":376,"F":388,"G":399,"A":894,"B":684,"C":848,"L":430,"M":683,"N":429,"O":309,"H":360,"I":507,"J":241,"K":244,"T":598,"W":240,"V":199,"P":651,"S":947,"R":531,"f":1089,"g":1979,"d":2581,"e":8201,"b":1267,"c":2589,"a":8669,"n":6047,"o":5561,"l":3930,"m":2503,"j":208,"k":699,"h":3040,"i":6685,"w":672,"v":711,"u":2926,"t":5635,"s":4238,"r":5985,"p":1752,"z":245,"y":1310,"x":420,"এলা":514,"ক। ":614,"ঐতি":234," l":233," m":235," o":526," i":296," d":312," f":245," a":616," b":228," c":315," t":765," p":493," s":393," r":292," J":235," K":227," H":321," I":408," N":324," O":231," L":375," M":610," B":602," C":719," A":760," F":332," G":372," D":404," E":296," S":771," R":448," P":577," W":226," T":506,"ا":336,"Co":219,"উ":9536,"এ":27004,"ঃ":514,"ং":15654,"ঁ":2141,"ই":18078,"আ":10399,"অ":13627,"চ":16928,"ছ":9717,"ঘ":1893,"ঙ":3088,"ঞ":3794,"ট":30397,"জ":27267,"ঝ":790,"ও":9899,"ঐ":344,"খ":8927,"গ":20205,"ঔ":204,"ক":78264,"ন":86702,"ফ":6705,"প":46274,"ভ":18017,"ব":70605,"য":69717,"ম":48748,"ড":11071,"ঠ":3258,"ণ":10122,"ঢ":578,"থ":13519,"ত":69776,"ধ":12543,"দ":33053,"স":56165,"হ":25168,"়":39420,"া":175719,"ি":114763,"র":156970,"ল":52543,"Ma":241,"শ":24901,"ষ":17272,"ৎ":1686,"্":145506,"ৌ":1675,"ো":19195,"ৈ":1879,"ে":113569,"ৃ":4705,"ূ":5615,"ু":27604,"ী":22483,"৭":2295,"৬":2181,"৯":5541,"৮":2706,"৩":2127,"২":3895,"৫":2215,"৪":1978,"১":8858,"০":4887,"৷":254,"In":199,"।":25409,"Th":253,"b ":285,"a ":1054,"i ":350,"ge":332,"ga":252,"he":773,"ha":472,"gh":243,"g ":316,"ea":343,"ec":205,"ed":350,"de":506,"di":332,"এছা":284,"h ":358,"el":402,"en":818,"em":278,"et":317,"es":737,"er":1291,"ca":315,"e ":2258,"be":225,"da":210,"f ":445,"ct":266,"co":263,"ci":201,"ch":362,"ce":347,"c ":220,"এটি":2063,"d ":875,"at":923,"as":401,"ar":1026,"al":912,"ai":231,"am":360,"an":1536,"ac":322,"ad":251,"ag":317," ১":7243,"nt":573,"ns":267,"no":211," ৯":250," ৮":298," ৭":311," ৬":364," ৫":417," ৪":387," ৩":666," ২":2429,"of":407,"om":400,"on":1111,"ol":393,"ot":205,"os":230,"ou":252,"or":795,"r ":1035,"pe":220,"lo":359," ঢ":514," ড":1933,"ll":307," ঠ":230," ধ":2352," দ":9809," থ":3157," ত":9984," ফ":3863," প":28296," ন":11037," য":6674," ম":15133," ভ":9746," ব":28530," ল":4344," র":8416,"o ":382," শ":7392,"ma":464,"mb":342," স":27697,"me":396," হ":13181,"mi":232,"একট":5136,"একজ":1493,"na":524,"একক":204,"nc":255,"nd":698,"ne":429,"ng":599,"ni":414,"একা":295," ।":1901,"m ":412," এ":26382," উ":6430,"এখা":461," অ":13571," আ":10126," ই":6407,"li":503," জ":9765," ট":1754,"le":571," ঘ":968," ছ":3198," চ":4584," ক":22576,"la":589," ঔ":203," গ":7129," খ":2591," ঐ":343," ও":5820,"n ":1470,"ht":261,"hu":305,"hi":274,"ho":238,"id":213,"ic":689,"ia":484,"ig":326,"ie":307,"ir":212,"is":631,"it":565,"il":352,"in":1040,"io":639,"l ":743,"এবং":4187,"এপ্":233,"y ":803,"ve":326,"x ":253,"ul":217,"ur":460,"us":340,"ut":207,"um":443,"un":241,"to":352,"tr":334,"te":708,"ti":997,"th":909,"ta":488,"ss":271,"st":614,"se":325,"এমন":239,"si":361,"rt":257,"ry":236,"ro":438,"ri":872,"re":667,"rd":212,"ra":839,"t ":920,"s ":1577,"এদে":277,"এই ":3233," �":274,"এক ":911,"এর ":2530,"উনি":291,"উন্":360,"উদ্":622,"উপা":289,"উপর":460,"উপন":490,"উরো":279,"উল্":354,"উৎস":239,"উৎপ":278,"উৎ":563,"উর":543,"উপ":2213,"উস":347,"উল":530,"উক":210,"উজ":216,"উচ":390,"উট":503,"উত":973,"উন":827,"উদ":753,"ইন":1626,"ইব":205,"ইম":223,"ইয":591,"ইর":537,"ইল":683,"ইস":741,"উই":375,"এছ":287,"এখ":589,"এক":9005,"এম":465,"এব":4214,"এপ":275,"এন":320,"এদ":306,"এট":2154,"এই":3366,"ছর ":262,"ংশ":1413,"ংস":1069,"ংল":2600,"ংর":2237,"ংব":422,"ংঘ":236,"ংগ":523,"ংখ":753,"ংক":757,"ঁর":727,"ৎ ":620,"ইত":723,"ইট":668,"ইড":241,"ইজ":233,"আস":454,"আল":1088,"ইক":452,"আয":323,"অ্":840,"আম":516,"আর":1419,"আফ":296,"আব":708,"আধ":289,"আদ":422,"আন":1004,"ইউ":806,"ইং":2317,"আছ":551,"অস":568,"আগ":507,"আক":520,"অল":207,"অর":1154,"অভ":838,"অব":2125,"অফ":360,"অপ":433,"অন":3326,"অধ":754,"আই":743,"অথ":198,"অঞ":597,"অত":316,"অক":381,"অঙ":287,"অং":592,"ঙা":410,"চট":225,"চত":213,"চন":791,"ঙ্":2470,"৩ ":950,"চর":577,"চল":2320,"চী":679,"চু":374,"চা":2514,"চি":4076,"চে":985,"চ্":2421,"ছব":324,"৪ ":965,"ছর":393,"জগ":223,"ঘট":513,"গা":2158,"গস":249,"গি":590,"গী":642,"গু":2497,"গো":907,"গ্":3546,"গে":1783,"১ ":1256,"চক":208,"২ ":1015,"ঝা":497,"গোল":259,"৭ ":899,"টক":320,"টপ":202,"টন":552,"ঞ্":2042,"টব":433,"ঞা":1608,"টা":3413,"৮ ":899,"টর":259,"জন":5856,"ছে":4099,"জধ":330,"ছি":2738,"ছু":520,"গ্র":2862,"ছা":810,"গ্য":399,"জর":278,"৫ ":1051,"জল":300,"ছো":231,"জম":242,"জয":281,"জো":267,"জে":2691,"জু":830,"জী":975,"জি":3650,"জা":4328,"৬ ":1035,"জ্":3683,"গের":766,"ওয":1866,"ক।":744,"ঐত":235,"এশ":223,"এল":685,"এর":2874,"এস":513,"ক্":11379,"খন":635,"কে":8760,"কো":2299,"কী":339,"কি":3416,"কা":12553,"কৃ":1298,"কু":1325,"কস":263,"গর":1257,"০ ":1923,"গল":479,"খে":1085,"গন":254,"গব":429,"খ্":2313,"গঠ":636,"খি":334,"খা":2637,"খু":253,"গড":267,"গত":723,"গণ":853,"কল":1752,"খক":249,"কভ":207,"কম":1122,"কয":216,"কর":9263,"কব":584,"কথ":578,"কদ":234,"কন":446,"কত":422,"কট":5691,"কজ":1571,"কক":371,"পক":777,"নো":1529,"নৈ":281,"ন্":18037,"পথ":251,"পত":1140,"পন":1278,"পদ":1221,"পড":271,"পট":230,"পঞ":305,"নর":378,"৷ ":217,"নন":368,"ধে":496,"নপ":524,"নব":764,"নভ":385,"ধ্":3005,"নম":449,"নয":630,"নু":2067,"নী":3272,"নে":8228,"নস":741,"নি":10383,"না":9095,"বচ":430,"বছ":386,"বঙ":528,"বক":640,"ফল":533,"ফর":691,"পৌ":262,"প্":14673,"পো":493,"বপ":409,"বন":1731,"ফে":647,"বদ":516,"বত":795,"ম।":432,"ফু":801,"বড":289,"ফি":694,"ফা":803,"পশ":1040,"পস":313,"পল":236,"পম":204,"পর":6275,"পে":1955,"পৃ":469,"পূ":1624,"বই":313,"পি":1741,"পা":6024,"পু":2560,"পী":458,"বং":4443,"ভর":238,"ব্":5813,"মক":698,"মগ":375,"ভি":3000,"ভা":10166,"মত":941,"র।":1267,"ভু":429,"ভূ":887,"মণ":247,"ভে":813,"মন":1866,"মদ":356,"মধ":1657,"মব":565,"ভো":225,"মপ":208,"বব":837,"ফো":381,"বয":356,"ফ্":917,"বর":3916,"বল":2788,"বশ":202,"বহ":1569,"বস":2742,"বা":13175,"বি":13020,"বী":1634,"বু":748,"য।":373,"বৃ":690,"বে":6807,"বৈ":512,"বো":1000,"ভব":322,"রগ":376,"রক":3370,"রও":363,"রট":549,"যা":13478,"য়":35819,"রজ":517,"রচ":1226,"যস":235,"রঙ":226,"রদ":1297,"রধ":882,"রত":4811,"রথ":1065,"ল।":866,"যু":2179,"রণ":2957,"যি":521,"রম":1030,"রয":914,"যো":1322,"রব":1734,"যৌ":231,"রভ":428,"রপ":736,"রন":1614,"যে":3967,"মল":265,"যক":743,"ময":969,"ভ্":389,"মর":648,"মহ":1061,"মস":593,"মূ":1527,"মৃ":459,"যত":845,"যদ":240,"ঙাল":366,"মা":10507,"মি":3934,"মী":801,"মু":1962,"মো":1026,"যব":1634,"মৌ":259,"যম":725,"ম্":4843,"যন":902,"মে":4592,"ড়":3388,"ডা":1047,"ডি":1739,"ঠি":877,"ঠা":1161,"৯ ":835,"টে":2184,"ঠন":362,"ট্":2862,"টো":693,"টু":322,"টি":14501,"ণী":616,"ণি":777,"দ।":351,"ণা":1071,"তক":615,"ণত":463,"ঢা":467,"ড্":286,"ডো":228,"ডে":1416,"ত।":1976,"থা":4072,"থি":1849,"ন।":3725,"থন":275,"তে":6636,"তৈ":476,"থব":514,"তো":511,"ত্":11770,"থম":1047,"দক":1036,"তা":9664,"তি":11706,"তৃ":563,"তী":2327,"তু":1609,"তথ":334,"তত":465,"তন":1191,"ণে":1383,"ণ্":713,"তম":1948,"তব":730,"তল":215,"তর":2978,"নট":466,"ধা":3367,"নত":805,"নদ":832,"ধি":1235,"ধী":706,"ধু":525,"দো":593,"দ্":6813,"ধন":270,"দে":7678,"দৈ":238,"নক":807,"নগ":632,"ধর":1424,"নও":200,"দস":274,"দশ":433,"দূ":217,"ধত":234,"দৃ":215,"দী":1553,"দু":1592,"দা":3394,"দি":3189,"থে":3273,"দন":421,"ছে ":1536,"জন ":2036,"দল":773,"দর":900,"থ্":402,"হন":525,"সে":5003,"সী":651,"সু":1284,"সূ":479,"হণ":341,"হত":965,"সৃ":307,"সা":9422,"সি":3847,"সহ":471,"হচ":543,"হল":1575,"স্":12917,"হম":242,"হর":1722,"হয":5736,"সো":556,"হে":854,"হৃ":361,"়।":3879,"হু":683,"হি":3582,"হী":535,"হা":4314,"হ্":705,"হো":256,"়ক":477,"়ত":462,"়ন":1084,"া।":1709,"়ম":211,"়র":208,"ি।":908,"়ু":308,"়ী":503,"াং":3360,"ছু ":408,"াঁ":1482,"়ি":1178,"়া":8455,"়ো":436,"াই":3512,"়ে":7915,"াউ":581,"াও":962,"াক":5532,"়্":205,"াচ":1101,"াছ":570,"াজ":4489,"াখ":950,"াগ":2341,"াঙ":862,"াথ":1203,"াত":6156,"াণ":1382,"ী।":871,"াড":1549,"িং":1140,"াঠ":366,"াট":1770,"াঞ":296,"াভ":908,"াব":4643,"াফ":353,"াপ":2739,"ান":18665,"িউ":1255,"াধ":2334,"াদ":5045,"িক":11087,"িখ":922,"াল":10227,"িও":631,"ার":30756,"িএ":231,"াম":7273,"ায":8979,"িছ":516,"িজ":2552,"াস":5883,"িচ":1754,"াহ":2942,"িগ":546,"াশ":1723,"াষ":6041,"িট":1865,"িড":639,"িত":12239,"িণ":858,"িদ":2453,"িন":8978,"িধ":463,"িপ":1269,"িব":2698,"িফ":329,"িম":2790,"িভ":1825,"াৎ":271,"িয":6762,"ির":6023,"িল":5504,"ীক":817,"িশ":3794,"ীগ":209,"িষ":2506,"িস":4278,"িহ":941,"ুই":664,"ীদ":382,"ীত":1782,"ীপ":505,"ীন":1798,"ীয":4089,"িৎ":317,"ীম":347,"ীব":842,"ীল":338,"ীর":2436,"ুগ":541,"ুক":2294,"ুখ":360,"ুজ":362,"ুট":930,"র্":19499,"রো":2284,"রে":15121,"রূ":464,"রু":2561,"রী":3578,"রা":17630,"রি":11429,"রস":1549,"রহ":932,"রশ":407,"রল":454,"লয":931,"লম":504,"লব":520,"লন":1184,"লত":1046,"লট":257,"লচ":904,"লগ":240,"লক":1697,"ল্":3376,"লো":3053,"লে":9966,"লী":1486,"লু":671,"লা":9534,"লি":6096,"লস":278,"শক":649,"শব":592,"শন":1507,"ষক":239,"শর":329,"স।":308,"শত":558,"শট":242,"শে":3570,"ষম":327,"শ্":3936,"শো":350,"ষয":380,"সক":584,"শহ":1521,"শু":848,"শী":865,"সং":3293,"শি":3267,"শা":2252,"ষণ":732,"সন":1003,"ষে":1161,"সফ":273,"সব":1306,"সভ":427,"সম":3977,"ষ্":5176,"সর":1993,"সল":739,"হক":226,"সঙ":453,"ষা":5502,"ষি":1250,"সত":301,"সদ":563,"ৎস":482,"ৎপ":325,"্শ":719,"্স":2033,"্ষ":4401,"্ম":4357,"্ভ":1012,"্র":30653,"্য":25184,"্ল":2305,"্ত":14406,"্ণ":1498,"্ড":2937,"্ঠ":1626,"্ট":7869,"্ঞ":1750,"্ব":10276,"্প":3453,"্ন":2407,"্ধ":2336,"্দ":4738,"্থ":5602,"ৌল":231,"ৌর":302,"্ছ":784,"্জ":1742,"্চ":4084,"্গ":2822,"্ঘ":304,"্ক":3885,"োয":771,"োর":1773,"োল":1204,"োব":621,"োভ":229,"োম":910,"োষ":455,"োস":598,"োহ":395,"োঝ":365,"োজ":493,"োড":367,"োট":842,"োদ":303,"োত":491,"োপ":763,"োন":1801,"োধ":294,"োক":1031,"োচ":456,"োগ":1362,"গঠি":273,"খান":1047,"গঠন":344,"ৈর":614,"েশ":5180,"েল":4085,"ের":25063,"েয":1289,"েম":1903,"েভ":263,"েব":2130,"েপ":618,"েন":8060,"ৈত":343,"েহ":255,"েষ":1422,"েস":1309,"েও":994,"েখ":1729,"েক":4154,"েই":1429,"েউ":248,"েট":1435,"েড":786,"েত":1719,"েদ":505,"েগ":411,"েছ":3105,"েজ":2937,"ে।":4052,"ৃথ":340,"ৃত":2338,"ৃহ":465,"ৃষ":756,"ূত":331,"ূপ":414,"ূম":517,"ূর":2094,"ূল":1187,"ূহ":378,"ুত":1038,"ুড":349,"ুন":1462,"ুদ":1447,"ুব":643,"ুপ":599,"ুয":1183,"ুর":4508,"ুভ":198,"ুম":876,"ুল":3399,"ুস":1262,"ুষ":874,"ুশ":279,"কেট":226,"কেন":738,"কের":985,"কোন":942,"ক্ত":2888,"ক্ট":455,"ক্ষ":3987,"ক্ল":430,"ক্র":1781,"ক্য":806,"ক্স":679,"কিছ":467,"কাহ":230,"কিন":953,"কাব":216,"কায":382,"কার":4553,"কাল":1045,"কাশ":812,"কিল":233,"কিস":331,"কুল":267,"কুর":331,"কুম":261,"কৃত":1025,"কৃষ":259,"কে।":511,"কেই":224,"৮০":213,"৭১":230,"৯৯":610,"৯০":387,"৯৫":459,"৯৬":482,"৯৭":685,"৯৮":501,"৯১":321,"৯২":343,"৯৩":359,"৯৪":434,"৮৯":204,"৮৮":200,"গুর":367,"গুল":1495,"৩০":296,"২৪":220,"গীত":413,"৫০":212,"গান":542,"গার":291,"গায":230,"১৬":326,"১৫":332,"১৮":996,"১৭":415,"১২":292,"১১":238,"১৪":244,"১৩":256,"১৯":3860,"২০":1142,"০০":1455,"১০":452,"গস্":210,"্র।":452,"্যক":735,"্মা":901,"্মী":308,"্মি":327,"্যন":687,"্মে":324,"্যত":807,"্যম":709,"্যব":1629,"্রচ":617,"্যস":234,"্রক":1414,"্যু":488,"্যি":244,"্যা":10276,"্রজ":405,"গবে":354,"্বত":419,"্বন":491,"্বপ":359,"্বব":695,"্বর":1021,"্বা":1809,"্য।":373,"্বি":841,"্বী":559,"্বে":1025,"্বো":213,"্মগ":199,"্ভা":239,"্লা":616,"্লি":367,"গরে":240,"্লে":524,"্রব":578,"্যো":301,"্রম":577,"্রভ":304,"্রয":464,"্রত":2154,"্রণ":361,"্রদ":1113,"্রথ":1019,"্রন":758,"্যে":2625,"্রধ":869,"্রপ":383,"্রা":4952,"্রি":3967,"্রী":1344,"্রু":685,"খ্য":1786,"্রশ":293,"খ্র":509,"্রস":463,"্রহ":735,"্রো":780,"্রে":2630,"্ষণ":271,"গণি":276,"্ষি":1061,"্ষা":679,"্ষে":819,"্ষম":296,"গড়":257,"্শন":406,"খেল":564,"্সে":379,"্সি":438,"গত ":349,"খা ":665,"কজন":1518,"খে ":204,"। ":19077,"কলে":496,"কল্":282,"করত":378,"করণ":527,"করে":4422,"করা":2433,"কলক":407,"কর্":871,"কম্":406,"কাজ":532,"কাছ":278,"কাত":521,"কান":552,"গে ":596,"গর ":415,"কটি":5240,"ঃ ":340,"ং ":5055,"কবি":409,"ই ":6830,"উ ":312,"কথা":468,"ও ":6690,"এ ":847,"চ ":769,"খ ":354,"গ ":1629,"ক ":11586,"ত ":12607,"ণ ":3613,"ঠ ":296,"ড ":1706,"ট ":3372,"জ ":1903,"ব ":3202,"ফ ":611,"প ":1284,"ন ":20715,"ধ ":808,"দ ":2233,"থ ":1007,"কা ":1480,"ল ":8450,"ওয়":1861,"ভ ":587,"ম ":6723,"কি ":282,"য ":4809,"র ":54334,"হ ":1019,"় ":12855,"শ ":3191,"খন ":279,"কে ":5302,"ষ ":1259,"স ":4530,"া ":26160,"ী ":7142,"ু ":2897,"ি ":25805,"ো ":2835,"্ ":1910,"কল ":227,"ে ":37314,"তর ":887,"ডার":368,"ডিস":334,"ডিয":319,"তন ":462,"ণে ":593,"তম ":977,"ড়া":1089,"ড়ি":682,"ড়ে":451,"১৯০":231,"১৯১":267,"তু ":433,"১৯৪":366,"১৯৫":361,"১৯২":273,"১৯৩":285,"১৯৮":393,"১৯৯":498,"১৯৬":398,"১৯৭":613,"তী ":298,"তে ":4456,"ডের":406,"ডেন":243,"তি ":2372,"তা ":2207,"ঠান":465,"ঠিত":686,"২০০":842,"দ। ":304,"ণা ":499,"টাব":261,"টান":241,"টিক":566,"টার":1384,"টাল":237,"টিত":421,"টিন":246,"টিশ":274,"টির":990,"টাই":256,"ডে ":223,"ণত ":291,"টোব":204,"ট্র":2092,"ট্য":377,"ট্ট":258,"টেম":345,"টের":443,"টেল":237,"ঞান":1488,"০০০":220,"ত। ":1457,"ডি ":251,"ঞ্চ":1271,"ঞ্জ":764,"টবল":369,"ড় ":618,"টি ":10700,"জ্ঞ":1748,"টা ":567,"জ্য":1590,"টে ":252,"ঠন ":205,"ঝায":280,"ঠা ":199,"ছোট":215,"জয়":273,"টন ":232,"জান":513,"জাত":1408,"জার":1034,"জিক":202,"জিত":234,"জীব":714,"জুল":281,"জুন":236,"জেন":272,"জেল":1183,"জের":493,"চ্চ":1435,"চ্ছ":780,"ছবি":323,"ছিল":2594,"ছাড":436,"জধা":329,"ছেন":778,"জনপ":400,"জনী":266,"জন্":2129,"ছে।":1579,"জি ":2005,"৭১ ":202,"ঙ্গ":2125,"ঙ্ক":292,"জা ":285,"চনা":453,"চন্":198,"জে ":350,"চলি":385,"চলে":500,"চলচ":862,"চরি":276,"চিত":2401,"চাল":768,"চার":933,"চিম":924,"The":204,"চেয":454,"চীন":632,"পৃথ":340,"মক ":218,"পূর":1563,"পুর":1770,"পুত":228,"বংশ":207,"পের":502,"পেন":312,"পশ্":957,"পাশ":229,"পিত":257,"পাক":348,"পান":707,"পিউ":331,"পাত":414,"পাদ":424,"পাধ":205,"পার":1335,"পাল":505,"পায":279,"পাও":244,"পরে":327,"পর্":1509,"বী ":353,"পরি":2733,"য। ":281,"পরব":233,"বে ":2767,"পন্":745,"পদা":548,"পদ্":310,"বা ":2607,"বি ":661,"পঞ্":305,"বর ":1036,"পড়":268,"বল ":484,"পত্":595,"পতি":403,"বন ":272,"ন্ত":3819,"ন্ড":1774,"ন্ট":1141,"ন্স":716,"ন্থ":421,"ন্দ":2863,"ন্ধ":523,"ন্ন":1373,"ন্ম":1138,"ন্য":3823,"ম। ":348,"নেক":580,"নেত":451,"নের":3172,"নৈত":249,"নোব":216,"নুয":400,"নুষ":645,"নুস":424,"নিস":305,"নীত":622,"নীয":704,"নীর":259,"নিত":220,"নিজ":306,"নির":1511,"নিয":1391,"নিব":223,"নিম":242,"নাথ":204,"নী।":283,"নাট":333,"নিক":1359,"নাল":376,"নার":884,"নায":377,"নাম":2830,"নিউ":358,"নান":283,"নাই":230,"না।":209,"পে ":392,"বং ":4190,"নয়":574,"ধ্য":2527,"পি ":200,"নভে":235,"ধ্ব":397,"নপ্":410,"নদী":606,"ধের":201,"নতা":305,"নটি":399,"al ":424,"পর ":878,"ধার":1388,"ধিক":529,"ধান":1581,"ধুন":236,"ধীন":509,"and":351,"an ":333,"নগর":339,"ধর্":622,"নকা":301,"দ্দ":285,"দ্র":1656,"দ্য":1705,"দ্ব":1242,"দ্ভ":351,"দ্ধ":1443,"ধরণ":208,"ধরন":300,"দোল":241,"ধতি":214,"দেখ":386,"দেশ":3340,"দেব":447,"দের":2400,"দেয":262,"দীর":527,"দুই":388,"দুর":229,"পক ":215,"ati":407,"দর্":464,"০০ ":448,"দলে":293,"নী ":1022,"দশক":219,"নে ":2592,"নো ":750,"দস্":252,"দায":342,"দান":683,"দিক":556,"দার":1068,"দিয":527,"দিন":460,"ন্ ":258,"থিত":1239,"থিব":279,"থান":907,"থার":216,"থাপ":342,"থের":228,"থেক":1914,"থ্য":318,"নি ":3145,"না ":2078,"ত্ত":2681,"ত্ব":1563,"ত্র":5371,"ত্য":1749,"ত্ম":200,"১০ ":255,"ধি ":218,"দক্":833,"ধে ":246,"থাক":949,"তাঁ":801,"তি।":212,"তাক":433,"তা।":273,"তিত":496,"তিন":2758,"তিব":469,"তিয":323,"তির":977,"তিহ":514,"তিষ":915,"তিস":234,"তাদ":379,"তান":831,"তাব":456,"তায":251,"তার":2316,"তাল":484,"তিক":1444,"তুল":199,"তুর":359,"তীর":252,"তীয":1406,"তৃত":307,"তের":1547,"তৈর":454,"থবি":319,"তর্":666,"দী ":536,"তরা":670,"তরে":281,"দু ":322,"ণ্ড":418,"দা ":406,"তমা":738,"দি ":552,"ণ্য":221,"তবে":327,"দে ":324,"ণিত":430,"দর ":218,"তন্":360,"ণের":721,"তথ্":200,"তত্":385,"থা ":838,"ন। ":2670,"থে ":916,"তকে":220,"ঢাক":444,"থম ":866,"রকা":1796,"লত ":375,"রচন":240,"রকে":273,"রক্":515,"লন ":417,"য়ক":403,"য়ন":1056,"য়ত":420,"রজা":344,"য়র":205,"য়ম":204,"য়ো":399,"য়ে":7441,"য়ু":262,"য়ী":440,"য়ি":464,"য়া":7334,"রচল":394,"রচি":290,"য়।":3790,"যুগ":221,"যুক":999,"যুদ":441,"যাক":626," । ":1068,"যাট":233,"যাত":1108,"যাদ":399,"যান":2143,"যাপ":621,"যাব":244,"যাম":399,"যাল":1187,"যার":1249,"যায":1288,"যিক":236,"যাস":933,"রটি":492,"যিন":243,"রথম":978,"রত্":227,"রতে":1478,"রতি":1816,"রতী":622,"রণে":668,"রণা":328,"রণত":263,"রপত":250,"ion":522,"রন্":434,"রনা":231,"রনে":444,"যেম":243,"যের":1196,"রধা":866,"রদে":667,"রদা":349,"রভা":262,"লি ":1224,"রবা":369,"রবি":326,"লা ":2174,"রবর":362,"যোগ":846,"রমা":242,"লী ":505,"রয়":679,"লে ":3437,"রহণ":319,"রস্":785,"লো ":880,"ল্ ":250,"রাই":307,"রাক":460,"রাচ":539,"রাজ":2647,"রাখ":212,"রাণ":570,"রাপ":301,"রাব":230,"রান":1015,"রিক":1905,"রার":655,"রাম":984,"রায":1265,"রিজ":258,"রাস":948,"রিচ":1336,"রাহ":202,"রাশ":201,"রাষ":1115,"রিত":670,"রিট":394,"রিম":358,"রিব":807,"রিন":213,"রিস":680,"রিয":1449,"রিল":310,"রীক":256,"রীয":402,"রীর":289,"রুয":235,"রুত":425,"রূপ":403,"রে।":1025,"রেন":1720,"রের":3004,"রেল":310,"রেশ":198,"রেছ":854,"রেজ":2190,"রেট":233,"রেক":207,"রেস":215,"রোম":235,"রোপ":366,"রোগ":263,"র্ণ":1236,"র্ত":1912,"র্ড":513,"র্ট":567,"র্ব":2473,"র্ধ":342,"র্ন":427,"র্থ":1669,"র্দ":480,"র্ল":198,"র্য":1584,"র্ম":2023,"র্ভ":416,"র্স":595,"র্ষ":411,"র্শ":716,"র্চ":340,"র্জ":782,"র্ক":1521,"র্ঘ":284,"র্গ":673,"লকা":488,"লচ্":855,"লক্":395," Ma":238,"he ":468,"লতে":397," Co":216,"লটি":222,"লনা":313," Th":233,"লয়":917,"লম্":237,"লাই":481,"লাক":628,"লান":256,"লাদ":1657,"লাস":227,"লিখ":198,"লিক":746,"লার":1528,"লাভ":487,"লায":309,"লাম":488,"লিপ":223,"লিন":254,"লিত":865," of":374," an":214,"লে।":338,"লেও":259,"লের":1832,"লেন":1829,"লেজ":413,"লেখ":829,"লেক":269,"লীন":213,"লিম":293,"লিয":577,"লির":379,"ing":263,"লীয":377,"in ":228,"ল্প":990,"ল্ড":206,"ল্য":834,"ল্ল":756,"স। ":246,"লোচ":236,"লোম":202,"লোর":312,"লোয":202,"লোক":706,"শন ":671,"hum":238," th":574,"মত ":200,"মন ":560,"ফরা":505,"র। ":1082,"প্ত":664,"প্য":216,"প্র":13060,"প্ল":304,"প্ট":290,"পৌর":203,"ভা ":303,"বনি":284,"er ":443,"বনে":288,"বন্":515,"es ":375,"ফেব":217,"বচে":366,"বঙ্":528,"ent":254,"বছর":385,"ফুল":203,"বড়":273,"ফুট":478,"ফার":283,"বলে":833,"মে ":1672,"বস্":2112,"বহু":351,"বহৃ":337,"বহা":619,"ফোর":229,"ববি":644,"বপূ":262,"মি ":517,"ফ্র":706,"মা ":428,"বয়":292,"মী ":371,"বলা":524,"বলত":367,"বর্":2305,"বৃহ":359,"বেক":203,"বেল":404,"বের":931,"বেশ":937,"বেষ":386,"বোঝ":365,"য় ":12199,"বাই":313,"বাং":2410,"বান":279,"বাদ":861,"বাধ":440,"বার":1699,"বাম":198,"বায":216,"বাঙ":403,"বাজ":214,"বাচ":199,"বিত":730,"বিদ":1525,"বিধ":272,"বিন":362,"বিপ":215,"বিব":278,"বিভ":1099,"বিয":248,"বির":389,"বাল":212,"বিখ":512,"বিক":671,"বাহ":872,"বিচ":203,"বাস":937,"বিজ":1409,"রও ":300,"বীর":327,"বীপ":377,"বিষ":640,"বিস":375,"বিশ":2344,"ভাই":208,"রম ":217,"মগ্":268,"রন ":224,"যে ":1841,"রত ":384,"রণ ":1177,"ল। ":672,"ব্র":1160,"ব্য":3166,"ব্দ":1107,"যা ":1724,"রা ":4296,"রি ":1578,"ভ্য":238,"মবঙ":362,"মন্":664,"ভেম":223,"মধ্":1555,"মতা":307,"ভূম":496,"ভুক":240,"ভাগ":718,"ভার":2311,"ভাব":1222,"ভাষ":4828,"ভিয":254,"ভিন":1250,"ভিত":372,"ed ":213,"মাই":303,"মিন":199,"মিত":518,"মিট":419,"মাস":327,"মিক":501,"মাল":710,"মার":1752,"মান":3429,"মাত":575,"মাধ":574,"মাণ":369,"মাজ":400,"রো ":310,"মহা":801,"মস্":221,"রে ":4008,"যক্":405,"রু ":494,"রী ":1730,"ময়":958,"মৌল":207,"ম্র":226,"ম্য":700,"ম্ভ":243,"ম্ম":363,"ম্ব":1398,"ম্প":1633,"যমে":422,"যন্":880,"যবস":455,"যবহ":926,"মেন":347,"মের":1210,"মিশ":280,"মিল":414,"মীয":198,"মুখ":293,"মুক":494,"মুদ":244,"মুস":261,"মূল":1049,"লক ":453,"মূহ":371,"মৃত":392,"যতম":572,"সলা":347,"re ":199,"হচ্":536,"সাং":239,"সাম":709,"সায":256,"সার":942,"সাল":2557,"সিক":500,"সাধ":708,"সান":242,"সাব":458,"সিত":294,"সাহ":727,"সাই":326,"সাথ":746,"সাগ":439,"সীম":247,"সিন":266,"সির":308,"়ক ":242,"সূর":236,"সত্":211,"সদস":228,"ষেত":374,"সন্":338,"ষের":340,"সবচ":368,"সবা":223,"ষ্ক":275,"ষ্ণ":260,"ষ্ট":2671,"সভা":316,"ষ্ঠ":1554,"সময":711,"ষ্য":236,"সমা":613,"সমূ":375,"সম্":1299,"সরক":840,"সর্":566,"়ন ":433,"হিন":860,"হিস":1250,"হাদ":208,"হান":202,"হাম":272,"হায":221,"হার":1136,"হাস":659,"হিত":775,"হাজ":205,"হৃত":340,"সেই":206,"সেম":294,"সেব":1065,"সেপ":226,"সেন":564,"হতে":419,"সৃষ":262,"হত্":332,"সের":1064,"হয়":5615,"া। ":1439,"স্ক":1611,"স্ট":2276,"স্ত":2428,"স্থ":3469,"স্প":563,"স্ব":1109,"স্য":561,"হলে":402,"হলো":289,"হরে":416,"াগ ":429,"াও ":315,"াক ":329,"়া ":2568,"়ি ":218,"হ্য":294,"হের":399,"াই ":889,"়ে ":3374,"়ী ":420,"ি। ":726,"াল ":1724,"িও ":348,"ার ":13003,"াম ":1773,"াভ ":401," ১০":339,"াব ":352," ৩০":204," ১২":242,"়নে":202," ১৩":220," ১৪":201," ১৫":282," ১৬":286," ১৭":371," ১৮":940," ১৯":3783,"াহ ":308," ২০":1046,"াস ":1335,"াশ ":330,"িক ":5573,"িং ":561,"ী। ":724,"াট ":241,"াজ ":536,"াপ ":211,"ান ":5410,"াদ ":431,"াণ ":268,"াত ":1005," এই":3270," উৎ":560,"ng ":235,"শে ":909," উদ":684," উন":258," উত":959," উচ":373," উল":389," উপ":2191," ইত":669," আস":453," আয":322," অ্":835," আম":509," আর":1370," আল":1083," আধ":289," আদ":422," আন":905," ইউ":775," আফ":294," আব":706," উই":304," ইস":476," ইল":209,"nd ":283," ইর":274," ইয":287," ইন":731," অঙ":286," অক":379," অত":316," অঞ":596," অপ":429," অফ":358," অব":2124," অথ":198," আই":665," অধ":751," অন":3315," অল":206," আক":517," অভ":831," অর":1146," আছ":551," আগ":504," অস":564," ইং":2299,"শি ":343," অং":591,"শী ":336,"ষণ ":226," জ্":458," জা":2134," জি":334," জী":551," জু":646," জে":1308," জো":204," জল":229," ছো":231," ছি":1929," ছা":399," জন":3064," ছব":300," চা":848," চি":755," চে":275," চল":1081," চর":388," ২ ":205," ১ ":302," গ্":1853," গো":546," গে":282," গু":745," গি":203," ঘট":372," গা":986," গব":363," খ্":684," খে":607," গণ":589," গঠ":432," খা":596," খু":226," কা":3277," কি":1369," কু":537," কৃ":271," কে":1060," কো":1627," ক্":2305," কন":254," কথ":522," কব":408," কর":8000," কম":804," কল":1072," ওয":520," এস":383," এশ":223," এর":2835," এল":645," ঐত":234," এছ":286," এখ":588," এক":8988," এম":399," এব":4212," এপ":273," এন":256," এদ":302," এট":2135," ফে":435," বন":336," ফা":463," বড":264," ফি":389," ফু":675," বছ":364," পো":225," পৌ":257," প্":11714," ফর":609," ফল":347," বে":1555," বো":669," বৈ":440," বি":7573," বা":7403," বৃ":550," বু":368," বস":531," বহ":441," ফ্":592," বল":1749," বর":1400," মধ":1533," মন":645," ভে":259," ভূ":442,"শক্":282," মত":465," ভা":7371," ভি":710,"শকে":227," ব্":3507," ম্":464," মৌ":257," মো":583," মে":1083," মৃ":318," মূ":802," মু":1346," মি":1118," মা":4397,"সন ":262,"ষে ":226," মহ":923," মর":212," ধা":649," নদ":623," দে":2058," দৈ":202," দ্":1551," ধর":1182," নগ":253," না":3576," নি":3394," নী":245," নে":783," ধ্":292," নভ":274,"of ":368," পঞ":266," পড":228," পদ":913," পত":409," ন্":272," নো":287,"সব ":353," পু":1605," পা":3594," পি":495," পে":490," বই":234," পূ":997," পৃ":428," পর":4825," পশ":1001," তব":328," তত":253," তথ":332," তি":2653," তা":3867," তু":325," দক":818," ত্":257," তৈ":474,"ষা ":1614," তে":303," থা":1028," দল":654," দর":261," থে":1958," দি":1421," দা":849," দু":810," দশ":293," টা":272," টি":345," টে":546," ট্":234," ডি":734," ডা":459," ডে":331," ঢা":461," হয":5669," স্":3980," সো":334," হল":1487," সি":1221," সা":6870," সহ":408," হচ":537," সে":1681," হত":623," সৃ":267," সূ":402," সু":981," সী":216," সব":807," সম":3235," সন":329," সর":1532," সঙ":383," সত":207," সদ":329," হো":212," হ্":276," হা":1267," হি":1859," হে":312," শক":251," শা":1199," শি":1188," সং":2886," শু":645," শহ":1500," সক":314," শে":357," শ্":581," শত":509," শব":479,"on ":532," রি":295," রা":3902," রে":556," রু":295," রূ":298," রো":452," রক":302," যা":2826," রচ":547," যু":1137," যি":261," রয":403," রব":219," যো":242," যৌ":226," যে":1328," লি":670," লা":967," লে":811," লো":608," লক":345,"স্ ":266," ও ":4589,"শাখ":311,"শাস":697,"শিত":339,"হর ":728,"শাল":206,"শিক":625,"le ":226,"শহর":1422,"সে ":658,"হন ":263,"সী ":240," এ ":753,"হণ ":261,"সি ":877,"শব্":493,"সা ":275,"শনে":210,"শনা":303,"শতক":230,"শতা":251,"শটি":231,"mb ":231,"ষাত":229,"ষিণ":767,"ষায":1936,"ষার":498,"ষাব":214," স ":339,"ষা।":354,"সঙ্":452," র ":316,"়। ":2782,"ষয়":378,"হী ":223,"শ্র":601,"শ্য":362,"শ্ব":1645,"ষমত":242,"শ্চ":1032,"শেষ":751,"শের":1660,"ষণা":351,"শুর":445,"সংব":214,"হল ":715,"সংস":804,"শিয":443,"শিল":531,"সংক":417,"সংগ":424,"সংখ":731,"সংঘ":234,"শিষ":385,"ুসা":325,"ুষ্":372,"ুসল":207,"ূমি":444,"ূলক":257,"ূলত":394,"ে। ":2956,"ূর্":1748,"ুক্":1812,"ুটি":275,"ুড়":269,"ুটব":355,"ুপ্":205,"ুনি":328,"ুদ্":1024,"ুত্":534,"ুলা":360,"ুলি":957,"ুলো":677,"ুরস":488,"ুরা":502,"ুরু":902,"ুরে":214,"ুর্":475,"ুয়":999,"ুমা":378,"ৃহত":230,"েও ":751,"েক ":677,"েজ ":442,"েট ":476,"েড ":233,"েত ":298,"েন ":2383,"ৃতি":667,"ৃথি":275,"ৃত্":530,"ৃষ্":660,"েই ":990,"াচী":479,"াঙা":392,"াঙ্":430,"াজে":291,"াজ্":1187,"াজা":637,"াজি":350,"াজধ":330,"াজন":429,"াছে":290,"াখা":510,"াগর":531,"ীন ":996,"াকা":1674,"াকৃ":214,"াকি":417,"াক্":624,"াকে":1517,"াগা":228,"াগু":251,"াগে":340,"ীত ":407,"াওয":538,"ুই ":270,"াপে":239,"াপি":216,"াপা":332,"াপ্":438,"াবল":223,"াব্":798,"াবে":1282,"াবি":607,"াবা":550,"ামক":357,"ার।":391,"াথে":795,"াদা":503,"াদি":433,"াদী":233,"াদে":2459,"াদ্":228,"ানক":226,"াধি":292,"াধী":407,"াধা":767,"িউট":438,"াধ্":701,"ানম":216,"ানব":285,"ানা":935,"ানি":1277,"ানী":1247,"ানু":818,"ানে":2964,"ান্":3257,"ানো":483,"াপক":268,"াপন":257,"াণী":233,"াণি":288,"াতী":444,"াতি":1024,"াতা":904,"াতন":242,"াণে":256,"ান।":554,"াত্":1070,"াতে":889,"ীর ":1512,"াটি":687,"াঞ্":296,"াট্":216,"াড়":1116,"িন ":1620,"িদ ":390,"িত ":5217,"িণ ":578,"িজ ":216,"়ের":988,"়েল":232,"াইল":290,"িস ":391,"়েন":224,"়েত":314,"়েছ":1789,"িশ ":479,"াইক":267,"াইন":398,"াইট":347,"়িত":393,"াঁর":695,"াংশ":369,"িল ":1310,"াংল":2405,"়াড":214,"াৎ ":225,"়াল":346,"়াম":294,"ির ":3233,"়ার":2357,"়ান":720,"়াত":233,"িম ":623,"িহা":558,"ংশ ":608,"িস্":1568,"িসা":395,"িসি":199,"িসে":1302,"ূল ":317,"িরি":292,"িরা":288,"ির্":1412,"িলা":304,"িলি":409,"িলে":1534,"িল্":792,"িলো":314,"িশন":236,"িষ্":1946,"িশি":298,"িশে":548,"িষয":343,"িশ্":1581,"ীবন":365,"ীমা":216,"ীয়":4081,"ীর্":354,"ীরে":207,"ৃত ":857,"umb":265,"ীতি":604,"ীতে":555,"ীদে":242,"ীনত":266,"াহা":469,"িচা":733,"াহী":229,"িচি":688,"াহি":1315,"াস্":708,"াসে":596,"াসি":1223,"াসা":419,"াসী":261,"াষ্":1171,"াসন":338,"াষা":4713,"াশি":537,"ুর ":1172,"িটি":780,"িটা":629,"িজ্":1437,"িজে":324,"িছু":475,"ালক":364,"ালয":874,"ালন":396,"ারী":1146,"ারি":1557,"ারা":1659,"ার্":5067,"ারে":2345,"ায়":8671,"ারক":300,"ারন":347,"ারণ":1235,"ারত":1942,"ামি":443,"ামা":748,"াম্":609,"ামে":1488,"ামী":317,"াশন":211,"িখ্":507,"ুন ":561,"িকা":1858,"িকি":198,"িকে":1068,"িক্":1074,"ালী":661,"ালা":739,"ালি":1410,"ালো":249,"ালে":2817,"াল্":297,"িপি":219,"িপ্":331,"িনে":565,"িনা":408,"িনি":2917,"িনী":642,"িনয":241,"িন্":1908,"িমি":290,"িম্":258,"িমে":259,"িমা":636,"িল।":473,"িয়":6354,"িবা":741,"িবী":293,"িবি":361,"িবে":347,"িবর":417,"িভি":889,"িভা":580,"িমব":382,"tio":423,"thu":236,"িদ।":233,"ুল ":613,"িডি":222,"িত।":1272,"িদ্":1210,"িদে":199,"িনট":209,"িধা":259,"ter":251,"িতী":319,"িতি":339,"িতা":574,"িতে":1209,"িত্":2907,"the":401,"ঁর ":674,"্র ":2468,"্য ":4783,"োঝা":363,"্ম ":1055,"্ব ":1443,"্প ":322,"োগ্":438,"্ন ":1119,"্স ":725,"্ষ ":348,"ংশে":259,"ংসদ":201,"্ট ":1640,"্জ ":260,"্চ ":547,"ংলা":2409,"ংরে":2127,"্দ ":444,"োকে":215,"্ধ ":572,"্ত ":2310,"ংবা":245,"্থ ":613,"্ণ ":661,"্ঠ ":206,"্ড ":1005,"্ক ":455,"্গ ":569,"ংক্":310,"ংখ্":741,"্ঞা":1605,"্জা":489,"্জি":208,"্জে":211,"্জন":340,"্ছে":599,"্ত।":316,"্ডি":303,"্ডা":322,"্ডে":556,"্ঠা":754,"্ঠি":421,"্টি":1218,"্টা":1173,"্টো":370,"্ট্":1738,"্টে":710,"্দ্":1049,"আর ":224,"্দো":445,"্দা":237,"্দি":541,"্দী":307,"্দে":669,"্দু":407,"্ধত":232,"্দর":217,"্থা":2220,"্থি":1299,"্থে":279,"্থন":231,"্তে":276,"্থব":329,"্ত্":1963,"্তা":1329,"্তি":2212,"্তী":445,"্তু":740,"্তৃ":252,"্তন":456,"্তম":911,"্তর":2524,"্বক":355,"্প্":259,"্পে":443,"্পা":452,"্পী":215,"্পি":552,"্পর":340,"্পন":198,"অংশ":574,"্না":360,"্নি":236,"্ধে":424,"্ধি":256,"্ধা":305,"অক্":342,"ইন ":402,"অঙ্":286,"্কি":934,"্কা":944,"্কৃ":335,"্কে":282,"্গত":198,"্চা":486,"্চি":1888,"্চল":770,"্গে":523,"্গী":329,"্গা":318,"অফ ":315,"ংস্":691,"োনো":215,"োনা":252,"োপা":229,"োমা":248,"োমি":242,"োয়":741,"োরি":210,"োর্":375,"োলা":267,"োলন":236,"েছি":696,"েছে":2368,"েজি":1969,"ইংর":2136,"েক্":594,"আমে":217,"েকে":2070,"েখক":248,"অ্য":839,"আয়":311,"েখা":876,"আবি":225,"েকট":292,"ইউন":276,"আধু":208,"ইউর":313,"�":1457,"আন্":621,"আলো":342,"আর্":422,"আরব":337,"ইটি":293,"অঞ্":595,"অবস":1435,"অভি":728,"আইন":296,"অধি":383,"অনু":1003,"অধ্":210,"অন্":1608,"অনে":547,"আকা":241,"েশ ":1172,"েস ":337,"অর্":1060,"েষ ":526,"েল ":879,"আছে":543,"ের ":22964,"অস্":383,"োর ":595,"োন ":788,"োট ":334,"ৈরী":206,"ৈরি":250,"উচ্":373,"ৈতি":259,"োগ ":311,"উটা":283,"োক ":321,"উত্":954,"েস্":373,"ইত্":216,"ইতি":273,"ইন্":608,"েরা":505,"েরি":683,"েলে":417,"েলি":409,"েলা":1478,"েলো":239,"ইয়":586,"েশন":465,"েশী":418,"েশি":582,"েষণ":370,"ইরা":327,"েশে":1749,"েষ্":229,"েন্":1439,"েনে":324,"েনি":277,"েনী":276,"েনা":409,"েপ্":238,"েবে":991,"েব্":240,"েয়":1277,"েম্":732,"েমি":222,"েমন":290,"ইসল":353,"েটি":246,"েডি":240,"েতা":376,"েত্":488,"েন।":2598},"n_words":[1969690,2210879,1502429],"name":"bn","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/br.json b/contrib/languages-data/br.json
new file mode 100644
index 0000000..7216a43
--- /dev/null
+++ b/contrib/languages-data/br.json
@@ -0,0 +1 @@
+{"freq":{"D":4197,"E":6232,"F":3169,"G":6172,"A":11736,"B":9621,"C":6059,"L":5259,"M":7896,"N":3931,"O":1684,"H":4028,"I":4207,"J":2162,"K":6749,"U":5511,"T":3855,"W":1494,"V":2500,"Q":172,"P":6579,"S":9102,"R":4212,"Y":1003,"X":512,"Z":488,"f":8875,"g":61989,"d":78008,"e":313803,"b":31588,"c":24054,"Fed":26,"a":272837,"n":176116,"o":149856,"l":87783,"Fei":28,"m":47338,"j":4813,"Fel":48,"k":50019,"h":76286,"i":117218,"w":12251,"v":50149,"Fer":128,"u":89344,"t":103349,"s":73080,"r":187209,"q":653,"p":29670,"z":76339,"y":13415,"x":1593,"²":87,"É":103,"Æ":28,"Á":35,"ß":56,"Ö":36,"Fil":72,"ï":72,"î":145,"Fin":82,"í":410,"ì":30,"ë":83,"ê":3087,"Fir":91,"é":1358,"è":308,"ç":177,"æ":55,"å":31,"ä":104,"ã":91,"â":320,"á":471,"à":154,"ü":417,"ú":104,"û":28,"ø":33,"ù":12711,"ö":186,"ô":883,"ò":64,"ó":303,"ð":41,"ñ":13114,"ē":47,"ā":179,"ı":57,"ī":94,"œ":24,"ł":28,"ō":129,"ŋ":29,"š":64,"ū":60,"Fas":33,"Fal":52,"Far":46,"Fao":28,"Ere":25,"Eri":80,"Ess":28,"Est":94,"ɪ":92,"Eti":95,"Ern":44,"ɑ":37,"ɔ":56,"ə":176,"Esk":29,"ɛ":92,"Eus":234,"Eur":413,"Evi":62,"Eve":178,"ʲ":56,"Etr":88,"ʁ":32,"ʃ":63,"Eug":29,"ʒ":27,"ˈ":252,"ː":126,"El ":42,"Eil":157,"Eis":24,"̪":34,"́":93,"En ":194,"Ele":79,"μ":110,"ν":272,"ο":341,"Eng":26,"θ":54,"ι":256,"Ene":282,"κ":130,"λ":194,"δ":108,"ε":164,"η":171,"α":367,"β":37,"γ":77,"έ":64,"ά":97,"Emr":46,"ί":114,"ή":58,"Τ":24,"Emi":47,"Π":49,"Σ":33,"Ema":198,"Emb":80,"Μ":29,"Κ":60,"Emg":113,"Δ":26,"Ε":25,"Α":38,"Eli":87,"Er ":626,"Eos":308,"ω":68,"ώ":38,"ύ":56,"ό":91,"σ":144,"Enn":75,"ς":295,"ρ":254,"Eno":35,"π":118,"χ":49,"φ":41,"υ":86,"τ":173,"ь":74," l":8798," m":10929," n":4691," o":10291,"я":110," h":33728,"ш":32," i":6397," j":591," k":16456,"ы":48," d":33789," e":65040,"х":29," f":3147,"ц":29," g":22398,"ч":83,"р":242," a":67851,"с":221," b":15502,"т":159," c":6570,"у":98," y":7164," x":75," z":11510," u":16972," t":10075," w":4118," v":14078," q":70," p":14721," s":18295," r":10530,"К":52,"М":37,"П":37,"Б":35,"А":40,"В":28," J":2150," K":6707,"Gel":33," H":4011,"Gem":30," I":4197," N":3921," O":1674," L":5228," M":7864," B":9590," C":6014,"Р":27," A":11706,"С":33," F":3158," G":6126," D":4174," E":6209,"л":194," Z":483,"к":212,"й":67," Y":1001," X":511,"и":383,"п":52,"Gev":46,"о":298,"н":235,"м":105," S":9083,"г":57,"Ger":233," R":4196,"в":187," Q":170,"б":58," P":6551,"а":436,"Geo":116," W":1492,"з":42," V":2493,"Gen":420," U":5507,"е":253," T":3831,"д":87," à":46," é":34,"Gla":87,"Gha":39," Á":35," Æ":28,"Gio":61,"Gin":304," É":99,"Gil":36,"Gir":54," Ö":36,"Giu":33,"Öst":25,"ա":37,"ה":46,"ו":58,"ד":24,"א":35,"ב":25,"מ":26,"ל":40,"י":72,"נ":26,"ש":34,"ר":42,"ת":30,"Gan":365,"Gal":400,"Gau":32,"Gar":135,"Gai":41,"Gae":31,"Gab":63,"و":95,"ي":184,"ف":25,"ق":36,"ل":178,"م":124,"ن":112,"ه":24,"Ful":27,"د":75,"ج":40,"ح":59,"ت":33,"ب":101,"ة":62,"ا":268,"ع":53,"ش":34,"س":58,"ز":26,"ر":129,"Fro":29," ˈ":146," Κ":60," Μ":29," Α":37," Ε":25," Δ":26,"Flo":56,"Fla":330," А":40," Б":35," В":28,"Fra":928," К":52,"Fri":125," М":37,"Fre":98," П":37,"A ":488," τ":29," σ":25,"Fon":41,"For":176,"Fou":55," Σ":33," Π":49,"F ":73,"Da":716,"Cu":144,"Cy":124,"Cl":234,"Co":885,"Cr":171,"Ce":272,"Ch":926,"Ci":142,"G ":58,"Ec":91,"Ed":137,"Ea":89,"Eb":325,"Du":515,"Dy":40,"Do":666,"Dr":253,"De":778,"Dj":30,"Di":924,"Dh":31,"Fe":317,"H ":28,"Fa":315,"Ez":36,"Eu":755,"Ev":284,"Ex":30,"Er":901,"Et":209,"Es":214,"En":714,"Em":545,"Ep":30,"Eo":336,"Ei":245,"El":312,"Ek":28,"Eg":106,"Ge":937,"Ga":1188,"I ":590,"Fu":93,"Fr":1205,"Fo":347,"Bé":40,"Fl":420,"Fi":321,"B ":138," Р":27," С":33,"II ":397,"C ":1616,"Av":131,"Au":241,"Ar":3352,"IIv":103,"Aq":24,"At":426,"As":207,"D ":116,"Ba":1056,"Az":390,"Ay":27,"Ae":93,"Af":391,"Ag":119,"Ah":33,"Ab":268,"Ac":90,"Ad":209,"Am":688,"An":2172,"Ao":672,"Ap":114,"Ai":113,"Ak":96,"Al":1482,"Hir":30,"Hit":39,"His":54,"Bu":358,"Br":4619,"Hiz":25,"Ca":1444,"E ":930,"Bh":52,"Bi":459,"Be":1718,"Hil":64,"Bo":882,"Him":29,"Bl":208,"Hip":24,"Ku":315,"Ky":42,"Kn":26,"Kl":164,"Kr":780,"Ko":1315,"Le":1264,"Li":865,"N ":201,"La":1228,"Lu":362,"Ly":48,"Hé":41,"Ll":170,"Lo":978,"Me":1994,"Mi":714,"া":34,"O ":130,"Ma":3078,"Mc":41,"My":178,"Mu":254,"Mo":1332,"Ni":470,"Ne":1118,"Na":820,"P ":94,"Hel":244,"Ny":33,"Hei":53,"Nu":54,"No":1155,"Hea":31,"Ok":98,"Ol":141,"Om":48,"Kê":117,"On":29,"Og":35,"Oi":37,"Heo":48,"Oc":47,"Od":68,"Hem":56,"Of":58,"Hen":313,"Hes":59,"Oa":47,"Her":732,"Ob":101,"Gi":574,"Gh":58,"Gl":175,"Gr":739,"Go":862,"Gu":340,"Gy":27,"Gw":1133,"J ":134,"क":25,"Ha":1093,"He":1705,"Hi":350,"Ho":415,"न":34,"Hu":232,"म":31,"Hy":84,"र":42,"K ":222,"Ib":34,"Ia":195,"Id":60,"Ic":25,"Ie":25,"ि":28,"Ig":31,"ा":67,"Io":60,"Ip":79,"Im":319,"In":1032,"Il":240,"ी":28,"Iw":209,"Iv":208,"्":59,"Is":368,"It":343,"Ir":161,"Ja":558,"L ":80,"Iz":313,"Ji":84,"Je":437,"Jo":529,"Had":25,"Haa":25,"Hab":131,"Ju":263,"Hal":46,"Ka":2038,"Hai":67,"Han":276,"M ":61,"Ham":53,"Kh":46,"Har":175,"Ki":287,"Haw":72,"Ke":1388,"Hau":54,"Ut":31,"Ur":2997,"Un":1813,"Uk":44,"Ul":286,"Uh":133,"Ty":53,"Tu":261,"Tr":772,"Pê":43,"To":427,"Th":476,"Ti":378,"Te":503,"Ta":643,"Tc":179,"V ":168,"Sw":47,"Sy":84,"St":1759,"Sv":127,"Su":798,"Wo":96,"Wi":335,"Sã":44,"Wh":28,"Wa":422,"We":489,"Vv":57,"Y ":42,"Vo":153,"Rí":30,"Vr":262,"Vu":27,"Vi":641,"Vl":43,"Ré":29,"X ":71,"Va":400,"Ve":664,"Mé":39,"Qi":24,"Pt":32," م":36,"Pu":130,"Pr":893,"Gus":31,"Ps":36,"Guy":41,"S ":168,"Py":52,"Pe":1224,"Gua":82,"Pf":25,"Pa":1933,"Gue":50,"Gui":57,"Lé":26,"Pl":487,"Po":937," ع":30,"Pi":432,"Ph":183,"Os":107,"Ot":70,"Ou":210," ا":123,"Op":33,"Or":316,"R ":57," ب":27,"Ow":50,"Kö":34,"Se":618,"Sc":261,"Si":922,"Sh":171,"Sm":46,"Sl":65,"Sk":511,"Sr":25,"Sp":346,"So":664,"Ru":420,"U ":31,"Sa":2188,"Gwy":105,"Gwe":600,"Re":1198,"Gwa":119,"Ri":358,"Rh":169,"Ro":1382,"Gwi":279,"Qu":112,"T ":45,"Ra":453,"Mü":57,"Gre":245,"Gri":43,"Gra":263,"Wü":24,"b ":1746,"Gru":58,"Gro":111,"a ":60184,"Ye":198,"Ya":330,"Yo":220,"Yv":36,"Yu":107,"Ys":24,"Gle":29,"Wy":25,"Glo":28,"Xa":37,"Xi":32,"Xv":91,"Gon":36,"Got":28,"Gou":537,"Gor":76,"Za":103,"Ze":159,"Zi":46,"Zo":34,"Zu":68,"Vî":53,"i ":7854,"fy":71,"gd":123,"ge":7989,"ga":15000,"gb":76,"bé":25,"fl":324,"fg":36,"ff":279,"fi":1447,"Ini":174,"fr":1282,"fu":204,"ft":110,"Int":68,"fo":1405,"j ":410,"gy":86,"gw":1619,"hd":50,"Ipa":70,"he":10707,"hc":25,"ha":23155,"gn":2093,"gm":47,"gl":848,"gk":33,"gi":2102,"gh":734,"gg":126,"gu":2092,"Ion":24,"gt":232,"gs":102,"gr":2162,"go":4456,"dt":122,"du":4207,"dv":101,"dw":187,"dy":171,"dz":34,"g ":20767,"ea":2356,"eb":1541,"ec":3453,"ed":9236,"de":16457,"Ili":95,"dd":600,"dg":78,"df":47,"di":9920,"dh":144,"dk":37,"dj":268,"dm":38,"dl":101,"do":5797,"dn":55,"dp":25,"ds":337,"dr":5697,"ew":376,"ex":279,"eu":16675,"añ":10099,"ev":9709,"ey":390,"ez":32964,"fa":1334,"h ":20209,"Ind":651,"fe":1499,"eh":357,"eg":13356,"ef":502,"ee":1354,"el":17644,"ek":11355,"ej":161,"ei":7025,"ep":3262,"eo":10164,"Imp":288,"en":41069,"em":6548,"et":30716,"es":7806,"aï":43,"er":32314,"aí":40,"eq":26,"ca":1129,"e ":52189,"bw":39,"by":108,"bs":76,"br":5995,"bu":921,"bn":37,"bo":2286,"bj":43,"bl":2570,"bh":95,"bi":1782,"bb":73,"bc":25,"bd":41,"be":10849,"db":36,"da":12451,"f ":728,"cy":83,"cu":421,"ct":371,"cs":66,"cq":94,"cr":201,"co":1158,"cm":65,"ck":578,"cl":127,"ci":937,"ch":6076,"ce":962,"cc":154,"c ":11462,"Il ":108,"az":3455,"ay":921,"ba":4699,"d ":20779,"at":6319,"as":6537,"ar":41906,"aq":80,"ax":163,"aw":608,"av":4883,"au":1374,"ak":2454,"al":17234,"ai":2830,"aj":361,"ao":6749,"ap":2035,"am":7774,"an":56453,"ac":2422,"ad":15598,"aa":848,"ab":3465,"ag":10609,"ah":821,"ae":5484,"af":819,"nu":699,"nt":21400,"ns":1811,"nr":251,"nq":24,"np":48,"no":7098,"nn":19156,"q ":55,"nz":429,"ny":637,"nw":203,"nv":5870,"oe":3732,"of":1198,"oc":1802,"od":2285,"oa":9317,"ob":1946,"om":6712,"on":16021,"kê":868,"ok":1528,"ol":8383,"oi":1324,"oj":257,"og":1807,"oh":346,"hō":59,"m²":87,"ot":2276,"os":3836,"ov":1541,"ou":23053,"op":1601,"oo":428,"or":10770,"oq":29,"r ":67149,"ox":54,"ow":323,"oz":3965,"oy":250,"pe":10529,"lá":71,"lâ":211,"pg":46,"pa":6028,"pl":2140,"lé":135,"pm":43,"lê":44,"pn":29,"po":2880,"ph":570,"JK ":124,"pi":1360,"pk":99,"lo":6423,"ln":63,"hê":112,"lm":544,"hé":48,"Ida":24,"ll":11038,"ls":344,"lr":74,"lp":246,"lw":139,"lv":1307,"lu":1154,"lt":965,"lz":374,"ly":435,"hô":24,"o ":32003,"md":124,"ma":12097,"mb":2996,"mg":320,"mh":71,"me":8356,"mf":56,"mk":42,"iè":109,"ml":68,"mi":2710,"mn":103,"iê":44,"mm":2195,"ié":44,"mp":2828,"mo":3261,"mr":185,"mt":73,"ms":373,"mv":54,"mu":2992,"iñ":2018,"ió":60,"mw":25,"mz":3729,"my":129,"p ":1373,"na":11870,"nb":331,"nc":1094,"nd":5068,"ne":23869,"nf":226,"ng":4164,"nh":269,"ni":8151,"nj":156,"nk":1557,"nl":441,"Iañ":171,"nm":245,"jv":61,"ju":124,"jp":52,"jo":305,"jk":39,"ki":2545,"kh":260,"kg":81,"kf":26,"ke":6882,"ka":6948,"m ":4186,"kw":151,"ky":90,"ks":726,"kt":473,"ku":843,"kv":64,"ko":9911,"kp":26,"kr":4134,"kk":98,"kl":851,"km":424,"gé":31,"gê":1037,"kn":60,"li":10539,"lh":850,"lk":281,"lj":172,"há":26,"le":14802,"ld":725,"lg":609,"lf":458,"hâ":37,"la":12823,"lc":685,"lb":380,"n ":59707,"hr":1218,"hs":140,"hp":166,"hv":118,"hw":571,"ht":657,"hu":1645,"hk":45,"hh":50,"hi":5476,"hn":188,"ho":10678,"hl":290,"dé":99,"hm":114,"id":3097,"ic":2459,"ib":1666,"ia":11918,"ih":739,"ig":3475,"if":619,"ie":9527,"hy":234,"k ":14348,"iq":109,"ir":3605,"is":9955,"it":5964,"iu":555,"eñ":707,"iv":5470,"iw":1706,"eó":29,"ix":145,"ii":261,"ij":2240,"ik":4708,"il":5550,"im":2753,"in":11502,"io":6614,"ip":861,"je":2425,"ji":570,"iz":11203,"iy":170,"l ":21633,"ja":657,"eü":82,"xi":170,"tê":725,"xo":25,"té":97,"tí":24,"xt":32,"wy":307,"ww":48,"z ":21079,"xa":108,"xe":71,"tá":25,"wg":44,"wi":1081,"wl":28,"sé":62,"wn":92,"wo":182,"wr":234,"ws":56,"wu":29,"ró":27,"vy":51,"y ":1750,"wa":5770,"wd":28,"wc":24,"we":3809,"vl":249,"rè":27,"ré":171,"vi":8261,"vu":454,"vr":4428,"vs":56,"rí":84,"vn":703,"vo":3736,"uz":3040,"uy":131,"ux":133,"uw":90,"uv":260,"uu":46,"rá":48,"ve":20616,"vc":54,"va":5019,"x ":1065,"ui":1518,"uj":140,"uk":544,"ul":3378,"ue":6663,"uf":196,"ug":1545,"uh":578,"uq":27,"ur":24906,"us":15250,"ut":2185,"um":4282,"un":10596,"uo":187,"up":344,"ty":255,"tz":187,"tu":2687,"tt":851,"tw":83,"tv":571,"ub":1283,"ua":3200,"ud":3820,"uc":669,"w ":395,"to":5713,"tn":87,"tm":65,"pé":34,"tl":859,"ts":475,"tr":9199,"tp":62,"tg":52,"tf":46,"te":12319,"td":69,"tk":150,"ti":7302,"th":2179,"où":12686,"v ":6249,"tb":27,"tc":753,"ta":12012,"su":1038,"oñ":231,"sv":158,"ss":1448,"st":12827,"sy":140,"sz":27,"sw":107,"sl":448,"sk":7635,"sn":138,"sm":219,"sp":1928,"so":3042,"sr":249,"sq":42,"sd":77,"sc":650,"sf":62,"se":5114,"sh":1087,"sg":84,"si":4429,"rz":4399,"u ":4057,"sa":6559,"sb":201,"rr":4376,"rs":928,"rt":3750,"ru":2612,"rv":3731,"rw":117,"nô":738,"rx":60,"ry":577,"rq":53,"rp":204,"ro":20230,"rn":4389,"rm":2207,"né":104,"rl":1174,"rk":1313,"nç":124,"rj":101,"ri":15929,"rh":211,"rg":1755,"rf":439,"re":30793,"ná":35,"rd":2128,"rc":2074,"rb":1039,"ra":14858,"t ":46228,"qu":495,"IXv":50,"mé":40,"má":30,"qa":26,"s ":25101,"px":514,"py":38,"pt":406,"pu":902,"pv":51,"pp":317,"lí":31,"pr":1879,"ps":278,"Huo":27,"Hun":82,"IX ":33,"IV ":67,"zz":96,"Hor":50,"zg":66,"zh":24600,"zi":3857,"Hou":36,"zb":56,"zd":80,"ze":9700,"Hog":24,"za":2558,"Hom":30,"Hon":65,"Hol":98,"zv":69,"zs":223,"vî":50,"zr":75,"zu":767,"zt":86,"zo":11706,"zn":961,"zp":33,"zk":64,"zm":26,"vé":27,"zl":48,"yg":40,"ye":6849,"yf":63,"uß":34,"yc":113,"yd":279,"ya":1435,"yb":30,"yw":102,"uñ":26,"yu":169,"yt":99,"ys":381,"yr":208,"yp":102,"yo":385,"yn":492,"ué":33,"ym":158,"yl":291,"yk":38,"yi":226,"Arg":47,"Are":40,"Arc":158,"Ard":33,"Ara":192,"Aro":63,"Arm":115,"Arn":27,"Ark":40,"Ari":62,"Apo":55,"Ate":49,"Ath":31,"Atl":295,"Ast":51,"Ass":62,"Arr":26,"Art":101,"Arv":191,"Arz":47,"Avi":58,"Ave":35,"Aus":28,"Aur":33,"Aug":87,"Aze":65,"Azg":34,"Azi":259,"Bag":26,"Bah":39,"Bai":39,"Bal":109,"Ban":147,"Bab":28,"Bac":26,"Bad":61,"Bae":28,"Bay":41,"Bar":222,"Bas":70,"Bav":68,"Bau":37,"ი":28,"ა":24,"Abe":71,"Aba":68,"Abd":28,"Ada":49,"Ado":41,"Ade":35,"Aet":37,"Aga":26,"Afr":330,"Afg":29,"Agn":27,"Air":36,"Al ":187,"Aka":25,"Ala":402,"Alb":169,"An ":1075,"Ali":70,"Alj":93,"Alc":41,"Ale":147,"Alf":49,"Alt":68,"All":74,"Alo":28,"Alp":44,"Ame":389,"Amb":31,"Ama":131,"Ams":36,"Ang":158,"Anj":44,"Ana":144,"And":189,"Anv":85,"Ant":227,"Ann":136,"Aod":213,"Aoz":43,"Ar ":2118,"Aos":340,"Bul":44,"Bun":25,"Bur":93,"Bue":26,"Bry":28,"Bru":240,"² ":87,"Cad":56,"Cae":102,"Cal":419,"Cam":89,"Cas":109,"Car":279,"Cat":102,"Can":103,"Cap":77,"Bea":77,"Bez":111,"Bev":81,"Bes":27,"Bet":78,"Ber":307,"Ben":164,"Bel":409,"Bei":28,"Beh":46,"Beg":25,"Bed":301,"Bib":28,"Bho":30,"Biz":33,"Bil":34,"Big":27,"Bih":155,"Bis":50,"Blo":51,"Ble":54,"Bla":69,"Bre":1870,"Bra":444,"Bro":1738,"Bri":270,"Bod":38,"Bob":30,"Boh":28,"Bol":142,"Bon":77,"Bor":114,"Bos":60,"Bot":49,"Bou":215,"Cyn":66,"Cun":40,"De ":65,"Der":69,"Des":46,"Dev":53,"Deu":57,"Dei":45,"Del":63,"Dem":103,"Den":92,"Dea":33,"Deg":26,"Da ":59,"Dam":34,"Dan":165,"Dao":76,"Dar":80,"Das":35,"Dav":72,"Dag":36,"Dal":37,"Cho":45,"Chr":125,"Che":89,"Chi":180,"Cia":24,"Chu":34,"án":119,"áp":29,"ái":34,"Cit":26,"ár":44,"ân":32,"ât":38,"âr":214,"ão":73,"Châ":25,"Cle":39,"Cla":74,"Cel":27,"Cen":52,"Cer":110,"à ":84,"á ":100,"ße":42,"Cha":389,"Cri":33,"Cra":30,"Cre":37,"Cro":36,"Clu":45,"Ös":26,"Cos":59,"Cor":145,"Com":218,"Col":101,"Coo":25,"Con":166,"Cou":45,"アアア":142,"ôg":720,"ôn":58,"ór":29,"ón":140,"ñs":2047,"ñv":891,"ó ":30,"ñc":70,"ña":31,"ñe":28,"ñj":75,"ñi":45,"ño":24,"ñ ":9866,"în":118,"Dyf":26,"ín":68,"ío":42,"ís":37,"ï ":28,"ía":79,"Egi":77,"í ":88,"êr":2927,"êt":62,"él":92,"éo":59,"ép":36,"ém":53,"én":95,"és":71,"ét":52,"ér":121,"év":46,"Edo":32,"éa":40,"éd":33,"éc":28,"ée":94,"Edw":28,"ég":45,"èn":34,"èr":89,"ès":41,"èg":41,"ê ":35,"Ecu":41,"Ech":27,"ço":86,"ça":53,"é ":426,"Ebr":307,"FE ":165,"Eas":54,"än":24,"ān":45,"Dia":97,"Dic":35,"Dis":208,"Dio":61,"Din":67,"Dim":39,"Dil":24,"Die":74,"Diz":31,"Diw":77,"Div":53,"Ἀ":34,"Dug":54,"Dub":25,"Dun":35,"Dul":31,"ể":119,"ün":95,"ül":25,"üs":76,"ür":103,"üt":25,"üh":27,"Dro":31,"ая ":51,"Du ":257,"ù ":12666,"Dre":155,"Dra":31,"ú ":25,"ör":30,"ön":34,"öl":28,"Dou":218,"Don":216,"Dom":60,"Dor":74,"Ned":28,"Nei":28,"Nev":419,"Neu":75,"Nes":32,"Ner":24,"Nep":142,"Ne ":61,"Nat":66,"Nav":46,"Naz":46,"Nij":37," Ἀ":34,"Nig":136,"Nie":35,"Nic":62,"Nis":25,"Nin":27,"Nik":43,"Nil":32,"New":165,"Mya":136,"Nao":129,"Nap":76,"Nar":28,"Nam":105,"Nan":35,"Nag":59,"Nac":31,"Naf":83,"Nou":26,"Nov":31,"Nor":936,"Not":36,"Nob":26,"Ofi":38,"Oki":39,"Oke":28,"PA ":54,"Obe":44,"Obl":33,"Ott":28,"Our":31,"Oug":38,"Ouz":52,"Owa":32,"ī ":39,"Oli":92,"Kêr":116,"Ore":54,"Ori":92,"Plo":191,"Ple":106,"Pla":137,"Pin":52,"Pis":27,"Pir":97,"Pie":124,"Pic":33,"Pho":45,"Phi":80,"Pha":29,"Peu":34,"Ped":27,"Per":261,"Pet":89,"Pem":45,"Pen":568,"Pep":35,"Pel":55,"Pat":67,"Pas":301,"Par":587,"Pau":112,"Pad":32,"Pan":90,"Pap":281,"Pao":63,"Pal":137,"Pak":62,"Pa ":68,"ōn":33,"ōg":26,"Pyr":31,"Pue":30,"Pro":300,"Pri":187,"Pre":189,"Pru":64,"Pra":117,"Poa":24,"Pob":66,"Pol":187,"Pom":34,"Pon":112,"Pot":43,"Pos":46,"Pou":45,"Pop":34,"Por":207,"Pow":35," ال":101,"Rac":24,"Rad":28,"Ram":45,"Ran":131,"Mün":49,"ū ":25,"Qui":31,"Que":38,"Isa":61,"Ita":306,"Isl":122,"Isr":84,"Ist":57,"Ira":83,"Iwe":203,"Ive":180,"Iza":28,"Ize":269,"Jaf":31,"Jac":102,"Jap":114,"Jan":80,"Jam":74,"Jak":53,"Jen":25,"Jer":130,"Jez":68,"Jea":161,"あ":41,"Jos":134,"Jor":74,"Jon":50,"ア":232,"Joh":156,"Joa":30,"Jud":39,"Jua":30,"Jus":24,"Jul":90,"Kad":30,"Kae":24,"Kab":38,"Kam":138,"Kal":163,"Kao":26,"Kan":725,"Kat":130,"Kas":218,"Kar":359,"Kaz":45,"Kav":60,"Kev":87,"Ker":517,"Ken":236,"Kel":56,"Kem":388,"Keb":26,"Kir":52,"Kin":57,"Kip":33,"Kim":33,"Klo":52,"Kle":50,"Kla":37,"Kon":258,"Kom":385,"Kol":122,"Kor":327,"Kou":52,"Koz":35,"Kre":355,"Kra":64," ア":26,"Kri":101,"Kro":256,"Kub":35,"Kum":133,"Kur":37,"Kuz":44,"Lev":68,"Les":97,"Leo":254,"Len":235,"Lem":27,"Lei":51,"Lee":34,"Lec":60,"Led":43,"Lau":43,"Lav":32,"Laz":27,"Le ":163,"Lak":70,"Lag":26,"Lat":40,"Lar":56,"Lao":55,"Lap":40,"Lam":58,"Lan":301,"Lad":25,"Lab":87,"La ":252,"Liè":30,"Lla":63,"Lly":57,"Lib":95,"Lia":47,"Lic":25,"Lig":265,"Lil":27,"Lim":38,"Lin":62,"Lis":51,"Lit":44,"Liv":31,"Leó":27,"Liz":35,"Lus":43,"Luk":34,"Lui":42,"Lun":30,"Lud":36,"Luc":61,"Lou":225,"Los":44,"Lot":45,"Loi":45,"Loe":103,"Lod":29,"Loc":62,"Lor":83,"Lop":24,"Lon":123,"Lom":37,"Lok":34,"Mei":50,"Men":173,"Mel":118,"Mes":68,"Mer":275,"Meu":589,"Met":52,"Mec":298,"Meg":27,"Med":94,"Mez":157,"Man":225,"Mao":80,"Mal":272,"Mar":1145,"Mas":64,"Mag":82,"Mad":169,"Mae":313,"Mak":35,"Mah":26,"Mai":71,"Mac":91,"Mab":145,"May":42,"Maz":26,"Mau":69,"Mat":103,"Mod":41,"Moh":29,"Mol":46,"Mon":319,"Mos":55,"Mor":620,"Mou":67,"Mik":53,"Mid":40,"Mig":28,"Mic":152,"Mir":97,"Mis":48,"Mil":93,"Min":123,"Mun":35,"Mur":34,"Mus":85,"çoi":70,"ège":37,"XXv":30,"Wor":34,"Wol":42,"Wil":179,"Win":40,"кая":36,"ère":70,"Wei":43,"Wel":28,"Wer":38,"Wes":242,"Wen":65,"São":44,"ène":27,"Was":35,"War":185,"Wat":32,"Wal":100,"ès ":37,"Vve":56,"ée ":61,"Vre":153,"Vro":82,"ées":30,"Vol":32,"Vis":41,"Vit":32,"Viê":40,"Vla":26,"éli":30,"éra":34,"éri":30,"éné":31,"Zel":58,"éon":34,"Zeu":41,"és ":43,"之":110,"並":62,"丘":32,"专":41,"三":151,"丁":73,"Yve":30,"Yun":35,"Yuz":30,"Yor":126,"You":64,"ə ":47,"ən":29,"Yan":129,"Yao":137,"êr ":2352,"на ":30,"êt ":44,"êrb":145,"êri":399,"Yez":130,"Xve":91,"Syl":30,"Stê":62,"Swa":35,"Sve":120,"Sur":43,"Sus":66,"Sum":29,"Sul":65,"Sun":33,"Sui":93,"Sua":196,"Str":447,"Stu":77,"Sti":57,"Sto":80,"Sta":876,"Ste":130,"Ten":39,"Tem":32,"Teo":30,"Tei":34,"Tel":40,"Tch":179,"ão ":72,"Tam":45,"Tan":101,"Tar":82,"Tai":60,"Tal":111,"Tad":57,"UA ":109,"Ski":27,"Sko":315,"Skr":81,"Ska":32,"Ske":30,"She":24,"Sha":39,"Sim":44,"Sil":48,"Sik":59,"Sig":31,"Sir":75,"Sin":458,"Sie":35,"Sib":58,"Sic":35,"Ser":108,"Set":39,"Sev":58,"Sen":87,"Sel":55,"Sei":79,"Sea":28,"Su ":163,"Spa":218,"Spi":30,"Spe":42,"Sok":47,"Soc":33,"Shō":35,"Sou":218,"Sov":76,"Sol":47,"Som":59,"Son":59,"Sor":31,"Slo":44,"Smi":34," 三":55,"Ruz":48,"Rus":272,"Sai":243,"Sak":41,"Sam":85,"Sal":156,"Sac":54,"Sab":28,"Sco":35,"Sci":29,"Sch":148,"Sav":198,"Sau":25,"Sar":106,"San":782,"Sao":292,"ови":34,"SI ":31,"Res":34,"Ret":176,"Reu":24,"Rev":40,"Rho":37,"Rhe":37,"Rio":64,"Riv":24,"Ric":118,"Rhy":27,"Rec":25,"Red":61,"Rei":69,"Reg":63,"Ren":134,"Rep":458,"Rol":26,"Roe":42,"Roa":103,"Rob":85,"Roc":84,"Rod":47,"Roy":27,"Rou":398,"Rot":44,"Ros":112,"Ron":40,"Rom":236,"Rop":51,"SS ":38," 之":38,"SO ":28,"Vel":35,"Ven":128,"ска":39,"Van":75,"Val":141,"Var":74,"Vig":36,"Vic":64,"Vie":116,"Vir":42,"Vil":82,"Vin":104,"Ver":120,"Veu":295,"Ul ":233,"Ukr":40,"Uhe":126,"VIv":68,"Una":1019,"Uni":82,"Unv":67,"Un ":607,"Uru":37,"Ur ":2862,"VI ":43,"Teu":28,"Tex":28,"Ter":100,"Tet":30,"Tes":38,"Tha":76,"The":234,"Thi":28,"Tho":61,"Tib":58,"Tie":85,"Tim":40,"Tin":33,"Tit":30,"Tir":57,"Pêr":43,"Tor":68,"Tol":28,"Tom":36,"Ton":31,"Tos":58,"Tou":94,"çai":30,"Tro":186,"Tri":158,"Tre":263,"Tra":119,"Tur":141,"Tun":31,"Tud":24,"ßen":37,"вич":50,"ああ":28,"bja":26,"bl ":407,"biz":216,"bis":45,"bit":50,"bio":107,"bir":75,"bik":31,"bil":84,"bin":157,"big":45,"bih":300,"bo ":54,"blo":711,"ble":426,"bli":615,"bla":374,"bod":104,"boa":63,"bob":195,"bok":24,"bol":151,"boe":45,"biñ":26,"br ":57,"bon":174,"bom":59,"bor":273,"bot":53,"bos":68,"bou":909,"be ":93,"bam":27,"ban":1032,"bak":29,"bal":315,"bai":138,"bag":46,"bah":28,"bae":46,"bac":146,"bad":553,"bab":75,"án ":76,"baz":28,"bay":39,"bau":28,"bat":179,"bas":257,"bar":1084,"bao":351,"bea":32,"bi ":70,"bei":36,"beg":400,"bed":495,"bec":37,"ber":2194,"bep":146,"ben":1724,"bem":27,"bel":314,"bek":240,"bez":495,"beu":260,"bev":173,"bes":62,"bet":4067,"ápa":25,"bia":300,"bic":30,"bid":59,"bie":109,"bha":32,"ca ":318,"car":108,"cas":148,"cat":71,"cau":32,"can":105,"cap":50,"cad":36,"cam":25,"cal":102,"ce ":316,"bri":446,"bro":1440,"bra":977,"bre":2698,"bu ":43,"bru":350,"bsb":24,"boù":29,"bua":43,"bur":380,"bul":68,"bun":41,"buh":76,"bug":103,"but":41,"bus":55,"by ":41,"bwe":29,"aka":732,"am ":551,"ake":317,"aki":154,"akh":42,"aji":29,"ajo":39,"al ":3797,"aja":58,"aje":72,"aii":42,"aik":32,"ail":319,"aim":51,"ain":893,"aio":50,"air":137,"ais":419,"ait":72,"aiw":30,"ak ":494,"aig":79,"aie":120,"aid":145,"aia":64,"ahi":68,"ahu":48,"ahr":32,"aho":95,"aj ":136,"ahe":52,"aha":329,"agg":35,"agh":38,"agi":75,"agr":25,"agu":180,"agn":1483,"ago":271,"aog":68,"aol":416,"aok":392,"aod":271,"aoe":315,"anv":4249,"anu":193,"anz":183,"any":89,"ano":519,"ann":3597,"anm":210,"ant":14921,"ans":274,"ane":6479,"anf":35,"ang":1077,"anh":35,"ani":1074,"anj":41,"ank":665,"ap ":252,"ana":1898,"anb":31,"anc":428,"and":2346,"amu":63,"amt":28,"amz":526,"amm":715,"amo":419,"amp":357,"ams":92,"amk":30,"amh":31,"ami":521,"amd":32,"ame":864,"amb":405,"ama":2994,"ao ":87,"alz":151,"alv":701,"alu":178,"alt":325,"als":46,"alp":40,"alo":509,"alm":105,"all":4383,"alk":84,"alg":71,"ali":1896,"alj":57,"alc":477,"ald":244,"ale":1440,"ala":2477,"alb":121,"an ":17839,"aks":123,"akr":86,"aku":87,"akt":174,"ako":115,"aba":962,"abe":777,"abi":225,"abj":25,"abl":133,"abo":381,"abr":134,"abs":38,"abu":59,"ae ":1012,"aca":108,"aaa":146,"aan":41,"aal":34,"aas":35,"aar":52,"aat":389,"ad ":6605,"ac ":497,"aa ":55,"ab ":625,"afo":34,"afr":170,"aff":71,"afe":127,"afi":38,"ai ":239,"aga":729,"agd":27,"age":701,"aen":254,"ael":271,"aes":112,"aer":1001,"aeg":510,"aed":61,"aek":374,"ah ":119,"âte":26,"afa":255,"aet":1430,"aez":325,"ado":1590,"adr":147,"adh":35,"adj":43,"adi":235,"add":93,"ade":2997,"ag ":6969,"adw":46,"ady":24,"ads":127,"adt":40,"adu":2557,"adv":56,"acq":55,"aco":101,"ack":112,"aci":183,"ach":1027,"ace":95,"acc":47,"ada":865,"af ":45,"âre":203,"act":55,"acu":55,"acr":44,"azo":96,"azi":660,"azt":49,"aze":890,"azh":640,"aza":204,"azz":51,"axi":40,"az ":790,"ayo":91,"aym":25,"ays":58,"aya":179,"aye":142,"ba ":235,"bc ":24,"aqu":35,"at ":1878,"arg":317,"arf":40,"are":2408,"ard":882,"arc":571,"arb":175,"ara":1504,"arp":87,"aro":893,"arn":1123,"arm":342,"arl":474,"anç":111,"ark":510,"ari":1893,"aru":121,"arv":1540,"arw":27,"arx":54,"arr":912,"ars":166,"art":2208,"au ":260,"asa":726,"ary":114,"arz":1296,"asi":144,"ash":125,"asc":74,"ase":577,"aso":106,"asp":56,"ask":546,"asm":40,"aon":341,"aoo":69,"aot":235,"aou":1726,"aor":81,"aos":156,"ar ":24115,"aoz":2346,"apa":418,"alá":31,"ape":187,"api":98,"aph":42,"apl":49,"apo":648,"app":89,"apr":28,"aps":87,"apt":56,"apu":61,"as ":2697,"ava":1022,"ax ":65,"aux":32,"aut":155,"arí":35,"avr":39,"avo":170,"avi":298,"ave":2923,"awe":103,"ay ":311,"awa":237,"awr":48,"awi":50,"av ":320,"aoù":110,"ata":1010,"asu":48,"aoñ":77,"ast":1024,"ass":241,"asy":30,"atl":96,"atr":172,"ato":465,"ate":653,"atc":36,"ati":1306,"ath":184,"aw ":56,"auc":67,"aub":40,"att":149,"ats":44,"atu":213,"aul":182,"aum":36,"aun":74,"aur":161,"aus":113,"aud":112,"aug":24,"νη":32,"νο":29,"ολ":24,"πο":32,"ος":144,"ια":35,"ιο":30,"λι":25,"λα":24,"μα":24,"Wür":24,"ει":25,"ος ":144,"ης":30,"αι":28,"αλ":29,"αν":33,"αρ":25,"ία":31,"ς ":295,"ν ":51,"ι ":25,"η ":101,"Zub":27,"Vîn":53,"α ":96,"アア":185,"ρος":25,"ий ":24,"ων":25,"ич ":51,"το":28,"στ":31,"σι":30,"ρα":33,"ρο":43,"jed":50,"jer":491,"jek":59,"jen":344,"jet":1133,"jañ":33,"jev":42,"ji ":46,"νη ":24,"jk ":28,"jad":45,"jab":30,"eüs":65,"jap":102,"jar":45,"jal":43,"jan":113,"je ":203,"jou":81,"jon":49,"jor":36,"jpg":45,"ск":72,"jio":97,"jin":152,"jik":26,"jie":63,"jia":71,"jib":29,"ти":34,"та":24,"ст":55,"jo ":26,"itl":152,"itr":124,"ito":200,"itu":156,"itt":128,"its":52,"itz":70,"ity":61,"isk":433,"ism":94,"isl":110,"iso":129,"isn":43,"isp":496,"iss":497,"isr":129,"isu":29,"ist":3083,"ioù":3206,"iv ":786,"ita":909,"itc":84,"ite":544,"ith":138,"iti":736,"ivr":37,"ivo":342,"ivy":27,"iwe":401,"iwa":1260,"ius":378,"iur":33,"ium":58,"eñv":618,"iva":1287,"ivc":49,"ix ":114,"ivi":262,"ive":2609,"ipr":52,"ipo":75,"ipp":126,"ipu":31,"ipt":148,"ipi":84,"ipl":28,"is ":2724,"ion":1454,"iop":130,"ior":153,"ios":108,"iot":56,"iou":688,"iog":34,"iol":67,"iom":41,"ipa":143,"ipe":76,"iov":38,"ir ":701,"iru":31,"irv":32,"irr":117,"iro":258,"irm":24,"iri":682,"isi":977,"ish":314,"ise":310,"isc":169,"isb":24,"isa":291,"eñ ":32,"iqu":102,"ire":787,"irg":98,"ira":461,"ird":151,"irc":108,"it ":2544,"ünc":44,"ja ":94,"iya":79,"iye":30,"iz ":5713,"eón":26,"iwo":27,"izu":26,"izo":212,"izi":1082,"izh":3028,"ize":643,"izd":61,"iza":353,"kig":118,"kim":79,"kil":109,"kia":534,"kic":156,"kib":31,"kie":104,"kiz":138,"kl ":27,"kin":273,"kip":95,"kir":270,"kis":131,"kit":175,"keñ":73,"km ":262,"ki ":169,"kgr":43,"kha":110,"kho":57,"kea":37,"ked":186,"kef":27,"keg":356,"kei":93,"kek":447,"kem":714,"kel":791,"keo":32,"ken":1794,"kes":29,"ker":582,"kañ":225,"keu":150,"ket":897,"kev":353,"kez":64,"ke ":191,"kra":328,"kre":866,"kt ":39,"ksa":52,"kse":61,"ku ":65,"kro":809,"kri":2084,"kr ":32,"koz":381,"kov":45,"km²":85,"kot":54,"kou":3139,"kos":289,"kor":683,"kop":87,"kon":591,"kom":3155,"kol":453,"kog":42,"koe":63,"kod":39,"ks ":105,"ể ":40,"klê":29,"kiñ":43,"kme":37,"koa":172,"kob":78,"kni":26,"gêr":1036,"kke":25,"kka":30,"ko ":451,"kle":274,"kla":324,"klo":157,"jus":37,"jvi":55,"joù":51,"ển":39,"kaz":128,"kav":240,"kay":33,"kat":219,"kar":717,"kas":328,"kap":68,"kan":1716,"kao":135,"kal":381,"kam":95,"kai":52,"kae":493,"kad":451,"kac":24,"kab":44,"kaa":150,"ka ":1336," Ga":1180," Ge":925," I ":68," Fo":346," Fu":93," Fr":1201," Fi":320," Fl":419," Bé":40," Ha":1090," He":1698," Gw":1125," Gy":27," J ":114," Go":857," Gr":735," Gu":336," Gh":58," Gi":573," Gl":175," Ig":31," Id":54," Ic":25," Ib":34," Ia":193," K ":68," Hy":84," Hu":232," Ho":413,"ha ":7925," Hi":350," Ji":83," Je":435," L ":45," Ja":555," Iz":312," Iv":30," Iw":209," Ir":161," Is":368," It":343," Im":318," In":1030," Io":58," Ip":79," Il":238,"ham":356,"han":1921," M ":27,"hao":51,"hap":104,"hai":128," Ka":2025,"hak":57,"hal":1676,"hau":60," Ke":1373,"hav":45," Ki":281,"har":1216,"has":180," Kh":45,"hat":349," Jo":527,"haf":27," Ju":260,"hae":120,"hag":6703,"hab":93,"had":772,"hac":45," N ":136," La":1222," Le":1251," Li":860," Kl":163," Kn":26," Ko":1313," Kr":777,"haz":119," Ku":314," Ky":42," Mc":41," Ma":3069," O ":70," Mi":710," Me":1982,"he ":1293," Lo":976," Hé":41," Ll":170," Ly":47," Lu":361," Ne":1118," P ":29,"а ":89," Na":819," Ni":468," Mo":1327," My":178," Mu":252,"hek":226,"hel":1281,"hei":155,"heg":164,"heh":30,"hee":27,"hef":31,"hec":228,"hed":545,"hea":87,"heb":137,"hez":301,"hev":363," A ":247,"hañ":1086,"heu":277,"het":353,"hes":174,"her":2530,"hep":235,"heo":107,"hen":1633,"hem":192,"hi ":652," B ":90," C ":1547," Ao":672," Ap":113," Am":685," An":2165," Ak":92," Al":1478," Ai":112,"hho":32," Ag":119," Ah":33," Ae":93," Af":391," Ac":90," Ad":207," Ab":268," Ba":1051," D ":80," Az":390," Ay":27," Av":131," Au":240," At":426," As":207," Ar":3347," Aq":24,"hig":75," Be":1711,"hie":591,"hid":70,"hic":42," Bi":459," Bh":52,"hia":407,"hip":35,"hio":751,"hin":985,"him":81," Bl":207," Bo":879,"hil":302,"hik":334," Br":4609,"heñ":300," Bu":358,"his":80,"hit":151,"hir":469," E ":710," Ca":1420,"hiz":309," Ce":271," Ci":142," Ch":923," Cl":231," Cr":170," Co":876," Cu":142," Cy":124," F ":30," Da":708,"dé ":30," Di":920," Dh":31," Dj":29," De":772," Dr":253," Do":662," Dy":40," Du":514," Ea":86,"hn ":80," Eb":324," Ec":91," Ed":137," G ":24,"hla":88,"hle":80," El":312," Ek":28," Ei":244,"hli":24," Eg":106," Et":209," Es":211," Er":900,"hlo":42," Ep":30," Eo":336," En":713," Em":541," Ez":35," Ex":30," Eu":753," Ev":283," Fe":315,"ho ":375,"hma":28," Fa":314,"gma":28,"go ":326," Xi":32,"glo":131," Xa":37,"к ":31,"gle":234," Wy":25,"gli":102,"gn ":547,"gla":326," Wo":96," Wi":334," Wh":28," Sã":44," We":489," Wa":422,"й ":42," Y ":37,"gog":60," Vî":53,"goe":139," Zu":66,"god":130,"gob":25,"goa":87," Zo":34,"gnu":34," Ze":158,"gno":502," Zi":46,"gni":42," Za":103," Ys":24,"gne":835," Yv":36," Yu":107,"gna":107," Yo":219," Ya":330,"giñ":44," Ye":198,"gs ":28," Wü":24,"goz":125,"gom":197,"н ":39,"gol":442,"gon":445,"gop":26,"gos":176,"gor":446,"gou":1457,"got":31,"gu ":30," a ":21743,"р ":30,"gro":187,"gra":972,"gri":140,"gre":811,"в ":40," Kö":34," Ow":50," Ou":210," Os":107,"gto":52," Ot":70," Or":316," Op":33," Po":935,"gui":62," Lé":26," Pl":484,"gum":1179," Pi":431,"gul":63," Ph":183," Pf":25,"gua":208," Pe":1214,"gue":151," Pa":1928," Ny":33," Nu":54," No":1153," Ol":141," Ok":98," Kê":117," On":29," Om":48," Og":35," Oi":37," Od":68," Oc":46," Of":53,"goù":281," Ob":100,"gta":158," Oa":47," Ra":453," Mü":57," Qu":112,"gwi":305," Ro":1377,"gwe":769," Re":1194,"gwa":450," Ri":357," Rh":169," Py":52,"guz":57," S ":53," Pr":893,"gur":63," Ps":36,"gus":176," Pt":32," Pu":129,"gun":35," Qi":24," Mé":39," Sy":84," Sw":45," Sv":127," Su":797," St":1757," Tc":179," Ta":642," V ":60," Th":473," Ti":377," Te":498," Tr":768," Pê":42," To":425,"gwr":80," Ru":419," Sa":2186," Sh":170," Si":919," Sc":261," Se":617," So":661," Sp":344," Sr":25," Sk":511," Sl":64," Sm":46," Va":397,"и ":29," Ve":661," Vi":639," Vl":43," Ré":29," Rí":30," Vo":153," Vu":27," Vr":262," Tu":257," Ty":52," Uh":133," Uk":44," Ul":286," Un":1810," Ur":2995," Ut":31," ja":187,"iak":59,"ь ":37," l ":123,"iam":250,"ial":435,"iao":55," iz":369,"ian":2252," ji":41,"ias":133,"iar":101," je":186,"iat":251,"iav":48," im":1409," in":589," il":111,"ic ":292," ij":104," iw":110," eñ":47,"iab":139,"iac":49," iv":1031,"iad":1539," is":1920," it":397,"iae":74,"iag":66," ir":67,"ibl":171,"ibi":176," ka":2835,"ibo":69," m ":227," kh":68,"ibr":42," ki":623,"ibu":38," ke":3570,"iaz":346," jo":32," jp":45,"id ":450,"iba":203,"ibe":842," ju":73," gw":1502," ha":17502," he":5230," gi":303," gl":364," gr":1597," go":1953," gu":1259,"ia ":5683," ib":41," id":83," hh":31,"ib ":56," hi":2337," hl":138," dé":33," ho":3333," hr":812," hs":28," ht":49," hu":259," hv":30," hw":461,"iet":779,"iañ":362,"ieu":132," ni":874,"iez":2096," ng":65," nd":30,"iel":957,"iem":34," ne":1817,"ien":1604," na":567," p ":133,"ier":945,"ies":635,"ied":391,"ieg":782,"iek":656," mu":464,"ig ":786," mo":1518," ok":126," ol":37," om":64," on":28," kê":858,"ifo":196," oc":81," of":821," oa":4658,"ifr":40," ob":921,"ife":146," ny":25,"ifi":60," nu":87," no":744,"ifa":82," le":3053,"icr":29,"ics":47,"ict":94,"icu":113," li":1853," n ":401,"ico":205,"ick":115," la":2328," ku":562,"ici":153,"ich":835,"icc":26,"ice":192," kw":51," gê":983," km":359,"ie ":398," kl":472,"ica":324," kr":1806," ko":5162," me":2877," mi":1053,"idu":47,"idt":41," o ":1644,"idr":25,"я ":78,"ido":268," ma":4658," mb":31,"idj":93," lu":242,"idi":732,"idg":33,"ide":470,"idd":28,"ida":806," lo":922," hê":99,"if ":38," ae":329," af":85," ag":91," aa":47," ab":707," ac":52,"iid":64," ad":591," am":775," an":17401," ao":767," ap":227," ai":30," ak":164," al":3609," av":170," au":58," aw":48," ar":20177," at":205," as":260," d ":4135," ba":1861," az":121," ay":27,"il ":1052,"ija":76," bi":786,"ije":1542," be":6354,"iji":263,"ijk":37," bo":1153," bl":938,"ijo":76," bu":368," br":3925," ca":216," e ":25254,"im ":142,"ika":1513,"igd":59,"ige":995,"iga":163,"ii ":140,"igl":24,"igh":464,"igi":116,"igu":89,"igr":67,"igo":308,"ign":346,"ij ":175," b ":27,"ihe":29,"iha":639,"ik ":1216," c ":4919," er":3046,"imo":147," et":1614,"imm":32," es":229," en":8597," em":1862,"imp":1483," eo":8201," ei":428," el":392,"ime":314," ek":63," ee":65,"imi":166," eg":182," fe":423,"ip ":40,"inc":134," h ":3324,"ind":570,"ina":1494,"inb":38," fa":598," ez":592,"imu":37," eu":10144," añ":114," ev":3777," fu":86,"inn":252,"ino":647," fr":528,"int":619," fo":762,"ins":144,"inf":56,"ine":1835," fl":235,"inh":55,"ing":949," fi":471,"ini":1294,"inl":79,"ink":143," ge":2883," ga":11426,"ioc":83,"iod":24,"inu":125," i ":72,"inv":58,"inw":47,"iny":86,"inz":27," cl":32,"iko":750," cm":63,"ikl":38," co":306," cr":36,"iki":142," ce":115," ch":782,"ike":750," ci":27,"ila":338,"ilb":25," da":8512,"in ":2762," cu":44,"ikt":69,"iku":27,"iks":110," do":1895,"ilo":265,"ill":771," dr":4372,"ilm":190,"ilh":762,"ilg":84," de":7284,"ili":920,"ild":111," dj":44," di":6211,"ile":746,"ima":192,"imb":131," g ":57,"ч ":54," ec":181," ed":82,"io ":457," ea":36," eb":166," du":1193,"ilt":54,"ilu":55,"ilv":37,"ль":30,"ми":24," zo":10353,"ла":32," zu":52,"hpe":153,"ле":32," vî":50," zr":49,"ли":31,"hok":47,"hol":1123,"hom":566,"hon":2468,"hog":52," za":226,"ко":31,"hos":71," ze":301,"hot":72,"hou":757,"hov":24,"hop":42," zi":383,"hor":720," yo":88,"ка":66," yi":44,"hoa":863," yu":97,"ки":34,"hof":24,"hoe":202,"hod":73,"hoc":104," ye":6259,"hno":26," ya":550," z ":116,"ин":40,"hmi":26,"ик":30,"ий":28,"ич":63,"ис":27,"ри":28,"hul":35,"hui":68,"hug":38,"ро":50,"hue":39,"hud":36,"hua":110,"htt":30,"htr":24,"hth":32,"hte":44,"hoù":3009,"ор":27,"hsk":47,"hse":48,"ол":26,"ом":26,"ов":76,"hu ":78,"но":24,"hro":118,"hre":688,"ни":34,"hri":212,"ht ":446,"на":57,"hra":123," ru":826," u ":26," sa":3355," se":1968," sc":29,"hya":32," si":1161," sh":141,"hyn":30," sl":74," sk":4322," sp":1119," so":1278," qu":41,"ви":64," t ":34," ra":3095," re":3577," ri":679," ro":2258," pu":120," pr":1459," s ":93," px":514,"hy ":25,"hwa":53,"hwe":448,"hwi":38," ot":171,"hum":675," ou":1145,"hun":195,"hus":124," op":32," or":427,"hur":95,"ан":44," r ":35,"huz":52," pe":8359," lâ":203,"hve":93," pa":1775,"ар":50," pl":522," po":1472,"ая":51," pi":232," ph":49," wa":3108," we":680," y ":92," wr":58," wo":36," wi":155," x ":44," va":783," ve":5168," vo":2288," vr":1722," vu":274," vi":3544," vl":184,"ес":33," tu":746," us":51," ur":9744," un":6030," uk":30," ul":727," ug":49," uh":238," ta":1167," tc":512," v ":32,"hys":42," sy":28," st":3800," sv":112," su":738,"ев":42," tr":3171," ts":55," to":979," th":740," ti":1088," te":1484,"ffu":30,"ffr":34,"fi ":42,"ffe":36,"ffa":40,"ffi":49,"feu":64,"fet":66,"fes":65,"fer":173,"fed":176,"fem":46,"fen":488,"fel":95,"fei":43,"faz":24,"fas":26,"far":201,"fam":247,"fan":218,"fal":202,"faf":76,"fae":67,"fad":24,"ff ":48,"fe ":207,"fa ":121,"euß":32,"eye":26,"exa":64,"ez ":11148,"exi":24,"ezv":39,"ezu":194,"eza":980,"ezo":388,"ezs":35,"eze":3326,"ezh":15689,"ezi":1049,"eta":840,"ete":2318,"etc":33,"eti":591,"eth":157,"etn":58,"etk":85,"esp":93,"esn":37,"eso":125,"est":1767,"ess":313,"esw":24,"eoù":227,"ev ":1299,"eua":81,"añc":45,"euc":54,"eub":98,"eue":418,"eud":347,"eui":37,"euk":25,"añj":24,"eum":58,"eul":367,"eun":223,"eto":294,"etr":1675,"ets":34,"ett":196,"etu":68,"ew ":174,"eve":3071,"eva":742,"evo":119,"evn":657,"evi":2428,"añv":131,"euv":34,"euw":32,"eut":225,"eur":2890,"añs":950,"eus":10810,"ex ":155,"euz":639,"eux":42,"ewi":41,"ewe":45,"evr":1345,"ey ":241,"ewa":59,"epe":148,"epi":164,"eph":135,"er ":13257,"epa":1461,"eou":101,"eot":99,"eos":32,"eor":152,"eom":25,"eol":459,"eop":50,"eon":456,"aï ":27,"es ":2160,"ept":42,"epu":540,"epk":92,"epm":31,"epp":25,"erk":270,"erl":341,"eri":3029,"erg":379,"erh":59,"ere":4269,"erf":51,"erc":973,"erd":244,"era":623,"erb":525,"et ":24274,"esk":805,"esl":29,"esh":133,"esi":729,"esb":38,"esc":99,"esd":28,"ese":255,"añ ":8904,"eu ":196,"esa":1077,"erz":1304,"ery":87,"erv":1357,"eru":118,"erw":40,"err":913,"ert":421,"ers":473,"ern":1502,"erm":1055,"ené":32,"erp":58,"ero":907,"eki":62,"ekk":24,"ekn":35,"eko":95,"ekr":46,"eks":87,"ekt":111,"eku":26,"ekv":40,"en ":13354,"elb":25,"ela":722,"eld":150,"elc":91,"elf":231,"ele":2592,"eli":634,"elg":331,"elh":32,"elm":94,"elk":40,"ell":2765,"elo":483,"elp":30,"elu":36,"elv":411,"els":97,"elt":214,"ely":60,"elw":63,"eo ":8202,"emb":1318,"ema":1014,"emg":174,"eme":1638,"emd":78,"emm":295,"emo":323,"emi":315,"eiñ":116,"emp":387,"emr":84,"ems":145,"ep ":539,"emz":52,"enf":38,"ene":2818,"enh":96,"eng":662,"enb":206,"ena":1493,"end":451,"enc":313,"eno":539,"enn":14198,"enk":534,"enl":51,"eni":918,"enw":44,"enu":60,"env":509,"ens":884,"ent":3408,"enr":123,"enz":173,"eny":79,"eog":63,"eof":24,"eod":154,"eoc":58,"ego":514,"ege":1345,"egg":36,"egi":351,"eha":29,"egr":44,"egu":61,"eho":118,"ehe":119,"ek ":10245,"eib":27,"eic":80,"eia":58,"eis":824,"eir":312,"eim":59,"eil":495,"ein":413,"eik":24,"eie":159,"eid":119,"eig":28,"eif":24,"eja":35,"el ":8480,"eiz":3938,"eit":125,"eke":323,"eka":196,"em ":600,"ejv":55,"gl ":37,"giz":64,"geñ":32,"git":101,"gis":73,"gir":65,"gil":150,"gim":29,"gip":131,"gin":251,"gio":137,"gid":69,"gie":386,"gia":396,"ght":390,"ghe":49,"gha":96,"ggi":48,"gge":26,"gga":25,"gi ":107,"gen":1791,"geo":127,"get":861,"geu":25,"gañ":198,"ger":1658,"ges":96,"gev":466,"gh ":112,"gez":763,"gea":34,"gee":129,"ged":397,"gei":27,"geg":174,"gef":25,"gem":219,"gel":631,"gek":101,"gda":29,"gde":64,"ge ":374,"gaz":228,"gab":25,"gac":227,"gad":295,"gae":60,"gai":52,"gas":335,"gar":551,"gau":35,"gat":92,"gaw":34,"gav":257,"gak":29,"gam":131,"gal":2464,"gao":44,"gan":9608,"ga ":256,"fyd":28,"fur":99,"foù":33,"fud":31,"ft ":72,"fra":395,"fre":158,"fri":574,"fro":118,"fou":46,"for":444,"fon":658,"fol":97,"fiñ":47,"fle":49,"fla":138,"flu":41,"flo":72,"fic":49,"fie":47,"fil":220,"fin":188,"fis":652,"fiz":82,"da ":6428,"dd ":287,"de ":2341,"dac":243,"dad":448,"dak":29,"dal":1050,"dai":58,"dag":91,"dah":25,"dae":431,"dat":150,"das":255,"dar":633,"dap":31,"dao":544,"dan":1203,"dam":247,"daz":25,"day":28,"dav":252,"dda":76,"dde":59,"ddi":92,"cul":78,"cum":26,"cua":46,"ctu":25,"cto":91,"cti":132,"cte":35,"cta":38,"cus":153,"cur":31,"cks":24,"cki":24,"ckh":38,"ckl":31,"cla":32,"cle":35,"co ":254,"ció":40," Ös":26,"coc":27,"con":207,"col":208,"com":88,"cor":91,"cos":24,"cop":59,"cot":43,"cou":64,"cs ":52,"cqu":76,"cre":28,"cra":43,"cri":37,"cro":75,"cci":49,"cch":32,"cca":25,"cea":54,"ch ":1041,"cer":68,"ces":90,"cen":91,"cep":71,"cel":179,"ci ":114," à ":31,"cha":1216,"chw":69,"chu":220,"chy":27,"cia":161,"ck ":309,"cie":45,"cid":63,"che":1551,"chl":55,"chi":619,"cho":838,"chm":39,"chn":35,"chs":66,"cht":150,"chr":27,"cil":43,"cis":63,"ciu":47,"cin":138,"cio":74,"cip":52,"cm ":60,"ckg":40,"cke":49,"ed ":5626,"eba":92,"ebe":535,"ebi":41,"ebl":61,"ebo":59,"ebr":613,"ebu":44,"ec ":2202,"eac":38,"eae":30,"ead":335,"eaj":43,"ean":370,"eal":95,"eam":25,"ear":95,"eas":306,"eat":155,"eau":133,"eb ":70,"ea ":600,"efi":24,"efl":30,"efo":31,"efa":48,"efe":192,"ei ":147,"ega":544,"efr":31,"eft":47,"eek":223,"een":103,"eel":114,"eeg":74,"eed":85,"eez":83,"eer":364,"eeu":74,"eañ":33,"eet":55,"edi":547,"edd":184,"ede":1648,"ône":24,"eda":435,"eg ":10353,"ης ":30,"edu":46,"edo":498,"edr":110,"eck":63,"ech":877,"eci":30,"eca":36,"ee ":124,"ef ":54,"ecu":28,"ect":87,"eco":53,"dwi":54,"dwa":71,"dy ":65,"dve":63,"dur":2497,"dus":115,"ôge":65,"ôg ":650,"dor":380,"don":562,"dom":53,"dol":90,"dok":40,"dov":47,"dou":1198,"dot":33,"dos":174,"dr ":37,"ds ":105,"diñ":81,"dne":34,"doa":651,"dob":26,"doc":36,"doe":85,"dof":83,"dog":66,"doù":1790,"dst":102,"dth":36,"dun":88,"dui":24,"dul":36,"due":53,"dug":195,"dua":33,"dud":657,"dri":262,"dra":338,"dt ":63,"dre":2258,"du ":415,"dro":2685,"dru":64,"dge":30,"dic":158,"did":92,"dia":1269,"dib":212,"dhi":25,"der":1355,"des":551,"det":713,"deu":1098,"dañ":173,"dev":161,"dez":1065,"dh ":36,"deb":164,"dea":43,"ded":268,"dec":29,"dee":39,"deh":79,"deg":839,"dei":311,"del":1180,"dek":575,"den":4056,"dem":143,"dep":1276,"deo":109,"di ":921,"dle":38,"dla":33,"do ":378,"dja":113,"dje":32,"div":643,"diu":39,"diw":1350,"diz":227,"dim":106,"din":658,"dio":629,"dir":425,"dis":1432,"dit":85,"die":331,"dif":129,"dig":697,"dij":34,"dik":146,"dil":186,"djo":47,"dji":63,"rgu":84,"rhe":42,"rha":43,"rho":37,"rga":269,"ri ":1054,"rgi":254,"rgh":25,"rge":313,"rgr":39,"rgo":143,"rgn":33,"ret":1781,"res":1419,"rev":451,"reu":637,"rañ":634,"rew":29,"rez":4363,"rey":46,"rfa":34,"rfe":82,"rfo":105,"rdu":42,"rds":36,"rdr":49,"rg ":501,"reb":334,"rea":765,"ree":432,"ref":61,"rec":185,"red":2063,"rei":3864,"reg":1192,"rem":696,"ren":3754,"rek":863,"rel":1453,"raí":24,"rer":622,"reo":489,"rep":96,"rf ":139,"rda":144,"rcu":51,"rct":28,"rdo":204,"rdi":421,"rde":436,"rdd":43,"re ":4982,"rbu":43,"rco":34,"rck":34,"rci":70,"rch":168,"rce":81,"rca":52,"raw":45,"rax":26,"ray":119,"raz":505,"rd ":675,"rao":698,"rap":82,"rar":100,"ras":1323,"rat":510,"rau":85,"rav":268,"rbi":84,"rbo":149,"rba":160,"rbe":368,"rc ":1530,"raj":38,"rai":247,"rah":111,"rag":290,"ran":3063,"ram":504,"ral":609,"rak":382,"rab":518,"raf":167,"rae":1511,"rad":877,"rac":186,"rs ":209,"rpe":55,"rpa":30,"rr ":2111,"rpi":29,"ror":29,"ros":301,"rot":175,"rom":591,"ron":1309,"roo":54,"rop":562,"roy":41,"roz":110,"rou":3431,"rov":876,"row":38,"rob":47,"roa":1313,"rod":180,"roc":330,"roi":658,"rol":1503,"rok":136,"rof":62,"roe":745,"rog":203,"rno":714,"rns":59,"rp ":29,"rna":543,"rne":675,"rni":416,"rmo":173,"rms":31,"riñ":374,"rmu":93,"ro ":7159,"rma":433,"rme":1192,"née":33,"rmi":74,"rlu":43,"rlo":138,"rli":304,"rle":290,"rla":227,"rn ":1159,"rkt":34,"rks":233,"rko":43,"nço":78,"rki":200,"rke":237,"nça":45,"rka":144,"rm ":168,"né ":43,"rji":52,"riy":25,"riz":637,"rix":34,"rl ":114,"rip":57,"rio":1388,"rit":468,"ris":758,"riv":1682,"reñ":136,"riu":84,"rig":797,"rij":35,"rii":29,"ril":176,"rik":1404,"rin":891,"rim":123,"ria":2221,"rib":276,"ric":406,"rid":500,"rie":2396,"rif":33,"rhy":25,"rk ":341,"rwe":28,"rwi":26,"rz ":165,"nôg":719,"rxi":44,"rya":37,"ryd":24,"rye":33,"ruj":46,"rui":38,"rug":115,"ruf":51,"rud":504,"ruc":43,"run":197,"rum":500,"ruz":176,"rus":640,"rut":37,"rva":250,"rvi":144,"rve":2286,"rvo":598,"rvu":116,"rwa":33,"ry ":297,"rsk":47,"rsi":111,"rso":78,"rsa":75,"rsh":36,"rse":187,"rta":1407,"rv ":292,"roù":324,"rst":71,"rsu":30,"rto":172,"rte":398,"rth":232,"rti":358,"rub":27,"rts":30,"rtr":52,"rtu":274,"rty":48,"rt ":661,"rqu":49,"rro":294,"rri":521,"rre":814,"rra":478,"ru ":88,"rry":64,"rnô":734,"rru":33,"saa":40,"sab":110,"sac":68,"sad":583,"sae":48,"sah":202,"sai":49,"sak":47,"sal":151,"sam":325,"sba":28,"sbe":56,"sao":1562,"san":655,"sau":75,"sat":83,"sas":62,"sar":136,"sav":1196,"sa ":294,"ón ":120,"rze":238,"rza":44,"rys":28,"ryn":36,"rzu":242,"rzo":285,"rzh":3355,"rzi":42,"sha":218,"sho":75,"she":193,"shi":238,"si ":134,"sga":31,"siz":115,"siv":61,"sie":664,"sid":67,"sic":44,"sia":1269,"sk ":453,"shu":43,"sit":87,"siu":25,"sir":71,"sis":396,"sip":30,"sin":816,"sio":143,"sil":161,"sim":118,"sik":81,"sif":26,"sig":38,"scu":28,"sde":25,"sbo":44,"sbu":48,"se ":1133,"sca":87,"sce":27,"sci":63,"sch":258,"sco":151,"sev":298,"sex":103,"sey":30,"ser":460,"ses":45,"set":371,"seu":286,"sañ":784,"sez":268,"sh ":215,"sfe":27,"sea":33,"sei":175,"seg":263,"see":27,"sed":76,"sec":35,"seb":48,"sep":184,"sen":385,"sem":202,"sel":465,"sek":164,"spo":97,"spr":39,"spe":713,"spl":66,"spi":316,"spa":666,"sot":45,"sou":522,"sov":37,"sol":98,"som":49,"son":1595,"sor":66,"sos":59,"sok":83,"soc":102,"su ":474,"oñ ":47,"sra":211,"st ":1732,"squ":36,"ss ":102,"slo":67,"sla":311,"sle":41,"ski":664,"skl":119,"sko":2999,"skr":1826,"skw":33,"sm ":26,"ska":795,"ske":666,"sno":26,"sne":81,"smo":24,"smu":27,"so ":138,"sma":85,"sme":47,"swi":31,"stê":656,"syl":43,"où ":12664,"sse":388,"ssa":244,"sso":146,"ssk":287,"ssi":196,"soñ":40,"soù":83,"ste":2322,"sth":144,"sta":2593,"sto":981,"sti":600,"stl":193,"stu":875,"str":2529,"sua":28,"sty":49,"sub":34,"oñi":37,"sul":41,"sum":35,"oñj":48,"suk":24,"sup":43,"sun":37,"oñs":58,"sus":65,"sur":177,"sve":118,"swa":45,"tai":326,"tak":108,"tal":1349,"tae":282,"taf":24,"tag":929,"taa":126,"tab":88,"tac":107,"tad":2010,"taz":91,"taw":31,"tav":87,"tau":39,"tat":200,"tas":62,"tar":434,"tap":82,"tao":273,"tan":1117,"tam":1603,"tch":742,"te ":835,"tde":54,"가":25,"ta ":886,"pa ":891,"pe ":5856,"ñs ":1564,"par":1873,"pat":120,"pas":123,"pav":27,"pac":192,"pad":413,"pab":42,"pag":683,"pak":52,"pal":693,"pai":106,"paj":24,"pao":79,"pap":202,"pan":368,"ñve":535,"phe":74,"pha":123,"ñvo":30,"phu":39,"phr":28,"pho":83,"phi":104,"pi ":45,"ñva":108,"ph ":70,"peu":284,"pev":140,"pez":279,"lâr":206,"pg ":46,"pea":32,"pec":47,"ñse":342,"ped":132,"ñso":78,"ñsk":24,"pen":1324,"pem":148,"pep":93,"peo":50,"per":657,"pet":110,"pes":749,"peg":57,"pei":35,"ñv ":186,"pel":375,"pek":88,"pla":359,"pli":1393,"ple":204,"plo":62,"plu":79,"lé ":29,"pke":90,"piz":28,"pl ":35,"phy":24,"pia":162,"pid":57,"pic":35,"pie":64,"pig":24,"pik":145,"pil":44,"pin":212,"pio":73,"pir":52,"pis":281,"pit":78,"poz":26,"por":318,"pop":53,"pou":712,"pot":112,"pos":143,"poi":42,"pon":117,"pol":852,"pob":151,"poe":85,"pod":72,"ps ":90,"ppi":46,"ppo":48,"ppa":34,"ppe":129,"pmu":27,"léo":25,"po ":44,"lêr":32,"poù":33,"pta":38,"pse":81,"psi":47,"pub":539,"pte":76,"pti":106,"pto":37,"pra":108,"pt ":114,"pri":323,"pre":639,"pro":765,"pur":73,"pus":78,"pun":48,"pui":25,"pul":44,"pve":49,"px ":512,"ñch":70,"ñie":36,"ñje":37,"qua":38,"que":308,"qui":93,"ra ":1887,"rb ":171,"ngo":851,"ngi":105,"ngl":237,"ngk":32,"ngu":135,"ngw":30,"ngr":91,"ngt":218,"ngs":57,"ni ":1357,"nge":730,"ngh":68,"nga":534,"nho":63,"ndé":24,"nha":101,"nhe":40,"neg":4522,"nei":121,"nel":551,"nek":2286,"nen":946,"nem":826,"nep":201,"neo":130,"ner":1985,"net":4017,"nes":379,"nev":416,"neu":170,"nañ":1277,"ndy":43,"ng ":876,"nea":366,"neb":309,"nec":60,"ned":1326,"nfo":32,"nfr":50,"ney":67,"nez":3411,"new":34,"nh ":31,"nfa":53,"nfe":39,"nct":33,"nco":81,"nci":162,"nce":259,"nch":229,"nca":55,"ne ":1685,"nbu":121,"ndu":84,"ndr":785,"nds":105,"ndo":360,"ndj":35,"ndh":32,"ndi":1021,"nde":859,"nda":686,"nak":317,"nal":306,"nam":483,"nan":2654,"nao":557,"nap":104,"nar":273,"nac":195,"nad":1521,"nae":258,"naf":53,"nag":110,"nai":155,"nc ":218,"nab":40,"naa":59,"nbo":33,"nbe":67,"nd ":930,"nba":62,"nav":633,"nau":104,"nat":458,"nas":202,"naz":46,"nay":25,"naw":28,"na ":1960,"mys":60,"mze":3020,"mza":29,"mzo":28,"ión":47,"mz ":627,"nya":139,"nye":172,"nyd":33,"nyi":90,"nwy":29,"nz ":54,"ny ":96,"nwe":103,"nwa":28,"nvi":206,"nvr":769,"nve":1542,"nva":229,"nuk":42,"num":58,"nun":42,"nus":273,"nut":34,"nuo":31,"nur":33,"nue":31,"ntv":525,"nto":1560,"ntu":79,"nts":43,"ntr":315,"nti":1416,"nth":89,"noù":2243,"nv ":3093,"nta":1595,"nte":2871,"nsu":30,"nso":629,"nst":225,"nse":99,"nsh":34,"nsi":150,"nsl":25,"nsk":93,"nsc":24,"nsa":96,"nu ":32,"nro":55,"nri":74,"nre":51,"nra":38,"nt ":12714,"ns ":309,"noc":89,"nod":56,"noa":38,"nob":77,"nog":78,"noe":57,"nol":692,"noi":46,"nop":48,"nom":155,"non":174,"not":100,"nos":158,"nor":724,"nov":93,"nou":960,"noz":603,"nne":3434,"nnf":24,"nng":83,"nna":2167,"nnb":29,"nnl":257,"nno":2232,"nni":788,"nnu":55,"nnw":29,"nnv":750,"nns":30,"nny":190,"nma":220,"niñ":141,"nn ":8975,"nla":142,"nle":246,"no ":590,"nke":412,"nkl":44,"nki":148,"nka":174,"nko":102,"nji":28,"nje":58,"nja":37,"nij":187,"nii":52,"nig":259,"nif":46,"nie":1515,"nid":227,"nic":163,"nib":35,"nia":931,"nk ":577,"niz":425,"niu":62,"niv":452,"nis":490,"nit":167,"nio":561,"nip":48,"nim":73,"nin":410,"nik":222,"nil":245,"ogr":98,"ogu":47,"ogw":32,"ogi":341,"ogh":25,"ogl":28,"ogo":182,"ogn":136,"oga":327,"oge":333,"oi ":52,"ohn":97,"oha":110,"ohe":80,"ois":208,"oir":111,"oit":33,"oio":472,"oin":74,"oic":40,"oid":101,"oie":65,"ok ":497,"oia":64,"oji":26,"oje":210,"ol ":836,"oce":86,"och":219,"oci":82,"ock":164,"oco":62,"oe ":1455,"oca":59,"ode":729,"odi":133,"odo":481,"odr":62,"îng":102,"of ":276,"oda":139,"oek":523,"oel":99,"oeg":409,"oei":96,"oer":123,"oet":567,"oen":280,"ody":27,"odu":112,"oed":76,"og ":209,"ofi":656,"ofo":33,"off":41,"ofe":71,"ofa":98,"oa ":4888,"ob ":87,"oc ":1030,"oan":167,"oal":46,"د ":37,"oad":1167,"oab":38,"oba":84,"oaz":875,"od ":515,"oav":413,"oar":1366,"oas":78,"oat":178,"obo":44,"obr":26,"obl":489,"obi":75,"obe":1082,"ة ":62,"nza":121,"nze":101,"nzi":83,"nzo":54,"oye":54,"oya":58,"ア ":34,"oz ":1606,"owy":28,"osé":34,"own":58,"owi":26,"ozn":946,"ozo":150,"oze":284,"ozh":546,"ozi":67,"oza":314,"oyo":26,"otu":34,"oud":1277,"oue":5169,"oub":196,"ouc":182,"oua":2059,"ow ":55,"oti":138,"oth":123,"ote":529,"ott":209,"ots":48,"otr":253,"oto":262,"ost":1461,"osu":49,"ota":312,"otc":28,"otd":52,"ov ":88,"osi":113,"osh":114,"osk":89,"ose":258,"oss":101,"osl":24,"oso":58,"oy ":63,"owa":47,"owe":66,"ovi":977,"ovo":29,"ouv":131,"ouz":1500,"ova":237,"ove":156,"oug":359,"ouf":27,"oui":317,"ouh":91,"ouk":182,"oum":209,"oul":878,"oun":651,"oup":85,"ous":409,"our":6950,"out":1285,"opo":193,"opi":160,"ope":218,"oph":160,"opa":474,"os ":1217,"opu":67,"opr":35,"opt":102,"ops":55,"ool":28,"ook":38,"ood":29,"or ":2242,"oou":106,"oot":29,"oor":62,"ork":157,"orl":90,"orm":333,"orn":1523,"oro":324,"orp":42,"orr":385,"orc":140,"ord":621,"ore":615,"orf":322,"org":321,"orh":51,"ori":818,"orj":52,"ou ":989,"osa":126,"osc":72,"ort":651,"ors":149,"orv":169,"oru":79,"orz":1153,"ory":50,"m² ":87,"ot ":217,"orb":59,"ora":374,"oqu":24,"ola":324,"old":111,"olc":37,"on ":5066,"oli":1224,"oll":2546,"olk":71,"olf":87,"ole":1214,"ols":36,"olt":143,"olm":61,"olo":1329,"olp":55,"oly":42,"olz":49,"olu":107,"oka":179,"om ":504,"oki":242,"oke":140,"okr":149,"oks":49,"oko":156,"oku":24,"ona":884,"ond":452,"onc":43,"onf":34,"one":3599,"ong":560,"oni":1468,"onk":97,"onn":763,"ono":501,"kêr":864,"onr":43,"ons":267,"ont":1954,"onu":48,"onv":33,"onw":26,"ony":75,"onz":24,"oma":931,"ome":375,"omb":242,"omi":259,"omm":204,"omp":243,"omo":225,"omt":26,"omu":437,"omy":53,"op ":79,"omz":3124,"la ":919,"ín ":37,"ío ":34,"le ":1172,"lch":45,"lf ":74,"lde":258,"lda":59,"ldo":59,"ldi":82,"lc ":543,"laa":65,"lab":398,"lac":162,"lad":1652,"lae":620,"lah":63,"lag":146,"laj":30,"lai":415,"lak":509,"lan":2333,"lam":1300,"lap":113,"lao":231,"lar":488,"lat":874,"las":665,"law":53,"lau":121,"lav":584,"lay":168,"laz":407,"lba":109,"ld ":178,"lbe":121,"lbi":34,"lbo":51,"lbr":32,"kve":49,"kuz":54,"kus":27,"kur":73,"kum":373,"kul":55,"kuk":29,"kwe":50,"kwa":63,"ky ":36,"kta":48,"kte":66,"koù":73,"kst":110,"ksi":64,"ksh":31,"kso":236,"kui":61,"ktr":79,"kti":78,"kto":145,"lpo":25,"llé":30,"lpe":41,"lph":43,"ls ":64,"lol":35,"lok":82,"lon":530,"lom":263,"lop":132,"lor":169,"lod":397,"loc":191,"loe":311,"log":539,"loi":42,"lpa":108,"los":243,"lot":351,"lou":939,"lov":116,"low":33,"loz":39,"loa":974,"lob":38,"hêr":108,"lmo":82,"lme":64,"lma":126,"liñ":39,"lth":34,"lti":120,"lto":44,"ltr":148,"ltu":28,"lud":25,"luc":61,"lub":38,"lua":26,"lue":102,"lsi":26,"lsk":38,"lso":26,"lsp":40,"lst":74,"lv ":131,"loù":329,"lta":234,"lte":150,"lu ":99,"ía ":69,"lre":43,"lt ":168,"lho":64,"lhe":195,"lha":164,"lgr":51,"lgo":96,"lge":35,"lgi":310,"li ":426,"lga":89,"lfr":35,"hât":29,"lfo":49,"lfi":33,"lfe":223,"lh ":403,"lez":1792,"ley":71,"lex":59,"leu":303,"lañ":440,"lev":593,"les":1142,"let":618,"ler":868,"leo":157,"lep":113,"lem":177,"len":2262,"lek":1292,"lel":46,"lei":313,"lej":65,"leg":1616,"lef":51,"lee":58,"led":429,"lec":1300,"leb":38,"lea":170,"llp":41,"lls":74,"llu":48,"llt":30,"llv":37,"lly":81,"lo ":582,"lla":2233,"lld":27,"lle":2420,"llg":38,"lli":505,"llk":31,"llo":648,"lko":32,"lka":49,"lke":60,"lki":37,"lm ":216,"lje":160,"ll ":4679,"lit":627,"lis":471,"lir":32,"lip":162,"lio":552,"lin":703,"lim":91,"liz":963,"liv":703,"leñ":35,"liu":80,"lic":150,"lid":231,"lia":1696,"lib":56,"lk ":62,"lik":678,"lil":31,"lij":1519,"lig":150,"lie":910,"lif":182,"ma ":1461,"mb ":591,"mac":81,"mab":327,"mah":35,"mai":110,"maj":31,"mak":56,"mad":427,"mae":311,"mag":364,"mar":2123,"mas":161,"mal":298,"mam":124,"man":3821,"mao":145,"maz":94,"may":26,"maw":44,"mat":618,"mba":789,"mbl":109,"mbi":183,"mbe":196,"mbr":812,"mbo":192,"me ":317,"mbu":84,"mde":59,"mdr":39,"med":147,"mee":66,"meg":191,"mea":32,"mañ":1356,"meu":785,"met":1513,"mes":334,"mer":2018,"mem":72,"mel":331,"meo":40,"men":1920,"mei":92,"mek":209,"mfe":26,"mez":216,"lva":154,"lve":509,"lvi":71,"luk":41,"lup":28,"lun":99,"lum":66,"lut":69,"lus":336,"lur":47,"ly ":109,"lwe":65,"lvo":48,"lvr":387,"lz ":240,"lwy":34,"lyd":32,"lzi":27,"lza":56,"lze":26,"lyw":60,"lym":32,"lyn":84,"mpi":89,"mph":46,"mpe":243,"mpr":173,"mpo":95,"mpl":1375,"mpv":50,"mps":40,"ms ":55,"mog":45,"moc":36,"moe":45,"mod":89,"mon":521,"mok":123,"moj":168,"mol":78,"mor":813,"mos":78,"mot":89,"mou":587,"mpa":496,"mre":137,"msa":83,"mu ":25,"iñ ":870,"ία ":28,"mst":158,"msk":28,"moù":263,"my ":29,"mur":71,"mus":152,"iñs":1025,"mut":27,"iñv":102,"mui":254,"mul":102,"mun":2223,"muz":74,"mgl":91,"mga":206,"mi ":131,"min":284,"mio":42,"mil":696,"mir":198,"mis":93,"mit":243,"miz":272,"mic":110,"mib":47,"mia":86,"mig":63,"mie":196,"mid":59,"mik":104,"mo ":123,"ièr":50,"ièg":36,"mka":24,"mm ":956,"moa":84,"iêt":40,"mno":39,"mp ":171,"mmu":105,"mmi":51,"mmo":164,"mma":602,"mme":253,"zre":26,"zro":39,"vîn":50,"zte":64,"zoù":296,"zu ":289,"zst":24,"zsk":181,"zve":38,"zue":58,"zul":186,"zun":25,"zuz":70,"zus":84,"zzo":29,"アア ":34,"zgi":47,"zi ":507,"zhi":1481,"zhe":1520,"zha":1503,"zed":202,"zeg":368,"zei":32,"zeb":38,"zdo":72,"zh ":14328,"zev":108,"zañ":742,"zeu":132,"zet":3672,"zes":33,"zez":352,"zen":1263,"zel":1371,"zek":305,"zer":980,"ze ":719,"zab":47,"zad":456,"zac":32,"zbe":30,"zam":89,"zan":193,"zak":37,"zal":175,"zar":107,"zao":47,"zap":54,"zas":136,"zat":105,"zoa":32,"zor":53,"zon":351,"zol":126,"zou":503,"ال":111,"zo ":10196,"ziñ":115,"zne":946,"vé ":24,"zka":30,"zhp":154,"zho":5307,"zhu":132,"zhs":28,"zhv":75,"zia":766,"zif":31,"zie":248,"zid":445,"zig":100,"zin":109,"zim":41,"zil":353,"zik":113,"zio":362,"zir":38,"zis":168,"zit":62,"ziv":302,"ziw":30,"yré":24,"yuz":72,"yth":46,"yst":49,"yso":29,"ysi":55,"yre":24,"ys ":180,"yr ":41,"you":78,"yot":24,"yon":122,"za ":184,"ywe":49,"ye ":71,"uße":32,"ych":49,"yde":28,"ydd":122,"yek":64,"yeg":55,"yer":66,"yen":76,"yez":6381,"yev":24,"ya ":614,"yd ":35,"yar":58,"yan":276,"yao":135,"yal":52,"yam":35,"yak":60,"yah":28,"yn ":147,"yla":30,"yle":56,"yli":31,"yll":66,"ylv":44,"yo ":48,"ymp":27,"ymo":24,"yna":31,"ync":35,"yne":111,"ر ":42,"yi ":39,"yin":88,"yie":31,"têr":714,"ي ":49,"ن ":50,"xi ":24,"té ":71,"xis":56,"xic":25,"xas":26,"xan":47,"wy ":42,"wys":53,"wyn":117,"wyd":57,"wiñ":39,"sé ":43,"wn ":51,"wre":95,"wri":41,"wor":37,"wol":24,"wr ":62,"we ":50,"wez":464,"wev":251,"wes":47,"wer":638,"weu":60,"wen":755,"wel":613,"wek":43,"wei":37,"weg":50,"wed":29,"wee":32,"wec":686,"wg ":36,"wi ":52,"wir":257,"wis":49,"wit":92,"wig":63,"wie":25,"wid":55,"wic":39,"win":99,"wil":166,"wia":38,"vye":26,"wa ":134,"wan":232,"wal":641,"wam":29,"wak":26,"way":43,"wat":35,"war":4112,"was":65,"waz":81,"wag":31,"wai":112,"wae":48,"wad":65,"ría":36,"vru":57,"vro":1897,"vri":160,"vre":1493,"vra":602,"vsk":45,"vus":30,"vuz":29,"vui":173,"vug":63,"vuh":82,"via":194,"vio":268,"vir":66,"vik":47,"vil":263,"vin":176,"vig":93,"vih":155,"vij":194,"vic":93,"vid":194,"vie":274,"viz":2997,"viv":29,"vit":2282,"vis":84,"ré ":55,"vn ":395,"vle":69,"vlo":168,"vo ":111,"rén":27,"viñ":739,"vne":256,"voa":40,"vod":80,"voe":1131,"vog":240,"voj":30,"voi":76,"vol":72,"von":458,"vor":641,"vot":54,"vos":31,"vou":673,"vr ":206,"vi ":33,"vez":7355,"ver":1988,"ves":130,"vet":4050,"veu":266,"vañ":660,"vev":1006,"vei":24,"veg":189,"ven":1240,"vel":2563,"vek":66,"vea":33,"vef":97,"vee":38,"ved":919,"ve ":570,"val":181,"vak":44,"van":668,"vam":45,"vao":84,"var":1289,"vat":272,"vas":122,"vc ":49,"vab":24,"vae":81,"vad":569,"vai":39,"vag":535,"va ":291,"uzu":185,"uzk":33,"uzi":114,"uzh":1308,"uze":459,"uzs":159,"uzo":65,"uza":122,"uzb":30,"uya":31,"uz ":475,"uwe":24,"ux ":103,"uvi":37,"uva":39,"uve":100,"uy ":40,"uvr":49,"usk":479,"ush":47,"usi":500,"use":355,"usc":41,"usa":194,"ust":519,"uss":119,"uso":25,"uth":105,"uti":264,"ute":147,"uta":215,"utt":43,"uts":71,"utu":47,"uto":157,"utr":26,"us ":12885,"ut ":1028,"urb":53,"ura":377,"urd":124,"urc":80,"ure":1465,"urh":24,"urg":533,"uri":1365,"url":187,"urk":288,"urm":291,"urn":83,"uro":614,"urr":1978,"urs":53,"urt":279,"uru":206,"urv":409,"ury":73,"urz":552,"uny":30,"uon":91,"upa":32,"ur ":15791,"upi":44,"upe":63,"upl":36,"umu":1652,"uiñ":100,"umi":41,"umm":904,"umo":35,"uma":310,"umb":711,"ume":152,"ulz":123,"unt":110,"uns":56,"unv":98,"unu":56,"unk":57,"uni":755,"uno":443,"unn":67,"unc":29,"und":259,"una":1290,"ung":318,"une":411,"up ":59,"uks":39,"ukr":56,"uku":44,"uko":46,"ukl":49,"uki":91,"uke":38,"um ":382,"uka":71,"ulv":25,"ulu":113,"ult":103,"uls":40,"ulo":116,"ulm":63,"ull":318,"uli":404,"ulg":56,"ule":376,"ulf":33,"ula":220,"un ":6539,"uid":76,"uie":35,"uig":40,"uik":27,"uil":155,"uin":77,"uio":78,"uis":287,"uk ":47,"uia":185,"uje":42,"uit":103,"uiz":66,"ul ":1239,"uja":41,"ugh":41,"ugi":31,"uge":364,"ugo":67,"ugl":36,"ui ":204,"uga":481,"uhe":503,"ugu":149,"ugr":53,"uha":28,"uco":39,"ucu":26,"uf ":24,"uda":492,"udd":57,"ude":1332,"udi":309,"ubo":31,"uca":58,"ue ":1086,"uce":33,"uci":47,"uch":262,"uck":41,"uet":1306,"uev":34,"uer":664,"ues":294,"uez":1264,"uff":67,"ufo":28,"udu":45,"udo":178,"ug ":229,"udw":33,"ueg":387,"uee":179,"ued":155,"uen":478,"uel":271,"uek":451,"ub ":86,"ua ":372,"uay":86,"uat":78,"uas":116,"uar":925,"uam":161,"ual":78,"uan":1051,"ubi":95,"ubl":610,"ube":190,"uba":184,"ud ":1278,"uai":32,"uaf":53,"uag":34,"uad":128,"uc ":83,"tze":26,"tyl":53,"ty ":132,"tve":544,"tur":438,"tus":231,"tut":38,"tul":25,"tun":47,"tum":651,"tua":71,"tud":498,"tug":248,"tz ":81,"twe":33,"ts ":141,"tre":2978,"tt ":64,"tra":1337,"tri":813,"tru":143,"tro":3557,"tu ":352,"tsa":33,"tse":38,"tsc":49,"tsi":40,"tsk":31,"tso":25,"tsu":28,"toù":279,"tta":120,"tte":308,"tti":121,"tto":83,"ttp":30,"tts":31,"to ":479,"tiñ":48,"tp ":30,"tna":56,"toe":80,"tod":25,"toc":284,"toi":54,"tog":32,"tou":859,"tov":27,"tos":326,"tot":83,"toz":55,"tow":30,"tom":159,"ton":1563,"tok":134,"tol":338,"tor":741,"top":94,"tr ":310,"til":274,"tik":947,"tif":26,"tie":481,"tig":104,"tir":140,"tiq":33,"tit":209,"tis":284,"tin":1250,"tim":79,"tio":509,"thy":51,"thu":684,"tia":703,"tib":461,"tic":184,"tid":98,"tiz":85,"tiu":34,"tiv":312,"tl ":113,"tka":29,"tke":102,"tla":426,"tle":255,"tem":281,"ten":1056,"teo":96,"tep":78,"tei":518,"tek":1918,"tel":1926,"tee":33,"tef":26,"teg":307,"tea":74,"tec":37,"ted":237,"th ":395,"tez":423,"tev":112,"teu":243,"tañ":1689,"tet":90,"tes":340,"ter":3607,"ti ":957,"tho":239,"thr":44,"the":408,"thi":81,"tha":171," アア":26,"並 ":30,"三 ":49,"üs ":59,"之 ":48},"n_words":[2305059,2819439,2226880],"name":"br","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/cs.json b/contrib/languages-data/cs.json
new file mode 100644
index 0000000..e6017de
--- /dev/null
+++ b/contrib/languages-data/cs.json
@@ -0,0 +1 @@
+{"freq":{"D":14883,"E":9868,"F":10603,"G":9505,"A":22558,"B":22848,"C":16455,"L":15978,"M":26754,"N":16476,"O":10801,"H":14139,"I":10554,"J":20213,"K":19386,"U":5533,"T":19206,"W":5350,"V":19456,"P":31985,"S":35760,"R":13848,"Y":1260,"Z":7679,"f":46799,"g":61772,"d":331353,"e":908926,"b":175631,"c":304749,"a":764805,"n":728035,"o":890229,"l":442609,"m":328621,"j":233650,"k":440118,"h":259891,"i":521717,"w":9425,"v":451146,"u":342714,"t":548551,"s":514164,"r":514188,"q":1288,"p":289793,"z":213126,"y":191798,"x":11788,"í":321621,"é":145384,"á":236804,"ý":129650,"ú":13969,"ó":5647,"ě":138011,"ď":2560,"Č":7702,"č":98194,"ř":105343,"Ř":1405,"ň":6555,"Ž":1649,"ž":81570,"ť":3583,"Š":3572,"š":60015,"ů":48717," l":32594," m":62986," n":107131," o":79037," h":32176," i":15946," j":127655," k":89036," d":65217," e":11656," f":19925," g":5629,"р":1443," a":90604,"с":1165," b":47368," c":20381," z":85758," u":20743," t":59923," w":1272," v":146123," p":164476," s":159790," r":52674," J":20168," K":19243," H":14014," I":10456," N":16390," O":10658," L":15894," M":26594," B":22644," C":16189," A":22443," F":10496," G":9381," D":14695," E":9809," Z":7643," Y":1252,"и":1928,"о":2213,"н":1523," S":35443," R":13762,"в":1172," P":31807,"а":2293," W":5251," V":19363," U":5519,"е":1554," T":19078," ú":11862," č":34240," Č":7692," ž":9995," Ž":1647," Š":3567," š":7349," ř":11127," Ř":1403,"A ":3554,"Da":2148,"Co":3211,"Ce":1274,"Ch":3956,"Do":2726,"De":2027,"Di":1704,"Ev":1845,"Ge":1462,"Ga":1606,"I ":2501,"Fr":2934,"Fo":1409,"Fi":1671,"C ":2405,"Au":1413,"Ar":2392,"As":1426,"D ":1568,"Ba":3474,"Am":1706,"An":3282,"Al":3092,"By":2085,"Bu":2103,"Br":3950,"Ca":2582,"Bi":1362,"Be":3232,"Bo":3007,"Ku":1175,"Kl":1430,"Kr":2544,"Ko":4086,"Le":3365,"Li":3491,"La":2847,"Lu":1426,"Lo":2548,"Me":3379,"Mi":4015,"O ":1633,"Ma":8187,"Mo":4597,"Ni":1492,"Ne":3393,"Na":4223,"P ":1532,"No":2609,"Ob":1767,"Gr":1729,"Ha":2931,"He":2201,"Ho":2992,"Hr":1212,"In":2920,"Ja":4326,"L ":1226,"Ji":2201,"Je":8576,"Jo":1796,"Ju":1244,"Ka":5031,"M ":1302,"Tu":1161,"Tr":2534,"To":2275,"Th":2120,"Ti":1361,"Te":3239,"Ně":1339,"Ta":2435,"V ":4585,"St":6172,"Sv":1697,"Su":1422,"Wi":1615,"Wa":1205,"Vy":1686,"Vo":1294,"Vi":2012,"Va":1654,"Ve":3321,"Pr":7552,"S ":3112,"Pe":2524,"Pa":5499,"Pl":1516,"Po":7183,"Pi":1236,"Os":1168,"Or":1234,"R ":1716,"Se":3431,"Sc":1219,"Si":1856,"Sl":2584,"Sk":1212,"Sp":2686,"So":3053,"Ru":1888,"Sa":3516,"Re":2582,"Ná":1404,"Ro":3655,"Ra":2507,"b ":5436,"Zá":1325,"a ":217661,"Př":2063,"Za":1646,"Ze":1335,"i ":90037,"fy":1326,"ge":8367,"ga":7746,"bý":4615,"fi":13067,"ač":14478,"fr":4738,"fu":2046,"ft":1555,"fo":9138,"bí":5183,"j ":5728,"gy":1560,"dá":9259,"he":13421,"ha":16982,"bě":8276,"gn":2020,"gl":5355,"gi":10280,"gh":1578,"gu":3569,"gr":6781,"cí":27801,"go":4942,"du":15000,"dv":6535,"dy":13097,"g ":5595,"ea":6165,"eb":20255,"ec":32017,"ed":50406,"de":36807,"dd":1592,"di":24806,"dh":1175,"dk":3086,"dm":3275,"dl":11857,"do":40054,"dn":44007,"dp":2177,"ds":6618,"dr":15992,"ew":1387,"ex":5658,"eu":3247,"ev":22848,"ey":1497,"ez":19185,"fa":3278,"h ":68060,"fe":4502,"bá":2585,"eh":9007,"eg":5751,"ef":3385,"ee":1918,"el":51949,"ek":22014,"ej":27806,"ei":3682,"ep":11390,"eo":6058,"en":103592,"em":54636,"et":33365,"es":44385,"er":89316,"ca":4919,"e ":246825,"bv":2259,"by":26507,"bs":4261,"br":9964,"bu":11101,"bn":7303,"bo":26250,"bj":2483,"bl":11541,"bi":8921,"bc":3044,"bd":1995,"be":13547,"dc":2442,"db":1918,"da":22450,"f ":2962,"cy":1677,"cu":2117,"ct":4541,"cr":1140,"co":10175,"cn":2818,"ck":45489,"ci":24104,"ch":106800,"ce":56738,"c ":9636,"az":14206,"ay":1595,"ba":13431,"d ":32945,"at":54683,"as":30808,"ar":43773,"ax":1424,"av":34552,"au":9271,"ak":35472,"al":52089,"ai":4202,"aj":15958,"ap":12841,"am":28407,"an":76603,"ac":26092,"ad":38615,"ab":9081,"ag":6085,"ah":10143,"ae":4121,"af":3809,"nu":14720,"nt":27386,"ns":25831,"ič":7118,"jí":23302,"no":57801,"nn":8311,"nz":2252,"ny":17200,"oe":1227,"ká":21030,"of":6319,"oc":21110,"od":69409,"oa":1996,"ob":47136,"ké":50289,"om":27583,"on":44356,"ok":26216,"ol":47354,"oi":2071,"oj":17875,"og":10937,"oh":13386,"ot":27460,"os":56160,"ov":112940,"ou":73196,"op":21879,"oo":2643,"or":53147,"r ":22705,"ox":1211,"ow":2081,"oz":27164,"lá":15416,"pe":15026,"pa":25836,"ký":35391,"pc":1302,"pl":12514,"lé":7239,"pn":3046,"po":81579,"ph":1918,"pi":14831,"eň":2211,"lo":61865,"ln":20483,"hé":2618,"lm":4934,"ll":7520,"ls":8702,"dů":3418,"lu":15908,"lt":4743,"lz":1355,"ly":9717,"o ":138337,"ma":25523,"eř":2784,"mb":4786,"hý":1516,"me":40325,"iá":5235,"ml":2209,"eš":4400,"mi":28342,"mn":8112,"mm":1350,"mp":4986,"mo":29298,"mr":1370,"ií":2470,"ms":3506,"mu":15591,"my":5631,"p ":4419,"na":97771,"nc":16128,"nd":14047,"ne":57176,"já":1158,"nf":2490,"ež":8504,"ng":13163,"ni":45831,"nk":8297,"jv":3866,"ju":1525,"eč":10324,"js":11164,"jn":5134,"jo":3496,"jm":7494,"ki":3600,"ke":11246,"kd":5064,"kc":3108,"ka":39832,"m ":95402,"ců":2079,"ky":30906,"kt":39344,"ku":33102,"kv":3857,"ko":71233,"kr":19435,"kl":18948,"km":3641,"kn":3161,"li":57563,"lk":10732,"le":77448,"há":9650,"ld":2521,"lf":1159,"la":65008,"dř":2617,"lc":1721,"lb":2945,"n ":43523,"hr":15902,"dí":8010,"hv":1306,"ht":2875,"hu":12412,"hi":7867,"hn":5204,"ho":73846,"hl":13059,"hm":1256,"dé":2586,"id":15111,"ic":65339,"dý":1471,"ib":5266,"ař":4683,"ia":10547,"ih":5555,"ig":6211,"if":3335,"eá":1247,"ie":19796,"hy":6927,"k ":33844,"ir":8411,"is":36532,"it":38607,"iu":2679,"iv":18333,"aš":1639,"ii":6209,"ij":4185,"ik":23953,"il":27036,"im":9173,"in":70373,"io":12386,"ip":4762,"je":121519,"až":7470,"ji":17967,"iz":8625,"l ":42283,"bř":2940,"ja":17600,"dě":13132,"xi":2953,"té":13825,"tí":14980,"pů":6675,"xt":1600,"z ":24530,"př":32632,"ož":13354,"tá":14232,"nž":1154,"oš":2365,"sé":1859,"sí":5664,"rč":2879,"nů":2947,"vy":26292,"vz":6628,"y ":96008,"rý":11309,"oř":9947,"wa":2470,"sá":2703,"we":1453,"vl":7295,"ré":10396,"vk":3052,"nš":1672,"vi":26093,"mž":1362,"vu":5429,"vr":8123,"vs":6209,"vn":24780,"vo":37739,"uz":6709,"mů":3291,"uv":3489,"ve":47740,"rá":30974,"vc":2056,"va":51428,"pě":4347,"x ":2742,"ui":1687,"uj":20016,"uk":7056,"ul":12653,"ue":2392,"ug":1937,"uh":10244,"ur":17880,"us":20839,"ut":14862,"um":13455,"un":12543,"up":13452,"ty":13294,"tz":1768,"tu":25069,"tt":3192,"lů":2536,"tv":17643,"ub":11697,"ua":2086,"ud":12808,"uc":6671,"w ":1552,"to":60100,"tn":18581,"tm":1809,"tl":6035,"ts":8087,"oč":8726,"tr":38784,"pí":3163,"te":85042,"pá":2981,"tk":8430,"ti":58581,"lš":2281,"th":6153,"v ":66402,"tb":1873,"tc":1257,"ta":58747,"ně":45741,"su":6657,"sv":12247,"ss":3706,"st":145711,"sy":7700,"ků":5193,"sl":22654,"sk":92692,"sn":8865,"sm":8745,"sp":20097,"so":35354,"sr":1708,"nč":1796,"sc":4718,"se":52884,"sh":2118,"si":15634,"rz":2926,"u ":121101,"mě":22506,"sa":15498,"kř":3224,"sb":1371,"ný":29623,"rr":2082,"rs":10884,"rt":11161,"ru":27381,"rv":11951,"ry":11066,"ní":119912,"rp":2592,"ro":116857,"rn":21334,"rm":10574,"né":26518,"rl":3217,"rk":6663,"ri":39119,"jš":5173,"rh":1365,"rg":7113,"iž":5869,"ná":41081,"re":36192,"rd":6788,"rc":6965,"mý":1831,"rb":3000,"ra":81349,"t ":42900,"mí":11983,"mé":6077,"iš":4295,"má":9760,"lý":4646,"s ":40054,"py":1779,"pt":3558,"pu":7668,"hů":2019,"lí":8097,"pr":57083,"ps":5342,"zý":1387,"zá":15336,"už":12787,"vš":5164,"zí":7043,"vů":2483,"uš":4102,"tš":4235,"vě":25419,"tř":15220,"tů":3818,"rž":1454,"vý":29752,"zh":2175,"zi":13719,"rš":2140,"zb":2038,"zd":9382,"ze":35435,"vá":31782,"tě":14576,"za":24684,"yz":2224,"rů":8094,"zv":6153,"zy":3702,"zs":3415,"uč":6510,"zr":1709,"zu":4066,"zt":1618,"zo":8118,"zn":27029,"ví":13118,"zp":5782,"zk":6049,"zm":2157,"vé":19370,"zl":2273,"yh":1425,"yc":3822,"yd":3200,"tý":6426,"yb":3071,"yv":3371,"yu":1291,"yt":7847,"ys":11111,"yr":3273,"yp":5097,"yn":5498,"ym":4086,"yl":23502,"yk":5878,"zš":1160,"yř":1210,"yš":3131,"ěžn":1182,"ám":8120,"án":28607,"áp":5503,"áj":1550,"ák":7074,"ál":25430,"áh":2531,"áb":3286,"ác":7677,"ád":11483,"áz":13195,"áv":12832,"ár":9720,"át":14731,"ás":12931,"á ":61028,"íč":1699,"ód":1198,"ón":1579,"éž":3751,"ív":7836,"íz":3776,"ín":7431,"ím":27488,"íp":2283,"ír":5728,"ít":5219,"ís":9896,"íh":10920,"ík":10708,"íl":6160,"íj":1652,"íc":36122,"íd":6139,"íb":1336,"í ":168172,"áž":2247,"él":1388,"áš":2438,"ém":16118,"én":5964,"ét":2091,"ér":4128,"ář":4474,"éd":1164,"éh":25602,"é ":80173,"áč":1328,"úč":1787,"ýc":33402,"ýz":2981,"ýv":5852,"ýs":1457,"ýt":1801,"ýr":2669,"ým":16378,"úz":3790,"ý ":58367,"ús":1640,"íš":1631,"íř":2525,"íž":4061,"ě ":45418,"ěh":2633,"ěj":8834,"ěd":3131,"ěc":1612,"ěz":2258,"ěs":10757,"ět":15295,"ěv":1529,"ěr":3724,"ěk":7417,"ěl":9917,"ěm":5852,"ěn":11047,"ěž":3582,"ěř":2184,"ýš":1701,"Če":5023,"či":11688,"čk":4324,"čl":2919,"če":26348,"ča":7771,"ď ":1420,"č ":2311,"čá":8501,"čn":16418,"čo":3182,"čt":3695,"ču":2648,"čí":6991,"ň ":2419,"š ":1248,"ří":28277,"řá":2554,"řn":1352,"ři":14008,"řs":2960,"ře":44896,"řa":3989,"ší":18419,"ť ":1589,"še":8882,"ša":2373,"šn":3181,"šk":3546,"šl":1812,"ši":4496,"št":7997,"ňu":1474,"ř ":4182,"žs":2453,"žn":8213,"žo":1725,"že":16587,"žd":1803,"žk":1495,"ži":11885,"ž ":17370,"ží":12326,"žá":1346,"ů ":27572,"ům":2630,"ůl":1327,"ůs":3997,"ův":4803,"ůz":2205,"šš":1500,"ůž":1940,"čás":7287,"čí ":1292,"čít":2030,"čís":1405,"čtv":1158,"čuj":2526,"ční":7738,"čné":1325,"čný":1279,"čně":1729,"ěji":2384,"ěko":1330,"ěkd":1747,"ělo":1511,"ěle":2558,"ěkt":1458,"ěme":3785,"ěls":1244,"ětš":3901,"ěta":1179,"ěto":2030,"ěný":1390,"ěst":9587,"ějš":4291,"ění":3741," Ga":1588," Ge":1446," Fo":1382," Fr":2926," Fi":1650," Ha":2917," He":2194," Gr":1717," Hr":1206," Ho":2982," Ji":2194," Je":8552," Ja":4310," In":2868," Ka":5019," Jo":1792," Ju":1242," La":2820," Le":3350," Li":3464," Kl":1383," Ko":4081," Kr":2542," Ku":1171," Ma":8133," Mi":3991," Me":3365," Lo":2542," Lu":1423," Ne":3375," Na":4212," Ni":1486," Mo":4565," Am":1697," An":3275," Al":3081," Ba":3436," Au":1408," As":1420," Ar":2370," Be":3213," Bi":1350," Bo":2981," Br":3935," Bu":2094," By":2085," Ca":2514," Ce":1267," Ch":3924," Co":3154," Da":2135," Di":1684," De":2001," Do":2665," Ev":1844," Př":2056," Wi":1593," Wa":1184," Vy":1684," Ze":1330," Za":1643," a ":53430," Zá":1320," Os":1166," Or":1230," Po":7138," Pl":1502," Pi":1231," Pe":2516," Pa":5464," No":2595," Ob":1764," Ra":2495," Ro":3640," Re":2567," Ná":1395," Pr":7523," Sv":1696," Su":1408," St":6100," Ně":1337," Ta":2426," V ":3746," Th":2108," Ti":1351," Te":3210," Tr":2526," To":2242," Ru":1883," Sa":3503," Si":1839," Sc":1168," Se":3406," So":3031," Sp":2664," Sk":1209," Sl":2578," Va":1650," Ve":3306," Vi":1986," Vo":1286," Tu":1143," ja":14129," dě":2397," bř":1481," ji":8392," až":2786," je":93301," in":6001," it":1175," ka":8683," m ":1247," kd":3115," ke":1672," jm":2892," js":6707," ha":1600," bě":2145," he":2509," dá":1402," cí":2459," gr":1666," k ":5918," hi":2201," hl":5156," ho":6862," dí":1705," hr":6552," hu":3075," ni":2426," ne":33309," na":43193," my":1417," mu":2661," mo":9457," mn":2424," ok":4623," oc":1666," od":14999," of":1957," ob":22345," no":3710," le":12637," li":8468," la":3675," dř":1310," kv":1829," ku":2340," kt":24717," kn":2205," km":3399," kl":3809," kr":8199," ko":16866," me":11211," mi":4779," ml":1339," o ":7191," ma":9419," dů":1272," lo":3118," ab":1328," am":3804," an":6347," ap":1198," ak":1935," al":5289," au":3694," ar":3304," at":1332," as":2951," ba":3819," bi":2222," be":2412," bo":3752," by":19260," bu":3054," br":3187," en":1401," el":2125," fa":1772," ex":2023," fu":1558," fr":2890," fo":4715," fi":6203," ge":1763," bý":3017," i ":5273," co":2133," ce":6243," ch":5478," da":4034," do":19631," dn":2609," dl":1361," dr":8732," de":7028," di":4948," dv":4297," du":1944," vý":12688," zk":2698," ví":2124," zp":3908," zn":6476," rů":1942," zv":2770," tě":2639," za":18239," zd":2577," vá":3060," ze":8914," tý":1167," té":5230," pů":4589," z ":20036," př":29127," vš":2884," už":1794," zá":11557," tř":3845," vě":4981," ru":2498," ry":1961," u ":3351," sa":3585," mě":10287," kř":2709," se":38618," sc":1843," si":3884," sn":1263," sm":2713," sl":8492," sk":7519," sr":1447," sp":14637," so":12794," mí":3137," ra":2856," ná":11985," re":10229," ro":31079," ní":1455," pr":44618," ps":1456," s ":10437," má":2572," os":6522," ot":1283," ov":1214," op":3355," or":4357," oz":5841," lá":1294," pe":2741," pa":10148," pl":5963," po":56929," vy":18421," vz":6214," sí":2537," pě":1240," va":1643," ve":20353," uv":1309," mů":1354," vo":5885," vr":1606," vi":2111," vl":5324," ty":2521," tv":3467," tu":1727," us":1203," ur":2603," um":2673," un":1389," ta":9553," ně":9405," v ":54233," sy":4875," st":28273," sv":11047," su":1768," pí":1932," tr":6193," to":5957," th":1725," ti":1444," pá":1635," te":10751," Če":5018," čí":2219," čá":5494," čt":2423," če":12226," čl":2635," či":5961," ča":2613,"ňuj":1462," ús":1372," úz":3683," úč":1533," šk":1284," řa":1815," ře":4682," ří":3390," ži":4198," že":3745,"Evr":1575,"Fra":1982,"šší":1462,"ším":3926,"šíc":3060,"Byl":1846,"šíř":1269,"ší ":8873,"Cha":1224,"Nov":1143,"Par":1206,"Pro":2287,"Pra":2828,"Pod":1248,"Pol":1195,"ůzn":1758,"ůso":2743,"ůvo":3624,"Jed":1974,"Jeh":1143,"Je ":2998,"ům ":1924,"Kar":1595,"Mar":2760,"řez":1271,"řev":1736,"řet":1507,"řes":3291,"řen":6131,"řel":1284,"ři ":4648,"řed":13088,"řej":1316,"řek":2349,"řeb":1351,"řec":3115,"ře ":3029,"řad":2628,"ého":25401,"ém ":11396,"řsk":2573,"áře":1219,"ému":2422,"éna":1816,"éno":1417,"éri":1326,"Sta":2172,"Spo":1540,"Slo":1556,"Vel":1392,"The":1321,"šec":1390,"šen":2332,"še ":1844,"šak":1283,"šti":2097,"šní":1582,"ško":1848,"šin":1751,"ště":3036,"átu":1328,"bje":2435,"áte":1666,"átk":2625,"átn":2002,"áto":1624,"bit":1438,"ást":9035,"bil":2518,"bo ":10795,"ávi":1211,"bli":4783,"ávn":2176,"bla":3565,"áva":2824,"bod":1239,"bol":1903,"boj":1202,"blí":1257,"ázv":1186,"ává":2557,"áze":5164,"bor":3758,"bov":1176,"bou":1297,"álk":1854,"áln":9542,"álo":2573,"álu":1211,"ákl":2414,"bal":2672,"áko":1419,"án ":4623,"ále":4254,"bar":2156,"áli":1317,"áns":1949,"bdo":1660,"áno":2096,"ány":1316,"ánu":1360,"ámo":1148,"bce":1301,"ána":2621,"áni":1332,"bec":2884,"ber":2768,"ben":2163,"bez":1903,"ápa":4357,"ámě":1142,"ání":9623,"át ":2061,"áro":4387,"árn":3371,"ámý":1162,"áda":1737,"ách":3663,"áce":1349,"ádá":1824,"ábě":1152,"ca ":1459,"ák ":1153,"ál ":2023,"ce ":37517,"ám ":1274,"bri":1805,"bro":1589,"bra":3506,"bu ":2220,"bní":3217,"bsk":1200,"bsa":1785,"bur":1260,"bud":1860,"bvy":1413,"by ":4307,"byl":18860,"byv":1815,"am ":2742,"ake":1229,"aji":2410,"al ":7387,"adě":2234,"aje":2039,"ak ":5398,"ahu":2375,"ahr":2506,"aha":1562,"ací":4252,"aké":5289,"anu":1370,"any":1635,"ano":3773,"ann":1847,"ant":5679,"ans":3967,"ají":6907,"ane":2662,"ang":5036,"ani":8977,"ank":1763,"ana":4560,"anc":5148,"and":5394,"amu":1194,"amo":2812,"amn":1598,"ami":3729,"ame":8001,"ama":1900,"aly":1179,"als":1998,"alo":7874,"all":1742,"ali":6637,"ale":6714,"ala":3987,"alb":1604,"an ":5076,"aku":1247,"akt":3867,"ako":12804,"abe":1141,"abs":1339,"ae ":1943,"ad ":4944,"afi":1777,"age":1161,"ael":1206,"ado":1550,"adl":1340,"adn":6465,"adi":3651,"ade":3282,"ady":1580,"adu":2552,"aco":1432,"aci":3453,"ach":6559,"ace":7532,"ada":3897,"arš":1293,"azu":1350,"aze":2771,"azy":2655,"até":1146,"apř":2101,"ba ":3672,"azý":1373,"avě":1439,"atř":2572,"at ":6248,"aná":3560,"are":2437,"ard":3288,"arc":2227,"ara":4391,"aro":4126,"arn":1320,"arm":1290,"ané":4536,"arl":1695,"ark":1851,"ází":4141,"ari":4139,"ars":2390,"art":3122,"asa":1162,"amě":1185,"asi":3609,"aný":5688,"ase":1167,"aso":1587,"asn":2528,"ask":1257,"ar ":2605,"alá":1216,"ape":1349,"alé":1624,"apo":2941,"as ":2058,"alý":1783,"ava":3516,"aut":3726,"avs":1166,"avo":2741,"avn":5290,"avi":3981,"ave":3850,"avy":1957,"avu":1943,"av ":1604,"ata":3060,"aně":2278,"ast":13117,"atn":1969,"atk":1885,"atr":1563,"ato":5121,"ate":11112,"ati":10351,"alš":1631,"atu":2078,"aur":1970,"řád":2065,"ří ":6262,"řív":1184,"říz":2145,"říc":1240,"říp":1616,"řís":1609,"řím":2008,"řík":1716,"říd":3286,"říž":1421,"až ":2790,"jeh":3911,"jej":4795,"jed":12881,"jek":1841,"jem":2659,"jen":5483,"jev":2523,"ji ":4802,"ažd":1260,"bře":2487,"děl":3685,"jak":10723,"děn":1255,"děj":2198,"jaz":2568,"je ":83650,"jme":2592,"jno":1257,"jov":2720,"jin":3546,"jih":2344,"jic":2414,"ito":2582,"itu":2506,"its":1828,"ity":1661,"isk":2395,"ism":2509,"isl":2097,"iso":2022,"ist":15778,"ině":2331,"ita":3897,"ite":5726,"iti":3568,"ivo":2672,"ivn":3856,"ium":1381,"iva":2483,"ivi":1448,"ive":2466,"is ":3839,"ion":6938,"iro":1733,"irm":1303,"ise":1488,"iný":1514,"iná":3960,"it ":3805,"dě ":4195,"itý":2115,"ití":1214,"ité":1984,"itá":1643,"izo":2349,"iza":2404,"km ":2378,"kdy":3252,"kej":1210,"kem":3800,"ket":1376,"ke ":1607,"kce":1649,"kde":1621,"kra":5687,"kre":3334,"kt ":1234,"ku ":19613,"kro":2033,"kov":11417,"kou":9659,"kos":4285,"kor":1310,"kop":1321,"kon":10573,"kom":4572,"kol":7494,"úze":3480,"klá":2870,"kni":1628,"klu":1477,"ko ":14736,"kle":2073,"kla":8470,"klo":1299,"jso":6864,"jsk":2959,"ečn":6772,"jmé":3893,"eče":1504,"již":2514,"kaz":1148,"kat":3393,"kar":1707,"kap":1616,"kan":2047,"kal":2034,"kam":1509,"kac":1503,"ka ":20272,"jvě":1879,"ha ":4741,"bě ":2724,"han":1905,"hal":1464,"har":2922,"běh":2126,"he ":2188,"dá ":2060,"dál":1753,"dáv":1525,"her":2617,"dán":2244,"hem":3497,"his":2398,"běž":1211,"hla":5571,"hle":2243,"hlo":1435,"ho ":41844,"gli":3394,"cí ":18956,"cíc":3683,"gra":4857,"cím":1211,"ial":1377,"ian":1754,"ic ":2965,"ibl":1394,"ia ":4182,"ien":1656,"aří":2027,"ier":1153,"ifi":1720,"ict":1675,"ick":33133,"ici":3981,"ich":6375,"ice":11279,"ie ":13367,"ica":1513,"ido":1643,"idl":1498,"ide":3474,"ida":1748,"il ":4856,"ika":5332,"ii ":5869,"igi":1176,"icí":1825,"iho":2925,"ik ":2696,"imo":1396,"ime":1223,"inc":4325,"ind":2184,"ina":8223,"inn":2167,"ino":5250,"int":2630,"ins":4625,"inf":1536,"ine":3495,"ing":4248,"ini":3264,"ink":1380,"iká":1715,"inu":3551,"iny":6607,"iko":2695,"ikl":2333,"ila":2431,"in ":6095,"iky":3097,"ikt":1203,"iku":2133,"ilo":3489,"ill":1963,"iln":1971,"ilm":2591,"ili":3988,"ile":1526,"hok":1282,"hol":1704,"hot":1459,"hou":1879,"hov":5549,"hor":3514,"dí ":3801,"hod":8706,"hni":1267,"huj":1855,"hud":2458,"hu ":4702,"hro":1617,"hrn":1403,"hra":7196,"díl":2411,"hyb":1243,"hož":1183,"hy ":3116,"hum":1365,"hrá":2941,"evš":1525,"ext":1409,"etí":2247,"exi":1437,"ezn":2836,"eze":2667,"ezi":7262,"eně":1924,"eta":2522,"ete":4466,"eti":3348,"etn":2128,"esp":1404,"esn":2468,"eso":1292,"est":7801,"ev ":3656,"eto":3093,"etr":4039,"erá":7485,"eve":6187,"evn":2175,"eré":8410,"evi":2322,"erý":10219,"er ":7795,"eor":1444,"es ":7550,"epu":2593,"elé":1267,"epr":1430,"eri":9740,"erg":1674,"ere":2706,"ená":5438,"era":5702,"et ":4584,"emí":4275,"esk":11348,"esi":1360,"ený":6913,"ese":3215,"emě":2087,"erz":1749,"erv":3468,"eru":1909,"ení":16758,"ert":1878,"ers":3684,"ern":7190,"erm":2474,"ené":4823,"ero":6299,"ekl":1253,"eko":2098,"ekt":6413,"en ":12357,"ela":2337,"ele":12125,"eli":4792,"elm":1412,"eln":3331,"elk":5262,"ell":1825,"elo":2426,"els":2269,"ema":1842,"eme":2383,"emn":1840,"emo":2184,"emi":4395,"ene":4580,"ena":4918,"end":1571,"enc":3468,"eno":6243,"enn":2162,"eni":2669,"ens":7642,"ent":12967,"ejí":5227,"eny":2102,"egi":1835,"ej ":1214,"edí":1331,"eho":5425,"ek ":7971,"ein":1192,"el ":8637,"ejs":2194,"ejn":3303,"ejm":2020,"eji":2735,"em ":31739,"ejv":3622,"gin":1170,"gio":1330,"gie":2007,"gic":2475,"gii":1170,"gen":3213,"býv":3044,"být":1189,"gan":3061,"ga ":1144,"íže":1335,"íž ":1472,"fun":1501,"fra":2633,"ače":3487,"ačo":2232,"ačn":3017,"aču":2194,"for":4925,"fot":2015,"bí ":2241,"fic":2581,"fil":3337,"fik":2015,"fin":1574,"fir":1363,"da ":6390,"de ":6350,"dal":3374,"daj":1518,"dat":3084,"dan":2182,"dce":1208,"ctv":1837,"cně":1190,"cko":3237,"chá":6943,"cky":4430,"ciá":2350,"co ":1224,"cká":6180,"cké":15214,"cov":2906,"cou":2423,"cký":12856,"ch ":62063,"ces":2551,"cen":5710,"cem":1899,"cel":4240,"ci ":10328,"cha":4867,"chu":1777,"chy":1469,"cia":1607,"cie":2249,"che":3452,"chl":1932,"chi":2422,"cho":11778,"chn":2940,"cht":1572,"chr":1838,"ed ":3026,"ebn":2564,"ebo":12455,"ec ":4754,"edl":1223,"edm":1145,"edn":17139,"edi":5930,"ede":6652,"eda":1810,"edy":1655,"eds":1915,"edo":3095,"eck":8996,"ech":12275,"eci":1513,"ecn":1350,"dy ":11421,"drá":1267,"dvo":3395,"dva":1139,"dor":1289,"dop":2114,"dom":1869,"dol":1600,"dok":1496,"dov":6030,"dou":1896,"dos":2695,"dpo":1574,"dna":2226,"dne":2707,"dni":1534,"dno":8163,"dob":7531,"dst":2606,"íře":1323,"dně":4952,"duc":1342,"dné":1547,"dra":2227,"dná":2916,"du ":8252,"dro":2166,"dní":18004,"dru":7228,"dsk":2773,"dic":1916,"dia":1263,"der":3512,"des":2009,"dev":1882,"deb":1947,"dec":1456,"del":3171,"den":6683,"dem":3354,"di ":2989,"dle":4381,"dla":1946,"do ":8212,"dlo":2844,"div":1579,"din":4766,"dio":1325,"dis":2494,"die":1484,"rga":2761,"rgi":1143,"ižn":3600,"ret":1327,"res":5841,"nás":2254,"náv":1475,"rez":1829,"náz":3803,"rač":1242,"iž ":1205,"rea":1647,"nác":1291,"rec":1925,"reg":1788,"nám":4897,"rem":3700,"ren":2252,"rek":1273,"nál":3115,"nár":3570,"rep":3246,"rdi":1273,"ná ":14630,"re ":2546,"rch":3978,"rce":1159,"raz":3775,"rd ":1372,"ras":1475,"rat":6479,"rav":9308,"raj":4559,"rah":1855,"ran":13598,"ram":4289,"ral":1943,"rak":3296,"rab":1667,"raf":2398,"rad":5238,"rac":4788,"ros":9033,"rot":4473,"rom":4735,"ron":3499,"rop":5057,"roz":10708,"rou":3738,"rov":17134,"rob":3794,"rod":11155,"roc":8820,"ní ":79844,"roj":4450,"rol":2078,"rok":8745,"rof":1452,"rog":2059,"rno":1887,"rnu":1168,"rna":1957,"rež":1163,"rni":1331,"ném":2468,"rmo":1370,"jší":4959,"ro ":11368,"rma":3339,"riá":1833,"néh":5048,"né ":18438,"raž":1181,"rit":3911,"ris":3258,"rig":1476,"rik":1662,"rin":2410,"ria":2007,"ric":8579,"rid":1291,"rie":3806,"rk ":1359,"ryc":1171,"ruh":6961,"rum":1222,"ruk":1449,"rus":3302,"rva":1915,"rve":2525,"rvn":4272,"ry ":6263,"rsk":6183,"rně":2168,"rst":1339,"rto":1577,"rti":1692,"rub":1147,"rtu":1152,"ním":14655,"ník":6579,"níh":7211,"rmá":1574,"níc":9560,"rt ":1561,"rné":1326,"ru ":6692,"rní":7275,"sah":2489,"měl":2079,"sam":2091,"nýc":8901,"ným":4766,"měn":2340,"san":1523,"sau":1548,"měs":9181,"měr":2410,"sa ":1633,"mě ":2287,"ruž":1312,"ný ":15629,"si ":3518,"měř":1748,"sin":1775,"sil":2216,"se ":32266,"sch":2059,"sev":4239,"ser":1773,"sed":1877,"kří":1234,"sen":1348,"sem":1612,"sel":1714,"spo":10422,"spr":1930,"spe":2260,"spi":1842,"ský":19060,"sou":17153,"sov":4194,"ské":26627,"son":1432,"ská":8637,"sof":1375,"soc":1565,"sob":5198,"su ":2847,"sní":1788,"sné":1153,"st ":11395,"slo":9919,"slu":2567,"sky":9407,"sla":4078,"sle":3342,"skl":1707,"sko":9797,"sku":8770,"ska":4807,"ske":1405,"sni":1352,"smu":2054,"stí":6180,"sté":3269,"stá":6877,"syn":1327,"sys":2806,"stě":3272,"své":1798,"smě":1301,"ste":9093,"sně":1417,"sta":23517,"stn":5308,"sto":14050,"sti":18925,"stl":1904,"stv":6645,"stu":6506,"str":16919,"sts":1291,"ků ":4430,"sty":1890,"sva":1646,"svo":1261,"sy ":1657,"něj":4079,"tak":7734,"něk":4566,"něl":1306,"tal":4758,"tac":1310,"tad":1311,"tba":1700,"tav":8149,"tat":3631,"tas":1264,"tar":5738,"tan":5175,"něn":3988,"něm":4105,"te ":2317,"svě":5020,"stř":6161,"éž ":3711,"ně ":25254,"ta ":12315,"ký ":20031,"ouž":6121,"ozá":1627,"pa ":1199,"ově":6255,"lá ":1917,"kýc":10186,"kým":4999,"par":3389,"pat":3635,"pad":7309,"pak":1199,"pal":1582,"pam":1204,"pan":2500,"pec":1647,"lád":4346,"lán":1802,"pen":1946,"per":4148,"lát":1414,"pla":3524,"ple":1738,"plo":2559,"lé ":2839,"pic":1267,"pin":4585,"pis":3803,"poz":3018,"por":4291,"pop":2679,"pov":4058,"pou":6174,"pot":2565,"pos":5385,"poj":5321,"poh":2575,"pom":2458,"pon":2428,"pok":1746,"pol":12866,"pob":1330,"poc":1426,"pod":13049,"láš":1492,"po ":3633,"lí ":2188,"psk":1538,"hů ":1499,"pub":3056,"poč":3912,"pra":9290,"prv":4970,"psa":1291,"pu ":1292,"pri":1503,"pre":2679,"pro":30809,"prá":4755,"poř":1288,"py ":1312,"prů":1838,"lý ":2387,"má ":3426,"mát":1458,"mán":1640,"išt":1515,"mén":4741,"mí ":3982,"mís":2771,"mír":1517,"mín":2069,"ra ":9350,"eži":1875,"ngl":4299,"ni ":2504,"nge":1442,"nej":9246,"nek":1332,"nen":1766,"nem":4884,"nep":1631,"ner":3084,"net":2605,"nes":3430,"ež ":1798,"ng ":3190,"neb":12487,"nec":2031,"ned":1356,"nfo":1282,"nač":7033,"nez":1422,"nco":2679,"nci":5166,"nce":5874,"ne ":3607,"ndo":1521,"ndi":1883,"nde":1639,"nda":1497,"nak":1732,"nal":3376,"nam":6213,"nan":1425,"nap":3429,"nar":1416,"nac":3724,"nad":3749,"nd ":2404,"nav":1415,"nat":3143,"nas":1417,"naz":1632,"na ":53204,"mys":1461,"mož":1630,"nož":1329,"ny ":15609,"nuj":1460,"nut":2319,"nto":3206,"ntu":1650,"ntr":2728,"nti":4061,"nta":3294,"nte":4199,"nst":3785,"nsk":18177,"nu ":7672,"ičn":2337,"nné":1576,"nt ":3215,"jím":1621,"jíc":10934,"noh":2321,"nol":1273,"nom":2363,"not":3595,"nos":13323,"nor":2008,"nov":9782,"nou":8746,"než":1270,"nno":1485,"nič":2090,"jí ":9261,"neš":1391,"no ":7023,"nka":1355,"nko":1306,"eží":3799,"nih":1649,"nie":1760,"nic":12884,"niz":2240,"niv":1387,"nis":3526,"nit":3056,"nin":2133,"nik":6308,"ogr":3240,"ogi":4525,"odí":1145,"ohl":1659,"oho":4168,"oha":1909,"obě":2552,"oj ":1676,"ok ":1263,"ohy":1525,"ojo":1142,"ojm":1998,"oji":1192,"oje":7276,"odě":2068,"obř":1215,"ol ":1430,"oce":9242,"och":4480,"oci":2151,"ock":1389,"obs":1745,"obv":2042,"obu":1622,"oby":3426,"ká ":15983,"ode":4045,"odl":3777,"odi":2898,"odo":4607,"odp":1587,"odn":14263,"ods":1272,"odr":1258,"of ":1489,"oda":2197,"kán":1145,"ody":1671,"odv":1246,"odu":5226,"ofi":1587,"obí":2810,"oba":1485,"od ":15166,"obo":2639,"obr":2915,"obl":4657,"obn":3328,"obj":2067,"obi":2455,"obd":1788,"obc":2803,"obe":3958,"ový":12973,"orů":1170,"ové":14190,"ozn":6156,"ozl":1181,"ouč":4572,"ozo":1460,"ozd":2636,"oze":3842,"ová":16410,"ozi":1341,"oty":1225,"oud":1374,"oub":1598,"ouc":1601,"otk":1156,"oti":3017,"ote":2572,"oto":6726,"otn":1824,"ost":32198,"ota":1788,"otb":1150,"ov ":3056,"osi":1402,"osk":1491,"ose":1306,"osp":1323,"osm":1339,"osl":4606,"oso":3077,"ovy":1368,"ovi":10902,"ovn":5524,"ovo":6751,"ovs":3742,"ouz":3479,"ova":19936,"ove":6197,"ouh":2082,"oun":1766,"oup":1457,"ous":3223,"out":2402,"opo":2402,"opi":2216,"ope":2692,"opa":2307,"os ":1967,"opu":1150,"opr":3218,"olí":1626,"ops":1833,"or ":5513,"ork":1330,"orm":4795,"orn":2809,"oro":4330,"ord":1649,"ore":3272,"oná":1885,"org":3417,"ori":6242,"ou ":37147,"omě":1557,"osa":2963,"ort":3045,"ors":1818,"oru":3220,"ory":1722,"omá":2183,"ora":3762,"ívá":2944,"íze":1985,"ola":2087,"on ":7329,"oli":8828,"ole":10375,"ols":1995,"oln":2388,"olo":8650,"oly":1795,"olu":2611,"oka":1642,"om ":1785,"íst":3954,"ké ":30775,"ísl":1496,"oke":1629,"íta":1783,"okr":4661,"oko":4002,"íků":1200,"oku":8537,"ona":3352,"ond":1711,"onc":1948,"one":2668,"ong":1188,"oni":3548,"ono":3674,"ons":4146,"ont":3048,"onu":1779,"ony":1528,"oma":2266,"ome":3465,"omi":2447,"kéh":13733,"kém":5563,"omp":1767,"omo":5000,"omu":2175,"íva":3319,"la ":20221,"ími":2048,"íms":1206,"íns":1474,"ím ":20382,"kvě":1156,"ín ":1890,"íle":1385,"le ":15329,"íro":1439,"írk":1295,"íse":1140,"lac":1653,"lad":8838,"ípa":1629,"lan":4986,"lam":1157,"lat":5358,"las":7702,"lav":8047,"krá":4170,"kup":4964,"kum":1330,"kul":2572,"ky ":27061,"ích":18473,"kte":27582,"íce":2062,"ídl":1751,"ců ":1941,"ktr":2141,"ktu":2012,"kti":3026,"kto":1589,"kyt":2072,"ík ":3909,"ící":14415,"ího":9884,"lok":1275,"lon":1573,"lké":1191,"lom":1730,"lod":1650,"loh":1236,"log":5594,"lký":1221,"los":3409,"lou":5777,"lov":17182,"lni":1153,"lež":2967,"lič":1203,"lmi":1186,"ltu":1148,"dů ":1701,"lub":1957,"lsk":6596,"lně":2708,"lné":1553,"lní":11343,"lu ":4880,"lný":1549,"liš":1271,"li ":6590,"lez":2008,"ház":6195,"lev":1768,"les":3807,"let":9902,"ler":1221,"lem":5693,"len":8028,"lek":3907,"lej":1379,"led":7715,"lec":1964,"eň ":1751,"lo ":9285,"lla":1318,"lle":1365,"lli":1334,"lko":2458,"lky":2194,"lka":1525,"leč":4689,"hé ":1767,"lm ":1297,"ll ":1176,"lit":5959,"lis":4420,"lin":6175,"liz":1555,"liv":3315,"lic":7730,"lid":2928,"lia":1350,"lik":5494,"lig":1241,"lie":1691,"ma ":3621,"mac":1223,"maj":1332,"mar":1328,"mal":2994,"man":3894,"mat":5599,"me ":1813,"med":1375,"mec":4748,"met":4900,"mer":5988,"mem":1269,"iál":4186,"men":11122,"mez":6394,"ly ":6413,"lož":5290,"moc":2506,"mob":1461,"mod":2374,"mon":2073,"mov":3077,"mor":1679,"mos":2024,"mot":2793,"mou":1328,"mní":1927,"mu ":8025,"msk":2845,"moř":3176,"my ":2400,"mus":1983,"mun":1943,"mi ":12425,"ešn":1222,"min":3830,"mil":1861,"mis":1525,"ešt":1436,"mic":2749,"mo ":1420,"ií ":2195,"mno":2859,"tří":4710,"tši":1374,"tší":2640,"ůže":1655,"Čes":3563,"vě ":4735,"věz":1289,"vět":11745,"tři":1228,"věk":2228,"tře":8474,"věd":2018,"výr":2025,"výs":1250,"vým":2640,"výz":2611,"výš":1486,"čas":5969,"zná":4232,"uča":1905,"zní":1327,"víc":1941,"čen":7616,"čel":3765,"čet":1680,"čes":6771,"čer":2858,"zný":1440,"zu ":1323,"zsk":2646,"učá":1855,"či ":4977,"rů ":3148,"zuj":1535,"čit":1616,"způ":1462,"čin":2747,"růz":1961,"čka":1735,"zyk":1998,"čle":1629,"čov":3019,"vý ":6798,"čno":3388,"výc":8997,"zi ":5006,"zač":1189,"zej":2847,"zev":2458,"zen":7446,"ván":11057,"zem":8432,"vál":2700,"zel":1300,"zer":1349,"vá ":12375,"ze ":9310,"zde":1205,"zab":1246,"zac":2566,"těj":1439,"zah":1842,"těn":1733,"zal":3135,"těl":1552,"zas":1298,"ví ":7249,"zor":1418,"zov":3172,"zpr":1480,"vém":1518,"rší":1390,"véh":3281,"zna":13601,"zni":4038,"vé ":13437,"zko":1259,"zkr":2046,"zdí":1212,"zař":1286,"zin":1828,"zit":1344,"zdě":2035,"těž":1441,"yrá":1302,"yva":1858,"ytu":1246,"ytv":1652,"yto":1723,"yst":4390,"yso":1389,"ysk":1190,"ysl":1771,"tě ":5527,"za ":5089,"ych":2524,"tým":1477,"tý ":2037,"týc":2116,"ykl":2523,"yla":6390,"ylo":2929,"yly":1314,"yl ":10326,"tím":1511,"tém":3602,"tí ":11245,"též":3726,"pův":2875,"půs":2701,"ože":4937,"ožn":1513,"oži":1506,"tán":1850,"pří":8711,"tát":4498,"ož ":2063,"tál":2187,"té ":3506,"xis":1170,"př ":1535,"tá ":1819,"při":7219,"pře":14856,"sér":1644,"síd":1352,"rče":1253,"oří":2027,"vzn":2563,"vzd":1455,"vyk":2012,"vyr":1739,"vyd":1992,"vys":3366,"vyt":1966,"vyv":1398,"rý ":8817,"vyš":2053,"rýc":1165,"oři":1304,"oře":3041,"vní":15429,"vro":2455,"vrc":1706,"vst":1754,"vsk":4082,"vu ":2977,"nů ":2566,"vně":2710,"vuj":1416,"vy ":4952,"voř":3595,"vil":1507,"vin":6325,"vic":2325,"vid":2543,"viz":1532,"vit":4611,"vis":2095,"ré ":7668,"vla":2806,"vo ":2147,"vna":1206,"vno":1207,"vni":1709,"vod":8774,"voj":3429,"vol":2921,"vor":1476,"vot":1529,"vov":1657,"vou":5924,"voz":2714,"vlá":2701,"vi ":1398,"ver":8833,"ves":2177,"rát":2382,"ráv":4812,"mž ":1199,"rán":2154,"ven":9163,"vem":1785,"rál":6073,"vel":5581,"ráb":1423,"ved":2640,"rác":2253,"ve ":12748,"rá ":6574,"val":7096,"van":12007,"var":2621,"vat":8883,"pěv":1148,"vaz":1671,"vac":2111,"vad":1154,"vaj":2879,"můž":1309,"va ":9872,"uze":1855,"uzs":2041,"utí":1162,"urč":2415,"usk":3978,"use":1494,"umě":1413,"ust":3573,"ute":1270,"mů ":1642,"uto":4678,"us ":7285,"ut ":1210,"ura":2035,"ure":1155,"uri":1389,"urn":1235,"uro":1540,"uru":1160,"ury":1381,"upi":4594,"upe":1451,"upn":1306,"umo":1169,"umb":1314,"ume":1179,"ují":7869,"unk":1600,"uni":2964,"uko":1228,"um ":4532,"ult":2459,"ulo":1228,"uhé":1788,"uli":1303,"ula":1664,"uje":11818,"ucí":1196,"uho":1206,"ude":2608,"udi":1376,"ubo":1407,"uce":1140,"uch":2624,"uh ":1827,"udo":2156,"ub ":1463,"ubl":3299,"ud ":1221,"trů":1395,"tví":6726,"typ":2401,"ty ":7625,"tvr":1530,"očí":2254,"tvo":4249,"trá":2691,"tva":2695,"tur":4631,"tuj":2396,"tup":3639,"tud":1736,"pís":1810,"oče":1902,"tre":1691,"tra":10732,"tné":1153,"oči":1195,"tri":3505,"tru":3242,"tro":11520,"očn":1259,"tní":9445,"tu ":7164,"tný":1437,"tsk":6630,"tně":1562,"lů ":2152,"to ":14761,"lší":1771,"tna":1155,"tno":1779,"toh":1166,"tou":3172,"tov":8578,"tos":1217,"tom":3677,"ton":3319,"tok":1791,"tol":3953,"tor":9066,"top":2079,"til":1384,"tik":3154,"tit":2068,"tis":2205,"tin":7629,"tio":2588,"thu":1152,"tic":11897,"teč":1644,"tiv":5666,"tko":1227,"tka":3033,"tli":2690,"tky":1508,"tla":1310,"tem":4570,"ten":3204,"teo":1298,"tej":1972,"tek":2779,"tel":12611,"tec":6325,"ted":1925,"tev":1290,"ter":37331,"ti ":14561,"tač":2083,"the":1375,"ží ":4449,"žíc":1641,"žív":5443,"yšš":1364,"zýv":1380,"žil":1490,"živ":2602,"žit":2707,"žij":1345,"žov":1554,"že ":5417,"žel":1619,"žen":7485,"záv":1903,"záp":4090,"zák":3066,"uži":2255,"ýt ":1211,"ýro":1252,"ým ":11479,"ými":4161,"žní":2919,"ých":32937,"žsk":1623,"žně":2362,"vší":1568,"zí ":4964,"vša":1280,"vše":1958,"uží":6581,"tů ":3433,"ýzn":2182,"ývá":1959,"ýva":2863,"uše":1228},"n_words":[11333226,13010717,8780627],"name":"cs","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/cy.json b/contrib/languages-data/cy.json
new file mode 100644
index 0000000..a3e1cb8
--- /dev/null
+++ b/contrib/languages-data/cy.json
@@ -0,0 +1 @@
+{"freq":{"Fe ":1539,"D":11333,"E":9591,"F":9129,"G":20345,"A":20881,"B":12687,"C":28561,"L":15711,"M":24164,"N":7579,"O":5361,"H":6663,"I":5301,"J":2920,"K":1955,"U":5682,"T":9914,"W":5139,"V":1175,"Q":331,"P":10920,"S":18433,"R":8785,"Y":6381,"X":156,"Z":358,"f":102454,"g":124045,"d":297794,"e":305845,"b":42206,"c":70455,"Fed":56,"a":340057,"n":303994,"o":193011,"Fei":80,"l":181438,"m":70473,"j":732,"Fel":228,"k":5358,"h":111415,"i":224421,"Fen":103,"w":133473,"v":3889,"u":84011,"Fer":95,"t":96401,"s":97156,"r":275486,"q":393,"p":27532,"z":1653,"y":272195,"x":1194,"Ffo":187,"Ffl":265,"Ffi":428,"Ffe":213,"Ffa":46,"Ffy":47,"Ffu":103,"Ffr":1430,"²":91,"Î":41,"É":51,"À":445,"Fil":43,"ï":652,"î":418,"Fin":48,"í":198,"ë":206,"ê":1481,"Fir":87,"é":1125,"è":159,"ç":67,"æ":50,"ä":70,"Fie":143,"â":2559,"á":343,"à":109,"ü":121,"ú":96,"ù":63,"ö":248,"ô":3067,"ò":62,"ó":166,"ē":42,"ā":208,"ī":83,"ō":123,"ŵ":1259,"ŷ":505,"ū":152,"Faw":386,"Fan":141,"Fal":84,"Far":105,"Fae":78,"Fac":114,"Fai":52,"Erb":44,"Eri":162,"Eth":197,"Eti":126,"Ers":44,"Ery":541,"Esg":154,"Eup":59,"Eur":69,"Eva":147,"Ewr":684,"ˈ":50,"Eit":40,"El ":81,"Eif":65,"Eig":43,"Eid":393,"Ein":91,"Eil":45,"Eis":204,"́":115,"Elf":104,"Ele":70,"μ":79,"ν":135,"ο":198,"Eng":69,"ι":95,"κ":64,"λ":95,"δ":38,"ε":83,"η":58,"α":182,"γ":42,"ά":48,"ί":75,"Emi":37,"Emm":39,"Eml":53,"Eli":163,"Ell":61,"Er ":143,"Enw":854,"ό":61,"σ":58,"ς":162,"ρ":107,"π":56,"Eni":89,"υ":45,"τ":85," l":10735,"ь":66," m":19257," n":29792," o":33276,"я":41," h":11620,"Cân":42," i":18800," j":98," k":864," d":30278," e":20097," f":18797," g":42173,"ч":59,"р":207," a":65561,"с":158," b":14900,"т":99," c":26227,"у":60," y":105922," u":7227," t":11355," w":8138," v":206," p":10362," s":19137," r":34831," J":2904," K":1945," H":6645,"Gel":559,"Gem":88," I":5291," N":7566," O":5351," L":15685,"Gei":1020," M":24137," B":12669," C":28504," A":20837,"С":45," F":9117," G":20264," D":11311," E":9519,"Gea":46,"л":170,"к":145," Z":352," Y":6373,"й":66," X":151,"и":260,"о":284,"н":180,"м":52,"г":58," S":18405,"Ger":213," R":8770,"в":159,"б":38," Q":329," P":10887,"а":295,"Geo":264," W":5129," V":1168,"Gen":357," U":5681,"е":260," T":9888,"д":81," â":1668," ô":1446,"Gla":300,"Gha":77," À":445," É":45," Î":41,"Gil":60," ŵ":72,"ה":41,"ו":50,"י":46,"Gaz":46,"Gan":372,"Gal":398,"Gam":70,"Gat":72,"Gar":516,"Gas":89,"Gai":99,"Gae":1402,"Gad":59,"Gab":57,"و":196,"ي":291,"ف":45,"ق":54,"ك":45,"ل":362,"م":201,"ن":219,"ه":67,"خ":38,"د":148,"ج":68,"ح":77,"ت":86,"ب":192,"ة":126,"ا":513,"أ":38,"ع":70,"ط":52,"ش":46,"س":144,"ز":53,"ر":230,"Fry":84,"Fro":64,"Fyn":271,"Fwy":63,"Flo":39,"Fla":61,"Fle":45,"Fra":319,"Fri":101,"Fre":191,"A ":1088,"Foo":73,"For":307,"Fou":44,"Foe":122,"F ":59,"Da":2498,"Cu":272,"Cw":585,"Cy":7372,"Cl":935,"Cn":42,"Co":2666,"Cr":1941,"Ce":3498,"Ch":2757,"Ci":277,"G ":104,"Ec":68,"Ed":412,"Ea":170,"Eb":590,"Dw":249,"Du":804,"Dy":1018,"Do":759,"Dr":592,"De":2160,"Dd":958,"Di":1824,"Dh":64,"Fe":2231,"H ":155,"Fa":1155,"Eu":257,"Ev":184,"Ew":725,"Ex":44,"Er":1074,"Et":355,"Es":277,"En":1155,"Em":230,"Ep":63,"Ei":1604,"El":634,"Ef":1230,"Eg":302,"Ge":2651,"Câ":42,"Ga":3313,"I ":676,"Fy":369,"Fw":115,"Fu":118,"Fr":789,"Fo":691,"Fl":182,"Fi":488,"Fh":75,"Ff":2731,"Hi ":175,"B ":244," С":45,"II ":307,"C ":911,"Av":93,"Au":228,"Aw":1145,"Ar":2753,"Aq":45,"At":330,"As":660,"D ":151,"Ba":2551,"Az":66,"Ay":58,"Ae":324,"Af":1863,"Ag":155,"Ab":1337,"Ac":626,"Ad":579,"Am":1812,"An":1298,"Ao":86,"Ap":230,"Ai":814,"Al":5324,"Hir":98,"By":401,"Bw":425,"His":66,"Bu":657,"Br":3063,"Ca":7250,"E ":255,"Bh":283,"Bi":359,"Be":2806,"Hig":55,"Bo":949,"Hil":184,"Him":44,"Bl":698,"Hin":102,"Ku":119,"Ky":127,"Kn":99,"Kr":62,"Ko":131,"Le":1967,"Li":730,"N ":179,"La":1295,"Lu":321,"Lw":47,"Ly":372,"Ll":8526,"Lo":2189,"Me":3068,"Iâ":58,"Mh":722,"Mi":742,"O ":337,"Dŷ":53,"Ma":14605,"Mc":91,"My":761,"Mw":171,"Mu":488,"Mr":121,"Mo":2328,"Nh":168,"Ni":590,"Ng":1975,"Ne":1732,"Na":1719,"P ":102,"Hel":181,"Ny":139,"Hei":49,"Nu":77,"Hed":82,"No":968,"Heb":135,"Hea":71,"Ol":406,"On":113,"Og":270,"Oh":42,"Oc":104,"Od":68,"Oe":2267,"Hen":857,"Of":86,"Hes":38,"Oa":53,"Ob":54,"Her":204,"Gi":265,"Gh":165,"Gl":1637,"Gr":2454,"Go":2441,"Gu":315,"Gy":1683,"Cô":47,"Gw":5026,"J ":134,"Ha":1868,"He":1744,"Hi":789,"Ho":750,"Hu":453,"Hw":138,"Hy":775,"र":40,"K ":39,"Ib":42,"Ia":328,"Id":236,"If":69,"Ie":177,"ा":57,"Io":765,"Im":55,"In":884,"Il":79,"Iw":647,"्":49,"Is":833,"It":59,"Ir":229,"Ja":937,"L ":90,"Ji":72,"Je":327,"Jo":1211,"Had":42,"Fô":38,"Jr":44,"Hae":300,"Ju":221,"Haf":198,"Hal":109,"Ka":451,"Hai":43,"M ":130,"Han":221,"Ham":150,"Kh":108,"Har":406,"Ki":402,"Haw":74,"Ke":308,"Hau":114,"Ut":49,"Ur":136,"Un":2471,"Ul":77,"Uc":2348,"W ":83,"Gyr":41,"Ty":474,"Tw":270,"Tu":595,"Gym":1100,"Tr":1825,"Ts":360,"Gyn":163,"Pê":44,"To":757,"Gyd":159,"Th":1526,"Ti":598,"Te":1081,"Gyf":142,"Ta":2097,"V ":167,"Sw":1250,"Sy":524,"St":1498,"Su":393,"Wr":303,"Wo":329,"Wi":1101,"Wh":150,"Wl":140,"Wa":840,"We":1710,"Y ":1318,"Vo":87,"Vi":334,"X ":53,"Va":335,"Grŵ":144,"Ve":206,"Uw":129,"Pw":190,"Pu":305," م":63,"Pr":1739,"S ":764,"Py":119,"Gua":65,"Pe":3405,"Pa":1592,"Gui":67,"Pl":525,"Po":1922,"Pi":467,"Ph":393,"Os":135,"Ot":47,"Ou":45," ا":223,"Op":59,"Or":342,"Gŵ":111,"R ":160," ب":53,"Ow":296,"Oy":53,"Se":1837,"Sc":412,"Si":2884,"Sh":621,"Sg":470,"Sm":109,"Sl":145,"Sk":83,"Sr":86,"Sp":262,"So":559,"Ru":535,"Ry":288,"Gwr":140,"Rw":509,"U ":156,"Sa":6147,"Sb":343,"Gwy":2266,"Gwe":1233,"Re":503,"Ri":460,"Gwa":335,"Rh":3082,"Gwo":134,"Gwn":52,"Gwl":709,"Ro":2605,"Gwi":108,"Qu":256,"Mô":742,"T ":147,"Ra":632,"Tŷ":84,"Gre":610,"Gri":267,"Gra":555,"Glö":96,"b ":3328,"Gru":159,"Gro":691,"a ":51218,"Ye":72,"Ya":90,"Yn":2049,"Ym":722,"Yo":129,"Yu":66,"Yr":659,"Ys":1224,"Gle":844,"Wy":417,"Glo":100,"Gly":254,"Goo":37,"Gol":183,"Gom":67,"Got":62,"Gor":1252,"Za":73,"Ze":78,"Zi":43,"Zu":39,"God":47,"Gog":472,"Goc":85,"i ":38903,"Inv":100,"fw":1978,"fy":10068,"Ioa":44,"gd":137,"ge":9102,"gf":868,"ga":16120,"gb":189,"fl":2826,"fg":93,"ff":11831,"fi":6912,"fs":46,"bî":58,"fr":8643,"fu":2341,"ft":461,"Int":85,"fo":10368,"fn":3165,"j ":48,"gy":9888,"gw":9737,"hf":267,"dâ":43,"hg":350,"hd":1537,"he":14852,"hb":108,"ha":19081,"gn":416,"gm":64,"gl":7224,"gi":3529,"gh":4830,"gg":166,"Ion":616,"gu":1382,"gt":170,"gs":165,"gr":6508,"Ior":53,"go":16514,"dt":42,"du":6001,"dw":8140,"dy":12449,"g ":36598,"ea":4976,"eb":3180,"ec":2955,"ed":44549,"de":16065,"dd":84524,"dg":311,"df":1708,"di":29599,"dh":972,"dm":105,"dl":3195,"do":14938,"dn":1255,"ds":484,"dr":8475,"ew":8995,"ex":424,"eu":11450,"ev":608,"ey":2052,"ez":173,"fa":9608,"h ":26458,"Ind":510,"fd":990,"fe":14177,"fb":102,"eh":1107,"eg":15169,"ef":16678,"ee":943,"el":20723,"ek":164,"ej":41,"ei":27830,"ep":1574,"eo":4494,"en":34977,"em":3744,"et":14347,"es":16689,"er":35301,"ca":8269,"e ":32522,"bw":1267,"by":5500,"bs":99,"br":4597,"bu":1226,"bn":48,"bo":4368,"bl":4571,"bh":358,"bi":1556,"bb":144,"bd":48,"be":6777,"dc":38,"db":94,"da":17715,"f ":18520,"cy":8823,"cw":614,"cu":488,"ct":1580,"cs":549,"cr":2262,"co":5637,"cm":48,"cn":51,"ck":930,"cl":1111,"ci":1587,"ch":26374,"ce":3041,"cc":166,"c ":8637,"az":312,"ay":1012,"ba":8027,"d ":90625,"at":8544,"as":12110,"ar":38569,"ax":139,"aw":9171,"av":851,"au":19336,"ak":678,"al":15639,"ai":28376,"aj":129,"ao":569,"ap":1859,"am":10300,"an":51341,"ac":11287,"ad":20781,"aa":163,"ab":3228,"ag":4220,"ah":1181,"ae":38165,"af":10668,"nu":1924,"nt":12389,"ns":2901,"nr":2284,"Ifa":40,"np":53,"no":14100,"nn":11240,"nz":130,"ny":12248,"nw":13858,"nv":170,"oe":17244,"of":2843,"oc":4164,"od":16702,"oa":475,"ob":4169,"om":3404,"on":23684,"ok":359,"ol":31043,"oi":1499,"og":11702,"oh":1022,"ot":1836,"m²":86,"os":6644,"ov":478,"ou":2364,"op":7305,"oo":863,"or":19871,"r ":97506,"gŵ":65,"ox":76,"ow":1905,"oz":69,"oy":522,"pe":4549,"lâ":62,"pa":7810,"pl":881,"lê":981,"po":1801,"ph":2260,"pi":2423,"lo":8056,"ln":84,"lm":2285,"ll":25680,"ls":592,"lr":78,"lp":247,"lw":7516,"lv":181,"lu":4776,"lt":3355,"Idd":114,"lz":47,"ly":9229,"hô":55,"o ":32658,"dŵ":331,"Idr":57,"mc":302,"md":1313,"ma":12774,"mb":1263,"mg":609,"mh":1304,"me":11191,"iâ":71,"mf":130,"ml":2070,"mi":3619,"mn":407,"mm":469,"mp":1485,"mo":3170,"mr":6007,"mt":173,"ms":740,"mu":2756,"Iei":49,"iô":76,"mw":2666,"my":4653,"p ":2056,"Ies":43,"na":18605,"nb":1462,"nc":2771,"nd":8926,"ne":28781,"nf":2039,"ng":19411,"nh":3006,"ni":14670,"nj":135,"nk":382,"nl":948,"nm":771,"Ieu":53,"fô":44,"ju":54,"fî":107,"jo":72,"ki":563,"kh":206,"gâ":69,"ke":713,"ka":582,"m ":12759,"ky":143,"ks":215,"ku":123,"ko":255,"kr":68,"kk":45,"kl":117,"km":709,"kn":46,"gê":84,"li":13502,"lh":333,"lk":174,"há":37,"le":25308,"ld":3291,"lg":727,"lf":1785,"hâ":88,"la":22265,"lc":1274,"lb":4658,"n ":129973,"dî":57,"hr":4162,"hs":206,"hw":8162,"ht":520,"hu":3049,"hi":7126,"hn":1183,"ho":8111,"hl":2073,"hm":233,"dé":370,"id":14418,"ic":5942,"ib":1579,"ia":26223,"ih":218,"ig":11702,"if":9533,"ie":4390,"hy":13222,"k ":1329,"iq":80,"ir":24552,"is":9056,"it":12154,"iu":514,"iv":491,"iw":5398,"ix":147,"ii":107,"ij":103,"ik":308,"il":12119,"im":1505,"in":29957,"eë":45,"io":13225,"ip":837,"je":73,"Iar":71,"ji":138,"iz":277,"iy":334,"Ian":45,"l ":44324,"ja":217,"Iai":126,"xi":200,"xo":38,"té":52,"tî":45,"xt":49,"wy":36348,"ww":54,"z ":355,"xa":159,"tâ":87,"xe":70,"wg":664,"wi":6651,"wl":4892,"sé":43,"wm":1763,"wn":9601,"sê":51,"wo":2007,"wp":170,"wr":12516,"ws":2657,"sï":46,"wt":289,"wu":53,"rô":114,"y ":43036,"wb":421,"wa":8291,"wd":1844,"wc":1767,"wf":218,"we":14515,"ré":173,"vi":1082,"râ":57,"vu":50,"vr":48,"rï":40,"rë":66,"vo":243,"uz":117,"uy":100,"ux":122,"uw":1502,"uv":98,"ve":1501,"va":690,"x ":538,"ui":873,"uk":137,"ul":2825,"ue":842,"uf":864,"ug":2077,"uh":77,"ur":9114,"us":3544,"ut":1125,"um":1436,"un":9776,"uo":2173,"up":276,"ty":2334,"tz":134,"tu":4774,"tt":1060,"tw":874,"ub":587,"ua":3026,"ud":3346,"uc":1723,"w ":28552,"lŷ":175,"to":4656,"tn":397,"pê":160,"tm":136,"tl":749,"ts":824,"tr":9852,"tp":42,"tg":191,"tf":144,"te":8913,"tk":40,"ti":6270,"th":34123,"v ":135,"tb":311,"tc":225,"ta":7641,"su":1034,"ss":1407,"Hyw":74,"st":10674,"sy":11549,"sw":1131,"sl":613,"sk":338,"sn":4104,"sm":342,"sp":272,"so":5309,"sr":218,"sd":421,"sc":590,"sf":301,"se":6958,"Hyd":590,"sh":1353,"sg":5428,"si":7703,"rz":64,"u ":38069,"sa":4420,"sb":869,"rr":3139,"rs":2573,"nï":350,"rt":9794,"ru":5227,"rv":239,"rw":8072,"ry":10076,"rq":40,"rp":904,"ro":15586,"në":57,"rn":7321,"né":106,"rm":2771,"rl":5238,"nç":49,"rk":493,"ri":23922,"rh":9744,"rg":2743,"rf":5415,"re":22722,"rd":11710,"rc":3476,"rb":1538,"ra":23899,"t ":12191,"mô":280,"qu":314,"Hwn":76,"mâ":48,"s ":31741,"hŷ":72,"px":78,"lô":60,"py":248,"lö":131,"pt":1306,"pu":631,"pw":742,"pp":348,"pr":1765,"ps":228,"hū":85,"Hun":62,"Hum":61,"Huw":52,"Hug":142,"tŷ":48,"IV ":48,"wâ":83,"rŵ":459,"zz":70,"Hor":65,"zh":71,"zi":170,"Hou":114,"zb":37,"ze":198,"za":330,"Hom":49,"yy":47,"Hon":146,"Hol":122,"zu":71,"zo":149,"How":58,"yg":2872,"yh":854,"ye":236,"yf":12232,"yc":3592,"yd":50329,"་":38,"ya":2335,"yb":822,"yw":21420,"yu":51,"yt":2465,"ys":12384,"yr":22478,"yp":196,"yo":362,"yn":74012,"ym":17663,"yl":4484,"yk":65,"IR ":39,"yi":95,"Arg":192,"Are":81,"Arf":161,"Arc":172,"Ard":330,"Ara":645,"Arb":38,"Arm":93,"Ark":49,"Arl":228,"Ari":157,"App":76,"Apo":76,"Ath":193,"Atl":65,"Ast":95,"Ass":90,"Asi":362,"Art":218,"Arw":129,"Avo":40,"Aur":46,"Aug":57,"Awy":60,"Aws":918,"Awd":150,"Bai":38,"Bal":342,"Ban":571,"Bab":49,"Bac":149,"Bad":45,"Bae":152,"Baf":40,"Bay":40,"Bar":705,"Bat":87,"Bas":189,"CC ":319,"BE ":140,"Abr":43,"Aca":159,"Abe":1076,"Aba":80,"Act":324,"Ada":89,"Ach":90,"Adn":48,"Adr":99,"Add":86,"Ade":142,"Ael":205,"Aer":53,"Afo":1268,"Aff":460,"Afg":44,"Ago":41,"Aif":105,"Ail":151,"Ain":442,"Air":55,"Al ":98,"Am ":62,"Ala":129,"Alb":3548,"An ":121,"Alg":86,"Ali":71,"Ale":206,"Alf":52,"Alu":44,"Alm":666,"All":174,"Alp":112,"Amg":74,"Ame":1251,"Ama":68,"Amw":99,"Ams":41,"Ang":201,"Ani":79,"Ana":70,"And":209,"Ant":265,"Ann":197,"Ar ":131,"Aon":53,"But":65,"Bul":39,"Bur":114,"Buc":38,"Bui":58,"Bu ":164,"Bry":805,"Bru":120,"Brw":176,"Byd":283,"² ":91,"Bwl":182,"Bwr":151,"DA ":97,"Cab":43,"Cad":213,"Cae":1032,"DD ":40,"Cal":219,"Cam":227,"Caf":480,"Cai":366,"Cas":915,"Car":1877,"Cat":223,"Can":1350,"Cap":159,"Caw":62,"Bea":150,"Bet":175,"Ber":402,"Ben":733,"Bel":274,"Bei":579,"Bed":337,"Bhr":46,"Bhu":62,"Bil":57,"Bis":38,"Bir":54,"Bha":56,"Bhe":84,"Ài":440,"Blu":41,"Blo":83,"Ble":43,"Bla":500,"Bre":500,"Bra":479,"Bro":495,"Bri":461,"Bod":96,"Bob":38,"Bol":50,"Bon":223,"Boo":44,"Bor":128,"Bos":53,"Bou":115,"Bow":40,"Cyc":57,"Cyd":52,"Cys":83,"Cyr":75,"Cyt":157,"Cyn":1224,"Cym":4367,"Cyl":131,"Cyf":1040,"Cyh":146,"Cur":60,"Cul":80,"Îl":39,"Cwm":473,"De ":401,"Dey":467,"Der":96,"Des":74,"Deu":61,"Dew":73,"Del":66,"Dem":67,"Den":121,"Dea":123,"Dec":105,"Ded":67,"Def":186,"Deh":65,"Ddy":46,"Ddw":51,"Ddu":131,"Ddi":405,"Dda":107,"Dde":168,"Dam":55,"Dan":185,"Dar":231,"Dat":83,"Dav":367,"Daw":158,"Dad":40,"Daf":217,"Dae":153,"Dai":40,"Dal":797,"Cho":221,"Chr":239,"Che":165,"Chi":233,"Chl":64,"Chu":124,"Chw":858,"Chy":89,"án":75,"ái":74,"Cit":58,"Cil":97,"âl":86,"ân":336,"âd":45,"âu":90,"ât":37,"âr":140,"Cle":140,"Cla":218,"Cea":48,"Cei":925,"Cef":291,"Cem":52,"Cel":239,"Cen":423,"Cer":1421,"Cha":693,"â ":1741,"Cri":276,"Cra":198,"Cre":385,"DU ":59,"Cry":52,"Cru":632,"Cro":352,"Cly":91,"Cli":57,"Clo":101,"Clw":225,"Clu":93,"Coc":117,"Coi":141,"Cof":107,"Coe":237,"Cop":81,"Cos":46,"Cor":407,"Com":238,"Col":320,"Con":652,"Cou":119,"アアア":57,"ôr":762,"ôl":1584,"ôn":601,"ón":72,"Dyw":75,"ïo":107,"Dyd":48,"Dyc":49,"în":138,"Dym":227,"Dyn":66,"Dys":44,"Dyf":415,"ïa":463,"Dyl":71,"Dwy":227,"îl":47,"îm":141,"î ":61,"ëw":53,"Egl":261,"Efr":172,"í ":45,"Efy":881,"ëd":42,"Ei ":672,"ëe":56,"ên":983,"êl":271,"êm":109,"êr":91,"él":41,"ép":354,"ém":39,"én":68,"ér":78,"ée":104,"ég":64,"Edw":243,"Edm":38,"èr":41,"ès":40,"Ef ":107,"ço":37,"é ":170,"Ebr":548,"Ear":39,"Eas":62,"ān":56,"Dia":78,"Dic":59,"Dis":198,"Dir":57,"Dio":56,"Din":930,"Dig":90,"Dif":46,"Diw":144,"Dug":61,"Dub":66,"Dun":80,"Dul":102,"Dui":91,"ể":69,"ür":59,"Duw":89,"Dur":65,"Dro":45,"Dru":57,"Dry":63,"Du ":131,"öy":129,"Dri":53,"Dre":173,"Dra":145,"Doc":73,"Dr ":38,"Dou":83,"Dow":41,"Dol":168,"Don":111,"Dom":58,"Dor":76,"Ned":55,"Nev":46,"Neu":46,"Nep":47,"Neo":67,"Ne ":182,"Nat":1075,"Nig":61,"Nid":87,"Nic":115,"Nin":78,"Nha":51,"Nhr":45,"Ngh":1380,"Nge":64,"Ni ":64,"Ngw":317,"Ngo":144,"New":1034,"Myn":603,"Nar":52,"Nan":259,"Nad":69,"OC ":70,"Nyf":95,"OS ":481,"Nor":455,"Not":37,"Nof":207,"Oes":2225,"Ogw":150,"Ogo":40,"Ogl":71,"Off":71,"PA ":37,"Oak":42,"Owa":159,"Owe":132,"ī ":39,"Oyk":41,"Oly":162,"Oli":119,"Ope":42,"Ora":40,"Ori":47,"Org":37,"Orl":66,"Plw":64,"Pla":369,"Pin":50,"Pil":40,"Pit":98,"Pie":81,"Pic":69,"Pho":86,"Phi":138,"Phr":55,"Phe":40,"Pea":46,"Ped":89,"Per":357,"Pet":166,"Pen":2562,"Pel":49,"Pat":108,"Pas":79,"Par":525,"Pau":125,"Pab":54,"Pac":49,"Pan":158,"Pap":93,"Pal":184,"Pak":50,"Gŵy":103,"ō ":54,"Pyr":65,"Pwy":112,"Pwl":49,"Pum":82,"Pur":41,"Pug":63,"Pro":243,"Pri":768,"Pre":133,"Pry":452,"Pra":116,"Pob":179,"Pol":106,"Pon":298,"Poi":41,"Pot":63,"Pos":39,"Por":358,"Pow":700," ال":192,"Rac":38,"Rad":90,"Rai":42,"Ram":56,"Ran":141,"ū ":77,"Môr":383,"Môn":358,"ŷ ":235,"ŵl":41,"ŵn":87,"ŵp":443,"ŵr":393,"Que":103,"ŵy":196,"Quo":68,"ŷn":164,"ŷr":76,"Isa":101,"Ise":233,"Isl":207,"Isr":118,"Is ":89,"Ira":166,"Iwe":600,"Jac":132,"Jap":343,"Jan":70,"Jam":257,"Jen":67,"Jer":98,"Jea":66,"Jim":37,"あ":42,"Jos":121,"Jon":426,"ア":100,"Joh":529,"Jr ":44,"Jul":71,"Jun":38,"Kan":77,"Kat":68,"Kas":44,"Kar":90,"Ker":43,"Ken":91,"Kel":37,"Kin":187,"Kil":92,"Kha":59,"Kno":77,"Kyl":83,"Lew":127,"Lev":111,"Les":58,"Ler":63,"Lep":994,"Leo":78,"Lei":113,"Leg":43,"Lee":38,"Lea":111,"Lau":47,"Law":91,"Le ":50,"Lai":41,"Lag":42,"Laf":70,"Las":63,"Lar":55,"Lao":55,"Lam":49,"Lan":381,"Lad":104,"La ":83,"Lli":81,"Llo":1177,"Lla":2896,"Lle":1205,"Lly":1369,"Llw":218,"Llu":548,"Lib":127,"Lia":147,"Lim":38,"Lin":147,"Lit":62,"Lun":66,"Luc":59,"Llŷ":126,"Llê":901,"Lou":107,"Los":46,"Loi":64,"Loe":137,"Loc":1370,"Lor":55,"Lon":140,"Lom":91,"Dŷ ":53,"Lyd":44,"Lyn":107,"Lyo":122,"Mei":183,"Meh":551,"Men":120,"Mel":122,"Mes":98,"Mer":331,"Mew":209,"Meu":40,"Met":107,"Mec":52,"Mea":441,"Med":618,"Mex":110,"Man":377,"Mao":42,"Mal":270,"Mam":89,"Mar":903,"Mas":96,"Mag":89,"Mad":118,"Mae":10140,"Mah":50,"Mai":677,"Mac":250,"Mab":124,"May":55,"Max":37,"Mau":68,"Mat":309,"Maw":799,"Moc":42,"Mod":56,"Moe":284,"Moh":40,"Mon":614,"Moo":54,"Mos":98,"Mor":865,"Mou":90,"Mot":83,"Mid":69,"Mic":171,"Mit":41,"Mis":80,"Mil":151,"Min":107,"Mhe":110,"Mha":155,"Mhu":72,"Mhr":108,"Mho":230,"Mun":41,"Mul":66,"Mur":72,"Mus":84,"Mud":92,"Mwr":59,"Wyd":174,"Wyn":117,"Wys":47,"Wre":214,"Wor":81,"Woo":67,"Wob":86,"Wla":90,"Whi":104,"Wil":763,"Win":87,"Wic":114,"ère":37,"Web":41,"Wed":39,"Wef":896,"Wei":199,"Wel":99,"Wer":95,"Wes":154,"Wen":117,"Wav":42,"Was":76,"War":117,"Wau":61,"Wat":101,"Wal":281,"ès ":39,"égi":56,"ée ":55,"ées":42,"éné":40,"épa":346,"之":120,"両":62,"並":68,"丘":45,"三":165,"丁":71,"ên ":981,"êm ":104,"êl ":267,"Ysb":66,"Ysg":820,"Yst":328,"Yr ":655,"Ym ":65,"Yn ":880,"Yml":71,"Ymr":46,"Ymd":103,"Yme":273,"Ymg":39,"Yma":48,"Yng":84,"You":59,"Yny":1064,"êr ":90,"ëdi":40,"ëeg":55,"Syl":57,"Syr":267,"Sys":57,"Syd":49,"Swy":845,"Swi":176,"Swe":125,"Swa":46,"Sur":47,"Sus":41,"Sum":39,"Sul":51,"Sun":61,"Str":352,"Stu":76,"Sti":42,"Sto":284,"Sta":278,"Ste":342,"Teg":58,"Tec":38,"Ten":65,"Tei":113,"Tel":116,"Tam":59,"Tan":88,"Tat":47,"Tar":132,"Taw":121,"Tay":77,"Tai":74,"Tal":579,"Taf":220,"Tac":506,"Shi":200,"She":116,"Sho":72,"Sgu":236,"Sha":163,"Sim":72,"Sil":92,"Sis":59,"Sir":1857,"Sio":82,"Sin":79,"Sid":94,"Sie":50,"Sib":37,"Sia":245,"Ser":173,"Sgo":69,"Sgi":37,"Sen":310,"Sel":127,"Sem":38,"Sei":620,"Sef":291,"Sea":62,"Sro":50,"TV ":60,"Spa":48,"Spi":76,"Spe":51,"St ":109,"Sof":96,"Soc":69,"Sou":117,"Sol":60,"Som":57,"Son":51,"Sla":57,"Slo":44,"Sma":50,"Siô":76,"Rws":463," 三":37,"Ryg":65,"Ryd":65,"Ryf":105,"Rus":66,"Rut":39,"Ruf":288,"Saf":103,"Sai":1382,"Sah":39,"Sam":100,"Sal":155,"Sac":72,"Sae":3187,"Sad":43,"Sco":119,"Sci":39,"Sch":160,"Sca":61,"Sat":44,"Sau":57,"Sar":112,"San":631,"Sba":324,"Res":39,"SJ ":70,"Rho":422,"Rhu":416,"Rhi":208,"Rha":817,"Rhe":292,"Riv":41,"Ric":257,"Rhy":836,"Rhw":49,"Ras":58,"Ree":51,"Red":74,"Rei":43,"Reg":46,"Ren":42,"SH ":92,"Rea":48,"Roi":127,"Roe":1456,"Rog":56,"Roa":53,"Rob":347,"Roc":57,"Roy":48,"Rou":63,"Ros":111,"Ron":57,"Rom":81,"SS ":37,"ST ":48,"SN ":103,"SO ":61,"Vai":45,"Ven":60,"Van":61,"Val":137,"Vic":94,"Vir":48,"Vin":38,"Ver":74,"Und":296,"Une":566,"Uni":136,"Uno":738,"Un ":695,"Urd":80,"Uwc":129,"Twi":39,"Twr":136,"Twy":43,"Tyw":284,"Tyd":45,"Uch":2348,"Teu":41,"Tex":55,"Ter":240,"Tes":46,"Tey":148,"Tha":114,"The":818,"Thi":43,"Tho":319,"Thr":141,"Tib":106,"Til":75,"Tim":107,"Tit":38,"Tir":127,"Pêl":41,"Tor":142,"Tok":42,"Tom":170,"Ton":77,"Tow":45,"Tou":86,"Try":61,"Tru":44,"Tro":153,"Tri":147,"Tre":944,"Tra":432,"Tsi":288,"Tur":62,"Tun":253,"Tud":152,"вич":38,"bl ":1207,"biw":40,"bit":41,"bio":221,"bir":39,"bil":108,"bin":244,"big":37,"bo ":73,"bly":678,"blw":152,"blo":1344,"ble":366,"bli":117,"bla":667,"bod":1326,"bob":1364,"bol":373,"br ":444,"bon":391,"bor":263,"bot":170,"bos":79,"bou":98,"be ":98,"ban":3482,"bal":155,"bai":223,"bag":40,"bae":396,"bac":337,"bad":90,"bab":43,"án ":53,"baw":50,"bau":111,"bat":176,"bas":114,"bar":2113,"bi ":299,"bei":410,"beg":385,"bed":758,"bec":63,"ber":2389,"ben":1862,"bel":400,"bet":283,"bh ":143,"bia":315,"bid":38,"bie":42,"bha":140,"byw":545,"ca ":1177,"car":1065,"cas":275,"cat":148,"can":2308,"cap":69,"caw":70,"cae":1029,"cad":459,"cam":299,"cal":289,"caf":386,"cai":547,"ce ":639,"bri":1804,"bro":531,"bra":416,"bre":653,"bry":598,"bu ":507,"brw":83,"bur":297,"bul":47,"bum":71,"bud":56,"but":37,"bus":107,"by ":89,"bwl":66,"bwm":84,"bwr":254,"bwy":699,"byd":1610,"byc":1183,"bym":110,"byn":951,"byg":624,"bys":89,"byt":107,"byr":125,"aka":125,"am ":3780,"ake":153,"aki":95,"akh":61,"al ":3450,"ail":1481,"aim":47,"ain":6784,"aeë":39,"air":2398,"ais":581,"ait":4733,"ak ":74,"aig":842,"aif":1555,"aid":7159,"aic":285,"aia":54,"agw":108,"aha":670,"agl":348,"agf":578,"agg":47,"agh":77,"agi":70,"agu":99,"agn":135,"ago":1155,"aoi":58,"aol":79,"aod":56,"aob":43,"anw":1721,"anu":408,"anz":40,"any":320,"ano":3132,"ann":2928,"anm":62,"ant":3519,"ans":1375,"anr":1027,"ane":3419,"anf":695,"ang":2452,"anh":308,"ani":1520,"ank":171,"anl":302,"ap ":434,"ana":2445,"anb":852,"anc":833,"and":2956,"amu":51,"amw":69,"amm":156,"aml":1316,"amo":343,"amp":711,"ams":445,"amr":285,"amh":93,"ami":272,"amg":269,"amd":204,"ame":669,"amb":378,"amc":173,"ama":982,"ao ":50,"aly":92,"alw":828,"alu":142,"alt":188,"als":144,"alp":44,"alo":371,"alm":88,"all":3308,"alk":68,"alg":97,"ali":1102,"alc":202,"ald":305,"ale":2251,"alf":125,"ahâ":37,"ala":2417,"alb":252,"an ":20701,"ako":53,"aba":276,"abe":698,"abh":50,"abi":212,"abl":167,"abo":216,"abr":69,"abw":100,"aby":812,"ae ":13921,"aca":158,"ad ":6988,"âr ":137,"ac ":6256,"ab ":524,"afn":71,"afo":1900,"afr":202,"aff":533,"afe":74,"afi":258,"afl":340,"ai ":2298,"aga":251,"age":183,"afu":183,"afw":88,"afy":180,"aeo":298,"aen":2354,"aem":216,"ael":2752,"aes":3677,"aer":2715,"aeg":1463,"aed":256,"ah ":232,"afa":399,"afb":52,"aew":200,"aet":9257,"ado":1424,"adr":412,"adl":478,"adn":878,"adh":309,"adg":37,"adi":858,"add":2640,"âu ":90,"adf":220,"ade":1321,"aea":937,"ag ":1062,"adw":1093,"ady":61,"adu":1508,"aco":112,"acl":41,"ack":236,"aci":176,"ach":3246,"ace":317,"ada":2456,"af ":6324,"act":455,"acu":37,"acr":47,"acs":87,"azi":49,"aza":106,"awy":402,"ayn":40,"ays":51,"aya":127,"aye":44,"ân ":313,"ba ":592,"âl ":80,"at ":1109,"arh":141,"arg":679,"arf":1468,"are":1082,"ard":3174,"arc":1261,"arb":599,"ara":2389,"arp":203,"aro":1162,"arn":2998,"arm":191,"arl":1098,"anç":39,"ark":227,"ari":1312,"aru":190,"arv":109,"arw":1994,"arr":856,"ars":273,"art":3281,"au ":18148,"asa":524,"ary":397,"asg":690,"asi":753,"ash":208,"asc":78,"ase":115,"asd":54,"aso":389,"asn":260,"ask":48,"asm":43,"aon":63,"aor":71,"aos":40,"ar ":13348,"apa":506,"ape":190,"api":138,"aph":68,"apo":90,"app":56,"apt":39,"apu":247,"as ":6724,"ava":74,"ax ":61,"aut":123,"avo":39,"avi":469,"ave":206,"awe":760,"awd":1261,"ay ":572,"awa":150,"awr":2782,"aws":410,"awn":857,"awl":555,"awf":124,"awg":51,"awi":82,"atb":229,"ata":746,"asu":218,"ast":1476,"ass":281,"asy":50,"asw":75,"atm":41,"atr":340,"ato":500,"âd ":43,"ate":809,"atc":82,"ati":760,"atg":130,"ath":1938,"aw ":1576,"att":185,"ats":96,"atu":1218,"atw":81,"aty":152,"aul":281,"aun":217,"aur":199,"aus":70,"aud":91,"aug":56,"ος":78,"ος ":78,"ς ":162,"α ":57,"アア":78,"Tŷ ":84,"ич ":40,"jan":42,"ск":52,"itl":241,"itr":65,"ito":86,"itu":48,"itt":152,"itz":75,"ity":126,"iw ":670,"ism":50,"isl":41,"iso":170,"isn":521,"iss":179,"ist":1922,"isw":48,"isy":118,"ita":465,"itc":77,"ite":660,"ith":9523,"iti":268,"iwd":40,"iwe":1260,"iwb":73,"iwc":99,"iwa":49,"iwl":119,"iwm":233,"iwi":130,"iwg":119,"ius":367,"ium":95,"iva":70,"ix ":95,"ivi":57,"ive":318,"ipp":108,"ips":45,"ipt":130,"ipi":109,"ipl":76,"is ":2910,"ion":4410,"iop":79,"ior":107,"ios":73,"iot":52,"iog":969,"iol":2152,"ipe":92,"ir ":16769,"irw":755,"irs":63,"irt":106,"irr":47,"iro":1087,"inë":53,"irn":1040,"irk":39,"irl":116,"iri":2683,"isi":1027,"ish":225,"isg":710,"ise":398,"isd":236,"isc":102,"isb":45,"isa":207,"iry":54,"iqu":69,"irf":117,"ire":717,"irg":231,"ira":269,"ird":247,"irc":70,"it ":213,"ja ":42,"iyn":234,"iya":51,"iwy":1090,"iwn":438,"iwt":68,"iwr":950,"iws":49,"izo":45,"iza":90,"kil":38,"kin":180,"kis":60,"km ":586,"ki ":114,"kha":65,"kel":108,"ken":115,"kes":50,"ker":140,"gân":47,"ke ":171,"ku ":48,"km²":84,"ks ":103,"gêm":78,"ko ":44,"kle":37,"kla":44," Ài":440,"fîn":104,"kar":68,"kan":58,"kai":44,"ka ":196," Ga":3309," Câ":42," Ge":2633," Fy":369," Fw":115," I ":222," Fo":687," Fu":118," Fr":788," Fh":75," Fi":485," Ff":2729," Fl":182," Ha":1860," He":1741," Cô":47," Gw":5007," Gy":1682," J ":50," Go":2436," Gr":2427," Gu":311," Gh":165," Gi":263," Gl":1637," If":69," Ie":177," Id":236," Ib":42," Ia":327," Hy":775," Hw":138," Hu":453," Ho":749," Hi":788,"ha ":291," Ji":72," Je":327," L ":42," Ja":936,"ان ":40," Iw":647," Ir":229," Is":832," It":59," Im":55," In":879," Io":765," Il":78,"ham":591," M ":53,"han":7727,"hao":141,"hap":50,"hai":1115," Ka":449,"hak":48,"hal":1482," Ke":303,"hau":1360,"haw":364," Ki":398,"har":2065,"has":612," Kh":108,"hat":129," Jo":1198," Ju":220,"haf":786,"hae":657," Jr":44,"hag":961,"hab":62," Fô":38,"had":434,"hac":101," La":1290," Le":1952," Li":729," Kn":99," Ko":131," Kr":62," Ku":119,"hbe":70," Ky":127," Mc":91," Ma":14593," Dŷ":53," O ":158," Mi":739," Mh":722," Me":3065," Iâ":58,"he ":1354," Lo":2188," Ll":8523," Ly":372," Lw":47," Lu":320," Ne":1728,"а ":68,"hdr":51," Na":1715," Ng":1975," Nh":168," Ni":588," Mr":121,"hda":91," Mo":2327," My":759,"hde":1261," Mu":485," Mw":171,"hel":2523,"hei":954,"heg":242,"hee":57,"hef":2098,"hec":90,"hed":1184,"hea":360,"heb":374,"hey":61,"hew":93," A ":750,"heu":445,"het":177,"hes":532,"her":1843,"heo":518,"hen":1636,"hem":246,"hfe":111,"hfa":83,"hi ":668," B ":201,"hga":94," C ":232,"hgy":60,"hgr":144," Ao":85," Ap":230," Am":1811," An":1296," Al":5312," Ai":814," Ag":155," Ae":323," Af":1863," Ac":625," Ad":573," Ab":1334," Ba":2550," D ":49," Az":66," Ay":58," Aw":1144," Av":92," Au":228," At":329," As":660," Ar":2746," Aq":45,"hig":403,"hif":269," Be":2802,"hie":154,"hid":60,"hic":96," Bi":359,"hia":2000," Bh":283,"hip":49,"hio":970,"hin":783,"him":84," Bl":695," Bo":944,"hil":345," Br":3059,"hiw":167," Bu":657,"his":167,"hit":155," Bw":425,"hir":643," By":399," E ":42," Ca":7220," Ce":3495," Ci":275," Ch":2756," Cn":42," Cl":933," Cr":1937," Co":2656," Cu":270," Cy":7369," Cw":584," Da":2496," Di":1823," Dh":64," De":2156," Dd":954," Dr":592," Do":753," Dy":1018," Du":803," Dw":249,"hn ":464," Ea":167," Eb":590," Ec":68," Ed":412,"hla":1210," El":634,"hle":156," Ei":1604,"hli":122," Eg":302," Ef":1230," Et":355,"hll":37," Es":277," Er":1072,"hlo":125," Ep":63," En":1110," Em":230,"hlu":57,"hlw":214," Ew":723," Ex":43,"hly":120," Eu":257," Ev":183,"ho ":75," Fe":2231,"hma":89," Fa":1154,"go ":385,"glw":795,"glu":132,"gly":189,"glo":190,"gle":4369,"gli":289," Wy":416," Wr":303,"gla":882," Wo":325," Wl":140," Wi":1099," Wh":149," We":1708," Wa":840," Y ":1307,"й ":51,"gog":3238,"gof":323,"goe":182,"god":705,"goc":155,"gob":234," Ze":78,"gno":66," Zi":43,"gni":87," Za":73," Ys":1224," Yr":658,"gne":142," Yu":65,"gna":64," Yo":129," Yn":2049," Ym":722," Ya":90," Ye":72,"gs ":64,"gr ":1217,"gom":145,"gol":3429,"gon":261,"gop":2404,"gos":1215,"gor":3586,"got":51,"gow":97,"gu ":507," a ":25270,"gro":1041,"gry":291,"grw":824,"gru":108,"gra":788,"gri":1359,"gre":514," Tŷ":84," Gŵ":111," Oy":53," Ow":296," Ou":45," Os":133,"gto":136," Ot":47," Or":342," Op":59," Po":1917," Pl":523," Pi":467," Ph":390,"gul":65," Pe":3404,"gua":83,"gue":160," Pa":1581,"gst":47," Ny":139," Nu":77," No":968," Ol":406," On":113," Oh":42," Og":269," Od":68," Oc":104," Of":85," Oe":2267," Ob":54," Oa":50,"gy ":41," Ra":627," T ":37,"grë":37," Mô":741," Qu":254,"gwm":270,"gwl":1426,"gwo":40,"gwn":213,"gwi":169," Ro":2598," Re":502,"gwe":1487," Ri":459,"gwa":1284," Rh":3078,"gwb":37," Py":119," S ":95,"gur":278," Pr":1731,"gus":127," Pu":304,"gun":41," Pw":190," Sy":523," Sw":1250," Su":392," St":1487," Ta":2095," V ":40,"gyb":143,"gyd":2260," Th":1521," Ti":598,"gyf":2828,"gyh":327," Te":1075," Tr":1823," Ts":358,"gyl":394,"gym":698,"gyn":1911," Pê":44," To":755," Rw":509,"gwr":1885," Ry":288," Ru":535," Sb":343,"gwy":2799," Sa":6139," Sg":470," Sh":618," Si":2880," Sc":411," Se":1836," So":559," Sp":260," Sr":86," Sk":82," Sl":145," Sm":109," Uw":129," Va":334," Ve":204,"grŵ":298," Vi":332,"gwâ":55," Vo":86," Tu":591,"gyw":46," Tw":270," Ty":472,"gyt":130,"gys":648,"gyr":399," Uc":2348," Ul":77," Un":2470," Ur":136," Ut":49,"iai":1947,"iam":862,"ial":272,"ian":2596,"iap":95,"ias":109,"iar":462,"iau":4682,"iat":304,"iaw":277," io":61," in":191," il":42,"ic ":408,"iac":97," is":434,"iad":7064,"iae":3103,"iaf":180,"iag":47,"ibl":135,"ibi":164,"ibo":40," m ":291,"ibr":68," ki":48," gâ":47," ke":53,"id ":3479,"iba":122,"ibh":49,"ibe":421," gy":7787," gw":7430," ha":2143," he":3350," gi":324," gl":929," gr":2189," go":7824,"ia ":4053," gu":50," hy":3013," ia":812," id":384," if":86," ie":418," hi":741,"ib ":99," dé":347," ho":683," dî":55," hu":549," hw":571,"iet":350,"ieu":94," nh":758," ni":1109," ng":3326,"iel":338," ne":8387,"ien":309," na":2311,"ier":266,"ies":434,"ied":632,"ieg":134," my":3937,"ief":37,"iei":784," mw":1538," mu":301,"ig ":8077," mo":510,"iec":167,"ifw":100," ol":455," on":1046," og":435," oh":320,"ify":395,"ifo":628," oc":254," od":327," oe":5934," of":830,"ibî":56,"ifr":76,"iff":1472,"ife":1414,"ifl":67," nw":267," ny":117,"ifi":808," no":795,"ifd":919,"ifa":410," le":1343,"icr":127,"ics":52,"ict":213,"icu":74," li":214," n ":12651,"ico":318,"ick":291,"icl":187," la":1153,"ici":221,"ich":1469,"ice":249," gê":83,"ie ":736," km":670,"ica":2272,"iby":308,"idy":707," me":5050," iâ":40,"idw":100," mh":245," mi":1482," ml":96," dŵ":166," o ":21338,"idr":79,"ido":1466," ma":5425,"idl":131," lu":101,"idi":544," lw":161,"idh":303,"idg":97,"idf":47," ly":443,"ide":336,"idd":6222," ll":7015,"ida":808," lo":202,"if ":3195," ae":587," af":757," ag":1206," ab":262," ac":6621," ad":2600," am":4335," an":2101," ap":337," ai":1226," al":881," au":347," aw":854," ar":16211," at":1462," as":456," d ":109," ba":1485,"il ":1444," bi":247," be":2463," bo":2899," bl":1371," by":2374," bw":689," bu":556," br":2722," ca":4243," e ":202,"im ":424,"ika":52,"ige":189,"iga":404,"ii ":78,"igl":50,"igh":415,"igi":940,"igu":58,"igr":271,"igo":654,"ign":71,"igy":145,"igw":328,"iha":170,"ik ":64," c ":146,"imo":119," er":1863," et":513,"iml":45," es":187," en":8086," em":89," ep":59,"imp":49," ei":5329,"ime":308," el":685," ef":594," eh":46,"imi":96," eg":690,"ip ":126," fe":5001,"inc":980," h ":491,"ind":417,"ina":4133," fa":2334,"inb":394," eu":1065," ew":44,"inn":805," fu":348,"inm":467,"ino":1100," fr":934," fo":1170,"int":968,"ins":292,"inf":90," fl":1325,"ine":1198,"ing":1306," fi":507,"ini":3171,"inl":130," ff":4585,"ink":95," ge":5478," ga":9934,"ioc":84,"eëd":39,"iod":501,"ioe":189,"inu":82," i ":16205,"inw":155,"iny":618," fy":1075," fw":1451," cl":534,"iko":49," cn":41," cm":41," co":4045," cr":1706,"iki":37," ce":1490,"ike":47," ch":4604," ci":218,"ila":556," da":4373,"in ":13332," cu":135," cw":407," cy":8555," do":719,"ilo":375,"ill":4682," dr":3411,"ilm":1379,"ilg":87," de":5281," dd":7964,"ili":1564,"ild":117," di":3478,"ilf":180,"ile":398,"ima":234,"imb":55,"ч ":45," ec":224," ed":232,"io ":4492," ea":139,"ilw":317," dw":1202,"ily":579," du":387,"ils":48,"ilt":144,"ilu":39," dy":2712,"ль":41,"hs ":119,"hr ":234,"how":182,"hoy":58,"hol":2341,"hom":352,"hon":1206,"hog":250,"ко":46,"hoi":337,"hos":458,"hot":44,"hou":76,"hoo":77,"hop":78,"hor":447," yn":41808," ym":5719,"hob":365," yw":15955,"hof":49," yr":12153,"hoe":1138,"hod":477,"hoc":66," ys":1828,"hni":112,"hno":169," yc":300," yd":4387,"hna":273," tî":44,"hne":101," wy":632,"ич":41,"hmo":38,"dép":339,"hui":111,"hug":114,"ро":38,"huf":332,"hud":119,"hua":173,"hub":85,"hw ":113,"hto":62,"ов":62,"hu ":783,"hru":48,"hry":410,"dîm":54," tŷ":48,"hro":487,"hre":1207,"hri":715,"ht ":286,"на":37,"hra":1003," ry":627," rw":43," u ":582," sa":1147," sb":90,"hyf":1088," se":3484,"hyh":50," sc":46,"hyg":114,"hyb":55," si":1946,"hyd":3013," sh":72," sg":392,"hyc":428,"hyn":4010," sm":89,"hym":1892," sp":64,"hyl":212," so":173,"hwr":536,"ви":49," mô":277," t ":111,"hwy":1500," ra":1107," re":611," ri":136," rh":8680," ro":1221," pw":499," pu":181," pr":1508," s ":280," px":78," lô":57,"hy ":169," py":162,"hwa":1012,"hwc":43,"hwe":1597,"hwi":274,"hwn":2898,"hwm":129," os":109,"hum":201,"hun":510,"hus":102,"hut":107," op":73," or":2059,"hur":322,"ан":37," r ":22194," gŵ":50," pe":3258," pa":944," pl":527," po":1095," pi":236," ph":1652," wa":778," we":3663," rô":102," y ":23704," wr":987," wo":70," wi":144," wn":253," sê":49," wl":1199," va":38," ve":41," uw":917," vo":48," vi":53," ré":57," uc":1154,"ер":44," w ":359," ty":390," tw":61," tu":2121," ur":1014," un":3364," ug":115," ta":1382,"hyt":162,"hys":337," sw":506,"hyr":987," sy":9830," st":735,"hyw":645," su":140,"ев":40," tr":2664," pê":155," to":326," th":2240," ti":314," te":1558,"ffu":1091,"fft":332,"ffw":181,"ffr":1792,"fi ":462,"ffy":559,"ffe":1545,"ffa":542,"ffl":109,"ffo":2073,"ffi":2487,"fet":126,"fes":342,"fer":2706,"fey":110,"few":292,"fec":310,"fed":1584,"fen":1795,"fel":4370,"fei":2068,"feg":51,"fia":821,"fga":38,"fgh":43,"fbw":52,"fba":41,"faw":347,"fau":218,"fas":178,"fat":439,"far":1398,"fam":349,"fan":3403,"fal":398,"fai":692,"fag":106,"fae":493,"fad":65,"fac":250,"fab":257,"ff ":917,"fdd":922,"fe ":363,"fa ":921,"eyr":966,"eys":88,"eyd":96,"eye":40,"exa":133,"ez ":93,"ewy":1561,"ews":78,"ewr":200,"exi":125,"eta":251,"ete":326,"eti":221,"eth":10791,"etn":119,"esp":56,"esn":3171,"eso":767,"est":1731,"esu":390,"ess":369,"esy":588,"esw":148,"ev ":37,"eua":272,"euc":47,"eub":64,"eue":51,"eud":983,"eug":110,"eul":766,"euo":648,"eun":167,"eto":117,"etr":1100,"ets":101,"ett":277,"etu":42,"etw":145,"ety":70,"ew ":378,"eve":268,"eva":81,"evi":156,"euw":78,"eut":118,"eur":164,"eus":156,"ex ":115,"euy":40,"ewi":3673,"ewe":71,"ewo":68,"ewn":2724,"ey ":759,"ewa":76,"epe":37,"epi":1038,"eph":182,"er ":10194,"epa":97,"eor":285,"eol":3405,"eop":55,"eon":268,"es ":7548,"ept":38,"epp":38,"erk":43,"erl":567,"eri":3199,"erg":452,"erh":118,"ere":2066,"erf":1312,"erc":671,"erd":1936,"era":2569,"erb":725,"et ":700,"esl":60,"esm":38,"esf":58,"esg":225,"esh":104,"esi":604,"esb":120,"esc":62,"esd":86,"ese":321,"eu ":7692,"esa":201,"ery":875,"erv":60,"eru":112,"erw":570,"err":1317,"ert":3212,"ers":1524,"ern":900,"erm":1779,"erp":114,"ero":902,"en ":8133,"elb":58,"ela":658,"eld":2120,"elf":432,"ele":1904,"eli":827,"elg":144,"elm":49,"ell":3094,"elo":1131,"elp":43,"elu":92,"els":154,"elt":533,"ely":937,"elw":957,"eo ":144,"emb":119,"ema":883,"eme":719,"eml":44,"emo":887,"emi":335,"emp":59,"emy":94,"enf":555,"ene":2395,"enh":570,"eng":830,"enb":102,"ena":618,"end":929,"enc":514,"eno":688,"enm":181,"enn":3626,"enk":49,"enl":180,"eni":1683,"enw":7750,"enu":97,"ens":549,"ent":4104,"enr":449,"enz":60,"eny":811,"eog":53,"eoe":80,"eod":76,"egl":298,"ego":998,"egn":70,"ege":153,"egf":89,"egi":546,"eha":71,"egr":1255,"egu":49,"egw":168,"egy":248,"ehe":930,"ek ":72,"eib":184,"eic":354,"eia":266,"eip":43,"eis":1705,"eir":7366,"eim":111,"eil":1530,"eio":37,"ein":3860,"eid":2056,"eig":925,"eif":224,"el ":7486,"eit":4122,"em ":460,"gl ":325,"git":112,"gis":343,"gir":83,"gil":269,"gip":42,"gin":207,"gio":1071,"gid":96,"gie":91,"gig":167,"gia":697,"ghy":1825,"ghw":91,"ghr":368,"ght":218,"gho":144,"ghe":333,"gha":1381,"gga":62,"gfy":612,"gi ":259,"gfa":38,"gfe":149,"gen":960,"get":83,"ger":2409,"ges":210,"gh ":361,"ged":221,"gei":2736,"geg":490,"gef":158,"gem":288,"gel":883,"gdd":63,"ge ":527,"gbi":171,"gac":62,"gad":298,"gae":2618,"gaf":171,"gai":1331,"gas":141,"gar":1053,"gau":316,"gat":114,"gaw":58,"gam":111,"gal":587,"gan":8980,"gap":38,"ga ":202,"fyw":141,"fys":392,"fwy":1473,"fwr":292,"fyr":1571,"fyl":222,"fyn":2565,"fym":71,"fyg":53,"fyd":4950,"fyf":43,"fur":1066,"fus":43,"fwn":147,"fta":48,"fud":244,"ful":86,"fun":259,"fug":165,"ft ":323,"fra":2027,"frg":79,"frd":59,"fre":1841,"fri":1485,"fu ":398,"fro":1336,"fru":72,"frw":347,"fry":995,"for":3487,"fos":53,"fot":51,"fon":2342,"fol":717,"bî ":57,"fr ":375,"öyn":126,"fna":74,"fne":58,"fnd":51,"fnf":190,"fnu":70,"fni":126,"fno":916,"fod":2910,"fog":67,"foe":238,"fny":1148,"fle":446,"fn ":482,"fla":336,"fli":205,"flu":96,"flo":137,"fo ":353,"fly":274,"flw":1207,"fid":83,"fic":52,"fie":361,"fih":95,"fig":170,"fil":1986,"fin":1391,"fio":735,"fir":153,"fis":223,"fit":156,"fiw":182,"fl ":121,"ffî":106,"da ":2613,"dd ":49936,"de ":2554,"dbw":38,"dac":62,"dad":291,"dal":2984,"dai":3023,"dag":237,"dae":1159,"daf":139,"dat":359,"das":169,"dar":1300,"dan":1183,"dam":229,"day":43,"daw":457,"dau":3395,"dda":2729,"dde":2964,"ddf":808,"ddg":83,"ddh":205,"ddi":10903,"ddl":174,"ddo":8493,"ddr":255,"ddu":2078,"ddw":3166,"ddy":2549,"df ":92,"cul":125,"cud":84,"ctu":101,"ctr":127,"cto":775,"cti":292,"cte":79,"cta":132," Îl":39,"cwl":44,"cwm":218,"cwr":96,"cwe":39,"cy ":84,"cus":119,"cym":1147,"cyl":309,"cyh":309,"cyf":2990,"cyt":158,"cys":475,"cyr":140,"cyn":2866,"cyd":196,"cyc":79,"cwy":42,"cyw":55,"cks":58,"cki":49,"ckl":44,"cla":175,"cle":277,"clu":110,"clw":122,"cli":85,"clo":200,"cly":91,"co ":406,"cod":670,"coe":175,"cof":150,"cob":38,"coc":219,"con":391,"col":301,"com":217,"cor":495,"cos":91,"cop":2154,"cot":157,"cou":73,"cs ":78,"ct ":59,"cre":794,"cra":272,"cri":112,"cru":57,"cro":299,"crw":405,"cry":237,"csa":239,"cso":73,"csi":110,"cci":42,"cca":45,"cea":95,"ch ":6978,"cer":588,"ces":90,"cen":351,"cep":40,"cem":115,"cel":207,"ceg":95,"cef":174,"cei":440,"ced":138,"ci ":202," â ":1666,"chb":78,"cha":3502,"chd":1364,"chw":2809,"chu":496,"chy":2533,"cia":409,"ck ":502,"cie":97,"chg":225,"che":3420,"chf":102,"chl":1181,"chi":932,"cho":903,"chm":87,"chn":386,"chs":64,"cht":200,"chr":1036,"ciw":72,"cil":222,"cis":143,"cit":38,"cin":98,"cio":66,"cip":70,"cm ":41,"cke":114,"ed ":6517,"eba":152,"ebe":278,"ôn ":543,"ebi":94,"ebl":52,"ebo":143,"ebr":165,"ebu":150,"eby":658,"ec ":78,"eac":183,"eag":286,"eaf":76,"eae":48,"ead":313,"eak":38,"eai":225,"ean":995,"eal":663,"eam":55,"ear":1078,"eas":85,"eat":353,"eau":164,"ôl ":1565,"eb ":1399,"ea ":342,"efi":914,"efn":2061,"efo":588,"ôr ":759,"efa":1730,"efe":318,"eff":454,"efy":3744,"ei ":4933,"ega":438,"efr":578,"eft":52,"een":324,"eel":41,"eed":37,"ees":61,"eer":37,"eet":97,"edl":1308,"edn":55,"edi":6449,"edd":25528,"ede":614,"ône":45,"edf":354,"eda":429,"eg ":10783,"edw":785,"edy":568,"edu":747,"edo":543,"edr":525,"eck":79,"ech":1424,"eci":144,"ece":39,"eca":88,"ee ":253,"ef ":6146,"ect":300,"ecs":273,"ecr":59,"eco":330,"dyg":246,"dyf":430,"dyl":322,"dym":701,"dyn":2076,"dys":437,"dyr":98,"dyd":3079,"dyc":204,"dwy":4886,"dwi":169,"dwe":581,"dwc":74,"dwr":1205,"dwl":78,"dwa":882,"dy ":4499,"duw":218,"dur":2144,"dus":635,"dyw":304,"dor":959,"dop":990,"don":2366,"dom":277,"dol":5215,"dow":54,"dos":197,"dr ":1131,"ds ":246,"dna":932,"dne":151," ôl":1443,"dno":132,"doc":100,"dod":1351,"doe":1307,"dog":979,"dso":55,"dw ":187,"dun":58,"duo":40,"dul":211,"dug":82,"dud":167,"dri":671,"dra":2006,"dre":2155,"drw":337,"dry":370,"du ":2339,"dro":1676,"dru":89,"dsh":71,"dha":318,"dge":156,"dgr":60,"dic":115,"did":295,"dia":3817,"dib":66,"dhe":140,"der":2766,"des":358,"det":47,"deu":912,"dew":315,"dey":177,"dh ":410,"dfa":511,"deb":1171,"dea":111,"ded":492,"dec":320,"def":1498,"dee":65,"deh":309,"deg":755,"dei":1883,"del":971,"den":795,"dem":336,"dep":41,"deo":150,"dfu":76,"dfr":146,"dfw":52,"di ":5212,"dga":45,"dfe":259,"dff":43,"dfo":480,"dle":505,"dla":968,"ddŵ":75,"do ":972,"dly":168,"dlw":464,"dlu":272,"dlo":314,"dli":219,"dl ":277,"diu":39,"diw":2623,"dim":202,"din":4906,"dio":2802,"dip":63,"dir":4606,"dis":563,"dit":87,"die":231,"dif":421,"dig":3101,"dil":392,"rgy":156,"rgu":68,"rgw":86,"rhe":939,"rha":3805,"rhi":381,"rhu":44,"rho":350,"rfy":869,"rga":737,"ri ":1789,"rgl":177,"rgi":149,"rgh":39,"rge":464,"rgr":243,"rgo":165,"ret":318,"res":1898,"rev":41,"reu":562,"rew":191,"rey":132,"rfa":309,"rfb":45,"rfe":703,"rff":1405,"rfi":631,"rfl":49,"rfo":973,"rdu":122,"rds":156,"rdr":70,"rdy":486,"rdw":84,"rg ":415,"reb":115,"rea":724,"ree":306,"ref":6655,"rec":445,"red":2528,"rei":2138,"reg":1012,"rem":807,"ren":2559,"rek":37,"rel":299,"rer":65,"reo":299,"rf ":340,"rda":1352,"rcy":41,"rcu":73,"rct":79,"rdo":260,"rdi":824,"rde":771,"rdd":6359,"re ":1508,"rby":562,"rbw":42,"rbu":39,"rbr":42,"rco":44,"rci":169,"rch":2439,"rce":170,"rca":64,"raw":994,"ray":146,"rd ":1148,"rap":92,"rar":127,"ras":623,"rat":596,"rau":1291,"rav":72,"rbi":70,"rbh":58,"rbo":108,"rba":80,"rbe":489,"rc ":329,"rai":5122,"rah":144,"rag":225,"ran":2411,"ram":999,"ral":1583,"rak":64,"rab":559,"raf":654,"rae":3686,"rad":1700,"rac":707,"rpo":41,"rs ":905,"rpw":58,"rpe":88,"rpa":573,"rr ":283,"rpi":50,"ror":608,"ros":1473,"rot":256,"rom":276,"ron":2092,"roo":125,"rop":851,"roy":52,"rou":509,"rov":96,"row":190,"rob":80,"roa":132,"rod":1050,"roc":437,"roi":253,"rol":2368,"rof":352,"roe":2453,"nëe":50,"rog":406,"rno":1027,"rns":37,"rnw":59,"rny":151,"rna":1374,"rng":123,"rne":1846,"rni":383,"rmo":210,"rms":122,"rmw":43,"ro ":1459,"rma":1380,"rme":127,"née":69,"rmi":264,"rly":277,"rlu":234,"rlo":220,"rll":3618,"rli":209,"rld":49,"rle":285,"rla":139,"rn ":2146,"rks":56,"rke":72,"rka":46,"rm ":554,"riz":47,"riw":213,"rix":39,"rl ":122,"rip":190,"rio":1986,"rir":398,"rit":599,"ris":1179,"riv":51,"riu":72,"rig":1620,"ril":699,"rik":45,"rin":1456,"rim":124,"ria":4427,"rib":415,"ric":2311,"rid":570,"rie":637,"rif":5003,"rhw":2579,"rhy":1627,"rk ":218,"rwg":50,"rwe":1416,"rwi":121,"rwp":99,"rwo":224,"rwn":561,"rwm":69,"rws":144,"rwr":297,"rwy":3684,"ryc":824,"ryd":2082,"ryf":329,"rui":70,"rug":759,"ruf":148,"rud":120,"ruc":119,"run":62,"rum":94,"rus":185,"rut":66,"rva":46,"rvi":51,"rve":108,"rwa":160,"rwb":38,"rwc":56,"ry ":1037,"rsi":244,"rso":354,"rsa":276,"nïa":320,"rsh":49,"rse":307,"rsy":112,"rta":429,"rst":96,"rsw":71,"rtn":57,"rto":132,"rte":612,"rth":6409,"rti":407,"rw ":1096,"rts":105,"rtr":278,"rty":80,"rt ":1113,"rqu":38,"rro":150,"rri":1366,"rre":437,"rra":424,"ru ":3404,"rry":313,"rru":55,"rrw":61,"sab":56,"sac":80,"sad":50,"sae":126,"saf":1227,"sai":432,"sak":39,"sal":122,"sam":242,"sba":268,"sbe":72,"sbi":64,"san":708,"sau":254,"sat":40,"sar":175,"say":42,"saw":457,"sa ":271,"ón ":59,"ryw":1557,"rys":874,"ryt":114,"ryr":190,"ryl":141,"rym":301,"ryn":2332,"ryg":165,"sha":130,"sgu":121,"sgw":280,"sgy":278,"sho":46,"sht":41,"she":119,"shi":361,"si ":461,"sga":258,"sgo":1819,"sgl":367,"sgr":1098,"sgf":73,"sge":250,"sgi":173,"siw":382,"siy":220,"sie":542,"sid":204,"sic":303,"sib":67,"sia":2244,"sk ":65,"sit":100,"sir":1014,"sis":167,"sin":210,"sio":756,"sil":315,"sim":40,"sif":99,"sig":463,"sda":48,"sdi":60,"sbr":83,"sbo":63,"sbu":82,"sby":154,"se ":534,"sca":115,"sci":60,"sch":110,"sco":172,"sex":75,"sey":83,"ser":1507,"ses":213,"set":136,"sfa":40,"sh ":444,"sff":143,"sfo":43,"sdr":221,"sg ":658,"sdy":51,"sei":619,"seg":442,"sef":1510,"see":37,"sed":256,"sec":40,"seb":56,"sep":89,"sen":717,"sem":62,"sel":456,"spo":55,"shū":62,"spe":66,"spi":54,"spa":52,"sou":66,"sol":527,"son":872,"sor":102,"sos":82,"soe":1016,"sod":829,"sog":1454,"sof":55,"soc":64,"su ":192,"sra":159,"st ":1362,"ss ":400,"sli":58,"slo":45,"sky":41,"sla":296,"sle":133,"ski":51,"sko":40,"ska":87,"sna":146,"sni":454,"sne":3482,"smo":123,"so ":95,"sma":85,"sme":38,"swr":60,"swo":44,"swm":55,"swl":42,"swi":39,"swe":46,"swy":714,"syc":54,"syd":2408,"syn":267,"syt":47,"sys":246,"sym":497,"syl":933,"sse":383,"ssa":154,"sso":142,"ssi":225,"ste":2086,"sta":1396,"stn":164,"sto":1089,"sti":810,"stl":111,"stu":468,"stw":328,"str":1798,"sty":866,"sul":51,"sut":42,"sus":58,"sur":508,"sy ":6998,"swa":56,"tai":1174,"tal":1078,"tae":112,"taf":985,"tag":117,"tab":136,"tac":100,"tad":428,"tbl":227,"tba":61,"taw":293,"tau":148,"tat":257,"tas":137,"tar":586,"tan":960,"tam":161,"tch":187,"te ":1064,"ta ":832,"pa ":4575,"pe ":114,"par":781,"pat":75,"pas":659,"pau":82,"pac":45,"pal":108,"pai":175,"pao":47,"pap":107,"pan":983,"phe":446,"pha":149,"phw":418,"phu":46,"phr":313,"pho":428,"phl":179,"phi":110,"pi ":55,"ph ":99,"lân":38,"pea":218,"pec":49,"ped":316,"pen":982,"per":2223,"pet":177,"pes":77,"pei":78,"pel":255,"pla":375,"ple":218,"plo":82,"ply":40,"plw":76,"plu":38,"phy":51,"pia":550,"pid":1048,"pig":55,"pil":126,"pin":161,"pio":130,"pis":45,"pit":95,"por":199,"pop":88,"pot":107,"pos":99,"pon":96,"pol":295,"pob":661,"poe":47,"ps ":77,"hū ":64,"ppi":61,"ppl":52,"ppe":110,"po ":38,"lên":948,"psi":40,"pte":1044,"pti":107,"pto":69,"pra":67,"pt ":38,"pry":124,"pri":908,"pre":259,"pro":331,"pwr":173,"pwt":58,"pwl":73,"pwy":372,"lôn":57,"pur":257,"pus":49,"pum":128,"pul":54,"px ":78,"pyn":54,"pyt":61,"pys":41,"löy":126,"qua":43," ŵy":55,"que":158,"qui":76,"môr":276,"ra ":2000,"ngo":2653,"ngi":310,"ngl":561,"ngu":161,"ngw":591,"ngr":197,"ngt":147,"ngs":104,"ni ":1483,"nfy":113,"nge":1187,"ngh":2527,"nga":1043,"ngd":98,"nho":72,"nhy":526,"nhw":208,"nhr":112,"nha":1076,"ngy":306,"nhi":540,"nhe":398,"neg":4193,"nei":264,"nel":633,"nen":226,"nem":48,"ner":1242,"net":231,"nes":2270,"neu":7331,"ndw":46,"ndy":248,"ng ":9397,"nea":151,"neb":716,"nec":67,"ned":7692,"nef":63,"nfi":141,"nfo":387,"nfr":376,"nfu":53,"ney":242,"new":803,"nh ":42,"nfa":429,"nff":206,"nfe":298,"ncr":37,"nct":92,"nco":109,"nci":267,"ncl":53,"nce":463,"nch":147,"nca":318,"ne ":2471,"nby":393,"nbu":88,"ndu":125,"ndr":932,"nds":144,"ndo":502,"ndl":46,"ndi":866,"nde":1235,"ndd":869,"nda":858,"ncy":93,"ncw":75,"nal":455,"nam":268,"nan":722,"nar":638,"nac":478,"nad":788,"nae":541,"naf":1404,"nag":190,"nah":42,"nai":1969,"nc ":1075,"nab":949,"nbr":49,"nbe":186,"nd ":2894,"nba":661,"nau":1855,"nat":502,"nas":4427,"naw":889,"ïo ":43,"na ":2341,"myt":139,"mys":130,"myr":47,"myn":3805,"myl":119,"mwr":262,"mwy":1376,"mwn":673,"iôn":44,"myd":160,"myf":39,"myg":57,"nyb":39,"nyf":51,"nyd":8463,"nyc":85,"nwy":4103,"nwi":220,"nwl":60,"nwo":499,"nwr":453,"nws":101,"ny ":647,"nwg":196,"nwe":332,"nwa":1807,"nve":125,"nul":284,"num":42,"nus":314,"nty":316,"ntw":74,"nw ":6013,"nto":464,"ntu":146,"ïau":380,"nts":154,"ntr":2835,"nti":615,"nth":322,"ntf":38,"ntl":64,"ïai":46,"nta":1485,"nte":453,"nsw":86,"nsy":58,"nso":522,"nst":197,"nse":102,"nsh":86,"nsg":81,"nsi":297,"nsl":63,"nsk":44,"nsc":44,"nsa":259,"nu ":1112,"nry":402,"nrw":38,"nro":38,"nri":910,"nrh":586,"nre":48,"nra":250,"nt ":5305,"ns ":965,"noc":164,"nod":2808,"nob":37,"nog":705,"noe":174,"nof":371,"nol":6920,"noi":38,"nom":300,"non":374,"nos":284,"nor":454,"nov":50,"nou":48,"noy":55,"nne":1057,"nna":2821,"nno":1554,"nnh":61,"nni":1551,"nnu":754,"nnw":1776,"nns":44,"nny":700,"nme":498,"nma":211,"nli":40,"nll":561,"nn ":872,"nla":116,"nle":42,"nly":168,"no ":1182,"ndŵ":83,"nki":55,"nka":58,"nja":59,"nig":2276,"nif":815,"nie":187,"nid":544,"nic":341,"nib":233,"nia":3864,"nk ":137,"niw":268,"niu":56,"niv":47,"nis":599,"nit":132,"nir":305,"nio":1796,"nim":150,"nin":984,"nil":468,"ogr":173,"ogw":78,"ogi":448,"ogl":3819,"ogo":455,"oga":1857,"oge":298,"ogf":54,"oi ":372,"oho":221,"ohn":495,"oha":89,"ohe":171,"ogy":317,"ois":279,"oir":340,"oit":58,"oin":128,"oic":86,"oid":128,"ok ":75,"ol ":19814,"oce":82,"och":2669,"oci":126,"ock":228,"ocl":42,"oco":167,"ocr":146,"ocs":68,"oby":47,"oe ":182,"oca":256,"ode":503,"odf":76,"odl":80,"odi":1787,"odo":830,"odr":901,"oct":65,"of ":754,"odd":3893,"oda":1191,"oel":498,"oeg":1830,"oer":92,"oes":1111,"oet":190,"oen":118,"ody":243,"odu":44,"odw":617,"oed":13126,"og ":4103,"oea":48,"ofn":179,"ofi":339,"ofr":130,"ofo":135,"off":461,"ofe":578,"ofa":165,"oa ":66,"ob ":762,"îl ":44,"oc ":243,"îm ":136,"oan":72,"د ":50,"oad":143,"în ":132,"oba":133,"od ":6442,"oat":50,"obo":63,"obr":327,"obl":2204,"obh":60,"obi":128,"obe":364,"nyn":289,"nym":173,"nyr":93,"nyt":58,"nys":1816,"nyw":409,"ة ":125,"oyd":162,"oya":64,"owy":855,"owr":86,"ows":37,"owl":62,"own":249,"oyw":120,"oty":64,"oud":41,"oub":57,"oua":56,"ow ":296,"oti":143,"oth":272,"ote":190,"ott":255,"ots":65,"oto":157,"osy":68,"ost":354,"osw":61,"ota":156,"otb":49,"ov ":55,"osi":398,"osh":69,"ose":562,"osg":209,"osf":80,"oss":202,"osl":48,"oso":1488,"osn":40,"oy ":77,"owa":86,"owe":136,"ovi":136,"ouv":47,"ouz":37,"ox ":45,"ova":45,"ove":172,"oug":154,"oui":121,"oul":126,"oun":623,"ous":235,"our":437,"out":170,"opo":179,"opi":77,"opl":66,"ope":333,"oph":125,"opa":4605,"os ":1942,"opu":42,"opr":39,"opt":1017,"oon":48,"ool":124,"oom":107,"ook":101,"ood":178,"or ":3226,"oot":121,"oor":76,"ork":80,"orl":3097,"orm":633,"orn":345,"oro":694,"orp":53,"orr":343,"orc":174,"ord":2956,"ore":929,"orf":1329,"org":734,"ori":1436,"ou ":122,"osa":670,"osc":72,"osb":238,"ort":1200,"onï":278,"ors":432,"oru":96,"orw":1133,"ory":220,"ot ":344,"m² ":86,"orb":51,"ora":612,"olb":704,"ola":1373,"old":430,"olc":45,"on ":11898,"oli":4195,"oll":572,"olk":46,"olf":383,"ole":1242,"olh":134,"olg":141,"olt":40,"olm":61,"olo":538,"oly":709,"olu":128,"olw":315,"oka":38,"om ":318,"oke":60,"ona":1663,"ond":1690,"onc":97,"onf":152,"one":971,"ong":675,"oni":1332,"onn":630,"ono":1646,"ons":312,"ont":1089,"onw":477,"ony":591,"oma":643,"oo ":37,"ome":1050,"omb":125,"omi":554,"omm":137,"omp":83,"omo":222,"omy":55,"op ":681,"la ":957,"le ":2080,"lbw":166,"lby":104,"lca":76,"lch":1070,"lco":59,"lf ":105,"lde":318,"lda":44,"ldi":2042,"ldw":120,"lds":45,"ldr":142,"lab":92,"lac":485,"lad":4285,"laf":737,"lae":2342,"lah":70,"lag":63,"lai":3136,"lal":87,"lan":5318,"lam":265,"lap":59,"lar":293,"lat":184,"las":992,"law":1750,"lau":895,"lav":69,"lay":112,"lba":3961,"ld ":464,"lbe":138,"lbo":179,"ky ":83,"llê":46,"lpe":48,"lph":58,"ls ":154,"lol":56,"lon":635,"lom":296,"lop":83,"loo":45,"lor":335,"lod":1459,"loc":229,"lof":189,"loe":1682,"log":1629,"loi":55,"lpa":73,"los":162,"lot":96,"lou":84,"low":187,"loy":165,"lob":68,"lmo":40,"lmi":265,"lme":52,"lma":729,"lti":1579,"lto":98,"ltw":61,"ltu":263,"lw ":187,"lud":183,"luc":42,"lub":52,"lue":48,"lsh":144,"lso":43,"lst":89,"lta":499,"lte":173,"lu ":1258,"lse":40,"lsa":39,"lt ":544,"lhe":114,"lha":157,"lgy":51,"lgw":38,"lgo":53,"lge":216,"li ":2035,"lga":220,"lfy":168,"lfr":121,"hân":48,"lfo":118,"lff":365,"lfe":288,"lfa":558,"ley":287,"lew":3221,"lex":112,"leu":338,"lev":58,"les":787,"let":455,"ler":483,"leo":2599,"lem":257,"len":2864,"lei":2402,"leg":1759,"lef":740,"lee":51,"led":5984,"lec":496,"leb":37,"lea":202,"lg ":94,"lls":93,"llu":1630,"llt":2306,"llw":877,"lly":1536,"lo ":540,"lla":2622,"lle":6817,"llf":78,"llg":60,"llh":61,"lli":2844,"llo":886,"lke":43,"lm ":1156,"ll ":5712,"lit":378,"lis":440,"lir":1910,"lip":176,"lio":1087,"lin":1276,"lim":71,"liz":69,"liw":673,"liv":56,"liu":80,"lic":203,"lid":245,"lia":3198,"lk ":57,"lil":79,"lig":359,"lie":476,"lif":562,"dŵr":302,"ma ":1637,"mb ":111,"mac":103,"mab":110,"mai":606,"mad":228,"mae":4308,"mag":118,"map":57,"mar":872,"mas":453,"mal":309,"mam":99,"man":1111,"maw":438,"mau":1430,"mat":779,"mba":96,"mbl":58,"mbi":109,"mbe":242,"mbr":269,"mbo":281,"me ":239,"mca":193,"mbu":37,"mch":98,"mda":65,"mde":487,"mdd":657,"mdr":53,"med":792,"meg":711,"mea":488,"met":1049,"mew":2214,"mes":747,"mer":2496,"mel":276,"men":1156,"mei":882,"mff":63,"mey":41,"lva":80,"lve":64,"luo":1302,"lun":1166,"lum":206,"lus":297,"lur":87,"ly ":477,"lwa":59,"lwb":141,"lwc":190,"lwe":381,"lwg":148,"lwi":478,"lwr":546,"lwn":220,"lwm":69,"lwy":4971,"lws":76,"lyb":68,"lyc":94,"lyd":945,"lyf":715,"lyg":1379,"hôn":42,"lyw":939,"lym":387,"lyn":3191,"lys":623,"lyt":263,"lyr":52,"mpi":350,"mph":82,"mpe":82,"mpl":56,"mpw":155,"mps":75,"mpt":51,"ms ":365,"ëwy":48,"moc":146,"moe":79,"mod":296,"mon":1356,"mol":107,"mor":527,"mos":228,"mot":129,"mou":82,"mpa":381,"mre":767,"mro":136,"mru":3069,"mry":585,"msa":38,"mu ":71,"mse":222,"mra":1424,"mth":126,"mud":331,"mst":38,"mwe":56,"mwd":160,"mwa":49,"my ":104,"mur":150,"mus":72,"mun":2037,"mho":82,"mhl":147,"mhw":50,"mha":708,"mhe":231,"mgy":321,"mgu":85,"mi ":264,"mge":117,"ml ":374,"min":367,"mio":78,"mil":1349,"mir":63,"mis":321,"miw":182,"mit":116,"mic":53,"mia":447,"mig":178,"mie":110,"mo ":102,"mlw":137,"mly":764,"mlu":98,"mle":90,"mla":506,"mni":274,"mne":46,"mmy":53,"mp ":159,"mmu":54,"mmo":59,"mma":119,"mme":106,"wâr":55,"rŵp":443,"zer":50,"ze ":49,"zab":52,"zan":41,"zar":39,"zon":65,"ان":54,"ال":222,"ÂR ":38,"yw ":17417,"yty":72,"ytu":126,"yts":74,"ytr":121,"yti":312,"yth":1577,"yta":92,"ysw":48,"ysy":639,"yst":2007,"ysu":113,"yso":937,"ysl":39,"ysg":2082,"ysf":43,"ysi":788,"ysb":191,"yse":204,"ysa":193,"yry":38,"yri":760,"yro":57,"yrn":1071,"yrs":46,"yrr":204,"yrw":139,"yrc":853,"yrd":987,"yra":2367,"yrh":40,"yre":633,"yrf":123,"ys ":5008,"ypr":60,"yr ":15016,"ype":40,"yon":159,"yny":6318,"ynu":364,"ynw":625,"za ":99,"ywe":457,"ywb":47,"ywa":1109,"ywi":571,"ywu":40,"ywo":763,"ywy":914,"yby":49,"ye ":65,"yca":52,"ych":3349,"ybi":137,"ybl":102,"ybo":259,"ybr":110,"ybu":37,"yf ":458,"yda":2864,"yde":440,"ydf":99,"ydd":28126,"ydi":506,"ycl":60,"yg ":747,"ydo":278,"ydr":1154,"ydl":894,"ydn":127,"ydw":276,"ydy":4654,"yfa":1668,"yfl":563,"yfi":251,"yfe":2566,"yff":1314,"yer":49,"ya ":226,"yb ":51,"yad":44,"yaf":1692,"ybe":40,"yd ":10831,"yan":80,"yal":55,"ym ":3225,"yke":46,"yn ":47219,"yla":317,"ylc":792,"yld":121,"yle":201,"ylf":185,"yli":297,"yll":1653,"ylo":75,"ylw":219,"ylv":55,"ylu":111,"ymc":84,"ymb":284,"yma":1393,"yo ":95,"ymh":887,"ymi":79,"ymf":49,"ymg":221,"ymd":943,"yme":789,"ymr":5566,"ymp":236,"ymo":286,"yml":267,"ymy":322,"ymu":2206,"ymt":125,"ymw":638,"yna":930,"ynd":320,"ync":116,"yni":1100,"ynl":244,"yne":1666,"ynf":195,"yng":7196,"ynh":1161,"ynr":509,"ynt":1723,"ynn":2787,"yno":1438,"yfw":48,"yfu":361,"ر ":53,"yfy":1325,"yfo":259,"yfn":833,"yfr":2540,"ygi":282,"ygl":131,"yga":105,"yge":40,"ygb":169,"ygw":75,"ygy":125,"ygo":197,"ygr":629,"ygu":321,"yhy":100,"yho":721,"yie":41,"yl ":434,"tîm":43,"ي ":83,"ن ":99,"xic":123,"xas":52,"xan":82,"wun":41,"wy ":2968,"wsl":55,"wst":1039,"wsa":63,"sïa":42,"wsi":378,"wse":138,"wto":60,"wty":66,"wth":52,"wyt":933,"wys":3904,"wyr":4356,"wyl":1023,"wym":72,"wyn":4153,"wyo":82,"wyd":14813,"wyb":349,"wyi":44,"wyf":1666,"wyg":83,"wya":1824,"wly":42,"wma":78,"wmb":41,"wmn":259,"wmp":187,"wmw":171,"wna":104,"wne":924,"wnd":93,"wnc":60,"wni":118,"wng":2908,"wnn":120,"wno":71,"wns":141,"wnt":347,"wm ":924,"wlc":168,"wla":2176,"wn ":4514,"wll":145,"wli":167,"wlg":67,"wle":1242,"wlf":163,"ws ":798,"wre":539,"wrc":168,"wrd":645,"wri":768,"wrg":40,"wt ":38,"wra":424,"wrt":1573,"wrs":66,"wry":875,"wrn":594,"wrp":460,"wro":897,"wod":593,"wog":540,"wob":230,"woc":119,"wny":106,"wor":104,"wol":250,"woo":72,"won":68,"wpa":41,"wr ":5367,"wpi":70,"wdh":39,"wdi":51,"wf ":39,"wdd":251,"wcl":96,"wch":1526,"we ":503,"wca":54,"wbl":50,"wfe":98,"wes":112,"wer":1889,"wen":943,"wel":1143,"wei":2145,"wef":1122,"weg":153,"wed":6219,"ŵyr":67,"web":67,"wec":102,"wg ":400,"ŵyl":106,"wdw":185,"wdu":669,"wgr":102,"wga":86,"wi ":316,"wir":738,"wis":414,"wit":92,"wl ":621,"wig":187,"wie":106,"wid":262,"wic":128,"wio":346,"win":3305,"wil":311,"wia":359,"ŵr ":390,"wb ":178,"wa ":108,"wan":225,"wal":228,"way":128,"wau":1633,"war":1946,"was":655,"wba":42,"wd ":517,"wbe":49,"wai":2037,"wah":560,"wae":306,"wad":321,"ŵl ":39,"ŵn ":84,"ŵp ":436,"rôl":105,"via":62,"vil":101,"vin":110,"vic":81,"vid":205,"vie":209,"viu":39,"vis":116,"rég":55,"von":93,"rëw":49,"ver":485,"ves":72,"rân":40,"ven":296,"vel":96,"ve ":424,"val":75,"van":302,"var":50,"va ":105,"uwy":107,"uz ":39,"uwc":992,"uwi":160,"ux ":77,"uve":61,"usk":46,"ush":85,"usi":90,"use":207,"usc":42,"usa":109,"ust":299,"uss":159,"usn":82,"uth":300,"uti":56,"ute":122,"uta":55,"uw ":191,"utt":64,"uts":49,"utu":93,"uto":58,"us ":2318,"ut ":194,"ura":273,"urd":1224,"urc":65,"urf":869,"ure":327,"urg":261,"uri":598,"urk":37,"urn":280,"uro":357,"urr":296,"urs":65,"urt":95,"urw":145,"ury":156,"uny":251,"uoi":80,"uog":87,"uoe":102,"uod":142,"uon":143,"uol":447,"uos":1109,"ur ":3905,"upi":63,"upe":58,"ump":113,"uml":95,"umo":125,"uma":120,"umb":206,"ume":97,"uly":76,"unt":210,"uns":43,"unr":208,"unw":94,"uni":1229,"uno":252,"unn":56,"unc":85,"und":1356,"una":363,"ung":102,"une":2052,"unf":97,"up ":41,"um ":501,"ulu":685,"ult":59,"ulo":75,"ull":591,"uli":246,"ulh":41,"ule":57,"ulf":75,"uld":43,"ula":311,"un ":3244,"uid":128,"uil":132,"uim":38,"uin":123,"uir":49,"uis":142,"uic":91,"uit":55,"ul ":450,"ugh":287,"ugi":102,"uge":143,"ugo":87,"ugl":146,"uga":343,"ugu":87,"uha":41,"uda":109,"udd":714,"udf":71,"ude":80,"udi":647,"udn":61,"ubu":38,"uca":70,"ue ":164,"uce":83,"uci":43,"uch":1304,"uck":68,"uer":86,"ues":64,"uff":193,"ufe":472,"ufa":148,"udu":77,"udo":207,"ug ":773,"udw":146,"uee":83,"ued":160,"uen":92,"uel":100,"ub ":91,"ua ":2061,"uat":48,"uar":137,"ual":68,"uan":197,"ubl":79,"ube":47,"ubh":79,"uba":126,"ud ":1095,"lŷr":43,"lŷn":132,"uai":74,"uag":90,"uad":147,"uae":87,"tyw":139,"tyf":106,"tyl":97,"tym":66,"tyn":537,"typ":46,"tyr":569,"tys":55,"ty ":569,"tur":1378,"tus":150,"tun":319,"tum":55,"tua":1979,"tud":365,"tyd":81,"tz ":50,"twy":340,"tws":164,"twr":164,"ts ":407,"tre":3761,"tt ":218,"tra":1795,"tri":1062,"tru":77,"tro":965,"ŷr ":76,"tu ":359,"trw":624,"try":340,"tsa":45,"tse":80,"tsi":56,"tsh":38,"tta":92,"tte":273,"tth":42,"tti":111,"ttl":73,"tto":97,"tts":49,"tty":41,"tma":64,"to ":461,"tmo":40,"tne":85,"tna":134,"tno":155,"ŷn ":163,"pêl":157,"tod":481,"toc":94,"toi":46,"tog":93,"tob":164,"tou":42,"tos":66,"tow":75,"tom":329,"ton":1238,"tol":136,"tor":1235,"top":118,"tr ":1173,"til":111,"tif":143,"tie":133,"tig":337,"tir":1682,"tit":55,"tis":234,"tin":490,"tim":109,"tio":758,"thy":2521,"thu":967,"thw":450,"tia":1118,"tic":242,"tid":55,"tiw":232,"tiu":49,"tiv":155,"tl ":156,"tlo":100,"tll":46,"tla":185,"tle":205,"tem":745,"ten":263,"tep":86,"tei":537,"tel":1208,"tef":162,"teg":330,"tea":52,"teb":257,"tec":118,"ted":382,"tfo":51,"th ":17104,"tey":122,"tev":62,"teu":225,"tes":349,"ter":2833,"tgo":37,"ti ":249,"tga":83,"thn":220,"tho":2822,"thl":355,"thm":78,"thr":1553,"ths":111,"thf":157,"thg":113,"thd":135,"the":2439,"thi":2871,"tha":2130,"AS ":68,"AU ":49,"BC ":162,"両 ":46,"Àit":437,"三 ":61,"tŷ ":48,"之 ":53},"n_words":[3639934,4414620,3548300],"name":"cy","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/da.json b/contrib/languages-data/da.json
new file mode 100644
index 0000000..9f5fa9e
--- /dev/null
+++ b/contrib/languages-data/da.json
@@ -0,0 +1 @@
+{"freq":{"D":31963,"E":16857,"F":19795,"G":13674,"A":25625,"B":25733,"C":16797,"L":17275,"M":22557,"N":16361,"O":9041,"H":24577,"I":12015,"J":9674,"K":25355,"U":7743,"T":19455,"W":7598,"V":12180,"P":17849,"S":49336,"R":17256,"Y":1840,"Z":1413,"f":221888,"g":349504,"d":493190,"e":1436845,"b":166372,"c":57573,"a":587527,"n":746797,"o":488340,"l":487393,"m":293320,"j":42561,"k":286164,"h":119659,"i":613022,"w":11095,"v":176622,"u":183003,"t":621215,"s":567338,"r":850344,"q":1336,"p":139855,"z":8890,"y":86732,"x":5939,"Å":1900,"Ø":2784,"é":2737,"æ":62497,"å":62645,"ä":1464,"ü":1583,"ø":71666,"ö":1579," l":35858," m":66057," n":25041," o":98877," h":49372," i":113380," j":6194," k":49552," d":112412," e":192889," f":104584," g":23827," a":95172," b":68676," c":8562," u":24358," t":58677," w":1082," v":42657," p":52152," s":123480," r":22226," J":9634," K":25242," H":24422," I":11943," N":16289," O":8943," L":17176," M":22363," B":25561," C":16566," A":25498," F":19659," G":13552," D":31783," E":16778," Z":1391," Y":1835," S":49095," R":17196,"а":1047," P":17741," W":7540," V":12125," U":7720," T":19345," å":3941," æ":1756," ø":5351," Å":1898," Ø":2782,"A ":4032,"F ":2501,"Da":6717,"Cl":971,"Co":3707,"Ce":1101,"Ch":3540,"Do":1568,"De":17413,"Di":2020,"Fe":1146,"Fa":2183,"Eu":1618,"Er":1008,"Et":1623,"En":5373,"El":1343,"Ge":1911,"Ga":1871,"I ":3326,"Fr":4782,"Fo":3851,"Fl":1651,"Fi":2355,"C ":1471,"Au":1064,"Ar":2625,"As":2462,"Ba":5317,"Aa":1418,"Am":3646,"An":3500,"Al":3794,"By":1617,"Bu":1524,"Br":3812,"Ca":3808,"Bi":1743,"Be":3864,"Bo":3085,"Bl":1436,"Kl":1826,"Kr":1747,"Ko":8878,"Le":2792,"Li":3203,"La":5071,"Lu":1291,"Lo":2732,"Me":3324,"Mi":3899,"Ma":7557,"Mu":1568,"Mo":3310,"Ni":2023,"Ne":2682,"Na":2422,"No":4840,"Gi":973,"Gr":3579,"Go":1190,"Gu":1554,"Ha":6824,"He":6629,"Hi":1514,"Ho":4306,"Hu":1589,"In":3339,"Is":1195,"Ja":2348,"L ":1217,"Je":1924,"Jo":2770,"Ju":1199,"Ka":3252,"M ":1238,"Ki":2897,"Ke":1081,"Un":2388,"Ty":1556,"Tr":2242,"To":2011,"Th":4228,"Ti":1419,"Te":2146,"Ta":2888,"Sy":2480,"St":10943,"Sv":1640,"Su":1509,"Wo":997,"Wi":1997,"Wa":1860,"We":1192,"Vo":988,"Vi":3450,"Va":1711,"Ve":4334,"Pr":4633,"S ":1850,"Pe":2483,"Pa":3757,"Po":2392,"Pi":1096,"Or":1531,"Kø":2561,"Se":2517,"Sc":1974,"Si":2298,"Sh":1188,"Sl":1960,"Sk":3103,"Sp":1977,"So":7973,"Ru":1537,"Sa":4145,"Re":4467,"Ri":2574,"Ro":3970,"Ra":2483,"b ":7025,"a ":44327,"Yo":1017,"Sø":1957,"bø":1832,"i ":102076,"fy":1735,"gd":2544,"ge":89823,"ga":13601,"gb":1002,"bæ":1473,"fj":1452,"fl":7833,"ff":4196,"fi":15203,"bå":1373,"fh":1152,"fs":1525,"fr":23015,"fu":3863,"ft":15592,"fo":54992,"j ":3361,"gy":1588,"he":22182,"ha":30491,"gn":17858,"gl":6422,"gi":15986,"gh":4618,"gg":16392,"gu":4973,"gt":15761,"gs":16863,"gr":16831,"go":3646,"dt":26827,"du":5081,"dv":6155,"dy":2631,"g ":111365,"ea":8665,"eb":9181,"ec":4319,"ed":75055,"de":218146,"dd":6228,"dg":5250,"df":2420,"di":20969,"dh":1605,"dk":4450,"dj":1187,"dm":1943,"dl":12465,"do":7994,"dn":3058,"ds":29024,"dr":12885,"ew":2085,"ex":1370,"eu":3377,"ev":22509,"ey":2555,"ez":1521,"fa":10884,"h ":6139,"fe":8693,"eh":3909,"eg":21663,"ef":12940,"ee":3853,"el":91687,"ek":16142,"ej":9330,"ei":7265,"ep":6194,"eo":4491,"en":269379,"em":29799,"et":141301,"es":67128,"er":363438,"ca":5613,"e ":256545,"by":15888,"bs":2188,"br":14152,"bu":7411,"bo":14081,"bj":2063,"bl":20954,"bi":8379,"bb":2090,"be":49808,"db":9039,"da":25268,"f ":48816,"cu":1080,"ct":2015,"co":4730,"ck":5340,"ci":7534,"ch":11084,"ce":13957,"c ":1829,"az":1010,"ay":3869,"ba":15017,"d ":83491,"at":56481,"as":18978,"ar":71549,"av":20381,"au":6593,"ak":8835,"al":58276,"ai":4230,"aj":1819,"ap":5563,"am":30979,"an":131829,"ac":6807,"ad":19629,"aa":1902,"ab":11909,"ag":21569,"ah":1853,"ae":3053,"af":53343,"nu":5799,"nt":31384,"ns":60694,"nr":1599,"no":15192,"nn":14628,"jø":1567,"ny":4222,"nv":2608,"oe":2046,"of":9995,"oc":5289,"od":18682,"oa":1343,"ob":5028,"om":68251,"on":52834,"ok":6768,"ol":36629,"oi":1528,"kæ":2283,"og":81449,"oh":1849,"ot":9072,"os":11517,"ov":19468,"ou":6113,"op":19430,"oo":2822,"or":109199,"kø":2883,"r ":303465,"ow":2461,"pe":19212,"pf":1434,"pa":11174,"pl":7694,"po":10483,"ph":1997,"pi":13460,"lå":3180,"læ":7117,"lo":18381,"lm":9161,"ll":54007,"ls":28805,"lr":2728,"lp":1954,"lv":5002,"lu":8894,"lt":13794,"ly":5517,"hø":4198,"o ":13391,"ma":28771,"mb":6952,"mh":1531,"me":79970,"mf":2516,"mk":2192,"ml":2687,"mi":18259,"mn":1374,"mm":27337,"mp":7909,"mo":10889,"mr":3179,"mt":7052,"ms":5339,"mu":15391,"my":1487,"p ":7331,"na":26783,"nb":2591,"nc":5256,"nd":115468,"ne":83213,"nf":2882,"ng":65185,"nh":5629,"ni":34813,"jæ":2156,"nj":1002,"nk":9744,"nl":3415,"nm":3852,"jy":1603,"ju":2882,"js":1575,"jo":3376,"jl":1331,"fæ":2974,"ki":13129,"gå":3551,"ke":62501,"kb":1624,"ka":33369,"m ":58614,"fø":16135,"ky":2485,"ks":11787,"kt":15601,"ku":9611,"kv":2686,"ko":24231,"kr":19731,"kk":11850,"kl":9428,"km":2041,"kn":3763,"li":70659,"hå":1642,"lh":1724,"lk":6427,"hæ":2028,"le":98770,"ld":24706,"lg":4872,"lf":2530,"la":47170,"lb":6128,"gø":1910,"n ":242497,"hr":2287,"hv":9222,"ht":1466,"hu":6980,"dæ":994,"hj":3040,"hi":7688,"hn":1182,"ho":15697,"id":25624,"ic":10962,"ib":6095,"ia":12086,"ig":59928,"if":7649,"ie":27731,"hy":1292,"k ":53472,"dø":3867,"ir":14180,"is":62018,"it":24973,"iu":2399,"iv":18874,"ik":30570,"il":63431,"im":7762,"in":99762,"io":27519,"ip":4261,"je":15216,"jd":2875,"få":1004,"iz":977,"l ":57442,"ja":3097,"tæ":3691,"tå":3749,"z ":1958,"sø":4093,"wi":999,"så":7469,"sæ":4934,"y ":17470,"rø":5891,"wa":2989,"we":1786,"ræ":12201,"rå":5052,"vi":25451,"vs":4402,"vn":10855,"vo":8670,"ve":64168,"vd":1809,"va":25040,"x ":2713,"ui":2354,"uk":3985,"ul":11807,"ue":7203,"uf":1429,"ug":8656,"ur":17879,"us":20761,"ut":8713,"um":12141,"un":39154,"up":6927,"ty":12222,"tz":1053,"tu":11275,"tt":18340,"tv":2718,"ub":5076,"ua":3814,"ud":21845,"uc":3424,"w ":2415,"to":25039,"tn":4704,"tm":1160,"tl":7804,"ts":12139,"tr":31125,"tf":1908,"te":116420,"tj":2329,"pæ":1318,"på":21476,"ti":89903,"th":7738,"v ":18910,"tb":1612,"ta":40351,"su":4879,"sv":6120,"ss":17007,"st":117117,"sy":9417,"sl":10786,"sk":90052,"sn":3225,"sm":6568,"sp":18285,"so":39577,"sr":1826,"sd":1411,"sc":3973,"sf":3067,"se":58416,"sh":5513,"sg":1978,"sj":1417,"si":34046,"u ":4030,"sa":20739,"sb":7435,"rr":10396,"rs":33077,"rt":26331,"ru":25205,"rv":5600,"ry":5622,"rp":2368,"ro":34074,"rn":23969,"rm":13546,"rl":8366,"rk":22061,"næ":3458,"nå":1618,"ri":66660,"rh":5594,"rg":18288,"rf":5902,"re":122058,"rd":25935,"rc":2308,"rb":9120,"ra":54038,"t ":215608,"mø":1338,"qu":972,"mæ":2988,"må":5133,"lø":4835,"s ":87017,"pt":4539,"pu":4408,"pp":6074,"pr":20775,"ps":3450,"zi":1962,"ze":1041,"za":1046,"væ":10057,"yg":7090,"ye":6589,"yd":11050,"ya":1357,"yb":1013,"tø":6010,"yt":3483,"ys":11680,"yr":5643,"yp":2457,"yn":4794,"ym":2136,"yl":4209,"yk":3734,"År":1234,"å ":31271,"Øs":1854,"æv":1666,"æs":6691,"ær":17270,"æt":3678,"æn":8602,"æk":5318,"æl":6220,"æg":5488,"æd":2878,"ån":3755,"ås":1047,"år":11909,"åe":2110,"ål":3106,"åb":1474,"åd":6195,"ør":20663,"øs":5971,"øv":2281,"øj":4875,"øk":983,"øn":5238,"øl":3077,"øm":1629,"øb":5904,"øg":2353,"øe":2140,"ød":12599,"ø ":2608," Ga":1860," Ge":1892," I ":2530," Fo":3830," Fr":4772," Fi":2339," Fl":1645," Ha":6811," He":6623," Go":1179," Gr":3557," Gu":1534," Hu":1582," Ho":4265," Hi":1510," Je":1919," Ja":2336," Is":1181," In":3319," Ka":3227," Ke":1072," Ki":2888," Jo":2762," Ju":1199," La":5040," Le":2776," Li":3188," Kl":1820," Ko":8866," Kr":1743," Ma":7451," Mi":3885," Me":3307," Lo":2724," Lu":1285," Ne":2661," Na":2409," Ni":2020," Mo":3297," Mu":1554," Am":3639," An":3489," Al":3774," Aa":1413," Ba":5300," Au":1061," As":1298," Ar":2597," Be":3849," Bi":1733," Bl":1427," Bo":3066," Br":3787," Bu":1519," By":1610," Ca":3749," Ce":1081," Ch":3522," Co":3667," Da":6687," Di":2014," De":17357," Do":1505," El":1337," Et":1617," Er":1002," En":5365," Eu":1616," Fe":1132," Fa":2149," Sø":1956," Wo":983," Wi":1981," We":1181," Wa":1857," Yo":1015," Kø":2558," Or":1513," Po":2365," Pi":1093," Pe":2468," Pa":3735," No":4834," Ra":2473," Ro":3958," Re":4459," Ri":2570," Pr":4618," Sy":2476," Sv":1637," Su":1503," St":10885," Ta":2881," Th":4217," Ti":1411," Te":2130," Tr":2230," To":1975," Ru":1535," Sa":4126," Sh":1168," Si":2284," Sc":1958," Se":2497," So":7960," Sp":1954," Sk":3100," Sl":1954," Va":1707," Ve":4322," Vi":3432," Vo":984," Ty":1554," Un":2381," ja":1664," få":1001," in":18970," ik":2700," is":1666," ka":10401," fø":12841," ki":4060," ke":5007," jo":1235," fæ":1545," ju":2042," ha":16862," he":5872," gi":1512," gr":8357," gu":1731," dø":1180," hi":2424," hj":2649," ho":5868," hu":1995," hv":8333," ne":2244," na":5975," mu":3617," mo":5717," om":10735," og":60338," of":4764," ny":1418," nu":1920," no":8819," le":3614," li":12024," hå":1306," la":9917," kv":1456," ku":4034," km":1736," kl":2396," kr":4395," ko":12549," me":34653," mi":5228," hø":2581," ma":10653," lu":1103," ly":1210," lo":1691," af":47341," ad":2058," am":5476," an":9937," ap":1120," ak":1320," al":5943," au":1423," ar":5256," at":12715," ba":5812," bi":2763," be":21652," bo":2959," bl":16483," by":7166," bu":1513," br":6861," ca":2244," er":79580," et":23500," en":65390," el":13704," ek":1980," ef":3944," eg":1315," fe":2371," fa":4879," fu":1922," fr":19782," fo":43206," fl":5239," fi":8516," bå":1058," ge":4611," ga":3502," bø":1211," i ":86016," fy":1194," co":1546," ce":1786," ci":1062," da":14483," do":1734," dr":1947," de":85238," di":4041," dy":1311," væ":5438," tæ":998," sø":1940," ru":2426," sa":10706," se":8834," si":12012," sm":1227," sl":3070," sk":10508," sp":8070," so":28866," ra":1828," re":10118," nå":1108," ri":1551," næ":2233," ro":3386," pr":10950," lø":1660," s ":1355," må":3141," ov":4934," op":11111," or":3538," kø":1518," pe":4046," pa":3870," pl":3433," po":4785," lå":2186," pi":1184," læ":1975," så":2369," sæ":2010," va":14642," ve":13203," vo":1548," vi":7015," ræ":1334," ud":14434," ty":5331," tv":1505," un":6550," ta":3468," sy":6433," st":20417," sv":2066," su":1555," tr":6706," to":3836," th":1962," på":21281," ti":28004," te":4720," År":1234," Øs":1852," år":3117," øs":2023,"Årh":1078,"Fil":1000,"Et ":1446,"Eur":1467,"En ":3727,"Fra":1859,"Fre":1891,"For":2210,"Hel":1134,"Her":3429,"Han":2509,"Ind":1212,"Øst":1835,"Hol":1719,"As ":1183,"Bay":1295,"Amt":2376,"And":1120,"Bye":1086,"Car":1184,"Ber":1053,"EF ":1275,"De ":1771,"Det":5562,"Den":6802,"Dan":4528,"Chr":1315,"Cha":1198,"New":1218,"Nor":4041,"Køb":2308,"Par":1507,"Pro":2885,"SA ":2039,"SAs":1153,"Joh":1172,"Kir":1207,"Kon":1132,"Kom":6411,"Lan":2730,"Man":1218,"Mar":2635,"Søn":999,"Syd":1794,"Sve":1310,"Str":999,"Sti":2399,"Sto":1709,"Sta":3406,"Ste":1353,"Tab":1274,"Sog":4806,"Sch":1301,"San":1256,"Reg":1718,"Ros":1109,"åen":1002,"åde":4772,"ål ":1259,"ånd":1467,"åne":2057,"åle":1126,"åre":1034,"ård":2059,"år ":6325,"Ver":993,"Ves":1255,"Uni":1434,"æde":2099,"Tys":1072,"æge":1409,"ægt":2411,"æll":2133,"ækk":2758,"æld":2168,"æns":1603,"æng":3355,"ænd":2342,"ær ":2495,"æsk":1388,"æse":1314,"ært":1291,"ærk":2357,"ære":6027,"ærd":1139,"ætt":1489,"æst":1815,"ævn":987,"The":2590,"bje":1890,"bil":1901,"bin":2468,"blo":1344,"ble":11852,"bli":3619,"bla":2817,"bol":3467,"bog":1211,"bor":5893,"bbe":1641,"be ":1587,"ban":5529,"bal":1468,"bag":1151,"bas":2113,"bar":1770,"bej":2245,"beg":2119,"bef":974,"bed":1902,"ber":10275,"ben":6123,"bel":6286,"bez":990,"bev":1288,"bes":5570,"bet":8125,"ca ":2352,"ce ":2816,"bri":2216,"bro":1760,"bra":1230,"bre":2307,"bru":5207,"bur":1375,"bun":1225,"bum":1745,"by ":6807,"bye":2309,"byg":4928,"am ":2554,"al ":9518,"ain":1470,"aj ":1010,"agt":3576,"anv":1364,"anu":1902,"ann":5158,"anm":3219,"ant":6958,"ans":23197,"ane":4038,"ang":12562,"ani":4609,"ank":4263,"ana":2737,"anc":2097,"and":39458,"amt":2475,"amm":7240,"aml":2016,"amp":2062,"ami":2635,"ame":7259,"ama":1553,"alv":1423,"alt":4977,"als":1981,"alr":2167,"alm":1522,"all":6438,"alg":1439,"ali":5360,"ald":6029,"ale":8208,"ala":1794,"alb":2973,"an ":17044,"akt":3338,"abe":4887,"ad ":3260,"ab ":2197,"aft":1820,"afs":1107,"afh":1105,"afi":1171,"age":8240,"adm":1030,"adi":3138,"ade":6627,"ag ":3450,"ads":1953,"ack":1318,"ach":1746,"ace":1610,"af ":43757,"aye":1830,"at ":17462,"are":4543,"ard":3649,"arb":2537,"ara":2794,"arm":1244,"arl":1801,"ark":7252,"ari":4342,"arv":1066,"arr":1598,"ars":2045,"art":9040,"au ":1067,"asi":1827,"ase":2004,"ask":1161,"ar ":24485,"as ":3170,"avd":1374,"avn":8812,"avi":1508,"ave":5203,"ay ":1009,"av ":1155,"ata":1987,"ast":4061,"ass":3627,"ato":1943,"ate":7241,"ati":16074,"att":4875,"ats":1271,"atu":2072,"jer":5562,"jek":1076,"jem":1846,"jen":1901,"jet":1025,"jan":1060,"je ":1645,"jde":2262,"jor":2291,"fær":971,"fæl":1337,"itu":1780,"itt":1519,"isk":22022,"ism":1473,"iss":2416,"ist":15812,"iv ":2118,"ita":3145,"ite":4554,"iti":5826,"ium":1528,"iva":1011,"ivi":1481,"ive":12562,"ipt":1296,"is ":8187,"ion":21894,"irk":6542,"isi":1236,"ise":3850,"isa":1650,"ire":2379,"it ":2641,"kil":3875,"kib":1205,"kin":2032,"kir":1742,"går":2699,"kis":983,"km ":1307,"ked":1397,"kel":4377,"ken":10401,"kes":1300,"ker":9106,"ket":3222,"ke ":28879,"kra":3109,"kre":5594,"kt ":3134,"kse":1892,"kro":1365,"kri":8221,"kov":1285,"kor":3059,"kon":5299,"kom":8214,"kol":2985,"ks ":2322,"kni":2413,"kke":10909,"klu":1988,"kle":1997,"kla":3032,"kli":1244,"jyl":1350,"jul":1124,"kba":1487,"kat":1697,"kar":1404,"kas":971,"kan":11432,"kal":6243,"kam":1128,"kab":5579,"ka ":2541,"før":6664,"fød":7830,"føl":1523,"ham":1255,"han":5336,"hal":2007,"hav":7061,"har":10554,"he ":3841,"hel":2400,"hed":6692,"her":3250,"hen":2397,"hin":1447,"his":2209,"hje":2112,"gle":3411,"gn ":5722,"gla":1189,"gni":1875,"gne":9082,"gs ":1501,"gsb":1331,"gsk":1233,"gru":5085,"gra":4027,"gt ":9775,"gre":2783,"gst":2115,"gte":3224,"gti":1404,"grø":1000,"gså":4604,"gus":1029,"græ":2818,"ial":2412,"ian":3001,"ic ":1068,"ibo":1076,"ølg":1529,"øn ":1089,"id ":2460,"ibe":2142,"ia ":3660,"øje":1642,"iet":2644,"iel":2523,"ien":9683,"ier":4656,"ig ":12921,"ift":4749,"ør ":3471,"ici":1501,"ich":2160,"ice":1664,"ie ":4513,"ica":1606,"ids":2132,"idt":1926,"idl":3701,"idi":1064,"ide":10399,"idd":1704,"ønd":1642,"øst":4212,"il ":24604,"im ":977,"ika":7541,"ige":19623,"iga":1060,"igh":3457,"igi":1414,"igg":9753,"igt":6078,"igs":1981,"ign":1642,"øre":4495,"ørs":6046,"ørr":1963,"ørt":1508,"ik ":5837,"ørn":1192,"ime":1521,"ind":23824,"ina":3988,"int":3500,"ins":7738,"ine":6155,"ing":36509,"ini":3271,"ink":1131,"iod":1188,"ikl":2029,"ikk":5095,"ike":2959,"ikb":1260,"in ":7601,"ikt":1780,"iks":1269,"ilo":3003,"ill":15072,"ilk":1504,"øve":1134,"ilm":4941,"ilh":993,"ili":3480,"ild":2323,"ile":1492,"io ":1215,"ils":1343,"hol":6737,"hov":3711,"hri":1405,"hvo":4960,"hum":1081,"hun":1070,"hus":3072,"hve":1568,"hvi":2196,"døs":1339,"død":1050,"ffe":2245,"ffi":1018,"fes":1566,"fer":1682,"fen":1036,"fas":1001,"fat":2864,"far":1600,"fam":1418,"fan":1048,"fal":1044,"ezi":1048,"evæ":1177,"eta":3145,"ete":8817,"eti":2099,"esp":2172,"est":19859,"ødt":7615,"ess":3934,"esv":1081,"ev ":11307,"etr":1299,"ets":3278,"ett":4721,"ety":1926,"øen":1132,"ew ":1416,"eve":5776,"eva":1631,"øge":1365,"evi":1173,"øj ":1036,"ey ":1638,"er ":229484,"epa":1085,"eor":1469,"eol":1033,"ød ":1370,"es ":24995,"øbe":3709,"ept":1210,"erk":1552,"erl":2388,"eri":19046,"erg":5102,"erh":1890,"ere":36721,"erf":2743,"erd":3362,"era":4483,"erb":2464,"et ":110723,"esk":4356,"esl":1065,"esi":2775,"øde":1837,"ese":1937,"erv":2764,"eru":2635,"err":4661,"ert":4521,"ers":15033,"ern":16903,"erm":1970,"ero":1448,"egå":1006,"ekr":1629,"eks":4054,"ekt":4817,"en ":174071,"ela":1844,"eld":1116,"ele":6394,"eli":11128,"ell":22146,"elv":2251,"els":21491,"elt":4815,"emb":3551,"ema":1766,"eme":3075,"emm":3388,"emo":1300,"emi":2222,"emt":1052,"emp":1313,"ems":1366,"enf":1295,"ene":9879,"enh":4546,"eng":3572,"enb":1247,"ena":1550,"end":24965,"enc":1233,"enn":5553,"enk":1120,"eni":2823,"ens":17443,"ent":14377,"enr":1262,"egn":6321,"ege":4459,"egi":3950,"eha":1147,"egr":1976,"eho":1258,"eis":2241,"ein":2075,"ejd":2400,"el ":14709,"ejs":1068,"ejl":1164,"eje":2612,"em ":8110,"gis":2194,"giv":4136,"gin":973,"gio":2383,"gie":1747,"ghe":2580,"gge":15510,"gi ":1776,"gen":19301,"geo":1146,"get":6054,"ger":26388,"ges":3499,"gem":1137,"gel":6514,"gde":1133,"ge ":22940,"gad":1042,"gar":1035,"gav":1126,"gam":1188,"gan":5170,"fte":8906,"fun":1949,"ft ":4358,"fra":16811,"fre":2876,"fri":2352,"for":48487,"fol":2294,"fod":2203,"fle":2554,"fla":1188,"flo":1580,"fly":1416,"båd":1117,"fic":1488,"fil":4788,"fik":1836,"fin":3119,"fir":1330,"fis":991,"da ":2439,"dbr":1439,"dbo":3204,"de ":51365,"dby":3214,"dal":1321,"dag":2881,"dat":1966,"dan":12624,"dam":1233,"dda":1566,"dde":3541,"com":1147,"ch ":2261,"cer":3808,"ces":1087,"cen":3123,"cha":1908,"cia":1876,"ck ":2373,"cie":1532,"che":2404,"cke":1007,"ed ":35220,"ebe":1273,"ebo":1021,"ebr":1930,"eal":1397,"eat":1418,"efi":1531,"efo":3934,"eft":4164,"een":1037,"edl":1944,"edi":2262,"ede":18829,"eda":1072,"edt":1636,"eds":6584,"edr":1723,"eci":1115,"ece":1005,"dyr":1288,"dvi":2440,"dve":2150,"don":1291,"dom":2065,"ds ":4746,"dmi":1103,"dni":1787,"dst":6032,"dsp":2071,"dti":2638,"dte":2421,"dtr":1052,"duk":1051,"duc":1998,"dri":1805,"dra":1347,"dt ":18171,"dre":7141,"dro":1133,"dsk":3064,"dsb":2127,"dsa":1334,"dse":2268,"dgi":2184,"dia":1005,"der":53839,"des":7277,"det":27574,"deb":1003,"deh":997,"del":20136,"den":48394,"dem":1809,"di ":1057,"dga":1116,"dle":3278,"dla":1839,"dkr":1756,"dli":6294,"din":2827,"dio":1576,"dis":4268,"dit":1114,"die":2342,"dig":3302,"rhu":1754,"rho":1270,"rga":2811,"ri ":2041,"rgi":1032,"rge":3583,"ret":16711,"res":7617,"rev":3074,"rfa":1970,"rfo":1099,"rds":1442,"rdv":999,"rg ":8008,"reb":1759,"rea":2289,"ref":3055,"red":12454,"rei":2319,"reg":5210,"rem":3072,"ren":13272,"rek":1859,"rel":3506,"rer":8658,"rep":1174,"rda":1335,"rdl":1187,"rdi":2720,"rde":7501,"re ":32700,"rbu":1055,"rbr":1012,"rd ":6821,"ras":1016,"rat":6278,"rbi":1442,"rba":1156,"rbe":2916,"rag":1612,"ran":8553,"ram":2785,"ral":3419,"rak":1777,"raf":2970,"rad":2370,"rs ":4872,"ros":1298,"rot":1028,"rom":2663,"ron":3183,"rop":3118,"rov":4008,"rod":3399,"roc":1755,"rol":2335,"rof":1572,"rog":3109,"rna":2635,"rne":13393,"rni":1274,"ro ":1858,"rma":3375,"rme":5215,"rli":2882,"rla":2011,"rn ":3815,"rks":2467,"rko":1321,"rki":1210,"rke":7276,"rka":1044,"rm ":1940,"næs":1010,"nær":1381,"rip":1436,"rio":1666,"når":1017,"rit":3128,"ris":7929,"riv":2443,"rig":7130,"ril":1328,"rik":9410,"rin":12007,"rim":1705,"ria":2312,"rib":1119,"ric":1788,"rid":1336,"rie":7227,"rif":1151,"rk ":5973,"rug":4144,"rue":1823,"rup":5078,"run":5278,"rum":2612,"ruk":1154,"rus":1691,"rva":1035,"rvi":1176,"rve":2295,"ry ":1399,"rsk":6329,"rsl":985,"rsi":2088,"rso":2553,"rsa":1053,"rse":2094,"rta":1313,"rst":8534,"rte":6848,"rti":3566,"rts":2294,"rt ":7502,"rri":1525,"rre":6580,"sag":1459,"sal":1002,"sam":8006,"sbe":2183,"san":3521,"sat":3398,"ryk":1357,"sho":1591,"sie":1150,"sid":5798,"sk ":33968,"sit":3034,"sis":4442,"sin":4733,"sio":2930,"sik":3367,"sig":4968,"sby":2385,"se ":14360,"sch":1769,"ser":12739,"ses":2382,"set":2238,"sfo":1331,"sep":1174,"sen":14543,"sem":1108,"sel":4270,"sek":1218,"spo":1957,"spr":2788,"spe":2284,"slæ":1631,"spi":8255,"spa":1108,"sol":1008,"som":26634,"son":4747,"sor":1570,"sog":2156,"st ":15643,"sli":981,"slu":1369,"sky":1101,"sla":3115,"sle":2180,"ski":4168,"skl":1367,"sko":3585,"skr":6518,"sku":2545,"ska":8399,"ske":25508,"sni":1817,"sma":1205,"sme":2348,"stæ":1164,"stå":3319,"syd":3714,"stø":3565,"syn":1352,"sys":1619,"sse":5630,"ssa":1137,"sso":1208,"ssi":4603,"ste":30468,"sta":15172,"stn":1107,"sto":7688,"sti":13739,"stj":1076,"stl":3640,"stu":1392,"str":12753,"sty":2236,"sva":1612,"sve":1375,"svi":1847,"tal":8954,"tag":2922,"tab":1032,"tad":2960,"tat":8263,"tar":2918,"tan":5101,"tam":1561,"te ":23992,"ta ":2303,"pa ":1106,"køb":1295,"pe ":3064,"par":4298,"pan":2233,"lå ":2263,"pec":1027,"pen":2557,"per":8319,"pel":1218,"pla":4596,"ple":1353,"læg":3305,"læn":1040,"lær":1638,"pil":7538,"pis":1494,"por":2733,"pop":989,"pos":979,"pol":3456,"ppe":4792,"pst":1130,"pte":1387,"pti":1074,"pri":5464,"pre":2422,"pro":9553,"pun":1123,"præ":2064,"løb":1835,"løs":1373,"mæn":1058,"mær":1139,"mål":1925,"mån":1657,"ra ":16943,"ngi":1084,"ngl":1961,"ngs":7820,"ni ":1743,"nge":21530,"ngd":1418,"nha":2979,"nhe":1266,"nel":5371,"nek":1429,"nen":7128,"nem":2182,"ner":14377,"net":9752,"nes":8157,"ng ":25397,"ned":2644,"nfo":1407,"nce":2989,"ne ":28324,"ndt":10291,"ndr":5292,"nds":9100,"ndo":2080,"ndl":3623,"ndk":1968,"ndi":3782,"nde":48861,"ndb":3975,"nda":1742,"nal":7713,"nan":1050,"nar":1016,"nd ":19511,"nav":5158,"nat":4037,"nas":1015,"na ":3264,"nve":1870,"nus":1183,"nto":1311,"ntr":3209,"nti":2579,"ntl":1237,"nta":2683,"nte":11676,"nsp":982,"nst":8074,"nse":6294,"nsi":1207,"nsk":23035,"nsb":1130,"nt ":5652,"ns ":13305,"nog":1664,"nom":1719,"nor":6469,"nov":1161,"nne":10196,"nni":1944,"nma":3344,"nli":1178,"nla":1269,"nke":3504,"nkt":1750,"jæl":1755,"nie":2401,"nk ":979,"niv":1911,"nis":7071,"nit":1685,"nio":1012,"nin":14093,"nik":1101,"ogs":5045,"ogr":2357,"ogi":3275,"ogl":1041,"ogn":7366,"oge":2091,"ol ":1201,"oci":1344,"ock":2154,"ode":4999,"ods":1493,"of ":2459,"odb":2119,"odu":2868,"og ":59080,"oft":2765,"off":2072,"ofe":1326,"od ":3992,"obe":1539,"nyt":1313,"ote":1650,"ott":1354,"oto":1594,"ost":2113,"ov ":1326,"osk":985,"ose":1440,"ovi":1957,"ovs":2306,"ove":11716,"oun":1213,"our":1852,"opl":974,"ope":1372,"opf":1235,"opa":1354,"os ":2264,"opr":3026,"opt":1105,"ops":1721,"or ":31324,"ork":2427,"orl":1427,"orm":8791,"orn":2079,"ord":14176,"ore":6583,"orf":2453,"org":9163,"orh":1310,"ori":4799,"ort":7693,"ors":6893,"orv":1018,"ot ":1242,"orb":3261,"ora":1624,"old":10067,"on ":17437,"oli":4334,"oll":3644,"olk":2982,"ole":3254,"ols":1497,"olm":1298,"olo":4008,"oka":1272,"om ":32517,"ona":4226,"ond":1810,"one":10149,"ong":2553,"oni":2635,"ono":1620,"ons":4853,"ont":2499,"oma":2195,"ome":4668,"omi":1702,"omh":1267,"omm":13262,"omk":1807,"omp":2560,"omr":2620,"oms":1669,"op ":2057,"la ":1857,"gør":1749,"le ":18274,"lde":9460,"ldt":2484,"lds":2126,"lac":1163,"lad":4353,"lag":4790,"lan":22791,"lam":1129,"lar":1164,"lat":2523,"las":2596,"lav":1728,"lba":1173,"ld ":5587,"lbo":1181,"lbu":1855,"kvi":1085,"kva":1167,"kun":3121,"kul":1926,"kte":2817,"kst":2005,"kso":1301,"kue":1399,"ktr":1143,"kti":4258,"kto":1463,"ktø":1090,"ls ":1795,"lok":1028,"lom":2968,"lod":2232,"log":4064,"los":1196,"lot":1245,"lov":1376,"lmi":1310,"lme":1724,"lti":975,"ltu":1430,"lub":2283,"lsk":4901,"lst":4539,"lta":1027,"lte":2079,"lse":13829,"lre":2112,"lt ":6296,"lge":2455,"li ":1269,"lev":14315,"les":5717,"let":5891,"ler":27840,"lem":6534,"len":6881,"lek":2168,"leg":1270,"led":4289,"lla":4918,"lle":37621,"lli":6167,"lke":3500,"hæn":1282,"lm ":3777,"ll ":2166,"lit":4876,"lis":3436,"lin":9392,"hån":1024,"liv":2785,"lia":1325,"lik":2160,"lil":1691,"lig":34196,"lie":4111,"ma ":1277,"mag":1391,"mar":6718,"mas":1505,"mal":1831,"man":9443,"mat":3311,"mbe":3163,"me ":4862,"med":21647,"meg":1181,"met":7243,"mes":4874,"mer":15160,"mel":6023,"men":16847,"lut":1751,"lys":1458,"høj":2362,"hør":1554,"mpe":2509,"ms ":982,"mod":4612,"mon":1609,"mor":1053,"mt ":4527,"mst":1872,"mrå":2725,"mus":3381,"mul":1155,"mun":9999,"mhe":1090,"min":6763,"mil":3215,"mis":2297,"mid":2156,"mle":1475,"mkr":1669,"mmu":9493,"mme":16141,"vær":6621,"zir":1025,"væg":1206,"ytt":1956,"yst":3197,"ysk":5330,"yre":2022,"yr ":1010,"yde":3637,"yer":2527,"yen":3113,"yd ":1599,"ykk":1287,"yll":1699,"ynd":1352,"ygg":4471,"tør":4175,"tær":1046,"tæn":1204,"tår":2373,"sæt":2008,"så ":5902,"sær":1369,"røn":1765,"rød":1365,"vst":2580,"vir":2222,"vik":2041,"vil":2648,"vin":4397,"vig":1873,"råd":3726,"vid":3491,"vis":5672,"ræs":2950,"ræn":2483,"ræk":2614,"vn ":4721,"vne":3693,"vns":1255,"vok":971,"vor":5368,"ver":19395,"ves":5050,"vet":6585,"vej":1947,"ven":7443,"vem":1052,"vel":2142,"ved":12182,"vde":1584,"ve ":6698,"val":2588,"van":4077,"var":14654,"vat":1120,"usk":1540,"usi":3385,"use":2336,"ust":3006,"uss":1778,"uti":1400,"ute":1658,"us ":6486,"ut ":1499,"ure":2206,"urg":1747,"uri":1026,"urn":1614,"uro":2454,"urt":1085,"ur ":2594,"upp":2843,"umm":1471,"umb":1127,"ume":1120,"uns":1753,"unk":1906,"uni":2675,"und":15503,"una":2326,"ung":3315,"une":7027,"up ":2829,"ukt":2014,"um ":5774,"ult":2011,"uli":1842,"ule":1205,"uld":1327,"un ":2311,"uge":2701,"ugl":1024,"ugu":1009,"ugt":1589,"udb":1145,"udd":1881,"ude":3316,"udg":3260,"udi":978,"ue ":1121,"uce":1989,"uer":2229,"ues":1633,"udv":2056,"uds":2177,"udt":1177,"uel":1000,"ub ":1257,"uar":1965,"ubl":1141,"ubb":1133,"ud ":2370,"typ":1409,"tyr":2052,"tys":3953,"ty ":1687,"træ":3121,"tur":4777,"tut":1397,"tun":1107,"tud":1198,"tyd":2007,"ts ":4804,"tre":5166,"tra":6691,"tri":4615,"tru":5093,"tro":3770,"try":1478,"tv ":1126,"tte":14611,"to ":3441,"tni":3342,"tne":990,"tof":1468,"tod":1141,"tog":1011,"ton":2739,"tol":1446,"tor":9016,"top":1001,"til":27566,"tik":5022,"tif":3307,"tie":1611,"tig":3596,"tit":2754,"tis":8549,"tin":4128,"tio":14633,"thu":1021,"tia":1323,"tid":6831,"tiv":4885,"tje":1256,"tli":4586,"tla":1138,"tle":1766,"tem":5581,"ten":11717,"tek":2453,"tel":3444,"teg":5502,"ted":5198,"th ":1733,"tet":9121,"tes":3522,"ter":41030,"på ":20701,"ti ":3596,"the":2230},"n_words":[9674395,11309170,8090238],"name":"da","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/de.json b/contrib/languages-data/de.json
new file mode 100644
index 0000000..df96884
--- /dev/null
+++ b/contrib/languages-data/de.json
@@ -0,0 +1 @@
+{"freq":{"D":565432,"E":313466,"F":303463,"G":350968,"A":448295,"B":452862,"C":218833,"L":272371,"M":385363,"N":224760,"O":171336,"H":264836,"I":169730,"J":158568,"K":336559,"U":140118,"T":243403,"W":221377,"V":203138,"Q":15370,"P":333284,"S":807982,"R":282432,"Y":16379,"X":9646,"Z":96079,"f":1052401,"g":2011313,"d":3686275,"e":12779820,"b":1254948,"c":2286896,"a":5009484,"n":7859499,"o":2714321,"l":3150399,"m":2035145,"j":85201,"k":1042619,"h":3128015,"i":7539051,"w":833806,"v":645462,"u":3015383,"t":5309288,"s":5151894,"r":6424621,"q":20503,"p":756433,"z":844832,"y":283697,"x":72882,"Ü":10506,"ß":107796,"Ö":13881,"í":8823,"é":38695,"ä":358815,"á":12062,"ü":397123,"ö":249595," l":98174," m":186199," n":127225," o":112357," h":112970," i":1142912," j":27689," k":107425," d":1491422," e":826321," f":160920," g":218820," a":569210," b":294630," c":26306," z":200898," u":467824," t":56510," w":322251," v":371555," p":57089," s":347463," r":56139," J":156270," K":328020," H":253550," I":146651," N":212076," O":159226," L":260916," M":370185," B":437655," C":196607," A":413316," F":291285," G":334059," D":543793," E":298852," Z":93212," Y":15401," S":736164," R":264582," Q":14562," P":316134," W":214460," V":189915," U":132267," T":228797," ä":10185," ö":18488," ü":31572," Ö":12893," Ü":10208,"A ":22760,"Da":90985,"Cl":9563,"Co":64248,"Ch":45017,"G ":11776,"Du":11229,"Do":28882,"Dr":19902,"De":166635,"Di":188563,"Fe":38252,"Fa":46184,"Eu":13931,"Er":67899,"Es":30499,"En":32393,"Ei":81512,"El":17463,"Ge":151044,"Ga":33294,"I ":13640,"Fu":26023,"Fr":58870,"Fo":32048,"Fl":33310,"Fi":36767,"B ":10700,"C ":14623,"Au":73540,"Ar":58134,"At":9258,"As":17569,"D ":9913,"Ba":87454,"Ab":25174,"Ad":11158,"Am":21201,"An":55559,"Ap":14324,"Al":66179,"Bu":55758,"Br":45253,"Ca":33967,"Bi":31885,"Be":143843,"Bo":32121,"Bl":15582,"Ku":30221,"Kl":25047,"Kr":46024,"Ko":56995,"Le":49783,"Li":55021,"La":89578,"Lu":17469,"Hö":9378,"Lo":26047,"Me":64987,"Mi":68245,"Ma":116460,"Mu":29273,"Mo":46837,"Ni":23664,"Ne":44049,"Na":66351,"P ":9001,"No":53500,"Ok":13364,"Ol":9042,"Ob":22144,"Gi":11262,"Gl":11809,"Gr":74845,"Go":17306,"Gu":11007,"Ha":86398,"He":62608,"Hi":21572,"Dé":12333,"Ho":42207,"Hu":11017,"Im":11278,"In":78332,"Is":10695,"Ja":77663,"Je":11227,"Jo":24131,"Ju":32751,"Ka":74688,"Ki":37983,"Ke":14966,"Um":8765,"Un":54096,"Tu":10377,"US":39827,"Tr":32100,"To":24574,"Th":35305,"Ti":19953,"Te":52263,"Ta":27705,"V ":9454,"Sy":15026,"St":193472,"Su":18450,"Wo":21476,"Wi":47351,"Wa":44055,"We":77082,"Vo":36505,"Vi":22665,"Va":14430,"Ve":104056,"Mä":13878,"Pu":10703,"Pr":83211,"S ":48175,"Pe":31242,"Pf":21717,"Pa":56041,"Pl":15401,"Po":48441,"Pi":16961,"Ph":15188,"Os":17404,"Or":58036,"R ":11116,"Kö":16414,"Se":64611,"Sc":120209,"Si":90352,"Sh":10028,"Sp":54743,"So":39076,"Ru":19547,"Sa":63564,"Re":106782,"Ri":24028,"Rh":15491,"Ro":44332,"Qu":12167,"SA":9500,"Ra":38143,"Mü":11354,"b ":47610,"a ":292619,"Sü":22392,"Ze":31894,"Zi":10309,"Zu":19954,"Zw":11243,"i ":187041,"ge":787899,"ga":94286,"fl":45063,"fg":17796,"ff":81186,"fi":73360,"fs":18166,"fr":68277,"fu":19769,"ft":140067,"fo":55471,"he":941699,"hb":12648,"ha":293469,"gn":22363,"gl":78298,"gk":13841,"gi":130788,"gh":20146,"gg":10845,"gu":58531,"gt":76690,"gs":110120,"gr":125748,"go":29970,"dt":100722,"du":88105,"dw":22283,"g ":403400,"ea":71177,"eb":165228,"ec":131420,"aß":17528,"ed":151386,"de":1831853,"dg":9266,"di":398882,"dh":11418,"dk":27018,"dl":45536,"do":69104,"dn":16795,"ds":46937,"dr":54467,"ew":66696,"ex":29126,"eu":212719,"ev":25427,"ey":30641,"ez":98518,"fa":86743,"h ":463041,"fe":136348,"eh":196157,"eg":232810,"ef":68699,"ee":58149,"el":553287,"ek":82123,"ei":1578594,"ep":49360,"eo":41207,"en":2309793,"em":396788,"et":380175,"es":772223,"er":2971302,"ca":38911,"bz":11720,"e ":2082290,"bs":31297,"br":69597,"bu":78689,"bt":20985,"bo":31335,"bl":45392,"bg":11719,"bi":136887,"be":567764,"db":9156,"da":176290,"f ":156375,"cu":10926,"ct":17144,"co":32151,"ck":133244,"ci":25902,"ch":1933159,"ce":45478,"c ":18586,"az":20071,"ay":30095,"ba":134727,"d ":722683,"at":393648,"as":323772,"ar":487358,"ax":8820,"aw":12659,"av":27657,"au":496496,"ak":51121,"al":571224,"ai":78558,"ap":48798,"am":283487,"an":921405,"ac":186135,"ad":167659,"aa":47275,"ab":98737,"ag":124899,"ah":148615,"ae":30527,"af":117969,"nu":74735,"nt":478161,"ns":309893,"nr":20689,"no":90394,"nn":190420,"nz":136146,"ny":13727,"nw":36447,"nv":14218,"oe":10022,"of":64141,"oc":78480,"od":135885,"oa":14277,"ob":51225,"om":176521,"on":667125,"ok":27868,"ol":220141,"oi":24320,"og":71416,"oh":66434,"ot":88470,"os":118609,"ov":54069,"ou":69753,"op":68308,"oo":24531,"or":433879,"r ":1993661,"ow":58643,"oz":19078,"kö":10263,"pe":109404,"pf":29697,"pa":81077,"kü":12292,"pl":26694,"po":71059,"ph":41272,"lä":41208,"pi":98645,"lo":124488,"ln":33301,"lm":52275,"ll":300411,"ls":178640,"lr":10222,"lp":20136,"lv":15360,"lu":97812,"lt":212969,"lz":28989,"hö":49290,"ly":20012,"o ":123676,"iß":9834,"ma":207608,"hü":11610,"mb":77222,"me":410943,"mf":12265,"ml":10335,"mi":238709,"mm":108620,"mp":57358,"mo":55705,"mt":36178,"ms":30817,"mu":48362,"p ":27276,"na":304264,"nb":61884,"nc":42156,"nd":1104110,"ne":802403,"nf":54695,"ng":588331,"nh":45861,"ni":342822,"nk":102273,"nl":39986,"nm":15381,"ki":44691,"ke":186644,"ka":155408,"fü":107647,"m ":660573,"ks":34677,"kt":124687,"ku":45967,"ko":74144,"kr":80423,"kl":44918,"km":33455,"li":562165,"hä":31781,"lh":12595,"lk":36060,"le":501436,"ld":86618,"lg":41891,"lf":32687,"la":312768,"lc":19357,"lb":59934,"n ":2940210,"hr":245096,"hs":72646,"hw":53275,"ht":189239,"hu":73471,"hk":8886,"hi":180846,"hn":163077,"ho":102479,"hl":130889,"hm":50687,"id":80042,"eß":9854,"ic":534870,"ib":46123,"ia":115654,"ih":35970,"ig":262184,"if":71985,"ie":1150659,"hy":13546,"k ":140095,"ir":170539,"is":1295546,"it":570135,"iu":24806,"iv":65304,"ik":165059,"il":298251,"im":289814,"in":1828758,"io":214368,"ip":33386,"je":30947,"fä":14922,"iz":46043,"l ":312169,"ja":19963,"tä":51288,"xi":15762,"z ":144384,"sü":19128,"sä":14628,"wi":176498,"wo":47417,"wu":73059,"rö":38038,"y ":99455,"wa":198781,"rü":63451,"we":246633,"rä":41054,"vi":76382,"vo":299196,"uz":17658,"ve":197477,"va":38019,"x ":21295,"ui":24830,"uk":24301,"ul":92478,"ue":71615,"uf":143869,"ug":74097,"uh":10714,"ur":375391,"us":360270,"ut":230981,"um":178633,"un":921062,"up":64261,"ty":34073,"tz":131414,"tu":172266,"tt":172596,"tw":53648,"ub":43047,"ua":44618,"ud":39409,"uc":128427,"w ":36904,"to":167744,"tn":11002,"tm":18841,"tl":110769,"ts":280176,"tr":215121,"tp":9922,"tg":32640,"tf":25417,"te":1203919,"tk":15341,"ti":420789,"pä":14388,"th":119533,"v ":14300,"tb":22514,"ta":360262,"su":36834,"sv":12561,"ss":289880,"st":1222626,"sy":19371,"sz":17514,"sw":26360,"sl":40519,"sk":49450,"sm":28595,"sp":131491,"so":116426,"sr":21109,"sd":16337,"sc":827974,"oß":20955,"sf":20622,"se":456715,"sh":47160,"sg":56341,"si":322120,"nö":10879,"rz":76103,"u ":108662,"sa":109787,"sb":46727,"rr":85173,"rs":267473,"rt":385464,"ru":174667,"rv":18995,"rw":59814,"ry":19301,"rp":30886,"ro":256692,"rn":187691,"rm":96942,"rl":93135,"rk":131828,"ri":506618,"rh":58397,"rg":185502,"rf":74192,"re":658545,"rd":287337,"rc":86223,"rb":105570,"ra":423969,"t ":1627252,"mö":8912,"qu":17893,"mä":18127,"lü":10612,"s ":1183348,"pt":54028,"pu":30332,"pp":47091,"pr":99340,"ps":12738,"zä":9571,"zö":21005,"wä":21780,"zi":99549,"zb":8992,"ze":213386,"za":20320,"zw":56096,"zu":170081,"zt":32056,"zo":13315,"ye":15230,"uß":27824,"ya":11551,"tü":14407,"yt":8983,"ys":38183,"yr":10616,"yp":13299,"yn":15755,"ym":18550,"yl":11401,"äc":27469,"Üb":9926,"ßb":17940,"ße":45162,"ßt":14023,"ép":12471,"ät":44405,"äu":30285,"äl":29341,"än":69784,"äs":15220,"är":37279,"äd":10841,"äg":13766,"äf":13624,"äh":39944,"ün":62239,"üs":17438,"ür":122114,"üt":16867,"üb":39622,"üc":24866,"üg":9236,"üd":43250,"üh":43336,"öß":14937,"ör":67546,"ös":54393,"ön":22557,"öl":13380,"öm":9902,"öh":14808,"öf":13450," Ga":33168," Ge":150714," Fo":31900," Fu":25992," Fr":58787," Fi":36574," Fl":33229," Ha":86258," He":62512," Go":17225," Gr":74652," Gu":10914," Gi":11192," Gl":11774," Hu":10983," Ho":42078," Dé":12331," Hi":21530," Je":11187," Ja":77578," Is":10643," Im":11222," In":77993," Ka":74525," Ke":14835," Ki":37849," Jo":24054," Ju":32722," La":89327," Le":49575," Li":54819," Kl":25011," Ko":56929," Kr":45960," Ku":30157," Ma":116111," Mi":68090," Me":64810," Lo":25969," Hö":9372," Lu":17420," Ne":43845," Na":66181," Ni":23603," Mo":46674," Mu":29161," Ap":14296," Am":21144," An":55404," Al":65979," Ad":11104," Ab":25090," Ba":87247," Au":73430," At":9235," As":17394," Ar":57936," Be":143584," Bi":31731," Bl":15450," Bo":31991," Br":45134," Bu":55625," Ca":33546," Ch":44814," Cl":9441," Co":63874," Da":90740," Di":188080," De":166199," Dr":19868," Do":28619," Du":11174," El":17420," Ei":81331," Es":30480," Er":67840," En":32299," Eu":13899," Fe":38170," Fa":46079," Sü":22387," Wo":21330," Wi":47141," We":76915," Wa":43907," Zu":19747," Zw":11238," Ze":31828," Zi":10283," Kö":16410," Os":17332," Or":58009," Po":48260," Pl":15331," Pi":16916," Ph":15079," Pf":21701," Pe":31180," Pa":55779," No":53429," Ol":9032," Ok":13357," Ob":22103," Ra":37989," Mü":11350," Qu":12048," Ro":44140," Re":106629," Ri":23976," Rh":15474," Pr":83041," Pu":10674," Mä":13877," Sy":14940," Su":18414," St":192451," Ta":27626," Th":35151," Ti":19864," Te":52071," US":39409," Tr":31991," To":24396," Ru":19497," Sa":63396," Sh":9926," Si":90133," Sc":119896," Se":64226," So":38900," Sp":54582," Va":14382," Ve":103873," Vi":22523," Vo":36449," Tu":10321," Um":8742," Un":54031," je":12055," im":207684," in":455196," is":439990," it":10403," ka":23448," fü":85353," ha":41553," he":33774," gi":8908," gl":12278," gr":31732," ih":21981," hi":13984," ho":9452," ni":22730," ne":14510," na":48500," od":67197," of":20927," nu":10713," no":19195," le":19412," li":42027," la":25753," ku":12534," km":15800," kl":13510," ko":18379," me":26065," mi":107786," ma":27726," ab":26560," am":69685," an":87925," al":101826," au":256133," ba":8860," bi":45326," be":204994," br":14611," bz":9809," er":81246," et":21454," es":15402," en":55368," ei":604709," eh":21203," fr":36324," ge":155650," ch":11442," da":104895," dr":12440," de":1066350," di":258330," du":31949," zu":137283," zw":39753," ze":10132," sü":17881," ru":14611," se":68014," sc":24771," si":106825," sp":27646," so":41648," re":21422," pr":15398," po":16457," wa":91486," we":72625," wu":68805," wi":72114," ve":73784," vo":278143," vi":13609," um":38406," un":417759," st":39848," tr":12290," th":15725," te":8776," Üb":9908," ös":13916," üb":31186,"Fer":11772,"Fil":18131,"Fam":19152,"Es ":23376,"Eur":10652,"Ein":58080,"Eis":15092,"Er ":28197,"Ent":11856,"Gem":51515,"Geb":17190,"Ges":25022,"Ger":10345,"Gen":10761,"Gat":12348,"Fuß":15716,"Flu":13729,"Fra":28567,"Fri":10250,"Fre":14074,"For":20619,"Dép":12076,"Hei":15430,"Her":23305,"Hal":8751,"Han":15746,"Hau":30716,"Gre":10954,"Gra":17232,"Gru":19222,"Gro":14185,"Int":14682,"Ins":19291,"Ind":10322,"In ":14119,"Hoc":11500,"Art":17165,"Aut":13725,"Aus":25209,"Aug":12878,"Auf":14417,"Ban":13867,"Bad":10629,"Bar":8982,"Bas":9343,"Bau":14505,"Als":17057,"Alt":10907,"Ant":9903,"Bun":26145,"Bur":11846,"Car":9966,"Bez":18543,"Bes":11001,"Ber":42078,"Bei":10336,"Beg":14069,"Bre":9847,"Bra":12567,"Dez":9022,"Der":97305,"Deu":28251,"Das":56242,"Chr":9059,"Chi":9883,"Cha":13525,"Com":14437,"Cou":18879,"Die":157652,"Dre":10394,"Dor":10511,"Neu":14698,"Nat":19706,"Nie":12050,"New":10171,"Nam":20762,"Nac":12244,"Nov":10049,"Nor":35685,"Okt":8850,"Obe":15435,"Ort":33777,"Ost":13628,"Pla":10533,"Pfl":9335,"Pfa":9319,"Per":13707,"Par":21713,"Pro":54127,"Pri":9311,"Pol":21608,"Mär":10626,"Jan":11806,"Jah":47539,"Joh":10371,"Jul":9776,"Jun":10796,"Kan":19278,"Kar":13767,"Kir":12749,"Kil":10995,"Kla":9457,"Kon":17432,"Kom":17455,"Kre":22963,"Kri":9328,"Kul":11508,"Lei":11844,"Lan":56584,"Lin":9234,"Lis":12636,"Men":8765,"Met":15567,"Man":15071,"Mal":9073,"Mar":33951,"Mai":15148,"Mon":12046,"Mit":29518,"Min":10015,"Mus":19078,"Süd":20812,"Wil":11137,"Wie":10497,"Wei":14673,"Wel":14947,"Wer":11351,"Wes":22169,"Was":9091,"Wal":11499,"Vor":17501,"Vol":11987,"Zei":17372,"Str":23794,"Stu":11271,"Sta":109473,"Ste":21247,"Tei":19885,"Sit":13505,"Sie":53287,"Ser":9549,"Sep":9592,"Sei":11021,"See":8773,"Spi":16479,"Spr":13123,"Sch":114345,"San":12822,"Rhe":11094,"Rec":10719,"Rei":15453,"Reg":42336,"Rom":8900,"Ver":96243,"Uni":18259,"Unt":29085,"The":20642,"US ":31557,"Tra":12824,"bis":44646,"bil":24249,"bin":15696,"ble":9072,"bli":23577,"be ":15350,"ban":20200,"bal":24830,"bah":12251,"bac":14708,"bau":19849,"bar":18594,"bei":74577,"bef":14886,"bed":10076,"ber":182051,"ben":84728,"bel":16201,"bek":17699,"bez":50353,"bew":10065,"bes":47672,"bet":15480,"bie":27376,"bge":10311,"bzw":9957,"ca ":13669,"ce ":20992,"bt ":13539,"bri":17955,"bra":14742,"bre":12676,"bru":12222,"bst":12657,"bur":42376,"bun":13485,"am ":57496,"al ":67162,"ain":24754,"ais":13530,"ahm":10952,"ahn":21966,"ahl":22279,"ahr":74469,"anu":15776,"anz":50908,"ano":9997,"ann":95807,"ant":49762,"ans":31362,"ane":16424,"ang":74734,"ani":87171,"ank":29449,"ana":30762,"anc":17098,"and":264094,"amt":12217,"amm":42375,"amp":12259,"ami":43004,"ame":78887,"amb":11361,"ama":13307,"alz":12079,"alt":72681,"als":84009,"all":87642,"ali":76090,"ald":16999,"ale":62596,"ala":15461,"alb":20979,"an ":124982,"akt":19509,"abe":38207,"ae ":18008,"aat":34161,"ad ":15688,"ab ":9708,"aft":77933,"aff":9982,"ai ":14358,"age":54066,"adi":20643,"ade":24209,"ag ":17784,"adt":77282,"ack":9729,"ach":145520,"ace":11312,"aye":9954,"at ":58813,"are":33081,"ard":28108,"arc":10050,"arb":19113,"ara":24649,"aro":9777,"arl":16356,"ark":32666,"ari":41626,"arr":11665,"ars":12931,"art":72580,"au ":29621,"asi":19652,"ase":11784,"ar ":124263,"as ":155043,"aut":27334,"ay ":9249,"ata":11051,"ast":27543,"ass":73183,"ato":14091,"ate":62804,"ati":113716,"ath":22536,"auc":75721,"att":42075,"ats":11481,"atu":18974,"atz":18097,"aum":14540,"aup":28995,"aus":149891,"aue":19995,"auf":98508,"itu":19936,"itt":50909,"its":25338,"itz":41656,"ism":9438,"iss":46444,"ist":567635,"ita":30952,"ite":83935,"itg":12750,"iti":55617,"ium":14795,"ivi":10241,"ive":36462,"is ":120788,"ion":169772,"irt":10794,"irk":24645,"isi":17517,"ish":14382,"ise":39335,"isc":434611,"isa":10445,"ire":14375,"ird":46859,"irc":21452,"it ":186965,"itä":20849,"izi":17119,"ize":13649,"kir":9307,"kis":9934,"km ":11509,"ki ":9579,"kei":25216,"kel":18319,"ken":38622,"ker":44613,"key":12420,"ke ":26956,"kra":15849,"kre":43910,"kt ":30020,"kri":11594,"kon":15402,"kom":22572,"ks ":9057,"kma":10480,"kle":18061,"kla":10851,"kat":15038,"für":75640,"kan":77012,"kal":10319,"füh":22557,"ka ":18181,"han":36578,"hal":38788,"hau":37347,"har":21603,"hat":27886,"haf":79189,"hab":9073,"he ":213315,"hel":11970,"hei":73898,"hec":9261,"heu":16839,"hes":34182,"her":185057,"heo":9547,"hen":324438,"hem":37228,"hie":35579,"hic":13745,"hin":28238,"hil":15183,"his":35328,"hl ":14867,"hn ":19219,"hla":28455,"hle":29146,"hli":16905,"hlo":9385,"hlu":9270,"gle":20084,"gli":40281,"gke":10880,"gs ":14145,"gsb":9179,"gsg":11045,"gro":10882,"gru":14203,"gra":25992,"gt ":53228,"gri":24893,"gre":15248,"gst":11232,"gss":10565,"gte":17882,"grö":13018,"grü":19508,"gus":11665,"gun":24014,"ial":21666,"ian":25164,"iat":10334,"ibt":8975,"id ":10846,"ibe":11815,"ia ":36994,"iet":30400,"iel":84385,"ien":123978,"ier":122946,"ies":42285,"ied":66086,"ieg":57690,"ief":8944,"ieh":10954,"ig ":34684,"iec":11989,"ieb":29946,"ift":20078,"iff":26227,"ick":25712,"ich":454249,"ie ":537544,"ica":15678,"ide":33142,"ida":13186,"ieß":9409,"il ":53074,"im ":225234,"ika":58499,"ige":123608,"iga":12204,"igk":10797,"igi":12120,"igu":13208,"igt":19283,"ign":9138,"ihe":9068,"ihr":18848,"ik ":39429,"imm":16766,"ime":10380,"ind":156874,"ina":40189,"inn":28145,"ino":11205,"int":30667,"ins":51098,"inf":10131,"ine":425715,"inh":13926,"ing":93214,"ini":56092,"inl":11619,"ink":12759,"inw":21089,"inz":40189,"ike":34841,"in ":793598,"ilo":17887,"ill":36002,"ilm":27378,"ili":52680,"ild":30027,"ile":21718,"ima":11905,"io ":15324,"hr ":43100,"hol":19495,"hor":9009,"hof":13254,"hoc":13003,"hni":15320,"hnu":17265,"hne":77095,"hme":30297,"hul":13709,"htu":10990,"hts":18003,"hti":8985,"hte":48599,"hst":18465,"hse":23032,"hrt":21145,"hre":87475,"hri":38046,"ht ":81034,"hwa":10659,"hwe":33406,"hum":9803,"hun":26557,"ffe":34346,"fes":13275,"fer":33138,"fen":43852,"fel":18220,"fge":10643,"fas":17434,"fan":11879,"fal":21835,"fah":14992,"ff ":20858,"fe ":14115,"ewä":9115,"eze":57686,"ezi":30267,"eta":11725,"ete":74739,"eti":14295,"eso":11977,"est":128414,"ess":60542,"eue":13801,"eug":13587,"etr":30198,"ett":26303,"etw":18129,"etz":35905,"ew ":11648,"eut":121638,"eur":19487,"ewi":9238,"ewe":18343,"ey ":16388,"erö":10701,"er ":1578023,"eor":17275,"es ":365853,"ept":13647,"epu":9001,"erk":45646,"erl":56900,"eri":118551,"erg":76890,"erh":33322,"ere":140461,"erf":36150,"erd":42731,"era":58724,"erb":63293,"et ":137716,"esi":26333,"esc":45868,"ese":67043,"esa":12858,"erz":20677,"erv":14305,"eru":46595,"erw":47997,"err":51760,"ert":122552,"ers":190257,"ern":139910,"erm":29678,"erp":11457,"ero":15582,"ekt":37882,"en ":1453342,"elb":18844,"ela":19480,"eld":20851,"elc":11326,"elf":9116,"ele":71198,"eli":22926,"elm":9454,"eln":17751,"ell":119434,"els":30215,"elt":58131,"ehö":33094,"eiß":9569,"emb":35370,"ema":36279,"eme":116815,"emi":15129,"enf":15937,"ene":83278,"enh":15362,"eng":41769,"enb":41104,"ena":52256,"end":112707,"eno":10734,"enn":31346,"enk":30270,"enl":8977,"eni":28822,"ens":113000,"ent":181395,"enr":9992,"enz":28843,"ege":66150,"egi":51407,"egr":34898,"egt":35267,"ehm":19125,"ehr":37443,"ehe":47146,"eib":17640,"eic":136250,"eht":23060,"eis":113764,"eim":26661,"eil":79634,"ein":827667,"eih":11402,"eie":15075,"eid":22147,"eig":23796,"el ":101100,"eiz":17159,"eit":174763,"efü":9260,"eka":19118,"em ":162112,"gis":26824,"gin":15255,"gio":29407,"gie":31403,"gen":239581,"get":10602,"ger":96243,"ges":76492,"gew":20591,"geb":50080,"geh":39393,"geg":29832,"gef":16578,"gem":34840,"gel":53887,"ge ":89868,"gab":8826,"gar":15096,"gan":29626,"ga ":12696,"frü":9222,"fte":22800,"ftl":11215,"fun":10974,"fts":18001,"ft ":64901,"fra":26521,"fre":14454,"fri":12770,"for":35090,"fol":14926,"fla":12837,"flu":9809,"fil":11145,"fin":25920,"da ":17513,"de ":197184,"das":76737,"dar":15587,"dam":14425,"ckl":11099,"chä":10997,"ch ":425455,"cha":118322,"chw":51871,"chu":40552,"ck ":31790,"che":671441,"chl":70583,"chi":103178,"cho":20768,"chm":11901,"chn":86780,"chs":68005,"cht":155579,"chr":43254,"cke":51895,"ed ":20514,"eba":8988,"ebe":60498,"ebi":30463,"ebr":17022,"ebu":9376,"eat":10811,"efi":14121,"efe":11305,"ei ":84962,"ega":9107,"eer":10701,"edi":25198,"ede":66213,"eg ":9875,"eck":32288,"ech":81379,"aße":12922,"ee ":17886,"dwe":14218,"dur":31652,"dor":21874,"don":10905,"ds ":13347,"dun":25608,"dri":12382,"dt ":65936,"dre":16392,"dsc":11728,"der":838163,"des":213577,"det":50672,"deu":71719,"del":34612,"den":286768,"dem":104609,"dkr":22973,"dli":27133,"din":14965,"dis":37105,"die":268782,"dig":18151,"rhe":18402,"rha":21798,"rga":19288,"rgi":14417,"rge":50622,"ret":20023,"res":34677,"reu":13265,"rfa":14831,"rfo":9599,"rg ":68697,"rea":10702,"rec":32717,"raß":12628,"rei":196090,"reg":17320,"rem":14552,"ren":145744,"rer":33774,"rf ":19644,"rdn":11514,"rdl":8810,"rdi":14216,"rde":136404,"re ":80803,"rbr":9368,"rch":72681,"rd ":68822,"rap":8929,"rar":8924,"ras":13690,"rat":45378,"rau":34010,"rbi":13086,"rba":21515,"rbe":36419,"rai":10459,"rag":25026,"ran":87263,"ram":18048,"ral":31964,"rab":8938,"raf":18926,"rad":17310,"rac":30425,"rs ":37799,"ros":12129,"rot":11834,"rom":12509,"ron":32055,"rop":24374,"rov":26322,"rod":18103,"rol":12320,"rof":8772,"rog":9738,"rns":12992,"rna":25318,"rne":34331,"rni":11673,"ro ":10671,"rma":31109,"rme":19061,"rmi":10506,"rli":33691,"rle":11502,"rla":23240,"rn ":74871,"rks":11567,"rke":27812,"rm ":16219,"rit":36924,"ris":65792,"rig":20472,"ril":13027,"rik":52744,"rin":60346,"ria":21040,"ric":47382,"rie":98836,"rif":31298,"rk ":38902,"rwe":21555,"rz ":21038,"ruc":12341,"rup":19176,"run":74339,"rum":15222,"rus":14979,"rwa":28214,"ry ":12420,"rsi":21022,"rso":11863,"rsp":15246,"rsc":59984,"roß":19513,"rsa":9024,"rse":16931,"rta":10712,"rst":71108,"rte":100313,"rth":11159,"rti":23089,"rua":8770,"rts":42716,"rtr":18145,"rt ":139444,"rro":10371,"rri":12758,"rre":38535,"rra":9354,"sam":29233,"sbe":20860,"san":12568,"sat":17995,"rze":22030,"sha":10198,"sho":11670,"sge":44768,"sie":44959,"sic":57522,"sit":30295,"sis":59876,"sin":51565,"sio":15881,"sik":20202,"se ":74178,"oße":10850,"sch":817718,"ser":52310,"ses":13066,"set":25883,"seu":12734,"sei":61074,"seh":13958,"see":12215,"sen":104567,"sem":11510,"sel":53325,"spo":10155,"spr":34328,"spe":13644,"spi":46629,"spa":12483,"sow":18519,"sol":10161,"son":33802,"sor":15536,"sre":10655,"st ":524603,"ss ":36271,"sla":20592,"ski":8870,"ska":11035,"so ":11408,"swe":11000,"stä":18451,"sse":110613,"ssa":12522,"sso":12151,"ssi":42403,"sst":37624,"ste":300373,"stf":9710,"sta":116653,"sto":29905,"sti":57375,"stl":38070,"stu":17599,"str":60741,"sun":13352,"sve":8840,"tal":50681,"tag":11693,"taa":33781,"tad":80208,"tbe":10504,"tau":10504,"tat":27116,"tar":23114,"tan":59689,"tam":12219,"te ":230506,"ta ":18923,"pe ":20233,"par":30140,"pan":20821,"läc":9211,"län":14161,"phi":11788,"pen":21299,"per":31265,"pel":10530,"pla":12193,"pie":61526,"por":21385,"pol":21212,"ppe":33120,"pub":8976,"pte":12552,"pts":12222,"pra":20767,"pri":21158,"pre":14559,"pro":24883,"ra ":23418,"ngi":9157,"ngl":32203,"ngs":81612,"ni ":15548,"nge":150664,"nga":14715,"nha":17909,"nhe":16114,"neh":18303,"nel":12298,"nen":124634,"nem":32650,"ner":149716,"net":57023,"nes":53346,"neu":10682,"ng ":256902,"nfo":9002,"nfa":11653,"nce":12344,"nch":15880,"ne ":310275,"nbu":15121,"ndt":9026,"ndu":21347,"ndr":11275,"nds":26772,"ndo":18073,"ndl":16133,"ndk":23064,"ndi":40257,"nde":333302,"nda":14652,"nal":58896,"nam":18687,"nan":39807,"nar":14659,"nac":41308,"nad":10470,"nah":12864,"nbe":18026,"nd ":563951,"nba":15883,"nau":11932,"nat":30766,"na ":34549,"nz ":37982,"nwo":19169,"nve":9948,"nun":33102,"nur":9530,"nua":9860,"nty":16624,"ntw":17792,"nto":23385,"nts":23619,"ntr":27593,"nti":33190,"nth":12699,"ntl":20923,"nta":24938,"nte":156191,"nsp":11166,"nst":73244,"nse":37942,"nsi":11408,"nsc":47879,"nsa":13614,"nsb":9715,"nt ":97967,"ns ":52604,"nom":14191,"nor":17522,"nne":52732,"nni":11264,"nnt":52262,"nns":10154,"nli":11747,"nn ":35057,"nla":20752,"no ":11518,"nke":16356,"nkm":14368,"nkt":16086,"nkr":10153,"nig":30567,"nie":44175,"nic":23391,"nia":9121,"niv":12854,"nis":144657,"nit":17226,"nin":9152,"nik":13041,"ogr":16058,"ogi":20348,"oge":19173,"ohl":8865,"ohn":34927,"och":36284,"ock":28973,"ode":93686,"of ":22612,"odu":15579,"off":17101,"nzö":20567,"obe":24203,"nze":40975,"nzi":11904,"owi":20813,"ozi":9366,"ow ":10053,"oti":8974,"oth":10089,"ote":15159,"ott":13896,"oto":13207,"ost":26924,"ose":16100,"oss":15828,"ovi":27674,"ove":15851,"oun":22912,"our":13749,"oph":11323,"opa":9142,"os ":20626,"or ":51491,"ork":11076,"orm":35190,"orn":17190,"ord":64827,"ore":22869,"orf":23363,"org":24171,"ori":35553,"ort":57075,"ors":22259,"ora":13071,"ola":9778,"old":10163,"on ":380926,"oli":42117,"oll":31645,"olk":11956,"ole":12997,"olg":16083,"olo":30821,"om ":33290,"ona":50054,"ond":30722,"one":41633,"ong":12616,"oni":29183,"onn":12427,"ono":12443,"ons":40673,"ont":24319,"oma":20160,"ome":26297,"omi":13337,"omm":35850,"omp":19252,"omo":11157,"la ":19811,"lb ":10692,"le ":103052,"lch":16014,"lde":27222,"lac":9475,"lag":29983,"lan":114878,"lar":12710,"lat":31357,"las":30573,"lau":19611,"lba":8767,"ld ":26676,"lbe":17468,"kur":12239,"kun":14388,"kte":20495,"ktr":9556,"ktu":9324,"kti":30020,"kto":15493,"ls ":107647,"lon":9380,"lom":14967,"log":27324,"los":22267,"lti":11054,"ltu":37425,"lug":9766,"lsp":9315,"lst":14411,"lte":61749,"lsc":14894,"lt ":63451,"lge":22548,"li ":14630,"les":24231,"let":14648,"ler":100316,"lem":16242,"len":91180,"lek":12889,"lei":58951,"leg":20421,"leb":11626,"lls":25768,"llu":11230,"llt":17543,"lla":17394,"lle":120640,"lli":30333,"ln ":17395,"lm ":19332,"ll ":39906,"lit":40830,"lis":62203,"lin":51826,"lic":170755,"lia":13089,"lik":12998,"lig":41309,"lie":114815,"ma ":15588,"mar":19548,"mal":49935,"man":57586,"mat":30614,"mbe":37028,"me ":36368,"met":20898,"mes":11403,"mer":74569,"mel":10134,"men":133736,"mei":94533,"meh":11820,"mfa":9540,"lve":9179,"lun":37355,"lus":22607,"lz ":10714,"hör":33720,"mpf":9424,"ms ":14667,"mon":13165,"mt ":15320,"mte":10917,"mus":17368,"mun":19505,"min":21460,"mil":28731,"mis":23371,"mit":124264,"mig":9701,"mie":12587,"mmu":13317,"mmt":13008,"mme":52772,"zt ":19861,"zte":9874,"zu ":51921,"zw ":9786,"zug":11138,"zur":37734,"zum":31618,"zun":15443,"zus":11821,"zwe":22370,"zwi":20415,"zei":75965,"zeu":11441,"zes":9884,"zen":43803,"zem":9613,"zel":10023,"zer":23898,"ze ":15773,"zia":11658,"zie":29129,"zig":9649,"zir":18395,"yst":17491,"ußb":14873,"uße":8940,"yer":10735,"tän":11031,"tät":24189,"süd":17890,"wur":70546,"woh":26117,"wes":27777,"wer":58987,"wen":21225,"wel":20147,"wei":83410,"weg":12299,"wir":55249,"wis":33738,"wie":34766,"wic":21374,"wa ":19878,"röß":14321,"wan":13038,"rün":30393,"wal":31257,"war":103047,"rüh":12452,"rüc":12801,"vin":25318,"vie":18884,"von":226355,"vom":22538,"vor":35872,"räg":10007,"ver":144066,"ven":17663,"vem":8808,"ve ":14670,"van":10775,"usi":22265,"usg":18900,"use":25762,"usa":16989,"ust":38480,"uss":45929,"usp":9564,"uti":13537,"ute":38886,"utz":15915,"uts":91328,"uto":20186,"us ":143584,"ut ":20376,"urd":76437,"urc":36382,"ure":16506,"urg":50861,"uri":14126,"urn":9214,"uro":18224,"urs":13370,"urt":10114,"urz":14868,"ur ":74659,"upp":21507,"upt":28543,"ums":9212,"umb":12929,"ume":14114,"umf":9413,"unt":60343,"uns":11344,"unk":20377,"uni":19103,"und":450828,"ung":313251,"ukt":12384,"um ":104918,"ult":19564,"uli":15580,"ule":13595,"ula":10852,"uge":19918,"uft":12442,"ugu":12915,"uf ":70919,"ude":15376,"ue ":12656,"uch":109126,"uck":11441,"uer":24349,"ufe":13329,"ufg":14459,"uen":13989,"uel":10929,"uar":22497,"ubl":12721,"tzt":23155,"tzu":9903,"tze":24373,"ty ":26382,"twa":23393,"tur":43568,"tun":79448,"tum":9791,"tz ":51212,"twi":14763,"twe":10501,"ts ":32847,"tre":34672,"tt ":15987,"tra":75255,"tri":43316,"tru":19097,"tro":30104,"tsc":125606,"tsg":8757,"tsp":10708,"tst":44445,"tte":89035,"tti":9403,"ttu":15931,"to ":13838,"tob":10974,"tom":9404,"ton":34822,"tor":56677,"tik":35201,"tie":28029,"tig":44003,"tit":14488,"tis":66050,"tin":30024,"tim":13478,"tio":114009,"thu":10251,"tiv":26072,"tli":78058,"tla":9588,"tle":19198,"tem":46999,"ten":278529,"tei":81154,"tel":91292,"teh":26143,"tec":9644,"tfa":9048,"th ":15758,"tet":40557,"tes":39452,"ter":318918,"tgl":9160,"tge":13122,"tho":19217,"the":36369,"tha":11467,"zös":20559,"zäh":9158,"épa":12146,"ähl":11359,"ähr":16538,"äch":26798,"äng":21244,"änd":29875,"ält":11293,"ät ":14412,"ärz":9551,"äte":10008,"äuf":9858,"ßte":10219,"ßen":17047,"ßer":9786,"ßba":15077,"ße ":16400,"Übe":9600,"üdl":8991,"ühe":8949,"übe":35449,"ück":19105,"ünd":28744,"ür ":75451,"üns":8869,"ühr":24588,"ößt":9800,"öst":25495,"ösi":20607,"ört":26727,"örd":14857,"öni":10085,"öff":11553,"ürt":8780},"n_words":[87197534,99298261,71857404],"name":"de","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/en.json b/contrib/languages-data/en.json
new file mode 100644
index 0000000..f0c64a5
--- /dev/null
+++ b/contrib/languages-data/en.json
@@ -0,0 +1 @@
+{"freq":{"D":662077,"E":559834,"F":608348,"G":582849,"A":1412556,"B":916671,"C":1498503,"L":649564,"M":1059392,"N":665299,"O":394009,"H":682378,"I":978047,"J":471262,"K":369000,"U":380950,"T":1402307,"W":505824,"V":256072,"Q":51938,"P":880443,"S":1630016,"R":708545,"Y":131400,"X":37224,"Z":73521,"f":5238032,"g":4381944,"d":8729953,"e":27848709,"b":3669334,"c":7841280,"a":23418136,"n":19713516,"o":18673929,"l":10669664,"m":6170962,"j":262547,"k":1633239,"h":10134148,"i":20570816,"w":3362380,"v":2275926,"u":6637499,"t":19408712,"s":16004058,"r":16873084,"q":170855,"p":4621926,"z":397471,"y":4124069,"x":440231,"é":58984," l":780297," m":1079276," n":608651," o":3436911," h":813084," i":3938466," j":78084," k":215197," d":941771," e":740834," f":1754540," g":440359," a":5405782," b":1659799," c":1744132," y":83514," u":322099," t":5058192," w":1900306," v":255788," q":29653," p":1560368," s":2415857," r":962057," J":460420," K":336817," H":654413," I":868613," N":607763," O":345142," L":596797," M":988580," B":847481," C":1361375," A":1263874," F":559603," G":550525," D":599768," E":493444," Z":67911," Y":122764," S":1468740," R":648863," Q":47311," P":804972," W":486015," V":212635," U":359652," T":1336813,"A ":159383,"Da":104182,"Cu":44169,"Cl":60734,"Co":426605,"Cr":65402,"Ce":60359,"Ch":258329,"Ci":62395,"Ed":38074,"Ea":55403,"Du":53228,"Do":65822,"Dr":31956,"De":174035,"Di":112559,"Fe":83008,"Fa":58998,"Eu":42215,"En":125985,"Em":27259,"El":44031,"Ge":134248,"Ga":73890,"I ":75735,"Fr":140633,"Fo":106181,"Fl":37525,"Fi":77847,"B ":28902,"C ":86756,"Au":138720,"Ar":132765,"At":36103,"As":94697,"D ":41383,"Ba":186506,"Af":42815,"Ac":49109,"Ad":33381,"Am":167256,"An":117416,"Ap":56752,"Ai":30417,"Al":127646,"Bu":74783,"Br":187559,"Ca":304473,"E ":36124,"Bi":58070,"Be":129264,"Bo":112086,"Bl":40686,"Ko":45338,"Le":127247,"Li":124649,"La":147235,"Lu":31872,"Lo":122679,"Me":130273,"Mi":151239,"O ":29173,"Ma":405967,"Mu":71213,"Mo":145746,"Ni":43349,"Ne":178415,"Na":143656,"P ":44679,"No":180123,"Ol":47193,"On":40436,"Oc":53105,"Gi":29349,"Gr":130751,"Go":73286,"Gu":48214,"Ha":132380,"He":218817,"II":40680,"Hi":92944,"Ho":127334,"Hu":45621,"K ":32711,"In":253915,"Is":60970,"It":340082,"Ir":53269,"Ja":154594,"L ":34197,"Je":52293,"Jo":101484,"Ju":117086,"Ka":74254,"M ":37484,"Ki":74159,"Ke":51086,"Un":220729,"Tu":36890,"Tr":84116,"US":35610,"To":93419,"Th":850306,"Ti":41292,"Te":99569,"Ta":74444,"V ":32754,"Sw":37200,"Sy":30043,"St":291796,"Su":96654,"Wo":87820,"Wi":99188,"Wh":33410,"Wa":135830,"We":94221,"Vi":82206,"Va":46571,"Ve":37939,"Pu":36227,"Pr":165318,"S ":81629,"Pe":109002,"Pa":220340,"Pl":35710,"Po":107653,"Pi":44494,"Ph":43729,"Or":50633,"R ":32772,"Se":177109,"Sc":119654,"Si":88455,"Sh":112206,"Sp":78520,"So":160970,"Ru":53200,"Sa":155395,"Re":186160,"Ri":92732,"Ro":164209,"Qu":37827,"T ":30323,"Ra":101065,"b ":134105,"a ":2991353,"Yo":71760,"i ":355519,"gd":31577,"ge":721279,"ga":370090,"fl":84104,"ff":180681,"fi":526440,"fr":391564,"fu":82822,"ft":147685,"fo":920673,"gy":74018,"he":4842012,"ha":1007728,"gn":127025,"gl":188283,"gi":377563,"gh":367310,"gg":29821,"gu":248240,"gt":38716,"gs":80931,"gr":275281,"go":177983,"du":289181,"dv":26927,"dw":35481,"dy":74747,"g ":1213593,"ea":1274992,"eb":152577,"ec":841985,"ed":2289411,"de":1320778,"dd":64948,"dg":51752,"di":930651,"dm":40343,"dl":51794,"do":273396,"ds":196380,"dr":141993,"ew":250035,"ex":221189,"eu":76879,"ev":357914,"ey":221145,"fa":236961,"h ":1529402,"fe":315742,"eh":48317,"eg":294475,"ef":194847,"ee":528675,"el":1146347,"ek":64162,"ei":272994,"ep":332185,"eo":187165,"en":2427008,"em":679038,"et":814658,"es":2395636,"er":4179896,"eq":41573,"ca":1058365,"e ":8494237,"by":527627,"bs":58904,"br":217191,"bu":297420,"bo":472510,"bl":324534,"bi":213219,"bb":35633,"be":846369,"da":395082,"f ":2316051,"cy":71900,"cu":238303,"ct":817847,"cs":93807,"cr":254963,"co":1252186,"ck":305854,"cl":251999,"ci":687756,"ch":1106571,"ce":1086909,"cc":111912,"c ":457702,"az":72598,"ay":450984,"ba":470554,"d ":4698126,"at":2664116,"as":2180049,"ar":2492347,"ax":35262,"aw":117859,"av":238618,"au":228752,"ak":197752,"al":2475728,"ai":590954,"aj":50310,"ap":330102,"am":913130,"an":4857931,"ac":715176,"ad":654223,"ab":325448,"ag":433896,"ah":83166,"ae":126319,"af":115896,"nu":217006,"nt":1825754,"ns":843426,"nr":30719,"no":643421,"nn":264870,"nz":30433,"ny":179007,"nv":69977,"oe":69809,"of":2379880,"oc":501750,"od":362363,"oa":174782,"ob":174628,"om":1179222,"on":3432632,"ok":132945,"ol":912097,"oi":135962,"og":247625,"oh":71587,"ot":549343,"os":498414,"ov":435331,"ou":1258409,"op":505111,"oo":421044,"or":2962572,"r ":3075136,"ox":44665,"ow":555778,"oy":75842,"pe":765233,"pa":591162,"pl":399253,"po":579386,"ph":211053,"pi":302192,"lo":761247,"lm":115044,"ll":1129598,"ls":295312,"lp":38509,"lw":35880,"lv":66473,"lu":281619,"lt":228186,"ly":725738,"o ":1535371,"ma":968442,"mb":416311,"me":1451345,"mi":591681,"mm":262444,"mp":468344,"mo":504581,"ms":133483,"mu":261893,"my":62036,"p ":307798,"na":1160193,"nb":27967,"nc":750937,"nd":2690580,"ne":1275364,"nf":87191,"ng":1746068,"nh":37213,"ni":1104016,"nk":109355,"nl":88734,"nm":67028,"ju":44053,"jo":70049,"ki":171410,"kh":26225,"ke":323852,"ka":109789,"m ":983735,"ky":30837,"ks":106786,"ko":46397,"kl":33919,"km":32936,"kn":143680,"li":1420180,"lk":42132,"le":1534709,"ld":351541,"lg":35629,"lf":59414,"la":1421955,"lb":105992,"n ":6374219,"hr":171058,"hw":45980,"ht":193546,"hu":172799,"hi":1052052,"hn":92044,"ho":773733,"hl":46194,"hm":30448,"id":496380,"ic":1849130,"ib":159645,"ia":1169835,"ig":507511,"if":219645,"ie":800933,"hy":82019,"k ":515132,"ir":647827,"is":3249081,"it":1893192,"iu":65009,"iv":578092,"ix":50259,"ik":78883,"il":1065515,"im":394260,"in":4877222,"io":1592954,"ip":270764,"je":50702,"iz":128734,"l ":1934675,"ja":50690,"xi":69443,"xp":49241,"xt":57882,"z ":53637,"xa":40308,"xe":33587,"wh":393518,"wi":429474,"wn":279417,"wo":263879,"wr":100589,"ws":68714,"vy":28150,"y ":3097451,"wa":1012808,"we":415691,"vi":599499,"vo":108876,"ve":1210480,"va":267555,"x ":126181,"ui":189592,"uk":33957,"ul":459768,"ue":281220,"uf":33361,"ug":230153,"ur":968763,"us":936891,"ut":635083,"um":403450,"un":943875,"up":204022,"ty":567576,"tu":446554,"tt":322006,"tw":175696,"ub":264839,"ua":340676,"ud":207254,"uc":282987,"w ":304612,"to":1515648,"tm":51220,"tl":201040,"ts":524135,"tr":893262,"te":2648213,"ti":2353666,"th":4782590,"v ":29410,"tb":79573,"tc":66980,"ta":1180046,"su":341240,"ss":646227,"st":2324937,"sy":102577,"sw":27118,"sl":145841,"sk":108573,"sn":41499,"sm":113246,"sp":316999,"so":700411,"sc":280155,"se":1450470,"sh":709009,"si":1101025,"u ":116514,"sa":232414,"sb":34898,"rr":265014,"rs":761380,"rt":886247,"ru":271327,"rv":158410,"rw":40021,"ry":553810,"rp":87486,"ro":1594919,"rn":650417,"rm":435749,"rl":279627,"rk":261286,"ri":2008460,"rh":32041,"rg":276649,"rf":65494,"re":2611877,"rd":466156,"rc":300833,"rb":95933,"ra":1639206,"t ":3468815,"qu":157005,"s ":7219728,"pt":158295,"pu":265747,"pp":173102,"pr":683558,"ps":94491,"zi":58211,"ze":102677,"za":82368,"zo":27033,"ye":187250,"yc":42157,"yd":28943,"ya":95795,"yt":30002,"ys":153061,"yr":36606,"yp":60686,"yo":54105,"yn":59856,"ym":80995,"yl":75727,"yi":42920,"一":42790," Ga":73074," Ge":133297," Fo":105500," Fr":140138," Fi":76403," Fl":37253," Ha":131725," He":218212," Go":72637," Gr":129383," Gu":47617," Gi":28529," Hu":45416," Ho":126780," II":28541," Hi":92512," Je":52048," Ja":154158," Ir":53197," Is":59983," It":339895," In":252514," Ka":73450," Ke":49770," Ki":73234," Jo":100944," Ju":116938," La":145853," Le":125799," Li":123371," Ko":45184," Ma":403673," Mi":150289," Me":129337," Lo":122023," Lu":31698," Ne":176925," Na":142533," Ni":43042," Mo":144948," Mu":70352," A ":81995," Ap":56659," Am":166952," An":116734," Al":126650," Ai":30214," Af":42668," Ac":48855," Ad":33141," Ba":185398," Au":138482," At":35966," As":93159," Ar":131824," Be":128479," Bi":57574," Bl":40368," Bo":111176," Br":186728," Bu":74408," Ca":300905," Ce":60031," Ci":62055," Ch":257143," Cl":59706," Cr":64523," Co":423247," Cu":43407," Da":103400," Di":111836," De":173171," Dr":31720," Do":63383," Du":52940," Ea":55250," Ed":37865," El":43749," En":125075," Em":27148," Eu":42050," Fe":82594," Fa":58222," Wo":87042," Wi":98480," Wh":33053," We":93550," Wa":135146," Yo":71619," a ":1606658," Or":50217," Po":106745," Pl":35261," Pi":44206," Ph":42887," Pe":108520," Pa":218936," No":179642," Ol":47101," On":40029," Oc":53053," Ra":100251," Qu":37258," Ro":163370," Re":185467," Ri":92437," Pr":164608," Pu":36045," Sy":29848," Sw":37042," Su":96395," St":289114," Ta":73883," Th":847432," Ti":40916," Te":98615," Tr":83585," US":33813," To":92734," Ru":52977," Sa":154637," Sh":111397," Si":87885," Sc":118544," Se":176317," So":160018," Sp":77680," Va":46297," Ve":37650," Vi":81481," Tu":35914," Un":220424," im":47481," in":2124350," is":1535535," it":174191," ki":30820," jo":31616," ju":31891," ha":241917," he":193790," gi":28626," gr":121630," go":63783," gu":29424," hi":208416," ho":110982," hu":32878," ne":121038," na":174050," mu":147201," mo":231837," on":505803," of":2275616," nu":46047," no":232717," le":140717," li":204448," la":217455," kn":129366," km":28301," me":218768," mi":119142," ma":335287," lo":197009," af":58077," ag":45613," ab":70854," ac":136746," ad":69502," am":31088," an":1904322," ap":69597," ai":40857," al":285645," au":57146," ar":316462," at":262226," as":400242," ba":223899," bi":52056," be":361253," bo":296458," bl":27386," by":488337," bu":118496," br":85339," ca":237167," es":49439," en":130005," em":28882," el":86812," fe":73621," fa":169421," ev":56247," ex":119951," fu":49639," fr":341564," fo":746598," fl":49512," fi":320292," ge":92358," ga":85124," cl":107340," co":825577," cr":113302," ce":122706," ch":161778," ci":78624," da":76510," cu":74858," do":65310," dr":48010," de":365522," di":285811," ed":47217," ea":97123," du":78958," ye":60017," ru":58222," sa":63365," se":417230," sc":118543," si":220940," sh":125764," sn":27188," sm":42001," sp":191590," so":270927," qu":28948," ra":127133," re":580460," ri":81201," ro":102378," pu":110299," pr":488102," s ":285424," ot":53672," ou":36365," ov":43302," op":81547," or":335005," ow":28576," pe":132888," pa":238807," pl":194868," po":256546," pi":40433," ph":50321," wa":774642," we":177231," wr":78668," wo":147411," wi":321826," wh":387560," va":54528," ve":51196," vo":40129," vi":105281," ty":26790," tw":69959," us":113343," up":41302," un":149473," ta":60356," sy":54778," st":354336," su":191136," tr":168050," to":791933," th":3629714," ti":77241," te":209958,"Feb":38336,"Eur":34468,"Eng":96944,"Ger":62713,"Geo":33733,"Gen":26192,"Fra":61982,"Fre":54323,"For":54660,"II ":31692,"His":27636,"Hig":26895,"He ":118997,"Her":26981,"Har":34188,"Gre":62525,"Gra":36710,"Int":46918,"Ind":83336,"In ":64014,"Hou":31497,"Arm":26565,"Apr":40277,"Ass":43311,"Aus":73786,"Aug":44651,"Bar":42165,"Afr":28749,"Ame":138898,"Cal":50030,"Car":57990,"Cat":29319,"Can":84273,"Ber":26471,"Bra":33017,"Bro":31697,"Bri":93113,"Dec":45892,"Chr":35874,"Chi":73091,"Cit":38778,"Cen":43272,"Cha":89919,"Cor":34743,"Com":89536,"Col":68792,"Con":74902,"Cou":101975,"Eas":34019,"Dis":41703,"Nat":69269,"New":118080,"Nov":49796,"Nor":97601,"Oct":42904,"Oly":28739,"Pla":26686,"Per":28472,"Pen":27579,"Par":98820,"Pro":69398,"Pri":36739,"Pre":41911,"Pol":34263,"Ita":30913,"Isl":36514,"It ":288844,"Jap":37112,"Jan":53046,"Joh":48960,"Jul":46542,"Jun":47295,"Kin":45793,"Lea":42060,"Lan":29809,"Lin":26493,"Lon":35381,"Man":49459,"Mar":140591,"May":50344,"Mon":37435,"Mic":30191,"Min":35884,"Mus":28824,"Wor":61067,"Wil":46697,"Wes":53739,"War":53697,"Wal":33670,"Yor":47584,"Str":31655,"Sta":170783,"Ste":27246,"She":47105,"Sha":26436,"Ser":32863,"Sep":45219,"Spa":35346,"Sou":76587,"Rus":29193,"Sco":37321,"Sch":55225,"San":42456,"Riv":36843,"Rep":36734,"Rom":34841,"Uni":206527,"The":741033,"Thi":55958,"Tra":32863,"bit":27600,"bil":34265,"bin":32338,"bly":27084,"ble":108674,"bli":153281,"boo":43455,"bor":211694,"bot":33326,"bou":74106,"be ":86424,"ban":103832,"bal":111539,"bac":27179,"bas":116354,"bee":53796,"bec":41647,"ber":372832,"bel":55314,"bes":39082,"bet":76601,"bia":36531,"ca ":71415,"car":71337,"cas":54417,"cat":224104,"can":218593,"cap":33144,"cad":26654,"cam":39022,"cal":268647,"ce ":489631,"bri":35233,"bro":46239,"bra":53964,"bre":38618,"bru":40872,"bur":52830,"bum":61942,"bui":38535,"but":82424,"bus":32037,"by ":517575,"am ":158006,"ake":79597,"aki":36034,"ajo":27606,"al ":1032287,"ail":109529,"ain":263108,"air":75684,"agu":57502,"ago":34773,"anu":71865,"any":116307,"ano":50033,"ann":83296,"ant":217929,"ans":132995,"ane":74984,"ang":153929,"ani":212965,"ank":54539,"ana":169898,"anc":203858,"and":1922995,"amm":33827,"amo":43526,"amp":91990,"ams":33255,"ami":135603,"ame":296400,"amb":32800,"ama":60146,"aly":26784,"alt":66067,"als":156873,"alo":44306,"all":466989,"ali":291050,"ale":104464,"ala":82861,"alb":63960,"an ":1345264,"aba":26164,"abe":27729,"abi":43800,"abl":95465,"abo":73249,"ae ":67161,"ad ":136134,"aft":64376,"aff":27842,"ai ":26417,"aga":52212,"age":211927,"ael":26991,"ado":32963,"adi":130468,"ade":130763,"ack":93180,"aci":48413,"ach":115898,"ace":139311,"acc":38863,"ada":60436,"act":176914,"azi":34919,"ays":38857,"aye":87163,"at ":514237,"arg":65842,"are":299717,"ard":199526,"arc":122863,"ara":130356,"aro":62443,"arn":35663,"arm":40703,"arl":119272,"ark":89799,"ari":222546,"arr":82306,"ars":76523,"art":349130,"ary":226746,"asi":41320,"ash":45659,"ase":201692,"aso":57197,"ask":29635,"ar ":287761,"apa":53686,"ape":43062,"aph":44106,"app":80291,"as ":1288188,"ava":37165,"aut":60210,"avi":60667,"ave":94995,"ay ":243951,"awa":44906,"ata":58365,"ast":279617,"ass":158911,"atr":35326,"ato":75056,"ate":773247,"ati":841381,"ath":123599,"aw ":33098,"att":81527,"atu":81250,"aus":27235,"jec":34085,"jor":28420,"itl":30516,"ito":44622,"itu":81601,"itt":81137,"its":101906,"ity":314774,"ism":33565,"isl":34570,"iso":38810,"iss":83951,"ist":561559,"ita":125707,"ite":286513,"ith":287933,"iti":312557,"ium":39769,"iva":48200,"ix ":27697,"ivi":104219,"ive":414501,"is ":1834908,"ion":1320795,"ior":32367,"iou":46819,"ipa":66227,"ir ":124191,"irs":127746,"isi":109243,"ish":348637,"ise":68787,"isc":59122,"ire":165519,"ird":33893,"irc":26906,"it ":178657,"ize":56700,"iza":46289,"kin":80796,"ker":49025,"ket":62341,"key":30606,"ke ":83682,"ks ":77920,"kno":134011,"ka ":41749,"ha ":26541,"ham":91101,"han":137177,"hai":28881,"hal":42170,"hav":58023,"har":141850,"has":120534,"hat":219394,"had":38461,"he ":3774627,"hel":72126,"hei":75673,"hed":104387,"hea":95197,"hey":47349,"hes":81158,"her":373103,"heo":27238,"hen":82591,"hem":53106,"hig":49277,"hie":27426,"hic":185273,"hip":92853,"hin":140164,"hil":96538,"his":233818,"hit":44255,"hir":71717,"hn ":39410,"ho ":150031,"go ":38352,"gle":57486,"gli":56142,"gn ":28550,"gla":57467,"gne":39182,"gs ":62491,"gov":34838,"gro":78380,"gra":130928,"gre":50553,"gui":30022,"gua":43805,"gue":71236,"gy ":58853,"gus":48388,"iam":54702,"ial":191889,"ian":404233,"iat":85021,"ic ":400287,"ibl":26901,"ibu":29254,"id ":86767,"ibe":47065,"ia ":367003,"iet":50527,"iel":55976,"ien":102314,"ier":62637,"ies":307456,"ied":65179,"ifo":38546,"iff":31829,"ife":37449,"ifi":68240,"ics":90231,"ict":157327,"icu":32368,"ico":35459,"ick":77376,"ici":172909,"ich":222330,"ice":143792,"ie ":67990,"ica":469571,"idi":26260,"ide":209151,"ida":78657,"il ":176048,"im ":36451,"iga":31696,"igh":217420,"igi":75927,"ign":93132,"imp":47812,"ime":121995,"imi":41657,"ip ":77179,"inc":218718,"ind":114260,"ina":201512,"inn":50515,"ino":47946,"int":230917,"ins":127823,"inf":38004,"ine":357110,"ing":1178957,"ini":149339,"inv":30106,"ike":31715,"ila":50860,"in ":2015240,"ilo":31568,"ill":253039,"ilm":70454,"ili":107918,"ild":57179,"ile":87652,"ima":76627,"io ":93766,"ilw":27998,"ily":104441,"ilt":33467,"how":39080,"hol":72031,"hom":45604,"hon":34484,"hos":48768,"hou":71727,"hoo":99559,"hor":109022,"hro":61086,"hre":44973,"hri":42093,"ht ":111042,"hy ":31885,"hum":60594,"hur":46747,"ffe":52374,"ffi":65666,"fes":54841,"fer":101678,"fea":31829,"fam":97344,"fac":45076,"ff ":28943,"fe ":33600,"ext":42471,"exa":34847,"ews":32501,"exp":39847,"exi":35172,"eta":52929,"ete":109151,"eti":91823,"eth":57374,"esp":36940,"est":416254,"ess":250577,"etr":52595,"ett":68152,"etw":81955,"ety":27383,"ew ":152274,"eve":202090,"evi":96051,"ex ":29419,"ey ":184413,"epe":34508,"er ":1640997,"epa":43995,"eor":48705,"eop":40243,"es ":1236398,"ept":72575,"epu":31828,"epr":49381,"erl":52014,"eri":370698,"erg":54150,"ere":274035,"erf":42571,"erc":40232,"era":262856,"erb":34102,"et ":183510,"equ":41108,"esi":112993,"esc":38776,"ese":189976,"ery":53481,"erv":128955,"err":75913,"ert":119791,"ers":454490,"ern":296552,"erm":132045,"ero":60052,"en ":515700,"ela":94480,"eld":88206,"ele":239396,"eli":77371,"ell":173610,"elo":78428,"els":41004,"ely":58392,"emb":232607,"ema":57740,"eme":113005,"emo":49872,"emi":71126,"emp":43662,"ene":128335,"eng":56090,"ena":53361,"end":141343,"enc":194004,"eno":33355,"enn":56995,"eni":62631,"enu":41299,"ens":118987,"ent":917089,"ege":50969,"egi":112559,"ek ":35390,"eir":69644,"ein":59892,"eig":51339,"el ":184102,"em ":65326,"gis":39618,"gin":123942,"gio":71590,"gic":30033,"gia":32230,"ght":158712,"gen":144495,"ger":93467,"ges":68825,"gh ":114066,"ged":27606,"gdo":27360,"ge ":289041,"gas":27663,"gar":46583,"gat":28511,"gam":47831,"gal":29045,"gan":93331,"fte":73646,"ful":30197,"ft ":41960,"fre":30903,"fri":38796,"fro":298934,"fou":123314,"for":682161,"foo":52751,"fol":40754,"fic":131579,"fie":53477,"fil":79707,"fin":56926,"fir":119031,"da ":86635,"de ":228034,"dal":30100,"dae":42405,"dat":50729,"dar":41780,"dan":30719,"day":44277,"cul":69156,"ctu":58584,"cts":42767,"ctr":37744,"cto":137353,"cti":262008,"cte":91935,"cy ":49469,"cus":28281,"cur":73843,"cla":60174,"cle":56340,"clu":89562,"clo":27081,"co ":51753,"coa":30142,"con":327550,"col":81553,"com":369257,"cor":122655,"cov":34759,"cot":39747,"cou":101684,"cs ":92018,"ct ":169935,"cre":89454,"cra":42776,"cri":68130,"cro":41753,"cco":29437,"cce":37436,"cea":26454,"ch ":441284,"cer":71180,"ces":153825,"cen":136988,"cem":49322,"cel":32556,"ced":67120,"cha":166560,"chu":28019,"cia":202890,"ck ":147449,"cie":145817,"che":138582,"chi":112580,"cho":124613,"chn":32667,"cil":38009,"cis":28214,"cit":63842,"cin":40251,"cip":75068,"cke":66684,"ed ":1971122,"ebr":59143,"eac":57373,"eag":49669,"ead":94691,"ean":92098,"eal":72020,"eam":67571,"ear":238611,"eas":225475,"eat":190659,"ea ":97410,"efo":27942,"efe":67003,"ega":48293,"eek":39188,"een":176170,"eed":37937,"eer":50775,"eet":39266,"edi":120330,"ede":54090,"edu":36419,"ech":57443,"eci":121874,"ece":81981,"eca":34537,"ee ":109231,"ecu":34646,"ect":304913,"eco":150097,"dy ":58601,"dur":62131,"don":63380,"dom":45245,"ds ":164811,"duc":125723,"dra":35355,"dre":38221,"dge":45767,"dic":54056,"dia":144125,"der":278249,"des":154773,"dev":51103,"dea":34588,"ded":141601,"def":26810,"del":51924,"den":154840,"dem":42344,"dep":50458,"dle":29935,"do ":34660,"div":36865,"din":170282,"dio":57931,"dir":47066,"dis":162456,"dit":74391,"die":65117,"dif":26218,"rga":62829,"ri ":41893,"rgi":33226,"rge":101036,"ret":62986,"res":357790,"rev":42186,"rfo":28887,"rds":61017,"rg ":35933,"rea":244476,"ree":171216,"ref":62077,"rec":149809,"red":206208,"reg":86193,"rem":55031,"ren":193932,"rel":146404,"rep":68957,"rdi":57659,"rde":80880,"re ":634483,"rch":164110,"rce":59020,"rd ":218199,"rap":56827,"rar":34987,"ras":43044,"rat":260279,"rai":83507,"rag":34608,"ran":265077,"ram":80300,"ral":264928,"rab":26255,"rad":103893,"rac":129420,"rpo":42976,"rs ":371607,"ros":58364,"rot":58999,"rom":337721,"ron":118148,"roo":26751,"rop":109260,"rou":191436,"rov":99028,"row":41881,"roa":55237,"rod":87645,"roc":66994,"rol":65046,"rof":62861,"rog":49055,"rnm":34849,"rna":118629,"rne":61259,"rni":62594,"ro ":47142,"rma":145311,"rme":117007,"rmi":32749,"rly":65353,"rli":51474,"rld":75350,"rle":27840,"rn ":330287,"rks":33396,"rke":46561,"rm ":73619,"rio":79616,"rit":248352,"ris":163763,"riv":53880,"rig":116494,"ril":65623,"rin":255678,"rim":52709,"ria":159780,"rib":60454,"ric":377068,"rid":60669,"rie":181610,"rk ":140894,"ruc":35769,"run":39647,"rum":30319,"rus":32542,"rva":27044,"rvi":62552,"rve":65133,"ry ":509562,"rsi":95891,"rso":35794,"rsh":28300,"rse":60300,"rta":48186,"rst":132558,"rtm":26912,"rte":66419,"rth":190521,"rti":127728,"rua":38894,"rts":59598,"rty":55015,"rt ":230530,"rro":33066,"rri":69478,"rre":97252,"rra":30166,"san":29214,"sha":36063,"sho":85008,"she":117943,"shi":166608,"sid":90354,"sic":111518,"sia":80711,"sit":152904,"sis":68592,"sin":187018,"sio":187210,"sim":26375,"sig":69827,"scr":41549,"se ":323378,"sci":38993,"sch":76599,"sco":58271,"sev":36211,"ser":204330,"ses":72706,"set":56749,"sh ":261718,"sea":111064,"sed":251715,"sec":65010,"sen":110589,"sem":35416,"sel":52566,"spo":60807,"spe":154402,"spi":31807,"spa":33041,"sou":97501,"sol":40004,"som":46889,"son":213206,"sor":49003,"soc":65715,"st ":788491,"ss ":186657,"sla":91489,"sm ":27910,"sna":26235,"so ":118506,"sma":61408,"sys":38243,"sse":101933,"ssa":44414,"sso":82323,"ssi":178639,"ste":308026,"sta":321451,"sto":159447,"sti":202749,"stl":26896,"stu":53381,"str":353511,"sts":54520,"sub":56163,"suc":48817,"sul":26601,"sup":33543,"sus":37393,"sur":52388,"tai":93008,"tak":29757,"tal":158887,"tag":31026,"tab":60969,"tba":73096,"tat":294451,"tar":154358,"tan":148320,"tch":55824,"te ":378459,"ta ":92521,"pe ":62967,"par":214065,"pat":33109,"pac":31182,"pal":68434,"pai":39322,"pan":124559,"phe":35230,"pho":33690,"phi":46768,"pea":63524,"pec":127586,"ped":38131,"pen":76105,"peo":27260,"per":254210,"pet":53680,"pla":213772,"pli":36447,"ple":105992,"plo":26352,"phy":39410,"pic":63629,"pin":46550,"pio":41668,"pit":34080,"por":147134,"pop":69835,"pos":94042,"pon":41677,"pol":103325,"ps ":57068,"ppo":34542,"ppe":56579,"pub":101611,"pte":60939,"pti":34138,"pri":140547,"pre":169814,"pro":351896,"pur":26601,"put":38175,"pul":69919,"qua":54197,"que":60004,"qui":38739,"ra ":108610,"ngi":45147,"ngl":153748,"ngu":51932,"ngt":37246,"ngs":65275,"ni ":33062,"nge":128219,"nga":40074,"ngd":28313,"nel":35766,"nen":30704,"ner":137621,"net":62476,"nes":167423,"ng ":1115424,"nea":48684,"ned":116572,"nee":30138,"nfo":27425,"ney":30714,"new":36703,"nct":31942,"nco":45249,"nci":91672,"ncl":65296,"nce":360792,"nch":88213,"ne ":474805,"ndu":39629,"ndr":33212,"nds":71457,"ndo":66887,"ndi":147835,"nde":264119,"nda":82618,"ncy":32567,"nal":361592,"nam":140852,"nan":41261,"nar":46225,"nad":72347,"nag":33449,"nai":32626,"nd ":1932876,"nat":199905,"na ":142173,"ny ":146188,"nve":33106,"num":41405,"nus":46108,"nua":61979,"nty":82414,"nto":84185,"ntu":58438,"nts":118194,"ntr":155494,"nti":190207,"nth":52495,"ntl":47080,"nta":166897,"nte":282517,"nsu":48967,"nst":113470,"nse":49755,"nsh":53144,"nsi":90541,"nt ":574346,"ns ":352877,"nol":27739,"nom":44734,"non":40510,"not":75828,"nor":109034,"now":162156,"nov":41537,"nne":95671,"nna":27532,"nni":61016,"nme":53495,"nly":54407,"no ":47600,"nic":132031,"nia":137970,"nk ":44974,"niz":43878,"niv":81629,"nis":146601,"nit":215552,"nio":55312,"nin":128702,"ogr":72732,"ogi":44598,"ohn":44056,"ogy":47192,"oin":47463,"ok ":58105,"ol ":109628,"oce":39082,"oci":84252,"ock":92930,"oca":158968,"occ":29313,"ode":77618,"of ":2204484,"odu":81510,"oft":39655,"off":67447,"ofe":46203,"oad":59707,"od ":90721,"obe":66424,"ows":30274,"own":272385,"owi":30580,"ow ":104160,"oti":37576,"oth":132550,"ote":72682,"ott":42106,"oto":39358,"ost":134462,"ota":47221,"otb":59122,"osi":37738,"ose":115458,"oss":51862,"owe":69263,"ovi":113638,"ove":271311,"oug":99629,"oul":30838,"oun":383251,"oup":67359,"ous":158478,"our":185741,"out":265981,"opo":44332,"opi":26584,"opl":40248,"ope":150764,"oph":34738,"os ":57513,"opu":63831,"ool":97648,"ook":83743,"ood":64494,"or ":897485,"oot":80611,"ork":137621,"orl":77957,"orm":213505,"orn":233282,"oro":42745,"orp":39756,"orr":27698,"orc":40804,"ord":187024,"ore":161271,"org":83794,"ori":179394,"ort":324283,"ors":65851,"ory":92826,"ot ":75528,"ora":99397,"ola":48347,"old":74258,"on ":1693252,"oli":168296,"oll":152650,"ole":57272,"olo":130918,"olu":51650,"om ":355568,"ona":295881,"ond":125458,"onc":37498,"onf":27843,"one":226758,"ong":180887,"oni":86395,"onl":50368,"onn":32769,"ono":57151,"ons":362489,"ont":151478,"ony":30391,"oma":89149,"ome":169344,"omb":33063,"omi":82283,"omm":164677,"omp":213665,"omo":39559,"op ":54660,"la ":85626,"le ":464484,"lf ":36467,"lde":36942,"ldi":28865,"lab":36068,"lac":84352,"lag":63397,"lai":31387,"lan":374156,"lar":145329,"lat":207187,"las":100146,"law":29496,"lay":143764,"ld ":226258,"lbu":63795,"ls ":147230,"lon":87604,"lop":54959,"lor":52442,"loc":148176,"log":104122,"los":46581,"low":85754,"lth":30060,"lti":36200,"lud":61069,"lub":38036,"lue":31016,"lso":110778,"lt ":61158,"li ":29468,"ley":50488,"lev":69397,"les":177287,"let":59346,"ler":75366,"lem":45662,"len":64181,"leg":67495,"led":96172,"lec":118736,"lea":145235,"lls":32993,"llu":31038,"lly":155319,"lo ":32182,"lla":120635,"lle":228951,"lli":124524,"llo":71927,"lm ":62508,"ll ":345769,"lit":221318,"lis":254417,"lin":196823,"liz":27437,"liv":42501,"lic":149255,"lia":177899,"lig":44454,"lie":64153,"lif":67751,"ma ":56941,"mb ":40338,"mai":57333,"mad":33441,"mag":32391,"mar":102764,"mas":36713,"mal":90910,"man":305273,"mat":146584,"mbl":26607,"mbi":34922,"mbe":250295,"me ":304088,"med":139063,"mea":33106,"met":85213,"mes":108882,"mer":308315,"mem":71604,"men":353373,"lve":34044,"lum":34502,"lus":42702,"ly ":633235,"lwa":32178,"lym":34483,"mpi":86987,"mpe":67319,"mpo":59756,"mpl":65250,"mpu":32461,"ms ":108315,"mod":36169,"mon":129400,"mol":27569,"mov":30527,"mor":62243,"mos":67786,"mot":45357,"mou":50940,"mpa":78503,"my ":44622,"mus":79882,"mul":30159,"mun":115137,"min":162859,"mil":151172,"mis":47781,"mit":48071,"mic":78126,"mmu":68704,"mmi":37447,"mmo":49689,"mma":30077,"mme":66787,"zed":29181,"zat":36341,"yst":59562,"ysi":32556,"ys ":46066,"ype":26271,"yea":55238,"yed":50493,"yer":52194,"ymp":36974,"yin":35501,"wo ":61016,"wn ":223496,"ws ":47661,"wri":85490,"wor":132758,"wes":66741,"wer":121358,"wel":48031,"wed":28630,"wee":76113,"whe":65586,"whi":161339,"who":156434,"wit":254528,"win":75777,"way":79562,"war":108109,"was":721522,"via":29199,"vil":95978,"vin":109107,"vic":63884,"vid":74216,"vie":42322,"vis":102225,"vol":39542,"ver":416461,"ves":68869,"ven":139602,"vem":57889,"vel":128051,"ved":73656,"ve ":294814,"val":55056,"van":50667,"var":47368,"vat":46841,"usi":127657,"use":190347,"ust":198208,"uss":43860,"uth":195687,"uti":76065,"ute":88005,"us ":269807,"ut ":151960,"ura":72791,"urc":49405,"ure":175159,"urg":39826,"uri":124329,"urn":72450,"uro":43771,"urr":69334,"urs":30553,"urt":42019,"ury":47606,"ur ":96286,"upp":28633,"umm":33986,"uma":31954,"umb":94738,"ume":46174,"uly":40527,"unt":202586,"uni":169488,"unc":71870,"und":253741,"ung":45523,"une":67239,"up ":99562,"um ":146010,"ult":82523,"ull":33469,"ule":32343,"ula":137681,"un ":32424,"uil":50329,"uis":33184,"uit":37221,"ul ":41234,"ugh":110128,"ugu":55721,"uct":69612,"ude":66964,"udi":73540,"uca":35169,"ue ":120838,"uce":56991,"uch":53880,"ues":42067,"uen":42045,"ub ":41295,"uat":42205,"uar":119614,"ual":88610,"ubl":124785,"uag":32210,"typ":31249,"ty ":506251,"tur":219279,"tut":36373,"tua":41791,"tud":63876,"two":81070,"twe":71315,"ts ":465295,"tre":115840,"tra":308513,"tri":221592,"tru":53183,"tro":123988,"try":66671,"tta":29674,"tte":122526,"tti":34349,"ttl":41524,"tme":35635,"to ":731436,"tly":63791,"tob":48945,"tow":72085,"tom":29936,"ton":150493,"tor":304450,"too":27044,"top":29122,"til":53505,"tie":69965,"tit":108778,"tis":139521,"tin":255051,"tim":81225,"tio":971575,"thu":55673,"tia":79923,"tic":276929,"tiv":181662,"tla":29767,"tle":95143,"tem":124758,"ten":154275,"tel":93554,"tee":36958,"tea":71475,"tec":49230,"ted":637757,"th ":648546,"tes":193644,"ter":809390,"ti ":33930,"tho":134583,"thr":87801,"the":3415279,"thi":110969,"tha":240340},"n_words":[260942223,308553243,224934017],"name":"en","type":"latin","flags":["ascii"]} \ No newline at end of file
diff --git a/contrib/languages-data/es.json b/contrib/languages-data/es.json
new file mode 100644
index 0000000..3c87107
--- /dev/null
+++ b/contrib/languages-data/es.json
@@ -0,0 +1 @@
+{"freq":{"D":116547,"E":296654,"F":128129,"G":121338,"A":269964,"B":167407,"C":324676,"L":239740,"M":232523,"N":101570,"O":64162,"H":81475,"I":139475,"J":79180,"K":34991,"U":62793,"T":132122,"W":35590,"V":93781,"Q":9803,"P":223906,"S":275410,"R":140692,"Y":15953,"X":27052,"Z":15636,"f":602083,"g":868874,"d":3877179,"e":8874725,"b":812441,"c":2912236,"a":7916083,"n":5177793,"o":5444424,"l":3848407,"m":1698678,"j":201978,"k":118503,"h":478078,"i":4816050,"w":47097,"v":521828,"u":2624688,"t":3108332,"s":4177405,"r":4307485,"q":288923,"p":1539196,"z":257162,"y":639511,"x":118459,"²":9268,"Á":8848,"í":298098,"é":232623,"è":9587,"á":241717,"ú":87005,"ó":542725,"ñ":141698," l":925296," m":342591," n":169035," o":224265," h":153575," i":177440," j":57642," k":20882," d":2112466," e":1686012," f":346033," g":128207," a":615947," b":110799," c":841620," y":413167," z":8894," u":509709," t":263819," v":109427," q":191724," p":764411," s":537300," r":229895," J":77814," K":33034," H":77965," I":96403," N":91185," O":55452," L":232579," M":222195," B":158843," C":308303," A":248654," F":121663," G":115612," D":105557," E":280970," Z":14852," Y":15119," X":18791," S":256762," R":131715," Q":9140," P":211939," W":33301," V":79914," U":57665," T":122063," á":26258," é":11575," ú":11089," Á":8823,"A ":15731,"Da":13474,"Cu":13670,"Cl":11053,"Co":80802,"Cr":14045,"Ce":15444,"Ch":44811,"Ci":15164,"Ed":7168,"Du":7509,"Do":14187,"De":29603,"Di":25181,"Fe":19702,"Fa":9899,"Eu":12005,"Es":99956,"En":28786,"El":96353,"Ge":16269,"Ga":21659,"I ":25126,"Fu":23434,"Fr":30636,"Fo":10911,"Fi":10813,"C ":12178,"Au":18315,"Ar":39333,"At":7482,"As":13866,"D ":7805,"Ba":43405,"Ac":8941,"Am":14049,"An":29222,"Ai":9057,"Al":51515,"Bu":16123,"Br":22960,"Ca":88236,"E ":7609,"Bi":10052,"Be":24042,"Bo":25983,"Le":25199,"Li":24708,"La":111930,"Lu":14073,"Lo":44117,"Me":30598,"Mi":26023,"Ma":85855,"Mu":18246,"Mo":36242,"Ni":10392,"Ne":11654,"Na":26109,"Nu":10474,"No":24130,"Gi":8606,"Gr":21868,"Go":14614,"Gu":21771,"Ha":19115,"He":16228,"II":18780,"Hi":11870,"Ho":13033,"Hu":9391,"In":34662,"Is":11404,"Ja":18733,"Je":8830,"Jo":22070,"Ju":22045,"Ka":8249,"Un":40236,"VI":7056,"Tu":7329,"Tr":17446,"To":19269,"Th":18313,"Ti":12341,"Te":21087,"Ta":17232,"V ":8314,"St":15377,"Su":36610,"Wi":9415,"Wa":8415,"Vi":28361,"X ":8775,"Va":20820,"Ve":16615,"Mé":10810,"Pu":11178,"Pr":36130,"S ":9671,"Pe":30523,"Pa":58336,"Pl":10964,"Po":27786,"Pi":20810,"Or":15403,"Se":51296,"Sc":7275,"Si":21213,"So":22342,"Ru":9143,"Sa":70712,"Re":46909,"Ri":14206,"Ro":31238,"Qu":8096,"Ra":15098,"b ":16974,"a ":2807777,"i ":67723,"cá":7737,"ge":97443,"ga":124206,"fl":18249,"fi":107495,"fr":77696,"fu":123473,"fo":79998,"có":10267,"he":77025,"ha":138548,"gn":24260,"cé":16005,"gl":50338,"gi":128766,"gh":9410,"gu":117756,"gr":97493,"cí":12742,"go":114696,"du":66523,"g ":31294,"ea":126973,"eb":49402,"ec":261065,"ed":147913,"de":2140534,"di":323631,"dm":7851,"do":556174,"dr":51266,"ex":62245,"eu":25398,"añ":100385,"ev":65967,"ey":26516,"ez":45865,"fa":71295,"h ":31038,"fe":72710,"eg":169863,"ef":34480,"ee":18956,"el":821140,"ej":32037,"ei":38528,"ep":114508,"eo":63152,"en":1446857,"em":183658,"et":133793,"es":1262459,"er":848036,"eq":18834,"aí":17948,"ca":518269,"e ":2816707,"bs":7367,"br":179247,"bu":47229,"bo":65362,"bl":117456,"bi":135239,"be":67612,"da":466302,"f ":15176,"cu":158454,"ct":134054,"cr":79888,"co":691321,"ck":21843,"cl":46854,"ci":780154,"ch":115875,"ce":230782,"cc":44533,"c ":23067,"az":30061,"ay":59587,"ba":136047,"d ":184141,"at":211115,"as":567489,"ar":691307,"aq":9531,"av":48388,"au":72787,"ak":11907,"al":659633,"ai":51423,"aj":50703,"ao":8102,"ap":77232,"am":297433,"an":805183,"ac":374463,"ad":631439,"ab":144055,"ag":85149,"ah":13118,"ae":43681,"af":21278,"nu":35537,"nt":779819,"ns":140899,"nq":11336,"no":368063,"nn":24107,"nz":31492,"ny":8769,"nv":18820,"oe":24400,"of":39013,"oc":180019,"od":93463,"oa":18159,"ob":123195,"om":334368,"on":689571,"ol":230551,"oi":24324,"oj":12159,"og":57063,"oh":10801,"m²":9191,"ot":93533,"os":699509,"ov":85328,"ou":52929,"op":77349,"oo":15677,"or":691238,"r ":505089,"ox":9374,"ow":10366,"oz":11222,"oy":19941,"lá":14567,"pe":253260,"pa":326975,"pl":67808,"lé":20848,"po":370364,"ph":12436,"pi":101456,"lo":395718,"lm":47465,"ll":193235,"ls":13390,"lp":10103,"lv":15446,"lu":77232,"lt":59369,"ly":9017,"o ":1816298,"ma":323604,"mb":142619,"me":329898,"mi":236057,"mm":8572,"ié":32355,"mp":132049,"mo":207044,"mu":126184,"ió":324878,"p ":13048,"na":683565,"nc":291584,"nd":277346,"ne":311456,"nf":32397,"ng":105777,"ni":309689,"nj":11352,"nk":8262,"nm":8232,"ju":57258,"fí":9213,"jo":44644,"ki":14501,"ke":14800,"ka":12118,"m ":52750,"gó":7060,"gí":14289,"gé":21878,"km":14607,"li":353994,"le":344098,"ld":22533,"lg":19778,"lf":10189,"la":1037960,"lc":15579,"lb":23157,"gú":7200,"n ":1645057,"hr":7436,"dí":25020,"ht":8163,"hu":24887,"hi":87814,"ho":64406,"dé":12258,"id":350652,"ic":527484,"ib":58759,"ia":458387,"ig":130847,"if":53944,"ie":349344,"k ":32633,"ir":119098,"is":363824,"it":317793,"iu":42028,"eñ":26834,"iv":114899,"eó":10608,"ij":12968,"ik":8874,"il":242265,"im":150543,"in":518205,"io":362002,"ip":90246,"je":41244,"iz":77284,"l ":1070460,"ja":41449,"xi":40771,"té":18357,"xp":12413,"tí":29392,"tó":61297,"xt":20556,"só":7077,"z ":55163,"tá":43968,"sé":9023,"sí":13754,"ró":22130,"y ":494983,"wa":13090,"ré":8377,"vi":184322,"rí":60498,"vo":65147,"uz":11349,"uy":29308,"ux":7768,"uv":14743,"rá":28876,"ve":131509,"va":112048,"x ":22885,"ui":101034,"uj":10219,"ul":130016,"ue":529325,"ug":37794,"ur":196055,"us":135539,"ut":75455,"um":62826,"un":688515,"uo":9672,"up":46545,"tu":178275,"tt":18266,"pó":8915,"ub":69875,"ua":157161,"ud":82055,"uc":86063,"w ":7730,"pú":12838,"to":492464,"tl":11756,"ts":11628,"tr":358077,"te":662982,"ti":404422,"th":33828,"tb":14576,"ta":636505,"su":163099,"ss":27514,"st":570360,"sl":23077,"sk":8256,"sm":35452,"sp":146933,"so":177522,"sd":21641,"sc":98040,"se":328771,"sh":16915,"si":316726,"rz":18465,"u ":97650,"nú":7450,"sa":193871,"rr":132506,"rs":76964,"rt":263306,"ru":82530,"rv":23971,"nó":12071,"ry":14114,"ní":12258,"rq":18301,"rp":16192,"ro":473902,"rn":71986,"né":9922,"rm":118903,"rl":28428,"rk":11610,"ri":592813,"rg":83834,"rf":13796,"ná":7351,"re":689451,"rd":86475,"rc":86488,"rb":27212,"ra":734384,"t ":94930,"mú":13777,"mó":11660,"qu":286748,"mí":7526,"mé":17124,"má":63634,"s ":1974557,"pt":30720,"pu":78951,"ló":19506,"lí":47975,"pr":228655,"ps":9053,"zó":8970,"zi":7657,"za":111482,"zu":8538,"zo":36191,"ye":23683,"ya":28514,"uí":10738,"yo":35768,"ué":9949,"² ":9260,"án":62336,"ál":20287,"áf":8040,"ác":15525,"ár":16431,"át":16282,"ás":49088,"á ":27321,"óg":11484,"ód":8472,"ór":13833,"ón":382883,"óm":11589,"ól":14360,"ó ":71778,"ña":56328,"ño":76416,"ín":33019,"ím":10418,"ío":16786,"ít":28434,"ís":34065,"íf":8702,"íc":18334,"íd":7650,"ía":105050,"í ":14487,"él":11392,"én":55163,"és":52145,"ét":12035,"ér":34634,"éx":11216,"éc":11450,"é ":16494,"ún":21741,"úl":7364,"út":8499,"ús":12464,"úb":12720,"ú ":7505,"一":7134," Ga":21582," Ge":16178," Fo":10836," Fu":23266," Fr":30592," Fi":10737," Ha":19064," He":16181," Go":14559," Gr":21740," Gu":21689," Gi":8540," Hu":9382," Ho":12969," Hi":11833," Je":8808," Ja":18701," Is":11343," In":34554," Ka":8187," Jo":22025," Ju":22012," La":111649," Le":25083," Li":24432," Ma":85482," Mi":25830," Me":30504," Lo":44008," Lu":14044," Ne":11540," Na":26019," Ni":10343," Mo":36161," Mu":18130," Am":14012," An":29155," Al":51409," Ai":9020," Ac":8911," Ba":43282," Au":18286," At":7457," As":13733," Ar":39211," Be":23973," Bi":9959," Bo":25824," Br":22869," Bu":16073," Ca":87586," Ce":15402," Ci":15105," Ch":44683," Cl":10942," Cr":13930," Co":80501," Cu":13492," Da":13336," Di":25027," De":29418," Do":13935," Du":7489," Ed":7143," El":96116," Es":98653," En":28605," Eu":11992," Fe":19655," Fa":9820," Wi":9336," Wa":8349," a ":151113," Or":15358," Po":27637," Pl":10897," Pi":20772," Pe":30452," Pa":58153," Nu":10454," No":24018," Ra":15024," Qu":7915," Ro":30970," Re":46794," Ri":14181," Pr":36030," Pu":11158," Mé":10807," Su":36537," St":14732," Ta":17163," Th":18237," Ti":12302," Te":20935," Tr":17351," To":19041," Ru":9128," Sa":70550," Si":21147," Sc":7137," Se":51093," So":22241," Va":20787," Ve":16562," Vi":28257," Tu":7181," Un":40136," ja":7912," im":16980," in":113728," is":9470," it":12073," ju":44169," ha":71347," he":17319," gr":47141," go":9321," gu":12156," id":8758," hi":30296," ho":20186," hu":9280," ni":9094," ne":10791," na":40693," mu":56375," mo":34821," oc":24698," of":15543," ob":20507," nu":10795," no":85354," le":29852," li":28752," la":626496," gé":18729," km":14107," me":61144," mi":48266," o ":71023," ma":85649," lu":16286," ll":24228," lo":186366," ag":19771," ab":21502," ac":45971," ad":18011," am":14490," an":45970," ap":24125," al":116261," au":26320," ar":40593," at":8359," as":24315," ba":52391," bi":11562," bo":13161," br":12518," ca":157850," e ":10258," er":14855," eq":8506," es":539820," en":614552," em":20859," ej":8137," el":384203," fe":21573," fa":54639," añ":28328," ex":38515," fu":116273," fr":61229," fo":35270," fl":10710," fi":33918," ge":20831," ga":13109," cl":15585," co":462664," cr":30029," ce":31950," ch":12983," ci":56484," da":13407," cu":60477," do":35749," de":1878742," di":146031," ed":14319," du":20922," té":10997," tí":7323," ru":8901," sa":21596," se":189308," si":104740," so":62800," qu":191524," mú":9473," ra":15504," re":170810," ro":18871," pu":38713," pr":177892," lí":9079," má":36795," ot":17424," or":52385," pe":108853," pa":137566," pl":26836," po":242173," pi":19178," y ":406137," va":23514," ve":32789," vo":9768," vi":36979," ub":13189," tu":8296," us":12351," ut":9609," un":466456," ta":42321," su":139495," tr":62181," to":30228," th":7488," ti":31564," te":60962," ár":9835," ál":11624,"Fer":7120,"Es ":20079,"Est":40499,"Esp":30688,"Eur":9266,"El ":89488,"En ":18934,"Gar":7699,"Fue":18788,"Fra":22586,"II ":12947,"Gue":7438,"Gra":11989,"Int":10608,"Ind":7667,"Arg":10516,"Bar":13123,"Ale":9873,"Alt":8713,"And":7070,"Ant":10595,"Cal":12289,"Cam":10802,"Cas":14393,"Car":17937,"Can":11389,"Ber":7187,"Chi":15815,"Cen":8021,"Cha":15858,"Cor":11229,"Com":17537,"Col":10868,"Con":24853,"Nac":11329,"Nue":8768,"Nor":14767,"Pla":8680,"Per":15205,"Par":20522,"Pro":17204,"Pri":7235,"Pre":7665,"Méx":8860,"Jos":9116,"Las":10322,"La ":78398,"Los":18518,"Med":8474,"Man":10902,"Mar":33709,"Mad":9566,"Mon":15534,"Mun":7799,"Su ":13104,"Sai":11675,"Sal":8578,"Se ":18407,"San":32494,"Rep":8544,"Val":13044,"Vil":8238,"Uni":27767,"The":12538,"bit":22599,"bio":8582,"bil":7159,"bo ":7079,"blo":8255,"ble":28350,"bli":27042,"bla":52392,"bol":20729,"bié":25416,"bor":11023,"be ":11171,"ban":26993,"bal":10177,"baj":17768,"bas":16282,"bar":17411,"ber":30080,"bia":11473,"bic":15037,"bie":16653,"ca ":145181,"car":45351,"cas":41067,"cat":14539,"can":94395,"cap":14546,"cac":16670,"cab":10529,"cad":51771,"cam":20670,"cal":48767,"ce ":35472,"bri":27454,"bro":15368,"bra":26651,"bre":105758,"bur":7145,"bum":12078,"am ":8792,"ajo":16624,"al ":274116,"aja":11561,"aje":17588,"ain":19507,"ais":7210,"agu":12491,"ago":28675,"anu":10190,"anz":18126,"ano":79509,"ann":7583,"ant":195066,"ans":17426,"ane":17755,"ang":19562,"ani":47254,"ana":70957,"anc":101262,"and":90623,"amo":12426,"amp":25988,"ami":57872,"ame":88516,"amb":41071,"ama":44395,"alu":9523,"alt":15595,"alo":13492,"alm":37560,"all":38280,"alg":10767,"ali":96797,"alc":7683,"ald":9255,"ale":82063,"ala":31798,"an ":104955,"aba":35637,"abe":14437,"abi":26160,"abl":18696,"abo":13547,"abr":22582,"ae ":28175,"aca":16009,"ad ":111062,"aga":11365,"ado":272062,"adr":17031,"adi":20049,"ade":31132,"adu":7585,"aco":10755,"aci":224093,"ach":10349,"ace":30766,"acc":10027,"ada":154737,"act":41870,"aza":10801,"ayo":25160,"aya":7832,"ba ":19878,"aqu":8935,"at ":7824,"arg":22636,"are":34407,"ard":31172,"arc":32672,"ara":95441,"aro":19229,"arn":7316,"arm":9270,"arl":13962,"ari":83070,"arq":11629,"arr":48459,"ars":9729,"art":119072,"asa":20743,"arz":11183,"asi":22274,"asc":10580,"ase":16265,"aso":13076,"ar ":100623,"apa":19232,"ape":7495,"api":11786,"apo":11441,"apr":7236,"as ":405615,"ava":9975,"aut":18769,"arí":13656,"avi":12074,"ave":14502,"ay ":13782,"ata":37974,"ast":48943,"atr":18116,"ato":31490,"ate":31650,"ati":40488,"atu":16694,"aun":7152,"aur":9494,"aus":7903,"jer":7889,"je ":13092,"jo ":28253,"ito":98982,"itu":57305,"eña":13429,"iud":31065,"ism":26830,"isl":10534,"iso":8290,"isp":12009,"ist":177840,"ita":90631,"ite":23119,"iti":14600,"ivo":27807,"eño":12432,"iva":29645,"ivi":28311,"ive":26878,"ipo":17187,"ipi":23278,"is ":44742,"ion":119701,"ior":15897,"ios":44527,"ipa":22022,"ir ":23370,"iri":18435,"isi":29089,"ise":10419,"isc":18799,"ire":21390,"ira":17066,"ja ":14889,"iza":56748,"km²":9037,"gía":13787,"gén":19973,"jul":9258,"jun":20993,"jue":11292,"ha ":23020,"ham":8200,"han":11281,"har":13249,"has":16766,"hab":28769,"hac":13453,"he ":23960,"her":16542,"hin":11792,"hil":16082,"his":17036,"ho ":17368,"go ":53446,"glo":15485,"gle":10695,"gla":7868,"gob":7093,"gni":7865,"gió":54574,"cés":10035,"glé":12339,"gon":9593,"gos":23034,"gru":17556,"gra":45297,"gri":10321,"gre":9359,"gui":14307,"gua":27766,"gue":27456,"gur":7400,"gun":20675,"iam":7087,"ial":56085,"ian":52472,"ias":29422,"iar":10235,"ic ":7674,"iac":10109,"iad":14476,"ibl":8505,"ibi":8164,"ibr":11003,"ibu":11957,"id ":12492,"ibe":11766,"ia ":260890,"iet":7563,"iel":8991,"iem":41853,"ien":137287,"ier":54165,"ies":19531,"ied":10903,"ieg":9534,"ifo":15425,"ife":9766,"ifi":21950,"ict":9719,"ico":124072,"ici":106469,"ich":18974,"ice":11245,"ie ":46477,"ica":217183,"ido":115195,"idi":11705,"ide":56950,"ida":143468,"il ":29082,"ige":12181,"iga":13218,"igl":16846,"igi":27180,"igu":25326,"igo":7750,"ign":13613,"imo":21293,"imp":19247,"ime":39882,"imi":27512,"inc":70215,"ind":21266,"ina":105516,"ino":52448,"int":60589,"ins":15850,"inf":13445,"ine":33717,"ing":45586,"ini":36645,"iod":9277,"inv":7546,"ila":17117,"in ":30692,"ilo":12791,"ill":61746,"ili":73439,"ile":24670,"ima":28669,"io ":149275,"día":18502,"hom":8228,"hos":7240,"hor":7815,"hum":8623,"fes":10458,"fer":25898,"fec":9155,"feb":8937,"fam":38352,"ext":18108,"ez ":27186,"exp":10760,"exi":15850,"eza":8196,"eta":35458,"ete":19089,"eti":13610,"esp":91539,"eso":20238,"est":182109,"aña":34948,"año":60064,"eto":13724,"etr":18301,"epú":7797,"eve":10842,"eva":20123,"evo":10163,"evi":21593,"erí":14511,"ey ":19031,"epe":9088,"er ":98611,"epa":47324,"eos":9539,"eor":7614,"eon":7647,"emá":15175,"es ":719197,"ept":16454,"epo":8010,"epr":9962,"elí":11498,"eri":85302,"erg":11010,"ere":38036,"erf":9607,"erc":31567,"erd":14302,"era":127455,"erb":7502,"et ":12616,"equ":18716,"aís":14636,"esi":46520,"esc":45995,"esd":20115,"ese":28957,"esa":79475,"erv":17764,"err":53481,"ert":62557,"ers":52410,"ern":44545,"erm":26731,"erp":8929,"ero":107826,"en ":609899,"ela":41071,"ele":43829,"eli":19783,"ell":40806,"elo":21079,"eo ":20655,"emb":41784,"ema":33787,"eme":18576,"emo":14924,"emi":17348,"emp":34242,"ene":94564,"eng":10179,"ena":41586,"end":51128,"enc":77537,"eno":37094,"eni":23612,"ens":46652,"ent":408861,"enz":9528,"ego":28111,"egi":66647,"egr":10013,"egu":24464,"ein":14332,"eja":7548,"el ":611266,"ejo":12599,"eje":7469,"gin":15815,"gio":12026,"gid":10945,"gic":10164,"gen":57105,"ger":8119,"ge ":11010,"gad":14992,"gas":7988,"gar":19699,"gal":9045,"gan":23906,"ga ":30329,"fue":90978,"fun":22112,"fra":50839,"fre":13616,"fri":7302,"for":59914,"flo":7804,"fic":57976,"fil":9492,"fin":19583,"da ":212873,"de ":1556339,"dad":141389,"dal":9998,"dae":14141,"das":36381,"dar":9501,"dan":12985,"dam":10048,"cul":40621,"cue":32934,"cua":33441,"ctu":33625,"ctr":9939,"cto":44819,"cti":20454,"cte":9828,"cta":10659,"cur":10605,"cuy":7974,"cla":14341,"clu":14576,"co ":135190,"ció":209489,"con":251381,"col":25189,"com":184452,"cor":25629,"cos":36158,"cre":24696,"cri":31616,"cro":8764,"cci":38750,"cea":16334,"ch ":11118,"cer":30048,"ces":71319,"cen":32729,"cep":7727,"cel":19017,"ced":7879,"cha":26138,"cia":152068,"ck ":13234,"cie":87706,"cid":70600,"che":19885,"chi":20475,"cho":21852,"cil":9620,"cim":9340,"cir":8507,"cis":9341,"cit":8942,"ciu":26750,"cin":17134,"cio":103441,"cip":45552,"ed ":10329,"ebr":19067,"eae":10702,"ead":15707,"eal":19761,"eas":12813,"eat":7412,"ea ":28070,"efi":9256,"efe":15631,"ega":22427,"edi":50600,"ede":34956,"eda":21258,"edo":13652,"edr":9154,"ech":18820,"eci":89707,"ece":32005,"ecc":12316,"eca":7852,"ecu":14868,"ect":45028,"eco":21596,"dur":19421,"dor":54124,"don":18997,"dou":11963,"dos":96466,"doc":7141,"duc":27199,"dri":13291,"dra":9547,"dre":11310,"dro":9953,"dic":47817,"did":11186,"dia":35235,"der":49978,"des":90830,"deb":8351,"dec":9801,"def":7698,"del":228490,"den":85908,"dem":11684,"dep":50829,"deo":9414,"do ":341916,"div":13995,"din":11969,"dio":35658,"dir":18800,"dis":80875,"dit":9088,"die":14888,"dif":14152,"rga":23318,"ri ":7113,"rge":24597,"rgo":16017,"ret":23611,"res":140547,"rev":10637,"rfi":8102,"rea":43224,"ref":13241,"rec":55408,"red":15261,"rei":7333,"reg":68764,"rem":14749,"ren":51706,"rel":19501,"rer":18242,"rep":14882,"rda":8172,"rdo":13679,"rdi":9872,"rde":29313,"re ":155218,"rci":19308,"rch":7367,"rce":15953,"rca":25800,"rd ":11954,"rar":15085,"ras":50104,"rat":31492,"rav":8075,"rag":12449,"ran":146320,"ram":21212,"ral":52901,"rab":19532,"raf":7329,"rad":60032,"rac":43442,"rs ":11514,"ros":47929,"rot":15483,"rom":15707,"ron":42744,"rop":32071,"rov":41024,"rod":21257,"roc":21679,"rol":20667,"rof":11386,"rog":9763,"rno":13811,"rna":28042,"rne":12661,"rni":8562,"ro ":146408,"rma":60914,"rme":24156,"rmi":20846,"rla":8699,"riz":10435,"rio":82353,"rit":87431,"ris":26175,"rig":32503,"ril":18939,"rin":44691,"rim":37251,"ria":74022,"rib":16779,"ric":53993,"rid":24912,"rie":40300,"rtí":7814,"ruc":9429,"rup":19339,"rus":10637,"rva":8468,"rvi":10259,"ry ":9879,"rsi":16295,"rso":23313,"rse":13455,"rta":69414,"rto":22364,"rte":83807,"rti":44079,"rtu":12987,"nía":7088,"rt ":11603,"rqu":18224,"rro":28597,"rri":26111,"rre":31796,"rra":36840,"sad":15576,"sal":12684,"san":10863,"sas":11791,"sar":21220,"sa ":86942,"rzo":11621,"sie":10373,"sid":35818,"sic":33044,"sia":21182,"sit":47886,"señ":8377,"sis":26321,"sin":18695,"sio":18884,"sil":10556,"sim":10529,"sig":27477,"scr":25119,"scu":15948,"sde":20004,"se ":142516,"sca":14799,"sco":24677,"ser":42063,"ses":15571,"seg":17308,"sec":10078,"sep":14018,"sen":28636,"sem":7053,"sel":10295,"spu":8777,"spo":14448,"spe":51710,"spa":61850,"sol":12812,"son":47400,"sor":10955,"sos":12831,"soc":12868,"sob":16402,"su ":57230,"st ":10396,"sla":17876,"smo":24196,"sió":31343,"so ":44831,"stá":22167,"ste":88396,"sta":179440,"sto":49757,"sti":69501,"stu":17380,"str":118933,"sub":9390,"sul":7853,"sup":14503,"sus":22170,"sur":23147,"tal":73680,"tag":7515,"tab":14559,"tac":23593,"tad":77862,"tat":7964,"tas":38619,"tar":41661,"tan":67218,"tam":79203,"te ":284941,"tbo":12533,"ta ":179252,"pa ":14872,"pe ":7492,"par":158326,"pas":11530,"pac":12615,"pal":23552,"pan":11657,"pañ":59522,"pec":57888,"pen":14739,"peo":7168,"per":107782,"paí":9997,"pes":10236,"pel":18436,"pla":29458,"pli":11132,"ple":15398,"plo":9048,"pic":11535,"pie":8675,"pin":12467,"pio":24607,"pit":12508,"por":173351,"pop":7479,"pos":33566,"pon":17239,"pol":27348,"pob":43702,"pod":7343,"lés":13215,"po ":39204,"pub":10168,"pti":14042,"lít":16259,"líc":10958,"lín":8248,"pri":52537,"pre":58569,"pro":108338,"put":7419,"pul":11629,"pue":30894,"mát":8971,"más":38263,"mán":10709,"mér":9468,"que":227040,"qui":47867,"quí":8097,"ra ":231606,"mús":8775,"ncé":9514,"ngo":7965,"ngl":22080,"ngu":15031,"ni ":7789,"nge":13697,"nga":8189,"nen":11810,"neo":11720,"ner":65372,"net":8127,"nes":80792,"ng ":18654,"nea":12399,"nec":28654,"nfo":10116,"nez":7381,"nco":17791,"nci":146898,"ncl":10729,"nce":61430,"nch":9730,"nca":12577,"ne ":57285,"ndu":7292,"ndr":12605,"ndo":65513,"ndi":38911,"nde":54981,"nda":66093,"ncu":16386,"nal":70953,"nam":8218,"nan":12157,"nar":27443,"nac":46410,"nad":35305,"naj":8529,"nd ":14500,"nat":21175,"nas":35377,"na ":398062,"ión":292845,"ntó":37020,"nve":10579,"nue":9759,"nto":136463,"ntu":7064,"ntr":93850,"nti":67583,"nta":102723,"nte":292971,"nso":9665,"nst":30664,"nse":26710,"nsi":26727,"nsa":11053,"nt ":24387,"nqu":11277,"ns ":13747,"noc":38585,"nom":49236,"nos":42446,"nor":29422,"nov":19327,"nne":10554,"no ":158177,"nif":10693,"nie":9958,"nid":51044,"nic":69869,"nia":31660,"niz":14512,"niv":15892,"nis":28903,"nit":8537,"nio":20490,"nim":13518,"ogr":18966,"ogo":8779,"ol ":38964,"oce":17267,"och":8312,"oci":56829,"ock":8945,"oco":9047,"oca":41229,"ode":14611,"odi":11580,"odo":26635,"ocu":9100,"oct":12604,"of ":7638,"oda":9018,"oes":12161,"odu":18410,"ofi":7512,"ofe":9434,"oba":8003,"obr":25648,"obl":50514,"obi":12794,"obe":8533,"nza":20160,"ote":14228,"otr":17397,"oto":14558,"ost":31662,"ota":19189,"osi":18739,"ose":15508,"oso":14720,"ovi":57483,"orí":9114,"ove":16922,"oun":14708,"our":10655,"opo":9237,"opi":14838,"ope":13312,"opa":10564,"os ":567590,"opu":9913,"olí":19344,"or ":251715,"orm":64912,"orn":13575,"oro":14041,"orr":19323,"ord":30127,"ore":48358,"org":21075,"ori":66357,"osa":21383,"ort":65622,"m² ":9185,"ora":46259,"ola":30987,"on ":190943,"oli":25682,"oll":17825,"ole":17479,"olo":38775,"olu":11781,"ogí":11606,"ona":104550,"ond":38242,"onc":23216,"onf":9258,"one":74442,"ong":11339,"onj":7098,"oni":32298,"ono":52521,"ons":50424,"ont":59611,"onv":7285,"oma":33534,"ome":21150,"omb":45255,"omi":25678,"omp":41070,"omo":82407,"omu":58045,"la ":610751,"gún":7107,"le ":58880,"lab":11220,"lac":64631,"lad":24486,"lag":7484,"lan":69592,"lam":24610,"lar":41436,"lat":24612,"las":124120,"lbu":12473,"lon":23409,"lom":10655,"lor":23128,"loc":26941,"log":23375,"los":171022,"lme":34946,"lti":12566,"lto":11513,"ltu":9464,"luc":12530,"lug":7366,"lta":15082,"lgu":7617,"lev":15617,"les":91476,"let":11653,"ler":15144,"lem":26257,"len":30658,"leg":14703,"lec":26734,"lea":8424,"lo ":93016,"lla":74669,"lle":47097,"lli":13816,"llo":31751,"ll ":10675,"lit":23261,"lis":29055,"lio":17662,"lin":23082,"lim":8340,"liz":32987,"lic":48449,"lid":38365,"lia":73957,"lib":11554,"lig":10555,"lie":8152,"ma ":72422,"mac":13622,"mad":34215,"mar":42452,"mas":23691,"mal":12192,"man":63359,"may":20759,"mat":18421,"mba":10176,"mbi":42599,"mbr":71287,"me ":12825,"med":24892,"met":16848,"mes":19527,"mer":60276,"men":167009,"mex":7078,"lva":8274,"lus":8634,"mpi":8494,"mpe":21376,"mpr":14573,"mpo":32948,"mpl":23586,"mpu":10144,"mod":10208,"mon":23296,"mor":11861,"mos":17488,"mpa":16447,"ió ":27428,"mus":8143,"mun":83842,"min":50965,"mil":54218,"mis":20947,"mit":22265,"mic":21219,"mie":39274,"mo ":116388,"ién":27517,"zad":34608,"zac":11303,"zan":7115,"zar":10021,"zon":9132,"zo ":17792,"yor":11730,"za ":34624,"ya ":16470,"yo ":17934,"ués":7665,"tín":7356,"tér":8916,"tón":43482,"tán":19529,"xim":7461,"xic":17547,"tá ":17299,"sí ":7618,"rís":7245,"río":10310,"ría":28707,"rón":9102,"via":12659,"vil":17697,"vin":37424,"vic":9152,"vid":23929,"vie":21693,"viv":7676,"vis":25393,"vo ":33305,"vol":10345,"vos":8624,"ver":42481,"ves":9401,"ven":28441,"vel":14981,"ve ":15016,"val":14785,"van":9128,"var":20642,"vas":9770,"vad":11386,"va ":31522,"uye":9933,"uy ":8325,"usi":15007,"use":7793,"usa":13025,"ust":26126,"uso":9699,"uti":16202,"uta":12332,"uto":18658,"us ":49172,"ura":67622,"ure":8908,"urg":11824,"uri":19845,"uro":19849,"ur ":25587,"upe":15682,"upo":17158,"uma":10015,"ume":13099,"unt":24294,"uni":64830,"uno":27366,"unc":10467,"und":48857,"una":267868,"une":8988,"um ":20862,"ult":21405,"ulo":17762,"uli":17880,"ula":47246,"un ":217532,"uid":13351,"uie":12890,"uil":7952,"uin":8672,"uip":8207,"uis":11000,"ueñ":8502,"uit":13101,"uga":15579,"ugu":12215,"uct":12213,"uda":37565,"udi":17314,"ubr":12417,"uca":7683,"ue ":294774,"uce":7415,"ucc":8218,"uci":25554,"uch":12624,"uev":15569,"uer":46622,"ues":36069,"ueg":19032,"ued":16010,"ueb":8677,"uen":38522,"uel":28446,"púb":11593,"ua ":14731,"uat":7912,"uar":13380,"ual":41530,"uan":21270,"ubi":18564,"ubl":12045,"ud ":8479,"uad":38503,"tur":45480,"tul":9339,"tub":10280,"tua":54339,"tud":17240,"tug":8561,"tre":58815,"tra":116327,"tri":81313,"tru":16821,"tro":72914,"to ":261732,"tod":17241,"tos":55424,"tom":10133,"ton":20463,"tor":94895,"til":28629,"tie":38174,"tig":17405,"tir":9162,"tit":18235,"tis":12154,"tin":45124,"tim":14112,"tip":10749,"tio":15498,"tia":11282,"tic":87300,"tid":24642,"tiv":46603,"tem":32370,"ten":66323,"tel":23707,"teg":9394,"tea":9283,"tec":13760,"th ":7178,"tes":66100,"ter":127000,"the":10645,"éti":7504,"éxi":11167,"éne":20764,"én ":28809,"éri":13573,"érm":8428,"és ":45633,"áti":14202,"án ":27367,"álb":11283,"áni":16413,"ás ":40719,"úsi":8717,"útb":7936,"úbl":12344,"ún ":11275,"óni":12420,"ólo":7805,"ón ":361830,"ógi":7057,"ño ":26522,"ños":18182,"ñol":28121,"ña ":43408,"íti":20855,"íst":9984,"íne":7430,"ín ":16078,"ío ":10995,"ís ":12371,"ícu":14416,"ías":8590,"ía ":90214},"n_words":[70286890,82926999,60413548],"name":"es","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/et.json b/contrib/languages-data/et.json
new file mode 100644
index 0000000..533fe04
--- /dev/null
+++ b/contrib/languages-data/et.json
@@ -0,0 +1 @@
+{"freq":{"D":3604,"E":9114,"F":3428,"G":4270,"A":13995,"B":6049,"C":4433,"L":9649,"M":10132,"N":6533,"O":2986,"H":6531,"I":5403,"J":4779,"K":13384,"U":2720,"T":11888,"W":2107,"V":9682,"Q":266,"P":11422,"S":15988,"R":6826,"Y":596,"X":461,"Z":653,"f":12728,"g":77107,"d":141364,"e":425996,"b":40246,"c":10761,"a":496421,"n":259681,"o":223900,"l":256122,"m":147242,"j":63354,"k":181311,"h":68975,"i":428108,"w":2663,"v":88215,"u":213745,"t":258531,"s":344443,"r":180124,"q":542,"p":61630,"z":3134,"y":5816,"x":1328,"²":200,"Å":72,"Ä":128,"Á":51,"Ü":1682,"ß":76,"Õ":259,"Ö":167,"í":335,"ì":56,"ë":66,"é":703,"è":77,"ç":89,"æ":123,"å":190,"ä":42375,"ã":61,"á":537,"à":86,"ü":30512,"ú":88,"ø":326,"ö":8126,"ô":78,"õ":40539,"ó":356,"ð":63,"ñ":47,"ē":125,"ė":44,"Ā":44,"ā":654,"č":90,"ı":50,"ī":336,"ş":74,"ń":54,"ł":96,"ō":306,"Ž":57,"ž":1084,"Š":453,"š":2423,"ū":213,"ǎ":46,"ə":85,"́":52,"μ":94,"ν":197,"ο":232,"ι":145,"κ":95,"λ":130,"δ":59,"ε":108,"η":90,"α":274,"γ":53,"ά":69,"ί":80,"ω":53,"ό":79,"σ":105,"ς":232,"ρ":156,"π":59,"υ":62,"τ":124," l":23928,"ь":570," m":38337,"э":66,"ю":76," n":13986," o":51528,"я":590," h":8823,"ш":180," i":10696," j":29617," k":61170,"ы":415,"ф":166," d":4378," e":23050,"х":292,"ц":201," f":3289," g":1891,"ч":785,"р":2377," a":29616,"с":1744,"т":1384," b":2333," c":602,"у":739," y":132," x":127," z":161," u":3277," t":29638," w":134," v":33321,"і":130,"ё":86," q":53," p":29550," s":33963," r":13644,"И":192,"Л":141,"К":318,"Н":192,"М":293,"П":266,"О":115,"Б":203,"А":448,"Г":287,"В":339,"Е":66,"Д":138,"З":71," J":4764," K":12980," H":6494,"Ш":75," I":5378,"Ю":47," N":6511,"Я":49," O":2964," L":9614," M":10069,"Э":79," B":6003,"Т":143," C":4377,"У":66,"Р":154," A":13953,"С":395," F":3376," G":4219,"Ч":60,"Ф":101," D":3559," E":9045,"Х":61,"л":1746," Z":652,"к":1535," Y":594,"й":835," X":432,"и":3293,"п":426,"о":3362,"н":2533,"м":786,"г":616," S":15877," R":6777,"в":1990," Q":264,"б":447," P":11356,"а":3415," W":2074,"з":300," V":9645,"ж":145," U":2711,"е":2779," T":11830,"д":957," ä":1217," õ":2318," ö":381," ü":7159," Ā":44," Á":51," Å":72," Ä":128," Ö":164," Õ":255," Ü":1674," ž":64," Ž":57," Š":453," š":125,"ն":64,"ա":118,"ו":53,"י":74,"ר":53,"و":125,"ي":278,"ف":48,"ل":301,"م":185,"ن":161,"ه":60,"د":159,"ح":106,"ت":46,"ب":196,"ة":66,"ا":449,"ع":100,"ش":48,"س":100,"ر":186," А":444," Б":203," В":336," Г":277," Д":137," Е":66," З":71," И":192," К":315," Л":139," М":291," Н":191," О":113,"A ":1491," П":265,"F ":209,"Da":636,"Cu":143,"Cl":241,"Co":1022,"Cr":170,"Ce":161,"Ch":852,"Ci":177,"G ":207,"Ec":84,"Ed":303,"Ea":53,"Du":151,"Do":581,"Dr":153,"De":809,"Di":576,"Fe":322,"H ":271,"Fa":384,"Eu":1055,"Ev":105,"Ex":60,"Er":534,"Et":156,"Es":507,"En":458,"Em":268,"Ep":67,"Ei":131,"El":689,"Ek":108,"Ee":3423,"Eh":79,"Eg":183,"Ge":880,"Ga":592,"I ":1170,"Fu":158,"Fr":833,"Fo":411,"Fl":194,"Fj":45,"Fi":490," б":52," г":58," в":53," д":53," и":51," к":97," н":58," м":69," п":140,"B ":254," о":107," Р":152," С":394," Т":143," У":64," Ф":99," Х":61," Ч":60," Ш":75," Э":77," Ю":47," Я":48,"C ":532," с":79,"Av":172,"Au":837,"Ar":1361,"At":438,"As":1550,"D ":280,"Ba":1524,"Az":50,"Ae":128,"Af":163,"Ag":213,"Ah":170,"Aa":1076,"Ab":517,"Ac":135,"Ad":407,"Am":1404,"An":1589,"Ap":221,"Ai":278,"Aj":171,"Ak":327,"Al":2030,"Bu":590,"Br":896,"Ca":1008,"E ":421,"Bi":534,"Be":1094,"Bo":722,"Bl":189,"Kv":67,"Ku":1642,"Gö":61,"Ky":76,"Kn":47,"Kl":321,"Kr":1260,"Ko":2411,"Le":1403,"Hä":127,"Li":2750,"N ":331,"Gü":52,"La":1509,"Lu":547,"Ly":58,"Hõ":54,"Lo":1151,"Me":1361,"Dž":49,"Mi":1243,"O ":457,"Ma":3890,"Hü":108,"Mc":62,"My":97,"Mu":881,"Mo":1371,"Jä":515,"Ni":728,"Ne":1120,"Na":1009,"P ":412,"Ny":45,"Jõ":269,"Nu":172,"No":1468,"Ok":125,"Ol":339,"Om":123,"On":147,"Oh":110,"Kä":196,"Oi":46,"Od":88,"Of":51,"Jü":111,"Ob":154,"Gi":310,"Gl":141,"Gr":855,"Go":370,"Gu":494,"Gy":49,"J ":73,"Ha":1920,"He":1336,"Hi":1244,"Ho":963,"Hu":297,"Hy":59,"K ":351,"Dü":69,"Id":463,"Ig":87,"Io":55,"Im":143,"In":1490,"Il":192,"Ii":222,"Iv":146,"Is":449,"It":383,"Ir":247,"Ja":1273,"L ":301,"Ji":117,"Je":477,"Jo":991,"Ju":835,"Fü":63,"Ka":3521,"Fö":134,"M ":228,"Kh":87,"Ki":1209,"Ke":1467,"Us":124,"Ut":61,"Ur":136,"Um":60,"Un":420,"Uk":201,"Ul":110,"Ud":58,"Pü":264,"W ":104,"Pö":45,"Ty":76,"Põ":1331,"Tv":68,"Tu":682,"Tr":728,"Ts":169,"To":953,"Pä":660,"Th":747,"Ti":538,"Te":1883,"Ta":4354,"V ":865,"Sy":110,"St":1249,"Sv":158,"Su":1472,"Wo":216,"Wi":599,"Wh":48,"Sä":63,"Rü":62,"Wa":532,"We":428,"Rõ":64,"Vo":452,"Vu":47,"Rä":65,"Vi":1892,"Vl":115,"X ":185,"Va":3312,"Ve":2352,"Uu":347,"Mä":340,"Lü":129,"Pt":77,"Pu":606," م":56,"Pr":1526,"Ps":58,"S ":641,"Lõ":671,"Py":58,"Pe":1263,"Pa":2287,"Kü":279,"Pl":289,"Po":1247," ع":51,"Pi":1126,"Ph":224,"Lä":1080,"Os":432,"Ot":174,"Ou":77," ا":173,"Oo":46,"Op":127,"Or":500,"R ":183," ب":58,"Kõ":416,"Kö":81,"Se":3048,"Sc":514,"Si":1380,"Sh":515,"Sm":85,"Sl":182,"Sk":219,"Sp":264,"So":1693,"Ru":426,"Nõ":704,"U ":154,"Sa":3549,"Nü":46,"Re":985,"Ri":1129,"Nä":146,"Rh":72,"Ro":1769,"Qu":132,"Mõ":229,"T ":213,"Ra":1769,"Mü":121,"Tš":407,"b ":14100,"a ":101956,"Tõ":185,"Tö":129,"Tü":236,"Ya":114,"Yo":251,"Yu":48,"Z ":57,"Sõ":312,"Sö":54,"Sü":198,"Xi":96,"Tä":315,"Tō":50,"Za":127,"Ze":122,"Zh":62,"Vä":634,"Zi":61,"Zo":49,"Zu":94,"Võ":414,"bö":50,"i ":67391,"gd":177,"ge":11091,"ga":17189,"bü":159,"fj":156,"fl":261,"fg":90,"ff":336,"fi":3834,"bä":233,"fs":80,"fr":799,"fu":505,"ft":358,"fo":2004,"j ":178,"gy":115,"he":15992,"ha":9696,"gn":1067,"gm":312,"gl":3003,"gk":631,"gj":169,"gi":15351,"gh":514,"gg":188,"gv":123,"gu":12657,"gt":248,"gs":906,"gr":3204,"gp":765,"go":1593,"dt":600,"du":15412,"dv":357,"dw":178,"dy":147,"dz":76,"g ":7339,"ea":10628,"eb":4711,"ec":855,"ed":12410,"de":25494,"dd":200,"dg":178,"di":20365,"dh":181,"dk":488,"dj":432,"eK":296,"dm":1746,"dl":1518,"do":2346,"dn":649,"dp":77,"ds":1055,"dr":3163,"ew":508,"ex":366,"eu":1294,"ev":10732,"ey":521,"aõ":117,"ez":242,"fa":820,"aü":201,"h ":1684,"fe":1177,"eh":9506,"eg":11045,"ef":1060,"ee":30805,"el":44868,"ek":15889,"ej":1876,"ei":11617,"ep":3591,"eo":4613,"en":30509,"em":17859,"et":21350,"es":49834,"er":30979,"eq":50,"ca":1187,"e ":97234,"by":175,"bs":372,"br":3541,"bu":2050,"bt":50,"bn":97,"bo":1301,"bj":462,"bk":63,"bl":1788,"bh":62,"bi":4858,"bb":142,"bd":109,"be":5461,"db":83,"da":21215,"f ":1128,"cz":59,"cy":123,"cu":367,"ct":474,"cs":86,"cr":248,"co":978,"cm":60,"ck":1210,"cl":163,"ci":834,"ch":3094,"ce":1021,"cc":199,"c ":475,"az":442,"ay":578,"ba":4886,"d ":44215,"at":32525,"as":52565,"ar":30297,"aq":50,"ax":179,"aw":265,"av":11952,"au":7185,"ak":18860,"al":53791,"ai":13608,"aj":8772,"ao":2265,"ap":5317,"am":18369,"an":38127,"ac":1403,"ad":21341,"aa":42535,"ab":9670,"ag":6434,"ah":11107,"ae":3739,"af":1986,"nu":11347,"nt":12596,"ns":4828,"nr":469,"np":151,"no":4692,"nn":17075,"q ":46,"nz":355,"jö":51,"ny":360,"jõ":2814,"nw":57,"nv":314,"oe":1975,"of":1882,"oc":974,"od":6416,"oa":902,"ob":2993,"om":10938,"on":61110,"ok":4533,"ol":25486,"oi":2595,"oj":1236,"og":7246,"kä":1444,"oh":4211,"ot":7307,"hō":45,"m²":200,"os":13528,"ov":3905,"ou":1255,"op":3770,"oo":36377,"or":18504,"r ":11002,"ox":103,"ow":446,"kö":136,"oz":161,"oy":237,"kõ":3255,"pe":8582,"pf":53,"kü":2678,"pa":9082,"pl":1823,"pm":182,"lé":50,"pn":209,"po":8371,"lä":2964,"ph":615,"pi":8736,"pk":178,"lo":11129,"ln":1607,"lm":6318,"ll":19635,"ls":3471,"lr":311,"lp":708,"lv":1889,"lu":12030,"lt":11142,"lz":55,"ly":347,"hõ":638,"o ":5439,"hü":580,"ma":35693,"mb":5069,"mg":52,"dž":411,"mh":80,"me":23829,"mf":109,"hā":79,"mk":376,"ml":264,"mi":40706,"eš":58,"mj":183,"mn":780,"mm":2852,"mp":2268,"mo":4294,"mr":80,"mt":306,"ms":1112,"mv":97,"mu":8559,"iõ":260,"my":155,"p ":1639,"na":27821,"iü":173,"nb":439,"nc":950,"nd":30717,"ne":35835,"nf":748,"ež":190,"ng":16763,"nh":382,"jä":4035,"ni":36973,"nj":320,"nk":2416,"nl":419,"nm":235,"jt":102,"ju":8195,"jn":184,"jo":2230,"jm":158,"kj":201,"ki":13463,"kh":323,"kf":60,"ke":20124,"kd":44,"kb":60,"fü":589,"ka":27697,"m ":10376,"fö":145,"ky":200,"gõ":66,"gö":54,"ks":20790,"kt":6626,"ku":25416,"kv":1230,"ko":25420,"kp":298,"kr":3838,"kk":4692,"kl":3700,"km":2283,"kn":633,"li":57000,"hä":827,"lh":474,"lk":2832,"lj":4573,"le":37295,"ld":8105,"lg":4950,"lf":372,"gü":68,"la":31670,"lc":95,"lb":823,"n ":49802,"hr":621,"hs":195,"hv":2638,"hw":103,"ht":6604,"hu":4453,"hj":2882,"hk":5994,"hh":1052,"hi":9059,"hn":1001,"ho":2898,"hl":335,"hm":902,"dé":47,"id":25348,"ic":2524,"ib":3313,"dü":272,"ia":15748,"ih":2638,"ig":11090,"if":1034,"ie":3981,"hy":186,"dõ":82,"k ":16382,"iq":93,"ir":11061,"is":67525,"it":26582,"iu":2821,"iv":6458,"iw":71,"ix":158,"ii":25093,"aš":218,"ij":1535,"ik":35922,"il":29934,"im":19741,"in":53773,"io":9587,"ip":3170,"je":2855,"až":108,"ji":256,"fä":213,"iz":330,"iy":72,"eõ":59,"l ":31660,"eü":118,"ja":41991,"tä":3641,"xi":267,"té":49,"xt":76,"sö":241,"sõ":3307,"ww":59,"z ":542,"sü":7550,"xa":160,"xe":69,"sä":382,"oš":66,"wi":436,"sé":45,"wn":69,"wo":143,"ws":111,"vv":89,"rõ":329,"vy":59,"rö":278,"y ":2468,"rø":55,"wa":716,"rü":1335,"rā":49,"we":364,"vl":251,"vm":177,"ré":93,"vj":46,"vk":322,"rä":668,"vi":9510,"vg":104,"vt":88,"vu":4481,"vr":127,"vs":891,"vp":221,"rí":55,"vn":468,"vo":2748,"uz":144,"uy":48,"ux":121,"uv":4464,"uu":16971,"ve":10196,"vd":60,"va":34297,"x ":556,"ui":3667,"uj":2076,"uk":5480,"ul":16978,"ue":1272,"uf":259,"ug":4237,"uh":3669,"ur":13993,"us":50648,"ut":13872,"um":10739,"un":13735,"uo":494,"up":1948,"ty":350,"põ":3410,"tz":299,"pö":275,"tu":27878,"tt":4340,"tw":94,"tv":618,"ub":5783,"pü":734,"ua":3006,"ud":21571,"uc":397,"w ":514,"to":11780,"tn":1099,"tm":2505,"tl":5476,"ts":21273,"tr":8177,"tp":259,"tg":91,"tf":183,"te":48864,"tk":949,"tj":657,"ti":30264,"pä":3196,"th":1862,"v ":7055,"tb":129,"tc":91,"ta":41990,"su":20276,"sv":2315,"ss":8143,"st":67054,"sy":172,"sz":75,"sw":139,"sl":2942,"sk":10212,"sn":1521,"sm":4289,"sp":2885,"so":5912,"sr":794,"sd":174,"sc":930,"sf":485,"se":62785,"sh":1322,"sg":140,"sj":1063,"kš":54,"si":30745,"rz":159,"u ":17617,"sa":24031,"nü":262,"sb":374,"rr":3973,"rs":3109,"rt":7363,"ru":7114,"rv":5729,"rw":105,"ry":602,"nõ":791,"rp":648,"ro":11751,"rn":3712,"rm":4085,"né":46,"rl":1539,"rk":3920,"nç":57,"rj":4639,"ri":41739,"rh":803,"nä":2052,"iž":51,"rg":9073,"mā":44,"rf":347,"re":19761,"rd":6093,"rc":505,"rb":1857,"mü":731,"ra":27209,"t ":40235,"mõ":3716,"mö":208,"qu":338,"iš":133,"lā":51,"mä":4794,"lü":1913,"s ":83808,"px":82,"lõ":2040,"py":45,"lö":179,"pt":1489,"pu":3682,"pp":2089,"pr":5736,"hū":50,"ps":1178,"yō":62,"uš":116,"tš":984,"vö":298,"zz":105,"vü":141,"vä":6112,"zh":68,"zi":492,"ze":455,"uü":74,"za":431,"uõ":62,"zy":50,"võ":10296,"zs":86,"zu":144,"zo":270,"zn":53,"zm":49,"yg":56,"ye":201,"yc":105,"yd":131,"tü":1113,"ya":457,"yb":52,"tö":1641,"tõ":1288,"yt":138,"ys":416,"yr":185,"yp":88,"yo":142,"yn":260,"ym":152,"yl":325,"yk":76,"yi":121,"რ":63,"ო":50,"ნ":45,"ი":145,"ლ":57,"ე":73,"ა":150,"² ":198,"án":134,"ä ":222,"ár":59,"äb":871,"Üh":852,"Ül":582,"á ":83,"Ü ":171,"Õi":103,"Õh":47,"Õp":48,"Ök":67,"アアア":314,"ö ":312,"õ ":118,"ón":116,"ín":74,"ía":57,"í ":51,"én":82,"és":48,"ér":80,"ää":6210,"é ":189,"ät":739,"äv":318,"äm":341,"äl":3418,"äo":127,"än":4791,"äp":377,"äs":977,"är":10885,"äe":2551,"äd":252,"äg":1326,"äi":4721,"äh":3974,"äk":203,"üü":2774,"ān":168,"ār":63,"õš":48,"ā ":58,"öö":3301,"ể":72,"üo":89,"ün":5015,"üm":1677,"ül":5410,"üs":2571,"ür":1208,"üp":727,"üv":138,"üt":1243,"üb":167,"üa":118,"üf":48,"üg":358,"üd":830,"ük":2636,"üh":5389,"øy":105,"õõ":933,"öv":135,"ør":74,"øn":53,"õr":3591,"õs":403,"õt":1896,"õm":351,"õn":2300,"õo":45,"õp":1548,"õi":12796,"õj":1645,"õk":165,"õl":2454,"õe":2011,"õg":861,"õh":4577,"õb":159,"õd":527,"öt":705,"ör":689,"ös":334,"öp":203,"ön":372,"öl":253,"öm":99,"ök":454,"öi":89,"ög":138,"öe":80,"öd":707,"öb":82,"ḩ":72,"õz":69,"õv":176,"õu":3856,"ī ":118,"īn":58,"š ":464,"še":571,"ša":283,"šo":109,"šk":120,"ši":439,"šu":129,"št":112,"Ša":47,"Šo":91,"Šv":165,"ōk":47,"ō ":113,"žu":64,"žo":59,"že":84,"ža":274,"ži":342,"ž ":81,"ū ":64,"ი ":59,"あ":71,"ア":442,"가가 ":46," ア":53,"乙":67,"之":249,"丹":47,"临":68,"並":138,"丘":83,"专":99,"三":373,"丁":204," 三":86," 丁":77,"倉":51," 之":80,"ああ":46,"ος":107,"ος ":107,"ς ":231,"α ":91,"アア":377,"ян":79,"ый":89,"ье":98,"ьс":48,"ьн":72,"ха":91,"ци":80,"че":89,"ск":479,"сл":56,"со":89,"се":96,"си":145,"рь":62,"са":149,"рс":87,"рт":59,"ру":71,"тр":186,"то":214,"те":148,"тв":71,"ти":150,"та":205,"сс":83,"ст":346,"ур":101,"ус":74,"ул":50,"ун":50,"уд":52,"уб":58,"ть":45,"фо":51," Ga":590," Ge":860," I ":313," Fo":396," Fu":156," Fr":828," Fi":477," Fl":192," Fj":45," Ha":1914," He":1326," Gy":49," Go":366," Gr":850," Gu":492," Gi":307," Gl":139," Ig":87," Id":463," Dü":69," Hy":59," Hu":297," Ho":962," Hi":1244," Ji":117," Je":475," L ":50," Ja":1269," Iv":146," Ir":247," Is":445," It":383," Im":143," In":1484," Io":55," Ii":222," Il":189," M ":95," Fö":134," Fü":63," Ka":3502," Ke":1459," Ki":1200," Kh":87," Jo":986," Ju":831," N ":212," Gü":52," La":1495," Le":1398," Hä":127," Li":2733," Kl":320," Kn":46," Ko":2402," Kr":947," Kv":67," Ku":1637," Gö":60," Ky":76," Mc":62," Ma":3863," Hü":108," O ":93," Mi":1232," Dž":49," Me":1354," Lo":1146," Ly":58," Hõ":54," Lu":546," Ne":1115," P ":83,"а ":653," Na":1003," Jä":515," Ni":722," Mo":1366," My":96," Mu":873," A ":257," B ":117," C ":205," Ap":221," Am":1400," An":1581," Ak":327," Al":2024," Ai":275," Aj":171," Ag":212," Ah":170," Ae":126," Af":163," Ac":133," Ad":404," Aa":1075," Ab":516," Ba":1515," D ":91," Az":50," Av":171," Au":835," At":438," As":1512," Ar":1360," Be":1090," Bi":531," Bl":186," Bo":717," Br":891," Bu":586," E ":253," Ca":996," Ce":158," Ci":166," Ch":848," Cl":225," Cr":170," Co":1011," Cu":137," F ":65," Da":635," Di":564," De":805," Dr":153," Do":564," Du":150," Ea":51," Ec":83," Ed":300," G ":69," El":689," Ek":107," Ei":131," Eh":78," Eg":183," Ee":3409," Et":156," Es":505," Er":526," Ep":67," En":453," Em":267," Ex":60," Eu":1052," Ev":105," Fe":322," Fa":376," H ":102," Xi":80," Tä":312," Sü":194,"к ":130," Sõ":311," Sö":54,"Ив":73," Wo":210," Wi":583," Wh":45," Sä":63," We":424," Rü":62," Wa":532,"й ":637," Rõ":64," Zu":94,"Ле":48," Võ":412," Zo":49,"Ку":52," Ze":122," Zh":62," Vä":632," Zi":58,"Ко":96,"м ":90," Za":127," Yu":48,"Ка":72," Yo":250," Tü":235," Ya":113," Tö":129," Tõ":184,"л ":110,"Ни":84,"Мо":47,"о ":255," Tš":404,"Ма":85,"Ми":98," Tō":50,"н ":434,"Па":47,"Пе":79,"По":48," a ":162,"с ":91,"р ":253," R ":62,"в ":424,"Ан":87," Kõ":416," Kö":81,"Ал":161," Ou":77," Os":431," Ot":172," Or":497," Oo":46,"Аб":44," Op":127," Po":1235," Pl":283," Pi":1123," Ph":212," Lä":1080," Pe":1250," Pa":2281," Kü":278," Ny":45," Jõ":269," Nu":172," No":1462," Ol":339," Ok":124," On":146," Om":122," Oh":110," Kä":196," Oi":46," Od":87," Of":45," Ob":154," Jü":111," Mõ":229,"Вл":51," Ra":1755," Mü":121," T ":66,"д ":86,"Ви":55," Qu":130," Ro":1766," Re":978," Ri":1126," Nä":145," Rh":72," Lõ":669," Py":57," S ":141,"Бо":74,"г ":58," Pr":1513," Ps":52," Pt":76," Pu":606,"Ва":94," Lü":129," Mä":338," Sy":110," Sv":156," Su":1464," St":1223," Ta":4345," V ":98," Pä":659," Th":744," Ti":535," Te":1863," Tr":722," Ts":168," To":951," Nõ":702," Ru":426,"Ге":81,"Гр":45," Sa":3543," Nü":45,"е ":239," Sh":511," Si":1350," Sc":495," Se":3040," So":1682," Sp":263," Sk":219," Sl":182," Sm":84," Uu":347," Va":3308," X ":95,"и ":220," Ve":2347," Rä":65," Vi":1878," Vl":115," Vo":452," Vu":47," Tu":679," Tv":68," Ty":76," Põ":1331," Pö":45," W ":53," Pü":263," Ud":58," Uk":200," Ul":110," Um":60," Un":417," Ur":135," Us":122," Ut":61," ja":20477," l ":68,"ь ":130," io":82," im":299," in":4488," il":865," ii":156," is":1859," it":246," fü":370," ka":13902," fö":77," m ":441," ki":4980," ke":11286," jn":167," jo":514," jm":152," jt":94," ju":2953," ha":2640," he":1467," gl":111," gr":524," go":49," k ":146,"ы ":90," ib":56," dü":205," id":1072," ig":443," hi":1143," ho":971," hu":967," jä":3184," ni":6713," nd":118," ne":1909," na":1142," p ":48," mu":3133," mo":1838," mm":67," ok":758," ol":7096," om":1854," on":34245," oh":172," kä":1164," oj":83," of":466," ob":937," jõ":1996," nu":369," nt":126," no":1262," nn":71," le":1944," lk":65," hä":549," li":6989," n ":104," la":5115," kv":246," ku":9586," km":1544," kl":989," kr":1633," ko":10755," me":3580," dž":65," mi":14942,"я ":380," hü":385," ma":7824," lu":810," hõ":577," lo":3309," ae":363," af":69," ag":306," ah":192," aa":4258,"Ст":49," ab":842," ac":46," ad":192," am":855," an":2331," ap":575," ai":1079," aj":1910," ak":483," al":5568," av":698," au":1797," ar":3514," at":197," as":4137," d ":153," ba":664," 가가":46," bi":566," be":162," bo":152," bl":78," bu":241," br":141," ca":117," e ":147,"х ":65," b ":66,"т ":156,"Ро":57,"Се":84,"Со":64," er":2077," et":1438," es":2246," en":2378," em":344," ep":75," ei":852," el":2672," ek":498," ef":88," ee":3011," eh":5575," eg":105," fe":136," fa":280," eu":119," ev":98," fu":315," fr":185," fo":525," fl":88," fj":44," fi":1214," bä":181," ge":720," ga":319," i ":993," cm":54," co":181," ch":69," da":137," do":440," dr":125," de":2173," eK":291," di":908,"ч ":585," ed":721," eb":183," du":101,"ль":338,"ма":140,"ме":79,"ми":161," vö":80,"лл":50,"ло":200," vü":80,"ла":291," zo":50," zu":50,"ле":285," võ":8473,"ли":217,"кс":169,"ко":414," vä":4759,"ка":276,"ки":323," tõ":919," tö":1012," tü":520,"йс":53,"ия":126," tä":3294,"им":129,"ин":324,"ик":203," sõ":2172,"ил":197," sö":81,"ии":45,"ий":374,"ич":632,"их":108,"ит":98,"ир":115,"ис":145," sü":5410,"ри":296,"рк":48,"рн":83,"ро":406,"ра":317,"рг":121,"рд":63,"ре":208,"пр":87,"по":93,"па":67,"пе":55,"ос":221,"ор":399,"оп":51,"от":68,"ок":58,"ол":330,"ом":142,"он":240,"ой":87,"ов":918,"ог":103,"од":117,"ое":100,"об":67,"ны":111,"нт":92,"нс":154,"но":345,"нн":108,"нк":74,"ни":279,"не":99,"нг":60,"нд":193,"на":391," tš":115,"мо":119,"ге":109," ru":423," nõ":449,"ги":57," u ":56,"го":156," sa":6645," nü":110," se":7005,"гу":45," sc":47," si":2589," sh":125," sl":105," sk":246," sp":570," so":1701,"да":103,"ве":150,"ви":646," mõ":2887," mö":122,"вн":105,"во":155," t ":61," mü":545," ra":5080,"вс":81," re":2103," ri":3272," nä":1534,"га":89," ro":1241,"бе":93," pu":1946," pr":4024," ps":246,"бо":55," s ":402," lö":96," px":82," lõ":1781," mä":2882," lü":990,"ва":236," os":2510,"ад":144," ot":471,"ае":61,"аз":52," oo":592," op":302,"аб":45,"ав":150," or":1834,"аг":45,"ам":81,"ан":559,"ай":127," kõ":2868,"ак":68,"ал":255," kö":73," pe":3640," kü":1892," pa":3723,"ас":154,"ар":332,"ат":153," pl":542," po":5154,"ая":206,"ба":89," pi":4298," lä":2715," rü":649," rõ":177," y ":67," rö":76,"ив":76,"иг":52,"ид":49,"ие":98," sä":314," x ":107," va":12420," ve":4280," uu":1024," vo":1317,"за":70," vu":102," rä":537," vi":1606," vk":47," vm":99,"ет":178," pü":607,"ес":139,"ер":394,"ео":53,"ен":381,"ем":79,"ел":192," pö":222," põ":3044,"ек":180,"ей":122," tu":3292,"ее":74," us":346," um":867," un":274," uk":117," ul":323," uj":60," ta":4816,"др":169,"до":115,"ди":121," sy":46," st":1273,"де":141," su":5118,"ев":407," tr":1088," ts":358,"ед":85," to":2603," pä":1847," th":387," ti":918," te":10216," Õp":48," Õh":47," Õi":101," Ök":64," Ü ":59," Ül":578," Üh":851," är":293," ää":835," öe":60," ök":219," õl":119," õi":938," õh":408," õp":648," õn":74," öö":76," ür":128," ül":2321," üm":508," ük":1493," üh":2633," 가":51,"가":148,"ōky":44," Šv":165," Šo":91," Ša":47," ža":50,"د ":87,"ة ":66,"ان":46,"ال":176,"ر ":45,"ي ":68,"ن ":97," アア":51,"AS ":102," ко":46,"BA ":64," по":46," Ро":57," Пе":79," Па":47," По":48,"가가":97," Ст":49," Се":84," Со":64,"AO ":53," Ан":87," Ал":161," Аб":44," Ва":94," Бо":74," Ви":55," Вл":50," Ге":81," Гр":45," Ка":71," Ив":73," Мо":47," Ни":84," Ко":95," Ку":52," Ле":47," Ма":83," Ми":98,"Fel":59,"Fer":121,"Fil":156,"Fin":145,"Fir":68,"Fan":92,"Fal":44,"Fai":50,"Era":62,"Eri":232,"Est":182,"Ern":93,"Esi":185,"Eur":956,"Eva":52,"Ehi":44,"Ele":225,"Ela":50,"Eks":56,"End":51,"Ena":97,"Eng":50,"Ene":66,"Emm":77,"Ema":72,"Eli":126,"Ell":63,"Ent":53,"Ger":95,"Geo":499,"Gen":130,"Gio":51,"Gil":46,"Ива":71,"Öko":66,"Gan":54,"Gal":104,"Gam":62,"Gar":101,"Gab":56,"Flo":75,"Fla":65,"Fra":377,"Fri":218,"Fre":186,"Foo":47,"Fon":48,"For":124,"IP ":48,"II ":579,"His":301,"Hii":583,"Hil":89,"Hin":100,"Hel":430,"Hei":240,"Hea":65,"Hen":145,"Her":270,"Haa":140,"Hab":68,"Hal":209,"Hai":81,"Han":256,"Ham":105,"Har":682,"Hau":60,"Gus":86,"Gua":87,"Gui":119,"Grö":69,"Gre":178,"Gri":80,"Gra":216,"Gru":170,"Gro":104,"Gol":71,"Got":58,"Gor":73,"Ing":355,"Inf":74,"Ini":62,"Int":306,"Ins":151,"Ill":72,"Ind":377,"Imp":71,"Iis":118,"Iir":94,"Ida":408,"Hum":47,"IV ":90,"Hor":121,"Hoo":71,"Hom":80,"Hon":86,"Hol":356,"Arg":115,"Arh":59,"Are":119,"Arc":52,"Ara":180,"Arm":142,"Arn":51,"Ark":65,"Ari":84,"App":49,"Apo":82,"Ate":50,"Atl":232,"Asu":1049,"Ast":103,"Ass":139,"Ase":89,"Art":168,"Arv":91,"Aru":68,"Ava":88,"Aut":116,"Aus":402,"Aug":149,"Bai":96,"Bak":55,"Bal":343,"Ban":110,"Bab":69,"Bad":75,"Bar":269,"Bat":57,"Bas":115,"Bau":47," пр":54,"Aar":73,"Aas":446,"Aaf":295,"Aad":67,"Abr":47,"Abe":61,"Aba":83,"Abd":79,"Abi":49,"Ada":90,"Adv":74,"Ado":56,"Ade":47,"Aer":49,"Aeg":46,"Age":54,"Afg":70,"Ain":50,"Air":92,"Al ":56,"Aja":140,"Aka":119,"Akt":63,"Ala":283,"Alb":201,"Alg":178,"Ali":85,"Ale":472,"Alf":50,"Alu":59,"Alt":115,"All":206,"Alp":102,"Ame":1051,"Amb":81,"Ama":69,"Ang":173,"Ani":70,"Ana":136,"And":351,"Ans":55,"Ant":450,"Ann":191,"Bus":55,"Bul":94,"Bur":160,"Bud":75,"Мих":58,"Bru":68,"Cal":198,"Cam":87,"Cas":93,"Car":284,"Cat":48,"Can":91,"Bea":78,"CH ":77,"Ber":414,"Ben":187,"Bel":194,"Bil":84,"Bis":73,"Bir":85,"Bio":136,"CO ":51,"Bla":86,"Bre":116,"Bra":266,"Bro":108,"Bri":271,"Ник":72,"Bol":76,"Bon":63,"Bor":154,"Bos":66,"Bou":71,"Õig":83,"EE ":44,"Det":56,"Des":76,"Dev":50,"Del":63,"Dem":159,"Den":66,"Dep":48,"ан ":122,"Dan":112,"Dar":75,"Dav":137,"ай ":47,"Chr":190,"Che":117,"Chi":112,"Cit":59,"Cla":86,"Cen":74,"Cha":324,"Cri":46,"DV ":45,"Clu":51,"Cor":172,"Com":152,"Col":177,"Con":247,"Cou":57,"FA ":50,"ади":61,"аев":60,"Ege":46,"Egi":118,"али":51,"аль":66,"ано":90,"Ees":3340,"анд":117,"ани":46,"Edu":80,"Edw":63,"Ede":52,"FC ":88,"Dia":62,"Dis":117,"Dio":47,"Dig":58,"Die":54,"Div":45,"ая ":201,"Dre":44,"Dra":54,"Пет":51,"Doy":59,"Don":122,"Dom":77,"Doo":55,"Dor":65,"Nee":123,"Nen":81,"Nel":76,"Nei":118,"Nev":47,"Neu":75,"Net":49,"Nat":157,"Nii":60,"Nig":95,"Nic":90,"Nim":92,"Nik":161,"Jär":285,"New":291,"Nap":53,"Nar":263,"Nan":47,"Nag":44,"Nad":109,"Jõe":111,"Jõg":96,"Jää":129,"OS ":46,"Nov":157,"Nor":978,"Noo":108,"Але":136,"Kär":84,"Obe":69,"Jür":85,"Ott":71,"Ote":47,"Kõi":90,"Kõr":209,"Oli":46,"Ole":65,"On ":49,"Oma":92,"Olü":53,"Ope":62,"Ora":58,"Osc":88,"Osa":64,"Ord":51,"Ori":67,"Org":113,"Ost":56,"Osl":81,"Oss":50,"Lää":780,"Ple":51,"Pla":187,"Pin":124,"Pil":73,"Pik":86,"Pii":306,"Pir":78,"Pih":114,"Pie":94,"Pho":46,"Phi":103,"Läh":64,"Lät":200,"Pea":138,"Ped":50,"Per":283,"Pet":350,"Pen":99,"Pel":65,"Pee":136,"Pat":127,"Pas":71,"Par":721,"Pav":48,"Pau":155,"Paa":107,"Pad":62,"Pan":241,"Pai":107,"Pal":313,"Kül":163,"Pak":101,"Lõu":592,"Pto":68,"Pun":149,"Pur":44,"Pue":49,"Puh":59,"Puu":79,"Pro":404,"Pri":219,"Pre":257,"Pra":604,"Pol":248,"Pom":45,"Poh":46,"Pot":67,"Pos":114,"Poo":305,"Por":187,"RO ":68,"Mär":85," ال":145,"Män":102,"Вла":51,"SA ":770,"Вас":49,"Raa":184,"Rad":71,"Rai":65,"Rah":536,"Ram":65,"Mün":49,"Ran":128,"Rak":98,"SD ":47,"Mõn":60,"Mõi":70,"Isa":72,"Ise":51,"Ita":340,"Isl":190,"Ira":139,"Iva":115,"Jac":79,"Jaa":509,"Jar":46,"Jan":141,"Jam":143,"Jal":74,"Jak":74,"Jen":64,"Jer":96,"Jea":76,"Jee":58,"KP ":62,"Jos":138,"Jor":57,"Joo":51,"Jon":103,"Joh":467,"Joa":63,"Jug":45,"Juh":63,"Juu":106,"Jur":60,"Jul":218,"Jum":146,"Föd":118,"Kaa":255,"Kad":123,"Kab":68,"Kai":254,"Kah":71,"Kag":66,"Kam":188,"Kal":368,"Kak":46,"Kap":63,"Kan":466,"Kau":239,"Kat":178,"Kas":307,"Kar":687,"Ker":134,"Kes":554,"Ket":49,"Ken":126,"Kel":51,"Kem":44,"Kei":251,"Keh":55,"Kee":118,"Kir":431,"Kit":75,"Kin":160,"Kiv":96,"Kil":83,"Kih":48,"Kii":162,"Klo":53,"Kli":49,"Kle":53,"Kla":98,"Klu":55,"Koo":227,"Kon":442,"Kom":339,"Kol":258,"Kos":162,"Kor":274,"Kop":81,"Kog":69,"Kod":97,"Kok":46,"Koi":60,"Koh":156,"Kr ":310,"Kot":63,"Kre":312,"Kra":186,"Kri":267,"Kro":87,"Kru":51,"Kui":148,"Kul":317,"Kun":277,"Kur":369,"Kuu":241,"Kva":53,"Lev":81,"Les":48,"Lep":73,"Leo":109,"Len":251,"Lem":45,"Lei":135,"Leh":50,"Lee":284,"Lea":56,"Lau":185,"Le ":49,"Lak":57,"Lai":145,"Lag":77,"Lah":103,"Lae":57,"Las":93,"Lar":49,"Lap":96,"Lam":60,"Lan":154,"Lad":56,"Laa":70,"La ":89,"ML ":59,"Lib":76,"Lie":81,"Lih":74,"Lig":47,"Lii":1363,"Lil":58,"Lim":58,"Lin":573,"Lip":63,"Lis":111,"Lit":56,"Liv":63,"Lut":62,"Luu":51,"Luk":48,"Lui":44,"Lud":75,"Luc":57,"Lou":126,"Los":88,"Lot":49,"Loh":45,"Lor":56,"Loo":276,"Lon":195,"Lom":53,"Lok":47,"NA ":78,"Mei":81,"Meh":138,"Men":72,"Mel":110,"Mes":92,"Mer":322,"Met":201,"Med":80,"Mee":102,"Man":356,"Mal":230,"Mar":1023,"Mas":191,"Mag":212,"Hüd":45,"Mad":279,"Maj":106,"Mak":124,"Mah":61,"Mai":109,"Mac":68,"Maa":607,"Max":73,"Mau":82,"Mat":240,"Mod":67,"Moh":45,"Mol":99,"Mon":346,"Moo":103,"Mos":352,"Mor":136,"Mot":44,"Mih":126,"Mik":120,"Mic":227,"Mit":88,"Mis":98,"Mil":150,"Min":245,"Mul":70,"Muh":59,"Muu":174,"Mur":95,"Mus":281,"Tän":126,"Täh":115,"лав":45,"лад":53,"ль ":49,"TÜ ":47,"ääm":126,"ääl":395,"ään":2004,"ääk":111,"ääd":51,"ääg":344,"ääb":251,"äät":82,"ääv":158,"äär":2288,"ääs":242,"ää ":101,"Sõn":78,"кса":105,"Sõj":66,"Sõr":64,"ков":99,"кол":81,"кий":216,"Wor":93,"Wol":71,"Wil":257,"Win":131,"кая":100,"Wei":45,"Weh":50,"Wes":117,"Was":59,"War":74,"Wat":77,"Wal":182,"йск":48,"ко ":58,"Vor":103,"Voo":53,"Vol":205,"Vis":61,"Vit":59,"Vla":113,"ная":59,"ое ":83,"ндр":145,"ой ":73,"Väl":103,"Väi":301,"Väs":46,"Vär":64,"ов ":254,"мир":58,"мов":45,"ман":65,"Yor":178,"You":47,"Töö":118,"Tür":179,"льн":72,"на ":209,"льс":44,"三三":54,"Tõn":45,"лов":104,"лив":45,"лен":58,"лек":145,"ра ":49,"Sve":99,"Suu":805,"Sur":85,"Sul":92,"Sup":46,"Sun":86,"Sud":52,"Str":229,"Stu":106,"Sto":184,"Sta":395,"Ste":226,"Tee":206,"Teh":108,"Teg":101,"Tea":178,"Ted":50,"Ten":61,"Tem":174,"Teo":71,"Tei":326,"Tel":114,"Tam":134,"Tan":148,"Tat":66,"Tar":920,"Tav":135,"Tai":140,"Tal":1376,"Tag":78,"Taa":310,"Tab":46,"Tad":48,"Ta ":667,"ори":54,"оро":72,"Ska":93,"Shi":143,"She":124,"Sha":138,"Sim":113,"Sil":212,"Sii":184,"Sih":45,"Sig":69,"Sis":180,"Sir":66,"ост":84,"Sin":138,"Sie":46,"Sib":150,"оль":64,"Ser":245,"Sev":74,"оло":46,"оли":53,"ола":72,"Sep":56,"Sen":68,"Sel":786,"Sem":74,"Sei":87,"Sed":193,"See":1098,"Sea":119,"TV ":56,"äb ":221,"Spa":44,"Spe":59,"Spo":69,"Sof":45,"Soc":51,"Sot":162,"Sou":50,"Sol":103,"Som":77,"Son":96,"Soo":896,"TO ":49,"Slo":106,"äga":245,"äev":922,"ова":78,"äet":56,"äes":339,"äea":55,"äed":62,"äel":380,"äht":1060,"äid":319,"äib":55,"Nõu":525,"ähe":2126,"ähk":56,"ähi":637,"Nõm":138,"äge":296,"ägi":746,"äe ":449,"äbi":596,"ный":73,"SV ":478,"Rus":63,"äda":144,"Rum":92,"äde":48,"änn":112,"änu":150,"Sag":67,"Sai":131,"Sah":102,"Sak":1209,"Sam":213,"Sal":247,"Saa":614,"Sab":44,"Sad":48,"ämm":58,"äna":797,"äni":236,"äng":1228,"äne":1640,"änd":528,"äpp":49,"äps":237,"Sco":47,"Sch":374,"äol":71,"Sav":86,"Sau":94,"Sar":178,"San":281,"är ":134,"äit":1511,"äis":253,"äir":146,"äin":438,"äil":189,"äik":1090,"äij":45,"äig":275,"äie":224,"äiv":81,"ове":46,"älu":94,"ält":126,"ови":380,"äli":683,"älj":1874,"овн":48,"овс":48,"äme":148,"ого":52,"äki":100,"TA ":65,"älg":52,"äld":112,"äle":277,"ävi":138,"SI ":60,"Res":61,"äva":99,"Rev":69,"Näi":96,"нов":187,"ор ":52,"Ris":101,"Rii":508,"Rin":80,"Ric":153,"Rid":55,"ärg":1735,"äri":1473,"ärj":460,"ärk":656,"ärm":139,"ära":1650,"ärd":69,"äre":1128,"Ras":57,"Rau":107,"Rap":154,"äsi":369,"äsk":115,"äse":53,"нко":46,"он ":72,"ärv":1633,"ärn":371,"äru":76,"ärt":890,"ärs":325,"ärr":78,"ätm":48,"äti":288,"ätk":94,"äst":200,"äss":64,"äsu":88,"Rec":53,"Rei":185,"Reg":77,"Ren":63,"Rel":79,"Rep":45,"ätt":141,"äts":56,"Rea":67,"Roh":92,"Rob":131,"Roc":60,"Ros":150,"Rom":120,"Roo":904,"SS ":97,"SO ":46,"нск":92,"сан":105,"Vab":795,"Vaa":45,"Vai":341,"Vah":167,"Vel":86,"Ven":1754,"Vee":94,"сил":46,"ски":218,"ска":112,"ско":106,"Vas":243,"Van":479,"Val":835,"Var":232,"Vih":45,"Vig":101,"Vii":222,"Vid":54,"Vic":79,"Vie":71,"Vir":448,"Vil":366,"Vik":195,"Vin":105,"сов":48,"Ver":155,"Ves":127,"Ukr":194,"Ung":127,"Uni":210,"VR ":44,"Uus":233,"Uur":54,"Uue":46,"Tve":67,"Põl":225,"Põh":1046,"Püh":181,"рий":44,"рис":45,"Pür":45,"ров":215,"VI ":61,"Ter":274,"Tes":65,"The":470,"Päi":111,"Tho":103,"Pär":428,"Tih":59,"Tii":133,"Tim":77,"Pää":51,"Too":201,"Tor":167,"Tol":64,"Tom":75,"Ton":64,"Toi":63,"Tru":76,"Tro":136,"Tri":130,"Tre":108,"Tra":219,"Tur":159,"Tuu":92,"Tul":130,"Tun":106,"ši ":155,"šet":55,"šev":66,"šel":50,"šer":71,"šee":52,"ван":86,"šeh":151,"ša ":68,"šat":45,"Šve":145,"вск":69,"вна":76,"вич":569,"šii":58,"ва ":83,"аси":52,"ато":47,"во ":47,"bje":380,"bja":66,"bis":218,"bit":234,"biv":187,"bio":537,"bip":80,"bir":67,"bik":204,"bil":578,"bim":157,"bin":301,"bii":222,"bij":70,"bo ":72,"blo":54,"ble":274,"bli":550,"bn ":58,"bla":847,"ев ":82,"bod":50,"bok":45,"bol":236,"ей ":78,"bon":137,"boo":67,"bor":247,"bot":147,"bos":56,"bou":65,"be ":205,"bam":130,"ban":365,"bak":284,"bal":373,"bai":111,"baj":47,"bah":73,"bac":66,"bad":310,"baa":286,"án ":48,"bav":70,"bat":316,"bas":493,"bar":1055,"bi ":1023,"bee":78,"bed":61,"bec":51,"ber":2586,"ben":167,"bem":51,"bel":642,"bek":156,"bes":968,"bet":314,"bia":533,"bib":89,"bid":126,"bie":117,"bha":48,"дро":45,"ет ":44,"ca ":308,"car":187,"cas":78,"cat":96,"can":142,"cal":161,"ce ":506,"bri":2225,"bro":86,"bra":389,"bre":198,"bu ":80,"bru":574,"bso":114,"bse":67,"bst":74,"дим":49,"bub":112,"bur":697,"bul":127,"bun":75,"bum":298,"bud":140,"but":54,"bus":333,"дор":45,"by ":122,"aka":1199,"am ":1037,"ake":1191,"aki":1056,"ajo":642,"aju":375,"al ":9437,"aja":7056,"aje":136,"aaž":50,"aik":874,"ail":2117,"aim":1277,"ain":2391,"aio":97,"air":125,"ais":922,"ait":1457,"aiu":220,"aiv":79,"ak ":324,"aig":556,"aie":299,"aid":1846,"aia":317,"ahn":114,"ahk":234,"ahl":63,"ahi":395,"ahj":153,"ahh":236,"ahu":752,"ahv":2279,"ahs":82,"aht":765,"ahr":54,"aho":103,"ahe":4825,"aha":671,"agi":522,"agr":165,"agu":1145,"agn":333,"ago":338,"akä":47,"aol":220,"aok":108,"anu":1617,"anz":132,"ajõ":234,"any":60,"ano":601,"ann":2713,"anm":52,"ant":3838,"ans":1284,"anr":93,"ane":2890,"ang":1938,"ajä":246,"ani":8218,"anj":107,"ank":687,"anl":132,"ap ":65,"ana":2831,"anc":369,"and":7769,"amu":491,"amt":68,"amm":1132,"aml":63,"amo":227,"amp":318,"ams":141,"amj":57,"ami":6164,"adž":93,"ame":2122,"amb":788,"ama":4891,"ao ":126,"alv":587,"alu":1954,"alt":2286,"als":1219,"alr":157,"alp":242,"alo":1054,"aln":583,"alm":1124,"all":6989,"alk":711,"alg":2752,"alh":218,"ahä":77,"ali":9008,"alj":729,"ald":4688,"ale":3211,"alf":70,"ala":5975,"alb":322,"an ":2385,"akv":79,"aks":8379,"akr":533,"aku":957,"akt":1317,"ako":2809,"akp":79,"akk":543,"akl":109,"aba":1901,"abe":658,"abi":1296,"abl":140,"abo":165,"abr":186,"abs":170,"abu":92,"ae ":265,"aca":68,"aab":988,"aaf":943,"aag":470,"aad":3122,"aae":152,"aaj":148,"aak":2014,"aai":1233,"aan":4143,"aal":7268,"aam":1949,"aas":6140,"aar":5151,"aap":765,"aav":886,"aau":47,"aat":3358,"ad ":6207,"ac ":51,"aa ":3615,"ab ":4871,"afo":83,"afr":355,"aft":134,"aff":76,"afe":44,"afi":983,"ai ":831,"aga":2664,"agd":52,"age":905,"aen":249,"ael":441,"aes":76,"aer":147,"aeg":1173,"aee":54,"aed":90,"aek":78,"aeh":45,"ah ":172,"afa":57,"aev":851,"aet":163,"ado":272,"adr":398,"adl":409,"adm":484,"adj":65,"adi":3217,"ade":4451,"ag ":78,"ads":103,"adu":2361,"adv":107,"aco":80,"ack":227,"aci":112,"ach":400,"ace":151,"acc":48,"ada":2882,"af ":128,"act":96,"azo":46,"azi":122,"гор":67,"Šot":73,"avä":555,"aza":97,"др ":56,"avõ":220,"avö":76,"azz":52,"atä":55,"axi":69,"asõ":365,"asü":91,"atõ":66,"atö":93,"atü":106,"aya":58,"aye":71,"ba ":651,"atš":101,"at ":2216,"amõ":135,"arh":476,"anä":68,"arg":455,"arf":44,"are":3776,"ard":1446,"arc":202,"arb":519,"ara":2982,"amü":61,"arp":144,"aro":486,"arn":587,"arm":512,"arl":676,"ark":1022,"anç":55,"arj":940,"ari":5535,"aru":930,"arv":2193,"arr":543,"ars":766,"art":3541,"au ":240,"asa":1531,"anõ":84,"ary":130,"asi":2682,"ash":208,"asc":73,"ase":3414,"aso":395,"asn":174,"asp":282,"ask":962,"asj":302,"asm":178,"asl":169,"aot":356,"aor":55,"aos":1245,"ar ":2000,"akü":111,"apa":845,"ape":400,"api":685,"aph":50,"apn":98,"apl":208,"apo":517,"app":368,"apr":632,"aps":286,"apt":144,"apu":179,"alõ":58,"as ":16624,"alü":198,"amä":398,"ava":6484,"ax ":62,"aut":985,"avs":163,"avo":221,"avl":84,"avi":977,"ave":501,"ay ":224,"arü":99,"awa":117,"arö":60,"avy":49,"arõ":46,"avu":364,"av ":2085,"ata":5828,"asu":6462,"ast":15872,"ass":1978,"asr":56,"asv":597,"atm":137,"atn":57,"atk":174,"atl":435,"atr":525,"ato":1488,"ate":6661,"ati":3865,"ath":254,"apä":659,"aua":310,"aub":254,"att":452,"ats":3184,"atv":82,"atu":6581,"apõ":81,"aul":1069,"aum":65,"aun":225,"aup":177,"aur":292,"aus":715,"aud":1021,"aue":64,"auf":46,"aug":1111,"auh":299,"auk":196,"Tōk":46,"Tšu":55,"Tši":63,"Tše":232,"Ühi":101,"Ühe":728,"Võr":202,"Või":74,"еев":71,"Üli":294,"Üld":58,"Üle":201,"еви":205,"ени":90,"енн":63,"ерн":50,"ерг":55,"екс":141,"ель":84,"етр":46,"ико":106,"ина":63,"ими":56,"иль":70,"ист":52,"иха":59,"ка ":82,"ив ":46,"ие ":51,"ий ":339,"ин ":106,"ич ":575,"ия ":116,"jee":65,"jer":66,"jek":711,"jel":510,"jem":199,"jen":398,"jes":189,"jet":51,"jev":327,"ji ":46,"jad":794,"jaa":1553,"jab":52,"jat":1299,"jas":1811,"jav":659,"jap":328,"jao":620,"jar":182,"jal":3824,"eük":52,"jak":1121,"jan":2708,"jam":517,"jah":222,"jag":412,"jaj":121,"jai":178,"jaz":54,"je ":209,"jms":73,"jne":167,"jok":49,"joo":1404,"jon":315,"jor":223,"jia":91,"jm ":66,"fää":209,"itn":116,"itm":881,"itl":1066,"itr":232,"ito":1263,"itu":2865,"itt":840,"its":4841,"itz":125,"ity":156,"ipõ":57,"iud":60,"isk":1562,"isj":238,"ism":2164,"isl":320,"iso":452,"isn":380,"üdr":236,"isp":759,"iss":1297,"isr":458,"isu":1149,"üdn":80,"ist":16111,"üdi":214,"isv":570,"iv ":471,"ita":4226,"ite":4040,"ith":114,"ipä":225,"iti":2606,"itj":280,"ivs":528,"ivo":234,"ivn":346,"ivu":91,"ühe":2500,"irü":49,"iwa":47,"ühh":198,"ühi":1384,"üha":330,"iup":58,"ius":891,"ium":1351,"iul":72,"iut":86,"iva":1768,"ix ":96,"ügi":120,"ivi":1653,"ive":618,"üga":160,"ipr":172,"ipo":148,"ipp":426,"ipu":354,"ips":94,"ipt":204,"ipi":209,"ipl":275,"ilõ":45,"is ":17934,"ion":1904,"ioo":5170,"iop":63,"ior":225,"ios":453,"iot":121,"iog":121,"iok":89,"iol":466,"iom":85,"ipa":243,"ikü":95,"ipe":504,"iov":51,"ikõ":47,"ir ":371,"iru":697,"irv":50,"irs":78,"irt":114,"iro":264,"irm":315,"irn":410,"irk":973,"irl":76,"iri":2513,"irj":2489,"isi":2505,"ish":182,"ise":17961,"isc":260,"isa":2662,"üda":146,"iu ":159,"inõ":65,"iqu":72,"übi":94,"imä":330,"ire":897,"inä":160,"irg":342,"ira":660,"ird":488,"irc":67,"it ":2043,"imõ":379,"ünn":379,"üno":56,"ünk":61,"üng":61,"ünd":3848,"üp ":170,"üna":270,"ümp":310,"ümm":93,"ümn":303,"itš":421,"ümi":74,"üme":56,"ümf":54,"ümb":638,"ülr":51,"üpo":95,"üpp":83,"üpe":67,"üpi":189,"ünt":213,"ja ":25408,"ül ":45,"itü":46,"itõ":50,"itö":136,"isü":229,"ühr":50,"isõ":144,"ühm":571,"üht":317,"üla":1693,"üle":1718,"üld":476,"ülg":296,"üli":561,"가가가":51,"ülj":132,"ülm":94,"üll":158,"ülo":65,"üks":2052,"ivõ":345,"ivä":146,"üki":160,"ize":56,"ükl":190,"ükk":136,"iza":126,"üm ":72,"kii":670,"kih":644,"kik":66,"kij":101,"kim":281,"kil":445,"kk ":305,"kia":138,"kib":147,"kie":46,"kid":1460,"kiv":697,"kin":1384,"kip":117,"kir":3701,"kis":700,"kit":731,"kiu":59,"kja":72,"km ":1345,"ki ":1992,"kha":79,"kj ":53,"kho":116,"kea":512,"ked":219,"kee":6776,"keh":762,"kei":247,"kek":94,"kej":56,"kem":268,"kel":1252,"ken":530,"kep":122,"kes":5621,"ker":962,"ket":387,"kev":86,"füü":427,"ke ":2101,"kra":1232,"kre":1046,"kt ":481,"ksa":2425,"kse":4827,"ku ":4118,"kro":695,"kru":92,"kri":683,"kpa":66,"kr ":45,"kov":236,"km²":193,"kot":103,"kos":501,"kor":2423,"kop":353,"koo":4137,"kon":8471,"kom":1331,"kol":1558,"kok":400,"koj":93,"koh":2395,"kog":2107,"koe":132,"kod":636,"ööp":158,"öör":516,"öös":287,"ööt":635,"ks ":9327,"ööv":121,"ööb":59,"ööd":419,"öög":123,"kpo":74,"kpi":89,"öök":163,"ööm":71,"ööl":141,"öön":269,"kme":409,"kmi":180,"koa":59,"kob":69,"kne":492,"öö ":232,"kku":1792,"kke":775,"kka":353,"kko":768,"kki":577,"klu":516,"ko ":325,"kma":69,"kle":259,"kla":1096,"klo":288,"kli":1510,"jut":1345,"jus":656,"juv":119,"juu":1477,"jul":461,"juk":80,"jun":529,"jum":545,"jur":167,"jub":134,"juh":1325,"jug":89,"jud":292,"ju ":890,"jt ":90,"kav":342,"kat":1523,"kau":1221,"kar":907,"füs":56,"kas":4782,"kap":253,"kan":1568,"kao":86,"kal":1965,"kam":275,"kaj":243,"kak":404,"kah":1019,"kai":1321,"kag":415,"kae":301,"kad":579,"kab":194,"kaa":1822,"ka ":8332,"föd":110,"ha ":1068,"ham":380,"han":1084,"hap":399,"hai":439,"haj":90,"hak":332,"hal":1865,"hau":155,"hav":82,"har":1689,"has":829,"hat":244,"haf":57,"hae":123,"hag":90,"hab":58,"haa":386,"had":171,"he ":2035,"hek":639,"hel":4104,"hei":379,"heg":134,"hee":149,"hed":836,"hea":267,"hev":124,"het":632,"hes":861,"her":737,"hep":112,"heo":122,"hen":4035,"hem":646,"hi ":803,"hhi":366,"hho":288,"hha":319,"hk ":5071,"hig":50,"hie":100,"hid":172,"hic":69,"hib":91,"hia":196,"hip":104,"hio":120,"hin":1818,"him":313,"hil":853,"hik":677,"hii":392,"hiv":50,"his":1677,"hit":1195,"hir":223,"hja":2408,"hje":45,"hju":407,"hka":48,"hm ":244,"hke":354,"hkl":54,"õõd":171,"hkr":47,"hku":160,"hkv":87,"hn ":212,"õõt":409,"õõr":156,"õõs":83,"hla":75,"hle":125,"hli":44,"ho ":78,"hma":426,"gma":96,"go ":269,"gme":81,"glu":61,"glo":135,"gle":108,"gli":2195,"gla":352,"gko":571,"gno":68,"gni":103,"gne":544,"gna":249,"gmi":89,"glü":44,"gs ":53,"gpa":728,"gol":120,"goo":292,"gon":121,"gos":166,"gor":359,"got":49,"gov":53,"ый ":88,"gu ":2469,"gse":607,"gro":161,"gru":407,"gra":1940,"gt ":50,"gri":373,"gre":257,"gto":66,"gug":104,"gui":112,"guk":582,"gum":403,"gul":521,"gua":174,"gub":77,"gue":138,"gud":1131,"gy ":51,"guv":167,"gut":1041,"guu":62,"gur":250,"gus":4478,"gup":144,"gun":555,"guo":121,"gvi":72,"iai":81,"iah":71,"iak":307,"iaj":141,"iam":367,"ial":852,"iao":48,"dün":222,"ian":848,"iap":68,"ias":2407,"iar":200,"iau":97,"iat":663,"iav":133,"ic ":272,"iaa":1098,"iab":61,"iac":55,"iad":147,"iae":81,"iag":258,"ibl":123,"ibi":192,"ibo":94,"ibn":54,"ibr":97,"ibu":187,"id ":9461,"iba":199,"ibe":483,"ia ":7687,"ib ":1767,"iet":453,"iev":106,"iel":576,"iem":193,"ien":617,"ier":409,"ies":329,"iee":79,"ied":249,"iek":65,"ig ":195,"iec":79,"ifu":44,"ifo":208,"ife":183,"ifi":363,"ifa":59,"icr":74,"ics":57,"ict":114,"icu":98,"ico":150,"ick":168,"ici":96,"ich":758,"ice":186,"ie ":546,"ica":492,"idu":2141,"ids":89,"idr":54,"ido":139,"idm":73,"idn":61,"idl":140,"idi":1042,"idg":52,"ide":8097,"ida":3662,"iib":405,"iia":256,"iif":48,"iig":4040,"iid":2067,"iie":248,"iik":3861,"aši":75,"iin":2367,"iil":963,"iim":756,"iis":1521,"iir":3180,"iip":91,"iiv":1986,"iiu":178,"iit":2428,"il ":6055,"ija":1055,"ije":50,"ijo":86,"iju":112,"im ":1389,"ika":6374,"ige":1448,"iga":2716,"ii ":657,"igl":90,"igm":62,"igh":187,"igi":4211,"igu":1608,"igr":133,"igo":112,"ign":238,"ihe":679,"iha":299,"ihk":182,"ihm":65,"ihh":111,"ihi":253,"iht":687,"ihu":144,"iho":52,"ik ":7328,"imo":225,"imn":64,"imm":144,"iml":60,"ims":252,"imp":192,"idž":109,"ime":7025,"imk":210,"imi":4107,"ip ":128,"inc":157,"ind":2723,"ina":4896,"inb":48,"imt":70,"imu":1850,"inn":6824,"ino":606,"inr":85,"int":1835,"ins":667,"inf":422,"ine":12596,"ijä":97,"inh":82,"ing":10735,"inj":49,"ini":4649,"inl":74,"ink":241,"ioa":102,"inu":4443,"inv":62,"iny":57,"ьев":85,"iko":1851,"ikn":407,"ikm":345,"ikl":1543,"ikk":1983,"iki":2101,"ikh":52,"ike":2715,"ila":1146,"ilb":76,"in ":2156,"ikv":103,"ikt":211,"iku":8444,"ikr":276,"iks":1858,"ilp":103,"ilo":1161,"ill":5963,"ilk":54,"iln":65,"ilm":2925,"ilh":117,"ilj":833,"ili":8126,"ild":409,"ile":1595,"ima":3230,"imb":81,"io ":525,"ils":210,"ilt":420,"ilu":306,"ilv":181,"hol":538,"hom":193,"hon":119,"hoi":327,"hos":103,"hot":60,"hou":76,"hov":70,"hoo":630,"hop":48,"hor":259,"hob":86,"hof":75,"hoe":47,"hod":56,"hni":355,"hno":248,"hnu":57,"hna":53,"hiü":45,"hme":82,"hmi":96,"øya":60,"huk":196,"hul":1525,"hua":81,"htu":834,"htt":114,"hts":526,"htr":54,"htp":46,"htn":93,"htm":114,"htl":468,"hti":872,"htk":120,"hte":1012,"hta":927,"hv ":86,"hst":88,"hu ":482,"hrm":49,"hro":62,"hre":114,"hri":254,"ht ":1267,"hra":59,"hvu":1043,"hwa":51,"hum":318,"hun":86,"hus":694,"hut":421,"hur":174,"huu":56,"huv":197,"hva":963,"hve":131,"hvi":258,"hvk":76,"fi ":130,"ffe":80,"ffi":97,"fes":255,"fer":227,"fen":116,"fek":256,"fel":93,"fia":527,"bän":220,"fga":88,"far":57,"fan":78,"fak":63,"aül":78,"fal":91,"fai":103,"aüh":102,"fac":50,"faa":159,"ff ":69,"fe ":58,"etš":80,"fa ":46,"etü":153,"etö":63,"etõ":90,"eys":46,"esü":78,"exa":132,"ez ":79,"esõ":110,"etä":142,"exi":65,"evõ":561,"evä":227,"ezi":53,"eta":4185,"ete":3476,"eti":2019,"eth":183,"epä":212,"etn":244,"etl":824,"etk":122,"esp":119,"esn":50,"eso":307,"est":15172,"esu":447,"esr":46,"ess":2291,"esw":67,"ev ":1881,"eud":58,"euk":45,"eum":209,"eto":650,"etr":1191,"ets":1555,"ett":1369,"etu":2430,"etv":60,"epõ":75,"ew ":294,"eve":552,"eva":4542,"evo":358,"evk":55,"evi":1332,"eut":162,"eur":217,"eus":372,"ex ":72,"evu":1013,"ey ":327,"erü":78,"epe":305,"epi":674,"elä":86,"eph":149,"er ":4533,"ekü":242,"epa":593,"eot":431,"eos":1290,"eor":477,"eom":115,"eol":658,"eok":113,"eop":87,"eoo":298,"eon":217,"emä":170,"es ":19178,"ept":722,"eps":46,"epu":101,"epl":50,"epp":174,"epo":236,"epr":147,"erk":288,"erl":431,"eri":8620,"erj":357,"erg":1464,"enä":125,"erh":103,"ere":3347,"erf":89,"erc":92,"erd":390,"era":2994,"erb":550,"et ":2344,"emõ":84,"esk":3814,"esl":149,"esm":728,"esh":61,"esi":3721,"esc":105,"ese":2495,"esa":695,"erz":44,"enõ":60,"ery":69,"erv":749,"eru":688,"err":901,"ert":1083,"ers":969,"ern":1144,"erm":1039,"erp":165,"ero":598,"eki":846,"ekk":441,"ekl":132,"eko":1441,"ekr":148,"eks":6278,"ekt":2822,"eku":1116,"ekv":84,"en ":1782,"elb":55,"ela":2671,"eld":1386,"elf":47,"ele":10866,"eli":5612,"elj":466,"elg":374,"elm":278,"eln":482,"elk":721,"ell":3694,"elo":690,"elp":87,"elu":1130,"elv":392,"els":698,"elt":4591,"eo ":141,"emb":1631,"ema":5853,"eme":1702,"emm":105,"emn":126,"emo":658,"emi":4057,"emj":89,"emk":49,"emu":471,"emp":531,"ems":73,"ep ":59,"enf":67,"ene":6080,"enh":138,"ejä":142,"eng":640,"enb":279,"ena":1669,"end":9301,"enc":218,"eno":318,"enn":1302,"enk":163,"enl":78,"eni":2414,"enj":57,"enu":916,"ens":876,"ent":3483,"enr":170,"enz":141,"ejõ":394,"eny":49,"eog":500,"eod":99,"eob":44,"egl":195,"ego":223,"egn":128,"ege":2106,"egi":861,"eha":1078,"egr":208,"egs":221,"egu":1523,"ehn":541,"ehm":64,"ehk":5017,"ehr":86,"eho":88,"ehe":590,"ehi":903,"ehh":383,"ek ":209,"eib":48,"eic":64,"eia":95,"eht":613,"eip":60,"eis":4050,"eir":116,"eim":514,"eil":387,"ein":1247,"eii":45,"eik":96,"eie":63,"eid":2255,"eig":71,"eja":948,"el ":10310,"eit":602,"eiu":148,"eiv":45,"ejo":57,"eje":77,"eke":406,"eka":1625,"em ":2077,"eju":213,"gjo":58,"öta":380,"ötl":187,"ötm":68,"gji":57,"gl ":95,"öst":232,"git":756,"gis":1905,"gir":78,"giv":225,"gil":1090,"gim":411,"gij":262,"gik":642,"gip":428,"gin":524,"gio":325,"gid":469,"gie":61,"gib":455,"gih":62,"gii":53,"gig":185,"örl":72,"gia":2005,"örs":50,"ght":152,"öra":67,"örd":139,"gha":121,"öri":170,"ös ":62,"gga":46,"gi ":5321,"öpm":55,"ör ":108,"gen":1194,"geo":730,"get":321,"geu":170,"ger":573,"ges":288,"gev":1367,"gh ":77,"gee":427,"ged":631,"geb":241,"gei":271,"gem":670,"gel":1762,"gek":61,"gej":286,"öni":115,"gda":70,"önd":159,"gde":51,"ömi":48,"ge ":1988,"gaz":58,"öli":74,"gab":187,"gad":306,"gai":51,"gaa":657,"gas":1340,"gar":467,"bür":50,"gat":540,"gav":357,"gak":106,"gaj":153,"gam":272,"gal":712,"gan":2019,"öko":300,"gap":72,"ga ":9681,"öid":47,"ögi":100,"öel":65,"fur":44,"fta":81,"fti":45,"fun":331,"ft ":139,"fra":194,"fre":107,"fri":412,"fro":53,"for":851,"fos":114,"fot":258,"fon":233,"foo":243,"fol":120,"fla":75,"fli":57,"fo ":79,"fid":70,"fic":72,"fie":44,"fii":98,"fil":1891,"fik":301,"fin":213,"fir":227,"fis":69,"fit":113,"fjo":141,"õla":152,"õle":240,"õlg":110,"õli":251,"õlk":134,"õll":298,"õlm":435,"õlt":294,"õlu":48,"õlv":413,"õju":446,"õkk":88,"õne":668,"õng":76,"õna":926,"õnd":49,"õnn":136,"õni":203,"õnk":86,"õnu":108,"da ":7682,"õmb":51,"õmm":206,"õpe":513,"õpi":263,"õpu":202,"de ":10486,"õpp":472,"dad":930,"daa":286,"dab":1585,"dak":424,"dal":2308,"dai":94,"daj":449,"dag":307,"dae":79,"dat":1521,"das":1349,"dar":435,"dap":141,"dao":265,"dan":602,"dam":1690,"dav":923,"õri":60,"õrj":81,"õrk":196,"dde":47,"õrg":1625,"õra":75,"õrb":100,"õrd":333,"õt ":51,"õrv":475,"õru":187,"õrr":267,"õrm":46,"õda":213,"õde":121,"cul":68,"ctu":66,"õe ":1023,"cto":118,"cti":143,"cy ":83,"õet":187,"ões":152,"õel":416,"õen":137,"õdu":141,"cus":131,"cur":49,"õhk":139,"õhe":49,"õhj":2576,"õhi":1112,"õge":171,"õgi":631,"õi ":5148,"õja":1187,"õiv":218,"õit":583,"õis":2033,"õim":1468,"õik":622,"õie":258,"õid":332,"õig":1596,"õib":513,"õhu":552,"cks":55,"cki":151,"ckh":122,"cla":52,"cle":64,"co ":192,"con":154,"col":137,"com":86,"cor":129,"cos":67,"cot":61,"cs ":57,"öd ":110,"ct ":57,"cra":52,"cro":105,"ödi":95,"öda":96,"öde":377,"õst":108,"cci":47,"õu ":118,"õsa":163,"õtu":46,"õtt":671,"cea":61,"õte":379,"õta":58,"õtm":273,"õtl":100,"õtj":101,"õus":427,"õut":47,"ch ":578,"ces":126,"õua":73,"cen":106,"õue":77,"õud":589,"õug":83,"cel":80,"õuk":759,"õul":55,"õun":1596,"ci ":105,"õva":82,"õve":54,"cha":386,"chw":86,"õtü":68,"chu":131,"cia":158,"ck ":487,"cie":98,"che":615,"chl":138,"chi":411,"cho":153,"chm":45,"chn":57,"chs":85,"cht":224,"chr":50,"cis":100,"cin":92,"cm ":52,"cke":185,"cka":54,"õzs":53,"ed ":5646,"eba":352,"ebe":121,"ebi":253,"ebl":44,"ebo":77,"ebr":684,"ebu":73,"eab":210,"eaa":424,"eag":132,"eae":122,"ead":2747,"eak":483,"eaj":63,"eai":99,"eah":87,"ean":733,"eao":48,"eal":1914,"eam":934,"ear":190,"eas":399,"eap":99,"eav":225,"eat":919,"eau":103,"eb ":2987,"ea ":655,"efi":173,"efo":256,"efa":50,"efe":291,"eff":54,"ei ":1558,"ega":4991,"eft":49,"eej":166,"eek":1260,"eeh":72,"een":1833,"eel":7467,"eem":2642,"eeb":812,"eea":50,"eeg":511,"eed":1212,"ees":3817,"eer":4899,"eep":225,"eev":114,"eet":2444,"edi":900,"ede":2096,"eda":2449,"eg ":436,"edu":707,"edo":157,"edr":231,"eck":136,"ech":210,"eci":98,"ee ":3170,"ef ":57,"ect":151,"eco":81,"dwi":78,"dvu":88,"dwa":83,"dy ":101,"dve":99,"duv":409,"duu":64,"dur":309,"dut":208,"dus":9107,"dva":107,"dor":276,"doo":134,"don":412,"dom":146,"dol":150,"dok":340,"dow":80,"dov":88,"dot":70,"dos":91,"dr ":97,"ds ":128,"dmi":448,"dmu":204,"dne":328,"dni":181,"dnu":82,"dsu":120,"dso":79,"dte":403,"dun":212,"dum":683,"dul":361,"duk":455,"õbe":86,"dub":375,"dua":181,"dud":1109,"dri":1541,"drh":50,"dra":532,"dt ":79,"dre":399,"du ":1711,"dro":385,"dru":114,"dsi":69,"dsa":56,"dse":425,"dha":80,"dge":90,"dgl":50,"dic":84,"did":1413,"dia":1187,"der":1254,"des":3846,"det":1487,"dev":307,"deb":74,"dea":150,"ded":103,"def":181,"dee":433,"deg":743,"dei":268,"del":2624,"dek":762,"den":1386,"dem":499,"dep":286,"deo":437,"di ":4266,"dle":97,"dla":684,"dko":283,"dki":67,"dme":789,"dma":248,"do ":334,"dlu":270,"dli":419,"dja":309,"div":260,"diu":154,"dim":305,"din":5058,"dio":737,"dip":162,"dir":107,"dis":3167,"dit":1143,"die":158,"dif":137,"dig":325,"dii":121,"dik":678,"dil":754,"dka":57,"dju":74,"eKr":290,"rgu":1208,"rhe":141,"näd":70,"rha":144,"näg":90,"näh":356,"näi":1177,"rhi":386,"när":92,"rho":51,"näo":109,"rga":2008,"ri ":5455,"rgk":63,"rgl":133,"rgi":2791,"rgh":68,"rge":1513,"rgs":57,"rgr":60,"rgo":122,"rgm":113,"rgn":141,"ret":637,"res":3334,"rev":282,"reu":99,"müü":230,"rfa":51,"rfe":44,"rfi":65,"rfo":51,"rdu":636,"rds":305,"rdr":70,"rg ":613,"reb":76,"rea":973,"ree":2061,"ref":247,"rec":94,"red":566,"rei":666,"reg":889,"rem":1488,"ren":1721,"rek":855,"rel":1476,"rer":134,"reo":148,"rep":269,"rda":591,"rcu":58,"rdo":137,"rdn":203,"rdk":56,"rdm":90,"rdl":149,"rdi":1642,"rde":1014,"re ":3338,"rbu":230,"rbr":301,"rch":187,"rce":98,"rd ":981,"rao":70,"rap":222,"mür":81,"rar":113,"ras":2388,"müt":281,"rat":2331,"rau":610,"rav":669,"rbi":388,"rbl":48,"rbo":167,"rba":241,"rbe":348,"raj":969,"rai":575,"rah":2331,"rag":255,"ran":3582,"mün":66,"ram":1127,"ral":1912,"rak":1264,"rab":296,"raa":4053,"raf":126,"rae":588,"rad":970,"rac":160,"rpu":103,"rpr":48,"rpo":141,"rs ":316,"rpe":48,"rpa":108,"rr ":55,"rpi":60,"ror":118,"ros":627,"rot":969,"rom":675,"ron":932,"roo":2596,"rop":430,"rou":140,"rov":1089,"row":56,"rob":239,"roa":117,"rod":447,"roc":250,"roj":238,"roi":109,"rol":630,"rok":255,"rof":315,"roe":51,"roh":391,"rog":576,"rno":131,"rns":89,"rnu":569,"rna":702,"riü":55,"rež":141,"rne":1143,"rni":528,"rjä":49,"rmo":236,"rms":58,"rmu":175,"ro ":398,"rma":1727,"rme":627,"rmi":820,"rlo":160,"nää":74,"rli":332,"rld":67,"rle":265,"rla":365,"rn ":318,"rgõ":53,"rkv":211,"rku":265,"rkt":136,"rks":128,"rkm":44,"rko":1059,"rki":431,"rkk":66,"rke":333,"rka":216,"rm ":305,"rju":905,"rjo":50,"rja":2847,"rje":754,"riz":44,"rl ":207,"rip":347,"rio":550,"rir":293,"rit":4861,"ris":3546,"riv":668,"riu":963,"rih":61,"rig":747,"rij":404,"rii":5626,"ril":3640,"rik":3181,"rin":3123,"rim":2490,"ria":2118,"rib":484,"ric":609,"rid":1612,"rie":633,"rif":87,"rk ":868,"rtü":56,"rug":66,"rue":78,"rud":172,"ruc":52,"rup":396,"run":460,"rum":712,"rul":170,"ruk":513,"ruu":777,"ruv":117,"rus":1487,"rut":261,"rva":1548,"rvi":795,"rve":1159,"rvp":144,"rvl":46,"rvu":1089,"rwa":50,"ry ":413,"rsk":243,"rsi":519,"rso":171,"rsc":53,"rsa":171,"rsh":62,"rse":561,"rta":274,"rv ":755,"rst":515,"rss":168,"rsu":190,"rtl":107,"rtm":58,"rtn":92,"rto":217,"rte":1543,"rth":281,"rti":1255,"rub":87,"rua":490,"rts":1053,"rtr":65,"rtu":1315,"rtt":57,"rt ":810,"rro":167,"rri":784,"rre":476,"rra":2127,"ru ":1080,"rry":114,"rru":164,"saa":4775,"sab":140,"sad":632,"sae":148,"sag":380,"sah":226,"sai":500,"saj":656,"sak":2002,"sal":1925,"sam":2292,"sbe":92,"sap":131,"óni":51,"san":1221,"sau":105,"sat":1244,"sas":2183,"sar":1496,"sav":213,"sa ":3636,"ón ":53,"rze":44,"nõu":458,"nõr":64,"nõl":71,"nõi":66,"rvü":53,"sha":261,"sho":138,"she":80,"shi":370,"si ":5360,"sgr":60,"sja":511,"siv":324,"sju":179,"sjo":145,"sfä":198,"sie":142,"sid":2303,"sic":71,"sib":74,"sia":2206,"sk ":538,"shu":57,"sit":1318,"siu":225,"sir":212,"sis":2918,"sip":251,"sin":1981,"sio":5046,"sil":1686,"sim":1606,"sij":144,"sik":2762,"sih":204,"sii":1362,"sif":138,"sig":349,"sda":47,"sdo":45,"sbo":51,"sbu":115,"se ":21232,"sca":191,"sci":49,"sch":453,"sco":130,"sev":717,"sey":55,"ser":1068,"ses":5796,"set":1978,"seu":254,"nüü":243,"sh ":174,"sfi":86,"sfo":68,"sea":1804,"sei":2222,"seh":77,"seg":1695,"see":2074,"sed":3928,"seb":254,"sep":903,"seo":645,"sen":2101,"sem":2170,"sel":10065,"sek":3272,"sej":332,"spu":121,"spo":772,"spr":203,"spe":493,"spi":353,"spa":687,"skü":66,"sot":518,"sou":51,"sov":164,"skõ":91,"sol":473,"som":95,"son":907,"soo":2060,"sop":124,"sor":585,"sos":135,"sog":196,"sof":91,"skä":80,"soa":50,"soc":67,"sob":104,"su ":776,"sri":388,"sre":70,"sra":250,"st ":19438,"smõ":57,"smä":672,"ss ":961,"sli":1453,"slo":247,"slu":141,"sla":864,"sle":179,"ski":1330,"skk":688,"skj":46,"skm":251,"skl":222,"sko":1611,"skn":98,"skp":86,"sks":70,"skr":240,"sku":1933,"skv":335,"sm ":509,"ska":1203,"ske":1189,"sno":114,"sjõ":49,"snu":95,"sna":125,"sni":235,"sjä":141,"sne":888,"smo":245,"smu":122,"so ":126,"sma":1250,"smi":1155,"sme":256,"swi":78,"ssü":155,"ssö":85,"ssõ":135,"stõ":73,"stö":240,"svä":103,"svõ":164,"svö":80,"sse":2400,"ssa":886,"sso":712,"ssk":48,"ssi":2279,"ssu":260,"sst":65,"ste":12441,"stf":123,"spä":99,"sta":12430,"stm":389,"stn":305,"sto":736,"sti":11558,"stj":94,"stk":190,"stl":1467,"stv":203,"stu":4098,"str":2584,"sts":374,"sua":230,"sud":110,"sub":2124,"suh":654,"sug":926,"sul":1089,"sum":385,"suk":274,"sup":112,"sun":473,"suu":3246,"sut":3768,"sus":4456,"sur":769,"suv":746,"sva":1352,"sve":77,"svi":156,"svo":72,"svu":249,"sy ":64,"tai":1177,"taj":1586,"tak":4223,"tal":2877,"tae":166,"taf":74,"tag":732,"tah":420,"taa":2158,"tab":1730,"tac":49,"tad":1273,"tba":65,"tav":3721,"tau":131,"tat":7322,"tas":2453,"tar":1636,"tap":159,"tao":157,"tan":2647,"tam":2975,"tch":74,"te ":12807,"ta ":4265,"сто":56,"стр":66,"ств":57,"pa ":804,"ста":51,"pe ":459,"par":1885,"küt":108,"pat":370,"küs":137,"pas":347,"pau":44,"pad":102,"paa":872,"pab":98,"pag":125,"pae":47,"pak":331,"kül":2089,"pal":1654,"pai":924,"paj":61,"pap":44,"küm":165,"pan":1223,"kün":49,"phe":92,"läb":569,"pha":122,"lät":92,"pho":86,"phi":121,"läh":1032,"pi ":489,"küü":67,"ph ":74,"lä ":65,"pea":2994,"peb":54,"pec":50,"ped":165,"pen":309,"per":2019,"pet":866,"pes":209,"pee":735,"pej":56,"pei":56,"pel":221,"pek":193,"pla":985,"pli":241,"lää":1062,"ple":335,"plo":204,"pko":106,"тан":48,"phy":67,"pia":480,"pid":668,"pie":55,"pig":83,"pii":2460,"pik":984,"pil":982,"pim":106,"pin":1660,"pio":63,"pir":110,"pis":249,"pit":216,"por":922,"pop":279,"poo":3433,"pot":178,"pos":489,"poi":69,"poj":56,"pom":75,"pon":250,"pok":78,"pol":2011,"poe":314,"ps ":115,"ppu":86,"ppi":265,"ppl":73,"ppo":76,"ppa":134,"ppe":738,"тер":49,"pme":107,"po ":62,"pni":140,"pne":50,"pp ":583,"psu":187,"pta":45,"pse":289,"psi":131,"pso":44,"ptu":183,"pub":112,"pud":66,"pte":686,"pti":246,"pto":149,"pts":116,"pra":973,"pru":93,"psa":102,"pu ":143,"pri":927,"pre":785,"pro":2923,"psü":231,"lõh":137,"pur":257,"pus":270,"put":156,"pun":658,"pui":197,"pul":391,"puh":552,"px ":81,"тов":47,"puu":696,"тор":53,"lõp":469,"lõi":316,"lõu":988,"тро":97,"löö":149,"lüm":359,"lül":150,"lük":99,"lüh":844,"lüü":275,"mäe":703,"mäg":380,"män":1220,"mäl":265,"mär":1565,"mää":557,"qua":76,"que":158,"qui":88,"mõe":265,"mõi":1422,"mõj":443,"mõl":89,"mõn":454,"mõr":58,"mõt":400,"mõõ":559,"ra ":2421,"möö":180,"rb ":55,"ngo":283,"ngj":112,"eži":152,"ngi":2407,"ngl":2197,"ngk":485,"ngv":53,"ngu":1971,"ngr":466,"ngt":111,"ngs":158,"ni ":8348,"nge":1413,"ngh":131,"nga":1383,"ngd":44,"nho":71,"jäl":102,"jät":125,"jär":2920,"nha":193,"jäi":81,"nhe":51,"neg":216,"nei":559,"nel":854,"nek":392,"nen":1253,"nem":2691,"nep":231,"neo":254,"ner":1383,"net":1372,"nes":1547,"nev":1928,"neu":153,"ndv":49,"ng ":5293,"nea":209,"neb":1735,"ned":454,"nee":1283,"nfi":63,"nfo":399,"nfl":72,"ney":104,"nez":55,"nfe":71,"nco":82,"nci":160,"nce":327,"nch":174,"ne ":18973,"nbu":170,"ndt":47,"ndu":3781,"ndr":1767,"nds":245,"ndn":73,"ndo":567,"ndl":703,"ndm":736,"ndj":199,"ndk":76,"ndi":9574,"nde":2285,"nda":5981,"ncy":54,"nak":847,"nal":1444,"iül":54,"nam":1141,"nan":791,"nao":524,"nap":666,"nar":696,"nac":71,"nad":869,"nae":164,"naf":95,"nag":661,"nah":165,"iüh":71,"nai":421,"naj":121,"nab":249,"naa":1844,"nbe":157,"nd ":4383,"nba":54,"nav":684,"nau":173,"nat":1014,"nas":5608,"na ":9461,"iõp":167,"가 ":51,"nya":69,"jõe":1151,"nyi":68,"jõg":658,"nz ":90,"nsü":49,"ny ":159,"nux":47,"nve":213,"nuk":391,"nul":392,"num":581,"nun":66,"nui":73,"nuj":150,"nus":1407,"nut":211,"nuv":56,"nur":279,"nua":567,"nud":6307,"nto":594,"ntn":44,"ntu":1152,"nts":3190,"ntr":629,"nti":1551,"nth":107,"ntl":53,"nta":1333,"nte":1954,"nsu":194,"nsp":183,"nso":192,"nst":1654,"nss":87,"nse":418,"nsh":72,"nsi":398,"nsk":423,"nsa":519,"nsb":47,"nu ":581,"nri":214,"nra":73,"nt ":1663,"npr":44,"ns ":389,"nod":52,"nob":45,"nog":106,"nof":50,"nok":61,"nol":406,"noi":73,"noo":957,"nop":56,"nom":241,"non":176,"not":137,"nos":276,"nor":508,"nov":777,"nr ":71,"nne":1409,"nna":9091,"nnm":60,"nno":142,"nni":2211,"nnu":1256,"nns":51,"nme":49,"nma":125,"ndž":60,"nli":99,"jää":730,"nn ":2722,"nla":220,"no ":545,"nke":134,"nkl":53,"nki":267,"nka":177,"nku":165,"nko":182,"nks":69,"nkt":721,"nkr":123,"nji":51,"nja":120,"njo":51,"nij":103,"nii":935,"nih":114,"nig":327,"nif":70,"nie":199,"nid":1206,"nic":153,"nib":61,"nia":1970,"nk ":386,"nix":49,"niu":138,"niv":309,"nis":5167,"nit":1025,"nir":49,"nio":206,"nip":222,"nim":5797,"nin":5067,"nik":3655,"nil":1691,"ogs":64,"ogr":1310,"ogu":2080,"ogt":71,"ogi":2325,"ogl":51,"ogo":115,"ogn":110,"oga":201,"oge":329,"oi ":80,"oht":1247,"kät":121,"käs":428,"kär":67,"ohv":181,"ohu":379,"ohk":157,"ohj":63,"käi":618,"ohi":105,"oho":123,"ohn":161,"oha":1453,"käe":74,"ohe":234,"ois":197,"oir":72,"oiu":71,"oit":264,"oin":102,"oik":52,"oim":998,"oil":59,"oid":535,"ok ":100,"oia":64,"oju":137,"oje":372,"oja":659,"ol ":1399,"oce":44,"och":80,"oci":105,"ock":501,"oco":61,"obs":62,"obu":202,"oe ":109,"ode":1123,"odk":64,"odi":1047,"odo":182,"odn":64,"ods":77,"odr":58,"of ":621,"oda":609,"oel":103,"oeg":287,"oer":126,"oes":47,"oet":688,"oen":399,"odu":2480,"oee":44,"og ":463,"ofi":557,"oft":95,"ofo":118,"off":59,"ofe":202,"oa ":93,"ob ":139,"oan":65,"oam":51,"oal":156,"oak":80,"oaj":111,"oba":165,"od ":481,"oar":70,"oas":64,"obo":116,"obr":390,"obl":782,"obj":309,"obi":400,"obe":361,"jõk":47,"jõu":952,"nza":48,"nze":88,"nzi":54,"kõi":997,"otü":52,"oya":45,"osü":192,"ows":73,"own":56,"kõr":1655,"kõv":81,"oyl":61,"kõl":116,"kõn":322,"otu":561,"ow ":114,"otl":94,"otk":48,"otj":93,"oti":581,"oth":115,"ote":630,"ott":288,"ots":2879,"otr":134,"oto":793,"otm":209,"ost":1657,"osu":161,"ota":435,"otb":45,"ov ":391,"osi":662,"osh":53,"osk":556,"ose":1045,"osf":200,"osp":69,"oss":687,"osm":201,"osl":192,"oso":739,"osn":672,"oy ":48,"owa":44,"owe":70,"ovi":1506,"ovg":80,"ovn":57,"ovo":225,"ovu":110,"ovs":142,"ox ":49,"ova":496,"ove":767,"oug":79,"oui":114,"oul":81,"oun":169,"oup":53,"ous":121,"our":234,"out":116,"opo":355,"opp":88,"opi":377,"opk":103,"opl":127,"ope":659,"oph":152,"opa":1002,"os ":1676,"olü":473,"opu":229,"opr":90,"opt":153,"ops":80,"oon":7646,"ool":6457,"oom":3730,"ooj":448,"ook":1060,"oof":591,"oog":2684,"ood":3903,"oob":808,"ooa":118,"or ":1504,"oov":321,"oot":2071,"oos":2409,"oor":2416,"oop":1217,"ork":293,"orl":106,"orm":1403,"orn":581,"oro":441,"orp":251,"orr":1904,"orc":46,"ord":2640,"ore":923,"orf":173,"org":2150,"ori":3030,"orj":99,"ou ":125,"osa":4585,"onü":106,"ort":1076,"ors":316,"orv":260,"oru":308,"ory":79,"omä":44,"ot ":233,"m² ":198,"omö":62,"orb":147,"ora":643,"ola":1318,"old":420,"on ":39161,"olj":64,"oli":8483,"oll":1578,"olk":349,"olf":126,"ole":2988,"kää":76,"olg":132,"ols":1007,"olt":1144,"olm":1122,"oln":368,"olo":2600,"olu":1563,"olv":65,"ofü":93,"oka":297,"om ":370,"okk":531,"oki":231,"oke":623,"okr":399,"oks":957,"oko":247,"okl":55,"okt":664,"oku":340,"ona":1303,"ond":3491,"onf":139,"one":1232,"ong":640,"oni":6425,"onl":72,"onk":273,"onn":4600,"ono":803,"onp":75,"onr":44,"ons":816,"ont":1242,"onu":172,"onv":154,"ony":92,"oma":3656,"oo ":462,"ome":2088,"omb":266,"omi":1618,"omm":613,"omp":670,"omn":106,"omo":342,"omt":108,"omu":606,"oms":246,"op ":270,"la ":5991,"le ":12620,"lf ":96,"ldg":52,"lde":484,"lda":2949,"ldo":95,"ldn":65,"ldm":61,"ldk":243,"ldj":62,"ldi":712,"ldu":1916,"ldt":53,"lds":102,"ldr":58,"laa":1373,"lab":404,"lac":145,"lad":1522,"lae":732,"lah":1640,"lag":388,"laj":204,"lai":1062,"lal":666,"lak":450,"lan":3578,"güm":58,"lam":1233,"lap":307,"lao":158,"lar":440,"kyō":50,"lat":2238,"las":6761,"lau":1302,"lav":840,"lay":76,"lba":164,"ld ":1107,"lbe":196,"lbi":58,"lbo":45,"lbu":264,"lbr":50,"kvi":101,"kve":117,"kva":933,"kuv":271,"kuu":2381,"kut":1032,"kus":4511,"kur":510,"kup":256,"kun":2636,"kum":1199,"kul":2990,"kuk":399,"kuj":1401,"ky ":44,"kta":184,"kte":370,"ksp":172,"ksu":1368,"kst":539,"ksk":77,"ksj":60,"ksi":1286,"ksh":44,"kso":186,"ksn":51,"ksm":52,"ksl":97,"kub":299,"kud":842,"kug":145,"kuh":220,"kui":2011,"kua":93,"ktr":1077,"kts":955,"ktu":547,"kti":1734,"kto":1139,"gõz":53,"ksü":131,"kyl":45,"lpo":71,"lpe":157,"lpi":167,"lph":47,"ls ":120,"llü":71,"lpt":60,"lol":75,"lok":121,"lon":328,"lom":526,"lop":192,"loo":6341,"lor":264,"lod":57,"loc":108,"loe":826,"loh":62,"log":273,"loi":81,"los":787,"lot":130,"lov":353,"lkõ":156,"lni":82,"lne":1147,"lob":91,"lnu":266,"lmn":103,"lmi":1561,"lme":771,"lma":2491,"liõ":138,"lmu":534,"lms":87,"lti":513,"lto":103,"ltr":71,"lts":498,"ltu":933,"lud":201,"lub":1064,"lua":79,"lug":367,"lue":62,"lsi":290,"lsk":106,"lso":81,"lss":61,"lst":139,"lsu":210,"lv ":77,"lta":252,"lte":588,"lri":102,"lu ":1464,"lse":1684,"lsa":658,"ía ":44,"lt ":7949,"lra":58,"lhu":155,"häv":119,"lho":73,"här":83,"lhe":124,"häi":143,"lha":82,"lgu":794,"lgs":300,"lgn":56,"lgl":54,"lgr":91,"lgo":86,"lgp":735,"lge":1193,"lgi":247,"li ":8571,"lga":1157,"lfr":46,"lfi":78,"lfa":51,"ley":127,"lex":145,"leu":72,"lev":2102,"les":6911,"let":1229,"ler":467,"leo":191,"lep":522,"lem":2382,"len":1713,"lek":2924,"lel":939,"lei":662,"lej":650,"leh":741,"leg":615,"lef":123,"lee":784,"led":276,"lec":56,"leb":814,"lea":143,"lg ":112,"llp":58,"lls":97,"llu":829,"lly":101,"lo ":301,"lla":3994,"lle":7577,"hää":354,"lli":4723,"llk":46,"llo":330,"lko":770,"lku":106,"lka":667,"lke":205,"lki":380,"lkl":73,"lju":697,"ljo":186,"lm ":614,"lje":847,"ll ":1528,"lja":2809,"lit":3006,"lis":12972,"lir":127,"lip":448,"lio":191,"lin":10479,"lim":654,"liz":82,"liv":165,"liu":274,"lic":156,"lid":1460,"lia":1378,"lib":211,"lk ":259,"lik":8420,"lil":1193,"lii":5503,"lij":133,"lig":496,"lih":443,"lie":227,"lif":188,"hõõ":63,"ma ":4860,"mb ":129,"maa":11983,"mac":100,"mab":268,"mah":237,"чес":55,"mai":1139,"maj":980,"mak":919,"hüd":167,"mad":1582,"mae":114,"maf":51,"mag":417,"hüp":189,"map":146,"mar":885,"mas":3246,"mal":2280,"hüm":55,"mam":324,"man":2399,"mav":524,"mat":3086,"mba":251,"mbl":329,"mbi":269,"mbe":2044,"mbr":1530,"mbo":263,"me ":2925,"mbu":182,"med":1125,"mee":3676,"meg":594,"mea":100,"met":4426,"mev":67,"mep":72,"mes":2849,"mer":2169,"mem":157,"mel":1042,"men":2636,"mei":782,"meh":405,"mek":592,"hüü":97,"lva":713,"lve":678,"lvi":208,"lul":347,"luk":457,"lup":150,"luo":49,"lun":505,"lum":505,"lut":683,"lus":3490,"lur":311,"luv":1330,"luu":781,"ly ":163,"lrü":75,"lvk":62,"hõb":45,"hõi":206,"hõl":310,"mpi":452,"mph":51,"mpe":554,"mpr":66,"mpo":299,"mpl":310,"mpu":89,"mps":61,"ms ":217,"mog":59,"mob":84,"moe":58,"mod":191,"moo":1204,"mon":658,"mok":297,"mom":61,"mol":409,"mov":44,"mor":367,"mos":365,"mot":173,"mpa":252,"msa":75,"mu ":287,"mse":406,"mtu":96,"mud":268,"mub":235,"mst":77,"msu":90,"mso":47,"msi":100,"mte":76,"my ":109,"mur":342,"mus":2450,"mut":357,"muu":2318,"muv":182,"mui":212,"muj":55,"muk":66,"mul":772,"mum":89,"mun":762,"džu":46,"dža":146,"mi ":3313,"dži":112,"mjo":107,"min":5859,"mio":159,"mil":5861,"mim":121,"mir":250,"mis":17764,"mip":65,"miv":72,"mit":2152,"mic":74,"mia":602,"mig":164,"mie":74,"mid":2544,"mik":1090,"mii":356,"mo ":151,"mli":87,"mle":44,"mla":87,"mn ":55,"mko":173,"mka":66,"mm ":329,"mnu":50,"mni":67,"mna":105,"mne":450,"mmy":82,"mp ":57,"mmu":480,"mmi":519,"mmo":72,"mma":633,"mme":654,"tš ":386,"tše":222,"tši":133,"tša":111,"zst":65,"võt":836,"võs":96,"võr":892,"võn":91,"võl":153,"või":7974,"võe":178,"vöö":291,"võõ":64,"vür":134,"zi ":155,"väl":2397,"väh":435,"väi":1090,"väg":389,"väe":751,"zen":74,"zer":96,"ze ":109,"zab":50,"uüh":53,"zan":66,"zar":62,"zon":61,"zo ":48,"vää":478,"vär":503,"väs":44,"zia":44,"zin":86,"yst":121,"ysi":65,"ys ":119,"ylä":47,"za ":111,"tüü":554,"yer":56,"ya ":162,"töö":1589,"tüt":130,"tür":79,"yan":90,"tük":102,"tüh":106,"yn ":51,"yle":91,"ylo":45,"yne":47,"yin":76,"tõe":238,"tõm":70,"tõl":200,"tõk":57,"tõu":195,"tõs":109,"tõt":177,"tõr":100,"tän":706,"täh":1822,"täi":727,"xi ":91,"süü":243,"xim":52,"tär":60,"täp":255,"söö":193,"süs":1324,"süt":71,"sük":585,"sül":70,"süm":231,"sün":4272,"xan":102,"süd":144,"süh":336,"süg":181,"sõl":377,"sõj":1120,"sõp":113,"sõn":939,"sõd":233,"sõi":351,"sää":83,"ws ":61,"wor":58,"rüü":199,"wer":86,"wen":48,"säi":178,"wit":44,"wig":128,"wic":50,"win":68,"rõh":178,"röö":217,"wa ":114,"wan":110,"rün":141,"rüo":64,"wal":97,"rük":165,"way":55,"rüt":71,"war":196,"rüh":567,"vri":47,"vsu":116,"vst":161,"vse":403,"vsk":184,"vu ":181,"vut":793,"vus":2887,"vud":63,"vum":73,"vuk":112,"vul":243,"vy ":55,"vib":72,"via":291,"vio":50,"vip":54,"vir":136,"vik":721,"vil":657,"vim":530,"vin":1205,"vig":116,"vih":89,"vii":1360,"vic":51,"vid":522,"vie":84,"viv":99,"vit":1084,"vis":990,"vka":64,"vko":169,"vkj":47,"vla":98,"rää":402,"vli":75,"vo ":249,"vms":66,"vne":367,"vna":66,"voj":65,"vol":333,"vok":90,"von":554,"voo":711,"vor":540,"vos":69,"vpa":151,"rän":173,"vi ":1273,"vgo":75,"veo":115,"ver":1595,"ves":902,"vet":333,"vei":261,"veg":47,"ven":2478,"vem":547,"vel":383,"vek":127,"vea":44,"vee":1876,"ved":504,"ve ":854,"val":8945,"vak":669,"van":2026,"vam":407,"vap":226,"var":1747,"vat":2727,"vas":3460,"vav":166,"vaa":1097,"vab":718,"vae":151,"vad":3860,"vai":1190,"vaj":399,"vag":98,"vah":2967,"va ":3296,"uvõ":268,"uvä":105,"usõ":228,"usü":545,"uuk":89,"uun":1174,"uul":3063,"uum":754,"uub":214,"uua":104,"uug":75,"uud":1301,"uue":262,"ux ":63,"uus":2149,"uur":5973,"uup":59,"uuv":44,"uut":1103,"uvi":326,"uvo":58,"uva":2014,"uve":246,"uvu":186,"usl":1435,"usm":668,"usj":225,"usk":1361,"ush":270,"usi":2532,"usf":104,"usg":54,"usd":49,"use":12729,"usc":44,"usa":1306,"uu ":547,"uv ":1217,"usv":989,"usu":604,"ust":10829,"uss":984,"usr":193,"usp":536,"uso":240,"usn":79,"utk":47,"utl":151,"utm":97,"utn":50,"uth":113,"upä":61,"uti":1297,"ute":1328,"uta":4946,"utz":51,"upõ":82,"utt":121,"uts":1366,"utv":93,"utu":2313,"uto":959,"utr":121,"us ":14520,"umä":125,"ut ":606,"urb":400,"ura":699,"urd":430,"ure":1850,"urg":757,"urj":163,"uri":3709,"url":67,"urk":267,"urm":383,"urn":372,"uro":1163,"urr":134,"urs":299,"urt":336,"uru":948,"urv":239,"uol":59,"uot":45,"uor":83,"uos":126,"ukü":84,"upa":311,"ur ":1500,"upi":356,"upe":239,"upo":146,"upp":283,"upr":65,"upl":50,"upu":118,"ump":59,"umu":239,"umi":3810,"umm":293,"umo":96,"uma":2384,"umb":1389,"ume":800,"unt":871,"uns":729,"unu":969,"unk":808,"uni":2914,"uno":48,"unn":618,"unc":54,"und":1856,"una":2903,"ung":491,"ujä":52,"une":1105,"up ":125,"uks":640,"ukr":117,"uku":265,"ukt":617,"uko":1820,"ukk":131,"ukl":81,"uki":384,"uke":268,"um ":1368,"uka":800,"uju":1494,"ulu":2649,"ult":2105,"uls":130,"ulp":70,"ulo":139,"uln":45,"ulm":184,"ull":465,"ulk":645,"ulj":361,"uli":2783,"ulg":972,"ule":2072,"uld":337,"ula":1652,"un ":198,"uid":849,"uie":51,"uig":91,"uil":85,"uim":112,"uin":284,"uis":365,"uht":1356,"uhu":769,"uk ":199,"uiv":119,"uit":151,"ul ":2180,"uja":426,"ugh":62,"ugi":162,"uge":590,"ugo":56,"ugl":46,"ui ":1419,"uga":910,"uhi":560,"uhe":222,"uho":49,"uhk":168,"ugu":2195,"ugr":50,"uha":362,"uct":46,"uda":566,"ude":2760,"udi":687,"udm":209,"ubu":163,"uca":46,"ue ":313,"uci":50,"uch":100,"uck":66,"uet":68,"uer":113,"ues":193,"püü":159,"ufo":48,"ufi":80,"udu":959,"uds":80,"udt":385,"udo":126,"ug ":76,"udw":51,"uee":109,"uen":148,"uel":155,"pöö":270,"ub ":3808,"põõ":56,"ua ":229,"uat":112,"uas":207,"püs":236,"uar":1172,"pür":52,"uam":44,"ual":222,"uan":196,"ubi":508,"ubj":82,"ubl":189,"ube":338,"uba":529,"ud ":15469,"uak":53,"püh":249,"uai":64,"uad":62,"uaa":347,"tvõ":44,"tvä":73,"tzi":57,"tze":47,"põh":2641,"põi":53,"põl":539,"põr":58,"ty ":254,"trü":150,"tvu":92,"tvo":55,"tve":93,"tva":221,"tur":775,"tus":7900,"tut":545,"tuu":2287,"tuv":618,"tuj":83,"tui":121,"tul":1640,"tuk":346,"tun":1878,"tum":1084,"tup":139,"tub":376,"tua":183,"tud":7345,"tuh":72,"tug":512,"tsü":333,"tz ":121,"tsõ":45,"ts ":1169,"tre":529,"tt ":402,"tra":1815,"tri":3128,"tru":752,"tro":1685,"tu ":1887,"tsa":1214,"tse":5653,"tsc":70,"tsi":9068,"tsj":165,"tsm":159,"tsk":220,"tsl":83,"tso":358,"tsu":2329,"tst":212,"tta":440,"tte":2371,"tti":343,"ttl":46,"tto":143,"ttp":55,"tts":70,"ttu":276,"tme":1276,"tma":236,"to ":641,"tmu":55,"tmo":97,"tmi":756,"tni":437,"tne":300,"tp ":49,"tna":202,"tnu":77,"tno":51,"tof":48,"toe":213,"tod":315,"toc":161,"toi":1340,"toh":46,"tog":150,"toa":50,"tov":120,"tos":269,"tot":150,"tom":456,"ton":1159,"tok":224,"tol":1335,"tor":2218,"too":2528,"top":205,"tkü":52,"tij":48,"tii":1991,"til":2148,"tik":5067,"tif":152,"tie":142,"tih":290,"tig":255,"tir":123,"tit":780,"tis":3212,"tin":1198,"tim":859,"tip":427,"tio":925,"thu":237,"tia":704,"tic":223,"tid":1567,"tiu":100,"tiv":496,"tja":582,"tki":109,"tko":214,"tku":135,"tka":199,"tke":169,"tli":1813,"pää":190,"tlu":1138,"tla":898,"tle":1553,"tem":2042,"ten":994,"teo":974,"tep":235,"tei":2648,"tej":88,"tek":2590,"tel":4094,"tee":4577,"tef":53,"teg":3418,"teh":952,"tea":2198,"tec":57,"ted":509,"tfo":51,"tfa":78,"th ":373,"tev":754,"tet":709,"tes":4048,"ter":4969,"ti ":9417,"tho":135,"thr":60,"pär":1925,"päe":892,"the":496,"thi":183,"päi":92,"tha":245,"ān ":65,"üüp":380,"üür":188,"üüs":557,"üüt":421,"üüa":70,"üüb":92,"üüd":395,"üüg":68,"üüh":62,"üüm":176,"üül":234,"üün":56,"並 ":57,"žik":69,"žis":93,"三 ":165,"žii":61,"žan":88,"丁 ":49,"žaa":84,"ži ":55,"üve":65,"ürs":166,"ürt":60,"üro":72,"ürk":57,"ürg":241,"üri":378,"üre":55,"üra":58,"üs ":51,"ütt":66,"ütu":70,"üti":151,"ütl":183,"ütm":69,"üto":339,"üta":158,"üte":90,"üss":79,"üst":1343,"üsi":1015,"之 ":88},"n_words":[4341644,4941492,4175920],"name":"et","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/eu.json b/contrib/languages-data/eu.json
new file mode 100644
index 0000000..ca99f7e
--- /dev/null
+++ b/contrib/languages-data/eu.json
@@ -0,0 +1 @@
+{"freq":{"D":12636,"E":31626,"F":41733,"G":24671,"A":49254,"B":32314,"C":24793,"L":25422,"M":31193,"N":19703,"O":18871,"H":20234,"I":17795,"J":9363,"K":12674,"U":5646,"T":16850,"W":3928,"V":9991,"Q":1167,"P":21838,"S":30092,"R":9657,"Y":2494,"X":2077,"Z":5511,"f":39789,"g":194786,"d":496450,"e":1172759,"b":212072,"c":39575,"a":1770525,"n":845494,"o":639351,"l":395235,"m":170469,"j":17374,"k":463043,"h":111048,"i":893736,"w":6134,"v":18456,"u":464998,"t":693891,"s":298290,"r":847892,"q":2827,"p":125668,"z":349493,"y":24758,"x":24609,"²":121,"´":110,"É":768,"Á":135,"í":1797,"ì":278,"ë":206,"ê":267,"é":6456,"è":4480,"ç":407,"ä":628,"â":543,"á":1849,"à":612,"ü":636,"ú":522,"ù":141,"ö":568,"ô":3132,"ò":128,"ó":1337,"ñ":2145,"ē":126,"ā":699,"ī":353,"ō":492,"š":115,"ū":427,"ǎ":126,"́":383,"μ":233,"ν":549,"ο":715,"ι":506,"κ":273,"λ":400,"δ":171,"ε":392,"η":259,"α":749,"γ":157,"ά":203,"ί":245,"ω":158,"ό":170,"σ":279,"ς":638,"ρ":505,"π":199,"φ":112,"υ":181,"τ":391," l":25023,"ь":254," m":32235," n":11168," o":23087,"я":187," h":48767," i":53849," j":11234,"ы":136," k":40202," d":213721," e":179842," f":18132," g":36773,"ч":225,"р":831," a":103410,"с":671," b":117209,"т":523," c":1597,"у":285," y":441," x":499," z":66161," u":47027," t":18277," w":614," v":880," q":216," p":25257," s":29279," r":2003,"К":143,"А":116,"В":111," J":9317," K":12636," H":20099," I":17702," N":19654," O":18798," L":25277," M":31072," B":32085," C":24661," A":48997,"С":176," F":41666," G":24576," D":12574," E":30981,"л":615," Z":5479,"к":647," Y":2491,"й":238," X":2059,"и":1033,"п":143,"о":1160,"н":760,"м":252," S":29950,"г":226," R":9631,"в":628," Q":1162,"б":218," P":21762,"а":1315," W":3894," V":9966," U":5599,"е":865," T":16758,"д":267," Á":134," É":748,"ו":119,"י":145,"و":342,"ي":584,"ف":132,"ق":122,"ل":670,"م":459,"ن":404,"ه":130,"د":321,"ج":158,"ح":170,"ت":166,"ب":429,"ة":205,"ا":1078,"ع":171,"ش":113,"س":238,"ز":110,"ر":498," А":116," В":111," К":143,"A ":2297,"F ":196,"Da":1856,"Ct":233,"Cu":578,"Cy":474,"Cl":878,"Cn":233,"Co":5762,"Cr":1202,"Ce":1047,"Ch":5833,"Ci":545,"G ":191,"Ec":112,"Ed":572,"Ea":166,"Eb":131,"Du":1120,"Do":3494,"Dr":849,"De":2359,"Di":1558,"Fe":1310,"H ":330,"Fa":940,"Ez":530,"Eu":5638,"Ev":124,"Ex":216,"Er":7909,"Et":707,"Es":7502,"En":1222,"Em":589,"Ep":195,"Ei":284,"El":2242,"Ek":1089,"Eg":1307,"Ge":4595,"Ga":8150,"I ":1890,"Fu":637,"Fr":34398,"Fo":1536,"Bé":204,"Fl":778,"Fi":1608,"B ":768," С":176,"C ":551,"Av":693,"Au":3356,"Ar":7948,"At":1721,"As":1845,"D ":286,"Ba":9918,"Az":821,"Ay":124,"Af":752,"Ag":995,"Ah":190,"Ab":1086,"Ac":453,"Ad":9951,"Am":3102,"An":4722,"Ap":642,"Ai":1869,"Ak":667,"Al":6525,"Bu":1690,"Br":3204,"Ca":6336,"E ":494,"Bi":4638,"Be":8067,"Bo":3491,"Bl":641,"Ku":829,"Kl":325,"Kr":801,"Ko":3438,"Le":5234,"Li":3143,"N ":436,"La":7002,"Lu":2001,"Ly":440,"Hé":453,"Ll":255,"Lo":6499,"Me":5309,"Mi":2877,"O ":483,"Ma":13695,"Mu":2429,"Iñ":132,"Mo":5680,"Ni":1359,"Ne":2222,"Na":11411,"P ":389,"Nu":354,"No":2161,"Ok":8891,"Ol":1020,"Om":122,"On":616,"Oh":168,"Oi":1025,"Od":228,"Of":128,"Ob":229,"Gi":3338,"Gl":406,"Gr":2264,"Go":2393,"Gu":2262,"Gy":276,"Cô":768,"Gw":139,"J ":172,"Ha":6267,"He":6657,"Hi":2597,"Ho":2280,"Hu":1033,"Hy":385,"Dô":481,"K ":469,"Ib":698,"Id":259,"Ig":331,"ा":121,"Ip":1284,"Im":160,"In":4524,"Ik":340,"Il":578,"Iv":141,"Is":1295,"It":2278,"Ir":2852,"Ja":2843,"L ":365,"Iz":862,"Ji":328,"Je":1151,"Jo":2759,"Fé":111,"Ju":1950,"Ka":4370,"M ":322,"Kh":198,"Ki":1106,"Ke":755,"Us":185,"Ut":139,"Ur":1743,"Un":1309,"Uk":242,"Ul":214,"Uh":348,"Ud":554,"Ty":618,"Tx":2595,"Tu":924,"Tr":2492,"Pé":178,"To":2471,"Th":1746,"Ti":995,"Te":1911,"Ta":2499,"V ":486,"Sy":260,"St":1667,"Su":1641,"Wo":340,"Wi":1463,"Wh":118,"Sé":155,"Sè":286,"Wa":880,"We":721,"Vo":900,"Vi":3787,"Ré":151,"X ":613,"Va":2746,"Ve":1775,"Uz":154,"Mé":297,"Qi":124,"Pt":120," م":129,"Pu":1219,"Pr":2635,"Ps":373,"S ":435,"Py":117,"Pe":2734,"Pa":5996,"Lé":212,"Pl":1086,"Po":3314,"Pi":2764,"Ph":767,"Os":761,"Ot":444,"Ou":336,"Ov":109," ا":406,"Op":333,"Or":2924,"R ":202," ب":155,"Ox":154,"Oz":622,"Se":3621,"Sc":1754,"Si":1956,"Sh":1018,"Sl":127,"Sk":128,"Sp":738,"So":2835,"Ru":564,"U ":162,"Sa":12778,"Re":1715,"Ri":1242,"Rh":1366,"Ro":2679,"Qu":659,"T ":234,"Ra":1463,"b ":834,"a ":418310,"Ye":164,"Ya":233,"Yo":1349,"Yv":208,"Yu":192,"Qū":205,"Xe":154,"Xa":263,"Xi":357,"Za":1318,"Ze":1350,"Zh":191,"Zi":1026,"Zo":305,"Zu":1064,"i ":75419,"gd":272,"ge":26042,"ga":31469,"gb":147,"bé":122,"fl":595,"fg":114,"ff":773,"fi":7825,"fr":4122,"fu":1726,"ft":461,"fo":4955,"j ":153,"gy":318,"gz":158,"he":25799,"ha":32541,"gn":3188,"gm":257,"gl":1391,"gi":27050,"gh":1165,"gg":388,"gu":20475,"gt":292,"gs":391,"gr":5318,"cí":127,"go":71226,"dt":177,"du":58038,"dw":263,"dy":552,"g ":4240,"ea":90696,"eb":7278,"ec":2369,"ed":20845,"de":134965,"dd":530,"dg":314,"di":46366,"dh":268,"dm":9929,"dl":187,"do":27634,"dn":167,"ds":538,"dr":5710,"ew":1215,"aô":1078,"ex":2335,"añ":358,"eu":10850,"ev":2468,"ey":2501,"ez":30732,"fa":13204,"h ":3096,"fe":4135,"eh":10866,"eg":27624,"ef":2297,"ee":7857,"el":32829,"ek":58090,"ej":999,"ei":12396,"ep":39259,"eo":5671,"en":302618,"em":15409,"et":97162,"es":80312,"er":181118,"aí":122,"ca":4335,"e ":126190,"by":276,"bs":905,"br":4958,"bu":10932,"bn":150,"bo":17651,"bj":339,"bl":4003,"bg":166,"bh":132,"bi":50340,"bb":248,"bd":240,"be":37939,"da":201401,"f ":1461,"cy":423,"cu":1624,"ct":2467,"cs":124,"cq":317,"cr":939,"co":4763,"ck":1858,"cl":657,"ci":5204,"ch":8436,"ce":4449,"cc":457,"c ":2780,"aB":147,"aE":312,"az":38405,"ay":2915,"ba":82298,"d ":7993,"at":138353,"as":38078,"ar":234987,"aq":229,"ax":1187,"aw":605,"av":3400,"au":28656,"ak":137603,"al":150790,"ai":71296,"aj":772,"ao":1106,"ap":9383,"am":53259,"an":300148,"ac":7614,"ad":15049,"aa":638,"ab":24555,"ag":67873,"ah":5467,"ae":11540,"af":5163,"nu":3790,"nt":87610,"ns":6135,"nr":517,"nq":161,"np":6374,"no":18233,"nn":4720,"q ":190,"nz":1004,"ny":2628,"nx":162,"nw":144,"nv":608,"oe":35138,"of":3518,"oc":3225,"od":6707,"oa":47947,"ob":8716,"om":15442,"on":56967,"ok":34406,"ol":29688,"oi":15827,"oj":392,"og":7848,"oh":2732,"hō":150,"m²":115,"ot":18274,"os":23317,"ov":2405,"ou":8969,"op":11236,"oo":1321,"or":55477,"oq":186,"r ":35138,"ox":827,"ow":1113,"oz":6401,"oy":1012,"lá":134,"pe":17100,"pa":48349,"pl":4223,"lè":695,"lé":417,"po":11630,"ph":5789,"pi":13999,"lo":23311,"ln":225,"lm":2191,"hé":316,"ll":14813,"ls":1249,"lp":1461,"lw":166,"lv":1388,"lu":13777,"lt":11988,"lz":205,"ly":1619,"hô":322,"o ":239621,"ià":195,"ma":42350,"mb":1990,"mg":110,"me":60276,"iá":151,"hā":111,"iè":1602,"mi":30640,"mn":549,"mm":1879,"ié":152,"mp":2538,"mo":12040,"mt":139,"ms":406,"mu":11394,"iñ":300,"ió":290,"iö":115,"my":536,"p ":1016,"na":89833,"nb":5719,"nc":5864,"nd":77531,"ne":54150,"nf":1744,"ng":20200,"jä":211,"nh":729,"ni":56445,"nj":622,"nk":3770,"nl":20585,"nm":259,"ju":1296,"jo":3182,"ki":41699,"kh":562,"ke":15259,"ka":61958,"m ":4321,"ky":241,"kz":9751,"ks":428,"kt":5287,"ku":47447,"ko":209669,"kr":3078,"kk":1095,"kl":2132,"km":634,"kn":545,"li":47744,"lh":523,"lk":11786,"lj":263,"le":105272,"ld":63524,"lg":2023,"hâ":367,"lf":1145,"la":65016,"lc":483,"lb":3705,"n ":374467,"hr":1439,"hs":188,"hw":153,"ht":906,"hu":4025,"hi":26055,"hn":657,"ho":11118,"hl":1392,"dé":379,"hm":308,"id":28391,"ic":6403,"ib":22289,"ia":177930,"ih":1356,"ig":10103,"if":3391,"ie":37824,"hy":1010,"k ":62634,"iq":528,"ir":40658,"is":41795,"it":88626,"iu":1861,"eñ":229,"iv":1736,"iw":111,"ix":1809,"ii":901,"ij":1221,"ik":65069,"il":61174,"im":14806,"in":101260,"io":30374,"ip":9370,"je":1468,"ji":1295,"iz":65709,"iy":270,"l ":19248,"ja":9289,"xk":501,"xi":8569,"xo":2798,"té":228,"tí":194,"xt":269,"xu":1487,"wy":210,"ww":215,"z ":36063,"xa":3213,"xe":3776,"wi":721,"sè":421,"sé":400,"wn":243,"wo":287,"ws":184,"vy":165,"rô":347,"y ":9063,"wa":1578,"we":945,"rè":474,"ré":955,"vi":5771,"vu":143,"vr":1081,"vs":130,"rí":340,"vo":1538,"uz":19517,"uy":977,"ux":2209,"uv":1191,"ve":4565,"va":4080,"x ":3431,"ui":4962,"uj":404,"uk":15052,"ul":13460,"ue":46850,"uf":629,"ug":6475,"uh":2527,"ur":58487,"us":31658,"ut":35746,"um":6170,"un":31293,"uo":392,"up":1366,"ty":2264,"tx":11953,"tz":106075,"tu":90564,"tt":2560,"tw":311,"ub":8862,"ua":92243,"ud":39922,"uc":1798,"w ":1243,"to":23073,"tn":499,"tm":498,"tl":1525,"ts":16546,"tr":26770,"tp":196,"tg":181,"tf":199,"te":85168,"ti":60433,"th":5295,"v ":481,"tb":891,"tc":244,"ta":203261,"su":14353,"oñ":152,"sv":152,"ss":4629,"st":55835,"sy":515,"sz":245,"sw":227,"sl":3070,"sk":52536,"sn":1906,"sm":3054,"sp":8612,"so":18734,"sr":212,"sq":219,"sd":220,"sc":2520,"sf":603,"se":20022,"sh":2489,"sg":1046,"sj":125,"si":25419,"rz":597,"u ":41051,"sa":30547,"sb":1326,"rr":133906,"rs":4346,"rt":84154,"ru":29410,"rv":1353,"rw":161,"rx":118,"ry":1952,"rq":209,"rp":2429,"ro":53014,"rn":9800,"né":358,"rm":7397,"rl":6227,"rk":10937,"nç":272,"rj":253,"ri":132692,"rh":504,"rg":8992,"rf":596,"ná":137,"re":124545,"rd":16275,"rc":2417,"rb":5362,"ra":172109,"t ":54109,"qu":2270,"mé":209,"qi":160,"s ":48513,"py":188,"pz":138,"pt":2020,"pu":7547,"pp":706,"lí":113,"pr":9929,"ps":2339,"zè":218,"zz":285,"zg":277,"zh":568,"zi":104084,"zb":829,"ze":72397,"za":55517,"zy":159,"zu":15760,"zt":33176,"zo":6743,"zp":2277,"zk":17416,"zl":2872,"yg":376,"ye":1014,"yc":830,"་":163,"yd":772,"ya":1650,"yu":249,"uñ":753,"yt":384,"ys":908,"yr":1417,"yp":1406,"yo":623,"yn":833,"ym":693,"ué":150,"yl":2141,"yk":360,"yi":1221,"Éc":146,"Ét":205,"àn":218,"án":687,"ái":148,"ál":138,"ár":132,"ât":411,"à ":197,"á ":169,"アアア":219,"ö ":127,"ôm":838,"ôn":1427,"ón":676,"ó ":189,"ña":982,"ñe":634,"ño":321,"ín":439,"ío":159,"ís":174,"ía":492,"í ":198,"éz":196,"él":236,"éo":163,"ém":241,"én":464,"és":320,"ét":218,"ér":1074,"év":257,"èz":354,"éa":185,"éd":183,"éc":258,"ée":453,"ég":228,"èn":113,"èr":1733,"ès":933,"èv":648,"èg":419,"ço":189,"ça":162,"é ":1538,"är":266,"ān":289,"ür":178,"ú ":187,"ôt":781,"ī ":109,"īn":117,"ōn":111,"ō ":172,"ū ":285,"ア":394,"乙":208,"之":886,"丹":146,"临":277,"並":364,"丘":302,"专":624,"三":1377,"丁":715,"万":373," 丘":117," 专":142," 三":359," 丁":261,"倉":119," 之":199,"大":136,"ος":294,"ος ":290,"ς ":631,"ν ":159,"α ":219,"アア":304,"ск":207,"ст":182," Ga":8127," Ge":4583," I ":500," Fo":1533," Fu":636," Fr":34376," Fi":1595," Fl":773," Bé":203," Ha":6216," He":6623," Cô":768," Gw":139," Gy":275," J ":124," Go":2376," Gr":2249," Gu":2252," Gi":3327," Gl":404," Ig":331," Id":259," Ib":692," K ":344," Hy":384," Dô":481," Hu":1024," Ho":2274," Hi":2583," Ji":326," Je":1147," L ":160," Ja":2821," Iz":849," Iv":140," Ir":2841," Is":1292," It":2274," Im":160," In":4497," Ip":1284," Ik":324," Il":575," M ":119," Ka":4363," Ke":748," Ki":1096," Kh":198," Fé":110," Jo":2739," Ju":1944," N ":227," La":6984," Le":5208," Li":3050," Kl":325," Ko":3426," Kr":791," Ku":827," Ma":13659," O ":208," Mi":2855," Me":5297," Lo":6484," Hé":453," Ll":255," Ly":440," Lu":1998," Ne":2204,"а ":256," Na":11386," Ni":1355," Mo":5669," Mu":2390," Iñ":129," A ":442," B ":131," C ":208," Ap":641," Am":3098," An":4660," Ak":544," Al":6496," Ai":1863," Ag":994," Ah":190," Af":752," Ac":451," Ad":9947," Ab":1037," Ba":9875," Az":814," Ay":124," Av":691," Au":3286," At":1715," As":1839," Ar":7825," Be":7803," Bi":4537," Bl":636," Bo":3476," Br":3193," Bu":1682," Ca":6284," Ce":1039," Ci":542," Ch":5831," Cn":231," Cl":859," Cr":1191," Co":5729," Cu":573," Ct":233," Cy":473," Da":1841," Di":1550," De":2338," Dr":847," Do":3472," Du":1118," Ea":161," Eb":127," Ec":112," Ed":572," El":2228," Ek":1058," Ei":282," Eg":1300," Et":695," Es":7460," Er":7880," Ep":193," En":1209," Em":587," Ez":526," Ex":211," Eu":5218," Ev":121," Fe":1297," Fa":936," H ":110," Xi":356," Xe":150," Xa":262," Wo":331," Sé":155," Sè":285," Wi":1455," Wh":115," We":715," Wa":873,"й ":179," Zu":1054," Zo":304," Ze":1333," Zh":191," Zi":1025," Za":1318," Qū":205," Yv":208," Yu":192," Yo":1348," Ya":232," Ye":164,"н ":112," a ":8305,"в ":122," Oz":621," Ox":152," Ou":336," Ov":109," Os":758," Ot":443," Or":2903," Op":333," Po":3289," Lé":212," Pl":1082," Pi":2760," Ph":764," Pe":2717," Pa":5978," Nu":351," No":2154," Ol":1015," Ok":8850," On":612," Om":121," Oh":168," Oi":1023," Od":227," Of":119," Ob":227," Ra":1451," Qu":655," Ro":2671," Re":1711," Ri":1240," Rh":1366," Py":112," Pr":2625," Ps":372," Pt":120," Pu":1219," Qi":123," Mé":297," Sy":259," Su":1637," St":1657," Ta":2484," V ":128," Th":1731," Ti":988," Te":1895," Tr":2477," To":2450," Pé":178," Ru":561," Sa":12748," Sh":1015," Si":1935," Sc":1747," Se":3599," So":2830," Sp":735," Sk":123," Sl":127," Uz":153," Va":2741," X ":118," Ve":1772," Vi":3778," Ré":151," Vo":900," Tu":917," Ty":618," Tx":2595," Ud":553," Uh":347," Uk":211," Ul":214," Un":1298," Ur":1698," Us":185," Ut":139," ja":7967," l ":317," iz":22098," je":446," ip":2762," im":117," in":7560," ik":2938," il":408," is":831," it":3666," ir":5256," ka":6507," m ":200," ki":4014," ke":272," jo":2357," ju":380," ha":15787," he":11845," gi":2915," gl":237," gr":2766," go":3412," gu":3064," ib":2762," ia":184," id":4126," ih":124," ig":680," hi":13695," ho":6235," hu":1046," ni":298," ne":1575," na":6665," mu":7219," mo":3550," ok":389," ol":651," om":239," on":4235," oh":1483," oi":1278," od":151," of":1484," ob":692," nu":269," no":1940," le":6898," li":3011," n ":330," la":8148," ku":1542," km":546," kl":781," kr":1091," ko":25212," me":9646," mi":3328,"я ":139," ma":8141," lu":3558," lo":2463," af":170," ag":1571," ah":1271," ab":5206," ad":2544," am":1664," an":37224," ap":2038," ai":1625," ak":945," al":7726," au":6652," ar":17725," at":1816," as":2486," d ":1527," ba":61785," az":4149," bi":31183," be":18081," bo":1808," bl":186," bu":3139," br":903," ca":373," e ":115," er":27809," et":48874," es":44512," en":3242," em":2495," ep":416," el":3773," ek":9675," eh":379," eg":10971," fe":664," fa":8966," ez":8057," eu":4631," ex":330," fu":1398," fr":2586," fo":1255," fl":294," fi":2901," ge":12607," ga":11697," i ":197," co":450," cr":121," ch":186," da":131136," do":2083," dr":141," de":49829," di":15772,"ч ":164," ed":12800," ea":1283," eb":259," du":13144," zo":627,"ла":160," zu":10484," za":4120,"ко":120," ze":24755," zi":26100,"ка":189,"ин":111,"ич":173," xe":141," xa":124,"ро":162,"ра":136,"ов":295,"но":113," ru":136," sa":10788," se":2907," sc":161," si":3486," sh":276," sp":132," so":7153," qu":182,"ви":184," ra":630," re":402," ri":131," ro":580," pu":1255," pr":7926," ps":237," s ":207," os":6513," ot":1043," op":380," or":3982,"ан":144," ox":142," oz":163," pe":3345,"ас":113," pa":3424," pl":1311," lè":553," po":4955," pi":2498," we":277," y ":323," x ":114," va":276," ve":153," uz":1119," vo":221," vi":176," ud":33375,"ер":141," tx":4570," tu":424," us":349," ut":439," ur":7726," um":156," un":1012," uk":258," ul":113," ug":588," uh":1778," ta":4238," st":216," su":3722,"ев":111," tr":2688," to":1288," th":540," ti":1534," te":2817," Ét":203," Éc":146,"가":192,"د ":145,"ة ":203,"ان":117,"ال":397,"ي ":165,"ن ":205,"가가":128,"Fed":255,"Fel":133,"Fen":110,"Fer":565,"Fil":503,"Fin":531,"Fis":168,"Ezk":195,"Fal":128,"Far":152,"Era":611,"Erd":679,"Ere":283,"Eri":170,"Erk":205,"Ess":209,"Est":2782,"Esp":2765,"Err":5471,"Ern":111,"Esc":150,"Esk":1235,"Eus":2856,"Eur":2345,"Etx":413,"Ez ":126,"El ":382,"Ele":228,"Ela":199,"Eki":655,"Eko":224,"Emi":244,"Ema":110,"Elo":131,"Eli":450,"Elk":412,"Ent":296,"Enp":155,"Eno":127,"Gek":939,"Gel":126,"Geh":141,"Ger":1702,"Get":138,"Geo":759,"Gen":514,"Gla":168,"Gin":141,"Gip":1442,"Gil":209,"Gir":724,"Giz":302,"Gaz":575,"Gan":238,"Gal":1278,"Gam":113,"Gau":462,"Gar":4067,"Gas":607,"Gai":282,"Gab":184,"Fut":140,"Fun":140,"Fro":202,"Flo":355,"Fla":260,"Fra":33371,"Fri":275,"Fre":391,"Fon":318,"For":678,"Fou":190,"II ":948,"Hir":971,"Hit":208,"His":306,"Hiz":289,"Hig":126,"Hil":265,"Hip":128,"Heg":983,"Hel":358,"Hei":184,"Hez":131,"Hem":285,"Hen":358,"Hes":115,"Her":3806,"Hal":393,"Hai":353,"Han":786,"Ham":279,"Has":208,"Har":1000,"Hau":2610,"Gym":213,"Côt":760,"Gur":199,"Guz":741,"Gua":381,"Gue":128,"Gui":276,"Gre":836,"Gri":154,"Gra":882,"Gro":247,"Glo":115,"Gon":350,"Gol":169,"Goi":319,"Got":125,"Gou":140,"Gor":655,"Gob":162,"Ing":1090,"Inf":149,"Inp":655,"Int":602,"Ins":196,"Ipa":1240,"Ill":286,"Ind":1394,"Ika":163,"Ign":110,"Ida":112,"Iba":263,"Ibe":221,"Hyd":269,"Dôm":479,"Hur":125,"Hun":272,"IX ":263,"IV ":188,"Hor":488,"Hou":158,"Hot":121,"Hom":153,"Hon":528,"Hol":339,"Arg":594,"Are":343,"Arc":161,"Ard":862,"Ara":1880,"Arb":146,"Aro":492,"Arm":450,"Arn":154,"Ark":223,"Ari":603,"Apo":201,"Ate":172,"Ath":126,"Atl":685,"Atr":335,"Ast":557,"Ass":118,"Ata":133,"Ash":115,"Asi":444,"Ask":171,"Asp":121,"Arr":879,"Art":881,"Avi":122,"Ave":440,"Auz":146,"Aut":408,"Aus":617,"Aur":352,"Aud":619,"Aug":208,"Aub":618,"Aza":115,"Azp":147,"Azk":248,"Bai":588,"Bak":203,"Bal":660,"Ban":555,"Bad":228,"Baz":195,"Bay":133,"Bar":2121,"Bat":3332,"Bas":1042,"Bav":137,"Bau":120,"Aca":159,"Abe":336,"Aba":192,"Ada":158,"Adm":9191,"Adi":259,"Aga":387,"Afr":600,"Agu":144,"Agi":187,"Ain":131,"Ais":678,"Air":369,"Ait":206,"Aiz":109,"Al ":169,"Aka":163,"Aki":200,"Ala":306,"Alb":472,"An ":143,"Alg":117,"Ali":138,"Alj":125,"Ald":917,"Ale":1684,"Alf":227,"Alt":401,"Alm":128,"All":581,"Alo":136,"Alp":727,"Ame":1656,"Ami":117,"Ama":372,"Amo":119,"Amp":481,"Ang":544,"Ana":309,"And":973,"Ant":1538,"Ano":454,"Ann":178,"Apa":132,"Bus":154,"Bul":138,"Bur":532,"Bud":110,"Bue":186,"Bru":377,"Cab":148,"Cad":128,"Cal":1977,"Cam":634,"Cai":113,"Cas":692,"Car":985,"Cau":156,"Cat":248,"Can":647,"Cap":209,"Bea":612,"Bes":339,"Bet":367,"Ber":3897,"Ben":412,"Bel":1201,"Bei":190,"Beh":504,"Bid":263,"Bie":141,"Bib":135,"Bia":112,"Biz":1309,"Bil":1194,"Big":333,"Bis":146,"Bir":303,"Bio":163,"Bi ":209,"Blo":124,"Bla":371,"Bre":742,"Bra":786,"Bro":433,"Bri":733,"Bol":414,"Boi":402,"Bon":360,"Bor":485,"Bos":337,"Bot":216,"Bou":715,"Cyr":266,"Cur":113,"Der":132,"Des":202,"Deu":416,"Del":171,"Dem":199,"Den":399,"Deb":142,"EB ":427,"Dam":258,"Dan":487,"Dar":179,"Dat":205,"Dav":212,"EBe":213,"Dal":132,"Cho":209,"Chr":241,"Che":1052,"Chi":641,"Cit":117,"Châ":343,"Cle":128,"Cla":350,"Cel":174,"Cen":305,"Cer":368,"Cha":3073,"Cri":128,"Cra":131,"Cre":406,"Cro":283,"Cte":231,"Clu":134,"Cne":218,"Cos":145,"Cor":1166,"Com":567,"Col":1978,"Con":778,"Cou":568,"Drô":329,"Egu":578,"Egi":536,"Edi":139,"Dia":142,"Dis":216,"Dip":284,"Dio":112,"Din":121,"Die":165,"Dub":337,"Dun":167,"Dur":225,"Dra":223,"Dou":638,"Don":1289,"Dom":330,"Dor":804,"Neu":356,"Ner":122,"Nat":287,"Nav":137,"Naz":944,"Nig":111,"Nic":160,"Nik":161,"Nil":110,"New":858,"Nap":194,"Nar":6638,"Nan":255,"Nag":310,"Naf":1923,"Niè":290,"Nov":113,"Nor":1256,"Not":129,"Nob":153,"Ois":735,"Ohi":123,"Okz":8717,"Oze":547,"Oli":468,"Ola":182,"Ono":139,"Ond":186,"Or ":694,"Ope":128,"Ora":139,"Ort":156,"Ore":134,"Ord":215,"Ori":184,"Org":128,"Orn":527,"Oro":257,"Oso":146,"Ple":223,"Pla":722,"Pin":245,"Pil":160,"Pir":1128,"Phy":140,"Pie":535,"Pic":114,"Pia":119,"Phi":231,"Phr":148,"Ped":255,"Per":814,"Pet":525,"Pen":376,"Pel":246,"Pat":265,"Pas":1075,"Par":2304,"Pau":404,"Pac":140,"Pan":351,"Pal":580,"Pue":117,"Puy":565,"Pro":1149,"Pri":464,"Pre":388,"Pru":183,"Pse":223,"Pra":311,"Pol":1076,"Pom":111,"Pon":385,"Poi":164,"Pot":148,"Pos":133,"Pou":137,"Pop":124,"Por":717," ال":346,"Rai":109,"Ram":376,"Ran":155,"Qui":200,"Que":317,"Irl":1084,"Iri":152,"Iru":781,"Ita":1023,"Isl":300,"Isr":115,"Ist":111,"Ira":616,"Isè":386,"Itu":232,"Its":696,"Itz":194,"Iza":331,"Ize":360,"Jac":241,"Jav":138,"Jau":467,"Jat":166,"Jar":142,"Jap":555,"Jan":172,"Jam":243,"Jak":114,"Jai":339,"Jer":194,"Jes":231,"Jea":485,"Jos":894,"Jor":132,"Jon":241,"Jok":251,"Joh":504,"Joa":335,"Jua":434,"Jus":123,"Jur":582,"Jul":330,"Jun":115,"Kai":138,"Kam":110,"Kal":524,"Kap":120,"Kan":885,"Kat":1063,"Kas":158,"Kar":856,"Kaz":123,"Ker":134,"Ken":177,"Kem":181,"Kir":246,"Kin":214,"Kil":130,"Kim":109,"Kla":112,"Kon":1335,"Kom":296,"Kol":432,"Kos":152,"Kor":646,"Kop":181,"Kre":127,"Kri":277,"Kro":231,"Kul":139,"Kur":185,"Let":186,"Les":516,"Ler":221,"Lep":592,"Leo":391,"Len":128,"Lem":125,"Lek":123,"Lei":412,"Leg":497,"Leh":498,"Lea":127,"Lau":560,"Lav":171,"Le ":868,"Lak":109,"Lag":212,"Las":259,"Lat":204,"Lar":562,"Lap":493,"Lam":255,"Lan":1366,"Lac":384,"Lab":307,"MA ":123,"La ":1524,"Hér":386,"Lib":309,"Lif":129,"Lie":116,"Lig":406,"Lim":186,"Lin":293,"Lio":514,"Lis":187,"Lit":211,"Liv":131,"Liz":228,"Luz":124,"Lur":628,"Lui":341,"Luc":199,"Loz":203,"Lou":492,"Los":296,"Lot":763,"Loi":2917,"Log":117,"Lor":274,"Lon":848,"NA ":1300,"Lyg":192,"Men":1204,"Mel":250,"Mes":401,"Mer":455,"Meu":1170,"Met":319,"Med":480,"Mex":492,"Man":1508,"Mal":618,"Mar":6226,"Mas":420,"Mag":310,"Mad":1098,"Mak":109,"Mah":123,"Mai":913,"Mac":300,"May":341,"Max":125,"Maz":151,"Mau":432,"Mat":548,"Mod":145,"Mol":190,"Mon":2148,"Mos":1550,"Mor":687,"Mou":317,"Mot":148,"Mik":294,"Mig":214,"Mic":585,"Mit":157,"Mir":210,"Mis":141,"Mil":404,"Min":480,"Mun":792,"Mul":124,"Mut":117,"Mur":267,"Mus":488,"Mug":257,"Xià":110,"Xab":116,"ège":392,"XX ":192,"Wor":153,"Sèv":275,"èvr":599,"Wik":612,"Wil":429,"Win":144,"ère":1699,"Wes":294,"Was":125,"War":190,"Wal":229,"ès ":918,"之三":148,"ée ":379,"éco":116,"Vos":480,"Vol":182,"èze":351,"Vit":132,"évi":152,"Zor":154,"Zeh":114,"Zee":161,"Zar":377,"Zah":179,"Zal":136,"éli":109,"éra":508,"ére":128,"éri":214,"Zie":209,"Zib":154,"Zin":153,"Zel":160,"Zen":285,"Zer":305,"és ":183,"Zab":118,"Qū ":204,"Yve":193,"Yor":688,"Yon":444,"三三":257,"三之":120,"丁三":109,"Sur":139,"Sul":122,"Sue":239,"Sui":315,"Str":342,"Sto":163,"Sta":507,"Ste":414,"Ten":223,"Teo":128,"Tei":132,"Tel":274,"Tan":225,"Tar":886,"Tai":174,"Tak":137,"Tal":334,"Shi":203,"She":176,"Sha":388,"Sim":208,"Sil":273,"Sis":185,"Sir":222,"Sin":296,"Sie":134,"Sib":121,"Ser":500,"Sev":217,"Saô":1077,"Sen":362,"Sel":120,"Sem":121,"Sei":1420,"Seg":141,"Sph":400,"Soc":136,"Sob":257,"Sou":293,"Soz":190,"Sol":243,"Som":871,"Son":154,"Sor":289,"Sai":5007,"Sam":232,"Sal":766,"Sab":129,"Sco":113,"Sci":1122,"Sch":335,"Sax":145,"Sav":636,"Sat":139,"Sau":359,"Sar":1149,"Sas":116,"San":2399,"ови":110,"Res":126,"Rhi":762,"Rha":154,"Rio":118,"Riv":186,"Ria":109,"Rib":112,"Ric":281,"ärv":206,"Rec":114,"Rei":155,"Reg":177,"Rem":135,"Ren":337,"Rea":142,"Rob":311,"Roc":325,"Rod":194,"Roy":117,"Rou":411,"Ros":328,"Rom":283,"Rhô":310,"Vel":146,"Ven":718,"Vau":379,"Vas":132,"Van":188,"Val":1311,"Var":518,"Vig":138,"Vic":317,"Vie":959,"Vip":162,"Vir":171,"Vil":1162,"Vin":200,"Ver":591,"Ukr":188,"Uha":317,"Uni":1067,"Uro":222,"Urt":224,"Urr":361,"Urk":120,"Uri":138,"Urd":163,"Ura":118,"Txa":393,"Typ":542,"Txi":1766,"Txe":254,"Uda":518,"VI ":183,"Ter":439,"Tes":138,"Tha":159,"The":888,"Thi":181,"Tho":316,"Tib":109,"Tim":132,"Tir":125,"Pér":118,"Tor":335,"Tok":213,"Tol":699,"Tom":137,"Tow":179,"Tou":411,"Tro":782,"Tri":446,"Tre":367,"Tra":727,"Tur":461,"aEu":198,"вич":155,"àn ":157,"bje":319,"biz":22873,"bis":1045,"bit":2242,"bio":874,"bir":655,"bik":563,"bil":6749,"bin":3474,"big":994,"bih":468,"bo ":895,"blo":238,"ble":654,"bli":2524,"bn ":137,"bla":489,"boa":985,"bok":9893,"bol":1370,"boi":169,"boe":180,"bs ":545,"bon":636,"bor":1560,"bot":625,"bos":584,"bou":296,"boz":129,"be ":1452,"ban":2718,"bak":3303,"bal":5966,"bai":7451,"bag":126,"bae":319,"bac":240,"bad":1178,"bab":550,"án ":329,"baz":996,"bax":116,"bat":43650,"bas":1155,"bar":12363,"bea":507,"áng":124,"bi ":1854,"bei":305,"beh":2621,"beg":284,"bed":218,"bec":150,"ber":19208,"ben":1757,"bel":1463,"bek":859,"bez":2002,"bes":5139,"bet":1729,"bia":2180,"bid":4505,"bie":1454,"bgu":159,"ca ":1045,"car":483,"cas":288,"cat":226,"cau":179,"can":540,"cap":126,"cad":117,"cak":136,"cam":158,"cal":436,"cag":240,"ce ":1311,"bri":2477,"bro":342,"bra":732,"bre":1164,"bu ":155,"bru":149,"bst":212,"bur":7454,"bul":964,"buk":216,"bum":126,"but":176,"bus":313,"buz":1067,"by ":112,"aka":3860,"am ":849,"ake":2522,"aki":3275,"akh":163,"ajo":184,"aix":538,"aiz":1199,"al ":8930,"aja":231,"aje":125,"aik":2668,"ail":16466,"aim":225,"ain":22745,"aio":2613,"aip":526,"air":1724,"ais":1887,"ait":6762,"ak ":35410,"aig":330,"aie":1989,"aid":440,"aib":241,"aia":7883,"ahi":1974,"ahu":235,"aho":676,"aha":1972,"agh":132,"agi":3410,"agr":297,"agu":6313,"agn":992,"ago":50304,"aoi":123,"anu":801,"anz":393,"any":212,"anp":1062,"ano":2915,"ann":1154,"ant":46628,"ans":1575,"ane":3569,"anf":114,"ang":4075,"ani":22937,"anj":269,"ank":984,"anl":19832,"ana":6367,"anb":593,"anc":2276,"and":13202,"amu":416,"amm":343,"amo":901,"amn":123,"amp":1235,"ams":133,"ami":9813,"ame":32856,"amb":651,"ama":5578,"ao ":342,"alv":863,"alu":1501,"alt":3287,"als":169,"alp":282,"alo":1294,"alm":822,"all":1873,"alk":1496,"alg":595,"ali":14200,"alc":178,"ald":57921,"ale":42820,"alf":261,"ala":12819,"alb":844,"akz":215,"an ":170348,"akr":254,"aku":2585,"akt":1284,"ako":87760,"aba":5416,"abd":115,"abe":8846,"abi":6317,"abl":275,"abo":739,"abr":604,"abs":119,"abu":1781,"ae ":7943,"aca":415,"aar":137,"ad ":651,"ac ":1453,"ab ":142,"afo":576,"afr":334,"aff":141,"afe":133,"afi":865,"ai ":2679,"aga":4252,"age":1716,"aen":488,"aem":396,"ael":880,"aer":1127,"ah ":241,"âte":302,"afa":2878,"aet":117,"ado":2055,"adr":1175,"adm":643,"adi":3700,"ade":1430,"adu":1159,"acq":143,"aco":349,"ack":319,"aci":488,"ach":1442,"ace":785,"ada":3736,"act":1592,"acu":160,"acr":160,"azp":1356,"azo":562,"azi":7825,"azl":2452,"azk":3543,"azu":329,"azt":5546,"aze":1070,"aza":3082,"azz":159,"axi":278,"axo":276,"axu":131,"az ":12242,"axa":121,"ayo":136,"ays":139,"aya":303,"aye":536,"ba ":1649,"aqu":160,"at ":33651,"arg":3519,"are":45955,"ard":3975,"arc":902,"arb":1084,"ara":18217,"arp":378,"aro":3911,"arn":4158,"arm":1625,"arl":1757,"ark":3693,"anç":220,"ari":25386,"aru":371,"arv":202,"arr":46687,"ars":715,"art":52635,"au ":3167,"asa":2228,"ary":240,"arz":155,"asg":196,"asi":3937,"ash":374,"asc":404,"ase":1361,"aso":2684,"asp":509,"ask":3157,"asm":635,"asl":1054,"aon":151,"ar ":18677,"apa":1371,"ape":2743,"api":1618,"aph":271,"apl":270,"apo":1684,"app":159,"apr":133,"aps":163,"apt":114,"apu":638,"as ":4585,"ava":748,"ax ":148,"auz":2790,"aux":633,"auv":282,"aut":4572,"arí":190,"avo":638,"avi":1105,"ave":561,"ay ":1280,"awa":295,"ata":5981,"asu":3797,"ast":11887,"ass":921,"atm":119,"atl":299,"atr":577,"ato":3893,"ate":18483,"ati":19404,"ath":563,"aua":391,"auc":470,"aub":143,"att":437,"ats":845,"atu":34742,"atx":565,"atz":18521,"aul":1718,"aum":331,"aun":1227,"aup":140,"aur":6720,"aus":1048,"aud":2046,"aue":1117,"aug":815,"auk":876,"Zur":179,"Zuz":173,"Zub":282,"ич ":157,"jer":179,"jek":326,"jel":120,"jen":358,"jab":248,"jat":1353,"jas":739,"jau":579,"jap":648,"jar":1747,"jal":123,"jak":490,"jan":459,"jai":2132,"jaz":157,"joa":281,"joe":132,"jol":254,"jok":1248,"jot":496,"jor":115,"jio":380,"jin":276,"jia":219,"jo ":284,"itm":162,"itr":368,"ito":2098,"itu":32563,"itt":399,"its":2904,"itz":12198,"ity":169,"itx":1037,"eña":150,"isk":1566,"ism":1753,"isl":341,"iso":671,"isn":727,"isp":164,"iss":773,"isu":768,"ist":19392,"ita":22759,"ite":6146,"ith":384,"iti":6015,"ivo":164,"ius":683,"iur":210,"ium":623,"iva":357,"ix ":589,"ivi":458,"ive":562,"ipo":429,"ipp":233,"ipu":2084,"ips":176,"ipt":634,"ipi":478,"ipl":363,"is ":8540,"ion":6034,"iop":428,"ior":622,"ios":459,"iot":1950,"iog":146,"ioi":472,"iok":992,"iol":1372,"ipa":3058,"ipe":1493,"iox":130,"ioz":931,"ir ":1292,"iru":3334,"irt":277,"irr":1309,"iro":2123,"irm":114,"irk":440,"irl":941,"iri":11303,"isi":1862,"ish":426,"ise":2016,"isc":611,"isb":459,"isa":1271,"iqu":500,"ire":8235,"irg":171,"ira":10391,"ird":162,"it ":1135,"ja ":330,"iya":143,"ixo":397,"ixk":401,"ixa":122,"ixe":126,"iz ":3294,"izu":662,"izt":21049,"izp":401,"izo":1014,"izl":132,"izk":5110,"izi":6510,"izg":172,"ize":7802,"iza":19239,"kig":334,"kik":669,"kim":840,"kil":2662,"kia":6057,"kib":168,"kie":1433,"kid":1763,"kiz":579,"kin":6825,"kio":719,"kip":627,"kir":776,"kis":461,"kit":2888,"km ":412,"ki ":14622,"kha":202,"kho":112,"kea":556,"kee":629,"kei":150,"kek":347,"kel":306,"keo":173,"ken":1434,"kes":365,"ker":2731,"ket":6095,"kez":458,"ke ":1678,"kra":849,"kre":293,"ku ":1173,"kro":731,"kri":1134,"koz":874,"kov":156,"km²":111,"kot":1230,"kos":1322,"kor":2545,"kop":563,"koo":115,"kon":8009,"kom":1476,"kol":2022,"kok":4933,"koi":1642,"koe":2023,"kod":227,"kiö":114,"koa":15917,"kob":165,"kni":322,"kno":187,"kko":963,"klu":109,"ko ":166136,"kle":410,"kla":832,"klo":486,"kli":257,"jur":356,"jua":336,"jud":181,"kaz":1190,"kat":17259,"kau":137,"kar":12671,"kas":3232,"kap":656,"kan":3807,"kal":6938,"kam":178,"kak":2781,"kai":3049,"kag":867,"kae":232,"kad":1070,"kab":331,"ka ":7200,"ha ":447,"ham":2422,"han":4212,"hap":332,"hai":2518,"hak":119,"hal":2041,"hau":3280,"har":11639,"has":2581,"hat":743,"hae":346,"hag":159,"hab":197,"had":140,"hac":168,"haz":745,"he ":3675,"hek":291,"hel":1598,"hei":419,"heg":3098,"hec":165,"hed":603,"hea":271,"heb":248,"hez":537,"hev":153,"het":258,"hes":714,"her":9041,"heo":124,"hen":3743,"hem":423,"hi ":2061,"hig":352,"hie":1401,"hid":400,"hic":295,"hia":1281,"hip":234,"hio":194,"hin":1789,"him":161,"hil":1588,"hik":564,"hii":240,"his":2862,"hit":1675,"hir":8155,"hiz":2483,"hn ":341,"hle":145,"hlo":1029,"ho ":201,"gma":140,"go ":26432,"glo":370,"gle":317,"gli":281,"gla":304,"gog":353,"goe":28566,"god":354,"gob":600,"goa":4107,"gny":701,"gno":312,"gni":265,"gne":942,"gna":765,"goz":493,"goi":1856,"gok":1203,"gom":158,"gol":1108,"gon":1318,"gos":518,"gor":3038,"got":736,"gu ":476,"gro":244,"gra":2474,"gri":255,"gre":2219,"gto":194,"gui":372,"gul":293,"gua":626,"gue":1210,"gud":543,"gy ":143,"guz":1681,"gut":1860,"gur":3120,"gus":2569,"gun":7491,"gzh":121,"iak":58472,"iam":434,"ial":8884,"ian":17793,"iap":140,"ias":645,"iar":18221,"iau":120,"iat":2790,"ic ":497,"iab":177,"iac":288,"iad":428,"iag":1598,"ibl":211,"ibi":2091,"ibo":10573,"ibr":326,"ibu":3077,"iaz":1327,"id ":402,"iba":3588,"ibe":2135,"ia ":66121,"iet":4538,"ieu":535,"iev":194,"iez":349,"iel":657,"iem":109,"ien":15589,"ier":10660,"ies":673,"iee":130,"ied":378,"ieg":332,"iei":288,"iek":1321,"ig ":225,"iea":142,"ifo":600,"iff":119,"ife":501,"ifi":1639,"ifa":243,"icr":332,"ict":263,"icu":365,"ico":799,"ick":329,"ici":329,"ich":1326,"ice":649,"ie ":1575,"ica":1267,"idu":909,"idr":287,"ido":1820,"idi":703,"idg":234,"ide":10967,"ida":12783,"iid":341,"il ":2896,"ija":210,"iji":583,"ijo":114,"im ":459,"ika":16573,"ige":670,"iga":3099,"ii ":406,"igl":239,"igh":367,"igi":1126,"igu":1200,"igr":300,"igo":1186,"ign":1443,"ihe":190,"iha":384,"ihu":414,"iho":241,"ik ":16195,"imo":875,"imm":120,"ime":3172,"imi":1266,"inc":1845,"ind":5098,"ina":14551,"inb":1383,"imu":195,"inn":283,"inp":1720,"ino":3873,"int":13017,"ins":1000,"inf":881,"ine":11821,"inh":165,"ing":7209,"ini":17586,"inl":550,"ink":1408,"ioa":6380,"iob":288,"ioc":123,"iod":290,"ioe":954,"inu":1280,"inv":181,"iny":1050,"iko":21260,"ikl":464,"iki":4819,"ike":2410,"ila":10425,"ilb":1403,"in ":15912,"ikz":303,"ikt":193,"iku":2386,"ikr":256,"ilo":4660,"ill":6632,"ilk":7089,"ilm":637,"ilh":252,"ilg":475,"ili":12072,"ild":890,"ile":8785,"ima":8169,"imb":209,"io ":8484,"ils":121,"ilt":2974,"ilu":1485,"hol":600,"hom":373,"hon":2777,"hog":384,"hoi":120,"hos":554,"hot":751,"hou":397,"hop":188,"hor":3428,"hob":344,"hoa":178,"hod":259,"hoc":115,"dée":208,"hua":295,"hth":235,"hry":189,"hro":550,"hri":326,"ht ":301,"hra":124,"hyd":145,"hyl":274,"hy ":139,"hum":243,"hun":759,"hus":732,"hut":205,"hur":1153,"ffe":167,"ffi":176,"fes":515,"fer":1274,"fed":363,"fen":469,"fek":715,"fel":165,"fia":757,"fga":109,"fas":196,"far":2984,"fam":8183,"fan":210,"fak":148,"fal":486,"fab":275,"ff ":133,"fe ":256,"fa ":176,"exu":444,"eyr":541,"exa":425,"ez ":12298,"exo":122,"aôn":1078,"exi":839,"exe":123,"ezu":483,"ezb":637,"eza":6134,"ezp":276,"ezt":764,"eze":550,"ezi":5130,"ezk":3961,"ezl":261,"eta":71146,"ete":2569,"eti":4795,"eth":290,"etn":393,"esp":3014,"esn":776,"eso":556,"est":11662,"esu":1628,"ess":912,"ev ":129,"aña":175,"eud":448,"euf":120,"eui":361,"euk":128,"eul":175,"eun":111,"eto":1392,"etr":5219,"ets":1121,"ett":760,"etu":368,"etx":2177,"etz":1163,"ew ":848,"eve":525,"eva":377,"evo":137,"evi":1107,"euv":304,"eut":239,"eur":2218,"eus":5316,"ex ":208,"eux":721,"ey ":1366,"epe":753,"epi":3559,"eph":603,"er ":6409,"epa":30994,"eot":118,"eor":1271,"eom":283,"eol":759,"eok":178,"eop":146,"eon":704,"es ":12081,"ept":728,"eps":204,"epu":1540,"epo":499,"erk":2593,"erl":1568,"eri":9060,"erg":1467,"ere":16377,"erf":172,"erc":707,"erd":4566,"era":40409,"erb":2356,"et ":5547,"esk":37262,"esl":821,"esm":224,"esf":120,"esh":127,"esi":2406,"esb":376,"esc":377,"ese":3688,"eu ":330,"esa":3808,"erz":150,"ery":187,"erv":411,"eru":942,"err":62362,"ert":11444,"ers":2709,"ern":3239,"erm":1856,"erp":616,"ero":11175,"eki":8183,"ekk":958,"ekl":121,"ekn":504,"eko":32688,"ekr":109,"ekt":3089,"eku":1445,"ekz":206,"en ":169009,"elb":764,"ela":7880,"eld":843,"elf":319,"ele":6128,"eli":2171,"elg":473,"elm":234,"elk":1631,"ell":3887,"elo":1639,"elu":1274,"elv":114,"els":360,"elt":1391,"ely":133,"eo ":860,"emb":233,"ema":8298,"eme":1995,"emo":994,"emi":1594,"emu":1285,"emp":192,"emy":190,"enf":133,"ene":16371,"enh":206,"eng":2970,"enb":2943,"ena":41627,"end":44877,"enc":765,"eno":2247,"enp":1606,"enn":1479,"enk":297,"eni":2624,"enu":163,"ens":1539,"ent":12453,"enr":290,"eoa":338,"enz":249,"eny":137,"eog":358,"eoe":128,"eod":110,"ego":7214,"ege":2822,"egi":10698,"eha":3263,"egr":361,"egu":4291,"eho":226,"ehe":4597,"ehi":2102,"ek ":6903,"eic":110,"eia":366,"ehu":489,"eis":293,"eir":418,"eim":319,"eil":722,"eio":204,"ein":3804,"eik":189,"eij":125,"eie":157,"eid":253,"eig":270,"eja":123,"el ":3223,"eiz":933,"eit":1960,"eke":1121,"eka":2666,"em ":347,"eju":612,"giz":1772,"git":4426,"gis":854,"gir":652,"gil":2535,"gim":660,"gik":1305,"gip":786,"gin":5407,"gio":349,"gid":465,"gie":433,"gia":5348,"ght":231,"ghe":212,"gha":252,"ggi":124,"gi ":1669,"gen":9845,"geo":710,"get":173,"ger":4696,"ges":1287,"gh ":273,"gez":400,"gea":1197,"gee":110,"gei":542,"geh":1437,"geg":119,"gel":2960,"gek":219,"ge ":1979,"gaz":2792,"gab":1043,"gad":132,"gae":256,"gai":4004,"gas":712,"gar":10218,"gau":1336,"gat":2266,"gak":724,"gam":476,"gal":2723,"gan":2119,"ga ":2307,"fur":143,"fus":174,"fut":598,"fun":655,"ftw":197,"ft ":115,"fra":2359,"fre":282,"fri":962,"fro":345,"fru":141,"for":3048,"fos":110,"fon":526,"fol":298,"foa":306,"fle":119,"fla":235,"flu":128,"fo ":357,"fic":126,"fie":126,"fil":1255,"fik":2128,"fin":1035,"fis":784,"fit":221,"fiz":1025,"da ":83681,"dd ":135,"de ":14213,"dac":1167,"dad":591,"dab":383,"dak":4129,"dal":34505,"dai":1813,"dag":46918,"dae":7409,"dat":4371,"das":372,"dar":7697,"dap":284,"dan":1747,"dam":370,"daz":3020,"dau":2348,"ddo":115,"cul":269,"cty":1123,"ctu":376,"cto":282,"cti":251,"cta":224,"cy ":306,"cus":776,"cur":130,"cut":132,"cla":138,"cle":140,"clu":182,"clo":110,"co ":929,"ció":186,"cod":133,"con":497,"col":516,"com":276,"cor":390,"cos":170,"cop":272,"cot":155,"cou":1017,"cqu":221,"cra":135,"cri":123,"cru":146,"cro":386,"cci":190,"cea":325,"ch ":1027,"cey":125,"cer":590,"ces":263,"cen":434,"cep":383,"cel":570,"cha":1044,"chw":110,"chu":371,"chy":392,"cia":581,"ck ":966,"cie":258,"cid":1189,"che":2829,"chl":124,"chi":830,"cho":437,"cht":276,"chr":456,"cil":203,"cis":344,"cin":1476,"cio":340,"cke":298,"ed ":562,"eba":3556,"ebe":1499,"ebg":158,"ebi":808,"ebo":188,"ebr":497,"ebu":253,"ec ":189,"eag":605,"eae":255,"ead":117,"eak":5006,"ean":53210,"eal":880,"ear":5919,"eas":238,"eat":764,"eau":1023,"eaz":319,"eb ":149,"ea ":21907,"efi":386,"efo":313,"efa":131,"efe":1060,"ei ":1830,"ega":1847,"eek":733,"eei":184,"een":2608,"eel":320,"eer":450,"eet":3066,"edi":3117,"edd":167,"ede":1453,"ône":1395,"eda":1186,"edu":758,"edo":12587,"edr":806,"eck":189,"ech":649,"eci":177,"eca":138,"ôme":834,"ee ":210,"ecu":151,"ect":278,"eco":286,"dyl":181,"dwa":144,"dy ":206,"dré":154,"dur":2122,"dut":4392,"dus":429,"duz":254,"dor":3977,"dop":318,"don":1432,"dom":352,"dol":488,"dok":827,"doz":381,"dou":190,"dot":620,"dos":896,"ds ":300,"dmi":9816,"doa":2009,"dob":254,"doc":225,"doe":371,"doi":1104,"dun":1704,"dum":256,"dui":109,"dul":792,"duk":1954,"due":5451,"dug":218,"dua":33103,"dri":1545,"dra":754,"dre":1584,"du ":7012,"dro":1415,"dge":253,"dic":166,"did":286,"dia":9396,"dib":563,"ôte":774,"der":7825,"des":2086,"det":1562,"dez":1959,"deb":3140,"dea":45982,"dec":379,"def":455,"dee":1641,"deg":553,"dei":1488,"del":2329,"dek":7786,"den":11216,"dem":540,"dep":31070,"deo":330,"di ":3931,"do ":13628,"diu":152,"diz":2212,"dim":351,"din":3293,"dio":1674,"dip":225,"dir":6847,"dis":1910,"dit":6852,"die":3603,"dif":136,"dig":733,"dik":2769,"dil":919,"rgu":720,"rhi":113,"rho":158,"rga":1667,"ri ":30074,"rgi":2575,"rgh":113,"rge":1357,"rgo":1529,"ret":4609,"res":7061,"rev":255,"reu":617,"rez":4457,"rey":281,"rfo":144,"rdu":1468,"rds":187,"rdy":168,"rg ":780,"reb":423,"rea":5485,"ree":1399,"ref":864,"rec":235,"red":748,"rei":1098,"reg":3035,"reh":258,"rem":1526,"ren":57161,"rek":7795,"rel":1168,"rer":1714,"reo":216,"rep":4742,"rf ":117,"rda":944,"rcu":273,"rdo":1564,"rdi":6252,"rde":3329,"re ":19143,"rbu":270,"rco":298,"rci":264,"rch":425,"rce":466,"rca":271,"ray":301,"raz":13993,"rd ":1974,"rao":152,"rap":758,"rar":3528,"ras":8250,"rat":19217,"rau":2390,"rav":257,"rbi":2074,"rbo":540,"rba":974,"rbe":1362,"raj":132,"rai":9556,"rah":192,"rag":3181,"ran":42992,"ram":1927,"ral":6492,"rak":9219,"rab":9119,"raf":1044,"rae":295,"rad":2088,"rac":1198,"rpu":497,"rpo":196,"rs ":1994,"rpe":728,"rpa":194,"rpi":260,"rph":358,"ror":761,"ros":1392,"rot":2296,"rom":2243,"ron":4833,"roo":173,"rop":4279,"roy":129,"roz":1007,"rou":490,"rov":593,"row":119,"rob":3623,"roa":6616,"rod":811,"roc":757,"roi":1273,"rol":3561,"rok":9546,"rof":646,"roe":542,"rog":1119,"rno":850,"rnu":670,"rna":1877,"rne":4341,"rni":878,"riè":489,"rmo":557,"rmu":500,"ro ":5889,"rma":3800,"rme":1506,"rmi":806,"rlu":114,"rlo":918,"rli":1036,"rld":114,"rle":728,"rla":3010,"rn ":837,"rku":647,"rko":637,"nço":166,"rki":3202,"rke":2368,"rka":2965,"né ":169,"riz":2281,"rix":382,"rl ":161,"rip":367,"rio":4087,"rir":622,"rit":7871,"ris":4325,"riv":160,"riu":525,"rig":1164,"rij":116,"ril":4692,"rik":14158,"rin":4139,"rim":739,"ria":43986,"rib":3119,"ric":1194,"rid":2808,"rie":4686,"rif":356,"rk ":865,"rya":116,"rui":249,"rug":753,"rue":502,"rud":764,"ruc":148,"rur":560,"rup":118,"run":1514,"rum":622,"rul":119,"ruk":7712,"ruz":1278,"rus":3454,"rut":2747,"rva":299,"rvi":571,"rve":398,"rrè":303,"ry ":1109,"rsk":111,"rsi":579,"rso":234,"rsa":310,"rsb":123,"rsh":119,"rse":377,"rta":37313,"rst":238,"rto":962,"rte":16058,"rth":1501,"rti":3170,"rtz":8628,"rub":137,"rua":4504,"rts":3910,"rtr":191,"rtu":8524,"rtx":1302,"rt ":2199,"rqu":185,"rro":8594,"rrh":151,"rri":59865,"rre":25667,"rra":27202,"ru ":3378,"rry":287,"rru":11643,"sab":386,"sac":395,"sad":276,"sag":755,"sai":9052,"sak":1159,"sal":1071,"sam":544,"sba":383,"sbe":333,"san":1883,"sau":475,"sat":4584,"sas":2876,"sar":2841,"saz":146,"sa ":3578,"ón ":558,"rze":132,"rza":167,"ruñ":573,"ryp":120,"ryn":159,"sha":293,"sho":242,"she":237,"shi":909,"si ":2705,"sga":355,"sge":484,"siz":297,"sie":2276,"sid":627,"sic":145,"sib":219,"sia":4506,"sk ":177,"shu":112,"sit":1781,"sir":233,"sis":2072,"sin":2608,"sio":1888,"sil":990,"sim":204,"sik":4135,"sig":316,"scu":158,"sbo":201,"sbu":282,"se ":3435,"sca":504,"sci":309,"sch":523,"sco":757,"sex":360,"sey":232,"ser":1632,"ses":1995,"set":590,"seu":341,"sez":2416,"sh ":330,"sfe":412,"sfo":111,"sea":404,"sei":1006,"seg":464,"sed":159,"sep":278,"seo":240,"sen":2206,"sem":882,"sel":1814,"sek":1181,"spo":227,"spe":2584,"spl":190,"spi":825,"spa":4514,"sot":612,"sou":390,"sov":121,"soz":627,"sol":1417,"som":367,"son":3257,"sop":239,"sor":5121,"soe":201,"sof":846,"soi":556,"sok":569,"soa":1638,"sob":130,"su ":1117,"sra":160,"st ":1348,"squ":206,"ss ":363,"sli":115,"slo":268,"sla":1331,"sle":1317,"ski":965,"sko":4744,"skr":385,"sku":35531,"ska":8859,"ske":1695,"sna":476,"sni":281,"sne":1033,"smo":1970,"so ":2308,"sma":633,"smi":193,"sme":185,"sze":109,"sse":1415,"ssa":845,"sso":606,"ssi":746,"ssu":264,"ssy":167,"ste":10155,"sta":12361,"sto":3482,"sti":12235,"stl":148,"stu":1961,"str":13494,"sua":1580,"sue":1113,"sub":323,"sui":142,"sug":112,"sul":894,"sum":1240,"suk":305,"sup":126,"sun":3563,"sut":122,"sus":779,"sur":2717,"sy ":268,"tai":3219,"tak":18285,"tal":9875,"tae":444,"taf":399,"tag":801,"tab":681,"tac":177,"tad":501,"taz":975,"tav":112,"tau":653,"tat":11576,"tas":3495,"tar":13099,"tap":321,"tan":42023,"tam":31263,"tch":169,"te ":16458,"tbo":740,"ta ":64927,"ozè":187,"ño ":122,"pa ":796,"pe ":842,"par":37920,"pat":1309,"pas":522,"pau":187,"paz":432,"pac":116,"pad":210,"pag":296,"pak":754,"pal":618,"pai":3956,"pap":195,"pan":730,"phe":439,"pha":737,"phu":411,"pht":217,"pho":650,"phl":1004,"phi":1924,"pi ":440,"ph ":172,"pez":1232,"pea":802,"ped":964,"pen":3866,"per":5212,"pet":1004,"pes":475,"peg":247,"pei":178,"pel":1577,"pek":338,"pla":1480,"pli":1410,"ple":493,"lès":615,"plo":475,"plu":342,"lé ":113,"piz":190,"phy":150,"pia":822,"pid":4283,"pic":144,"pie":488,"pig":222,"pik":760,"pil":704,"pin":1928,"pio":428,"pir":1470,"pis":933,"pit":729,"por":1550,"pop":436,"pot":370,"pos":1322,"pon":1624,"pok":216,"pol":3723,"poa":264,"poe":553,"pod":317,"ps ":1451,"ppi":156,"ppe":293,"po ":711,"ñak":177,"psi":410,"pso":126,"ptu":244,"pub":1994,"pte":196,"pti":290,"pto":1097,"pra":285,"ña ":328,"psa":122,"pri":796,"pre":2596,"pro":6098,"ñea":278,"ñek":147,"pur":933,"pus":334,"put":783,"pun":714,"pui":142,"pul":574,"puz":1734,"pzi":136,"qua":275,"que":1375,"qui":483,"ra ":35431,"ngo":4685,"ngi":806,"ngl":421,"ngu":2685,"ngr":203,"ngt":258,"ngs":265,"ni ":1052,"nge":3635,"ngh":311,"nga":2908,"ngd":140,"ndé":266,"jär":205,"nha":305,"ngy":127,"ngz":152,"nhe":147,"neg":587,"nei":353,"nel":533,"nek":3786,"nen":5074,"nem":1122,"neo":478,"ner":12607,"net":5039,"nes":2224,"nev":226,"neu":1060,"ng ":2807,"nea":4726,"neb":129,"nec":148,"ned":379,"nee":138,"nfi":148,"nfo":733,"nfl":131,"ney":289,"nez":3775,"nfa":134,"nfe":453,"nct":118,"nco":700,"nci":1712,"nce":1222,"nch":1193,"nca":298,"ne ":11184,"nbu":575,"ndu":36540,"ndr":2320,"nds":159,"ndo":4551,"ndi":10387,"nde":10020,"nda":11174,"ncy":130,"ncu":199,"nak":8535,"nal":3860,"nam":614,"nan":1735,"nar":6462,"nac":646,"nad":808,"nae":224,"naf":779,"nag":2948,"nah":1954,"nai":1342,"nab":663,"nbo":866,"nbe":572,"nbi":1065,"nd ":1618,"nba":2467,"nav":317,"nau":412,"nat":3667,"nas":994,"naz":1755,"nay":387,"na ":51123,"mys":135,"ión":245,"nya":287,"nyi":1026,"nz ":115,"ny ":1034,"nvi":456,"nuk":355,"num":238,"nus":716,"nur":159,"nua":755,"nue":380,"ntx":477,"ntz":53194,"nto":2625,"ntu":3025,"nts":4328,"ntr":1766,"nti":3678,"nth":439,"nta":4382,"nte":6419,"nsu":131,"nsm":113,"nso":335,"nst":1117,"nse":442,"nsh":183,"nsi":830,"nsk":396,"nsa":275,"nu ":553,"nri":259,"nt ":6455,"nqu":144,"npe":1521,"npi":336,"npl":1447,"npo":1351,"npr":973,"npu":200,"ns ":1708,"noc":367,"nod":214,"noa":1389,"nob":252,"nog":132,"noe":258,"nok":336,"nol":1301,"noi":432,"nop":687,"nom":2855,"non":1208,"not":726,"nos":1504,"nor":1438,"nov":276,"nou":160,"noz":134,"npa":532,"nne":2767,"nna":510,"nno":193,"nni":412,"niè":150,"nn ":470,"nla":655,"nle":19805,"no ":4249,"nke":316,"nki":526,"nka":819,"nku":162,"nko":1451,"nji":179,"nje":125,"nja":149,"nig":202,"nif":230,"nie":1873,"nid":1634,"nic":527,"nib":1314,"nia":21004,"nk ":215,"niz":435,"niu":163,"niv":129,"nis":11960,"nit":1609,"nir":121,"nio":1723,"nim":7615,"nin":1144,"nik":2843,"nil":338,"ogr":1140,"ogu":264,"ogi":3112,"ogl":187,"ogo":1114,"ogn":260,"oga":472,"oge":994,"oi ":1867,"ohi":1496,"oho":168,"ohn":393,"oha":408,"ohe":156,"oix":147,"ois":1228,"oir":3115,"oit":1745,"oin":2360,"oik":468,"oil":486,"oig":146,"oih":139,"oid":551,"oie":762,"ok ":552,"oia":1590,"ol ":1542,"oiz":813,"oce":551,"och":579,"oci":236,"ock":754,"oco":220,"obu":191,"oe ":120,"oca":267,"ode":1086,"odi":779,"odo":1366,"odr":254,"of ":673,"oda":1173,"oek":753,"oel":238,"oem":135,"oei":191,"oer":727,"oes":154,"oet":2140,"oen":30360,"ody":112,"odu":1464,"og ":109,"ofi":1327,"oft":279,"ofo":462,"off":123,"ofe":486,"oa ":23154,"oc ":215,"oan":5469,"oal":1996,"oak":7788,"oai":315,"oag":249,"oaf":179,"oad":180,"oba":971,"oaz":457,"od ":223,"oar":7570,"oat":134,"obo":225,"obr":314,"obl":583,"obj":240,"obi":4309,"obe":1621,"nza":293,"oye":159,"oya":225,"oxi":323,"oxe":134,"oxa":143,"oz ":1399,"osé":257,"own":214,"ozt":385,"ozk":775,"ozo":304,"oze":1252,"ozi":1536,"oza":425,"oty":475,"otz":3173,"otx":166,"otu":1228,"oud":180,"oue":152,"oub":647,"ouc":291,"oua":149,"ow ":180,"oti":1497,"oth":414,"ote":1876,"ott":442,"ots":1405,"otr":282,"oto":1274,"ost":3761,"osu":155,"ota":4700,"ov ":231,"osi":954,"osh":196,"osk":831,"ose":2342,"osg":604,"osf":232,"osp":1434,"oss":573,"osm":148,"osl":159,"oso":2977,"osn":183,"oy ":351,"owe":287,"ovi":716,"ouv":335,"oux":333,"ouz":161,"ova":475,"ove":679,"oug":264,"oui":410,"oul":669,"oun":317,"oup":179,"ous":984,"our":2427,"out":459,"opo":1448,"opi":1772,"opl":283,"ope":1031,"oph":2032,"opa":1875,"os ":3052,"opu":866,"opt":221,"ops":1174,"ool":150,"ook":209,"ood":215,"or ":2026,"oor":211,"ork":1333,"orl":371,"orm":2863,"orn":1125,"oro":1987,"orp":1026,"orr":8683,"orc":229,"ord":4422,"ore":7122,"orf":275,"org":1446,"ori":6643,"ou ":585,"osa":5006,"osc":282,"ort":9314,"ors":420,"orv":338,"oru":798,"ory":148,"ot ":1204,"orb":627,"ora":3882,"oqu":170,"ola":6129,"old":2635,"on ":9316,"oli":5769,"oll":736,"olk":977,"olf":204,"ole":2038,"ols":202,"olt":171,"olm":155,"olo":6056,"olp":119,"oly":472,"olu":2010,"oka":5664,"om ":243,"oki":11899,"oke":354,"okr":376,"oko":14411,"okl":115,"okz":240,"okt":241,"oku":440,"onb":522,"ona":9127,"ond":7536,"onc":465,"onf":339,"one":4593,"ong":1876,"oni":6318,"onk":362,"onn":1364,"ono":3870,"onp":1137,"ons":1225,"ont":7687,"onu":301,"onv":206,"ony":150,"onz":159,"oma":3483,"ome":4236,"omb":366,"omi":2449,"omm":1081,"omp":326,"omo":1653,"omu":1205,"op ":248,"la ":13924,"ín ":239,"ío ":118,"le ":29964,"lca":121,"lf ":110,"lde":53459,"lda":3988,"ldo":569,"ldi":2939,"ldu":1826,"lab":1670,"lac":584,"lad":550,"lae":479,"lah":118,"lag":1101,"lai":2536,"lal":149,"lak":5012,"lan":12052,"lam":935,"lap":544,"lar":14549,"lat":4673,"las":2232,"lax":140,"lau":1492,"lav":391,"lay":294,"laz":1246,"lba":511,"ld ":516,"lbe":387,"lbi":632,"lbo":1200,"lbu":874,"kuz":165,"kut":584,"kus":1014,"kur":1078,"kup":125,"kun":4291,"kum":688,"kul":2168,"kuk":191,"kta":288,"kte":291,"kub":403,"kud":351,"kue":274,"kui":374,"kua":34372,"ktr":750,"ktu":1773,"kti":747,"kto":1401,"kzi":9745,"llé":141,"lpe":1008,"lpi":131,"lph":125,"ls ":415,"lol":163,"lok":422,"lon":2710,"lom":2677,"lop":1545,"lor":2845,"lod":422,"loc":168,"loe":182,"log":3913,"loj":116,"loi":515,"lpa":119,"los":1717,"lot":1891,"lou":253,"lov":277,"low":117,"loa":880,"lob":424,"lmo":204,"lmi":305,"liè":124,"lme":519,"lma":675,"lti":598,"lto":617,"lts":230,"ltu":1223,"ltz":6760,"ltx":178,"lub":1731,"lua":1210,"lue":348,"lso":139,"lst":144,"lta":966,"lte":571,"lu ":1176,"ía ":420,"lt ":691,"lhe":161,"lha":201,"lgu":141,"lgo":926,"lge":122,"lgi":369,"li ":1303,"lga":398,"hât":347,"lfo":360,"lfa":358,"lez":813,"ley":359,"lex":439,"leu":374,"lev":222,"les":5236,"let":2340,"ler":37899,"leo":696,"lep":680,"lem":3502,"len":2886,"lek":3605,"lel":174,"lei":663,"leh":2756,"leg":2211,"lef":184,"lee":849,"led":224,"lec":200,"leb":1036,"lea":7810,"lls":158,"llu":240,"lly":586,"lo ":1720,"lla":2591,"lle":6822,"lli":1822,"llo":1275,"lko":462,"lku":282,"lka":8759,"lke":746,"lki":1220,"lm ":353,"lje":187,"ll ":864,"lit":5235,"lis":2569,"lir":147,"lip":493,"lio":1757,"lin":3474,"lim":313,"liz":2323,"lix":128,"liv":211,"liu":172,"lic":425,"lid":704,"lia":18816,"lib":1181,"lk ":177,"lik":3840,"lij":389,"lig":725,"lie":1726,"lif":1422,"ma ":5443,"mb ":125,"mac":227,"mab":228,"mah":164,"mai":4561,"mak":1772,"mad":637,"mae":329,"mag":822,"mar":5241,"mas":1187,"mal":8071,"man":6689,"maz":985,"mat":5384,"mba":280,"mbl":156,"mbi":246,"mbe":401,"mbr":369,"mbo":218,"me ":3651,"mbu":137,"iàn":147,"med":627,"mee":219,"mea":1209,"met":5777,"mes":758,"mer":3360,"mel":403,"meo":147,"men":42606,"mei":212,"mek":454,"mez":354,"luz":1919,"lva":1011,"lve":149,"lvi":156,"lul":394,"luk":181,"lun":936,"lum":401,"lut":259,"lus":2243,"lur":2397,"lux":127,"ly ":703,"lwa":119,"lyc":377,"hôn":315,"mpi":239,"mph":670,"mpe":165,"mpr":155,"mpo":233,"mpl":143,"mps":237,"ms ":126,"mog":116,"moi":170,"moe":126,"mod":1434,"mon":2086,"mop":136,"mok":406,"mol":594,"mor":1418,"mos":401,"mot":1295,"mou":184,"mpa":321,"moz":332,"mu ":428,"mug":1811,"mua":312,"iña":144,"my ":184,"mur":529,"mus":3234,"mut":703,"mui":219,"mul":1435,"mun":2365,"mi ":314,"min":12260,"mio":152,"mil":9541,"mir":441,"mis":679,"mit":1601,"mic":192,"mia":1625,"mig":137,"mie":282,"mid":994,"mik":1980,"mo ":1194,"ièv":316,"ièr":845,"ièg":369,"moa":1809,"mno":287,"mp ":117,"mmu":161,"mmo":170,"mma":310,"mme":989,"zte":6650,"zti":2572,"zta":21734,"ztu":2068,"zto":118,"zu ":506,"zub":389,"zua":330,"zuh":271,"zue":6583,"zur":964,"zuk":745,"zul":747,"zun":824,"zuz":1928,"zut":2275,"zy ":123,"zga":145,"zi ":3451,"zed":168,"zee":198,"zeg":625,"zeh":1509,"zei":931,"zej":597,"zea":4408,"zeu":174,"zet":1270,"zes":644,"zez":804,"zen":47887,"zel":1984,"zek":4796,"zer":3205,"zep":260,"zeo":158,"ze ":2487,"zab":1288,"zad":174,"zac":133,"zae":141,"zaz":774,"zbe":719,"zai":5026,"zag":3454,"zah":516,"zam":162,"zan":15520,"zak":3577,"zal":4025,"zar":6420,"zap":297,"zau":482,"zat":4596,"zoa":1994,"zot":212,"zor":775,"zon":1162,"zok":314,"zol":134,"zoi":237,"zoe":174,"zpe":474,"zpa":171,"zpi":1584,"zo ":1317,"zke":2202,"zka":3073,"zko":8159,"zki":1871,"zku":2058,"zle":2768,"zho":218,"zib":216,"zia":43966,"zie":3257,"zid":476,"zig":450,"zin":2910,"zim":160,"zil":633,"zik":1618,"zio":10069,"zip":247,"zir":2573,"zis":301,"zit":32762,"ziz":641,"uñe":454,"uña":238,"yst":148,"yro":439,"yrt":231,"yra":176,"yre":199,"ys ":403,"yph":1017,"ypt":141,"yon":185,"za ":8723,"ye ":281,"ych":465,"yda":109,"yco":120,"ydr":356,"yer":179,"yen":319,"ya ":555,"yar":131,"yas":121,"yan":296,"yal":138,"yko":242,"yn ":138,"yla":109,"yle":229,"yli":146,"yll":153,"ylv":131,"ylu":1217,"ymn":245,"yne":176,"yno":210,"yi ":127,"ygo":225,"yin":1034,"tín":140,"xu ":295,"xo ":287,"xor":162,"xot":335,"xon":327,"xoa":1198,"xua":342,"xur":507,"xi ":475,"xen":218,"xer":312,"xet":179,"xea":732,"xeb":306,"xek":523,"xel":128,"xka":395,"té ":122,"xir":793,"xis":324,"xit":138,"xil":657,"xik":2037,"xin":2912,"xim":206,"xia":292,"xie":194,"xa ":433,"xe ":943,"xat":249,"xar":321,"xak":258,"xal":109,"xap":662,"xan":503,"xab":132,"xag":128,"wyn":124,"sé ":310,"wn ":153,"sèr":390,"wor":110,"wer":265,"web":234,"wic":120,"win":111,"wil":123,"wa ":219,"wan":193,"wal":111,"way":199,"war":517,"ría":207,"vre":805,"vsk":110,"vy ":146,"rôm":333,"via":694,"vir":147,"vil":2088,"vin":415,"vig":216,"vic":152,"vid":247,"vie":427,"vit":352,"vis":236,"ré ":295,"rèz":299,"vo ":165,"viè":127,"voi":624,"von":289,"vi ":294,"vey":349,"ver":1032,"ves":311,"vet":138,"veg":284,"ven":884,"vel":488,"ve ":552,"val":523,"vak":167,"van":773,"var":579,"vat":112,"vad":796,"vai":119,"va ":528,"uzk":3402,"uzi":824,"uze":3853,"uzt":5216,"uzo":1817,"uza":960,"uya":140,"uxe":167,"uxi":112,"uz ":3148,"ux ":1723,"uvi":405,"uva":124,"uve":444,"uy ":638,"uvr":124,"usk":7231,"ush":119,"usi":7071,"use":2014,"usc":130,"usa":632,"usu":203,"ust":3020,"uss":762,"usp":226,"uso":148,"uth":327,"uti":2642,"ute":9518,"uta":14646,"utb":708,"utx":862,"utz":2136,"utt":153,"uts":624,"utu":1500,"uto":1475,"utr":268,"us ":9795,"ut ":685,"urb":881,"ura":8810,"urd":1157,"urc":233,"ure":3312,"urg":1560,"uri":5079,"url":279,"urk":2210,"urm":306,"urn":368,"uro":1987,"urp":319,"urr":7898,"urs":288,"urt":8830,"uru":8984,"ury":170,"upa":176,"ur ":5462,"upi":127,"upe":490,"umi":217,"umo":363,"uma":1860,"umb":322,"ume":1972,"unt":5181,"uns":142,"unk":576,"uni":2485,"uno":187,"unn":124,"unc":158,"und":3967,"una":6699,"unb":140,"ung":1494,"une":4343,"up ":152,"ukr":242,"uku":113,"ukt":390,"uko":10845,"ukl":257,"uki":593,"uke":693,"um ":1084,"uka":1067,"ulu":958,"ult":3635,"ulp":177,"ulo":533,"ulm":122,"ull":366,"uli":995,"ulg":169,"ule":1208,"ula":4220,"un ":5328,"uid":216,"uil":962,"uin":1030,"uir":110,"uis":870,"uk ":736,"uia":127,"uji":118,"uit":797,"uiz":145,"ul ":480,"uja":166,"ugh":140,"ugi":893,"uge":534,"ugo":167,"ui ":164,"uga":3668,"uhi":142,"ugu":650,"uha":2168,"uco":199,"uf ":132,"uda":33965,"ude":2755,"udi":1792,"ubo":127,"ubs":680,"ubr":1485,"ubu":129,"uca":203,"ue ":1044,"uce":126,"uci":117,"uch":496,"ucl":185,"uck":125,"uet":3427,"uev":148,"uer":1318,"ues":754,"uez":197,"uff":155,"udu":480,"udo":382,"ued":519,"ueb":168,"uen":35965,"uel":1556,"uek":1147,"uei":226,"ub ":167,"ua ":13559,"uaz":311,"uat":379,"uar":5388,"ual":33640,"uan":34449,"ubi":1182,"ubl":2352,"ube":1089,"uba":1426,"ud ":222,"uak":3352,"uai":215,"uag":257,"uad":292,"tzi":45858,"tzo":1861,"tzu":3457,"tza":20297,"tze":31257,"tyl":1154,"typ":482,"ty ":368,"twa":207,"tuz":3752,"tx ":291,"tur":8124,"tus":1524,"tut":13606,"tul":337,"tuk":1405,"tun":1118,"tum":153,"tub":959,"tua":12060,"tud":397,"tue":26377,"tug":593,"txu":656,"txo":1698,"txi":4866,"txe":2626,"txa":1807,"tz ":3179,"ts ":1033,"tre":1843,"tt ":158,"tra":13798,"tri":2959,"tru":540,"tro":7382,"tu ":19929,"tsa":4489,"tse":2634,"tsi":2184,"tso":2535,"tsu":3293,"tta":369,"tte":927,"tti":344,"tto":194,"tts":160,"to ":2344,"tmo":266,"tni":243,"tna":149,"toe":252,"tod":580,"toc":190,"toi":544,"tog":246,"tob":359,"toa":986,"tou":170,"tos":463,"tot":514,"toz":229,"tom":728,"ton":2972,"tok":1652,"tol":2657,"tor":7132,"top":659,"til":1069,"tik":17820,"tif":454,"tie":8056,"tig":462,"tir":1298,"tiq":263,"tit":1526,"tis":1599,"tin":3308,"tim":1547,"tip":247,"tio":1317,"thu":295,"tia":4907,"tib":10581,"tic":667,"tid":1081,"tiz":562,"tiv":202,"tla":841,"tle":497,"tem":2289,"ten":15717,"teo":777,"tei":891,"tek":9552,"tel":3847,"tee":1553,"teg":2380,"teh":241,"tea":12289,"teb":185,"ted":602,"th ":729,"tez":2099,"tev":241,"teu":131,"tet":1953,"tes":1845,"ter":11656,"ti ":3145,"tho":471,"thr":250,"the":2245,"thi":275,"tha":655,"ān ":123,"专 ":284,"並 ":155,"三 ":381,"丁 ":173,"万 ":297,"zèr":200,"之 ":273},"n_words":[10862913,12392263,10572779],"name":"eu","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/fa.json b/contrib/languages-data/fa.json
new file mode 100644
index 0000000..6acdfa3
--- /dev/null
+++ b/contrib/languages-data/fa.json
@@ -0,0 +1 @@
+{"freq":{"٬":970,"پ":57795,"و":425936,"ي":735907,"ً":1698,"َ":4564,"ُ":2202,"ف":144242,"ق":86257,"ك":2176,"ل":186547,"م":383367,"ن":505254,"ه":501741,"ّ":1083,"ِ":3108,"ٔ":10105,"خ":65646,"د":475980,"ج":87595,"ح":43181,"ت":373990,"ث":6476,"ب":289206,"ة":899,"ئ":9436,"ا":1111531,"آ":65724,"أ":1456,"ء":1000,"غ":15714,"ع":91539,"ظ":7379,"ط":45348,"ض":9683,"ص":43813,"ش":310066,"س":370480,"ز":155766,"ر":687985,"ذ":11817,"،":43281,"؛":1040,"۲":32624,"۳":25683,"۰":28783,"۱":63091,"۶":17216,"۷":19045,"۴":17256,"۵":18080,"۸":24510,"۹":42744,"گ":80261,"ک":285107,"چ":25230,"ژ":11180," ،":7698," گ":30777," ک":148021," ۸":2786," ۹":2591," ۱":47632," ۲":17985," ۳":5368," ۴":3375," ۵":3167," ۶":2742," ۷":2925," ن":61607," ه":83469," ل":8629," م":155700," ق":22376," ك":1185," ف":28537," ي":42307," و":134423," ص":10068," ش":162851," ط":22340," ض":1200," ر":51088," ذ":1128," س":118308," ز":19079," ع":23850," غ":6362," ا":284499," آ":62929," ج":29359," ح":15097," خ":24724," د":158860," ب":152331," ت":57718," ژ":5441," چ":17369," پ":40145,"کا ":2349,"کت ":2257,"ژوئ":1909,"ژي ":877,"ژه ":1588,"ژان":2068,"گيت":1091,"گير":2604,"کلا":1036,"کلي":1793,"کوچ":1236,"کيل":3244,"کمي":2269,"کنا":990,"کند":3501,"کنو":1241,"کنن":2076,"کوم":883,"کوه":1843,"کرا":939,"کشف":46287,"کشو":4351,"کست":856,"کزي":3453,"کرد":5509,"کار":7408,"کات":897,"کان":3075,"کاي":1649,"کام":1480,"کال":1004,"کبي":947,"کتر":1843,"کتا":2868,"کتب":2985,"کم ":997,"کل ":1774,"که ":49534,"کن ":1159,"کي ":15027,"کز ":2384,"کس ":931,"گور":1227,"گون":2525,"گوي":1532,"گلي":2047,"گفت":2210,"گست":862,"گري":2038,"گرو":2216,"گزا":1135,"گرد":3447,"گرا":2354,"گرف":2543,"گذش":1619,"گذا":1619,"گار":1384,"گاه":8669,"گان":3798,"گي ":4154,"گر ":3440,"ا، ":2188,"پان":3222,"پاد":1299,"پار":1859,"۹۹ ":1855,"۹۸ ":1731,"۹۷ ":955,"۹۶ ":844,"۹۲ ":854,"۹۱ ":894,"ئي ":825,"ات ":10939,"اح ":1044,"اج ":937,"۹۰ ":940,"پرا":1102,"پرد":1609,"پرو":1607,"اب ":5407,"ئن ":919,"پاي":4002,"پتا":3629,"پنج":5682,"۸۹ ":864,"۸۸ ":992,"۸۳ ":1400,"۸۵ ":947,"۸۰ ":846,"۸۱ ":878,"اي ":55848,"بت ":1678,"اه ":14391,"او ":4256,"با ":13059,"ان ":92173,"اً ":1671,"اف ":1676,"پيو":992,"پيش":2404,"پير":873,"ام ":15467,"ال ":24240,"اق ":2509,"اع ":1667,"اط ":832,"ار ":39983,"اد ":13676,"اش ":1063,"از ":51085,"اس ":3550,"اسک":971,"بي ":8868,"ت، ":2631,"اضي":879,"ارگ":1672,"اصل":2729,"اسپ":971,"اعا":905,"اعت":1127,"اطل":2273,"اعي":1202,"۹۹۹":1558,"۹۹۸":1457,"ادا":1809,"احي":1808,"اخت":6534,"احم":966,"احت":888,"ارب":1131,"ارا":5657,"ادي":8285,"اده":8465,"ادب":902,"بق ":16057,"ادر":1545,"ادش":1271,"ازي":4465,"است":105076,"ازه":1433,"ازن":1098,"اسا":2986,"ازم":1894,"ازد":2915,"اري":13153,"اره":35608,"ازا":956,"ارن":1623,"ارو":2419,"ارم":2814,"ارز":893,"ارس":5553,"ارش":1311,"ارص":1726,"بل ":1849,"ارد":9154,"ارت":4139,"اشن":1730,"ارک":47474,"اشي":1130,"اشد":5423,"به ":41777,"اسم":943,"اسل":2665,"اشا":865,"اشت":3533,"اسي":5578,"بن ":2042,"اسر":1003,"بع ":2965,"ائي":1624,"ابا":1519,"ابت":1082,"ابر":3253,"ابع":2818,"ابي":1834,"ابل":1424,"ابق":1003,"ابو":1132,"اتر":1045,"اثر":1056,"اتي":2176,"اتو":1748,"اجر":1089,"ئيه":1089,"بر ":33260,"آب ":1128,"آثا":830,"آبا":1911,"آزا":908,"آذر":4073,"آن ":10516,"آما":970,"آلم":1164,"آمر":3443,"آمد":1740,"آمو":1076,"آنه":16519,"آور":2786,"آهن":945,"آغا":1416,"چه ":1398,"جزي":1617,"خت ":2202,"حل ":927,"جرا":1045,"جري":3087,"جار":1185,"جاد":1192,"جان":5597,"جام":2439,"جاه":2146,"جاي":1843,"حال":1317,"جنو":3614,"جها":2193,"جمي":1716,"جمع":1997,"جمه":2045,"جمو":2028,"خش ":8012,"حي ":1372,"د، ":4587,"خي ":1696,"دا ":1969,"خه ":1063,"حسي":842,"حزب":975,"حرم":1392,"حده":1001,"حدو":1445,"جنگ":1615,"جوا":942,"جود":1945,"ختر":883,"خان":4015,"خار":1012,"خاب":828,"حمد":2876,"در ":87864,"دد ":1157,"دن ":19684,"تگا":1213,"خست":1155,"ده ":55594,"دو ":4399,"ر، ":2027,"دي ":13399,"خته":2886,"ختل":1697,"خدا":1090,"دل ":1012,"دم ":1329,"خلي":855,"دان":9861,"دام":1683,"دال":1294,"داي":1703,"دار":14265,"داز":1707,"داس":1080,"داش":2836,"داخ":1028,"داد":4687,"خوا":3065,"خود":3802,"خور":1687,"دبي":1292,"رج ":831,"درگ":1116,"رت ":5024,"رد ":14918,"را ":12544,"دست":3550,"دسا":1658,"رب ":1902,"دشا":1262,"درس":984,"درج":847,"درا":1451,"دري":2646,"درو":1489,"دول":1633,"دون":991,"دوم":3136,"دوي":2445,"دهم":3138,"دوا":1480,"دود":1919,"دور":3332,"دهٔ":2250,"دها":1941,"دهد":1019,"دني":1772,"دهس":4371,"دمي":947,"دند":2383,"ذار":1352,"رف ":16580,"ديو":1392,"ديم":1100,"دين":2614,"ديا":931,"ديد":3091,"دير":977,"رش ":1790,"رس ":3514,"رز ":943,"دما":1034,"دلب":839,"اقي":1204,"افي":974,"اقع":6290,"الل":1166,"ان،":3374,"الي":6506,"الن":1060,"اما":3065,"الم":2177,"اله":1801,"امب":7272,"امر":1231,"الت":2172,"الب":1010,"الا":3840,"الد":1099,"الع":862,"افز":1059,"افر":840,"افت":2627,"اين":37088,"ايل":1631,"ايي":11337,"ايه":1592,"ايس":842,"انگ":3856,"ايش":2420,"ايا":4829,"ايت":2340,"ايج":4814,"انک":896,"ايد":1943,"ايز":1142,"اير":12515,"، ":41769,"بخش":8437,"اپ ":1341,"اهر":1198,"اهش":1637,"اني":16598,"باي":4838,"انه":7584,"انو":5568,"اهد":836,"انق":1002,"اها":4950,"بان":6032,"انن":2230,"بال":3191,"اند":8054,"باد":2479,"باز":4359,"انز":1366,"بار":4430,"انش":5048,"باش":6504,"انس":4593,"باس":1834,"انص":1766,"امن":1026,"انا":2522,"امه":4047,"امو":1040,"انت":3295,"بات":1162,"امي":5640,"انج":1903,"امل":2517,"اوي":1440,"اول":3794,"اور":2685,"الک":1567,"اوت":2392,"اهي":3496,"اوا":1252,"امپ":1281,"تا ":5488,"بسي":1565,"برگ":2294,"ايگ":1002,"بدا":1231,"برا":9676,"برد":2157,"برخ":1084,"برو":884,"برن":1758,"بري":2753,"بزر":3175,"بست":2397,"تر ":6150,"بعد":1118,"بني":1199,"بنا":1507,"بند":1464,"ا ":87899,"ب ":20308,"ح ":3843,"خ ":4523,"د ":139796,"بين":3092,"بيم":902,"بيس":5590,"بيش":2092,"ت ":137024,"بيا":1448,"بير":1134,"ث ":1238,"بور":941,"ج ":5421,"بود":10929,"تي ":8134,"اک ":1290,"تن ":2136,"ته ":15461,"تم ":2196,"ثر ":1178,"؛ ":940,"تبر":18681,"تال":1624,"تبا":2109,"تان":31858,"تام":3922,"تاه":3675,"تاي":3199,"تاد":4918,"تاش":890,"تار":5154,"تاب":4019,"تخا":898,"تحد":1114,"تري":7869,"ترو":1889,"ترا":2202,"ترن":839,"جا ":15939,"تصا":1521,"تشر":878,"ترک":1945,"تصد":3250,"تشک":1423,"تعد":829,"تغي":867,"اژه":1048,"تفا":3357,"تقا":902,"تقو":993,"تهر":2967,"تها":877,"ثار":853,"تند":2837,"تمي":4044,"تلف":1376,"تما":2359,"تلا":1147,"تيم":1139,"تين":1737,"تيا":863,"تون":1309,"تول":2859,"تور":2332,"توس":3860,"توا":4394,"پس ":2219,"تيک":1005,"اکن":1376,"اکت":3167,"اکس":936,"جه ":2283,"جي ":1241,"حت ":897,"پت":4019,"پا":14295,"پس":3568,"پد":1266,"پر":7121,"گ ":9285,"پو":2856,"په":1230,"پن":7042,"پي":8594,"پل":1723,"ک ":69398,"مک":3060,"ي،":6981,"وي":29381,"وو":995,"يب":4572,"يا":96279,"يع":3352,"يز":10370,"نگ":14413,"يس":23115,"يش":9793,"يص":2081,"يخ":3819,"يد":17899,"ير":33847,"يت":13742,"يج":6291,"يح":1015,"نک":2007,"يه":14576,"ين":90729,"وچ":1800,"يو":11558,"يق":4160,"يم":13074,"يل":23315,"يف":3191,"وپ":2022,"يي":14011,"وژ":952,"وک":2822,"پ ":2440,"وگ":1347,"يچ":1842,"يپ":878,"يژ":1013,"يک":36397,"يگ":5574,"فع":2000,"فض":915,"فز":1100,"فر":14718,"فس":1042,"فد":1178,"فت":17008,"فا":9831,"قع":6601,"قط":1598,"قش":1281,"قس":1226,"قر":7195,"قد":3410,"قت":1605,"في":8057,"قب":1853,"فه":2747,"فو":4734,"فن":1501,"قا":9232,"فل":2016,"فق":1179,"ل،":834,"قي":10959,"قل":3288,"قم":3612,"قه":3627,"قو":3406,"لق":1885,"لف":2767,"لط":1095,"لع":1660,"لد":3211,"لح":1516,"لز":963,"لس":4361,"لر":1340,"لا":28241,"لت":4427,"م،":1074,"لب":3816,"مع":25874,"مغ":887,"مص":1370,"مط":1157,"مف":852,"مق":2922,"مل":8979,"مت":12979,"لو":11064,"مج":4572,"ن،":5004,"لي":31085,"لم":9307,"لل":2270,"له":9135,"مب":9205,"لن":3401,"ما":76186,"مز":2088,"مر":23149,"مش":4307,"مس":6882,"مخ":2590,"مح":8770,"مد":9936,"نظ":3887,"نع":1089,"نط":1891,"نم":4839,"نق":3830,"نف":2887,"نج":25669,"ه،":2861,"مي":78888,"نت":9394,"مو":18704,"مه":13616,"نب":2742,"نا":34982,"من":11960,"نص":3159,"نش":8281,"نس":7940,"نز":3319,"نر":3727,"ند":45085,"نخ":1510,"مپ":1882,"هف":7528,"هل":3945,"هم":15355,"وئ":3203,"وا":39119,"هن":8401,"نه":35756,"هب":1561,"ها":75779,"نن":4681,"هت":1153,"نو":29290,"هج":3131,"ني":39040,"هد":4476,"هز":15706,"هر":26999,"هش":9098,"هس":7389,"هص":1630,"وع":5118,"وق":2152,"وف":3392,"ون":20803,"وه":7280,"ول":18645,"وم":15237,"هي":7368,"وت":7324,"مچ":1311,"هو":6697,"هه":1141,"وب":8255,"ود":37999,"وح":978,"لک":3276,"وج":4297,"وس":20717,"لگ":942,"وز":11168,"ور":41200,"وط":1483,"وض":931,"وص":985,"هٔ":10019,"وش":6987,"چ ":1702,"خو":10471,"دت":1296,"دا":45095,"خه":1201,"دب":2008,"دخ":1393,"خي":3813,"خل":2639,"خط":1338,"خر":3213,"خد":1570,"خص":2448,"خش":9472,"خس":1563,"تگ":2018,"ده":71326,"دو":21727,"ر،":2098,"دي":30900,"دف":1399,"دق":1055,"دل":3645,"دم":4441,"دن":24729,"ذا":2266,"دد":1619,"در":101792,"دش":2498,"دس":6715,"جي":2986,"جو":5371,"حت":1860,"حا":5566,"جن":6990,"جه":5500,"حب":830,"جل":2106,"جم":10111,"جس":1114,"جز":2725,"جر":5927,"جد":2691,"بک":2261,"خت":10589,"حي":3848,"د،":4720,"تک":1874,"حم":4596,"خا":9456,"حو":1605,"حق":1478,"حل":2638,"حص":1359,"حز":1102,"حر":3367,"حس":2640,"حد":3791,"اژ":1490,"تغ":1106,"تف":4099,"تم":9655,"تل":4375,"تق":4035,"تو":18858,"ته":20979,"تن":7644,"ثا":1467,"تج":1247,"تح":3912,"تر":24685,"تخ":2986,"تد":1435,"تش":4284,"تص":5949,"تس":1020,"تع":2873,"جا":32040,"جب":888,"جت":900,"تي":16351,"اک":9717,"ثر":1600,"اگ":2405,"بع":5160,"به":44819,"بن":6697,"بل":4961,"بق":17149,"بخ":9337,"بد":4503,"اً":1680,"بت":3170,"اي":143043,"اه":29729,"او":19244,"بط":1115,"بش":1384,"بز":4117,"بس":4538,"بر":58307,"اپ":4030,"تا":68031,"تب":22518,"بو":16880,"بي":27336,"ت،":2720,"ا،":2256,"ائ":3170,"از":65740,"ار":177821,"اد":40749,"اض":1760,"اص":6315,"اش":15541,"اس":126776,"ات":19285,"ئو":1601,"اب":23054,"ئن":1012,"اخ":9486,"اح":6972,"اج":4954,"ئي":4511,"اث":1612,"اف":9698,"اق":12353,"ام":46608,"ان":164721,"با":53255,"ال":53226,"اع":7941,"اغ":1903,"اط":5875,"اظ":989,"آب":4072,"آث":840,"ٔ ":10096,"آذ":4112,"آر":1739,"آس":1327,"آز":1293,"آغ":1527,"آف":915,"آل":2867,"آم":7747,"آو":3278,"آن":29292,"آه":1034,"آي":2113,"غي":2298,"عي":7120,"غر":4096,"عل":6634,"عم":5201,"غا":3393,"عن":4842,"عه":5570,"عت":3100,"عد":3656,"عر":22149,"عض":1560,"عا":7561,"عب":3193,"ظا":1618,"طل":3586,"طق":2226,"شگ":3723,"ظر":2221,"شک":5793,"طي":2187,"طه":1305,"طو":2844,"طر":3523,"ضي":1604,"سک":2829,"طا":2296,"طب":17352,"ضو":1678,"سپ":5713,"صل":3684,"صف":2604,"ضر":876,"زگ":1068,"صو":4924,"صن":1205,"ضا":2719,"صي":2649,"شف":46579,"شص":3397,"شش":4939,"شع":1023,"رک":59526,"صد":15627,"صر":2425,"رگ":10959,"شم":40834,"شن":7520,"صا":3027,"شه":20211,"شو":16524,"صت":1929,"شي":10582,"سع":1222,"سط":4899,"دگ":4253,"سف":2299,"رپ":863,"سي":82351,"شت":20229,"رچ":966,"سو":9339,"شب":2714,"سه":5822,"شا":14113,"سن":5406,"سم":5040,"سل":6428,"شر":11318,"شد":71050,"شخ":1686,"زش":2575,"زه":4419,"سب":3293,"زن":6220,"سا":40397,"ست":157670,"زو":2068,"زم":7708,"سد":1583,"سر":10053,"زي":19577,"سخ":1157,"دک":1541,"رس":21469,"رش":6643,"رر":874,"رز":5465,"رص":2328,"رض":1657,"رع":995,"رل":1461,"رق":7164,"رف":22038,"رو":41463,"ره":48555,"زب":4854,"زا":26314,"رن":9102,"رم":13248,"ري":64671,"زر":4516,"زد":7554,"ذر":4784,"ذش":1625,"رآ":1627,"رب":15211,"را":79279,"رت":10552,"رج":4101,"ذي":851,"رخ":3587,"رح":1384,"حک":1290,"رد":33206,"ف ":71140,"ع ":15503,"غ ":1398,"ص ":2346,"ط ":7167,"ظ ":1176,"ر ":213685,"ز ":65089,"س ":20728,"ش ":24544,"ً ":1683,"ي ":278753,"ه ":289561,"ن ":207616,"و ":126016,"ق ":24581,"م ":45325,"ل ":55421,"۳۰":2261,"۳۱":1756,"۳۲":1475,"۲۷":1672,"۲۶":1845,"۲۹":1827,"۲۸":1810,"۲۳":1837,"۲۲":2072,"۲۵":2015,"۲۴":2310,"۴۲":932,"۴۳":828,"۴۰":1179,"۴۱":851,"۳۹":874,"۳۸":3317,"۳۷":1442,"۳۶":1469,"۳۵":1620,"۳۴":1319,"۳۳":1412,"۱۰":3498,"۰۱":1251,"۰۰":7262,"۰۳":960,"۰۲":955,"۰۵":991,"۰۴":954,"۰۷":971,"۰۶":1030,"۰۹":925,"۰۸":1126,"۲۰":6610,"۲۱":2252,"۱۴":2656,"۱۳":8246,"۱۲":3243,"۱۱":2760,"۱۸":4100,"۱۷":2716,"۱۶":2670,"۱۵":2740,"۱۹":21618,"۷۴":853,"۷۳":1099,"۷۶":1037,"۷۵":996,"۷۰":1130,"۷۲":902,"۷۱":971,"۶۸":828,"۶۹":864,"۸۷":1462,"۸۶":1335,"۸۵":1706,"۸۴":1157,"۸۳":1867,"۸۲":1238,"۸۱":1392,"۸۰":1426,"۷۹":1133,"۷۷":1205,"۷۸":1210,"۵۰":1455,"۵۲":836,"۵۱":867,"۵۴":849,"۵۳":863,"۴۴":839,"۴۵":921,"۴۸":900,"۴۹":832,"۶۱":861,"۶۰":1533,"۶۵":892,"۵۷":978,"۵۸":881,"۵۵":896,"۵۶":829,"۵۹":829,"۸۹":1660,"۸۸":1649,"۹۰":1972,"۹۳":2257,"۹۴":1978,"۹۱":1915,"۹۲":1938,"۹۷":3928,"۹۸":5762,"۹۵":1920,"۹۶":2485,"۹۹":9457,"۸ ":9234,"۷ ":8195,"۹ ":8815,"کي":21693,"کس":3863,"کش":52864,"کر":10838,"کز":5951,"کد":1167,"کت":10947,"کو":9889,"که":50638,"کن":11029,"کم":5441,"کل":6076,"کب":1527,"کا":21352,"گف":2232,"گل":4014,"گن":1256,"گه":1120,"گو":7699,"گي":9941,"گذ":3598,"گز":1726,"گر":18517,"گش":860,"گس":1433,"گا":16359,"۵ ":8279,"۶ ":7892,"۳ ":8356,"۴ ":8000,"۱ ":8359,"۲ ":7902,"۰ ":11742,"چي":2258,"چن":2891,"چه":10435,"چو":1224,"چک":1330,"چا":2573,"ژا":3316,"ژه":1974,"ژي":1330,"ژو":2750,"چين":1123,"چني":1160,"چها":6583,"چهل":2101,"چند":1391,"چار":831," ، ":7275," خ ":829," و ":107907," حا":2893," جن":6563," جه":2834," جل":883," جم":4408," جو":1700," جد":1121," جز":2226," جر":1081," جا":5217," تي":2383," اک":4067," خل":1477," خي":1014," خو":9064," دا":22315," خر":1716," خد":1173," خط":1264," حق":954," تک":1140," حم":1000," خا":5629," حو":858," حز":987," حر":1265," حس":1494," حد":1443," به":41159," بن":5266," بل":1919," بع":1479," بز":3261," بس":2873," بر":23110," اي":60506," اه":1491," او":12009," بخ":8934," بد":2040," اق":1653," اف":3495," ال":9180," با":32041," ان":13519," ام":5995," اط":2539," اع":2096," اد":2220," ار":5819," از":47219," اس":93321," اش":2136," اص":3471," اب":2944," ات":1821," اث":1151," اج":1670," اح":1812," اخ":1959," تو":11109," ته":3565," تن":2119," تم":1148," تل":1644," تق":2105," تع":1944," تش":1637," تص":1087," تر":7158," تخ":1066," تج":1001," تح":1857," تا":10536," تب":1758," بو":12627," بي":13492," آم":7369," آل":2819," آو":2892," آه":1013," آن":28846," آي":1817," آذ":4109," آر":1633," آث":831," آب":3531," آغ":1486," آس":1300," آز":1278," شک":1757," طو":1797," عض":849," عر":3450," عا":1962," عب":2630," غر":3490," عل":4955," عم":2914," عن":2426," غي":1131," سط":837," سف":999," سي":58158," شب":2526," سه":2789," سو":5501," سم":1003," سن":2452," شا":6389," سل":2036," شر":8538," شخ":981," شد":64662," شص":1903," شش":4604," شو":11010," شي":3635," شم":35388," شن":2904," شه":16707," صد":3342," سپ":4409," صف":1164," صو":2043," صن":981," طب":16842," طر":1891," دس":4487," در":93973," دي":6271," دو":15789," ده":8568," دن":1430," دل":1101," حک":1033," را":15933," رس":3186," رش":1541," ري":3051," رف":1064," رو":19239," زب":3421," زا":2852," رم":1079," زي":3601," سد":907," سر":6965," زم":4334," سب":1207," زن":2927," سا":27260," ست":1210," ۲۱":1529," ۲۰":5779," ۱۹":20860," ۱۵":1981," ۱۶":1865," ۱۷":1946," ۱۸":3370," ۱۱":2053," ۱۲":2491," ۱۳":7504," ۱۴":1848," ۳۰":1169," ۲۸":971," ۲۹":1004," ۲۶":1106," ۲۷":943," ۲۴":1600," ۲۵":1220," ۲۲":1386," ۲۳":1186," ۱۰":2514," ۷ ":1034," ۶ ":835," ۹ ":923," ۸ ":918," پل":1189," پن":6100," پو":1775," پي":6231," پا":11101," پر":5507," پد":1204," پس":3103," لا":2038," مل":2782," مق":2722," مع":22757," مط":1111," مص":1331," مس":5314," مش":3603," مر":12704," مد":3474," مح":8486," مخ":2396," لي":1375," مج":4247," لو":1161," مت":6834," ما":10891," مب":1057," نف":1957," نق":2575," نم":3573," نظ":2990," نخ":1392," نر":962," نز":1060," نس":1428," نش":1872," نص":1002," نا":13232," من":6720," مه":4441," مو":8861," مي":39390," هف":7486," هن":3049," وا":10473," هم":7875," هر":3177," هز":15471," هس":2372," هش":6627," نو":12914," ها":28631," نه":5384," ني":6690," هج":2983," فر":9595," فع":1613," فا":3850," فل":1560," قا":3036," فو":3446," قب":1076," في":4001," قد":1747," قر":6081," قم":3484," قل":1179," قو":1126," يک":22683," وج":1401," هي":1336," هو":2401," وس":1484," وز":1216," ور":1520," ول":1293," مک":1516," وي":5201," يا":14595," نگ":1352," يو":1770," ۱ ":999," ۵ ":990," ۴ ":1026," ۳ ":1037," ۲ ":1134," کت":2907," کر":7960," کش":52338," کل":3525," کن":7370," کم":2467," کو":5973," که":48270," کي":3060," کب":1107," کا":11187," گا":3551," گل":1221," گف":2153," گي":3912," گو":4533," گر":10421," گذ":2142," ژا":2692," ژو":2172," چا":2040," چي":1498," چن":1727," چه":9110,"۶۰ ":1024,"۲۰۰":3965,"۳۸۳":931,"۲۸ ":1132,"۲۹ ":1101,"۲۶ ":1164,"۲۷ ":991,"۲۴ ":1524,"۲۵ ":1126,"۰۰۰":1569,"۲۳ ":1040,"۲۲ ":1199,"۲۱ ":1215,"۲۰ ":1572,"۳۱ ":837,"۱۹۸":3784,"۱۹۹":7313,"۱۹۳":1178,"۱۹۲":829,"۱۹۵":879,"۱۹۴":946,"۱۹۷":2720,"۱۹۶":1373,"۱۳۸":2507,"۳۰ ":1283,"۰۰ ":2532,"۱۲ ":1158,"۱۱ ":1041,"۱۳ ":1130,"۱۴ ":1282,"۱۵ ":1270,"۱۶ ":1284,"۱۷ ":1225,"۱۸ ":1153,"۱۹ ":1122,"۱۰ ":1507,"فر ":1868,"فت ":4170,"قع ":6048,"فار":2458,"فاد":2141,"فاع":864,"فتا":2708,"فته":4673,"فتم":1892,"فتص":1637,"فه ":1283,"في ":1976,"عرو":1419,"عرف":15753,"عرب":2186,"عرا":897,"عدا":1093,"عبد":885,"عات":1421,"عال":2039,"عبا":1366,"عاد":1232,"غرب":3482,"عيت":1749,"عمل":1014,"عنا":1018,"عمو":1947,"عنو":2161,"غان":879,"عني":1446,"غاز":1437,"علا":887,"علي":2182,"علو":970,"عما":1036,"علم":1702,"غير":1279,"صي ":884,"شما":36739,"شهر":16753,"شنا":4807,"صاد":1085,"شمي":1876,"رکت":2292,"رکز":5506,"رکي":1378,"شور":5763,"شود":8535,"شهو":994,"شون":1223,"شير":1302,"شيد":1499,"شين":1156,"ضي ":873,"رگا":1053,"رگذ":955,"رگر":1703,"رگز":904,"سپت":3558,"سپا":1035,"صول":885,"صور":2307,"ضاي":1046,"صلي":1295,"طي ":1075,"طه ":1047,"ظر ":999,"طبق":15902,"طرا":958,"عت ":1441,"عد ":843,"طلس":913,"طلا":2183,"طقه":1497,"طور":1362,"ظام":977,"شکي":1859,"شکل":1326,"شگا":3095,"عه ":4310,"عي ":3265,"زد ":1250,"ري ":28175,"رن ":1348,"ذشت":1620,"زب ":861,"ره ":40871,"رو ":1622,"ديگ":2478,"ديک":1038,"رق ":1211,"ذرب":3735,"رم ":3802,"رجه":899,"ردم":1119,"رده":3960,"ردن":2001,"ردي":2386,"ردا":4731,"رتب":927,"ربي":3967,"رتي":854,"راک":1311,"ران":23644,"ربا":5086,"راه":2985,"راو":1025,"راي":12001,"ربر":1171,"راف":1247,"رام":1664,"رال":1192,"راب":3402,"رائ":1115,"راح":1002,"رات":2813,"رار":4645,"راد":1759,"راز":1040,"راس":2348,"زش ":1033,"رفت":3769,"سر ":1698,"زي ":8767,"رصد":2148,"رشي":1128,"ست ":82115,"رسا":1284,"رست":10190,"رشت":1369,"رسم":1031,"رسي":3685,"زه ":3212,"رزش":844,"سم ":1194,"زدي":1099,"زده":4341,"ريک":4778,"روپ":963,"ريه":3010,"ريو":908,"رين":6859,"ريل":1861,"ريق":887,"ريا":4939,"ريخ":2752,"ريت":1617,"ريز":1529,"رنگ":1040,"ريس":1451,"روه":2711,"روي":3959,"ري،":1064,"روف":1550,"رون":2388,"روم":1326,"رور":969,"روز":5199,"رود":4045,"رهٔ":1083,"روس":7942,"روش":1626,"رهن":1422,"روا":2639,"زاي":1000,"زان":1010,"رها":4101,"زبا":3526,"زار":18512,"رند":2708,"زاد":3573,"رمي":2432,"رنا":1487,"رما":3636,"رمز":959,"سط ":3314,"رقي":4879,"شد ":26077,"شر ":1296,"شش ":1401,"سن ":1047,"سه ":5126,"زرگ":3314,"شت ":4426,"سي ":13405,"ستگ":1285,"شف ":46304,"زيک":1213,"سري":988,"سرو":889,"سرا":2106,"دگي":1827,"شه ":1221,"دگا":1978,"زما":4471,"سام":2387,"سال":16867,"سان":4639,"ساي":1790,"زند":3098,"ساخ":3059,"ساس":1619,"ساز":3943,"زنا":1261,"زمي":2413,"ساب":1251,"ستا":39209,"ستب":15399,"ستر":2167,"ست،":1386,"ستف":2194,"ستن":2498,"زيا":1067,"ستم":1666,"ستي":2717,"سته":2853,"ستو":1258,"زيس":843,"زير":3410,"رگ ":3861,"صر ":1320,"سلا":2748,"سمت":1146,"شي ":3088,"صت ":1755,"صد ":14363,"رک ":48005,"شرق":5570,"صل ":1169,"شده":27411,"شدن":16377,"شصت":1903,"شرک":1669,"شصد":1494,"ششم":1955,"ششص":1494,"شتا":2276,"سوم":2657,"سوي":1270,"شتر":2340,"شتص":1588,"شبه":953,"شاه":3881,"شان":2849,"شام":1210,"شار":1758,"سند":1687,"شاخ":862,"سمي":1232,"شخص":1423,"سيل":1493,"سين":2125,"سيم":1189,"سيق":1188,"سيو":851,"شتم":2005,"سيص":1797,"سنگ":854,"سيس":2418,"سير":939,"سيد":2026,"شتي":869,"سيا":50847,"شته":5322,"يکا":4258,"يکم":1999,"يکي":12431,"يگر":3114,"يگا":1094,"يچ ":1170,"وچک":1078,"يقي":1448,"يلا":5819,"يلي":2362,"يلم":2163,"يما":2897,"يله":1142,"يلو":2447,"يند":2629,"ينج":15434,"ينا":1201,"يمن":836,"ينت":984,"يمي":2048,"ينو":1227,"يني":2869,"ينه":1667,"يهٔ":843,"يوس":866,"يوا":1011,"يون":3682,"ينگ":934,"وپا":979,"يصد":1797,"يشه":906,"يشت":1412,"يسي":2944,"نگي":1776,"نگل":2493,"يسه":1037,"يسن":1399,"يزي":2379,"يست":11437,"يزه":870,"يسا":894,"نگا":2016,"يره":2645,"يرو":2532,"يري":2723,"يزد":1123,"يعي":880,"يجا":4729,"يتا":2607,"يتي":884,"يده":2751,"يدن":1012,"يدي":1302,"يرا":13163,"يرد":1088,"يخي":912,"يدا":2030,"يدل":878,"ياف":1602,"يال":2762,"يان":11075,"يبا":1219,"يام":955,"ياه":1109,"ياي":3214,"ياس":2249,"يار":48948,"ياز":1404,"ياد":1729,"يات":2044,"ياب":1358,"يک ":14180,"وز ":3949,"ور ":11858,"ود ":25900,"نقش":885,"هٔ ":10011,"نفر":1610,"وش ":1865,"وس ":2204,"نمو":979,"نند":4531,"هاس":15525,"هار":8046,"نما":2997,"ها،":830,"وع ":1666,"نقل":1074,"وط ":936,"نيز":2529,"نيس":1397,"نير":1331,"نيا":4775,"ني،":840,"نوي":5351,"نون":2065,"نور":1647,"نود":1940,"نوب":3858,"نهم":1851,"نوا":6754,"نوع":1934,"نوش":1776,"نوز":853,"هاي":33269,"هان":4405,"نها":19728,"نهص":1628,"هدا":927,"وف ":1708,"نيک":1404,"نين":1804,"نيم":949,"نيو":887,"هجر":2343,"هست":7247,"وم ":4571,"هري":1954,"هشت":6878,"ون ":8984,"هرا":3739,"هرم":1056,"هره":1437,"هزا":15536,"هرس":8280,"ول ":5820,"وي ":9400,"ي، ":6814,"وه ":3934,"هشم":1388,"هصد":1629,"معي":1766,"معم":1600,"معن":1773,"معر":16933,"معا":1278,"هد ":1760,"هر ":8315,"مقا":1363,"منا":1581,"ناب":867,"مند":1285,"ناخ":1262,"منت":1332,"منط":1759,"ناس":3745,"نار":1067,"ملل":1010,"مله":1083,"ملي":2015,"مهم":1073,"مهو":1618,"موا":1351,"موج":939,"مور":2297,"موز":1395,"مود":1287,"موس":2124,"موع":2058,"نام":12986,"نان":3607,"ناي":2070,"نتش":1291,"نتر":1378,"مون":1614,"مول":1953,"ميل":5431,"ميد":1645,"مير":1559,"ميا":3140,"نتي":1863,"نجم":2428,"نجا":19529,"مين":23588,"ندا":3431,"ند،":1301,"نخس":1148,"ندو":1099,"نده":6796,"ندر":1455,"ندس":875,"ندي":2668,"هل ":2462,"نزد":2165,"هم ":4541,"نسا":1177,"نست":1454,"ندگ":2266,"نسو":981,"نشا":1366,"وب ":4521,"نصد":1688,"وت ":2984,"هي ":3587,"نطق":1746,"نظا":1022,"نظر":1892,"نشگ":2179,"ونا":1554,"ومي":5536,"وند":2535,"ولي":4939,"ومت":2798,"يع ":1026,"ولت":1350,"ولا":2437,"ولد":1454,"يش ":4417,"يس ":3929,"نگ ":4653,"يق ":1173,"ويژ":863,"يف ":1505,"ويچ":897,"مکا":1239,"ويي":925,"وين":2044,"ويه":2289,"ويم":1400,"ويز":853,"ويس":4678,"وير":1085,"وني":3417,"ونه":1782,"وها":1116,"يي ":11355,"يه ":12282,"يو ":1257,"يم ":5175,"ين ":61177,"يل ":7962,"هفت":6908,"هنگ":2686,"واژ":1094,"مچن":1005,"وتب":889,"وجو":2093,"لکت":1073,"هنر":1330,"وار":3450,"واز":2127,"هند":2551,"واد":2014,"همي":4888,"وئي":1673,"واب":2536,"وئن":956,"هما":980,"هور":2961,"وبي":1735,"همچ":1222,"هوا":1704,"واه":871,"واي":1826,"واق":6339,"وال":1232,"وان":9374,"وام":2251,"وري":9228,"وست":8309,"وزه":1349,"وزن":1058,"وزي":1584,"يا ":15452,"وسط":3323,"وسي":3688,"وشت":1878,"يب ":1599,"ود،":1062,"ودر":855,"ودن":1173,"ودي":1606,"وده":3280,"ورا":3355,"ورز":1216,"ورش":1426,"ورد":3566,"ورت":2597,"وره":2981,"ورو":1136,"يد ":7931,"ير ":7783,"يز ":4213,"يت ":7713,"يج ":845,"يخ ":2265,"وعي":1123,"وعه":1942,"لد ":1738,"لس ":1850,"لت ":2709,"لا ":1918,"لب ":937,"م، ":1039,"لف ":1009,"له ":6963,"ما ":2248,"لم ":2584,"قمر":3187,"قلا":1065,"قيق":1249,"قوي":1155,"فعا":1470,"فرد":869,"فرا":3846,"فرو":1305,"فري":1360,"فرم":1232,"فره":1581,"فزا":918,"قه ":2804,"قي ":7279,"فيز":835,"فيل":2814,"قدي":1049,"قرن":906,"قرا":4415,"قال":936,"قاب":1236,"فوت":1061,"فور":2032,"قان":1259,"لله":1097,"لما":2054,"ماع":870,"لمي":1688,"مات":1140,"مار":39512,"لند":1567,"ماد":1634,"لمل":950,"نس ":921,"نش ":1280,"ند ":23182,"ه، ":2793,"مي ":37976,"نج ":1914,"ني ":21211,"مشه":1314,"مرک":5626,"نو ":876,"نه ":11112,"مسي":1271,"ها ":10366,"مست":1149,"مسا":1677,"مري":7089,"مرو":1152,"مرد":2093,"مرا":2087,"مرب":1215,"مدي":1420,"مدر":931,"مخت":1330,"مدا":1385,"محر":1442,"محل":1270,"محم":2301,"ليو":834,"لين":2134,"مجم":2201,"مجل":977,"ليت":1597,"ليا":2311,"متو":1555,"ليس":3144,"ليد":1495,"ليل":1108,"لوي":1612,"لوم":2898,"متر":3763,"متح":1213,"مال":4589,"مام":1588,"مان":14800,"ماه":2044,"ماي":4595,"مبر":7239,"مد ":3240,"لو ":1092,"مت ":2632,"ن، ":4905,"لي ":12663,"مه ":7560,"نا ":1758,"من ":2003,"نت ":1363,"مل ":2783,"لاح":1096,"لاد":4785,"لار":867,"لاب":1067,"لات":2844,"لاق":1169,"لاف":888,"لاس":997,"لاع":939,"لبر":1058,"لاي":1580,"لام":3536,"لان":2170,"لال":908},"n_words":[8069793,10004435,6796528],"name":"fa","type":"arab"} \ No newline at end of file
diff --git a/contrib/languages-data/fi.json b/contrib/languages-data/fi.json
new file mode 100644
index 0000000..2c587bd
--- /dev/null
+++ b/contrib/languages-data/fi.json
@@ -0,0 +1 @@
+{"freq":{"D":15745,"E":23140,"F":13820,"G":13964,"A":40046,"B":22964,"C":22892,"L":28771,"M":37698,"N":19249,"O":13807,"H":28813,"I":21256,"J":16698,"K":43440,"U":7805,"T":36899,"W":9333,"V":24290,"P":38188,"S":77764,"R":24157,"Y":10043,"X":1821,"Z":1858,"f":29615,"g":68514,"d":158713,"e":1194104,"b":43435,"c":44492,"a":1814181,"n":1349748,"o":934203,"l":885783,"m":433706,"j":314669,"k":691662,"h":239330,"i":1579260,"w":11494,"v":295575,"u":751889,"t":1169051,"s":1099978,"r":475344,"q":1633,"p":248621,"z":7860,"y":250687,"x":8131,"é":2342,"ä":426790,"ö":56679,"š":2606," l":70812," m":89685," n":43615," o":180629," h":39470," i":21438," j":161666," k":172428," d":8045," e":69090," f":6572," g":3193," a":72510," b":5870," c":2712," y":39165," u":13744," t":117334," v":98965," p":89643," s":126171," r":34331," J":16375," K":41900," H":27293," I":17045," N":17648," O":11538," L":26471," M":33953," B":21331," C":19265," A":36093," F":12406," G":12952," D":13481," E":21275," Z":1697," Y":9517," S":71766," R":22636," P":35182," W":8721," V":22294," U":6908," T":34378," ä":2798,"A ":2604,"Da":2655,"Co":4892,"Ch":3205,"Do":1694,"De":2688,"Di":2497,"Fa":1585,"Eu":3024,"Et":2349,"Es":2046,"En":2950,"El":2967,"Ge":2029,"Ga":2319,"I ":2891,"Fr":2159,"Fo":2415,"Fi":2245,"C ":2932,"Au":2202,"Ar":3564,"As":1696,"D ":1819,"Ba":4270,"Am":2679,"An":4463,"Al":7577,"Bu":1605,"Br":4547,"Ca":3954,"Bi":1625,"Be":3488,"Bo":2874,"Ku":5528,"Ky":1627,"Kr":2024,"Ko":6559,"Le":5504,"Hä":4384,"Li":5491,"La":6972,"Lu":2444,"Lo":3355,"Me":4707,"Mi":5018,"Ma":12276,"Mu":3221,"Mo":4384,"Ni":3815,"Ne":4302,"Na":2575,"P ":1899,"No":3154,"Gr":2750,"Ha":6094,"He":6887,"II":1871,"Hi":2221,"Ho":2925,"In":4312,"Is":2133,"It":2586,"Ja":4546,"Jo":4525,"Ju":3085,"Ka":13363,"M ":1871,"Ki":5554,"Ke":4172,"Un":1530,"Tu":4391,"Tr":2204,"To":3873,"Th":5318,"Ti":2566,"Te":3907,"Ta":5984,"St":4721,"Su":13153,"Wi":2315,"Wa":2313,"Vu":2187,"Vi":4932,"Va":6002,"Ve":5654,"Pu":2910,"Pr":3317,"S ":3929,"Pe":5861,"Pa":6975,"Po":6867,"Pi":3820,"Se":20320,"Si":6786,"Sh":1709,"So":3993,"Ru":2993,"Sa":10443,"Re":3883,"Ri":2393,"Ro":4731,"Ra":6455,"b ":4269,"a ":521046,"Yh":4604,"Tä":2310,"i ":187878,"ge":9008,"ga":10572,"fi":6772,"fr":2666,"fo":3746,"hd":23028,"he":38383,"ha":33064,"gn":1824,"gl":5520,"gi":17723,"gh":2779,"gu":2320,"gr":3324,"go":5014,"du":5956,"dy":8630,"g ":5897,"ea":17917,"eb":2211,"ec":3419,"ed":14005,"de":60149,"di":25899,"do":15043,"ds":2160,"dr":3184,"ew":1828,"eu":16676,"ev":21342,"ey":5808,"fa":2058,"h ":5313,"fe":3904,"eh":18148,"eg":3976,"ef":1593,"ee":73047,"el":141223,"ek":30007,"ej":4585,"ei":64027,"ep":7092,"eo":9816,"en":311708,"em":29073,"et":103119,"es":102393,"er":103228,"ca":4693,"e ":102313,"br":3098,"bu":11145,"bo":3540,"bl":1718,"bi":4927,"be":5208,"da":16197,"f ":4183,"cu":1883,"ct":2366,"co":5044,"ck":6770,"ci":3249,"ch":7835,"ce":5416,"c ":2562,"az":1738,"ay":3699,"ba":5388,"d ":12256,"at":82974,"as":103098,"ar":90264,"av":32524,"au":43994,"ak":51088,"al":188447,"ai":159817,"aj":32823,"ao":2348,"ap":28311,"am":44148,"an":226249,"ac":6773,"ad":13018,"aa":139980,"ab":3846,"ag":5956,"ah":19189,"ae":6848,"af":2453,"nu":19006,"nt":79878,"ns":48167,"nr":2630,"np":4065,"no":34422,"nn":76172,"ny":10224,"nv":4829,"oe":6505,"of":6585,"oc":5124,"od":21394,"oa":12733,"ob":3163,"om":49202,"on":220535,"ok":63100,"ol":88347,"oi":101083,"oj":13195,"og":7134,"oh":26450,"kä":31248,"ot":48882,"os":73162,"ov":18707,"ou":20507,"op":25063,"oo":22586,"or":48012,"r ":15286,"ow":2927,"kö":4127,"oy":1883,"pe":39610,"pa":50151,"pl":2749,"po":19855,"lä":49440,"ph":2309,"pi":40558,"lo":48127,"lm":34905,"ll":141072,"ls":4754,"lp":5969,"lv":6764,"lu":74348,"lt":40738,"ly":10157,"o ":46103,"ma":107323,"mb":5971,"iä":11662,"me":68275,"mi":93628,"mm":26269,"mp":15152,"mo":20189,"mu":39868,"iö":6377,"my":13711,"p ":3699,"na":88104,"nc":4077,"nd":15417,"ne":115628,"nf":1558,"ng":26517,"nh":5001,"jä":31523,"ni":80364,"nj":4418,"nk":37841,"nl":4563,"nm":4904,"ju":18610,"jo":83376,"ki":95289,"ke":65747,"ka":163517,"m ":8821,"ky":14212,"ks":65607,"kt":5970,"ku":90075,"ko":85957,"kr":5885,"kk":48142,"kl":2002,"kn":1816,"li":159675,"hä":6582,"lk":30410,"lj":7180,"le":81668,"ld":2991,"la":154382,"lb":9921,"n ":656910,"hr":2420,"ht":42173,"hu":10726,"hj":14036,"hk":3966,"dä":2655,"hi":25267,"hn":1754,"ho":13688,"hl":1555,"hm":8011,"id":26485,"ic":10236,"ib":2776,"ia":70345,"eä":2738,"ih":17748,"ig":6859,"if":2884,"ie":58968,"hy":6074,"k ":7419,"ir":40208,"is":262822,"it":142893,"iu":5141,"iv":30487,"ii":85125,"ij":23513,"ik":78594,"il":97254,"im":65870,"in":288307,"io":39520,"ip":11864,"je":25016,"ji":8870,"l ":13054,"ja":144917,"tä":77533,"sä":41776,"vy":5926,"y ":23865,"wa":2667,"rä":7609,"vi":49369,"vu":43029,"vo":19782,"uv":30695,"uu":71208,"ve":29647,"va":118391,"x ":5327,"ui":22148,"uj":5727,"uk":45990,"ul":59174,"ue":19283,"ug":2138,"uh":10868,"ur":38607,"us":87239,"ut":63413,"um":30283,"un":78305,"uo":96118,"up":18134,"ty":54901,"tu":95699,"tt":139450,"ub":2312,"ua":10701,"ud":16569,"uc":1658,"w ":2157,"to":89637,"tl":2100,"ts":24815,"tr":16390,"te":142275,"tk":13593,"ti":135644,"th":11168,"pä":16825,"ta":258052,"su":49746,"sv":13201,"ss":117492,"st":215124,"sy":12228,"sl":7481,"sk":39138,"sn":1670,"sm":5723,"sp":7200,"so":30621,"sr":2112,"sc":2341,"se":139811,"sh":5275,"sj":2476,"si":171906,"nö":1934,"u ":38729,"sa":155918,"rr":9979,"rs":10921,"rt":23316,"ru":25282,"rv":12255,"ry":7951,"rp":3591,"ro":37223,"rn":7182,"rm":9935,"rl":3938,"rk":30938,"rj":32055,"ri":107434,"rh":7195,"nä":20276,"rg":6568,"re":35490,"rd":7095,"rc":2086,"rb":1938,"ra":66068,"t ":86930,"mä":30120,"s ":74921,"px":2388,"py":4209,"lö":4854,"pt":2602,"pu":33575,"pp":18507,"pr":6312,"ps":3220,"yä":2288,"yö":17077,"vä":26642,"yy":14572,"yh":29899,"ye":9120,"yd":4538,"ya":1921,"tö":12963,"yv":6669,"yt":27603,"ys":23779,"yr":6697,"yp":4560,"yn":15218,"ym":10818,"yl":20580,"yk":18251,"yi":8456,"ä ":141477,"äa":1925,"ö ":8048,"ää":49135,"ät":13038,"äv":12353,"äy":16618,"äm":13914,"äl":20535,"äo":2938,"än":46392,"äp":3824,"äs":18046,"är":23375,"äe":2662,"äi":25110,"äh":12610,"äk":13701,"äj":7869,"öö":1835,"öy":2134,"öt":2960,"ör":2049,"ös":12022,"ön":8537,"öl":2694,"öm":1579,"ök":3180,"öh":1825,"öi":4386," Ga":2281," Ge":2007," Fo":2388," Fr":2153," Fi":2200," Ha":6075," He":6845," Gr":2717," Ho":2907," Hi":2219," Ja":4528," Is":2125," It":2580," In":4291," Ka":13310," Ke":4144," Ki":5524," Jo":4519," Ju":3077," La":6921," Le":5486," Hä":4383," Li":5091," Ko":6537," Kr":2018," Ku":5494," Ky":1626," Ma":12197," Mi":4965," Me":4668," Lo":3341," Lu":2432," Ne":4266," Na":2563," Ni":3790," Mo":4358," Mu":3189," Am":2670," An":4363," Al":7546," Ba":4234," Au":2195," As":1683," Ar":3519," Be":3463," Bi":1601," Bo":2840," Br":4522," Bu":1591," Ca":3824," Ch":3183," Co":4800," Da":2631," Di":2463," De":2676," Do":1656," El":2955," Et":2345," Es":2034," En":2918," Eu":3021," Fa":1558," Tä":2303," Wi":2296," Wa":2289," Yh":4598," Po":6838," Pi":3788," Pe":5834," Pa":6894," No":3136," Ra":6410," Ro":4711," Re":3856," Ri":2382," Pr":3297," Pu":2902," Su":13126," St":4422," Ta":5960," Th":5301," Ti":2528," Te":3868," Tr":2184," To":3832," Ru":2990," Sa":10406," Sh":1689," Si":6699," Se":20290," So":3959," Va":5984," Ve":5638," Vi":4890," Vu":2183," Tu":4371," ja":74622," in":3365," il":6233," is":1907," it":4582," ka":41758," ki":21829," ke":20426," jo":60523," ju":13868," ha":11651," he":10326," hy":3213," ih":2463," hi":4639," ho":2199," hu":2999," jä":12144," ni":12782," ne":5502," na":3130," my":11678," mu":27269," mo":4701," ol":30354," om":3788," on":110013," kä":15552," oh":5375," oi":2526," of":2700," ny":2745," nu":2351," no":7612," le":7943," hä":3832," li":11414," n ":4861," la":17994," ku":35062," ky":5412," kr":3070," ko":26681," me":12340," mi":8054," ma":22033," lu":14733," ly":2009," lo":4509," am":2608," an":4850," ai":10388," aj":3176," al":26047," av":3055," au":2714," ar":5107," as":8761," er":7980," et":8124," es":9200," en":13349," ei":4106," el":17426," fi":2104," de":3555," di":1711," ed":3633," vä":7263," ym":1774," yl":9242," yk":6097," yh":19431," tä":4808," ru":4569," ry":2522," sa":23518," se":22713," si":27049," so":8389," ra":14800," re":3082," ri":4303," nä":4180," ro":4580," pu":9566," pr":4112," s ":2297," px":2387," py":2739," mä":2284," os":9956," ot":1666," ov":7670," op":3466," or":1769," pe":20224," pa":14359," po":10381," pi":14437," lä":10166," sä":4588," va":34158," ve":7508," uu":2456," vo":6724," vu":30542," vi":12396," ty":5276," tu":19858," us":4512," ur":1664," ul":2520," ta":38855," sy":5905," st":3683," su":25086," to":18439," th":5224," pä":9746," ti":8862," te":13642," ää":2036,"Ete":1912,"Eur":2720,"For":1614,"Hel":2818,"Int":2022,"Alb":1684,"Bri":2185,"Nor":1531,"Per":1731,"Par":1783,"Poh":2941,"Ran":1941,"Kal":1586,"Kan":2320,"Kau":1621,"Kar":2566,"Kir":1567,"Kun":1618,"Hän":3486,"Mar":3439,"ään":18825,"ääk":4426,"äät":2724,"äär":4190,"ääs":2129,"ää ":10320,"Vuo":2097,"Yhd":3250,"Suo":9477,"Sta":1658,"Sen":4830,"äht":3405,"ähd":1555,"ähe":3536,"ähi":1621,"Ruo":1578,"änn":2773,"äns":2545,"änt":2999,"Sak":1930,"ämi":2674,"äni":2131,"äjä":6711,"äne":3700,"Se ":11672,"äs ":1575,"ämä":7782,"äos":2009,"äka":1964,"äis":10749,"äin":8551,"äiv":1941,"ält":2814,"äli":6173,"älk":2158,"äll":6310,"äks":2395,"äki":2767,"än ":27597,"äve":2052,"ävi":2773,"ärä":1957,"äyt":13595,"äri":6126,"ärj":7258,"ät ":5667,"äsi":4968,"äse":2119,"ärv":3216,"äst":3327,"äss":3767,"ätt":2038,"ävä":6336,"Ven":3558,"Val":1869,"The":3947,"Tur":1761,"ber":2683,"ce ":2305,"bri":1634,"bum":8758,"aka":6269,"ake":7201,"aki":5604,"aji":6912,"ajo":3036,"al ":4672,"aja":18366,"aje":2029,"aih":3713,"aik":14735,"ail":13159,"aim":1862,"ain":38398,"ais":49469,"ait":14001,"aiv":4010,"aid":2730,"ahm":1834,"aht":3966,"ahd":4670,"aha":3293,"anu":4220,"ano":5725,"ann":13433,"anm":2516,"ant":14494,"ans":18785,"ane":3551,"ang":3334,"anh":3197,"ani":13808,"anj":1817,"ank":6571,"ana":14302,"anc":1814,"and":6936,"amm":4533,"amo":2056,"amp":3132,"ami":8254,"ame":4479,"ama":17130,"alv":4153,"alu":15472,"alt":16363,"alo":7802,"alm":6123,"all":43431,"alk":10647,"ali":18156,"ale":9704,"ala":40225,"alb":7609,"an ":106763,"aks":12179,"aku":8003,"akt":1737,"ako":3232,"akk":5053,"ae ":1542,"aaj":5209,"aak":5208,"aai":4807,"aan":46070,"aal":13155,"aam":4542,"aas":4039,"aar":7457,"aav":4444,"aat":11110,"aa ":30258,"ai ":13611,"ael":1895,"adi":4196,"ack":1788,"ada":3189,"at ":26536,"arh":2305,"are":4235,"ard":3175,"ara":7612,"arp":1607,"aro":2059,"arm":2145,"arl":1651,"ark":11086,"arj":11397,"ari":15453,"aru":3626,"arv":4834,"arr":3315,"ars":2606,"art":6849,"asa":4886,"asi":8287,"ase":5553,"aso":1854,"ask":4888,"ar ":2391,"apa":10451,"ape":2308,"api":2097,"app":6574,"apu":2355,"as ":6418,"ava":19468,"aut":8391,"avo":2634,"avi":5086,"avu":2827,"ata":7640,"asu":5324,"ast":26494,"ass":30398,"asv":4804,"atk":3390,"ato":4227,"ate":6056,"ati":12192,"att":13242,"ats":1884,"atu":3704,"aul":4866,"aup":10709,"aur":1674,"aus":6854,"aud":2731,"auh":2050,"auk":3794,"jel":4529,"jen":8431,"jes":7295,"jet":1556,"ji ":2297,"jaa":5054,"jat":4403,"jas":4641,"jal":7569,"jak":4713,"jan":11104,"jai":10124,"jou":4303,"joh":4795,"joe":2427,"jol":3356,"jok":24849,"joi":16729,"jon":9520,"jot":6559,"jos":6649,"jia":1786,"itk":2090,"ito":6246,"itu":9149,"itt":43813,"its":13426,"ity":9715,"isk":6128,"ism":2973,"isl":1855,"iso":4721,"isp":1964,"iss":30541,"isu":9410,"ist":90841,"isy":1690,"ita":17408,"ite":16460,"iti":5058,"ivo":1659,"ivu":2547,"isä":6664,"iva":11789,"ivi":6748,"ive":3576,"ipp":2283,"ipu":1889,"ilä":2764,"ilö":2966,"is ":8294,"ion":10934,"iop":2837,"ios":4401,"iot":1986,"ikä":1979,"ioi":3973,"iol":2452,"ipa":1808,"ipe":1630,"irt":2993,"iro":1572,"irk":4448,"iri":6484,"irj":10801,"isi":26753,"ise":59209,"isa":5742,"ire":2058,"inä":3666,"ira":5520,"it ":5916,"ja ":90836,"itä":10709,"ivä":3912,"kii":3407,"kik":3240,"kij":2090,"kim":2775,"kil":12288,"kia":4204,"kie":9439,"kiv":2180,"kin":14280,"kio":1639,"kir":13427,"kis":5163,"kit":6283,"ki ":12342,"kea":4096,"kee":4463,"keh":4191,"kei":5212,"kem":2757,"kel":5236,"ken":9401,"kes":10316,"ker":7027,"keu":3655,"ket":2834,"kev":1653,"ke ":2125,"kre":1700,"ksa":4741,"kse":20115,"ku ":3951,"kot":3952,"kou":3762,"kos":5488,"kor":5268,"koo":4082,"kon":11074,"kom":3392,"kol":7289,"kok":7259,"koj":1821,"koi":14746,"koh":3422,"koe":1980,"kku":3944,"kke":3841,"kka":15414,"kko":10268,"kki":11721,"ko ":7803,"jul":11521,"kat":4283,"kau":16540,"kar":4289,"kas":12031,"kap":6608,"kan":20757,"kal":9712,"kam":1710,"kak":4624,"kah":2946,"kai":21386,"kaa":14665,"ka ":40830,"ha ":1581,"han":4990,"hai":2275,"hal":6945,"har":4587,"hah":1631,"haa":1766,"he ":6389,"hdo":2167,"hdy":6548,"hde":8081,"hdi":4168,"hel":4032,"hei":7575,"hee":1552,"het":1965,"her":2750,"hen":6953,"hem":2103,"hin":8148,"his":2803,"hit":5065,"hja":5248,"hje":3366,"hjo":5005,"gle":2192,"gla":2321,"gra":1797,"ial":8370,"ian":13283,"ias":7499,"ic ":1670,"iaa":4991,"ia ":28344,"iet":9042,"iel":9758,"iem":2939,"ien":20960,"ier":2897,"ies":3794,"ied":2257,"ieh":1857,"iek":1604,"eä ":1619,"ich":1751,"ie ":2310,"ica":1971,"idi":2094,"ide":16311,"ida":3705,"iid":1915,"iik":12079,"iih":2036,"iin":34905,"iil":1871,"iim":1601,"iis":3357,"iir":7370,"iip":1650,"iiv":3159,"iit":11463,"ija":15115,"ijo":3212,"ika":17282,"ii ":2634,"igh":2049,"ihe":3769,"iha":1816,"ihm":2273,"ihi":4976,"iht":1902,"imo":4420,"imm":10403,"imp":1982,"ime":14110,"imi":21669,"ind":1904,"ina":17571,"imu":3688,"inn":9244,"ino":7309,"int":21117,"ins":3814,"ine":62207,"ijä":3525,"ing":10847,"ini":7961,"ink":6694,"ioa":2138,"inu":2993,"inv":2235,"iko":8699,"ikk":20680,"iki":6034,"ike":7607,"ila":13660,"in ":123439,"iku":7225,"iks":5910,"ilp":4235,"ilo":6040,"ill":27290,"ilm":11405,"ilj":1888,"ili":8485,"ile":2304,"ima":7267,"io ":5938,"ilt":4359,"ilu":7728,"hmä":2450,"hol":1937,"hon":2485,"hoi":2109,"hmi":2965,"hmo":1850,"hty":9392,"htu":3085,"hto":2163,"hti":9794,"hte":8765,"hta":4518,"htä":1902,"huo":1796,"hum":3543,"hyv":1693,"etä":4989,"evä":1690,"eta":9667,"ete":9398,"eti":3164,"est":38413,"ess":23541,"eud":1841,"eto":4814,"etr":5809,"ets":2385,"ett":36313,"etu":5501,"ety":2077,"erä":4263,"eve":1697,"eva":7898,"evi":4106,"euv":2196,"eut":3244,"eur":5135,"eus":2033,"esä":2111,"ey ":1541,"evy":4434,"elä":9755,"er ":8358,"eor":2097,"eol":1627,"emä":1894,"es ":8151,"erk":7307,"eri":25262,"erg":2239,"erh":1625,"enä":7730,"ere":5494,"era":4710,"et ":16346,"esk":9110,"esi":14483,"ese":2664,"erv":1561,"eru":10609,"err":4242,"ert":7186,"ers":4983,"ern":2916,"erm":2674,"ero":5460,"eki":2975,"ekk":1967,"ekn":1637,"eko":1843,"eks":9087,"ekt":2330,"en ":216998,"ela":5301,"ele":9915,"eli":28185,"elj":2641,"elm":10985,"ell":36182,"elo":8595,"elu":6803,"els":2931,"elt":10887,"ely":1692,"ema":7235,"eme":2562,"emm":5417,"emo":1546,"emi":5347,"emp":2813,"ene":8165,"eng":2981,"ena":5955,"end":1729,"enn":9973,"enk":6752,"eni":7989,"ens":15471,"ent":18077,"ekä":5887,"ehd":2924,"ehi":4544,"eht":6057,"eis":18168,"eim":3841,"eil":7078,"ein":8482,"eik":4720,"eid":4335,"eja":2716,"el ":2656,"eit":8348,"öss":1852,"gis":3146,"gin":6890,"gia":4299,"ght":1865,"ös ":7736,"gen":2252,"ger":1575,"ön ":5342,"gas":2278,"gan":2134,"fri":1576,"for":2137,"fil":1743,"da ":1964,"de ":3246,"daa":2346,"dal":1725,"das":2350,"dan":3795,"ck ":3175,"ed ":1799,"ean":1991,"eal":1781,"eat":2428,"ea ":2811,"ei ":3830,"een":37770,"eel":9070,"ees":6176,"eet":6503,"edi":2493,"ede":4270,"edu":2200,"edo":1603,"ee ":9590,"dys":6375,"dus":2698,"don":2736,"dol":2034,"dos":3976,"dia":2516,"der":2863,"des":8657,"det":3714,"del":8280,"dek":2312,"den":24947,"dem":1603,"di ":1662,"din":5138,"dio":5021,"dis":6234,"rhe":3181,"rha":2376,"näj":3467,"näk":2091,"näi":1855,"ri ":14183,"rgi":2577,"ret":2238,"res":3703,"nä ":7166,"rea":2509,"ree":2885,"rei":4803,"ren":6049,"rel":2990,"rdi":1756,"re ":2848,"rd ":1882,"ras":4899,"rat":4668,"rau":3008,"raj":2533,"rai":2122,"ran":13051,"ral":4384,"rak":6377,"raa":6078,"rad":3325,"rs ":1794,"ros":3755,"rot":3070,"rom":2805,"ron":4039,"roo":4243,"rov":1686,"roc":2312,"roi":1996,"rna":1810,"rne":1656,"rni":2033,"ro ":2723,"rma":3198,"rme":1785,"riä":3343,"rmi":2119,"rko":8356,"rki":6693,"rkk":7003,"rke":4145,"rka":1913,"rjo":6361,"rja":16764,"rje":7814,"rio":2073,"rit":12816,"ris":12311,"rii":3347,"ril":5111,"rik":9617,"rin":19152,"rim":2324,"ria":8783,"ric":1741,"rie":2399,"näy":2474,"ruo":2205,"run":2801,"ruu":3504,"rus":12209,"rva":2385,"rvi":3707,"rve":3560,"rvo":2466,"ry ":2410,"rsi":3925,"rta":4520,"rto":3696,"rte":2990,"rti":3436,"rtt":1850,"rt ":1812,"rro":1797,"rre":2394,"rra":3831,"saa":11689,"sai":4605,"sak":3935,"sal":8568,"sam":4120,"san":10417,"sat":2386,"sas":5453,"sar":9940,"sav":3217,"sa ":88364,"ryh":2885,"si ":33316,"siv":2877,"sie":3768,"sia":14727,"sit":13812,"sis":15595,"sin":22898,"sio":5975,"sil":7537,"sim":10671,"sij":10631,"sik":6439,"sii":15427,"se ":7339,"sev":3309,"ser":1954,"ses":19353,"set":8009,"seu":4708,"sei":5713,"see":16044,"sen":44413,"sem":6469,"sel":8992,"sek":9571,"spa":2113,"sot":3315,"sol":2285,"son":3580,"sop":1591,"sos":3124,"sod":2739,"sof":1780,"soi":4463,"st ":1959,"sli":1829,"sla":3001,"ski":7594,"sko":4863,"sku":10246,"ska":6718,"ske":7078,"sma":1538,"siä":2875,"smi":2339,"ssä":20775,"stä":20839,"stö":6003,"syn":3354,"syy":2751,"sse":2037,"ssa":81782,"sso":2391,"ssi":7227,"ste":34759,"sta":74007,"sto":18448,"sti":27582,"stu":19044,"str":3547,"sty":7941,"suk":6559,"suo":12490,"suu":15921,"sut":2418,"sva":8410,"svi":2330,"tai":23079,"taj":9294,"tak":4197,"tal":15729,"taa":33105,"tav":11283,"tau":2602,"tat":3910,"tas":5911,"tar":15686,"tap":4732,"tan":13725,"tam":13647,"te ":4200,"ta ":96137,"pa ":1685,"par":4524,"paa":2641,"pah":2670,"pak":1785,"pal":14678,"pai":9231,"pan":6844,"läp":1604,"län":3940,"läm":2523,"läi":5398,"läh":6121,"pi ":2782,"lä ":21592,"per":21853,"pet":1794,"pel":8616,"lää":3392,"pia":3590,"pid":1828,"pie":3232,"pii":5873,"pil":2184,"pin":5017,"pis":5110,"pit":5090,"por":1632,"poi":2560,"poh":4180,"pol":4180,"ppu":2417,"ppi":4768,"ppa":7267,"ppe":1921,"pro":3960,"pur":1783,"pus":1680,"pun":10901,"puo":7107,"pul":1657,"puh":2589,"px ":2372,"puu":2017,"mä ":7257,"mäi":6703,"män":3876,"mäs":1650,"mää":5076,"ra ":5712,"ngi":7939,"ngl":4558,"ni ":6341,"nge":2486,"nga":3203,"jän":3177,"jäl":4462,"jäs":2729,"jär":9906,"nha":2067,"nei":4215,"nel":5383,"nen":63104,"ner":2732,"net":11819,"nes":5079,"neu":1692,"ng ":3916,"nee":7398,"jä ":5912,"nce":1684,"ne ":6951,"ndo":1777,"ndi":2475,"nde":2171,"nda":1665,"nak":1691,"nal":6875,"nan":9972,"nai":6761,"naa":4824,"nd ":4059,"nat":3023,"nas":6250,"na ":39872,"myö":9305,"iö ":2528,"ntä":3484,"nsä":4958,"nva":1615,"num":1758,"nus":3629,"nut":8387,"nty":5978,"nto":10581,"ntu":4377,"ntt":4440,"nti":13349,"nta":21023,"nte":11336,"nso":2855,"nss":4946,"nse":1616,"nsi":11891,"nsk":3706,"nsa":12409,"nnä":1593,"nt ":1924,"ns ":1897,"nol":2730,"noi":8508,"nom":2361,"non":3636,"not":1919,"nos":4123,"nne":16039,"nna":31714,"nno":6361,"nni":12231,"nnu":4777,"nme":1667,"nma":1873,"jää":2989,"nla":2791,"no ":2124,"nke":2631,"nki":13461,"nka":13056,"nko":2285,"nja":2100,"nii":4930,"nie":2772,"nia":6492,"nis":11071,"nit":5273,"nim":13793,"nin":16562,"nik":3354,"nil":2835,"ogi":4118,"oi ":3679,"oht":5278,"käs":5110,"kär":1779,"ohj":13237,"oho":2246,"oiv":1619,"ois":23808,"oir":1655,"oit":25419,"oin":13137,"oik":5181,"oim":14211,"oil":4642,"oih":1702,"oid":5498,"käy":11169,"oje":6434,"oja":5307,"ock":3160,"ode":9371,"odi":1977,"odo":4669,"of ":2718,"oda":3354,"oel":2434,"oen":2647,"ofi":1682,"kä ":7989,"oa ":4765,"oal":3299,"nyk":2547,"nyt":4401,"nvä":1760,"otu":2486,"otk":3192,"oti":5562,"ote":5798,"ott":10457,"ots":2991,"oto":4571,"ost":14630,"ota":8496,"osi":11836,"osk":3907,"ose":3031,"oss":12304,"oso":3047,"ovi":3135,"ova":10956,"ove":2498,"ouk":4542,"oul":4682,"oun":1887,"ous":2951,"out":1762,"opp":4159,"opi":5727,"ope":4607,"opa":3651,"os ":5287,"opu":2089,"oon":7289,"ool":1601,"oom":1672,"or ":1961,"oot":2091,"oos":2166,"oop":3454,"ork":3647,"orm":3104,"orn":1628,"oro":2190,"ord":2513,"ore":3159,"org":2171,"ori":12794,"osa":15087,"ort":3782,"ot ":3256,"ora":3303,"ola":6203,"on ":146201,"oli":32153,"oll":16426,"ole":10977,"kää":1980,"olt":1744,"olm":4749,"olo":5842,"oly":2026,"olu":4176,"oka":27584,"okk":2554,"oki":3937,"oke":2183,"oks":6588,"oko":8846,"oku":9166,"ona":6281,"one":7607,"ong":1915,"oni":10795,"onk":9689,"onn":20892,"ono":2022,"ons":4191,"ont":4754,"oma":18393,"ome":15724,"omi":7452,"omu":1601,"la ":45674,"le ":23672,"laa":8510,"lah":2594,"laj":7083,"lai":41589,"lal":1802,"lak":2624,"lan":15670,"lam":1915,"lat":4073,"las":9219,"lau":6037,"lbu":8824,"kuv":12915,"kuu":20315,"kut":5564,"kus":5968,"kup":2448,"kuo":2056,"kun":18716,"kul":5464,"kuk":1991,"ksi":34515,"kso":2000,"kue":1696,"kui":3945,"kti":3367,"kyi":1827,"kyl":5052,"llä":17871,"lok":9030,"lon":3666,"lom":4409,"lop":2359,"log":4285,"loi":6423,"lpa":4160,"los":4127,"lot":1540,"lou":2403,"ljä":2124,"lmi":8971,"lme":5378,"lma":14737,"lti":5458,"ltt":2378,"lue":13574,"lsi":2865,"lta":20774,"lu ":5034,"lmä":3813,"hän":3427,"li ":38792,"lev":10595,"les":3513,"let":2489,"ler":1630,"lem":3429,"len":7655,"lek":2031,"lel":2977,"lei":7201,"leh":3152,"lee":7823,"llu":3725,"lo ":3012,"lla":45545,"lle":23098,"lli":38165,"llo":8003,"lko":3906,"lku":3582,"lka":14232,"lke":3142,"lki":3385,"ljo":1577,"lje":1833,"ll ":2046,"lit":6604,"lis":35039,"lip":2086,"lio":3340,"lin":25643,"lim":2511,"liv":3460,"lia":6855,"lik":3266,"lil":3081,"lii":12989,"lij":5839,"lie":2208,"ma ":15016,"mb ":3008,"maa":25280,"mah":1595,"mai":6076,"mak":3049,"mar":3413,"mas":6492,"mal":15003,"man":17338,"mat":9985,"me ":2801,"mee":1671,"met":10390,"mes":9381,"mer":13701,"mel":4799,"men":18334,"mei":2490,"iä ":10723,"lve":3316,"lvi":1540,"luk":6577,"lui":2536,"luo":7753,"lun":5262,"lut":6301,"lus":6284,"luv":9141,"luu":4990,"ltä":6950,"lyh":1732,"lym":1752,"mpi":5319,"moo":2276,"mon":6257,"mpa":1715,"mmä":7757,"mpä":2564,"mua":1679,"mus":7282,"mut":3853,"muu":6734,"mui":3639,"muk":6397,"muo":5752,"mi ":13252,"min":18710,"mil":5082,"mis":25017,"mit":6945,"mia":4076,"mie":6732,"mik":3016,"mii":3025,"mo ":3241,"mmi":7415,"mma":5671,"mme":2557,"väl":6253,"vä ":5493,"vää":1811,"vän":2110,"vät":4019,"yvi":1744,"yty":2014,"ytt":9230,"yte":5446,"ysv":6099,"yst":4193,"ysi":1754,"yri":2968,"yt ":5088,"ymä":1567,"ys ":5219,"ylä":5863,"yny":2515,"yvä":4334,"yyl":1824,"yys":2076,"yyp":2141,"yyt":1690,"yy ":2993,"ye ":3730,"yde":2513,"yee":3728,"yks":11428,"yky":3192,"yn ":4172,"yle":4657,"yli":5834,"yll":2070,"ymi":2202,"ymp":3885,"yne":1918,"ynt":3210,"yi ":2699,"yhm":2893,"yhd":7827,"yht":15566,"yis":3850,"tön":2023,"tös":1710,"tö ":2585,"täm":6823,"tän":1885,"täh":1876,"täj":3398,"tä ":34022,"tää":13926,"täy":1596,"tär":1635,"täv":7438,"sää":2844,"sä ":25127,"säl":3359,"säk":2119,"säv":1927,"vuo":32425,"vun":2514,"vul":3123,"vy ":2051,"via":2642,"vio":1556,"vir":4221,"vil":2731,"vin":6040,"vii":6538,"vie":4226,"vit":3912,"vis":7340,"voi":10127,"von":1808,"vos":2863,"räi":2223,"vi ":2985,"ver":6645,"ves":2270,"ven":5385,"vel":7217,"ve ":1928,"val":26751,"vak":1862,"van":11048,"vap":1548,"var":8953,"vat":20575,"vas":8886,"vaa":7863,"vai":7839,"va ":18614,"uuk":1525,"uun":9740,"uul":10457,"uud":5759,"uus":11398,"uur":9061,"uut":11944,"uvi":3760,"uvo":2424,"uva":18498,"uvu":5385,"usl":2474,"usk":5912,"usi":8604,"use":5829,"usa":2048,"uu ":9094,"ust":26922,"uss":6964,"utk":3663,"uti":2698,"ute":5204,"uta":9377,"utt":14676,"uts":1830,"utu":7065,"uto":3590,"us ":19751,"ut ":13926,"ura":6710,"ure":2812,"urh":2266,"uri":10072,"uro":4160,"uru":1961,"uod":11474,"uon":20919,"uol":10441,"uom":19569,"uok":4826,"uot":13021,"uor":5597,"uos":6389,"upe":2549,"upu":10042,"umi":12508,"uma":5258,"umb":3567,"ume":3010,"unt":10189,"unu":3435,"unk":6506,"uni":5890,"unn":17239,"und":1617,"una":4707,"ung":5391,"une":2125,"uks":15716,"uku":7377,"uko":2518,"ukk":6046,"uki":2552,"uke":1610,"um ":2055,"uka":8994,"ulu":17346,"ult":3742,"ulo":2329,"ull":6128,"ulk":14188,"uli":3634,"ule":2174,"ula":6501,"un ":17537,"uin":4870,"uis":6404,"uht":1985,"uhu":1855,"uje":1776,"uit":1991,"uja":2763,"ui ":2881,"uha":2389,"ude":10287,"udi":3267,"ue ":4574,"uet":2172,"uee":7376,"työ":3659,"ua ":4041,"uas":1855,"tyv":3209,"tyy":6164,"tye":7861,"tyi":5279,"tyk":3846,"tym":2979,"tyn":4512,"tys":6857,"ty ":6437,"tur":3509,"tus":10527,"tut":5144,"tuu":8595,"tuv":3012,"tuj":1880,"tui":4969,"tul":5059,"tuk":7469,"tun":13048,"tum":4274,"tuo":5296,"tua":2491,"tud":2510,"ttö":2890,"ttä":18426,"tra":3772,"tri":7061,"tro":3179,"tu ":15401,"tsa":2112,"tse":11636,"tsi":5021,"tsu":1775,"tta":37250,"tte":15326,"tti":29985,"tto":7420,"ttu":17456,"tty":9077,"to ":10851,"tiö":3363,"toj":3671,"toi":21414,"tkä":1786,"toa":3897,"tos":4171,"tot":2445,"tom":3430,"ton":10160,"tok":5162,"tol":5536,"tor":8049,"too":2936,"top":1727,"tii":20914,"til":10225,"tik":4482,"tie":14185,"tit":2814,"tis":9101,"tin":13829,"tim":2695,"tio":13178,"thu":3317,"tia":5511,"tiv":2672,"tki":4098,"tka":4817,"pää":9932,"tem":2906,"ten":19694,"teo":3855,"tei":13515,"tek":5492,"tel":25974,"tee":16061,"teh":4545,"th ":1573,"tet":23609,"tes":1638,"ter":13778,"ti ":27214,"pär":2295,"the":3039,"päi":2568,"yön":1725,"yös":8569,"yöh":1625,"yä ":1547},"n_words":[15184556,16912812,13033049],"name":"fi","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/fr.json b/contrib/languages-data/fr.json
new file mode 100644
index 0000000..ee27c50
--- /dev/null
+++ b/contrib/languages-data/fr.json
@@ -0,0 +1 @@
+{"freq":{"D":116102,"E":120408,"F":121384,"G":111406,"A":238769,"B":169565,"C":285959,"L":345504,"M":204991,"N":98243,"O":65813,"H":89479,"I":164982,"J":77783,"K":45111,"U":61602,"T":117987,"W":42348,"V":73826,"Q":14589,"P":196230,"S":238878,"R":128546,"Y":18365,"X":15982,"Z":13917,"f":625832,"g":841835,"d":2804209,"e":9206578,"b":583197,"c":1872040,"a":5160230,"n":5070939,"o":3525396,"l":3535844,"m":1717377,"j":148113,"k":150020,"h":692514,"i":4746975,"w":64787,"v":659313,"u":3519294,"t":4403208,"s":4479915,"r":4208721,"q":428703,"p":1557910,"z":87206,"y":309342,"x":204437,"É":44421,"ï":13787,"î":22540,"ê":38749,"é":1751958,"è":218696,"ç":101170,"â":19710,"à":277569,"û":11801,"ù":7682,"ô":30698,"œ":8733," l":1232220," m":320276," n":242807," o":228420," h":94558," i":171950," j":98827," k":16664," d":1997342," e":1227865," f":345437," g":129922," a":696872," b":124278," c":594010," y":7295," u":524523," t":219372," w":8382," v":133967," q":138426," p":728591," s":616330," r":286749," J":75893," K":42648," H":85423," I":133876," N":84973," O":54606," L":335414," M":194304," B":161567," C":266111," A":214898," F":109790," G":105282," D":105238," E":106937," Z":13187," Y":17633," X":12755," S":214333," R":117248," Q":13527," P":183259," W":40332," V":62074," U":57563," T":104796," à":274935," î":9932," é":209998," ê":8160," É":44088,"A ":15041,"Da":22440,"Cl":11868,"Co":65041,"Cr":12941,"Ce":29429,"Ch":51129,"Ci":6652,"Du":7894,"Do":13538,"De":21502,"Di":17162,"Fe":8715,"Fa":10235,"Eu":11691,"Es":11694,"En":18617,"El":32899,"Ge":14692,"Ga":20211,"I ":15646,"Fr":44421,"Fo":14809,"Fl":7090,"Fi":9942,"C ":21532,"Au":19631,"Ar":26711,"At":6889,"As":11086,"D ":7702,"Ba":35723,"Am":14264,"An":31376,"Al":37486,"Bu":10594,"Br":27743,"Ca":55063,"Bi":10547,"Be":29116,"Bo":31176,"Bl":7237,"Le":129356,"Li":24542,"La":91448,"Lu":9746,"Lo":35045,"Me":21247,"Mi":27542,"Ma":80328,"Mu":9199,"Mo":39904,"Ni":11148,"Ne":15700,"Na":15519,"No":31277,"On":6946,"Gi":8289,"Gr":22075,"Go":10803,"Gu":15739,"Ha":29460,"He":19781,"II":12760,"Hi":8034,"Ho":13597,"In":22404,"Il":64459,"Is":6832,"It":7525,"Ja":19690,"L ":40055,"Je":19939,"Jo":19144,"Ju":9216,"Ka":11065,"Un":43594,"Tr":15916,"To":17150,"Th":22226,"Ti":6718,"Te":13283,"Ta":12863,"V ":9181,"St":21014,"Su":20295,"Wi":11355,"Wa":11101,"We":6825,"Vi":21333,"Ré":14676,"Va":15441,"Ve":10793,"Pr":25219,"S ":10997,"Pe":15558,"Pa":62058,"Pl":7693,"Po":24431,"Pi":18433,"Ph":7701,"Ou":6784,"Or":12154,"Se":26414,"Sc":10127,"Si":15134,"Sh":8197,"Sp":6886,"So":23717,"Ru":6655,"Sa":56793,"Re":19635,"Ri":11472,"Ro":36071,"Qu":12001,"Ra":14786,"b ":23149,"a ":715548,"Yo":8948,"i ":246446,"ge":153192,"ga":79217,"bé":13640,"fl":14307,"ff":44067,"fi":98156,"fr":118819,"fu":33433,"fo":104257,"he":135815,"ha":144005,"gn":87721,"cé":45974,"cè":9053,"gl":38751,"gi":139568,"gh":16776,"gu":62416,"gr":93539,"go":42919,"du":266692,"g ":46447,"ea":84524,"eb":15026,"ec":142148,"ed":28114,"de":1288806,"di":214915,"dm":11618,"do":95110,"ds":13995,"dr":52545,"ew":13282,"ex":53989,"eu":311458,"ev":34640,"ey":24158,"ez":12097,"fa":75922,"h ":43257,"fe":44423,"eg":22246,"ef":23663,"ee":14949,"el":261273,"ei":65911,"ep":62933,"eo":13735,"en":1013332,"em":298038,"et":487438,"aï":8404,"es":1468583,"aî":9959,"er":561408,"ca":174843,"e ":4165476,"by":10945,"bs":8710,"br":109877,"bu":41947,"bo":48835,"bl":80107,"bi":56922,"be":65791,"da":250547,"f ":44719,"cy":11950,"cu":58711,"ct":159489,"cs":7871,"cq":7986,"cr":84742,"co":398094,"ck":30588,"cl":49217,"ci":192973,"ch":228563,"ce":285004,"cc":27518,"c ":86602,"az":14616,"ay":48306,"ba":99079,"d ":356687,"at":402587,"as":164469,"ar":580946,"aq":13150,"ax":8225,"av":102326,"au":313806,"ak":16011,"al":437033,"ai":491839,"aj":9782,"ao":15479,"ap":112014,"am":169689,"an":984305,"ac":155600,"ad":94574,"ab":78438,"ag":129146,"ah":12966,"ae":24602,"af":19038,"nu":48492,"nt":781378,"ns":405957,"nr":20379,"nq":10241,"no":177623,"nn":194465,"nz":7383,"ny":16039,"nv":29268,"oe":7051,"of":34594,"oc":112798,"od":60253,"oa":9189,"ob":47107,"om":320717,"on":978231,"ok":8561,"ol":182032,"oi":182944,"og":68867,"oh":10520,"ot":93297,"os":119510,"ov":56208,"ou":495961,"op":94277,"oo":30133,"or":397296,"oq":6742,"r ":618020,"ox":6981,"ow":11940,"oy":26184,"pe":191299,"pa":336824,"lè":14938,"pl":106081,"lé":66665,"po":236682,"ph":72167,"pi":70246,"lo":174519,"hé":40769,"lm":23015,"hè":8383,"ll":348027,"ls":35163,"lp":13659,"lv":7890,"lu":105730,"lt":35681,"ly":20798,"o ":123736,"ma":245390,"mb":101583,"me":421982,"iè":67396,"mi":181182,"mm":149391,"ié":40043,"mp":121199,"mo":139179,"mt":11044,"ms":10324,"mu":103041,"my":7264,"p ":21047,"na":254840,"nc":212957,"nd":247872,"ne":645258,"nf":26140,"ng":123325,"ni":252369,"nk":9186,"ju":35235,"jo":42449,"fé":31064,"ki":18400,"ke":23430,"ka":17710,"m ":101180,"ko":10158,"gé":41926,"li":366432,"le":1033424,"ld":17364,"lg":18291,"lf":6672,"la":653743,"lb":19702,"n ":1301691,"hr":18002,"ht":13842,"hu":32067,"hi":111052,"hn":14386,"ho":82392,"hl":7215,"dé":209539,"id":89273,"ic":228813,"ib":37956,"ia":126735,"ig":111904,"if":68146,"ie":498651,"hy":15296,"k ":41221,"iq":189141,"ir":210609,"is":657338,"it":513778,"iu":11403,"iv":116504,"ix":21397,"ik":10626,"il":316521,"im":87035,"in":595401,"io":414857,"ip":60451,"je":33885,"iz":8646,"l ":543603,"ja":25992,"xi":22025,"tè":15940,"té":190035,"xp":12895,"xt":10929,"z ":20913,"xa":6853,"xe":16992,"wi":9300,"sé":88528,"rô":7192,"y ":99277,"wa":16319,"we":7882,"rè":35139,"ré":242093,"vi":188474,"vu":6788,"vr":43893,"rê":8320,"vo":57579,"ux":99749,"uv":71595,"ve":234012,"va":93360,"x ":120860,"ui":230155,"uj":8246,"ul":117513,"ue":407659,"uf":8137,"ug":28998,"ur":515467,"us":227928,"ut":191898,"um":73719,"un":620155,"up":64955,"ty":19621,"tu":167178,"tt":77237,"ub":50581,"ua":51699,"ud":59261,"uc":57765,"w ":13938,"to":173702,"pé":53976,"tl":8663,"pè":21910,"ts":102393,"tr":321129,"te":602785,"ti":612679,"th":96741,"v ":6801,"où":7384,"tb":13422,"tc":11543,"oû":9564,"ta":328001,"su":146804,"ss":210582,"st":753670,"sy":23798,"sl":12189,"sk":12848,"sn":9845,"sm":19680,"sp":78469,"so":222811,"sq":17522,"sc":60065,"se":403227,"sh":19364,"si":325826,"u ":509520,"sa":131590,"rr":83294,"rs":168244,"rt":289803,"ru":65582,"rv":27573,"ry":20802,"rq":9625,"rp":15297,"ro":341033,"rn":84154,"né":177701,"rm":89580,"rl":33141,"rk":16207,"nç":94772,"ri":487470,"rg":66626,"rf":12647,"re":763207,"rd":96680,"rc":70956,"rb":28741,"ra":505286,"t ":1634972,"qu":422607,"mê":10277,"mé":91911,"mè":10378,"s ":1913426,"pt":48826,"pu":65796,"pp":68088,"pr":230460,"ps":19103,"zi":10741,"ze":10497,"za":12596,"zo":10854,"vé":17663,"ye":16656,"yc":12861,"ya":27740,"yt":7956,"ys":37929,"yr":15599,"yp":14150,"yo":8853,"yn":14467,"ué":92385,"ym":16691,"yl":12792,"Ét":26242,"ât":11174,"à ":276969,"éé":15388,"îl":9937,"êm":12225,"êt":20220,"él":51000,"éo":28234,"ép":86639,"ém":44293,"én":55491,"és":107336,"ét":140932,"éq":14455,"ér":173640,"év":51715,"éb":22743,"éa":37956,"éd":70000,"éc":110985,"éf":19719,"ée":230327,"ég":99948,"èm":27722,"èn":11542,"èr":67257,"ès":34878,"èt":16045,"èv":6684,"èg":9962,"èc":26515,"ço":8440,"ça":89329,"é ":372600,"ût":9917,"ù ":7557,"ôt":12300,"œu":7751,"一":9376," Ga":20103," Ge":14539," Fo":14685," Fr":44293," Fi":9743," Fl":7048," Ha":29378," He":19697," Go":10717," Gr":21904," Gu":15637," Gi":8161," Ho":13528," Hi":7850," Je":19786," L ":35977," Ja":19604," Is":6751," It":7499," In":22151," Il":64299," Ka":10962," Jo":19013," Ju":9181," La":90821," Le":128061," Li":24245," Ma":79831," Mi":27364," Me":21128," Lo":34906," Lu":9698," Ne":15538," Na":15371," Ni":11097," Mo":39732," Mu":9105," C ":11416," Am":14164," An":31164," Al":37253," Ba":35535," Au":19508," At":6802," As":10968," Ar":26331," Be":28995," Bi":10366," Bl":7174," Bo":30964," Br":27614," Bu":10535," Ca":54551," Ce":29219," Ch":50964," Cl":11738," Cr":12852," Co":64524," Da":22266," Di":16958," De":21321," Do":13217," Du":7843," El":32823," Es":11656," En":18307," Eu":11642," Fe":8673," Fa":10157," Wi":11260," We":6753," Wa":11001," Yo":8931," a ":54049," Ou":6671," Or":12055," Po":24205," Pl":7593," Pi":18345," Ph":7574," Pe":15479," Pa":61683," No":30970," On":6799," Ra":14631," Qu":11878," Ro":35894," Re":19473," Ri":11439," Pr":25007," Su":20225," St":20131," Ta":12776," Th":22049," Te":13049," Tr":15810," To":17022," Sa":56588," Sh":8106," Si":14799," Sc":9966," Se":26149," So":23336," Sp":6786," Va":15389," Ve":10579," Vi":21221," Ré":14605," Un":43425," ja":19099," l ":229130," je":16653," im":17160," in":90112," il":33523," is":6946," it":13099," jo":28748," fé":16770," ju":33224," ha":21033," gr":51361," go":9168," gu":9190," hi":17321," dé":147001," ho":26370," hu":11315," ne":11993," na":26573," mu":26866," mo":87695," on":19443," oc":20805," of":17687," ob":10074," no":93418," le":471758," li":50278," n ":10043," la":426807," gé":20899," me":31539," mi":31668," ma":94790," lu":10283," lo":32481," af":11272," ag":11352," ab":10657," ac":41475," ad":17435," am":34528," an":80768," ao":8597," ap":47388," ai":15942," al":42510," av":56190," au":168550," ar":49629," at":13572," as":18267," d ":197071," ba":44107," bi":13538," be":11213," bo":15457," bl":6713," bu":7813," br":18993," ca":62413," et":342046," es":474914," en":328723," em":8216," el":14313," fe":12823," fa":58915," eu":10657," ex":33671," fu":26386," fr":103021," fo":71702," fl":7457," fi":46439," ge":16870," ga":14369," cl":20052," co":290856," cr":32388," ce":54595," ch":72895," ci":26401," da":165596," cu":9192," cy":6653," do":48107," dr":11453," de":1120211," di":79077," du":223778," té":10776," ru":15648," sa":43475," se":95411," sc":20238," si":97963," sp":20322," so":138476," qu":138128," ra":22818," re":79730," ri":12843," né":88113," ro":35414," pu":26022," pr":170981," s ":23984," mê":10232," mé":23004," ou":87553," op":9623," or":44807," pe":62711," pa":204808," lé":7304," pl":62779," po":141674," pi":14858," ph":17219," sé":19742," va":14483," ve":21390," vo":19720," vi":65672," ré":111803," ut":13716," un":501687," ta":13187," où":7340," sy":15329," st":19632," su":114473," tr":64495," pé":8718," to":29268," th":29162," ti":14570," te":41930," Ét":26213," à ":274908," êt":8158," év":15420," éq":11336," ét":84332," ép":12565," él":18021," éd":13385," éc":33500," ég":11829," îl":9913,"Eur":7732,"En ":11566,"Ell":27452,"Fra":34835,"For":6793,"II ":8125,"Her":7505,"Hau":10362,"Gra":11197,"Ind":7362,"Il ":58031,"Bar":7002,"Bas":7999,"All":7697,"Ang":8461,"Cal":11943,"Car":9781,"Can":10563,"Ber":8508,"Bel":7820,"Bou":9492,"Dan":8952,"Chi":7925,"Cen":6692,"Cet":9727,"Cha":27535,"Cor":8959,"Com":17296,"Col":7940,"Con":15750,"Cou":6779,"New":7835,"Nor":17044,"Pie":8398,"Par":29868,"Pro":9444,"Ita":6863,"Jea":13129,"Les":29698,"Le ":85946,"Lan":7053,"La ":64716,"Lou":8888,"Loi":7362,"Man":8487,"Mar":33987,"Mon":17275,"Mic":7917,"Sud":6967,"Sta":7499,"Son":7414,"Sai":21598,"San":8752,"Val":6713,"Uni":28327,"Un ":8531,"The":11621,"bit":13396,"bil":9380,"ble":36471,"bli":30327,"bor":11093,"bou":14756,"be ":11661,"ban":14082,"bal":23571,"bat":8684,"bas":21924,"bar":8519,"bec":7102,"ber":20504,"bel":11445,"bie":9676,"ca ":7169,"car":22613,"cat":26927,"can":25116,"cap":8531,"cad":7391,"cal":21842,"cai":32564,"ce ":152975,"bri":16551,"bra":7577,"bre":74429,"bum":11288,"but":12601,"by ":8395,"am ":12290,"al ":83242,"ail":20993,"ain":126816,"air":72297,"ais":165056,"ait":69408,"aie":8332,"acé":7099,"agi":9624,"agn":32293,"ago":7536,"anv":10025,"ano":12803,"ann":37434,"ant":198270,"ans":198981,"ane":14041,"ang":47054,"ani":45704,"ana":30499,"anc":97402,"and":98722,"amm":17042,"amp":23230,"ami":37168,"ame":15452,"amb":9570,"ama":14537,"alt":7552,"alo":12342,"all":56591,"ali":100273,"ale":107350,"ala":20659,"alb":12381,"an ":76640,"abe":9364,"abi":15595,"abl":19961,"abo":9408,"abr":8483,"ae ":16557,"ac ":8751,"aff":8179,"ai ":16674,"aga":7441,"age":55470,"ado":7415,"adm":9823,"adi":21924,"ade":15733,"aci":10019,"ach":16602,"ace":28163,"acc":10028,"ada":13056,"act":43926,"até":6761,"ays":15243,"aya":8788,"aqu":12595,"at ":36684,"arg":12036,"are":21800,"ard":35646,"arc":26378,"arb":8126,"ara":32792,"aro":14440,"arn":8384,"arm":11551,"arl":15187,"anç":91515,"ari":62241,"aru":7211,"arq":6770,"arr":20556,"ars":16927,"art":134713,"au ":118568,"asi":7486,"ase":11574,"ar ":120208,"api":10314,"aph":15548,"apo":12040,"app":42097,"apr":9346,"as ":37163,"amé":27799,"ava":24401,"aux":51164,"aut":60224,"avr":9575,"avo":10413,"avi":13083,"ave":40354,"ay ":10427,"ata":13379,"aoû":8368,"ast":22288,"ass":51465,"atr":15439,"ato":12284,"ate":44406,"ati":192812,"ath":17738,"auc":6672,"att":16852,"ats":23409,"atu":15332,"aul":9810,"aum":7753,"aur":11354,"aus":19732,"aud":9105,"jet":8119,"jeu":19460,"jan":10693,"fév":8788,"fér":12432,"jou":35022,"itr":12118,"ito":10908,"itu":87676,"itt":10859,"its":10569,"ism":13926,"iso":19540,"isp":7420,"iss":50166,"ist":125656,"ita":62348,"ite":57973,"ith":6663,"iti":60924,"isé":36821,"iva":20746,"ix ":16528,"ivi":25932,"ive":55895,"is ":216454,"ion":363871,"iol":6761,"ipa":18735,"ipe":13929,"ir ":34075,"iro":12325,"iné":18867,"iri":8297,"isi":28415,"ise":101298,"isc":8611,"isa":28074,"iqu":188846,"ire":120253,"ira":8690,"irc":8347,"it ":122098,"ité":60635,"gén":18856,"jus":7076,"jui":19529,"ham":23355,"han":29050,"hau":9292,"har":26334,"hab":12359,"he ":52407,"hel":8715,"hef":7684,"hes":11848,"her":19826,"hie":13715,"hin":15050,"hil":13377,"his":19375,"hiq":6944,"dé ":18734,"déb":7151,"déc":35759,"go ":7689,"cée":7075,"céd":16748,"gle":10717,"gli":6716,"gla":15951,"gno":9326,"gni":10059,"gne":50271,"gna":8886,"gou":8451,"gro":28558,"gra":41927,"gre":10583,"gui":7723,"gue":37672,"ial":35445,"ian":23438,"iat":15622,"ic ":12032,"ibl":8749,"ibu":7144,"ia ":21961,"ieu":43579,"iel":22526,"ien":138676,"ier":92906,"ies":16655,"iff":14012,"ifi":22387,"ict":17983,"icu":12669,"ico":11245,"ick":6729,"ici":37411,"ich":23588,"ice":26902,"ie ":169464,"ica":66266,"idi":10544,"ide":31414,"ida":13896,"if ":16206,"il ":58832,"ige":6657,"igh":9785,"igi":21897,"igu":11721,"ign":41717,"idé":15214,"imp":16150,"ime":18408,"imi":12406,"inc":41459,"ind":21942,"ina":36398,"ino":12991,"int":75915,"ins":44722,"inf":11189,"ine":105011,"ing":34736,"ini":40626,"ila":9244,"in ":120797,"ilo":10879,"ill":136561,"ilm":14297,"ili":45030,"ile":18930,"ima":15975,"io ":17855,"ils":11236,"hol":10188,"hom":18219,"hon":13352,"hor":8438,"dév":10462,"dée":10321,"déf":8940,"dém":9364,"dép":53945,"dér":20431,"dés":15740,"ht ":8492,"hum":15199,"ffe":7771,"ffi":11627,"fes":12569,"fer":7877,"fam":26250,"fai":22917,"fac":6862,"ext":9480,"ez ":9334,"exp":11699,"exi":10548,"eta":8136,"ete":7713,"eti":12647,"esp":33176,"est":489584,"ess":57266,"eul":7484,"ett":36757,"ew ":8505,"eve":10111,"eva":7495,"evi":7979,"euv":6803,"eut":10699,"eur":177418,"eus":14865,"eux":38960,"ey ":17748,"er ":177809,"es ":856843,"ept":19543,"epu":14012,"elé":11860,"epr":16261,"erl":8779,"eri":17387,"erg":13600,"ere":10113,"erc":19130,"era":14672,"erb":9918,"et ":392437,"esc":6799,"eu ":31850,"erv":19978,"err":46160,"ert":38461,"ers":78150,"ern":42947,"erm":29874,"ero":7440,"en ":352124,"ela":11282,"ele":8299,"eli":13751,"elg":9644,"ell":93604,"elo":21097,"els":10727,"emb":53506,"ema":21014,"eme":149235,"emi":31587,"emp":25298,"ene":6850,"ena":20675,"end":38663,"enc":42120,"enn":54421,"eni":9389,"enu":8876,"env":9265,"ens":48323,"ent":371396,"enr":18661,"eil":18380,"ein":23896,"eig":7921,"el ":58534,"gis":10364,"giq":10802,"gin":20723,"gio":57044,"gie":22112,"ght":8779,"gen":26739,"ger":14731,"ges":19429,"ge ":72294,"gar":9985,"gal":16426,"gan":17016,"ga ":6882,"fus":7249,"fut":21703,"fra":95189,"fri":7633,"for":45771,"fon":27020,"foo":11436,"foi":12370,"fic":24328,"fil":25221,"fin":20051,"ffé":6845,"da ":14864,"de ":945562,"dai":12767,"dae":8896,"dat":11688,"dan":177922,"cul":27372,"ctu":20091,"ctr":11700,"cto":18088,"cti":59878,"cte":30379,"cré":24660,"cla":14396,"cle":15005,"clu":9348,"ché":8767,"co ":9868,"cié":10273,"con":120854,"col":28490,"com":148564,"cor":20969,"cou":37141,"cs ":7503,"cqu":7392,"ct ":7934,"cra":8267,"cri":32184,"cro":10112,"cci":7074,"cea":6989,"ch ":14828,"cer":17030,"ces":36299,"cet":8987,"cen":24433,"cem":11283,"cel":14093,"cha":54442,"cia":30366,"ck ":14947,"cie":55297,"cid":7429,"che":76684,"chi":31813,"cho":10460,"chn":7132,"cir":7170,"cis":7169,"cit":9428,"cin":16999,"cip":23717,"cke":8035,"ed ":9999,"ec ":39615,"ean":16858,"eau":41781,"eff":6903,"ech":13667,"ef ":8454,"ect":52251,"eco":14578,"dur":7123,"don":36748,"dom":8813,"dou":8007,"ds ":11646,"dmi":10099,"doc":6954,"dui":12311,"duc":16031,"dri":6694,"dra":6827,"dre":21768,"du ":216503,"dro":13106,"dic":10492,"dia":18227,"der":20812,"des":229646,"deu":19332,"dev":7361,"del":7755,"den":24656,"dep":13089,"di ":9657,"do ":8222,"div":14424,"din":10138,"dio":11927,"dir":13444,"dis":29420,"dit":31296,"die":29137,"dif":14101,"rga":14359,"ri ":10992,"rgi":6778,"rge":19992,"ret":19172,"res":111968,"reu":15390,"rg ":10612,"rea":7335,"rec":29178,"reg":9663,"rem":41279,"ren":49926,"rel":19572,"rer":6909,"rep":18635,"rdi":12299,"rde":11104,"re ":401675,"rco":8273,"rch":25796,"rce":14227,"rd ":49795,"rap":22951,"ras":10406,"rat":61280,"rav":14677,"rbe":7330,"rai":37514,"rag":12671,"ran":191224,"ram":14978,"ral":42593,"rab":8579,"rad":17757,"rac":18041,"rs ":108005,"ros":12152,"rot":13239,"rom":20207,"ron":45128,"rop":30358,"rou":56658,"rov":25371,"rod":19072,"roc":18144,"roi":33087,"rol":7836,"rof":11414,"rog":8618,"rna":26829,"rne":26551,"rni":14539,"riè":7237,"nér":15715,"ro ":12314,"rma":27965,"née":51678,"rme":33077,"rmi":8879,"rle":10727,"rla":9260,"nça":87637,"né ":83238,"rip":8171,"rio":12752,"riq":23356,"rit":49518,"ris":61412,"riv":19174,"rig":30515,"ril":14567,"rin":32694,"rim":9133,"ria":17879,"rib":8141,"ric":64397,"rid":9964,"rie":84165,"rk ":9714,"ruc":9646,"rus":10779,"rvi":9893,"rve":7324,"ry ":13884,"rsi":16639,"rso":15966,"rse":12239,"rta":21499,"rto":8213,"rte":72339,"rth":9356,"rti":82832,"rts":8395,"rtu":7199,"rmé":10475,"rt ":67054,"rqu":9541,"rro":9520,"rri":15769,"rre":35748,"rra":13162,"sac":6821,"sai":15869,"san":31853,"sat":19662,"sa ":20364,"si ":23508,"sie":23358,"sid":13927,"sic":8539,"sit":96579,"sis":11253,"siq":13014,"sin":20761,"sio":46892,"sil":9278,"sig":20886,"scr":8195,"se ":185840,"sca":7156,"sci":11470,"sco":9152,"ser":33436,"ses":32384,"seu":18614,"sea":8788,"sei":15724,"sec":11236,"sep":15495,"sen":31564,"sem":23920,"sel":11350,"spo":15398,"spe":6813,"spa":15911,"sou":38682,"sol":8864,"son":104745,"sor":19583,"soi":7516,"soc":22585,"st ":465410,"squ":17455,"sla":8458,"siè":10850,"sme":14217,"stè":10526,"sys":7499,"où ":7381,"sse":73979,"ssa":25709,"sso":20070,"ssi":65158,"ssu":9524,"ste":86262,"sta":32395,"spé":11204,"sto":25947,"sti":51286,"spè":18097,"str":64918,"sud":12096,"sui":13416,"sul":7775,"sup":9440,"sur":74845,"tai":73243,"tal":50318,"tag":11022,"tab":9930,"oût":9507,"tba":12570,"tat":51850,"tar":13094,"tan":65935,"tam":9043,"tch":9033,"te ":229351,"ta ":14870,"pe ":50543,"par":234438,"pat":9116,"pas":16097,"pay":9300,"pag":21493,"pal":18293,"pan":7834,"phe":9797,"pha":8008,"pho":13907,"phi":21732,"peu":15687,"pen":18365,"per":42656,"pet":8398,"pes":12928,"pel":20669,"pla":26028,"pli":9068,"ple":16401,"plo":8634,"plu":41224,"lé ":12711,"phy":6923,"pio":10675,"pir":7633,"pit":8267,"por":43095,"pop":8694,"pou":69236,"pos":40507,"poi":8423,"pon":18257,"pol":30878,"ps ":10703,"ppo":8721,"ppa":15829,"ppe":26475,"lév":6751,"lég":7457,"lée":13954,"pub":20867,"pte":18796,"pti":14976,"pri":39208,"pre":41906,"pro":84520,"pui":23064,"pul":10006,"prè":17131,"pré":40058,"mé ":9595,"mêm":9754,"mée":9397,"méd":12609,"mét":11009,"mér":37720,"qu ":18907,"qua":24258,"que":267169,"qui":99689,"qué":9551,"ra ":22507,"ngl":20995,"ngu":17741,"ni ":11818,"nge":18618,"nga":7486,"ndé":17148,"nel":18137,"nen":6872,"nem":17361,"ner":14652,"net":8307,"nes":51020,"neu":14088,"ng ":26468,"nfo":8493,"nct":9124,"nco":15938,"nci":43333,"nce":105515,"nch":14886,"ne ":496030,"ndu":10752,"ndr":20713,"ndo":10176,"ndi":34156,"nde":56096,"nda":37414,"nal":54750,"nan":26247,"nar":12696,"nad":12377,"nag":10703,"nai":30209,"nd ":51106,"nau":12991,"nat":48959,"na ":18193,"nté":16371,"ny ":7135,"nvi":18890,"nve":7658,"nue":12712,"nto":18893,"nts":41197,"ntr":72123,"nti":46425,"nta":41915,"nte":98733,"nsu":6773,"nso":9167,"nst":29806,"nse":30814,"nsi":22894,"nsc":6653,"nu ":15262,"nné":23464,"nre":13930,"nt ":427954,"nqu":6929,"ns ":274209,"nol":9967,"noi":11530,"nom":60135,"non":15485,"not":12329,"nor":18442,"nov":11258,"nou":9106,"nne":96393,"nna":28812,"nni":16118,"nnu":18917,"no ":11795,"nif":7747,"nie":43637,"nic":15379,"niv":18250,"nis":60898,"nit":15799,"niq":25898,"nio":9717,"nim":8520,"nin":8256,"ogr":17742,"ogi":23895,"ogn":7673,"oi ":11549,"ois":67090,"oir":45748,"oit":23632,"oin":16797,"ol ":10751,"och":11107,"oci":26047,"ock":12402,"oca":13060,"occ":8851,"ode":18688,"oct":13097,"of ":10906,"odu":17793,"off":9016,"ofe":9519,"obr":10777,"obi":7081,"oye":7463,"oya":11498,"osé":9652,"oue":24956,"ouc":7361,"oti":7550,"ote":10769,"oto":12004,"opé":11013,"ost":16493,"ota":13987,"otb":12106,"osi":13992,"ose":15543,"oss":13431,"ovi":29641,"ouv":52919,"ove":15893,"oug":6908,"oui":10077,"oul":19390,"oup":35466,"ous":44108,"our":142367,"out":30934,"opo":10738,"opp":14092,"ope":8783,"oph":15160,"os ":20357,"opu":8920,"or ":16010,"oot":13483,"ork":6806,"orm":40453,"orn":11249,"orr":9505,"orc":7615,"ord":43495,"ore":17630,"org":20050,"ori":52286,"ou ":81997,"ort":98537,"ors":17665,"omé":6755,"ot ":11871,"ora":19855,"ola":10628,"on ":413868,"oli":37368,"oll":18132,"ole":20481,"olo":39631,"olu":16814,"om ":32719,"ona":46449,"ond":64776,"onc":25758,"onf":7947,"one":18885,"ong":21441,"oni":23679,"onn":93395,"ono":18517,"ons":103783,"ont":114426,"ony":7110,"oma":27935,"ome":15382,"omb":20959,"omi":18767,"omm":120540,"omp":47351,"omo":9810,"omt":10618,"la ":409873,"le ":650000,"lab":8668,"lac":18171,"lag":11480,"lai":40035,"lan":67550,"lam":6667,"lar":10935,"lat":30778,"las":17285,"ld ":8004,"lbu":11677,"llé":6984,"ls ":27190,"lon":33452,"lom":9149,"lop":13744,"lor":20550,"loc":9951,"log":34746,"loi":10734,"los":7562,"lié":10397,"héo":7867,"lti":6832,"lub":8118,"lue":8111,"li ":8120,"leu":30125,"les":193810,"let":24641,"ler":15063,"lem":54863,"len":12174,"lec":18621,"lo ":9843,"lla":30932,"lle":228673,"lli":25721,"llo":13913,"lm ":12927,"ll ":25434,"lit":56867,"lis":75238,"liq":19529,"lio":7237,"lin":24717,"lim":6784,"liv":7668,"lic":17589,"lia":14317,"lib":8142,"lig":13313,"lie":66363,"lif":7647,"ma ":11524,"mb ":9981,"mai":41522,"mag":14210,"mar":34078,"mas":7661,"mal":10704,"man":59224,"mat":42658,"mba":7370,"mbl":12331,"mbr":52251,"mbo":7187,"me ":138747,"met":15733,"mes":29770,"mer":17959,"mem":10190,"men":192406,"lui":7872,"lut":8740,"lus":46191,"mpi":17291,"mpr":7983,"mpo":28665,"mpl":18530,"mps":7384,"mpt":6961,"ms ":7519,"moi":10596,"mod":8999,"mon":39570,"mor":34638,"mot":9918,"mou":10824,"mpa":13773,"mmé":6819,"mus":17057,"mul":7053,"mun":72605,"mi ":7506,"min":39720,"mil":42847,"mis":18157,"miq":9951,"mit":10387,"mie":22944,"ièr":39842,"ièm":14714,"ié ":12315,"mmu":65417,"iét":13514,"miè":11922,"mma":8876,"mme":60859,"yst":10800,"ys ":16246,"yen":7179,"yan":8923,"ué ":23787,"uéb":10376,"uée":52636,"tér":21675,"tés":15841,"tél":10092,"tée":11073,"tèr":7376,"tèm":7268,"té ":120644,"sée":25892,"sér":11984,"sé ":33254,"réé":15299,"vri":19364,"vre":15576,"vra":7557,"vir":10748,"vil":38576,"vin":28833,"vic":9378,"vid":11495,"vie":25896,"vit":11987,"vis":24817,"ré ":18385,"rès":23870,"réc":11353,"rée":15149,"réf":7765,"réa":27396,"rén":7293,"rég":60534,"rét":7637,"rés":42404,"rép":7979,"voi":21769,"vol":18467,"ver":63394,"ves":12081,"ven":32893,"vem":17149,"vel":23577,"vea":6695,"vec":28438,"ve ":39545,"val":17132,"van":23469,"var":6981,"vai":21124,"uté":12504,"ux ":89405,"uve":48851,"uvr":11649,"usi":30125,"use":23041,"ust":23088,"uss":29122,"uti":33117,"ute":37011,"uto":14655,"utr":16881,"us ":97767,"ut ":58626,"ura":21007,"ure":61643,"urg":15629,"uri":18044,"urn":17785,"uro":15520,"urs":51288,"urt":11255,"ur ":263487,"upe":34564,"uma":8198,"umb":11805,"ume":16466,"uni":34130,"una":10830,"une":273015,"um ":21626,"ult":20316,"uli":14760,"ule":23978,"ula":21824,"un ":279711,"uil":20013,"uin":15450,"uip":9137,"uis":42982,"uit":34639,"ul ":11106,"ui ":87556,"uct":19646,"ude":13798,"udi":11213,"ue ":252733,"uch":11064,"ueu":15091,"uer":16125,"ues":65033,"uen":9815,"uel":31495,"ub ":7072,"uat":10437,"uar":8894,"uan":14849,"ubl":25725,"ud ":22607,"tué":65263,"typ":6932,"ty ":7203,"tré":9963,"tur":39432,"tut":8273,"tud":13647,"tue":18585,"ts ":96064,"tre":117480,"tra":79361,"tri":46224,"tru":15862,"tro":41105,"tta":10904,"tte":35818,"ttr":6953,"pée":8023,"péc":10887,"to ":12714,"pér":17722,"tiè":7263,"toi":21756,"tob":11135,"tou":31577,"tom":9528,"ton":34164,"tor":25952,"til":20545,"tif":22436,"tie":47885,"tir":8477,"tiq":62118,"tit":40283,"tis":20099,"tin":33045,"tim":9651,"tio":235114,"thu":11875,"tia":8957,"tic":25452,"tiv":30333,"thé":16932,"pèc":18042,"tem":70682,"ten":37907,"tel":13125,"tec":11914,"th ":10711,"teu":75009,"tes":56728,"ter":82954,"ti ":26511,"tho":14369,"the":16191,"Éta":23397,"ège":6746,"èce":21913,"ère":67047,"ète":9916,"ène":11368,"ème":27688,"ès ":31596,"édé":24476,"éga":13682,"égi":62907,"écé":13741,"édi":27690,"éce":13512,"éci":16144,"ée ":173192,"écu":7487,"écr":22748,"éco":23770,"éfi":6764,"ées":42716,"éen":8156,"ébe":6903,"éal":20524,"éve":11864,"évi":9923,"éré":8391,"évo":13471,"évr":9038,"éta":51936,"éti":16412,"éte":7748,"étr":10893,"étu":6864,"été":39872,"éme":8284,"émi":12731,"émo":10213,"éna":8958,"éni":9260,"éli":8934,"éle":12690,"éma":11177,"éo ":6991,"équ":14445,"éra":46783,"ére":13799,"éri":83768,"éro":16442,"éné":26266,"ése":23781,"ési":28918,"épa":51263,"épu":10400,"épo":7454,"élé":15180,"és ":45990,"ême":12159,"êtr":10139,"çai":86644,"ût ":9033,"ôte":8179,"île":9715,"éé ":7606},"n_words":[66338594,78580813,56850284],"name":"fr","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/ga.json b/contrib/languages-data/ga.json
new file mode 100644
index 0000000..73d5eb1
--- /dev/null
+++ b/contrib/languages-data/ga.json
@@ -0,0 +1 @@
+{"freq":{"fág":374,"fái":240,"fán":17,"fás":43,"Bá ":22,"D":3670,"E":1972,"F":3366,"G":4685,"A":5367,"B":7488,"C":9211,"L":3210,"M":5923,"N":2064,"O":1295,"H":1083,"I":7769,"J":910,"K":695,"U":762,"T":5807,"W":639,"V":513,"Q":100,"P":3001,"S":6241,"R":3017,"Y":140,"X":81,"Z":115,"f":10484,"g":35594,"Fea":348,"d":35722,"e":80219,"b":22235,"c":48153,"a":205992,"n":112182,"o":48493,"Fei":128,"l":55972,"m":27549,"j":233,"k":1737,"h":92271,"i":133366,"w":1117,"v":1580,"u":29979,"Fer":45,"t":60494,"s":65408,"r":82828,"q":111,"p":7714,"z":576,"y":2584,"x":359,"Fha":60,"Í":207,"É":2404,"Fho":14,"Fhe":48,"Fhi":99,"Á":813,"fá ":15,"Fia":48,"Ú":160,"Fhr":107,"Ö":14,"Ó":660,"Fil":63,"Fin":103,"í":22652,"ë":49,"Fio":121,"é":21077,"Fir":16,"è":30,"ç":26,"å":17,"ä":37,"ã":17,"â":15,"á":26187,"à":53,"ü":78,"ú":11150,"ù":19,"ö":111,"ò":31,"ó":13717,"ð":28,"ñ":23,"ē":22,"féi":573,"ā":45,"ı":15,"ī":25,"ł":23,"че":14,"ō":29,"š":15,"ū":19,"ск":41,"то":18,"ст":21,"Fat":14,"féa":120,"Far":31,"Fao":27,"Fad":18,"Fai":28,"itn":37,"ipé":81,"itl":172,"itr":218,"ipí":17,"ito":21,"itu":20,"itt":39,"its":92,"itz":45,"ity":81,"Eri":32,"ism":40,"isl":100,"iso":33,"isn":120,"isp":63,"iss":59,"ist":1852,"iv ":26,"ita":62,"itc":21,"ite":2146,"ith":4604,"iti":613,"irí":823,"Esp":20,"iré":62,"ɪ":16,"isé":44,"ius":114,"ium":34,"Ern":19,"iva":35,"ix ":29,"ə":39,"ivi":35,"ɛ":15,"ive":99,"ilí":349,"ipp":21,"ips":24,"ipt":76,"ipi":79,"Eur":16,"ilé":72,"Eva":15,"Eve":20,"is ":8915,"ion":1312,"iop":36,"ior":150,"ios":247,"iot":378,"iof":15,"iog":41,"iol":50,"iom":483,"ipe":96,"ir ":10292,"irv":17,"irs":353,"irt":1857,"iní":139,"iro":33,"irp":47,"irm":319,"iné":195,"irn":286,"irk":27,"irl":227,"iri":2132,"isi":1776,"ish":105,"isf":50,"ise":1193,"isc":1361,"isb":24,"isa":22,"iu ":157,"iqu":19,"ˈ":48,"imí":77,"imé":209,"irf":27,"ire":4833,"irg":383,"irb":139,"ira":42,"ird":294,"irc":279,"ː":17,"it ":669,"El ":21,"ünc":20,"Eil":84,"Eis":23,"́":82,"ja ":23,"μ":37,"ν":74,"ο":100,"Eng":16,"θ":18,"ι":51,"κ":26,"λ":62,"δ":20,"ε":36,"η":30,"α":107,"β":19,"γ":29,"έ":19,"ά":19,"ί":36,"ití":138,"Emi":14,"ité":82,"Eo ":105,"isí":148,"Eli":30,"Ell":16,"Eor":261,"Eol":17,"Eog":51,"ivé":51,"Eoi":40,"Eoc":19,"iza":25,"ω":17,"ύ":15,"ό":27,"σ":40,"ς":68,"ρ":66,"π":27,"φ":26,"υ":24,"τ":48," l":8985,"ь":28," m":7148,"э":32," n":15246," o":1707,"я":18," h":3099," i":18215," k":158,"ы":19," d":12271," e":3594,"х":18," f":6713,"ц":14," g":7608,"ч":57,"р":135," a":48402,"úsa":108,"с":109,"kie":20," b":10871,"т":81," c":12912,"у":41," y":42," z":14," u":1021,"kin":66," t":9735," w":69," v":164," p":2643," s":15941," r":5178,"úta":17,"úth":153,"km ":105,"ús ":237,"ki ":55,"И":15,"К":15,"П":17,"А":21,"В":16,"Д":14," J":910," K":682," H":1081," I":7545," N":2056," O":1160," L":3193," M":5910," B":6902," C":8023,"úra":117,"kha":23," A":4758,"С":21," F":3245," G":4206," D":3525," E":1742,"Gea":306,"л":108," Z":111,"к":120," Y":140,"й":72," X":80,"и":188,"п":19,"о":212,"н":176,"м":39,"Cás":28," S":5918,"г":35," R":3015,"úrs":169,"в":118,"Ger":46," Q":98,"б":29," P":2788,"а":242,"Geo":92," W":633,"з":23," V":510,"Gen":35," U":701,"úrt":165,"е":136," T":5610,"д":38," á":890," í":1985," é":7791,"únt":454,"gái":142,"gál":25,"kel":20,"gán":133,"ken":22," ó":1986,"kes":25,"gás":18,"Cé ":51,"ker":42,"ket":22," ú":1600,"key":21,"úpa":264,"úr ":181,"úpl":35,"Gla":76,"ke ":58,"Ghe":106,"Gha":320,"úlr":31,"Gho":18,"últ":26,"Ghl":88,"Ghi":30,"Ghu":20,"Ghr":519," Á":761,"úna":570,"Gio":40,"Gin":21," É":1719," Í":149,"Gil":20," Ó":649,"únm":24," Ö":14,"hÁi":46," Ú":148,"únd":18,"ku ":25,"kov":15,"kol":18,"ks ":25,"hÍs":21,"hÍo":25,"ה":23,"ו":21,"א":14,"ל":17,"י":36,"géi":147,"úsá":344,"útó":18,"gí ":33,"ר":17,"Gao":19,"Gal":229,"ָ":14,"géa":181,"ּ":16,"Gar":111,"Gai":188,"Gae":1050,"hÉi":662,"Gab":25," Ái":88," Ár":98," Át":543,"و":29,"Bó ":14,"ي":52,"fór":53,"fós":55,"fón":15,"ل":53,"م":48,"ن":39,"Fui":17,"Fua":59,"د":34,"ح":14,"ت":14,"ب":21,"ة":18,"ا":87,"fói":40,"ع":14,"س":25,"fód":32,"ر":41,"Bót":22," ˈ":34,"fío":57,"fír":23,"fís":212,"kar":18,"kan":18,"Béa":950,"Flo":29,"Fhé":16,"Fla":23,"Fle":15,"fút":14,"ka ":67," А":21," В":16,"Fra":387," Д":14," И":15," К":15,"Fri":39,"Fre":40,"A ":162," П":17,"Bío":106,"Foi":71,"Foo":31,"For":64," α":14," Ga":1406,"F ":106," Bú":24," Ge":490," Cá":50,"Da":525,"Dé ":27,"Cu":433," Bó":52," I ":184,"Cy":22,"Cl":805," Bí":110,"Cn":52," Fo":226,"Co":2077," Fu":111,"Cr":445," Fr":436," Fh":371," Fi":384,"Ce":563," Fl":82,"Ch":2601," Bé":806,"Ci":377," Ha":296,"G ":74," Dá":241," He":239,"Ed":73,"Ea":766," Có":81," Gw":17,"Du":229," Cú":124," J ":15," Go":230,"Do":507," Cí":18,"Dl":37," Gr":356,"Dr":164," Gu":114," Gh":1129," Gi":149,"De":754," Gl":184,"Di":257," Cé":88," Gn":25,"Dh":350,"cúp":30,"cúr":63,"Fe":557,"Bá":48," Id":99,"H ":53,"Déa":55,"Fa":174," Ia":276," Dú":161," K ":14,"cúi":261,"Eu":46,"cúl":23,"Ev":36,"Ex":17,"Er":77," Dó":33," Hu":86,"Es":55,"En":53,"Em":36,"Eo":495," Dí":30," Ho":194,"Ei":150,"El":97," Dé":97," Hi":140,"ha ":3363,"Cá":56,"Ge":520," Ji":54,"Ga":1705," Fá":64," Je":87,"Bú":28,"I ":379," Ja":245,"Bó":57,"Fu":117," Iv":17,"Fr":485,"Bí":112," Ir":128," Is":5161,"Fo":242," It":14," Im":262,"Bé":956,"Fl":84," In":364," Io":352,"Fi":403,"Déi":15,"Fh":373," Il":46,"ham":346," M ":16,"han":1176,"hao":727,"hap":30," Ka":137,"hai":6276,"hal":414," Ke":187," Ki":123,"har":2134,"has":684,"B ":234,"hat":707," Kh":43," Fé":97," С":21," Jo":392,"II ":143," Ju":72,"hae":331," Jr":24,"hag":201,"hab":166,"had":297," Fó":592,"hac":666," N ":85," La":605," Le":483," Li":406,"C ":207,"hba":15," Kn":16," р":15,"hd ":21," Ko":48," Kr":33,"hbh":223," Ku":37," Mc":98,"Au":67," Ma":1326,"Ar":549,"At":157," O ":116,"As":333,"D ":237," Mi":402,"Ba":1930," Mh":1153," Me":1282,"Ae":65,"Af":126,"dá ":242,"he ":2867,"Ag":62," Lo":611,"Ab":180," Hé":19,"Ac":138,"Ad":72," Ll":24,"Am":211,"An":878," Ly":37,"Ao":683,"Ap":36,"Ai":875,"hch":612," Lu":335,"Al":673,"By":31,"Hit":18," Ne":228,"а ":44,"Bu":392," Iú":336,"Br":787," Na":383,"Ca":1185,"E ":51," Ni":185,"Bh":1538,"Bi":171,"Be":744,"hda":17,"Hig":22," Mo":403," My":20,"hde":119,"Bo":264,"Hil":25,"Bl":146," Mu":284,"hdi":36,"Hio":16,"hdh":155,"dál":94,"Ku":37,"hel":99,"hei":1299,"dái":241,"Kn":16,"hea":3474,"Kr":33,"Hay":19,"Ko":50,"Le":491,"hew":16," A ":89,"Li":408,"N ":133,"dát":17,"hes":67,"her":184,"dár":15,"haí":305,"heo":669,"hen":86,"dán":99,"hem":16,"La":609,"Lu":335,"Ly":38,"hfh":57,"hfi":58,"Hé":19,"Ll":24,"Lo":614,"hfe":113,"Dá ":15,"hfa":66,"Me":1284,"hi ":227,"Mh":1155,"Mi":403," B ":218,"hfr":27,"O ":148,"hfu":664,"hfo":149,"Ma":1327,"Mc":98,"hgh":63,"My":20,"Mu":285,"Mo":406," C ":40,"Ni":186,"Ne":229,"Na":387,"P ":39,"Iú":345," Ao":620," Ap":32," Am":200," An":855,"Dál":164,"Hel":19," Al":487," Ai":787,"Hei":32,"Dái":50," Ag":59," Ae":60,"Nu":291," Af":79," Ac":123,"No":499," Ad":67,"Hea":28," Ab":148," Ba":1677,"Ol":192," D ":160,"On":32,"Oi":439,"Oc":20,"Od":23,"Hen":60," Au":67," At":140," As":265," Ar":510,"Her":56,"Ob":21,"Gi":165,"hig":33," Be":713,"heá":241,"Gh":1136,"hie":19,"Cé":98,"hid":79,"Gn":26,"hic":130," Bi":165,"Gl":218," Bh":1534,"hia":602,"Cí":19,"hip":47,"hio":429,"Gr":417,"hin":549,"Go":237," Bl":127,"him":81,"Gu":127," Bo":242,"hil":430," Br":722," Bu":374,"Có":94,"Gw":18,"his":511,"Cú":234,"hit":119,"J ":18,"hir":287," By":31,"Ha":296," Ca":1052,"Dá":246,"He":239,"Hi":140," Ce":495," Ci":334,"Dé":102," Ch":2590,"Ho":194," Cn":49,"Dí":30," Cl":788," Cr":398,"Hu":87,"hfé":14," Co":1382,"Dó":33," Cu":398,"hfó":38," Cy":22,"K ":26,"Cúi":194,"Dú":193," F ":85,"Ib":14,"Ia":309,"Id":105,"dé ":34,"hm ":163," Da":489," Di":255,"Io":441," Dh":344,"Im":285,"In":411," De":739,"Il":49,"Iv":17," Dr":151,"Is":5163," Dl":36,"It":14," Do":485,"Ir":130,"Ja":246,"L ":51," Du":227,"hn ":148," Ea":709,"Ji":54," Ed":71,"Fá":72,"Je":87,"hla":719," El":96,"hle":240,"Jo":392," Ei":130,"hli":2004,"Fé":97,"Fó":594," Es":55,"hlo":95," Er":76,"Jr":24," Eo":356," En":50,"Ju":72," Em":35,"Hal":38,"hlu":153,"Ka":139," Ex":17,"M ":83," Eu":45,"Han":37,"Ham":35," Ev":35,"Kh":43,"ho ":34," Fe":551," Bá":46,"Har":97,"Ki":125,"hma":28,"déa":414," Fa":169,"hmc":14," H ":25,"Ke":192,"céa":541," Tó":45,"go ":1892,"Ur":48," Tí":110,"Un":145," Té":54,"Ul":134,"Ui":83,"glu":75,"glo":14," Tá":2087,"ghé":59,"Ua":134,"ghá":14,"gle":54," Sú":23," Só":54,"Pó":40,"gli":33,"Tu":566," Wy":14,"Tr":269," Sí":148,"Pé":28,"gla":380," Wo":76,"To":290,"Th":1249," Sé":182,"Ti":174," Wi":221,"Pá":248,"Te":461," Wh":33," We":93," Sá":44,"Ta":319,"Úis":14," Wa":161," Rú":279,"V ":65,"Sw":14,"й ":53,"Sy":24,"St":806," Ró":193,"Su":269,"Sí":182,"Wo":77,"gob":19,"Wi":224,"cí ":85,"Wh":33,"Sé":185," Ví":26,"Wa":162,"hOs":21,"Rú":279," Ze":18,"We":94,"Sá":50," Zi":16,"Ró":193,"hOl":21,"Y ":15,"hOi":42,"gni":28," Za":30,"Vo":36,"Rí":362,"gne":64," Uí":112,"giú":420,"gna":72,"Vi":167," Yo":40,"Ré":222,"X ":33,"Va":115," Ya":26,"Ve":93,"Rá":59," Ye":18," Tú":22,"gmh":16,"céi":99,"Má":456,"gs ":22,"Qa":19,"glú":20,"úba":28,"úd ":102,"Mé":25,"Gré":160," م":17,"Pu":37,"Pr":303,"Lí":32,"glé":50,"о ":14,"Lú":395,"S ":72,"Gua":19,"Pe":309,"Lá":163,"Pa":425,"Gui":35,"Lé":53,"Pl":114,"Po":635,"úc ":28,"Pi":190,"Ph":604,"Os":120,"Ot":24," ا":42,"н ":32,"Op":26,"gon":18,"Or":118,"R ":47,"gos":16,"gor":55,"Ow":22,"Ox":16,"Se":734,"Sc":406,"Si":351,"Sh":551,"Sg":27,"Sn":28,"Sm":30,"Sl":185,"Sk":30,"Sr":230,"úda":213,"Oí":18,"Sp":350,"So":189," a ":8371,"gní":96,"Ru":470,"р ":23,"úcr":20,"Ry":20,"gné":36,"Cór":28,"U ":28,"gnó":47,"Sa":1373,"úca":22,"Re":192,"gra":412,"Ná":110,"Ri":337,"gt ":23,"úch":359,"Rh":17,"Cón":21,"gri":67,"Né":26,"Ro":461,"gre":39,"gná":104,"Ní":182,"Cói":21,"Mí":197,"cío":21,"cín":25,"Qu":70,"Mó":140,"Grú":25,"T ":38,"cís":24,"Mú":20,"cít":16,"Ra":236,"Mü":22," R ":39,"в ":19," Ow":22," Ox":16,"gto":41," Os":95," Ot":23," Or":112," Op":26,"Gre":42," Po":543," Lé":53," Pl":107,"Gri":35," Pi":185," Ph":599,"có ":37,"gua":61," Pe":292," Lá":163,"Gra":99," Pa":406,"gue":52," Nu":290," No":498,"gth":443," Ol":168," On":31,"gte":20," Oi":369," Od":23," Oc":18," Ob":21,"hUa":22,"gta":637,"gy ":14,"grú":149," Ra":236," Mü":22," Mú":20," Mí":196," Mó":140," Qu":68," Né":26,"cón":291,"cói":80," Ní":182,"údá":21,"b ":1035," Ro":460," Re":192," Ná":110,"cód":14," Ri":337," Rh":17," S ":17," Lú":395,"Gru":17,"Úcr":31,"gut":34," Lí":32,"Gro":24,"gur":328," Pr":285,"gus":7532," Pu":36,"gré":40,"a ":43319," Mé":25,"Úda":15," Qa":19,"grá":39," Má":456," Sy":24,"Céi":17,"Tú":25,"úl ":165," Sw":14," Su":231," St":802,"úit":93,"Ye":18,"úis":399," Ta":308,"Ya":26,"úir":269," Th":1246,"Yo":40," Ti":162," Pá":205," Te":438," Tr":254,"Giú":44," To":282,"Uí":112," Pé":25,"Gle":76,"cós":67,"Ghá":16,"Sú":30,"cór":164," Ry":20,"cóp":18,"Wy":14,"Só":54," Ru":470,"Tá":2089," Sa":1332,"gsú":141," Sg":27," Sh":547,"úin":409,"Glu":16,"Té":56," Si":309,"úil":1485," Sc":405," Se":630,"úig":450," So":181,"Tó":46," Sp":350,"Go ":71,"Céa":22,"úic":75," Oí":17," Sr":217,"úid":61," Sk":30," Sl":167," Sm":30,"Tí":131," Sn":28,"ún ":707,"úla":387," Va":113," X ":22," Ve":93,"Gor":57," Rá":59," Vi":167," Ré":221," Rí":362," Vo":36," Tu":471," Pó":39,"Za":30,"Ze":18," Ua":104,"Zi":16,"Ví":26," Ui":69," Ul":132," Un":136,"God":16," Ur":43,"Gob":16,"iai":2569,"i ":7943,"iam":415,"ial":677,"bó":145,"ian":964,"fy":18,"gc":1529,"gd":28,"hE":204,"cá":614,"ge":2607,"hF":113,"ias":115,"iar":1266,"gf":27," fá":646,"Iod":189,"bú":25,"hA":434,"iat":624,"ga":4696,"gb":67," io":626,"bé":131," im":729,"fl":50," in":3888," il":62,"ff":123,"fi":830,"ic ":568,"fh":1623,"Ini":60,"fr":651,"iab":149,"Inn":16,"fu":1205,"ft":48,"iac":136,"Int":32,"iad":874," is":4270,"fo":1220,"Ins":58," it":46,"bí":296,"iag":42," ir":76,"ibl":72,"j ":21,"cú":400," fú":16,"ibi":111,"gy":22,"có":705," m ":25,"hf":1201,"hg":74,"ibr":504,"hd":373,"dá":713,"he":9095," hÁ":49,"hb":245,"hc":639," gá":43,"ha":17871,"hO":88,"gn":464,"gm":31,"cé":763,"gl":642," fí":105,"gi":1171,"Iom":65,"hI":157,"gh":5251,"gg":41,"id ":2038," fé":577,"Ion":38,"gu":8062," fó":153,"hU":40,"gt":1166,"gs":210,"Ios":63,"gr":790,"cí":192,"Ior":55,"ibh":672,"ibe":76,"go":2060,"dt":1341,"du":779,"dw":67," có":303,"dúl":59," cú":315,"dy":102,"dúi":243,"g ":4176,"dúr":62," ha":566,"ea":25239,"eb":101,"dún":41,"ec":179," he":103," dá":331,"ed":379,"de":6172," gi":157,"dd":146,"dg":84," hI":155," gh":896,"df":76,"di":2454," cé":299," hO":88," gn":249,"dh":7736," gl":224,"dm":58," cí":22,"dúc":72," gr":303,"dl":217,"do":1640," go":1885,"dn":21,"ia ":680," gu":367,"ds":128,"Ill":14," hU":40,"dr":674,"ew":190,"ex":98,"eu":111,"ev":181,"ey":348,"ez":71," dú":293," ia":1300,"fa":1504,"Inb":19,"Inc":22," id":903,"h ":22971,"Ind":128,"fe":932,"bá":362,"gC":1143,"eh":31,"eg":159," hi":225,"ib ":82,"ef":77,"ee":269,"el":1747,"ek":71," dé":333,"ei":8910," ho":149,"ep":165,"Imp":113," dí":197,"eo":5144,"en":2705,"Imr":156,"em":248,"et":550," ht":48," hu":37,"es":1134,"er":2470,"aí":5441," dó":278,"iet":28,"cb":18,"ca":3365," ni":34,"e ":26562," ng":227,"iel":96," nd":382,"eál":273," ne":306,"ien":60,"eán":1781,"by":43,"bs":46,"br":1157," na":8256,"ier":82,"eár":16,"bu":707,"ies":49,"bt":15," iú":22,"eád":40,"bo":363,"ied":26,"bp":167,"ieg":18,"eái":390,"bl":1578,"и́":22," mu":97,"cG":32,"bh":11551,"bi":450,"ig ":954,"cC":35,"bb":61," mo":107," mn":21,"eác":154," nO":20,"be":1029,"dc":62,"db":293," ol":129,"da":3031,"In ":39," on":26,"f ":269," oi":794,"ifo":72," oc":122,"cy":33,"ibé":23," of":143,"cu":1138,"ct":536,"ibí":99,"ifr":59,"cs":370," ob":69,"dT":187,"cr":1405,"co":2084,"iff":28,"ife":99,"cm":91,"cn":208,"ck":346,"ifi":245,"cl":898,"ci":1417," nu":480,"ch":26341," no":28,"ce":4885,"cf":27,"cc":47," le":4947," há":141,"icr":56,"c ":2108,"ics":128,"ict":334,"icu":19," li":530," n ":179,"icn":56,"ico":53,"ick":139,"icl":81," la":460," mB":573,"icm":54,"ici":351,"ich":719,"ice":1644," gé":72,"eá ":311,"ie ":209," km":111,"ica":132," hÉ":663," hÍ":55," me":369," nG":425,"az":83,"ay":273," mh":955," nI":25,"ba":2824," mi":457,"ids":14,"d ":7267,"idt":208,"at":6355,"as":7371,"idr":97," hú":26,"ar":22782,"ido":20," nA":40,"ax":41,"idm":20," ma":2135,"aw":83," mb":882,"av":283," nD":124," nE":15,"au":284,"ak":145," lu":291,"idi":1657,"al":6550,"idh":1444,"ai":28977,"aj":44,"idg":41,"bP":203,"ao":4221,"ap":476,"ide":714,"idd":27,"am":5107,"an":31829,"idb":243,"ac":15816," hé":28,"ad":8059,"ida":56,"aa":74," lo":308,"ab":2446,"ag":12220,"ah":164,"ae":3237,"af":197,"nu":707," ae":136,"nt":6330,"ns":1273," ag":9107,"nr":644,"np":33," ab":662,"no":556," ac":887,"nn":13453," ad":142," am":1152," an":14934," bP":203," ao":494,"nz":50," ap":19," ai":2945,"ny":183,"nw":29,"nv":31," al":169,"oe":87,"of":346,"oc":3503,"od":1311," ar":6072,"oa":84,"ob":1794," at":2168," as":1109,"om":4389,"on":9178," d ":589," ba":1293,"ok":119,"ol":4027,"oi":8724,"oj":15,"og":1022,"il ":4239,"oh":246," bh":6443,"ot":1050," bi":158,"os":2328,"ov":241,"ou":522,"op":291,"oo":309," be":526,"or":3909," bp":167,"r ":26836," bo":185," bl":423,"ox":26,"ow":261,"oz":32,"oy":150,"ifí":195,"lá":2770,"pe":951," bu":524,"pa":1122,"pc":34," br":367,"pl":469," ca":803,"lé":691,"po":724,"ph":1119,"im ":440,"pi":497,"ika":15,"lo":964,"ln":33,"ige":988,"lm":247,"hé":1982,"ll":4477,"ls":671,"iga":43,"lr":97,"hí":2469,"ii ":17,"lp":55,"hó":1143,"lw":14,"lv":106,"icé":38,"lu":1017,"lt":2738,"Idi":92,"igh":3179,"hö":28,"igi":784,"ly":208,"igg":14,"hú":882,"igu":24,"nA":42,"igt":17,"o ":4410,"nD":124,"mc":74,"icí":106,"nE":15,"md":20,"igo":16,"ma":4924,"mb":1185,"ign":27,"mg":14,"mh":8902,"nI":25,"me":1225,"iá":23,"mf":48,"nG":433,"ml":170,"mi":1584," b ":71,"mn":295,"nO":21,"mm":166,"mp":948,"mo":376,"mr":638,"mt":67,"ms":365,"mu":293,"idé":131,"ió":116,"my":76,"p ":418,"iú":2995,"na":20087,"nb":151,"nc":1202,"nd":1981,"ne":5883,"idí":295,"nf":133,"ng":2368,"nh":89,"ni":2645,"ik ":21,"nj":39,"nk":107," c ":38,"nl":294,"nm":973,"imo":55,"imn":120," et":14,"imm":26,"fó":227,"ims":271," en":17,"imr":401," eo":211,"imp":429,"fí":323," ei":675,"imf":18,"ime":312,"jo":35,"fé":693,"imi":716,"imh":994," gC":1143,"ki":194,"kh":56," fe":599,"ip ":117," bá":278,"inc":452," h ":53,"ke":227,"gá":342,"ind":132,"hÁ":50,"ina":1555," fa":1032,"inb":24,"ka":183,"imt":31,"fú":18,"m ":2796,"а́":17," fu":504,"inn":3418,"gó":548,"inm":801,"ky":50,"ino":76," fr":394,"gö":27,"ks":87," bí":103,"inr":60,"ku":55," fo":815,"int":1223,"ins":402,"inf":38,"ko":112," bé":67," fl":33,"hÍ":55,"ine":2867,"inh":20,"gí":69,"ing":553,"kr":26," fi":426,"kk":18," fh":1406,"hÉ":663,"ini":1121,"kl":45,"gé":329,"inl":22,"km":117,"ink":22," cá":319,"li":6865," ge":143,"lh":37," gc":1511,"lk":48," hE":204,"há":2539,"le":11033," ga":579,"iob":126,"ioc":151,"ld":296,"lg":924,"iod":20,"lf":136," hA":434,"la":10153,"mB":573,"inu":20," i ":6264,"lc":271,"lb":620,"iny":32,"gú":29,"n ":44303," bó":54,"igí":38,"hr":4331," cl":537,"hs":439," cn":57,"hp":68,"iko":20,"dí":554,"igé":189," co":1181,"dó":754,"hw":24," cr":447,"ht":5712,"hu":4839,"iki":22,"hk":20,"hi":3686," ce":1239,"ike":19,"hn":1628," ch":6174,"ho":3923,"hl":3740,"hm":370," ci":449,"dé":521,"ila":50,"id":7038,"ilb":41,"ic":4509,"ib":1678,"ia":8556,"ih":33,"in ":9648,"ig":6298," da":665,"if":738,"eá":2972,"ie":631," dT":187," cu":711,"hy":66,"dú":564,"k ":471,"iq":21," do":1320,"ir":22505,"ilo":20,"is":15956,"it":9092," dl":128,"ill":1658,"iu":350," dr":195,"iv":271,"eó":36,"ilm":27,"ix":45,"ii":26,"ilg":818,"ij":27,"ik":120," de":4551,"ili":1289,"il":13368,"ild":34,"im":4227,"in":22865,"ilc":72,"ilf":25,"io":3173," di":196,"ile":3808," dh":1220,"ip":561,"ima":59,"je":23,"fá":699,"imb":35,"ч ":34,"Iar":271," ea":2630,"io ":128,"ji":36,"iz":73,"iy":39," dt":1102," du":676,"ils":248,"ilt":514,"l ":9412,"ja":59,"ilv":45,"ль":14,"hlú":76,"xi":32,"té":533,"Dúi":63,"tí":1793,"Dún":93,"tó":758,"ми":19,"xt":17,"wy":58,"hlé":82,"ww":87,"só":186,"sú":492,"z ":155,"hph":63,"Dúc":24,"hló":128,"xa":66,"tá":3852,"xe":16,"hlí":29,"wi":117," ví":29,"how":16,"wl":15,"sé":2086,"ла":34,"wn":96,"hlá":205,"wo":78,"ле":18,"sí":985,"wr":17,"ws":41,"wt":17,"ró":676,"hol":348,"hom":626,"hon":716,"hog":133,"hoi":807,"y ":1513,"ко":39,"hos":256,"rú":779,"hot":43,"wa":203,"hop":25,"we":124,"sá":626,"hor":473,"ré":2568,"dí ":288,"vi":359,"hob":241,"vr":17,"vs":40,"ки":31,"rí":3930,"hod":30,"vn":14,"vo":75,"hoc":113,"hni":60," tú":106,"uz":23,"ux":50,"uv":36,"uu":19,"rá":2009,"ve":416,"hno":28,"va":328,"hnu":19,"x ":157,"ui":6290,"uk":43," té":207,"ul":621,"ue":219,"uf":51,"ug":2486,"uh":17,"hiú":107," tó":85,"ur":1536,"us":8283,"hna":260,"ut":560,"um":955,"un":1808,"uo":20,"hne":600," tí":564,"up":67,"ty":172,"hme":74," sú":61,"tz":64,"hmh":34,"déi":40,"tu":987,"hmi":33,"ик":15,"tt":384,"tw":33,"pó":261," ww":43," só":70,"ий":37,"ub":394,"ua":4295,"ич":35,"ud":441," tá":647,"uc":433,"w ":201,"pú":26,"to":1115,"tn":113,"hul":73,"pé":170,"hui":1613,"tm":76,"tl":375,"hug":337,"ts":494,"tr":2509,"pí":75,"ро":21,"tp":64,"huc":17,"ра":26,"tf":21,"hua":936,"pá":620,"te":7229,"hub":28,"ti":2504,"th":14962,"dó ":211,"v ":84,"oú":35,"tb":247,"tc":62,"ta":12304,"htt":53,"su":1468,"htr":64,"ss":379,"st":5067,"sy":34,"sz":28,"sw":26,"sl":560,"hth":78,"sk":149,"tO":20,"sn":666,"sm":219,"sp":789,"so":753,"hte":89,"sr":416,"tS":312,"oí":60,"sd":35,"sc":3980,"hta":2099,"sf":132,"se":3651,"sh":1228,"tI":37,"sg":76,"si":3838,"rz":27,"u ":794,"hst":33,"nú":375,"tA":118,"sa":7835,"sb":93,"hsp":79,"rr":1332,"ос":14,"ор":16,"rs":1355,"rt":5685,"ru":2167,"rv":57,"nó":3141,"rw":39,"rx":21,"ry":348,"ní":1203,"hse":111,"rp":389,"hsf":34,"ол":16,"ом":15,"hsc":29,"ro":1820,"rn":1077,"он":24,"rm":1323,"né":446,"rl":1840,"rk":221,"nç":15,"hsh":62,"ri":5360,"ов":54,"rh":40,"hu ":168,"rg":994,"rf":169,"hsa":44,"re":8800,"ná":2686,"rd":1927,"rc":1015,"hnú":19,"rb":776,"hru":236,"ra":9135,"t ":8611,"mú":53,"hnó":36,"hné":52,"mó":1228,"qu":91,"hní":184,"но":37,"hro":198,"mí":387,"mé":353,"hre":546,"hná":193,"Dón":21,"nÉ":14,"hri":328,"má":744,"ht ":2941,"на":27,"hra":1213,"dír":63,"s ":29560,"lú":564,"dís":17,"pt":179,"pu":104,"díl":15,"ló":456,"dín":53,"pp":84,"lí":989,"pr":684,"dío":88,"ps":108," ru":1283,"htó":70,"Hum":14," nó":2812,"htú":179," sa":4679," nú":48," tA":117," sf":15," se":1093," sc":1162," si":1192," tI":37," sh":787," tO":20," sn":472," sm":68," sl":198," sr":276," tS":311," sp":393," so":340," mí":271,"dór":16," mó":1033,"ви":37," t ":559," mú":31," ra":690," re":138," ná":1297,"htá":92," ri":660," né":46," ro":398," ní":458," pu":29,"hrí":388," pr":520," lí":138," lú":62,"hró":88," s ":76,"hy ":43,"hrú":276," má":90,"dóc":22," mé":92,"dói":476," nÉ":14," os":81," ot":23,"hum":273,"hun":734,"hus":58,"hut":20,"ав":21,"hur":315," or":279,"ан":35," r ":14,"ай":20," lá":1959," pe":214,"IV ":29,"hrá":475," pa":99," lé":265," pl":119," po":531,"hré":565," pi":69," ph":761," rú":25," sá":66," ró":83," y ":28," sí":561," sé":1980," va":18,"dú ":74," ve":45," rá":141,"zz":17," vo":28," rí":368," vi":31," ré":1371,"Hor":17,"zh":29,"zi":69," ua":370,"ес":16,"ер":17,"Hou":27,"ze":68,"vá":31,"ен":14,"ек":14,"za":84," pó":35," tu":399,"Hol":53," ur":39," um":41,"zo":52,"ví":54," ui":526,"vé":90,"yg":21," ta":838,"ye":100,"uá":49,"yf":14,"yc":51,"yd":69,"ya":136,"yb":15,"tú":630,"huí":244," st":1040,"о́":24," su":1397,"yw":35," pí":25,"ев":18," tr":1087," ts":243,"yt":19," pé":22,"ys":107,"yr":61,"uí":380," to":270," th":2550,"yp":14,"yo":56," ti":480,"yn":135,"ym":43,"ué":20,"yl":73," pá":201," te":1172,"yk":16,"yi":42,"Are":14,"Arc":17,"Ard":151,"Ara":133,"Arm":66,"gCú":110,"ffy":15,"ffe":33,"ffi":21,"bás":179,"fer":28,"faí":35,"feo":21,"fea":451,"bán":27,"As ":53,"bái":111,"fei":332,"fhr":56,"fhu":70,"Ath":59,"Atl":73,"fia":38,"Ast":182,"Ass":25,"fhe":162,"fhi":169,"fho":376,"fha":173,"Art":45,"Asa":26,"gCu":31,"gCr":41,"gCl":15,"gCo":682,"gCi":43,"gCe":68,"gCa":115,"Óg ":20,"Aus":24,"far":64,"fao":645,"fan":33,"Aug":23,"fai":206,"fad":258,"fac":71,"Ógl":34,"ff ":26,"fe ":57,"fa ":140,"Ba ":684,"eys":19,"exa":49,"ez ":54,"ews":23,"ewt":15,"Bai":726,"Bal":86,"Ban":106,"Bay":32,"Bar":140,"Bat":24,"Bas":44,"eta":28,"ete":77,"eti":18,"eth":90,"eso":16,"est":189,"ess":89,"eto":24,"etr":56,"ets":16,"ett":84,"etu":17,"ew ":88,"eve":60,"eva":24,"evo":14,"Aca":46,"evi":62,"eut":17,"Abh":106,"eur":21,"eus":24,"ex ":32,"Ada":23,"Ach":57,"ey ":289,"ewa":22,"epe":15,"eph":65,"er ":853,"epa":17,"eos":35,"eor":359,"Ado":17,"eom":21,"eol":1069,"eon":124,"aíd":33,"es ":655,"Aer":41,"epp":19,"erl":77,"eri":131,"erg":65,"erh":16,"ere":93,"ená":60,"Aga":17,"erf":70,"erc":31,"erd":48,"era":154,"erb":43,"et ":98,"Afr":106,"aít":187,"aío":2153,"aín":34,"aíl":74,"esh":15,"esi":19,"esc":30,"ese":21,"esa":23,"ery":28,"erv":17,"err":112,"ert":175,"ers":225,"ern":128,"erm":66,"erp":23,"ero":49,"eks":14,"en ":1506,"Aib":300,"elb":39,"ela":302,"eld":65,"Aif":16,"Aig":121,"elf":14,"Aid":28,"ele":76,"eli":66,"elm":20,"Aim":21,"Ail":68,"Ain":61,"ell":274,"Ais":61,"elo":57,"Air":154,"Ait":29,"els":60,"elt":152,"Al ":17,"eo ":814,"emb":18,"ema":39,"eme":32,"emo":20,"emi":38,"emp":22,"ene":80,"enh":19,"eng":17,"enb":18,"ena":216,"end":83,"enc":57,"eno":22,"enn":128,"eni":51,"ens":89,"ent":221,"enr":49,"Ala":73,"Alb":377,"aí ":2937,"An ":507,"eoi":2441,"eog":109,"Ali":19,"eod":36,"eoc":94,"Ale":50,"Alt":15,"ego":31,"ege":29,"All":35,"egi":20,"Alp":16,"Amh":81,"Ama":63,"ek ":36,"Ang":71,"eib":67,"eic":489,"Ana":16,"And":78,"eip":74,"eis":2272,"eir":1778,"eim":365,"eil":1930,"ein":197,"Ant":93,"Anr":30,"eid":455,"eig":136,"Ann":43,"eif":17,"Aoi":37,"el ":546,"Aod":28,"eit":1087,"Ar ":54,"Aon":605,"em ":43,"cé ":113,"Bré":15,"Brá":20,"Bus":17,"Bul":20,"Bun":205,"Bur":29,"git":32,"gis":154,"gir":14,"Bui":33,"gil":38,"gip":32,"gin":265,"Bua":24,"gio":95,"gid":19,"gig":26,"gia":23,"ghu":53,"ghr":185,"ght":98,"ghs":14,"hIm":15,"hIn":39,"ghn":371,"hIo":62,"gho":20,"ghi":177,"ghl":361,"ghe":310,"hIa":25,"gha":654,"ghc":42,"ghd":172,"gcú":68,"gcó":255,"gcé":46,"Bru":49,"gcá":19,"gfo":19,"gaí":93,"cán":103,"gen":61,"hFo":15,"geo":88,"ger":72,"hFr":49,"ges":26,"cás":29,"Byr":24,"Buí":18,"gh ":2708,"gea":414,"gei":17,"cái":420,"hFi":19,"gel":59,"cál":28,"hEo":135,"hEa":45,"hEi":18,"gcr":73,"gco":270,"gcl":65,"gcu":132,"gca":128,"ge ":1840,"gci":53,"gch":18,"gce":392,"Brú":19,"gba":47,"gab":28,"gac":522,"gad":1739,"gai":436,"gas":69,"gar":258,"gat":20,"gal":128,"gao":46,"gan":561,"hAf":47,"hAb":16,"öl ":26,"hAs":63,"hAr":24,"hAo":33,"hAn":19,"hAl":173,"hAi":35,"ga ":749,"Cab":19,"Cad":42,"Cae":22,"Cal":103,"Cam":61,"Cai":194,"Cas":82,"Car":213,"Cat":300,"Cao":35,"Can":62,"Cap":27,"Bea":469,"bót":50,"Ber":55,"Ben":44,"Bel":48,"Bei":56,"fré":32,"frá":23,"fur":17,"bói":55,"bón":20,"Át":545,"Ár":101,"Bhi":15,"Bhr":197,"Ái":134,"Bhu":117,"Bho":33,"fud":114,"fua":289,"fun":18,"fui":747,"Bin":21,"Bil":27,"Big":43,"Bir":28,"Bio":15,"ft ":28,"fra":143,"fre":342,"fri":80,"bío":179,"bít":82,"Bha":357,"Bhe":114,"Bhó":20,"for":412,"fos":28,"Bhí":533,"fon":18,"Blo":17,"foi":441,"Bhé":114,"fol":56,"béi":58,"fiú":44,"foc":169,"Bhá":20,"fog":33,"Bli":37,"Bla":57,"fhá":105,"fhé":430,"fhí":42,"Bre":227,"fo ":28,"Bra":117,"fhó":29,"Bro":105,"béa":62,"Bri":217,"fid":30,"fic":105,"fie":34,"fig":207,"fil":119,"fin":52,"fio":69,"fir":44,"fis":60,"fit":18,"Bob":22,"Bol":34,"Boi":21,"Bon":27,"Boo":20,"Bor":28,"Bou":17,"Boy":27,"övs":26,"Ó ":534,"Óg":62,"Ói":16,"da ":574,"dd ":46,"Cur":28,"dbh":268,"Crí":132,"de ":3405,"dac":276,"dad":34,"dal":63,"dai":439,"Cua":50,"dae":36,"dat":54,"das":53,"dar":472,"Cui":121,"dao":417,"dan":122,"dam":218,"Cun":15,"day":18,"Cum":170,"dde":19," Éi":1640,"ddi":20,"ddl":19,"ddy":21,"Ís":59,"Ío":123,"dch":53," Éa":60,"cun":17,"Éa":62,"cul":63,"cum":194,"cui":390,"cua":56,"ctr":133,"De ":104,"cto":75,"cti":62,"cte":57,"cth":78,"cta":42,"dTu":94,"dTr":14,"dTí":21,"Der":45,"Des":14,"Dei":367,"crú":31,"cy ":21,"Dem":25,"Den":24,"Dea":77," Ís":38," Ío":95,"cré":18,"crí":673,"crá":38,"cus":41,"cur":146,"Éi":2322,"cté":51,"Dam":30,"É ":47,"Dan":89,"Dao":73,"Dar":147,"Dav":75,"Dai":34,"Dal":29," Ó ":534,"cks":22,"Chn":17,"Cho":745,"Chr":206,"ài":32,"Che":174,"cki":21,"Chi":387,"Chl":125,"cla":165,"Cia":62,"cle":66,"chá":513," Óg":54,"Chu":175," Ói":14,"cky":14,"ám":250,"chí":120,"án":4189,"Cin":46,"áp":43,"Cio":16,"Cit":56,"ái":8489,"clu":187,"chó":321,"ál":1018,"áe":16,"cli":29,"áf":24,"ág":444,"Cil":132,"áb":389,"ché":598,"ác":287,"clo":44,"ád":259,"cme":28,"cmh":19,"co ":63,"ár":1109,"chú":264,"át":991,"ás":613,"ció":20,"cni":17,"cne":34,"cna":87,"ciú":86,"DN ":19,"ão":15,"cnu":17,"coi":647,"cod":26,"Cla":68,"cog":48,"cob":20,"con":157,"col":164,"com":586,"cor":131,"cos":144,"cot":56,"cou":17,"clá":107,"Cea":371,"clé":67,"Cei":73,"clí":17,"Cel":18,"cló":73,"Ceo":53,"Cen":23,"clú":142,"cs ":59,"cmí":15,"ct ":18,"cre":91,"cná":17,"cra":247,"cri":85,"á ":8021,"cru":138,"cro":73,"cu ":204," Úc":26," Úi":16,"dTe":20,"cse":49,"csa":161," Ús":41,"Cha":583," Úr":42,"csi":66,"Clú":19,"Cló":22,"Cri":31,"cea":2323,"Cra":92,"Cre":44,"Ús":41,"Úr":44,"ch ":8976,"cer":23,"ces":38,"ceo":483,"cen":28,"Úc":31,"Úd":15,"caí":154,"cel":32,"Úi":18,"Cru":51,"cei":451,"Cro":66,"ci ":143," á ":180,"Co ":17,"Chú":59,"chb":68,"cha":3675,"chd":28,"Cli":497,"Clo":33,"Ché":62,"Chó":43,"Clu":118,"chu":2165,"Ciú":36,"cia":133,"ck ":201,"ceá":740,"cie":25,"che":1628,"chl":544,"chi":421,"cho":1057,"chm":18,"chn":85,"chs":16,"cht":5396,"chr":402," ár":46,"Coi":126,"Cog":108,"cil":150,"Cno":22," áb":129,"cim":18,"cig":39," áf":23,"cir":45,"cis":192," ái":498,"cit":27,"ciu":24,"cin":312,"cio":113,"Cnu":23,"cip":31,"cke":40,"Clá":26,"Cos":39,"Cor":193,"Com":288,"Col":188,"Coo":21,"Con":1012,"Cou":24,"ed ":132,"ós":524,"ót":285,"eba":20,"óv":19,"ebe":20," é ":6560,"ec ":22,"eac":4120,"eab":592,"eag":1122,"ób":33,"eaf":21,"eae":15,"ead":1421,"eai":31,"ean":6477,"ói":3006,"eal":1578,"óg":454,"eam":933,"ear":3955,"eas":1483,"eap":208,"ód":167,"óc":115,"ór":1746,"óp":56,"eat":955,"eau":27,"ón":1301,"óm":776,"ól":143,"íú":73,"ea ":2266,"ó ":5058,"eff":24,"ei ":24,"ega":26," í ":1887,"een":63,"eel":21," éi":443,"eed":16,"eer":16,"eaí":20,"eet":19,"edh":19,"edi":38,"ín":1128,"ím":31,"edd":25,"ede":40,"ío":6870,"ír":1028,"ít":709,"ís":504,"eg ":16," éa":781,"edy":19,"éú":20,"edo":20,"edr":22,"ech":43,"ee ":87,"íg":26,"íf":18,"íl":526,"ect":37,"íc":229,"íd":182,"ía":41,"eco":18,"íb":395,"dtú":90,"ës":14,"dtá":26,"í ":10858,"dté":16,"dtí":383," ís":30," ío":64,"dwi":23,"él":14,"éi":5025,"dró":22,"éo":14,"ép":16,"én":18,"és":18,"dwa":33,"ét":14,"ér":37,"dy ":84,"ë ":21,"drá":86,"éb":23,"éa":6775,"ée":17,"Edw":27,"ég":14," ór":14," ón":600," óg":50," ói":41,"é ":9003,"Ear":38,"Eas":45,"Eal":32,"Ean":323," ó ":1226,"Eag":152,"áí":25,"Eac":31,"Eab":120,"FC ":14,"dor":65,"don":416,"dom":343,"dol":27,"dos":18,"Dhú":133,"dlí":99,"Diú":26,"ds ":70,"dlú":36," ós":29,"dió":66,"diú":99,"doc":36,"dog":20,"doi":23,"Dhá":14," úd":122,"Dho":65,"Dhe":46,"dso":14,"Dia":55," ús":341,"dto":27," úr":105,"Dic":15,"dti":49,"dte":287,"Dhu":42,"dta":63,"Dis":41,"Dir":17,"Din":17,"Dil":14,"dtu":328,"dtr":51,"Die":22,"dun":14,"dui":527,"dul":80,"dub":16,"dua":69,"duc":14," ú ":1012,"dri":103,"dra":170,"dre":124,"du ":18,"dro":93,"dru":42,"Dha":18,"Dub":47,"Dua":39,"dha":260,"dhb":24,"Dun":28,"Dui":58,"dge":49,"ün":29,"ür":27,"Drá":15,"dhu":100,"dic":27,"dia":293,"dhi":80,"dhe":442,"dhr":104,"dho":62,"dhn":28,"dhm":310,"dhl":50,"daí":220,"der":172,"des":36,"dh ":5455,"dfa":52,"dea":701,"Dro":81,"dei":449,"del":62,"den":1153,"deo":76,"úp":306,"ún":1833,"úm":23,"úl":626,"di ":53,"út":197,"ús":719,"úr":674,"úi":3261,"úg":21,"úd":355,"úb":49,"úc":437,"dle":41,"dhá":244,"dla":24,"öv":27,"Dlí":35,"dmh":31,"dhú":132,"do ":629,"Dre":21,"Dra":14,"dhó":20,"dhí":53,"dhé":348,"ú ":2631,"Doi":47,"Dob":22,"dim":32,"din":294,"dio":40,"dir":1209,"dis":147,"dit":21,"deá":44,"die":30,"dif":40,"dig":48,"dil":18,"ör":19,"öl":32,"Dr ":17,"Dou":14,"Don":78,"Dom":200,"Dor":64,"rdá":19,"órá":63,"Nea":38,"rha":17,"Nei":30,"rdí":26,"rbó":47,"rga":146,"ri ":89,"rgi":106,"óva":19,"rgh":71,"rcá":36,"rge":229,"rgt":18,"rgn":54,"nát":222,"ret":35,"res":104,"rew":25,"rey":30,"óth":174,"rfa":28,"óta":81,"rfh":18,"rfi":21,"rfo":63,"rdu":18,"Nas":14,"Nat":55,"rds":35,"Nav":15,"rg ":211,"rea":4475,"ósa":37,"ree":55,"nác":45,"rec":16,"red":50,"nád":57,"nái":868,"rei":578,"reg":26,"rem":34,"nám":42,"ren":116,"nán":213,"óst":232,"rek":22,"nál":147,"rel":65,"raí":921,"rer":16,"nár":33,"reo":588,"óra":749,"Nig":14,"rda":143,"órc":14,"Nic":75,"Nia":22,"rdo":44,"órt":164,"órs":81,"rdi":66,"Nio":18,"rdh":169,"rdc":33,"rde":172,"ós ":247,"re ":2165,"ná ":1044,"ómá":33,"rbu":15,"rbr":21,"rco":14,"rcl":38,"rci":49,"rch":211,"rce":106,"rca":315,"ópa":18,"ór ":620,"rd ":1105,"rao":356,"rap":41,"rar":34,"ras":586,"rat":223,"rav":21,"rbi":53,"rbh":278,"rbo":26,"rba":184,"rbe":37,"rc ":189,"New":90,"rai":2238,"rah":38,"rag":65,"ran":640,"ram":134,"ral":89,"rak":15,"rab":86,"raf":72,"rae":116,"rad":543,"rac":1041,"rlí":86,"rpo":23,"rs ":166,"rlá":17,"rpe":23,"rpa":218,"rr ":255,"rph":42,"ros":108,"rot":96,"rom":262,"ron":135,"roo":19,"rop":32,"rou":48,"rov":34,"row":44,"rod":23,"roc":87,"ní ":309,"roi":637,"rol":45,"rok":18,"rog":54,"rno":16,"rnr":15,"rns":15,"riú":336,"rp ":54,"rna":252,"rne":186,"rni":111,"rml":62,"rmo":22,"rmu":15,"ro ":90,"rma":359,"néa":182,"rme":66,"rmh":95,"rmi":88,"néi":221,"Nao":110,"Nap":16,"Nai":48,"rlo":46,"rli":111,"rle":214,"rla":1273,"rn ":133,"rku":14,"rgí":14,"rks":24,"rki":19,"rgá":88,"rke":35,"rka":14,"né ":24,"rm ":191,"Iúi":305,"ótó":16,"Na ":69,"riz":14,"rl ":40,"rip":32,"rio":414,"rit":613,"ris":438,"riu":30,"rig":165,"ril":27,"rik":16,"rin":795,"rim":139,"rdú":27,"ria":919,"rib":71,"ric":942,"rid":144,"reá":343,"rie":109,"rif":21,"rk ":90,"nóg":29,"óg ":175,"rsá":56,"nól":18,"nói":152,"rwi":20,"nón":22,"rsé":16,"nót":37,"nós":133,"Nua":266,"rté":20,"rtí":355,"rtú":33,"rui":134,"rug":1110,"ruf":14,"rud":172,"ruc":68,"run":22,"rum":55,"óch":56,"rus":59,"rut":246,"óca":32,"rva":21,"rrá":17,"rve":15,"óda":35,"nóc":14,"nód":16,"ry ":291,"rrú":28,"ódh":24,"rsk":16,"rsi":104,"rso":40,"rsm":24,"rsc":217,"rsa":287,"rsh":40,"rse":211,"rta":707,"rst":140,"roí":25,"rtl":15,"rto":55,"ód ":76,"rte":314,"rth":1771,"rti":96,"nó ":2699,"rua":226,"rts":27,"rtr":15,"óbh":15,"rty":31,"rmé":23,"nín":101,"níl":49,"rmá":327,"níc":30,"rt ":2182,"rmó":19,"nít":71,"nío":623,"rné":17,"rro":35,"rri":84,"rre":71,"rná":209,"rra":630,"rry":95,"rrt":19,"rnó":52,"rní":19,"rrs":58,"sc ":327,"sab":21,"sac":654,"sad":54,"sae":23,"sag":75,"sai":592,"sal":68,"sam":141,"tAo":24,"ómh":688,"óma":33,"ónt":14,"sbh":16,"ónn":18,"sao":275,"óng":28,"san":1622,"sat":24,"sas":19,"sar":67,"óna":427,"óp ":28,"sa ":3740,"núi":159,"ógá":48,"tAb":14,"tAi":42,"nús":88,"óla":44,"nún":35,"ón ":775,"óim":116,"óil":212,"óig":90,"óis":269,"óir":1464,"óit":73,"óin":227,"nú ":75,"óip":36,"ódó":17,"óib":55,"óic":15,"óid":394,"ól ":82,"óiv":47,"ógr":30,"óga":179,"Nor":78,"ógt":15,"Nol":338,"Nob":19,"sha":314,"scó":82,"sho":57,"shr":63,"she":203,"shi":156,"shl":100,"si ":38,"scé":347,"scí":15,"scá":85,"sgh":17,"sfé":91,"seá":101,"sie":17,"sid":18,"sic":138,"sib":26,"sia":514,"sk ":19,"shu":50,"sit":136,"sir":78,"sis":198,"sin":1068,"sio":159,"sil":94,"sim":57,"sig":443,"scr":521,"sct":40,"scu":23,"Oid":16,"Oib":81,"Oif":26,"Oil":208,"Oir":93,"sbu":25,"se ":1138,"sca":692,"sce":1106,"sci":152,"sch":129,"sco":383,"scn":26,"scl":50,"sey":40,"ser":32,"set":23,"sh ":132,"sea":1145,"sei":128,"see":18,"saí":406,"sep":38,"seo":837,"sen":41,"sel":32,"slé":30,"slí":29,"spr":39,"sph":19,"slá":31,"spe":143,"spl":89,"spi":78,"spa":83,"sol":68,"son":394,"sor":16,"sof":18,"soi":82,"oí ":21,"soc":67,"sru":24,"tSu":38,"sné":54,"tSl":17,"tSi":42,"tSe":104,"sná":21,"sre":22,"tSa":39,"sra":197,"st ":494,"oíc":18,"ss ":98,"sli":105,"shí":50,"shó":29,"slu":19,"sgö":26,"sky":14,"sla":192,"shá":17,"sle":131,"ski":48,"ska":40,"sna":496,"siú":811,"sne":60,"smu":24,"so ":31,"sma":98,"smh":17,"smi":37,"swi":14,"stí":140,"sté":91,"stá":507,"stú":37,"stó":65,"suí":51,"Owa":15,"szo":14,"sse":71,"ssa":61,"sso":60,"ssi":66,"íú ":73,"spá":57,"ste":1218,"sta":1246,"spé":48,"stm":17,"sto":201,"sti":488,"stl":22,"stu":43,"spó":185,"str":448,"sua":60,"sub":21,"sui":1192,"sul":46,"sun":29,"srá":131,"tSí":34,"sy ":15,"tai":2305,"tal":289,"tae":1311,"tag":109,"tab":44,"tac":773,"tad":69,"Oll":155,"tbh":212,"tba":28,"tat":53,"tas":487,"tar":1806,"tap":16,"tao":264,"tan":292,"tam":111,"tch":56,"Ont":15,"te ":2398,"oú ":35,"가":17,"Ope":20,"Ora":17,"ta ":3767,"Osc":19,"Ord":25,"Org":16,"Ost":52,"Osp":25,"Léi":39,"Plu":14,"pa ":431,"Phá":131,"Pla":56,"Pin":38,"Pil":15,"Pir":16,"Pie":43,"Pic":15,"Pia":16,"Phl":25,"Pho":215,"pch":27,"Phi":49,"Phr":41,"Phe":38,"lá ":1233,"pe ":40,"Pha":68,"par":151,"pat":36,"pas":37,"pac":105,"pad":34,"pag":31,"pal":41,"pai":149,"pan":23,"Pea":55,"phe":133,"pha":75,"Lár":36,"Per":28,"Pet":70,"Pen":31,"phr":237,"Pei":90,"pho":285,"Lái":95,"phl":74,"phi":39,"pi ":15,"ph ":45,"Lá ":25,"pea":396,"Pat":75,"Pas":17,"Par":135,"Pau":68,"pen":39,"lán":235,"lám":61,"per":79,"lár":422,"paí":59,"lát":214,"lás":33,"lái":556,"pei":268,"pel":21,"pla":221,"ple":113,"phá":114,"phí":17,"Pac":17,"phé":19,"Pan":18,"phó":39,"Pai":26,"Pal":24,"lé ":37,"phy":24,"pia":37,"pid":51,"pic":45,"peá":30,"pin":38,"pio":53,"pir":179,"pit":35,"por":56,"RA ":16,"pop":47,"pos":23,"poi":71,"pol":359,"pob":89,"ps ":19,"plí":36,"ppi":17,"plé":24,"plá":39,"ppe":27,"léi":437,"léa":199,"lí ":340,"Lú ":20,"pta":38,"pso":42,"ló ":37,"pub":20,"pte":41,"pti":15,"pth":22,"pto":20,"Pró":17,"lís":63,"lít":64,"pra":40,"pt ":15,"lío":233,"lín":268,"Prí":87,"pri":56,"pre":46,"pro":19,"lór":25,"lóv":14,"lóg":58,"lói":259,"lón":22,"Pro":75,"pus":14,"pun":14,"Pri":61,"Pre":38,"prá":19,"prí":405,"pró":88,"lú ":248,"Lí ":19,"Pob":269,"Pol":146,"Poi":14,"Pot":15,"Por":97,"Pow":27,"lún":59,"lút":57,"lúd":46,"lúc":14,"lúi":104,"Mí ":149," ال":36,"má ":30,"mát":17,"más":38,"mán":258,"mái":372,"Mái":98,"Már":312,"nÉi":14,"RC ":57,"Lún":278,"Lút":94,"Má ":17,"méi":105,"méa":226,"mí ":94,"mít":19,"míc":15,"mín":39,"mío":44,"míl":152,"Rac":22,"Rai":49,"Ram":22,"Ran":22,"Mün":20,"mó ":604,"Qué":14,"que":47,"qui":28,"móg":26,"mói":77,"mór":457,"món":35,"Qui":16,"Que":32,"Mói":32,"Mór":89,"Míc":34,"múi":17,"ra ":1834,"rb ":96,"ngn":22,"Isa":18,"ngo":25,"ngi":39,"ngl":144,"ngu":44,"ngr":50,"ncí":19,"ngt":83,"ngs":41,"ni ":60,"Iri":72,"nge":211,"ngf":22,"ngh":86,"nga":974,"ndí":14,"ndé":38,"ndó":18,"nha":48,"Isl":17,"ndá":65,"neg":14,"nei":19,"Is ":5111,"nel":42,"nen":32,"neo":447,"ner":75,"naí":1112,"net":45,"nes":120,"ndy":16,"ng ":349,"nea":1340,"ned":36,"nfh":67,"Ire":45,"ney":97,"nfa":14,"nfe":14,"nco":31,"nci":177,"nce":204,"nch":428,"nca":121,"ne ":3335,"ndu":14,"ndr":118,"nds":22,"ndo":64,"ndl":20,"ndi":217,"nde":329,"nda":522,"ncy":16,"ncu":14,"nal":159,"nam":348,"nan":389,"nao":175,"nap":14,"nar":134,"nac":2846,"nad":430,"nag":54,"nai":1426,"nc ":149,"nab":27,"nbe":25,"nbh":87,"nd ":363,"nat":73,"nas":642,"na ":12179,"iúd":104,"iúc":163,"iún":891,"iúl":283,"iúi":885,"iúr":175,"iút":14,"iú ":460,"Jac":51,"Jar":14,"Jan":31,"Jam":111,"msí":64,"ión":27,"ntú":67,"Jer":36,"nyi":30,"nté":57,"ntí":102,"Jea":24,"Fái":49,"ntó":92,"nz ":20,"ntá":73,"nsí":44,"nrú":15,"ny ":111,"nvi":15,"nró":30,"nrí":45,"nui":29,"nus":32,"nua":572,"nty":14,"nto":113,"ntu":65,"Jim":41,"ntr":159,"nti":241,"nth":35,"nta":3920,"nte":565,"nsp":24,"nso":37,"nst":156,"nse":204,"nsh":22,"nsi":157,"nsl":20,"nsk":16,"nsc":131,"nsa":161,"nnú":15,"nry":35,"nní":68,"nné":19,"nri":70,"nná":276,"nre":31,"nra":389,"nt ":782,"Féi":77,"nph":26,"ns ":220,"noc":72,"nod":15,"nol":15,"noi":128,"nom":39,"non":20,"nos":23,"nor":35,"nov":43,"Féa":15,"nne":1006,"nnf":20,"nna":2785,"nnc":59,"nnl":27,"nnm":15,"nno":36,"nni":557,"nnu":27,"nnt":268,"nns":21,"nnr":29,"nny":46,"nma":16,"nmn":111,"nmh":268,"niú":176,"nli":38,"nn ":8140,"nla":210,"nle":19,"no ":106,"ngá":45,"nm ":546,"ngó":117,"ngé":45,"Jos":64,"Jon":27,"nja":20,"ア":18,"Joh":203,"Joe":27,"Jr ":24,"Joy":18,"nig":486,"nie":54,"neá":219,"nid":36,"nic":278,"nia":209,"ndú":111,"nk ":33,"Joa":15,"niu":167,"niv":27,"nis":728,"nit":168,"nir":17,"nio":42,"nim":85,"nin":41,"nil":25,"ogr":39,"ogh":289,"ogl":14,"oga":398,"oge":56,"oi ":566,"odó":26,"ohn":167,"oha":45,"odá":193,"ohe":14,"ois":848,"oir":3309,"oit":255,"oin":1191,"oim":354,"oil":1296,"oif":183,"oig":99,"oib":236,"oic":136,"oid":224,"ok ":22,"ofá":17,"Jul":20,"ol ":417,"och":2829,"oci":28,"ock":58,"ocl":57,"ocr":39,"ocs":92,"oe ":46,"oca":254,"odg":15,"odh":743,"ode":33,"odo":22,"odr":47,"Fóm":579,"of ":150,"oda":77,"og ":142,"oft":18,"obá":17,"ofa":122,"oa ":16,"ob ":24,"oc ":64,"oan":17,"oad":17,"oba":305,"od ":80,"obr":83,"obl":440,"obh":750,"obi":26,"obe":98,"ة ":18,"Kan":19,"Kat":31,"Kar":34,"Kev":15,"Key":15,"oyd":15,"oya":32,"Ken":62,"Kel":29,"Kea":20,"otá":130,"osú":142,"owy":22,"osó":14,"ows":15,"osé":18,"LG ":24,"own":87,"oyl":19,"Kin":36,"otu":20,"ow ":44,"oth":323,"ott":66,"ots":23,"Kil":36,"otr":21,"oto":30,"ost":375,"ota":316,"otb":27,"ov ":33,"osh":28,"ose":75,"éú ":20,"osf":14,"osp":42,"oss":42,"osr":51,"osl":98,"oso":32,"osn":18,"oró":83,"oy ":46,"owa":27,"osá":17,"owe":32,"ovi":50,"oré":21,"ox ":19,"ova":45,"orá":51,"ove":70,"oug":50,"oui":57,"oul":26,"oun":65,"ous":31,"our":113,"out":79,"opo":21,"ítí":24,"ope":29,"olá":174,"oph":46,"opa":58,"opc":14,"olú":33,"os ":781,"oló":41,"opt":22,"oon":34,"ool":43,"oom":21,"ísí":16,"ook":30,"ood":67,"or ":345,"Khö":26,"oot":38,"oos":16,"oor":30,"ork":35,"orl":75,"orm":172,"orn":219,"oro":53,"orp":272,"orr":167,"orc":278,"ord":273,"oná":62,"ore":96,"orf":22,"org":242,"ori":126,"onú":23,"ou ":33,"osa":407,"osc":122,"ort":590,"ors":62,"oru":46,"onó":59,"ory":25,"omá":246,"ot ":44,"orb":118,"ora":509,"omó":111,"ola":1711,"old":55,"ítr":15,"olc":90,"on ":2106,"íth":89,"oli":70,"oll":644,"íti":59,"olf":34,"ole":55,"olg":25,"olr":30,"ols":26,"olt":377,"olm":81,"olo":58,"olu":29,"Kra":14,"íst":16,"om ":147,"ísh":22,"ísl":16,"oke":30,"ogá":20,"íte":478,"ona":897,"ond":268,"onc":148,"onf":22,"one":117,"ong":260,"oni":64,"onl":192,"onn":1898,"onm":22,"ono":40,"onr":302,"ons":354,"ont":2217,"ony":50,"oma":307,"ome":52,"omc":38,"omb":131,"omi":43,"omh":2862,"omm":49,"oml":86,"omp":156,"omn":22,"omo":20,"omr":44,"oms":14,"op ":47,"la ":2317,"ímh":16,"íne":254,"íni":119,"ínt":26,"ín ":561,"íle":271,"íli":20,"ílt":21,"le ":6126,"há ":229,"lca":35,"ít ":29,"lch":115,"lf ":34,"íní":128,"íre":446,"lde":34,"íri":75,"lda":20,"ldo":21,"íse":228,"ldi":15,"ldr":25,"íon":1757,"íol":105,"íom":1097,"íof":67,"lc ":14,"íog":60,"mBa":252,"íob":247,"lab":207,"íoc":2004,"lac":1652,"íod":679,"lad":283,"laf":29,"mBe":30,"lae":75,"lah":15,"lag":166,"lai":2053,"ír ":462,"mBl":19,"lan":820,"lam":183,"lap":14,"lao":133,"mBo":21,"mBr":58,"íos":602,"lar":224,"íor":199,"lat":121,"íot":31,"las":488,"lau":20,"mBu":18,"lav":45,"lay":20,"lba":399,"ld ":134,"lbe":35,"lbh":90,"ís ":182,"lbo":37,"gói":147,"Les":14,"Leo":49,"Len":15,"ky ":41,"Lei":85,"Ísi":55,"Lee":32,"Lea":169,"ífh":16,"ích":119,"íci":16,"Lau":16,"íce":62,"gó ":16,"íde":34,"Le ":36,"göl":26,"Lai":234,"Lat":22,"Lar":18,"Íos":108,"Lao":78,"Lan":50,"Lac":25,"íl ":179,"Lab":43,"Íoc":14,"gól":15,"gór":363,"ídí":15,"La ":40,"llí":62,"llá":36,"lph":21,"llú":34,"ls ":42,"híc":20,"lló":16,"lon":282,"lom":21,"loo":18,"lor":72,"loc":108,"log":67,"loi":139,"los":46,"lot":18,"lou":14,"low":25,"loy":15,"hí ":1705,"lmo":15,"héi":849,"lme":15,"lmh":99,"héa":1120,"lma":25,"liú":299,"ML ":17,"lth":18,"lti":78,"Lib":24,"lto":55,"Lia":109,"íd ":117,"Lif":21,"ltr":42,"Lim":18,"Lin":50,"Lio":85,"hó ":54,"Lit":21,"luc":142,"Liv":18,"Leó":14,"lub":148,"lua":385,"íbh":382,"lsi":100,"íc ":20,"lso":25,"lst":34,"lta":1256,"ltb":209,"lte":433,"lse":63,"lsc":185,"lsa":41,"híl":84,"ía ":22,"hír":114,"hís":67,"hín":94,"hío":369,"lt ":143,"lra":80,"lhe":14,"Lui":93,"Lua":43,"Lud":14,"Luc":96,"Lug":29,"lge":761,"lcá":58,"lgh":15,"li ":52,"lga":54,"lfr":17,"lfo":14,"lbá":32,"lfa":28,"ley":85,"lex":35,"lev":17,"les":180,"hás":82,"let":44,"hát":26,"laí":1250,"hár":49,"ler":93,"leo":208,"háp":15,"lem":28,"mBé":150,"len":233,"hán":441,"lek":16,"hál":49,"hái":1498,"lei":1619,"hág":33,"leg":26,"hád":25,"lec":18,"háb":67,"lea":1609,"lg ":39,"lls":203,"llt":63,"lly":96,"lo ":74,"lla":935,"llc":15,"lle":424,"lli":518,"llm":62,"llo":69,"ln ":15,"lgá":15,"lgé":23,"Lou":71,"Lov":14,"Los":30,"lm ":60,"Loi":15,"ll ":1866,"Loc":214,"Lor":30,"Lon":181,"lit":361,"lis":292,"lir":387,"lip":69,"lio":164,"lin":417,"lim":385,"liz":19,"liv":28,"liu":30,"lic":201,"lid":19,"lia":3592,"lib":16,"lk ":16,"lil":22,"lig":150,"leá":626,"lie":64,"lif":258,"ma ":472,"húc":86,"húd":16,"húi":311,"húl":55,"hún":169,"hús":42,"húr":40,"mac":612,"mai":598,"mad":253,"mae":32,"mag":35,"mar":1718,"mas":175,"mal":86,"mam":24,"man":516,"mao":49,"mat":79,"mba":240,"mbl":503,"mbi":32,"mbe":81,"mbr":65,"mbo":25,"me ":182,"nDa":34,"nDe":15,"mbu":66,"nDo":14,"mch":67,"mdh":15,"mea":458,"met":27,"maí":234,"mes":142,"mer":79,"meo":30,"men":88,"mei":43,"nGe":21,"nGa":292,"mh ":1936,"Lyn":18,"mbó":35,"nGr":52,"mbí":90,"mbé":18,"nGl":33,"lva":29,"lve":18,"lvi":21,"lui":189,"lun":17,"lum":19,"lus":40,"Mei":735,"ly ":90,"Men":20,"Mel":55,"Mer":24,"hód":18,"hóg":71,"hói":282,"Mea":58,"Med":17,"lsú":44,"hór":401,"lsí":98,"hón":68,"hót":81,"hós":139,"ltú":88,"ltí":65,"ltó":312,"lvé":32,"lyw":29,"lym":15,"lyn":33,"luí":23,"Man":130,"Mao":38,"Mal":52,"Mar":371,"Mas":30,"Mag":40,"Mad":44,"Mak":15,"Mah":18,"Mai":143,"höv":26,"Mac":269,"McG":26,"McC":34,"hú ":144,"Mau":18,"Mat":68,"mpi":170,"mph":15,"mlá":60,"mpe":309,"mpr":28,"mlí":34,"mpl":157,"mps":56,"mpt":19,"ms ":24,"Mhá":43,"moi":28,"mod":31,"mon":137,"mol":22,"mor":39,"Mhí":40,"mot":24,"mou":21,"mpa":114,"Mhó":86,"mná":23,"mre":320,"Mol":14,"Mon":181,"Moo":22,"mní":56,"Mos":37,"Mor":60,"Mou":21,"msa":14,"mse":115,"mra":22,"Mik":20,"Meá":332,"Mid":25,"Mic":162,"mth":59,"Mit":15,"ió ":67,"Mis":38,"Mil":49,"Min":33,"Mio":27,"Mhi":56,"mst":20,"Mhe":373,"Mha":337,"msh":17,"msi":78,"mpá":18,"Mhu":157,"Mho":39,"mrí":232,"my ":60,"mur":22,"mus":69,"mui":145,"mun":25,"mrá":27,"mho":79,"nIo":16,"mhp":53,"mhr":513,"mhl":221,"mhn":220,"mhs":290,"mht":62,"mhu":94,"Mun":14,"nDú":32,"Mul":17,"mhb":105,"mha":2867,"Mum":29,"Mui":108,"mhg":44,"mhi":314,"mhd":145,"Mur":63,"Mus":20,"mhc":392,"mhf":40,"mhe":469,"mi ":37,"min":375,"mio":137,"mil":143,"mir":224,"mis":103,"mit":76,"mic":290,"mia":46,"meá":129,"mie":35,"mid":17,"mhú":61,"mo ":38,"mhó":274,"mhí":205,"mhé":50,"mle":33,"mhá":463,"mla":28,"mni":119,"nOi":18,"mna":22,"mne":64,"mmy":33,"miú":91,"mmi":21,"mmo":23,"mma":33,"mme":29,"Téa":54,"Tái":16,"Tá ":2060,"Súi":16,"Sói":47,"кий":26,"Sín":135,"Sío":26,"Wor":21,"Woo":16,"Wol":21,"Séa":149,"Whi":17,"Wik":16,"Wil":138,"Win":32,"TÉ ":39,"Sé ":26,"Sái":26,"Wes":45,"Rúi":254,"Was":16,"War":22,"Wal":73,"Róm":117,"Rói":56,"éi ":35,"vír":30,"éig":552,"éif":26,"éid":461,"éic":46,"éib":77,"éin":788,"éim":366,"éil":829,"éiv":16,"éit":178,"éir":1174,"éis":462,"éip":14,"ée ":15,"Vol":22,"Rí ":158,"Río":178,"ébe":15,"éal":1141,"éam":173,"éan":937,"éar":2079,"éas":293,"éad":1071,"éag":793,"éac":146,"éab":116,"Ré ":18,"Réi":59,"Réa":133,"zi ":14,"vái":19,"zer":17,"ze ":17,"zab":15,"zar":14,"zon":14,"Áth":545,"ال":39,"véa":37,"véi":51,"Ára":86,"之":46,"Áit":16,"Áis":79,"Áir":27,"yst":15,"三":49,"丁":24,"uío":293,"ys ":50,"yon":20,"uí ":65,"za ":23,"Uíb":33,"ywo":25,"ye ":18,"yce":17,"ydd":20,"uái":30,"yes":20,"yer":38,"uán":14,"ya ":45,"tún":49,"túr":132,"tús":176,"túi":132,"You":19,"yd ":20,"Uí ":79,"yan":35,"yal":16,"yn ":41,"yle":30,"uéb":14,"ymo":15,"yne":26,"на ":23,"yin":31,"Tóg":25,"tín":149,"tío":154,"tíl":50,"tír":606,"téi":199,"téa":299,"tí ":809,"ن ":19,"Tír":114,"tú ":120,"tói":556,"tóg":93,"tón":40,"tór":36,"tán":115,"tár":64,"tát":339,"tád":22,"táb":174,"tál":25,"tái":738,"Stá":384,"té ":28,"Srá":139,"Sut":21,"Sur":14,"Sum":35,"Sul":19,"Sun":21,"Sui":19,"Sua":83,"Str":47,"Stu":20,"Sti":47,"Sto":43,"Sta":139,"Ste":88,"Spá":239,"sún":98,"súl":153,"súi":176,"súr":33,"Tea":279,"Pád":61,"Ten":26,"Páp":16,"Teo":19,"Tei":25,"Pái":139,"Tel":19,"ão ":15,"Tao":30,"Tam":21,"Tan":19,"Tar":51,"sú ":27,"Tai":37,"Tal":28,"Tag":65,"tá ":2371,"xas":26,"xan":25,"ww ":43,"Shi":41,"Shl":44,"She":97,"Shr":17,"Sho":28,"Sha":212,"Sim":55,"Sil":27,"Sir":43,"Sio":65,"Sin":55,"Seá":101,"Sib":15,"Sic":20,"Sia":23,"wys":21,"wyn":15,"Ser":18,"Sev":15,"Scé":14,"Sgi":14,"www":43,"sór":14,"TG ":26,"sói":127,"Scr":150,"Seo":77,"оно":15,"Sei":86,"Sea":356,"Sra":58,"séa":65,"séi":59,"Sru":15,"TV ":15,"Spi":26,"Spe":25,"sé ":1940,"Spr":18,"Spo":16,"Slé":27,"Sló":23,"Oíc":14,"wn ":71,"St ":18,"síc":29,"ws ":28,"sít":28,"sío":289,"sín":66,"Soc":18,"séú":20,"Sou":43,"Sol":26,"Som":19,"Son":17,"Sor":26,"Sla":28,"sí ":546,"Shó":16,"wor":22,"woo":34,"Shé":39,"Sli":87,"Smi":23,"sás":15,"wer":23,"sár":23,"sán":45,"wen":17,"sál":14,"wel":39,"sái":497,"Rya":15,"sác":17,"Éad":37,"Rus":14,"Rug":352,"wic":33,"Rua":57,"win":29,"Sag":39,"Sai":92,"Sam":333,"Sal":39,"TD ":24,"Sac":45,"Sab":15,"Sco":64,"Sci":16,"Sch":44,"Sca":96,"Sat":14,"Sar":19,"Sas":352,"San":139,"Sao":103,"rú ":278,"rúi":83,"rún":65,"rús":14,"rúp":255,"Éid":18,"Éig":34,"Éis":16,"Éir":2239,"wa ":20,"rúc":52,"rúd":20,"wan":20,"wal":16,"ови":23,"way":37,"war":66,"Sa ":139,"wai":21,"rís":49,"rít":177,"rír":17,"río":1887,"rín":63,"ríd":89,"ríc":21,"ríb":349,"ном":14,"нов":15,"vsg":26,"Rio":16,"Riv":17,"Rit":19,"ró ":41,"Ris":24,"Rin":78,"Ria":85,"Ric":56,"ríú":71,"Rat":26,"Ray":17,"rót":49,"rós":27,"róc":18,"ród":40,"róg":49,"ról":47,"rói":314,"róp":18,"róm":19,"rón":46,"Red":20,"Rei":34,"Nái":100,"Reg":14,"Ren":19,"Rep":16,"Rea":41,"via":20,"vil":33,"vin":52,"vic":53,"vid":56,"vie":19,"Roi":64,"Rog":14,"Roa":15,"Rob":88,"Roc":23,"Rod":16,"Ní ":122,"vit":24,"vis":41,"Roy":25,"Ros":122,"Ron":28,"ré ":111,"Níl":43,"réa":1091,"réi":1345,"rí ":1171,"vol":16,"von":30,"Néi":23,"ver":165,"ves":15,"rás":53,"rát":69,"vei":42,"rái":750,"rán":608,"ven":46,"rám":112,"vel":29,"rál":140,"rád":32,"rác":51,"ve ":67,"rá ":172,"val":28,"van":117,"var":26,"vac":33,"vad":16,"vai":30,"ски":27,"va ":51,"Van":14,"Val":47,"Vic":66,"Vir":27,"Vil":15,"Vin":17,"Ver":42,"Rát":50,"ux ":34,"uve":14,"usk":14,"ush":22,"use":38,"usc":28,"usa":33,"unú":106,"ust":89,"uss":40,"usp":33,"uth":369,"uti":18,"uta":20,"Uil":36,"utt":29,"us ":7938,"Ung":31,"Uni":93,"ut ":27,"urb":96,"ura":189,"urc":62,"ure":45,"urg":90,"uri":53,"url":38,"urn":75,"uro":30,"urp":15,"urr":39,"urt":87,"ury":23,"Ula":97,"ur ":612,"umh":128,"umm":17,"uma":512,"umb":45,"ume":17,"unt":128,"uns":24,"unr":31,"uni":30,"unn":30,"unc":40,"und":110,"una":569,"ung":63,"up ":17,"um ":194,"ulu":22,"ult":111,"ulo":15,"ull":75,"uli":38,"ulg":25,"ule":17,"ulf":16,"ula":104,"un ":599,"uid":1064,"uig":163,"uil":876,"uim":340,"uin":941,"uip":23,"uir":1098,"uis":276,"uic":175,"uib":54,"uit":1252,"ul ":152,"ugh":92,"ugb":50,"uge":17,"ugl":17,"uga":1558,"ugu":24,"ugt":630,"uha":14,"Pól":20,"uda":51,"udd":19,"ude":22,"udi":21,"Trá":40,"ubs":20,"Trí":30,"uca":20,"ue ":74,"uci":14,"uch":302,"uck":27,"uer":27,"uff":30,"udo":19,"ug ":71,"uee":24,"uen":19,"uel":34,"ub ":137,"ua ":465,"uat":139,"uas":240,"uar":105,"uam":28,"ual":155,"uan":118,"ubl":49,"ubh":89,"uba":46,"ubb":14,"Ua ":15,"ud ":277,"uai":2599,"uag":16,"uad":53,"uac":355,"uc ":20,"Uac":82,"Uai":24,"tuá":17,"ty ":159,"trú":23,"tró":28,"tré":177,"trí":558,"trá":247,"tur":96,"tus":32,"tut":21,"tui":74,"tua":318,"tug":303,"tz ":31,"pós":16,"two":15,"pór":40,"VI ":18,"tsí":19,"pói":180,"Tex":30,"Pár":28,"Ter":20,"ts ":73,"pío":31,"pín":27,"tre":362,"tt ":71,"tra":441,"tri":301,"tru":91,"tro":254,"tu ":58,"tsa":41,"tse":67,"tsc":23,"tsi":118,"Tha":57,"tsr":40,"The":480,"tso":36,"Thi":133,"tsu":19,"Tho":176,"Thr":22,"tta":38,"Thu":219,"tte":89,"tth":14,"tti":29,"ttl":27,"tto":23,"Tia":32,"ttp":53,"tts":17,"Tio":42,"Tim":32,"Tit":14,"péa":48,"tma":53,"thú":153,"to ":131,"péi":112,"tne":54,"tiú":375,"tp ":53,"tna":46,"toc":35,"Thá":100,"toi":111,"tog":48,"tob":29,"Thí":47,"tos":71,"tow":49,"tom":35,"ton":323,"tok":16,"tol":18,"tor":167,"top":31,"Péi":18,"til":72,"tie":23,"teá":53,"tig":135,"tir":335,"tit":122,"tis":104,"tin":198,"tim":310,"tio":296,"thy":18,"thu":979,"tia":170,"tic":139,"Tor":40,"Tol":19,"Tom":78,"Ton":21,"Tos":30,"tiu":19,"tiv":17,"Tog":21,"Toi":20,"tli":127,"thé":99,"thí":114,"thó":153,"tla":91,"thá":326,"tle":130,"tem":15,"ten":53,"pán":35,"teo":1066,"tep":28,"tei":324,"pái":517,"tel":48,"tee":15,"tea":2655,"Tro":44,"ted":69,"Tri":35,"Tre":38,"Tra":56,"th ":2421,"tev":28,"tes":48,"pás":47,"ter":367,"taí":552,"ti ":132,"thn":307,"tho":317,"thl":87,"thm":25,"thr":797,"ths":70,"Tur":38,"thf":27,"thg":15,"thd":15,"the":3504,"thi":663,"Tui":81,"Tul":23,"thc":164,"thb":46,"tha":4634,"Tua":184,"Tug":194,"вич":31,"àid":20,"́н":22,"́ч":14,"áth":595,"bis":88,"bit":81,"ást":38,"bir":14,"bil":25,"bin":47,"áta":69,"bhú":16,"bhó":78,"bhé":79,"bhí":1218,"árú":23,"bhá":244,"ble":36,"bli":926,"bla":511,"boc":23,"bol":50,"boi":15,"bog":74,"blí":56,"bpo":38,"bpr":43,"bló":16,"bpa":22,"bpl":21,"bon":33,"bor":45,"bou":52,"bbe":22,"ált":86,"cCa":16,"be ":60,"áma":128,"ámh":113,"bam":96,"ban":414,"bal":290,"bai":1183,"bac":109,"bad":23,"án ":2509,"ála":655,"bas":43,"bar":115,"ánr":41,"áns":33,"ánt":45,"ánm":44,"ánn":32,"bea":484,"ánd":17,"ánc":23,"ána":1388,"ás ":419,"bei":63,"ár ":430,"bec":19,"ber":213,"baí":52,"beo":79,"bel":41,"bet":26,"ápa":41,"bh ":1159,"bhf":1122,"ása":96,"bhi":557,"bhl":1983,"bho":90,"bhn":402,"bhs":39,"bhr":844,"bhu":498,"árt":347,"bht":84,"árs":39,"bia":125,"ásc":33,"át ":311,"bhF":112,"árn":34,"ári":39,"bha":2405,"bhe":575,"ára":142,"bhc":24,"áda":66,"ách":270,"ág ":24,"ádr":77,"áfa":23,"ága":18,"ágt":374,"ca ":266,"áig":19,"áid":933,"áib":37,"áic":28,"ádú":57,"car":214,"cas":83,"cat":333,"can":421,"cao":97,"cap":21,"ál ":251,"cac":181,"áip":28,"cab":30,"áis":885,"cae":15,"áir":1722,"cad":202,"áit":897,"cam":73,"cal":252,"áim":58,"cag":16,"áil":1302,"cai":992,"áin":2550,"ce ":736,"cbh":16,"bri":171,"bro":48,"bra":206,"bre":499,"bru":63,"bst":23,"bpá":25,"bua":84,"bur":99,"bun":439,"bui":16,"bus":26,"brá":19,"brí":91,"bró":17,"by ":34,"brú":28,"ád ":33,"ábl":63,"ábh":315,"buí":16,"aka":23,"am ":799,"ake":39,"agá":70,"ajo":14,"afó":14,"al ":1358,"ail":3174,"aim":543,"ain":7349,"aip":101,"air":5994,"ais":2778,"ait":3108,"aiv":22,"adú":40,"ak ":15,"adó":396,"aig":2716,"aif":67,"aid":1852,"aic":689,"aib":509,"aho":27,"acó":14,"ahe":17,"adá":61,"aha":38,"agl":179,"agm":19,"agh":478,"agr":310,"ags":144,"agt":22,"agu":7542,"agn":79,"ago":48,"aoi":1842,"aof":47,"aog":45,"aol":452,"aom":117,"bPa":18,"aod":21,"bPe":17,"aob":407,"aoc":48,"anu":67,"anz":22,"any":20,"ano":191,"ann":7828,"anm":43,"ant":1573,"ans":231,"anr":74,"ane":93,"anf":24,"ang":1024,"anh":18,"ani":113,"ank":64,"anl":23,"ap ":67,"ana":2087,"anb":49,"anc":353,"and":712,"amu":114,"amm":39,"amo":44,"amp":203,"ams":48,"amh":2467,"ami":50,"ame":171,"amb":62,"ama":964,"alv":19,"alu":30,"alt":1341,"als":101,"alr":23,"alo":60,"alm":34,"all":1574,"alk":22,"alg":42,"ali":175,"alc":59,"ald":89,"ale":108,"ala":1139,"alb":131,"an ":16537,"agó":420,"ako":19,"aba":124,"abe":33,"abh":1558,"abi":26,"abl":14,"abo":23,"abr":55,"ae ":1425,"aca":623,"ad ":2107,"ac ":653,"ab ":562,"afo":26,"ai ":46,"aga":645,"agc":16,"age":73,"acá":56,"aen":85,"aem":24,"ael":663,"aes":47,"aer":113,"aed":15,"aei":796,"ah ":42,"afa":69,"aet":39,"ado":66,"adr":74,"adl":17,"adh":3910,"adi":55,"add":28,"adc":24,"adf":52,"ade":47,"ag ":2140,"ady":15,"ads":15,"adt":14,"adu":21,"aco":19,"acn":59,"acl":31,"ack":100,"aci":53,"ach":13609,"ace":48,"ada":1067,"adb":26,"af ":23,"act":78,"acu":209,"acr":43,"acs":125,"azi":25,"avá":25,"aza":20,"asó":66,"asú":74,"atá":1959,"ayn":15,"ató":45,"aya":22,"aye":50,"ba ":436,"amó":47,"at ":114,"arg":165,"arf":40,"aná":386,"are":86,"ard":1063,"arc":302,"arb":337,"ara":1215,"arp":24,"aro":41,"arn":274,"arm":678,"arl":1377,"ark":117,"anç":14,"ari":179,"aru":23,"arv":15,"anó":82,"arx":20,"arr":979,"ars":293,"art":2216,"au ":32,"anú":140,"asa":1535,"ary":92,"asg":16,"asi":30,"ash":50,"asc":579,"ase":61,"bPá":43,"asd":17,"aso":16,"asn":49,"asp":71,"ask":20,"asm":52,"asl":40,"aon":895,"bPo":90,"aot":134,"aor":130,"bPr":17,"aos":25,"ar ":12570,"apa":112,"ape":18,"alá":30,"api":16,"aph":25,"apo":27,"app":15,"apt":53,"aló":74,"as ":3979,"alú":82,"amá":22,"ava":69,"ax ":19,"arí":40,"avi":97,"ará":352,"ave":39,"asá":71,"arú":93,"ay ":128,"awa":22,"aró":146,"ata":651,"aoú":35,"ast":501,"ass":83,"asr":26,"atm":35,"atn":41,"atl":30,"atr":129,"ato":56,"apá":67,"ate":101,"ati":172,"ath":2853,"aw ":16,"att":85,"ats":25,"atu":25,"apó":24,"aul":83,"aur":29,"aus":26,"aud":27,"aug":19,"ον":16,"ος":28,"ος ":28,"ον ":15,"BC ":31,"AC ":14,"Úrs":30,"AD ":23,"ς ":68,"AM ":43,"Úsá":39,"ν ":30,"ÁC ":14,"α ":44,"三 ":16,"アア":15,"ий ":34,"之 ":27,"ич ":33,"το":15,"ρα":19},"n_words":[1346129,1623635,1308708],"name":"ga","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/hi.json b/contrib/languages-data/hi.json
new file mode 100644
index 0000000..3f575b1
--- /dev/null
+++ b/contrib/languages-data/hi.json
@@ -0,0 +1 @@
+{"freq":{"ौद्":366,"ोली":389,"ोर्":465,"ोमी":388,"्तम":479,"्तन":384,"्तर":2639,"्त्":1391,"्थल":430,"्ता":2582,"्ती":679,"्ति":2706,"्तु":954,"्थि":2834,"्था":3750,"्द्":869,"्दी":1870,"्दू":492,"्दे":907,"्ट्":2750,"्टे":7603,"्टी":634,"्टि":711,"्तक":539,"्बन":468,"्मक":445,"्यक":2308,"्मा":2864,"्मि":1000,"्रं":360,"्मी":437,"्यत":638,"्यय":385,"्यम":558,"्रक":2642,"्यव":609,"्रच":370,"्रज":367,"्यु":778,"्या":6530,"्धा":551,"्धि":463,"्ना":572,"्पन":572,"्पा":591,"्प्":3747,"्गत":419,"्का":992,"्कृ":823,"्कर":450,"्जा":403,"्ञा":1890,"्टर":764,"्चि":1116,"ोंन":705,"ौर ":706,"ं":116003,"ः":743,"ँ":3754,"आ":15110,"इ":21960,"अ":27216,"ऊ":1116,"ई":8670,"उ":14369,"ए":27071,"ओ":3623,"ऑ":695,"ऐ":1181,"ग":41430,"ख":13380,"क":215389,"औ":11975,"छ":5467,"च":24607,"घ":4688,"ट":40532,"ञ":2576,"झ":1114,"ज":58287,"ठ":3871,"ड":21061,"ढ":1924,"ण":16159,"त":129370,"थ":26984,"द":62970,"ध":21789,"न":137720,"प":89801,"फ":9525,"्ग ":934,"ब":39694,"भ":28885,"म":108014,"य":96626,"र":228209,"ल":79901,"व":82288,"ष":22409,"श":41726,"ह":118206,"स":149246,"़":11159,"ि":139433,"ा":290518,"े":193119,"ू":22463,"ृ":6345,"ी":110466,"ु":44034,"ौ":4425,"्":228350,"ो":68898,"ै":59521,"ॉ":2831,"।":45019,"०":5718,"१":6322,"्क ":738,"६":1789,"७":1738,"८":1943,"९":4350,"२":3762,"३":1587,"४":1437,"५":1969,"्न ":1500,"ोजन":549," ख":3003," ग":13738," औ":11821," क":119739," ओ":938," ऐ":1154," ऑ":666," ट":5844," ज":31363," झ":690," च":7337," छ":3781," घ":3397," इ":19284," आ":12686," अ":26917," ए":21008," ऊ":806," उ":13706," ई":1344,"्फ ":515,"्म ":3545,"ोड़":535,"्य ":9507,"्र ":7477,"ोते":895,"ोती":1270,"ोता":2537," २":2506," ३":490," १":5155," ।":6381,"्व ":2678,"ोनो":409,"ोने":1142," प":54146," फ":4760," न":21169," म":54702," य":22634," ब":23290,"्ष ":1625," भ":19961," ड":2225," द":21719," ध":2884," त":12822," थ":8999," ह":72412," स":78945," ल":14663," र":25548," श":12231," व":27670,"्स ":1196,"्च ":578,"्ट ":1436,"्ञ ":468,"्ड ":1049,"्ठ ":1088,"्ण ":1354,"्थ ":1814,"ोगि":441,"ोगो":559,"्त ":4240,"्ध ":2352,"्द ":1588,"्सा":631,"्ष्":370,"्हे":874,"्हो":949,"्स्":3660,"्ली":1260,"्ला":820,"्ले":511,"्रद":1892,"्रथ":561,"्रत":2596,"्रण":709,"्यू":989,"्रप":384,"्रम":2375,"्रभ":518,"्यो":1594,"्रय":1925,"्रव":977,"्रश":494,"्रह":1076,"्रस":1694,"्रा":9061,"्रि":3453,"्री":4034,"्रै":386,"्रे":9372,"्रो":1860,"्षण":540,"्षे":3114,"्षि":1460,"्षा":1408,"्वत":875,"्वर":1044,"्वव":540,"्वप":609,"्वी":1298,"्शन":1985,"्वे":430,"्वि":632,"्वा":6601,"०० ":579,"२००":1298,"१९९":360,"१९६":420,"१९७":415,"ंत ":1187,"ंड ":1010,"ंग ":1786,"ंक ":375,"ंभ ":447,"ंद ":452,"ंश ":574,"ंस ":426,"ंह ":485,"आप ":923,"अंत":806,"अंग":1106,"अक्":469,"इन ":809,"ंचत":1845,"ंचा":2297,"ंग्":1940,"ंगा":789,"ंगी":464,"ं। ":4299,"ंको":451,"ंक्":1624,"ंख्":1085,"ंगल":563,"ंटे":1825,"ंडि":369,"ंजा":609,"ंने":716,"ंबं":450,"ंपा":1007,"ंत्":1705,"ंति":457,"ंतर":775,"ंद्":705,"ंदी":875,"ंदि":932,"ंदर":556,"ंबर":485,"ंयु":368,"ंस्":2087,"ंसा":366,"�":2362,"थे ":819,"दन ":568,"थी ":791,"था ":4899,"त् ":405,"थम ":523,"तंत":641,"तो ":1178,"ं ":59244,"ः ":691,"ताओ":415,"तान":1545,"ताब":607,"तार":557,"तिक":1335,"दो ":833,"तिज":375,"तिन":613,"ँ ":2207,"तिय":1006,"तिर":505,"तिह":787,"आ ":1631,"नई ":675,"धन ":595,"दी ":4353,"दू ":610,"तरी":448,"तरा":413,"तरह":445,"दा ":857,"तर्":854,"दि ":1483,"तमा":404,"तमि":373,"ण्ड":966,"णों":461,"दर ":576,"दल ":742,"तथा":2577,"तत्":438,"णाल":368,"थ ":4109,"द ":7269,"ध ":3357,"न ":42221,"ड ":6850,"था।":3059,"ठ ":1539,"थाप":1123,"ण ":9157,"थान":1787,"त ":29579,"थित":2566,"धी ":490,"ज ":2925,"दक्":984,"ञ ":468,"धा ":464,"नट ":1785,"ट ":6235,"धि ":2431,"थवा":400,"च ":2310,"छ ":1066,"क ":31192,"तों":455,"ग ":7730,"ख ":1929,"त्स":696,"त्व":1830,"त्प":791,"त्र":11745,"त्य":2756,"त्म":761,"त्त":3587,"ए ":4565,"तीन":446,"तीय":3767,"तीस":449,"ई ":6870,"ै ":18248,"दार":977,"दान":855,"दाय":435,"दिल":1253,"दिश":630,"दिर":1086,"े ":101676,"दिय":1540,"दित":1005,"दिन":569,"न् ":786,"दुर":430,"ू ":2066,"दूर":671,"दूस":592,"ि ":14043,"नी ":5102,"ी ":75983,"ु ":3758,"ा ":97285,"़ ":2122,"दस्":366,"ह ":13867,"ने ":12971,"स ":15469,"ष ":2544,"श ":4366,"व ":7299,"ल ":18408,"नि ":406,"दर्":1109,"ना ":7450,"र ":61007,"थी।":911,"य ":22189,"म ":13770,"भ ":1015,"नव ":411,"ब ":2705,"फ ":1049,"थे।":1504,"प ":5685,"टना":394,"डल ":378,"डी ":506,"डा ":698,"टर्":362,"ड़ ":675," �":578,"टती":1811,"ञान":1620,"ट्र":7028,"ट्ट":617,"टेश":7359,"ढ़ ":636,"टीक":471,"णु ":439,"णी ":648,"णा ":355,"तक ":1940,"ति ":4868,"ता ":14635,"तु ":1219,"ती ":9089,"ते ":6248,"तन ":484,"ड़े":606,"ड़ी":1222,"ड़ा":1450,"डिय":1027,"तम ":378,"तर ":1819,"जे ":3791,"जा ":1910,"ज़ ":425,"जी ":1578,"चेन":368,"चीन":1126,"चुन":405,"जो ":3111,"चिक":478,"चित":1154,"चार":1698,"चाल":2122,"चिम":876,"जंक":1375,"च्च":632,"छोट":541,"जन्":1234,"टक ":496,"जधा":578,"जनस":468,"जनी":553,"छूट":1815,"जस्":392,"टन ":357,"जहा":441,"ज़ी":477,"जिस":3147,"जिल":1663,"जिन":942,"जित":402,"जार":493,"जाब":559,"जान":2165,"जात":5721,"जैस":743,"ज्य":2693,"ज्ञ":2457,"जीव":1088,"टर ":1798,"टे ":2183,"टि ":384,"टी ":1658,"टा ":740,"ंघ":459,"ंख":1287,"ंग":7039,"ंक":4013,"ंड":2775,"ंट":3074,"ंज":1716,"ंच":5039,"केन":435,"ंश":984,"ंस":3555,"ंह":717,"ंव":676,"् ":2686,"खने":369,"ंध":1776,"ंद":4729,"ंन":763,"ंथ":399,"ंत":5036,"ंय":471,"ंप":2156,"ंभ":782,"ंब":1984,"ो ":17876,"ँच":523,"कोई":460,"कों":1045,"आत":950,"इं":1275,"कृत":1818,"आज":527,"अस":694,"अव":2719,"आक":698,"अल":1444,"अर":2196,"अभ":1051,"अम":1289,"अब":415,"अप":3105,"आई":372,"अध":2198,"अन":4741,"अथ":432,"इत":740,"इट":387,"आस":422,"इक":606,"आव":1056,"आम":383,"आय":903,"आर":1257,"आप":1303,"कृष":566,"आद":1231,"आध":1054,"आन":511,"अं":2476,"आं":508,"अत":785,"अक":781,"अग":575,"ं।":6630,"उप":2592,"उस":2266,"इन":2901,"इल":681,"इस":13512,"आ।":435,"ईस":387,"उत":2684,"उन":3336,"उद":1042,"कोच":453,"कोड":3985,"एं":534,"कोल":359,"एँ":425,"कोश":370,"एक":15570,"क्ट":677,"क्त":3551,"क्य":529,"क्श":1460,"क्ष":8266,"क्र":2787,"क्स":4316,"ओर":445,"ओं":2444,"ऐस":714,"एव":3043,"एल":360,"एस":427,"गर":2860,"गय":3634,"गल":1055,"गव":557,"खे":1130,"खो":528,"गम":494,"ख्":2681,"गभ":616,"खा":2157,"गठ":414,"खि":516,"खी":461,"घं":1798,"गढ":741,"गण":767,"गत":1406,"गद":372,"गए":383,"क्":22746,"कै":618,"के":37285,"खन":943,"गई":554,"को":17366,"कॉ":440,"कि":12948,"की":20919,"गं":513,"का":35945,"कृ":2446,"कु":3003,"कू":402,"कस":766,"कव":956,"कह":3138,"कल":2049,"खक":372,"कम":1276,"कर":12591,"कप":515,"कथ":448,"कन":1065,"कड":382,"खं":419,"कत":2310,"कट":630,"और":11428,"कई":777,"कं":723,"चु":923,"ची":1816,"जं":1545,"चि":3214,"चा":5438,"चे":1055,"चौ":426,"च्":1117,"० ":1980,"जग":567,"गढ़":657,"चत":1993,"चन":1667,"चर":750,"चल":1479,"घा":478,"चं":451,"। ":33428,"गति":373,"चक":424,"गा":3720,"गह":412,"गी":1381,"गि":1175,"गु":1958,"गो":2483,"ग्":4970,"गे":620,"गै":364,"घर":423,"टन":928,"ञा":1890,"टत":1909,"टा":1764,"टल":396,"टर":2646,"४ ":778,"झा":416,"३ ":798,"टक":827,"जो":3936,"जै":1167,"जे":4339,"जू":402,"जी":3381,"जु":839,"जा":12605,"जि":7022,"ज़":2064,"घंट":1796,"२ ":974,"ज्":5651,"जन":5056,"जध":594,"छू":1841,"छा":395,"जह":523,"जस":483,"१ ":954,"जर":910,"जल":806,"जब":740,"छो":713,"जय":526,"जम":644,"ठा":568,"५ ":1070,"टे":10515,"ठन":369,"ट्":8239,"टो":662,"टी":2663,"टि":1894,"ड़":5317,"डा":1474,"डि":2023,"डी":831,"डल":514,"६ ":930,"गभग":596,"तं":976,"तः":367,"ढ़":1322,"७ ":816,"ड्":669,"डो":499,"डे":781,"डु":358,"णि":851,"णी":758,"णु":577,"णा":1135,"तक":2674,"८ ":811,"णन":433,"९ ":917,"तव":480,"ति":11566,"ता":20016,"तु":2358,"ती":14469,"तत":547,"तथ":2628,"तप":364,"तन":1343,"ण्":1311,"तम":1364,"णो":495,"तल":566,"तर":5109,"थव":456,"था":11678,"थी":1994,"थि":3214,"ते":6961,"तो":1865,"त्":23294,"थम":587,"थल":512,"दक":1341,"दस":512,"दश":363,"दृ":463,"दू":2227,"दु":1528,"दी":5182,"दि":8815,"दा":4680,"थो":462,"थे":2431,"दन":776,"दल":1038,"दर":2412,"थ्":491,"धा":4715,"नट":1830,"नत":1382,"खें":445,"नद":975,"धी":935,"धि":5621,"धु":599,"दो":2067,"दौ":382,"द्":15486,"नई":683,"दे":6364,"धन":912,"नक":3879,"नग":1441,"धर":1283,"नर":642,"नल":477,"नव":1823,"नन":633,"नप":443,"ध्":2713,"नम":981,"नी":7183,"पं":1090,"नु":2801,"ने":15948,"नस":1797,"नह":1786,"ना":16910,"नि":11710,"पक":1243,"नो":2338,"न्":16749,"पत":2217,"पन":4849,"पद":2167,"पड":542,"पट":576,"पश":1179,"पह":3589,"पस":581,"पल":544,"पय":630,"पर":12235,"बई":388,"पे":1223,"पै":501,"पू":3275,"पृ":1255,"पा":8076,"पि":1966,"पी":1550,"बं":1558,"पु":5232,"फल":545,"फर":388,"प्":26390,"पौ":356,"पो":892,"बन":3146,"फे":662,"बद":869,"फि":861,"बड":1139,"फी":943,"फा":687,"बज":3798,"फ़":1678,"फो":368,"फ्":1074,"बर":2061,"बल":818,"गया":3134,"भग":1053,"बस":1999,"बह":1601,"बि":1756,"बा":6278,"बु":789,"बी":1868,"मं":2240,"बे":1048,"बै":523,"बो":1031,"भर":525,"ब्":3946,"मक":1369,"भव":561,"भी":5128,"यं":862,"भि":1870,"भा":13836,"मत":879,"मण":522,"भू":2019,"भु":462,"मन":2122,"मध":940,"मद":711,"भौ":389,"भो":416,"मल":693,"यक":2875,"भ्":381,"मय":1214,"मर":1260,"मश":384,"मस":725,"मह":3586,"मृ":555,"यत":1327,"मू":1671,"ख्य":2528,"यद":785,"मि":6854,"मा":15028,"मु":5029,"मी":3565,"रं":1975,"मो":1712,"यम":1479,"म्":5925,"मे":33130,"यन":1546,"यप":564,"मै":980,"रख":1463,"यव":775,"रग":385,"रक":5780,"यय":399,"यर":1126,"या":29338,"रज":1027,"यह":10437,"रच":1272,"रद":2279,"रथ":786,"रत":13386,"यू":1784,"रण":4290,"यु":3494,"यी":833,"यि":596,"रय":1959,"रम":3670,"रभ":636,"यो":8068,"रब":740,"रप":839,"ये":5448,"रन":4212,"लम":734,"लय":1888,"लब":457,"लन":1624,"लत":1279,"लग":2035,"लक":1497,"र्":27657,"रो":5583,"रै":502,"रॉ":358,"रे":18745,"गां":472,"रू":3986,"लं":804,"री":10641,"रु":2131,"रि":9664,"रा":30478,"रह":3970,"रस":3428,"रश":605,"रव":1845,"रल":964,"वं":3802,"ल्":5184,"लो":3980,"चन ":356,"लै":601,"ले":6664,"लु":426,"ली":6031,"लि":10617,"ला":9912,"लव":4226,"शब":1373,"वो":877,"वै":1045,"शन":10719,"वे":7461,"व्":3139,"शर":623,"वह":1716,"वव":621,"वश":922,"वस":1629,"वृ":595,"शत":728,"वा":18296,"वि":15882,"वी":3437,"वप":710,"वन":1882,"वध":1955,"शक":894,"वल":726,"वर":5872,"वय":393,"वज":391,"वक":455,"वत":1673,"षे":3151,"सन":2988,"सप":658,"सभ":1408,"सब":1580,"सम":9143,"ष्":6623,"सर":4398,"सल":818,"सव":526,"षा":3622,"सट":463,"षि":1967,"सत":766,"सद":867,"शै":363,"शे":1186,"श्":6992,"शो":892,"षय":360,"सक":10544,"शह":1374,"शी":1148,"सं":12512,"शु":1050,"शा":4600,"शि":3780,"गवा":390,"षण":1876,"है":50827,"हे":2158,"हु":7130,"ही":5060,"हि":5996,"हा":10508,"ह्":694,"हो":8889,"हन":827,"से":22788,"सु":1954,"सी":5308,"हत":2956,"सू":1760,"चल ":435,"सि":5630,"सा":11391,"सह":931,"सस":498,"हव":548,"हस":476,"हल":1812,"हम":833,"स्":33294,"हर":3284,"सै":435,"सो":953,"़त":356,"ात":13410,"ाथ":2015,"ाण":2291,"ाठ":433,"िं":3492,"ाड":1607,"ाट":1520,"ाब":2717,"ाभ":451,"ाप":4193,"ाफ":948,"ान":23672,"ाद":7103,"गुर":518,"ाध":1672,"ाव":4478,"िख":1296,"िक":17099,"ाल":13682,"ार":38751,"ाय":6583,"िए":2749,"ाम":8252,"िज":2534,"ाह":4362,"िच":656,"ास":7258,"ाष":4485,"िग":560,"ाश":1899,"़ी":1936,"ां":6856,"़ा":2260,"़ि":839,"ाँ":2505,"़ो":448,"ाइ":2283,"ाई":2655,"़े":807,"ाउ":506,"ाओ":2028,"ाक":3759,"़्":460,"ाए":1076,"ाच":1779,"ाज":7904,"ाग":3365,"ाख":925,"ीड":730,"ुं":2936,"ा।":4894,"ुई":845,"ीत":2135,"ुआ":2021,"ीप":1050,"ीन":3683,"ुए":1050,"ीम":1042,"ीय":7229,"ीब":439,"ील":1370,"ुओ":472,"ीर":2043,"ुग":719,"ुख":2469,"ीव":1580,"गीत":642,"ुक":2798,"ुज":614,"ुछ":884,"ीस":1060,"ुट":385,"िट":1437,"ीं":3186,"गिक":498,"िण":1209,"गाल":415,"ित":17874,"िद":3772,"िध":2081,"िन":9694,"िप":1804,"िब":505,"िभ":1498,"िम":3142,"िर":6200,"िय":18344,"िल":8308,"ीक":2676,"िश":4603,"िव":3402,"िस":8212,"िष":1899,"ीच":800,"िह":1477,"ीज":592,"ीट":1012,"ेव":2738,"ेश":12536,"ैक":523,"ेल":9698,"ेय":725,"ेर":2606,"ेम":795,"ेब":355,"ेप":664,"ेन":6508,"ैद":560,"ैत":376,"ैज":408,"ेह":634,"ेस":4417,"ेष":1173,"ैर":490,"ैल":1189,"ैन":999,"े।":1956,"ॉन":366,"ैस":1223,"ें":31320,"ेक":2519,"ेख":1744,"गों":941,"ेट":1393,"ेड":787,"ैं":11503,"ेत":4873,"ेद":951,"ेग":371,"ेज":1606,"ृत":2832,"ृष":1881,"गोल":437,"ुत":2428,"ुण":594,"ुड":541,"ुन":2123,"ुध":368,"ुद":1873,"ुप":1093,"ुर":6463,"ुम":1318,"ग्ल":427,"ुल":2104,"ुष":645,"ुस":2362,"ुव":1118,"ग्र":3597,"ूच":523,"ूट":2497,"ूत":791,"ी।":1527,"ून":1187,"ूप":2981,"ूब":457,"ूम":650,"ूर":5314,"ूल":1317,"ूष":1076,"ूस":930,"ूह":483,"्व":16523,"्श":2596,"्ष":9849,"्स":6797,"्ह":2245,"्भ":823,"्म":10658,"्य":26066,"्र":57172,"्ल":4149,"ची ":462,"ौत":437,"ौद":469,"ोर":2026,"ोल":2394,"ोब":410,"ोम":1176,"ोस":741,"ोष":453,"ोह":815,"ोश":547,"ोव":518,"्ण":2398,"्त":17955,"्ड":2149,"्ट":15553,"्ठ":1556,"्ञ":2460,"्फ":981,"्ब":2118,"्प":6879,"्ध":4156,"्न":3593,"्थ":9797,"्द":7520,"ौर":1279,"्ज":1507,"्च":2621,"्ग":2050,"्क":4547,"ों":15790,"ै।":21739,"ोई":504,"ॉर":357,"ॉल":485,"ोज":1283,"ोड":4892,"ोट":1429,"ोद":429,"ोत":5502,"ोप":971,"ोध":664,"ोन":2390,"ोक":1863,"ोच":771,"ोग":4303,"००":2062,"१०":362,"१८":536,"१९":2833,"२०":1612,"९६":475,"९५":380,"९८":427,"९७":465,"९९":438,"जन ":1003,"जब ":495,"चती":1825,"चना":962,"गई ":460,"के ":35060,"का ":19876,"कि ":2326,"की ":19870,"कल ":368,"कम ":375,"कर ":3798,"और ":11340,"कित":562,"किन":799,"काफ":651,"काम":601,"कार":8097,"काल":1674,"काश":1174,"काव":390,"कास":610,"किप":365,"किय":4402,"किल":674,"किस":2621,"कीय":409,"कुछ":880,"कुल":589,"कुम":405,"कां":514,"कान":492,"कहत":1259,"कहल":373,"कहा":1242,"कला":836,"गी ":381,"कवि":783,"कम्":429,"कर्":878,"गा ":602,"करा":372,"करे":537,"करत":2060,"करण":990,"करन":3111,"कथा":387,"कता":1005,"कते":720,"गर ":1575,"खंड":407,"गत ":745,"खा ":869,"कंप":421,"को ":9463,"ओं ":2427,"एवं":2948,"एक्":3566,"कई ":776,"ओर ":406,"उसक":842,"उसे":365,"उपन":475,"उपय":550,"एँ ":417,"एक ":11574,"आदि":993,"आधा":542,"आर्":522,"आवश":646,"उन ":374,"इति":479,"अथव":383,"अधि":1401,"अनु":2084,"अध्":641,"अन्":1665,"अने":519,"अपन":2305,"इस ":3686,"अभि":793,"अमे":560,"अलग":451,"अर्":1665,"अवध":1855,"उन्":1250,"उद्":651,"उनक":1565,"उत्":2559,"उस ":606,"इनक":960,"इन्":786,"इसक":6193,"इसम":1168,"इसी":419,"इसे":1083,"इस्":411,"ेश्":531,"ेशो":391," ओर":436," कं":518," कई":773," और":11423," कन":365," कम":1021," कर":8774," कल":965," कव":757," कह":3125," की":14772," कि":9638," का":20292," कृ":765," कु":2268," के":32645," कै":557," गई":551," को":14690,"ेशन":8153," गए":382," क्":4800,"ेवा":915," एक":15549," एव":3020," ऐस":714,"ै। ":17713,"ोई ":492," चौ":410," जग":453," चु":690," ची":475," जं":1512," चि":1251," चा":1105," चे":563," जर":355," जल":669," जब":682," छो":700," जह":416," छू":1830," जन":2746," ज्":953," जि":5438," जा":9396," जु":639," जी":1466," जो":3609," जै":1100," गर":416," गय":3607," खे":428," गण":526," घं":1793," खा":713," गो":922," ग्":1394," गु":1570," गा":1413," । ":3304," चल":837," चर":408," अं":2469," अब":411," अप":3085," अन":4720," अध":2196," अथ":432," आक":690,"ेता":622," अल":1382," अर":2119," अम":1263," अभ":1046," आज":524," अस":670," अव":2716," आत":825,"ेत्":3123," इं":1185," अक":778," अग":573," आं":486," अत":780,"ेपा":393,"ेन्":1060,"ेना":477,"ेयर":374," इस":13352," इल":428," इन":2514,"ों ":14584," इत":729," आस":419," आर":1183," आम":378," आय":881," आव":1052," आन":491," आद":1214," आध":1052," आप":1294,"ेलव":3726," उस":2203," उप":2582," उन":3214," उद":990," उत":2673,"ेरि":755," वा":3866," वी":554," वि":11396," वृ":367," शत":617," वस":497," वह":1459," व्":2079," शर":525," वे":1801," वै":880," शब":1343," शु":774," सं":11345," शि":1818," शा":2156," शह":1270," सक":1827," श्":1605,"ैसे":672," शक":380," वर":2508," लग":1362," ला":1268," लि":5639," ले":1874," लो":1934," रख":1043," रज":470," या":6370," यह":10383," रच":735," यू":702," यु":969," यो":670," ये":2040," रा":8448," रि":478," रह":2139," रे":6398," रू":2374," रु":449," लं":500," रो":897,"ोर ":430," हो":7690," हि":3553," ही":2071," हा":1189," हु":3748," हे":505," है":50701," सम":6692," सभ":777," सब":1516," सन":1812," सर":2776," सट":433," सद":698," सत":571," हम":409," स्":16029," हर":704," सो":571," सा":6452," सि":2848," सह":892," से":17371," सी":1037," सु":1695," सू":1297," दर":820," थे":2324," दू":1295," दृ":400," दी":559," दु":776," दा":748," दि":4337," दक":983," त्":639," तो":1173," था":4674," थी":1702," तम":403," तर":977," तथ":2624," तत":379," ता":833," ति":580,"ोग ":2225," ती":840," तु":355," तक":1670," डा":569," डि":508," ट्":3950,"ोच ":425," मो":707," मे":29038," मै":859," मू":889," यद":611," मा":5157," मि":3653," मी":694," रं":422," मु":2959," मह":3455," मर":397," भौ":360," मन":1109," मध":857," भू":1742," भी":4132," भा":10893," भर":368," ब्":1174," बे":646," बै":464," बो":710," बा":3995," बि":1259," बी":1196," मं":1754," बु":605," भग":427," बह":1547," बस":477," फ्":403," बर":576," बल":422," बद":685," बन":2421," फे":481," फा":416," बज":3787," फ़":822," फि":728," बड":1101," प्":19145," पो":399," पि":711," पा":4190," पु":2921," पी":496," बं":729," पे":601," पै":416," पृ":1225," पू":2103,"ोड ":4034," पर":10662," पश":1088," पह":3539," पड":455," पट":420," पद":1827," पत":1117," न्":487," नह":1717," नि":5215," ना":4741," पं":1035," नी":536," ने":3413," ध्":387," नव":592," धा":800," नद":808," दे":3244," द्":5503," दो":1318," धर":1010," नग":840,"ेंट":533,"ेंद":417," ई ":361,"ेल ":4266,"ेर ":787,"ेस ":3757,"ेष ":525,"ेश ":2529,"े। ":1586,"ेकि":361,"ेक्":824,"ैंड":368,"ैं।":6253,"ेज़":466,"ेजी":411," व ":823,"ृष्":1735,"ें ":29625,"ृति":1045,"ृत्":636,"ेन ":4101,"ेद ":371,"ेज ":388,"ेट ":613,"ैं ":4429,"ेक ":779,"ुष्":405,"ुवा":512,"ुला":407,"ुरा":1009,"ुरस":482,"ुर्":818,"ुरु":791,"ुरू":359,"ुमा":600,"ुनि":743,"ुना":530,"ुद्":1350,"ुत्":627,"ूसर":611,"ूषण":1023,"ूर्":2859,"ूरी":482,"ूटत":1807,"ुस्":700,"ुसा":1149,"िकल":459,"ाला":1369,"ालि":2288,"ाली":1852,"ाले":1192,"ावि":362,"ावा":595,"ुत ":1429,"िका":3179,"िकि":844,"िकी":876,"ावर":399,"िको":367,"िक्":1744,"ाषा":2056,"ासन":527,"ाष्":2240,"ाशि":573,"ासक":514,"ाहि":1577,"ासि":587,"ासा":469,"ासी":381,"ाहर":429,"ास्":1433,"िज्":1407,"ुर ":2020,"िता":1136,"िति":559,"ुल ":595,"ित्":3274,"िद्":3069,"िधि":924,"िधा":871,"िनट":1785,"िना":538,"िनि":795,"िन्":3490,"िपी":396,"िभि":546,"िभा":779,"ियन":355,"ियम":677,"िमी":551,"िमा":800,"ियो":3134,"िये":2350,"िया":10338,"िर्":2740,"िरा":423,"िलत":401,"िले":1007,"िल्":2138,"िलो":516,"िला":1775,"िसम":787,"िष्":1088,"ी। ":1157,"िशा":959,"िश्":1982,"िशे":677,"िसक":1052,"िवा":1432,"िवे":381,"ीका":588,"िवर":417,"ूप ":2670,"िहा":1270,"ून ":537,"िसे":557,"िसी":1508,"िस्":2756,"ीडि":468,"ुंब":372,"ुंच":1928,"ीटर":673,"ूल ":705,"ीति":750,"ूर ":759,"ुआ।":435,"ूह ":393,"ुओं":451,"ीर्":408,"ीमा":484,"ुक्":1845,"ीवन":532,"ुख्":1068,"ृत ":890,"ीं ":2774,"ित ":12024,"िण ":703,"िन ":1347,"िल ":1061,"ीक ":803,"ांग":686,"ांस":610,"िम ":1068,"ांत":1630,"िय ":463,"िर ":1319,"ीच ":562,"ाएँ":397,"िश ":406,"िस ":650,"ा। ":4040,"ुआ ":1392,"ीत ":751,"ागर":849,"ाक्":373,"ाकि":1058,"ाका":641,"ाकर":472,"ाओं":1931,"िंद":1176,"ाड़":911,"िंह":577,"ुए ":894,"िंग":982,"ाटक":402,"ाजा":802,"ाजि":402,"ाज्":2259,"ीप ":552,"ाजन":843,"ाजध":594,"ाजस":382,"ाची":795,"ीन ":2404,"ुई ":687,"ाने":2242,"ाना":1838,"ानि":2065,"ानी":2791,"ुख ":1161,"ानव":590,"ानस":622,"ाध्":419,"ापन":842,"ान्":1734,"ानो":532,"ादी":544,"ादा":388,"ादि":1157,"ानत":633,"ाधि":410,"ानक":524,"ाति":524,"ाता":4840,"ाती":1316,"ाते":918,"ात्":3549,"ील ":655,"ीय ":6964,"ीर ":855,"ारी":1527,"ारि":1016,"ारा":6234,"ारस":394,"ार्":5244,"ारो":922,"ारू":465,"ारे":707,"ालय":1677,"ामि":517,"ामा":1032,"ायक":414,"ाम्":377,"ायन":449,"ामी":544,"ाया":1613,"ायी":366,"ारक":420,"ारण":1386,"ारत":7763,"ाबा":770,"ाब्":555,"ामक":498,"ुछ ":876,"ापा":517,"ाप्":901,"ाफी":651,"ां ":1658,"़ी ":1669,"ाई ":2362,"़े ":628,"हों":935,"है।":21737,"होत":4643,"होन":1130,"ह्म":356,"़ा ":1290,"ाँ ":1565,"ाग ":1189,"ाथ ":1651,"ाद ":3112,"ाण ":964,"ात ":1397,"ान ":8758,"ाज ":1013,"ाश ":423,"ाव ":1068,"िक ":8100,"ाह ":929,"ास ":2767,"ाम ":3709,"िए ":2584,"ाब ":747,"ाल ":3360,"ार ":11273,"ाय ":1113,"समु":515,"समा":1211,"समू":429,"सरक":1452,"समे":1977,"सम्":2440,"सर्":919,"सबस":1305,"समय":1034,"ष्य":501,"ष्ण":694,"ष्ठ":1362,"सभी":533,"सभा":753,"ष्ट":3181,"षेत":2985,"सन्":672,"हत्":826,"हते":1441,"सेन":499,"सें":450,"सीम":423,"सिर":645,"सूर":646,"सूच":437,"सां":361,"साह":1353,"सिद":1352,"सित":526,"साध":457,"सार":1977,"साम":1236,"साय":536,"सिक":706,"साग":441,"सिं":774,"साथ":1234,"सहा":390,"ससे":419," १८":501," १९":2789," २०":1512,"हला":582,"हले":581,"हरा":369,"स्व":2405,"स्य":682,"स्म":485,"स्ल":386,"स्थ":6865,"स्प":4184,"स्ट":9331,"स्त":5132,"स्क":2303,"सेव":720,"हैं":10612,"हें":816,"हुत":957,"हुई":797,"हुए":951,"हुं":1933,"हुआ":1762,"हिन":1798,"हित":1670,"हीं":1893,"हास":1027,"हाव":428,"हार":1703,"हान":874,"हिं":1060,"हाँ":938,"हां":1185,"हे ":650,"है ":18074,"सकी":3365,"सका":2827,"सके":2127,"सत्":357,"षिण":921,"सटी":407,"षित":444,"हो ":1330,"शेष":769,"शों":440,"हा ":1528,"सकत":1521,"ही ":2790,"श्व":2006,"श्र":1917,"श्य":1102,"श्च":1235,"शहर":1243,"से ":20466,"सी ":3799,"हर ":1676,"संच":2031,"शिय":571,"संक":898,"संग":1492,"संख":1034,"शिव":510,"संघ":401,"संब":719,"संप":1355,"संय":444,"संस":2227,"शित":453,"शाह":383,"शास":1407,"शिक":1243,"शाल":480,"वेश":377,"वेद":482,"शता":469,"सर ":563,"सा ":1620,"व्य":2741,"शब्":1328,"वों":401,"वर्":3433,"षा ":2805,"वरी":565,"ववि":516,"शक्":405,"वश्":752,"वस्":1030,"सन ":1657,"वाद":1110,"वान":862,"वाच":378,"विक":1766,"वाल":2753,"वास":1153,"वार":6241,"वाय":555,"वाम":390,"वित":981,"विद":1842,"विध":1382,"विज":1255,"वाह":694,"विच":389,"वीं":679,"विष":806,"विश":2580,"विस":375,"विव":625,"विभ":1112,"वीप":490,"वता":365,"वधि":1770,"वपू":515,"शा ":783,"षण ":1678,"शी ":479,"वंश":456,"शन ":10109,"वे ":5027,"वि ":440,"वा ":1780,"वी ":1441,"ल्प":398,"ल्म":761,"ल्य":463,"ल्ल":1775,"लोक":1046,"लोग":827,"लों":837,"वह ":1195,"लोम":397,"वल ":387,"लेख":932,"लेक":881,"लिय":3022,"वर ":730,"लाक":373,"लात":527,"लिए":2317,"लाल":465,"लिख":696,"लिक":529,"लित":2228,"लिप":379,"वन ":938,"लवे":3731,"लना":388,"लता":462,"लती":425,"वं ":2957,"लगभ":597,"लगा":459,"रेस":3665,"रोग":432,"रों":2025,"र्श":1019,"र्व":3266,"र्स":388,"र्ष":1579,"र्म":3443,"र्भ":362,"र्य":2627,"र्थ":2552,"र्द":1098,"र्ध":360,"र्न":513,"र्फ":531,"र्ब":360,"र्ट":763,"र्ड":507,"र्ण":1686,"र्त":1633,"र्ग":1726,"र्क":924,"र्ज":853,"र्च":479,"रीक":801,"रिव":843,"रिय":2208,"रीय":1696,"रीर":445,"रुप":452,"रूप":2871,"रें":640,"रेज":901,"रेल":6023,"रेन":3698,"रसा":555,"रसि":1094,"रहत":499,"रहा":649,"रहे":489,"रस्":963,"ले ":3374,"रां":1067,"रान":1776,"राप":737,"रात":590,"राण":691,"राज":5799,"राच":861,"राक":432,"रिट":356,"रित":976,"राष":2127,"रास":461,"राम":1378,"राय":1077,"रार":802,"रिक":2624,"राव":460,"ला ":4369,"रयो":985,"रयु":635,"रम्":388,"रमा":570,"रमु":1096,"रवा":666,"ली ":4841,"रने":2577,"रना":959,"रदे":1187,"रदा":576,"रभा":427,"योज":542,"योग":2669,"यों":3685,"या।":1162,"युक":1333,"युत":419,"युद":407,"यान":790,"याप":561,"यात":2560,"याद":888,"यास":655,"यिक":384,"याल":1423,"यार":471,"याय":535,"रता":1516,"रति":1618,"रती":4166,"रते":811,"रत्":447,"रथम":512,"लय ":1619,"यूट":437,"यून":379,"रणा":516,"रचन":643,"रक्":994,"रखे":433,"रजि":411,"यां":725,"याँ":525,"यहा":1528,"लन ":775,"रे ":1761,"मस्":459,"महा":2097,"महत":756,"यक्":1167,"रु ":369,"यका":377,"री ":6215,"मृत":432,"मूह":442,"मूल":727,"मुद":734,"मुख":2339,"रंभ":445,"मुं":391,"मीट":638,"रंग":680,"मिल":1800,"मित":732,"मिन":2223,"मार":2023,"माल":612,"मिक":732,"रो ":378,"माण":1011,"माध":371,"मात":697,"मान":5551,"माज":766,"मां":436,"मों":431,"लग ":468,"यदि":507,"में":27306,"मेर":842,"मेल":1950,"यता":597,"रका":3522,"म्र":418,"म्प":956,"म्ब":1357,"म्म":1860,"रत ":4510,"रण ":3158,"या ":17033,"यु ":436,"यी ":716,"भिन":962,"भाव":792,"भाष":2240,"भार":7799,"भाग":1261,"रम ":841,"यंत":450,"ये ":4867,"मध्":813,"भूम":411,"भूष":1010,"रल ":672,"रा ":9045,"मर्":364,"रह ":1087,"मा ":1358,"भगव":386,"मी ":1758,"मे ":1785,"यन ":960,"बहु":1203,"बसे":1366,"बाल":449,"बार":924,"बाद":1868,"बां":380,"यर ":634,"मंत":477,"मंद":933,"बिह":404,"बीच":558,"मंड":367,"यम ":869,"रक ":417,"यह ":8676,"बोल":432,"ब्र":1124,"ब्द":1896,"प्य":460,"प्र":23891,"प्त":1355,"भा ":824,"भी ":4811,"मन ":544,"फ़ि":394,"बजे":3647,"फिल":452,"मय ":1094,"बड़":987,"बदल":617,"यक ":1025,"फेर":414,"बना":1428,"बन्":526,"फ्र":464,"पहु":2050,"पहल":1024,"पश्":999,"बी ":558,"बा ":412,"पर्":1203,"परम":473,"पयो":528,"परि":1837,"परा":615,"मक ":839,"पृष":917,"पुस":378,"पूर":2834,"पुत":428,"पुर":3647,"पीड":441,"बंग":529,"बंध":817,"पास":574,"पित":787,"पाक":1033,"पान":519,"पात":387,"पाद":1538,"पार":1007,"पाल":950,"पक्":435,"फी ":766,"न्त":2330,"न्ध":640,"न्न":1957,"न्द":3701,"न्य":2792,"न्म":1190,"न्ह":2015,"नों":1406,"पद्":1215,"पदा":458,"भग ":605,"पनी":874,"पना":1055,"पन्":743,"पने":1546,"बर ":1142,"पत्":1249,"पति":612,"पड़":437,"नमे":432,"नवर":402,"नसं":440,"नदी":699,"ध्य":2164,"नुस":1344,"पंज":567,"नीत":724,"नेप":393,"नेत":472,"नेक":552,"बई ":387,"नही":1669,"नसभ":406,"निव":567,"निर":2352,"निय":1801,"निध":617,"नित":1221,"नाव":392,"निक":2637,"नाय":738,"नाम":2777,"नार":680,"नान":715,"नात":423,"नाथ":458,"नाट":450,"नाड":359,"नाग":557,"द्द":569,"द्व":5706,"द्र":2473,"द्य":2403,"द्म":1046,"द्ध":2842,"धर्":969,"नका":1135,"नकी":794,"नके":1054,"देख":471,"देव":1125,"देश":3481,"दों":393,"दोन":372,"धित":713,"धिय":418,"धार":1754,"धिक":1763,"धान":1811,"पर ":7035,"नता":441,"नते":555,"नगर":1205},"n_words":[3436892,4107546,2722787],"name":"hi","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/hr.json b/contrib/languages-data/hr.json
new file mode 100644
index 0000000..c42497a
--- /dev/null
+++ b/contrib/languages-data/hr.json
@@ -0,0 +1 @@
+{"freq":{"D":8690,"E":5169,"F":5109,"G":8706,"A":14720,"B":12556,"C":10193,"L":6822,"M":14153,"N":10511,"O":10729,"H":9113,"I":13451,"J":5359,"K":11989,"U":4743,"T":10711,"W":3179,"V":5850,"Q":585,"P":15109,"S":23054,"R":9319,"Y":1450,"X":701,"Z":4251,"f":16193,"g":88687,"d":148916,"e":436858,"b":60942,"c":64868,"a":586818,"n":342270,"o":415815,"l":177559,"m":139424,"j":258985,"k":204303,"h":49394,"i":501698,"w":7347,"v":140168,"u":209613,"t":201984,"s":238514,"r":271701,"q":1755,"p":126499,"z":85944,"y":9646,"x":2022,"í":886,"é":1412,"á":1674,"ó":546,"đ":8171,"ć":16773,"Č":895,"č":46294,"Ž":887,"ž":23555,"Š":1432,"š":26211," l":9617," m":20259," n":49295," o":45630," h":4791," i":61249," j":56434," k":46966," d":31478," e":6507," f":5286," g":21986,"р":912," a":19142,"с":742," b":17477,"т":629," c":5752," z":19710," u":45274," t":21468," w":820," v":15539," p":67380," s":79773," r":20001," J":5152," K":11465," H":8434," I":11848," N":9837," O":7822," L":5986," M":13514," B":11861," C":9435," A":13018," F":4631," G":8304," D":7573," E":4598,"л":648," Z":4104,"к":817," Y":1407," X":526,"и":1209,"о":1253,"н":943," S":19621," R":8667,"в":563," Q":537," P":14387,"а":1816," W":3048," V":5326," U":4477,"е":917," T":9972," č":5596," Č":887," ž":5028," Ž":838," Š":1403," š":4901,"A ":857,"Da":1321,"Co":1834,"Cr":1328,"Ce":588,"Ch":1843,"Du":764,"Do":1328,"Dr":1133,"De":1138,"Di":1002,"Fe":631,"H ":574,"Fa":563,"Eu":879,"Ge":716,"Ga":972,"I ":821,"Fr":1113,"Fo":779,"Fi":649,"Au":808,"Ar":1684,"At":729,"As":560,"D ":829,"Ba":2710,"Af":545,"Am":1341,"An":1467,"Al":1655,"Bu":1011,"Br":2042,"Ca":2160,"Bi":1321,"Be":1685,"Bo":2115,"Ku":844,"Kr":1748,"Ko":2488,"Le":881,"Li":1283,"La":1685,"Lu":593,"Lo":1036,"Me":1818,"Mi":1887,"O ":2633,"Ma":5853,"Mu":793,"Mo":1972,"Nj":820,"Ni":1096,"Ne":1404,"Na":3401,"No":2003,"Ob":956,"Gl":526,"Gr":2253,"Go":1536,"Gu":881,"Gv":612,"Ha":1340,"He":1374,"Hi":570,"Ho":905,"IS":2154,"Hr":2959,"Hu":536,"In":4619,"Is":978,"Ja":1450,"L ":681,"Iz":610,"Je":1026,"Jo":711,"Ju":1563,"Ka":3681,"Ki":952,"Tu":1072,"Tr":1314,"To":1609,"Th":1212,"Ti":993,"Te":1282,"Ta":1685,"St":1916,"Sv":940,"Su":1303,"Wi":722,"Wa":1107,"Vo":662,"Vi":1040,"Va":1022,"Ve":1522,"Pu":746,"Pr":3018,"S ":720,"Pe":1604,"Pa":2898,"Pl":816,"Po":3639,"Pi":867,"Os":790,"Ov":672,"Op":1011,"Or":735,"Se":1283,"Sj":776,"Si":1385,"Sh":659,"Sl":1203,"Sr":1393,"Sp":846,"So":1277,"Ru":1026,"U ":1401,"Sa":3386,"Re":2977,"Ri":1491,"Ro":1097,"SO":2074,"Ra":1418,"b ":1522,"a ":202747,"Za":2365,"Ze":593,"i ":135497,"ađ":2045,"gd":857,"ge":5429,"ga":10157,"ać":1902,"fi":4106,"fs":661,"fr":2013,"ač":10342,"fu":639,"fo":2404,"j ":16449,"he":4881,"ha":5859,"gn":997,"gl":6895,"gi":6788,"gh":714,"gu":4729,"gr":14473,"go":15835,"du":5516,"dv":2741,"g ":19299,"ea":3988,"eb":4045,"ec":3773,"ed":22606,"de":10224,"dg":624,"di":30316,"dj":3005,"dm":1005,"dl":806,"do":12624,"dn":15638,"ds":4759,"dr":13255,"ew":844,"eu":1755,"ev":10791,"ey":838,"ez":15435,"fa":1476,"h ":20168,"fe":2011,"eh":1279,"eg":7983,"ef":1181,"ee":1826,"el":24518,"ek":14461,"ej":2546,"ei":1534,"ep":5699,"eo":3348,"en":44674,"em":19226,"et":18394,"es":16599,"er":32266,"ca":14154,"e ":160018,"br":6048,"bu":5481,"bn":1412,"bo":6809,"bj":1695,"bl":6540,"bi":13442,"be":6031,"db":626,"da":22107,"f ":1430,"cu":2623,"ct":683,"cr":1908,"co":2679,"ck":2130,"ci":21232,"ch":4164,"ce":10558,"c ":2983,"az":15697,"ay":1948,"ba":10149,"d ":22851,"at":29447,"as":22343,"ar":30709,"aw":1498,"av":28227,"au":3869,"ak":15591,"al":35013,"ai":3078,"aj":16895,"ao":6448,"ap":10100,"am":17849,"an":70440,"ac":13980,"ad":21936,"aa":799,"ab":3879,"ag":6276,"ah":2814,"ae":1668,"af":2077,"nu":8144,"nt":10450,"ns":14593,"ič":14290,"no":45797,"nn":1399,"nz":952,"ny":715,"oe":678,"of":2150,"oc":4342,"od":40790,"oa":2451,"ob":12953,"om":28810,"on":23346,"ok":14795,"ol":17826,"oi":3961,"oj":40426,"og":22833,"oh":1159,"ot":12114,"os":23349,"ov":31034,"ou":2922,"op":11191,"oo":1568,"or":33494,"jč":526,"r ":11028,"ow":1054,"oz":7848,"pe":6182,"pa":15403,"pl":8816,"pn":1287,"po":36246,"ph":606,"pi":11043,"pj":1476,"lo":18651,"ln":5989,"lm":1222,"ll":3111,"ls":2651,"lu":8623,"lt":2358,"o ":61519,"ma":31717,"mb":2862,"dž":894,"me":24452,"ml":1963,"eš":3371,"mi":10591,"mj":5104,"mn":1665,"mp":2261,"mo":11753,"mr":1000,"ms":2000,"mu":3945,"p ":2149,"na":87240,"nc":5972,"nd":11672,"ne":33576,"nf":891,"ež":1706,"ng":7899,"ni":54450,"nj":27833,"nk":2801,"ić":2802,"jv":1605,"eđ":3970,"ju":19590,"eč":2671,"js":8691,"jn":2724,"jo":4699,"eć":4631,"jk":855,"ki":28341,"ke":16969,"kc":1026,"ka":37663,"m ":37899,"ks":2734,"kt":4893,"ku":14523,"kv":1615,"ko":63668,"kr":10108,"kl":4333,"km":1244,"kn":1083,"li":41243,"lk":741,"lj":24378,"le":17764,"ld":889,"lg":643,"la":37196,"lb":2414,"n ":24345,"hr":2448,"hv":1089,"ht":807,"hu":1937,"hi":4652,"hn":862,"ho":4536,"id":4699,"ic":20781,"ib":2814,"ia":4852,"ih":18991,"ig":5433,"if":1923,"ie":2060,"k ":14010,"ir":12272,"is":21664,"it":16417,"iu":848,"iv":14019,"aš":3959,"ij":57037,"ik":23850,"il":22797,"im":27723,"in":47299,"io":10059,"ip":4978,"je":134763,"až":1877,"ji":26676,"iz":19026,"l ":6966,"ja":39549,"z ":7997,"ož":2534,"oš":2893,"wi":861,"rč":1709,"y ":3377,"wa":2878,"we":956,"vl":4881,"vj":3877,"vk":562,"vi":24026,"vu":3419,"vr":6622,"vs":1342,"vn":9959,"vo":21455,"uz":3112,"uv":1349,"ve":22017,"vc":593,"va":35406,"x ":785,"ui":1893,"uj":4079,"uk":4559,"ul":5933,"pć":2758,"ue":1973,"ug":8425,"uh":1544,"ur":9708,"us":10059,"ut":7404,"um":7672,"un":8708,"uo":923,"up":11863,"tu":9961,"tt":1393,"tv":7233,"ođ":1449,"ub":5211,"ua":3882,"ud":5733,"uc":1692,"w ":858,"to":28843,"tn":5773,"tm":575,"tl":1860,"oć":686,"ts":8790,"tr":16932,"oč":4671,"tp":896,"te":22658,"tk":2453,"tj":1700,"ti":34096,"th":3097,"v ":4865,"ta":38983,"su":13292,"sv":6451,"ss":1881,"st":55588,"sl":7897,"sk":52271,"sn":7159,"sm":3278,"sp":5046,"so":5247,"sr":3449,"sc":1522,"se":24429,"sh":2381,"sj":4697,"si":10381,"rz":1007,"u ":87089,"sa":16094,"rr":1162,"rs":8936,"rt":5687,"ru":17434,"rv":8435,"ry":930,"rp":1334,"ro":34834,"rn":10194,"rm":2849,"rl":1655,"rk":2924,"rj":894,"ri":44000,"rh":1312,"rg":3269,"iž":1025,"re":32485,"rd":3070,"rc":1885,"rb":2322,"ra":64303,"t ":14252,"qu":1390,"iš":6125,"s ":15910,"pt":1447,"pu":6628,"pp":658,"pr":27731,"ps":2712,"už":5556,"uš":2233,"rž":4670,"zg":1072,"rš":1722,"zi":22879,"zb":2089,"zd":2018,"ze":4698,"za":19270,"zv":4353,"uč":5083,"zr":2246,"zu":2322,"zo":3146,"zn":8371,"zm":2468,"zl":2200,"uć":2114,"ye":523,"ya":1738,"yo":520,"á ":773,"é ":601,"ć ":1732,"ći":5749,"ću":1057,"ća":4248,"će":3354,"či":7487,"čj":2822,"čk":13325,"čl":554,"če":5735,"ča":5202,"č ":1473,"đe":2942,"đa":1580,"čn":7182,"ču":1818,"đu":2924,"š ":1063,"šć":1355,"še":2480,"ša":1860,"šp":635,"šn":3132,"šk":2964,"šl":591,"ši":3525,"šu":567,"št":7272,"žu":2110,"žn":3428,"žb":852,"že":4351,"ža":6201,"ži":5092,"đen":1909,"đer":597,"đu ":1688,"đun":708,"čun":831," Ga":961," Ge":702," Fo":742," Fr":1109," Fi":643," Ha":1336," He":1372," Go":1534," Gr":2247," Gu":878," Gv":612," Gl":525," Hu":528," IS":2110," Hr":2946," Ho":895," Hi":565," Je":1023," Ja":1447," Iz":607," Is":977," In":4605," Ka":3666," Ki":914," Jo":707," Ju":1556," La":1670," Le":867," Li":1265," Ko":2481," Kr":1745," Ku":843," Ma":5817," Mi":1882," Me":1804," Lo":1032," Lu":587," Ne":1391,"а ":562," Na":3399," Nj":820," Ni":1084," Mo":1968," Mu":787," Am":1333," An":1450," Al":1637," Af":533," Ba":2689," Au":794," At":724," As":558," Ar":1651," Be":1676," Bi":1296," Bo":2110," Br":2033," Bu":1004," Ca":2140," Ce":585," Ch":1828," Cr":1326," Co":1819," Da":1296," Di":998," De":1131," Dr":1125," Do":1300," Du":759," Eu":878," Fe":623," Fa":554," Wi":714," Wa":1098," Ze":593," Za":2362," a ":4274," Ov":665," Os":789," Or":732," Op":1008," Po":3623," Pl":812," Pi":864," Pe":1599," Pa":2881," No":2002," Ob":956," Ra":1412," Ro":1089," Re":2970," Ri":1491," Pr":3007," Pu":743," Sv":937," Su":1301," St":1875," Ta":1677," Th":1189," Ti":990," Te":1277," Tr":1295," To":1604," Ru":1022," Sa":3371," U ":1282," Sh":655," Si":1373," Sj":776," Se":1273," So":1270," Sp":836," Sr":1393," Sl":1201," Va":1018," Ve":1517," Vi":1035," Vo":661," Tu":1066," ja":1334," iz":11704," je":51007," im":3911," in":3359," il":7148," is":4046," ka":7800," ki":1128," jo":810," ju":3163," ha":604," he":608," gl":3276," gr":8161," go":6544," ih":786," ig":1120," hi":648," ho":696," hr":1744," nj":3220," ni":1874," ne":6451," na":34503," mu":1067," mo":4787," mn":528," ok":4011," on":1068," od":15714," of":856," ob":6528," no":2407," le":827," lj":2291," li":2543," la":2172," ku":1972," kn":829," km":1048," kl":1410," kr":4664," ko":26854," me":3676," mi":2188," mj":1905," o ":1165," ma":4891," lo":1177," am":2645," an":1498," ak":848," al":3416," au":1442," ar":1418," at":615," ba":2490," bi":5965," be":1110," bo":2421," bl":1053," bu":691," br":3222," ca":552," et":559," en":2154," el":1166," ek":584," fa":621," fr":941," fo":883," fi":1669," ge":1007," gd":551," ga":1134," i ":27993," cr":1408," ce":1531," ci":1052," da":4574," do":6878," dr":6375," de":2927," dj":1233," di":5389," dv":2035," du":1555," zn":2259," zr":767," zv":702," za":12881," ze":1108," zb":633," už":620," ru":1327," u ":32404," sa":9931," se":16886," sj":3429," si":2544," sn":698," sm":1890," sl":3915," sk":4532," sr":2957," sp":2436," so":944," ra":7698," re":4140," ri":4141," ro":2280," pu":1716," pr":23703," s ":3500," os":3950," ot":2810," ov":1276," op":3149," or":1765," oz":1224," pe":1664," pa":2832," pl":5604," po":28804," pi":966," pj":1218," va":1250," ve":3149," uz":1739," vo":2912," vr":3315," vi":2766," vj":724," vl":1147," ud":648," tv":902," tu":1007," us":1218," ut":721," ur":562," up":1243," um":849," un":999," uk":1055," ul":803," ug":653," ta":2808," st":9162," sv":5803," su":10580," tr":4273," to":2817," th":929," ti":2177," te":5426," če":1938," čl":530," či":1960," št":1607," ši":1148," šp":620," ži":2289," že":880," žu":1586,"Eur":771,"Fra":831,"Her":776,"Gra":1106,"Gor":695,"Ind":3698,"Hrv":2864,"ISO":2034,"Ara":578,"šće":547,"Ame":663,"Car":617,"Bra":541,"Bri":609,"Bos":708,"Chi":621,"Cha":594,"Nal":560,"Nje":555,"Nov":1094,"Opć":766,"Per":568,"Par":726,"Pro":696,"Pri":721,"Pre":828,"Juž":532,"Kal":564,"Kan":524,"Kar":752,"Kra":872,"Man":821,"Mal":573,"Mar":1189,"Mad":576,"Zag":834,"Sta":754,"Sje":767,"Srb":562,"Sre":533,"Slo":726,"Rus":582,"San":791,"Rep":1934,"SO ":2049,"Vel":771,"The":928,"šen":553,"še ":976,"što":1559,"šte":2571,"šti":879,"šta":884,"štv":1041,"ško":1181,"šnj":2421,"šin":692,"šir":1400,"ški":826,"bje":654,"bja":1027,"bit":2079,"biv":585,"bio":1738,"bil":3046,"bin":696,"bij":1580,"blj":1003,"bli":4296,"bla":688,"bol":1257,"boj":638,"bog":903,"bič":654,"bno":610,"bor":1232,"be ":1311,"ban":1537,"bal":1792,"bav":860,"bar":843,"bi ":1256,"ber":918,"ben":1811,"ca ":10450,"car":656,"can":1075,"ce ":5505,"bri":882,"bro":2066,"bra":1828,"bu ":732,"bum":1939,"buh":796,"aka":2848,"am ":2209,"ake":1450,"aki":683,"aji":1629,"ajn":1118,"ajs":690,"aju":3559,"ajv":1325,"al ":2419,"aja":1758,"aje":2556,"ain":623,"ak ":2449,"aj ":1426,"aha":673,"agr":1324,"agu":525,"ago":1080,"anu":1384,"ano":4034,"ant":2834,"ans":7065,"ane":2342,"ang":1485,"ani":9755,"anj":10858,"ank":1213,"ana":11419,"anc":2218,"and":3683,"amo":1853,"ami":1538,"ame":3781,"amb":801,"ama":5199,"ao ":5760,"alu":766,"alt":538,"als":661,"alo":1706,"aln":4903,"all":806,"ali":7309,"alj":2503,"ale":2883,"ala":6700,"alb":1948,"an ":10347,"aku":618,"akt":1211,"ako":4291,"aba":762,"abi":791,"ae ":719,"aca":3812,"ad ":3345,"ac ":1484,"afs":539,"afi":554,"ai ":709,"aga":937,"age":778,"ado":1169,"adr":1028,"adn":3155,"adi":2550,"ade":1153,"ads":935,"adu":1422,"aci":6004,"ach":727,"ada":5689,"azn":685,"azi":7453,"azl":1063,"azv":1013,"azu":592,"aze":804,"aza":1040,"azb":1010,"aya":527,"ba ":2218,"at ":2601,"arh":564,"are":1147,"ard":1409,"ara":6301,"aro":3575,"arn":1712,"ark":1048,"ari":4189,"aru":579,"ars":3100,"art":1393,"asa":826,"asi":1226,"ash":669,"ase":3354,"asn":1043,"asp":540,"ask":1064,"asl":534,"ar ":2545,"apa":4410,"api":1243,"apo":975,"apr":526,"aps":650,"apu":900,"as ":2262,"ava":8033,"aut":1108,"avo":1516,"avn":4358,"avl":3013,"avi":4083,"ave":3087,"ay ":531,"awa":719,"avu":1466,"av ":1100,"ata":3446,"ast":9058,"atn":1424,"atk":562,"atr":1028,"ato":2245,"ate":2791,"ati":6984,"ats":4995,"atu":1171,"aus":601,"jeg":2764,"jed":10627,"jec":1103,"jer":2418,"jek":4521,"jel":6299,"jem":5086,"jen":9322,"jez":10141,"jes":4061,"jet":3631,"jev":7451,"jač":750,"ji ":16045,"ažn":588,"jat":632,"jav":2585,"jal":1858,"jak":822,"jan":5965,"jam":1147,"je ":61320,"ješ":1621,"jni":1204,"joj":1250,"jom":1082,"jiv":803,"jim":3670,"jin":1845,"jih":2364,"ječ":1963,"jeć":1172,"eća":1346,"eće":940,"eći":1617,"ito":2406,"itu":706,"its":544,"isk":1391,"isl":604,"iso":539,"isn":807,"isp":569,"isu":779,"ist":10340,"iv ":1925,"ita":3250,"ite":2457,"iti":4001,"ivo":1642,"ivn":1941,"iva":3738,"ivi":1447,"ive":1512,"is ":1700,"ion":3221,"ipa":1930,"ir ":678,"iro":1457,"iri":1540,"isi":976,"ish":564,"ise":556,"isc":574,"isa":1555,"ire":1683,"ira":5142,"it ":890,"ja ":23119,"iz ":4867,"izu":877,"izv":2232,"izr":793,"izo":774,"izn":759,"izm":2019,"izl":668,"izi":2042,"izd":806,"iza":2198,"kih":5870,"kim":2628,"kin":753,"km ":900,"ki ":16671,"ked":578,"ke ":14258,"kci":941,"kra":4133,"kre":1063,"ku ":4982,"kro":1000,"kru":1711,"kri":1186,"kov":2238,"kot":547,"kor":2342,"kop":1083,"kon":3505,"kom":6854,"kol":2347,"koj":25320,"kog":6388,"kod":903,"knj":735,"ko ":9564,"kla":1495,"klo":701,"klj":755,"eđu":2748,"još":665,"jve":805,"eđe":812,"juj":863,"jug":1520,"jud":1914,"jsk":7836,"jst":574,"eči":628,"ju ":11452,"eče":573,"kaz":825,"kat":974,"kar":1304,"kas":686,"kan":1885,"kao":3422,"kal":1067,"kam":671,"kak":633,"kad":1513,"kac":594,"juž":1378,"ka ":22352,"juč":696,"juć":551,"ha ":913,"han":1101,"har":926,"he ":1979,"her":717,"hin":716,"hit":527,"go ":855,"gle":1361,"gla":3728,"god":2754,"gom":912,"gon":803,"gos":769,"gor":1668,"gov":5098,"gu ":1902,"gru":1369,"gra":9421,"gre":1427,"grč":1302,"ian":1493,"ic ":604,"iba":598,"ibe":551,"ia ":1869,"ifi":824,"ih ":17225,"icu":773,"ici":6028,"ich":645,"ice":4697,"ie ":604,"ica":6876,"idi":521,"ide":1150,"ida":944,"ašn":1360,"ašt":576,"il ":644,"ija":16185,"ije":24598,"iji":7426,"ijo":787,"ijs":5406,"iju":2114,"im ":10462,"ika":7961,"ige":790,"iga":952,"igi":547,"igr":1302,"iho":1185,"ik ":6489,"imo":723,"imp":613,"ime":3556,"imi":856,"inc":1602,"ind":1666,"ina":9909,"ino":2039,"int":1177,"ins":3389,"ine":10127,"ing":2574,"inj":1865,"ini":6764,"inu":1508,"iko":2904,"iki":780,"ike":2426,"ila":3734,"in ":2960,"iku":1705,"ilo":2029,"ill":863,"ilm":540,"ilj":1386,"ili":10385,"ile":865,"ima":9207,"io ":4778,"ils":642,"ilu":562,"hov":1213,"hrv":1503,"hva":1061,"fer":566,"ez ":610,"ezu":527,"eza":1009,"ezn":818,"eze":1246,"ezi":9990,"eta":3704,"ete":898,"eti":2144,"etn":2390,"etl":530,"etk":539,"esn":1243,"est":6673,"eto":1100,"etr":912,"ets":1480,"etu":996,"etv":760,"eve":4353,"eva":1803,"evo":961,"evn":623,"evi":1701,"er ":4432,"epa":580,"es ":2168,"epu":2109,"epo":647,"epr":650,"eri":7215,"erg":698,"ere":1639,"erc":963,"era":4007,"et ":2066,"esk":1461,"esm":595,"esi":709,"ese":1073,"esa":706,"erv":605,"eru":1435,"ert":600,"ers":1367,"ern":3520,"erm":720,"ero":2614,"eki":802,"eko":2778,"eks":1589,"ekt":2568,"eku":789,"en ":4377,"ela":3182,"ele":2918,"eli":4944,"elj":5961,"ell":698,"elo":2967,"elu":1458,"ema":3624,"eme":6131,"eml":1258,"emo":840,"emi":1481,"ene":3561,"eng":1640,"ena":7266,"end":836,"enc":1218,"eno":6088,"eni":7665,"enj":3407,"enu":1213,"ens":1955,"ent":3912,"ego":1890,"egi":1521,"ek ":753,"el ":1052,"ejs":573,"eke":1782,"eka":2350,"em ":3991,"gl ":618,"gij":2556,"gin":523,"gi ":831,"gen":1325,"ger":827,"gdj":702,"ge ":1704,"gar":946,"gal":662,"gan":1740,"ga ":4609,"ađa":855,"ađe":658,"fra":927,"ača":1970,"ače":950,"ačk":3501,"fri":721,"ači":1814,"fsk":622,"aču":958,"for":1658,"ač ":583,"aće":585,"aća":1021,"fil":896,"fik":566,"fin":592,"da ":9164,"de ":2788,"dal":1347,"daj":785,"dat":824,"dar":1385,"dan":5150,"dam":626,"dav":567,"cus":960,"co ":549,"cu ":955,"cea":713,"ch ":519,"ces":1492,"cen":997,"ceg":685,"ci ":6216,"cha":750,"ck ":1290,"che":856,"chi":796,"cij":9280,"cim":1128,"cir":569,"cio":1059,"ed ":1254,"eba":923,"ebe":581,"ean":810,"ea ":773,"duž":619,"ega":1085,"edn":7588,"edi":4719,"ede":1425,"eda":3130,"eg ":2170,"eds":1513,"edo":1202,"eci":935,"eca":935,"dvi":741,"dvo":690,"dva":1103,"drž":4140,"don":1767,"dom":1052,"dol":970,"dok":659,"dov":1189,"dos":617,"diš":1956,"dna":2624,"dne":953,"dni":3564,"dnj":1612,"dno":6490,"dob":1518,"dst":1252,"dra":1195,"dre":1145,"du ":2280,"dru":5517,"dsk":2664,"dic":3889,"der":1371,"des":966,"den":1450,"di ":4014,"do ":2419,"dje":2862,"dim":541,"din":5519,"dio":1844,"dis":1126,"dij":7896,"rađ":876,"rga":1240,"ri ":5834,"rgi":678,"ret":1800,"res":1271,"rev":690,"rez":931,"rać":609,"rač":1555,"reb":1913,"rea":805,"ree":521,"red":6133,"reg":1745,"rem":2802,"ren":2797,"rek":1056,"rel":612,"rep":724,"rdi":692,"re ":4525,"rce":928,"raz":4813,"rd ":686,"rap":1294,"ras":1520,"rat":4060,"rav":4237,"rbi":882,"raj":3616,"rag":678,"ran":11269,"ram":1816,"ral":3163,"rak":1775,"raf":1147,"rad":6716,"rac":1934,"ros":2073,"rot":1439,"rom":2618,"ron":1652,"rop":1763,"roz":1266,"rov":2438,"rob":584,"roa":633,"rod":8265,"roc":1470,"roj":2300,"roi":1622,"rol":793,"rok":933,"rog":1195,"rno":2932,"rič":3609,"rna":1479,"rne":1371,"rnj":845,"rni":2710,"ro ":1317,"rma":1223,"rmi":655,"rlo":551,"rn ":540,"rkv":521,"rka":695,"ređ":981,"reć":667,"raž":797,"rje":601,"riz":799,"rip":1837,"rio":889,"rir":879,"rit":2510,"ris":3174,"riv":1004,"rig":523,"rij":11045,"raš":597,"ril":1430,"rik":1383,"rin":1848,"rim":1849,"ria":651,"rib":805,"ric":1431,"rid":583,"rug":4029,"rup":1471,"run":520,"ruk":844,"rus":1028,"rva":5151,"rvi":1279,"rve":1037,"rvo":605,"ry ":556,"rsk":5201,"rta":726,"rst":2436,"rti":871,"rt ":714,"ru ":2471,"sad":738,"sam":1694,"san":1142,"sat":604,"sas":3039,"sav":1014,"sa ":5478,"ruž":889,"ruš":721,"ruč":2722,"rzi":592,"shi":607,"si ":1399,"sje":4603,"sis":550,"sin":1783,"sil":686,"sim":836,"sij":991,"se ":14671,"ser":1178,"set":665,"sh ":559,"seb":745,"sel":4338,"spo":2263,"spr":578,"spe":611,"spa":672,"sov":613,"son":650,"sob":1342,"su ":8627,"sre":2282,"st ":3926,"slj":632,"sli":1331,"slo":1832,"slu":1752,"sla":2093,"ski":17159,"skl":527,"sko":15307,"skr":553,"sku":5305,"ska":7019,"ske":5955,"sno":3094,"sna":922,"sni":2255,"sne":697,"smj":1004,"sma":1116,"ste":2346,"sta":16592,"sto":9996,"sti":9753,"stv":3654,"stu":2357,"str":6456,"sus":1535,"sva":788,"sve":1415,"svi":1441,"svj":804,"svo":1576,"taj":996,"tak":2161,"tal":4191,"tac":782,"tav":5990,"tat":1128,"tar":4163,"tan":6149,"tam":583,"te ":6089,"ta ":9339,"pa ":1847,"pe ":1308,"par":1595,"pat":647,"pad":4813,"pal":580,"pan":3273,"pi ":571,"per":1812,"pet":668,"pla":1804,"plj":816,"pli":1041,"ple":3986,"plo":1052,"pje":1454,"pij":889,"pin":4572,"pis":2255,"poz":2191,"por":5631,"pop":1191,"pov":2463,"pot":1881,"pos":3892,"poj":1457,"pog":612,"pom":933,"pon":975,"pok":1642,"pol":3220,"pod":6429,"po ":2082,"psk":1887,"pub":2165,"pti":586,"poč":712,"pra":3170,"prv":1850,"pri":7027,"pre":6249,"pro":8677,"put":1060,"pun":676,"pul":698,"iše":1062,"išn":1267,"išt":2503,"qui":570,"ra ":9271,"ngo":698,"ngl":1753,"ni ":16130,"nga":751,"nej":882,"nek":2622,"nen":652,"nep":734,"ner":1509,"net":1026,"nes":1293,"ng ":2367,"nač":2757,"nez":1162,"nci":2911,"nce":931,"ne ":19985,"ndu":534,"ndo":1052,"ndi":5037,"nde":837,"nda":1627,"ncu":940,"nak":2307,"nal":5099,"nam":1658,"nan":1732,"nap":971,"nar":3446,"nac":3781,"nad":1618,"nag":771,"naj":3818,"nd ":1313,"nav":631,"nat":3622,"nas":6912,"naz":2805,"na ":42477,"mož":877,"nut":1280,"nto":781,"ntr":1002,"nti":2109,"nta":2295,"nte":1843,"nst":1771,"nsk":10758,"nu ":5412,"ičn":3327,"ičk":8118,"iči":1329,"iča":967,"nt ":937,"niš":631,"ns ":584,"noa":520,"nog":6035,"noj":3137,"nom":6601,"nos":6166,"nor":535,"nov":4240,"nič":2517,"ića":608,"no ":15032,"nka":782,"nji":3978,"nje":12914,"nja":6205,"ić ":1370,"nju":1904,"njs":879,"njo":1594,"nij":6542,"naš":1014,"nih":6201,"nic":5054,"niz":1589,"nis":1359,"nit":879,"nir":714,"nim":4415,"nin":1303,"nik":4325,"ogr":2375,"ogu":995,"ogi":1476,"ogl":635,"ogo":1802,"oga":1651,"oj ":12832,"ois":1145,"oim":653,"ok ":1418,"oju":1294,"ojs":576,"ojo":759,"ojn":1052,"oji":10382,"oje":6734,"oja":6046,"ol ":664,"oiz":1012,"oce":973,"oci":1114,"ock":1115,"obu":893,"ode":1274,"odi":8132,"odo":921,"odn":4204,"ods":1138,"odr":4259,"of ":888,"oda":2801,"odu":910,"og ":13004,"oan":559,"oba":2782,"od ":14807,"obo":933,"obr":1117,"obl":1881,"obn":555,"obj":1316,"obi":2599,"oz ":529,"ozn":3312,"ozi":1205,"oza":1421,"oti":1557,"ote":985,"otr":921,"otp":614,"oto":4095,"ost":8908,"ota":912,"ov ":1004,"osi":1554,"ose":1141,"osl":1958,"oso":1519,"osn":2717,"ovj":940,"ovi":7022,"ovn":2887,"ovr":829,"ovo":6487,"ova":7007,"ove":3130,"opć":1972,"opo":690,"opi":1778,"opl":1169,"ope":858,"opa":775,"os ":1176,"opu":936,"opr":664,"ops":861,"or ":2008,"orm":1284,"orn":2821,"oro":4229,"ord":815,"ore":2892,"org":1291,"ori":7402,"osa":1092,"ort":1833,"ors":1607,"oru":1306,"ot ":522,"ora":3600,"ola":1856,"on ":4329,"olj":2400,"oli":4446,"ole":1119,"ols":789,"olo":3261,"olu":1664,"oka":2684,"om ":17269,"oke":734,"okr":3164,"oko":3273,"oku":2145,"ona":3954,"ond":619,"one":1941,"ong":1053,"onj":777,"oni":2936,"ono":2047,"ons":1609,"ont":1171,"onu":585,"oma":2281,"ome":2911,"omi":1374,"omp":729,"omo":1785,"la ":9649,"le ":3757,"lac":1275,"lad":1494,"lag":644,"lak":731,"lan":3996,"lam":781,"lar":880,"lat":2578,"las":3015,"lav":4030,"laz":5016,"lbu":1947,"kva":701,"kup":5219,"kul":1164,"ksi":936,"ktr":973,"kođ":596,"ktu":654,"kti":1269,"kuć":703,"lok":712,"lon":791,"lom":1506,"lop":558,"log":2016,"lov":4118,"lno":1832,"lni":2255,"lne":712,"lob":538,"lič":1936,"lna":885,"ltu":737,"lub":608,"lsk":2111,"lu ":2550,"liš":638,"lj ":865,"li ":12760,"les":1936,"let":657,"lem":4158,"len":1436,"lek":2007,"led":614,"lo ":4167,"lla":657,"lle":663,"lli":529,"ljs":545,"lju":4071,"lje":10652,"ll ":619,"lja":4832,"lji":2209,"lit":2066,"lis":1339,"lin":3083,"lim":1508,"liz":1420,"lic":2147,"lia":609,"lik":6521,"lij":3213,"lig":541,"ma ":15645,"luž":1294,"mac":714,"maj":623,"mak":533,"mar":837,"mas":812,"mal":2079,"man":3666,"mat":2524,"mbi":905,"me ":5240,"med":673,"met":3414,"mer":3791,"mel":710,"men":6548,"mač":1334,"lum":944,"loš":641,"mpi":753,"mog":976,"mon":1050,"mor":2726,"mos":601,"mot":653,"mu ":1457,"msk":1673,"mun":787,"mi ":1004,"međ":2505,"mje":5038,"min":2090,"mil":697,"mir":1036,"mis":832,"ešt":1289,"mit":939,"ešk":669,"mij":1629,"mo ":1420,"mlj":1522,"mno":900,"rža":4071,"ča ":974,"čan":800,"čar":558,"čaj":693,"uča":545,"zra":1677,"če ":853,"učj":2470,"uči":687,"čav":1111,"čen":1159,"čet":1250,"čes":595,"zu ":768,"zva":612,"zvi":939,"či ":1484,"zvo":1982,"zum":625,"čij":837,"čic":566,"čit":844,"čin":2554,"čko":4001,"čka":1912,"čke":2028,"čki":5096,"čju":1626,"češ":556,"čla":542,"čni":1699,"čno":3306,"čna":751,"čne":1006,"zi ":3239,"zem":1449,"zer":1084,"ze ":973,"zbo":641,"zda":1154,"zac":771,"zbe":843,"zaj":1076,"zam":819,"zan":918,"zal":859,"zap":3007,"zav":531,"zas":565,"zon":794,"zme":1381,"zna":6328,"zno":753,"zič":837,"zni":942,"zla":762,"uća":616,"zli":1013,"ući":766,"zic":1241,"zij":2320,"rši":715,"zin":1079,"zil":1675,"zik":7193,"zir":891,"ziv":3073,"za ":7407,"ya ":644,"ože":1175,"ću ":879,"ćin":2919,"ći ":2008,"rčk":1144,"oš ":708,"ošk":698,"vrš":1066,"wa ":713,"viš":1245,"vrt":868,"vrs":1358,"vri":916,"vre":806,"vsk":989,"vu ":2689,"vir":547,"vil":1092,"vim":1483,"vin":3680,"vih":903,"vij":3997,"vic":888,"vid":903,"vit":817,"vis":1291,"već":1856,"vje":3831,"vla":1440,"vlj":3383,"vo ":3548,"vne":801,"vna":1303,"vno":3450,"vić":621,"vni":4024,"vod":2717,"vog":1164,"voj":3212,"vol":582,"vom":1226,"vor":5059,"vot":928,"voz":1030,"vi ":4716,"vač":1765,"vez":1580,"ver":4932,"vet":804,"vać":849,"ven":3207,"vel":1857,"ved":700,"ve ":5319,"val":1770,"vak":779,"van":5697,"var":1878,"vat":5563,"vac":751,"vaj":1521,"va ":12711,"uz ":1098,"usk":1818,"usi":743,"ust":3150,"uti":590,"ute":706,"uta":1448,"uto":1403,"us ":1654,"ut ":910,"ura":1400,"ure":1189,"uri":1158,"urn":947,"uro":1486,"urs":550,"uru":580,"upa":2662,"upi":4500,"upe":980,"upo":759,"upr":748,"upn":605,"umi":603,"umj":671,"uma":1379,"umb":688,"ume":724,"unu":749,"uni":993,"und":654,"una":2557,"up ":535,"uko":757,"ukl":657,"um ":2138,"uka":702,"uju":1023,"ult":1215,"pći":2432,"uli":687,"ula":1753,"uhv":776,"uje":2393,"ugi":1126,"uge":731,"ugo":2301,"ugl":583,"uga":1630,"ugu":1137,"uda":924,"udi":2722,"ue ":562,"uci":556,"ug ":588,"ua ":834,"ual":550,"uan":840,"ubl":2337,"tvu":537,"tvr":1039,"tvo":2139,"tve":1133,"ođe":1121,"tva":2106,"tur":2517,"tup":876,"tud":1103,"tre":2014,"oče":731,"tra":6680,"će ":1908,"oči":564,"tri":3139,"tru":1417,"tro":2790,"očn":2463,"tu ":3006,"tsk":7605,"toč":2840,"ćen":895,"to ":5478,"tni":2342,"ća ":3122,"tna":778,"tič":3116,"tno":1901,"toc":579,"toj":1710,"toi":723,"tog":984,"tov":1949,"tom":1889,"ton":1847,"tok":3601,"tol":1732,"tor":3179,"top":785,"ćan":554,"tij":2254,"til":813,"tik":1009,"tih":692,"tir":1327,"tit":1406,"tis":916,"tin":3058,"tim":1630,"tio":1426,"tic":2096,"tiv":2430,"tje":1395,"tko":583,"tka":775,"tla":700,"tem":1590,"ten":2148,"tek":931,"tel":2393,"th ":723,"tet":818,"ter":4781,"ti ":9529,"the":1111,"živ":2628,"žni":894,"žno":1666,"že ":1842,"žbe":582,"žav":3922,"žan":794,"ži ":866,"žen":1273,"užb":678,"uže":864,"uži":889,"užn":2163,"žup":1446,"ušt":795},"n_words":[5153330,5928363,4281211],"name":"hr","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/hu.json b/contrib/languages-data/hu.json
new file mode 100644
index 0000000..74a903a
--- /dev/null
+++ b/contrib/languages-data/hu.json
@@ -0,0 +1 @@
+{"freq":{"D":13198,"E":18562,"F":13797,"G":13466,"A":98368,"B":24663,"C":21970,"L":15870,"M":28130,"N":17752,"O":8481,"H":13530,"I":14875,"J":7703,"K":21077,"U":4237,"T":19473,"W":4434,"V":10406,"P":22295,"S":35593,"R":12093,"Y":1303,"X":1480,"Z":4534,"f":108322,"g":317259,"d":216557,"e":996693,"b":219927,"c":121096,"a":1016786,"n":579901,"o":467276,"l":670215,"m":298372,"j":133885,"k":454857,"h":131050,"i":506132,"w":6657,"v":203150,"u":147118,"t":700784,"s":637262,"r":536637,"q":1410,"p":129287,"z":429255,"y":242745,"x":7186,"É":1936,"Á":2646,"í":56628,"é":327830,"á":372527,"ü":54519,"ú":35451,"ö":107023,"ó":117740,"ő":81190,"ű":27405," l":36854," m":86086," n":55262," o":27009," h":46816," i":30318," j":25021," k":101750," d":20317," e":90097," f":58652," g":17434,"р":1511," a":216424,"с":1133," b":38681," c":32807," z":5071," u":7248," t":73839," v":67319," p":25823," s":75268," r":41479," J":7511," K":20282," H":12799," I":11800," N":16438," O":7214," L":14736," M":26568," B":23082," C":19809," A":94593," F":12742," G":12445," D":12015," E":17107,"л":1197," Z":3901,"к":1191," Y":1159,"и":1900,"о":2301,"н":1605," S":32225,"в":1255," R":11088,"а":2749," P":20591," W":4161," V":9030," U":3712,"е":1700," T":17702," á":18269," í":4682," é":57281," ö":8570," ó":3751," ü":2190," ú":3722," Á":2459," É":1858," ő":1879,"A ":56345,"Da":1481,"Cs":2770,"Co":4107,"Ce":1232,"Ch":2651,"Du":1186,"Do":1533,"De":2255,"Di":1628,"Fe":3066,"Fa":1242,"Ez":1912,"Eu":1996,"Er":1727,"El":1696,"Eg":3224,"Ge":1854,"Ga":2079,"I ":2334,"Fr":2249,"Bí":1300,"Fo":1548,"Fi":1170,"B ":1118,"C ":1489,"Au":1680,"Ar":2744,"Ba":3859,"Az":15235,"Ac":1577,"Am":2415,"An":3658,"Al":3816,"Bu":2584,"Br":2522,"Ca":4180,"Bi":1847,"Be":4063,"Bo":3301,"Kr":1187,"Ko":2867,"Le":3145,"Li":2545,"La":4095,"Lo":1996,"Me":3040,"Mi":3545,"Ma":10861,"Mu":1674,"Mo":3340,"Ne":3381,"Na":3799,"Ny":3322,"No":1352,"Ol":1338,"Gr":1713,"Go":1444,"Gy":1283,"Ha":3592,"He":2397,"II":1513,"Dé":1163,"Ho":2637,"In":4213,"Is":1406,"Ja":1924,"Je":1294,"Jo":1519,"Ka":3817,"Ki":3017,"Ke":1650,"Tu":1174,"Tr":2483,"To":1807,"Th":2533,"Ti":1466,"Te":2749,"Ta":2278,"V ":1123,"Sz":12584,"St":3173,"Wi":1325,"Vi":2150,"Va":2270,"Ve":1656,"Pr":2035,"S ":1580,"Pe":2500,"Pa":5475,"Po":2747,"Pi":1727,"Or":1697,"Kö":2908,"Se":1702,"Sc":1225,"Si":1430,"Sp":1130,"So":1614,"Sa":3389,"Re":2307,"Né":1630,"Ro":2833,"Ra":1803,"b ":16527,"a ":242203,"bö":1784,"i ":117842,"bó":6751,"ge":24009,"gf":2198,"ga":25428,"gb":5764,"bé":2495,"ff":1119,"fi":9218,"fr":3611,"fu":1929,"fo":23259,"j ":9703,"gy":101312,"gz":1512,"dá":7767,"he":27807,"ha":38767,"gn":5176,"gm":1339,"cé":2844,"gl":4237,"gk":2851,"gj":4145,"gi":17202,"gh":4135,"gg":3431,"gv":1711,"gu":3889,"gt":2295,"gs":3570,"gr":6943,"cí":4683,"go":14059,"dt":3427,"du":4251,"dv":2240,"dz":1317,"g ":45451,"ea":6887,"eb":8727,"ec":7541,"ed":20502,"de":31392,"dd":1273,"di":23518,"dj":21891,"do":18282,"dn":1552,"ds":5503,"dr":4675,"ex":2202,"eu":3957,"ev":16778,"ey":1469,"ez":42514,"fa":14955,"h ":4414,"bá":5025,"fe":22515,"eh":6429,"eg":88315,"ef":4014,"ee":2573,"el":138888,"ek":55065,"ej":8422,"ei":11205,"ep":13034,"eo":3448,"en":123802,"em":38648,"et":109362,"es":66857,"er":112461,"ca":6636,"e ":99285,"bs":1600,"br":4886,"bu":6371,"bo":9382,"bl":2460,"bi":10991,"bb":21421,"bd":1747,"be":56227,"db":1466,"da":33610,"f ":3042,"cu":2732,"ct":3944,"cs":39940,"cr":1412,"co":5597,"ck":3543,"ci":18130,"ch":10237,"ce":11684,"cc":1311,"c ":4126,"az":52261,"ay":1779,"ba":62023,"d ":20001,"at":73551,"as":35129,"ar":73118,"av":6465,"au":6414,"ak":76786,"al":112498,"ai":30999,"aj":22921,"ao":1282,"ap":21474,"am":34363,"an":99045,"ac":9905,"ad":30543,"aa":1143,"ab":12613,"ag":47002,"ah":3611,"ae":13109,"af":4252,"nu":6592,"nt":48640,"ns":9377,"no":16464,"nn":10307,"nz":2533,"ny":61139,"gő":1259,"jó":1888,"ká":8866,"of":2805,"oc":6235,"od":12616,"ob":7903,"jú":2913,"ké":20553,"om":30097,"on":54876,"ok":33935,"ol":57719,"oi":1954,"og":16189,"oh":1599,"ot":26104,"os":55594,"ov":15603,"ou":3364,"op":11263,"oo":1729,"or":73234,"kí":3018,"kú":6878,"r ":40419,"ow":1527,"kó":3434,"kö":37773,"oz":36942,"lá":37200,"pe":16459,"pa":16699,"kü":4521,"pc":1541,"pl":5060,"lé":24839,"po":17893,"ph":4251,"pi":8966,"pj":2198,"pk":2115,"lo":28055,"ln":6206,"lm":14166,"hé":2687,"ll":38085,"ls":10986,"lr":2355,"hí":1890,"lv":12472,"lu":8529,"lt":43305,"lz":1116,"ly":54235,"o ":10039,"ma":45288,"mb":11466,"me":75090,"iá":11532,"ml":3581,"mi":28248,"mn":1344,"mm":4364,"mp":6217,"mo":16948,"ms":1960,"mu":7004,"fő":4516,"ió":9250,"mz":4734,"p ":5660,"na":50058,"nb":3564,"nc":11966,"nd":37316,"ne":56223,"já":30832,"nf":2395,"ng":19748,"nh":1725,"ni":26541,"nj":1322,"nk":8011,"nl":3257,"jé":12386,"jt":5016,"ju":2496,"jn":3362,"jo":4214,"jl":2908,"fé":13187,"ki":30779,"kh":2892,"ke":42527,"gá":10089,"kc":1158,"kb":6547,"fü":2239,"ka":34729,"m ":27269,"fö":2942,"jz":1414,"gó":4573,"dő":5431,"gö":3372,"ks":5542,"kt":6729,"ku":13047,"kv":1892,"ko":37181,"gí":1165,"kr":7237,"kk":6940,"kl":3302,"gé":11029,"km":3318,"kn":3693,"li":35208,"lh":5532,"lk":11727,"lj":5903,"le":93067,"há":9402,"ld":10308,"lg":5358,"lf":2782,"la":70668,"lc":4145,"lb":4771,"gú":1727,"n ":153664,"hr":1649,"dí":4512,"dó":5588,"ht":1620,"hu":2691,"hi":9568,"hn":1454,"ho":21034,"dé":9871,"id":22338,"ic":13315,"ib":5712,"ia":30517,"ih":1907,"ig":15930,"if":9201,"eá":1335,"ie":7180,"dö":1970,"hy":1869,"k ":156709,"ir":14921,"is":42797,"it":23842,"iu":6023,"iv":7173,"bő":3863,"ii":3341,"ij":1228,"ik":46279,"il":27833,"im":8040,"in":53934,"io":8559,"ip":5320,"fá":1392,"je":23937,"jd":2673,"iz":8720,"l ":91636,"ja":22570,"xi":1625,"té":34673,"tí":4348,"tó":18526,"sö":2411,"só":2652,"sú":4418,"z ":78459,"sü":3136,"tá":43166,"wi":1167,"sé":27679,"sí":6485,"ró":12762,"rö":8024,"y ":64628,"rú":3555,"rü":11432,"wa":1485,"sá":25703,"ré":19532,"vi":19889,"vt":2486,"rí":1306,"vo":15593,"mű":8899,"nő":6159,"ve":52514,"rá":28508,"va":38110,"x ":2815,"ui":1873,"uk":4884,"ul":23432,"ue":1945,"ug":7493,"ur":14404,"us":30910,"ut":12436,"um":11242,"un":8785,"up":2459,"ty":4352,"tu":13656,"tt":50460,"tv":7094,"ub":2532,"ua":1854,"pü":4993,"ud":8185,"uc":2378,"to":52152,"tn":2524,"pé":3054,"tm":2785,"tl":6382,"ts":8729,"tr":17609,"pí":3835,"tf":1412,"te":87104,"pá":7384,"tk":5205,"tj":7119,"ti":40828,"th":9478,"v ":9584,"tb":6187,"tc":1163,"ta":76171,"su":4875,"sv":2378,"ss":22768,"st":26226,"sz":174760,"lő":14903,"sl":2577,"sk":7704,"sn":2526,"sm":5506,"sp":4512,"so":26449,"sr":3213,"sd":1132,"sc":3372,"sf":1789,"se":34881,"sh":2900,"si":20642,"nö":2962,"rz":5827,"u ":5228,"sa":40529,"sb":5688,"rr":6334,"rs":21922,"rt":47601,"ru":7619,"rv":8829,"nó":1459,"kő":1313,"ry":3017,"rp":2572,"ro":47249,"rn":9437,"né":20022,"rm":20825,"rl":4279,"rk":7954,"rj":4223,"ri":45410,"rh":2668,"rg":7757,"rf":3837,"re":67985,"ná":12107,"rd":9230,"rc":7289,"rb":5639,"ra":55047,"t ":159290,"mú":1197,"mó":2093,"qu":1194,"mí":1861,"mé":15355,"má":27550,"lü":5880,"s ":157503,"lö":4000,"pt":5892,"pu":3980,"ló":16644,"pp":2547,"lí":3237,"pr":7474,"ps":3440,"yő":1310,"zü":5181,"zú":1280,"zá":31274,"yú":1973,"yü":3112,"zé":19023,"zí":5603,"zö":10272,"zó":20498,"vű":1517,"yá":13547,"yí":1801,"yé":5272,"yó":2584,"vő":2313,"tű":4358,"vö":1632,"zz":3816,"sű":1526,"tő":14413,"zf":1314,"zg":2806,"zh":3151,"zi":25268,"zb":1632,"zd":4010,"ze":71368,"vá":29898,"za":30193,"yz":1903,"zv":1698,"rű":2676,"ső":8855,"zs":12765,"zu":3963,"zt":36837,"zo":19843,"zn":6608,"ví":2765,"zp":2246,"zk":4768,"vé":17783,"zm":2937,"zl":11760,"yg":1834,"yh":1922,"ye":40237,"uá":1400,"yf":2056,"yc":1116,"ya":26830,"tü":4027,"yb":3764,"tú":2750,"tö":12442,"rő":4773,"yv":4653,"yu":3843,"yt":3497,"ys":7991,"yr":1838,"yp":1398,"yo":12260,"yn":3914,"ym":1973,"yl":2832,"yk":2442,"yi":17109,"ző":8108,"yű":1984,"Ál":1439,"ám":8760,"án":50938,"áp":2119,"áj":7251,"ák":18338,"ál":59120,"ág":29984,"áh":4265,"áb":34134,"ác":4739,"ád":14308,"áz":7779,"áv":4411,"áu":1148,"ár":52483,"át":21141,"ás":47264,"á ":2111,"óz":2079,"ós":5892,"ót":4344,"óv":1296,"óa":1234,"ób":3397,"ój":4542,"ói":2619,"óg":4159,"óf":2044,"ód":5384,"óc":1867,"ór":2907,"óp":3032,"ón":2969,"óm":2752,"ól":15303,"ók":6328,"ó ":44110,"ív":3409,"íz":3248,"ín":4459,"ím":4915,"ír":8949,"ít":21446,"íg":1483,"ík":1149,"íl":1523,"íj":1882,"éz":4750,"ék":26204,"él":30309,"éj":1119,"ép":16482,"ém":6476,"én":36437,"és":84643,"ét":19103,"ér":22242,"év":16532,"éb":17184,"éd":3694,"éc":1321,"éh":4561,"ég":31689,"é ":4130,"úz":1141,"ün":2126,"ül":37857,"ür":1396,"üt":3250,"üz":1153,"üg":2645,"ük":3502,"úl":1994,"új":3185,"út":2986,"ús":1934,"úr":1723,"úa":6976,"úg":2023,"öv":8122,"öz":26938,"ú ":9189,"öt":9388,"ör":17835,"ös":7442,"ön":10766,"öl":8906,"öm":1336,"ök":5801,"ög":3454,"öd":2116,"öb":4016,"ő ":26112,"őv":2032,"őr":2463,"ős":10918,"őt":2983,"őz":2353,"őd":2691,"őb":2103,"őg":1179,"őe":1463,"ől":9494,"ők":4433,"őj":2432,"ői":2735,"őn":2715,"ű ":13555,"űk":1684,"űe":1599,"űs":1405,"űr":1219,"űv":2905,"一":1237," Ál":1439," Ga":2065," Ge":1837," Bí":1298," Fo":1535," Fr":2248," Fi":1161," Ha":3587," He":2393," Gy":1279," Go":1438," Gr":1700," Ho":2631," Dé":1163," Je":1288," Ja":1919," Is":1400," In":4206," Ka":3809," Ke":1641," Ki":3002," Jo":1515," La":4057," Le":3135," Li":2524," Ko":2859," Kr":1178," Ma":10822," Mi":3523," Me":3025," Lo":1988," Ne":3354," Na":3783," Mo":3325," Mu":1669," A ":54120," Am":2399," An":3650," Al":3798," Ac":1573," Ba":3855," Az":15203," Au":1680," Ar":2712," Be":4051," Bi":1842," Bo":3283," Br":2520," Bu":2577," Ca":4107," Ce":1230," Ch":2631," Co":4068," Cs":2760," Da":1472," Di":1622," De":2234," Do":1490," Du":1183," El":1692," Eg":3214," Er":1720," Ez":1906," Eu":1996," Fe":3060," Fa":1224," Wi":1310," a ":114242," Kö":2904," Or":1691," Po":2728," Pi":1722," Pe":2496," Pa":5457," Ny":3316," No":1346," Ol":1338," Ra":1786," Né":1627," Ro":2824," Re":2297," Pr":2030," Sz":12567," St":3106," Ta":2270," Th":2520," Ti":1460," Te":2728," Tr":2468," To":1786," Sa":3377," Si":1425," Sc":1193," Se":1692," So":1606," Sp":1107," Va":2262," Ve":1647," Vi":2142," Tu":1156," ja":1594," je":10035," in":4519," il":2853," is":10067," ir":2442," fü":1523," ka":7871," fö":1646," ki":17579," ke":13720," jo":1938," fé":4201," gy":6161," ha":19271," he":6504," cé":1611," cí":4405," gr":1159," id":3467," ig":3043," hi":3276," dé":2801," ho":8126," dí":1823," ne":18984," já":7979," na":5781," fő":4149," mu":2133," mo":4422," ok":1477," ol":6794," ké":11002," of":1186," ny":10296," no":1409," há":4031," le":17706," li":2353," la":7930," ku":2128," gö":2536," km":2432," kr":1168," ko":9227," me":30782," mi":10796," ma":18644," lo":1099," hí":1381," ad":3539," am":14965," an":8296," ak":3658," al":19507," au":2171," ar":2746," as":2187," ba":7443," az":39674," bi":2851," be":18442," bo":2566," bu":1302," er":6741," es":6369," en":1270," em":3989," el":22279," eg":37244," fe":16017," bá":1342," fa":10457," ez":6961," fu":1228," fr":2101," fo":11286," bé":1167," fi":3986," ge":1465," ga":1837," ci":1469," da":2485," cs":21597," do":1754," de":6050," di":2713," vé":3902," ví":1450," vá":10832," ze":2004," tö":9167," té":2457," tá":3920," nö":1405," sa":2016," se":2587," sp":2365," so":5755," ra":2479," re":19463," né":13996," ro":3461," pr":3595," má":8283," mé":4247," os":10567," kí":1660," or":4197," kö":27883," pe":2664," lá":2021," kü":3017," pa":5441," lé":4236," po":4686," pi":1682," ró":1121," rö":2141," va":20643," ve":9072," nő":1649," mű":3958," vo":11603," vi":8365," ré":10099," tu":3046," ut":3907," ur":1101," ta":23929," sz":51220," st":2457," su":1760," tr":1755," pé":1624," to":2214," ti":2324," te":19251," pá":3006," át":2658," ál":11522," év":4437," ér":4015," és":39564," én":2210," ép":2514," él":3293," ír":3245," ók":1246," ön":2076," ös":3774," ót":1104," új":1846,"ől ":7803,"ője":1778,"ők ":2201,"őne":2127,"ősz":2500,"ős ":1549,"ősí":1918,"ősö":1177,"őtt":1492,"őbb":1227,"ői ":1993,"Áll":1152,"Eur":1570,"Ez ":1152,"Fra":1443,"Bír":1270,"Int":2403,"Az ":14884,"Ame":1184,"Bud":1205,"Car":1146,"Cse":1140,"Egy":3037,"Nem":1251,"Nag":2455,"Nye":1809,"űvé":1367,"Köz":2207,"Pas":2299,"Par":1124,"űve":1432,"űkö":1244,"Lad":1325,"Mar":2612,"Mag":3698,"éhe":3730,"ék ":14028,"ége":6434,"égb":1296,"égi":4433,"égy":1459,"él ":2635,"éde":1275,"ég ":10442,"ébe":12999,"éba":2195,"éve":6732,"ésé":4210,"év ":4758,"ész":20296,"éte":4730,"élő":1204,"ést":1350,"ésr":1270,"épü":1343,"épí":1687,"étr":1526,"éze":2382,"érő":1195,"éme":4235,"ép ":2557,"éne":8073,"éni":1139,"ént":3323,"ény":11240,"ékh":1465,"éke":3632,"éko":2177,"égé":2685,"én ":10291,"éli":2101,"éle":4096,"élk":1173,"ély":3515,"élt":1344,"ére":3670,"érf":1845,"ét ":8716,"ért":4687,"érs":1117,"éri":1359,"ése":7910,"ési":1788,"épe":3380,"ér ":1942,"éps":1130,"élé":8715,"és ":42926,"ésű":1277,"Szl":3665,"Sza":1311,"Sze":2596,"The":1583,"ául":1145,"áts":1437,"áto":1839,"biz":1578,"áss":1306,"ást":1822,"ász":3797,"áró":1875,"ásá":7057,"árá":5593,"áva":2230,"bol":1519,"átó":1225,"áté":2725,"áz ":1260,"ázi":1122,"bor":3409,"áza":2446,"áll":11176,"bbi":2623,"ált":12559,"bba":2240,"ály":14827,"be ":8763,"áma":1434,"ban":38532,"bal":3567,"baj":2665,"áku":1940,"bad":1154,"án ":13154,"ála":2623,"bar":1493,"áli":4198,"álh":2859,"áno":1436,"ány":14546,"ánu":1287,"ámo":1867,"ána":12065,"bda":1545,"áni":2143,"ájá":1672,"áló":2208,"ás ":11271,"bi ":2519,"ár ":6296,"ber":6263,"ben":26680,"bel":7243,"bes":1771,"bet":1775,"ása":4529,"árt":3175,"árs":2861,"áso":5295,"ási":3907,"ásb":2951,"át ":7693,"ámí":1237,"áro":12322,"árm":2982,"ári":3231,"ára":5673,"ács":1314,"áci":2905,"ág ":10677,"ádj":10076,"ábó":2969,"ága":2054,"ágb":3476,"ca ":1909,"ágo":3161,"ági":2804,"ák ":11048,"áho":4041,"ája":2888,"ál ":2859,"ágá":1372,"áki":3435,"ce ":2292,"ám ":1135,"bri":1367,"bra":1211,"bur":1175,"bum":2164,"ád ":2121,"ábo":1867,"ább":3556,"ába":23979,"ajz":1349,"aka":2195,"am ":2190,"agá":2325,"aki":3799,"afé":1601,"ajn":3205,"ajt":2171,"al ":17585,"aja":1802,"ajd":2547,"ain":2508,"ait":1256,"ak ":42544,"adó":2898,"aj ":6376,"agy":29473,"adá":3539,"agj":1385,"ago":2554,"anu":1841,"any":6283,"ano":1477,"ann":3140,"ant":4051,"ans":1439,"ane":1370,"ajá":1272,"ang":7307,"ani":3989,"ank":1265,"ana":3610,"anc":3826,"and":5977,"amm":1385,"amo":3002,"amp":1099,"ami":6863,"ame":12652,"amb":1439,"ama":2681,"alv":1163,"alu":2639,"alt":1806,"alr":1104,"alo":6119,"alm":6541,"all":3854,"alk":5861,"ali":4354,"alc":1755,"ale":2695,"ala":25236,"alb":2678,"an ":50569,"akr":1682,"aku":2116,"akt":1697,"ako":5426,"akk":1936,"aba":1856,"abb":3158,"abd":1557,"ae ":11230,"ad ":2344,"ai ":23587,"aga":3600,"ado":2879,"adi":3222,"ag ":2506,"adt":1359,"aci":1651,"ach":1191,"ace":1933,"ada":7719,"acs":1177,"azo":3899,"azi":1454,"azt":1117,"azg":1460,"aza":1771,"azd":1290,"azz":1187,"az ":35775,"asú":1719,"atá":8809,"ató":6586,"azá":1314,"ba ":10534,"bb ":12760,"at ":13186,"are":1125,"ard":2003,"arc":2552,"ara":12278,"aro":4590,"arm":1949,"arl":1467,"ark":2718,"ari":4156,"aru":1342,"arr":1237,"art":20941,"asa":1245,"aso":1347,"ar ":7389,"akö":2063,"apa":2996,"akú":5988,"alá":20616,"ape":1622,"apc":1186,"apj":1950,"api":1102,"apo":2602,"apt":1225,"aló":3345,"as ":5129,"ava":2849,"aut":1403,"avi":1292,"ará":1634,"asá":3004,"arú":1364,"asz":10871,"atb":1153,"ata":7891,"ast":1692,"ass":5488,"atl":1204,"apí":1933,"atr":1521,"ato":8994,"apá":1330,"ate":2686,"ati":9317,"ath":1161,"att":4210,"aur":1366,"jed":1383,"jel":12132,"jez":1371,"jes":2679,"jai":1887,"je ":3339,"fér":1780,"fél":10120,"job":1168,"jno":2503,"jog":1359,"jle":1493,"itr":1237,"ito":1407,"itt":1847,"isk":1278,"ism":3684,"iss":1661,"ist":5127,"isz":6268,"ita":2682,"ite":2144,"iti":1991,"ius":3051,"ium":2277,"inő":1958,"iva":2264,"ügg":1434,"irá":5579,"ive":1942,"is ":16119,"ion":3768,"ilá":4340,"iro":2666,"ise":2504,"ire":1148,"ira":1291,"it ":5468,"ült":4085,"ülé":2870,"izá":1108,"ja ":16271,"ül ":11962,"ből":3435,"ük ":2123,"itá":2293,"üle":10807,"izo":1300,"ize":1447,"kif":1538,"kik":1160,"kil":1182,"kia":2258,"kin":1795,"kir":2041,"kis":3777,"kit":1528,"úsz":1202,"km ":2029,"ki ":6708,"khe":1740,"ked":3722,"kel":6145,"gál":2697,"ken":2595,"gán":1121,"kes":2444,"gás":1179,"ker":9767,"gár":1661,"ket":5479,"kez":5201,"ke ":3579,"kra":2981,"koz":3084,"kot":2662,"kos":4432,"kor":12163,"kon":4605,"kom":1428,"kol":2622,"kok":2194,"kod":1381,"gés":1802,"kna":1132,"kiá":3411,"géb":1114,"gép":2262,"gén":2847,"kke":2328,"kka":1690,"kbe":2008,"kba":2863,"kat":6062,"kar":2888,"kas":1212,"kap":3160,"kan":1437,"kal":4211,"kai":6519,"füg":1411,"ka ":5099,"föl":2822,"gyé":3341,"gyá":1929,"gyü":2858,"ha ":1609,"han":3792,"haj":2261,"hal":5044,"har":4293,"has":5102,"hat":10778,"hag":1327,"had":1308,"gyű":1250,"he ":2263,"hel":6775,"heg":1841,"hez":5105,"het":6443,"dás":3281,"dár":1447,"her":2201,"hiv":1273,"gla":1955,"gna":2223,"gió":1365,"cél":1462,"úak":6906,"gok":1862,"gol":3486,"gon":2487,"gos":1601,"got":1210,"gra":2375,"gre":1699,"cím":4480,"gsz":2045,"gy ":37866,"gus":1307,"gya":12084,"gye":12610,"gyh":1320,"gyi":8702,"gyk":1565,"gym":1148,"gyo":4080,"úgó":1191,"gys":4096,"iai":5083,"ial":1462,"ian":2020,"iat":1462,"iad":1714,"id ":1248,"ibe":1135,"ia ":14894,"ig ":5385,"ifo":5710,"ife":1711,"ics":1889,"ici":1201,"ich":2336,"ice":1195,"ie ":1156,"ica":2195,"ide":4043,"ida":9550,"ika":8060,"ige":3208,"iga":2520,"ii ":1284,"idé":1418,"ik ":22104,"imp":1521,"inc":1595,"ind":5305,"ina":5669,"ino":2798,"int":11506,"ine":3645,"ing":3990,"ini":2901,"ink":1323,"iká":2172,"inu":1420,"iko":1515,"ike":2523,"ila":1887,"in ":5995,"idő":1933,"iku":4841,"ilo":2015,"ill":7614,"ilm":1881,"ili":3110,"ile":2438,"hoz":7697,"hol":2205,"hon":1202,"hog":3581,"hos":1589,"hor":2048,"dék":1979,"dél":3343,"dés":3202,"dó ":3604,"dít":1872,"díj":1758,"fen":1212,"bán":1642,"fek":1857,"fel":10451,"fej":4036,"fia":1202,"ező":2970,"fal":3069,"faj":8821,"etű":3115,"ető":6086,"ezé":5371,"etü":1875,"esü":1885,"ez ":6988,"eté":6862,"erű":1796,"ezd":1515,"erő":1876,"evé":4056,"ezt":2772,"eze":13658,"ezh":1979,"ezi":1930,"etb":3254,"eta":1302,"ete":14739,"eti":8705,"etl":3489,"etk":2537,"est":5914,"ess":3758,"elő":7242,"esz":10684,"epü":3424,"etr":2609,"ets":2876,"ett":18422,"etv":2394,"erá":1228,"eve":8161,"eré":4519,"eur":1117,"esí":1394,"erü":9565,"epe":2041,"epi":1162,"er ":12986,"es ":29789,"elü":5526,"epl":1338,"elé":3820,"erk":1552,"eri":12722,"erj":2568,"erg":1768,"ere":20225,"erc":2570,"erd":1315,"era":2862,"erb":2020,"et ":26086,"emé":3302,"esi":1683,"ese":5978,"erz":2655,"erv":3920,"err":1968,"ert":6946,"ers":4641,"ern":2834,"erm":4064,"ené":1314,"ero":1595,"eki":1276,"egé":3370,"ekt":2496,"en ":52180,"ela":1439,"ele":27299,"eli":3526,"elj":2109,"elh":1530,"ehé":1275,"elm":4378,"eln":2146,"elk":2863,"ell":9606,"elo":1349,"elv":9038,"els":6778,"elt":4823,"ely":23022,"emb":4386,"ema":1264,"eme":6876,"eml":2503,"emi":1303,"emp":1725,"emz":4591,"ene":6376,"eng":4545,"end":20330,"enc":3429,"ejé":1183,"enn":3397,"enl":1598,"eni":2643,"ens":3074,"ent":14391,"eny":3762,"egk":1751,"egj":2290,"egn":2926,"ege":4339,"egf":1828,"egi":1596,"egh":1485,"egr":1473,"egs":1610,"egt":1312,"egy":47388,"edé":2069,"ehe":2302,"ek ":33389,"ein":1947,"el ":15987,"eit":1126,"ejt":1562,"ejl":2010,"eje":2098,"eke":7178,"ekb":1825,"em ":9006,"ött":6080,"gje":1874,"öss":4328,"gja":1441,"git":1317,"gia":2402,"ört":3402,"örz":1373,"öná":1325,"gha":1393,"ös ":1742,"gi ":5691,"ör ":1444,"gen":2834,"get":4617,"ger":3980,"ges":2578,"gel":2023,"öny":3378,"gek":1625,"önb":1254,"ge ":3243,"gbe":1509,"öld":3665,"gaz":3489,"gba":3483,"ölt":2040,"gas":2172,"gar":1902,"gat":6541,"gal":3993,"ga ":3541,"ök ":2555,"ög ":1983,"ból":6420,"fra":1911,"özö":5568,"özé":2985,"for":12637,"fon":1546,"fol":3847,"özi":2766,"öze":2220,"özs":4673,"özp":1522,"özl":1118,"övé":1547,"fog":4106,"özt":1797,"örü":1396,"fil":2271,"fin":1712,"övi":1986,"öve":4144,"örö":5177,"da ":3729,"de ":4718,"dal":6673,"dae":8117,"dat":1797,"das":1500,"dar":6693,"dap":1398,"cti":1331,"csá":1388,"ciá":1338,"ció":4016,"cs ":2427,"öbb":3612,"cse":2576,"csa":18273,"cso":5464,"csi":2762,"csk":1549,"cea":1428,"ch ":1376,"cer":1367,"cen":1114,"ci ":1364,"cha":1323,"cia":3115,"ck ":1275,"che":1523,"chi":1879,"ed ":1845,"ebb":4696,"ebe":1132,"eae":1455,"ea ":1332,"efo":1148,"ei ":4004,"ega":1438,"edi":4113,"ede":6411,"eg ":8600,"ech":1220,"ect":1158,"ecs":1616,"dor":1576,"don":3362,"dom":3952,"dol":1610,"dot":2319,"djá":9950,"djé":9456,"dsz":3860,"dul":1851,"dt ":1182,"dia":1805,"der":2519,"des":1303,"det":5300,"dez":2835,"del":4174,"den":3777,"di ":3532,"dja":1819,"din":2017,"dig":1984,"dik":7071,"rga":1547,"ri ":7426,"rgi":1214,"rge":1233,"ret":2640,"res":5836,"óta":1799,"rfi":1520,"red":5622,"ósz":1200,"reg":2954,"rem":1197,"ren":20766,"rek":3056,"rel":1449,"nál":7068,"rep":4197,"rde":1294,"ós ":2694,"re ":12803,"rci":1283,"rce":1814,"ópa":1701,"rd ":1700,"ras":2085,"rat":3432,"rba":1762,"rbe":1412,"raj":1914,"rai":1551,"rag":1242,"ran":7745,"ram":2669,"ral":2852,"rak":6278,"rab":2185,"rad":2745,"rac":1357,"rs ":1186,"ror":2542,"ros":13308,"rot":1270,"rom":4552,"ron":3143,"rop":1359,"roz":4272,"rov":1366,"rod":2730,"roc":1647,"rol":2608,"rok":2835,"rog":1810,"rny":1765,"rna":1947,"rne":1337,"rni":1311,"nél":1390,"ném":3320,"nép":2342,"név":7964,"rma":4453,"rme":7442,"riá":1542,"nég":1166,"rla":1110,"rke":1521,"rje":2570,"rit":2732,"ris":3358,"ril":1221,"rik":5031,"rin":5644,"ria":3879,"ric":1848,"rid":1666,"rie":1106,"rif":2877,"rk ":1195,"rsé":1351,"rtá":1222,"rté":4693,"rul":1250,"rus":2059,"rva":1143,"rve":3364,"ry ":1179,"rsa":2647,"rse":2444,"rsz":11249,"rta":3812,"rtj":2461,"rto":16676,"rte":3860,"rti":1302,"óba":1208,"rmé":1800,"rmá":4220,"rt ":8024,"rre":1962,"rra":1492,"sab":1704,"sai":1775,"sak":2444,"sal":14426,"óma":1626,"sba":3589,"sap":1951,"san":2277,"ójá":1365,"sat":1742,"sas":1771,"sa ":7871,"óko":1362,"ók ":2577,"ól ":13016,"ója":2231,"rvá":1234,"rze":1233,"ógi":2210,"ói ":1956,"növ":1656,"rzs":1300,"rvé":1214,"si ":8670,"sid":1199,"sil":1988,"sik":1599,"rző":1316,"se ":7438,"ser":4397,"set":1567,"seb":2714,"sen":5233,"sem":1195,"sel":3187,"sek":2815,"spo":1141,"spa":1306,"sol":2274,"son":4142,"sop":2367,"sor":6863,"sod":2739,"sok":5478,"st ":5212,"sle":1305,"sko":1582,"ska":1477,"ske":1576,"sme":3505,"ssé":2608,"ssá":1740,"sz ":8870,"lőt":1770,"lős":2401,"stá":1316,"sza":14173,"svá":1185,"sze":35971,"szo":9131,"szt":27632,"szu":2442,"szi":9598,"szl":6370,"szk":3277,"szn":4368,"sse":3446,"ssa":2570,"ssz":9449,"ste":4883,"sta":3478,"sto":1355,"sti":2035,"str":2018,"sug":1404,"lő ":3629,"sré":1266,"tai":1258,"tak":4038,"tal":17363,"tag":2404,"tbe":3293,"tba":1669,"tat":5289,"tar":19006,"tan":4747,"te ":9032,"ta ":15756,"szá":22866,"szü":3008,"szé":7434,"szí":4720,"szö":3515,"szó":5026,"ozó":12847,"ozá":2383,"kúa":5586,"pa ":1509,"pcs":1390,"par":3913,"pat":2664,"kül":3938,"pai":1203,"pap":1164,"pan":2344,"pha":1287,"láb":2286,"ped":1503,"lád":12468,"lán":2463,"pen":1323,"per":4376,"lát":1961,"pes":3470,"lás":5210,"lág":4159,"pel":1327,"lál":4812,"plo":1196,"pia":1507,"por":4386,"pos":1197,"pon":4538,"pol":3552,"lét":2548,"lés":5210,"lén":1237,"lék":10125,"lég":1202,"pjá":1227,"psz":1291,"plő":1147,"ló ":6853,"pte":2465,"lít":2551,"pri":1456,"pre":1161,"pro":3235,"lós":1760,"lóg":2572,"pus":1929,"lód":1195,"lön":2431,"lül":5325,"már":3617,"más":7074,"mán":10674,"máj":1466,"még":1185,"mét":1869,"mér":2829,"més":1202,"mél":2519,"mén":4256,"mít":1327,"mód":1366,"ra ":14404,"ngo":3206,"ni ":7437,"nge":3949,"ngy":1650,"jáh":2978,"neg":1502,"nel":1635,"nek":17344,"ják":1953,"ján":4452,"nem":11176,"jár":7148,"ner":1420,"net":3267,"ját":5567,"nes":2093,"nev":9590,"ng ":3420,"jáb":8030,"ncs":2007,"nci":3666,"nce":2318,"ne ":3513,"ndr":1401,"nds":3385,"ndo":2353,"ndj":9787,"ndi":2220,"nde":6999,"nda":1524,"nak":21124,"nal":2777,"nap":2250,"nae":1236,"nag":5845,"nai":2208,"nd ":4041,"nba":1457,"nat":2233,"na ":6605,"mze":3907,"iós":1792,"iój":1245,"nyb":1400,"nya":6957,"nye":12400,"nyi":4978,"nté":5969,"ny ":9196,"nul":2608,"nus":1717,"nty":1140,"nto":3218,"nti":4462,"ntj":1553,"nta":2379,"nte":5991,"nsz":1210,"nse":1158,"nt ":15590,"ns ":1607,"nká":1105,"nok":4021,"nop":1219,"nos":3762,"nov":1101,"nne":2064,"nna":2413,"nni":1136,"nny":1741,"niá":1146,"jéb":6198,"jén":1719,"jéh":3083,"nle":1203,"no ":1255,"nka":1134,"nid":1544,"nic":1428,"nia":2714,"nk ":1863,"nis":2046,"nik":2308,"ogr":1697,"ogl":2068,"oga":3293,"ogy":4073,"ok ":16218,"ol ":3707,"ock":1291,"ode":1450,"odi":3096,"odo":1217,"of ":1189,"oda":2951,"kál":1176,"kán":1252,"káb":1589,"nyí":1276,"obb":3582,"nyo":6087,"nyv":3444,"nyu":2805,"nyt":1370,"nys":1658,"ntő":1470,"jú ":1653,"nyá":1816,"nyé":1280,"oz ":6136,"osí":1185,"köv":1585,"köt":1840,"ozt":1266,"köz":23037,"kön":3656,"köl":1449,"kör":4099,"ozo":1583,"köd":1357,"ová":8369,"ozi":2582,"oza":5292,"ott":13732,"oto":1925,"osz":17179,"ost":2095,"ítő":1100,"oss":3713,"oso":1195,"orú":1625,"osá":2471,"ovi":1122,"ova":1834,"ove":1321,"orá":3688,"íté":2953,"opo":3410,"olá":1371,"ító":2222,"os ":17347,"oló":3495,"opt":1890,"or ":6588,"ítá":3043,"orm":10619,"orn":2184,"oro":8033,"orr":2134,"ord":2929,"oná":1361,"ore":1109,"org":1652,"ori":4752,"osa":4828,"ort":5009,"ors":11126,"orv":1840,"omá":8665,"ot ":4277,"orb":1551,"ora":2591,"ízi":1186,"íto":2971,"ola":3942,"old":2725,"olc":1133,"on ":14829,"oli":4956,"oll":1909,"olg":2986,"ols":1101,"olt":11581,"oln":1426,"olo":2069,"oly":9282,"olu":1943,"okb":1937,"oka":3243,"om ":4999,"okk":1292,"okr":1131,"oks":2384,"íte":4142,"oko":2728,"ív ":1366,"okt":1201,"író":2841,"ona":4624,"ond":2206,"one":1152,"ong":1637,"oni":3479,"ono":2799,"ons":1750,"ont":8687,"ony":4114,"oma":1728,"omb":2831,"omi":1564,"omm":1333,"kék":1183,"kép":5158,"kén":3827,"írá":2059,"omo":2114,"két":4231,"kés":3743,"ímű":2569,"la ":5399,"le ":4396,"lcs":2466,"írt":1125,"lda":1565,"ldi":1183,"lab":1923,"lad":1437,"lag":3296,"laj":1625,"lal":2932,"lak":15098,"lan":5737,"lam":7155,"lap":6224,"lat":10133,"las":4338,"ld ":2188,"lbu":2257,"dő ":2164,"kus":4984,"kul":4809,"ksá":2221,"ksz":1790,"gó ":1820,"gör":2727,"llí":1578,"llá":2364,"lló":2888,"lon":1455,"lom":5872,"lor":1462,"log":1806,"los":2605,"lot":1131,"lov":8327,"lne":1797,"ljá":1491,"lmi":2811,"lme":1597,"lma":4921,"lna":1780,"lto":3300,"lsz":1378,"lta":8440,"lte":3310,"lu ":1887,"lmé":1286,"lre":1816,"lt ":21002,"ldá":1177,"lha":4212,"li ":5160,"ház":2853,"lev":1937,"les":5249,"let":23881,"hár":2058,"lep":3720,"lem":6723,"len":17272,"lek":3977,"lel":1659,"leh":1502,"leg":14221,"háb":1477,"lla":10657,"lle":8630,"lli":2652,"llo":1404,"lko":4418,"lka":2131,"lke":2520,"lgá":2219,"lje":1805,"ll ":3110,"lja":2062,"lit":2107,"lis":6935,"lin":2766,"lim":1567,"lid":1319,"lia":3789,"lik":2243,"lyó":1710,"ma ":5513,"mai":2750,"maj":1342,"mad":7309,"mag":9145,"mar":2289,"mas":1406,"mal":2064,"man":2348,"maz":4614,"mat":4172,"mba":2545,"mbe":4662,"me ":1765,"meg":18261,"iáb":6342,"met":6641,"mes":7852,"mer":9128,"mel":20048,"men":4193,"mek":1223,"mez":3626,"lva":1622,"lve":3172,"lul":1401,"lus":1938,"ly ":11120,"lvt":1957,"ltá":1280,"lsó":1363,"lya":5925,"lyb":1133,"lyi":2296,"lye":12093,"lté":1277,"lvá":1133,"lyo":1466,"lyn":2379,"lys":1166,"lyt":1183,"lső":6342,"lyá":9265,"mpi":1413,"mpl":1595,"mon":2735,"mok":3026,"mos":2616,"mot":1595,"moz":1966,"ió ":2881,"mus":1737,"mut":1302,"mun":2084,"mi ":6215,"min":10577,"mil":1543,"mit":2000,"mia":1836,"mik":1748,"mma":1863,"tő ":4350,"től":2706,"tős":1505,"tőn":1924,"sű ":1297,"zná":3356,"zt ":3494,"víz":2022,"zte":5806,"zti":1839,"zta":2495,"ztr":2410,"zto":1911,"zsi":1572,"ső ":5513,"zul":1547,"ztá":11872,"zté":2691,"rű ":1401,"zsé":5276,"zza":1374,"zga":2049,"zi ":4058,"zhe":2186,"zet":19059,"zes":1480,"zen":10515,"ván":2981,"zem":3342,"zel":4337,"vál":6881,"zek":2284,"vák":7162,"vár":8676,"zer":19180,"ze ":4791,"zda":1164,"zab":2585,"zad":1388,"zak":6902,"zal":1294,"zat":8108,"zot":2710,"zor":2657,"zom":1281,"zon":5431,"zok":2301,"zol":2432,"zpo":1664,"vén":5253,"véd":2112,"vég":2913,"vét":1848,"vés":2023,"zió":1200,"zke":1119,"zle":1674,"zlo":7817,"zig":3735,"zin":2369,"zil":1212,"zik":7225,"zis":1133,"yve":2842,"yug":2984,"ysz":2663,"yok":1279,"yom":2043,"yol":1936,"yos":2298,"yob":2167,"za ":3143,"ről":1883,"ysé":2805,"ye ":3860,"yei":1140,"yek":4530,"yed":1362,"yes":5416,"yer":1837,"yen":2702,"yel":9386,"yez":2468,"yet":5968,"ya ":3038,"yag":1568,"ybe":1927,"yba":1297,"yar":8873,"yan":3850,"tül":2293,"yal":1350,"yak":3842,"yai":1283,"yhá":1178,"yne":2282,"yi ":5728,"yik":7147,"tív":1267,"téz":2544,"tér":3350,"tét":2097,"tés":9077,"tén":6927,"tél":1509,"ték":6622,"töb":3419,"tör":6044,"tó ":7302,"tól":4069,"tán":3887,"tár":7602,"tás":11209,"táv":1125,"ták":3376,"tál":11084,"táj":1226,"sú ":1586,"sül":2209,"só ":1339,"sök":1579,"ség":20101,"sén":1506,"sér":1918,"sét":1189,"sít":5115,"sár":2700,"sát":1861,"sán":1607,"ság":11788,"sáb":4560,"rög":2754,"rök":1818,"röv":1876,"rúg":1325,"rül":11029,"ró ":3203,"vtu":1747,"ról":2801,"róp":2151,"vir":1492,"vil":4652,"vid":4003,"vis":3046,"réb":2703,"rég":2911,"rés":10041,"vol":10664,"von":3245,"vi ":1300,"vez":9822,"ver":7753,"rás":8381,"ves":3105,"vet":5802,"veg":1557,"rág":1665,"ven":5494,"rán":5049,"rál":4428,"vel":6375,"vek":3167,"ráb":2450,"ve ":7296,"val":9128,"van":3020,"var":2245,"vat":2280,"vas":2577,"vad":1432,"vag":11793,"va ":3234,"műv":2529,"műk":1274,"nős":2336,"utó":1685,"utá":2191,"női":1248,"nő ":1717,"mű ":3511,"uró":2193,"usz":5860,"ust":1128,"uta":3446,"uto":1342,"us ":17567,"ura":2400,"urg":1418,"uri":1391,"ulá":1430,"uma":1476,"umb":1113,"unk":2047,"uni":1375,"um ":4397,"ult":3281,"ula":2897,"uk ":1640,"ul ":10317,"uga":4002,"uda":1985,"udo":3577,"pül":4756,"tvá":1162,"trá":1500,"tve":3654,"tur":1775,"tus":2989,"tul":1430,"tum":1120,"tud":4114,"tté":1199,"ttá":1546,"tsé":3581,"pít":3633,"tre":3344,"tt ":30966,"tra":4155,"tri":3477,"tro":2690,"tsz":2410,"tta":4727,"tte":6235,"tti":1842,"to ":1211,"tjá":2955,"tos":4461,"tot":7575,"tkö":1448,"toz":15230,"tom":3113,"ton":3995,"tok":4029,"tol":3082,"tor":5700,"til":2521,"tik":5482,"tis":1923,"tin":5413,"tio":2102,"tiz":1109,"tja":3013,"tke":1731,"tla":1565,"tle":4022,"tem":4106,"pán":1462,"ten":6944,"tei":1486,"tek":7221,"pál":1428,"tel":10568,"teg":2075,"th ":1102,"tet":11629,"tes":7803,"ter":19863,"pár":2273,"ti ":14986,"the":2658,"tha":1854,"ző ":4019,"zöt":4004,"zöv":1986,"zör":1244,"zül":3298,"zás":4765,"zár":4252,"zám":6211,"zál":1821,"zág":9532,"záz":1557,"yüt":2463,"zó ":13810,"vű ":1466,"zít":1657,"zín":2823,"zép":2406,"zér":1246,"zés":6463,"zén":2337,"zél":1670,"zék":2328,"ülö":2566,"ütt":2514,"ülő":1827,"yáb":1633,"yán":7191,"yár":2315,"tű ":2810,"vő ":1785,"yó ":1102,"yéb":2439},"n_words":[10929783,12338513,8457220],"name":"hu","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/id.json b/contrib/languages-data/id.json
new file mode 100644
index 0000000..818be59
--- /dev/null
+++ b/contrib/languages-data/id.json
@@ -0,0 +1 @@
+{"freq":{"D":29898,"E":10468,"F":12839,"G":16234,"A":46756,"B":45618,"C":22345,"L":19113,"M":41589,"N":19093,"O":9096,"H":16882,"I":48750,"J":24390,"K":57099,"U":11179,"T":39498,"W":10090,"V":6089,"Q":1321,"P":55685,"S":65231,"R":21435,"Y":7072,"X":1468,"Z":2501,"f":36925,"g":365165,"d":485064,"e":840847,"b":246328,"c":70473,"a":2100570,"n":996656,"o":305761,"l":430448,"m":362974,"j":64260,"k":378208,"h":280863,"i":864649,"w":50622,"v":31442,"u":508685,"t":522270,"s":482635,"r":589018,"q":2068,"p":245252,"z":10934,"y":153007,"x":3585,"é":2192," l":29162," m":98895," n":17327," o":22384," h":14754," i":38006," j":15556," k":88933," d":239066," e":8087," f":10185," g":12086," a":116958," b":88325," c":8643," y":62811," z":1713," u":21958," t":81597," w":11107," v":3008," p":104408," s":135289," r":13775," J":23985," K":55803," H":16124," I":42065," N":16611," O":7318," L":17821," M":38172," B":43424," C":19640," A":41294," F":11657," G":14895," D":28065," E":8736," Z":2373," Y":6861," S":61440," R":19599," Q":1212," P":52451," W":9492," V":4735," U":10169," T":36300,"ا":1474,"A ":3175,"Da":7353,"Co":3999,"Ce":1371,"Ch":3483,"Ci":2915,"Du":2494,"Do":1602,"De":6798,"Di":6852,"Fe":1983,"Fa":1382,"Er":1543,"Ge":3292,"Ga":2686,"I ":4993,"Fr":1829,"Fo":1312,"Fi":2806,"C ":1897,"Au":2055,"Ar":4506,"As":2577,"D ":1256,"Ba":19721,"Ag":2188,"Ab":1488,"Ac":1714,"Ad":1447,"Am":5332,"An":4093,"Ap":1629,"Al":4784,"Bu":4215,"Br":3076,"Ca":4296,"Bi":2492,"Be":8472,"Bo":3000,"Ku":3549,"Kr":1685,"Ko":9825,"Le":3207,"Li":3131,"La":6311,"Lu":1669,"Lo":2427,"Me":8886,"Mi":3917,"Ma":15663,"Mu":4166,"Mo":3510,"Ni":1382,"Ne":3573,"Na":5130,"P ":1286,"Nu":1142,"No":3476,"Ok":1174,"Gr":2085,"Go":1818,"Gu":2419,"Ha":5613,"He":2039,"II":2521,"Hi":2733,"Ho":2454,"Hu":2080,"Ib":1304,"Ia":3224,"In":24134,"Is":3276,"It":2124,"Ir":1207,"Ja":10872,"Je":6536,"Jo":2100,"Ju":3132,"Ka":22425,"M ":1852,"Ki":1891,"Ke":13666,"Ut":2746,"Un":3136,"Tu":2495,"Tr":1946,"To":2609,"Th":2978,"Ti":8403,"Te":9113,"Ta":6615,"V ":1165,"Sy":1338,"St":3734,"Su":10264,"Wo":1285,"Wi":2707,"Wa":3162,"We":1219,"Vi":1573,"Pu":4020,"Pr":10851,"S ":1994,"Pe":15767,"Pa":12458,"Po":3505,"Pi":2342,"Or":1349,"Se":18572,"Si":5355,"Sh":2006,"Sp":1564,"So":3044,"Ru":2528,"Sa":8331,"Re":4622,"Ri":1937,"SM":1268,"Ro":3385,"T ":1420,"Ra":5456,"b ":5402,"a ":312256,"Ya":1618,"Yo":2051,"Yu":2161,"i ":261146,"ge":17252,"ga":100134,"fi":10254,"fr":1949,"fu":1399,"ft":1686,"fo":4136,"he":8947,"ha":50757,"gn":2069,"gl":1807,"gk":15524,"gi":27056,"gh":4457,"gg":24987,"gu":19428,"gs":5762,"gr":8256,"go":7739,"du":22038,"dy":1437,"g ":127207,"ea":8168,"eb":53877,"ec":18556,"ed":14792,"de":45859,"dd":1120,"di":143405,"do":23309,"ds":1430,"dr":3663,"ew":5523,"eu":3102,"ev":3178,"ey":2387,"fa":4277,"h ":162608,"fe":4040,"eh":16895,"eg":14415,"ef":2165,"ee":3361,"el":59121,"ek":27553,"ej":8227,"ei":5982,"ep":21232,"eo":11093,"en":149316,"em":59371,"et":34760,"es":60149,"er":212475,"ca":27397,"e ":42795,"br":3158,"bu":59966,"bo":6681,"bl":3420,"bi":20522,"be":67748,"da":219838,"f ":6725,"cu":3577,"ct":1798,"co":4648,"ck":2674,"ci":8787,"ch":7724,"ce":8123,"c ":1919,"az":1822,"ay":22914,"ba":74854,"d ":16186,"at":148965,"as":93999,"ar":156214,"aw":19383,"av":2545,"au":33011,"ak":85647,"al":170145,"ai":50563,"aj":9897,"ap":26307,"am":87049,"an":494871,"ac":7825,"ad":111448,"aa":15819,"ab":26392,"ag":30377,"ah":179811,"ae":6490,"af":4082,"nu":12849,"nt":61715,"ns":21631,"no":11352,"nn":7780,"nz":1307,"ny":39684,"oe":1902,"of":3903,"oc":2742,"od":8904,"oa":2254,"ob":5836,"om":19983,"on":61017,"ok":11631,"ol":35874,"oi":1595,"og":9163,"oh":4041,"ot":23779,"os":12530,"ov":13798,"ou":5529,"op":9093,"oo":2776,"or":39625,"r ":75373,"ow":2582,"oy":1610,"pe":68224,"pa":90070,"pl":2952,"po":12602,"ph":1712,"pi":13857,"lo":16018,"ln":1524,"lm":6776,"ll":7645,"ls":1781,"lu":24730,"lt":3200,"ly":1874,"o ":23320,"ma":94460,"mb":26120,"me":93044,"ml":1630,"mi":26777,"mn":2215,"mm":2111,"mp":24012,"mo":8478,"mu":28574,"p ":11473,"na":77075,"nc":11437,"nd":56574,"ne":39080,"nf":2387,"ng":238792,"ni":56068,"nj":13796,"nk":3871,"ju":15959,"jo":2256,"ki":23132,"kh":4590,"ke":56774,"ka":127350,"m ":50860,"ky":1397,"ks":9835,"kt":10395,"ku":22809,"ko":27878,"kr":3523,"kk":1592,"kl":3754,"km":2765,"kn":3265,"li":60829,"lk":2913,"le":44927,"ld":2217,"la":192799,"lb":1849,"n ":334938,"hr":1702,"ht":1424,"hu":19954,"hk":1872,"hi":19755,"hn":2129,"ho":5306,"hl":1668,"id":19549,"ic":8689,"ib":13081,"ia":81624,"ih":9019,"ig":10843,"if":6720,"ie":7742,"k ":77693,"ir":31102,"is":68271,"it":40678,"iu":5028,"iv":4491,"iw":2077,"ij":2865,"ik":61148,"il":48904,"im":30225,"in":117362,"io":16681,"ip":11977,"je":5678,"ji":4743,"iz":1367,"iy":1200,"l ":56560,"ja":34513,"z ":1416,"wi":10306,"wo":1229,"y ":11456,"wa":31269,"we":3398,"vi":15930,"vo":1681,"ve":8269,"va":3932,"x ":1658,"ui":4586,"uj":4080,"uk":44283,"ul":29290,"ue":2818,"uf":1241,"ug":8297,"uh":9753,"ur":39636,"us":36462,"ut":30979,"um":25849,"un":84334,"up":32836,"ty":2279,"tu":61107,"tt":3054,"ub":9153,"ua":56784,"ud":13238,"uc":2268,"w ":1917,"to":20692,"tn":2005,"tl":1402,"ts":2489,"tr":20074,"te":90779,"tk":2568,"ti":55141,"th":5635,"ta":176003,"su":25989,"ss":5417,"st":32797,"sy":2246,"sw":1219,"sl":3160,"sk":6158,"sn":2055,"sm":3761,"sp":3394,"so":6557,"sc":1962,"se":100046,"sh":4276,"si":103815,"u ":69102,"sa":100996,"rr":2177,"rs":13384,"rt":29454,"ru":41105,"rv":1161,"rw":1931,"ry":4130,"rp":3731,"ro":34136,"rn":12088,"rm":13211,"rl":14368,"rk":15615,"rj":5203,"ri":102805,"rh":3147,"rg":10415,"rf":1249,"re":30519,"rd":11856,"rc":2704,"rb":14366,"ra":143028,"t ":74952,"qu":1325,"s ":74416,"pt":3193,"pu":23123,"pr":13816,"ps":1371,"zi":2204,"za":3196,"ye":4611,"ya":121025,"yu":3182,"ys":1566,"yo":3138,"yi":2638,"一":3113," Ga":2665," Ge":3275," Fo":1303," Fr":1822," Fi":2802," Ha":5588," He":2029," Go":1803," Gr":2062," Gu":2405," Ib":1298," Ia":3220," Hu":2077," Ho":2444," II":1738," Hi":2728," Je":6511," Ja":10843," Ir":1206," Is":3262," It":2123," In":24109," Ka":22393," Ke":13629," Ki":1879," Jo":2086," Ju":3129," La":6277," Le":3183," Li":3097," Ko":9805," Kr":1679," Ku":3543," Ma":15603," Mi":3888," Me":8848," Lo":2414," Lu":1657," Ne":3562," Na":5110," Ni":1377," Mo":3486," Mu":4147," Ap":1628," Am":5319," An":4070," Al":4777," Ag":2185," Ac":1711," Ad":1437," Ab":1471," Ba":19679," Au":2049," As":2567," Ar":4490," Be":8434," Bi":2483," Bo":2979," Br":3065," Bu":4209," Ca":4213," Ce":1368," Ci":2897," Ch":3462," Co":3963," Da":7319," Di":6832," De":6775," Do":1546," Du":2487," Er":1536," Fe":1977," Fa":1366," Wo":1267," Wi":2689," We":1211," Wa":3147," Yu":2156," Yo":2043," Ya":1612," Or":1346," Po":3467," Pi":2336," Pe":15730," Pa":12421," Nu":1139," No":3465," Ok":1169," Ra":5409," SM":1160," Ro":3363," Re":4599," Ri":1930," Pr":10827," Pu":4013," Sy":1332," Su":10248," St":3681," Ta":6595," Th":2952," Ti":8388," Te":9090," Tr":1942," To":2586," Ru":2526," Sa":8312," Sh":1982," Si":5333," Se":18549," So":3026," Sp":1555," Vi":1551," Tu":2469," Un":3130," Ut":2746," ja":5269," je":2335," in":24841," il":1562," is":2175," it":1708," ka":17469," kh":1422," ki":3127," ke":41380," ju":6965," ha":6291," gr":1294," gu":1601," ib":1921," ia":3559," hi":4380," hu":2034," ne":5231," na":8812," mu":5887," mo":2739," ol":12623," of":1811," no":1685," le":5736," li":3289," la":14587," ku":3200," km":2201," kl":1852," ko":17290," me":68772," mi":4388," ma":16460," lu":3991," lo":1345," ag":1308," ab":1554," ad":60411," an":11749," ap":1720," ak":5080," al":4506," aw":1213," ar":2749," at":18641," as":3479," ba":29062," bi":8314," be":42050," bo":2101," bu":6066," ca":3227," en":1364," ek":1536," fo":1399," fi":5527," ge":4483," ga":3124," co":1499," ce":1281," da":93500," do":1270," de":28323," di":109723," du":4523," za":1414," ya":62479," ru":1822," sa":30188," se":78628," si":6482," sp":1184," so":1196," ra":4660," re":4944," ro":1171," pu":5843," pr":10257," or":4567," pe":52401," pa":26959," po":5413," pi":2294," wa":3549," wi":6744," tu":4514," ut":3268," um":2172," un":12469," ta":18998," st":3288," su":11745," tr":2305," to":2153," th":1170," ti":9886," te":42313,"Fil":1716,"Ger":1344,"II ":2070,"Han":1227,"Har":1306,"Ing":3276,"Int":1699,"Ind":17352,"Ia ":3146,"Ara":1730,"Aus":1437,"Bah":2306,"Bal":1493,"Ban":5008,"Bar":4958,"Bat":1605,"Agu":1534,"Ame":4085,"Ang":1356,"Ber":2116,"Ben":1538,"Bel":2649,"Bri":1173,"Des":2124,"Dal":2000,"Cha":1289,"Dia":1644,"Neg":1421,"Nov":1144,"Per":8357,"Pem":1634,"Pen":3260,"Pas":1289,"Par":2408,"Pad":2035,"Pan":1988,"Pul":1756,"Pro":7786,"Pre":1253,"Pol":1113,"Ita":1993,"Isl":1234,"Jan":1394,"Jak":1458,"Jep":2416,"Jer":2596,"Jaw":4820,"Jul":1161,"Kab":8059,"Kal":5302,"Kan":1250,"Kat":1399,"Kar":2399,"Ker":1724,"Kep":1438,"Kel":1127,"Kec":4407,"Kon":1421,"Kom":1142,"Kor":1136,"Kot":4453,"Lau":1252,"Men":2499,"Mer":1254,"Man":2093,"Mal":2386,"Mar":3752,"Mas":1454,"Mus":1132,"Yun":1506,"一一":1315,"Sur":1395,"Sum":1934,"Sul":1400,"Sun":2471,"Sta":2065,"Ten":4508,"Tan":2075,"Sin":1312,"Ser":4461,"Sep":1413,"Sel":5571,"Sem":1247,"Seb":1132,"Rus":1267,"Sam":1292,"San":2004,"Rep":1522,"Rom":1304,"Uni":2252,"Uta":2484,"Ter":1521,"The":1859,"Tim":5598,"bis":1615,"bit":1641,"bil":2430,"bin":3990,"bih":2863,"bli":2292,"bol":2158,"ban":15219,"bak":1477,"bal":2565,"bai":2045,"bag":18039,"bah":11866,"bad":2660,"baw":2021,"bat":5802,"bas":2119,"bar":6280,"beb":2207,"bed":1156,"ber":44372,"ben":5468,"bel":6123,"bek":1139,"bes":5623,"bia":3415,"bid":1129,"ca ":1339,"car":5372,"can":2310,"cam":11583,"ce ":2031,"bu ":2451,"bua":22699,"bup":9962,"bur":1967,"bul":1642,"buk":3601,"bun":4326,"bum":1823,"buh":2607,"but":6527,"aka":36392,"am ":27176,"aki":3783,"akh":2140,"al ":32136,"aja":7514,"aik":1989,"ain":9801,"air":2314,"ais":1591,"ait":2311,"ak ":26644,"ahk":1403,"ahi":5150,"ahu":13410,"aha":23711,"agi":8426,"agu":2010,"anu":3628,"any":11108,"ano":1219,"ann":4127,"ant":18067,"ans":3241,"ane":1770,"ang":133187,"ani":10218,"anj":5111,"ank":1863,"ap ":5145,"ana":17620,"anc":4861,"and":16448,"amu":2163,"amp":6554,"ami":2968,"ame":2230,"amb":4802,"ama":38387,"alu":3924,"alo":1291,"all":1612,"ali":20004,"ale":3013,"ala":101814,"an ":259764,"aks":3177,"aku":4198,"akt":4736,"aba":6181,"abi":1719,"abu":12017,"ae ":1556,"aca":2365,"aan":12880,"aat":2166,"ad ":3820,"ab ":3122,"afi":1142,"ai ":28231,"aga":15818,"age":1445,"aer":3528,"ah ":131776,"adi":10057,"ade":2164,"ach":1161,"ace":1304,"ada":91074,"ayu":1260,"aya":17559,"ba ":1212,"at ":42706,"arg":2616,"are":4682,"ard":2694,"ara":50200,"aro":1477,"arn":2327,"arl":1195,"ark":4616,"ari":39797,"aru":4864,"ars":1240,"art":10017,"au ":21880,"asa":28749,"ary":2056,"asi":28272,"ase":1504,"ask":2058,"ar ":23543,"apa":12599,"api":3283,"apu":2004,"as ":18260,"aut":2942,"ay ":1252,"awa":16458,"awi":1328,"ata":52964,"asu":4212,"ast":4229,"ass":1429,"asy":1183,"atk":1321,"ato":2873,"ate":14253,"ati":9780,"aua":1165,"atu":20191,"aup":1527,"aus":1167,"jen":2753,"jad":5570,"jaa":2051,"jab":1150,"jar":5491,"jal":2507,"jak":2749,"jan":3425,"jo ":1155,"ito":1897,"itu":7300,"ism":1635,"isu":1798,"ist":11957,"ita":13383,"ite":4451,"iti":3900,"iwa":1676,"ium":1147,"iun":2068,"ivi":1396,"ive":2266,"ipu":1250,"ipi":1541,"is ":23018,"ion":11007,"ipa":2509,"ipe":2501,"ir ":10536,"irk":1299,"iri":9503,"isi":11065,"ise":5218,"isa":7947,"ire":1684,"ira":4533,"it ":5451,"ja ":5112,"kim":1495,"kil":1164,"kin":2100,"kir":1475,"kis":1425,"kit":4346,"km ":1129,"ki ":8986,"khi":1575,"keb":2854,"kec":8512,"ked":2012,"kek":1191,"kem":4126,"kel":7010,"ken":6683,"kep":3726,"kes":1631,"ker":5150,"ket":3912,"ke ":5970,"ksa":1691,"ku ":4682,"kot":9086,"kon":4219,"kom":5415,"kol":2400,"koh":1184,"ks ":1167,"kny":1410,"kka":1328,"ko ":1260,"kla":1828,"juk":1809,"jun":1767,"jum":1474,"jua":1961,"jug":4990,"kaw":1325,"kat":13174,"kar":11177,"kas":4851,"kap":2757,"kan":62967,"kal":5122,"kam":1705,"kai":2738,"kad":1254,"kab":2943,"ka ":12884,"ha ":2059,"ham":2420,"han":14832,"hak":1134,"hal":1973,"har":6037,"has":12725,"hat":1841,"haa":1690,"had":1765,"he ":2996,"her":1241,"hi ":1844,"hid":1896,"hin":3929,"hir":6378,"hka":1709,"go ":1252,"gku":1576,"gor":1284,"got":2192,"gsa":3115,"gu ":1706,"gra":2940,"gri":3408,"gur":1410,"gus":2168,"gun":9840,"iam":1372,"ial":6166,"ian":16893,"iap":1176,"ias":3614,"iat":1370,"ibi":1510,"ibu":4263,"id ":2050,"iba":3456,"ibe":2500,"ia ":44404,"ier":1178,"ies":1321,"ifi":1731,"ih ":5574,"ich":1544,"ie ":1390,"ica":1838,"idu":1993,"idi":3417,"ide":3199,"ida":6661,"if ":2797,"il ":7886,"ija":1503,"im ":3888,"ika":23689,"iga":3238,"igu":3825,"iha":2592,"ik ":17419,"imp":2775,"ime":1812,"imi":1994,"ind":4767,"ina":9693,"imu":7350,"inn":1767,"ino":1354,"int":9752,"ins":11171,"ine":2698,"ing":27890,"ini":22825,"ink":1199,"iny":3792,"iko":1474,"iki":7614,"ike":5441,"ila":14804,"in ":16335,"iku":2328,"ill":2052,"ilm":5734,"ili":13211,"ile":1187,"ima":9934,"io ":1907,"hny":1246,"hub":1262,"hun":11971,"hus":1117,"fat":1257,"eta":14851,"ete":4068,"eti":4646,"est":3135,"esu":1622,"ess":1214,"etu":1676,"evi":1298,"ey ":1761,"ewa":3433,"epe":3912,"er ":19709,"epa":9668,"eor":7717,"es ":8727,"ept":1302,"epu":3274,"erk":7438,"erl":11827,"eri":25792,"erj":4499,"erg":3829,"erh":2837,"ere":6493,"erc":1371,"erd":7020,"era":35580,"erb":12574,"et ":5262,"esi":21745,"ese":3800,"esa":15867,"eru":21470,"ert":16167,"ers":10880,"ern":7132,"erm":8938,"erp":2943,"ero":2183,"eki":2304,"ekn":1342,"eko":2677,"eks":2683,"ekt":2574,"eku":2083,"en ":21818,"ela":27699,"ele":4883,"eli":4636,"ell":1619,"elo":2885,"elu":8272,"emb":14950,"ema":7624,"eme":9102,"emo":1229,"emi":9424,"emu":4943,"emp":7479,"ene":6504,"eng":44211,"ena":14758,"end":12541,"enc":3844,"eni":5826,"enj":6735,"enu":4939,"ens":2961,"ent":14866,"eny":6205,"ege":2018,"egi":2598,"ehi":1472,"ek ":3218,"ein":1306,"eja":5845,"el ":6722,"eke":1577,"eka":8278,"em ":3699,"gka":11656,"git":1200,"gin":2402,"gio":1293,"gia":7505,"gha":1865,"ggu":3578,"ggr":3634,"ggo":2263,"ggi":3715,"gga":11266,"gi ":10809,"gen":4341,"ger":4633,"gem":1548,"gel":2369,"ge ":1412,"gah":5405,"gai":13025,"gas":1879,"gar":10391,"gat":2080,"gam":3520,"gal":5628,"gan":35620,"gap":1320,"ga ":16084,"for":2821,"fil":3873,"fik":2255,"fis":1267,"da ":32556,"de ":5057,"dak":4551,"dal":74713,"dah":2729,"dae":2768,"dat":2016,"das":2265,"dar":32215,"dap":6287,"dan":52470,"dam":1256,"day":1876,"ch ":1540,"cer":1324,"cha":1339,"ck ":1406,"che":1327,"chi":1177,"cil":1660,"cis":2553,"ed ":1578,"eba":14092,"ebe":8142,"ebi":3337,"ebr":1132,"ebu":26049,"ea ":1655,"ei ":1901,"ega":7504,"eh ":13927,"edi":3728,"ede":1589,"eda":4173,"edu":2208,"eci":1667,"eca":14054,"dus":1247,"don":16622,"dok":1141,"dun":4005,"dup":1272,"dul":1434,"duk":5272,"dua":3278,"dud":2657,"dra":1669,"du ":1319,"did":3706,"dia":10670,"dib":6007,"der":4755,"des":7802,"del":1249,"dek":2240,"den":19332,"dem":1173,"di ":63731,"do ":1904,"dim":2715,"din":3800,"dio":1486,"dip":4666,"dir":8308,"dis":11623,"dit":5425,"dig":4149,"dik":8070,"dil":3331,"rha":2148,"rga":5059,"ri ":42938,"rge":1577,"ret":2887,"res":4698,"rg ":1147,"rea":2388,"rej":1691,"ren":4953,"rek":3031,"rda":4163,"rdi":3109,"re ":2800,"rbu":1584,"raw":1318,"rd ":2133,"rap":3154,"rar":1749,"ras":8094,"rat":11399,"rbi":1817,"rba":6850,"rbe":3317,"raj":2995,"rai":2568,"rah":9741,"rag":1442,"ran":36962,"ram":3611,"ral":3814,"rak":7284,"rab":2762,"raa":1221,"raf":1467,"rad":6359,"rs ":1283,"rpe":1250,"ros":2826,"rot":1395,"rom":1135,"ron":3571,"rop":2512,"rov":9832,"rod":2363,"rog":1358,"rny":1495,"rna":5736,"rmu":1260,"ro ":1992,"rma":9740,"rle":9267,"rla":2281,"rn ":1378,"rki":1226,"rke":4730,"rka":5975,"rja":3509,"rip":1275,"rio":1652,"rit":4490,"ris":9168,"ril":2736,"rik":14926,"rin":9510,"rim":1709,"ria":5936,"rib":1211,"ric":1356,"rid":1193,"rie":1621,"rk ":1641,"rya":1832,"ruh":2011,"rup":16158,"run":3050,"rum":2131,"ruk":1365,"rus":4704,"rut":3005,"ry ":1765,"rsi":3955,"rsa":2467,"rse":3931,"rta":11752,"rte":3710,"rti":7667,"rua":2111,"rtu":2198,"rt ":1478,"ru ":3013,"saa":3516,"sah":3766,"sai":1253,"sak":1265,"sal":14489,"sam":5305,"san":11536,"sat":14608,"sas":2791,"sar":11521,"saw":1247,"sa ":26535,"shi":1399,"si ":37996,"sid":1884,"sia":23497,"sit":3002,"siu":2036,"sir":1424,"sis":6271,"sin":5946,"sio":5312,"sil":3164,"sim":1558,"sik":5796,"sih":1154,"sif":1869,"se ":1750,"ser":7000,"ses":3161,"set":3675,"seh":1426,"sed":1550,"sec":2609,"seb":38986,"sep":5282,"seo":6619,"sen":4315,"sem":4262,"sel":6726,"sek":6032,"sej":4109,"spe":1482,"son":1660,"st ":1847,"ss ":1345,"sli":1139,"sla":1693,"ska":2806,"smi":1231,"sme":1407,"sya":1486,"ssa":1575,"ste":6115,"sta":5899,"sto":1411,"sti":6848,"stu":1610,"str":8471,"sua":5405,"sum":1276,"suk":5658,"sun":2892,"sut":1776,"sus":2249,"sur":1552,"tai":3534,"tak":12396,"tal":4630,"tah":16201,"tab":1684,"tau":16309,"tat":1881,"tas":11773,"tar":15390,"tap":2003,"tan":48487,"tam":7817,"te ":2719,"ta ":30047,"pa ":5126,"par":4662,"pat":19772,"pas":2458,"pad":19421,"pak":16990,"pal":3741,"pai":3130,"pan":11608,"pi ":3245,"pen":20640,"pem":8687,"per":28237,"pes":2547,"pel":2885,"pil":1386,"pin":3678,"pis":1169,"por":1652,"pop":1746,"pos":1337,"pon":1531,"pok":1819,"pol":2768,"pua":1330,"pub":1942,"pte":1271,"pri":2203,"pre":1413,"pro":9219,"pur":2238,"pus":1843,"put":3676,"pun":4895,"pul":5693,"ra ":33891,"ngo":1466,"ngi":5434,"ngk":15500,"ngu":4341,"ngs":5639,"ni ":27725,"nge":5034,"ngg":24626,"ngh":3138,"nga":45848,"neg":5658,"nen":1469,"ner":3870,"net":1984,"nes":16997,"ng ":124434,"nci":3629,"nce":1815,"nca":3309,"ne ":3872,"ndu":6040,"ndr":1343,"ndo":17880,"ndi":8610,"nde":4365,"nda":13200,"ncu":1191,"nak":8053,"nal":12498,"nam":10027,"nan":12711,"nar":2327,"nah":2814,"nai":1146,"nd ":3909,"nat":2083,"nas":5218,"na ":14205,"nya":32061,"nye":2679,"nyi":2079,"nul":1194,"nun":2161,"nus":2178,"nur":1832,"nua":1507,"nto":2680,"ntu":18006,"ntr":1617,"nti":5215,"nta":22900,"nte":7428,"nst":1796,"nse":1390,"nsi":12613,"nt ":2342,"ns ":1443,"nol":1247,"nom":2318,"nny":5115,"no ":2934,"nka":1820,"nja":9105,"nju":2843,"nia":6702,"niv":1591,"nis":7959,"nit":2077,"nin":2190,"nik":2713,"ogr":1789,"ogi":3408,"ok ":3549,"ol ":2833,"ode":3528,"of ":1858,"odu":2149,"oh ":1516,"obe":1842,"ote":1332,"oto":2549,"ota":16038,"osi":2975,"ose":2201,"oso":1148,"ovi":10349,"ove":2391,"oun":1267,"ope":1909,"opa":1279,"os ":1902,"opu":1750,"or ":7039,"orm":2622,"oro":1416,"ord":1550,"ore":2472,"org":2231,"ori":3198,"ort":1972,"ora":11685,"ola":5593,"on ":12856,"oli":4593,"ole":13717,"olo":5388,"oka":1673,"om ":1677,"oko":1983,"ona":7292,"ond":2230,"one":17406,"ong":6397,"oni":2889,"ono":3029,"ons":2868,"ont":3055,"oma":3069,"ome":2094,"omb":1474,"omi":2626,"omp":4769,"omo":1445,"omu":1988,"la ":8248,"le ":2787,"lah":85384,"lag":1642,"lai":7338,"lal":2146,"lak":4119,"lan":18336,"lam":23788,"lap":1829,"lar":2452,"lat":9794,"las":5560,"law":2074,"lau":6152,"lay":9124,"kut":2035,"kus":1523,"kur":1949,"kup":1223,"kun":2321,"kum":1981,"kul":1651,"kuk":1717,"kte":1325,"ksi":4460,"kua":1980,"ktr":1281,"ktu":2650,"kti":1994,"kto":2651,"lok":1231,"lon":2362,"lom":2722,"log":4305,"lny":1353,"lmu":1446,"lua":4860,"lta":1301,"lu ":3267,"li ":7633,"lev":1141,"les":2673,"let":9284,"ler":1546,"lem":2195,"len":2677,"lek":2289,"leh":12733,"leb":3217,"lla":1957,"lle":1326,"lli":1227,"lka":1991,"lm ":4360,"ll ":1780,"lit":5426,"lis":7079,"lir":1515,"lip":1370,"lin":6938,"lim":6542,"lia":5362,"lik":11324,"lih":1781,"ma ":21495,"mah":2164,"mai":3887,"mak":3420,"mad":1850,"mar":3160,"mas":10730,"mal":1793,"man":22564,"mat":17173,"mba":10647,"mbi":2114,"mbe":6808,"me ":2693,"mbu":4891,"med":1539,"met":2294,"mes":1768,"mer":26378,"mem":15519,"mel":4982,"men":35186,"luk":2200,"lui":1193,"lun":1652,"lum":2455,"lus":1551,"lur":4662,"mpi":3325,"mpe":2325,"mpo":2760,"mpu":6558,"mod":1185,"mon":1666,"mor":1409,"mpa":7594,"mu ":1626,"mud":2039,"mua":1660,"mur":7325,"mus":3149,"muk":2006,"mul":2774,"mum":2269,"mun":4790,"mi ":4545,"min":4316,"mil":8646,"mis":1673,"mit":1227,"mia":1700,"mik":1662,"mla":1505,"mny":1911,"zam":1194,"yu ":1225,"ya ":33584,"yat":1924,"yar":2043,"yan":64564,"yak":4682,"yah":7474,"yai":2823,"yi ":1127,"wi ":1323,"wil":6167,"wa ":9713,"wan":5957,"wal":2130,"wak":2165,"wat":1765,"war":3553,"was":1762,"wah":1890,"vin":9845,"vis":1830,"ver":3522,"usi":7004,"use":1166,"usa":6560,"usu":2654,"ust":3983,"uti":2003,"ute":2083,"uta":7869,"utu":1851,"utr":2712,"us ":12050,"ut ":11892,"ura":9315,"uri":1909,"urk":1245,"uru":7051,"uny":2077,"upa":26147,"ur ":13684,"upu":1725,"ump":1843,"umu":2414,"umi":1414,"uml":1462,"umn":1617,"uma":4069,"umb":3461,"ume":1307,"unt":12867,"unu":2087,"uni":6942,"unc":1388,"und":1967,"una":10560,"ung":21765,"une":1335,"up ":3437,"uks":1606,"uku":6493,"uko":1586,"uki":1626,"um ":7458,"uka":7728,"uju":3230,"ulu":3880,"ult":1653,"uli":4656,"ula":12496,"un ":20290,"uk ":22268,"ul ":3263,"ui ":2114,"uga":6118,"uha":3495,"uda":4275,"udi":2630,"ubu":2471,"uh ":4643,"udu":3551,"ua ":5771,"uat":8034,"uas":4449,"uar":6223,"ual":1917,"uan":9121,"ubl":2137,"uba":1945,"uah":19677,"ty ":1866,"tur":4927,"tus":3183,"tuj":1345,"tul":1791,"tuk":15174,"tun":2681,"tum":1823,"tua":3005,"ts ":1124,"tra":8844,"tri":6158,"tru":1229,"tro":2826,"tu ":22458,"to ":3427,"tny":1162,"tob":1155,"ton":2911,"tok":1669,"tol":2048,"tor":4842,"til":2355,"tik":7507,"tif":2658,"tig":1707,"tit":1315,"tis":2474,"tin":8673,"tim":3584,"tio":3164,"tia":2861,"tid":3087,"tiv":1124,"tka":2458,"tem":9574,"ten":16531,"tek":2322,"tel":6089,"th ":1540,"tet":1414,"tes":1167,"ter":45895,"ti ":10357,"the":1811},"n_words":[11077227,12709440,9643042],"name":"id","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/is.json b/contrib/languages-data/is.json
new file mode 100644
index 0000000..5fd568b
--- /dev/null
+++ b/contrib/languages-data/is.json
@@ -0,0 +1 @@
+{"freq":{"Aþe":44,"D":2516,"E":3951,"F":4399,"G":3894,"A":5835,"B":5986,"C":2240,"L":4227,"M":5323,"N":3677,"O":1245,"H":8629,"I":1566,"J":2209,"K":4574,"U":1075,"T":3307,"W":992,"V":3210,"Q":137,"P":2862,"S":11572,"R":3778,"Y":369,"X":126,"Z":178,"f":81003,"g":100010,"d":53734,"e":187693,"b":27005,"c":5204,"a":263792,"n":216937,"o":68668,"l":133209,"m":99769,"j":33645,"Fel":69,"k":84595,"h":42906,"Fen":29,"i":195706,"w":1317,"v":51876,"u":132172,"Fer":95,"Bár":54,"t":142015,"s":170723,"r":261058,"q":297,"p":24810,"z":1088,"y":26307,"x":2164,"Óði":53,"²":107,"Í":3558,"É":51,"Æ":142,"Á":1654,"Ý":39,"Þ":3879,"Ú":294,"Ö":390,"Ó":667,"Fil":53,"í":48796,"Fin":139,"Fim":41,"Fir":29,"é":6992,"è":36,"æ":24190,"å":50,"ä":93,"á":41692,"à":32,"þ":19539,"ü":180,"ý":7865,"ú":17028,"Fja":106,"ø":114,"ö":23897,"Fis":39,"ó":30363,"ð":97792,"Eyj":294,"Eys":84,"Eyr":71,"ā":102,"Eyv":33,"ī":50,"ō":51,"š":31,"ū":35,"Fan":34,"Fal":50,"Far":50,"Að ":48,"Erl":41,"Elí":30,"Aða":120,"Eur":33,"ˈ":29,"Evr":590,"Ein":322,"Eir":81,"FL ":35,"́":73,"Eld":72,"Ekk":41,"μ":51,"End":39,"ν":134,"ο":164,"Eng":344,"ι":127,"κ":62,"λ":87,"δ":52,"ε":87,"η":55,"α":162,"ά":44,"ί":34,"Π":31,"Ell":80,"Er ":52,"ω":34,"ό":43,"Ens":32,"σ":73,"ς":149,"ρ":115,"π":60,"υ":43,"τ":83," l":13188,"ь":55," m":17205," n":9875,"я":55," o":24158," h":25667," i":1873," j":2491," k":10596,"ы":69," d":4603,"х":30," e":51762," f":25096," g":9131,"ч":38,"р":151,"с":242," a":22647," b":11739,"т":158," c":312,"у":71," y":1803," x":58," z":29," u":8441," t":16530," w":71," v":22667," p":2352," s":52785," r":7093,"К":36,"Önn":32,"М":32," J":2203," K":4567," H":8585," I":1555," N":3667," O":1188," L":4213,"Gei":146," M":5293," B":5951," C":2201,"Р":33,"С":41," A":5808," F":4383," G":3869," D":2495," E":3934,"л":143,"к":145," Z":177," Y":365,"й":61,"и":226," X":119,"п":31,"о":257,"н":183,"м":63,"г":31," S":11508,"Ges":41," R":3756,"в":111,"Ger":69,"б":78," Q":135," P":2823,"а":337,"Geo":100,"з":43," W":981,"Gen":80," V":3197,"е":181," U":1070,"д":70," T":3273," æ":1194," á":19136," í":21823," é":48," ö":1771," ó":1123," ý":438," þ":16844," ú":4082,"Gla":37," Á":1653," Æ":142," É":49," Í":3550,"Gil":84," Ó":667," Ö":387," Ú":293," Þ":3878," Ý":38,"Búr":44,"ה":29,"ו":29,"Ölf":40,"י":40,"Gal":86,"Gam":94,"Gat":36,"Gau":70,"Gar":113,"Gag":40,"Fræ":55,"Frá":36,"و":81,"ي":130,"ل":172,"م":100,"ن":92,"Ful":38,"د":85,"ح":41,"ت":33,"ب":97,"ا":239,"ع":51,"ش":31,"س":65,"ر":96,"Fru":56,"Fro":37,"Fyr":272,"Fyl":44,"Frí":39,"Bók":91,"Flu":50,"Flo":84,"Flj":91,"Fla":98,"Fle":86,"Bær":105,"Bæj":56,"Fra":556," К":36,"Fri":161," М":32,"Fre":118,"A ":369,"Fló":77,"Fos":51,"For":261,"Fjö":95," Π":31,"Fjó":52,"F ":112,"Da":796,"Cu":62,"Cl":179,"Dæm":64,"Co":498,"Cr":129,"Ce":91,"Ch":432,"Ci":87,"G ":277,"Ed":148,"Ea":44,"Du":121,"Dy":59,"Do":199,"Dr":201,"De":344,"Dj":72,"Di":273,"Aþ":47,"Bá":77,"Fe":260,"H ":152,"Fa":299,"Ey":536,"Hlu":52,"Eu":56,"Ev":632,"Ex":43,"Er":232,"Et":30,"Að":243,"Es":92,"En":512,"Em":79,"Ei":511,"El":310,"Ek":54,"Ef":192,"Hlj":179,"Eg":213,"Ge":554,"Ga":560,"Bú":144,"Hjá":32,"I ":270,"Bö":36,"Fy":316,"Bó":133,"Fu":127,"Fr":1134,"Bí":44,"Fo":403,"Fl":550,"Fj":284,"Bæ":200,"Fi":347,"B ":133," о":38," Р":33," С":41,"II ":73,"C ":290,"Av":40,"Au":691,"Ar":661,"At":288,"As":369,"D ":98,"Ba":1277,"Af":349,"Ag":60,"Aa":38,"Ab":97,"Ac":57,"Ad":101,"Am":376,"An":599,"Ap":140,"Ai":48,"Ak":259,"Al":997,"By":66,"Hit":42,"His":29,"Bu":149,"Br":1094,"Ca":498,"E ":99,"Hja":125,"Bi":297,"Be":746,"Hil":53,"Bo":829,"Bl":365,"Hin":97,"Bj":353,"Hip":31,"Kv":148,"Ku":90,"Gö":52,"Ky":159,"Kn":101,"Kl":180,"Kr":557,"Gí":124,"Ko":534,"Há":323,"Le":589,"Lj":88,"Hæ":71,"Li":540,"N ":121,"La":1290,"Lu":200,"Hó":220,"Ly":57,"Hö":355,"Hé":115,"Lo":531,"Hí":33,"Me":774,"Mi":929,"Mj":65,"O ":134,"Hú":1094,"Ma":1619,"Mc":44,"My":193,"Mu":181,"Ið":49,"Mo":567,"Nj":57,"Ni":233,"Já":37,"Ne":516,"Na":464,"P ":275,"Jö":155,"Hel":374,"Jó":681,"Hei":302,"Nu":48,"No":1524,"Hea":36,"Ok":35,"Ol":56,"On":43,"Oh":30,"Od":80,"Ká":86,"Hen":69,"Of":128,"Jú":86,"Hes":54,"Her":215,"Gi":186,"Gj":38,"Gn":42,"Gl":211,"Gr":968,"Go":225,"Gu":618,"Gv":43,"Gy":51,"J ":42,"Ha":2194,"He":1196,"Hi":344,"Dæ":71,"Hj":179,"Hl":321,"Hn":67,"Ho":620,"Hr":479,"Hu":359,"Hv":448,"Dó":65,"Dö":29,"K ":93,"Dý":44,"Ic":56,"Im":36,"In":761,"Il":85,"Is":113,"Eð":29,"Ja":517,"L ":154,"Fá":50,"Je":231,"Jo":341,"Fí":29,"Fæ":128,"Fé":158,"Fó":50,"Hag":49,"Haf":163,"Ju":93,"Hal":249,"Ka":1200,"Fö":37,"Han":1145,"M ":133,"Ham":103,"Har":211,"Kj":148,"Ki":277,"Hau":84,"Ke":369,"Ut":32,"Ur":47,"Up":198,"Um":210,"Un":333,"Ul":39,"W ":56,"Ty":102,"Guð":291,"Pó":142,"Tv":79,"Tu":195,"Pí":33,"Tr":409,"Pé":116,"To":307,"Th":490,"Ti":314,"Tj":67,"Pá":120,"Te":275,"Ta":376,"V ":127,"Sy":96,"St":1866,"Sv":974,"Su":942,"Sí":179,"Wo":95,"Wi":310,"Sæ":90,"Wh":42,"Sé":40,"Wa":265,"Rú":290,"We":159,"Sá":78,"Ró":242,"Rö":41,"Vo":162,"Rí":181,"Úlf":48,"Vi":653,"Ré":30,"X ":104,"Úkr":30,"Va":711,"Rá":45,"Ve":1144,"Má":196,"Lý":96,"Grá":30,"Græ":154,"Mæ":40,"Gun":117,"Ló":43,"Pu":82,"Pr":335,"Lí":171,"Lú":62,"S ":238,"Lö":102,"Lá":110,"Pe":379,"Gua":30,"Pa":545,"Pl":227,"Po":355,"Pi":180,"Ph":126,"Gul":70,"Læ":49,"Os":65,"Ot":31," ا":112,"Op":55,"Kí":164,"Or":413,"R ":119,"Kú":67,"Kó":214,"Ox":46,"Kö":70,"Se":576,"Sc":214,"Sj":305,"Si":722,"Sh":148,"Sn":277,"Sm":127,"Sl":218,"Sk":1512,"Sp":372,"So":351,"Ru":106,"Nó":36,"U ":71,"Sa":1406,"Ný":191,"Nú":101,"Mý":140,"Re":1401,"Ná":93,"Ri":256,"Rh":36,"Næ":38,"Ro":345,"Ní":89,"Mí":43,"Qu":74,"Grí":106,"Mó":77,"Mö":80,"T ":79,"Mú":128,"Gró":29,"Mü":48,"Ra":615,"Gre":171,"Gri":130,"Gra":178,"b ":287,"Gru":62,"Gro":42,"a ":54859,"Gnú":34,"Tú":51,"Tö":63,"Yf":66,"Ya":33,"Yo":164,"Gle":35,"Sú":126,"Sö":178,"Só":163,"Glo":38,"Tá":38,"Sý":65,"Té":39,"Tæ":41,"Tó":200,"Tí":127,"Vö":72,"Gol":33,"Got":52,"Za":29,"Ze":49,"Vé":39,"Ví":264,"bö":256,"i ":42436,"fv":268,"bó":1425,"fy":4976,"gd":575,"ge":6266,"gf":407,"bú":1553,"ga":14643,"gb":298,"bý":357,"fk":117,"Ing":198,"fj":4059,"bæ":1840,"fm":226,"fl":4455,"fg":129,"ff":584,"fi":7839,"fh":173,"fs":2374,"fr":11940,"Inn":76,"fu":5606,"ft":4797,"Int":121,"fo":2567,"fn":8061,"Ins":29,"bí":505,"j ":69,"gy":295,"hf":59,"he":10053,"dá":141,"hb":32,"ha":7740,"gn":2900,"gm":495,"gl":2693,"gk":130,"gj":2483,"gi":7861,"gh":415,"gg":3399,"gv":803,"gu":10964,"gt":1944,"gs":3263,"gr":4868,"go":889,"dt":82,"du":6343,"dv":327,"dw":47,"dy":153,"g ":30278,"ea":698,"eb":435,"ec":343,"ed":647,"de":2371,"dd":1771,"dg":199,"df":200,"di":10756,"dh":181,"dk":373,"dj":161,"dm":128,"dl":570,"do":802,"dn":635,"ds":4031,"Ill":42,"dr":2023,"aó":39,"ew":295,"ex":734,"eu":292,"ev":474,"ey":6435,"aö":53,"ez":75,"fa":5499,"h ":796,"aú":77,"Ind":243,"fd":62,"bá":511,"fe":3046,"fb":250,"aþ":342,"eh":101,"eg":10758,"ef":8971,"ee":293,"aá":119,"el":7041,"ek":4309,"aæ":251,"ej":35,"ei":27077,"ep":1965,"eo":285,"en":19500,"em":17579,"et":5721,"að":23009,"es":9022,"er":48228,"eq":33,"aí":380,"ca":618,"e ":4753,"by":1518,"bs":81,"br":3631,"bu":941,"bn":48,"bo":3424,"bj":538,"bl":1526,"bi":1485,"bb":255,"be":3254,"db":244,"da":11080,"f ":7700,"cy":44,"cu":238,"ct":305,"cs":53,"cr":172,"co":503,"cm":37,"ck":446,"cl":64,"ci":425,"ch":1138,"ce":634,"cc":89,"c ":322,"az":129,"ay":341,"ba":3436,"d ":7343,"at":7217,"as":9693,"ar":51349,"aq":51,"ax":603,"aw":133,"av":2668,"au":7198,"ak":4372,"al":17385,"ai":546,"aj":229,"ao":105,"ap":1426,"am":11839,"an":36124,"ac":749,"ad":1746,"aa":265,"ab":2521,"ag":8003,"ah":1785,"ae":748,"af":17312,"nu":14078,"nt":3812,"ns":14817,"nr":473,"np":59,"no":4303,"nn":28163,"q ":62,"jö":4211,"nz":284,"ny":302,"nw":40,"jó":9027,"nv":1399,"oe":103,"ká":825,"of":4047,"oc":382,"od":513,"oa":97,"ob":315,"jú":991,"om":2016,"on":7027,"ok":2788,"ol":2476,"oi":174,"oj":70,"kæ":150,"og":21658,"oh":219,"ot":3409,"m²":103,"os":2326,"ov":311,"ou":631,"op":1077,"oo":305,"or":15125,"kí":733,"kú":380,"r ":98244,"ox":157,"ow":306,"kó":2090,"oz":31,"kö":623,"oy":162,"lá":981,"pe":1831,"pf":85,"pg":70,"pa":2152,"pb":43,"ký":317,"pl":1650,"lé":431,"pm":157,"pn":706,"po":550,"ph":775,"pi":1889,"pj":95,"læ":985,"pk":33,"lo":2737,"ln":490,"hé":660,"lm":1436,"ll":11846,"ls":4614,"lr":408,"hí":33,"lp":267,"hó":925,"lv":1365,"lu":9341,"lt":2655,"hö":2143,"lz":40,"ly":1089,"hú":1293,"o ":2043,"md":435,"ma":10956,"hý":74,"mb":2218,"mg":109,"mh":618,"me":10367,"iá":30,"mf":545,"mk":703,"ml":1350,"mi":7819,"mj":551,"mn":514,"mm":1678,"mp":1250,"mo":647,"mr":361,"mt":1727,"ið":22433,"ms":4462,"mv":384,"mu":3055,"my":2358,"p ":1704,"na":24623,"nb":713,"iþ":48,"nc":512,"nd":28901,"ne":5566,"já":2743,"nf":978,"ng":20121,"nh":697,"ni":18092,"nj":745,"nk":1747,"nl":1765,"nm":750,"fó":868,"fð":1006,"ju":3468,"fí":157,"jo":90,"fé":1623,"fæ":1715,"Dýr":35,"gæ":257,"kj":4569,"ki":11936,"kh":360,"kg":80,"kf":253,"ke":4269,"gá":474,"kd":187,"fþ":53,"kb":82,"ka":13629,"fú":58,"m ":40783,"fö":400,"gó":341,"Ice":47,"ky":1074,"gö":726,"ks":1875,"kt":2936,"gð":1600,"ku":9598,"kv":2762,"ko":4051,"gí":357,"kr":3178,"kk":5618,"kl":2343,"km":1291,"kn":2921,"li":12184,"lh":740,"lk":1973,"hæ":1097,"lj":3578,"há":1640,"le":15573,"ld":8186,"lg":1902,"lf":2514,"la":24339,"lc":60,"gþ":48,"lb":662,"gú":251,"n ":36944,"hr":2163,"hs":37,"dí":356,"hv":2491,"dó":1251,"hw":52,"ht":210,"hu":1451,"dæ":682,"hj":645,"hi":2023,"hn":524,"ho":1432,"hl":4630,"hm":61,"id":1046,"ic":1301,"ib":562,"dý":970,"ia":1044,"ih":1334,"ig":5591,"if":2163,"ie":864,"dö":206,"hy":512,"dú":164,"k ":6366,"iq":49,"ir":20666,"is":11681,"eð":11441,"it":9396,"iu":360,"iv":434,"eó":56,"ix":96,"ii":120,"ij":65,"ik":5708,"il":15253,"im":4416,"in":45324,"io":856,"ip":2056,"je":151,"fá":352,"ji":208,"iz":110,"iy":35,"l ":12639,"ja":12542,"tæ":3059,"xi":212,"xn":82,"xl":137,"té":125,"xp":31,"tí":3010,"tó":2681,"xt":373,"xu":39,"sö":1819,"só":1189,"ww":36,"sú":642,"z ":258,"sþ":235,"sý":1460,"xa":252,"xf":48,"xe":64,"tá":725,"wi":159,"sæ":968,"sé":1567,"wn":54,"wo":89,"sí":2536,"ws":87,"ró":3050,"rö":1573,"y ":1711,"rú":1794,"rý":331,"wa":277,"rþ":186,"sá":1042,"we":162,"ré":1220,"ræ":6045,"vi":9174,"rð":12661,"vu":610,"vr":778,"rí":6044,"vn":41,"vo":2712,"uz":51,"uy":35,"ux":78,"uv":609,"uu":36,"rá":5069,"ve":15820,"va":13225,"x ":601,"ui":242,"uj":51,"uk":1254,"ul":3466,"ue":357,"uf":941,"ug":3646,"uh":528,"ur":34056,"us":5972,"ut":4027,"um":24785,"un":17263,"uo":98,"up":2929,"ty":774,"tz":87,"pö":182,"tu":13668,"tt":12048,"tw":67,"pó":246,"tv":2024,"ub":622,"pý":31,"ua":273,"ud":463,"uc":259,"w ":311,"pú":114,"to":3449,"tn":2675,"Hví":124,"tm":438,"tl":2066,"ts":2200,"tr":5330,"pí":449,"tp":67,"tg":495,"tf":506,"te":7118,"pá":304,"td":76,"tk":513,"pæ":160,"tj":2442,"ti":20846,"th":1669,"v ":113,"tb":686,"tc":63,"ta":23969,"su":3147,"sv":4635,"ss":7478,"st":37992,"oð":819,"sy":891,"sl":8316,"sk":20851,"sn":2092,"sm":2339,"sp":2463,"so":2878,"sr":592,"sd":792,"nþ":148,"sc":305,"sf":1106,"se":20267,"sh":1736,"sg":382,"sj":2608,"si":7701,"rz":66,"nö":212,"u ":23701,"nú":1706,"sa":9556,"ný":623,"sb":788,"rr":3768,"rs":10257,"rt":4340,"ru":11212,"rv":1275,"rw":51,"nó":446,"ry":982,"ní":1001,"rp":1277,"ro":2458,"rn":7451,"rm":2415,"né":65,"rl":3496,"rk":5904,"rj":1626,"næ":1092,"ri":23025,"rh":1825,"rg":5415,"rf":4667,"re":10640,"ná":1602,"rd":1531,"rc":245,"mþ":95,"rb":1584,"mý":131,"ra":19142,"t ":24652,"mú":288,"mö":974,"mó":746,"qu":172,"mí":442,"mæ":806,"Dóm":48,"má":3190,"Hrú":49,"lý":968,"lþ":583,"s ":20225,"lú":270,"Hró":53,"py":298,"lö":2699,"pt":1531,"pu":2231,"pv":50,"ló":1489,"pp":4302,"lí":4068,"pr":1694,"ps":1356,"Hun":69,"Hum":33,"Hve":94,"Hva":208,"Hug":182,"yð":586,"xá":78,"Hri":58,"Hre":99,"Hra":144,"xí":115,"Hlí":31,"vö":1344,"zz":45,"Hor":96,"zi":76,"uþ":108,"ze":74,"za":123,"Hon":37,"Hol":265,"zu":44,"zo":77,"ví":5140,"How":29,"zk":225,"væ":2362,"vé":379,"yg":1822,"ye":103,"uá":36,"yf":2085,"yc":47,"tþ":77,"yd":149,"ya":182,"yb":37,"tý":358,"tú":766,"tö":3200,"yv":43,"yt":1440,"uð":6077,"ys":971,"Hof":73,"yr":6845,"yp":342,"yo":75,"yn":4442,"ym":390,"yl":1475,"yk":1688,"yj":1636,"uæ":73,"yi":92,"Arg":37,"Are":29,"Ara":128,"Arn":175,"Ari":100,"Alþ":272,"App":83,"Atl":210,"Ast":30,"Ass":31,"Art":46,"Aus":516,"Auk":30,"Aug":34,"Así":208,"Auð":65,"Óly":64,"Óma":39,"Óla":271,"Bak":55,"Bal":91,"Ban":640,"Bar":205,"Bas":64,"Ósk":37,"Afr":254,"Afg":30,"Air":29,"Akr":70,"Aku":116,"Ala":63,"Alb":62,"Alg":63,"Ale":88,"Alf":93,"Alm":57,"All":93,"Ame":231,"Ama":30,"Ang":85,"Ani":38,"Ana":32,"And":211,"Ant":112,"Ann":73,"Bur":43,"Bry":34,"Bru":46,"Byg":36,"Íþr":48,"² ":107,"Brú":53,"Cal":54,"Cam":73,"Cas":34,"Car":150,"Cat":30,"Can":52,"Bea":37,"Á ":621,"Bey":33,"Bet":31,"Ber":266,"Ben":118,"Bel":130,"Bei":52,"Ás":264,"Ár":416,"Bib":37,"Ág":72,"Ál":120,"Bja":184,"Bil":56,"Bis":37,"Bir":67,"CO ":32,"Bla":79,"Blö":85,"Bló":44,"Bre":500,"Bra":209,"Bro":90,"Bri":92,"Bjö":145,"Bol":62,"Boo":38,"Bor":456,"Bos":43,"Bot":40,"Blá":56,"Ól":368,"Óm":42,"Ód":30,"Íþ":49,"Íb":166,"Ít":191,"Ís":2281,"Ír":171,"De ":44,"Dev":30,"Í ":655,"Dei":35,"Del":38,"Den":31,"Dan":318,"Dar":51,"Æv":33,"Dav":95,"Æt":40,"Dag":64,"Dal":141,"Áð":37,"Chr":82,"Che":50,"Chi":122,"ám":763,"án":1343,"áp":70,"Cit":32,"ái":182,"ák":1231,"ál":5705,"áe":70,"áf":560,"ág":530,"áh":466,"áa":140,"áb":249,"ád":63,"áv":511,"áu":67,"ár":6093,"át":2849,"ás":1809,"Þæ":99,"Þá":57,"Þý":326,"Þó":307,"æ ":266,"Cla":106,"Cen":41,"Þa":1137,"Þe":606,"Þi":250,"Þj":278,"Þo":417,"Þu":34,"Þr":200,"Þy":38,"Þv":74,"á ":17048,"Cha":121,"Cri":38,"Úr":59,"Út":101,"Úk":30,"Úl":56,"Ós":82,"Öl":88,"Ön":69,"Ör":97,"Ös":31,"Óð":61,"Öx":50,"Cor":85,"Com":79,"Col":95,"Con":100,"Cou":37,"ós":1395,"ót":2821,"アアア":152,"óu":215,"óv":375,"ö ":331,"ðá":72,"óa":487,"ób":406,"ój":46,"ói":177,"óh":245,"óg":351,"óf":1069,"óe":87,"ód":134,"ór":3904,"óp":1247,"ón":3499,"óm":3062,"ól":4371,"ók":2164,"íð":2645,"íó":153,"íþ":437,"ðr":2069,"ðs":3396,"ðp":30,"ðv":1344,"ðt":171,"ðu":11785,"ó ":1141,"ðl":1590,"ðm":400,"ðn":620,"ðo":29,"ðh":571,"ði":13616,"ðj":880,"ðk":295,"ðd":125,"ðe":430,"ðf":1240,"ðg":495,"ða":17064,"ðb":974,"ív":134,"íu":2135,"ín":2687,"ím":2131,"íp":113,"ío":119,"ír":336,"éð":84,"ít":1441,"ís":5447,"ð ":40406,"íg":419,"íh":106,"íe":115,"íf":1316,"ík":5310,"íl":945,"íd":173,"ía":767,"íb":632,"Egy":100,"Egi":67,"Eft":63,"í ":21136,"æð":6732,"Efn":44,"ék":185,"él":2012,"én":89,"és":167,"ét":1646,"ér":1779,"éu":87,"éb":41,"éf":124,"ég":31,"Edd":90,"Ef ":37,"é ":624,"æv":174,"æx":41,"æs":710,"ær":4290,"æt":3260,"æn":1633,"æp":103,"æk":1777,"æj":307,"æm":1338,"æl":1029,"æg":1139,"æf":427,"æi":109,"æd":792,"áð":1784,"áæ":82,"FC ":43,"ýð":730,"þá":1196,"þí":45,"þæ":401,"þé":152,"þó":497,"þú":323,"þý":829,"þö":79,"Dia":31,"Dis":97,"ýl":453,"ýk":129,"ýj":265,"ýf":75,"ýd":43,"ýa":30,"ýv":32,"ýs":2406,"ýt":324,"ýr":1984,"ýp":92,"ým":480,"ýn":556,"þj":1460,"þi":534,"þe":4740,"þa":4830,"þv":1549,"þy":303,"þr":1838,"úð":432,"þu":272,"þo":314,"ý ":120,"ün":52,"ür":71,"þ ":167,"ая ":31,"úp":309,"ún":2987,"úm":548,"úl":849,"úk":435,"úv":113,"úu":46,"öð":2279,"út":3137,"ús":2168,"úr":2334,"úa":1374,"úi":308,"úf":229,"úg":271,"úd":160,"úe":84,"úb":118,"óð":3019,"öx":90,"ør":44,"Dre":57,"Dra":80,"óþ":62,"ú ":1027,"Djú":38,"ðþ":39,"ðö":84,"öt":1074,"öu":59,"ör":3745,"ös":255,"öp":100,"ön":3508,"öl":4517,"öm":308,"ök":1689,"öf":2560,"ög":3309,"öd":29,"Don":53,"Nef":35,"Net":32,"Nes":42,"Nas":29,"Nat":57,"Nau":55,"Nic":32,"Neð":45,"Nin":66,"Nik":35,"New":193,"Myn":130,"Nar":29,"Naf":146,"Jök":86,"Jóh":104,"Jót":38,"Jór":29,"Jón":432,"Jól":33,"Njá":30,"OS ":34,"Nor":1398,"Not":36,"Kár":32,"Odd":60,"Oft":93,"Jör":55,"Júl":36,"Orð":204,"Kór":59,"Kóp":83,"Kól":42,"Oxf":36,"Kín":133,"Org":32,"Ork":38,"Pla":169,"Læk":37,"Pin":45,"Pho":31,"Phi":60,"Lár":54,"Per":153,"Pet":79,"Pen":61,"Pat":43,"Par":152,"Pau":49,"Pan":54,"Pap":35,"Pal":72,"Pak":41,"Lög":81,"Pro":96,"Pri":96,"Pre":57,"Líf":63,"Pol":31,"Pot":52,"Por":129," ال":82,"Mál":122,"Már":34,"Lýð":56,"SA ":48,"Múl":61,"Mús":34,"Raf":70,"Rad":29,"Rag":108,"Mün":41,"Ran":132,"Que":34,"Isl":57,"Jaf":30,"Jac":46,"Jar":64,"Jap":87,"Jan":110,"Jam":61,"Jak":50,"Jen":40,"Jer":68,"Jes":43,"Fél":151,"あ":61,"Fær":114,"Jos":55,"Jon":44,"ア":218,"Joh":167,"Jul":35,"Kam":65,"Kal":223,"Kap":39,"Kan":178,"Kau":134,"Kat":86,"Kas":91,"Kar":285,"Ker":45,"Ket":54,"Ken":76,"Kep":34,"Kel":35,"Kef":52,"Kir":145,"Kin":55,"Kja":80,"Kna":56,"Kle":57,"Kla":45,"LP ":162,"Kon":176,"Kom":44,"Kol":166,"Kos":29,"Kor":44,"Kr ":66,"Kjö":29,"Gís":50,"Gín":44,"Kra":61," ア":29,"Kri":234,"Kro":41,"Kyn":37,"Kyr":107,"Kró":49,"Íbú":146,"Kve":46,"Kvi":48,"Ísa":105,"Írl":73,"Hát":37,"Let":29,"Hás":150,"Les":33,"Íta":188,"Len":37,"Hál":31,"Hák":37,"Lei":237,"Ísl":2081,"Ísr":41,"Lau":192,"Lax":122,"Íra":76,"Le ":30,"Lag":87,"Lat":43,"Lao":32,"Lam":35,"Lan":545,"La ":46,"Hér":92,"ML ":30,"Lim":33,"Lin":79,"Lis":104,"Lit":139,"Lun":86,"Lou":37,"Los":48,"Lof":40,"Lon":229,"Ljó":82,"Hör":55,"NA ":58,"Hún":965,"Hól":152,"Höf":247,"Men":138,"Mel":83,"Mes":43,"Mer":93,"Met":61,"Meg":45,"Med":40,"Mex":54,"Hús":107,"Man":190,"Mal":118,"Mar":589,"Mas":67,"Mag":199,"Mad":42,"Mak":33,"Mai":40,"Mac":73,"Max":34,"Mat":90,"Mið":393,"Mol":36,"Mon":128,"Mos":92,"Mor":172,"Mot":32,"Mjó":31,"Mik":116,"Mic":123,"Með":113,"Mis":52,"Mil":62,"Min":98,"Mun":57,"Mus":51,"Ték":31,"Sýr":30,"Sög":42,"Sön":96,"Sók":35,"Sól":71,"Síð":58,"Wor":55,"Wil":110,"Win":83,"Wii":32,"Wei":29,"Wes":69,"Rús":196,"Rún":33,"Was":36,"War":47,"Wal":110,"Róm":176,"ék ":30,"él ":93,"éf ":34,"éfa":46,"Vol":51,"Rík":122,"Vis":32,"Veð":31,"Ráð":30,"Við":210,"ést":58,"étr":52,"étt":1263,"étu":131,"Vík":49,"Vín":30,"Vís":51,"Víe":38,"ékk":137,"éla":1571,"én ":35,"éli":43,"élt":37,"éra":473,"érh":132,"ét ":143,"érs":261,"éru":58,"éu ":77,"ér ":713,"élö":131,"之":47,"並":35,"三":94,"Yor":108,"You":39,"æð ":244,"Yfi":66,"æða":580,"æði":5160,"æðu":478,"æðs":139,"æðr":99,"Tóm":44,"Tón":101,"Töl":55,"Tím":60,"Svæ":31,"Sví":230,"Suð":653,"Stó":139,"Stö":59,"Stæ":32,"Þýs":303,"Svi":95,"Sve":238,"Sva":354,"Sum":80,"Sup":30,"Sun":82,"Sty":51,"Str":276,"Stu":161,"Stj":104,"Sti":42,"Sto":146,"Sta":383,"Spá":114,"Ste":372,"Ten":40,"Pál":100,"Tel":34,"Tan":53,"Tal":83,"Skj":90,"Ski":81,"Sko":140,"Skr":62,"Sku":31,"Ska":549,"Ske":116,"Shi":29,"Sha":56,"Sim":64,"Sil":74,"Sig":433,"Sin":41,"Ser":78,"Set":31,"Sey":30,"Sen":51,"Sel":128,"Sem":35,"Sei":38,"Seg":32,"Snæ":134,"Spa":50,"Skú":52,"Spi":49,"Spe":34,"Slí":40,"Slé":43,"Sló":37,"Smá":43,"St ":29,"Sjá":155,"Sjö":30,"Sjó":78,"Sno":80,"Ská":150,"Sog":33,"Sof":40,"Sou":36,"Sov":61,"Skó":140,"Sol":36,"Son":39,"Skí":37,"Sky":37,"Smi":30,"обл":31,"Sag":99,"Saf":35,"Sam":545,"Sal":86,"Sco":36,"Sch":99,"Sax":56,"Sav":41,"Sau":95,"Sar":46,"San":244,"Nýj":100,"SI ":80,"Rey":1063,"Nát":43,"Rit":75,"Ric":60,"Rau":132,"Mýv":29,"Mýr":98,"SG ":207,"Rei":54,"Reg":63,"Rob":50,"Roc":43,"Ros":47,"Rom":33,"áæt":78,"Ven":79,"Veg":63,"Vei":40,"Vat":207,"Van":47,"Val":271,"Var":102,"Vig":35,"Vir":57,"Vil":122,"Vik":42,"Vin":79,"áð ":233,"Ver":208,"Ves":624,"Vet":35,"Um ":130,"áða":140,"áðg":34,"áðh":171,"áði":225,"áðu":798,"áðs":118,"Und":51,"Ung":70,"Uni":181,"Upp":182,"æg ":91,"ædd":776,"æfi":141,"Pól":106,"æfe":104,"æfa":50,"Tró":34,"Trö":38,"Tyr":85,"ægj":69,"ægi":110,"æga":135,"ægr":102,"ægt":328,"ægu":162,"æju":29,"æfð":38,"Tví":41,"æin":69,"æja":272,"æl ":48,"æli":334,"æll":56,"ælt":61,"ælu":92,"æma":80,"æmd":120,"æki":736,"ækj":217,"ækk":69,"ækn":356,"ægð":100,"ækt":237,"æku":92,"æn ":110,"æla":272,"æld":77,"ænu":227,"ænt":81,"æns":354,"ænm":30,"ænl":161,"ænn":155,"æni":89,"æng":118,"ænd":99,"æna":190,"æmu":71,"æmt":313,"æmi":690,"æpl":30,"ær ":1321,"æsi":41,"æsl":32,"æsk":45,"ærr":162,"ærs":892,"ært":38,"æru":151,"æra":217,"æri":815,"ære":175,"ætt":2050,"ætu":294,"æta":147,"æti":456,"ætl":282,"æst":554,"Tex":43,"Ter":34,"ævi":136,"æva":29,"ærð":475,"Tha":29,"The":298,"æxl":41,"Tho":106,"Til":161,"Tim":38,"Tin":49,"Tit":29,"Pét":113,"Tor":78,"Tom":37,"Tos":37,"Tjö":35,"Try":40,"Tro":45,"Tri":44,"Tre":49,"Tra":74,"Tur":46,"Tun":86,"Þrí":29,"Þrá":37,"Þve":45,"ああ":41,"áum":45,"átu":169,"bju":54,"átt":1937,"bja":121,"áti":93,"átr":47,"bis":218,"bit":57,"áss":31,"ást":312,"bir":182,"bik":50,"bil":637,"bin":115,"áta":175,"ble":145,"áve":79,"árá":48,"bli":128,"árð":42,"bla":414,"áva":365,"bol":146,"bog":60,"átö":52,"bjö":156,"bjó":189,"átí":162,"ásö":38,"blí":66,"bló":334,"ávö":33,"blö":167,"blá":155,"blæ":36,"bon":36,"bor":2681,"bot":163,"álk":336,"áll":88,"álm":334,"bbi":37,"álp":80,"áls":457,"álu":194,"álv":69,"bba":96,"be ":48,"áma":116,"bbu":35,"ámi":57,"bam":50,"ban":1603,"bak":368,"bal":62,"ákn":429,"bah":47,"áko":53,"áku":34,"bac":32,"bad":40,"áks":66,"ákv":456,"án ":338,"ála":688,"ágú":123,"álf":1089,"bau":182,"ále":131,"bat":32,"áld":466,"bas":167,"bar":589,"áli":555,"álh":106,"álg":55,"áns":123,"ánn":67,"ánu":150,"áms":328,"ámu":55,"ánd":108,"ána":282,"áni":194,"ás ":143,"álí":29,"bi ":52,"bei":420,"ár ":710,"ber":1931,"ben":97,"bel":260,"bek":34,"bey":64,"álæ":106,"bes":144,"bað":32,"bet":171,"ása":364,"árv":81,"áru":387,"árr":36,"árs":356,"ásk":711,"ási":75,"bib":31,"áse":31,"bif":37,"át ":80,"árm":104,"árn":118,"árk":58,"árl":107,"ári":3035,"árg":32,"árh":61,"áre":46,"árf":67,"árd":141,"ára":516,"áfa":149,"ábó":66,"áfu":225,"áfe":61,"áfr":66,"áhe":82,"áha":37,"ágs":40,"ágr":106,"águ":57,"ca ":193,"áhu":51,"áhr":211,"car":60,"cas":39,"cat":48,"can":86,"ál ":816,"áir":32,"cal":71,"cag":30,"áin":106,"ce ":210,"áka":90,"ám ":162,"brj":92,"bri":289,"bro":253,"bra":622,"bre":1260,"bry":35,"bu ":44,"bru":150,"boð":197,"bur":454,"bun":269,"bus":56,"áa ":42,"brá":62,"bræ":116,"bré":110,"bró":94,"brö":90,"by ":34,"brú":361,"áan":33,"áar":36,"ábe":37,"áby":43,"byl":138,"byg":1051,"bys":77,"ábr":30,"byr":205,"aka":724,"am ":1285,"ake":553,"aki":430,"afæ":30,"afé":292,"afð":193,"afí":36,"afó":34,"al ":1641,"aja":65,"ail":51,"ain":143,"air":55,"ais":50,"ak ":482,"ahy":29,"Þór":281,"adó":89,"adý":30,"ahl":129,"adæ":81,"ahv":94,"adí":88,"ahr":268,"aho":81,"agv":46,"ahe":110,"aha":506,"agj":32,"agk":31,"agl":139,"agf":101,"agg":36,"agi":919,"agr":297,"ags":903,"agt":139,"agu":274,"agn":1177,"ago":116,"anv":402,"anu":572,"ajö":66,"ano":128,"ann":7640,"anm":237,"ant":646,"ans":2276,"anr":62,"ane":475,"anf":63,"ang":1850,"anh":50,"ani":508,"anj":69,"ank":389,"anl":325,"ap ":71,"ana":1067,"anb":53,"anc":186,"and":12634,"amu":36,"amt":977,"amv":106,"amy":55,"amm":631,"aml":867,"amo":77,"amn":326,"amp":188,"ams":715,"amr":194,"amk":501,"amh":353,"ami":326,"amf":296,"amg":65,"amd":67,"ame":964,"amb":945,"ama":1875,"ahú":86,"ahö":183,"aly":37,"alv":169,"alu":337,"alt":530,"als":1039,"alr":40,"alp":48,"alo":66,"aln":159,"alm":385,"all":4558,"alk":134,"alg":346,"alh":154,"ali":1259,"ald":2532,"ale":741,"ahá":51,"alf":139,"ala":1875,"alb":102,"agö":58,"an ":5852,"agó":47,"akv":56,"aks":182,"akr":86,"aku":48,"akt":192,"agð":277,"ako":186,"akk":529,"akj":37,"akm":75,"akl":519,"aba":247,"abb":91,"abe":147,"abi":501,"abj":35,"abl":134,"abo":208,"abr":215,"abu":41,"aby":70,"ae ":196,"aaf":34,"aal":29,"aar":39,"ad ":170,"ac ":69,"afn":2524,"afm":183,"afo":148,"afr":1163,"abí":243,"aft":662,"afs":676,"aff":148,"abá":34,"afe":261,"afh":109,"afj":658,"abæ":120,"afi":1278,"afl":1145,"afk":73,"ai ":55,"abú":61,"aga":1887,"agb":56,"age":255,"afu":287,"abó":285,"afy":94,"ael":113,"aef":43,"aei":56,"ah ":67,"afa":1337,"afb":140,"aey":189,"ado":94,"adr":85,"adi":100,"add":86,"ade":268,"ag ":1083,"ads":38,"aco":29,"ack":101,"aci":77,"ach":162,"ace":117,"ada":509,"af ":5722,"act":62,"avé":39,"aví":1064,"avö":101,"atæ":44,"axi":148,"axl":39,"axn":54,"atí":446,"Þát":29,"axt":48,"asó":42,"asö":150,"az ":37,"axa":101,"asý":141,"atá":67,"auð":861,"atö":90,"ató":78,"aya":40,"aye":40,"ba ":118,"bb ":35,"axá":78,"Þær":73,"amó":187,"amí":123,"amú":39,"at ":305,"amö":61,"arh":891,"arg":845,"arf":2166,"are":429,"ard":705,"amþ":87,"arc":90,"arb":562,"ara":2184,"arp":734,"aro":178,"arn":1542,"arm":909,"arl":1420,"ark":1210,"arj":31,"ari":2716,"aru":75,"arv":495,"anó":79,"aní":198,"arr":610,"ars":2758,"art":990,"au ":547,"anú":173,"asa":1107,"ary":75,"asi":329,"ash":111,"anþ":38,"ase":151,"aso":291,"asn":37,"asp":70,"ask":904,"asj":57,"asm":56,"asl":51,"akí":50,"aos":34,"akó":58,"ar ":24659,"apa":379,"ape":84,"alæ":30,"api":60,"aph":36,"apl":48,"apo":29,"app":288,"alí":469,"apr":187,"apu":54,"aló":52,"alö":117,"as ":455,"alþ":280,"alý":31,"amá":227,"amæ":216,"ava":456,"ax ":56,"aut":633,"arð":2612,"arí":1423,"avo":132,"aré":99,"avn":36,"avi":270,"aræ":58,"ave":508,"ará":337,"arþ":116,"arú":66,"ay ":120,"awa":61,"arö":154,"aró":106,"así":119,"atb":114,"ata":1386,"asu":87,"ast":4650,"ass":502,"asy":41,"asv":193,"atn":1309,"atk":61,"atl":50,"atr":203,"ato":180,"ate":387,"atf":38,"ati":491,"ath":330,"att":952,"ats":86,"atv":191,"atu":294,"apö":55,"aul":90,"aum":222,"aun":895,"aup":528,"aur":125,"aus":1761,"auf":120,"aug":671,"auk":646,"ος":71,"ος ":71,"ς ":149,"ν ":35,"Víð":50,"α ":38,"Völ":29,"アア":185,"Það":730,"Þet":130,"Þes":144,"Þei":197,"Þeg":77,"Þin":237,"Þor":406,"Þjó":274,"Þau":111,"Þar":262,"jen":38,"fán":134,"fás":32,"jað":148,"ji ":76,"jad":61,"jaf":886,"jab":64,"jat":92,"jas":441,"jav":807,"jap":160,"jar":3369,"jal":1563,"jak":139,"jan":1062,"jam":60,"jah":111,"fá ":59,"fél":1458,"fék":83,"fæð":108,"ск":66,"fé ":69,"fæt":90,"fæs":142,"fær":603,"fæd":750,"ст":78,"itn":80,"eðn":127,"itm":37,"itl":270,"eðl":410,"eðr":140,"itr":160,"ito":53,"itv":30,"eðv":33,"eðu":188,"itu":683,"itt":1458,"its":233,"eðs":115,"itz":45,"ity":146,"isk":905,"isj":46,"ism":523,"isl":359,"iso":62,"isn":159,"isp":55,"iss":785,"isr":186,"isu":36,"ist":4346,"isv":198,"eða":6425,"ita":1646,"ite":114,"eðf":92,"itg":57,"ith":325,"eði":184,"iti":2561,"eðj":70,"irð":784,"irí":91,"isá":31,"ius":175,"ium":116,"iva":54,"ix ":65,"ivi":61,"iræ":48,"ive":235,"ilí":122,"ipp":147,"ipu":275,"ips":70,"ipt":795,"ipi":107,"ipl":30,"ilö":36,"is ":1869,"ion":501,"ior":56,"ipa":275,"ipe":46,"ir ":13932,"iru":61,"irv":49,"irs":247,"irt":855,"iní":40,"irr":770,"iro":49,"irm":92,"irn":320,"irk":1284,"irl":451,"iri":354,"isi":625,"ish":290,"isg":63,"isf":275,"ise":129,"isd":44,"isc":101,"isb":133,"isa":319,"iu ":46,"iqu":44,"irf":191,"ire":187,"irh":41,"irg":228,"irb":292,"ira":203,"eð ":3577,"it ":1254,"ünc":35,"ja ":3453,"ití":89,"isú":45,"iza":39,"kif":29,"kim":46,"kil":1176,"kk ":327,"úse":29,"úsa":165,"úsd":34,"kie":30,"úsn":29,"úsi":225,"úsk":37,"úsl":41,"kin":2085,"úsu":332,"kip":1436,"kir":1170,"kis":1049,"keð":39,"úss":498,"kit":45,"úst":268,"úta":123,"öði":221,"úti":93,"úth":65,"kja":2742,"útg":318,"útf":37,"gæf":29,"útd":41,"útb":80,"útl":88,"öðl":46,"km ":337,"öðv":305,"útv":167,"kju":1220,"öðu":713,"útu":109,"útr":45,"gæs":64,"öðr":512,"gæt":90,"úts":144,"úpv":33,"ús ":380,"ki ":3751,"kgr":34,"khl":111,"út ":1602,"öð ":459,"úra":87,"kha":33,"úrg":32,"úrf":29,"úre":62,"úrk":41,"kdó":160,"úrl":34,"úrs":50,"úní":148,"kho":40,"úrv":36,"úrt":29,"úru":307,"kef":136,"gáf":290,"únu":54,"keg":32,"kei":325,"kem":386,"kel":195,"ken":1184,"kep":493,"kes":42,"gár":91,"ker":1044,"gát":46,"ket":42,"kað":419,"key":170,"kfa":29,"úpa":59,"úr ":1479,"kaþ":93,"úpu":52,"úps":35,"kfr":105,"úlí":196,"úpi":33,"ke ":136,"úlu":109,"úml":92,"úmm":51,"úme":150,"úmi":46,"úp ":32,"úna":560,"úms":48,"úið":119,"únn":35,"úni":599,"kra":945,"kre":121,"kt ":1361,"gð ":585,"kmö":30,"gíu":82,"gít":108,"gís":71,"gíg":49,"kse":36,"ksf":65,"kry":53,"ku ":2256,"knú":46,"kro":107,"kru":154,"kri":1166,"kkó":35,"km²":99,"kot":277,"kos":362,"kor":407,"kop":46,"kon":1246,"kom":1243,"kol":168,"ks ":435,"klú":36,"kló":34,"klæ":58,"kið":1002,"kmy":402,"úsí":34,"kna":1143,"kme":201,"kmi":111,"gæð":40,"knu":434,"kjó":69,"kjö":344,"útí":111,"kob":49,"kjá":188,"kne":96,"kni":971,"kkv":35,"kku":899,"kkt":826,"kks":204,"kkr":232,"úum":39,"kn ":153,"kke":95,"kkb":29,"kka":637,"kkn":107,"kko":71,"kkl":305,"kkj":140,"kkh":101,"kki":1529,"klu":225,"khó":38,"khú":66,"ko ":34,"kma":92,"úrí":36,"kle":411,"úve":99,"kla":939,"klo":62,"kli":470," Ál":120," Ág":72," Ár":415," Ás":264,"fór":224,"fót":90,"fós":48,"fón":31,"fól":411,"jus":220,"juv":64,"jul":273,"jun":745,"jum":817,"jur":390," Á ":609,"fðu":130,"jub":47,"juh":41,"jug":89,"juf":31,"jud":50,"fða":298,"fði":429,"fðb":90,"ju ":578,"fín":37,"fð ":55,"kbr":32,"kav":64,"kat":183,"kau":363,"kar":1751,"kas":646,"kap":274,"kan":720,"kal":1475,"kam":694,"kak":79,"kah":74,"kaf":322,"kag":595,"kad":110,"kab":52,"föð":56,"fús":45,"ka ":5563,"fóð":29,"för":70,"fös":57,"föt":31,"fön":37,"föl":135," Ga":560," Bú":143," Ge":549," Fy":316," Bó":133," I ":60," Bö":36," Bí":44," Fo":401," Fu":126," Fr":1133," Fi":346," Fl":547," Fj":284," Bæ":200," Ha":2193,"gyð":102," He":1194," Gy":51," Go":223," Gr":964," Gu":613," Gv":43," Gi":180," Gj":38," Gl":210," Gn":42," Ic":56," Dý":44," K ":33," Dö":29," Hv":447," Dó":65," Hu":336," Hr":478," Hn":67," Ho":620," Hl":320," Dæ":71," Hj":179,"ha ":73," Hi":343," Fá":50," Je":228," L ":32," Ja":516," Is":113," Im":35," In":759," Il":85,"ham":273,"han":2326," M ":34," Fö":37," Ka":1198,"hal":899,"hau":175," Ke":368," Ki":275,"har":340," Kj":147,"has":39,"hat":96," Fé":158," Fæ":127," Fí":29," Jo":341,"haf":2871," Ju":93,"hae":47,"hag":381," Fó":49," N ":56," La":1289," Há":323," Le":588," Li":537," Lj":87," Hæ":71," Kl":180," Kn":101," Ko":532," Kr":556," Gí":124," Kv":148," Ku":90," Gö":52," Ky":159," Mc":44," Ma":1610," Hú":1094," O ":35," Mi":927," Mj":65," Me":769,"he ":479," Hí":33," Lo":529," Hé":115," Hö":355," Ly":57," Hó":219," Lu":199," Já":37," Ne":513,"а ":49," Na":464," Nj":57," Ni":233,"hf ":56," Ið":49," Mo":559," My":192," Mu":180,"hel":997,"hei":4035,"dái":46,"heg":81,"hef":1628,"heb":56,"hey":346," A ":141,"het":78,"hes":156,"her":942,"hen":1048,"hem":44,"hi ":50," B ":65," C ":121," Ap":140," Am":374," An":597," Ak":256," Al":992," Ai":48," Ag":60," Af":348," Ac":57," Ad":101," Aa":38," Ab":95," Ba":1274," D ":30," Av":40," Au":690," At":285," As":369," Ar":655,"hig":37," Be":741,"hic":69," Bi":297,"hia":38,"hip":34," Bj":353,"hin":765," Bl":363,"him":136,"hil":183," Bo":816," Br":1093," Bu":149,"his":35,"hit":297," By":64,"hir":173,"hja":45," E ":35," Ca":484,"dæg":46," Ce":87," Ci":85," Ch":431," Cl":177," Cr":127,"dæm":557,"dæl":69," Co":494," Cu":55," F ":53," Da":794," Di":273," Dj":72," De":342," Dr":200," Do":188," Dy":59," Du":120," Ea":43,"hn ":90," Ed":147," G ":36,"hla":226,"hle":106," El":310," Ek":54," Ei":508,"hli":228," Eg":213," Ef":192,"hlj":1680," Et":30," Að":243," Es":91," Er":231," En":509," Em":78,"hlu":2009," Ey":536," Ex":43," Eu":55," Ev":632," Bá":77,"ho ":29," Fe":260," Aþ":47," Fa":298," H ":70,"gma":107," Tó":200,"go ":107,"gme":79," Tí":126," Té":39,"glu":412," Tæ":41," Tá":38,"gls":98," Sý":65,"gle":410," Sú":125," Só":162,"gli":346,"glj":57," Sö":178,"gn ":453,"gla":791," Sí":179," Wo":94,"ggð":568," Sé":40," Sæ":89," Wi":308," Wh":41," We":157," Sá":78,"gko":58," Wa":263," Rú":290,"й ":44," Rö":41," Ró":242,"gog":30," Vé":39,"gjö":146," Ví":264,"gnu":408,"gnv":46,"gns":98," Ze":49,"gni":304,"gnh":31," Za":29,"gnf":78,"gjá":29,"gne":63,"gna":1067,"úa ":496," Yo":162,"gmy":138,"gið":678," Ya":33," Yf":65," Tö":59," Tú":51,"glö":41,"úbb":42,"gs ":890,"gmá":109,"glý":46,"glæ":66,"úar":691,"glí":50,"úaf":94," Vö":72,"gon":51,"gos":180,"gor":133,"got":57,"údd":41,"gsb":67,"úde":39,"gsa":85,"gu ":2071,"gnú":139,"gsk":208,"gsh":92,"gsi":265,"gsf":96,"úda":42,"gsd":42," a ":135,"gse":69,"gro":36,"gru":389,"gra":798,"gt ":1312,"gri":532,"grj":36,"gre":1498," R ":43," Kú":67,"úel":31," Kö":70," Kó":213," Ox":46,"gto":58," Os":65," Ot":31," Kí":164," Or":369," Op":55," Po":343,"guf":112," Pl":227,"guh":41," Læ":47,"gum":2005," Pi":180," Ph":118,"gul":515," Lá":110,"gua":30," Pe":377,"gue":40," Pa":536,"gsv":75," Jö":155,"gst":295,"goð":247,"gsu":50," Jó":679,"gsl":236," Nu":48,"gsm":187,"gsr":110,"gss":198," No":1523,"gsp":47," Ol":55," Ok":35," On":42," Oh":30," Od":79," Of":128," Ká":86," Jú":86,"gta":478,"gy ":31,"grú":33," Mö":79,"gró":151," Mü":48," Ra":615," Mú":128,"grö":62,"úgu":29," Mí":43," Mó":77," Qu":74,"grí":914," Næ":38," Ní":89," Ro":344,"gsæ":96," Re":1399," Ná":93," Mý":140," Ri":247," Rh":36," Lö":101," S ":42," Lú":62," Lí":171,"gur":4137," Pr":333,"úfu":85,"gus":223," Pu":82,"gun":1431," Ló":43," Mæ":40,"gré":42,"úi ":32,"græ":199,"gvi":54,"úga":156," Lý":96,"grá":134,"gve":169," Má":195,"gva":364," Sy":96," Sv":974," Su":940," St":1837,"gtö":37," Ta":373," Tj":67,"úfé":45," Th":487," Ti":313," Pá":120," Te":268," Tr":405," Pí":33," To":304,"gyp":133," Pé":116," Nó":36," Ru":106," Nú":101," Ný":191," Sa":1402," Sh":147,"úin":138," Si":722," Sj":303," Sc":205," Se":571," So":351," Sp":370," Sk":1509," Sl":218," Sm":127," Sn":277,"gvö":64,"ún ":1473,"úla":181," Va":709,"úkr":68," X ":47," Rá":45," Ve":1143,"úku":33,"úlk":110," Vi":651,"úll":94," Ré":29," Rí":181," Vo":162,"úlf":51,"úli":36," Tu":190," Pó":142," Tv":79," Ty":102,"guð":201,"úm ":49,"úkl":39," Ul":39,"gví":39," Um":208,"úkd":157," Un":333,"úka":42," Up":196,"gvé":77," Ur":46," Ut":32," ja":1316,"ь ":33,"iam":73,"ial":59,"ian":234,"ias":35," fá":200,"iat":58," in":1466," il":75,"ic ":168," is":51," eð":5981,"ibl":77,"ibj":53,"ibi":39," ka":1977,"dýp":59," fö":181," m ":286," gæ":107,"dýr":894," kj":379,"ibr":56," ki":467," gá":35," ke":1446,"ibu":44," fí":60," fæ":1198," fé":473,"id ":120," fó":715,"iba":30,"ibe":53," ju":184," gy":80," ha":4851,"dún":44," he":6396," dá":76," gi":256," gj":246," gl":267," gr":1982," go":395,"ia ":441," gu":299," hy":49," dö":163," dú":43," dý":362," hi":1085," dæ":326," hj":450," hl":3298," hn":173," ho":514," hr":1011," hu":968," hv":1445," dó":235,"iet":44," ni":345,"iel":71," ne":1620," já":131,"ien":104," na":898,"ier":105,"ies":63,"ied":45,"ief":45," my":1128,"iei":96," mu":272," ið":125,"ig ":2070," mo":182," mm":37," ok":208," ol":65,"ifu":195," og":20861," kæ":55,"ifo":174," od":33," ká":29," of":1574," jú":303,"ifs":149,"ift":225,"ifr":86," jö":359,"iff":48,"ife":72,"ifl":73," jó":154,"ifj":45," ny":134,"ifi":210," nu":54," no":3458,"ifa":566," le":2444," há":1191," lj":434," hæ":774,"icr":57,"ics":37,"ict":48,"icu":82," li":1725," n ":78,"ico":79,"ick":89," la":3474," kv":906," ku":88,"ici":64,"ich":282," gö":256," ky":352,"ice":85," gó":105," kn":246," km":437,"ie ":195,"ica":290," kl":476," kr":671," gí":129,"iby":31," ko":2181," me":6197," mi":3548," mj":459,"я ":46," o ":46," hú":877," ma":2672," hý":48," lu":44,"idi":32," hó":647,"idg":43," ly":173,"ide":137,"idd":384," hö":1280," hé":535,"ida":195," lo":635,"if ":187," af":6312," ag":34," ab":43," am":104," an":1387," ap":172," ak":50," al":2511," au":1349," ar":311," at":450," as":67," d ":204," ba":1579,"il ":5889," bi":592," be":1105," bo":1713," bl":752," bj":275," by":1013," bu":123," br":1739," ca":62," e ":310,"im ":624,"ika":773,"ige":85,"ibý":65,"iga":307,"ii ":75,"igl":259,"igh":129,"igi":601,"igf":50,"igg":668,"igu":503,"igs":49,"igr":192,"ign":307," b ":35,"ihe":1081,"iha":82,"ihl":41,"ik ":458,"imo":50," er":28966,"imn":59,"imm":338," að":9614," et":46,"iml":33," en":4649,"ims":1087," em":145,"imp":63,"imf":72," ei":5897," el":620,"ime":190," ek":1007,"imk":47," ef":2947," eh":31,"imi":735," eg":82,"ip ":220," bá":176," fe":1285,"inc":97,"ind":1888,"ina":1616," fa":994,"inb":291," ey":827,"imu":329," ev":138,"imy":162," fu":598,"inn":11792,"inm":78," fr":7589,"ino":134,"inr":53," bí":110," fo":1797,"int":511,"ins":4795,"inf":252,"ine":220," fl":2079,"inh":343,"ing":10532," fj":2182," bæ":1033,"inj":88," fi":969,"ini":973,"inl":365,"ink":753," ge":3864," ga":1049," bý":113," bú":500,"inu":3925," i ":55,"inv":49," bö":146,"iny":29," fy":4531," bó":727,"iko":33,"ikn":392," cm":36,"ikm":465,"ikl":338," co":79,"ikk":205,"ikj":227,"iki":1110,"ikh":77,"ikf":51," ch":40,"ike":140,"ila":590,"ilb":133," f ":217," da":1144,"in ":6373,"ikv":111,"igð":208,"ikt":88,"iku":599,"ikr":111,"iks":474," do":59,"ilo":79,"ilr":73,"ill":2619,"ilk":107," dr":748,"iln":105,"ilm":73,"ilh":348,"ilg":369,"ilj":142," de":781,"ili":745,"ild":1200," dj":84," di":134,"ilf":193,"ile":949,"ima":446,"imb":81,"io ":167," dv":45,"ily":95," du":109,"ils":420,"ilt":422,"ilu":204,"ilv":300," dy":47,"hlý":34," vö":461,"hlí":219," vé":126," væ":160," ví":914,"ла":36,"hol":488,"hom":68,"hon":246,"hou":37,"hop":49,"hor":323," yn":69,"hjó":152,"ка":44,"dí ":50,"hjú":47," yr":29," yt":52," ys":51,"hni":63," tú":84," tö":739," yf":1550,"hno":44,"hns":33," tæ":256," tó":1014,"hna":38,"hne":91,"hjá":398," tí":1122," sú":468," sö":1067," só":322," tá":346," sý":615,"hið":78,"hug":970,"ра":31,"dó ":54,"ос":32,"ов":39,"hnú":49,"об":32,"hru":56,"hry":131,"hní":30,"hro":60,"hre":864,"hri":511,"ht ":90,"на":33,"hra":356,"dís":130,"díu":40,"dín":33," ru":152," ry":31," nó":204," u ":29," nö":39," sa":4334," ný":430," nú":672," se":17046,"hyg":213," sj":1921," si":1522," sn":851," sm":466," sl":415," sk":4722,"hyl":69," sp":941," so":265," mí":73,"dót":514,"dór":123," mó":335," mö":457," t ":217," mú":64," ra":1335," mý":45," ná":971," re":1271," ri":857," næ":757," né":32," ro":246," ní":127," ló":69,"hvo":342,"hrí":33," pu":136," pr":628," lí":1770," lú":49," s ":184,"бл":47," lö":1089,"hy ":40," má":1082," lý":420,"dóe":37," mæ":323,"dón":58,"dóm":399," os":39,"hum":74,"hun":188,"hus":67," op":322," kí":261,"hur":44," or":937,"ан":42," kú":89," kó":124," kö":324," pe":303,"hva":437," lá":392,"hve":1438," pa":204,"hrá":47,"ас":31," ký":34," lé":191," pl":325,"ая":31," po":204," pi":60,"hræ":36," læ":234," rý":32," rú":443," sá":460," ró":306," rö":312," sí":1634," sæ":382," sé":1534," x ":48," va":8042," ve":5521," rá":293," vo":1291," rí":972," vi":6117," ræ":385," ré":421,"hvö":38," pö":85," ty":60," pó":129," tv":1119," tu":588," ut":212," ur":156," up":1941," um":4447," un":1589," ul":34," ta":1361,"hyr":120," sy":488," st":9503," sv":2024," su":1520," pí":89," tr":698,"dök":34,"hví":219,"dön":95," to":200," th":207," ti":6298," tj":75," te":2098," pá":51,"dög":48,"óðv":149,"óðu":414,"ffr":124,"fi ":1557,"óðm":36,"óðr":133,"óðs":215,"óðe":41,"ffe":40,"óðf":210,"óðg":53,"óðh":71,"óði":237,"ffa":53,"óðk":45,"óðl":242,"óða":595,"ffi":118,"óðb":40,"fet":34,"fað":546,"bát":196,"fes":284,"fer":1310,"fey":29,"feb":142,"fen":195,"bál":229,"fel":818,"fei":90,"fhe":36,"fhl":44,"aþó":58,"bæ ":192,"fga":65,"aþá":42,"fge":31,"fbr":125,"fbe":30,"fas":294,"fat":65,"far":1064,"fam":53,"fan":667,"fal":741,"fah":32,"fag":101,"faf":45,"óð ":473,"ff ":51,"aþj":48,"aþi":69,"eyð":283,"exí":65,"fa ":1735,"aút":75,"ext":253,"eyr":688,"eys":419,"eyp":145,"eyn":309,"eym":192,"eyj":1341,"eyk":1077,"eyi":44,"eyf":333,"eyg":150,"esú":56,"exa":95,"ez ":34,"eyt":859,"aöl":43,"etb":29,"eta":600,"aða":2137,"aðf":330,"ete":97,"aðe":248,"etj":138,"eti":412,"aði":1858,"eth":57,"aðh":97,"aðg":231,"aðn":42,"etn":430,"aðl":109,"etl":247,"esp":60,"eso":32,"est":3452,"ess":1992,"eum":32,"eto":75,"aðr":337,"etr":663,"ets":81,"aðs":858,"ett":1496,"etu":910,"aðu":2529,"aðv":31,"ew ":192,"eve":76,"erá":64,"eva":56,"evo":39,"evi":74,"eus":66,"ex ":266,"esí":96,"eró":30,"erí":413,"evr":168,"erð":2864,"ey ":517,"erú":35,"epi":31,"eph":64,"er ":26993,"epa":67,"eor":111,"eon":46,"aíd":31,"es ":1150,"ept":231,"epl":35,"epp":1389,"elí":74,"erk":1822,"erl":415,"eri":1572,"erj":619,"erg":583,"erh":55,"ere":120,"erf":1459,"erc":66,"erd":76,"era":1234,"erb":160,"et ":347,"að ":14115,"emí":69,"aín":58,"esj":33,"esk":983,"esf":32,"esh":83,"esi":419,"esb":59,"esc":36,"ese":247,"eu ":59,"esa":148,"ery":35,"erv":161,"eru":4787,"err":409,"ení":97,"ert":870,"ers":1962,"ern":801,"erm":315,"erp":109,"ero":113,"eki":1159,"ekj":180,"ekk":2070,"ekn":178,"eks":107,"ekt":94,"egð":96,"eku":210,"en ":4249,"ela":212,"eld":1621,"elf":107,"ele":208,"eli":198,"elj":279,"elg":458,"elm":97,"ell":1350,"elo":54,"elp":39,"elu":227,"els":1013,"elt":302,"eo ":32,"emb":731,"ema":216,"eme":166,"emd":57,"emm":172,"emo":53,"emi":212,"eið":2061,"emu":458,"emp":64,"ems":115,"enf":36,"ene":240,"enh":41,"eng":2379,"enb":71,"ena":189,"end":2651,"enc":88,"eno":72,"enn":4534,"enk":42,"eni":177,"enj":285,"enu":71,"ens":3046,"ent":778,"enr":50,"aí ":153,"enz":239,"egj":131,"egl":570,"ego":116,"egn":789,"ege":76,"egg":655,"egi":1272,"egr":290,"egs":216,"egt":454,"egu":1949,"egy":33,"ehf":31,"ek ":94,"eip":47,"eis":901,"eir":2011,"eim":2679,"eil":1098,"ein":8206,"eik":3123,"eid":336,"eig":1161,"eif":473,"el ":631,"eit":4783,"efð":193,"aæt":246,"efá":91,"eka":127,"em ":15176,"gfú":42,"ötu":646,"ött":156,"ötn":84,"gju":374,"ötv":30,"gfé":47,"ötl":36,"gl ":222,"öst":120,"gja":1901,"geð":49,"git":30,"gis":534,"gir":792,"ösu":31,"gil":555,"gim":55,"gin":2704,"ösk":74,"gib":68,"gdý":42,"gif":68,"örl":66,"örn":587,"örp":62,"örs":47,"öru":372,"ört":84,"örv":91,"öry":39,"ght":99,"öt ":64,"gho":33,"örd":65,"ghe":48,"örf":205,"örg":409,"gha":53,"öri":68,"örk":384,"ggv":60,"ggu":598,"ggt":85,"ggs":75,"ggl":31,"ggj":980,"ggi":554,"ggd":40,"gge":45,"gga":155,"gfr":149,"öpu":50,"öpp":29,"ölæ":50,"gi ":2174,"gbú":105,"gfa":30,"gaþ":60,"gfe":34,"ör ":105,"gfi":32,"gen":859,"get":826,"gað":150,"ger":1807,"ges":57,"gey":329,"gh ":58,"gaö":31,"gg ":105,"geb":47,"gei":373,"geg":382,"gef":1039,"gel":153,"gek":42,"gdr":32,"öns":279,"önt":102,"önk":81,"gdu":56,"önn":762,"gda":103,"önd":1211,"öng":1010,"gdi":40,"ömu":145,"ömm":54,"búð":112,"öml":44,"ge ":230,"ölv":557,"öku":681,"ökv":108,"öld":1331,"gd ":235,"ölb":96,"ölm":167,"öll":1141,"ölg":41,"ölf":107,"ölu":629,"öls":201,"býl":253,"gbl":50,"gbr":62,"býr":47,"öln":31,"gab":121,"gad":130,"öfð":390,"gaf":494,"gag":255,"gah":95,"gaa":32,"gas":712,"gar":5221,"öki":70,"gau":40,"ökl":108,"ökk":171,"gat":286,"ökf":51,"gav":94,"gak":142,"ögð":187,"gam":445,"gal":380,"gan":1061,"gap":30,"öl ":78,"búd":35,"búa":689,"ga ":4778,"ök ":413,"bús":74,"bún":284,"búi":183,"ögu":1452,"ögs":34,"ögr":85,"öfu":1400,"bú ":35,"öfr":36,"ögm":139,"ögn":233,"ögi":82,"ögl":33,"ögf":50,"ögg":153,"öfn":464,"öfl":142,"fuð":815,"ög ":817,"bök":39,"bör":116,"bön":81,"ftæ":51,"bót":132,"fsí":41,"fyr":4233,"fyl":713,"ftó":31,"fræ":3955,"fvi":41,"fré":80,"fve":203,"frá":3018,"fur":2375,"fus":91,"ból":159,"bók":1011,"bón":65,"frö":89,"frí":321,"fró":162,"fta":843,"fti":2157,"fst":735,"fss":270,"fsm":103,"fsv":103,"fuf":32,"fub":43,"fuk":55,"ful":355,"fum":292,"fun":902,"fug":341,"fts":95,"ftu":254,"ft ":1106,"fra":2612,"fre":467,"frj":180,"fri":247,"bíl":118,"bíu":86,"bís":172,"fu ":237,"fsa":40,"fsd":78,"fse":170,"fsf":61,"fsh":40,"fsi":147,"fsj":32,"fsk":78,"fro":66,"fru":674,"for":2274,"fos":163,"fol":50,"fló":302,"flæ":88,"fmæ":37,"flú":30,"fs ":273,"flö":79,"fið":525,"fna":2112,"fne":55,"fjá":227,"fnd":868,"fng":60,"fnf":164,"fns":126,"fnv":120,"fjó":323,"bæð":272,"fnt":253,"fnu":1196,"fnh":30,"fni":1953,"fjö":1748,"fle":884,"fn ":987,"fla":445,"flj":256,"fli":90,"flu":781,"flo":1007,"fly":309,"öxt":60,"fma":136,"öun":59,"fil":143,"fin":1853,"fim":307,"fir":2556,"fis":690,"báð":53,"fit":83,"feð":29,"fl ":57,"fja":1696,"bæk":87,"ffæ":88,"bæj":204,"bæi":95,"bæn":134,"bær":563,"bæt":285,"fju":37,"örð":1075," Áð":37,"da ":2461,"dd ":334,"dba":39,"dbr":34,"dbo":31,"de ":373,"dad":43,"dab":91,"dak":98,"dal":1432,"dag":946,"dah":92," Æt":40,"dae":100,"daf":226,"dat":82," Æv":33,"das":501,"dar":2968,"dap":31,"dan":878,"dam":444,"dav":102,"dau":223,"dda":212,"ddi":256,"dds":49,"ddu":785,"cul":53,"cum":33,"ctu":33,"cto":64,"cti":110," Í ":611," Ír":171," Ít":191," Ís":2273," Íb":166,"cus":90," Íþ":49,"cks":45,"ckl":29," Ól":368," Óm":42," Ód":30," Ós":82,"co ":104," Ön":69," Ör":95," Ös":31," Öl":87,"con":78,"col":55,"com":44,"cor":78," Öx":50,"cot":33," Óð":61,"cs ":45,"ct ":31,"cra":34,"cri":30,"cro":75," Úl":56," Úk":30,"öf ":67," Út":101," Úr":58,"cea":68,"ch ":237,"cer":51,"ces":67,"cet":54,"cen":40,"cel":70," Þv":74," Þy":38,"ci ":39," á ":11624," Þo":417," Þr":200," Þu":34," Þi":250," Þj":278," Þa":1137," Þe":606,"cha":138,"chw":37,"chu":59,"cia":66,"ck ":218,"cie":48,"che":262,"chi":127,"cho":60,"chn":39,"cht":62," át":481," ás":462," ár":4236," áv":94,"cil":37," áb":154," ág":152," áh":379," áf":139," ák":361,"cis":51," ál":132,"cin":65," án":237,"cm ":36," Þó":307,"cke":55," Þý":325," Þá":57," Þæ":99,"ed ":214," æt":868,"ebe":38," æs":38," æx":33,"ebr":277," æv":104," áæ":48,"eae":45," áð":574,"ead":32,"eak":31,"ean":82,"eal":37,"ear":81,"eas":45,"eat":104,"eau":29,"ea ":112,"efi":869,"efj":94,"efl":96," æð":121,"efn":3337,"efa":155,"eff":31,"ei ":96,"ega":3391,"efr":52,"eft":1944,"efs":237," í ":19047,"efu":1553,"een":87,"eed":29,"aár":34,"eet":35,"edi":112,"edd":31,"ede":88,"eda":35,"eg ":669,"edo":29,"edr":32,"eck":48,"ech":72,"ee ":58,"ef ":241,"ect":77,"eco":32,"dys":33,"duð":125," ít":165," ís":1832," ír":55,"dsö":53," íb":414," íl":31,"dró":35,"drú":32,"dy ":43,"dvi":51,"drá":54,"dve":148,"drí":37,"dræ":56,"dré":44,"duv":56,"dur":3307,"dus":104,"dva":88," ór":37," ól":197," óm":33," óh":75," óg":40," óf":82," óe":43," ób":82," íþ":190,"ðár":38,"dor":109,"don":304,"dom":32,"dow":49,"dos":38," öf":73," ök":39," öl":609," ön":266," ör":241,"ds ":1667,"dmi":55," ót":48," ós":171," óv":94,"dið":637,"dna":116,"dni":37,"dnu":50,"ðöl":79,"djú":102,"dsv":141,"dss":318,"dst":207,"dsr":47,"dsp":43," úr":1554," öð":482," út":2473,"dun":333,"dum":1677,"dul":119,"duf":29,"duh":45,"duc":33," óþ":38,"dri":390,"dra":458,"dt ":40,"dná":398,"dre":520,"dry":91,"du ":383,"dro":176,"dru":103,"dsh":112,"dsj":55,"dsi":362,"dsl":272,"dsk":119,"dsn":42,"dsm":154,"dsb":51,"dsa":115,"dsd":51,"dsf":53,"dse":98," þo":209," þu":239," þr":1309," þy":189," þv":1520," þa":4775," þe":4557,"dge":78," þi":195,"dgo":49," þj":806,"dic":66,"dia":76,"dhe":43,"der":294,"des":336,"det":32,"dað":218,"dey":65,"dea":33," þ ":166,"deg":113,"dei":506,"del":110,"den":250,"dem":107,"dfr":39," ým":413,"di ":5025,"dbú":64,"daþ":31,"dfj":56,"dle":192," þö":45," þý":700,"dla":206," þú":310," þó":393," þé":142," þæ":322," þá":953,"dkn":291,"dgæ":30,"do ":152,"dlu":40,"dli":96,"dim":46,"din":1824,"dio":43,"dir":2045,"dis":659,"die":47,"dik":52,"dil":68,"rgv":83,"rgu":338,"órí":40,"rhe":181,"óve":202,"rha":190,"óvi":58,"rdæ":68,"rhl":419,"rhr":181,"órð":209,"rhu":32,"óró":29,"rho":90,"rdí":30,"rfy":82,"rbó":57,"rfu":285,"rft":34,"óun":156,"rbý":31,"rga":1056,"rbú":62,"ri ":5309,"rgl":50,"rgi":863,"rgj":52,"óva":44,"rge":305,"rgf":73,"rgs":155,"rgt":31,"rgr":329,"rað":905,"ret":461,"nát":321,"ótm":41,"nás":32,"res":811,"reu":56,"óti":277,"ótl":55,"rey":1506,"óte":161,"rfa":643,"óta":345,"rfe":164,"rfi":1445,"rfj":246,"rbæ":241,"ótu":133,"rfl":183,"rfo":103,"ópí":47,"rfr":289,"óts":159,"rfs":230,"ótt":1174,"rdu":33,"ósm":64,"rds":39,"ósl":49,"ósk":121,"ósi":102,"rg ":1633,"óse":107,"rea":127,"ósa":223,"ree":99,"ref":289,"red":93,"rei":2743,"reg":939,"nág":58,"rem":334,"nám":587,"nán":98,"ren":1042,"rek":593,"nák":43,"óst":252,"ósu":45,"nál":173,"rel":158,"raí":49,"rep":672,"rf ":244,"óra":323,"órb":108,"rda":542,"mþy":72,"óre":82,"órf":38,"rcu":48,"órh":40,"óri":369,"órn":1251,"rdo":29,"órr":30,"óní":98,"órt":130,"rdj":41,"órs":264,"órv":45,"óru":242,"rdi":87,"rde":130,"ós ":169,"ná ":105,"re ":456,"rby":65,"rbu":32,"mýr":102,"rbr":230,"rch":57,"rce":64,"ót ":306,"ópa":167,"ór ":572,"rd ":235,"ópi":56,"rap":103,"rar":1340,"ópe":33,"ras":718,"rat":554,"rau":1255,"rav":117,"rbi":30,"ólí":218,"rbj":47,"óps":195,"rbl":48,"rbo":361,"rba":200,"rbe":117,"ópu":659,"rai":43,"rah":258,"rag":405,"ran":2332,"ram":2670,"ral":525,"rak":489,"rab":445,"raf":954,"rae":86,"rad":211,"rac":77,"rpu":127,"rpr":62,"rps":409,"rlí":129,"rpo":41,"rs ":1328,"rlö":185,"rlá":68,"rpe":36,"rpa":113,"rkú":29,"rr ":186,"rlæ":104,"rpi":226,"rkí":63,"ror":58,"ros":443,"rot":445,"rom":128,"ron":188,"rop":81,"roy":30,"rou":80,"rov":35,"row":39,"rod":104,"roc":48,"ní ":161,"rjú":69,"rol":93,"rok":333,"rof":43,"rog":34,"rno":60,"rnl":105,"rnm":279,"rnr":31,"rns":265,"rnv":34,"rjó":300,"næð":35,"rnt":68,"rnu":692,"rjö":122,"rp ":163,"rnb":71,"rna":2301,"rng":308,"rnf":82,"rne":499,"rjá":342,"rnd":175,"rni":1126,"rml":167,"rmo":35,"rms":59,"rið":4639,"rmu":102,"rmy":87,"rhö":75,"ro ":119,"rhú":92,"rma":627,"rme":281,"rmi":222,"rly":31,"rhó":65,"rlu":136,"rls":75,"rlo":52,"rlm":64,"rhé":29,"rlj":29,"rhæ":116,"rli":393,"rld":50,"rhá":110,"rle":987,"rla":936,"rn ":955,"rgö":94,"rkv":63,"rku":705,"rkr":89,"rgí":51,"rgð":52,"rkt":84,"rks":157,"rkn":154,"rkm":132,"rko":161,"rkj":1072,"rki":911,"rkl":152,"rke":391,"rgá":37,"rkf":85,"rka":797,"rm ":127,"né ":49,"næs":243,"nær":514,"rju":301,"næt":37,"rfð":143,"rfæ":63,"næm":31,"rfé":382,"rja":455,"næf":123,"næg":56,"ótí":55,"rl ":136,"rip":130,"rio":79,"rir":3442,"náð":134,"reð":86,"rit":1626,"ris":1095,"riu":44,"rih":31,"rig":346,"ril":173,"rik":328,"rin":4044,"rim":103,"ria":195,"rdý":141,"rib":57,"ric":218,"rid":143,"rie":134,"rif":777,"rhv":114,"rdó":74,"ósí":50,"rhy":62,"rk ":749,"rsá":81,"nól":35,"rsæ":104,"nót":79,"rsí":75,"rsó":269,"nóv":141,"rsö":95,"rtá":75,"rsý":233,"ófa":126,"rtæ":610,"óev":37,"rtí":153,"óað":58,"ófo":31,"rtó":34,"ófr":36,"rtö":112,"ófl":62,"rtú":143,"ófi":133,"ryd":53,"ófe":115,"ruh":42,"rug":140,"ruf":120,"rud":31,"ruc":30,"rur":213,"rup":45,"run":1571,"rum":2163,"rul":208,"ruv":97,"rus":350,"rut":38,"rva":375,"rvi":233,"rræ":279,"óf ":138,"rve":420,"rrá":98,"rrí":167,"rré":62,"rvo":44,"ry ":262,"rsk":1287,"rsl":476,"rsi":446,"rsj":74,"rso":176,"rsp":74,"rsm":68,"rsn":93,"rsd":88,"rsa":290,"rsb":37,"rsh":96,"rse":406,"rsf":44,"rsy":38,"rta":441,"rst":3509,"rss":354,"rsv":330,"rsu":120,"óar":52,"óas":59,"óat":33,"rtl":50,"óan":31,"rto":127,"rte":220,"rth":122,"rti":364,"nó ":64,"óbe":269,"rub":37,"óba":44,"rts":104,"rtr":57,"rtu":242,"nín":81,"rmá":390,"níf":29,"nía":80,"rlý":62,"rmú":75,"rt ":1301,"óa ":210,"rmö":79,"rmó":68,"níu":415,"nít":33,"nís":81,"rro":51,"rri":789,"rrk":38,"rná":49,"rre":213,"rra":1384,"ru ":5901,"rry":72,"rnö":83,"rru":200,"rnó":42,"rrv":80,"rní":108,"ómv":228,"sab":56,"óms":871,"sac":50,"óið":33,"ómu":162,"saf":750,"sag":708,"ómp":698,"sak":243,"ómk":72,"sal":365,"sam":4354,"ómi":78,"ómb":34,"óme":237,"óma":424,"óly":43,"sba":82,"sbe":82,"ónu":716,"ónv":387,"óns":366,"sbi":51,"nýj":114,"ónn":30,"nýl":85,"ónl":482,"sbl":34,"óni":207,"óng":36,"san":529,"sau":114,"sat":94,"óne":67,"sas":112,"ónd":59,"sar":943,"óna":507,"sav":82,"óp ":40,"núa":192,"ókr":63,"óks":153,"óku":123,"sa ":884,"óki":198,"núi":49,"ókm":151,"ókn":521,"óka":293,"ryð":51,"óm ":126,"óls":237,"ólu":206,"ný ":52,"ólk":565,"óll":128,"óli":764,"ólm":135,"nús":141,"núp":46,"óla":1201,"ólg":69,"nún":502,"núm":106,"ólf":325,"óle":64,"núl":72,"núv":68,"ón ":402,"nút":151,"nú ":311,"óin":48,"ók ":536,"óju":31,"ól ":221,"óge":44,"nöf":78,"ógi":34,"ógr":30,"rys":58,"ófs":262,"ruð":130,"ófu":99,"ryk":114,"ryl":29,"ói ":77,"ryn":55,"ryg":260,"óga":70,"óhe":37,"rvö":60,"ódí":33,"ógu":78,"rví":87,"nöl":72,"óha":107,"sha":333,"sho":46,"shr":178,"she":190,"shi":231,"sdæ":76,"shl":141,"si ":1082,"sga":69,"sfy":34,"sbó":49,"sgr":89,"sge":55,"nþá":53,"sja":169,"sl ":180,"sju":30,"sfé":34,"sid":34,"sic":68,"sdý":41,"sia":58,"sk ":1182,"shy":73,"sdó":359,"shv":58,"sit":427,"sir":292,"sis":245,"sin":3946,"sio":58,"sil":235,"sig":687,"nþr":30,"núð":42,"sda":221,"sde":55,"nýr":57,"sbr":123,"sbo":107,"sbu":68,"nýs":66,"nýt":153,"sby":61,"se ":199,"sca":45,"sch":105,"sco":77,"sex":134,"sey":275,"ser":203,"ses":89,"set":1580,"sað":68,"sfa":35,"sh ":114,"sbæ":68,"sfj":305,"sfi":93,"sfe":87,"sfo":31,"sfr":321,"sfl":100,"sea":30,"sei":316,"seg":634,"sef":91,"sep":228,"sen":611,"sem":15452,"sel":291,"sek":38,"saæ":40,"sló":155,"spu":111,"spy":259,"slö":108,"spo":83,"slé":92,"slí":192,"spr":262,"slá":52,"spe":881,"slæ":31,"spj":61,"spi":313,"skú":111,"ský":245,"spa":131,"sou":29,"skó":1556,"skö":121,"sol":30,"son":2333,"sop":47,"skí":253,"sor":180,"ská":595,"sof":60,"skæ":55,"sjú":230,"soc":29,"su ":308,"snú":516,"sní":36,"snæ":42,"sri":40,"sre":88,"sra":90,"smö":45,"oð ":82,"st ":7277,"smó":46,"smí":94,"smæ":45,"smá":328,"ss ":1445,"sli":370,"slo":65,"shó":47,"slu":1624,"sgö":54,"sky":454,"sn ":57,"sla":2777,"sle":2574,"shá":42,"ski":2451,"skj":398,"skl":38,"sko":719,"sks":67,"skr":1500,"sku":3717,"skt":636,"skv":172,"sm ":71,"ska":5867,"ske":638,"sjö":285,"sjó":945,"sna":202,"sni":260,"snj":58,"sne":753,"sjá":875,"smo":37,"smy":129,"smu":253,"sið":390,"shú":35,"sma":628,"sly":46,"shö":100,"smi":324,"sme":283,"ssí":67,"ssý":210,"ssö":35,"stí":390,"sté":71,"stæ":1897,"stá":53,"stý":228,"stú":122,"stö":1473,"stó":1005,"syn":591,"suð":680,"sys":40,"syl":37,"syk":82,"íþj":172,"sví":186,"íþr":259,"svæ":826,"svö":99,"sse":290,"sný":67,"ssa":788,"ssn":223,"sso":1722,"ssl":175,"ssj":54,"ssk":305,"ssi":507,"ssv":116,"ssu":314,"sst":880,"ssp":30,"ssy":180,"ste":1923,"spá":57,"stf":191,"sth":38,"sta":8651,"oða":349,"stb":39,"stm":156,"stn":133,"oðn":31,"sto":2031,"sti":2147,"oði":87,"spæ":137,"stj":1787,"stk":49,"stl":62,"stv":37,"oðu":86,"stu":5049,"str":2496,"spí":66,"sts":239,"oðs":85,"sty":391,"sum":731,"sun":1042,"sus":74,"sur":86,"sva":378,"sve":2028,"srá":129,"svi":516,"svo":582,"srí":153,"tai":47,"tak":1550,"tal":2080,"tae":43,"taf":1306,"tag":148,"tah":147,"tab":188,"tad":164,"tbe":34,"tba":41,"tav":183,"tau":127,"tat":143,"tas":1447,"tar":3875,"tap":85,"tan":2188,"tam":380,"tch":58,"te ":264,"tbo":47,"tbr":96,"tbu":123,"tda":38,"가":30,"ta ":6721,"syð":69,"ký ":32,"kút":86,"kúr":39,"kúl":122,"pa ":507,"oxí":40,"íð ":441,"kýr":177,"pe ":53,"lá ":83,"kýj":35,"par":415,"pat":41,"pas":115,"pav":105,"pac":29,"pad":30,"paf":53,"pak":38,"pal":112,"pap":68,"pan":400,"phe":36,"pha":447,"phi":81,"pi ":583,"íði":373,"íðn":37,"íðu":289,"íðs":99,"íó ":37,"pen":326,"lán":85,"lár":68,"per":484,"lát":267,"pað":131,"pes":31,"lás":107,"peg":29,"íða":1262,"lág":191,"pel":119,"lák":76,"pek":637,"pla":766,"pn ":39,"pli":59,"ple":140,"pgö":32,"pja":37,"læg":317,"læm":64,"læk":184,"læp":38,"læt":66,"lær":134,"phy":56,"íói":42,"pil":337,"pin":465,"pir":132,"pis":61,"pit":31,"por":171,"pop":61,"pos":65,"pon":36,"pol":74,"plö":432,"ps ":194,"pps":271,"ppr":373,"ppu":465,"ppt":170,"pph":431,"ppi":417,"ppl":285,"ppn":424,"ppa":270,"ppe":128,"ppf":56,"ppg":58,"pið":157,"lét":224,"lés":58,"lén":49,"lék":31,"ðuð":62,"pma":115,"lí ":172,"pnu":104,"læð":115,"ðví":60,"pni":409,"pp ":769,"pna":137,"pss":172,"pst":155,"pta":402,"pse":49,"psi":86,"psn":40,"psk":239,"pso":41,"ptu":116,"ló ":63,"pub":42,"pte":249,"pti":394,"ppí":56,"líu":494,"lís":172,"lít":407,"pra":73,"pt ":185,"líf":935,"plý":155,"lía":90,"lín":354,"lím":87,"lík":1027,"pru":359,"psa":36,"pu ":453,"pri":135,"pre":517,"pro":73,"lór":49,"lót":52,"lóv":63,"lói":51,"lók":52,"lóm":402,"lón":87,"psþ":115,"ptö":103,"pur":707,"pus":125,"put":29,"líð":255,"pum":165,"pun":389,"puk":38,"pul":139,"pve":36,"prí":173,"lóa":225,"pró":302,"lóð":350,"pyr":266,"puð":74,"lög":1326,"lön":726,"lök":35,"löt":398,"ðju":320,"ðfæ":123,"ðfé":50,"ðji":36,"lús":36,"lút":54,"löð":154,"ðja":484,"ðis":677,"ðir":1217,"lúb":39,"ðin":3092,"ðim":112,"ðil":607,"ðig":61,"ðih":890,"ðid":32,"ðib":80,"ðdý":38,"ðlu":118,"ðhö":65,"ðle":272,"ðli":420,"ðla":663,"ðgö":37,"ðko":89,"ðke":36,"ðki":33,"ðka":49,"ðnu":157,"ðne":31,"ðna":212,"lúð":37,"ðni":136,"ðið":1005,"lþj":313,"lþi":147,"ðmu":161,"ðma":32,"má ":229,"lýt":37,"lýs":582,"ðmi":61,"lýð":232,"ðs ":277,"ðlæ":31,"mát":62,"már":47,"más":73,"mán":166,"mál":2457,"lþý":88,"ðri":810,"ðsa":58,"ðu ":1241,"ðru":553,"ðmæ":37,"ðra":457,"ðto":53,"mæl":428,"ðub":33,"mæt":102,"mær":230,"ðsk":523,"ðsl":355,"ðsm":81,"ðsp":39,"ðse":398,"ðsf":48,"ðsh":47,"ðsi":217,"ðta":31,"ðst":779,"ðss":159,"ðsv":67,"ðsy":35,"ðré":42,"ðræ":84,"ðvi":83,"ðsá":34,"ðrú":35,"ðul":76,"ðuk":30,"ðuf":35,"ðuh":32,"ðug":67,"ður":7875,"ðus":182,"ðun":524,"ðum":1484,"ðuv":112,"ðve":852,"ðva":303,"ðsö":131,"mís":58,"míu":36,"mín":126,"míl":29,"qua":30,"míð":92,"ða ":7924,"que":67,"qui":57,"mób":30,"mót":447,"món":57,"ðal":1210,"ðak":71,"ðan":1528,"ðam":182,"ðab":236,"ðad":86,"ðaf":336,"ðae":34,"ðah":156,"ðag":69,"ðba":46,"ðas":709,"ðat":38,"ðar":3764,"ðau":278,"ðav":103,"ðbo":469,"ðbr":88,"ðbu":122,"mög":91,"mön":242,"mör":555,"móð":114,"ðaá":38,"ðei":268,"ðen":29,"ðað":227,"ðey":75,"ðfa":52,"ðfe":384,"ðaþ":30,"ðfl":57,"ðfj":46,"ðbæ":76,"ðfr":439,"ði ":5726,"ðbó":84,"ðge":167,"ðga":93,"múl":68,"ra ":5434,"ðgu":32,"ðgr":104,"mún":52,"ðhe":185,"mús":74,"ðha":43,"múr":37,"ðho":42,"ðhv":60,"ngn":161,"ngm":109,"ngo":35,"ngj":642,"ngi":2358,"ngl":694,"ngk":66,"ngv":376,"ngu":4585,"ngr":646,"ngt":285,"ngs":1362,"ni ":6328,"nfy":38,"nge":478,"ngf":42,"ngh":122,"nga":4769,"ngb":69,"ngd":467,"ndí":91,"nhl":58,"ndó":106,"nhv":283,"nha":96,"nhi":29,"nhe":69,"ndá":32,"neg":62,"nei":195,"nel":88,"jál":1132,"ják":79,"nen":108,"ján":150,"nem":287,"jár":442,"ner":236,"ját":39,"nað":2058,"net":289,"nes":1695,"jás":35,"jáv":273,"neu":45,"ndv":196,"ng ":2515,"nea":43,"ned":41,"nef":1415,"nfj":30,"nfl":71,"nfr":379,"nfu":33,"ney":381,"naú":37,"naþ":36,"nfa":160,"nfe":55,"nco":35,"nci":82,"nce":184,"nch":109,"ne ":377,"já ":476,"nbu":51,"ndu":3833,"ndr":595,"nds":3224,"ndn":520,"ndo":373,"ndl":456,"ndm":49,"ndk":337,"ndh":138,"ndi":7817,"ndf":105,"ndg":40,"nde":320,"ndd":33,"ndb":149,"nda":5929,"nak":181,"nal":486,"nam":345,"nan":1706,"nap":74,"nar":7258,"nac":33,"nad":255,"nae":106,"naf":1278,"nag":145,"nah":376,"naj":48,"nab":189,"nbo":78,"nbl":83,"nbr":60,"nbe":288,"nbj":30,"nd ":4229,"nav":232,"nau":139,"nat":892,"nas":978,"na ":7656,"myr":42,"myn":2237,"msó":36,"msæ":58,"msö":32,"msý":70,"iðö":77,"mtö":286,"mtí":120,"ntö":45,"ntý":78,"ntí":118,"ntó":67,"nsö":46,"nz ":37,"jö ":94,"nsý":72,"nsæ":244,"jól":491,"jóm":1532,"jón":1264,"nsí":40,"jór":2056,"jós":611,"jót":508,"nsó":384,"ny ":83,"jóf":63,"nsá":38,"nré":59,"nvi":93,"nrí":48,"nve":746,"nva":467,"nul":144,"num":5240,"nun":1626,"nug":63,"nuh":50,"nus":525,"nuv":51,"nur":921,"nua":31,"nub":32,"nuf":123,"nud":81,"jó ":290,"nto":119,"ntu":324,"nts":226,"ntr":104,"nti":309,"nth":63,"nta":596,"nte":421,"nsu":61,"nsv":84,"nsy":38,"nsn":109,"nsm":56,"nsp":46,"nso":41,"nst":1135,"nss":607,"nsf":108,"nse":233,"nsh":91,"nsg":31,"nsj":37,"nsi":284,"nsl":375,"nsk":4454,"nsd":160,"nsa":227,"nsb":64,"nu ":4087,"nní":45,"nri":142,"nná":48,"nra":58,"nt ":1180,"nmö":126,"nmá":266,"nlí":51,"ns ":5692,"nok":465,"noi":41,"nom":35,"non":64,"not":1703,"nos":55,"nor":1577,"nnd":365,"nne":296,"nnf":115,"nng":84,"nna":7028,"nnb":60,"nnl":189,"nno":53,"nnh":72,"nni":6598,"nnk":88,"nnu":1613,"nnt":643,"njó":114,"nnv":87,"nns":1691,"nnr":157,"nny":33,"nme":186,"nma":83,"nið":1052,"nmy":38,"nli":460,"nn ":8805,"nla":365,"nle":833,"no ":154,"nke":308,"ngá":73,"nki":143,"nka":436,"ngó":130,"nku":284,"nky":63,"ngö":59,"nko":60,"nkt":196,"nfæ":48,"nja":244,"nfö":56,"nju":309,"nih":310,"nig":1566,"nif":72,"nie":59,"nid":40,"nic":97,"ndý":173,"nia":97,"ndú":78,"nk ":108,"niu":46,"niv":147,"nis":1128,"nit":151,"neð":186,"nir":1570,"nio":41,"nip":29,"nim":227,"nin":4763,"nik":72,"nil":175,"ogs":89,"ogr":32,"ogu":44,"ogi":191,"ogl":33,"ogn":32,"oga":158,"oge":37,"ogg":37,"ohn":133,"oha":44,"ois":44,"oid":32,"ok ":125,"kær":45,"kæn":58,"ol ":139,"och":58,"oci":55,"ock":125,"oco":29,"ode":87,"odi":36,"odo":53,"of ":366,"odd":103,"oda":50,"kák":62,"kál":657,"odu":43,"og ":20914,"ofn":1635,"ofi":70,"ofu":107,"ofs":67,"oft":1402,"off":70,"ofa":240,"júk":314,"jún":149,"júl":154,"júg":38,"júf":76,"ob ":50,"júp":173,"د ":39,"oba":45,"od ":57,"obi":55,"obe":61,"nyr":104,"nuð":960,"nyt":33,"jöf":126,"jög":408,"nvæ":34,"jök":280,"nzk":209,"jöl":1329,"jör":1720,"jöu":59,"jöt":144,"nvö":31,"jóð":2051,"jú ":59,"oya":34,"ア ":30,"kót":31,"kór":108,"kós":45,"ows":60,"kól":1488,"own":47,"köt":45,"köm":42,"ové":67,"kön":66,"kök":33,"köl":238,"kör":99,"köp":65,"otu":454,"otv":37,"kó ":88,"ow ":78,"otl":108,"otk":150,"oti":132,"oth":76,"ote":93,"ott":415,"ots":82,"oto":46,"otn":229,"ost":320,"ota":1324,"ov ":34,"osi":138,"osh":54,"osk":246,"ose":94,"osf":59,"oss":397,"oso":80,"osn":180,"oy ":61,"kóg":196,"owe":53,"ovi":54,"orí":119,"orð":3303,"ox ":46,"ova":32,"orá":31,"ove":83,"oug":45,"oui":32,"oul":47,"oun":72,"ous":64,"our":135,"out":62,"kíð":62,"opn":231,"opo":41,"opp":155,"opi":251,"ope":40,"oph":87,"opa":52,"ítö":52,"os ":473,"olí":72,"opt":43,"ool":48,"ook":64,"ood":45,"or ":435,"okö":33,"ítá":50,"íví":47,"ork":465,"orl":173,"orm":710,"orn":1435,"oro":81,"orp":260,"orr":707,"orc":31,"ord":318,"ore":423,"orf":203,"org":2899,"ori":386,"ou ":62,"osa":139,"ort":719,"ors":867,"orv":93,"oru":1072,"ory":62,"kíl":193,"kín":122,"kíf":146,"ot ":172,"m² ":103,"orb":110,"ora":164,"kís":41,"kír":55,"olb":58,"ola":137,"old":173,"ítr":55,"ítt":43,"on ":2981,"ítu":112,"oli":166,"oll":558,"olk":48,"íti":396,"éði":33,"ítl":29,"olf":120,"ole":105,"ols":62,"olt":453,"olm":30,"olo":103,"oly":34,"olu":106,"olv":29,"oka":194,"ísu":62,"íst":50,"om ":494,"okk":1938,"ísi":469,"oki":67,"ísl":2033,"ísk":2316,"íta":480,"oks":33,"okt":196,"oku":100,"ona":1076,"ond":305,"one":201,"ong":207,"oni":201,"onn":87,"ono":62,"ons":298,"ont":181,"onu":1209,"ony":41,"oma":392,"ome":121,"íus":154,"íur":45,"omb":46,"omi":303,"íun":207,"íum":114,"íul":62,"omm":235,"íuh":63,"omp":59,"omn":42,"omo":42,"omu":187,"íva":36,"oms":54,"op ":66,"íma":1376,"íme":53,"la ":3270,"ími":124,"íms":207,"ímu":159,"ímy":31,"ína":797,"íne":65,"íni":88,"gús":189,"íns":125,"ínu":752,"ínv":127,"ím ":33,"íka":550,"íkj":759,"íkl":107,"góð":87,"íki":1509,"íkr":40,"íks":66,"ígð":53,"íkt":208,"íku":987,"ín ":498,"íle":41,"íld":62,"íla":168,"íll":70,"íli":51,"íls":42,"há ":44,"le ":477,"lby":30,"ít ":92,"éð ":29,"íra":38,"lf ":135,"írs":66,"ldf":77,"ldg":92,"lde":90,"ldb":66,"lda":1630,"ísa":218,"ldn":83,"ldm":49,"ldl":84,"íu ":1396,"ldk":29,"ldi":1971,"ínó":48,"ldv":44,"ldu":1350,"íru":32,"lds":601,"ldr":327,"íon":35,"lab":255,"lac":81,"lad":110,"laf":643,"lae":76,"lah":156,"lag":2929,"lai":54,"ír ":94,"lal":147,"lak":210,"íkó":69,"lan":8388,"lam":321,"lap":48,"lar":1445,"íos":29,"lat":1416,"las":1408,"lax":34,"lau":1468,"lav":277,"lay":48,"lba":94,"ld ":1392,"lbe":135,"ís ":151,"lbo":49,"íló":181,"ípu":47,"lbr":155,"kvi":617,"kve":660,"krá":299,"kva":323,"íet":48,"kuv":29,"kut":72,"íg ":45,"kus":117,"kur":3957,"kup":220,"kun":523,"kum":1574,"kul":461,"ífi":116,"ífl":51,"ksá":78,"ífr":77,"ífs":158,"íft":46,"ífu":123,"ífv":204,"ky ":34,"krú":51,"kró":126,"krö":33,"kvu":31,"ífa":134,"krí":69,"íff":195,"ífe":48,"gða":277,"kta":448,"kte":85,"kss":123,"ksv":50,"koð":120,"kst":476,"ksk":82,"ksi":136,"ksh":75,"ksm":107,"ksl":35,"kub":39,"kuf":44,"kug":52,"gó ":40,"íf ":93,"kts":31,"ktu":499,"gðu":319,"gði":360,"kti":182,"íde":43,"ída":53,"kto":78,"gön":268,"göm":74,"göl":30,"kvæ":806,"gög":58,"kuð":160,"kyr":42,"kyn":662,"ík ":916,"göt":244,"kví":214,"gör":37,"íl ":215,"kvö":70,"ígu":52,"ígs":41,"ígr":36,"ígi":50,"gól":117,"íga":62,"íbú":416,"íhy":55,"kyl":290,"któ":189,"llí":79,"lpe":30,"lph":37,"ls ":828,"llý":48,"lló":29,"lpu":35,"lok":1278,"lon":100,"lom":31,"lop":33,"lor":130,"loc":48,"lof":376,"log":101,"lpa":98,"los":137,"lot":189,"lou":29,"lkó":30,"low":39,"lni":132,"ljá":29,"lne":42,"ljú":103,"ljó":2991,"hæð":207,"lnu":78,"lmi":211,"hél":52,"lme":420,"lma":336,"lna":176,"lmu":132,"hér":502,"lms":88,"hét":88,"lið":1440,"íba":75,"lti":430,"ltr":104,"lts":172,"ltu":112,"lub":100,"luh":51,"lug":511,"luf":94,"íbe":59,"lue":48,"lsh":178,"lsi":379,"lsj":39,"lsk":731,"lsl":62,"lsm":64,"lsn":115,"lss":277,"lst":946,"lsu":92,"lsv":167,"ías":50,"íal":40,"ían":123,"lta":329,"lte":303,"lri":85,"lu ":1523,"lsf":58,"lse":116,"lsd":64,"lsb":58,"lsa":85,"ía ":451,"lmö":34,"lre":29,"lt ":907,"lra":219,"ldó":125,"lhl":81,"lho":108,"lhe":309,"lhj":75,"lha":33,"lgu":75,"lgr":315,"lge":419,"lgj":181,"lgi":219,"li ":3368,"lga":439,"lbú":63,"lfv":36,"lfr":406,"lfs":536,"lft":177,"lfu":334,"lfo":43,"lfj":68,"lfl":58,"lfi":167,"lfb":34,"lfe":71,"lbá":43,"lfa":315,"ley":649,"lex":80,"leu":35,"lev":29,"les":811,"hás":508,"lað":1400,"hát":389,"let":264,"ler":229,"hár":141,"lep":51,"lem":83,"len":3455,"lek":97,"hál":239,"lei":4428,"laæ":29,"leg":4445,"lef":174,"lea":40,"lls":642,"llr":182,"llu":1097,"llt":609,"llv":107,"lly":78,"lo ":81,"lla":2838,"llb":51,"lld":150,"lle":713,"llg":128,"llh":48,"lli":2397,"llj":272,"llk":55,"llm":34,"lln":58,"llo":157,"lgí":110,"lko":96,"lku":305,"lks":187,"lky":33,"lka":170,"lke":74,"lki":628,"lkj":58,"lkn":60,"hæs":106,"hær":33,"lju":52,"hæt":240,"lm ":74,"ll ":1950,"lja":366,"hæk":32,"hæf":189,"hæg":247,"leð":89,"lit":720,"háð":110,"lis":1592,"lir":354,"lip":103,"lin":2452,"lim":235,"liv":30,"liu":91,"lic":99,"lid":33,"lia":157,"ldý":49,"lib":33,"lk ":248,"lik":141,"lil":61,"lig":590,"lie":155,"lif":428,"ma ":1535,"hún":569,"hús":642,"mb ":57,"mab":539,"mah":39,"mak":53,"mad":43,"mae":55,"maf":42,"mag":278,"mar":1996,"mas":392,"mal":260,"mam":64,"man":3392,"mav":111,"mat":635,"mba":627,"md ":63,"mbl":54,"mbi":46,"mbe":570,"hýs":53,"mbr":131,"mbo":170,"me ":192,"mbu":134,"mby":39,"húð":63,"mda":120,"mde":37,"mdu":54,"mdi":69,"med":51,"mef":118,"meg":340,"met":614,"mað":1182,"maí":169,"mes":450,"mer":1104,"mel":116,"men":2186,"mei":865,"maæ":41,"mfe":123,"mfa":71,"mey":48,"mfr":56,"mbí":46,"mbæ":186,"mfj":29,"lva":264,"lve":225,"lvi":125,"lul":121,"luk":133,"lun":888,"lum":1455,"lut":2348,"lus":353,"lur":1152,"luv":69,"ly ":105,"lsá":123,"hóf":338,"lræ":29,"lvu":443,"lsí":62,"hóp":300,"hól":244,"lyk":142,"lyf":122,"ltö":53,"hög":61,"höf":1628,"höl":107,"lvæ":179,"lym":116,"lyn":76,"lys":63,"luð":344,"lyt":300,"lyr":91,"hön":281,"lví":102,"hör":44,"mpi":49,"mpe":55,"mpr":35,"mpl":736,"mpu":43,"mps":68,"ms ":324,"mon":182,"mol":39,"mor":175,"mos":64,"mou":30,"mpa":70,"mri":72,"mru":37,"msa":120,"mu ":328,"mse":173,"msb":39,"mmá":75,"mt ":874,"ið ":16040,"mmú":58,"mra":69,"mtu":58,"iðu":850,"iðv":49,"iðs":853,"iðt":99,"mpí":104,"iðr":196,"iðm":78,"iðn":200,"iðk":105,"iðl":242,"iði":994,"mti":77,"iðj":659,"iðh":142,"mss":231,"mst":639,"msu":130,"msp":568,"msj":68,"msk":329,"msl":119,"msm":255,"msf":81,"msh":58,"msi":108,"iðd":64,"iðf":164,"mte":29,"iðe":79,"iðb":270,"iða":1209,"mta":215,"msv":831,"mvi":66,"mræ":144,"msá":58,"my ":29,"mur":1133,"mus":94,"mul":139,"mum":344,"mun":905,"mva":32,"mve":268,"mhl":52,"mhv":209,"mha":163,"mdæ":65,"mhe":130,"mi ":1234,"mbö":37,"mfy":45,"mfé":153,"min":1199,"mil":1646,"mir":185,"mis":788,"mit":132,"með":4188,"mic":43,"mik":821,"mo ":39,"mlu":66,"mli":71,"mle":912,"mla":218,"mgö":49,"mky":51,"mkv":450,"mki":55,"mke":47,"mm ":233,"mjö":303,"mjú":31,"mnu":39,"mjó":184,"mni":205,"mna":60,"mne":186,"mmy":31,"mms":259,"mmu":172,"mið":1660,"mmt":327,"mmi":66,"mmo":32,"mma":212,"mme":102,"xár":48,"xík":59,"xá ":30,"víd":42,"vía":47,"vít":367,"vís":868,"vív":36,"víu":65,"vír":31,"víl":52,"vík":1324,"vín":173,"vím":40,"víg":117,"víþ":176,"víð":290,"vö ":226,"vöx":63,"vöt":84,"vör":454,"völ":332,"vök":88,"アア ":30,"vöð":43,"ví ":1408,"væð":1152,"Ásg":52,"Ást":139,"ال":93,"vél":295,"vét":58,"Ára":30,"Ári":106,"Árm":44,"Árn":150,"væt":40,"vær":197,"zku":191,"væg":225,"væm":521,"væl":58,"væn":135,"Ágú":61,"Álf":73,"yrð":218,"yvi":33,"uðu":1637,"ytu":186,"uðv":232,"uðs":298,"ytt":292,"ytr":53,"uðr":201,"uðm":148,"uðn":68,"uðk":47,"uðl":128,"ytj":179,"yti":525,"uði":226,"uðg":34,"uðf":78,"uðb":470,"uða":444,"yta":127,"yss":114,"yst":379,"ysu":42,"ysk":93,"ysl":34,"ysi":128,"ynþ":31,"ysa":75,"yrk":276,"yrj":263,"yri":3304,"yrp":48,"yrn":424,"yrt":41,"yrs":1207,"yrr":562,"yru":38,"uð ":1944,"yra":258,"yrg":59,"ys ":74,"ypi":37,"ypt":168,"yps":42,"yr ":47,"yjó":34,"uðá":45,"týr":322,"yf ":34,"ydd":64,"yfa":39,"yfl":41,"yfi":1843,"yfj":42,"yer":35,"ya ":46,"túd":45,"tún":89,"töð":1090,"túr":354,"túg":124,"túl":110,"yan":50,"uæt":68,"yju":348,"ykh":37,"yki":114,"ykj":958,"ykk":283,"yks":31,"ykr":37,"yku":91,"ygð":31,"ykt":79,"yn ":65,"yld":242,"yle":30,"ylf":29,"ylg":269,"ylk":497,"yll":160,"ylv":38,"ylt":92,"yma":72,"ymi":55,"yms":49,"ymp":115,"yna":50,"ynd":2435,"yni":439,"ynj":135,"ynl":57,"ynf":55,"yng":529,"yns":204,"ynt":58,"ynn":253,"ر ":29,"yft":54,"ygi":89,"ygg":1542,"ygl":52,"ygj":36,"yin":76,"yja":1253,"tím":1376,"tín":493,"tíl":107,"tíu":187,"tís":148,"tíf":31,"tíg":107,"xtu":66,"xta":178,"xti":64,"tæð":489,"xne":38,"xna":31,"xið":31,"tét":65,"ي ":37,"xlu":43,"töf":197,"tön":65,"tök":743,"töl":1034,"tör":51,"ن ":58,"tóð":129,"tó ":38,"tíð":431,"tóm":77,"tón":860,"tók":231,"tól":186,"tób":197,"tóf":35,"tór":803,"tót":73,"sþá":113,"xfo":37,"tán":117,"tár":67,"ták":413,"tál":58,"xla":43,"tæt":90,"tær":1323,"tæk":1071,"tæp":54,"xin":82,"súl":58,"sút":44,"súr":97,"xa ":51,"sú ":352,"sýr":127,"sýs":821,"sýn":431,"sýk":65,"sþi":29,"tá ":55,"xas":32,"xar":46,"xan":81,"síð":895,"sök":67,"söl":167,"söm":115,"sön":374,"söf":68,"sög":1012,"sót":79,"sós":78,"són":245,"sól":222,"sók":506,"sér":1103,"séu":75,"sæt":356,"sé ":343,"wn ":40,"síg":43,"ws ":58,"sía":114,"sís":110,"síu":323,"sík":31,"sím":100,"síl":50,"sín":753,"wor":29,"rúð":68,"rþe":39,"rþi":59,"sá ":403,"rýt":47,"rýs":98,"rýn":71,"rým":56,"wer":56,"sár":272,"sát":78,"sál":193,"sæl":271,"sæk":59,"sæn":178,"wic":43,"rön":643,"röl":81,"rös":68,"rög":86,"röf":82,"rök":168,"róð":187,"rú ":190,"rúi":57,"rún":199,"rúm":310,"rús":221,"röð":352,"rút":141,"wa ":41,"rúa":419,"rúg":52,"rúf":57,"way":31,"war":95,"wai":35,"ríu":241,"rív":32,"rís":1547,"rít":76,"réð":35,"rír":32,"río":31,"rím":339,"rín":121,"rík":2512,"ríl":176,"ríh":63,"ríf":39,"ríb":73,"ría":168,"rð ":1989,"vu ":91,"rðv":334,"ró ":38,"rðr":296,"rðs":472,"rðu":3047,"rðg":39,"rðf":294,"rði":2526,"rðh":35,"rðm":67,"rðl":335,"rða":3018,"rðb":53,"rðe":35,"ríð":409,"vus":33,"vur":48,"vuo":30,"ríó":57,"vum":75,"vun":101,"vul":145,"vró":716,"rót":502,"rós":105,"róu":170,"róa":188,"róf":441,"rók":92,"ról":88,"rói":38,"rój":32,"róp":739,"róm":241,"rón":128,"vip":108,"vir":486,"vik":659,"vil":263,"vin":1308,"vig":107,"vic":33,"vid":89,"vif":41,"veð":482,"ráð":822,"vit":214,"vis":416,"ræn":875,"ræl":54,"ræm":47,"ræk":248,"ræf":40,"ræg":184,"ré ":89,"ræt":349,"ræv":35,"vo ":541,"réf":119,"rét":867,"við":5233,"rés":59,"ræð":4116,"rí ":42,"vog":188,"vol":114,"vok":77,"von":122,"vop":132,"vor":1409,"vot":61,"vi ":117,"vex":159,"vep":209,"ver":6921,"rár":206,"rás":260,"ves":1169,"vet":255,"rát":145,"vað":288,"ráv":43,"rái":33,"vei":2637,"veg":1474,"ven":751,"rán":98,"vem":147,"vel":1100,"rák":60,"vek":61,"ráb":38,"ráa":38,"vef":239,"ve ":176,"rá ":3167,"val":1084,"vak":173,"van":556,"vam":93,"var":9040,"vat":1043,"vas":74,"vav":70,"vax":219,"Áðu":37,"vaf":68,"vad":53,"vag":69,"va ":334,"uvö":31,"uví":69,"utæ":43,"utí":62,"ux ":46,"uvi":38,"uva":134,"uve":291,"urá":81,"urí":54,"urð":755,"usl":30,"usk":240,"ush":40,"usi":109,"use":190,"usa":273,"usv":66,"ust":3311,"uss":141,"usp":64,"usn":74,"utl":60,"utn":112,"uth":88,"uti":783,"upá":53,"ute":116,"utf":76,"uta":1193,"utt":454,"uts":66,"utv":219,"utu":201,"uto":43,"utr":39,"us ":1167,"ulý":49,"umá":434,"umæ":45,"umö":32,"ut ":328,"urb":253,"ura":213,"urd":67,"urf":235,"ure":339,"urh":524,"urg":224,"urj":30,"uri":1241,"url":562,"urk":192,"urm":112,"urn":504,"uro":90,"urp":40,"urr":390,"urs":1140,"urt":381,"uru":95,"urv":93,"ury":37,"uor":53,"upa":89,"ur ":26291,"upi":81,"upe":58,"ulá":50,"upp":2005,"ulí":46,"upm":133,"ulö":48,"ups":167,"ums":276,"umr":110,"umu":278,"umt":34,"umv":35,"umi":189,"umh":215,"umk":61,"umm":97,"uml":76,"umn":32,"uma":561,"umb":222,"umd":81,"ume":325,"umf":103,"uhö":65,"unt":112,"uns":75,"unv":62,"unu":1350,"unk":272,"uni":524,"unn":2545,"unc":32,"und":5296,"una":2110,"unb":61,"ung":2211,"une":148,"umy":38,"up ":193,"uku":56,"ukt":40,"ukn":82,"ukk":98,"uki":102,"uke":171,"um ":21458,"uka":181,"ulu":90,"ult":54,"uls":171,"ulr":34,"ulo":30,"ull":717,"ulj":36,"uli":154,"ulg":31,"uhá":62,"ule":1154,"ulf":47,"uld":119,"ula":290,"un ":2337,"uin":39,"uis":52,"uk ":357,"ufé":80,"uit":37,"ul ":138,"ugf":52,"ugg":223,"ugh":68,"ugi":102,"ugb":111,"uge":80,"ugn":84,"ugl":480,"ugm":142,"ufu":94,"ubó":48,"ufr":151,"uft":32,"uga":631,"ufy":58,"uhe":55,"uhr":48,"uhl":89,"ugv":167,"ugu":342,"ugt":564,"ugs":232,"ugr":64,"uha":44,"uct":39,"uda":130,"udd":31,"ude":123,"udi":40,"ubo":41,"ubr":59,"uca":53,"ue ":86,"uch":32,"uck":49,"uer":36,"ues":38,"uff":37,"ufe":30,"ufa":76,"ufo":48,"ubæ":33,"ufj":77,"ufi":29,"ufl":123,"ug ":195,"uef":41,"uel":43,"uar":31,"ual":30,"uan":43,"ubl":130,"ube":67,"uba":61,"ubb":30,"tvö":262,"tvæ":181,"tví":220,"pön":127,"pör":37,"tyk":37,"tyr":286,"tuð":583,"tyt":114,"ty ":229,"trö":549,"trý":35,"trú":459,"tró":64,"tré":135,"træ":183,"trí":436,"tvo":138,"trá":56,"tve":811,"tvi":179,"tva":215,"tur":6286,"tus":346,"tut":290,"tul":115,"tuk":92,"tun":2035,"tum":1349,"tub":54,"tud":78,"tuf":48,"tuh":68,"tug":285,"ttú":315,"ttá":38,"tz ":61,"pós":50,"two":32,"pól":144,"pía":68,"ts ":398,"tmæ":47,"tmá":108,"pít":55,"pír":83,"píp":39,"píu":114,"tre":326,"tt ":3973,"tra":1427,"trj":59,"tri":1095,"tru":188,"tro":207,"tu ":1914,"try":105,"tsa":40,"tsb":48,"tse":120,"tsd":95,"tsi":93,"tsj":49,"tsh":220,"tsm":42,"tsk":176,"tsl":69,"tsp":261,"tsv":50,"tsu":54,"toð":80,"tst":183,"tss":149,"tta":1700,"ttb":296,"tte":221,"ttf":40,"tth":142,"tti":1896,"ttk":177,"ttl":443,"ttm":78,"ttn":145,"tto":50,"ttp":29,"tts":368,"ttr":107,"ttu":1777,"ttt":56,"ttv":68,"tme":42,"tma":134,"to ":193,"thö":259,"tið":769,"tmi":41,"tni":1019,"tne":174,"tjá":158,"tna":432,"tmy":32,"tjó":1537,"tns":509,"tnu":113,"tof":1728,"toc":34,"tog":155,"tjö":220,"tos":79,"tom":29,"ton":377,"tok":74,"tol":63,"tor":456,"top":73,"tlæ":32,"til":6806,"tik":46,"tif":55,"tie":49,"tig":294,"tir":3353,"tit":169,"tis":756,"tin":2196,"tim":103,"tio":397,"thy":59,"thu":143,"thv":123,"tia":90,"tic":138,"tid":39,"tfæ":31,"tju":44,"pæn":140,"tiv":71,"tja":445,"tkv":226,"tku":148,"Ætt":33,"tke":39,"tgá":259,"Ævi":29,"tli":156,"tlu":263,"tn ":382,"tla":1003,"thá":36,"tle":549,"tem":310,"pán":114,"ten":1249,"tep":34,"tei":1039,"tek":688,"tel":552,"tef":480,"teg":799,"ted":71,"tfl":34,"tfj":120,"tfi":62,"tfe":37,"tbá":198,"tfa":120,"th ":216,"tey":170,"tex":159,"tað":3001,"tet":95,"tes":176,"ter":920,"pár":60,"tgr":38,"tge":143,"ti ":5397,"tbý":74,"tga":31,"tfr":38,"tho":74,"thl":29,"thr":34,"the":336,"thi":41,"tha":162,"þús":303,"þýs":403,"þýt":31,"þýð":375,"ýða":33,"ýðr":52,"ýði":302," アア":29,"ýðs":32,"ýðu":113,"ýðv":179,"þét":142,"þör":71,"þól":58,"þór":44,"þót":75,"þó ":313,"þá ":592,"þær":240,"þæt":143,"þát":560," об":31,"þrá":74,"þve":149,"þró":578,"þrí":169,"þræ":66,"þur":132,"þun":127,"þrý":79,"því":1355,"þyr":48,"þyn":97,"þyk":156,"三 ":35,"ýnt":39,"ýr ":423,"ýrd":38,"ýra":596,"ýrl":53,"ýri":416,"ýrt":38,"ýrs":37,"ýju":59,"ýki":49,"ýli":248,"ýle":93,"ými":230,"ýms":227,"ýni":230,"ýnd":168,"ýna":59,"ýja":206,"úða":105,"úðu":100,"úði":56,"úð ":64,"þro":69,"þre":237,"þrj":165,"þri":320,"þjó":1391,"þol":52,"þot":31,"þor":186,"þjá":53,"þin":513,"xíð":39,"þeg":905,"þei":1439,"þek":897,"þen":84,"það":1705,"þet":98,"þes":1283,"þar":1999,"þan":673,"þak":46,"yða":47,"yðj":117,"yði":287,"yðr":29,"þau":382,"yðs":65,"ýrð":81,"ýva":31,"ýti":83,"ýtt":104,"ýtu":56,"ýta":55,"ýsl":836,"ýsk":757,"ýsi":434,"ýst":220,"ýru":217,"ýsa":87},"n_words":[2831948,3326341,2728347],"name":"is","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/it.json b/contrib/languages-data/it.json
new file mode 100644
index 0000000..6371347
--- /dev/null
+++ b/contrib/languages-data/it.json
@@ -0,0 +1 @@
+{"freq":{"D":94947,"E":67243,"F":92398,"G":104535,"A":191905,"B":119758,"C":221153,"L":203562,"M":160653,"N":84931,"O":53123,"H":49392,"I":191375,"J":30427,"K":33649,"U":59673,"T":99599,"W":33788,"V":71164,"Q":12325,"P":151391,"S":231227,"R":109065,"Y":11206,"X":16332,"Z":10278,"f":518611,"g":920085,"d":2363397,"e":6056669,"b":494596,"c":2150346,"a":6155041,"n":4141668,"o":4513622,"l":3766393,"m":1338197,"j":22860,"k":110896,"h":452585,"i":5976536,"w":49843,"v":585518,"u":1754682,"t":3725316,"s":2419415,"r":3264659,"q":101625,"p":1281861,"z":475996,"y":114701,"x":30592,"È":18809,"ì":9651,"é":22484,"è":321778,"à":93588,"ù":30198,"ò":27447,"ó":6194," l":332142," m":240472," n":389032," o":184207," h":31309," i":549255," k":13980," d":1574321," e":446281," f":277424," g":138336," a":703335," b":75718," c":722915," z":6698," u":458470," t":223758," v":125009," q":64842," p":570240," s":706865," r":244819," J":29715," K":31685," H":46792," I":159850," N":78119," O":44164," L":197779," M":152789," B":113710," C":207226," A":159909," F":86857," G":100079," D":87417," E":59714," Z":9584," Y":10426," X":11662," S":213852," R":103083," Q":11829," P":141839," W":31372," V":61312," U":56839," T":91377," è":311053," È":18792,"A ":27808,"Da":15987,"Cu":5620,"Cl":7833,"Co":63300,"Cr":10676,"Ce":10902,"Ch":29704,"Ci":11685,"Du":6024,"Do":13966,"De":15564,"Di":21598,"Fe":11756,"Fa":10362,"Eu":7860,"Es":8695,"En":5820,"El":6080,"Ge":15850,"Ga":14081,"I ":31564,"Fu":8170,"Fr":19816,"Fo":12748,"Fi":15192,"C ":10682,"Au":9085,"Ar":21848,"As":8912,"D ":5691,"Ba":27783,"Am":10014,"An":18633,"Al":34245,"Bu":8230,"Br":18440,"Ca":56504,"Bi":8237,"Be":19580,"Bo":19802,"Le":25820,"Li":19778,"La":88640,"Lu":9671,"Lo":25268,"Me":19948,"Mi":23733,"Ma":59219,"Mu":9761,"Mo":30287,"Ni":7514,"Ne":23857,"Na":17893,"No":19720,"Gi":22185,"Gl":5829,"Gr":18092,"Go":9531,"Gu":8537,"Ha":17031,"He":8149,"II":13036,"Ho":8979,"In":30727,"Il":66084,"Is":9021,"It":12730,"Ja":8255,"L ":27703,"Jo":9534,"Ka":7944,"Un":28071,"Tr":15672,"To":14627,"Th":17010,"Ti":6315,"Te":15797,"Ta":10754,"UA":13360,"V ":7192,"St":32757,"Su":12977,"Wi":7930,"Wa":7819,"Vo":7254,"Vi":20021,"Va":13663,"Ve":14211,"Pu":5612,"Pr":25455,"S ":9118,"Pe":16282,"Pa":38366,"Po":20730,"Pi":22494,"Or":10767,"Se":20797,"Sc":24538,"Si":21736,"Sh":7322,"Sp":9077,"So":16973,"Ru":7444,"Sa":46373,"Re":30166,"Ri":14281,"Ro":32671,"Qu":10556,"Ra":11841,"b ":11012,"a ":2254795,"i ":1483483,"ge":88624,"ga":67666,"fl":8575,"ff":35291,"fi":134642,"fr":78002,"fu":41990,"fo":66122,"he":195889,"ha":63814,"gn":70158,"gl":120822,"gi":234393,"gh":24919,"gg":70910,"gu":55229,"gr":73387,"go":71768,"du":52294,"g ":29202,"ea":91863,"eb":24191,"ec":137759,"ed":128701,"de":782039,"dd":10103,"di":854362,"do":146993,"ds":9108,"dr":32135,"ew":7817,"ex":7461,"eu":18301,"ev":52077,"ey":12656,"ez":26509,"fa":71893,"h ":27064,"fe":57833,"eg":172770,"ef":19354,"ee":16955,"el":899715,"ei":89786,"ep":27874,"eo":41068,"en":625515,"em":132418,"et":253878,"es":434221,"er":718824,"eq":6537,"ca":399283,"e ":2094441,"br":59095,"bu":40562,"bo":29867,"bl":42370,"bi":141440,"bb":48744,"be":43765,"da":306546,"f ":15395,"cu":62362,"ct":11024,"cq":7195,"cr":59094,"co":609419,"ck":21526,"cl":55423,"ci":319342,"ch":241963,"ce":223557,"cc":104089,"c ":16295,"az":131617,"ay":15144,"ba":66599,"d ":145378,"at":664795,"as":217940,"ar":494405,"av":77540,"au":53901,"ak":11813,"al":679169,"ai":64110,"ap":84421,"am":172413,"an":700812,"ac":102427,"ad":97495,"ab":100158,"ag":138596,"ah":7533,"ae":26244,"af":30789,"nu":38735,"nt":634979,"ns":87660,"nq":7056,"no":374486,"nn":92022,"nz":76063,"ny":7751,"nv":12211,"oe":12640,"of":30817,"oc":106098,"od":71766,"oa":9353,"ob":27798,"om":291133,"on":807692,"ok":6829,"ol":302572,"oi":49498,"og":89760,"oh":6209,"ot":113582,"os":173480,"ov":109964,"ou":37230,"op":117448,"oo":15155,"or":452213,"r ":177153,"ow":12656,"oz":6935,"oy":5914,"pe":242170,"pa":234524,"pl":24851,"po":213425,"ph":7578,"pi":128997,"lo":210541,"lm":44463,"ll":700194,"ls":12139,"lp":10018,"lv":11056,"lu":64908,"lt":90822,"ly":7335,"o ":1645460,"ma":263312,"mb":52388,"me":350394,"mi":170562,"mm":42029,"mp":106005,"mo":153835,"mu":105465,"p ":14900,"iù":27078,"na":431794,"nc":198192,"nd":220346,"ne":784634,"nf":25150,"ng":109071,"ni":335162,"nk":7990,"ki":12045,"ke":15323,"ka":11574,"m ":76534,"ko":6777,"km":10394,"li":478088,"le":450199,"ld":17670,"lg":10033,"lf":7624,"la":677771,"lc":22009,"lb":26576,"n ":671873,"hr":6559,"ht":9126,"hu":11107,"hi":93831,"hn":5725,"ho":18964,"id":103779,"ic":450726,"ib":38087,"ia":461965,"ig":105629,"if":52751,"ie":183937,"k ":32183,"ir":101293,"is":299704,"it":487446,"iu":44781,"iv":129892,"ik":7413,"il":319428,"im":199167,"in":697560,"io":541230,"ip":107291,"iz":100179,"l ":908043,"ja":6741,"z ":9054,"tà":85596,"wi":6023,"vv":10000,"y ":65822,"wa":12380,"we":6401,"vi":166809,"vo":78376,"uz":20664,"ve":175153,"va":133125,"x ":19811,"ui":79181,"ul":72491,"ue":88946,"uf":10300,"ug":23193,"ur":134766,"us":100555,"ut":118876,"um":71437,"un":561799,"uo":58100,"up":47527,"ty":10365,"tu":164890,"tt":381279,"ub":48232,"ua":156844,"ud":43002,"uc":34894,"w ":10150,"to":743121,"tl":8532,"ts":9120,"tr":295588,"te":580573,"ti":577841,"th":33664,"ta":698330,"su":137066,"sv":16162,"ss":213977,"st":447357,"sl":9152,"sk":10657,"sm":16619,"sp":87152,"so":218207,"sc":157523,"sf":9364,"se":363584,"sh":16912,"si":401810,"rz":16932,"u ":59520,"sa":125213,"rr":72120,"rs":82412,"rt":207644,"ru":63955,"rv":21724,"ry":12041,"rp":15121,"ro":352219,"rn":64837,"rm":75961,"rl":22967,"rk":10623,"ri":614338,"rg":44544,"rf":10278,"re":622304,"rd":75640,"rc":66009,"rb":30426,"ra":583297,"t ":103375,"qu":99430,"s ":156468,"pu":55098,"pp":84071,"pr":255723,"ps":6191,"zz":76375,"zi":221598,"ze":20937,"za":99515,"zo":37143,"ya":7485,"È ":18782,"à ":92132,"ò ":26518,"ì ":8878,"é ":9885,"è ":314074,"ù ":29830," Ga":14003," Ge":15739," I ":16475," Fo":12451," Fu":8160," Fr":19775," Fi":15103," Ha":16981," He":8121," Go":9484," Gr":17959," Gu":8468," Gi":22095," Gl":5798," Ho":8931," L ":24925," Ja":8214," Is":8984," It":12700," In":30290," Il":65842," Ka":7887," Jo":9477," La":88189," Le":25663," Li":19618," Ma":58913," Mi":23636," Me":19838," Lo":25207," Lu":9633," Ne":23724," Na":17829," Ni":7466," Mo":30189," Mu":9681," A ":5979," Am":9971," An":18493," Al":34065," Ba":27690," Au":9040," As":8840," Ar":21718," Be":19500," Bi":8121," Bo":19624," Br":18373," Bu":8165," Ca":56136," Ce":10868," Ci":11600," Ch":29611," Cl":7713," Cr":10596," Co":62928," Da":15272," Di":21504," De":15426," Do":13718," Du":5966," El":6042," Es":8654," En":5749," Eu":7835," Fe":11726," Fa":10272," Wi":7870," Wa":7761," a ":110030," Or":10710," Po":20608," Pi":22451," Pe":16220," Pa":38161," No":19629," Ra":11758," Qu":10458," Ro":32569," Re":30042," Ri":14247," Pr":25353," Su":12949," St":32098," Ta":10678," UA":13306," Th":16944," Ti":6253," Te":15675," Tr":15562," To":14480," Ru":7431," Sa":46303," Sh":7238," Si":21606," Sc":24393," Se":20659," So":16887," Sp":9003," Va":13628," Ve":14047," Vi":19885," Vo":7231," Un":27982," l ":52713," im":22658," in":295683," il":151179," is":14605," it":18185," ha":23853," gi":32407," gl":14007," gr":40025," go":5702," gu":9712," id":6931," ne":275511," na":25351," mu":20050," mo":56127," ol":7856," om":7514," og":8277," oc":9930," of":10734," nu":14604," no":68303," le":52423," li":35936," la":143866," km":9911," me":51316," mi":29028," o ":42065," ma":81024," lu":16900," lo":28809," af":7222," ag":11338," ab":64448," ac":16193," ad":25196," am":20786," an":81988," ap":29511," ai":8950," al":183776," av":17618," au":20712," ar":36060," at":28661," as":31346," d ":20488," ba":32317," bi":8838," be":7889," bo":6391," br":13522," ca":114268," e ":219384," er":25454," et":12895," es":52932," en":13079," ep":5911," el":14730," fe":18710," fa":56978," fu":35844," fr":63957," fo":40785," fi":57936," ge":26299," ga":8284," i ":33110," cl":12326," co":356836," cr":19933," ce":24853," ch":106561," ci":55235," da":211997," cu":28570," do":30811," de":613533," di":670575," ec":32925," ed":39893," du":23486," ru":12186," sa":17824," se":114506," sc":42709," si":147221," sp":37866," so":80656," qu":64678," ra":31475," re":103663," ri":75848," ro":21093," pu":36089," pr":197319," os":7051," ot":10340," ov":5976," op":14803," or":49602," pe":114303," pa":91709," po":66694," pi":52800," va":18746," ve":35685," vo":17500," vi":51221," tu":14428," us":12554," ut":9880," un":408903," ul":5818," ta":13154," st":115114," sv":15397," su":116897," tr":76246," to":12684," th":12060," ti":18079," te":75278," È ":18766," è ":310990,"Eur":6532,"Gio":7152,"Fra":14090,"For":6200,"II ":9095,"Gra":9265,"Int":6283,"In ":9661,"Il ":64107,"Bas":6130,"Alt":6625,"Cal":8462,"Cam":7263,"Cas":9838,"Car":10034,"Can":7902,"Chi":9585,"Cen":5742,"Cha":12533,"Cor":7549,"Com":13539,"Col":6543,"Con":19924,"Dis":7107,"Nel":12164,"Nor":11147,"Per":7527,"Par":12134,"Pro":9921,"Pre":7483,"Que":6463,"Ita":11783,"Le ":10050,"La ":72068,"Man":6976,"Mar":24416,"Mon":12589,"Sta":17125,"UA ":13253,"Si ":7499,"Sai":6635,"Sco":15601,"San":19570,"Reg":7644,"Rom":11444,"Ven":6082,"Val":6880,"Uni":19062,"The":12364,"Tra":6212,"bit":77836,"bil":24167,"bli":33195,"bor":7965,"bbl":30259,"bbe":6273,"be ":7531,"bbr":5663,"ban":14724,"bal":6018,"bat":9698,"bas":13208,"bar":7268,"ber":18766,"bia":7881,"ca ":152253,"car":46789,"cas":12364,"cat":57544,"can":40911,"cap":15975,"caz":8224,"cam":17670,"cal":28832,"ce ":48205,"bri":12930,"bro":7660,"bra":15040,"bre":21272,"bum":18883,"am ":7918,"al ":120078,"ain":14219,"aio":8047,"agl":13391,"agg":47904,"agi":14545,"agn":23437,"ago":14318,"anz":27062,"ano":80778,"ann":47068,"ant":158501,"ans":9496,"ane":24020,"ang":15547,"ani":57213,"ana":44733,"anc":100934,"and":75292,"amm":16889,"amo":10418,"amp":23753,"ami":24192,"ame":55568,"amb":14366,"ama":14741,"alt":24272,"alo":9405,"alm":19827,"all":157126,"ali":107974,"alc":17413,"ald":5979,"ale":157930,"ala":17702,"alb":21166,"an ":45440,"abb":6484,"abi":71766,"abo":7042,"ae ":8196,"ad ":23773,"aff":9278,"afi":13627,"ai ":19697,"aga":10288,"age":5874,"aes":7300,"ado":12933,"adr":12192,"adi":16689,"ade":11907,"acq":6540,"aco":8070,"aci":10043,"ach":7116,"ace":12782,"acc":33357,"ada":10503,"acr":5776,"azi":114771,"azz":11554,"at ":6931,"arg":6155,"are":73118,"ard":28986,"arc":21226,"ara":44876,"aro":14834,"arn":6502,"arm":9201,"arl":10615,"ari":92056,"arr":17226,"ars":9758,"art":120284,"asa":12008,"asi":18399,"asc":31970,"ase":8737,"ar ":13924,"api":8043,"apo":15567,"app":40715,"as ":11094,"ava":20339,"aut":20581,"avo":14059,"avi":12649,"ave":19075,"ay ":7096,"avv":6341,"ata":122912,"ast":41964,"ass":66552,"atr":11363,"ato":251278,"ate":43614,"ati":96839,"att":98894,"atu":21242,"aur":8293,"aus":7205,"ito":68066,"itu":73079,"itt":66676,"ism":9170,"iso":15645,"isp":27507,"iss":18851,"ist":119446,"ita":148002,"ite":23677,"iti":31286,"ivo":23510,"ius":6126,"ium":7439,"iun":5847,"iut":11741,"iva":34645,"ivi":28519,"ive":42061,"ipo":9185,"ipi":8806,"is ":24623,"ion":300543,"ior":37852,"ios":7307,"ipa":67260,"ipe":7498,"iov":6740,"iro":6686,"iri":10262,"isi":24222,"ise":9930,"isc":25942,"isa":8063,"ire":38092,"ira":11739,"irc":16765,"ità":54075,"izz":49660,"izi":46425,"km ":6935,"ha ":20767,"ham":6367,"han":10716,"har":10403,"he ":154969,"het":5812,"her":11167,"hi ":17347,"hie":13497,"hia":17408,"hin":6740,"hil":8553,"hit":6455,"go ":24875,"gle":14019,"gli":101433,"gno":22948,"gni":13103,"gne":7169,"gna":23696,"gol":13409,"gon":12443,"gru":16613,"gra":38555,"gre":12522,"gui":11363,"gua":15610,"gue":16931,"gur":5768,"iam":18301,"ial":41395,"ian":67167,"ias":18220,"iar":13087,"iat":27828,"ic ":7518,"iac":6819,"ibi":9969,"ibr":7455,"ibu":6636,"iaz":6940,"ibe":7052,"ia ":244503,"iet":16601,"iem":9287,"ien":42565,"ier":26963,"ies":15028,"iff":8035,"ife":10392,"ifi":25651,"ico":88289,"ici":66289,"ich":34469,"icc":12587,"ice":31036,"ie ":52789,"ica":189725,"ido":7371,"idi":19231,"ide":49954,"ida":12753,"il ":154014,"igl":35255,"igh":7328,"igi":24518,"igu":7702,"ign":12535,"imo":36242,"imm":6781,"imp":19629,"ime":69205,"imi":17819,"inc":72864,"ind":26998,"ina":84297,"ino":48220,"int":56511,"ins":16076,"inf":11205,"ine":59357,"ing":57070,"ini":52386,"ioc":18011,"inv":7003,"ila":12536,"in ":183514,"ilo":8196,"ill":25371,"ilm":20062,"ili":45534,"ile":34997,"ima":41217,"io ":144991,"ilu":7192,"ffe":9164,"ffi":11476,"fes":8200,"fer":25051,"fia":6078,"fas":16462,"fat":6791,"far":5672,"fam":16618,"fan":7946,"età":10656,"ezz":10468,"ezi":12836,"eta":25147,"ete":11093,"eti":19239,"esp":10424,"eso":7266,"est":82383,"ess":82490,"eto":8361,"etr":19381,"ett":136219,"eve":12042,"eva":15707,"evo":6501,"evi":15209,"eur":6939,"ey ":9468,"er ":129403,"eor":7000,"es ":37108,"epu":6598,"eri":107300,"erg":9054,"ere":60624,"erf":7449,"erc":19250,"era":90400,"et ":14734,"equ":6340,"esi":40808,"esc":25003,"ese":123684,"esa":17222,"erz":5997,"erv":15974,"err":35138,"ert":35208,"ers":54254,"ern":33474,"erm":28661,"erp":7756,"ero":53515,"en ":21820,"ela":13747,"ele":33688,"eli":13565,"ell":470580,"elo":7068,"eo ":15585,"emb":16989,"ema":19885,"eme":23252,"emo":10570,"emi":32972,"emp":19288,"ene":64829,"eng":5768,"ena":16730,"end":39915,"enc":6461,"eno":18323,"enn":25781,"eni":17944,"enu":8919,"ens":28995,"ent":321381,"enz":41200,"egl":30932,"ego":7039,"egn":18128,"egg":11766,"egi":67882,"egu":13973,"el ":344686,"giu":12377,"gis":8322,"gin":21943,"gio":125636,"gic":7947,"gia":27457,"ght":6164,"ghi":6769,"ghe":8042,"ggi":57223,"gge":11031,"gi ":14154,"gen":34367,"get":10150,"ger":10735,"ge ":13833,"gar":8686,"gat":8038,"gan":16471,"ga ":14336,"fra":61392,"fu ":20614,"for":37768,"fon":16839,"fic":48822,"fig":8947,"fil":22511,"fin":27532,"da ":136024,"de ":72497,"dal":94565,"dai":6473,"dat":26742,"dar":7604,"dan":11831,"cun":8420,"cul":6650,"cui":21314,"cur":6089,"cla":11488,"cli":30380,"co ":105436,"cog":6308,"con":172605,"col":60224,"com":159257,"cor":35975,"cos":24601,"cop":26258,"cqu":7001,"cre":13257,"cra":6453,"cri":26431,"cro":12281,"cci":18510,"cch":12126,"cco":24015,"cca":13415,"cce":29193,"ch ":7853,"cer":13344,"ces":73387,"cen":46057,"cel":12841,"ced":6856,"ci ":29747,"cha":6310,"cia":75274,"ck ":12816,"cie":22050,"cid":13163,"che":152251,"chi":60026,"cil":6171,"cir":15709,"cis":9575,"cit":48155,"ciu":9832,"cin":18966,"cio":18635,"cip":33626,"ed ":34167,"ebb":8622,"ebr":5973,"ean":5957,"eal":11897,"eat":17904,"ea ":31185,"efi":7167,"ei ":71643,"ega":13314,"edi":40474,"ede":36599,"ecl":13991,"eci":22204,"ece":13292,"ecc":23316,"eca":6015,"ee ":7306,"eco":39931,"dur":10435,"dut":6584,"duz":6137,"dor":7110,"dop":9690,"don":12153,"dov":9865,"dot":18765,"ds ":7341,"due":12229,"dri":5744,"dra":9112,"dre":9661,"dro":6212,"dic":30670,"dia":36200,"der":31418,"des":28656,"det":11258,"dec":5661,"def":6138,"deg":19555,"dei":52282,"del":486810,"den":37113,"deo":6268,"di ":549365,"do ":64324,"div":20925,"diz":13866,"din":21333,"dio":28106,"dip":45197,"dir":19963,"dis":42122,"dit":11525,"die":7033,"dif":12635,"rga":12245,"ri ":84427,"rgi":7768,"rge":8891,"rgo":7492,"ret":51197,"res":79309,"rev":9081,"rfi":5771,"rds":5842,"rea":31808,"rec":19082,"red":9839,"reg":68923,"rem":13549,"ren":40494,"rel":11321,"rda":7163,"rdo":8984,"rdi":19945,"rde":7719,"re ":253820,"rco":11417,"rci":11839,"rch":14217,"rca":18883,"raz":24666,"rd ":22088,"rap":12296,"rar":11138,"ras":20760,"rat":94270,"rav":8469,"rbi":17348,"rai":8283,"rag":15270,"ran":117955,"ram":19333,"ral":33868,"raf":16450,"rad":21432,"rac":15856,"rpr":6371,"rs ":8921,"ros":16960,"rot":12347,"rom":18807,"ron":34271,"rop":27028,"rov":41355,"rod":22675,"roc":16930,"roi":16969,"rol":9837,"rof":8044,"rog":13121,"rno":15812,"rna":22371,"rne":10164,"rni":10544,"ro ":93036,"rma":38511,"rme":10556,"rmi":17565,"rla":7860,"riz":22621,"rio":45747,"rit":48367,"ris":50562,"riv":16323,"rig":24728,"ril":9910,"rin":43294,"rim":44423,"ria":61665,"rib":8788,"ric":75415,"rid":11323,"rie":41936,"rif":9551,"rk ":5699,"rup":18084,"rus":6787,"rut":6215,"rva":7812,"rvi":6676,"rve":5812,"ry ":9246,"rsi":24369,"rso":28714,"rsa":7860,"rse":7118,"rta":21350,"rto":33655,"rte":47947,"rti":80543,"rt ":10303,"rro":11867,"rri":17046,"rre":16669,"rra":22088,"sal":9289,"san":12698,"sat":15313,"sar":8168,"sa ":55092,"rzo":7427,"si ":104255,"siv":14099,"sie":10542,"sid":14077,"sic":27928,"sia":21469,"sit":73146,"sis":21720,"sin":20546,"sio":35066,"sil":9310,"sim":20855,"sig":13260,"scr":20145,"se ":150356,"sca":15235,"sce":19663,"sci":47092,"sch":12701,"sco":34425,"ser":44878,"ses":5630,"set":14434,"seg":23827,"sed":9878,"sec":24038,"sen":42117,"sem":26599,"spo":16816,"spe":38890,"spi":8159,"spa":13861,"sot":9626,"sol":26000,"son":52029,"sop":6399,"sor":13321,"soc":11137,"su ":15907,"st ":20759,"smo":8359,"so ":74484,"sse":52141,"ssa":30610,"sso":53075,"ssi":64743,"ssu":6194,"ste":66836,"sta":130887,"sto":49939,"sti":78547,"stu":12684,"str":82216,"sua":16483,"sud":8747,"suc":8338,"sul":27351,"sup":11429,"suo":21188,"sur":6713,"svi":7639,"svo":5978,"tal":71028,"tag":17728,"taz":13579,"tav":10674,"tat":99866,"tas":10040,"tar":29860,"tan":110605,"tam":12199,"te ":231097,"ta ":297475,"pa ":11956,"pe ":6955,"par":126674,"pat":12618,"pas":6581,"pag":17667,"pal":29532,"pan":6881,"pi ":12873,"pec":11403,"pen":11283,"per":160544,"pet":24231,"pes":8448,"pli":9280,"ple":7659,"pia":16405,"pic":12371,"pie":7531,"pin":8248,"pio":15425,"pir":5858,"pit":11446,"por":30452,"pop":13161,"pot":6913,"pos":33968,"poi":5808,"pon":18443,"pol":43083,"poc":5679,"ppr":9310,"ppi":6671,"ppo":29526,"ppa":24732,"ppe":7033,"po ":47633,"più":26636,"pub":29314,"pra":8794,"pri":70446,"pre":86755,"pro":88068,"put":6255,"pun":7305,"qua":45208,"que":36643,"qui":15702,"ra ":147867,"ngo":19829,"ngl":15051,"ngu":15413,"ni ":109423,"nge":12593,"ngh":7552,"nga":7243,"neg":12898,"nei":14330,"nel":251833,"nen":19286,"nem":6968,"ner":29386,"net":10607,"nes":22568,"ng ":19777,"nea":10437,"nfi":6288,"nco":13364,"nci":58066,"ncl":16907,"nce":63829,"nch":32511,"nca":8638,"ne ":381891,"ndu":5794,"ndr":9579,"ndo":51716,"ndi":44494,"nde":40952,"nda":41143,"nal":55844,"nam":6348,"nan":9718,"nar":20198,"nag":9973,"nd ":21375,"nat":56733,"nas":7852,"naz":16138,"na ":222844,"iù ":27049,"nve":7397,"num":9332,"nut":9355,"nto":104437,"ntr":54646,"nti":140368,"nta":97179,"nte":203922,"nso":6392,"nse":23034,"nsi":31575,"nt ":19040,"nqu":6994,"ns ":8742,"nol":12956,"nom":37789,"non":21278,"not":13854,"nos":16869,"nor":18667,"nov":9880,"nne":24481,"nna":15245,"nno":16579,"nni":28556,"no ":229968,"nif":7757,"nie":7902,"nic":33089,"nia":32535,"niz":16719,"niv":11731,"nis":28026,"nit":37389,"nio":10735,"nim":18502,"ogr":18460,"ogi":20692,"ogo":10374,"ogn":10473,"oge":8216,"ogg":8490,"oi ":11644,"oir":5807,"oid":15471,"ol ":6484,"oce":16535,"och":7177,"oci":18254,"ock":8603,"oco":10221,"oca":18569,"occ":17275,"ode":8882,"odi":13909,"odo":23349,"of ":9685,"oda":9043,"odu":9647,"obi":7836,"nza":33286,"nze":8647,"nzi":18694,"nzo":13784,"oti":8184,"ote":14816,"ott":46439,"oto":17159,"ost":54505,"ota":15242,"osi":22379,"ose":9147,"oss":25102,"oso":10625,"ovi":33581,"ova":26672,"ove":33842,"oun":6402,"our":8721,"opo":34311,"opp":9241,"ope":36127,"os ":8709,"opr":18087,"or ":15387,"orm":33668,"orn":18923,"oro":17981,"orr":13295,"ord":34705,"ore":83573,"org":19011,"ori":84933,"osa":10641,"osc":18331,"ort":43269,"ors":13700,"orb":16181,"ora":29450,"ola":56821,"on ":116999,"oli":48219,"oll":21894,"ole":18360,"olt":34437,"olo":80006,"olu":13657,"ona":73743,"ond":62359,"onc":11700,"onf":10972,"one":239015,"ong":9187,"oni":83803,"onn":10167,"ono":78642,"ons":27379,"ont":70151,"oma":36812,"ome":71129,"omb":8981,"omi":23159,"omm":10870,"omp":38351,"omo":18571,"omu":77250,"la ":502615,"le ":302428,"lci":6000,"lcu":7598,"lab":6177,"lac":10059,"lan":31109,"lam":6780,"lar":23739,"lat":27051,"las":21662,"lav":9746,"laz":14283,"ld ":6147,"lbu":19083,"lpi":7148,"lon":12253,"lom":6131,"lor":18602,"loc":10691,"log":22625,"los":5990,"lme":20869,"lti":14409,"lto":12474,"ltr":18612,"lta":21939,"lte":12298,"li ":126320,"lev":10337,"les":31571,"let":25767,"ler":10315,"lem":6948,"len":14630,"leg":16217,"lo ":116614,"lla":366876,"lle":95889,"lli":26880,"llo":51317,"lm ":18058,"ll ":147686,"lit":48006,"lis":18952,"lio":27570,"lin":48858,"lim":8378,"liz":23775,"liv":6284,"lic":53902,"lia":69213,"lib":8948,"lig":7530,"lie":14089,"ma ":77830,"mag":30019,"mar":19705,"mas":10652,"mal":8482,"man":52234,"maz":7709,"mat":39139,"mba":7166,"mbi":12940,"mbr":16743,"me ":86094,"med":14225,"met":29612,"mes":11952,"mer":34722,"mem":5945,"men":156332,"lup":7071,"luo":8412,"lun":8513,"lus":8431,"mpi":23542,"mpe":12689,"mpr":10131,"mpo":28943,"mpl":10446,"mod":11161,"mon":36802,"mol":12121,"mor":12412,"mos":9186,"mot":7018,"mpa":14258,"mus":15113,"mun":79273,"mi ":19370,"min":44712,"mil":12857,"mis":11697,"mit":12512,"mic":16950,"mia":19829,"mig":18385,"mo ":49530,"mmi":13753,"mma":13393,"mme":10561,"zzo":8684,"zza":55596,"zi ":8253,"ze ":11813,"zaz":5784,"zat":34965,"zon":11265,"zo ":22095,"zia":26147,"zie":6922,"zio":172644,"za ":45565,"tà ":85470,"vve":6371,"via":17063,"vil":14796,"vin":31306,"vic":6909,"vid":11446,"vie":12590,"viz":5900,"vit":13527,"vis":26147,"vo ":24368,"vol":30564,"vor":8888,"vi ":11655,"ver":60261,"ves":9627,"ven":39876,"vel":9674,"ve ":30393,"val":16987,"van":19808,"vam":6468,"var":11852,"vat":11790,"va ":54959,"uzi":17041,"usi":20123,"use":8740,"usc":8066,"usa":11990,"ust":15108,"uss":11959,"uti":17848,"ute":8898,"uta":15229,"utt":27857,"uto":37177,"us ":15875,"ura":49253,"ure":16806,"urg":6288,"uri":11168,"uro":17633,"uog":8121,"uol":7555,"uov":6581,"ur ":9581,"upe":12180,"upp":25887,"umb":6348,"ume":24760,"uo ":15893,"unt":14471,"uni":39506,"uno":18306,"una":118143,"ung":13147,"une":69547,"um ":26907,"ult":16466,"ull":18007,"ula":8409,"un ":268027,"uin":7055,"uis":8264,"uit":22856,"ul ":12226,"ui ":25799,"udi":18251,"ue ":29020,"ucc":13459,"uer":10140,"ues":18932,"uff":8520,"uen":11155,"uel":14803,"ua ":25910,"uat":61583,"uar":12747,"ual":28454,"uan":11940,"ubi":6952,"ubb":30175,"ud ":7866,"uad":8171,"ty ":9152,"tur":35100,"tut":19514,"tui":7703,"tun":9180,"tua":67153,"tud":14358,"ttà":16625,"tre":43645,"tra":110475,"tri":61377,"tru":14883,"tro":62451,"tta":55228,"tte":61167,"tti":63839,"tto":137927,"ttr":17103,"ttu":21997,"to ":551865,"tog":6135,"tos":6930,"tom":8273,"ton":24892,"tol":23856,"tor":90775,"til":16701,"tif":7476,"tie":11090,"tig":6347,"tir":6554,"tit":35734,"tis":12185,"tin":33317,"tim":54555,"tip":9392,"tio":18064,"tia":7439,"tic":92757,"tiz":5615,"tiv":40919,"tem":30137,"ten":60350,"tel":26672,"tea":13828,"tec":11962,"ted":10497,"th ":7445,"tes":26748,"ter":139440,"ti ":212534,"the":11405},"n_words":[55820958,65476626,49460182],"name":"it","type":"latin","flags":["diacritics"]}
diff --git a/contrib/languages-data/lt.json b/contrib/languages-data/lt.json
new file mode 100644
index 0000000..53c8ce9
--- /dev/null
+++ b/contrib/languages-data/lt.json
@@ -0,0 +1 @@
+{"freq":{"ūdu":237,"ūdo":102,"ūdi":671,"ūda":331,"D":8684,"E":5566,"F":3594,"G":7812,"A":19441,"B":11003,"C":5201,"L":15923,"M":12432,"N":7515,"O":2898,"H":3427,"I":6943,"J":8588,"K":17851,"U":3005,"T":11425,"W":951,"V":11358,"Q":106,"P":20146,"S":16914,"R":10122,"Y":441,"X":1264,"Z":1079,"f":17928,"g":102928,"d":143421,"e":363311,"b":73275,"c":39025,"Fed":229,"a":700709,"n":341806,"o":414082,"l":214716,"m":192290,"j":140065,"k":242084,"h":15703,"i":803156,"w":1450,"v":137211,"Fer":122,"u":266546,"t":305542,"s":495974,"r":352720,"q":282,"p":146221,"z":20490,"y":92511,"x":937,"ūgn":67,"ūgi":136,"­":175,"²":445,"Fil":257,"Fin":83,"í":134,"é":265,"ä":96,"á":197,"à":65,"ü":86,"ú":64,"ö":70,"ó":158,"ė":94567,"ę":9237,"ā":164,"ą":20880,"Č":1561,"č":23545,"ī":70,"Į":732,"į":24134,"ō":147,"ų":90375,"Ž":2451,"ž":44691,"Š":7077,"š":70424,"Ū":76,"ū":25506,"Fal":71,"Far":66,"ūbr":365,"Fab":73,"Eri":121,"Est":230,"Eti":120,"Eur":1489,"Ekv":92,"Ele":243,"Eko":63,"μ":79,"ν":128,"ο":198,"ι":119,"κ":65,"λ":109,"ε":95,"α":204,"Ent":76,"σ":88,"ς":168,"ρ":114,"π":65,"τ":94,"ь":90," l":18168," m":39544," n":26364,"я":147," o":7927," h":2325," i":50039," j":10300,"ы":100," k":69096," d":33767," e":11412,"ц":67," f":7201,"ч":71," g":24899,"р":400,"с":408," a":48847,"т":349," b":21159,"у":184," c":5176," y":9991," x":101," z":358," u":9812," t":37056," w":140," v":43820," p":74653," s":59667," r":27511," J":8568," K":17825," H":3382,"Gel":228," I":6930," N":7489," O":2834,"Geg":68," L":15878," M":12380," B":10949," C":5111,"Ged":147,"С":93," A":19385," F":3572," G":7762," D":8642," E":5548,"л":327,"к":381," Z":1072,"й":180," Y":439,"и":579," X":1253,"п":122,"о":721,"н":471,"м":156,"г":113," S":16766,"в":327,"Ger":157," R":10064,"б":111," Q":106,"а":757," P":20081,"з":104,"Geo":114," W":927,"Gen":295," V":11305,"е":519," U":2992,"д":172," T":11383," ą":118," č":811," Č":1561," ė":303,"Gla":64,"Gib":75,"Gin":94,"Gil":90,"Gir":131," ž":11466," Ž":2446," ų":74," Ū":76," ū":618," Š":7067," š":19822," Į":722," į":17189,"šų ":222,"Gan":127,"Gal":319,"Gam":189,"Gaj":89,"Gau":103,"Gar":244,"Gai":76,"Gab":77,"ي":99,"ل":106,"م":65,"ن":73,"ب":63,"ا":149,"ر":65,"šūn":177,"Flo":97,"Fra":186,"Fri":67,"Fre":105,"A ":1074,"For":525,"F ":250,"Da":2187,"Cu":139,"Cy":71,"Cl":158,"Co":831,"Cr":212,"Ce":857,"Ch":1267,"Ci":314,"G ":220,"Ed":144,"Dv":187,"Du":605,"Dz":157,"Dy":151,"Do":667,"Dn":85,"Dr":666,"De":900,"Di":2059,"Fe":543,"H ":197,"Fa":428,"Eu":1613,"Ev":98,"Ex":73,"Er":363,"Et":189,"Es":386,"En":323,"Em":143,"Ep":87,"Ei":125,"El":573,"Ek":234,"Eg":301,"Ge":1133,"Ga":1414,"I ":1579,"Fu":220,"Fr":454,"Fo":751,"Fl":218,"Fi":559,"B ":521," С":93,"II ":866,"C ":1065,"Av":215,"Au":2682,"Ar":1944,"At":910,"As":631,"D ":380,"Ba":3819,"Az":731,"Af":806,"Ag":282,"Ab":409,"Ac":198,"Ad":552,"Am":1804,"An":2621,"Ap":788,"Ai":520,"Aj":72,"Ak":583,"Al":1985,"Bu":1123,"Br":1448,"Ca":774,"E ":242,"Bi":838,"Be":1548,"Bo":1206,"Bl":293,"Him":100,"Hip":99,"Kv":191,"Ku":1466,"Ky":76,"Kn":86,"Kl":830,"Kr":1957,"Ko":2668,"Le":2735,"Li":7587,"N ":208,"La":2652,"Lu":489,"Ly":250,"Lo":724,"Dū":73,"Me":1902,"Dž":420,"Mi":1792,"O ":471,"Ma":4741,"My":151,"Mu":533,"Mo":1998,"Ni":720,"Ež":273,"Ne":1803,"Na":1866,"P ":639,"Hel":151,"Ny":218,"Nr":561,"Nu":901,"No":779,"Ok":221,"Ol":265,"Om":89,"On":141,"Ją":106,"Oc":104,"Od":117,"Of":81,"Hen":73,"Ob":214,"Her":326,"Gi":594,"Gl":239,"Gr":2248,"Go":422,"Gu":450,"Gv":487,"Gy":327,"J ":126,"Ha":879,"He":851,"Hi":517,"Ho":543,"Hu":227,"Hy":110,"K ":521,"Ib":64,"Id":102,"ा":68,"Ig":304,"Im":227,"In":1584,"Ik":112,"Il":691,"Aš":111,"्":69,"Iv":75,"Is":624,"It":425,"Ir":401,"Ja":1783,"L ":366,"Dė":112,"Iz":149,"Ji":1102,"Je":435,"Jo":1434,"Bū":168,"Ju":1787,"Hal":105,"Ka":7199,"M ":415,"Han":93,"Ham":121,"Har":160,"Ki":1470,"Hav":88,"Ke":841,"Us":82,"Mū":111,"Ut":266,"Ur":330,"Uo":118,"Up":176,"Un":265,"Uk":462,"Ul":82,"Ug":88,"W ":71,"Ty":98,"Tv":72,"Gyv":237,"Tu":954,"Tr":1269,"To":1061,"Th":580,"Ti":954,"Te":1633,"Ta":3988,"V ":1768,"Sy":113,"St":1651,"Kū":349,"Sv":277,"Su":2454,"Wo":154,"Wi":274,"Wh":63,"Wa":211,"We":128,"Vy":520,"Vo":1251,"Vi":4324,"Vl":71,"X ":603,"Va":2935,"Ve":1368,"Uz":69,"Gva":257,"Gvi":230,"Iš":781,"Jį":73,"Pu":558,"Pr":4041,"Ps":108,"Kė":213,"S ":1115,"Pe":1585,"Gua":94,"ėžt":110,"Pa":7638,"Gud":68,"Pl":1104,"Po":1535,"Pi":2755,"Ph":112,"ėži":636,"Os":252,"Ot":63,"Op":146,"Or":562,"Jė":115,"R ":1502,"Se":1883,"Sc":300,"Si":1578,"Sh":180,"Sn":82,"Sm":286,"Sl":299,"Sk":1024,"Sr":63,"Sp":553,"So":1804,"Ru":2114,"Jū":181,"Ry":867,"Mė":156,"U ":260,"Jų":127,"Sa":3106,"Re":2289,"Ri":771,"Ro":1443,"Lė":71,"T ":426,"Ra":1932,"Gre":184,"Gri":532,"Gra":983,"Vė":216,"b ":435,"Už":349,"Gru":297,"Gro":162,"a ":85927,"Sė":67,"Yo":68,"Yr":183,"Z ":96,"Są":373,"Gol":68,"Sū":95,"Tė":65,"Za":514,"Ze":220,"Zi":68,"Zo":74,"Rū":230,"aė":112,"i ":72374,"fy":103,"gd":980,"ge":8770,"ga":24984,"Inf":65,"fl":318,"fg":67,"ff":102,"fi":4585,"fr":1236,"ač":2434,"fu":1782,"ft":398,"fo":4629,"Int":267,"Ins":77,"j ":778,"bę":413,"bė":6516,"gy":5384,"gz":596,"he":3549,"ha":2884,"gn":1795,"gm":391,"gl":2484,"bą":398,"gi":19462,"gh":277,"gv":1056,"gu":4898,"gt":1721,"gs":555,"gr":11904,"gp":278,"go":7511,"du":8502,"dv":2782,"dy":7581,"dz":385,"g ":2674,"ea":3623,"eb":2049,"ec":2366,"ed":9131,"de":15402,"dg":238,"Ilg":538,"di":31560,"dh":99,"dk":208,"dm":1790,"dl":210,"do":13968,"dn":150,"ds":320,"dr":6349,"ew":210,"ex":138,"eu":1022,"ev":4073,"ey":228,"ez":1754,"fa":1478,"h ":675,"Ind":876,"fe":2397,"eh":155,"eg":6969,"ef":1143,"ee":377,"el":20835,"ek":14743,"ej":2777,"ei":27949,"ep":4431,"Imp":87,"eo":2521,"en":55954,"em":13103,"et":36257,"es":24051,"er":37980,"ca":1365,"bz":182,"e ":76709,"by":382,"bs":381,"br":2910,"bu":7186,"bt":432,"bn":67,"bo":6948,"bj":1259,"bl":3193,"bi":8863,"bd":129,"be":9419,"da":31040,"f ":470,"cy":129,"cu":428,"ct":550,"cr":288,"co":1164,"cm":472,"ck":632,"cl":149,"ci":18836,"ch":6035,"ce":6724,"cc":125,"Iki":75,"c ":600,"az":2975,"ay":393,"ba":18685,"d ":4075,"at":26947,"as":91731,"ar":60917,"ax":94,"aw":113,"av":20318,"au":60068,"ak":21549,"al":70175,"ai":71518,"aj":9247,"ao":400,"ap":19804,"am":34880,"an":55907,"ac":11205,"ad":17624,"aa":279,"ab":6312,"ag":11672,"ah":654,"ae":3599,"af":1358,"nu":16862,"nt":40253,"ns":5838,"ič":2386,"nr":387,"np":87,"no":25328,"nn":299,"iė":81,"fų":79,"nz":353,"ny":7794,"nv":1496,"oe":521,"of":2043,"oc":3010,"od":8826,"oa":712,"ob":3847,"om":18799,"on":30734,"ok":15352,"ol":18423,"gš":230,"oi":681,"oj":37495,"ją":2263,"og":8954,"oh":445,"ot":12743,"m²":434,"os":91367,"gū":892,"ov":9611,"ou":825,"op":6292,"oo":622,"or":24858,"jė":931,"gų":1741,"ję":663,"r ":33992,"ox":93,"ow":356,"oz":1228,"oy":145,"pd":423,"pe":8966,"Ign":235,"gž":880,"pg":158,"pa":38606,"pc":152,"pl":7261,"pm":79,"pn":207,"po":11055,"ph":742,"pi":24691,"ką":1329,"pj":353,"pk":273,"lo":17284,"ln":7253,"lm":1772,"ll":1349,"ls":6461,"dū":239,"lp":828,"lv":2437,"lu":5702,"lt":6648,"dų":1603,"lz":63,"ly":8935,"gę":183,"gė":2321,"o ":113444,"md":246,"ma":41541,"mb":3140,"mg":95,"dž":9497,"me":27128,"mf":234,"mk":64,"ml":224,"mi":27912,"eš":4905,"mn":423,"mm":415,"mp":5761,"mo":36514,"mt":1706,"ms":5086,"mu":8277,"gį":95,"mz":185,"my":1666,"hė":71,"p ":5826,"na":39063,"nb":286,"nc":4749,"nd":18751,"ne":20163,"nf":1054,"ež":4639,"ng":14841,"nh":168,"ni":75394,"ią":1524,"nj":255,"nk":20248,"nl":177,"nm":383,"jy":130,"dį":460,"ju":6480,"eč":2839,"bū":3463,"jo":44031,"ki":36060,"kh":180,"kg":90,"ke":8017,"kd":1358,"kc":1366,"ka":55987,"m ":17519,"bų":1576,"ky":6899,"ks":8217,"cū":948,"kt":10687,"ku":20625,"eį":147,"kv":1386,"ko":29827,"kr":12212,"kl":14764,"km":7611,"kn":695,"li":63975,"lh":105,"lk":4623,"lj":376,"gą":598,"le":15356,"ld":6869,"lg":3321,"lf":324,"la":33178,"lc":275,"lb":5099,"fė":82,"cų":73,"n ":4899,"hr":575,"hs":179,"ht":431,"hu":755,"hi":2459,"hn":698,"ho":2226,"hl":151,"hm":207,"id":19607,"ic":4825,"ib":3663,"ia":66700,"ih":166,"ig":7460,"if":2042,"ie":61163,"hy":403,"k ":2911,"cė":232,"ir":43336,"is":76128,"it":24630,"iu":21162,"iv":9558,"ix":156,"aš":6163,"ii":607,"dą":519,"ij":49065,"ik":38607,"il":20049,"im":33709,"in":120546,"io":45938,"ip":8649,"je":37765,"až":6015,"ji":8060,"iz":7107,"dė":5783,"dę":233,"l ":5566,"ja":28492,"są":2757,"xi":80,"pš":499,"sč":2453,"pū":264,"rį":1236,"ww":153,"rę":1878,"z ":658,"rė":10366,"ož":891,"šį ":141,"nž":291,"oš":750,"wi":168,"rą":1320,"wn":67,"wo":98,"rč":361,"ws":118,"vy":6021,"nų":7287,"vz":413,"y ":1490,"wa":302,"we":130,"vl":94,"vk":143,"nš":207,"vi":38809,"mž":705,"vu":2039,"pį":89,"vr":219,"vs":237,"vn":98,"vo":18294,"pė":6097,"mų":3539,"uz":1745,"ux":118,"uv":14825,"ve":15404,"va":45949,"pę":314,"x ":613,"mš":101,"ui":3816,"uj":5784,"pą":197,"uk":12269,"ul":12572,"ue":864,"uf":303,"ėšr":96,"ug":10980,"lž":345,"uh":117,"ur":34041,"pč":97,"us":51555,"mū":487,"ut":11996,"um":9287,"un":10381,"uo":35316,"up":8302,"ty":13326,"lų":4078,"tz":73,"nį":1610,"tu":28297,"tt":447,"tw":77,"tv":4512,"ub":4404,"ua":1611,"ud":10289,"uc":1769,"w ":260,"to":36142,"tn":718,"tm":1109,"tl":1570,"ts":3686,"lū":569,"tr":20341,"oč":849,"tp":243,"tg":277,"tf":256,"te":29101,"tk":518,"tj":104,"lš":275,"ti":64800,"th":1088,"v ":492,"nę":1600,"kų":5621,"nė":25365,"tb":1204,"tc":76,"ta":61331,"su":18269,"sv":2479,"kū":2551,"ss":694,"st":45024,"sy":1283,"sz":67,"sl":4978,"sk":13287,"sn":3503,"sm":4814,"sp":7479,"so":8499,"sr":2349,"nč":4435,"sd":318,"sc":661,"sf":460,"se":19656,"sh":403,"sj":177,"ną":1594,"kš":7202,"si":47811,"rz":154,"jų":7476,"mė":4474,"u ":18088,"mę":301,"sa":23030,"sb":164,"mč":71,"rr":385,"jū":2283,"rs":5538,"rt":15475,"lį":874,"ru":13317,"rv":2004,"ry":13171,"rp":4617,"ro":31644,"rn":3965,"rm":7673,"rl":1174,"rk":4790,"mą":3705,"rj":224,"ri":79105,"rh":115,"rg":5620,"iž":716,"rf":267,"re":18683,"rd":4525,"rc":1831,"rb":9032,"ra":68430,"t ":8619,"lę":789,"ių":36065,"lė":8350,"kį":293,"qu":225,"iū":2482,"lč":267,"lą":1161,"iš":22230,"kė":5197,"s ":278368,"kę":593,"px":100,"py":2379,"pt":3572,"pu":6072,"jį":548,"pv":639,"pp":160,"kč":96,"pr":22301,"ps":4103,"IX ":219,"zę":74,"zė":693,"už":4062,"yč":2372,"vū":491,"vų":1508,"uš":1079,"tž":178,"IV ":137,"tš":139,"sų":1592,"vė":5454,"vę":713,"tų":12691,"tū":4346,"vą":321,"rž":1586,"rų":3959,"sū":452,"vč":186,"Hor":71,"zg":207,"zi":7116,"rš":2441,"zb":85,"zd":1810,"ze":985,"tę":1071,"za":3048,"Hom":79,"yz":303,"Hon":129,"tė":5769,"Hol":87,"zy":69,"rū":3551,"zr":101,"uč":735,"zu":1165,"tį":886,"zo":1897,"zn":153,"zk":67,"zm":1647,"zl":136,"pž":94,"yg":3318,"ye":94,"yc":258,"yd":2201,"ya":257,"yb":11044,"sę":282,"sė":3998,"pų":330,"yv":6788,"sį":230,"yt":11083,"ys":9392,"yr":12733,"yp":1504,"yo":203,"yn":5647,"ym":5531,"yl":2089,"yk":7720,"yj":6842,"tą":2285,"yi":98,"Arg":584,"Are":73,"Arc":130,"Ara":258,"Arm":184,"Ark":155,"Ari":111,"šči":2647,"Apo":92,"Atl":385,"Asu":66,"Ast":264,"Art":193,"Avi":81,"Aut":259,"Aus":646,"Auk":724,"Aug":852,"Ats":131,"yž":529,"Aze":82,"zū":96,"Azi":595,"zų":230,"yš":830,"šė ":158,"šės":100,"šėj":66,"Bag":85,"Bah":87,"Bai":88,"Bak":129,"Bal":1866,"Ban":272,"Bab":93,"Bad":64,"Bar":458,"Bat":90,"Bas":130,"Bau":99,"Abi":168,"Ada":67,"Ach":87,"Adm":203,"Adr":71,"Aga":104,"Afr":719,"šą ":73,"Air":332,"Aka":93,"Akm":194,"Ala":187,"Alb":193,"Alg":65,"Ali":208,"Ale":201,"Alt":143,"Aly":220,"Alk":99,"Alp":174,"Ame":1274,"Ama":241,"Ang":426,"Ank":145,"Ana":165,"And":549,"Any":154,"Ant":913,"Api":205,"Apa":86,"Buv":78,"But":90,"Bul":113,"Bur":231,"Bud":102,"Bue":112,"Bui":103,"Bru":71,"² ":443,"Cac":84,"Cad":64,"Cal":67,"Cam":66,"Car":189,"Can":84,"Cap":73,"Bet":72,"Ber":374,"Ben":312,"Bel":409,"Baž":77,"Bil":78,"Bir":299,"Bla":117,"Bre":103,"Bra":609,"Bro":169,"Bri":429,"Bol":312,"Bon":72,"Bor":174,"Bos":142,"Bot":70,"Dei":68,"Del":74,"Dem":131,"Dep":64,"Deg":63,"Dam":137,"Dan":260,"Dar":403,"Dau":478,"Dab":105,"Dai":135,"Dal":128,"Cho":100,"Chr":185,"Che":286,"Chi":117,"Chu":119,"Cit":67,"Cen":563,"Cer":118,"Cha":378,"Cro":65,"DP ":99,"Cor":127,"Com":218,"Col":154,"Con":133,"アアア":149,"FA ":185,"ón":71,"Dze":98,"Egi":189,"é ":79,"Dni":78,"Daž":276,"ą ":16360,"Dia":66,"Did":1104,"Dis":102,"Dir":98,"Din":76,"Die":275,"Dub":146,"Dun":110,"Dvi":80,"Dus":63,"Dva":68,"Dru":132,"Dri":158,"Dra":241,"Dod":83,"ES ":63,"Don":122,"Dom":99,"Nem":374,"Nev":213,"Neu":81,"Net":114,"ęs":3467,"Ner":231,"Nep":196,"ėz":158,"ę ":5692,"ėc":117,"ėd":1595,"ėl":4129,"ėk":704,"ėn":2911,"ėm":2033,"ėg":1127,"ėj":12119,"ėt":2531,"ės":37968,"ėv":406,"ėp":181,"ėr":671,"čų":69,"Nat":155,"Nau":859,"−":522,"Nig":114,"Niu":165,"ūšy":191,"Nik":109,"ūši":1752,"ėč":63,"Eže":263,"New":68,"ėž":954,"ėš":258,"Nar":128,"Nam":132,"Nag":68,"Nac":163,"Nuo":622,"ąl":250,"ąj":1470,"ąm":118,"ąn":83,"ąs":1224,"ąr":696,"ąv":409,"Nyd":143,"Či":814,"Če":428,"Ča":214,"či":21375,"čk":154,"če":803,"ča":249,"č ":409,"ąž":190,"čo":121,"ču":90,"ė ":26547,"ųjų":1526,"OS ":98,"Nr ":561,"Nov":113,"Nor":386,"Not":108,"čė":116,"įž":184,"Ją ":106,"Oke":81,"Och":90,"Obe":83,"Į ":110,"į ":15271,"Įe":85,"Įk":141,"įg":465,"įl":469,"įk":1035,"įm":672,"įd":140,"įe":122,"Oli":102,"Įs":221,"įr":905,"įp":182,"įv":1510,"įs":2207,"įt":836,"Ori":91,"Org":75,"Oro":67,"Ost":83," −":501,"Po ":75,"Plu":194,"š ":7298,"Plo":267,"Pli":66,"Ple":71,"Pla":439,"Pin":159,"Pil":450,"Paš":81,"Pir":463,"Pie":1313,"Pic":67,"šį":142,"Per":807,"Pet":236,"Pen":132,"Pel":183,"šč":2693,"šą":98,"šė":408,"Pat":292,"Pas":767,"Par":1014,"Pav":466,"Pau":206,"Pag":484,"Pab":107,"Pad":139,"Pan":697,"Pam":122,"Pap":991,"Paj":94,"Pal":1658,"Pak":297,"šg":221,"še":5219,"šd":199,"ša":5331,"šb":74,"šo":2052,"šp":366,"šm":981,"šn":613,"šk":10936,"šl":1712,"ši":16792,"šv":1932,"šu":1367,"št":7877,"šs":1233,"šr":983,"šy":1194,"Še":341,"Ša":886,"Šl":99,"Šo":70,"Ši":3391,"Šk":111,"Jėz":95,"Šu":220,"Št":155,"Šv":1543,"Šr":75,"Prū":117,"Kėd":210,"ō ":76,"Pun":96,"Pus":118,"Pue":76,"Pro":479,"Pri":882,"Pre":234,"Jį ":73,"Pra":2269,"Pod":91,"Pol":415,"Pon":120,"Pot":100,"Pos":67,"Pop":119,"Por":237,"žr":232,"žs":417,"žt":693,"žu":2236,"žn":2332,"žo":2063,"žp":171,"žv":1312,"žy":1527,"žb":100,"že":6838,"žd":1624,"ža":3699,"žk":289,"žm":1670,"žl":158,"ži":16869,"Žm":91,"Žy":116,"Žu":122,"Žv":178,"Ža":482,"Ži":399,"Že":955,"RS ":294,"Išs":132,"Išt":197,"ž ":1080,"žų":376,"Iš ":185,"žį":113,"žą":94,"žė":568,"SA ":111,"ūči":238,"Rad":254,"Rai":78,"Raj":66,"Rag":90,"Ram":470,"Ran":143,"ū ":138,"šų":238,"šū":280,"ūg":553,"ūd":1733,"ūb":446,"ūz":1060,"ūs":1429,"ūt":1461,"ūv":180,"ūp":172,"ūr":9049,"ūk":1535,"ūl":283,"ūm":598,"ūn":4467,"ų ":88718,"ūč":245,"ųj":1568,"ūš":2032,"ūž":93,"Ita":399,"Isp":323,"Isl":135,"įpr":139,"įma":63,"įmo":598,"Ira":230,"įsi":807,"įsk":80,"įkū":191,"įst":1175,"įta":351,"įte":278,"įtr":77,"įtv":117,"įra":525,"įre":301,"įro":74,"Izr":81,"įga":117,"įgy":242,"Dėl":85,"įei":119,"įdu":68,"įku":691,"Jav":80,"Jau":108,"Jas":71,"įla":456,"Jap":569,"Jan":213,"Jam":221,"Jak":178,"Jel":102,"Jer":74,"Jis":502,"Jie":135,"Ji ":425,"あ":83,"Jo ":148,"Įst":114,"Įsi":88,"Jos":270,"Jon":566,"ア":227,"Joh":97,"Būd":72,"Įku":124,"Jug":66,"Jup":99,"Juo":530,"Jur":232,"Juk":63,"Jun":554,"Įei":85,"Kad":98,"Kab":68,"Kai":1116,"Kam":412,"Kal":1098,"Kap":211,"Kan":625,"Kau":985,"Kat":242,"Kas":171,"Kar":1509,"Kaz":340,"国 ":82,"Ker":168,"Ket":93,"Ken":88,"Kel":250,"Kir":183,"Kit":124,"Kin":537,"Kip":63,"Kie":93,"Kil":126,"Kli":79,"Kle":64,"Kla":523,"Klu":106,"Kon":597,"Kom":332,"Kol":450,"Kos":140,"Kor":624,"Kop":80,"Kov":100,"LR ":167,"Kre":575,"Kra":414,"Kri":437,"Kro":173,"Kru":166,"Kry":125,"Kub":74,"Kul":193,"Kun":174,"Kup":188,"Kur":371,"Kva":100,"Les":78,"Lep":69,"Leo":93,"Len":1971,"Lei":123,"Lau":223,"Laz":170,"MC ":504,"Lai":279,"Lat":571,"Lao":67,"Lap":196,"Lam":138,"Lan":215,"Lab":226,"Kuž":106,"La ":159,"Lib":176,"Lia":138,"Lie":6085,"Lig":81,"Lim":100,"Lin":276,"Lit":89,"Liu":198,"Liv":74,"Luk":93,"Lua":75,"Los":103,"Lot":96,"Lon":143,"ūgš":200,"ūs ":568,"ūpi":97,"ūra":1708,"ūse":158,"ūry":183,"ūru":361,"ūro":3121,"ūri":1981,"ūks":471,"ūka":130,"ūki":544,"ūkl":137,"ūma":254,"ūna":1137,"ūmi":118,"ūdž":157,"Lyg":63,"ūnu":353,"ūni":1251,"ūne":67,"ūno":452,"ūrė":579,"ūrę":259,"Mek":335,"Men":159,"Mel":164,"Mes":186,"Mer":401,"Met":158,"Med":222,"Mez":64,"ūzi":829,"Džo":137,"ūrų":466,"Dža":76,"Dži":72,"Dže":84,"ūkš":76,"ūsi":141,"ūną":68,"ūst":200,"ūnė":216,"ūta":74,"ūte":79,"Man":463,"Mal":406,"ūti":914,"Mar":1387,"Mas":288,"Mag":384,"Mad":147,"Maj":79,"Mak":226,"Mah":72,"Mai":142,"ūty":109,"Mac":86,"ūmų":111,"ūvi":114,"Mau":95,"Mat":219,"ūnų":861,"ūrą":276,"Mok":227,"Mol":370,"Mon":426,"Mos":65,"Mor":209,"Mot":313,"Moz":127,"ūzų":144,"NR ":79,"Mik":194,"Mie":142,"Mia":99,"Mic":168,"Mit":95,"Mir":79,"Mis":99,"Mil":131,"Min":395,"ūtų":160,"ūsų":232,"Maž":500,"Mur":145,"Mus":128,"Miš":117,"Sąj":308,"XX ":204,"XV ":71,"кий":66,"Wor":95,"Wil":72,"Win":96,"XI ":73,"War":82,"Vyr":158,"Vyt":150,"Viž":91,"Vok":795,"Vol":265,"Viš":70,"Vis":391,"Vit":100,"čkų":69,"Zar":256,"Zel":141,"之":194,"並":122,"三":326,"丁":115,"Yra":183,"三三":72,"Sve":90,"Sva":109,"Sur":111,"Sus":251,"Suv":802,"Sum":113,"Suk":74,"Sup":79,"Suo":198,"Sun":92,"Sud":274,"Suc":101,"Sub":64,"Str":249,"Stu":148,"Sti":184,"Sto":157,"Sta":653,"Ste":219,"Ten":81,"Tei":320,"Tel":387,"Tek":128,"Tam":147,"Tan":110,"Tar":749,"Tau":535,"Tai":1790,"Tak":96,"Tal":105,"Tad":89,"Ski":160,"Skr":105,"Sku":184,"Ska":362,"Sim":140,"Sil":118,"Sir":117,"Sin":192,"Sid":156,"Sie":159,"Sib":270,"Sic":64,"Ser":315,"Sep":68,"Sen":642,"Sel":68,"Sem":92,"Sei":374,"TV ":98,"Kūn":234,"国":86,"Spa":162,"TS ":71,"Spi":75,"Spe":132,"Spo":74,"Sof":72,"Sok":963,"Soc":156,"Sol":99,"Son":84,"Sos":108,"Sky":69,"Slo":158,"Smi":70,"Sma":84," 三":86,"Ryt":687,"Ryg":105,"Jų ":127,"Jūr":171,"Rus":1573,"Rug":83,"Rud":101,"Rum":121,"Sak":119,"Sam":191,"Sal":563,"Sac":85,"Sco":107,"Sav":191,"Sat":80,"Sau":518,"Sar":175,"San":807,"Mėn":80,"Rač":79,"SI ":77,"Res":1390,"Rio":84,"Rin":77,"Rib":131,"Rie":98,"Ras":159,"Rau":279,"Rec":68,"Red":71,"Rei":156,"Reg":116,"Ren":82,"Rok":173,"Rob":64,"Rod":82,"SR ":568,"Ros":182,"Rom":573,"SS ":87,"čų ":69," 之":78,"SO ":120,"Vai":313,"Vad":65,"Vel":220,"Ven":367,"Vei":167,"ски":73,"Vas":102,"Van":212,"Val":695,"Vak":682,"Var":578,"Vaš":70,"Vid":634,"Vie":542,"Vir":184,"Vil":1774,"Vik":201,"Vin":109,"Ver":344,"Ves":109,"Ukr":322,"Ukm":131,"Uni":208,"Uru":89,"Ura":110,"Ute":210,"Upė":66,"VD ":82,"VI ":165,"Ter":267,"Tet":74,"Tes":113,"The":467,"Tib":113,"Tie":67,"Tik":149,"Til":74,"Tim":101,"Tin":67,"Tir":87,"Tiu":72,"Tor":126,"Tok":232,"Tol":111,"Tom":115,"Ton":98,"Tru":94,"Tro":116,"Tri":330,"Tre":156,"Tra":520,"Tur":537,"Tun":69,"šga":146,"ši ":168,"šel":123,"šer":281,"šei":4330,"ša ":145,"še ":72,"šas":283,"šar":241,"šau":451,"šac":68,"šai":175,"šak":893,"šal":2584,"šam":125,"šan":259,"Švč":176,"Šve":932,"Švi":88,"što":898,"štr":99,"šte":875,"šti":1747,"škų":474,"šta":1289,"šun":139,"šuo":95,"šul":170,"šum":70,"šus":224,"šuj":65,"šuk":112,"štu":1243,"šty":912,"švi":810,"šve":671,"šva":235,"šut":236,"šuv":83,"švy":117,"špa":109,"šką":187,"špl":90,"šon":165,"šor":222,"šos":190,"šiš":128,"škė":457,"šre":107,"šra":138,"šri":577,"šiū":137,"šių":982,"šsi":699,"šsp":63,"šsk":346,"šru":91,"šsa":81,"šmė":100,"šku":531,"šky":77,"ško":2173,"šle":715,"šli":309,"šla":532,"šme":157,"šeš":251,"šmi":172,"šo ":946,"šma":81,"šmo":366,"šni":181,"šią":67,"šne":70,"šna":94,"šny":138,"šno":80,"šoj":128,"šok":166,"šom":149,"šia":7811,"šie":268,"šin":2480,"šio":1156,"šil":344,"šim":653,"šik":111,"šiu":646,"šir":121,"šis":1523,"šdė":102,"ška":2905,"ški":3659,"ške":337,"štų":188,"štė":332,"štį":112,"šyt":666,"šys":236,"šym":121,"štą":94,"ęs ":3114,"ęsi":184,"ęst":139,"bje":1237,"baž":702,"bis":206,"bit":262,"biu":413,"bio":445,"bip":90,"bir":575,"bik":90,"bil":1165,"bim":181,"bin":2312,"bij":590,"bo ":831,"blo":199,"ble":254,"bli":2467,"bla":154,"bod":66,"bok":148,"bol":1634,"boj":747,"bež":91,"biš":132,"bon":209,"bom":266,"bor":294,"bot":239,"bos":2270,"be ":431,"bam":151,"ban":1542,"bak":168,"bal":1952,"bai":1573,"baj":127,"bac":144,"bad":71,"baz":180,"bau":383,"bat":206,"bas":1718,"bar":1644,"bda":77,"bi ":151,"bei":2412,"ber":929,"ben":2915,"bel":329,"bek":139,"bev":245,"bes":726,"bet":875,"bia":875,"bib":123,"ėdo":412,"bie":263,"ėdi":261,"ėda":433,"buč":70,"bzd":176,"− ":502,"ca ":297,"buž":82,"car":324,"cas":107,"cat":74,"can":137,"cac":140,"cal":88,"ce ":305,"bių":787,"bri":867,"bro":454,"bra":835,"bre":106,"bry":134,"bu ":188,"bru":157,"bso":103,"bse":75,"bta":77,"bst":94,"bti":224,"btr":68,"buo":284,"bur":719,"bul":328,"buk":109,"bum":669,"bui":94,"bud":276,"buv":3564,"but":190,"bus":430,"brė":285,"byl":146,"bys":99,"abų":128,"aka":10783,"am ":1782,"ake":779,"akc":437,"aki":1290,"aji":120,"ėza":80,"ajo":4851,"aju":423,"aiz":904,"al ":2864,"adė":694,"aja":2169,"aje":126,"aik":6012,"ail":788,"aim":3610,"ain":2341,"aip":4816,"air":2272,"ais":9853,"ait":1788,"aiv":2833,"ėsč":100,"ak ":92,"aig":1833,"aid":2977,"aib":279,"ahi":72,"aho":88,"ėvi":102,"aj ":639,"abė":174,"agv":331,"agy":177,"ėva":79,"aha":210,"agl":74,"agm":87,"agi":657,"ėnų":1107,"agr":3158,"agu":296,"agn":741,"ago":977,"anu":591,"anz":118,"any":235,"ano":3129,"ann":125,"anm":118,"ant":17439,"ans":1524,"aič":1223,"anr":199,"ane":1374,"ang":3590,"ani":4762,"anj":123,"ank":4229,"ap ":76,"ana":4088,"anc":1441,"and":6903,"amu":630,"ėtų":311,"amt":459,"amy":484,"amz":131,"amm":86,"amo":3569,"amp":1185,"ams":2382,"ami":5086,"adž":530,"ame":5511,"amb":1223,"ama":10174,"ao ":130,"adų":80,"agė":263,"aly":3612,"alv":1945,"alu":1766,"alt":2672,"als":5664,"alp":263,"alo":3517,"aln":4883,"alm":300,"all":289,"alk":1267,"alg":429,"ali":17593,"alc":176,"ald":5080,"ale":3230,"Šan":75,"ala":5633,"alb":4521,"Šal":318,"Šak":157,"an ":1340,"Šau":91,"ėtį":118,"akv":76,"aky":230,"aks":329,"akr":1125,"Šar":99,"aku":724,"akt":1972,"ako":1771,"akn":187,"akm":413,"akl":356,"aba":2740,"abe":207,"abi":1011,"abl":158,"abo":330,"abr":414,"abs":143,"abu":396,"abz":181,"ae ":3025,"aca":72,"ad ":1165,"ac ":144,"ab ":182,"afo":102,"afr":63,"aft":132,"afi":672,"ai ":28957,"aga":3563,"age":494,"afy":94,"aen":65,"ael":136,"aei":134,"afa":115,"ado":1953,"adr":251,"adm":1262,"adi":7918,"ade":803,"ady":144,"adu":285,"adv":445,"ack":137,"aci":7737,"ach":900,"ace":1807,"ada":1767,"act":213,"azm":131,"azo":425,"azi":1049,"arš":379,"azl":78,"auč":212,"arū":95,"atė":313,"aze":155,"azg":73,"aza":391,"azd":314,"ėcė":111,"arų":1990,"arž":560,"apš":64,"asč":85,"arė":673,"arę":105,"atą":166,"ėgo":219,"ėgi":261,"apų":68,"asė":868,"ėga":161,"avų":296,"ėkt":249,"Šta":114,"ėn ":149,"Šv ":293,"ėla":197,"ėli":1870,"auž":306,"ba ":7831,"azė":203,"ėly":383,"ėgė":107,"Šud":88,"ėme":99,"ėdž":122,"ėmi":1028,"ėms":611,"atū":905,"ėja":1671,"ėl ":1089,"ėdė":151,"ėji":1633,"ėje":5476,"atš":113,"asų":302,"ėjo":1675,"avė":225,"avę":277,"ėju":616,"atž":107,"ėki":72,"auš":245,"atų":809,"ėkl":135,"ėkm":115,"alč":210,"alė":817,"at ":1471,"ėpa":71,"arg":513,"aiž":139,"are":1358,"ard":2315,"arc":907,"arb":6774,"ara":6684,"arp":3635,"aro":3645,"arn":1263,"ėgų":151,"arm":594,"arl":418,"ark":2565,"amą":414,"ėję":513,"arj":115,"ari":8416,"alį":375,"aru":2631,"arv":357,"Šie":85,"Šia":1607,"arr":83,"ars":1153,"ajū":107,"art":6116,"au ":2801,"asa":3470,"ary":2268,"ajų":380,"akš":168,"asi":8082,"ash":79,"ase":867,"asd":82,"aso":710,"asn":256,"asp":307,"ask":1323,"aną":115,"asm":1404,"ės ":35616,"asl":491,"ėnu":240,"ėno":201,"aos":78,"agū":453,"ajė":352,"ar ":4730,"ėni":505,"agų":187,"apa":1300,"ėne":311,"ape":335,"ėna":175,"apd":357,"aką":119,"api":6467,"apg":138,"apl":1752,"apk":227,"apo":1183,"apr":1671,"aps":2502,"apt":510,"apu":404,"apv":201,"apy":1793,"akė":76,"as ":68022,"Ši ":192,"aiš":684,"alą":244,"ėny":160,"ėto":359,"ava":4842,"apė":321,"amų":755,"auz":79,"ėty":82,"aut":4104,"Ško":96,"avo":3989,"ėta":441,"avi":8394,"amž":667,"ėti":999,"ave":644,"ay ":151,"Šeš":127,"avy":790,"anų":720,"avu":579,"arč":154,"arą":182,"anž":85,"anė":190,"akų":454,"ata":3198,"asu":274,"ast":4236,"ass":235,"anč":3200,"asy":280,"asv":107,"Šil":407,"atm":389,"atn":97,"atk":211,"atl":931,"ėra":344,"Šip":64,"atr":904,"Šio":192,"ato":2724,"Šim":69,"atp":71,"ėlė":390,"ate":2024,"atf":194,"Šis":287,"Šir":132,"ėri":244,"ati":4429,"atg":129,"Šiu":193,"ath":131,"alų":2328,"ėsn":115,"aub":219,"att":75,"ats":3002,"alū":263,"atv":1480,"atu":978,"ėst":385,"aty":1953,"aul":3204,"aum":246,"aun":1743,"auo":93,"aup":243,"ėmė":214,"aur":9854,"aus":15793,"ėjų":416,"aud":4737,"ėsi":184,"aug":7354,"ėse":1427,"auj":2881,"auk":5802,"ος":93,"ος ":93,"Vėl":110,"ς ":168,"ėči":63,"Rūd":99,"α ":85,"Sūd":82,"アア":188,"ий ":71,"jei":143,"jer":295,"jek":1582,"jen":63,"jet":69,"jev":166,"ji ":3840,"aža":639,"ažd":237,"aže":494,"aži":1113,"ažn":2141,"ažo":516,"ažu":326,"ažy":107,"jad":82,"izė":124,"jas":2376,"jav":90,"jau":1901,"jap":205,"jar":157,"jak":120,"jan":2657,"jam":5082,"jag":120,"jai":1573,"je ":35197,"ažį":104,"ažų":112,"jog":127,"jok":177,"joj":10950,"jon":4897,"jom":502,"jot":90,"jos":20733,"jor":274,"ск":151,"jis":383,"jim":2557,"jin":736,"jie":392,"ст":86,"jo ":6159,"itn":63,"itm":290,"itk":77,"ioč":215,"itr":489,"ito":5645,"itv":127,"inį":1158,"itu":2883,"itt":74,"ity":2401,"ilų":166,"iub":80,"iuc":232,"ilž":195,"iud":174,"iuj":1926,"iuk":566,"iui":902,"isk":1115,"iną":194,"ism":716,"isl":291,"iso":747,"isp":344,"iss":63,"ikū":653,"inč":492,"isu":1429,"ist":10402,"isv":554,"isy":283,"ikų":1702,"inė":20322,"inę":1356,"ita":3101,"ite":2150,"ith":115,"iti":4567,"ivo":225,"inų":1093,"ivy":313,"ivu":89,"irą":66,"inž":95,"ius":6964,"iur":510,"ium":511,"iul":202,"iuo":4507,"iun":199,"imų":1088,"iuz":146,"iut":1006,"iuv":85,"iva":5713,"ipė":458,"ix ":136,"ivi":1601,"ive":1142,"ipr":458,"ipo":522,"ipu":252,"ips":676,"ipt":461,"ipi":440,"iką":480,"igž":771,"ipl":309,"ikę":172,"is ":52601,"ikė":859,"ion":5845,"iop":388,"ior":133,"ios":8346,"igū":296,"iot":445,"iog":457,"iją":1411,"ioj":3109,"iok":271,"iol":667,"iom":700,"iję":103,"ipa":679,"ipe":332,"iov":183,"igų":264,"ioz":68,"ir ":24988,"iru":470,"irv":355,"irs":602,"ijū":154,"irt":3182,"iro":972,"irp":210,"irm":1827,"irn":109,"irk":393,"iri":3779,"imą":1476,"ikš":1311,"isi":2798,"ish":114,"ise":370,"isd":71,"isc":198,"isa":1339,"iu ":2871,"imė":266,"ijų":2456,"iry":540,"ire":616,"irg":245,"irb":684,"ira":1224,"ird":228,"it ":169,"ilė":771,"ilę":379,"ašų":113,"iuš":73,"dėt":870,"dės":969,"itų":1031,"dėj":1386,"dėm":219,"dėn":263,"dėl":1132,"dę ":83,"ivė":295,"isų":290,"ivų":75,"dęs":150,"ja ":13854,"itą":217,"ipų":79,"isė":1814,"isę":115,"isą":156,"irę":81,"irė":261,"irž":446,"irų":76,"izu":821,"itį":88,"irū":86,"izo":451,"izn":83,"izm":1490,"izi":1251,"irš":1690,"izg":80,"ize":116,"izd":791,"iza":1796,"itę":494,"itė":412,"dė ":795,"kaš":89,"kik":107,"kij":2691,"kim":1590,"kil":1486,"kia":4293,"kie":2272,"kiv":63,"kin":5088,"kio":2376,"kip":128,"kir":3853,"kis":920,"kit":2704,"kiu":661,"kaž":63,"km ":5939,"ki ":3521,"kdy":223,"kg ":77,"kea":110,"ked":124,"kei":560,"kel":3590,"ken":372,"kep":138,"kes":439,"ker":661,"ket":958,"ke ":844,"kci":1331,"kda":895,"kdo":201,"kra":4663,"kre":1055,"klė":428,"kt ":364,"kių":2205,"kiš":1820,"klą":309,"ksa":796,"kse":134,"kry":1180,"kmė":162,"ku ":438,"kro":983,"kru":318,"kri":3232,"kov":891,"km²":423,"kot":816,"kos":8611,"kor":653,"kop":332,"koo":136,"kon":3138,"kom":3814,"kol":1241,"kok":423,"koj":2216,"koh":148,"kod":1418,"ks ":271,"kmu":184,"kme":718,"kmi":91,"kny":405,"kni":155,"kią":80,"klu":945,"ko ":5671,"kly":141,"kle":472,"kla":7712,"klo":1474,"kli":2387,"bų ":1576,"jyb":83,"jus":1760,"jun":1716,"juo":1236,"jur":161,"juj":263,"jui":107,"jud":642,"būn":290,"būk":89,"būr":626,"būs":338,"būt":996,"eči":2678,"dį ":451,"ju ":392,"būd":1044,"kaz":236,"kav":389,"kat":1246,"kau":876,"kar":10769,"kas":6028,"kap":1244,"kan":2442,"kal":11413,"kam":1843,"kaj":116,"kak":332,"kai":11174,"kag":106,"kad":1450,"kac":439,"kab":221,"ka ":5403," Ga":1407,"bėt":104,"bės":4168,"bėn":81,"bėm":94," Ge":1126,"bėj":670," I ":269,"bėg":144,"bėc":110," Fo":747," Fu":218,"bę ":399," Fr":454," Fi":557," Fl":215," Ha":877," He":844," Gy":327," J ":93," Go":421," Gr":2243," Gu":448," Gv":487," Gi":589," Gl":237," Ig":304," Id":102," Ib":64,"guž":295," K ":135," Hy":110," Hu":227," Ho":536,"ha ":309," Hi":515," Ji":1102," Je":431," L ":82," Ja":1776," Iz":149," Dė":110," Iv":74," Ir":401," Is":623," It":425," Im":226," In":1579," Aš":111," Ik":112," Il":691,"ham":166," M ":181,"han":534,"hai":104," Ka":7194,"haj":70,"hal":255,"hau":98," Ke":839," Ki":1466,"har":550,"has":169,"hat":88," Jo":1429," Ju":1786," Bū":168,"hab":63,"had":70," La":2647," Le":2732," Li":7563," Kl":830," Kn":86," Ko":2662," Kr":1957," Kv":191," Ku":1464," Ky":76," Ma":4719," O ":103," Mi":1782," Dž":420," Me":1896," Dū":73,"he ":653," Lo":721," Ly":249," Lu":488," Ne":1796,"а ":158," P ":142," Na":1859," Ež":273," Ni":719," Mo":1991," My":150," Mu":530,"hel":177,"hei":83,"hev":142," A ":384,"het":95,"hes":72,"her":871,"heo":356,"hen":157,"hem":647,"hi ":89," B ":160," C ":148," Ap":776," Am":1804," An":2616," Ak":583," Al":1977," Ai":520," Aj":72," Ag":281," Af":804," Ac":198," Ad":550," Ab":408," Ba":3796," D ":97," Az":731," Av":213," Au":2682," At":910," As":629," Ar":1939," Be":1541,"hie":74,"hid":370," Bi":835,"hia":94,"hip":216,"hin":342," Bl":291,"him":102,"hil":142," Bo":1202,"hij":135," Br":1443," Bu":1122,"his":79,"hit":343,"hir":118," E ":80," Ca":760," Ce":856," Ci":314," Ch":1252," Cl":153," Cr":197," Co":818," Cu":130," Cy":71," Da":2178," Di":2044," De":898," Dr":663," Do":658," Dn":85," Dy":151," Dz":157," Du":604," Dv":187," Ed":144," G ":65," El":573," Ek":234," Ei":124," Eg":301," Et":188," Es":385,"hlo":65," Er":362," Ep":87," En":317," Em":142," Ex":72," Eu":1612," Ev":98," Fe":541,"ho ":213,"hma":111," Fa":424," H ":80,"gma":128,"go ":1512,"gme":171," Są":373,"glo":300,"gle":181,"gli":959,"gla":443," Wo":146," Wi":267," We":123," Wa":210,"й ":139," Vy":520," Rū":228,"gog":103," Zo":73," Ze":220,"gno":577," Zi":68," Tė":65,"gni":373," Za":514," Yr":183,"gne":245,"gna":449," Yo":68," Sė":67,"giš":189,"gpj":199," Vė":216,"о ":63,"gr ":135," Sū":95,"goj":625,"gom":579,"gol":244,"gon":466,"gos":3067,"gor":332,"got":295,"gov":127,"gu ":264," a ":1580,"gro":326," Už":349,"gry":429,"gru":2484,"gra":4017,"gių":416,"glė":83,"gri":3118,"gre":877," R ":90," Jė":115,"gty":102,"glų":187,"gto":154," Os":251," Or":560," Op":146," Po":1526,"gui":113," Pl":1099," Pi":2753,"gum":1158," Ph":102,"gul":484," Pe":1581,"gua":121,"gub":128," Pa":7618,"gue":64,"gst":140," Ny":218," Nr":561," Nu":899," No":770," Ol":265," Ok":221," On":138," Om":89," Ją":106,"gti":868," Od":117," Oc":104," Of":78," Ob":214,"gta":465," Lė":70," Ra":1919,"gvo":218," Ro":1436,"grą":89," Re":2283," Ri":769," Kė":213," S ":144,"guv":99,"gut":67,"gur":151," Pr":4036," Ps":108,"gus":1460," Jį":73," Pu":558,"gun":99,"guo":261," Iš":781,"gvi":98,"gva":685," Sy":113," Sv":277," Su":2443,"grį":216," St":1586," Kū":349," Ta":3980,"gsė":181," V ":128,"gyb":293,"gyd":154," Th":577," Ti":951," Te":1623," Tr":1262,"gyl":77,"gyn":197," To":1060," Ry":866," Jū":181," Ru":2112," Sa":3099," Jų":127,"е ":76," Mė":156," Sh":178," Si":1574," Sc":292," Se":1876," So":1791," Sp":549," Sr":63," Sk":1023," Sl":298," Sm":286," Sn":82,"grū":76," Uz":68," Va":2931," X ":111,"и ":92," Ve":1358," Vi":4303," Vl":71," Vo":1248," Tu":948,"gyv":4200," Tv":72," Ty":93,"gys":254,"bė ":1092,"gzi":495," Ug":88," Uk":462," Ul":82," Un":258," Uo":118," Up":172," Ur":330," Us":82," Mū":111," Ut":265," ja":1311,"iai":12600,"iak":2489,"iaj":841,"iam":6822," dė":1157,"ial":2617," iz":109,"ian":3835," ji":819,"iap":102,"ias":3810,"iar":592,"iau":21521," je":150,"iat":599,"iav":1428," im":762," in":5586," ik":3172," il":1403,"ic ":202," aš":299,"iab":113,"iac":1098," is":1471,"iad":246," it":186," ir":24988,"iag":1048,"ibl":199," ka":22789,"ibi":638,"ibo":741," m ":8180," kg":76,"ibr":360," ki":5307," ke":3545,"ibu":210," jo":2067,"id ":126,"iba":407," bū":2814,"ibe":354," ju":2553," gy":4128," bė":99," ha":363," he":768," gi":1781," gl":388," gr":5317," go":132,"ia ":6457," gu":260," gv":148," k ":116," id":308," ie":140," hi":591," ho":365," hu":138,"iet":17789,"iev":1199," ni":198," ež":2821,"iel":363,"iem":1768," ne":7202,"ien":11441,"iep":530," na":5031,"ier":804,"ies":9563,"ied":842,"ieg":235," my":64,"iek":3632,"iej":2087," mu":1417," mo":7083," mm":210,"ieb":372," ok":189," ol":297," ją":256," oj":106,"ifo":912," od":155," of":644," ob":1166,"ife":169," ny":106,"ifi":598," nu":12007," no":1368,"ifa":104," le":2689,"icr":108,"icu":77," li":4372,"ico":274,"ick":163," la":7451," kv":492," ku":12047,"ici":1931," kt":309,"ich":922," ky":193,"ice":275," kn":370,"ie ":6151," km":6372," kl":2310,"ica":672," kr":4858," ko":8974," me":9020,"idy":563," dž":138," eš":161,"idu":2683," mi":7229,"idv":63," ml":158," gė":342,"я ":102," o ":1212,"idr":494,"ido":937,"idm":185," ma":4864," lu":92,"idi":2189," ly":1967,"ide":3721,"ida":3570," lo":631," ag":258,"aša":470," ab":508,"iid":108," ac":88,"aše":79," ad":1540," am":912,"idą":63,"ašk":334," an":3700," ap":10764,"aši":756," ai":453,"ašo":756," ak":1627,"ašl":96,"ašm":98,"iim":317," al":1456," av":222," au":6951," ar":9760," at":6438,"ašu":283,"ašt":1920," as":2056," d ":1323,"ašy":781," ba":4469,"il ":133,"idė":1297,"ija":13519," bi":992,"ije":95," be":6829,"iaž":208,"iji":627," bo":424," bl":292,"ijo":29798,"ieč":1593," by":148,"ibū":258," bu":4118,"iju":808," br":918," ca":122,"ibų":249," e ":82,"im ":74,"ika":5325,"ige":301,"iga":1803,"ii ":82,"igl":111,"igm":118,"igh":167,"igi":1406,"igu":322,"igt":307,"igr":206,"igo":701,"ign":206,"dą ":495,"ibė":80,"igy":112,"ik ":1008,"imo":10288," er":623,"imn":310," et":431," es":1712," en":778,"ims":264," em":226," ep":239,"imp":910,"idž":3127," ei":683,"imf":152,"ime":1400," el":1508,"imd":91," ek":915," ef":150,"imi":2582,"ieš":2115," eg":645," fe":604,"ip ":3767,"inc":1582,"ind":4737,"ina":9499," fa":728,"imt":970,"imu":2073," eu":161," ev":173,"imy":291," fu":1594,"inm":71,"ino":4836," fr":209,"ašč":176," fo":1905,"int":7918,"ins":1597,"inf":613," fl":132,"ine":2681,"iež":504,"ing":5394," fi":1973,"ini":42857,"ink":9182," ge":4705," ga":6719,"ioc":68,"iod":332,"inu":851,"inv":260,"iny":2814,"ašė":95,"iko":8237," cm":466,"ikm":102,"ikl":4355," co":131,"iki":8010," ce":2733,"ike":332," ch":821," ci":727,"ila":1055,"in ":677," da":11917,"iky":490," cu":64,"ikt":1187,"iku":935,"ikr":1885,"iks":1325," do":606,"ilp":157,"ilo":1045,"ill":367," dr":1386,"ilk":807,"iln":1654,"ilm":942,"ilg":1557,"ilj":190," de":4787,"ili":7113,"ild":935," di":7861,"ile":510,"ima":10774,"imb":476," g ":911,"idų":320,"igė":179,"io ":24537," dv":1810,"ily":138," du":1631,"ils":105,"ilt":617,"ilu":734," dy":1060,"ilv":228," sū":347," zo":229," rū":2140,"hol":535," tė":174,"hom":130,"hon":117,"ко":78," tę":151,"hos":152,"hot":89,"hov":154,"hop":73,"hor":425," yp":378,"ка":91,"ки":93," yr":9482," sė":250,"hni":331,"hno":260," są":2199," pū":86,"ин":66," ww":64,"ий":86," ož":66,"ра":115,"hua":109,"htt":75," už":2934,"hst":134,"ол":71,"ов":122,"hry":109,"но":71,"hro":216,"ни":79,"hri":103," tū":466,"ht ":128,"на":78,"hra":72," vė":1295," ru":1285," jū":1646," ry":5467," mė":866," jų":977," sa":11017," sf":110," se":4482," sc":162," si":4889," sn":168," sm":866," sl":1116," sk":5183," sr":2054,"hyl":73," sp":3493," so":1538,"ве":64," lė":404," t ":238," ra":8074," re":7211," ri":2152," mą":77," ro":1078," pv":390," pu":2293," jį":257," pr":18161," ps":398," s ":108," px":100," lą":374," iš":11701," os":113,"hum":232," ov":65,"hun":68," op":537," or":3230," jė":239,"ан":71,"ал":63," pe":3678," pa":29839,"ар":72," pl":4167," po":4807," pi":10334," pj":71," vy":3074," y ":87," x ":92," va":20321," ve":4247," pė":140," vo":600,"cėl":111," vu":113," vi":14126,"ер":72,"ен":65," ty":567," tv":701," tu":2991," mū":318," ur":214," uo":741," up":3500," un":1322," ul":71," ug":910," ta":10594," nė":349,"hyt":188," st":5898," kū":1165," sv":1124," su":14551," tr":3585," lū":96," to":1975," th":302," ti":5451," te":9733,"fes":435,"fer":701,"fed":364,"fen":78,"fek":526,"fel":82," Ča":214," Či":814," Če":428,"fga":67,"faz":69,"fas":187,"far":115,"žų ":324,"fan":250,"fak":286,"fal":119,"fai":102,"ezė":113,"fe ":71,"etų":4160,"esų":111,"evė":459,"evų":124,"fa ":109,"esė":92,"erį":83,"etą":266,"erė":268,"erč":118," če":566,"esč":286," či":215,"epš":367,"esą":106,"ezu":126,"erų":398,"erž":348,"etė":1071,"eza":64,"esį":127,"ezo":319,"eze":135,"erš":197,"ezi":795,"eta":5454,"enę":98,"ete":1059,"elš":173,"eti":4554,"etn":281,"esp":1704,"esn":1467,"eso":815,"est":4994,"esu":549,"enč":626,"ess":157,"esy":202,"enė":1439,"ekų":115,"eud":63,"euk":102,"eto":2887,"etr":2354,"ett":92,"enį":206,"etu":10440,"etv":1625,"ety":134,"ew ":79,"eve":542,"eva":902,"evo":715,"evi":699,"eut":158,"eur":366,"eus":130,"epė":63,"emų":168,"erą":156,"evr":154,"ey ":158,"evy":281,"enų":1350,"epe":177,"epc":101,"epi":513,"eph":64,"er ":2219,"epa":1258,"eot":67,"eor":538,"eom":87,"eol":734,"eop":149,"eon":218,"eiš":1009,"ekė":269,"es ":8000,"ekę":92,"ept":436,"eps":96,"epu":165,"epo":412,"epr":642,"erk":587,"erl":418,"eri":10880,"emą":219,"erg":1284,"ere":1184,"erf":77,"erc":545,"erd":659,"era":5900,"erb":981," ėj":280,"elę":124,"elė":2221,"et ":1493,"ekį":64,"esj":92,"eną":643,"esk":317,"esm":259,"ekš":67,"esi":2143,"ese":290,"esa":2125,"ery":234,"emė":1363,"ejų":738,"erv":903,"elį":206,"eru":488,"err":173,"ert":1937,"ers":2707,"ern":1320,"erm":1142,"erp":330,"ero":2197,"eki":1989,"ekl":411,"ekm":65,"eko":1042,"ekr":192,"eks":1896,"ekt":4207,"eku":345,"ekv":315,"eky":288,"en ":710,"elb":215,"ela":746,"eld":365,"elf":88,"ele":3424,"eli":8265,"elj":72,"elg":477,"elm":261,"eln":456,"elk":671,"ell":316,"elo":518,"elu":87,"elv":84,"els":353,"elt":670,"ely":474,"eo ":170,"egė":73,"edų":106,"emb":301,"ema":2008,"edž":1549,"eme":1016,"emd":90,"emo":2200,"emi":2274,"emt":191,"emu":703,"emp":946,"ems":853,"emy":348,"ene":2754,"eng":1816,"enb":111,"ena":6485,"end":4440,"enc":1042,"eno":4885,"enm":89,"enk":5159,"enl":92,"eni":6797,"enu":670,"env":1006,"ens":1444,"ent":11928,"enr":107,"eič":280,"enz":118,"eny":1624,"eog":245,"eod":89,"egl":182,"ego":396,"ege":449,"egi":3283,"egz":550,"ebė":292,"egr":181,"egu":769,"egy":91,"ek ":622,"eic":241,"eip":170,"eis":5132,"eir":124,"eim":4571,"eil":568,"ein":1861,"eik":5224,"eid":1499,"eig":1358,"eja":503,"edė":140,"el ":389,"eiz":76,"eit":977,"eiv":749,"ejo":153,"eji":683,"eje":65,"eke":125,"ekc":170,"eka":2352,"em ":150,"ejy":83,"eju":497," į ":8903,"gl ":220,"giu":292,"git":144,"gis":1794,"gir":399," Į ":86,"gil":204,"gim":1321,"gij":3190,"gik":123,"gip":251,"gin":4262,"gio":2880,"gie":143,"gia":3193,"ght":134,"bą ":394,"gi ":457,"gen":3944,"geo":416,"get":168,"ger":1023,"ges":352,"geb":197,"gei":69,"geg":248,"gem":66,"gel":1714,"gdo":74,"gdy":714,"gda":73,"ge ":442,"gab":134,"gac":108,"gad":87,"gai":1374,"gas":1974,"gar":1729,"gau":1337,"gat":335,"gav":268,"gam":2232,"gal":9014,"gan":3227,"ga ":2878," įv":1493," įt":810," įs":1931," įr":895," įp":146," Įs":221," įd":139," įe":113," įg":452," įk":1033," įl":469," įm":645,"fys":93," Įe":85," Įk":141,"fut":1079," įž":65,"fun":535,"fto":75,"ft ":191,"ača":64,"fra":263,"ačk":85,"ači":1928,"fri":745,"fro":133,"for":3388,"fos":117,"fot":172,"fon":490,"fol":145,"ač ":218,"fla":74,"fli":110,"fo ":188,"fic":473,"fig":118,"fij":466,"fil":952,"fik":605,"fin":1043,"fir":67,"fit":88,"fiz":526,"da ":3356,"de ":955,"dac":195,"dad":96,"dab":1199,"dak":155,"dal":5226,"dai":1928,"dag":146,"dae":1242,"dat":189,"das":3048,"dar":5510,"dan":2672,"dam":1100,"dav":1026,"dau":2555,"cul":112,"cto":80,"cti":127,"cta":124,"cus":68,"cur":66,"cko":72,"co ":242,"con":95,"col":102,"com":76,"cor":163,"cos":201,"cop":83,"ciš":133,"cro":177,"cea":1680,"ch ":208,"cer":417,"ces":658,"cet":70,"cen":3061,"cep":179,"cel":113,"ceg":78,"cha":955,"chu":89,"cia":2358,"ck ":279,"cie":95,"cid":198,"che":1211,"chl":85,"chi":1035,"cho":966,"chm":91,"chn":602,"chs":167,"cht":194,"chr":188,"civ":350,"cij":9068,"cik":433,"cil":163,"cif":247,"cis":123,"cit":220,"cin":3236,"cio":1497,"cip":355,"cm ":459,"ed ":292,"eba":313,"ebe":309,"ebi":367,"ebo":120,"ebr":205,"ebu":212,"eag":73,"eae":1640,"eak":217,"ean":215,"eal":365,"ear":134,"eap":89,"eat":373,"eau":64,"ea ":188,"efi":76,"efo":293,"efa":70,"efe":515,"ei ":3954,"ega":737,"eed":76,"edi":2343,"ede":757,"eda":2475,"edy":257,"edu":236,"edo":437,"edr":226,"eck":65,"ech":882,"eci":846,"ece":95,"eca":89,"ect":147,"eco":118,"dyg":104,"dyk":422,"dym":1573,"dyn":853,"dys":311,"dyt":634,"dyd":692,"dyb":2813,"drė":138,"drą":94,"dvy":133,"dvi":1481,"dve":133,"dvo":211,"duv":159,"dur":2035,"dut":428,"dus":534,"dva":622,"dvė":188,"dzi":233,"dor":1140,"don":1347,"dom":1366,"dol":160,"dok":455,"dow":94,"dov":1045,"dot":521,"dos":2310,"ds ":126,"diš":216,"deš":1558,"dmi":1463,"dni":66,"dob":88,"doe":78,"doj":2119,"dum":210,"duo":2102,"duj":255,"dui":167,"dul":493,"duk":700,"dug":90,"dub":205,"dua":83,"dri":926,"dra":1708,"dre":173,"dry":71,"du ":892,"dro":2362,"dru":787,"dge":113,"dic":606,"did":3151,"dia":454,"der":1524,"des":572,"det":124,"dev":201,"deb":268,"dea":123,"ded":1735,"deg":595,"dei":369,"del":1942,"dek":567,"den":3068,"dem":709,"dep":764,"deo":162,"di ":260,"dme":216,"do ":2940,"div":255,"diu":103,"diz":157,"dim":1824,"din":11534,"dio":390,"dip":85,"dir":773,"dis":1529,"dit":234,"die":4136,"dif":150,"dij":4732,"dik":613,"dil":259,"dka":165,"daž":1194,"ižy":171,"rbė":133,"rgu":182,"mą ":3290,"rga":2638,"iža":147,"ri ":2109,"rgl":72,"iži":186,"rgi":812,"rbą":117,"rge":1017,"rgo":391,"ret":1342,"res":1263,"rev":180,"rez":564,"rfi":80,"rfo":87,"rač":122,"rdu":515,"rds":64,"rdv":219,"reb":105,"rea":608,"ree":109,"ref":432,"rec":183,"red":291,"rei":3237,"lėš":194,"rej":92,"reg":2937,"rem":808,"ren":2189,"rek":801,"rel":806,"rer":127,"reo":63,"rep":489,"rda":868,"rcu":85,"rdo":569,"rdi":1074,"rde":192,"re ":1146,"rby":63,"rbt":207,"rbu":464,"rco":114,"rci":353,"rch":1033,"rce":143,"raz":577,"rd ":418,"rao":69,"rap":372,"rar":206,"ras":10157,"rat":2588,"rau":2880,"rav":918,"rbi":1182,"rbl":168,"rbo":606,"rba":5655,"rbe":227,"raj":5239,"rai":3447,"rah":84,"rag":1079,"ran":8258,"ram":2837,"ral":3029,"rak":1364,"rab":639,"raf":875,"rae":172,"rad":3322,"rac":2843,"rpt":814,"rpu":483,"rpr":64,"rps":98,"rpo":225,"rkė":212,"rs ":981,"rpe":180,"rpa":257,"rką":115,"rpi":679,"rgž":70,"ror":154,"ros":5272,"rot":753,"rom":1205,"ron":1539,"rop":2336,"roz":106,"rgų":68,"rou":123,"rov":2454,"row":67,"rob":326,"roa":194,"rod":1480,"roc":897,"roj":2662,"roi":152,"rol":1077,"rok":757,"rof":604,"roe":102,"rog":1825,"rno":408,"rič":321,"rnu":106,"rny":471,"rp ":1398,"rna":1385,"rež":195,"rne":498,"rią":492,"rni":813,"rmo":1239,"rmu":712,"rdų":72,"ro ":7225,"rgė":150,"rma":2602,"rme":740,"rdž":190,"rmi":1316,"rly":64,"rlo":116,"rli":270,"rld":79,"rle":65,"rla":454,"rn ":72,"rky":100,"rkv":63,"rku":184,"rkt":264,"rks":74,"rko":615,"rki":550,"rkl":319,"rke":407,"rka":1300,"rbų":71,"mąs":197,"reč":569,"mąj":193,"raž":304,"rje":132,"riz":580,"rix":85,"rdė":104,"rip":333,"rio":6643,"rit":7580,"ris":4114,"riv":457,"riu":3503,"rig":767,"rij":7031,"rdą":70,"raš":3717,"rii":272,"ril":196,"rik":6018,"rin":15816,"rim":1853,"ria":7346,"rib":1061,"ric":629,"rid":769,"rie":6574,"rif":330,"rk ":142,"rož":108,"jūč":213,"rsė":70,"jų ":7448,"ryb":1588,"ryd":120,"rui":138,"rug":582,"rud":269,"rur":65,"rup":2436,"ruo":2554,"run":372,"rum":1117,"ruk":710,"ruz":216,"ruv":66,"rus":3390,"rut":405,"rva":500,"rmų":317,"rvi":416,"rve":467,"rvo":162,"rnų":71,"rvu":153,"ry ":248,"rsk":222,"rsl":371,"rkš":158,"rsi":1214,"rso":508,"rsm":221,"jūn":185,"rsa":416,"rse":236,"rkų":131,"rta":4621,"rst":803,"jūr":1765,"rsu":138,"rtm":86,"rtn":140,"rto":2230,"rte":485,"rth":95,"rti":4648,"rub":105,"rtr":75,"rtu":1225,"rty":412,"riš":698,"rt ":258,"rių":3254,"mči":71,"rmą":324,"rre":91,"riž":94,"rra":105,"lį ":872,"ru ":632,"rmė":253,"sac":151,"sad":119,"sag":124,"sai":543,"saj":93,"sak":1089,"sal":3421,"sam":580,"ryš":526,"sap":78,"san":3811,"sau":3166,"sat":201,"sas":1914,"sar":840,"sav":6083,"sa ":609,"ryč":1308,"rvų":140,"mėg":132,"rvė":96,"rsų":124,"mę ":264,"rtų":277,"mėt":142,"mės":1624,"mėl":387,"mėm":110,"mėj":427,"mėn":862,"rtė":217,"mė ":740,"rys":1185,"ryt":5807,"ryo":78,"ryp":869,"ryk":138,"ryl":68,"rym":161,"ryn":188,"ryj":340,"rtą":376,"rtį":75,"ną ":1490,"shi":127,"si ":5878," 가":71,"siv":352,"sjo":73,"nąj":98,"sie":2665,"sid":3080,"sic":542,"sib":166,"sia":6343,"kšt":3974,"sit":1385,"siu":976,"sir":1172,"sis":6798,"sip":235,"sin":3201,"kšn":209,"sio":4893,"kšo":166,"kšl":78,"sil":885,"kšm":294,"sim":839,"sij":3056,"sik":1642,"sii":91,"sif":333,"sig":288,"sda":106,"sdi":133,"se ":12437,"ryž":454,"sce":120,"sci":144,"sch":242,"ser":955,"ses":253,"set":159,"seu":67,"sez":169,"sh ":92,"sfe":280,"sfo":80,"sei":720,"sed":68,"sep":203,"sen":3202,"sem":199,"sel":216,"sek":790,"spu":1602,"skė":68,"spy":159,"spo":957,"spr":582,"spe":1156,"spi":670,"sjė":92,"spa":2100,"sot":100,"sov":81,"sol":244,"som":660,"son":563,"sop":70,"sor":222,"sos":1164,"sod":242,"sof":371,"soj":156,"soc":1068,"su ":4472,"smė":297,"sru":176,"sro":178,"sri":1777,"nči":4255,"siž":207,"sra":172,"sių":1944,"slė":668,"st ":506,"siū":210,"siš":352,"ss ":140,"sli":978,"slo":811,"slu":296,"sky":900,"sla":1643,"sle":271,"ski":4050,"skl":344,"sko":1593,"skr":2283,"sku":545,"skv":165,"ska":2455,"ske":580,"kšč":2245,"sno":220,"sny":134,"snu":80,"sna":277,"sp ":78,"sni":2039,"sią":193,"sne":141,"smo":495,"smu":706,"so ":3418,"sma":829,"smi":538,"sme":1637,"syb":334,"kų ":5583,"syn":63,"syt":71,"sys":136,"syk":221,"stą":248,"stę":193,"syv":319,"stė":1387,"nė ":11072,"kūg":75,"sse":159,"ssa":96,"kūn":731,"sso":72,"ssi":140,"kūs":84,"kūr":1568,"snė":370,"skų":182,"ste":4835,"sta":11594,"sto":6918,"sti":7618,"snį":163,"stu":1797,"str":5446,"sua":240,"sty":3413,"slų":200,"sud":2168,"sue":135,"sub":489,"sui":78,"suf":148,"sug":205,"sul":523,"sum":815,"suj":169,"suk":1852,"sup":534,"sun":407,"suo":1175,"sut":1183,"sus":2952,"sur":378,"suv":210,"spė":63,"smų":124,"sva":1159,"sve":370,"svi":243,"svo":286,"svu":88,"svy":123,"tai":9707,"taj":152,"tak":2532,"tal":2568,"tag":213,"tab":401,"tac":699,"tad":469,"tav":1620,"tau":2680,"tat":2848,"tas":15287,"tar":6407,"tap":484,"tan":2917,"tam":3320,"te ":2806,"tbo":1115,"nę ":1530,"svė":126,"가":170,"stų":690,"nėm":645,"nėn":104,"nėl":246,"nėj":3435,"stū":94,"nės":9359,"nėt":145,"nėr":303,"ta ":7677,"suž":77,"nęs":68,"ovų":204,"jęs":545," št":72," šv":1206," šu":230," ši":9378,"pa ":504," šl":369," šo":352," ša":3536," še":4400," Šv":1543," Šu":220," Št":147," Šr":75," Šo":70," Šl":99," Šk":111,"otų":425," Ši":3390," Še":340,"jės":91," Ša":886,"jėg":559,"ovę":68,"ję ":118,"ovė":1571,"pdo":296,"pci":139,"pe ":192,"par":5103,"pat":3653,"pas":6180,"pav":2954,"pau":1209,"pac":165,"pad":1171,"paa":84,"pab":319,"pag":4995,"pak":1745,"pal":2163,"pai":635,"paj":498,"pap":2360,"pam":953,"pan":2096,"ozė":194,"pha":125,"ką ":1267,"pho":90,"phi":136,"gžd":845,"pač":638,"pec":747,"ped":487,"pen":679,"per":4439,"pet":193,"pes":135,"pei":362,"pel":1215,"pek":276,"pla":1821,"pli":2030,"ple":626,"plo":1805,"ply":260,"plu":196,"pkr":163,"paž":694,"phy":268,"pib":499,"pia":510,"pid":180,"pie":9064,"pig":92,"paš":357,"pij":468,"pik":263,"pil":4079,"pim":702,"pin":2920,"pio":1030,"pir":1825,"pis":533,"pit":515,"piu":812,"poz":347,"pr ":158,"por":1711,"pop":634,"pov":195,"pot":290,"pos":1432,"poj":445,"pog":134,"pom":76,"pon":986,"pok":315,"pol":1919,"pob":138,"poe":104,"poc":74,"pod":139,"po ":1517,"psu":123,"pst":87,"pta":1134,"pse":76,"psi":725,"psn":657,"psk":1854,"ptu":124,"pty":174,"pua":68,"pub":1670,"pte":200,"pti":1385,"pto":309,"plū":91,"pra":5043,"pių":716,"plė":383,"piš":241,"pjū":231,"pru":94,"psa":387,"pu ":180,"jį ":548,"kči":82,"pri":9679,"pre":1608,"pro":5447,"poš":212,"pož":275,"pyg":410,"pur":225,"pus":2091,"put":192,"pun":96,"puo":365,"pup":123,"puk":97,"pul":563,"px ":100,"pva":210,"pvz":390,"kę ":172,"kėj":853,"kėn":99,"kėl":172,"kės":2221,"ptū":85,"kėt":269,"puš":167,"pyn":242,"pyj":119,"pyl":1154,"pyk":93,"pyv":70,"pyr":87,"pyt":97,"kė ":1457,"prū":156,"puč":92,"kęs":421,"lą ":721,"iš ":6273,"ląs":439,"išd":131,"iše":155,"išg":193,"iša":318,"išm":207,"išl":1019,"išo":261,"išn":213,"iši":468,"išk":9640,"išt":673,"išv":411,"išp":207,"išs":1093,"išr":686," Ži":399," Že":954," Ža":482," Žy":115," Žu":120," Žv":178," Žm":91," ži":2342," žm":1523," ža":1939," že":2052," žy":592," žv":1024," žo":907," žu":959,"iū ":112," ųj":70,"kį ":292,"lči":255,"iūt":153,"iūr":499,"iūn":1343,"iūl":184,"que":92,"qui":80," šį":69,"ių ":35485,"lė ":2188," ūk":511,"iųj":519,"lėg":91,"lėj":539,"lėd":122,"lę ":354,"lėv":65,"lėt":585,"lės":3311,"lėn":507,"lėm":235,"lėl":106,"lėk":280,"ra ":13206,"lęs":405,"ngo":2538,"ngi":2530,"eži":1071,"ngl":1130,"ngv":440,"ngu":1035,"ežu":89,"ngr":450,"ežt":146,"ngt":1224,"ngs":123,"ni ":526,"eže":2899,"nge":541,"eža":160,"nga":2804,"ežd":87,"nha":74,"ią ":1321,"neg":409,"nei":1033,"nel":523,"nek":262,"nen":435,"nem":377,"nep":854,"neo":246,"ner":1909,"net":3370,"nes":2353,"nev":652,"neu":225,"ndy":107,"ng ":636,"nea":242,"neb":233,"nec":126,"ned":530,"nef":72,"nfo":543,"nfl":92,"nfr":63,"nez":420,"nfe":266,"nco":109,"nci":2521,"nce":603,"nch":181,"ne ":5137,"nbu":79,"ndu":1568,"ndr":3362,"nds":72,"ndo":1993,"ndi":4736,"nde":2551,"nda":1898,"nak":285,"nal":3006,"nam":4510,"nan":2181,"nap":314,"nar":1177,"nac":1442,"nad":530,"nae":88,"naf":82,"nag":711,"nai":2792,"naj":138,"nc ":104,"nab":100,"nbe":88,"nd ":453,"nav":974,"nau":2982,"nat":1500,"nas":9068,"naz":237,"na ":6125,"muš":82,"myr":132,"myn":393,"myl":81,"가 ":71,"mzd":138,"myk":132,"myb":757,"fų ":79,"nyb":586,"ntą":186,"nyj":775,"nyi":66,"nyg":399,"ny ":199,"nvi":1009,"nux":72,"nve":361,"nva":85,"nuk":354,"nul":255,"num":449,"nun":66,"nug":200,"nui":231,"nus":1958,"nut":401,"nuv":180,"nuo":11505,"nur":161,"nty":1460,"ntv":111,"nto":3213,"ntu":1006,"nts":156,"ntr":4642,"nti":13882,"nth":149,"ntg":70,"nta":6698,"nte":3240,"nsu":225,"nkų":1281,"nsp":335,"nso":235,"nst":1864,"nse":182,"nkš":139,"nsi":456,"nsl":149,"nsk":533,"nsa":277,"nu ":681,"nro":72,"iči":2192,"nri":70,"nra":165,"nių":11248,"nt ":3145,"niū":1065,"niš":1026,"nką":172,"ns ":1187,"nkė":2543,"nod":224,"nog":120,"nok":84,"nol":827,"noi":70,"noj":1649,"nop":168,"nom":2167,"non":256,"not":507,"nos":4995,"nor":1222,"nov":1035,"ngų":481,"noz":151,"nne":81,"než":127,"nni":71,"nme":211,"nma":125,"neš":331,"ndž":626,"ngą":233,"nla":91,"ndų":534,"ngė":262,"no ":11621,"ndū":86,"nke":587,"nkl":1886,"nki":5054,"nkc":507,"nka":2908,"nku":626,"neį":137,"nky":223,"nko":1384,"nks":891,"ncū":939,"nkt":1190,"nkr":206,"iąj":181,"naž":94,"nja":76,"ndė":404,"njo":112,"ndą":97,"nij":5825,"naš":624,"nig":783,"nif":163,"nie":779,"nid":254,"nic":409,"nia":7994,"nk ":388,"niz":2108,"niu":3647,"niv":780,"nis":15940,"nit":387,"nir":72,"nio":7526,"nip":69,"nim":4778,"nin":8141,"nik":1554,"nil":147,"ogs":78,"ogr":2210,"ogu":324,"ogi":3408,"ogl":119,"ogo":286,"ogn":64,"oga":839,"oge":235,"oho":167,"oha":70,"ohe":76,"obė":124,"oj ":68,"ją ":2223,"gšt":167,"odą":88,"oid":309,"ok ":122,"oju":555,"obū":144,"ojo":4241,"oji":3887,"oje":21105,"oja":6655,"odė":378,"ol ":243,"oce":779,"och":261,"oci":1478,"ock":185,"obs":81,"obu":232,"odg":104,"ode":1122,"odk":163,"odi":1127,"odo":1175,"odr":68,"of ":299,"oda":2306,"oel":84,"oet":66,"oeu":79,"ody":417,"odu":589,"og ":261,"ofi":842,"oft":141,"ofo":158,"ofe":379,"ofa":95,"oa ":103,"nyč":782,"oac":66,"oba":242,"od ":72,"oar":69,"oat":135,"obo":229,"obl":199,"obj":1006,"obi":1248,"obe":189,"nyn":1169,"nyk":427,"nyr":226,"nyt":99,"nys":2919,"ntė":672,"nzi":126,"nzo":87,"ntį":195,"nsų":145,"ntū":104,"ntų":1010,"gų ":1699,"osė":119,"orė":417,"ows":92,"orą":78,"orų":130,"orū":115,"ozo":156,"ozi":309,"jė ":190,"otė":267,"oza":345,"otą":235,"olų":162,"oty":482,"otu":369,"ow ":66,"oti":3180,"ote":1275,"ott":72,"otr":264,"oto":2263,"otn":65,"okų":103,"ost":1854,"osu":63,"osv":137,"ota":3370,"onė":1445,"osi":4557,"okš":697,"osk":117,"oną":313,"ose":10057,"osf":255,"osp":155,"oss":88,"gūr":160,"osm":327,"osl":114,"oso":811,"osn":165,"gūn":307,"ovy":172,"onų":1034,"ovi":2936,"onš":70,"ovo":1368,"ovu":118,"ovs":154,"omų":264,"opė":75,"ox ":72,"ova":2312,"ove":473,"oun":122,"oup":69,"ous":97,"our":160,"out":83,"opo":1809,"opi":1192,"opl":208,"ope":871,"oph":333,"opa":456,"os ":71850,"okė":208,"opu":410,"opr":78,"opt":255,"ops":178,"ool":67,"ood":82,"or ":364,"ojė":85,"ogų":154,"oor":180,"ork":297,"orl":173,"orm":3710,"orn":376,"oro":1427,"orp":329,"orc":113,"ord":1011,"ore":636,"orf":147,"org":2280,"ori":8425,"omą":125,"omė":233,"ojų":757,"ou ":101,"osa":489,"gūb":364,"ort":1674,"ors":672,"orv":193,"oru":254,"ory":137,"olą":105,"m² ":432,"ot ":142,"olė":893,"orb":205,"ora":1549,"olę":89,"okį":63,"ola":1063,"old":223,"on ":1119,"oli":7420,"oll":172,"olk":163,"olf":87,"ole":1072,"olg":102,"olt":96,"olm":81,"oln":64,"olo":5241,"oly":304,"odų":160,"ogė":746,"olu":523,"oka":1176,"om ":163,"oki":2742,"oke":340,"okr":516,"oks":1989,"oko":845,"okl":512,"oky":4231,"okt":115,"oku":1484,"ona":5705,"ond":604,"onc":437,"onf":314,"one":3050,"ong":677,"oni":5683,"onk":450,"ono":5947,"ons":941,"ont":1350,"onu":771,"onv":131,"ony":229,"onz":95,"oma":4058,"ome":2921,"omb":322,"omi":4219,"odž":830,"omp":1868,"omo":2822,"omu":462,"oms":893,"omy":305,"op ":190,"la ":3891,"kyč":84,"ksų":107,"kvė":157,"ktų":516,"ktū":882,"le ":987,"lci":77,"lde":301,"lda":470,"ldo":1438,"ldi":380,"ldu":86,"lab":885,"lac":354,"lad":233,"lag":301,"laj":360,"lai":7095,"lal":283,"lak":544,"lan":3678,"lam":672,"lap":1033,"lar":290,"lat":1022,"las":4962,"lau":4662,"lav":1958,"lay":83,"laz":248,"lba":2195,"ld ":140,"lbe":120,"kyš":114,"lbi":260,"lbo":1044,"lbu":643,"kvi":414,"kve":103,"kva":547,"kuv":68,"kut":500,"kus":1457,"kur":11466,"kup":293,"kuo":1084,"kun":573,"kum":831,"kul":3442,"krą":130,"kvo":154,"kta":2002,"kte":286,"cūz":939,"ksp":237,"ksu":217,"kst":1638,"ksi":1031,"kso":492,"ksn":481,"ksm":470,"ksl":1822,"kub":70,"kui":142,"kty":770,"klų":780,"ktr":1113,"ktu":872,"kti":2012,"kto":1582,"kyt":491,"kyr":514,"kys":407,"ktė":87,"krū":445,"kuč":63,"krų":140,"cų ":73,"kyb":405,"kyd":213,"kyk":2742,"kyj":122,"ktą":149,"kym":1532,"kyl":177,"ksč":304,"lpo":78,"lpn":97,"lpi":224,"lkė":206,"ls ":120,"lpt":73,"lok":724,"lon":920,"lom":534,"lop":556,"lor":335,"lod":121,"loc":79,"log":3426,"loj":732,"loi":69,"lpa":143,"los":2610,"lot":1612,"lou":74,"lov":512,"lno":410,"lią":195,"lni":4011,"lež":588,"lne":64,"lob":276,"lny":212,"lnu":597,"lmo":217,"lmi":148,"lme":109,"ldž":460,"lma":561,"lna":1027,"lmu":116,"lti":1386,"lto":827,"ltr":83,"loč":162,"ltu":191,"lty":140,"lub":724,"lkš":220,"lsi":157,"lsk":87,"lso":116,"dūr":138,"lkū":97,"lst":3935,"lsv":208,"lnė":167,"lkų":772,"lta":1286,"lte":434,"lu ":327,"lmė":408,"lsa":135,"liš":786,"liū":227,"lių":4116,"lt ":76,"lbė":158,"gą ":583,"lgu":66,"lgy":105,"lgo":332,"lge":222,"lbą":82,"lgi":1731,"li ":2175,"lga":584,"lač":341,"lfi":69,"lfa":89,"lez":63,"leu":86,"lev":356,"les":733,"let":811,"ler":662,"leo":285,"lep":84,"lem":868,"len":3529,"lek":1990,"lel":445,"lei":2618,"leg":368,"lef":117,"led":360,"lec":77,"ldy":3239,"lls":89,"llu":67,"lo ":4298,"lla":282,"lle":203,"lli":227,"llo":104,"lko":1340,"lku":68,"lks":80,"ln ":170,"lka":889,"lke":67,"lki":640,"lkl":70,"lbų":491,"lje":178,"ll ":250,"lja":125,"lit":2868,"lis":6123,"lip":320,"lio":6518,"lin":10945,"lim":1681,"ldė":95,"liz":1115,"liv":318,"liu":3368,"lic":390,"lid":423,"lia":9617,"lib":160,"lik":3451,"laš":117,"lij":2779,"lig":956,"lie":4974,"lif":303,"ma ":8966,"gęs":112,"mb ":104,"lvų":129,"mac":657,"mai":3495,"maj":238,"mak":310,"mad":111,"mag":648,"mar":995,"mas":16980,"mal":1077,"mam":360,"man":2648,"maz":236,"mav":409,"mau":158,"mat":2440,"mba":403,"mbl":299,"mbi":672,"mbe":160,"mbr":361,"mbo":656,"me ":4745,"mbu":328,"mdo":70,"mdi":69,"med":2275,"meg":289,"mec":166,"met":7146,"mes":916,"mer":2496,"mem":105,"mel":455,"men":7636,"mei":219,"mez":71,"mfo":93,"lmų":71,"lpė":80,"lva":657,"lve":135,"lvi":553,"luk":80,"lui":136,"lup":118,"luo":731,"lun":291,"lum":619,"lut":328,"lus":1933,"lur":73,"luv":66,"lnų":522,"ly ":90,"lvo":651,"lyb":79,"lyd":329,"dų ":1583,"ltą":68,"lyj":2459,"lyk":200,"lyg":1880,"lsč":1443,"gė ":262,"lyv":644,"lyp":129,"lym":88,"lyn":910,"lys":1218,"lyt":728,"gėl":263,"gėj":295,"gėg":66,"gę ":71,"lvė":163,"ltų":271,"gėr":187,"ltū":1732,"gės":1188,"mpi":2031,"mph":67,"mpe":889,"mpr":78,"mpo":545,"mpl":367,"mpu":163,"ms ":4671,"mog":880,"moc":111,"mob":923,"mod":511,"mon":3483,"mok":6158,"moj":1912,"mom":231,"mol":407,"mor":477,"mos":8683,"mot":735,"mpa":1170,"mu ":1142,"gį ":94,"miš":1246,"mių":389,"mt ":92,"mto":342,"mtm":156,"mti":479,"mso":66,"msi":145,"mta":393,"mur":124,"mus":1181,"mut":199,"mui":800,"mul":740,"mum":317,"mun":878,"muo":1470,"muz":1069,"mpė":146,"džo":105,"dža":265,"mga":68,"eš ":414,"mi ":2927,"dži":8825,"dže":173,"meč":426,"mbū":66,"maž":1405,"min":8866,"mio":695,"ešo":276,"mil":606,"mim":392,"ešm":321,"mir":512,"mis":3697,"ešp":103,"ešt":271,"mit":1456,"ešu":171,"miu":166,"mic":65,"mia":1046,"eša":283,"mig":185,"eše":206,"mie":3684,"mid":149,"mik":627,"ešk":262,"mij":1021,"eši":2201,"maš":150,"mo ":11787,"mln":124,"mm ":215,"mna":274,"meš":73,"vėž":564,"tša":98,"įve":191,"įva":943,"įvy":373,"tūr":3760,"tūn":64,"tūk":418,"sūn":301,"sūr":75,"sų ":1585,"vė ":1134,"Ček":141,"Čer":143,"vęs":499,"Čiu":96,"vėr":82,"vės":1539,"Čik":109,"vėd":103,"Čil":228,"vėm":70,"vėn":180,"vėp":105,"Čia":184,"vėj":505,"vėl":1093,"vę ":211,"rža":90,"rže":425,"rži":124,"ržo":186,"ržu":185,"ržy":332,"vą ":302,"ržų":128,"vč ":177,"zra":92,"zmą":73,"uči":685,"ršū":168,"čem":518,"rūd":121,"rūg":209,"tį ":848,"rūt":179,"rūs":316,"rūv":85,"rūk":126,"rūn":147,"rūm":473,"rūp":93,"zmų":92,"zuo":830,"zul":113,"čia":6305,"čiu":3945,"čin":247,"rų ":3880,"čio":4188,"rūš":1748,"rųj":78,"čią":252,"čių":6037,"čiū":111,"zga":86,"rš ":371,"zdu":307,"zdy":84,"zdo":187,"zeu":81,"zen":162,"zel":75,"zer":347,"ze ":133,"zda":224,"zdi":311,"zde":65,"zac":1511,"zai":89,"zam":101,"zan":208,"zal":69,"zar":93,"zau":206,"zav":191,"zas":215,"zos":160,"zot":82,"zon":766,"zol":127,"zo ":283,"zma":593,"zmo":623,"zme":95,"zdž":483,"zna":91,"zmu":127,"rša":115,"zia":212,"zie":245,"zid":291,"zic":125,"zij":2812,"rši":542,"zin":555,"zil":400,"zik":1298,"ršk":199,"zio":150,"zis":547,"ršt":245,"zit":198,"ršu":471,"yvu":368,"ynų":327,"yvy":146,"yvo":316,"yve":3277,"yvi":885,"yva":1082,"ymų":245,"ytu":3236,"yto":736,"yti":2477,"yta":2174,"ynė":548,"yst":1756,"yną":129,"ysk":104,"ysl":216,"ysi":1379,"ykš":318,"yse":638,"sį ":224,"ymė":134,"ymą":285,"yri":1443,"yro":395,"yru":239,"ylė":72,"yra":10061,"yre":153,"ys ":5011,"ykę":197,"ypt":748,"ygų":179,"ypa":451,"yop":72,"yny":165,"ynu":272,"tęs":608,"yvū":473,"yvų":140,"za ":200,"tėl":64,"tėn":94,"tėm":94,"tėj":863,"ytų":2089,"tės":2264,"tėv":185,"tę ":460,"yzd":234,"yrų":212,"ytą":75,"tė ":2107,"ytė":146,"ysč":138,"yrė":91,"ybi":1941,"ybo":1219,"yda":247,"yde":256,"ydi":267,"ydo":238,"ydr":112,"ydy":152,"ya ":66,"sęs":72,"rįž":117,"ybe":375,"yba":1117,"ydį":315,"ybų":501,"yka":208,"ykd":1149,"yki":766,"ykl":3525,"yko":378,"yks":518,"yku":494,"yn ":82,"yla":244,"yli":1437,"ygą":95,"yll":79,"ylo":78,"yma":917,"ydų":119,"ymi":564,"ydž":370,"yme":67,"ymo":2861,"ymu":353,"yna":1411,"yni":762,"yne":693,"yno":1069,"ygi":962,"ygl":225,"ybą":107,"yga":678,"ybė":5361,"tą ":2180,"ygo":549,"ygu":328,"ybę":375,"yin":78,"tąj":82,"yje":6777,"sči":2451,"pūs":102,"rį ":999,"sė ":786,"sėd":208,"sėk":228,"sėj":581,"sės":2025,"sėt":101,"sę ":210,"rįs":115,"pų ":324,"są ":505,"ožy":124,"ože":207,"oža":97,"oži":226,"rėž":276,"sąv":390,"sąr":690,"sąs":356,"sąj":351,"sąm":110,"sąl":240,"sąn":78,"pši":366,"ręs":410,"rėg":75,"rėl":160,"rėj":1557,"rėn":386,"rėm":119,"rėt":168,"rės":6344,"rę ":1445,"rėd":146,"ww ":76,"rąž":93,"rė ":1022,"www":76,"ws ":91,"rči":327,"nžu":77,"rą ":1079,"nži":161,"rąj":82,"ošt":122,"oši":132,"oše":197,"oša":95,"vyl":67,"vyk":2515,"vyn":610,"vyr":1040,"vyd":92,"vyj":96,"vys":564,"vyt":65,"vyz":235,"nųj":77,"war":74,"viš":606,"vro":160,"vių":2220,"vsk":219,"vu ":109,"pį ":89,"vus":1133,"vuo":278,"vum":127,"vul":245,"vz ":411,"nų ":7207,"vyb":651,"vož":79,"via":688,"vio":508,"vir":3137,"vik":248,"vil":1104,"vim":2439,"vin":4831,"vig":137,"nši":88,"vij":1156,"vic":102,"vid":2656,"vie":10516,"viz":527,"viv":2783,"viu":186,"vit":264,"vis":3495,"važ":200,"vka":106,"vo ":7394,"vež":182,"vič":254,"voj":2810,"vol":423,"vok":960,"von":160,"vor":253,"vot":192,"vos":5796,"ąra":686,"vi ":818,"ąjį":197,"vač":96,"mži":691,"ąją":384,"ver":3346,"ves":601,"vet":199,"vej":358,"vei":3674,"veg":178,"ven":4988,"vel":635,"vek":95,"ved":568,"ąmo":118,"ąna":83," − ":501,"ve ":425,"ąly":225,"val":10598,"vak":4952,"van":3055,"vam":251,"vap":91,"var":5998,"vat":733,"vas":2439,"vav":423,"vau":1068,"vaz":104,"vab":435,"vac":68,"vad":6655,"vai":6631,"vaj":535,"ąju":613,"vag":166,"uvų":343,"ąja":230,"va ":1330,"utų":550,"pės":2048,"pėj":200,"pėm":214,"pėn":79,"uvę":282,"pėd":630,"usų":229,"pę ":262,"uvė":461,"urž":199,"mūš":172,"mųj":304,"urų":123,"urš":132,"uzi":1325,"usį":91,"uza":114,"uzd":74,"utė":482,"pė ":2903,"urį":786,"mų ":3232,"usę":91,"usė":442,"usą":115,"uož":251,"urė":7194,"urę":1318,"uoš":293,"umų":278,"upę":254,"ux ":100,"upė":4550,"uvi":2426,"uvk":94,"uvo":8973,"uva":1382,"uve":227,"uvy":103,"unų":111,"uvu":401,"usl":117,"usm":292,"usk":802,"ukš":4009,"usi":16156,"mūg":65,"usd":94,"use":242,"usa":1585,"unė":100,"ukų":215,"usy":348,"usv":157,"usu":325,"ust":2671,"uss":74,"mūs":116,"ukū":174,"mūr":92,"usr":197,"uso":1941,"uti":3065,"ute":1211,"uta":1298,"utb":1100,"ulų":95,"uty":105,"uts":108,"utu":465,"uto":2462,"utr":654,"uoč":243,"us ":25324,"ukė":207,"ut ":98,"ulė":940,"urb":304,"ura":1290,"urd":135,"ure":433,"urg":609,"umą":619,"uri":13437,"pči":95,"urk":677,"urm":174,"urn":819,"uro":2323,"urp":92,"urs":335,"urt":2267,"uru":270,"ulį":98,"urv":125,"ury":263,"ujų":216,"uog":115,"uod":1799,"uob":148,"uop":102,"uon":468,"uol":1850,"uom":2229,"uoj":2916,"ują":85,"uok":472,"uot":3875,"uos":9387,"upa":260,"ugų":171,"uoz":64,"ur ":478,"uką":82,"upi":1336,"upe":880,"upo":164,"upr":184,"upy":138,"upt":113,"upu":131,"ump":675,"umu":499,"umi":565,"umo":1874,"uma":2535,"umb":630,"ume":1015,"udž":589,"uly":692,"ugė":81,"uo ":11031,"ugę":74,"unt":286,"uns":125,"unu":78,"unk":1010,"uni":2717,"uno":1046,"unc":101,"und":514,"una":778,"ung":2769,"une":377,"up ":104,"uks":488,"ukr":338,"uku":1424,"ukt":1044,"uko":768,"ukm":63,"ukl":276,"uki":835,"ukc":193,"uke":516,"um ":371,"uka":1363,"ubų":81,"uju":231,"ulv":76,"ulu":205,"ult":2376,"uls":117,"ulp":163,"ulo":319,"ulm":98,"ulk":1598,"uli":4261,"ulg":118,"ule":258,"ula":686,"un ":91,"uid":65,"uik":63,"uil":83,"uin":83,"uis":152,"uic":69,"ąvo":220,"uje":2229,"uji":238,"ujo":901,"ąve":118,"ąva":67,"uit":183,"uiz":104,"ul ":100,"udė":847,"uja":1817,"ugi":2043,"ąsi":447,"lži":190,"ugd":727,"uge":478,"ugn":281,"ugo":862,"ugp":256,"ugl":79,"ui ":2863,"uga":3449,"ugy":234,"ugv":81,"ugu":1147,"ugs":210,"ugr":96,"uha":63,"pą ":193,"ąst":536,"uda":2525,"ude":508,"udi":1843,"ubo":317,"ubt":123,"ubr":94,"ubu":247,"ue ":115,"uci":1337,"uch":229,"uer":154,"ufo":165,"udu":90,"udr":145,"udo":3114,"ug ":571,"udy":189,"udz":149,"uen":154,"uel":163,"uei":67,"tyč":85,"tuš":104,"ua ":181,"uau":101,"uar":229,"ual":389,"uan":260,"ubi":273,"ubj":203,"ubl":1778,"ube":224,"uba":739,"uac":101,"trų":261,"trū":89,"tyv":1223,"tyg":105,"tyj":2629,"tyk":592,"tyl":66,"tym":1125,"tyn":745,"tyr":732,"tys":1652,"tyt":1130,"tvė":152,"ty ":191,"tvy":70,"tve":795,"tvi":1569,"tva":1876,"tur":3389,"tus":4707,"tut":476,"tuv":8023,"tui":321,"tul":676,"tuk":242,"tun":200,"tum":1081,"tup":188,"tuo":4132,"tub":127,"tua":342,"tud":729,"tuc":1071,"tug":166,"tyb":2995,"lų ":4053,"trė":213,"trą":137,"ts ":284,"tiš":751,"tre":865,"tt ":65,"tra":6670,"tri":3774,"oči":767,"tru":2118,"tro":4397,"nį ":1595,"tu ":2200,"try":1680,"tsa":349,"lūd":70,"tsi":864,"lūn":117,"lūk":105,"tsk":627,"tsp":242,"tst":1087,"lūs":124,"tte":91,"ttp":75,"tme":480,"tma":182,"to ":9437,"tmo":148,"tmi":171,"tni":215,"tne":192,"tp ":76,"tna":162,"tno":94,"tod":622,"toc":126,"toj":3705,"tog":985,"tob":201,"tov":2012,"tos":4057,"tot":663,"toz":74,"tom":2117,"ton":1654,"tok":1091,"tol":2595,"tor":6343,"top":234,"tr ":100,"tpa":67,"tij":3726,"lši":179,"taš":256,"til":880,"tik":5456,"tif":144,"tie":4693,"tig":115,"tir":719,"tit":1932,"tis":9550,"tin":18689,"tim":2115,"tip":1038,"tio":733,"thu":146,"tia":180,"tib":66,"tic":399,"tid":346,"taž":90,"tiz":264,"tiu":112,"tiv":191,"tko":75,"tku":65,"tka":159,"tli":717,"tla":468,"tle":281,"tem":2567,"ten":1501,"teo":504,"tep":206,"tei":4906,"tek":2029,"tel":3046,"teg":305,"tea":212,"teb":306,"tec":578,"ted":149,"tfo":195,"th ":187,"tez":110,"tet":1080,"tes":640,"ter":7958,"ti ":12579,"tga":173,"tač":773,"tho":100,"the":357,"thi":68,"tha":124,"yži":382,"zūr":65,"žė ":79,"zų ":230,"žėj":93,"žės":302,"yšu":129,"yšk":285,"yši":281,"žįs":104,"AR ":460,"AT ":96,"AV ":1248,"zę ":74,"zės":255,"BA ":84,"AB ":129,"가가":99,"Žie":168,"Žmo":87,"Žal":202,"Žai":105,"Žem":792,"ža ":218,"vųj":76,"vų ":1432,"Žva":101,"žli":98,"žka":118,"žin":2776,"žim":518,"žik":242,"žir":69,"žio":3609,"žiu":1205,"žis":284,"ždė":292,"žia":4861,"žie":756,"三 ":108,"žpa":73,"žos":282,"zė ":373,"žny":789,"žoj":209,"žol":428,"žod":694,"žmo":1552,"žią":130,"žni":1016,"žna":456,"žo ":203,"ždž":217,"žde":109,"žda":511,"žas":486,"žba":64,"žai":1499,"žal":557,"žan":496,"žar":129,"ži ":73,"žer":3193,"žes":178,"žet":91,"žei":363,"žel":344,"žem":2279,"žen":275,"ždy":272,"ždu":70,"ždi":76,"už ":1010,"uža":101,"užd":391,"uže":68,"užs":382,"užt":281,"užr":186,"užu":95,"užk":126,"uži":622,"užp":129,"užn":124,"žys":63,"žym":728,"žyg":68,"žyd":166,"žyb":321,"žtv":87,"žud":129,"žuv":1113,"žut":66,"žur":382,"žuo":217,"žve":120,"žva":821,"žvi":225,"žvy":103,"žra":185,"žių":1603,"žiū":506,"žsi":337,"žta":167,"žte":70,"žti":234,"yči":2350,"vūn":458,"užė":257,"tųj":219,"tų ":12460,"ušė":81,"之 ":65,"ušt":63,"ušk":90,"uši":396,"uša":109,"tžv":91},"n_words":[6266541,7160065,6094403],"name":"lt","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/lv.json b/contrib/languages-data/lv.json
new file mode 100644
index 0000000..4442428
--- /dev/null
+++ b/contrib/languages-data/lv.json
@@ -0,0 +1 @@
+{"freq":{"ūci":182,"ūdu":26,"ūdr":44,"ūdo":28,"ūde":716,"ūda":90,"D":3059,"E":2166,"F":1831,"G":1835,"A":5495,"B":3355,"C":2145,"L":4567,"M":3974,"N":1829,"O":1507,"H":1820,"I":2327,"J":1801,"K":4616,"U":852,"T":4509,"W":365,"V":3715,"Q":38,"P":4759,"S":5447,"R":3339,"Y":65,"X":234,"Z":1747,"f":8161,"g":29438,"d":58295,"e":131519,"b":30435,"Fed":124,"c":22500,"a":237698,"n":95937,"o":88352,"l":73126,"m":69285,"j":43124,"k":78380,"Fel":22,"h":5764,"Fen":23,"i":199506,"w":553,"v":49552,"u":101354,"Fer":33,"t":118682,"s":178123,"r":127678,"q":293,"p":52161,"z":35062,"y":1297,"x":396,"²":54,"Fil":64,"í":35,"Fin":30,"Fir":23,"é":81,"ç":26,"ä":65,"â":42,"á":40,"ü":91,"ö":83,"ó":42,"Fiz":28,"ē":27728,"Ē":143,"Ā":409,"ā":76597,"Č":300,"č":1638,"ı":70,"ķ":3799,"Ķ":255,"Eze":42,"Ļ":56,"ļ":8048,"Ģ":128,"ģ":3732,"Ī":133,"ī":34346,"Ģer":28,"Ģeo":39,"Ģen":28,"ş":30,"ņ":7202,"Ņ":124,"ō":55,"Ž":134,"ž":4793,"Š":1043,"š":16459,"Ū":37,"ū":7903,"Equ":185,"Eri":30,"Ern":25,"Eur":70,"Eir":595,"́":38,"Ele":79,"Eks":36,"Eko":28,"μ":50,"ν":83,"End":31,"ο":116,"ι":60,"κ":31,"λ":61,"δ":42,"ε":43,"η":40,"α":118,"β":23,"γ":49,"ί":37,"Emi":22,"Eli":48,"ω":27,"Ent":23,"ό":36,"σ":41,"ς":97,"ρ":66,"π":29,"φ":23,"υ":22,"τ":47," l":9777,"ь":65," m":8646," n":13651," o":3224,"я":84," h":1222," i":26578," j":3868," k":23420,"ы":62," d":13444," e":3216,"х":31,"ц":23," f":3413," g":7521,"ч":56,"р":268," a":22507,"с":262," b":8086,"т":170," c":4249,"у":116," y":33," x":65," z":3907," u":13570," t":13126," w":93," v":19041," q":33," p":25406," s":20595," r":6772,"HK ":23,"Ūde":29,"Л":28,"К":30,"Н":52,"М":37,"О":27,"Б":29,"А":26,"В":29," J":1795," K":4607," H":1796," I":2309," N":1813," O":1473," L":4536," M":3956," B":3324," C":2058,"Р":35," A":5469,"С":193," F":1822," G":1815,"Ф":23," D":3031," E":2156,"л":187," Z":1739,"к":221,"й":81," Y":64," X":228,"и":382,"п":57,"о":387,"н":271,"м":85," S":5368,"г":53,"Ger":28," R":3325,"в":222,"б":51," Q":37," P":4729,"а":390,"Geo":50," W":350,"з":76," V":3691,"Gen":23," U":847,"е":335," T":4452,"д":133,"šža":24," ā":501," Ā":409," č":467," Č":299," Ē":143," ē":282,"HL ":22," ž":311," Ž":134," Ū":37," ū":565," Š":1042," š":2083," Ļ":56," ļ":329," ķ":663," Ķ":254," Ī":131," ī":702," Ģ":128," ģ":1094," Ņ":120," ņ":41,"י":25,"Gan":49,"Gal":115,"Gam":34,"Gau":47,"Gar":70,"Gai":64,"Fut":176,"و":37,"ي":53,"ل":68,"م":47,"ن":33,"د":35,"ب":27,"ة":22,"ا":101,"س":25,"ر":50,"Frī":41,"šūn":142,"Flo":40," А":25," Б":28," В":29,"Fra":343," К":29,"Fri":44," Л":27," М":37," Н":52," О":26,"Fre":87,"A ":493,"For":93," α":37,"F ":117,"Da":634,"Cu":37,"Cl":54,"Co":292,"Cr":43,"Ce":347,"Ch":135,"Ci":209,"G ":59,"Ed":95,"Eb":22,"Du":106,"Dz":229,"Do":208,"Dr":103,"De":400,"Di":679,"IP ":41,"Fe":227,"H ":468,"Fa":110,"Ez":43,"Eu":85,"Ev":42,"Ex":29,"Er":98,"Eq":185,"Et":48,"Es":44,"En":123,"Em":90,"Ei":677,"El":193,"Ek":93,"Eg":26,"Ge":158,"Bā":56,"Ga":460,"I ":434,"Fu":230,"Fr":536,"Fo":212,"Fl":70,"Fi":194," о":23,"B ":133," Р":34," С":176," Ф":23,"II ":185,"C ":377," с":29,"Av":74,"Au":759,"Ar":519,"At":405,"As":243,"D ":170,"Ba":1010,"Az":34,"Af":50,"Ag":51,"Ah":37,"Ab":147,"Ac":82,"Ad":148,"Am":328,"An":739,"Ap":260,"Ai":200,"Ak":154,"Al":535,"Hit":49,"His":22,"Bu":240,"Br":526,"Ca":279,"E ":106,"Bi":333,"Be":447,"Bo":341,"Hil":23,"Bl":116,"Him":40,"Hip":27,"Kv":49,"Ku":433,"Kl":267,"Kr":1045,"Ko":785,"Gā":24,"Le":274,"Li":1187,"N ":114,"La":2106,"Lu":187,"Ly":26,"Lo":258,"Me":656,"Hā":28,"Dž":311,"Mi":487,"O ":256,"Ma":1352,"My":39,"Hē":35,"Mu":257,"Mo":497,"Ni":163,"Ne":339,"Na":331,"P ":315,"Hel":44,"Hei":53,"Nu":37,"No":488,"Ok":85,"Ol":289,"On":42,"Og":53,"Oh":28,"Od":50,"Jā":93,"Hen":52,"Of":29,"Ob":41,"Her":113,"Gi":68,"Gl":88,"Gr":518,"Go":173,"IA ":51,"Gu":165,"Gv":31,"Bē":72,"J ":48,"Ha":442,"He":346,"Hi":235,"Ho":305,"Bī":48,"Hu":55,"Cē":86,"K ":99,"Ib":23,"Id":25,"Dā":72,"Ie":161,"Ig":193,"Im":77,"In":597,"Ik":26,"Il":62,"Dē":23,"Iv":35,"Is":56,"It":130,"Ir":159,"Ja":606,"L ":193,"Iz":255,"Je":180,"Jo":238,"Hab":30,"Ju":235,"Hal":59,"Hai":24,"Ka":1308,"Han":77,"M ":140,"Ham":27,"Har":76,"Ki":234,"Hav":40,"Ke":175,"Mū":173,"Ur":133,"Up":46,"Un":191,"Uk":57,"Ul":29,"Ug":37,"Pā":146,"W ":36,"Tu":367,"Nī":78,"Tr":348,"Lū":27,"To":528,"Th":183,"Ti":330,"Tj":24,"Te":412,"Ta":1094,"V ":507,"Bēr":58,"Grī":41,"Sy":41,"St":811,"Sv":222,"Su":204,"Wo":58,"Wi":99,"Wa":68,"Rā":30,"We":47,"Vo":206,"Pī":38,"Vu":40,"Vi":1349,"Vl":38,"X ":140,"Va":795,"Ve":528,"Pē":222,"Uz":97,"Lā":40,"Pu":149,"Pr":574,"Ps":29,"Gus":31,"S ":509,"Pe":330,"Kā":157,"Pa":1460,"Gui":25,"Pl":246,"Po":522,"Pi":543,"Ph":56,"Gul":51,"Os":199,"Ot":132," ا":37,"Ov":28,"Op":65,"Or":179,"R ":242,"šņu":59,"Jē":211,"Nā":75,"Se":449,"Sc":140,"Si":396,"Sh":52,"Sn":22,"Sm":71,"Sl":112,"Sk":318,"Mī":29,"Sp":308,"So":292,"Ru":198,"Jū":171,"Mē":139,"U ":122,"Sa":1320,"Grā":24,"Re":684,"Mā":133,"Ri":309,"Rh":25,"Ro":567,"Lī":215,"Qu":25,"Lē":28,"T ":118,"Ra":377,"Gre":140,"Gri":112,"Gra":91,"Vī":58,"b ":1431,"Gru":54,"Gro":39,"a ":45978,"Sē":77,"Tā":889,"Yo":26,"Tē":78,"Sī":40,"Z ":47,"Rē":60,"Sā":102,"Pū":23,"Rī":858,"Gol":28,"Got":33,"Vā":356,"Vē":122,"Za":126,"Ze":604,"Zi":554,"Tī":49,"Zo":72,"Zu":34,"God":26,"Rū":43,"Zv":222,"šķā":29,"i ":20844,"aē":38,"gd":52,"ge":675,"bā":1878,"ga":11955,"gb":23,"Ing":34,"Inf":36,"fl":161,"fg":38,"ff":54,"aā":32,"fi":2006,"šķē":67,"fs":184,"fr":742,"ač":195,"fu":915,"ft":143,"Int":119,"fo":1630,"Ins":32,"j ":431,"aķ":196,"gz":413,"bē":351,"Ir ":43,"cā":282,"he":981,"aļ":2174,"ha":893,"gn":346,"gm":103,"gl":1302,"gk":23,"aģ":126,"gi":1023,"gh":75,"gg":39,"gv":152,"gu":2538,"gt":345,"gs":1978,"gr":3115,"šķī":179,"aī":165,"go":1269,"dt":34,"du":3460,"dv":153,"dy":39,"dz":8558,"g ":378,"Ima":40,"ea":831,"eb":2145,"ec":2362,"ed":5035,"de":6326,"dd":61,"dg":22,"di":7508,"dh":33,"dk":50,"dm":437,"dl":81,"do":4137,"dn":387,"dp":40,"ds":1737,"dr":2632,"ew":72,"ex":66,"eu":138,"ev":3202,"ey":101,"ez":2104,"fa":698,"h ":326,"Inc":26,"Ind":290,"fe":924,"eh":574,"eg":1917,"ef":579,"ee":260,"el":8559,"ek":9435,"ej":1682,"ei":6080,"ep":1928,"eo":1018,"Imp":27,"en":17231,"em":12435,"et":9035,"es":13667,"er":13266,"cb":46,"ca":1325,"e ":9104,"bv":92,"by":23,"bs":478,"br":1873,"bu":2780,"bn":90,"bo":2478,"bp":134,"bj":321,"bk":204,"bl":923,"bi":5030,"bb":37,"bd":41,"be":3600,"da":13240,"f ":227,"cy":80,"cu":688,"ct":320,"cs":106,"cr":118,"co":534,"cp":28,"cm":65,"ck":210,"cl":78,"ci":9940,"ch":496,"ce":3798,"cc":58,"c ":1617,"Zī":28,"az":1867,"ay":138,"ba":7096,"d ":1260,"at":15301,"as":46560,"ar":22107,"šķe":37,"ax":44,"aw":25,"av":4016,"au":11606,"šķa":58,"ak":5105,"al":13877,"ai":16755,"aj":4377,"ao":99,"ap":6109,"šķi":792,"am":7193,"an":13591,"ac":1971,"ad":7331,"aa":123,"šķu":61,"ab":3333,"ag":2079,"ah":438,"ae":799,"af":317,"nu":3766,"nt":6816,"ns":6634,"ič":143,"nr":301,"hī":43,"np":87,"no":13676,"nn":366,"nz":252,"ny":68,"iē":45,"nv":1299,"oe":196,"jā":6443,"of":937,"Dān":37,"oc":1480,"od":5344,"oa":231,"ob":1666,"Dār":22,"om":4560,"on":9439,"ok":3012,"ol":6437,"oi":175,"gš":291,"iģ":203,"oj":3579,"og":1772,"oh":231,"Iga":176,"ot":8233,"m²":53,"gū":417,"os":7017,"ov":1717,"ou":258,"op":3592,"oo":225,"or":8642,"iķ":475,"r ":23056,"ox":43,"jē":253,"ow":104,"oz":1358,"oy":26,"pd":241,"pe":2624,"kā":9200,"pg":761,"gž":90,"pa":12196,"pb":92,"iļ":351,"pc":59,"pl":2446,"iņ":2232,"pm":337,"pn":123,"po":2922,"ph":171,"pi":8931,"pj":212,"pk":534,"lo":5141,"ln":1238,"eņ":1120,"lm":668,"ll":864,"ls":6050,"dū":151,"lr":53,"fī":68,"lp":774,"lv":2188,"lu":3183,"lt":2348,"lz":417,"ly":68,"gē":101,"o ":13523,"md":123,"ma":11255,"mb":1027,"dž":490,"mg":152,"me":8257,"mf":62,"hā":197,"mk":81,"ml":84,"mi":7529,"eš":2372,"mj":254,"mn":294,"mm":568,"mp":1937,"mo":3385,"mt":1499,"ms":4163,"mv":67,"Iek":27,"mu":4579,"hē":85,"mz":26,"Ied":23,"Ier":30,"p ":1187,"Ies":23,"na":11100,"nb":173,"nc":1308,"gļ":378,"nd":3610,"ne":6367,"nf":496,"iā":1355,"ež":1461,"ng":2033,"nh":159,"ni":10637,"nj":103,"nk":1409,"nl":178,"nm":119,"jv":54,"jt":24,"ju":6450,"jr":40,"eč":136,"bū":646,"js":1745,"jp":49,"dī":2457,"jn":108,"jo":2718,"jl":29,"jk":33,"ki":2861,"eģ":460,"kh":87,"kg":105,"fā":86,"ke":1430,"kd":52,"kc":519,"kb":27,"ka":21595,"m ":14604,"fē":229,"ky":24,"cū":51,"ks":6047,"kt":4783,"ku":9917,"kv":498,"ko":8251,"kp":53,"eī":44,"kr":3493,"kk":49,"kl":2204,"dņ":25,"km":1006,"kn":291,"li":12111,"lh":104,"lk":789,"lj":234,"le":5914,"ld":1879,"lg":521,"lf":344,"gā":2261,"la":14259,"eļ":2084,"lc":198,"lb":519,"n ":12333,"eķ":397,"hr":242,"hs":176,"bī":818,"ht":159,"hu":177,"hi":1019,"hn":345,"ho":773,"hl":97,"aņ":799,"hm":55,"id":6988,"ic":1972,"ib":840,"ia":987,"ih":374,"ig":1651,"if":662,"dā":3118,"ie":43851,"hy":49,"cē":786,"k ":3032,"cī":1751,"iq":24,"ir":20819,"is":22114,"it":6916,"iu":210,"iv":1914,"dē":1715,"ix":47,"ii":76,"aš":1551,"ij":18002,"ik":8262,"il":7272,"im":5227,"in":12037,"io":3152,"ip":1361,"je":2277,"jd":65,"až":1131,"eā":550,"ji":2430,"iz":8191,"l ":908,"ja":18785,"xi":24,"pš":285,"rģ":265,"pū":149,"rī":5569,"xt":23,"rē":1505,"ww":72,"z ":3167,"ož":188,"sā":1658,"nž":71,"wi":65,"oš":2487,"šīs":96,"šīr":28,"wo":40,"šīn":152,"ws":45,"vv":135,"vz":34,"y ":470,"wa":100,"pļ":34,"rā":7122,"we":66,"vl":36,"vm":57,"oņ":431,"vj":161,"vk":55,"vi":15879,"nš":154,"vg":67,"vu":1511,"vr":110,"vs":665,"nū":52,"vp":135,"pī":487,"vn":277,"vo":2198,"šīb":168,"uz":3239,"ux":47,"pē":3898,"uv":1138,"uu":163,"ve":6316,"vd":128,"vc":24,"oļ":231,"va":13478,"oķ":34,"x ":250,"oģ":891,"ui":330,"mš":154,"uj":549,"uk":2577,"ul":3970,"ue":159,"pā":3136,"uf":42,"ug":2904,"lž":37,"uh":49,"oī":103,"ur":8826,"mū":945,"us":8406,"ut":3554,"um":10369,"un":12883,"uo":66,"up":1299,"ty":109,"tz":181,"tu":9581,"tt":1263,"tw":38,"tv":2682,"ub":1129,"ua":219,"ud":1801,"uc":777,"w ":104,"to":9558,"tn":1900,"tm":416,"tl":676,"lū":587,"ts":7290,"tr":9995,"oč":89,"nī":2204,"tp":327,"tg":235,"tf":75,"te":8645,"td":205,"tk":771,"tj":107,"ti":16287,"lš":164,"th":431,"v ":977,"šī ":103,"tb":1541,"tc":58,"ta":17648,"su":2355,"sv":864,"ss":1501,"kū":96,"st":22568,"sy":29,"sz":152,"sw":23,"nē":1546,"sl":2038,"sk":10907,"sn":958,"sm":2964,"sp":4072,"so":1970,"sr":95,"nč":276,"mī":785,"sd":451,"sc":379,"sf":321,"nā":8240,"se":3495,"sh":197,"sg":86,"sj":78,"si":5921,"kš":1455,"rz":732,"u ":34678,"sa":11922,"sb":260,"rr":239,"rs":5613,"jū":1021,"rt":4290,"ru":7404,"rv":1310,"Cēs":56,"mē":2975,"ry":160,"lī":4107,"rp":1821,"ro":10142,"rn":1729,"kņ":114,"rm":3278,"rl":568,"rk":1664,"rj":126,"jš":27,"ri":15479,"rh":439,"rg":2074,"iž":275,"mā":4269,"rf":127,"re":6968,"rd":2322,"rc":590,"kļ":994,"rb":2090,"ra":18562,"t ":6209,"lē":2235,"qu":266,"kī":37,"iš":513,"lā":5128,"s ":99264,"kē":55,"pz":438,"pt":1346,"pu":3008,"pv":476,"pp":139,"jī":54,"pr":4678,"ps":1123,"zī":3820,"zā":1002,"IX ":48,"šģi":28,"zē":1137,"už":113,"uš":785,"IV ":42,"tš":221,"uļ":295,"uņ":333,"vī":926,"tū":1454,"vā":2847,"tļ":184,"uģ":166,"rž":79,"zz":61,"sū":160,"Bīb":25,"vē":3474,"tņ":104,"uā":341,"Hor":54,"zg":379,"zi":7929,"rš":1224,"zb":206,"Hou":27,"zc":577,"zd":606,"ze":4296,"za":1506,"Hom":27,"Hon":23,"Hok":34,"Hol":58,"zv":1734,"uē":34,"rū":541,"zs":1624,"uč":23,"zr":375,"zu":648,"zt":460,"zo":976,"zn":964,"tī":5903,"zp":657,"zk":332,"zj":64,"zm":1200,"zl":791,"yg":24,"ye":62,"tā":11917,"rļ":24,"yc":58,"yd":22,"ya":64,"rķ":279,"sē":2104,"tē":2602,"yt":35,"ys":126,"yr":30,"sī":1152,"yp":31,"yo":86,"yn":43,"ym":26,"rņ":171,"yl":66,"yi":22,"Arg":44,"Arh":40,"Are":22,"Ard":29,"Ara":25,"Arm":42,"Ari":31,"Šīs":37,"Apv":89,"Apo":23,"Atk":35,"Atl":83,"Atr":91,"Ato":22,"Ast":53,"Ata":28,"Asi":33,"Aso":76,"Art":56,"Avo":25,"Aut":94,"Aus":451,"Aug":149,"Alū":46,"Arā":51,"šēj":249,"zš":91,"zū":24,"zņ":392,"Šķi":29,"Atš":29,"Bak":26,"Bal":501,"Ban":57,"Bab":25,"Baz":45,"Bar":119,"Bas":49,"Bau":40,"Abr":31,"Act":23,"Ada":38,"šā ":313,"Adr":25,"šād":125,"šāk":184,"šām":60,"šās":183,"šāv":22,"Afg":26,"Aiz":104,"Aka":34,"Ala":26,"Alb":79,"Ale":93,"Alf":40,"Alt":55,"All":36,"Alp":29,"Ame":197,"Ama":49,"Šī ":93,"Ang":135,"Ana":95,"And":123,"Ant":228,"Ann":55,"Ar ":101,"ОН ":57,"Bul":41,"Bur":79,"Blū":22,"Bru":49,"Brī":92,"² ":53,"Brā":30,"Cal":37,"Cam":27,"Car":39,"Cau":23,"Can":86,"Cap":22,"CH ":201,"Bez":37,"Ber":134,"Ben":73,"Bel":45,"Bie":67,"Biz":31,"Šād":55,"Bil":27,"Bir":42,"Bio":33,"Blo":34,"CP ":24,"CO ":32,"Bla":25,"Bre":53,"Bra":125,"Bro":55,"Bri":92,"Bol":40,"Bon":23,"Bor":50,"Bos":71,"Bov":25,"Der":41,"Des":26,"Dei":40,"Del":44,"Dem":45,"Den":34,"Deg":23,"Dan":52,"Dar":42,"Dat":63,"Dau":261,"Dab":33,"Chr":32,"Cit":38,"Cir":22,"Cil":47,"Cel":26,"Cen":195,"Cet":27,"Cer":41,"Cha":50,"Cor":106,"Com":69,"Col":30,"Con":48,"FA ":41,"ģip":59,"ģin":756,"ģio":352,"ģit":52,"ģis":449,"ģim":77,"ģij":982,"Dze":94,"Dzi":93,"ģi ":28,"ģer":123,"ģel":28,"ģeo":161,"ģen":236,"Edv":25,"Edu":28,"ģa ":65,"Dzī":27,"āg":215,"Daž":59,"āf":379,"āc":3184,"ād":3136,"āb":862,"āp":472,"ām":4331,"ān":2529,"āk":5743,"āl":5581,"āj":3661,"āv":1981,"āt":4781,"ās":7161,"ār":6698,"āz":697,"Dib":24,"Dis":48,"Din":26,"Āb":22,"Dig":22,"Ād":23,"Āg":35,"Āf":113,"Die":362,"Āz":143,"Ār":50,"Div":37,"ā ":23455,"Dub":30,"ür":23,"ая ":29,"ģu ":60,"СН ":64,"Dra":31,"Dob":31,"ES ":40,"Don":29,"Dom":61,"OH ":75,"Nea":33,"ēģ":190,"Nei":26,"Net":40,"Ner":49,"Nep":30,"Neo":28,"Ēģ":41,"Nat":30,"−":111,"īdi":166,"īdr":32,"īds":186,"īdu":170,"īda":185,"īde":154,"Nin":27,"Nik":51,"ēž":23,"īci":383,"ūšu":22,"īca":276,"ūša":102,"īce":148,"ībn":58,"ēš":773,"ībi":22,"ību":1222,"ībv":37,"īd ":59,"ībe":36,"ība":3544,"ēļ":376,"New":31,"ēķ":56,"ēņ":27,"īcā":40,"ībā":960,"īgi":496,"īga":1676,"īgu":405,"īgo":298,"īgs":407,"īdz":1603,"īla":22,"Nam":26,"īgā":813,"īle":67,"īli":124,"Nac":121,"īka":42,"īkl":241,"īko":106,"īks":151,"īja":305,"ģu":81,"īji":84,"īju":700,"īdī":105,"īm ":246,"ģi":2798,"ģe":601,"ģa":66,"īdā":112,"īcī":70,"Ģe":106,"āļ":238,"āņ":597,"āč":113,"Či":48,"Če":124,"Ča":35,"Ču":25,"či":150,"če":516,"Īri":58,"ča":247,"āš":567,"āž":93,"No ":69,"ē ":2668,"Īsl":26,"čo":33,"čs":42,"ču":562,"ēz":349,"ēt":4896,"ēs":1279,"ēv":258,"čū":27,"Čī":26,"OS ":28,"ēm":2242,"ēn":867,"ēk":1496,"ēl":2212,"ēr":3544,"ēp":82,"Nov":61,"ēc":1136,"ēd":833,"Nor":163,"ēj":4209,"ēg":174,"Nos":50,"Not":22,"ēb":25,"Nok":22,"Ēr":64,"Nob":33,"ķe":541,"ķa":229,"ķu":510,"ķs":30,"ķo":78,"Odi":27,"ķi":1644,"ļ ":195,"Ogr":36,"Jān":51,"ļa":2291,"ļi":696,"ļk":39,"ļj":32,"ļe":71,"ļd":32,"ķā":31,"Ķī":170,"ļr":180,"ļs":48,"ļl":36,"ļo":646,"ķē":171,"ļu":2107,"ļv":154,"Ļe":28,"Oke":30,"ķī":541,"ļķ":37,"ļī":23,"ļģ":54,"ļē":91,"PA ":36,"ļā":656,"ļš":133,"ļū":169,"ļļ":89,"ļņ":176,"ņv":39,"ņi":716,"ņk":42,"ņj":25,"ņo":260,"ņn":28,"ņu":2226,"ņs":60,"ņr":139,"Ņu":43,"ņa":1766,"ņb":29,"ņd":47,"ņe":432,"Otr":101,"Ovi":22,"ī ":3304,"ģē":120,"ģī":39,"Jēk":140,"Īs":48,"Īr":60,"īb":5893,"Jēz":37,"īc":960,"īd":2819,"īj":1202,"īg":4125,"īl":317,"īk":663,"īn":1786,"īm":2002,"īp":453,"īr":757,"īt":3369,"īs":1871,"īv":3050,"īz":500,"ģīt":31,"Oli":250,"ı ":35,"Jāņ":29,"īģ":38,"īč":97,"īņ":368,"PS ":25,"īļ":122,"īš":592,"Ope":52,"Org":41,"Ost":30," −":67,"Osk":35,"Osm":120,"Ķe":55,"š ":1382,"Ple":55,"Pla":145,"Pil":83,"Paš":73,"Pit":22,"Pir":196,"QL ":27,"Pie":153,"Pha":28,"ģēt":22,"ģēr":24,"Kār":70,"ģēl":37,"šī":572,"šģ":28,"Per":109,"šļ":41,"Pet":30,"Pen":59,"Pek":25,"Pel":45,"šķ":1261,"Šī":134,"Kā ":42,"šā":893,"šē":273,"Šķ":45,"Šā":65,"Paz":31,"Pat":54,"Pas":386,"Par":468,"Pav":27,"Pau":41,"Pad":103,"Pan":44,"Pap":26,"Pal":61,"Pak":46,"šg":29,"še":180,"šd":202,"ša":5315,"šo":571,"šp":217,"šm":110,"šn":89,"šk":157,"šl":112,"ši":1623,"šv":170,"šu":2014,"št":354,"šs":357,"šr":148,"šz":36,"Še":104,"Ša":199,"Šo":132,"Ši":163,"Št":29,"Šv":81,"ņš":324,"Ņū":40,"ņģ":77,"Prū":54,"ņē":348,"ņķ":245,"ņā":297,"Prā":22,"Pro":228,"Pri":121,"Pre":78,"Pra":58,"Pol":274,"Pos":25,"Pop":24,"Por":59,"žr":80,"žs":96,"žu":691,"žn":24,"žo":773,"že":338,"ža":902,"žk":75,"žm":27,"ži":613,"Ža":46,"Ži":23,"Že":35,"RS ":104," ال":23,"ž ":41,"žū":26,"žņ":118,"žī":67,"žģ":48,"žā":790,"SA ":43,"Rad":110,"Rai":35,"Rag":24,"Rak":35,"ū ":140,"šņ":106,"Ūd":31,"šž":26,"šū":152,"ūg":56,"ūd":971,"ūc":206,"ūz":629,"ūs":815,"ūt":957,"ūv":256,"ūp":130,"ūr":2415,"ūk":417,"ūl":106,"ūm":112,"ūn":373,"ūš":146,"ūž":54,"Isl":22,"Irā":62,"Iva":31,"Izv":72,"Izr":29,"Itā":112,"Jac":27,"Jav":26,"Jau":217,"Jap":129,"Jan":40,"Jam":45,"Jag":28,"Ja ":27,"Jel":80,"Jer":41,"Jor":27,"Jon":25,"ア":23,"Joh":102,"KS ":29,"Jug":32,"Jup":83,"Jur":42,"Jum":22,"Kad":24,"Kab":22,"Kai":30,"Kam":79,"Kal":190,"Kap":52,"Kan":182,"Kau":72,"Kat":99,"Kas":57,"Kar":357,"Kaz":79,"Ker":28,"Ken":34,"Kem":29,"Kei":25,"Kir":46,"Kin":77,"Kij":27,"Kli":34,"Kle":26,"Kla":44,"Klu":132,"Kon":195,"Kom":112,"Kol":59,"Kos":58,"Kor":100,"Kop":106,"Kod":29,"Kok":40,"Kre":35,"Kra":70,"Kri":714,"Kro":35,"Kru":113,"Krā":28,"Kul":67,"Kur":259,"Kva":27,"Let":23,"Leo":37,"Lej":26,"Led":38,"Lau":64,"Lak":51,"Lai":103,"Lag":34,"Lat":1556,"Lar":29,"Lap":25,"Lam":24,"Lan":56,"Lab":56,"ML ":46,"Lib":28,"Lie":695,"Lim":30,"Lin":97,"Lit":29,"Liv":157,"Lut":24,"Luk":24,"Lud":28,"Lug":27,"Lor":22,"Lon":64,"Lok":22,"ūs ":24,"ūpn":62,"ūra":1491,"ūt ":294,"ūsd":328,"ūnā":49,"ūrs":28,"ūru":316,"ūrv":45,"ūrm":32,"ūrn":45,"ūrg":22,"ūri":123,"ūks":153,"ūku":78,"ūka":36,"ūko":44,"ūma":25,"ūli":71,"ūmu":36,"ūna":154,"ūns":22,"ūnu":55,"ūni":45,"Mek":48,"Mei":30,"Men":40,"Mel":144,"Mer":74,"Met":75,"ūtā":37,"Med":58,"Mez":22,"ūzi":561,"Džo":145,"ūzu":22,"ūtī":135,"Džu":25,"Dže":88,"ūsi":58,"ūsm":68,"ūsu":37,"ūst":237,"ūta":126,"ūte":28,"Man":129,"Mal":86,"ūti":73,"Mam":22,"Mar":393,"ūto":32,"Mas":145,"ūtn":56,"Mag":45,"ūts":52,"Mad":99,"Mak":70,"Mai":65,"ūtu":111,"Mac":23,"īzā":35,"īzē":29,"ūvi":42,"Maz":108,"Mat":72,"ūve":114,"ūrā":242,"īvā":296,"ītī":76,"īzi":177,"īze":130,"ītē":23,"Mod":95,"Mol":50,"Mon":122,"Mos":23,"Mor":85,"Mot":31,"Moz":28,"NS ":43,"īvī":139,"Mež":62,"īvē":41,"Mih":26,"Mik":53,"Mie":24,"īrā":42,"Mic":50,"īvn":200,"īvm":36,"īvp":28,"īvo":1046,"īvu":146,"Mir":23,"īvs":154,"Mil":64,"Min":133,"īva":467,"īve":148,"īvi":298,"ūvē":84,"ītā":552,"NO ":49,"īz ":81,"īsā":29,"īlī":22,"īrs":118,"īru":139,"īri":268,"īsa":30,"īnā":148,"īmē":659,"Mur":22,"Mus":124,"īra":120,"īt ":399,"ītu":205,"īnī":56,"ītr":41,"īts":565,"ītn":23,"īto":115,"īti":392,"īst":852,"īss":36,"īmī":130,"īsl":28,"īsk":119,"īsi":181,"īkš":28,"īte":151,"īta":808,"īne":41,"īni":507,"īno":33,"īns":285,"īnu":169,"īme":446,"īmi":343,"īmj":45,"īmo":31,"īmu":34,"īms":40,"īna":462,"īkā":26,"īs ":468,"ījā":106,"īpa":426,"Sāk":57,"Rēz":54,"XX ":24,"кий":32,"Wor":39,"Wil":32,"Win":29,"War":28,"Vul":29,"Vor":36,"Vol":134,"Viņ":206,"Viļ":33,"Vis":230,"Vit":48,"Vla":36,"ču ":517,"Zaļ":27,"Zie":391,"Zin":62,"Zil":29,"Zel":41,"Zem":525,"之":22,"Tēr":50,"三":24,"на ":32,"Tā ":634,"Tāl":53,"Tās":150,"Sēr":28,"Sēl":24,"Rīg":820,"Svē":102,"Sys":26,"Sve":45,"Spē":56,"Sul":33,"Spā":87,"Str":115,"Stu":53,"Sti":60,"Sto":87,"Sta":349,"Ste":80,"Teh":22,"Tec":22,"Tem":25,"Teo":25,"Tei":33,"Tel":43,"Tam":51,"Tan":40,"Tas":583,"Tar":26,"Tau":65,"Tai":63,"Taj":81,"Tak":27,"Tal":71,"Sko":119,"Sku":25,"Ska":110,"Sim":41,"Sil":58,"Sig":27,"Sir":32,"Sin":63,"Sid":28,"Sie":22,"Sib":28,"Nāc":29,"Ser":100,"Sen":122,"Sel":26,"Sem":34,"Sek":22,"TV ":22,"Spa":42,"TS ":22,"Spi":27,"Spe":35,"Spo":27,"Soc":53,"Sol":25,"Som":114,"Son":25,"Slo":56,"Smi":29,"TP ":38,"Mēr":47,"Mēn":64,"Jūr":108,"Jūl":32,"SV ":351,"Run":31,"Rum":56,"Rub":29,"Sai":29,"Sah":30,"Sak":57,"Sam":35,"Sal":189,"Sac":23,"Sab":29,"Sae":27,"Sad":27,"Sco":41,"Sci":36,"Sch":39,"Sav":262,"Sat":49,"Sau":200,"Sar":143,"Sas":60,"San":93,"ови":22,"Mār":46,"Māk":28,"SI ":51,"Res":30,"Rie":159,"Rau":22,"Rec":23,"Red":33,"Rei":58,"Ren":27,"Rep":321,"Rob":99,"Rod":49,"SR ":106,"Ros":76,"Ron":23,"Rom":176,"SS ":24,"Līg":32,"Līd":101,"čūs":27,"Reģ":30,"SO ":60,"Pēc":116,"Pēt":62,"Vai":65,"Vel":97,"Ven":119,"Vei":37,"Vec":107,"ски":39,"ска":25,"ско":22,"Vas":82,"Van":50,"Val":406,"Var":89,"Vaš":22,"Vid":291,"Vie":179,"Vir":65,"Vil":141,"Vik":70,"Vin":38,"Ver":44,"Ves":70,"Ukr":55,"Ung":47,"Uni":125,"Urs":73,"Mūz":68,"Mūs":71,"Ērg":30,"Uz ":28,"Čīl":23,"ēcī":76,"ēdā":27,"Trī":27,"ēj ":73,"ēgu":33,"ēgt":73,"ēkl":37,"ēko":23,"ēki":215,"ēku":376,"ēkt":35,"ēks":180,"ēm ":819,"ēka":504,"ēji":561,"ēju":643,"ējs":400,"ējr":35,"ējp":24,"ējo":477,"ēl ":109,"ēdē":151,"ējd":44,"ēja":1223,"ēmu":365,"ēma":558,"ēmi":221,"ēli":303,"ēlo":171,"ēln":72,"ēls":126,"ēlu":113,"ēla":172,"ēle":379,"ēc ":930,"ēci":41,"ēdu":26,"ēdz":296,"Pāv":29,"Pār":113,"ēda":98,"VI ":40,"ēde":49,"ēdi":142,"Ter":125,"Tet":23,"ēvs":32,"ērē":61,"ērā":124,"ērķ":215,"ēsē":144,"The":138,"ērī":91,"ēsā":29,"ēze":148,"Tib":26,"Tie":134,"ēsī":28,"Tim":41,"ētā":792,"Tir":25,"ēzi":135,"ērš":71,"ētī":178,"ēzu":41,"ēvē":143,"To ":233,"ēne":214,"ēni":174,"ēna":102,"ēns":137,"ēnu":91,"Top":34,"ējā":633,"Tor":80,"Tom":55,"Tos":31,"ēkā":56,"ēr ":112,"ējī":33,"ēpo":30,"ēlā":185,"ēs ":404,"ēt ":252,"ēlē":492,"Nīd":49,"ēra":721,"ērb":56,"ērd":31,"ēre":37,"ēmā":142,"ēri":669,"ērk":23,"Tro":39,"ērn":117,"ērp":23,"ēro":293,"ēlī":53,"ērt":389,"Tri":83,"ērs":217,"ērv":101,"ēru":97,"ēmē":83,"Tre":69,"ērz":31,"Tra":108,"ēnā":38,"ēkš":30,"ēsl":23,"ēsu":45,"ēst":566,"ēnē":31,"ēte":130,"ēta":1832,"ētn":71,"ēto":79,"ēti":598,"ētk":66,"ētu":414,"ētr":34,"ēts":396,"Tur":208,"Tuv":48,"Tuk":33,"Tul":26,"šdz":152,"šde":24,"ši ":834,"šel":77,"ša ":338,"šas":321,"šau":228,"šah":27,"šai":236,"šaj":205,"šam":126,"šan":3757,"ēģi":168,"Šve":65,"šte":79,"šta":234,"šst":29,"šum":55,"šus":57,"švi":22,"šve":23,"šva":72,"špa":34,"škā":87,"špi":125,"šos":132,"šot":31,"Ēģi":41,"špu":29,"šs ":266,"šre":66,"šru":52,"šsa":28,"šu ":1883,"šla":74,"šme":77,"вич":26,"šo ":346,"šni":32,"šno":38,"šie":334,"šif":27,"šin":270,"šim":31,"šis":80,"ēļa":42,"ēļ ":153,"ēķi":49,"švā":48,"šze":25,"ēļu":170,"bju":45,"bje":249,"biz":35,"bis":145,"bit":116,"bio":155,"bir":59,"bik":26,"bil":410,"bin":496,"bij":2367,"bo ":36,"blo":48,"ble":47,"bli":610,"bla":73,"bku":154,"bok":22,"bol":1514,"boj":283,"bni":61,"bež":545,"bs ":212,"biļ":23,"bpi":124,"biņ":28,"bon":217,"bor":108,"bot":84,"bos":35,"bov":23,"be ":419,"bam":32,"ban":81,"bak":69,"bal":1602,"bai":173,"baj":47,"bag":46,"bac":26,"bad":23,"baz":174,"bau":36,"bat":100,"bas":2875,"bar":192,"azī":447,"bi ":165,"bei":323,"bed":22,"ber":482,"ben":195,"bel":118,"bek":85,"bez":361,"bes":245,"bet":713,"bib":56,"bie":860,"brū":72,"− ":42,"buļ":79,"−C":44,"ca ":396,"car":55,"cas":201,"cat":29,"cau":337,"can":76,"cab":38,"cam":74,"cal":52,"ce ":431,"cba":35,"blē":52,"bri":385,"bro":61,"bra":298,"bre":171,"bu ":1529,"blī":48,"bru":342,"bsk":33,"bso":54,"bse":25,"bst":77,"bur":350,"bul":250,"bun":25,"bum":299,"bud":74,"but":64,"bus":88,"boļ":39,"bva":41,"brā":110,"brī":410,"aka":880,"am ":2221,"ake":96,"akc":136,"aki":53,"aji":783,"ajo":724,"ajs":38,"adī":860,"aju":73,"adē":277,"aiz":832,"al ":300,"aja":469,"aje":41,"aij":61,"aik":1200,"ail":137,"aim":532,"ain":809,"aip":45,"acī":139,"air":752,"ais":4315,"ait":1125,"aiv":51,"aig":508,"adā":819,"aie":25,"aid":426,"aic":101,"aib":25,"ahi":35,"ahs":67,"aht":36,"abī":81,"ahr":22,"abē":108,"aha":93,"agl":88,"abā":487,"agi":39,"agr":218,"ags":92,"agu":84,"agn":169,"ago":152,"ajā":2224,"anv":34,"anu":718,"anz":65,"ano":394,"ann":140,"ant":1775,"ans":674,"anr":123,"ane":123,"ang":611,"anh":30,"ani":1185,"anj":22,"ank":275,"anl":23,"ap ":332,"ana":3035,"anc":401,"and":1589,"amu":130,"amm":408,"amo":175,"amp":185,"ams":603,"ami":532,"ahā":45,"adž":91,"ame":775,"amb":217,"ama":1351,"ao ":25,"alv":1226,"alu":519,"alt":791,"als":1977,"alp":208,"alo":1383,"aln":628,"alm":121,"all":272,"alk":231,"alg":97,"ali":1208,"alc":61,"ald":1013,"ale":584,"alf":96,"agā":163,"ala":1330,"alb":172,"Šaj":103,"an ":784,"aks":1610,"akr":121,"Šar":24,"aku":213,"akt":789,"ako":151,"akn":91,"akm":124,"akl":47,"aba":1096,"abe":95,"abi":630,"abl":44,"abo":176,"abp":131,"abr":109,"abs":116,"abv":34,"abu":136,"ae ":655,"aca":38,"aau":77,"ad ":480,"ac ":37,"afr":25,"aft":53,"afi":105,"ai ":5352,"aga":835,"age":107,"ael":27,"aei":29,"ah ":32,"afa":26,"ado":609,"adr":151,"adl":29,"adn":49,"adm":304,"adi":802,"ade":142,"adz":161,"ads":379,"adu":517,"ack":35,"aci":939,"ach":47,"ace":507,"ada":1586,"act":88,"acu":34,"azn":207,"azm":31,"azo":137,"azi":230,"arš":263,"azu":55,"atī":1487,"azs":68,"atē":140,"aze":46,"aza":217,"azd":25,"atņ":23,"avē":54,"avā":267,"arž":51,"arī":2864,"arē":195,"az ":62,"asā":83,"asī":143,"asē":82,"Šob":33,"atā":330,"aye":23,"auž":79,"ba ":1526,"azā":276,"atū":468,"auņ":57,"avī":79,"atš":186,"auš":105,"at ":261,"alē":137,"arh":312,"aiž":27,"arg":217,"amā":271,"are":686,"ard":512,"arc":143,"akļ":72,"arb":1417,"ara":2574,"arp":1342,"aro":671,"arn":121,"arm":206,"arl":146,"ark":821,"arj":66,"ari":685,"aru":580,"Šie":27,"arv":25,"amē":73,"alī":910,"arr":34,"ars":430,"art":883,"au ":159,"asa":1301,"ary":36,"anā":994,"asf":22,"akš":415,"asi":550,"ash":28,"asc":35,"ase":553,"aso":90,"asn":297,"amī":170,"asp":375,"ask":709,"asm":50,"asl":59,"ar ":6199,"apb":88,"apa":727,"Šep":43,"akā":218,"ape":72,"apd":220,"aiļ":41,"apj":56,"api":222,"aph":30,"apg":734,"aiņ":241,"apm":273,"Šei":27,"apl":157,"apk":491,"apo":44,"app":36,"apr":520,"aps":483,"apt":349,"apu":118,"apv":358,"apz":415,"as ":38125,"alā":492,"aiš":99,"ava":1146,"ax ":25,"auz":73,"auv":33,"aut":1629,"avs":157,"apī":65,"avr":23,"avo":299,"avp":76,"anš":45,"avi":843,"ave":232,"ay ":49,"Šo ":72,"avv":82,"avu":357,"arā":580,"av ":331,"atb":314,"ata":845,"asu":217,"ast":3489,"ass":194,"anč":183,"anē":182,"asv":32,"atm":195,"atn":153,"atk":493,"atl":205,"anī":82,"atr":1790,"ato":1405,"atp":124,"ate":661,"atf":33,"Šis":89,"atc":32,"atd":107,"ati":895,"atj":54,"atg":171,"ath":38,"auc":511,"att":979,"ats":614,"alū":32,"atv":2153,"atu":973,"atz":132,"aul":1187,"aum":53,"aun":922,"aup":26,"aur":568,"aus":1232,"aud":1114,"apā":234,"aug":1739,"auj":444,"auk":1608,"Vēs":43,"Vēr":24,"ος":59,"ος ":59,"ς ":97,"ν ":22,"Zvi":157,"Zva":55,"α ":62,"Vār":66,"Vāc":268,"ēša":752,"Н ":147,"ий ":35,"ич ":26,"jeb":1453,"jer":106,"jek":399,"jel":23,"jen":64,"jet":57,"jev":88,"eān":337,"eāl":99,"eāt":94,"ji ":637,"aža":89,"aži":139,"ažk":37,"ažo":204,"ažr":77,"ažu":75,"ļģi":26,"jad":67,"izē":509,"jas":8640,"jau":557,"jap":53,"jar":34,"jak":65,"jan":85,"jam":931,"dēš":109,"jai":593,"izņ":119,"izš":65,"jda":46,"jni":88,"jol":22,"jon":751,"jom":89,"jot":574,"jos":350,"jor":71,"js ":1627,"dīb":475,"dīc":65,"jiņ":29,"jpu":34,"ск":94," zī":183,"jis":272,"jie":1464,"то":24,"ст":46,"jko":24,"jo ":387,"ažā":482,"itm":69,"itl":241,"itr":232,"inī":182,"itp":32,"ito":983,"itu":694,"itt":30,"its":171,"ity":36,"imš":47,"isk":6070,"ism":951,"isl":163,"iso":147,"isn":115,"imī":127,"isp":195,"iss":145,"inč":41,"isu":253,"ist":3658,"isv":78,"inē":330,"ita":561,"ite":695,"ith":30,"iti":844,"ilš":47,"inū":22,"ivs":26,"ivr":48,"ipī":32,"ivp":30,"ivo":171,"ivv":36,"ivu":117,"irā":703,"inž":48,"ius":102,"iur":38,"ium":46,"ipē":33,"iva":204,"ivd":29,"dē ":186,"ix ":38,"ivi":487,"inš":34,"ivj":45,"ivk":27,"ive":291,"ipr":208,"ipo":63,"ipu":70,"ips":96,"ipt":115,"ipi":127,"igž":89,"ipl":132,"cīb":793,"cīg":288,"ilā":148,"is ":8439,"ion":1944,"iop":23,"ašī":283,"ior":31,"igū":45,"ios":84,"iot":138,"ijā":2772,"iog":37,"iok":51,"iol":177,"iom":40,"ipa":229,"ikā":1450,"ipe":167,"iov":23,"ir ":14672,"imē":165,"iru":216,"irs":599,"irt":236,"ilī":94,"irr":24,"iro":905,"irm":929,"irn":124,"irk":171,"iri":558,"isi":459,"ikš":84,"ish":36,"inā":3097,"ise":117,"isc":158,"isb":151,"isa":605,"irz":314,"cīt":112,"cīn":190,"cīj":23,"cīk":97,"imā":119,"ire":99,"irg":402,"irb":90,"ira":259,"ird":185,"irc":39,"it ":228,"ilē":113,"cīz":63,"itū":160,"ivī":107,"izā":483,"dēļ":202,"ja ":7647,"itā":1332,"irī":150,"dēs":22,"dēt":124,"dēv":144,"dēj":602,"dēn":29,"dēm":188,"cīņ":106,"dēl":80,"isā":208,"irē":31,"iz ":254,"ivā":236,"itļ":164,"izz":43,"izu":55,"izv":798,"itī":222,"izr":274,"izs":752,"izt":200,"izp":558,"izo":166,"izn":131,"izm":1077,"izl":692,"izk":139,"izj":45,"irš":41,"izi":513,"izg":245,"ize":159,"izd":376,"izc":301,"izb":41,"iza":170,"itē":104,"isī":84,"eģi":394,"kij":77,"kil":129,"kia":33,"kie":620,"eģe":52,"kin":233,"ļās":38,"kip":24,"kir":36,"kis":66,"ļām":48,"keā":336,"km ":504,"kga":63,"ki ":1566,"kgr":28,"kaļ":69,"kaķ":31,"kho":57,"kaņ":374,"kej":204,"kel":71,"ken":50,"kes":27,"ker":86,"ket":457,"ļā ":507,"fāt":30,"ke ":111,"kci":508,"kdi":30,"kra":920,"kre":163,"klē":111,"kt ":222,"eīn":30,"klā":396,"ksa":250,"kse":83,"kmē":88,"ku ":2253,"kro":296,"klī":65,"kru":265,"kri":985,"koz":38,"kov":64,"kot":513,"km²":49,"kos":513,"kor":259,"kop":1110,"koo":61,"kon":1216,"kom":1199,"kol":578,"kok":286,"koj":143,"koh":29,"kog":34,"koe":27,"kod":306,"ks ":1859,"kme":260,"kne":183,"klu":183,"kls":75,"ko ":1627,"kle":136,"kla":494,"klo":153,"kli":573,"ļēj":87,"dīš":155,"još":397,"jus":320,"jum":2601,"jur":89,"jvi":27,"jsi":45,"būv":217,"būs":22,"būt":380,"eču":47,"ju ":3050,"dīg":518,"dīj":452,"dīn":38,"dīt":668,"dīz":28,"jra":36,"eče":56,"kaz":48,"kav":162,"kat":1010,"kau":385,"kar":1806,"kas":6913,"kap":301,"kan":621,"kal":1136,"kam":395,"kaj":1054,"kak":37,"kai":2823,"kad":430,"kab":198,"ka ":3723,"juš":280," Ga":455," Bā":54," Ge":158," I ":103,"guļ":52," Fo":211," Fu":230," Fr":535,"ļš ":124," Fi":190," Fl":70," Ha":441,"aķu":37," He":345," Bē":72,"aķi":22,"guš":31," J ":40," Go":172,"aķe":127," Gr":516," Gu":162," Gv":31," Gi":68," Gl":86," Ig":193," Dā":72," Ie":156," Id":25," Ib":23," K ":22," Cē":86," Hu":54," Bī":48," Ho":304,"ha ":116," Hi":232," Je":179," Ja":604," Iz":255," Iv":35," Dē":23," Ir":159," Is":56," It":130," Im":77," In":594," Ik":26," Il":59,"ham":85," M ":45,"han":159,"hai":42," Ka":1303,"hal":100," Ke":174,"hau":35," Ki":234,"har":152,"has":51,"hat":24,"aļ ":30," Jo":237," Ju":233,"hae":23," N ":55," La":2100," Le":272," Gā":24," Li":1175," Kl":266," Ko":784," Kr":1044," Kv":49," Ku":433," Ma":1344," O ":55,"aļu":335," Mi":483,"aļv":40," Dž":308," Me":654," Hā":28,"he ":281," Lo":254,"aļa":796," Ly":24," Lu":187,"aļi":184," Ne":334," P ":63,"а ":100," Na":328," Ni":163,"cā ":43," Mo":496," My":39," Mu":257," Hē":35,"hek":42,"hel":84,"hei":58," A ":102,"С ":41,"het":54,"her":196,"heo":49,"hen":64,"hem":51,"cāk":136,"cām":35,"hi ":25," B ":49,"cās":52," C ":203," Ap":260," Am":327," An":736," Ak":149," Al":533," Ai":198," Ag":50," Ah":36," Af":50," Ac":82," Ad":148,"aļā":556," Ab":144," Ba":1003,"aļē":89," D ":68," Az":34," Av":74," Au":758," At":403," As":243," Ar":519," Be":444,"hie":34,"hid":182,"hic":23,"hib":51," Bi":327,"hip":128,"hin":98," Bl":114,"him":44,"hil":44," Bo":338,"hij":70," Br":524," Bu":238,"his":79,"hit":143,"aļķ":22," E ":56," Ca":270," Ce":344," Ci":203," Ch":131," Cl":40," Cr":39," Co":292," Cu":37," F ":24," Da":629," Di":673," De":398," Dr":102," Do":202," Dz":227," Du":106,"hn ":22," Eb":22," Ed":95," El":193," Ek":92," Ei":676," Eg":26," Et":48," Es":44," Er":98,"hlo":75," Eq":185," En":121," Em":90," Ez":43," Ex":28," Eu":85," Ev":42," Fe":226,"aņa":180," Fa":110," H ":140,"aņd":30,"gma":49,"go ":413,"gme":45," Rī":858," Pū":23,"glu":49,"glo":94," Sā":102,"gls":48," Z ":22,"gle":241," Rē":60,"gli":344,"gla":305," Wo":56," Wi":95," We":44," Rā":30," Wa":65,"й ":60," Rū":43,"gog":33," Zu":31,"god":68," Zv":222," Zo":72,"aģē":26," Tī":49," Ze":604,"gno":38," Zi":551,"gni":35,"bāš":52," Za":124," Sī":40," Tē":78,"gna":34," Yo":26," Tā":889," Sē":77," Vī":58,"gs ":1150,"glā":76,"о ":42," Vē":121,"н ":27,"gol":71,"gon":119,"gos":60,"gor":256," Vā":354,"got":83,"gov":43,"gu ":870,"gnā":78," a ":72,"р ":25,"glī":122,"gro":67,"gru":596,"gra":755,"gt ":56,"gri":552,"gre":213,"aīs":153,"gtu":35," R ":37," Jē":211," Ov":27," Os":199,"gto":46," Ot":132," Or":178,"gts":50," Op":65," Po":516," Pl":245,"gum":401," Pi":541," Ph":55,"gul":224," Kā":157,"gua":28," Pe":329,"gub":86," Pa":1447,"gud":66,"gst":733,"gnē":109,"gsn":33," Nu":37," No":486," Ol":289," Ok":85," On":42," Oh":28," Og":53," Od":50," Jā":93," Of":27," Ob":41,"gta":111," Ra":371,"д ":27," Lē":28," Qu":23,"goš":45," Lī":215," Ro":565," Re":681,"grā":671," Ri":308," Rh":25," Mā":133," S ":101,"guv":148,"gur":196," Pr":570," Ps":28,"gus":188," Pu":147,"gun":194,"gvi":84," Lā":40,"gva":31,"bēr":112," Sy":39,"grī":79,"bēt":95," Sv":220," Su":203," St":793," Ta":1092," V ":48," Tj":24," Th":177," Ti":329," Te":399," Tr":346," Lū":27,"ļūt":29,"ļūs":33," Nī":78," To":521," Mē":139," Jū":171," Ru":196,"ļūd":92,"grē":114," Sa":1313," U ":30,"е ":61," Sh":52," Si":395," Sc":134," Se":442," Nā":75," So":290," Sp":307," Mī":29,"bēj":25," Sk":316," Sl":112," Sm":70,"bēm":82," Sn":22," Pē":221,"grū":45," Uz":96," Va":790," X ":41,"и ":54," Ve":526," Vi":1347," Vl":38," Pī":38," Vo":205," Vu":40," Tu":361," W ":26," Pā":146," Ug":37," Uk":57," Ul":29,"gzd":28," Un":191," Up":44," Ur":133,"gzn":376," Mū":173," ja":682,"cēš":42,"iak":26,"ь ":24,"iam":59,"ial":119," iz":4946,"ian":220,"ias":27," je":1511,"iat":108," im":336," in":1344," ik":136," il":190,"ic ":183," dē":302," iv":22," is":92," it":111," cī":105,"iag":25," ir":14600,"ibl":76," ka":7945,"ibi":285,"ibo":30," m ":158,"ibr":57," ki":241," fā":22,"ibu":58," ke":72,"iaz":26," jo":407," dī":55,"id ":25,"iba":66," bū":407,"ibe":125," ju":119," bē":90," ha":132," he":218," aģ":61," gl":287," gr":1494," go":110," gu":155,"ia ":270," gs":71," cē":148," id":175," dā":101," ie":4455," ig":94," hi":297," hl":48," ho":304," bī":86," hr":108," ht":33," hu":35,"iet":4005,"iev":1792," ni":63,"iez":227,"iel":3446,"iem":8120," ne":2383,"ien":7676,"iep":380," na":986,"ier":1197," p ":50,"ies":2658,"iee":100,"ied":2745,"ieg":992,"iek":4508," mu":445,"iej":39,"iea":35," mo":718,"iec":1164," mm":35,"ieb":72," ok":512," ol":76," on":40,"dāv":49," og":135,"ifo":180,"dām":160," of":307," jā":140,"dān":30,"ifs":22,"dās":334,"dāt":198,"dār":218,"ifr":46," ob":243,"dāf":28,"ife":75,"dāl":53,"dāj":251,"ifi":245," nu":108," no":9749," gā":212," le":458,"icr":40,"ics":32,"ict":31," li":2852,"icu":63," n ":74,"ico":88,"ick":37," eļ":37,"icl":22," la":3298," kv":261," ku":4521,"ici":543," cū":37,"ich":68,"ice":118,"ie ":1698," km":554,"ica":238," kl":531," kr":1914," ko":4523," me":1393,"idz":312," dž":28,"idu":626," mi":1711,"ids":336,"я ":56,"idr":632,"ido":1694,"idp":27,"idm":53," ma":1802,"idn":29," lu":115,"idi":317," lv":27,"ide":638,"idd":33,"ida":1167," lo":437,"dā ":1681," af":23," ag":155,"aša":197,"idā":226," ab":209," ac":105," ad":392," am":414," an":722,"aši":302," ap":4517,"ašn":47," ai":846," ak":503,"ašl":70," al":593," av":171,"ašs":53," au":2262,"ašr":63,"icī":260," ar":6236,"ašv":86," at":4655,"ašu":167," as":496," d ":35," ba":1482,"idē":338,"il ":50,"ija":12639," bi":2895,"ije":54," be":1390,"iji":165," bo":133," bl":146,"idī":348,"ijs":425," bu":382,"iju":1715," br":766," ca":410," e ":62,"im ":312,"ika":3320,"ige":58,"iga":284,"ii ":23,"igl":56,"igm":40,"igh":40,"igi":68,"igu":73,"igs":27,"igr":58,"igo":33,"ign":102,"ij ":163,"igz":407," b ":53,"iha":58,"ihi":54,"ihs":70," Zī":28,"iho":104,"ibī":64,"ik ":182,"icē":211," c ":63," er":23,"imo":88,"imn":195," et":118," es":227,"ims":83," en":251," em":85," ep":55,"imp":593,"idž":50,"imf":36," ei":45," el":849,"ime":522," ek":494," ef":77,"imi":811,"ieš":1802," fe":288,"ip ":24,"inc":281,"ind":539,"ina":1252," fa":282," ez":655,"imt":1282,"imu":237," ev":61," fu":829,"inn":34,"ino":729," fr":351,"ļļu":39,"inr":24,"int":1230," fo":740,"ins":796,"inf":318,"ašā":181,"ine":633," fl":69,"inh":61,"ing":589,"iež":570,"inj":23," fi":803,"dāš":37,"ini":1119,"ink":89," ge":43,"cī ":27," bā":233,"ioa":45," ga":5014,"iod":270,"ļļa":40,"inu":278,"inv":33," i ":44,"iko":225,"ikn":25," cm":50,"ikm":90,"ikl":215," co":68,"iki":144,"ikg":62," ce":962," ch":25,"ike":59,"ikd":24," ci":2295,"ila":297,"ilb":52," da":4272,"in ":197,"ieķ":378,"ikv":30," cu":46,"ikt":677,"iku":1045,"ikr":195,"iks":380," do":494,"ilp":256,"ilo":443,"ill":168," dr":234,"ilk":121,"iln":265,"ilm":237,"ieņ":198,"ilh":58,"ilg":175," de":1205,"ilj":175,"ili":579,"ild":409,"ilc":81,"ieļ":23,"igā":206," di":3087,"ile":125,"ima":242,"imb":308,"ч ":32," g ":36,"io ":204," eb":89," du":128,"ilz":47,"idū":60," dz":3456,"ils":2044,"ilt":346,"ilu":127,"ilv":752,"ль":27,"hs ":112,"bīg":65," sū":56,"bīd":30," vē":1059,"ми":22,"bīb":430,"ло":25," vā":1227," tī":340," zo":142,"ла":30," zu":23,"ле":23," rū":113,"ли":29," zv":544,"hok":186,"hol":226,"hom":35,"hon":30," za":193,"ко":45,"hos":37," ze":740,"hot":32," zi":2010,"hop":25,"hor":75,"ка":54," tē":247,"ки":52," sī":84,"hni":146," sē":228," tā":1796,"hno":151,"aņu":241," pū":63," rī":133,"aņe":35,"hme":29,"ин":34,"aņi":61," ww":36,"ий":38," sā":685,"ич":34,"aņo":46,"ри":31,"ро":53,"ра":42,"ре":36,"htt":39," zā":53,"hst":50,"ос":22,"ор":37,"ол":32,"aņģ":36,"ов":87,"hu ":43,"ое":22,"aņē":59,"нс":22,"но":31,"hro":158,"aņā":78,"ни":29,"hri":23," tū":103," vī":192,"ht ":30,"на":44,"bīr":30,"bīt":96,"bīs":120,"bīn":33,"cēt":118," ru":206," jū":849," mē":678," u ":116," sa":6813," nā":219," sf":46," se":1483," sc":42," si":1753," sh":33," sn":112," sm":289," sl":596," sk":1357," mī":111," sp":2239," so":351,"ве":29," qu":23,"ви":41," lē":170," t ":61,"во":24," ra":2342," kļ":152," re":2229," mā":826," ri":726,"cēn":32," kņ":49,"cēm":41,"cēl":179," ro":913,"cēj":228," lī":2107," pu":1266," pr":3156," ps":120," s ":86," lā":230,"ļņu":56,"ва":29," os":114," gū":38," ot":330,"ļņi":27,"hum":50," op":221,"ав":30," or":1147,"ан":46,"ļņa":76," jē":154,"ал":24," pe":651,"cē ":120," kā":2571," pa":8411,"ар":38," pc":24," pl":1389,"ая":29," po":1148," pi":5908," rā":73," x ":60," va":7044," ve":2127," pē":1285," uz":2903," vo":159," pī":83," vu":52," vi":7158,"ес":31,"ер":46,"ен":52," tv":38," tu":790," mū":750," ut":55," ur":52," up":232," un":10091," ug":66," pā":1765," ta":1745," st":3025," kū":39," sv":412," su":879," nī":29," tr":1203," lū":49," to":1082," th":155," ti":3365," te":2144,"aāt":29,"fi ":28,"fes":176,"fer":133,"feo":25,"fed":168,"ņi ":361,"feb":35,"fen":84,"fek":185,"fel":37," Ča":35,"ņin":63," Či":48,"ņie":244," Če":124,"fga":33,"faz":23,"fas":28,"ezī":69,"far":40,"fan":93,"fak":97,"fal":45,"fai":36,"ņbr":28,"fab":102,"ņem":387,"ņda":33,"fe ":34,"ņos":47,"ņot":124,"ņoj":49," Čī":26,"evē":231,"evī":174," ēz":45," ēt":31,"ņra":135,"fa ":173," Ēr":64,"ņs ":24," ēd":44," ēr":63," ēk":65,"esē":107,"esī":292,"erņ":62,"etā":683," Ču":25,"ez ":111,"erē":156,"erī":393," če":400,"ņji":23,"erģ":206," ča":23,"esā":195,"ezv":29,"ezu":188,"evā":40,"eza":38,"ezd":35,"etē":183,"ezm":33,"ezn":141,"ezo":141,"ezp":31,"etī":89,"erū":27,"ezt":29,"eze":817,"ezg":53,"erš":24,"ezi":201,"ezk":125,"etb":351,"eta":734,"ete":624,"eti":845,"elš":89,"eth":26,"etn":170,"etl":29,"esp":338,"emī":84,"esn":43,"eso":286,"est":937,"ņoš":22,"esu":212,"enč":28,"esr":33,"ess":340,"enē":213,"esv":35,"ev ":24,"epā":130,"emš":50,"eto":1378,"enī":969,"etr":698,"ets":222,"ett":58,"etu":1584,"etv":168,"ew ":38,"eve":83,"eva":376,"evo":146,"evn":29,"evi":1543,"enš":59,"eut":31,"eus":24,"ex ":26,"erā":709,"evu":397,"evr":24,"evs":91,"ey ":55,"epe":71,"ekā":290,"epi":258,"epj":76,"eph":37,"er ":1104,"epa":258,"eot":23,"egū":271,"eos":34,"eor":322,"ņu ":2105,"eom":53,"eol":176,"eop":39,"eon":61,"elā":1125,"es ":8988,"ept":197,"eps":32,"epu":444,"epl":47,"epo":71,"epr":190,"erk":66,"erl":157,"eri":2539,"erg":167,"ere":536,"emā":318,"erf":37,"ekļ":537,"erc":178,"erd":99,"era":1626,"erb":238,"et ":1093,"elē":112,"esk":278,"esl":110,"esm":388,"enā":1113,"ņus":89,"esi":374,"ekš":656,"esc":24,"ese":289," Ēģ":41,"esa":337,"erz":36,"ery":29,"erv":331,"eru":391,"emē":508,"ņve":32,"err":36,"elī":103,"ert":417,"ers":1648,"ern":467,"erm":855,"erp":67,"ero":595,"eki":366,"ekl":427,"ekm":150,"ekn":62,"eko":416,"ekr":305,"eks":1397,"ekt":1750,"eku":811,"ekv":109,"en ":402,"elb":111,"ela":1449,"eld":81,"egā":92,"elf":85,"ele":1367,"eli":980,"elj":22,"elg":122,"elm":98,"eln":210,"elk":50," Ģe":106,"ell":209,"elo":182,"elp":198,"elu":464,"elv":59,"els":567,"elt":409,"elz":306,"eo ":63,"emb":176,"ema":412,"edž":46,"emg":143,"ehā":103,"eme":2607,"emd":28," ģe":323,"emn":25,"emo":348," ģi":768,"emi":389,"emj":71,"emt":133,"emu":60,"emp":493,"ems":89,"ep ":41,"ene":1004,"eng":203,"enb":97,"ena":2039,"end":388,"enc":397,"eno":1358,"enp":40,"enm":45,"enn":52,"enk":331,"enl":88,"eni":1040,"enu":734,"env":1058,"ens":2464,"ent":2847,"ehī":26,"enr":79,"enz":110,"egš":47,"eog":110,"eof":26,"ejā":89,"eod":60,"egl":203,"ego":210,"ege":31,"egi":29,"eaģ":26,"egr":83,"egs":36,"egt":159,"egu":482,"egv":44,"ehn":276,"eho":29,"ecā":200,"ehi":58,"ek ":1542,"eib":23,"eic":317,"ecē":53,"ecī":715,"eis":229,"eir":180,"eim":142,"eil":42,"ein":253,"eih":26,"eik":735,"eie":54,"eid":2544,"eig":207,"edā":218,"eja":808,"el ":113,"edē":86,"eiz":614,"eit":215,"eiv":56,"ejs":177,"ebū":23,"edī":123,"ejo":81,"ejn":76,"eji":80,"eke":31,"ekc":120,"eka":402,"em ":6412,"eju":260,"ejv":23,"gaž":22,"git":41,"gis":49," īs":256,"gin":40,"aģe":63,"gie":240," īp":415,"aģi":37,"ght":32," Īr":60," Īs":47,"gaļ":93,"bās":254,"bāt":57,"bāz":178,"gi ":519,"bāj":44,"bāl":44,"bāk":295,"bān":43,"bām":208,"gen":244,"ger":118,"ges":34,"gel":54,"gej":38,"bā ":646,"ge ":89,"gab":646,"gad":2505,"gai":779,"gas":1682,"gar":786,"gau":305,"gat":362,"gav":387,"gaj":108,"gam":98,"gal":1339,"gan":1451,"ga ":1326,"frī":26," ķē":68,"aēl":30," ļo":206," Ķī":169,"frē":39," ļa":118,"žņu":81," ķī":292,"Ņuj":43,"fut":631," Ļe":28,"fta":36,"fun":189,"ft ":61,"fra":303,"fre":65,"fri":213," Ķe":55,"fu ":37," ķi":28,"fro":69,"aču":132," ķe":273,"for":1089,"fos":41,"fot":77,"fon":280,"fol":75,"ņak":24,"ņam":33,"ņas":543,"ņaz":78," ņe":38," Ņu":39,"fs ":175,"ņai":51,"ņa ":1018,"fle":24,"fla":25,"fli":29,"flo":42,"fic":294,"fig":49,"fij":235,"fil":435,"fik":181,"fin":189,"fir":37,"fis":214,"fiz":287,"da ":3140,"de ":574,"dac":25,"dab":346,"dak":25,"dal":823,"dai":150,"dag":58,"dae":523,"dat":711,"das":2562,"dar":1471,"dap":69,"dan":60,"dam":271,"dau":771,"dda":35,"cul":28,"cum":72,"cty":34,"cto":75,"cti":74,"cte":48,"cus":46,"cyo":60,"ceļ":571,"cle":31,"co ":36,"ciā":742,"cog":76,"con":50,"col":50,"com":55,"cor":52,"cop":28,"cot":97,"ciņ":28,"cs ":85,"ct ":32,"cro":67,"cu ":436,"cea":28,"ch ":124,"cer":171,"ces":699,"cet":115,"cen":1090,"cep":69,"cek":125,"cem":35,"cel":374,"ced":32,"ci ":164,"cha":72,"cia":105,"ck ":59,"cie":767,"che":97,"chi":53,"cho":24,"chn":24,"civ":69,"cij":3939,"cik":179,"cil":946,"cim":32,"cif":53,"cir":29,"cis":561,"cit":901,"ciu":34,"cin":303,"cio":768,"cip":250,"cm ":47,"cke":32,"cka":44,"ed ":130,"eba":49,"ebe":168,"ebi":87,"ebk":175,"ebo":22,"ebr":247,"ebs":44,"eak":105,"ean":79,"eal":83,"ear":51,"eas":36,"eap":54,"dzī":2001,"eat":203,"eau":46,"eb ":1292,"dzā":72,"dzē":373,"ea ":66,"efi":131,"efo":131,"efa":30,"efe":198,"ei ":338,"ega":166,"efs":31,"eej":99,"een":27,"edi":335,"ede":1042,"eda":646,"edz":1195,"eds":41,"edu":254,"edo":120,"edr":685,"eck":45,"ech":44,"eci":633,"ece":209,"eca":89,"ee ":59,"ecu":116,"ect":70,"ecp":24,"eco":62,"drī":349,"dz ":1238,"doš":285,"drā":113,"dy ":22,"dvi":33,"dve":34,"doņ":93,"dur":107,"dus":742,"dva":54,"duš":31,"dzv":49,"dzs":170,"dzu":124,"dzo":52,"dzn":78,"duā":33,"dzi":2913,"dze":1097,"dza":294,"dor":118,"don":402,"dom":421,"dol":232,"dok":188,"dow":29,"dov":34,"dot":762,"dos":455,"diņ":89,"dpo":22,"ds ":1372,"dmi":319,"dne":44,"dni":257,"diā":75,"dob":64,"dod":87,"doe":23,"doj":533,"dnī":68,"dun":24,"dum":173,"dul":78,"duk":158,"dug":33,"dub":73,"dua":23,"duc":94,"dri":640,"diž":52,"dra":350,"dre":139,"du ":1832,"dro":640,"drs":87,"dru":292,"dsi":206,"dsk":23,"dsm":48,"daļ":1548,"dic":163,"dia":195,"dib":246,"der":1382,"des":875,"det":70,"dev":375,"deb":147,"dea":22,"ded":49,"dec":38,"def":128,"deh":29,"deg":170,"dej":161,"dei":89,"del":244,"dek":100,"den":1101,"dem":139,"dep":138,"deo":95,"di ":661,"deļ":51,"dko":25,"dma":88,"do ":341,"deņ":285," Āz":143,"div":810,"diu":34," Ār":50,"dim":118,"din":621,"dio":398,"dip":35,"dir":59,"dis":577,"dit":40,"die":2511," Āf":113,"dif":53," Āg":35,"dig":41," Ād":23," Āb":22,"dij":481,"dik":129,"dil":24," ār":250," āt":154," āp":26,"daž":632," ād":29,"ižu":32,"rgu":304,"raļ":69,"rhe":62,"raķ":99,"rha":55,"rhi":273,"rbī":529,"māz":34,"raē":35,"māt":506,"iža":92,"rga":943,"jš ":26,"ri ":1063,"ižk":30,"rgi":83,"iže":26,"rge":111,"rbā":31,"rgs":168,"rgo":78,"ret":778,"res":696,"rev":103,"reu":25,"rez":384,"rfa":28,"māc":507,"māj":191,"māk":627,"māl":154,"mām":145,"mān":158,"rfo":45,"mār":102,"mās":220,"rdu":246,"rds":349,"rg ":40,"rdz":219,"reb":22,"rea":246,"ree":37,"ref":132,"rec":144,"red":582,"rei":822,"rej":271,"reg":233,"rem":185,"ren":387,"rek":208,"rel":259,"rer":29,"reo":23,"rep":144,"mā ":1508,"rda":365,"kļu":259,"rct":35,"rdo":96,"rdn":63,"rdi":228,"rde":305,"re ":399,"ņām":78,"ņās":22,"rbu":231,"rbs":88,"rco":80,"kļo":52,"kļi":136,"rci":226,"rch":63,"rce":59,"rca":34,"kļa":421,"rax":25,"raz":76,"rd ":116,"rap":69,"rar":28,"ras":4121,"rat":1049,"rau":810,"rav":242,"rbi":268,"rbo":531,"rba":232,"rbe":87,"raj":717,"lēš":123,"ņā ":197,"rai":643,"rah":53,"rag":283,"ran":1071,"ram":1238,"ral":520,"rak":1567,"rab":128,"raf":152,"rad":1141,"rac":100,"rpt":521,"rpu":97,"rpr":64,"rpp":23,"rpo":72,"rs ":2853,"rpe":30,"rpa":43,"riķ":45,"riņ":229,"rpl":24,"rpi":90,"rkā":93,"ror":63,"ros":500,"rot":765,"rom":435,"ron":900,"roo":23,"rop":1052,"roz":140,"rou":41,"rov":266,"rob":624,"roa":30,"rod":1744,"roc":512,"lī ":122,"roj":521,"roi":35,"riģ":56,"rol":216,"rok":423,"rof":236,"roe":58,"rog":545,"rno":46,"rns":56,"rnu":187,"rp ":573,"rgļ":35,"rna":307,"rež":135,"riā":400,"rne":333,"rni":347,"māš":32,"rmk":26,"māņ":51,"rmo":304,"rms":203,"kņu":24,"rmu":139,"ro ":308,"kņa":79,"rma":773,"rme":418,"rdž":57,"rmi":499,"reš":174,"rls":24,"rlo":32,"rli":126,"rgā":217,"ižā":28,"rld":35,"rle":24,"rla":201,"rn ":84,"rku":265,"rkt":251,"rks":248,"kļū":96,"rkn":35,"rko":61,"rki":73,"reģ":361,"rkl":58,"rke":63,"rka":377,"rbū":24,"reč":45,"rdī":30,"rja":44,"reā":83,"raž":305,"rje":48,"riz":128,"rdē":22,"rip":74,"rio":330,"rit":1293,"ris":2568,"riv":81,"riu":29,"rih":80,"rig":98,"rij":2366,"raš":66,"ril":99,"rik":885,"rin":683,"rim":290,"ria":161,"rib":107,"ric":188,"rid":231,"rie":3740,"rif":114,"rdā":161,"rk ":63,"roš":448,"mēd":92,"rož":23,"rsā":44,"mēj":294,"mēm":78,"mēl":71,"mēn":131,"mēs":84,"mēr":1231,"mēt":199,"rsē":59,"rtā":193,"rpā":40,"rud":44,"ruc":33,"rur":31,"roī":36,"rup":644,"run":357,"rum":1852,"ruk":350,"ruz":42,"rpē":106,"rus":878,"rut":84,"rva":432,"mē ":626,"rvi":327,"rve":210,"rvs":22,"līč":64,"roņ":62,"rvu":63,"ry ":107,"rsk":108,"rsl":46,"jūl":25,"rsi":451,"rkš":28,"rso":354,"rsp":142,"rsm":215,"rsn":57,"jūd":35,"rsd":23,"rsa":148,"rse":69,"rnā":130,"rta":1012,"jūt":39,"rst":685,"rss":51,"jūr":881,"rnē":53,"rsv":73,"rsu":128,"rtl":32,"rtn":59,"rto":248,"rnī":124,"rte":168,"rth":30,"rti":792,"rts":333,"rtr":97,"roč":76,"rtu":328,"līm":186,"līn":460,"riš":83,"līj":172,"līg":392,"līc":164,"līd":1562,"līb":213,"rt ":437,"līz":66,"līv":121,"līt":444,"rri":36,"rre":54,"rmā":762,"rra":96,"ru ":2726,"rmē":77,"rlī":45,"sab":330,"sac":442,"sad":310,"sag":154,"sai":857,"mēš":100,"saj":138,"sak":466,"sal":1082,"sam":248,"sbi":144,"rzī":41,"sap":162,"san":223,"sau":2277,"sat":285,"sas":1660,"sar":747,"sav":1100,"saz":48,"sa ":1042,"mēģ":49,"rsū":33,"rvē":141,"ruš":59,"ruņ":155,"rze":250,"ruā":39,"rza":50,"rtē":197,"līš":88,"rvā":75,"rzs":23,"rtī":278,"rzi":284,"saņ":59,"sho":31,"shi":22,"si ":826,"kš ":48,"sga":35,"nāv":198,"nāt":1807,"nās":1024,"sgr":33,"saī":148,"siz":24,"saž":44,"sie":635,"sid":107,"kšd":149,"sic":34,"sib":36,"kša":264,"sia":56,"sk ":23,"kšt":50,"sit":335,"kšu":53,"sir":69,"sis":1127,"kšs":52,"kšp":164,"sip":27,"sin":756,"kšn":32,"sio":148,"sil":229,"kšl":23,"kšm":75,"sim":594,"sij":423,"sik":99,"kšk":106,"sih":129,"sif":65,"sig":89,"nā ":1513,"sda":48,"sdi":356,"sbu":40,"se ":542,"sca":22,"sce":66,"sci":117,"sch":88,"sco":39,"sev":271,"ser":334,"ses":362,"set":54,"sfa":26,"sez":117,"sh ":43,"nāj":705,"nād":289,"nāc":166,"sfo":68,"nār":126,"nāl":1149,"nāk":422,"nām":427,"sdz":22,"sei":104,"seg":72,"sed":36,"sec":62,"sep":119,"sen":630,"sem":62,"sel":189,"sek":324,"sej":83,"spu":61,"spo":584,"siņ":49,"kšņ":49,"spr":198,"skā":2459,"spe":377,"spl":61,"spi":604,"kšķ":38,"spa":151,"sot":106,"sov":22,"sol":116,"som":91,"son":503,"sor":227,"sos":35,"sod":41,"sof":59,"kšģ":28,"soj":40,"soc":383,"su ":834,"smē":44,"nču":194,"sjū":46,"slī":114,"kšž":25,"smā":173,"nča":33,"sra":53,"st ":664,"slē":278,"mīt":118,"mīn":84,"mīk":32,"mīl":44,"mīg":202,"slā":344,"mīb":243,"mīd":29,"ss ":875,"sli":399,"slo":148,"slu":68,"sfē":192,"sla":592,"sle":81,"ski":1153,"sko":1104,"sks":400,"skr":170,"sku":481,"ska":4650,"ske":410,"sno":53,"kšē":181,"sns":27,"sna":41,"nāš":340,"sni":516,"sng":29,"sne":180,"kšā":24,"smo":229,"shē":40,"sms":242,"smu":340,"so ":75,"sma":1140,"smi":429,"seš":62,"sme":291,"soš":166,"nēt":549,"nēs":72,"nēm":160,"nēj":281,"nēz":87,"stā":3123,"sze":116,"suā":44,"stē":996,"stī":1710,"svā":52,"sse":108,"ssa":247,"sso":61,"ssk":34,"ssi":62,"ssv":26,"kūv":24,"smī":51,"ssp":22,"snē":32,"ste":1133,"sta":3898,"stm":49,"stn":212,"sto":1125,"stp":85,"sti":2606,"stl":22,"stv":62,"stu":1969,"str":3208,"snī":38,"sts":1496,"sud":63,"sub":69,"spā":202,"sug":619,"sul":139,"sum":128,"sup":49,"sun":42,"sus":127,"sur":82,"spē":1745,"suv":25,"nē ":308,"sva":373,"sve":113,"svi":110,"soņ":53,"spī":75,"nēš":36,"tai":828,"taj":702,"tak":124,"tal":256,"taf":30,"tag":177,"tab":171,"tac":309,"tad":222,"tbi":216,"tba":91,"tav":303,"tau":931,"tat":537,"tas":3364,"tar":1920,"tap":52,"tan":395,"tam":934,"tce":25,"te ":724,"tbo":1157,"tbr":38,"tdi":61,"tda":95,"svē":166,"suņ":88,"stū":113,"svī":41,"ta ":6165,"ozā":24," št":206," ši":160,"pa ":730," šo":253," ša":314,"iķu":54," še":87,"iķi":331," Šv":81,"iķa":30," Št":29,"iķe":39," Šo":131," Ši":163," Še":104,"ovī":25," Ša":199,"ovē":187,"osū":27," Šķ":45,"pdr":26," šā":147,"iļu":60," Šī":134,"kā ":4239,"iļi":22,"pci":27," Šā":65,"iļa":27,"pe ":174,"par":3884,"ozī":528,"pat":548,"pas":2092,"pav":354,"pau":149,"paz":430,"pba":51,"pac":124,"pad":181,"paa":70,"pab":25,"pag":415,"pak":786,"pal":445,"pai":59,"pap":138,"pam":682,"pan":195,"pc ":24,"iļā":27,"paļ":30,"pha":38,"paņ":40,"phi":54,"pga":664,"kāz":22,"pi ":142,"pgr":35,"kād":424,"kāb":641,"kāc":170,"kāp":196,"kān":152,"kāt":32,"kās":1168,"kār":1333,"paā":29,"kām":189,"kāl":251,"kāk":39,"kāj":89,"pea":50,"pec":223,"ped":72,"pdz":192,"pen":158,"per":1152,"pet":33,"pes":264,"pei":23,"pel":224,"pek":146,"peļ":35,"pla":1300,"pli":106,"pgā":32,"ple":241,"plo":77,"plu":29,"pka":321,"pko":110,"pja":69,"iļņ":141,"pjo":49,"pju":63,"pie":4300,"pig":22,"paš":762,"pij":105,"pil":2275,"pin":133,"pio":305,"pir":966,"pis":387,"pit":151,"poz":106,"por":630,"pop":247,"pot":153,"pos":221,"poj":154,"pog":70,"pon":210,"pok":25,"pol":767,"pod":51,"ps ":241,"ppu":25,"ppl":37,"piņ":42,"pkā":95,"iņu":475,"iņs":42,"iņo":51,"kāņ":216,"iņi":157,"iņj":22,"pme":52,"pma":68,"iņa":746,"po ":65,"gžņ":89,"pni":63,"pne":24,"psu":31,"pst":387,"iņķ":170,"pta":564,"pse":42,"ņķo":49,"ņķi":80,"psi":158,"psk":78,"ņķa":92,"pso":28,"ptu":237,"ptv":61,"pub":494,"pte":121,"pti":177,"pto":59,"pnī":27,"plū":201,"plē":184,"pra":501,"jīg":32,"plā":202,"pmē":120,"pru":34,"psa":72,"pu ":321,"pmā":37,"iņā":150,"pri":1091,"pre":942,"pro":1797,"plī":101,"poš":65,"pož":46,"prē":65,"prī":60,"psē":31,"pur":145,"pmū":26,"pus":720,"put":341,"pum":233,"pun":290,"ņģē":37,"iņš":323,"pul":349,"poļ":35,"pva":35,"pvi":404,"prā":125," Ņū":40,"ptū":22,"pzi":58,"ptī":28,"prū":42,"ņģe":27,"pzī":361,"lā ":1304,"lār":401,"lās":590,"lāt":209,"lām":367,"lān":215,"lāp":55,"lāj":219,"lāk":1056,"lāg":85,"lād":144,"lāc":180,"lāv":61,"lāz":36,"iša":70,"iši":31,"išu":56," Ži":23," Že":35," Ža":46,"lāč":85,"lāš":22," ža":153," že":29," žu":96,"lāņ":65,"išķ":223,"išņ":36,"ņēm":347,"qua":27,"lē ":420,"quu":124,"que":29,"qui":79," šī":215," šķ":525,"lēd":76,"lēc":25,"lēm":141,"lēn":178,"lēk":87,"lēl":48,"lēj":178,"lēg":138,"lēs":295,"lēt":383,"lēr":42,"lēp":64," šū":130," Ūd":31," ūd":556,"ra ":3874,"ežo":374,"ngo":130,"ngi":47,"eži":331,"ngl":303,"ngv":61,"ežu":144,"ngu":183,"ngr":212,"ngt":36,"ngs":108,"ni ":1261,"nge":139,"eža":263,"nga":229,"ncē":85,"nha":38,"nhi":24,"nhe":54,"neg":80,"nej":92,"nei":201,"nel":264,"nek":293,"nen":247,"nem":97,"nep":336,"neo":76,"ner":698,"net":418,"nes":1590,"nev":218,"ndz":31,"ng ":184,"nea":224,"neb":55,"nec":44,"ned":153,"nfi":27,"nfo":273,"iān":89,"iāl":1004,"nfl":40,"nfr":48,"nez":35,"iāc":169,"nfe":88,"nci":571,"gļi":38,"nce":476,"nch":29,"nca":32,"ne ":1014,"nbu":52,"ndu":273,"ndr":359,"nds":68,"ndo":248,"ndi":727,"nde":437,"nda":780,"gļu":239,"nak":85,"nal":235,"nam":315,"nan":170,"nar":136,"nac":383,"nad":66,"nae":90,"naf":23,"nag":57,"nai":923,"naj":193,"nab":23,"nbe":61,"nd ":206,"nav":456,"nau":117,"nat":306,"nas":3073,"naz":22,"na ":4384,"muļ":23,"muš":43,"hēm":45,"mtā":193,"ntā":395,"nsē":48,"noš":193,"ny ":39,"nvi":1034,"nux":29,"nve":133,"nva":62,"nuk":32,"nul":25,"num":139,"nus":295,"noī":26,"nto":950,"ntu":416,"nts":925,"ntr":875,"nti":950,"nth":28,"nta":799,"nte":768,"nsu":59,"nsv":36,"nsp":162,"nso":88,"nst":540,"nss":27,"nkū":25,"nsf":27,"nse":131,"nsg":24,"nsi":192,"nsl":44,"nsk":554,"nsa":85,"nu ":3157,"nmē":32,"iču":24,"ičs":33,"nrs":66,"iči":25,"nri":78,"iča":38,"nra":85,"nt ":98,"hīd":23,"nkā":234,"niņ":82,"ns ":4213,"noc":58,"nod":505,"noa":58,"nob":42,"nog":200,"nof":35,"nok":292,"nol":424,"noj":304,"nop":85,"nom":610,"non":115,"not":1518,"nos":1301,"nor":435,"nov":804,"noz":696,"npa":23,"niķ":23,"nne":83,"nna":118,"ngļ":211,"nno":31,"nni":41,"nme":24,"nma":29,"iāņ":39,"ežģ":48,"neš":108,"ndž":62,"ežī":26,"ežā":198,"ngā":136,"nn ":32,"nla":94,"neļ":30,"nle":25,"no ":5915,"nke":26,"nki":45,"nkc":180,"nka":173,"nku":121,"nko":53,"gļū":52,"nks":81,"nkt":272,"nkr":126,"nja":39,"ndī":35,"njo":28,"nij":1586,"nig":62,"nif":36,"ndā":228,"nie":3147,"nid":123,"nic":83,"nia":73,"nk ":29,"niz":488,"ndē":98,"niv":242,"nis":2463,"nit":151,"nir":27,"nio":28,"nip":37,"nim":128,"nin":84,"nik":329,"nil":57,"ogs":183,"ogr":713,"ogu":116,"ogi":122,"ogl":96,"ogo":65,"ogn":46,"oga":163,"obā":56,"ņš ":323,"obī":23,"ocē":27,"oho":29,"ohn":25,"oha":113,"odē":120,"ois":41,"oin":31,"ocī":54,"odā":459,"iģi":165,"iģe":27,"ok ":31,"gša":114,"oju":1293,"odī":31,"ojo":586,"oji":109,"oje":183,"oja":1186,"ol ":51,"oce":578,"och":26,"oci":581,"ock":58,"oco":46,"obs":66,"obu":91,"oca":32,"ode":548,"odi":313,"odo":337,"odn":41,"ods":132,"odr":212,"ocy":51,"jā ":4204,"of ":167,"oda":2367,"oel":45,"oef":31,"oei":29,"oen":27,"odz":39,"odu":575,"jām":1042,"jān":27,"ofi":331,"jāj":22,"jās":923,"ofs":60,"oft":50,"ofo":72,"ofe":168,"jād":51,"jāb":32,"ofa":27,"nzē":40,"oal":24,"oak":87,"oba":54,"od ":93,"obo":33,"obr":164,"obl":72,"obj":208,"obi":154,"obe":704,"nsī":306,"ntē":146,"ة ":22,"nza":50,"nzi":41,"nzo":53,"ntī":340,"nvā":46,"ntū":58,"otā":1010,"orķ":25,"orī":50,"opš":220,"jēj":29,"osā":36,"jēd":148,"orē":91,"ows":32,"ovā":78,"gūš":56,"ozu":30,"otī":57,"ozo":189,"oze":115,"ozi":143,"oza":288,"otē":115,"orņ":38,"otu":305,"oti":1556,"oth":36,"ote":602,"ott":59,"olū":125,"ots":723,"otr":369,"onī":135,"oto":694,"otn":222,"onē":180,"ost":506,"gūt":168,"osu":40,"ota":947,"osi":149,"osk":156,"ose":88,"osf":203,"onā":1526,"osp":73,"oss":48,"gūs":114,"gūr":45,"osm":456,"osl":225,"oso":109,"orā":211,"owe":22,"ovi":389,"ovg":31,"opī":120,"ovs":143,"opē":176,"ox ":29,"jē ":37,"ova":643,"ove":137,"opā":322,"oun":55,"ous":25,"our":37,"out":44,"opm":30,"opo":347,"opi":253,"opl":68,"ope":301,"okā":278,"oph":40,"opa":835,"os ":3856,"opu":475,"opr":71,"opt":86,"ops":93,"ojā":198,"or ":99,"oot":22,"oor":77,"ork":107,"orl":52,"orm":1086,"orn":222,"oro":248,"orp":128,"olī":129,"orr":39,"orc":49,"okļ":89,"ord":492,"ore":247,"omā":452,"orf":64,"org":872,"orh":26,"ori":1689,"osa":935,"ort":723,"ors":691,"orv":185,"oru":295,"omē":208,"orz":33,"ory":23,"olā":244,"ot ":1449,"m² ":52,"orb":148,"ora":653,"olē":98,"olb":29,"ola":1498,"old":135,"on ":659,"oli":1155,"oll":82,"olk":110,"olf":57,"ogā":71,"ole":441,"olg":32,"ols":417,"olt":46,"olm":107,"oln":31,"olo":1146,"olu":413,"ogē":50,"oka":339,"om ":55,"oki":85,"okh":49,"oke":542,"okr":192,"oks":506,"oko":208,"okl":145,"okm":47,"okt":89,"oku":309,"ona":1194,"ond":224,"ogļ":73,"onc":136,"onf":111,"one":173,"ong":158,"oni":1530,"onk":159,"onn":56,"ono":626,"onr":30,"ons":1271,"ont":491,"onu":567,"onv":60,"gšē":65,"ony":39,"onz":36,"oma":1047,"ome":546,"omb":95,"omi":730,"omm":51,"omj":98,"omp":487,"omo":191,"omt":26,"omu":284,"oms":200,"op ":56,"la ":3657,"eķu":303,"kuļ":50,"ktū":291,"kuš":110,"eķi":82,"le ":615,"eļd":25,"lce":40,"eļe":30,"eļa":635,"lci":93,"eļi":128,"eļv":32,"eļu":421,"eļo":286,"eļr":165,"gā ":654,"lde":303,"lda":256,"ldo":68,"ldn":139,"ldi":101,"ldv":22,"ldu":71,"lds":71,"ldr":27,"lab":686,"lac":74,"lad":63,"lah":38,"lag":32,"laj":474,"lai":2379,"lak":285,"lan":766,"lam":314,"lap":246,"lar":131,"lat":1524,"las":2361,"lau":462,"lav":247,"lay":44,"laz":38,"lba":32,"ld ":81,"lbe":128,"lbu":171,"lbr":107,"kvi":68,"kve":97,"kva":316,"kuu":25,"kut":36,"kmū":49,"kus":576,"kur":4124,"kup":73,"kun":181,"kum":1594,"kul":627,"kuk":32,"kuj":25,"koš":92,"krā":588,"koņ":106,"kta":560,"kte":51,"knē":45,"kss":117,"kmī":26,"ksp":156,"ksu":120,"kst":1947,"ksk":31,"cūk":37,"ksi":433,"kso":59,"ksn":205,"ksm":118,"ksl":373,"ktr":699,"kts":634,"ktu":492,"kti":662,"kto":408,"ksī":103,"ktē":73,"kuģ":144,"krū":64,"ktī":473,"ksā":78,"krē":127,"ktā":172,"krī":67,"fēr":208,"ksē":49,"lpo":192,"liņ":107,"lps":165,"lkā":113,"lpe":42,"lpi":52,"dū ":66,"ls ":2358,"lpu":76,"lok":186,"lon":250,"lom":194,"lop":98,"lor":215,"lod":1255,"loc":172,"log":309,"loj":39,"liģ":139,"lpa":98,"los":267,"lot":233,"lov":125,"loz":171,"lno":58,"lnk":23,"lni":256,"lne":55,"lob":58,"lnv":34,"lnu":164,"lns":198,"eņo":30,"lmi":75,"eņi":138,"lme":73,"eņe":150,"eņb":28,"lma":269,"eņa":350,"gāļ":23,"lna":214,"lmu":99,"eņu":143,"eņr":134,"lms":33,"ltk":80,"lti":424,"ltn":68,"lto":53,"ltr":29,"lts":217,"lnī":100,"ltu":221,"lud":54,"luc":32,"lub":111,"lug":36,"lpā":54,"lsi":33,"lkš":22,"lsk":140,"lsm":135,"lsn":30,"lso":131,"lsp":34,"lss":53,"lst":1735,"lsu":49,"eņķ":69,"lv ":26,"lta":371,"lte":188,"lkņ":24,"lmā":69,"lu ":1807,"eņē":35,"lnā":79,"lse":64,"dūd":48,"lsa":85,"lt ":35,"lhe":59,"eļā":42,"lgu":44,"lgs":36,"lgt":25,"lgr":44,"lgo":65,"lbā":28,"lge":25,"li ":1126,"gāz":197,"lga":178,"gār":144,"lfr":22,"gās":198,"gāt":113,"gān":110,"gāj":129,"gāk":229,"gāl":68,"gām":215,"gāc":40,"gād":124,"lfa":193,"lez":112,"lev":95,"les":1626,"let":176,"ler":213,"leo":70,"lep":67,"lem":358,"len":329,"lek":1222,"lei":141,"lej":190,"leg":36,"lef":47,"led":254,"lec":71,"lea":23,"lfī":26,"lls":27,"ldū":50,"llu":78,"lo ":480,"lla":140,"lle":160,"lgā":68,"lli":182,"leņ":73,"llo":82,"lko":184,"lku":31,"lks":97,"lka":119,"lke":27,"lki":33,"eļš":95,"leģ":37,"lkl":39,"leč":51,"ljo":123,"ldī":542,"eļļ":42,"lje":30,"ll ":123,"lja":61,"eļņ":24,"lit":895,"lis":1779,"lir":22,"lip":94,"lio":99,"lin":517,"lim":573,"liz":308,"ldē":33,"liv":35,"lic":208,"lid":326,"lia":94,"lib":49,"lik":929,"laš":379,"eļģ":25,"lij":964,"lig":100,"lie":3537,"lif":131,"ldā":34,"ma ":2997,"mac":23,"mai":882,"maj":174,"mak":194,"mad":68,"mag":255,"map":23,"lzī":23,"mar":276,"mas":1459,"mal":305,"mam":184,"man":1880,"maz":508,"mat":1537,"mba":149,"mbl":29,"mbi":138,"mbe":47,"mbr":158,"mbo":287,"me ":570,"mbu":166,"mda":33,"mde":27,"mdo":23,"med":310,"meg":22,"met":1247,"mes":1625,"mer":791,"mem":29,"mel":157,"men":1391,"mei":147,"meh":105,"mek":209,"mez":28,"hār":41,"mfo":25,"hān":110,"lva":502,"lve":648,"lvi":41,"luk":23,"loģ":845,"lup":35,"luo":34,"lum":401,"lut":87,"lus":425,"lur":30,"loī":24,"ly ":28,"loš":29,"loņ":26,"lvo":84,"lvu":57,"lož":83,"ltā":300,"gēn":73,"lsē":1122,"lzi":33,"lza":40,"lzc":242,"lsī":24,"lvā":83,"ltī":58,"lzs":43,"lvē":725,"ltū":225,"mpi":549,"mpe":254,"mpr":55,"mpo":156,"miņ":109,"mpl":189,"mpu":71,"mps":29,"mpt":22,"ms ":3789,"mog":37,"moc":27,"mob":84,"mod":286,"mon":399,"mop":26,"mok":98,"moj":43,"mom":24,"mol":217,"mor":220,"mos":1307,"mot":190,"mpa":147,"ešķ":123,"miķ":31,"mmā":27,"mmē":68,"msa":67,"mu ":2868,"mt ":26,"mtu":58,"mts":51,"mti":48,"mug":77,"mpā":152,"mss":33,"mst":25,"msu":24,"msk":121,"mte":23,"mta":1054,"mvi":24,"moš":35,"mur":78,"mus":558,"mut":78,"mui":109,"mul":121,"mum":318,"mun":151,"mpē":250,"muz":106,"maņ":208,"džu":100,"džs":35,"mga":146,"dža":84,"mi ":937,"maģ":28,"dži":70,"dže":114,"mju":201,"ml ":23,"mje":32,"min":1418,"mio":29,"ešo":30,"mil":392,"mir":718,"ešs":32,"mis":1428,"ešv":29,"mit":475,"ešu":1242,"miz":31,"mic":35,"eša":400,"mig":64,"mie":997,"ešd":35,"mid":33,"mik":313,"mij":472,"maš":145,"eši":317,"mo ":301,"meņ":339,"meļ":1113,"džā":37,"mko":36,"mm ":35,"mni":192,"mna":27,"ešā":114,"mež":103,"mmu":74,"mma":291,"mme":33,"vī ":36,"tšķ":209,"uņi":48,"uņo":84,"uņu":131,"tūt":53,"tūr":1186,"tūk":66,"tūc":93,"vīn":52,"vīt":70,"vīr":237,"vīg":77,"vīd":56,"vīb":202,"vīz":153,"vē ":134,"īču":22,"īča":67,"sūt":95,"vēl":461,"vēk":693,"vēj":244,"vēc":27,"Čeh":54,"vēs":501,"vēt":425,"vēr":790,"vēn":31,"vēģ":76,"Čik":22,"uļo":37,"uļv":73,"uļu":71,"uļi":36,"uļa":32,"vēš":46,"tļa":51,"tļu":59,"tļi":52,"tļo":22,"vā ":578,"vāj":43,"vāc":471,"vāl":70,"vāk":140,"vān":38,"vām":138,"vār":1021,"vāt":107,"vās":151,"vāv":76,"ržu":43,"uģa":52,"uģi":59,"uģu":36,"tņu":72,"ča ":145,"zra":210,"zru":92,"zjū":39,"tīd":188,"zlā":29,"tīb":1094,"zs ":189,"tīt":1190,"tīs":433,"tīv":934,"tīr":77,"tīk":237,"tīn":159,"tīm":268,"tīg":275,"tīj":350,"zte":106,"zti":66,"ztu":134,"čem":249,"ztv":96,"čet":144,"znī":263,"čer":57,"zsa":231,"rūd":56,"znā":215,"zmē":101,"zu ":183,"rūt":48,"zst":360,"rūs":59,"zsv":33,"znē":22,"zsl":25,"zsk":600,"rūk":50,"rūn":90,"rūm":55,"zsp":47,"rūp":88,"zva":673,"zvi":325,"zve":584,"či ":69,"zul":164,"zum":137,"zpē":51,"zus":63,"čie":24,"zsā":72,"tīņ":237,"zrā":47,"tīģ":32,"zzi":39,"zuā":39,"īģe":36,"tīš":166,"zvē":121,"rūš":31,"čs ":40,"zgl":134,"zga":142,"rš ":626,"zi ":317,"uār":73,"uāl":181,"zaļ":79,"zgu":59,"zef":34,"zej":232,"zdz":26,"zeb":60,"zdo":186,"uāc":50,"zes":430,"zen":285,"zem":1268,"zel":450,"zek":170,"zer":903,"ze ":308,"zce":422,"zbr":68,"zda":107,"zde":234,"zci":73,"zab":39,"īļu":88,"zai":49,"tēš":90,"zah":57,"zam":111,"zan":24,"zak":23,"zar":328,"zau":128,"zav":27,"zas":275,"zat":43,"zod":22,"zob":42,"tī ":235,"zos":43,"zot":69,"zor":28,"zop":22,"zom":26,"zon":281,"zol":154,"zof":144,"ziļ":81,"ziķ":104,"zpa":78,"zpr":45,"ال":32,"zpl":262,"ziņ":261,"zkā":32,"zpi":173,"zo ":48,"zma":918,"zme":35,"zmi":84,"zna":38,"zmu":41,"zno":68,"zne":198,"ršā":79,"zni":156,"zbū":81,"zka":34,"zkl":48,"zkr":193,"zla":642,"zgā":25,"zli":78,"zeļ":55,"zeņ":47,"rša":176,"zie":1720,"zid":135,"zij":625,"rši":100,"zin":1174,"zim":1926,"zil":123,"zik":669,"ršo":29,"zio":48,"zcī":73,"ršr":51,"zir":324,"zis":229,"zit":58,"ršu":90,"ziv":58,"tē ":318,"yst":46,"rņā":31,"sīk":51,"sīj":64,"sīm":27,"sīn":31,"sīs":28,"sīt":105,"sīv":85,"sīb":502,"sīd":98,"sīg":56,"tāž":25,"yon":65,"sī ":32,"tēģ":40,"īņā":28,"za ":232,"āža":23,"āžu":50,"īņu":254,"īņa":76,"sīļ":23,"tēk":46,"tēl":335,"tēm":821,"tēn":44,"tēj":348,"tēt":204,"tēs":73,"tēv":63,"tēr":175,"tēz":37,"īša":499,"īšu":52,"tā ":2912,"tāc":148,"tād":527,"tāk":538,"tāl":1306,"tāj":1760,"rķu":27,"rķe":31,"rķi":179,"yla":29,"tāļ":48,"rņa":66,"rņu":41,"tāš":25,"tāv":1332,"tāz":22,"tāp":79,"tām":538,"tān":411,"tās":1054,"tāt":740,"tār":419,"rīz":162,"rīn":188,"rīj":40,"rīk":162,"rīl":26,"rīv":387,"rīs":243,"rīt":190,"rīg":665,"rīb":593,"rīd":236,"rīc":210,"pūt":23,"sāļ":43,"rī ":2361,"rīš":55,"pūš":30,"sē ":157,"sēj":316,"sēk":27,"sēm":89,"sēn":52,"sēd":44,"sēs":30,"sēr":135,"sēt":1211,"ožu":58,"oža":54,"sāt":137,"sās":67,"sār":68,"sāp":24,"sān":37,"pš ":215,"sāc":43,"sāj":28,"sāk":643,"sāl":103,"sām":92,"ožā":27,"pšu":22,"rģi":226,"pša":33,"rģe":23,"rēķ":47,"sā ":333,"rēš":122,"rē ":110,"ww ":36,"āļu":173,"āļi":45,"www":36,"rēt":449,"rēs":31,"rēm":93,"rēn":56,"rēj":380,"rēk":33,"rēl":48,"rēd":88,"rāļ":69,"rāņ":44,"ošā":198,"rāš":23,"ws ":30,"ošī":35,"wor":28,"rā ":1557,"pļa":22,"rāk":869,"rāl":881,"rām":402,"rān":164,"rāg":38,"rāj":67,"rāc":432,"rād":816,"rāf":309,"rāb":117,"wer":27,"nže":53,"rār":31,"rāp":24,"rāv":135,"rās":553,"rāt":492,"rāz":74,"ošs":181,"ošu":115,"oši":415,"ošo":147,"oša":1342,"vze":33,"vvē":32,"āņi":41,"āņu":468,"vuš":26,"āņa":65,"war":33,"pīt":24,"pīr":89,"pīn":23,"pīl":42,"viš":195,"pīg":88,"pīd":81,"pīb":56,"vs ":473,"vri":37,"vst":90,"vsk":63,"vu ":956,"vus":141,"vmū":35,"vum":201,"vuk":61,"vul":71,"vva":94,"voš":86,"pīļ":68,"āša":527,"via":49,"nša":32,"vio":33,"vir":905,"vik":41,"vil":371,"vin":313,"vig":52,"vij":2101,"vic":31,"vid":1832,"vie":6969,"viz":69,"nšu":53,"vit":157,"nšt":29,"vis":1620,"vji":56,"vju":71,"vo ":339,"oņa":59,"veš":46,"oņi":162,"oņu":145,"viā":31,"vna":31,"vni":238,"vič":61,"vod":34,"voj":423,"vol":130,"vok":149,"von":248,"vor":27,"vot":667,"vos":64,"vpa":23,"viļ":118,"vpi":79,"viņ":404,"vpr":25,"vaņ":38,"vaļ":151,"vi ":493,"vgo":31,"ver":992,"ves":531,"vet":45,"vej":30,"vei":2843,"ven":961,"vem":34,"vel":158,"vek":42,"ved":123,"vec":250,"oļu":172," − ":35,"vda":91,"uzņ":254,"ve ":176,"val":3215,"vak":43,"van":198,"vam":39,"var":1672,"vat":135,"vas":1579,"uzē":26,"vab":47,"vad":1354,"vai":3704,"vaj":267,"pēļ":144,"va ":982,"mūž":31,"uvē":31,"uvā":81,"uzv":164,"uzl":72,"uzk":39,"urš":530,"uzi":73,"uzg":36,"uze":128,"uzt":192,"uzs":479,"uzr":64,"utī":62,"uzm":28,"utē":47,"uzb":144,"uzc":22,"uzd":109,"utā":171,"usī":47,"pēt":433,"usē":169,"usā":96,"pēd":188,"pēc":941,"pēj":691,"urģ":36,"pēm":32,"pēl":736,"pēk":384,"pēr":307,"urī":204,"urē":196,"uz ":1335,"urā":908,"umš":52,"uum":27,"upā":104,"upē":50,"ux ":33,"uus":124,"uvi":141,"uvo":31,"uva":357,"uve":337,"upī":23,"uvu":135,"usl":182,"usm":71,"usj":49,"usk":104,"mūk":26,"ukš":120,"usi":609,"unā":343,"usd":32,"use":127,"usa":206,"mūz":468,"usz":100,"usv":36,"usu":107,"ust":2158,"mūs":296,"uss":395,"mūr":69,"umī":24,"uso":49,"mūn":33,"utn":283,"uth":28,"uti":652,"ute":201,"uta":279,"utb":797,"utt":67,"uts":90,"utv":22,"utu":102,"uto":580,"utr":30,"unī":25,"us ":3775,"ulā":424,"oīd":99,"ulē":350,"ut ":97,"urb":65,"ura":1608,"urd":30,"urc":141,"umā":913,"ure":246,"urg":216,"uiž":110,"urj":22,"uri":851,"url":46,"urk":327,"urn":303,"uro":291,"urp":107,"ulī":38,"urs":344,"urt":245,"uru":1149,"urv":97,"umē":24,"urz":200,"unz":34,"ugš":195,"ujā":42,"uor":22,"upa":461,"ur ":573,"upj":23,"upi":127,"ukā":121,"upe":293,"upo":29,"upu":109,"ump":111,"ums":2443,"umu":2190,"umt":47,"umv":39,"umi":1048,"umk":31,"pāņ":98,"umm":30,"uml":29,"umo":1060,"umn":30,"uma":1511,"umb":191,"umd":66,"ume":317,"udž":74,"unt":48,"uns":163,"unr":34,"unv":30,"unu":158,"unl":25,"unk":486,"uni":663,"uno":67,"ugļ":37,"unc":25,"und":348,"una":264,"ung":150,"une":47,"up ":25,"uks":271,"uku":836,"ukt":791,"uko":36,"ukl":91,"uki":42,"ukc":69,"um ":175,"uka":140,"ulv":63,"ulu":136,"ult":666,"uls":122,"ulp":39,"ulo":29,"ulm":42,"ull":58,"ulk":243,"uli":203,"ulg":48,"ule":823,"ulf":35,"ugā":140,"ulc":29,"uld":110,"ula":397,"ulb":50,"un ":9898,"uid":83,"oģe":28,"udā":23,"oģi":861,"uil":24,"uis":38,"ucē":126,"ķa ":175,"mša":94,"uji":27,"ujo":73,"udē":94,"ul ":24,"uja":223,"ubā":54,"ugi":71,"ugo":30,"ugl":76,"pār":1984,"pās":27,"uga":887,"ugu":553,"ugs":799,"uha":22,"uj ":134,"uco":23,"pā ":453,"uda":124,"ude":104,"udi":282,"ķir":550,"ķis":433,"ķin":48,"ubs":106,"ķij":62,"ķim":24,"ubu":76,"ķie":106,"ķid":184,"uca":88,"ue ":35,"uce":66,"uci":96,"uer":36,"pāc":26,"pān":369,"pām":68,"pāj":65,"udu":86,"āču":68,"udr":142,"udo":32,"ug ":37,"udz":752,"uel":29,"ķet":39,"ķes":25,"ķer":270,"ķen":35,"ķel":35,"tuš":25,"ua ":31,"tzī":84,"uar":51,"ual":31,"uan":35,"ubi":59,"ubj":31,"ubl":522,"ube":111,"uba":77,"uag":22,"uc ":316,"ķi ":217,"tvā":37,"tzi":22,"tuā":134,"ttī":266,"trū":52,"ttē":198,"tza":38,"ttā":196,"nīš":44,"tyl":39,"tvē":108,"ķu ":470,"ty ":51,"toņ":39,"tve":395,"toļ":81,"tvi":1976,"tva":158,"tur":1909,"tus":646,"tuv":673,"ķo ":45,"tul":280,"tuk":47,"tun":72,"ķeš":66,"tum":1336,"tub":43,"tud":188,"tuc":22,"tug":34,"tpū":25,"trī":313,"trē":132,"two":22,"toš":435,"trā":1294,"nīc":430,"nīd":58,"nīb":728,"ts ":6404,"nīj":53,"nīg":331,"nīt":121,"nīs":25,"nīr":102,"nīn":27,"nīm":63,"tlē":35,"tre":229,"tt ":106,"oča":27,"tra":1972,"tri":1251,"oči":28,"trs":398,"tru":1883,"tro":2406,"tlī":43,"tu ":4075,"tmē":32,"tsa":92,"tse":174,"lūc":74,"lūd":44,"lūg":25,"lūk":137,"tsk":152,"tsl":24,"lūp":24,"tsp":162,"tsv":27,"tsu":24,"lūt":49,"tst":123,"lūs":138,"lūz":46,"tnē":293,"tta":34,"tte":67,"tti":291,"ttl":22,"tto":22,"tnī":27,"ttp":39,"ķus":35,"tma":80,"to ":1892,"tms":32,"tmo":123,"tml":23,"tmi":87,"tni":753,"tne":504,"tp ":39,"tna":35,"tns":112,"tnu":123,"tno":42,"tof":38,"tod":192,"toc":137,"toj":671,"tog":69,"nī ":207,"tob":116,"tov":52,"tos":547,"tot":998,"toz":23,"tom":540,"ton":588,"tok":294,"tol":346,"tor":2083,"top":326,"tpe":28,"tkā":47,"tpi":87,"tiķ":219,"tpa":63,"tpl":32,"tiņ":164,"tpr":24,"tij":940,"til":601,"tik":1958,"tif":82,"tie":4403,"tig":30,"tir":195,"tit":273,"tis":2277,"tin":593,"tim":104,"tip":378,"tio":312,"tia":75,"lša":104,"tic":277,"tid":70,"teā":78,"tju":24,"tiz":110,"lšu":27,"tiv":158,"tja":44,"tki":59,"tkl":212,"tko":27,"tkr":96,"tku":37,"tka":280,"tli":341,"teņ":82,"tla":152,"tgā":22,"tle":76,"tem":480,"ten":587,"teo":285,"tep":65,"tei":672,"tej":47,"tek":587,"tel":542,"teg":123,"teh":268,"teb":48,"tec":92,"ted":53,"tfo":35,"th ":68,"tet":45,"tes":1146,"ter":2575,"tgr":47,"ti ":2942,"tga":132,"tač":123,"tho":40,"the":191,"thi":29,"taļ":38,"tha":41," α ":31,"āpa":64,"āpe":142,"ākā":884,"āpi":34,"ājē":32,"ār ":34,"ājā":201,"āno":72,"āns":365,"ānu":327,"āre":82,"āmā":78,"ārg":36,"āra":470,"ārb":33,"ākļ":103,"ārd":904,"ārm":72,"ārn":280,"āro":74,"ārp":114,"āri":461,"ārk":89,"ārl":118,"āt ":234,"ālā":1101,"āpu":44,"āps":37,"ās ":6616,"āld":44,"āgā":35,"āle":181,"āli":1326,"āla":1215,"āks":485,"ākt":61,"āku":495,"āko":642,"āki":268,"āka":1716,"ām ":3525,"ājs":680,"ābū":29,"ādī":293,"āju":1139,"āji":502,"ājo":89,"ājp":22,"āni":891,"āna":615,"ānd":34,"āmu":42,"āms":160,"āmi":68,"āma":396,"ālu":633,"āls":579,"ālo":317,"ārē":202,"āsā":40,"āpš":36,"ārī":93,"āvu":149,"žās":33,"ārā":289,"žād":455,"žāk":194,"žām":34,"ātē":54,"āze":265,"ārš":275,"āzi":216,"āzu":50,"zšķ":85,"ātī":67,"ātā":439,"ārņ":54,"āv ":562,"žā ":38,"āta":627,"āst":95,"āsu":62,"ātn":541,"ātr":318,"āto":42,"āte":469,"āti":825,"ārz":92,"āsa":252,"ālī":22,"ārr":75,"ārs":844,"ārt":1313,"āru":144,"ārv":488,"ānā":124,"āsi":27,"ākš":30,"āvd":67,"āva":113,"āpē":74,"āvs":30,"āvo":161,"āvj":71,"āvi":182,"āve":205,"āts":679,"ātu":408,"Āzi":142,"āfi":261,"āfr":63,"āfs":37,"āga":57,"āgs":41,"āgo":45,"āj ":38,"ābē":84,"ācē":37,"āk ":1029,"ādā":677,"ācī":199,"ādē":157,"āja":844,"āje":33,"ābi":54,"ābj":26,"ābe":514,"āba":24,"āca":27,"ābu":87,"āci":2513,"ācb":31,"āda":636,"ācu":337,"ādn":33,"ādo":104,"āde":246,"ādi":416,"āds":129,"ādu":390,"ļin":31,"ļie":252,"ļi ":260,"zņe":149,"ļda":25,"ļav":42,"ļau":478,"ļai":47,"ļas":391,"ļam":83,"ļa ":1215,"āzē":88,"zņē":241,"žīm":24,"žģī":37,"ātņ":41,"āvē":149,"āvā":189,"āvī":87,"AU ":30,"":23,"zēš":55,"BA ":99,"ķī ":22,"ļot":251,"ļos":305,"ļiņ":134,"ļoj":28,"ļri":165,"Ķīn":125,"ļu ":1814,"zīš":29,"ļsk":34,"ļve":81,"ļvi":28,"ļum":33,"ļus":125,"ļuv":123,"ķēr":58,"ķēn":27,"ķēd":39,"ļoš":38,"Žan":29,"zīs":345,"zīt":108,"zīv":1468,"zīl":49,"zīj":29,"zīm":1070,"zīd":65,"zīc":55,"zīb":324,"zīg":242,"AP ":26,"zāc":457,"zā ":57,"ža ":310,"zāk":210,"zāl":65,"zām":26,"zān":65,"zās":75,"žku":26,"zē ":172," С ":34,"žis":66,"ķīd":71,"ķīr":23,"ķīs":84,"žie":84,"ķīn":49,"ķīm":271,"žos":33,"žot":153,"žor":51,"žkā":37,"Āge":33,"žon":63,"žoj":249,"Āfr":113,"zēl":36,"zēj":212,"zēm":74,"zēt":500,"zēs":27,"zēr":47,"žni":23,"žo ":82,"žas":293,"žag":66,"žai":25,"žan":122,"žam":29,"ži ":383,"žer":29,"žes":28,"žet":61,"žei":43,"žel":23,"žen":74,"uža":37,"užu":32,"žus":34,"žur":90,"žoš":80," Н ":25,"žs ":70,"žre":77,"žu ":492,"ušā":109,"ušu":56,"ušo":68,"uši":230,"uša":268},"n_words":[2106271,2428959,2036825],"name":"lv","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/mr.json b/contrib/languages-data/mr.json
new file mode 100644
index 0000000..2f32d7e
--- /dev/null
+++ b/contrib/languages-data/mr.json
@@ -0,0 +1 @@
+{"freq":{"णां":79,"णाच":69,"थे ":276,"तका":129,"था ":217,"ोल्":72,"ोलि":66,"ोला":81,"ोर्":136,"ोबर":168,"णता":197,"तंत":167,"ोव्":196,"तो ":552,"णजे":209,"्तर":330,"ं ":115,"्त्":568,"्ते":64,"्थळ":62,"तां":107,"्ता":389,"ः ":139,"्ती":202,"ताक":90,"्ति":145,"्तु":82,"ताच":344,"्दर":73,"तात":957,"तान":226,"्थि":115,"ताम":63,"्था":569,"तार":104,"ताल":160,"तिक":153,"्थे":76,"तीं":64,"्दा":73,"а":72,"्धत":82,"तिह":109,"तीच":131,"तिस":79,"्ट्":957,"्टो":150,"्टे":199,"्टी":161,"्टा":154,"्टि":89,"इ ":181,"तसे":151,"धन ":83,"्तक":79,"दी ":549,"्यं":164,"्मन":141,"्यक":342,"दू ":113,"्मा":304,"्मि":128,"्रं":90,"्यत":102,"्मे":88,"्यप":104,"्यम":126,"्रक":379,"्यव":153,"्रज":340,"्यु":195,"्या":9729,"्धा":165,"तया":65,"्ने":69,"तरे":84,"्ना":140,"तरा":162,"तरर":91,"्पन":111,"दा ":131,"्पर":85,"तले":94,"तला":76,"्पा":80,"ण्य":949,"ا":67,"तमि":228,"्का":190,"्कृ":130,"्कर":75,"दर ":72,"णून":182,"तदा":122,"्ञा":188,"दल ":81,"तत्":79,"णार":809,"णात":73,"्गा":89,"्चि":216,"्चा":100,"णुक":137,"्चन":73,"थ ":288,"द ":645,"ध ":495,"न ":5349,"ड ":671,"थील":100,"ठ ":317,"थाप":202,"ण ":1146,"थान":236,"त ":6358,"थित":69,"धी ":120,"ज ":339,"दक्":266,"ञ ":122,"धा ":98,"ट ":1322,"ं":13502,"ः":232,"ँ":308,"आ":6891,"इ":1323,"अ":4895,"ऊ":224,"ऋ":69,"ई":607,"उ":1473,"घ ":103,"ए":2374,"ओ":357,"ऑ":510,"ऐ":63,"ग":7424,"ख":2772,"क":20151,"औ":108,"छ":206,"च":12137,"घ":1111,"ट":6169,"ञ":320,"झ":981,"ज":8381,"ठ":2153,"ड":3743,"ढ":440,"ण":6906,"त":27784,"थ":2477,"द":9254,"ध":4679,"न":18404,"प":12607,"फ":1499,"ब":5130,"्ग ":233,"भ":4247,"म":15733,"य":22975,"र":35779,"ळ":2991,"ल":18817,"व":16505,"ष":4156,"श":6670,"ह":17826,"स":18689,"ऽ":116,"ि":18218,"ा":66714,"थवा":122,"े":31183,"ॅ":601,"ू":4430,"ृ":896,"ी":21357,"च ":843,"ु":7163,"ौ":422,"्":44808,"ो":8505,"ै":961,"ॉ":916,"०":1926,"१":3312,"्क ":91,"६":758,"७":829,"८":1020,"९":2445,"२":1611,"३":695,"४":690,"५":705,"क ":4288,"ग ":912,"ख ":378,"त्स":89,"त्व":365,"त्प":68,"त्र":2148,"त्य":1716,"त्म":72,"त्त":815,"त्न":66,"ம":77,"த":78,"ா":90,"ி":93,"ர":79,"तून":246,"க":89,"तुर":76,"ॉर्":122,"்":211,"तीत":68,"तीन":98,"तीय":438,"तील":2444,"तेल":85,"ई ":267,"ै ":165,"्न ":140," ख":774," ग":1923," औ":107,"ोजी":152," क":6366," ओ":265," ऑ":506," ट":487," ज":3863," झ":541," च":2075," छ":154," घ":462," इ":1199," आ":6783," अ":4797," ए":2269," ऊ":68," उ":1326," ई":99,"दार":306,"दिग":99,"दान":93,"दाच":70,"दिल":153,"दिव":143,"्फ ":77,"दिर":79,"े ":16291,"दीच":62,"नंत":140,"दुर":84,"दुस":164,"ोठे":238,"्म ":214,"ू ":729,"्य ":1111,"्र ":868,"ि ":987,"नी ":1364,"ोणा":90,"ी ":12189,"ोत्":125,"ोतो":104,"ोते":752,"ोती":123,"ोता":310,"ु ":263,"ा ":17571,"ोधन":72," ८":95," ९":93," ६":115," ७":115," ४":132," ५":115," २":1066," ३":209," ०":70," १":2735,"्व ":312," प":6615," फ":822," न":2997," म":6902," य":3998,"्ष ":149," ब":2263," भ":2543," ठ":228," ड":419,"ह ":265," द":3178," ध":461," त":3644," थ":112,"ोपा":104," ह":7628," स":7738," ल":2032," र":3046," श":2241," व":5406,"ने ":992,"्स ":322,"स ":1697,"ष ":215,"थ्व":73,"श ":822,"व ":2502,"्च ":156,"दरम":106,"ळ ":724,"्ट ":345,"ल ":4586,"्ञ ":122,"दर्":246,"ना ":828,"्ड ":88,"र ":6250,"य ":2749,"्ठ ":232,"म ":1312,"्ण ":136,"ोका":76,"ोकस":391,"भ ":104,"ब ":134,"्थ ":178,"फ ":180,"्त ":411,"्ध ":328,"प ":291,"्द ":119,"टना":76,"डू ":84,"डी ":137,"डा ":115,"ठे ":245,"ठी ":788,"ञान":146,"टोब":135,"ट्य":132,"ट्र":1051,"ट्ट":87,"टेड":67,"टेक":65,"तः ":64,"डे ":134,"टां":121,"टें":125,"टात":71,"टार":107,"टिक":111,"ठ्य":96,"णी ":301,"णि ":782,"णा ":114,"्ह्":230,"्सि":71,"्हण":631,"्हा":235,"्हि":103,"्हे":192,"्स्":66,"ठिक":87,"डणु":118,"्लि":183,"्ली":63,"्ला":198,"्ले":93,"्रद":304,"्रथ":62,"्रत":188,"्रण":89,"्यू":155,"्रप":582,"्ये":1183,"्रम":557,"्यो":94,"्रल":83,"्रव":174,"्रश":146,"्रह":141,"्रस":352,"्रा":1606,"्रु":153,"्रि":802,"्लं":83,"्री":882,"्रे":549,"्रो":151,"्र्":109,"्षण":152,"्शि":77,"्षे":198,"्षि":311,"्षा":344,"्षी":88,"्वत":172,"्वज":70,"्वर":132,"्वी":309,"्वे":282,"्वि":88,"्वा":1006,"ड्य":63,"ता ":928,"ती ":831,"ते ":2330,"णे ":394,"डाच":85,"डात":62,"डिय":72,"डिस":128,"डील":75,"तर ":662,"डून":170,"जे ":259,"जा ":105,"जी ":439,"चीन":184,"चित":727,"चार":281,"चाल":111,"चिन":184,"चिम":207,"जगा":147,"च्य":2911,"च्च":223,"जन्":279,"टक ":89,"जनत":86,"जधा":220,"टन ":79,"जाग":68,"जां":62,"जिल":278,"जास":142,"जार":66,"जान":157,"जात":654,"जाण":139,"जरा":68,"जर्":134,"जवळ":124,"जोड":73,"ज्य":1261,"ज्ञ":320,"जीव":120,"जुन":111,"जुल":121,"जून":124,"टर ":126,"झाल":422,"टी ":174,"टा ":105,"ठा ":104,"ंघ":431,"ंख":180,"ंग":1699,"ंक":394,"ंड":876,"केत":187,"ंट":262,"ंज":187,"ंच":1704,"केच":132,"केट":130,"ँड":81,"ंश":217,"ंस":555,"ंव":461,"् ":692,"ंध":318,"ंद":1200,"ंन":966,"ंथ":109,"ंत":1644,"ंम":356,"केल":701,"ंप":555,"ंभ":92,"ंब":953,"ो ":900,"ँग":72,"आण":837,"आढ":90,"इं":419,"कृत":201,"अस":1528,"अश":181,"आग":71,"अव":109,"आक":127,"अल":143,"अर":282,"अभ":461,"अम":428,"अप":77,"अध":195,"अन":311,"अथ":144,"इत":287,"आश":86,"आह":3959,"आल":245,"आय":156,"आर":178,"आप":132,"आफ":117,"कृष":91,"आध":103,"अं":259,"आं":194,"कें":67,"अत":143,"अण":68,"अक":80,"उर":65,"उप":257,"उल":75,"ऊन":63,"ऊर":66,"इस":98,"उं":94,"खले":96,"उच":118,"उत":389,"उन":70,"उद":171,"कोण":88,"कोल":77,"एक":1893,"एख":65,"एप":100,"क्क":92,"क्ट":201,"क्त":320,"क्य":169,"क्ष":1291,"क्र":523,"क्स":174,"ऑस":91,"ऑफ":67,"ऑक":141,"ऑग":114,"गर":430,"गल":148,"गळ":139,"गव":115,"खे":276,"गप":74,"गम":74,"ख्":538,"गट":66,"खा":505,"खि":83,"गड":87,"खी":84,"गण":286,"गत":154,"खर":91,"क्":2945,"खल":175,"के":1566,"खन":94,"कॅ":149,"को":604,"कॉ":161,"कि":847,"की":630,"गं":74,"का":3922,"कृ":301,"कु":375,"कू":79,"कस":544,"कव":162,"कश":95,"कल":330,"खक":117,"कम":167,"कर":1579,"कप":84,"कथ":172,"कन":175,"कड":224,"खं":163,"कण":68,"कत":150,"कक":66,"कं":129,"ओळ":135,"ची":1602,"चि":1243,"चा":1991,"चे":2243,"चौ":88,"च्":3182,"० ":717,"जग":235,"जक":120,"चन":187,"चर":81,"चल":81,"चव":64,"घा":367,"घे":138,"चं":156,"घट":138,"गा":1549,"गस":133,"घड":69,"गी":328,"गि":172,"गू":80,"गु":294,"गो":373,"ग्":1223,"गे":325,"घर":90,"टन":228,"ञा":188,"टा":652,"टल":177,"टर":206,"खाद":93,"४ ":416,"झा":550,"झि":70,"टच":62,"३ ":377,"टक":258,"जो":179,"जे":499,"जू":189,"जी":702,"जु":275,"जा":1637,"जि":541,"२ ":421,"ज्":1646,"जन":601,"खान":75,"जध":220,"खाल":87,"जस":63,"जव":226,"१ ":522,"जर":265,"जल":91,"जय":81,"जम":98,"डच":80,"ठा":219,"ठि":103,"ठव":67,"५ ":430,"टे":445,"ट्":1360,"टो":293,"टी":396,"टि":297,"डा":413,"डि":310,"डी":307,"डल":139,"डळ":73,"डव":71,"६ ":435,"ठ्":106,"डम":84,"डर":65,"ठी":879,"डत":76,"डण":186,"डन":65,"ठे":307,"तं":195,"तः":94,"ढा":78,"णज":221,"णक":97,"७ ":492,"ढळ":87,"ड्":156,"डो":138,"डॉ":73,"डे":278,"डू":298,"डु":64,"णि":910,"णी":405,"णु":223,"णू":264,"णा":1365,"तक":277,"८ ":406,"णप":91,"णत":272,"९ ":481,"तव":72,"तस":223,"ति":798,"ता":3296,"तू":362,"तु":325,"ती":4269,"दं":76,"तद":128,"तत":106,"तप":146,"तन":88,"णे":550,"तम":332,"ण्":982,"तय":70,"तल":255,"तळ":94,"तर":1232,"थव":138,"था":849,"थी":145,"थि":153,"ते":2652,"तो":665,"त्":5456,"खेळ":198,"थळ":77,"दक":326,"दृ":82,"धत":108,"दू":226,"दु":367,"नं":194,"दी":767,"दि":707,"दा":1004,"थे":411,"दन":81,"दव":72,"दल":156,"दर":524,"थ्":121,"धा":812,"नच":194,"नत":190,"नद":208,"धी":622,"नड":78,"धि":310,"धू":73,"धु":110,"दो":255,"द्":1977,"धन":165,"दे":1492,"नक":199,"नग":152,"धर":260,"धल":79,"नर":109,"नल":207,"नव":362,"धे":84,"नय":86,"नम":96,"ध्":1350,"नी":1612,"पं":155,"नु":263,"गळ्":76,"ने":2153,"नस":171,"ना":2993,"नि":1438,"पक":409,"नो":289,"नै":75,"न्":1414,"पत":355,"पण":229,"पन":409,"पद":339,"पड":106,"पट":647,"पश":225,"पह":161,"पस":114,"पल":219,"पर":957,"बई":129,"पे":273,"पै":233,"पॉ":77,"पू":594,"पृ":102,"पॅ":69,"पा":1577,"पि":254,"बं":239,"पी":379,"पु":752,"बच":72,"फळ":72,"प्":3134,"पो":211,"बन":164,"फे":175,"बद":156,"बत":64,"फु":80,"फि":95,"फा":147,"फो":62,"फ्":393,"बर":861,"बह":109,"बि":244,"बा":902,"बु":115,"मं":347,"बी":172,"बे":403,"बॉ":75,"बो":197,"भर":75,"ब्":696,"मक":100,"गरा":120,"गरी":78,"मग":69,"मज":82,"मच":80,"यं":250,"भि":399,"भा":2637,"मत":229,"भू":282,"मण":132,"भे":177,"मन":439,"मध":1532,"मद":153,"भौ":79,"भो":73,"मल":215,"यक":493,"भ्":94,"मर":604,"मह":1126,"यत":203,"मृ":108,"मू":271,"मॅ":66,"ख्य":500,"मि":1010,"यट":74,"मा":2886,"मु":1239,"रं":435,"मी":463,"मो":679,"यम":237,"म्":1246,"यन":341,"मे":875,"यप":140,"मै":72,"यव":231,"रख":99,"रग":70,"रक":670,"यल":98,"यर":147,"या":14036,"रज":391,"रच":375,"यस":75,"रध":68,"रद":356,"रथ":65,"रत":1594,"यू":189,"रण":1061,"यु":675,"यी":118,"यि":121,"रय":68,"रम":767,"रभ":78,"रब":97,"यो":389,"रफ":64,"रप":674,"ये":1927,"रन":85,"गाच":88,"लय":143,"लब":72,"लन":141,"लढ":74,"गात":319,"लच":92,"लग":102,"लक":174,"र्":5914,"रो":716,"रॉ":92,"रे":1413,"गां":194,"रू":387,"लं":299,"री":2531,"रु":575,"रि":1770,"रा":6808,"रह":201,"रस":797,"रश":254,"रव":382,"रल":306,"रर":139,"ळ्":150,"ळे":264,"ळा":565,"वं":149,"ळी":258,"ळू":62,"ळव":128,"ल्":2173,"लो":767,"चन ":87,"लू":62,"ळण":107,"ळत":82,"लै":137,"ले":3399,"ळन":66,"लु":220,"ली":1530,"लि":1122,"ला":2983,"लव":72,"ळख":135,"ळक":78,"वो":76,"शब":124,"वै":95,"वे":911,"शन":181,"शव":74,"षक":96,"व्":1218,"शर":79,"वश":88,"गस्":118,"वस":571,"वू":99,"वृ":105,"शत":95,"वा":3822,"वि":1941,"वी":773,"वन":263,"शक":187,"वळ":226,"वल":308,"वर":1642,"वय":91,"वज":113,"वच":65,"वक":74,"वण":225,"वत":410,"वड":312,"वट":66,"सन":187,"षे":500,"सप":188,"सभ":348,"ष्":1679,"सम":806,"सर":1179,"सल":632,"सव":108,"सच":99,"षा":600,"षि":375,"षी":127,"सण":108,"सत":526,"सद":113,"सध":64,"शे":245,"श्":798,"शो":167,"षय":121,"सक":135,"शह":705,"सं":1892,"शी":422,"शु":72,"शा":1405,"शि":914,"षण":199,"हे":6136,"हॅ":73,"हु":162,"ही":1380,"हि":1219,"हा":3206,"ह्":871,"हो":1733,"हन":73,"से":980,"सु":643,"सी":265,"हत":249,"सू":619,"हण":691,"सि":714,"सा":2701,"सह":169,"हव":64,"हस":71,"हम":90,"स्":3801,"हय":143,"हर":898,"सै":65,"सो":298,"ात":6409,"ाथ":69,"ाढ":83,"ाण":917,"ाठ":965,"िं":1222,"ाड":364,"ाट":442,"ाब":339,"ाभ":170,"ाप":1160,"ान":3426,"ाद":925,"ाध":343,"गुर":68,"ाव":2259,"िख":96,"िक":2722,"ाळ":594,"ाल":2412,"ार":6697,"ाय":1081,"ाम":1731,"िज":229,"ाह":1051,"िच":184,"ास":2359,"ाष":1328,"ाश":369,"िग":160,"ां":4665,"ाँ":113,"ाऊ":66,"ाइ":80,"ाई":239,"ाउ":96,"ाक":602,"ाच":4155,"ाझ":79,"ाज":2078,"ाग":1046,"ाख":206,"ीण":65,"ीठ":87,"ुं":330,"गिर":64,"ीत":755,"ीप":214,"ीन":664,"ीम":234,"ीय":1104,"ील":3494,"ीर":265,"ुग":140,"ुख":527,"ीव":305,"ुक":528,"गीत":150,"ुज":89,"ीस":155,"ुट":125,"िट":204,"गाव":277,"ीं":192,"िड":79,"गाल":127,"िण":383,"ित":1967,"गाय":92,"िद":477,"िध":193,"िन":1260,"िप":347,"िब":126,"घटन":111,"िभ":130,"िम":634,"िर":694,"िय":1422,"िल":1282,"ीक":319,"िळ":425,"िश":552,"िव":770,"िस":721,"िष":365,"िह":250,"ीच":727,"ीज":85,"ेव":793,"ेश":1384,"ेळ":324,"ैक":225,"ेल":2762,"ेय":96,"ेर":672,"ेम":206,"ेब":175,"ेप":128,"ेन":599,"ैद":75,"ॉं":62,"ेह":98,"ेस":492,"ेष":313,"ॉक":64,"गेल":207,"ैव":63,"ैन":90,"ॉन":105,"ॉट":71,"ॅर":127,"ॅन":107,"ें":768,"ेक":722,"ेख":448,"ेट":491,"ेड":164,"ेत":2226,"ेण":224,"ेद":177,"ेथ":380,"ेग":137,"ेच":757,"ेज":75,"ृथ":73,"ृत":438,"ृष":221,"गोल":138,"ुत":238,"ुण":210,"ुढ":64,"ुड":67,"ूं":81,"ुन":719,"ुध":64,"ुद":440,"ुप":192,"ुर":1052,"ुम":183,"ुळ":257,"ग्ल":179,"ूक":85,"ुल":420,"ुष":125,"ुस":491,"ुव":264,"ग्र":670,"ूच":81,"ग्द":71,"ग्न":84,"ूत":126,"ून":1612,"ूप":102,"ूम":143,"ूर":779,"ूल":112,"ूळ":67,"ूह":94,"चा ":1450,"्व":2745,"्श":274,"्ष":1475,"्स":788,"्ह":1660,"्म":1315,"्य":13761,"्र":9096,"्ल":826,"ची ":1393,"ोर":443,"ोय":88,"ोल":499,"ोब":307,"ोम":148,"ोस":168,"ोष":97,"ोह":121,"ोळ":97,"ोश":68,"ोव":325,"्ण":384,"्त":2563,"्ड":225,"्ट":2277,"्ठ":301,"्झ":104,"्ञ":320,"्फ":142,"्ब":125,"्प":569,"्ध":714,"्न":549,"्थ":1118,"्द":543,"ौर":111,"्ज":211,"्च":600,"्ग":488,"्क":801,"ों":135,"ॉर":159,"ॉल":165,"ोज":285,"ोड":211,"ोट":232,"ोठ":384,"ोद":77,"ोण":200,"ोत":1519,"ोप":280,"ोध":220,"ोन":394,"ोक":697,"ोच":112,"ोग":276,"चे ":2125,"घात":147,"घाच":71,"चंद":96,"घेत":69,"०६":62,"०७":100,"०४":65,"००":610,"०१":76,"०८":75,"०९":89,"१०":143,"१५":108,"१६":128,"१७":139,"१८":321,"११":87,"१२":105,"१३":94,"१४":109,"१९":1554,"२१":65,"२०":603,"८०":89,"८६":64,"९०":106,"९६":205,"९५":154,"९८":288,"९७":177,"९२":135,"९१":161,"९४":175,"९३":129,"८८":69,"८९":125,"९९":420,"३०":86,"२३":65,"२२":68,"२५":80,"२७":76,"२६":63,"२८":64,"५०":65,"६०":76,"जन ":67,"चना":79,"के ":109,"का ":542,"ओळख":133,"की ":396,"खक ":110,"कर ":232,"कन ":100,"काह":112,"किन":92,"काम":326,"कार":1020,"काय":62,"काल":221,"काळ":222,"काश":148,"कास":81,"किम":109,"किल":76,"कीय":113,"कां":335,"काद":84,"कात":158,"काण":89,"किं":328,"काच":171,"कवी":78,"कसं":90,"कशा":76,"कसभ":296,"कला":81,"कल्":115,"गी ":87,"कर्":219,"गा ":89,"करा":112,"करू":89,"करी":67,"三":73,"करत":181,"करण":522,"कथा":145,"कडे":62,"कडू":62,"गर ":134,"खंड":160,"कंप":101,"एप्":100,"एका":149,"एखा":65,"ऑफ ":63,"ऑस्":91,"ऑगस":114,"ऑक्":139,"०९ ":83,"०८ ":72,"०७ ":92,"ऊर्":63,"११ ":69,"१२ ":81,"१३ ":67,"१० ":113,"१६ ":75,"१७ ":70,"१४ ":80,"१५ ":76,"०० ":116,"०४ ":63,"एक ":1511,"आणि":780,"आपल":113,"आफ्":114,"२००":459,"आले":120,"आर्":98,"आहे":3954,"इति":65,"इतर":127,"अथव":122,"अति":63,"१९१":93,"१९८":221,"१९९":347,"१९६":128,"१९७":149,"१९४":143,"१९५":124,"१९२":109,"१९३":101,"अधि":146,"अने":154,"अभि":344,"अभ्":70,"अमे":278,"अमि":68,"अर्":160,"आका":68,"असल":335,"असत":304,"असण":73,"अशा":75,"अशी":64,"असे":343,"असू":247,"असा":90,"इंड":62,"इंग":258,"आढळ":87,"००९":79,"००७":86,"் ":95,"उच्":113,"उद्":130,"३० ":73,"उत्":376,"१९ ":73,"२७ ":71,"ऊन ":62,"२८ ":62,"२५ ":73,"इस्":65,"२० ":116,"ेशा":514,"ेशि":132," ओळ":134," कं":103,"ेष्":213," खं":102," कथ":124," कम":84," कर":956,"ैकी":208," कल":112," कव":97," कि":683," का":1385," कृ":78,"ेवर":160," कु":249," के":933," कॅ":146," को":333," कॉ":122," क्":525,"ेवा":214," एक":1893," एख":65," एप":100," ऑक":140," ऑग":114," ऑफ":67,"ेस्":75," ऑस":91," चौ":88," च्":195," जग":208," ची":128," चि":913," चा":258," चे":120," जर":148," जम":72," १ ":124," जन":432," ज्":461," २ ":80," जि":352," जा":1062," जू":107," जु":247," जी":146," जे":116," जो":163," ३ ":78," झा":447," खे":215," गण":130," गट":64," खा":206," घर":66," गो":192," ग्":328," गे":212," गु":208," गा":404," चं":92," घे":131," घा":63,"ंट ":79," अं":254,"ेथे":251,"ेथी":97," अप":73,"ेते":395," अन":300," अध":194," अथ":143," आक":124,"ेता":130," अल":139,"ेती":391," अर":236," अम":426," अभ":454," अस":1524," अव":109," अश":180," आग":68,"ेत्":247," आढ":90," आण":837," इं":409,"ेणा":90," अक":80," आं":187,"ेण्":83," अण":68," अत":140,"ेतल":69,"६० ":62,"ंत ":465,"ंड ":204,"ेने":94,"ेन्":114,"ेनि":72,"ंग ":304,"ेब्":104,"ेळा":116," इस":89,"ेल्":528,"ेलि":98,"ेली":343," इत":279,"ेला":618," आह":3959,"ेले":852," आश":86," आर":162," आय":154," आल":243,"ंच ":100," आध":97," आप":129," आफ":117," उल":66," उप":255," उद":168," उत":384,"ेरि":321,"ेरी":87," उच":118,"ंघ ":80," वा":841,"ोत ":71," वि":1279," शत":75," वस":293," व्":476," शर":73," वे":251," वै":84," शब":119," सं":1488," शि":338," शा":307," शह":692," शे":119," श्":176," शो":63," शक":86," वर":325," वन":73,"ोन ":196," ला":353," लि":318," ले":317," लो":622," या":2852," रच":72," यु":309," यो":83," ये":604," रश":94," रा":1893," रि":79," रस":83," रे":153," रु":64," रो":269,"ंद ":67,"ोर ":89," हो":1650," ह्":509," हि":526," ही":801," हा":1542," हे":1892,"ोल ":64," सम":640," सप":137," सर":815," सद":67," सध":64," सत":74," स्":1248," हर":99," हय":132," सो":189," सा":1287," सि":219," सह":142," से":175," सी":112," सु":443," सू":90," दर":173," दृ":63," दु":235," दा":117,"ोक ":70," दि":467," दक":264," त्":1085," तो":143," ते":655," तय":65," तम":233," तर":237," तत":72," ता":327," ति":182,"ोग ":64," ती":172," तु":116," तस":163," ७ ":73," ६ ":68," डि":141," टे":67," ट्":135," टो":93," ५ ":63," ठि":88," ४ ":80," टा":72," मो":576," म्":694," मे":240," मै":64," मू":114," मृ":78," मा":1083," मि":294," रं":86," मु":622," मह":1114," मल":145," मर":508," भौ":66," मन":77," मध":589," मत":139," भू":208," भा":1980," ब्":283," बे":267," बो":108," बा":493," बि":111," मं":217," बह":106," फ्":212," बर":139," बद":64," बन":145," फे":128," फा":98," फु":65," फि":73," बच":70," प्":2295," पो":147," पि":143," पा":818," पु":600," बं":158," पे":127," पॉ":75," पृ":96," पू":277," पर":492," पश":206," पह":159," पड":69," पट":81," पद":228," पत":76," न्":105," नो":178," पक":276," नि":731," ना":905," पं":133," ने":304," नव":130," धा":110," नद":157," दे":1033," द्":193," दो":206," धर":205,"ॅरि":66,"ेंट":68,"ेंद":165,"ेंब":371,"८९ ":68," द ":78," इ ":175,"ेल ":174,"८० ":71,"ेर ":66,"ेस ":273,"ेश ":484,"ेव ":85," स ":136,"ेच्":192,"ेचे":150,"ेचा":92,"ेची":66,"ेकड":79,"ेखन":78,"ेक्":200,"ेखक":117,"ेटा":76,"ॉन ":63,"९८ ":66,"९९ ":71,"९६ ":77,"९१ ":71," व ":1440,"ृष्":213,"ृथ्":72,"ृती":81,"ृत्":188,"अंत":115,"ेत ":845,"ेन ":118,"ेट ":229,"ेड ":81,"९९६":63,"ेक ":319,"ेख ":111,"ेच ":240,"आंत":108,"ुष्":79,"ुसर":130,"ुवा":185,"ुळे":148,"ुला":65,"ुलै":113,"ुरा":109,"ुरस":95,"ुरो":111,"ुर्":256,"ुरु":156,"ुरू":78,"ुमा":140,"ुप्":76,"ुनि":101,"ुनी":63,"ुना":87,"ुद्":358,"ुत्":100,"ंच्":531,"ंचा":365,"ंची":306,"ंचे":348,"ंग्":429,"ंगा":332,"ंघट":78,"ंगी":166,"ंघा":242,"ंका":93,"ंख्":149,"ंगल":89,"ंगण":71,"ंडा":180,"ंडळ":71,"ंना":305,"ंनी":632,"ंपै":165,"ंबई":124,"ूर्":429,"ंपा":63,"ंपर":71,"ंपन":86,"ंत्":351,"ंता":140,"ंती":82,"ंतर":401,"ंद्":289,"ंदी":266,"ंदू":116,"ंदा":71,"ंदि":113,"ंदर":105,"ंबर":448,"ंमध":305,"ंस्":364,"ंसा":99,"ुस्":136,"ंवर":89,"ुसा":139,"ंवा":298,"ंशो":74,"ाळी":71,"ाळा":189,"िकन":68,"ाला":601,"ालि":128,"ाली":458,"ालु":126,"ाल्":170,"ाले":279,"ावि":118,"ावी":70,"ावा":374,"ाव्":94,"ावे":148,"ावण":64,"िका":572,"ावर":588,"ावल":82,"िको":101,"िक्":215,"िके":418,"ाषा":187,"ासत":84,"ाषे":285,"ाष्":810,"ाशि":80,"ाशी":94,"िग्":108,"ासक":92,"ाही":306,"ाहि":257,"िचा":69,"ाह्":62,"ुन ":350,"ासि":69,"ासा":570,"ासु":84,"ाहत":75,"ासू":218,"ास्":488,"िजे":63,"ींच":65,"िणे":98,"िता":173,"िती":211,"ित्":984,"िद्":408,"िना":177,"िनि":68,"िनी":230,"िने":334,"िन्":135,"िपी":179,"िभा":112,"ियन":201,"िमे":82,"ियम":83,"िमी":103,"िमा":152,"ियो":66,"िया":833,"िर्":245,"िरी":73,"िरा":70,"िले":194,"िल्":507,"िला":170,"िली":83,"िसर":109,"िष्":172,"िषय":111,"िश्":136,"िशे":76,"िवस":72,"िवा":161,"िवि":81,"ीका":68,"िवड":238,"ीचे":213,"ीची":91,"िहि":83,"िहा":133,"ीचा":113,"ून ":1516,"ीच्":262,"िसे":127,"िस्":228,"ुंब":177,"ीती":162,"ूर ":255,"ीने":102,"ीरा":80,"ीवर":113,"ुक्":227,"ुका":154,"ुख्":276,"ृत ":101,"ीला":83,"ित ":454,"िण ":169,"िन ":167,"िध ":64,"िल ":211,"िळ ":165,"ीक ":118,"ांड":163,"ांच":1386,"ांक":146,"ांग":206,"ांस":157,"ांव":124,"िम ":197,"ांम":292,"ांब":110,"ांप":219,"ांन":910,"ांद":82,"ांध":110,"ांत":540,"िय ":84,"िर ":71,"िश ":190,"िस ":90,"ागा":289,"ीत ":405,"ागर":198,"ाखा":79,"ाक्":63,"ाका":132,"ाकर":80,"िंद":422,"ाडू":108,"िंव":279,"ाठी":839,"िंग":264,"ाटक":140,"ाजा":172,"ाजी":169,"ाज्":867,"ाजध":220,"ाजव":101,"ाचा":713,"ाची":807,"ाचे":1121,"ाच्":1245,"ाजक":91,"ीन ":336,"ाने":765,"ाना":350,"ानि":81,"ानी":337,"ानु":71,"ानल":114,"ानव":85,"ुख ":209,"ाध्":84,"ापन":141,"ान्":247,"ादी":203,"ानं":83,"ानत":66,"ाधि":74,"ाधा":87,"ानच":62,"ानक":91,"ाद्":145,"ाता":198,"ाती":1964,"ातू":202,"ाणे":107,"ाण्":123,"ातल":106,"ाते":315,"ात्":194,"ातो":153,"ील ":3327,"ीय ":1038,"ातं":85,"ीर ":82,"ाणी":153,"ाणा":214,"ारी":709,"ारि":99,"ारा":1018,"ारस":180,"ार्":1216,"ारे":318,"ालय":129,"ामा":334,"ायक":105,"ाम्":126,"ायन":99,"ामी":72,"ारं":64,"ामु":161,"ाया":101,"ारच":84,"ारख":74,"ारक":98,"ारण":233,"ारत":1080,"ाबा":143,"ामध":252,"ामन":110,"ापर":215,"ापू":104,"ापी":79,"ापा":183,"ाप्":78,"ीस ":70,"ाई ":106,"होत":1397,"होण":97,"ह्म":91,"ह्य":743,"ाक ":79,"ाग ":209,"ाद ":178,"ाण ":173,"ात ":2992,"ान ":859,"ाज ":109,"ाच ":165,"ाट ":100,"ाव ":432,"िक ":1026,"ाळ ":124,"ास ":473,"ाम ":369,"ाल ":196,"ार ":1261,"ाय ":102,"सले":457,"सल्":84,"समु":164,"समा":227,"समू":87,"सम्":80,"सरा":98,"सर्":792,"सप्":140,"सभे":108,"ष्य":98,"ष्ण":139,"ष्ठ":291,"सभा":231,"ष्ट":1050,"सध्":62,"षेत":366,"हत्":160,"सेच":158,"सेन":81,"सें":161,"सीम":75,"सुन":150,"हणू":177,"सुर":133,"सुम":66,"हणत":183,"सून":460,"हणज":210,"सां":148,"साह":175,"सिद":219,"साध":112,"सार":316,"साम":328,"साय":147,"सिक":101,"साव":80,"साल":226,"साग":141,"सिं":119,"साठ":416,"सात":110," १२":89," ११":78," १४":90," १३":81," १६":113," १५":99," १८":311," १७":121," १९":1529," २०":568,"हरा":269,"हर्":68,"स्व":512,"हया":139,"स्य":64,"स्ल":87,"स्थ":759,"स्प":226,"स्ट":559,"स्त":1031,"स्क":382,"सेव":68," १०":100,"हेत":649,"हें":134,"हिल":252,"हिन":68,"हित":204,"हास":245,"हाव":84,"हाय":65,"हार":516,"हाम":139,"हान":148,"हिं":378,"हात":102,"हे ":5167,"षां":107,"सणा":80,"सतो":69,"सते":145,"सता":120,"सत्":153,"षाच":186,"षिण":264,"शेष":68,"शोध":130,"हा ":1574,"ही ":1273,"श्व":169,"श्र":226,"श्य":73,"श्च":243,"शहर":683,"से ":365,"सी ":115,"हर ":423,"संत":74,"शिय":317,"संक":81,"संग":244,"शिव":108,"संख":143,"संघ":417,"संब":106,"संप":91,"संस":355,"संश":79,"शात":171,"शाच":270,"शास":375,"शिक":213,"शां":163,"वेळ":85,"वेल":78,"वेश":67,"वेग":99,"वेद":75,"शतक":63,"वृत":90,"सा ":185,"व्ह":547,"व्य":626,"शब्":122,"वर्":390,"षा ":197,"वरा":67,"वरू":76,"वरी":175,"षी ":90,"वले":122,"वसा":93,"वसल":181,"वस्":189,"सन ":83,"वां":120,"वात":562,"वाद":207,"वान":200,"वाच":234,"वाज":104,"विक":171,"वाल":80,"वास":154,"वाप":201,"वार":452,"वाय":134,"वाम":62,"वित":108,"विन":71,"विद":197,"विध":115,"विज":139,"वाह":173,"विच":82,"विष":207,"विश":185,"वीच":70,"विल":73,"विव":106,"विम":91,"विभ":103,"वीप":110,"वडू":94,"वडण":125,"वणा":82,"वण्":85,"वती":67,"वता":101,"वना":93,"शा ":95,"षण ":107,"शी ":306,"वंश":67,"ळात":171,"ळाड":66,"षक ":73,"ळ्य":142,"शन ":88,"वे ":211,"वा ":653,"वी ":367,"ल्प":162,"ल्य":1022,"ल्ल":251,"ल्व":86,"ल्स":80,"ल्ह":320,"ळना":64,"लेश":64,"लोक":579,"वळ ":98,"लेल":1062,"लेख":355,"लिह":91,"लिश":121,"लिय":136,"लील":65,"लुक":126,"वर ":789,"लां":197,"लाग":85,"लाच":70,"लिं":80,"लाप":66,"लाव":113,"लिक":135,"लास":65,"लिन":79,"लिप":197,"वत ":73,"ळखल":119,"ळे ":200,"ळा ":98,"ळी ":148,"रेस":151,"रोज":164,"रोप":113,"र्श":238,"र्व":1106,"र्स":68,"र्ष":184,"र्म":630,"र्य":552,"र्ल":103,"र्थ":343,"र्द":88,"र्ध":121,"र्न":169,"र्फ":104,"र्ट":116,"र्ड":113,"र्ण":231,"र्त":217,"र्ग":412,"र्क":189,"र्ज":148,"र्च":115,"रीक":109,"रिय":267,"रिल":109,"रिस":109,"रीत":101,"लंड":195,"रीय":372,"रील":210,"रुन":72,"रुप":68,"रुव":148,"रून":145,"रूप":76,"रें":77,"रेक":125,"रेट":62,"रेल":183,"रसा":80,"रसि":236,"रस्":202,"ले ":1654,"लै ":119,"रां":490,"रान":176,"राठ":475,"राट":76,"रात":566,"राण":185,"राज":1474,"राच":427,"रिट":93,"रित":141,"राष":801,"रास":112,"राह":156,"राम":258,"राय":119,"रिक":669,"राव":320,"रलि":71,"ला ":1877,"ररा":122,"रम्":122,"रमा":263,"रमु":208,"रसं":149,"रशि":112,"रशा":97,"रवा":143,"रले":92,"ली ":1229,"रपट":518,"येष":193,"येण":75,"येथ":336,"येत":173,"रदे":228,"योग":175,"युक":84,"युर":149,"युन":102,"युद":90,"याच":1413,"याम":178,"यान":560,"याप":224,"यात":1507,"याद":157,"यास":414,"याव":251,"यिक":66,"याल":203,"यार":164,"रता":731,"रति":74,"रती":443,"रते":65,"रत्":99,"लय ":91,"रणा":308,"रणे":72,"रण्":354,"रचन":71,"यवस":94,"रक्":111,"रजा":126,"रजी":94,"यां":1609,"रज्":107,"रच्":105,"लन ":62,"रे ":511,"महा":885,"महत":141,"यक्":261,"रू ":76,"मले":80,"री ":1362,"मृत":96,"मूह":88,"मुळ":150,"मुल":69,"मुद":185,"र् ":478,"मुख":511,"रंप":65,"रंथ":84,"मुं":140,"रंग":175,"मिळ":390,"मित":196,"मिन":63,"मार":537,"माल":161,"मिक":114,"माव":85,"माह":78,"माण":213,"मात":178,"मान":677,"माच":73,"माज":179,"मां":206,"मोठ":381,"में":100,"मेर":300,"मेल":94,"यत्":103,"रका":358,"म्र":173,"म्य":205,"म्ह":682,"म्म":64,"रत ":129,"यू ":94,"रण ":190,"या ":6887,"भिन":289,"भाव":89,"भाष":512,"भार":1098,"भाग":467,"रम ":63,"यंत":200,"ये ":856,"मधी":410,"मध्":993,"भेव":86,"मनी":88,"भूत":65,"भूम":113,"मतद":118,"मरा":500,"रा ":574,"भ्य":77,"मा ":144,"बर्":157,"मी ":268,"यन ":212,"मे ":125,"बहु":71,"बिय":93,"बाज":80,"बाब":72,"बार":87,"बाद":98,"बां":115,"मंत":114,"मंद":83,"मंड":86,"यम ":64,"रक ":67,"बेर":65,"बेट":143,"बोध":67,"ब्र":411,"ब्द":139,"प्र":2788,"प्त":110,"प्ट":127,"भा ":239,"मण ":66,"मन ":141,"बच्":64,"यक ":87,"बद्":68,"बनव":83,"फेब":101,"फ्र":309,"पहि":119,"पश्":206,"पल्":82,"बी ":93,"बा ":68,"पर्":325,"परि":145,"परा":78,"परं":68,"�":133,"पैक":208,"पृथ":73,"पुस":66,"पूर":541,"पुण":116,"पुत":63,"पुर":350,"पीठ":86,"बंग":106,"बंध":84,"पास":320,"पाच":82,"पाण":105,"पान":76,"पात":141,"पाद":66,"पार":155,"पाल":72,"पाय":64,"पां":102,"पक्":286,"न्न":159,"न्य":422,"न्म":293,"न्ह":93,"न्स":215,"नोव":141,"पद्":103,"पदा":135,"पनी":76,"पना":113,"पणे":87,"बर ":537,"पत्":186,"पती":143,"पटा":141,"नले":90,"नवी":100,"पी ":206,"नदी":146,"ध्य":1211,"नुस":148,"नीच":95,"निस":85,"नेत":475,"नेश":67,"नेव":132,"नेक":163,"बई ":90,"निव":276,"निर":218,"निय":224,"नास":64,"नाह":82,"नाव":453,"निक":308,"नाय":79,"नाम":119,"नार":159,"नात":134,"नाट":176,"नाड":71,"नाच":211,"नाग":125,"नां":144,"द्द":122,"द्व":204,"द्र":575,"द्य":488,"द्ध":540,"धर्":201,"पट ":338,"देव":241,"देश":972,"देण":76,"दोन":169,"धात":66,"धार":222,"धिक":210,"धान":301,"धील":412,"पर ":71,"नता":99,"नगर":128,"पण ":79,"नच्":79},"n_words":[573395,652939,442285],"name":"mr","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/ms.json b/contrib/languages-data/ms.json
new file mode 100644
index 0000000..216d2c3
--- /dev/null
+++ b/contrib/languages-data/ms.json
@@ -0,0 +1 @@
+{"freq":{"D":19572,"E":5893,"F":8324,"G":10072,"A":33896,"B":36297,"C":14172,"L":22330,"M":42865,"N":13239,"O":4712,"H":13017,"I":28961,"J":16075,"K":40946,"U":8179,"T":34101,"W":6762,"V":4890,"Q":1060,"P":46573,"S":49712,"R":17401,"Y":3291,"X":535,"Z":1931,"f":21309,"g":267175,"d":316152,"e":658563,"Feb":550,"b":199729,"Fed":85,"c":52318,"a":1572499,"n":737214,"o":176914,"l":322914,"m":264030,"j":47182,"Fel":252,"k":282587,"h":223514,"i":560632,"w":38651,"v":16909,"Fer":162,"u":382193,"t":354175,"s":303833,"r":438412,"q":1773,"p":194581,"z":11534,"y":121423,"x":2864,"²":570,"É":92,"ß":112,"Fil":1891,"Fin":201,"í":159,"ë":593,"é":1443,"Fir":84,"è":291,"ç":161,"ä":134,"Fie":111,"â":119,"á":238,"ü":561,"ö":608,"ô":979,"ó":201,"ā":412,"ī":194,"ō":211,"ū":189,"Fas":81,"Fat":132,"Faz":217,"Fan":166,"Fal":106,"Far":151,"Fai":104,"Eri":93,"Est":118,"Ero":494,"ə":85,"Eur":130,"El ":142,"́":146,"Ele":136,"Eks":136,"Eko":134,"ν":86,"ο":114,"Eng":424,"α":136,"Emp":510,"Emi":118,"Ent":129,"ς":105,"ρ":83," l":23228," m":77479," n":10921,"я":98," o":15254," h":12749," i":30341," j":11345," k":65931," d":162664," e":5363," f":6534," g":7177,"р":170,"с":230," a":48871,"т":130," b":61081,"у":83," c":5105," y":44531," x":98," z":926," u":11761," t":69161," w":8735," v":1459," q":106," p":65928," s":95750," r":8778," J":16031," K":40896," H":12965,"Gel":192,"Gem":153," I":28925," N":13195," O":4683," L":22273," M":42791," B":36217," C":14046," A":33790," F":8281," G":9989," D":19485," E":5868,"л":171,"к":214," Z":1920," Y":3279,"и":251," X":518,"о":231,"н":181," S":49556,"в":121," R":17319,"Ger":418,"б":106," Q":1053,"а":416," P":46450," W":6726,"Geo":314," V":4864,"Gen":396,"е":192," U":8167," T":33952,"Gha":143," É":91,"Gil":114,"Gir":155,"י":86,"Gan":153,"Gal":238,"Gam":239,"Gaj":170,"Gay":116,"Gar":385,"Gad":110,"Gab":156,"و":473,"ي":643,"ف":123,"ق":139,"ك":121,"ل":805,"م":542,"ن":527,"ه":177,"د":337,"ج":123,"ح":205,"ت":219,"ب":378,"ة":256,"ا":1272,"أ":90,"ع":180,"ش":82,"س":358,"ر":545,"Flo":106,"Fle":177,"Fra":351,"Fri":170,"Fre":163,"A ":1902,"Fon":180,"For":435,"F ":465,"Da":7113,"Cu":422,"Cy":178,"Cz":168,"Cl":327,"Co":2781,"Cr":469,"Ce":818,"Ch":3522,"Ci":1462,"G ":398,"Ec":309,"Ed":294,"Ea":212,"Du":1497,"Do":1267,"Dr":1224,"De":3041,"Di":4070,"Dh":92,"Fe":1229,"H ":344,"Fa":1374,"Eu":254,"Ev":110,"Ex":177,"Er":726,"Et":118,"Es":290,"En":930,"Em":828,"Ei":91,"El":547,"Ek":289,"Eh":94,"Ge":1691,"Ga":1915,"I ":1850,"Fu":334,"Fr":767,"Fo":867,"Fl":397,"Fi":2562,"B ":922,"II ":729,"C ":1235,"Av":219,"Au":1348,"Aw":291,"Ar":3162,"Aq":140,"At":570,"As":2214,"D ":620,"Ba":15003,"Az":1110,"Ay":310,"Ae":197,"Af":831,"Ag":889,"Ah":1002,"Ab":2089,"Ac":1560,"Ad":704,"Am":3400,"An":3447,"Ap":1094,"Ai":809,"Aj":96,"Ak":735,"Al":5346,"By":81,"Hir":81,"Hit":715,"His":108,"Bu":5575,"Br":2582,"Ca":2733,"E ":430,"IM ":85,"Bh":473,"Bi":1727,"Hid":126,"Be":7488,"Hig":156,"Hij":131,"Hik":136,"Hil":325,"Bo":2057,"Bl":490,"Hin":334,"Ku":4706,"Ky":121,"Kl":380,"Kr":939,"Ko":4380,"Le":2149,"Li":1937,"N ":1325,"La":11882,"Lu":2055,"Ly":128,"Lo":3499,"Me":9475,"Mi":2501,"O ":757,"Ma":21779,"Mc":109,"My":256,"Mu":4039,"Mo":3221,"Ni":961,"Ng":163,"Ne":3358,"Na":4580,"P ":1043,"Hel":170,"Ny":154,"Nu":712,"No":2652,"Hea":82,"Ok":544,"Ol":491,"Om":176,"On":254,"Og":727,"Oc":198,"Hen":128,"Of":107,"Her":410,"Ob":148,"Gi":647,"Gh":173,"Gl":218,"Gr":2002,"Go":908,"Gu":1935,"Cô":450,"J ":232,"Ha":4713,"He":1106,"Hi":2278,"Ho":2093,"Hu":2027,"Hy":124,"K ":459,"Ib":1386,"Ia":4453,"Id":210,"Ic":100,"Ip":196,"Im":670,"In":13413,"Ik":260,"Il":316,"IC ":92,"Is":3633,"It":1183,"Ir":732,"Ja":4941,"L ":509,"Ji":423,"Je":6467,"Jo":1642,"Had":169,"Hab":110,"Ju":2146,"Hal":335,"Hak":118,"Haj":534,"Hai":91,"Ka":14796,"M ":4357,"Han":529,"Ham":347,"Kh":882,"Hat":137,"Kg":139,"Has":393,"Har":1041,"Ki":2010,"Haw":106,"Ke":11600,"Hau":497,"Us":180,"Ut":2903,"Ur":241,"Up":83,"Um":385,"Un":2182,"Uk":86,"Ul":440,"Ud":348,"Ub":107,"W ":157,"Tu":2321,"Tr":1038,"Ts":110,"To":1817,"Th":2070,"Ti":5675,"Te":14128,"Ta":5699,"V ":895,"Côt":448,"Sw":424,"Sy":2644,"St":2134,"Su":9154,"Wo":465,"Wi":2174,"Wh":136,"Wa":2486,"Rü":119,"We":1119,"Y ":91,"Vo":1700,"Vi":1229,"X ":350,"Va":902,"Ve":612,"Uz":98,"Qa":209,"Qi":91,"Gun":810,"Pu":4399," م":146,"Gur":224,"Pr":5037,"S ":1438,"Py":150,"Gua":329,"Pe":16586,"Pa":11390," س":132,"Gui":240,"Pl":433,"Po":3845,"Pi":2607,"Ph":402,"Gul":81,"Os":257,"Ot":149,"Ou":112," ا":503,"Op":211,"Or":706,"R ":526," ب":135,"Se":15468,"Sd":126,"Sc":1468,"Si":4011,"Sh":1862,"Sm":89,"Sl":322,"Sk":239,"Sr":539,"Sp":508,"So":1552,"Ru":1209,"U ":260,"Sa":7125,"Re":2654,"Ri":1317,"Rh":2165,"Ro":2065,"Qu":662,"T ":567,"Ra":4724,"Gre":1162,"Gra":462,"b ":4501,"Gro":225,"a ":210782,"Ye":197,"Ya":1244,"Yo":982,"Yu":587,"Glo":89,"Sü":116,"Gon":110,"Gol":142,"Gor":139,"Za":844,"Ze":342,"Zi":166,"Zo":133,"Zu":270,"God":87,"i ":164203,"gd":604,"ge":16197,"ga":76046,"fk":164,"Ing":1603,"Inf":85,"fl":433,"fg":113,"ff":772,"fi":6374,"Ini":272,"fs":278,"fr":1213,"fu":556,"ft":567,"fo":1845,"Int":557,"Ins":318,"j ":686,"gy":603,"hd":641,"he":6433,"hb":328,"ha":43380,"gn":1513,"gm":169,"gl":1474,"gk":8707,"gj":210,"gi":15794,"gh":2760,"gg":14722,"gu":11142,"gt":306,"gs":5117,"gr":2072,"go":5967,"dt":171,"du":14415,"dv":115,"dw":512,"dy":480,"dz":256,"g ":103343,"Ima":490,"ea":5087,"eb":49697,"ec":13590,"aß":86,"ed":9261,"de":23626,"Ili":91,"dd":794,"dg":528,"di":100113,"dh":461,"dk":453,"dm":900,"dl":344,"do":13268,"dn":327,"ds":769,"dr":1709,"ew":3500,"ex":782,"eu":1721,"ev":2199,"ey":1335,"ez":1091,"fa":3015,"h ":128258,"Ind":10245,"fe":1363,"eh":14770,"eg":12184,"ef":868,"ee":2807,"el":48944,"ek":16873,"ej":6455,"ei":5276,"ep":15359,"Imp":84,"eo":5366,"en":99039,"em":39155,"et":34806,"es":39797,"er":193978,"eq":83,"ca":17842,"Ika":175,"e ":34293,"by":357,"bs":314,"br":1904,"bu":54229,"bt":124,"bn":513,"bo":5898,"bj":669,"bk":504,"bl":1565,"bi":16194,"bb":374,"bd":1187,"be":37633,"dc":103,"db":909,"da":137597,"f ":4203,"cy":378,"cu":2092,"ct":1148,"cs":129,"cr":524,"co":3401,"ck":2599,"cl":364,"ci":8015,"ch":6428,"ce":6430,"cc":744,"c ":1819,"az":2511,"ay":31162,"ba":73332,"d ":17846,"at":94936,"as":52637,"ar":111647,"aq":379,"ax":268,"aw":17249,"av":1840,"au":24752,"ak":88368,"al":109743,"ai":38299,"aj":9650,"ao":666,"ap":27642,"am":64984,"an":422498,"ac":4688,"ad":56503,"aa":11612,"ab":20825,"ag":21007,"ah":136447,"ae":8470,"af":2405,"nu":8816,"nt":38718,"ns":10123,"nr":1326,"np":780,"no":5581,"nn":4359,"q ":402,"nz":372,"ny":25233,"nw":221,"nv":296,"oe":721,"of":2166,"oc":1646,"od":3040,"oa":966,"ob":2206,"om":14310,"ké":379,"on":34226,"ok":8406,"ol":25149,"oi":2136,"oj":754,"og":4180,"oh":4267,"m²":560,"ot":8011,"os":7149,"ov":6595,"ou":3897,"op":4862,"oo":1785,"or":25529,"r ":54659,"ox":186,"ow":1588,"oz":259,"oy":682,"pe":46877,"pa":95395,"pl":1464,"pm":86,"pn":114,"po":7399,"ph":1253,"pi":8396,"pk":282,"lo":8490,"ln":692,"lm":890,"ll":5253,"ls":1529,"lp":1203,"lw":82,"lv":509,"lu":18399,"lt":2503,"lz":145,"ly":913,"hô":463,"o ":11973,"Idr":97,"ma":65762,"mb":18534,"me":75198,"mf":114,"mk":292,"ml":733,"iè":101,"mi":11715,"mn":744,"mm":2305,"mp":19625,"mo":4147,"mr":122,"mt":113,"ms":473,"mu":20069,"my":327,"p ":5732,"na":48153,"nb":1690,"nc":10083,"nd":46633,"ne":28218,"nf":758,"ng":182026,"nh":406,"ni":30289,"nj":7927,"nk":2659,"nl":387,"nm":699,"ju":11519,"jr":154,"jo":1281,"jl":522,"Ibr":373,"ki":14104,"kh":3991,"Ibu":746,"ke":44562,"kb":86,"ka":108120,"Ibn":142,"m ":42736,"fö":176,"kw":208,"ky":1117,"ks":3853,"kt":4143,"ku":13823,"ko":15955,"kp":103,"kr":1887,"kk":1128,"kl":2495,"km":1674,"kn":2015,"li":41181,"lh":268,"lk":1910,"lj":112,"le":52683,"ld":1859,"lg":478,"lf":422,"la":150999,"lc":220,"lb":2609,"n ":280458,"hr":1238,"hs":425,"hw":450,"ht":1165,"hu":13402,"hk":2033,"hi":14383,"hn":1231,"ho":3729,"hl":3211,"hm":1374,"id":12680,"ic":6147,"ib":7125,"ia":71186,"ih":7528,"ig":7857,"if":3795,"ie":5178,"hy":546,"k ":62478,"iq":398,"ir":20180,"is":34419,"it":31463,"iu":2005,"iv":2344,"iw":1960,"ix":314,"ii":592,"ij":1917,"ik":34589,"il":33625,"im":21680,"in":69236,"io":6266,"ip":10779,"je":4970,"ji":4809,"iz":1664,"iy":922,"l ":29072,"ja":22803,"xi":565,"xo":133,"té":82,"xt":102,"ww":150,"z ":1327,"xa":185,"Ia ":4390,"xe":102,"wi":8007,"wn":252,"wo":532,"ws":319,"wu":517,"vy":83,"rö":281,"y ":7880,"wa":23770,"we":3150,"ré":209,"vi":7596,"vu":82,"vr":113,"vo":1142,"uz":1379,"uy":326,"ux":309,"uw":287,"uv":269,"uu":95,"ve":5122,"va":1988,"x ":1392,"ui":3189,"uj":3571,"uk":26920,"ul":24992,"ue":2226,"uf":806,"ug":5578,"uh":8031,"uq":123,"ur":29560,"us":17527,"ut":20841,"um":18481,"un":56650,"uo":172,"up":38959,"ty":1373,"tz":380,"tu":37333,"tt":1895,"tw":249,"tv":82,"ub":6360,"ua":55927,"ud":9184,"uc":1791,"w ":1480,"to":8847,"tn":1031,"tm":404,"tl":983,"ts":1145,"tr":7210,"tp":463,"tf":106,"te":79676,"td":166,"tk":3583,"ti":36996,"th":4853,"v ":487,"tb":138,"tc":212,"ta":118349,"su":13746,"ss":3665,"st":16805,"sy":3492,"sw":1296,"sl":3506,"sk":3092,"sn":2184,"sm":1932,"sp":2237,"so":3953,"sr":496,"sd":136,"sc":1307,"sf":360,"se":78194,"sh":2942,"sg":539,"sj":1138,"si":55053,"rz":381,"u ":48453,"sa":62323,"sb":776,"rr":2104,"rs":8875,"rt":16822,"ru":47839,"rv":432,"rw":832,"ry":2374,"rp":3350,"ro":15649,"rn":7372,"rm":10746,"né":180,"rl":23672,"rk":11585,"rj":2659,"ri":56760,"rh":2483,"rg":8124,"rf":712,"re":18314,"rd":8278,"rc":2216,"rb":22662,"ra":108362,"t ":48210,"qu":886,"qi":117,"qa":200,"s ":44138,"px":487,"py":85,"pt":2000,"pu":18990,"pp":606,"pr":4620,"ps":516,"Hut":228,"Hus":197,"Hun":189,"Hub":115,"Hul":888,"IV ":93,"zz":209,"Hor":109,"zh":286,"zi":2979,"Hou":88,"zb":112,"Hos":142,"ze":1129,"za":3097,"yy":396,"Hon":251,"yz":105,"Hol":986,"zu":864,"zo":647,"zn":118,"zm":124,"zl":139,"yg":94,"yh":122,"ye":3480,"yc":226,"yd":229,"ya":87677,"yb":95,"yu":3240,"yt":295,"ys":11634,"yr":594,"yp":206,"yo":1324,"yn":309,"ym":262,"yl":472,"yi":2320,"Arg":299,"Are":112,"Ard":238,"Ara":1408,"Arm":211,"Ari":140,"Aqu":111,"Apr":696,"Atl":206,"Ast":233,"Ass":149,"Ata":113,"Ash":102,"Asi":970,"Art":249,"Asa":432,"Aut":127,"Aus":928,"Awa":270,"Aze":184,"Azi":185,"Azu":461,"Aye":93,"Aya":126,"CA ":82,"Bag":475,"Bah":1958,"Bai":201,"Bak":335,"Bal":892,"Ban":3541,"Bab":232,"Bac":93,"Bad":566,"Baw":87,"Bay":277,"Bar":4153,"Bat":1275,"Bas":494,"Bau":97,"Abu":317,"Aca":95,"Abb":90,"Aba":147,"Abd":1143,"Abj":137,"Abi":121,"Ada":244,"Ace":1263,"Adv":82,"Adi":102,"Aer":103,"Age":96,"Aga":333,"Afr":636,"Afg":103,"Aha":85,"Agu":275,"Ago":91,"Ahm":452,"Ahl":397,"Air":528,"Al ":922,"Aka":166,"BN ":115,"Akt":218,"Ala":509,"Alb":985,"Alg":101,"Ali":409,"Ale":191,"Alu":88,"Alt":207,"Alm":117,"All":368,"Alo":177,"Alp":987,"Ame":2205,"Ami":153,"Ama":359,"Amu":131,"Amp":255,"Ang":606,"Ani":107,"Ana":284,"And":339,"Anu":311,"Ant":1304,"Ann":89,"Api":115,"Apa":124,"But":135,"Buk":2249,"Bul":563,"Bum":345,"Bun":442,"Bur":1072,"Bud":212,"Bua":152,"Bru":298,"² ":569,"DA ":144,"Cab":88,"Cal":547,"Cam":344,"Cas":261,"Car":564,"Cat":148,"Can":269,"Cap":134,"Bea":159,"Beb":103,"Bes":855,"Bet":145,"Ber":2024,"Ben":734,"Bel":2825,"Bek":161,"Bei":136,"Bid":98,"Bia":120,"Bin":573,"Bil":164,"Bis":119,"Bit":82,"Bir":213,"Bio":85,"Bha":303,"Bhd":125,"Bla":207,"Bre":159,"Bra":1009,"Bro":237,"Bri":839,"Bol":345,"Bon":245,"Boo":88,"Bor":302,"Bos":100,"Bot":109,"Bou":267,"Boy":81,"Cze":166,"De ":309,"Der":103,"Des":621,"Dev":116,"Dew":569,"Del":214,"Dem":195,"Den":384,"Dea":94,"Dam":225,"Dan":256,"Dar":1495,"Das":103,"Dat":1195,"Dau":165,"Dav":161,"Day":273,"Dae":1400,"Dah":132,"Dag":88,"Dai":144,"Dal":1240,"Cho":141,"Chr":187,"Che":558,"Chi":1206,"Chu":173,"Cin":942,"Cit":148,"Cir":82,"Cla":127,"Cem":85,"Cen":218,"Cer":238,"ße":95,"Cha":1124,"Cre":111,"Cro":154,"Coc":102,"Cos":83,"Cor":310,"Com":467,"Col":444,"Con":500,"Cou":501,"アアア":393,"ôn":487,"FA ":123,"ër":184,"ém":402,"én":144,"ér":118,"ë ":295,"ée":141,"èr":124,"Edi":123,"Eck":180,"é ":327,"Eas":113,"Do ":123,"ān":100,"Dia":606,"Dib":90,"Dic":128,"Dit":98,"Dis":1199,"Dir":367,"Dip":183,"Din":297,"Dil":85,"Dig":111,"Di ":495,"Dua":114,"Dun":670,"ể":84,"ür":122,"Dur":174,"Dus":152,"üg":114,"üd":115,"Dra":745,"ôt":453,"ör":194,"Dr ":339,"öe":242,"Dou":90,"Dol":113,"Don":404,"Dom":132,"Neg":1871,"Neu":122,"Net":159,"Nas":509,"Nat":234,"Nav":84,"Naz":81,"Nig":91,"Nic":169,"Nin":95,"Nik":104,"New":731,"Mya":98,"Nar":191,"Nak":108,"Nam":1815,"Nan":583,"Nag":142,"Nai":85,"Naj":112,"Nab":285,"Nun":101,"Nur":239,"Nus":207,"Nov":710,"Nor":1237,"Noo":205,"Oct":88,"Ogo":658,"Okt":487,"PA ":297,"Oli":227,"Oma":141,"Ope":136,"Ora":305,"Ori":90,"Pla":316,"Pin":1148,"Pil":423,"Pis":85,"Pie":157,"Pic":109,"Pia":318,"Phi":161,"Ped":98,"Per":10202,"Pes":304,"Pet":530,"Pem":974,"Pen":3021,"Pej":89,"Pek":281,"Pel":589,"Peg":132,"Pay":578,"Pat":250,"Pas":1115,"Par":2423,"Pav":366,"Pau":192,"Pah":601,"Pag":90,"Pad":1577,"Pan":1375,"Pam":171,"Pap":263,"Pal":1707,"Pak":472,"RA ":106,"ō ":90,"Pyr":101,"Puc":123,"Pun":218,"Pur":154,"Put":1136,"Pus":493,"Pue":122,"Pul":1812,"Pro":3699,"Pri":366,"Pre":620,"Pra":231,"Pok":1401,"Pol":629,"Pon":245,"Pot":126,"Pos":316,"Pop":197,"Por":490,"Pow":89," ال":410,"RM ":2481,"RI ":158,"RC ":114,"SA ":131,"Rab":140,"Rad":225,"Rai":103,"Raj":807,"Rah":353,"Ram":356,"Ran":504,"Rak":486,"Que":146,"Qur":321,"Isa":108,"Ita":1073,"Isk":108,"Ism":235,"Isl":1820,"Isr":240,"Ist":853,"Ipo":169,"Ira":444,"Ire":182,"Jac":170,"Jab":343,"Jar":133,"Jan":881,"Jam":523,"Jal":1059,"Jak":191,"Jel":140,"Jem":105,"Jen":531,"Jep":1274,"Jer":4043,"Jea":98,"Jay":337,"Jaw":691,"Jin":94,"あ":296,"Jos":163,"ア":682,"Joh":1111,"KR ":83,"Jua":218,"Jul":729,"Jun":707,"Jum":163,"Kad":192,"Kab":4567,"Kai":155,"Kam":1464,"Kal":3891,"Kak":93,"Kaj":109,"Kap":368,"Kan":1394,"Kau":151,"Kat":284,"Kas":307,"Kar":941,"Kaz":84,"Kaw":364,"Kay":157,"Ker":1245,"Kes":641,"Ket":677,"Ken":331,"Kep":862,"Kej":260,"Kel":1647,"Kem":873,"Kec":1697,"Keb":1923,"Ked":783,"Kg ":138,"Kea":187,"가가 ":106,"Kis":136,"Kir":134,"Kit":181,"Kin":961,"Kid":196,"Kil":86,"Kim":88,"Kha":610,"Kla":249,"Kon":712,"Kom":706,"Kol":252,"Kor":569,"Kot":1614,"Kob":83,"Kre":356,"Kra":141," ア":106,"Kri":351,"Kua":1867,"Kub":239,"Kuc":105,"Kui":101,"Kum":474,"Kul":245,"Kun":284,"Kup":104,"Kus":108,"Kur":232,"Kut":677,"Les":206,"Lep":84,"Leo":109,"Len":99,"Lem":434,"Lee":107,"Leb":345,"Lau":1422,"Law":147,"Lay":85,"Le ":130,"Lak":332,"Lal":252,"Lag":398,"Lah":252,"Las":115,"Lat":437,"Lar":140,"Lao":109,"Lap":5633,"Lam":637,"Lan":870,"Lad":118,"Lab":297,"MA ":87,"La ":382,"MP ":115,"Lib":174,"Lia":116,"Lig":184,"Lim":361,"Lin":364,"Lit":120,"Liv":87,"Lui":82,"Lum":953,"Lub":167,"Lua":151,"Lud":173,"Luc":91,"Lou":140,"Los":104,"Lot":137,"MS ":96,"Loi":929,"Lor":631,"Lon":654,"Lom":266,"Lok":116,"NG ":89,"Mek":105,"Mei":562,"Men":2125,"Mem":236,"Mel":2611,"Mes":469,"Mer":1020,"Met":202,"Mec":896,"Meg":95,"Med":514,"Mex":375,"Man":1319,"Mal":11656,"Mam":120,"Mar":2036,"Mas":1315,"Mag":286,"Mad":347,"Maj":610,"Mak":620,"Mah":703,"Mai":531,"Mac":771,"May":201,"Maz":85,"Mau":200,"Mat":615,"Mod":145,"Moh":709,"Mon":755,"Mos":127,"Mor":550,"Mou":293,"Mot":160,"Mik":122,"Mid":149,"Mic":276,"Mit":90,"Mir":149,"Mis":255,"Mil":498,"Min":689,"NO ":270,"Muz":273,"Mun":209,"Mul":257,"Muk":242,"Muh":672,"Mut":94,"Mur":403,"Mus":909,"Mud":234,"Mua":519,"Süd":110,"Wor":178,"Won":85,"Wik":155,"Wil":1358,"Win":285,"Wit":159,"ère":118,"Wei":137,"Wes":721,"Wah":96,"Rüg":112,"Was":81,"War":783,"Wat":152,"Wan":460,"Wak":137,"Wal":495,"ées":91,"Vos":513,"Vor":961,"Vol":89,"Vis":137,"émo":374,"Zea":181,"Zai":220,"Zam":306,"éné":87,"乙":83,"之":243,"並":163,"丘":125,"三":343,"丁":209,"Yus":150,"Yun":286,"Yor":143,"Yon":245,"Yog":442,"Yam":136,"Yan":424,"Yah":251,"Syr":106,"Sys":90,"Sya":1981,"Sye":220,"Swi":148,"Swe":223,"Sur":638,"Sus":89,"Sum":1496,"Sul":3001,"Suk":631,"Sup":214,"Sun":2363,"Sud":110,"Sub":100,"Sua":114,"Str":327,"Stu":188,"Sto":152,"Sta":593,"Ste":575,"Teb":161,"Tea":82,"Ten":4201,"Tem":412,"Teo":86,"Tel":885,"Tek":246,"Tap":195,"Tam":976,"Tan":1919,"Tas":214,"Tat":100,"Tar":410,"Taw":155,"Tau":110,"Tai":272,"Taj":110,"Tak":203,"Tal":221,"Tah":219,"Tab":349,"TM ":239,"Shi":244,"She":230,"Sho":202,"Sha":998,"Sim":492,"Sil":414,"Sik":106,"Sit":261,"Sis":302,"Sir":341,"Sip":89,"Sin":1116,"Sid":166,"Sie":82,"Sib":139,"Sia":209,"Ses":148,"Ser":1397,"Set":535,"Sdn":124,"Sep":1336,"Sen":771,"Seo":93,"Sel":5561,"Sem":1319,"Sej":326,"Sek":1892,"Sei":202,"Seg":241,"Sed":118,"See":220,"Seb":744,"Sec":184,"Sea":113,"Sri":518,"TV ":585,"Spa":92,"Spi":83,"Spe":180,"St ":234,"Sou":282,"Sov":188,"Sol":221,"Som":81,"Son":251,"Sos":91,"Sle":97,"TO ":91,"Slo":108,"TP ":81," 三":99,"Rus":662,"Rum":183,"Sai":1025,"Sah":128,"Sak":152,"Sam":586,"Sal":707,"Saa":187,"Sab":832,"Sco":163,"Sch":1149,"Say":155,"Sat":343,"Sau":314,"Sar":891,"Sas":105,"San":1065,"Sao":85,"SI ":111,"Res":223,"Rev":110,"Rhi":1420,"Rhe":239,"Rio":130,"Riv":156,"Rin":102,"Ria":192,"Ric":211,"Rid":172,"Rat":181,"Ras":224,"Rap":81,"Raw":128,"Raz":189,"Ray":672,"Red":114,"Rei":85,"Rem":165,"Ren":489,"Rek":84,"Rep":799,"Rob":171,"Roc":116,"Roy":88,"Rou":105,"Rot":215,"Ros":222,"Rom":673,"SS ":126,"SM ":492,"Rhô":457,"Ven":208,"Van":91,"Val":299,"Var":384,"Vic":157,"Vie":298,"Vir":90,"Vil":217,"Ver":240,"Una":261,"Und":396,"Uni":1237,"Unt":117,"Ula":82,"Ulu":237,"Uma":96,"Umu":153,"Uta":2604,"Uth":159,"Uda":328,"Ter":7545,"Tet":82,"Tha":659,"The":1090,"Tho":125,"Tid":104,"Tig":175,"Tim":3943,"Tin":840,"Tit":106,"Tir":130,"UN ":205,"Tor":170,"Tok":365,"Tol":171,"Tom":154,"Ton":166,"Tow":90,"Tou":95,"Tob":83,"Toj":142,"Tro":148,"Tri":323,"Tre":155,"Tra":341,"Tur":534,"Tuj":92,"Tul":101,"Tum":183,"Tun":599,"Tua":360,"Tuh":175,"ße ":87,"ああ":178,"bka":467,"bje":424,"bja":244,"bis":253,"bit":3310,"bio":260,"bir":1273,"bil":2147,"bim":123,"bin":2755,"bih":1923,"bij":157,"bo ":233,"blo":96,"ble":283,"bli":991,"bn ":236,"bla":135,"bok":140,"bol":2635,"boj":88,"boh":101,"bni":142,"bon":529,"bom":143,"bor":733,"bot":679,"bou":178,"bbe":102,"bba":131,"be ":164,"ban":31880,"bak":943,"bal":1621,"bai":843,"baj":91,"bag":10103,"bah":10190,"bac":413,"bad":1572,"baa":211,"bab":844,"bd ":101,"bay":335,"baw":1427,"bau":218,"bat":5257,"bas":903,"bar":5076,"bap":179,"beb":1241,"bea":94,"bdu":982,"bi ":755,"beg":153,"ber":24299,"ben":3312,"bel":3864,"bek":999,"bez":543,"bes":2425,"bet":286,"bia":1988,"bib":110,"bic":100,"bid":732,"ca ":950,"car":3573,"cas":256,"cat":393,"cau":90,"can":1363,"cap":475,"cay":362,"cac":97,"cab":306,"cad":172,"cak":355,"cam":7892,"cal":653,"cai":202,"cah":474,"ce ":1976,"bri":257,"bro":107,"bra":750,"bre":165,"bu ":2368,"bru":584,"bsi":118,"bua":29593,"bup":5795,"bur":3331,"bul":1177,"buk":2032,"bun":2518,"bum":1048,"buh":3221,"bud":788,"but":1884,"bus":270,"by ":195,"aka":43030,"am ":24035,"ake":622,"aki":2473,"akh":1543,"ajl":518,"aji":1471,"ajo":353,"aju":766,"aiz":292,"al ":13357,"aja":6197,"aij":188,"aik":1144,"ail":1086,"aim":441,"ain":8906,"aip":152,"air":1407,"ais":498,"ait":2354,"aiw":1231,"ak ":31786,"ahy":162,"ahw":146,"aid":269,"aib":213,"aia":983,"ahm":450,"ahn":440,"ahk":975,"ahl":1517,"ahi":2365,"ahu":8672,"ahs":174,"ahr":292,"aj ":128,"ahe":94,"aha":18129,"agh":136,"agi":6470,"agu":1752,"agn":508,"ago":378,"aq ":213,"anu":3170,"anz":119,"any":7283,"anp":517,"ano":698,"ann":2528,"anm":135,"ant":12519,"ans":983,"ane":1046,"ang":113274,"ani":5398,"anj":3436,"ank":1191,"anl":96,"ap ":2995,"ana":12204,"anb":114,"anc":5840,"and":21721,"amu":780,"amm":801,"aml":124,"amo":344,"amn":175,"amp":5004,"ams":199,"amk":171,"ami":1889,"ame":1030,"amb":3055,"ama":27062,"ao ":367,"alv":310,"alu":2781,"alt":434,"als":328,"alp":100,"alo":978,"aln":492,"alm":207,"all":932,"alk":853,"alg":121,"ali":15559,"alc":93,"ald":621,"ale":1649,"ala":69871,"alb":641,"an ":229755,"akw":163,"aky":793,"aks":1768,"akr":316,"aku":1710,"akt":1506,"ako":830,"akn":464,"akk":560,"akm":163,"akl":477,"aba":7456,"abe":538,"abi":1679,"abj":107,"abk":443,"abl":125,"abo":245,"abr":124,"abu":7287,"ae ":779,"aca":1070,"aah":88,"aan":10415,"aar":216,"aat":274,"ad ":3530,"ac ":681,"aa ":82,"ab ":2395,"aft":225,"afs":96,"aff":132,"afi":651,"ai ":18726,"aga":10764,"age":668,"aen":91,"ael":376,"aer":6596,"aed":290,"ah ":102722,"afa":559,"ado":684,"adr":137,"adm":127,"adh":132,"adi":4970,"add":126,"ade":739,"adz":85,"adu":655,"aco":144,"ack":348,"aci":257,"ach":779,"ace":635,"acc":89,"ada":44005,"adb":874,"af ":525,"act":193,"acu":251,"azi":1065,"aze":259,"azh":86,"aza":605,"az ":156,"ayi":95,"ayo":257,"ays":11200,"ayu":2352,"aya":15557,"aye":177,"ba ":1121,"at ":26256,"arh":224,"arg":2527,"arf":134,"are":909,"ard":1182,"arc":474,"arb":558,"ara":39644,"arp":171,"aro":691,"arn":2199,"arm":521,"arl":1083,"ark":4152,"arj":275,"ari":23309,"aru":4058,"arv":121,"arr":588,"ars":575,"art":3752,"au ":15214,"asa":23237,"ary":969,"asi":8187,"ash":536,"asc":171,"ase":633,"aso":149,"asn":586,"asp":182,"ask":1160,"asj":1090,"asm":720,"asl":278,"aos":96,"ar ":23228,"apa":19852,"ape":224,"api":1939,"aph":119,"apl":112,"apk":212,"apo":341,"app":137,"apt":214,"apu":1299,"as ":9010,"ava":423,"ax ":100,"aut":4080,"avo":84,"avi":800,"ave":328,"awe":1878,"ay ":1141,"awa":14399,"awi":616,"ata":35618,"asu":2536,"ast":2146,"ass":961,"asr":128,"asy":799,"atm":81,"atn":277,"atk":1026,"atl":166,"atr":347,"ato":1267,"ate":9872,"atd":105,"ati":6436,"ath":634,"aw ":128,"aua":793,"auc":85,"aub":89,"att":488,"ats":175,"atu":11913,"aul":410,"aum":504,"aun":617,"aup":666,"aur":353,"aus":360,"aud":602,"aue":165,"auf":124,"auh":298,"auk":81,"ς ":105,"Zul":114,"Zui":82,"アア":532,"jer":102,"jek":688,"jel":278,"jem":250,"jen":3319,"jet":96,"ji ":1240,"jad":2581,"jaa":1731,"jab":2775,"jat":469,"jas":271,"jau":469,"jar":2821,"jal":1171,"jak":1017,"jan":2215,"jam":827,"jah":1005,"jag":216,"jaj":319,"jai":194,"jay":698,"jaw":545,"joh":284,"jol":87,"jon":105,"jor":354,"ск":92,"jis":137,"jir":304,"jin":516,"jik":249,"jil":131,"jid":1157,"jia":666,"jib":155,"jo ":323,"jli":517,"itk":2435,"itr":182,"ito":856,"itu":5709,"itt":386,"its":183,"itz":235,"ity":553,"iub":115,"isk":246,"ism":665,"isl":386,"iso":280,"isn":315,"iss":300,"üdl":82,"isu":481,"ist":4387,"isy":1010,"ita":6918,"ite":4203,"ith":286,"iti":5204,"iwa":1799,"ius":385,"iur":92,"ium":973,"iul":90,"iva":248,"üge":111,"ix ":251,"ivi":886,"ive":1052,"ipo":113,"ipp":136,"ipu":557,"ips":84,"ipt":701,"ipi":916,"ipl":145,"is ":15255,"ion":3684,"iop":132,"ior":170,"ios":146,"iot":81,"iol":260,"ipa":5888,"ipe":1538,"ir ":5942,"iru":594,"iro":343,"irm":96,"irn":148,"irk":679,"irl":158,"iri":4503,"isi":4078,"ish":1001,"ise":2265,"isc":471,"isb":98,"isa":2790,"iqu":224,"ire":1729,"irg":93,"ira":5164,"ird":90,"irc":278,"it ":3986,"ja ":3286,"iya":450,"iz ":428,"가가가":123,"izi":398,"ize":101,"iza":490,"iyy":293,"kim":1100,"kil":906,"kia":289,"kib":224,"kiy":103,"kin":1562,"kip":238,"kir":1682,"kis":850,"kit":2779,"km ":794,"ki ":4083,"khl":192,"khi":1569,"khe":118,"khb":203,"kha":646,"khu":413,"kht":143,"kho":147,"kea":516,"keb":1435,"kec":7220,"ked":1408,"kee":288,"keg":389,"keh":350,"kek":487,"kej":496,"kem":2040,"kel":6287,"ken":4505,"kep":5037,"kes":1492,"ker":4473,"keu":171,"ket":2676,"kew":229,"key":94,"kh ":421,"ke ":4792,"kra":370,"kre":451,"ksa":750,"kse":201,"ku ":3568,"kro":241,"kri":758,"kov":81,"km²":556,"kot":1484,"kos":215,"kor":487,"kop":296,"koo":92,"kon":2602,"kom":4887,"kol":2649,"kok":1924,"koh":327,"kod":435,"ks ":713,"kmu":121,"kna":192,"kny":1047,"kni":375,"kno":375,"kka":941,"kki":112,"klu":484,"ko ":261,"kma":107,"kle":1198,"kla":510,"klo":121,"kli":164,"jut":411,"jul":111,"juk":2111,"jun":1303,"jum":727,"jur":705,"jua":1342,"juh":227,"jug":3370,"jud":682,"ju ":351,"jra":136,"kaw":5261,"kay":507,"kat":7066,"kau":601,"kar":4772,"kas":2281,"kap":1840,"kan":65462,"kal":3979,"kam":1403,"kaj":351,"kak":575,"kah":502,"kai":1379,"kae":207,"kad":826,"kac":140,"kab":1684,"kaa":721,"ka ":8387,"för":174," Ga":1910," Ge":1684," I ":473," Fo":858," Fu":333," Fr":765," Fi":2557," Fl":395," Ha":4690," He":1102," Cô":450," J ":133," Go":899," Gr":1994," Gu":1933," Gh":173," Gi":642," Gl":218," Id":208," Ic":100," Ib":1384," Ia":4449," K ":99," Hy":123," Hu":2023," Ho":2089,"ha ":1152," Hi":2274," Ji":422," Je":6459," L ":153," Ja":4925,"ان ":99," Ir":730," Is":3629," It":1183," Im":670," In":13399," Ip":196," Ik":259," Il":313,"ham":2871,"han":12119," M ":391,"hap":427,"hai":1915," Ka":14783,"haj":358,"hak":893,"hal":1601," Ke":11588,"hau":162,"haw":466," Ki":2005,"har":6324," Kg":139,"has":7477," Kh":876,"hat":1258," Jo":1631," Ju":2140,"haf":92,"hae":147,"hah":683,"hag":2675,"hab":563,"haa":110,"had":1334," N ":147," La":11856," Le":2143," Li":1930," Kl":380,"hba":248,"hd ":532," Ko":4376,"haz":127," Kr":939,"hay":446," Ku":4698," Ky":121," Mc":109," Ma":21745," Mi":2499," Me":9466,"he ":2354," Lo":3490," Ly":128," Lu":2053," Ne":3341," P ":266," Na":4572," Ng":163," Ni":955," Mo":3210," My":254," Mu":4029,"hek":90,"hel":428,"hei":620,"hee":103,"hea":136,"heb":90," A ":513,"het":112,"hes":177,"her":867,"hen":757,"hem":241,"hi ":1051," B ":222," C ":269," Ap":1093," Am":3390," An":3440," Ak":734," Al":5334," Ai":808," Aj":96," Ag":886," Ah":1001," Ae":195," Af":829," Ac":1559," Ad":700," Ab":2084," Ba":14981," D ":146," Az":1109," Ay":309," Aw":291," Av":216," Au":1348," At":566," As":2194," Ar":3152," Aq":140," Be":7481,"hie":97,"hid":1769,"hic":143,"hib":221," Bi":1723," Bh":473,"hia":286,"hip":145,"hin":4765,"him":801," Bl":487," Bo":2041,"hil":602,"hik":165,"hij":260," Br":2580," Bu":5571,"his":190,"hit":603," By":81,"hir":2908," E ":101," Ca":2706," Ce":812," Ci":1452," Ch":3498," Cl":318," Cr":466," Co":2741," Cu":414," Cy":178," Cz":168," F ":213,"hka":1860," Da":7101," Di":4059," Dh":92," De":3032," Dr":1222," Do":1228,"hko":119," Du":1487,"hn ":373," Ea":212," Ec":309," Ed":292," G ":103,"hla":260,"hle":884," El":545," Ek":289," Ei":91,"hli":1784," Eh":94," Et":117," Es":289," Er":725," En":922," Em":827,"hlu":168," Ex":176," Eu":253," Ev":109,"ho ":208," Fe":1224,"hma":998," Fa":1363," H ":142,"gma":83,"go ":689,"glo":121," Sü":116,"gle":434,"gli":352,"gla":485," Wo":453,"gku":1265," Wi":2169," Wh":135," We":1105,"gko":307," Wa":2483," Rü":119,"gkr":129,"gke":128,"gki":641,"gog":193," Zu":270," Zo":131,"gny":427," Ze":341," Zi":163,"gni":98," Za":841,"gne":632,"gna":169," Yu":587," Yo":978," Ya":1240," Ye":195,"gs ":118,"gol":750,"gon":466,"gos":757,"gor":1882,"got":513,"gov":355,"gsa":3814,"gu ":1642,"gsi":463," a ":258,"gro":268,"gra":1121,"gri":124,"gre":287," R ":136,"gtu":144," Ou":112,"gto":99," Os":256," Ot":149," Or":704," Op":210,"gug":107," Po":3822,"gui":166," Pl":428,"guk":111," Pi":2601,"gum":149,"gul":272," Ph":397,"gua":691," Pe":16557,"gub":432,"gue":251," Pa":11353,"gsu":450," Ny":144,"gsl":154," Nu":712," No":2649," Ol":489," Ok":544," On":249," Om":175," Og":727," Oc":198," Of":98," Ob":147," Ra":4674," T ":97,"grö":241," Qu":659," Ro":2055," Re":2651," Ri":1308," Rh":2165," Py":149," S ":208,"gut":128," Pr":5024,"gur":850,"gus":381," Pu":4397,"gun":5744," Qi":91," Qa":209," Sy":2644," Sw":421," Su":9144," St":2094," Ta":5685," V ":104,"gya":453," Th":2050," Ti":5667," Te":14107," Tr":1030," Ts":110," To":1798," Ru":1202," Sa":7117," Sh":1855," Si":3998," Sc":1446," Sd":126," Se":15446," So":1545," Sp":492," Sr":539," Sk":238," Sl":322," Sm":87," Uz":98," Va":900," X ":203," Ve":610," Vi":1221," Vo":1699," Tu":2312," Ub":106," Ud":348," Uk":86," Ul":439," Um":384," Un":2175," Up":83," Ur":240," Us":180," Ut":2902," ja":4995,"iai":1585,"iah":796,"iak":522," l ":89,"iam":821,"ial":13523,"ian":10128," ji":417,"iap":745,"ias":2287,"iar":1823," je":1297,"iau":2147,"iat":661,"iaw":324," im":275," in":10019," ik":452," il":277,"ic ":717,"iaa":121,"iac":153,"iad":926," is":1283," it":1237,"iag":345,"ibl":93,"ibi":920," ka":14437,"ibo":150," m ":97,"ibn":261," kh":806,"ibr":155," ki":2905," ke":34954,"ibu":1830,"id ":2411,"iba":1857,"ibb":83,"ibe":991," ju":4534," ha":7358," he":560," gi":259," gl":145," gr":258," go":642,"ia ":33962," gu":802," ib":1091," ia":15199," id":186," hi":2680,"ib ":619," ho":310," hu":1664,"iet":538,"ieu":98," ni":525,"iel":330," ne":5519,"ien":485," na":3187,"ier":702,"ies":1302,"ied":208,"iej":108," mu":4333,"ig ":949," mo":1128," ok":83," ol":9058," on":137," oc":95,"ifo":215," of":1351,"ifs":152," ob":344,"iff":148,"ife":181,"ifk":145," ny":301,"ifi":877," nu":298,"ih ":4063," no":914,"ifa":630," le":4083,"icr":152,"ics":105,"ict":304,"icu":180," li":1439,"ico":699,"ick":394," la":15380," ku":3370,"ici":517,"ich":943,"ice":662," km":1352,"ie ":1007,"ica":1361," kl":234," kr":296," ko":7504," me":61077,"idz":82,"idu":1218," mi":1388,"я ":83,"idr":191,"ido":222,"idm":710," ma":9348," lu":1062,"idi":1336,"idg":114,"ide":1853,"idd":108,"ida":4190," lo":1081,"if ":1278," ag":1211," ah":1386," ab":756," ac":279," ad":16501," am":705,"iik":146," an":6006," ap":1132," ai":662," aj":154," ak":2442," al":2643," au":286," aw":789," ar":1148," at":10219," as":1761," d ":615," ba":24359," 가가":103," ay":339,"il ":5677,"ija":979," bi":4229," be":24484,"iji":355," bo":2914," bl":104,"ijr":146," bu":4711,"iju":223," br":115," ca":1681," e ":109,"im ":3199,"ika":13080,"ige":492,"iga":1804,"ii ":168,"igh":815,"igi":732,"igu":2207,"igs":159,"igr":98,"igo":173,"ign":291,"iha":2790,"ihi":368,"ihu":86,"ik ":10067,"imo":347," er":382,"imn":174," et":757," en":1147," em":632," ep":143,"imp":1571," el":748,"ime":2177," ek":760,"imi":887," fe":189,"ip ":575,"inc":592,"ind":2963,"ina":9402," fa":1094,"inb":149,"imu":5422,"inn":244," fu":221,"inm":85," fr":306,"ino":771," fo":731,"int":4295,"ins":5288,"inf":140,"ine":3802," fl":155,"inh":91,"ing":16551," fi":3762,"inj":440,"ini":10223,"inl":149,"ink":392," ge":3276,"iq ":87," ga":1706,"inu":307," i ":105,"iny":1962,"iko":560,"ikn":492," co":867,"ikl":232,"ikk":91,"iki":2641,"ikh":399," ce":822," ch":277,"ike":4994," ci":740,"ila":13068," da":59600,"in ":11178," cu":456,"ikt":166,"iku":1306,"ikr":147,"iks":241," do":472,"ilo":508,"ill":1539,"ilk":662," dr":364,"ilm":266,"ilh":97," de":14986,"ili":5625,"ild":212," di":83692,"ile":5293,"ima":6615,"imb":1113," ed":217,"io ":1306,"ily":94," du":2752,"ilu":254," zo":121,"hol":201,"hom":180,"hon":510," za":721,"hos":188,"hot":110,"hou":148,"hoo":173,"hor":1332,"hny":592,"ка":100," ya":44378,"hmu":111,"hme":81,"hmi":93,"huk":240,"hul":645,"hui":194,"huj":251,"hud":233,"hua":247,"hub":485,"htt":96,"hte":100,"hta":151,"hsi":131,"hu ":365,"hru":143,"hro":130,"hre":90,"hri":265,"ht ":578,"hra":480," ru":967," sa":14469," se":65781," si":4360,"hya":165," sh":165," sk":326," sp":1202," so":731," ra":4498," re":1765," ri":882," ro":601," pu":2878," pr":3226," s ":516," px":487,"hwa":232,"hwe":84,"hwi":126," ot":94,"hup":170,"hum":860,"hun":7536,"hus":555,"hut":907," op":389,"hur":599," or":3496," pe":36568," pa":17415," pl":444," po":3206," pi":1442," wa":2574," we":290," wu":266," wi":5410," va":145," ve":377," vo":304," vi":594," ud":384," ub":143," tu":2924," us":324," ut":1972," ur":128," up":118," um":752," un":7312," uk":109," ul":360," ta":11300," sw":110," sy":970," st":1611," su":5235," tr":1008," to":641," th":1478," ti":5407," te":46306,"fi ":402,"ffa":97,"ffi":302,"fes":299,"fer":388,"fen":164,"fel":156,"fia":152,"fgh":100,"fas":143,"fat":355,"far":311,"fam":394,"fan":307,"fak":149,"fal":282,"fai":192,"fah":482,"ff ":196,"fe ":96,"fa ":136,"exa":138,"ez ":176,"ews":100,"ewu":103,"exi":402,"ezu":133,"eza":588,"eta":22591,"ete":2221,"eti":3058,"eth":196,"etn":374,"esp":92,"eso":358,"est":2090,"esu":1224,"ess":1053,"esy":124,"esw":856,"eum":158,"eun":154,"eto":293,"etr":602,"ets":101,"ett":299,"etu":1742,"etw":105,"ew ":783,"eve":355,"eva":165,"evo":467,"evi":1067,"eut":155,"eur":163,"eus":217,"eup":121,"ex ":120,"ewi":120,"ey ":956,"ewa":2199,"epe":2484,"epi":447,"eph":179,"er ":9482,"epa":7540,"eor":3629,"eol":252,"eop":100,"eon":238,"es ":6607,"ept":643,"epu":3262,"epp":83,"epo":201,"erk":5331,"erl":22148,"eri":15969,"erj":2346,"erg":1984,"erh":2054,"ere":4833,"erf":289,"erc":938,"erd":5231,"era":31612,"erb":21481,"et ":3047,"esk":117,"esl":106,"esh":228,"esi":13843,"esc":110,"ese":3188,"eu ":111,"esa":9499,"erz":234,"ery":101,"erv":184,"eru":33986,"erw":586,"err":623,"ert":10703,"ers":7669,"ern":3919,"erm":8575,"erp":1899,"ero":1667,"eki":1159,"ekn":672,"eko":2990,"eks":1079,"ekt":1163,"eku":1531,"en ":13321,"elb":797,"ela":24547,"eld":531,"elf":99,"ele":4153,"eli":4806,"elg":122,"elm":87,"ell":1081,"elo":901,"elu":8313,"els":167,"elt":96,"eo ":687,"emb":9560,"ema":5467,"eme":4035,"emo":659,"emi":3299,"emu":2900,"emp":6552,"ep ":352,"ene":3392,"enh":146,"eng":28628,"enb":1340,"ena":10322,"end":7875,"enc":2582,"eno":673,"enp":206,"enm":121,"enn":571,"enk":219,"eni":4899,"enj":3357,"enu":2960,"ens":1346,"ent":11262,"enr":1199,"enz":142,"eny":4231,"eog":153,"egl":90,"ego":878,"ege":3600,"egi":1125,"ej ":300,"eha":360,"egr":182,"egu":379,"ehk":131,"ehe":137,"ehi":1102,"ek ":1904,"eic":82,"eis":698,"eir":227,"eim":272,"eil":160,"ein":1673,"eik":182,"eij":135,"eid":126,"eif":187,"eja":2806,"el ":2913,"ejo":362,"eje":2245,"eke":565,"eka":5600,"em ":6452,"eju":647,"gka":6162,"gja":188,"giu":105,"git":589,"gis":912,"gir":419,"gil":557,"gim":123,"gik":836,"gin":1308,"gio":226,"gic":109,"gig":81,"gia":2714,"ghu":271,"ght":623,"ghi":183,"örd":174,"gha":1099,"ggu":2160,"ggr":373,"ggo":583,"ggi":2148,"gge":1788,"gga":7632,"gi ":7590,"gen":3765,"geo":235,"get":640,"ger":6793,"ges":802,"gh ":228,"gea":90,"ged":153,"geb":235,"gem":770,"gel":1459,"gek":139,"gdo":544,"ge ":840,"gab":488,"gac":87,"gad":225,"gag":159,"gah":4102,"gai":9563,"gaa":525,"gas":1063,"gar":7736,"gau":156,"gat":1264,"gaw":574,"gay":288,"gak":392,"gaj":331,"gam":2696,"gal":2392,"gan":31725,"gap":1128,"ga ":11059,"fte":85,"fta":174,"fsw":151,"fun":317,"ft ":199,"fra":229,"fre":154,"fri":738,"for":1053,"fot":84,"fon":311,"foo":101,"fol":96,"fli":95,"flo":92,"fic":314,"fie":153,"fil":3257,"fik":1106,"fin":217,"fir":98,"fis":84,"fiz":345,"fka":157,"da ":27370,"dbi":867,"de ":2083,"dad":217,"daa":403,"dak":3335,"dal":29698,"dai":362,"dag":474,"dah":2813,"dae":5534,"daf":161,"dat":1149,"das":913,"dar":25178,"dap":4274,"dan":33544,"dam":453,"day":1071,"daw":132,"dau":298,"ddh":97,"ddi":484,"cup":101,"cun":117,"cul":407,"cum":151,"cuk":241,"cub":107,"cua":246,"ctu":131,"cto":255,"cti":335,"cy ":279,"cus":161,"cur":169,"cut":119,"cks":91,"ckl":1007,"cle":147,"co ":813,"con":891,"col":233,"com":238,"cor":311,"cos":92,"cop":111,"cot":159,"cou":214,"cs ":109,"ct ":200,"cre":115,"cra":101,"öe ":241,"cro":203,"cu ":125,"ccu":84,"cci":124,"cce":280,"cea":276,"ch ":1178,"cer":686,"ces":635,"cet":263,"cen":347,"cep":121,"cem":134,"cel":216,"ceh":1264,"cec":129,"ced":146,"ci ":407,"cha":755,"chw":84,"chu":185,"chy":82,"cia":390,"ck ":760,"cie":186,"che":1276,"chl":916,"chi":1006,"cho":416,"cht":142,"civ":87,"cil":1184,"cir":431,"cis":3697,"cit":186,"cin":441,"cio":99,"cip":618,"cke":401,"ed ":1818,"eba":11892,"ebe":4137,"ebi":2395,"ebo":229,"ebr":729,"ebs":109,"ebu":29935,"eac":114,"eag":133,"eae":222,"ead":560,"eak":120,"ean":542,"eal":407,"eam":160,"ear":468,"eas":200,"eat":650,"eau":189,"eb ":216,"ea ":1116,"efi":144,"efo":203,"efe":130,"ei ":1170,"ega":5716,"eft":107,"eek":338,"een":939,"eel":102,"eem":212,"eec":90,"eed":132,"eh ":12714,"eer":155,"eet":128,"edi":2073,"ede":990,"ône":474,"eda":2175,"eg ":84,"edu":1494,"edo":329,"edr":119,"eck":1080,"ech":443,"eci":1191,"aße":85,"ece":504,"eca":9518,"ee ":526,"ef ":96,"ecu":162,"ect":304,"eco":162,"dwi":284,"dwe":130,"dwa":87,"dy ":371,"dur":258,"dut":214,"dus":523,"dor":515,"don":9812,"dom":753,"dol":193,"dok":339,"dow":266,"dos":301,"ds ":392,"dmi":140,"dun":2327,"dup":672,"dul":1503,"duk":2609,"dua":2723,"dud":2109,"duc":125,"dri":381,"dra":600,"dt ":116,"dre":255,"du ":1066,"dro":350,"dsb":197,"dha":216,"dge":491,"dic":635,"did":1487,"dia":6402,"dib":2542,"dhi":87,"ôte":452,"der":2200,"des":4618,"dew":286,"deb":180,"dea":352,"ded":138,"dec":284,"def":91,"deg":127,"del":494,"dek":1033,"den":10314,"dem":474,"dep":132,"deo":515,"di ":54108,"dle":103,"dn ":127,"dkr":218,"dma":725,"do ":591,"dli":128,"div":279,"diu":557,"diw":190,"dim":977,"din":2811,"dio":830,"dip":2297,"dir":2791,"dis":3174,"dit":5839,"die":663,"dif":84,"dig":2704,"dih":643,"dii":332,"dij":600,"dik":7651,"dil":2398,"dka":218,"rgu":308,"rhe":129,"rha":1676,"rhi":196,"rhu":353,"rho":81,"rfu":133,"rga":3299,"ri ":22851,"rgk":87,"rgi":456,"rge":1062,"rgo":543,"ret":1067,"res":2048,"rev":326,"reu":161,"rew":103,"rez":84,"rey":120,"rfa":107,"rfi":182,"rdu":156,"rds":117,"rdy":187,"rdw":115,"rg ":2166,"reb":133,"rea":1054,"ree":677,"raß":86,"rec":165,"red":517,"rei":890,"rej":223,"reg":650,"reh":132,"rem":580,"ren":2521,"rek":2078,"rel":624,"rer":249,"rep":182,"rf ":161,"rda":3685,"rcu":188,"rdo":180,"rdi":1361,"rde":1385,"re ":3550,"rbu":880,"rco":88,"rci":124,"rch":578,"rce":252,"rca":846,"raw":990,"ray":1762,"raz":756,"rd ":938,"rap":1559,"raq":133,"rar":213,"ras":4078,"rat":7211,"rau":320,"rav":203,"rbi":3232,"rbo":304,"rba":16478,"rbe":1672,"raj":2760,"rai":1892,"rah":13007,"rag":489,"ran":31080,"ram":3117,"ral":1707,"rak":4425,"rab":2634,"raa":1169,"raf":913,"rae":299,"rad":1746,"rac":391,"rpu":377,"rpo":1155,"rs ":614,"rpe":834,"rpa":577,"rpi":203,"ror":104,"ros":1224,"rot":661,"rom":519,"ron":1956,"roo":126,"rop":1379,"rou":319,"rov":4057,"row":385,"rob":171,"roa":181,"rod":471,"roc":332,"roj":312,"roi":110,"rol":484,"rok":385,"rof":362,"roe":206,"roh":152,"rog":606,"rno":175,"rnm":315,"rny":606,"rna":2876,"rnf":173,"rne":905,"rnk":114,"rni":698,"rmo":257,"rmu":1254,"ro ":1007,"rma":7923,"rme":289,"née":98,"rmi":310,"rlu":660,"rlo":176,"rli":1376,"rld":143,"rle":19283,"rla":1739,"rn ":1339,"rku":436,"rko":237,"rki":661,"rke":2263,"rkh":746,"rka":6421,"rm ":579,"rju":506,"rja":1818,"rje":230,"riz":119,"rix":133,"rl ":182,"rip":4424,"rio":461,"rit":2934,"ris":4313,"riv":103,"riu":208,"rih":81,"rig":839,"ril":887,"rik":7517,"rin":4389,"rim":859,"ria":3082,"rib":675,"ric":726,"rid":481,"rie":868,"rif":442,"rk ":625,"rwo":83,"rya":582,"ruj":1475,"rui":92,"ruh":1229,"rug":133,"ruf":342,"rud":184,"ruc":88,"rup":29747,"run":3036,"rum":1098,"rul":804,"ruk":474,"ruy":112,"rus":2650,"rut":1552,"rva":107,"rvi":143,"rve":148,"rwa":662,"ry ":1503,"rsk":111,"rsi":2063,"rso":232,"rsa":1745,"rsb":121,"rsh":121,"rse":3167,"rta":6444,"rst":183,"rsu":233,"rtn":93,"rto":428,"rte":1378,"rth":768,"rti":3790,"rub":489,"rua":1248,"rts":116,"rtu":2053,"rty":170,"rt ":1334,"rro":201,"rri":312,"rre":320,"rra":913,"ru ":2865,"rry":211,"saa":2288,"sab":137,"sac":82,"sad":132,"saf":204,"sah":1841,"sai":2187,"sak":765,"sal":6322,"sam":2727,"sba":93,"sbe":123,"sap":108,"san":12183,"sau":293,"sat":8021,"sas":1598,"sar":5174,"say":288,"saw":965,"sa ":16822,"rzo":153,"sha":523,"sho":148,"she":167,"shi":687,"si ":12617,"sge":511," 가":110,"sji":1115,"sie":915,"sid":1093,"sic":184,"sib":161,"sia":23942,"sk ":160,"sit":1085,"siu":88,"sir":1920,"sis":2215,"sip":1088,"sin":2640,"sio":1252,"sil":1595,"sim":1250,"sik":1183,"sih":825,"sif":646,"sig":163,"sbu":405,"se ":1244,"sca":245,"sch":553,"sco":283,"sew":140,"ser":3709,"ses":2265,"set":1888,"seu":109,"sh ":916,"sfe":203,"sea":187,"sei":253,"seh":881,"seg":534,"see":90,"sed":838,"sec":1466,"seb":39199,"sep":2892,"seo":3063,"sen":3081,"sem":4327,"sel":4711,"sek":3390,"sej":3739,"spo":247,"spe":1275,"spi":326,"spa":153,"sot":113,"sou":89,"sol":388,"som":96,"son":1077,"sop":111,"sor":696,"sos":286,"sod":99,"sof":189,"sok":132,"soa":82,"su ":534,"sra":377,"st ":1090,"ss ":420,"sli":668,"slo":101,"slu":163,"sky":86,"sla":2463,"sle":93,"ski":268,"sko":166,"skr":273,"sku":127,"ska":1853,"sny":1578,"sna":139,"sni":143,"sne":275,"smo":84,"so ":382,"sma":631,"smi":706,"sme":415,"swi":860,"sye":1001,"sya":1661,"syu":126,"syt":127,"syi":154,"syh":85,"sse":728,"ssa":1040,"sso":702,"ssi":559,"ste":4462,"sta":2889,"stm":167,"sto":598,"stp":330,"sti":3901,"stu":237,"str":2762,"sua":2264,"sud":961,"sub":558,"suc":350,"suh":294,"sul":629,"sum":580,"suk":2918,"sup":206,"sun":1377,"sut":148,"sus":1037,"sur":1688,"sy ":170,"swa":407,"tai":2173,"taj":254,"tak":21498,"tal":2397,"taf":141,"tag":153,"tah":9752,"taa":959,"tab":947,"tac":174,"tad":1206,"tay":304,"taw":490,"tau":9325,"tat":797,"tas":2227,"tar":10465,"tap":1660,"tan":37185,"tam":5004,"tch":182,"te ":3845,"tda":113,"가":342,"ta ":10995,"pa ":2327,"pe ":281,"par":2140,"pat":12545,"pas":2802,"pau":127,"pay":722,"pac":225,"pad":20739,"pab":507,"pag":366,"pah":868,"pak":30609,"pal":3287,"pai":927,"pap":231,"pam":148,"pan":16670,"phe":134,"pha":505,"pho":147,"phi":165,"pi ":1888,"pea":108,"pec":204,"ped":430,"pen":13849,"pem":3764,"pep":189,"per":20160,"pet":458,"pes":2868,"peg":272,"pej":473,"pel":2806,"pek":821,"pla":686,"pli":230,"ple":359,"plo":124,"pka":268,"phy":98,"pia":417,"pid":110,"pic":128,"pih":341,"pik":502,"pil":749,"pim":153,"pin":1830,"pio":81,"pir":975,"pis":530,"pit":518,"por":689,"pop":931,"pou":100,"pot":165,"pos":307,"poh":486,"pom":970,"pon":661,"pok":948,"pol":1646,"ps ":173,"ppi":85,"ppe":181,"po ":181,"pny":87,"pta":664,"psi":179,"pua":664,"pub":846,"pte":935,"pti":180,"pto":107,"pra":188,"pu ":412,"pri":989,"pre":663,"pro":2699,"pur":2588,"pus":1163,"put":1968,"pun":6347,"puk":143,"pul":4553,"px ":483,"رة ":90,"qua":138,"que":442,"qui":261,"ra ":25300,"ngn":355,"ngo":1514,"ngj":195,"ngi":2924,"ngl":959,"ngk":8614,"ngu":2703,"ngr":216,"ngt":156,"ngs":4906,"ni ":12040,"nge":2966,"ngg":14541,"ngh":1384,"nga":40112,"ngd":572,"nha":170,"neg":5789,"nei":292,"nel":1414,"nek":199,"nen":1045,"nem":406,"neo":158,"ner":1969,"net":1133,"nes":9659,"neu":163,"ng ":99565,"nea":314,"ned":91,"nfo":133,"nfl":104,"ney":393,"nez":159,"nfe":134,"nco":417,"nci":4468,"nce":1649,"nch":449,"nca":2247,"ne ":4670,"nbu":1352,"ndu":4122,"ndr":524,"nds":418,"ndo":10238,"ndi":3985,"nde":2188,"nda":19923,"ncy":259,"ncu":421,"nak":4741,"nal":5652,"nam":4608,"nan":8824,"nap":273,"nar":1813,"nac":196,"nad":1121,"naf":114,"nag":573,"nah":2523,"nai":809,"naj":140,"nab":295,"naa":932,"nbe":130,"nd ":4902,"nba":120,"nau":433,"nat":2102,"nas":1399,"nay":387,"naw":275,"na ":10713,"가 ":113,"nya":20158,"nye":1690,"nyi":1836,"ny ":576,"nwa":109,"nvi":88,"nve":165,"nuk":445,"nul":732,"num":373,"nun":1462,"nug":514,"nuh":642,"nus":1343,"nut":397,"nur":346,"nua":1039,"nub":164,"nty":389,"nto":1247,"ntu":9594,"nts":152,"ntr":493,"nti":4527,"nth":267,"nta":14735,"nte":5062,"nsu":721,"nsy":119,"nsn":707,"nso":443,"nst":1059,"nse":539,"nsh":129,"nsi":4092,"nsk":184,"nsa":240,"nsb":162,"nu ":1047,"nre":1118,"nt ":2047,"npl":189,"ns ":1395,"noh":123,"nol":621,"noi":90,"nop":117,"nom":1343,"non":340,"not":257,"nos":240,"nor":470,"nov":416,"npa":520,"nne":1190,"nna":293,"nno":98,"nni":227,"nny":2270,"nme":425,"nma":233,"nn ":167,"nla":231,"no ":1002,"nke":98,"nki":218,"nka":1483,"nku":327,"nji":842,"nje":264,"nja":4869,"nfö":173,"nju":1774,"njo":126,"nig":119,"nie":150,"nid":172,"nic":456,"nia":3904,"nk ":304,"niy":199,"niu":157,"niv":669,"nis":5362,"nit":1992,"nio":219,"nip":112,"nim":789,"nin":1570,"nik":1402,"nil":494,"ogr":689,"ogu":135,"ogt":141,"ogi":1196,"ogo":280,"ogn":241,"oga":366,"oge":265,"oi ":269,"oho":915,"ohn":385,"ohd":380,"oha":945,"ogy":484,"ois":234,"oir":1071,"oin":141,"oid":154,"ok ":3077,"ojo":198,"oje":321,"oja":137,"ol ":1539,"oce":85,"och":315,"oci":139,"ock":424,"oco":179,"oe ":208,"oca":188,"occ":131,"ode":810,"odi":278,"odo":173,"of ":1293,"oda":299,"oei":98,"oen":92,"odu":321,"og ":188,"ofi":103,"oft":130,"oh ":1421,"off":261,"ofe":269,"oa ":251,"ob ":122,"oan":123,"oal":112,"د ":141,"oad":171,"oba":366,"od ":778,"obo":201,"obl":99,"obj":270,"obi":241,"obe":749,"nyo":518,"nyu":376,"ة ":254,"nza":129,"oya":277,"ア ":125,"ows":189,"own":225,"owi":119,"oyo":90,"ow ":425,"otl":114,"oti":361,"oth":207,"ote":516,"ott":294,"oto":989,"otn":85,"ost":425,"ota":4428,"ov ":225,"osi":886,"osh":144,"osk":108,"ose":885,"osg":503,"osf":111,"osp":283,"oss":236,"osl":105,"oso":466,"oy ":167,"owa":312,"owe":160,"ovi":3651,"ovo":102,"ova":317,"ove":2144,"oug":180,"oui":99,"oul":186,"oun":730,"oup":135,"ous":420,"our":774,"out":586,"opo":452,"opi":468,"opl":109,"ope":851,"oph":272,"opa":651,"os ":2032,"opu":974,"opt":304,"ops":100,"oon":251,"ool":149,"oké":372,"ook":216,"ood":234,"or ":5219,"oot":168,"oor":392,"ork":389,"orl":178,"orm":1207,"orn":616,"oro":810,"orp":1180,"orr":678,"orc":95,"ord":1125,"ore":1134,"orf":206,"org":773,"ori":1675,"ou ":243,"osa":542,"osc":127,"ort":1677,"ors":277,"oru":170,"orw":94,"ory":594,"m² ":559,"ot ":726,"orb":185,"ora":7028,"ola":3886,"old":231,"on ":7262,"oli":2647,"oll":548,"olk":89,"olf":106,"ole":11812,"ols":765,"olt":108,"olo":2453,"olu":604,"oka":760,"om ":1649,"oki":234,"oke":184,"okr":214,"oks":233,"oko":2518,"oky":119,"okt":111,"oku":358,"ona":2279,"ond":1093,"onc":169,"onf":186,"one":9980,"ong":5686,"onj":101,"oni":1220,"onk":139,"onn":659,"ono":1193,"ons":1416,"ont":1746,"onu":81,"onv":152,"ony":334,"oma":1160,"oo ":109,"ome":1328,"omb":1378,"omi":1343,"omm":1191,"kém":368,"omp":2047,"omo":486,"omu":3500,"op ":434,"la ":8163,"le ":2220,"lf ":112,"lde":213,"lda":235,"ldk":204,"ldi":123,"ldr":97,"laa":247,"lab":751,"lac":351,"lad":420,"laf":100,"lah":41764,"lag":1649,"laj":805,"lai":4507,"lal":1588,"lak":3883,"lan":17980,"lam":20229,"lap":6088,"lar":2085,"lat":9127,"las":2647,"law":2684,"lau":5752,"lav":163,"lay":19664,"laz":215,"lba":1538,"ld ":768,"lbe":124,"lbu":792,"kut":2151,"kus":252,"kur":1183,"kup":264,"kun":678,"kum":2053,"kul":690,"kuk":537,"kwa":145,"ky ":133,"kta":496,"ksy":183,"ksp":152,"ksu":782,"ksi":749,"kso":141,"kub":97,"kud":134,"kue":85,"kug":115,"kui":193,"kua":1683,"ktr":757,"ktu":771,"kti":1117,"kto":892,"kyo":126,"kya":786,"lpe":958,"ls ":210,"lok":357,"lon":1793,"lom":1385,"lop":336,"lor":590,"loc":124,"loh":134,"log":1685,"loi":125,"lpa":103,"los":237,"lot":172,"lou":94,"lov":170,"low":135,"lob":141,"lny":581,"lma":261,"lmu":271,"lti":290,"lto":86,"lud":118,"luc":152,"lub":115,"lua":5030,"luh":485,"lue":130,"lst":756,"lta":1323,"lte":276,"lu ":2269,"lsa":202,"lt ":289,"lha":114,"lge":136,"lgi":113,"li ":9076,"lga":143,"lfo":84,"ley":274,"lew":115,"lex":130,"leu":205,"lev":943,"les":2125,"let":19322,"ler":519,"leo":109,"lep":1429,"lem":5627,"len":2626,"lek":1317,"lel":468,"lei":110,"lej":185,"leh":11245,"leg":545,"lef":234,"lee":98,"led":158,"lec":148,"leb":2160,"lea":321,"lls":96,"llu":129,"lly":170,"lo ":657,"lla":1553,"lle":1246,"lli":666,"llo":405,"lka":1572,"lki":104,"lm ":121,"ll ":825,"lit":3181,"lis":3844,"lir":917,"lip":802,"lio":243,"lin":3503,"lim":5664,"liz":154,"liv":116,"liu":162,"lic":650,"lid":486,"lia":4614,"lib":614,"lik":3976,"lil":385,"lig":376,"lih":1595,"lie":274,"lif":337,"ma ":13530,"mb ":646,"maa":523,"mac":144,"mah":1764,"mai":2470,"maj":1120,"mak":2827,"mad":1643,"mag":284,"map":110,"mar":1635,"mas":5847,"mal":1537,"mam":694,"man":17867,"maz":97,"may":121,"mau":220,"mat":13126,"mba":6501,"mbi":2016,"mbe":4139,"mbr":177,"mbo":1368,"me ":1188,"mbu":3564,"med":795,"mee":451,"meg":240,"mec":162,"met":1241,"mew":781,"mes":766,"mer":37899,"mem":9158,"mel":2698,"men":19175,"mek":226,"mej":156,"lva":321,"lve":96,"lul":249,"luk":1418,"lui":805,"lun":825,"lum":2206,"lut":262,"lus":1161,"lur":2937,"ly ":453,"lya":87,"hôn":460,"lym":81,"mpi":2087,"mpe":1217,"mpr":185,"mpo":1192,"mpl":253,"mpu":8478,"ms ":187,"mod":652,"mon":1281,"mok":188,"mol":218,"mor":397,"mos":294,"mot":314,"mou":108,"mpa":5954,"mu ":410,"mud":1128,"mua":910,"my ":225,"mur":5500,"mus":917,"mut":295,"mui":219,"muk":990,"mul":2530,"mum":892,"mun":5382,"muz":686,"ër ":162,"mi ":2729,"min":2074,"mil":3082,"mim":282,"mir":325,"mis":572,"mit":382,"mic":168,"mia":690,"mig":85,"mid":185,"mik":768,"mo ":213,"ièr":83,"mla":599,"mka":269,"mny":568,"mmu":120,"mmi":85,"mma":792,"mme":1112,"ور":123,"zue":132,"zur":484,"zza":91,"アア ":119,"zi ":161,"zha":131,"zec":167,"zen":271,"zer":331,"ze ":110,"بن":82,"zaa":88,"zah":203,"zam":678,"zan":280,"zak":354,"zal":148,"zar":263,"zon":176,"zog":154,"ان":141,"ال":473,"zia":91,"zin":101,"zim":257,"zil":726,"zik":1108,"ziu":148,"ziz":173,"yré":84,"yum":99,"yun":210,"yur":228,"yus":82,"yua":148,"yti":119,"yst":183,"ysi":10703,"yu ":2215,"yri":256,"ys ":580,"yok":119,"yol":401,"yon":283,"za ":754,"سو":84,"yya":320,"را":86,"رة":90,"ye ":102,"yeb":310,"yed":369,"yer":661,"yen":1160,"yem":89,"yel":428,"ya ":19984,"yaa":759,"yaw":106,"yat":1718,"yar":4015,"yas":133,"yap":186,"yan":47012,"yal":266,"yam":513,"yak":3202,"yah":7404,"yai":2092,"yn ":94,"yle":84,"yll":91,"ylo":98,"yo ":269,"ymp":86,"ر ":108,"yi ":1148,"yhu":83,"yin":165,"yim":134,"yik":137,"yid":111,"yia":421,"ي ":156,"xon":87,"ه ":83,"ن ":270,"م ":108,"xic":389,"xan":91,"wuj":426,"wn ":203,"ws ":263,"wor":251,"woo":89,"we ":145,"wes":1940,"wer":159,"wel":124,"wei":92,"wed":219,"web":258,"wi ":588,"wir":85,"wit":297,"wig":1003,"win":269,"wil":5229,"wia":82,"röe":241,"wa ":2659,"wan":4391,"wal":2126,"wam":340,"waj":117,"wak":1437,"way":450,"wat":2105,"wau":86,"war":2035,"was":5549,"wai":624,"wah":1233,"wab":269,"via":591,"vir":115,"vil":492,"vin":3415,"vic":160,"vid":769,"vie":296,"vit":347,"vis":1097,"vo ":146,"rén":87,"vol":481,"vok":203,"vi ":88,"ver":1914,"ves":180,"vet":142,"ven":1162,"vem":496,"vel":571,"ve ":410,"val":198,"van":369,"var":225,"vat":140,"vas":121,"vad":278,"va ":432,"uzi":987,"uya":163,"ux ":226,"uve":119,"uwa":153,"usl":307,"usk":402,"ush":174,"usi":3335,"use":449,"usa":3092,"usu":1400,"ust":2174,"uss":842,"uso":135,"usn":368,"utn":101,"uth":367,"uti":1850,"ute":2173,"uta":5785,"utt":161,"uts":114,"utu":1541,"uto":578,"utr":336,"us ":4638,"ut ":7583,"urb":296,"ura":6397,"urd":90,"urc":329,"ure":519,"urh":114,"urg":2337,"uri":1172,"urk":876,"urn":367,"uro":302,"urr":119,"urs":201,"urt":279,"uru":5838,"ury":103,"uny":2843,"upa":36302,"ur ":9875,"upi":131,"upe":318,"upu":963,"ump":3501,"umu":874,"umi":1006,"umk":95,"uml":587,"umo":90,"umn":298,"uma":2885,"umb":3220,"ume":429,"uly":87,"unt":6686,"uns":519,"unu":1520,"unk":553,"unj":444,"uni":4206,"uno":217,"unn":144,"unc":664,"und":1844,"una":5768,"ung":15003,"une":463,"up ":964,"uku":3080,"ukt":432,"uko":385,"ukn":323,"ukk":353,"ukl":355,"uki":2400,"ukh":102,"uke":89,"um ":5354,"uka":5655,"uju":2959,"ulu":2906,"ult":1554,"ulo":336,"ull":522,"ulk":198,"uli":2651,"ulg":85,"ule":183,"ula":12934,"un ":15688,"uid":135,"uih":107,"uil":237,"uin":352,"uis":395,"uhu":184,"uk ":13558,"uji":209,"uit":287,"ul ":3175,"uja":272,"ugh":152,"ugi":310,"uge":603,"ugo":103,"ui ":1412,"uga":3887,"uhi":252,"uhr":221,"uhn":192,"uhk":782,"ugu":316,"uha":3353,"uct":124,"ucu":163,"uf ":428,"uda":2440,"udd":449,"ude":159,"udg":308,"udi":1380,"udk":154,"ubo":151,"ubu":2193,"uca":268,"ue ":523,"ucc":281,"uci":281,"uch":381,"uck":113,"uer":235,"ues":185,"uh ":2913,"ufa":96,"udu":2586,"udo":102,"udw":165,"uen":605,"uel":282,"ub ":287,"ua ":4778,"uay":140,"uat":3203,"uas":2848,"uar":5742,"uam":156,"ual":2564,"uan":6824,"ubi":233,"ubj":123,"ubl":997,"ube":118,"uba":1808,"ud ":1277,"uak":209,"uah":28489,"uai":435,"uad":168,"uac":88,"tze":150,"ty ":1209,"tur":2470,"tus":1028,"tut":762,"tuj":696,"tul":1448,"tuk":8612,"tun":1428,"tum":1714,"tup":264,"tub":1391,"tua":2911,"tud":425,"tue":190,"tuh":346,"tug":473,"tz ":139,"two":138,"ts ":530,"tre":416,"tt ":153,"tra":2815,"tri":1776,"tru":429,"tro":1657,"tu ":13080,"tsu":260,"tta":304,"tte":701,"tti":139,"ttl":217,"tto":116,"ttp":95,"tme":175,"tma":120,"to ":1901,"tni":159,"tne":114,"tp ":97,"tna":265,"tny":365,"tno":94,"toc":160,"toi":105,"toh":336,"tob":555,"tov":117,"tos":314,"tot":99,"tow":94,"tom":534,"ton":1740,"tok":501,"tol":501,"tor":1363,"top":201,"tph":309,"til":1110,"tik":4794,"tif":1686,"tie":154,"tih":1384,"tig":1070,"tir":325,"tiq":113,"tit":1049,"tis":1131,"tin":5400,"tim":2572,"tip":149,"tio":1887,"thu":783,"tia":2252,"tib":138,"tic":397,"tid":1658,"tiw":247,"tiu":167,"tiv":497,"tka":3394,"tli":157,"tla":512,"tle":246,"tem":4640,"ten":10423,"teo":252,"tep":379,"tei":1032,"tek":974,"tel":4483,"teg":559,"teh":223,"tea":171,"teb":211,"tec":87,"ted":825,"th ":1378,"tet":1024,"tes":1074,"ter":49142,"ti ":8689,"tho":286,"thm":261,"thr":109,"the":1215,"thi":258,"tha":367," アア":100,"AS ":389,"BB ":113,"BC ":165,"AC ":83,"AB ":193,"가가":229,"AM ":100,"AL ":81,"AN ":625,"AP ":166,"三 ":110,"あああ":113},"n_words":[8074753,9381487,7891301],"name":"ms","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/ne.json b/contrib/languages-data/ne.json
new file mode 100644
index 0000000..9c1aba5
--- /dev/null
+++ b/contrib/languages-data/ne.json
@@ -0,0 +1 @@
+{"freq":{"ेश्":122,"ेष्":121,"दन ":955," कम":272," कर":171," कल":142," कि":313,"ेशक":101," का":2467," कृ":130," कु":353," के":400," को":746," क्":668,"ेवा":114,"ेवी":117," एक":2232," एम":84," एस":66," एव":82,"ेही":103," चौ":73," चु":87," ची":65," चि":272," चा":174," छन":436,"थी ":98," जर":83,"तको":395," छो":111," जस":287," १ ":94,"तका":126," छा":110," जन":834," ज्":137," २ ":106," जि":1031," जा":297," जु":184," जी":93," जे":102,"था ":777," छ।":1176," जो":201," ३ ":81," झा":69," गर":1726," खो":93," खे":113," गण":154," गत":96," खा":152," घर":283," गौ":72," गो":262," ग्":152,"ैति":183," गु":176," गा":503," । ":1729,"दछ ":91," चल":134,"णमा":89," चर":93," चन":91,"ोल्":69," अं":111,"ोलि":69,"ोला":99,"ोलन":95," अप":72," अन":738," अध":277," अथ":96," आक":83,"ेता":128," अल":77," अर":427," अम":148," अभ":65,"ोर्":161," अस":323," अव":627,"ेत्":416," अक":100," अग":100," अत":71," अञ":333,"ेपा":2443,"ेन्":518,"ोबा":73,"थल ":101,"ेना":77,"ंग ":129,"ेमा":93," एउ":305,"त् ":79,"थम ":65," इल":78," इन":145,"ेली":90,"ेला":163,"तो ":127," आर":92," आय":68," आन":91," आद":140," आध":94," आफ":173," उह":329,"ेर्":71," उप":369," उन":300," उद":118," उत":328,"ेरि":117," उच":89,"ेरै":136,"ंघ ":125," वा":585," वी":67," वि":1830," वु":93,"्तर":540," व्":234," शर":146,"ं ":520,"ैशा":79," वै":119," शब":115," शु":75,"्तो":97," सं":1338,"्त्":507," शि":206," शा":289,"्तै":88," शह":107,"्थल":136,"्ता":424," सक":211,"्ती":153," श्":338,"ताक":144,"्ति":514,"्तु":122,"्थ्":92,"तान":131,"्दछ":254,"्थि":236,"ताम":82,"्था":656,"तार":119,"ताल":259,"तिक":679,"्थी":106,"ताह":141,"्दो":111,"्द्":640," वट":348,"तिन":112,"तिम":220,"ँ ":497,"तिब":72,"्दा":387,"तिल":64," वर":390,"्दी":92,"्दि":109,"तिर":95,"्दू":82,"तीक":85,"तिह":119," वन":80,"्दै":77,"्दे":123,"्ट्":320," लग":144,"्टी":291,"्टा":74," लल":75,"्डल":215," लु":106," ला":569," लि":251," ले":262,"इ ":252," लो":97,"्डक":81,"्णक":65," या":166," यस":2003," यह":103," यि":70,"धन ":65," यु":243," यो":2991,"्तक":221," रा":1721," रह":505," रे":111," रू":232,"दै ":146," रु":253," रो":152,"्बन":84,"दी ":721,"्बि":65,"्मक":109,"्मन":119,"्यक":669,"्मा":313,"्मि":89,"्मी":132,"्यत":93,"्मे":80,"्यम":169,"्रक":2903,"्यव":169,"्यस":223,"्यह":70,"्रज":90,"्यु":227,"्या":1225," हो":3798,"्धा":79,"्धि":77,"्धु":64,"तया":66," हि":324," हा":314," हु":1822," हे":105,"्ने":717,"्नु":381,"तरी":81,"्ना":84,"तरा":100,"्नो":104," सम":2258," सभ":305," सब":222,"्पन":76," सन":367,"्पत":195," सर":306,"दा ":445," सद":429," सत":75,"तर्":239," स्":1058," हर":182,"्फब":94," सो":117,"्पा":1027," सा":1310," सि":468," सह":215," से":264,"्प्":73," सु":497," सू":97,"दि ":71," दल":430," दर":95," दु":249," दा":223,"ोक ":69," दि":387," दक":141," त्":468," ते":64," था":199," थि":1391," तर":320," तथ":624," ता":221,"्क्":76," ति":156,"ोग ":226,"तमा":143,"्को":75," ती":93,"्का":162,"्कृ":122,"्की":66,"ण्ड":575,"ैनि":66," ठु":72," ठू":98," डा":80,"ँको":90," टे":70,"ैभन":80,"तपु":159," ५ ":87," ठा":328,"तन्":137," ४ ":90,"७१ ":81," मो":133," मे":182,"्जा":70," मा":2199," मि":226," मी":71," मु":393,"्जी":68,"्छ।":1182," मह":623," भो":79," मन":295," भे":81," मध":395,"ैमा":131," भु":80," मण":175," भू":134," भि":128," भा":1679," मज":70," भर":94," ब्":200," भद":64," बे":149," भन":724," बै":74," बो":103,"्ञा":169," बा":593," बि":392," मं":65," बी":67," बु":143," भग":64," बह":81," बस":196,"्टर":85," फ्":93," भए":2250," बर":162,"दल ":217," बन":332," फा":77," बज":106,"तथा":612," बढ":88," प्":5303," पो":110," फल":67," पि":113," पा":1007," पु":600," पे":81," पृ":77," पू":312,"ौं ":109," पर":802," पश":191," पह":651," पछ":122," पन":637," पद":103,"णाल":87," पत":3652,"ौँ ":65," न्":72," नै":168," नि":657," ना":582," पं":74,"्चल":413," ने":2763," धे":138,"्छन":169," नय":92," नर":78,"्चि":231," नव":75,"्चा":256," धा":133,"्। ":184," नद":186," दै":64," दे":732," धन":78," द्":298," दो":108," धर":161," नग":197,"थ ":223,"दछ।":89,"द ":562,"ध ":304,"दछन":85,"न ":5506,"ड ":307," छ ":575,"ठ ":146,"थाल":88,"थिए":221,"थाप":321,"ण ":876,"थान":166,"थिय":1145,"त ":3231,"थित":211,"धी ":65,"ज ":416,"दक्":143,"ट ":1241,"धि ":82,"ं":3288,"ः":73,"ँ":2200,"आ":1341,"इ":1404,"अ":4126,"ऋ":113,"ई":1594,"उ":3478,"घ ":163,"ए":6874,"ओ":539,"ऐ":69,"ग":8111,"ख":2732,"क":41084,"औ":111,"छ":5495,"च":3790,"ङ":845,"घ":968,"ट":4637,"ञ":822,"झ":319,"ज":7684,"ठ":1551,"ङ ":292,"ड":2764,"ढ":364,"ण":2637,"त":20032,"थ":4532,"द":12121,"ध":3905,"न":29922,"प":22723,"फ":1251,"्ग ":136,"ब":5465,"भ":7912,"म":22298,"य":17139,"र":42105,"ल":18318,"व":12132,"ष":4104,"श":6654,"ह":13763,"स":20352,"ि":29268,"ा":66742,"े":14995,"थवा":92,"ू":2666,"ृ":815,"ी":11554,"च ":186,"ु":12537,"ौ":1228,"्":49626,"ो":28329,"ै":2713,"।":9235,"०":2064,"१":1785,"्क ":72,"छ ":1403,"६":623,"७":619,"८":678,"९":972,"२":2170,"३":565,"४":638,"५":791,"क ":5557,"ेल ":167,"ग ":763,"ख ":276,"ेर ":327,"त्स":67,"त्व":197,"त्प":68,"त्र":5347,"त्य":758,"त्त":449,"ए ":239,"तीय":69,"द् ":189,"ेस ":70,"उ ":287,"ई ":1278,"ेश ":318,"ै ":1366,"्न ":467,"नै ":335," ख":612," ग":3519," औ":90," क":6153," ओ":97," ऐ":65," ट":271," ज":3597," झ":164," च":1230," छ":2605," घ":502," इ":461," आ":1284," अ":3988," ए":2988," ऋ":77," उ":1784," ई":92,"दार":162,"नो ":211,"दान":85,"दीक":207,"दिर":84,"े ":3604,"दिन":243,"न् ":626,"दुई":79,"ौँम":309,"दुर":326,"ेकप":97,"्म ":374,"ू ":585,"ेका":344,"ेको":828,"्य ":1998,"ेखि":464,"्र ":942,"ेखा":65,"ि ":3525,"नी ":612,"ेटि":110,"ी ":7016,"नु ":137,"ोत्":73,"ु ":1173,"ा ":19102,"ँउ ":227," ८":76," ९":73," ६":100," ७":169," ४":169," ५":165," २":1308," ३":186," ०":277," १":1083," ।":2222,"्व ":234,"ेजी":71,"दस्":318," प":14289," फ":558," न":5247," म":5367," य":5733,"्ष ":268," ब":2996," भ":5836," ढ":70," ठ":526," ड":306,"ह ":271," द":2964," ध":656," त":2303," थ":1702," ह":6806," स":10170," ल":1764," र":4971," श":1531," व":3977,"ने ":1547,"्स ":116,"स ":1459,"ष ":373,"थ्य":109,"श ":446,"्छ ":694,"व ":550,"्च ":74,"दरम":107,"्ट ":232,"ल ":2508,"दलह":69,"नि ":671,"दर्":91,"दलक":97,"ना ":698,"ँग ":83,"र ":6666,"्ड ":120,"य ":3205,"्ठ ":89,"म ":1712,"्ण ":171,"ोखर":85," र ":1581,"ब ":130,"्थ ":94,"फ ":86,"्त ":382,"्ध ":250,"प ":228,"ो। ":2741,"्द ":129,"ृष्":136,"डल ":72,"डी ":201,"डा ":124,"ञ्ज":95,"ञ्च":499,"ृति":122,"ृत्":80,"ञान":153,"ेत ":110,"ठमा":517,"ट्र":339,"ट्ट":87,"ढी ":85,"ेक ":82,"अक्":91,"टिन":116,"अञ्":333,"टीक":104,"ुवा":235,"ूको":117,"ुला":154,"ुलो":69,"ुल्":89,"ुले":184,"ुरा":226,"ुरो":69,"ुर्":237,"ुरी":67,"ुरु":159,"ुरम":86,"ुम्":151,"ुरक":79,"ुमा":298,"णी ":110,"ँदै":80,"ुभय":124,"णा ":67,"डलक":101,"ुभए":420,"ुप्":75,"ुपा":81,"ुपम":101,"ुन्":1483,"ुनै":129,"ुने":151,"ुनु":515,"ुनि":176,"ुना":77,"ुद्":231,"्सा":87,"्से":64,"्ष्":65,"ँमा":361,"्स्":64,"ठाउ":291,"ठूल":104,"्ला":1132,"्लो":92,"्ले":68,"ंचा":97,"्रद":218,"्रथ":82,"्रत":719,"्रण":90,"्रप":72,"्रन":77,"्ये":172,"्रध":142,"्रम":518,"्यो":226,"ंग्":118,"्रय":179,"्रव":165,"्रश":109,"्रह":151,"्रस":389,"्रा":928,"ंगा":83,"्रि":4096,"्री":584,"ठुल":72,"्रै":117,"्रे":422,"्रो":266,"्षक":74,"ीहर":197,"्षर":74,"ंगठ":68,"्षे":409,"्षि":335,"्षा":151,"्षी":92,"्वक":82,"ुग्":67,"्वत":106,"्वर":184,"्वव":84,"्वम":76,"्वय":164,"्वी":175,"्वा":1398,"डौँ":365,"डौं":74,"ूला":68,"ूलो":82,"ति ":1256,"ता ":557,"णको":70,"तु ":113,"ूमि":65,"ूमा":80,"ती ":337,"ूर्":396,"ूपम":142,"तै ":115,"ते ":88,"ूद्":136,"ंमा":238,"ंस्":240,"तर ":294,"ुस्":101,"ुसा":127,"ुहु":425,"ंवि":168,"ाला":279,"ालि":277,"ाली":1573,"ाल्":175,"ाले":372,"छ। ":2152,"ावि":101,"ाशन":833,"िकृ":90,"िका":4480,"ाशक":884,"िको":207,"िक्":271,"ाषा":1077,"ासन":94,"ासद":191,"ाष्":296,"ाशि":795,"ासक":88,"ाही":67,"ाहि":278,"ाहा":83,"ुन ":257,"ासि":356,"ासी":103,"ाहर":380,"ास्":236,"जा ":203,"चलक":256,"िज्":95,"चर्":65,"िजय":135,"जी ":92,"ुर ":568,"ङ्ग":239,"ितप":65,"छन्":559,"िता":146,"िति":411,"ित्":493,"िद्":368,"िधि":130,"िधा":249,"िना":260,"िनि":160,"िनी":149,"िने":242,"िनु":78,"िन्":1022,"चीन":87,"िभि":96,"िभा":91,"ियन":73,"िम्":132,"ियम":114,"िमा":914,"ियो":1315,"िया":360,"िर्":332,"िरा":176,"िले":173,"िल्":1069,"िलो":573,"चित":254,"िला":204,"चाय":66,"चार":351,"चाल":100,"चिव":79,"चिन":100,"चिम":183,"िष्":174,"िश्":356,"िशे":71,"िवा":232,"ीका":189,"ीको":490,"िहा":169,"िस्":142,"ूल ":110,"ीति":294,"च्च":109,"जमा":145,"ीद्":108,"छोर":78,"जन्":230,"जनक":98,"जनत":72,"जधा":122,"जनव":107,"जनै":185,"जनी":264,"जना":168,"जदु":67,"ीमा":357,"ीया":65,"ुक्":205,"ुको":176,"ुका":171,"ुख्":130,"ृत ":152,"ीले":86,"ीला":83,"जस्":157,"जसल":64,"ित ":1545,"िण ":71,"जिल":993,"जिक":146,"जार":186,"जान":79,"जात":246,"जर्":77,"जयी":89,"ाँउ":242,"ाँच":67,"ाँक":137,"िन ":363,"जवा":81,"ाइन":178,"िल ":92,"जोड":89,"ाउँ":218,"ाउं":237,"ाउन":366,"ीक ":78,"ज्य":351,"ाँस":67,"ांग":75,"ज्ञ":223,"ाँल":66,"िम ":169,"िय ":243,"िर ":148,"जील":69,"जीव":121,"ाएक":100,"जुन":124,"टर ":108,"िव ":68,"िस ":86,"ाग्":194,"ागु":64,"ागि":189,"ीत ":128,"ागर":78,"ागम":84,"ाको":3606,"ाक्":157,"ाका":303,"ाओव":397,"ाडी":141,"िंह":72,"ाठम":478,"ाटन":82,"टी ":233,"टा ":741,"ाजा":131,"ाजि":146,"ाज्":244,"ाजन":456,"ाजध":126,"ाजव":83,"ीन ":153,"ाचन":103,"ाङ्":108,"ुई ":73,"ाने":97,"ाना":169,"ानि":283,"ानी":322,"ानु":103,"ानव":65,"ुख ":149,"ानस":161,"ानम":146,"ाध्":103,"ापन":228,"ान्":702,"ानो":110,"ादे":93,"ादी":738,"ादु":180,"ादन":914,"ानक":94,"ाद्":158,"ाति":178,"ाता":140,"ाती":68,"ाण्":134,"ात्":408,"ादक":75,"ीय ":427,"ाडौ":440,"ारी":407,"ारि":163,"ारा":1342,"ार्":2184,"ारे":69,"ालक":765,"ालम":256,"ालय":324,"ालद":78,"ामा":889,"ायक":72,"ाम्":197,"ायण":151,"ायत":156,"ाया":71,"ारक":244,"ारम":182,"ारण":184,"ारत":376,"ाबा":106,"िएक":373,"ामक":80,"ामय":74,"ामम":66,"ापा":186,"ाप्":232,"ुङ ":64,"ंघ":232,"ँस":83,"ौ ":65,"ंख":88,"ंग":634,"ंक":193,"ँल":78,"ंच":134,"केन":240,"ँद":154,"ँड":80,"ँम":363,"ंस":389,"ंह":86,"ंव":201,"् ":1181,"ंत":94,"ंम":241,"केह":83,"ँक":178,"ो ":20357,"ँग":166,"ँच":76,"ँउ":243,"कृत":264,"अस":341,"अव":627,"आक":84,"अल":82,"अर":461,"अभ":65,"आए":70,"अम":151,"घर ":251,"अप":74,"अध":281,"अन":771,"अथ":96,"इत":80,"ाइ ":156,"इए":81,"आय":68,"आर":95,"आफ":174,"कृष":151,"आद":141,"आध":94,"आन":112,"ाई ":1080,"अं":111,"अञ":333,"अत":71,"अक":101,"अग":107,"उम":66,"उप":377,"उह":331,"ऋत":72,"इन":366,"इर":70,"इल":137,"इस":102,"उँ":251,"उं":238,"उक":79,"उच":90,"उट":311,"उत":340,"हो।":2612,"उन":717,"उद":141,"कोट":86,"एउ":305,"एक":5494,"एम":113,"एप":66,"क्क":84,"क्त":425,"क्य":129,"क्न":67,"क्ष":1267,"क्र":426,"क्स":110,"ाँ ":310,"ए।":67,"एव":82,"एर":147,"एस":80,"गर":2258,"गल":135,"गव":76,"खे":215,"गन":66,"खो":112,"गम":263,"ख्":345,"खा":431,"गठ":95,"खि":580,"खी":82,"खु":123,"ाग ":123,"गढ":68,"गण":181,"गत":326,"गको":91,"खर":150,"क्":2722,"खम":75,"गक":127,"कै":214,"के":654,"को":14452,"कि":691,"की":474,"का":12832,"कृ":425,"कु":618,"कू":91,"कस":66,"कव":78,"कह":90,"कल":273,"कम":415,"कर":463,"कप":247,"कन":76,"कत":154,"ाङ ":105,"कक":112,"ओव":398,"चु":124,"ची":172,"चि":731,"चा":795,"छन":717,"चौ":90,"च्":216,"चो":89,"० ":345,"जक":89,"चन":324,"ङ्":397,"चर":129,"चल":599,"घा":108,"गण्":66,"ङम":69,"गते":76,"। ":7045,"गा":955,"गी":218,"गि":310,"गु":302,"गो":351,"गौ":84,"ग्":809,"गे":161,"गै":75,"घर":335,"टन":160,"ञ्":596,"ञा":169,"टा":992,"टर":235,"४ ":302,"झा":105,"३ ":263,"टक":135,"जो":225,"जे":208,"छ।":2479,"जी":387,"जु":351,"जा":939,"जि":1330,"२ ":297,"ज्":670,"जन":1499,"खान":74,"जद":90,"जध":126,"छि":255,"छा":205,"जस":334,"जव":93,"१ ":351,"जर":120,"जल":76,"गठन":79,"छो":118,"जय":160,"जम":205,"ठा":393,"डक":133,"ाथ ":102,"५ ":351,"ठन":83,"टे":203,"ठम":522,"ट्":498,"टो":148,"टी":408,"टि":258,"डा":364,"डि":270,"डी":265,"डल":256,"६ ":313,"ाद ":282,"ठु":73,"ठू":104,"ढी":114,"ढा":72,"ाण ":70,"णक":119,"७ ":328,"ड्":123,"डौ":474,"डो":86,"डे":171,"णि":132,"णी":170,"णा":196,"ात ":197,"तक":594,"८ ":352,"णम":93,"तव":80,"९ ":264,"तह":110,"ति":2878,"ता":1722,"तु":292,"ती":680,"तथ":624,"तप":208,"तन":233,"तम":226,"ण्":631,"तय":73,"तल":91,"तर":935,"थव":101,"दछ":271,"था":1584,"थी":140,"थि":1694,"ते":197,"तै":125,"तो":181,"थम":91,"त्":7154,"थल":165,"खेल":103,"दक":289,"दस":347,"दू":145,"दु":584,"दी":1175,"दि":689,"दा":996,"दन":1011,"दव":74,"दल":480,"दर":324,"दम":71,"थ्":174,"नज":74,"धा":907,"नत":94,"नद":266,"धी":98,"धि":828,"ान ":852,"धु":156,"दो":255,"दौ":70,"द्":2831,"दे":1366,"धन":183,"दै":256,"नक":650,"नग":337,"धर":218,"नर":137,"नल":223,"नव":289,"ाज ":178,"धे":169,"नन":100,"नप":96,"नब":64,"नय":150,"नम":477,"ध्":787,"पं":75,"नी":1179,"नु":1548,"ने":4740,"नस":329,"नह":97,"ना":2286,"नि":2559,"पक":159,"नो":316,"नै":540,"न्":7416,"पत":4116,"पन":1101,"पद":120,"न।":118,"पछ":277,"पट":80,"पश":213,"पह":665,"पस":130,"पल":132,"पम":272,"पर":1026,"पे":166,"पू":427,"पृ":79,"पा":5213,"पि":272,"पी":141,"पु":1182,"फल":131,"फर":70,"प्":6004,"फब":94,"पो":196,"बन":449,"फे":70,"बत":86,"फु":76,"बढ":88,"फू":67,"फा":131,"बज":120,"फ्":316,"भए":2692,"बर":285,"बल":88,"भक":78,"भग":82,"बस":232,"बह":195,"बि":521,"बा":1458,"बु":218,"मं":75,"बी":142,"भद":79,"भन":878,"बे":211,"गरम":92,"बै":272,"बो":141,"गरप":94,"भय":191,"भर":147,"गरे":321,"ब्":486,"मक":339,"गरी":134,"गरि":502,"मग":77,"मज":101,"भि":375,"भा":2323,"मत":163,"मण":318,"भू":158,"भु":153,"गर्":821,"मन":572,"भे":108,"मध":481,"मद":72,"भो":89,"मप":65,"मल":187,"यक":819,"मम":151,"भ्":133,"मय":212,"मर":128,"मस":126,"मह":673,"मृ":68,"यत":299,"मू":114,"यण":158,"ख्य":245,"यद":91,"मि":1113,"मा":10729,"ाट ":785,"मु":840,"मी":348,"रं":73,"मो":273,"यम":527,"म्":2421,"मे":530,"यन":231,"यप":89,"मै":88,"यव":183,"रख":100,"रग":116,"रक":3696,"यल":80,"यर":101,"या":2168,"रज":106,"यह":189,"रच":84,"गमा":170,"यस":2268,"रध":173,"रद":310,"रथ":102,"रत":1236,"रण":521,"यु":657,"यी":175,"यि":191,"रय":191,"रम":1296,"रभ":77,"रब":131,"यो":5061,"रप":226,"रन":174,"ये":226,"लम":462,"लय":351,"लब":81,"लद":123,"लन":243,"गाउ":130,"लच":65,"लग":235,"लक":1487,"र्":6868,"रो":703,"रै":297,"रे":1245,"गाँ":224,"रू":1254,"री":1735,"रु":1451,"रि":5597,"रा":5311,"रह":868,"रस":574,"रश":114,"रव":321,"रल":193,"वं":88,"िक ":1742,"ल्":1854,"लो":1078,"चन ":69,"ले":2083,"लु":247,"ली":1940,"लि":1013,"ला":3708,"लल":168,"लह":138,"लस":69,"शब":122,"वै":233,"शन":968,"वे":271,"षक":102,"शम":116,"व्":325,"शर":176,"वह":113,"वव":90,"वस":368,"वु":106,"वा":3817,"वि":2520,"वी":434,"वप":72,"वन":350,"वध":419,"शक":1037,"वल":144,"वर":826,"वय":169,"वम":117,"वक":170,"वत":227,"वट":383,"ाह ":84,"सन":601,"षे":417,"सप":72,"सभ":377,"सब":248,"ष्":905,"सम":2673,"सर":429,"सल":312,"सव":65,"हक":127,"सच":90,"षा":1291,"षि":415,"षी":120,"िङ ":81,"ास ":619,"सत":83,"सद":673,"शे":152,"श्":1168,"षर":77,"सग":64,"सक":2028,"शह":143,"सं":1482,"शी":150,"शु":136,"शा":630,"शि":1111,"सँ":111,"षण":89,"हे":701,"हु":2366,"ही":279,"हि":1570,"हा":1720,"िए ":165,"ाम ":560,"ह्":87,"हो":3850,"से":478,"हन":162,"सु":586,"सी":345,"हत":108,"सू":105,"सि":1173,"चल ":150,"सा":2163,"सह":275,"हज":68,"हल":85,"स्":3576,"हर":1967,"सै":149,"सो":307,"ात":1181,"ाथ":320,"ाण":404,"ाठ":517,"िं":132,"ाड":760,"ाल ":1016,"ाट":1019,"ाब":231,"ाभ":93,"ाप":949,"ान":3357,"ाद":2589,"ाध":265,"गुर":71,"िख":71,"ाव":435,"िक":7298,"ाल":5422,"ार":6852,"ाय":812,"ाम":2329,"िए":681,"िज":372,"ाह":1078,"िच":143,"िङ":184,"ास":1934,"ाष":1391,"ाश":2631,"िग":115,"ां":267,"ाँ":1104,"ाइ":698,"ाई":1298,"ाउ":964,"ाओ":403,"ाक":4316,"ाए":185,"गुन":74,"ाच":265,"ाज":1659,"ाग":1013,"ाख":288,"ाङ":244,"ाघ":70,"ार ":1305,"ुँ":124,"ीद":163,"गिर":76,"ुई":88,"ीत":508,"ीप":148,"ीन":313,"ीम":461,"ीय":520,"ीब":79,"ील":304,"ीर":227,"ुग":188,"ुख":347,"ीव":147,"गीत":100,"ुक":691,"ीह":209,"ीस":78,"ुङ":145,"ुट":214,"िट":172,"ाय ":70,"िण":163,"ित":2827,"िद":484,"गाय":88,"िध":410,"िन":2481,"िप":245,"िब":181,"िभ":260,"िम":1517,"िर":1038,"िय":2231,"िल":2254,"ीक":872,"िश":533,"िव":468,"िस":542,"िष":301,"िह":253,"ेव":427,"ेश":820,"ेल":681,"सला":171,"ेर":844,"ेम":230,"ेब":84,"ेप":2592,"सले":93,"ेन":806,"ैत":257,"ैज":79,"ेह":196,"ेस":274,"ेष":206,"ैश":81,"ैर":64,"ैल":110,"ैभ":84,"ैम":134,"ैन":181,"ेक":1516,"ेख":723,"ेट":234,"ेड":98,"ेत":801,"ेद":151,"ेग":72,"ेज":215,"समु":85,"समि":387,"समा":459,"ृथ":65,"ृत":389,"ृष":194,"गोर":127,"सरक":167,"गोल":71,"समे":69,"सम्":1386,"ुत":130,"ुण":70,"ुन":2984,"ुद":383,"ुब":76,"ुप":397,"ुर":1722,"ुम":556,"ुभ":589,"ूक":155,"ग्ल":85,"ुल":730,"ुष":88,"ुस":363,"ुव":409,"ग्र":367,"ुश":84,"ुह":538,"सर्":105,"ग्न":160,"ूद":147,"ून":67,"ूप":214,"ूम":171,"ूर":453,"ूल":388,"ूह":74,"सबै":187,"्व":2939,"्श":82,"्ष":1678,"्स":579,"्ह":81,"्भ":100,"्म":1441,"्य":5693,"्र":13669,"्ल":1547,"्।":480,"समय":115,"ष्म":69,"ष्ण":122,"ष्ठ":171,"ष्ट":483,"सभा":365,"सद्":160,"ौत":91,"ोर":513,"ोल":515,"सदस":314,"ोब":119,"ोम":148,"ोस":122,"ोह":153,"सदर":107,"ोश":104,"ोव":69,"्ण":427,"्त":3287,"्ड":711,"्ट":1245,"्ठ":192,"्ञ":223,"्फ":210,"्ब":412,"्प":1700,"्ध":702,"्न":1912,"षेत":396,"्थ":1433,"ो।":3741,"्द":2288,"ौल":71,"ौर":128,"्ज":453,"्छ":2152,"्च":1134,"्ग":579,"्ख":113,"्क":866,"सन्":376,"ोज":238,"ौं":159,"ोड":185,"ोट":228,"ौँ":401,"ोद":66,"ोत":169,"ोप":186,"ोध":78,"ोन":98,"ोख":109,"ोक":287,"ोग":453,"हत्":91,"सेन":118,"छन ":99,"सुन":123,"सिर":66,"सिम":95,"सिन":119,"सुर":172,"साह":147,"सिद":138,"सान":163,"साद":218,"साप":131,"सार":311,"साम":357,"सिक":403,"साल":247,"सिं":91,"साथ":94,"सहर":108," १८":152," १९":516," २०":1012,"०६":103,"०४":114,"०५":217,"०२":500,"०३":93,"००":262,"०१":255,"१०":116,"१५":81,"१७":82,"१८":226,"१२":66,"१४":88,"१९":555,"२०":1126," ७१":72,"हरू":762,"हरु":794,"हरि":77,"ङमा":65,"७१":90,"स्व":373,"९०":64,"छि ":189,"९६":77,"९५":70,"९८":66,"९१":112,"स्र":74,"स्य":402,"स्न":77,"स्थ":1019,"स्प":226,"स्ट":105,"स्त":727,"९९":85,"स्क":351,"२३":69,"२२":66,"२५":83,"२७":122,"२६":80,"२९":70,"२८":115,"सोज":81,"सेव":93," १०":76," ०५":134,"हेक":471,"हेन":82,"जन ":98,"हुन":2172,"हुँ":68,"हिम":124,"हिल":691,"हिन":285,"हित":161,"हास":142,"हिक":125,"हाल":183,"हार":140,"हान":72,"हाद":225,"हाड":113,"हाँ":430,"चना":64,"चन्":135,"सचि":83,"सकि":79,"सका":115,"सको":1596,"सक्":148,"कै ":184,"का ":4257,"कि ":84,"षाक":932,"की ":394,"षिण":127,"हो ":1122,"षिक":187,"ओवा":396,"शेष":71,"हा ":81,"ही ":218,"श्व":419,"श्र":376,"श्य":86,"श्च":215,"शहर":129,"सी ":177,"हर ":124,"संग":215,"संख":67,"संघ":224,"संव":197,"संस":296,"शाख":89,"शित":796,"शाह":92,"शास":187,"सँग":107,"शिक":120,"सो ":69,"शार":66,"वैश":69,"वेश":81,"वुल":80,"शर्":111,"सा ":76,"व्य":273,"शब्":114,"वर्":460,"षा ":211,"किन":120,"काम":260,"कार":2072,"काल":377,"किक":104,"काश":2575,"कास":405,"किस":96,"कुन":166,"कुर":99,"कुम":159,"ववि":79,"शको":75,"कान":97,"काठ":489,"काक":1844,"वस्":272,"वहा":72,"सन ":103,"गि ":141,"वाद":803,"वान":150,"वाच":132,"विक":476,"वाल":112,"वास":237,"कला":91,"वार":1366,"वाम":64,"वित":78,"विन":80,"विद":224,"विध":297,"विज":192,"वाह":64,"गी ":84,"विष":89,"विश":380,"विस":90,"विम":83,"विर":102,"विभ":130,"वीर":70,"कम्":158,"कमा":129,"कर्":161,"गा ":93,"करण":105,"कपु":78,"कपा":103,"वधि":404,"कता":70,"वमा":91,"गर ":180,"वयम":124,"सं ":192,"शी ":67,"गत ":162,"खि ":378,"वटा":364,"खी ":68,"खा ":132,"को ":13998,"शन ":893,"वि ":102,"एमा":79,"वा ":501,"वी ":188,"ल्न":81,"ल्प":107,"ल्य":159,"ल्ल":1131,"एवं":67,"एकी":88,"एका":367,"एकि":98,"एको":2830,"लोक":92,"लेख":158,"शक ":887,"लेक":101,"लेट":101,"लिम":70,"लुम":71,"वर ":134,"लाई":880,"लाइ":94,"लाग":378,"लाक":490,"लाम":331,"लाल":114,"लिक":155,"लाह":236,"लिङ":98,"लित":141,"लिन":92,"वन ":146,"लहर":130,"एउट":304,"ललि":72,"लमा":361,"लद्":81,"लका":156,"वं ":67,"लगा":129,"लको":1159,"लक्":65,"रैम":78,"रेस":80,"रेष":101,"रोप":82,"रोग":68,"र्श":68,"र्व":558,"र्स":92,"र्ष":409,"र्म":544,"र्य":1351,"र्थ":286,"र्द":429,"र्न":750,"र्फ":154,"र्ट":316,"र्ण":301,"र्त":237,"र्ग":285,"र्ख":68,"र्क":192,"र्ज":198,"र्छ":112,"र्च":149,"रीक":121,"रिव":148,"रिय":408,"रीम":87,"रीय":210,"रुक":171,"रुम":97,"रुप":191,"रुल":94,"रूक":141,"रूद":136,"रूप":207,"रूम":82,"रूल":87,"रेक":307,"रेज":78,"रेल":67,"रेर":97,"रेन":80,"रसा":265,"रसि":109,"रहे":467,"रहर":171,"रस्":71,"ले ":1440,"राई":127,"रान":448,"राप":77,"रात":83,"राण":115,"राख":96,"राज":1244,"राक":104,"रिन":282,"रित":70,"राष":293,"राह":72,"राम":287,"रिए":176,"राय":214,"राल":136,"रिक":3842,"लो ":852,"रला":76,"लि ":69,"ला ":807,"रयो":160,"रम्":75,"रमा":715,"रमु":241,"रवा":142,"रले":64,"ली ":1594,"रधा":161,"येक":79,"रदे":155,"रमण":71,"यो।":1126,"रबा":70,"योग":340,"रपा":108,"युक":77,"युर":75,"युन":112,"युद":113,"याङ":69,"याक":103,"याम":150,"यान":128,"याप":111,"यात":120,"याद":68,"यास":84,"यिक":111,"याल":253,"यार":124,"याय":70,"यिन":65,"रति":595,"रत्":114,"रथम":70,"लय ":258,"युव":76,"रतक":192,"रणा":86,"यसक":1478,"यसै":79,"यसल":145,"रगत":68,"यवस":100,"रक्":100,"रको":493,"रजा":74,"याँ":127,"यहा":98,"यस्":93,"लन ":125,"रे ":127,"महे":70,"महा":221,"महि":223,"महत":77,"यको":226,"यक्":275,"रू ":452,"रु ":592,"यका":195,"मले":67,"री ":1056,"मुद":90,"मुख":298,"मुक":206,"मिल":98,"मित":526,"मिन":83,"मार":349,"माल":263,"मिक":100,"रो ":251,"मास":338,"मिट":74,"माण":135,"माड":449,"माध":70,"माथ":111,"मात":198,"मान":717,"माओ":399,"माक":73,"माज":262,"रै ":187,"मोर":76,"यन्":64,"यद्":67,"मेत":64,"मेर":107,"मेल":71,"रका":2994,"यमि":88,"ऋतु":72,"म्र":98,"म्य":162,"यमा":180,"म्प":1201,"म्ब":311,"म्म":335,"रत ":106,"मको":161,"रण ":320,"या ":441,"उहा":325,"यी ":158,"भिन":116,"भित":82,"भाव":75,"यो ":3433,"भास":199,"भाष":1089,"भार":394,"भाग":184,"उपत":120,"रम ":92,"मजद":66,"ये ":122,"मध्":444,"मना":80,"मन्":256,"भूम":78,"मण्":216,"मती":69,"मयि":71,"रा ":1447,"०० ":67,"रि ":116,"ममा":144,"मा ":6826,"बर्":135,"यत ":118,"भक्":69,"मी ":185,"यन ":118,"बस्":105,"बहा":133,"बाल":77,"बाह":65,"बास":89,"बिन":70,"बाट":745,"बाग":79,"बार":117,"बिह":66,"यम ":171,"बेल":114,"यस ":334,"भनि":212,"भने":219,"भन्":420,"बैभ":80,"भयो":166,"ब्र":111,"ब्य":100,"ब्द":140,"एर ":132,"प्य":67,"प्र":5545,"प्त":252,"भा ":133,"मन ":82,"बजा":98,"मय ":66,"यक ":69,"बने":91,"बना":154,"बन्":175,"एक ":2017,"भएक":2580,"फ्न":128,"फ्र":99,"आदि":87,"पहि":540,"पहा":109,"पश्":183,"आन्":94,"पर्":460,"आफ्":129,"इएक":72,"२०२":474,"परि":275,"२०१":229,"परा":119,"२००":176,"पमा":255,"�":311,"फबा":94,"पोख":90,"उन ":91,"पृथ":65,"पुस":117,"पूर":347,"पुग":79,"पुर":744,"पित":80,"पाक":119,"पान":108,"पात":131,"पाद":966,"पार":420,"पाल":2675,"पाइ":114,"बै ":101,"अथव":91,"१९१":90,"१९९":68,"१९६":68,"अधि":111,"अनि":120,"अनु":226,"अध्":166,"अन्":365,"न्च":76,"न्छ":1928,"न्त":1150,"न्ट":89,"न्ध":228,"न्न":383,"न्थ":109,"न्द":1479,"न्य":282,"न्म":233,"न्स":147,"न्।":473,"नैत":179,"अमे":66,"अरू":140,"अर्":230,"अवस":167,"अवध":415,"पनी":71,"पना":230,"पनि":614,"पन्":133,"असो":81,"अस्":166,"पत्":3819,"पति":105,"पता":130,"पछि":199,"पा ":229,"नले":102,"नया":92,"नवा":108,"नन्":74,"०२७":93,"०२८":90,"नदी":180,"धेर":139,"नमा":333,"ध्य":728,"नुह":426,"उटा":296,"नुप":72,"नुभ":561,"नुस":135,"निस":101,"निष":82,"नीत":296,"नेप":2459,"नेत":159,"नेक":221,"फल ":80,"उनी":101,"उनु":99,"उने":183,"उनल":77,"उद्":70,"उनक":91,"नस्":108,"निर":312,"निय":235,"निम":64,"निन":391,"उत्":329,"निध":76,"नाल":106,"निक":384,"नाम":395,"नार":232,"नाथ":91,"नाक":103,"नाउ":100,"नाइ":96,"द्द":65,"द्व":1093,"द्र":692,"द्य":285,"द्ध":407,"धर्":124,"नकप":71,"इने":87,"इन्":241,"नका":130,"नको":327,"इला":81,"देख":530,"देव":249,"देश":451,"दैन":81,"१८ ":75,"दोल":117,"२७ ":96,"धित":72,"धिम":418,"धार":244,"धिक":138,"धान":515,"२८ ":93,"नता":76,"नगर":305,"न। ":91,"उंम":222},"n_words":[614665,704688,490631],"name":"ne","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/nl.json b/contrib/languages-data/nl.json
new file mode 100644
index 0000000..6f4124a
--- /dev/null
+++ b/contrib/languages-data/nl.json
@@ -0,0 +1 @@
+{"freq":{"D":295391,"E":101171,"F":127232,"G":105530,"A":221658,"B":178586,"C":193400,"L":124657,"M":162943,"N":137196,"O":83395,"H":199830,"I":105198,"J":61744,"K":84088,"U":22509,"T":111990,"W":76478,"V":96433,"P":145276,"S":218863,"R":114676,"Y":11382,"Z":56681,"f":358313,"g":1325585,"d":2549536,"e":9521882,"b":675678,"c":885504,"a":4428747,"n":5174765,"o":3044475,"l":2174413,"m":1301830,"j":487381,"k":957853,"h":1157193,"i":4146613,"w":618312,"v":1136367,"u":1226136,"t":3691802,"s":3058116,"r":3420928,"q":14380,"p":897097,"z":331215,"y":211813,"x":48411,"²":22665,"ï":7308,"í":7570,"ë":58538,"é":54151,"è":10784,"ä":6605,"á":12476,"ü":12035,"ö":12215,"ô":9257,"ó":9442," l":108039," m":271385," n":113728," o":359305," h":429647," i":863484," j":55299," k":151715," d":1022256," e":865604," f":60480," g":362770,"р":6363," a":271750," b":268848," c":85027," z":129440," u":151127," t":287237," w":287907," v":736662," p":253369," s":272741," r":136806," J":58839," K":80686," H":197094," I":88140," N":131673," O":75874," L":120300," M":156963," B":171895," C":177462," A":209281," F":122253," G":98641," D":289943," E":96010," Z":55679," Y":11046,"и":8389,"о":8805,"н":6323," S":205129," R":108525," P":136936,"а":10448," W":74232," V":87489," U":20421,"е":6745," T":105528," é":7785,"A ":9463,"Da":14497,"Cu":5824,"Cl":7287,"Co":57452,"Cr":6279,"Ce":10546,"Ch":35435,"Ci":5986,"Du":33850,"Do":13572,"Dr":5649,"De":198026,"Di":16644,"Fe":6923,"Fa":6211,"Eu":11420,"Er":8327,"En":17423,"El":7917,"Ee":19326,"Ge":21620,"Ga":13267,"I ":14576,"Fr":73223,"Fo":9924,"Fi":11740,"C ":12680,"Au":12206,"Ar":23539,"Ba":28977,"Am":55987,"An":22709,"Al":28901,"Bu":10850,"Br":34564,"Ca":38690,"Bi":11873,"Be":44722,"Bo":26862,"Bl":5807,"Kr":9527,"Ko":15418,"Le":21847,"Li":19176,"La":31976,"Lu":12150,"Lo":28450,"Me":24563,"Mi":26609,"Ma":59234,"Mu":7133,"Mo":28002,"Ni":14152,"Ne":59633,"Na":17889,"No":27858,"Ol":9544,"Gi":5965,"Gr":28322,"Go":11925,"Gu":7667,"Ha":30031,"He":95269,"II":10036,"Hi":30742,"Ho":27037,"Hu":6810,"In":31628,"Is":5913,"It":15844,"Ja":19607,"Je":8912,"Jo":19365,"Ju":6270,"Ka":21546,"Ki":7967,"Ke":10121,"Un":6926,"Tu":6976,"Tr":10760,"Ts":9146,"To":14361,"Th":19966,"Ti":10118,"Te":13194,"Ta":10688,"V ":6513,"St":35202,"Su":9985,"Wo":6733,"Wi":21345,"Wa":15248,"We":22896,"Vo":12732,"Vi":16383,"Vl":8595,"Va":15790,"Ve":22438,"Pr":15997,"S ":6945,"Pe":17479,"Pa":35566,"Po":30054,"Pi":14013,"Oo":13070,"Op":6344,"Or":9483,"Se":15433,"Sc":16133,"Si":16384,"Sh":6081,"Sl":6946,"Sp":19561,"So":16434,"Ru":16114,"Sa":42083,"Re":17106,"Ri":17272,"Rh":6524,"Ro":32707,"Ra":12538,"b ":17772,"a ":223995,"Yo":6270,"Ze":12835,"Zi":6830,"Zu":10609,"Zw":13009,"i ":117137,"gd":23906,"ge":622849,"ga":53344,"fl":6356,"fg":6573,"ff":17069,"fi":41042,"fs":14314,"fr":11929,"fu":6506,"ft":32454,"fo":24009,"j ":85100,"he":517783,"ha":140501,"gn":22691,"gl":11647,"gi":125274,"gh":17920,"gg":9680,"gu":24589,"gt":35908,"gs":36659,"gr":72480,"go":35093,"dt":42741,"du":30903,"dw":10868,"g ":209248,"ea":48028,"eb":81350,"ec":86712,"ed":198243,"de":1265409,"dd":20989,"dg":6934,"di":281697,"dh":8510,"dk":6604,"do":126396,"ds":99321,"dr":53877,"ew":31588,"ex":17157,"eu":69138,"ev":102331,"ey":15758,"ez":66338,"fa":30179,"h ":79224,"fd":18426,"fe":38002,"eh":28491,"eg":168269,"ef":50847,"ee":952339,"el":605553,"ek":125240,"ei":156611,"ep":103855,"eo":27881,"en":2016971,"em":320296,"et":590121,"es":318427,"er":1223846,"ca":46321,"e ":2100607,"bs":5841,"br":58403,"bu":44669,"bo":58794,"bl":25851,"bi":87840,"bb":9494,"be":276525,"da":122637,"f ":90198,"cu":23421,"ct":84409,"cr":11311,"co":70369,"ck":28358,"cl":14976,"ci":82756,"ch":396490,"ce":87968,"c ":16305,"az":12546,"ay":18941,"ba":76157,"d ":427738,"at":401880,"as":187038,"ar":458993,"aw":6737,"av":29958,"au":65384,"ak":138064,"al":362702,"ai":53889,"aj":7035,"ap":71362,"am":172784,"an":1131186,"ac":99694,"ad":102907,"aa":662152,"ab":30531,"ag":85506,"ah":10878,"ae":23635,"af":47867,"nu":31030,"nt":435391,"ns":284103,"nr":14733,"no":95599,"nn":67675,"nz":11234,"ny":9083,"nw":88548,"nv":13925,"oe":163750,"of":83411,"oc":46361,"od":56125,"oa":14423,"ob":29747,"om":159575,"on":522696,"ok":52068,"ol":157894,"oi":33928,"og":70046,"oh":14048,"ot":108603,"m²":22643,"os":81239,"ov":88551,"ou":131520,"op":166634,"oo":370075,"or":483257,"r ":577062,"ow":27555,"oz":6517,"oy":5912,"pe":152921,"pg":12730,"pa":116725,"pl":132966,"po":60048,"ph":12783,"pi":48220,"lo":99408,"lm":24809,"ll":128927,"ls":119057,"lp":13038,"lv":15042,"lu":37402,"lt":126691,"ly":17100,"o ":155926,"md":19425,"ma":206926,"mb":60844,"me":492457,"mi":91034,"mm":32963,"mp":51654,"ië":52728,"mo":56705,"mt":19679,"ms":35581,"mu":25483,"my":6479,"p ":141470,"na":201890,"nb":21559,"nc":69092,"nd":473575,"ne":270200,"nf":9979,"ng":259192,"nh":15116,"ni":168878,"nk":43137,"nl":16899,"nm":6930,"jv":11516,"ju":22953,"js":16655,"jn":70736,"jo":13034,"jk":115620,"ki":53710,"kh":5862,"ke":182363,"ka":118671,"m ":155936,"kw":7713,"ks":46927,"kt":131352,"ku":22530,"ko":60158,"kr":23259,"kk":20620,"kl":29172,"km":27830,"kn":6593,"li":338485,"lh":5934,"lk":34257,"le":302236,"ld":95337,"lg":36498,"lf":20122,"la":398196,"lc":8500,"lb":22338,"n ":2543334,"hr":30422,"ht":107458,"hu":34327,"hi":99792,"hn":9649,"ho":105347,"id":125660,"ic":175476,"ib":14735,"ia":95834,"ig":142126,"if":16130,"ie":477304,"k ":193827,"ir":54494,"is":722822,"it":333187,"iu":13498,"iv":35131,"iw":8456,"ix":5720,"ij":364327,"ik":93516,"il":169468,"im":40230,"in":879621,"io":144997,"ip":26974,"je":41083,"jd":31240,"jf":10910,"iz":17029,"l ":299819,"ja":41826,"xi":7194,"z ":13752,"wi":58960,"wn":7387,"wo":161887,"ws":8197,"y ":100628,"wa":155603,"wd":6545,"we":181101,"vl":34784,"ré":7748,"vi":112804,"vr":17131,"vo":180923,"uz":15014,"ux":8237,"uw":44437,"uv":7894,"uu":52115,"ve":243007,"va":525240,"x ":20927,"ui":271759,"uk":10265,"ul":55545,"ue":25713,"ug":30034,"ur":165500,"us":143221,"ut":46184,"um":51805,"un":100952,"up":10064,"ty":46646,"tz":8930,"tu":100531,"tt":45242,"tw":33429,"tv":9249,"ub":30475,"ua":33926,"ud":45500,"uc":29841,"w ":28312,"to":174226,"tm":7826,"tl":14300,"ts":211260,"tr":144811,"tg":20068,"te":770637,"tk":6386,"tj":6074,"ti":257739,"th":76245,"v ":8380,"tb":23716,"ta":261366,"su":28639,"sv":9064,"ss":138251,"st":562933,"sy":14252,"sz":5928,"sw":6865,"sl":43368,"sk":33909,"sn":9526,"sm":15035,"sp":74738,"so":59040,"sr":6378,"sd":20593,"sc":205927,"se":425364,"sh":26990,"sg":6293,"sj":16338,"si":98258,"rz":20638,"u ":30645,"sa":44187,"sb":15979,"rr":71892,"rs":228779,"rt":187734,"ru":73560,"rv":44462,"rw":18490,"ry":14720,"rp":27084,"ro":286166,"rn":60686,"né":6017,"rm":70366,"rl":117376,"rk":74650,"ri":419079,"rh":16037,"rg":84498,"rf":11818,"re":392586,"rd":257870,"rc":25722,"rb":33548,"ra":285112,"t ":1445388,"qu":12987,"s ":1176543,"pt":24055,"pu":20157,"pp":39364,"pr":99751,"ps":14478,"zi":116688,"ze":78298,"za":31109,"zu":15798,"zo":49094,"ye":6127,"yc":6275,"ya":8170,"ys":18578,"yr":9415,"yp":6946,"yn":9021,"ym":12848,"yl":8352,"² ":22661,"éé":7636,"én":13925,"ë ":43502,"é ":8856,"一":6916," Ga":13216," Ge":21536," Fo":9883," Fr":73181," Fi":11695," Ha":29997," He":95105," Go":11879," Gr":28204," Gu":7630," Gi":5928," Hu":6795," Ho":26988," II":6497," Hi":30725," Je":8879," Ja":19556," Is":5895," It":15839," In":31514," Ka":21414," Ke":10042," Ki":7899," Jo":19317," Ju":6264," La":31862," Le":21646," Li":18941," Ko":15401," Kr":9517," Ma":59034," Mi":26529," Me":24479," Lo":28411," Lu":12123," Ne":59527," Na":17815," Ni":14127," Mo":27929," Mu":7091," Am":55975," An":22679," Al":28811," Ba":28861," Au":12111," Ar":23451," Be":44588," Bi":11811," Bl":5783," Bo":26752," Br":34502," Bu":10803," Ca":38284," Ce":10523," Ci":5893," Ch":35359," Cl":7183," Cr":6209," Co":57213," Cu":5704," Da":14480," Di":16596," De":197691," Do":13329," Du":33818," El":7895," Ee":19260," Er":8309," En":17358," Eu":11409," Fe":6899," Fa":6133," Wo":6660," Wi":21285," We":22829," Wa":15180," Zu":10598," Zw":13007," Ze":12821," Zi":6778," Yo":6266," Or":9466," Oo":13036," Op":6330," Po":29977," Pi":14002," Pe":16009," Pa":35403," No":27810," Ol":9537," Ra":12376," Ro":32636," Re":17067," Ri":17238," Rh":6521," Pr":15952," Su":9973," St":34839," Ta":10649," Th":19903," Ti":10088," Te":13125," Tr":10698," Ts":9134," To":14233," Ru":16099," Sa":42032," Sh":6028," Si":16291," Sc":16056," Se":15382," So":16378," Sp":19500," Sl":6935," Va":15750," Ve":22303," Vi":16318," Vl":8585," Vo":12506," Tu":6926," Un":6865," ja":26885," in":468125," is":385652," ka":34673," ki":8773," ke":15418," ju":21208," ha":22481," he":342958," gr":38760," go":7687," hi":19625," ho":31679," hu":11200," ni":13445," ne":8341," na":55514," mu":11657," mo":19517," ok":9997," om":21439," on":84624," of":50488," nu":5724," no":28282," le":24338," li":33990," la":36262," ku":10447," km":25898," kl":14578," kr":8283," ko":24113," me":105531," mi":19475," ma":109683," lo":6645," af":19870," aa":54537," ac":15329," an":19082," ap":11762," al":65830," au":17140," ar":47665," ba":14963," bi":41311," be":166033," bo":19486," bl":6588," bu":9669," br":10013," ca":7354," e ":8358," er":11018," et":7087," en":320059," ei":19716," el":9897," ee":469095," fe":12855," fa":15977," fo":5946," fi":16445," ge":303157," ga":6353," co":32521," ce":11923," ch":6021," ci":15694," da":51374," do":72769," dr":11539," de":740739," di":128156," du":9386," zo":26839," zu":11699," za":9390," ze":21870," zi":55419," ru":5958," sa":9420," se":22896," sc":26028," si":12277," sl":6445," sp":31376," so":18785," ra":8521," re":91305," ri":13940," ro":16553," pr":57908," s ":10712," ou":7125," ov":18421," oo":33765," op":114061," or":11383," pe":20368," pa":24621," pl":113070," po":21982," wa":118659," we":90321," wo":59080," wi":19128," va":481040," ve":76969," vo":131543," vr":12217," vi":22015," vl":9586," tw":15968," tu":14723," ui":144553," ta":9836," sy":6615," st":111138," su":9964," tr":11805," to":58623," th":14839," ti":18789," te":138599," éé":7588,"Eur":9786,"Eng":13833,"Fra":63542,"II ":7320,"Hij":21364,"Het":68660,"Her":7011,"Gri":6704,"Gra":6848,"Gro":9812,"Ind":7590,"In ":14751,"Hon":6999,"Hol":6098,"Bar":6393,"Alp":5725,"Ame":44932,"Ams":6535,"Ant":6487,"Cal":8797,"Car":7946,"Ber":10327,"Bel":16135,"Bra":7962,"Bri":11166,"Bou":5646,"De ":168413,"Dez":5967,"Dee":6161,"Chi":8065,"Cen":7181,"Cha":14656,"Cor":5822,"Com":7136,"Con":6541,"Cou":26493,"Een":16908,"Dui":29012,"Ned":43974,"Nat":6561,"New":6417,"Nor":10502,"Noo":11547,"Oly":6017,"Oos":10737,"Per":5867,"Par":11966,"Poo":9123,"Ita":15181,"Joh":7769,"Lan":8583,"Man":6806,"Mar":24247,"Mon":9837,"Mid":7429,"Wil":9135,"Wer":5760,"Wes":9343,"Vla":7545,"Ze ":6085,"Sta":16762,"Sin":5858,"Spa":11230,"Rus":11905,"Sai":9817,"Sch":12674,"San":8826,"Rij":7380,"Rom":5676,"Ver":12912,"Uni":5818,"The":11210,"Tsj":8380,"bis":7734,"bin":13224,"bij":36202,"bli":10301,"bla":6463,"boe":8360,"bon":5829,"bor":10260,"bou":11481,"bbe":6915,"ban":15026,"bal":21186,"baa":7632,"bas":9398,"bar":8019,"beh":12808,"beg":7886,"bee":12385,"bed":18175,"ber":72483,"ben":9807,"bel":14825,"bek":19147,"bev":17462,"bes":52050,"bet":13589,"bie":17101,"ca ":12312,"car":7029,"cat":5698,"ce ":27218,"bri":6013,"bro":7591,"bra":10008,"bre":7181,"bru":26538,"bur":21107,"bum":8070,"am ":47277,"ake":12657,"al ":74743,"ail":5969,"ain":18706,"air":10562,"ais":6111,"ak ":11574,"agt":10140,"agn":7735,"anu":16559,"ano":8039,"ann":16248,"ant":55265,"ans":143853,"ane":9267,"ang":46635,"ani":27973,"ank":15316,"ap ":25134,"ana":20565,"anc":16651,"and":178626,"amm":8429,"amp":20053,"ams":9681,"ami":28047,"ame":31772,"amb":6788,"ama":9230,"alv":6355,"alt":35976,"als":49851,"all":30537,"ali":53705,"alc":5747,"ald":14462,"ale":38623,"ala":12213,"alb":11148,"an ":555423,"aks":6554,"akt":91851,"abe":6263,"abi":6431,"ae ":14725,"aaf":9902,"aag":19367,"aad":5831,"aak":81870,"aan":153725,"aal":41110,"aam":28835,"aas":6793,"aar":125046,"aat":183677,"ad ":41911,"afs":8757,"age":22392,"adi":11143,"ade":19021,"ag ":18180,"ach":45489,"ace":13447,"ada":5737,"af ":10774,"act":17739,"at ":119635,"are":27820,"ard":36005,"arc":10579,"arb":9145,"ara":15993,"aro":16161,"arn":8226,"arm":7787,"arl":11777,"ark":11271,"ari":63150,"arr":46599,"ars":14646,"art":89778,"au ":9748,"asi":6239,"ase":7343,"ar ":70080,"apa":7428,"app":11428,"apr":10161,"as ":104628,"aut":12900,"avi":8222,"ave":10618,"ay ":8013,"ata":7505,"ast":27312,"ass":19856,"ato":8717,"ate":41676,"ati":71342,"ath":9399,"att":6300,"ats":113866,"atu":9691,"aug":10264,"Zwe":6862,"Zui":9741,"jec":10400,"jk ":66346,"jaa":10547,"jar":7927,"jan":13106,"jd ":7717,"je ":16212,"jde":16692,"jns":7518,"js ":7214,"jn ":50205,"jks":11628,"jke":29064,"itt":7963,"its":47572,"ity":14389,"iss":57247,"ist":78828,"ita":16110,"ite":28930,"itg":14085,"iti":26080,"ium":6105,"ivi":15930,"ive":12826,"is ":431822,"ion":45782,"ir ":7395,"isi":11995,"ish":6473,"ise":13845,"isc":89426,"isa":6882,"ire":19679,"it ":148319,"iwo":7073,"ize":7008,"kin":23405,"ki ":12463,"kee":5719,"kel":22383,"ken":67439,"ker":29057,"ke ":43164,"kt ":89985,"kse":13533,"kri":8432,"km²":22590,"kor":8329,"kon":7670,"kom":21374,"ks ":13043,"kke":14372,"kle":13907,"kla":8538,"jul":9669,"jun":10013,"jve":8311,"jst":5685,"kan":26234,"kam":11248,"kaa":47537,"ka ":11569,"ham":9052,"han":20214,"hap":31501,"hal":11321,"har":17692,"haa":16281,"had":6694,"he ":78504,"hel":11001,"hei":31175,"hee":28005,"het":305470,"her":18043,"hen":9035,"hem":8872,"hie":14941,"hin":11368,"hil":13952,"hij":15137,"his":12860,"gne":9405,"gna":8227,"gon":7132,"gsd":8056,"gro":30942,"gra":24600,"gt ":25727,"gri":8059,"gre":7606,"gst":8398,"gus":11645,"ial":9076,"ian":13694,"iat":6807,"iaa":21077,"id ":39722,"ia ":31791,"iet":21274,"ieu":11845,"iev":7556,"iel":12150,"iem":5824,"ien":50882,"ier":48258,"ies":24371,"ied":23758,"ief":9097,"iek":35699,"ig ":26250,"ict":38543,"icu":11018,"ico":6343,"ici":9264,"ich":59319,"ice":15194,"ie ":201388,"ica":19828,"idi":13237,"ide":28170,"idd":13665,"ida":12788,"il ":19641,"ijd":30969,"ije":8022,"ijf":10866,"ijk":115115,"ijn":70104,"ijs":16040,"ijv":11367,"im ":7764,"ika":50912,"igd":11510,"ige":49060,"igh":9499,"igi":9202,"igt":11964,"ign":9284,"ij ":80452,"ik ":10619,"ime":7803,"inc":33586,"ind":41153,"ina":22287,"inn":14337,"ino":8487,"int":30127,"ins":19772,"ine":33694,"ing":150853,"ini":14846,"ink":10381,"ioe":9616,"inw":77340,"ikk":7357,"ike":6423,"ila":17614,"in ":405294,"ikt":10786,"ilo":7646,"ill":44186,"ilm":12944,"ili":39663,"ild":8065,"ile":5842,"ima":5906,"io ":71745,"hol":9679,"hou":16715,"hoo":29440,"hor":9181,"hoe":5979,"hui":10622,"hts":6548,"hth":9745,"hti":8381,"hte":21726,"hre":7796,"hri":17051,"ht ":49854,"hum":5764,"ffe":6190,"ffi":6818,"feb":9777,"fen":6207,"fam":16628,"fde":6413,"eze":22604,"ezi":32218,"etb":16830,"eta":11379,"ete":33498,"eti":8945,"eth":6048,"esp":11543,"est":111009,"ess":14653,"etr":9667,"ets":6968,"ett":13669,"ew ":6999,"eve":54896,"eva":7494,"evo":21109,"evi":14484,"euw":17939,"eur":21284,"ewe":9375,"ewo":5810,"ey ":9860,"epe":6552,"er ":333095,"epa":50447,"eor":9695,"es ":93024,"ept":12657,"erk":43542,"erl":61155,"eri":98353,"erg":28574,"erh":11977,"ere":86052,"erd":117197,"era":27762,"erb":15858,"et ":468269,"esl":15451,"esi":12100,"esc":18281,"ese":16953,"erz":13202,"erv":34773,"erw":12350,"err":17024,"ert":36463,"ers":189637,"ern":29282,"erm":20928,"erp":10525,"ero":16315,"eks":13762,"ekt":9522,"en ":1373884,"ela":20546,"eld":60315,"elf":9763,"ele":68456,"eli":59882,"elg":17311,"elk":7255,"ell":29565,"elo":7411,"els":48151,"elt":76311,"emb":35339,"ema":13115,"eme":213429,"emd":11991,"emi":9436,"ep ":11271,"ene":28524,"enh":7989,"eng":11580,"enb":15906,"ena":20359,"end":77370,"enc":8130,"eno":20895,"enn":18048,"enk":9463,"enl":7298,"eni":29460,"enw":7353,"ens":81592,"ent":262064,"enr":11987,"ego":6580,"ege":50512,"egi":74199,"eho":15902,"ek ":28609,"eis":7935,"eil":21113,"ein":30892,"eid":39967,"eig":6260,"el ":169543,"eiz":7508,"eit":10871,"eke":48960,"em ":17335,"gis":18136,"gin":20458,"gio":63608,"gie":9618,"gge":6516,"gep":6253,"gen":119618,"geo":6626,"get":5657,"ger":43727,"ges":41787,"gev":26798,"gew":8731,"gez":31827,"gee":9960,"ged":10354,"geb":49406,"geh":10722,"gem":117467,"gel":56245,"gek":5735,"gde":10792,"ge ":59115,"gd ":7871,"gaa":11487,"gan":14683,"ft ":25649,"for":15464,"fic":10500,"fie":5840,"fil":14418,"da ":11415,"de ":684651,"daa":13978,"dag":8773,"dae":7793,"dat":38507,"dan":11068,"dam":12288,"dde":16386,"cti":18928,"cte":10141,"cus":11312,"clu":9316,"co ":6846,"con":15186,"com":22591,"ct ":40340,"cea":6367,"ch ":51967,"ces":6110,"cen":17525,"cem":10124,"cha":49925,"cia":10705,"ck ":10471,"cie":34119,"che":90373,"chi":44310,"cho":18169,"cht":101737,"chr":21384,"cit":10702,"ed ":30096,"eba":5684,"ebe":6342,"ebi":14653,"ebo":12775,"ebr":30709,"eau":7092,"ei ":13562,"eft":21131,"eek":11355,"een":586471,"eel":149087,"eem":7130,"eef":23903,"eed":19633,"ees":22270,"eer":87604,"eeu":10006,"eet":5732,"edi":16531,"ede":93412,"eda":9229,"eg ":11274,"eds":11135,"edo":6698,"edr":18455,"ech":31918,"eci":6720,"ece":11419,"ee ":19989,"ef ":10385,"ect":16877,"eco":6380,"dwe":6219,"dor":15099,"doo":63254,"don":8072,"dom":7397,"ds ":31717,"dië":7942,"doc":6254,"doe":8162,"dst":15322,"dui":6443,"duc":8131,"dri":17764,"dra":19458,"dt ":39781,"dro":6043,"dsc":11185,"dse":26450,"dic":13497,"dia":8442,"der":192741,"des":13390,"dez":10434,"dec":12054,"dee":111320,"del":33701,"den":135340,"dep":43953,"di ":6337,"do ":6228,"din":21044,"dio":7262,"dis":79203,"dit":10793,"die":85548,"dig":22881,"rha":6318,"rga":14305,"ri ":27658,"rgi":6243,"rge":20743,"ret":8087,"res":26172,"rev":8098,"rdt":29462,"rg ":25954,"rea":7679,"ree":20672,"rec":16838,"red":11064,"rei":15261,"reg":76191,"ren":91873,"rek":7399,"rel":18716,"rda":12870,"rdo":6848,"rdi":16416,"rde":80930,"re ":58121,"rch":10326,"rd ":94954,"ras":7988,"rat":17581,"rbi":10377,"rbe":8163,"rag":7326,"ran":87841,"ram":11943,"ral":17891,"raa":43729,"rad":10061,"rac":17496,"rs ":114967,"rpe":6605,"ros":6652,"rot":12676,"rom":9424,"ron":74395,"roo":21289,"rop":13868,"rou":11473,"rov":34172,"rod":10405,"roc":11802,"rol":11209,"roe":27621,"rog":7078,"rno":7457,"rp ":10130,"rna":17955,"rne":14489,"rni":6410,"rmo":6164,"ro ":8067,"rma":25601,"rme":14051,"rlo":10187,"rli":43352,"rle":10232,"rla":47955,"rn ":7785,"rko":8091,"rke":15492,"rm ":8899,"rip":6607,"rio":7162,"rit":17759,"ris":26171,"riv":7770,"rig":10543,"rij":60217,"ril":12296,"rik":53920,"rin":39022,"ria":15453,"ric":60714,"rid":8288,"rie":42534,"rk ":23012,"rwe":7689,"rwi":6024,"rui":23623,"rug":6639,"rum":7291,"rus":7171,"rva":8120,"rve":6455,"rvl":18128,"rvo":6709,"ry ":9896,"rsi":7125,"rso":10142,"rsp":13174,"rsc":16580,"rse":16920,"rta":7455,"rst":29072,"rto":9036,"rte":64198,"rth":8072,"rti":18904,"rua":9937,"rts":6565,"rt ":52830,"rro":41654,"rri":6371,"rre":14498,"sam":7637,"sat":5699,"shi":8681,"sje":10417,"sie":18702,"sit":8909,"sis":17171,"sin":18433,"sig":6448,"sdi":9330,"se ":245903,"sch":188901,"sco":5845,"ser":23257,"sh ":6367,"sei":5692,"see":12427,"sep":12185,"sen":46205,"sem":40984,"sel":17905,"spo":12356,"spr":14426,"spe":34342,"spi":6360,"son":16537,"soo":14382,"st ":68174,"sla":28873,"ski":9665,"ske":6724,"sme":5826,"sse":83731,"ssa":7327,"sso":6523,"ssi":25931,"ste":153650,"sta":134037,"sto":24680,"sti":36773,"stu":49771,"str":76216,"sus":9037,"tai":6439,"tal":42563,"taa":79421,"tad":33209,"tba":18995,"tat":25594,"tar":9576,"tan":24920,"te ":245946,"ta ":14225,"pe ":6304,"par":65094,"paa":13416,"pan":11481,"pge":11800,"pec":7014,"pen":27399,"per":54393,"pes":8975,"pee":13839,"pel":26450,"pla":118636,"ple":6523,"pij":6479,"pio":11106,"pis":8712,"por":10136,"poo":7190,"pon":6968,"pol":16543,"ppe":30477,"pub":6405,"pte":13585,"pri":21984,"pre":14246,"pro":59321,"que":5834,"ra ":17662,"ngr":7754,"ngt":6149,"ngs":24028,"ni ":14308,"nge":85205,"nga":9441,"nha":5738,"nel":9354,"nen":38308,"nem":7134,"ner":92669,"net":10080,"nes":17246,"ng ":100478,"nee":12320,"nci":34445,"nce":13701,"nch":8196,"ne ":58920,"nbu":5898,"ndr":9897,"nds":53318,"ndo":10563,"ndi":65506,"nde":170234,"nda":12836,"nal":19401,"nam":19197,"nan":5995,"nad":6211,"naa":50268,"nbe":7893,"nd ":125333,"nat":22785,"na ":37268,"nwo":81256,"ny ":6026,"num":6116,"nua":11671,"nty":25239,"ntw":10449,"nto":19506,"nts":8743,"ntr":16443,"nti":22690,"nta":19603,"nte":152583,"nsu":9180,"nst":31383,"nse":133099,"nsc":14189,"nri":8599,"nt ":140657,"ns ":66627,"noe":11549,"noo":19104,"nom":9568,"nov":12875,"nne":42144,"nna":5969,"nni":9362,"nië":12014,"nla":8754,"no ":10798,"nke":14949,"nkr":6212,"nig":15471,"nie":28378,"nic":7027,"nia":7786,"nk ":5831,"niv":5902,"nis":35078,"nin":21782,"ogr":8544,"ogi":8951,"oge":15686,"oiw":7018,"ois":5765,"oir":6590,"ok ":26496,"ol ":12318,"och":15236,"ock":8061,"ode":16327,"ods":9122,"of ":51317,"oek":14333,"oel":8698,"oem":15572,"oeg":10025,"oer":17651,"oet":21912,"oen":24849,"oep":14118,"odu":7865,"oed":12901,"og ":18782,"off":7132,"ofd":11478,"oal":5868,"od ":7190,"obe":14031,"own":6969,"oud":23948,"oth":6889,"ote":16152,"ott":8284,"ots":8865,"oto":6753,"ost":29293,"ota":7234,"ose":6693,"ovi":34676,"ouw":19904,"ove":41666,"oun":29232,"ous":5891,"our":16621,"out":8877,"opp":20304,"ope":17398,"opg":12331,"os ":12938,"oon":17563,"ool":15370,"oom":10898,"ook":25159,"ooi":6071,"oof":13207,"oog":15636,"ood":7845,"or ":123656,"oot":15244,"oos":12524,"oor":222103,"oop":6797,"ork":12244,"orl":12130,"orm":37127,"orn":13425,"orp":13450,"ord":94351,"ore":21033,"org":20784,"ori":18821,"ou ":6777,"ort":47654,"ors":16087,"ot ":44568,"m² ":22639,"ora":14333,"ola":5947,"old":6867,"on ":90823,"oli":23091,"oll":13695,"olk":19645,"ole":13281,"olg":11475,"ols":13292,"olo":16700,"om ":28845,"okt":10174,"ona":29547,"ond":144852,"one":97393,"ong":27845,"oni":32623,"onn":9985,"ono":10092,"ons":14560,"ont":39227,"oma":10713,"ome":31512,"omb":6304,"omi":12146,"omm":12297,"omp":15805,"omt":12605,"oms":11302,"op ":79508,"la ":16552,"le ":71663,"lf ":5640,"lde":29685,"laa":121868,"lac":19544,"lad":5854,"lag":14890,"lai":7568,"lak":20347,"lan":133760,"lar":5948,"lat":13164,"las":13782,"ld ":39055,"lbu":9341,"kun":16374,"kwa":5842,"kte":24985,"kst":6852,"kto":10642,"lpe":6305,"ls ":61450,"lon":9738,"lom":7994,"loo":10759,"lor":5860,"loe":7778,"log":16741,"los":5784,"lië":7455,"lub":9599,"lst":25217,"lte":6994,"lse":20318,"lt ":100197,"lge":14115,"lgi":13560,"li ":13675,"lev":15162,"les":15192,"let":7945,"ler":20832,"lem":10140,"len":59840,"lei":20020,"leg":19493,"lee":17156,"led":8570,"lec":6605,"lo ":6322,"lla":18593,"lle":62567,"lli":22272,"llo":6485,"lks":5894,"lki":11362,"lm ":10134,"ll ":9099,"lit":19124,"lis":19817,"lip":6344,"lin":41169,"lic":12334,"lia":22468,"lij":95486,"lig":34482,"lie":40870,"ma ":13279,"maa":102890,"mar":12201,"mal":14307,"man":28303,"mat":12974,"md ":13491,"mbe":34190,"me ":20464,"med":7669,"mee":138208,"met":69732,"mes":6603,"mer":72670,"mel":11918,"men":146532,"mei":13406,"lve":5791,"lym":7416,"mpi":18269,"mpe":6753,"mpo":7375,"ms ":12121,"moe":5824,"mon":14109,"mt ":13658,"mst":14316,"muz":8520,"min":20688,"mil":24349,"mis":8333,"mit":6248,"mig":6106,"mie":5852,"mid":8678,"ië ":43280,"mma":6643,"mme":17999,"zui":10334,"zee":6519,"zet":5935,"zen":14435,"zel":8377,"ze ":24809,"zan":8216,"zoo":6798,"zon":12147,"zoe":7835,"zie":38234,"zic":12949,"zij":41913,"yst":6749,"ys ":5784,"ymp":7552,"wn ":6464,"wod":7070,"wor":46676,"woo":12376,"won":83488,"woi":6978,"wes":10367,"wer":76690,"wet":5976,"wen":7541,"wel":18559,"weg":14286,"wee":29661,"wit":7604,"win":7741,"wij":18720,"wat":9640,"war":13183,"was":80249,"waa":29854,"vro":6819,"vil":13878,"vin":41497,"vie":14100,"vis":16850,"vla":21113,"vli":6207,"voe":25269,"vol":30369,"von":6589,"voo":92340,"vor":15339,"ver":131519,"ven":47801,"vem":9979,"vel":12213,"vee":14817,"ve ":13676,"val":33641,"van":455616,"vat":5892,"vaa":8390,"uzi":9073,"uwe":13657,"uwd":6413,"uur":48711,"usi":5945,"use":7901,"ust":24814,"uss":30621,"ute":13524,"uw ":14870,"uto":6696,"us ":63461,"ut ":6738,"ure":14305,"urg":25394,"uri":10844,"url":29792,"uro":10683,"urt":8342,"ur ":31687,"umb":6654,"unt":32613,"uns":8285,"uni":16955,"und":14286,"um ":25713,"ult":7652,"uli":14819,"ule":6125,"ula":6204,"un ":7353,"uid":29632,"uik":15052,"uis":15533,"uit":185570,"ugu":11498,"ude":20442,"udi":6485,"ue ":5767,"uch":9124,"ub ":8370,"uar":23535,"ubl":7698,"ty ":41728,"tur":6970,"tus":27504,"tuu":41013,"tud":6157,"twi":6118,"twe":21595,"ts ":120548,"tre":20553,"tra":31398,"tri":58554,"tru":9236,"tro":22667,"tse":38669,"tsc":8839,"tst":17796,"tte":25599,"to ":11397,"toe":15554,"tob":10290,"tot":36145,"tow":6284,"ton":34234,"tor":20776,"tij":29417,"tie":72965,"tig":17350,"tit":9886,"tis":23784,"tin":21180,"tio":32273,"thu":8037,"tic":17261,"tle":5983,"tem":61681,"ten":116036,"tei":13819,"tek":11255,"tel":112278,"tee":16372,"teg":10988,"ted":9251,"th ":11037,"teu":8413,"tes":8744,"ter":133625,"tge":17358,"tho":11376,"the":28515,"tha":9247,"én ":8357,"één":7621},"n_words":[56157687,65372177,47614417],"name":"nl","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/no.json b/contrib/languages-data/no.json
new file mode 100644
index 0000000..757fc8e
--- /dev/null
+++ b/contrib/languages-data/no.json
@@ -0,0 +1 @@
+{"freq":{"D":67298,"E":31132,"F":46085,"G":30699,"A":51244,"B":53160,"C":43132,"L":37752,"M":50816,"N":44173,"O":29040,"H":46630,"I":28485,"J":19280,"K":42898,"U":16316,"T":46770,"W":15084,"V":29381,"Q":2727,"P":33476,"S":98250,"R":33889,"Y":5384,"X":3613,"Z":4254,"f":354544,"g":680290,"d":751947,"e":2892960,"b":291896,"c":77603,"a":1296256,"n":1640496,"o":1106000,"l":1037936,"m":608425,"j":148253,"k":716879,"h":226424,"i":1317183,"w":19861,"v":434834,"u":407647,"t":1430364,"s":1268259,"r":1753294,"q":3203,"p":350711,"z":18142,"y":203712,"x":10440,"²":2235,"Å":2838,"Ø":5129,"í":2375,"é":5770,"æ":27321,"å":157055,"ä":7866,"á":3631,"à":2467,"ü":2355,"ø":174400,"ö":5464,"ó":2058," l":79574," m":127350," n":66294," o":215607," h":91447," i":286253," j":14786," k":126841," d":163994," e":389283," f":217023," g":55928,"р":2932," a":164625,"с":2244," b":141242," c":8452," u":55940," t":125598," v":100006," p":120765," s":286186," r":49569," J":19242," K":42735," H":46471," I":28413," N":43960," O":28901," L":37540," M":50591," B":52872," C":42604," A":51010," F":45901," G":30503," D":67077," E":31023,"л":2294," Z":4215,"к":2313," Y":5370," X":3570,"и":3772,"о":4246,"н":2997," S":97781,"в":2674," R":33770," Q":2713,"а":4944," P":33276," W":14969," V":29296,"е":3204," U":16274," T":46489," å":24460," ø":14695," Å":2834," Ø":5122,"A ":7964,"Da":7415,"Co":15888,"Ch":7594,"Do":3375,"Dr":2174,"De":44977,"Di":3865,"Fe":2733,"Fa":3197,"Eu":2611,"Et":2524,"En":8728,"El":2986,"Ge":3683,"Ga":4272,"I ":9052,"Fy":2932,"Fr":8731,"Fo":9701,"Fl":2786,"Fj":2610,"Fi":5958,"C ":3097,"Au":3599,"Ar":6717,"As":3221,"Ba":9331,"Am":2967,"An":7679,"Al":7567,"By":3184,"Bu":3866,"Br":7415,"Ca":9043,"Bi":3113,"Be":11885,"Bo":6452,"Bl":2411,"Ku":2113,"Kr":5137,"Ko":7893,"Le":5322,"Li":8435,"La":9531,"Lu":3278,"Lo":6840,"Me":7383,"Mi":7116,"Ma":15908,"Mu":2812,"Mo":8218,"Ni":3496,"Ne":6662,"Na":6162,"No":20842,"Ol":2369,"Gr":7570,"Go":2542,"Gu":4480,"Ha":15522,"He":10062,"Hi":2665,"Ho":7455,"Hu":4099,"In":7421,"Is":2952,"Ir":2049,"Ja":4954,"L ":8381,"Je":2911,"Jo":5435,"Ju":2288,"Ka":7574,"M ":2289,"Ki":8846,"Ke":2144,"Un":4807,"W ":3224,"Ty":3546,"Tr":8937,"To":6250,"Th":6517,"Ti":3470,"Te":5634,"Ta":4262,"V ":2848,"Sy":2174,"St":18668,"Sv":4781,"Su":4159,"Wi":3769,"Wa":4239,"We":2978,"Vi":7414,"Va":4893,"Ve":9972,"Pr":5274,"S ":4165,"Pe":5166,"Pa":8226,"Po":5247,"Pi":2410,"Os":6058,"Op":2483,"Or":3270,"Se":7511,"Sc":2844,"Si":5113,"Sh":4054,"Sl":2066,"Sk":6894,"Sp":4048,"So":11528,"Ru":4355,"Sa":11798,"Re":6477,"Ri":4763,"Ro":9292,"Ra":4607,"b ":4101,"a ":140249,"Yo":2095,"Sø":4628,"Xi":2231,"bø":2634,"i ":283647,"fy":8788,"gd":9301,"ge":149531,"ga":32513,"fj":12532,"fl":14388,"ff":8428,"bå":3627,"fi":25866,"fr":47803,"fu":8120,"ft":23386,"fo":101580,"j ":2796,"gy":3318,"he":44416,"ha":66151,"gn":19968,"gl":10219,"gj":12645,"gi":33050,"gh":8862,"gg":35921,"gv":2904,"gu":9872,"gt":8325,"gs":33529,"gr":39232,"go":8838,"dt":33464,"du":13333,"dv":5679,"dy":4144,"g ":242136,"ea":17194,"eb":16045,"ec":4860,"ed":108322,"de":299418,"dd":13784,"df":2274,"di":40822,"dh":3225,"dk":3004,"dm":5071,"dl":19375,"do":17360,"dn":4555,"ds":31560,"dr":31419,"ew":4011,"ex":3115,"eu":5509,"ev":23237,"ey":5833,"fa":23302,"h ":12462,"fe":26099,"eh":7528,"eg":49132,"ef":18320,"ee":8701,"el":199057,"ek":50272,"ei":44782,"ep":20746,"eo":9834,"en":582197,"em":46141,"et":320684,"es":146652,"er":695279,"ca":10711,"e ":500246,"by":34947,"br":29081,"bu":14609,"bo":17004,"bl":54931,"bi":19197,"bb":5548,"be":63706,"db":3584,"da":47371,"f ":9303,"cu":2103,"ct":3488,"co":8095,"ck":10053,"ci":6056,"ch":18251,"ce":8996,"c ":3649,"az":2995,"ay":5287,"ba":35656,"d ":150657,"at":91048,"as":69325,"ar":172213,"av":121888,"au":14940,"ak":30193,"al":135078,"ai":12693,"aj":2677,"ao":2477,"ap":29140,"am":63029,"an":267240,"ac":9777,"ad":37337,"aa":2792,"ab":12532,"ag":40280,"ah":4661,"ae":7154,"af":16520,"nu":7442,"nt":92694,"ns":124075,"nr":3026,"no":52873,"nn":98330,"jø":13856,"ny":13664,"nv":3459,"oe":7618,"of":17045,"oc":8436,"od":26153,"oa":3647,"ob":11128,"om":188777,"on":123342,"ok":27219,"ol":75500,"oi":3014,"og":163889,"oh":4581,"ot":32564,"m²":2226,"os":28900,"ov":41178,"ou":20519,"op":34634,"oo":5024,"or":234371,"r ":604948,"ow":4407,"oy":2125,"pe":55903,"pa":27179,"pl":13813,"pn":2118,"po":25761,"ph":3721,"lä":2860,"pi":29184,"lå":4732,"læ":3598,"lo":47607,"lm":11687,"ll":125327,"ls":54920,"lp":4961,"lv":17299,"lu":16394,"lt":42371,"ly":13905,"hø":10888,"o ":38269,"ma":52812,"mb":14541,"me":153343,"mf":5780,"mk":2245,"ml":5863,"mi":35740,"mn":2923,"mm":59754,"mp":15073,"mo":24316,"mr":8156,"mt":8238,"ms":13233,"mu":35413,"my":4118,"p ":18651,"na":63523,"nb":10179,"nc":6842,"nd":155382,"ne":210727,"nf":8666,"ng":145804,"nh":6700,"ni":70736,"nj":3359,"nk":18937,"nl":14559,"nm":3689,"ju":8303,"jo":49289,"kj":21128,"gå":9251,"ki":30043,"kh":3599,"ke":149522,"ka":75120,"m ":148254,"fø":39544,"ky":6546,"ks":31928,"kt":45246,"ku":18940,"kv":7398,"ko":71373,"kr":37123,"kk":42089,"kl":19159,"km":10210,"kn":9496,"li":148110,"lh":5181,"lk":27090,"lj":4011,"le":206454,"ld":27006,"lg":10977,"lf":6606,"la":117768,"lb":9872,"n ":508629,"hr":3491,"hv":8983,"ht":2318,"hu":13547,"hj":3760,"hi":16514,"hn":2241,"ho":28328,"id":53829,"ic":16512,"ib":6764,"ia":42013,"ig":91825,"if":12391,"ie":49167,"hy":2076,"k ":129587,"dø":14496,"ir":32357,"is":145060,"it":66902,"iu":3847,"iv":28029,"ii":2629,"ij":2484,"ik":79231,"il":114997,"im":14324,"in":224355,"io":25793,"ip":8856,"je":54644,"ji":3380,"l ":114552,"ja":10997,"xi":3017,"tå":6348,"z ":3331,"sø":16304,"så":13214,"wi":2306,"y ":43971,"rø":14616,"wa":4985,"we":2854,"vh":2078,"rå":14698,"vi":56505,"vt":2733,"vs":5579,"vn":14394,"vo":12525,"ve":142305,"vd":3441,"va":71132,"x ":3406,"ui":4966,"uk":21664,"ul":26959,"ue":11327,"uf":3039,"ug":10093,"ur":41090,"us":50014,"ut":47498,"um":19336,"un":102304,"up":9818,"ty":27626,"tu":28284,"tt":93499,"tv":14967,"ub":13117,"ua":11212,"ud":11995,"uc":2853,"w ":4541,"to":63371,"tn":11784,"tm":2957,"tl":16216,"ts":42441,"tr":65044,"tg":7470,"tf":6385,"te":283104,"td":2594,"tk":2587,"tj":4066,"ti":143857,"på":54079,"th":14355,"v ":99425,"tb":9067,"ta":95087,"su":13370,"sv":19448,"ss":49933,"st":245546,"sy":11979,"sl":28878,"sk":191864,"sn":6441,"sm":11084,"sp":37365,"so":108796,"sr":4235,"sd":5831,"sc":5435,"sf":8108,"se":148118,"sh":13228,"sg":2307,"sj":47719,"si":74309,"nø":2380,"u ":10252,"sa":44310,"sb":10119,"rr":19009,"rs":78883,"rt":82606,"ru":50990,"rv":11610,"ry":11156,"rp":5076,"ro":69758,"rn":31836,"rm":21775,"rl":20062,"rk":47716,"næ":4051,"ri":132605,"nå":4464,"rh":6919,"rg":35522,"rf":12509,"re":256059,"rd":66008,"rc":4262,"rb":18116,"ra":128776,"t ":465642,"mø":2258,"qu":2140,"må":7691,"lø":8427,"s ":146195,"pt":7260,"pu":11071,"pp":34390,"pr":51296,"ps":7773,"vå":3732,"za":2150,"væ":7493,"yg":16794,"ye":16616,"yd":10423,"ya":7947,"tø":14974,"yt":13580,"ys":21183,"yr":13910,"yp":7213,"yn":9715,"ym":5956,"yl":14093,"yk":8767,"yi":4209,"² ":2226,"å ":86331,"Øs":2809,"ær":24440,"åp":3244,"ån":3530,"åt":7047,"ås":2737,"år":24111,"åe":2573,"ål":6435,"åk":4798,"åd":11397,"än":3083,"øy":21642,"ør":54354,"øs":14419,"øp":5028,"øv":7903,"øt":3062,"øk":5634,"øn":8821,"øl":4965,"øm":4740,"øe":2478,"ød":34288,"ø ":2765,"之":2379,"专":2738,"三":3026," Ga":4247," Ge":3655," Fy":2931," I ":6572," Fo":9668," Fr":8721," Fi":5928," Fl":2776," Fj":2609," Ha":15503," He":10039," Go":2524," Gr":7534," Gu":4458," Hu":4096," Ho":7447," Hi":2663," Je":2895," Ja":4943," Is":2884," In":7378," Ka":7550," Ke":2113," Ki":8813," Jo":5426," Ju":2284," La":9489," Le":5267," Li":8383," Ko":7880," Kr":5128," Ku":2108," Ma":15819," Mi":7093," Me":7355," Lo":6813," Lu":3268," Ne":6630," Na":6059," Ni":3491," Mo":8183," Mu":2787," Am":2962," An":7665," Al":7546," Ba":9272," Au":3596," As":2828," Ar":6669," Be":11852," Bi":3097," Bl":2409," Bo":6415," Br":7392," Bu":3840," By":3181," Ca":8810," Ch":7563," Co":15800," Da":7402," Di":3846," De":44915," Dr":2166," Do":3289," El":2979," Et":2521," En":8708," Eu":2608," Fe":2724," Fa":3178," Xi":2228," Sø":4627," Wi":3744," We":2962," Wa":4211," Yo":2085," Os":5937," Or":3262," Op":2481," Po":5205," Pi":2406," Pe":5104," Pa":8187," No":20722," Ol":2367," Ra":4586," Ro":9267," Re":6462," Ri":4754," Pr":5244," Sy":2169," Sv":4778," Su":4150," St":18500," Ta":4246," Th":6493," Ti":3455," Te":5588," Tr":8905," To":6181," Ru":4349," Sa":11773," Sh":4026," Si":5095," Sc":2823," Se":7484," So":11504," Sp":4034," Sk":6885," Sl":2061," Va":4882," Ve":9942," Vi":7397," Ty":3541," Un":4794," ja":4722," in":34065," ik":4460," is":2212," ka":20700," fø":33238," kj":12239," gå":5771," ki":10103," jo":2540," ju":4702," ha":40329," he":11962," gi":4943," gj":7272," gr":19932," gu":2109," dø":7855," hi":3888," hj":2877," ho":10427," hu":4249," hv":8245," ne":7120," na":12532," my":2654," mu":6899," mo":13920," ok":2382," ol":3868," om":17822," og":141787," of":9223," ob":2162," ny":3021," no":33974," le":11218," li":27765," la":25410," kv":4363," ku":8567," km":9490," kl":4277," kr":10076," ko":42531," me":70036," mi":10095," hø":6629," ma":17890," lu":2098," lo":3520," ad":3981," am":9313," an":19228," ap":2222," ak":3076," al":9858," av":89135," au":3693," ar":13091," at":7530," ba":13278," bi":6342," be":29509," bo":6968," bl":40680," by":21015," bu":2338," br":16487," ca":3932," er":153435," et":63825," en":123670," ei":4836," el":30282," ek":4165," eg":3293," fe":8817," fa":9591," fu":3699," fr":43218," fo":78205," fl":9728," fj":6013," bå":2550," fi":13808," ge":3067," ga":9278," i ":237285," fy":7547," da":14064," do":2621," dr":4255," de":122624," di":7830," dy":2072," væ":4635," sø":12596," ru":9067," sa":19419," se":29156," sj":3216," si":17713," sl":7113," sk":21745," sp":19430," so":87507," ra":3039," re":25505," ri":2962," nå":3726," næ":2896," ro":6266," pu":2169," pr":27989," lø":2099," må":3587," ov":11878," op":17267," or":7497," pe":8091," pa":7750," pl":5274," po":9080," pi":4926," lä":2565," så":2458," va":41741," ve":33600," vo":3210," vi":14710," ty":6900," tu":3382," ut":27318," un":21502," ta":8589," sy":5687," st":43167," sv":7377," su":2425," tr":14296," to":9228," th":2455," på":53815," ti":62638," te":14306," Øs":2807," å ":15026," år":6691," øs":6280," øy":4235,"Fin":2794,"Eur":2192,"En ":4024,"Eng":3095,"Fyl":2572,"Fra":3815,"Fre":2185,"Fol":3808,"For":3758,"Hel":2604,"Han":5694,"Har":2577,"Ind":2121,"Øst":2801,"Bar":2295,"And":2152,"Car":2172,"Ber":4934,"De ":3390,"Det":18734,"Den":19203,"Dan":2408,"Cha":2366,"Cou":8345,"OL ":7076,"New":2594,"Nor":18501,"Osl":4630,"Par":3554,"Pro":2168,"SA ":4753,"Joh":2414,"Kar":2117,"Kin":6314,"Kon":2546,"Kom":2130,"Kri":2217,"Lan":3221,"MW ":2903,"Lon":2680,"Man":2484,"Mar":5164,"Sør":3934,"Sve":3734,"Str":2300,"Sto":5079,"Sta":5059,"Ste":2680,"Sko":2364,"Som":4932,"Rus":2959,"San":3826,"än ":2530,"Rom":2726,"åde":9646,"ål ":2493,"Vei":2618,"åle":2191,"Vin":2983,"åre":3629,"ård":2572,"år ":14295,"Ves":3148,"ått":3275,"Uni":3264,"Tys":2479,"ær ":4620,"ært":3809,"ære":8422,"The":4012,"Tro":3352,"bis":3511,"bil":4902,"bin":5018,"blo":2551,"ble":31774,"bli":12888,"bla":6396,"bok":3234,"bor":4144,"bbe":3093,"ban":9995,"bak":3210,"bal":7414,"bas":4309,"bar":6636,"bei":5432,"beg":3804,"ber":17512,"ben":4646,"bel":3914,"bev":2185,"bes":8730,"bet":9507,"ca ":4720,"ce ":3774,"bri":5623,"bra":2313,"bre":4128,"bru":13492,"bur":2443,"bun":2536,"bum":3444,"by ":8318,"byd":2738,"bye":5451,"byg":13374,"byp":2127,"am ":5629,"ake":6088,"al ":22413,"ain":3128,"ak ":2688,"agt":4288,"anu":3347,"ano":2720,"ann":21260,"ant":16537,"ans":34997,"ane":10156,"ang":34028,"ani":10694,"ank":5395,"anl":5003,"ap ":5392,"ana":7215,"anc":2796,"and":71633,"amt":3120,"amm":13306,"aml":4533,"amp":3404,"ami":6970,"ame":12312,"amb":2328,"ama":4584,"alv":2901,"alt":13338,"als":4480,"all":28689,"alg":3345,"ali":12960,"ald":4724,"ale":19089,"ala":6862,"alb":4116,"an ":35027,"aks":3242,"akt":7856,"akk":3187,"abl":2978,"ae ":2689,"ad ":7446,"aft":9172,"afi":2584,"ai ":3427,"aga":2271,"age":11424,"adm":2981,"adi":6411,"add":5083,"ade":7303,"ag ":10292,"ach":2606,"ada":2399,"at ":12379,"are":11421,"ard":7601,"arb":5329,"ara":7138,"arn":4304,"arm":4027,"arl":3706,"ark":12605,"ari":9703,"arr":4082,"ars":6026,"art":19222,"asi":5334,"ase":4616,"ask":2682,"asj":20288,"ar ":65150,"apa":4754,"ape":8916,"app":2255,"as ":8910,"ava":2465,"aut":2096,"avn":11841,"avi":3695,"ave":7468,"ay ":2288,"av ":85297,"ata":4831,"ast":9209,"ass":12223,"ato":4866,"ate":21584,"ati":14442,"att":15952,"ats":2979,"atu":5294,"aug":3319,"jer":9769,"jek":2468,"jel":9659,"jem":3205,"jen":18638,"jan":3394,"je ":3919,"jon":34428,"jor":11663,"itu":2594,"itt":16474,"its":2537,"ity":2756,"isk":51560,"isj":4819,"ism":2335,"iss":5696,"ist":35845,"iv ":3591,"ita":7242,"ite":15976,"iti":9684,"iva":2856,"ivi":3570,"ive":13599,"ipp":2048,"is ":12915,"ion":14718,"ir ":6919,"irk":9296,"isi":4522,"ish":2413,"ise":12425,"isa":4298,"ire":7035,"it ":2143,"kil":3815,"kk ":10113,"kin":8354,"kip":2377,"kir":5305,"går":7228,"kis":2484,"kje":16281,"km ":7242,"kel":6677,"ken":15267,"kes":10195,"ker":29257,"ket":9398,"ke ":65844,"kra":11143,"kre":8431,"kt ":12793,"kse":4519,"kro":2450,"kri":13226,"kot":2092,"km²":2186,"kor":5874,"kon":13827,"kom":31756,"kol":9129,"ks ":3032,"kny":2137,"kjø":3238,"kni":5015,"kke":25288,"klu":3574,"kle":4386,"kla":6195,"kli":3140,"jul":2456,"kat":3942,"kar":4133,"kas":2638,"kap":17702,"kan":20091,"kal":14105,"kam":2092,"ka ":6242,"før":14702,"fød":22348,"føl":2302,"ha ":2473,"ham":3082,"han":12460,"hal":3654,"hav":6160,"har":25645,"had":4958,"he ":6327,"hel":5939,"hei":3009,"het":10209,"her":5545,"hen":7004,"hin":2784,"his":4361,"hje":2935,"gle":2190,"gn ":3125,"gla":3596,"gjø":2529,"gni":2494,"gne":11554,"gs ":3264,"gru":12215,"gra":9273,"gt ":5657,"gre":14854,"gst":4516,"gså":9706,"gus":2418,"ial":4669,"ian":8957,"øke":2125,"ølg":2438,"id ":6821,"ibe":2226,"ia ":20599,"iet":4979,"iel":5273,"ien":17663,"ier":7794,"ies":2351,"ig ":24484,"ift":6890,"ør ":13261,"øpe":2597,"ømm":2930,"ich":3831,"ie ":7138,"ica":2979,"ids":3533,"idr":3478,"ønn":3150,"idl":8890,"ide":18687,"idd":2770,"ida":3822,"ønd":2352,"øst":12196,"il ":45404,"im ":2951,"ika":14540,"ige":27059,"iga":2145,"igh":5558,"igi":2075,"igg":17522,"igs":3095,"ign":3006,"øre":10636,"ørs":12878,"ørt":2899,"ik ":6168,"ime":2903,"ind":10332,"ina":14021,"inn":35697,"ino":2582,"int":10160,"ins":19112,"ine":16471,"ing":71844,"ini":7560,"ink":2146,"iod":2244,"iny":3773,"ikl":3957,"ikk":22590,"ike":13916,"in ":20575,"ikt":9421,"iks":3390,"ilo":2911,"ill":23109,"ilk":2217,"øve":6392,"ilm":4862,"ilh":3403,"ili":8732,"ild":3649,"ile":4123,"io ":2961,"ils":3090,"ilt":4137,"ørø":2134,"øy ":5016,"hol":9462,"hov":6717,"øye":5083,"øya":3991,"hvo":3807,"hun":2960,"hus":4616,"hve":2835,"døs":2243,"død":8337,"ffe":3959,"ffi":2131,"fes":3648,"fer":4921,"fem":2188,"fen":2692,"fek":3989,"fel":4217,"fat":5769,"far":3333,"fam":3942,"fal":2369,"eta":7185,"ete":19743,"eti":2646,"etn":3543,"esp":3357,"eso":2351,"est":46619,"ødt":21901,"ess":8652,"esv":2859,"etr":2759,"ets":9941,"ett":37893,"ety":3237,"ew ":2555,"eve":11454,"eva":2230,"evi":2446,"ey ":3642,"er ":431766,"epa":2788,"eor":2578,"eol":2061,"ød ":7105,"es ":40484,"ept":2696,"epu":5041,"epr":2644,"erk":15694,"erl":6414,"eri":32585,"erg":11340,"erh":2771,"ere":60500,"erf":5751,"erd":9568,"era":10673,"erb":3057,"et ":226950,"esk":8307,"esi":10382,"øde":3134,"ese":10631,"erv":4618,"eru":2639,"err":6355,"ert":31938,"ers":27995,"ern":17742,"erm":4003,"ero":2745,"ekk":7221,"ekn":2204,"ekr":3313,"eks":10659,"ekt":15683,"en ":368973,"ela":6680,"eld":5215,"ele":18548,"eli":14582,"elg":2392,"ell":56973,"elo":2842,"elv":10426,"els":37467,"elt":17365,"emb":7048,"ema":3872,"eme":6179,"emm":4042,"emo":2614,"emi":3720,"emp":3266,"ems":2210,"enf":4002,"ene":43923,"enh":3807,"eng":13869,"ena":2893,"end":24201,"enn":18704,"enk":3130,"eni":5336,"ens":38448,"ent":42518,"egn":9280,"ege":7883,"egg":4893,"egi":7310,"eha":2568,"egr":3736,"eis":4556,"eim":2591,"ein":6378,"eie":6921,"eid":6829,"el ":17607,"eke":3082,"em ":6695,"gje":8413,"git":6162,"gis":5345,"gin":2581,"gio":4769,"ghe":4290,"gge":29980,"gi ":3980,"gen":31946,"get":10089,"ger":52848,"ges":4527,"gg ":2739,"gel":10922,"gde":3615,"ge ":33172,"gas":3030,"gar":3339,"gat":3035,"gam":2397,"gal":2375,"gan":11800,"ga ":3023,"fyl":7035,"fte":8757,"fun":3666,"ftv":7006,"ft ":3864,"fra":35021,"fre":5013,"fri":6048,"for":85448,"fot":5733,"fol":6844,"fle":5300,"fly":4535,"fil":5351,"fik":2963,"fin":6087,"fir":2657,"fis":4937,"fje":5241,"fjo":6945,"da ":8120,"de ":60427,"dal":11372,"dag":6176,"dat":3189,"dar":2553,"dan":8876,"dde":9644,"co ":2205,"ch ":3122,"cha":2322,"ck ":3691,"che":4052,"chi":2081,"cke":2693,"ed ":55809,"eba":2196,"ebe":2240,"ebr":3393,"eal":3627,"eat":2589,"efi":2051,"efo":4434,"efe":4659,"ei ":7806,"een":2210,"edl":3176,"edi":4356,"ede":17856,"eda":3620,"eg ":6746,"eds":6107,"edr":4047,"dve":3394,"dus":5910,"don":4573,"dom":4754,"ds ":5042,"dmi":3335,"dni":2776,"dst":4471,"dte":2377,"duk":4468,"dri":5473,"dra":3627,"dt ":28714,"dre":18932,"dsk":5623,"dia":3579,"der":53613,"des":6374,"det":47571,"del":41297,"dek":2450,"den":73325,"dem":4267,"di ":2586,"dle":5037,"dla":2367,"dli":11521,"din":4634,"dio":3152,"dis":11321,"die":2789,"dig":4896,"rga":6400,"ri ":3980,"rgi":2280,"rge":14583,"ret":27902,"res":20361,"rev":6468,"rfa":4418,"rds":3327,"rdv":2179,"rg ":6581,"rea":5698,"ree":2115,"ref":5778,"red":9285,"rei":3228,"reg":12569,"rem":5839,"ren":35981,"rek":8172,"rel":4778,"rer":20318,"rep":8966,"rda":5684,"rdl":3552,"rdi":4958,"rde":19629,"re ":73979,"rbu":2376,"rd ":17240,"ras":7043,"rat":9184,"rav":3015,"rbi":3863,"rbe":5664,"rag":2711,"ran":19588,"ram":7456,"ral":8750,"rak":4191,"raf":12560,"rad":5981,"rs ":8675,"ros":5462,"rom":6997,"ron":6959,"rop":5525,"rov":9073,"rod":9193,"rol":4308,"rof":2729,"rog":4064,"rna":5941,"rne":13070,"ro ":3052,"rma":5722,"rme":6882,"rli":7449,"rle":2859,"rla":5982,"rn ":5061,"rks":2379,"rko":2612,"rki":3108,"rke":16934,"rka":2079,"rm ":3363,"nær":3936,"rio":3486,"rit":8311,"ris":17487,"riv":5194,"rig":8523,"ril":2995,"rik":24390,"rin":21593,"rim":2649,"ria":5352,"ric":2697,"rid":3352,"rie":12327,"rif":3440,"rdø":2401,"rk ":15343,"rup":6154,"run":12301,"rum":4066,"ruk":11189,"rus":6418,"rva":2898,"rvi":2442,"rve":4784,"ry ":3420,"rsk":24627,"rsi":6764,"rso":5206,"rse":3955,"rta":2228,"rst":17535,"rte":20285,"rti":8810,"rua":2277,"rts":5729,"rt ":35053,"rri":3179,"rre":7643,"rra":4011,"sak":3680,"sal":2138,"sam":15315,"sbe":2623,"san":6993,"sat":4359,"sas":3809,"sa ":2445,"ryk":2399,"sha":2088,"shi":3201,"sje":5058,"sjo":34525,"sie":4330,"sid":8350,"sia":4578,"sk ":75137,"sit":9775,"sis":16636,"sin":9850,"sik":9289,"sda":2876,"sby":2166,"se ":19672,"sch":2395,"ser":35054,"ses":4849,"set":9508,"sfo":2877,"seg":6362,"sep":3173,"sen":41736,"sem":4309,"sel":10690,"sek":3507,"spo":8496,"spr":5617,"spe":5862,"spi":12660,"spa":2392,"sol":2063,"som":85971,"son":9679,"sor":2986,"sjø":3410,"st ":39863,"ss ":6097,"sli":3561,"slo":6037,"slu":2332,"sky":2874,"sla":9825,"sle":4538,"ski":8246,"skj":4074,"skl":2812,"sko":10936,"skr":10481,"sku":3651,"ska":22269,"ske":46720,"sma":2688,"sme":3758,"stå":5531,"stø":6651,"syn":2627,"sys":2439,"syk":3044,"sse":15330,"ssa":2824,"sso":3173,"ssl":2943,"ssk":3059,"ssi":8369,"sst":2689,"ste":66334,"stf":2230,"sta":38189,"stn":2344,"sto":16688,"sti":20167,"stl":6400,"stu":3784,"str":25344,"sty":4151,"sun":2773,"sut":4111,"sva":4458,"sve":9604,"svi":2603,"tak":2812,"tal":21730,"tab":3588,"tad":7590,"tba":6294,"tav":2813,"tat":16792,"tas":5433,"tar":7233,"tan":11218,"tam":2411,"te ":58351,"ta ":8154,"pa ":2052,"pe ":5528,"par":10024,"pas":4090,"pan":5764,"län":2721,"pen":10042,"per":17920,"pet":8954,"pes":3294,"pel":3770,"pla":9218,"lær":3470,"pil":11406,"pin":7171,"pis":4864,"por":10569,"pol":7076,"ppr":5964,"ppl":2678,"ppe":9671,"pp ":3799,"pub":5466,"pte":2999,"pri":10360,"pre":12096,"pro":23686,"prå":2961,"løp":3837,"mål":4336,"ra ":37023,"ngi":2137,"ngl":4160,"ngr":2861,"ngt":2215,"ngs":15959,"ni ":4401,"nge":45140,"nga":2868,"ngd":4050,"nhe":2871,"nel":12450,"nen":27435,"ner":33385,"net":21165,"nes":25627,"ng ":54251,"ned":4860,"nfo":4282,"nce":2258,"ne ":72733,"nby":5798,"ndt":4870,"ndr":12413,"nds":12659,"ndo":5458,"ndl":2779,"ndi":6968,"nde":56337,"nda":5748,"nal":12130,"nan":2828,"nar":2470,"nad":2448,"nd ":38901,"nav":9346,"nat":5154,"nas":7061,"na ":15576,"nyi":3711,"nua":2135,"nty":8509,"nto":3515,"ntr":8273,"nti":5299,"ntl":2516,"nta":6663,"nte":28676,"nsp":2257,"nst":14726,"nss":3014,"nse":21534,"nsj":2053,"nsi":3034,"nsl":2182,"nsk":39260,"nsa":2994,"nt ":24653,"ns ":20153,"noe":3540,"nom":6845,"nor":29131,"nov":2911,"nne":42931,"nna":2105,"nnb":5914,"nnl":4919,"nno":3889,"nni":5129,"nnk":2501,"nns":7691,"nma":2510,"nli":4969,"nn ":14981,"nla":6112,"nle":2679,"no ":2847,"nke":3676,"nkr":4191,"nia":6220,"niv":5311,"nis":15475,"nit":2397,"nin":25664,"nik":2124,"ogs":10014,"ogr":5237,"ogi":5246,"ogn":3334,"oge":2102,"ok ":7509,"ol ":2411,"ock":4853,"ode":7882,"of ":4012,"oen":3787,"odu":8565,"og ":134463,"oft":3759,"off":4398,"ofe":2299,"od ":2196,"obl":2263,"obe":3133,"nyt":4280,"jøe":2169,"jør":5841,"ote":3613,"ott":4146,"ots":2202,"oto":3139,"ost":4414,"otb":5531,"osi":2968,"ose":3901,"oss":3129,"ovi":8837,"ove":25154,"oun":9480,"our":3311,"opp":20548,"ope":3654,"opa":2135,"os ":5236,"or ":56677,"ork":4146,"orl":2445,"orm":10647,"orn":3102,"orr":3175,"ord":42679,"ore":13522,"orf":4509,"org":16810,"ori":9530,"ort":20720,"ors":26414,"m² ":2219,"ot ":7999,"orb":6069,"ora":3709,"ola":2381,"old":10833,"on ":35130,"oli":8827,"oll":6799,"olk":9387,"ole":9725,"ols":3475,"olm":2556,"olo":8072,"oly":3133,"oka":3954,"om ":107366,"okk":3118,"oks":3517,"okt":2231,"ona":9739,"ond":6224,"one":22668,"ong":10465,"oni":5288,"ono":3591,"ons":14426,"ont":6196,"oma":5119,"ome":6397,"omb":2444,"omi":3410,"omf":2921,"omm":37033,"omp":3178,"omr":6915,"oms":5919,"la ":5793,"le ":57070,"lde":10166,"lds":2282,"lad":2570,"lag":17793,"lan":58009,"lar":3088,"lat":7640,"las":9906,"lav":2641,"lba":2444,"ld ":6657,"lbu":3945,"kvi":3664,"kva":2672,"kur":2362,"kun":5248,"kul":5655,"kte":10988,"kst":5101,"ksj":6623,"ksi":2580,"ktu":4360,"kti":6575,"kto":3659,"kys":2223,"ls ":2559,"lok":3826,"lon":2509,"lom":14627,"log":6983,"los":2313,"lov":2717,"lme":3805,"lto":5869,"ltu":2394,"lub":3807,"lsk":13894,"lst":8591,"lv ":4083,"lta":2646,"lte":5587,"lse":19815,"lt ":21046,"lge":4495,"li ":4167,"lev":4182,"les":12321,"let":13065,"ler":55284,"lem":5548,"len":25387,"lek":9666,"lei":2319,"leg":6026,"led":5805,"lls":3758,"lo ":5941,"lhø":2797,"lla":5878,"lle":65120,"lli":10408,"llk":3343,"llo":12212,"lke":16528,"lkn":2938,"lm ":3452,"lje":2230,"ll ":17353,"lit":14639,"lis":9353,"lir":3649,"lin":18549,"liv":2248,"lia":5498,"lik":12256,"lig":60844,"lie":7273,"ma ":3762,"mai":2118,"mar":11919,"mas":4336,"mal":3128,"man":16587,"mat":5209,"mbe":7011,"me ":8290,"med":37095,"met":13813,"mes":7734,"mer":33081,"mel":15249,"men":33002,"mfa":2204,"lva":3696,"lve":4358,"lut":2852,"ly ":2856,"lym":3245,"lys":2102,"høy":5492,"hør":3562,"mpi":3576,"mpe":4413,"ms ":2196,"mod":2416,"mon":3257,"mor":2125,"mot":10268,"mt ":3487,"mst":3459,"mrå":7270,"mus":6700,"mun":25496,"min":11788,"mil":8071,"mis":4562,"mid":4108,"mle":3222,"mmu":24025,"mme":31877,"vær":6865,"ytt":5943,"yte":2764,"yst":6626,"ysk":7598,"yrk":2373,"yre":3443,"ypr":2466,"yr ":3697,"ype":2571,"ye ":2367,"yde":4954,"yer":2533,"yen":9116,"ya ":4898,"ykk":3634,"ylk":9083,"ymp":3267,"ygg":10777,"ygd":2788,"yin":3851,"tør":7504,"tøv":4228,"tår":3969,"sør":11014,"så ":11194,"røs":2166,"røn":4125,"røy":2397,"vir":2566,"råk":3246,"vik":8483,"vil":4773,"vin":16505,"vid":3526,"råd":8908,"vit":4787,"vis":10758,"vn ":5466,"vne":6818,"vol":2228,"vok":2535,"vor":4361,"ver":56905,"ves":12440,"vet":7953,"vei":7599,"veg":2114,"ven":15144,"vem":2207,"vel":5384,"ved":23018,"ve ":6401,"val":6677,"van":11534,"var":40025,"vat":3521,"va ":5156,"utø":4075,"usi":6124,"use":8793,"ust":8108,"uss":8811,"utg":6539,"ute":5272,"utt":5236,"uts":3096,"utv":4169,"us ":10711,"ut ":5919,"ure":6307,"urg":2892,"uri":2750,"urn":3039,"uro":3768,"urr":2556,"ur ":5709,"upp":6292,"ume":4177,"unt":9250,"uns":3775,"unk":3135,"uni":6920,"unn":14136,"und":31193,"ung":4670,"une":22585,"uks":4735,"ukt":5470,"uke":5765,"um ":8855,"ult":4324,"ull":5554,"uli":4543,"ule":2555,"un ":3708,"ugl":2152,"ugu":2387,"ude":2985,"udi":2221,"ue ":2256,"ues":2288,"uar":4525,"uan":2334,"ubl":5816,"ubb":3409,"ud ":2293,"typ":2478,"tyr":5657,"tys":4599,"ty ":11596,"tve":8575,"tvi":4656,"tur":14995,"tus":2293,"tun":2106,"tud":2813,"ts ":7521,"tre":18842,"tt ":26597,"tra":17913,"tri":10126,"tru":4763,"tro":7380,"try":2883,"tse":4787,"tsk":2860,"tsu":3903,"tst":7808,"tta":2344,"tte":44536,"tti":2906,"tts":8940,"to ":8546,"tni":6514,"tne":2955,"tob":2085,"ton":7030,"tok":6614,"tol":4307,"tor":21379,"til":54443,"tik":6372,"tif":3581,"tie":3348,"tig":5426,"tit":4010,"tis":16494,"tin":10509,"tio":5365,"tia":3220,"tid":14688,"tiv":7883,"tje":3021,"tli":7658,"tla":4715,"tle":2120,"tem":9786,"ten":44932,"tei":3878,"tek":5962,"tel":9339,"teg":7791,"ted":11925,"tfo":3043,"th ":3609,"tet":33199,"tes":6700,"ter":82933,"tgi":4175,"på ":52716,"ti ":4056,"the":4578},"n_words":[20399254,23799460,17069273],"name":"no","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/pa.json b/contrib/languages-data/pa.json
new file mode 100644
index 0000000..3699c75
--- /dev/null
+++ b/contrib/languages-data/pa.json
@@ -0,0 +1 @@
+{"freq":{"ਾਜ਼":17,"ਾਜਧ":12,"ਾਜਾ":13,"ੀਲ ":18,"D":18,"E":20,"F":29,"G":23,"A":52,"B":35,"C":59,"L":38,"M":47,"N":37,"O":37,"H":31,"I":60,"U":14,"T":45,"W":18,"P":47,"S":55,"R":24,"f":103,"g":125,"d":221,"e":655,"b":81,"c":194,"ੀਰ ":33,"a":561,"n":492,"o":398,"l":256,"m":260,"j":16,"k":37,"h":252,"i":537,"w":71,"v":42,"u":236,"t":467,"s":328,"r":400,"p":124,"z":13,"y":106,"x":19,"ਿਆ।":22,"ਾਤਰ":17,"ਾਤਾ":27,"ਾਤੀ":12,"ਿਆਣ":31,"ਿਆਦ":20,"ਿਆਨ":47,"ਿਆਲ":13,"ਿਆਰ":28,"ਿਆਵ":14,"ਿਆਸ":12,"ਾਣਕ":12,"ਾਣੂ":104,"ਿਆਂ":76,"ਾਣੀ":40,"ਾਣਿ":27,"ਾਣਾ":22,"ਾਨੇ":13,"ਾਨਾ":31,"ਿਊਟ":14,"ਾਨੀ":48,"੩੧ ":18," । ":128,"੩੦ ":23,"ਾਦੀ":22,"ਾਦਾ":19,"ਾਨਕ":26,"ੁਝ ":21,"।ਇਸ":14,"ium":27,"is ":22,"ion":36,"ੀਤ ":15,"਼੍ਰ":22,"੦੦੮":24,"ੀਪ ":375,"ੀਨ ":57,"ੀਮ ":51,"ਟੀ ":57,"ਜੋਂ":19,"ਾਗਰ":20,"ਾਕੀ":370,"ਾਕਿ":34,"ਟਾ ":22,"੨੩ ":13," m":24," o":57," h":17," i":57," d":21," e":15," f":30," a":133," b":21," c":34," t":120," w":40," p":41," s":54,"੨੨ ":14," r":16,"੨੧ ":14,"੨੫ ":14,"਼ਸੀ":29," H":25," I":54," N":32," O":31," L":21," M":43,"੨੪ ":13," B":32," C":54," A":46," F":17," G":21," D":15," E":17,"ਿਰ ":73," S":47," R":22,"਼ਹਿ":50," P":44," W":15," T":36,"ਜੀਵ":20,"੨੭ ":13,"੨੬ ":13,"ਿਲ ":25,"ਜੁਲ":54,"ਜਿੰ":30,"੨੯ ":16,"ਜੂਨ":35,"਼ਾਂ":36,"਼ਾਹ":32,"੨੮ ":15,"਼ਿਆ":14,"ੀਕ ":34,"ਜ਼ਮ":12,"਼ਿਲ":75,"ਜ਼ਾ":33,"ਜ਼ਿ":88,"ਾਂਦ":190,"ਜਾਂ":231,"ਜ਼ੀ":72,"ਾਂਤ":20,"ਾਂਸ":16,"ਜਾਬ":125,"਼ੁਰ":27,"ਜਾਤ":13,"ਜਾਣ":55,"ਜਾਦ":21,"ਜਿਮ":22,"ਜਿਲ":12,"ਜੀਅ":19,"ਿਸ ":72,"ਾਅਦ":30,"ਜਿਸ":70,"ਜਿਹ":19,"ਾਇਆ":59,"ਾਇਣ":62,"੨੦ ":14,"ਾਇਲ":17,"ਟਰ ":67,"ਾਈਟ":14,"ਾਈਡ":27,"ਾਉਂ":14,"ਾਈਨ":29,"ਾਉਣ":23,"਼ਬਦ":26,"ੀਂ ":52,"ਿਤ ":82,"ਟਨ ":17,"ਚੰਦ":21,"੧੮ ":12,"ਛੋਟ":13,"੧੯ ":14,"ੀਆ ":53,"ਿਨ ":1101,"਼ਰਵ":39,"ਜਨਮ":32,"ਜਨਵ":39,"ਜਨਸ":13,"ਜਦੋ":15,"ਜਧਾ":12,"ਿਬ ":69,"ੀਤਾ":41,"Co":14,"ੀਤੀ":42,"ੁਆਰ":46," In":28," Ma":13,"he ":69,"ਾੜੀ":12,"ੀਟਰ":38,"Ma":13,"Ol":12," Co":14,"In":29,"L ":13,"।":1840,"ੀਕਲ":22,"ੂਪ ":25,"ੀਕਨ":13,"ਿਲੀ":21,"ਿਲਾ":83,"Th":15,"ਿਲ੍":13,"ਿਲੋ":46,"ਿਲੇ":38,"ੂਨ ":35,"ਿਲਦ":12,"ਵ":3164,"ਲ":4004,"ਰ":5772,"ਿ":5178,"ਾ":10004,"਼":1111,"ਹ":4455,"ਸ":5034,"ਦ":5926,"ਧ":297,"ਤ":3377,"ਥ":271,"ਢ":42,"ਣ":777,"ਠ":69,"ਡ":853,"ਮ":2207,"ਯ":188,"ਬ":2031,"ਭ":430,"ਪ":1926,"ਫ":303,"ਨ":4410,"ਕ":3798,"ਗ":2047,"ਖ":660,"ਐ":62,"ਓ":66,"ਝ":61,"ਜ":2041,"ਟ":625,"ਘ":134,"ਛ":70,"ਚ":1522,"ਅ":1321,"ਆ":1270,"ਇ":2053,"ਂ":2880,"ਏ":233,"ਈ":490,"ਉ":422,"ਊ":48,"ੱ":1924,"ੰ":2555," a ":18,"੦":200,"੧":273,"੪":62,"ਿਮਨ":20,"੫":77,"੨":239,"੩":93,"੮":92,"੯":96,"੬":69,"੭":58,"੨੦੦":38,"ੜ":191,"ੀ":4636,"ੁ":1825,"ੂ":1115,"ੇ":3761,"ੈ":2081,"ੋ":1676,"ੌ":206,"੍":1046,"ੂਲ ":22,"ੀਜਿ":21," Ol":12,"ੂਰ ":25,"b ":18,"ਿਹਾ":91,"a ":87,"ਿਸੇ":29,"ਿਸ਼":57,"ਿਸਾ":23," Th":15,"ਿਸਤ":34,"ਚੌਂ":12,"ਚੋਂ":38,"ਿਵੇ":63,"ਿਵਾ":13,"ੀਕਾ":45,"i ":27,"ਿਟੀ":38,"ge":13," in":40,"ic ":12,"fi":14," is":12,"fo":16,"ਚਾਈ":23,"he":95,"ha":27,"gh":19,"go":13,"g ":34,"ea":30,"ਚਾਰ":34,"ec":17,"ਚਾਲ":16,"ed":45,"de":37,"di":36,"ia ":20,"ev":13,"h ":33,"Ind":16,"ee":14,"el":41,"ei":12,"en":68,"em":14,"et":19,"ੀਆਂ":178,"es":53,"er":114,"ੀਅਤ":34,"ca":15,"e ":169,"ਚੀਨ":33,"be":16,"da":20,"f ":44," of":43,"ct":18,"cs":14,"co":29,"ck":12,"ch":22,"ce":32,"c ":16,"ics":14,"d ":98,"at":58,"as":29,"ar":52,"al":54,"ai":16,"am":68,"an":97,"ac":18,"ad":16,"ab":12,"ਿਨਾ":26,"nt":47,"ns":26," am":51," an":24,"ਿਨੇ":13,"ੈ। ":755,"of":44,"om":42,"on":91,"ol":28," ਅ":1094,"os":19," ਇ":1749," ਆ":274,"ou":18,"or":54," ਏ":119,"r ":72," ਉ":313,"ow":13," ਈ":30," ਕ":1460,"pe":18," ਖ":182," ਗ":918,"ਿਨ੍":12," ਐ":59," ਓ":47," ਜ":1208," ਝ":16," ਟ":112,"po":12," ਘ":42,"pi":19," ਚ":259," ਛ":39,"lo":16,"ਜੋ ":82,"ll":20,"igh":12,"ly":15,"o ":26,"ma":17,"mb":17,"me":42,"mi":19,"mp":27,"mu":48,"na":37,"nc":21,"nd":68,"ne":28,"ng":50,"ni":32,"ਿਤਾ":77,"ਿਤੀ":12," ।":190,"ੀਅਮ":40,"m ":58,"ੀਅਨ":27,"ਿਥਿ":27,"ine":16,"ing":33,"li":35,"le":33,"ld":21,"la":31,"n ":137," co":22,"ht":18,"hu":17,"hi":29,"ho":16,"id":13,"ic":60,"ia":36,"ig":20,"in ":38,"ie":22,"k ":17,"ir":14,"is":39,"it":46,"iu":27,"il":21,"in":129,"io":41,"l ":48,"ਾਰਕ":16,"ਾਰਚ":41,"ਾਰਤ":101,"ਾਰਨ":27,"wo":13,"ਾਮਿ":15,"y ":61,"wa":12,"ve":29,"ਾਰੀ":54,"ur":21,"us":14,"ਾਰਾ":51,"ut":16,"um":56,"un":15,"ty":14,"ਾਰੇ":39,"ua":16,"to":25,"ts":12,"tr":30,"te":65,"ti":67,"ਾਬਕ":377,"th":116,"ta":28,"st":52,"se":26,"sh":12,"si":30," ੨":195," ੩":56," ੪":26," ੫":29,"u ":55,"ਚਰਲ":20," ੬":17," ੭":14," ੮":15," ੯":19,"rs":22,"rt":24,"ry":21," ੧":210,"ro":42,"ri":64,"ਚਲਾ":12,"re":47,"rd":17,"ੁਤ ":26,"ra":24,"t ":72," ਹ":2872," ਸ":2699," ਵ":2408,"ਾਬਲ":23,"ht ":12," ਰ":498," ਲ":715," ਬ":868," ਭ":339,"s ":155,"ੁਣ ":18," ਮ":1134," ਯ":133," ਨ":1023," ਪ":1056," ਫ":184," ਤ":951," ਥ":21,"ਜੇ ":27," ਦ":4245," ਧ":86," ਡ":55,"ਾਬੀ":48," ਢ":16,"pr":12,"ਾਬਾ":23,"ਿਗਿ":35,"ੁਰ ":13," s ":16,"ਾਹੀ":40,"ਾਹਿ":88,"ਿਚਾ":17,"hum":14,"ਾਸਿ":34,"ਾਸ਼":82,"ਾਸੇ":16,"ਜਾ ":63,"ਚਨਾ":19,"ਾਲੇ":56,"ਾਲਾ":46,"ਾਲੀ":54,"ਜ਼ ":52,"ਜੀ ":155,"ਿਖੇ":23,"ਾਵਾ":47,"ਿਖਾ":19,"ਾਵਲ":13,"ਿਕਾ":29," th":90,"ym":13,"ਿਕਸ":14,"ਹਾਂ":94,"ਹੀਰ":14,"ਹੀਨ":13,"ਹੁਣ":12,"ਹੁਤ":24,"ਹਿਰ":60,"ਹਿਲ":50,"ਹਿਨ":24,"ਹਿਬ":75,"ਹਿਸ":19,"ਹਾਲ":13,"ਹਾਰ":25,"ਹਾਨ":18,"ਹਿਤ":27,"ਹੀਂ":44,"ਹਾਸ":57,"ਹਾਈ":26,"ਜਨ ":22,"਼ਨ ":23,"ਹਰਿ":36,"ਹਨਾ":25,"er ":48,"es ":28,"ਸ੍ਰ":19,"ers":16,"en ":20,"ਸਿੱ":77,"ਸਿੰ":78,"ਸੂਰ":32,"ਸੂਬ":12,"ਸਿਧ":13,"ਸਿਰ":19,"ਹਨ।":438,"ent":21,"ght":14,"ਸੀ।":71,"ਸ਼ਨ":37,"ਸ਼ਤ":17,"ਸ਼ਰ":12,"ਸ਼ਬ":26,"ਸ਼ਹ":65,"ਸ਼ੁ":34,"ਸ਼ੀ":30,"ਸਾਂ":13,"ਸ਼ਿ":19,"ਸ਼ਾ":149,"ਸ਼ਖ":29,"ਸ਼ਟ":21,"ਸਾਨ":18,"ਸਾਰ":46,"ਸਿਕ":33,"ਸਿਖ":14,"ਸਾਲ":1144,"ਸਾਹ":90,"ਸਿਟ":36,"ਸੀਅ":29,"ਸਾਇ":73,"ਸ਼ੇ":16,"ਸ਼੍":23,"ਸਿਆ":13,"ਸਰਕ":13,"ਸਮੇ":23,"ਸਮਾ":24,"ਗੜ੍":19,"ਸਰੀ":12,"ਸਲਾ":21,"ਸਤੰ":39,"ਸਦੇ":13,"ਸਦਾ":17,"ਸਦੀ":15,"ਸਨੂ":15,"ਗੁਰ":167,"ਗੋਬ":26,"for":13,"ਸਟਾ":14,"ਗ੍ਰ":502,"ਸਨ।":37,"ਚਾ ":12,"ਸਤਾ":54,"ਸਥਾ":37,"ਸਥਿ":13,"ਗਰੇ":16,"ਗਰਾ":30,"ਗਰੀ":415,"ਿਚ ":162,"ਾਹ ":17,"ਾਸ ":54,"਼ਟਰ":18,"ਿਕ ":134,"ਕੰਪ":19,"ਾਲ ":1369,"੧੬ ":14,"cti":12,"ਗਸਤ":63,"੧੭ ":15,"ਾਰ ":238,"੧੪ ":14,"੧੫ ":13,"੧੧ ":12,"ਾਮ ":62,"ਗਿਆ":104,"੧੨ ":16,"ਗਾਂ":14,"੧੩ ":12,"਼ਖ਼":29,"ਾਬ ":99,"੧੦ ":14,"ਹੱਦ":12,"ਖਾਂ":21,"ਖ਼ਸ":29,"ਖਿਆ":40,"ਾਨ ":155,"com":13,"ਗਣਿ":17,"ਿਆ ":232,"ਾਦ ":29,"੦੮ ":24,"cs ":14,"ਾਣ ":28,"ਾਤ ":23,"ਖੇਡ":37,"ਖੇਤ":37,"ਾਜ ":52,"ਖੋਂ":14,"ed ":32,"ਕੌਮ":15,"ਾਗ ":23,"ਕ੍ਰ":37,"ਸੰਸ":37,"੦੦ ":21,"ਘਰ ":13,"ਾਈ ":117,"ਸੰਖ":18,"ਕਾਂ":22,"ਸੰਬ":52,"ਸੰਤ":12,"ਕਿਤ":57,"ਾਂ ":1353,"਼ੀ ":81,"ਹੋਏ":48,"ਹੋਇ":70,"ਹੋਈ":20,"ਕਾਲ":30,"ਕਾਸ":23,"ਕਾਬ":25,"ਕਾਰ":103,"ਕੀਤ":87,"ਹੋਰ":25,"ਹੋਣ":23,"ਕਿਲ":58,"ਕਿਸ":74,"ਕਿਹ":19,"ਕੁਝ":20,"ਕੁੱ":13,"dia":15,"਼ਾ ":51,"ਹਿੰ":31,"ਹੈ।":922,"ਕੇਸ":12,"ਹੁੰ":468,"ਕੋਈ":24," ਅਪ":47," ਅਨ":20," ਆਉ":20," ਅਧ":17,"ੱਖੀ":14,"ੱਖਾ":16," ਅਮ":64," ਅਰ":39," ਅਸ":17,"ੱਖਰ":18,"ਕਦੀ":15," ਅਜ":33," ਅਗ":67," ਅਕ":54," ਅਤ":411,"ੰਸਥ":15,"ਗਾ ":22,"ਕਨੀ":13,"ੰਸਾ":14,"re ":16,"ੱਚੋ":29,"ੱਛਮ":12,"ਕਤੀ":13,"ਕਤੂ":34,"ੱਜੋ":16,"rs ":13," ਉੱ":56," ਉਹ":51," ਉਸ":69," ਇੱ":255," ਇੰ":25," ਉਪ":25," ਉਨ":48," ਉਤ":19," ਏ।":18," ਇਹ":533," ਇਸ":723," ਇਲ":20," ਅੰ":217," ਅੱ":34,"ਾ। ":18," ਇਨ":39," ਇਤ":23," ਇਥ":21,"rig":16,"ਗਰ ":22," ਆਮ":22,"ੱਖਣ":15," ਆਰ":19," ਇਕ":94," ਆਦ":14," ਆਪ":76," ਆਬ":12,"ਸਟ ":13," ਕਰ":183," ੨ ":19," ਕਲ":399," ਕਹ":19," ਕਾ":86," ਕੀ":109," ਕਿ":260," ਕੁ":68," ਗਈ":32," ਕੇ":73," ਕੈ":22," ਕੋ":64," ਕੌ":24," ੩ ":14," ਕ੍":23," ਗਏ":21,"ਕਸ਼":26," ਓਲ":31," ਕਈ":21," ੧ ":21,"ry ":17,"ਸਤ ":69,"ਕਲਾ":14,"ਵਰਗ":12,"ਕਲੰ":379," ਏਫ":13," ਏਨ":15," ਏਸ":15,"ਵਰਸ":36," ਐਗ":22," ਏਲ":14,"ਵਰਤ":32,"ਵਰੀ":76," ਜ਼":109," ਜਿ":159," ਜਾ":364," ਜੁ":64," ਜੀ":148," ਜੂ":37," ਜੇ":15," ਜੋ":96," ਜੰ":39,"ਸਨ ":38,"ਕਰਕ":17," ੮ ":14,"ਕਰਦ":41,"ੱਡਾ":23,"ਕਰਨ":55," ਚਿ":16," ਚਾ":51," ਚੁ":16," ਚੀ":38," ਚੰ":37," ਚੱ":12," ਜਰ":13," ਛੋ":14,"ਕਰੀ":18," ੯ ":15,"ਕਰੋ":16," ਜਨ":89," ਜਦ":19,"ਕਲਚ":23," ੬ ":12,"ੱਤਰ":42," ਚਲ":15," ੭ ":13,"ਗੀ ":17,"ਸਭ ":25," ਗਰ":23," ਕੰ":44," ੪ ":15," ਖੇ":84," ਗਣ":16," ਖਾ":28," ਖਿ":13,"ੱਥੇ":12," ਖ਼":15,"ੱਤਾ":19," ਘਰ":13," ਗ੍":436," ੫ ":15," ਗੋ":35," ਗੁ":188," ਗਿ":86,"ੱਤੇ":24," ਤਰ":31,"ੰਖਿ":16," ਤਾ":41," ਤਿ":24,"ੰਗਾ":15,"ੰਗਰ":20," ਤਕ":27," ਤੱ":66," ਦਰ":56," ਦੂ":37," ਦਾ":950," ਦਿ":1171," ਦੀ":419," ਦੁ":51," ਦਸ":54," ਤੌ":38," ਤੋ":497," ਤੇ":160," ਟੀ":56," ਟਰ":13,"ੰਗ੍":60," ਮਈ":37," ਬੇ":15," ਬੋ":36," ਬਾ":492," ਬਿ":25," ਬੀ":32," ਬੁ":18," ਭਗ":25," ਬਹ":34," ਫ੍":14," ਪੱ":30," ਪੰ":163," ਬਲ":14," ਬਰ":25," ਬਨ":14," ਫ਼":66," ਪੜ":16," ਫਾ":12," ਫਿ":18," ਫੁ":20," ਬਣ":71," ਪੋ":18," ਪ੍":133," ਮੌ":17," ਮੋ":24," ਮੈ":37," ਮੀ":19," ਮੁ":481," ਮਾ":131," ਮਿ":97," ਮਹ":95," ਮਸ":25," ਬੰ":15,"st ":12," ਭੌ":12," ਮਨ":31," ਮਤ":12," ਭਾ":230," ਭੀ":16," ਬ੍":13," ਭਰ":19," ਨਹ":38," ਨਿ":106," ਨਾ":302," ਨੂ":313," ਨੇ":127," ਦੱ":14," ਨਵ":51,"ਗਏ ":15," ਦੇ":1425," ਦੋ":31," ਧਰ":56," ਪੁ":69," ਪੀ":18," ਪਿ":76," ਪਾ":125," ਪੈ":40," ਪੇ":13," ਪੂ":34," ਪਰ":200," ਪਹ":57," ਪਟ":13,"ਕੋ ":15," ਲੱ":19,"ਈ ":351,"ੰਦੇ":65," ਵਖ":19,"ੰਦੀ":31,"ੰਦਾ":409," ਵਰ":63," ਵਧ":14," ਵੀ":103," ਵਿ":1204," ਵਾ":793,"ੰਦਰ":40," ਵਸ":13," ਵੇ":12," ਵੈ":16," ਵੰ":18," ਵੱ":108," ਸਕ":45,"ੰਬਰ":125," ਯਾ":23," ਰਚ":14," ਯੂ":72,"ਖੇ ":23," ਮੰ":56," ਰਾ":142," ਰਿ":33," ਰਸ":75," ਰਹ":56," ਰੇ":12," ਲਈ":76," ਰੁ":23,"str":14,"ੰਪਿ":40," ਰੂ":37," ਰੋ":38," ਲਗ":21," ਰੱ":19," ਲਾ":29," ਲਿ":55," ਲੀ":380,"ਏ ":114," ਲੇ":15," ਲੈ":20," ਲੋ":37,"Oly":12,"ਗਤ ":30,"ਲੰਪ":29,"ਲੰਡ":379,"ਲੱਗ":14,"ਂ ":2409,"ੰਜੀ":16,"ੰਜਾ":127," ਸਭ":32," ਸਬ":19," ਸਮ":69," ਸਨ":75," ਸਪ":16," ਸਰ":55," ਸਟ":13," ਸਤ":61," ਸਥ":20," ਸਦ":12,"ੰਤਰ":46," ਹਰ":55," ਸ੍":21," ਸੋ":27,"ਅ ":12," ਸਾ":1296," ਸਿ":210," ਸ਼":257,"ਖੀ ":31," ਹਨ":519," ਸੇ":33," ਸੂ":56," ਸੀ":135," ਸੁ":40," ਹੋ":233,"ੰਡਲ":14,"ੰਡੀ":26," ਸੱ":24,"ਆ ":416," ਸੰ":107,"ੰਡਾ":15," ਹਾ":41," ਹਿ":46," ਹੀ":66," ਹੁ":491," ਹੇ":15," ਹੈ":1352,"ੰਡਰ":381,"ਟ ":103,"ਝ ":29," ਏ ":29,"ਜ ":145,"ਕਸ ":17,"ਚ ":1074,"ਘ ":72,"ਹਿ ":31,"ਗ ":162," ਚ ":20,"ਹਾ ":59,"ਖ ":140,"ਕ ":1139,"ਹੀ ":112,"ਓ ":12,"ੀ। ":55,"pic":12,"ਕਰ ":44,"ਵੰਡ":19,"ਰ ":1488,"ਵੱਖ":25,"ਹੇ ":27,"ਵੰਬ":36,"ਵੱਜ":17,"ਵੱਡ":34,"ਭ ":31,"ਹੈ ":423,"ਵੱਲ":20,"ਸਕਦ":19,"ਗਈ ":25,"ਮ ":350,"ਫ ":31,"ਕੇ ":100,"ਬ ":211,"ਓਲੰ":29,"ਪ ":461,"ਧ ":64,"ਨ ":1907,"ਥ ":48,"ਕਾ ":68,"ਦ ":171,"ਕਿ ":66,"ਣ ":217,"ਕੀ ":396,"ਹੋ ":30,"ਤ ":537,"ਠ ":22,"ਡ ":132,"ਖਣ ":24,"ਾ ":2802,"ਵਿਸ":21,"ਵਾਰ":49,"ਵਿਕ":13,"ੁ ":14,"ਵਾਲ":96,"ਵਾਸ":13,"ਵਿਗ":34,"ਵਿਖ":22,"ਵਿਚ":200,"ੀ ":2902,"ਵਿਦ":16,"ਿ ":127,"ਵਾਨ":14,"ਵਾਂ":742,"ਹ ":633,"ਸਰ ":20,"਼ ":152,"ਵ ":71,"ਸਾ ":23,"ਵੇਦ":48,"ਸ ":997,"ਵੇਂ":22,"ਸ਼ ":84,"ਲ ":1735,"ਸਸ ":13,"ਵਿੱ":870,"ਸੇ ":56,"ਹਨ ":79,"ਂ।":15,"ਕਨ ":15,"ੋ ":213,"ਸੀ ":89,"੍ ":22,"ਹਰ ":16,"ੇ ":2940,"ੈ ":444,"ੂ ":303,"ਕਟ ":14,"ਰਸ਼":34,"ਰਸਾ":69,"ਰਸਿ":50,"ਰਹਿ":56,"ng ":24,"ਰਹੇ":17,"nce":16,"ne ":14,"ndi":15,"ਰਾਂ":87,"ਰਾਜ":89,"ਰਾਬ":13,"ਰਾਨ":31,"ਰਿਆ":72,"ਰਾਣ":19,"ਰਾਸ":25,"ਰਾਹ":24,"ਰਿਕ":21,"ਰਾਵ":14,"ਰਾਮ":35,"nd ":32,"ਰਿਤ":27,"ਰੀਆ":20,"ਰਮਾ":139,"ਮੰਡ":22,"ਮੰਨ":14,"ਲੇ ":121,"ਰਵਾ":29,"ਰਵਰ":37,"ਰੈਲ":39,"ਰੋਜ":15,"ਰੋਮ":30,"ਰੋੜ":12,"nte":17,"ੰਕ ":58,"ns ":12,"ਰੀਸ":12,"ਰੀਬ":18,"ਰੀਕ":94,"ਰਿਸ":18,"ਰਿਹ":15,"ੰਗ ":69,"। ":1125,"ਕਈ ":21,"ਰੂਪ":26,"ੰਘ ":71,"ਰੈਗ":367,"ਰੈਕ":12,"ਰੇਗ":15,"ਰੇਜ":79,"ੰਜ ":22,"ਲਮ ":13,"ਰਕੇ":20,"of ":41,"ਲਣ ":22,"ਰਕਾ":23,"ਐਗਰ":22,"ਰਨਾ":16,"ਲਾ ":163,"ਰਦੁ":28,"ਰਦੀ":16,"ਰਦਾ":42,"ਰਦੇ":29,"ਰਮਨ":15," ਈ ":16,"ਰਬੀ":20,"ਲੀ ":116,"ਯੋਗ":18," ਆ ":21,"on ":41,"ਰਤਾ":15,"ਰਤਿ":15,"ਰਤੀ":63,"ona":13,"ons":12,"ਯੂਨ":34,"ਯੂਰ":14,"ਯੂਲ":19,"ਲੀਪ":371,"ੱਤ ":62,"ਲੀਵ":25,"ੱਦ ":13,"ਲਾਂ":56,"ਲਾਈ":52,"ਵਲ ":18,"ਲਿਆ":33,"ਲੀਅ":22,"ld ":18,"ਲੀਆ":17,"ਲਾਵ":16,"ਲਿਖ":25,"ੱਧ ":29,"ਵੇ ":16,"ਲ੍ਹ":16,"ੱਲ ":23,"ਵਾ ":26,"ਲੋਮ":22,"ਲੋਂ":20,"ਲੋਕ":25,"ਲੋਗ":23,"ਵੀ ":125,"ਲੇਖ":15,"ਲੈਂ":19,"ੰਤ ":18,"mb ":14,"ਵਖ ":12,"ੰਡ ":46,"ਲਚਰ":23,"mer":15,"ਲਗਾ":13,"ੰਨ ":21,"lym":12,"ੰਧ ":16,"।ਇ":25,"ੰਦ ":43,"ੰਥ ":14,"mpi":14,"ਰੱਖ":17,"ੱਕ ":255,"ਰੰਥ":15,"ਵਨ ":15,"ੱਖ ":91,"mu ":47,"ੱਚ ":827,"ਆਉਂ":16,"ਅਨੁ":13,"੍ਹਾ":96,"ਇਸ ":693,"ਰਾ ":103,"ਮਨੇ":20,"ਇਹ ":527,"ਮਨੁ":16,"ਮਨੀ":13,"ਅਤੇ":407,"ਰਲ ":24,"ਭਾਈ":15,"ਭਾਗ":17,"ਭਾਰ":147,"ਭਾਵ":14,"ਭਾਸ":44,"ਅਜਿ":15,"ਇਲ ":16,"ਰਨ ":81,"ਰਮ ":38,"ਅਰਥ":18,"ਅਮਰ":51,"ਯਾ ":16,"ਬੋਲ":29,"ਰਡ ":23,"ਅਪ੍":39,"ਰਤ ":82,"ਬ੍ਰ":15,"੍ਰੋ":27,"੍ਰੇ":90,"੍ਰੈ":416,"ਰਣ ":20,"੍ਰਮ":14,"੍ਰਦ":16,"੍ਰੀ":65,"੍ਰਿ":60,"੍ਰਾ":30,"੍ਰਸ":16,"੍ਰਹ":30,"ਰਥ ":20,"੍ਰੰ":13,"ਮੇਂ":24,"ਆਨੀ":21,"ਏ। ":23,"ਆਪਣ":51,"ਆਦਿ":12,"ਆਦਾ":18,"ਆਣਾ":27,"ਮੈਨ":17,"ਮਾਨ":46,"ਮਾਤ":26,"ਮਾਣ":116,"ਮਿਕ":21,"ਮਾਰ":62,"ਮਾਲ":18,"ਮਿਥ":28,"ਮਾਂ":39,"ਇਆ।":31,"ਮੁੱ":38,"ਮਿਲ":41,"ਮਿਸ":19,"ਮੀਟ":34,"ਮੁਖ":14,"ਮੁਕ":29,"ਮੁਤ":380,"ਰੋ ":14,"ਮਹਾ":43,"ਮਹਿ":32,"ਮਹੀ":13,"ਆਵਾ":13,"ਰੂ ":145,"ਭੌਤ":12,"ਆਰਾ":46,"ਰੀ ":651,"ਬੰਧ":25,"ਰੇ ":75,"ਲਈ ":78,"ਉਣ ":23,"ਮਰੀ":43,"ਇਥੇ":16,"ਬਣਾ":43,"ਇਨ੍":18,"ਪੜ੍":12,"ਫੁਟ":13,"ਫ਼ਰ":40,"ਇਣਕ":56,"ਉਸ ":51,"ਉਹ ":40,"ਈਆਂ":13,"ਇਤਿ":19,"ਮਨ ":35,"ਇਸਦ":16,"ਪ੍ਰ":176,"ਪੈਦ":15,"ਅੰਤ":31,"ਅੰਦ":15,"ਅੰਗ":91,"ਅੰਕ":65,"ਭੀ ":15,"ਇਲਾ":16,"ਈਨਾ":22,"ਰਜ ":32,"ਰਚ ":44,"ਬੀਜ":26,"ਰਗ ":14,"ਬਿੰ":25,"ਬਾਦ":22,"ਬਾਰ":32,"ਬਾਬ":12,"ਰਕ ":18,"ਬਾਕ":372,"ਬਾਲ":27,"ymp":12,"ਈਡਰ":15,"ਬਾਅ":27,"ਬਾਈ":14,"ਉਂਦ":22,"ਬਹੁ":25,"ਭਗਤ":25,"ਮੇ ":12,"ਬਲਾ":20,"ਪੱਛ":12,"ਪੰਜ":152,"ਬਰਾ":15,"ਫ੍ਰ":14,"ਮੀ ":18,"ਮਾ ":32,"ਸਮ":81,"ਸਭ":32,"ਸਬ":22,"ਸਫ":15,"ਸਪ":32,"ਸਨ":101,"ਸਵ":19,"ਸਲ":51,"ਸਰ":94,"ਾ।":27,"ਸਟ":58,"ਸਦ":49,"ਸਤ":207,"ਸਥ":50,"ਸਕ":73,"ਵੱ":108,"ਵੰ":62,"ਵੈ":22,"ਵੇ":119,"ਉਨ੍":43,"ਵਸ":16,"ਵਿ":1234,"ਵੀ":154,"ਵਾ":1015,"ਵਨ":21,"ਵਧ":14,"ਵਰ":201,"ਵਲ":39,"ਵਖ":19,"ਲੰ":416,"ਲੱ":23,"ਲੜ":14,"ੌਰ ":34,"ਲੋ":133,"ਲ੍":24,"ਲੇ":163,"ਲੈ":43,"ਲਿ":106,"ਲਾ":368,"ਲੁ":15,"ਲੀ":580,"ਲਹ":12,"ਰੱ":21,"ਰੰ":40,"ਲਵ":25,"ਲਬ":13,"ਲਮ":25,"ਲਤ":16,"ਲਣ":25,"ਲਦ":24,"ਲਚ":23,"ਲਕ":25,"ਲਗ":28,"ਰੈ":428,"ਰੋ":128,"ਰ੍":22,"ਰੀ":839,"ਰੁ":36,"ਰੂ":205,"ਲਈ":79,"ਰੇ":213,"ਰਹ":94,"ਰਸ":179,"ਰਿ":221,"ਰਾ":502,"ਮੱ":15,"ਮੰ":60,"ਰਲ":37,"ਰਵ":92,"ੀਤ":114,"ੁਆ":59,"ੀਦ":20,"ੀਬ":34,"ੀਮ":70,"ੀਨ":92,"ੀਪ":396,"ੁਖ":20,"ੀਵ":67,"ੁਕ":56,"ੀਰ":67,"ੀਲ":39,"ੇ।":14,"ੁਝ":24,"ੁਟ":25,"ੀਸ":29,"ੁਜ":22,"ੀਆ":241,"ਿਥ":45,"ਿਦ":23,"ਿਣ":20,"ੀਅ":117,"ਿਤ":204,"ੀਂ":64,"ਿਡ":14,"ਿਟ":58,"ਿਮ":51,"ਿਬ":77,"ਬਰ ":175,"ਿਪ":16,"ਿਧ":20,"ਿਨ":1162,"ੀਗ":18,"ਿਵ":98,"ੀਕ":136,"ਿਲ":265,"ਿਰ":130,"ੀਟ":47,"ਾੜ":27,"ੀਜ":44,"ਿਹ":111,"ਿਸ":263,"ਾਡ":16,"ਇੰਟ":13,"ਾਣ":249,"ਿਅ":43,"ਾਤ":101,"ਿਆ":517,"ਾਦ":96,"ਾਧ":15,"ਿਉ":16,"ਾਨ":318,"ਿਊ":33,"ਾਪ":44,"ਾਬ":594,"ਾਮ":104,"ਾਰ":660,"ਾਲ":1569,"ਾਵ":109,"ਿਖ":72,"ਿਕ":215,"ਿਗ":45,"ਾਹ":169,"ਿਚ":202,"ਾਸ":245,"ਿਜ":25,"਼ਾ":203,"਼ਿ":116,"ਾਅ":32,"਼ੀ":109,"ਾਂ":1630,"਼ੁ":43,"ਾਈ":221,"ਾਉ":53,"਼ੇ":19,"ਾਇ":185,"਼ੋ":16,"਼ੈ":21,"ਬਲ ":16,"਼੍":25,"ਾਕ":443,"ਾਗ":63,"ਾਖ":18,"ਾਜ":126,"ਾਚ":13,"਼ਨ":40,"਼ਤ":27,"਼ਟ":24,"਼ਸ":37,"਼ਹ":70,"਼ਵ":12,"਼ਰ":65,"਼ਬ":33,"਼ਮ":21,"਼ਖ":29,"ਹੱ":25,"ਹੰ":15,"ਹੂ":17,"ਹੁ":520,"ਹੈ":1355,"ਹੇ":48,"ਹਾ":329,"ੀ।":87,"ਹੀ":203,"ਹਿ":360,"ਸੰ":167,"ਸੱ":26,"ਹੋ":237,"ਹੌ":13,"ਸੂ":66,"ਸੁ":43,"ਸੀ":222,"ਸੇ":97,"ਹਨ":550,"ਸਹ":12,"ਸਸ":15,"ਸਿ":313,"ਸਾ":1477,"ਸ਼":623,"ਹਲ":14,"ਸੋ":30,"ਹਰ":67,"ਹਮ":13,"ਸ੍":26,"ਦਸ":57,"ਦਿ":1225,"ਦਾ":1748,"ਦੁ":90,"ਦੀ":615,"ਦੂ":48,"ਥੇ":53,"ਦਨ":53,"ਥੋ":13,"ਦਰ":129,"ਤੰ":44,"ਦਲ":21,"ਤੱ":67,"ਧਾ":77,"ਨਜ":12,"ਨਡ":15,"ਧੀ":16,"ਧਿ":28,"ਨਦ":13,"ਨਤ":17,"ਦੇ":1613,"ਦੋ":54,"ਨੁਸ":12,"ਧਰ":65,"ਨਕ":44,"ਤਸ":17,"ਤਵ":19,"ਤੀ":231,"ਤੂ":46,"ਤਿ":96,"ਤਾ":728,"ਣੇ":53,"ਤਨ":20,"ਤਪ":13,"ਤਤ":12,"ਤਰ":186,"ਤਲ":37,"ਤਮ":27,"ਉਹਨ":12,"ਥੀ":15,"ਥਿ":46,"ਥਾ":70,"ਤੋ":518,"ਤੇ":631,"ਨੁੱ":16,"ਤੌ":43,"ਇੱਕ":245,"ਣਕ":72,"ਨੂੰ":329,"ਡੇ":30,"ਣਾ":137,"ਣੂ":108,"ਣਿ":62,"ਣੀ":85,"ਤਕ":48,"ਨੇਂ":14,"ਟ੍":17,"ਟੇ":26,"ਨੇਜ":20,"ਟੀ":130,"ਟਿ":29,"ਡਿ":29,"ਡੀ":68,"ਨ।":476,"ਡਾ":86,"ਡਲ":20,"ਡਰ":410,"ਡਦ":12,"ਮਰ":69,"ਮਲ":18,"ਬੰ":44,"ਮਸ":29,"ਮਹ":96,"ਮੁ":499,"ਮੀ":82,"ਮਿ":154,"ਮਾ":391,"ਮੂ":19,"ਮੈ":49,"ਮੇ":59,"ਮ੍":15,"ਮੌ":19,"ਮੋ":28,"ਰਕ":76,"ਰਖ":17,"ਰਗ":45,"ਰਚ":72,"ਰਜ":81,"ਰਟ":18,"ਯਾ":37,"ਰਡ":28,"ਯੂ":73,"ਰਣ":36,"ਯੁ":14,"ਰਥ":37,"ਰਤ":198,"ਰਦ":131,"ਰਨ":116,"ਰਫ":22,"ਰਪ":20,"ਯੋ":27,"ਰਬ":46,"ਰਮ":236,"ਪੰ":164,"ਬਲ":53,"ਪੱ":37,"ਬਰ":215,"ਫ੍":15,"ਬਹ":34,"ਭਗ":29,"ਬੁ":20,"ਬੀ":120,"ਬਿ":55,"ਬਾ":578,"ਬੋ":46,"ਬੈ":14,"ਬੇ":32,"ਮਈ":41,"ਮਕ":17,"ਬ੍":15,"ਭਰ":22,"ਮਜ":12,"ਮਤ":30,"ਭਾ":256,"ਭੀ":18,"ਭਿ":14,"ਭੌ":12,"ਮਦ":18,"ਮਨ":98,"ਪਲ":21,"ਨੰ":12,"ਪਹ":57,"ਪਸ":12,"ਪਰ":222,"ਪੂ":52,"ਪੈ":44,"ਪੇ":23,"ਪੀ":37,"ਪੁ":90,"ਪਾ":168,"ਪਿ":130,"ਬਕ":380,"ਪੋ":22,"ਪ੍":176,"ਨ੍ਹ":75,"ਫਰ":17,"ਫਲ":16,"ਬਦ":46,"ਬਨ":25,"ਫ਼":106,"ਪੜ":21,"ਫਾ":21,"ਫਿ":21,"ਫੁ":23,"ਬਣ":71,"ਨਵ":102,"ਦੱ":14,"ਨਲ":14,"ਨਰ":14,"ਨਮ":39,"ਨੇ":196,"ਨੂ":341,"ਨੀ":200,"ਨੁ":33,"ਨਾ":539,"ਨਿ":163,"ਨਸ":36,"ਨਹ":40,"ਨ੍":89,"ਨੈ":12,"ਨੋ":19,"ਪਨ":15,"ਪਣ":56,"ਪਤ":28,"ਪਟ":24,"ਬਦ ":20,"ਕਰ":232,"ਕਮ":20,"੨ ":59,"ਕਲ":439,"ਕਨ":33,"ਕਦ":33,"ਕਟ":31,"ਕਤ":65,"੧ ":77,"ਕਈ":21,"ਕੱ":19,"ਕੰ":48,"ਗਲ":25,"ਗਰ":511,"੪ ":44,"ਖੋ":29,"ਖੇ":111,"ਗਦ":22,"ਗਣ":22,"ਗਤ":41,"ਖੀ":35,"ਖਾ":81,"ਖਿ":54,"ਖ਼":60,"ਕੜ":15,"ਕ੍":37,"ਗਏ":21,"ਖਰ":32,"ਕੌ":24,"੩ ":43,"ਕੋ":104,"ਗਈ":32,"ਖਦ":15,"ਕੈ":22,"ਕੇ":135,"ਕੁ":80,"ਕੂ":15,"ਖਣ":34,"ਖਤ":12,"ਕਾ":323,"ਕਿ":322,"ਕੀ":517,"ਕਹ":19,"ਕਵ":12,"ਕਸ":64,"ਏਲ":14,"ਐਗ":22,"ਪਹਿ":45,"ਏਸ":17,"ਓਲ":31,"੦ ":87,"ਜੇ":38,"ਜੈ":24,"ਜੋ":120,"ਜ਼":304,"ਜਾ":547,"ਜਿ":213,"ਜੀ":234,"ਜੁ":64,"ਜੂ":46,"ਪਿਤ":20,"ੰ ":328,"ਪਾਸ":19,"ਪਿਕ":29,"ਜਦ":23,"ਪਾਰ":18,"ਜਧ":12,"ਜਨ":116,"ਪਿਊ":14,"ਪਾਣ":18,"੯ ":52,"ਛੋ":17,"ਚੱ":12,"ਪਾਕ":32,"ਚੰ":38,"ਜਲ":16,"ਜਰ":27,"ਟਬ":13,"ਟਨ":25,"ਪਾਈ":14,"ਟਾ":78,"ਟਰ":121,"ਜੰ":41,"ੋਗਰ":27,"ਗੜ":29,"੬ ":50,"ਚਕ":12,"ਗਸ":65,"ਗਾ":75,"ਗੂ":16,"ਗੁ":197,"ਗੀ":24,"ਗਿ":132,"ਪੂਰ":40,"ਗੇ":24,"ਗ੍":505,"ਗੋ":52,"੫ ":50,"ਘਰ":19,"ਚਿ":28,"ਚਾ":99,"ਚੁ":16,"ਚੀ":48,"ਚੇ":17,"ਪਿੰ":41,"ਪੁਰ":63,"ਚੋ":42,"ਚੌ":23,"੮ ":73,"ਛਮ":12,"ਜਗ":13,"ਚਨ":19,"ਚਰ":32,"੭ ":49,"ਚਲ":24,"ਅਤ":454,"ਆਂ":297,"ਅਜ":34,"ਅਗ":69,"ਅਕ":61,"ਆਣ":33,"ਇਆ":135,"ਆਖ":12,"ਅਸ":23,"ਅਮ":113,"ਅਰ":63,"ਅਲ":18,"ਅਦ":42,"ਆਇ":12,"ਅਧ":17,"ਆਉ":22,"ਅਨ":55,"ਅਪ":47,"ਪਣੇ":34,"ਈਆ":13,"ਇਥ":21,"ਇਤ":25,"ਇਣ":62,"ਆਸ":22,"ਪਣੀ":15,"ਆਰ":95,"ਆਮ":24,"ਇਕ":101,"ਆਵ":17,"ਆਲ":16,"ਆਨ":54,"ਆਦ":35,"ਆਬ":13,"ਆਪ":78,"ਆ।":54,"ਬਾ ":22,"ਂਕ":18,"ਂਗ":31,"ਈ।":12,"ਂਟ":25,"ਂਡ":36,"ਂਸ":30,"ਂਤ":24,"ਂਦ":244,"ਉੱ":56,"ੋੜ ":16,"ਬੀ ":78,"ਏਨ":15,"ਏਫ":13,"ਇਨ":48,"ਅੱ":34,"ਇਲ":37,"ਅੰ":220,"ਈਟ":14,"ਇਹ":542,"ਮਈ ":41,"ਇਸ":741,"ਉਂ":46,"ਈਡ":27,"ਈਨ":32,"ਏ।":33,"ਬੇ ":13,"ਉਦ":21,"ਉਨ":51,"ੋਇਆ":71,"ਉਣ":28,"ਉਤ":19,"ਉਪ":26,"ੜ ":44,"ਇੱ":255,"ਇੰ":32,"ਉਸ":76,"ਉਹ":56,"ਊਟ":14,"ਪਰਮ":113,"ਨਕਸ":15,"ਪਤ ":12,"ਧਰਮ":31,"ਧਰਤ":22,"ਪਰ ":49,"ਧਾਂ":14,"ਉੱਤ":40,"ੋਬਿ":25,"ਦੁਆ":48,"ਦਿੱ":27,"ਦੁਨ":24,"ੋਮੀ":24,"ਦੂਸ":12,"ਦੂਜ":16,"ੋਮਨ":25,"੦੦":66,"੦੮":25,"੧੦":19,"੧੩":12,"੧੪":15,"੧੧":14,"੧੨":18,"੧੭":17,"੧੮":24,"੧੫":17,"੧੬":17,"੧੯":42,"੨੧":14,"੨੦":53,"ਦਾਂ":14," ੧੮":24," ੧੭":16,"ਦਾਨ":17," ੧੬":17," ੧੫":16," ੧੪":15," ੧੩":12," ੧੨":16," ੧੧":13,"ੋਲੀ":19,"ਦਾਰ":26," ੧੯":41," ੨੦":53," ੨੧":14,"ਦਿਆ":31," ੨੭":12," ੨੬":12," ੨੯":17," ੨੮":14," ੨੩":13," ੨੨":14," ੨੫":13," ੨੪":14,"ਦਾਸ":19," ੩੦":25," ੩੧":16,"ਦੀਆ":75,"ਦਿਨ":1097,"umb":16,"੍ਹ ":21," ੧੦":19,"ਦੋਂ":17,"੩੧":18,"੩੦":26,"um ":31,"੨੯":17,"੨੮":15,"੨੭":13,"੨੬":13,"੨੫":14,"੨੪":14,"੨੩":13,"੨੨":14,"ੱਤ":172,"ੱਢ":13,"ੱਡ":45,"ੱਧ":49,"ੱਦ":25,"ੱਥ":25,"ੱਗ":23,"ੱਕ":286,"ੱਖ":185,"ੱਛ":18,"ੱਜ":30,"ੰਸ":52,"ੱਚ":879,"ੱਟ":28,"ੱਠ":21,"ੱਲ":52,"ੱਸ":22,"ੰਥ":18,"ੰਤ":85,"ੰਧ":36,"ੰਦ":618,"ੰਡ":502,"ੰਬ":155,"ੰਮ":32,"ੰਨ":45,"ੰਪ":49,"ੰਕ":75,"ੰਖ":19,"ੰਗ":218,"ਦੇਸ":44,"ਦੇਵ":25,"ੰਜ":177,"ੰਟ":22,"ੰਘ":85,"ਨਵਰ":42,"ਨੀਆ":32,"ਨੀਅ":13,"ਨਿਵ":62,"ਨੀਕ":15,"ਨਿਆ":24,"ਬਕ ":378,"ਨਾਨ":32,"ਨਾਲ":179,"ਨਾਵ":16,"ਨਿਕ":18,"ਨਾਮ":38,"ੜ੍":38,"ਨਾਂ":88,"ੜੀ":29,"ੜਾ":29,"ੜੇ":20,"ty ":13,"ਨਸੰ":13,"ਨਹੀ":36,"ਏਫ ":13,"ਨਵੰ":36,"ੁਦ":13,"ੂਆ":13,"ੁਨ":42,"ੁਤ":412,"ੁਣ":26,"ੁਮ":18,"ੁਰ":279,"ੁਸ":47,"ਿੰ":258,"ੁਲ":97,"ਿੱ":1026,"ੈ।":922,"ੂਜ":17,"ੂਨ":78,"ੂਦ":14,"ੂਰ":124,"ਏਲ ":13,"ੂਬ":50,"ੂਪ":31,"ੂਸ":23,"ੂਲ":53,"ੁੰ":490,"ੁੱ":117,"ੂੰ":335,"ੇਂ":94,"ੇਕ":12,"ੇਖ":36,"ੇਦ":53,"ੇਤ":52,"ੇਡ":47,"ੈਂ":49,"ੇਟ":14,"ੇਜ":108,"ੇਗ":24,"ੇਰ":47,"ੇਲ":44,"ੈਕ":39,"ੇਵ":52,"ੈਗ":377,"ਧਾਰ":30,"ੇਨ":27,"ਧਾਨ":15,"ਧਿਆ":15,"ੈਣ":13,"ੈਦ":24,"ੇਸ":109,"ੈਟ":17,"ੈਲ":73,"ੈਰ":17,"ੈਨ":38,"ੈਸ":23,"ੋਂ":617,"ੋਇ":77,"ੋਈ":46,"tio":34,"thu":15,"ੋਟ":27,"ੌਂ":47,"ੋਡ":13,"ੋਜ":32,"ੋਧ":14,"ੋਨ":32,"ੋਪ":22,"ੋਣ":30,"ੋਤ":20,"ੋਏ":51,"ੋਗ":55,"ੋਚ":14,"ੋਕ":33,"ੌਜ":14,"ੌਤ":23,"ੋਮ":61,"ੋਬ":36,"ੋਲ":71,"ੋਰ":92,"ੋਵ":23,"ੋਹ":24,"ੋਸ":21,"ੋੜ":31,"ੌਮ":17,"ੌਰ":50,"ted":14,"੍ਹ":146,"੍ਰ":834,"ter":25,"the":71,"ਆ। ":38,"ਤੂਬ":34,"ੇਲ ":13,"ੇਰ ":12,"ਤੀਆ":19,"ੇਸ ":12,"ਤਿਹ":21,"ਤਿਆ":30,"ਤਾਨ":49,"ਤਿਕ":18,"ਤਾਬ":383,"ਨਕ ":14,"ਤਾਰ":18,"ਤਾਂ":61,"ੇਵ ":19,"ਨਮ ":32,"ਤੌਂ":25,"ਤੋਂ":485,"ੈਨ ":18,"ਤੌਰ":15,"ੈਲ ":41,"ਨਾ ":113,"ਥਾਂ":13,"ਥਿਤ":12,"ਥਿਹ":26,"ਥਾਨ":18,"ਦਸੰ":39,"ਨੀ ":107,"ੇਗਰ":13,"ਦਰਿ":26,"ਨੇ ":143,"ਤੰਬ":41,"ਤੱਕ":12,"ਤੱਤ":53,"ਂਟ ":12,"ੇਜੀ":27,"ੇਜ਼":73,"ੇਡਾ":24,"ਂਡ ":22,"ੈਂਡ":21,"ੇਡਦ":12,"ਂਗ ":18,"ੇਦਨ":48,"ੇਤੀ":16,"ੇਤਰ":23,"ੋਂ ":614,"ਤੋ ":20,"ਣਕਾ":14,"ਤਕਨ":13,"ਥੇ ":48,"ਦਨ ":52,"ੈਕਟ":16,"ੋਈ ":39,"ce ":16,"ੈਗਰ":366,"ੋਏ ":45,"ਥਾ ":23,"ੇਵਾ":19,"ੇਸ਼":80,"am ":12,"ੋਕ ":20,"al ":27,"ਣਿਤ":17,"ਣਿਆ":40,"and":32,"amu":47,"an ":23,"ੈਦਾ":18,"ਦਰ ":34,"ਣਾਇ":20,"ੌਂ ":41,"ੋਣ ":17,"ਦੋ ":25,"ਤਰਾ":24,"ਤਰੀ":36,"ੋਪ ":15,"ਦੇ ":1528,"at ":19,"ਦੀ ":510,"as ":15,"ੋਰ ":44,"ੋਲ ":12,"ਦਾ ":1619,"ati":17,"ਦਿ ":15,"ੜਾ ":14,"ੁਟਬ":13,"ੀਸਟ":12,"ੀਵਨ":13,"ੀਵਰ":32,"ੁਕਾ":27,"ਤਕ ":19,"ੜੀ ":26,"ੀਮਾ":12,"�":48,"ੁਰਦ":34,"ਡਦੀ":12,"ੜੇ ":18,"ੁਨਿ":12,"ੁਨੀ":20,"ਣੀ ":69,"ੁਤਾ":380,"ਣਾ ":86,"ਡਰਾ":13,"ਣੂ ":105,"ਣੇ ":51,"ਆਂ ":277,"ੁਸ਼":14,"ਂਦਰ":14,"ਂਦੀ":34,"ਂਦਾ":168,"ਂਦੇ":20,"ੈ।ਇ":14,"ੂੰ ":327,"ਤਰ ":83,"ਅਨ ":34,"ਡਾਂ":31,"ੁਰੂ":142,"ੁਰਾ":23,"ੁਰਸ":21,"ਿੰਦ":69,"ਿੰਡ":45,"ਿੰਘ":68,"ਿੰਗ":49,"ਡੀਅ":12,"ਿੱਤ":44,"ਿੱਧ":18,"ਿੱਖ":61,"ਅਤ ":35,"ਿੱਚ":863,"ੁਲਾ":52,"ੁਲੀ":15,"ਅਦ ":27,"ੂਰਜ":24,"ੂਰਬ":14,"ੂਬਰ":34,"ਤਾ ":184,"ਤੀ ":175,"ੂਨਿ":14,"ੂਨੀ":19,"ਤੇ ":613,"ਜੰਤ":20,"ਜੰਗ":15,"ਡਰ ":388,"ਆਨ ":22,"ੂਲੀ":19,"ਇਆ ":100,"ਆਪ ":24,"ਅਮ ":45,"ੁੱਖ":54,"ਅਰ ":17,"ੁੰਦ":474,"ਡਲ ":18,"ਟਬਾ":13,"ਡੀ ":25,"ਡਾ ":35,"ਨ। ":77,"ੇਂ ":52,"ਡੇ ":21,"ੜ੍ਹ":37,"ਟਰੀ":23,"ਟਾਂ":13,"ਟਾਇ":14,"ਅਕਤ":41,"ਅਕਾ":15,"ਅਗਸ":63,"ਆਮ ":19,"ਆਰ ":21,"ਣਕ ":58,"ਟਿਆ":13,"ਇਕ ":86,"ਟੀਮ":54,"ਟ੍ਰ":17},"n_words":[112478,136533,89577],"name":"pa","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/pl.json b/contrib/languages-data/pl.json
new file mode 100644
index 0000000..3ddb2f6
--- /dev/null
+++ b/contrib/languages-data/pl.json
@@ -0,0 +1 @@
+{"freq":{"D":76546,"E":48349,"F":59499,"G":97533,"A":113161,"B":112599,"C":110974,"L":74803,"M":131493,"N":86269,"O":63924,"H":50582,"I":84977,"J":54373,"K":109538,"U":33818,"T":71466,"W":122172,"V":29111,"P":225853,"S":204600,"R":87715,"Y":5329,"X":17841,"Z":57656,"f":154643,"g":760584,"d":1304112,"e":3541226,"b":467514,"c":1792078,"a":3833729,"n":2529442,"o":3537464,"l":1140017,"m":1150421,"j":965827,"k":1483844,"h":558914,"i":3732766,"w":2322567,"v":42704,"u":1069834,"t":1619824,"s":1819258,"r":2204575,"p":1082020,"z":1828688,"y":1372315,"x":13069,"é":11522,"á":5335,"ü":5350,"ö":4590,"ó":396123,"ę":271422,"ć":60605,"ą":370153,"ś":279181,"Ś":15543,"ń":123462,"ł":591480,"Ł":9157,"ż":255164,"Ż":6256,"ź":20372," l":139321," m":204189," n":260999," o":286725," h":33638," i":177645," j":136928," k":213385," d":243257," e":37159," f":50076," g":203060,"р":5358," a":148427," b":95754," c":133679," z":321847," u":97002," t":172994," w":969288," p":661390," s":339763," r":209751," J":52915," K":104698," H":47495," I":52640," N":80062," O":56243," L":69035," M":123296," B":105665," C":84648," A":95983," F":55238," G":73890," D":70098," E":42554," Z":55304,"к":5011," X":11746,"и":6265,"о":7740,"н":5938," S":182443," R":76715," P":214389,"а":9102," W":115231," V":20154," U":30651,"е":5223," T":64394," ż":19337," Ż":6158,"ęśc":13317," ś":51887," Ś":15346," Ł":9064," ł":17467,"A ":12056,"Da":8795,"Cz":10287,"Co":11705,"Ce":7894,"Ch":21362,"Ci":4668,"G ":10044,"Du":4576,"Do":17200,"Dr":5145,"De":9899,"Di":5692,"GC":12708,"Fe":4746,"Eu":8774,"Ge":6043,"Ga":8896,"I ":22055,"Fr":25795,"Fo":4852,"Fi":6166,"B ":5264,"C ":19133,"Au":8116,"Ar":11839,"Ba":23485,"Am":7571,"An":13752,"Al":15057,"Bu":9080,"Br":16403,"Ca":12445,"Bi":14211,"Be":12406,"Bo":15278,"Ku":5947,"Gó":8233,"Kr":17850,"Ko":31811,"Le":13922,"Li":15420,"La":13916,"Lu":9458,"Lo":10236,"Me":12645,"NG":5663,"Mi":29200,"O ":4778,"Ma":41474,"Mu":5320,"Mo":18522,"Ni":19737,"Ne":7072,"Na":24418,"P ":7412,"No":17077,"Ol":5489,"Od":7399,"PG":5183,"Ob":6870,"Gm":6336,"Gr":14758,"Go":7919,"Ha":11794,"He":11337,"II":18254,"Hi":5668,"Ho":9461,"In":12447,"Ja":16186,"L ":4866,"Je":18291,"Jo":8139,"Ka":25307,"M ":4990,"Ki":5813,"Un":7211,"VI":4920,"W ":15791,"Tu":5606,"Tr":10180,"To":9177,"Th":7956,"Te":9915,"Ta":10132,"V ":6364,"Sz":17267,"Sy":6237,"St":34755,"Su":6926,"Wo":12341,"Ws":4692,"Wi":28290,"Wa":16039,"We":8452,"Vi":5052,"X ":6678,"Pu":5975,"Pr":25703,"S ":11205,"Pe":7756,"Pa":34023,"Po":99382,"Pi":15484,"Os":6465,"Or":5855,"Se":13365,"Sc":7388,"Si":11311,"Sk":5337,"Sp":7038,"So":10398,"Ru":5061,"Rz":5267,"Sa":22530,"Re":17869,"Ro":19242,"Ra":13392,"b ":46793,"a ":1141880,"Wy":14854,"Za":17995,"Sł":20200,"i ":569208,"bó":4844,"gd":5266,"ge":34328,"ga":71454,"ać":8134,"fi":50271,"fr":12808,"fu":7503,"fo":23920,"j ":229252,"bę":5925,"có":12082,"gw":10035,"he":43730,"ha":50953,"gn":13137,"gm":84581,"gl":15240,"bą":7905,"gi":77017,"gh":5124,"gu":37274,"gr":81461,"go":201815,"du":56828,"dw":18508,"dy":56165,"dz":249442,"g ":46202,"ea":22981,"eb":20189,"ec":208980,"ed":135395,"de":106238,"dd":7144,"di":38677,"dk":20908,"dm":14590,"dl":41406,"do":162506,"dn":125390,"dp":5891,"ds":17240,"dr":43512,"ew":111327,"eu":15671,"ev":5608,"ey":6513,"ez":85170,"fa":15386,"h ":260015,"fe":16025,"eg":232853,"ef":12705,"ee":8458,"el":150281,"ek":109837,"ej":291522,"ei":21799,"ep":45340,"eo":16834,"en":264211,"em":150465,"et":92062,"es":172911,"er":319668,"ca":105725,"e ":876959,"by":41730,"bs":19604,"br":47012,"bu":41705,"bn":8407,"bo":39861,"bl":23444,"bi":69531,"be":48450,"dc":10689,"db":7638,"da":129820,"f ":12810,"cz":335815,"cy":107837,"cu":14533,"ct":15111,"co":51690,"cn":22110,"ck":80920,"ci":275500,"cj":115414,"ch":414464,"ce":179393,"c ":34764,"az":93181,"ay":6838,"ba":43868,"d ":139962,"at":193861,"as":161357,"ar":314898,"aw":119971,"av":7852,"au":36829,"ak":99776,"al":185695,"ai":24623,"aj":148717,"ap":33608,"am":123297,"an":453452,"ac":219676,"ad":149992,"ab":34080,"ag":38473,"ah":5887,"ae":10185,"af":22452,"nu":23194,"nt":96712,"ns":36134,"nr":4954,"no":152706,"nn":38516,"ny":249928,"oe":4808,"of":20662,"oc":101234,"od":293823,"oa":8273,"ob":82692,"ię":123546,"om":108472,"on":318937,"ok":137945,"ol":232069,"oi":39360,"ją":114675,"oj":102542,"og":59938,"oh":7680,"ot":70221,"os":163642,"ov":6489,"ou":18871,"op":86592,"oo":8474,"or":254276,"ję":19551,"r ":119247,"ow":537395,"kó":27850,"oz":62053,"pe":42396,"pa":107114,"pc":8415,"pl":33619,"pn":11150,"po":424408,"ph":5594,"ił":21358,"pi":86460,"ką":16568,"iń":19637,"lo":81679,"ln":79071,"lm":9683,"ll":32626,"ls":113022,"hó":9384,"lu":76951,"lt":13727,"o ":461084,"dź":5330,"eś":75573,"mc":10115,"ma":121066,"mb":17385,"me":89660,"mi":333776,"mn":11655,"mm":6313,"mp":24445,"mo":86555,"ms":11588,"mu":41658,"ió":8421,"my":18150,"p ":18753,"na":529472,"nb":5437,"nc":75955,"nd":63112,"ne":249253,"eż":40953,"nf":7374,"ng":54651,"gł":47489,"ni":692138,"ią":60315,"nk":45573,"jw":6916,"ju":20103,"js":79998,"jn":43747,"jo":26898,"jm":12943,"dł":15854,"ki":412466,"ke":10550,"kc":19574,"ka":229229,"m ":343638,"kw":12402,"gó":15509,"ks":37081,"kt":82034,"ku":103176,"ko":241820,"kr":97482,"kl":22672,"km":9933,"eł":15052,"li":209435,"eń":19329,"lk":31238,"le":182455,"ld":11857,"lg":4525,"la":173548,"lc":6468,"lb":16636,"n ":138864,"hr":17682,"dó":7283,"ht":5694,"hu":10241,"cą":8049,"ań":46393,"ał":115023,"hi":35646,"hn":15999,"ho":81700,"id":59864,"ic":201937,"aś":6694,"ib":17573,"ia":315115,"ig":21890,"if":5037,"ie":1136068,"hy":6565,"k ":111343,"ir":31982,"is":139600,"it":68750,"iu":39428,"iv":6169,"iw":23202,"ii":58678,"ij":14562,"dą":5425,"ik":74901,"il":54561,"im":169852,"in":280897,"io":133204,"ip":17035,"aź":5844,"jc":7619,"aż":13250,"je":192771,"jd":20070,"ji":76024,"iz":35250,"l ":52041,"ja":94283,"są":9617,"tó":52700,"wz":9495,"wy":175884,"só":9957,"rę":14692,"z ":208589,"oż":86148,"wi":428233,"pł":15785,"rą":23256,"wk":11864,"wn":80692,"wo":213117,"wr":13531,"ws":100565,"wu":6452,"ró":58140,"y ":457874,"wa":271461,"oś":108251,"wc":28885,"we":125766,"oń":26624,"oł":121062,"vi":10565,"uz":15301,"uw":4835,"ve":14040,"va":7846,"x ":7519,"ui":5984,"uj":65669,"uk":36207,"ul":35319,"ue":10441,"ug":25222,"ur":117881,"us":71220,"ut":47463,"um":45758,"un":53367,"up":30095,"ty":157784,"tz":6072,"tu":85903,"tt":12443,"pó":31075,"tw":127984,"ub":63557,"ua":10045,"ud":56663,"uc":35683,"w ":734288,"to":228890,"tn":41685,"tl":10672,"ts":8380,"tr":134088,"te":190777,"kż":7374,"tk":33051,"ti":40445,"th":19993,"nę":6037,"ta":271329,"su":26228,"ss":16223,"st":440956,"sy":40789,"sz":185716,"sw":7100,"sl":4597,"sk":362869,"sn":10482,"sm":8241,"sp":72745,"so":57024,"sc":108490,"se":50637,"sh":8501,"ną":21831,"sj":10343,"si":154595,"rz":330244,"u ":285894,"mę":6084,"sa":75170,"rr":8977,"rs":83617,"rt":71318,"ru":79284,"nó":11750,"rw":35761,"ry":128901,"rp":13436,"ro":332963,"rn":56890,"rm":40806,"rl":14366,"rk":29749,"ri":79907,"kł":24507,"rg":38504,"rf":5604,"re":197175,"iż":7681,"rd":33902,"rc":41834,"jś":4511,"rb":12273,"ra":338168,"t ":152714,"lę":5780,"mó":8836,"lą":12242,"s ":109715,"kę":4736,"py":18332,"pt":9892,"pu":38310,"pr":207712,"ps":10578,"ył":33312,"yń":8163,"zą":41404,"zę":35066,"uż":24054,"wł":15632,"zó":6325,"wę":7097,"uł":11203,"wą":11216,"wó":72388,"zg":11821,"zi":142936,"sł":33435,"zb":26540,"zc":34117,"zd":18869,"ze":326759,"tę":19569,"za":220009,"yz":12669,"zw":54934,"zy":216261,"zr":6357,"zu":31373,"zt":75735,"zo":93735,"zn":149848,"zp":13878,"zk":54785,"zj":15781,"zm":18594,"zl":6622,"yg":10175,"yf":6219,"yc":219665,"yd":32633,"yb":20679,"yw":52492,"yt":47612,"ys":92766,"yr":14041,"yp":20913,"yn":66941,"ym":103256,"yl":20573,"yk":74193,"yj":45803,"tą":4964,"yż":9849,"yś":4897,"zł":13341,"ów":161116,"ób":6381,"ój":5872,"ód":78413,"óc":5492,"ór":65730,"ól":16369,"ć ":59245,"ąd":23612,"ąc":135745,"ą ":117382,"óż":8608,"ół":34199,"ęb":8256,"ęc":19527,"ęd":32895,"ęg":11685,"ęk":11873,"ęp":16514,"ęt":22906,"ęs":9306,"ę ":96973,"ęz":11858,"ęś":18258,"ęż":7758,"ąg":22092,"ąs":12009,"ąt":12097,"ąz":17763,"ąż":21246,"ł ":59292,"łu":50120,"łt":4875,"ły":40224,"łk":15386,"łn":20167,"ło":189649,"łe":22223,"łc":7515,"ń ":16610,"ła":119873,"ńs":75332,"ńc":31072,"łó":33659,"śl":22153,"śn":19502,"śr":30296,"św":24072,"śc":92180,"Św":8984,"ś ":45778,"ść":40444,"łę":5560,"łą":10057,"źn":6441,"źd":5042,"ż ":20942,"żu":6315,"ży":40208,"żo":72896,"żn":19632,"żs":5910,"że":31935,"ża":27767,"żą":12050,"ąża":15191,"ęci":10430,"ędz":16930,"ędu":4976,"ęzy":9477,"ępu":8551,"ęks":6674,"ęst":6004,"łec":8332,"ła ":38802,"łan":5477,"ład":27517,"łac":9469,"ław":16966,"łas":4900,"łoż":68106,"łoś":19952,"ły ":21321,"ływ":5534,"łu ":9627,"łoń":16090,"łud":14450,"ług":11020,"łow":21447,"łos":9246,"łon":8256,"łno":15191,"łod":6015,"łka":8122,"ło ":21533," Ga":8827," Ge":5971," I ":4633," Fo":4786," Fr":25744," Fi":5918," Ha":11737," He":11280," Go":7898," Gr":14678,"ńcó":5655," Gm":6328," Ho":9424," II":9642," Hi":5632," Je":18244," Ja":16067," In":12303," Ka":25219," Ki":5713," Jo":8117," La":13845," Le":13833," Li":15290," Ko":31759," Kr":17816," Ku":5884," Gó":8232," Ma":41192," Mi":29080," Me":12575," NG":5574," Lo":10195," Lu":9428," Ne":6981," Na":24184," Ni":19681," Mo":18451," Mu":5268," Am":7370," An":13699," Al":15025," Ba":23126," Au":8090," Ar":11761," Be":12343," Bi":14133," Bo":15194," Br":16331," Bu":9050," Ca":12220," Ce":7873,"ńcz":4853," Ci":4561," Ch":21239," Co":11557," Cz":10255,"ńce":15534," Da":8706," Di":5639," De":9843," Dr":5105," Do":16986," Du":4568," Eu":8764," Fe":4580," Wy":14786," Ws":4674," Wo":12135," Wi":28155," We":8391," Wa":15970," Sł":20157," Za":17851," a ":33564," Os":6455," Or":5834," Po":99137," Pi":15394," Pe":7708," Pa":33893," No":17025," Ol":5468," Od":7369," PG":4658," Ob":6845," Ra":13288," Ro":19091," Re":17774," Pr":25548," Pu":5949," Sz":17219," Sy":6223," Su":6907," St":33766," Ta":10078," Th":7821," Te":9806," Tr":10135,"ńst":9625," To":9099,"ńsk":65260," Ru":5052," Sa":22468," Rz":5260," Si":11239," Sc":7241," Se":13251," So":10324," Sp":6969," Sk":5318," Vi":4980," Tu":5482," W ":13679," Un":6885," ja":24161," je":81709," im":8378," in":25684," is":7255," ka":27845," ki":9981," gw":7366," j ":15130," ha":5671," he":5463," gm":81906," gr":42165," go":8008," hi":10476," hr":4991," gł":25015," ni":54044," na":181345," mu":9539," mo":21735," ok":44273," on":4611," og":5234," od":78508," of":7838," ob":36134," no":11764," le":15807," li":34396," la":36395," ku":10205," kt":33550," ks":6999," kw":7071," km":9336," kl":10638," kr":29471," ko":64410," me":15598," mi":86248," o ":18463," ma":52176," lu":42219," lo":7327," ad":8368," am":10571," an":21235," ak":7205," al":15945," au":9619," ar":12965," as":16623," ba":14299," bi":14788," be":7689," bo":7340," by":20334," bu":7038," br":13094,"łuż":7054," ca":5242," el":9782," fa":6215," fu":5153," fr":6875," fo":9292," fi":17828," ge":6292," ga":16786," i ":123913," co":7528," ce":14033," ch":18489," ci":24028," da":17665," cy":4833," cz":55270," do":83523," dn":7531," dl":8990," dr":18598," de":38500," di":6649," dw":10226," du":6950," dz":27524," dy":8440," zm":8262," zo":12577," zn":26389," zw":23230," za":89284," zd":5616," ze":19904," zb":9872," zi":4717," sł":12543," są":8630," z ":107756," wy":79384," wz":8136," wł":13150," uż":7103," ru":4802," ry":6135," rz":19343," sa":14213," se":14117," si":84234," sk":26716," sp":31645," so":11139," ra":15542," re":46718," ro":89060," pu":7297," pr":169964," os":21891," ot":4775," op":16859," or":30563," r ":7257," ję":8026," oz":4712," pe":8239," pa":50520," pl":22703," po":329900," pi":36353," wa":17275," we":29944," wc":10065," ró":17486," wr":10056," wo":81271," ws":27715," wi":85470," pł":9284," w ":591873," ty":20926," tw":5981," pó":15190," tu":7399," us":6046," ut":8073," ur":32556," uk":7357," ul":5488," ta":20050," sw":5466," sz":26144," sy":14406," st":72132," su":7093," tr":22707," to":40581," th":4944," te":42334,"łów":28362," ła":6906,"łąc":7474," Św":8979," śr":24528," św":19258," że":5562," ży":7572,"GC ":12479,"Eur":7453,"Fra":21145,"II ":13664,"Her":4920,"Gra":4795,"Gmi":6178,"Bar":4693,"Bra":4520,"Cha":6684,"Dol":5831,"Nie":14925,"Now":7712,"Nor":6043,"PGC":4549,"Pie":5284,"Par":14525,"Poł":4889,"Pro":6911,"Prz":9834,"Pod":4669,"Pol":63410,"Pow":4846,"Rad":5296,"Jan":4677,"Jes":5251,"Kar":5182,"Kon":5190,"Kra":7526,"Koś":5328,"Gór":8184,"NGC":5540,"Mar":15306,"Mon":4650,"Mie":5310,"Mis":5057,"Wys":5223,"Wie":13213,"War":8050,"Sło":18142,"Str":5084,"Sta":19220,"Sie":4994,"Sai":4713,"Sch":5152,"Ros":4617,"Uni":6296,"The":5470,"bio":14193,"bli":16339,"bor":8274,"bow":6707,"bar":9707,"bec":8656,"ber":14109,"bel":6901,"bez":6239,"bia":7547,"bie":21077,"ca ":84053,"cac":4624,"był":19691,"ce ":116313,"bro":10887,"bra":11568,"bry":5366,"brz":9909,"bsz":5466,"bst":6710,"bur":11294,"bum":8181,"bud":9663,"by ":10708,"aka":5857,"am ":8178,"akc":4640,"aki":8717,"ajo":5413,"ajm":4898,"ajw":6681,"aju":12289,"al ":15001,"aja":6999,"ajd":19542,"aje":5093,"ain":11213,"ak ":12839,"aj ":9008,"agr":6880,"ago":6833,"ają":56423,"anu":7044,"any":57351,"ano":19191,"ann":5995,"ant":15544,"ans":10106,"ane":52267,"ang":21057,"ani":105534,"ank":9718,"ana":52659,"anc":30280,"and":28542,"amo":13297,"amp":4556,"ami":37349,"ame":32849,"ama":9031,"alo":8151,"aln":38047,"all":6281,"ali":35299,"ale":28164,"ala":17506,"alb":10365,"an ":25897,"akt":18470,"ako":21085,"abi":5944,"abs":7005,"ae ":4965,"ad ":24951,"ac ":5802,"afi":15590,"aga":5833,"ado":9262,"adm":7603,"adi":7361,"ade":12127,"ady":8896,"adz":20389,"ack":10335,"acj":42190,"aci":10932,"ach":91172,"ada":31955,"acz":24570,"acy":17521,"azo":11989,"azu":9191,"azw":14194,"aza":4626,"azd":9097,"azy":7482,"az ":24099,"ba ":9607,"at ":32096,"are":9332,"ard":17168,"arc":24556,"ara":29532,"aro":21478,"arn":16904,"arm":8931,"arl":5792,"ark":13034,"ari":18485,"aru":4729,"anó":4796,"ars":25559,"art":41017,"asa":22529,"ary":12822,"arz":32133,"asi":8550,"aso":5103,"ask":9308,"ar ":9731,"api":8704,"apo":6161,"as ":15649,"aut":8273,"awa":20666,"aws":12931,"awn":15933,"awo":12547,"awi":30734,"asz":13814,"ata":29924,"ść ":40186,"ast":57235,"asy":6947,"atk":6138,"atr":8063,"ato":27910,"ate":18972,"akż":7374,"ati":9455,"aw ":7482,"atu":20800,"aty":21623,"auk":5346,"Świ":8845,"ści":91962,"śni":16799,"śli":5074,"śro":7633,"śre":20628,"ślą":5357,"świ":21339,"jeg":6331,"jej":4751,"jed":34575,"jen":6642,"jew":58578,"jes":38109,"ji ":74801,"jal":7148,"jak":18567,"jaw":9057,"aźd":4510,"je ":30364,"jdu":17294,"jmu":7356,"jna":5939,"jny":19293,"jne":11196,"jow":13446,"jon":8504,"ito":6446,"ity":14838,"isk":17265,"ist":59047,"isz":10261,"ita":8740,"ite":12873,"iwe":4858,"iwi":4661,"ius":4598,"ium":6118,"is ":13674,"ion":48352,"iop":4645,"ior":18228,"ios":6430,"iot":6025,"ipc":5025,"ikó":5855,"iow":26535,"isi":4518,"isa":8456,"iu ":24955,"ire":4546,"ira":6326,"ja ":38334,"izy":4891,"izo":6649,"izm":4986,"iza":9287,"kim":128411,"kic":24931,"kie":126958,"dłu":9193,"km ":8293,"ki ":116429,"kań":16086,"kcj":11696,"kra":24036,"kre":18070,"kry":8710,"krz":7284,"ku ":72509,"kro":6037,"kow":64383,"kos":4831,"kor":8298,"kop":10012,"kon":28370,"kom":18077,"kol":18048,"kok":4910,"klu":5529,"ko ":39683,"kle":4640,"kla":7237,"jsk":40065,"jsz":8170,"ju ":15477,"jsc":28565,"kaz":5339,"kat":10533,"kar":21380,"kan":15572,"kal":9168,"kam":6459,"kad":5909,"kac":12564,"ka ":110422,"ha ":5601,"han":8170,"har":13309,"he ":11732,"her":6722,"ał ":24279,"cą ":7445,"ań ":4989,"ałe":8027,"ała":29091,"ało":22494,"his":8897,"ały":11839,"ańc":7994,"ańs":33252,"go ":160963,"god":4635,"gni":4919,"gmi":81864,"gos":5577,"gor":5388,"gow":8194,"gu ":24037,"gro":9171,"grz":5388,"gry":6845,"gru":20718,"gra":31694,"gwi":7275,"ców":11188,"iaj":4940,"iam":4548,"ial":8076,"ian":22005,"ias":25407,"iar":9378,"iat":20282,"ic ":6481,"iac":9488,"iad":9916,"iaz":7546,"id ":16421,"ia ":151774,"iet":14221,"iew":12649,"iel":59211,"iem":53337,"ien":34053,"ier":72550,"ies":27602,"ied":28632,"ieg":53169,"iek":15875,"iej":135012,"iec":108952,"icy":13868,"ict":6284,"icj":4509,"ick":18455,"ici":5695,"ich":42480,"ice":16432,"ie ":425948,"ica":18194,"ide":5884,"ida":21189,"icz":59703,"ijs":6419,"im ":137877,"ika":22144,"ii ":57658,"ibą":4979,"iał":33757,"ik ":16857,"imp":4698,"imi":11049,"inc":8772,"ind":7280,"ina":57100,"inn":9746,"ino":8623,"int":11381,"ins":5688,"ine":11157,"ież":14661,"ing":19340,"ini":80940,"iny":19314,"iko":7564,"iki":9122,"in ":24060,"ilo":5237,"ill":10795,"ień":7511,"ilm":5161,"ili":7909,"ieś":51901,"io ":7198,"how":11191,"hol":4719,"hor":6372,"hod":42292,"hni":9648,"hra":5522,"dów":6742,"fia":9588,"ewó":56727,"ez ":38609,"ews":7573,"eze":8964,"ezi":7185,"ezj":6360,"eta":10069,"etn":9079,"esp":11621,"eso":4594,"est":61386,"esz":24722,"eto":22583,"etr":10559,"ety":8345,"ew ":5885,"ewi":12221,"ewo":9270,"ewn":8260,"ewa":5038,"er ":34400,"epa":15408,"eją":6188,"es ":24102,"epu":5035,"epr":8058,"eri":17872,"erg":8537,"ere":16329,"erc":5702,"era":33404,"erb":5967,"et ":10917,"esk":6857,"esi":11275,"esa":4705,"erz":21989,"ery":23381,"eru":5751,"erw":27139,"ert":7815,"ers":24326,"ern":18633,"erm":6952,"erp":6669,"ero":30451,"eki":4817,"eko":5980,"eks":10430,"ekt":18859,"eku":6941,"en ":29550,"ela":11200,"ele":25422,"eli":15142,"eln":10771,"elk":19485,"ell":8172,"elo":10644,"elu":7577,"els":11724,"emc":9224,"ema":7081,"eme":7011,"emo":5689,"emi":25674,"ene":12261,"ena":8388,"end":7106,"enc":29197,"eno":4952,"enn":11782,"enk":5319,"eni":82488,"egł":18236,"ens":6693,"ent":47936,"ego":161252,"egi":24719,"ej ":196844,"egu":4666,"ek ":37209,"ein":6239,"el ":16434,"ejs":51321,"ejo":11481,"ejm":7156,"eje":7376,"eka":10766,"em ":73512,"gio":21748,"gie":12191,"gic":6767,"gii":7756,"gia":4982,"bą ":6293,"gi ":9182,"gen":12957,"gel":5153,"gar":6319,"gat":9377,"gaj":4881,"gal":8113,"gan":14815,"ga ":14947,"fun":4925,"fra":5853,"for":16996,"fic":8573,"fil":7378,"fik":6148,"ać ":7866,"czą":12269,"da ":43439,"czł":5966,"de ":15481,"dby":5063,"czę":22437,"dal":5635,"daj":6034,"dar":8045,"dan":24834,"daw":15145,"dcz":6407,"ctw":8655,"cy ":54622,"cus":4654,"cym":5360,"cyj":21911,"cyc":12366,"cz ":10143,"czy":47263,"czk":8277,"czn":94844,"czo":18714,"cza":38835,"cze":61835,"cki":56501,"chó":9144,"co ":6133,"cni":7419,"cią":23459,"cne":4800,"cny":4546,"cow":30428,"cję":4623,"cez":4969,"ch ":250098,"cer":7119,"ces":6923,"cen":13171,"cej":5254,"cel":7686,"ceg":6128,"ci ":63750,"cha":21398,"cia":18501,"cie":119729,"che":16574,"chi":12341,"cho":63783,"chn":12122,"chr":6017,"ciw":5743,"cja":35582,"ciu":5042,"cin":7038,"cio":9246,"cka":9993,"cji":61836,"ed ":7038,"ebi":5142,"ec ":12319,"dzą":8566,"ega":9608,"edl":5574,"edn":51333,"ede":12015,"eda":5582,"edz":17029,"edy":9486,"eds":6095,"ecj":5624,"eck":33379,"ech":31282,"eci":73659,"ece":7263,"ecz":26497,"ect":5183,"ecn":7076,"dyn":9974,"dys":5848,"dyc":5708,"dy ":20176,"dzy":15488,"dzt":56173,"dzo":7609,"dzk":13570,"dzi":114188,"dze":17343,"dza":11866,"dor":6162,"dom":7183,"dol":8096,"dok":4665,"doz":5829,"dow":41632,"dos":5363,"dmi":12552,"dna":7598,"dne":6153,"dni":85396,"dno":17791,"dny":5925,"dob":7583,"dst":8461,"duj":19180,"duk":7030,"dra":4842,"drz":4777,"du ":16409,"dro":12893,"dru":8195,"dia":9387,"der":13407,"del":4853,"dek":8141,"den":21584,"dem":8020,"dep":15043,"dle":22301,"dla":14281,"dko":7298,"do ":56353,"dio":7032,"die":6315,"rga":13673,"rgi":6819,"ret":5375,"res":16392,"rez":11929,"rg ":8389,"rdz":8372,"rea":6346,"rec":6681,"red":24223,"rej":11845,"reg":27968,"rem":8867,"ren":18517,"rek":5432,"rep":5124,"rcz":7138,"re ":17858,"rci":7329,"rch":9881,"rce":4547,"rca":6614,"raw":21807,"raz":29604,"rd ":6953,"ras":7862,"rat":14245,"raj":22570,"rai":5724,"ran":52894,"ram":15553,"ral":18585,"rak":14728,"rab":9860,"raf":17050,"rad":12311,"rac":33149,"rpn":4743,"ros":15610,"rot":7718,"rom":11621,"ron":23271,"rop":15650,"roz":22873,"row":46481,"rob":7374,"rod":59324,"roc":14309,"roj":6342,"roi":15518,"rol":7056,"rok":41872,"rog":11103,"rno":5279,"rny":6200,"rna":10748,"rne":9659,"rni":18640,"ro ":7330,"rma":13967,"reś":7847,"rmi":12636,"rla":5043,"rki":6506,"rka":6181,"riu":5278,"rii":14143,"rin":4886,"ria":19003,"kła":22407,"ric":4906,"rie":5586,"rwo":5268,"rws":10740,"nów":11177,"rz ":14438,"ryb":4752,"ryc":13818,"rug":6274,"rud":6357,"ruc":5354,"rup":16289,"run":6993,"rum":5890,"ruk":5290,"rus":7391,"rwa":7707,"rwc":4554,"ry ":30832,"rsk":40769,"rsz":11669,"rta":19705,"rst":9077,"rto":10205,"rte":6255,"rti":6031,"rtu":4870,"rty":7882,"rt ":8136,"ru ":8305,"rzę":9880,"sad":8969,"sam":12218,"san":8261,"sar":4951,"sa ":26422,"rzą":15714,"rze":166268,"rza":14480,"rzc":6541,"ryw":7519,"rys":7800,"ryt":14172,"ryk":17536,"rym":6312,"ryn":5422,"rzy":76075,"rzo":14189,"ną ":19254,"si ":6951,"sie":37561,"sia":7216,"sk ":6051,"sin":6607,"se ":4707,"sce":55140,"sch":21250,"sco":24974,"ser":11407,"sen":9224,"spo":28928,"spr":6284,"spe":6485,"spi":8532,"sow":15205,"son":7183,"sok":5885,"się":66198,"sob":7483,"su ":8612,"skł":11877,"st ":49786,"ski":241685,"sko":33991,"skr":5762,"sku":9283,"ska":44791,"sią":6562,"sz ":5749,"syn":5333,"sys":6083,"sza":29086,"stę":16122,"sze":30322,"szc":22167,"szp":5112,"szo":8648,"szt":14601,"szk":19152,"szy":32568,"ste":40839,"sta":110434,"stn":10874,"sto":55229,"sti":7248,"stk":11925,"stu":7827,"spó":13799,"stw":35135,"str":55283,"sty":30683,"sy ":5848,"tak":16147,"tal":13415,"tac":30341,"tad":4771,"taw":14561,"tat":11340,"tar":27950,"tan":40782,"tam":18213,"te ":12262,"ta ":47407,"jęz":8303,"pa ":7030,"ową":6832,"ję ":5820,"pca":4745,"par":31402,"pas":19500,"pac":4930,"pad":8610,"pal":4646,"pan":7407,"ką ":14644,"pań":7582,"pec":4830,"per":13425,"pej":4544,"pla":21906,"ple":4628,"ińs":18739,"pie":29091,"iłk":7468,"pio":5728,"pir":4743,"pis":14610,"poz":13437,"pow":94565,"por":15922,"pop":9036,"pot":7235,"pos":20600,"poj":4594,"pom":22714,"pon":6930,"pok":5372,"pol":56274,"poc":15903,"pod":44935,"pił":8661,"po ":9878,"pni":8916,"pub":6917,"pra":25179,"prz":116916,"pu ":5270,"pre":13822,"pro":44748,"put":4642,"puj":8407,"poł":82723,"py ":16720,"ląs":7826,"mów":5303,"ra ":38610,"ngi":4912,"ni ":23063,"nge":9484,"ią ":10996,"neg":51936,"nej":47963,"nek":13069,"nem":5441,"ner":9906,"net":21971,"nes":4536,"ndy":5147,"ng ":20546,"eży":7778,"nci":16716,"ncj":34860,"nce":9629,"ne ":77120,"eż ":13347,"ndr":6665,"ndo":5824,"ndi":8916,"nde":9089,"nda":6965,"nak":5404,"nal":20391,"nam":4778,"nan":15192,"nap":5117,"nar":14632,"nac":26022,"nad":18221,"nag":7300,"naj":37748,"nd ":10560,"nau":6824,"nat":12135,"nas":8986,"naz":14802,"naw":6217,"na ":315412,"moż":6857,"nyc":66044,"ny ":143123,"noś":21837,"nty":9827,"nto":7930,"ntu":6194,"ntr":10458,"nta":17677,"nte":16419,"nst":10393,"nu ":12191,"nt ":17005,"ns ":4556,"noc":23507,"nom":6299,"nos":13706,"nor":4839,"now":33236,"nne":10038,"nna":4663,"nni":6637,"nię":6603,"nny":11159,"głó":21438,"no ":23520,"nki":7495,"nkc":4543,"nka":8221,"nku":6543,"nko":6939,"eżą":8181,"iąg":20115,"iąz":16848,"nii":15486,"nie":303754,"nic":58316,"nia":115515,"niz":10946,"niu":18651,"nis":21291,"nio":33347,"gło":21325,"nim":8023,"nin":5902,"nik":44866,"ogr":14773,"ogi":15434,"ogo":4557,"oga":7462,"ją ":20069,"oid":31094,"ok ":8415,"ojs":6215,"ojn":8775,"oje":68149,"jąc":92228,"oce":7209,"och":19137,"oci":8093,"ock":7255,"ocn":14524,"obs":6969,"oby":5467,"ode":8229,"odk":11049,"odl":22987,"odo":21990,"odp":5300,"odn":35244,"ods":5690,"odr":4605,"ocz":25360,"of ":6324,"odc":6738,"odb":6509,"oda":13640,"odz":61637,"ody":9005,"odu":13447,"ofi":6001,"ięk":9725,"ięc":10576,"ięd":13216,"ięt":12440,"od ":47389,"obo":8696,"obr":12863,"obl":4516,"obn":4740,"obi":10703,"obe":11907,"nym":35914,"ię ":65182,"owy":75588,"osó":4510,"ków":24138,"ows":40460,"own":11322,"owo":53873,"owi":138750,"ozy":6621,"ozw":5341,"ozn":9119,"osł":6618,"ozb":6448,"oty":9500,"ote":7310,"otr":5518,"oto":11888,"otn":8681,"osz":15296,"ost":51897,"ota":6378,"osi":14132,"osk":7450,"ose":9237,"osp":5848,"oso":13740,"oró":5598,"owc":6201,"owa":124010,"owe":69144,"opo":27225,"opi":9321,"ope":10070,"opa":11132,"os ":8863,"opu":5185,"opr":8595,"or ":15758,"ork":4605,"orm":16889,"orn":6157,"oro":17951,"orc":4621,"ord":8495,"ore":9264,"org":11803,"ori":12364,"osa":8380,"ort":15205,"ors":22187,"oru":7711,"orz":34348,"ory":13310,"ora":31522,"ola":10017,"on ":32944,"oli":42609,"ole":26115,"ols":92465,"oln":14866,"olo":22365,"olu":5605,"oka":12522,"om ":7335,"oki":5566,"okr":39604,"oko":19282,"ogó":4810,"oku":40113,"ona":86386,"ond":4877,"one":29526,"oni":52759,"onk":6206,"onn":4626,"ono":16680,"ons":8848,"ont":10876,"onu":6969,"ony":38958,"oma":14351,"ome":9149,"omi":22956,"omp":9870,"omo":23578,"omu":5636,"la ":29248,"le ":27634,"lac":9698,"lak":7036,"lan":38327,"lam":5773,"lar":10967,"lat":37023,"las":14320,"ld ":4729,"lbu":8442,"koł":12944,"kul":5213,"kuj":7417,"kwi":7243,"krą":16281,"koś":9426,"kró":6396,"kte":5141,"ksz":11351,"ksi":5778,"kty":11231,"ktr":5362,"ktu":7021,"kto":8566,"krę":7083,"gól":4564,"gór":7545,"któ":37445,"lok":4711,"lon":16670,"log":15244,"lot":5895,"low":11925,"lno":13692,"lni":16273,"leż":15711,"lne":15681,"lny":18798,"lna":11882,"lud":5047,"lub":38824,"lsk":52175,"lu ":15294,"lsc":50391,"li ":18475,"lew":7878,"les":10216,"let":6569,"ler":6678,"lem":11032,"len":14544,"lek":14961,"lej":10577,"leg":25572,"lec":7985,"lla":5033,"lle":8878,"lli":5642,"lko":12509,"eńs":10006,"lka":5102,"lki":9010,"ll ":4740,"lit":23937,"lis":17192,"lip":7716,"lin":28801,"lim":6337,"liz":9545,"liw":7216,"lic":38891,"lia":7515,"eń ":8305,"lik":8729,"lii":5868,"ma ":13583,"mac":9833,"eś ":42823,"maj":8752,"mar":13792,"mas":6086,"mal":6141,"man":12874,"maz":13085,"mat":15538,"me ":5269,"eśc":9065,"eśn":12795,"eśl":8074,"mcz":8811,"met":12557,"mer":18450,"men":33777,"lut":5364,"hód":6854,"mpi":5561,"miń":6040,"moc":8677,"mod":4841,"mon":7419,"mow":13205,"mor":22691,"mu ":12663,"msk":7954,"my ":8132,"muj":8914,"mun":5245,"muz":6245,"mał":6431,"mi ":36519,"min":109517,"mis":7482,"mit":5081,"mic":8508,"mia":35513,"mie":85656,"mię":17160,"mni":5567,"wą ":9465,"wód":59175,"wór":6020,"źdz":4969,"zta":5990,"ztw":56537,"zu ":8060,"zuj":5309,"zur":6895,"zy ":52536,"zwa":16748,"zwi":19757,"zwy":6194,"zyw":7464,"zys":27664,"zym":15419,"zyn":18487,"zyk":17972,"zyl":5653,"zyj":5225,"zyc":26077,"zyd":4698,"zył":4808,"zi ":13438,"zał":9317,"zgr":4705,"zec":34126,"zed":16305,"zeg":13280,"zej":12005,"zeb":5093,"zdo":9053,"zes":26033,"zez":38687,"zew":14699,"zen":40191,"zem":9711,"zel":6475,"zek":13631,"zer":16763,"ze ":43444,"zch":5824,"zbi":11943,"zcz":25210,"zac":28997,"zaw":17852,"zaj":18917,"zam":9131,"zan":18287,"zak":6986,"zal":7485,"zar":15092,"zap":6541,"zas":21035,"zny":34365,"zos":17618,"zon":31868,"zow":26765,"zpo":4827,"ześ":12448,"zmi":5909,"zna":56249,"zno":5266,"zne":31393,"zni":17047,"zm ":5171,"zka":12896,"zko":15029,"zki":16987,"zeń":5484,"zib":8100,"zia":21275,"sła":11498,"zie":58529,"zin":14757,"sło":8768,"zio":8015,"słu":9392,"zja":5084,"zji":8382,"yty":7484,"ytu":9487,"yto":6663,"źni":4961,"yte":4862,"yta":5297,"ysz":7085,"yst":44231,"yso":5982,"ysp":7012,"ysk":11600,"tęp":15379,"za ":37664,"yzn":5243,"ywa":26508,"ywi":7128,"ywn":5067,"yce":6751,"ych":133947,"yci":10363,"ycz":50193,"yda":15003,"żaj":15742,"yck":5423,"ycj":6652,"że ":16017,"yjs":11565,"yka":23821,"ym ":69059,"yki":7812,"ykl":4610,"yko":9261,"yn ":8183,"yli":7761,"ymi":14584,"yms":4916,"yna":17425,"yni":13954,"yno":4822,"żen":5078,"yk ":13655,"yjn":27661,"tów":16736,"tór":35547,"są ":7387,"ożo":65228,"oży":4895,"oże":6845,"wy ":42246,"wsp":8605,"wsz":18473,"wst":10921,"wsc":10075,"wsk":48535,"rąż":16283,"wys":19278,"wym":21711,"wyk":10398,"wyn":5788,"wyd":14033,"wyb":6431,"wyc":28770,"sów":5741,"woś":26886,"wo ":32774,"wna":7537,"wne":21421,"wni":31749,"wią":20802,"wno":4837,"wka":4779,"wrz":6674,"wod":23078,"wię":20120,"wny":9160,"wow":8334,"wor":17209,"wol":5953,"woj":77026,"wcz":7095,"ośl":6671,"wch":7307,"we ":41958,"ośc":57589,"wca":6919,"wer":9502,"wej":33041,"weg":17678,"wed":4775,"wał":10230,"ość":33539,"wi ":5607,"pły":9060,"wis":10022,"wiz":5342,"wie":266248,"wid":6688,"wic":20177,"win":12633,"wia":38637,"wa ":75032,"wan":92067,"wal":11772,"waj":5843,"wat":12396,"war":32065,"wac":6719,"róż":7313,"wad":9110,"rów":28327,"ród":6783,"ról":5670,"oła":8333,"oło":72193,"ołe":8524,"ołu":20038,"ońc":21153,"ver":4892,"uzy":7699,"usk":8960,"usz":14475,"ust":14884,"ute":10402,"utw":6376,"uto":11126,"us ":13666,"ura":9619,"ure":5261,"urg":10533,"uro":16796,"urs":7537,"ury":7673,"urz":9377,"ują":40031,"upa":5160,"ur ":29893,"upy":9651,"umi":4996,"ume":5036,"unk":16732,"uni":8824,"und":6279,"une":6969,"uko":7594,"um ":20351,"uka":4715,"ult":6004,"uli":7259,"ula":8673,"uje":18646,"uja":4953,"ugi":6634,"ucz":7047,"udn":21514,"uch":15770,"udo":10709,"udz":6914,"ub ":30727,"pół":22236,"ubl":8802,"ube":7781,"tyw":7322,"tyj":4614,"tyk":20395,"tyl":6759,"tym":7939,"tyn":8462,"typ":6891,"tys":5969,"tyt":6677,"twó":5271,"ty ":25638,"twa":27679,"tur":19565,"tun":9184,"tyc":48848,"two":26167,"pól":4883,"twi":61091,"tre":5469,"tra":36373,"tri":6420,"tru":11588,"tro":23044,"trz":32044,"tu ":25361,"try":9123,"to ":54716,"tni":28308,"toc":7538,"toi":15874,"tos":10191,"tow":38410,"tom":6513,"ton":10929,"tok":8683,"tol":14627,"tor":34275,"top":9511,"tin":4571,"tio":8819,"tki":10465,"tko":7816,"tka":9207,"tle":5081,"tem":19837,"ten":10208,"tej":7771,"tek":9091,"tel":10074,"kże":7372,"teg":14724,"tec":7568,"ter":68917,"the":6053,"tał":27215,"żąc":10258,"zło":10911,"ył ":11114,"ży ":9032,"zęś":17553,"życ":6860,"yła":8647,"żyn":4577,"żyw":6989,"yły":6907,"yńs":6994,"żni":7269,"żon":68291,"zęd":7177,"zęs":4784,"ższ":5421,"ząd":13275,"ząc":15081,"zą ":5842,"ząt":4518,"uży":12473,"zów":5144,"wła":9785,"óżn":6877,"ół ":8476,"ółn":14744,"ów ":112668,"óra":6553,"óre":15215,"órn":8319,"óry":15904,"ór ":5776,"ówk":5666,"ówn":37744,"ódz":62547,"ód ":10612,"ób ":4595,"óln":9187,"ągu":15768,"ądz":6409,"ącz":8602,"ący":47223,"ące":25280,"ąca":47593,"ązk":9275,"ąza":4528,"ąsk":8772,"ątk":5004},"n_words":[44927968,50956492,36530760],"name":"pl","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/pt.json b/contrib/languages-data/pt.json
new file mode 100644
index 0000000..5203a33
--- /dev/null
+++ b/contrib/languages-data/pt.json
@@ -0,0 +1 @@
+{"freq":{"D":84998,"E":149956,"F":100967,"G":92106,"A":261572,"B":132089,"C":229572,"L":100908,"M":162595,"N":83569,"O":115743,"H":53006,"I":86169,"J":66643,"K":27233,"U":54108,"T":95611,"W":30243,"V":63770,"Q":7457,"P":198289,"S":200312,"R":102796,"Y":7968,"X":13707,"Z":9504,"f":420930,"g":646820,"d":3123972,"e":5421795,"b":522239,"c":1736812,"a":5855900,"n":2824771,"o":4680584,"l":1547279,"m":2105049,"j":94292,"k":164391,"h":490405,"i":3472398,"w":44026,"v":454134,"u":1899338,"t":2255026,"s":2945228,"r":3035316,"q":191465,"p":1075380,"z":175309,"y":94919,"x":113550,"²":81071,"É":13693,"Á":9519,"í":209507,"ê":67610,"é":451892,"è":5136,"ç":214936,"ã":317848,"â":37738,"á":187832,"à":26465,"ü":5538,"ú":40758,"ô":23605,"õ":33734,"ó":129413," l":122890," m":228220," n":368310," o":307851," h":135081," i":114948," j":44060," k":90787," d":1636882," e":668130," f":265709," g":86172," a":553690," b":90195," c":638137," u":471782," t":166023," v":73530," q":119308," p":552163," s":358148," r":175170," J":65167," K":25787," H":50638," I":64806," N":77349," O":109788," L":96197," M":155484," B":122895," C":211790," A":232527," F":95595," G":81500," D":78496," E":140343," Z":9058," Y":7429," X":9721," S":182999," R":96218," Q":6965," P":187689," W":28455," V":54881," U":50226," T":85637," á":66191," à":25954," é":287636," ú":5989," Á":9484," É":13626,"A ":78627,"Da":9542,"Cu":6462,"Cl":8427,"Co":62457,"Cr":10302,"Ce":12150,"Ch":23975,"Ci":8714,"Ed":5016,"Do":10375,"De":23490,"Di":17511,"GC":5835,"Fe":14358,"Fa":9224,"Eu":7410,"Es":76180,"En":6707,"Em":8917,"El":11312,"Ge":12507,"Ga":11449,"I ":12898,"Fu":6893,"Fr":18796,"Fo":24543,"Fl":6042,"Fi":9055,"C ":14112,"Au":8700,"Ar":20685,"At":7248,"As":11784,"D ":5749,"Ba":33758,"Ag":5894,"Ab":6380,"Ac":5079,"Am":13589,"An":22486,"Al":38766,"Bu":7329,"Br":31392,"Ca":66526,"E ":5464,"Bi":7032,"Be":17271,"Bo":16310,"Le":17360,"Li":18877,"La":20252,"Lu":9478,"Lo":21075,"Me":18166,"NG":5897,"Mi":22675,"O ":59245,"Ma":62167,"Mu":14513,"Mo":23160,"Ni":6791,"Ne":10091,"Na":16341,"P ":5903,"No":30608,"Ol":8785,"Gi":5566,"Gr":19098,"Go":9769,"Gu":13341,"Ha":12782,"He":10653,"II":8816,"Hi":6333,"Ho":11804,"In":21460,"Ja":20297,"Je":5208,"Jo":23876,"Ju":10807,"Ka":6332,"Um":5798,"Un":17690,"Tu":6480,"Tr":12997,"To":12084,"Th":10714,"Ti":5247,"Te":19234,"Ta":10759,"UA":15545,"V ":6036,"St":11159,"Su":24311,"Wi":7399,"Sã":16301,"Wa":7041,"Vi":19753,"Va":11369,"Ve":14368,"Pr":23453,"S ":9133,"Pe":21497,"Pa":48708,"Pl":5522,"Po":57527,"Pi":15167,"Os":8721,"Or":9792,"Se":27899,"Sc":6387,"Si":12578,"Sh":5677,"So":14955,"Sa":39239,"Re":33199,"Ri":19155,"Ro":19693,"Qu":6140,"T ":5229,"Ra":8949,"b ":50847,"a ":2181662,"i ":168232,"ge":70306,"ga":87148,"bé":16913,"fl":8986,"fi":74478,"fr":59032,"fu":27785,"fo":117850,"dá":5256,"he":73854,"ha":210111,"gn":21302,"cê":5480,"gl":18191,"cç":8441,"gi":116568,"gh":7414,"gu":120239,"gr":61974,"cí":25911,"go":74169,"du":48733,"g ":26553,"ea":109055,"eb":27192,"ec":114155,"ed":73900,"de":1384296,"di":203085,"dm":36748,"do":671969,"ds":6146,"dr":29810,"ew":5596,"ex":56960,"eu":52678,"ev":46416,"ey":9665,"ez":25288,"fa":47754,"h ":22234,"fe":50703,"eg":161615,"ef":22289,"ee":12045,"el":215678,"aç":106998,"ej":12312,"ei":173784,"ep":64753,"eo":27810,"en":615409,"em":301673,"et":118849,"es":604827,"er":508227,"eq":14954,"aí":13564,"ca":322979,"e ":2017914,"bs":7191,"br":100433,"bu":28485,"bo":46844,"bl":22439,"bi":100249,"be":49470,"da":652111,"f ":9921,"cu":52185,"ct":31468,"cr":50241,"co":536282,"ck":15123,"cl":33604,"ci":328384,"ch":65018,"ce":215952,"c ":14270,"az":18902,"ay":13866,"ba":74690,"d ":43293,"at":195898,"as":441529,"ar":395911,"aq":7417,"av":49201,"au":56223,"ak":8531,"al":413966,"ai":134662,"aj":6551,"ao":31258,"ap":60289,"am":235425,"an":557322,"ac":100712,"ad":561485,"ab":137795,"ag":58484,"ah":7769,"ae":25110,"af":14102,"nu":24993,"nt":522402,"ns":188107,"no":344725,"nn":15103,"nz":5976,"ny":7562,"nv":17155,"oe":16285,"of":27077,"oc":103353,"od":80574,"oa":21115,"ob":40172,"om":360608,"on":341990,"ol":141927,"oi":109737,"oj":7341,"og":52000,"oh":5263,"ot":49005,"m²":81019,"os":507875,"ov":77341,"ou":126561,"op":64022,"oo":11470,"or":489375,"r ":303972,"ox":5797,"ow":11301,"oz":5717,"pe":192004,"lá":12510,"pa":207724,"pl":31072,"lé":9270,"lê":6741,"po":262183,"ph":9247,"pi":69522,"lo":175844,"lm":34639,"ll":48668,"ls":10434,"lp":6052,"lv":20781,"lu":42856,"lt":38629,"ly":6623,"o ":2000634,"ma":495019,"mb":74173,"iã":65284,"me":284803,"iá":6420,"iç":26049,"mi":135455,"mm":5698,"mp":92721,"mo":132550,"mu":111484,"p ":8620,"na":437221,"nc":193122,"nd":299787,"ne":119195,"nf":17343,"ng":76996,"nh":95890,"ni":224974,"nj":6755,"nk":6105,"ju":23595,"fí":5753,"jo":21637,"ki":12050,"ke":10814,"ka":8676,"m ":610401,"ko":5815,"gé":5229,"km":88160,"gê":12280,"li":257137,"lh":65430,"le":173629,"ld":17597,"lg":14230,"lf":5772,"la":234272,"lc":7172,"lb":14242,"n ":112737,"ht":6380,"hu":13446,"hi":44146,"dê":5692,"ho":87126,"dé":6382,"id":307392,"ic":326922,"ib":24713,"ia":390572,"ig":76460,"if":27051,"ie":76108,"k ":20205,"ir":178791,"is":322059,"it":260738,"iu":12350,"iv":114919,"ix":17582,"ik":5904,"eç":8830,"il":161901,"im":114838,"in":395415,"io":223313,"ip":55696,"je":12713,"iz":80563,"l ":281164,"ja":26393,"xi":25832,"tê":5442,"xo":8355,"té":18873,"xp":7291,"tí":13967,"tó":26791,"xt":10508,"z ":28371,"xc":16696,"xa":14045,"tã":10541,"tâ":11963,"tá":22795,"xe":8953,"sã":39079,"wi":7047,"sé":18935,"sí":5599,"ró":40309,"y ":44219,"wa":10938,"sá":5639,"ré":9925,"rç":11146,"vi":94338,"rã":8523,"vr":9835,"rí":29707,"rê":8332,"vo":58913,"uz":14799,"ux":5541,"uv":5858,"ve":130840,"rá":18536,"va":112071,"x ":15240,"ui":98859,"uj":5584,"ul":126725,"ue":171782,"ug":35554,"ur":125836,"us":87304,"ut":96190,"um":493013,"un":209874,"up":26206,"ty":5162,"tu":132302,"tt":15069,"pó":5919,"ub":41087,"ua":116112,"ud":26863,"uc":23327,"w ":7530,"pú":7028,"to":326748,"pé":22989,"tl":9288,"ts":8324,"tr":260066,"te":557844,"ti":259190,"th":25055,"ta":456660,"su":99840,"ss":133040,"st":452789,"sl":7670,"sk":8628,"sm":23244,"sp":74972,"so":134893,"sq":6495,"sd":6659,"sc":72024,"se":292294,"sh":13868,"si":227032,"u ":130375,"sa":133204,"sb":7374,"rr":74864,"rs":43158,"rt":193480,"ru":48286,"rv":18129,"ry":9625,"ní":7223,"rq":17132,"rp":10793,"ro":344915,"rn":53960,"rm":73617,"né":7825,"rl":15891,"nç":36309,"rk":7248,"ri":409856,"nã":10618,"rg":47427,"rf":6634,"ná":10363,"re":480208,"rd":53701,"rc":51264,"rb":27665,"ra":596837,"t ":61702,"mú":6929,"qu":189373,"mí":22028,"mé":15139,"má":8216,"mã":9155,"s ":1150995,"pt":9571,"pu":45299,"ló":14756,"lí":26351,"pr":175784,"ps":6678,"zi":14725,"ze":25298,"vá":5473,"za":80538,"zo":10261,"ví":24973,"ya":6552,"ys":5731,"yr":7114,"uí":16711,"uê":10113,"uç":8683,"² ":81055,"É ":12059,"ã ":5975,"ál":15193,"ác":6980,"ád":5021,"áv":6927,"áx":6998,"ár":84913,"át":10444,"ás":9114,"âm":6223,"ân":30468,"ão":308442,"à ":22252,"á ":25281,"ós":7667,"ôm":5559,"ôn":14217,"ói":30866,"óg":6061,"ód":6423,"ór":23808,"óp":8429,"ón":16555,"ól":8310,"ív":5979,"ín":37401,"ím":8935,"íp":23888,"ío":18396,"ít":18007,"ís":21982,"íf":5629,"íl":22187,"íc":11953,"íd":15269,"çõ":21574,"çã":131394,"ên":31853,"êm":5074,"ês":26627,"él":7493,"ém":24476,"én":6312,"és":8228,"ét":9564,"ér":30738,"éd":7542,"éc":31275,"ço":21084,"ça":38992,"é ":308148,"ún":5650,"ús":10148,"úb":7903,"õe":33653," Ga":11344," Ge":12440," Fo":24420," Fu":6872," Fr":18748," Fi":9007," Fl":5988," Ha":12738," He":10614," Go":9722," Gr":18935," Gu":13279," Gi":5513," Ho":11760," Hi":6304," Je":5193," Ja":20267," In":21384," Ka":6274," Jo":23796," Ju":10788," La":20143," Le":17264," Li":18789," Ma":61882," O ":56830," Mi":22595," Me":18100," NG":5699," Lo":21011," Lu":9458," Ne":9995," Na":16245," Ni":6760," Mo":23061," Mu":14430," A ":56821," Am":13551," An":22425," Al":38669," Ag":5878," Ac":5031," Ab":6362," Ba":33333," Au":8681," At":7218," As":11678," Ar":20591," Be":17207," Bi":6949," Bo":16224," Br":31284," Bu":7302," Ca":66026," Ce":12123," Ci":8645," Ch":23864," Cl":8347," Cr":10239," Co":62169," Cu":6320," Da":9487," Di":17388," De":23395," Do":10149," El":11281," Es":76102," En":6655," Em":8884," Eu":7403," Fe":14330," Fa":9149," Wi":7332," Sã":16295," Wa":6979," a ":159392," Os":8676," Or":9772," Po":57372," Pl":5480," Pi":15147," Pe":21311," Pa":48469," No":30533," Ol":8779," Ra":8887," Qu":6094," Ro":19582," Re":33096," Ri":19127," Pr":23377," Su":24284," St":10640," Ta":10684," UA":14219," Th":10656," Ti":5222," Te":19124," Tr":12898," To":12004," Sa":39159," Sh":5584," Si":12511," Sc":6287," Se":27797," So":14864," Va":11341," Ve":14300," Vi":19683," Tu":6372," Um":5787," Un":17669," ja":9410," im":11343," in":69292," il":7613," it":11439," jo":14428," ju":16927," ha":94775," gr":27016," go":7301," gu":5619," hi":11250," ho":15720," ne":9802," na":144146," mu":41919," mo":24736," on":8262," oc":7515," of":9658," ob":9893," nu":6152," no":190063," le":13833," li":23905," la":21762," gê":9642," km":87834," me":40602," mi":24825," o ":110021," ma":77227," lu":6557," lo":47174," ag":9251," ab":11690," ac":15052," ad":42206," am":26285," an":52016," ao":28506," ap":22430," al":29345," av":5103," au":17149," ar":22234," at":33443," as":63845," ba":35433," bi":6595," be":6338," bo":7718," br":28760," ca":66098," e ":241281," er":11510," et":6896," es":129714," en":44317," em":145368," el":17511," fe":16956," fa":35767," ex":43118," fu":24478," fr":49866," fo":99597," fi":29419," ge":13813," ga":14076," cl":11956," co":383123," cr":20621," ce":57678," ch":14448," ci":62716," da":258975," cu":12445," do":238864," de":1021034," di":92269," ed":7807," du":15330," sa":10534," se":193144," si":34152," so":24812," qu":119214," mú":6854," ra":8844," re":136757," nã":9894," ri":7562," ro":14358," pu":7134," pr":135007," lí":6039," os":51421," ou":60851," op":5907," or":37748," pe":115763," pa":93579," pl":9433," po":171127," pi":7518," sã":13196," sé":13507," va":8521," ve":22883," vo":11121," vi":22819," us":9297," ut":7175," um":444668," un":7084," ta":23369," su":56894," tr":33259," to":19262," th":6383," ti":9760," te":60106," É ":12023," à ":21932," ár":53832," ál":7915," é ":284550,"GC ":5727,"Est":57555,"Esp":12609,"Eur":5804,"Ele":5431,"Em ":5626,"Ger":5645,"Fra":12238,"Foi":10640,"For":9044,"II ":6274,"Gra":10045,"Int":5948,"Amé":5088,"Bai":6138,"Bar":6969,"Ale":12779,"Alt":5502,"Ant":8344,"Cal":7739,"Cam":11986,"Cas":10156,"Car":12659,"Cat":5648,"Can":7596,"Bra":20750,"Den":4991,"Chi":5974,"Cen":6997,"Cha":10337,"Cor":7280,"Com":13375,"Col":5164,"Con":24999,"Dis":5047,"Nov":9350,"Nor":13657,"Os ":6443,"Per":7552,"Par":15095,"Pau":10573,"Pal":5074,"Pro":8377,"Pol":9021,"Pos":21873,"Por":13937,"Jan":9313,"Jos":5715,"Jog":5307,"Lan":5276,"NGC":5668,"Man":7960,"Mar":25108,"Mon":7943,"Min":6224,"Mun":8484,"São":16298,"Sul":8673,"UA ":15445,"Sai":5389,"San":15530,"Rio":11763,"Val":5795,"Vil":5627,"Ver":7558,"Uni":16626,"Ter":5697,"The":7418,"Tra":5593,"bit":68890,"bil":5849,"bo ":5900,"bli":15859,"bol":16054,"bor":6814,"be ":8829,"ban":15903,"bal":9064,"bai":9744,"bas":9817,"bar":9261,"ber":18628,"bel":6221,"ca ":99422,"car":23995,"cas":22552,"cat":6662,"can":45210,"cap":11240,"cad":24057,"cam":12174,"cal":52109,"ce ":17573,"bri":17515,"bro":31772,"bra":35217,"bre":13206,"bur":6170,"bum":7625,"am ":33009,"aix":9798,"al ":180975,"ain":14382,"aio":19228,"air":8502,"ais":60324,"aia":5333,"ago":14405,"anu":5896,"ano":74765,"ant":129336,"ans":13455,"ane":21736,"ang":13849,"anh":31372,"ani":21090,"ana":43001,"anc":58051,"and":76029,"amo":8042,"amp":20451,"ami":6934,"ame":89963,"amb":25048,"ama":26496,"ao ":25066,"alt":10265,"alo":6981,"alm":23414,"all":8441,"alg":7598,"alh":11152,"ali":86609,"ald":5965,"ale":20569,"ala":17691,"an ":23333,"aba":10377,"abe":9561,"abi":55871,"abo":5404,"abr":10253,"ae ":16585,"aca":7889,"ab ":40485,"ai ":5437,"aga":6957,"age":17901,"ado":217915,"adr":8357,"adm":35408,"adi":9559,"ade":147619,"adu":9382,"aco":11338,"aci":33260,"ach":9385,"ace":11134,"ada":120249,"act":10015,"até":8607,"ató":5142,"ba ":7163,"aqu":7206,"amí":18994,"arg":7520,"are":23525,"ard":21302,"arc":12276,"ara":72529,"aro":8382,"arn":4993,"arm":6518,"arl":7205,"anç":24866,"ari":33288,"arq":9519,"arr":16718,"art":76275,"asa":6049,"asi":39800,"asc":20114,"ase":9902,"ar ":59582,"apa":10937,"alá":8272,"ape":8629,"api":9537,"apo":6995,"apr":7853,"as ":272706,"ava":17923,"aut":16096,"arç":7034,"avi":9973,"ave":10994,"ata":23226,"ast":51032,"ass":26073,"atr":16524,"ato":21654,"ate":18241,"ati":65753,"atu":18602,"aul":12887,"aus":5992,"jet":6359,"jan":5629,"jog":9592,"ito":67486,"itu":23834,"ism":9783,"isp":8481,"iss":14904,"ist":137228,"ita":115808,"ite":14296,"iti":9264,"ivr":6273,"ivo":19750,"isã":7956,"iva":52169,"ivi":13782,"ive":20602,"ipo":7067,"is ":98587,"ion":54065,"ior":20141,"ios":21003,"ipa":29796,"ipe":6360,"ir ":14429,"irr":7079,"iro":63701,"iri":6848,"ise":5201,"isc":10067,"isa":6981,"iu ":6072,"ire":22818,"ira":48023,"ja ":10608,"ixa":7209,"itâ":5590,"iz ":5440,"iza":68411,"km ":6889,"ki ":6235,"km²":80918,"gên":12105,"jul":5237,"jun":11429,"ha ":54934,"ham":13772,"han":7220,"har":10472,"has":8814,"hab":92261,"he ":16106,"hei":5849,"hec":22728,"her":9140,"hin":8200,"his":10200,"ho ":40610,"go ":31208,"cçã":8036,"gna":9579,"giã":60677,"gos":20347,"gov":5137,"gru":8390,"gra":31116,"gre":13275,"cíp":21053,"gui":6993,"gua":14762,"gue":29368,"gun":46422,"guê":7234,"iai":5206,"iam":5743,"ial":29154,"ian":36430,"ias":36377,"iad":18351,"ibu":6231,"ibe":5599,"ia ":236755,"ien":14065,"ier":7590,"ies":5878,"ied":5977,"iaç":6604,"ife":6300,"ifi":12350,"icu":5216,"ico":76077,"ici":43382,"ich":7968,"ice":7761,"ie ":27537,"ica":143651,"ido":58535,"idi":8244,"ide":55475,"ida":173600,"il ":28740,"im ":15779,"ige":5893,"iga":14060,"igi":13843,"igu":5678,"icí":20778,"igo":8521,"ign":14724,"imo":12687,"imp":12501,"ime":38876,"imi":7471,"inc":33960,"ind":23589,"ina":69514,"ino":22963,"int":49765,"ins":15328,"inf":7949,"ine":18603,"inh":19122,"ing":32761,"ini":53005,"inu":8866,"ila":11890,"in ":16212,"ilo":7283,"ill":18435,"ilm":5403,"ilh":18906,"ili":24989,"ile":27199,"ima":19294,"io ":111339,"hom":5231,"hos":8051,"hor":13996,"hum":7243,"fes":5911,"fer":17361,"fei":6807,"fam":20697,"ext":9301,"ez ":8287,"exp":6312,"exi":7134,"exc":16402,"eze":10386,"eta":21527,"ete":19661,"eti":12522,"esp":45070,"est":117047,"ess":36172,"eto":18111,"etr":18837,"eve":21643,"eva":5679,"evi":13265,"eus":10550,"eró":29181,"erí":19798,"ey ":6913,"er ":64588,"epa":35649,"açõ":12626,"eon":5303,"es ":232451,"epr":6051,"enç":6954,"eri":53238,"erg":9605,"ere":26129,"erc":25527,"erd":8470,"era":58345,"et ":10811,"equ":14020,"aís":9202,"esm":7907,"esi":25700,"esc":27105,"esd":5456,"ese":28703,"eu ":29870,"esa":56958,"erv":11462,"err":29563,"ert":32707,"ers":31036,"ern":27425,"erm":19525,"ero":27972,"en ":19630,"ela":57757,"ele":31916,"eli":10608,"elh":19021,"ell":12189,"elo":35477,"eo ":6077,"emb":29421,"ema":25633,"eme":11201,"emo":8721,"emi":9916,"emp":20050,"ene":11086,"enh":10752,"ena":27141,"end":70599,"enc":33249,"eno":17065,"eni":7335,"env":8603,"ens":111144,"ent":263509,"açã":84894,"ecç":6791,"ego":8230,"egi":67782,"egr":7999,"egu":58530,"eia":6927,"eis":10113,"eir":97869,"eio":5873,"ein":12110,"eja":8929,"el ":27319,"eit":17574,"em ":185457,"gin":9613,"gio":6344,"gic":7282,"gia":12807,"gen":19808,"ger":11186,"gem":16786,"ge ":8655,"gad":9856,"gas":7737,"gar":10290,"gal":15478,"gan":11614,"ga ":19137,"fut":9060,"fun":15140,"fra":39155,"fre":11644,"for":36958,"foi":67238,"bém":15726,"fic":31195,"fil":13806,"fin":10276,"fis":5455,"da ":386105,"de ":1113463,"dad":172007,"dal":5052,"dae":8716,"das":50851,"dan":7654,"dam":5121,"cul":25961,"cto":6036,"cti":5064,"cta":7890,"cur":7265,"cla":7926,"clu":9168,"cli":8238,"co ":77432,"con":114590,"col":16761,"com":260529,"cor":23539,"cos":23065,"cre":8780,"cri":27018,"cro":6255,"cea":7114,"ch ":7362,"cer":21587,"ces":47284,"ceu":5073,"cen":80958,"caç":9087,"cel":14050,"cei":7700,"cha":18824,"cia":84536,"ck ":7938,"cie":24606,"cid":84373,"che":14945,"chi":8861,"cim":6171,"cis":5750,"cin":23953,"cio":45369,"cip":32198,"ebo":13090,"ead":6607,"ean":5576,"eal":11859,"eat":4993,"ea ":57325,"efe":11937,"ei ":8171,"ega":11879,"edi":19796,"ede":22533,"eda":8333,"edo":8760,"ecl":7415,"eci":35040,"ece":14294,"ecu":5255,"ect":14984,"eco":11351,"dur":9714,"duz":6967,"dor":36541,"don":5597,"dos":101787,"diç":6127,"dmi":36030,"dua":7331,"dri":7071,"dra":5496,"dre":6857,"dro":6768,"dic":13696,"did":7827,"dia":53753,"der":26564,"des":61760,"dez":6564,"dec":9473,"def":5074,"dei":8639,"del":8957,"den":70173,"dem":10237,"dep":38994,"do ":499886,"div":11366,"din":7506,"dio":15896,"dir":15370,"dis":34873,"dit":6840,"dif":6362,"rga":9198,"ri ":5474,"rgi":6642,"rge":9033,"não":10106,"rgo":7927,"ret":18367,"res":88172,"rev":9016,"rg ":6617,"rea":66834,"ref":8795,"rec":27943,"red":8922,"rei":27335,"reg":80521,"rem":9993,"ren":24469,"raç":16220,"rel":14080,"nár":5684,"rep":8548,"rda":6649,"rdo":6927,"rdi":10382,"rde":15782,"re ":57889,"rci":7180,"rce":9738,"rca":20138,"rd ":8761,"rar":6504,"ras":65057,"rat":52626,"rav":11402,"rbi":16469,"rba":5129,"rai":12729,"rag":7162,"ran":101236,"ram":28519,"ral":30318,"rab":8082,"raf":5059,"rad":42430,"rac":11489,"rs ":6575,"ros":32986,"rot":8757,"rom":13204,"ron":14637,"rop":15939,"rou":5772,"rov":31049,"rod":14281,"roc":11812,"rol":6958,"rof":9288,"nçã":8381,"rog":7781,"rno":11572,"rna":23454,"rne":8579,"rmo":8053,"ro ":148425,"rma":34000,"rme":10601,"rmi":14095,"nça":24469,"riz":8006,"rio":44890,"rit":46897,"ris":18600,"rig":17891,"ril":10251,"rin":40225,"rim":24339,"ria":73821,"rib":7744,"ric":60975,"rid":13442,"rie":17826,"rup":9891,"rus":5606,"rva":5441,"rvi":6787,"ry ":6483,"rsi":7243,"rso":14017,"rta":49041,"rto":14307,"rte":57600,"rti":26907,"rtu":25752,"rt ":7638,"rqu":17086,"rro":14834,"rri":9266,"rre":21970,"rra":23654,"sad":9694,"san":5546,"sas":9730,"sar":5392,"sa ":76681,"sid":59438,"sic":18066,"sia":15495,"sit":17633,"sis":13892,"sin":13902,"sio":8799,"sil":41142,"sim":8594,"sig":13420,"scr":14392,"scu":5033,"sde":5601,"se ":99012,"sca":8201,"sce":12058,"sci":8690,"sco":17645,"ser":25139,"ses":9941,"set":9939,"seu":20458,"seg":45509,"sed":9373,"sen":34160,"sem":9716,"sel":5712,"spo":14510,"spe":12903,"spi":7965,"spa":15990,"sol":6214,"son":13806,"sor":8549,"sos":34321,"soa":5579,"soc":9148,"sob":10181,"st ":5646,"squ":6464,"smo":15213,"so ":33266,"ssã":5427,"stá":10797,"stã":5171,"stó":7858,"sse":16316,"ssa":18526,"sso":30852,"ssi":23013,"ssu":30134,"ste":119936,"sta":117494,"spé":16790,"sto":24915,"sti":39339,"stu":7367,"str":101454,"sua":22858,"sub":11326,"sui":27030,"sul":9138,"sup":6250,"sur":5128,"tai":5662,"tal":56545,"tad":72109,"tat":5635,"tas":21431,"tar":22207,"tan":74838,"tam":58144,"te ":182044,"ta ":108579,"pa ":8506,"pe ":6181,"par":108733,"pas":6728,"pac":5622,"pal":27847,"pan":19668,"láx":5857,"pec":9282,"pen":12180,"per":70169,"paí":6811,"pet":6411,"pes":15522,"pel":52058,"pla":10972,"ple":7590,"plo":5321,"pic":9020,"pin":7594,"pio":22913,"pir":7271,"pit":9245,"por":117200,"pop":23863,"pos":31387,"pon":14947,"pol":21044,"pod":12218,"po ":20718,"lês":5033,"pub":5596,"lít":10135,"pri":47853,"pre":42479,"pro":71854,"put":8461,"pul":26066,"mão":6273,"mér":6337,"míl":18580,"qua":26640,"que":121041,"qui":34023,"ra ":178784,"mús":6480,"ngo":5638,"ngl":12535,"ngu":12950,"ni ":5382,"nge":10906,"nga":5764,"nho":23066,"nha":43874,"nhe":24382,"nei":17925,"naç":10350,"nen":5777,"ner":21401,"net":5963,"nes":11864,"ng ":14902,"nco":14783,"nci":80668,"ncl":5010,"nce":69648,"nch":7394,"nca":5561,"ne ":24199,"ndr":9286,"ndo":94739,"ndi":27820,"nde":77208,"nda":61455,"nal":50464,"nam":7322,"nan":6772,"nar":8444,"nac":9251,"nad":24850,"nag":7446,"nai":6712,"nd ":10811,"nat":14635,"nas":35071,"na ":240257,"ny ":5694,"nsã":7776,"nvo":7994,"nve":5937,"num":5422,"nut":6804,"nto":104039,"ntu":19077,"ntr":64145,"nti":32354,"nta":52915,"nte":218759,"nso":37106,"nst":25404,"nse":18037,"nsi":55540,"nsa":5243,"nt ":11783,"ns ":20234,"nom":29572,"not":5470,"nos":46255,"nor":21961,"nov":11058,"nne":6439,"no ":209889,"nid":27726,"nic":53412,"nia":29100,"niz":6361,"niv":8244,"nis":48078,"nio":9439,"nim":8021,"nin":5656,"ogr":11168,"ogi":9366,"ogo":16032,"oga":7825,"oi ":78403,"ois":10929,"oje":5415,"ol ":15035,"oce":8227,"oci":14692,"oco":7976,"oca":52787,"ode":20265,"odi":6628,"odo":29483,"of ":5154,"oda":5699,"oes":5612,"odu":11939,"ofi":9565,"oa ":8425,"obr":14956,"oví":22195,"ote":8672,"oto":8649,"ost":25890,"ota":9466,"osi":9200,"oss":38213,"oso":8522,"ovi":11009,"ovo":6748,"ova":12269,"ove":21270,"ous":5680,"our":9672,"out":17508,"opo":7367,"ope":8960,"opa":6108,"os ":396479,"opu":24150,"oló":8455,"olí":12107,"or ":155288,"orm":37181,"orn":14580,"oro":7726,"orr":17850,"ord":17760,"ore":31129,"org":12371,"ori":29601,"ou ":76190,"osa":9113,"ort":72366,"m² ":81009,"orb":15491,"ora":44778,"ola":17798,"on ":35303,"oli":16249,"ole":10370,"olo":20061,"olu":6559,"olv":8428,"om ":129315,"ona":61961,"ond":42834,"onc":16183,"one":12259,"onh":24723,"ong":8421,"oni":11825,"ono":11037,"ons":37243,"ont":48875,"oma":22403,"ome":35533,"omb":7493,"omi":12319,"omp":29869,"omo":47819,"omu":62085,"la ":68801,"le ":30025,"lac":17518,"lad":10810,"lag":5216,"lan":32402,"lam":5387,"lar":20429,"lat":13280,"las":19631,"ld ":5145,"lbu":8512,"lon":10273,"lor":10947,"loc":44637,"log":14014,"los":17433,"lme":28363,"lti":6810,"lto":6807,"ltu":6502,"lub":6858,"lta":10527,"lho":31407,"lhe":5407,"lha":24850,"lgu":5605,"lev":8296,"les":20832,"let":10999,"ler":5749,"lem":18432,"len":11746,"laç":21723,"lei":31609,"leg":6064,"lec":6739,"lo ":60493,"lla":8917,"lle":13121,"lli":8289,"ll ":7390,"lit":15034,"lis":21665,"lio":5219,"lin":27495,"lim":6007,"liz":58462,"liv":7687,"lic":27450,"lid":13334,"lia":47125,"lig":7684,"ma ":312319,"mai":41100,"mad":18812,"mar":24207,"mas":19571,"mal":6120,"man":38877,"mat":11364,"mba":7627,"mbi":6786,"mbr":26443,"mbo":5222,"me ":30188,"med":7143,"met":15815,"mes":16176,"mer":33993,"mem":6344,"mel":7946,"men":135706,"mei":21546,"maç":6372,"mbé":15876,"lva":5641,"lve":5109,"lvi":6861,"mpi":7951,"mpe":15384,"mpr":13673,"mpo":26049,"mpl":10714,"içã":16989,"mod":7097,"mon":12314,"mor":10927,"mos":10632,"mpa":8900,"mui":6353,"mul":5806,"mun":87543,"ião":65139,"min":67980,"mil":11247,"mis":7225,"mit":9135,"mic":17167,"mia":5205,"mo ":71790,"vín":22165,"zem":7471,"zaç":7006,"zad":51320,"zon":5106,"uíd":7919,"uçã":7240,"za ":11487,"uês":7863,"tón":8038,"tór":13405,"tão":9310,"tân":11804,"tár":6234,"té ":9213,"xim":6106,"xia":6380,"xa ":5938,"xce":15412,"tá ":7267,"séc":5490,"sér":7647,"são":39006,"río":17761,"rói":28842,"róp":5175,"via":10682,"vil":9868,"vim":5528,"vid":18086,"vis":17869,"rço":7357,"vo ":18229,"vol":13908,"vos":6610,"rão":7259,"vez":5961,"ver":46986,"ves":7125,"vei":6022,"ven":17920,"vem":8996,"vel":15436,"ve ":14933,"val":11487,"van":5589,"var":6549,"vas":5897,"vad":9906,"va ":59685,"uzi":7134,"utó":6849,"usi":7047,"use":5760,"usa":11181,"ust":14662,"uss":6347,"uti":10782,"ute":14945,"uta":12601,"utu":10263,"uto":22443,"utr":11202,"us ":31309,"ura":55930,"ure":6495,"urg":9235,"uri":7874,"uro":13305,"ur ":5714,"upe":7143,"upo":9984,"uma":271540,"ume":7805,"unt":10008,"uni":47036,"und":70729,"una":50258,"unh":9065,"um ":201899,"ult":13394,"ulo":24134,"uli":6278,"ulh":9149,"ula":44299,"uil":6385,"uin":8363,"uip":5474,"uis":7425,"uia":5524,"uit":14697,"ul ":16765,"ui ":29672,"uga":10215,"ugu":19601,"uda":6128,"ude":5928,"ubr":7378,"uca":5380,"ue ":100430,"uer":13174,"ues":23125,"udo":7316,"uen":9436,"uel":7358,"púb":6723,"ua ":33662,"uas":11459,"uar":8119,"ual":23558,"uan":11328,"ubl":7183,"ube":7379,"uai":5297,"uad":12055,"tur":41731,"tus":5101,"tui":5138,"tul":5524,"tub":7669,"tua":23176,"tud":7868,"tug":23030,"tre":31460,"tra":95106,"tri":60532,"tru":9663,"tro":48136,"péc":16451,"to ":188185,"tod":9530,"tou":5354,"tos":40605,"tom":5946,"ton":10767,"tor":47860,"til":14634,"tig":9433,"tir":6653,"tit":12090,"tis":7919,"tin":26516,"tim":13089,"tip":5927,"tio":9200,"tia":5519,"tic":53927,"tid":12974,"tiv":62127,"tem":39133,"ten":72655,"tei":7527,"taç":9386,"tel":22078,"teg":5010,"teb":12305,"tec":8043,"th ":5190,"tes":82988,"ter":103003,"the":7042,"ço ":13183,"ém ":22415,"édi":7117,"éci":19647,"écu":6135,"éti":5315,"éri":21433,"ênc":16544,"êne":10481,"ês ":26475,"ção":131304,"ão ":306157,"ça ":19591,"çad":10194,"áti":8887,"áve":5112,"áxi":6942,"álb":7607,"áli":5630,"ári":27812,"áre":51471,"âni":17684,"úsi":6572,"úbl":7260,"ões":32837,"ôni":11441,"óri":16535,"óno":6648,"óni":7908,"óid":28287,"íti":12961,"íst":6738,"ínc":23886,"íng":5418,"íli":21725,"íod":17716,"ípi":21652,"ís ":6687,"íci":9019,"íde":5475,"çõe":21567},"n_words":[49778514,58587553,42469388],"name":"pt","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/ro.json b/contrib/languages-data/ro.json
new file mode 100644
index 0000000..6cbc837
--- /dev/null
+++ b/contrib/languages-data/ro.json
@@ -0,0 +1 @@
+{"freq":{"D":14206,"E":15258,"F":12709,"G":12710,"A":32029,"B":21807,"C":31096,"L":13478,"M":26711,"N":11195,"O":8313,"H":8411,"I":15485,"J":5893,"K":4820,"U":8262,"T":15341,"W":4878,"V":8350,"Q":589,"P":22438,"S":30214,"R":23700,"Y":1325,"X":1901,"Z":1899,"f":89747,"g":84289,"d":270437,"e":933940,"b":76172,"c":339926,"a":811577,"n":568157,"o":427370,"l":434670,"m":224465,"j":16485,"k":15291,"h":47000,"i":839847,"w":7236,"v":72644,"u":459551,"t":525861,"s":321476,"r":567673,"q":978,"p":199529,"z":55715,"y":12885,"x":14642,"²":280,"Î":4242,"É":135,"ß":93,"î":69716,"í":464,"é":1581,"è":290,"ç":171,"ä":380,"ã":94,"â":33080,"á":1243,"à":152,"ü":921,"ú":132,"ö":583,"ó":581,"ñ":107,"Ă":92,"ă":161614,"ā":266,"ć":89,"ı":136,"ī":130,"ş":69795,"ł":112,"ō":143,"š":140,"ţ":70418,"ū":99,"ǎ":351,"Ș":1233,"Ț":844,"ə":124,"ˈ":89,"́":280,"μ":153,"ν":326,"ο":443,"ι":274,"κ":132,"λ":209,"δ":99,"ε":222,"η":124,"α":406,"γ":123,"ά":91,"ί":118,"ό":118,"σ":174,"ς":360,"ρ":245,"π":122,"υ":100,"τ":203," l":54431,"ь":237," m":47832," n":26804," o":49358,"я":249," h":3267," i":28305," j":7583," k":2023,"ы":136," d":156105," e":67780,"х":112," f":51072,"ц":117," g":13468,"ч":249,"р":882," a":134125,"с":828," b":11159,"т":574," c":116614,"у":311," y":145," x":142," z":3353," u":43415," t":26873," w":1170," v":14303,"і":84," q":90," p":90611," s":84706," r":35629,"К":133,"Н":83,"М":137,"П":110,"Б":103,"А":114,"В":101," J":5857," K":4744," H":8304," I":15348," N":11099," O":8211," L":13327," M":26422," B":21644," C":30661,"Р":101," A":31745,"С":176," F":12566," G":12513," D":13925," E":15142,"л":649," Z":1885,"к":766," Y":1319,"й":327," X":1885,"и":1244,"п":176,"о":1327,"н":924,"м":284," S":29902,"г":187," R":23579,"в":707," Q":580,"б":205," P":22131,"а":1480," W":4821,"з":146," V":8268," U":8214,"е":1081," T":15157,"д":332," î":68405," Ă":90," É":134," Î":4219," ţ":2046," ş":40504,"ה":105,"ו":138,"א":92,"ל":89,"י":166," Ș":1227," Ț":843,"ר":104,"و":163,"ي":302,"ل":318,"م":220,"ن":216,"د":146,"ح":87,"ب":195,"ا":469,"ع":89,"س":117,"ر":219," А":114," Б":103," В":101," К":131," М":137,"A ":4154," П":110,"F ":568,"Da":2015,"Cu":1954,"Cy":113,"Cl":1608,"Co":8574,"Cr":1776,"Ce":2628,"Ch":2680,"Ci":1438,"G ":509,"Ec":471,"Ed":733,"Ea":729,"Du":1500,"Do":1908,"Dr":850,"De":3817,"Di":2658,"Fe":1913,"H ":435,"Fa":1302,"Eu":3402,"Ev":451,"Ex":957,"Er":513,"Et":184,"Es":2728,"En":827,"Em":497,"Ep":280,"Ei":240,"El":1974,"Ef":84,"Eg":290,"Ge":3177,"Cá":254,"Câ":416,"Ga":1765,"I ":3107,"Fu":513,"Fr":2787,"Fo":1889,"Fl":1292,"Fi":1643,"Bâ":129,"B ":735," Р":100," С":176,"C ":1390,"Av":423,"Au":2210,"Aw":103,"Ar":4794,"At":636,"As":1391,"D ":881,"Ba":4270,"Az":208,"Ae":279,"Af":611,"Ag":465,"Ah":119,"Ab":553,"Ac":3223,"Ad":1097,"Am":1924,"An":3647,"Ap":1490,"Ai":410,"Ak":87,"Al":4305,"Bu":3694,"Br":3092,"Ca":7562,"E ":767,"Bi":3068,"Be":2955,"Bo":2204,"Bl":574,"Ku":328,"Ky":124,"Kn":99,"Kl":149,"Kr":348,"Ko":662,"Le":1965,"Gă":138,"Li":3218,"N ":854,"La":3044,"Lu":1753,"Ly":104,"Ll":318,"Lo":1983,"Me":2961,"Mi":4169,"O ":1394,"Ma":9710,"Mc":201,"My":160,"Mu":2543,"Mo":4971,"Ni":1761,"Ne":2385,"Na":2454,"P ":736,"Q ":153,"Nu":750,"No":2851,"Ol":1045,"Om":198,"On":376,"Oh":91,"Oc":1033,"Od":206,"Of":171,"Oa":113,"Ob":305,"Gi":853,"Bă":725,"Gh":757,"Gl":444,"Gr":2243,"Go":1170,"Gu":1179,"Gy":91,"J ":354,"Ha":2364,"Dâ":128,"He":1745,"Că":581,"Hi":659,"Ho":1571,"Hr":249,"Hu":882,"Hy":112,"K ":358,"Ib":100,"Ia":1254,"Id":117,"Ie":183,"Ig":92,"Io":1307,"Im":1022,"In":3678,"Il":431,"Iu":1265,"Iv":162,"Is":1017,"It":790,"Ir":549,"Ja":1638,"L ":645,"Iz":203,"Ji":337,"Je":630,"Jo":1867,"Ju":935,"Ka":1128,"M ":833,"Kh":125,"Fă":159,"Ki":625,"Ke":541,"Ut":114,"Ur":420,"Up":83,"Um":91,"Un":5673,"Ul":182,"Pă":635,"Uc":368,"W ":229,"Ty":98,"Tu":1266,"Tr":2949,"To":1782,"Th":2212,"Ti":1500,"Te":2836,"Pâ":232,"Ta":1163,"V ":1269,"Sw":112,"Sz":144,"Sy":285,"St":5254,"Su":2579,"Wr":229,"Wo":691,"Wi":1273,"Ră":813,"Wh":234,"Wa":1039,"Sâ":220,"We":859,"Y ":147,"Vo":713,"Vr":133,"Vu":111,"Râ":933,"Vi":1974,"Vl":297,"X ":1079,"Va":2283,"Ve":1567,"Lă":421,"Pu":736,"Pr":4511,"S ":1482,"Pe":2677,"Pa":6072,"Pl":931,"Po":3427,"Pi":1539,"Ph":445,"Os":331,"Ot":294,"Ou":154," ا":202,"Ov":120,"Op":423,"Or":1772,"R ":901,"Kö":111,"Sf":926,"Se":3730,"Sc":1718,"Si":2479,"Sh":841,"Nă":243,"Sm":167,"Sl":471,"Sk":113,"Sp":3075,"So":2382,"Ru":1542,"U ":435,"Sa":3082,"Re":6657,"Ri":1089,"Mă":769,"Rh":203,"Ro":9754,"Qu":329,"T ":645,"Mü":187,"Ra":1639,"Wü":172,"b ":5072,"a ":212743,"Ye":128,"Tă":194,"Ya":168,"Yo":679,"Yu":120,"Z ":124,"Tâ":356,"Să":464,"Vă":154,"Za":383,"Ze":392,"Zi":317,"Vâ":376,"Zo":278,"i ":179645,"gd":205,"ge":13414,"câ":4391,"ga":11234,"gb":146,"fl":4423,"ff":711,"bâ":136,"fi":21259,"fr":4539,"fu":3903,"ft":996,"fo":28249,"j ":2040,"gy":251,"dâ":214,"he":10237,"ha":6256,"gn":1318,"gm":382,"gl":4043,"gi":18379,"gh":4326,"bă":2234,"gg":253,"gv":296,"gu":7182,"gt":188,"gs":381,"gr":9341,"cî":121,"go":3415,"dt":248,"du":15963,"dv":200,"dw":264,"dy":391,"dz":86,"g ":5818,"ea":59379,"eb":4646,"ec":30872,"ed":13772,"de":117725,"dd":311,"dg":263,"di":66800,"dh":111,"dj":215,"dm":1691,"eM":86,"dl":291,"do":13587,"dn":210,"ds":959,"dr":6694,"ew":1340,"ex":7454,"eu":5496,"ev":8185,"ey":1419,"ez":15187,"fa":8662,"h ":3714,"fe":10587,"eh":1815,"eg":14417,"ef":4744,"ee":3471,"el":50297,"ek":775,"ej":687,"ei":30112,"ep":15349,"eo":6294,"en":63988,"em":24435,"et":20416,"es":79718,"er":87843,"ca":58257,"e ":363120,"by":328,"bs":1372,"br":14880,"bu":7988,"bt":195,"bn":117,"bo":5650,"bl":6057,"bm":100,"bi":12351,"bb":308,"bc":108,"bd":241,"be":6318,"db":261,"da":18159,"f ":3092,"cz":119,"cy":195,"cv":646,"cu":45953,"ct":20001,"cs":291,"cq":100,"cr":11320,"co":42083,"cm":243,"cn":187,"ck":2747,"cl":5298,"ci":32813,"ch":13925,"ce":51893,"cc":2745,"c ":18619,"aP":84,"aC":95,"az":9879,"ay":1659,"ba":11641,"d ":22818,"at":93723,"as":21874,"ar":105841,"aq":90,"ax":1099,"aw":499,"av":8565,"au":21651,"ak":1416,"al":87116,"ai":18198,"aj":3933,"ao":374,"ap":15113,"am":19806,"an":87690,"ac":26532,"ad":16915,"aa":641,"ab":6836,"ag":7060,"ah":1692,"ae":3048,"af":7326,"nu":32036,"nt":67674,"ns":20163,"nr":1134,"np":113,"no":17936,"nn":2298,"q ":85,"nz":1824,"ny":836,"nw":156,"nv":2089,"oe":1765,"of":5818,"oc":21548,"od":10761,"oa":21531,"ob":5951,"om":43493,"on":56179,"ok":801,"ol":30865,"oi":8296,"oj":309,"og":8118,"oh":1090,"m²":273,"ot":11364,"os":27341,"ov":12192,"ou":7338,"op":17725,"oo":1964,"or":85929,"r ":49086,"ox":1908,"ow":1237,"oz":3227,"oy":555,"pe":42586,"lâ":532,"pa":25762,"pc":98,"pl":7066,"lé":112,"po":22942,"ph":1089,"pi":11751,"lo":40730,"ln":822,"lm":3343,"ll":6217,"ls":1221,"lp":822,"lv":2229,"lu":48097,"lt":11451,"lz":261,"ly":862,"o ":37058,"mc":90,"ma":44395,"mb":17891,"eş":9887,"hă":248,"me":40474,"mf":257,"eţ":7364,"ml":160,"iè":112,"mi":28493,"mn":4278,"mm":955,"mp":17238,"mo":11698,"mt":225,"ms":601,"mu":21624,"my":299,"p ":5018,"na":37617,"nb":568,"nc":20662,"nd":32264,"ne":45632,"nf":4807,"ng":14388,"nh":479,"ni":65371,"nj":736,"nk":1221,"nl":1126,"nm":330,"ju":7523,"jo":2219,"jl":509,"bţ":531,"ki":1735,"kh":269,"fă":1862,"cş":118,"gâ":388,"kf":113,"ke":1642,"ka":1464,"m ":15258,"ky":316,"ks":661,"kt":260,"ku":385,"ko":995,"kr":318,"kk":161,"cţ":4949,"kl":292,"km":1457,"li":54376,"lh":237,"gă":2958,"lk":383,"lj":135,"le":70613,"ld":4148,"lg":2038,"lf":1236,"hâ":94,"la":55152,"lc":2339,"lb":4110,"n ":164001,"hr":1429,"hs":252,"hw":257,"ht":1909,"hu":2313,"că":22340,"hi":13680,"hn":1591,"ho":3277,"hl":404,"hm":291,"id":10613,"ic":65212,"ib":4937,"ia":63950,"dă":2289,"ih":1392,"ig":9132,"aş":9011,"if":6185,"ie":62856,"hy":354,"k ":4658,"iq":165,"eî":135,"ir":13340,"is":36911,"it":62439,"iu":24448,"iv":15138,"iw":97,"ix":744,"ii":33158,"ij":1160,"aţ":19894,"ik":1190,"il":42983,"im":26881,"in":122257,"io":22518,"ip":8638,"je":1168,"fâ":980,"ji":658,"iz":12018,"iy":146,"l ":109782,"ja":1735,"pţ":503,"să":10006,"xi":3113,"xo":371,"té":92,"tî":216,"xp":1513,"xt":2105,"xu":441,"ww":338,"z ":3980,"xc":491,"xa":1180,"tâ":2269,"xe":1609,"oş":1503,"ră":15639,"wi":755,"oţ":1099,"sé":93,"wn":249,"wo":518,"sî":86,"wr":185,"ws":439,"vy":161,"y ":7000,"wa":1979,"sâ":398,"we":941,"vl":85,"ré":180,"nţ":14138,"vi":18551,"râ":2794,"nş":236,"vu":1822,"vr":1272,"rî":85,"vs":260,"rí":114,"vn":113,"vo":6165,"uz":4516,"uy":132,"ux":811,"uw":93,"uv":2854,"uu":147,"ve":20356,"va":13083,"x ":3188,"ui":36731,"uj":940,"mţ":245,"uk":412,"ul":105505,"ue":3558,"uf":779,"ug":3958,"pă":8327,"uh":313,"ur":41535,"us":16414,"ut":20794,"um":22214,"un":77098,"uo":207,"up":12202,"ty":1026,"tz":478,"tu":41090,"tt":2205,"tw":494,"tv":129,"ub":9499,"ua":12620,"ud":10993,"uc":12122,"w ":1362,"to":36489,"tn":678,"tm":622,"tl":1983,"ts":1249,"tr":54687,"tp":270,"pâ":2492,"tf":804,"te":135412,"td":104,"lţ":1071,"tk":89,"ti":57663,"th":5025,"v ":5358,"tb":1584,"tc":809,"ta":57242,"su":20142,"sv":111,"ss":3065,"st":121541,"sy":354,"sz":231,"sw":164,"sl":2146,"sk":1369,"sn":714,"sm":3046,"sp":10131,"so":10359,"sr":365,"sd":200,"sc":21675,"sf":2108,"se":33580,"sh":1776,"nă":16784,"sg":163,"si":25637,"rz":902,"u ":55858,"sa":21555,"sb":549,"rr":1909,"rs":9764,"rt":23612,"ru":36664,"rv":2469,"rw":210,"ry":1487,"rp":2476,"ro":43127,"rn":8536,"rm":17803,"né":122,"rl":4592,"nç":103,"rk":1642,"rj":246,"ri":114051,"mă":7183,"rh":1830,"rg":8792,"rf":1291,"nâ":643,"re":125851,"rd":9049,"rc":6829,"rb":4129,"ra":67426,"t ":87746,"qu":761,"mé":97,"iţ":6799,"mâ":11991,"lă":11508,"iş":3714,"s ":29143,"px":1165,"py":91,"pt":7489,"pu":17959,"pp":664,"pr":42710,"ps":1182,"ză":8661,"zâ":331,"uţ":3113,"xă":383,"uş":1691,"vă":2660,"zz":266,"sţ":297,"zf":124,"vâ":2188,"rş":635,"zg":128,"uă":2010,"zi":14356,"zb":1622,"zd":236,"ze":7824,"vá":88,"za":10358,"zv":1283,"zy":102,"zu":1714,"zt":122,"zo":3689,"zn":131,"rţ":3519,"zm":105,"zl":93,"tă":34729,"ye":514,"yc":205,"yd":240,"ya":763,"yb":116,"yw":120,"yu":112,"yt":228,"ys":611,"yr":295,"yp":159,"yo":376,"yn":435,"ym":285,"yl":584,"yi":202,"² ":275,"Î ":96,"În":3944,"Îm":147,"án":283,"ác":265,"ár":117,"ás":83,"âl":535,"âm":971,"ân":23884,"âi":324,"âu":2278,"ât":1719,"âr":2512,"à ":112,"á ":120,"アアア":92,"ón":162,"ó ":105,"în":66365,"ín":93,"îi":280,"îl":220,"îm":1778,"ía":115,"î ":282,"âş":625,"él":95,"én":106,"és":86,"ér":215,"ée":141,"èr":89,"é ":392,"ăc":1712,"ăd":1353,"ăi":1593,"ăj":113,"ăg":390,"ăm":1728,"ăn":1902,"ăl":2480,"ăr":15029,"ăp":813,"ăv":254,"ău":2670,"ăt":8950,"ăs":5402,"ăz":2233,"ăb":253,"Ă ":96,"ă ":108634,"ün":238,"ür":317,"ör":105,"ön":99,"öl":118,"îş":473,"ăţ":4694,"ăş":1107,"şo":1467,"şt":10708,"şu":3626,"şi":42994,"şn":265,"şm":86,"şa":1934,"şc":1152,"şe":2787,"ş ":4403,"şă":140,"ţe":5578,"ţi":45859,"ţu":4697,"ţa":7179,"ţ ":1101,"ţă":5758,"ア":157,"ǎ ":253,"Ți":110,"Ța":292,"Ș ":94,"Și":132,"Șo":113,"Șt":390,"Ț ":94,"Șc":133,"Șa":135,"Șe":144,"Ță":266,"之":98,"三":194,"丁":102,"ος":175,"ος ":175,"ς ":359,"ν ":94,"α ":133,"アア":124,"ск":290,"та":92,"ст":158," Ga":1748," Câ":415," Ge":3154," Cá":254," I ":808," Fo":1868," Fu":504," Fr":2770," Fi":1622," Bâ":129," Fl":1278," Ha":2353," He":1731," Dâ":128," Gy":84," J ":235," Go":1158," Gr":2202," Gu":1163," Bă":725," Gh":755," Gi":841," Gl":431," Ig":92," Ie":183," Id":117," Ib":97," Ia":1248," K ":217," Hy":109," Hu":880," Hr":248," Ho":1557," Că":576," Hi":652," Ji":335," Je":625," L ":250," Ja":1632," Iz":203," Iu":1263," Iv":159," Ir":548," Is":1004," It":785," Im":1017," In":3630," Io":1287," Il":430," M ":262," Ka":1125," Ke":531," Ki":614," Kh":124," Fă":159," Jo":1852," Ju":930," N ":206," La":3007," Le":1942," Gă":138," Li":3175," Kl":144," Kn":98," Ko":661," Kr":346," Ku":321," Ky":124," Mc":200," Ma":9571," O ":853," Mi":4107," Me":2921," Lo":1967," Ll":318," Ly":104," Lu":1743," Ne":2351," P ":197,"а ":320," Na":2442," Ni":1749," Mo":4944," My":159," Mu":2527," A ":2659," B ":442," C ":439," Ap":1483," Am":1916," An":3623," Ak":86," Al":4275," Ai":406," Ag":455," Ah":117," Ae":277," Af":571," Ac":3199," Ad":1079," Ab":530," Ba":4239," D ":276," Az":205," Aw":102," Av":410," Au":2198," At":631," As":1386," Ar":4774," Be":2935," Bi":3051," Bl":566," Bo":2181," Br":3074," Bu":3675," E ":267," Ca":7457," Ce":2586," Ci":1423," Ch":2660," Cl":1578," Cr":1755," Co":8451," Cu":1909," Cy":110," F ":199," Da":1990," Di":2603," De":3729," Dr":841," Do":1853," Du":1487," Ea":726," Ec":468," Ed":725," G ":189," El":1961," Ei":240," Eg":286," Ef":83," Et":180," Es":2724," Er":511," Ep":280," En":810," Em":492," Ex":936," Eu":3393," Ev":441," Fe":1900," Fa":1274," H ":241," Să":460," Tâ":354,"к ":95," Wr":229," Wo":669," Wi":1265," Ră":813," Wh":231," We":849," Sâ":220," Wa":1026," Y ":114,"й ":231," Zo":278," Ze":389," Vâ":375," Zi":312," Za":381," Yu":120," Yo":676," Ya":164," Tă":194," Ye":128," Wü":172,"о ":126,"н ":131," Vă":153," a ":36875," R ":300,"в ":116," Kö":111," Ou":147," Ov":103," Os":331," Ot":292," Or":1764," Op":422," Po":3392," Pl":908," Pi":1531," Ph":433," Pe":2652," Pa":6041," Q ":118," Nu":741," No":2833," Ol":1044," On":368," Om":193," Oh":91," Od":204," Oc":1030," Of":168," Ob":298," Oa":113," Mü":186," Ra":1619," T ":190," Qu":324," Ro":9725," Re":6614," Ri":1081," Mă":767," Rh":203," S ":442," Pr":4358," Pu":723," Lă":359," Sz":142," Sy":283," Sw":111," Su":2564," St":5193," Ta":1158," V ":299," Th":2181," Ti":1481," Pâ":232," Te":2810," Tr":2917," To":1756," Ru":1539," Sa":3068," U ":212,"е ":128," Sh":834," Nă":243," Si":2443," Sc":1686," Se":3700," Sf":922," So":2361," Sp":3054," Sk":111," Sl":463," Sm":160," Va":2273," X ":295,"и ":98," Ve":1551," Vi":1939," Râ":933," Vl":295," Vo":709," Vu":110," Vr":132," Tu":1248," Ty":98," W ":133," Uc":367," Pă":634," Ul":182," Um":91," Un":5644," Ur":414," Ut":114," ja":470," l ":435," iz":399," io":137," ip":103," im":2637," in":15177," il":210," iu":2496," is":1936," it":606," ir":289," ka":124," m ":620," fă":1009," ki":195," gâ":148," jo":1049," ju":5955," ha":692," he":341," gi":362," gh":227," bă":568," gl":415," gr":4643," cî":96," go":322," gu":1027," ia":2677," id":624," ie":296," dă":97," aş":533," că":5014," hi":694," ho":634," hr":508," ht":170," ni":1017," ne":2985," na":3334," p ":138," mu":6108," mo":5925," ol":415," om":2225," on":472," oc":2425," od":198," of":2241," oa":721," ob":2230," nr":213," nu":8582," no":5141," le":5248," gă":633," li":8299," n ":4015," la":23505," km":1385," me":7797," eş":107," mi":5999," o ":24882,"я ":186," ma":19097," lu":7950," lo":8243," ae":559," af":3722," ag":621," ab":871," ac":11612," ad":3606," am":3470," an":10041," ap":7602," ai":935," aj":609," al":27010," av":3789," au":7831," ax":133," ar":7993," at":2213," as":3512," d ":818," ba":2886," az":461," bi":2225," be":668," bo":1156," bl":302," bu":1418," br":1785," ca":30202," e ":424," c ":92," er":1837," et":1185," es":44667," en":2470," em":716," ep":805," ei":565," el":3115," ef":535," eg":319," fe":3014," fa":5201," eu":1087," ev":1448," ex":4580," fu":2227," fr":3215," fo":24062," fl":777," fi":11367," ge":4653," câ":3570," ga":964," i ":494," cl":2039," cm":100," co":29681," cr":3733," cc":163," ce":15081," ch":1829," ci":2979," f ":104," da":4946," cu":21872," do":6118," dr":1888," de":93131," di":45684,"ч ":147," ec":2186," ed":1091," ea":295," eb":276," du":3328,"ль":104," vă":281," zo":1012,"ла":120,"ли":124,"ко":168," ze":739," zb":153," zi":1040," zf":120," vâ":719,"ка":188,"ки":145," tă":177," să":3897,"ин":138,"ик":85," ww":167,"ий":142," tâ":352,"ич":160,"ри":89,"ро":151,"ра":156,"ре":83,"ос":104,"ор":119,"ол":94,"ов":307," uş":254,"но":129,"ни":117,"на":162," ru":1689," sa":12280," sf":678," se":17401," sc":5833," si":9687," sh":138," nă":1457," sl":456," sp":5202," so":3426,"ви":183,"во":101," ra":2561," re":17790," ri":1713," mă":1822," ro":8367," pu":4423," pr":28214," ps":399," s ":2092," px":1165," lă":254," mâ":207," os":197," ot":110," op":1754," or":11013,"ан":170," ox":160,"ал":98," pe":25650," lâ":421," pa":9591,"ар":142," pl":2462," po":12520,"ая":98," pi":2615," wa":449," sâ":329," we":213," wr":124," wi":111," ră":1828," x ":110," va":2539," ve":4691," uz":162," vo":2209," vr":384," vu":171," râ":1545," vi":3164," uc":310,"ес":86,"ер":137,"ен":133," tu":1391," us":154," ut":956," ur":2048," um":561," un":37329," ul":1504," pă":1596," ta":986," st":9754," su":13217,"ев":109," tr":8292," to":2637," th":2187," ti":4195," te":6492," pâ":1627,"Țăr":258," Î ":96," Îm":146," În":3919," în":65579," îl":200," îm":1722," îi":236," î ":223," îş":424," Ă ":90," ţa":471," şt":950," şo":136," şi":37976," şc":219," şe":338," şa":764," ţi":391," ţe":161," ţă":1008," Ță":265," Ța":292," Ți":110," Șe":142," Șc":133," Șa":134," Ț ":92," Șt":387," Șo":113," Și":132," Ș ":91,"ال":185,"ي ":110,"ن ":132,"AS ":90,"BC ":103,"Feb":468,"Fed":536,"Fel":141,"Fer":333,"Fes":98,"Bâr":109,"Fil":558,"Fin":381,"Fir":91,"Fie":114,"Ext":451,"Fam":192,"Fan":103,"Fal":98,"Far":165,"Fac":313,"Fab":84,"Era":97,"Eri":131,"Est":2527,"Eur":3119,"Eva":149,"Eve":110,"Eug":130,"Exp":186,"Exi":117,"Exc":103,"Evu":86,"El ":710,"Ele":546,"Enc":124,"Eng":174,"Ene":132,"Emi":260,"Elv":237,"Eli":210,"Epi":189,"Ent":184,"Câm":221,"Cân":103,"Các":221,"Ger":1707,"Geo":684,"Gen":482,"Gla":99,"Ghe":506,"Băl":198,"Ghi":176,"Băt":252,"Gil":105,"Gir":238,"Giu":159,"Gaz":102,"Gal":559,"Gam":141,"Gav":159,"Gar":269,"Gab":147,"Fun":252,"Fru":86,"Fro":141,"Flo":463,"Fla":640,"Fra":1957,"Fri":249,"Fre":285,"Fon":202,"Fot":362,"For":809,"Fox":86,"II ":1544,"Dâm":116,"Căl":223,"His":116,"Hil":108,"Hel":236,"Hei":175,"Hea":139,"Hen":243,"Hes":101,"Her":541,"Hal":221,"Hai":98,"Han":325,"Ham":243,"Har":682,"Hau":104,"Gur":143,"Guv":353,"Gua":120,"Gui":207,"Gre":662,"Gri":261,"Gra":547,"Gru":280,"Gro":360,"ţă ":4334,"Glo":227,"Goo":112,"Gol":285,"Got":84,"ţăt":101,"Gor":250,"ţăr":1089,"ţăm":207,"Inv":138,"Ioa":445,"Inf":292,"Ini":100,"Int":1173,"Ins":803,"Ion":531,"Ios":98,"Ior":163,"Ili":165,"Ill":92,"Inc":150,"Ind":606,"Imp":850,"In ":158,"Iaş":404,"Ier":125,"Ian":535,"Ial":83,"Hun":383,"Hum":90,"IX ":412,"Hug":91,"IV ":391,"IT ":137,"Hor":215,"Hou":138,"Hot":160,"Hom":127,"Hon":117,"Hol":365,"Hr ":133,"Arg":385,"Arh":350,"Are":1696,"Arc":194,"Ard":100,"Ara":584,"Arm":402,"Ari":241,"Apo":195,"Apr":477,"Ate":96,"Atl":247,"Ast":296,"Ass":154,"Asi":398,"Aso":236,"Art":517,"Au ":88,"Avi":111,"Ave":127,"Aut":413,"Aus":786,"Aur":237,"Apă":85,"Aug":551,"Bai":170,"Bal":441,"Ban":574,"Bab":153,"Bac":358,"Bad":439,"Baz":111,"Bay":92,"Bar":873,"Bat":155,"Bas":347,"Bav":161,"CD ":118,"Abr":91,"Aca":558,"Act":175,"Ada":136,"Ace":1958,"Acc":188,"Adu":103,"Adm":250,"Ado":166,"Adr":161,"Ade":100,"Afa":103,"Aer":222,"Age":153,"Afr":381,"Agr":117,"Air":208,"Al ":217,"Ala":198,"Alb":1104,"Alg":107,"Ali":309,"Alc":89,"Ale":1016,"Alf":175,"Alt":180,"Alm":145,"All":248,"Alp":203,"Ame":1318,"Amb":85,"Ama":153,"Ang":951,"Ani":130,"Ana":326,"And":938,"Ant":732,"Ann":166,"Apa":378,"But":87,"Bus":115,"Buz":155,"Bul":406,"Bun":176,"Bur":449,"Buc":1806,"Bud":195,"Bru":284,"Bră":148,"Ca ":119,"Cab":93,"Cal":738,"Cam":777,"Cas":1102,"Car":1723,"Cau":101,"Cat":1040,"Can":883,"Cap":564,"Bea":150,"Bet":104,"Ber":735,"Ben":285,"Bel":1128,"Bib":207,"Bil":178,"Bih":202,"Bis":1833,"Bir":155,"Bio":98,"Blo":128,"CN ":94,"CO ":85,"Bla":274,"Bre":357,"Bra":1081,"Bro":311,"Bri":819,"Bog":147,"Boe":111,"Bol":190,"Bon":168,"Boo":93,"Bor":292,"Bos":170,"Bot":287,"Bou":149,"Cuv":164,"Cur":363,"Cup":386,"Cul":405,"De ":453,"Dez":104,"Der":87,"Det":101,"Des":342,"Dev":138,"Deu":92,"Del":231,"Dem":343,"Den":254,"Dep":258,"Dea":199,"Dec":652,"Dam":105,"Dan":549,"Dar":174,"Dat":151,"Dav":338,"Dac":187,"Dal":97,"Chr":305,"Che":298,"Chi":1153,"Cip":91,"Cin":138,"Cio":191,"Cit":251,"Ciu":252,"Civ":93,"DN ":145,"Cle":88,"Cla":363,"Cea":181,"Ceh":224,"Cel":451,"Cen":839,"Cet":187,"Cer":452,"Cha":723,"Cri":558,"Cra":341,"Cre":321,"Cu ":179,"Cru":182,"Cro":263,"Cli":120,"Clo":93,"şă ":93,"Clu":841,"Coc":83,"Coa":119,"Cod":224,"Cop":211,"Cos":228,"Cor":1033,"Com":2355,"Col":727,"Coo":87,"Con":2677,"Cou":282,"Cot":116,"Cov":111,"Ea ":436,"FA ":238,"Egi":189,"FI ":85,"Edu":88,"Edi":420,"Eco":139,"Ech":197,"ţur":116,"Eas":194,"ţui":121,"ţul":4343,"FC ":245,"ţar":732,"ţat":1470,"Deş":155,"ţe ":1287,"ţa ":4852,"Dia":147,"Dic":236,"Dis":609,"Dir":129,"Dio":102,"Din":623,"Dim":132,"Die":129,"Div":260,"ţit":365,"ţiu":2393,"ţir":210,"Duc":160,"ţin":4093,"ţio":5262,"Dup":205,"ţil":2341,"ţim":464,"Dun":357,"Dum":429,"ţii":7297,"ţif":426,"EX ":88,"Dur":97,"ая ":95,"ţia":6859,"ţie":10551,"ţei":1511,"ţen":180,"ţel":1678,"ţes":190,"Dre":250,"Dra":255,"ţi ":5299,"Doi":184,"Dob":158,"Dou":102,"Dol":170,"Don":271,"Dom":358,"Dor":224,"ţea":461,"Nea":360,"Neg":223,"Nev":114,"Neu":159,"Net":137,"Nep":85,"Nas":96,"Nat":327,"Nav":140,"Nig":112,"Nic":784,"Nis":298,"Nin":99,"Nik":102,"Naţ":910,"New":711,"Nap":289,"Nam":92,"Num":477,"OS ":93,"Nou":391,"Nov":172,"Nor":1220,"Not":118,"Noi":520,"Nob":138,"Înc":117,"Oct":512,"Înt":212,"Ode":88,"PC ":107,"Oce":295,"Împ":136,"Obe":86,"În ":3310,"Oto":155,"Olt":287,"Oli":317,"Ola":252,"Ono":115,"One":96,"Ope":278,"Ora":462,"Ort":215,"Osc":136,"Ord":181,"Ori":268,"Org":211,"Peş":117,"Plo":152,"Ple":87,"Pla":572,"Pin":170,"Pit":107,"Pir":96,"Pie":423,"Pic":155,"Pia":306,"Pho":97,"Phi":218,"Ped":83,"Per":522,"Pet":738,"Pen":513,"Pel":85,"Pe ":267,"Pat":391,"Pas":174,"Par":3240,"Pav":117,"Pau":327,"Pac":294,"Pan":274,"Pap":271,"Pal":567,"Pub":157,"Pur":83,"Put":143,"Pro":1436,"Pri":1498,"Pre":1075,"Pru":161,"Pra":285,"Pod":203,"Poa":126,"Pol":793,"Pom":101,"Pon":142,"Poi":169,"Pot":185,"Pos":133,"Pop":623,"Por":507," ال":165,"Lăc":323,"SA ":143,"Rac":113,"Rad":382,"Rai":238,"Ram":106,"Mün":150,"Ran":107,"SD ":117,"Que":155,"Isa":101,"Irl":266,"Ita":687,"Isl":175,"Isr":173,"Ist":357,"Ira":132,"Iug":109,"Iva":127,"Iul":580,"Iun":435,"Izv":134,"Jae":83,"Jac":383,"Jap":315,"Jan":217,"Jam":257,"Jer":140,"Jea":190,"Jim":122,"Jos":531,"Jon":151,"Joh":545,"Joc":288,"Jud":261,"Jus":113,"Jur":102,"Jul":125,"Jun":115,"Kal":121,"Kan":96,"Kat":124,"Kar":336,"Ken":149,"Kir":86,"Kin":185,"Kie":90,"Kon":94,"Kos":163,"Kre":87,"Kra":118,"Lew":83,"Lev":90,"Let":83,"Les":120,"Leo":277,"Len":176,"Lei":93,"Leg":332,"Lee":87,"Lea":120,"Lau":147,"Laz":87,"Le ":90,"Las":86,"Lat":109,"Lar":154,"Lam":95,"Lan":338,"Lac":259,"Lab":85,"La ":1181,"Lle":258,"Lib":362,"Lic":139,"Lie":89,"Lig":414,"Lim":410,"Lin":461,"Lis":431,"Lit":301,"Liv":198,"MI ":96,"Lux":121,"Lup":107,"Lum":137,"Lun":558,"Lud":146,"Luc":356,"Lou":179,"Lov":137,"Los":244,"Lot":89,"Loc":483,"Lor":132,"Lon":361,"Meh":106,"Men":147,"Mem":162,"Mel":209,"Mes":271,"Mer":299,"Met":461,"Mec":135,"Med":595,"Mex":216,"Man":1131,"Mal":342,"Mar":4668,"Mas":368,"Mag":383,"Mad":271,"Maj":121,"Mai":903,"Mac":356,"May":100,"Max":155,"Mau":132,"Mat":338,"Mod":176,"Mol":1838,"Mon":1277,"Mos":318,"Mor":392,"Mou":115,"Mot":179,"Mih":609,"Mik":106,"Mij":92,"Mid":157,"Mig":86,"Mic":936,"Mit":232,"Mir":321,"Mis":279,"Mil":373,"Min":651,"Muz":367,"Mun":1060,"Mul":94,"Mur":508,"Mus":300,"NU ":88,"Săl":172,"Săr":113,"Târ":305,"XX ":206,"XV ":95,"Wre":180,"Wor":386,"Wol":146,"Whi":111,"Răd":101,"Răz":528,"Răs":89,"Wil":518,"Win":333,"Wie":86,"Wit":97,"ère":85,"Web":84,"Wes":374,"Sân":132,"Was":121,"War":230,"Wat":92,"Wal":290,"Vra":93,"ée ":112,"Vol":202,"Voi":185,"Vis":124,"Vit":135,"Vla":243,"Ziu":91,"Zon":137,"Zee":92,"Vâl":191,"Vâr":128,"âşt":551,"Yor":388,"You":200,"на ":94,"Stă":112,"Sys":110,"Stî":142,"Sur":132,"Sus":204,"Sul":83,"Sup":346,"Sun":219,"Sue":270,"Sud":589,"Suc":243,"Sub":164,"Str":664,"Stu":292,"Sti":105,"Sto":394,"Sta":2644,"Ste":728,"Teh":119,"Tea":170,"Tec":113,"Ten":121,"Tem":164,"Teo":264,"Tel":395,"Tan":144,"Tat":111,"Tar":189,"Tai":83,"Tal":112,"UA ":470,"Sfâ":498,"Shi":108,"She":197,"Năs":219,"Sho":130,"Sha":300,"Sim":365,"Sil":252,"Sig":138,"Sit":102,"Sis":180,"Sir":233,"Sin":438,"Sie":127,"Sib":370,"Sfi":241,"Sez":141,"Ser":760,"Sev":325,"Sf ":135,"Scr":179,"Sep":495,"Sen":217,"Sel":98,"Sem":99,"Sec":315,"Sea":111,"TV ":299,"Spa":2255,"Spi":211,"Spe":223,"Spr":121,"Spo":178,"Sof":165,"Soa":134,"Soc":483,"Sou":289,"Sov":352,"Sol":260,"Som":154,"Son":175,"Sor":138,"Sla":128,"Slo":264,"Roş":199,"Rus":998,"Sai":239,"Sam":225,"Sal":380,"Sab":86,"Se ":674,"Sco":358,"Sci":132,"Sch":680,"Sca":185,"Sax":130,"Sav":134,"Sat":330,"Sau":85,"Sar":201,"San":758,"ови":108,"TA ":103,"Rez":129,"Res":168,"Ret":90,"Rev":440,"Măn":337,"Măr":265,"Rhe":108,"Riv":111,"Rin":182,"Ric":330,"Rap":96,"Ref":116,"Rec":405,"Red":97,"Rei":117,"Reg":1929,"Rem":121,"Ren":298,"Rel":102,"Rep":1978,"Rea":273,"Rol":118,"Rob":343,"Roc":197,"Rod":127,"Roy":87,"Rot":120,"Ros":294,"Rom":7794,"SS ":239,"SO ":102,"Reş":139,"Vel":106,"Ven":242,"Vec":214,"ски":101,"Vas":382,"Van":141,"Val":1157,"Var":247,"Vic":330,"Vie":288,"Vir":168,"Vil":251,"Vin":215,"Râu":810,"Ver":423,"Ves":364,"Păm":274,"Păd":123,"Ung":582,"Uni":3993,"Un ":792,"VD ":93,"Ucr":344,"VI ":204,"Tex":109,"Ter":953,"Tes":175,"Pân":88,"Pâr":129,"Tha":108,"The":1600,"Thi":130,"Tho":227,"Tib":83,"Tim":723,"Tin":109,"Tit":147,"Tir":112,"Top":132,"Tor":259,"Tok":94,"Tol":279,"Tom":280,"Ton":126,"Tot":116,"Tou":121,"Tru":137,"Tro":183,"Tri":361,"Tre":328,"Tra":1888,"Tur":781,"Tul":107,"Tun":93,"Tud":91,"вич":136,"biz":191,"bis":684,"bit":942,"biu":329,"bio":569,"bir":494,"baţ":118,"bil":3719,"bin":1111,"bii":385,"beş":168,"bo ":147,"blu":442,"şa ":427,"blo":266,"ble":593,"bli":4345,"bla":309,"boa":326,"bol":798,"boi":1231,"bog":143,"biş":155,"biţ":97,"şe ":206,"şca":505,"şal":128,"şan":384,"şap":297,"bon":331,"bom":116,"bor":1299,"bot":233,"bos":94,"bov":171,"şar":126,"şas":296,"bou":197,"box":118,"şat":99,"be ":463,"ban":1063,"bal":2277,"bai":156,"baj":238,"bac":607,"bab":295,"án ":151,"baz":1371,"bat":704,"bas":635,"bar":766,"bdi":107,"bea":119,"bi ":576,"bei":94,"bee":100,"bec":120,"ber":2623,"ben":568,"bel":997,"bes":190,"bet":600,"bia":703,"bib":169,"bic":703,"bie":1278,"áce":223,"şnu":136,"buţ":387,"buş":122,"şoa":686,"şor":235,"şov":403,"ca ":12197,"car":20537,"cas":995,"cat":4867,"cau":568,"can":4982,"cap":1522,"caz":602,"cav":140,"cac":106,"cab":250,"cad":1721,"cam":769,"cal":7164,"caf":88,"cai":123,"şu ":299,"ce ":13744,"bri":8393,"bro":479,"şco":241,"bra":1280,"bre":732,"bu ":154,"şea":100,"bru":3685,"şed":992,"bso":227,"bse":314,"şez":265,"şev":87,"bst":592,"şel":294,"şef":250,"şer":161,"şen":171,"şi ":40695,"bur":1502,"bul":939,"bun":893,"bum":1812,"bui":746,"buc":290,"but":536,"bus":305,"buz":102,"şcă":238,"şie":311,"şii":223,"by ":257,"şia":83,"bră":230,"şit":599,"şir":239,"şin":563,"şil":101,"aka":186,"am ":1398,"ake":307,"aki":175,"ajo":811,"aju":1101,"al ":30968,"aja":297,"aje":549,"adă":644,"ail":713,"aim":241,"ain":2042,"aio":622,"air":263,"ais":301,"ait":135,"aiu":141,"ak ":275,"aie":333,"aid":284,"aic":780,"aib":127,"aia":581,"ahn":102,"ahi":154,"acă":1581,"ahu":103,"aho":221,"aj ":993,"adâ":150,"ârâ":145,"aha":317,"agl":117,"agm":128,"agh":501,"abă":348,"agi":1223,"agr":589,"agu":482,"agn":543,"ago":775,"anu":8793,"anz":399,"any":226,"ano":1227,"ann":872,"ant":7062,"ans":5187,"ane":4573,"ang":1746,"anh":83,"ani":17950,"anj":319,"ank":595,"ap ":209,"ana":3541,"anc":4078,"and":7634,"amu":1451,"amm":250,"amo":500,"amn":539,"amp":1577,"ams":162,"ami":2968,"ame":7488,"amb":1018,"ama":1598,"alz":98,"alv":263,"alu":3357,"alt":3527,"als":199,"alp":187,"alo":2252,"aln":118,"alm":409,"all":1429,"alk":125,"alg":190,"agă":206,"ali":13360,"alc":1323,"ald":503,"ale":15313,"alf":367,"ala":3376,"alb":2405,"an ":14955,"aku":88,"ako":128,"acţ":1206,"ârş":461,"aba":554,"abe":900,"abi":2311,"abl":317,"abo":655,"abr":755,"abs":332,"abu":239,"ae ":1225,"aca":773,"aaa":86,"aal":84,"aar":122,"ad ":1372,"ânt":3632,"şur":726,"ânu":94,"ânz":207,"şul":2492,"âns":100,"ac ":1040,"ştr":244,"âmt":99,"şti":6783,"şte":3540,"âmp":537,"âng":896,"âne":1748,"şta":102,"ând":5547,"âni":6072,"ab ":204,"ânc":472,"âna":293,"afr":149,"aft":132,"aff":130,"afe":369,"afi":1982,"afl":2509,"ai ":11311,"aga":899,"age":1103,"afu":111,"âur":92,"âul":1913,"aen":112,"ael":563,"aes":133,"aer":621,"ah ":371,"âte":342,"afa":1595,"ado":765,"ârs":182,"adr":1167,"ârt":98,"adm":1348,"adj":104,"adi":2476,"ârz":238,"âu ":266,"ade":2813,"ag ":301,"ână":2880,"adt":92,"adu":1730,"aco":1240,"acl":100,"ack":689,"aci":1819,"ach":1150,"ace":7711,"ât ":1298,"acc":1560,"ârb":268,"ada":3740,"ârf":276,"ârg":288,"af ":209,"acv":115,"ârn":129,"act":5256,"acu":1523,"ârl":126,"acr":543,"azo":188,"azi":1570,"arţ":1475,"azu":425,"aze":593,"avâ":822,"aza":1518,"azd":83,"azz":139,"avă":214,"asă":1423,"axi":362,"axo":262,"az ":338,"axa":101,"atâ":459,"ată":12359,"âi ":95,"ays":88,"aya":143,"aye":215,"ân ":1771,"âlc":198,"ba ":2953,"âmb":183,"ază":4808,"âln":280,"âin":172,"at ":24564,"amă":335,"arh":1164,"arg":1153,"arf":116,"are":42152,"ard":2497,"arc":2307,"arb":848,"ara":6502,"arp":413,"aro":1047,"arn":629,"arm":1225,"arl":2466,"anç":90,"ark":583,"ari":10760,"aru":1463,"arv":129,"arr":585,"ars":566,"art":13731,"au ":13208,"asa":2050,"ary":262,"arz":166,"asi":2001,"ană":4433,"ash":375,"asc":1871,"ase":2773,"aso":520,"asn":213,"asp":360,"ask":118,"asm":175,"asl":121,"ar ":8918,"apa":2757,"ape":1246,"api":1462,"aph":142,"apl":456,"apo":1977,"app":137,"apr":2890,"aps":139,"apt":1565,"apu":270,"as ":1819,"ală":6884,"ava":1432,"ax ":193,"auz":385,"aux":116,"aut":2879,"avr":217,"avo":325,"anţ":3393,"avi":1654,"anş":96,"ave":2326,"ay ":775,"awa":162,"avy":131,"avu":1095,"ară":4267,"av ":216,"ata":4096,"asu":901,"ast":6214,"ass":610,"asy":113,"atm":162,"alţ":185,"atl":201,"atr":2501,"ato":5035,"ate":24962,"atf":138,"atc":92,"ati":8684,"ath":466,"aw ":115,"aua":249,"auc":144,"att":328,"ats":134,"atu":9240,"aul":521,"aum":109,"aun":296,"aur":1060,"aus":649,"aud":358,"aug":1299,"apă":1796,"amţ":177,"Wür":170,"ка ":90,"ий ":128,"ич ":137,"jec":116,"jel":228,"jen":194,"fâr":376,"fân":580,"ji ":99,"jat":278,"jap":309,"jar":183,"jan":170,"jaz":89,"je ":321,"joa":246,"joc":694,"joz":167,"jos":164,"jor":690,"jit":146,"jin":221,"jaţ":96,"bţi":531,"jo ":103,"jlo":496,"itm":199,"itl":818,"itr":663,"ito":6294,"itu":8009,"itt":280,"its":137,"itz":191,"ity":420,"iub":117,"iuc":160,"iua":221,"iud":230,"ipă":404,"isk":89,"ism":2124,"isl":772,"iso":596,"isn":143,"isp":881,"iss":507,"isr":101,"isu":227,"ist":19214,"iv ":3796,"ita":17021,"itc":116,"ite":7821,"ith":307,"iti":4614,"ivo":234,"ivu":281,"ius":647,"iur":519,"ium":322,"iul":7954,"iun":9474,"iva":1811,"ix ":362,"ivi":3232,"inţ":4376,"ive":4336,"ipr":212,"ipo":338,"ipp":144,"ipu":386,"ips":239,"ipt":541,"ipi":1233,"aţă":1525,"ipl":436,"is ":3857,"ion":10900,"iop":116,"ior":1475,"ios":402,"iot":422,"iou":159,"iog":188,"iol":1112,"ipa":2480,"ipe":948,"iov":274,"ir ":799,"iru":345,"irs":100,"irt":172,"iro":730,"irm":483,"eîn":134,"irk":94,"irl":333,"iri":2227,"isi":1374,"ish":315,"ină":2514,"isf":103,"ise":3028,"isc":2092,"isa":444,"iu ":4493,"iqu":132,"ilă":788,"inâ":388,"ire":5707,"imă":577,"irg":179,"ira":895,"irc":947,"it ":8220,"ünc":112,"iză":370,"ja ":337,"ită":7145,"ixt":86,"isă":796,"ixe":89,"iz ":119,"ivă":1340,"izu":147,"izv":208,"izo":952,"izi":2792,"ize":859,"iza":6312,"kil":158,"kin":314,"kir":111,"kis":140,"km ":1110,"ki ":642,"făc":542,"făr":519,"kel":127,"ken":233,"kes":118,"ker":360,"ket":144,"fă ":122,"gân":291,"kfu":88,"ke ":452,"kra":120,"kre":109,"kt ":111,"kov":198,"km²":262,"kol":93,"ks ":224,"făş":535,"cţi":4945,"ko ":243,"jut":351,"jus":113,"jul":508,"jun":382,"jum":207,"jur":1124,"jud":4033,"juc":683,"kar":128,"kan":145,"kai":124,"kad":96,"ka ":524,"ha ":718,"ham":492,"han":1154,"hai":779,"hak":92,"hal":402,"hau":169,"har":1141,"has":93,"hat":192,"hae":287,"hag":86,"hab":90,"he ":3331,"hel":915,"hei":851,"hee":120,"hed":164,"hea":469,"hez":295,"hev":85,"het":520,"hes":378,"her":1057,"heo":670,"hen":622,"hem":280,"că ":14688,"hi ":1098,"dân":197,"căi":96,"căl":499,"căd":116,"căz":100,"căp":117,"căs":168,"căr":2073,"cău":418,"căt":3820,"hie":477,"hid":863,"hic":510,"hib":86,"hia":1153,"hip":1598,"hio":209,"hin":1334,"him":1433,"hil":698,"hii":152,"hiu":429,"hiv":233,"his":738,"hit":1266,"hir":636,"hiz":150,"hn ":414,"hle":150,"ho ":172,"gma":139,"go ":432,"gme":179,"glo":451,"gle":2336,"gli":771,"gn ":157,"gla":327,"gog":138,"goa":126,"gnu":85,"gno":150,"gni":185,"câş":533,"gne":487,"gna":170,"geş":236,"gs ":160,"gol":308,"gon":427,"gos":413,"gor":814,"got":99,"gov":160,"gu ":263,"gro":318,"gru":1821,"gra":4239,"gri":471,"gre":2106,"gto":159,"gui":168,"gum":179,"gul":1451,"gua":163,"gue":323,"gy ":135,"gră":290,"guv":925,"gur":1708,"gus":1778,"gvi":266,"iam":458,"ial":8383,"ian":7496,"ias":341,"iar":3383,"iau":106,"iat":2458,"ic ":11072,"iab":156,"iac":516,"iad":260,"iag":132,"ibl":375,"ibi":1287,"ibo":95,"ibr":438,"ibu":1167,"iaz":367,"id ":1405,"iba":208,"ibe":1115,"ia ":37192,"iet":2227,"ieu":92,"iev":591,"iew":117,"iez":102,"iel":766,"iem":1503,"ien":2907,"iep":154,"ier":4481,"ies":1526,"ied":283,"ief":128,"iei":9680,"aş ":2678,"ig ":320,"iec":2094,"ifu":325,"ifo":500,"ifr":208,"iff":103,"ife":1805,"ifi":2918,"dă ":1783,"ifa":105,"icr":452,"ics":163,"ict":2757,"icu":1908,"ico":2006,"ick":421,"icl":554,"ici":10582,"ich":1701,"ice":8554,"ie ":34669,"ica":13932,"idu":1244,"idr":458,"ido":239,"idi":1327,"idg":110,"ide":3884,"ida":1312,"if ":119,"iic":157,"iaş":207,"iaţ":1579,"idă":222,"iin":4917,"iil":3254,"iit":1059,"il ":2791,"ija":114,"iji":265,"ijl":490,"ijo":87,"im ":1225,"ika":140,"aţa":1077,"aşe":616,"ige":485,"aşc":90,"aşa":330,"iga":1279,"ii ":23496,"igm":125,"igh":1464,"igi":3078,"aşi":1717,"icâ":83,"igu":863,"aşu":2407,"aşt":704,"igr":211,"aşo":348,"igo":379,"ign":421,"dăc":87,"iha":624,"ică":10472,"ihi":105,"dău":132,"iho":423,"dăr":156,"ik ":233,"imo":864,"imn":359,"imm":132,"imp":4726,"ieş":578,"imf":123,"ime":4318,"ieţ":356,"imi":3490,"ip ":1099,"inc":6719,"ind":7549,"ina":7598,"imu":2173,"inn":213,"inm":129,"ino":1626,"int":13234,"ins":3523,"inf":1447,"ine":10869,"inh":83,"ing":4928,"ini":7437,"inl":374,"ink":229,"ioa":4790,"ioc":312,"iod":317,"inu":2953,"inv":735,"iny":91,"inz":228,"iko":237,"icţ":367,"aţi":16926,"iki":186,"ike":152,"aţe":270,"ila":1467,"ilb":98,"in ":44693,"ilo":9977,"ill":1766,"ilm":2143,"igă":285,"ilh":99,"ili":9285,"ild":180,"ile":12078,"ima":4046,"imb":4580,"io ":1569,"ily":129,"ilt":143,"ilu":581,"ilv":1166,"hiş":314,"how":106,"hol":376,"hom":292,"hon":245,"hos":162,"hot":319,"hou":159,"hov":191,"hoo":89,"hop":132,"hor":542,"hoe":88,"hoc":86,"hni":581,"hno":368,"hne":86,"heţ":86,"hul":158,"hua":109,"htt":211,"hte":132,"hro":140,"hre":117,"hri":377,"ht ":1261,"hra":577,"hiţ":84,"hy ":109,"hwa":155,"hum":1416,"hus":146,"hur":185,"fi ":1715,"ffe":164,"ffi":135,"feu":99,"fet":117,"fes":1040,"fer":4398,"fec":1006,"fed":383,"feb":993,"fem":538,"fen":538,"fel":943,"fib":88,"fia":458,"faz":136,"fas":102,"fat":182,"far":402,"fap":490,"fam":1470,"fan":745,"fal":367,"fai":144,"fac":2220,"fab":663,"ff ":157,"fe ":258,"euş":129,"eză":2753,"fa ":130,"exu":301,"ext":1548,"etă":1447,"exa":920,"ez ":2054,"ews":115,"exp":1323,"epţ":305,"esă":354,"exi":1412,"exc":376,"exe":917,"ezv":899,"ezu":584,"evă":275,"eza":830,"ezo":739,"eze":4121,"ezi":2997,"erţ":230,"eta":3771,"epâ":360,"ete":2601,"etc":489,"eti":2973,"eth":248,"etn":363,"esp":1890,"eso":846,"est":55654,"esu":952,"ess":1010,"ev ":251,"euc":86,"eud":222,"epă":213,"eum":115,"eul":878,"eun":627,"eto":805,"etr":2728,"ets":89,"ett":511,"etu":1024,"etw":102,"ew ":746,"eve":1810,"eva":1191,"evo":1396,"enţ":4104,"evi":2580,"eut":437,"eur":1309,"eus":264,"ex ":508,"ewi":114,"eră":1931,"evr":439,"ey ":1079,"ewa":103,"epe":1003,"epi":751,"eph":288,"er ":7653,"epa":1109,"eot":142,"eos":392,"eor":2351,"eom":197,"eol":739,"eop":176,"eon":471,"elă":165,"eiţ":151,"es ":5970,"ept":3637,"epu":3456,"epl":448,"epp":122,"epo":532,"epr":2782,"erk":178,"erl":756,"eri":23643,"erg":1582,"emă":409,"erh":102,"ere":9729,"erf":436,"erc":1961,"erd":628,"era":10049,"erb":1083,"et ":2968,"esk":95,"esl":106,"esf":637,"enă":298,"esh":139,"esi":1831,"esb":88,"esc":5445,"ese":2325,"eu ":1098,"esa":2035,"erz":182,"ery":156,"erv":1894,"eru":1701,"erw":96,"err":761,"ert":2020,"ers":6219,"ern":5307,"erm":6392,"erp":946,"ero":1890,"eki":83,"ecţ":1050,"en ":4825,"elb":109,"ela":2761,"eld":314,"elf":84,"ele":22448,"eli":2942,"elg":1037,"egă":470,"elm":166,"eln":87,"ell":1629,"elo":7272,"elu":2902,"elv":133,"els":286,"elt":287,"eo ":496,"emb":7566,"ema":2057,"eme":3406,"emn":2329,"emo":1281,"ehă":102,"emi":3276,"emu":1154,"emp":1201,"ems":94,"ep ":186,"ene":5641,"enh":144,"eng":1760,"enb":358,"ena":2023,"end":1865,"enc":997,"eno":950,"enn":450,"enk":132,"eni":7113,"enu":3506,"ens":2167,"ent":26508,"enr":218,"eoa":176,"enz":462,"eog":396,"eod":283,"eoc":172,"egl":156,"ego":650,"egn":92,"ege":2159,"ecâ":351,"egi":6402,"egh":185,"egr":796,"egu":603,"ehn":787,"eho":111,"ehe":163,"ehi":357,"ecă":398,"ek ":382,"eic":203,"eia":920,"eis":644,"eir":131,"eim":251,"eil":863,"ein":887,"eii":140,"edă":119,"eaţ":502,"eie":409,"eid":483,"eig":145,"eaş":214,"eja":336,"el ":7309,"eit":209,"eiu":167,"eke":85,"em ":1443,"eju":102,"giz":650,"giu":3192,"git":384,"gis":1283,"gir":91,"gil":280,"gim":999,"gaţ":227,"gip":292,"gin":3668,"gio":683,"gid":91,"gie":1712,"gic":1515,"gii":481,"gia":2426,"bău":105,"băr":238,"ght":1266,"băt":475,"băn":99,"ghi":1143,"ghe":1448,"gha":167,"cât":1186,"gi ":458,"câi":94,"cân":2264,"câm":225,"gen":4213,"geo":702,"get":450,"ger":3349,"ges":353,"gh ":216,"bă ":1169,"gea":351,"geb":118,"gem":253,"gel":950,"gda":132,"ge ":2096,"gaz":511,"gby":87,"gas":177,"gar":1662,"gau":115,"gat":2233,"gaj":154,"gam":133,"gal":1275,"gan":2759,"ga ":1522,"îşi":423,"fuz":427,"fur":551,"fus":129,"ful":658,"fun":1914,"ftw":294,"ft ":458,"fra":2347,"fre":553,"fri":632,"fiţ":85,"fro":290,"fru":551,"for":8285,"fos":13009,"fot":1551,"fon":2006,"fol":2314,"flă":1101,"fiş":196,"feţ":166,"foc":121,"foa":663,"fle":282,"fla":1128,"fli":157,"flu":1381,"flo":329,"fic":5576,"fie":1608,"fig":239,"fii":4052,"fil":2800,"faţ":1332,"fin":1931,"fir":582,"fis":97,"fit":276,"fiu":394,"fix":216,"fiz":803,"cuţ":360,"da ":5431,"dba":199,"de ":81669,"dac":568,"dad":169,"dal":1004,"dai":112,"daj":204,"dag":131,"dae":325,"dat":5418,"dar":2392,"dap":257,"dan":788,"dam":518,"day":88,"dav":93,"dau":120,"cup":2017,"cun":3394,"cul":7448,"cum":2248,"cui":2182,"cuf":114,"cub":97,"cuc":191,"cua":310,"ctu":3820,"ctr":1989,"cto":4340,"cti":3912,"cte":2267,"cta":977,"coţ":184,"cră":288,"cy ":117,"cve":360,"cvi":112,"cva":166,"cus":289,"cur":6209,"cut":5650,"cuv":696,"ctă":260,"Șco":105,"cks":236,"ckh":87,"cla":1563,"cle":639,"clu":1691,"cli":610,"clo":419,"ceş":351,"cmi":98,"co ":1054,"cni":95,"cod":494,"coe":86,"cof":296,"cog":235,"coa":845,"cob":156,"coc":134,"con":13122,"coo":319,"col":4664,"com":13771,"cor":2665,"cos":422,"cop":2252,"cov":597,"cot":422,"cou":153,"cs ":186,"clă":363,"cqu":87,"ct ":2355,"cre":3245,"cra":1687,"cri":4361,"cru":523,"cro":1167,"cu ":14527,"cci":346,"cca":249,"cce":1997,"cea":5425,"cez":1350,"ch ":1051,"cev":105,"cer":3600,"ces":6831,"cet":999,"ceu":226,"cen":3570,"cep":2469,"cem":1612,"cel":8680,"cei":1228,"ceh":124,"cee":645,"ced":641,"ci ":3845,"Ști":135,"cha":1202,"Ște":246,"chw":220,"chu":202,"cia":8398,"ck ":1749,"cie":2502,"cid":710,"cic":613,"che":3039,"chl":127,"chi":6835,"cho":240,"chn":170,"chs":144,"cht":297,"ciz":320,"civ":411,"caţ":977,"cil":1200,"cim":273,"cif":735,"caş":448,"cii":3633,"cir":703,"cis":679,"cit":1218,"ciu":1361,"cin":2022,"cio":753,"cip":3172,"cm ":96,"cke":198,"ed ":964,"eba":172,"ebe":338,"ebi":493,"ebo":152,"ebr":2253,"ebu":793,"ec ":965,"eac":1155,"eag":683,"eae":329,"ead":219,"eak":166,"ean":4116,"eal":2295,"eam":780,"ear":779,"eas":2985,"eap":381,"eav":360,"eat":1944,"eau":812,"eaz":4296,"eb ":255,"ea ":37128,"efi":943,"efl":126,"efo":468,"efa":467,"efe":1778,"eff":109,"ei ":24037,"ega":2463,"eft":125,"efu":229,"eek":131,"een":318,"eel":266,"eea":861,"eed":149,"eer":336,"eep":102,"eet":278,"edi":6693,"ede":2885,"eda":988,"eg ":239,"edu":500,"edo":913,"edr":363,"ecl":455,"eck":180,"ech":2954,"eci":4375,"ece":4160,"eca":1262,"ee ":711,"ef ":336,"ecv":378,"ecu":2928,"ect":7556,"ecr":556,"eco":3176,"dwi":90,"dwa":128,"dy ":306,"dve":124,"dur":1540,"dut":183,"dus":2699,"duş":94,"dor":982,"dop":268,"don":1156,"dom":1511,"dol":361,"dox":527,"dow":256,"dov":2135,"dou":2054,"dos":139,"ds ":634,"diţ":1218,"diş":157,"deţ":4616,"dmi":1594,"dne":111,"doa":843,"dob":218,"doc":617,"dof":143,"doi":1173,"Țar":280,"dun":278,"dum":103,"dup":2223,"dui":145,"dul":3772,"dub":195,"dua":160,"duc":3456,"dri":740,"dra":1452,"dt ":159,"dre":2042,"du ":925,"dro":713,"dru":1503,"dge":174,"dic":3233,"did":167,"dia":4181,"der":5158,"des":4508,"det":654,"deu":131,"dev":1104,"dez":1907,"deb":451,"dea":1065,"ded":225,"dec":2801,"def":691,"dee":144,"deg":147,"dej":85,"dei":744,"del":2089,"den":4447,"dem":1462,"dep":2099,"deo":975,"di ":446,"dle":126,"dla":95,"deş":387,"do ":923,"dja":87,"div":1213,"diu":1854,"diz":147,"dim":595,"din":41384,"dio":1557,"dip":176,"dir":1178,"dis":3885,"dit":1572,"die":1235,"dif":1731,"dig":295,"dii":366,"daţ":232,"dil":110,"rgu":556,"rhe":302,"rj ":109,"rha":273,"rcă":369,"rhi":1066,"măr":2035,"măs":677,"măt":499,"măn":620,"rfu":326,"rga":2411,"ri ":15694,"rgi":1524,"rgh":746,"rbă":345,"rge":1425,"rgo":215,"ret":2582,"res":5915,"rev":1477,"reu":1156,"rew":92,"rez":5275,"rey":127,"mă ":3118,"rfa":103,"rfe":202,"rfi":117,"nân":632,"rfo":213,"rdu":767,"rds":382,"rg ":1476,"reb":516,"rea":18674,"ree":777,"ref":1708,"rec":5448,"red":1424,"rei":3494,"rej":104,"reg":5973,"rem":2573,"ren":2941,"rek":143,"rel":3576,"rer":308,"reo":342,"rep":4378,"rf ":235,"rda":852,"rcu":1210,"rct":172,"rdo":623,"rdi":1685,"rde":964,"re ":51071,"rbu":436,"rco":232,"rci":1263,"rch":609,"rce":1470,"rca":1029,"ray":96,"raz":629,"rd ":3327,"rap":865,"rar":2929,"ras":1358,"rat":9747,"rau":625,"rav":765,"rbi":1165,"rbo":575,"rba":839,"rbe":545,"rc ":287,"raj":371,"rai":1767,"rah":296,"rag":920,"ran":9556,"ram":2869,"ral":5890,"rak":183,"rab":1135,"raf":2847,"rae":350,"rad":3282,"rac":2468,"rpu":387,"rpr":755,"rpo":278,"rs ":1565,"rpe":182,"rpa":307,"rpi":136,"ror":724,"ros":1102,"rot":1489,"rom":6745,"ron":3428,"roo":164,"rop":6403,"roz":338,"rou":752,"rov":4132,"row":176,"rox":732,"rob":828,"roa":1307,"rod":3362,"roc":2541,"roi":1094,"rol":1987,"rof":1423,"roe":237,"roh":90,"rog":1247,"rno":327,"rnu":701,"rp ":237,"rna":2969,"rne":1758,"rni":1303,"reţ":1190,"rmo":436,"rmu":522,"ro ":2108,"rma":9885,"rme":3062,"reş":4337,"rmi":1348,"rls":94,"rlo":266,"rgă":207,"rli":413,"rld":353,"rle":420,"rla":2658,"rn ":698,"rks":134,"rke":254,"rm ":799,"riz":1053,"rl ":211,"rip":540,"rio":5018,"rir":448,"rit":7787,"ris":4701,"riv":1528,"riu":2574,"rdă":172,"rig":3530,"raş":5092,"rij":321,"rii":7673,"ril":8587,"rik":126,"raţ":2453,"rin":9619,"rim":5400,"ria":7770,"rib":1255,"ric":11723,"rid":1224,"rie":15408,"rif":356,"rk ":829,"roş":363,"rsă":312,"rui":1637,"rug":366,"rud":206,"ruc":1749,"rur":434,"rup":3037,"run":754,"rum":1184,"rul":9603,"ruz":121,"rux":123,"rus":1703,"rut":1017,"rva":737,"rvi":837,"rve":659,"rvo":119,"ry ":1178,"rsk":120,"rsi":1656,"rso":2085,"rsc":90,"rsa":860,"rnă":515,"rsh":97,"rse":1189,"rta":3149,"rst":426,"rsu":1104,"rto":1146,"rte":6836,"rth":550,"rti":6445,"rua":1449,"rts":182,"rtr":166,"rtu":1485,"rtt":156,"riţ":1112,"riş":356,"rmâ":178,"rt ":2203,"rro":293,"rmă":1457,"rri":340,"rre":470,"rra":339,"ru ":12849,"rry":345,"sc ":2232,"sab":243,"sac":340,"sad":91,"sag":137,"sai":83,"saj":124,"sal":1207,"sam":459,"sbe":121,"san":693,"sau":8354,"sat":4809,"sas":155,"sar":1304,"sa ":3104,"ón ":120,"ruş":110,"rze":109,"rtă":914,"rzi":434,"sha":218,"sho":144,"năr":653,"năs":2089,"năt":775,"she":136,"scă":1232,"shi":636,"năl":262,"si ":1523,"sfâ":443,"siv":865,"sie":984,"sid":975,"sic":666,"sib":419,"sia":1164,"sk ":250,"nău":325,"sit":6826,"siu":1567,"sir":294,"sis":2018,"sip":93,"sin":2491,"sio":987,"sil":2030,"sim":1237,"sih":314,"sii":129,"sif":302,"sig":587,"scr":3431,"scu":7764,"sbu":216,"se ":12289,"sca":1105,"sce":729,"sci":701,"sch":1873,"sco":2491,"sex":343,"sey":122,"ser":5132,"ses":902,"set":368,"seu":253,"sfa":100,"sez":381,"sh ":403,"nă ":12507,"sfi":168,"sfe":495,"sfo":294,"sea":1239,"sei":529,"see":104,"sed":775,"sec":2976,"seb":398,"sep":1669,"seo":135,"sen":1576,"sem":2294,"sel":1265,"spu":788,"spo":1513,"spr":1609,"spe":3508,"spi":782,"spa":1327,"sou":172,"sov":534,"sol":1405,"som":99,"son":2390,"sop":133,"sor":985,"sos":101,"sod":147,"sof":666,"soa":1403,"soc":1613,"su ":139,"sra":283,"st ":20718,"ss ":769,"sli":132,"slo":207,"slu":183,"sla":1454,"sle":117,"ski":424,"sfă":558,"sko":122,"sm ":736,"ska":303,"sna":139,"sni":276,"sne":177,"smo":155,"seţ":118,"smu":996,"so ":251,"sma":281,"seş":442,"smi":416,"sme":397,"soţ":294,"stâ":421,"stă":4284,"sse":604,"ssa":443,"sso":336,"ssi":591,"ssu":98,"ste":53928,"spâ":306,"stf":531,"sta":11097,"stm":85,"sto":3769,"sti":8907,"stl":380,"stu":4079,"str":12855,"sty":123,"sud":1668,"sue":207,"sub":3715,"suc":517,"spă":179,"suf":275,"sul":3772,"sum":370,"sup":2843,"sun":3645,"sut":208,"sus":705,"sur":1681,"suv":86,"sy ":155,"tai":479,"taj":164,"tal":7011,"taf":116,"tag":288,"tab":988,"tac":696,"tad":477,"tc ":460,"tba":1494,"tax":122,"tav":190,"tau":320,"tat":21973,"tas":470,"tar":6168,"tap":132,"tan":6974,"tam":968,"tch":206,"te ":79600,"suş":110,"ta ":8238,"oză":144,"pa ":2141,"oxă":260,"pe ":11904,"par":10822,"pat":2004,"pas":403,"pay":83,"pac":752,"pad":108,"pab":102,"pag":502,"pal":1642,"pai":83,"pap":190,"pam":183,"pan":5506,"phe":183,"pha":169,"pho":127,"phi":253,"pi ":238,"ph ":186,"lân":506,"pea":2043,"pec":3297,"ped":445,"pen":9052,"per":11036,"pet":1065,"pes":1059,"pei":1300,"pel":911,"pla":2428,"pli":1624,"ple":1254,"plo":453,"plu":926,"phy":102,"pia":1002,"pid":288,"pic":1543,"pie":1703,"pii":550,"paţ":844,"pil":610,"pin":728,"pio":918,"pir":800,"pis":848,"pit":1452,"piu":839,"poz":1416,"por":4692,"pop":3746,"pov":414,"pot":1327,"pos":1095,"poi":429,"pog":111,"pom":116,"pon":1731,"pol":4578,"poa":1462,"poe":647,"poc":495,"pod":314,"ps ":147,"ppe":285,"peş":178,"po ":105,"pta":1006,"pse":287,"psi":410,"pso":85,"ptu":1231,"pub":3609,"pte":2615,"pti":445,"pto":319,"pra":2308,"pt ":1178,"plă":316,"pru":251,"psa":94,"pri":13510,"pre":11982,"pro":14449,"pră":140,"ptă":561,"pur":1199,"pus":1351,"put":2590,"pun":2005,"pul":6398,"px ":1151,"puş":180,"puţ":344,"mân":11918,"lă ":9449,"iş ":380,"lăc":209,"lăd":384,"lăr":280,"lău":157,"lăt":576,"işa":158,"işc":590,"işe":100,"işi":457,"işo":462,"işn":143,"işu":248,"işt":1090,"iţe":420,"iţi":4507,"iţa":1141,"lăţ":94,"iţă":635,"qua":125,"que":368,"qui":218,"ra ":10628,"rb ":100,"ngo":269,"ngi":1205,"ngl":3065,"ngv":280,"ngu":1282,"ngr":328,"ngt":167,"ngs":258,"ni ":7085,"nge":1701,"ncâ":95,"ngh":727,"nga":1437,"nha":227,"ncă":762,"neg":502,"nei":3106,"nel":2471,"nen":1433,"nem":658,"nep":189,"neo":674,"ner":4256,"net":2067,"nes":1905,"nev":324,"neu":474,"ng ":2660,"nea":6161,"nec":713,"ned":727,"nee":362,"nef":188,"nfi":1313,"nfo":1463,"nfl":766,"nfr":253,"nfu":123,"ney":389,"nez":1743,"nex":158,"nfa":94,"nfe":701,"ncr":242,"nct":978,"nco":1033,"nci":6536,"ncl":974,"nce":5421,"nch":1298,"nca":697,"ne ":15507,"nbu":225,"ndu":3412,"ndr":2268,"nds":282,"ndo":1104,"ndi":3514,"nde":6309,"ndb":189,"nda":4140,"ncy":92,"ncu":853,"nal":8677,"nam":764,"nan":953,"nap":143,"nar":3126,"nac":272,"nad":606,"nag":295,"nah":123,"nai":648,"naj":750,"nc ":203,"nbe":172,"nd ":10185,"nav":597,"nau":373,"nat":5278,"nas":876,"naz":192,"na ":10566,"muş":108,"moţ":117,"nyi":101,"ntă":3762,"nz ":336,"ntâ":712,"nsă":705,"noţ":101,"ny ":515,"noş":120,"nvi":271,"nux":134,"nve":1249,"nva":170,"nul":11142,"num":9716,"nun":659,"nui":2359,"nus":463,"nut":1282,"nuu":86,"nuv":87,"nur":524,"nua":2364,"nue":133,"nuc":341,"nty":205,"nto":1327,"ntu":5345,"nts":104,"ntr":20429,"nti":5120,"nth":232,"nta":5926,"nte":13468,"nsu":2110,"nsn":130,"nsm":304,"nsp":737,"nso":628,"nst":5554,"nsf":385,"nse":1387,"nsh":116,"nsi":3286,"nsl":126,"nsk":220,"nsc":408,"nsa":2214,"nu ":2393,"nru":101,"nri":188,"nre":465,"nt ":10809,"niţ":1334,"niş":600,"ns ":1634,"noc":153,"nod":134,"noa":802,"nob":170,"nog":217,"nol":681,"noi":1449,"nop":228,"nom":2958,"non":580,"not":496,"nos":3264,"nor":3599,"nov":557,"nou":744,"noz":170,"nr ":214,"nne":796,"nna":357,"nno":163,"nni":320,"nny":106,"nme":156,"nma":86,"neţ":165,"neş":1107,"nli":309,"ngă":685,"nn ":394,"nla":407,"no ":1211,"nlo":244,"nkf":88,"nke":129,"ncţ":1362,"nki":120,"nka":125,"nkt":91,"înţ":344,"nje":175,"nja":194,"nju":195,"nii":3788,"ndă":489,"naş":348,"nig":141,"nif":691,"nie":6578,"nid":183,"nic":6607,"nib":123,"nia":13403,"nk ":409,"niz":1868,"nix":112,"niu":1843,"niv":2033,"nis":4310,"nit":7595,"nir":583,"nio":766,"nim":2048,"nin":1227,"naţ":2629,"nik":97,"nil":2118,"ogr":2357,"ogu":334,"ogi":3138,"ogl":113,"ogo":258,"ogn":118,"oga":345,"ogd":87,"oge":523,"oi ":1939,"ohi":99,"oho":87,"ohn":440,"oha":219,"ocă":128,"ohe":86,"ois":381,"oir":138,"oiu":600,"oit":118,"oin":312,"oaţ":87,"oil":1036,"oaş":230,"odă":270,"înă":265,"oii":90,"oic":184,"oid":249,"oie":2586,"ok ":171,"oia":306,"obţ":455,"ol ":1950,"oiz":87,"oce":1371,"och":606,"oci":2584,"ock":1201,"ocl":282,"ocm":112,"oco":593,"împ":1524,"ocr":595,"obs":247,"obu":209,"oe ":171,"oca":5500,"occ":161,"îmb":234,"ode":1683,"odi":1388,"înv":645,"odo":1005,"înt":5727,"înr":506,"îns":1196,"odr":168,"ocu":4799,"înl":252,"oct":1467,"îng":438,"înf":1221,"of ":1300,"înd":673,"înc":2936,"îna":811,"oda":803,"oel":100,"oem":170,"oes":95,"oet":442,"oen":174,"ody":97,"odu":3682,"og ":550,"ofi":1691,"ofu":93,"oft":569,"ofo":587,"oez":194,"off":117,"ofe":1241,"ofa":91,"nzâ":121,"ob ":212,"îl ":203,"nză":237,"oc ":1888,"oap":522,"oan":2626,"oam":565,"oal":517,"oai":251,"oad":3000,"oac":410,"în ":51095,"oba":769,"od ":1364,"oar":8811,"oas":2028,"oat":2201,"obo":275,"obr":179,"obl":493,"obi":2216,"obe":608,"nza":270,"nze":236,"nuă":93,"nzi":410,"nzo":124,"nvă":302,"îi ":276,"otă":208,"oya":123,"oxi":1058,"oxe":145,"oz ":239,"ows":299,"own":215,"oră":461,"owi":83,"ovă":83,"orţ":934,"ozo":459,"oze":129,"ouă":1877,"ozi":1701,"oza":446,"otu":494,"oua":772,"ow ":278,"olţ":86,"oti":981,"oth":247,"ote":1814,"ott":429,"ots":89,"otr":617,"oto":1723,"ost":14678,"osu":176,"ota":1624,"otb":1480,"ov ":924,"osi":2418,"osh":105,"onă":692,"ose":1383,"osf":245,"osp":184,"oss":270,"osm":217,"osl":251,"oso":685,"osn":103,"oy ":170,"owa":92,"owe":145,"ovi":5242,"onţ":708,"ovo":380,"ovs":162,"ouv":83,"ox ":338,"ova":2953,"ove":2093,"orâ":88,"oug":124,"oui":176,"oul":710,"oun":598,"oup":188,"ous":347,"our":791,"out":476,"opo":1954,"opp":88,"opi":1812,"opl":160,"ope":4845,"oph":214,"opa":1115,"os ":2802,"opu":4445,"opr":1065,"opt":660,"ops":95,"oon":154,"ool":264,"ook":218,"ood":367,"or ":26373,"oot":145,"oor":339,"oop":114,"ork":590,"orl":449,"orm":7578,"orn":1037,"oro":968,"orp":872,"orr":316,"orc":286,"ord":5185,"ore":2827,"orf":352,"org":3181,"omă":1147,"ori":16103,"orj":132,"ou ":829,"osa":293,"osc":3302,"ort":5002,"ors":359,"orv":285,"oru":3894,"ory":244,"olă":311,"omâ":10245,"m² ":271,"ot ":1395,"orb":953,"ora":7147,"ola":2095,"old":2473,"olc":184,"on ":8021,"olj":96,"oli":6258,"oll":628,"olk":148,"olf":429,"ole":2350,"olg":83,"ols":165,"olt":1104,"olm":146,"oln":127,"olo":7169,"oly":117,"olu":4417,"olv":270,"om ":2396,"oki":92,"oke":139,"ona":10266,"ond":4478,"onc":1725,"onf":1659,"one":3635,"ong":702,"onj":211,"oni":6032,"onl":252,"onn":355,"ono":3183,"ons":6444,"ont":4103,"onu":2421,"onv":453,"ony":208,"onz":212,"oma":4275,"oo ":114,"ome":3698,"omb":2289,"omi":3631,"omm":249,"omp":6241,"omn":706,"omo":1504,"omu":6796,"op ":1017,"la ":23354,"lb ":181,"le ":40374,"lce":389,"lca":429,"lcl":134,"lch":83,"lci":164,"lcu":522,"lco":212,"lf ":227,"lde":317,"lda":182,"ldo":2003,"ldi":182,"ldu":132,"lab":636,"lac":1028,"lad":507,"lae":470,"lah":161,"lag":391,"laj":309,"lai":336,"lal":366,"lan":7785,"lam":2475,"lap":209,"lar":3822,"lat":4828,"las":2223,"lax":136,"lau":569,"lav":737,"lay":251,"lba":949,"ld ":1067,"lbe":538,"lbi":174,"lbo":115,"lbu":1909,"ky ":197,"kso":178,"lpi":329,"lph":128,"ls ":385,"lpt":166,"lon":2782,"lom":608,"lop":364,"loo":88,"lor":17380,"lod":381,"loc":8139,"log":3938,"loi":220,"los":2548,"lot":440,"lou":173,"lov":661,"low":107,"loz":395,"lni":416,"lne":136,"loa":1027,"lob":416,"lmo":117,"lmi":101,"leţ":90,"lme":740,"leş":291,"lma":385,"lna":86,"lmu":506,"lth":83,"lti":1438,"lto":515,"ltr":114,"ltu":1401,"lud":310,"luc":1053,"lub":644,"lua":671,"lug":175,"lue":1286,"lsi":131,"lso":99,"lst":136,"lta":1833,"lte":3356,"lu ":1358,"lse":84,"lsa":93,"liţ":378,"ía ":107,"liş":264,"lt ":1778,"găt":711,"gău":109,"găr":308,"găs":508,"lhe":118,"lcă":318,"lj ":83,"lgo":101,"lge":272,"lgi":975,"lbă":158,"li ":1453,"lga":583,"lfu":196,"lfo":121,"gă ":1042,"lfa":415,"lez":1735,"ley":333,"lex":1098,"leu":326,"lev":1072,"les":2557,"let":1046,"ler":1357,"leo":330,"lep":98,"lem":2259,"len":1650,"lek":93,"lel":1961,"lei":1502,"leg":2868,"lef":302,"lee":148,"led":336,"lec":2987,"leb":331,"lea":5194,"lls":187,"llu":109,"lly":238,"lo ":746,"lla":922,"lle":1509,"lli":1027,"llo":392,"ln ":118,"lm ":1367,"ll ":1537,"lit":10843,"lis":3477,"lir":181,"lip":665,"lio":847,"lin":3950,"lim":4821,"liz":3596,"liv":266,"liu":800,"lic":6840,"lid":584,"lia":5811,"lib":815,"lk ":118,"lik":113,"laţ":3598,"lil":379,"lii":609,"lig":1016,"laş":630,"lie":5757,"lif":704,"ma ":3957,"mb ":1486,"mac":332,"mai":9505,"maj":528,"mad":521,"mae":84,"mag":1419,"mar":6545,"mas":1035,"mal":1356,"mam":301,"man":9186,"max":174,"mat":6955,"mba":2864,"mbl":374,"mbi":1641,"mbe":588,"mbr":9100,"mbo":699,"me ":5049,"mbu":511,"med":2490,"meg":129,"mea":1453,"mec":632,"eş ":976,"met":2646,"mes":1043,"mer":5757,"mem":2715,"mel":4206,"men":12391,"mei":953,"hă ":127,"mez":105,"mex":145,"mfo":85,"luz":516,"lva":1356,"lve":558,"lvi":203,"lul":6901,"luj":787,"lui":26427,"lup":481,"lun":1806,"lum":1944,"lut":376,"lus":1014,"lur":909,"lux":157,"luv":232,"ly ":423,"ltă":720,"luţ":840,"mpi":1771,"mpe":1922,"mpr":847,"mpo":2882,"mpl":2315,"mpu":2400,"mps":120,"mpt":100,"ms ":246,"mog":102,"moc":469,"mob":459,"mod":2074,"mon":3526,"mom":281,"mol":693,"mov":314,"mor":1244,"mos":450,"mot":668,"mou":110,"mpa":3076,"mnâ":163,"miş":960,"miţ":248,"mto":98,"mst":145,"mnă":376,"moş":151,"my ":245,"mur":1183,"mus":438,"mut":273,"mpă":755,"mul":9511,"mun":7969,"muz":1842,"eşt":6641,"eşu":215,"eşa":195,"mi ":621,"eşi":561,"mbă":561,"eşe":1204,"min":5735,"mio":138,"mil":3851,"mir":1594,"mis":1523,"mit":6241,"miu":551,"miz":90,"mix":115,"eţ ":436,"mic":3196,"mia":589,"mig":156,"maş":321,"mif":173,"mie":1057,"mid":167,"maţ":1711,"mij":400,"mii":918,"mo ":311,"mn ":1067,"eţu":4047,"eţi":1636,"eţe":1089,"eţa":97,"mm ":112,"moa":417,"mnu":348,"mni":697,"mna":983,"mne":568,"mmy":134,"mp ":917,"mmo":100,"mma":189,"meş":407,"mme":257,"ăţă":263,"vă ":1622,"văr":375,"văz":128,"sţi":293,"văţ":340,"ziţ":770,"zvo":1221,"zua":152,"zur":207,"zul":782,"zut":213,"zz ":105,"rşi":489,"zi ":1628,"vâr":386,"vân":1791,"zec":677,"zei":499,"zea":690,"zdu":96,"uă ":1773,"zeu":674,"zes":98,"zen":2310,"zel":386,"zer":601,"ze ":1470,"zbo":1342,"zbu":165,"zf ":120,"zac":138,"zah":127,"zam":95,"zan":432,"zar":1712,"zau":120,"zat":4549,"zoa":213,"zot":117,"zor":484,"zom":97,"zon":1721,"zol":318,"zof":297,"rţă":100,"zo ":122,"rţe":443,"rţa":169,"rţi":2629,"rţu":83,"rţ ":87,"uăz":94,"zib":96,"zia":1295,"zie":586,"zid":234,"zic":2773,"zii":433,"zin":2192,"zim":92,"zil":806,"zaţ":846,"zio":247,"zir":91,"zis":507,"zit":1014,"ziu":1413,"yst":189,"ysi":92,"ys ":152,"tăţ":3951,"yon":90,"za ":1974,"ywo":86,"ye ":94,"tă ":27261,"yer":209,"ya ":253,"yan":149,"yn ":128,"yle":187,"yo ":86,"yne":94,"tăl":523,"tăn":89,"tăm":184,"tăz":367,"tăt":383,"tăr":1670,"tăp":108,"yin":96,"tîn":184,"xt ":219,"xtr":935,"xtu":126,"xte":487,"xti":276,"xpr":511,"xpo":137,"xpl":405,"xpe":325,"xon":265,"ăţi":4022,"ăţe":252,"ăşe":185,"ăşi":118,"ăşo":168,"ăşu":379,"ăşt":194,"xul":217,"xua":180,"săm":195,"săn":94,"săi":149,"tât":368,"târ":286,"tân":1123,"să ":6878,"tâi":116,"tâl":280,"tâm":92,"xem":547,"xer":167,"xec":276,"xel":184,"pţi":497,"xis":976,"xil":150,"xim":851,"xid":122,"xic":439,"xig":85,"săr":830,"săs":204,"săp":211,"săt":291,"său":989,"xcl":154,"xce":187,"xe ":275,"xat":91,"xan":620,"xac":149,"ww ":168,"www":168,"oţi":926,"oţe":90,"wn ":181,"ws ":286,"wre":154,"wor":208,"woo":193,"răş":270,"răţ":90,"sân":187,"ră ":10046,"wer":171,"wel":112,"wei":114,"web":177,"oş ":108,"răb":152,"răd":292,"răc":168,"răg":121,"răj":88,"răi":753,"răn":163,"răm":369,"oşt":362,"oşu":230,"oşa":206,"oşi":435,"sâr":193,"wis":119,"wig":94,"wic":83,"win":151,"rău":112,"răs":796,"răt":330,"răr":765,"răz":877,"wa ":123,"wan":150,"wal":599,"way":113,"war":714,"viţ":222,"viş":91,"vro":102,"vri":167,"vre":765,"vra":124,"vsk":216,"vut":1014,"vul":660,"vy ":150,"via":1245,"nţ ":202,"vio":597,"vir":425,"vaţ":256,"vil":1004,"vin":4712,"vig":218,"vii":288,"vic":1489,"vid":1022,"vie":1344,"viz":1818,"viu":343,"vit":1797,"vis":1422,"nţa":4036,"nţe":2750,"nţi":4753,"nţu":270,"vo ":269,"veţ":303,"veş":233,"voa":145,"voc":768,"vod":247,"voi":344,"vol":2372,"von":307,"vor":1265,"vot":169,"vos":108,"nţă":2104,"vi ":279,"rât":86,"râu":1467,"râr":88,"vez":158,"ver":5755,"ves":2284,"vet":135,"râm":188,"rân":915,"vei":383,"veh":194,"veg":396,"ven":3368,"vel":1246,"vea":1522,"ved":733,"vec":1443,"ve ":2034,"val":1848,"van":2141,"var":1789,"vat":1229,"vas":355,"vaz":94,"vac":381,"vad":175,"vai":89,"uză":207,"uzâ":97,"va ":4378,"usţ":288,"uzu":128,"uzi":2633,"uvâ":532,"uze":512,"uza":616,"ută":2040,"uxe":267,"usă":1693,"uz ":167,"ură":2750,"upă":3161,"ux ":398,"uvi":580,"unţ":1150,"uve":1576,"urâ":83,"ună":2252,"ush":108,"usi":1228,"use":1795,"usc":466,"usa":280,"uu ":89,"usu":380,"ust":4305,"uss":282,"uso":94,"ulţ":391,"uth":338,"uti":1787,"ute":2406,"uta":1302,"utt":99,"uts":136,"utu":1616,"uto":3287,"utr":237,"us ":5190,"ulă":561,"ut ":7246,"urb":404,"ura":6173,"urd":294,"urc":802,"ure":4226,"umă":1226,"urg":1586,"uri":11434,"urk":100,"urm":1472,"urn":1161,"uro":4600,"urp":138,"urr":101,"urs":2050,"urt":1427,"uru":770,"ury":127,"urz":103,"unz":236,"upa":1044,"ur ":1460,"păş":90,"upi":117,"upe":1889,"upo":171,"upr":3078,"upl":134,"upt":549,"upu":1021,"ump":261,"umu":1472,"umi":4694,"umo":274,"umn":200,"uma":1213,"umb":1802,"ume":7127,"unt":4936,"uns":365,"unu":4147,"unk":173,"uni":10730,"uno":4024,"unn":89,"unc":3006,"und":2612,"una":3959,"ung":1996,"une":10686,"up ":833,"ucţ":871,"um ":3644,"ulu":21791,"ult":5763,"uls":125,"ulp":273,"ulo":572,"ulm":181,"ull":195,"uli":2417,"ugă":238,"ulg":558,"ule":2260,"ulc":332,"ula":6764,"ulb":106,"un ":26513,"uid":145,"uie":791,"uaţ":284,"uil":227,"uin":534,"uir":346,"uis":305,"păt":304,"păs":368,"uk ":108,"mţ ":178,"uia":763,"uit":3995,"ul ":62901,"ucâ":92,"ugh":328,"ugi":103,"ugb":125,"uge":384,"ugo":240,"ufu":127,"ui ":29161,"uga":410,"păi":109,"ucă":1628,"păd":164,"păr":2478,"păm":252,"ugu":1856,"uha":84,"uj ":675,"uco":175,"ucr":1114,"uct":1068,"ucu":1790,"uda":563,"ude":4922,"udi":2446,"ubo":209,"ubm":96,"ubs":626,"ubt":149,"ubu":425,"uca":1046,"ue ":469,"uce":1968,"ucc":519,"uci":711,"uch":237,"ucl":385,"uck":119,"uer":254,"ues":246,"pă ":4354,"uff":102,"ufe":113,"ufi":109,"ufl":99,"udu":394,"udo":402,"ug ":96,"ued":469,"uea":236,"uen":1166,"uel":339,"ub ":2463,"tuţ":713,"ua ":1456,"uat":4356,"uar":3767,"ual":1937,"uan":471,"ubi":491,"ubl":4031,"ube":197,"uba":209,"ubc":93,"ubd":102,"ud ":2013,"uad":93,"uc ":293,"tze":90,"tyl":114,"tuş":101,"ty ":780,"twa":283,"trâ":424,"tur":9303,"tus":270,"tut":920,"tui":1608,"tul":18833,"tun":827,"tum":175,"tua":5218,"tud":1979,"tue":106,"tug":400,"tz ":249,"two":138,"toţ":88,"tră":1759,"toş":149,"ts ":552,"tiţ":743,"tiş":169,"tre":13329,"tt ":332,"tra":10048,"tri":7435,"tru":15106,"tro":4167,"tu ":332,"tsc":140,"tsu":91,"tst":102,"tta":206,"tte":678,"tti":236,"ttl":111,"tto":150,"ttp":214,"tts":95,"teş":321,"tme":92,"tma":135,"to ":1187,"tmo":189,"tni":304,"tne":121,"tp ":211,"tna":149,"tno":89,"tof":98,"tod":1039,"toc":696,"toi":238,"tog":562,"tob":142,"toa":4019,"tou":153,"tov":167,"tos":511,"tot":1088,"tox":87,"tom":2691,"ton":3163,"tol":1579,"tor":17808,"top":634,"tr ":2261,"tii":1559,"til":2704,"taţ":1027,"tif":641,"tie":3100,"taş":87,"tig":849,"tir":1365,"tit":3982,"tis":1483,"tin":5976,"tim":3231,"tip":1320,"tio":2187,"thu":1487,"tia":853,"tib":219,"tic":12965,"tid":1357,"tiz":272,"tiu":313,"tiv":7387,"lţi":875,"tli":252,"tlu":762,"tla":333,"tle":501,"tem":5382,"ten":2986,"teo":833,"tep":232,"tei":2254,"tej":251,"tel":8378,"tee":83,"tef":321,"teg":901,"teh":698,"tea":10690,"teb":131,"tec":1715,"ted":479,"pân":2359,"tfo":164,"tfe":489,"tfa":104,"th ":1080,"tez":721,"tex":467,"tev":339,"teu":83,"tet":387,"tes":961,"ter":17000,"ti ":4750,"pâr":90,"tho":249,"thr":172,"the":1255,"thi":201,"tha":314,"ăpi":117,"ăr ":639,"ărg":110,"ări":6448,"ărc":171,"ăre":1669,"ărb":346,"ăra":1104,"ăpu":119,"ăpt":200,"ălă":333,"ămâ":1238,"ăsp":468,"ăsu":674,"ăst":933,"ăta":498,"ăte":138,"ăpâ":120,"ăti":182,"ălţ":396,"ăto":3911,"ăro":211,"ăru":1791,"ărt":197,"ărs":101,"ău ":1645,"ăsc":1578,"ăse":603,"ănă":998,"ăsi":241,"ăut":251,"ărâ":207,"ătu":1360,"ătr":2034,"ăud":174,"ăul":112,"ăsă":746,"ără":1000,"ăzu":203,"ărţ":804,"ăzi":480,"ăze":118,"ăzd":88,"ăzb":1237,"ătă":689,"ăuţ":165,"ăce":168,"ăca":435,"ăci":357,"ăcu":469,"ăde":218,"ădi":504,"ădu":392,"ăi ":268,"ăia":213,"ăie":323,"ăcă":141,"ăl ":131,"ăin":267,"ăit":151,"ădă":164,"ăil":231,"ăgă":140,"ăle":134,"ălc":90,"ăld":111,"ăla":258,"ălb":84,"ăma":126,"ămi":144,"ăli":530,"ălu":247,"ăne":410,"ănc":84,"ăni":157,"ză ":7802,"zăr":410,"zăt":197,"zău":117,"zân":327,"ürt":162,"uş ":157,"xă ":377,"uţi":2814,"uşu":97,"uşt":92,"uşo":225,"uşe":140,"uşc":117,"uşi":662},"n_words":[8213281,9706468,7977386],"name":"ro","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/ru.json b/contrib/languages-data/ru.json
new file mode 100644
index 0000000..944e03f
--- /dev/null
+++ b/contrib/languages-data/ru.json
@@ -0,0 +1 @@
+{"freq":{"D":12949,"E":10306,"F":8444,"G":9635,"A":18320,"B":11812,"C":18729,"L":10133,"M":18025,"N":9350,"O":8480,"H":7852,"I":36250,"K":4555,"U":4649,"T":15138,"W":7160,"V":11078,"P":15071,"S":24620,"R":10933,"X":14985,"f":12270,"g":18709,"d":30289,"e":107154,"b":13058,"c":32324,"a":88793,"n":72221,"o":77565,"l":46717,"m":27155,"k":11440,"h":29689,"i":78874,"w":9055,"v":9950,"u":33414,"t":65000,"s":53078,"r":72396,"p":19491,"z":3846,"y":15658,"x":6168,"́":88446,"ь":418052,"э":80439,"ю":190059," o":3966,"я":756306,"ш":164289,"щ":122233,"ъ":10879,"ы":569718,"ф":163421,"х":364215,"ц":246632,"ч":362951,"р":2010938," a":3919,"с":1933070,"т":1865754,"у":741481," t":5450,"ё":73990,"И":44043,"Л":43881,"К":96587,"Н":57501,"М":84561,"П":106733,"О":53561,"Б":68591,"А":88041,"Г":58489,"В":87514,"Е":15516,"Д":50036,"З":22723,"Ж":9219,"Ш":21342," H":6358," I":14743,"Ю":9600," N":6330,"Я":13672," O":4736," L":7278,"Э":22580," M":13492,"Т":45273," B":8977,"У":23856," C":13678,"Р":75819," A":12906,"С":148731,"Ц":10320," F":6714,"Ч":15862," G":7609,"Ф":36383," D":9169,"Х":20421," E":6845,"л":1373509,"к":1271742,"й":669125,"и":2990916," X":9849,"п":840034,"о":3520831,"н":2386826,"м":997958,"г":667312," S":16699,"в":1520663," R":8106,"б":419069,"а":2917550," P":10356," W":6218,"з":510343,"ж":242497," V":5002,"е":2737376,"д":981077," T":11223," А":74197," Б":66425," В":84271," Г":55440," Д":46951," Е":13561," Ж":8620," З":21445," И":40121," К":90300," Л":41939," М":81442," Н":51890," О":47190," П":101697,"Co":3768,"I ":12785," б":96560," а":114180," г":188817," в":459004," е":32581," д":177939," з":81625," ж":31277," й":4814," и":369580," л":56771," к":233167," н":240854," м":177553," п":455990," о":265653," Р":63171," С":125419," Т":41894," У":21149," Ф":32286," Х":19430," Ц":9599," Ч":15364," Ш":14654," Э":20859," Ю":9415," Я":13422,"C ":3880," т":131031," у":81588," р":185096," с":401251," ц":26307," ч":72261," ф":71734," х":30938," ш":31141," ю":9238," я":35568," э":51707,"Ma":3809,"II":10412,"VI":3879,"Th":4668,"X ":5588,"S ":3829,"a ":12175,"i ":4290,"he":9109,"g ":4678,"ea":4162,"ec":3745,"de":5482,"h ":4087,"el":5609,"en":10589,"et":4586,"es":8452,"er":17575,"e ":29556,"f ":4058,"ch":4543,"ce":3891,"c ":3917,"d ":10245,"at":8929,"as":4202,"ar":11277,"al":8675,"am":4333,"an":13622,"ac":3987,"nt":7777,"ns":3721,"of":4484,"om":4497,"on":14150,"ol":4531,"os":3740,"ou":4174,"or":10679,"r ":11184,"ll":5038,"o ":6694,"ma":4022,"me":5234,"na":4836,"nd":7645,"ne":5797,"ng":6066,"ni":4386,"m ":5088,"li":5421,"le":7129,"la":5868,"n ":18576,"ic":8588,"ia":4454,"ie":3873,"k ":4036,"is":6529,"it":5343,"il":4505,"in":14086,"io":6997,"l ":9526,"y ":8801,"ve":4522,"x ":4062,"ur":4265,"us":4229,"to":4590,"te":9298,"ti":9123,"th":7132,"ta":6068,"st":7180,"ro":7763,"ri":8232,"re":8155,"ra":7850,"t ":13376,"s ":22614,"́в":5893,"́д":3913,"́й":4535,"́м":3791,"́л":9850,"́н":15831,"́р":9052,"́с":4186,"́т":6032,"К ":4112,"О ":4223,"А ":10572,"В ":18440,"ья":15305,"ью":22763,"эл":12007,"эк":10864,"эр":3980,"эн":8177,"эт":23791,"юб":4435,"юг":4684,"юд":5402,"юз":4895,"юл":6476,"юн":7127,"юр":4992,"юс":3863,"ют":24842,"юч":8444,"ющ":38941,"яж":4621,"яе":26767,"яд":10334,"яв":18798,"яб":17686,"ян":29130,"ям":15306,"ял":9628,"яз":22046,"ях":9639,"яс":6449,"ят":41978,"яр":7070,"яю":12156,"ящ":14702,"щи":42463,"ще":53089,"ща":18897,"ъе":8051,"ый":115257,"ык":21356,"ые":44083,"ыд":3688,"ыв":20957,"ыш":9845,"ыч":6916,"ых":88893,"ыт":10971,"ыс":15933,"ыр":7427,"ып":8198,"ын":9614,"ым":40979,"ыл":22285,"ьк":10517,"ьд":4034,"ье":17860,"ьз":12362,"ьб":9420,"ьш":16573,"ьс":28658,"ьт":14115,"ьн":93902,"ьм":17794,"хе":5192,"хи":15842,"хн":9628,"хо":65229,"хр":8875,"хс":5190,"ху":8026,"фф":4256,"ха":22339,"ци":120621,"цк":12640,"цо":4729,"цу":6546,"ца":29752,"це":46729,"чл":3954,"чн":40293,"чи":44063,"чк":5394,"чт":9754,"цы":8412,"че":142550,"ча":74585,"шо":6953,"шн":6580,"шк":7840,"шл":7340,"ши":40508,"шт":17550,"ше":40253,"чё":6388,"ша":19003,"ск":366421,"см":15604,"сл":64229,"со":139920,"сн":41282,"ср":13223,"сп":78471,"св":31144,"рё":5020,"сб":4953,"се":102280,"си":96102,"рш":7321,"ры":45781,"рь":12146,"са":63371,"ря":46850,"рр":14529,"рс":52200,"рт":57700,"ру":105041,"рх":14747,"рц":4365,"тн":65472,"тм":3983,"тл":9354,"тк":17989,"тс":84335,"тр":155968,"то":273813,"те":235202,"тд":4232,"тв":124455,"сё":6893,"ти":199676,"сы":9941,"сь":18264,"та":266195,"ся":89398,"тб":4549,"су":38119,"сф":3977,"сс":88379,"ст":540751,"сч":5733,"сш":4240,"сх":8473,"сц":3788,"ур":58652,"уп":39068,"ут":30026,"ус":56106,"ум":19318,"ул":47107,"ун":41314,"уи":4250,"уз":23986,"ук":30175,"уд":42687,"уг":33231,"уж":19840,"уе":15435,"уа":10851,"тя":20201,"уб":26698,"ув":5212,"ты":37440,"ть":74835,"тч":3850,"тт":6006,"ту":57964,"фу":8016,"фр":15690,"фо":29756,"фи":45878,"фе":25068,"фа":16306,"ую":32657,"ущ":17182,"уч":38493,"уш":8215,"ух":10059,"ёт":8989,"ён":22517,"ёр":12412,"ёл":4881,"ём":5538," II":3728," Ma":3710,"а ":616120,"Р ":10559,"С ":6434,"Ис":5682,"Ин":6739,"к ":120218,"Из":4203,"Ив":3823,"й ":507309,"Ле":11105,"Ли":8362,"Ла":6540,"Ку":8515,"Ко":21705,"м ":237652,"Кр":10087,"Ки":8491,"Ка":27548,"л ":64722,"На":20616,"Не":6593,"Ни":9075,"Му":3926,"Мо":21246,"о ":404148,"Ма":25177,"Ми":12922,"Ме":9781,"Ло":5700,"н ":135483,"Лу":3901,"Па":15703,"Пе":17521,"Пи":4313,"Пл":3876,"с ":91974,"По":29660,"р ":85184,"Ос":6849,"Ор":3751,"От":4582,"Об":5237,"Он":3901,"Ол":8342,"Но":9337,"п ":8470,"в ":366455,"Ам":4416,"Ан":11755,"Ал":14756,"Ав":5862,"Ба":12391,"Ар":8812,"б ":7229,"д ":64292,"Во":11807,"Ве":14286,"Ви":9847,"Га":7333,"Бо":12192,"г ":35799,"Бр":14227,"Бе":12321,"Би":3781,"Ва":10241,"Бу":4848,"Ди":3742,"Дж":8980,"Де":6705,"ж ":6087,"До":7659," Th":4595,"Ев":5669,"Ге":11433,"Гр":11587,"Го":10198,"е ":450788,"Да":6449,"и ":574901,"За":11852,"з ":53551,"ь ":133456,"е́":15696,"ы ":130752,"и́":16589,"ё ":6265,"Яв":5487,"ю ":63842,"я ":490850,"Эт":4232,"Ст":11217,"Су":5933,"Та":9444,"Ти":3758,"Те":7001,"ф ":6051,"То":6501,"Тр":6028,"Ту":4280,"Ук":4529,"х ":211491,"Пр":23361,"Ра":11292,"Ре":10364,"Ри":7652,"СР":8553,"т ":206463,"Ро":21953,"Ру":5315,"СС":13685,"СШ":6418,"Са":18244,"Св":4560,"Си":8236,"Се":17477,"у ":114144,"Со":23127,"Це":5862,"ш ":6132,"а́":24040,"Че":6673,"ША":6418,"ц ":11865,"Фр":9086,"Фе":6645,"Фи":4961,"Ха":6486,"ч ":21859,"Хо":3899,"ль":255358,"лы":10808,"лю":22731,"мб":8994,"ля":104684,"ма":130982,"лё":9664,"ме":172608,"ми":140084,"мл":6471,"лл":23806,"лн":13959,"ло":163517,"лс":10825,"лт":4507,"лу":35325,"ла":158459,"лж":4211,"ле":205849,"лд":3683,"лг":5390,"лк":10879,"ли":255185,"км":7931,"кн":9321,"кк":4284,"кл":31050,"кр":60959,"кс":31731,"ко":383857,"кт":66101,"ку":41628,"кц":9152,"ка":219076,"кж":14683,"ки":210538,"кв":18587,"ке":35320,"йн":21970,"йо":19765,"йр":3918,"йк":4778,"йл":4612,"йш":7189,"йс":69884,"йт":6056,"ия":163126,"ию":29435,"ищ":4465,"иш":6262,"у́":5838,"йд":4105,"ио":61311,"ип":31810,"им":121890,"ин":226639,"ик":130372,"ил":122490,"ии":133366,"ий":180537,"иц":62824,"ич":108247,"иф":12791,"их":84356,"ит":178471,"иу":4714,"ир":73443,"ис":176435,"ри":235653,"рк":23089,"рл":11046,"рм":37955,"рн":63599,"ро":363111,"рп":4622,"ра":429505,"рб":11729,"рв":24770,"рг":34028,"рд":18157,"ре":279331,"рж":10477,"пь":4901,"пы":11436,"пр":223152,"пп":19192,"пт":6272,"пс":5990,"пу":32229,"пи":56651,"пн":7871,"по":247631,"пл":31561,"ою":7698,"оя":26385,"па":81900,"пе":95232,"ощ":8932,"ош":11972,"оч":34323,"оц":13939,"оэ":5861,"ос":271102,"ор":307758,"оп":60188,"оо":12934,"ох":15431,"оф":19467,"оу":5082,"от":162519,"ок":93705,"ол":228863,"ом":207667,"он":199509,"ож":50393,"оз":54318,"ои":31770,"ой":205541,"ов":359690,"нё":5120,"ог":196172,"од":289964,"ое":90767,"ня":35079,"оа":5488,"об":119246,"нь":18690,"ны":243593,"нц":33951,"нч":4962,"нт":90634,"нс":97833,"нф":7203,"ну":26904,"но":402371,"нн":163382,"нр":4657,"нк":27682,"нз":4096,"ни":399245,"не":163042,"нг":32092,"нд":56240,"мё":3976,"нв":7556,"на":409072,"мя":17291,"мь":4683,"мы":32208,"му":54274,"мс":10476,"мп":42062,"мо":86128,"мн":19897,"мм":15488,"ге":39235,"гд":7818,"ги":64206,"гн":5587,"го":307537,"гл":27873,"гм":4198,"гр":76619,"гу":22493,"дв":21897,"да":163117,"вг":7437,"ве":186015,"ви":112054,"вк":13130,"вл":60895,"вм":3919,"вн":71300,"во":212143,"вп":4179,"вр":34804,"вс":46803,"ву":21309,"вт":16905,"вх":16990,"вш":21902,"вы":73241,"вь":4164,"га":61976,"вя":19117,"вё":4031,"би":27123,"бе":43659,"бр":63607,"бн":9059,"бо":89921,"бл":44500,"бу":21528,"бс":7153,"ва":210954,"бы":35755,"бъ":8402,"бщ":13191,"ад":83035,"ае":38603,"аж":22477,"аз":106308,"аб":41193,"ав":144996,"аг":30571,"ам":106564,"ан":372615,"ап":45088,"аи":20853,"ай":49922,"ак":101386,"ал":220430,"ах":49626,"аф":15259,"ач":35451,"ац":44113,"ас":161556,"ар":181597,"ау":19997,"ат":182329,"ба":30240,"ая":142316,"аю":29293,"ащ":9761,"аш":12824,"зс":6594,"зр":12483,"зу":17726,"зк":4107,"зи":46141,"зо":48202,"зн":42107,"зм":17172,"зл":7624,"ив":68972,"иг":39987,"зя":3693,"иа":42640,"иб":20393,"иж":14048,"из":124756,"ид":43626,"ие":128645,"зы":33503,"жо":5749,"жс":4009,"жу":6473,"жи":33668,"жн":29430,"за":99672,"зв":49033,"жё":3790,"зг":3804,"зд":32492,"зе":22277,"еф":7165,"еу":4109,"ет":190550,"ес":211743,"ер":322951,"еп":23032,"ео":22288,"ен":392846,"ем":131477,"ел":205217,"ек":97566,"ей":99541,"еи":5939,"ез":47927,"еж":36786,"ее":34825,"жд":33436,"же":89890,"жа":21003,"ея":12292,"ещ":8165,"еч":24475,"еш":11291,"ех":17181,"ец":24531,"дс":40426,"др":39150,"ду":60618,"дн":71944,"дм":11329,"дп":4848,"до":102671,"ди":135662,"дл":31466,"дк":7688,"де":155382,"дд":3909,"о́":17497,"дж":10265,"еб":21454,"ев":101298,"дё":3920,"ег":62011,"ед":133360,"дь":6956,"еа":12623,"дя":7893,"дш":3897,"ды":16920," Яв":5485," ар":13568," ба":10710," ав":18221," ад":9137," ал":10374," ак":9085," ан":12535," ам":9443," ап":7635," бу":7250," ва":6199," бы":23597," би":6156," бе":13045," бр":8596," бо":19871," бл":6242," вт":4810," вх":16859," га":5609," вы":32582," ви":13232," ве":27020," во":59783," вс":13316," вр":12829," вл":4862," вк":4169," вн":5530," дв":15611," да":12181," гу":4609," го":108387," гл":8578," гр":29774," ге":11208," ги":5523," ед":6587," ег":8307," до":35767," др":17969," ду":6602," де":43890," ди":14485," дл":25835," же":14087," за":57830," зв":5213," жи":10062," зн":7274," иг":13915," из":63621," ил":26819," ин":23235," им":21022," ис":39365," их":4602," ию":10395," ка":46349," ки":10920," кр":19920," ко":101579," кн":6962," км":7101," кл":11050," ку":8839," ла":5400," ли":22008," ле":16122," ло":3818," ме":49176," ми":21788," лю":4902," ма":35222," мо":29145," мн":7010," му":26862," ни":8598," не":49337," на":155501," но":20718," ок":24600," он":5812," од":28184," об":63971," от":54560," ор":15811," ос":31704," оп":15294," по":164771," пл":14871," пи":11060," пе":48196," па":21414," Ре":10285," Ра":11205," Ро":21862," Ри":7649," Пр":23192," Пе":17464," Па":15586," с ":51024," По":29407," Пл":3863," Пи":4292," От":4556," Ос":6825," Ор":3729," Те":6933," Ти":3719," То":6441," Тр":5998," Ст":11107," Су":5919," Та":9379," Св":4548," Си":8202," Се":17430," у ":4211," Со":23029," Ру":5277," СС":6857," СШ":6404," Са":18190," Фр":9072," Фи":4937," Фе":6613," Ук":4507," Ту":4270," Це":5856," Хо":3885," Ха":6470," Че":6653," Эт":4213," Ба":12348," Ар":8769," в ":244550," Ан":11697," Ам":4402," Ал":14708," Ав":5843," Ва":10204," Бу":4830," Бо":12018," г ":7009," Бр":14207," Бе":12299," Би":3719," а ":10629," Ев":5656," Ди":3719," Дж":8956," Де":6645," До":7584," Га":7281," Ве":14245," Ви":9808," Во":11759," Да":6384," Ге":11389," Го":10155," Гр":11539," Ис":5607," Ин":6668," к ":15541," Ки":8443," Ка":27418," и ":157390," За":11752," й ":4397," Ив":3814," Из":4180," Му":3903," Мо":21203," о ":7496," На":20476," Не":6542," Ни":9034," Но":9287," Об":5188," Он":3896," Ол":8330," Ко":21565," Кр":10028," Ку":8476," Ла":6504," Ле":11080," Ли":8319," Ло":5687," Лу":3887," Ма":25091," Ме":9720," Ми":12850," В ":17470,"II ":7232," ра":87254," ре":41392," ро":33130," пр":177960," пс":3748," пу":8711," св":25177," си":22645," се":47025," сл":16395," см":6218," ск":7720," сп":19254," ср":11080," сн":4707," со":89660," ру":13053," са":14863," ти":6431," те":38937," то":20252," тр":22707," ст":53938," су":12938," та":27006," ук":4204," ус":11044," уп":6557," ур":3864," ун":4890," ул":9872," ту":5700," фо":12379," фр":10534," фу":6744," фе":9085," фи":20989," фа":9016," уч":17593," хр":4873," хо":7233," ху":6092," ха":4633," ци":3784," це":18264," чт":6942," чи":6042," че":25547," ча":26885," ша":3808," шт":13545," эк":9725," эл":8825," эт":18390," юг":4276," ян":5667," яз":10931," яв":12558,"аз ":5770,"ад ":7406,"Явл":5341,"ав ":5373,"ам ":14841,"ан ":31178,"ак ":26116,"ал ":21311,"ай ":4239,"авш":9597,"авт":9645,"ага":8115,"аго":7572,"ада":10767,"ади":12630,"аде":10895,"адм":7561,"адн":6641,"адь":3777,"аем":7453,"ает":25830,"аже":6271,"ажд":5556,"аба":3999,"або":17739,"абр":6755,"ава":11200,"авн":21035,"авл":26690,"авс":4043,"аво":12671,"аве":7803,"авг":5378,"ави":21691,"ало":13331,"алл":5996,"ала":22307,"али":48595,"але":13975,"амм":7674,"амо":8686,"амы":4469,"аля":5952,"ама":7226,"аль":73407,"ами":28699,"аме":19983,"анн":37419,"ано":18426,"анс":39188,"ант":20680,"анц":15483,"аны":7448,"ань":4229,"ана":29768,"анд":25091,"анг":9722,"ани":99562,"ане":11955,"анк":9278,"азр":7025,"азо":13307,"азн":9789,"азл":4533,"ази":10605,"азе":3818,"азд":5347,"азв":17619,"аза":10657,"аин":6236,"аиб":4114,"азы":8101,"айс":4889,"айо":18909,"айн":4793,"акт":16378,"ако":12308,"акж":14674,"аки":5796,"ака":8072,"ах ":31165,"ас ":6451,"ар ":4822,"ат ":18392,"ая ":136821,"ба ":4571,"Пол":8430,"Пор":6259,"При":5525,"Пре":6670,"Про":7157,"Рос":14485,"Рас":4325,"Рес":3751,"Сан":8587,"ССС":5587,"ССР":7399,"США":6399,"Сос":6600,"Сов":3947,"Сев":6002,"Сер":4458,"Ста":4121,"Укр":4080,"Фед":3907,"Фра":5981,"ША ":6384,"а́н":5689,"лам":6144,"лан":15129,"лас":29897,"лат":7877,"ме ":7744,"ля ":50911,"ма ":24448,"лав":17972,"лаг":7649,"лад":11258,"лы ":5725,"ль ":38018,"кую":3788,"кус":4925,"кул":8752,"ктя":6372,"кци":8948,"кре":6974,"кра":17934,"кри":5557,"кру":16231,"кро":7021,"лу ":5744,"кры":6225,"кса":6759,"кси":5075,"кта":5223,"кте":4216,"кти":10807,"кто":14126,"ктр":8094,"кту":4971,"кла":9863,"кло":3891,"ло ":19382,"клю":8248,"кни":4141,"ког":53446,"ков":40843,"ком":58598,"кон":29711,"коп":5499,"кор":15168,"кос":5742,"кот":33255,"кое":17818,"кой":65146,"кол":25577,"кие":12731,"кин":10160,"ким":14411,"кий":81796,"ких":35222,"ле ":19103,"кже":14673,"ли ":56435,"ква":7601,"ках":4702,"кат":7467,"кар":8017,"кам":8824,"кан":23037,"как":20319,"кал":11552,"каз":10972,"кая":36891,"каб":6283,"кад":5211,"ла ":40418,"йши":4432,"йся":5983,"йск":44468,"йст":14738,"кт ":8556,"ку ":9345,"йны":8079,"кс ":4079,"йон":18007,"ко ":19496,"км ":4431,"ки ":43486,"ке ":21167,"иям":6120,"иях":4963,"нва":6008,"од ":33146,"ная":57610,"нах":9504,"нац":5517,"нау":8200,"нач":22853,"ог ":6440,"нан":6213,"нам":8570,"нал":25338,"нат":9581,"нас":17157,"нар":16443,"нап":11298,"над":9721,"нак":5181,"наи":4066,"наз":20814,"нде":5878,"нда":8693,"нгл":9288,"нге":3994,"нга":3873,"ое ":58821,"ней":12822,"нек":4897,"нем":9855,"нен":19222,"нер":13236,"нес":9300,"нет":5791,"нег":6018,"нев":4773,"нее":5864,"нди":11172,"ндо":5836,"ндр":8570,"нив":5586,"нии":26244,"низ":11864,"ник":36850,"ний":18207,"ниг":4932,"ние":74066,"ок ":21671,"ой ":187929,"ны ":35382,"нь ":8902,"ня ":11152,"мый":5262,"мых":5149,"мыш":3735,"ов ":92702,"нт ":11498,"мпо":5172,"мпи":13008,"мпе":6698,"мпа":7653,"мот":4724,"ну ":6285,"мпь":3765,"мск":6474,"мун":14544,"мул":3800,"муз":9742,"мик":7960,"мил":4443,"мии":4851,"мич":7046,"мин":28165,"мир":17908,"мит":4140,"но ":62002,"мму":3810,"мно":11450,"мод":4772,"мог":5035,"мов":6153,"мой":4768,"мож":6250,"мон":9329,"мом":4520,"мол":6893,"мос":8228,"мор":8527,"нд ":4096,"мац":4061,"ляю":8990,"мая":9013,"лял":3785,"мал":9300,"ляе":22475,"мат":21959,"мас":5866,"ляр":4924,"мар":12011,"ман":23543,"люч":8195,"мец":4888,"мес":13130,"мет":17123,"мен":50820,"ни ":19557,"мер":29417,"мез":5962,"меж":12542,"мед":8168,"не ":38359,"лён":4990,"мы ":9798,"льн":93765,"на ":153332,"мя ":11127,"льм":14104,"льк":7925,"льз":12358,"льд":3831,"льб":7821,"лья":5362,"льш":14923,"льт":14050,"льс":23881,"лощ":4920,"му ":16323,"лок":5500,"лог":21990,"лод":5038,"лож":18221,"лор":4516,"лос":11300,"лот":8157,"лом":9527,"лон":6111,"лов":35245,"луч":9778,"луж":6113,"лся":9183,"лиг":4087,"лив":6013,"лиз":10430,"лии":15024,"лим":9347,"лий":6287,"лик":17837,"лез":6815,"лей":13214,"лев":9323,"лег":4328,"лед":13548,"лее":8350,"лес":4757,"лер":5420,"ми ":51806,"лен":60261,"лем":10309,"лек":22756,"лет":13846,"лли":5515,"лла":3879,"лле":7201,"лиц":18482,"лич":17052,"лис":14036,"лит":29052,"лин":18545,"лия":7551,"лко":3705,"пас":4125,"оящ":5066,"ояб":5483,"пад":12246,"пал":13951,"рг ":4862,"оян":3986,"пан":12047,"пар":16176,"ре ":17661,"ра ":44965,"оюз":3745,"пий":7004,"пио":5573,"пис":24111,"пла":9596,"пле":8010,"ро ":9844,"пло":9025,"пед":4071,"ри ":19004,"пер":55359,"пес":4491,"печ":4195,"пец":6528,"ори":37323,"орд":5846,"оре":20523,"орг":13852,"орс":6547,"оро":76961,"орм":17625,"орн":11312,"опу":5280,"ора":20394,"опе":10688,"опи":6454,"опо":11431,"опр":11046,"оор":3720,"опа":4525,"отд":3834,"оте":8329,"отк":7498,"отл":4401,"оти":8154,"ото":45196,"отн":11373,"отр":8447,"осу":15201,"отв":5315,"ота":11881,"осе":5320,"оси":7543,"оск":15979,"осл":18494,"осн":17062,"осо":11923,"осп":4485,"осс":25551,"ост":122012,"ору":9047,"орт":19622,"оры":19028,"оря":6038,"осв":3854,"омм":4672,"оми":12071,"оме":14471,"ома":19204,"оля":5909,"оль":52719,"олу":10916,"олн":13122,"по ":36706,"оло":54184,"олл":5951,"олж":3930,"оле":21412,"оли":27404,"ола":9666,"окр":16259,"окт":7805,"оку":5419,"око":16003,"ооб":4126,"онс":12501,"онт":8231,"онц":5168,"они":16817,"онк":3985,"оно":19353,"онн":16949,"она":44812,"онд":5190,"оне":18296,"омо":11288,"омп":18935,"ому":11330,"оше":5000,"пы ":9126,"очи":4013,"оче":6235,"очн":15161,"оща":4802,"офе":5906,"офи":7745,"оты":3797,"оце":7042,"оци":5856,"охо":7896,"нят":6570,"няе":3761,"ова":79059,"обы":7576,"общ":12061,"объ":7061,"обр":22953,"обо":16065,"обн":4315,"обл":17532,"оби":5216,"обе":11035,"ных":63297,"ные":28042,"ным":27407,"ный":86741,"па ":9132,"оит":3855,"оис":6658,"оим":4265,"ойс":6061,"ойн":7471,"оке":4198,"ока":10728,"ожи":4007,"ожн":11252,"озд":12188,"озв":5021,"ози":6006,"озн":9808,"оиз":10355,"одн":39144,"оди":52480,"оду":22578,"одр":3912,"одс":13394,"одо":20011,"оды":5756,"оед":4040,"оев":9413,"одя":6688,"оен":8911,"оек":3819,"оже":24113,"ожд":8010,"ове":40857,"овк":4823,"овл":5356,"ови":24912,"ово":42247,"овн":14676,"овр":5254,"овс":17603,"овы":17385,"ога":5667,"овя":4334,"огд":4881,"оги":20707,"огл":3937,"ого":132906,"огр":15611,"ода":58550,"оде":17121,"от ":31995,"ноя":6122,"нос":44637,"нор":3684,"нол":4077,"ном":37955,"ной":71982,"ное":29866,"ног":57728,"нод":4260,"нов":54046,"нны":69394,"нно":59505,"ор ":24212,"нни":6472,"нна":20166,"нко":5353,"он ":29089,"нкт":5008,"нка":4499,"ом ":104456,"ния":84752,"нию":8268,"нир":5623,"нис":17636,"нит":8771,"ним":17208,"нин":5963,"нич":11022,"них":9088,"ниц":23421,"нце":5996,"нци":17698,"нцу":5080,"ную":9327,"нфо":3773,"нтя":5255,"нут":4116,"нта":17028,"нте":10866,"нти":10987,"нто":10091,"нтр":15601,"нск":63751,"нст":16529,"сам":12116,"сан":12409,"ряд":7767,"сат":6375,"сво":14816,"те ":15248,"све":4519,"свя":9460,"сев":8467,"сел":21166,"ти ":46330,"сен":10794,"сем":7371,"сет":3970,"сер":16452,"сис":11359,"сит":9045,"сий":14008,"сии":9496,"син":6230,"сил":7978,"сим":6904,"скв":8290,"ски":131938,"ска":46869,"сли":3844,"сле":23336,"сла":10537,"ску":8391,"ско":157451,"сме":6005,"слу":9361,"сло":15249,"то ":29238,"сня":3968,"соб":15821,"сов":27452,"соз":9281,"сок":8059,"сом":3851,"сно":26195,"тр ":8572,"сны":3885,"спе":13580,"спа":4761,"спи":4683,"сос":21656,"сор":6482,"соо":5100,"сон":7291,"соц":4226,"сре":10218,"ту ":8116,"спу":6484,"спо":39170,"спр":6952,"рри":9506,"роц":5562,"рош":3692,"рот":12263,"роф":6754,"рох":5095,"роп":12392,"рос":28947,"ст ":16255,"рта":13460,"рст":15956,"рто":4691,"рти":13560,"рск":18805,"рсо":4243,"рси":7173,"рую":4208,"рту":7614,"рук":7417,"руг":20792,"руд":5213,"руж":6808,"руп":20038,"рус":13228,"рхи":6150,"сь ":14955,"ся ":83299,"та ":65491,"рыт":4303,"рых":5096,"рый":5524,"рые":4885,"тв ":5853,"рад":12120,"раж":10828,"раз":53144,"раб":18768,"рав":32412,"рам":17118,"ран":56502,"раи":8567,"рай":21830,"рак":10369,"рал":23925,"рах":8851,"раф":10748,"рац":10644,"рас":32059,"рат":31627,"рая":5022,"ращ":3710,"рбу":3964,"рва":4083,"пью":3789,"реб":6936,"рев":19639,"рег":20011,"ред":53953,"реа":5135,"рет":10005,"рес":14958,"реп":5228,"си ":5105,"рен":15519,"рем":25502,"рел":11664,"рек":12317,"рей":7082,"рез":11794,"реж":11533,"ржа":3754,"реч":6532,"рво":6073,"се ":6838,"рвы":9033,"рга":14493,"рге":4800,"рги":4021,"рия":13567,"рию":4048,"рио":6627,"рим":11681,"рин":20707,"рик":20481,"рил":3781,"рии":19852,"рий":6871,"рич":8657,"рит":22893,"рир":3686,"рис":16618,"рка":4003,"риа":11930,"риг":4581,"рив":4645,"рид":5404,"риз":6655,"ск ":4696,"рни":9740,"рна":12681,"рок":11103,"рол":12179,"ром":23332,"рон":16716,"рож":7867,"роз":4092,"рои":18841,"рой":14224,"ров":68815,"рог":16486,"род":57888,"рое":10921,"рны":14512,"рно":18416,"рла":4760,"рко":4371,"рми":8750,"рма":15602,"со ":3903,"ппы":6617,"пра":26585,"при":52484,"пре":53035,"ру ":6954,"про":87029,"поп":4906,"пор":17269,"пос":28755,"пот":5403,"поэ":3864,"ппа":6927,"рт ":5103,"The":3957,"под":23464,"пов":12924,"пон":8324,"пом":6183,"пол":68859,"пок":4701,"поз":9391,"пуб":8028,"пус":5784,"пут":5691,"пул":3714,"ры ":15775,"са ":16206,"ря ":29812,"рь ":4987,"вар":15705,"ват":16606,"вая":7821,"ваю":8055,"ге ":6711,"вае":10913,"вав":4587,"ван":65485,"вал":18282,"быт":4123,"быч":3846,"был":18105,"га ":14808,"бъе":7043,"бще":10673,"вы ":7062,"бур":7473,"бря":21845,"вто":16228,"все":9508,"вск":27242,"вст":7859,"вра":8893,"вре":17466,"вро":6581,"вол":13485,"вок":4441,"вой":25086,"вои":3759,"воз":12251,"вое":18754,"вод":28316,"вог":6893,"вов":10300,"вны":14251,"вор":11524,"вос":15224,"воп":3740,"вом":8183,"вни":4616,"вне":8106,"вна":13669,"вно":25418,"вля":30877,"вле":21719,"вла":3874,"го ":153532,"вкл":4066,"вка":4191,"вич":20022,"вия":4450,"виж":4984,"вил":7784,"вин":10044,"вис":5933,"вит":12981,"вид":14202,"вес":15630,"вет":22334,"вер":37523,"вен":36297,"ги ":6381,"вел":6076,"век":17069,"вед":16797,"вгу":5781,"ва ":46587,"ающ":19084,"ают":9616,"ачи":5855,"бы ":3828,"ащи":4099,"аще":3705,"ауч":3787,"аук":4642,"афи":6466,"ахо":7975,"ача":13504,"аче":11447,"аци":41912,"апр":13134,"апа":11764,"апо":3746,"апи":8487,"арх":7888,"арс":18625,"арт":22698,"арь":4529,"аря":9072,"аре":7851,"ард":6979,"ара":21925,"арн":5879,"арм":4715,"аро":18219,"ари":19835,"арл":4156,"арк":7904,"асс":17733,"аст":73387,"ась":5439,"ата":17783,"аси":4710,"асе":9811,"асл":4318,"асп":18234,"асн":7236,"ату":10612,"аты":6297,"ать":13519,"ате":41110,"ати":28209,"атн":5152,"ато":21641,"атр":7249,"бол":22933,"бом":6832,"бой":5049,"бор":14402,"бот":13748,"бно":3859,"бро":3695,"бри":6979,"бре":4187,"бра":24036,"бла":17775,"бли":15168,"бле":4490,"во ":31067,"ви ":4378,"бес":5023,"бер":15432,"бел":4899,"бит":4733,"бил":3862,"ве ":17457,"даю":4004,"дах":4951,"дан":21423,"дам":3898,"дар":18766,"дат":8984,"дви":7106,"дал":8909,"дав":5333,"ев ":8761,"дек":7455,"дей":9763,"дем":5618,"дел":26493,"ден":32485,"дер":21083,"ей ":52679,"дес":4085,"дет":5015,"дея":8136,"дву":4656,"дво":4445,"ее ":25157,"ез ":4626,"ды ":12967,"дь ":4234,"ех ":3705,"дст":24032,"дск":13686,"дро":6400,"дру":10418,"дре":7577,"дра":7639,"ет ":49624,"дун":5024,"ец ":6315,"ен ":23878,"дия":4777,"диц":4594,"ем ":32646,"диа":3739,"див":3812,"дим":5230,"дин":31699,"ел ":6867,"дио":3858,"дис":4374,"дит":31866,"дии":5066,"дил":7726,"ек ":8976,"дны":11957,"дож":6200,"дов":21587,"дос":6949,"дор":10512,"дол":8011,"док":5562,"дон":5574,"дом":8881,"дна":11456,"дни":9495,"дне":7494,"ер ":23656,"дно":27634,"для":23991,"дми":9130,"вып":7258,"вый":9785,"вым":6060,"вых":10632,"выс":8860,"al ":4210,"да ":70397,"вяз":5698,"гал":7699,"вят":8003,"ган":16467,"гар":4062,"де ":21659,"гда":5300,"вую":5927,"вхо":16916,"вше":5430,"вша":3781,"вши":12432,"вые":6905,"гон":3740,"гол":6702,"гос":13811,"гот":3744,"гор":32983,"гов":9981,"год":69990,"гру":17285,"ду ":34904,"гро":3890,"гра":40520,"гре":6189,"гус":6354,"ген":9872,"гер":7302,"ди ":8816,"гии":7384,"гио":11815,"гич":7565,"гих":4581,"гла":12247,"до ":13248,"гли":6452,"жан":5670,"еят":7722,"за ":17832,"еще":5057,"жит":5272,"жив":7938,"жис":4312,"жес":7568,"жет":3940,"жду":12980,"жел":6716,"жен":38841,"жде":11274,"жда":4735,"жск":3945,"жно":12596,"жни":4413,"жны":6194,"жур":3704,"ежи":6716,"ежд":16924,"еду":5404,"едс":18751,"еза":4036,"езн":5938,"езо":10562,"езд":5637,"ези":9501,"ева":10267,"еви":11883,"еве":16643,"еат":3835,"дящ":4359,"его":29841,"еда":12960,"еде":22810,"еди":27267,"едо":10112,"едн":12312,"евн":10381,"же ":22970,"ево":16350,"евр":9678,"евс":6183,"евы":4574,"еге":5203,"еги":14109,"ент":47594,"енс":9645,"енц":4019,"ени":132856,"ено":11907,"енн":84711,"ена":18893,"емя":7341,"ене":16352,"енд":7186,"емь":4324,"емы":10998,"еор":5073,"ены":6940,"ень":8052,"епо":3992,"ерх":5155,"ерр":10216,"ерс":17748,"ерт":9897,"ерл":3718,"ерм":10872,"ерн":25326,"еро":19985,"ери":40656,"ерк":7071,"ерг":7482,"ерж":8654,"ере":44413,"ера":37126,"ерв":20819,"ерб":7899,"ейн":5541,"ейс":18646,"еки":4636,"еко":11088,"ект":27081,"екс":15558,"ейш":5926,"ека":19196,"ели":21048,"ело":19740,"еле":41611,"ела":9361,"емл":3751,"емо":8767,"еми":9433,"ему":3888,"емп":6664,"ель":74971,"еме":23861,"еля":16614,"ема":12283,"елё":3840,"ехн":5852,"ецк":9009,"еци":7333,"еча":3853,"ечн":3687,"ече":11632,"еше":3757,"есе":5474,"еск":73489,"есн":5953,"есп":9644,"есс":15061,"ест":79162,"еся":5372,"ета":21699,"ети":10490,"ете":10421,"етр":11442,"ето":9221,"етн":10174,"етс":48640,"еты":4087,"ибо":7839,"иве":7414,"иви":3809,"ива":16306,"иал":20951,"иан":8939,"иже":6481,"идо":4070,"иев":3803,"ием":14066,"ией":9608,"игр":16265,"иго":4097,"ида":4859,"иде":14015,"иво":7871,"ивн":17940,"ивш":6456,"ига":5856,"иги":5053,"икл":4956,"икр":3804,"ико":26837,"ике":6203,"ики":16651,"ика":34115,"ийс":37107,"изм":10707,"изо":8828,"изн":6897,"изи":10371,"изд":4964,"иза":12843,"изв":18761,"ион":41500,"инц":6456,"ины":10703,"иня":5979,"иод":4236,"ине":14798,"ини":31286,"инн":3826,"ино":20897,"инс":24611,"инт":7204,"инф":4063,"ина":38691,"инд":6333,"инг":12127,"ими":15581,"име":21870,"имс":3708,"имо":9905,"имп":12615,"има":14708,"иль":23825,"или":46988,"иле":5170,"илс":4907,"илл":5632,"ило":11583,"ила":9561,"иси":5473,"иса":15121,"исх":5676,"ист":74674,"исс":10637,"исп":16317,"исо":5087,"исл":10745,"иск":11427,"ити":13816,"ите":56212,"ита":24816,"ись":6230,"иту":6461,"ито":18876,"итс":8689,"ипа":14414,"ипе":3791,"ира":11140,"ире":4647,"иру":8486,"ири":4818,"иро":27497,"иха":3800,"ихо":4447,"ице":7291,"ица":16979,"ици":25302,"ицы":6049,"ить":5325,"ифи":3877,"ичи":6000,"ичн":11017,"ича":4726,"иче":66707,"июн":5092,"июл":5316,"ка ":55202,"ив ":4402,"зав":10564,"ид ":4294,"зви":4153,"зве":16359,"зва":15033,"зац":9007,"зат":4555,"зап":11760,"зан":11015,"зам":3927,"зак":6826,"зде":5994,"зда":15915,"ие ":94285,"зво":7967,"ий ":139271,"зер":5619,"ии ":131775,"зем":4097,"из ":38626,"зид":4005,"зил":7018,"ил ":8940,"ик ":27882,"ин ":25833,"им ":24359,"зия":4934,"зит":4573,"зме":6834,"зли":4448,"зна":20529,"зни":4774,"зно":8729,"ир ":5392,"зны":4336,"зов":19642,"ис ":4567,"зон":5974,"зор":7083,"ит ":29545,"зра":8237,"зск":6535,"их ":71178,"зуе":5324,"ич ":17567,"зыв":8397,"ию ":18909,"зык":19420,"ия ":146910,"ьшо":3872,"ьши":3819,"ьше":5810,"ьян":5268,"ьют":3946,"ьма":3914,"ьна":7658,"ьни":4302,"ьно":44690,"ьны":33869,"ько":6149,"ion":5234,"ьзу":6190,"ьзо":5415,"ьту":4384,"ьст":6382,"ьск":16866,"ям ":5239,"ют ":10384,"эко":4504,"эле":8958,"это":14311,"ых ":86372,"he ":5907,"ыва":15848,"ье ":4950,"ые ":43888,"ыл ":8892,"ым ":27422,"ый ":115051,"ычн":4405,"ья ":5879,"ью ":17235,"ьбо":5724,"ьев":4427,"ьер":4030,"ьм ":7924,"ыка":9496,"ыла":5382,"ыми":11026,"ыпу":5527,"ысо":4693,"ённ":14954,"er ":6599,"es ":4533,"яза":3917,"яет":23887,"язы":12445,"явл":14387,"ябр":17050,"ях ":9267,"ютс":8148,"юте":4109,"юща":7138,"ющи":22555,"юще":8725,"юча":3971,"юля":5121,"ят ":7009,"юня":4749,"яют":4520,"яющ":7517,"яще":6980,"ящи":4896,"янс":6220,"янв":5611,"яни":5071,"ями":7921,"ярн":3999,"ято":4282,"яти":7069,"яте":9306,"уще":14055,"уча":15033,"учн":4074,"учи":4977,"уче":7765,"фес":5738,"фев":5319,"фер":4649,"ующ":10346,"фин":4428,"физ":4024,"фил":15705,"фик":3811,"фиц":4922,"фре":4266,"фра":6501,"фор":17745,"фон":4486,"ца ":19779,"це ":5162,"хан":4526,"хар":5112,"хра":4915,"хно":4200,"хни":3959,"хож":3747,"хов":4677,"ход":41602,"цы ":7957,"худ":5932,"сск":14412,"сси":32096,"ссо":7973,"сса":7005,"ссе":5970,"стн":20518,"сто":67851,"стр":70127,"ств":108223,"сте":31686,"сти":71515,"ста":85694,"сст":5885,"суд":15247,"сть":44809,"сты":4236,"сту":9860,"сущ":7215,"схо":7227,"ты ":16675,"ть ":55254,"тав":37473,"так":23239,"тал":22250,"там":8524,"тан":36504,"тай":3709,"тат":27244,"уг ":6960,"тар":13509,"тбо":4051,"тву":7862,"сёр":3801,"тво":31741,"тви":10153,"тве":41916,"тва":21790,"тех":6332,"тем":17566,"тел":76816,"тео":3852,"тен":11539,"тер":49417,"теп":3783,"тет":16644,"тек":8246,"тей":6713,"тив":26421,"тие":8972,"ук ":4128,"тка":5169,"тич":24503,"тия":7560,"тии":4630,"тий":5054,"тин":12258,"тик":14625,"тил":7099,"тир":5823,"тис":5149,"тип":5281,"тит":7029,"тки":3943,"тно":26382,"ток":11951,"тол":12417,"той":7374,"тны":17802,"тов":29921,"тог":9531,"тни":13284,"тна":5384,"ут ":4381,"тре":17053,"тра":54933,"три":18286,"тор":95430,"том":20153,"тон":9780,"ус ":4688,"точ":11868,"тоя":12927,"тст":5913,"тся":53866,"тро":41481,"тру":10653,"тск":19176,"туг":4888,"туп":6027,"тур":21906,"тью":9623,"тый":5760,"ую ":18777,"уго":4888,"уги":7037,"уга":8925,"уда":16459,"тяб":11568,"убл":8850,"убе":4682,"узы":8785,"узс":4917,"уже":5996,"ует":9543,"уем":3815,"уди":4265,"удо":10125,"уме":6894,"уль":15447,"уля":5536,"ули":11772,"укт":4529,"уко":4826,"ука":4047,"упп":14974,"упн":6779,"упр":5417,"ура":6947,"ург":8767,"уре":4620,"унк":5527,"уна":6858,"уни":18074,"уст":16092,"усс":16109,"ута":4019,"уры":4919,"урн":11401,"уро":5529,"уск":4624,"ших":7365,"шир":4576,"шин":5562,"ший":10857,"шен":11201,"шая":6983,"шта":14038,"щих":9037,"щие":5309,"щий":14604,"щин":3869,"щее":7976,"щей":4091,"щег":3871,"щен":12451,"щес":17297,"щая":10075,"щад":4691,"on ":6654,"цен":15360,"чи ":4183,"цел":4906,"цев":3998,"цес":4907,"цер":5569,"циа":13419,"ций":4974,"ции":29185,"цие":3800,"цип":12774,"цио":18812,"ция":20642,"ча ":3694,"цар":3955,"цуз":4865,"цко":4797,"цки":5412,"чем":5556,"чен":25493,"чел":10826,"чес":78218,"чер":6254,"чет":6065,"чле":3904,"чин":7894,"чив":3802,"чис":7524,"чит":8014,"ше ":5808,"чаю":4561,"час":36891,"чат":3981,"чал":9929,"чае":6846,"чны":14169,"чно":19736,"что":8092,"tio":4098,"Кар":6020,"Кра":3888,"Кор":4309,"Кон":5008,"Мар":7555,"Мин":4281,"Мос":11770,"Нас":4395,"Ник":4718,"Нов":5069,"Оли":6672,"Пар":6298,"СР ":8410,"Пет":6304,"Пер":7246,"Але":6898,"Бра":9561,"Бол":4163,"Бел":4463,"Вел":5787,"Гра":3978,"Гер":5952,"Джо":3835,"Евр":4076},"n_words":[36763344,40893832,29165701],"name":"ru","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/sl.json b/contrib/languages-data/sl.json
new file mode 100644
index 0000000..4131d88
--- /dev/null
+++ b/contrib/languages-data/sl.json
@@ -0,0 +1 @@
+{"freq":{"D":7697,"E":4326,"F":7483,"G":7482,"A":12478,"B":11289,"C":8095,"L":10635,"M":14480,"N":8171,"O":6564,"H":9046,"I":7126,"J":7268,"K":11757,"U":2722,"T":7824,"W":1905,"V":10234,"Q":288,"P":16672,"S":26591,"R":9751,"Y":433,"X":356,"Z":6077,"f":21271,"g":83131,"d":174445,"e":575804,"b":82172,"Fed":58,"c":56405,"a":599039,"n":410604,"o":476628,"l":247955,"m":151651,"j":244223,"k":242702,"Fel":92,"h":59332,"i":522865,"w":2524,"v":222867,"Fer":370,"u":129396,"t":241091,"s":268170,"r":314054,"q":489,"p":164089,"z":94676,"y":6418,"x":1284,"²":123,"Î":185,"É":129,"Á":62,"Fil":230,"í":2013,"Fin":151,"ë":81,"ê":309,"é":3055,"Fir":82,"è":584,"ç":174,"ä":365,"â":114,"á":3560,"à":133,"ü":557,"ú":870,"ø":79,"ö":488,"ô":419,"ò":234,"ó":1670,"ē":62,"đ":109,"Đ":87,"ā":111,"ć":1063,"Č":1470,"č":56247,"ŕ":73,"ő":245,"ł":69,"ō":86,"Ž":2008,"ž":30614,"Š":2528,"š":51762,"Fak":387,"Fal":70,"Far":111,"Eri":80,"Est":69,"Eti":62,"Ern":82,"Eur":142,"Eva":78,"Evr":794,"Ein":58,"́":345,"Ele":144,"Eko":86,"μ":124,"ν":240,"Ena":60,"ο":342,"ι":214,"κ":105,"λ":169,"δ":61,"ε":131,"η":95,"α":330,"β":59,"γ":116,"ά":79,"ί":88,"Emi":63,"Eli":70,"ό":99,"σ":119,"ς":264,"ρ":222,"π":98,"φ":61,"υ":78,"τ":148," l":19333,"ь":113," m":28795," n":57198,"я":125," o":41952," h":4815,"ш":89," i":55210," j":67383," k":54205,"ы":63," d":42036,"ф":78,"х":75," e":9710,"ц":79," f":9447," g":14912,"ч":253,"р":642,"с":417," a":19069," b":18884,"т":428," c":4312,"у":369," y":127," x":151," z":31775," u":19039," t":27202," w":136," v":56888," p":90919," s":76633," r":20918,"HK ":93,"И":69,"К":110,"Н":65,"М":132,"П":82,"Б":103,"А":111,"Г":80,"В":98,"Д":60," J":7259," K":11717," H":8994," I":7097," N":8139," O":6489," L":10542," M":14428," B":11242," C":8027,"С":106," A":12432," F":7459," G":7430," D":7664," E":4306,"л":503,"к":526," Z":6067,"й":182," Y":432," X":347,"и":1084,"п":111,"о":912,"н":714,"м":224,"г":177," S":26492," R":9719,"Ger":142,"в":657," Q":287,"б":103," P":16617,"а":1195,"з":72," W":1888,"Geo":282," V":10198,"Gen":320," U":2704,"е":796,"д":264," T":7802," č":5716," Č":1467," Đ":87,"Gla":255,"Gia":71,"HL ":80," Á":62,"Gio":131," É":127," Î":185,"Gir":61,"Giu":113," ž":7506," Ž":2008," Š":2526," š":7903,"ա":58,"Gan":63,"Gal":184,"Gam":76,"Gar":174,"Gab":108,"و":76,"ي":148,"ل":189,"م":127,"ن":117,"Fun":63,"د":71,"ب":101,"ا":258,"ر":124,"Flo":89,"Fla":62," А":111," Б":103," В":98," Г":71,"Fra":1714," Д":60," И":69," К":110,"Fri":169," М":132," Н":65,"A ":1521," П":82,"Fre":167,"For":1812," α":63,"F ":287,"Da":1043,"Cu":190,"Cv":64,"Cy":61,"Cl":275,"Co":1643,"Cr":347,"Ce":1243,"Ch":1233,"Ci":465,"G ":285,"Ed":331,"Ea":63,"Dv":143,"Du":856,"Do":1622,"Dr":1054,"De":1300,"Di":918,"Fe":729,"H ":307,"Fa":834,"Eu":245,"Ev":1000,"Er":320,"Et":115,"Es":220,"En":413,"Em":164,"Ep":63,"Ei":124,"El":403,"Ek":172,"Eg":130,"Ge":887,"Ga":902,"I ":771,"Fu":219,"Fr":2141,"Fo":2047,"Fl":225,"Fi":681,"B ":327," С":104,"II ":321,"C ":846,"Av":1043,"Au":489,"Ar":1294,"At":316,"As":365,"D ":417,"Ba":1844,"Az":412,"Af":338,"Ag":178,"Ah":87,"Ab":563,"Ac":172,"Ad":372,"Am":987,"An":1820,"Ap":398,"Ai":256,"Aj":81,"Ak":264,"Al":2250,"Hit":92,"Bu":797,"Br":2247,"Ca":1535,"E ":345,"Bi":1004,"Hid":63,"Be":2195,"Bo":1910,"Hil":81,"Bl":438,"Bj":272,"Hip":72,"Kv":109,"Ku":509,"Kn":259,"IE ":65,"Kl":445,"Kr":2275,"Ko":3816,"Le":3189,"Lj":1723,"Li":1298,"N ":264,"La":1624,"Lu":757,"Ly":76,"Lo":1512,"Me":2219,"Dž":96,"Mi":1869,"Ml":211,"O ":680,"Ma":6025,"Mc":133,"Mu":580,"Mr":101,"Mo":2460,"Nj":399,"Ni":828,"Já":138,"Ne":1598,"Na":2980,"P ":622,"Hel":136,"Ny":85,"Hei":136,"Nu":95,"No":1554,"Ok":258,"Ol":321,"Om":113,"On":187,"Og":200,"Oh":58,"Oc":80,"Od":1568,"Hen":142,"Her":380,"Ob":922,"Gi":553,"Gl":510,"Gr":1909,"Go":1659,"Gu":367,"Gv":83,"Gy":129,"Cô":91,"J ":297,"Ha":1563,"He":1085,"Hi":543,"Ho":701,"Hr":4064,"Hu":331,"K ":438,"Ib":78,"Id":108,"Ig":180,"Im":513,"In":1624,"Il":230,"Iv":415,"Is":1036,"It":703,"Ir":277,"Ja":1804,"L ":443,"Iz":883,"Ji":68,"Je":1541,"Jo":1290,"Hab":58,"Ju":2001,"Hal":126,"Haj":64,"Ka":2508,"Han":157,"M ":389,"Ham":119,"Har":293,"Ki":622,"Győ":92,"Hau":402,"Ke":544,"Us":272,"Ur":262,"Up":188,"Um":80,"Un":907,"Uk":147,"Ul":103,"W ":109,"Ty":77,"Tu":672,"Tr":1582,"To":1468,"Th":691,"Ti":704,"Te":1011,"Ta":1070,"V ":1619,"Côt":91,"Sw":66,"Sz":701,"Sy":147,"St":3308,"Sv":2528,"Su":652,"Wo":195,"Wi":724,"Wh":64,"Wa":405,"We":272,"Vz":150,"Vo":999,"Vr":488,"Vs":193,"Vu":139,"Vi":1620,"Vl":200,"X ":215,"Va":913,"Ve":3902,"Má":81,"Pt":177,"Pu":321,"Pr":3992,"S ":1529,"Pe":1571,"Pa":2197,"Gui":73,"Lé":59,"Pl":789,"Po":5526,"Pi":1027,"Ph":198,"Os":738,"Ot":263," ا":112,"Op":322,"Or":646,"R ":294,"Oz":78,"Se":2093,"Sc":592,"Si":1170,"Sh":254,"Sn":82,"Sm":234,"Sl":6573,"Sk":484,"Sr":839,"Sp":1374,"So":1617,"Ru":971,"Grč":81,"U ":245,"Sa":2421,"Re":3735,"Rd":107,"Ri":922,"Rh":128,"Ro":1660,"Qu":211,"T ":330,"Ra":1534,"Mü":85,"Gre":332,"Gri":94,"Gra":810,"Grb":113,"b ":3960,"Gru":126,"Gro":222,"a ":211856,"Yo":229,"Gle":73,"Z ":240,"Glo":95,"Gol":160,"Gor":907,"Gos":120,"Za":1749,"Zd":593,"Ze":881,"Zi":330,"Zg":337,"Vé":82,"Zm":60,"Zl":152,"Zo":102,"Zn":141,"Zu":99,"God":61,"Zr":80,"Zv":306,"i ":134220,"gd":128,"ge":5872,"ga":24650,"fj":134,"Inf":332,"fl":264,"ff":331,"fi":4842,"fs":419,"fr":3897,"ač":5802,"fu":678,"ft":369,"fo":2045,"Int":382,"fn":91,"j ":12717,"gy":387,"dá":73,"he":3525,"ha":5343,"gn":1368,"gm":166,"gl":6996,"gi":8855,"gh":503,"gg":192,"gv":88,"gu":2562,"gt":124,"gs":203,"bč":5121,"gr":10356,"cí":78,"go":15666,"dt":312,"du":5066,"dv":5016,"dw":173,"dy":144,"dz":369,"g ":4531,"Ima":116,"ea":2500,"eb":7309,"ec":6168,"ed":29381,"de":25427,"Ili":79,"dd":738,"dg":429,"di":24057,"dh":181,"dk":1177,"dj":1456,"dm":1512,"dl":1112,"do":16087,"dn":14568,"dp":1050,"ds":3632,"dr":13867,"ew":457,"ex":237,"eu":1040,"ev":19283,"ey":856,"ez":12153,"fa":2439,"h ":28964,"aú":301,"Ind":547,"fe":2538,"bá":91,"eh":4032,"eg":21346,"ef":1414,"ee":651,"Ime":316,"el":42176,"ek":16159,"ej":9407,"ei":1868,"ep":9382,"eo":3145,"en":61076,"em":38275,"et":32921,"es":22879,"er":40015,"ca":8624,"bz":65,"e ":176361,"bv":197,"by":89,"bs":2013,"br":6519,"bu":2540,"bt":104,"bn":3084,"bo":9119,"bj":937,"bk":143,"cL":77,"bl":11842,"bm":547,"bh":211,"bi":17028,"bb":158,"bd":603,"be":10505,"dc":181,"db":1176,"da":28148,"f ":2764,"cz":61,"cy":123,"cv":189,"cu":951,"ct":549,"cs":478,"cq":69,"cr":248,"co":5124,"cm":121,"cn":95,"ck":1329,"cl":245,"ci":15112,"ch":3183,"ce":12343,"cc":353,"c ":6745,"az":10124,"ay":618,"ba":6779,"d ":25792,"at":30196,"as":24786,"ar":39381,"ax":151,"aw":283,"av":30027,"au":2021,"ak":12407,"al":45711,"ai":1701,"aj":23033,"ao":367,"ap":6069,"am":13386,"an":71011,"ac":6592,"ad":26926,"aa":226,"ab":7824,"ag":7152,"ah":7624,"ae":1031,"af":1621,"nu":5151,"nt":10999,"ns":25512,"ič":13659,"nr":295,"np":329,"no":51231,"nn":1495,"nz":984,"ny":526,"nw":82,"jó":58,"nv":255,"oe":569,"ká":135,"of":3946,"oc":3247,"od":39028,"oa":584,"ob":20785,"om":19239,"on":23036,"ok":14837,"ol":27351,"oi":2548,"oj":11831,"og":11726,"oh":1523,"m²":117,"ot":18168,"os":30813,"ov":55859,"ou":2158,"op":10262,"oo":1164,"or":35498,"jč":131,"r ":20683,"ox":97,"kó":141,"ow":586,"kö":75,"oz":8175,"oy":187,"pd":65,"lá":269,"pe":10198,"pa":26706,"Igr":74,"pc":146,"pl":6471,"lé":474,"pn":2144,"po":53400,"ph":657,"pi":10229,"pj":64,"pk":185,"lo":38721,"ln":9297,"lm":1635,"hé":133,"ll":3272,"ls":3383,"lp":1148,"lv":458,"lu":4829,"lt":2975,"lz":195,"ly":422,"hô":58,"Idr":67,"o ":117173,"mc":91,"md":104,"ma":23441,"mb":4609,"dž":1661,"iá":58,"me":39489,"mf":152,"mk":917,"iè":113,"ml":1947,"mi":12739,"eš":4276,"mj":85,"mn":2345,"mm":463,"mp":2668,"mo":14761,"hč":66,"mr":915,"mt":95,"ms":4458,"mv":69,"mu":5539,"my":115,"p ":1343,"na":91417,"nb":277,"nc":8502,"nd":6315,"ne":43250,"já":118,"nf":942,"ež":5003,"ng":4878,"nh":345,"ni":72174,"nj":24738,"nk":3254,"ić":821,"nl":278,"nm":267,"jv":1409,"jt":127,"ju":14494,"eč":5728,"js":10383,"jp":458,"jn":4767,"jo":15551,"jl":168,"jm":397,"jk":398,"kj":752,"ki":56467,"kh":121,"ke":31686,"kd":477,"kc":1346,"ka":42561,"fü":98,"m ":32866,"jz":162,"ky":130,"ks":2035,"kt":5285,"ku":8273,"kv":2313,"ko":52366,"gí":90,"kr":10395,"kk":78,"kl":5176,"km":1749,"gé":62,"kn":1425,"dš":398,"li":49096,"lh":245,"lk":2648,"lj":32952,"le":34235,"há":163,"ld":976,"lg":1256,"lf":539,"hâ":70,"la":37335,"lc":2469,"lb":823,"n ":53330,"hr":1878,"hs":96,"dí":74,"hw":70,"ht":692,"hu":981,"hj":61,"hk":1523,"hi":3365,"hn":1375,"ho":9636,"hl":379,"dè":58,"dé":139,"hm":293,"id":5268,"ic":16189,"ib":3404,"ia":4880,"ih":24332,"ig":5375,"if":1125,"ie":2994,"hy":222,"dú":72,"k ":19129,"iq":110,"ir":15616,"dč":73,"is":20587,"it":17935,"iu":664,"iv":10504,"ix":194,"aš":9037,"ii":2454,"ij":44668,"ik":30544,"il":26890,"im":25029,"in":69797,"io":5253,"ip":4264,"jc":271,"jb":886,"je":114329,"jd":485,"až":1346,"bš":153,"ji":18381,"jh":438,"iz":20523,"l ":16581,"ja":44343,"tä":161,"pš":106,"xi":105,"té":252,"tí":147,"tó":164,"ww":69,"z ":12173,"xa":126,"ož":4663,"tá":178,"nž":333,"wi":424,"oš":3587,"sé":150,"wn":164,"wo":138,"sí":94,"rč":573,"ws":163,"ró":226,"rô":65,"vz":2935,"y ":2958,"wa":490,"sá":124,"we":348,"rè":76,"vl":5625,"vm":123,"ré":314,"vj":1191,"vk":1293,"vh":61,"nš":866,"vi":26446,"vg":688,"vt":1201,"vu":1200,"vr":6944,"vs":7139,"vp":572,"rí":254,"vn":18643,"vo":20029,"uz":834,"ux":225,"uv":1705,"uu":61,"ve":45929,"rá":407,"vd":234,"vc":816,"vb":122,"va":33280,"x ":755,"mš":2061,"ui":868,"uj":5428,"uk":2181,"ul":7777,"ue":1036,"uf":172,"ug":6200,"lž":478,"uh":1992,"ur":10254,"pč":78,"us":9210,"ut":2587,"um":3595,"un":5703,"uo":144,"up":17758,"ty":284,"tz":304,"tu":11745,"tt":1249,"tw":125,"pó":70,"tv":10431,"ub":7050,"ua":3183,"ud":8270,"uc":1029,"w ":442,"to":33340,"tn":10733,"pé":68,"tm":2491,"tl":2152,"ts":2917,"oč":7724,"tr":18852,"tp":111,"tg":72,"tf":115,"te":38423,"pá":79,"td":278,"tk":3275,"tj":2887,"lš":176,"ti":34585,"th":1742,"v ":46495,"tc":114,"ta":42943,"su":2438,"sv":4904,"ss":1521,"st":64846,"sy":100,"sz":642,"sw":95,"sl":11373,"sk":60513,"sn":5408,"sm":2123,"sp":12768,"so":15794,"nč":1620,"sr":3075,"sd":114,"sc":1363,"sf":304,"se":40074,"sh":762,"sj":389,"kš":233,"si":9098,"rz":2218,"u ":23884,"sa":10107,"sb":1404,"mč":427,"rr":991,"rs":12425,"rt":8497,"ru":11754,"rv":10562,"rw":89,"nó":68,"ry":877,"ní":142,"rp":731,"ro":31529,"rn":11113,"né":172,"rm":4944,"rl":1460,"rk":7297,"nç":122,"rj":7769,"jš":3373,"ri":44436,"rh":1257,"rg":4613,"iž":2554,"rf":351,"ná":202,"re":43763,"rd":5068,"rc":1442,"rb":2537,"ra":68896,"t ":20636,"qu":409,"mí":90,"lč":258,"mé":211,"iš":12414,"má":431,"s ":18299,"px":59,"pt":1806,"pu":4524,"ló":186,"pp":444,"lí":193,"pr":43317,"ps":1569,"Hum":81,"yő":97,"vž":60,"zá":92,"už":4277,"vš":416,"uš":1692,"Hrv":3876,"yí":60,"Hra":77,"rž":2418,"zz":215,"vč":549,"zg":2406,"Hor":93,"zh":2141,"zi":10015,"rš":3989,"zb":1418,"zc":70,"zd":3868,"ze":5649,"vá":315,"za":21344,"Hon":64,"Hok":73,"Hol":102,"zv":5632,"zs":530,"zr":2612,"uč":1996,"zu":2176,"zt":382,"zo":6065,"zn":9166,"ví":186,"zp":931,"zk":568,"zj":291,"zm":3200,"vé":151,"zl":2570,"ye":263,"yc":118,"yd":123,"ya":395,"yt":90,"ys":299,"Hoc":85,"yr":276,"yp":87,"yo":155,"yn":206,"ym":124,"yl":250,"yk":172,"yj":189,"yi":151,"Arg":159,"Arh":66,"Are":64,"Ard":164,"šču":315,"Ara":107,"Arm":203,"ščo":81,"ščn":64,"Ari":101,"šči":1771,"Apo":252,"šče":2731,"šča":1225,"Atl":156,"Ast":131,"Ass":80,"Art":149,"Avt":105,"Avs":709,"Ave":81,"Auv":73,"Aut":64,"Aug":99,"zš":796,"Azi":191,"Azu":144,"Bak":78,"Bal":229,"Ban":193,"Bab":79,"Bad":69,"Bar":430,"Bat":109,"Bas":111,"Bav":61,"Aba":278,"Ada":80,"Ado":61,"Afr":271,"Air":92,"Al ":78,"šće":104,"Aka":59,"Akv":95,"Ala":127,"Alb":257,"Ali":77,"Ale":389,"Alf":117,"Alt":76,"All":122,"Alo":87,"Alp":662,"Ame":656,"Ama":87,"Ang":426,"Ana":133,"And":409,"šč ":144,"Ant":584,"Ann":63,"Buz":68,"Buk":74,"Bur":210,"Bud":88,"Bru":110,"Bož":77,"² ":120,"DA ":787,"DD ":66,"Cal":180,"Cam":157,"Cas":187,"Car":397,"Cat":66,"Can":211,"Cap":85,"Bea":66,"Bes":153,"Ber":630,"Beo":125,"Ben":301,"Bel":559,"Biz":86,"Bje":262,"Bil":214,"Bis":245,"Bit":75,"Bio":71,"Blo":64,"Ble":72,"Bla":204,"Bre":545,"Bra":506,"Bro":313,"Bri":606,"Boh":133,"Bog":157,"Bol":225,"Boj":60,"Bon":91,"Bor":510,"Bos":176,"Bou":105,"Îl":184,"Der":74,"Des":135,"Dev":75,"Dek":70,"Del":334,"Dem":64,"Den":114,"Deb":67,"Dam":75,"Dan":339,"Dar":97,"Dav":176,"Dal":121,"Chr":147,"Che":154,"Chi":125,"ám":76,"án":655,"Cit":86,"áj":126,"Cir":117,"ák":172,"ál":309,"ác":115,"ád":136,"áz":116,"áv":123,"ár":751,"át":278,"ás":315,"ât":67,"Châ":62,"Cla":118,"Cel":355,"Cen":243,"Cer":450,"Ces":88,"à ":58,"á ":69,"Cha":633,"Cre":137,"Cor":603,"Com":262,"Col":181,"Con":211,"Cou":99,"ós":120,"ót":110,"óv":101,"ôm":59,"ôn":89,"Duš":60,"ód":99,"ór":148,"ón":273,"óm":66,"ól":124,"ók":78,"ó ":118,"Drž":130,"ív":88,"íz":94,"ín":324,"ír":134,"ít":126,"ís":96,"ík":80,"íl":246,"íj":224,"íd":88,"ía":72,"Egi":80,"ên":58,"êr":99,"éz":94,"ék":134,"él":241,"éj":87,"ém":138,"én":587,"és":167,"ét":261,"ér":344,"év":100,"éd":144,"ée":75,"Edw":93,"èn":67,"èr":126,"ço":94,"é ":306,"ät":162,"Do ":87,"ć ":449,"Dia":61,"Dic":151,"Dis":100,"Dir":87,"Dio":72,"Din":68,"Die":63,"Div":106,"Dub":388,"Dun":175,"ün":108,"ür":209,"Dvo":99,"Dru":220,"ún":88,"új":321,"úr":86,"Dre":102,"Dra":452,"íš":61,"Dob":200,"ôt":129,"ör":97,"Dou":72,"Dol":339,"Don":353,"Dom":230,"Dor":129,"Ned":79,"Nea":69,"Nem":513,"Nek":87,"Nev":71,"Neu":84,"Jás":105,"Nep":74,"Nas":351,"Nat":199,"Nav":148,"Nic":123,"Niz":179,"Nik":208,"OJ ":106,"New":248,"Nap":119,"Nar":283,"Nam":213,"Nan":68,"Nag":201,"Nah":254,"Naj":294,"Nad":104,"Na ":449,"ći":324,"OV ":124,"ća":95,"će":161,"Či":127,"Če":527,"Ča":315,"Ču":69,"Čr":316,"či":13203,"čj":3069,"čk":2033,"čl":1507,"če":9878,"ča":7103,"čb":207,"č ":3958,"đe":65,"Đu":65,"čn":11436,"čo":315,"čr":1069,"ču":2190,"čv":95,"Nji":74,"Nje":320,"Nov":818,"Nor":440,"Not":61,"Odv":1174,"Ogr":131,"Obč":247,"PL ":89,"Okr":139,"Nyí":60,"Obs":170,"Obi":89,"Obr":75,"Obo":60,"Od ":141,"Île":184,"Oto":151,"Oli":186,"Ont":118,"Ope":134,"Ore":118,"Org":93,"Ost":65,"Osj":214,"Osm":58,"Osn":93,"Po ":423,"š ":428,"Pli":67,"Ple":143,"Pla":466,"Pin":72,"Pik":79,"Pit":66,"Pis":66,"Pir":250,"Pie":165,"Phi":96,"Ped":76,"Per":309,"Pes":215,"Pet":518,"Pen":134,"Pel":66,"šč":6357,"šć":151,"Pat":125,"Pas":151,"Par":669,"Pav":183,"Pau":156,"Pad":61,"Pac":78,"Pan":157,"Pap":82,"Pal":206,"Pak":100,"še":3798,"ša":2845,"šo":760,"šp":1066,"šn":1922,"šk":22840,"šl":713,"ši":4197,"šj":684,"šv":183,"šu":178,"št":5493,"Še":283,"Ša":280,"Šm":149,"Šo":105,"Ši":182,"Šk":346,"Šu":75,"Št":406,"Šv":259,"Šp":321,"Pož":327,"Ptu":112,"Pro":707,"Pri":1370,"Pre":1114,"Prv":307,"Pru":73,"őr":96,"Pra":329,"Pod":622,"Pok":159,"Pol":1336,"Pom":102,"Pon":195,"Pog":199,"Poi":167,"Poj":93,"Pot":652,"Pos":334,"Pov":181,"Pop":91,"Por":385,"Poz":75,"žr":68,"žu":4511,"žn":3757,"žo":181,"žc":62,"žb":1014,"že":6604,"žd":91,"ža":4974,"žk":205,"žj":1185,"žl":145,"žg":130,"ži":6432,"Žu":1147,"Ža":131,"Ži":199,"Že":372,"RS ":268," ال":88,"ž ":1046,"Rac":68,"Rad":357,"Ram":58,"Mün":75,"Ran":79,"Rak":65,"Que":120,"Irs":68,"Ita":679,"Isl":158,"Ist":728,"Ira":123,"Inš":104,"Iva":381,"Izv":187,"Izr":254,"Izd":68,"Jac":161,"Jad":111,"Jav":92,"Jar":108,"Jap":208,"Jan":529,"Jam":191,"Jak":171,"Jel":130,"Jer":198,"Jes":136,"Jez":169,"Jea":145,"Izš":73,"Je ":506,"Jos":354,"Jor":75,"Jon":77,"Joh":406,"Jug":908,"Jud":70,"Jup":160,"Jur":234,"Jul":171,"Jož":199,"LA ":59,"Juž":280,"Kam":278,"Kal":271,"Kap":166,"Kan":372,"Kat":241,"Kas":83,"Kar":544,"Kaz":101,"Kav":68,"Ker":149,"Ken":111,"Kis":102,"Kir":77,"Kit":180,"Kin":85,"Klo":59,"Kli":73,"Kle":103,"Kla":125,"Kon":517,"Kom":582,"Kol":204,"Kos":201,"Kor":867,"Kop":552,"Kov":80,"Kot":149,"Koz":113,"Knj":131,"Kob":74,"Koc":70,"Kre":107,"Kra":1273,"Kri":388,"Krk":87,"Kro":118,"Krš":94,"Koš":77,"Koč":102,"Kul":64,"Kun":76,"Kur":69,"Kva":82,"Lev":100,"Let":1929,"Les":99,"Leo":137,"Len":143,"Lau":76,"Law":92,"Le ":107,"Lag":58,"Lah":83,"Las":91,"Lat":85,"Lar":113,"Lam":67,"Lan":280,"Lab":97,"La ":179,"Lju":1716,"Lib":144,"Lig":77,"Lim":128,"Lin":177,"Lip":125,"Lit":222,"Luk":154,"Lui":81,"Lun":67,"Lud":105,"Luc":114,"Lou":148,"Lov":122,"Los":65,"Lot":115,"MS ":61,"Loi":123,"Log":104,"Lor":123,"Lon":188,"Lok":174,"Lič":69,"Lež":206,"Meh":106,"Men":114,"Mel":123,"Mes":284,"Mer":173,"Met":220,"Med":836,"Mač":73,"Mez":77,"Man":399,"Mal":462,"Mar":2349,"Mas":191,"Mag":152,"Mad":1011,"Maj":111,"Mak":235,"Mai":69,"Mac":176,"McL":66,"Max":58,"Mau":68,"Mat":387,"Mla":152,"Mod":111,"Moh":80,"Moj":64,"Mol":94,"Mon":601,"Mos":493,"Mor":297,"Mou":64,"Mot":314,"Mih":227,"Mik":175,"Mic":218,"Mit":62,"Mir":154,"Mis":119,"Mil":386,"Min":343,"NK ":115,"Mur":270,"Mus":87,"Moš":81,"çoi":67,"Wor":66,"Wol":82,"Wik":71,"Wil":329,"Win":160,"ère":90,"Wes":77,"War":73,"Wal":129,"Vzh":106,"Vse":94,"Vrb":98,"Vra":69,"Vrh":141,"Vol":145,"Voj":511,"Vod":126,"Viš":166,"Več":124,"Vis":126,"Vit":109,"Vla":188,"Zla":131,"čuj":826,"čun":696,"Vél":58,"čut":108,"črk":423,"čre":113,"črt":288,"éte":58,"črn":193,"ču ":419,"Zna":123,"Zdr":568,"ény":95,"Zap":71,"Zar":126,"Zas":101,"Zav":104,"Zag":314,"Zah":111,"Zak":109,"Zal":123,"ékt":72,"én ":269,"éli":106,"Zgr":67,"Zgo":249,"éra":62,"Zim":226,"Zel":179,"Zem":564,"ов ":64,"Zač":142,"之":72,"三":107,"Zad":119,"Za ":213,"Yor":161,"на ":82,"Szo":98,"Sza":480,"Sys":60,"Sve":2328,"Sup":72,"Sud":142,"Str":416,"Stu":150,"Sti":77,"Sto":225,"Sta":2102,"Ste":244,"Teh":61,"Ten":108,"Tem":109,"Teo":88,"Tel":99,"Tek":83,"Tam":78,"Tan":81,"Tar":117,"Tak":138,"Tal":65,"Ta ":195,"Sko":62,"Skr":66,"Sku":204,"Ska":62,"Sha":82,"Sim":142,"Sil":93,"Sis":205,"Sir":188,"Sin":157,"Sib":59,"Sez":419,"Ses":154,"Ser":176,"Sev":340,"Sen":145,"Sel":241,"Sem":91,"Sei":131,"Sed":75,"Srb":301,"Sre":466,"TV ":116,"Sv ":98,"Spa":211,"Spl":481,"Spi":65,"Spe":164,"Spr":96,"Spo":313,"Sod":63,"Sok":63,"Soc":117,"Sob":124,"Sou":80,"Sov":270,"Sol":119,"Som":85,"Son":199,"Sop":149,"Sor":59,"Sla":201,"TO ":84,"Slo":6226,"Sli":73,"So ":69,"Rož":72,"Rus":577,"Rud":148,"Sai":290,"Sam":204,"Sal":223,"Sad":71,"Sco":63,"Sch":394,"Sav":417,"Sat":60,"Sau":74,"Sar":260,"San":495,"ови":138,"Rač":75,"SI ":76,"Res":82,"Rev":59,"нов":79,"Rim":348,"Rib":92,"Ric":128,"ät ":150,"Ras":66,"Rav":91,"Raz":271,"Rde":102,"SG ":79,"Rec":63,"Red":122,"Rei":91,"Reg":444,"Ren":132,"Rek":120,"Rep":2306,"Rog":117,"Rob":196,"Roc":78,"Rod":85,"Rou":102,"Ros":172,"Ron":234,"Rom":237,"SS ":448,"SO ":89,"Vel":2994,"Ven":157,"ски":72,"Vas":75,"Van":123,"Val":340,"Var":221,"Vid":128,"Vic":103,"Vie":59,"Vir":205,"Vil":211,"Vik":75,"Vin":206,"Ver":276,"Ves":184,"Ukr":102,"Uni":842,"Ura":80,"Ust":217,"Upo":141,"Trž":77,"Ter":179,"The":362,"Tho":193,"Tih":67,"Tim":73,"Tis":131,"Tir":83,"To ":334,"Top":167,"Tor":193,"Tok":64,"Tol":157,"Tom":179,"Ton":82,"Tou":87,"Tru":58,"Trs":127,"Tro":210,"Trn":82,"Tri":282,"Trg":65,"Tre":271,"Tra":244,"Tur":330,"Tuk":62,"ši ":887,"šev":379,"šem":193,"šel":270,"šen":350,"šes":555,"šer":80,"šeg":118,"šek":215,"ša ":1145,"še ":1531,"šar":175,"šav":187,"šah":216,"šaj":99,"šal":222,"šan":614,"Šve":109,"Švi":128,"što":95,"štr":65,"šte":2122,"šti":888,"šta":268,"šuj":72,"štv":1659,"štu":262,"švi":77,"šve":96,"špa":358,"šov":61,"špo":639,"št ":64,"šu ":58,"ško":7029,"šlj":265,"šla":312,"вич":166,"šo ":156,"šić":67,"šni":492,"šnj":513,"šne":308,"šna":327,"šič":190,"šno":281,"šol":499,"šic":246,"šib":61,"šin":843,"šil":171,"šim":108,"šik":69,"ših":534,"šit":122,"šir":718,"šje":233,"šja":121,"šji":274,"ška":2789,"ški":6172,"ške":6766,"́н":79,"cLa":71,"bju":136,"bje":482,"bja":288,"áto":72,"biz":60,"bis":206,"bit":738,"biv":1271,"bio":487,"bir":767,"ász":140,"bil":9355,"bim":90,"bin":672,"bij":500,"bo ":1111,"blj":4814,"blo":131,"ble":384,"bli":5531,"bn ":78,"bla":906,"bod":537,"bok":127,"bol":2141,"boj":794,"bog":423,"boh":73,"bič":489,"bno":809,"bna":528,"bni":923,"bne":712,"bmo":523,"biš":76,"bon":136,"bom":159,"bor":1964,"áza":65,"bot":328,"bos":204,"bov":492,"bou":88,"áln":82,"be ":2097,"bam":109,"ban":872,"bak":214,"bal":951,"baj":107,"áko":72,"bah":107,"bac":170,"án ":124,"baz":156,"bav":207,"bat":235,"bas":207,"bar":1233,"ánt":85,"bdo":308,"áno":61,"ány":67,"bde":133,"azš":326,"ánd":61,"bda":111,"bi ":1760,"bej":129,"beh":88,"ár ":304,"bec":216,"ber":3398,"ben":1839,"bel":842,"bez":109,"bes":1297,"bet":155,"baú":258,"bho":160,"bia":67,"bib":116,"bic":125,"áro":114,"ári":73,"áci":80,"buš":62,"ca ":7059,"car":308,"cas":90,"cat":115,"can":153,"cam":193,"cal":177,"cah":276,"ce ":3980,"bri":1142,"bro":951,"brn":171,"bra":2116,"bre":1080,"bu ":321,"brs":91,"bru":767,"bsk":752,"bso":165,"bse":534,"bst":433,"boč":145,"bur":549,"bul":151,"bum":224,"buj":619,"bud":138,"buc":60,"bus":181,"bve":153,"by ":66,"bož":173,"aka":2233,"am ":1501,"ake":764,"akc":307,"aki":595,"ajk":170,"ajl":126,"aji":853,"ajo":3508,"ajp":438,"ajm":203,"ajn":1032,"ajs":1806,"ajt":95,"aju":1775,"ajv":1227,"al ":5680,"ajb":861,"aja":4631,"ajd":410,"ajc":106,"aje":1321,"ajh":414,"ail":145,"ain":656,"air":120,"ais":193,"ak ":1483,"ahk":1261,"ahl":62,"ahi":99,"ahu":67,"aht":170,"aho":1733,"aj ":3209,"agy":182,"aha":1141,"agl":148,"agm":59,"agi":325,"agr":2630,"agu":186,"agn":486,"ago":1560,"anu":2488,"anz":168,"ano":4496,"ann":617,"ant":2576,"ans":8949,"ane":3760,"ang":2192,"anh":58,"ani":12168,"anj":11003,"ank":1463,"ap ":82,"ana":5327,"anc":5066,"and":2328,"amu":229,"amm":99,"amo":2039,"amn":424,"amp":456,"ams":594,"amk":68,"ami":1934,"adž":1259,"ame":3673,"amb":620,"ama":1428,"ao ":196,"alv":164,"alu":459,"alt":357,"als":1671,"alp":219,"alo":3988,"aln":5663,"alm":581,"all":563,"alk":590,"alg":235,"ali":12519,"adš":352,"alj":2204,"alc":1638,"ald":278,"ale":2748,"alf":59,"Šam":81,"ala":5384,"alb":313,"an ":7080,"aks":331,"akr":499,"aku":1530,"akt":954,"ako":3028,"akn":93,"akl":239,"aba":528,"abe":1164,"abi":922,"abl":2808,"abn":285,"abo":950,"abr":232,"abs":510,"abu":83,"ae ":446,"aca":87,"ad ":1592,"ac ":892,"ab ":93,"afo":67,"afr":130,"aft":161,"afs":248,"aff":75,"afe":70,"afi":418,"ai ":215,"aga":832,"age":434,"ael":296,"ah ":2832,"afa":85,"ado":1208,"adr":500,"adl":177,"adk":221,"adn":2260,"adm":341,"adg":67,"adj":251,"adi":2931,"add":208,"adc":70,"ade":1675,"ag ":119,"adz":178,"ads":309,"adu":492,"aco":154,"ack":237,"aci":3927,"ach":405,"ace":463,"acc":103,"ada":12281,"adb":350,"af ":280,"act":73,"azn":554,"azm":458,"azp":368,"azo":464,"arš":186,"azi":1682,"azl":1367,"azk":65,"azv":987,"azu":362,"azr":436,"azt":209,"azs":228,"aze":313,"azg":172,"aza":617,"Špa":241,"azb":60,"azd":489,"avč":119,"azz":92,"az ":748,"ayl":61,"aye":106,"Šta":81,"Šte":144,"ba ":1670,"Štu":63,"at ":1858,"arh":471,"arg":219,"are":1974,"ard":2757,"arc":477,"arb":245,"ara":3352,"arp":61,"aro":2967,"arn":2439,"arm":683,"arl":640,"anç":98,"ark":1199,"arj":3449,"ajš":619,"ari":4347,"aru":188,"arv":516,"arr":314,"ars":3622,"art":3803,"au ":230,"asa":521,"ary":261,"akš":197,"asi":1326,"ash":118,"asc":90,"asb":1150,"ase":7736,"aso":654,"asn":756,"asp":358,"ask":223,"asm":118,"asl":1005,"ar ":4861,"apa":403,"Šen":181,"ape":643,"api":1573,"aph":75,"apn":71,"apl":264,"apo":1357,"app":73,"apr":1152,"aps":75,"apt":76,"apu":142,"as ":2234,"avc":158,"avb":92,"ava":3257,"ax ":66,"aux":63,"aut":447,"avs":1220,"avt":891,"avr":171,"Ško":256,"avo":1996,"avn":8963,"avk":259,"avl":3158,"avi":5083,"anš":152,"avj":154,"avg":550,"ave":2196,"Šma":126,"ay ":236,"awa":58,"avz":193,"avu":60,"arč":77,"awn":81,"anž":60,"av ":1266,"ata":1793,"asu":583,"ast":7399,"ass":309,"anč":577,"atm":280,"atn":621,"atk":2238,"atl":115,"atr":682,"ato":4303,"ate":7456,"alš":59,"ati":8392,"atj":90,"ath":285,"att":232,"ats":603,"atu":979,"aul":252,"aum":70,"aun":59,"aur":182,"aus":178,"aud":100,"auk":85,"ος":138,"ος ":138,"ς ":264,"ν ":77,"Zve":255,"α ":118,"еви":63,"ий ":84,"ич ":167,"až ":103,"jeg":1394,"jej":1047,"jed":501,"jec":86,"jep":94,"jer":828,"jek":653,"jel":447,"jem":3606,"jen":7756,"jez":1544,"jes":258,"jet":1512,"jev":3484,"jač":189,"ji ":9921,"aža":200,"ažd":84,"aže":590,"ažj":94,"aži":70,"ažn":142,"jhe":109,"jhn":247,"jad":100,"jat":493,"jas":144,"jav":1972,"jap":253,"jar":116,"jal":4024,"jak":772,"jan":7670,"jam":569,"jah":261,"jaj":1388,"jaz":69,"jbo":832,"jce":82,"je ":90645,"izš":397,"jci":81,"jde":159,"jda":105,"jna":637,"ješ":262,"jmo":79,"jni":1155,"jne":1049,"jič":83,"jno":1777,"eč ":1193,"jol":65,"jon":462,"jos":83,"jor":101,"jpo":317,"jpr":102,"ск":123,"jiv":675,"jit":167,"jis":201,"jim":790,"jin":1168,"bšk":61,"jik":186,"jil":336,"jaš":1441,"jij":72,"jig":518,"jih":2539,"jic":888,"те":99,"ječ":100,"ст":65,"ул":59,"jn ":65,"jko":128,"jka":118,"jo ":14224,"jma":181,"jlo":62,"itn":496,"itm":98,"itl":62,"itk":153,"itr":810,"ito":1788,"itv":1047,"itu":620,"itt":146,"its":553,"itz":130,"ity":105,"isk":1191,"ism":148,"isl":637,"iso":1260,"isn":596,"isp":122,"iss":283,"inč":105,"isu":398,"ist":8943,"isz":119,"iv ":665,"ita":4085,"itd":78,"ite":3294,"ith":151,"iti":3232,"itj":163,"ivo":647,"ivn":1608,"ivu":289,"inž":250,"ius":336,"ium":162,"iva":2833,"ix ":146,"inš":354,"ivi":1461,"ivj":69,"ivk":187,"ivl":505,"ive":1813,"ipr":269,"ipo":673,"ipp":99,"ipu":59,"ips":60,"ipt":181,"ipi":261,"ipl":628,"is ":1968,"ion":2321,"iop":95,"ior":98,"ios":95,"iot":214,"iog":97,"iok":133,"iol":587,"iom":79,"ipa":1086,"ipe":338,"iov":126,"ir ":1030,"iru":579,"irs":381,"irt":62,"iro":1658,"irn":1433,"irk":3666,"iri":1414,"irj":549,"isi":572,"ish":146,"ise":765,"isc":429,"isa":2843,"iu ":69,"iqu":102,"ilč":111,"ire":909,"irg":67,"ira":3405,"irc":132,"it ":737,"ünc":71,"ivč":108,"ür ":68,"ivš":143,"ja ":24669,"iz ":4501,"izu":488,"izv":2556,"izr":1033,"izs":209,"izp":299,"izo":951,"izn":244,"izm":1841,"izl":440,"izk":366,"izj":155,"irš":113,"izi":2488,"izh":387,"izg":268,"ize":500,"izd":905,"izb":341,"iza":2030,"kaš":110,"kih":6752,"kij":103,"kim":1029,"kil":157,"kie":91,"kiv":97,"kin":454,"kip":232,"kir":103,"kis":399,"kit":420,"kaž":123,"kje":723,"km ":698,"ki ":46414,"ked":200,"keg":3700,"kej":985,"kem":8923,"kel":218,"ken":160,"kes":145,"ker":562,"ket":264,"kev":340,"key":87,"kač":293,"ke ":15947,"kci":1307,"kda":435,"kra":4811,"krb":350,"kre":318,"kt ":433,"ksa":263,"kse":141,"ku ":1833,"kro":1818,"krv":182,"kri":2242,"koz":528,"kov":6012,"km²":98,"kot":4172,"kos":954,"kor":1260,"kop":980,"koo":149,"kon":3042,"kom":2571,"kol":3036,"kok":1914,"koj":101,"koh":100,"kog":105,"kof":1624,"kod":500,"ks ":201,"kmu":212,"kme":268,"kmo":345,"koc":91,"kob":135,"kne":116,"kni":115,"knj":1121,"klu":447,"ko ":24162,"kma":99,"kle":646,"kla":1462,"klo":710,"kli":1093,"klj":712,"jvo":146,"jut":62,"jus":126,"jul":555,"jun":698,"jur":257,"jve":859,"jvi":346,"joč":433,"jub":2164,"juj":406,"jug":1198,"jud":1091,"jsk":8627,"jst":1435,"ečj":1199,"ečk":404,"eči":887,"ečn":355,"ečo":63,"eču":68,"ju ":6445,"jse":265,"jiš":131,"eča":390,"eče":1036,"již":650,"kaz":528,"kav":355,"kat":5637,"für":83,"kar":3117,"kas":410,"kap":270,"kan":1808,"kal":3239,"kam":632,"kaj":559,"kak":434,"kah":366,"kai":81,"kad":433,"kac":404,"juž":772,"ka ":23634,"juč":601,"jze":62," Ga":900," Ge":881," I ":207," Fo":2042," Fu":219," Fr":2139," Fi":677," Fl":224," Ha":1561," He":1083," Cô":91," Gy":129," J ":84," Go":1655," Gr":1898," Gu":360," Gv":83," Gi":550," Gl":510," Ig":179," Id":108," Ib":77," K ":116," Hu":329," Hr":4060," Ho":699,"ha ":480," Hi":543," Ji":67," Je":1541," L ":110," Ja":1800," Iz":882," Iv":415," Ir":276," Is":1032," It":703," Im":512," In":1618," Il":228,"ham":322,"han":759," M ":155,"hai":79," Ka":2499,"haj":1477,"hal":392," Ke":540,"hau":83," Ki":619,"har":950,"has":60,"hat":80," Jo":1288," Ju":1997,"haf":95,"hae":137,"hab":68,"had":64," N ":83," La":1551," Le":3180," Li":1291," Lj":1723," Kl":443," Kn":255," Ko":3810," Kr":2271," Kv":109," Ku":508," Mc":133," Ma":6006," O ":269," Ml":210," Mi":1860," Dž":96," Me":2209,"he ":889," Lo":1511," Ly":76," Lu":757," Já":138," Ne":1590,"а ":247," P ":279," Na":2970," Nj":398," Ni":827," Mr":101," Mo":2454," Mu":575,"hek":94,"hel":368,"hei":125,"heb":110," A ":314,"het":73,"hes":152,"her":719,"heo":145,"hen":330,"hem":187,"hi ":171," B ":212," C ":442," Ap":398," Am":984," An":1817," Ak":264," Al":2235," Ai":256," Aj":81," Ag":178," Ah":85," Af":338," Ac":171," Ad":366," Ab":561," Ba":1832," D ":134," Az":412," Av":1041," Au":488," At":316," As":363," Ar":1291," Be":2192,"hie":110,"hid":239,"hic":86," Bi":1002,"hia":98,"hip":210,"hio":70," Bj":272,"hin":407,"him":147," Bl":437," Bo":1900,"hil":201,"hik":78,"hij":109," Br":2242," Bu":794,"his":178,"hit":605,"hir":188," E ":92," Ca":1521," Ce":1243," Ci":464," Ch":1220," Cl":266," Cr":346," Co":1629," Cu":189," Cv":64," Cy":61," F ":123," Da":1039," Di":915," De":1297," Dr":1054,"hkr":181," Do":1606,"hko":1232," Du":855," Dv":142," Ea":63,"hn ":214," Ed":330," G ":65,"hla":193," El":398," Ek":172," Ei":122," Eg":129," Et":115," Es":219," Er":320," Ep":63," En":408," Em":164," Eu":245," Ev":1000," Fe":726,"ho ":80,"hma":181," Fa":830," H ":159,"gma":95,"go ":1697,"glo":304," Z ":182,"gle":2479,"gli":810,"glj":228,"gla":2978," Wo":190," Wi":722," Wh":62," We":269," Wa":403,"й ":140," Vz":150,"gog":353," Zr":80," Zu":99,"god":1736," Zv":306,"gob":81," Vé":82," Zm":60," Zl":152," Zo":102," Zn":141," Zd":593," Ze":880," Zg":337,"gič":79,"gno":136," Zi":327,"gni":103,"gnj":147," Za":1745,"gne":563,"gna":300," Yo":229,"gs ":62,"о ":66,"goz":506,"goj":282,"н ":123,"gom":421,"gol":311,"gon":470,"gos":3146,"gor":2864,"got":348,"gov":2780,"gu ":806," a ":458,"р ":62,"gro":564,"grm":63,"gru":125,"bču":59,"grs":221,"gra":6878,"grb":61,"bči":4969,"gri":272,"gre":942," R ":122,"в ":115," Oz":78," Os":737,"gto":98," Ot":263," Or":643,"goč":401," Op":320," Po":5507," Lé":59," Pl":785," Pi":1026,"gul":122," Ph":190,"gua":77," Pe":1565,"gub":149," Pa":2188,"gue":188," Ny":85," Nu":95," No":1550," Ol":321," Ok":257," On":182," Om":107," Oh":58," Og":200," Od":1566," Oc":80," Ob":907," Ra":1525," Mü":85," T ":114," Qu":210,"új ":254,"goš":89," Ro":1658," Re":3734," Rd":107," Ri":916," Rh":128," S ":293," Pr":3986,"gur":145,"gus":676," Pt":177," Pu":321,"gun":145," Má":81," Sz":701," Sy":146," Sw":66," Sv":2526," Su":652," St":3290," Ta":1068," V ":1176,"gya":99," Th":689," Ti":702," Te":1006," Tr":1582,"gyk":115," To":1461," Ru":970," Sa":2415," U ":88,"е ":69," Sh":246," Si":1155," Sc":584," Se":2090," So":1610," Sp":1368," Sr":838," Sk":482," Sl":6566," Sm":233," Sn":82," Va":911,"и ":110," X ":144," Ve":3898," Vi":1616," Vl":200," Vo":997," Vu":138," Vr":488," Vs":192," Tu":669," Ty":77,"grš":1077," Uk":147," Ul":103," Um":80," Un":907," Up":188," Ur":251," Us":272," ja":3029," l ":65,"iam":240,"ial":1334," iz":12947,"ian":961," ji":1172,"ias":76,"iar":121," je":57692,"iat":484," io":104," ip":64," im":7503," in":30104," il":218,"ic ":1048,"iac":130," is":755," it":1465,"iag":159," ir":117,"ibl":600," fü":65,"ibi":274," ka":7604,"ibo":889," m ":545,"ibn":134,"ibr":170," kj":700," ki":23502,"ibu":174," ke":816," jo":1895,"id ":690,"iba":621,"ibe":314," ju":3473," ha":275," he":621," gi":540," gl":3153," gr":2915," go":2718,"ia ":1100," gu":114," k ":178," ib":60," id":224," ig":1379,"ib ":92," hi":1051," hk":181," hl":97," ho":1169," hr":955," hu":193,"iet":196,"ieu":64," nj":2131," ni":1612,"iel":266," ne":7311,"ien":545," na":42601,"ier":585,"ies":172,"ied":168,"ieg":71,"и́":65," mu":575," mr":220,"ig ":209," mo":4823," mn":787," mm":80," ok":4185," ol":657,"ifu":62," om":674," on":135," og":566," oh":232,"ifo":187," oc":331," od":6669," of":580," ob":12756,"ifr":67,"ife":161,"ifi":406,"ih ":22740," nu":342," no":2512," np":247,"ifa":63," le":11324," lj":1128,"icr":71,"ics":75,"ict":217," li":1970,"icu":137,"icn":80," n ":309,"ico":974,"ick":188," la":3283," kv":554," ku":716,"ici":3252,"ich":665,"ice":3027," kn":1159,"ie ":718," km":1016,"ica":6322," kl":1067," kr":5141," ko":11511," me":12878," dž":64,"idu":106," mi":1837,"ids":75," ml":614,"я ":69,"idr":346," o ":887,"ido":732," ma":6202,"idn":485," lu":270,"idi":434,"idg":71,"ide":1356,"ida":711," lo":1132," af":94," ag":154,"aša":593," ab":327," ac":59," ad":284,"aše":251," am":1331,"ašk":6766," an":2354,"aši":200," ap":760,"ašn":368,"iin":107," ak":715,"iim":2227," al":6858," av":1858," au":60," ar":1254," at":457,"ašt":146," as":1847," d ":320," ba":1756," az":58,"il ":5175,"ija":12085," bi":10040,"ije":12707," be":1503,"iji":5965," bo":2432," bl":915,"ijo":6465,"ijs":4363," bu":186,"iju":165," br":1850," ca":173," e ":85,"im ":3856,"ika":6461,"ige":666,"iga":1104,"aš ":78,"igl":183,"igh":214,"igi":601,"igu":193,"igr":1537,"igo":249,"ign":280,"ij ":2784,"т ":70," b ":94,"ihe":100,"iha":530,"ihi":87,"iho":729,"ik ":9136,"у ":103," c ":70," er":65,"imo":1733,"imn":223," et":360," es":185," en":4762,"ims":2333," em":109," ep":251,"imp":926,"imf":91," el":1413,"ime":9448," ek":723,"imk":790,"imi":2451,"ip ":337," fe":1007,"inc":748,"ind":925,"ina":10483," fa":1370,"imu":325,"а́":109," ev":586," fu":480,"inn":113," fr":3146,"ino":5575,"ašč":462," fo":718,"int":1271,"ins":4445,"inf":316," fl":126,"ine":5190,"inh":91,"ing":1310,"inj":1193," fi":2355,"ini":4151,"ink":465," ge":1321," ga":4003,"iod":295,"inu":433," i ":115,"inv":62,"inz":59," cl":64,"iko":4804," cm":100,"ikl":979," co":254,"iki":3287," ce":2735," ch":72,"ike":4355," ci":552,"ila":7057,"ilb":66," f ":61,"in ":31978," da":5215,"ikv":58," cv":150,"ikt":162,"iku":799,"ikr":245,"iks":94," do":7621," dn":241,"ilo":4202,"ill":1121,"ilk":457," dr":7206,"iln":1875,"ilm":587,"ilh":107," de":11720,"ilj":766,"ili":2960,"ild":76,"ilc":146," di":5516,"ile":1135,"ima":2334,"imb":293,"ч ":172," ed":1001,"io ":829," dv":2054," du":1905,"ils":366,"ilt":85,"ilu":372,"ilv":125,"ль":58," vč":201," zm":655," zl":487,"ла":90," zo":151," zn":3542," zu":280,"ле":83," uč":541," zr":511,"ли":66," zv":1609,"hok":781,"hol":497,"hom":250,"hon":167," za":14763,"ко":128,"hos":66," zd":1135,"hot":761," ze":976," zb":804,"hov":2393,"ку":58,"hop":61," zi":186,"hor":342," zg":1780,"ка":91,"ки":115,"hod":3942,"hni":523,"hno":289,"hnu":58,"hna":106,"hne":92," z ":4835,"ин":98,"ик":81,"ий":90," ož":121,"ич":191,"ри":87,"ро":86,"ра":120,"ре":58,"htt":71,"hto":63,"htn":72,"hte":153," už":81,"ор":104,"ол":82,"ов":288,"hu ":190,"hrv":430,"но":113,"hro":110,"hre":73,"ни":87,"hri":399,"ht ":188,"на":128,"hra":653,"hiš":212," ru":1131," u ":188," sa":1902," se":18649," sc":148," si":3326," sh":137," sn":739," sm":996," sl":6094," sk":4932," sr":1993," sp":10659," so":10507,"ви":220," t ":95," ra":7736," re":7014," rd":178," ri":2298," ro":2318," pt":209," pu":582," pr":31964," ps":336," s ":4586," px":59,"hy ":96,"ва":103,"ад":75," os":4746," ot":1470,"hum":246," ov":185,"hun":84,"hus":107," op":2076," or":2176,"hur":142,"ан":143,"ак":68," oz":3235," pe":3830," pa":8196,"ар":96," pl":2669," po":40217," pi":2589," y ":101," vz":1551," x ":129," va":2039," ve":6450," uv":1301," vn":129," vo":6456," vp":464," vr":2767," vs":3256," vi":2299," vk":283," vl":673," vm":85," ud":201,"ет":79,"ер":77,"ен":63," tv":236," tu":5865," us":2479," ut":152," ur":2875," up":8796," um":682," un":617," uk":539," ul":238," ug":179," ta":4305," v ":29906," st":7117," sv":3870,"о́":61," su":765,"ев":143," oč":210," tr":3796," tl":193," to":2461," th":371," ti":1200," tk":120," te":8405,"fi ":65,"ffe":78,"ffi":69,"fes":342,"fer":516,"fed":154,"feb":634,"fen":145,"fek":403,"fel":89," Ča":315," Či":126," Če":525,"faz":112,"fat":64,"far":143,"fan":669,"fak":920,"fal":122,"ff ":62,"fe ":105," Đu":65,"fa ":189,"aúj":298," ču":115," čr":759,"eyr":71," Ču":69,"exa":83,"ez ":873," Čr":316," če":1273," čl":1304," či":437," ča":1729,"ezu":507,"eza":1493,"ezd":639,"ezn":2303,"ezo":3022,"eví":88,"euč":161,"eze":1111,"erš":61,"ezi":1770,"eta":9837,"ete":2260,"etd":101,"etj":1541,"eti":2887,"elš":67,"eth":104,"etn":3058,"etl":581,"etk":435,"esp":164,"esn":1865,"eso":719,"est":9497,"esu":218,"enč":254,"esr":69,"ess":341,"ev ":4515,"emš":2022,"eto":5019,"etr":1803,"ets":629,"ett":335,"etu":861,"etv":352,"ew ":247,"eve":3581,"evd":110,"evc":189,"eva":3659,"evo":978,"evn":851,"evl":113,"evk":344,"evj":120,"enš":165,"evi":3431,"euv":65,"eut":92,"eur":130,"eus":174,"ex ":78,"evu":106,"evr":568,"evs":223,"evt":159,"ey ":565,"evz":91,"epe":218,"epi":485,"eph":184,"er ":9909,"epa":2146,"eos":79,"eor":822,"eom":284,"eol":485,"eop":91,"eon":196,"es ":3592,"ept":897,"epu":2717,"epl":115,"epn":163,"elé":65,"epp":110,"epo":984,"epr":1096,"erk":1130,"erl":288,"ejš":1389,"eri":6828,"erj":1569,"erg":1075,"erh":74,"ere":2880,"erf":64,"erc":387,"erd":237,"era":3623,"erb":290,"et ":2922,"esj":60,"esk":362,"esl":322,"esm":246,"esi":510,"esc":198,"ese":3032,"eu ":71,"esa":1428,"erz":1265,"ery":65,"erv":377,"eru":570,"emč":403,"err":420,"ert":841,"ers":1249,"ern":2752,"erm":750,"erp":166,"ero":2923,"eki":619,"ekl":439,"ekm":687,"eko":1561,"ekr":138,"eks":912,"ekt":2647,"eku":579,"ekv":164,"en ":7706,"elb":111,"ela":2552,"eld":165,"elc":327,"elf":63,"ele":5343,"eli":7355,"elj":12080,"elg":274,"ehé":68,"elm":113,"eln":478,"elk":390,"ell":946,"elo":6099,"elu":1481,"els":386,"elt":135,"eo ":194,"emb":2938,"ema":2409,"edž":154,"eme":2514,"emd":94,"eml":1047,"emn":341,"emo":1461,"emi":2775,"emu":847,"emp":655,"ems":675,"ep ":100,"ene":4626,"enh":70,"eng":267,"enb":171,"ena":6583,"end":853,"enc":1426,"eno":7193,"enn":350,"enk":486,"enl":165,"eni":11655,"enj":3527,"enu":1052,"ens":9125,"ent":4611,"enr":131,"enz":343,"eog":377,"eod":241,"eob":101,"egl":515,"ego":1324,"egn":116,"ege":392,"egi":3464,"ej ":1349,"eha":369,"egr":207,"egu":231,"egy":59,"ehn":551,"ehr":149,"eho":984,"ehi":163,"ek ":3893,"eic":68,"eis":179,"eir":92,"eim":143,"eil":148,"ein":591,"eid":153,"eja":1145,"el ":3464,"eiz":90,"eit":89,"ejs":725,"ejo":1467,"ejn":247,"ebš":60,"eji":700,"eje":1829,"ekd":397,"eke":552,"ekc":78,"eka":3433,"em ":19835,"eju":371,"gl ":122,"git":143,"gis":130,"gir":72,"gim":692,"gij":4610,"gik":62,"gip":136,"gin":441,"gio":182,"gie":59,"gib":460,"gih":484,"gia":72,"ght":197,"gha":74,"ggi":66,"gač":142,"gi ":1057,"gen":1655,"geo":621,"get":148,"ger":535,"ges":132,"gh ":98,"geb":124,"geg":139,"gem":133,"gel":572,"gej":96,"gda":62,"ge ":1547,"gac":68,"gad":460,"gah":82,"gas":134,"gar":537,"gat":408,"gaj":272,"gam":198,"gal":701,"gan":2184,"ga ":19235,"fur":60,"fte":66,"fun":414,"ft ":152,"ačb":149,"ača":451,"fra":3096,"fre":206,"ače":1435,"ačj":77,"ačk":163,"fri":439,"ači":1402,"fsk":383,"fro":120,"ačn":463,"ačr":174,"aču":1108,"fov":120,"for":1149,"fos":82,"fot":167,"fon":272,"fol":119,"ač ":265,"fič":180,"fla":71,"fic":219,"fie":63,"fig":93,"fij":889,"fil":1101,"fik":288,"fin":542,"fit":105,"fiz":1064,"fja":62,"db ":97,"da ":14922,"dbe":281,"dba":322,"dbi":164,"dbo":232,"de ":3505,"dac":99,"dal":2088,"daj":1290,"dag":370,"dah":122,"dae":198,"dat":2652,"dar":1727,"dan":3799,"dam":251,"dav":343,"dda":258,"dde":307,"dce":69,"cul":97,"cto":99,"cti":246," Îl":184,"cy ":79,"cve":165,"cus":130,"cur":63,"cks":59,"cko":94,"cla":60,"cle":108,"co ":1155,"con":195,"col":193,"com":102,"cor":143,"cos":2856,"cot":60,"cou":95,"cs ":326,"cqu":61,"cro":134,"cu ":453,"cci":147,"cca":61,"cea":385,"ch ":526,"cev":1776,"cer":935,"ces":1318,"cet":77,"cen":988,"cep":196,"cej":81,"cem":898,"cel":1340,"ceg":119,"ced":151,"ci ":2670,"cha":428,"chw":61,"chu":146,"cia":974,"ck ":555,"cie":148,"cid":94,"che":837,"chl":72,"chi":339,"cho":135,"chm":179,"chn":116,"cht":110,"civ":109,"cij":6609,"cik":850,"cil":290,"cim":128,"cif":150,"cih":145,"cir":298,"cis":476,"cit":587,"cin":475,"cio":727,"cip":225,"cm ":87,"cke":249,"cka":102,"ed ":7632,"eba":410,"ebe":1030,"ebi":1286,"ebl":102,"ebn":705,"ebo":328,"ebr":1109,"ebu":659,"ec ":3986,"eac":70,"eag":60,"eae":122,"ead":77,"eak":245,"ean":473,"eal":308,"ear":308,"eas":76,"eap":68,"eat":181,"eau":166,"eb ":1452,"ea ":232,"efi":249,"efo":206,"efa":145,"efe":483,"ei ":95,"ega":14282,"een":162,"eh ":1490,"eer":68,"eev":58,"edk":372,"edl":222,"edm":691,"edn":4723,"edh":90,"edi":2962,"edj":165,"ede":4374,"ône":77,"eda":2590,"edb":168,"eg ":627,"edt":128,"eds":1765,"edv":775,"edu":552,"edp":144,"edo":1120,"edr":537,"eck":137,"ech":121,"eci":430,"ece":981,"eca":75,"ee ":129,"ef ":166,"ecu":64,"ect":136,"eco":75,"dož":69,"dvs":488,"dwa":102,"dy ":102,"dvi":1792,"dve":1124,"dvo":628,"dur":65,"dus":252,"dva":885,"duš":167,"drž":2000,"dzo":143,"dzi":74,"dze":62,"dor":425,"dop":136,"don":666,"dom":1025,"dol":2617,"dok":396,"doz":167,"dow":104,"dov":2766,"dot":187,"dos":834,"dr ":66,"dpi":182,"dpr":535,"dpo":228,"ds ":150,"diš":1126,"dmi":400,"dmo":356,"dna":1989,"dne":2475,"dni":3893,"dež":1743,"dnj":2661,"dno":3481,"dič":320,"dob":2688,"doc":83,"dod":289,"dog":505,"dst":1636,"dso":71,"dte":147,"dun":61,"duj":151,"dul":104,"duk":196,"duh":1475,"duc":137,"dri":615,"dra":2046,"dt ":61,"dre":1648,"du ":2258,"dro":1558,"drs":225,"dru":5537,"dsk":1257,"dse":412,"dge":82,"dgo":243,"dic":712,"did":62,"dia":709,"dho":97,"ôte":98,"der":1069,"des":1368,"det":180,"dev":559,"dez":147,"deb":135,"dea":152,"ded":131,"dec":783,"def":191,"dej":655,"del":8200,"dek":748,"den":2287,"dem":894,"dep":1917,"deo":172,"di ":7874,"dle":116,"dla":418,"dko":292,"dkr":290,"dki":137,"dme":496,"dma":184,"do ":2879,"dlo":208,"dlj":134,"dli":196,"dja":348,"dje":967,"div":552,"diu":58,"diz":86,"dim":344,"din":3211,"dio":320,"dip":467,"dir":3392,"dis":608,"dit":557,"die":123,"dif":117,"dig":180,"dih":219,"dij":1635,"dik":285,"dil":1003,"dka":143,"dke":226,"dju":78,"deč":365,"rgy":70,"rgu":403,"rhe":114,"rha":142,"rhi":372,"rhu":89,"rhn":68,"rho":225,"iža":226,"rga":1747,"ri ":6205,"rgl":58,"iži":369,"rgi":606,"ižj":188,"iže":597,"rge":583,"rgo":351,"ižn":1012,"rgn":87,"ret":1927,"res":1802,"rev":1974,"reu":328,"rez":1267,"rh ":216,"rfi":77,"rfo":61,"rač":1077,"rdu":102,"rds":107,"rg ":550,"iž ":81,"reb":2051,"rea":620,"ree":132,"ref":492,"rec":915,"red":9699,"rei":333,"rej":1975,"reg":4478,"reh":767,"rem":2494,"ren":2516,"rek":2213,"rel":1062,"rer":139,"reo":180,"rep":1325,"rda":399,"rcu":159,"rdo":364,"rdn":359,"rdi":2193,"rde":564,"re ":2956,"rbu":279,"rbs":360,"rci":266,"rch":173,"rce":392,"rca":244,"raz":7036,"rd ":765,"rap":609,"rar":714,"ras":1404,"rat":4465,"rau":89,"rav":10605,"rbi":646,"rbo":438,"rba":318,"rbe":206,"rc ":65,"raj":3231,"rai":122,"rah":639,"rag":448,"ran":10813,"ram":1883,"ral":3846,"rak":1045,"rab":4754,"raf":1035,"rae":188,"rad":7153,"rac":1106,"rpu":173,"rpo":96,"rs ":406,"rpe":109,"rpa":103,"ror":143,"ros":2112,"rot":1454,"rom":2391,"ron":2361,"roo":117,"rop":1736,"roz":633,"rou":216,"rov":3453,"row":90,"rob":744,"roa":69,"rod":3371,"roc":719,"roj":1187,"roi":1061,"rol":591,"rok":1027,"rof":772,"roe":143,"rog":1831,"rno":2826,"jšč":126,"rič":1125,"rns":80,"rnu":76,"rna":2170,"rež":741,"rne":1804,"rnj":672,"rni":3001,"rmo":552,"rmu":1839,"ro ":1744,"rma":1564,"rme":357,"rmi":362,"reš":389,"rlo":204,"rlj":61,"rli":272,"rle":264,"rla":328,"rn ":268,"rkv":475,"rku":166,"rkt":89,"rks":59,"rkn":66,"nço":87,"rko":671,"rki":446,"rke":698,"rka":3906,"rm ":91,"reč":850,"rju":560,"rji":433,"rja":3706,"raž":658,"rje":3010,"riz":1024,"rl ":197,"rip":1323,"jšo":114,"rio":714,"rir":527,"rit":2886,"ris":2096,"riv":1041,"riu":103,"rih":1145,"rig":966,"rij":4080,"raš":557,"jši":1298,"rii":2351,"ril":2003,"rik":1886,"jšn":99,"rin":1694,"rim":3965,"jša":896,"ria":1305,"rib":1637,"ric":1600,"rid":1029,"rie":837,"jše":763,"rif":96,"rk ":603,"roš":477,"rož":1407,"ruj":117,"ruh":77,"rug":3066,"rud":146,"ruc":91,"rup":276,"run":235,"rum":444,"rul":77,"ruk":405,"ruz":173,"rus":1264,"rut":93,"rva":5491,"rvi":1217,"rve":3036,"rvo":519,"rvn":190,"ry ":643,"rsk":7463,"rsi":153,"rso":356,"rsa":151,"rse":525,"rta":677,"rst":3047,"rtm":1893,"rtn":868,"rto":742,"rte":567,"rth":222,"rti":1649,"rub":82,"rua":643,"rts":92,"roč":1581,"rtu":402,"rtv":101,"riš":1946,"rt ":931,"rro":107,"mči":401,"rri":145,"rre":246,"riž":633,"rra":237,"ru ":1442,"rry":141,"sab":104,"sac":80,"sad":165,"saj":415,"sak":747,"sal":910,"sam":1606,"sba":85,"sbe":912,"sbi":124,"san":1213,"sat":1223,"sas":75,"sar":861,"sav":254,"sa ":1996,"ruž":2445,"ón ":126,"ruš":505,"rze":647,"rza":241,"ryj":60,"rzo":76,"rzi":1047,"sha":61,"sho":59,"shr":72,"she":68,"shi":263,"si ":1301,"sje":270,"siv":155,"seč":117,"sid":220,"sic":368,"sia":89,"sk ":194,"sit":181,"sir":202,"sis":1373,"sip":188,"sin":800,"kšn":168,"sio":244,"sil":1296,"sim":507,"sij":849,"sik":175,"sih":502,"saš":156,"sif":95,"sig":135,"sbo":112,"sbu":112,"se ":9988,"sca":143,"sce":180,"sci":213,"sch":525,"sco":197,"sev":1928,"ser":946,"ses":2057,"set":1089,"sez":2688,"sh ":159,"sfe":159,"sfo":66,"sei":64,"seh":402,"seg":907,"sed":3642,"sec":285,"seb":3164,"sep":883,"sen":803,"sem":2072,"sel":7746,"sek":379,"sej":258,"spu":64,"spo":2044,"spr":1492,"spe":1038,"spl":748,"spi":220,"spa":7101,"sot":293,"sou":74,"sov":1364,"sol":487,"som":341,"son":987,"sop":193,"sor":556,"sos":106,"sod":1527,"sof":102,"sok":589,"soj":102,"soc":318,"sob":183,"su ":1066,"nčn":750,"nči":273,"nče":232,"sre":2496,"srb":412,"nča":248,"st ":5603,"ss ":196,"sli":984,"slo":6036,"slu":603,"sla":2779,"sle":851,"ski":17344,"skl":1236,"sko":15385,"skr":668,"sku":2764,"skv":191,"ska":8402,"ske":14199,"sič":233,"sno":1962,"sna":435,"sni":1800,"snj":102,"sež":377,"sne":1079,"smo":133,"smr":229,"smu":290,"so ":8342,"sma":396,"smi":413,"sme":609,"sz ":97,"sza":136,"sze":92,"szt":71,"sse":338,"ssa":253,"sso":262,"ssi":329,"ste":4516,"sta":14712,"std":76,"stm":122,"stn":2769,"sto":9875,"sti":10401,"stj":760,"stk":173,"stl":589,"stv":6827,"stu":585,"soč":125,"str":7508,"sub":140,"suh":69,"sul":130,"sum":64,"suj":230,"sup":155,"sun":61,"sur":243,"sve":2955,"svi":91,"svo":1687,"tai":89,"taj":1653,"tak":1296,"tal":5739,"taf":62,"tag":120,"tah":112,"tab":357,"tac":360,"tad":115,"td ":92,"taz":72,"tav":4146,"tat":2093,"tas":324,"tar":2532,"tap":65,"tan":6147,"tam":415,"tch":88,"te ":3456,"tde":172,"ta ":16983,"ovš":201," št":2696," šv":175," ši":477,"pa ":6188," šk":1366," šo":471," šp":977," ša":246," še":1284," Šv":259," Šu":75," Št":405," Šp":321," Šo":105," Šm":149," Šk":346," Ši":182," Še":283," Ša":280,"ovč":87," šč":159,"pci":101,"pe ":794,"par":3565,"pat":325,"pas":565,"pav":120,"paz":284,"pac":108,"pad":8267,"pah":117,"pak":187,"pal":492,"paj":410,"pap":279,"pan":5531,"phe":97,"pha":91,"pho":59,"phi":118,"pi ":445,"ph ":153,"pev":467,"pač":83,"pea":111,"pec":334,"ped":485,"pen":956,"pep":58,"per":2309,"pet":1059,"lás":63,"pes":1184,"peh":707,"pel":589,"pek":319,"pla":1692,"plj":349,"pli":1265,"ple":1082,"plo":1730,"piz":105,"peč":83,"phy":92,"pia":104,"pid":74,"pic":81,"pih":119,"pij":931,"pik":108,"pil":764,"pin":2417,"pio":83,"pir":404,"pis":3588,"pit":455,"poz":877,"pr ":382,"por":6507,"pop":763,"pov":2801,"pou":108,"pot":2623,"pos":4743,"poi":280,"poj":1082,"pog":1928,"pom":2891,"pon":1163,"pok":980,"pol":5783,"pob":187,"poe":84,"pod":14532,"ps ":70,"plé":235,"ppo":61,"ppe":190,"lén":278,"peš":381,"po ":5444,"pič":147,"pno":567,"pnj":141,"pež":186,"pni":1125,"pne":206,"pna":93,"pse":98,"psi":273,"psk":942,"pso":79,"ptu":77,"pub":2962,"pte":795,"pti":568,"pto":218,"poč":106,"pra":8108,"pt ":65,"piš":190,"prv":4499,"prs":213,"prt":260,"pru":88,"pu ":357,"pri":11604,"pre":12368,"pro":5665,"poš":266,"pož":71,"pur":71,"pus":357,"put":68,"pun":78,"pul":238,"px ":59,"puš":308,"már":283,"iš ":113,"iše":281,"iša":220,"išl":273,"išn":372,"iši":543,"išk":6388,"išj":568,"išt":1077," Ži":199," Že":372," Ža":131," Žu":1147,"išć":99,"išč":2416," ži":1623," žl":73," ža":202," že":1263," žu":4184,"mén":60,"lčn":96,"qua":66,"que":223,"qui":88,"ra ":5234,"rb ":124,"ežn":508,"ngo":291,"ežj":186,"ngi":146,"eži":1395,"ngl":1754,"ežk":163,"ngu":231,"ngr":157,"ngt":104,"ngs":96,"ni ":21439,"eže":1628,"nge":731,"ngh":66,"nga":200,"eža":393,"nha":154,"nj ":974,"nhe":59,"neh":90,"neg":6715,"nej":1610,"nei":60,"nel":292,"nek":1786,"nen":595,"nem":5832,"nep":765,"neo":239,"ner":2095,"net":1642,"nes":1470,"nev":638,"neu":164,"ndv":192,"ež ":616,"ng ":946,"nea":167,"neb":386,"nec":499,"ned":247,"nfo":286,"nfr":73,"nač":1875,"ney":179,"nez":634,"nfa":314,"nfe":121,"nco":3080,"nci":1963,"nck":74,"nce":1597,"nch":251,"nca":769,"ne ":16673,"nbu":104,"ndu":365,"ndr":735,"nds":176,"ndo":673,"ndi":1187,"nde":738,"nda":1053,"ncu":165,"nak":2105,"nal":3824,"nam":2274,"nan":3844,"nap":1555,"nar":4271,"nac":538,"nad":2224,"nag":2320,"nah":1010,"nai":103,"naj":4748,"nc ":424,"nab":409,"nbe":101,"nd ":921,"nav":1486,"nau":169,"nat":1633,"nas":9591,"naz":549,"na ":45810,"muč":308,"mož":442,"nyi":95,"nz ":124,"nož":611,"ny ":285,"nve":127,"nuk":69,"num":110,"nun":197,"nuj":664,"nus":215,"nut":274,"nua":1914,"nud":60,"ntv":66,"nto":1165,"ntn":394,"ntu":151,"nts":282,"noč":122,"ntr":823,"nti":1732,"nth":129,"ntj":84,"nta":1740,"nte":2274,"nsp":149,"nso":136,"nst":4879,"nsf":59,"nse":185,"nsi":206,"nsk":18646,"nsc":84,"nsa":368,"nu ":1386,"iču":99,"ičn":8484,"ičk":636,"njš":744,"iči":660,"nri":146,"niž":206,"iče":649,"iča":1089,"nt ":1816,"niš":3134,"npr":251,"ns ":498,"noc":81,"nod":149,"noa":98,"nob":153,"nog":856,"nof":95,"nok":237,"nol":557,"noi":86,"noj":94,"noo":307,"nop":217,"nom":2492,"non":367,"not":1702,"nos":5345,"nor":656,"nov":7837,"noz":157,"ič ":1910,"nne":487,"než":185,"nna":175,"nić":78,"nno":127,"nni":311,"nič":1334,"nma":70,"neš":91,"ići":278,"nlj":152,"nn ":230,"no ":28820,"nke":357,"nki":201,"nm ":145,"nkc":434,"nka":820,"nku":97,"nko":712,"nkt":84,"nkr":119,"nji":3942,"njk":62,"nje":10740,"nja":5163,"ić ":428,"nju":1491,"neč":59,"njs":1007,"njo":572,"nij":12636,"naš":856,"nih":7675,"nig":117,"nif":78,"nie":143,"nid":118,"nic":4360,"nia":191,"nk ":305,"niz":1781,"niv":1064,"nis":1438,"nit":923,"nir":658,"nio":254,"nim":2568,"nin":1324,"nik":9988,"nil":489,"obč":4860,"ogr":2005,"ogu":149,"ogi":1571,"ogl":820,"ogo":3723,"ogn":253,"oga":915,"oge":466,"ohr":268,"ohl":58,"ohi":154,"oho":271,"ohn":289,"oha":244,"ohe":84,"oj ":853,"ois":217,"oir":158,"oit":129,"oin":114,"oim":560,"oid":870,"ok ":1437,"ojz":107,"ojv":145,"oju":183,"ojs":1259,"ojo":151,"ojn":3019,"ojm":105,"oji":1325,"oje":1842,"oja":2676,"ol ":641,"oiz":309,"oce":920,"och":145,"oci":1214,"ock":526,"oco":76,"obs":723,"obv":173,"obu":288,"oca":63,"odg":241,"ode":2090,"odk":490,"odl":499,"odi":2311,"odj":811,"odo":2908,"odp":861,"odm":304,"odn":5885,"ods":542,"odt":73,"odr":2425,"of ":1863,"odd":348,"odc":60,"odb":552,"oda":3554,"oel":114,"oen":93,"odz":124,"odv":669,"odu":1390,"og ":1603,"ofi":975,"ofj":119,"ofs":134,"oft":92,"ofo":143,"oh ":61,"oev":59,"off":75,"ofe":287,"ofa":151,"oa ":64,"ob ":1309,"oc ":81,"oam":62,"oak":62,"oba":1124,"od ":12681,"oar":69,"obo":1557,"obr":1689,"obl":2408,"obn":1165,"obm":522,"obh":168,"obj":732,"obi":1733,"obd":521,"obe":1617,"nza":121,"nze":101,"nzi":167,"nzo":136,"nzu":292,"oz ":588,"ows":139,"own":81,"ozv":174,"ozm":90,"ozn":2233,"ozl":77,"ouč":124,"ozo":660,"ozd":363,"oze":956,"ozj":60,"orš":73,"ozi":1844,"oza":922,"otu":189,"oud":99,"ouc":89,"ow ":91,"otl":190,"otj":79,"oti":1413,"oth":99,"ote":2905,"ott":253,"ots":334,"otr":1161,"oto":3788,"otn":1790,"ost":12934,"osu":137,"osv":805,"ota":1246,"ov ":9875,"osi":707,"osk":3606,"ose":3687,"osf":145,"osp":608,"oss":155,"onč":622,"osr":908,"osm":266,"osl":2901,"oso":665,"osn":920,"ovz":1051,"owe":104,"ovj":805,"ovi":6438,"ovn":6797,"ovl":1128,"ovk":184,"ovr":972,"ovp":60,"ovo":2629,"ovs":1169,"ova":8912,"ovc":416,"ove":14862,"olž":454,"oug":85,"oui":145,"oul":129,"oun":202,"ous":295,"our":376,"out":124,"opn":423,"opo":1059,"opi":1707,"opk":100,"opl":563,"ope":1736,"oph":167,"opa":1127,"os ":1104,"opu":601,"opr":1200,"opt":284,"ops":761,"ook":176,"ood":98,"or ":2915,"oot":78,"oos":261,"oor":234,"ork":374,"orl":98,"orm":2841,"orn":2246,"oro":2344,"orp":318,"orr":124,"orc":198,"ord":1013,"ore":2187,"orf":214,"org":1762,"ori":3690,"orj":1749,"ou ":161,"osa":1004,"osc":65,"ort":1525,"ors":2919,"orv":191,"oru":646,"orz":678,"ory":80,"m² ":116,"ot ":4531,"orb":248,"ora":6836,"olč":72,"ola":1198,"old":256,"olc":287,"on ":4034,"olj":3299,"oli":8521,"oll":233,"olk":1067,"olf":200,"ole":2539,"olh":61,"olg":662,"ols":682,"olt":165,"olm":216,"oln":1077,"olo":4662,"olp":199,"olz":68,"olu":478,"okc":353,"oka":3564,"om ":4672,"oki":527,"oke":1295,"okr":2091,"oks":347,"oko":2939,"okl":420,"okv":386,"okt":717,"oku":681,"ona":3944,"ond":566,"onc":719,"onf":152,"one":1109,"ong":354,"onj":581,"oni":3533,"onk":153,"onn":242,"ono":2372,"ons":2332,"ont":1027,"onu":380,"onv":124,"ony":108,"onz":388,"oma":3776,"ome":4659,"omb":385,"omi":1226,"omm":146,"oml":116,"omp":471,"omn":379,"omo":1779,"omt":60,"omu":612,"omr":152,"oms":494,"op ":353,"la ":14406,"kuž":64,"ína":99,"ín ":66,"ílo":74,"kuš":168,"le ":7709,"lce":1290,"lca":262,"lci":518,"lcs":234,"lf ":143,"őr ":86,"lde":127,"lda":104,"ldo":98,"ldi":83,"lab":207,"lac":514,"lad":2741,"lah":1475,"lag":668,"laj":371,"lai":176,"lal":158,"lak":556,"lan":4164,"lam":493,"lap":158,"lao":145,"lar":855,"lat":1921,"las":3314,"lau":130,"lav":3720,"lay":77,"laz":215,"lba":139,"ld ":368,"lbe":231,"lbi":66,"lbo":79,"lbu":240,"kvi":678,"kve":490,"kva":1108,"kus":268,"kur":144,"kup":2763,"kun":201,"kum":211,"kul":2114,"kuj":269,"koš":266,"ky ":76,"kta":268,"kte":244,"ksp":155,"kst":183,"ksi":550,"kso":225,"ksn":135,"kuh":70,"ktr":1174,"koč":387,"ktu":785,"kti":841,"ktn":114,"kto":1394,"krš":290,"íja":164,"kož":132,"lpo":59,"lps":155,"lpe":443,"lpi":97,"lph":69,"ls ":116,"lol":70,"lok":331,"lon":766,"lom":1330,"lop":499,"lor":376,"lod":293,"loc":74,"loh":68,"log":2821,"loj":149,"lpa":140,"los":423,"lot":1148,"lou":85,"lov":16172,"loz":578,"lno":1801,"lić":59,"lnj":113,"lni":3993,"lež":843,"lne":1558,"lob":711,"lič":1905,"lmo":111,"lmi":113,"leš":1366,"lme":146,"lma":647,"lp ":83,"lna":1726,"lmu":66,"hér":83,"lms":222,"lti":179,"lto":145,"ltr":80,"loč":1559,"lts":67,"ltu":477,"luc":178,"lub":512,"lug":80,"lue":65,"lsk":2568,"lso":81,"lst":393,"lta":305,"lte":1274,"ljš":529,"liž":879,"lu ":1284,"liš":2924,"ía ":66,"lt ":201,"lhe":70,"lj ":3087,"lha":78,"lgo":251,"lge":200,"lgi":319,"li ":11443,"lga":376,"lfr":59,"lač":361,"hât":62,"lfo":59,"lfi":89,"lfa":90,"ház":65,"lez":1147,"ley":245,"lex":102,"lev":543,"les":1759,"let":8204,"ler":672,"leo":160,"lep":918,"lem":1328,"len":1784,"lek":2164,"lel":63,"lei":115,"lej":216,"leh":60,"leg":676,"lef":95,"led":2297,"lec":1220,"leb":105,"lea":97,"lg ":68,"lls":71,"llu":110,"lly":120,"lo ":8977,"lla":550,"lle":786,"lli":672,"llo":334,"lko":692,"lku":60,"ln ":62,"lka":768,"lke":292,"lki":170,"ljs":1060,"leč":258,"lju":2873,"ljo":339,"ljn":358,"lm ":297,"lje":13663,"ll ":501,"lja":8525,"ljk":59,"laž":224,"lji":2324,"lit":3130,"lis":1861,"lir":252,"lip":265,"lio":237,"lin":3453,"lim":1141,"liz":841,"liv":1015,"liu":90,"lic":2658,"lid":143,"lia":493,"lib":162,"lk ":580,"lik":9667,"dšk":353,"lil":332,"laš":191,"lij":4137,"lig":667,"lih":765,"lie":274,"lif":220,"ma ":6156,"luž":388,"mb ":135,"mac":507,"mah":62,"maj":3291,"mak":294,"mad":706,"mag":926,"mar":1529,"mas":537,"mal":1048,"mam":116,"man":2972,"maz":61,"mat":4492,"mba":322,"mbi":346,"mbe":1994,"mbr":581,"mbo":391,"mbn":601,"me ":3191,"mbu":131,"mde":94,"med":6987,"meg":105,"mec":75,"met":4065,"mev":70,"mes":5031,"mer":4601,"mem":950,"mel":2712,"meo":80,"men":7879,"meh":365,"mek":1496,"mej":987,"mez":436,"mač":311,"mfo":74,"luz":65,"lva":240,"lve":110,"lvi":73,"luk":109,"luj":862,"lun":128,"lum":243,"lut":187,"lus":353,"ly ":206,"loš":1311,"ltä":132,"lož":762,"lza":60,"luč":58,"luš":63,"mpi":770,"mpe":599,"mpo":176,"mpl":480,"mpu":59,"mpt":60,"ms ":130,"mog":546,"mob":319,"mod":647,"mon":864,"mok":221,"moj":62,"mom":240,"mol":590,"mov":1124,"mor":2946,"mos":831,"mot":853,"mou":135,"mpa":312,"moz":59,"mre":315,"mrl":143,"mrt":287,"mu ":1595,"miš":516,"moč":1268,"mso":95,"msk":4107,"moš":595,"my ":71,"mur":404,"mus":157,"mut":85,"mul":1968,"mun":583,"muz":183,"dža":1295,"mi ":4700,"dži":218,"meč":117,"maž":60,"min":1835,"ešn":268,"mio":71,"mil":655,"mim":87,"mir":615,"mis":564,"mit":679,"ešt":114,"miz":89,"mic":185,"eša":172,"eše":255,"mie":124,"mid":113,"ešk":2738,"mik":628,"mij":927,"maš":150,"eši":154,"mih":245,"mo ":2844,"mlj":1176,"mle":124,"mla":591,"mki":710,"mka":116,"mm ":88,"ešč":404,"mič":519,"mni":612,"mnm":116,"mno":883,"mna":379,"mne":300,"meš":231,"mma":103,"mme":132,"Če ":73,"Čep":77,"Češ":144,"Črn":256,"rža":2068,"rže":110,"rži":130,"ča ":1709,"čal":131,"čam":203,"čan":1056,"včn":97,"čar":720,"čas":2119,"čaj":758,"čak":128,"vče":97,"vča":256,"zre":618,"uče":431,"uča":350,"zra":1527,"če ":2812,"uču":301,"zro":341,"učn":253,"uči":543,"čat":86,"čav":96,"čba":88,"čbe":68,"víl":96,"ziš":265,"zte":155,"čeg":102,"čen":2815,"čem":362,"čel":601,"ček":421,"čev":1119,"čet":926,"čes":137,"čer":105,"zto":106,"čep":104,"zse":225,"zu ":354,"zst":173,"zva":406,"zvi":2202,"zve":1702,"či ":940,"zvr":346,"zvo":918,"zuj":323,"čez":81,"zur":194,"zul":378,"zum":402,"zun":254,"zus":115,"čij":767,"čih":292,"čic":410,"čk ":142,"čit":716,"čis":78,"čin":8235,"čil":961,"čim":249,"čko":555,"čkr":195,"čka":504,"zzo":59,"čke":298,"čki":315,"zza":68,"čjo":263,"čju":736,"čja":509,"čje":920,"čji":637,"češ":144,"člo":587,"čo ":156,"čle":236,"čla":635,"čob":73,"čič":263,"čić":71,"čni":3672,"čno":2056,"čna":1890,"čne":3797,"čiš":160,"zgl":160,"zi ":788,"zač":773,"zha":357,"zgu":93,"zgr":400,"zgo":1653,"zej":178,"zdr":1230,"zdj":83,"zdo":198,"zdn":338,"zet":178,"zen":780,"ván":80,"zem":1646,"zel":858,"vár":138,"zer":530,"ze ":1270,"zbo":448,"zbi":532,"zbu":82,"zbr":206,"zda":703,"zdi":107,"zde":1030,"zab":392,"zad":779,"zac":850,"zaz":93,"zd ":136,"zbe":87,"zai":81,"zaj":329,"zag":420,"zah":1609,"zam":289,"zan":1549,"zak":553,"zal":882,"zar":791,"zap":901,"zav":812,"zas":719,"zat":657,"zod":127,"zob":347,"zor":519,"zom":118,"zon":2846,"zol":213,"zof":467,"zpe":152,"zpa":113,"zoz":300,"zov":527,"zpr":114,"zpo":448,"ال":104,"zo ":291,"zma":791,"zmn":81,"zmo":266,"zme":1501,"zmi":301,"zna":6251,"zmu":231,"zno":574,"zič":152,"ršč":1404,"zne":577,"zni":1693,"zka":85,"zko":152,"zkl":76,"zki":60,"zku":97,"zla":344,"zli":1716,"zle":63,"zlo":375,"zho":1743,"rša":147,"zia":58,"rše":205,"zid":236,"zic":70,"zij":1942,"ršj":95,"zaš":153,"rši":594,"ršn":88,"zin":229,"zim":197,"zil":483,"zik":2477,"ršk":1367,"zio":176,"zir":1419,"zis":483,"zit":356,"ziv":530,"zja":105,"zje":158,"yst":102,"ysi":60,"yro":69,"yon":68,"za ":8558,"ye ":58,"yer":80,"ya ":130,"yar":104,"yku":71,"yle":59,"yi ":114,"yje":66,"yja":58,"ن ":69,"ožu":71,"ože":1171,"oža":441,"ožb":309,"ožn":777,"oži":853,"ožj":690,"ožg":76,"ož ":115,"té ":62,"tät":153,"ći ":280,"xan":70,"ošč":370,"wn ":129,"ws ":103,"rče":97,"rči":287,"wor":59,"wer":110,"wel":68,"nže":273,"oš ":92,"wis":86,"ošt":692,"oši":152,"oše":119,"wic":65,"ošo":62,"ošn":419,"win":68,"ošk":1384,"oša":203,"vzo":115,"vzn":60,"vzr":280,"vzp":183,"vzg":93,"vze":217,"vzd":233,"vrš":1631,"vzh":1577,"vza":88,"wal":60,"war":181,"viš":902,"vrt":231,"vrs":1856,"vrn":62,"vro":1339,"vri":65,"vrh":387,"vre":731,"vra":482,"vso":118,"vst":1541,"vse":2779,"vsk":1809,"vsi":97,"vu ":881,"vsa":763,"vto":922,"vtr":73,"voč":112,"vts":62,"vul":228,"via":94,"vk ":203,"vio":100,"vir":2583,"vik":86,"vil":3700,"vim":485,"vin":3336,"vig":158,"vih":947,"vaš":4759,"vij":2768,"vic":1147,"vid":829,"vie":118,"nše":75,"vja":192,"viz":807,"nšt":314,"vit":1128,"vis":1138,"več":3002,"vje":763,"vka":371,"vju":170,"vko":123,"vke":182,"vkl":325,"vla":601,"vle":130,"vlo":204,"vlj":4644,"vo ":5275,"vme":74,"rén":67,"veš":647,"vež":91,"vne":3166,"vna":1676,"vno":7416,"vić":274,"vnj":103,"vni":6241,"nšč":326,"vič":1157,"vob":458,"vod":2329,"vog":94,"voj":6252,"vol":768,"vok":302,"von":637,"vom":445,"vor":1714,"vot":255,"vos":442,"vov":212,"voz":481,"vpi":58,"vpl":277,"vpr":143,"vgu":581,"vi ":4563,"vač":184,"vey":59,"vez":2827,"ver":4490,"ves":828,"vet":5958,"vdo":109,"vej":290,"veh":446,"veg":583,"rán":91,"ven":15616,"vem":1214,"vel":2593,"vek":749,"ved":1600,"vec":1132,"vcu":59,"vca":96,"ve ":3712,"vci":431,"vce":194,"val":4949,"vak":400,"van":6929,"vam":248,"var":2538,"vat":726,"vas":1162,"vaz":61,"vac":503,"vad":923,"vai":77,"vaj":1562,"vah":328,"va ":7766,"uzi":197,"urš":301,"uze":286,"uza":111,"urč":177,"ux ":175,"uva":93,"uve":379,"uvr":1131,"usl":98,"usm":167,"usk":1090,"usi":676,"use":332,"usa":242,"usu":60,"ust":3507,"uss":205,"usp":414,"uso":223,"usn":197,"utl":65,"utn":316,"uth":139,"uti":197,"ute":579,"uta":264,"utt":69,"uts":102,"uto":194,"utr":115,"us ":1853,"ut ":342,"urb":124,"ura":3309,"urd":58,"ure":1237,"urg":652,"urj":220,"uri":925,"url":67,"urk":122,"urm":63,"urn":892,"uro":433,"urs":478,"urt":136,"uru":97,"ury":61,"upa":4858,"ur ":618,"upi":1896,"upe":339,"upo":4232,"upr":4636,"upl":66,"upn":1389,"umr":152,"umu":107,"umi":296,"umo":158,"uma":367,"umb":237,"ume":1232,"unt":159,"uns":232,"unk":535,"unj":73,"uni":1649,"uno":147,"unc":267,"und":490,"una":1305,"ung":160,"une":190,"up ":120,"uks":65,"ukr":140,"uku":79,"ukt":476,"uko":214,"ukn":60,"ukl":92,"uki":118,"ukc":123,"uke":60,"um ":819,"uka":277,"ulu":100,"ult":2100,"uls":66,"ulo":296,"ull":127,"ulk":70,"ulj":355,"uli":1131,"ule":1791,"ulf":70,"uld":69,"ula":1090,"un ":304,"ukv":294,"uig":66,"mšk":1972,"uil":118,"uin":78,"uir":59,"uis":202,"uk ":155,"uje":4627,"uji":154,"ujo":137,"ujs":75,"uit":156,"ul ":297,"uja":181,"ugh":94,"ugi":1041,"lži":298,"uge":696,"lžn":64,"ugo":2841,"ugl":86,"uga":776,"uhi":80,"uho":1525,"ugu":295,"uha":183,"uj ":60,"uda":313,"ude":487,"udj":82,"udi":5813,"udn":106,"ubo":133,"ubn":112,"ubs":149,"ubr":361,"uca":72,"ue ":195,"uce":114,"uci":418,"uch":129,"uck":88,"uer":142,"ues":152,"uh ":100,"uds":601,"udo":411,"ug ":215,"ued":59,"uen":108,"uel":148,"ub ":374,"uar":2656,"ual":170,"uan":113,"ubi":368,"ubl":4817,"ube":341,"uba":207,"ud ":197,"trž":74,"ty ":183,"tvu":608,"tvo":3047,"tve":2705,"tvi":733,"tva":3299,"tur":2565,"tus":332,"tut":264,"tuj":422,"tul":174,"tun":135,"tum":104,"tub":74,"tua":118,"tud":5466,"tuc":67,"tug":184,"tz ":159,"toš":66,"ts ":250,"tiš":235,"tmá":241,"trd":395,"tre":2501,"oče":1653,"tt ":133,"oča":1047,"tra":4316,"trj":88,"očj":1644,"očk":616,"očl":103,"će ":102,"trm":72,"trg":328,"tri":4216,"oči":1373,"trs":733,"oču":60,"tru":1087,"trt":467,"tro":4048,"trn":84,"očn":740,"tu ":1700,"try":92,"tsc":117,"tsk":2341,"tta":143,"tte":299,"tti":218,"tto":129,"ttp":71,"tts":81,"toč":930,"tma":1947,"to ":6420,"tmo":86,"tmi":151,"teš":130,"tež":380,"tni":5182,"tne":1620,"ća ":63,"tp ":71,"tna":1556,"tič":5076,"tno":2333,"tod":350,"toc":762,"toj":799,"toi":353,"toh":82,"tog":270,"tob":807,"tou":124,"tov":5531,"tos":1124,"tot":329,"toz":100,"tom":1530,"ton":1758,"tok":2514,"tol":3650,"tor":3549,"top":2051,"oč ":359,"tij":686,"til":753,"tik":2804,"tif":115,"tie":119,"tih":1408,"tig":120,"tir":1113,"tit":785,"tis":2436,"tin":2891,"tim":668,"tip":613,"tio":1015,"thu":192,"tia":170,"tib":79,"tic":1448,"tid":147,"tji":207,"tju":277,"teč":443,"tjo":443,"tiz":729,"tiv":1588,"tje":1033,"tja":867,"tki":1874,"tko":496,"tku":163,"tka":483,"tke":192,"tlj":122,"tli":831,"tlo":345,"tla":447,"tle":327,"tem":4827,"ten":1364,"teo":775,"tep":103,"tei":263,"tej":504,"tek":2989,"tel":4105,"tef":109,"teg":1181,"teh":652,"tea":89,"teb":91,"tec":139,"ted":342,"th ":436,"tez":323,"tev":3243,"tet":1626,"tes":590,"ter":10610,"ti ":9411,"tho":148,"the":481,"thi":96,"tha":151,"zšl":321,"zši":325,"zše":149,"Živ":99,"yőr":92,"Žel":286,"ža ":604,"Žup":1073,"žko":65,"žle":58,"žlj":64,"žju":249,"žke":66,"žin":1764,"žim":160,"žil":222,"žir":92,"živ":1723,"žit":230,"žis":155,"žja":222,"žje":479,"žji":178,"žic":387,"žig":82,"žij":171,"žič":188,"žnj":265,"žni":1372,"žno":1191,"žna":269,"žne":657,"žo ":93,"žeš":277,"že ":817,"žbi":268,"žbe":388,"žbo":86,"žav":2001,"žba":234,"žaj":196,"žal":97,"žan":339,"žar":1500,"žga":114,"ži ":1033,"žev":1038,"žej":69,"žek":75,"žel":1057,"žem":871,"žen":2078,"žef":104,"ždi":82,"žuž":106,"užb":674,"uže":844,"užu":116,"uži":1380,"užn":1159,"žuj":195,"žup":4080,"žiš":141,"žu ":73,"všč":108,"všk":96,"vše":78,"vši":66,"yír":60,"ušč":304,"ušt":314,"ušn":118,"ušk":172,"uši":260,"uše":213,"uša":205},"n_words":[5788075,6773679,5606921],"name":"sl","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/so.json b/contrib/languages-data/so.json
new file mode 100644
index 0000000..1be8d8b
--- /dev/null
+++ b/contrib/languages-data/so.json
@@ -0,0 +1 @@
+{"freq":{"YO ":13,"jec":34,"jee":32,"D":313,"E":183,"F":66,"G":214,"A":673,"B":249,"C":240,"L":152,"M":367,"N":163,"O":122,"H":180,"I":236,"J":129,"K":173,"U":82,"T":107,"W":226,"V":11,"Q":76,"P":22,"S":486,"R":114,"Y":96,"X":120,"Z":10,"f":458,"g":2154,"d":5233,"e":4497,"b":2102,"c":900,"a":24510,"n":3878,"o":5982,"l":3786,"m":2460,"j":397,"k":2897,"h":3132,"i":6615,"w":2306,"v":27,"u":3829,"t":1545,"s":2871,"r":2895,"q":718,"p":77,"z":23,"y":3607,"x":1698,"jaa":13,"jab":16,"jar":10,"jam":12,"Xam":11,"joo":14,"Xas":10,"jis":14,"jir":95,"jii":13,"jid":17,"jo ":15,"Far":12,"isk":69,"ism":12,"isl":25,"iso":22,"isu":42,"ist":67,"ita":17,"is ":71,"ion":20,"ir ":84,"irs":56,"irt":28,"iro":22,"irk":32,"iri":56,"isi":32,"ish":96,"ise":18,"isb":17,"Wux":23,"isa":134,"ire":16,"ira":131,"iyi":10,"iyo":394,"iya":423,"iye":65,"ixi":16," l":598," m":880,"kii":161," n":189," o":537," h":365," i":795," j":267," k":1328," d":1214," e":328," f":95," g":401," a":1317," b":593," c":361," y":296," x":283," u":599," t":376," w":1834," q":291," p":20," s":807," r":112,"km ":14," J":125," K":142," H":119," I":161," N":93," O":34," L":81," M":322," B":217,"khd":24," C":229,"kha":11," A":275," F":59," G":169," D":236," E":41," Z":10," Y":40," X":90," S":438," R":66," Q":69," P":18," W":211," U":33," T":83,"kee":20,"key":11,"kh ":38,"Web":10,"Waa":56,"ku ":434,"kor":15,"Wax":40,"koo":94,"War":17,"XEE":11,"مد":16,"Gal":22,"و":25,"ي":76,"ف":13,"ق":12,"ل":77,"م":62,"ن":31,"ه":13,"د":46,"ح":26,"ب":37,"ة":21,"ا":98,"أ":11,"ع":29,"ش":21,"س":23,"ر":49,"kar":49,"kas":30,"kan":49,"kal":143,"kam":32,"kad":48,"kac":14,"kab":10,"kaa":81,"ka ":1268,"A ":83," Ga":53," Ge":18,"Da":59,"DU":11,"Cu":18,"Co":13,"DE":11," Fi":17,"Ce":13,"DH":15,"Ci":23," Ha":35,"Du":13,"EY":13," Go":61," Gu":12,"EG":11,"De":45,"EE":45,"EL":14,"Di":29,"Dh":36,"H ":16,"GA":19,"Fa":23," IY":15,"Er":12," Ho":29,"ha ":334," Hi":37,"Ge":18," Ji":25,"Ga":53,"حم":18,"HA":35,"I ":13," Ja":63," KA":16," Is":32," It":29,"GM":12," In":35,"Fi":17,"ham":43,"han":102," Ka":28,"hal":48,"haw":17,"hax":44,"haq":58," Ki":19,"har":45,"has":76," Kh":10," Ju":19,"hah":12,"hab":77,"haa":189,"had":144,"hac":36,"AS":15,"AR":23," MA":17,"AX":27," La":22,"AY":15,"BA":11," Li":11,"C ":10,"AD":43,"AA":51,"AB":14,"AG":11," Ko":23,"AH":23,"hay":333,"AL":37," Ku":26,"AM":13,"AN":35," Ma":180,"Ax":18,"Ar":12,"D ":22,"بن":10," Mi":27,"Ba":101,"CA":15,"Af":65,"بد":10,"he ":25,"Aa":22,"Ab":33,"Ad":10,"Am":17," Lu":25,"Al":38," Ne":14,"Bu":30," Na":32,"Ca":127,"DA":43,"E ":30,"Bi":19,"Be":25,"hda":27,"Bo":30,"Hin":18," Mu":78,"hel":22,"Ku":26,"hee":112,"Ko":23,"hey":26,"hex":72,"Li":11,"N ":26,"her":11,"MA":41,"La":22,"Lu":25,"hi ":27,"Mi":27,"NK":10,"ال":51,"O ":34,"NA":12,"Ma":180,"Mu":79,"Ne":14,"Na":32," Am":16," Al":38,"Nu":16," Af":65,"No":12,"OO":18," Ad":10," Aa":22," Ab":33," Ba":101," CA":12," Ax":18," Ar":12,"hig":23," Be":25,"hid":12," Bi":19,"hin":40,"Go":61,"him":17,"Gu":12," Bo":30,"hii":170," Bu":30,"his":24,"hir":31,"Ha":35," Ca":127,"Hi":37," Ce":13," DE":10," Ci":23,"IN":12,"Ho":29," DH":13,"IS":10," Co":12," Cu":18,"IY":20," Da":59," Di":29," Dh":36,"In":36," De":45,"Is":32,"It":30,"Ja":63,"KA":33," Du":13,"Ji":25," Er":12,"Ju":19,"LA":35,"Ka":28,"Kh":10,"ho ":53,"Har":14,"Ki":19,"LE":16," Fa":23,"gma":64,"go ":32," Xi":13," Xa":51,"UU":11,"yuu":26," Wu":23,"To":11,"Th":10," Wi":15," We":12,"Ta":37," Wa":133,"St":13,"Su":23,"Wu":23,"gob":97,"Wi":16,"Wa":133,"XA":19,"We":12,"XE":12,"Y ":18,"yst":29," Yu":14,"yso":15," Ya":10,"WA":26,"gmo":41,"ysa":93,"Qa":26,"Qo":17," م":12,"RA":10,"S ":18," ع":21," ا":48,"goo":52,"R ":20," ب":13,"gsa":14,"gu ":229,"Si":17,"Sh":86,"gsi":12,"So":180,"Ru":12,"U ":11,"Sa":70,"TA":13,"Re":13,"SH":11,"Ro":11,"yoo":24,"Qu":16,"SA":16,"Ra":20,"gud":22," Nu":16," No":12,"gta":43," Ra":20," Qu":16,"b ":130," Ro":11," Re":13,"guu":20,"gun":12,"a ":5909," Qo":17," Qa":26,"شي":10," Su":23," St":13," Ta":37,"Ya":10," Th":10,"Yu":14," To":11," Ru":12," Sa":70,"Xa":51,"YO":15," Sh":86," Si":17,"Xi":13," So":180," WA":20,"ري":12,"Gob":48," ja":60,"i ":853,"ye ":36,"ian":11," iy":365," ji":127,"ge":93," je":47,"ga":1135,"fk":16,"Ing":16," im":15," in":148," il":54," ii":23,"ic ":14,"fi":49,"fr":45,"fu":47,"ft":29,"fo":18," is":155," ka":688," kh":13,"hd":44,"he":286," ki":46," ke":11,"ha":1580,"gn":11,"gm":108," jo":14,"gl":15,"gi":72,"id ":171,"gu":305,"iba":32,"gt":52,"gs":27,"gr":15," ju":17,"go":196,"du":188,"dw":36,"dy":13,"g ":83," ha":190,"ea":16,"eb":72,"yee":61,"ec":51," he":28,"ed":360,"de":252,"dd":113,"di":494,"dh":632,"dk":189,"dl":33," go":117,"do":234,"dn":22," gu":55,"ia ":36,"ex":102,"ey":554,"fa":110,"h ":441," id":15,"fe":17,"eh":54,"ib ":32,"eg":202," hi":20,"ee":1263,"el":242,"ek":35," ho":120,"ei":12,"yey":26,"en":172,"em":31,"et":26,"es":93,"er":287,"ya ":266,"ca":427," ni":37,"e ":881," ne":15,"bs":21," na":54,"br":36,"bu":104,"bt":55,"bn":18,"bo":234,"bk":30,"bl":13," mu":48,"ig ":10,"bi":355,"bb":15,"bd":41,"be":201,"db":11,"da":2087," og":18,"f ":98,"cy":18," of":16,"cu":41,"ct":11,"cs":27,"co":62,"cm":24,"cn":13,"cl":19,"ci":73," nu":10,"ch":33," no":73,"ce":64,"cd":20,"yad":111,"yag":10," le":91,"c ":51,"yaa":287," la":334,"icm":22," ku":465,"ici":14," km":14,"ica":25," ko":88," me":49,"az":10,"ay":1458,"idu":13," mi":187,"ba":817,"d ":893,"at":134,"as":580,"yd ":29,"ido":43,"ar":1307,"aq":237," ma":590,"ax":1066,"aw":157,"idk":12,"yay":52," lu":25,"ak":76,"al":1647,"idi":35,"yaw":11,"idh":19,"ai":29,"aj":59,"yar":45,"am":590,"an":1951,"yaq":50,"yan":13,"ac":260,"ida":140,"ad":2243,"aa":4171," lo":138,"ab":630,"ag":664,"ah":1152,"yah":134,"af":128,"iib":15,"nu":38,"iic":11,"nt":263," af":45,"ns":59," ah":473," aa":208,"iig":13," ab":31,"iid":50,"no":160,"nn":18," ad":49,"q ":34," am":103," an":18,"iik":48,"iin":164,"ny":57,"yka":17,"iil":93," al":21,"iim":26,"iis":199,"iir":65,"of":78,"iiq":14,"oc":29," ax":10,"od":156," ar":26,"ob":291," aq":21," as":29,"om":340,"on":186," ba":344,"ok":16,"ol":273," ay":246,"og":129,"il ":80,"ot":41,"os":90," bi":107,"op":10,"oo":1738," be":63,"or":236,"oq":49,"yn ":105," bo":34,"r ":475,"ox":10,"ow":125,"oy":128," bu":35,"pa":14," ca":238,"im ":21,"ika":50,"lo":386,"ige":10,"lm":39,"ll":110,"ls":27,"iga":247,"ii ":339,"lw":14,"lu":48,"igi":31,"yo ":488,"ly":56,"igu":13,"igt":12,"o ":2012,"ma":1465,"mb":52,"mh":21,"me":199,"mk":39,"mi":333,"mp":19,"mo":102,"yna":98,"mu":85,"ihi":82,"yni":14,"na":851,"nb":30,"yne":30,"nc":10,"nd":137,"ne":107,"nf":30,"ng":58,"ynt":29,"ni":213,"nk":312,"nl":21,"imo":20,"ju":17,"jo":31," ee":295,"imi":21,"ki":203,"kh":95,"ke":48,"ind":29,"ina":80," fa":48,"yga":15,"ka":1778,"yi ":19,"m ":103," fu":10,"ino":13,"kt":20," fo":12,"ku":558,"int":102,"ins":10,"ko":130,"ine":14,"ing":16," fi":17,"ini":10,"km":16,"ink":82," ge":36,"li":577,"lk":332,"le":352," ga":186,"ld":23,"lg":22,"inu":15,"la":1306,"lb":52,"iny":13,"n ":1478," co":22,"ht":11,"hu":92,"ikh":54," ce":15,"hi":387,"hn":16,"ho":217," ci":36,"ila":160,"id":471,"ic":103,"yin":59,"ib":108,"ia":61,"ih":88,"in ":262,"ig":350," da":424,"if":21,"yih":49,"yig":21," cu":34,"hy":12,"k ":24,"iq":21," do":45,"ilo":13,"ir":438,"is":630,"it":49,"ill":18,"ilk":32,"ix":28,"ilm":12,"ii":1062,"ij":21,"ik":134," de":120,"ili":51,"il":385,"im":170,"in":663,"io":30," di":70,"yir":13," dh":511,"ima":76,"je":69,"ji":178,"iy":896," du":39,"l ":398,"ja":82,"xi":123,"xo":56,"xm":34,"xw":27,"xu":185,"xb":18,"xa":850,"xe":161,"xd":67,"wg":11,"wi":81,"how":15,"wl":60,"wo":26,"wu":102,"hog":13,"y ":1137,"wa":1722,"wd":13,"hoo":55,"we":185,"hor":60," yi":55," yu":13,"uy":12,"ux":164,"uw":34,"uu":720," ye":13,"ve":10," ya":211,"x ":140," xo":33,"uj":15,"uk":28,"ul":200,"uf":20," xi":90,"ug":210,"uh":16,"uq":90,"ur":259,"hna":12," xu":39,"us":114,"ut":54,"um":90,"un":214,"tu":47,"ub":104,"ua":11,"ud":145,"uc":17," xe":16,"w ":59," xa":103,"to":175,"hul":37,"tr":25,"te":120,"ti":246,"th":37,"ta":784,"su":111,"ss":19,"st":173,"sw":12,"sl":47,"sk":106,"sm":25,"so":371,"sr":10,"sc":17,"se":101,"sh":456,"ي ":20,"xme":19,"si":404,"xma":13,"u ":1296,"sa":722,"sb":21,"rr":20,"rs":115,"rt":160,"ru":77,"rw":11,"rx":11,"ry":27,"ro":144,"rn":40,"rm":32,"rl":22,"rk":200,"ri":397,"hu ":11,"rg":35,"re":258,"rd":49,"rc":12,"rb":25,"ra":754,"t ":51,"qu":35,"qs":10,"xoo":44,"qo":163,"IYO":15,"qi":33,"qe":23,"qa":334,"qd":61,"s ":240,"pu":15,"pr":14," ru":12," u ":194," sa":221," se":17," si":157," sh":112," so":259," qu":21,"xya":13," ra":48," re":33,"ن ":17," ro":11," qe":14," qa":168," qo":69," qi":18," oo":464," or":10,"huu":29," wa":1582," we":88," wo":12," wu":102," wi":39," uu":195,"xud":12,"xuu":133,"Hoo":12," tu":36," us":16," ur":10,"م ":11," um":12," un":11," ug":131,"yg":19," ta":231,"ye":133,"yd":48,"ya":998,"yb":27,"xwe":21,"xy":17," su":25,"yu":34,"ys":166," to":18," th":15," ti":62,"yo":522,"yn":280," te":11,"yk":19,"yi":189,"fee":11,"xey":58,"xee":54,"far":32,"fad":21,"faa":24,"Suu":12,"Axm":14,"xir":17,"xis":13,"xil":26,"xii":17,"xid":14,"xig":24,"Sta":10,"xa ":169,"eyb":17,"eya":63,"eys":74,"Tal":11,"eyn":163,"eyo":14,"eyk":10,"xda":51,"eyd":16,"eye":14,"exa":10,"exd":12,"exe":51,"xe ":46,"xar":38,"Ban":18,"Baa":14,"Bad":22,"xam":54,"xan":16,"Bar":23,"xay":166,"xba":16,"xaa":341,"xad":27,"xag":13,"wux":100,"Aas":11,"Shi":22,"She":12,"Sha":50,"ex ":21,"Af ":19,"ey ":159,"er ":103,"es ":21,"eri":33,"ere":30,"era":49,"Afr":32,"esh":28,"esa":10,"ers":11,"ern":14,"ekh":16,"en ":89,"ela":47,"ele":26,"eli":17,"ell":42,"elo":15,"emb":19,"ena":28,"wla":53,"eny":12,"egm":90,"ego":14,"egt":11,"Som":32,"Soo":136,"woq":10,"el ":65,"wda":13,"Buu":11,"Bur":11,"we ":12,"gir":17,"gii":26,"wey":124,"wee":27,"gey":15,"gee":44,"wi ":14,"wis":10,"wii":22,"Sal":11,"gab":12,"gac":45,"gad":26,"DA ":20,"gaa":436,"gar":35,"gay":21,"gal":70,"gan":69,"ga ":388,"San":27,"wa ":22,"Cab":27,"waq":26,"wan":30,"wal":39,"wax":715,"way":45,"Cal":18,"war":52,"was":18,"Car":40,"waa":581,"wad":168,"Bel":10,"fur":37,"Bis":12,"fri":39,"fii":15,"Boo":10,"fka":13,"da ":918,"de ":22,"dad":131,"daa":159,"dab":19,"dal":113,"WAX":16,"dag":65,"dah":101,"dar":51,"dan":291,"dam":39,"day":61,"dax":79,"daw":32,"Cum":10,"dda":74,"dde":11,"ddi":17,"cun":14,"EEY":13,"EEL":14,"EGM":11,"Deg":30,"cyo":15,"uxu":126,"Daa":22,"Dag":10,"Dal":10,"uxa":15,"uun":88,"uul":63,"uum":13,"uug":15,"uud":50,"uux":10,"ux ":12,"uus":29,"uur":74,"uuq":18,"uut":24,"uwa":28,"co ":26,"cma":23,"ush":13,"usi":11,"use":13,"uu ":316,"usu":26,"uso":11,"uti":16,"uta":19,"cod":10,"com":11,"uqa":33,"uqd":36,"ura":37,"ure":10,"uri":31,"urk":17,"urt":32,"uru":37,"ur ":39,"csi":14,"uma":56,"unt":32,"unk":27,"uni":11,"una":85,"cel":30,"uka":13,"cee":17,"uls":10,"ulo":20,"ull":14,"ulk":27,"uli":14,"ule":16,"ula":26,"un ":29,"che":12,"ul ":36,"ciy":12,"cii":28,"uga":40,"ugu":128,"ugs":11,"ed ":184,"ebi":20,"uf ":13,"uda":33,"udi":12,"eb ":12,"udu":37,"ug ":18,"ega":53,"ub ":32,"eek":25,"een":99,"eel":138,"eem":18,"eeb":23,"eeg":65,"eed":229,"eey":113,"eh ":42,"ees":56,"eer":157,"edk":18,"edi":12,"ede":22,"eda":72,"uba":39,"ubb":11,"edu":15,"ud ":36,"edo":11,"ecl":12,"ece":25,"ee ":319,"dwe":25,"dwa":11,"duu":57,"tuu":22,"doo":96,"dow":37,"tri":10,"The":10,"dna":12,"to ":75,"Dhe":14,"Dhu":12,"dun":12,"dul":20,"dug":23,"too":69,"du ":45,"tii":59,"tig":10,"tir":66,"dha":335,"tio":16,"tic":26,"dhu":33,"dib":25,"dhi":112,"dhe":122,"dho":21,"der":19,"dex":18,"dey":16,"dee":48,"deg":96,"den":15,"di ":38,"dle":11,"dla":17,"tee":36,"dku":14,"dki":33,"do ":77,"ter":36,"diy":39,"din":26,"ti ":29,"dir":60,"dis":51,"dig":42,"dii":165,"dil":12,"dka":134,"the":16,"rga":14,"ri ":48,"rge":14,"rey":42,"ree":110,"rda":15,"rdh":16,"re ":77,"rco":10,"rax":25,"ray":99,"rar":15,"ras":44,"rat":10,"rba":11,"rah":41,"ran":54,"ram":17,"rak":12,"rab":82,"raa":165,"rad":87,"rs ":11,"roo":48,"rna":16,"rne":11,"rni":10,"ro ":63,"rma":23,"Nab":15,"rla":13,"rku":10,"rko":10,"rki":41,"rke":18,"rka":117,"riy":58,"ris":28,"rig":31,"rii":110,"rik":46,"rin":21,"ric":16,"rya":13,"rur":10,"run":18,"ruu":10,"ry ":11,"rsi":16,"rsa":63,"rsh":15,"rta":110,"rto":18,"rte":11,"rti":11,"rub":12,"saa":120,"sab":11,"sad":52,"sag":23,"sah":11,"sal":49,"sam":47,"sbi":14,"san":191,"sas":14,"sar":33,"say":43,"sa ":99,"sha":242,"sho":46,"she":41,"shi":83,"si ":68,"siy":42,"sid":91,"shu":10,"sil":13,"sim":38,"sii":82,"sig":32,"se ":61,"sh ":17,"see":14,"sow":16,"som":59,"soo":214,"soc":14,"su ":25,"sla":30,"sku":37,"ska":59,"so ":55,"sma":15,"حمد":15,"ste":15,"sta":66,"sto":28,"sti":41,"sub":11,"suf":12,"sug":13,"sul":11,"suu":22,"tal":42,"tag":10,"tah":87,"taa":194,"tad":13,"tay":60,"tar":33,"tan":31,"tam":13,"te ":13,"ta ":272,"bka":23,"biy":71,"bis":28,"bir":12,"bil":48,"bin":31,"big":38,"bii":37,"bo ":47,"bol":129,"bna":15,"boo":24,"bba":12,"be ":19,"ban":61,"bal":43,"bah":27,"bad":232,"baa":96,"bab":12,"bay":35,"bax":34,"bas":10,"bar":156,"bdi":25,"bdu":11,"bi ":69,"bee":145,"ber":11,"bey":12,"ca ":55,"car":35,"cas":13,"can":24,"cay":13,"cab":20,"cad":53,"caa":145,"cal":33,"cag":16,"bri":13,"bra":15,"bsa":11,"bta":33,"bti":13,"bur":20,"bul":12,"buu":52,"aka":19,"am ":40,"aki":23,"aji":27,"ajo":16,"qa ":12,"al ":136,"ahi":41,"qar":20,"qay":16,"aho":10,"qad":44,"qab":47,"qaa":149,"ahd":20,"qan":14,"qal":17,"ahe":26,"aha":697,"agm":13,"agt":24,"agu":76,"ago":29,"aq ":22,"qdi":38,"qda":17,"any":23,"ano":51,"ann":10,"ant":70,"ans":32,"ane":21,"ang":10," ال":46,"ani":87,"ank":185,"ana":385,"anb":26,"and":92,"amu":23,"amo":10,"amk":32,"amh":19,"ami":82,"ame":93,"amb":16,"ama":257,"aly":20,"qey":14,"alo":160,"alm":17,"all":22,"alk":165,"alg":17,"ali":424,"ald":14,"ale":110,"ala":480,"alb":42,"an ":924,"aba":194,"abd":37,"abe":56,"abi":146,"abk":18,"abo":40,"abt":38,"abu":36,"aca":130,"aab":114,"aac":13,"aaa":15,"aaf":38,"aag":64,"aad":398,"aaj":28,"aak":21,"aah":75,"aan":742,"aal":743,"aam":113,"aas":211,"aar":259,"aaq":41,"aaw":32,"aat":37,"aay":89,"aax":19,"ad ":334,"qiy":15,"ac ":19,"aa ":1110,"qii":10,"ab ":33,"afr":11,"aft":15,"afi":18,"aga":458,"age":12,"ah ":325,"afa":38,"ado":85,"adl":23,"adk":153,"adn":12,"adh":26,"adi":223,"add":96,"ade":66,"ag ":29,"adw":22,"adu":44,"aci":16,"ace":10,"Qar":12,"acd":15,"ada":1138,"af ":19,"acy":15,"acs":19,"qor":48,"qoo":60,"qof":24,"axi":13,"axm":15,"axo":15,"axu":15,"axa":702,"axb":16,"axd":50,"axe":90,"ayi":11,"ayo":52,"ayn":115,"ays":84,"ayu":13,"axy":16,"axw":26,"ayb":10,"aya":151,"ayg":11,"ayd":32,"aye":26,"ba ":84,"qur":24,"at ":11,"arg":25,"are":96,"ard":30,"arb":14,"ara":357,"aro":72,"arn":19,"arm":17,"arl":10,"ark":135,"ari":153,"aru":20,"ars":39,"art":72,"asa":99,"ary":14,"asi":106,"ash":156,"ase":12,"aso":31,"ask":17,"ar ":198,"as ":80,"aqa":111,"aqi":13,"aqo":51,"ax ":98,"awe":20,"ay ":932,"awa":46,"awl":31,"awi":33,"ata":37,"asu":12,"ast":33,"ato":18,"ate":17,"ra ":58,"ati":34,"ngi":20,"ni ":47,"Isl":11,"neh":11,"ng ":11,"nee":16,"nfu":25,"ney":14,"ne ":43,"ndh":18,"ndi":22,"nan":17,"nac":45,"nad":83,"nah":41,"nab":18,"naa":131,"Ito":28,"nbe":15,"nd ":69,"AXE":10,"AY ":10,"nba":11,"AXA":12,"nay":47,"nax":11,"na ":412,"Jab":13,"Jan":13,"Jam":22,"KA ":11,"KAL":10,"nya":38,"AAL":13,"ADA":25,"nuu":21,"nto":13,"nti":37,"nta":176,"nte":24,"nsi":15,"nsa":22,"AHA":14,"noo":67,"noq":18,"nna":11,"ALA":17,"nle":12,"no ":59,"nki":22,"nka":271,"AN ":16,"nii":13,"nih":11,"nig":39,"niy":10,"nis":15,"nim":17,"nin":39,"ogu":24,"oga":60,"Jub":11,"ol ":60,"oco":11,"odi":15,"of ":38,"oda":43,"ofe":10,"LA ":12,"د ":29,"oba":86,"od ":60,"obo":134,"obi":38,"ة ":21,"oyi":94,"oya":10,"owl":29,"ow ":45,"ost":14,"ota":10,"ose":28,"os ":15,"oon":114,"ool":98,"oom":198,"oof":13,"oog":60,"ood":123,"oob":124,"or ":39,"ooy":111,"oow":16,"oot":14,"oos":65,"oor":31,"Koo":13,"ore":44,"ori":14,"osa":11,"ort":21,"oqo":37,"oqd":11,"ora":61,"ola":52,"on ":52,"olk":99,"ole":20,"olo":14,"oly":10,"ona":28,"onf":25,"oni":16,"onk":11,"ons":12,"ont":14,"oma":298,"oo ":749,"omp":12,"la ":241,"le ":159,"laa":281,"lab":61,"lac":11,"lad":232,"laf":10,"lah":96,"lag":116,"lal":23,"lan":88,"lam":27,"las":21,"lay":70,"lba":15,"lbe":31,"kuw":22,"kuu":18,"kun":22,"kul":14,"kto":17,"MAD":13,"lom":11,"loo":176,"lmo":12,"lmi":13,"lma":10,"lsh":13,"Luu":11,"li ":92,"lga":16,"ley":29,"leh":35,"lee":98,"lo ":165,"lla":49,"lle":32,"lka":311,"lki":14,"lis":19,"lin":48,"lim":15,"liy":204,"lid":28,"lia":24,"lib":24,"lil":40,"lii":17,"lig":30,"ma ":133,"maa":361,"mac":36,"mah":24,"mad":229,"mag":226,"mar":193,"mas":14,"mal":133,"man":32,"may":23,"max":25,"mba":26,"mbe":10,"me ":19,"med":68,"mee":72,"mey":24,"luq":12,"luu":17,"مد ":15,"lya":33,"lyo":10,"Mar":22,"Mas":10,"Mag":51,"Mad":20,"Maa":17,"Max":25,"moo":35,"muq":17,"muu":16,"mul":10,"Mux":13,"mhu":20,"Muq":24,"Mud":14,"mi ":19,"min":17,"mil":14,"mis":11,"miy":27,"mig":18,"mid":170,"mij":10,"mii":25,"mo ":60,"mka":33},"n_words":[94077,109135,83288],"name":"so","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/sq.json b/contrib/languages-data/sq.json
new file mode 100644
index 0000000..d14e9b5
--- /dev/null
+++ b/contrib/languages-data/sq.json
@@ -0,0 +1 @@
+{"freq":{"D":2951,"E":2593,"F":3211,"G":3160,"A":5867,"B":4180,"C":1638,"L":2905,"M":4987,"N":3399,"O":1229,"H":1921,"I":3066,"J":1447,"K":6663,"U":1057,"T":3497,"W":518,"V":1916,"Q":766,"P":4792,"S":7666,"R":2957,"Y":251,"X":569,"Z":898,"f":14736,"g":26110,"d":50009,"e":167179,"b":16462,"c":10479,"a":127255,"n":113931,"o":70925,"l":52723,"m":56721,"j":47732,"k":46937,"h":74009,"i":153617,"w":760,"v":25346,"u":51367,"t":143796,"s":94347,"r":123043,"q":14863,"p":40169,"z":11328,"y":8479,"x":1327,"Ë":504,"Ç":673,"Fil":198,"ë":138684,"ç":2147,"Fja":191,"Evr":427,"ο":279,"α":259," l":7602," m":19599," n":36778," o":2746," h":2298," i":13440," j":4180," k":16392," d":18386," e":22497," f":7928," g":6344," a":8088," b":5231," c":3040," z":2178," u":2660," t":29267," v":9770," q":6524," p":21985," s":22399," r":6234," J":1246," K":6396," H":1660," I":2018," N":2979," O":928," L":2729," M":4746," B":3697," C":1374," A":5029," F":3007," G":2987," D":2696," E":2074," Z":866," Y":222," X":455," S":7145," R":2508," Q":712," P":4598," W":483," V":1621," U":882," T":3065,"Gje":385," ç":814,"Gji":334," ë":7724,"Gju":218," Ç":648," Ë":309,"Gja":201,"Fra":312,"A ":555,"For":238,"Da":377,"Co":247,"Ch":193,"Du":229,"Do":262,"Dr":303,"De":624,"Di":445,"Dh":185,"Fe":366,"Fa":366,"Ev":475,"El":231,"Ga":322,"I ":544,"Fr":441,"Fo":342,"Fl":194,"Fj":196,"Fi":584,"BA":277,"C ":177,"Au":418,"Ar":625,"At":240,"Ba":1072,"Af":186,"Am":331,"An":632,"Ai":327,"Aj":203,"Al":611,"Bu":398,"Br":503,"Ca":335,"E ":302,"Bi":247,"Be":668,"Bo":499,"Ku":625,"Ky":228,"Kr":523,"Ko":2407,"Le":440,"Li":756,"La":432,"Lu":465,"Lo":267,"Me":757,"Mi":743,"Ma":1783,"Mu":398,"Mo":476,"Nj":248,"Ni":232,"Nd":207,"Ne":392,"Na":320,"No":435,"Gj":1228,"Gr":581,"Go":208,"Gu":206,"Ha":528,"He":303,"II":206,"Hi":261,"Ho":278,"In":443,"Is":481,"It":205,"Ja":385,"Je":182,"Jo":291,"Ju":322,"Ka":1334,"Kj":278,"Ki":240,"Un":312,"Tu":198,"Tr":315,"To":375,"Pë":415,"Th":341,"Ti":609,"Te":638,"Ta":357,"St":538,"Su":265,"Wi":200,"Vi":295,"Va":291,"Ve":412,"Qe":273,"Pu":185,"Pr":997,"Pe":613,"Pa":1058,"Po":615,"Pi":181,"Kë":300,"Se":537,"Si":523,"Sh":3256,"Sk":218,"Sp":262,"So":375,"Ru":267,"Rr":230,"Sa":579,"Re":810,"Ri":264,"Në":821,"Ro":386,"Ra":365,"Gre":313,"b ":407,"a ":28448,"Xh":218,"i ":33796,"ge":656,"ga":7640,"fj":347,"fl":382,"fi":3904,"fs":1052,"fr":816,"fu":1400,"ft":1033,"fo":1696,"bë":2042,"j ":4723,"hf":238,"he":19006,"ha":4203,"gl":638,"gj":7884,"gi":492,"gu":1606,"gr":2543,"cë":728,"go":1368,"du":2130,"dy":756,"g ":1263,"ea":970,"eb":315,"ec":617,"ed":3466,"de":5126,"di":8348,"dh":17111,"dj":1045,"dm":281,"do":4139,"ds":204,"dr":2528,"ew":200,"ex":233,"eu":592,"ev":3916,"ez":1764,"fa":1842,"h ":2770,"fe":1185,"eh":617,"eg":2448,"ef":474,"el":4768,"ek":4482,"ej":2627,"ei":482,"ep":1848,"eo":855,"en":13277,"em":3767,"et":19558,"es":10616,"er":15019,"eq":1060,"ca":1358,"e ":72317,"br":1628,"bu":1107,"bo":1748,"bj":242,"bl":1253,"bi":2183,"be":1978,"da":2581,"f ":429,"co":479,"ck":313,"ci":4788,"ch":654,"ce":1081,"c ":318,"az":1629,"ay":236,"ba":3411,"d ":2270,"at":11806,"as":8047,"ar":20865,"aq":1562,"av":2909,"au":845,"ak":4327,"al":8585,"ai":935,"aj":4288,"ap":2172,"am":3780,"an":16760,"ac":1088,"ad":3850,"ab":980,"ag":1063,"ah":884,"ae":194,"af":1439,"nu":1486,"nt":6015,"ns":1390,"jë":8682,"no":2581,"nn":299,"q ":809,"ny":268,"nx":351,"oe":253,"of":894,"oc":773,"od":2400,"oa":235,"ob":841,"om":4147,"on":11462,"ok":1665,"ol":4317,"oi":918,"oj":2100,"og":2207,"oh":3303,"ot":2941,"os":4887,"ov":2272,"ou":371,"op":2168,"oo":285,"kë":4598,"or":15900,"oq":456,"r ":22499,"oz":751,"pe":3763,"pa":6781,"pl":799,"lë":3066,"po":4293,"pi":2577,"pj":1679,"lo":4348,"hë":4530,"lm":750,"ll":8126,"ls":381,"lu":2680,"lt":1030,"ly":235,"o ":4990,"ma":6825,"mb":3911,"me":13242,"iç":252,"mi":11206,"mj":687,"mp":1128,"mo":2299,"mr":960,"mt":825,"ms":189,"mu":2606,"p ":820,"na":4803,"nc":1647,"nd":14065,"ne":6011,"nf":360,"ng":8085,"ni":9080,"nj":10024,"nk":623,"jy":374,"jv":200,"jt":2182,"ju":2055,"jr":203,"js":193,"jn":989,"fë":409,"jo":2391,"jm":201,"kj":232,"ki":2183,"ke":4651,"ka":7856,"m ":4929,"ky":261,"ks":1435,"kt":3378,"ku":6445,"ko":7127,"gë":1191,"kr":3597,"kl":551,"km":406,"kn":219,"li":13297,"lk":295,"lj":608,"le":5962,"ld":260,"lg":209,"la":6555,"lb":400,"n ":21363,"hr":608,"hs":268,"hp":932,"hq":3267,"hv":498,"ht":18411,"hu":3735,"hj":1397,"hk":4243,"hi":5167,"hn":261,"ho":1827,"dë":2976,"hm":1740,"id":1902,"ic":1580,"ib":803,"ia":5265,"ih":920,"ig":1500,"if":611,"ie":1546,"hy":320,"k ":3634,"iq":788,"ir":3308,"is":14282,"it":23622,"iu":865,"iv":2165,"ij":2183,"eç":454,"ik":8974,"il":7041,"im":10832,"in":18975,"io":3678,"ip":5685,"je":13383,"ji":4180,"iz":2757,"l ":3714,"ja":7433,"xh":647,"të":37539,"z ":666,"së":6850,"y ":1757,"wa":196,"vl":237,"vj":627,"vi":5977,"vr":700,"rë":7665,"vo":819,"uz":609,"uv":188,"ve":12217,"vd":565,"va":1908,"x ":256,"ui":282,"uj":950,"uk":1866,"ul":3717,"ue":2301,"uf":993,"ug":1106,"uh":1399,"uq":312,"ur":9836,"us":3067,"ut":2692,"um":2889,"un":3916,"që":4381,"up":1314,"ty":1124,"tu":7405,"tt":246,"ub":1079,"ua":6884,"ud":1227,"uc":369,"w ":197,"pë":9674,"to":7414,"tm":513,"ts":310,"tr":5518,"te":16922,"tj":1903,"ti":17731,"th":4122,"v ":372,"tb":272,"ta":11255,"su":1269,"sv":254,"ss":508,"st":7650,"sy":189,"sl":488,"sk":843,"sn":215,"sm":700,"sp":784,"so":3394,"sq":194,"sc":195,"se":5448,"sh":35309,"sj":384,"si":11685,"u ":4144,"sa":3279,"rr":4311,"rs":2938,"rt":4337,"ru":3266,"rv":415,"ry":2015,"rq":210,"rp":488,"ro":7859,"në":24491,"rn":818,"rm":3136,"rl":428,"rk":1734,"rj":983,"ri":20960,"rh":205,"rg":2062,"rf":941,"re":16088,"rd":2048,"rc":625,"rb":1536,"ra":14927,"t ":30601,"qy":921,"qu":769,"më":7228,"qj":229,"qi":4293,"qe":2941,"qa":297,"s ":14264,"pt":2183,"pu":2168,"pr":4557,"ps":322,"zë":930,"zg":391,"zh":1019,"zi":2576,"zb":204,"ze":1031,"za":1391,"zy":222,"zu":773,"zo":1184,"vë":1622,"zj":202,"zm":366,"ye":1323,"yt":1297,"ys":927,"yr":1681,"yp":187,"yn":208,"ym":190,"yl":202,"Art":179,"Aut":217,"Bas":348,"Ai ":287,"Ame":229,"Ber":210,"Bot":188,"Ës":295,"Çm":287,"アアア":185,"ër":16534,"çë":180,"ëp":625,"ëm":1771,"ën":7250,"ël":747,"ëz":648,"ëv":1509,"ës":19082,"ët":3696,"ëh":461,"ëj":261,"ëd":282,"ë ":85317,"çm":262,"çi":247,"çe":269,"ça":442,"ç ":230,"Nob":257,"Per":268,"Pas":194,"Par":336,"Pro":243,"Pri":459,"Pre":232,"Ish":253,"Ita":196,"ア":259,"Jug":202,"Ka ":178,"Kal":215,"Kar":206,"Kjo":277,"Kon":248,"Kom":428,"Kos":1014,"Kor":248,"Ky ":207,"Lig":189,"Mal":292,"Mar":396,"Maq":235,"Mad":210,"Min":180,"Ësh":295,"çmi":226,"ëhe":440,"ëm ":968,"ël ":264,"ëll":356,"ën ":3829,"ënt":362,"ëng":554,"ënd":1225,"ëna":195,"ëmi":309,"ëmb":216,"ëdh":247,"Sta":228,"Shk":524,"Shq":1268,"Sht":446,"Ser":197,"Rep":364,"Në ":708,"Uni":285,"The":241,"Tir":480,"Për":406,"çan":218,"bje":222,"bis":202,"bim":300,"bin":241,"ble":244,"bli":768,"bol":404,"bot":888,"be ":208,"ban":768,"baj":338,"baz":296,"bas":738,"bar":627,"bi ":668,"ber":314,"bel":434,"bet":408,"ca ":491,"cav":177,"cak":266,"ce ":197,"bri":501,"bra":276,"bre":593,"bur":304,"bul":254,"aka":205,"am ":292,"ake":288,"aki":184,"afë":206,"ajo":282,"ajt":819,"al ":1170,"aja":239,"ak ":563,"ahi":215,"aj ":1938,"agj":197,"ago":189,"ano":603,"ant":917,"ans":360,"ane":1253,"ang":643,"ani":2814,"anj":226,"ana":956,"anc":511,"and":1227,"amu":220,"amp":346,"ami":877,"ame":814,"amb":181,"ama":345,"alt":205,"alo":479,"all":1341,"ali":1967,"ale":1756,"ala":461,"an ":2721,"aks":181,"aku":485,"akt":1546,"ako":591,"aba":207,"abe":271,"aft":183,"afi":534,"ai ":361,"adm":223,"adh":1978,"adi":505,"ade":333,"aci":658,"ada":241,"azi":334,"aze":285,"azh":332,"atë":1708,"azë":204,"at ":2869,"arg":323,"are":2127,"ard":768,"ara":2483,"aro":307,"anë":3784,"arm":193,"arl":201,"ark":631,"arj":231,"ari":2009,"aru":212,"arr":782,"ars":590,"art":1488,"asa":351,"asi":536,"ash":2587,"ar ":6112,"apa":246,"alë":446,"apo":1058,"as ":2094,"aqe":844,"aqi":214,"aqj":186,"amë":284,"ava":276,"aut":366,"arë":1850,"avi":245,"ave":2116,"ata":707,"asu":211,"ast":1173,"atr":379,"ato":1090,"apë":205,"ate":982,"ati":2851,"atu":362,"aty":276,"アア":222,"jed":481,"jeo":227,"jer":1714,"jek":498,"jel":290,"jen":1698,"jes":2300,"jet":2586,"jev":418,"ji ":371,"jat":998,"jas":292,"jar":261,"jal":704,"jak":272,"jan":2402,"je ":2467,"joh":964,"jon":213,"fër":208,"jit":1409,"jis":586,"jim":228,"jin":428,"jik":319,"jih":193,"jo ":859,"ito":557,"ipë":1386,"itu":2464,"iud":180,"iso":211,"ist":3511,"iv ":257,"ita":1245,"ite":1604,"ith":1252,"iti":3696,"itj":291,"irë":662,"iut":194,"iva":248,"ivi":385,"ive":1169,"ilë":642,"ipt":1558,"ipi":226,"is ":537,"ion":2522,"ikë":1269,"ior":219,"ipa":1258,"ipe":572,"ir ":232,"inë":1756,"iro":194,"iri":583,"isi":327,"ish":5381,"ise":279,"isa":571,"iu ":213,"imë":180,"ire":227,"ira":796,"it ":11180,"ja ":2116,"itë":896,"isë":2854,"izu":355,"izo":367,"izm":294,"izi":684,"iza":492,"kim":794,"kin":237,"kip":221,"kis":422,"km ":346,"ken":688,"ket":291,"ke ":3158,"kra":414,"kre":377,"kt ":208,"kry":877,"ku ":990,"kro":261,"kru":297,"kri":1336,"kov":192,"gët":303,"kos":230,"kor":588,"kon":2239,"kom":1483,"gël":243,"kol":575,"koh":623,"kod":386,"gë ":240,"ko ":239,"kla":195,"jtë":442,"jut":199,"jtj":180,"jtu":499,"jua":282,"juh":823,"jug":449,"jta":181,"jti":303,"jnë":775,"jt ":307,"kat":718,"kar":404,"kas":189,"kan":1846,"kal":946,"kam":249,"kak":195,"ka ":2506,"jyr":194," Ga":321," Fo":338," Fr":441," Fi":583," Fl":194," Fj":187," Ha":528," He":302," Go":206," Gr":579," Gu":204," Gj":1228," Ho":278,"ha ":1068," Hi":261," Je":182," Ja":384," Is":478," It":205," In":442,"ham":267,"han":292,"hap":354," Ka":1328," Ki":237,"har":380," Kj":278,"has":365,"hat":909," Jo":291," Ju":322," La":426," Le":440," Li":748," Ko":2406," Kr":523," Ku":622," Ky":226," Ma":1780," Mi":739," Me":757,"he ":11876," Lo":267," Lu":464," Nd":206," Ne":390," Na":320," Nj":246," Ni":229," Mo":474," Mu":395,"hek":391,"hel":179,"hej":224,"het":2896,"hes":233,"her":715,"heq":210,"hen":1147,"hem":522,"hfa":178,"hi ":531," Am":329," An":631," Al":605," Ai":327," Aj":202," Af":185," Ba":1069," Au":417," At":239," Ar":624," Be":668," Bi":243,"hin":747,"him":565,"hil":181," Bo":497," Br":498," Bu":397,"his":436,"hit":1517,"hir":335," Ca":327,"hje":1227," Ch":191," Co":245,"hka":488," Da":377," Di":443,"hke":445," Dh":185," De":621,"hki":188," Dr":303,"hkr":988," Do":261,"hko":663,"hku":970," Du":229," El":231," Ev":475," Fe":366,"dë ":206," Fa":359,"cë ":308," Xh":218,"gli":277," Wi":198,"gjë":270,"gon":186,"cës":254,"gos":180,"gor":539," a ":180,"gru":420,"gra":1045,"gri":269,"gre":682," Kë":300," Po":615," Pi":181,"gul":267," Pe":612," Pa":1054," No":435," Ra":364," Në":821," Ro":384," Re":809," Ri":264," Pr":996,"gur":313,"gus":310," Pu":185," Qe":273," Su":263," St":520," Ta":356," Th":340," Ti":608," Te":634," Tr":311," To":373," Pë":415," Rr":230," Ru":266," Sa":578," Sh":3248," Si":519," Se":536," So":373," Sp":261," Sk":217," Va":291," Ve":411," Vi":293," Tu":194," Un":311," ja":2600,"ial":691,"ian":1176," je":756," in":1156," is":1788," it":245," ka":3973,"ibr":249," ki":698," ke":222," jo":218," ju":582," ha":507," he":538," gj":3833," gr":1301,"ia ":2790," gu":290," hi":627," dë":296," ho":209," hu":257," nj":8261," ni":252," ng":5742," nd":3078,"iel":198," ne":1135,"ien":214," na":398,"ier":413," mu":1092," mo":751," of":214,"ifi":230," nu":833," no":407," le":585," li":3581," la":1142," ku":2373,"ici":344,"ich":192," ky":232," km":370,"ie ":357," kl":275,"ica":291," kr":2025," ko":4104," me":6491," mi":1023," mj":371," ma":2979," mb":1927," lu":944,"idi":290,"idh":520,"ide":545," ll":324," lo":338," af":314," ad":323," am":508," an":1204," ap":925," ai":237," aj":220," ak":811," al":253," au":397," ar":1194," at":742," as":478," ba":1617,"il ":230,"ija":200," bi":487,"ije":259," be":279," bo":1013," bu":403,"iju":269," br":668," ca":214," e ":17735,"im ":2164,"eça":298,"ika":1647,"iga":202,"igj":563,"igu":244,"icë":252,"ij ":910,"ihe":650,"ik ":1689,"imo":407," et":343," es":291," en":262," em":840," el":514,"ime":1522," ek":723,"imi":5114,"ip ":230," fe":444,"ind":2986,"ina":1004," fa":908,"imt":634," fu":1154," fs":638,"ino":317,"ijë":223," fr":355,"int":466," fo":1164," bë":626,"ins":260,"inf":213,"ine":1441," fl":312,"ing":422," fj":334," fi":2335,"inj":211,"ini":1146,"iq ":474," ga":620,"inu":257," i ":9701,"iko":537,"iki":386,"ike":2366," ci":2530,"ila":1202," da":481,"in ":7905,"ikt":220,"iku":640," do":799,"ilo":291,"ill":1867," dr":678,"ilm":536," de":1917,"ilj":319,"ili":1491," di":2033," dh":10408,"ile":229,"ima":293," ed":1307,"io ":303," du":817," dy":651,"hpr":194," vë":300," zo":209,"hpe":278," zy":211," za":245,"dës":713," zb":179,"hkë":454," zh":535,"hoq":360,"hor":539,"dër":1831," zg":360," të":19747,"hme":1293,"hul":218,"hua":438,"hty":181,"htu":748,"htr":565,"htm":182,"hti":611,"htj":309,"hte":3639,"hta":810,"hsh":200,"hro":441,"ht ":2187,"hqi":3061," ru":213," rr":1833," u ":1417," sa":775," se":1591," si":4496," sh":10305," sk":290," sp":362," so":432," qu":416," qy":665," t ":287," ra":1563," re":1537," ri":358,"htë":9010," në":16417," ro":441," pu":708," pr":3527," qe":1420," më":4730," os":1270,"hum":1047," kë":1907," or":779,"hur":1522," pe":1738," pa":4231,"hvi":461," pl":373," lë":638," po":2196," pi":398," pj":1282," së":2396," va":567," ve":3244," vd":524," vo":419," rë":271," vi":3777," vj":545," vl":199," ty":371," tu":272," us":358," që":3732," uj":303," ta":476," st":879," su":385," tr":1582," to":411," pë":7156," th":1109," ti":1065," tj":633," te":3252,"fes":283,"fer":357,"faq":704,"fam":375,"fak":216,"ezë":244,"ez ":343,"etë":2083,"ezu":183,"eza":184,"ezo":208,"eze":303,"ezi":227,"eta":1185,"ete":1425,"etj":331,"eti":2581,"eth":892,"eso":355,"est":534,"esv":178,"eto":1111,"etr":567,"etu":501,"eve":3023,"erë":1553,"evi":375,"esë":1317,"epe":182,"er ":1621,"eor":217,"ekë":226,"eqi":478,"es ":2822,"epu":459,"elë":206,"epr":500,"eri":4116,"erg":266,"ere":548,"era":1561,"erb":418,"et ":8412,"emë":345,"esm":180,"esh":1754,"esi":1453,"ese":746,"eu ":239,"esa":635,"err":624,"ert":290,"ers":1252,"ern":432,"erm":871,"enë":512,"ero":390,"eki":248,"ekn":209,"eko":404,"egë":220,"eks":618,"ekt":1016,"eku":676,"en ":3312,"ela":426,"ele":1238,"eli":682,"ell":702,"elu":266,"emb":225,"ema":440,"eme":707,"emo":232,"emi":771,"emr":507,"ene":497,"ena":430,"end":3296,"enc":779,"eno":271,"eni":647,"enj":342,"ens":203,"ent":2477,"eog":217,"egj":763,"ego":445,"ej ":1063,"egu":355,"ek ":360,"eja":234,"el ":689,"ejt":732,"eka":189,"em ":322,"gju":704,"gjy":338,"gje":2061,"gji":3096,"gja":1196,"gim":207,"gaz":249,"gar":602,"gat":279,"gan":669,"ga ":5443,"ftë":453,"fus":270,"fut":207,"fun":698,"fra":187,"fri":223,"fsh":1041,"bër":688,"for":1268,"bët":491,"bëh":244,"fil":1245,"fik":505,"fin":217,"fis":186,"fit":614,"fiz":441,"fja":339,"da ":488," Çm":287,"de ":479,"dal":409,"dat":201,"dar":567,"dan":203," Ës":295,"ces":191,"ci ":188,"cia":487,"cil":2535,"cio":782,"ean":206,"eat":241,"ega":205,"edh":1894,"edi":471,"ede":284,"eda":183,"edo":295,"eci":308," çm":240,"dyt":203,"dy ":376,"dur":609," ës":7645,"dor":1271,"don":593,"dom":180,"dok":214,"dos":301,"dmi":238,"dod":448,"duk":778,"dua":191,"dri":384,"dra":292,"dre":608,"dry":602,"dro":361,"dha":308,"dhu":653,"dia":304,"dhj":875,"dhi":1463,"dhe":12079,"der":1468,"des":314,"det":819,"dh ":431,"deg":199,"del":210,"dek":243,"den":513,"dem":246,"di ":1814,"do ":585,"dhë":1093,"dje":856,"dim":1054,"din":518,"dio":362,"diq":424,"dis":945,"dit":1503,"dik":388,"rga":634,"ri ":4362,"rgj":700,"ret":2035,"res":1012,"rev":322,"rez":402,"rfa":431,"rbë":484,"rfs":234,"rg ":266,"rea":282,"rej":1398,"reg":1272,"reh":210,"rem":230,"ren":1060,"rek":369,"req":525,"rdo":695,"rdi":283,"rdh":548,"re ":5892,"rca":229,"rd ":234,"rap":224,"raq":331,"ras":547,"rat":1584,"rav":877,"rbi":290,"rba":177,"rbe":317,"raj":258,"rah":340,"ran":2020,"ram":546,"ral":554,"rak":530,"rab":220,"raf":580,"rad":1584,"rs ":444,"ror":1129,"ros":215,"nës":866,"nët":322,"rot":314,"rom":357,"ron":1148,"nën":960,"rop":794,"rov":207,"rod":677,"roc":243,"roj":419,"roi":191,"rol":305,"rok":233,"rof":239,"roh":382,"rog":307,"rne":248,"rmo":182,"rmu":349,"ro ":340,"në ":21951,"rmb":202,"rma":1098,"rme":349,"rmi":261,"rku":392,"rko":294,"rki":195,"rke":238,"rja":228,"rje":742,"riz":365,"rip":239,"rio":403,"rit":2857,"ris":2678,"riv":179,"riu":462,"rih":296,"rig":343,"rij":533,"ril":452,"rik":1578,"rin":1760,"rim":1881,"ria":1506,"rie":292,"rk ":231,"rtë":586,"rye":967,"rue":227,"rur":358,"rup":598,"rus":297,"rve":228,"rrë":463,"rsi":760,"rso":384,"rsa":288,"rsh":704,"rse":194,"rta":523,"rto":212,"rte":551,"rth":191,"rti":1091,"rua":870,"rtu":307,"rt ":743,"rmë":369,"rri":1024,"rrj":335,"rre":1446,"rra":326,"rru":264,"saj":699,"san":214,"sat":207,"sa ":1426,"rys":615,"sha":1190,"shm":1425,"sho":753,"shp":846,"shq":1997,"shr":422,"sht":17551,"she":1082,"shf":232,"shi":1551,"shk":3709,"si ":3797,"sje":269,"sid":277,"sia":412,"shu":1255,"sit":1207,"sir":214,"sis":1576,"sip":1484,"sin":587,"sio":712,"sim":613,"sik":262,"se ":3292,"ser":504,"set":189,"sh ":1239,"sen":315,"spo":232,"spe":251,"sot":202,"sov":1143,"son":618,"sor":814,"st ":714,"shë":1713,"sla":211,"ske":209,"sma":252,"sme":323,"stë":596,"sse":192,"ste":1041,"sta":1312,"sto":730,"sti":1254,"stu":543,"str":1367,"sua":229,"sue":178,"sur":369,"sve":238,"taj":547,"tal":816,"tav":260,"tat":899,"tas":209,"tar":4736,"tan":685,"te ":5516,"tbo":238,"ta ":2326,"pa ":427,"pe ":427,"par":1922,"pat":286,"pas":2622,"pak":267,"pan":389,"pi ":351,"per":1941,"pet":184,"pes":328,"pla":231,"plo":258,"pje":1425,"pju":181,"pia":517,"pik":379,"pin":244,"pio":290,"pit":346,"poz":244,"lër":206,"por":810,"pop":489,"lëv":248,"lët":262,"lës":678,"pos":377,"lën":745,"pon":243,"pol":674,"lë ":644,"po ":983,"pta":1447,"pub":598,"pti":317,"pto":261,"pra":544,"pri":797,"pre":1386,"pro":1746,"pun":526,"pul":588,"qar":179,"qe ":553,"qet":282,"qev":185,"qer":357,"qen":825,"qed":256,"qi ":290,"qit":232,"qip":2962,"qis":408,"më ":5423,"mës":389,"mër":628,"mën":311,"qua":358,"quh":192,"qyt":799,"ra ":3887,"ncë":386,"ngj":549,"ngl":339,"ngu":279,"ngr":213,"ni ":1941,"nge":207,"nga":5420,"ndë":2245,"nen":1014,"ner":340,"net":653,"nes":253,"nev":297,"ng ":438,"nez":219,"nci":280,"nce":340,"nca":359,"ne ":2694,"ndu":806,"ndr":1255,"ndo":1605,"ndj":868,"ndi":3173,"nde":1242,"nda":1141,"nal":768,"nar":589,"nd ":1287,"nav":195,"nat":905,"nas":219,"na ":1164,"ntë":218,"nxh":177,"nuk":387,"num":496,"nua":315,"nto":514,"ntr":261,"nti":1877,"nta":460,"nte":1912,"nst":368,"nsi":197,"nt ":562,"nom":633,"non":199,"jës":1000,"jër":345,"nor":452,"nov":244,"një":7204,"jë ":6765,"no ":251,"ngë":383,"nji":251,"nje":980,"nja":288,"njo":1016,"nie":218,"nic":209,"nia":391,"niz":506,"niv":445,"nis":1563,"nit":1162,"nim":624,"nin":724,"nik":917,"ogr":753,"ogj":624,"oi ":722,"ohu":860,"ohe":1613,"oj ":252,"ojn":599,"oje":249,"oja":193,"ol ":185,"oci":316,"odh":931,"ode":218,"odi":416,"of ":206,"odu":227,"ofe":213,"obe":363,"otë":804,"ovë":1008,"ozi":268,"otu":316,"oti":443,"ote":245,"oto":318,"opë":263,"ost":299,"ota":201,"osh":522,"ose":1363,"oso":1152,"ovi":292,"orë":1117,"ova":484,"ove":273,"oqë":260,"opo":219,"opi":438,"ope":206,"os ":591,"opu":565,"kën":763,"okë":307,"or ":3887,"kët":1025,"kës":1506,"kër":189,"orm":1202,"onë":412,"orr":382,"orc":230,"ord":255,"ore":3570,"org":551,"ori":2679,"ort":629,"oru":235,"ot ":268,"ora":318,"ola":251,"on ":2800,"oli":1087,"oll":1071,"ole":276,"olo":874,"ohë":394,"oka":403,"ogë":224,"oku":205,"ona":1138,"ond":187,"one":812,"onj":684,"oni":2539,"ojë":384,"ono":475,"ons":340,"ont":1418,"oma":638,"kë ":735,"ome":460,"omb":682,"omi":782,"omp":538,"omo":183,"omu":542,"la ":1588,"le ":1504,"lan":809,"lam":463,"lar":790,"lat":987,"las":448,"lav":342,"lba":191,"kut":329,"kus":201,"kur":1434,"kup":473,"kun":397,"kul":958,"ky ":230,"kth":196,"kte":440,"ksi":671,"kuf":336,"kua":615,"ktr":411,"ktu":532,"kti":608,"kto":809,"llë":931,"lon":395,"hën":1076,"hëm":639,"lor":960,"hër":431,"log":828,"loj":558,"loi":197,"hës":946,"lot":202,"lmi":262,"ltu":283,"lua":941,"luf":367,"lue":177,"lsi":217,"li ":2003,"lez":179,"lev":346,"les":310,"let":699,"ler":367,"lem":300,"len":674,"lek":623,"lls":213,"llu":548,"hë ":1014,"lla":1048,"lle":383,"lli":2212,"llo":1314,"lm ":208,"lje":416,"ll ":881,"lja":178,"lit":1752,"lis":1010,"lir":324,"lio":179,"lin":3413,"lim":1223,"liz":412,"lic":188,"lid":353,"lia":455,"lib":286,"lik":872,"lig":270,"ma ":556,"maj":406,"mak":224,"mad":655,"mar":852,"mas":295,"mal":684,"man":1367,"mat":1185,"mba":779,"mbl":257,"mbi":849,"mbe":401,"mbr":294,"me ":7511,"mbu":225,"med":203,"met":1333,"mev":393,"mes":776,"mer":1044,"mel":355,"men":1333,"mbë":676,"lum":373,"mpj":180,"mpi":353,"ëtë":434,"mpo":182,"mon":405,"mor":732,"mos":301,"mri":623,"mra":263,"mua":310,"mta":664,"mur":234,"mul":232,"mun":1237,"muz":308,"ës ":6600,"ëpi":229,"ër ":5836,"mi ":2352,"ëse":300,"ësa":205,"ërt":763,"ërs":761,"ërp":333,"ënë":345,"ëro":630,"ërr":212,"mje":546,"ërk":679,"ërm":698,"ërg":545,"ëri":1847,"min":2761,"ërf":795,"mil":488,"mim":764,"ërd":700,"mir":472,"ëra":457,"mis":535,"ërb":763,"ët ":1589,"mit":2427,"mik":541,"mij":213,"ëti":364,"ëto":188,"ëpë":198,"ëta":971,"ësh":8836,"ësi":2416,"ëso":446,"ësu":202,"ërë":456,"ëve":1277,"zua":506,"zyr":215,"zgj":372,"zi ":247,"zet":248,"ze ":310,"zan":199,"zak":278,"zat":320,"vës":631,"zon":428,"zoh":205,"vë ":592,"zmi":236,"zhv":458,"zim":386,"zik":591,"zis":248,"zit":331,"yrë":392,"yte":900,"ysh":673,"yrt":226,"yra":227,"yre":469,"za ":311,"ytë":280,"yes":421,"yer":268,"yeq":202,"të ":32538,"tëp":239,"tër":1782,"tët":229,"tës":1610,"tëv":188,"tën":440,"tëm":288,"xha":211,"xhi":228,"Çmi":287,"së ":5936,"sëm":198,"sën":372,"vro":528,"veç":250,"vil":586,"vin":191,"vic":177,"viz":481,"vit":3504,"vis":321,"vje":622,"rë ":3679,"vog":268,"rën":1295,"rët":460,"rës":1368,"rëv":444,"rëz":253,"vep":456,"ver":1289,"vet":682,"ven":1374,"vel":341,"vdi":416,"ve ":7283,"val":324,"var":488,"va ":428,"uzi":400,"urë":434,"ush":1610,"usi":271,"ust":434,"uti":207,"ute":336,"utb":206,"uto":619,"us ":390,"umë":772,"ut ":820,"ura":1239,"ure":247,"urg":207,"uri":1038,"uro":455,"unë":434,"urr":247,"urt":615,"qër":295,"qës":313,"ur ":4520,"upi":325,"upt":364,"umr":331,"umi":458,"uma":183,"umb":309,"ume":412,"që ":3460,"uni":521,"ujë":192,"und":1633,"una":206,"up ":226,"uku":221,"ukt":232,"uke":519,"ult":579,"uhë":641,"ull":1873,"uli":199,"ula":307,"un ":316,"uk ":422,"ujt":177,"ugo":283,"uft":506,"uhe":480,"uha":223,"udh":352,"udi":562,"ues":1871,"ufi":398,"ug ":248,"ua ":660,"uar":4550,"ual":287,"uan":427,"ubl":681,"uaj":694,"tyr":784,"tur":3146,"tul":217,"tua":1367,"tud":541,"tue":838,"tre":747,"tra":1541,"tri":1729,"tru":612,"tro":732,"tu ":717,"tme":342,"to ":593,"pë ":222,"toj":273,"toh":424,"pës":324,"tom":222,"ton":869,"tok":323,"tol":198,"për":8954,"tor":3712,"tij":916,"til":200,"tik":1967,"tit":3342,"tis":395,"tin":4094,"tim":1479,"tio":325,"thu":272,"tia":238,"tiv":962,"tje":1286,"tja":349,"thë":407,"tem":772,"ten":475,"tek":589,"tel":426,"th ":586,"tev":1136,"tet":4737,"tes":527,"ter":1775,"ti ":2991,"tho":197,"ths":192,"the":828,"thi":467,"tj ":257,"tha":760,"BA ":261,"zë ":417},"n_words":[1760559,2076420,1518161],"name":"sq","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/sr.json b/contrib/languages-data/sr.json
new file mode 100644
index 0000000..4164f4e
--- /dev/null
+++ b/contrib/languages-data/sr.json
@@ -0,0 +1 @@
+{"freq":{"Зид":109,"Зим":114,"Зла":429,"­":109,"²":236,"í":92,"é":175,"á":86,"ó":108,"đ":1104,"ā":126,"Штр":168,"ć":1891,"Што":91,"Č":99,"č":4854,"Шта":397,"Ште":107,"Зај":154,"Зап":1249,"Зал":315,"Зак":122,"Зах":97,"Зас":107,"ž":2132,"Зав":113,"Зад":197,"Заг":404,"Š":118,"š":2843,"Шпр":125,"Зво":126,"Зве":122,"Збо":90,"Зем":492,"Зел":167,"Зег":102,"Шум":153,"Шко":107,"Шиб":265,"Шир":110,"Шпа":200,"Шле":1282,"Шма":86,"Зор":122,"Шва":417,"Шве":146,"Шал":83,"Шап":121,"Шам":410,"Шат":197,"Шар":556,"μ":130,"ν":216,"ο":340,"ι":229,"κ":120,"λ":186,"δ":101,"ε":168,"η":99,"α":396,"Шен":174,"Шер":106,"γ":103,"ά":102,"Шек":94,"Зуб":86,"σ":141,"ς":235,"ρ":204,"π":100,"τ":159,"ш":94950,"ф":29785,"х":46445,"ц":68619,"ч":65628,"р":436156,"с":355754,"т":321024,"у":311365,"џ":1310,"љ":35971,"ј":308122,"ћ":19548,"њ":47495,"Или":225,"ђ":11394,"Ј":17149,"Љ":527,"Њ":798,"Ћ":246,"Џ":1148,"Ђ":626,"И":5904,"Л":8844,"К":19940,"Н":11262,"М":15719,"П":32017,"О":9935,"Б":27505,"А":14540,"Г":10785,"В":12826,"Е":5430,"Д":13309,"З":6537,"Ж":1460,"Ш":7137,"Т":9123,"У":5062,"Р":18801,"С":31653,"Изр":126,"Ц":3268,"Ч":1717,"Ф":11461,"Х":16282,"л":188879,"к":286042,"и":710913,"п":216690,"о":641802,"н":481173,"м":187091,"г":133308,"в":226917,"б":84282,"а":837777,"з":105986,"ж":36683,"е":690251,"д":236545," ć":228," č":635," Č":97,"Инд":349,"Инс":87,"Инт":201,"Има":166,"Име":214,"Имп":88," ž":304," Š":118," š":593,"Ива":318,"Иак":105,"Изе":84,"Изв":129,"ل":83,"ا":150,"Ита":429,"Ирс":118,"Ира":123,"Ист":900," А":14517," Б":27484," В":12808," Г":10748," Д":13291," Е":5410," Ж":1459," З":6528," И":5862," К":19904," Л":8835," М":15676," Н":11244," О":9864," П":31853," Ђ":625," Љ":525," Ј":17141," Ћ":244," Њ":798," Џ":1146," α":83," б":24266," а":18333," г":25445," в":20362," е":9609," д":51232," з":20203," ж":6804," и":86886," л":8350," к":56435," н":57584," м":28129," п":94827," о":106930," Р":18752," С":31602," Т":9091," У":5047," Ф":11453," Х":15981," Ц":3264," Ч":1710," Ш":7130," ћ":799," њ":14264," љ":998," ј":100589," џ":140," т":19933," у":90576," р":27310," с":117389," ц":5947," ч":5875," ф":9403," х":4218," ш":6192,"Кад":128,"Кат":231,"Кап":162,"Као":168,"Кас":339,"Кар":1306,"Кал":923,"Как":158,"Кан":452,"Кам":420,"Каз":105,"Кај":141,"Ла ":84,"Куп":172,"Кур":248,"Кук":124,"Кул":160,"Кут":93,"Куз":132,"Кри":449,"Крс":142,"Кро":295,"Кру":300,"Кра":1299,"Кре":190,"Коњ":201,"Кош":104,"Кох":109,"Кот":211,"Кос":2912,"Кор":985,"Коп":201,"Кон":882,"Ком":414,"Кол":434,"Коз":193,"Код":100,"Ков":122,"Коб":165,"Кне":149,"Клу":161,"Кла":296,"Кли":168,"Кле":200,"Кло":88,"Кис":96,"Кир":248,"Кин":302,"Кер":95,"Кел":116,"Кен":184,"Лај":133,"Лав":93,"Лан":943,"Лау":243,"Лаз":155,"Лак":94,"Лен":108,"Леп":116,"Лео":127,"Лес":143,"Лет":239,"НК ":130,"Либ":219,"Лив":97,"Лим":199,"Лит":89,"Лис":112,"Лип":144,"Лин":222,"Лич":200,"Лиц":223," ја":1826," је":93577,"Лоа":476," љу":968,"Лоз":115,"Лон":298," њи":786,"Лот":96,"Лор":361," ње":13337," јо":756," ју":4367,"НУ ":87," њо":101," ће":734,"Луд":233,"Луц":83,"Лук":374," ра":8653," ре":13918," ри":921," пј":104," ро":2604," пр":45536," пт":139," пс":608," пу":1943," св":6584," си":3294," се":23555," сл":3656," см":1501," ск":1772," сп":2834," ср":16502," сн":827," со":2475," ру":1082," са":24552," тз":108," ти":1237," сј":428," тв":397," те":6410," то":2768," тр":4085," тк":114," сц":352," сх":94," ст":16984," су":11465," та":3595," тј":250," ук":888," уз":918," уж":105," уг":826," уд":840," ус":2032," ут":916," уп":1401," ур":494," ун":1058," уо":216," ул":992," ум":1152," ту":890," ув":475," уб":353," фо":1385," фр":763," фу":2034," фе":853," уј":191," фи":2888," фл":239," фа":1231," уч":922," уш":182," хр":1331," хо":601," ху":177," хи":633," хе":1026," хл":116," ха":273," цр":1684," ци":926," цв":109," це":2162," ца":1024," чу":291," чо":353," чл":773," чи":1933," че":1951," чв":130,"На ":568," ча":434,"Мал":538,"Мак":940,"Мад":118,"Маз":83,"Мас":129," шв":112,"Мар":1490," ша":327,"Мат":232,"Ман":533," ше":436," шк":548," ши":2335," шп":265,"Мач":118," шт":1839," шу":256,"Мађ":286,"Маг":109,"Мај":781,"Мез":119,"Мед":226,"Мес":117,"Мет":642,"Мел":120,"Мек":1028,"Мер":390,"Мен":198,"Међ":362,"Мла":173,"Мик":159,"Мил":998,"Мих":270,"Мин":381,"Мит":253,"Мис":115,"Мир":309,"Моз":210,"Мод":99,"Мос":300,"Мор":533,"Мол":130,"Мон":516,"² ":229,"Мур":119,"Нац":137,"Наг":213,"Нак":122,"Наз":173,"Нал":1049,"Нас":418,"Нап":155,"Нар":455,"Нев":147,"Од ":276,"Нај":450,"Нин":100,"Ниш":535,"Нид":112,"Ник":531,"Нем":539,"Нек":205,"Нер":93,"Нор":920,"Нов":1182,"Ноб":88,"аз ":491,"Ној":432,"ад ":7870,"ав ":760,"Он ":195,"аг ":169,"аб ":148,"ао ":7785,"ап ":117,"ам ":2217,"ан ":10929,"ак ":3202,"ал ":1554,"РЈ ":221,"Оаз":95,"Ово":323,"Ове":241,"Обр":159,"Обе":308,"Оби":118,"Ова":554,"Па ":301,"Оли":280,"Окр":343,"Оде":172,"авц":213,"аву":2060,"ага":974,"аги":364,"авј":94,"ављ":3634,"аге":519,"агр":1237,"агл":272,"агм":99,"агн":334,"аго":977,"агу":495,"ада":15920,"ади":3490,"аде":2838,"аду":3144,"адр":910,"адс":1188,"адо":1331,"адм":506,"адн":4147,"Опш":1002,"Ора":138,"Орг":87,"адњ":419,"аел":202,"Орл":171,"би ":1331,"аер":148,"аес":355,"ажа":324,"аже":556,"ажд":168,"Осв":136,"ажи":552,"Оск":121,"ажн":580,"Осн":326,"Осм":190,"ај ":2186,"Ост":672,"ажу":110,"Опе":165,"аба":540,"По ":294,"абл":408,"або":463,"абр":657,"абе":439,"аби":574,"ава":9441,"абу":169,"авн":5705,"авл":255,"абљ":86,"авк":265,"авс":891,"Оно":129,"авр":714,"бе ":991,"аво":2548,"аве":14288,"авд":102,"авг":575,"Она":164,"ави":16802,"алс":1372,"алт":594,"алп":116,"алн":9423,"бо ":129,"ало":1695,"алм":639,"алц":195,"алу":774,"алф":120,"алв":200,"алб":659,"ала":10846,"алк":547,"али":7292,"але":3111,"алд":1018,"алг":361,"амс":415,"аму":268,"амн":217,"амо":1815,"амп":780,"амб":552,"ама":5553,"ами":1178,"аме":2791,"анр":169,"ано":13152,"ану":5736,"анс":7532,"ант":3514,"анч":263,"анш":366,"анц":5831,"ана":8086,"анд":4157,"анг":1007,"ани":15172,"анз":207,"ане":2427,"анл":86,"анк":1530,"амј":97,"анџ":130,"аор":109,"аоц":85,"аоб":176,"анђ":145,"аон":418,"азу":880,"азо":538,"аљ ":458,"азн":730,"азм":274,"ажњ":102,"азл":1135,"ази":7146,"азе":707,"азг":86,"азд":502,"азб":116,"азв":1306,"аза":1339,"ањ ":588,"бл ":88,"аил":149,"аку":907,"акт":1696,"акц":534,"акш":91,"акл":273,"акн":144,"акм":1192,"ако":4032,"акс":2478,"акр":275,"акв":306,"аке":1255,"аки":556,"ака":2323,"ах ":1503,"аф ":127,"ć ":324,"ач ":958,"ац ":3942,"Пал":2844,"Пак":125,"Пар":781,"ас ":1347,"Пан":253,"Пав":221,"Паз":97,"Пас":164,"Пат":127,"ар ":7524,"ау ":935,"ат ":5190,"СР ":195,"Пла":286,"Пли":97,"Пле":203,"ба ":1983,"Пиј":157,"Пит":117,"Пик":455,"Пин":119,"Пир":890,"аш ":324,"Пећ":96,"Пет":882,"Пес":116,"Пер":500,"Пен":104,"Пел":147,"Пчи":125,"Џеј":167,"Са ":86,"ТВ ":182,"Поб":94,"Пов":174,"Поа":233,"Пож":269,"Под":816,"Пок":135,"Поз":146,"Пон":117,"Пол":384,"Пом":1052,"Пој":141,"Пољ":527,"Пор":439,"Поп":218,"Пот":421,"Пос":2261,"При":2140,"Пре":8305,"Прв":934,"Про":918,"Пра":388,"Прњ":91,"Рим":354,"Рис":97,"Риб":123,"Риј":90,"Рот":220,"Рођ":159,"Роз":94,"Рон":456,"Ром":159,"Роб":215,"č ":155,"Рог":222,"Род":147,"Рач":90,"Рас":264,"Рат":180,"Раш":217,"Рад":602,"Раз":142,"Рав":210,"Ран":169,"Рам":88,"Рак":105,"Рај":3464,"Рег":203,"Реч":123,"Реп":6121,"Рен":315,"Рем":99,"Рек":143,"Акв":410,"Ака":112,"Але":602,"Алз":106,"Али":168,"Алб":402,"Ала":116,"Алт":379,"Алп":608,"Амб":86,"Ама":100,"Аме":543,"Анд":310,"Анг":149,"Сад":314,"Саб":121,"Ана":188,"Сав":772,"Сан":433,"Сао":220,"Сал":228,"Сам":255,"Сак":1964,"Анс":83,"Ант":461,"Сау":85,"Сас":150,"Сар":597,"Анх":325,"Анђ":98,"Апо":93,"Рус":825,"Рум":494,"Рук":87,"Руд":247,"Ага":84,"Адм":355,"Ази":247,"Азу":140,"То ":455,"Сло":1071,"Сме":197,"Сма":124,"Ску":98,"Ско":232,"Сли":85,"Сла":799,"Ска":113,"Срп":4201,"Спо":152,"Срб":3494,"Сре":1205,"Сон":87,"Соц":253,"Спа":137,"Спл":118,"Спи":169,"Сов":165,"Сок":210,"Сом":238,"Сол":177,"Сев":1163,"Џор":106,"Џон":221,"Све":1041,"Сви":102,"Сва":118,"Сил":112,"Сим":270,"Син":185,"Сир":125,"Сис":300,"ВЈ ":118,"Сек":103,"Сел":387,"Сед":576,"Сер":251,"Сем":83,"Сен":765,"Бру":152,"Бри":492,"Бро":346,"Бра":1319,"Брд":115,"Бре":491,"š ":86,"Тар":199,"Тау":93,"Тан":138,"Так":280,"Бос":5750,"Бон":160,"Бор":452,"Бок":92,"Бол":109,"Бож":132,"Тај":125,"Боб":83,"Бод":117,"Бог":382,"Бој":140,"Бла":154,"Биб":121,"Бис":191,"Бит":519,"Бир":183,"Бин":118,"Био":412,"Бил":291,"šć":176,"Ајф":238,"Ајх":175,"Биј":195,"Беч":173,"БиХ":287,"Ста":912,"Сти":143,"Сте":486,"Стр":358,"Сто":282,"Сту":125,"Без":118,"Суд":249,"Бел":521,"Бек":95,"Бен":320,"Сун":370,"Бео":1668,"Бер":907,"Суп":90,"Бет":112,"Бањ":358,"Бај":268,"Бач":119,"Бау":104,"Бат":131,"Бар":635,"Тро":185,"Бал":373,"Трн":93,"Бан":484,"Три":351,"Бад":1635,"Тре":550,"Баб":113,"Бав":2112,"Тра":430,"Тул":87,"Тур":462,"Топ":250,"Том":239,"Тор":137,"Ток":171,"Тит":113,"Тих":87,"Тир":1042,"Тим":218,"Сје":411,"Афр":304,"Аус":436,"Аут":151,"Ати":273,"Атл":356,"ФК ":379,"Ата":245,"Арв":84,"Арг":112,"Ара":348,"Тек":85,"Ард":374,"Тео":217,"Ари":102,"Тел":164,"Тес":146,"Тер":251,"Арм":86,"Арт":149,"Арх":112,"Врх":140,"Врб":141,"Вра":229,"Вул":164,"Вук":248,"Вор":124,"Вод":118,"Вол":243,"Вог":102,"Вој":653,"Вид":142,"Вин":286,"Вик":124,"Вил":360,"Виз":162,"Вит":360,"Вир":1331,"Вис":149,"Виш":338,"Бје":189,"Виј":137,"Вла":763,"Вер":279,"Вес":907,"Вел":1090,"Вен":217,"Вез":142,"Тут":88,"Вај":429,"Ваљ":108,"Вас":189,"Вар":502,"Вал":502,"Ван":167,"Уст":141,"Упр":237,"Унт":105,"Уни":389,"Укр":132,"Бук":174,"Бул":140,"Бур":592,"Буг":354,"Буд":232,"Буз":86,"Буј":88,"Гис":85,"Др ":85,"До ":877,"Гли":92,"Гла":528,"Фак":100,"Фар":105,"Гра":1813,"Грб":96,"Гре":246,"Гри":146,"Гро":269,"Гру":261,"Грч":384,"Год":132,"Гор":2705,"Гос":206,"Гот":135,"Гол":158,"Фер":187,"Фед":1501,"Фил":509,"Фир":118,"Фин":97,"Ује":238,"Фле":186,"Фоч":111,"Фон":130,"Фор":279,"Фра":5371,"Фри":175,"Гал":207,"Фре":186,"Гар":335,"Гац":85,"Гај":99,"Фуд":232,"Гер":273,"Гео":143,"Ген":166,"ア":111,"Диј":83,"Дит":144,"Дир":130,"Дис":105,"Дио":94,"Дип":97,"Дим":140,"Дин":131,"Ен ":161,"Ер ":150,"Хар":369,"Хас":91,"Хал":153,"Хам":142,"Хан":247,"Доњ":1845,"Дон":322,"Дом":284,"Дол":169,"Дор":179,"Доб":482,"Дри":109,"Дре":135,"Држ":378,"Дра":512,"Дру":869,"Дун":151,"Дуб":334,"Дуг":84,"Душ":140,"Хај":361,"Хип":87,"Хил":214,"Хел":153,"Хен":110,"Хем":86,"Хер":6097,"Хес":450,"Хрв":3210,"Хри":165,"Хое":93,"Хор":121,"Хол":1488,"Хох":95,"Дал":436,"Дав":86,"Дан":479,"Дам":130,"Дар":218,"Дво":149,"Хун":181,"Дев":98,"Дем":212,"Дел":197,"Дер":133,"Ден":132,"Деч":93,"Дес":132,"Деј":200,"Цар":168,"Ест":109,"Епи":167,"Епа":99,"Енг":213,"Енд":111,"Еми":93,"Еле":145,"Ели":91,"Цер":116,"Цет":121,"Цел":161,"Цен":445,"Црв":222,"Црк":191,"Црн":1110,"Еге":91,"Еги":147,"Евр":809,"Чар":97,"Чеш":130,"Чес":89,"Чет":114,"Чел":87,"Жан":101,"За ":264,"Жив":194,"Жир":124,"Жен":115,"Жер":101,"лај":601,"лањ":114,"лбр":87,"лбу":532,"лва":233,"лаз":5049,"лаж":295,"лал":90,"лам":651,"лак":571,"лар":955,"лан":5765,"лау":200,"лас":3936,"лат":4598,"лаш":484,"лач":526,"лац":1124,"лах":123,"лба":351,"лбе":269,"лађ":217,"лго":231,"лги":122,"лда":152,"лде":413,"лди":101,"лдб":91,"ме ":4180,"лви":91,"лге":138,"лга":84,"ма ":25410,"лаб":226,"лав":5743,"лаг":702,"лад":2173,"кши":181,"кућ":321,"куј":354,"кус":263,"кут":125,"куш":199,"кул":1691,"кум":252,"кун":133,"лц ":163,"куп":1701,"кур":384,"кци":1655,"кош":268,"кох":141,"кођ":505,"коњ":158,"кој":46468,"лт ":421,"крв":211,"кре":1287,"кра":4330,"кри":1665,"кру":15138,"лу ":3102,"кро":885,"крс":287,"крш":123,"кса":656,"ксе":256,"кси":900,"ксо":2107,"ксн":118,"кст":498,"ксп":269,"ксу":269,"кта":463,"кте":692,"кти":1369,"ктн":244,"кто":1404,"ктр":790,"кту":4907,"кла":1570,"ло ":9505,"кло":812,"кли":774,"кле":1389,"клу":905,"кми":1187,"кне":270,"кни":220,"коб":278,"ког":8112,"ков":3602,"кну":107,"ком":10764,"кон":3999,"лс ":170,"коо":112,"коп":907,"кор":2532,"кос":446,"кот":489,"код":1658,"кож":96,"коз":151,"кок":257,"кол":2479,"кид":186,"кив":177,"кин":661,"ким":2487,"кил":447,"киј":370,"кић":129,"кињ":125,"лм ":881,"кип":134,"кир":398,"кис":271,"ких":13568,"киш":101,"ле ":4253,"кво":109,"кви":2552,"кед":760,"кеа":264,"кеј":173,"ли ":11349,"кен":534,"кел":235,"кет":218,"кер":419,"кањ":169,"лд ":1252,"кај":91,"кве":793,"ква":1006,"кат":1296,"кау":101,"кар":2613,"кас":751,"као":4364,"кап":295,"кам":527,"кан":1752,"как":575,"кал":1288,"каж":125,"каз":991,"каш":243,"кац":648,"каб":115,"кав":369,"кад":1624,"лб ":126,"ла ":11363,"кт ":360,"ку ":4779,"ићу":167,"кс ":487,"иљу":112,"иња":733,"иње":1591,"ињи":131,"ињс":371,"ко ":11086,"ињу":98,"ића":1051,"иће":486,"ићи":1351,"ићк":102,"ије":18649,"ија":26005,"км ":243,"ијо":1259,"ији":11828,"ијс":4230,"ију":3169,"иља":489,"иље":460,"иљн":91,"иљк":320,"ки ":15059,"ке ":16319,"нва":203,"од ":26954,"нбу":2054,"нбе":511,"нај":4359,"нањ":237,"нађ":92,"нба":320,"нах":325,"нац":1791,"нау":1444,"нач":3335,"наш":893,"нао":86,"ог ":17762,"нан":525,"нам":1269,"нал":7610,"нат":5956,"нас":17476,"нар":3487,"нап":984,"наж":99,"нае":414,"над":1326,"наг":937,"нак":1382,"наи":83,"наз":1926,"нде":1146,"нда":1820,"нгу":126,"нгт":100,"нго":298,"нгс":87,"нгр":162,"нгл":815,"нги":1043,"нге":1637,"нгв":164,"нга":390,"нви":125,"нве":215,"неш":165,"нже":149,"нек":2076,"нел":235,"нем":1023,"нен":721,"нео":346,"неп":695,"нер":1881,"нес":797,"нет":980,"неу":227,"неф":87,"нец":104,"нег":385,"нев":413,"неб":279,"неа":117,"нез":804,"неж":223,"нед":404,"нди":1494,"оз ":466,"ндо":1219,"ндс":442,"ндр":728,"нду":438,"нив":1382,"низ":2908,"ник":12010,"нид":108,"ниг":102,"ниж":128,"нзо":88,"ок ":1611,"нза":186,"нзи":538,"нзе":237,"неј":878,"оа ":446,"ов ":1648,"нав":745,"наб":169,"об ":174,"мпс":94,"мпо":410,"нт ":1409,"мпл":372,"моћ":512,"мпи":804,"мпе":520,"мпа":795,"моч":117,"моц":131,"моу":89,"мот":526,"мрт":332,"мро":192,"ну ":15940,"мпј":83,"мрл":89,"мре":223,"мск":2421,"三":157,"丁":86,"нц ":300,"мун":1141,"мул":381,"муз":898,"мус":132,"мур":117,"муч":159,"муш":457,"нш ":306,"мци":104,"мењ":500,"мељ":149,"мећ":198,"нк ":147,"мик":370,"мил":886,"миз":88,"мид":103,"мич":1375,"миц":536,"миш":727,"мин":3851,"мим":241,"мио":91,"мис":825,"мир":1661,"мит":950,"миј":1533,"мињ":179,"мић":122,"Љуб":331,"мла":438,"но ":31892,"мна":346,"мне":116,"мни":309,"мно":823,"моб":172,"мод":539,"мог":1314,"мов":616,"моз":267,"мож":885,"мон":1010,"мом":420,"мол":368,"мок":358,"мос":2613,"мор":2662,"нс ":374,"маљ":216,"мај":1350,"маћ":267,"мањ":980,"мбе":1510,"мби":327,"мбу":200,"мбо":455,"нд ":1390,"мбр":845,"маш":262,"мац":1084,"мач":12755,"мах":140,"мађ":207,"мба":1378,"мал":1443,"мак":415,"мад":341,"мат":3328,"мас":545,"мар":1689,"нг ":1021,"мао":200,"ман":7905,"мам":150,"маг":535,"Јов":351,"меј":166,"међ":2287,"мех":195,"меш":617,"мес":3136,"мет":3201,"мен":7145,"ни ":34617,"мер":4603,"мек":210,"мел":176,"мем":128,"мед":1023,"мев":268,"мде":84,"не ":27271,"Јон":92,"лшт":1289,"Јад":173,"Јаб":184,"Јав":91,"Јаг":89,"Јан":158,"Јап":177,"Јак":157,"Јас":196,"лце":157,"лци":100,"лцл":114,"лха":148,"Јел":179,"на ":57921,"Јез":144,"Јед":11541,"Јев":137,"Јер":216,"лпи":612,"лој":304,"лпс":100,"лош":1201,"лоч":242,"му ":1793,"лни":2506,"лне":2152,"лно":2998,"лму":125,"лмс":356,"лна":907,"лоз":717,"лок":686,"лог":3009,"лод":251,"лож":580,"лоп":543,"лор":387,"лос":501,"лот":376,"лол":105,"лом":1605,"лон":784,"мс ":231,"лну":1948,"лов":5701,"лоб":930,"лут":258,"лус":300,"луч":438,"луц":348,"лул":94,"лук":379,"луп":108,"луо":194,"лун":148,"лум":724,"луг":281,"луб":1026,"луж":723,"луз":136,"луј":149,"лфр":88,"лфе":131,"лфи":86,"лфа":117,"лст":128,"лск":1883,"лса":106,"лту":760,"лто":95,"лти":229,"лте":881,"лта":559,"лећ":152,"лза":115,"лиг":1118,"лид":159,"либ":192,"лив":576,"лиз":1529,"лиж":310,"лил":191,"лим":1734,"лик":11361,"лдо":133,"лдс":84,"леа":196,"леб":189,"леж":344,"лез":1467,"лев":722,"Југ":1256,"лег":313,"лед":1839,"Јуж":557,"лес":1364,"лер":1174,"леп":226,"Јур":133,"лео":258,"ми ":812,"лен":2926,"лем":1370,"лел":128,"Јул":83,"лек":2393,"леш":105,"леч":125,"леф":108,"леу":115,"лет":1196,"леј":185,"леђ":138,"лкр":243,"кљу":617,"књи":1112,"мо ":1125,"лма":835,"лмо":236,"лме":86,"лих":503,"лиц":2558,"лич":2364,"лиш":305,"лис":1427,"лит":3381,"лиф":251,"лин":3445,"лио":440,"лип":305,"лир":152,"лић":519,"лиј":3989,"лко":146,"лка":612,"пас":269,"пат":651,"пац":228,"паш":107,"пав":93,"пад":14312,"паг":84,"паж":106,"пак":284,"пам":140,"пал":338,"пао":90,"рг ":5715,"пан":3250,"пар":6224,"пап":134,"пај":236,"пањ":487,"рд ":408,"ођа":440,"ре ":3735,"ође":2085,"ра ":11124,"рб ":143,"пил":207,"пик":84,"пир":430,"пио":186,"рл ":93,"пин":427,"пит":636,"пис":10327,"пиш":133,"оја":11213,"оје":9859,"ојв":646,"оји":9116,"пиј":1084,"ојк":186,"ојн":1254,"пић":150,"ојм":157,"ојо":880,"ојс":808,"оју":1151,"ојт":161,"ојц":174,"ојш":127,"оља":608,"оље":731,"ољи":259,"ољс":462,"рн ":846,"ољо":106,"ољн":234,"ољу":345,"пла":1577,"оња":1770,"оњи":617,"пли":958,"пле":749,"оње":264,"пло":805,"ро ":1261,"оће":109,"оћи":182,"пед":320,"пев":364,"ри ":8692,"пер":2735,"пес":743,"пет":618,"пек":270,"пел":585,"пен":599,"пец":408,"пеш":318,"пећ":129,"рк ":422,"пиз":88,"ори":9358,"орз":85,"орд":1293,"оре":3647,"орв":124,"орг":2732,"орс":1818,"оро":1886,"пу ":455,"орп":370,"орм":2476,"орн":1996,"опљ":214,"орк":370,"опш":35593,"опу":756,"опт":202,"опс":1008,"орб":607,"ора":4078,"нџа":125,"опе":978,"опи":8685,"опн":248,"опо":1046,"опр":619,"опл":534,"оос":176,"оор":106,"опа":674,"оте":1270,"отк":354,"отл":140,"оти":2195,"осј":1754,"ото":1607,"отп":577,"отн":337,"оту":206,"отр":1043,"отс":417,"осф":169,"осу":428,"отв":376,"ота":1137,"орђ":198,"осе":1479,"оси":1742,"орј":175,"оск":422,"осл":4652,"орњ":1483,"осм":2386,"осн":8035,"осо":1921,"осп":343,"оср":231,"орџ":88,"ост":12354,"ору":1182,"орт":1154,"орф":1119,"орц":137,"оса":1233,"осв":640,"омн":289,"омл":99,"оми":1500,"оме":4149,"омб":447,"ома":3081,"олш":1216,"олц":286,"олу":1267,"олф":220,"олс":304,"олт":167,"олн":318,"по ":2745,"оло":4064,"олк":138,"олм":112,"оле":1477,"оли":4441,"олд":233,"ола":2070,"окр":16250,"окс":337,"окт":753,"оку":1463,"око":3712,"онс":1810,"онт":1672,"онц":357,"ону":5169,"онф":228,"они":5926,"онз":280,"онк":246,"пр ":139,"оно":2541,"она":7689,"онв":188,"онб":88,"онд":758,"онг":338,"оне":1433,"омо":1831,"омп":1100,"омс":686,"ому":652,"оша":382,"оши":323,"оше":324,"ошк":1178,"ошл":182,"ошт":538,"оча":344,"очи":758,"оче":1097,"очн":3498,"оцј":1629,"очу":83,"ошћ":293,"офе":855,"офа":229,"оуч":265,"офт":139,"офо":217,"офс":147,"офи":650,"оуг":194,"оуп":90,"оуз":104,"оца":171,"оце":938,"оци":1593,"оха":159,"охе":282,"охр":88,"охи":547,"охо":201,"оба":2774,"оат":253,"оар":591,"оан":147,"овг":89,"ова":11175,"обу":821,"обр":1721,"обо":1375,"обн":584,"обл":2580,"оби":2413,"обз":108,"обе":1299,"Њен":125,"па ":2144,"Њег":294,"оис":1849,"оим":1253,"оћ ":160,"нљи":100,"оки":368,"окл":297,"оке":600,"ока":1581,"окв":1772,"ожи":330,"ој ":44878,"ожђ":85,"озв":134,"ози":1003,"озе":676,"оза":1830,"ожј":112,"озо":1076,"озн":2962,"оид":1008,"оиз":729,"Њуј":106,"одм":327,"одн":4424,"одл":481,"оди":8562,"оду":1182,"одр":3378,"одс":582,"одо":1134,"одњ":237,"оен":147,"пи ":1059,"оже":1557,"ожд":83,"ожа":629,"ове":6236,"обљ":196,"овл":133,"ови":13959,"обј":733,"ово":5370,"пе ":1159,"овн":11503,"ову":980,"овр":1263,"овс":742,"овч":112,"овц":697,"ога":1828,"нђе":269,"оге":687,"оги":1839,"овј":1495,"овљ":259,"огл":656,"огн":99,"ого":1533,"огр":4802,"огт":183,"огу":1193,"одб":375,"ода":2535,"одг":540,"одв":356,"оде":2259,"оу ":269,"нро":158,"нри":135,"ноћ":177,"от ":628,"нпр":136,"ној":19200,"нот":238,"нос":7574,"ноф":97,"нош":420,"ноп":192,"нор":440,"нон":275,"ноо":172,"ос ":2917,"нол":379,"ном":7402,"нок":125,"ноз":97,"нож":162,"ног":6852,"нод":143,"ноб":292,"нов":16704,"ноа":173,"ор ":3231,"оп ":397,"нла":112,"нли":105,"он ":5316,"нко":648,"нкр":96,"нкт":107,"нку":283,"мљу":151,"нкц":621,"мља":354,"нка":850,"нки":337,"мљи":386,"мље":717,"нке":437,"ом ":22463,"нић":400,"ниј":12404,"мје":4955,"нир":517,"нис":2501,"нит":625,"нил":406,"ним":4578,"нин":1402,"нио":253,"ол ":661,"нич":2890,"ниш":4604,"ниф":141,"них":6787,"ниц":5694,"ншт":373,"нче":230,"нчи":115,"нча":188,"нце":702,"нци":2151,"нца":651,"нцу":5237,"нхе":115,"нха":822,"ош ":929,"нхо":113,"нфе":414,"нфо":322,"нуа":555,"нум":116,"нул":184,"нук":195,"нус":137,"нуо":120,"нут":1477,"нсф":160,"нта":3191,"нте":2285,"нти":6056,"нтс":456,"нту":323,"нто":871,"нтн":454,"нтр":2081,"нса":348,"нсб":322,"нск":21333,"нси":1508,"нсе":243,"нст":2320,"нср":148,"нсп":163,"нсо":221,"оф ":272,"саз":225,"сак":730,"сал":373,"сам":1848,"сан":1886,"сао":546,"саб":166,"сав":12830,"саг":142,"сад":723,"сач":444,"сар":487,"сат":1303,"сас":2383,"сау":119,"сац":438,"сањ":362,"сај":140,"тд ":136,"сба":232,"сбе":307,"сва":1042,"сбу":542,"сво":2081,"те ":6016,"свр":218,"све":3473,"сви":792,"рђа":295,"свј":110,"рђе":430,"сдо":330,"себ":683,"сев":3037,"сег":95,"сез":327,"сед":1857,"сел":1535,"сек":671,"сеп":618,"ти ":10122,"сео":160,"сен":1313,"сем":200,"сет":704,"сес":115,"сер":1186,"сец":168,"сеч":136,"сељ":14369,"сеј":202,"сећ":267,"сењ":205,"сиг":253,"сид":147,"сив":302,"рја":155,"рје":274,"сич":228,"сис":1247,"сит":283,"сир":1615,"сих":600,"сиф":177,"сик":192,"син":1494,"сио":533,"сил":738,"сим":906,"скв":125,"ске":7702,"ски":22959,"рљи":86,"ска":16636,"сиј":1688,"сиљ":91,"сињ":146,"сић":118,"рњи":454,"сли":1408,"рње":277,"сле":1944,"рња":1169,"сла":3221,"ску":1728,"скр":678,"ско":23421,"скл":305,"сми":426,"сме":837,"сма":1176,"АЈ ":245,"слу":1437,"сло":3155,"то ":10724,"скљ":114,"сна":1061,"сне":1878,"сни":6625,"смо":2197,"смр":275,"соб":1187,"сов":2446,"сок":416,"сол":242,"сом":308,"тр ":138,"сно":4183,"сну":121,"спе":1119,"спа":703,"спл":139,"спи":697,"соф":144,"сор":646,"соп":287,"сон":2669,"соц":896,"сре":14058,"ту ":6826,"сро":147,"сри":173,"спу":160,"спо":2462,"спр":728,"рпу":136,"рпс":6643,"рпр":93,"су ":17733,"роц":2456,"роч":246,"рош":621,"рот":1629,"роу":489,"роф":1077,"роп":1727,"рор":221,"рос":2133,"ст ":4278,"рпо":293,"рој":2351,"рођ":922,"рпе":118,"рпа":88,"рта":1010,"рст":3136,"рсх":160,"ртн":153,"ртм":4491,"ртс":303,"ртр":100,"рто":516,"рте":1621,"ртв":127,"рти":1376,"рса":168,"рск":8319,"рсл":95,"рсо":156,"рсн":119,"рсб":203,"рсд":115,"рсе":238,"рси":314,"руш":1238,"руч":948,"рфи":96,"руј":409,"рфе":114,"рфо":135,"рту":311,"руа":505,"руб":149,"руз":135,"рук":1051,"руг":17782,"руд":207,"руж":1022,"руп":2147,"рус":775,"рут":180,"рум":339,"рун":279,"рца":147,"рце":6006,"рцг":111,"рцв":90,"рци":365,"рцо":146,"рха":299,"рхе":608,"рхи":1027,"рхо":272,"АД ":350,"рху":132,"ршт":219,"ршн":146,"рши":1285,"ршк":166,"рше":452,"рша":332,"рчк":1174,"рчи":96,"рча":87,"АП ":94,"та ":24494,"рад":15222,"рае":112,"раж":1067,"раз":4405,"раб":340,"рав":5728,"раг":886,"рам":3023,"ран":15671,"рао":659,"рап":559,"рак":1642,"рал":3181,"рах":410,"раф":1317,"рач":1292,"рац":2825,"рас":1831,"рар":136,"рау":311,"рат":5078,"рба":695,"раш":666,"рби":3548,"рбе":475,"рађ":1238,"рбо":328,"рањ":1384,"раћ":751,"рај":4293,"раљ":1551,"рбу":351,"рва":5445,"рдс":106,"рду":314,"рдн":105,"рдо":475,"рди":1053,"рде":834,"реб":1934,"рев":1509,"рег":7841,"ред":21162,"реа":613,"рдф":196,"реф":4512,"реу":146,"рет":2436,"рес":1742,"рер":230,"реп":1358,"си ":1735,"рео":463,"рен":3928,"рем":12715,"рел":899,"рек":3409,"реи":116,"рез":1388,"реж":768,"рже":168,"ређ":1332,"ржа":14645,"реч":1314,"реш":535,"рех":110,"рец":293,"рвн":173,"се ":16756,"рво":1461,"рве":1500,"рви":1598,"рга":2899,"рву":294,"рвц":130,"рго":513,"рге":622,"рги":440,"рдв":110,"рда":653,"ргу":233,"риш":1433,"пје":186,"рио":1442,"сл ":142,"рип":10260,"рим":2792,"рин":3442,"рик":1650,"рил":1573,"риц":1301,"рич":2529,"риф":333,"рих":484,"рит":3451,"рир":799,"рис":3789,"пља":339,"рка":895,"рки":242,"пљи":88,"ркв":1089,"рке":414,"пље":324,"риј":9688,"рињ":113,"рић":295,"рза":189,"ржи":374,"реј":705,"рећ":815,"рењ":498,"риб":747,"риг":881,"рив":1495,"рид":656,"риз":1001,"риж":129,"рзи":967,"рзо":108,"ск ":101,"рнк":113,"рни":2602,"рне":1342,"рнб":105,"рна":2309,"рму":235,"рмс":114,"рмо":317,"рок":955,"рол":836,"ром":2696,"рон":2160,"рож":168,"роз":1124,"рои":2993,"ров":3353,"рог":2212,"род":5445,"роа":193,"роб":830,"рну":227,"рно":3320,"рле":211,"рли":396,"рла":605,"рку":269,"ркт":206,"рко":589,"рме":465,"рми":1450,"рма":1859,"рло":695,"со ":105,"рлс":110,"пра":4356,"прв":2403,"при":15354,"пре":17090,"про":11337,"ру ":5752,"поп":7996,"пор":3359,"пос":5163,"пот":2258,"пох":118,"поч":1072,"пош":263,"пој":1887,"пољ":1013,"рт ":1206,"пно":461,"пое":108,"под":4064,"пог":482,"пов":2442,"поб":402,"рс ":235,"пон":1141,"пом":1383,"пол":3072,"пок":1589,"поз":2822,"оћу":157,"оћн":181,"пне":188,"пна":116,"пни":227,"пуш":177,"рч ":111,"пуњ":88,"пуб":6548,"пту":139,"пун":422,"рц ":99,"пус":393,"пут":1505,"пук":115,"пул":497,"пта":194,"пст":361,"рх ":203,"пто":209,"пти":395,"пте":647,"пса":91,"прс":111,"пру":342,"псо":194,"рф ":929,"пск":7893,"пси":594,"псе":135,"пшт":36823,"пче":98,"пци":231,"пхо":118,"са ":8016,"вао":651,"вар":4393,"вас":221,"ват":5276,"вац":1889,"вач":2332,"ваљ":162,"вај":2438,"вањ":3929,"ге ":1625,"вад":467,"ван":5104,"вам":274,"вал":2173,"вак":1047,"ваз":458,"важ":509,"га ":15953,"бух":516,"буш":92,"бун":179,"бум":473,"бул":126,"бук":180,"бус":106,"бур":3775,"буд":350,"буг":130,"буб":88,"врђ":432,"вск":2068,"вст":354,"вра":364,"врд":211,"вре":2511,"ври":173,"врл":169,"врс":1575,"гу ":3405,"вро":1134,"врх":345,"врт":434,"врш":1750,"вођ":799,"вој":4438,"вољ":474,"вол":443,"вок":137,"воз":273,"вод":3129,"вог":1645,"вов":377,"воб":187,"воа":91,"вот":981,"воу":181,"вор":3081,"вос":856,"вом":1630,"вон":796,"вни":13918,"вне":1253,"вна":1125,"вну":306,"вно":4495,"гр ":172,"вла":1894,"го ":717,"вко":88,"бље":277,"вка":213,"бља":290,"вић":3354,"вињ":156,"виј":2182,"виш":1683,"виц":1060,"вич":586,"бје":512,"виђ":131,"бја":446,"виз":1047,"вил":1408,"вик":313,"вин":8161,"вим":1946,"вио":870,"гл ":123,"вис":1551,"вир":2019,"вит":1026,"вих":1104,"вив":88,"виг":1474,"вид":967,"већ":2150,"веш":575,"веч":174,"вец":113,"вес":693,"вет":3813,"вер":6835,"вен":4778,"ги ":980,"вео":533,"вел":2599,"вем":722,"век":1758,"вез":13652,"веж":185,"вег":170,"вед":952,"веб":100,"вде":113,"вда":83,"вгу":597,"ва ":17876,"бак":147,"баз":278,"бан":881,"бам":152,"бал":2686,"бав":1092,"баб":84,"ачн":488,"ачи":1962,"ачк":16987,"ачу":748,"аше":304,"аша":706,"ашн":83,"ашк":841,"аши":740,"ашт":708,"ашћ":92,"ашњ":1176,"афа":126,"ауч":504,"ауц":95,"аус":384,"аут":1011,"аур":188,"аун":269,"аул":105,"аум":107,"аук":567,"ауз":840,"ауе":242,"ауд":139,"ахв":170,"аха":309,"афр":145,"афс":527,"афи":664,"афе":332,"аца":895,"аце":128,"ахо":294,"аху":91,"ахт":162,"ахи":208,"ача":2246,"ачв":93,"аче":1192,"аци":9797,"ацк":104,"апу":221,"апс":380,"апт":123,"апр":1018,"апе":351,"апа":4871,"апо":620,"апл":119,"апи":905,"арх":1394,"арц":510,"арс":6391,"арт":6536,"ару":699,"аса":1276,"арш":342,"аре":1441,"ард":1546,"арб":390,"ара":7231,"арг":210,"арв":113,"арн":2120,"арм":468,"арп":116,"аро":3543,"бу ":460,"ари":4397,"арл":1030,"арк":1412,"асу":214,"аст":10621,"ата":4681,"аси":1271,"асе":15255,"асл":611,"аск":500,"асп":712,"асо":748,"асн":1469,"ату":1549,"ауб":95,"атв":223,"ате":3408,"ати":9099,"атл":474,"атк":530,"атн":1447,"атм":93,"ато":2391,"атс":4507,"атр":1269,"бож":165,"бол":831,"бом":290,"бок":146,"бог":924,"бод":695,"бов":463,"бор":1861,"бон":136,"бот":221,"бос":184,"бни":336,"бна":134,"вр ":135,"бно":688,"брн":93,"бро":2384,"ву ":4383,"брз":284,"аџи":118,"бри":1202,"бре":580,"брд":174,"бра":2962,"бру":739,"бољ":521,"бој":722,"бођ":100,"аља":600,"аљи":242,"аље":1718,"ајњ":180,"аљо":121,"аљу":134,"биш":118,"бич":725,"ајб":530,"аја":2689,"ајг":83,"ајв":1567,"аје":3121,"ајд":438,"ајз":386,"аји":1314,"биј":4013,"биљ":565,"ајк":465,"бињ":212,"ајл":672,"бић":203,"ајм":1497,"ајн":7063,"ајо":227,"ајп":549,"ајр":167,"ајс":1200,"ајт":407,"ају":3999,"ајф":161,"ајх":127,"ајц":375,"ајч":467,"аћа":316,"аћи":320,"аће":606,"бла":1682,"ања":4048,"бли":8473,"ањи":632,"бле":479,"ање":5548,"бло":178,"ањо":120,"во ":5192,"ањс":162,"ању":1132,"беј":91,"беђ":131,"ви ":16486,"бео":120,"бен":1184,"бет":151,"бес":248,"бер":3715,"без":706,"бед":465,"бел":853,"бек":184,"бег":170,"биц":333,"бир":422,"бит":1370,"бис":240,"бим":119,"бил":6721,"био":4368,"бин":1064,"биб":102,"бив":1525,"бзи":113,"ве ":5887,"бањ":123,"бај":140,"бат":207,"бас":218,"бар":2176,"бац":280,"бах":889,"баш":126,"бач":343,"ађу":117,"ађи":181,"ађе":817,"ађо":95,"ађа":1192,"дбе":122,"дби":132,"дај":799,"даљ":844,"дањ":368,"дба":1697,"дач":129,"дац":201,"даш":393,"дан":4942,"дам":540,"дап":114,"ег ":1363,"дао":250,"дар":1892,"дат":1152,"дви":424,"два":1043,"две":625,"дбр":248,"ед ":1208,"дбо":137,"еб ":222,"дак":211,"дал":853,"дав":818,"даг":152,"ев ":380,"дев":361,"деб":169,"дек":279,"дем":734,"дел":3577,"део":1026,"ден":4057,"дер":3109,"деп":4528,"деј":406,"дећ":331,"дељ":596,"деф":396,"дес":1778,"дет":316,"деш":161,"дец":815,"еж ":110,"дго":498,"дво":732,"ез ":996,"дгр":142,"еа ":228,"ех ":100,"дсј":100,"дсх":127,"дст":2337,"еф ":157,"дск":2156,"дсе":669,"дсб":213,"дрш":85,"еу ":86,"дро":606,"дру":3934,"држ":14521,"дре":1505,"дри":545,"дрв":213,"дра":1821,"доњ":201,"ет ":2403,"еш ":152,"дфр":132,"дха":112,"дуј":1810,"еч ":332,"дућ":111,"дур":108,"дус":249,"дуц":108,"дух":450,"душ":198,"дуа":101,"дуб":168,"дув":92,"дуг":396,"дуж":674,"дуз":216,"дук":334,"дул":103,"дун":303,"ец ":275,"гље":108,"ен ":11412,"диш":14641,"дич":217,"диц":1561,"диф":139,"ем ":4511,"дињ":631,"дић":425,"диј":3395,"диг":489,"див":514,"дим":783,"дин":8562,"ел ":2175,"дио":1175,"дип":178,"дир":430,"дис":935,"дит":511,"дид":113,"диз":259,"дик":313,"дил":663,"ек ":825,"доб":1676,"дож":120,"доз":192,"дов":1958,"дог":363,"дод":480,"доп":210,"ес ":778,"дос":743,"дор":1358,"дол":780,"док":1240,"дон":1598,"дом":1333,"дош":123,"дна":2901,"дни":4153,"дне":1150,"дно":18824,"ер ":5027,"дну":350,"дма":127,"дме":197,"дми":757,"дмо":469,"еп ":122,"дла":144,"дле":110,"дли":278,"ео ":1513,"дло":145,"длу":219,"да ":21560,"гад":292,"гал":396,"гам":317,"гањ":112,"гај":141,"гађ":246,"гач":132,"гаш":101,"гас":227,"гат":525,"гау":144,"ган":3079,"гао":220,"гар":1084,"гво":89,"де ":3882,"гви":147,"где":607,"гда":83,"вук":158,"вуч":103,"вуј":109,"вца":402,"вце":222,"вци":497,"вцу":236,"вше":911,"вшо":105,"вши":448,"гон":372,"гол":312,"гом":499,"гос":1688,"гот":321,"гор":1661,"гош":127,"гов":8734,"год":5194,"гог":627,"гои":426,"гоз":827,"гно":115,"др ":192,"гну":250,"гна":134,"гне":319,"гни":146,"гру":1851,"ду ":4304,"гро":436,"грч":867,"грб":106,"гра":17750,"гри":166,"гре":1035,"гој":432,"гоњ":233,"гту":131,"гто":97,"гсл":98,"гсб":103,"гха":104,"гућ":509,"гуј":168,"гус":706,"гур":326,"гум":88,"гун":84,"гул":187,"губ":222,"гет":151,"геј":143,"геб":295,"гед":238,"гез":107,"ген":3069,"гел":241,"гер":454,"ди ":2469,"гео":722,"гим":517,"гис":230,"гит":213,"гин":376,"гио":6793,"гип":234,"вља":2757,"вље":1142,"вљу":193,"вљи":116,"вја":130,"вје":4514,"гич":129,"гих":425,"гиј":3453,"гињ":124,"гме":124,"гма":140,"гле":1228,"гла":3020,"глу":599,"гло":280,"до ":2950,"гли":311,"жањ":128,"жба":86,"жбе":201,"жај":441,"жби":131,"жан":1034,"жал":90,"жар":134,"жав":14295,"зв ":108,"ος":117,"за ":8613,"ешћ":540,"еју":172,"еља":1079,"еље":14036,"ељк":93,"ељи":403,"ељн":90,"ељу":544,"ељс":195,"жит":83,"жич":92,"жиц":127,"жиш":102,"еја":612,"ејв":144,"еје":348,"еји":839,"жиј":121,"ејк":85,"ејл":91,"ејм":173,"ος ":117,"ејн":153,"ејо":108,"ејс":626,"ејт":90,"жив":4026,"жир":301,"жис":135,"жио":111,"жим":106,"жин":429,"жил":151,"жењ":609,"жељ":130,"жем":122,"жел":174,"зи ":3995,"жен":1361,"жев":999,"жег":248,"еђу":2825,"жди":88,"жде":98,"зе ":1198,"еђе":1098,"еђи":194,"еђа":194,"иХ ":284,"зу ":549,"жрт":88,"жно":1981,"жни":671,"жне":211,"жна":151,"зо ":136,"ењу":566,"ењс":222,"ењи":241,"ење":2556,"ења":1959,"ећо":157,"ећу":200,"ећи":1696,"ећа":1119,"еће":1229,"ς ":235,"жуп":1922,"жут":87,"жуј":115,"жа ":368,"еак":214,"ежа":300,"ежн":187,"еј ":684,"ежи":857,"еже":462,"еду":2402,"едр":262,"едс":2840,"жи ":752,"Ђур":142,"едњ":1513,"α ":172,"езу":767,"еин":162,"еир":106,"езб":299,"еза":1560,"езн":11980,"ељ ":395,"езо":593,"езв":1298,"ежђ":133,"езг":91,"езд":473,"езе":1288,"ези":2451,"ебу":238,"ебр":919,"ебо":119,"ебн":548,"ева":2735,"ебљ":174,"еви":3027,"еве":5206,"еар":187,"Ђор":119,"еат":111,"еал":333,"еан":354,"еба":904,"еби":695,"ебе":787,"егр":445,"его":7667,"егу":205,"едв":232,"едб":129,"еда":4271,"еде":3246,"едг":91,"еди":18581,"едо":1827,"едн":17721,"едм":296,"едл":104,"евн":1038,"же ":1650,"ево":1640,"евр":650,"евс":729,"еву":170,"евц":292,"ега":1064,"еге":504,"еги":7687,"егл":144,"евљ":87,"ент":9797,"ену":1523,"енр":171,"енс":2955,"енц":1314,"енч":83,"енф":253,"енх":510,"емљ":1229,"енк":694,"енл":133,"ени":9840,"ено":12442,"енг":675,"енв":171,"енб":2610,"ена":7015,"енз":581,"ене":4034,"енд":1652,"еос":145,"енљ":86,"еол":657,"еом":497,"еон":274,"еоп":167,"еор":1067,"еоб":155,"еод":197,"еог":2215,"енш":218,"епт":829,"епу":6492,"епр":922,"епо":878,"епи":567,"епе":474,"епа":4865,"ерш":235,"еру":849,"ерф":172,"ерх":186,"ерц":6125,"ерп":274,"ерс":1727,"ерт":854,"ерл":469,"ерм":1235,"ерн":3846,"жу ":162,"еро":4629,"ери":8354,"ерз":950,"ерк":557,"ерд":412,"ерг":2878,"ере":2380,"ера":7017,"ерв":952,"ерб":428,"дје":329,"ећ ":287,"еке":903,"еки":816,"дљи":143,"екл":1752,"еко":3685,"екр":251,"ект":6659,"екс":2140,"еку":831,"ека":2874,"екв":261,"елн":382,"елм":190,"елк":306,"ели":7167,"дњи":405,"елх":90,"дњу":139,"елу":1959,"елт":171,"елс":443,"ело":4882,"дњо":511,"екц":169,"елд":739,"дње":682,"еле":3555,"елг":118,"дња":475,"ела":3270,"елб":151,"емо":949,"емн":211,"еми":2164,"ему":676,"емп":302,"емс":396,"еме":3254,"ема":24141,"емб":3007,"ехн":475,"ехо":95,"еха":297,"еца":115,"еци":860,"еце":786,"еча":464,"ецу":92,"ечк":150,"ечн":680,"ечи":726,"ече":483,"еша":500,"ешт":2328,"ешн":216,"ешк":621,"ечј":98,"еши":203,"еше":271,"еса":822,"есв":112,"есе":1676,"еси":981,"еск":1340,"ерњ":218,"есл":277,"есн":2049,"есм":470,"есп":273,"есо":641,"есу":289,"ест":11967,"есх":103,"ета":4650,"етв":788,"ети":2810,"ете":1561,"етл":282,"етк":797,"етр":1300,"етп":147,"ето":2098,"етн":2013,"ету":4517,"етс":2919,"етх":191,"еуд":103,"еук":143,"етљ":101,"етњ":220,"еуз":123,"еут":176,"еур":88,"ефе":4639,"ефа":304,"ефо":247,"ефи":517,"И ":215,"ибу":225,"ибл":386,"ибо":357,"ибр":131,"иве":1702,"иви":1641,"ива":4282,"иак":116,"иби":356,"ибе":609,"иба":325,"К ":688,"Л ":171,"М ":193,"ижа":175,"иже":927,"ижи":106,"ижн":162,"Н ":104,"идо":324,"идн":616,"иду":243,"идс":126,"идр":302,"игр":1750,"игс":188,"игм":91,"игл":124,"О ":291,"иго":261,"игн":387,"игу":383,"ида":820,"иди":470,"идл":111,"иде":1216,"ивр":422,"иво":1982,"ивн":2641,"П ":278,"ивш":1479,"иву":230,"ига":1246,"ивј":1819,"ивљ":215,"иги":748,"иге":674,"икл":343,"икс":296,"икр":200,"ико":2826,"ике":3772,"А ":487,"ики":922,"ика":16078,"ић ":4301,"зје":128,"Б ":209,"アア":86,"В ":325,"изу":761,"изр":754,"изм":2464,"изл":469,"иљ ":194,"изо":941,"изн":934,"изи":1871,"изд":704,"изг":522,"изе":216,"иза":4363,"изв":1974,"изб":400,"Г ":98,"иј ":194,"иом":122,"ион":10502,"иок":107,"иол":450,"иор":162,"иоп":96,"иот":219,"инц":1099,"инх":152,"иод":791,"иог":169,"Д ":487,"ине":13774,"инж":128,"ини":19900,"имљ":315,"инк":386,"ино":2092,"инс":14428,"инт":1353,"ину":1584,"инф":389,"ина":25163,"Е ":100,"инв":151,"инд":1081,"инг":3441,"ими":1186,"име":4873,"иму":393,"имс":660,"имо":935,"имп":1016,"имн":293,"илу":422,"имб":441,"има":9275,"или":9465,"илд":262,"иле":1239,"илс":244,"илт":96,"илм":1501,"илн":453,"ило":6433,"икш":116,"ику":1972,"икт":253,"ила":5198,"илв":95,"иси":951,"исе":634,"иса":3378,"исц":582,"исх":121,"ису":8052,"ист":15326,"исп":745,"исо":651,"исн":740,"исм":208,"исл":783,"иск":1574,"ити":3953,"итв":152,"итд":137,"ите":7069,"ита":4182,"итб":246,"Ј ":683,"иту":1129,"ито":3248,"итс":285,"итр":493,"итл":226,"итк":519,"итн":680,"итм":218,"ипа":10217,"ипе":470,"иоц":158,"ипт":214,"ипс":176,"ипр":120,"ипу":126,"ипи":313,"ипо":546,"ипл":633,"ира":4573,"ирг":162,"ире":985,"ирс":409,"иру":1957,"ирт":1182,"ирх":398,"ирц":88,"ири":2830,"ирк":377,"ирн":181,"ирм":121,"иро":1739,"ихв":144,"иха":399,"ихи":202,"ихе":169,"ихт":106,"ихо":1238,"ице":3082,"ица":6758,"ици":5856,"ицу":455,"ицо":160,"ифа":124,"ифе":473,"ифи":670,"ифо":192,"ифр":1763,"ифх":92,"ишљ":461,"ишњ":1193,"ишћ":818,"ичу":145,"ичи":1889,"ичк":7995,"ичн":3458,"ича":2409,"иче":1493,"ишу":252,"ишт":18544,"ишк":400,"иши":415,"ишн":213,"иша":620,"ише":2071,"ка ":39689,"ив ":1671,"зав":1036,"заб":470,"зад":497,"заг":167,"заз":185,"збу":130,"ид ":941,"збо":708,"зањ":158,"зви":1871,"зве":860,"зва":825,"заш":441,"зач":117,"зац":2143,"зах":250,"зау":217,"зат":525,"зас":773,"зар":677,"зап":3143,"иг ":1521,"зао":108,"зан":1598,"зам":1010,"зал":371,"зак":756,"зби":265,"зај":1067,"збе":327,"зго":178,"згр":481,"зде":188,"зди":94,"зда":994,"здв":108,"зву":144,"зво":1776,"звр":300,"згл":149,"жђа":128,"зећ":156,"зељ":93,"зеј":114,"зет":340,"зер":989,"зел":607,"зен":1338,"зем":1233,"зду":396,"из ":14002,"здр":240,"зид":143,"зив":2326,"зим":680,"зил":849,"зик":1839,"зир":341,"зио":440,"ил ":1271,"зин":605,"ик ":5538,"ин ":3321,"им ":10232,"зиј":1899,"жје":87,"жја":87,"зиц":570,"зиш":103,"зич":979,"зит":988,"ип ":474,"зму":125,"зми":443,"зме":1661,"зма":563,"зло":335,"ио ":7229,"зли":1074,"жња":91,"зла":578,"зна":5348,"зне":283,"зни":867,"зно":12141,"ир ":1849,"зов":1108,"зод":97,"зом":405,"зол":148,"ис ":1009,"зон":699,"зор":503,"зоф":485,"ит ":777,"зра":882,"зре":249,"зро":128,"их ":25752,"зул":422,"зуз":162,"зуч":91,"зур":195,"иц ":714,"зум":622,"зуј":637,"ич ":252,"иш ":433,"шћ":1913,"шњ":2514,"шљ":564,"хе":2594,"хи":2850,"хл":164,"хн":515,"хо":3887,"хр":1677,"хс":192,"хт":445,"ху":632,"фх":116,"ха":4269,"хв":996,"ци":25708,"цн":139,"цк":319,"цл":158,"цр":1743,"цо":468,"цу":6249,"хш":139,"цб":191,"ца":10662,"це":15022,"цг":117,"цв":296,"чл":795,"чм":103,"чн":8899,"чо":443,"чи":8450,"цј":1649,"чк":26459,"чу":1749,"че":8488,"ча":7145,"чв":283,"шо":406,"шп":317,"шн":886,"шк":4442,"чњ":99,"шл":364,"ши":6327,"чј":809,"шч":215,"шт":64295,"шу":691,"ше":5258,"шв":269,"ша":3738,"ск":73819,"рљ":151,"рј":516,"рћ":85,"см":5030,"сл":11320,"рњ":1982,"со":9678,"сн":13876,"ср":17094,"рџ":149,"сп":6049,"св":7773,"сб":1179,"сд":436,"рђ":833,"сг":131,"се":44770,"си":12767,"рш":2918,"са":33351,"рс":13440,"рт":11412,"ру":33347,"рф":1371,"рх":2668,"рц":7114,"рч":1587,"тн":6309,"тм":5097,"тл":1743,"тк":2545,"тс":9086,"тр":21007,"тп":835,"то":37020,"те":36966,"тд":161,"тв":12471,"сј":2502,"ти":79813,"тз":114,"та":69266,"тб":348,"су":21121,"сф":694,"ст":82417,"сх":1012,"сц":1134,"ур":15198,"уп":9618,"ут":7405,"ус":13259,"ум":6429,"ул":6994,"тњ":299,"уо":641,"ун":8172,"уи":452,"уз":4197,"ук":5623,"тљ":189,"тј":481,"уд":6767,"уг":23579,"уж":4717,"уе":639,"уа":1983,"уб":10441,"ув":1407,"ту":17614,"тф":592,"тх":390,"фу":2311,"фт":307,"фс":704,"уџ":113,"фр":3373,"фо":3036,"ућ":1931,"уњ":301,"фл":420,"уљ":267,"уј":6353,"фи":5806,"фе":8951,"уђ":267,"фа":2762,"уч":4458,"уш":3354,"ух":1368,"уц":1110,"уф":272,"џе":228,"џа":369,"џи":363,"њу":2111,"њо":1184,"њс":793,"ћи":4498,"ћк":165,"ће":4723,"ћа":3529,"љу":3257,"љс":735,"јџ":83,"љн":429,"љо":421,"јњ":183,"љк":483,"њи":5652,"ње":25640,"ња":11305,"ћу":1016,"ћн":479,"ћо":225,"јш":265,"ља":6896,"љи":2197,"ље":20267,"јл":855,"јк":776,"ји":23301,"јп":587,"јо":3515,"јн":8532,"јм":1846,"ју":14903,"јт":676,"јс":6913,"јр":268,"јч":534,"јц":574,"јх":352,"јф":416,"ја":44066,"јб":617,"јв":2381,"јг":134,"јд":673,"је":146704,"јз":552,"ђу":3076,"ђо":146,"ђи":555,"ђе":5047,"ђа":2396,"Ју":2303,"Јо":811,"Ја":1517,"Је":12368,"Љу":413,"Њу":211,"Њи":110,"Ње":449,"а ":310619,"Р ":421,"С ":431,"Т ":144,"У ":1965,"Ђа":97,"Ф ":135,"Ђо":174,"Ђу":229,"Х ":378,"Ц ":102,"Ир":324,"Ис":1207,"Ит":448,"Им":514,"Ин":925,"к ":12340,"Ик":105,"Ил":414,"Из":699,"Иг":191,"Ив":390,"Иб":105,"Иа":105,"Зу":210,"Зр":87,"Зо":292,"Зл":467,"Зи":401,"Ле":1326,"Ли":1997,"Ла":2339,"Ку":1279,"Кл":913,"Кн":244,"Ко":7323,"м ":42843,"Кр":2869,"Ке":678,"Кв":150,"Ки":1180,"Ка":4883,"л ":7046,"На":4229,"Не":1665,"Ни":1775,"Мр":192,"Му":528,"о ":97605,"Мо":2251,"Ма":5725,"Мл":210,"Ми":2919,"Ме":3482,"н ":35328,"Ло":1918,"Књ":126,"Лу":1170,"Па":5381,"Пе":2174,"Пи":1995,"Пл":687,"По":7950,"с ":9421,"Оп":1309,"р ":19435,"Ос":1730,"Ор":832,"От":286,"Ох":85,"Оа":99,"Об":952,"Од":735,"Ов":1319,"Ог":84,"Оз":108,"Ом":122,"Он":644,"Ок":613,"Ол":521,"п ":2907,"Но":2845,"Ну":145,"в ":4786,"Ап":242,"Ам":917,"Ан":1871,"Ак":748,"Ал":2798,"Аз":496,"Ад":582,"Ав":348,"Аг":213,"Аб":271,"Ба":6851,"Ах":118,"Аф":342,"Ау":783,"Ат":942,"Ас":250,"Ар":1905,"б ":1710,"Џо":462,"Џи":97,"Џе":413,"Вл":769,"Вр":797,"Во":1407,"д ":41112,"Ве":3053,"Ви":3647,"Бј":217,"Гв":148,"Га":1178,"Ву":646,"Ај":796,"Бл":374,"Бо":7744,"г ":28611,"Бр":3072,"Бе":4333,"Би":2691,"Ва":2337,"Бу":2024,"Ди":1375,"Де":1690,"Ду":1074,"Др":2279,"ж ":448,"До":4484,"Ег":317,"Ед":208,"Ев":965,"Ги":403,"Ге":866,"Гр":3342,"Гл":769,"е ":235734,"Го":3576,"Гу":327,"Да":1824,"Дв":240,"Жи":530,"и ":186242,"Жу":184,"Зб":101,"Зв":315,"За":3384,"Зе":1084,"Ек":250,"Ем":298,"Ел":532,"з ":16545,"Ен":728,"Ер":728,"Еп":349,"Ет":201,"Ес":301,"Еу":95,"Жа":255,"Же":368,"Шл":1326,"Шк":142,"Ши":626,"Шп":414,"Шо":300,"Шм":122,"Шу":240,"Шт":864,"Шв":589,"Ша":1762,"Ше":616,"Ст":2384,"Су":1195,"Та":1272,"Сј":431,"Ти":1785,"Тв":83,"Те":1308,"ф ":1728,"То":1598,"Тр":1845,"Ту":877,"Ук":239,"Уз":91,"Уд":151,"Уг":187,"Ут":114,"Ус":287,"Ур":216,"Уп":276,"х ":27795,"Ун":652,"Ум":120,"Ул":137,"Пр":12883,"Пс":86,"Пу":267,"Пч":135,"Ра":5749,"Ре":7578,"Ри":1147,"Пљ":86,"т ":18408,"Ро":2099,"Ру":1955,"Са":5493,"Св":1412,"Си":1482,"Се":3616,"См":461,"Сл":2083,"Ск":586,"Ср":8976,"Сп":690,"Со":1389,"у ":163872,"Цв":112,"Це":942,"Ца":232,"ш ":2445,"Цр":1575,"Ци":219,"Че":601,"Ча":537,"Чу":105,"Чо":83,"Чи":273,"Фа":508,"ц ":5646,"Фо":828,"Фр":5849,"Фу":461,"Фе":1908,"Фи":915,"Уј":251,"Фл":377,"Ха":1787,"Хр":3553,"ч ":1834,"Хо":2226,"Ху":405,"Хи":666,"Хе":7160,"мб":4820,"ма":59507,"ме":28451,"мд":107,"ми":14537,"мл":591,"мк":113,"књ":1128,"кћ":91,"лм":2649,"лн":10609,"ло":27832,"лп":842,"лс":2735,"лт":3191,"лу":8870,"лф":579,"лх":258,"лц":781,"лш":1376,"лв":428,"лб":1478,"ла":45745,"ле":21446,"лд":2395,"лг":595,"кљ":675,"лк":1280,"ли":48289,"лз":191,"км":1561,"кн":662,"кл":5535,"кр":24074,"кс":5786,"ко":94888,"кх":134,"кт":10243,"ку":10596,"кш":274,"кц":1692,"ка":57877,"ки":34381,"кв":4521,"ке":19190,"ћ ":4823,"иџ":130,"иј":65389,"иљ":1736,"ињ":3046,"ић":7527,"иш":25602,"иђ":179,"њ ":751,"ио":20368,"ип":13525,"им":29885,"ин":88842,"ик":32559,"ил":26990,"зј":195,"иц":17142,"ич":17794,"иф":3473,"их":28103,"ит":23396,"ир":17174,"ис":35020,"ри":59126,"пј":236,"рк":4376,"пљ":819,"рл":2194,"рм":4605,"рн":11097,"ро":38332,"рп":7588,"ра":86875,"рб":5621,"рв":10734,"рг":10628,"рд":4407,"ре":76600,"рж":15319,"рз":1501,"пш":36896,"пч":155,"пр":51210,"пт":1690,"пс":9353,"пф":93,"пу":10490,"пц":293,"пх":131,"ој":81044,"пк":142,"ољ":2891,"пи":15225,"пн":1028,"по":43591,"оњ":2765,"пл":4178,"оћ":912,"па":29402,"пе":8683,"ођ":2674,"ош":4387,"оч":6007,"оц":4534,"ос":40837,"ор":39900,"нџ":226,"оп":51144,"оо":430,"ох":1509,"оф":2614,"оу":1143,"от":10383,"ок":28856,"нљ":100,"ол":17145,"ом":36538,"он":34353,"ож":2953,"оз":8406,"ои":5016,"ов":55896,"нђ":336,"ог":30691,"од":53599,"ое":545,"оа":1915,"об":14901,"нц":9206,"нх":1153,"нш":840,"нч":575,"нт":17280,"нс":27485,"нф":1024,"ну":19119,"но":93266,"џ ":107,"нр":483,"нп":198,"нл":270,"нм":157,"мј":5028,"мљ":1678,"нк":3713,"нз":1151,"ни":94588,"не":40351,"нж":264,"нг":6034,"нд":8864,"нб":3109,"нв":605,"на":115144,"мш":112,"мц":179,"мф":180,"му":5395,"мс":2960,"мр":1007,"мп":3351,"мо":14075,"мн":1634,"ге":7187,"гд":811,"вј":4674,"вљ":4218,"гз":88,"ги":13875,"гн":968,"го":22475,"гл":5570,"гм":306,"гс":345,"гт":294,"гр":22464,"гх":130,"гу":5955,"гш":129,"дг":708,"дв":2905,"дб":2411,"да":35561,"вг":764,"вд":274,"ве":45977,"ви":47733,"бј":964,"бљ":690,"вк":390,"вл":2091,"бњ":88,"вн":21104,"во":25024,"вр":9405,"вс":2479,"ву":5035,"вч":192,"вц":1358,"вш":1624,"га":23320,"гв":336,"бз":146,"би":23648,"бе":9199,"ађ":2448,"бд":119,"аџ":248,"бр":8596,"бн":1295,"бо":7576,"бл":10968,"ањ":12252,"аћ":1384,"ај":30020,"аљ":3397,"бу":6812,"бс":108,"ва":49542,"ад":42109,"ае":747,"аж":2528,"аз":15348,"аб":3706,"ав":58459,"аг":5540,"ам":16256,"ан":80786,"ђ ":91,"ао":8763,"ап":8907,"аи":605,"ак":19402,"ал":40672,"ах":2983,"аф":2148,"ач":24712,"ац":14985,"ас":34373,"ар":47766,"ау":5673,"ат":35129,"ба":12073,"аш":5217,"зр":1331,"иХ":289,"зу":2900,"жј":268,"зи":16003,"љ ":1114,"зо":3900,"зн":18686,"зм":2905,"жњ":238,"зл":2056,"ив":18317,"иг":7473,"иа":237,"иб":2549,"иж":1542,"из":30460,"ид":5409,"ие":162,"жо":177,"ј ":48286,"жр":91,"жу":2380,"еј":4470,"жи":6697,"ењ":5600,"ељ":16871,"жн":3064,"ећ":4760,"за":24448,"зб":1569,"зв":5905,"жђ":219,"зг":986,"зд":2099,"зе":6370,"еф":6020,"еу":929,"ет":26936,"ес":21968,"ер":49978,"еп":15485,"ео":7262,"ен":68315,"ем":41012,"дњ":2216,"ел":25958,"ек":21254,"дљ":162,"дј":352,"еи":490,"ез":21921,"еж":2298,"жд":302,"же":5505,"еђ":4350,"жа":16716,"жб":477,"еч":3067,"еш":5057,"ех":1267,"ец":2261,"дс":5798,"др":23445,"ду":9933,"дн":27401,"дм":1566,"до":15789,"ди":38559,"гњ":86,"дл":918,"гљ":169,"де":26907,"дз":137,"еб":4721,"ев":16201,"ег":19300,"ед":54814,"еа":1573,"дф":213,"дх":157,"ђи ":137,"ђе ":1005,"ђа ":629,"ђор":83,"ђењ":699,"ђив":257,"ђев":272,"ђен":2693,"ђел":175,"ђањ":91,"ђај":335,"ђач":140,"ђар":463,"ђан":442,"ђав":223,"ђу ":1809,"ушћ":97,"ха ":436,"уха":230,"ухо":341,"ухв":534,"уце":178,"уци":804,"уча":728,"учу":355,"учн":763,"учи":534,"уче":1175,"уша":487,"ушт":1121,"ушн":189,"учј":690,"ушк":628,"уше":325,"уши":195,"хе ":209,"уђе":173,"фес":847,"фед":157,"феб":471,"фер":989,"фен":546,"фел":1161,"фек":4515,"фар":175,"фам":285,"фан":502,"фал":512,"фак":464,"фаз":142,"фаб":126,"фло":155,"уће":434,"ућа":330,"уње":115,"уња":132,"фов":93,"ући":802,"ућн":239,"ућу":84,"фиг":103,"ујо":136,"ују":1186,"ује":4514,"фиј":646,"уља":126,"фин":837,"фит":117,"фир":149,"физ":617,"фил":2015,"фик":680,"уја":305,"фиц":170,"фич":243,"фур":105,"фуд":1381,"фун":652,"фхо":95,"фри":594,"фре":160,"фра":725,"фот":183,"фор":2033,"фол":134,"фон":348,"фтв":101,"фск":663,"ху ":202,"фро":157,"фру":1715,"ца ":8462,"хер":413,"хел":163,"ци ":5975,"хео":230,"хем":656,"хен":509,"хев":99,"це ":3788,"хва":984,"хал":405,"хам":134,"хан":387,"хар":360,"хау":611,"хаф":91,"хаг":129,"хаи":135,"хај":1188,"хри":538,"хра":374,"хрв":394,"хоф":175,"хол":640,"хом":223,"хок":106,"хоп":195,"хор":321,"хно":141,"хни":346,"хов":1243,"ход":493,"хла":98,"хиј":1039,"хид":139,"хит":284,"хип":394,"хин":106,"хим":207,"хич":96,"хшт":83,"хум":204,"хте":253,"хсф":89,"хро":311,"цу ":1011,"срп":2456,"стм":124,"сто":18048,"стр":8594,"ств":4838,"сте":5340,"сти":11681,"ста":26605,"сур":199,"сус":280,"суп":505,"сум":91,"сун":122,"сук":182,"сул":102,"суд":629,"сув":97,"суб":180,"суа":202,"стф":499,"сту":2121,"сфо":135,"суј":205,"суђ":102,"сфе":494,"суш":111,"сут":178,"схо":131,"сха":631,"сце":455,"сци":572,"сца":94,"уа ":129,"уб ":744,"тав":5969,"тад":668,"таб":573,"так":3250,"тал":4047,"там":556,"тан":13504,"таз":140,"тау":158,"тат":1577,"таф":105,"тап":100,"тао":464,"уг ":985,"тас":541,"тар":5550,"тач":1717,"тац":1165,"таш":269,"уд ":223,"тањ":1253,"тај":2712,"таљ":208,"тбу":301,"уж ":93,"тву":884,"твр":969,"уе ":91,"тво":2793,"тви":271,"тве":1074,"тва":6461,"теф":236,"тех":457,"теч":207,"теш":240,"тем":4118,"тел":3005,"тео":771,"тен":2926,"тер":8164,"теп":335,"тет":6112,"тес":265,"тед":132,"тег":425,"теж":433,"теи":123,"тез":220,"тек":983,"тев":356,"уз ":472,"тиг":306,"тив":3595,"тид":114,"тиб":241,"ук ":347,"тзв":108,"тељ":925,"тењ":110,"теј":125,"тј ":244,"тећ":113,"тка":747,"тке":238,"тиј":2163,"тиљ":126,"тињ":565,"тић":314,"ум ":1118,"тиш":108,"тич":4693,"тиц":1520,"сја":94,"сје":2389,"тиз":382,"тим":1937,"тин":41889,"тик":1442,"тил":706,"тир":1382,"тис":676,"тио":325,"ул ":556,"тип":584,"тиф":212,"тих":863,"тит":5421,"тлу":83,"уо ":134,"тло":233,"тме":159,"тма":4738,"ткр":261,"тку":270,"тко":705,"ун ":634,"тки":298,"тле":453,"тли":403,"тла":526,"тну":158,"тно":2527,"ур ":576,"тоз":134,"тод":371,"ток":2271,"тол":1112,"тои":1244,"тов":1934,"тог":1199,"тоб":664,"тмо":118,"уп ":544,"тне":555,"тни":2279,"тна":775,"тпу":223,"тпр":138,"тпо":276,"ут ":1197,"тре":2615,"трг":210,"трв":1452,"тра":8057,"трк":102,"три":3258,"трж":83,"тор":6320,"тос":262,"тот":172,"том":2063,"тон":1779,"ус ":844,"топ":627,"тох":508,"точ":3504,"тој":1764,"тпи":109,"тст":153,"ух ":119,"тро":3120,"тру":1654,"тск":8634,"уф ":92,"уш ":86,"тхо":231,"тха":116,"туг":141,"туд":536,"тул":330,"тум":385,"туп":985,"тур":6672,"тун":177,"туа":281,"туј":202,"тфо":135,"тут":228,"тус":171,"туц":235,"тфа":417,"фа ":267,"уво":94,"уви":157,"убј":98,"убљ":204,"уве":506,"ува":512,"уго":4228,"угл":556,"уги":1217,"уге":625,"уга":13358,"уда":1291,"угу":2310,"угр":109,"уал":444,"уац":116,"уан":156,"уар":1082,"уби":913,"убл":6600,"уба":618,"убе":181,"убр":360,"убо":453,"убу":129,"узе":1365,"узд":102,"ужј":100,"узи":1311,"узо":105,"узр":146,"узв":99,"уза":304,"уже":711,"ужи":830,"ужн":2081,"ужа":406,"ужб":369,"уер":98,"уел":160,"уен":239,"уду":209,"удс":407,"уди":1350,"уде":530,"удв":207,"удб":1648,"удр":214,"удо":485,"удн":163,"умб":202,"уме":1673,"уми":518,"ума":1270,"улс":111,"улт":1736,"улу":266,"тњи":173,"ули":840,"улк":196,"укљ":356,"уло":814,"ула":1767,"уле":373,"укс":201,"укт":434,"укр":138,"укц":281,"уку":549,"уки":197,"тљи":143,"уко":1171,"укл":262,"уке":369,"ука":1212,"тје":162,"уис":163,"упо":628,"фт ":102,"упн":661,"упу":323,"упс":196,"упр":1269,"упш":224,"ура":1239,"ург":4051,"урб":165,"ури":4983,"уре":1096,"упљ":436,"урк":103,"уоп":85,"уос":180,"упа":3575,"упе":909,"упи":580,"унс":388,"уно":440,"унк":639,"унц":425,"уну":749,"унт":86,"уоб":88,"умо":176,"умр":281,"умс":122,"уму":713,"уна":2155,"уни":1710,"унг":142,"унд":396,"уне":181,"утр":371,"утс":99,"уту":188,"уст":2999,"усу":97,"усо":230,"усп":514,"уто":1313,"утн":844,"ути":1135,"уте":570,"утв":274,"ута":1163,"урш":100,"урс":607,"урт":171,"уру":316,"урн":928,"уро":361,"усл":737,"уск":5922,"усн":112,"усм":134,"уси":776,"усв":94,"усе":189,"уса":477,"шло":150,"шку":123,"чња":95,"шке":532,"шки":1029,"шко":1898,"шов":116,"шни":421,"шно":249,"шна":90,"шењ":346,"шка":843,"чју":367,"шић":458,"шиј":103,"чје":235,"чја":148,"ших":85,"шиф":1728,"шиц":139,"шир":819,"шин":1269,"шио":130,"шил":146,"шва":162,"шев":523,"шег":353,"шес":305,"шет":83,"шен":657,"шај":207,"шањ":397,"шав":680,"шах":118,"шар":232,"шао":182,"шан":312,"шам":104,"шал":110,"шчи":96,"штр":216,"што":1671,"шту":218,"шум":244,"шпа":238,"шој":123,"штв":5051,"ште":3994,"шти":37563,"шта":15543,"шћу":343,"шће":923,"шћа":591,"шњи":882,"шње":857,"шњо":257,"шња":479,"шље":424,"шља":86,"цге":112,"цве":139,"цва":144,"че ":1166,"цен":2833,"чи ":1951,"цел":553,"цем":639,"цег":5749,"цес":608,"цер":167,"цеп":196,"циљ":469,"циј":13318,"циз":111,"циг":120,"цив":149,"цип":664,"цио":2122,"цис":184,"цир":296,"цил":157,"цик":239,"цин":417,"цим":814,"цит":296,"циф":257,"ча ":1135,"цањ":160,"цбу":108,"цај":333,"цар":1250,"цам":251,"цус":5145,"цла":132,"цко":142,"цна":120,"цог":137,"цом":173,"црв":233,"црн":309,"црк":917,"чу ":279,"црт":240,"чењ":1369,"чев":703,"чег":147,"чем":184,"чен":757,"чек":99,"чел":592,"чес":1070,"чер":91,"чео":115,"ши ":1042,"чет":1463,"чеш":599,"чко":16119,"чки":5326,"чке":2191,"чла":792,"чку":322,"чић":352,"чињ":485,"цје":1643,"чиј":1063,"чка":2488,"чим":154,"чил":179,"чио":107,"чин":2262,"чив":224,"чит":1082,"чиц":356,"ша ":940,"чвр":128,"ше ":2479,"чва":109,"чај":1188,"час":347,"чак":300,"чар":1858,"чан":988,"чав":1025,"чне":725,"чна":776,"чов":351,"чни":2024,"чну":146,"чно":5223,"чув":312,"чуј":364,"чун":594,"шу ":312,"џер":98,"џиј":83,"џа ":116,"ћим":236,"ћин":617,"ћих":347,"ћел":450,"ћен":999,"ћем":203,"ћер":180,"ћењ":211,"ћев":207,"ћег":174,"ћој":145,"ћно":252,"ћни":156,"ђус":133,"ђут":104,"ђун":548,"ђуј":344," ар":1531," ас":1495," ат":789," ау":1060," аф":120," ба":1782," аг":204," ав":1075," ае":132," ад":525," аз":106," ал":2165," ак":1263," ан":1456," ам":1558," ап":880," бу":724," ва":1641," би":13849," бе":1509," бр":3177," бо":2496," бл":699," ву":141," га":1017," ви":3100," ве":6374," во":2889," вр":3580," вл":1525," дв":1695," да":6529,"њск":762," гу":273," го":5940," гл":2733," гр":13040," ге":1414," гд":614," вј":1068," ги":217,"ћу ":880," ев":376," ег":176," дн":118," до":8150," др":17143," ду":1399," де":12761," ди":3032," же":911," жа":185,"њој":397," еф":184," ес":167," ет":301," еп":599," ер":161," ен":4749," ел":1099," ем":401," дј":115," ек":993," зд":192," зг":134," зе":1269,"њом":267," за":14329," зв":844," зб":573," жу":2033,"њов":202," жр":88," жи":3507," зи":192," зл":353," зо":272," зн":1754," зр":158,"њев":193," иа":112,"њег":1511," иг":1416," ид":487," из":20909," ил":6136," ик":91," ин":3129," им":4445," ит":477," ис":6583," ир":131," их":414,"ће ":2252," ка":9529," ки":953," кв":334," ке":139," кр":4983," ко":34802," кн":423," км":304," кл":2025," ку":1563," ла":2233," кљ":83," ли":3146," ле":1584," лу":259,"њиц":291," књ":1120," кћ":91," ло":1091,"њих":951," ме":6800," ми":3211,"њић":169," мл":339," ма":5632," мо":4593," мн":508," му":1999," мр":337," ни":1843,"њиж":739," не":7050,"њим":687,"њин":185," мј":4621," на":44905,"њив":145,"њиг":412," ну":317," но":3063," нп":131," ол":413," ок":19137," он":950," ом":368," оз":712," ог":275," ов":1780," од":28239," об":6316," ош":91," оц":139," оч":225," оф":198," от":977,"њер":162,"ћи ":3090," ор":2923,"њен":1992," ос":5852,"њем":13062," оп":36889," по":35696,"њењ":256,"�":306," пл":2060," пи":1443," пе":3239," па":2758," Ре":7567," Ра":5738," Ро":2092," Ри":1146," Пљ":86," Пу":260," Пр":12819," Пс":86," Пч":135," Пе":2104," Па":5377," По":7935," с ":397," Пл":685," Пи":1992," Ох":85," От":286," Ос":1726," Ор":831," Оп":1276," Те":1302," Ти":1784," Сј":431," То":1597," Тр":1837," Ст":2368," Су":1194," Та":1268," Св":1404," Си":1481," Се":3613," Сл":2081," См":461," Ск":586," Сп":690," Ср":8968," Со":1387," у ":76526," Ру":1950," Са":5488," Фр":5848," Фу":460," Фо":827," Уј":250," Фи":914," Фл":377," Фе":1907," Фа":506,"ћањ":215,"ћај":354," Ут":114," Ус":286," Ур":216," Уп":276," Ун":652," Ум":118," Ул":135," Ук":237," Уз":91," Уд":150," Уг":187," Ту":874," Цр":1572,"ћав":181," Ци":218," Ца":232," Цв":112," Це":942," Хр":3549," Хо":2223," Ху":404," Хи":661," Хе":7159," Ха":1786,"ћан":574,"њуј":187," Ше":614," Шв":589," Ша":1760," Шо":300," Шп":414," Шм":122," Шк":141," Шл":1326," Ши":625," Шт":864," Шу":240," Че":600," Ча":535," Чо":83," Чи":272," Чу":105," Ба":6848," Аф":342," Ах":117," Ат":942," Ау":782," Ар":1901," Ас":250," Ап":241," Ан":1869," Ам":916," Ал":2792," Ак":746,"љиш":98," Аз":496,"љиц":161," Ад":581," Аг":213," Ав":348," Аб":270,"љим":98," Ва":2332,"љин":408,"љив":745," Бу":2024," Бо":7741," Бр":3064," Ај":794," Бл":374," Бе":4332," Би":2681,"јње":140," а ":3795,"љке":117," Џи":97," Џо":461,"љка":256," Џе":412,"љни":159," Ед":207," Ег":317," Ев":963,"љно":170," Ди":1374," Де":1685," Ду":1072," Др":2274," До":4482," Же":368," Жа":255," Ек":250," Ел":532," Ем":298," Ес":301," Ет":200," Еу":95," Ен":715," Еп":349," Ер":727," Ву":631," Гв":147," Га":1175," Ве":3050," Ви":3645," Бј":217," Вл":762," Вр":797," Во":1406," д ":244," Гу":327,"љон":115,"љоп":103," Да":1819," Дв":240," Ге":857," Ги":403," Гл":761," е ":253," Го":3576," Гр":3337," Ис":1205," Ир":324,"љас":133," Ит":448,"љал":133," Ин":917,"љам":117," Им":500,"љан":705,"љао":106," Ил":412,"љак":314," Ик":105,"љаш":109,"љач":143," Ки":1180,"љав":466," Ке":678," Кв":150," Ка":4863," и ":42425," Жу":183," Жи":529,"ње ":8309," За":3380," Зб":101," Зв":313," Зе":1083," Зр":87," Зу":210,"љањ":379," Зи":400,"љај":470," Зо":292," Зл":467," Ив":384," Иг":188," Иа":105," Иб":105," Из":696," Мр":192," Му":523," о ":1337," Мо":2248,"љев":1384," На":4218," Не":1661," Ни":1773," Ну":144," п ":1147," Но":2843," Од":732," Ог":84," Ов":1314," Об":952," Оа":97," Он":642," Ом":122," Ол":520," Ок":611," Оз":107," Кл":909," Кн":244," Ко":7314," Кр":2862," Ку":1279," Ла":2334," Ле":1325," Ли":1997," н ":236,"њи ":1901," Ло":1918," Књ":125,"љен":9209,"љем":246," Лу":1168," Ма":5708," Ме":3478,"љењ":432," Мл":210," Ми":2909," У ":1834," С ":120,"ћа ":2102,"њав":306," Ђу":228,"њац":140,"њач":180," Ђо":174,"њан":194,"њак":423," Ђа":97," Љу":412,"њу ":1891," Ја":1517," Је":12367," Ју":2300,"љст":94,"љск":641," Јо":807,"људ":763,"љуб":277,"ључ":650,"љуј":500," Њи":110," Ње":449," Њу":211,"јна":3212,"јнб":154,"јно":645,"јну":95,"јни":1500,"јне":363,"јма":315,"јмо":183,"јмс":112,"јло":133,"јли":84,"јле":319,"јка":209,"јко":160,"јке":100,"јис":141,"јих":1002,"јић":104," А ":144,"јил":86,"јин":1068,"јим":1235,"јио":131," Б ":110,"јзе":206,"јзн":168,"јес":4944,"јет":1944,"јен":3254,"љи ":494,"јеп":114,"јер":1439,"јеч":245,"јеш":1172,"јењ":122,"јељ":149,"јећ":109,"јез":2070,"јек":1138,"јел":2648,"јем":1164,"јев":2265,"јег":367,"јед":10980,"јдо":91,"јду":106,"јде":199," М ":110,"јво":585,"ље ":8612,"јви":622,"јве":956,"јва":185,"јбо":385,"јањ":589,"јај":268,"јаш":298,"јак":354,"јал":3500,"јам":1141,"јан":2625,"јао":110,"јап":268,"јар":388,"јас":623,"јат":622,"јац":247,"јач":377,"јаг":137,"јав":1529,"јад":132,"ља ":3633,"ња ":9657,"јшт":110,"јца":205,"јцн":120,"јче":458,"јућ":704,"јфе":389,"јхс":103,"јте":185,"јул":555,"јум":460,"јун":673,"јур":125,"јус":143,"јут":86,"југ":1799,"јуж":1274,"љу ":927,"јра":101,"јск":5249,"јст":654,"јсе":169,"јси":92,"јон":495,"јом":1166,"јор":180,"јнш":97,"јој":768,"јпо":385,"још":604,"јм ":973,"јн ":2046,"ји ":19262,"је ":112273,"јд ":136,"ја ":30644,"ју ":8819,"јц ":119,"јс ":427,"јт ":210},"n_words":[7556811,8870200,7383394],"name":"sr","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/stop_words b/contrib/languages-data/stop_words
new file mode 100644
index 0000000..0aad88d
--- /dev/null
+++ b/contrib/languages-data/stop_words
@@ -0,0 +1,4153 @@
+{
+ "fr": [
+ "seront",
+ "eûmes",
+ "mes",
+ "auront",
+ "étante",
+ "notre",
+ "été",
+ "eusses",
+ "même",
+ "vous",
+ "furent",
+ "nous",
+ "votre",
+ "avions",
+ "ayants",
+ "par",
+ "eussions",
+ "eues",
+ "moi",
+ "fut",
+ "fussiez",
+ "serions",
+ "sommes",
+ "qu",
+ "serons",
+ "serai",
+ "aurions",
+ "ayantes",
+ "qui",
+ "aurons",
+ "avaient",
+ "aurai",
+ "fûtes",
+ "avez",
+ "sont",
+ "aurait",
+ "aura",
+ "eusse",
+ "eût",
+ "étiez",
+ "était",
+ "ait",
+ "étés",
+ "tes",
+ "dans",
+ "aux",
+ "serait",
+ "est",
+ "sera",
+ "aurais",
+ "elle",
+ "êtes",
+ "auraient",
+ "eue",
+ "étée",
+ "avais",
+ "fussent",
+ "eus",
+ "ayant",
+ "serais",
+ "toi",
+ "ayez",
+ "soyons",
+ "seraient",
+ "fusses",
+ "étions",
+ "avons",
+ "eurent",
+ "auriez",
+ "aie",
+ "une",
+ "seriez",
+ "fûmes",
+ "eux",
+ "leur",
+ "ton",
+ "pour",
+ "étaient",
+ "étant",
+ "ses",
+ "ont",
+ "soit",
+ "eûtes",
+ "soyez",
+ "soient",
+ "aient",
+ "eussiez",
+ "eut",
+ "avec",
+ "sur",
+ "fussions",
+ "seras",
+ "mon",
+ "ayante",
+ "ayons",
+ "ces",
+ "aviez",
+ "aies",
+ "auras",
+ "fusse",
+ "fût",
+ "avait",
+ "suis",
+ "serez",
+ "étantes",
+ "fus",
+ "pas",
+ "étants",
+ "eussent",
+ "aurez",
+ "étées",
+ "étais"
+ ],
+ "id": [
+ "mungkinkah",
+ "gunakan",
+ "sebelumnya",
+ "penting",
+ "percuma",
+ "sekecil",
+ "masalahnya",
+ "teringat",
+ "sedangkan",
+ "melalui",
+ "boleh",
+ "kamilah",
+ "depan",
+ "amatlah",
+ "bermaksud",
+ "rasanya",
+ "harusnya",
+ "bermacam-macam",
+ "betulkah",
+ "berarti",
+ "dari",
+ "lanjut",
+ "haruslah",
+ "tegasnya",
+ "keinginan",
+ "mengibaratkan",
+ "dikatakan",
+ "tadinya",
+ "terdahulu",
+ "tunjuk",
+ "naik",
+ "yang",
+ "berapakah",
+ "berikutnya",
+ "diakhiri",
+ "sekaligus",
+ "mengibaratkannya",
+ "sayalah",
+ "guna",
+ "bukanlah",
+ "terjadilah",
+ "belumlah",
+ "bagaimanapun",
+ "sesampai",
+ "semisal",
+ "bukannya",
+ "selanjutnya",
+ "kedua",
+ "kenapa",
+ "seseorang",
+ "menyampaikan",
+ "kembali",
+ "menjadi",
+ "benarlah",
+ "itukah",
+ "pastilah",
+ "sekadar",
+ "memberi",
+ "katanya",
+ "bermula",
+ "tinggi",
+ "meski",
+ "sebegini",
+ "digunakan",
+ "tersampaikan",
+ "bawah",
+ "seenaknya",
+ "bahwa",
+ "diberikannya",
+ "sejak",
+ "menginginkan",
+ "setempat",
+ "bagai",
+ "mempergunakan",
+ "mau",
+ "disampaikan",
+ "banyak",
+ "dipunyai",
+ "memihak",
+ "seperti",
+ "mengucapkannya",
+ "sekalipun",
+ "sajalah",
+ "ialah",
+ "dibuat",
+ "dimulainya",
+ "dia",
+ "ditunjuk",
+ "pula",
+ "ada",
+ "inikah",
+ "kepadanya",
+ "bagaikan",
+ "agak",
+ "menyiapkan",
+ "kamu",
+ "seringnya",
+ "sekiranya",
+ "bertanya",
+ "suatu",
+ "aku",
+ "rata",
+ "ibaratkan",
+ "meyakinkan",
+ "tentulah",
+ "wong",
+ "sepanjang",
+ "tak",
+ "tidakkah",
+ "memerlukan",
+ "menunjuki",
+ "sendirinya",
+ "padanya",
+ "keseluruhannya",
+ "keadaan",
+ "sangat",
+ "melakukan",
+ "masing-masing",
+ "sebegitu",
+ "tentunya",
+ "secara",
+ "bung",
+ "begitu",
+ "dua",
+ "disini",
+ "sebetulnya",
+ "lainnya",
+ "masih",
+ "tempat",
+ "mengatakannya",
+ "jelaslah",
+ "semacam",
+ "memungkinkan",
+ "tepat",
+ "terhadapnya",
+ "datang",
+ "sinilah",
+ "turut",
+ "kapankah",
+ "sejauh",
+ "tandasnya",
+ "kelihatannya",
+ "semuanya",
+ "jelasnya",
+ "ditunjukkan",
+ "jumlah",
+ "awal",
+ "lalu",
+ "asalkan",
+ "berujar",
+ "karenanya",
+ "memperlihatkan",
+ "secukupnya",
+ "mendatangi",
+ "sehingga",
+ "apakah",
+ "hingga",
+ "bilakah",
+ "atau",
+ "tertuju",
+ "menyatakan",
+ "tetap",
+ "ingin",
+ "kalaulah",
+ "perlukah",
+ "tiba",
+ "ujar",
+ "mempunyai",
+ "asal",
+ "dong",
+ "inginkan",
+ "menunjuknya",
+ "akankah",
+ "soal",
+ "diperbuat",
+ "atas",
+ "macam",
+ "menanyakan",
+ "hendak",
+ "buat",
+ "tambahnya",
+ "diminta",
+ "semata",
+ "jadinya",
+ "ibaratnya",
+ "akhirnya",
+ "seterusnya",
+ "setengah",
+ "menjawab",
+ "ternyata",
+ "tandas",
+ "ditanyakan",
+ "hendaklah",
+ "nyaris",
+ "jumlahnya",
+ "cukuplah",
+ "kemudian",
+ "ketika",
+ "begitupun",
+ "ditunjuknya",
+ "tanpa",
+ "kesampaian",
+ "mempersiapkan",
+ "memisalkan",
+ "diketahui",
+ "umumnya",
+ "inginkah",
+ "seorang",
+ "sesudah",
+ "cuma",
+ "perlu",
+ "berkenaan",
+ "berlalu",
+ "seberapa",
+ "jawab",
+ "waktunya",
+ "pihak",
+ "sebagaimana",
+ "maka",
+ "ataupun",
+ "memperkirakan",
+ "tetapi",
+ "bakalan",
+ "kini",
+ "sedang",
+ "siapapun",
+ "dipersoalkan",
+ "yakin",
+ "apalagi",
+ "sebagainya",
+ "terus",
+ "pun",
+ "bagaimanakah",
+ "mana",
+ "dekat",
+ "nyatanya",
+ "terlalu",
+ "tambah",
+ "apa",
+ "jadilah",
+ "sedikit",
+ "tidak",
+ "masing",
+ "keterlaluan",
+ "serta",
+ "hendaknya",
+ "serupa",
+ "semampu",
+ "ibarat",
+ "sampai-sampai",
+ "diakhirinya",
+ "berkeinginan",
+ "dini",
+ "sekitar",
+ "disinilah",
+ "menunjukkan",
+ "sendirian",
+ "seingat",
+ "ibu",
+ "ditujukan",
+ "apaan",
+ "tuturnya",
+ "sekurangnya",
+ "masihkah",
+ "melainkan",
+ "ditunjuki",
+ "terdapat",
+ "mirip",
+ "sangatlah",
+ "sebuah",
+ "memintakan",
+ "disebut",
+ "dikira",
+ "begitulah",
+ "diucapkannya",
+ "mempertanyakan",
+ "sudahlah",
+ "balik",
+ "bersama",
+ "kapan",
+ "seluruh",
+ "dapat",
+ "didapat",
+ "tiap",
+ "kira-kira",
+ "dimintai",
+ "malahan",
+ "karena",
+ "kala",
+ "mengucapkan",
+ "menghendaki",
+ "mengatakan",
+ "memulai",
+ "menuju",
+ "semakin",
+ "bersama-sama",
+ "ucap",
+ "menunjuk",
+ "mengungkapkan",
+ "terutama",
+ "keseluruhan",
+ "diperlihatkan",
+ "toh",
+ "memastikan",
+ "beginian",
+ "bisakah",
+ "apabila",
+ "satu",
+ "lanjutnya",
+ "terjadinya",
+ "katakanlah",
+ "ditunjukkannya",
+ "mulanya",
+ "kita",
+ "bermacam",
+ "selama-lamanya",
+ "itu",
+ "melihat",
+ "beginikah",
+ "keluar",
+ "pernah",
+ "sesuatunya",
+ "berjumlah",
+ "anda",
+ "sekurang-kurangnya",
+ "nantinya",
+ "belakangan",
+ "antar",
+ "kira",
+ "sering",
+ "apatah",
+ "entahlah",
+ "bolehkah",
+ "sedemikian",
+ "sesegera",
+ "oleh",
+ "terlebih",
+ "dikarenakan",
+ "cara",
+ "beri",
+ "semata-mata",
+ "lama",
+ "betul",
+ "dibuatnya",
+ "tanyanya",
+ "nah",
+ "misal",
+ "persoalan",
+ "mulailah",
+ "pertanyaan",
+ "kepada",
+ "terhadap",
+ "menanti",
+ "dijawab",
+ "walaupun",
+ "berkehendak",
+ "khususnya",
+ "sebut",
+ "biasa",
+ "kalaupun",
+ "seharusnya",
+ "berikan",
+ "sebabnya",
+ "misalkan",
+ "dimulailah",
+ "berbagai",
+ "masa",
+ "hanya",
+ "mengetahui",
+ "sekali",
+ "ini",
+ "dirinya",
+ "diingat",
+ "menggunakan",
+ "terlihat",
+ "sesudahnya",
+ "dilalui",
+ "sebaik",
+ "akhir",
+ "mampu",
+ "jika",
+ "antara",
+ "justru",
+ "bersiap-siap",
+ "diibaratkannya",
+ "lagi",
+ "setidak-tidaknya",
+ "manakala",
+ "luar",
+ "kebetulan",
+ "dikatakannya",
+ "setiba",
+ "besar",
+ "merasa",
+ "inilah",
+ "sesama",
+ "sejenak",
+ "jauh",
+ "hari",
+ "semasa",
+ "tertentu",
+ "yaitu",
+ "saatnya",
+ "bisa",
+ "sebagai",
+ "kemungkinan",
+ "dimaksudnya",
+ "cukupkah",
+ "sebanyak",
+ "kecil",
+ "sewaktu",
+ "kalau",
+ "walau",
+ "baru",
+ "berawal",
+ "menantikan",
+ "umum",
+ "sekitarnya",
+ "jelas",
+ "semua",
+ "itulah",
+ "sudahkah",
+ "semampunya",
+ "selalu",
+ "jangan",
+ "sama-sama",
+ "semaunya",
+ "sini",
+ "sedikitnya",
+ "sampai",
+ "tahu",
+ "diberikan",
+ "diingatkan",
+ "tentu",
+ "tentang",
+ "segala",
+ "segera",
+ "lagian",
+ "seluruhnya",
+ "sebenarnya",
+ "jelaskan",
+ "tapi",
+ "menaiki",
+ "tersebutlah",
+ "ditambahkan",
+ "sementara",
+ "makanya",
+ "kalian",
+ "enggak",
+ "kurang",
+ "diinginkan",
+ "berupa",
+ "seperlunya",
+ "dijelaskannya",
+ "punya",
+ "dimaksudkannya",
+ "memang",
+ "setelah",
+ "menyeluruh",
+ "berlainan",
+ "mengingatkan",
+ "pukul",
+ "saja",
+ "dilakukan",
+ "berapalah",
+ "paling",
+ "meyakini",
+ "menuturkan",
+ "makin",
+ "sekali-kali",
+ "olehnya",
+ "tahun",
+ "ditanya",
+ "daripada",
+ "menyangkut",
+ "ungkapnya",
+ "bagian",
+ "lamanya",
+ "terbanyak",
+ "caranya",
+ "malah",
+ "kamulah",
+ "siap",
+ "ingat-ingat",
+ "berturut",
+ "saling",
+ "manalagi",
+ "melihatnya",
+ "tampaknya",
+ "kapanpun",
+ "dimungkinkan",
+ "dituturkannya",
+ "sesuatu",
+ "diperlukan",
+ "diri",
+ "sekalian",
+ "mereka",
+ "kelihatan",
+ "bahkan",
+ "baik",
+ "namun",
+ "terjadi",
+ "dulu",
+ "mempersoalkan",
+ "keduanya",
+ "jawaban",
+ "empat",
+ "kok",
+ "didatangkan",
+ "bukan",
+ "berlangsung",
+ "begini",
+ "saya",
+ "berakhir",
+ "dipergunakan",
+ "wahai",
+ "belum",
+ "termasuk",
+ "lebih",
+ "pantas",
+ "mengira",
+ "terakhir",
+ "bakal",
+ "merupakan",
+ "panjang",
+ "ucapnya",
+ "ke",
+ "benar",
+ "bapak",
+ "sepihak",
+ "pasti",
+ "adapun",
+ "amat",
+ "mengapa",
+ "selamanya",
+ "bolehlah",
+ "diperbuatnya",
+ "kami",
+ "bagi",
+ "maupun",
+ "beberapa",
+ "mendatangkan",
+ "sepantasnyalah",
+ "tegas",
+ "kiranya",
+ "siapa",
+ "diantara",
+ "menandaskan",
+ "dalam",
+ "disebutkan",
+ "berapapun",
+ "ditegaskan",
+ "demikian",
+ "meminta",
+ "harus",
+ "telah",
+ "mula",
+ "menanyai",
+ "selain",
+ "sama",
+ "sebutlah",
+ "bukankah",
+ "seolah",
+ "sela",
+ "membuat",
+ "selaku",
+ "seketika",
+ "kata",
+ "hanyalah",
+ "biasanya",
+ "diibaratkan",
+ "diketahuinya",
+ "sebutnya",
+ "ditanyai",
+ "sebelum",
+ "ditandaskan",
+ "menjelaskan",
+ "benarkah",
+ "dimisalkan",
+ "pentingnya",
+ "sebab",
+ "menegaskan",
+ "masalah",
+ "mengenai",
+ "tanyakan",
+ "tampak",
+ "disebutkannya",
+ "misalnya",
+ "juga",
+ "entah",
+ "tanya",
+ "lewat",
+ "semula",
+ "siapakah",
+ "sebaliknya",
+ "ungkap",
+ "pertama",
+ "dikerjakan",
+ "berdatangan",
+ "kemungkinannya",
+ "enggaknya",
+ "mulai",
+ "padahal",
+ "sambil",
+ "semisalnya",
+ "katakan",
+ "berikut",
+ "mendapatkan",
+ "supaya",
+ "usah",
+ "agaknya",
+ "beginilah",
+ "tiga",
+ "tiba-tiba",
+ "waduh",
+ "usai",
+ "dimaksud",
+ "kitalah",
+ "demikianlah",
+ "kelima",
+ "tadi",
+ "seolah-olah",
+ "nanti",
+ "selama",
+ "hampir",
+ "bulan",
+ "dijelaskan",
+ "akan",
+ "seusai",
+ "sekadarnya",
+ "andalah",
+ "merekalah",
+ "pertama-tama",
+ "pertanyakan",
+ "rasa",
+ "bila",
+ "berkali-kali",
+ "diperkirakan",
+ "diucapkan",
+ "terasa",
+ "berapa",
+ "jangankan",
+ "kelamaan",
+ "agar",
+ "ataukah",
+ "pak",
+ "diberi",
+ "sampaikan",
+ "setinggi",
+ "menyebutkan",
+ "menurut",
+ "dengan",
+ "menanya",
+ "lima",
+ "sepertinya",
+ "dituturkan",
+ "diperlukannya",
+ "berakhirlah",
+ "dimulai",
+ "sempat",
+ "sejumlah",
+ "setidaknya",
+ "dialah",
+ "adalah",
+ "berlebihan",
+ "lah",
+ "sudah",
+ "berturut-turut",
+ "ujarnya",
+ "mengakhiri",
+ "awalnya",
+ "setiap",
+ "sendiri",
+ "sana",
+ "diungkapkan",
+ "sebaik-baiknya",
+ "memperbuat",
+ "berkata",
+ "meskipun",
+ "tutur",
+ "diantaranya",
+ "yakni",
+ "akulah",
+ "mungkin",
+ "demi",
+ "bersiap",
+ "dimaksudkan",
+ "menambahkan",
+ "berakhirnya",
+ "mengerjakan",
+ "tidaklah",
+ "setibanya",
+ "semasih",
+ "kasus",
+ "janganlah",
+ "jadi",
+ "jikalau",
+ "sebisanya",
+ "dilihat",
+ "minta",
+ "sebaiknya",
+ "soalnya",
+ "mendatang",
+ "bertanya-tanya",
+ "dipertanyakan",
+ "ikut",
+ "terkira",
+ "terdiri",
+ "dipastikan",
+ "antaranya",
+ "sesekali",
+ "rupanya",
+ "hal",
+ "berada",
+ "sesaat",
+ "tersebut",
+ "sebesar",
+ "adanya",
+ "mampukah",
+ "ingat",
+ "sebagian",
+ "bekerja",
+ "untuk",
+ "waktu",
+ "pihaknya",
+ "jawabnya",
+ "bahwasanya",
+ "wah",
+ "mengingat",
+ "segalanya",
+ "menanti-nanti",
+ "pada",
+ "kinilah",
+ "memberikan",
+ "dahulu",
+ "mendapat",
+ "begitukah",
+ "perlunya",
+ "cukup",
+ "belakang",
+ "teringat-ingat",
+ "artinya",
+ "bertutur",
+ "sepantasnya",
+ "lain",
+ "tengah",
+ "bagaimana",
+ "akhiri"
+ ],
+ "en": [
+ "until",
+ "his",
+ "those",
+ "who",
+ "any",
+ "you've",
+ "it's",
+ "that",
+ "aren",
+ "after",
+ "down",
+ "couldn",
+ "itself",
+ "ours",
+ "about",
+ "whom",
+ "won't",
+ "isn't",
+ "what",
+ "this",
+ "wouldn't",
+ "than",
+ "its",
+ "don't",
+ "too",
+ "such",
+ "wasn",
+ "by",
+ "doing",
+ "you'd",
+ "you",
+ "some",
+ "where",
+ "you'll",
+ "against",
+ "she",
+ "been",
+ "into",
+ "but",
+ "that'll",
+ "they",
+ "more",
+ "wasn't",
+ "and",
+ "here",
+ "to",
+ "during",
+ "him",
+ "my",
+ "our",
+ "she's",
+ "if",
+ "yourself",
+ "hers",
+ "there",
+ "out",
+ "yours",
+ "aren't",
+ "should've",
+ "because",
+ "own",
+ "couldn't",
+ "these",
+ "should",
+ "ourselves",
+ "few",
+ "them",
+ "haven",
+ "between",
+ "both",
+ "shouldn't",
+ "just",
+ "wouldn",
+ "when",
+ "didn",
+ "off",
+ "hasn",
+ "how",
+ "did",
+ "once",
+ "above",
+ "yourselves",
+ "you're",
+ "other",
+ "below",
+ "shan't",
+ "with",
+ "don",
+ "haven't",
+ "through",
+ "again",
+ "each",
+ "ain",
+ "then",
+ "myself",
+ "the",
+ "won",
+ "we",
+ "can",
+ "now",
+ "their",
+ "herself",
+ "only",
+ "mustn",
+ "very",
+ "shan",
+ "themselves",
+ "were",
+ "didn't",
+ "hasn't",
+ "nor",
+ "not",
+ "does",
+ "weren't",
+ "mightn't",
+ "having",
+ "doesn't",
+ "needn't",
+ "further",
+ "while",
+ "before",
+ "hadn't",
+ "mustn't",
+ "why",
+ "theirs",
+ "your",
+ "himself",
+ "which",
+ "being",
+ "from",
+ "up",
+ "it",
+ "same",
+ ],
+ "ar": [
+ "هنالك",
+ "لم",
+ "أنت",
+ "بكما",
+ "هيا",
+ "اللتان",
+ "اللواتي",
+ "بمن",
+ "وهو",
+ "فلا",
+ "عليه",
+ "بهما",
+ "لعل",
+ "حيث",
+ "اللتين",
+ "هنا",
+ "كأي",
+ "لاسيما",
+ "كيت",
+ "آها",
+ "سوف",
+ "لستما",
+ "أنا",
+ "الذي",
+ "لستم",
+ "إلا",
+ "ذاك",
+ "هيت",
+ "إيه",
+ "كلاهما",
+ "فإن",
+ "والذي",
+ "هو",
+ "الذين",
+ "يا",
+ "فإذا",
+ "مما",
+ "هؤلاء",
+ "إذ",
+ "ولكن",
+ "أو",
+ "هذان",
+ "عند",
+ "وما",
+ "التي",
+ "أينما",
+ "بها",
+ "ماذا",
+ "لسن",
+ "إنه",
+ "بس",
+ "ثمة",
+ "بنا",
+ "كليكما",
+ "فيه",
+ "لدى",
+ "لستن",
+ "هناك",
+ "ذان",
+ "تلكم",
+ "بي",
+ "منه",
+ "لهم",
+ "ليسا",
+ "إليك",
+ "تين",
+ "وإن",
+ "اللذين",
+ "إذا",
+ "لكم",
+ "حين",
+ "وإذا",
+ "عدا",
+ "تلكما",
+ "قد",
+ "أكثر",
+ "به",
+ "دون",
+ "إما",
+ "ها",
+ "عل",
+ "هاتين",
+ "ذلكن",
+ "على",
+ "كلتا",
+ "كليهما",
+ "لهن",
+ "هذين",
+ "بك",
+ "ذينك",
+ "كأنما",
+ "كيفما",
+ "اللذان",
+ "لكن",
+ "مذ",
+ "لما",
+ "ولا",
+ "آي",
+ "هاته",
+ "أولاء",
+ "نحن",
+ "هي",
+ "عليك",
+ "تينك",
+ "هذه",
+ "كي",
+ "ذانك",
+ "أي",
+ "هاتي",
+ "أيها",
+ "ذلكم",
+ "هن",
+ "آه",
+ "فيم",
+ "ذين",
+ "نحو",
+ "إذما",
+ "إليكن",
+ "إلى",
+ "ليسوا",
+ "أن",
+ "هاهنا",
+ "كذلك",
+ "ذلكما",
+ "عما",
+ "هكذا",
+ "لسنا",
+ "أوه",
+ "لئن",
+ "لكيلا",
+ "فيها",
+ "سوى",
+ "ذواتي",
+ "لا",
+ "أنتن",
+ "إي",
+ "ذي",
+ "هل",
+ "اللتيا",
+ "إن",
+ "في",
+ "حاشا",
+ "كل",
+ "ذه",
+ "حبذا",
+ "خلا",
+ "بما",
+ "كأين",
+ "ذواتا",
+ "والذين",
+ "منذ",
+ "لوما",
+ "هذا",
+ "بكن",
+ "هاتان",
+ "لك",
+ "أنتما",
+ "أقل",
+ "اللائي",
+ "أنتم",
+ "كذا",
+ "لن",
+ "نعم",
+ "بهن",
+ "له",
+ "أولئك",
+ "أف",
+ "ذوا",
+ "ته",
+ "لست",
+ "بل",
+ "كما",
+ "لكي",
+ "مهما",
+ "بلى",
+ "حيثما",
+ "عن",
+ "ومن",
+ "اللاتي",
+ "هما",
+ "ذا",
+ "بين",
+ "شتان",
+ "لي",
+ "ممن",
+ "تي",
+ "بهم",
+ "حتى",
+ "كلما",
+ "ليستا",
+ "ذات",
+ "ليت",
+ "إنما",
+ "هيهات",
+ "فيما",
+ "ريث",
+ "بعض",
+ "لنا",
+ "ما",
+ "ليست",
+ "مع",
+ "ذو",
+ "لها",
+ "إذن",
+ "عسى",
+ "أم",
+ "لولا",
+ "هذي",
+ "إليكما",
+ "هم",
+ "ذلك",
+ "بيد",
+ "فمن",
+ "بخ",
+ "ولو",
+ "مه",
+ "تلك",
+ "إليكم",
+ "من",
+ "بعد",
+ "متى",
+ "ليس",
+ "لهما",
+ "منها",
+ "هلا",
+ "بماذا",
+ "هاك",
+ "لكنما",
+ "لكما",
+ "إنا",
+ "غير",
+ "كيف",
+ "ألا",
+ "ثم",
+ "كأن",
+ "كلا",
+ "لو",
+ "وإذ"
+ ],
+ "no": [
+ "noka",
+ "ingi",
+ "kva",
+ "deira",
+ "uten",
+ "vært",
+ "noko",
+ "noe",
+ "noen",
+ "mellom",
+ "somt",
+ "varte",
+ "nokon",
+ "hossen",
+ "kvifor",
+ "deg",
+ "medan",
+ "siden",
+ "å",
+ "hvem",
+ "sånn",
+ "deires",
+ "korleis",
+ "vere",
+ "blei",
+ "kvi",
+ "inkje",
+ "mykje",
+ "sjøl",
+ "korso",
+ "hoss",
+ "fordi",
+ "blitt",
+ "hvilke",
+ "sidan",
+ "verte",
+ "hvilken",
+ "inni",
+ "etter",
+ "somme",
+ "hvordan",
+ "eit",
+ "vors",
+ "deim",
+ "enn",
+ "kven",
+ "nå",
+ "dykkar",
+ "hva",
+ "inn",
+ "dykk",
+ "ble",
+ "kvar",
+ "både",
+ "eitt",
+ "bare",
+ "hvorfor",
+ "sitt",
+ "vore",
+ "hver",
+ "opp",
+ "kvarhelst",
+ "samme",
+ "hjå",
+ "hennar",
+ "før",
+ "hadde",
+ "dere",
+ "ved",
+ "båe",
+ "eg",
+ "seg",
+ "vort",
+ "nokre",
+ "kom",
+ "begge",
+ "nokor",
+ "slik",
+ "elles"
+ ],
+ "kz": [
+ "өзім",
+ "дүрс",
+ "емес",
+ "мен",
+ "қайсыбір",
+ "күрт",
+ "сарт-сұрт",
+ "ғана",
+ "әрине",
+ "сіздердің",
+ "сол",
+ "әншейін",
+ "барша",
+ "бізге",
+ "әлдене",
+ "сен",
+ "дүңк",
+ "ешқашан",
+ "алатау",
+ "әлдеқашан",
+ "ай",
+ "паһ-паһ",
+ "сіз",
+ "пай",
+ "әй",
+ "біздердің",
+ "алайда",
+ "өзінің",
+ "арбаң-арбаң",
+ "арс",
+ "сізге",
+ "өзіне",
+ "ірк",
+ "кірт",
+ "сарт",
+ "моһ",
+ "сыңқ",
+ "болп",
+ "алдақашан",
+ "ешқайсы",
+ "батыр-бұтыр",
+ "арсалаң-арсалаң",
+ "бәрі",
+ "былп",
+ "кә",
+ "құрау-құрау",
+ "біз",
+ "өзімнің",
+ "мыңқ",
+ "ешқандай",
+ "ешкім",
+ "менің",
+ "сіздер",
+ "шіңк",
+ "шаңқ-шұңқ",
+ "тарбаң-тарбаң",
+ "өй",
+ "аһа",
+ "жалт-жалт",
+ "әйтпесе",
+ "беу",
+ "осылай",
+ "сізбен",
+ "олардың",
+ "тағы",
+ "әлденеше",
+ "пай-пай",
+ "уау",
+ "сенің",
+ "арнайы",
+ "қолп",
+ "өз",
+ "бірақ",
+ "жаракімалла",
+ "бірдеме",
+ "ана",
+ "ә",
+ "кейбір",
+ "эх",
+ "сондай",
+ "біздер",
+ "бүгжең-бүгжең",
+ "өзге",
+ "ойпырмай",
+ "өзің",
+ "тырс",
+ "онымен",
+ "жалп",
+ "ештеме",
+ "менімен",
+ "шаңқ-шаңқ",
+ "әлдеқалай",
+ "мына",
+ "әрне",
+ "тыңқ",
+ "әрқайсы",
+ "барлық",
+ "пфша",
+ "ырс",
+ "сенімен",
+ "бізбен",
+ "әркім",
+ "құрау",
+ "жоқ",
+ "жалт-жұлт",
+ "уай",
+ "сона",
+ "қыңқ",
+ "қорс",
+ "түге",
+ "күңк",
+ "менде",
+ "анау",
+ "қаңғыр-күңгір",
+ "кәне",
+ "еш",
+ "бұндай",
+ "пішту",
+ "сіздерге",
+ "қалт-қалт",
+ "шәйт",
+ "оһо",
+ "олармен",
+ "сорап",
+ "тәк",
+ "кейбіреу",
+ "қап",
+ "борт",
+ "мынау",
+ "әттегенай",
+ "сіздермен",
+ "дегенмен",
+ "бүйт",
+ "сенде",
+ "эй",
+ "сонау",
+ "түгел",
+ "біздерден",
+ "қайқаң-құйқаң",
+ "сенен\tонан",
+ "барқ",
+ "гүрс",
+ "арс-ұрс",
+ "бар",
+ "ох",
+ "шек",
+ "алақай",
+ "пырс",
+ "шіркін",
+ "қош-қош",
+ "саңқ",
+ "сонымен",
+ "осынау",
+ "тек",
+ "біздерге",
+ "морт",
+ "әрқалай",
+ "маңқ",
+ "сіздерден",
+ "олар",
+ "себебі",
+ "желп",
+ "қалт-құлт",
+ "ол",
+ "мұндай",
+ "әлдеқайдан",
+ "әттең",
+ "мышы",
+ "солай",
+ "салаң-сұлаң",
+ "қана",
+ "біздермен",
+ "кәнеки",
+ "уа",
+ "әрбір",
+ "құр",
+ "мәссаған",
+ "ал",
+ "ыңқ",
+ "е",
+ "осы",
+ "сізден",
+ "ура",
+ "әукім",
+ "пішә",
+ "шырт",
+ "митың-митың",
+ "біреу",
+ "әлдекім",
+ "шаңқ",
+ "әттеген-ай",
+ "далаң-далаң",
+ "өзіме",
+ "қаңқ-қаңқ",
+ "кәһ",
+ "өзі",
+ "әйткенмен",
+ "онда",
+ "өйткені",
+ "ербелең-ербелең",
+ "тарс-тұрс",
+ "япырмай",
+ "па",
+ "күллі",
+ "қаңқ-құңқ",
+ "кәні",
+ "тарс",
+ "бүкіл",
+ "айтпақшы",
+ "ыржың-тыржың",
+ "ах",
+ "бұл",
+ "әйда",
+ "ие",
+ "бізден",
+ "сенен",
+ "қызараң-қызараң",
+ "үйт",
+ "ешбір",
+ "астапыралла",
+ "тағыда",
+ "дәнеңе",
+ "таңқ",
+ "ырқ",
+ "масқарай"
+ ],
+ "hu": [
+ "szinte",
+ "lehetett",
+ "talán",
+ "csak",
+ "nagy",
+ "ill",
+ "illetve",
+ "kívül",
+ "cikkeket",
+ "utána",
+ "nagyobb",
+ "sokkal",
+ "vagy",
+ "melyek",
+ "aztán",
+ "ennek",
+ "õ",
+ "azonban",
+ "azért",
+ "míg",
+ "éppen",
+ "jobban",
+ "mellett",
+ "benne",
+ "után",
+ "ehhez",
+ "mindenki",
+ "mikor",
+ "azon",
+ "ison",
+ "alatt",
+ "pedig",
+ "hanem",
+ "neki",
+ "elsõ",
+ "igen",
+ "szemben",
+ "lenne",
+ "számára",
+ "lenni",
+ "vagyis",
+ "egyéb",
+ "ezek",
+ "még",
+ "szerint",
+ "mindent",
+ "milyen",
+ "másik",
+ "maga",
+ "ott",
+ "új",
+ "mindig",
+ "nekem",
+ "volna",
+ "jó",
+ "vissza",
+ "ismét",
+ "rá",
+ "azt",
+ "egy",
+ "emilyen",
+ "vannak",
+ "én",
+ "voltunk",
+ "ellen",
+ "belül",
+ "magát",
+ "valami",
+ "így",
+ "egyik",
+ "mivel",
+ "viszont",
+ "nagyon",
+ "ezen",
+ "valaki",
+ "kellett",
+ "és",
+ "ezzel",
+ "persze",
+ "sok",
+ "õk",
+ "abban",
+ "miért",
+ "arról",
+ "át",
+ "újra",
+ "össze",
+ "volt",
+ "azok",
+ "ilyen",
+ "sokat",
+ "néha",
+ "ekkor",
+ "ill.",
+ "cikk",
+ "erre",
+ "egyetlen",
+ "vagyok",
+ "voltam",
+ "amelyeket",
+ "annak",
+ "ezt",
+ "semmi",
+ "ez",
+ "ahhoz",
+ "minden",
+ "lett",
+ "ilyenkor",
+ "aki",
+ "egyre",
+ "ahogy",
+ "amolyan",
+ "keressünk",
+ "egész",
+ "mintha",
+ "néhány",
+ "nincs",
+ "õket",
+ "teljes",
+ "által",
+ "hogy",
+ "mely",
+ "között",
+ "egyes",
+ "legyen",
+ "amíg",
+ "amelyek",
+ "továbbá",
+ "akkor",
+ "valamint",
+ "voltak",
+ "amelynek",
+ "bár",
+ "hogyan",
+ "azután",
+ "tehát",
+ "ezért",
+ "utolsó",
+ "saját",
+ "lesz",
+ "jól",
+ "általában",
+ "hiszen",
+ "vele",
+ "tovább",
+ "ami",
+ "elõször",
+ "újabb",
+ "nélkül",
+ "cikkek",
+ "elõ",
+ "elõtt",
+ "amelyekben",
+ "elég",
+ "kell",
+ "keresztül",
+ "amely",
+ "már",
+ "felé",
+ "ebben",
+ "több",
+ "mert",
+ "való",
+ "amelyet",
+ "azzal",
+ "úgy",
+ "legalább",
+ "eddig",
+ "amikor",
+ "arra",
+ "ahol",
+ "olyan",
+ "közül",
+ "ugyanis",
+ "lehet",
+ "itt",
+ "amit",
+ "mint",
+ "akik"
+ ],
+ "tr": [
+ "niye",
+ "mu",
+ "belki",
+ "nasıl",
+ "niçin",
+ "mü",
+ "ile",
+ "nerede",
+ "aslında",
+ "nerde",
+ "bazı",
+ "gibi",
+ "şey",
+ "yani",
+ "mı",
+ "kez",
+ "hep",
+ "defa",
+ "ama",
+ "neden",
+ "ise",
+ "diye",
+ "veya",
+ "için",
+ "hepsi",
+ "tüm",
+ "çok",
+ "birkaç",
+ "sanki",
+ "acaba",
+ "şu",
+ "çünkü",
+ "hiç",
+ "eğer",
+ "nereye"
+ ],
+ "it": [
+ "avrai",
+ "erano",
+ "suoi",
+ "fossero",
+ "siete",
+ "fecero",
+ "sarà",
+ "fareste",
+ "faceva",
+ "nello",
+ "aveva",
+ "quelle",
+ "starei",
+ "stando",
+ "faresti",
+ "tutti",
+ "eri",
+ "avevamo",
+ "sarebbe",
+ "miei",
+ "fanno",
+ "avessimo",
+ "sul",
+ "furono",
+ "ebbero",
+ "stesti",
+ "quanti",
+ "fu",
+ "nostra",
+ "come",
+ "sarei",
+ "vostra",
+ "facevamo",
+ "queste",
+ "faremo",
+ "nelle",
+ "stiamo",
+ "nei",
+ "che",
+ "non",
+ "faremmo",
+ "tutto",
+ "stavamo",
+ "avendo",
+ "avessi",
+ "starete",
+ "negli",
+ "stava",
+ "avrei",
+ "facessi",
+ "aveste",
+ "facemmo",
+ "avevate",
+ "ebbi",
+ "avemmo",
+ "quanto",
+ "è",
+ "nella",
+ "anche",
+ "stetti",
+ "negl",
+ "sono",
+ "quella",
+ "avevo",
+ "faranno",
+ "farò",
+ "ero",
+ "agli",
+ "facesse",
+ "facciamo",
+ "stiate",
+ "stai",
+ "stesse",
+ "nell",
+ "stavi",
+ "farete",
+ "sue",
+ "avrà",
+ "staremo",
+ "facesti",
+ "col",
+ "stessero",
+ "sarai",
+ "più",
+ "farebbero",
+ "stavate",
+ "feci",
+ "starebbe",
+ "mia",
+ "avevi",
+ "questa",
+ "dal",
+ "avrebbe",
+ "hai",
+ "vostre",
+ "sei",
+ "abbiano",
+ "starai",
+ "facessimo",
+ "mio",
+ "faccia",
+ "nostre",
+ "loro",
+ "stavo",
+ "faceste",
+ "stette",
+ "dall",
+ "sullo",
+ "siate",
+ "faccio",
+ "dove",
+ "farebbe",
+ "siano",
+ "saresti",
+ "eravate",
+ "avesti",
+ "avute",
+ "avuti",
+ "questo",
+ "facciate",
+ "stessi",
+ "dagl",
+ "sareste",
+ "abbia",
+ "dalla",
+ "tue",
+ "farà",
+ "dello",
+ "stanno",
+ "foste",
+ "fosti",
+ "abbiamo",
+ "facevi",
+ "essendo",
+ "avrebbero",
+ "sulle",
+ "avrò",
+ "sui",
+ "tra",
+ "degli",
+ "avranno",
+ "saremmo",
+ "avrete",
+ "staremmo",
+ "nel",
+ "avuto",
+ "starebbero",
+ "fai",
+ "sugli",
+ "saremo",
+ "siamo",
+ "agl",
+ "facciano",
+ "quello",
+ "delle",
+ "farei",
+ "stessimo",
+ "stettero",
+ "sta",
+ "quanta",
+ "facessero",
+ "lei",
+ "vostri",
+ "perché",
+ "sto",
+ "avete",
+ "sulla",
+ "avremmo",
+ "avessero",
+ "hanno",
+ "sarete",
+ "sugl",
+ "stia",
+ "facevano",
+ "abbiate",
+ "dell",
+ "sarò",
+ "facendo",
+ "stareste",
+ "staranno",
+ "saranno",
+ "dallo",
+ "eravamo",
+ "sull",
+ "della",
+ "ebbe",
+ "vostro",
+ "degl",
+ "stavano",
+ "avremo",
+ "contro",
+ "nostro",
+ "quelli",
+ "staresti",
+ "avresti",
+ "questi",
+ "dagli",
+ "stemmo",
+ "coi",
+ "stiano",
+ "quante",
+ "steste",
+ "avuta",
+ "suo",
+ "chi",
+ "fummo",
+ "gli",
+ "dov",
+ "fece",
+ "tuoi",
+ "fossi",
+ "dai",
+ "starà",
+ "dalle",
+ "quale",
+ "facevo",
+ "starò",
+ "avesse",
+ "farai",
+ "sarebbero",
+ "avevano",
+ "fossimo",
+ "facevate",
+ "avreste",
+ "allo"
+ ],
+ "de": [
+ "musste",
+ "ob",
+ "ihrem",
+ "jeden",
+ "solchen",
+ "seine",
+ "sich",
+ "sonst",
+ "kann",
+ "seinen",
+ "eine",
+ "indem",
+ "für",
+ "solche",
+ "dann",
+ "unser",
+ "würden",
+ "dieselben",
+ "im",
+ "hab",
+ "derselbe",
+ "können",
+ "würde",
+ "seiner",
+ "nun",
+ "jenen",
+ "zum",
+ "ihrer",
+ "derselben",
+ "nichts",
+ "welchen",
+ "deinem",
+ "über",
+ "gewesen",
+ "jetzt",
+ "ihm",
+ "oder",
+ "eurer",
+ "mir",
+ "anderm",
+ "jene",
+ "dieselbe",
+ "jede",
+ "etwas",
+ "hatte",
+ "welche",
+ "wirst",
+ "ihnen",
+ "eurem",
+ "sondern",
+ "derer",
+ "von",
+ "hatten",
+ "selbst",
+ "weg",
+ "hin",
+ "euch",
+ "keinem",
+ "einen",
+ "einmal",
+ "einigen",
+ "noch",
+ "diesem",
+ "aller",
+ "wenn",
+ "wo",
+ "meinem",
+ "ist",
+ "einige",
+ "eures",
+ "allem",
+ "unseres",
+ "seines",
+ "zwischen",
+ "denn",
+ "dessen",
+ "wollen",
+ "wir",
+ "weil",
+ "unserem",
+ "sind",
+ "haben",
+ "sie",
+ "ich",
+ "ihres",
+ "sollte",
+ "manche",
+ "hat",
+ "aber",
+ "habe",
+ "anderen",
+ "manchen",
+ "ihn",
+ "sein",
+ "nicht",
+ "hinter",
+ "dort",
+ "wird",
+ "auf",
+ "machen",
+ "anderem",
+ "demselben",
+ "uns",
+ "bei",
+ "keines",
+ "damit",
+ "bin",
+ "mancher",
+ "ihr",
+ "anderes",
+ "eines",
+ "deines",
+ "anderer",
+ "dasselbe",
+ "manches",
+ "dazu",
+ "bist",
+ "allen",
+ "muss",
+ "warst",
+ "anders",
+ "manchem",
+ "meinen",
+ "diesen",
+ "zwar",
+ "einiges",
+ "unsere",
+ "einigem",
+ "und",
+ "jedes",
+ "mich",
+ "kein",
+ "also",
+ "diese",
+ "ins",
+ "meine",
+ "meiner",
+ "dieser",
+ "unseren",
+ "jenes",
+ "dein",
+ "zur",
+ "einiger",
+ "aus",
+ "zu",
+ "wollte",
+ "bis",
+ "keiner",
+ "welches",
+ "dich",
+ "andern",
+ "ohne",
+ "sehr",
+ "deine",
+ "jener",
+ "welchem",
+ "deinen",
+ "wieder",
+ "jedem",
+ "ander",
+ "euer",
+ "ihren",
+ "eure",
+ "keinen",
+ "durch",
+ "jenem",
+ "anderr",
+ "jeder",
+ "deiner",
+ "welcher",
+ "unter",
+ "mein",
+ "gegen",
+ "dies",
+ "denselben",
+ "keine",
+ "werde",
+ "einig",
+ "nur",
+ "seinem",
+ "daß",
+ "solches",
+ "weiter",
+ "solchem",
+ "einer",
+ "desselben",
+ "werden",
+ "während",
+ "nach",
+ "war",
+ "auch",
+ "einem",
+ "ihre",
+ "meines",
+ "solcher",
+ "viel",
+ "dieses",
+ "soll",
+ "euren",
+ "könnte",
+ "öffnen",
+ "fahren",
+ "fährt",
+ "anhängen",
+ "anhang",
+ "angehangen",
+ "nächste",
+ "nämlich",
+ "spät",
+ "später",
+ "gehören",
+ "hören",
+ "früh",
+ "früher",
+ "möglich",
+ "plötzlich",
+ "schön",
+ "überall",
+ "zurück",
+ "fünf",
+ "außen",
+ "außer",
+ "spaß",
+ "süß",
+ "hallo",
+ "guten",
+ "freundliche",
+ "freundlichen",
+ "besten"
+ ],
+ "sv": [
+ "varför",
+ "deras",
+ "någon",
+ "inom",
+ "ert",
+ "något",
+ "inte",
+ "till",
+ "dess",
+ "vilket",
+ "att",
+ "än",
+ "hon",
+ "vilka",
+ "dessa",
+ "vid",
+ "vars",
+ "denna",
+ "sina",
+ "vara",
+ "är",
+ "vilken",
+ "sådant",
+ "vilkas",
+ "där",
+ "jag",
+ "ju",
+ "mycket",
+ "dina",
+ "utan",
+ "sedan",
+ "detta",
+ "vad",
+ "vem",
+ "och",
+ "samma",
+ "några",
+ "våra",
+ "varje",
+ "vårt",
+ "mellan",
+ "varit",
+ "sitta",
+ "själv",
+ "sådana",
+ "över",
+ "kunde",
+ "för",
+ "mina",
+ "här",
+ "åt",
+ "från",
+ "icke",
+ "allt",
+ "hade",
+ "blivit",
+ "när",
+ "ej",
+ "hur"
+ ],
+ "ru": [
+ "сам",
+ "тогда",
+ "него",
+ "мой",
+ "ну",
+ "он",
+ "мне",
+ "впрочем",
+ "чтобы",
+ "не",
+ "себя",
+ "моя",
+ "она",
+ "нас",
+ "к",
+ "про",
+ "была",
+ "какой",
+ "иногда",
+ "было",
+ "хорошо",
+ "чем",
+ "всю",
+ "они",
+ "здесь",
+ "чуть",
+ "тот",
+ "об",
+ "над",
+ "них",
+ "опять",
+ "надо",
+ "теперь",
+ "был",
+ "уже",
+ "вам",
+ "если",
+ "тебя",
+ "чтоб",
+ "почти",
+ "тоже",
+ "ему",
+ "или",
+ "вот",
+ "по",
+ "можно",
+ "но",
+ "перед",
+ "три",
+ "есть",
+ "конечно",
+ "там",
+ "ли",
+ "да",
+ "их",
+ "а",
+ "один",
+ "всего",
+ "ж",
+ "никогда",
+ "эту",
+ "же",
+ "без",
+ "у",
+ "потом",
+ "его",
+ "от",
+ "тут",
+ "так",
+ "какая",
+ "чего",
+ "этой",
+ "то",
+ "тем",
+ "что",
+ "им",
+ "сейчас",
+ "нибудь",
+ "через",
+ "всегда",
+ "этого",
+ "того",
+ "ни",
+ "даже",
+ "уж",
+ "с",
+ "ты",
+ "будто",
+ "зачем",
+ "при",
+ "вдруг",
+ "свою",
+ "разве",
+ "под",
+ "этот",
+ "нельзя",
+ "много",
+ "в",
+ "может",
+ "всех",
+ "как",
+ "вас",
+ "были",
+ "нет",
+ "ведь",
+ "из",
+ "два",
+ "нее",
+ "быть",
+ "после",
+ "больше",
+ "эти",
+ "лучше",
+ "на",
+ "более",
+ "для",
+ "за",
+ "ее",
+ "куда",
+ "том",
+ "все",
+ "между",
+ "до",
+ "себе",
+ "где",
+ "мы",
+ "ней",
+ "другой",
+ "когда",
+ "со",
+ "ним",
+ "будет",
+ "только",
+ "хоть",
+ "раз",
+ "потому",
+ "и",
+ "во",
+ "ничего",
+ "бы",
+ "этом",
+ "совсем",
+ "еще",
+ "кто",
+ "наконец",
+ "меня",
+ "такой",
+ "вы"
+ ],
+ "az": [
+ "olan",
+ "onların ",
+ "bəzi",
+ "ay",
+ "arasında",
+ "harada",
+ "haqqında",
+ "onsuzda",
+ "beş",
+ "otuz",
+ "əslində",
+ "faiz",
+ "heç",
+ "yaxşı",
+ "çox",
+ "yəni",
+ "bilər",
+ "kimi",
+ "təəssüf",
+ "xan",
+ "yoxdur",
+ "kimə",
+ "sən",
+ "olmuşdur",
+ "həmin",
+ "nəhayət",
+ "isə",
+ "olmadı",
+ "yalnız",
+ "bəy",
+ "üç",
+ "onun",
+ "onu",
+ "buradan",
+ "yenə",
+ "olmaz",
+ "edir",
+ "ən",
+ "ı",
+ "ondan",
+ "sizlər",
+ "özü",
+ "bəlkə",
+ "iki",
+ "lap",
+ "deyil",
+ "bəli",
+ "bizim",
+ "doqquz",
+ "həm",
+ "amma",
+ "sənin",
+ "ilk",
+ "hə",
+ "mənə",
+ "düz",
+ "olur",
+ "indi",
+ "sizin",
+ "xanım",
+ "həmişə",
+ "bir",
+ "və",
+ "artıq",
+ "etmək",
+ "nə",
+ "olduğu",
+ "məhz",
+ "səhv",
+ "gilə",
+ "olaraq",
+ "dəqiqə",
+ "buna",
+ "idi",
+ "altmış",
+ "səkkiz",
+ "zaman",
+ "mən",
+ "bəzən",
+ "bundan",
+ "dək",
+ "niyə",
+ "qədər",
+ "yeddi",
+ "əlbəttə",
+ "ildə",
+ "onlardan",
+ "yoxsa",
+ "dörd",
+ "dedi",
+ "saniyə",
+ "xeyr",
+ "istifadə",
+ "bütün",
+ "qırx",
+ "ü",
+ "obirisi",
+ "etmə",
+ "olar",
+ "öz",
+ "əgər",
+ "mirşey",
+ "bizlər",
+ "bunların",
+ "biraz",
+ "oradan",
+ "cı",
+ "lakin",
+ "ə",
+ "sənə",
+ "üçün",
+ "bunun",
+ "altı",
+ "hər",
+ "sonra",
+ "çünki",
+ "edən",
+ "doqsan",
+ "sadəcə",
+ "əlli",
+ "də",
+ "etdi",
+ "dən",
+ "yüz",
+ "belə",
+ "görə",
+ "iyirmi",
+ "olsun",
+ "elə",
+ "onlar",
+ "səksən",
+ "yetmiş",
+ "yox",
+ "oldu",
+ "qarşı",
+ "ilə",
+ "ona",
+ "bunu",
+ "bax",
+ "cü"
+ ],
+ "fi": [
+ "sinun",
+ "teiltä",
+ "nämä",
+ "teissä",
+ "näille",
+ "heistä",
+ "olisit",
+ "minusta",
+ "minulle",
+ "tuona",
+ "tätä",
+ "kenen",
+ "miltä",
+ "teihin",
+ "siinä",
+ "keitä",
+ "häneen",
+ "kenet",
+ "siihen",
+ "vaikka",
+ "meidät",
+ "sinut",
+ "tuon",
+ "hän",
+ "joiden",
+ "sinua",
+ "vaan",
+ "niille",
+ "keiksi",
+ "minussa",
+ "tästä",
+ "tallä",
+ "siitä",
+ "jolla",
+ "keihin",
+ "olisimme",
+ "jotka",
+ "mutta",
+ "keissä",
+ "tuossa",
+ "keiltä",
+ "tuolta",
+ "ovat",
+ "noille",
+ "missä",
+ "mille",
+ "olette",
+ "olimme",
+ "meille",
+ "kanssa",
+ "keinä",
+ "teidän",
+ "heillä",
+ "minulta",
+ "tähän",
+ "teitä",
+ "itse",
+ "jolle",
+ "niissä",
+ "keneen",
+ "niiltä",
+ "oli",
+ "hänet",
+ "olin",
+ "minuun",
+ "teidät",
+ "niiksi",
+ "keille",
+ "siksi",
+ "tässä",
+ "niihin",
+ "johon",
+ "tälle",
+ "joina",
+ "niinä",
+ "kenenä",
+ "eivät",
+ "noista",
+ "meihin",
+ "mistä",
+ "noiksi",
+ "meissä",
+ "noihin",
+ "hänen",
+ "näitä",
+ "meiltä",
+ "minulla",
+ "näiltä",
+ "meitä",
+ "poikki",
+ "meidän",
+ "vai",
+ "teille",
+ "näissä",
+ "näihin",
+ "näiksi",
+ "jota",
+ "noilla",
+ "olit",
+ "mitä",
+ "tältä",
+ "joissa",
+ "joilta",
+ "näinä",
+ "olisi",
+ "joita",
+ "niitä",
+ "kuin",
+ "olisin",
+ "kenessä",
+ "olen",
+ "olisivat",
+ "sinussa",
+ "minua",
+ "noiden",
+ "heitä",
+ "heidän",
+ "teillä",
+ "mikä",
+ "jos",
+ "joka",
+ "nyt",
+ "sen",
+ "sinulta",
+ "tuoksi",
+ "jona",
+ "joksi",
+ "tämän",
+ "niiden",
+ "täksi",
+ "ette",
+ "keneltä",
+ "keistä",
+ "tuosta",
+ "sille",
+ "keneksi",
+ "siltä",
+ "tuolla",
+ "ollut",
+ "keillä",
+ "sinulle",
+ "häntä",
+ "tuotä",
+ "kenestä",
+ "joille",
+ "hänellä",
+ "kenelle",
+ "josta",
+ "minun",
+ "kuka",
+ "tämä",
+ "olet",
+ "sinusta",
+ "heissä",
+ "tuohon",
+ "että",
+ "olleet",
+ "minut",
+ "teistä",
+ "heiltä",
+ "nuo",
+ "olivat",
+ "heihin",
+ "näiden",
+ "jossa",
+ "joista",
+ "tai",
+ "noina",
+ "mukaan",
+ "niillä",
+ "häneltä",
+ "joihin",
+ "sekä",
+ "joiksi",
+ "yli",
+ "emme",
+ "näistä",
+ "meillä",
+ "millä",
+ "heille",
+ "mitkä",
+ "noin",
+ "hänessä",
+ "sitä",
+ "jonka",
+ "miksi",
+ "noilta",
+ "noissa",
+ "ketä",
+ "näillä",
+ "meistä",
+ "sinuun",
+ "noita",
+ "tänä",
+ "olisitte",
+ "mihin",
+ "hänestä",
+ "olla",
+ "heidät",
+ "keiden",
+ "sinulla",
+ "olemme",
+ "niistä",
+ "olitte",
+ "joilla",
+ "jolta",
+ "koska",
+ "kenellä",
+ "tuolle",
+ "hänelle"
+ ],
+ "da": [
+ "noget",
+ "hos",
+ "anden",
+ "jo",
+ "thi",
+ "været",
+ "ind",
+ "sit",
+ "hende",
+ "blive",
+ "bliver",
+ "havde",
+ "jer",
+ "hvad",
+ "af",
+ "ud",
+ "nogle",
+ "ham",
+ "hendes",
+ ],
+ "es": [
+ "estuvierais",
+ "esa",
+ "fuimos",
+ "tuviese",
+ "habíais",
+ "hubieses",
+ "nosotras",
+ "estuviera",
+ "tenemos",
+ "eso",
+ "estáis",
+ "habiendo",
+ "otra",
+ "habrías",
+ "seamos",
+ "fueron",
+ "eras",
+ "fueras",
+ "estando",
+ "hubieras",
+ "están",
+ "otro",
+ "sentida",
+ "tuvimos",
+ "tuviera",
+ "sintiendo",
+ "tenidas",
+ "suya",
+ "tuvieron",
+ "erais",
+ "tuvieseis",
+ "contra",
+ "tenido",
+ "tuviésemos",
+ "cuando",
+ "durante",
+ "las",
+ "serán",
+ "sí",
+ "vuestra",
+ "hemos",
+ "fuéramos",
+ "estabais",
+ "estemos",
+ "desde",
+ "tuviéramos",
+ "suyo",
+ "estada",
+ "quienes",
+ "sobre",
+ "soy",
+ "todos",
+ "tuvierais",
+ "estados",
+ "nuestra",
+ "estuvieran",
+ "tendríamos",
+ "esas",
+ "hubieron",
+ "tengáis",
+ "fuera",
+ "pero",
+ "hubieseis",
+ "tendrían",
+ "fueseis",
+ "esos",
+ "estábamos",
+ "también",
+ "hubiésemos",
+ "tuyas",
+ "fueran",
+ "estés",
+ "habré",
+ "eran",
+ "habido",
+ "vuestras",
+ "hubiéramos",
+ "estuvo",
+ "antes",
+ "algunos",
+ "habidas",
+ "nuestras",
+ "estuvimos",
+ "hubierais",
+ "unos",
+ "estarás",
+ "estaremos",
+ "tuyos",
+ "tanto",
+ "hubiese",
+ "estuvieses",
+ "tuvieses",
+ "estaba",
+ "les",
+ "sentid",
+ "estaríamos",
+ "estuviésemos",
+ "estarían",
+ "hayas",
+ "seas",
+ "quien",
+ "seríais",
+ "teniendo",
+ "tendrás",
+ "tendremos",
+ "tuvieras",
+ "teníais",
+ "estoy",
+ "hubiera",
+ "estaban",
+ "hubimos",
+ "esto",
+ "cual",
+ "sentidas",
+ "tendrá",
+ "serás",
+ "sentido",
+ "mí",
+ "habrán",
+ "tendrías",
+ "habían",
+ "ese",
+ "otros",
+ "tus",
+ "tengamos",
+ "había",
+ "habrá",
+ "tendréis",
+ "tienes",
+ "estar",
+ "hubisteis",
+ "míos",
+ "los",
+ "estuviéramos",
+ "otras",
+ "tendrán",
+ "muchos",
+ "eres",
+ "hayáis",
+ "mías",
+ "porque",
+ "habríamos",
+ "hasta",
+ "fueses",
+ "estuvieras",
+ "tuve",
+ "estabas",
+ "nosotros",
+ "teníamos",
+ "nada",
+ "estaréis",
+ "habríais",
+ "habidos",
+ "tuviesen",
+ "tened",
+ "nuestro",
+ "tenías",
+ "tengo",
+ "hubo",
+ "hubiste",
+ "vuestro",
+ "habremos",
+ "estará",
+ "algunas",
+ "ellos",
+ "suyos",
+ "habida",
+ "serías",
+ "estarías",
+ "estuviste",
+ "habéis",
+ "tengas",
+ "estás",
+ "él",
+ "vosostros",
+ "tú",
+ "estuvisteis",
+ "estuviesen",
+ "estuvieron",
+ "tuvieran",
+ "muy",
+ "fuésemos",
+ "ellas",
+ "suyas",
+ "fuisteis",
+ "estarán",
+ "estadas",
+ "nuestros",
+ "hayan",
+ "seréis",
+ "tuyo",
+ "mucho",
+ "fuesen",
+ "habréis",
+ "hubiesen",
+ "siente",
+ "hay",
+ "tendríais",
+ "tuviste",
+ "tuvo",
+ "fuerais",
+ "tenéis",
+ "ante",
+ "estado",
+ "estaría",
+ "poco",
+ "vuestros",
+ "fuese",
+ "tendré",
+ "esté",
+ "mío",
+ "tienen",
+ "tenida",
+ "yo",
+ "tuya",
+ "tiene",
+ "mía",
+ "estad",
+ "hubieran",
+ "seáis",
+ "habrás",
+ "tenidos",
+ "habías",
+ "estos",
+ "vosostras",
+ "estén",
+ "tendría",
+ "tenga",
+ "habíamos",
+ "fuiste",
+ "sentidos",
+ "seré",
+ "mis",
+ "qué",
+ "tengan",
+ "habrían",
+ "estaríais",
+ "tuvisteis",
+ "hayamos",
+ "algo",
+ "estuviese",
+ "haya",
+ "tenía",
+ "serían",
+ "estéis",
+ "estuve",
+ "sería",
+ "estuvieseis",
+ "habría",
+ "estaré",
+ "tenían",
+ "hube",
+ "fue",
+ "donde"
+ ],
+ "pt": [
+ "nossas",
+ "depois",
+ "sou",
+ "houveria",
+ "em",
+ "há",
+ "tiverem",
+ "teu",
+ "foi",
+ "fossem",
+ "meus",
+ "nosso",
+ "nós",
+ "fomos",
+ "eles",
+ "tivéssemos",
+ "hajam",
+ "estivermos",
+ "sejam",
+ "esteja",
+ "tivermos",
+ "estavam",
+ "vocês",
+ "houverão",
+ "formos",
+ "minha",
+ "estivemos",
+ "houveremos",
+ "estivera",
+ "tivéramos",
+ "hei",
+ "terão",
+ "tem",
+ "fora",
+ "aquela",
+ "essas",
+ "também",
+ "num",
+ "houvesse",
+ "dele",
+ "nossos",
+ "tiver",
+ "estejamos",
+ "tém",
+ "pelos",
+ "mas",
+ "estivesse",
+ "teve",
+ "teus",
+ "tivesse",
+ "estive",
+ "pelas",
+ "tuas",
+ "tiveram",
+ "nossa",
+ "ao",
+ "tivessem",
+ "aquilo",
+ "houveríamos",
+ "teríamos",
+ "uma",
+ "aquele",
+ "mesmo",
+ "tenham",
+ "estiverem",
+ "teria",
+ "minhas",
+ "tinha",
+ "isto",
+ "serei",
+ "seriam",
+ "muito",
+ "numa",
+ "fôssemos",
+ "esses",
+ "houverem",
+ "já",
+ "qual",
+ "forem",
+ "temos",
+ "só",
+ "tenha",
+ "teriam",
+ "lhe",
+ "tinham",
+ "elas",
+ "seria",
+ "aos",
+ "houveriam",
+ "aqueles",
+ "são",
+ "terei",
+ "delas",
+ "houvéssemos",
+ "dos",
+ "às",
+ "essa",
+ "estivéramos",
+ "tive",
+ "havemos",
+ "estiveram",
+ "suas",
+ "isso",
+ "tínhamos",
+ "tivera",
+ "houve",
+ "você",
+ "estou",
+ "estes",
+ "houver",
+ "seus",
+ "estivessem",
+ "nas",
+ "estiver",
+ "tivemos",
+ "não",
+ "serão",
+ "foram",
+ "tenho",
+ "houvera",
+ "houvemos",
+ "lhes",
+ "aquelas",
+ "houverá",
+ "quem",
+ "esteve",
+ "pela",
+ "fôramos",
+ "estava",
+ "quando",
+ "ela",
+ "houvermos",
+ "houvessem",
+ "houveram",
+ "hão",
+ "pelo",
+ "deles",
+ "estejam",
+ "até",
+ "esse",
+ "haja",
+ "seja",
+ "terá",
+ "estivéssemos",
+ "tenhamos",
+ "houvéramos",
+ "estávamos",
+ "houverei",
+ "teremos",
+ "estão",
+ "sejamos",
+ "seu",
+ "hajamos"
+ ],
+ "ne": [
+ "आफू",
+ "यी",
+ "रहेका",
+ "चाहिए",
+ "तिमी",
+ "अन्यत्र",
+ "तुरुन्तै",
+ "छ",
+ "थियो",
+ "त्सपछि",
+ "हुन्",
+ "भन्ने",
+ "आत्म",
+ "अरुलाई",
+ "नौ",
+ "थिए",
+ "बरु",
+ "जहाँ",
+ "बिशेष",
+ "लगभग",
+ "र",
+ "राख्छ",
+ "कि",
+ "यसबाहेक",
+ "पनि",
+ "पहिल्यै",
+ "कसरी",
+ "तिर",
+ "नजिकै",
+ "न",
+ "पटक",
+ "बिरुद्ध",
+ "देखिन्छ",
+ "तापनी",
+ "गर्नुपर्छ",
+ "जान",
+ "त्यो",
+ "पछि",
+ "निम्न",
+ "म",
+ "साँच्चै",
+ "सबै",
+ "भन्छु",
+ "जब",
+ "उदाहरण",
+ "पाँचौं",
+ "सात",
+ "संगै",
+ "पक्का",
+ "बाहेक",
+ "औं",
+ "एकदम",
+ "राम्रो",
+ "कसैले",
+ "एक",
+ "प्रतेक",
+ "जसलाई",
+ "गर्दै",
+ "पूर्व",
+ "बारे",
+ "हुन्छ",
+ "यस",
+ "छु",
+ "सधै",
+ "तिनिहरुलाई",
+ "दिनुभएको",
+ "एउटै",
+ "भए",
+ "पहिले",
+ "उहालाई",
+ "सक्छ",
+ "थिएन",
+ "तेस्रो",
+ "को",
+ "भित्र",
+ "रहेको",
+ "यदि",
+ "साथ",
+ "ले",
+ "माथि",
+ "छू",
+ "हुने",
+ "वास्तवमा",
+ "भन्",
+ "गरी",
+ "उप",
+ "गर्छ",
+ "तिनी",
+ "अर्थात्",
+ "तपाईको",
+ "अझै",
+ "नै",
+ "ओठ",
+ "भन्नुभयो",
+ "किनभने",
+ "आयो",
+ "हरेक",
+ "आजको",
+ "चाहन्छु",
+ "समय",
+ "आदि",
+ "अर्थात",
+ "कम से कम",
+ "कोही",
+ "नयाँ",
+ "राखे",
+ "बीच",
+ "ती",
+ "देखि",
+ "तर",
+ "अब",
+ "बने",
+ "भने",
+ "बाहिर",
+ "तत्काल",
+ "अक्सर",
+ "वरीपरी",
+ "जे",
+ "पक्कै",
+ "भित्री",
+ "दिए",
+ "दुई",
+ "त",
+ "बीचमा",
+ "जसमा",
+ "सायद",
+ "स्पष्ट",
+ "गर्नु",
+ "दिनुहुन्छ",
+ "अन्तर्गत",
+ "तपाई",
+ "यहाँ",
+ "पहिलो",
+ "के",
+ "अर्को",
+ "यसरी",
+ "गर्ने",
+ "कतै",
+ "गरौं",
+ "नि",
+ "मा",
+ "यद्यपि",
+ "जबकि",
+ "सही",
+ "प्लस",
+ "त्सैले",
+ "गर्न",
+ "उनले",
+ "गैर",
+ "यहाँसम्म",
+ "निम्ति",
+ "गए",
+ "लाई",
+ "जताततै",
+ "यसको",
+ "चाले",
+ "आए",
+ "यसो",
+ "उनको",
+ "चाहनुहुन्छ",
+ "पर्थ्यो",
+ "जस्तोसुकै",
+ "तीन",
+ "मलाई",
+ "तिनीहरुको",
+ "यथोचित",
+ "रूप",
+ "ठीक",
+ "फेरी",
+ "कृपया",
+ "गर्छु",
+ "केही",
+ "भन्छन्",
+ "गरि",
+ "संग",
+ "भन",
+ "गरेर",
+ "गयौ",
+ "यस्तो",
+ "गरेको",
+ "आफ्नो",
+ "जसबाट",
+ "सम्भव",
+ "प्रति",
+ "जो",
+ "क्रमशः",
+ "मुख्य",
+ "पर्छ",
+ "लागि",
+ "छैन",
+ "आफ्नै",
+ "या",
+ "जुन",
+ "निम्नानुसार",
+ "रही",
+ "मात्र",
+ "अनुसार",
+ "छन्",
+ "देखेर",
+ "जस्तो",
+ "कुरा",
+ "साथै",
+ "देखेको",
+ "हरे",
+ "कुनै",
+ "हो",
+ "आफूलाई",
+ "यति",
+ "अरु",
+ "तथा",
+ "शायद",
+ "सम्म",
+ "हुन",
+ "त्यहाँ",
+ "सोही",
+ "दोस्रो",
+ "यसपछि",
+ "कहाँबाट",
+ "जस्तै",
+ "चार",
+ "मेरो",
+ "पाँच",
+ "यो",
+ "पर्याप्त",
+ "तदनुसार",
+ "जसले",
+ "देखियो",
+ "अन्य",
+ "भन्दा",
+ "तल",
+ "भएको",
+ "देखे",
+ "धेरै",
+ "गरेका",
+ "अगाडी",
+ "छौं",
+ "जाहिर",
+ "नत्र",
+ "निर्दिष्ट",
+ "किन",
+ "सबैलाई",
+ "सो",
+ "भर",
+ "सारा",
+ "कहिलेकाहीं",
+ "तेस्कारण",
+ "जसको",
+ "पछिल्लो",
+ "अलग",
+ "कसै",
+ "तिनीहरू",
+ "अन्यथा",
+ "सट्टा"
+ ],
+ "nl": [
+ "zich",
+ "zou",
+ "hebben",
+ "reeds",
+ "ge",
+ "veel",
+ "worden",
+ "daar",
+ "aan",
+ "want",
+ "zonder",
+ "het",
+ "omdat",
+ "niet",
+ "voor",
+ "iets",
+ "zijn",
+ "zij",
+ "wordt",
+ "nog",
+ "dit",
+ "ik",
+ "tegen",
+ "heb",
+ "ben",
+ "ze",
+ "ons",
+ "uw",
+ "maar",
+ "hij",
+ "iemand",
+ "naar",
+ "geweest",
+ "doen",
+ "mijn",
+ "een",
+ "werd",
+ "zo",
+ "mij",
+ "haar",
+ "door",
+ "meer",
+ "dus",
+ "niets",
+ "kon",
+ "zal",
+ "toen",
+ "uit",
+ "ook",
+ "wezen",
+ "wil",
+ "met",
+ "geen",
+ "bij",
+ "moet",
+ "onder",
+ "zelf",
+ "eens",
+ "toch",
+ "kunnen",
+ "altijd",
+ "deze",
+ "wat",
+ "heeft"
+ ],
+ "ro": [
+ "fii",
+ "aceea",
+ "tăi",
+ "acestia",
+ "altfel",
+ "citi",
+ "atita",
+ "numai",
+ "cîtva",
+ "toata",
+ "pînă",
+ "către",
+ "acelea",
+ "atunci",
+ "acest",
+ "îmi",
+ "imi",
+ "după",
+ "putini",
+ "parca",
+ "aia",
+ "unele",
+ "atit",
+ "decit",
+ "fi",
+ "altceva",
+ "ala",
+ "două",
+ "tuturor",
+ "cei",
+ "să",
+ "prin",
+ "citeva",
+ "căci",
+ "unuia",
+ "doilea",
+ "mîine",
+ "deja",
+ "inca",
+ "mele",
+ "unul",
+ "ea",
+ "până",
+ "uneori",
+ "ăstea",
+ "alte",
+ "tale",
+ "unui",
+ "ceilalti",
+ "care",
+ "deci",
+ "sai",
+ "dată",
+ "prima",
+ "vostru",
+ "cind",
+ "abia",
+ "aceşti",
+ "iar",
+ "oriunde",
+ "nostru",
+ "pai",
+ "fiţi",
+ "ati",
+ "chiar",
+ "atata",
+ "dacă",
+ "tocmai",
+ "despre",
+ "cat",
+ "ca",
+ "nici",
+ "nimeni",
+ "toţi",
+ "pina",
+ "vă",
+ "deşi",
+ "drept",
+ "doi",
+ "niste",
+ "fără",
+ "noştri",
+ "nimic",
+ "acea",
+ "său",
+ "atat",
+ "cînd",
+ "sint",
+ "cîţi",
+ "isi",
+ "şi",
+ "aş",
+ "celor",
+ "mâine",
+ "nou",
+ "patru",
+ "cea",
+ "lor",
+ "voştri",
+ "pe",
+ "foarte",
+ "anume",
+ "cine",
+ "dintr-",
+ "mereu",
+ "în",
+ "inapoi",
+ "undeva",
+ "toate",
+ "aceeasi",
+ "adica",
+ "sunt",
+ "carora",
+ "noua",
+ "uneia",
+ "deasupra",
+ "dintr",
+ "alta",
+ "câţi",
+ "cum",
+ "toată",
+ "că",
+ "îţi",
+ "cărui",
+ "asupra",
+ "câtva",
+ "unei",
+ "va",
+ "dau",
+ "fata",
+ "acestei",
+ "zice",
+ "zi",
+ "nouă",
+ "câte",
+ "acela",
+ "fara",
+ "vouă",
+ "toti",
+ "printr-",
+ "totul",
+ "atitea",
+ "atitia",
+ "desi",
+ "cît",
+ "acei",
+ "fiu",
+ "ii",
+ "îi",
+ "orice",
+ "căror",
+ "caruia",
+ "oricând",
+ "acel",
+ "ceva",
+ "cand",
+ "pot",
+ "patra",
+ "cite",
+ "mea",
+ "mai",
+ "insa",
+ "aceia",
+ "catre",
+ "ale",
+ "cel",
+ "poate",
+ "îl",
+ "puţin",
+ "asta",
+ "mult",
+ "ăsta",
+ "avem",
+ "vreo",
+ "doar",
+ "tău",
+ "unora",
+ "atare",
+ "pentru",
+ "daca",
+ "alea",
+ "aţi",
+ "treilea",
+ "careia",
+ "avea",
+ "alti",
+ "ălea",
+ "altul",
+ "spre",
+ "apoi",
+ "aceştia",
+ "multa",
+ "ţi",
+ "noastră",
+ "mă",
+ "unor",
+ "oricât",
+ "ul",
+ "inainte",
+ "face",
+ "fim",
+ "intre",
+ "oricine",
+ "sau",
+ "citiva",
+ "aceste",
+ "voastră",
+ "unii",
+ "noastre",
+ "acestui",
+ "multă",
+ "aveţi",
+ "dintre",
+ "fie",
+ "altcineva",
+ "sale",
+ "spate",
+ "ar",
+ "eşti",
+ "sintem",
+ "ţie",
+ "ori",
+ "ului",
+ "sunteţi",
+ "oricare",
+ "sub",
+ "intr",
+ "puţină",
+ "asa",
+ "totusi",
+ "acestea",
+ "atatia",
+ "aceasta",
+ "voastre",
+ "atatea",
+ "fiecare",
+ "trei",
+ "mulţi",
+ "lângă",
+ "incit",
+ "oricum",
+ "multi",
+ "multe",
+ "nişte",
+ "cărei",
+ "astfel",
+ "aici",
+ "cita",
+ "oricînd",
+ "ba",
+ "unu",
+ "cineva",
+ "dă",
+ "treia",
+ "altii",
+ "unde",
+ "mei",
+ "ăştia",
+ "cam",
+ "acesta",
+ "iti",
+ "aibă",
+ "primul",
+ "dupa",
+ "acele",
+ "această",
+ "sa-mi",
+ "fost",
+ "cele",
+ "ceea",
+ "prea",
+ "totuşi",
+ "cit",
+ "dar",
+ "acelasi",
+ "puţina",
+ "peste",
+ "cât",
+ "astea",
+ "lîngă",
+ "săi",
+ "oricît",
+ "suntem",
+ "sa-ti",
+ "vreun",
+ "cumva",
+ "cîte",
+ "ăla",
+ "pic",
+ "tine",
+ "avut"
+ ],
+ "cs": [
+ "ačkoli",
+ "ahoj",
+ "alespoň",
+ "anebo",
+ "aspoň",
+ "během",
+ "blízko",
+ "bohužel",
+ "brzo",
+ "brzy",
+ "bude",
+ "budeme",
+ "budeš",
+ "budete",
+ "budou",
+ "budu",
+ "byl",
+ "byla",
+ "byli",
+ "bylo",
+ "byly",
+ "bys",
+ "být",
+ "celá",
+ "celé",
+ "celý",
+ "česká",
+ "či",
+ "čísel",
+ "čísla",
+ "číslo",
+ "dál",
+ "dále",
+ "další",
+ "dát",
+ "děkujeme",
+ "děkuji",
+ "dělá",
+ "dělat",
+ "dík",
+ "díky",
+ "dnes",
+ "dobrý",
+ "docela",
+ "dokud",
+ "druhá",
+ "druhé",
+ "druhý",
+ "druzí",
+ "hned",
+ "hodně",
+ "chce",
+ "chceme",
+ "chceš",
+ "chcete",
+ "chci",
+ "chtějí",
+ "chtít",
+ "já",
+ "jak",
+ "jaká",
+ "jaké",
+ "jaký",
+ "jde",
+ "jeho",
+ "její",
+ "jejich",
+ "jemu",
+ "jen",
+ "jenom",
+ "jenž",
+ "jestli",
+ "jestliže",
+ "ještě",
+ "jí",
+ "jich",
+ "jím",
+ "jiná",
+ "jinak",
+ "jiné",
+ "jiný",
+ "jít",
+ "již",
+ "jsem",
+ "jsi",
+ "jsme",
+ "jsou",
+ "jste",
+ "každá",
+ "každé",
+ "každý",
+ "kde",
+ "kdo",
+ "kdy",
+ "když",
+ "kolik",
+ "kromě",
+ "která",
+ "které",
+ "který",
+ "kteří",
+ "kvůli",
+ "má",
+ "mají",
+ "málo",
+ "mám",
+ "máme",
+ "máš",
+ "máte",
+ "mé",
+ "mě",
+ "měl",
+ "mém",
+ "mezi",
+ "mí",
+ "místo",
+ "mít",
+ "mně",
+ "mnou",
+ "moc",
+ "mohl",
+ "mohou",
+ "mohu",
+ "možná",
+ "možné",
+ "možný",
+ "můj",
+ "musí",
+ "může",
+ "můžeme",
+ "můžeš",
+ "můžete",
+ "nad",
+ "nade",
+ "nám",
+ "námi",
+ "naproti",
+ "nás",
+ "náš",
+ "naše",
+ "naši",
+ "našich",
+ "našim",
+ "našimi",
+ "nebo",
+ "nebyl",
+ "nebyla",
+ "nebyli",
+ "nebyly",
+ "něco",
+ "nějak",
+ "nějaká",
+ "nějaké",
+ "nějakou",
+ "nějaký",
+ "někam",
+ "někde",
+ "někdo",
+ "nemají",
+ "nemáme",
+ "nemáš",
+ "nemáte",
+ "neměl",
+ "němu",
+ "není",
+ "než",
+ "nich",
+ "nikam",
+ "nikde",
+ "nikdo",
+ "ním",
+ "nimi",
+ "nová",
+ "nové",
+ "nový",
+ "pak",
+ "paní",
+ "pod",
+ "podle",
+ "pokud",
+ "pořád",
+ "potom",
+ "pozdravem",
+ "proč",
+ "prosím",
+ "prostě",
+ "proti",
+ "protože",
+ "prvé",
+ "první",
+ "před",
+ "přejeme",
+ "přeji",
+ "přes",
+ "přese",
+ "při",
+ "příloha",
+ "přílohou",
+ "přílohy",
+ "příloze",
+ "přiložená",
+ "přiložené",
+ "přiloženém",
+ "přiložený",
+ "přiložených",
+ "přímo",
+ "sám",
+ "skoro",
+ "smí",
+ "smíme",
+ "smíš",
+ "smíte",
+ "snad",
+ "spolu",
+ "svá",
+ "svého",
+ "svému",
+ "svoje",
+ "svoji",
+ "svou",
+ "svůj",
+ "tady",
+ "také",
+ "takhle",
+ "taková",
+ "takové",
+ "takový",
+ "takto",
+ "taky",
+ "takže",
+ "tamhle",
+ "tamhleto",
+ "tamto",
+ "tě",
+ "tebou",
+ "teď",
+ "tedy",
+ "tento",
+ "této",
+ "tobě",
+ "tohle",
+ "trošku",
+ "třeba",
+ "tvá",
+ "tvé",
+ "tvoje",
+ "tvůj",
+ "určitě",
+ "už",
+ "vám",
+ "vámi",
+ "vás",
+ "váš",
+ "vaše",
+ "vaši",
+ "vaší",
+ "vážená",
+ "vážené",
+ "vážení",
+ "vážený",
+ "včera",
+ "vědět",
+ "však",
+ "vše",
+ "všeho",
+ "všechen",
+ "všechna",
+ "všechno",
+ "všechny",
+ "všichni",
+ "vždy",
+ "zač",
+ "zatím",
+ "zatímco",
+ "zda",
+ "zítra",
+ "žádám",
+ "žádná",
+ "žádné",
+ "žádný",
+ "že"
+ ],
+ "sl": [
+ "avgust",
+ "bila",
+ "bile",
+ "bili",
+ "bilo",
+ "biti",
+ "blizu",
+ "bomo",
+ "boste",
+ "bova",
+ "boš",
+ "brez",
+ "celi",
+ "celo",
+ "daleč",
+ "deset",
+ "težak",
+ "težka",
+ "težki",
+ "težko",
+ "često",
+ "četrta",
+ "četrtek",
+ "četrti",
+ "četrto",
+ "čigav",
+ "šest",
+ "šesta",
+ "šesti",
+ "šesto",
+ "štiri",
+
+ ],
+}
diff --git a/contrib/languages-data/sv.json b/contrib/languages-data/sv.json
new file mode 100644
index 0000000..dbcdfcb
--- /dev/null
+++ b/contrib/languages-data/sv.json
@@ -0,0 +1 @@
+{"freq":{"D":81614,"E":57703,"F":65307,"G":58568,"A":101184,"B":88354,"C":71198,"L":68342,"M":88836,"N":55651,"O":30870,"H":85820,"I":53417,"J":41709,"K":67803,"U":32439,"T":73345,"W":29206,"V":48264,"P":64540,"S":183432,"R":54519,"Y":8819,"X":3563,"Z":5647,"f":573497,"g":796472,"d":1393951,"e":3122256,"b":426497,"c":439610,"a":2769748,"n":2605268,"o":1513455,"l":1576016,"m":1015154,"j":171593,"k":981769,"h":549646,"i":1975038,"w":38244,"v":702277,"u":641405,"t":2130744,"s":2008119,"r":2700083,"q":5367,"p":544314,"z":28076,"y":232177,"x":48501,"Å":5245,"Ö":9894,"é":11827,"å":315734,"ä":562602,"ü":3814,"ö":429328," l":92862," m":210837," n":78082," o":280066," h":123124," i":363557," j":42422," k":154387," d":271388," e":324754," f":359134," g":79790," a":307677," b":150256," c":22076," y":5622," u":95544," t":189989," v":181674," p":161981," s":494735," r":78305," J":40952," K":63008," H":83187," I":43524," N":52283," O":27251," L":64682," M":82331," B":82669," C":63320," A":84049," F":59275," G":56022," D":77148," E":54653," Z":5389," Y":8451,"и":3560,"о":3458," S":163518," R":51071," P":59385,"а":4143," W":28309," V":40400," U":30253," T":68207," å":25155," ä":214016," ö":25267," Å":5217," Ö":9837,"A ":15096,"F ":4011,"Da":13315,"Cl":3705,"Co":16319,"Ce":3786,"Ch":12336,"Ed":3462,"Do":5286,"De":40359,"Di":6082,"Fe":3321,"Fa":6453,"Eu":5433,"Er":5959,"En":12408,"El":5544,"Ge":7613,"Ga":6615,"I ":11291,"Fr":15828,"Fo":6429,"Fl":4338,"Fi":8854,"B ":3976,"C ":5688,"Au":4696,"Ar":9978,"As":4625,"D ":3210,"Ba":14125,"Ad":3948,"Am":3905,"An":15253,"Al":15279,"Bu":5535,"Br":13255,"Ca":15038,"Bi":5634,"Be":16801,"Bo":13961,"Bl":4691,"Ku":4607,"Gö":6526,"Kr":7039,"Ko":8264,"Le":10243,"Li":14030,"La":13793,"Lu":7481,"Lo":9958,"Me":11631,"Mi":11197,"Ma":32867,"Mu":4647,"Mo":11964,"Ni":7405,"Ne":10761,"Na":8237,"P ":3714,"Ny":3435,"No":15326,"Ol":4960,"Gr":10737,"Go":5035,"Gu":8716,"Ha":34281,"He":15981,"II":4483,"Hi":4225,"Ho":12929,"Hu":4943,"K ":4256,"In":13186,"Is":3759,"Ja":10130,"Je":5266,"Jo":15004,"Ju":4068,"Ka":19254,"Fö":6773,"M ":4536,"Ki":6491,"Ke":4271,"Up":4420,"Un":6136,"Ty":4282,"Tu":3610,"US":9839,"Tr":7442,"To":8743,"Th":13440,"Ti":6372,"Te":8278,"Ta":5863,"V ":6654,"Sy":4319,"St":38584,"Sv":20759,"TV":4696,"Su":6305,"Wo":3225,"Wi":9262,"Wa":7328,"We":4981,"Vi":10913,"Va":8115,"Ve":4995,"Pr":8839,"S ":6544,"Pe":11351,"Pa":14568,"Po":7593,"Pi":4124,"Or":4779,"Se":8850,"Sc":5830,"Si":8063,"Sh":4489,"Sk":9057,"Sp":7173,"So":11179,"Ru":3598,"Sa":15777,"Re":10230,"Ri":6924,"Ro":13031,"SA":10320,"Ra":8269,"b ":9647,"a ":475159,"Yo":5029,"Sö":3933,"Vä":9609,"bö":8648,"i ":355245,"fy":6327,"gd":12439,"ge":171880,"gf":3700,"ga":96208,"gb":3970,"fj":3473,"fl":21495,"ff":12688,"bå":4938,"fi":46296,"bä":6295,"fr":73976,"fu":7874,"ft":39725,"fo":44582,"j ":14814,"gy":3273,"he":67749,"ha":96551,"gn":17191,"gl":17131,"gj":3942,"gi":45303,"gh":13879,"gg":27749,"gu":21320,"gt":19887,"gs":51929,"gr":63119,"go":20646,"dt":3864,"du":15217,"dv":9506,"dy":5840,"g ":162227,"ea":25581,"eb":31218,"ec":33642,"ed":130488,"de":463243,"dd":77303,"dg":4171,"df":3528,"di":70271,"dh":3483,"dk":3943,"dj":7587,"dm":5988,"dl":16308,"do":35107,"dn":14977,"ds":66886,"dr":50264,"ew":8421,"ex":20287,"eu":8550,"ev":26002,"ey":12032,"fa":41395,"h ":210556,"fe":29350,"eh":9823,"eg":38958,"ef":28657,"ee":11526,"el":240548,"ek":41076,"ej":4317,"ei":17866,"ep":29857,"eo":14539,"en":790036,"em":80183,"et":324009,"es":162582,"er":632238,"ca":14258,"e ":392431,"by":19363,"br":42376,"bu":29326,"bo":47200,"bl":35898,"bi":33086,"bb":11237,"be":121628,"db":6093,"da":116179,"f ":20176,"cu":4292,"ct":5938,"co":13763,"ck":84782,"ci":26831,"ch":224513,"ce":44300,"c ":6962,"az":4871,"ay":9160,"ba":47465,"d ":342945,"at":219537,"as":105952,"ar":493878,"ax":4752,"aw":3514,"av":157525,"au":25162,"ak":38031,"al":200396,"ai":14768,"aj":13462,"ap":42925,"am":154793,"an":490452,"ac":25353,"ad":157516,"aa":3218,"ab":17368,"ag":71089,"ah":8954,"ae":7341,"af":19685,"nu":26246,"nt":115132,"ns":230163,"nr":8077,"no":73752,"nn":76927,"jö":13694,"ny":12036,"nv":25830,"oe":5372,"of":28837,"oc":231970,"od":38865,"oa":7955,"ob":24414,"om":258987,"on":206933,"ok":32662,"ol":124858,"oi":5452,"kå":10126,"oj":3321,"og":35630,"kä":17405,"oh":11968,"ot":63544,"os":44373,"ov":30611,"ou":23986,"op":33718,"oo":9810,"or":224835,"r ":824959,"ow":7736,"kö":11089,"pe":96760,"pg":3698,"pa":50445,"pl":23211,"pn":3697,"po":41753,"ph":7115,"lä":54929,"pi":19978,"lå":14272,"lo":57125,"ln":12525,"lm":43916,"ll":251645,"ls":70931,"lp":6851,"lv":21024,"lu":27907,"lt":40778,"hö":19038,"ly":18029,"o ":47987,"md":3318,"ma":128458,"mb":42798,"mg":3435,"mh":5236,"me":195253,"mf":8701,"mk":4460,"ml":20883,"mi":57579,"mn":28583,"mm":63704,"mp":23791,"mo":40002,"mr":10561,"mt":14449,"ms":22455,"mu":35770,"my":8276,"p ":30112,"na":190258,"nb":8963,"nc":14129,"nd":274187,"ne":121340,"nf":12519,"ng":233628,"jä":20365,"nh":9404,"ni":141019,"nj":5464,"nk":26276,"nl":21248,"nm":4189,"ju":37966,"jo":14819,"gå":19638,"ki":37850,"kh":15721,"ke":98699,"ka":235170,"m ":249455,"fö":204508,"ky":14497,"gö":7789,"ks":31609,"kt":90094,"ku":21062,"kv":10463,"ko":98732,"kr":52042,"kl":32687,"km":8676,"kn":24158,"li":194145,"hå":8698,"lh":8848,"hä":12770,"lk":26056,"lj":22122,"le":171301,"ld":48550,"lg":7505,"lf":10995,"la":251597,"lb":23488,"n ":932062,"hr":6279,"ht":5130,"hu":27638,"hj":3586,"dä":13148,"då":7895,"hi":28029,"hn":5105,"ho":40609,"hl":4048,"id":81847,"ic":45251,"ib":12524,"ia":49113,"ig":122539,"if":22472,"ie":80770,"dö":42849,"hy":3755,"k ":153527,"ir":29485,"is":203197,"it":107910,"iu":7188,"iv":58134,"ix":3284,"ii":3814,"ik":122112,"il":185111,"im":21187,"in":352330,"io":82024,"ip":11363,"je":24984,"få":5419,"fä":10915,"iz":3965,"l ":174554,"ja":31220,"tä":25279,"xi":4781,"tå":13823,"xt":11670,"sö":12206,"z ":7095,"xa":4078,"xe":8556,"sä":22008,"wi":3916,"så":23262,"rö":18726,"y ":50286,"wa":8836,"we":6979,"vl":8639,"rä":38698,"rå":61767,"vi":89468,"vt":4439,"vu":11778,"vr":3344,"vs":16546,"vn":4095,"vo":9064,"uv":14989,"ve":165060,"vd":3502,"va":154942,"x ":11441,"ui":6645,"uk":16033,"ul":41627,"ue":11748,"ug":17372,"ur":68778,"us":77613,"ut":71499,"um":42001,"un":129135,"up":38948,"ty":33055,"tu":36176,"tt":213949,"tv":19150,"ub":16763,"ua":23575,"ud":29487,"uc":9950,"w ":8904,"to":108377,"tn":19214,"tm":5831,"tl":17707,"ts":67257,"tr":105466,"tg":11122,"tf":8472,"te":298793,"tk":5397,"tj":8416,"ti":267169,"på":66737,"th":27907,"v ":145502,"tb":14691,"ta":250228,"su":13835,"sv":52306,"ss":78482,"st":354837,"sy":19796,"sl":47405,"sk":296865,"sn":10814,"sm":23482,"sp":69363,"so":182040,"sr":7516,"sd":11605,"sc":12968,"sf":11899,"se":125434,"sh":21233,"sg":4215,"sj":15800,"si":100666,"nö":3871,"u ":12792,"sa":104819,"sb":15311,"rr":35803,"rs":138379,"rt":88500,"ru":68780,"rv":15260,"ry":24021,"rp":8643,"ro":113836,"rn":86552,"rm":33875,"rl":42446,"rk":70077,"rj":9549,"ri":267243,"nå":6703,"nä":22292,"rh":9656,"rg":57899,"rf":20073,"re":276377,"rd":70096,"rc":6855,"rb":28710,"ra":275879,"t ":541739,"mö":6674,"qu":3576,"må":14078,"mä":17850,"s ":365755,"lö":8782,"pt":22670,"pu":12367,"pp":63825,"pr":69684,"ps":16974,"vä":52142,"zi":3697,"vå":20606,"za":3995,"yg":18262,"yf":3366,"yc":9359,"yd":14899,"ya":7829,"tö":15637,"yt":14189,"ys":34208,"yr":23125,"yp":6499,"yn":12472,"ym":10053,"yl":9018,"yk":5316,"å ":92466,"äc":6371,"Ös":5286,"ö ":8541,"én":3279,"åv":3406,"ån":81191,"åt":17902,"ås":6794,"år":43331,"åg":13520,"åe":3379,"ål":17121,"åk":8868,"åd":24022,"ät":25342,"äv":21618,"äx":8215,"äm":18322,"äl":34618,"än":82390,"äp":6744,"äs":34864,"är":272645,"äd":8120,"äg":24196,"äk":15670,"öv":17529,"öt":11908,"ör":194602,"ös":15048,"öp":9470,"ön":13721,"öl":7954,"öm":7511,"öj":4231,"ök":5353,"ög":10789,"öd":115283,"一":3570," Ga":6575," Ge":7551," I ":6565," Fo":6382," Fr":15807," Fi":8815," Fl":4322," Ha":34254," He":15951," Go":5001," Gr":10670," Gu":8675," Hu":4927," Ho":12906," Hi":4166," Je":5254," Ja":10108," Is":3740," In":13126," Fö":6755," Ka":19203," Ke":4195," Ki":6438," Jo":14976," Ju":4056," La":13680," Le":10167," Li":13966," Ko":8252," Kr":7023," Ku":4598," Gö":6518," Ma":32732," Mi":11127," Me":11596," Lo":9918," Lu":7453," Ne":10698," Na":8152," Ni":7391," Mo":11908," Mu":4615," Am":3890," An":15211," Al":15206," Ad":3934," Ba":14044," Au":4688," As":4526," Ar":9902," Be":16738," Bi":5612," Bl":4680," Bo":13888," Br":13201," Bu":5515," Ca":14547," Ce":3768," Ch":12294," Cl":3646," Co":16148," Da":13258," Di":6045," De":40274," Do":5127," Ed":3453," El":5523," Er":5935," En":12351," Eu":5425," Fe":3308," Fa":6390," Sö":3929," Wi":9211," We":4953," Wa":7296," Vä":9600," Yo":5022," Or":4762," Po":7530," Pi":4113," Pe":11321," Pa":14488," Ny":3423," No":15289," Ol":4952," Ra":8225," Ro":12950," Re":10191," Ri":6904," Pr":8806," Sy":4310," Sv":20662," TV":4398," Su":6295," St":38280," Ta":5840," Th":13403," Ti":6347," Te":8213," US":9725," Tr":7389," To":8642," Ru":3585," Sa":15732," Sh":4450," Si":8029," Sc":5776," Se":8798," So":11126," Sp":7111," Sk":9040," Va":8077," Ve":4954," Vi":10865," Tu":3571," Ty":4268," Un":6118," Up":4407," ja":13806," få":4608," fä":3840," in":71154," is":5201," ka":33444," fö":177054," gå":6683," ki":4333," ke":3269," jo":4366," ju":20755," ha":54753," he":14640," gi":5926," gr":27602," gu":3989," dö":38312," id":4924," dä":12563," då":6671," hi":6661," ho":7645," hu":13035," ne":4922," na":17184," my":5673," mu":11845," mo":16924," ok":9806," ol":7701," om":30114," kä":11926," oc":192923," of":14077," ny":4446," nu":5991," no":24830," le":9968," hä":8278," li":23241," la":18204," kv":5049," ku":9406," ky":7530," km":5693," kl":6892," kr":12841," ko":48113," me":100026," mi":15594," ma":40584," hö":10368," lo":3574," ad":5086," am":17761," an":46161," ap":9654," ak":3619," al":18398," av":131499," au":10184," ar":19850," at":35597," ba":19765," bi":15445," be":42377," bo":12025," bl":19693," by":8036," br":16393," e ":3326," et":57787," en":197679," el":30240," ef":10304," eg":6206," fe":13408," fa":16157," ex":7709," fu":4697," fr":64090," fo":25076," fl":15697," bå":3785," fi":26001," ge":16400," ga":6755," i ":270663," bö":6965," fy":5095," ce":5878," ci":5961," da":13590," do":7611," dr":6511," de":166176," di":13002," vä":22816," yt":3626," tä":6373," sö":6783," ru":4704," ry":5559," sa":28037," se":37040," sj":9848," si":31916," sm":3812," sl":17774," sk":41581," sp":27636," so":132908," mö":3254," ra":6370," re":30692," ri":11458," nå":5590," nä":11487," ro":9439," pu":3945," pr":32472," ps":3376," s ":6480," mä":6492," må":7724," or":14441," kö":4010," pe":10275," pa":12607," pl":6656," po":22401," lå":10106," lä":20032," rö":4188," så":14446," sä":12510," va":97104," ve":11661," rä":4287," vi":43226," ty":10671," tv":8751," tu":4474," ut":38370," ur":7073," up":19177," un":29257," ta":17457," sy":11768," st":66122," sv":36306," su":3592," tr":20781," to":9251," th":7205," på":64716," ti":85531," te":14430," Ös":5280," år":16994," åt":5457," än":4390," äl":3304," är":187240," äv":12511," ös":4997," öv":12751,"Fin":4121,"Eri":4081,"Eur":4760,"En ":5663,"Eng":3595,"Öst":5201,"Fra":6710,"Fre":3915,"Hel":4484,"Her":3363,"Han":20213,"Har":3270,"Gra":3275,"Ind":4121,"Hon":4948,"Alb":3593,"And":5658,"Car":7211,"Ber":6424,"De ":4186,"Det":11763,"Den":16516,"Dan":4412,"Cha":4718,"Cou":3952,"New":5646,"Nor":12355,"Per":3613,"Pet":3811,"Par":5960,"Pro":3506,"SA ":9788,"Joh":8659,"För":6368,"Kal":4090,"Kar":6529,"Göt":5300,"Lin":4919,"Man":3676,"Mal":3953,"Mar":11224,"Söd":3244,"Wil":4251,"Väs":5464,"Yor":3342,"Sve":19151,"Str":3346,"Sto":17434,"Sta":8398,"Ste":4189,"TV ":4653,"äga":3677,"äge":8468,"ägg":3687,"äck":6350,"änn":6181,"äns":11234,"ämn":7168,"äms":3895,"äng":10799,"änd":34964,"äpp":6344,"Sch":3849,"San":4701,"är ":211888,"älv":4078,"äll":14085,"äkt":7912,"äkn":3856,"äld":3913,"än ":11102,"äve":13254,"ävl":6172,"äxt":6157,"ärk":4071,"ärl":8853,"ärm":4235,"ära":8245,"ärd":6093,"äre":3225,"ärn":6279,"ärs":4830,"äst":24789,"ätt":18556,"åde":18255,"ågo":4233,"åna":9220,"ång":25948,"åll":6899,"ån ":40117,"åre":4622,"åt ":4155,"ård":7005,"år ":24938,"åte":4737,"ått":4014,"Upp":4062,"Tys":3259,"The":8834,"USA":9498,"bis":3915,"bil":16029,"bin":3462,"ble":7136,"bli":9670,"bla":13299,"bok":5630,"bol":12338,"bor":13470,"bbe":3314,"ban":15887,"bas":6749,"bar":10200,"beg":3230,"ber":59590,"ben":7083,"bel":10187,"bes":11143,"bet":17916,"ca ":4920,"ce ":7249,"bri":10671,"bro":7219,"bra":3701,"bre":3338,"bru":13260,"bur":4739,"bun":4581,"bum":12276,"by ":6960,"byg":8032,"aka":4483,"am ":15544,"ake":4072,"al ":27066,"ain":4963,"aj ":9347,"ags":6901,"agn":5084,"anv":11075,"anu":13080,"ano":4964,"ann":22378,"ant":22398,"ans":71523,"ane":7517,"anf":3613,"ang":11657,"ani":15707,"ank":9321,"anl":8379,"ap ":4055,"ana":13418,"anc":5238,"and":127973,"amt":9744,"amm":21193,"aml":16952,"amo":3480,"amn":18156,"amp":5088,"amh":4267,"ami":11713,"amf":3558,"ame":25638,"amb":4074,"ama":6369,"alv":3571,"alt":10698,"als":5130,"alo":3365,"alm":9277,"all":37991,"alk":5184,"ali":21293,"ald":6403,"ale":25866,"ala":19113,"alb":12779,"an ":132556,"akt":15324,"ad ":44388,"aft":3394,"aff":3221,"afi":4365,"aga":8161,"age":18047,"adm":3257,"adi":7918,"ade":78504,"ag ":17369,"ads":8986,"ack":8031,"ach":4677,"ace":4160,"ada":3703,"af ":3707,"at ":35834,"are":94208,"ard":11975,"arb":9854,"ara":27724,"aro":4452,"arn":18125,"arm":4137,"arl":13544,"ark":16487,"ari":37821,"arr":8387,"ars":21054,"art":35107,"asi":5423,"ase":5556,"ask":3632,"ar ":171511,"apa":8845,"ape":7645,"app":4146,"apr":9022,"as ":45411,"ava":3381,"avs":7927,"avi":4830,"ave":5436,"ay ":3818,"av ":125744,"ata":7543,"ast":21648,"ass":11560,"ato":8615,"ate":21603,"ati":48093,"att":63033,"ats":14038,"atu":6856,"aur":3255,"aug":8539,"jer":3396,"jen":6427,"fäl":3203,"fär":6347,"jan":14542,"je ":5239,"jor":7802,"itu":3337,"itt":20776,"ity":3491,"isk":74955,"ism":4171,"iss":12117,"ist":54624,"iv ":6421,"ita":12733,"ite":17827,"iti":20829,"ius":3596,"iva":11600,"ivi":7901,"ive":20061,"is ":19720,"ion":60198,"ir ":3453,"irk":5584,"isi":5287,"ish":4776,"ise":7902,"isa":8098,"ire":6030,"it ":10727,"ja ":4918,"kil":7942,"kiv":4415,"kin":6276,"gån":6463,"går":10145,"kis":6751,"kho":12716,"kel":5730,"ken":19496,"kes":3520,"ker":29895,"ket":13180,"key":3901,"ke ":13529,"kra":8741,"kre":6291,"kt ":28682,"ksa":5304,"ksd":4444,"kro":5638,"kri":26131,"kot":3776,"kor":12173,"kon":20182,"kom":35824,"kol":9820,"ks ":3388,"kna":9166,"kni":12636,"klu":5282,"kla":15409,"kli":6794,"jul":9684,"jun":10738,"jur":5620,"kat":7694,"kar":20696,"kas":4078,"kap":15909,"kan":45226,"kal":21132,"kam":3425,"kad":6591,"ka ":105354,"för":137814,"föd":63483,"ha ":4594,"ham":7622,"han":27562,"hal":5140,"hav":3856,"har":31426,"had":5793,"he ":12764,"hel":8034,"het":17433,"her":10299,"hen":4180,"hem":4216,"då ":5689,"där":12388,"hin":3839,"his":5950,"gli":6286,"gla":6605,"gni":3551,"gna":6982,"gs ":13333,"gon":4582,"gor":4044,"gsk":4034,"gru":20067,"gra":18317,"gt ":15114,"gre":14703,"gst":6826,"gsm":3229,"gus":10096,"grä":4757,"ial":7807,"ian":12040,"ic ":4032,"ibl":3325,"id ":28142,"ibe":3733,"ia ":17620,"iet":8507,"iel":7300,"ien":33298,"ier":11523,"ies":3414,"ig ":23812,"ift":11636,"ick":11163,"ici":4587,"ich":7049,"ice":6107,"ie ":9770,"ica":5181,"ids":4482,"idr":4877,"idn":3610,"idi":9977,"ide":12497,"ida":10207,"il ":15253,"ika":38088,"ige":28072,"iga":25749,"igh":9261,"igi":4197,"igg":9635,"igt":12741,"ik ":16104,"ime":3935,"ind":16350,"ina":18896,"inn":24862,"ino":16003,"int":17796,"ins":22420,"inf":4654,"ine":12860,"ing":135978,"ini":12277,"inl":6319,"ink":4368,"inv":10239,"ike":30688,"ila":3896,"in ":36372,"ikt":12966,"iks":9950,"ilo":5583,"ill":91188,"ilk":6078,"ilm":10451,"ilj":12364,"ili":8683,"ild":15424,"ile":4047,"io ":6788,"ils":3516,"hol":17007,"hon":3609,"hri":3262,"hum":3706,"hus":7589,"huv":8614,"död":37434,"fes":6198,"fer":3302,"feb":8379,"fat":13524,"far":5867,"fam":7188,"fal":3610,"ext":4721,"exe":4418,"eta":15891,"ete":27600,"eti":4494,"esp":9596,"est":29943,"ess":19229,"ev ":6530,"etr":3912,"ets":17582,"ett":72343,"ety":4212,"ew ":5717,"eve":6119,"eva":3607,"evi":3537,"ey ":7588,"elä":5847,"er ":284777,"eor":5154,"es ":68641,"ept":9611,"epp":4218,"epr":4795,"erk":19481,"erl":7951,"eri":65274,"erg":20456,"erh":4470,"enä":3549,"ere":6711,"erf":4436,"era":63522,"erb":6250,"et ":167149,"esk":4221,"esi":7482,"ese":7369,"erv":5509,"err":9816,"ert":13225,"ers":49047,"ern":40094,"erm":7787,"ero":5248,"eki":3784,"eko":7026,"ekt":14604,"en ":561421,"ela":36480,"ele":19021,"eli":7809,"eln":6244,"ell":67565,"els":32415,"elt":6933,"emb":26969,"ema":5588,"eme":8121,"emm":3946,"emo":6706,"emi":7410,"emp":5979,"ene":8729,"enh":4318,"eng":9041,"enb":3805,"ena":11883,"end":16477,"eno":10317,"enn":10256,"eni":7932,"ens":78815,"ent":44100,"enr":3651,"ege":11795,"egi":9515,"egr":5273,"eis":3601,"ein":4280,"el ":33296,"em ":8949,"öte":5512,"gjo":3575,"öst":10660,"git":4157,"gis":8031,"giv":6533,"gin":4809,"gio":4718,"gic":3318,"gif":3672,"örs":34131,"öra":7478,"örb":6194,"örd":7646,"ghe":6517,"öre":25188,"örf":10182,"örj":5505,"örk":3758,"ggn":3889,"gge":11686,"gga":4221,"gi ":4509,"öpi":3748,"ör ":72432,"gen":70552,"get":15734,"ger":32986,"ges":7614,"gel":15568,"gde":4040,"ge ":21010,"ön ":5859,"öm ":3264,"gas":5062,"gar":34878,"gat":4458,"gan":12485,"ga ":26894,"ögs":4071,"ödr":4070,"bör":6225,"frå":39311,"frä":4616,"fte":17736,"fta":8409,"fun":3567,"ft ":9311,"fra":14521,"fri":9705,"for":23834,"fot":7918,"fol":8477,"fle":7024,"flo":3656,"fly":6056,"fic":5656,"fil":11844,"fik":3556,"fin":14513,"fis":3388,"öve":12758,"da ":24533,"dd ":64224,"de ":113906,"dad":11525,"dal":5199,"dag":13716,"dat":8203,"das":4305,"dar":13228,"dan":22463,"dam":6624,"dda":3257,"dde":6028,"cks":6976,"ckh":12838,"ckn":5609,"ckl":6273,"öd ":37381,"öde":6673,"ödd":63162,"ch ":194706,"cer":9280,"cen":9841,"cem":8851,"cha":5427,"cia":6069,"ck ":16616,"cie":3837,"che":9375,"chi":3577,"cir":4840,"cke":18899,"cka":6286,"ed ":60806,"ebo":7090,"ebr":11024,"ean":3203,"eat":4659,"ea ":3260,"efo":3657,"eft":11252,"edl":4748,"edi":6332,"edd":3460,"ede":13539,"eda":22012,"edr":4279,"eck":13530,"eci":3498,"ece":9407,"dvä":3688,"dor":6126,"don":6243,"dom":6715,"ds ":21182,"dmi":3679,"dni":12167,"dst":6539,"duc":4844,"dri":7974,"dra":24350,"dre":6617,"dro":7925,"dsk":11908,"dia":4497,"der":69482,"des":48453,"det":49713,"dec":9237,"del":42235,"den":110439,"dem":7457,"dle":5138,"dla":5392,"dli":4421,"din":7458,"dio":6335,"dis":13125,"dit":3226,"die":4951,"dig":14250,"dju":3500,"näm":4493,"när":11567,"näs":3945,"rga":9150,"ri ":24498,"rgi":4148,"rge":9457,"rgs":6893,"ret":18000,"res":20396,"rev":5759,"rfa":11243,"rds":5163,"rg ":18322,"rea":6436,"red":13615,"reg":14934,"rem":4053,"ren":33831,"rek":11220,"rel":8037,"rer":8988,"rep":9910,"rda":6312,"rdn":3815,"rdi":6203,"rde":19138,"re ":105451,"rbu":4396,"rd ":16211,"rar":16692,"ras":12995,"rat":27144,"rav":4107,"rbe":10058,"rag":4677,"ran":40758,"ram":18971,"ral":14704,"rak":6953,"rab":3374,"raf":9108,"rad":32041,"rs ":30266,"rr ":4068,"rlä":3700,"ror":5868,"ros":5853,"rot":10704,"rom":7155,"ron":11508,"rop":10007,"rov":7683,"rod":8744,"roc":6434,"rol":8031,"rof":6006,"rog":7715,"rna":45582,"rne":7715,"rni":4580,"ro ":5673,"rma":11324,"rme":6990,"rli":7045,"rld":8530,"rle":4342,"rla":6287,"rn ":13803,"rks":6561,"rko":6401,"rki":4271,"rke":8992,"rka":21148,"rm ":4966,"rja":6006,"rl ":5718,"rio":4391,"rit":18193,"ris":26661,"riv":11699,"rig":24904,"någ":4585,"ril":11007,"rik":49805,"rin":29008,"rim":3215,"ria":11330,"ric":6800,"rid":7589,"rie":22621,"rif":3813,"rk ":11840,"rup":12524,"run":19578,"rum":7564,"ruk":6602,"rus":3564,"rva":5265,"rvi":3304,"ry ":6473,"rsk":18149,"rsi":7847,"rso":8438,"rsp":7925,"rsa":13567,"rse":4696,"rta":8444,"rst":26007,"rss":3568,"rte":14318,"rth":3501,"rti":15874,"rua":8438,"rts":3730,"rt ":25088,"rri":6493,"rre":7456,"rra":8312,"sak":4471,"sal":8723,"sam":44053,"san":6209,"sat":10892,"sar":8127,"sa ":10458,"rys":4892,"sho":4590,"shi":4912,"sju":4204,"sie":4171,"sid":6241,"sk ":90513,"sit":12645,"sis":10405,"sin":20425,"sio":10464,"sik":13865,"sig":10296,"sda":5839,"sde":3304,"se ":11017,"sch":5874,"ser":32108,"ses":3294,"set":6663,"sed":10233,"sep":9935,"sen":30980,"sel":3990,"spo":4913,"spr":13840,"slä":13781,"spe":39400,"spa":4154,"som":124363,"son":32242,"sor":7813,"skå":7923,"soc":6710,"st ":46043,"ss ":9668,"sli":3748,"slo":3319,"slu":6612,"sky":3403,"sla":13504,"sle":3801,"ski":13633,"skl":5724,"sko":16207,"skr":17282,"sku":3455,"skt":14119,"sfö":5604,"ska":106537,"ske":8526,"sjö":6673,"sni":4681,"sjä":3770,"sma":7643,"sme":3479,"stå":11548,"stä":10967,"syd":3945,"stö":8124,"sys":5192,"svä":4118,"sse":11502,"ssa":10371,"sso":16553,"ssl":3516,"ssi":9151,"sst":3551,"ssp":3681,"ste":53976,"sta":88293,"stn":5297,"sto":24174,"sti":35352,"stu":7051,"str":43893,"sty":3941,"sva":8262,"sve":36524,"tal":35934,"tag":10048,"tad":34944,"tav":3738,"tat":22284,"tas":6110,"tar":38387,"tan":25694,"tam":3270,"te ":29236,"tbo":6832,"ta ":57380,"pa ":4951,"par":19834,"pas":4154,"pan":9819,"läg":9136,"lär":4866,"läp":5670,"län":20491,"läk":8254,"pen":15225,"per":22100,"pet":5946,"pel":41073,"pla":14119,"pin":6560,"lån":4645,"pis":3811,"låt":5580,"por":8039,"pop":3841,"pos":4574,"pol":17194,"pps":6631,"ppt":7044,"ppl":4401,"ppa":4036,"ppe":13206,"pp ":11986,"pub":4680,"pte":14474,"pru":4841,"psa":5456,"pri":16288,"pre":11182,"pro":27839,"prå":5570,"män":8000,"mäs":4442,"mål":5318,"mån":4648,"ra ":70040,"ngl":9266,"ngr":3616,"ngt":3834,"ngs":25507,"ni ":11042,"nge":55986,"nga":25508,"ngd":5985,"jäl":6437,"jär":7781,"nhe":3391,"neh":3788,"nel":8830,"nen":21817,"ner":30394,"net":14471,"nes":10240,"ng ":83505,"neb":3747,"ned":3616,"nce":6161,"ne ":14860,"ndr":19487,"nds":29703,"ndo":7016,"ndl":5881,"ndi":11752,"nde":95443,"nda":25018,"nal":16599,"nam":15523,"nan":10186,"nar":23024,"nad":14500,"nd ":63041,"nat":21915,"nas":10608,"na ":68634,"ny ":3572,"num":3857,"nus":4249,"nua":9868,"nty":4644,"nto":5785,"ntr":13553,"nti":10623,"nta":12228,"nte":31914,"nsp":4485,"nst":23393,"nss":4316,"nse":13545,"nsi":5055,"nsl":4609,"nsk":92387,"nsa":6209,"nri":3216,"nt ":21290,"ns ":54594,"nom":26142,"nor":19884,"nov":9987,"nne":21868,"nna":22185,"nno":3272,"nni":10650,"nns":9117,"nli":10002,"nn ":5037,"nla":5932,"no ":3529,"nke":3380,"ngå":3281,"nkt":4581,"nkr":3354,"nfö":4263,"nie":10939,"nia":3660,"niv":6000,"nis":23117,"nit":5715,"nio":3817,"nin":62497,"nik":4407,"ogr":10374,"ogi":6891,"ohn":3565,"kän":11729,"oha":5651,"kåd":7525,"ok ":3568,"ol ":3524,"och":189597,"oci":4767,"ock":32660,"ode":11255,"of ":8033,"odu":7243,"og ":7992,"oft":6611,"off":4118,"ofe":5148,"od ":6498,"obe":13311,"nvä":12582,"nvå":8371,"jör":3329,"köp":4602,"ote":5241,"ott":18376,"ots":4197,"oto":5671,"ost":8837,"ota":4879,"otb":6812,"osi":3701,"ose":3675,"oss":3221,"ovi":7790,"ove":12742,"oun":6352,"our":5919,"opp":7590,"ope":5756,"opa":4338,"os ":9800,"or ":33039,"ork":5444,"orm":13532,"orn":10771,"orr":11303,"ord":31398,"ore":8252,"org":22560,"ori":16030,"osa":3703,"ort":28485,"ors":16477,"ot ":11261,"ora":9653,"ola":11528,"on ":79327,"oli":25700,"oll":18015,"olk":10298,"ole":5201,"ols":3442,"olm":16608,"olo":12711,"oly":3210,"oka":3360,"om ":168644,"oke":3223,"okr":4246,"okt":9964,"ona":12877,"ond":7866,"one":32789,"ong":8629,"oni":7172,"ono":6812,"ons":23539,"ont":10940,"oma":9422,"ome":9673,"omb":3421,"omi":4970,"omm":30299,"omk":3355,"omp":5523,"omr":7897,"oms":4749,"op ":3482,"la ":37263,"le ":15487,"lde":11008,"lda":11000,"lds":6297,"ldr":3289,"lac":3318,"lad":21513,"lag":21635,"lan":79933,"lam":3972,"lar":35655,"lat":17536,"las":17797,"lba":3463,"ld ":9319,"lbu":12650,"kvi":3264,"kva":4810,"kus":3478,"kun":6039,"kul":5875,"kså":3945,"kta":4658,"kte":14929,"kti":15289,"kto":12389,"kyr":9176,"gör":5454,"ls ":8684,"lom":6295,"lor":5534,"lod":5577,"log":11812,"los":3900,"lot":3451,"lni":4907,"lme":6188,"lma":4184,"lms":5047,"lti":5165,"lub":5423,"lsk":12154,"lss":5166,"lst":11990,"lta":4364,"lte":4556,"lse":12155,"lsa":4733,"lt ":15274,"häl":3201,"här":4070,"li ":11521,"lev":10258,"les":8492,"let":19531,"ler":49986,"lem":6956,"len":25248,"lek":6522,"led":11323,"lls":18041,"llt":8125,"llv":5128,"lhö":4242,"lla":46043,"lle":53525,"llh":4880,"lli":11251,"lln":3548,"lkr":4019,"ln ":4441,"lke":7097,"lm ":18320,"lje":9323,"ll ":78820,"lja":3349,"lit":24473,"lis":16271,"lin":35763,"liv":5593,"lic":4558,"lia":6013,"lik":14533,"hål":7111,"lig":48482,"lie":9952,"ma ":10795,"maj":9292,"mar":31731,"mas":5427,"mal":5574,"man":40392,"mat":14610,"mbe":28393,"me ":3701,"med":67946,"met":18659,"mes":8746,"mer":42686,"mel":15875,"men":33082,"lva":3742,"lve":7060,"lun":3206,"lut":8771,"lyg":4277,"hög":7787,"hör":6448,"mpi":3263,"mpe":6539,"mpo":3301,"ms ":6348,"mod":5091,"mon":5448,"mok":4020,"mor":4951,"mot":12947,"mt ":8137,"mst":8336,"mrå":8280,"mus":13621,"mun":18113,"mfö":3853,"min":18579,"mil":15426,"mis":6696,"mit":4808,"mli":14006,"mla":4634,"mn ":9157,"mni":3641,"mne":9027,"mmu":16790,"mma":26822,"mme":13993,"vå ":7032,"väg":7022,"vän":13490,"vär":10896,"väs":8080,"väx":7295,"vån":8847,"ytt":4640,"yta":3770,"yst":7068,"ysk":15364,"yrk":11273,"yra":3515,"yde":3835,"yck":8174,"ya ":3709,"ygg":9192,"xte":5064,"tör":12969,"täl":6340,"xem":4510,"tår":5717,"tär":4618,"täv":5108,"söd":5258,"så ":8428,"sån":9241,"sät":7762,"röm":5410,"rör":3335,"vs ":5015,"vud":9343,"rät":6983,"råk":6071,"vik":5560,"vil":12065,"rån":39221,"vin":14334,"råd":11141,"vid":23789,"vit":5788,"vis":16981,"vli":3664,"rän":8989,"räk":4195,"räm":4496,"räd":4698,"ver":61891,"vet":10570,"ven":64039,"vem":8723,"vec":5468,"ve ":6856,"val":12123,"van":15891,"var":99905,"vat":6662,"va ":9694,"uvu":9284,"usi":13675,"use":6334,"ust":23739,"utg":8418,"uti":3815,"ute":6086,"uta":8659,"utt":3405,"uts":5227,"utv":5038,"us ":21399,"ut ":10089,"ura":3453,"ure":6299,"urg":5379,"uri":5728,"urn":5209,"uro":7154,"urs":7342,"ur ":12946,"upp":32257,"umb":4012,"ume":9374,"unt":7357,"unk":5142,"uni":16744,"unn":5336,"und":51074,"ung":19464,"une":3661,"ukt":5611,"um ":17518,"ult":6645,"ull":6732,"uli":10267,"un ":12274,"ugu":9934,"ude":4661,"udi":4906,"uce":4932,"uds":4702,"udo":3784,"uar":18907,"ubl":5383,"ubb":6201,"två":7001,"typ":3540,"tyr":3920,"tys":8073,"ty ":8321,"trö":5020,"tve":6253,"trä":9790,"tur":14907,"tun":3943,"tud":5778,"tyd":4220,"ts ":26900,"tre":12611,"tt ":116227,"tra":36819,"tri":14508,"tru":8774,"tro":12216,"try":3989,"tse":5043,"tsk":5040,"tst":3437,"tta":33856,"tte":27085,"tti":9635,"ttn":5259,"tts":5893,"ttr":3221,"to ":4895,"tni":11444,"tjä":4558,"tna":3229,"tod":3739,"toc":13127,"tog":4722,"tob":9412,"tom":4310,"ton":13637,"tol":5901,"tor":39112,"til":75428,"tik":17763,"tif":5408,"tie":6576,"tig":8660,"tit":6965,"tis":30207,"tin":14073,"tio":41843,"thu":4634,"tia":4024,"tic":3207,"tid":19952,"tiv":14860,"tli":6668,"tla":6760,"tem":18447,"ten":56126,"tek":5853,"tel":9950,"teb":4880,"tec":5494,"th ":6931,"tex":3539,"tet":22695,"tes":9480,"ter":115723,"tgi":4603,"på ":62169,"ti ":13119,"the":7679},"n_words":[31862602,36956776,26222440],"name":"sv","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/sw.json b/contrib/languages-data/sw.json
new file mode 100644
index 0000000..e5cefd2
--- /dev/null
+++ b/contrib/languages-data/sw.json
@@ -0,0 +1 @@
+{"freq":{"jer":348,"jen":305,"ji ":6234,"D":1805,"E":874,"F":1081,"G":1202,"A":4461,"B":2717,"C":2251,"L":1530,"M":12761,"N":2782,"O":860,"H":1677,"I":2605,"J":2641,"K":12188,"U":3120,"T":5185,"W":4730,"V":1116,"P":2090,"S":3343,"R":1632,"Y":517,"Z":395,"f":11048,"g":13829,"d":15034,"e":46694,"Feb":214,"b":19688,"c":9784,"a":289584,"n":90468,"o":57043,"l":42025,"m":53651,"j":21456,"k":76835,"h":32492,"i":164978,"w":60984,"v":3863,"u":57506,"t":40551,"s":35298,"r":27443,"p":13501,"z":18893,"y":38832,"x":501,"jar":185,"jan":137,"jaw":201,"é":167,"jim":1500,"jin":4267,"jil":163,"jij":492,"jia":221,"jib":3854,"ito":288,"itu":317,"itw":269,"isp":140,"ist":592,"ita":1061,"ite":213,"iti":334,"ivy":133,"iwa":2430,"ius":183,"ipo":224,"ipi":265,"is ":521,"ion":720,"iop":279,"ipa":165,"ipe":219,"iro":173,"iri":997,"isi":902,"ish":5756,"isa":694,"ire":164,"ira":314,"ja ":1529,"iyo":4644,"iye":227,"izo":242,"izi":413,"iza":568," l":8602,"kif":518," m":27935," n":19872," o":327,"kik":333," h":7652," i":9059,"kij":166,"kim":258," j":5212,"kil":389," k":27977," d":1010," e":802," f":914,"kia":390," g":257," a":6533," b":1252," c":2191,"kiw":279," y":17767," z":2257,"kin":442," u":4361,"kio":148," t":2402,"kip":379," w":34366," v":1482,"kis":520," p":2154,"kit":315," s":6097," r":837,"ki ":2193," J":2627," K":12017," H":1638," I":2128," N":2678," O":803," L":1487," M":12665," B":2646," C":2112," A":4277," F":1046," G":1172," D":1740," E":782," Z":375," Y":513,"и":142," S":3229," R":1588,"а":137," P":2015," W":4707," V":1031," U":3052," T":5117,"kea":156,"kem":150,"ke ":1988,"ku ":187,"kri":520,"kon":141,"koa":3734,"ko ":1214,"ل":165,"ا":240,"juu":155,"jul":257,"jum":177,"kaz":5045,"kaw":137,"kat":14149,"kar":374,"kas":316,"kan":2795,"kao":197,"kal":354,"kam":1048,"kad":160,"kab":375,"ka ":19783," Ga":196,"Da":365," Ge":229,"Co":364," Fr":177,"Ch":770," Ha":622," He":218," Go":142,"Do":469," Gr":177," Gu":142,"De":497,"Di":169,"Fe":311," Id":148,"Fa":160," Hu":173," Ho":177," II":154,"ha ":2668," Hi":392,"Ge":229," Ji":535,"Ga":198," Je":286,"I ":397," Ja":792,"Fr":177," Ir":284," Is":141," It":181," In":316," Ik":143," Il":224,"ham":522,"han":444,"hap":154," Ka":2225,"hai":238,"haj":163,"hak":611,"hal":314," Ke":708," Ki":3568,"har":1714,"has":255,"hat":148," Jo":255,"II ":207," Ju":691,"hag":267,"hab":181,"had":740," La":231," Le":207," Li":441," Ko":414," Ku":695," Kw":4009,"Au":181," Ma":4258," Mb":461,"Ar":475,"As":222," Mk":3388,"Ba":771," Mi":685," Mj":478," Me":615,"Af":445,"he ":544,"Ag":372," Lo":213,"Am":241,"An":463,"Ap":290," Lu":315,"Al":840," Ne":518,"Bu":429,"Br":278," Na":464,"Ca":592," Ni":435,"Bi":308," Mt":420,"Be":362," Mp":146," Mo":643,"Bo":282," Mu":471," Mw":545,"Ku":695,"Kw":4009,"Ko":415,"hez":299,"Le":210,"Li":441,"hes":336,"her":275,"hen":226,"hem":395,"La":231,"Lu":315,"Lo":213,"Me":621,"hi ":3880,"Mi":690,"Mj":478,"Mk":3388,"Ma":4263,"Mb":461,"Mw":546,"Mu":475,"Mt":420,"Mp":146,"Mo":643,"Ni":437,"Ne":518,"Na":466," Ap":290," Am":240," An":463," Al":833,"Ny":247," Ag":372," Af":443,"No":466," Ba":766,"Ok":277," Au":181," As":222," Ar":474," Be":362," Bi":308,"hio":2603,"Gr":177,"Go":143,"hin":1991,"him":244,"hil":432,"Gu":142," Bo":282,"hii":230," Br":278," Bu":429,"his":266,"hir":394,"Ha":622," Ca":582,"hiy":239,"He":219,"II":286,"Hi":393," Ch":768,"Ho":179,"Hu":173," Co":362,"K ":152,"Id":148," Da":365," Di":167,"In":317," De":495,"Ik":143,"Il":226,"Is":141,"It":181," Do":469,"Ir":284,"Ja":792,"Ji":536,"Je":286,"Jo":255,"Ju":691,"Ka":2234,"Has":225,"ho ":334," Fe":311,"Ki":3577," Fa":159,"Ke":708,"Us":172,"Ut":325,"Ur":181,"go ":920,"Un":355,"Uk":150,"Ul":189,"Ui":244,"Uj":249,"Uh":170,"Uf":251,"Uc":175,"Tu":237,"To":205,"Th":275,"Te":258," Wi":3377,"Ta":3841," We":188," Wa":1003,"St":260,"Su":178,"Wi":3380,"Wa":1003,"We":189," Zi":141," Za":152,"Vi":670," Yo":250,"Pr":150,"Pe":270,"goz":233,"Pa":858,"Po":195,"Pi":163,"gom":190,"gon":205,"gos":279,"gor":306,"Se":532,"gu ":424,"Si":424,"Sh":518,"So":239,"Ru":370,"Sa":668,"Re":188,"Ri":138,"Ro":385,"Ra":354," Po":195,"guj":253," Pi":163," Pe":270," Pa":857," Ny":247," No":466," Ok":277," Ra":354,"b ":211," Ro":385,"gwe":166," Re":188," Ri":138,"gwa":280,"guz":429," Pr":150,"a ":143240," Su":178," St":248," Ta":3838," Th":274,"Yo":250," Te":257," To":205," Ru":370," Sa":668," Sh":517," Si":421," Se":528," So":239," Vi":666," Tu":231,"Za":152,"Zi":141," Uc":175," Uf":251," Uh":170," Ui":243," Uj":249," Uk":150," Ul":189," Un":355," Ur":181," Us":172," Ut":325," ja":134,"iak":142,"i ":52347,"ian":874," ji":4522,"ias":364,"ge":1928,"iar":235," je":226,"ga":2900," im":145," in":3363," ik":274," il":4878,"fi":1075,"fr":504,"fu":1927,"fo":752,"ibl":142,"ibi":603," ka":16147,"gw":483," ki":3027,"he":2541,"ibu":4111,"ha":8898,"gl":145,"gi":1836,"gh":1233,"gu":1858,"iba":566," ju":300,"go":2336,"du":838,"dw":136,"g ":607," ha":1606,"ea":1091,"eb":539," he":144,"ec":251,"ed":686,"de":1841,"di":4816,"dh":617,"do":1639,"ia ":9119,"dr":203,"ew":912,"ex":163,"eu":261,"ev":332,"ey":739,"ez":1828,"fa":6104,"h ":704," id":219,"fe":174,"eh":737," hi":990,"eg":644,"ef":303,"ee":307,"el":2120,"ek":2577,"ej":155," ho":139,"ei":650,"ep":643,"eo":1165,"en":9965,"em":2423,"et":1296," hu":4749,"es":2258,"er":4147," nj":147,"ca":364," ni":9330,"e ":10467," ng":147," nd":690,"bw":843," nc":2455," na":6269,"br":408,"bu":5373,"bo":2905," mw":6857,"bl":321," mu":4335," mt":648," ms":331,"bi":2134," mp":280," mo":680," mn":1501,"be":1280," mm":157,"ifu":393,"da":3239,"f ":246,"ifo":606," of":164,"co":390," ny":523,"ck":301,"ci":283,"ch":7388,"ce":365,"ifa":585," le":184,"c ":192," li":859," la":7153," ku":5668,"ich":830," kw":2736," km":140,"ica":140," ko":150," me":184," mf":368,"az":6015,"ay":5308," mi":1257,"ba":6016," mj":5191," mk":1617,"d ":1205,"at":22079,"as":4908,"ar":9773," ma":3449," mb":469,"aw":1490," mc":155,"av":414,"au":1667," lu":341,"ak":14678,"al":8458,"idi":551,"ai":5267,"aj":1998,"ao":6210,"ap":5739,"ide":157,"am":9111,"an":29556,"ac":1224,"ad":3126,"ida":813,"aa":1773,"ab":2568,"ag":1596,"ah":1414,"ae":682,"af":1092,"nu":591,"nt":1270,"ns":4895,"no":1318,"nn":478," am":1335," an":488,"nz":5093," ai":153,"iin":242,"ny":7307," aj":134," ak":183," al":2589,"of":4380," au":941,"oc":308,"od":678,"oa":4118,"ob":631," at":195," as":220,"om":1846,"on":3853,"ok":2328," ba":679,"ol":1930,"oi":1488,"oj":1425,"og":855,"oh":360,"ija":140,"ot":1280," bi":222,"os":1066,"ov":580,"ou":534,"ije":137,"op":845,"oo":318,"or":2938,"iji":1232,"r ":1622,"ow":244,"oz":397,"oy":154,"pe":836,"pa":6921,"po":1264,"ph":151,"pi":2193,"ika":13864,"lo":1408,"lm":337,"Ida":135,"ll":791,"ls":182,"iga":224,"ii ":525,"lu":868,"lt":178,"igh":170,"igi":384,"ly":147,"o ":24303,"mc":173,"igo":169,"ma":8274,"mb":6660,"mh":261,"me":2630,"mf":564,"mk":1733,"ml":210,"mi":3477,"mj":5199,"mn":1546,"mm":321,"mp":578,"ihe":138,"mo":6079,"mr":140,"mt":753,"ms":447,"mu":6394,"mw":6988,"ihi":187,"p ":352,"na":23279,"nc":2788,"nd":5575,"ne":2353,"ng":6858,"ni":24361,"nj":567,"nk":135,"imo":196," es":141," en":369,"ju":713,"imf":161,"ime":354," el":223,"jo":133,"imi":180,"ki":6922,"kh":154,"ind":834,"ke":2748,"ina":8001," fa":353,"ka":45110,"imu":392,"m ":727," fu":177,"kw":3124,"ino":181,"ks":210,"kt":463,"ku":10532,"ins":133,"ko":5804,"ine":479,"ing":1959,"kr":669," fi":274,"ini":4598,"km":156,"li":17984,"le":2997,"ld":221,"lf":159,"la":14880,"lb":250,"iny":275,"n ":3144,"iko":612,"hw":492,"ht":198,"hu":6825,"iki":2488,"hi":11111," ch":2090,"hn":150,"ho":1180,"ila":4379,"id":1813,"ic":1403,"ib":5595,"ia":11251,"ih":490,"in ":378,"ig":1252," da":146,"if":1790,"ie":672,"iku":2496,"k ":628,"ilo":373,"ir":1982,"is":9376,"it":2904,"ill":288,"iu":466,"iv":385,"iw":2556,"ii":989,"ij":1580,"ik":19966," de":224,"ili":8251,"il":13887,"im":4832,"in":17333,"io":4395,"ile":321,"ip":1169,"ima":914,"je":934,"imb":2471,"io ":2960,"ji":17145,"iz":1362,"iy":4997," du":302,"l ":1018,"ja":2368,"z ":191,"wi":1773,"wo":202,"vy":671," za":1702,"y ":1239,"wa":56175," zi":456,"we":2203,"vi":1632,"vu":418,"vo":138,"uz":1451,"uw":2877,"uv":252,"uu":3068," ye":258,"ve":578," ya":17428,"va":328,"x ":213,"ui":563,"uj":4429,"uk":1643,"ul":2575,"ue":357,"uf":741,"ug":901,"uh":626,"ur":1919,"us":3274,"ut":2784,"um":5397,"un":5099,"uo":368,"up":1077,"ty":166,"tu":2287,"tt":391,"tw":473,"ub":1112,"ua":2111,"ud":534,"uc":476,"w ":435,"to":4407,"huk":345,"hul":146,"tl":220,"ts":343,"tr":455,"te":2280,"ti":12092,"th":999,"ta":14867,"su":644,"ss":500,"st":1842,"sw":308,"sl":142,"sk":865,"sm":139,"sp":289,"so":683,"sc":179,"se":5649,"sh":8151,"si":4764,"u ":13704,"sa":7736,"rr":220,"rs":467,"rt":620,"ru":2279,"ry":287,"ro":1786,"rn":619,"rm":257,"rl":223,"rk":320,"ri":8157,"rg":403,"re":3855,"rd":556,"rc":143,"rb":136,"ra":5018,"t ":1231,"s ":3025,"pt":348,"pu":357,"pw":193,"pr":381," sa":589," se":4480," si":369," sh":318," ra":432," ri":188,"hwa":473,"huo":175,"hum":2789,"hun":282,"hus":506,"hur":418,"huu":1333," pe":176," pa":632," pi":931," wa":33135," we":275," vy":396," wi":862," vi":1013," uc":144,"zi":8597,"ze":368,"za":8043," tu":189,"zw":257," us":165," ut":249," up":502," um":247,"zu":272," un":1571," uk":210,"zo":952," ul":573," uh":139," ta":1410,"ye":2395,"ya":24129,"yu":306," to":170," th":289,"yo":5888," te":201,"yi":4283,"Apr":266,"Asi":146,"Aru":195,"far":316,"fam":283,"fan":4203,"fal":292,"fa ":488,"eya":259,"Bah":237,"Bar":140,"eza":1136,"ezo":172,"ezi":237,"eta":229,"ete":154,"eti":253,"est":247,"ett":212,"ew ":355,"evi":165,"ewe":148,"ey ":361,"ewa":358,"er ":615,"epa":149,"es ":640,"ept":299,"eri":650,"ere":660,"era":456,"Afr":406,"esh":359,"ese":306,"esa":279,"eru":498,"Ago":254,"ert":152,"ers":339,"eku":184,"en ":297,"ela":204,"ele":786,"eli":360,"ell":177,"eo ":852,"emb":1055,"ema":157,"eme":314,"emi":276,"emu":365,"ene":704,"eng":671,"ena":283,"end":498,"eno":221,"eni":486,"ens":4087,"ent":441,"eny":1803,"Ali":478,"ege":351,"Ame":158,"ehe":647,"Ana":176,"el ":260,"eke":267,"eka":1754,"giz":193,"gir":232,"gin":349,"gid":165,"ght":136,"gha":925,"gi ":572,"gen":204,"ger":781,"ge ":611,"gaz":140,"gar":155,"gan":693,"ga ":1334,"Cal":307,"fup":194,"Bib":137,"fua":317,"fum":143,"fun":167,"fri":445,"fu ":810,"for":356,"fo ":342,"fil":269,"fik":168,"fiz":146,"da ":1525,"de ":752,"dad":386,"dae":220,"dar":151,"dan":305,"dam":173,"Des":272,"Dar":167,"Chi":216,"Chu":136,"Cha":300,"ch ":165,"cha":2430,"chu":596,"ck ":143,"che":571,"chi":3152,"cho":370,"ed ":154,"ebr":313,"ea ":663,"ei ":346,"efu":197,"edi":297,"ee ":156,"don":150,"dom":308,"dol":151,"dog":335,"dun":335,"dha":302,"dia":330,"dhi":240,"der":146,"deg":261,"del":152,"di ":2661,"do ":429,"Dod":240,"diy":201,"din":291,"dis":387,"dik":302,"ri ":2373,"rez":420,"rea":148,"ref":154,"reh":266,"ren":163,"rek":1672,"re ":305,"rd ":213,"ras":256,"rat":173,"Ni ":218,"New":381,"rai":160,"ran":867,"ram":226,"rab":297,"rad":150,"ron":135,"rog":253,"rne":169,"rni":283,"ro ":593,"riw":166,"ris":508,"ril":300,"rik":1688,"rin":373,"ria":769,"rib":1011,"ric":160,"rk ":191,"ruf":262,"rum":452,"ruk":315,"rus":423,"ry ":194,"rse":228,"Nya":144,"rua":234,"rt ":160,"ru ":273,"sab":458,"sac":139,"san":482,"sas":180,"sa ":5643,"Nov":242,"sha":1745,"sho":271,"she":240,"shi":5099,"si ":1365,"siw":355,"sia":608,"shw":458,"shu":187,"sis":157,"sin":881,"sil":283,"sim":158,"sik":319,"sey":212,"ser":175,"set":147,"Okt":259,"seh":319,"sen":4083,"sem":335,"spa":151,"son":242,"su ":198,"st ":167,"sko":136,"ska":599,"so ":134,"ssa":198,"ste":192,"sta":295,"sto":444,"sti":401,"str":197,"swa":181,"tai":280,"taj":233,"tak":462,"tal":339,"taa":220,"tab":242,"taw":344,"tat":292,"tar":668,"tao":3872,"tan":641,"tam":288,"te ":507,"ta ":6480,"pa ":765,"pat":4120,"pak":235,"pap":248,"pam":300,"pan":895,"pi ":233,"ped":156,"Pap":368,"pia":789,"pil":189,"pin":267,"pis":162,"pit":144,"po ":743,"pte":287,"pri":298,"pwa":189,"Rai":176,"ra ":1932,"ngo":958,"ngi":1065,"ngu":1084,"ngw":363,"ni ":18823,"Iri":209,"nge":937,"nga":1742,"Ita":147,"neo":505,"nes":161,"ng ":405,"nch":2504,"ne ":911,"ndu":263,"ndo":574,"ndi":1835,"nde":1085,"nda":1162,"nak":251,"nal":257,"nam":1855,"nan":221,"nao":1457,"nap":185,"nac":183,"nad":288,"naf":402,"nai":158,"naj":196,"nd ":409,"nat":353,"nas":439,"nay":454,"na ":15738,"Jan":271,"Jam":281,"nya":1379,"Jer":215,"nye":1338,"nyi":4239,"nus":133,"nua":282,"Jim":174,"Jin":277,"nti":403,"nta":151,"nte":177,"nsi":211,"nsa":4269,"nt ":232,"ns ":140,"nne":236,"no ":948,"nji":138,"nja":269,"Joh":134,"nia":4199,"nis":530,"ogo":593,"ois":1291,"oji":173,"oja":1149,"Jul":285,"Jun":259,"odo":288,"of ":150,"ofu":134,"ofa":3991,"oa ":3810,"oan":188,"oba":375,"nza":3817,"nzi":1111,"Kai":144,"Kag":175,"Kal":167,"Kan":354,"Kat":474,"Kas":372,"Kar":232,"Ken":632,"ozi":165,"Kis":329,"Kir":165,"Kit":204,"Kin":148,"Kib":138,"Kia":309,"ote":378,"Kik":287,"Kil":453,"Kim":202,"oto":331,"Kig":295,"Kii":249,"ost":309,"ota":195,"ove":320,"opo":325,"os ":178,"or ":161,"Kon":197,"orn":300,"oro":673,"ore":188,"ori":369,"ort":147,"ora":378,"ola":427,"on ":838,"oli":431,"ole":357,"olo":331,"oka":1580,"oke":163,"oko":236,"oku":141,"ona":230,"ond":383,"one":151,"ong":860,"oni":784,"oma":766,"omb":303,"omi":249,"omo":182,"op ":143,"la ":8089,"le ":1011,"Kwa":3975,"laa":157,"lai":293,"lak":564,"lan":660,"lam":497,"lat":186,"lay":3727,"Kus":393,"lba":165,"kuz":236,"kuw":2713,"kuu":1305,"kut":1795,"kus":492,"kur":190,"kup":186,"kun":409,"kum":210,"kul":297,"kuj":187,"kwe":591,"kwa":2512,"kub":762,"kuf":233,"kuh":134,"kua":620,"kto":308,"lom":136,"loj":136,"lme":241,"Lin":225,"lug":350,"lu ":155,"li ":2787,"lez":192,"lew":193,"lev":140,"les":155,"leo":178,"lem":198,"len":254,"lek":133,"lo ":347,"lla":138,"lle":153,"lli":198,"ll ":147,"lit":241,"lis":337,"lip":257,"lio":738,"lin":627,"lim":922,"liz":411,"liy":4415,"liw":979,"lic":340,"lia":1497,"lik":2742,"lil":529,"lih":179,"lif":397,"ma ":2611,"mb ":139,"maa":449,"maj":397,"mak":522,"mad":206,"mae":140,"mag":342,"mar":439,"mas":613,"mal":159,"mam":161,"man":1055,"mat":406,"mba":3047,"mbi":361,"mbe":389,"mbo":2343,"me ":516,"mbu":267,"mch":170,"met":211,"mer":252,"men":492,"mfa":152,"mez":387,"mfu":373,"Mei":250,"Man":216,"Mar":1940,"Mas":472,"Mag":282,"Mak":206,"Mac":287,"Mbe":273,"mpi":142,"mon":163,"moj":1127,"mpa":160,"Mor":279,"mu ":1602,"mtu":175,"mto":226,"Mic":182,"Mis":147,"msh":144,"mta":228,"mwe":383,"mwi":345,"Mko":3178,"mwa":6205,"Mku":138,"Mji":464,"muj":3839,"muz":374,"mhu":232,"Mtw":147,"mi ":359,"mji":5175,"min":192,"mil":749,"Mwa":460,"mit":295,"mia":630,"mik":321,"mo ":4413,"mku":1038,"mko":539,"mna":1501,"mmo":145,"Wik":149,"Wil":3077,"Wan":148,"zwa":252,"zi ":5785,"zai":249,"zaj":254,"zam":177,"zan":3194,"zal":783,"zar":173,"zo ":612,"zia":533,"zin":815,"zil":197,"zik":548,"zis":240,"一":303,"yof":3874,"yot":286,"za ":2981,"ye ":1320,"yen":237,"ya ":21762,"yar":252,"yan":567,"yao":167,"yam":250,"yak":657,"yo ":973,"yin":213,"yik":3954,"一一":144,"Tan":3407,"Tab":164,"Shi":315,"Sin":201,"Sep":283,"we ":401,"wez":265,"wen":1037,"wim":286,"wil":741,"Sal":197,"vyo":257,"wa ":33121,"wap":4111,"wan":3901,"wal":617,"wam":169,"wak":9923,"way":141,"wat":368,"war":238,"was":172,"wai":2667,"wah":176,"vu ":165,"vya":351,"vil":200,"vin":183,"vit":187,"vis":284,"Rom":180,"vem":244,"Vij":328,"uzi":743,"uza":470,"Uje":235,"uwa":2760,"uvu":174,"ush":417,"usi":1319,"use":183,"usa":176,"uu ":2892,"usu":216,"ust":207,"uso":141,"uti":211,"ute":137,"uta":560,"Uin":218,"utu":215,"uto":1436,"us ":536,"Ung":252,"ura":183,"ure":140,"uri":491,"uru":630,"unz":137,"Ula":150,"upa":554,"upi":311,"umu":162,"umi":484,"umo":2705,"uma":686,"umb":661,"ume":297,"uo ":238,"uni":940,"und":747,"una":1741,"ung":1193,"uku":302,"uko":457,"uki":429,"uka":247,"ulu":258,"uli":1405,"ule":192,"ula":478,"ukw":139,"uhu":267,"uji":4010,"uja":302,"Utu":261,"ugh":514,"ufu":352,"uhi":136,"ugu":137,"udi":174,"ubw":695,"uch":343,"ufa":176,"ufi":189,"ua ":369,"uat":317,"uar":494,"uan":690,"uba":185,"Uch":175,"ty ":146,"twa":450,"tur":369,"tun":270,"tum":424,"Ufa":219,"ts ":214,"tu ":896,"The":164,"tts":142,"to ":986,"tob":268,"tom":167,"ton":281,"tok":1553,"tol":482,"tor":246,"tik":8147,"tis":158,"tin":351,"tio":199,"thu":171,"tia":156,"tem":384,"ten":273,"tel":171,"th ":160,"ter":432,"ti ":2389,"the":225,"thi":213,"biw":209,"bis":191,"bil":315,"bin":256,"bo ":2326,"bli":173,"bor":262,"be ":229,"bam":230,"ban":516,"bal":619,"bah":147,"baa":227,"bab":179,"bay":333,"bar":432,"bao":277,"bi ":662,"ber":216,"bel":151,"bey":251,"bia":222,"ce ":176,"bu ":4649,"bru":221,"bur":149,"bun":177,"bwa":786,"aka":10583,"am ":337,"ake":1982,"aki":644,"aji":1355,"aju":170,"al ":304,"aja":293,"ain":393,"air":222,"ais":2933,"aif":267,"aid":437,"ahi":308,"aha":751,"agh":475,"agu":395,"aoi":1233,"anu":344,"anz":4756,"any":4453,"ano":638,"ann":141,"ant":323,"ans":490,"ane":261,"ang":1660,"ani":7747,"anj":260,"ana":4702,"anc":133,"and":2300,"amu":1047,"amo":1890,"amp":179,"amh":222,"ami":838,"ame":637,"amb":1658,"ama":1868,"ao ":4649,"alo":269,"alm":262,"all":133,"ali":5324,"ale":476,"ala":1026,"alb":152,"an ":1167,"akr":376,"aku":502,"ako":215,"aba":751,"abe":140,"abi":660,"abo":208,"abu":582,"ae ":291,"aad":302,"aan":389,"aal":140,"aam":185,"aar":236,"aa ":361,"afi":303,"ai ":477,"aga":223,"age":227,"afu":225,"aen":162,"ael":172,"afa":411,"ado":269,"adh":288,"adi":1538,"ach":840,"ada":637,"azo":205,"azi":5401,"aza":186,"ayo":638,"aya":4140,"aye":284,"ba ":2178,"are":1998,"ard":317,"ara":2057,"aro":249,"ari":3153,"aru":316,"art":243,"au ":993,"asa":1084,"asi":1169,"ash":895,"ask":665,"ar ":568,"apa":4869,"api":162,"apo":406,"as ":271,"aut":148,"awa":1126,"awi":190,"ata":10070,"ast":167,"ass":197,"ato":634,"ate":225,"ati":9962,"ath":135,"atu":749},"n_words":[1316698,1560317,1165243],"name":"sw","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/tr.json b/contrib/languages-data/tr.json
new file mode 100644
index 0000000..5b8f9e4
--- /dev/null
+++ b/contrib/languages-data/tr.json
@@ -0,0 +1 @@
+{"freq":{"D":20468,"E":16965,"F":15212,"G":16978,"A":46399,"B":33447,"C":16597,"L":11614,"M":29609,"N":10463,"O":12233,"H":17902,"I":6437,"J":5964,"K":30894,"U":5006,"T":27864,"W":5022,"V":7330,"P":19836,"S":32251,"R":13212,"Y":12032,"Z":2430,"f":59766,"g":101779,"d":361700,"e":798707,"b":176889,"c":86462,"a":1029574,"n":718304,"o":261031,"l":623439,"m":275808,"j":10372,"k":334154,"h":85819,"i":785261,"w":7611,"v":106213,"u":275921,"t":334795,"s":315971,"r":646028,"q":1181,"p":80800,"z":105754,"y":255623,"x":3410,"²":1600,"Ç":4715,"Ü":2583,"Ö":3499,"î":1753,"é":1705,"ç":69086,"â":3605,"ü":157542,"ö":58180,"ğ":69432,"ı":399538,"İ":20804,"ş":111613,"Ş":5518," l":7220," m":26771," n":32386," o":55007," h":21337," i":59190," k":78855," d":82641," e":36355," f":17358," g":42591,"р":1499," a":67898,"с":1190," b":117978,"т":957," c":5691," y":76657," z":4398," u":11497," t":53835," v":55379," p":15531," s":57822," r":8817," J":5925," K":30750," H":17802," I":6388," N":10386," O":12150," L":11515," M":29458," B":33275," C":16419," A":46229," F":15133," G":16827," D":20328," E":16897,"л":1101," Z":2402,"к":1097," Y":11995,"и":2029,"о":2304,"н":1554," S":31973,"в":1191," R":13120," P":19696,"а":2550," W":4958," V":7287,"е":1678," U":4989," T":27682," ç":18143," ö":14326," ü":15280," Ç":4706," Ö":3487," Ü":2581," ı":6625," İ":20776," ş":10933," Ş":5509,"ي":1130,"ل":1185,"ن":965,"ا":2068,"A ":2665,"Da":3349,"Cu":2398,"Co":2920,"Ce":1839,"Ch":2248,"Do":2641,"De":6139,"Di":2765,"Fe":2199,"Fa":2297,"Ey":1528,"Er":2288,"Es":1409,"En":2354,"El":1347,"Ek":1512,"Ağ":1411,"Ge":3246,"Ga":2711,"Bü":1152,"I ":2251,"Bö":1239,"Fr":4109,"Fo":1223,"Fi":2407,"B ":1016,"C ":1464,"Av":3034,"Ar":5311,"At":1678,"As":1760,"D ":3341,"Ba":8245,"Ay":1472,"Af":1012,"Ab":1087,"Ad":2089,"Am":3467,"An":5110,"Ak":1709,"Al":8286,"Bu":5544,"Br":2172,"Ca":3556,"Bi":5816,"Be":4617,"Bo":2818,"Ku":5039,"Gö":1216,"Kr":1520,"Ko":3871,"Le":1961,"Li":3051,"Gü":3195,"La":2872,"Lo":1867,"Me":5154,"Mi":3881,"Ma":11016,"Mu":2388,"Mo":2807,"Ni":2456,"Ne":2578,"Na":1799,"P ":982,"No":1828,"Ok":1001,"Ol":1014,"Oc":1091,"Gi":1074,"Gr":2260,"Go":1145,"Ha":7669,"He":2472,"Hi":1980,"Ho":2868,"Dü":2155,"In":1300,"Ja":2318,"Jo":1650,"Ka":10073,"M ":1217,"Ki":1749,"Ke":1920,"Ul":1131,"Tu":1634,"Tr":1523,"To":2482,"Th":2500,"Ti":1452,"Te":4160,"Ta":4376,"V ":1033,"St":3068,"Su":2048,"Wi":1541,"Wa":1156,"Vi":2393,"Va":1979,"Ve":1434,"Pr":2222,"S ":1244,"Pe":2037,"Pa":5059,"Kü":1314,"Po":6260,"Pi":1406,"Os":2243,"Or":2331,"Kö":1204,"Se":4094,"Sc":1963,"Si":3179,"Sh":999,"Sp":940,"So":3019,"Ru":3002,"Sa":7772,"Re":2740,"Ri":1066,"Ro":3278,"Kı":2617,"Ra":1699,"Mü":1802,"b ":1498,"a ":176902,"Ye":1685,"Tü":7528,"Ya":4483,"Yo":1238,"Yu":2386,"bö":7750,"i ":134793,"ge":29308,"ağ":18792,"bü":7597,"ga":7799,"fl":1604,"ff":1053,"fi":9181,"fr":3323,"fu":6151,"ft":2020,"fo":3898,"he":11849,"ha":24248,"cü":3165,"gl":1385,"gi":16689,"gh":1246,"gu":5867,"gr":6009,"go":2702,"du":14570,"dy":2265,"g ":6921,"ea":4032,"eb":5387,"ec":6696,"ed":29273,"de":95875,"dd":2946,"di":58969,"dl":2907,"do":12268,"dr":3254,"ew":1275,"eu":1087,"ev":14605,"ey":30087,"ez":8326,"fa":8497,"h ":5851,"fe":6028,"eh":5944,"eg":3973,"ef":3639,"ee":1898,"el":56887,"ek":41993,"aç":5873,"ei":4450,"ep":3432,"eo":3151,"en":100462,"em":27716,"et":47279,"es":52024,"er":131153,"ca":18661,"e ":189222,"br":3182,"bu":23594,"bo":7205,"bl":1863,"bi":65897,"be":19027,"da":97587,"f ":5395,"cu":6717,"ct":1404,"cr":1803,"co":3001,"ck":3008,"ci":12969,"ch":6435,"ce":16707,"c ":1516,"az":19018,"ay":42685,"ba":34644,"d ":11781,"at":38962,"as":49850,"ar":157561,"av":10941,"au":2292,"ak":57066,"al":73836,"ai":6322,"aj":1917,"ap":20439,"am":37584,"an":184545,"ac":10260,"ad":37380,"aa":2080,"ab":13369,"ag":3250,"ah":16357,"ae":2002,"af":13702,"nu":18358,"nt":17557,"ns":13849,"nr":3598,"no":7521,"nn":3019,"nz":1833,"ny":12611,"hı":1036,"iğ":11558,"of":3409,"oc":3225,"od":4927,"ob":3075,"om":13542,"on":43168,"ok":10977,"kç":3190,"ol":59486,"oi":1195,"oj":3578,"og":3689,"oh":1112,"m²":1589,"ot":7864,"os":10639,"ov":4695,"ou":3530,"op":8235,"oo":1893,"or":32678,"r ":170339,"ow":1477,"oz":2422,"kö":4303,"oy":8048,"pe":6240,"kü":6018,"pa":15264,"pl":7151,"pm":1729,"po":6918,"ph":1672,"pi":5799,"lç":5891,"lo":10260,"lm":31640,"ll":28962,"ls":3083,"fı":9951,"lu":32826,"lt":9362,"ly":5619,"o ":14289,"mc":2028,"md":5246,"hü":1497,"ma":67090,"mb":3028,"eş":11455,"mh":2240,"me":45214,"iç":12229,"ml":15462,"mi":33126,"mm":2739,"mp":4822,"mo":6599,"ms":3576,"mu":11366,"gı":2023,"my":1356,"p ":11912,"na":47407,"nb":3039,"nc":18538,"nd":102802,"ne":51401,"nf":1257,"ng":12672,"ni":53546,"nk":5100,"nl":28972,"nm":10641,"dı":37931,"ki":41309,"eğ":6350,"ke":25138,"ka":45970,"m ":32916,"ky":1403,"gö":9935,"ks":8093,"kt":21304,"ku":22990,"ko":13147,"kr":3294,"kk":1610,"kl":25669,"km":5012,"kn":1431,"li":71923,"lk":10655,"le":122046,"ld":15695,"lg":12154,"lf":964,"la":144587,"gü":8845,"lc":1649,"lb":3731,"n ":243619,"hr":4105,"bı":1392,"ht":2266,"hu":4538,"hi":16646,"hn":1197,"ho":2852,"hl":3714,"hm":1584,"id":21397,"ic":9431,"ib":5859,"dü":13708,"ia":8374,"ih":7127,"ig":4782,"aş":24105,"if":4811,"ie":4567,"dö":3928,"k ":83031,"ir":107452,"is":40125,"it":22698,"iu":1112,"iv":5224,"cı":8286,"ii":1103,"ij":1085,"ik":40360,"eç":4865,"il":86504,"im":27783,"in":138518,"io":5300,"ip":8100,"je":1713,"ji":4659,"iz":21969,"iy":28989,"l ":42232,"ja":1035,"tç":1871,"ğlu":963,"rı":43249,"sö":2312,"z ":18621,"sü":4852,"oş":947,"wi":1830,"sç":1475,"pı":8243,"vv":1012,"vy":1154,"y ":15418,"wa":1412,"rü":8173,"ğla":2983,"vl":4067,"rç":3762,"vi":9343,"nş":1036,"vu":2607,"vr":5677,"vo":1424,"uz":10570,"uy":4807,"uv":2249,"ve":59460,"va":15490,"x ":1999,"ui":1273,"uk":6582,"ul":36937,"ue":3947,"oğ":10206,"ug":2004,"ğit":1160,"uh":1861,"ur":35478,"ğin":3622,"us":21592,"ut":7622,"um":12243,"un":39859,"up":6378,"ty":1262,"tu":11035,"tt":7164,"nı":44489,"ub":5081,"ua":1900,"ud":5831,"uc":4903,"w ":1459,"to":17485,"tm":6406,"tl":13327,"ts":2378,"tr":10162,"te":52587,"pç":1206,"tk":2785,"ti":53158,"th":4577,"v ":2143,"tb":2499,"ta":68248,"su":11906,"sv":930,"ss":3728,"st":34655,"sy":6610,"mı":21024,"sw":1258,"sl":8558,"sk":7111,"sn":1024,"sm":5546,"sp":5003,"so":11037,"sc":1810,"se":25089,"sh":2001,"si":63532,"rz":1185,"u ":42818,"sa":39618,"nü":10410,"rr":2777,"rs":8400,"rt":19992,"ru":21036,"rv":1617,"lı":53226,"ry":3722,"rp":1215,"ro":18824,"rn":5785,"rm":14124,"rl":25348,"rk":23839,"nç":1698,"rj":994,"ri":89903,"rh":1272,"rg":7962,"rf":1575,"re":45975,"rd":23811,"rc":3479,"rb":3537,"mü":12609,"ği ":7346,"ra":89176,"t ":36013,"kı":16796,"ğer":2592,"iş":22142,"lü":12025,"s ":31408,"pt":3720,"pu":1218,"pr":5018,"ps":1980,"ğun":2143,"ğus":1809,"ğum":954,"zı":6859,"zü":1410,"yı":27954,"yü":11063,"ğu ":5997,"zö":2165,"ğlı":7152,"ğre":1586,"uş":11784,"yâ":1423,"yö":4435,"ğiş":1548,"uğ":6358,"zg":2173,"rş":2653,"zi":13389,"zc":5363,"zd":2307,"ze":21253,"za":12673,"tı":26503,"zy":1958,"zu":2578,"zo":2918,"zm":3788,"zl":5972,"yg":2621,"ye":34397,"yd":5178,"tü":11302,"ya":90167,"yb":1899,"tö":1628,"sı":43368,"yv":967,"yu":8007,"ys":1270,"yr":3168,"yo":15057,"yn":5330,"ym":1117,"yl":13689,"yk":1018,"uç":1551,"yi":7908,"² ":1587,"ğı ":9316,"ğın":3141,"Çi":1350,"Ça":1454,"âl":1634,"Ün":1471,"Öz":1299,"çı":7148,"çü":4634,"î ":1515,"ço":5321,"çm":1158,"çl":3470,"çi":13394,"çe":14606,"ça":9969,"ç ":5469,"üş":3641,"üğ":1561,"üç":3404,"ün":27564,"üm":12601,"ül":12799,"üs":5069,"ür":27019,"üp":974,"üt":2907,"üz":15474,"üy":5418,"üc":2767,"üf":4960,"üd":3476,"öğ":1623,"ük":10837,"ü ":14372,"öy":3363,"öz":7289,"ör":11792,"ös":1882,"ön":12083,"öl":13407,"ök":2015,"ğ ":1364,"ğu":12250,"ğr":3621,"ğe":3325,"ğd":979,"ğa":3324,"ğl":11320,"ği":14714,"ğı":15723,"ğü":1540,"ığ":10531,"ış":19918,"ı ":102139,"İn":4248,"İm":1089,"İl":1863,"ın":99943,"ım":16215,"ıp":1333,"ıl":42504,"ık":17416,"ıf":1477,"ıb":1069,"ıd":8898,"ıc":3354,"İt":1484,"İr":962,"İs":7391,"ıy":7698,"ız":6701,"ıs":11942,"ıt":2777,"ır":43940,"şt":19651,"şu":2786,"şi":12692,"şl":8667,"şk":4781,"şm":7053,"şa":13302,"şe":7623,"Şu":1166,"Şa":2212,"Şe":1159,"ş ":18890,"şı":10920,"şü":1419," Ga":2694," Bü":1149," Ağ":1410," Ge":3227," Bö":1239," Fo":1207," Fr":4097," Fi":2393," Ha":7643," He":2466," Go":1137," Gr":2225," Gi":1067," Dü":2153," Ho":2861," Hi":1971," Ja":2310," In":1293," Ka":10047," Ke":1904," Ki":1728," Jo":1635," Gü":3188," La":2844," Le":1944," Li":3034," Ko":3863," Kr":1518," Ku":5027," Gö":1215," Ma":10963," Mi":3862," Me":5127," Lo":1852," Ne":2557," Na":1779," Ni":2442," Mo":2790," Mu":2369," Am":3459," An":5094," Ak":1699," Al":8247," Af":1009," Ad":2080," Ab":1075," Ba":8219," Ay":1468," Av":3029," At":1662," As":1754," Ar":5292," Be":4599," Bi":5804," Bo":2788," Br":2154," Bu":5532," Ca":3515," Ce":1828," Ch":2228," Co":2886," Cu":2385," Da":3328," Di":2745," De":6126," Do":2600," El":1340," Ek":1512," Es":1407," Er":2282," En":2338," Ey":1526," Fe":2195," Fa":2283," Wi":1521," Wa":1142," Yu":2383," Yo":1225," Tü":7497," Ya":4478," Ye":1681," a ":1377," Kö":1204," Os":2235," Or":2329," Po":6216," Pi":1404," Pe":2029," Pa":5030," Kü":1311," No":1821," Ol":1012," Ok":999," Oc":1088," Ra":1684," Mü":1801," Kı":2610," Ro":3268," Re":2723," Ri":1059," Pr":2202," Su":2042," St":2970," Ta":4359," Th":2488," Ti":1443," Te":4145," Tr":1513," To":2461," Ru":2999," Sa":7750," Sh":981," Si":3171," Sc":1940," Se":4078," So":2983," Va":1973," Ve":1421," Vi":2383," Tu":1612," Ul":1129," im":933," in":10201," ik":4076," il":20054," is":5383," ka":23639," ki":5592," ke":6355," eğ":1244," dı":1534," ha":11824," he":4045," gi":4830," gr":3277," dö":3823," dü":5597," id":1007," aş":930," hi":2120," ni":6427," nd":4080," ne":3117," na":1507," mu":1078," mo":2617," ok":1391," ol":35321," on":2426," of":1496," nu":1915," no":1087," le":1165," li":3624," gü":6331," la":1347," ku":18460," gö":9671," km":3282," ko":8297," me":8098," mi":3110," iç":9390," hü":1380," ma":6917," ad":10374," am":2886," an":8617," ai":1874," ak":2362," al":14807," ar":9911," at":1881," as":2441," d ":1776," ba":23010," ay":4744," bi":52706," be":11424," bo":2633," bu":15855," ca":1538," e ":1478," er":1285," et":4327," es":3101," en":6931," el":3328," ek":1681," aç":1971," fe":1277," fa":4483," ey":3388," ev":1229," fu":1090," fr":1901," fo":1846," fi":5962," ağ":1494," ge":16462," bü":4020," ga":1355," i ":957," bö":7468," ci":1625," da":23947," do":7108," de":26605," di":9385," ed":6003," du":2416," za":2588," yo":2330," sı":5340," ye":11522," tü":4969," ya":37365," sö":2004," sü":3398," yı":13517," yü":5685," yö":4365," nü":5050," sa":17392," se":8759," si":6463," sp":996," so":7754," kı":5201," ra":1785," mü":4235," re":3088," ro":2639," pr":3082," iş":3101," ot":2226," or":5403," oy":4366," kö":3581," pe":1612," kü":2525," pa":4625," pl":965," po":2333," pi":1501," va":2885," ve":49979," uz":2734," uy":2267," vi":955," nı":8640," tu":1641," un":2824," ul":1616," ta":27334," st":1812," su":2833," to":3720," th":1268," ti":1727," te":11491," İt":1484," İs":7380," İr":961," ın":4768," İn":4243," İl":1857," İm":1089," Çi":1348," Ça":1452," Öz":1296," Ün":1468," çe":4156," ça":3855," ço":4379," çi":1440," çı":3500," öl":1632," ön":4253," ör":1000," öz":4179," ür":2793," üs":1211," ün":2173," ül":2545," üz":3783," öğ":1347," üç":1563," şi":1684," şe":4952," şa":3567," Şa":2210," Şe":1156," Şu":1166,"İst":2249,"İta":1353,"İsp":2587,"İng":3329,"İmp":936,"ıca":1274,"ıda":1227,"ılı":15802,"ısa":1923,"ıra":2874,"ırl":2487,"ırm":2227,"ını":15301,"ımı":5151,"ıya":1118,"ırı":3492,"ız ":2539,"ıl ":1920,"ıdı":7570,"ık ":8525,"ıcı":2074,"ıla":12183,"ın ":32401,"ıld":3344,"ılm":5978,"ıll":2939,"ım ":3853,"ıka":2439,"ıkl":3108,"ıkt":1193,"ınl":2820,"ıp ":1063,"ınd":38554,"ına":8579,"ınm":927,"ıma":1331,"ımc":938,"ıml":3642,"ıs ":2159,"ır ":29899,"ıyı":1283,"ızı":1437,"ısı":6837,"ıyl":4411,"ızl":1239,"Çin":1025,"ığı":10132,"ış ":6594,"ışt":5588,"ışa":989,"ışm":2843,"ışı":2617,"Fil":1047,"Eyl":1137,"Eki":1120,"End":927,"Ağu":1069,"Gen":1382,"Gal":926,"Böl":1142,"Fra":3208,"II ":1197,"Haz":1331,"Her":938,"Hal":1040,"Har":940,"Dün":1753,"Hol":1623,"Ara":2840,"Avr":1832,"Bar":1214,"Bat":956,"BD ":2271,"Ada":1186,"Alm":3781,"Ame":2710,"Ana":1277,"Ant":1215,"Bu ":2451,"şıl":1169,"şık":1556,"şın":2204,"şı ":3074,"Bel":1045,"Bil":1105,"Baş":989,"Bir":3328,"Cum":2030,"Dev":1876,"Den":1321,"Cha":1022,"Doğ":1141,"Nis":1117,"Oca":1015,"Ort":943,"Osm":1787,"Par":1928,"Pro":985,"Por":4336,"Kan":1554,"Kas":1322,"Kar":2603,"Kon":1226,"Kra":1061,"Kur":1360,"Kuz":1311,"Gün":1851,"Mer":1131,"Man":1052,"Mar":3200,"May":1678,"Mil":1269,"çok":3674,"çeş":1324,"çla":1690,"çim":1273,"çin":7152,"Yun":1817,"Tür":7269,"Yar":1131,"Sta":1148,"Tem":1343,"Rus":2328,"Sch":1519,"Sav":1634,"San":1732,"Rom":1453,"Ulu":930,"The":1923,"ça ":3264,"çal":2740,"çe ":2241,"çes":2357,"çer":2328,"çev":1541,"çek":2799,"çi ":1341,"biy":1472,"bit":1248,"bir":44645,"bil":11436,"bin":1380,"baş":7621,"şa ":1752,"bol":2761,"şam":1786,"şan":2545,"şar":3761,"boy":1344,"ban":1510,"bak":1768,"bal":1557,"baz":1091,"bay":1165,"azı":4805,"bat":3330,"bas":2581,"bar":2153,"Şub":979,"bi ":3364,"ber":3678,"ben":1194,"bel":7483,"bes":1722,"bağ":8489,"şla":4267,"şle":3059,"şma":4426,"şme":1364,"ca ":7674,"car":1245,"can":1925,"cak":4366,"ce ":9532,"bra":943,"bu ":5377,"şek":1911,"şeh":2781,"şi ":1181,"bur":1706,"bul":11239,"bun":1530,"buc":1673,"şid":1048,"şit":1478,"şir":1286,"şin":976,"şim":2323,"şil":1127,"şik":1949,"şki":1113,"şke":1220,"şka":1851,"aka":3723,"am ":5523,"aki":6742,"adı":13553,"al ":11243,"ail":1471,"air":932,"ait":1247,"acı":3398,"ak ":21809,"abı":1039,"ahi":5926,"aha":4061,"anu":1353,"any":6219,"ano":1022,"ann":1052,"anm":4730,"ant":4742,"ans":6131,"ane":2508,"ang":2190,"ani":4880,"ank":1605,"anl":12840,"ap ":1282,"ana":13099,"anb":2176,"anc":4224,"and":10443,"aml":2039,"amp":1789,"ami":2705,"ame":1745,"ama":13353,"aly":2496,"afı":8781,"alt":4529,"alm":2361,"all":3746,"alk":2064,"ali":6616,"ald":2079,"ale":5493,"ala":14214,"alb":2602,"an ":84156,"aks":970,"akt":8868,"akk":924,"akl":5036,"aba":3739,"abe":1525,"abi":3498,"abu":1168,"ae ":1012,"aca":2693,"ad ":1275,"ştu":4101,"şti":6965,"afi":1276,"ştı":7023,"ah ":1276,"ado":1356,"adl":2298,"adi":1602,"add":1987,"ade":4140,"ady":944,"ada":8970,"azi":2552,"azl":1262,"atı":8461,"aze":1261,"arş":2391,"aza":4710,"az ":2571,"ayn":2937,"ayl":1883,"ayr":2190,"ası":27693,"arı":35962,"aya":12842,"ayd":1055,"aye":1092,"âle":1449,"ba ":1088,"ayı":12386,"akı":5066,"at ":6328,"are":4111,"ard":9709,"arc":1066,"ara":45240,"arm":1435,"arl":5899,"ark":6902,"ari":9836,"alı":12304,"ars":1642,"art":5112,"asa":4734,"ary":1104,"asi":2326,"ask":1950,"ar ":19629,"apa":2097,"apm":1588,"apl":1156,"apo":1295,"apt":1792,"as ":3211,"ava":5024,"avr":943,"arç":1233,"avi":1237,"ay ":3414,"avu":1433,"apı":7957,"ata":3117,"ast":3088,"asy":2788,"amı":5927,"atm":1041,"apç":991,"atl":2228,"atr":1379,"ato":2437,"ate":2841,"ati":5438,"att":1422,"anı":18491,"Üni":1310,"Şar":1029,"ji ":1449,"jis":950,"itl":1849,"öğr":1364,"ito":956,"cı ":3493,"ism":1455,"ist":11668,"ita":3780,"ite":3995,"iti":3699,"cıl":1667,"üfu":4682,"iva":1157,"ive":2871,"ilç":2129,"is ":3578,"ion":3161,"ir ":71671,"irm":3093,"irk":1523,"irl":6644,"iri":12312,"isi":13092,"ise":2824,"isa":3166,"ire":3266,"ira":2633,"ird":1851,"it ":3783,"ünl":1704,"üni":930,"ünc":1623,"ünd":2862,"üne":6398,"üml":1023,"üme":1310,"ült":1479,"ür ":5331,"üny":3730,"iyi":1021,"ül ":1714,"iyl":2890,"iyo":3774,"iya":6327,"iye":14035,"üdü":2475,"ük ":5512,"cıy":983,"iz ":6631,"üle":2031,"ülk":2638,"üll":944,"üks":1988,"ün ":4631,"izm":2299,"izl":1160,"izi":3733,"izc":3709,"ükl":1155,"iza":1126,"üm ":2100,"kim":2642,"kil":4381,"kiy":3334,"kiz":4406,"kin":3419,"kis":1066,"kit":1608,"km ":1709,"ki ":15464,"eğe":1107,"eği":4944,"kel":2350,"ken":7944,"kes":1806,"ker":2097,"ket":3347,"kez":3142,"ke ":1164,"kra":1708,"kiş":2390,"kse":2982,"klı":3118,"km²":1573,"kor":1133,"kon":4434,"kom":1557,"kol":1873,"kle":9482,"kla":8235,"kli":3448,"dız":949,"dıy":1257,"dır":19814,"dın":2440,"dı ":5738,"kaz":1358,"kay":2732,"kat":3255,"kar":8976,"kas":2234,"kap":1914,"kan":7174,"kal":4961,"kam":982,"kad":3577,"kab":1637,"dış":1185,"ka ":4513,"dığ":4219,"ha ":3071,"cü ":1507,"ham":1081,"han":2727,"hak":1219,"hal":3743,"hav":968,"har":3475,"has":1125,"hat":1036,"haz":1057,"hay":1743,"he ":3168,"her":2799,"hen":1336,"hem":1060,"hi ":1444,"hip":4144,"hin":2793,"hil":1264,"hir":2302,"hle":2209,"ağı":5375,"gru":2866,"gra":2358,"gul":1221,"gue":2075,"dül":2427,"ian":1361,"dün":2104,"dür":3900,"ibi":3243,"düz":2369,"id ":1061,"iba":969,"ia ":4696,"aş ":1969,"ig ":1399,"ici":2058,"ich":1442,"ice":1186,"ie ":1088,"ica":1674,"idi":13477,"ide":3373,"ida":1813,"if ":1449,"düş":1411,"il ":5350,"im ":8509,"ika":5895,"aşa":4663,"aşl":3782,"aşm":1272,"aşk":2892,"aşt":1881,"ihl":1333,"ihi":3044,"ik ":16892,"iml":3173,"imp":924,"ime":2281,"imd":1567,"imi":5215,"ip ":4274,"inc":5308,"ind":30574,"ina":3996,"aşı":5939,"ino":995,"int":1436,"ins":2898,"ine":16354,"ing":3240,"ini":20786,"inl":1777,"iko":990,"ikl":5637,"iki":4685,"eçi":1614,"eçe":1013,"ila":2643,"in ":47078,"ikt":2801,"ilo":1062,"ill":5588,"ilk":4218,"ilm":10420,"ilg":3153,"ili":20573,"ild":2510,"ile":24870,"ima":2176,"io ":955,"ily":1842,"hri":2688,"hur":2425,"dör":1115,"dön":2425,"fes":1014,"fer":1326,"far":1401,"eyâ":1363,"fa ":1026,"eyb":1488,"eya":8242,"eyl":966,"eyi":4871,"eyd":2007,"eye":3542,"ez ":1852,"ezo":1005,"ezi":2727,"eta":1696,"ete":2836,"eti":16248,"etm":3594,"etl":4019,"etk":1732,"est":3861,"ess":1296,"esw":1087,"ev ":1032,"etr":1981,"ett":2719,"eve":1133,"eva":954,"evl":3752,"erç":1693,"evi":3588,"evr":2587,"ey ":4964,"er ":28073,"es ":5122,"erk":5212,"erl":5910,"eri":48739,"erg":2108,"ere":9775,"erc":963,"erd":6317,"era":4204,"erb":1053,"et ":9016,"açı":2094,"esk":1828,"esl":1542,"esm":989,"esi":30715,"ese":2043,"esa":1128,"ert":1494,"ers":4580,"ern":2620,"erm":2340,"eki":11678,"ekl":5425,"açl":1635,"ekn":1222,"eko":1029,"eks":1790,"ekt":6067,"en ":46837,"ela":1161,"eld":2163,"ele":20479,"eli":10508,"elm":1130,"ell":5734,"els":966,"ema":2676,"eme":6814,"eml":3228,"emm":1228,"emi":6693,"ene":8543,"eng":1269,"ena":1544,"end":6506,"enc":1546,"enm":2094,"enk":1143,"enl":4600,"eni":13441,"ens":1925,"ent":5851,"enz":1180,"egu":1647,"ehr":2303,"ehi":2131,"ek ":9883,"aç ":1092,"ein":1910,"el ":11201,"eke":1777,"eka":935,"em ":2604,"öst":1805,"gis":1807,"gir":1360,"gil":5370,"önü":972,"geç":2375,"gin":1021,"gib":2269,"ört":1016,"öre":4482,"ölü":2891,"ölç":2720,"gi ":2386,"ör ":949,"gen":4928,"ger":3542,"ges":4599,"ağa":1186,"gel":7731,"ağl":9979,"önc":1623,"öne":7107,"ge ":2812,"ağ ":1140,"gaz":1147,"ölg":6551,"gar":1277,"büy":3008,"büm":2701,"gan":1892,"böl":7461,"fus":4728,"fut":963,"öyü":1007,"fre":2029,"for":2226,"öze":3441,"özl":926,"örü":2439,"fil":4052,"da ":52350,"de ":46250,"dak":5123,"dal":2767,"dah":2619,"das":1177,"dar":3775,"dan":21526,"dam":1680,"day":1218,"dde":1533,"cul":955,"cus":1172,"cre":977,"cu ":1846,"ch ":1055,"ces":998,"cek":1367,"cel":1540,"ci ":4802,"ck ":1696,"che":1266,"chl":1164,"cil":1965,"cis":1558,"cin":1363,"ed ":1577,"ebe":1387,"ebi":2611,"efe":1034,"edi":16774,"ede":8053,"eda":1066,"eci":1467,"ece":3057,"dyo":1078,"dur":5671,"duğ":3327,"don":972,"dol":1859,"dok":977,"diğ":3682,"doğ":5145,"dra":1187,"dlı":1288,"du ":2273,"dağ":1454,"der":6369,"des":1911,"dev":2820,"ded":1153,"del":2710,"dek":4272,"den":20219,"dem":1663,"di ":6071,"dla":1190,"do ":1033,"diz":1879,"diy":5465,"din":3013,"dir":24487,"dis":2432,"dik":1433,"dil":6616,"değ":2872,"rga":1818,"ri ":19287,"rgi":1489,"rge":959,"ret":5334,"res":4290,"rev":1391,"rdu":2054,"rg ":1524,"rec":1008,"red":1084,"reg":1836,"ren":6966,"rek":4543,"rel":1942,"rda":4941,"rdi":3615,"rde":5880,"re ":11153,"ray":2290,"müz":3407,"rd ":1472,"rap":1757,"rar":2927,"ras":9820,"rat":4222,"rbi":1042,"rba":1205,"mün":3661,"ran":10600,"ram":4044,"ral":5875,"rak":15812,"rab":1433,"raf":10463,"rad":2346,"rac":1216,"rs ":1268,"ros":1512,"rot":967,"rom":2121,"ron":1936,"rol":2443,"rkç":1463,"rog":1323,"rna":1248,"rne":1926,"ro ":1843,"rma":6313,"rme":4031,"rmi":1407,"rlu":1118,"rli":4427,"rle":9208,"rla":6927,"rki":3194,"rkl":1461,"rke":5713,"rka":1543,"rdı":3751,"riy":3959,"rit":1940,"ris":5765,"rih":5101,"müş":1050,"raş":1079,"ril":8019,"rik":4914,"rin":23896,"rim":3407,"ria":1426,"rdü":1150,"ric":1498,"rid":4314,"rk ":5291,"lıl":999,"lık":8208,"lın":9443,"lım":1113,"lır":2906,"rya":1228,"rup":3018,"run":1284,"rum":2945,"rul":5547,"ry ":1364,"rsi":2903,"rsa":1025,"rta":4786,"rte":4662,"rti":1858,"lı ":21277,"rub":2412,"rt ":3885,"rkı":3548,"ru ":1610,"rlı":1952,"sab":1070,"sad":1069,"nüf":4498,"sah":5074,"sal":4864,"nüm":1062,"san":6337,"nün":1717,"sat":1316,"sar":2328,"say":5210,"sav":1790,"sa ":4002,"nü ":1409,"lış":2586,"lığ":4204,"si ":18237,"sağ":1738,"siz":1136,"siy":3676,"sid":3408,"sia":1857,"sit":3495,"sis":2770,"sin":18799,"sil":2083,"sim":2963,"sik":1925,"se ":2953,"ser":3758,"ses":2131,"sen":2086,"sem":1040,"sel":4229,"sek":2392,"spo":1131,"spa":2896,"son":6601,"su ":5315,"st ":3445,"slu":1935,"sla":3121,"sle":1769,"ski":2726,"ske":1671,"sma":2231,"smi":1955,"mın":4103,"swi":1113,"stü":1161,"sya":2595,"syo":3350,"ste":9865,"sta":9233,"sto":2278,"sti":3825,"stl":934,"str":1911,"mı ":4565,"sun":2855,"tak":3619,"tal":5550,"tab":2400,"tad":4726,"tay":1623,"tat":1503,"tas":2835,"tar":17331,"tap":1242,"tan":13930,"tam":2113,"te ":6912,"tbo":1793,"ta ":6535,"mış":10404,"pa ":2055,"par":4696,"kül":1832,"küm":1078,"pan":3919,"per":2301,"küç":956,"pla":3634,"ple":961,"plu":1414,"lçe":2541,"piy":2262,"por":1907,"pon":1081,"pol":1613,"lçü":2547,"pti":1099,"pra":1013,"pro":2802,"ptı":1375,"lü ":3559,"lüm":2580,"lül":1275,"lük":1015,"iş ":5374,"işi":4832,"işk":1088,"işl":2771,"işt":5462,"kı ":2158,"kıl":1124,"kım":2874,"kıs":2851,"kın":2434,"kıy":1259,"ra ":9857,"mü ":1475,"ngi":4542,"ni ":12017,"nge":1612,"ncü":1344,"nel":4863,"nek":1315,"nen":2962,"nem":4576,"ner":2342,"net":4444,"nes":2204,"ng ":2974,"ned":1504,"ney":4102,"nci":4377,"nce":5236,"nca":4009,"ne ":19615,"nbu":2438,"ndu":1472,"ndr":1044,"ndo":1041,"ndi":6541,"nde":35967,"nda":48013,"ncu":1481,"nak":1647,"nal":3192,"nam":1052,"nan":13136,"nar":2141,"nad":2145,"nd ":2524,"nat":3318,"nas":1441,"nay":1104,"na ":14803,"muş":4297,"nya":10222,"nun":5903,"nus":1510,"nuc":968,"nto":981,"ntr":1143,"nti":4165,"nta":1967,"nte":3585,"nmı":2809,"nst":987,"nse":1596,"nsi":1251,"nsa":3394,"nu ":4463,"nlı":4882,"nra":2468,"nt ":2440,"niş":970,"ns ":1946,"nlü":1339,"nom":2114,"nne":1221,"nme":1944,"nma":4498,"nmi":973,"nli":2503,"nla":14247,"nle":4649,"nlu":1296,"nka":1369,"ndı":4155,"nic":987,"ndü":1181,"niy":1052,"niz":3806,"ncı":1170,"niv":2157,"nis":1976,"nir":2294,"nim":1125,"nin":20306,"nik":2175,"nil":944,"ogr":1717,"ok ":4093,"oji":2722,"ol ":4011,"ock":1051,"ode":1635,"of ":1645,"iği":9467,"iğe":1929,"obi":987,"nsı":2063,"köy":1994,"oyu":4442,"oyn":1402,"oto":3517,"osy":1120,"ost":1210,"osu":1204,"ovi":1205,"ova":1326,"opl":2685,"os ":3304,"çıl":1025,"çık":4300,"or ":3792,"ork":1035,"orl":1782,"orm":2268,"ord":2039,"ore":1088,"org":1822,"ori":2004,"ort":8149,"oru":2784,"m² ":1578,"ora":1605,"ola":24963,"old":3573,"olc":925,"on ":12904,"oli":2380,"oll":2123,"ole":1383,"ols":1253,"olm":3895,"olo":4194,"olu":8661,"om ":1843,"kçe":1715,"okt":974,"oku":2095,"ona":2977,"ond":1625,"one":1566,"oni":1789,"onl":1840,"ono":2239,"onr":2464,"ons":1619,"ont":1468,"onu":7635,"ony":1563,"oma":3785,"ome":1724,"omi":1625,"omo":1197,"la ":11386,"le ":19263,"lde":3058,"ldi":2187,"ldu":3454,"lab":1412,"lac":926,"lad":1565,"lah":1109,"lak":1159,"gün":5583,"lan":43556,"lam":9121,"lar":57894,"lat":2773,"las":2417,"lay":4711,"ld ":958,"kuz":3055,"kur":7540,"kul":7562,"kta":6976,"kte":6402,"ksi":2396,"ktr":1368,"kti":2217,"gös":1720,"gör":5828,"ktı":2504,"lon":1166,"liğ":4787,"loj":2682,"lmi":5563,"lme":4109,"leş":5292,"lma":11588,"lmu":3175,"lst":1073,"lmı":3998,"lta":1114,"lte":1773,"lu ":7438,"llı":1332,"liş":3210,"lt ":1120,"lge":7089,"lgi":3114,"li ":16772,"lbü":2715,"lga":999,"ley":2404,"lev":1310,"les":3725,"let":9791,"ler":45886,"lem":4975,"len":16099,"lek":2724,"led":5291,"lec":1154,"lo ":939,"lla":11633,"lle":6955,"lli":5594,"lke":3179,"lm ":1714,"ldı":4073,"ll ":1428,"lit":1562,"lis":4164,"lir":4755,"lin":9226,"lim":4866,"liz":3501,"liy":1446,"lid":1846,"lia":946,"lk ":5135,"lik":9172,"lil":1002,"laş":3620,"ma ":9869,"mac":2119,"mak":8947,"mad":3045,"mar":2763,"mas":9043,"mal":4481,"mam":1405,"man":15375,"may":3231,"mat":2471,"me ":6043,"mda":1303,"mde":1460,"mdi":1706,"med":2057,"eş ":1039,"met":4942,"mes":5862,"mer":6138,"mel":3135,"men":5922,"mek":5999,"maç":1261,"mey":2238,"çüm":2197,"çük":1139,"luk":1924,"lup":1601,"lun":8327,"lum":1008,"lus":2340,"fın":8889,"lya":3819,"luğ":2080,"ltı":2446,"luş":5045,"mpi":1862,"mod":1329,"mon":1220,"mpa":1588,"mu ":1170,"miş":7571,"mun":1116,"muz":1291,"mhu":2129,"eşm":1123,"eşt":1717,"mi ":6405,"eşi":5221,"min":6521,"mil":2577,"mir":1395,"mis":1028,"mcı":1046,"mit":1204,"mid":2070,"mik":1507,"mlu":936,"mli":2428,"mle":5174,"mla":5844,"içi":8066,"içe":2201,"mmu":1223,"uğu":5772,"tı ":4071,"zun":1539,"tıl":3307,"tın":2882,"tır":10360,"tıs":1266,"zyo":962,"tığ":1565,"zi ":3100,"zet":1053,"zey":4798,"zen":2488,"zel":3272,"zer":6678,"ze ":1245,"zce":3650,"zde":1081,"zam":2429,"zan":2260,"zak":1080,"zar":3022,"zon":1273,"zme":978,"rşı":2081,"zla":2044,"zgü":956,"zle":1804,"zin":1617,"zik":2532,"zir":1408,"zis":1290,"yum":1152,"yun":4454,"sı ":14744,"ynı":1581,"ylü":1204,"yol":2904,"yor":1353,"yon":7554,"yrı":1788,"sıd":1928,"sıl":1373,"sım":1355,"sır":2328,"sın":16353,"sız":2708,"sıy":986,"ye ":8402,"yda":1138,"yed":1753,"yes":4583,"yer":7744,"yen":3683,"yel":1262,"yet":5659,"ya ":30890,"rış":2333,"yba":1101,"yaz":5444,"yay":4498,"yat":3352,"yar":6010,"tür":5785,"yas":4408,"yap":11061,"tün":1161,"yan":10579,"yal":5013,"tüm":1022,"yak":2748,"ydı":961,"yla":6744,"yle":4770,"yo ":1109,"yna":2824,"yi ":2354,"ygu":1552,"yin":3381,"yaş":2680,"tör":1578,"rı ":14655,"rım":1734,"rın":16497,"rıl":3423,"rıs":1452,"sür":2901,"söz":1499,"sça":1363,"wig":1137,"rü ":1492,"rün":2219,"rül":1328,"vru":1839,"vri":1244,"vre":1449,"vra":964,"pıl":3787,"pım":2266,"vil":989,"vaş":2523,"viz":960,"vis":1122,"rça":1249,"rçe":1701,"vle":3463,"vi ":1409,"vey":5886,"ver":8628,"vet":1190,"ven":1213,"ve ":40015,"val":1203,"van":2336,"var":3116,"va ":2329,"uzu":1436,"uze":4115,"uyu":1005,"uza":1667,"uyg":1403,"uya":1017,"uz ":2525,"usç":954,"uva":1017,"usl":3168,"usa":1561,"usu":5699,"ust":2940,"utb":1431,"us ":4706,"ut ":1302,"ura":2627,"urd":1061,"urg":1888,"uri":2897,"urm":935,"uro":982,"urt":1354,"uru":9050,"upa":2517,"ur ":10047,"umu":1897,"umh":2127,"uml":1645,"uma":1385,"unu":6201,"unl":2635,"unm":1881,"unc":2040,"und":5507,"una":8329,"up ":2721,"ukl":1910,"um ":2957,"ulu":12477,"ult":959,"ulm":2979,"ull":6976,"ula":6334,"un ":10772,"uk ":2310,"ul ":3821,"uha":1114,"ucu":1955,"udi":996,"ubu":2444,"uca":2065,"oğu":5104,"oğr":1470,"ues":1785,"udu":3053,"oğa":1679,"oğl":957,"uba":1499,"tur":6494,"nır":2256,"nıl":5540,"nın":19519,"nım":1872,"nıf":1036,"tre":1553,"tra":2671,"tri":1839,"tro":3501,"tte":1278,"tti":2653,"nı ":11130,"tme":3621,"tma":1695,"to ":1507,"tiğ":1447,"tos":1656,"tom":1446,"ton":2840,"tol":991,"tor":3257,"top":3422,"til":2825,"tik":5330,"tif":1158,"taş":2105,"tir":10735,"tis":1905,"tin":8295,"tim":3897,"tio":1845,"tic":1549,"tid":1024,"tiy":1388,"tki":2141,"pça":1027,"tli":1970,"tla":3863,"tle":6323,"tem":6048,"ten":3191,"tei":1620,"tek":7857,"tel":3048,"ted":3509,"th ":1012,"tes":4105,"ter":10441,"ti ":6632,"the":1663,"üşü":1357,"üğü":1444,"zı ":1890,"zıl":2487,"üç ":948,"zöl":2079,"üçü":1824,"yı ":2418,"yım":2005,"yıl":14867,"yın":3653,"yıs":2728,"yük":5408,"yüz":3683,"yön":3982,"üzö":2079,"üyü":3989,"üzi":2596,"üze":7514,"üye":1004,"ütü":1249,"üsü":1036,"ürü":4447,"ürk":7688,"ürl":1211,"üre":5286,"ümü":6554,"ülü":2192,"üs ":1086,"üst":1769,"ünü":3990,"uş ":2901,"uşu":1743,"uşt":4122,"uşa":1668,"yâl":1370},"n_words":[9192208,10449574,7620193],"name":"tr","type":"latin","flags":["diacritics"]}
diff --git a/contrib/languages-data/uk.json b/contrib/languages-data/uk.json
new file mode 100644
index 0000000..3a0cdfd
--- /dev/null
+++ b/contrib/languages-data/uk.json
@@ -0,0 +1 @@
+{"freq":{"D":2686,"E":45335,"F":1931,"G":2251,"A":4545,"B":2792,"C":4259,"L":2592,"M":3964,"N":23611,"O":1971,"H":1743,"I":29162,"T":3063,"V":2399,"P":3688,"S":26730,"R":2282,"X":2695,"f":3088,"g":5064,"d":7920,"e":26938,"b":4218,"c":9825,"a":26142,"n":18042,"o":19695,"l":12837,"m":7697,"k":2703,"h":7896,"i":22805,"w":2062,"v":2426,"u":10520,"t":16747,"s":15016,"r":19222,"p":6493,"y":4268,"x":1842,"́":45988,"ь":314859,"ю":121712,"я":259056,"ш":55425,"щ":34352,"ф":70318,"х":152145,"ц":190380,"ч":157844,"р":767814,"с":616542,"т":784384,"у":465611," t":1584,"є":83027,"ї":138148,"і":1047876," p":1613,"Є":3706,"І":14787,"Й":2127,"Л":22219,"К":34440,"Н":39528,"М":28217,"П":47409,"О":15488,"Б":23123,"А":35851,"Г":15610,"В":31676,"Е":8630,"Д":40073,"З":11772,"Ж":3520,"Ш":12613," I":24900,"Ю":2003,"Я":3891," L":1894," M":2910," B":2299,"Т":14847," C":3270,"У":18037,"Р":27850," A":3247,"С":43546," F":1599,"Ц":5942,"Ч":5807," G":1797,"Ф":32198," D":1850,"Х":6705,"л":484150,"к":541520,"й":174507," X":1787,"и":768429,"п":339521,"о":1248992,"н":1274404,"м":381536,"г":226618," S":3616," R":1762,"в":579991,"б":162019," P":2630,"а":1251055,"з":257293,"ж":95383,"е":806087," T":2138,"д":454853,"Ґ":1990,"ґ":4623," А":31736," Б":22701," В":30948," Г":15174," Д":39397," Е":7917," Ж":3460," З":11194," Й":2051," К":32754," Л":21825," М":27530," Н":36830," О":14175," П":46014," Є":3619," І":13525,"EE":21655,"I ":2684," б":36681," а":43891," г":34855," в":154625," е":34325," д":117491," з":113779," ж":8768," й":6819," л":26339," к":61923," н":123337," м":87814," п":148797," о":74913," Р":21137," С":39028," Т":13848," У":16956," Ф":31501," Х":6271," Ц":5651," Ч":5720," Ш":10618," Ю":1964," Я":3806," ї":5196," і":83499," є":13641," т":92032," у":89129," р":101219," с":120858," ц":13483," ч":24594," ф":40198," х":11362," ш":8917," щ":18497," ю":2119," я":36739," Ґ":1984,"E ":21977,"NS":21624,"II":2127,"IN":21640,"SE":21638,"a ":4767,"i ":1589,"he":1740,"el":1571,"en":2535,"es":2308,"er":4659,"e ":6591,"d ":1983,"at":2458,"ar":2758,"al":2232,"an":3512,"ac":1574,"nt":1885,"on":3299,"or":2694,"r ":2633,"o ":1648,"na":1574,"ni":1555,"li":1877,"le":1913,"la":1826,"n ":4271,"ic":2243,"ia":1621,"is":1891,"in":3535,"io":1638,"l ":1929,"y ":1830,"us":2293,"um":1631,"te":2376,"ti":2183,"th":1945,"ta":1627,"st":1645,"ro":2012,"ri":2866,"re":1930,"ra":2584,"t ":3059,"s ":6868,"́ ":2290,"́в":3741,"́д":2078,"́к":2065,"́й":3031,"́м":2120,"́л":4305,"́н":8997,"́р":4428,"́с":2026,"́т":2528,"А ":2905,"В ":2961,"ьє":2885,"юв":4387,"юд":3425,"юр":3284,"ют":14456,"юч":4396,"ює":2722,"яд":6152,"яг":3108,"яв":2990,"юю":1542,"ян":10771,"ям":9107,"ял":2194,"як":26414,"яз":4329,"ях":3443,"ят":9738,"яр":2263,"яч":4477,"яє":2354,"ші":2899,"щи":3380,"ще":5720,"ща":2636,"що":19501,"щі":1589,"ьк":115778,"і́":5805,"ьб":1841,"ьв":2256,"ьш":4727,"ьс":24456,"ьт":4512,"ьн":50183,"ьм":4831,"ьп":5070,"ьо":12701,"фі":13912,"хи":2735,"хн":7629,"хо":17158,"хр":2067,"ху":3902,"ха":7837,"ци":27077,"хі":11714,"цт":3698,"цу":23008,"це":16672,"чл":2502,"чн":59778,"чо":6493,"ці":93529,"чи":16012,"чк":3842,"чу":3146,"ць":14520,"че":19603,"ця":6177,"ча":25750,"цю":1804,"шо":7889,"шн":3282,"шк":3613,"шл":2107,"ши":11930,"шт":3498,"шу":1879,"чч":1777,"чі":4713,"ше":8200,"ша":4745,"ск":15952,"см":3119,"сл":40211,"со":21185,"сн":22933,"сп":20278,"св":10260,"се":44728,"си":21636,"рі":55323,"рш":6088,"рю":2898,"са":14767,"ря":9090,"рр":2512,"рс":15180,"рт":37524,"ру":28810,"рх":8486,"рц":1712,"тн":17002,"тл":3183,"тк":8278,"тс":3967,"тр":48477,"то":75584,"те":71719,"тв":21013,"ти":129473,"сі":43933,"сь":82109,"та":153881,"ся":27435,"сю":2514,"су":15433,"сс":3485,"ст":199868,"сх":2897,"сц":3318,"ур":24405,"уп":9084,"ут":35010,"ус":11964,"ум":8171,"ул":17666,"ун":34351,"ті":36808,"уз":30153,"ук":20646,"уд":13417,"уг":6510,"уж":3934,"тю":2714,"уа":9949,"тя":7369,"уб":6901,"ув":18706,"ть":44694,"тт":6769,"ту":74115,"фу":3164,"фт":2346,"фр":25365,"фо":9781,"фе":6303,"ує":8184,"фа":4406,"ую":4341,"уч":6062,"уш":1880,"ух":4146,"іш":7237,"іц":26077,"іч":43837,"іє":28690,"їв":4260,"ію":3757,"ія":30142,"ін":70416,"ім":15479,"іл":34864,"ік":38962,"ій":44252,"із":21994,"іж":6237,"іх":3168,"іф":1617,"іт":52153,"іс":42072,"ір":18028,"іп":25092,"іо":58208,"ів":71042,"іг":5632,"ід":79007,"іа":11156,"іб":26830,"її":1916,"ії":55045,"їн":18326,"їх":2476,"єю":25186,"єм":3400,"єн":2727,"єт":11907,"єр":2623,"єд":3075,"єк":1686,"єв":4497," IN":21609,"а ":272631,"Р ":3804,"У ":2439,"Єв":1986,"Ів":2523,"Ін":2522,"Іл":1807,"к ":38380,"й ":109949,"Ле":2968,"Ла":4470,"Ку":2193,"Ко":9293,"м ":60876,"Кр":3269,"Ки":4604,"Ка":8275,"л ":10034,"Йо":1971,"На":28658,"Не":1779,"Мі":3387,"Мо":5475,"о ":171358,"Ма":9118,"Ль":1829,"Ми":3400,"Лі":2822,"Ме":3642,"н ":48334,"Ло":1601,"Лу":5897,"Па":5129,"Пе":6590,"По":6892,"с ":17750,"р ":43282,"Ос":1646,"Ор":1949,"Ні":1710,"Ол":3009,"Но":2027,"п ":4596,"в ":98488,"Ам":1689,"Ан":4049,"Ак":1666,"Ал":7021,"Ав":2330,"Ба":3984,"Ар":5755,"б ":28913,"Во":3994,"д ":30432,"Ве":7201,"Ви":2444,"Бі":2125,"Га":3476,"Бо":3837,"г ":7123,"Бр":2821,"Бе":3301,"Ва":4390,"Бу":4720,"Дж":1933,"Де":3400,"До":4198,"ж ":10290,"Ві":6492,"Ге":2702,"Гр":3253,"Го":2499,"е ":51777,"Да":23258,"и ":147695,"За":6302,"з ":38125,"Ен":2243,"ь ":77560,"е́":6375,"Ша":6268,"Ше":1818,"и́":5895,"ю ":78537,"я ":164366,"Ст":3877,"Су":1648,"Та":2989,"Сі":1751,"Те":3497,"То":1720,"ф ":2068,"Тр":1849,"Ук":9250,"х ":93221,"РС":2535,"Пр":9085,"Пу":2178,"Ра":2736,"Ре":2944,"Пі":12505,"Ри":1682,"СР":4182,"т ":74217,"Ро":9385,"Ру":1597,"СШ":1868,"Са":5497,"Св":1748,"Си":1634,"Се":8345,"Со":4417,"у ":182563,"Це":4350,"ш ":3438,"а́":12498,"Че":2490,"ША":1865,"Фр":25493,"Фе":1857,"Ха":2266,"ч ":13144,"ль":97373,"лю":13039,"мб":2910,"ма":43992,"ля":27148,"ме":56645,"лі":100876,"ми":33489,"мл":2674,"мк":1772,"лл":4063,"ло":47531,"лу":11894,"ла":56466,"ле":64714,"лк":3429,"ли":41628,"кі":53446,"км":1768,"кн":2648,"кл":18117,"кр":33069,"кс":9526,"ко":160847,"кт":21418,"ку":30802,"кц":4113,"ка":63693,"ки":83769,"кв":7446,"ке":9654,"йн":15208,"йо":12518,"йк":1565,"йл":1986,"йм":3040,"йс":17098,"ия":1553,"ищ":6034,"иш":3403,"иї":2680,"у́":2267,"иє":2892,"йд":1750,"йб":2838,"ип":31662,"им":32760,"ин":52951,"ик":67385,"ил":14249,"ий":79035,"иц":14181,"ич":30391,"иф":2474,"их":75898,"ит":52472,"ир":15232,"ис":66975,"ри":69723,"рк":9097,"рл":2627,"рм":12337,"рн":24183,"ро":132547,"рп":3819,"ра":159212,"рб":3520,"рв":5932,"рг":12811,"рд":11789,"ре":97753,"рж":5968,"пі":30461,"пр":61701,"пт":2856,"пс":1790,"пу":8363,"ої":42339,"пи":15610,"пн":6007,"по":70389,"пл":11153,"ою":42263,"оя":1743,"па":69692,"оє":2366,"пе":54812,"ощ":2597,"ош":4991,"оч":9700,"оц":9082,"ос":101143,"ор":94904,"оп":23531,"оо":1738,"ох":8570,"оф":6129,"от":23405,"ок":38039,"ол":59373,"ом":93606,"он":134604,"ож":13014,"ні":182081,"оз":26801,"ой":3869,"ов":119233,"ог":97492,"од":63564,"ое":3185,"ню":3932,"ня":95444,"об":38077,"нь":43392,"нц":54211,"нш":5309,"нч":1583,"нт":57484,"нс":58539,"нф":2536,"ну":21092,"но":153491,"нн":89867,"нк":13365,"мі":70237,"ни":149376,"не":44107,"нг":6365,"нд":19006,"на":216248,"му":49313,"мс":3566,"мп":12768,"мо":32397,"мн":5261,"мм":1713,"ге":9243,"ві":92696,"ги":4209,"гн":2639,"го":97928,"гл":6748,"гр":20635,"гу":9031,"дв":4924,"дб":1844,"да":33332,"вд":7688,"ве":48786,"вж":2823,"бі":16889,"ви":75994,"вк":5403,"вл":11314,"вн":39402,"є ":23248,"во":52805,"вп":2102,"вр":5405,"вс":21071,"ву":11922,"вт":7129,"вч":4263,"вц":1693,"га":24943,"вя":2583,"би":5176,"аї":20255,"ає":15656,"бе":15560,"бр":8733,"бн":4172,"бо":24369,"бл":17974,"бк":1672,"бу":17942,"бс":1904,"ва":79944,"ад":39043,"аж":6697,"аз":20777,"аб":18055,"ав":68705,"аг":13743,"ам":54378,"ан":186015,"ап":12553,"ай":22198,"ак":29485,"ал":112843,"ах":20860,"аф":6570,"ач":13762,"ац":41300,"ас":69626,"ар":92013,"ау":6764,"ат":76559,"ба":13923,"аю":6775,"аш":6416,"зт":3423,"зр":3339,"зп":3833,"зу":8061,"зк":2540,"зи":11355,"жі":2914,"зо":12242,"зн":22505,"зм":7721,"ив":25953,"иг":5812,"иб":4671,"иж":2800,"зі":9568,"из":10688,"ид":11644,"зь":26870,"жо":4282,"жу":4654,"еї":6545,"жи":9686,"жн":10415,"за":70302,"зб":4513,"зв":12953,"зг":1786,"зд":4686,"зе":9240,"еф":2808,"ет":48754,"ес":21933,"ер":123318,"еп":28664,"ї ":106019,"ео":7564,"ен":200818,"ем":25862,"ел":52238,"ек":44785,"ей":12736,"ез":15415,"ді":60214,"еж":8691,"же":36749,"ея":2396,"жа":10671,"еч":5677,"еш":1971,"ех":4631,"ец":12416,"дс":7505,"др":11609,"ду":13511,"дн":33993,"дм":3635,"дп":3899,"і ":217878,"до":70187,"ди":35057,"дл":9508,"дк":6162,"де":83166,"дз":2175,"гі":41117,"о́":9048,"дж":30353,"еб":4669,"ев":15708,"ег":29992,"ед":62088,"дя":4582,"еа":4570," ар":4635," ба":6422," аб":10662," ав":3504," ад":2033," ал":3311," ак":3721," ан":4873," ам":2325," бу":8984," ва":2979," бе":6626," бр":1983," бо":3438," бл":2347," ву":2932," га":4235," бі":6025," ви":31633," ве":7581," во":6259," є ":4843," вс":3250," вл":2695," дв":2986," да":3768," гу":2423," го":6549," гр":11679," ге":4251," ві":52058," до":45102," і ":33511," др":3031," де":37135," гі":3754," ди":4078," дл":8069," ел":3252," ек":24873," ді":8202," зд":2257," зе":1996," за":56665," зв":4345," зб":3336," жо":2604," жи":2736," зм":2726," зо":3274," зн":5228," зі":1585," йо":3069," ка":7109," кв":3523," кр":6792," ко":25840," кн":1684," кл":3559," ку":3737," ла":4131," кі":5040," ли":5653," ле":3016," ме":10350," лі":7010," ми":2991," лю":4680," ма":12887," мо":13377," му":26214," ни":3719," мі":18386," не":11146," на":101285," но":3271," ок":3237," оз":1668," ні":2363," од":9316," об":14076," от":1642," ор":5923," ос":30485," оп":3801," по":44438," пл":4775," пи":3225," пе":16844," па":8207," Ре":2937," Пі":12498," Ра":2732," Ро":9378," Ри":1682," Пу":2177," Пр":9060," Пе":6585," Па":5105," По":6869," Ос":1642," Ор":1944," р ":1902," Те":3490," Сі":1746," То":1716," Тр":1847," Ст":3864," Су":1643," Та":2975," Св":1745," Си":1629," Се":8339," Со":4411," у ":67844," Ру":1597," СШ":1865," Са":5492," Фр":25493," Фе":1855," Ук":9235," Це":4345," Ха":2264," Ше":1818," Ша":6265," Че":2486," я ":1678," Ба":3980," Ар":5747," в ":34140," Ан":4045," Ам":1689," Ал":7017," Ак":1665," Ав":2326," Ва":4387," Бу":4716," Бо":3831," Бр":2819," Бе":3297," а ":3713," Дж":1930," Де":3390," До":4191," Ен":2238," з ":28463," Га":3472," Ве":7185," Ви":2437," Бі":2122," Во":3987," Да":23253," Ге":2696," Ві":6479," Го":2496," Гр":3249," Йо":1970," Ки":4602," Ка":8259," За":6236," й ":3420," Мо":5471," На":28633," Не":1775," Мі":3382," Но":2025," Ол":3009," Ні":1705," Ко":9278," м ":2294," Кр":3260," Ку":2189," Ла":4467," Ле":2966," Ло":1600," Лу":5896," Ль":1829," Ма":9106," Лі":2816," Ме":3636," Ми":3397," У ":1855," Єв":1986," Ів":2522," Іл":1806," Ін":2499," В ":2749,"Шам":2267,"Шар":2655,"INS":21596,"Кар":1691,"Кор":2456,"Киї":2310," єк":1604," єд":2480," їх":2188," іс":5151," із":3768," ім":4088," ін":33850," її":1897,"Луа":4691," ра":11678," ре":33133," пі":17870," ри":2122," ро":38150," пр":49254," св":7646," рі":9410," си":9133," се":11547," сл":3167," см":1572," ск":6896," сп":12079," со":4130," ру":3107," са":3132," сі":4244," ти":4115," тв":2718," те":13347," то":7539," тр":8770," сх":2630," ст":41843," су":7442," сю":1727," та":50935," ук":7367," ті":1840," ус":2008," ут":2182," ун":1655," фо":4647," фр":23569," фу":2570," фа":2336," уч":1717," хр":1639," хо":1776," ху":1552," фі":5255," ха":2328," ци":1770," це":7308," чо":1959," чл":2302," чи":4976," ці":2356," че":5747," ча":8926,"Льв":1624,"Мар":4041," шт":1850," що":17557," ян":1885," яз":3165," як":23816," ят":2201,"Мик":1636,"Мон":1603,"Нас":21926,"ад ":3964,"ав ":5473,"EE ":21630,"ам ":4361,"ан ":7774,"ак ":2945,"ал ":2859,"ай ":1706,"Оле":2078,"авч":1579,"авт":2891,"ага":5972,"аві":2505,"аго":2958,"ада":8317,"ади":3697,"аде":2606,"аду":2146,"адс":2241,"адо":2036,"адм":1600,"адя":2411,"аді":3733,"би ":2028,"ажа":1619,"або":10777,"абе":1768,"ава":4168,"авн":9017,"авл":3400,"авс":2814,"ає ":8442,"аво":4040,"аве":24071,"ави":3333,"бо ":10197,"ало":3744,"алу":3259,"ала":5763,"али":6052,"акі":1706,"але":7492,"амо":2377,"амп":2753,"ама":2782,"аль":45514,"ами":10086,"аме":25683,"алі":34159,"анн":20969,"ано":8600,"ану":2414,"анс":11789,"ант":9369,"анц":47104,"ань":4811,"ана":9630,"анд":6858,"анг":2311,"ани":9657,"амі":2404,"ане":2623,"анк":2826,"ані":36761,"азу":2305,"азо":3692,"ази":2579,"азв":4425,"аза":2057,"азі":1724,"айс":1694,"айо":6278,"айн":1941,"айб":2377,"акт":6740,"ако":7371,"ака":2878,"ах ":10031,"Пар":1681,"ас ":3290,"ар ":5874,"ат ":4127,"СР ":2617,"ба ":1557,"Пет":1744,"Пер":2277,"Пол":2322,"РСР":2369,"При":2113,"Про":3790,"Пуа":1541,"Пік":2365,"Пір":3646,"Рос":2415,"Роз":1581,"Рон":3384,"Пів":4853,"Аль":5551,"Сан":1563,"США":1862,"Сер":1775,"Сен":4479,"Ста":1577,"Ард":2815,"Тер":1748,"Вол":2219,"Вер":4116,"Вел":1833,"Укр":9121,"Бур":2431,"Фра":24714,"Дан":22031,"ША ":1853,"а́н":3396,"Цен":2677,"Чер":1615,"NSE":21595,"лам":1827,"лан":5122,"лас":12503,"лат":2370,"ма ":7369,"ля ":13589,"лав":3656,"лад":10782,"кці":3993,"ль ":10569,"кул":3808,"кур":1627,"кою":25572,"кої":15644,"кре":3214,"кра":21169,"кри":4236,"кро":1703,"лу ":3819,"кса":1966,"кте":2165,"кти":3833,"кто":5082,"ктр":2798,"кту":3015,"кла":11761,"ло ":6223,"клю":1692,"клі":1698,"ког":16518,"ков":13165,"ком":15826,"кон":32464,"коп":2254,"кор":9858,"кос":2132,"кож":4742,"кол":8207,"ким":4246,"кий":27960,"ких":8957,"кві":3598,"ле ":3371,"кі ":7394,"ли ":7619,"кер":1599,"ква":2483,"ках":2083,"кат":2360,"кар":5682,"кам":2944,"кан":6220,"кал":2575,"каз":1777,"кад":1810,"ла ":13040,"Іва":2158,"йсь":13547,"кт ":1727,"ку ":18769,"йна":1852,"йно":3774,"йни":5036,"йов":2120,"йог":2770,"йон":6057,"ко ":7248,"иїв":2473,"ки ":39616,"ке ":4049,"йбі":2012,"од ":2452,"нах":3061,"нац":23841,"нау":3625,"наф":1792,"нач":7238,"ог ":2185,"нан":4567,"нам":2788,"нал":31101,"нат":2955,"нас":4427,"нар":6454,"нап":3620,"над":4116,"нак":1904,"най":5486,"наз":4802,"нде":1705,"нда":2557,"ож ":4256,"нгл":1926,"неї":3826,"нен":4632,"нер":5081,"нес":1886,"нет":1977,"нец":1734,"нев":2650,"нез":1549,"нді":4444,"ні ":100559,"ндр":3680,"мії":2239,"ник":15713,"ний":35532,"ок ":8024,"мір":2620,"міс":12191,"мік":1956,"мін":9879,"міч":24763,"між":4232,"нь ":34487,"ня ":85321,"ню ":1561,"ов ":5346,"нав":26133,"об ":3066,"мпо":1641,"нт ":27692,"мпе":2663,"мпа":3905,"ну ":12904,"мпі":1622,"мсь":1988,"мун":23434,"муз":3552,"ліз":3946,"лій":2290,"лік":3856,"лід":25282,"лів":5148,"літ":32189,"ліс":3353,"лін":5016,"лії":2011,"мис":2927,"мир":2021,"но ":17011,"мно":2222,"мод":1683,"мог":2124,"мов":8204,"мож":2842,"мон":3152,"мол":2071,"мор":4169,"нс ":2855,"має":2263,"ляє":1872,"мац":1743,"мал":3075,"мад":2394,"ляд":1770,"мат":7712,"мас":1664,"ляр":1752,"мар":2087,"лян":1603,"ман":6730,"люч":1579,"маг":1762,"люд":2454,"лют":2538,"мет":6826,"мен":33354,"ни ":22900,"мер":6133,"меж":1784,"мі ":2838,"не ":13395,"льп":5070,"льн":50159,"льо":2192,"на ":73406,"льм":2015,"льк":3474,"льш":4684,"льт":4502,"льс":6409,"му ":17492,"лок":1590,"лог":8013,"лод":3460,"лор":1601,"лос":2987,"лот":1979,"лом":2920,"лон":2251,"лов":10255,"луж":1851,"ків":10057,"кій":3619,"кіл":3020,"кіп":21719,"кін":3986,"кіс":1867,"лив":4838,"лиз":2163,"лик":4841,"лі ":6988,"леж":2754,"ми ":23542,"лен":36965,"лем":3035,"лек":7457,"лиц":3452,"лиш":2141,"лис":4376,"лин":3727,"лип":2133,"пат":1536,"пад":4424,"пал":24506,"пан":5073,"пар":26316,"ре ":2016,"ра ":10978,"пис":6483,"пла":3785,"пле":2508,"пло":1933,"ро ":4356,"пед":22902,"ри ":9926,"пер":22098,"печ":1592,"ори":10852,"опі":2491,"орд":2209,"оре":6973,"орг":5783,"орс":3875,"оро":12661,"орм":6382,"орн":2519,"опу":1571,"ора":4398,"опе":4273,"опи":2385,"опо":5348,"опа":3672,"осі":28049,"оте":2368,"оти":4264,"ото":4402,"отр":2511,"ота":2076,"орі":12506,"оси":1994,"оск":2565,"осл":27847,"осн":4625,"осо":4627,"осп":1773,"ост":18846,"ору":3825,"орт":4236,"оря":2411,"оми":3494,"олі":11097,"оме":3420,"ома":8391,"оля":2505,"олю":1788,"оль":6830,"олу":1961,"по ":3091,"оло":19189,"оле":3051,"оли":6076,"окі":1858,"ола":2363,"окр":3962,"оку":9695,"око":4015,"оні":30331,"онс":3562,"онт":3739,"ону":5669,"они":2616,"омі":28058,"оно":28802,"онн":3725,"она":36879,"онд":1886,"оне":2765,"омо":4256,"омп":5253,"ому":14955,"оча":2902,"очи":1547,"оці":5589,"офе":1770,"оце":2765,"офі":2407,"охо":3822,"оба":1591,"нят":1850,"ням":4193,"ова":23572,"обу":2657,"обр":3128,"обо":4431,"обн":1797,"обл":9987,"оби":2069,"ою ":40879,"ньо":6069,"па ":1808,"оки":2050,"оке":1657,"ока":3381,"ожн":3109,"озв":2698,"нів":6803,"ніз":4328,"ози":2146,"оза":1905,"озт":3392,"ніт":1771,"ніс":9671,"ніц":22879,"ніш":2881,"ніч":5481,"нік":1930,"ній":8198,"озм":1625,"нім":3177,"озн":3472,"озр":1991,"озп":1689,"нії":3375,"нія":2921,"одн":11088,"оди":15822,"одж":2752,"огі":6272,"оду":4615,"одо":6110,"ої ":40186,"пи ":6419,"оді":8902,"оже":2141,"обі":3439,"ове":4729,"овл":2741,"ови":27036,"ово":14293,"овн":9596,"овт":2617,"ову":5401,"овс":3796,"ога":1966,"ові":15249,"ого":77853,"огр":5122,"ода":5921,"оде":2118,"ної":16682,"ною":5373,"нос":8914,"ноп":1707,"ном":33988,"нок":2048,"нні":5141,"ног":41554,"нов":14623,"ння":64956,"нно":4474,"ор ":9067,"нни":8535,"нна":3011,"SEE":21595,"нко":3013,"он ":7906,"нку":1909,"нка":2370,"ом ":22352,"ним":10232,"нин":2727,"нич":2031,"них":49028,"ниц":6236,"нши":2462,"нці":27792,"нцу":22511,"нув":2948,"нті":3002,"нсь":25045,"нта":5969,"нте":3274,"нти":4476,"нту":1658,"нто":2721,"ох ":2654,"нтр":7071,"нст":25118,"сам":2553,"рям":1886,"сан":2886,"ряд":3770,"сво":2566,"свя":1631,"сві":5195,"сі ":3512,"сел":26979,"ти ":13731,"сен":1784,"сер":9261,"рів":9362,"рід":3016,"ріа":3973,"рій":3132,"різ":4766,"ріо":2440,"ріш":1652,"річ":3119,"рія":3137,"сис":3972,"сит":2195,"рії":5126,"син":2791,"сил":2927,"ска":1987,"сли":1774,"сла":3451,"ско":3258,"скл":4964,"слі":25511,"сля":1724,"слу":2674,"сло":3957,"то ":6783,"сни":4268,"сня":2333,"соб":4324,"сов":3112,"сні":1707,"сок":1638,"сно":9584,"тр ":4211,"сну":2111,"спе":2794,"сор":1743,"сон":2424,"соц":1647,"ту ":33552,"спі":4109,"спу":1543,"спо":6481,"спр":2669,"су ":3870,"роц":5357,"рот":5051,"роф":2524,"роп":4979,"рос":8219,"ст ":6153,"рпн":1909,"рсь":7310,"рта":23604,"рст":1652,"рти":2701,"рси":2327,"рух":1575,"рту":2197,"рті":2581,"рук":2570,"руг":2655,"руд":3764,"руп":2957,"рус":1928,"рхі":2138,"рхн":4291,"рши":1950,"сь ":1944,"та ":53895,"ся ":24676,"рад":5772,"раж":1608,"раз":3483,"рав":11695,"рам":5248,"ран":55851,"рай":6264,"рак":4056,"рал":6105,"рах":2738,"раф":3312,"рац":4988,"рас":1585,"рат":9434,"раї":18701,"рі ":5250,"рде":3344,"ргі":2098,"реб":1606,"рев":3520,"рег":26086,"ред":9949,"реа":2012,"рет":3016,"рес":6263,"си ":1816,"рен":12198,"рем":5175,"рел":2059,"рек":2766,"рез":6938,"рді":3024,"реж":3159,"ржа":4954,"реч":1677,"рец":2506,"рвн":2312,"рга":5371,"ргу":2442,"рим":4841,"рин":4051,"рик":6384,"рил":1738,"рий":1712,"рич":3064,"рит":7476,"рир":1816,"рис":11068,"рка":1676,"пів":7244,"під":9422,"риб":1872,"риг":1977,"рив":2681,"риз":3342,"піл":3701,"піс":2425,"рмі":3148,"рни":5921,"рне":1575,"рна":4926,"рок":13067,"рол":3802,"ром":8841,"рон":6202,"роз":14973,"рні":3617,"ров":13779,"рог":3625,"род":18027,"роб":8445,"рно":5153,"рко":1653,"рма":4259,"пра":9765,"при":17085,"пре":5034,"про":26547,"ру ":4801,"поп":1576,"пор":8542,"пос":5211,"пот":2142,"пох":2319,"поч":2731,"пош":1607,"рт ":1813,"под":5278,"пов":10451,"пня":3871,"пон":3944,"пом":2310,"пол":10620,"пок":1979,"поз":3035,"пуб":2176,"пус":1543,"пря":1882,"са ":2954,"вар":4866,"ват":3699,"вач":1754,"ває":1673,"вав":2297,"ван":27620,"вал":6746,"важ":2723,"га ":2428,"бут":2028,"бул":3726,"бур":1929,"буд":3912,"був":3407,"́н ":1990,"вся":2929,"вто":3926,"втн":2139,"вст":3544,"всь":11822,"гу ":1846,"вро":2652,"вою":2829,"вої":4164,"вол":4005,"вні":6649,"вод":5394,"вог":4666,"вов":1600,"вня":4991,"вор":9030,"вос":3392,"вом":2439,"вон":1910,"вни":11918,"вне":1755,"вна":3546,"вно":8046,"влі":1867,"вля":2045,"вле":3353,"вла":2838,"го ":74789,"вка":1932,"вищ":4417,"вич":10270,"виз":2384,"вий":5063,"вил":1963,"вик":6641,"вин":5363,"вим":3289,"вип":2153,"вис":3214,"вир":3300,"вит":2461,"вих":6912,"вив":1786,"виг":1941,"вид":7090,"біл":7304,"вец":2207,"вер":12033,"ги ":1955,"вел":4069,"вед":23850,"ві ":8320,"вде":5262,"ва ":20184,"ают":4986,"баг":2186,"аці":38441,"ашо":3325,"аук":3811,"аті":2815,"афт":2014,"ахо":3481,"афі":2408,"ача":3308,"аче":5139,"ахі":3064,"апр":2675,"апа":1903,"апо":2274,"апи":1583,"арх":2803,"арс":2742,"арт":27902,"аре":1829,"ард":4062,"ара":11795,"арн":4172,"аро":9064,"ари":4536,"арк":3550,"аст":19420,"ата":3063,"аси":3444,"арі":4603,"асе":23473,"асл":1823,"асо":2485,"асн":7019,"ату":6561,"ать":1702,"ате":4441,"асі":1565,"ати":34490,"атк":2377,"атн":2400,"́ль":1996,"ато":8833,"атр":2250,"бол":1926,"бор":3014,"бни":2582,"бро":2133,"ву ":2725,"бра":3585,"блі":3564,"бла":6820,"бли":3503,"бле":1613,"во ":6738,"ви ":5236,"аєт":5513,"бер":8329,"без":3385,"аїн":17837,"ве ":2027,"дає":2865,"дач":1562,"дан":4800,"дар":3127,"дат":2935,"дал":2067,"дав":3568,"дем":2009,"ден":35742,"дер":7440,"деп":22255,"дже":27277,"ей ":5110,"дво":2066,"ез ":2842,"ді ":6167,"дсь":3872,"дст":2708,"дрі":2262,"дро":2499,"дру":2324,"дра":1728,"дпо":2072,"ет ":25079,"дко":1659,"ен ":10176,"ем ":4029,"див":2382,"гіч":3315,"дин":11940,"дит":3892,"гії":2091,"о́в":1799,"гіо":23749,"гір":2225,"гід":1618,"дня":2562,"доб":2151,"дні":5130,"дов":8472,"дос":25281,"дор":2173,"док":2841,"дон":2228,"дом":6507,"дна":4835,"дмі":2603,"дни":7067,"дне":1547,"дно":10257,"ер ":7534,"для":8246,"да ":6361,"газ":2730,"гал":4240,"гат":2475,"вят":1631,"ган":6922,"де ":6060,"вул":1831,"вує":1869,"вча":1606,"вче":1615,"гол":4270,"гос":1776,"гор":5348,"гов":2480,"год":1736,"гру":5182,"ду ":6755,"гро":3158,"гра":8609,"гре":2241,"гун":2442,"ген":3541,"ди ":6794,"гео":1703,"вів":2194,"вік":22506,"віл":2297,"вій":5760,"від":28365,"віт":10410,"вір":2465,"віс":2374,"він":3115,"гля":1806,"до ":11544,"жав":4567,"за ":31382,"жит":2833,"жив":2319,"жин":1953,"жі ":1618,"жен":31144,"зу ":2376,"жно":2243,"жни":2399,"жна":2353,"жні":1682,"жов":2516,"ежа":1636,"ежн":1674,"ежи":2240,"едс":1966,"еї ":5804,"еді":22755,"дія":9605,"діє":22354,"дії":2193,"діл":4733,"езн":2478,"езп":2124,"дів":4952,"дій":3323,"ева":2033,"еви":2106,"еат":1536,"дян":2535,"еда":2858,"еде":24252,"еди":1957,"егі":24559,"едо":1597,"едн":2110,"евн":1571,"же ":2405,"ево":2649,"еві":2180,"ент":38060,"енс":2476,"енц":1664,"енк":2033,"ени":9106,"ено":4866,"енн":55011,"ена":6246,"емі":4010,"ене":8942,"енд":1893,"еор":1718,"ені":27559,"ень":28255,"епа":22831,"ерш":4377,"ерх":5234,"ерп":2312,"ерс":4919,"ерт":3088,"ерм":2838,"ерн":8048,"еро":3883,"ери":8614,"ерк":2529,"ерд":2064,"ерг":2332,"ерж":5295,"ере":27737,"ера":11283,"ерв":4795,"ерб":1797,"ейс":2515,"еко":24801,"ект":7509,"екс":5740,"ели":5579,"ело":1860,"еле":29499,"ела":1663,"емл":1627,"емо":2244,"емн":1709,"еми":2190,"елі":3500,"ель":6771,"еме":2758,"ема":4374,"ехн":2533,"ець":7827,"еці":2099,"ері":9985,"есн":2797,"есп":2017,"есо":1640,"ест":4336,"ета":3510,"есі":2297,"ети":3023,"ете":1577,"етр":4098,"ето":3229,"ету":1803,"ива":5433,"иді":1970,"иго":1963,"ида":2853,"ивс":1559,"иво":3293,"ивн":5798,"иві":2780,"икл":3021,"ико":10700,"ики":26994,"ика":9412,"изь":1617,"изн":4148,"ині":5661,"имі":1780,"ини":11405,"инн":2229,"ино":4309,"инс":2389,"ину":2117,"ина":9917,"ими":8406,"илі":2149,"имо":2293,"има":1943,"иль":2368,"икі":3590,"или":2166,"ило":1638,"ила":2574,"исе":1748,"иса":1892,"ист":43720,"исо":2640,"исл":3795,"иск":1974,"ити":3061,"ите":2660,"ита":3840,"ися":2391,"ись":3153,"иту":23904,"итт":1716,"ито":5821,"ипа":23599,"ипн":2085,"ире":1798,"иро":6079,"ихо":1721,"ицт":1948,"иць":3344,"ить":5744,"ище":2694,"иці":4564,"ичи":1676,"ичн":15682,"иця":3075,"ича":2349,"ка ":31484,"ив ":2824,"зав":2331,"заб":1710,"заг":2313,"ид ":1647,"зви":3864,"зва":3561,"зац":3958,"зах":3661,"зас":5087,"зап":2437,"зан":2419,"зал":3333,"зак":3350,"ий ":76977,"зер":2249,"зем":2575,"зді":2313,"зі ":4174,"зик":1993,"ик ":11149,"ин ":8446,"им ":13202,"зич":2000,"зна":12042,"зни":3492,"змі":3801,"зно":1802,"зня":3181,"зов":3243,"зон":2606,"зпе":1731,"зпо":1609,"зро":1734,"зта":3312,"их ":71895,"ич ":8041,"зьк":25702,"ьме":1976,"ьна":3031,"ьни":9629,"ьно":31998,"ька":11941,"ьке":2474,"ьки":32360,"ько":60206,"ькі":5969,"ьту":1942,"ься":17960,"ьсь":5340,"ьог":3861,"ьов":1687,"ьні":3450,"ьпи":4442,"як ":5568,"ям ":4621,"ює ":1572,"юва":4241,"юр ":1687,"er ":1536,"яка":3149,"яки":6457,"яко":4501,"які":4711,"яч ":2564,"юто":2149,"ях ":2449,"ють":10792,"ючи":2094,"янс":3724,"ями":2547,"ять":2425,"яти":2263,"уча":3150,"уєт":3972,"фес":1886,"фер":1827,"уют":3197,"фун":1616,"фра":23347,"фор":6458,"фік":1922,"філ":3022,"фіз":1799,"це ":3714,"хан":1700,"хар":2307,"хні":3188,"хов":2787,"ход":8360,"хня":2286,"сто":20885,"стр":14612,"ств":9052,"сте":10268,"сти":58155,"ста":42984,"сті":17422,"стя":1612,"стю":2000,"сть":9349,"сту":4755,"сце":2414,"ть ":23970,"тю ":2076,"ськ":76644,"тя ":4033,"сьм":2215,"сюр":1594,"ув ":2334,"тав":4291,"так":7239,"тал":7016,"там":24076,"тан":13432,"тат":26660,"тах":1715,"тар":4879,"таш":3389,"тво":10485,"тва":6033,"тех":2451,"тец":1853,"тем":5996,"тел":3217,"тен":2843,"тер":16300,"теп":1791,"тет":24922,"тек":2381,"тей":1760,"ті ":19387,"січ":2428,"тив":7452,"сій":4342,"ук ":2255,"сів":2280,"сіб":22500,"тка":1831,"тич":9159,"сії":1716,"тий":2070,"тин":8122,"тик":26054,"тил":2041,"тир":1541,"тис":25636,"тип":1715,"тит":24505,"тку":2518,"тко":2210,"тла":1548,"ур ":1549,"тно":2899,"тні":2826,"тод":1678,"ток":3298,"тол":4612,"тов":10097,"тог":4926,"тня":4250,"тни":4639,"тре":1761,"тра":13299,"три":6247,"тор":19453,"тос":2194,"том":6059,"тон":3478,"топ":3418,"тою":1721,"тсь":3003,"тро":10751,"тру":4586,"трі":6027,"тув":1637,"туп":2434,"тур":8289,"ття":4038,"тут":22979,"тьс":17965,"ує ":4149,"ува":15273,"уго":1668,"уар":5114,"уат":1738,"убл":2373,"узь":23065,"узе":1844,"тій":2025,"узи":2433,"тіл":1694,"тів":6962,"уді":1635,"удо":3223,"удн":2817,"уме":1763,"уль":5593,"уля":1849,"ули":2646,"уло":1791,"ула":2199,"укт":2271,"укр":7476,"уко":2812,"упн":1546,"ура":2553,"ург":3415,"ури":2733,"упа":1808,"унк":2435,"уні":25084,"умо":1836,"унд":2657,"уту":22317,"уст":2645,"утв":1947,"урн":4525,"уро":1539,"що ":17655,"шов":4224,"ших":3225,"шир":2574,"ший":2090,"ще ":2338,"шен":2816,"ші ":1719,"шта":1833,"щин":2002,"щен":2787,"цен":4287,"чи ":3745,"цев":1886,"цес":2327,"цер":1781,"ці ":10080,"хід":5700,"цип":23018,"ць ":4743,"ця ":4768,"ча ":2185,"цуз":22494,"цтв":3628,"ців":2368,"ціа":3468,"ціо":25893,"цій":6921,"ція":7173,"чен":10697,"чер":4797,"чі ":2791,"чле":2375,"чка":1597,"чин":4805,"ціє":1783,"ції":31741,"чис":2289,"цьк":8704,"ша ":1587,"ше ":3944,"чає":2804,"час":12463,"чат":2049,"чай":1572,"ща ":1848,"чні":4741,"чна":4519,"чня":2280,"чни":34595,"чно":11306,"us ":1758,"ію ":3147,"їв ":1974,"іал":5781,"іан":1803,"ія ":24064,"іде":1874,"ідж":22706,"ідк":2690,"ідн":11513,"ії ":54256,"ідр":2425,"ідп":3775,"ідо":5503,"івд":5855,"іве":2535,"ібн":1843,"івс":5463,"івн":8627,"ійн":9751,"ійс":12117,"іжн":2207,"ізо":1615,"ізн":5205,"ізм":1906,"ізи":1715,"іза":4100,"інд":1760,"іне":2360,"іна":3489,"інн":4758,"іно":3712,"імі":1726,"інф":1596,"інц":2809,"інс":24263,"інт":1810,"іль":17941,"іме":3447,"іля":2402,"ілі":1831,"імп":2489,"іле":1779,"ікі":22043,"іло":3466,"іка":6429,"іки":2076,"іко":1702,"ісц":2429,"іст":27606,"ісл":1997,"існ":3890,"іре":3719,"ірн":2678,"іпе":22100,"іні":5468,"іод":1626,"інш":3831,"інь":1631,"іон":51386,"ітн":4596,"іто":2264,"їх ":2010,"ітт":1981,"іту":1943,"іта":3402,"іте":26865,"іти":4695,"ішн":1712,"іше":2070,"ічн":39482,"іці":2111,"іци":22860,"іб ":22595,"ів ":41252,"ід ":15090,"із ":3976,"іж ":3000,"ій ":20369,"ік ":2836,"ім ":3593,"ін ":5333,"ір ":2760,"іх ":2419,"єю ":25174,"єдн":2196,"єкт":1631,"єть":11566,"її ":1912,"ією":24943,"іяч":2557,"ївс":1781,"їнс":8303,"їни":6169},"n_words":[15331232,17151725,12469252],"name":"uk","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/ur.json b/contrib/languages-data/ur.json
new file mode 100644
index 0000000..62296aa
--- /dev/null
+++ b/contrib/languages-data/ur.json
@@ -0,0 +1 @@
+{"freq":{"ٹ":7565,"پ":19909,"ٰ":348,"و":91887,"ي":164757,"ً":561,"َ":343,"ُ":600,"ـ":316,"ف":15396,"ق":19382,"ك":314,"ل":61015,"م":85213,"ن":73500,"ه":210,"ّ":218,"ِ":1208,"خ":10520,"د":42999,"ج":29799,"ح":15399,"ت":63424,"ث":2772,"ب":46160,"ئ":12622,"ا":210246,"ؤ":741,"آ":6835,"ء":3457,"غ":3607,"ع":26594,"ظ":4173,"ط":9560,"ض":4853,"ص":11646,"ش":17043,"س":59068,"ز":13283,"ر":100269,"ذ":2393,"،":8584,"؛":310,"ہ":81948,"ۃ":194,"ۂ":442,"ے":72723,"ۓ":1226,"۔":21285,"گ":13942,"ک":96247,"ھ":18217,"ں":33709,"چ":6431,"ڈ":3486,"ڑ":2986,"ژ":212," ،":2410," ۔":1318," ہ":33487," ھ":395," گ":5518," ک":67994," ن":12353," ل":8072," م":43729," ق":5408," ك":206," ف":4429," ي":9396," و":11189," ص":3954," ش":8375," ط":3503," ض":1387," ر":7523," ذ":1057," س":22509," ز":3628," ع":9286," ظ":270," غ":946," ا":57235," ء":2548," آ":6281," ج":19940," ح":6228," خ":5235," د":11120," ب":20334," ت":17083," ث":374," چ":3475," ڈ":1133," ٹ":932," پ":14925,"کا ":9732,"کت ":188," ، ":2340," ء ":2420," و ":1367," بھ":3806," جن":2286," حا":1179," جل":205," بہ":901," جم":727," جي":597," جو":3440," جد":255," جس":2699," جز":358," جر":340," اے":201," اہ":629," جا":6670," جب":836," تي":855," اک":704," بڑ":974," اگ":491," خل":943," تہ":199," تھ":4319," خي":280," خو":966," دا":1316," خص":169," خر":184," خد":283," خط":402," حق":234," حي":633," تک":1112," حم":243," خا":1564," حر":509," حس":437," حد":214," حض":717," حص":647," بے":210," بن":2795," بل":1071," بغ":190," بع":1385," اُ":270," بر":1803," اي":8027," او":11360," بد":254," بح":524," اق":467," اف":961," ال":4248," با":2739," ان":6816," ام":1333," اط":314," اع":876," اد":879," ار":1330," از":241," اس":10716," اش":424," اص":842," اض":219," اب":1381," ات":313," اث":175," اج":297," اح":507," اخ":805," تو":1190," بچ":211," تن":378," تم":482," تق":681," تف":194," تع":1588," تش":261," تص":444," تر":1553," تخ":375," تج":367," تح":884," تا":1192," تب":411," اپ":1467," اٹ":312," بو":485," بي":1555," آپ":775," آل":254," آن":292," آي":279," آخ":168," آر":246," آت":368," آج":212," آئ":402," آب":1319," آف":180," آس":249," آز":331," عہ":219," سے":9513," شک":419," طو":1200," ظا":186," عظ":314," عر":1083," عد":298," عث":218," عا":1056," عب":533," شہ":1939," عي":293," عل":2998," عم":1153," عن":249," عو":213," غي":270," سع":194," سط":261," دھ":223," سف":224," سي":1366," سو":980," سم":787," دہ":248," شا":2109," سن":832," سل":1069," شر":734," شخ":260," شع":372," شي":299," شم":1497," صا":255," صر":324," رک":1032," صح":380," صد":878," صل":252," صف":254," صو":1376," رہ":1328," ضر":190," ضل":1004," طا":301," طب":638," سک":803," طر":1051," دس":404," در":2025," دي":2436," دو":2534," دن":789," دل":217," ذر":344," جگ":269," حک":746," ذي":205," جہ":593," را":903," جھ":293," رس":334," ري":940," زر":261," رق":405," رو":1485," زب":1015," زا":201," رن":190," زي":935," سر":1466," زم":612," سب":960," سا":2953," زن":311," ست":266,"گھر":200,"گہ ":210," ہے":16945,"گوں":240,"گيا":1207," ٹا":182," پڑ":329," پن":536," پو":662," پي":2142," پا":3033," پت":168," پر":5135," پش":170," پس":187," ٹي":317," لف":878,"ں، ":508," لغ":168," لح":265," لئ":237," لا":1034," مل":1468," مق":1520," مف":313," مغ":611," مع":1708," مط":921," مض":238," مص":513," مس":1572," مش":1902," مر":2119," مز":309," مد":642," مذ":261," مح":1355," مخ":914," لي":2681," مث":406," مج":498," لو":614," مت":1246," ما":1837," نف":215," نق":288," نم":568," نظ":923," نس":373," نش":178," مم":601," من":1456," نا":2714," نب":179," مو":1833," مي":17546," نج":183," وا":3957," لڑ":168," نو":730," ني":646," فض":168," فر":1011," فا":763," فت":168," قس":291," قص":318," فل":520," قا":1019," فن":193," فو":398," قب":691," في":493," قد":579," قر":855," فٹ":171," قل":240," قو":565," قي":355," وہ":1582," يہ":4254," لک":550," وج":661," لگ":411," وس":430," وز":353," ور":376," وغ":293," ول":235," وق":563," وف":272," مک":558," وي":428," لے":206," مگ":362," مہ":365," يا":2941," نک":275," نگ":248," يع":665," نہ":1220," يو":1100," نے":2696," گھ":362," ہو":7614," ہم":409," ہن":745," ہي":5912," ہز":243," ہر":384," ہا":430," ہج":240," کت":582," کر":5496," کس":1214," کش":187," کل":1235," کن":341," کم":719," کو":6421," کي":15836," کا":11009," کئ":348," کہ":5408," کھ":713," گا":490," گئ":863," کے":17634," کچ":321," گن":214," گل":202," گي":1388," گو":519," گر":807," ۔ ":1018," پہ":1353," پھ":659," چت":240," چا":498," چل":252," چي":492," چو":467," چن":303," ڈا":359," ڈي":297," چھ":653," چک":241,"کٹر":175,"کيے":174,"کي۔":282,"کلا":210,"کلو":610,"کلي":168,"کيا":2343,"کيو":230,"کيم":299,"کين":186,"کيل":348,"کمي":186,"کمل":193,"کنا":202,"کوئ":495,"کور":176,"کوم":779,"کرا":507,"کتے":245,"کسي":1209,"کست":1507,"کزي":242,"کري":262,"کرت":1247,"کرد":421,"کرن":1152,"کرک":265,"کار":1107,"کائ":219,"کئي":281,"کان":309,"کام":558,"کال":425,"کتي":199,"کثر":209,"کتا":937,"کم ":404,"کل ":646,"کن ":715,"کي ":12997,"کو ":5510,"کر ":2045,"کز ":207,"کس ":278,"گور":172,"گري":1479,"گرو":225,"گرد":220,"گرا":170,"گاہ":224,"گئے":444,"گار":222,"گئي":576,"گي ":628,"کہل":354,"کہت":572,"کہا":1480,"کھا":567,"کھت":425,"کھو":307,"کھن":402,"کھي":483,"گر ":1024,"گا ":209,"کے ":18223,"کچھ":310,"کھ ":350,"کہ ":4209,"ا، ":388,"ھا۔":1135,"ھائ":250,"ں۔ ":2401,"ھان":195,"ھار":432,"عہد":221,"ھتے":283,"پان":548,"پار":326,"پاس":169,"پائ":259,"ہ، ":321,"پي ":246,"ئي ":3847,"ات ":5026,"اح ":512,"اج ":434,"ئم ":385,"پرو":236,"پري":221,"ئل ":289,"اب ":1615,"اء ":706,"اؤ ":172,"پاک":1528,"پنج":401,"پنا":177,"ئع ":214,"پور":484,"پني":552,"ئش ":369,"ھي ":3674,"ئر ":279,"بت ":362,"فر ":258,"ان ":9264,"با ":254,"فظ ":915,"ہم ":759,"اً ":534,"ہو ":977,"پنے":725,"اف ":395,"پيش":361,"پيد":1076,"ام ":5824,"ال ":3321,"ہي ":1279,"اق ":313,"اظ ":454,"اع ":255,"ہائ":399,"ہات":210,"ہار":307,"ار ":4602,"اخ ":174,"فت ":347,"اد ":2459,"ہاں":875,"اص ":271,"از ":1002,"اس ":6443,"ہان":212,"ارے":550,"اطا":177,"ھوا":216,"اسک":1125,"ت، ":322,"بي ":1989,"اضي":270,"بو ":188,"اصط":318,"اصل":1371,"ارہ":558,"اسے":795,"اعت":501,"اعد":311,"اعر":317,"ھوٹ":341,"ھيل":511,"اطي":187,"ھنے":355,"اعظ":290,"ھوں":204,"اعل":259,"قع ":1257,"فار":420,"ھيں":343,"فات":329,"فاظ":273,"ادا":654,"اخل":199,"اتھ":922,"ھي۔":577,"احي":174,"اخت":714,"احم":281,"احت":209,"ئي۔":457,"ہا ":1654,"ارا":821,"ادي":1677,"ئيں":363,"ادل":266,"ادب":182,"ادت":200,"بق ":617,"ادر":214,"اتے":523,"ازي":292,"است":1978,"اري":1870,"ارن":318,"ارو":652,"ہت ":499,"ارف":181,"ارس":382,"بل ":520,"ارد":993,"ارت":911,"ارک":342,"اشي":236,"اسم":259,"ادہ":769,"اسل":804,"اشا":172,"اسي":947,"ارٹ":196,"بن ":873,"بع ":355,"ائد":205,"ائر":395,"ائش":451,"ائع":219,"ائن":582,"ائم":404,"ائل":355,"ائي":3099,"ئنس":292,"ابت":325,"ابر":254,"اؤں":202,"ہد ":227,"ابي":488,"ابل":415,"ابق":678,"ابو":358,"ابن":262,"اتح":175,"اتا":3152,"ہر ":2189,"اثر":224,"ؤں ":253,"اتي":1903,"قت ":631,"ائے":1094,"اجا":187,"بر ":849,"في ":604,"عظي":276,"عظم":356,"پڑھ":180,"عري":274,"عرو":255,"عرب":766,"عرا":186,"عدا":424,"عدد":310,"عثم":220,"ظيم":431,"عبد":345,"عات":242,"شہو":678,"شہر":1734,"عال":536,"عام":681,"عاش":251,"عار":177,"صے ":201,"پہن":208,"پہل":895,"غرب":546,"طہ ":206,"عيا":204,"عيس":213,"پھر":309,"پھي":228,"عمل":531,"عمو":341,"عمي":283,"عمر":256,"غان":180,"عني":881,"غاز":188,"علا":1570,"علق":602,"علي":1229,"علو":368,"عما":1154,"علم":738,"آتا":178,"آئي":238,"آبا":1188,"آخر":190,"آزا":306,"ھر ":618,"آن ":225,"ھا ":1233,"غير":796,"آيا":202,"عے ":290,"آپ ":665,"عہ ":760,"جسم":317,"جزي":258,"خت ":294,"جرا":223,"جري":262,"شعب":197,"جزا":198,"جرم":182,"بکہ":367,"ثيت":188,"ذہب":185,"شما":1408,"جبک":363,"اہم":638,"جائ":411,"اہل":181,"جات":4030,"اہو":304,"جاب":370,"جاس":338,"اہي":375,"جار":388,"جان":1341,"جام":298,"رے ":1358,"بہت":604,"رکز":477,"بہا":219,"صبہ":185,"جما":288,"رکا":241,"صحا":176,"رکي":303,"صدر":477,"حال":384,"جنو":753,"ہے۔":9056,"صدي":342,"حاد":180,"حاص":632,"حاظ":206,"جمع":225,"جمو":257,"شيا":452,"خط ":169,"بھي":3076,"بھا":585,"ضي ":277,"صطل":323,"جسے":332,"رکھ":862,"صرف":317,"جسک":274,"زہ ":268,"طب ":280,"تک ":956,"خي ":172,"حرک":215,"حضر":635,"حصي":251,"دت ":251,"حدہ":191,"سٹي":260,"حسا":181,"دا ":1143,"حسن":171,"سٹر":203,"حرا":239,"حري":437,"طح ":229,"تہ ":472,"تھ ":814,"صوص":325,"ہے،":923,"صول":176,"حدي":195,"صور":542,"صوب":1118,"رہي":215,"رہن":216,"جمہ":270,"رہا":394,"جنگ":528,"جيس":335,"رہت":193,"صلا":189,"جود":775,"ضرت":619,"ضرو":182,"خان":852,"خاص":306,"خار":176,"حمد":801,"حقي":267,"تے ":3295,"رہے":389,"صيل":310,"دس ":176,"در ":1119,"دد ":492,"حصہ":359,"دن ":286,"ٹا ":239,"دو ":1561,"ظر ":265,"خصو":323,"ر، ":449,"دي ":2904,"حير":216,"ضلع":1009,"حيا":329,"ختل":505,"حيث":191,"ختي":341,"سے ":11230,"طان":608,"طال":493,"طاب":536,"طبي":334,"دل ":339,"دم ":215,"خلي":574,"سکت":863,"تہا":216,"سکا":323,"سکي":300,"خلا":443,"سکو":402,"دان":840,"دال":423,"تھے":1383,"دائ":580,"دار":1737,"داز":181,"داد":564,"طرح":391,"طرز":173,"طرف":300,"خوا":291,"خود":282,"طري":194,"ظم ":393,"شہ ":199,"سکے":358,"خيا":180,"عت ":511,"تھي":1189,"عد ":1126,"تھا":1925,"رج ":392,"ٹي ":724,"رت ":2032,"رد ":384,"طلب":258,"طلا":420,"عض ":255,"رح ":408,"طور":1195,"ظام":601,"طنت":356,"شکل":258,"را ":761,"دست":238,"رب ":643,"ظاہ":223,"درس":185,"درج":475,"درا":374,"دري":711,"جہ ":949,"درم":359,"درو":190,"دون":230,"ظري":241,"صہ ":469,"دور":724,"دوس":950,"دني":624,"دوں":249,"رف ":993,"ديو":334,"ديم":368,"دين":825,"ديل":228,"ديا":819,"ديت":278,"ديد":212,"ٹر ":1167,"رس ":290,"عي ":320,"دما":182,"جے ":273,"اقو":437,"اقي":177,"اقت":219,"اقا":176,"افي":307,"اقع":1323,"الق":182,"الف":464,"الل":695,"الي":1802,"ان،":219,"امت":248,"اما":469,"الن":229,"الم":986,"الو":429,"امر":725,"الج":351,"الت":351,"الب":611,"الا":1303,"الس":179,"الر":180,"الد":643,"الح":530,"الع":407,"ري ":3579,"جگہ":221,"افر":440,"افت":417,"ٹري":255,"انے":1223,"ايم":221,"انہ":1162,"ايو":177,"ايس":1069,"انگ":1569,"ايش":229,"امہ":269,"ايا":902,"ايت":316,"انک":290,"اير":272,"، ":8315,"بحي":221,"اوہ":305,"رو ":291,"بحر":243,"اقے":283,"ديگ":275,"اني":2788,"انو":1426,"ديک":357,"رق ":406,"باً":193,"اقہ":415,"بان":1448,"بال":507,"اند":1077,"باد":1658,"باز":180,"بار":1042,"انس":905,"امن":202,"انا":720,"انب":309,"امو":288,"انت":736,"بات":454,"امي":1316,"انج":263,"امل":979,"بائ":435,"الے":826,"بتد":219,"اوي":243,"انڈ":193,"انچ":276,"الہ":394,"اون":198,"اول":497,"اوق":178,"اور":10590,"الک":535,"رم ":316,"ذري":244,"اوا":211,"تا ":6076,"حکو":764,"حکم":277,"رحد":298,"ردو":947,"ردي":295,"ردا":370,"رتے":586,"رتا":498,"ايک":5551,"بدا":306,"راچ":302,"ربع":318,"بدي":235,"برا":685,"ربي":855,"ذيل":186,"راہ":513,"برط":276,"برق":294,"بري":212,"رتي":676,"راک":232,"جہا":530,"تر ":480,"ران":1556,"ربا":206,"راع":230,"راف":173,"رام":330,"رال":643,"راب":283,"رائ":562,"راج":355,"رات":605,"راث":218,"رار":318,"راد":611,"راص":190,"راس":245,"ہيں":6226,"بعد":1053,"تح ":170,"ہو۔":237,"جھي":195,"بعض":255,"رآن":223,"بني":578,"بنا":831,"ہوگ":285,"ہوں":722,"بند":385,"ا ":36261,"بلن":294,"ہنچ":177,"بلو":291,"بلي":191,"ہلے":442,"سر ":306,"بلا":185,"ب ":6862,"ء ":3203,"ہوئ":1384,"ہور":1098,"ہوت":2264,"ہوا":926,"ہون":793,"ہلي":355,"ہند":711,"ہمي":171,"ؤ ":175,"زي ":1743,"رطا":394,"ح ":1752,"ھے ":736,"ہلا":565,"خ ":788,"رسٹ":177,"د ":10666,"بوں":222,"بيٹ":246,"ست ":854,"ذ ":232,"بين":440,"بيل":268,"سا ":510,"رست":341,"بيع":188,"ت ":16193,"بيا":321,"بير":253,"رسي":336,"سب ":865,"رجہ":199,"ث ":408,"بول":436,"ج ":1790,"بلک":176,"سم ":677,"دہ ":2099,"تي ":4132,"سل ":210,"اک ":371,"ريہ":436,"تو ":819,"ٹلي":171,"ريک":876,"دھ ":232,"روں":847,"رپ ":187,"ريع":323,"رين":690,"ريل":294,"ريف":299,"ريق":376,"رنے":958,"تم ":196,"ريا":1581,"ريب":696,"ريخ":486,"ريت":171,"ريز":1341,"ريش":173,"رنگ":273,"روف":378,"رون":314,"روم":315,"روع":324,"رور":309,"روز":221,"روس":205,"روا":481,"اپن":1352,"ہزا":285,"ہري":293,"زبا":959,"زار":465,"رند":247,"زاد":421,"ہرا":229,"رمي":582,"رمن":177,"رنا":383,"زائ":318,"رما":229,"اٹل":178,"ہتے":651,"ثر ":284,"؛ ":306,"سط ":176,"رقي":620,"رقب":303,"ہجر":225,"ئے ":2320,"تبد":215,"تان":2396,"تبا":381,"تار":774,"تاب":521,"اچي":318,"دے ":517,"سن ":294,"ہے ":7219,"اں ":1697,"سو ":217,"تحر":360,"شت ":170,"تحا":210,"تحص":253,"سي ":3401,"تدا":329,"اہ ":1042,"تري":474,"ترا":469,"جا ":352,"تصا":242,"جب ":430,"تصو":215,"ترک":411,"اڑي":213,"ستہ":193,"تظا":196,"سري":335,"تعا":188,"سرح":298,"سرا":421,"تعم":1291,"تعل":965,"تعد":362,"دگي":389,"جد ":384,"شن ":295,"زما":353,"سام":409,"سال":748,"سان":757,"زند":314,"جس ":1740,"سائ":617,"زمي":343,"سات":868,"ساب":328,"ستا":2494,"تقا":197,"سجد":245,"تقر":347,"تقس":240,"ستع":1050,"زيا":694,"ستي":192,"ستو":208,"زيد":178,"اے ":199,"زير":703,"تمل":297,"ا۔ ":2384,"صر ":373,"تلف":484,"تما":488,"اۓ ":304,"سلا":945,"تين":302,"سمج":250,"سلي":236,"سمب":188,"سلم":718,"سما":225,"تيس":201,"تيا":780,"سلط":553,"سلس":275,"توا":248,"شي ":269,"سطح":252,"تيں":193,"پر ":4411,"اکي":196,"اکا":303,"اکس":1424,"سرے":373,"توں":404,"صد ":229,"ٹے ":294,"اگر":397,"جن ":581,"بڑا":364,"رک ":436,"بہ ":1879,"اکھ":210,"رہ ":1976,"جي ":406,"شرق":569,"شرو":345,"شري":335,"صل ":1226,"جو ":2871,"بڑي":256,"حت ":262,"ئے۔":600,"بڑے":246,"حد ":400,"ھے۔":835,"سوي":178,"شتر":182,"سوا":232,"سور":248,"شاع":346,"شام":808,"سمن":228,"شائ":191,"سند":323,"شاخ":186,"سمي":191,"ثلا":193,"شخص":263,"ثما":224,"ذکر":233,"اہر":591,"اہد":216,"سين":345,"سيم":308,"شاہ":679,"شتم":303,"سيد":218,"بے ":441,"سيا":755,"ڑا ":500,"ٹھ":533,"پت":249,"پا":3394,"پس":366,"پش":179,"پر":5467,"ٹے":304,"گ ":1702,"ٹک":243,"پڑ":366,"پو":978,"پن":2069,"پي":2662,"پل":260,"ٹي":1294,"ٹو":369,"ٹن":221,"ٹل":297,"ٹر":1769,"ٹا":643,"ک ":9401,"لے":2164,"يں،":396,"مگ":395,"مک":804,"نڈ":840,"وي":2835,"ي،":686,"نچ":605,"وو":294,"يب":1474,"يا":18357,"مہ":1477,"يئ":237,"يع":1465,"يز":2169,"يس":3006,"نگ":3438,"يش":1424,"يص":226,"يخ":630,"يد":2958,"ير":4679,"يت":2515,"يث":342,"يج":682,"نک":1176,"يح":396,"ين":5944,"يو":3744,"وچ":363,"نھ":192,"يق":966,"يم":3458,"نہ":3293,"يل":3575,"يف":992,"وپ":411,"وٹ":898,"وڑ":349,"نے":8036,"وڈ":316,"يٹ":1722,"وک":763,"پ ":1225,"يٰ":198,"وگ":857,"وہ":2577,"يچ":216,"يڈ":589,"وں":7039,"يپ":229,"و۔":276,"يک":8185,"يگ":581,"يہ":6841,"يں":24637,"ي۔":1652,"يۓ":606,"يے":1374,"فع":234,"فض":206,"فظ":975,"فر":2018,"فس":304,"فت":956,"فا":1934,"يہا":454,"قع":1462,"قط":251,"قص":525,"قس":674,"قر":1456,"قد":958,"قت":918,"في":1664,"قب":1358,"فو":716,"قا":2876,"فن":259,"فل":626,"فق":170,"فٹ":222,"قي":2110,"ل،":228,"قل":641,"قو":1321,"لق":994,"لف":1951,"لط":721,"يں۔":3302,"لغ":238,"لع":1546,"لد":859,"لج":430,"لح":953,"لز":212,"لس":945,"لر":265,"لئ":340,"لا":9180,"لت":997,"لب":1083,"مع":2282,"مغ":639,"مص":529,"مض":257,"مط":1001,"مف":327,"مق":1607,"مل":3799,"مت":2835,"لو":3381,"مج":828,"لي":9899,"ن،":378,"مث":441,"لم":3135,"لل":728,"مب":937,"لن":899,"ما":9656,"مز":517,"مر":3546,"مش":2018,"مس":1772,"مخ":947,"مح":1397,"مذ":278,"مد":1699,"نظ":1248,"نع":313,"نل":169,"قہ":970,"نم":856,"نق":463,"نف":515,"ے، ":1325,"نج":1093,"مي":23019,"نت":1838,"مو":3519,"نب":733,"نا":7139,"من":2588,"فہ":638,"مم":631,"نص":409,"نش":418,"نس":2278,"نز":337,"نر":275,"ند":4576,"قے":375,"مپ":203,"وئ":2114,"وا":9079,"نن":295,"نو":3979,"ني":8247,"و،":187,"ٹ ":1461,"لڑ":176,"وغ":354,"نٹ":588,"وع":906,"وق":1147,"وف":901,"ون":4062,"ول":2928,"وم":3387,"لہ":2193,"وت":3089,"وب":2645,"ود":2057,"لک":2192,"وح":206,"وج":2206,"لگ":620,"وس":2774,"وز":895,"ور":18253,"وط":241,"وض":250,"وص":445,"وش":613,"يکھ":326,"ڑ ":321,"يکن":476,"يکي":320,"يگر":286,"چ ":481,"يکہ":301,"ڈ ":843,"خو":1180,"دت":297,"دا":6455,"دب":338,"خي":646,"خل":1333,"خم":176,"تہ":894,"تھ":5631,"خط":526,"خر":474,"خد":295,"خص":603,"دو":4578,"ر،":467,"دي":7480,"دف":241,"دل":705,"دم":634,"ذا":327,"دن":1041,"تے":3362,"دع":168,"دد":552,"در":3958,"دش":266,"دس":695,"جي":1296,"جو":4658,"حت":458,"جن":2503,"حا":2227,"حب":298,"جل":349,"بہ":2825,"جم":1355,"بھ":4084,"ا۔":3031,"اۓ":335,"اے":213,"جس":2866,"جز":533,"جر":1045,"جد":723,"بک":601,"خت":1742,"حي":1318,"تک":1171,"حم":1392,"خا":1967,"خب":324,"حو":294,"حق":534,"حل":497,"حض":792,"حص":974,"بے":458,"حر":1330,"حس":621,"حد":976,"تف":294,"تم":1316,"تل":864,"تق":1058,"تو":2257,"بچ":213,"ثا":431,"تن":759,"تج":398,"تح":1528,"تر":2642,"تخ":631,"تد":422,"اڑ":396,"تش":356,"تص":699,"تس":246,"تظ":210,"تع":3085,"اں":1731,"ثل":246,"جا":8260,"ثم":243,"اہ":3738,"جب":952,"ثي":412,"تي":6367,"اک":3503,"ثر":502,"اگ":766,"بڑ":994,"ئے":3028,"ؤں":259,"بغ":203,"بع":1847,"بن":3272,"بم":183,"بل":2052,"بق":773,"بد":925,"اً":548,"بج":198,"بح":566,"بت":774,"اي":9647,"او":12882,"بط":281,"بص":181,"بز":199,"بس":224,"اُ":281,"بر":3415,"اپ":1892,"اٹ":526,"تا":10999,"اچ":510,"تب":1015,"بو":1546,"ت،":328,"بي":4832,"ئد":211,"ئر":536,"ئش":454,"ا،":402,"اء":804,"اؤ":588,"ئل":407,"ائ":7576,"ئم":430,"ئع":221,"از":2184,"ار":13604,"اد":7175,"اض":766,"اص":2408,"اش":1157,"اس":13319,"ات":12391,"اب":5157,"ئن":668,"اخ":1510,"اح":1946,"اج":1405,"اث":517,"ئي":5602,"اف":2468,"اق":3776,"ام":10852,"با":7594,"ان":23472,"ال":15347,"اع":2513,"اغ":318,"اط":844,"اظ":619,"آپ":794,"آئ":413,"آب":1377,"آت":374,"آج":214,"آخ":195,"آر":259,"آس":262,"آز":333,"آف":182,"آل":275,"آم":177,"آن":518,"آي":281,"عے":301,"ٰ ":195,"عہ":1012,"طہ":234,"غي":910,"غل":246,"عي":1223,"غر":737,"صے":202,"عق":239,"عل":4649,"صہ":487,"عم":2601,"غا":645,"عن":1246,"عو":588,"عث":370,"ظي":492,"عت":818,"عد":2145,"عز":175,"عر":1942,"عظ":634,"عض":358,"عا":2408,"عب":841,"ظم":515,"شہ":2846,"ظا":851,"طن":541,"سہ":287,"طل":743,"سے":11317,"ظر":549,"شک":686,"طي":706,"طو":1493,"ضم":182,"زہ":344,"ضل":1190,"رے":1402,"طر":1375,"ضي":496,"سک":2597,"طح":262,"طا":2021,"طب":814,"ضو":288,"سپ":375,"صل":1848,"صف":401,"صط":357,"سٹ":824,"ضر":887,"صو":2504,"رہ":3575,"صن":434,"ضا":582,"صي":766,"دے":538,"شع":464,"رک":3328,"صح":404,"صد":1147,"صر":962,"رگ":411,"شم":1789,"ذہ":258,"صا":794,"شن":774,"صب":345,"شو":358,"شي":1319,"سع":252,"سط":727,"دگ":457,"سف":448,"دھ":774,"رپ":455,"رٹ":425,"رڈ":213,"سي":6156,"شت":955,"رچ":305,"سو":1823,"شا":3056,"سن":1275,"دہ":2389,"سم":1996,"سل":3485,"شر":1691,"شد":243,"شخ":304,"ذک":274,"سب":1244,"سا":5309,"زن":603,"ست":5770,"زو":460,"زم":950,"زل":234,"سر":2705,"سج":282,"زي":3808,"دک":188,"رس":1755,"رش":298,"رر":201,"رز":482,"جے":273,"رط":461,"رص":252,"رض":357,"رل":427,"رق":1634,"رف":1286,"رو":5304,"زب":1100,"رن":2342,"زا":1886,"رم":1656,"ري":12753,"زر":534,"ذر":554,"جگ":285,"رآ":320,"جھ":599,"جہ":1619,"رب":2493,"را":9267,"رت":4343,"رج":1190,"ذي":412,"رخ":287,"رح":1014,"حک":1119,"رد":2466,"ف ":3475,"ع ":4219,"ڑي ":543,"غ ":230,"ص ":761,"ض ":697,"ط ":839,"ظ ":1429,"ر ":38086,"ز ":2573,"س ":11232,"ش ":1550,"ِ ":847,"ً ":542,"ي ":57056,"ن ":18794,"و ":15262,"ق ":2735,"م ":14225,"ل ":13265,"ينہ":361,"ينے":248,"يوں":1103,"وہ ":2016,"يقي":265,"يلا":274,"يلي":664,"يما":422,"يلو":184,"ينا":221,"نہي":1033,"يمي":391,"نہو":438,"يٹ ":181,"يني":579,"يقہ":243,"وچس":192,"يور":512,"يوا":288,"يون":715,"ينڈ":257,"وں ":6861,"ہے":17450,"يٰ ":189,"وٹي":197,"يعے":209,"وگ ":226,"يرہ":738,"يشي":298,"يشن":306,"يسي":670,"نگي":216,"نگل":243,"يسو":203,"يسر":205,"ے،":1342,"نگر":1405,"يزي":1326,"نگا":269,"يسا":524,"يرو":314,"يري":407,"يعن":649,"يسے":586,"۔ا":238,"يثي":187,"يا۔":1172,"نکا":269,"ے۔":10861,"يتي":187,"ياں":353,"يرا":565,"نکہ":250,"يدا":1270,"نے ":7909,"يال":283,"يان":856,"يبا":352,"يام":192,"ياس":748,"يار":881,"ياد":1222,"ياض":232,"ياء":241,"يات":2312,"ياب":173,"يائ":508,"کچ":345,"کٹ":398,"کي":17718,"ي۔ ":1247,"کس":3186,"کش":375,"کر":6688,"کز":505,"کث":328,"کت":1905,"کو":8220,"کن":1311,"کم":1336,"کل":1933,"کئ":359,"کب":357,"کا":13105,"يۓ ":585,"يے ":1282,"گہ":320,"گھ":468,"گل":520,"گن":309,"گو":1092,"گي":2325,"گز":223,"گر":3536,"گئ":1042,"گا":1385,"کے":18417,"کھ":2829,"کہ":6908,"ں،":524,"گے":188,"يہ ":6011,"يڈي":227,"ھے":1744,"ہز":291,"ہر":3373,"ہج":298,"ہت":1501,"ہد":644,"ہا":4675,"ہب":255,"ہي":8193,"ہل":1738,"ہم":1207,"ہن":1510,"ہو":9694,"ں۔":3376,"ھم":206,"ھل":258,"ھي":5556,"ھو":1393,"ھن":692,"ہ،":328,"ھر":965,"يں ":20810,"ھا":3850,"ھت":572,"ں ":29650,"ڈر":227,"چي":975,"چن":411,"چو":627,"وہا":172,"ڈا":533,"چک":371,"چہ":230,"چھ":1116,"ڈو":318,"ڈي":856,"پھ":684,"پہ":1407,"چس":237,"چا":849,"چت":269,"چل":282,"ڑا":619,"ۃ ":184,"ۂ ":441,"ڑے ":367,"يٹر":982,"ڑي":670,"يک ":5968,"ھ ":2275,"چے":173,"ہ ":29676,"وگو":199,"ڑک":206,"ڑھ":467,"ڑے":384,"ے ":59880,"ۓ ":1150,"۔ ":15851,"و۔ ":200,"وز ":174,"ور ":14043,"ود ":1161,"ڈي ":274,"لگ ":206,"وس ":423,"چين":228,"نما":532,"وع ":503,"نيا":1482,"نوي":380,"نون":235,"نور":347,"نوب":664,"نوا":373,"نٹ ":234,"وف ":378,"نيہ":422,"نيو":458,"نوں":1111,"وم ":918,"لہ ":1961,"ون ":1017,"چند":174,"ول ":1088,"نڈ ":285,"وي ":1068,"ي، ":669,"نچ ":177,"مغر":519,"معل":267,"معن":336,"معر":258,"مشہ":684,"معا":596,"چست":201,"مقا":854,"مقد":219,"چان":177,"ملت":253,"ملا":357,"چتر":233,"قے ":368,"منا":208,"نائ":404,"مند":339,"نات":285,"منت":213,"نار":304,"مما":299,"ملي":227,"موا":174,"ملک":619,"موج":660,"مور":225,"موس":304,"موع":240,"نام":2044,"نان":513,"ناي":217,"نتق":217,"نتظ":204,"نتخ":209,"مون":289,"موم":337,"مول":250,"ميل":269,"ميد":233,"مير":641,"ميا":812,"ميت":203,"نتي":206,"نجا":562,"مين":762,"ميٹ":952,"موں":292,"ندا":677,"ميں":16302,"ندو":448,"ندر":713,"ندي":569,"چار":237,"نسا":448,"چي ":376,"وا ":736,"ندگ":273,"ندہ":249,"نسي":397,"نسل":289,"ندھ":311,"وب ":577,"وت ":377,"نظا":416,"نظر":493,"وج ":244,"نظي":172,"لک ":878,"ونا":319,"ومي":882,"ونس":207,"وما":454,"ولي":616,"ولو":168,"ومت":782,"يع ":214,"ولا":396,"وقت":472,"وفا":282,"يش ":423,"يس ":618,"نگ ":881,"ويں":364,"يق ":261,"يف ":611,"مکم":188,"ونے":813,"نڈي":216,"چہ ":203,"ونک":263,"چھ ":368,"وني":567,"وٹ ":170,"ونو":241,"وڈ ":203,"يو ":296,"نہ ":1535,"يم ":2131,"ين ":3424,"مگر":341,"يل ":1806,"لے ":2120,"وئے":873,"واں":205,"وتي":759,"وتا":1184,"وبہ":880,"وجي":169,"وجو":858,"وا۔":228,"وار":699,"واز":279,"واد":234,"واج":227,"وئي":1076,"واب":205,"وائ":356,"وبي":420,"واي":171,"واق":1340,"وال":2546,"وان":598,"وبا":263,"وام":369,"وري":875,"مہ ":911,"وست":275,"لگا":240,"وزي":317,"يا ":8493,"وسر":713,"وسط":295,"ورپ":286,"ودہ":176,"وسي":366,"يب ":717,"ورہ":195,"لکي":197,"وتے":500,"ودي":279,"ورا":750,"وجہ":615,"ورس":204,"ورت":506,"لکہ":196,"ورن":218,"لکھ":453,"وغي":292,"يد ":1164,"ير ":2092,"يز ":500,"يت ":1838,"يج ":176,"يح ":219,"يخ ":423,"لد ":289,"قصب":216,"لت ":358,"لا ":1251,"قسم":285,"لب ":395,"قسي":239,"�":423,"لق ":599,"لف ":570,"چھو":396,"قوں":260,"ما ":498,"لم ":1252,"لع ":962,"قيق":283,"قيا":359,"قوم":305,"قوا":311,"قل ":209,"فرا":633,"فرو":181,"فري":361,"قي ":950,"ل، ":225,"فيص":180,"قبہ":284,"قدر":190,"قدي":366,"قري":696,"قرآ":222,"قرا":218,"قال":197,"قائ":507,"قاب":316,"قات":245,"فلم":218,"فوج":255,"قبو":176,"قبل":223,"قان":205,"قبا":188,"قام":703,"فٹ ":202,"لما":635,"ماع":315,"لمي":331,"مات":656,"مار":1346,"ماد":258,"لند":373,"مائ":379,"لفظ":904,"لفا":303,"نس ":426,"لعہ":239,"ند ":917,"لطا":204,"مي ":1920,"لطن":355,"مطل":234,"ني ":4359,"و، ":185,"مطا":719,"مصن":197,"مصر":194,"مرک":583,"نو ":281,"مذہ":183,"مشر":609,"مسل":679,"مشت":387,"مست":254,"مسج":242,"مسا":197,"قہ ":928,"مري":646,"مجھ":254,"مرا":743,"مرب":331,"مرت":186,"مدي":169,"ليے":1014,"ليۓ":470,"مدد":185,"ليک":538,"ليہ":571,"مخت":598,"لوگ":423,"محم":603,"لوں":448,"لوچ":226,"لين":425,"مجم":256,"لنے":191,"ليت":222,"متي":175,"ليا":836,"ماہ":260,"ليم":656,"مثل":227,"لوي":317,"لوم":894,"للہ":672,"متح":211,"متع":399,"مال":2414,"مام":560,"مان":1742,"مبا":175,"ماي":231,"مبر":417,"مر ":315,"مد ":873,"لو ":198,"ے۔ ":8070,"مت ":1234,"لي ":3595,"ن، ":362,"لسل":386,"نب ":260,"نا ":1656,"من ":398,"فہ ":313,"نت ":572,"لدي":343,"لحک":303,"مل ":1925,"لتي":176,"لاک":279,"لئے":321,"لاہ":228,"لتا":182,"لحا":247,"لاح":555,"لاد":170,"لاز":192,"لائ":397,"لات":751,"لاق":1241,"لاف":367,"مع ":244,"لاو":376,"لام":1294,"لان":427,"لبا":312},"n_words":[1602570,1999510,1324903],"name":"ur","type":"arab"} \ No newline at end of file
diff --git a/contrib/languages-data/vi.json b/contrib/languages-data/vi.json
new file mode 100644
index 0000000..d3a4d75
--- /dev/null
+++ b/contrib/languages-data/vi.json
@@ -0,0 +1 @@
+{"freq":{"D":18934,"E":10094,"F":9985,"G":23207,"A":42579,"B":48257,"C":72224,"L":45665,"M":47574,"N":70917,"O":10000,"H":57302,"I":15743,"J":5371,"K":22108,"U":5400,"T":104616,"W":8071,"V":30021,"Q":14744,"P":61692,"S":43824,"R":17604,"Y":3509,"X":6207,"Z":3364,"f":32992,"g":525146,"d":205922,"e":428748,"b":182777,"c":671453,"a":692878,"n":1382200,"o":437498,"l":442210,"m":420259,"j":4762,"k":100510,"h":937660,"i":759263,"w":22119,"v":178461,"u":410719,"t":952446,"s":286409,"r":444790,"q":33677,"p":174320,"z":16996,"y":192811,"x":29082,"²":689,"Î":330,"É":501,"Á":3556,"Â":2373,"Ý":1288,"ß":648,"Ú":1274,"Ô":1751,"í":49339,"ì":32480,"ê":66073,"é":38397,"è":3862,"ç":233,"ä":808,"ã":19808,"â":112923,"á":116051,"à":337783,"ü":3187,"ý":5197,"ú":10002,"ù":30808,"ö":2319,"ô":98885,"õ":1117,"ò":13544,"ó":78081,"ñ":390,"đ":198318,"Đ":41649,"ă":37793,"ā":557,"ĩ":9562,"ī":249,"ō":772,"ũ":5403,"ū":650,"ư":147894,"ơ":25607,"́":204,"ο":236,"ι":146,"λ":139,"α":217,"ς":203,"ρ":170," l":242575,"ь":159," m":173509," n":244497,"я":194," o":12009," h":134731," i":13655," j":644,"ы":142," k":76050," d":93354," e":7007," f":8319," g":49741,"ч":190,"р":638,"с":522," a":22488," b":142456,"т":441,"у":252," c":262807," y":1958," x":17955," z":365," u":1259," t":566783," w":4693," v":156401," q":28586," p":59115," s":94989," r":14869," J":5319," K":22031," H":57162," I":15633," N":70771," O":9935," L":45508," M":47408," B":48077," C":71972," A":42454,"С":156," F":9904," G":23091," D":18825," E":10050,"л":506,"к":551," Z":3238,"й":211," Y":3492,"и":885," X":6141,"о":889,"н":659,"м":235,"г":168," S":43565,"в":508," R":17493," Q":14713,"б":149,"а":1192," P":61533," W":8016," V":29925," U":5377,"е":792,"д":263," T":104253," á":1499," â":3662," í":541," ô":2220," ý":503," ă":627," Đ":41607," đ":198156," Â":2373," Á":3556," É":500," Î":330," Ô":1748," Ú":1274," Ý":1277," ư":1144,"ي":271,"ل":264,"م":187,"ن":142,"ا":398,"ر":178,"A ":2159,"F ":494,"Da":2569,"Cu":1591,"Cy":963,"Cl":1245,"Co":10944,"Cr":1746,"Ce":2576,"Ch":19039,"Ci":1005,"G ":637,"Ec":908,"Ed":361,"Ea":530,"Du":1267,"Dy":152,"Do":2470,"Dr":1247,"De":2217,"Di":2894,"Bà":792,"Bá":858,"Fe":902,"H ":455,"Fa":1831,"Eu":2126,"Ex":464,"Er":617,"Et":367,"Es":894,"En":771,"Em":460,"Ep":314,"Ei":335,"El":872,"Cá":2732,"Ge":2583,"Câ":290,"Cà":157,"Ga":3052,"I ":2174,"Fu":749,"Fr":2327,"Bí":387,"Bì":1773,"Fo":1217,"Bé":191,"Fl":1104,"Fi":1024,"B ":642," С":156,"C ":1227,"Av":428,"Au":2997,"Ar":5387,"Aq":1658,"At":1067,"As":1434,"D ":745,"Ba":13610,"Az":1190,"Ay":139,"Ae":265,"Af":962,"Ag":750,"Ah":192,"Ab":577,"Ac":1193,"Ad":611,"Am":2029,"An":10483,"Ap":626,"Ai":1117,"Ak":164,"Al":6955,"By":154,"Bu":3220,"Br":4012,"Ca":14082,"E ":3496,"Bh":176,"Bi":2488,"Be":3517,"Bo":4388,"Bl":876,"Gò":247,"Ku":658,"Gö":233,"Ky":391,"Kn":144,"Kl":231,"Kr":781,"Ko":952,"Hã":340,"Hà":3871,"Há":6912,"Le":5532,"Li":5716,"N ":1502,"La":11255,"Lu":1591,"Hó":455,"Hò":671,"Ly":614,"Bư":341,"Hé":321,"Lo":11096,"Hì":160,"Me":5622,"Mi":7578,"Cơ":380,"O ":859,"Ma":14564,"Cư":440,"My":979,"Mu":2726,"Mo":6001,"Nh":15252,"Ni":3602,"Ng":8870,"Ne":5246,"Na":14732,"P ":1136,"Dư":2196,"ưu ":1732,"Ny":327,"Nu":290,"No":7260,"Ok":174,"Ol":595,"Om":201,"On":483,"Oh":203,"Oi":748,"Oc":485,"Od":183,"Oe":270,"Oa":270,"Ob":773,"Gi":6635,"Gh":326,"Gl":496,"Gr":4171,"Go":1400,"Gu":2462,"Gy":286,"Cô":3480,"Có":402,"Cú":325,"J ":219,"Ha":7544,"Dâ":1283,"He":2483,"Hi":2709,"Ho":13976,"Hu":4398,"Hy":1927,"Dô":340,"K ":324,"Ib":265,"Id":176,"Ic":220,"Im":190,"In":3623,"Il":428,"Is":1691,"It":2092,"Ir":1003,"Ja":1797,"L ":642,"Ji":230,"Je":710,"Jo":1104,"Ju":1037,"Ka":2594,"M ":626,"Kh":4970,"ưng":2590,"Ki":2990,"Ke":1180,"Ut":638,"Ur":404,"Um":166,"Un":919,"Uk":408,"Ul":140,"Ug":456,"W ":203,"Ty":201,"Tw":155,"Tu":2972,"Tr":21381,"To":2690,"Th":42787,"Ti":3875,"Te":2952,"Ta":3822,"V ":758,"Sw":579,"Sy":613,"St":3687,"Su":1921,"Wo":921,"Wi":1647,"Wh":297,"Sé":358,"Sè":329,"Wa":2024,"Sâ":534,"Sá":354,"We":2017,"Sà":247,"Y ":322,"Lư":806,"Vo":1915,"Vu":318,"Vi":16219,"Ré":208,"X ":401,"Va":2214,"Ve":2267,"Uy":394,"Lă":215,"Mã":1030,"Má":214,"Lý":664,"Lü":151,"Mé":245,"Pt":637,"Pu":1475,"Pr":3920,"Ps":557,"S ":1842,"Py":3299,"Pe":2603,"Là":800,"Lã":279,"Pf":1720,"Lâ":833,"Pa":7810,"Lé":142,"Pl":1384,"Po":3931,"Lê":927,"Pi":2275,"Ph":31098,"Os":655,"Ot":301,"Ou":271,"Ov":238," ا":171,"Op":394,"Or":2938,"R ":429,"Kô":237,"Se":3444,"Sc":3122,"Si":2745,"Nă":720,"Sh":1345,"Sm":208,"Sl":522,"Sk":307,"Sr":367,"Sp":2000,"So":3317,"Ru":1170,"Nô":330,"Ry":155,"Nó":11015,"U ":337,"Hư":909,"Sa":11442,"Re":2085,"Ri":2194,"Rh":4441,"Ro":3835,"Qu":14208,"Mô":839,"T ":649,"Mù":252,"Ra":1904,"Mü":183,"Sơ":2030,"Sư":226,"Yê":502,"Xã":2079,"Wü":541,"Xô":549,"ưa ":1620,"Xí":179,"Vĩ":456,"b ":1444,"a ":226759,"Tư":1027,"Vũ":589,"Tù":160,"Tú":283,"Tô":962,"Xy":153,"Nư":140,"Ye":349,"Ya":435,"Yp":239,"Yo":874,"Sĩ":507,"Yv":187,"Yu":292,"Mư":225,"Só":240,"Sô":413,"Tâ":9624,"Tà":542,"Xe":229,"Tá":302,"Sü":308,"Xa":216,"Tê":676,"Xi":420,"Tò":164,"Xu":1093,"Tí":259,"Vù":202,"Vă":1172,"Za":1272,"Và":236,"Ze":739,"Zh":185,"Zi":458,"Vâ":484,"Tĩ":188,"Zu":199,"Võ":191,"i ":212782,"bó":2113,"cà":270,"gd":295,"cá":22579,"ge":11280,"câ":2231,"bú":210,"ga":12467,"bé":224,"fl":1616,"bã":190,"fg":215,"ff":1242,"fi":2497,"fs":160,"fr":3324,"fu":1164,"ft":684,"fo":4267,"bê":1228,"bí":976,"bì":10910,"j ":155,"cù":1871,"cú":386,"gy":681,"có":47444,"gw":161,"cô":5046,"dâ":28874,"dã":502,"dà":2537,"he":41577,"hb":191,"ha":43156,"gn":5754,"gm":577,"gl":1738,"gk":293,"gi":33763,"bă":263,"gh":11779,"gg":570,"cò":3292,"gu":10662,"gt":510,"gs":1016,"gr":2654,"go":8645,"dt":805,"du":4489,"dw":715,"dy":507,"g ":364697,"ea":15752,"eb":3649,"ec":8564,"ed":11541,"de":24490,"dd":556,"dg":298,"df":167,"di":32181,"dh":710,"dk":145,"dm":293,"dl":1043,"do":14691,"dn":264,"ds":1755,"dr":3533,"ew":3323,"ex":3581,"eu":6762,"ev":2827,"ey":3123,"ez":1216,"fa":5205,"h ":178273,"bà":2398,"bá":2351,"fe":2871,"eh":679,"eg":4460,"ef":1217,"ee":4546,"el":24442,"ek":799,"ej":311,"ei":13045,"ep":4148,"eo":14079,"en":49087,"em":10013,"et":12592,"es":40569,"er":51401,"eq":167,"ca":30360,"Xư":246,"e ":134279,"bw":303,"by":1138,"bs":628,"br":3660,"bu":6071,"bt":170,"bn":144,"bo":4357,"bl":1961,"bf":332,"bh":146,"bi":46026,"bb":920,"be":11159,"db":293,"da":38163,"f ":9188,"cy":1022,"cu":9388,"ct":7017,"cs":417,"cq":226,"cr":3098,"co":14606,"cm":248,"cn":200,"ck":4238,"cl":1934,"ci":14195,"ch":129179,"ce":13915,"cc":1731,"Vư":1101,"c ":282171,"az":1913,"ay":20261,"ba":27162,"d ":28581,"at":21123,"as":21308,"ar":46226,"aq":534,"ax":1123,"aw":1934,"av":3839,"au":17184,"ak":3208,"al":35399,"ai":30021,"aj":1274,"ao":21790,"ap":5292,"am":43172,"an":108096,"ac":18572,"ad":10374,"aa":985,"ab":4752,"ag":8446,"ah":2616,"ae":34413,"af":1148,"nu":5357,"nt":26825,"ns":10879,"nr":434,"nq":250,"np":246,"no":9747,"hĩ":3597,"nn":7993,"q ":277,"nz":1727,"dư":2566,"ny":2595,"nx":314,"nw":217,"nv":655,"oe":1614,"of":8008,"oc":9998,"od":5406,"oa":17234,"ob":3593,"ké":466,"om":14342,"kê":1778,"on":100425,"ok":1733,"kè":139,"ol":16267,"oi":9431,"oj":178,"og":4447,"oh":1218,"ot":8950,"m²":659,"hō":191,"os":13276,"ov":4515,"ou":16281,"kì":215,"op":11878,"oo":2896,"or":34485,"oq":166,"gũ":395,"kí":1225,"r ":23753,"ox":666,"ow":4522,"oz":1526,"oy":925,"là":155485,"lá":810,"pe":13642,"pf":220,"lâ":707,"lã":1014,"pa":8707,"ký":795,"lè":430,"pl":2303,"lé":393,"lê":819,"po":6064,"ph":59995,"pi":7555,"lo":48870,"ln":332,"hê":913,"lm":1499,"hé":1262,"hè":446,"ll":28312,"ls":5922,"hí":22956,"lp":3661,"hì":4519,"hó":4714,"lw":160,"hò":5943,"lv":1420,"lu":8343,"lt":3135,"bư":9431,"lz":2455,"hö":169,"ly":7272,"hô":9522,"hú":4659,"hù":1344,"o ":85116,"iß":177,"ià":688,"hü":788,"ma":19960,"mb":6117,"mg":163,"hă":499,"me":15863,"iá":6621,"cơ":2439,"iè":809,"ml":141,"mi":19185,"mn":579,"iê":19338,"mm":3914,"ié":147,"mp":6181,"mo":8059,"mt":201,"ms":1171,"mu":3841,"iô":221,"ió":194,"cư":776,"my":849,"p ":57766,"iú":305,"na":36440,"nb":2672,"nc":9194,"nd":29108,"ne":33625,"nf":985,"ng":422545,"nh":179739,"ni":21672,"nj":602,"nk":2141,"nl":3093,"nm":685,"ju":396,"jo":698,"ki":21387,"kh":39807,"gã":188,"gâ":871,"ke":3413,"gá":300,"gà":10665,"ka":4069,"m ":167316,"gó":565,"gô":2554,"ky":750,"ks":1201,"kt":384,"ku":945,"ko":1602,"cũ":2937,"kr":861,"kk":293,"kl":1148,"km":4687,"kn":974,"li":29575,"lh":510,"lk":1201,"le":28117,"há":50648,"hà":29720,"ld":3791,"hã":1199,"lg":997,"hâ":38191,"lf":811,"la":39723,"lc":719,"lb":2760,"n ":440934,"hr":3391,"hs":2273,"dò":649,"hw":1075,"ht":2057,"hu":96505,"hk":147,"hh":235,"că":337,"hi":59002,"hn":1134,"ho":33760,"hl":2514,"hm":909,"dé":1520,"id":34385,"ic":24617,"ib":3478,"ia":42159,"ih":435,"ig":7165,"if":3529,"ie":18418,"hy":2981,"dù":2108,"k ":5941,"iq":1214,"ir":10741,"is":30398,"it":19216,"iu":2880,"iv":3060,"iw":152,"eó":2057,"ix":1054,"ii":6395,"ij":559,"ik":1883,"il":38718,"im":11936,"in":70002,"io":10270,"ip":6214,"je":426,"ji":826,"iz":1625,"iy":348,"l ":17786,"ja":1818,"nơ":1994,"să":488,"xi":2236,"tê":7401,"xo":824,"té":201,"tì":8961,"xp":151,"tí":18614,"tò":332,"xt":572,"xu":3759,"mư":601,"sô":2011,"ww":348,"só":466,"z ":4461,"xc":234,"xa":2141,"tâ":11580,"tá":3122,"xe":1418,"tà":3376,"wh":1250,"ră":636,"wi":4079,"sè":470,"wl":199,"sé":203,"wn":1808,"sê":269,"wo":1048,"ws":516,"wt":183,"rò":801,"rõ":266,"rô":669,"lư":3756,"rö":667,"rù":551,"y ":123996,"rú":838,"rü":313,"wa":4838,"sâ":1032,"sá":4087,"we":3258,"rè":166,"ré":3552,"vi":14749,"râ":232,"rã":310,"vu":13359,"vr":721,"rì":2865,"rí":1219,"rê":17007,"vo":1414,"uz":955,"uy":40068,"ux":1932,"uw":188,"uv":1489,"ve":8349,"rá":1135,"rà":746,"va":5946,"x ":5085,"ui":7198,"uj":271,"uk":987,"ul":11719,"ue":8283,"uf":658,"ug":2145,"uh":313,"mũ":152,"ur":22852,"us":22737,"ut":11022,"um":8525,"un":37448,"uo":379,"up":2448,"ty":3384,"tz":1632,"tu":10835,"tt":6267,"tw":506,"tv":273,"ub":3534,"ua":12317,"ud":3186,"uc":4195,"w ":3929,"to":19941,"tn":373,"tm":750,"tl":2439,"ts":3488,"tr":168490,"lũ":299,"tp":612,"tg":165,"tf":340,"te":34125,"ti":45992,"th":244395,"v ":848,"tb":456,"tc":539,"ta":22518,"su":6738,"sv":185,"ss":11481,"st":25304,"sy":900,"sw":1422,"sl":2460,"sk":1966,"sn":3570,"sm":2007,"sp":6570,"so":7204,"sr":482,"sq":267,"oà":40273,"sd":636,"oß":224,"sc":6190,"sf":434,"se":16078,"oá":2469,"nă":29159,"sh":6976,"sg":779,"oã":161,"si":19668,"hư":18279,"rz":821,"u ":92839,"nú":1257,"sa":12158,"sb":1450,"rr":9518,"rs":7765,"rt":11056,"ru":33603,"rv":1083,"rw":648,"nó":3472,"nô":498,"ry":4815,"rq":206,"rp":2369,"lĩ":607,"ro":79309,"nê":766,"rn":10149,"né":3611,"rm":6127,"rl":2540,"nç":154,"rk":3126,"hơ":3447,"ri":45411,"rh":1015,"rg":11132,"nâ":296,"rf":2041,"re":31859,"rd":9664,"nà":32104,"า":167,"rc":4893,"rb":3024,"ra":48559,"mù":496,"t ":240212,"gư":23033,"mó":270,"mô":1896,"qu":33189,"ร":145,"mé":23563,"mì":570,"má":2261,"mã":976,"lý":2656,"mà":2647,"s ":83922,"lú":457,"lô":11642,"px":1137,"py":386,"pt":4331,"pu":3257,"lò":168,"pp":2831,"lí":1363,"pr":2803,"ps":2406,"hū":297,"vư":1017,"zè":171,"vũ":680,"tư":4765,"sơ":283,"yê":5321,"sư":822,"xá":866,"xâ":942,"xã":6617,"rư":7067,"xé":140,"vĩ":1312,"xí":212,"vă":2376,"vù":22603,"zz":263,"vâ":231,"zh":389,"zi":1581,"uý":467,"zb":261,"và":46943,"ze":2177,"za":3062,"yz":169,"vò":489,"võ":378,"vô":1031,"zu":1605,"zo":1512,"zn":191,"ví":191,"vì":1173,"zl":195,"yg":495,"yh":272,"tă":382,"ye":2354,"uá":1249,"uâ":5412,"yf":154,"yc":1911,"yd":753,"ya":3754,"yb":311,"tú":167,"tù":418,"nư":35853,"tô":1034,"xy":294,"yx":252,"uô":12078,"yu":633,"yt":872,"ys":3871,"yr":5371,"yp":2222,"sĩ":1785,"yo":1215,"yn":1357,"uê":505,"ym":1622,"ué":319,"yl":3067,"yi":713,"xư":530,"yū":142,"² ":668,"Á ":1673,"Áv":245,"Áo":1420,"Âu":2129,"Âm":150,"Îl":330,"àn":34510,"ào":9571,"àm":2558,"ài":40656,"ã ":15139,"ày":41288,"àu":3190,"ám":1101,"án":28484,"áo":4300,"áp":27542,"ái":6121,"ác":31049,"áy":2255,"áu":552,"át":6111,"âm":6839,"ân":71359,"âu":6947,"ât":496,"ây":27219,"ãn":2862,"ão":485,"ãi":672,"ãy":559,"à ":205928,"á ":8250,"ße":294,"Úc":1243,"Ý ":1285,"Ôn":1610,"アアア":304,"ôi":2574,"ôm":11781,"ôn":48478,"óa":2709,"õ ":900,"ói":993,"óc":731,"óp":293,"ón":6670,"óm":1531,"ô ":34032,"òa":5612,"òm":318,"òn":6337,"ó ":64834,"ña":207,"ò ":1069,"ín":13049,"ít":636,"ìn":20626,"ìm":8528,"íc":17864,"ía":12239,"í ":5056,"ên":48870,"êm":9168,"êu":3630,"éz":154,"ì ":3075,"él":169,"éo":764,"ép":2326,"ém":434,"én":3441,"és":341,"ét":23592,"ér":825,"év":213,"éb":181,"éd":152,"éc":499,"ée":3394,"ég":181,"èn":257,"èo":280,"èr":1433,"ès":603,"èv":444,"ê ":4050,"é ":1396,"Đưể":826,"è ":371,"är":168,"ăm":28997,"ăn":8563,"ăk":142,"đưể":28774,"ān":253,"ý ":5155,"ể":1508247,"ün":249,"ür":1564,"üt":177,"üc":207,"üd":322,"ùn":27807,"ùi":235,"ùa":1067,"úp":776,"ún":1686,"úy":219,"út":637,"úa":606,"ùy":248,"úi":1359,"úc":3127,"ù ":1286,"ú ":1439,"ôt":1632,"öt":180,"ör":366,"ös":634,"ön":312,"öl":155,"đĩ":597,"đó":3828,"đô":27703,"đú":182,"đă":220,"Đư":848," ể":80643,"đư":29695,"đơ":1908,"tưể":2647,"Đa":443,"Đo":196,"Đi":1420,"đe":478,"đa":1708,"đo":2470,"đi":17430,"Đ ":295,"Đì":322,"Đà":1638,"Đá":165,"Đâ":3555,"đí":392,"đì":460,"đê":8064,"đã":4348,"Đă":188,"đá":3730,"đâ":1604,"đà":1035,"Đô":12692,"đu":489,"Cể":8674,"Dể":759,"Bể":9876,"Hể":9168,"Gể":170,"Lể":3634,"Kể":5925,"ĩ ":4602,"ĩa":3526,"ĩn":1420,"ĩnh":1417,"dể":12219,"ĩa ":3523,"cể":67050,"bể":45726,"mể":131131,"ũ ":2155,"lể":23218,"kể":7585,"iể":166241,"hể":264262,"gể":17545,"Sể":1192,"Tể":7402,"Rể":525,"ō ":339,"Mể":5140,"vưể":475,"Nể":1896,"Xể":142,"Vể":1294,"rể":50434,"sể":54959,"tể":83275,"uể":92128,"nể":21381,"ū ":399,"oể":14719,"ũi":200,"ũn":2955,"vể":53705,"专专 ":169,"xể":2191,"yể":31245,"ơi":3100,"ơm":181,"ơn":18072,"ơ ":4182,"あ":270,"ア":496,"가가 ":224,"ươ":12314,"ư ":8494,"ưa":1622,"ưn":2592,"ưu":1733,"Để":19679,"để":91689,"Đan":304,"đa ":614,"ơm ":179,"ơn ":5934,"đai":310,"đan":664,"ơi ":3096,"đi ":743,"ơng":12128,"đen":294,"đo ":419,"đoà":1003,"đua":287,"乙":857,"乘":179,"之":2791,"丹":642,"临":665,"中":142,"並":1114,"丙":309,"丘":935,"丛":252,"专":2269,"且":143,"丈":196,"三":4224,"丁":2034,"万":989,"亞":563,"亂":327,"侏":215,"ểy ":15070,"ểu ":43780,"ểt ":179200,"Đây":3553,"Đào":412,"Đài":765,"Đà ":277,"Đôn":3441,"圓":161,"Đô ":9162,"Đìn":322,"冲":171,"đêm":8016," 丘":335,"đíc":372," 专":413," 三":1027," 丁":795,"đìn":459,"đây":1563,"đá ":2316,"倉":569,"đán":1131,"đã ":4325,"đào":392,"đài":257,"đàn":318,"đún":140," 倉":244,"ưể":121112," 侏":158,"đón":976,"đôi":461,"đôn":8728," 並":342," 临":320," 丹":351," 之":799," 乙":411,"đó ":2805,"đô ":18495," 亂":152,"大":197,"ểa ":61642,"ểc ":245982,"ểch":9163,"ểi ":112695,"ểk ":174,"ển ":203101,"ểm ":59200,"ểp ":25010,"ểng":124564,"ểnh":41635,"ểo ":9528,"Điể":1233,"ああ":156,"điể":16552,"đoể":920,"đăn":218,"đĩa":596,"ς ":202,"đơn":1904," ểc":15788," ển":5390," ểm":264," ểy":799,"アア":391," ể ":57997,"đươ":308,"đưa":612,"ск":150," vư":1017," xư":529," Áo":1420," Áv":245," Á ":1673," Ga":3043," Câ":289," Cá":2724," Ge":2570," Cà":157," I ":489," Bì":1773," Bí":387," Fo":1209," Fu":749," Fr":2319," Fi":1014," Fl":1075," Bé":190," Ha":7534," He":2474," Dâ":1282," Cô":3477," Có":398," Gy":285," Cú":325," J ":162," Go":1392," Gr":4156," Gu":2454," Gh":322," Gi":6622," Gl":488," Id":176," Ic":219," Ib":265," Hy":1924," Dô":340," Hu":4380," Ho":13959," Hi":2699," Ji":227," Je":694," L ":288," Ja":1791," Ir":1002," Is":1688," It":2091," Im":186," In":3565," Il":425," M ":229," Ka":2587," Ke":1169," Ki":2975," Kh":4951," Jo":1090," Ju":1029," N ":147," La":11244," Hà":3868," Há":6905," Le":5519," Hã":339," Li":5628," Kl":231," Kn":141," Ko":946," Kr":777," Gò":247," Ku":656," Gö":233," Ky":391," Ma":14506," O ":139," Cơ":380," Mi":7555," Me":5610," Hì":157," Lo":11086," Hé":321," Bư":341," Ly":614," Hó":455," Hò":670," Lu":1590," Ne":5220,"а ":296," P ":285," Na":14703," Ng":8844," Nh":15231," Ni":3593," Mo":5991," Cư":440," My":978," Mu":2713," A ":801," B ":314," C ":460," Ap":624," Am":2023," An":10467," Ak":163," Al":6944," Ai":1106," Ag":749," Ah":189," Ae":262," Af":957," Ac":1189," Ad":608," Ab":573," Ba":13586," D ":281," Az":1190," Ay":139," Av":424," Au":2992," At":1064," As":1419," Ar":5374," Aq":1657," Be":3504," Bi":2420," Bh":176," Bl":874," Bo":4376," Br":4006," Bu":3207," By":153," E ":175," Ca":14032," Ce":2572," Ci":1003," Ch":19001," Cl":1229," Cr":1732," Co":10896," Cu":1575," Cy":959," F ":217," Da":2548," Di":2876," De":2206," Dr":1245," Do":2436," Dy":152," Du":1262," Ea":529," Ec":908," Ed":358," G ":173," El":867," Ei":335," Et":365," Es":889," Er":614," Ep":314," En":763," Em":454," Ex":456," Eu":2122," Bà":785," Bá":846," Fe":894," Fa":1822," H ":178," Xu":1090," Tò":163," Tí":259," Tê":651," Xi":413," Tà":542," Xe":204," Tá":302," Tâ":9623," Sü":308," Xa":216," Só":240," Sô":411," Mư":225," Wo":905," Sé":358," Sè":329," Wi":1632," Wh":294," Sá":350," We":2007," Sâ":533," Sà":247," Wa":2017,"й ":151," Y ":281," Lư":805," Võ":190," Tĩ":188," Và":232," Ze":737," Vâ":484," Zh":184," Zi":450," Za":1260," Yv":187," Yu":290," Yp":238," Yo":868," Sĩ":506," Ya":432," Ye":349," Nư":140," Tô":962," Xy":153," Tú":282," Tù":160," Xí":179," Vĩ":455," Xô":549," Wü":541," Xã":2079," Vù":202," Vă":1148," Vũ":587," Tư":1025," a ":4962," Yê":501," Sư":226," Sơ":2029," R ":199," Kô":237," Ou":268," Ov":236," Os":655," Ot":299," Or":2938," Op":391," Po":3911," Lê":922," Lé":142," Pl":1371," Pi":2271," Ph":31029," Lã":279," Pf":1720," Lâ":831," Pe":2592," Là":800," Pa":7774," Dư":2196," Ny":327," Nu":290," No":7248," Ol":594," Ok":173," On":479," Om":200," Oh":202," Oi":748," Od":181," Oc":483," Oe":269," Ob":773," Oa":270," Ra":1894," Mü":182," T ":156," Mù":252," Mô":838," Qu":14182," Ro":3827," Re":2068," Ri":2185," Rh":4439," Py":3297," S ":230," Pr":3910," Ps":554," Pt":637," Pu":1471," Mé":244," Lý":663," Lü":151," Má":214," Lă":214," Mã":1026," Sy":607," Sw":578," Su":1915," St":3635," Ta":3810," V ":169," Th":42680," Ti":3850," Te":2931," Tr":21258," To":2625," Nó":11014," Ry":154," Nô":330," Ru":1162," Sa":11420," Hư":907," U ":154," Nă":720," Sh":1334," Si":2727," Sc":3070," Se":3434," So":3300," Sp":1984," Sr":367," Sk":306," Sl":517," Sm":206," Uy":393," Va":2209," X ":200," Ve":2261," Vi":16180," Ré":207," Vo":1912," Vu":317," Tu":2947," Tw":154," Ty":200," Ug":456," Uk":408," Ul":140," Um":165," Un":913," Ur":403," Ut":638," ja":224," l ":250," im":322," in":6639," is":5408," it":773," ka":329," m ":439," kh":38847," ki":18267," gâ":429," ke":373," gá":282," ju":151," cô":5020," có":47410," cú":382," cù":1871," ha":11491," dã":502," dâ":28872," he":621," dà":2524," gi":28288," bă":261," gh":729," gl":236," gr":899," go":250," gu":378," cò":3288," hy":276," dù":2107," că":333," hi":6669," dé":1144," ho":9851," ht":218," hu":16965," dò":646," nh":45517," ni":1927," ng":51853," ne":507," na":15500," cư":775," mu":1888," mo":1972," ké":437," on":1221," kê":1776," oc":375," of":6994," ob":204," dư":2565," nu":602," no":1173," hã":1017," hà":8670," há":923," le":1986," li":4644," la":5104," gó":504," kn":913," km":4476," cũ":2936," ko":246," me":1022," mi":9905," cơ":2426," hù":342,"я ":139," hú":203," ma":3748," lu":1700," hó":2026," hò":4796," ly":382," hô":283," bư":9431," hè":160," lo":35801," hì":2834," ag":160," ab":359," ac":560," ad":220,"Hểu":669," am":614,"Hểp":191," an":7056," ap":254," ai":278,"Hển":822," al":2427," au":545,"Hểi":3718," ar":2712,"Hểc":428," aq":275," at":395," as":1145," d ":2111," ba":20628," 가가":227," bi":39922," be":1810," bo":1086," bl":299," by":542," bu":1037," br":705," ca":18336," Xư":246,"Hể ":3004," Vư":1101," er":165," et":2265," es":168," en":1522," em":442," el":437," bà":2390," fe":289," bá":2320," fa":2087," eu":165," ex":489," fu":610," fr":2001," bí":964," bì":10908," fo":1760," bê":1224," bé":146," fl":771," fi":679," bã":190," ge":1821," cá":22553," câ":2231," cà":270," ga":2783," bú":206," bó":2103," cl":550," cm":154," co":7147," cr":532," ce":1064," ch":72993," ci":299," da":4945," cu":4863," cy":157," do":4301," dr":248," de":8416," di":19653," ec":238," ed":190," ea":304," du":2623," 三三":209," vù":22601," vă":2376," ví":173," vì":1170," võ":378," vô":1027," vò":489," và":46932," vâ":230," sĩ":1785,"ка":178," tù":416," tú":155," tô":909," nư":35851," tă":379," xo":207," tê":7377," nơ":1991," să":488," tò":331," xu":3665," tí":18540," tì":8960," mư":601," ww":167," só":461," sô":2005," tâ":11572," tà":3363," tá":3073," xe":994," xa":907," tư":4765," vũ":680," 三之":172," sư":822,"ов":213," yê":400," sơ":282," rư":159," vĩ":1312," xã":6617," xâ":941," xá":864," ru":574," nô":442," nó":3467," hư":1543," sa":4809," nú":1256," se":1352," sc":583," si":4586," sh":865," nă":29145," sn":2777," sm":592," sl":244," sp":4116," so":2264," qu":28552," mó":260,"ви":141," mô":1890," mù":492," ra":5117," re":1311," nà":32102," nâ":291," ri":1195," hơ":1747," né":300," ro":791," lĩ":607," nê":751," lò":161," pu":610," pr":1132," lí":1307," lú":446," s ":1052," lô":926," px":1128," py":148," mã":975," má":2233," mà":2646," lý":2654," mì":566," mé":12707," ot":153," kì":215," op":485," kí":1225," or":1862," lá":722," pe":891," là":155391," lã":1013," lâ":684," pa":1485," ký":784," lè":332," pl":606," po":832," lê":796," pi":462," ph":51377," wa":721," sâ":1031," sá":4080," we":797," rõ":266," y ":288," lư":3756," wo":408," wi":1339," wh":1197," sê":264," va":1100," ve":759," uy":141," vo":357," vu":12917," rã":285," vi":8649," ty":993," tu":4206," us":140," mũ":152," up":179," un":373," ta":1679," v ":243," sy":262," st":916," su":3809," lũ":299," tr":156052," to":6595," th":230858," ti":21975," te":899," đu":484," đa":1700," đe":474," đi":17419," đo":2466," Đá":164," Đà":1637," Đâ":3555," Đì":322," Đo":196," Đi":1416," Đa":443," đơ":1908," đư":29691," Đư":847," đí":392," đê":8032," đì":460," đá":3730," đâ":1604," đã":4347," Đă":188," đà":1035," Đô":12690," đă":220," đú":182," đô":27698," đó":3827," đĩ":596,"Lể ":569,"Lểi":178,"Lển":215,"Lểp":1280,"Lểc":1143," Bể":9869," Kể":5918," Hể":9143,"Kể ":5821," Gể":165," Cể":8663," Dể":757," Âm":150," Âu":2129," Îl":330," Ôn":1608,"Nể ":198,"Nển":165,"Nểi":1235," Úc":1243," Ý ":1274," áp":355," án":814," âm":3643," ít":431," ô ":353," ôn":1711,"Mể ":2527," ý ":503,"Mểc":807,"Mểt":1153,"Mểu":174,"Mển":285,"Mểi":146," ăn":626," 가":331," ươ":433," ưu":158,"가":830," hể":63957," gể":12621," lể":23214," kể":7585," cể":67038," dể":12218," bể":45724," Vể":1292," Tể":7366,"Rểp":306," Xể":142," Nể":1895," Lể":3615," Mể":5122," Rể":522," Sể":1093," vể":53702," xể":2191," yể":935," mể":131122," nể":21375," rể":4289," sể":54954," tể":83225,"Tểt":172,"Tểp":225,"Tểc":152,"Tểa":1075,"Tển":2741,"Tểi":245,"Tể ":2617,"Sể ":900,"Vểt":162,"Vển":442,"Vểi":163,"Vể ":500,"ưểt":382,"ưểu":183,"ưển":18802,"ưểm":9501,"ưểi":24657,"ưểc":67509," Để":19647," để":91683," ưể":501,"Ávi":245,"ال":152,"Áo ":1417,"三三 ":318,"三万 ":202,"Âu ":2118,"三专 ":318,"Âm ":149,"ươn":12134,"ươi":146,"가가":499,"Bể ":2169,"AO ":231,"AN ":149,"Bểt":140,"Bểc":4036,"Bển":2788,"Bểo":450,"Cể ":946,"Cểu":671,"Cểp":339,"Cểm":232,"Cển":5510,"Cểc":787,"Dể ":256,"Dểc":190,"Bà ":315,"Bá ":150,"Bài":310,"Bàn":152,"Bác":305,"Fel":162,"Fen":155,"Fer":312,"Fis":282,"Ext":290,"Fas":526,"Fal":176,"Far":151,"Fab":275,"Eri":170,"Ess":158,"Est":392,"Eth":271,"Eup":437,"Eur":1040,"El ":188,"Ele":184,"Eng":198,"Epi":248,"Ent":180,"Các":1801,"Ger":929,"Cát":277,"Geo":862,"Gen":359,"Gla":140,"Gha":245,"Gia":2324,"Gil":159,"Gir":424,"oể ":155,"oểt":2492,"Cá ":259,"oểc":3719,"Gan":147,"Gal":295,"Gam":240,"Gau":147,"Gar":1513,"oển":3941,"oểi":4411,"Gab":195,"Fus":252,"Fro":378,"Flo":588,"Fla":166,"Fle":174,"ũi ":200,"Fra":932,"Fri":349,"ũng":2953,"Fre":553,"Bín":197,"Bìn":1748,"Fon":217,"For":497,"Fou":186,"Dân":1220,"nểu":216,"II ":1176,"Hil":320,"Him":177,"Hin":145,"Hip":145,"nể ":1462,"Hel":390,"nểi":2679,"Hei":256,"nểm":15373,"nển":1144,"Hem":163,"Hen":196,"Hes":405,"nểa":304,"Her":528,"Cúp":236,"Hal":332,"Hai":401,"Han":467,"Ham":344,"Has":179,"Har":748,"Haw":533,"Hau":3745,"Côt":1585,"Guy":153,"Cô ":190,"Gua":732,"Có ":275,"Gui":1036,"Côn":1681,"Gre":1012,"Gri":151,"Gra":2331,"Gro":483,"Glo":169,"Giá":926,"Gon":156,"Goo":205,"Gol":188,"Gom":158,"Úc ":1233,"Inn":161,"Int":561,"Ins":197,"Ill":204,"Ind":2106,"mểc":13680,"mểm":11638,"mểi":2779,"mểu":420,"mểt":99606,"mển":1814,"mể ":1110,"Ibe":205,"亞 ":440,"Hyp":280,"Hyd":396,"Dôm":338,"Hy ":979,"Hun":464,"Huy":2731,"Hue":217,"IV ":160,"Hoà":1902,"Hor":294,"Hou":153,"Hom":153,"Hon":555,"Hok":148,"Hol":1270,"Hoa":8475,"Arg":562,"Are":201,"Arc":363,"Ard":1593,"Ara":960,"Arm":257,"Ari":480,"Aqu":1651,"Apo":203,"Ath":149,"Atl":649,"Ast":281,"Ass":392,"Asi":256,"Arr":175,"Art":235,"Ave":196,"Auv":549,"Aus":949,"Aur":157,"Aud":253,"Aug":204,"Aub":473,"lể ":2800,"Azu":1003,"lểp":5146,"lểt":155,"lểi":2389,"lển":5223,"lểa":430,"lểc":6489,"Ba ":889,"lểy":462,"Bai":215,"Bal":574,"Ban":6419,"Bac":146,"Bad":1148,"Bay":966,"Bar":929,"Bat":270,"Bas":1136,"Bau":155,"Abr":165,"Aca":233,"Acr":321,"Ach":200,"Ade":163,"Ai ":341,"Aga":200,"Afr":687,"Afg":186,"Air":488,"Ala":309,"Alb":681,"An ":1388,"Alg":185,"Ali":170,"Ale":299,"Alv":153,"Als":515,"Alt":493,"Alm":187,"All":366,"Alp":2942,"Ame":689,"Amb":210,"Ama":449,"Amp":158,"Anh":5927,"Ang":642,"Ana":331,"And":666,"Ant":595,"Ano":159,"Ann":219,"Bus":168,"Bul":625,"Bur":1284,"Buc":492,"Bru":412,"kể ":5372,"Cab":144,"kểt":1768,"Cae":179,"Cal":3283,"Cam":1757,"Cai":141,"Cas":3059,"Car":1359,"Cau":236,"Cat":533,"Cao":644,"Can":1671,"kểc":374,"Cap":594,"Bea":394,"Ber":1074,"Ben":559,"Bel":748,"Bin":143,"Bil":278,"Bis":150,"Bit":253,"Bir":234,"Blu":260,"CP ":196,"Biê":192,"CN ":900,"Bla":380,"Bre":721,"Bra":1413,"Bro":439,"Bri":863,"Bol":471,"Boi":232,"Bon":251,"Bor":635,"Bos":277,"Bot":225,"Bou":1517,"Cyp":409,"Cur":167,"Cub":268,"Cun":184,"Cup":164,"EE ":154,"Des":235,"Deu":424,"Del":231,"Dem":185,"Den":364,"Dam":166,"Dan":842,"Dar":270,"Dav":172,"Dal":154,"Cho":474,"Chr":351,"Che":539,"Chi":4142,"Cic":417,"Chu":975,"Cit":157,"Châ":2351,"Cle":204,"Cla":633,"iểt":19441,"iểu":22084,"iểy":197,"Cel":188,"iểm":11357,"iểp":3887,"iển":94986,"Cen":1295,"Cer":828,"iểc":3158,"iểi":7620,"iểa":1763,"Cha":5749,"Cri":167,"Cra":378,"Cre":395,"Chư":286,"Cro":546,"Chù":202,"Chú":672,"Chí":1230,"Coc":220,"Coe":184,"Cop":262,"Cos":1089,"Cor":1423,"Com":752,"Col":1893,"Con":3653,"Cou":867,"FA ":242,"Drô":335,"iể ":1731,"Edw":139,"Ông":1552,"Ect":200,"Ecu":469,"Eas":364,"Do ":273,"Diê":175,"Dic":144,"之三三":139,"Dit":152,"Dis":351,"hểo":739,"hểp":5189,"hểm":3873,"Dip":407,"hển":23819,"Dio":154,"hểy":11164,"hểu":6003,"Die":258,"hểt":14285,"hểa":1704,"hểi":18265,"hểc":23691,"Di ":179,"hể ":155404,"Dun":239,"Duy":294,"Du ":216,"Dri":386,"Dre":146,"Dra":192,"Dou":158,"Don":392,"Dom":257,"Dor":723,"Cươ":311,"Nev":160,"Neu":541,"Net":191,"Nep":895,"Neo":280,"Nas":316,"Nat":595,"Nav":156,"Nig":396,"Nie":1241,"Nic":337,"Nin":1068,"Nhi":250,"Nha":5926,"Nga":1678,"Ngh":1164,"Ngu":2389,"Ngo":273,"New":2499,"Myr":349,"xể ":903,"Mya":329,"Nak":165,"Nam":11941,"Nan":196,"Nag":208,"Na ":357,"xểp":906,"xểy":267,"Như":191,"Nym":236,"Nhó":149,"Nhà":676,"Nhâ":3285,"Ngô":642,"Ngà":567,"Ngâ":218,"Ngư":724,"Nhĩ":654,"Nov":205,"Ngũ":256,"Nor":4126,"Not":381,"Nob":171,"Noc":1779,"Oec":143,"Dươ":1846,"PG ":211,"Ois":715,"Ohi":151,"Oah":152,"Occ":238,"Obe":662,"Île":330,"Ott":171,"Ovu":166,"Kôn":235,"Oly":147,"Oli":201,"Giể":2035,"Ont":199,"Or ":550,"Opo":183,"Ora":158,"Ore":369,"Orc":304,"Ori":453,"Orn":566,"Ost":243,"Phú":926,"Phù":169,"Phò":192,"Phó":180,"Phá":21296,"Ple":538,"Phâ":188,"Pla":641,"Hiể":1343,"Lê ":891,"Pin":370,"Pit":142,"Phy":669,"Pie":506,"Pic":810,"Pho":672,"Phi":3036,"Pha":566,"Lãn":143,"vểy":569,"Lâm":707,"vển":2537,"vểt":17681,"Pfa":1704,"vểa":164,"vểc":13644,"vểi":7805,"Per":956,"Pet":409,"Pen":544,"Pel":140,"Lào":383,"vể ":11302,"Pay":1416,"Là ":298,"Pat":164,"Pas":888,"Par":1717,"Pau":274,"Pac":219,"Pan":548,"Pap":721,"Pal":1091,"Pak":192,"Pyr":3193,"Huể":468,"Pte":513,"Pun":221,"Pup":166,"Pue":171,"Puy":394,"Pro":2087,"Pri":425,"Pre":247,"Phư":678,"Pse":388,"Hoể":173,"Pra":808,"Pol":756,"Pom":148,"Pon":298,"Poi":1472,"Pot":227,"Por":309,"uểc":63038,"uểi":2825,"Lăn":196,"uểt":6647,"uển":17332,"Mã ":868,"uể ":2223,"Lý ":659,"Mùa":206,"SA ":174,"Ram":194,"Ran":617,"Quá":225,"Quâ":725,"Môn":650,"Quý":277,"Qua":706,"Qui":159,"Que":829,"Quy":457,"Ita":500,"Isl":594,"Isr":263,"It ":1456,"Ira":481,"Ire":363,"Isè":447,"tểm":507,"tển":38505,"tểi":9573,"tểc":3283,"tểa":923,"tể ":24294,"Jac":218,"Jav":319,"Jan":148,"Jam":357,"tểo":2243,"tểp":1747,"tểt":2043,"Jer":195,"Jea":217,"Biể":504,"Jos":191,"Jor":166,"Joh":334,"Jul":480,"sể ":37621,"sển":14954,"sểm":218,"sểc":1290,"sểa":158,"Kai":155,"Kam":181,"Kal":268,"Kan":358,"Kau":180,"Kat":153,"Kas":184,"Kar":496,"Kaz":203,"sểt":255,"sểp":188,"Ken":722,"Kir":203,"Kit":143,"Kin":907,"Kim":609,"Kho":491,"Khu":1438,"Kha":464,"Khi":230,"Chể":1285,"Khê":158,"Khá":594,"Khô":380,"Kon":253,"Kor":151,"Kre":398,"Gòn":201,"Cuể":281,"Kus":145,"Kur":182,"Hàn":1241,"Leu":183,"Les":367,"Lep":631,"Leo":339,"Len":152,"Hán":6856,"Lei":182,"rểc":1530,"Lea":285,"rển":39207,"rểm":291,"rểi":1137,"Hà ":2463,"Lau":510,"rể ":6793,"Le ":619,"Lak":163,"Lai":390,"Las":248,"Lat":742,"Lar":215,"Lao":140,"Lam":368,"Lan":4696,"Lac":165,"Lab":180,"倉 ":180,"La ":2717,"Liê":1950,"Hér":279,"Diể":335,"Lib":574,"Lie":155,"Lig":183,"Lim":511,"Lin":692,"Lio":171,"Lis":148,"Lit":507,"Liv":154,"Leó":2050,"Hãn":333,"rểt":1287,"Hòa":572,"Lud":184,"Luc":212,"Loà":1669,"Hìn":160,"Loz":163,"Lou":367,"Los":193,"Lot":580,"MS ":483,"Loi":3201,"Lor":2131,"Lon":1378,"Lom":271,"Loa":517,"Ma ":267,"Hóa":437,"Luâ":253,"Lyc":258,"Mei":187,"Men":179,"Mel":544,"Mes":227,"Mer":503,"Meu":1122,"Met":367,"Mec":579,"Meg":242,"Med":321,"Mex":918,"Man":2248,"Mal":1668,"Mar":4888,"Mas":469,"Mag":297,"Mad":710,"Mah":444,"Mai":1077,"Mac":532,"NE ":2847,"May":519,"Mau":531,"Mat":259,"Miê":150,"Mol":400,"Mon":1606,"Mos":1419,"Mor":654,"Mou":470,"Mot":424,"Moz":414,"Mid":2002,"Mic":1239,"Cơ ":351,"Mit":747,"Mir":161,"Mis":522,"Mil":319,"Min":1719,"Mun":171,"Mur":1848,"Mus":259,"Xià":174,"Phể":1906,"Tây":8477,"Tân":1027,"Tào":168,"Tàu":164,"Süd":288,"Sôn":404,"Sóc":214,"Wor":462,"Wol":216,"Séc":256,"Sèv":318,"Whi":209,"èvr":358,"Wik":174,"Wil":393,"Win":429,"Wie":149,"Wit":233,"ère":1422,"Sài":216,"Web":172,"Wei":433,"Lươ":358,"Wes":961,"Sân":446,"Was":264,"War":327,"Wal":818,"ès ":590,"Lưu":290,"Dưể":277,"èn ":152,"èo ":278,"ém ":263,"之丁":166,"QĐ ":168,"之三":478,"之万":178,"ée ":472,"之专":315,"之之":217,"ées":2904,"Vos":497,"Vor":516,"Vol":589,"éc ":283,"Nhể":3766,"Ngể":811,"Viê":165,"évi":139,"Tĩn":188,"ép ":1113,"Zea":402,"Zar":282,"Zam":723,"én ":239,"éo ":609,"éra":396,"ét ":23407,"éri":203,"éné":2942,"Zim":303,"Zel":153,"épa":1146,"Vân":484,"ên ":47944,"êm ":9120,"Yps":167,"Quể":10633,"Yve":176,"並三":153,"Sĩ ":505,"Yor":442,"You":143,"Yon":155,"ênh":268,"êng":589,"专专":219,"专三":225,"êu ":3625,"Yel":146,"三万":220,"三丁":213,"三三":777,"三专":394,"Tô ":277,"Xuâ":498,"三之":347,"Tôn":652,"Túc":179,"丁专":144,"丁丁":142,"Tên":660,"丁三":258,"Tín":160,"丁之":215,"Xuy":470,"ãi ":669,"Syn":143,"Syr":151,"Swi":186,"Swa":238,"Sur":243,"Sum":294,"Sul":327,"Sun":218,"Sud":207,"Str":573,"Stu":169,"Sti":380,"Sto":266,"Sta":800,"Ste":1346,"Ten":237,"Tel":150,"ãnh":975,"ãng":1319,"ão ":484,"Tam":802,"Tan":929,"Tas":235,"Tar":601,"ãn ":565,"Tai":151,"Tak":148,"Ski":144,"ãy ":559,"Khể":614,"Shi":359,"She":144,"Năm":661,"Sho":220,"Sha":353,"Sim":166,"Sil":271,"Sin":736,"Sie":417,"Sib":258,"Sic":157,"Ses":155,"Ser":590,"Sen":317,"Sel":193,"Hươ":208,"Sei":684,"Seg":435,"Sri":344,"TV ":172,"Spa":202,"Spi":140,"Sph":1036,"Spe":218,"Spr":184,"Sou":1403,"Sol":431,"Som":283,"Son":366,"Sor":294,"Kiể":703,"Slo":396,"Nôn":230,"Rus":514,"Nó ":10964,"Sai":3614,"Sam":307,"Sal":1166,"Saa":431,"Sac":786,"Sab":139,"Sco":422,"Sci":154,"Sch":2083,"Sca":289,"Sax":162,"Sav":348,"Sat":196,"Sau":596,"Sar":1013,"Sap":212,"San":1415,"Sao":173,"Hưn":444,"Sa ":225,"TA ":219,"Res":158,"Rhi":678,"Rhe":1850,"Riv":248,"Ris":546,"Rie":160,"Ric":612,"Red":234,"Rei":173,"Reg":216,"Ren":178,"Rep":253,"Rob":306,"Roc":356,"Rou":1296,"Rot":151,"Ros":295,"Rom":626,"SS ":368,"Rhô":1640,"Ven":831,"Vau":301,"Van":289,"Val":1014,"Var":300,"Vic":424,"Vie":766,"Vir":376,"Vil":850,"Vin":383,"Ver":694,"Vex":266,"Ukr":405,"Uni":646,"Miể":206,"Uy ":300,"Utt":447,"Tră":223,"Luể":194,"Trá":373,"Trà":189,"Trâ":155,"Trì":245,"Bưể":300,"Trư":1573,"Uga":441,"Tex":717,"Ter":1194,"Tha":1705,"The":9028,"Thi":2418,"Tho":351,"Thu":1030,"Til":178,"Tim":168,"Tin":432,"Thà":2636,"Thá":3215,"Liể":150,"Thü":723,"Thô":412,"Tiê":963,"Tor":743,"Tok":152,"Tol":222,"Tom":166,"Tou":295,"Thư":1102,"Tru":10246,"Tro":2116,"Tri":2061,"Tre":371,"Tra":816,"Toà":178,"Tuy":629,"Tur":1442,"Tun":141,"Mưể":195,"ày ":41264,"Tươ":169,"àu ":3188,"gểa":154,"gểc":1587,"gểi":5837,"gểp":336,"gểm":3968,"gển":2170,"gểy":212,"gểt":591,"ành":21823,"àng":7133,"gể ":2467,"ào ":9552,"àn ":5519,"àm ":2556,"ài ":40596,"dể ":1928,"bis":208,"bit":429,"biu":210,"bio":195,"biq":379,"bir":196,"bil":353,"bin":1156,"bii":441,"dểa":1072,"dểc":2450,"bo ":249,"dểi":171,"dểm":153,"dển":5451,"dểu":482,"dểy":237,"áy ":2251,"blo":143,"ble":788,"bli":464,"bla":343,"boa":544,"bol":192,"biê":756,"bon":625,"bom":252,"bop":200,"bor":571,"bot":181,"bos":169,"bou":707,"bbe":548,"be ":978,"áo ":4269,"ban":14532,"bal":489,"bai":195,"bac":1588,"bad":244,"bab":362,"án ":11049,"bay":2258,"bat":528,"bas":507,"bar":886,"bao":2943,"bea":250,"áp ":27496,"ánh":4002,"áng":13329,"bi ":208,"bei":269,"bee":755,"bed":546,"bec":391,"ber":5184,"ben":538,"bel":1263,"bek":171,"bes":289,"bet":259,"bfa":299,"áu ":550,"bia":2262,"bic":185,"bid":364,"át ":6081,"ách":6363,"áce":165,"ái ":6094,"ca ":5266,"car":2682,"cas":523,"cat":809,"cau":258,"can":2819,"cao":13635,"cap":271,"cac":552,"cae":343,"cad":152,"cam":352,"cal":1785,"cai":449,"ce ":3602,"ám ":1096,"bri":1032,"bro":469,"bra":914,"bre":597,"bru":440,"bsi":207,"bur":2940,"bul":333,"bun":198,"bum":722,"but":706,"bus":353,"by ":708,"bwe":288,"ác ":24475,"aka":675,"am ":27480,"ake":553,"aki":591,"akh":416,"aji":229,"ajo":231,"al ":6772,"aja":635,"aii":453,"ail":3696,"ain":10996,"air":827,"ais":1435,"ait":246,"ak ":354,"aig":201,"aid":285,"aic":180,"aia":148,"ây ":27201,"ahi":144,"ahu":249,"ahr":141,"aho":292,"aha":852,"agi":293,"agr":420,"agu":637,"agn":2270,"ago":1655,"aq ":141,"anu":911,"anz":946,"any":498,"ano":1266,"ann":1274,"anm":363,"ant":5267,"ans":1570,"ane":1647,"ang":23334,"anh":11504,"ani":4224,"anj":313,"ank":983,"ap ":200,"ana":4781,"anc":3852,"and":17225,"amu":239,"amm":581,"amo":823,"amp":3495,"ams":266,"ami":3273,"ame":2454,"amb":1921,"ama":1996,"ao ":21236,"alz":1868,"aly":679,"alv":524,"alu":519,"alt":1192,"als":583,"alp":280,"alo":1138,"alm":374,"all":4541,"alk":430,"alg":162,"ali":4610,"alc":275,"ald":946,"ale":3407,"alf":143,"ala":5532,"alb":1038,"an ":27435,"aku":180,"ako":211,"aba":1197,"abe":439,"abi":699,"abl":307,"abo":447,"abr":553,"abu":216,"abw":286,"ae ":31054,"aca":592,"aal":305,"aar":275,"ad ":1523,"ac ":1152,"âng":210,"ab ":257,"aft":152,"aff":222,"ai ":10857,"aga":1564,"age":954,"aeo":180,"aen":693,"ael":524,"aes":144,"aer":162,"Vươ":859,"aei":212,"ah ":482,"âte":406,"ado":1640,"adr":334,"adi":656,"âu ":6939,"ade":2368,"aea":378,"aec":197,"ag ":159,"ady":140,"adt":318,"adu":484,"aco":486,"ack":866,"aci":2053,"ach":4974,"ace":5973,"acc":246,"ada":2077,"af ":167,"act":895,"acu":424,"acr":438,"azo":203,"azi":604,"aze":208,"aza":466,"axi":240,"axo":326,"az ":215,"ayo":158,"ays":2306,"aya":1069,"aye":1477,"ân ":71107,"ba ":1967,"âm ":6828,"aqu":368,"at ":2191,"arh":179,"arg":1494,"are":4513,"ard":3817,"arc":1390,"arb":768,"ara":4118,"arp":777,"aro":1967,"arn":2152,"arm":452,"arl":768,"ark":1469,"ari":8943,"aru":469,"arv":290,"arr":2293,"ars":863,"art":4067,"au ":6244,"asa":466,"ary":947,"arz":380,"asi":1358,"ash":1142,"asc":1250,"ase":497,"aso":213,"asp":226,"ask":259,"asm":354,"aon":161,"ar ":3587,"apa":481,"ape":779,"api":791,"aph":710,"apl":205,"apo":627,"app":349,"apt":205,"apu":632,"as ":6065,"ava":1308,"ax ":315,"aux":509,"auv":229,"aut":4199,"avo":316,"avi":854,"ave":987,"awe":151,"ay ":14622,"awa":1004,"awi":403,"ata":3801,"asu":225,"ast":6128,"ass":2666,"atr":716,"ato":1406,"ate":3193,"atc":165,"ati":4840,"ath":1382,"aua":167,"auc":575,"aub":164,"att":609,"ats":390,"atu":1696,"aty":238,"aul":975,"aum":254,"aun":342,"aur":1043,"aus":1014,"aud":522,"aue":262,"aug":153,"aui":145,"bể ":9115,"Wür":527,"bểy":259,"bểt":2371,"bểu":602,"bểi":3447,"bểo":909,"Xã ":2077,"bển":18106,"bểc":10801,"Thể":19271,"Tiể":1514,"Trể":2534,"Vĩn":400,"Xíc":159,"Võ ":184,"cểu":3163,"cểt":394,"cểp":4267,"cểc":593,"cểa":46999,"cển":3734,"cểm":741,"cểi":1406,"Hưể":237,"Vùn":202,"cể ":5697,"Văn":1172,"Sư ":192,"Viể":12501,"Vũ ":518,"Tư ":617,"Tuể":240,"Xô ":542,"Yên":481,"Sơn":1964,"ji ":324,"jar":408,"jan":213,"biể":38109,"jo ":152,"itr":907,"ito":2297,"itu":456,"itt":1509,"its":449,"itz":828,"ity":598,"ism":206,"isl":191,"iso":476,"isp":337,"iss":3448,"ist":3624,"ita":3373,"itc":146,"ite":2019,"ith":2136,"iti":2423,"ivo":162,"ius":1319,"ium":1206,"iva":362,"ix ":837,"ivi":819,"ive":1549,"ipo":143,"ipp":1792,"ipu":171,"ipt":450,"ipi":254,"iph":349,"ipl":287,"ilô":10662,"is ":16454,"ion":4473,"iop":856,"ior":202,"ios":885,"iot":545,"iou":177,"ioi":173,"iol":823,"ipa":1057,"ipe":657,"ir ":1809,"iru":298,"irs":241,"ück":162,"iro":773,"irk":203,"irl":235,"iri":455,"isi":750,"ish":1324,"ise":1480,"isc":1025,"isa":361,"iqu":1196,"ire":3687,"irg":448,"ira":1107,"irc":652,"it ":1527,"ja ":441,"iya":187,"iz ":244,"eón":2056,"가가가":272,"izo":396,"ize":388,"iza":272,"kim":1056,"kil":10715,"kia":494,"kin":3453,"kip":325,"kir":466,"kis":481,"km ":3828,"chể":21772,"ki ":1183,"khi":3505,"út ":627,"khe":165,"kha":834,"khu":15081,"kho":6571,"gày":9377,"gái":216,"kel":215,"ken":731,"kes":174,"ker":740,"ket":226,"key":304,"gân":416,"gây":413,"ke ":634,"úp ":776,"gàn":962,"úng":1614,"kra":492,"kre":255,"kt ":229,"cũn":2434,"ểa":61690,"ku ":366,"km²":641,"kot":155,"kor":174,"kom":159,"kok":195,"ks ":691,"ể ":376292,"úy ":216,"cũ ":497,"kno":896,"kka":183,"khô":4285,"khó":360,"khí":730,"khú":467,"ko ":151,"khá":4090,"kle":735,"kla":228,"buể":254,"ểo":9548,"ểp":25031,"ểk":175,"ểm":59218,"ển":369560,"ểi":112828,"ểc":255331,"ểy":15079,"ểt":179295,"ểu":43805,"kaz":157,"gà ":270,"kat":204,"kar":238,"kas":219,"kan":739,"kal":182,"kam":185,"kai":366,"ka ":1184,"cùn":1840,"cúp":216,"ha ":8300,"ùng":27727,"ham":3795,"han":4769,"hao":550,"hap":378,"hai":4690,"hal":2163,"hau":2057,"hav":347,"har":4673,"has":721,"hat":786,"hae":501,"hag":376,"hab":337,"had":177,"hac":331,"hay":7652,"he ":18327,"dàn":837,"dài":1594,"hel":1664,"hei":3775,"hec":208,"hed":210,"hea":373,"hey":228,"hev":147,"het":420,"hes":679,"her":2719,"heo":10006,"hen":1600,"hem":538,"hi ":13142,"dây":222,"dân":28557,"dãy":398,"căn":327,"hig":332,"hie":278,"hid":1412,"hic":705,"hia":1555,"hip":346,"hio":999,"hin":3454,"him":2986,"ùy ":240,"hil":2244,"hik":232,"hii":300,"hiu":173,"his":1013,"hit":572,"hir":444,"hn ":286,"hla":242,"hle":1267,"hli":520,"hlo":240,"ho ":7256,"hma":350,"gma":174,"go ":1433,"giá":5603,"gme":307,"già":408,"glo":256,"gle":536,"gli":276,"gla":529,"gko":162,"gog":725,"gny":420,"ghĩ":2919,"gno":258,"gni":256,"gne":3863,"giú":291,"gna":708,"úa ":604,"giô":172,"gs ":297,"goz":262,"úc ":3114,"gom":139,"gol":384,"gon":1688,"gos":596,"gor":601,"gov":334,"gu ":148,"goà":839,"gro":663,"gra":1031,"gri":527,"gre":280,"gto":303,"gui":371,"gum":182,"gul":499,"có ":47334,"gua":512,"gue":1969,"gy ":222,"cô ":368,"guy":4577,"gur":185,"gus":453,"gun":159,"còn":3275,"úi ":1356,"côn":4644,"gyr":164,"iai":626,"iam":328,"ial":1014,"iao":1227,"ian":5609,"ias":397,"iar":265,"iat":696,"ic ":3037,"iac":1407,"iae":280,"ibl":173,"ibi":314,"ibo":164,"ibu":345,"id ":1469,"iba":306,"ibb":546,"ibe":1206,"ia ":29704,"iet":540,"ieu":473,"iel":1226,"ien":2075,"ier":2747,"ies":4767,"ied":1831,"ieg":187,"ig ":1226,"ifo":1413,"iff":148,"ife":614,"ifl":166,"ifi":693,"icr":1054,"ics":295,"ict":897,"icu":1741,"ico":2316,"ick":622,"ici":3591,"ich":3932,"ice":765,"ie ":3741,"ica":6048,"idu":239,"ids":175,"ido":567,"idi":2598,"ide":2455,"ida":26239,"iid":4750,"il ":3121,"ija":174,"iji":157,"im ":5476,"ika":371,"ige":770,"iga":742,"ii ":1370,"igm":416,"igh":923,"igi":710,"igu":318,"igs":187,"igr":220,"igo":300,"ign":1169,"iha":223,"ik ":169,"imo":742,"imm":158,"imp":567,"ime":1906,"imi":643,"ip ":810,"inc":1322,"ind":1704,"ina":4704,"inb":264,"imu":305,"inn":513,"ino":1469,"int":4526,"ins":1803,"inf":237,"ine":10837,"inh":13778,"ing":8141,"ini":3074,"inl":2046,"ink":351,"ioc":292,"iod":167,"inu":932,"inv":156,"inx":181,"iny":208,"iko":368,"iki":420,"ike":257,"ila":1297,"ilb":181,"in ":13262,"ilo":550,"ill":9682,"ilh":181,"ili":3385,"ild":355,"ile":3779,"ima":1310,"imb":551,"io ":1175,"ily":2239,"ils":2324,"ilu":323,"how":204,"hol":800,"hom":456,"hon":2401,"hoi":164,"hos":411,"hot":229,"hou":479,"hoo":159,"hop":241,"hor":2606,"hoa":5214,"hof":203,"hoe":206,"hod":441,"hoc":207,"hni":161,"hne":266,"dée":277,"hme":233,"hmi":203,"hiê":5040,"dép":1137,"hua":252,"htt":326,"htr":324,"hth":197,"hte":186,"hst":227,"hse":1600,"hoá":860,"hoà":1969,"hu ":18061,"hry":423,"hro":1154,"hre":321,"hri":577,"ùa ":1067,"ht ":687,"hra":468,"hya":190,"huê":158,"hyl":1269,"dòn":604,"hy ":171,"hwa":665,"hwe":313,"hum":443,"hun":1586,"hus":902,"hut":182,"hur":289,"huy":20578,"Vưể":237,"dùn":1738,"dù ":368,"hyt":176,"hys":223,"hyr":202,"huô":174,"ùi ":234,"ffe":313,"ffi":237,"fer":812,"báo":721,"bác":152,"fen":465,"bán":1218,"fel":1035,"fgh":188,"bà ":200,"fas":191,"far":156,"fam":2105,"fal":2041,"bày":158,"bàn":328,"bào":422,"bài":1285,"ff ":218,"eya":142,"ext":206,"eyr":325,"eyh":211,"eye":179,"exa":986,"ez ":316,"exi":1325,"exc":181,"ezu":350,"ezi":175,"eta":842,"ete":743,"eti":1014,"eth":666,"etn":165,"etl":616,"esp":684,"esn":283,"eso":340,"est":3808,"ess":1434,"esw":957,"ev ":202,"euc":337,"eud":540,"eui":263,"eum":201,"eto":465,"etr":999,"ets":459,"ett":1071,"etu":203,"etw":156,"etz":312,"ew ":2560,"eve":622,"eva":566,"evi":1144,"euv":222,"eut":564,"eur":1318,"eus":1354,"ex ":606,"euz":190,"eux":671,"ey ":1664,"ewa":232,"erö":586,"epe":234,"epi":495,"eph":665,"er ":9592,"epa":418,"eot":213,"eor":527,"eom":471,"eol":262,"eop":530,"eon":709,"es ":26910,"ept":1404,"epu":235,"epo":154,"erk":202,"erl":752,"eri":6361,"erg":3169,"erh":199,"ere":2445,"erf":557,"erc":1171,"erd":573,"era":3401,"erb":858,"et ":4591,"equ":162,"esl":202,"esh":1269,"esi":2021,"esc":975,"ese":753,"eu ":303,"esa":259,"erz":260,"ery":556,"erv":528,"eru":1012,"erw":278,"err":2985,"ert":1566,"ers":4623,"ern":4630,"erm":2107,"erp":282,"ero":2376,"eki":196,"en ":12018,"elb":206,"ela":2654,"eld":1258,"elf":146,"ele":1267,"eli":1798,"elg":160,"elm":262,"elk":404,"ell":9746,"elo":933,"elu":208,"els":642,"elt":280,"ely":341,"eo ":10593,"eiß":164,"emb":1047,"ema":1158,"eme":2947,"emm":256,"emo":539,"emi":1585,"emp":238,"ems":155,"emy":139,"enf":291,"ene":1945,"enh":484,"eng":730,"enb":2082,"ena":1558,"end":2473,"enc":2127,"eno":956,"enn":3389,"enk":343,"enl":192,"eni":2073,"enu":1664,"ens":4503,"ent":10489,"enr":274,"enz":420,"eny":515,"eoc":157,"ego":1228,"ege":478,"egg":139,"egi":670,"egr":229,"egu":327,"ek ":217,"eic":752,"eis":833,"eir":787,"eim":1035,"eil":952,"ein":4903,"eie":545,"eid":730,"eig":203,"eif":191,"el ":3452,"eiz":186,"eit":258,"em ":1658,"öst":587,"giu":194,"gis":185,"gil":310,"gin":1363,"gio":553,"gid":1547,"gic":199,"gia":8820,"ght":794,"băn":187,"gho":161,"ghi":3850,"ghe":189,"gha":488,"ggi":146,"gge":165,"câu":470,"cây":1629,"gi ":387,"gen":5062,"cán":798,"cáo":198,"ger":1519,"ges":1085,"gh ":607,"các":18946,"geb":333,"cái":701,"gem":243,"gel":544,"cá ":1776,"ge ":1808,"gae":147,"gai":1612,"gas":1474,"gar":1812,"gau":202,"gat":452,"gay":317,"gam":376,"gal":707,"gan":1692,"gap":295,"ga ":2819,"bút":167,"Tưể":239,"fur":264,"fus":239,"bón":2056,"ful":164,"fun":375,"ft ":439,"fra":309,"fre":507,"fri":815,"bín":794,"fro":1604,"fou":879,"for":2385,"fon":152,"fol":504,"bìn":10852,"bên":1143,"bí ":160,"fle":146,"fla":250,"fli":289,"flo":441,"fly":418,"fic":547,"fie":487,"fil":195,"fin":314,"fis":489,"da ":3697,"de ":8611,"dac":762,"dad":140,"dal":972,"dai":189,"dag":446,"dae":24708,"dat":480,"dar":398,"dap":183,"dan":4922,"dam":377,"cun":718,"cul":2298,"cum":328,"cua":546,"cty":284,"ctu":2164,"ctr":238,"cto":873,"cti":1458,"cte":647,"cta":670,"cy ":251,"cus":673,"cur":399,"cut":255,"cyc":140,"cks":263,"cki":166,"ckl":786,"cla":486,"chá":201,"cle":346,"châ":14860,"cky":262,"chí":8076,"chò":225,"clu":420,"chó":180,"cli":182,"ché":314,"chì":193,"clo":464,"chù":312,"co ":1548,"chú":1406,"coi":717,"cod":256,"coa":287,"cob":146,"coc":214,"con":3655,"col":1035,"com":2135,"cor":1035,"cos":565,"cop":801,"cot":443,"cou":1134,"coz":183,"cs ":366,"ct ":454,"cre":241,"cra":438,"chơ":592,"cri":479,"cro":1725,"chư":1623,"ccu":244,"cci":838,"cco":265,"cca":156,"cea":4824,"ch ":37737,"cer":1071,"ces":531,"cet":142,"cen":1512,"cep":386,"Xươ":220,"cel":1002,"ced":395,"cha":3033,"chw":543,"chu":6009,"chy":360,"cia":1452,"ck ":1564,"cie":3416,"cid":2524,"che":4294,"chl":1776,"chi":13490,"cho":7644,"chm":217,"chn":259,"chs":1959,"cht":448,"chr":711,"cil":2248,"cif":277,"cis":311,"cit":448,"cin":1085,"cio":750,"cip":1009,"cm ":153,"cke":753,"ed ":4935,"eba":285,"ebe":884,"ôn ":3722,"ebi":183,"ebo":301,"ebr":934,"ebs":216,"ebu":294,"ec ":362,"eac":334,"ôm ":271,"eag":170,"eaf":201,"eae":4646,"ead":460,"ean":1098,"eal":621,"ear":1037,"eas":698,"eat":882,"eau":1120,"eb ":303,"ea ":3947,"efo":223,"efe":249,"ei ":970,"ega":773,"een":1179,"eel":145,"eed":257,"ees":218,"eer":175,"eep":145,"eet":728,"edi":1118,"ede":1955,"ône":1757,"ông":42975,"eda":404,"eg ":341,"edt":284,"edo":1988,"edr":212,"eck":1046,"ech":639,"eci":3537,"ece":167,"eca":304,"ee ":1150,"ôme":730,"ecu":225,"ect":1210,"eco":767,"dwi":249,"dwe":183,"dwa":258,"dy ":278,"dur":468,"dus":275,"duy":1143,"ôi ":2569,"dor":2168,"dop":306,"don":2791,"dom":306,"dol":520,"dow":395,"dov":304,"dou":172,"dos":360,"ds ":1070,"doa":646,"doc":1372,"dog":488,"dun":458,"dul":398,"duc":230,"dri":731,"dra":523,"dt ":665,"dre":808,"du ":1084,"dro":986,"dha":326,"dge":229,"dic":469,"did":312,"dia":1872,"ôte":1594,"der":3477,"des":3199,"ômé":10659,"dea":701,"ded":323,"dec":231,"del":1465,"den":4157,"dem":911,"deo":220,"di ":3427,"dle":656,"dla":192,"do ":4210,"diu":260,"din":1244,"dio":353,"dis":2108,"dit":379,"die":2847,"dil":249,"rgy":150,"rgu":287,"rhe":221,"rha":208,"rho":196,"rga":532,"ri ":2301,"rgi":1202,"rgh":152,"rge":1773,"rgo":1290,"rgn":567,"ret":1055,"res":3724,"rev":398,"reu":912,"rex":297,"rey":288,"rfa":190,"rfl":255,"nân":179,"rdu":164,"rds":270,"rdr":159,"này":31504,"rg ":4949,"reb":939,"rea":2006,"ree":1006,"ref":247,"rec":523,"red":781,"rei":1635,"reg":1203,"rem":776,"ren":4076,"rel":1428,"rer":300,"reo":199,"rep":265,"rf ":1207,"rda":570,"rcu":715,"rct":554,"rdo":799,"nào":548,"rdi":1755,"rde":2543,"re ":9446,"rbu":335,"rco":417,"rci":274,"rch":1874,"rce":427,"rca":280,"ray":496,"raz":574,"rd ":2860,"rao":655,"rap":552,"raq":160,"rar":494,"ras":1804,"rat":1838,"rau":875,"rav":215,"rbi":1026,"rbo":389,"rba":580,"rbe":418,"rai":3535,"rah":190,"rag":1656,"ran":8266,"ram":1112,"ral":2485,"rak":217,"rab":584,"raf":217,"rae":749,"rad":1615,"rac":3183,"rpu":270,"rpo":754,"rs ":2330,"rpe":165,"rpa":404,"rpi":173,"rph":340,"ror":201,"ros":1675,"rot":1221,"rom":2423,"ron":57984,"lĩn":607,"roo":669,"rop":3656,"roy":149,"rou":1200,"rov":1781,"row":593,"rob":744,"roa":358,"rod":644,"roc":1935,"roi":481,"rol":951,"rof":216,"rog":648,"nên":713,"rno":358,"rns":159,"rna":1180,"rne":2695,"rni":1419,"riè":172,"ném":189,"riê":408,"rmo":497,"rms":184,"ro ":929,"rma":3617,"née":2905,"rme":709,"rmi":637,"rly":155,"rlo":157,"rli":457,"rld":399,"rle":351,"rla":627,"rn ":3576,"hơn":1695,"hơi":801,"rki":204,"rke":393,"rka":202,"né ":169,"rm ":302,"riz":418,"rix":394,"rl ":161,"rip":365,"rio":1072,"rit":3591,"ris":2983,"riv":318,"riu":456,"rig":1218,"rii":3008,"ril":1020,"rin":4090,"rim":450,"ria":5391,"rib":1175,"ric":5324,"rid":4461,"rie":2155,"rif":320,"rk ":1670,"hơ ":851,"rwe":155,"nói":685,"rz ":208,"hư ":5084,"nôn":377,"rya":216,"ryc":442,"rug":231,"rue":707,"ruc":377,"rup":171,"run":25178,"rum":705,"rul":242,"ruy":2675,"ruz":213,"rus":1414,"rva":295,"rvi":473,"rve":224,"rwa":347,"ry ":2417,"rsi":891,"rso":252,"roß":201,"rsc":361,"rsd":187,"rsa":1123,"rsb":257,"rsh":319,"rse":899,"rta":737,"óc ":726,"rst":546,"rto":415,"rtb":145,"rte":1749,"rth":2805,"rti":1215,"nó ":2643,"rub":265,"rts":371,"rtr":430,"rtu":180,"rtt":513,"rt ":1957,"óa ":2701,"rqu":185,"rro":1564,"rrh":197,"rri":1788,"rre":2216,"rra":3074,"ru ":477,"rry":300,"rru":147,"sab":140,"sac":1864,"sai":281,"sak":172,"sal":439,"sam":384,"sba":297,"sbe":425,"sao":776,"óng":4090,"san":1255,"sau":2887,"sat":228,"sas":328,"sar":660,"oà ":337,"óp ":288,"sa ":2015,"núi":1191,"óm ":1521,"ón ":2564,"rze":166,"hưa":582,"rys":387,"ryo":172,"ryp":167,"ryl":162,"ói ":993,"ryn":172,"hưn":1523,"sha":619,"năm":27971,"năn":1182,"sho":293,"shr":233,"sht":381,"she":876,"shi":1264,"si ":634,"oãn":159,"sge":580,"sie":368,"sid":739,"sic":687,"sia":3700,"sk ":591,"shw":300,"shu":294,"sit":709,"sis":3021,"sip":265,"sin":5091,"sio":465,"sil":1895,"sim":553,"sii":258,"sif":151,"sig":220,"scr":307,"scu":281,"òng":2223,"oài":34230,"oàn":5706,"sdo":465,"sbu":460,"se ":4575,"oá ":380,"sca":1351,"sce":236,"sci":1194,"sch":1792,"sco":719,"sey":198,"ser":1028,"ses":272,"set":418,"oát":273,"seu":551,"sh ":1939,"sfe":321,"sea":715,"sei":161,"see":284,"sed":245,"sec":229,"sep":202,"sen":3196,"oán":1638,"sem":1371,"sel":2101,"hươ":4318,"spo":375,"shū":247,"spr":259,"sph":157,"spe":3907,"spi":1229,"spa":351,"sot":292,"sou":945,"sol":467,"som":376,"son":1789,"sop":308,"sor":191,"sof":194,"soi":556,"soc":217,"su ":286,"sra":294,"st ":2960,"squ":256,"ss ":872,"sli":218,"sky":167,"kiể":2684,"sla":1569,"sle":323,"ski":288,"sks":212,"khể":3237,"ska":369,"sna":2980,"sni":207,"sne":250,"smo":583,"siê":264,"so ":947,"sma":1028,"smi":179,"swi":1048,"syn":165,"syl":224,"sse":3231,"soá":206,"ssa":1676,"sso":1413,"ssi":3339,"ssu":544,"ste":5951,"stf":168,"sth":161,"sta":3335,"sto":2068,"stp":174,"sti":4685,"stl":165,"stu":445,"str":4349,"sts":262,"sty":144,"sub":863,"sul":490,"sum":304,"sup":191,"sun":266,"sus":998,"sur":1961,"suy":191,"òa ":5602,"sy ":172,"swa":243,"tai":3180,"tak":236,"tal":2442,"tae":263,"tag":338,"tah":173,"tab":252,"tac":777,"tad":502,"tay":436,"tax":151,"tau":185,"tat":1298,"tas":231,"tar":1441,"tap":149,"tan":2995,"tam":442,"tch":446,"òm ":317,"te ":8300,"tbu":170,"òn ":4112,"ta ":6742,"ký ":793,"ozè":153,"pa ":420,"làm":1846,"làn":610,"lá ":624,"pe ":1111,"par":2272,"pat":396,"pas":155,"là ":152991,"pac":448,"pag":1801,"pal":1430,"pan":1032,"phe":584,"pha":2087,"phu":325,"phr":290,"pho":3352,"phn":208,"phi":5842,"pi ":273,"lãn":895,"ph ":150,"lâu":392,"lâm":192,"pea":417,"pec":3297,"ped":468,"pen":678,"per":2665,"pet":646,"pes":3121,"pel":638,"pla":748,"hiể":23255,"pli":245,"phâ":2752,"phá":7538,"ple":773,"lès":368,"phí":12064,"phê":328,"plo":417,"phé":481,"phò":631,"phó":453,"phy":787,"pia":524,"pid":686,"pic":879,"pie":144,"pil":547,"pin":2627,"pio":231,"pir":455,"pis":429,"pit":358,"por":1000,"pop":359,"pot":426,"pos":699,"pom":618,"pon":447,"pol":658,"poc":162,"pod":1012,"ps ":851,"hū ":248,"ppi":1358,"ppo":224,"ppe":777,"phú":224,"po ":176,"lí ":1217,"lên":761,"pta":227,"pse":181,"psi":853,"pso":273,"ptu":208,"pua":385,"pub":253,"puc":309,"pte":1527,"pti":1324,"pto":542,"ptr":151,"pra":304,"hoể":8808,"pt ":251,"phư":1981,"pri":732,"pre":749,"pro":835,"huể":53256,"lôn":185,"lôm":10657,"pur":817,"pus":345,"pun":212,"pul":573,"lô ":685,"px ":1130,"pyr":166,"lý ":2650,"lúc":371,"mà ":1433,"màn":167,"máy":1955,"mã ":867,"màu":877,"mét":23053,"méo":221,"mìn":517,"qua":5837,"mô ":773,"quy":3085,"que":2033,"qui":2180,"món":229,"môn":561,"môi":520,"quâ":3680,"quá":966,"quê":311,"quý":155,"mùa":374,"ra ":16536,"ngo":2830,"ngi":1741,"ngl":604,"ngk":252,"ngu":5422,"ngr":255,"ngt":407,"ngs":604,"ni ":1381,"nge":3237,"ngh":7783,"nga":2179,"nho":251,"ndé":306,"nhu":514,"nha":2433,"nhi":5815,"nhe":263,"neg":239,"nei":512,"nel":1599,"nen":680,"nem":262,"neo":282,"ner":1201,"net":904,"nes":4373,"nev":164,"neu":463,"ng ":357107,"nea":1462,"neb":278,"nec":300,"ned":346,"nee":182,"nfo":163,"ney":270,"nez":428,"nh ":132657,"nfe":333,"nct":449,"nco":534,"nci":1276,"ncl":257,"nce":3207,"nch":2383,"nca":570,"ne ":19277,"nbu":1513,"ndu":416,"ndr":1581,"nds":990,"ndo":2398,"ndl":619,"ndh":176,"ndi":4609,"nde":3318,"nda":1999,"ncy":170,"nal":1151,"nam":13111,"nan":913,"nar":774,"nac":1244,"nad":927,"nae":1024,"nag":697,"nai":3249,"nbo":145,"nbe":462,"nd ":12230,"nba":380,"nav":164,"nau":516,"nat":2078,"nas":580,"nay":2459,"na ":6835,"iúp":291,"가 ":328,"cư ":415,"iôn":176,"myc":163,"nya":523,"nyi":193,"nz ":267,"ny ":1333,"nvi":479,"nx ":222,"nul":298,"num":443,"nus":3075,"nut":506,"nty":550,"nto":2015,"ntu":367,"nts":334,"ntr":2166,"nti":3110,"nth":1365,"ntl":140,"nta":3026,"nte":4904,"nsu":730,"nsy":214,"nso":325,"nst":981,"nsf":180,"nse":669,"nsh":461,"nsi":2204,"nsl":567,"nsk":139,"nsc":152,"nsa":421,"nsb":373,"như":5336,"nt ":8221,"ngư":22262,"nqu":229,"ns ":3001,"noc":411,"nod":303,"hĩa":2885,"nob":365,"nol":370,"noi":432,"nop":786,"nom":551,"non":799,"not":862,"nos":776,"nor":939,"now":1063,"nov":236,"nou":281,"nne":5126,"nna":750,"nno":345,"nni":716,"nns":289,"nma":453,"niê":790,"nhâ":2593,"nn ":413,"nla":2159,"nhá":489,"nhà":6343,"nhó":1299,"nly":637,"no ":1110,"hĩ ":710,"nhì":354,"ngâ":223,"nke":271,"nki":377,"ngà":10009,"nka":598,"ngô":1896,"nkt":202,"nja":290,"nii":438,"nig":289,"nif":242,"nie":473,"nid":3745,"nic":2037,"nia":5524,"nk ":242,"nix":164,"niu":181,"niv":273,"nis":1435,"nit":1158,"nio":409,"nim":283,"nin":1178,"nik":149,"nil":357,"ogr":267,"ogu":214,"ogi":228,"ogl":349,"ogo":319,"ogn":1408,"oga":317,"oge":346,"oi ":1139,"ohn":303,"oha":229,"ohe":168,"ogy":195,"ois":1103,"oir":3370,"oit":1455,"oin":300,"oil":149,"oid":1389,"oie":174,"ok ":359,"ol ":657,"oce":820,"och":1882,"oci":330,"ock":741,"ocl":146,"oco":568,"ocr":154,"obu":172,"oe ":211,"oca":1110,"occ":493,"ode":1045,"odi":450,"odo":1089,"odr":210,"oct":1918,"ocy":317,"of ":6954,"oda":691,"dươ":848,"oen":367,"odu":285,"oed":204,"og ":551,"oft":282,"off":252,"ofe":161,"oa ":13629,"oc ":1250,"oan":1921,"oad":230,"oba":818,"od ":1168,"oar":504,"oas":324,"oat":240,"obo":175,"obr":145,"obl":325,"obi":968,"obe":655,"nym":211,"nza":905,"nze":176,"oya":255,"oxy":147,"oxi":202,"oz ":147,"guể":1293,"ows":383,"owl":142,"own":1769,"owi":183,"ozo":255,"oza":724,"otu":288,"oud":174,"oub":187,"ouc":419,"oua":154,"ow ":1038,"oti":1060,"oth":1496,"ote":697,"ott":720,"ots":282,"otr":764,"oto":966,"ost":2982,"osu":324,"ota":1031,"ov ":258,"osi":491,"osh":227,"ose":2073,"osg":493,"osp":425,"oss":826,"osm":547,"oso":413,"osn":140,"oy ":273,"owe":547,"ovi":954,"ovo":159,"ouv":263,"oux":242,"ova":705,"ove":2204,"oug":763,"oui":391,"oul":679,"oun":2537,"oup":387,"ous":2368,"our":3149,"out":2276,"opo":1519,"opp":188,"opi":1120,"opl":220,"ope":1422,"oph":3191,"opa":411,"os ":2470,"opu":314,"opt":1403,"ops":1255,"oon":440,"ool":189,"oom":247,"ook":283,"oog":197,"ood":513,"or ":3376,"oot":347,"oor":232,"ork":631,"orl":473,"orm":2975,"orn":1953,"oro":870,"orp":1047,"orr":2602,"orc":598,"ord":3003,"ore":2045,"orf":1239,"org":708,"ori":3584,"ou ":1707,"osa":1094,"osc":366,"ort":2762,"ors":1266,"oru":667,"ory":1225,"kín":427,"kíc":629,"ot ":1197,"goể":939,"m² ":639,"orb":599,"ora":2404,"oqu":150,"ola":2194,"old":517,"kê ":1566,"giể":12764,"on ":15999,"oli":2444,"oll":1993,"olf":295,"ole":1581,"ols":1043,"olt":149,"olm":148,"olo":2220,"oly":684,"olz":312,"olu":1314,"olv":159,"oka":251,"ghể":2336,"om ":2129,"okk":143,"oki":163,"oke":162,"oku":288,"ona":2358,"ond":2416,"onc":461,"onf":214,"one":2805,"ong":59905,"oni":4577,"onl":675,"onn":2036,"kên":176,"ono":1589,"ons":1629,"ont":3656,"onu":697,"onv":312,"ony":549,"gũ ":341,"kí ":158,"oma":3160,"oo ":152,"ome":1667,"omb":1058,"omi":1212,"omm":2408,"omp":760,"omo":798,"kéo":312,"omu":461,"omy":255,"op ":502,"kì ":200,"la ":10761,"ính":11878,"ín ":1003,"há ":535,"le ":10516,"lca":204,"ít ":628,"lch":163,"lf ":224,"lde":761,"ldb":159,"lda":161,"hào":150,"ldo":198,"hàn":21780,"hàm":435,"hài":213,"ldi":250,"lab":417,"lac":1406,"lad":738,"lae":401,"lah":182,"lag":424,"laj":292,"lai":1546,"lal":167,"lan":10384,"lam":1066,"lap":209,"lao":259,"lar":4422,"lat":2293,"las":1229,"law":585,"lau":512,"lav":645,"lay":1413,"lba":423,"hà ":7105,"ld ":1873,"lbe":635,"lbi":189,"lbo":527,"lbu":797,"gô ":876,"góc":144,"ky ":446,"ích":17834,"cuể":3904,"gôi":662,"góp":269,"gôn":1009,"hìn":3185,"llé":141,"hìm":147,"lpe":2886,"lpi":216,"lph":225,"ls ":3004,"híc":534,"hía":11942,"lok":158,"lon":3179,"lom":1001,"lop":1213,"lor":1602,"lod":250,"loc":750,"loe":167,"log":693,"loi":242,"los":940,"lot":496,"lou":372,"lov":598,"low":793,"hêm":276,"lob":302,"hí ":2215,"liê":1520,"hép":885,"lmo":317,"lme":324,"lma":282,"hì ":1109,"lti":486,"lto":187,"hó ":498,"lud":253,"luc":198,"lue":351,"lso":351,"lst":1066,"lta":363,"lte":462,"lu ":183,"lse":196,"loà":31500,"lsa":619,"ía ":12221,"hín":8169,"lt ":1144,"lhe":150,"lha":193,"hãn":1016,"lge":195,"li ":845,"lga":417,"hât":447,"hâu":5143,"hân":32338,"hâm":245,"lfe":172,"ley":541,"lex":375,"leu":725,"lev":279,"les":4804,"hát":4309,"let":890,"ler":1633,"leo":683,"háo":303,"lep":286,"háp":24791,"hám":208,"lem":456,"len":2814,"hán":13040,"lel":163,"lei":443,"hái":3807,"leg":371,"lef":140,"led":786,"hác":3511,"lec":514,"leb":324,"lea":958,"lls":526,"llu":1871,"lly":728,"lo ":703,"lla":8252,"llb":208,"lle":6069,"lli":4980,"llo":3130,"lks":330,"hê ":529,"diể":17680,"lka":390,"lm ":305,"ll ":2030,"hè ":211,"lit":1006,"lis":1811,"lip":1960,"lio":1480,"lin":3515,"lim":803,"liz":196,"liv":616,"liu":661,"lic":1528,"lid":3931,"lia":3554,"lk ":142,"lik":204,"lii":571,"lig":457,"lie":1623,"lif":1017,"ma ":3578,"húa":466,"húc":1589,"hún":1289,"hút":209,"húy":143,"mb ":269,"ìm ":8521,"mac":644,"mai":885,"maj":203,"mad":498,"mae":190,"ìn ":392,"mag":301,"hür":727,"mar":2810,"mas":488,"mal":1566,"man":6200,"maz":173,"mat":1698,"mba":981,"mbl":196,"mbi":1974,"mbe":1329,"mbr":351,"mbo":504,"me ":3769,"iá ":976,"iße":139,"mbu":270,"ình":20204,"iàn":475,"med":457,"meg":343,"mea":185,"iác":256,"mec":144,"met":1158,"iáp":2001,"mes":1080,"mer":2891,"iám":415,"mem":171,"mel":1053,"iáo":2698,"ián":260,"men":4045,"mei":235,"luy":242,"hòa":4573,"lva":958,"hô ":347,"lve":227,"lvi":160,"lul":235,"lun":153,"lum":1803,"lut":702,"lus":2401,"ly ":5413,"hóa":2227,"hòm":296,"hòn":1026,"lz ":1932,"hôi":152,"hón":422,"hóm":1446,"hôn":8959,"luô":144,"lyp":567,"lym":264,"lyn":185,"hù ":377,"hú ":960,"hùn":400,"hùa":429,"mpi":548,"mph":909,"mpe":699,"mpr":166,"mpo":339,"mpl":212,"mpu":526,"mps":257,"ms ":540,"moc":217,"mod":141,"mon":2835,"mop":418,"mol":919,"mor":1191,"mos":484,"mot":349,"mou":592,"mpa":2137,"mu ":149,"mua":252,"mst":192,"my ":346,"mur":466,"mus":478,"mul":387,"mun":1488,"hăm":190,"hăn":308,"mi ":443,"min":2275,"mil":2845,"mir":352,"mis":870,"mit":1161,"cơ ":2355,"mic":1181,"mib":195,"mia":637,"mie":192,"mid":343,"mo ":242,"ièr":623,"mm ":173,"iêu":3120,"mni":139,"iêm":793,"iên":15316,"mno":232,"mmu":465,"mmi":508,"miê":868,"mmo":992,"mma":550,"mme":1127,"xâm":158,"xây":774,"xã ":6613,"xác":753,"thể":105642,"tiể":16334,"vĩ ":1285,"Để ":4107,"suể":581,"Đểt":393,"Đểu":259,"Đểa":425,"Đểc":8666,"Đểi":2655,"Đển":2345,"Đểo":612,"văn":2376,"soể":246,"vô ":966,"vòn":456,"zue":365,"zur":1019,"ruể":225,"võ ":369,"hưể":6739,"vùn":22597,"vây":180,"vào":6478,"vàn":747,"zen":390,"zel":179,"zer":396,"ze ":541,"vài":326,"và ":39376,"zam":416,"zan":947,"zak":194,"zar":156,"ví ":164,"zon":909,"zo ":189,"vì ":1170,"zna":154,"riể":3937,"zia":217,"zie":194,"zin":147,"zil":497,"yré":2878,"yx ":207,"yth":348,"yst":455,"yso":284,"ysi":882,"yri":846,"yro":276,"yra":354,"yrg":171,"yre":315,"ys ":1775,"yph":446,"ypt":526,"ypr":406,"ypo":300,"ype":314,"yon":322,"uý ":440,"za ":604,"gưể":22834,"uôn":11800,"uôi":272,"quể":14720,"ye ":286,"uá ":683,"yca":261,"yce":225,"ych":303,"ycl":287,"yco":153,"yct":483,"ydr":465,"yer":937,"uán":332,"yen":509,"ya ":1624,"yat":304,"yan":957,"yal":151,"uê ":487,"yla":353,"yle":207,"yli":383,"yll":1078,"ylo":482,"ylv":243,"ylu":187,"yma":185,"sĩ ":1783,"yo ":184,"yme":163,"ymp":602,"ymn":164,"yna":220,"yne":184,"yno":257,"uân":5407,"yi ":308,"tăn":371,"yho":211,"yin":196,"tín":2825,"để ":24120,"tíc":15647,"đểy":569,"xtr":322,"đểu":7023,"đểt":4542,"đểa":6133,"đểi":9089,"đểc":6272,"đển":28910,"đểo":4093,"đểp":378,"đểm":552,"tên":7356,"tìn":624,"tìm":8334,"xon":351,"xoa":253,"tù ":245,"tô ":345,"tòa":324,"xuy":162,"xun":238,"tôn":573,"săn":488,"xi ":160,"tây":9977,"tâm":1507,"tán":178,"xem":772,"tác":2521,"tái":142,"tàn":309,"tàu":1978,"nơi":1970,"phể":19501,"xil":335,"xin":153,"xic":980,"xa ":336,"tài":1059,"xce":150,"xe ":304,"xas":712,"xan":855,"ww ":171,"www":171,"són":349,"sôn":1977,"wo ":144,"yểu":888,"yểt":2656,"yển":27686,"sên":250,"wn ":1674,"sèr":449,"ws ":411,"wor":456,"woo":295,"we ":324,"sân":742,"wes":568,"wer":566,"sáu":219,"sát":605,"wen":170,"sán":1258,"wel":193,"wei":504,"wed":169,"wee":268,"web":356,"sác":1906,"whe":719,"whi":453,"răn":535,"sâu":259,"wi ":419,"wit":1057,"wig":1125,"wid":265,"wic":172,"win":477,"wil":183,"rös":588,"lưu":884,"vuô":11318,"dưể":1608,"rùn":419,"wa ":340,"rúc":638,"wan":646,"wal":651,"way":264,"wat":547,"war":864,"was":494,"wai":552,"rüc":145,"ría":161,"vre":519,"rò ":634,"vua":1478,"vul":314,"rõ ":263,"ròn":162,"rôm":369,"via":1024,"vir":262,"vil":2132,"vin":903,"vig":163,"vic":424,"vid":453,"vie":338,"ngể":4112,"vit":212,"vis":359,"nhể":20937,"ré ":197,"niể":631,"rén":2896,"viê":2261,"rì ":216,"rên":16855,"rí ":838,"voi":343,"vol":291,"von":231,"vor":176,"rìn":2591,"vi ":814,"râu":162,"rãi":238,"ver":2511,"ves":443,"vet":169,"ràn":171,"rào":384,"rái":592,"rán":159,"ven":2308,"vel":746,"rác":292,"ve ":1535,"val":496,"vak":211,"van":1011,"var":683,"vat":444,"rà ":170,"vad":360,"vai":638,"va ":1421,"uyê":4853,"cưể":274,"uze":169,"uzn":158,"uya":181,"uxe":167,"uxo":172,"muể":253,"ux ":1397,"uvi":315,"uve":890,"uy ":4510,"usk":685,"ush":495,"usi":1086,"use":2256,"usc":486,"usa":422,"usu":188,"ust":2067,"uss":2125,"utm":305,"uth":2069,"uti":986,"ute":4039,"uta":488,"utt":513,"uts":241,"utu":168,"uto":356,"us ":12540,"ut ":1468,"urb":527,"ura":1745,"urc":370,"ure":2141,"urg":4458,"uri":3154,"urk":298,"urn":505,"uro":993,"urp":172,"urr":1188,"urs":436,"urt":1515,"uru":494,"ury":261,"ur ":3979,"uph":337,"upi":208,"upe":461,"upl":256,"umi":350,"umo":202,"uma":655,"umb":1047,"ume":427,"unt":1093,"uns":220,"uni":1404,"unn":257,"unc":850,"und":2117,"una":410,"ung":28650,"une":861,"up ":561,"uki":214,"um ":5338,"ulu":810,"ult":827,"ulo":594,"ulm":200,"ull":475,"uli":2981,"ulg":343,"ule":468,"ulc":170,"ula":2994,"ulb":224,"miể":7160,"un ":876,"uid":1911,"uil":817,"uin":1133,"uis":537,"uit":1805,"ul ":704,"ugh":635,"uge":202,"ugl":142,"ui ":422,"uga":249,"ugu":343,"uco":341,"uct":140,"ucu":155,"uda":464,"ude":797,"udi":503,"ubs":175,"ubr":196,"uca":423,"ue ":2123,"uce":268,"ucc":527,"uci":266,"uch":1204,"ucl":244,"uck":465,"uet":161,"uev":189,"uer":585,"ues":940,"uff":186,"udo":607,"udw":163,"uee":563,"ued":1142,"ueb":187,"uen":891,"uel":938,"ub ":179,"ua ":4516,"uay":450,"uat":643,"uar":339,"ual":262,"uan":4521,"ubi":473,"ubl":385,"ube":682,"ubf":310,"uba":515,"ud ":256,"uai":209,"uad":968,"tze":378,"tyl":438,"typ":380,"bưể":9335,"trư":5334,"trù":418,"ty ":2056,"trú":686,"trò":743,"trí":887,"trì":2564,"trê":16809,"trá":625,"trà":444,"tvi":145,"tuy":1965,"tur":1038,"tus":1579,"tut":187,"tui":1819,"tul":991,"tun":226,"tum":525,"tud":242,"tuc":215,"luể":1046,"tz ":904,"two":235,"tră":326,"ts ":2029,"lũn":205,"tre":2316,"loể":3367,"tt ":262,"tra":13075,"thơ":816,"tri":6787,"tru":17464,"tro":55540,"thư":5419,"tu ":263,"try":204,"toá":1052,"tsc":237,"toà":1057,"tsu":330,"tsw":157,"tta":873,"tte":2605,"tti":475,"ttl":380,"tto":705,"ttp":325,"tts":284,"thă":146,"tme":444,"tma":160,"thú":506,"thù":145,"to ":4755,"thô":2398,"tiê":3084,"tp ":325,"tna":209,"toe":205,"tod":450,"toc":558,"toi":213,"tog":204,"tob":196,"tou":1449,"tos":429,"tot":153,"tow":612,"tom":1725,"ton":3013,"tol":487,"tor":2310,"top":602,"tr ":327,"tii":370,"til":3476,"tif":586,"tie":501,"tig":741,"tir":204,"tiq":562,"tit":511,"tis":1648,"tin":5106,"tim":1567,"tip":239,"tio":2405,"thy":410,"thu":53072,"thw":183,"tia":1585,"tic":3507,"tid":1768,"tiu":237,"tiv":602,"tli":260,"thê":286,"thé":143,"thí":558,"thì":784,"liể":1722,"tla":967,"thâ":11890,"thà":11154,"tle":1005,"thá":12170,"tem":2257,"ten":2299,"tep":155,"tei":2098,"tel":2040,"tee":338,"teg":403,"tea":605,"tec":172,"ted":1492,"tfa":159,"th ":4032,"tev":154,"teu":168,"tet":184,"tes":3544,"ter":9430,"ti ":720,"tho":1935,"thm":205,"thr":908,"the":21046,"thi":5961,"tha":4235,"之三 ":187,"之万 ":161,"rưể":6336,"之专 ":252,"ăk ":141,"ăm ":28988,"ăn ":5275,"ăng":3284,"xưa":183,"vươ":541,"nưể":35806,"丘 ":240,"mưể":240,"专 ":1368,"xuể":3053,"並 ":377,"tươ":996,"lưể":2704,"三 ":1277,"丁 ":542,"万 ":771,"uyể":30286,"乙 ":247,"sư ":673,"tư ":1117,"zèr":162,"vũ ":670,"ürt":532,"üri":758,"viể":4820,"rươ":293,"rưn":427,"tuể":1284,"trể":43603,"yêu":353,"之 ":589,"yên":4958,"sơ ":163},"n_words":[13809827,17315344,13396979],"name":"vi","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/lc-btrie/CMakeLists.txt b/contrib/lc-btrie/CMakeLists.txt
new file mode 100644
index 0000000..a5fe9e5
--- /dev/null
+++ b/contrib/lc-btrie/CMakeLists.txt
@@ -0,0 +1,6 @@
+SET(LCTRIESRC btrie.c)
+ADD_LIBRARY(lcbtrie STATIC ${LCTRIESRC})
+
+SET(LCTRIE_CFLAGS "-DBUILD_RSPAMD")
+
+set_target_properties(lcbtrie PROPERTIES COMPILE_FLAGS "${LCTRIE_CFLAGS}") \ No newline at end of file
diff --git a/contrib/lc-btrie/btrie.c b/contrib/lc-btrie/btrie.c
new file mode 100644
index 0000000..81b69b2
--- /dev/null
+++ b/contrib/lc-btrie/btrie.c
@@ -0,0 +1,2644 @@
+/* Level-Compressed Tree Bitmap (LC-TBM) Trie implementation
+ *
+ * Contributed by Geoffrey T. Dairiki <dairiki@dairiki.org>
+ *
+ * This file is released under a "Three-clause BSD License".
+ *
+ * Copyright (c) 2013, Geoffrey T. Dairiki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * * Neither the name of Geoffrey T. Dairiki nor the names of other
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GEOFFREY
+ * T. DAIRIKI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*****************************************************************
+ *
+ * This code implements a routing table conceptually based on a binary
+ * trie structure. Internally, the trie is represented by two types
+ * of compound nodes: "multibit nodes", which contain the top few
+ * levels of an entire binary subtree; and "level compression" (LC)
+ * nodes which represent a (potentially long) chain of out-degree one
+ * (single child) binary nodes (possibly ending at a terminal node).
+ *
+ * The multibit nodes are represented using a "Tree Bitmap" structure
+ * (more on this below), which is very efficient --- both in terms of
+ * memory usage and lookup speed --- at representing densely branching
+ * parts of the trie. The LC nodes can efficiently represent long
+ * non-branching chains of binary trie nodes. Using both node types
+ * together results in efficient representation of both the sparse and
+ * dense parts of a binary trie.
+ *
+ * Graphically, here's the rough idea:
+ *
+ * ........
+ * .LC o .
+ * . / . LC nodes can
+ * . o . <= represent long chains
+ * . \ . of (non-branching) binary
+ * . o . trie nodes
+ * . / .
+ * . o .
+ * ......../.....
+ * .TBM o .
+ * . / \ . TBM nodes can represent
+ * . o * . <= several levels of densely
+ * . / \ . branching binary trie nodes
+ * . o o .
+ * ......./.....\.......
+ * .TBM o .. o LC.
+ * . / \ .. \ .
+ * . o o .. o .
+ * . / / \ .. \ .
+ * . * o *.. o .
+ * ...../....... / .
+ * . o LC. . o .
+ * . \ . .....\......
+ * . * . . o TBM.
+ * ........ . / \ .
+ * . o o .
+ * . / \ \ .
+ * .* * *.
+ * ...........
+ *
+ * Terminology
+ * -----------
+ *
+ * node
+ * Usually, in the comments below, "node" will be used to refer to
+ * a compound node: either a multibit (TBM) node or an LC node.
+ *
+ * "internal node" or "prefix"
+ * The terms "prefix" or "internal node" are used to refer to
+ * a node in the binary trie which is internal to a multibit (TBM)
+ * node.
+ *
+ * ----------------------------------------------------------------
+ *
+ * Internal Representation of the Nodes
+ * ====================================
+ *
+ * Multibit (TBM) Nodes
+ * ~~~~~~~~~~~~~~~~~~~~
+ *
+ * The multibit nodes are represented using a "Tree Bitmap" (TBM)
+ * structure as described by Eatherton, Dittia and Varghese[1]. See
+ * the paper referenced below for basic details.
+ *
+ * A multibit node, represents several levels of a binary trie.
+ * For example, here is a multibit node of stride 2 (which represent
+ * two levels of a binary trie.
+ *
+ * +------- | ------+
+ * | multi o |
+ * | bit / \ |
+ * | node / \ |
+ * | o * |
+ * +--- / \ - / \ --+
+ * O
+ *
+ * Note that, for a multibit node of stride S, there are 2^S - 1 internal
+ * nodes, each of which may have data (or not) associated with them, and
+ * 2^S "external paths" leading to other (possibly compound nodes).
+ * (In the diagram above, one of three internal node (the one denoted by "*")
+ * has data, and one of four extending paths leads to an external node
+ * (denoted by the 'O').)
+ *
+ * The TBM structure can represent these bitmaps in a very memory-efficient
+ * manner.
+ *
+ * Each TBM node consists of two bitmaps --- the "internal bitmap" and the
+ * "extending paths bitmap" --- and a pointer which points to an array
+ * which contains both the extending path ("child") nodes and any
+ * internal prefix data for the TBM node.
+ *
+ * +--------+--------+
+ * TBM | ext bm | int bm |
+ * Node +--------+--------+
+ * | pointer |----+
+ * +-----------------+ |
+ * |
+ * |
+ * +-----------------+ |
+ * | extending path | |
+ * | node[N-1] | |
+ * +-----------------+ |
+ * / ... / |
+ * / ... / |
+ * +-----------------+ |
+ * | extending path | |
+ * | node[0] | |
+ * +-----------------+<---+
+ * | int. data[M-1] |
+ * +-----------------+
+ * / ... /
+ * +-----------------+
+ * | int. data[0] |
+ * +-----------------+
+ *
+ * The extending paths bitmap (or "ext bitmap") has one bit for each
+ * possible "extending path" from the bottom of the multibit node. To
+ * check if a particular extending path is present, one checks to see if
+ * the corresponding bit is set in the ext bitmap. The index into the
+ * array of children for that path can be found by counting the number
+ * of set bits to the left of that bit.
+ *
+ * Similarly, the internal bitmap has one bit for each binary node
+ * which is internal to the multibit node. To determine whether there
+ * is data stored for an internal prefix, one checks the corresponding
+ * bit in the internal bitmap. As for extending paths, the index into
+ * the array of internal data is found by counting the number of set
+ * bits to the left of that bit.
+ *
+ * To save space in the node structure, the node data array is stored
+ * contiguously with the node extending path array. The single
+ * ("children") pointer in the TBM structure points to the beginning
+ * of the array of extending path nodes and to (one past) the end of
+ * the the internal data array.
+ *
+ * The multibit stride is chosen so that the entire TBM node structure fits
+ * in the space of two pointers. On 32 bit machines this means the stride
+ * is four (each of the two bitmaps is 16 bits); on 32 bit machines the
+ * stride is five.
+ *
+ * Note that there are only 2^stride - 1 internal prefixes in a TBM
+ * node. That means there is one unused bit in the internal bitmap.
+ * We require that that bit must always be clear for a TBM node. (If
+ * set, it indicates that the structure represents, instead, an LC
+ * node. See below.)
+ *
+ * ----------------------------------------------------------------
+ *
+ * Level Compression (LC) Nodes
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * LC nodes are used to represent a chain of out-degree-one (single
+ * child) prefixes in the binary trie. The are represented by a bit
+ * string (the "relative prefix") along with its length and a pointer
+ * to the extending path (the next node past the LC node.)
+ *
+ *
+ * Non-Terminal LC Node:
+ *
+ * +------------------+-------+
+ * | relative prefix |1|0|len|
+ * +------------------+-------+
+ * | ptr.child |--+
+ * +--------------------------+ |
+ * |
+ * |
+ * +--------------------------+ |
+ * | Next node - | |
+ * | either LC or TBM | |
+ * | | |
+ * +--------------------------+<-+
+ *
+ * The Relative Prefix
+ * -------------------
+ *
+ * The maximum relative prefix per LC node is selected so that (again)
+ * the entire node structure fits in the space of two pointers. On 32 bit
+ * machines, the maximum relative prefix is 24 bits; on 62 bit machines
+ * the limit is 56 bits.
+ *
+ * In the LC node structure, the relative prefix is stored as an array
+ * of bytes. To avoid some bit-shifting during tree searches, these
+ * bytes are byte-aligned with the global prefix. In other words, in
+ * general there are (pos % 8) "pad" bits at the beginning of the
+ * relative prefix --- where pos "starting bit" (or depth in the
+ * binary tree) of the LC node --- which really belong to the parent
+ * node(s) of the LC node. For efficiency (so that we don't have to
+ * mask them out when matching) we require that these pad bits be
+ * correct --- they must match the path which leads to the LC node.
+ *
+ * The relative prefix length stored in the LC node structure does not
+ * count the pad bits.
+ *
+ * Terminal Node Compression
+ * -------------------------
+ *
+ * For memory efficiency, we also support "terminal LC" nodes. When
+ * the extension path from an LC node consists a single terminal node,
+ * we store that terminal nodes data directly in the parent LC node.
+ *
+ * Instead of this:
+ *
+ * +------------------+-------+
+ * | relative prefix |1|0|len|
+ * +------------------+-------+
+ * | ptr.child |--+
+ * +--------------------------+ |
+ * |
+ * +--------------------------+ |
+ * | Terminal Node (TBM node, | |
+ * | empty except for the | |
+ * +--| root internal node.) | |
+ * | +--------------------------+<-+
+ * |
+ * +->+--------------------------+
+ * | terminal node data |
+ * +--------------------------+
+ *
+ * We can do this:
+ *
+ * +------------------+-------+
+ * | relative prefix |1|1|len|
+ * +------------------+-------+
+ * | terminal node data |
+ * +--------------------------+
+ *
+ * Terminal LC nodes are differentiated from non-terminal LC nodes
+ * by the setting of the is_terminal flag.
+ *
+ * Node Structure Packing Details
+ * ------------------------------
+ *
+ * The LC and TBM node structures are carefully packed so that the
+ * "is_lc" flag (which indicates that a node is an LC node)
+ * corresponds to the one unused bit in the internal bitmap of the TBM
+ * node structure (which we require to be zero for TBM nodes).
+ *
+ * ----------------------------------------------------------------
+ *
+ * References
+ * ==========
+ *
+ * [1] Will Eatherton, George Varghese, and Zubin Dittia. 2004. Tree
+ * bitmap: hardware/software IP lookups with incremental
+ * updates. SIGCOMM Comput. Commun. Rev. 34, 2 (April 2004),
+ * 97-122. DOI=10.1145/997150.997160
+ * http://doi.acm.org/10.1145/997150.997160
+ * http://comnet.kaist.ac.kr/yhlee/CN_2008_Spring/readings/Eath-04-tree_bitmap.pdf
+ *
+ ****************************************************************/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <setjmp.h>
+#if defined(TEST) && defined(NDEBUG)
+# warning undefining NDEBUG for TEST build
+# undef NDEBUG
+#endif
+#include <assert.h>
+
+#include "btrie.h"
+#include "libutil/mem_pool.h"
+
+#ifdef __SIZEOF_POINTER__
+#define SIZEOF_VOID_P __SIZEOF_POINTER__
+#else
+#if defined(__ILP32__) || defined(__ILP32) || defined(_ILP32)
+# define SIZEOF_VOID_P 4
+#elif defined(__ILP64__) || defined(__ILP64) || defined(_ILP64)
+# define SIZEOF_VOID_P 8
+#elif defined(__LLP64__) || defined(__LLP64) || defined(_LLP64) || defined(_WIN64)
+# define SIZEOF_VOID_P 8
+#elif defined(__LP64__) || defined(__LP64) || defined(_LP64)
+# define SIZEOF_VOID_P 8
+#elif defined(UINTPTR_MAX) && defined(UINT64_MAX) && (UINTPTR_MAX == UINT64_MAX)
+# define SIZEOF_VOID_P 8
+#else
+# define SIZEOF_VOID_P 4
+#endif
+#endif
+
+#if SIZEOF_VOID_P == 4
+# define TBM_STRIDE 4
+#elif SIZEOF_VOID_P == 8
+# define TBM_STRIDE 5
+#else
+# error "Unsupported word size"
+#endif
+
+#ifndef NO_STDINT_H
+# if TBM_STRIDE == 4
+typedef uint16_t tbm_bitmap_t;
+# else
+typedef uint32_t tbm_bitmap_t;
+# endif
+#else /* NO_STDINT_H */
+# if TBM_STRIDE == 4
+# if SIZEOF_SHORT == 2
+typedef short unsigned tbm_bitmap_t;
+# else
+# error "can not determine type for 16 bit unsigned int"
+# endif
+# else /* TBM_STRIDE == 5 */
+# if SIZEOF_INT == 4
+typedef unsigned tbm_bitmap_t;
+# elif SIZEOF_LONG == 4
+typedef long unsigned tbm_bitmap_t;
+# else
+# error "can not determine type for 32 bit unsigned int"
+# endif
+# endif
+#endif
+
+#define TBM_FANOUT (1U << TBM_STRIDE)
+#define LC_BYTES_PER_NODE (SIZEOF_VOID_P - 1)
+
+typedef union node_u node_t;
+
+/* The tbm_node and lc_node structs must be packed so that the the
+ * high bit (LC_FLAGS_IS_LC) of lc_flags in the the lc_node struct
+ * coincides with bit zero (the most significant bit) of tbm_node's
+ * int_bm. (This bit is how we differentiate between the two node
+ * types. It is always clear for a tbm_node and always set for an
+ * lc_node.)
+ */
+
+struct tbm_node
+{
+#ifdef WORDS_BIGENDIAN
+ tbm_bitmap_t int_bm; /* the internal bitmap */
+ tbm_bitmap_t ext_bm; /* extending path ("external") bitmap */
+#else
+ tbm_bitmap_t ext_bm; /* extending path ("external") bitmap */
+ tbm_bitmap_t int_bm; /* the internal bitmap */
+#endif
+ union
+ {
+ node_t *children; /* pointer to array of children */
+ const void **data_end; /* one past end of internal prefix data array */
+ } ptr;
+};
+
+struct lc_node
+{
+ /* lc_flags contains the LC prefix length and a couple of bit flags
+ * (apparently char-sized bit fields are a gcc extension)
+ */
+# define LC_FLAGS_IS_LC 0x80
+# define LC_FLAGS_IS_TERMINAL 0x40
+# define LC_FLAGS_LEN_MASK 0x3f
+#ifdef WORDS_BIGENDIAN
+ btrie_oct_t lc_flags;
+ btrie_oct_t prefix[LC_BYTES_PER_NODE];
+#else
+ btrie_oct_t prefix[LC_BYTES_PER_NODE];
+ btrie_oct_t lc_flags;
+#endif
+ union
+ {
+ node_t *child; /* pointer to child (if !is_terminal) */
+ const void *data; /* the prefix data (if is_terminal) */
+ } ptr;
+};
+
+union node_u
+{
+ struct tbm_node tbm_node;
+ struct lc_node lc_node;
+};
+
+struct free_hunk
+{
+ struct free_hunk *next;
+};
+
+#define MAX_CHILD_ARRAY_LEN (TBM_FANOUT + TBM_FANOUT / 2)
+
+struct btrie
+{
+ node_t root;
+
+ rspamd_mempool_t *mp;
+ struct free_hunk *free_list[MAX_CHILD_ARRAY_LEN];
+ jmp_buf exception;
+ /* mem mgmt stats */
+ size_t alloc_total; /* total bytes allocated from mempool */
+ size_t alloc_data; /* bytes allocated for TBM node int. prefix data */
+ size_t alloc_waste; /* bytes wasted by rounding of data array size */
+#ifdef BTRIE_DEBUG_ALLOC
+ size_t alloc_hist[MAX_CHILD_ARRAY_LEN * 2]; /* histogram of alloc sizes */
+#endif
+
+ /* trie stats */
+ size_t n_entries; /* number of entries */
+ size_t n_tbm_nodes; /* total number of TBM nodes in tree */
+ size_t n_lc_nodes; /* total number of LC nodes in tree */
+};
+
+/****************************************************************
+ *
+ * Memory management
+ *
+ * We will need to frequently resize child/data arrays. The current
+ * mempool implementation does not support resizing/freeing, so here
+ * we roll our own.
+ */
+
+static inline void _free_hunk(struct btrie *btrie, void *buf, unsigned n_nodes)
+{
+ struct free_hunk *hunk = buf;
+
+ hunk->next = btrie->free_list[n_nodes - 1];
+ btrie->free_list[n_nodes - 1] = hunk;
+}
+
+static inline void *
+_get_hunk(struct btrie *btrie, unsigned n_nodes)
+{
+ struct free_hunk *hunk = btrie->free_list[n_nodes - 1];
+
+ if (hunk != NULL)
+ btrie->free_list[n_nodes - 1] = hunk->next;
+ return hunk;
+}
+
+/* Get pointer to uninitialized child/data array.
+ *
+ * Allocates memory for an array of NDATA (void *)s followed by an
+ * array of NCHILDREN (node_t)s. The returned pointer points to to
+ * beginning of the children array (i.e. it points to (one past) the
+ * end of the data array.)
+ */
+static node_t *
+alloc_nodes(struct btrie *btrie, unsigned nchildren, unsigned ndata)
+{
+ size_t n_nodes = nchildren + (ndata + 1) / 2;
+ node_t *hunk;
+
+ assert(n_nodes > 0 && n_nodes <= MAX_CHILD_ARRAY_LEN);
+
+ hunk = _get_hunk (btrie, n_nodes);
+ if (hunk == NULL) {
+ /* Do not have free hunk of exactly the requested size, look for a
+ * larger hunk. (The funny order in which we scan the buckets is
+ * heuristically selected in an attempt to minimize unnecessary
+ * creation of small fragments)
+ */
+ size_t n, skip = n_nodes > 4 ? 4 : n_nodes;
+ for (n = n_nodes + skip; n <= MAX_CHILD_ARRAY_LEN; n++) {
+ if ((hunk = _get_hunk (btrie, n)) != NULL) {
+ _free_hunk (btrie, hunk + n_nodes, n - n_nodes);
+ goto DONE;
+ }
+ }
+ for (n = n_nodes + 1; n < n_nodes + skip && n <= MAX_CHILD_ARRAY_LEN;
+ n++) {
+ if ((hunk = _get_hunk (btrie, n)) != NULL) {
+ _free_hunk (btrie, hunk + n_nodes, n - n_nodes);
+ goto DONE;
+ }
+ }
+
+ /* failed to find free hunk, allocate a fresh one */
+ hunk = rspamd_mempool_alloc0 (btrie->mp, n_nodes * sizeof(node_t));
+ if (hunk == NULL)
+ longjmp (btrie->exception, BTRIE_ALLOC_FAILED);
+ btrie->alloc_total += n_nodes * sizeof(node_t);
+ }
+
+ DONE: btrie->alloc_data += ndata * sizeof(void *);
+ btrie->alloc_waste += (ndata % 2) * sizeof(void *);
+#ifdef BTRIE_DEBUG_ALLOC
+ btrie->alloc_hist[2 * nchildren + ndata]++;
+#endif
+
+ /* adjust pointer to allow room for data array before child array */
+ return hunk + (ndata + 1) / 2;
+}
+
+/* Free memory allocated by alloc_nodes */
+static void free_nodes(struct btrie *btrie, node_t *buf, unsigned nchildren,
+ unsigned ndata)
+{
+ size_t n_nodes = nchildren + (ndata + 1) / 2;
+
+ assert(n_nodes > 0 && n_nodes <= MAX_CHILD_ARRAY_LEN);
+
+ _free_hunk (btrie, buf - (ndata + 1) / 2, n_nodes);
+
+ btrie->alloc_data -= ndata * sizeof(void *);
+ btrie->alloc_waste -= (ndata % 2) * sizeof(void *);
+#ifdef BTRIE_DEBUG_ALLOC
+ btrie->alloc_hist[2 * nchildren + ndata]--;
+#endif
+}
+
+/* Debugging/development only: */
+#ifdef BTRIE_DEBUG_ALLOC
+static void
+dump_alloc_hist(const struct btrie *btrie)
+{
+ unsigned bin;
+ size_t total_alloc = 0;
+ size_t total_free = 0;
+ size_t total_bytes = 0;
+ size_t total_waste = 0;
+ size_t total_free_bytes = 0;
+
+ puts("hunk alloc free alloc wasted free");
+ puts("size hunks hunks bytes bytes bytes");
+ puts("==== ====== ====== ======== ======== ========");
+
+ for (bin = 1; bin < 2 * MAX_CHILD_ARRAY_LEN; bin++) {
+ size_t n_alloc = btrie->alloc_hist[bin];
+ size_t bytes = n_alloc * bin * sizeof(void *);
+ size_t waste_bytes = (bin % 2) * n_alloc * sizeof(void *);
+ size_t n_free = 0, free_bytes;
+ if (bin % 2 == 0) {
+ const struct free_hunk *hunk;
+ for (hunk = btrie->free_list[bin / 2 - 1]; hunk; hunk = hunk->next)
+ n_free++;
+ }
+ free_bytes = n_free * bin * sizeof(void *);
+
+ printf("%3zu: %6zu %6zu %8zu %8zu %8zu\n", bin * sizeof(void *),
+ n_alloc, n_free, bytes, waste_bytes, free_bytes);
+
+ total_alloc += n_alloc;
+ total_free += n_free;
+ total_bytes += bytes;
+ total_waste += waste_bytes;
+ total_free_bytes += free_bytes;
+ }
+ puts("---- ------ ------ -------- -------- --------");
+ printf("SUM: %6zu %6zu %8zu %8zu %8zu\n",
+ total_alloc, total_free, total_bytes, total_waste, total_free_bytes);
+}
+#endif
+
+/****************************************************************
+ *
+ * Bit twiddling
+ *
+ */
+
+static inline tbm_bitmap_t bit(unsigned b)
+{
+ return 1U << ((1 << TBM_STRIDE) - 1 - b);
+}
+
+/* count the number of set bits in bitmap
+ *
+ * algorithm from
+ * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+ */
+static inline unsigned count_bits(tbm_bitmap_t v)
+{
+ /* Count set bits in parallel. */
+ /* v = (v & 0x5555...) + ((v >> 1) & 0x5555...); */
+ v -= (v >> 1) & (tbm_bitmap_t) ~0UL / 3;
+ /* v = (v & 0x3333...) + ((v >> 2) & 0x3333...); */
+ v = (v & (tbm_bitmap_t) ~0UL / 5) + ((v >> 2) & (tbm_bitmap_t) ~0UL / 5);
+ /* v = (v & 0x0f0f...) + ((v >> 4) & 0x0f0f...); */
+ v = (v + (v >> 4)) & (tbm_bitmap_t) ~0UL / 17;
+ /* v = v % 255; */
+#if TBM_STRIDE == 4
+ /* tbm_bitmap_t is uint16_t, avoid the multiply */
+ return (v + (v >> 8)) & 0x0ff;
+#else
+ return (v * (tbm_bitmap_t) (~0UL / 255)) >> ((sizeof(tbm_bitmap_t) - 1) * 8);
+#endif
+}
+
+static inline unsigned count_bits_before(tbm_bitmap_t bm, int b)
+{
+ return b ? count_bits (bm >> ((1 << TBM_STRIDE) - b)) : 0;
+}
+
+static inline unsigned count_bits_from(tbm_bitmap_t bm, int b)
+{
+ return count_bits (bm << b);
+}
+
+/* extracts a few bits from bitstring, returning them as an integer */
+static inline btrie_oct_t RSPAMD_NO_SANITIZE extract_bits(const btrie_oct_t *prefix, unsigned pos,
+ unsigned nbits)
+{
+ if (nbits == 0)
+ return 0;
+ else {
+ unsigned v = (prefix[pos / 8] << 8) + prefix[pos / 8 + 1];
+ return (v >> (16 - nbits - pos % 8)) & ((1U << nbits) - 1);
+ }
+}
+
+static inline unsigned extract_bit(const btrie_oct_t *prefix, int pos)
+{
+ return (prefix[pos / 8] >> (7 - pos % 8)) & 0x01;
+}
+
+/* get mask for high n bits of a byte */
+static inline btrie_oct_t high_bits(unsigned n)
+{
+ return (btrie_oct_t) -(1U << (8 - n));
+}
+
+/* determine whether two prefixes are equal */
+static inline int prefixes_equal(const btrie_oct_t *pfx1,
+ const btrie_oct_t *pfx2, unsigned len)
+{
+ return (memcmp (pfx1, pfx2, len / 8) == 0
+ && (len % 8 == 0 ||
+ ((pfx1[len / 8] ^ pfx2[len / 8]) & high_bits (len % 8)) == 0));
+}
+
+/* determine length of longest common subprefix */
+static inline unsigned common_prefix(const btrie_oct_t *pfx1,
+ const btrie_oct_t *pfx2, unsigned len)
+{
+ /* algorithm adapted from
+ * http://graphics.stanford.edu/~seander/bithacks.html#IntegerLogLookup
+ */
+ static btrie_oct_t leading_zeros[] =
+ { 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, };
+ unsigned nb;
+
+ for (nb = 0; nb < len / 8; nb++) {
+ unsigned diff = *pfx1++ ^ *pfx2++;
+ if (diff != 0)
+ return 8 * nb + leading_zeros[diff];
+ }
+ if (len % 8) {
+ unsigned n = leading_zeros[*pfx1 ^ *pfx2];
+ if (n < len % 8)
+ return 8 * nb + n;
+ }
+ return len;
+}
+
+/****************************************************************
+ */
+
+static inline int is_empty_node(const node_t *node)
+{
+ return node->tbm_node.ext_bm == 0 && node->tbm_node.int_bm == 0;
+}
+
+static inline int is_lc_node(const node_t *node)
+{
+ return (node->lc_node.lc_flags & LC_FLAGS_IS_LC) != 0;
+}
+
+static inline int is_tbm_node(const node_t *node)
+{
+ return !is_lc_node (node);
+}
+
+/* is node a TBM node with internal data? */
+static inline int has_data(const node_t *node)
+{
+ return is_tbm_node (node) && node->tbm_node.int_bm != 0;
+}
+
+static inline unsigned base_index(unsigned pfx, unsigned plen)
+{
+ assert(plen < TBM_STRIDE);
+ assert(pfx < (1U << plen));
+ return pfx | (1U << plen);
+}
+
+/* initialize node to an empty TBM node */
+static inline void init_empty_node(struct btrie *btrie, node_t *node)
+{
+ memset(node, 0, sizeof(*node));
+ btrie->n_tbm_nodes++;
+}
+
+/* get pointer to TBM internal prefix data */
+static inline const void **
+tbm_data_p(const struct tbm_node *node, unsigned pfx, unsigned plen)
+{
+ unsigned bi = base_index (pfx, plen);
+
+ if ((node->int_bm & bit (bi)) == 0)
+ return NULL; /* no data */
+ else {
+ return &node->ptr.data_end[-(int) count_bits_from (node->int_bm, bi)];
+ }
+}
+
+/* add an element to the internal data array */
+static void tbm_insert_data(struct btrie *btrie, struct tbm_node *node,
+ unsigned pfx, unsigned plen, const void *data)
+{
+ /* XXX: don't realloc if already big enough? */
+ unsigned bi = base_index (pfx, plen);
+ unsigned nchildren = count_bits (node->ext_bm);
+ int ndata = count_bits (node->int_bm);
+ unsigned di = count_bits_before (node->int_bm, bi);
+ node_t *old_children = node->ptr.children;
+ const void **old_data_beg = node->ptr.data_end - ndata;
+ const void **data_beg;
+
+ assert((node->int_bm & bit (bi)) == 0);
+
+ node->ptr.children = alloc_nodes (btrie, nchildren, ndata + 1);
+ data_beg = node->ptr.data_end - (ndata + 1);
+ data_beg[di] = data;
+ node->int_bm |= bit (bi);
+
+ if (nchildren != 0 || ndata != 0) {
+ memcpy(data_beg, old_data_beg, di * sizeof(data_beg[0]));
+ memcpy(&data_beg[di + 1], &old_data_beg[di],
+ (ndata - di) * sizeof(data_beg[0])
+ + nchildren * sizeof(node_t));
+ free_nodes (btrie, old_children, nchildren, ndata);
+ }
+}
+
+/* determine whether TBM has internal prefix data for pfx/plen or ancestors */
+static inline int has_internal_data(const struct tbm_node *node, unsigned pfx,
+ unsigned plen)
+{
+# define BIT(n) (1U << ((1 << TBM_STRIDE) - 1 - (n)))
+# define B0() BIT(1) /* the bit for 0/0 */
+# define B1(n) (BIT((n) + 2) | B0()) /* the bits for n/1 and its ancestors */
+# define B2(n) (BIT((n) + 4) | B1(n >> 1)) /* the bits for n/2 and ancestors */
+# define B3(n) (BIT((n) + 8) | B2(n >> 1)) /* the bits for n/3 and ancestors */
+# define B4(n) (BIT((n) + 16) | B3(n >> 1)) /* the bits for n/4 and ancestors */
+
+ static tbm_bitmap_t ancestors[] =
+ { 0, B0(), B1(0), B1(1), B2(0), B2(1), B2(2), B2(3), B3(0), B3(1), B3(2),
+ B3(3), B3(4), B3(5), B3(6), B3(7),
+# if TBM_STRIDE == 5
+ B4(0), B4(1), B4(2), B4(3), B4(4), B4(5), B4(6), B4(7), B4(8), B4(
+ 9), B4(10), B4(11), B4(12), B4(13), B4(14), B4(15),
+# elif TBM_STRIDE != 4
+# error "unsupported TBM_STRIDE"
+# endif
+ };
+# undef B4
+# undef B3
+# undef B2
+# undef B1
+# undef B0
+# undef BIT
+
+ return (node->int_bm & ancestors[base_index (pfx, plen)]) != 0;
+}
+
+/* get pointer to TBM extending path */
+static inline node_t *
+tbm_ext_path(const struct tbm_node *node, unsigned pfx)
+{
+ if ((node->ext_bm & bit (pfx)) == 0)
+ return NULL;
+ else
+ return &node->ptr.children[count_bits_before (node->ext_bm, pfx)];
+}
+
+/* resize TBM node child array to make space for new child node */
+static node_t *
+tbm_insert_ext_path(struct btrie *btrie, struct tbm_node *node, unsigned pfx)
+{
+ unsigned nchildren = count_bits (node->ext_bm);
+ unsigned ci = count_bits_before (node->ext_bm, pfx);
+ int ndata = count_bits (node->int_bm);
+ node_t *old_children = node->ptr.children;
+ const void **old_data_beg = node->ptr.data_end - ndata;
+
+ assert((node->ext_bm & bit (pfx)) == 0);
+
+ node->ptr.children = alloc_nodes (btrie, nchildren + 1, ndata);
+ init_empty_node (btrie, &node->ptr.children[ci]);
+ node->ext_bm |= bit (pfx);
+
+ if (nchildren != 0 || ndata != 0) {
+ const void **data_beg = node->ptr.data_end - ndata;
+ memcpy(data_beg, old_data_beg,
+ ndata * sizeof(data_beg[0]) + ci * sizeof(node_t));
+ memcpy(&node->ptr.children[ci + 1], &old_children[ci],
+ (nchildren - ci) * sizeof(old_children[0]));
+ free_nodes (btrie, old_children, nchildren, ndata);
+ }
+
+ return &node->ptr.children[ci];
+}
+
+static inline int lc_is_terminal(const struct lc_node *node)
+{
+ return (node->lc_flags & LC_FLAGS_IS_TERMINAL) != 0;
+}
+
+static inline unsigned lc_len(const struct lc_node *node)
+{
+ return node->lc_flags & LC_FLAGS_LEN_MASK;
+}
+
+static inline void lc_init_flags(struct lc_node *node, int is_terminal,
+ unsigned len)
+{
+ assert((len & ~LC_FLAGS_LEN_MASK) == 0);
+ node->lc_flags = LC_FLAGS_IS_LC | len;
+ if (is_terminal)
+ node->lc_flags |= LC_FLAGS_IS_TERMINAL;
+}
+
+static inline void lc_add_to_len(struct lc_node *node, int increment)
+{
+ unsigned new_len = lc_len (node) + increment;
+ assert((new_len & ~LC_FLAGS_LEN_MASK) == 0);
+ node->lc_flags = (node->lc_flags & ~LC_FLAGS_LEN_MASK) | new_len;
+}
+
+static inline unsigned lc_shift(unsigned pos)
+{
+ return pos / 8;
+}
+
+static inline unsigned lc_base(unsigned pos)
+{
+ return 8 * lc_shift (pos);
+}
+
+static inline unsigned lc_bits(const struct lc_node *node, unsigned pos)
+{
+ return pos % 8 + lc_len (node);
+}
+
+static inline unsigned lc_bytes(const struct lc_node *node, unsigned pos)
+{
+ return (lc_bits (node, pos) + 7) / 8;
+}
+
+static inline unsigned lc_leading_bits(const struct lc_node *node, unsigned pos,
+ unsigned nbits)
+{
+ return extract_bits (node->prefix, pos % 8, nbits);
+}
+
+/* Initialize a new terminal LC node
+ *
+ * If prefix is too long to fit in a single LC node, then a chain
+ * of LC nodes will be created.
+ */
+static void init_terminal_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ const btrie_oct_t *prefix, unsigned len, const void *data)
+{
+ struct lc_node *node = &dst->lc_node;
+ unsigned nbytes = (len + 7) / 8;
+
+ while (nbytes - lc_shift (pos) > LC_BYTES_PER_NODE) {
+ memcpy(node->prefix, prefix + lc_shift (pos), LC_BYTES_PER_NODE);
+ lc_init_flags (node, 0, 8 * LC_BYTES_PER_NODE - pos % 8);
+ node->ptr.child = alloc_nodes (btrie, 1, 0);
+ pos += lc_len (node);
+ node = &node->ptr.child->lc_node;
+ btrie->n_lc_nodes++;
+ }
+
+ memcpy(node->prefix, prefix + lc_shift (pos), nbytes - lc_shift (pos));
+ lc_init_flags (node, 1, len - pos);
+ node->ptr.data = data;
+ btrie->n_lc_nodes++;
+}
+
+/* merge chains of multiple LC nodes into a single LC node, if possible.
+ *
+ * also ensure that the leading nodes in the LC chain have maximum length.
+ */
+static void coalesce_lc_node(struct btrie *btrie, struct lc_node *node,
+ unsigned pos)
+{
+ while (!lc_is_terminal (node) && lc_bits (node, pos) < 8 * LC_BYTES_PER_NODE
+ && is_lc_node (node->ptr.child)) {
+ struct lc_node *child = &node->ptr.child->lc_node;
+ unsigned spare_bits = 8 * LC_BYTES_PER_NODE - lc_bits (node, pos);
+ unsigned end = pos + lc_len (node);
+ unsigned shift = lc_shift (end) - lc_shift (pos);
+ if (lc_len (child) <= spare_bits) {
+ /* node plus child will fit in single node - merge */
+ memcpy(node->prefix + shift, child->prefix, lc_bytes (child, end));
+ lc_init_flags (node, lc_is_terminal (child),
+ lc_len (node) + lc_len (child));
+ node->ptr = child->ptr;
+ free_nodes (btrie, (node_t *) child, 1, 0);
+ btrie->n_lc_nodes--;
+ }
+ else {
+ /* can't merge, but can take some of children bits */
+ unsigned cshift = lc_shift (end + spare_bits) - lc_shift (end);
+
+ memcpy(node->prefix + shift, child->prefix,
+ LC_BYTES_PER_NODE - shift);
+ lc_add_to_len (node, spare_bits);
+ if (cshift)
+ memmove(child->prefix, child->prefix + cshift,
+ lc_bytes (child, end) - cshift);
+ assert(lc_len (child) > spare_bits);
+ lc_add_to_len (child, -spare_bits);
+
+ pos += lc_len (node);
+ node = child;
+ }
+ }
+}
+
+static void init_tbm_node(struct btrie *btrie, node_t *node, unsigned pos,
+ const btrie_oct_t pbyte, const void **root_data_p, node_t *left,
+ node_t *right);
+
+/* given an LC node at orig_pos, create a new (shorter) node at pos */
+static void shorten_lc_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ struct lc_node *src, unsigned orig_pos)
+{
+ assert(orig_pos < pos);
+ assert(lc_len (src) >= pos - orig_pos);
+ assert(dst != (node_t * )src);
+
+ if (lc_len (src) == pos - orig_pos && !lc_is_terminal (src)) {
+ /* just steal the child */
+ node_t *child = src->ptr.child;
+ *dst = *child;
+ free_nodes (btrie, child, 1, 0);
+ btrie->n_lc_nodes--;
+ }
+ else {
+ struct lc_node *node = &dst->lc_node;
+ unsigned shift = lc_shift (pos) - lc_shift (orig_pos);
+ if (shift) {
+ memmove(node->prefix, src->prefix + shift,
+ lc_bytes (src, orig_pos) - shift);
+ node->lc_flags = src->lc_flags;
+ node->ptr = src->ptr;
+ }
+ else {
+ *node = *src;
+ }
+ lc_add_to_len (node, -(pos - orig_pos));
+ coalesce_lc_node (btrie, node, pos);
+ }
+}
+
+/* convert LC node to non-terminal LC node of length len *in place*
+ *
+ * on entry, node must have length at least len
+ */
+static void split_lc_node(struct btrie *btrie, struct lc_node *node,
+ unsigned pos, unsigned len)
+{
+ node_t *child = alloc_nodes (btrie, 1, 0);
+
+ assert(lc_len (node) >= len);
+ shorten_lc_node (btrie, child, pos + len, node, pos);
+
+ lc_init_flags (node, 0, len);
+ node->ptr.child = child;
+ btrie->n_lc_nodes++;
+}
+
+/* convert non-terminal LC node of length one to a TBM node *in place* */
+static void convert_lc_node_1(struct btrie *btrie, struct lc_node *node,
+ unsigned pos)
+{
+ btrie_oct_t pbyte = node->prefix[0];
+ node_t *child = node->ptr.child;
+ node_t *left, *right;
+
+ assert(lc_len (node) == 1);
+ assert(!lc_is_terminal (node));
+
+ if (extract_bit (node->prefix, pos % 8))
+ left = NULL, right = child;
+ else
+ left = child, right = NULL;
+ init_tbm_node (btrie, (node_t *) node, pos, pbyte, NULL, left, right);
+ free_nodes (btrie, child, 1, 0);
+ btrie->n_lc_nodes--;
+}
+
+/* convert an LC node to TBM node *in place* */
+static void convert_lc_node(struct btrie *btrie, struct lc_node *node,
+ unsigned pos)
+{
+ unsigned len = lc_len (node);
+
+ if (len >= TBM_STRIDE) {
+ unsigned pfx = lc_leading_bits (node, pos, TBM_STRIDE);
+ struct tbm_node *result = (struct tbm_node *) node;
+
+ /* split to LC of len TBM_STRIDE followed by child (extending path) */
+ split_lc_node (btrie, node, pos, TBM_STRIDE);
+ /* then convert leading LC node to TBM node */
+ result->int_bm = 0;
+ result->ext_bm = bit (pfx);
+ btrie->n_lc_nodes--;
+ btrie->n_tbm_nodes++;
+ }
+ else if (lc_is_terminal (node)) {
+ /* convert short terminal LC to TBM (with internal data) */
+ unsigned pfx = lc_leading_bits (node, pos, len);
+ const void *data = node->ptr.data;
+ node_t *result = (node_t *) node;
+
+ init_empty_node (btrie, result);
+ tbm_insert_data (btrie, &result->tbm_node, pfx, len, data);
+
+ btrie->n_lc_nodes--;
+ }
+ else {
+ assert(len > 0);
+ for (; len > 1; len--) {
+ split_lc_node (btrie, node, pos, len - 1);
+ convert_lc_node_1 (btrie, &node->ptr.child->lc_node, pos + len - 1);
+ }
+ convert_lc_node_1 (btrie, node, pos);
+ }
+}
+
+static void insert_lc_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ btrie_oct_t pbyte, unsigned last_bit, node_t *tail)
+{
+ struct lc_node *node = &dst->lc_node;
+ btrie_oct_t mask = 1 << (7 - (pos % 8));
+ btrie_oct_t bit = last_bit ? mask : 0;
+
+ if (mask != 0x01 && is_lc_node (tail)) {
+ /* optimization: LC tail has room for the extra bit (without shifting) */
+ assert((tail->lc_node.prefix[0] & mask) == bit);
+ *node = tail->lc_node;
+ lc_add_to_len (node, 1);
+ return;
+ }
+
+ /* add new leading LC node of len 1 */
+ node->prefix[0] = pbyte | bit;
+ lc_init_flags (node, 0, 1);
+ node->ptr.child = alloc_nodes (btrie, 1, 0);
+ node->ptr.child[0] = *tail;
+ btrie->n_lc_nodes++;
+
+ if (is_lc_node (tail))
+ coalesce_lc_node (btrie, node, pos);
+}
+
+/* given:
+ * pbyte: the bits in the prefix between lc_base(pos) and pos
+ * pfx: the next TBM_STRIDE bits in the prefix starting at pos
+ * returns:
+ * the bits in the prefix between lc_base(pos + plen) and pos + plen
+ */
+static inline btrie_oct_t next_pbyte(btrie_oct_t pbyte, unsigned pos,
+ unsigned pfx)
+{
+ unsigned end = pos + TBM_STRIDE;
+
+ if (end % 8 != 0) {
+ btrie_oct_t nbyte = (btrie_oct_t) pfx << (8 - end % 8);
+ if (end % 8 > TBM_STRIDE)
+ nbyte |= pbyte & high_bits (pos % 8);
+ return nbyte;
+ }
+ return 0;
+}
+
+/* construct a new TBM node, given the data and children of the
+ * root prefix of the new node.
+ */
+static void init_tbm_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ const btrie_oct_t pbyte, const void **root_data_p, node_t *left,
+ node_t *right)
+{
+ struct tbm_node *node = &dst->tbm_node;
+ unsigned nchildren = 0;
+ unsigned ndata = 0;
+ node_t children[TBM_FANOUT];
+ const void *data[TBM_FANOUT - 1];
+ tbm_bitmap_t ext_bm = 0;
+ tbm_bitmap_t int_bm = 0;
+ unsigned i, d, pfx_base;
+
+ if (left && is_lc_node (left) && lc_len (&left->lc_node) < TBM_STRIDE)
+ convert_lc_node (btrie, &left->lc_node, pos + 1);
+ if (right && is_lc_node (right) && lc_len (&right->lc_node) < TBM_STRIDE)
+ convert_lc_node (btrie, &right->lc_node, pos + 1);
+
+ /* set internal data for root prefix */
+ if (root_data_p) {
+ data[ndata++] = *root_data_p;
+ int_bm |= bit (base_index (0, 0));
+ }
+ /* copy internal data from children */
+ for (d = 0; d < TBM_STRIDE - 1; d++) {
+ if (left && has_data (left)) {
+ for (i = 0; i < 1U << d; i++) {
+ const void **data_p = tbm_data_p (&left->tbm_node, i, d);
+ if (data_p) {
+ data[ndata++] = *data_p;
+ int_bm |= bit (base_index (i, d + 1));
+ }
+ }
+ }
+ if (right && has_data (right)) {
+ for (i = 0; i < 1U << d; i++) {
+ const void **data_p = tbm_data_p (&right->tbm_node, i, d);
+ if (data_p) {
+ data[ndata++] = *data_p;
+ int_bm |= bit (base_index (i + (1 << d), d + 1));
+ }
+ }
+ }
+ }
+
+ /* copy extending paths */
+ for (pfx_base = 0; pfx_base < TBM_FANOUT; pfx_base += TBM_FANOUT / 2) {
+ node_t *child = pfx_base ? right : left;
+ if (child == NULL) {
+ continue;
+ }
+ else if (is_lc_node (child)) {
+ unsigned pfx = pfx_base + lc_leading_bits (&child->lc_node, pos + 1,
+ TBM_STRIDE - 1);
+ /* child is LC node, just shorten it by TBM_STRIDE - 1 */
+ shorten_lc_node (btrie, &children[nchildren++], pos + TBM_STRIDE,
+ &child->lc_node, pos + 1);
+ ext_bm |= bit (pfx);
+ }
+ else if (!is_empty_node (child)) {
+ /* convert deepest internal prefixes of child to extending paths
+ * of the new node
+ */
+ for (i = 0; i < TBM_FANOUT / 2; i++) {
+ const void **data_p = tbm_data_p (&child->tbm_node, i,
+ TBM_STRIDE - 1);
+ node_t *left_ext = tbm_ext_path (&child->tbm_node, 2 * i);
+ node_t *right_ext = tbm_ext_path (&child->tbm_node, 2 * i + 1);
+ if (data_p || left_ext || right_ext) {
+ node_t *ext_path = &children[nchildren++];
+ unsigned pfx = pfx_base + i;
+ btrie_oct_t npbyte = next_pbyte (pbyte, pos, pfx);
+
+ ext_bm |= bit (pfx);
+ if (left_ext == NULL && right_ext == NULL) {
+ /* only have data - set ext_path to zero-length terminal LC node */
+ lc_init_flags (&ext_path->lc_node, 1, 0);
+ ext_path->lc_node.prefix[0] = npbyte;
+ ext_path->lc_node.ptr.data = *data_p;
+ btrie->n_lc_nodes++;
+ }
+ else if (data_p || (left_ext && right_ext)) {
+ /* have at least two of data, left_ext, right_ext
+ * ext_path must be a full TBM node */
+ init_tbm_node (btrie, ext_path, pos + TBM_STRIDE,
+ npbyte, data_p, left_ext, right_ext);
+ }
+ else if (left_ext) {
+ /* have only left_ext, insert length-one LC node */
+ insert_lc_node (btrie, ext_path, pos + TBM_STRIDE,
+ npbyte, 0, left_ext);
+ }
+ else {
+ /* have only right_ext, insert length-one LC node */
+ insert_lc_node (btrie, ext_path, pos + TBM_STRIDE,
+ npbyte, 1, right_ext);
+ }
+ }
+ }
+ btrie->n_tbm_nodes--;
+ free_nodes (btrie, child->tbm_node.ptr.children,
+ count_bits (child->tbm_node.ext_bm),
+ count_bits (child->tbm_node.int_bm));
+ }
+ }
+
+ assert(count_bits (int_bm) == ndata);
+ assert(count_bits (ext_bm) == nchildren);
+
+ node->ptr.children = alloc_nodes (btrie, nchildren, ndata);
+ memcpy(node->ptr.data_end - (int )ndata, data, ndata * sizeof(data[0]));
+ memcpy(node->ptr.children, children, nchildren * sizeof(children[0]));
+ node->ext_bm = ext_bm;
+ node->int_bm = int_bm;
+ btrie->n_tbm_nodes++;
+}
+
+static enum btrie_result add_to_trie(struct btrie *btrie, node_t *node,
+ unsigned pos, const btrie_oct_t *prefix, unsigned len, const void *data)
+{
+ for (;;) {
+ if (is_lc_node (node)) {
+ struct lc_node *lc_node = &node->lc_node;
+ unsigned end = pos + lc_len (lc_node);
+ unsigned cbits = common_prefix (prefix + lc_shift (pos),
+ lc_node->prefix, (len < end ? len : end) - lc_base (pos));
+ unsigned clen = lc_base (pos) + cbits; /* position of first mismatch */
+
+ if (clen == end && !lc_is_terminal (lc_node)) {
+ /* matched entire prefix of LC node, proceed to child */
+ assert(lc_len (lc_node) > 0);
+ node = lc_node->ptr.child;
+ pos = end;
+ }
+ else if (clen == end && len == end && lc_is_terminal (lc_node)) {
+ /* exact match for terminal node - already have data for prefix */
+ return BTRIE_DUPLICATE_PREFIX;
+ }
+ else {
+ assert(clen < end || (lc_is_terminal (lc_node) && len > end));
+ /* Need to insert new TBM node at clen */
+ if (clen > pos) {
+ split_lc_node (btrie, lc_node, pos, clen - pos);
+ node = lc_node->ptr.child;
+ assert(is_lc_node (node));
+ pos = clen;
+ }
+ convert_lc_node (btrie, &node->lc_node, pos);
+ }
+ }
+ else if (is_empty_node (node)) {
+ /* at empty TBM node - just replace with terminal LC node */
+ init_terminal_node (btrie, node, pos, prefix, len, data);
+ btrie->n_entries++;
+ btrie->n_tbm_nodes--;
+ return BTRIE_OKAY;
+ }
+ else {
+ struct tbm_node *tbm_node = &node->tbm_node;
+ unsigned end = pos + TBM_STRIDE;
+
+ if (len < end) {
+ unsigned plen = len - pos;
+ unsigned pfx = extract_bits (prefix, pos, plen);
+
+ if (tbm_data_p (tbm_node, pfx, plen) != NULL)
+ return BTRIE_DUPLICATE_PREFIX; /* prefix already has data */
+ else {
+ tbm_insert_data (btrie, tbm_node, pfx, plen, data);
+ btrie->n_entries++;
+ return BTRIE_OKAY;
+ }
+ }
+ else {
+ unsigned pfx = extract_bits (prefix, pos, TBM_STRIDE);
+
+ /* follow extending path */
+ node = tbm_ext_path (tbm_node, pfx);
+ if (node == NULL)
+ node = tbm_insert_ext_path (btrie, tbm_node, pfx);
+ pos = end;
+ }
+ }
+ }
+}
+
+static const void *
+search_trie(const node_t *node, unsigned pos, const btrie_oct_t *prefix,
+ unsigned len)
+{
+ /* remember last TBM node seen with internal data */
+ const struct tbm_node *int_node = 0;
+ unsigned int_pfx = 0, int_plen = 0;
+
+ while (node) {
+ if (is_lc_node (node)) {
+ const struct lc_node *lc_node = &node->lc_node;
+ unsigned end = pos + lc_len (lc_node);
+ if (len < end)
+ break;
+ if (!prefixes_equal (prefix + lc_shift (pos), lc_node->prefix,
+ end - lc_base (pos)))
+ break;
+
+ if (lc_is_terminal (lc_node))
+ return lc_node->ptr.data; /* found terminal node */
+
+ pos = end;
+ node = lc_node->ptr.child;
+ }
+ else {
+ const struct tbm_node *tbm_node = &node->tbm_node;
+ unsigned end = pos + TBM_STRIDE;
+ if (len < end) {
+ unsigned plen = len - pos;
+ unsigned pfx = extract_bits (prefix, pos, plen);
+ if (has_internal_data (tbm_node, pfx, plen)) {
+ int_node = tbm_node;
+ int_pfx = pfx;
+ int_plen = plen;
+ }
+ break;
+ }
+ else {
+ unsigned pfx = extract_bits (prefix, pos, TBM_STRIDE);
+ if (has_internal_data (tbm_node, pfx >> 1, TBM_STRIDE - 1)) {
+ int_node = tbm_node;
+ int_pfx = pfx >> 1;
+ int_plen = TBM_STRIDE - 1;
+ }
+ pos = end;
+ node = tbm_ext_path (tbm_node, pfx);
+ }
+ }
+ }
+
+ if (int_node) {
+ const void **data_p = tbm_data_p (int_node, int_pfx, int_plen);
+ while (data_p == NULL) {
+ assert(int_plen > 0);
+ int_pfx >>= 1;
+ int_plen--;
+ data_p = tbm_data_p (int_node, int_pfx, int_plen);
+ }
+ return *data_p;
+ }
+
+ return NULL;
+}
+
+struct btrie *
+btrie_init(rspamd_mempool_t *mp)
+{
+ struct btrie *btrie;
+
+ if (!(btrie = rspamd_mempool_alloc0 (mp, sizeof(*btrie)))) {
+ return NULL;
+ }
+
+ btrie->mp = mp;
+ btrie->alloc_total = sizeof(*btrie);
+
+ /* count the empty root node */
+ btrie->n_tbm_nodes = 1;
+
+ return btrie;
+}
+
+enum btrie_result btrie_add_prefix(struct btrie *btrie,
+ const btrie_oct_t *prefix, unsigned len, const void *data)
+{
+ enum btrie_result rv;
+ if ((rv = setjmp (btrie->exception)) != 0)
+ return rv; /* out of memory */
+
+ return add_to_trie (btrie, &btrie->root, 0, prefix, len, data);
+}
+
+const void *
+btrie_lookup(const struct btrie *btrie, const btrie_oct_t *prefix, unsigned len)
+{
+ return search_trie (&btrie->root, 0, prefix, len);
+}
+
+/****************************************************************
+ *
+ * btrie_stats() - statistics reporting
+ */
+
+#ifdef BTRIE_EXTENDED_STATS
+
+/* Define BTRIE_EXTENDED_STATS to get extra statistics (including
+ * trie depth). This statistics require a traversal of the entire trie
+ * to compute, and so are disabled by default.
+ */
+
+struct stats {
+ size_t max_depth;
+ size_t total_depth;
+#ifndef NDEBUG
+ size_t n_lc_nodes;
+ size_t n_tbm_nodes;
+ size_t n_entries;
+ size_t alloc_data;
+ size_t alloc_waste;
+#endif
+};
+
+static void
+node_stats(const node_t *node, size_t depth, struct stats *stats)
+{
+ if (depth > stats->max_depth)
+ stats->max_depth = depth;
+ stats->total_depth += depth;
+
+ if (is_lc_node(node)) {
+#ifndef NDEBUG
+ stats->n_lc_nodes++;
+#endif
+ if (!lc_is_terminal(&node->lc_node))
+ node_stats(node->lc_node.ptr.child, depth + 1, stats);
+#ifndef NDEBUG
+ else
+ stats->n_entries++;
+#endif
+ }
+ else {
+ unsigned i;
+ unsigned nchildren = count_bits(node->tbm_node.ext_bm);
+#ifndef NDEBUG
+ unsigned ndata = count_bits(node->tbm_node.int_bm);
+
+ stats->n_tbm_nodes++;
+ stats->n_entries += ndata;
+ stats->alloc_data += ndata * sizeof(void *);
+ stats->alloc_waste += (ndata % 2) * sizeof(void *);
+#endif
+ for (i = 0; i < nchildren; i++)
+ node_stats(&node->tbm_node.ptr.children[i], depth + 1, stats);
+ }
+}
+#endif /* BTRIE_EXTENDED_STATS */
+
+#ifndef NDEBUG
+static size_t count_free(const struct btrie *btrie)
+{
+ size_t total = 0;
+ unsigned sz;
+ for (sz = 1; sz <= MAX_CHILD_ARRAY_LEN; sz++) {
+ const struct free_hunk *free = btrie->free_list[sz - 1];
+ size_t n;
+ for (n = 0; free; n++)
+ free = free->next;
+ total += sz * n;
+ }
+ return total * sizeof(node_t);
+}
+#endif /* not NDEBUG */
+
+const char *
+btrie_stats(const struct btrie *btrie, guint duplicates)
+{
+ static char buf[128];
+ size_t n_nodes = btrie->n_lc_nodes + btrie->n_tbm_nodes;
+ size_t alloc_free = (btrie->alloc_total + sizeof(node_t) /* do not double-count the root node */
+ - n_nodes * sizeof(node_t) - btrie->alloc_data - btrie->alloc_waste
+ - sizeof(*btrie));
+#ifdef BTRIE_EXTENDED_STATS
+ struct stats stats;
+ double average_depth;
+
+ memset(&stats, 0, sizeof(stats));
+ node_stats(&btrie->root, 0, &stats);
+ average_depth = (double)stats.total_depth / n_nodes;
+
+#ifndef NDEBUG
+ /* check the node counts */
+ assert(stats.n_lc_nodes == btrie->n_lc_nodes);
+ assert(stats.n_tbm_nodes == btrie->n_tbm_nodes);
+ assert(stats.n_entries == btrie->n_entries);
+ assert(stats.alloc_data == btrie->alloc_data);
+ assert(stats.alloc_waste == btrie->alloc_waste);
+#endif /* not NDEBUG */
+#endif /* BTRIE_EXTENDED_STATS */
+
+#ifndef NDEBUG
+ /* check that we haven't lost any memory */
+ assert(alloc_free == count_free (btrie));
+#endif
+
+#ifdef BTRIE_DEBUG_ALLOC
+ dump_alloc_hist(btrie);
+#endif
+
+
+#ifdef BTRIE_EXTENDED_STATS
+ snprintf(buf, sizeof(buf),
+ "ents=%lu tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu"
+ " depth=%.1f/%lu"
+ ,(long unsigned)btrie->n_entries, (long unsigned)btrie->n_tbm_nodes,
+ (long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024,
+ (long unsigned)alloc_free, (long unsigned)btrie->alloc_waste
+ , average_depth, (long unsigned)stats.max_depth);
+#else
+ snprintf(buf, sizeof(buf),
+ "ents=%lu dup=%u tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu",
+ (long unsigned)btrie->n_entries,
+ duplicates,
+ (long unsigned)btrie->n_tbm_nodes,
+ (long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024,
+ (long unsigned)alloc_free, (long unsigned)btrie->alloc_waste
+ );
+#endif
+ buf[sizeof(buf) - 1] = '\0';
+ return buf;
+}
+
+/****************************************************************/
+
+#ifndef NO_MASTER_DUMP
+
+struct walk_context
+{
+ btrie_walk_cb_t *callback;
+ void *user_data;
+
+ btrie_oct_t prefix[(BTRIE_MAX_PREFIX + 7) / 8];
+};
+
+static void
+walk_node(const node_t *node, unsigned pos, struct walk_context *ctx);
+
+static void walk_tbm_node(const struct tbm_node *node, unsigned pos,
+ unsigned pfx, unsigned plen, struct walk_context *ctx)
+{
+ btrie_oct_t *prefix = ctx->prefix;
+ int pbyte = pos / 8;
+ btrie_oct_t pbit = 0x80 >> (pos % 8);
+ const void **data_p = tbm_data_p (node, pfx, plen);
+
+ if (pos >= BTRIE_MAX_PREFIX) {
+ /* This can/should not happen, but don't overwrite buffers if it does. */
+ return;
+ }
+
+ if (data_p)
+ ctx->callback (prefix, pos, *data_p, 0, ctx->user_data);
+
+ /* walk children */
+ if (plen < TBM_STRIDE - 1) {
+ /* children are internal prefixes in same node */
+ walk_tbm_node (node, pos + 1, pfx << 1, plen + 1, ctx);
+ prefix[pbyte] |= pbit;
+ walk_tbm_node (node, pos + 1, (pfx << 1) + 1, plen + 1, ctx);
+ prefix[pbyte] &= ~pbit;
+ }
+ else {
+ /* children are extending paths */
+ const node_t *ext_path;
+ if ((ext_path = tbm_ext_path (node, pfx << 1)) != NULL)
+ walk_node (ext_path, pos + 1, ctx);
+ if ((ext_path = tbm_ext_path (node, (pfx << 1) + 1)) != NULL) {
+ prefix[pbyte] |= pbit;
+ walk_node (ext_path, pos + 1, ctx);
+ prefix[pbyte] &= ~pbit;
+ }
+ }
+
+ if (data_p)
+ ctx->callback (prefix, pos, *data_p, 1, ctx->user_data);
+}
+
+static void walk_lc_node(const struct lc_node *node, unsigned pos,
+ struct walk_context *ctx)
+{
+ btrie_oct_t *prefix = ctx->prefix;
+ unsigned end = pos + lc_len (node);
+ btrie_oct_t save_prefix = prefix[lc_shift (pos)];
+
+ if (end > BTRIE_MAX_PREFIX) {
+ /* This can/should not happen, but don't overwrite buffers if it does. */
+ return;
+ }
+
+ /* construct full prefix to node */
+ memcpy(&prefix[lc_shift (pos)], node->prefix, lc_bytes (node, pos));
+ if (end % 8)
+ prefix[end / 8] &= high_bits (end % 8);
+
+ if (lc_is_terminal (node)) {
+ ctx->callback (prefix, end, node->ptr.data, 0, ctx->user_data);
+ ctx->callback (prefix, end, node->ptr.data, 1, ctx->user_data);
+ }
+ else
+ walk_node (node->ptr.child, end, ctx);
+
+ prefix[lc_shift (pos)] = save_prefix; /* restore parents prefix */
+ if (lc_bytes (node, pos) > 1)
+ memset(&prefix[lc_shift (pos) + 1], 0, lc_bytes (node, pos) - 1);
+}
+
+static void walk_node(const node_t *node, unsigned pos,
+ struct walk_context *ctx)
+{
+ if (is_lc_node (node))
+ walk_lc_node (&node->lc_node, pos, ctx);
+ else
+ walk_tbm_node (&node->tbm_node, pos, 0, 0, ctx);
+}
+
+/* walk trie in lexicographical order
+ *
+ * calls callback twice (once preorder, once postorder) at each prefix
+ */
+void btrie_walk(const struct btrie *btrie, btrie_walk_cb_t *callback,
+ void *user_data)
+{
+ struct walk_context ctx;
+
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.callback = callback;
+ ctx.user_data = user_data;
+
+ walk_node (&btrie->root, 0, &ctx);
+}
+
+#endif /* not NO_MASTER_DUMP */
+
+
+#ifdef TEST
+/*****************************************************************
+ *
+ * Unit tests
+ *
+ */
+#include <stdio.h>
+
+#ifndef UNUSED
+# define UNUSED __attribute__((unused))
+#endif
+
+/* bogus replacements mp_alloc for running self-tests */
+void *
+mp_alloc(UNUSED struct mempool *mp, unsigned sz, UNUSED int align)
+{
+ return malloc(sz);
+}
+
+#if 0
+# define PASS(name) puts("OK " name)
+#else
+# define PASS(name) fputs(".", stdout); fflush(stdout)
+#endif
+
+const char * pgm_name = "???";
+
+static void
+test_struct_node_packing()
+{
+ node_t node;
+
+ assert(sizeof(struct tbm_node) == 2 * sizeof(void *));
+ assert(sizeof(struct lc_node) == 2 * sizeof(void *));
+ assert(sizeof(node_t) == 2 * sizeof(void *));
+
+ /* The lc_node bit must be an alias for bit zero of int_bm, since
+ * that is the only unused bit in the TBM node structure.
+ */
+ memset(&node, 0, sizeof(node));
+ assert(node.tbm_node.int_bm == 0);
+ lc_init_flags(&node.lc_node, 0, 0);
+ assert(node.tbm_node.int_bm == bit(0));
+
+ PASS("test_struct_node_packing");
+}
+
+static void
+test_bit()
+{
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+ tbm_bitmap_t high_bit = ones ^ (ones >> 1);
+
+ assert(bit(0) == high_bit);
+ assert(bit(1) == high_bit >> 1);
+ assert(bit(8 * sizeof(tbm_bitmap_t) - 1) == 1);
+ PASS("test_bit");
+}
+
+static void
+test_count_bits()
+{
+ unsigned max_bits = sizeof(tbm_bitmap_t) * 8;
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+
+ assert(count_bits(0) == 0);
+ assert(count_bits(1) == 1);
+ assert(count_bits(2) == 1);
+ assert(count_bits(3) == 2);
+ assert(count_bits(ones) == max_bits);
+ assert(count_bits(~1) == max_bits - 1);
+
+ /* count_bits(0x5555....) */
+ assert(count_bits(ones / 3) == max_bits / 2);
+ /* count_bits(0x3333...) */
+ assert(count_bits(ones / 5) == max_bits / 2);
+ /* count_bits(0x0f0f...) */
+ assert(count_bits(ones / 17) == max_bits / 2);
+ /* count_bits(0x1010...) */
+ assert(count_bits(ones / 255) == max_bits / 8);
+
+ PASS("test_count_bits");
+}
+
+static void
+test_count_bits_before()
+{
+ unsigned max_bits = sizeof(tbm_bitmap_t) * 8;
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+ unsigned i;
+
+ for (i = 0; i < max_bits; i++) {
+ assert(count_bits_before(0, i) == 0);
+ assert(count_bits_before(ones, i) == i);
+ }
+
+ PASS("test_count_bits_before");
+}
+
+static void
+test_count_bits_from()
+{
+ unsigned max_bits = sizeof(tbm_bitmap_t) * 8;
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+ unsigned i;
+
+ for (i = 0; i < max_bits; i++) {
+ assert(count_bits_from(0, i) == 0);
+ assert(count_bits_from(ones, i) == max_bits - i);
+ }
+
+ PASS("test_count_bits_from");
+}
+
+static void
+test_extract_bits()
+{
+ static btrie_oct_t prefix[] = {0xff, 0x55, 0xaa, 0x00};
+ unsigned i;
+
+ for (i = 0; i < 32; i++)
+ assert(extract_bits(prefix, i, 0) == 0);
+
+ for (i = 0; i < 8; i++)
+ assert(extract_bits(prefix, i, 1) == 1);
+ for (i = 8; i < 16; i++)
+ assert(extract_bits(prefix, i, 1) == i % 2);
+ for (i = 16; i < 24; i++)
+ assert(extract_bits(prefix, i, 1) == (i + 1) % 2);
+ for (i = 24; i < 32; i++)
+ assert(extract_bits(prefix, i, 1) == 0);
+
+ assert(extract_bits(prefix, 2, 6) == 0x3f);
+ assert(extract_bits(prefix, 3, 6) == 0x3e);
+ assert(extract_bits(prefix, 4, 6) == 0x3d);
+ assert(extract_bits(prefix, 5, 6) == 0x3a);
+ assert(extract_bits(prefix, 6, 6) == 0x35);
+ assert(extract_bits(prefix, 7, 6) == 0x2a);
+ assert(extract_bits(prefix, 8, 6) == 0x15);
+
+ PASS("test_extract_bits");
+}
+
+static void
+test_high_bits()
+{
+ assert(high_bits(0) == 0x00);
+ assert(high_bits(1) == 0x80);
+ assert(high_bits(2) == 0xc0);
+ assert(high_bits(3) == 0xe0);
+ assert(high_bits(4) == 0xf0);
+ assert(high_bits(5) == 0xf8);
+ assert(high_bits(6) == 0xfc);
+ assert(high_bits(7) == 0xfe);
+ assert(high_bits(8) == 0xff);
+ PASS("test_high_bits");
+}
+
+static void
+test_prefixes_equal()
+{
+ btrie_oct_t prefix1[LC_BYTES_PER_NODE];
+ btrie_oct_t prefix2[LC_BYTES_PER_NODE];
+ unsigned i;
+ memset(prefix1, 0xaa, LC_BYTES_PER_NODE);
+ memset(prefix2, 0xaa, LC_BYTES_PER_NODE);
+
+ for (i = 0; i < 8 * LC_BYTES_PER_NODE; i++) {
+ assert(prefixes_equal(prefix1, prefix2, i));
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ assert(!prefixes_equal(prefix1, prefix2, 8 * LC_BYTES_PER_NODE));
+ assert(prefixes_equal(prefix1, prefix2, i));
+ if (i + 1 < 8 * LC_BYTES_PER_NODE)
+ assert(!prefixes_equal(prefix1, prefix2, i + 1));
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ }
+ PASS("test_prefixes_equal");
+}
+
+static void
+test_common_prefix()
+{
+ btrie_oct_t prefix1[LC_BYTES_PER_NODE];
+ btrie_oct_t prefix2[LC_BYTES_PER_NODE];
+ unsigned i;
+ memset(prefix1, 0x55, LC_BYTES_PER_NODE);
+ memset(prefix2, 0x55, LC_BYTES_PER_NODE);
+
+ for (i = 0; i < 8 * LC_BYTES_PER_NODE; i++) {
+ assert(common_prefix(prefix1, prefix2, i) == i);
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ assert(common_prefix(prefix1, prefix2, 8 * LC_BYTES_PER_NODE) == i);
+ if (i + 1 < 8 * LC_BYTES_PER_NODE)
+ assert(common_prefix(prefix1, prefix2, i+1) == i);
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ }
+ PASS("test_common_prefix");
+}
+
+static void
+test_base_index()
+{
+ assert(base_index(0,0) == 1);
+ assert(base_index(0,1) == 2);
+ assert(base_index(1,1) == 3);
+ assert(base_index(0,2) == 4);
+ assert(base_index(1,2) == 5);
+ assert(base_index(2,2) == 6);
+ assert(base_index(3,2) == 7);
+ PASS("test_base_index");
+}
+
+static void
+test_has_internal_data()
+{
+ struct tbm_node node;
+ unsigned plen, pfx, bi;
+ for (plen = 0; plen < TBM_STRIDE; plen++) {
+ for (pfx = 0; pfx < 1U << plen; pfx++) {
+ tbm_bitmap_t ancestor_mask = 0;
+ for (bi = base_index(pfx, plen); bi; bi >>= 1) {
+ node.int_bm = bit(bi);
+ ancestor_mask |= bit(bi);
+ assert(has_internal_data(&node, pfx, plen));
+ }
+ node.int_bm = ~ancestor_mask;
+ assert(!has_internal_data(&node, pfx, plen));
+ }
+ }
+ PASS("test_has_internal_data");
+}
+
+/****************************************************************/
+static const btrie_oct_t numbered_bytes[] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+};
+
+static void
+check_non_terminal_lc_node(struct lc_node *node, unsigned len)
+{
+ assert(is_lc_node((node_t *)node));
+ assert(!lc_is_terminal(node));
+ assert(lc_len(node) == len);
+}
+
+static void
+check_terminal_lc_node(struct lc_node *node, unsigned len, const void *data)
+{
+ assert(is_lc_node((node_t *)node));
+ assert(lc_is_terminal(node));
+ assert(lc_len(node) == len);
+ assert(node->ptr.data == data);
+}
+
+static void
+test_init_terminal_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node;
+ struct lc_node *head = &node.lc_node;
+
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ check_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE, data);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE) == 0);
+
+ init_terminal_node(btrie, &node, 7,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ check_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE - 7, data);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE) == 0);
+
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 2 * 8 * LC_BYTES_PER_NODE, data);
+ check_non_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE) == 0);
+ {
+ struct lc_node *child = &head->ptr.child->lc_node;
+ check_terminal_lc_node(child, 8 * LC_BYTES_PER_NODE, data);
+ assert(memcmp(child->prefix, &numbered_bytes[LC_BYTES_PER_NODE],
+ LC_BYTES_PER_NODE) == 0);
+ }
+
+ init_terminal_node(btrie, &node, 15,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE + 15, data);
+ check_non_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE - 7);
+ assert(memcmp(head->prefix, &numbered_bytes[1], LC_BYTES_PER_NODE) == 0);
+ {
+ struct lc_node *child = &head->ptr.child->lc_node;
+ check_terminal_lc_node(child, 7, data);
+ assert(child->prefix[0] == numbered_bytes[LC_BYTES_PER_NODE + 1]);
+ }
+
+ PASS("test_init_terminal_node");
+}
+
+static void
+test_coalesce_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node;
+ struct lc_node *head = &node.lc_node;
+
+ /* test merging */
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * (LC_BYTES_PER_NODE + 1), data);
+ check_non_terminal_lc_node(head, LC_BYTES_PER_NODE * 8);
+ lc_add_to_len(head, -8);
+ coalesce_lc_node(btrie, head, 8);
+ check_terminal_lc_node(head, LC_BYTES_PER_NODE * 8, data);
+ assert(head->prefix[LC_BYTES_PER_NODE - 1]
+ == numbered_bytes[LC_BYTES_PER_NODE]);
+
+ /* test bit stealing */
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * (2 * LC_BYTES_PER_NODE), data);
+ check_non_terminal_lc_node(head, LC_BYTES_PER_NODE * 8);
+ lc_add_to_len(head, -15);
+ coalesce_lc_node(btrie, head, 15);
+ check_non_terminal_lc_node(head, LC_BYTES_PER_NODE * 8 - 7);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE - 1) == 0);
+ assert(head->prefix[LC_BYTES_PER_NODE - 1]
+ == numbered_bytes[LC_BYTES_PER_NODE]);
+ {
+ struct lc_node *child = &head->ptr.child->lc_node;
+ check_terminal_lc_node(child, 8 * (LC_BYTES_PER_NODE - 1), data);
+ assert(memcmp(child->prefix, &numbered_bytes[LC_BYTES_PER_NODE + 1],
+ LC_BYTES_PER_NODE - 1) == 0);
+ }
+
+ PASS("test_coalesce_lc_node");
+}
+
+static void
+test_shorten_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node, shorter;
+
+ /* test shorten without shift */
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ memset(shorter.lc_node.prefix, 0xff, LC_BYTES_PER_NODE);
+ shorten_lc_node(btrie, &shorter, 7, &node.lc_node, 0);
+ check_terminal_lc_node(&shorter.lc_node, LC_BYTES_PER_NODE * 8 - 7, data);
+ assert(memcmp(shorter.lc_node.prefix, numbered_bytes, LC_BYTES_PER_NODE)
+ == 0);
+
+ /* test shorten with shift */
+ init_terminal_node(btrie, &node, 7,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ memset(shorter.lc_node.prefix, 0xff, LC_BYTES_PER_NODE);
+ shorten_lc_node(btrie, &shorter, 9, &node.lc_node, 7);
+ check_terminal_lc_node(&shorter.lc_node, LC_BYTES_PER_NODE * 8 - 9, data);
+ assert(memcmp(shorter.lc_node.prefix, &numbered_bytes[1],
+ LC_BYTES_PER_NODE - 1) == 0);
+
+ {
+ /* test child stealing */
+ struct lc_node head;
+ node_t tail, shorter;
+
+ lc_init_flags(&head, 0, 7);
+ head.ptr.child = &tail;
+ init_empty_node(btrie, &tail);
+
+ shorten_lc_node(btrie, &shorter, 7, &head, 0);
+ assert(is_empty_node(&shorter));
+ }
+
+ PASS("test_shorten_lc_node");
+}
+
+static void
+test_split_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ struct lc_node node;
+
+ init_terminal_node(btrie, (node_t *)&node, 1, numbered_bytes, 25, data);
+ split_lc_node(btrie, &node, 1, 8);
+ check_non_terminal_lc_node(&node, 8);
+ check_terminal_lc_node(&node.ptr.child->lc_node, 16, data);
+
+ /* test conversion of terminal to non-terminal */
+ init_terminal_node(btrie, (node_t *)&node, 7, numbered_bytes, 10, data);
+ split_lc_node(btrie, &node, 7, 3);
+ check_non_terminal_lc_node(&node, 3);
+ check_terminal_lc_node(&node.ptr.child->lc_node, 0, data);
+
+ PASS("test_split_lc_node");
+}
+
+static void
+test_convert_lc_node_1()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ struct lc_node head;
+
+ /* test tail is left */
+ lc_init_flags(&head, 0, 1);
+ head.prefix[0] = 0;
+ head.ptr.child = alloc_nodes(btrie, 1, 0);
+ init_terminal_node(btrie, head.ptr.child, 1, numbered_bytes, 1, data);
+ convert_lc_node_1(btrie, &head, 0);
+ {
+ node_t *result = (node_t *)&head;
+ assert(is_tbm_node(result));
+ assert(result->tbm_node.ext_bm == 0);
+ assert(result->tbm_node.int_bm == bit(base_index(0, 1)));
+ assert(*tbm_data_p(&result->tbm_node, 0, 1) == data);
+ }
+
+ /* test tail is right */
+ lc_init_flags(&head, 0, 1);
+ head.prefix[0] = 1;
+ head.ptr.child = alloc_nodes(btrie, 1, 0);
+ init_terminal_node(btrie, head.ptr.child, 8, numbered_bytes, 10, data);
+ convert_lc_node_1(btrie, &head, 7);
+ {
+ node_t *result = (node_t *)&head;
+ assert(is_tbm_node(result));
+ assert(result->tbm_node.ext_bm == 0);
+ assert(result->tbm_node.int_bm == bit(base_index(4, 3)));
+ assert(*tbm_data_p(&result->tbm_node, 4, 3) == data);
+ }
+
+ PASS("test_convert_lc_node_1");
+}
+
+static void
+test_convert_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node;
+
+ /* if (len >= TBM_STRIDE) */
+ init_terminal_node(btrie, &node, 7, numbered_bytes, TBM_STRIDE + 7, data);
+ convert_lc_node(btrie, &node.lc_node, 7);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == bit(0));
+ assert(node.tbm_node.int_bm == 0);
+ check_terminal_lc_node(&tbm_ext_path(&node.tbm_node, 0)->lc_node, 0, data);
+
+ /* if (lc_is_terminal(node)) */
+ init_terminal_node(btrie, &node, 0, numbered_bytes, 0, data);
+ convert_lc_node(btrie, &node.lc_node, 0);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(0, 0)));
+ assert(*tbm_data_p(&node.tbm_node, 0, 0) == data);
+
+ /* else */
+ lc_init_flags(&node.lc_node, 0, TBM_STRIDE - 1);
+ node.lc_node.prefix[0] = 0;
+ node.lc_node.ptr.child = alloc_nodes(btrie, 1, 0);
+ init_empty_node(btrie, node.lc_node.ptr.child);
+ tbm_insert_data(btrie, &node.lc_node.ptr.child->tbm_node, 0, 0, data);
+
+ convert_lc_node(btrie, &node.lc_node, 0);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(0, TBM_STRIDE - 1)));
+ assert(*tbm_data_p(&node.tbm_node, 0, TBM_STRIDE - 1) == data);
+
+ PASS("test_convert_lc_node");
+}
+
+static void
+test_insert_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node, tail;
+
+ /* test optimized case, last_bit == 0 */
+ init_terminal_node(btrie, &tail, 9, numbered_bytes, 17, data);
+ insert_lc_node(btrie, &node, 8, 0, 0, &tail);
+ check_terminal_lc_node(&node.lc_node, 9, data);
+ assert(memcmp(node.lc_node.prefix, &numbered_bytes[1], 2) == 0);
+
+ /* test optimized case, last_bit == 1 */
+ init_terminal_node(btrie, &tail, 7, &numbered_bytes[0x12], 15, data);
+ insert_lc_node(btrie, &node, 6, 0x10, 1, &tail);
+ check_terminal_lc_node(&node.lc_node, 9, data);
+ assert(node.lc_node.prefix[0] == 0x12);
+ assert(node.lc_node.prefix[1] == 0x13);
+
+ /* test with shift */
+ init_terminal_node(btrie, &tail, 0, numbered_bytes, 8, data);
+ insert_lc_node(btrie, &node, 7, 0x40, 1, &tail);
+ check_terminal_lc_node(&node.lc_node, 9, data);
+ assert(node.lc_node.prefix[0] == 0x41);
+ assert(node.lc_node.prefix[1] == numbered_bytes[0]);
+
+ /* test with TBM node */
+ init_empty_node(btrie, &tail);
+ insert_lc_node(btrie, &node, 6, 0x40, 0, &tail);
+ check_non_terminal_lc_node(&node.lc_node, 1);
+ assert(is_tbm_node(node.lc_node.ptr.child));
+
+ PASS("test_insert_lc_node");
+}
+
+static void
+test_next_pbyte()
+{
+ assert(next_pbyte(0xff, 0, 1) == 0x80 >> (TBM_STRIDE - 1));
+ assert(next_pbyte(0xff, 1, 1) == (0x80 | (0x80 >> TBM_STRIDE)));
+ assert(next_pbyte(0xff, 2, 1) == (0xc0 | (0x80 >> (TBM_STRIDE + 1))));
+ assert(next_pbyte(0xff, 8 - TBM_STRIDE, 1) == 0);
+ assert(next_pbyte(0xff, 9 - TBM_STRIDE, 1) == 0x80);
+
+ PASS("test_next_pbyte");
+}
+
+static void
+test_init_tbm_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ unsigned lr;
+ node_t node;
+
+ /* test root data */
+ init_tbm_node(btrie, &node, 0, 0, &data, NULL, NULL);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(0, 0)));
+ assert(*tbm_data_p(&node.tbm_node, 0, 0) == data);
+
+ for (lr = 0; lr < 2; lr++) {
+ node_t child;
+ node_t *left = lr ? NULL : &child;
+ node_t *right = lr ? &child : NULL;
+ unsigned base = lr ? (1U << (TBM_STRIDE - 1)) : 0;
+ unsigned pfx;
+
+ /* test with long LC node child */
+ init_terminal_node(btrie, &child, 1, numbered_bytes, TBM_STRIDE + 1, data);
+ init_tbm_node(btrie, &node, 0, 0, NULL, left, right);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == bit(base));
+ assert(node.tbm_node.int_bm == 0);
+ check_terminal_lc_node(&tbm_ext_path(&node.tbm_node, base)->lc_node,
+ 1, data);
+
+ /* test with short LC node children */
+ init_terminal_node(btrie, &child, 1, numbered_bytes, TBM_STRIDE - 1, data);
+ init_tbm_node(btrie, &node, 0, 0, NULL, left, right);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(base >> 1, TBM_STRIDE-1)));
+ assert(*tbm_data_p(&node.tbm_node, base >> 1, TBM_STRIDE-1) == data);
+
+ /* construct TBM node with all eight combinations of having data,
+ * left_ext and/or right_ext in its extending paths */
+ init_empty_node(btrie, &child);
+ for (pfx = 0; pfx < 8; pfx++) {
+ if (pfx & 1)
+ tbm_insert_data(btrie, &child.tbm_node, pfx, TBM_STRIDE - 1, data);
+ if (pfx & 2) {
+ btrie_oct_t prefix0 = 0;
+ init_terminal_node(btrie,
+ tbm_insert_ext_path(btrie, &child.tbm_node, 2*pfx),
+ TBM_STRIDE + 1,
+ &prefix0, TBM_STRIDE + 2, data);
+ }
+ if (pfx & 4) {
+ btrie_oct_t prefix0 = 0x80 >> TBM_STRIDE;
+ init_terminal_node(btrie,
+ tbm_insert_ext_path(btrie, &child.tbm_node, 2*pfx+1),
+ TBM_STRIDE + 1,
+ &prefix0, TBM_STRIDE + 3, data);
+ }
+ }
+ init_tbm_node(btrie, &node, 0, 0, NULL, left, right);
+ for (pfx = 0; pfx < 8; pfx++) {
+ unsigned base = lr ? (1U << (TBM_STRIDE - 1)) : 0;
+ node_t *ext_path = tbm_ext_path(&node.tbm_node, base + pfx);
+ if (pfx == 0)
+ assert(ext_path == NULL);
+ else if (pfx == 1)
+ check_terminal_lc_node(&ext_path->lc_node, 0, data);
+ else if (pfx == 2) {
+ check_terminal_lc_node(&ext_path->lc_node, 2, data);
+ assert(ext_path->lc_node.prefix[0] == 0);
+ }
+ else if (pfx == 4) {
+ check_terminal_lc_node(&ext_path->lc_node, 3, data);
+ assert(ext_path->lc_node.prefix[0] == (0x80 >> TBM_STRIDE));
+ }
+ else {
+ tbm_bitmap_t int_bm = 0;
+ assert(is_tbm_node(ext_path));
+ if (pfx & 1) {
+ int_bm |= bit(base_index(0, 0));
+ assert(*tbm_data_p(&ext_path->tbm_node, 0, 0) == data);
+ }
+ if (pfx & 2) {
+ int_bm |= bit(base_index(0, 2));
+ assert(*tbm_data_p(&ext_path->tbm_node, 0, 2) == data);
+ }
+ if (pfx & 4) {
+ int_bm |= bit(base_index(4, 3));
+ assert(*tbm_data_p(&ext_path->tbm_node, 4, 3) == data);
+ }
+ assert(ext_path->tbm_node.int_bm == int_bm);
+ }
+ }
+ }
+
+ PASS("test_init_tbm_node");
+}
+
+static void
+test_add_to_trie()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ enum btrie_result result;
+ unsigned pfx, plen;
+ node_t root;
+
+ /* test initial insertion */
+ init_empty_node(btrie, &root);
+ result = add_to_trie(btrie, &root, 0,
+ numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE, data);
+ assert(result == BTRIE_OKAY);
+ check_non_terminal_lc_node(&root.lc_node, 8 * LC_BYTES_PER_NODE);
+ check_terminal_lc_node(&root.lc_node.ptr.child->lc_node,
+ 8 * LC_BYTES_PER_NODE, data);
+
+ /* test can follow LC node to tail, and then detect duplicate prefix */
+ result = add_to_trie(btrie, &root, 0,
+ numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE, data);
+ assert(result == BTRIE_DUPLICATE_PREFIX);
+
+ /* test can insert new TBM node within existing LC node */
+ result = add_to_trie(btrie, &root, 0,
+ &numbered_bytes[1], 16, data);
+ assert(result == BTRIE_OKAY);
+ check_non_terminal_lc_node(&root.lc_node, 7);
+ assert(is_tbm_node(root.lc_node.ptr.child));
+
+ /* test can convert terminal LC node to TBM node */
+ init_terminal_node(btrie, &root, 0, numbered_bytes, 12, data);
+ result = add_to_trie(btrie, &root, 0, numbered_bytes, 24, data);
+ assert(result == BTRIE_OKAY);
+ check_non_terminal_lc_node(&root.lc_node, 12);
+ assert(is_tbm_node(root.lc_node.ptr.child));
+
+ /* test can insert internal prefix data in TBM node */
+ for (plen = 0; plen < TBM_STRIDE; plen++) {
+ for (pfx = 0; pfx < (1U << plen); pfx++) {
+ btrie_oct_t prefix0 = plen ? pfx << (8 - plen) : 0;
+ init_empty_node(btrie, &root);
+ init_terminal_node(btrie, tbm_insert_ext_path(btrie, &root.tbm_node, 0),
+ TBM_STRIDE,
+ numbered_bytes, 8, data);
+ result = add_to_trie(btrie, &root, 0, &prefix0, plen, data);
+ assert(result == BTRIE_OKAY);
+ assert(is_tbm_node(&root));
+ assert(root.tbm_node.ext_bm == bit(0));
+ assert(root.tbm_node.int_bm == bit(base_index(pfx, plen)));
+ assert(*tbm_data_p(&root.tbm_node, pfx, plen) == data);
+
+ result = add_to_trie(btrie, &root, 0, &prefix0, plen, data);
+ assert(result == BTRIE_DUPLICATE_PREFIX);
+ }
+ }
+
+ /* test can add extending paths to TBM node */
+ for (pfx = 0; pfx < (1U << TBM_STRIDE); pfx++) {
+ btrie_oct_t prefix0 = pfx << (8 - TBM_STRIDE);
+ init_empty_node(btrie, &root);
+ tbm_insert_data(btrie, &root.tbm_node, 0, 0, data);
+ result = add_to_trie(btrie, &root, 0, &prefix0, 8, data);
+ assert(result == BTRIE_OKAY);
+ assert(is_tbm_node(&root));
+ assert(root.tbm_node.ext_bm == bit(pfx));
+ assert(root.tbm_node.int_bm == bit(base_index(0, 0)));
+ check_terminal_lc_node(&tbm_ext_path(&root.tbm_node, pfx)->lc_node,
+ 8 - TBM_STRIDE, data);
+
+ result = add_to_trie(btrie, &root, 0, &prefix0, 8, data);
+ assert(result == BTRIE_DUPLICATE_PREFIX);
+ }
+
+ /* test can follow extending path */
+ init_empty_node(btrie, &root);
+ init_terminal_node(btrie,
+ tbm_insert_ext_path(btrie, &root.tbm_node, 0), TBM_STRIDE,
+ numbered_bytes, 8, data);
+ result = add_to_trie(btrie, &root, 0, numbered_bytes, 7, data);
+ assert(result == BTRIE_OKAY);
+ assert(root.tbm_node.ext_bm == bit(0));
+ assert(root.tbm_node.int_bm == 0);
+ check_non_terminal_lc_node(&root.tbm_node.ptr.children[0].lc_node,
+ 7 - TBM_STRIDE);
+
+ PASS("test_add_to_trie");
+}
+
+static void
+test_search_trie()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data01 = (void *)0xdead0001;
+ const void *data11 = (void *)0xdead0101;
+ const void *data = (void *)0xdeadbeef;
+ unsigned plen, pfx;
+ node_t root;
+
+ /* test can follow chain of LC nodes to an exact match */
+ init_empty_node(btrie, &root);
+ add_to_trie(btrie, &root, 0,
+ numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE, data);
+
+ assert(search_trie(&root, 0, numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE)
+ == data);
+ assert(search_trie(&root, 0, numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE + 1)
+ == data);
+ assert(search_trie(&root, 0, numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE - 1)
+ == NULL);
+ assert(search_trie(&root, 0, &numbered_bytes[1], 8 * 2 * LC_BYTES_PER_NODE)
+ == NULL);
+
+ /* test can follow extending path to an exact match */
+ for (pfx = 0; pfx < (1U << TBM_STRIDE); pfx++) {
+ btrie_oct_t prefix0 = pfx << (8 - TBM_STRIDE);
+ init_empty_node(btrie, &root);
+ tbm_insert_data(btrie, &root.tbm_node, 0, 1, data01);
+ tbm_insert_data(btrie, &root.tbm_node, 1, 1, data11);
+ add_to_trie(btrie, &root, 0, &prefix0, 8, data);
+ assert(search_trie(&root, 0, &prefix0, 8) == data);
+ /* test that last matching TBM internal prefix gets picked up */
+ if (prefix0 & 0x80)
+ assert(search_trie(&root, 0, &prefix0, 7) == data11);
+ else
+ assert(search_trie(&root, 0, &prefix0, 7) == data01);
+ prefix0 ^= 1 << (8 - TBM_STRIDE);
+ if (prefix0 & 0x80)
+ assert(search_trie(&root, 0, &prefix0, 8) == data11);
+ else
+ assert(search_trie(&root, 0, &prefix0, 8) == data01);
+ }
+
+ /* test finding of TBM internal prefixes */
+ init_empty_node(btrie, &root);
+ tbm_insert_data(btrie, &root.tbm_node, 0, 1, data01);
+ tbm_insert_data(btrie, &root.tbm_node, 1, 1, data11);
+
+ assert(search_trie(&root, 0, numbered_bytes, 0) == NULL);
+ for (plen = 1; plen < TBM_STRIDE; plen++) {
+ for (pfx = 0; pfx < (1U << TBM_STRIDE); pfx++) {
+ btrie_oct_t prefix0 = pfx << (8 - plen);
+ if (prefix0 & 0x80)
+ assert(search_trie(&root, 0, &prefix0, plen) == data11);
+ else
+ assert(search_trie(&root, 0, &prefix0, plen) == data01);
+ }
+ }
+
+ PASS("test_search_trie");
+}
+
+static int
+unit_tests()
+{
+ test_struct_node_packing();
+ test_bit();
+ test_count_bits();
+ test_count_bits_before();
+ test_count_bits_from();
+ test_extract_bits();
+ test_high_bits();
+ test_prefixes_equal();
+ test_common_prefix();
+ test_base_index();
+ test_has_internal_data();
+
+ test_init_terminal_node();
+ test_coalesce_lc_node();
+ test_shorten_lc_node();
+ test_split_lc_node();
+ test_convert_lc_node_1();
+ test_convert_lc_node();
+ test_insert_lc_node();
+ test_next_pbyte();
+ test_init_tbm_node();
+ test_add_to_trie();
+ test_search_trie();
+
+ puts("\nOK");
+ return 0;
+}
+
+/*****************************************************************
+ *
+ * btrie_dump: print out the trie structure (for testing)
+ *
+ */
+#define INDENT_FILL "....:....|....:....|....:....|....:....|"
+
+static void dump_node(const node_t *node, unsigned pos, btrie_oct_t *prefix,
+ int indent);
+
+static void
+dump_prefix(btrie_oct_t *prefix, unsigned len, int indent, const char *tail)
+{
+ unsigned i;
+
+ printf("%*.*s0x", indent, indent, INDENT_FILL);
+ for (i = 0; i < len / 8; i++)
+ printf("%02x", prefix[i]);
+ if (len % 8)
+ printf("%02x", prefix[len / 8] & high_bits(len % 8));
+ printf("/%u%s", len, tail);
+}
+
+/* the opposite of extract_bits, sets a short string of bits from integer */
+static void
+insert_bits(btrie_oct_t *prefix, unsigned pos, btrie_oct_t pfx, unsigned nbits)
+{
+ if (nbits != 0) {
+ unsigned v = (prefix[pos / 8] << 8) + prefix[pos / 8 + 1];
+ unsigned mask = (1U << nbits) - 1;
+ unsigned shift = 16 - (pos % 8) - nbits;
+ v = (v & ~(mask << shift)) | (pfx << shift);
+ prefix[pos / 8] = v >> 8;
+ prefix[pos / 8 + 1] = (btrie_oct_t)v;
+ }
+}
+
+static void
+dump_tbm_node(const struct tbm_node *node, unsigned pos,
+ btrie_oct_t *prefix, int indent)
+{
+ unsigned pfx = 0, plen = 0;
+
+ dump_prefix(prefix, pos, indent, " [tbm]\n");
+
+ for (;;) {
+ if (plen < TBM_STRIDE) {
+ const void **data_p = tbm_data_p(node, pfx, plen);
+ if (data_p) {
+ insert_bits(prefix, pos, pfx, plen);
+ dump_prefix(prefix, pos + plen, indent, "");
+ printf(" [%u/%u] (%s)\n", pfx, plen, (const char *)*data_p);
+ }
+ plen++;
+ pfx <<= 1;
+ }
+ else {
+ const node_t *ext_path = tbm_ext_path(node, pfx);
+ if (ext_path) {
+ insert_bits(prefix, pos, pfx, TBM_STRIDE);
+ dump_node(ext_path, pos + TBM_STRIDE, prefix, indent + 1);
+ }
+ while (pfx & 1) {
+ if (--plen == 0)
+ return;
+ pfx >>= 1;
+ }
+ pfx++;
+ }
+ }
+}
+
+static void
+dump_lc_node(const struct lc_node *node, unsigned pos,
+ btrie_oct_t *prefix, int indent)
+{
+ unsigned end = pos + lc_len(node);
+ btrie_oct_t save_prefix = prefix[lc_shift(pos)];
+
+ memcpy(&prefix[lc_shift(pos)], node->prefix, lc_bytes(node, pos));
+
+ if (lc_is_terminal(node)) {
+ dump_prefix(prefix, end, indent, "");
+ printf(" (%s)\n", (const char *)node->ptr.data);
+ }
+ else {
+ dump_prefix(prefix, end, indent, "\n");
+ dump_node(node->ptr.child, end, prefix, indent + 1);
+ }
+
+ prefix[lc_shift(pos)] = save_prefix;
+ if (lc_bytes(node, pos) > 1)
+ memset(&prefix[lc_shift(pos) + 1], 0, lc_bytes(node, pos) - 1);
+}
+
+static void
+dump_node(const node_t *node, unsigned pos, btrie_oct_t *prefix, int indent)
+{
+ if (is_lc_node(node))
+ dump_lc_node(&node->lc_node, pos, prefix, indent);
+ else
+ dump_tbm_node(&node->tbm_node, pos, prefix, indent);
+}
+
+static void
+btrie_dump(struct btrie *btrie)
+{
+ btrie_oct_t prefix[(BTRIE_MAX_PREFIX + 7) / 8];
+
+ memset(prefix, 0, sizeof(prefix));
+ dump_node(&btrie->root, 0, prefix, 0);
+ puts(btrie_stats(btrie));
+}
+
+/****************************************************************
+ *
+ * test program - just enough to construct a trie and preform a lookup
+ *
+ */
+
+#include <arpa/inet.h>
+
+static int
+parse_prefix(const char *arg, btrie_oct_t prefix[16], unsigned *len)
+{
+ char addrbuf[128];
+ return sscanf(arg, "%127[0-9a-fA-F:]/%u", addrbuf, len) == 2
+ && inet_pton(AF_INET6, addrbuf, prefix) == 1;
+}
+
+static int
+test_btrie(int argc, char *argv[])
+{
+ struct btrie *btrie = btrie_init(NULL);
+ int i;
+ btrie_oct_t prefix[16];
+ unsigned len;
+
+ for (i = 1; i < argc-1; i++) {
+ if (!parse_prefix(argv[i], prefix, &len)) {
+ fprintf(stderr, "Can not parse arg '%s'\n", argv[i]);
+ return 1;
+ }
+ btrie_add_prefix(btrie, prefix, len, argv[i]);
+ }
+
+ btrie_dump(btrie);
+
+ if (argc > 1) {
+ const void *data;
+
+ if (!parse_prefix(argv[argc-1], prefix, &len)) {
+ fprintf(stderr, "Can not parse arg '%s'\n", argv[argc-1]);
+ return 1;
+ }
+ data = btrie_lookup(btrie, prefix, 128);
+ printf("lookup(%s) => %s\n", argv[argc-1], (const char *)data);
+ }
+
+ return 0;
+}
+
+int
+main(int argc, char *argv[])
+{
+ if ((pgm_name = strrchr(argv[0], '/')) != NULL)
+ pgm_name++;
+ else
+ pgm_name = argv[0];
+
+ if (argc > 1)
+ return test_btrie(argc, argv);
+ else
+ return unit_tests();
+}
+
+#endif /* TEST */
diff --git a/contrib/lc-btrie/btrie.h b/contrib/lc-btrie/btrie.h
new file mode 100644
index 0000000..370a03e
--- /dev/null
+++ b/contrib/lc-btrie/btrie.h
@@ -0,0 +1,83 @@
+/* Level-Compressed Tree Bitmap (LC-TBM) Trie implementation
+ *
+ * Contributed by Geoffrey T. Dairiki <dairiki@dairiki.org>
+ *
+ * This file is released under a "Three-clause BSD License".
+ *
+ * Copyright (c) 2013, Geoffrey T. Dairiki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * * Neither the name of Geoffrey T. Dairiki nor the names of other
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GEOFFREY
+ * T. DAIRIKI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+#ifndef _BTRIE_H_INCLUDED
+#define _BTRIE_H_INCLUDED
+
+#include "config.h"
+
+#include <stdint.h>
+typedef uint8_t btrie_oct_t;
+
+/* maximum length of bit string btrie_walk() can handle
+ *
+ * note: this limit is necessitated by the use of fixed length buffers
+ * in btrie_walk() --- btrie_add_prefix() and btrie_lookup() impose no
+ * limit on the length of bitstrings
+ */
+#define BTRIE_MAX_PREFIX 128
+
+struct btrie;
+struct memory_pool_s;
+
+struct btrie * btrie_init(struct memory_pool_s *mp);
+
+enum btrie_result
+{
+ BTRIE_OKAY = 0,
+ BTRIE_ALLOC_FAILED = -1,
+ BTRIE_DUPLICATE_PREFIX = 1
+};
+
+enum btrie_result btrie_add_prefix(struct btrie *btrie,
+ const btrie_oct_t *prefix, unsigned len, const void *data);
+
+const void *btrie_lookup(const struct btrie *btrie, const btrie_oct_t *pfx,
+ unsigned len);
+
+const char *btrie_stats(const struct btrie *btrie, guint duplicates);
+
+#ifndef NO_MASTER_DUMP
+typedef void btrie_walk_cb_t(const btrie_oct_t *prefix, unsigned len,
+ const void *data, int post, void *user_data);
+
+void btrie_walk(const struct btrie *btrie, btrie_walk_cb_t *callback,
+ void *user_data);
+#endif /* not NO_MASTER_DUMP */
+
+#endif /* _BTRIE_H_INCLUDED */
diff --git a/contrib/libev/CMakeLists.txt b/contrib/libev/CMakeLists.txt
new file mode 100644
index 0000000..b2d33fe
--- /dev/null
+++ b/contrib/libev/CMakeLists.txt
@@ -0,0 +1,82 @@
+SET(LIBEVSRC ev.c)
+
+CHECK_INCLUDE_FILES(sys/types.h HAVE_SYS_TYPES_H)
+CHECK_INCLUDE_FILES(sys/inotify.h HAVE_SYS_INOTIFY_H)
+CHECK_INCLUDE_FILES(sys/epoll.h HAVE_SYS_EPOLL_H)
+CHECK_INCLUDE_FILES("sys/types.h;sys/event.h;sys/time.h" HAVE_SYS_EVENT_H)
+CHECK_INCLUDE_FILES(sys/queue.h HAVE_SYS_QUEUE_H)
+CHECK_INCLUDE_FILES(sys/stat.h HAVE_SYS_STAT_H)
+CHECK_INCLUDE_FILES(sys/signalfd.h HAVE_SYS_SIGNALFD_H)
+CHECK_INCLUDE_FILES(port.h HAVE_PORT_H)
+CHECK_INCLUDE_FILES(poll.h HAVE_POLL_H)
+CHECK_INCLUDE_FILES(memory.h HAVE_MEMORY_H)
+CHECK_INCLUDE_FILES(sys/select.h HAVE_SYS_SELECT_H)
+CHECK_INCLUDE_FILES(sys/eventfd.h HAVE_SYS_EVENTFD_H)
+CHECK_INCLUDE_FILES(sys/timerfd.h HAVE_SYS_TIMERFD_H)
+CHECK_INCLUDE_FILES(linux/fs.h HAVE_LINUX_FS_H)
+CHECK_INCLUDE_FILES(linux/aio_abi.h HAVE_LINUX_AIO_ABI_H)
+
+IF(HAVE_SYS_INOTIFY_H)
+ CHECK_SYMBOL_EXISTS(inotify_init "sys/types.h;sys/inotify.h" HAVE_INOTIFY_INIT)
+ENDIF()
+IF(HAVE_SYS_EPOLL_H)
+ CHECK_SYMBOL_EXISTS(epoll_ctl "sys/types.h;sys/epoll.h" HAVE_EPOLL_CTL)
+ENDIF()
+IF(HAVE_SYS_EVENT_H)
+ CHECK_SYMBOL_EXISTS(kqueue "sys/types.h;sys/event.h;sys/time.h" HAVE_KQUEUE)
+ENDIF()
+IF(HAVE_PORT_H)
+ CHECK_SYMBOL_EXISTS(port_create port.h HAVE_PORT_CREATE)
+ENDIF()
+IF(HAVE_POLL_H)
+ CHECK_SYMBOL_EXISTS(poll poll.h HAVE_POLL)
+ENDIF()
+IF(HAVE_SYS_SELECT_H)
+ CHECK_SYMBOL_EXISTS(select sys/select.h HAVE_SELECT)
+ENDIF()
+IF(HAVE_SYS_EVENTFD_H)
+ CHECK_SYMBOL_EXISTS(eventfd sys/eventfd.h HAVE_EVENTFD)
+ENDIF()
+IF(HAVE_SYS_SIGNALFD_H)
+ CHECK_SYMBOL_EXISTS(signalfd sys/signalfd.h HAVE_SIGNALFD)
+ENDIF()
+IF(HAVE_SYS_TIMERFD_H)
+ CHECK_SYMBOL_EXISTS(timerfd_create sys/timerfd.h HAVE_TIMERFD)
+ENDIF()
+IF(HAVE_LINUX_FS_H)
+ CHECK_SYMBOL_EXISTS(RWF_SUPPORTED linux/fs.h HAVE_KERNEL_RWF_T)
+ENDIF()
+CHECK_SYMBOL_EXISTS(time.h nanosleep HAVE_NANOSLEEP)
+
+# check first without rt
+CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
+
+CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_LIBRT)
+# then check with rt
+CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_CLOCK_GETTIME)
+CHECK_LIBRARY_EXISTS(m ceil "" HAVE_LIBM)
+
+CONFIGURE_FILE(config.h.in libev-config.h)
+
+IF(ENABLE_STATIC MATCHES "ON")
+ ADD_LIBRARY(rspamd-ev STATIC ${LIBEVSRC})
+ELSE()
+ ADD_LIBRARY(rspamd-ev SHARED ${LIBEVSRC})
+ENDIF()
+target_link_libraries(rspamd-ev "${RSPAMD_REQUIRED_LIBRARIES}")
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+ADD_DEFINITIONS("-DEV_CONFIG_H=\"libev-config.h\""
+ -DEV_MULTIPLICITY=1
+ -DEV_USE_FLOOR=1
+ -DEV_NO_THREADS=1 # We do not have threads in Rspamd!
+ -DEV_FEATURES=127 # Enable all features
+ )
+IF(HAVE_EVENTFD)
+ ADD_DEFINITIONS(-DEV_USE_EVENTFD=1)
+ENDIF()
+
+IF(ENABLE_FULL_DEBUG MATCHES "ON")
+ ADD_DEFINITIONS(-DEV_VERIFY=3)
+ENDIF()
+
+INSTALL(TARGETS rspamd-ev LIBRARY DESTINATION ${RSPAMD_LIBDIR})
diff --git a/contrib/libev/Changes b/contrib/libev/Changes
new file mode 100644
index 0000000..04e58ba
--- /dev/null
+++ b/contrib/libev/Changes
@@ -0,0 +1,529 @@
+Revision history for libev, a high-performance and full-featured event loop.
+
+4.25 Fri Dec 21 07:49:20 CET 2018
+ - INCOMPATIBLE CHANGE: EV_THROW was renamed to EV_NOEXCEPT
+ (EV_THROW sitll provided) and now uses noexcept on C++11 or newer.
+ - move the darwin select workaround highe rin ev.c, as newer versions of
+ darwin managed to break their broken select even more.
+ - ANDROID => __ANDROID__ (reported by enh@google.com).
+ - disable epoll_create1 on android because it has broken header files
+ and google is unwilling to fix them (reported by enh@google.com).
+ - avoid a minor compilation warning on win32.
+ - c++: remove deprecated dynamic throw() specifications.
+ - c++: improve the (unsupported) bad_loop exception class.
+ - backport perl ev_periodic example to C, untested.
+ - update libecb, biggets change is to include a memory fence
+ in ECB_MEMORY_FENCE_RELEASE on x86/amd64.
+ - minor autoconf/automake modernisation.
+
+4.24 Wed Dec 28 05:19:55 CET 2016
+ - bump version to 4.24, as the release tarball inexplicably
+ didn't have the right version in ev.h, even though the cvs-tagged
+ version did have the right one (reported by Ales Teska).
+
+4.23 Wed Nov 16 18:23:41 CET 2016
+ - move some declarations at the beginning to help certain retarded
+ microsoft compilers, even though their documentation claims
+ otherwise (reported by Ruslan Osmanov).
+
+4.22 Sun Dec 20 22:11:50 CET 2015
+ - when epoll detects unremovable fds in the fd set, rebuild
+ only the epoll descriptor, not the signal pipe, to avoid
+ SIGPIPE in ev_async_send. This doesn't solve it on fork,
+ so document what needs to be done in ev_loop_fork
+ (analyzed by Benjamin Mahler).
+ - remove superfluous sys/timeb.h include on win32
+ (analyzed by Jason Madden).
+ - updated libecb.
+
+4.20 Sat Jun 20 13:01:43 CEST 2015
+ - prefer noexcept over throw () with C++ 11.
+ - update ecb.h due to incompatibilities with c11.
+ - fix a potential aliasing issue when reading and writing
+ watcher callbacks.
+
+4.19 Thu Sep 25 08:18:25 CEST 2014
+ - ev.h wasn't valid C++ anymore, which tripped compilers other than
+ clang, msvc or gcc (analyzed by Raphael 'kena' Poss). Unfortunately,
+ C++ doesn't support typedefs for function pointers fully, so the affected
+ declarations have to spell out the types each time.
+ - when not using autoconf, tighten the check for clock_gettime and related
+ functionality.
+
+4.18 Fri Sep 5 17:55:26 CEST 2014
+ - events on files were not always generated properly with the
+ epoll backend (testcase by Assaf Inbal).
+ - mark event pipe fd as cloexec after a fork (analyzed by Sami Farin).
+ - (ecb) support m68k, m88k and sh (patch by Miod Vallat).
+ - use a reasonable fallback for EV_NSIG instead of erroring out
+ when we can't detect the signal set size.
+ - in the absence of autoconf, do not use the clock syscall
+ on glibc >= 2.17 (avoids the syscall AND -lrt on systems
+ doing clock_gettime in userspace).
+ - ensure extern "C" function pointers are used for externally-visible
+ loop callbacks (not watcher callbacks yet).
+ - (ecb) work around memory barriers and volatile apparently both being
+ broken in visual studio 2008 and later (analysed and patch by Nicolas Noble).
+
+4.15 Fri Mar 1 12:04:50 CET 2013
+ - destroying a non-default loop would stop the global waitpid
+ watcher (Denis Bilenko).
+ - queueing pending watchers of higher priority from a watcher now invokes
+ them in a timely fashion (reported by Denis Bilenko).
+ - add throw() to all libev functions that cannot throw exceptions, for
+ further code size decrease when compiling for C++.
+ - add throw () to callbacks that must not throw exceptions (allocator,
+ syserr, loop acquire/release, periodic reschedule cbs).
+ - fix event_base_loop return code, add event_get_callback, event_base_new,
+ event_base_get_method calls to improve libevent 1.x emulation and add
+ some libevent 2.x functionality (based on a patch by Jeff Davey).
+ - add more memory fences to fix a bug reported by Jeff Davey. Better
+ be overfenced than underprotected.
+ - ev_run now returns a boolean status (true meaning watchers are
+ still active).
+ - ev_once: undef EV_ERROR in ev_kqueue.c, to avoid clashing with
+ libev's EV_ERROR (reported by 191919).
+ - (ecb) add memory fence support for xlC (Darin McBride).
+ - (ecb) add memory fence support for gcc-mips (Anton Kirilov).
+ - (ecb) add memory fence support for gcc-alpha (Christian Weisgerber).
+ - work around some kernels losing file descriptors by leaking
+ the kqueue descriptor in the child.
+ - work around linux inotify not reporting IN_ATTRIB changes for directories
+ in many cases.
+ - include sys/syscall.h instead of plain syscall.h.
+ - check for io watcher loops in ev_verify, check for the most
+ common reported usage bug in ev_io_start.
+ - choose socket vs. WSASocket at compiletime using EV_USE_WSASOCKET.
+ - always use WSASend/WSARecv directly on windows, hoping that this
+ works in all cases (unlike read/write/send/recv...).
+ - try to detect signals around a fork faster (test program by
+ Denis Bilenko).
+ - work around recent glibc versions that leak memory in realloc.
+ - rename ev::embed::set to ev::embed::set_embed to avoid clashing
+ the watcher base set (loop) method.
+ - rewrite the async/signal pipe logic to always keep a valid fd, which
+ simplifies (and hopefully correctifies :) the race checking
+ on fork, at the cost of one extra fd.
+ - add fat, msdos, jffs2, ramfs, ntfs and btrfs to the list of
+ inotify-supporting filesystems.
+ - move orig_CFLAGS assignment to after AC_INIT, as newer autoconf
+ versions ignore it before
+ (https://bugzilla.redhat.com/show_bug.cgi?id=908096).
+ - add some untested android support.
+ - enum expressions must be of type int (reported by Juan Pablo L).
+
+4.11 Sat Feb 4 19:52:39 CET 2012
+ - INCOMPATIBLE CHANGE: ev_timer_again now clears the pending status, as
+ was documented already, but not implemented in the repeating case.
+ - new compiletime symbols: EV_NO_SMP and EV_NO_THREADS.
+ - fix a race where the workaround against the epoll fork bugs
+ caused signals to not be handled anymore.
+ - correct backend_fudge for most backends, and implement a windows
+ specific workaround to avoid looping because we call both
+ select and Sleep, both with different time resolutions.
+ - document range and guarantees of ev_sleep.
+ - document reasonable ranges for periodics interval and offset.
+ - rename backend_fudge to backend_mintime to avoid future confusion :)
+ - change the default periodic reschedule function to hopefully be more
+ exact and correct even in corner cases or in the far future.
+ - do not rely on -lm anymore: use it when available but use our
+ own floor () if it is missing. This should make it easier to embed,
+ as no external libraries are required.
+ - strategically import macros from libecb and mark rarely-used functions
+ as cache-cold (saving almost 2k code size on typical amd64 setups).
+ - add Symbols.ev and Symbols.event files, that were missing.
+ - fix backend_mintime value for epoll (was 1/1024, is 1/1000 now).
+ - fix #3 "be smart about timeouts" to not "deadlock" when
+ timeout == now, also improve the section overall.
+ - avoid "AVOIDING FINISHING BEFORE RETURNING" idiom.
+ - support new EV_API_STATIC mode to make all libev symbols
+ static.
+ - supply default CFLAGS of -g -O3 with gcc when original CFLAGS
+ were empty.
+
+4.04 Wed Feb 16 09:01:51 CET 2011
+ - fix two problems in the native win32 backend, where reuse of fd's
+ with different underlying handles caused handles not to be removed
+ or added to the select set (analyzed and tested by Bert Belder).
+ - do no rely on ceil() in ev_e?poll.c.
+ - backport libev to HP-UX versions before 11 v3.
+ - configure did not detect nanosleep and clock_gettime properly when
+ they are available in the libc (as opposed to -lrt).
+
+4.03 Tue Jan 11 14:37:25 CET 2011
+ - officially support polling files with all backends.
+ - support files, /dev/zero etc. the same way as select in the epoll
+ backend, by generating events on our own.
+ - ports backend: work around solaris bug 6874410 and many related ones
+ (EINTR, maybe more), with no performance loss (note that the solaris
+ bug report is actually wrong, reality is far more bizarre and broken
+ than that).
+ - define EV_READ/EV_WRITE as macros in event.h, as some programs use
+ #ifdef to test for them.
+ - new (experimental) function: ev_feed_signal.
+ - new (to become default) EVFLAG_NOSIGMASK flag.
+ - new EVBACKEND_MASK symbol.
+ - updated COMMON IDIOMS SECTION.
+
+4.01 Fri Nov 5 21:51:29 CET 2010
+ - automake fucked it up, apparently, --add-missing -f is not quite enough
+ to make it update its files, so 4.00 didn't install ev++.h and
+ event.h on make install. grrr.
+ - ev_loop(count|depth) didn't return anything (Robin Haberkorn).
+ - change EV_UNDEF to 0xffffffff to silence some overzealous compilers.
+ - use "(libev) " prefix for all libev error messages now.
+
+4.00 Mon Oct 25 12:32:12 CEST 2010
+ - "PORTING FROM LIBEV 3.X TO 4.X" (in ev.pod) is recommended reading.
+ - ev_embed_stop did not correctly stop the watcher (very good
+ testcase by Vladimir Timofeev).
+ - ev_run will now always update the current loop time - it erroneously
+ didn't when idle watchers were active, causing timers not to fire.
+ - fix a bug where a timeout of zero caused the timer not to fire
+ in the libevent emulation (testcase by Péter Szabó).
+ - applied win32 fixes by Michael Lenaghan (also James Mansion).
+ - replace EV_MINIMAL by EV_FEATURES.
+ - prefer EPOLL_CTL_ADD over EPOLL_CTL_MOD in some more cases, as it
+ seems the former is *much* faster than the latter.
+ - linux kernel version detection (for inotify bug workarounds)
+ did not work properly.
+ - reduce the number of spurious wake-ups with the ports backend.
+ - remove dependency on sys/queue.h on freebsd (patch by Vanilla Hsu).
+ - do async init within ev_async_start, not ev_async_set, which avoids
+ an API quirk where the set function must be called in the C++ API
+ even when there is nothing to set.
+ - add (undocumented) EV_ENABLE when adding events with kqueue,
+ this might help with OS X, which seems to need it despite documenting
+ not to need it (helpfully pointed out by Tilghman Lesher).
+ - do not use poll by default on freebsd, it's broken (what isn't
+ on freebsd...).
+ - allow to embed epoll on kernels >= 2.6.32.
+ - configure now prepends -O3, not appends it, so one can still
+ override it.
+ - ev.pod: greatly expanded the portability section, added a porting
+ section, a description of watcher states and made lots of minor fixes.
+ - disable poll backend on AIX, the poll header spams the namespace
+ and it's not worth working around dead platforms (reported
+ and analyzed by Aivars Kalvans).
+ - improve header file compatibility of the standalone eventfd code
+ in an obscure case.
+ - implement EV_AVOID_STDIO option.
+ - do not use sscanf to parse linux version number (smaller, faster,
+ no sscanf dependency).
+ - new EV_CHILD_ENABLE and EV_SIGNAL_ENABLE configurable settings.
+ - update libev.m4 HAVE_CLOCK_SYSCALL test for newer glibcs.
+ - add section on accept() problems to the manpage.
+ - rename EV_TIMEOUT to EV_TIMER.
+ - rename ev_loop_count/depth/verify/loop/unloop.
+ - remove ev_default_destroy and ev_default_fork.
+ - switch to two-digit minor version.
+ - work around an apparent gentoo compiler bug.
+ - define _DARWIN_UNLIMITED_SELECT. just so.
+ - use enum instead of #define for most constants.
+ - improve compatibility to older C++ compilers.
+ - (experimental) ev_run/ev_default_loop/ev_break/ev_loop_new have now
+ default arguments when compiled as C++.
+ - enable automake dependency tracking.
+ - ev_loop_new no longer leaks memory when loop creation failed.
+ - new ev_cleanup watcher type.
+
+3.9 Thu Dec 31 07:59:59 CET 2009
+ - signalfd is no longer used by default and has to be requested
+ explicitly - this means that easy to catch bugs become hard to
+ catch race conditions, but the users have spoken.
+ - point out the unspecified signal mask in the documentation, and
+ that this is a race condition regardless of EV_SIGNALFD.
+ - backport inotify code to C89.
+ - inotify file descriptors could leak into child processes.
+ - ev_stat watchers could keep an erroneous extra ref on the loop,
+ preventing exit when unregistering all watchers (testcases
+ provided by ry@tinyclouds.org).
+ - implement EV_WIN32_HANDLE_TO_FD and EV_WIN32_CLOSE_FD configuration
+ symbols to make it easier for apps to do their own fd management.
+ - support EV_IDLE_ENABLE being disabled in ev++.h
+ (patch by Didier Spezia).
+ - take advantage of inotify_init1, if available, to set cloexec/nonblock
+ on fd creation, to avoid races.
+ - the signal handling pipe wasn't always initialised under windows
+ (analysed by lekma).
+ - changed minimum glibc requirement from glibc 2.9 to 2.7, for
+ signalfd.
+ - add missing string.h include (Denis F. Latypoff).
+ - only replace ev_stat.prev when we detect an actual difference,
+ so prev is (almost) always different to attr. this might
+ have caused the problems with 04_stat.t.
+ - add ev::timer->remaining () method to C++ API.
+
+3.8 Sun Aug 9 14:30:45 CEST 2009
+ - incompatible change: do not necessarily reset signal handler
+ to SIG_DFL when a sighandler is stopped.
+ - ev_default_destroy did not properly free or zero some members,
+ potentially causing crashes and memory corruption on repeated
+ ev_default_destroy/ev_default_loop calls.
+ - take advantage of signalfd on GNU/Linux systems.
+ - document that the signal mask might be in an unspecified
+ state when using libev's signal handling.
+ - take advantage of some GNU/Linux calls to set cloexec/nonblock
+ on fd creation, to avoid race conditions.
+
+3.7 Fri Jul 17 16:36:32 CEST 2009
+ - ev_unloop and ev_loop wrongly used a global variable to exit loops,
+ instead of using a per-loop variable (bug caught by accident...).
+ - the ev_set_io_collect_interval interpretation has changed.
+ - add new functionality: ev_set_userdata, ev_userdata,
+ ev_set_invoke_pending_cb, ev_set_loop_release_cb,
+ ev_invoke_pending, ev_pending_count, together with a long example
+ about thread locking.
+ - add ev_timer_remaining (as requested by Denis F. Latypoff).
+ - add ev_loop_depth.
+ - calling ev_unloop in fork/prepare watchers will no longer poll
+ for new events.
+ - Denis F. Latypoff corrected many typos in example code snippets.
+ - honor autoconf detection of EV_USE_CLOCK_SYSCALL, also double-
+ check that the syscall number is available before trying to
+ use it (reported by ry@tinyclouds).
+ - use GetSystemTimeAsFileTime instead of _timeb on windows, for
+ slightly higher accuracy.
+ - properly declare ev_loop_verify and ev_now_update even when
+ !EV_MULTIPLICITY.
+ - do not compile in any priority code when EV_MAXPRI == EV_MINPRI.
+ - support EV_MINIMAL==2 for a reduced API.
+ - actually 0-initialise struct sigaction when installing signals.
+ - add section on hibernate and stopped processes to ev_timer docs.
+
+3.6 Tue Apr 28 02:49:30 CEST 2009
+ - multiple timers becoming ready within an event loop iteration
+ will be invoked in the "correct" order now.
+ - do not leave the event loop early just because we have no active
+ watchers, fixing a problem when embedding a kqueue loop
+ that has active kernel events but no registered watchers
+ (reported by blacksand blacksand).
+ - correctly zero the idx values for arrays, so destroying and
+ reinitialising the default loop actually works (patch by
+ Malek Hadj-Ali).
+ - implement ev_suspend and ev_resume.
+ - new EV_CUSTOM revents flag for use by applications.
+ - add documentation section about priorities.
+ - add a glossary to the documentation.
+ - extend the ev_fork description slightly.
+ - optimize a jump out of call_pending.
+
+3.53 Sun Feb 15 02:38:20 CET 2009
+ - fix a bug in event pipe creation on win32 that would cause a
+ failed assertion on event loop creation (patch by Malek Hadj-Ali).
+ - probe for CLOCK_REALTIME support at runtime as well and fall
+ back to gettimeofday if there is an error, to support older
+ operating systems with newer header files/libraries.
+ - prefer gettimeofday over clock_gettime with USE_CLOCK_SYSCALL
+ (default most everywhere), otherwise not.
+
+3.52 Wed Jan 7 21:43:02 CET 2009
+ - fix compilation of select backend in fd_set mode when NFDBITS is
+ missing (to get it to compile on QNX, reported by Rodrigo Campos).
+ - better select-nfds handling when select backend is in fd_set mode.
+ - diagnose fd_set overruns when select backend is in fd_set mode.
+ - due to a thinko, instead of disabling everything but
+ select on the borked OS X platform, everything but select was
+ allowed (reported by Emanuele Giaquinta).
+ - actually verify that local and remote port are matching in
+ libev's socketpair emulation, which makes denial-of-service
+ attacks harder (but not impossible - it's windows). Make sure
+ it even works under vista, which thinks that getpeer/sockname
+ should return fantasy port numbers.
+ - include "libev" in all assertion messages for potentially
+ clearer diagnostics.
+ - event_get_version (libevent compatibility) returned
+ a useless string instead of the expected version string
+ (patch by W.C.A. Wijngaards).
+
+3.51 Wed Dec 24 23:00:11 CET 2008
+ - fix a bug where an inotify watcher was added twice, causing
+ freezes on hash collisions (reported and analysed by Graham Leggett).
+ - new config symbol, EV_USE_CLOCK_SYSCALL, to make libev use
+ a direct syscall - slower, but no dependency on librt et al.
+ - assume negative return values != -1 signals success of port_getn
+ (http://cvs.epicsol.org/cgi/viewcvs.cgi/epic5/source/newio.c?rev=1.52)
+ (no known failure reports, but it doesn't hurt).
+ - fork detection in ev_embed now stops and restarts the watcher
+ automatically.
+ - EXPERIMENTAL: default the method to operator () in ev++.h,
+ to make it nicer to use functors (requested by Benedek László).
+ - fixed const object callbacks in ev++.h.
+ - replaced loop_ref argument of watcher.set (loop) by a direct
+ ev_loop * in ev++.h, to avoid clashes with functor patch.
+ - do not try to watch the empty string via inotify.
+ - inotify watchers could be leaked under certain circumstances.
+ - OS X 10.5 is actually even more broken than earlier versions,
+ so fall back to select on that piece of garbage.
+ - fixed some weirdness in the ev_embed documentation.
+
+3.49 Wed Nov 19 11:26:53 CET 2008
+ - ev_stat watchers will now use inotify as a mere hint on
+ kernels <2.6.25, or if the filesystem is not in the
+ "known to be good" list.
+ - better mingw32 compatibility (it's not as borked as native win32)
+ (analysed by Roger Pack).
+ - include stdio.h in the example program, as too many people are
+ confused by the weird C language otherwise. I guess the next thing
+ I get told is that the "..." ellipses in the examples don't compile
+ with their C compiler.
+
+3.48 Thu Oct 30 09:02:37 CET 2008
+ - further optimise away the EPOLL_CTL_ADD/MOD combo in the epoll
+ backend by assuming the kernel event mask hasn't changed if
+ ADD fails with EEXIST.
+ - work around spurious event notification bugs in epoll by using
+ a 32-bit generation counter. recreate kernel state if we receive
+ spurious notifications or unwanted events. this is very costly,
+ but I didn't come up with this horrible design.
+ - use memset to initialise most arrays now and do away with the
+ init functions.
+ - expand time-out strategies into a "Be smart about timeouts" section.
+ - drop the "struct" from all ev_watcher declarations in the
+ documentation and did other clarifications (yeah, it was a mistake
+ to have a struct AND a function called ev_loop).
+ - fix a bug where ev_default would not initialise the default
+ loop again after it was destroyed with ev_default_destroy.
+ - rename syserr to ev_syserr to avoid name clashes when embedding,
+ do similar changes for event.c.
+
+3.45 Tue Oct 21 21:59:26 CEST 2008
+ - disable inotify usage on linux <2.6.25, as it is broken
+ (reported by Yoann Vandoorselaere).
+ - ev_stat erroneously would try to add inotify watchers
+ even when inotify wasn't available (this should only
+ have a performance impact).
+ - ev_once now passes both timeout and io to the callback if both
+ occur concurrently, instead of giving timeouts precedence.
+ - disable EV_USE_INOTIFY when sys/inotify.h is too old.
+
+3.44 Mon Sep 29 05:18:39 CEST 2008
+ - embed watchers now automatically invoke ev_loop_fork on the
+ embedded loop when the parent loop forks.
+ - new function: ev_now_update (loop).
+ - verify_watcher was not marked static.
+ - improve the "associating..." manpage section.
+ - documentation tweaks here and there.
+
+3.43 Sun Jul 6 05:34:41 CEST 2008
+ - include more include files on windows to get struct _stati64
+ (reported by Chris Hulbert, but doesn't quite fix his issue).
+ - add missing #include <io.h> in ev.c on windows (reported by
+ Matt Tolton).
+
+3.42 Tue Jun 17 12:12:07 CEST 2008
+ - work around yet another windows bug: FD_SET actually adds fd's
+ multiple times to the fd_*SET*, despite official MSN docs claiming
+ otherwise. Reported and well-analysed by Matt Tolton.
+ - define NFDBITS to 0 when EV_SELECT_IS_WINSOCKET to make it compile
+ (reported any analysed by Chris Hulbert).
+ - fix a bug in ev_ebadf (this function is only used to catch
+ programming errors in the libev user). reported by Matt Tolton.
+ - fix a bug in fd_intern on win32 (could lead to compile errors
+ under some circumstances, but would work correctly if it compiles).
+ reported by Matt Tolton.
+ - (try to) work around missing lstat on windows.
+ - pass in the write fd set as except fd set under windows. windows
+ is so uncontrollably lame that it requires this. this means that
+ switching off oobinline is not supported (but tcp/ip doesn't
+ have oob, so that would be stupid anyways.
+ - use posix module symbol to auto-detect monotonic clock presence
+ and some other default values.
+
+3.41 Fri May 23 18:42:54 CEST 2008
+ - work around an obscure bug in winsocket select: if you
+ provide only empty fd sets then select returns WSAEINVAL. how sucky.
+ - improve timer scheduling stability and reduce use of time_epsilon.
+ - use 1-based 2-heap for EV_MINIMAL, simplifies code, reduces
+ codesize and makes for better cache-efficiency.
+ - use 3-based 4-heap for !EV_MINIMAL. this makes better use
+ of cpu cache lines and gives better growth behaviour than
+ 2-based heaps.
+ - cache timestamp within heap for !EV_MINIMAL, to avoid random
+ memory accesses.
+ - document/add EV_USE_4HEAP and EV_HEAP_CACHE_AT.
+ - fix a potential aliasing issue in ev_timer_again.
+ - add/document ev_periodic_at, retract direct access to ->at.
+ - improve ev_stat docs.
+ - add portability requirements section.
+ - fix manpage headers etc.
+ - normalise WSA error codes to lower range on windows.
+ - add consistency check code that can be called automatically
+ or on demand to check for internal structures (ev_loop_verify).
+
+3.31 Wed Apr 16 20:45:04 CEST 2008
+ - added last minute fix for ev_poll.c by Brandon Black.
+
+3.3 Wed Apr 16 19:04:10 CEST 2008
+ - event_base_loopexit should return 0 on success
+ (W.C.A. Wijngaards).
+ - added linux eventfd support.
+ - try to autodetect epoll and inotify support
+ by libc header version if not using autoconf.
+ - new symbols: EV_DEFAULT_UC and EV_DEFAULT_UC_.
+ - declare functions defined in ev.h as inline if
+ C99 or gcc are available.
+ - enable inlining with gcc versions 2 and 3.
+ - work around broken poll implementations potentially
+ not clearing revents field in ev_poll (Brandon Black)
+ (no such systems are known at this time).
+ - work around a bug in realloc on openbsd and darwin,
+ also makes the erroneous valgrind complaints
+ go away (noted by various people).
+ - fix ev_async_pending, add c++ wrapper for ev_async
+ (based on patch sent by Johannes Deisenhofer).
+ - add sensible set method to ev::embed.
+ - made integer constants type int in ev.h.
+
+3.2 Wed Apr 2 17:11:19 CEST 2008
+ - fix a 64 bit overflow issue in the select backend,
+ by using fd_mask instead of int for the mask.
+ - rename internal sighandler to avoid clash with very old perls.
+ - entering ev_loop will not clear the ONESHOT or NONBLOCKING
+ flags of any outer loops anymore.
+ - add ev_async_pending.
+
+3.1 Thu Mar 13 13:45:22 CET 2008
+ - implement ev_async watchers.
+ - only initialise signal pipe on demand.
+ - make use of sig_atomic_t configurable.
+ - improved documentation.
+
+3.0 Mon Jan 28 13:14:47 CET 2008
+ - API/ABI bump to version 3.0.
+ - ev++.h includes "ev.h" by default now, not <ev.h>.
+ - slightly improved documentation.
+ - speed up signal detection after a fork.
+ - only optionally return trace status changed in ev_child
+ watchers.
+ - experimental (and undocumented) loop wrappers for ev++.h.
+
+2.01 Tue Dec 25 08:04:41 CET 2007
+ - separate Changes file.
+ - fix ev_path_set => ev_stat_set typo.
+ - remove event_compat.h from the libev tarball.
+ - change how include files are found.
+ - doc updates.
+ - update licenses, explicitly allow for GPL relicensing.
+
+2.0 Sat Dec 22 17:47:03 CET 2007
+ - new ev_sleep, ev_set_(io|timeout)_collect_interval.
+ - removed epoll from embeddable fd set.
+ - fix embed watchers.
+ - renamed ev_embed.loop to other.
+ - added exported Symbol tables.
+ - undefine member wrapper macros at the end of ev.c.
+ - respect EV_H in ev++.h.
+
+1.86 Tue Dec 18 02:36:57 CET 2007
+ - fix memleak on loop destroy (not relevant for perl).
+
+1.85 Fri Dec 14 20:32:40 CET 2007
+ - fix some aliasing issues w.r.t. timers and periodics
+ (not relevant for perl).
+
+(for historic versions refer to EV/Changes, found in the Perl interface)
+
+0.1 Wed Oct 31 21:31:48 CET 2007
+ - original version; hacked together in <24h.
+
diff --git a/contrib/libev/LICENSE b/contrib/libev/LICENSE
new file mode 100644
index 0000000..2fdabd4
--- /dev/null
+++ b/contrib/libev/LICENSE
@@ -0,0 +1,37 @@
+All files in libev are
+Copyright (c)2007,2008,2009,2010,2011,2012,2013 Marc Alexander Lehmann.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Alternatively, the contents of this package may be used under the terms
+of the GNU General Public License ("GPL") version 2 or any later version,
+in which case the provisions of the GPL are applicable instead of the
+above. If you wish to allow the use of your version of this package only
+under the terms of the GPL and not to allow others to use your version of
+this file under the BSD license, indicate your decision by deleting the
+provisions above and replace them with the notice and other provisions
+required by the GPL in this and the other files of this package. If you do
+not delete the provisions above, a recipient may use your version of this
+file under either the BSD or the GPL.
diff --git a/contrib/libev/config.h.in b/contrib/libev/config.h.in
new file mode 100644
index 0000000..686a379
--- /dev/null
+++ b/contrib/libev/config.h.in
@@ -0,0 +1,112 @@
+/* config.h.in. Generated from configure.ac by autoheader. */
+
+/* Define to 1 if you have the `clock_gettime' function. */
+#cmakedefine HAVE_CLOCK_GETTIME 1
+
+/* Define to 1 to use the syscall interface for clock_gettime */
+#cmakedefine HAVE_CLOCK_SYSCALL 1
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#cmakedefine HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the `epoll_ctl' function. */
+#cmakedefine HAVE_EPOLL_CTL 1
+
+/* Define to 1 if you have the `eventfd' function. */
+#cmakedefine HAVE_EVENTFD 1
+
+/* Define to 1 if the floor function is available */
+#cmakedefine HAVE_FLOOR 1
+
+/* Define to 1 if you have the `inotify_init' function. */
+#cmakedefine HAVE_INOTIFY_INIT 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#cmakedefine HAVE_INTTYPES_H 1
+
+/* Define to 1 if linux/fs.h defined kernel_rwf_t */
+#cmakedefine HAVE_KERNEL_RWF_T 1
+
+/* Define to 1 if you have the `kqueue' function. */
+#cmakedefine HAVE_KQUEUE 1
+
+/* Define to 1 if you have the `rt' library (-lrt). */
+#cmakedefine HAVE_LIBRT 1
+
+/* Define to 1 if you have the <linux/aio_abi.h> header file. */
+#cmakedefine HAVE_LINUX_AIO_ABI_H 1
+
+/* Define to 1 if you have the <linux/fs.h> header file. */
+#cmakedefine HAVE_LINUX_FS_H 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#cmakedefine HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `nanosleep' function. */
+#cmakedefine HAVE_NANOSLEEP 1
+
+/* Define to 1 if you have the `poll' function. */
+#cmakedefine HAVE_POLL 1
+
+/* Define to 1 if you have the <poll.h> header file. */
+#cmakedefine HAVE_POLL_H 1
+
+/* Define to 1 if you have the `port_create' function. */
+#cmakedefine HAVE_PORT_CREATE 1
+
+/* Define to 1 if you have the <port.h> header file. */
+#cmakedefine HAVE_PORT_H 1
+
+/* Define to 1 if you have the `select' function. */
+#cmakedefine HAVE_SELECT 1
+
+/* Define to 1 if you have the `signalfd' function. */
+#cmakedefine HAVE_SIGNALFD 1
+
+/* Define to 1 if you have the `timerfd_create' function. */
+#cmakedefine HAVE_TIMERFD 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#cmakedefine HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#cmakedefine HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#cmakedefine HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#cmakedefine HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/epoll.h> header file. */
+#cmakedefine HAVE_SYS_EPOLL_H 1
+
+/* Define to 1 if you have the <sys/eventfd.h> header file. */
+#cmakedefine HAVE_SYS_EVENTFD_H 1
+
+/* Define to 1 if you have the <sys/event.h> header file. */
+#cmakedefine HAVE_SYS_EVENT_H 1
+
+/* Define to 1 if you have the <sys/inotify.h> header file. */
+#cmakedefine HAVE_SYS_INOTIFY_H 1
+
+/* Define to 1 if you have the <sys/select.h> header file. */
+#cmakedefine HAVE_SYS_SELECT_H 1
+
+/* Define to 1 if you have the <sys/signalfd.h> header file. */
+#cmakedefine HAVE_SYS_SIGNALFD_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#cmakedefine HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/timerfd.h> header file. */
+#cmakedefine HAVE_SYS_TIMERFD_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#cmakedefine HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#cmakedefine HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
diff --git a/contrib/libev/ev++.h b/contrib/libev/ev++.h
new file mode 100644
index 0000000..0e1b60d
--- /dev/null
+++ b/contrib/libev/ev++.h
@@ -0,0 +1,816 @@
+/*
+ * libev simple C++ wrapper classes
+ *
+ * Copyright (c) 2007,2008,2010,2018 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef EVPP_H__
+#define EVPP_H__
+
+#ifdef EV_H
+# include EV_H
+#else
+# include "ev.h"
+#endif
+
+#ifndef EV_USE_STDEXCEPT
+# define EV_USE_STDEXCEPT 1
+#endif
+
+#if EV_USE_STDEXCEPT
+# include <stdexcept>
+#endif
+
+namespace ev {
+
+ typedef ev_tstamp tstamp;
+
+ enum {
+ UNDEF = EV_UNDEF,
+ NONE = EV_NONE,
+ READ = EV_READ,
+ WRITE = EV_WRITE,
+#if EV_COMPAT3
+ TIMEOUT = EV_TIMEOUT,
+#endif
+ TIMER = EV_TIMER,
+ PERIODIC = EV_PERIODIC,
+ SIGNAL = EV_SIGNAL,
+ CHILD = EV_CHILD,
+ STAT = EV_STAT,
+ IDLE = EV_IDLE,
+ CHECK = EV_CHECK,
+ PREPARE = EV_PREPARE,
+ FORK = EV_FORK,
+ ASYNC = EV_ASYNC,
+ EMBED = EV_EMBED,
+# undef ERROR // some systems stupidly #define ERROR
+ ERROR = EV_ERROR
+ };
+
+ enum
+ {
+ AUTO = EVFLAG_AUTO,
+ NOENV = EVFLAG_NOENV,
+ FORKCHECK = EVFLAG_FORKCHECK,
+
+ SELECT = EVBACKEND_SELECT,
+ POLL = EVBACKEND_POLL,
+ EPOLL = EVBACKEND_EPOLL,
+ KQUEUE = EVBACKEND_KQUEUE,
+ DEVPOLL = EVBACKEND_DEVPOLL,
+ PORT = EVBACKEND_PORT
+ };
+
+ enum
+ {
+#if EV_COMPAT3
+ NONBLOCK = EVLOOP_NONBLOCK,
+ ONESHOT = EVLOOP_ONESHOT,
+#endif
+ NOWAIT = EVRUN_NOWAIT,
+ ONCE = EVRUN_ONCE
+ };
+
+ enum how_t
+ {
+ ONE = EVBREAK_ONE,
+ ALL = EVBREAK_ALL
+ };
+
+ struct bad_loop
+#if EV_USE_STDEXCEPT
+ : std::exception
+#endif
+ {
+#if EV_USE_STDEXCEPT
+ const char *what () const EV_NOEXCEPT
+ {
+ return "libev event loop cannot be initialized, bad value of LIBEV_FLAGS?";
+ }
+#endif
+ };
+
+#ifdef EV_AX
+# undef EV_AX
+#endif
+
+#ifdef EV_AX_
+# undef EV_AX_
+#endif
+
+#if EV_MULTIPLICITY
+# define EV_AX raw_loop
+# define EV_AX_ raw_loop,
+#else
+# define EV_AX
+# define EV_AX_
+#endif
+
+ struct loop_ref
+ {
+ loop_ref (EV_P) EV_NOEXCEPT
+#if EV_MULTIPLICITY
+ : EV_AX (EV_A)
+#endif
+ {
+ }
+
+ bool operator == (const loop_ref &other) const EV_NOEXCEPT
+ {
+#if EV_MULTIPLICITY
+ return EV_AX == other.EV_AX;
+#else
+ return true;
+#endif
+ }
+
+ bool operator != (const loop_ref &other) const EV_NOEXCEPT
+ {
+#if EV_MULTIPLICITY
+ return ! (*this == other);
+#else
+ return false;
+#endif
+ }
+
+#if EV_MULTIPLICITY
+ bool operator == (const EV_P) const EV_NOEXCEPT
+ {
+ return this->EV_AX == EV_A;
+ }
+
+ bool operator != (const EV_P) const EV_NOEXCEPT
+ {
+ return ! (*this == EV_A);
+ }
+
+ operator struct ev_loop * () const EV_NOEXCEPT
+ {
+ return EV_AX;
+ }
+
+ operator const struct ev_loop * () const EV_NOEXCEPT
+ {
+ return EV_AX;
+ }
+
+ bool is_default () const EV_NOEXCEPT
+ {
+ return EV_AX == ev_default_loop (0);
+ }
+#endif
+
+#if EV_COMPAT3
+ void loop (int flags = 0)
+ {
+ ev_run (EV_AX_ flags);
+ }
+
+ void unloop (how_t how = ONE) EV_NOEXCEPT
+ {
+ ev_break (EV_AX_ how);
+ }
+#endif
+
+ void run (int flags = 0)
+ {
+ ev_run (EV_AX_ flags);
+ }
+
+ void break_loop (how_t how = ONE) EV_NOEXCEPT
+ {
+ ev_break (EV_AX_ how);
+ }
+
+ void post_fork () EV_NOEXCEPT
+ {
+ ev_loop_fork (EV_AX);
+ }
+
+ unsigned int backend () const EV_NOEXCEPT
+ {
+ return ev_backend (EV_AX);
+ }
+
+ tstamp now () const EV_NOEXCEPT
+ {
+ return ev_now (EV_AX);
+ }
+
+ void ref () EV_NOEXCEPT
+ {
+ ev_ref (EV_AX);
+ }
+
+ void unref () EV_NOEXCEPT
+ {
+ ev_unref (EV_AX);
+ }
+
+#if EV_FEATURE_API
+ unsigned int iteration () const EV_NOEXCEPT
+ {
+ return ev_iteration (EV_AX);
+ }
+
+ unsigned int depth () const EV_NOEXCEPT
+ {
+ return ev_depth (EV_AX);
+ }
+
+ void set_io_collect_interval (tstamp interval) EV_NOEXCEPT
+ {
+ ev_set_io_collect_interval (EV_AX_ interval);
+ }
+
+ void set_timeout_collect_interval (tstamp interval) EV_NOEXCEPT
+ {
+ ev_set_timeout_collect_interval (EV_AX_ interval);
+ }
+#endif
+
+ // function callback
+ void once (int fd, int events, tstamp timeout, void (*cb)(int, void *), void *arg = 0) EV_NOEXCEPT
+ {
+ ev_once (EV_AX_ fd, events, timeout, cb, arg);
+ }
+
+ // method callback
+ template<class K, void (K::*method)(int)>
+ void once (int fd, int events, tstamp timeout, K *object) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, method_thunk<K, method>, object);
+ }
+
+ // default method == operator ()
+ template<class K>
+ void once (int fd, int events, tstamp timeout, K *object) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, method_thunk<K, &K::operator ()>, object);
+ }
+
+ template<class K, void (K::*method)(int)>
+ static void method_thunk (int revents, void *arg)
+ {
+ (static_cast<K *>(arg)->*method)
+ (revents);
+ }
+
+ // no-argument method callback
+ template<class K, void (K::*method)()>
+ void once (int fd, int events, tstamp timeout, K *object) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, method_noargs_thunk<K, method>, object);
+ }
+
+ template<class K, void (K::*method)()>
+ static void method_noargs_thunk (int revents, void *arg)
+ {
+ (static_cast<K *>(arg)->*method)
+ ();
+ }
+
+ // simpler function callback
+ template<void (*cb)(int)>
+ void once (int fd, int events, tstamp timeout) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, simpler_func_thunk<cb>);
+ }
+
+ template<void (*cb)(int)>
+ static void simpler_func_thunk (int revents, void *arg)
+ {
+ (*cb)
+ (revents);
+ }
+
+ // simplest function callback
+ template<void (*cb)()>
+ void once (int fd, int events, tstamp timeout) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, simplest_func_thunk<cb>);
+ }
+
+ template<void (*cb)()>
+ static void simplest_func_thunk (int revents, void *arg)
+ {
+ (*cb)
+ ();
+ }
+
+ void feed_fd_event (int fd, int revents) EV_NOEXCEPT
+ {
+ ev_feed_fd_event (EV_AX_ fd, revents);
+ }
+
+ void feed_signal_event (int signum) EV_NOEXCEPT
+ {
+ ev_feed_signal_event (EV_AX_ signum);
+ }
+
+#if EV_MULTIPLICITY
+ struct ev_loop* EV_AX;
+#endif
+
+ };
+
+#if EV_MULTIPLICITY
+ struct dynamic_loop : loop_ref
+ {
+
+ dynamic_loop (unsigned int flags = AUTO)
+ : loop_ref (ev_loop_new (flags))
+ {
+ if (!EV_AX)
+ throw bad_loop ();
+ }
+
+ ~dynamic_loop () EV_NOEXCEPT
+ {
+ ev_loop_destroy (EV_AX);
+ EV_AX = 0;
+ }
+
+ private:
+
+ dynamic_loop (const dynamic_loop &);
+
+ dynamic_loop & operator= (const dynamic_loop &);
+
+ };
+#endif
+
+ struct default_loop : loop_ref
+ {
+ default_loop (unsigned int flags = AUTO)
+#if EV_MULTIPLICITY
+ : loop_ref (ev_default_loop (flags))
+#endif
+ {
+ if (
+#if EV_MULTIPLICITY
+ !EV_AX
+#else
+ !ev_default_loop (flags)
+#endif
+ )
+ throw bad_loop ();
+ }
+
+ private:
+ default_loop (const default_loop &);
+ default_loop &operator = (const default_loop &);
+ };
+
+ inline loop_ref get_default_loop () EV_NOEXCEPT
+ {
+#if EV_MULTIPLICITY
+ return ev_default_loop (0);
+#else
+ return loop_ref ();
+#endif
+ }
+
+#undef EV_AX
+#undef EV_AX_
+
+#undef EV_PX
+#undef EV_PX_
+#if EV_MULTIPLICITY
+# define EV_PX loop_ref EV_A
+# define EV_PX_ loop_ref EV_A_
+#else
+# define EV_PX
+# define EV_PX_
+#endif
+
+ template<class ev_watcher, class watcher>
+ struct base : ev_watcher
+ {
+ #if EV_MULTIPLICITY
+ EV_PX;
+
+ // loop set
+ void set (EV_P) EV_NOEXCEPT
+ {
+ this->EV_A = EV_A;
+ }
+ #endif
+
+ base (EV_PX) EV_NOEXCEPT
+ #if EV_MULTIPLICITY
+ : EV_A (EV_A)
+ #endif
+ {
+ ev_init (this, 0);
+ }
+
+ void set_ (const void *data, void (*cb)(EV_P_ ev_watcher *w, int revents)) EV_NOEXCEPT
+ {
+ this->data = (void *)data;
+ ev_set_cb (static_cast<ev_watcher *>(this), cb);
+ }
+
+ // function callback
+ template<void (*function)(watcher &w, int)>
+ void set (void *data = 0) EV_NOEXCEPT
+ {
+ set_ (data, function_thunk<function>);
+ }
+
+ template<void (*function)(watcher &w, int)>
+ static void function_thunk (EV_P_ ev_watcher *w, int revents)
+ {
+ function
+ (*static_cast<watcher *>(w), revents);
+ }
+
+ // method callback
+ template<class K, void (K::*method)(watcher &w, int)>
+ void set (K *object) EV_NOEXCEPT
+ {
+ set_ (object, method_thunk<K, method>);
+ }
+
+ // default method == operator ()
+ template<class K>
+ void set (K *object) EV_NOEXCEPT
+ {
+ set_ (object, method_thunk<K, &K::operator ()>);
+ }
+
+ template<class K, void (K::*method)(watcher &w, int)>
+ static void method_thunk (EV_P_ ev_watcher *w, int revents)
+ {
+ (static_cast<K *>(w->data)->*method)
+ (*static_cast<watcher *>(w), revents);
+ }
+
+ // no-argument callback
+ template<class K, void (K::*method)()>
+ void set (K *object) EV_NOEXCEPT
+ {
+ set_ (object, method_noargs_thunk<K, method>);
+ }
+
+ template<class K, void (K::*method)()>
+ static void method_noargs_thunk (EV_P_ ev_watcher *w, int revents)
+ {
+ (static_cast<K *>(w->data)->*method)
+ ();
+ }
+
+ void operator ()(int events = EV_UNDEF)
+ {
+ return
+ ev_cb (static_cast<ev_watcher *>(this))
+ (static_cast<ev_watcher *>(this), events);
+ }
+
+ bool is_active () const EV_NOEXCEPT
+ {
+ return ev_is_active (static_cast<const ev_watcher *>(this));
+ }
+
+ bool is_pending () const EV_NOEXCEPT
+ {
+ return ev_is_pending (static_cast<const ev_watcher *>(this));
+ }
+
+ void feed_event (int revents) EV_NOEXCEPT
+ {
+ ev_feed_event (EV_A_ static_cast<ev_watcher *>(this), revents);
+ }
+ };
+
+ inline tstamp now (EV_P) EV_NOEXCEPT
+ {
+ return ev_now (EV_A);
+ }
+
+ inline void delay (tstamp interval) EV_NOEXCEPT
+ {
+ ev_sleep (interval);
+ }
+
+ inline int version_major () EV_NOEXCEPT
+ {
+ return ev_version_major ();
+ }
+
+ inline int version_minor () EV_NOEXCEPT
+ {
+ return ev_version_minor ();
+ }
+
+ inline unsigned int supported_backends () EV_NOEXCEPT
+ {
+ return ev_supported_backends ();
+ }
+
+ inline unsigned int recommended_backends () EV_NOEXCEPT
+ {
+ return ev_recommended_backends ();
+ }
+
+ inline unsigned int embeddable_backends () EV_NOEXCEPT
+ {
+ return ev_embeddable_backends ();
+ }
+
+ inline void set_allocator (void *(*cb)(void *ptr, long size) EV_NOEXCEPT) EV_NOEXCEPT
+ {
+ ev_set_allocator (cb);
+ }
+
+ inline void set_syserr_cb (void (*cb)(const char *msg) EV_NOEXCEPT) EV_NOEXCEPT
+ {
+ ev_set_syserr_cb (cb);
+ }
+
+ #if EV_MULTIPLICITY
+ #define EV_CONSTRUCT(cppstem,cstem) \
+ (EV_PX = get_default_loop ()) EV_NOEXCEPT \
+ : base<ev_ ## cstem, cppstem> (EV_A) \
+ { \
+ }
+ #else
+ #define EV_CONSTRUCT(cppstem,cstem) \
+ () EV_NOEXCEPT \
+ { \
+ }
+ #endif
+
+ /* using a template here would require quite a few more lines,
+ * so a macro solution was chosen */
+ #define EV_BEGIN_WATCHER(cppstem,cstem) \
+ \
+ struct cppstem : base<ev_ ## cstem, cppstem> \
+ { \
+ void start () EV_NOEXCEPT \
+ { \
+ ev_ ## cstem ## _start (EV_A_ static_cast<ev_ ## cstem *>(this)); \
+ } \
+ \
+ void stop () EV_NOEXCEPT \
+ { \
+ ev_ ## cstem ## _stop (EV_A_ static_cast<ev_ ## cstem *>(this)); \
+ } \
+ \
+ cppstem EV_CONSTRUCT(cppstem,cstem) \
+ \
+ ~cppstem () EV_NOEXCEPT \
+ { \
+ stop (); \
+ } \
+ \
+ using base<ev_ ## cstem, cppstem>::set; \
+ \
+ private: \
+ \
+ cppstem (const cppstem &o); \
+ \
+ cppstem &operator =(const cppstem &o); \
+ \
+ public:
+
+ #define EV_END_WATCHER(cppstem,cstem) \
+ };
+
+ EV_BEGIN_WATCHER (io, io)
+ void set (int fd, int events) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_io_set (static_cast<ev_io *>(this), fd, events);
+ if (active) start ();
+ }
+
+ void set (int events) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_io_set (static_cast<ev_io *>(this), fd, events);
+ if (active) start ();
+ }
+
+ void start (int fd, int events) EV_NOEXCEPT
+ {
+ set (fd, events);
+ start ();
+ }
+ EV_END_WATCHER (io, io)
+
+ EV_BEGIN_WATCHER (timer, timer)
+ void set (ev_tstamp after, ev_tstamp repeat = 0.) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_timer_set (static_cast<ev_timer *>(this), after, repeat);
+ if (active) start ();
+ }
+
+ void start (ev_tstamp after, ev_tstamp repeat = 0.) EV_NOEXCEPT
+ {
+ set (after, repeat);
+ start ();
+ }
+
+ void again () EV_NOEXCEPT
+ {
+ ev_timer_again (EV_A_ static_cast<ev_timer *>(this));
+ }
+
+ ev_tstamp remaining ()
+ {
+ return ev_timer_remaining (EV_A_ static_cast<ev_timer *>(this));
+ }
+ EV_END_WATCHER (timer, timer)
+
+ #if EV_PERIODIC_ENABLE
+ EV_BEGIN_WATCHER (periodic, periodic)
+ void set (ev_tstamp at, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_periodic_set (static_cast<ev_periodic *>(this), at, interval, 0);
+ if (active) start ();
+ }
+
+ void start (ev_tstamp at, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ set (at, interval);
+ start ();
+ }
+
+ void again () EV_NOEXCEPT
+ {
+ ev_periodic_again (EV_A_ static_cast<ev_periodic *>(this));
+ }
+ EV_END_WATCHER (periodic, periodic)
+ #endif
+
+ #if EV_SIGNAL_ENABLE
+ EV_BEGIN_WATCHER (sig, signal)
+ void set (int signum) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_signal_set (static_cast<ev_signal *>(this), signum);
+ if (active) start ();
+ }
+
+ void start (int signum) EV_NOEXCEPT
+ {
+ set (signum);
+ start ();
+ }
+ EV_END_WATCHER (sig, signal)
+ #endif
+
+ #if EV_CHILD_ENABLE
+ EV_BEGIN_WATCHER (child, child)
+ void set (int pid, int trace = 0) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_child_set (static_cast<ev_child *>(this), pid, trace);
+ if (active) start ();
+ }
+
+ void start (int pid, int trace = 0) EV_NOEXCEPT
+ {
+ set (pid, trace);
+ start ();
+ }
+ EV_END_WATCHER (child, child)
+ #endif
+
+ #if EV_STAT_ENABLE
+ EV_BEGIN_WATCHER (stat, stat)
+ void set (const char *path, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_stat_set (static_cast<ev_stat *>(this), path, interval);
+ if (active) start ();
+ }
+
+ void start (const char *path, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ stop ();
+ set (path, interval);
+ start ();
+ }
+
+ void update () EV_NOEXCEPT
+ {
+ ev_stat_stat (EV_A_ static_cast<ev_stat *>(this));
+ }
+ EV_END_WATCHER (stat, stat)
+ #endif
+
+ #if EV_IDLE_ENABLE
+ EV_BEGIN_WATCHER (idle, idle)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (idle, idle)
+ #endif
+
+ #if EV_PREPARE_ENABLE
+ EV_BEGIN_WATCHER (prepare, prepare)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (prepare, prepare)
+ #endif
+
+ #if EV_CHECK_ENABLE
+ EV_BEGIN_WATCHER (check, check)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (check, check)
+ #endif
+
+ #if EV_EMBED_ENABLE
+ EV_BEGIN_WATCHER (embed, embed)
+ void set_embed (struct ev_loop *embedded_loop) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_embed_set (static_cast<ev_embed *>(this), embedded_loop);
+ if (active) start ();
+ }
+
+ void start (struct ev_loop *embedded_loop) EV_NOEXCEPT
+ {
+ set (embedded_loop);
+ start ();
+ }
+
+ void sweep ()
+ {
+ ev_embed_sweep (EV_A_ static_cast<ev_embed *>(this));
+ }
+ EV_END_WATCHER (embed, embed)
+ #endif
+
+ #if EV_FORK_ENABLE
+ EV_BEGIN_WATCHER (fork, fork)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (fork, fork)
+ #endif
+
+ #if EV_ASYNC_ENABLE
+ EV_BEGIN_WATCHER (async, async)
+ void send () EV_NOEXCEPT
+ {
+ ev_async_send (EV_A_ static_cast<ev_async *>(this));
+ }
+
+ bool async_pending () EV_NOEXCEPT
+ {
+ return ev_async_pending (static_cast<ev_async *>(this));
+ }
+ EV_END_WATCHER (async, async)
+ #endif
+
+ #undef EV_PX
+ #undef EV_PX_
+ #undef EV_CONSTRUCT
+ #undef EV_BEGIN_WATCHER
+ #undef EV_END_WATCHER
+}
+
+#endif
+
diff --git a/contrib/libev/ev.c b/contrib/libev/ev.c
new file mode 100644
index 0000000..230445d
--- /dev/null
+++ b/contrib/libev/ev.c
@@ -0,0 +1,5678 @@
+/*
+ * libev event processing core, watcher management
+ *
+ * Copyright (c) 2007-2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/* this big block deduces configuration from config.h */
+#ifndef EV_STANDALONE
+# ifdef EV_CONFIG_H
+# include EV_CONFIG_H
+# else
+# include "config.h"
+# endif
+
+#ifdef __GNUC__
+#pragma GCC diagnostic ignored "-Wunused-value"
+#endif
+
+# if HAVE_FLOOR
+# ifndef EV_USE_FLOOR
+# define EV_USE_FLOOR 1
+# endif
+# endif
+
+# if HAVE_CLOCK_SYSCALL
+# ifndef EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 1
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 1
+# endif
+# endif
+# elif !defined EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 0
+# endif
+
+# if HAVE_CLOCK_GETTIME
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 1
+# endif
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+# else
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 0
+# endif
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+# endif
+
+# if HAVE_NANOSLEEP
+# ifndef EV_USE_NANOSLEEP
+# define EV_USE_NANOSLEEP EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_NANOSLEEP
+# define EV_USE_NANOSLEEP 0
+# endif
+
+# if HAVE_SELECT && HAVE_SYS_SELECT_H
+# ifndef EV_USE_SELECT
+# define EV_USE_SELECT EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_SELECT
+# define EV_USE_SELECT 0
+# endif
+
+# if HAVE_POLL && HAVE_POLL_H
+# ifndef EV_USE_POLL
+# define EV_USE_POLL EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_POLL
+# define EV_USE_POLL 0
+# endif
+
+# if HAVE_EPOLL_CTL && HAVE_SYS_EPOLL_H
+# ifndef EV_USE_EPOLL
+# define EV_USE_EPOLL EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_EPOLL
+# define EV_USE_EPOLL 0
+# endif
+
+# if HAVE_LINUX_AIO_ABI_H
+# ifndef EV_USE_LINUXAIO
+# define EV_USE_LINUXAIO 0 /* was: EV_FEATURE_BACKENDS, always off by default */
+# endif
+# else
+# undef EV_USE_LINUXAIO
+# define EV_USE_LINUXAIO 0
+# endif
+
+# if HAVE_LINUX_FS_H && HAVE_SYS_TIMERFD_H && HAVE_KERNEL_RWF_T
+# ifndef EV_USE_IOURING
+# define EV_USE_IOURING EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_IOURING
+# define EV_USE_IOURING 0
+# endif
+
+# if HAVE_KQUEUE && HAVE_SYS_EVENT_H
+# ifndef EV_USE_KQUEUE
+# define EV_USE_KQUEUE EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_KQUEUE
+# define EV_USE_KQUEUE 0
+# endif
+
+# if HAVE_PORT_H && HAVE_PORT_CREATE
+# ifndef EV_USE_PORT
+# define EV_USE_PORT EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_PORT
+# define EV_USE_PORT 0
+# endif
+
+# if HAVE_INOTIFY_INIT && HAVE_SYS_INOTIFY_H
+# ifndef EV_USE_INOTIFY
+# define EV_USE_INOTIFY EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_INOTIFY
+# define EV_USE_INOTIFY 0
+# endif
+
+# if HAVE_SIGNALFD && HAVE_SYS_SIGNALFD_H
+# ifndef EV_USE_SIGNALFD
+# define EV_USE_SIGNALFD EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_SIGNALFD
+# define EV_USE_SIGNALFD 0
+# endif
+
+# if HAVE_EVENTFD
+# ifndef EV_USE_EVENTFD
+# define EV_USE_EVENTFD EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_EVENTFD
+# define EV_USE_EVENTFD 0
+# endif
+
+# if HAVE_SYS_TIMERFD_H && HAVE_TIMERFD
+# ifndef EV_USE_TIMERFD
+# define EV_USE_TIMERFD EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_TIMERFD
+# define EV_USE_TIMERFD 0
+# endif
+
+#endif
+
+/* OS X, in its infinite idiocy, actually HARDCODES
+ * a limit of 1024 into their select. Where people have brains,
+ * OS X engineers apparently have a vacuum. Or maybe they were
+ * ordered to have a vacuum, or they do anything for money.
+ * This might help. Or not.
+ * Note that this must be defined early, as other include files
+ * will rely on this define as well.
+ */
+#define _DARWIN_UNLIMITED_SELECT 1
+
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <stddef.h>
+
+#include <stdio.h>
+
+#include <assert.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <time.h>
+#include <limits.h>
+
+#include <signal.h>
+
+#ifdef EV_H
+# include EV_H
+#else
+# include "ev.h"
+#endif
+
+#if EV_NO_THREADS
+# undef EV_NO_SMP
+# define EV_NO_SMP 1
+# undef ECB_NO_THREADS
+# define ECB_NO_THREADS 1
+#endif
+#if EV_NO_SMP
+# undef EV_NO_SMP
+# define ECB_NO_SMP 1
+#endif
+
+#ifndef _WIN32
+# include <sys/time.h>
+# include <sys/wait.h>
+# include <unistd.h>
+#else
+# include <io.h>
+# define WIN32_LEAN_AND_MEAN
+# include <winsock2.h>
+# include <windows.h>
+# ifndef EV_SELECT_IS_WINSOCKET
+# define EV_SELECT_IS_WINSOCKET 1
+# endif
+# undef EV_AVOID_STDIO
+#endif
+
+/* this block tries to deduce configuration from header-defined symbols and defaults */
+
+/* try to deduce the maximum number of signals on this platform */
+#if defined EV_NSIG
+/* use what's provided */
+#elif defined NSIG
+# define EV_NSIG (NSIG)
+#elif defined _NSIG
+# define EV_NSIG (_NSIG)
+#elif defined SIGMAX
+# define EV_NSIG (SIGMAX+1)
+#elif defined SIG_MAX
+# define EV_NSIG (SIG_MAX+1)
+#elif defined _SIG_MAX
+# define EV_NSIG (_SIG_MAX+1)
+#elif defined MAXSIG
+# define EV_NSIG (MAXSIG+1)
+#elif defined MAX_SIG
+# define EV_NSIG (MAX_SIG+1)
+#elif defined SIGARRAYSIZE
+# define EV_NSIG (SIGARRAYSIZE) /* Assume ary[SIGARRAYSIZE] */
+#elif defined _sys_nsig
+# define EV_NSIG (_sys_nsig) /* Solaris 2.5 */
+#else
+# define EV_NSIG (8 * sizeof (sigset_t) + 1)
+#endif
+
+#ifndef EV_USE_FLOOR
+# define EV_USE_FLOOR 0
+#endif
+
+#ifndef EV_USE_CLOCK_SYSCALL
+# if __linux && __GLIBC__ == 2 && __GLIBC_MINOR__ < 17
+# define EV_USE_CLOCK_SYSCALL EV_FEATURE_OS
+# else
+# define EV_USE_CLOCK_SYSCALL 0
+# endif
+#endif
+
+#if !(_POSIX_TIMERS > 0)
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 0
+# endif
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+#endif
+
+#ifndef EV_USE_MONOTONIC
+# if defined _POSIX_MONOTONIC_CLOCK && _POSIX_MONOTONIC_CLOCK >= 0
+# define EV_USE_MONOTONIC EV_FEATURE_OS
+# else
+# define EV_USE_MONOTONIC 0
+# endif
+#endif
+
+#ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME !EV_USE_CLOCK_SYSCALL
+#endif
+
+#ifndef EV_USE_NANOSLEEP
+# if _POSIX_C_SOURCE >= 199309L
+# define EV_USE_NANOSLEEP EV_FEATURE_OS
+# else
+# define EV_USE_NANOSLEEP 0
+# endif
+#endif
+
+#ifndef EV_USE_SELECT
+# define EV_USE_SELECT EV_FEATURE_BACKENDS
+#endif
+
+#ifndef EV_USE_POLL
+# ifdef _WIN32
+# define EV_USE_POLL 0
+# else
+# define EV_USE_POLL EV_FEATURE_BACKENDS
+# endif
+#endif
+
+#ifndef EV_USE_EPOLL
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 4))
+# define EV_USE_EPOLL EV_FEATURE_BACKENDS
+# else
+# define EV_USE_EPOLL 0
+# endif
+#endif
+
+#ifndef EV_USE_KQUEUE
+# define EV_USE_KQUEUE 0
+#endif
+
+#ifndef EV_USE_PORT
+# define EV_USE_PORT 0
+#endif
+
+#ifndef EV_USE_LINUXAIO
+# if __linux /* libev currently assumes linux/aio_abi.h is always available on linux */
+# define EV_USE_LINUXAIO 0 /* was: 1, always off by default */
+# else
+# define EV_USE_LINUXAIO 0
+# endif
+#endif
+
+#ifndef EV_USE_IOURING
+# if __linux /* later checks might disable again */
+# define EV_USE_IOURING 1
+# else
+# define EV_USE_IOURING 0
+# endif
+#endif
+
+#ifndef EV_USE_INOTIFY
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 4))
+# define EV_USE_INOTIFY EV_FEATURE_OS
+# else
+# define EV_USE_INOTIFY 0
+# endif
+#endif
+
+#ifndef EV_PID_HASHSIZE
+# define EV_PID_HASHSIZE EV_FEATURE_DATA ? 16 : 1
+#endif
+
+#ifndef EV_INOTIFY_HASHSIZE
+# define EV_INOTIFY_HASHSIZE EV_FEATURE_DATA ? 16 : 1
+#endif
+
+#ifndef EV_USE_EVENTFD
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 7))
+# define EV_USE_EVENTFD EV_FEATURE_OS
+# else
+# define EV_USE_EVENTFD 0
+# endif
+#endif
+
+#ifndef EV_USE_SIGNALFD
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 7))
+# define EV_USE_SIGNALFD EV_FEATURE_OS
+# else
+# define EV_USE_SIGNALFD 0
+# endif
+#endif
+
+#ifndef EV_USE_TIMERFD
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 8))
+# define EV_USE_TIMERFD EV_FEATURE_OS
+# else
+# define EV_USE_TIMERFD 0
+# endif
+#endif
+
+#if 0 /* debugging */
+# define EV_VERIFY 3
+# define EV_USE_4HEAP 1
+# define EV_HEAP_CACHE_AT 1
+#endif
+
+#ifndef EV_VERIFY
+# define EV_VERIFY (EV_FEATURE_API ? 1 : 0)
+#endif
+
+#ifndef EV_USE_4HEAP
+# define EV_USE_4HEAP EV_FEATURE_DATA
+#endif
+
+#ifndef EV_HEAP_CACHE_AT
+# define EV_HEAP_CACHE_AT EV_FEATURE_DATA
+#endif
+
+#ifdef __ANDROID__
+/* supposedly, android doesn't typedef fd_mask */
+# undef EV_USE_SELECT
+# define EV_USE_SELECT 0
+/* supposedly, we need to include syscall.h, not sys/syscall.h, so just disable */
+# undef EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 0
+#endif
+
+/* aix's poll.h seems to cause lots of trouble */
+#ifdef _AIX
+/* AIX has a completely broken poll.h header */
+# undef EV_USE_POLL
+# define EV_USE_POLL 0
+#endif
+
+/* on linux, we can use a (slow) syscall to avoid a dependency on pthread, */
+/* which makes programs even slower. might work on other unices, too. */
+#if EV_USE_CLOCK_SYSCALL
+# include <sys/syscall.h>
+# ifdef SYS_clock_gettime
+# define clock_gettime(id, ts) syscall (SYS_clock_gettime, (id), (ts))
+# undef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 1
+# define EV_NEED_SYSCALL 1
+# else
+# undef EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 0
+# endif
+#endif
+
+/* this block fixes any misconfiguration where we know we run into trouble otherwise */
+
+#ifndef CLOCK_MONOTONIC
+# undef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 0
+#endif
+
+#ifndef CLOCK_REALTIME
+# undef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+#endif
+
+#if !EV_STAT_ENABLE
+# undef EV_USE_INOTIFY
+# define EV_USE_INOTIFY 0
+#endif
+
+#if __linux && EV_USE_IOURING
+# include <linux/version.h>
+# if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+# undef EV_USE_IOURING
+# define EV_USE_IOURING 0
+# endif
+#endif
+
+#if !EV_USE_NANOSLEEP
+/* hp-ux has it in sys/time.h, which we unconditionally include above */
+# if !defined _WIN32 && !defined __hpux
+# include <sys/select.h>
+# endif
+#endif
+
+#if EV_USE_LINUXAIO
+# include <sys/syscall.h>
+# if SYS_io_getevents && EV_USE_EPOLL /* linuxaio backend requires epoll backend */
+# define EV_NEED_SYSCALL 1
+# else
+# undef EV_USE_LINUXAIO
+# define EV_USE_LINUXAIO 0
+# endif
+#endif
+
+#if EV_USE_IOURING
+# include <sys/syscall.h>
+# if !SYS_io_uring_register && __linux && !__alpha
+# define SYS_io_uring_setup 425
+# define SYS_io_uring_enter 426
+# define SYS_io_uring_register 427
+# endif
+# if SYS_io_uring_setup && EV_USE_EPOLL /* iouring backend requires epoll backend */
+# define EV_NEED_SYSCALL 1
+# else
+# undef EV_USE_IOURING
+# define EV_USE_IOURING 0
+# endif
+#endif
+
+#if EV_USE_INOTIFY
+# include <sys/statfs.h>
+# include <sys/inotify.h>
+/* some very old inotify.h headers don't have IN_DONT_FOLLOW */
+# ifndef IN_DONT_FOLLOW
+# undef EV_USE_INOTIFY
+# define EV_USE_INOTIFY 0
+# endif
+#endif
+
+#if EV_USE_EVENTFD
+/* our minimum requirement is glibc 2.7 which has the stub, but not the full header */
+# include <stdint.h>
+# ifndef EFD_NONBLOCK
+# define EFD_NONBLOCK O_NONBLOCK
+# endif
+# ifndef EFD_CLOEXEC
+# ifdef O_CLOEXEC
+# define EFD_CLOEXEC O_CLOEXEC
+# else
+# define EFD_CLOEXEC 02000000
+# endif
+# endif
+EV_CPP(extern "C") int (eventfd) (unsigned int initval, int flags);
+#endif
+
+#if EV_USE_SIGNALFD
+/* our minimum requirement is glibc 2.7 which has the stub, but not the full header */
+# include <stdint.h>
+# ifndef SFD_NONBLOCK
+# define SFD_NONBLOCK O_NONBLOCK
+# endif
+# ifndef SFD_CLOEXEC
+# ifdef O_CLOEXEC
+# define SFD_CLOEXEC O_CLOEXEC
+# else
+# define SFD_CLOEXEC 02000000
+# endif
+# endif
+EV_CPP (extern "C") int (signalfd) (int fd, const sigset_t *mask, int flags);
+
+struct signalfd_siginfo
+{
+ uint32_t ssi_signo;
+ char pad[128 - sizeof (uint32_t)];
+};
+#endif
+
+/* for timerfd, libev core requires TFD_TIMER_CANCEL_ON_SET &c */
+#if EV_USE_TIMERFD
+# include <sys/timerfd.h>
+/* timerfd is only used for periodics */
+# if !(defined (TFD_TIMER_CANCEL_ON_SET) && defined (TFD_CLOEXEC) && defined (TFD_NONBLOCK)) || !EV_PERIODIC_ENABLE
+# undef EV_USE_TIMERFD
+# define EV_USE_TIMERFD 0
+# endif
+#endif
+
+/*****************************************************************************/
+
+#if EV_VERIFY >= 3
+# define EV_FREQUENT_CHECK ev_verify (EV_A)
+#else
+# define EV_FREQUENT_CHECK do { } while (0)
+#endif
+
+/*
+ * This is used to work around floating point rounding problems.
+ * This value is good at least till the year 4000.
+ */
+#define MIN_INTERVAL 0.0001220703125 /* 1/2**13, good till 4000 */
+// #define MIN_INTERVAL 0.00000095367431640625 /* 1/2**20, good till 2200 */
+
+#define MIN_TIMEJUMP 1. /* minimum timejump that gets detected (if monotonic clock available) */
+#define MAX_BLOCKTIME 59.743 /* never wait longer than this time (to detect time jumps) */
+#define MAX_BLOCKTIME2 1500001.07 /* same, but when timerfd is used to detect jumps, also safe delay to not overflow */
+
+/* find a portable timestamp that is "always" in the future but fits into time_t.
+ * this is quite hard, and we are mostly guessing - we handle 32 bit signed/unsigned time_t,
+ * and sizes larger than 32 bit, and maybe the unlikely floating point time_t */
+#define EV_TSTAMP_HUGE \
+ (sizeof (time_t) >= 8 ? 10000000000000. \
+ : 0 < (time_t)4294967295 ? 4294967295. \
+ : 2147483647.) \
+
+#ifndef EV_TS_CONST
+# define EV_TS_CONST(nv) nv
+# define EV_TS_TO_MSEC(a) a * 1e3 + 0.9999
+# define EV_TS_FROM_USEC(us) us * 1e-6
+# define EV_TV_SET(tv,t) do { tv.tv_sec = (long)t; tv.tv_usec = (long)((t - tv.tv_sec) * 1e6); } while (0)
+# define EV_TS_SET(ts,t) do { ts.tv_sec = (long)t; ts.tv_nsec = (long)((t - ts.tv_sec) * 1e9); } while (0)
+# define EV_TV_GET(tv) ((tv).tv_sec + (tv).tv_usec * 1e-6)
+# define EV_TS_GET(ts) ((ts).tv_sec + (ts).tv_nsec * 1e-9)
+#endif
+
+/* the following is ecb.h embedded into libev - use update_ev_c to update from an external copy */
+/* ECB.H BEGIN */
+/*
+ * libecb - http://software.schmorp.de/pkg/libecb
+ *
+ * Copyright (©) 2009-2015,2018-2020 Marc Alexander Lehmann <libecb@schmorp.de>
+ * Copyright (©) 2011 Emanuele Giaquinta
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef ECB_H
+#define ECB_H
+
+/* 16 bits major, 16 bits minor */
+#define ECB_VERSION 0x00010008
+
+#include <string.h> /* for memcpy */
+
+#if defined (_WIN32) && !defined (__MINGW32__)
+ typedef signed char int8_t;
+ typedef unsigned char uint8_t;
+ typedef signed char int_fast8_t;
+ typedef unsigned char uint_fast8_t;
+ typedef signed short int16_t;
+ typedef unsigned short uint16_t;
+ typedef signed int int_fast16_t;
+ typedef unsigned int uint_fast16_t;
+ typedef signed int int32_t;
+ typedef unsigned int uint32_t;
+ typedef signed int int_fast32_t;
+ typedef unsigned int uint_fast32_t;
+ #if __GNUC__
+ typedef signed long long int64_t;
+ typedef unsigned long long uint64_t;
+ #else /* _MSC_VER || __BORLANDC__ */
+ typedef signed __int64 int64_t;
+ typedef unsigned __int64 uint64_t;
+ #endif
+ typedef int64_t int_fast64_t;
+ typedef uint64_t uint_fast64_t;
+ #ifdef _WIN64
+ #define ECB_PTRSIZE 8
+ typedef uint64_t uintptr_t;
+ typedef int64_t intptr_t;
+ #else
+ #define ECB_PTRSIZE 4
+ typedef uint32_t uintptr_t;
+ typedef int32_t intptr_t;
+ #endif
+#else
+ #include <inttypes.h>
+ #if (defined INTPTR_MAX ? INTPTR_MAX : ULONG_MAX) > 0xffffffffU
+ #define ECB_PTRSIZE 8
+ #else
+ #define ECB_PTRSIZE 4
+ #endif
+#endif
+
+#define ECB_GCC_AMD64 (__amd64 || __amd64__ || __x86_64 || __x86_64__)
+#define ECB_MSVC_AMD64 (_M_AMD64 || _M_X64)
+
+#ifndef ECB_OPTIMIZE_SIZE
+ #if __OPTIMIZE_SIZE__
+ #define ECB_OPTIMIZE_SIZE 1
+ #else
+ #define ECB_OPTIMIZE_SIZE 0
+ #endif
+#endif
+
+/* work around x32 idiocy by defining proper macros */
+#if ECB_GCC_AMD64 || ECB_MSVC_AMD64
+ #if _ILP32
+ #define ECB_AMD64_X32 1
+ #else
+ #define ECB_AMD64 1
+ #endif
+#endif
+
+/* many compilers define _GNUC_ to some versions but then only implement
+ * what their idiot authors think are the "more important" extensions,
+ * causing enormous grief in return for some better fake benchmark numbers.
+ * or so.
+ * we try to detect these and simply assume they are not gcc - if they have
+ * an issue with that they should have done it right in the first place.
+ */
+#if !defined __GNUC_MINOR__ || defined __INTEL_COMPILER || defined __SUNPRO_C || defined __SUNPRO_CC || defined __llvm__ || defined __clang__
+ #define ECB_GCC_VERSION(major,minor) 0
+#else
+ #define ECB_GCC_VERSION(major,minor) (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#endif
+
+#define ECB_CLANG_VERSION(major,minor) (__clang_major__ > (major) || (__clang_major__ == (major) && __clang_minor__ >= (minor)))
+
+#if __clang__ && defined __has_builtin
+ #define ECB_CLANG_BUILTIN(x) __has_builtin (x)
+#else
+ #define ECB_CLANG_BUILTIN(x) 0
+#endif
+
+#if __clang__ && defined __has_extension
+ #define ECB_CLANG_EXTENSION(x) __has_extension (x)
+#else
+ #define ECB_CLANG_EXTENSION(x) 0
+#endif
+
+#define ECB_CPP (__cplusplus+0)
+#define ECB_CPP11 (__cplusplus >= 201103L)
+#define ECB_CPP14 (__cplusplus >= 201402L)
+#define ECB_CPP17 (__cplusplus >= 201703L)
+
+#if ECB_CPP
+ #define ECB_C 0
+ #define ECB_STDC_VERSION 0
+#else
+ #define ECB_C 1
+ #define ECB_STDC_VERSION __STDC_VERSION__
+#endif
+
+#define ECB_C99 (ECB_STDC_VERSION >= 199901L)
+#define ECB_C11 (ECB_STDC_VERSION >= 201112L)
+#define ECB_C17 (ECB_STDC_VERSION >= 201710L)
+
+#if ECB_CPP
+ #define ECB_EXTERN_C extern "C"
+ #define ECB_EXTERN_C_BEG ECB_EXTERN_C {
+ #define ECB_EXTERN_C_END }
+#else
+ #define ECB_EXTERN_C extern
+ #define ECB_EXTERN_C_BEG
+ #define ECB_EXTERN_C_END
+#endif
+
+/*****************************************************************************/
+
+/* ECB_NO_THREADS - ecb is not used by multiple threads, ever */
+/* ECB_NO_SMP - ecb might be used in multiple threads, but only on a single cpu */
+
+#if ECB_NO_THREADS
+ #define ECB_NO_SMP 1
+#endif
+
+#if ECB_NO_SMP
+ #define ECB_MEMORY_FENCE do { } while (0)
+#endif
+
+/* http://www-01.ibm.com/support/knowledgecenter/SSGH3R_13.1.0/com.ibm.xlcpp131.aix.doc/compiler_ref/compiler_builtins.html */
+#if __xlC__ && ECB_CPP
+ #include <builtins.h>
+#endif
+
+#if 1400 <= _MSC_VER
+ #include <intrin.h> /* fence functions _ReadBarrier, also bit search functions _BitScanReverse */
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if ECB_GCC_VERSION(2,5) || defined __INTEL_COMPILER || (__llvm__ && __GNUC__) || __SUNPRO_C >= 0x5110 || __SUNPRO_CC >= 0x5110
+ #define ECB_MEMORY_FENCE_RELAXED __asm__ __volatile__ ("" : : : "memory")
+ #if __i386 || __i386__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("lock; orb $0, -1(%%esp)" : : : "memory")
+ #define ECB_MEMORY_FENCE_ACQUIRE __asm__ __volatile__ ("" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("" : : : "memory")
+ #elif ECB_GCC_AMD64
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mfence" : : : "memory")
+ #define ECB_MEMORY_FENCE_ACQUIRE __asm__ __volatile__ ("" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("" : : : "memory")
+ #elif __powerpc__ || __ppc__ || __powerpc64__ || __ppc64__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("sync" : : : "memory")
+ #elif defined __ARM_ARCH_2__ \
+ || defined __ARM_ARCH_3__ || defined __ARM_ARCH_3M__ \
+ || defined __ARM_ARCH_4__ || defined __ARM_ARCH_4T__ \
+ || defined __ARM_ARCH_5__ || defined __ARM_ARCH_5E__ \
+ || defined __ARM_ARCH_5T__ || defined __ARM_ARCH_5TE__ \
+ || defined __ARM_ARCH_5TEJ__
+ /* should not need any, unless running old code on newer cpu - arm doesn't support that */
+ #elif defined __ARM_ARCH_6__ || defined __ARM_ARCH_6J__ \
+ || defined __ARM_ARCH_6K__ || defined __ARM_ARCH_6ZK__ \
+ || defined __ARM_ARCH_6T2__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mcr p15,0,%0,c7,c10,5" : : "r" (0) : "memory")
+ #elif defined __ARM_ARCH_7__ || defined __ARM_ARCH_7A__ \
+ || defined __ARM_ARCH_7R__ || defined __ARM_ARCH_7M__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("dmb" : : : "memory")
+ #elif __aarch64__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("dmb ish" : : : "memory")
+ #elif (__sparc || __sparc__) && !(__sparc_v8__ || defined __sparcv8)
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("membar #LoadStore | #LoadLoad | #StoreStore | #StoreLoad" : : : "memory")
+ #define ECB_MEMORY_FENCE_ACQUIRE __asm__ __volatile__ ("membar #LoadStore | #LoadLoad" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("membar #LoadStore | #StoreStore")
+ #elif defined __s390__ || defined __s390x__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("bcr 15,0" : : : "memory")
+ #elif defined __mips__
+ /* GNU/Linux emulates sync on mips1 architectures, so we force its use */
+ /* anybody else who still uses mips1 is supposed to send in their version, with detection code. */
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ (".set mips2; sync; .set mips0" : : : "memory")
+ #elif defined __alpha__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mb" : : : "memory")
+ #elif defined __hppa__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("")
+ #elif defined __ia64__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mf" : : : "memory")
+ #elif defined __m68k__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("" : : : "memory")
+ #elif defined __m88k__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("tb1 0,%%r0,128" : : : "memory")
+ #elif defined __sh__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("" : : : "memory")
+ #elif defined __loongarch__ || __loongarch64
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("dbar %0 ": : "I"(0) : "memory")
+ #endif
+ #endif
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if ECB_GCC_VERSION(4,7)
+ /* see comment below (stdatomic.h) about the C11 memory model. */
+ #define ECB_MEMORY_FENCE __atomic_thread_fence (__ATOMIC_SEQ_CST)
+ #define ECB_MEMORY_FENCE_ACQUIRE __atomic_thread_fence (__ATOMIC_ACQUIRE)
+ #define ECB_MEMORY_FENCE_RELEASE __atomic_thread_fence (__ATOMIC_RELEASE)
+ #define ECB_MEMORY_FENCE_RELAXED __atomic_thread_fence (__ATOMIC_RELAXED)
+
+ #elif ECB_CLANG_EXTENSION(c_atomic)
+ /* see comment below (stdatomic.h) about the C11 memory model. */
+ #define ECB_MEMORY_FENCE __c11_atomic_thread_fence (__ATOMIC_SEQ_CST)
+ #define ECB_MEMORY_FENCE_ACQUIRE __c11_atomic_thread_fence (__ATOMIC_ACQUIRE)
+ #define ECB_MEMORY_FENCE_RELEASE __c11_atomic_thread_fence (__ATOMIC_RELEASE)
+ #define ECB_MEMORY_FENCE_RELAXED __c11_atomic_thread_fence (__ATOMIC_RELAXED)
+
+ #elif ECB_GCC_VERSION(4,4) || defined __INTEL_COMPILER || defined __clang__
+ #define ECB_MEMORY_FENCE __sync_synchronize ()
+ #elif _MSC_VER >= 1500 /* VC++ 2008 */
+ /* apparently, microsoft broke all the memory barrier stuff in Visual Studio 2008... */
+ #pragma intrinsic(_ReadBarrier,_WriteBarrier,_ReadWriteBarrier)
+ #define ECB_MEMORY_FENCE _ReadWriteBarrier (); MemoryBarrier()
+ #define ECB_MEMORY_FENCE_ACQUIRE _ReadWriteBarrier (); MemoryBarrier() /* according to msdn, _ReadBarrier is not a load fence */
+ #define ECB_MEMORY_FENCE_RELEASE _WriteBarrier (); MemoryBarrier()
+ #elif _MSC_VER >= 1400 /* VC++ 2005 */
+ #pragma intrinsic(_ReadBarrier,_WriteBarrier,_ReadWriteBarrier)
+ #define ECB_MEMORY_FENCE _ReadWriteBarrier ()
+ #define ECB_MEMORY_FENCE_ACQUIRE _ReadWriteBarrier () /* according to msdn, _ReadBarrier is not a load fence */
+ #define ECB_MEMORY_FENCE_RELEASE _WriteBarrier ()
+ #elif defined _WIN32
+ #include <WinNT.h>
+ #define ECB_MEMORY_FENCE MemoryBarrier () /* actually just xchg on x86... scary */
+ #elif __SUNPRO_C >= 0x5110 || __SUNPRO_CC >= 0x5110
+ #include <mbarrier.h>
+ #define ECB_MEMORY_FENCE __machine_rw_barrier ()
+ #define ECB_MEMORY_FENCE_ACQUIRE __machine_acq_barrier ()
+ #define ECB_MEMORY_FENCE_RELEASE __machine_rel_barrier ()
+ #define ECB_MEMORY_FENCE_RELAXED __compiler_barrier ()
+ #elif __xlC__
+ #define ECB_MEMORY_FENCE __sync ()
+ #endif
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if ECB_C11 && !defined __STDC_NO_ATOMICS__
+ /* we assume that these memory fences work on all variables/all memory accesses, */
+ /* not just C11 atomics and atomic accesses */
+ #include <stdatomic.h>
+ #define ECB_MEMORY_FENCE atomic_thread_fence (memory_order_seq_cst)
+ #define ECB_MEMORY_FENCE_ACQUIRE atomic_thread_fence (memory_order_acquire)
+ #define ECB_MEMORY_FENCE_RELEASE atomic_thread_fence (memory_order_release)
+ #endif
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if !ECB_AVOID_PTHREADS
+ /*
+ * if you get undefined symbol references to pthread_mutex_lock,
+ * or failure to find pthread.h, then you should implement
+ * the ECB_MEMORY_FENCE operations for your cpu/compiler
+ * OR provide pthread.h and link against the posix thread library
+ * of your system.
+ */
+ #include <pthread.h>
+ #define ECB_NEEDS_PTHREADS 1
+ #define ECB_MEMORY_FENCE_NEEDS_PTHREADS 1
+
+ static pthread_mutex_t ecb_mf_lock = PTHREAD_MUTEX_INITIALIZER;
+ #define ECB_MEMORY_FENCE do { pthread_mutex_lock (&ecb_mf_lock); pthread_mutex_unlock (&ecb_mf_lock); } while (0)
+ #endif
+#endif
+
+#if !defined ECB_MEMORY_FENCE_ACQUIRE && defined ECB_MEMORY_FENCE
+ #define ECB_MEMORY_FENCE_ACQUIRE ECB_MEMORY_FENCE
+#endif
+
+#if !defined ECB_MEMORY_FENCE_RELEASE && defined ECB_MEMORY_FENCE
+ #define ECB_MEMORY_FENCE_RELEASE ECB_MEMORY_FENCE
+#endif
+
+#if !defined ECB_MEMORY_FENCE_RELAXED && defined ECB_MEMORY_FENCE
+ #define ECB_MEMORY_FENCE_RELAXED ECB_MEMORY_FENCE /* very heavy-handed */
+#endif
+
+/*****************************************************************************/
+
+#if ECB_CPP
+ #define ecb_inline static inline
+#elif ECB_GCC_VERSION(2,5)
+ #define ecb_inline static __inline__
+#elif ECB_C99
+ #define ecb_inline static inline
+#else
+ #define ecb_inline static
+#endif
+
+#if ECB_GCC_VERSION(3,3)
+ #define ecb_restrict __restrict__
+#elif ECB_C99
+ #define ecb_restrict restrict
+#else
+ #define ecb_restrict
+#endif
+
+typedef int ecb_bool;
+
+#define ECB_CONCAT_(a, b) a ## b
+#define ECB_CONCAT(a, b) ECB_CONCAT_(a, b)
+#define ECB_STRINGIFY_(a) # a
+#define ECB_STRINGIFY(a) ECB_STRINGIFY_(a)
+#define ECB_STRINGIFY_EXPR(expr) ((expr), ECB_STRINGIFY_ (expr))
+
+#define ecb_function_ ecb_inline
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_VERSION(2,8)
+ #define ecb_attribute(attrlist) __attribute__ (attrlist)
+#else
+ #define ecb_attribute(attrlist)
+#endif
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_BUILTIN(__builtin_constant_p)
+ #define ecb_is_constant(expr) __builtin_constant_p (expr)
+#else
+ /* possible C11 impl for integral types
+ typedef struct ecb_is_constant_struct ecb_is_constant_struct;
+ #define ecb_is_constant(expr) _Generic ((1 ? (struct ecb_is_constant_struct *)0 : (void *)((expr) - (expr)), ecb_is_constant_struct *: 0, default: 1)) */
+
+ #define ecb_is_constant(expr) 0
+#endif
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_BUILTIN(__builtin_expect)
+ #define ecb_expect(expr,value) __builtin_expect ((expr),(value))
+#else
+ #define ecb_expect(expr,value) (expr)
+#endif
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_BUILTIN(__builtin_prefetch)
+ #define ecb_prefetch(addr,rw,locality) __builtin_prefetch (addr, rw, locality)
+#else
+ #define ecb_prefetch(addr,rw,locality)
+#endif
+
+/* no emulation for ecb_decltype */
+#if ECB_CPP11
+ // older implementations might have problems with decltype(x)::type, work around it
+ template<class T> struct ecb_decltype_t { typedef T type; };
+ #define ecb_decltype(x) ecb_decltype_t<decltype (x)>::type
+#elif ECB_GCC_VERSION(3,0) || ECB_CLANG_VERSION(2,8)
+ #define ecb_decltype(x) __typeof__ (x)
+#endif
+
+#if _MSC_VER >= 1300
+ #define ecb_deprecated __declspec (deprecated)
+#else
+ #define ecb_deprecated ecb_attribute ((__deprecated__))
+#endif
+
+#if _MSC_VER >= 1500
+ #define ecb_deprecated_message(msg) __declspec (deprecated (msg))
+#elif ECB_GCC_VERSION(4,5)
+ #define ecb_deprecated_message(msg) ecb_attribute ((__deprecated__ (msg))
+#else
+ #define ecb_deprecated_message(msg) ecb_deprecated
+#endif
+
+#if _MSC_VER >= 1400
+ #define ecb_noinline __declspec (noinline)
+#else
+ #define ecb_noinline ecb_attribute ((__noinline__))
+#endif
+
+#define ecb_unused ecb_attribute ((__unused__))
+#define ecb_const ecb_attribute ((__const__))
+#define ecb_pure ecb_attribute ((__pure__))
+
+#if ECB_C11 || __IBMC_NORETURN
+ /* http://www-01.ibm.com/support/knowledgecenter/SSGH3R_13.1.0/com.ibm.xlcpp131.aix.doc/language_ref/noreturn.html */
+ #define ecb_noreturn _Noreturn
+#elif ECB_CPP11
+ #define ecb_noreturn [[noreturn]]
+#elif _MSC_VER >= 1200
+ /* http://msdn.microsoft.com/en-us/library/k6ktzx3s.aspx */
+ #define ecb_noreturn __declspec (noreturn)
+#else
+ #define ecb_noreturn ecb_attribute ((__noreturn__))
+#endif
+
+#if ECB_GCC_VERSION(4,3)
+ #define ecb_artificial ecb_attribute ((__artificial__))
+ #define ecb_hot ecb_attribute ((__hot__))
+ #define ecb_cold ecb_attribute ((__cold__))
+#else
+ #define ecb_artificial
+ #define ecb_hot
+ #define ecb_cold
+#endif
+
+/* put around conditional expressions if you are very sure that the */
+/* expression is mostly true or mostly false. note that these return */
+/* booleans, not the expression. */
+#define ecb_expect_false(expr) ecb_expect (!!(expr), 0)
+#define ecb_expect_true(expr) ecb_expect (!!(expr), 1)
+/* for compatibility to the rest of the world */
+#define ecb_likely(expr) ecb_expect_true (expr)
+#define ecb_unlikely(expr) ecb_expect_false (expr)
+
+/* count trailing zero bits and count # of one bits */
+#if ECB_GCC_VERSION(3,4) \
+ || (ECB_CLANG_BUILTIN(__builtin_clz) && ECB_CLANG_BUILTIN(__builtin_clzll) \
+ && ECB_CLANG_BUILTIN(__builtin_ctz) && ECB_CLANG_BUILTIN(__builtin_ctzll) \
+ && ECB_CLANG_BUILTIN(__builtin_popcount))
+ /* we assume int == 32 bit, long == 32 or 64 bit and long long == 64 bit */
+ #define ecb_ld32(x) (__builtin_clz (x) ^ 31)
+ #define ecb_ld64(x) (__builtin_clzll (x) ^ 63)
+ #define ecb_ctz32(x) __builtin_ctz (x)
+ #define ecb_ctz64(x) __builtin_ctzll (x)
+ #define ecb_popcount32(x) __builtin_popcount (x)
+ /* no popcountll */
+#else
+ ecb_function_ ecb_const int ecb_ctz32 (uint32_t x);
+ ecb_function_ ecb_const int
+ ecb_ctz32 (uint32_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_IX86 || _M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanForward (&r, x);
+ return (int)r;
+#else
+ int r = 0;
+
+ x &= ~x + 1; /* this isolates the lowest bit */
+
+#if ECB_branchless_on_i386
+ r += !!(x & 0xaaaaaaaa) << 0;
+ r += !!(x & 0xcccccccc) << 1;
+ r += !!(x & 0xf0f0f0f0) << 2;
+ r += !!(x & 0xff00ff00) << 3;
+ r += !!(x & 0xffff0000) << 4;
+#else
+ if (x & 0xaaaaaaaa) r += 1;
+ if (x & 0xcccccccc) r += 2;
+ if (x & 0xf0f0f0f0) r += 4;
+ if (x & 0xff00ff00) r += 8;
+ if (x & 0xffff0000) r += 16;
+#endif
+
+ return r;
+#endif
+ }
+
+ ecb_function_ ecb_const int ecb_ctz64 (uint64_t x);
+ ecb_function_ ecb_const int
+ ecb_ctz64 (uint64_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanForward64 (&r, x);
+ return (int)r;
+#else
+ int shift = x & 0xffffffff ? 0 : 32;
+ return ecb_ctz32 (x >> shift) + shift;
+#endif
+ }
+
+ ecb_function_ ecb_const int ecb_popcount32 (uint32_t x);
+ ecb_function_ ecb_const int
+ ecb_popcount32 (uint32_t x)
+ {
+ x -= (x >> 1) & 0x55555555;
+ x = ((x >> 2) & 0x33333333) + (x & 0x33333333);
+ x = ((x >> 4) + x) & 0x0f0f0f0f;
+ x *= 0x01010101;
+
+ return x >> 24;
+ }
+
+ ecb_function_ ecb_const int ecb_ld32 (uint32_t x);
+ ecb_function_ ecb_const int ecb_ld32 (uint32_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_IX86 || _M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanReverse (&r, x);
+ return (int)r;
+#else
+ int r = 0;
+
+ if (x >> 16) { x >>= 16; r += 16; }
+ if (x >> 8) { x >>= 8; r += 8; }
+ if (x >> 4) { x >>= 4; r += 4; }
+ if (x >> 2) { x >>= 2; r += 2; }
+ if (x >> 1) { r += 1; }
+
+ return r;
+#endif
+ }
+
+ ecb_function_ ecb_const int ecb_ld64 (uint64_t x);
+ ecb_function_ ecb_const int ecb_ld64 (uint64_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanReverse64 (&r, x);
+ return (int)r;
+#else
+ int r = 0;
+
+ if (x >> 32) { x >>= 32; r += 32; }
+
+ return r + ecb_ld32 (x);
+#endif
+ }
+#endif
+
+ecb_function_ ecb_const ecb_bool ecb_is_pot32 (uint32_t x);
+ecb_function_ ecb_const ecb_bool ecb_is_pot32 (uint32_t x) { return !(x & (x - 1)); }
+ecb_function_ ecb_const ecb_bool ecb_is_pot64 (uint64_t x);
+ecb_function_ ecb_const ecb_bool ecb_is_pot64 (uint64_t x) { return !(x & (x - 1)); }
+
+ecb_function_ ecb_const uint8_t ecb_bitrev8 (uint8_t x);
+ecb_function_ ecb_const uint8_t ecb_bitrev8 (uint8_t x)
+{
+ return ( (x * 0x0802U & 0x22110U)
+ | (x * 0x8020U & 0x88440U)) * 0x10101U >> 16;
+}
+
+ecb_function_ ecb_const uint16_t ecb_bitrev16 (uint16_t x);
+ecb_function_ ecb_const uint16_t ecb_bitrev16 (uint16_t x)
+{
+ x = ((x >> 1) & 0x5555) | ((x & 0x5555) << 1);
+ x = ((x >> 2) & 0x3333) | ((x & 0x3333) << 2);
+ x = ((x >> 4) & 0x0f0f) | ((x & 0x0f0f) << 4);
+ x = ( x >> 8 ) | ( x << 8);
+
+ return x;
+}
+
+ecb_function_ ecb_const uint32_t ecb_bitrev32 (uint32_t x);
+ecb_function_ ecb_const uint32_t ecb_bitrev32 (uint32_t x)
+{
+ x = ((x >> 1) & 0x55555555) | ((x & 0x55555555) << 1);
+ x = ((x >> 2) & 0x33333333) | ((x & 0x33333333) << 2);
+ x = ((x >> 4) & 0x0f0f0f0f) | ((x & 0x0f0f0f0f) << 4);
+ x = ((x >> 8) & 0x00ff00ff) | ((x & 0x00ff00ff) << 8);
+ x = ( x >> 16 ) | ( x << 16);
+
+ return x;
+}
+
+/* popcount64 is only available on 64 bit cpus as gcc builtin */
+/* so for this version we are lazy */
+ecb_function_ ecb_const int ecb_popcount64 (uint64_t x);
+ecb_function_ ecb_const int
+ecb_popcount64 (uint64_t x)
+{
+ return ecb_popcount32 (x) + ecb_popcount32 (x >> 32);
+}
+
+ecb_inline ecb_const uint8_t ecb_rotl8 (uint8_t x, unsigned int count);
+ecb_inline ecb_const uint8_t ecb_rotr8 (uint8_t x, unsigned int count);
+ecb_inline ecb_const uint16_t ecb_rotl16 (uint16_t x, unsigned int count);
+ecb_inline ecb_const uint16_t ecb_rotr16 (uint16_t x, unsigned int count);
+ecb_inline ecb_const uint32_t ecb_rotl32 (uint32_t x, unsigned int count);
+ecb_inline ecb_const uint32_t ecb_rotr32 (uint32_t x, unsigned int count);
+ecb_inline ecb_const uint64_t ecb_rotl64 (uint64_t x, unsigned int count);
+ecb_inline ecb_const uint64_t ecb_rotr64 (uint64_t x, unsigned int count);
+
+ecb_inline ecb_const uint8_t ecb_rotl8 (uint8_t x, unsigned int count) { return (x >> ( 8 - count)) | (x << count); }
+ecb_inline ecb_const uint8_t ecb_rotr8 (uint8_t x, unsigned int count) { return (x << ( 8 - count)) | (x >> count); }
+ecb_inline ecb_const uint16_t ecb_rotl16 (uint16_t x, unsigned int count) { return (x >> (16 - count)) | (x << count); }
+ecb_inline ecb_const uint16_t ecb_rotr16 (uint16_t x, unsigned int count) { return (x << (16 - count)) | (x >> count); }
+ecb_inline ecb_const uint32_t ecb_rotl32 (uint32_t x, unsigned int count) { return (x >> (32 - count)) | (x << count); }
+ecb_inline ecb_const uint32_t ecb_rotr32 (uint32_t x, unsigned int count) { return (x << (32 - count)) | (x >> count); }
+ecb_inline ecb_const uint64_t ecb_rotl64 (uint64_t x, unsigned int count) { return (x >> (64 - count)) | (x << count); }
+ecb_inline ecb_const uint64_t ecb_rotr64 (uint64_t x, unsigned int count) { return (x << (64 - count)) | (x >> count); }
+
+#if ECB_CPP
+
+inline uint8_t ecb_ctz (uint8_t v) { return ecb_ctz32 (v); }
+inline uint16_t ecb_ctz (uint16_t v) { return ecb_ctz32 (v); }
+inline uint32_t ecb_ctz (uint32_t v) { return ecb_ctz32 (v); }
+inline uint64_t ecb_ctz (uint64_t v) { return ecb_ctz64 (v); }
+
+inline bool ecb_is_pot (uint8_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint16_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint32_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint64_t v) { return ecb_is_pot64 (v); }
+
+inline int ecb_ld (uint8_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint16_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint32_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint64_t v) { return ecb_ld64 (v); }
+
+inline int ecb_popcount (uint8_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint16_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint32_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint64_t v) { return ecb_popcount64 (v); }
+
+inline uint8_t ecb_bitrev (uint8_t v) { return ecb_bitrev8 (v); }
+inline uint16_t ecb_bitrev (uint16_t v) { return ecb_bitrev16 (v); }
+inline uint32_t ecb_bitrev (uint32_t v) { return ecb_bitrev32 (v); }
+
+inline uint8_t ecb_rotl (uint8_t v, unsigned int count) { return ecb_rotl8 (v, count); }
+inline uint16_t ecb_rotl (uint16_t v, unsigned int count) { return ecb_rotl16 (v, count); }
+inline uint32_t ecb_rotl (uint32_t v, unsigned int count) { return ecb_rotl32 (v, count); }
+inline uint64_t ecb_rotl (uint64_t v, unsigned int count) { return ecb_rotl64 (v, count); }
+
+inline uint8_t ecb_rotr (uint8_t v, unsigned int count) { return ecb_rotr8 (v, count); }
+inline uint16_t ecb_rotr (uint16_t v, unsigned int count) { return ecb_rotr16 (v, count); }
+inline uint32_t ecb_rotr (uint32_t v, unsigned int count) { return ecb_rotr32 (v, count); }
+inline uint64_t ecb_rotr (uint64_t v, unsigned int count) { return ecb_rotr64 (v, count); }
+
+#endif
+
+#if ECB_GCC_VERSION(4,3) || (ECB_CLANG_BUILTIN(__builtin_bswap32) && ECB_CLANG_BUILTIN(__builtin_bswap64))
+ #if ECB_GCC_VERSION(4,8) || ECB_CLANG_BUILTIN(__builtin_bswap16)
+ #define ecb_bswap16(x) __builtin_bswap16 (x)
+ #else
+ #define ecb_bswap16(x) (__builtin_bswap32 (x) >> 16)
+ #endif
+ #define ecb_bswap32(x) __builtin_bswap32 (x)
+ #define ecb_bswap64(x) __builtin_bswap64 (x)
+#elif _MSC_VER
+ #include <stdlib.h>
+ #define ecb_bswap16(x) ((uint16_t)_byteswap_ushort ((uint16_t)(x)))
+ #define ecb_bswap32(x) ((uint32_t)_byteswap_ulong ((uint32_t)(x)))
+ #define ecb_bswap64(x) ((uint64_t)_byteswap_uint64 ((uint64_t)(x)))
+#else
+ ecb_function_ ecb_const uint16_t ecb_bswap16 (uint16_t x);
+ ecb_function_ ecb_const uint16_t
+ ecb_bswap16 (uint16_t x)
+ {
+ return ecb_rotl16 (x, 8);
+ }
+
+ ecb_function_ ecb_const uint32_t ecb_bswap32 (uint32_t x);
+ ecb_function_ ecb_const uint32_t
+ ecb_bswap32 (uint32_t x)
+ {
+ return (((uint32_t)ecb_bswap16 (x)) << 16) | ecb_bswap16 (x >> 16);
+ }
+
+ ecb_function_ ecb_const uint64_t ecb_bswap64 (uint64_t x);
+ ecb_function_ ecb_const uint64_t
+ ecb_bswap64 (uint64_t x)
+ {
+ return (((uint64_t)ecb_bswap32 (x)) << 32) | ecb_bswap32 (x >> 32);
+ }
+#endif
+
+#if ECB_GCC_VERSION(4,5) || ECB_CLANG_BUILTIN(__builtin_unreachable)
+ #define ecb_unreachable() __builtin_unreachable ()
+#else
+ /* this seems to work fine, but gcc always emits a warning for it :/ */
+ ecb_inline ecb_noreturn void ecb_unreachable (void);
+ ecb_inline ecb_noreturn void ecb_unreachable (void) { }
+#endif
+
+/* try to tell the compiler that some condition is definitely true */
+#define ecb_assume(cond) if (!(cond)) ecb_unreachable (); else 0
+
+ecb_inline ecb_const uint32_t ecb_byteorder_helper (void);
+ecb_inline ecb_const uint32_t
+ecb_byteorder_helper (void)
+{
+ /* the union code still generates code under pressure in gcc, */
+ /* but less than using pointers, and always seems to */
+ /* successfully return a constant. */
+ /* the reason why we have this horrible preprocessor mess */
+ /* is to avoid it in all cases, at least on common architectures */
+ /* or when using a recent enough gcc version (>= 4.6) */
+#if (defined __BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
+ || ((__i386 || __i386__ || _M_IX86 || ECB_GCC_AMD64 || ECB_MSVC_AMD64) && !__VOS__)
+ #define ECB_LITTLE_ENDIAN 1
+ return 0x44332211;
+#elif (defined __BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \
+ || ((__AARCH64EB__ || __MIPSEB__ || __ARMEB__) && !__VOS__)
+ #define ECB_BIG_ENDIAN 1
+ return 0x11223344;
+#else
+ union
+ {
+ uint8_t c[4];
+ uint32_t u;
+ } u = { 0x11, 0x22, 0x33, 0x44 };
+ return u.u;
+#endif
+}
+
+ecb_inline ecb_const ecb_bool ecb_big_endian (void);
+ecb_inline ecb_const ecb_bool ecb_big_endian (void) { return ecb_byteorder_helper () == 0x11223344; }
+ecb_inline ecb_const ecb_bool ecb_little_endian (void);
+ecb_inline ecb_const ecb_bool ecb_little_endian (void) { return ecb_byteorder_helper () == 0x44332211; }
+
+/*****************************************************************************/
+/* unaligned load/store */
+
+ecb_inline uint_fast16_t ecb_be_u16_to_host (uint_fast16_t v) { return ecb_little_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_be_u32_to_host (uint_fast32_t v) { return ecb_little_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_be_u64_to_host (uint_fast64_t v) { return ecb_little_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_le_u16_to_host (uint_fast16_t v) { return ecb_big_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_le_u32_to_host (uint_fast32_t v) { return ecb_big_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_le_u64_to_host (uint_fast64_t v) { return ecb_big_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_peek_u16_u (const void *ptr) { uint16_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+ecb_inline uint_fast32_t ecb_peek_u32_u (const void *ptr) { uint32_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+ecb_inline uint_fast64_t ecb_peek_u64_u (const void *ptr) { uint64_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+
+ecb_inline uint_fast16_t ecb_peek_be_u16_u (const void *ptr) { return ecb_be_u16_to_host (ecb_peek_u16_u (ptr)); }
+ecb_inline uint_fast32_t ecb_peek_be_u32_u (const void *ptr) { return ecb_be_u32_to_host (ecb_peek_u32_u (ptr)); }
+ecb_inline uint_fast64_t ecb_peek_be_u64_u (const void *ptr) { return ecb_be_u64_to_host (ecb_peek_u64_u (ptr)); }
+
+ecb_inline uint_fast16_t ecb_peek_le_u16_u (const void *ptr) { return ecb_le_u16_to_host (ecb_peek_u16_u (ptr)); }
+ecb_inline uint_fast32_t ecb_peek_le_u32_u (const void *ptr) { return ecb_le_u32_to_host (ecb_peek_u32_u (ptr)); }
+ecb_inline uint_fast64_t ecb_peek_le_u64_u (const void *ptr) { return ecb_le_u64_to_host (ecb_peek_u64_u (ptr)); }
+
+ecb_inline uint_fast16_t ecb_host_to_be_u16 (uint_fast16_t v) { return ecb_little_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_host_to_be_u32 (uint_fast32_t v) { return ecb_little_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_host_to_be_u64 (uint_fast64_t v) { return ecb_little_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_host_to_le_u16 (uint_fast16_t v) { return ecb_big_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_host_to_le_u32 (uint_fast32_t v) { return ecb_big_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_host_to_le_u64 (uint_fast64_t v) { return ecb_big_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline void ecb_poke_u16_u (void *ptr, uint16_t v) { memcpy (ptr, &v, sizeof (v)); }
+ecb_inline void ecb_poke_u32_u (void *ptr, uint32_t v) { memcpy (ptr, &v, sizeof (v)); }
+ecb_inline void ecb_poke_u64_u (void *ptr, uint64_t v) { memcpy (ptr, &v, sizeof (v)); }
+
+ecb_inline void ecb_poke_be_u16_u (void *ptr, uint_fast16_t v) { ecb_poke_u16_u (ptr, ecb_host_to_be_u16 (v)); }
+ecb_inline void ecb_poke_be_u32_u (void *ptr, uint_fast32_t v) { ecb_poke_u32_u (ptr, ecb_host_to_be_u32 (v)); }
+ecb_inline void ecb_poke_be_u64_u (void *ptr, uint_fast64_t v) { ecb_poke_u64_u (ptr, ecb_host_to_be_u64 (v)); }
+
+ecb_inline void ecb_poke_le_u16_u (void *ptr, uint_fast16_t v) { ecb_poke_u16_u (ptr, ecb_host_to_le_u16 (v)); }
+ecb_inline void ecb_poke_le_u32_u (void *ptr, uint_fast32_t v) { ecb_poke_u32_u (ptr, ecb_host_to_le_u32 (v)); }
+ecb_inline void ecb_poke_le_u64_u (void *ptr, uint_fast64_t v) { ecb_poke_u64_u (ptr, ecb_host_to_le_u64 (v)); }
+
+#if ECB_CPP
+
+inline uint8_t ecb_bswap (uint8_t v) { return v; }
+inline uint16_t ecb_bswap (uint16_t v) { return ecb_bswap16 (v); }
+inline uint32_t ecb_bswap (uint32_t v) { return ecb_bswap32 (v); }
+inline uint64_t ecb_bswap (uint64_t v) { return ecb_bswap64 (v); }
+
+template<typename T> inline T ecb_be_to_host (T v) { return ecb_little_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_le_to_host (T v) { return ecb_big_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_peek (const void *ptr) { return *(const T *)ptr; }
+template<typename T> inline T ecb_peek_be (const void *ptr) { return ecb_be_to_host (ecb_peek <T> (ptr)); }
+template<typename T> inline T ecb_peek_le (const void *ptr) { return ecb_le_to_host (ecb_peek <T> (ptr)); }
+template<typename T> inline T ecb_peek_u (const void *ptr) { T v; memcpy (&v, ptr, sizeof (v)); return v; }
+template<typename T> inline T ecb_peek_be_u (const void *ptr) { return ecb_be_to_host (ecb_peek_u<T> (ptr)); }
+template<typename T> inline T ecb_peek_le_u (const void *ptr) { return ecb_le_to_host (ecb_peek_u<T> (ptr)); }
+
+template<typename T> inline T ecb_host_to_be (T v) { return ecb_little_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_host_to_le (T v) { return ecb_big_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline void ecb_poke (void *ptr, T v) { *(T *)ptr = v; }
+template<typename T> inline void ecb_poke_be (void *ptr, T v) { return ecb_poke <T> (ptr, ecb_host_to_be (v)); }
+template<typename T> inline void ecb_poke_le (void *ptr, T v) { return ecb_poke <T> (ptr, ecb_host_to_le (v)); }
+template<typename T> inline void ecb_poke_u (void *ptr, T v) { memcpy (ptr, &v, sizeof (v)); }
+template<typename T> inline void ecb_poke_be_u (void *ptr, T v) { return ecb_poke_u<T> (ptr, ecb_host_to_be (v)); }
+template<typename T> inline void ecb_poke_le_u (void *ptr, T v) { return ecb_poke_u<T> (ptr, ecb_host_to_le (v)); }
+
+#endif
+
+/*****************************************************************************/
+
+#if ECB_GCC_VERSION(3,0) || ECB_C99
+ #define ecb_mod(m,n) ((m) % (n) + ((m) % (n) < 0 ? (n) : 0))
+#else
+ #define ecb_mod(m,n) ((m) < 0 ? ((n) - 1 - ((-1 - (m)) % (n))) : ((m) % (n)))
+#endif
+
+#if ECB_CPP
+ template<typename T>
+ static inline T ecb_div_rd (T val, T div)
+ {
+ return val < 0 ? - ((-val + div - 1) / div) : (val ) / div;
+ }
+ template<typename T>
+ static inline T ecb_div_ru (T val, T div)
+ {
+ return val < 0 ? - ((-val ) / div) : (val + div - 1) / div;
+ }
+#else
+ #define ecb_div_rd(val,div) ((val) < 0 ? - ((-(val) + (div) - 1) / (div)) : ((val) ) / (div))
+ #define ecb_div_ru(val,div) ((val) < 0 ? - ((-(val) ) / (div)) : ((val) + (div) - 1) / (div))
+#endif
+
+#if ecb_cplusplus_does_not_suck
+ /* does not work for local types (http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2657.htm) */
+ template<typename T, int N>
+ static inline int ecb_array_length (const T (&arr)[N])
+ {
+ return N;
+ }
+#else
+ #define ecb_array_length(name) (sizeof (name) / sizeof (name [0]))
+#endif
+
+/*****************************************************************************/
+
+ecb_function_ ecb_const uint32_t ecb_binary16_to_binary32 (uint32_t x);
+ecb_function_ ecb_const uint32_t
+ecb_binary16_to_binary32 (uint32_t x)
+{
+ unsigned int s = (x & 0x8000) << (31 - 15);
+ int e = (x >> 10) & 0x001f;
+ unsigned int m = x & 0x03ff;
+
+ if (ecb_expect_false (e == 31))
+ /* infinity or NaN */
+ e = 255 - (127 - 15);
+ else if (ecb_expect_false (!e))
+ {
+ if (ecb_expect_true (!m))
+ /* zero, handled by code below by forcing e to 0 */
+ e = 0 - (127 - 15);
+ else
+ {
+ /* subnormal, renormalise */
+ unsigned int s = 10 - ecb_ld32 (m);
+
+ m = (m << s) & 0x3ff; /* mask implicit bit */
+ e -= s - 1;
+ }
+ }
+
+ /* e and m now are normalised, or zero, (or inf or nan) */
+ e += 127 - 15;
+
+ return s | (e << 23) | (m << (23 - 10));
+}
+
+ecb_function_ ecb_const uint16_t ecb_binary32_to_binary16 (uint32_t x);
+ecb_function_ ecb_const uint16_t
+ecb_binary32_to_binary16 (uint32_t x)
+{
+ unsigned int s = (x >> 16) & 0x00008000; /* sign bit, the easy part */
+ unsigned int e = ((x >> 23) & 0x000000ff) - (127 - 15); /* the desired exponent */
+ unsigned int m = x & 0x007fffff;
+
+ x &= 0x7fffffff;
+
+ /* if it's within range of binary16 normals, use fast path */
+ if (ecb_expect_true (0x38800000 <= x && x <= 0x477fefff))
+ {
+ /* mantissa round-to-even */
+ m += 0x00000fff + ((m >> (23 - 10)) & 1);
+
+ /* handle overflow */
+ if (ecb_expect_false (m >= 0x00800000))
+ {
+ m >>= 1;
+ e += 1;
+ }
+
+ return s | (e << 10) | (m >> (23 - 10));
+ }
+
+ /* handle large numbers and infinity */
+ if (ecb_expect_true (0x477fefff < x && x <= 0x7f800000))
+ return s | 0x7c00;
+
+ /* handle zero, subnormals and small numbers */
+ if (ecb_expect_true (x < 0x38800000))
+ {
+ /* zero */
+ if (ecb_expect_true (!x))
+ return s;
+
+ /* handle subnormals */
+
+ /* too small, will be zero */
+ if (e < (14 - 24)) /* might not be sharp, but is good enough */
+ return s;
+
+ m |= 0x00800000; /* make implicit bit explicit */
+
+ /* very tricky - we need to round to the nearest e (+10) bit value */
+ {
+ unsigned int bits = 14 - e;
+ unsigned int half = (1 << (bits - 1)) - 1;
+ unsigned int even = (m >> bits) & 1;
+
+ /* if this overflows, we will end up with a normalised number */
+ m = (m + half + even) >> bits;
+ }
+
+ return s | m;
+ }
+
+ /* handle NaNs, preserve leftmost nan bits, but make sure we don't turn them into infinities */
+ m >>= 13;
+
+ return s | 0x7c00 | m | !m;
+}
+
+/*******************************************************************************/
+/* floating point stuff, can be disabled by defining ECB_NO_LIBM */
+
+/* basically, everything uses "ieee pure-endian" floating point numbers */
+/* the only noteworthy exception is ancient armle, which uses order 43218765 */
+#if 0 \
+ || __i386 || __i386__ \
+ || ECB_GCC_AMD64 \
+ || __powerpc__ || __ppc__ || __powerpc64__ || __ppc64__ \
+ || defined __s390__ || defined __s390x__ \
+ || defined __mips__ \
+ || defined __alpha__ \
+ || defined __hppa__ \
+ || defined __ia64__ \
+ || defined __m68k__ \
+ || defined __m88k__ \
+ || defined __sh__ \
+ || defined _M_IX86 || defined ECB_MSVC_AMD64 || defined _M_IA64 \
+ || (defined __arm__ && (defined __ARM_EABI__ || defined __EABI__ || defined __VFP_FP__ || defined _WIN32_WCE || defined __ANDROID__)) \
+ || defined __aarch64__
+ #define ECB_STDFP 1
+#else
+ #define ECB_STDFP 0
+#endif
+
+#ifndef ECB_NO_LIBM
+
+ #include <math.h> /* for frexp*, ldexp*, INFINITY, NAN */
+
+ /* only the oldest of old doesn't have this one. solaris. */
+ #ifdef INFINITY
+ #define ECB_INFINITY INFINITY
+ #else
+ #define ECB_INFINITY HUGE_VAL
+ #endif
+
+ #ifdef NAN
+ #define ECB_NAN NAN
+ #else
+ #define ECB_NAN ECB_INFINITY
+ #endif
+
+ #if ECB_C99 || _XOPEN_VERSION >= 600 || _POSIX_VERSION >= 200112L
+ #define ecb_ldexpf(x,e) ldexpf ((x), (e))
+ #define ecb_frexpf(x,e) frexpf ((x), (e))
+ #else
+ #define ecb_ldexpf(x,e) (float) ldexp ((double) (x), (e))
+ #define ecb_frexpf(x,e) (float) frexp ((double) (x), (e))
+ #endif
+
+ /* convert a float to ieee single/binary32 */
+ ecb_function_ ecb_const uint32_t ecb_float_to_binary32 (float x);
+ ecb_function_ ecb_const uint32_t
+ ecb_float_to_binary32 (float x)
+ {
+ uint32_t r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 4);
+ #else
+ /* slow emulation, works for anything but -0 */
+ uint32_t m;
+ int e;
+
+ if (x == 0e0f ) return 0x00000000U;
+ if (x > +3.40282346638528860e+38f) return 0x7f800000U;
+ if (x < -3.40282346638528860e+38f) return 0xff800000U;
+ if (x != x ) return 0x7fbfffffU;
+
+ m = ecb_frexpf (x, &e) * 0x1000000U;
+
+ r = m & 0x80000000U;
+
+ if (r)
+ m = -m;
+
+ if (e <= -126)
+ {
+ m &= 0xffffffU;
+ m >>= (-125 - e);
+ e = -126;
+ }
+
+ r |= (e + 126) << 23;
+ r |= m & 0x7fffffU;
+ #endif
+
+ return r;
+ }
+
+ /* converts an ieee single/binary32 to a float */
+ ecb_function_ ecb_const float ecb_binary32_to_float (uint32_t x);
+ ecb_function_ ecb_const float
+ ecb_binary32_to_float (uint32_t x)
+ {
+ float r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 4);
+ #else
+ /* emulation, only works for normals and subnormals and +0 */
+ int neg = x >> 31;
+ int e = (x >> 23) & 0xffU;
+
+ x &= 0x7fffffU;
+
+ if (e)
+ x |= 0x800000U;
+ else
+ e = 1;
+
+ /* we distrust ldexpf a bit and do the 2**-24 scaling by an extra multiply */
+ r = ecb_ldexpf (x * (0.5f / 0x800000U), e - 126);
+
+ r = neg ? -r : r;
+ #endif
+
+ return r;
+ }
+
+ /* convert a double to ieee double/binary64 */
+ ecb_function_ ecb_const uint64_t ecb_double_to_binary64 (double x);
+ ecb_function_ ecb_const uint64_t
+ ecb_double_to_binary64 (double x)
+ {
+ uint64_t r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 8);
+ #else
+ /* slow emulation, works for anything but -0 */
+ uint64_t m;
+ int e;
+
+ if (x == 0e0 ) return 0x0000000000000000U;
+ if (x > +1.79769313486231470e+308) return 0x7ff0000000000000U;
+ if (x < -1.79769313486231470e+308) return 0xfff0000000000000U;
+ if (x != x ) return 0X7ff7ffffffffffffU;
+
+ m = frexp (x, &e) * 0x20000000000000U;
+
+ r = m & 0x8000000000000000;;
+
+ if (r)
+ m = -m;
+
+ if (e <= -1022)
+ {
+ m &= 0x1fffffffffffffU;
+ m >>= (-1021 - e);
+ e = -1022;
+ }
+
+ r |= ((uint64_t)(e + 1022)) << 52;
+ r |= m & 0xfffffffffffffU;
+ #endif
+
+ return r;
+ }
+
+ /* converts an ieee double/binary64 to a double */
+ ecb_function_ ecb_const double ecb_binary64_to_double (uint64_t x);
+ ecb_function_ ecb_const double
+ ecb_binary64_to_double (uint64_t x)
+ {
+ double r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 8);
+ #else
+ /* emulation, only works for normals and subnormals and +0 */
+ int neg = x >> 63;
+ int e = (x >> 52) & 0x7ffU;
+
+ x &= 0xfffffffffffffU;
+
+ if (e)
+ x |= 0x10000000000000U;
+ else
+ e = 1;
+
+ /* we distrust ldexp a bit and do the 2**-53 scaling by an extra multiply */
+ r = ldexp (x * (0.5 / 0x10000000000000U), e - 1022);
+
+ r = neg ? -r : r;
+ #endif
+
+ return r;
+ }
+
+ /* convert a float to ieee half/binary16 */
+ ecb_function_ ecb_const uint16_t ecb_float_to_binary16 (float x);
+ ecb_function_ ecb_const uint16_t
+ ecb_float_to_binary16 (float x)
+ {
+ return ecb_binary32_to_binary16 (ecb_float_to_binary32 (x));
+ }
+
+ /* convert an ieee half/binary16 to float */
+ ecb_function_ ecb_const float ecb_binary16_to_float (uint16_t x);
+ ecb_function_ ecb_const float
+ ecb_binary16_to_float (uint16_t x)
+ {
+ return ecb_binary32_to_float (ecb_binary16_to_binary32 (x));
+ }
+
+#endif
+
+#endif
+
+/* ECB.H END */
+
+#if ECB_MEMORY_FENCE_NEEDS_PTHREADS
+/* if your architecture doesn't need memory fences, e.g. because it is
+ * single-cpu/core, or if you use libev in a project that doesn't use libev
+ * from multiple threads, then you can define ECB_NO_THREADS when compiling
+ * libev, in which cases the memory fences become nops.
+ * alternatively, you can remove this #error and link against libpthread,
+ * which will then provide the memory fences.
+ */
+# error "memory fences not defined for your architecture, please report"
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+# define ECB_MEMORY_FENCE do { } while (0)
+# define ECB_MEMORY_FENCE_ACQUIRE ECB_MEMORY_FENCE
+# define ECB_MEMORY_FENCE_RELEASE ECB_MEMORY_FENCE
+#endif
+
+#define inline_size ecb_inline
+
+#if EV_FEATURE_CODE
+# define inline_speed ecb_inline
+#else
+# define inline_speed ecb_noinline static
+#endif
+
+/*****************************************************************************/
+/* raw syscall wrappers */
+
+#if EV_NEED_SYSCALL
+
+#include <sys/syscall.h>
+
+/*
+ * define some syscall wrappers for common architectures
+ * this is mostly for nice looks during debugging, not performance.
+ * our syscalls return < 0, not == -1, on error. which is good
+ * enough for linux aio.
+ * TODO: arm is also common nowadays, maybe even mips and x86
+ * TODO: after implementing this, it suddenly looks like overkill, but its hard to remove...
+ */
+#if __GNUC__ && __linux && ECB_AMD64 && !EV_FEATURE_CODE
+ /* the costly errno access probably kills this for size optimisation */
+
+ #define ev_syscall(nr,narg,arg1,arg2,arg3,arg4,arg5,arg6) \
+ ({ \
+ long res; \
+ register unsigned long r6 __asm__ ("r9" ); \
+ register unsigned long r5 __asm__ ("r8" ); \
+ register unsigned long r4 __asm__ ("r10"); \
+ register unsigned long r3 __asm__ ("rdx"); \
+ register unsigned long r2 __asm__ ("rsi"); \
+ register unsigned long r1 __asm__ ("rdi"); \
+ if (narg >= 6) r6 = (unsigned long)(arg6); \
+ if (narg >= 5) r5 = (unsigned long)(arg5); \
+ if (narg >= 4) r4 = (unsigned long)(arg4); \
+ if (narg >= 3) r3 = (unsigned long)(arg3); \
+ if (narg >= 2) r2 = (unsigned long)(arg2); \
+ if (narg >= 1) r1 = (unsigned long)(arg1); \
+ __asm__ __volatile__ ( \
+ "syscall\n\t" \
+ : "=a" (res) \
+ : "0" (nr), "r" (r1), "r" (r2), "r" (r3), "r" (r4), "r" (r5) \
+ : "cc", "r11", "cx", "memory"); \
+ errno = -res; \
+ res; \
+ })
+
+#endif
+
+#ifdef ev_syscall
+ #define ev_syscall0(nr) ev_syscall (nr, 0, 0, 0, 0, 0, 0, 0)
+ #define ev_syscall1(nr,arg1) ev_syscall (nr, 1, arg1, 0, 0, 0, 0, 0)
+ #define ev_syscall2(nr,arg1,arg2) ev_syscall (nr, 2, arg1, arg2, 0, 0, 0, 0)
+ #define ev_syscall3(nr,arg1,arg2,arg3) ev_syscall (nr, 3, arg1, arg2, arg3, 0, 0, 0)
+ #define ev_syscall4(nr,arg1,arg2,arg3,arg4) ev_syscall (nr, 3, arg1, arg2, arg3, arg4, 0, 0)
+ #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5) ev_syscall (nr, 5, arg1, arg2, arg3, arg4, arg5, 0)
+ #define ev_syscall6(nr,arg1,arg2,arg3,arg4,arg5,arg6) ev_syscall (nr, 6, arg1, arg2, arg3, arg4, arg5,arg6)
+#else
+ #define ev_syscall0(nr) syscall (nr)
+ #define ev_syscall1(nr,arg1) syscall (nr, arg1)
+ #define ev_syscall2(nr,arg1,arg2) syscall (nr, arg1, arg2)
+ #define ev_syscall3(nr,arg1,arg2,arg3) syscall (nr, arg1, arg2, arg3)
+ #define ev_syscall4(nr,arg1,arg2,arg3,arg4) syscall (nr, arg1, arg2, arg3, arg4)
+ #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5) syscall (nr, arg1, arg2, arg3, arg4, arg5)
+ #define ev_syscall6(nr,arg1,arg2,arg3,arg4,arg5,arg6) syscall (nr, arg1, arg2, arg3, arg4, arg5,arg6)
+#endif
+
+#endif
+
+/*****************************************************************************/
+
+#define NUMPRI (EV_MAXPRI - EV_MINPRI + 1)
+
+#if EV_MINPRI == EV_MAXPRI
+# define ABSPRI(w) (((W)w), 0)
+#else
+# define ABSPRI(w) (((W)w)->priority - EV_MINPRI)
+#endif
+
+#define EMPTY /* required for microsofts broken pseudo-c compiler */
+
+typedef ev_watcher *W;
+typedef ev_watcher_list *WL;
+typedef ev_watcher_time *WT;
+
+#define ev_active(w) ((W)(w))->active
+#define ev_at(w) ((WT)(w))->at
+
+#if EV_USE_REALTIME
+/* sig_atomic_t is used to avoid per-thread variables or locking but still */
+/* giving it a reasonably high chance of working on typical architectures */
+static EV_ATOMIC_T have_realtime; /* did clock_gettime (CLOCK_REALTIME) work? */
+#endif
+
+#if EV_USE_MONOTONIC
+static EV_ATOMIC_T have_monotonic; /* did clock_gettime (CLOCK_MONOTONIC) work? */
+static EV_ATOMIC_T monotonic_clock_id;
+#endif
+static EV_ATOMIC_T have_cheap_timer = 0;
+
+#ifndef EV_FD_TO_WIN32_HANDLE
+# define EV_FD_TO_WIN32_HANDLE(fd) _get_osfhandle (fd)
+#endif
+#ifndef EV_WIN32_HANDLE_TO_FD
+# define EV_WIN32_HANDLE_TO_FD(handle) _open_osfhandle (handle, 0)
+#endif
+#ifndef EV_WIN32_CLOSE_FD
+# define EV_WIN32_CLOSE_FD(fd) close (fd)
+#endif
+
+#ifdef _WIN32
+# include "ev_win32.c"
+#endif
+
+/*****************************************************************************/
+
+#if EV_USE_LINUXAIO
+# include <linux/aio_abi.h> /* probably only needed for aio_context_t */
+#endif
+
+/* define a suitable floor function (only used by periodics atm) */
+
+#if EV_USE_FLOOR
+# include <math.h>
+# define ev_floor(v) floor (v)
+#else
+
+#include <float.h>
+
+/* a floor() replacement function, should be independent of ev_tstamp type */
+ecb_noinline
+static ev_tstamp
+ev_floor (ev_tstamp v)
+{
+ /* the choice of shift factor is not terribly important */
+#if FLT_RADIX != 2 /* assume FLT_RADIX == 10 */
+ const ev_tstamp shift = sizeof (unsigned long) >= 8 ? 10000000000000000000. : 1000000000.;
+#else
+ const ev_tstamp shift = sizeof (unsigned long) >= 8 ? 18446744073709551616. : 4294967296.;
+#endif
+
+ /* special treatment for negative arguments */
+ if (ecb_expect_false (v < 0.))
+ {
+ ev_tstamp f = -ev_floor (-v);
+
+ return f - (f == v ? 0 : 1);
+ }
+
+ /* argument too large for an unsigned long? then reduce it */
+ if (ecb_expect_false (v >= shift))
+ {
+ ev_tstamp f;
+
+ if (v == v - 1.)
+ return v; /* very large numbers are assumed to be integer */
+
+ f = shift * ev_floor (v * (1. / shift));
+ return f + ev_floor (v - f);
+ }
+
+ /* fits into an unsigned long */
+ return (unsigned long)v;
+}
+
+#endif
+
+/*****************************************************************************/
+
+#ifdef __linux
+# include <sys/utsname.h>
+#endif
+
+ecb_noinline ecb_cold
+static unsigned int
+ev_linux_version (void)
+{
+#ifdef __linux
+ unsigned int v = 0;
+ struct utsname buf;
+ int i;
+ char *p = buf.release;
+
+ if (uname (&buf))
+ return 0;
+
+ for (i = 3+1; --i; )
+ {
+ unsigned int c = 0;
+
+ for (;;)
+ {
+ if (*p >= '0' && *p <= '9')
+ c = c * 10 + *p++ - '0';
+ else
+ {
+ p += *p == '.';
+ break;
+ }
+ }
+
+ v = (v << 8) | c;
+ }
+
+ return v;
+#else
+ return 0;
+#endif
+}
+
+/*****************************************************************************/
+
+#if EV_AVOID_STDIO
+ecb_noinline ecb_cold
+static void
+ev_printerr (const char *msg)
+{
+ write (STDERR_FILENO, msg, strlen (msg));
+}
+#endif
+
+static void (*syserr_cb)(const char *msg) EV_NOEXCEPT;
+
+ecb_cold
+void
+ev_set_syserr_cb (void (*cb)(const char *msg) EV_NOEXCEPT) EV_NOEXCEPT
+{
+ syserr_cb = cb;
+}
+
+ecb_noinline ecb_cold
+static void
+ev_syserr (const char *msg)
+{
+ if (!msg)
+ msg = "(libev) system error";
+
+ if (syserr_cb)
+ syserr_cb (msg);
+ else
+ {
+#if EV_AVOID_STDIO
+ ev_printerr (msg);
+ ev_printerr (": ");
+ ev_printerr (strerror (errno));
+ ev_printerr ("\n");
+#else
+ perror (msg);
+#endif
+ abort ();
+ }
+}
+
+static void *
+ev_realloc_emul (void *ptr, long size) EV_NOEXCEPT
+{
+ /* some systems, notably openbsd and darwin, fail to properly
+ * implement realloc (x, 0) (as required by both ansi c-89 and
+ * the single unix specification, so work around them here.
+ * recently, also (at least) fedora and debian started breaking it,
+ * despite documenting it otherwise.
+ */
+
+ if (size)
+ return realloc (ptr, size);
+
+ free (ptr);
+ return 0;
+}
+
+static void *(*alloc)(void *ptr, long size) EV_NOEXCEPT = ev_realloc_emul;
+
+ecb_cold
+void
+ev_set_allocator (void *(*cb)(void *ptr, long size) EV_NOEXCEPT) EV_NOEXCEPT
+{
+ alloc = cb;
+}
+
+inline_speed void *
+ev_realloc (void *ptr, long size)
+{
+ ptr = alloc (ptr, size);
+
+ if (!ptr && size)
+ {
+#if EV_AVOID_STDIO
+ ev_printerr ("(libev) memory allocation failed, aborting.\n");
+#else
+ fprintf (stderr, "(libev) cannot allocate %ld bytes, aborting.", size);
+#endif
+ abort ();
+ }
+
+ return ptr;
+}
+
+#define ev_malloc(size) ev_realloc (0, (size))
+#define ev_free(ptr) ev_realloc ((ptr), 0)
+
+/*****************************************************************************/
+
+/* set in reify when reification needed */
+#define EV_ANFD_REIFY 1
+
+/* file descriptor info structure */
+typedef struct
+{
+ WL head;
+ unsigned char events; /* the events watched for */
+ unsigned char reify; /* flag set when this ANFD needs reification (EV_ANFD_REIFY, EV__IOFDSET) */
+ unsigned char emask; /* some backends store the actual kernel mask in here */
+ unsigned char eflags; /* flags field for use by backends */
+#if EV_USE_EPOLL
+ unsigned int egen; /* generation counter to counter epoll bugs */
+#endif
+#if EV_SELECT_IS_WINSOCKET || EV_USE_IOCP
+ SOCKET handle;
+#endif
+#if EV_USE_IOCP
+ OVERLAPPED or, ow;
+#endif
+} ANFD;
+
+/* stores the pending event set for a given watcher */
+typedef struct
+{
+ W w;
+ int events; /* the pending event set for the given watcher */
+} ANPENDING;
+
+#if EV_USE_INOTIFY
+/* hash table entry per inotify-id */
+typedef struct
+{
+ WL head;
+} ANFS;
+#endif
+
+/* Heap Entry */
+#if EV_HEAP_CACHE_AT
+ /* a heap element */
+ typedef struct {
+ ev_tstamp at;
+ WT w;
+ } ANHE;
+
+ #define ANHE_w(he) (he).w /* access watcher, read-write */
+ #define ANHE_at(he) (he).at /* access cached at, read-only */
+ #define ANHE_at_cache(he) (he).at = (he).w->at /* update at from watcher */
+#else
+ /* a heap element */
+ typedef WT ANHE;
+
+ #define ANHE_w(he) (he)
+ #define ANHE_at(he) (he)->at
+ #define ANHE_at_cache(he)
+#endif
+
+#if EV_MULTIPLICITY
+
+ struct ev_loop
+ {
+ ev_tstamp ev_rt_now;
+ #define ev_rt_now ((loop)->ev_rt_now)
+ #define VAR(name,decl) decl;
+ #include "ev_vars.h"
+ #undef VAR
+ };
+ #include "ev_wrap.h"
+
+ static struct ev_loop default_loop_struct;
+ EV_API_DECL struct ev_loop *ev_default_loop_ptr = 0; /* needs to be initialised to make it a definition despite extern */
+
+#else
+
+ EV_API_DECL ev_tstamp ev_rt_now = EV_TS_CONST (0.); /* needs to be initialised to make it a definition despite extern */
+ #define VAR(name,decl) static decl;
+ #include "ev_vars.h"
+ #undef VAR
+
+ static int ev_default_loop_ptr;
+
+#endif
+
+#if EV_FEATURE_API
+# define EV_RELEASE_CB if (ecb_expect_false (release_cb)) release_cb (EV_A)
+# define EV_ACQUIRE_CB if (ecb_expect_false (acquire_cb)) acquire_cb (EV_A)
+# define EV_INVOKE_PENDING invoke_cb (EV_A)
+#else
+# define EV_RELEASE_CB (void)0
+# define EV_ACQUIRE_CB (void)0
+# define EV_INVOKE_PENDING ev_invoke_pending (EV_A)
+#endif
+
+#define EVBREAK_RECURSE 0x80
+
+/*****************************************************************************/
+
+#ifndef EV_HAVE_EV_TIME
+ev_tstamp
+ev_time (void) EV_NOEXCEPT
+{
+#if EV_USE_REALTIME
+ if (ecb_expect_true (have_realtime))
+ {
+ struct timespec ts;
+ clock_gettime (CLOCK_REALTIME, &ts);
+ return EV_TS_GET (ts);
+ }
+#endif
+
+ {
+ struct timeval tv;
+ gettimeofday (&tv, 0);
+ return EV_TV_GET (tv);
+ }
+}
+#endif
+
+inline_size ev_tstamp
+get_clock (void)
+{
+#if EV_USE_MONOTONIC
+ if (ecb_expect_true (have_monotonic))
+ {
+ struct timespec ts;
+ clock_gettime (monotonic_clock_id, &ts);
+ return ((ev_tstamp)ts.tv_sec) + ts.tv_nsec * 1e-9;
+ }
+#endif
+
+ return ev_time ();
+}
+
+#if EV_MULTIPLICITY
+ev_tstamp
+ev_now (EV_P) EV_NOEXCEPT
+{
+ return ev_rt_now;
+}
+#endif
+
+void
+ev_sleep (ev_tstamp delay) EV_NOEXCEPT
+{
+ if (delay > EV_TS_CONST (0.))
+ {
+#if EV_USE_NANOSLEEP
+ struct timespec ts;
+
+ EV_TS_SET (ts, delay);
+ nanosleep (&ts, 0);
+#elif defined _WIN32
+ /* maybe this should round up, as ms is very low resolution */
+ /* compared to select (µs) or nanosleep (ns) */
+ Sleep ((unsigned long)(EV_TS_TO_MSEC (delay)));
+#else
+ struct timeval tv;
+
+ /* here we rely on sys/time.h + sys/types.h + unistd.h providing select */
+ /* something not guaranteed by newer posix versions, but guaranteed */
+ /* by older ones */
+ EV_TV_SET (tv, delay);
+ select (0, 0, 0, 0, &tv);
+#endif
+ }
+}
+
+/*****************************************************************************/
+
+#define MALLOC_ROUND 4096 /* prefer to allocate in chunks of this size, must be 2**n and >> 4 longs */
+
+/* find a suitable new size for the given array, */
+/* hopefully by rounding to a nice-to-malloc size */
+inline_size int
+array_nextsize (int elem, int cur, int cnt)
+{
+ int ncur = cur + 1;
+
+ do
+ ncur <<= 1;
+ while (cnt > ncur);
+
+ /* if size is large, round to MALLOC_ROUND - 4 * longs to accommodate malloc overhead */
+ if (elem * ncur > MALLOC_ROUND - sizeof (void *) * 4)
+ {
+ ncur *= elem;
+ ncur = (ncur + elem + (MALLOC_ROUND - 1) + sizeof (void *) * 4) & ~(MALLOC_ROUND - 1);
+ ncur = ncur - sizeof (void *) * 4;
+ ncur /= elem;
+ }
+
+ return ncur;
+}
+
+ecb_noinline ecb_cold
+static void *
+array_realloc (int elem, void *base, int *cur, int cnt)
+{
+ *cur = array_nextsize (elem, *cur, cnt);
+ return ev_realloc (base, elem * *cur);
+}
+
+#define array_needsize_noinit(base,offset,count)
+
+#define array_needsize_zerofill(base,offset,count) \
+ memset ((void *)(base + offset), 0, sizeof (*(base)) * (count))
+
+#define array_needsize(type,base,cur,cnt,init) \
+ if (ecb_expect_false ((cnt) > (cur))) \
+ { \
+ ecb_unused int ocur_ = (cur); \
+ (base) = (type *)array_realloc \
+ (sizeof (type), (base), &(cur), (cnt)); \
+ init ((base), ocur_, ((cur) - ocur_)); \
+ }
+
+#if 0
+#define array_slim(type,stem) \
+ if (stem ## max < array_roundsize (stem ## cnt >> 2)) \
+ { \
+ stem ## max = array_roundsize (stem ## cnt >> 1); \
+ base = (type *)ev_realloc (base, sizeof (type) * (stem ## max));\
+ fprintf (stderr, "slimmed down " # stem " to %d\n", stem ## max);/*D*/\
+ }
+#endif
+
+#define array_free(stem, idx) \
+ ev_free (stem ## s idx); stem ## cnt idx = stem ## max idx = 0; stem ## s idx = 0
+
+/*****************************************************************************/
+
+/* dummy callback for pending events */
+ecb_noinline
+static void
+pendingcb (EV_P_ ev_prepare *w, int revents)
+{
+}
+
+ecb_noinline
+void
+ev_feed_event (EV_P_ void *w, int revents) EV_NOEXCEPT
+{
+ W w_ = (W)w;
+ int pri = ABSPRI (w_);
+
+ if (ecb_expect_false (w_->pending))
+ pendings [pri][w_->pending - 1].events |= revents;
+ else
+ {
+ w_->pending = ++pendingcnt [pri];
+ array_needsize (ANPENDING, pendings [pri], pendingmax [pri], w_->pending, array_needsize_noinit);
+ pendings [pri][w_->pending - 1].w = w_;
+ pendings [pri][w_->pending - 1].events = revents;
+ }
+
+ pendingpri = NUMPRI - 1;
+}
+
+inline_speed void
+feed_reverse (EV_P_ W w)
+{
+ array_needsize (W, rfeeds, rfeedmax, rfeedcnt + 1, array_needsize_noinit);
+ rfeeds [rfeedcnt++] = w;
+}
+
+inline_size void
+feed_reverse_done (EV_P_ int revents)
+{
+ do
+ ev_feed_event (EV_A_ rfeeds [--rfeedcnt], revents);
+ while (rfeedcnt);
+}
+
+inline_speed void
+queue_events (EV_P_ W *events, int eventcnt, int type)
+{
+ int i;
+
+ for (i = 0; i < eventcnt; ++i)
+ ev_feed_event (EV_A_ events [i], type);
+}
+
+/*****************************************************************************/
+
+inline_speed void
+fd_event_nocheck (EV_P_ int fd, int revents)
+{
+ ANFD *anfd = anfds + fd;
+ ev_io *w;
+
+ for (w = (ev_io *)anfd->head; w; w = (ev_io *)((WL)w)->next)
+ {
+ int ev = w->events & revents;
+
+ if (ev)
+ ev_feed_event (EV_A_ (W)w, ev);
+ }
+}
+
+/* do not submit kernel events for fds that have reify set */
+/* because that means they changed while we were polling for new events */
+inline_speed void
+fd_event (EV_P_ int fd, int revents)
+{
+ ANFD *anfd = anfds + fd;
+
+ if (ecb_expect_true (!anfd->reify))
+ fd_event_nocheck (EV_A_ fd, revents);
+}
+
+void
+ev_feed_fd_event (EV_P_ int fd, int revents) EV_NOEXCEPT
+{
+ if (fd >= 0 && fd < anfdmax)
+ fd_event_nocheck (EV_A_ fd, revents);
+}
+
+/* make sure the external fd watch events are in-sync */
+/* with the kernel/libev internal state */
+inline_size void
+fd_reify (EV_P)
+{
+ int i;
+
+ /* most backends do not modify the fdchanges list in backend_modfiy.
+ * except io_uring, which has fixed-size buffers which might force us
+ * to handle events in backend_modify, causing fdchanges to be amended,
+ * which could result in an endless loop.
+ * to avoid this, we do not dynamically handle fds that were added
+ * during fd_reify. that means that for those backends, fdchangecnt
+ * might be non-zero during poll, which must cause them to not block.
+ * to not put too much of a burden on other backends, this detail
+ * needs to be handled in the backend.
+ */
+ int changecnt = fdchangecnt;
+
+#if EV_SELECT_IS_WINSOCKET || EV_USE_IOCP
+ for (i = 0; i < changecnt; ++i)
+ {
+ int fd = fdchanges [i];
+ ANFD *anfd = anfds + fd;
+
+ if (anfd->reify & EV__IOFDSET && anfd->head)
+ {
+ SOCKET handle = EV_FD_TO_WIN32_HANDLE (fd);
+
+ if (handle != anfd->handle)
+ {
+ unsigned long arg;
+
+ assert (("libev: only socket fds supported in this configuration", ioctlsocket (handle, FIONREAD, &arg) == 0));
+
+ /* handle changed, but fd didn't - we need to do it in two steps */
+ backend_modify (EV_A_ fd, anfd->events, 0);
+ anfd->events = 0;
+ anfd->handle = handle;
+ }
+ }
+ }
+#endif
+
+ for (i = 0; i < changecnt; ++i)
+ {
+ int fd = fdchanges [i];
+ ANFD *anfd = anfds + fd;
+ ev_io *w;
+
+ unsigned char o_events = anfd->events;
+ unsigned char o_reify = anfd->reify;
+
+ anfd->reify = 0;
+
+ /*if (ecb_expect_true (o_reify & EV_ANFD_REIFY)) probably a deoptimisation */
+ {
+ anfd->events = 0;
+
+ for (w = (ev_io *)anfd->head; w; w = (ev_io *)((WL)w)->next)
+ anfd->events |= (unsigned char)w->events;
+
+ if (o_events != anfd->events)
+ o_reify = EV__IOFDSET; /* actually |= */
+ }
+
+ if (o_reify & EV__IOFDSET)
+ backend_modify (EV_A_ fd, o_events, anfd->events);
+ }
+
+ /* normally, fdchangecnt hasn't changed. if it has, then new fds have been added.
+ * this is a rare case (see beginning comment in this function), so we copy them to the
+ * front and hope the backend handles this case.
+ */
+ if (ecb_expect_false (fdchangecnt != changecnt))
+ memmove (fdchanges, fdchanges + changecnt, (fdchangecnt - changecnt) * sizeof (*fdchanges));
+
+ fdchangecnt -= changecnt;
+}
+
+/* something about the given fd changed */
+inline_size
+void
+fd_change (EV_P_ int fd, int flags)
+{
+ unsigned char reify = anfds [fd].reify;
+ anfds [fd].reify = reify | flags;
+
+ if (ecb_expect_true (!reify))
+ {
+ ++fdchangecnt;
+ array_needsize (int, fdchanges, fdchangemax, fdchangecnt, array_needsize_noinit);
+ fdchanges [fdchangecnt - 1] = fd;
+ }
+}
+
+/* the given fd is invalid/unusable, so make sure it doesn't hurt us anymore */
+inline_speed ecb_cold void
+fd_kill (EV_P_ int fd)
+{
+ ev_io *w;
+
+ while ((w = (ev_io *)anfds [fd].head))
+ {
+ ev_io_stop (EV_A_ w);
+ ev_feed_event (EV_A_ (W)w, EV_ERROR | EV_READ | EV_WRITE);
+ }
+}
+
+/* check whether the given fd is actually valid, for error recovery */
+inline_size ecb_cold int
+fd_valid (int fd)
+{
+#ifdef _WIN32
+ return EV_FD_TO_WIN32_HANDLE (fd) != -1;
+#else
+ return fcntl (fd, F_GETFD) != -1;
+#endif
+}
+
+/* called on EBADF to verify fds */
+ecb_noinline ecb_cold
+static void
+fd_ebadf (EV_P)
+{
+ int fd;
+
+ for (fd = 0; fd < anfdmax; ++fd)
+ if (anfds [fd].events)
+ if (!fd_valid (fd) && errno == EBADF)
+ fd_kill (EV_A_ fd);
+}
+
+/* called on ENOMEM in select/poll to kill some fds and retry */
+ecb_noinline ecb_cold
+static void
+fd_enomem (EV_P)
+{
+ int fd;
+
+ for (fd = anfdmax; fd--; )
+ if (anfds [fd].events)
+ {
+ fd_kill (EV_A_ fd);
+ break;
+ }
+}
+
+/* usually called after fork if backend needs to re-arm all fds from scratch */
+ecb_noinline
+static void
+fd_rearm_all (EV_P)
+{
+ int fd;
+
+ for (fd = 0; fd < anfdmax; ++fd)
+ if (anfds [fd].events)
+ {
+ anfds [fd].events = 0;
+ anfds [fd].emask = 0;
+ fd_change (EV_A_ fd, EV__IOFDSET | EV_ANFD_REIFY);
+ }
+}
+
+/* used to prepare libev internal fd's */
+/* this is not fork-safe */
+inline_speed void
+fd_intern (int fd)
+{
+#ifdef _WIN32
+ unsigned long arg = 1;
+ ioctlsocket (EV_FD_TO_WIN32_HANDLE (fd), FIONBIO, &arg);
+#else
+ fcntl (fd, F_SETFD, FD_CLOEXEC);
+ fcntl (fd, F_SETFL, O_NONBLOCK);
+#endif
+}
+
+/*****************************************************************************/
+
+/*
+ * the heap functions want a real array index. array index 0 is guaranteed to not
+ * be in-use at any time. the first heap entry is at array [HEAP0]. DHEAP gives
+ * the branching factor of the d-tree.
+ */
+
+/*
+ * at the moment we allow libev the luxury of two heaps,
+ * a small-code-size 2-heap one and a ~1.5kb larger 4-heap
+ * which is more cache-efficient.
+ * the difference is about 5% with 50000+ watchers.
+ */
+#if EV_USE_4HEAP
+
+#define DHEAP 4
+#define HEAP0 (DHEAP - 1) /* index of first element in heap */
+#define HPARENT(k) ((((k) - HEAP0 - 1) / DHEAP) + HEAP0)
+#define UPHEAP_DONE(p,k) ((p) == (k))
+
+/* away from the root */
+inline_speed void
+downheap (ANHE *heap, int N, int k)
+{
+ ANHE he = heap [k];
+ ANHE *E = heap + N + HEAP0;
+
+ for (;;)
+ {
+ ev_tstamp minat;
+ ANHE *minpos;
+ ANHE *pos = heap + DHEAP * (k - HEAP0) + HEAP0 + 1;
+
+ /* find minimum child */
+ if (ecb_expect_true (pos + DHEAP - 1 < E))
+ {
+ /* fast path */ (minpos = pos + 0), (minat = ANHE_at (*minpos));
+ if ( minat > ANHE_at (pos [1])) (minpos = pos + 1), (minat = ANHE_at (*minpos));
+ if ( minat > ANHE_at (pos [2])) (minpos = pos + 2), (minat = ANHE_at (*minpos));
+ if ( minat > ANHE_at (pos [3])) (minpos = pos + 3), (minat = ANHE_at (*minpos));
+ }
+ else if (pos < E)
+ {
+ /* slow path */ (minpos = pos + 0), (minat = ANHE_at (*minpos));
+ if (pos + 1 < E && minat > ANHE_at (pos [1])) (minpos = pos + 1), (minat = ANHE_at (*minpos));
+ if (pos + 2 < E && minat > ANHE_at (pos [2])) (minpos = pos + 2), (minat = ANHE_at (*minpos));
+ if (pos + 3 < E && minat > ANHE_at (pos [3])) (minpos = pos + 3), (minat = ANHE_at (*minpos));
+ }
+ else
+ break;
+
+ if (ANHE_at (he) <= minat)
+ break;
+
+ heap [k] = *minpos;
+ ev_active (ANHE_w (*minpos)) = k;
+
+ k = minpos - heap;
+ }
+
+ heap [k] = he;
+ ev_active (ANHE_w (he)) = k;
+}
+
+#else /* not 4HEAP */
+
+#define HEAP0 1
+#define HPARENT(k) ((k) >> 1)
+#define UPHEAP_DONE(p,k) (!(p))
+
+/* away from the root */
+inline_speed void
+downheap (ANHE *heap, int N, int k)
+{
+ ANHE he = heap [k];
+
+ for (;;)
+ {
+ int c = k << 1;
+
+ if (c >= N + HEAP0)
+ break;
+
+ c += c + 1 < N + HEAP0 && ANHE_at (heap [c]) > ANHE_at (heap [c + 1])
+ ? 1 : 0;
+
+ if (ANHE_at (he) <= ANHE_at (heap [c]))
+ break;
+
+ heap [k] = heap [c];
+ ev_active (ANHE_w (heap [k])) = k;
+
+ k = c;
+ }
+
+ heap [k] = he;
+ ev_active (ANHE_w (he)) = k;
+}
+#endif
+
+/* towards the root */
+inline_speed void
+upheap (ANHE *heap, int k)
+{
+ ANHE he = heap [k];
+
+ for (;;)
+ {
+ int p = HPARENT (k);
+
+ if (UPHEAP_DONE (p, k) || ANHE_at (heap [p]) <= ANHE_at (he))
+ break;
+
+ heap [k] = heap [p];
+ ev_active (ANHE_w (heap [k])) = k;
+ k = p;
+ }
+
+ heap [k] = he;
+ ev_active (ANHE_w (he)) = k;
+}
+
+/* move an element suitably so it is in a correct place */
+inline_size void
+adjustheap (ANHE *heap, int N, int k)
+{
+ if (k > HEAP0 && ANHE_at (heap [k]) <= ANHE_at (heap [HPARENT (k)]))
+ upheap (heap, k);
+ else
+ downheap (heap, N, k);
+}
+
+/* rebuild the heap: this function is used only once and executed rarely */
+inline_size void
+reheap (ANHE *heap, int N)
+{
+ int i;
+
+ /* we don't use floyds algorithm, upheap is simpler and is more cache-efficient */
+ /* also, this is easy to implement and correct for both 2-heaps and 4-heaps */
+ for (i = 0; i < N; ++i)
+ upheap (heap, i + HEAP0);
+}
+
+/*****************************************************************************/
+
+/* associate signal watchers to a signal */
+typedef struct
+{
+ EV_ATOMIC_T pending;
+#if EV_MULTIPLICITY
+ EV_P;
+#endif
+ WL head;
+} ANSIG;
+
+static ANSIG signals [EV_NSIG - 1];
+
+/*****************************************************************************/
+
+#if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
+
+ecb_noinline ecb_cold
+static void
+evpipe_init (EV_P)
+{
+ if (!ev_is_active (&pipe_w))
+ {
+ int fds [2];
+
+# if EV_USE_EVENTFD
+ fds [0] = -1;
+ fds [1] = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC);
+ if (fds [1] < 0 && errno == EINVAL)
+ fds [1] = eventfd (0, 0);
+
+ if (fds [1] < 0)
+# endif
+ {
+ while (pipe (fds))
+ ev_syserr ("(libev) error creating signal/async pipe");
+
+ fd_intern (fds [0]);
+ }
+
+ evpipe [0] = fds [0];
+
+ if (evpipe [1] < 0)
+ evpipe [1] = fds [1]; /* first call, set write fd */
+ else
+ {
+ /* on subsequent calls, do not change evpipe [1] */
+ /* so that evpipe_write can always rely on its value. */
+ /* this branch does not do anything sensible on windows, */
+ /* so must not be executed on windows */
+
+ dup2 (fds [1], evpipe [1]);
+ close (fds [1]);
+ }
+
+ fd_intern (evpipe [1]);
+
+ ev_io_set (&pipe_w, evpipe [0] < 0 ? evpipe [1] : evpipe [0], EV_READ);
+ ev_io_start (EV_A_ &pipe_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+ }
+}
+
+inline_speed void
+evpipe_write (EV_P_ EV_ATOMIC_T *flag)
+{
+ ECB_MEMORY_FENCE; /* push out the write before this function was called, acquire flag */
+
+ if (ecb_expect_true (*flag))
+ return;
+
+ *flag = 1;
+ ECB_MEMORY_FENCE_RELEASE; /* make sure flag is visible before the wakeup */
+
+ pipe_write_skipped = 1;
+
+ ECB_MEMORY_FENCE; /* make sure pipe_write_skipped is visible before we check pipe_write_wanted */
+
+ if (pipe_write_wanted)
+ {
+ int old_errno;
+
+ pipe_write_skipped = 0;
+ ECB_MEMORY_FENCE_RELEASE;
+
+ old_errno = errno; /* save errno because write will clobber it */
+
+#if EV_USE_EVENTFD
+ if (evpipe [0] < 0)
+ {
+ uint64_t counter = 1;
+ (void) !write (evpipe [1], &counter, sizeof (uint64_t));
+ }
+ else
+#endif
+ {
+#ifdef _WIN32
+ WSABUF buf;
+ DWORD sent;
+ buf.buf = (char *)&buf;
+ buf.len = 1;
+ WSASend (EV_FD_TO_WIN32_HANDLE (evpipe [1]), &buf, 1, &sent, 0, 0, 0);
+#else
+ (void) !write (evpipe [1], &(evpipe [1]), 1);
+#endif
+ }
+
+ errno = old_errno;
+ }
+}
+
+/* called whenever the libev signal pipe */
+/* got some events (signal, async) */
+static void
+pipecb (EV_P_ ev_io *iow, int revents)
+{
+ int i;
+
+ if (revents & EV_READ)
+ {
+#if EV_USE_EVENTFD
+ if (evpipe [0] < 0)
+ {
+ uint64_t counter;
+ (void) !read (evpipe [1], &counter, sizeof (uint64_t));
+ }
+ else
+#endif
+ {
+ char dummy[4];
+#ifdef _WIN32
+ WSABUF buf;
+ DWORD recvd;
+ DWORD flags = 0;
+ buf.buf = dummy;
+ buf.len = sizeof (dummy);
+ WSARecv (EV_FD_TO_WIN32_HANDLE (evpipe [0]), &buf, 1, &recvd, &flags, 0, 0);
+#else
+ (void) !read (evpipe [0], &dummy, sizeof (dummy));
+#endif
+ }
+ }
+
+ pipe_write_skipped = 0;
+
+ ECB_MEMORY_FENCE; /* push out skipped, acquire flags */
+
+#if EV_SIGNAL_ENABLE
+ if (sig_pending)
+ {
+ sig_pending = 0;
+
+ ECB_MEMORY_FENCE;
+
+ for (i = EV_NSIG - 1; i--; )
+ if (ecb_expect_false (signals [i].pending))
+ ev_feed_signal_event (EV_A_ i + 1);
+ }
+#endif
+
+#if EV_ASYNC_ENABLE
+ if (async_pending)
+ {
+ async_pending = 0;
+
+ ECB_MEMORY_FENCE;
+
+ for (i = asynccnt; i--; )
+ if (asyncs [i]->sent)
+ {
+ asyncs [i]->sent = 0;
+ ECB_MEMORY_FENCE_RELEASE;
+ ev_feed_event (EV_A_ asyncs [i], EV_ASYNC);
+ }
+ }
+#endif
+}
+
+/*****************************************************************************/
+
+void
+ev_feed_signal (int signum) EV_NOEXCEPT
+{
+#if EV_MULTIPLICITY
+ EV_P;
+ ECB_MEMORY_FENCE_ACQUIRE;
+ EV_A = signals [signum - 1].loop;
+
+ if (!EV_A)
+ return;
+#endif
+
+ signals [signum - 1].pending = 1;
+ evpipe_write (EV_A_ &sig_pending);
+}
+
+static void
+ev_sighandler (int signum)
+{
+#ifdef _WIN32
+ signal (signum, ev_sighandler);
+#endif
+
+ ev_feed_signal (signum);
+}
+
+ecb_noinline
+void
+ev_feed_signal_event (EV_P_ int signum) EV_NOEXCEPT
+{
+ WL w;
+
+ if (ecb_expect_false (signum <= 0 || signum >= EV_NSIG))
+ return;
+
+ --signum;
+
+#if EV_MULTIPLICITY
+ /* it is permissible to try to feed a signal to the wrong loop */
+ /* or, likely more useful, feeding a signal nobody is waiting for */
+
+ if (ecb_expect_false (signals [signum].loop != EV_A))
+ return;
+#endif
+
+ signals [signum].pending = 0;
+ ECB_MEMORY_FENCE_RELEASE;
+
+ for (w = signals [signum].head; w; w = w->next)
+ ev_feed_event (EV_A_ (W)w, EV_SIGNAL);
+}
+
+#if EV_USE_SIGNALFD
+static void
+sigfdcb (EV_P_ ev_io *iow, int revents)
+{
+ struct signalfd_siginfo si[2], *sip; /* these structs are big */
+
+ for (;;)
+ {
+ ssize_t res = read (sigfd, si, sizeof (si));
+
+ /* not ISO-C, as res might be -1, but works with SuS */
+ for (sip = si; (char *)sip < (char *)si + res; ++sip)
+ ev_feed_signal_event (EV_A_ sip->ssi_signo);
+
+ if (res < (ssize_t)sizeof (si))
+ break;
+ }
+}
+#endif
+
+#endif
+
+/*****************************************************************************/
+
+#if EV_CHILD_ENABLE
+static WL childs [EV_PID_HASHSIZE];
+
+static ev_signal childev;
+
+#ifndef WIFCONTINUED
+# define WIFCONTINUED(status) 0
+#endif
+
+/* handle a single child status event */
+inline_speed void
+child_reap (EV_P_ int chain, int pid, int status)
+{
+ ev_child *w;
+ int traced = WIFSTOPPED (status) || WIFCONTINUED (status);
+
+ for (w = (ev_child *)childs [chain & ((EV_PID_HASHSIZE) - 1)]; w; w = (ev_child *)((WL)w)->next)
+ {
+ if ((w->pid == pid || !w->pid)
+ && (!traced || (w->flags & 1)))
+ {
+ ev_set_priority (w, EV_MAXPRI); /* need to do it *now*, this *must* be the same prio as the signal watcher itself */
+ w->rpid = pid;
+ w->rstatus = status;
+ ev_feed_event (EV_A_ (W)w, EV_CHILD);
+ }
+ }
+}
+
+#ifndef WCONTINUED
+# define WCONTINUED 0
+#endif
+
+/* called on sigchld etc., calls waitpid */
+static void
+childcb (EV_P_ ev_signal *sw, int revents)
+{
+ int pid, status;
+
+ /* some systems define WCONTINUED but then fail to support it (linux 2.4) */
+ if (0 >= (pid = waitpid (-1, &status, WNOHANG | WUNTRACED | WCONTINUED)))
+ if (!WCONTINUED
+ || errno != EINVAL
+ || 0 >= (pid = waitpid (-1, &status, WNOHANG | WUNTRACED)))
+ return;
+
+ /* make sure we are called again until all children have been reaped */
+ /* we need to do it this way so that the callback gets called before we continue */
+ ev_feed_event (EV_A_ (W)sw, EV_SIGNAL);
+
+ child_reap (EV_A_ pid, pid, status);
+ if ((EV_PID_HASHSIZE) > 1)
+ child_reap (EV_A_ 0, pid, status); /* this might trigger a watcher twice, but feed_event catches that */
+}
+
+#endif
+
+/*****************************************************************************/
+
+#if EV_USE_TIMERFD
+
+static void periodics_reschedule (EV_P);
+
+static void
+timerfdcb (EV_P_ ev_io *iow, int revents)
+{
+ struct itimerspec its = { 0 };
+
+ its.it_value.tv_sec = ev_rt_now + (int)MAX_BLOCKTIME2;
+ timerfd_settime (timerfd, TFD_TIMER_ABSTIME | TFD_TIMER_CANCEL_ON_SET, &its, 0);
+
+ ev_rt_now = ev_time ();
+ /* periodics_reschedule only needs ev_rt_now */
+ /* but maybe in the future we want the full treatment. */
+ /*
+ now_floor = EV_TS_CONST (0.);
+ time_update (EV_A_ EV_TSTAMP_HUGE);
+ */
+#if EV_PERIODIC_ENABLE
+ periodics_reschedule (EV_A);
+#endif
+}
+
+ecb_noinline ecb_cold
+static void
+evtimerfd_init (EV_P)
+{
+ if (!ev_is_active (&timerfd_w))
+ {
+ timerfd = timerfd_create (CLOCK_REALTIME, TFD_NONBLOCK | TFD_CLOEXEC);
+
+ if (timerfd >= 0)
+ {
+ fd_intern (timerfd); /* just to be sure */
+
+ ev_io_init (&timerfd_w, timerfdcb, timerfd, EV_READ);
+ ev_set_priority (&timerfd_w, EV_MINPRI);
+ ev_io_start (EV_A_ &timerfd_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+
+ /* (re-) arm timer */
+ timerfdcb (EV_A_ 0, 0);
+ }
+ }
+}
+
+#endif
+
+/*****************************************************************************/
+
+#if EV_USE_IOCP
+# include "ev_iocp.c"
+#endif
+#if EV_USE_PORT
+# include "ev_port.c"
+#endif
+#if EV_USE_KQUEUE
+# include "ev_kqueue.c"
+#endif
+#if EV_USE_EPOLL
+# include "ev_epoll.c"
+#endif
+#if EV_USE_LINUXAIO
+# include "ev_linuxaio.c"
+#endif
+#if EV_USE_IOURING
+# include "ev_iouring.c"
+#endif
+#if EV_USE_POLL
+# include "ev_poll.c"
+#endif
+#if EV_USE_SELECT
+# include "ev_select.c"
+#endif
+
+ecb_cold int
+ev_version_major (void) EV_NOEXCEPT
+{
+ return EV_VERSION_MAJOR;
+}
+
+ecb_cold int
+ev_version_minor (void) EV_NOEXCEPT
+{
+ return EV_VERSION_MINOR;
+}
+
+/* return true if we are running with elevated privileges and should ignore env variables */
+inline_size ecb_cold int
+enable_secure (void)
+{
+#ifdef _WIN32
+ return 0;
+#else
+ return getuid () != geteuid ()
+ || getgid () != getegid ();
+#endif
+}
+
+ecb_cold
+unsigned int
+ev_supported_backends (void) EV_NOEXCEPT
+{
+ unsigned int flags = 0;
+
+ if (EV_USE_PORT ) flags |= EVBACKEND_PORT;
+ if (EV_USE_KQUEUE ) flags |= EVBACKEND_KQUEUE;
+ if (EV_USE_EPOLL ) flags |= EVBACKEND_EPOLL;
+ if (EV_USE_LINUXAIO ) flags |= EVBACKEND_LINUXAIO;
+ if (EV_USE_IOURING && ev_linux_version () >= 0x050601) flags |= EVBACKEND_IOURING; /* 5.6.1+ */
+ if (EV_USE_POLL ) flags |= EVBACKEND_POLL;
+ if (EV_USE_SELECT ) flags |= EVBACKEND_SELECT;
+
+ return flags;
+}
+
+ecb_cold
+unsigned int
+ev_recommended_backends (void) EV_NOEXCEPT
+{
+ unsigned int flags = ev_supported_backends ();
+
+#ifndef __NetBSD__
+ /* kqueue is borked on everything but netbsd apparently */
+ /* it usually doesn't work correctly on anything but sockets and pipes */
+ flags &= ~EVBACKEND_KQUEUE;
+#endif
+#ifdef __APPLE__
+ /* only select works correctly on that "unix-certified" platform */
+ flags &= ~EVBACKEND_KQUEUE; /* horribly broken, even for sockets */
+ flags &= ~EVBACKEND_POLL; /* poll is based on kqueue from 10.5 onwards */
+#endif
+#ifdef __FreeBSD__
+ flags &= ~EVBACKEND_POLL; /* poll return value is unusable (http://forums.freebsd.org/archive/index.php/t-10270.html) */
+#endif
+
+ /* TODO: linuxaio is very experimental */
+#if !EV_RECOMMEND_LINUXAIO
+ flags &= ~EVBACKEND_LINUXAIO;
+#endif
+ /* TODO: linuxaio is super experimental */
+#if !EV_RECOMMEND_IOURING
+ flags &= ~EVBACKEND_IOURING;
+#endif
+
+ return flags;
+}
+
+ecb_cold
+unsigned int
+ev_embeddable_backends (void) EV_NOEXCEPT
+{
+ int flags = EVBACKEND_EPOLL | EVBACKEND_KQUEUE | EVBACKEND_PORT | EVBACKEND_IOURING;
+
+ /* epoll embeddability broken on all linux versions up to at least 2.6.23 */
+ if (ev_linux_version () < 0x020620) /* disable it on linux < 2.6.32 */
+ flags &= ~EVBACKEND_EPOLL;
+
+ /* EVBACKEND_LINUXAIO is theoretically embeddable, but suffers from a performance overhead */
+
+ return flags;
+}
+
+unsigned int
+ev_backend (EV_P) EV_NOEXCEPT
+{
+ return backend;
+}
+
+#if EV_FEATURE_API
+unsigned int
+ev_iteration (EV_P) EV_NOEXCEPT
+{
+ return loop_count;
+}
+
+unsigned int
+ev_depth (EV_P) EV_NOEXCEPT
+{
+ return loop_depth;
+}
+
+void
+ev_set_io_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT
+{
+ io_blocktime = interval;
+}
+
+void
+ev_set_timeout_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT
+{
+ timeout_blocktime = interval;
+}
+
+void
+ev_set_userdata (EV_P_ void *data) EV_NOEXCEPT
+{
+ userdata = data;
+}
+
+void *
+ev_userdata (EV_P) EV_NOEXCEPT
+{
+ return userdata;
+}
+
+void
+ev_set_invoke_pending_cb (EV_P_ ev_loop_callback invoke_pending_cb) EV_NOEXCEPT
+{
+ invoke_cb = invoke_pending_cb;
+}
+
+void
+ev_set_loop_release_cb (EV_P_ void (*release)(EV_P) EV_NOEXCEPT, void (*acquire)(EV_P) EV_NOEXCEPT) EV_NOEXCEPT
+{
+ release_cb = release;
+ acquire_cb = acquire;
+}
+#endif
+
+/* initialise a loop structure, must be zero-initialised */
+ecb_noinline ecb_cold
+static void
+loop_init (EV_P_ unsigned int flags) EV_NOEXCEPT
+{
+ if (!backend)
+ {
+ origflags = flags;
+
+#if EV_USE_REALTIME
+ if (!have_realtime)
+ {
+ struct timespec ts;
+
+ if (!clock_gettime (CLOCK_REALTIME, &ts))
+ have_realtime = 1;
+ }
+#endif
+
+#if EV_USE_MONOTONIC
+ if (!have_monotonic)
+ {
+ struct timespec ts;
+
+ if (!clock_gettime (CLOCK_MONOTONIC, &ts)) {
+ have_monotonic = 1;
+ monotonic_clock_id = CLOCK_MONOTONIC;
+#define CHECK_CLOCK_SOURCE(id) do { \
+ if (!clock_gettime ((id), &ts) && \
+ !clock_getres ((id), &ts)) { \
+ if (ts.tv_sec == 0 && ts.tv_nsec < 10ULL * 1000000) { \
+ monotonic_clock_id = (id); \
+ have_cheap_timer = 1; \
+ } \
+ } \
+} while(0)
+#ifdef CLOCK_MONOTONIC_COARSE
+ CHECK_CLOCK_SOURCE(CLOCK_MONOTONIC_COARSE);
+#elif defined(CLOCK_MONOTONIC_FAST) /* BSD stuff */
+ CHECK_CLOCK_SOURCE(CLOCK_MONOTONIC_FAST);
+#elif defined(CLOCK_MONOTONIC_RAW_APPROX) /* OSX stuff */
+ CHECK_CLOCK_SOURCE(CLOCK_MONOTONIC_RAW_APPROX);
+#endif
+#undef CHECK_CLOCK_SOURCE
+ }
+ }
+#endif
+
+ /* pid check not overridable via env */
+#ifndef _WIN32
+ if (flags & EVFLAG_FORKCHECK)
+ curpid = getpid ();
+#endif
+
+ if (!(flags & EVFLAG_NOENV)
+ && !enable_secure ()
+ && getenv ("LIBEV_FLAGS"))
+ flags = atoi (getenv ("LIBEV_FLAGS"));
+
+ ev_rt_now = ev_time ();
+ mn_now = get_clock ();
+ now_floor = mn_now;
+ rtmn_diff = ev_rt_now - mn_now;
+#if EV_FEATURE_API
+ invoke_cb = ev_invoke_pending;
+#endif
+
+ io_blocktime = 0.;
+ timeout_blocktime = 0.;
+ backend = 0;
+ backend_fd = -1;
+ sig_pending = 0;
+#if EV_ASYNC_ENABLE
+ async_pending = 0;
+#endif
+ pipe_write_skipped = 0;
+ pipe_write_wanted = 0;
+ evpipe [0] = -1;
+ evpipe [1] = -1;
+#if EV_USE_INOTIFY
+ fs_fd = flags & EVFLAG_NOINOTIFY ? -1 : -2;
+#endif
+#if EV_USE_SIGNALFD
+ sigfd = flags & EVFLAG_SIGNALFD ? -2 : -1;
+#endif
+#if EV_USE_TIMERFD
+ timerfd = flags & EVFLAG_NOTIMERFD ? -1 : -2;
+#endif
+
+ if (!(flags & EVBACKEND_MASK))
+ flags |= ev_recommended_backends ();
+
+#if EV_USE_IOCP
+ if (!backend && (flags & EVBACKEND_IOCP )) backend = iocp_init (EV_A_ flags);
+#endif
+#if EV_USE_PORT
+ if (!backend && (flags & EVBACKEND_PORT )) backend = port_init (EV_A_ flags);
+#endif
+#if EV_USE_KQUEUE
+ if (!backend && (flags & EVBACKEND_KQUEUE )) backend = kqueue_init (EV_A_ flags);
+#endif
+#if EV_USE_IOURING
+ if (!backend && (flags & EVBACKEND_IOURING )) backend = iouring_init (EV_A_ flags);
+#endif
+#if EV_USE_LINUXAIO
+ if (!backend && (flags & EVBACKEND_LINUXAIO)) backend = linuxaio_init (EV_A_ flags);
+#endif
+#if EV_USE_EPOLL
+ if (!backend && (flags & EVBACKEND_EPOLL )) backend = epoll_init (EV_A_ flags);
+#endif
+#if EV_USE_POLL
+ if (!backend && (flags & EVBACKEND_POLL )) backend = poll_init (EV_A_ flags);
+#endif
+#if EV_USE_SELECT
+ if (!backend && (flags & EVBACKEND_SELECT )) backend = select_init (EV_A_ flags);
+#endif
+
+ ev_prepare_init (&pending_w, pendingcb);
+
+#if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
+ ev_init (&pipe_w, pipecb);
+ ev_set_priority (&pipe_w, EV_MAXPRI);
+#endif
+ }
+}
+
+EV_INLINE struct ev_loop *
+ev_default_loop_uc_ (void) EV_NOEXCEPT
+{
+ return ev_default_loop_ptr;
+}
+
+EV_INLINE int
+ev_is_default_loop (EV_P) EV_NOEXCEPT
+{
+ return EV_A == EV_DEFAULT_UC;
+}
+
+/* free up a loop structure */
+ecb_cold
+void
+ev_loop_destroy (EV_P)
+{
+ int i;
+
+#if EV_MULTIPLICITY
+ /* mimic free (0) */
+ if (!EV_A)
+ return;
+#endif
+
+#if EV_CLEANUP_ENABLE
+ /* queue cleanup watchers (and execute them) */
+ if (ecb_expect_false (cleanupcnt))
+ {
+ queue_events (EV_A_ (W *)cleanups, cleanupcnt, EV_CLEANUP);
+ EV_INVOKE_PENDING;
+ }
+#endif
+
+#if EV_CHILD_ENABLE
+ if (ev_is_default_loop (EV_A) && ev_is_active (&childev))
+ {
+ ev_ref (EV_A); /* child watcher */
+ ev_signal_stop (EV_A_ &childev);
+ }
+#endif
+
+ if (ev_is_active (&pipe_w))
+ {
+ /*ev_ref (EV_A);*/
+ /*ev_io_stop (EV_A_ &pipe_w);*/
+
+ if (evpipe [0] >= 0) EV_WIN32_CLOSE_FD (evpipe [0]);
+ if (evpipe [1] >= 0) EV_WIN32_CLOSE_FD (evpipe [1]);
+ }
+
+#if EV_USE_SIGNALFD
+ if (ev_is_active (&sigfd_w))
+ close (sigfd);
+#endif
+
+#if EV_USE_TIMERFD
+ if (ev_is_active (&timerfd_w))
+ close (timerfd);
+#endif
+
+#if EV_USE_INOTIFY
+ if (fs_fd >= 0)
+ close (fs_fd);
+#endif
+
+ if (backend_fd >= 0)
+ close (backend_fd);
+
+#if EV_USE_IOCP
+ if (backend == EVBACKEND_IOCP ) iocp_destroy (EV_A);
+#endif
+#if EV_USE_PORT
+ if (backend == EVBACKEND_PORT ) port_destroy (EV_A);
+#endif
+#if EV_USE_KQUEUE
+ if (backend == EVBACKEND_KQUEUE ) kqueue_destroy (EV_A);
+#endif
+#if EV_USE_IOURING
+ if (backend == EVBACKEND_IOURING ) iouring_destroy (EV_A);
+#endif
+#if EV_USE_LINUXAIO
+ if (backend == EVBACKEND_LINUXAIO) linuxaio_destroy (EV_A);
+#endif
+#if EV_USE_EPOLL
+ if (backend == EVBACKEND_EPOLL ) epoll_destroy (EV_A);
+#endif
+#if EV_USE_POLL
+ if (backend == EVBACKEND_POLL ) poll_destroy (EV_A);
+#endif
+#if EV_USE_SELECT
+ if (backend == EVBACKEND_SELECT ) select_destroy (EV_A);
+#endif
+
+ for (i = NUMPRI; i--; )
+ {
+ array_free (pending, [i]);
+#if EV_IDLE_ENABLE
+ array_free (idle, [i]);
+#endif
+ }
+
+ ev_free (anfds); anfds = 0; anfdmax = 0;
+
+ /* have to use the microsoft-never-gets-it-right macro */
+ array_free (rfeed, EMPTY);
+ array_free (fdchange, EMPTY);
+ array_free (timer, EMPTY);
+#if EV_PERIODIC_ENABLE
+ array_free (periodic, EMPTY);
+#endif
+#if EV_FORK_ENABLE
+ array_free (fork, EMPTY);
+#endif
+#if EV_CLEANUP_ENABLE
+ array_free (cleanup, EMPTY);
+#endif
+ array_free (prepare, EMPTY);
+ array_free (check, EMPTY);
+#if EV_ASYNC_ENABLE
+ array_free (async, EMPTY);
+#endif
+
+ backend = 0;
+
+#if EV_MULTIPLICITY
+ if (ev_is_default_loop (EV_A))
+#endif
+ ev_default_loop_ptr = 0;
+#if EV_MULTIPLICITY
+ else
+ ev_free (EV_A);
+#endif
+}
+
+#if EV_USE_INOTIFY
+inline_size void infy_fork (EV_P);
+#endif
+
+inline_size void
+loop_fork (EV_P)
+{
+#if EV_USE_PORT
+ if (backend == EVBACKEND_PORT ) port_fork (EV_A);
+#endif
+#if EV_USE_KQUEUE
+ if (backend == EVBACKEND_KQUEUE ) kqueue_fork (EV_A);
+#endif
+#if EV_USE_IOURING
+ if (backend == EVBACKEND_IOURING ) iouring_fork (EV_A);
+#endif
+#if EV_USE_LINUXAIO
+ if (backend == EVBACKEND_LINUXAIO) linuxaio_fork (EV_A);
+#endif
+#if EV_USE_EPOLL
+ if (backend == EVBACKEND_EPOLL ) epoll_fork (EV_A);
+#endif
+#if EV_USE_INOTIFY
+ infy_fork (EV_A);
+#endif
+
+ if (postfork != 2)
+ {
+ #if EV_USE_SIGNALFD
+ /* surprisingly, nothing needs to be done for signalfd, accoridng to docs, it does the right thing on fork */
+ #endif
+
+ #if EV_USE_TIMERFD
+ if (ev_is_active (&timerfd_w))
+ {
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &timerfd_w);
+
+ close (timerfd);
+ timerfd = -2;
+
+ evtimerfd_init (EV_A);
+ /* reschedule periodics, in case we missed something */
+ ev_feed_event (EV_A_ &timerfd_w, EV_CUSTOM);
+ }
+ #endif
+
+ #if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
+ if (ev_is_active (&pipe_w))
+ {
+ /* pipe_write_wanted must be false now, so modifying fd vars should be safe */
+
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &pipe_w);
+
+ if (evpipe [0] >= 0)
+ EV_WIN32_CLOSE_FD (evpipe [0]);
+
+ evpipe_init (EV_A);
+ /* iterate over everything, in case we missed something before */
+ ev_feed_event (EV_A_ &pipe_w, EV_CUSTOM);
+ }
+ #endif
+ }
+
+ postfork = 0;
+}
+
+#if EV_MULTIPLICITY
+
+ecb_cold
+struct ev_loop *
+ev_loop_new (unsigned int flags) EV_NOEXCEPT
+{
+ EV_P = (struct ev_loop *)ev_malloc (sizeof (struct ev_loop));
+
+ memset (EV_A, 0, sizeof (struct ev_loop));
+ loop_init (EV_A_ flags);
+
+ if (ev_backend (EV_A))
+ return EV_A;
+
+ ev_free (EV_A);
+ return 0;
+}
+
+#endif /* multiplicity */
+
+#if EV_VERIFY
+ecb_noinline ecb_cold
+static void
+verify_watcher (EV_P_ W w)
+{
+ assert (("libev: watcher has invalid priority", ABSPRI (w) >= 0 && ABSPRI (w) < NUMPRI));
+
+ if (w->pending)
+ assert (("libev: pending watcher not on pending queue", pendings [ABSPRI (w)][w->pending - 1].w == w));
+}
+
+ecb_noinline ecb_cold
+static void
+verify_heap (EV_P_ ANHE *heap, int N)
+{
+ int i;
+
+ for (i = HEAP0; i < N + HEAP0; ++i)
+ {
+ assert (("libev: active index mismatch in heap", ev_active (ANHE_w (heap [i])) == i));
+ assert (("libev: heap condition violated", i == HEAP0 || ANHE_at (heap [HPARENT (i)]) <= ANHE_at (heap [i])));
+ assert (("libev: heap at cache mismatch", ANHE_at (heap [i]) == ev_at (ANHE_w (heap [i]))));
+
+ verify_watcher (EV_A_ (W)ANHE_w (heap [i]));
+ }
+}
+
+ecb_noinline ecb_cold
+static void
+array_verify (EV_P_ W *ws, int cnt)
+{
+ while (cnt--)
+ {
+ assert (("libev: active index mismatch", ev_active (ws [cnt]) == cnt + 1));
+ verify_watcher (EV_A_ ws [cnt]);
+ }
+}
+#endif
+
+#if EV_FEATURE_API
+void ecb_cold
+ev_verify (EV_P) EV_NOEXCEPT
+{
+#if EV_VERIFY
+ int i;
+ WL w, w2;
+
+ assert (activecnt >= -1);
+
+ assert (fdchangemax >= fdchangecnt);
+ for (i = 0; i < fdchangecnt; ++i)
+ assert (("libev: negative fd in fdchanges", fdchanges [i] >= 0));
+
+ assert (anfdmax >= 0);
+ for (i = 0; i < anfdmax; ++i)
+ {
+ int j = 0;
+
+ for (w = w2 = anfds [i].head; w; w = w->next)
+ {
+ verify_watcher (EV_A_ (W)w);
+
+ if (j++ & 1)
+ {
+ assert (("libev: io watcher list contains a loop", w != w2));
+ w2 = w2->next;
+ }
+
+ assert (("libev: inactive fd watcher on anfd list", ev_active (w) == 1));
+ assert (("libev: fd mismatch between watcher and anfd", ((ev_io *)w)->fd == i));
+ }
+ }
+
+ assert (timermax >= timercnt);
+ verify_heap (EV_A_ timers, timercnt);
+
+#if EV_PERIODIC_ENABLE
+ assert (periodicmax >= periodiccnt);
+ verify_heap (EV_A_ periodics, periodiccnt);
+#endif
+
+ for (i = NUMPRI; i--; )
+ {
+ assert (pendingmax [i] >= pendingcnt [i]);
+#if EV_IDLE_ENABLE
+ assert (idleall >= 0);
+ assert (idlemax [i] >= idlecnt [i]);
+ array_verify (EV_A_ (W *)idles [i], idlecnt [i]);
+#endif
+ }
+
+#if EV_FORK_ENABLE
+ assert (forkmax >= forkcnt);
+ array_verify (EV_A_ (W *)forks, forkcnt);
+#endif
+
+#if EV_CLEANUP_ENABLE
+ assert (cleanupmax >= cleanupcnt);
+ array_verify (EV_A_ (W *)cleanups, cleanupcnt);
+#endif
+
+#if EV_ASYNC_ENABLE
+ assert (asyncmax >= asynccnt);
+ array_verify (EV_A_ (W *)asyncs, asynccnt);
+#endif
+
+#if EV_PREPARE_ENABLE
+ assert (preparemax >= preparecnt);
+ array_verify (EV_A_ (W *)prepares, preparecnt);
+#endif
+
+#if EV_CHECK_ENABLE
+ assert (checkmax >= checkcnt);
+ array_verify (EV_A_ (W *)checks, checkcnt);
+#endif
+
+# if 0
+#if EV_CHILD_ENABLE
+ for (w = (ev_child *)childs [chain & ((EV_PID_HASHSIZE) - 1)]; w; w = (ev_child *)((WL)w)->next)
+ for (signum = EV_NSIG; signum--; ) if (signals [signum].pending)
+#endif
+# endif
+#endif
+}
+#endif
+
+#if EV_MULTIPLICITY
+ecb_cold
+struct ev_loop *
+#else
+int
+#endif
+ev_default_loop (unsigned int flags) EV_NOEXCEPT
+{
+ if (!ev_default_loop_ptr)
+ {
+#if EV_MULTIPLICITY
+ EV_P = ev_default_loop_ptr = &default_loop_struct;
+#else
+ ev_default_loop_ptr = 1;
+#endif
+
+ loop_init (EV_A_ flags);
+
+ if (ev_backend (EV_A))
+ {
+#if EV_CHILD_ENABLE
+ ev_signal_init (&childev, childcb, SIGCHLD);
+ ev_set_priority (&childev, EV_MAXPRI);
+ ev_signal_start (EV_A_ &childev);
+ ev_unref (EV_A); /* child watcher should not keep loop alive */
+#endif
+ }
+ else
+ ev_default_loop_ptr = 0;
+ }
+
+ return ev_default_loop_ptr;
+}
+
+void
+ev_loop_fork (EV_P) EV_NOEXCEPT
+{
+ postfork = 1;
+}
+
+/*****************************************************************************/
+
+void
+ev_invoke (EV_P_ void *w, int revents)
+{
+ EV_CB_INVOKE ((W)w, revents);
+}
+
+unsigned int
+ev_pending_count (EV_P) EV_NOEXCEPT
+{
+ int pri;
+ unsigned int count = 0;
+
+ for (pri = NUMPRI; pri--; )
+ count += pendingcnt [pri];
+
+ return count;
+}
+
+ecb_noinline
+void
+ev_invoke_pending (EV_P)
+{
+ pendingpri = NUMPRI;
+
+ do
+ {
+ --pendingpri;
+
+ /* pendingpri possibly gets modified in the inner loop */
+ while (pendingcnt [pendingpri])
+ {
+ ANPENDING *p = pendings [pendingpri] + --pendingcnt [pendingpri];
+
+ p->w->pending = 0;
+ EV_CB_INVOKE (p->w, p->events);
+ EV_FREQUENT_CHECK;
+ }
+ }
+ while (pendingpri);
+}
+
+#if EV_IDLE_ENABLE
+/* make idle watchers pending. this handles the "call-idle */
+/* only when higher priorities are idle" logic */
+inline_size void
+idle_reify (EV_P)
+{
+ if (ecb_expect_false (idleall))
+ {
+ int pri;
+
+ for (pri = NUMPRI; pri--; )
+ {
+ if (pendingcnt [pri])
+ break;
+
+ if (idlecnt [pri])
+ {
+ queue_events (EV_A_ (W *)idles [pri], idlecnt [pri], EV_IDLE);
+ break;
+ }
+ }
+ }
+}
+#endif
+
+/* make timers pending */
+inline_size void
+timers_reify (EV_P)
+{
+ EV_FREQUENT_CHECK;
+
+ if (timercnt && ANHE_at (timers [HEAP0]) < mn_now)
+ {
+ do
+ {
+ ev_timer *w = (ev_timer *)ANHE_w (timers [HEAP0]);
+
+ /*assert (("libev: inactive timer on timer heap detected", ev_is_active (w)));*/
+
+ /* first reschedule or stop timer */
+ if (w->repeat)
+ {
+ ev_at (w) += w->repeat;
+ if (ev_at (w) < mn_now)
+ ev_at (w) = mn_now;
+
+ assert (("libev: negative ev_timer repeat value found while processing timers", w->repeat > EV_TS_CONST (0.)));
+
+ ANHE_at_cache (timers [HEAP0]);
+ downheap (timers, timercnt, HEAP0);
+ }
+ else
+ ev_timer_stop (EV_A_ w); /* nonrepeating: stop timer */
+
+ EV_FREQUENT_CHECK;
+ feed_reverse (EV_A_ (W)w);
+ }
+ while (timercnt && ANHE_at (timers [HEAP0]) < mn_now);
+
+ feed_reverse_done (EV_A_ EV_TIMER);
+ }
+}
+
+#if EV_PERIODIC_ENABLE
+
+ecb_noinline
+static void
+periodic_recalc (EV_P_ ev_periodic *w)
+{
+ ev_tstamp interval = w->interval > MIN_INTERVAL ? w->interval : MIN_INTERVAL;
+ ev_tstamp at = w->offset + interval * ev_floor ((ev_rt_now - w->offset) / interval);
+
+ /* the above almost always errs on the low side */
+ while (at <= ev_rt_now)
+ {
+ ev_tstamp nat = at + w->interval;
+
+ /* when resolution fails us, we use ev_rt_now */
+ if (ecb_expect_false (nat == at))
+ {
+ at = ev_rt_now;
+ break;
+ }
+
+ at = nat;
+ }
+
+ ev_at (w) = at;
+}
+
+/* make periodics pending */
+inline_size void
+periodics_reify (EV_P)
+{
+ EV_FREQUENT_CHECK;
+
+ while (periodiccnt && ANHE_at (periodics [HEAP0]) < ev_rt_now)
+ {
+ do
+ {
+ ev_periodic *w = (ev_periodic *)ANHE_w (periodics [HEAP0]);
+
+ /*assert (("libev: inactive timer on periodic heap detected", ev_is_active (w)));*/
+
+ /* first reschedule or stop timer */
+ if (w->reschedule_cb)
+ {
+ ev_at (w) = w->reschedule_cb (w, ev_rt_now);
+
+ assert (("libev: ev_periodic reschedule callback returned time in the past", ev_at (w) >= ev_rt_now));
+
+ ANHE_at_cache (periodics [HEAP0]);
+ downheap (periodics, periodiccnt, HEAP0);
+ }
+ else if (w->interval)
+ {
+ periodic_recalc (EV_A_ w);
+ ANHE_at_cache (periodics [HEAP0]);
+ downheap (periodics, periodiccnt, HEAP0);
+ }
+ else
+ ev_periodic_stop (EV_A_ w); /* nonrepeating: stop timer */
+
+ EV_FREQUENT_CHECK;
+ feed_reverse (EV_A_ (W)w);
+ }
+ while (periodiccnt && ANHE_at (periodics [HEAP0]) < ev_rt_now);
+
+ feed_reverse_done (EV_A_ EV_PERIODIC);
+ }
+}
+
+/* simply recalculate all periodics */
+/* TODO: maybe ensure that at least one event happens when jumping forward? */
+ecb_noinline ecb_cold
+static void
+periodics_reschedule (EV_P)
+{
+ int i;
+
+ /* adjust periodics after time jump */
+ for (i = HEAP0; i < periodiccnt + HEAP0; ++i)
+ {
+ ev_periodic *w = (ev_periodic *)ANHE_w (periodics [i]);
+
+ if (w->reschedule_cb)
+ ev_at (w) = w->reschedule_cb (w, ev_rt_now);
+ else if (w->interval)
+ periodic_recalc (EV_A_ w);
+
+ ANHE_at_cache (periodics [i]);
+ }
+
+ reheap (periodics, periodiccnt);
+}
+#endif
+
+/* adjust all timers by a given offset */
+ecb_noinline ecb_cold
+static void
+timers_reschedule (EV_P_ ev_tstamp adjust)
+{
+ int i;
+
+ for (i = 0; i < timercnt; ++i)
+ {
+ ANHE *he = timers + i + HEAP0;
+ ANHE_w (*he)->at += adjust;
+ ANHE_at_cache (*he);
+ }
+}
+
+/* fetch new monotonic and realtime times from the kernel */
+/* also detect if there was a timejump, and act accordingly */
+inline_speed void
+time_update (EV_P_ ev_tstamp max_block)
+{
+#if EV_USE_MONOTONIC
+ if (ecb_expect_true (have_monotonic))
+ {
+ int i;
+ ev_tstamp odiff = rtmn_diff;
+
+ mn_now = get_clock ();
+
+ /* only fetch the realtime clock every 0.5*MIN_TIMEJUMP seconds */
+ /* interpolate in the meantime */
+ if (ecb_expect_true (mn_now - now_floor < EV_TS_CONST (MIN_TIMEJUMP * .5)))
+ {
+ ev_rt_now = rtmn_diff + mn_now;
+ return;
+ }
+
+ now_floor = mn_now;
+ ev_rt_now = ev_time ();
+
+ /* loop a few times, before making important decisions.
+ * on the choice of "4": one iteration isn't enough,
+ * in case we get preempted during the calls to
+ * ev_time and get_clock. a second call is almost guaranteed
+ * to succeed in that case, though. and looping a few more times
+ * doesn't hurt either as we only do this on time-jumps or
+ * in the unlikely event of having been preempted here.
+ */
+ for (i = 4; --i; )
+ {
+ ev_tstamp diff;
+ rtmn_diff = ev_rt_now - mn_now;
+
+ diff = odiff - rtmn_diff;
+
+ if (ecb_expect_true ((diff < EV_TS_CONST (0.) ? -diff : diff) < EV_TS_CONST (MIN_TIMEJUMP)))
+ return; /* all is well */
+
+ ev_rt_now = ev_time ();
+ mn_now = get_clock ();
+ now_floor = mn_now;
+ }
+
+ /* no timer adjustment, as the monotonic clock doesn't jump */
+ /* timers_reschedule (EV_A_ rtmn_diff - odiff) */
+# if EV_PERIODIC_ENABLE
+ periodics_reschedule (EV_A);
+# endif
+ }
+ else
+#endif
+ {
+ ev_rt_now = ev_time ();
+
+ if (ecb_expect_false (mn_now > ev_rt_now || ev_rt_now > mn_now + max_block + EV_TS_CONST (MIN_TIMEJUMP)))
+ {
+ /* adjust timers. this is easy, as the offset is the same for all of them */
+ timers_reschedule (EV_A_ ev_rt_now - mn_now);
+#if EV_PERIODIC_ENABLE
+ periodics_reschedule (EV_A);
+#endif
+ }
+
+ mn_now = ev_rt_now;
+ }
+}
+
+int
+ev_run (EV_P_ int flags)
+{
+#if EV_FEATURE_API
+ ++loop_depth;
+#endif
+
+ assert (("libev: ev_loop recursion during release detected", loop_done != EVBREAK_RECURSE));
+
+ loop_done = EVBREAK_CANCEL;
+
+ EV_INVOKE_PENDING; /* in case we recurse, ensure ordering stays nice and clean */
+
+ do
+ {
+#if EV_VERIFY >= 2
+ ev_verify (EV_A);
+#endif
+
+#ifndef _WIN32
+ if (ecb_expect_false (curpid)) /* penalise the forking check even more */
+ if (ecb_expect_false (getpid () != curpid))
+ {
+ curpid = getpid ();
+ postfork = 1;
+ }
+#endif
+
+#if EV_FORK_ENABLE
+ /* we might have forked, so queue fork handlers */
+ if (ecb_expect_false (postfork))
+ if (forkcnt)
+ {
+ queue_events (EV_A_ (W *)forks, forkcnt, EV_FORK);
+ EV_INVOKE_PENDING;
+ }
+#endif
+
+#if EV_PREPARE_ENABLE
+ /* queue prepare watchers (and execute them) */
+ if (ecb_expect_false (preparecnt))
+ {
+ queue_events (EV_A_ (W *)prepares, preparecnt, EV_PREPARE);
+ EV_INVOKE_PENDING;
+ }
+#endif
+
+ if (ecb_expect_false (loop_done))
+ break;
+
+ /* we might have forked, so reify kernel state if necessary */
+ if (ecb_expect_false (postfork))
+ loop_fork (EV_A);
+
+ /* update fd-related kernel structures */
+ fd_reify (EV_A);
+
+ /* calculate blocking time */
+ {
+ ev_tstamp waittime = 0.;
+ ev_tstamp sleeptime = 0.;
+
+ /* remember old timestamp for io_blocktime calculation */
+ ev_tstamp prev_mn_now = mn_now;
+
+ /* update time to cancel out callback processing overhead */
+ time_update (EV_A_ EV_TS_CONST (EV_TSTAMP_HUGE));
+
+ /* from now on, we want a pipe-wake-up */
+ pipe_write_wanted = 1;
+
+ ECB_MEMORY_FENCE; /* make sure pipe_write_wanted is visible before we check for potential skips */
+
+ if (ecb_expect_true (!(flags & EVRUN_NOWAIT || idleall || !activecnt || pipe_write_skipped)))
+ {
+ waittime = EV_TS_CONST (MAX_BLOCKTIME);
+#if EV_USE_MONOTONIC
+ if (ecb_expect_true (have_monotonic)) {
+#if EV_USE_TIMERFD
+ /* sleep a lot longer when we can reliably detect timejumps */
+ if (ecb_expect_true (timerfd != -1))
+ waittime = EV_TS_CONST (MAX_BLOCKTIME2);
+#endif
+#if !EV_PERIODIC_ENABLE
+ /* without periodics but with monotonic clock there is no need */
+ /* for any time jump detection, so sleep longer */
+
+ waittime = EV_TS_CONST (MAX_BLOCKTIME2);
+#endif
+ }
+#endif /* EV_USE_MONOTONIC */
+ if (timercnt)
+ {
+ ev_tstamp to = ANHE_at (timers [HEAP0]) - mn_now;
+ if (waittime > to) waittime = to;
+ }
+
+#if EV_PERIODIC_ENABLE
+ if (periodiccnt)
+ {
+ ev_tstamp to = ANHE_at (periodics [HEAP0]) - ev_rt_now;
+ if (waittime > to) waittime = to;
+ }
+#endif
+
+ /* don't let timeouts decrease the waittime below timeout_blocktime */
+ if (ecb_expect_false (waittime < timeout_blocktime))
+ waittime = timeout_blocktime;
+
+ /* now there are two more special cases left, either we have
+ * already-expired timers, so we should not sleep, or we have timers
+ * that expire very soon, in which case we need to wait for a minimum
+ * amount of time for some event loop backends.
+ */
+ if (ecb_expect_false (waittime < backend_mintime))
+ waittime = waittime <= EV_TS_CONST (0.)
+ ? EV_TS_CONST (0.)
+ : backend_mintime;
+
+ /* extra check because io_blocktime is commonly 0 */
+ if (ecb_expect_false (io_blocktime))
+ {
+ sleeptime = io_blocktime - (mn_now - prev_mn_now);
+
+ if (sleeptime > waittime - backend_mintime)
+ sleeptime = waittime - backend_mintime;
+
+ if (ecb_expect_true (sleeptime > EV_TS_CONST (0.)))
+ {
+ ev_sleep (sleeptime);
+ waittime -= sleeptime;
+ }
+ }
+ }
+
+#if EV_FEATURE_API
+ ++loop_count;
+#endif
+ assert ((loop_done = EVBREAK_RECURSE, 1)); /* assert for side effect */
+ backend_poll (EV_A_ waittime);
+ assert ((loop_done = EVBREAK_CANCEL, 1)); /* assert for side effect */
+
+ pipe_write_wanted = 0; /* just an optimisation, no fence needed */
+
+ ECB_MEMORY_FENCE_ACQUIRE;
+ if (pipe_write_skipped)
+ {
+ assert (("libev: pipe_w not active, but pipe not written", ev_is_active (&pipe_w)));
+ ev_feed_event (EV_A_ &pipe_w, EV_CUSTOM);
+ }
+
+ /* update ev_rt_now, do magic */
+ time_update (EV_A_ waittime + sleeptime);
+ }
+
+ /* queue pending timers and reschedule them */
+ timers_reify (EV_A); /* relative timers called last */
+#if EV_PERIODIC_ENABLE
+ periodics_reify (EV_A); /* absolute timers called first */
+#endif
+
+#if EV_IDLE_ENABLE
+ /* queue idle watchers unless other events are pending */
+ idle_reify (EV_A);
+#endif
+
+#if EV_CHECK_ENABLE
+ /* queue check watchers, to be executed first */
+ if (ecb_expect_false (checkcnt))
+ queue_events (EV_A_ (W *)checks, checkcnt, EV_CHECK);
+#endif
+
+ EV_INVOKE_PENDING;
+ }
+ while (ecb_expect_true (
+ activecnt
+ && !loop_done
+ && !(flags & (EVRUN_ONCE | EVRUN_NOWAIT))
+ ));
+
+ if (loop_done == EVBREAK_ONE)
+ loop_done = EVBREAK_CANCEL;
+
+#if EV_FEATURE_API
+ --loop_depth;
+#endif
+
+ return activecnt;
+}
+
+void
+ev_break (EV_P_ int how) EV_NOEXCEPT
+{
+ loop_done = how;
+}
+
+void
+ev_ref (EV_P) EV_NOEXCEPT
+{
+ ++activecnt;
+}
+
+void
+ev_unref (EV_P) EV_NOEXCEPT
+{
+ --activecnt;
+}
+
+void
+ev_now_update (EV_P) EV_NOEXCEPT
+{
+ time_update (EV_A_ EV_TSTAMP_HUGE);
+}
+
+void
+ev_suspend (EV_P) EV_NOEXCEPT
+{
+ ev_now_update (EV_A);
+}
+
+void
+ev_resume (EV_P) EV_NOEXCEPT
+{
+ ev_tstamp mn_prev = mn_now;
+
+ ev_now_update (EV_A);
+ timers_reschedule (EV_A_ mn_now - mn_prev);
+#if EV_PERIODIC_ENABLE
+ /* TODO: really do this? */
+ periodics_reschedule (EV_A);
+#endif
+}
+
+/*****************************************************************************/
+/* singly-linked list management, used when the expected list length is short */
+
+inline_size void
+wlist_add (WL *head, WL elem)
+{
+ elem->next = *head;
+ *head = elem;
+}
+
+inline_size void
+wlist_del (WL *head, WL elem)
+{
+ while (*head)
+ {
+ if (ecb_expect_true (*head == elem))
+ {
+ *head = elem->next;
+ break;
+ }
+
+ head = &(*head)->next;
+ }
+}
+
+/* internal, faster, version of ev_clear_pending */
+inline_speed void
+clear_pending (EV_P_ W w)
+{
+ if (w->pending)
+ {
+ pendings [ABSPRI (w)][w->pending - 1].w = (W)&pending_w;
+ w->pending = 0;
+ }
+}
+
+int
+ev_clear_pending (EV_P_ void *w) EV_NOEXCEPT
+{
+ W w_ = (W)w;
+ int pending = w_->pending;
+
+ if (ecb_expect_true (pending))
+ {
+ ANPENDING *p = pendings [ABSPRI (w_)] + pending - 1;
+ p->w = (W)&pending_w;
+ w_->pending = 0;
+ return p->events;
+ }
+ else
+ return 0;
+}
+
+inline_size void
+pri_adjust (EV_P_ W w)
+{
+ int pri = ev_priority (w);
+ pri = pri < EV_MINPRI ? EV_MINPRI : pri;
+ pri = pri > EV_MAXPRI ? EV_MAXPRI : pri;
+ ev_set_priority (w, pri);
+}
+
+inline_speed void
+ev_start (EV_P_ W w, int active)
+{
+ pri_adjust (EV_A_ w);
+ w->active = active;
+ ev_ref (EV_A);
+}
+
+inline_size void
+ev_stop (EV_P_ W w)
+{
+ ev_unref (EV_A);
+ w->active = 0;
+}
+
+/*****************************************************************************/
+
+ecb_noinline
+void
+ev_io_start (EV_P_ ev_io *w) EV_NOEXCEPT
+{
+ int fd = w->fd;
+
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ assert (("libev: ev_io_start called with negative fd", fd >= 0));
+ assert (("libev: ev_io_start called with illegal event mask", !(w->events & ~(EV__IOFDSET | EV_READ | EV_WRITE))));
+
+#if EV_VERIFY >= 2
+ assert (("libev: ev_io_start called on watcher with invalid fd", fd_valid (fd)));
+#endif
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, 1);
+ array_needsize (ANFD, anfds, anfdmax, fd + 1, array_needsize_zerofill);
+ wlist_add (&anfds[fd].head, (WL)w);
+
+ /* common bug, apparently */
+ assert (("libev: ev_io_start called with corrupted watcher", ((WL)w)->next != (WL)w));
+
+ fd_change (EV_A_ fd, (w->events & EV__IOFDSET) | EV_ANFD_REIFY);
+ w->events &= ~EV__IOFDSET;
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_io_stop (EV_P_ ev_io *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ assert (("libev: ev_io_stop called with illegal fd (must stay constant after start!)", w->fd >= 0 && w->fd < anfdmax));
+
+#if EV_VERIFY >= 2
+ assert (("libev: ev_io_stop called on watcher with invalid fd", fd_valid (w->fd)));
+#endif
+ EV_FREQUENT_CHECK;
+
+ wlist_del (&anfds[w->fd].head, (WL)w);
+ ev_stop (EV_A_ (W)w);
+
+ fd_change (EV_A_ w->fd, EV_ANFD_REIFY);
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_timer_start (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ ev_at (w) += mn_now;
+
+ assert (("libev: ev_timer_start called with negative timer repeat value", w->repeat >= 0.));
+
+ EV_FREQUENT_CHECK;
+
+ ++timercnt;
+ ev_start (EV_A_ (W)w, timercnt + HEAP0 - 1);
+ array_needsize (ANHE, timers, timermax, ev_active (w) + 1, array_needsize_noinit);
+ ANHE_w (timers [ev_active (w)]) = (WT)w;
+ ANHE_at_cache (timers [ev_active (w)]);
+ upheap (timers, ev_active (w));
+
+ EV_FREQUENT_CHECK;
+
+ /*assert (("libev: internal timer heap corruption", timers [ev_active (w)] == (WT)w));*/
+}
+
+ecb_noinline
+void
+ev_timer_stop (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ assert (("libev: internal timer heap corruption", ANHE_w (timers [active]) == (WT)w));
+
+ --timercnt;
+
+ if (ecb_expect_true (active < timercnt + HEAP0))
+ {
+ timers [active] = timers [timercnt + HEAP0];
+ adjustheap (timers, timercnt, active);
+ }
+ }
+
+ ev_at (w) -= mn_now;
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_timer_again (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ EV_FREQUENT_CHECK;
+
+ clear_pending (EV_A_ (W)w);
+
+ if (ev_is_active (w))
+ {
+ if (w->repeat)
+ {
+ ev_at (w) = mn_now + w->repeat;
+ ANHE_at_cache (timers [ev_active (w)]);
+ adjustheap (timers, timercnt, ev_active (w));
+ }
+ else
+ ev_timer_stop (EV_A_ w);
+ }
+ else if (w->repeat)
+ {
+ ev_at (w) = w->repeat;
+ ev_timer_start (EV_A_ w);
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+ev_tstamp
+ev_timer_remaining (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ return ev_at (w) - (ev_is_active (w) ? mn_now : EV_TS_CONST (0.));
+}
+
+#if EV_PERIODIC_ENABLE
+ecb_noinline
+void
+ev_periodic_start (EV_P_ ev_periodic *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+#if EV_USE_TIMERFD
+ if (timerfd == -2)
+ evtimerfd_init (EV_A);
+#endif
+
+ if (w->reschedule_cb)
+ ev_at (w) = w->reschedule_cb (w, ev_rt_now);
+ else if (w->interval)
+ {
+ assert (("libev: ev_periodic_start called with negative interval value", w->interval >= 0.));
+ periodic_recalc (EV_A_ w);
+ }
+ else
+ ev_at (w) = w->offset;
+
+ EV_FREQUENT_CHECK;
+
+ ++periodiccnt;
+ ev_start (EV_A_ (W)w, periodiccnt + HEAP0 - 1);
+ array_needsize (ANHE, periodics, periodicmax, ev_active (w) + 1, array_needsize_noinit);
+ ANHE_w (periodics [ev_active (w)]) = (WT)w;
+ ANHE_at_cache (periodics [ev_active (w)]);
+ upheap (periodics, ev_active (w));
+
+ EV_FREQUENT_CHECK;
+
+ /*assert (("libev: internal periodic heap corruption", ANHE_w (periodics [ev_active (w)]) == (WT)w));*/
+}
+
+ecb_noinline
+void
+ev_periodic_stop (EV_P_ ev_periodic *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ assert (("libev: internal periodic heap corruption", ANHE_w (periodics [active]) == (WT)w));
+
+ --periodiccnt;
+
+ if (ecb_expect_true (active < periodiccnt + HEAP0))
+ {
+ periodics [active] = periodics [periodiccnt + HEAP0];
+ adjustheap (periodics, periodiccnt, active);
+ }
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_periodic_again (EV_P_ ev_periodic *w) EV_NOEXCEPT
+{
+ /* TODO: use adjustheap and recalculation */
+ ev_periodic_stop (EV_A_ w);
+ ev_periodic_start (EV_A_ w);
+}
+#endif
+
+#ifndef SA_RESTART
+# define SA_RESTART 0
+#endif
+
+#if EV_SIGNAL_ENABLE
+
+ecb_noinline
+void
+ev_signal_start (EV_P_ ev_signal *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ assert (("libev: ev_signal_start called with illegal signal number", w->signum > 0 && w->signum < EV_NSIG));
+
+#if EV_MULTIPLICITY
+ assert (("libev: a signal must not be attached to two different loops",
+ !signals [w->signum - 1].loop || signals [w->signum - 1].loop == loop));
+
+ signals [w->signum - 1].loop = EV_A;
+ ECB_MEMORY_FENCE_RELEASE;
+#endif
+
+ EV_FREQUENT_CHECK;
+
+#if EV_USE_SIGNALFD
+ if (sigfd == -2)
+ {
+ sigfd = signalfd (-1, &sigfd_set, SFD_NONBLOCK | SFD_CLOEXEC);
+ if (sigfd < 0 && errno == EINVAL)
+ sigfd = signalfd (-1, &sigfd_set, 0); /* retry without flags */
+
+ if (sigfd >= 0)
+ {
+ fd_intern (sigfd); /* doing it twice will not hurt */
+
+ sigemptyset (&sigfd_set);
+
+ ev_io_init (&sigfd_w, sigfdcb, sigfd, EV_READ);
+ ev_set_priority (&sigfd_w, EV_MAXPRI);
+ ev_io_start (EV_A_ &sigfd_w);
+ ev_unref (EV_A); /* signalfd watcher should not keep loop alive */
+ }
+ }
+
+ if (sigfd >= 0)
+ {
+ /* TODO: check .head */
+ sigaddset (&sigfd_set, w->signum);
+ sigprocmask (SIG_BLOCK, &sigfd_set, 0);
+
+ signalfd (sigfd, &sigfd_set, 0);
+ }
+#endif
+
+ ev_start (EV_A_ (W)w, 1);
+ wlist_add (&signals [w->signum - 1].head, (WL)w);
+
+ if (!((WL)w)->next)
+# if EV_USE_SIGNALFD
+ if (sigfd < 0) /*TODO*/
+# endif
+ {
+# ifdef _WIN32
+ evpipe_init (EV_A);
+
+ signal (w->signum, ev_sighandler);
+# else
+ struct sigaction sa;
+
+ evpipe_init (EV_A);
+
+ sa.sa_handler = ev_sighandler;
+ sigfillset (&sa.sa_mask);
+ sa.sa_flags = SA_RESTART; /* if restarting works we save one iteration */
+ sigaction (w->signum, &sa, 0);
+
+ if (origflags & EVFLAG_NOSIGMASK)
+ {
+ sigemptyset (&sa.sa_mask);
+ sigaddset (&sa.sa_mask, w->signum);
+ sigprocmask (SIG_UNBLOCK, &sa.sa_mask, 0);
+ }
+#endif
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_signal_stop (EV_P_ ev_signal *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ wlist_del (&signals [w->signum - 1].head, (WL)w);
+ ev_stop (EV_A_ (W)w);
+
+ if (!signals [w->signum - 1].head)
+ {
+#if EV_MULTIPLICITY
+ signals [w->signum - 1].loop = 0; /* unattach from signal */
+#endif
+#if EV_USE_SIGNALFD
+ if (sigfd >= 0)
+ {
+ sigset_t ss;
+
+ sigemptyset (&ss);
+ sigaddset (&ss, w->signum);
+ sigdelset (&sigfd_set, w->signum);
+
+ signalfd (sigfd, &sigfd_set, 0);
+ sigprocmask (SIG_UNBLOCK, &ss, 0);
+ }
+ else
+#endif
+ signal (w->signum, SIG_DFL);
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+#endif
+
+#if EV_CHILD_ENABLE
+
+void
+ev_child_start (EV_P_ ev_child *w) EV_NOEXCEPT
+{
+#if EV_MULTIPLICITY
+ assert (("libev: child watchers are only supported in the default loop", loop == ev_default_loop_ptr));
+#endif
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, 1);
+ wlist_add (&childs [w->pid & ((EV_PID_HASHSIZE) - 1)], (WL)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_child_stop (EV_P_ ev_child *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ wlist_del (&childs [w->pid & ((EV_PID_HASHSIZE) - 1)], (WL)w);
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+#endif
+
+#if EV_STAT_ENABLE
+
+# ifdef _WIN32
+# undef lstat
+# define lstat(a,b) _stati64 (a,b)
+# endif
+
+#define DEF_STAT_INTERVAL 5.0074891
+#define NFS_STAT_INTERVAL 30.1074891 /* for filesystems potentially failing inotify */
+#define MIN_STAT_INTERVAL 0.1074891
+
+ecb_noinline static void stat_timer_cb (EV_P_ ev_timer *w_, int revents);
+
+#if EV_USE_INOTIFY
+
+/* the * 2 is to allow for alignment padding, which for some reason is >> 8 */
+# define EV_INOTIFY_BUFSIZE (sizeof (struct inotify_event) * 2 + NAME_MAX)
+
+ecb_noinline
+static void
+infy_add (EV_P_ ev_stat *w)
+{
+ w->wd = inotify_add_watch (fs_fd, w->path,
+ IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF | IN_MODIFY
+ | IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO
+ | IN_DONT_FOLLOW | IN_MASK_ADD);
+
+ if (w->wd >= 0)
+ {
+ struct statfs sfs;
+
+ /* now local changes will be tracked by inotify, but remote changes won't */
+ /* unless the filesystem is known to be local, we therefore still poll */
+ /* also do poll on <2.6.25, but with normal frequency */
+
+ if (!fs_2625)
+ w->timer.repeat = w->interval ? w->interval : DEF_STAT_INTERVAL;
+ else if (!statfs (w->path, &sfs)
+ && (sfs.f_type == 0x1373 /* devfs */
+ || sfs.f_type == 0x4006 /* fat */
+ || sfs.f_type == 0x4d44 /* msdos */
+ || sfs.f_type == 0xEF53 /* ext2/3 */
+ || sfs.f_type == 0x72b6 /* jffs2 */
+ || sfs.f_type == 0x858458f6 /* ramfs */
+ || sfs.f_type == 0x5346544e /* ntfs */
+ || sfs.f_type == 0x3153464a /* jfs */
+ || sfs.f_type == 0x9123683e /* btrfs */
+ || sfs.f_type == 0x52654973 /* reiser3 */
+ || sfs.f_type == 0x01021994 /* tmpfs */
+ || sfs.f_type == 0x58465342 /* xfs */))
+ w->timer.repeat = 0.; /* filesystem is local, kernel new enough */
+ else
+ w->timer.repeat = w->interval ? w->interval : NFS_STAT_INTERVAL; /* remote, use reduced frequency */
+ }
+ else
+ {
+ /* can't use inotify, continue to stat */
+ w->timer.repeat = w->interval ? w->interval : DEF_STAT_INTERVAL;
+
+ /* if path is not there, monitor some parent directory for speedup hints */
+ /* note that exceeding the hardcoded path limit is not a correctness issue, */
+ /* but an efficiency issue only */
+ if ((errno == ENOENT || errno == EACCES) && strlen (w->path) < 4096)
+ {
+ char path [4096];
+ strcpy (path, w->path);
+
+ do
+ {
+ int mask = IN_MASK_ADD | IN_DELETE_SELF | IN_MOVE_SELF
+ | (errno == EACCES ? IN_ATTRIB : IN_CREATE | IN_MOVED_TO);
+
+ char *pend = strrchr (path, '/');
+
+ if (!pend || pend == path)
+ break;
+
+ *pend = 0;
+ w->wd = inotify_add_watch (fs_fd, path, mask);
+ }
+ while (w->wd < 0 && (errno == ENOENT || errno == EACCES));
+ }
+ }
+
+ if (w->wd >= 0)
+ wlist_add (&fs_hash [w->wd & ((EV_INOTIFY_HASHSIZE) - 1)].head, (WL)w);
+
+ /* now re-arm timer, if required */
+ if (ev_is_active (&w->timer)) ev_ref (EV_A);
+ ev_timer_again (EV_A_ &w->timer);
+ if (ev_is_active (&w->timer)) ev_unref (EV_A);
+}
+
+ecb_noinline
+static void
+infy_del (EV_P_ ev_stat *w)
+{
+ int slot;
+ int wd = w->wd;
+
+ if (wd < 0)
+ return;
+
+ w->wd = -2;
+ slot = wd & ((EV_INOTIFY_HASHSIZE) - 1);
+ wlist_del (&fs_hash [slot].head, (WL)w);
+
+ /* remove this watcher, if others are watching it, they will rearm */
+ inotify_rm_watch (fs_fd, wd);
+}
+
+ecb_noinline
+static void
+infy_wd (EV_P_ int slot, int wd, struct inotify_event *ev)
+{
+ if (slot < 0)
+ /* overflow, need to check for all hash slots */
+ for (slot = 0; slot < (EV_INOTIFY_HASHSIZE); ++slot)
+ infy_wd (EV_A_ slot, wd, ev);
+ else
+ {
+ WL w_;
+
+ for (w_ = fs_hash [slot & ((EV_INOTIFY_HASHSIZE) - 1)].head; w_; )
+ {
+ ev_stat *w = (ev_stat *)w_;
+ w_ = w_->next; /* lets us remove this watcher and all before it */
+
+ if (w->wd == wd || wd == -1)
+ {
+ if (ev->mask & (IN_IGNORED | IN_UNMOUNT | IN_DELETE_SELF))
+ {
+ wlist_del (&fs_hash [slot & ((EV_INOTIFY_HASHSIZE) - 1)].head, (WL)w);
+ w->wd = -1;
+ infy_add (EV_A_ w); /* re-add, no matter what */
+ }
+
+ stat_timer_cb (EV_A_ &w->timer, 0);
+ }
+ }
+ }
+}
+
+static void
+infy_cb (EV_P_ ev_io *w, int revents)
+{
+ char buf [EV_INOTIFY_BUFSIZE];
+ int ofs;
+ int len = read (fs_fd, buf, sizeof (buf));
+
+ for (ofs = 0; ofs < len; )
+ {
+ struct inotify_event *ev = (struct inotify_event *)(buf + ofs);
+ infy_wd (EV_A_ ev->wd, ev->wd, ev);
+ ofs += sizeof (struct inotify_event) + ev->len;
+ }
+}
+
+inline_size ecb_cold
+void
+ev_check_2625 (EV_P)
+{
+ /* kernels < 2.6.25 are borked
+ * http://www.ussg.indiana.edu/hypermail/linux/kernel/0711.3/1208.html
+ */
+ if (ev_linux_version () < 0x020619)
+ return;
+
+ fs_2625 = 1;
+}
+
+inline_size int
+infy_newfd (void)
+{
+#if defined IN_CLOEXEC && defined IN_NONBLOCK
+ int fd = inotify_init1 (IN_CLOEXEC | IN_NONBLOCK);
+ if (fd >= 0)
+ return fd;
+#endif
+ return inotify_init ();
+}
+
+inline_size void
+infy_init (EV_P)
+{
+ if (fs_fd != -2)
+ return;
+
+ fs_fd = -1;
+
+ ev_check_2625 (EV_A);
+
+ fs_fd = infy_newfd ();
+
+ if (fs_fd >= 0)
+ {
+ fd_intern (fs_fd);
+ ev_io_init (&fs_w, infy_cb, fs_fd, EV_READ);
+ ev_set_priority (&fs_w, EV_MAXPRI);
+ ev_io_start (EV_A_ &fs_w);
+ ev_unref (EV_A);
+ }
+}
+
+inline_size void
+infy_fork (EV_P)
+{
+ int slot;
+
+ if (fs_fd < 0)
+ return;
+
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &fs_w);
+ close (fs_fd);
+ fs_fd = infy_newfd ();
+
+ if (fs_fd >= 0)
+ {
+ fd_intern (fs_fd);
+ ev_io_set (&fs_w, fs_fd, EV_READ);
+ ev_io_start (EV_A_ &fs_w);
+ ev_unref (EV_A);
+ }
+
+ for (slot = 0; slot < (EV_INOTIFY_HASHSIZE); ++slot)
+ {
+ WL w_ = fs_hash [slot].head;
+ fs_hash [slot].head = 0;
+
+ while (w_)
+ {
+ ev_stat *w = (ev_stat *)w_;
+ w_ = w_->next; /* lets us add this watcher */
+
+ w->wd = -1;
+
+ if (fs_fd >= 0)
+ infy_add (EV_A_ w); /* re-add, no matter what */
+ else
+ {
+ w->timer.repeat = w->interval ? w->interval : DEF_STAT_INTERVAL;
+ if (ev_is_active (&w->timer)) ev_ref (EV_A);
+ ev_timer_again (EV_A_ &w->timer);
+ if (ev_is_active (&w->timer)) ev_unref (EV_A);
+ }
+ }
+ }
+}
+
+#endif
+
+#ifdef _WIN32
+# define EV_LSTAT(p,b) _stati64 (p, b)
+#else
+# define EV_LSTAT(p,b) lstat (p, b)
+#endif
+
+void
+ev_stat_stat (EV_P_ ev_stat *w) EV_NOEXCEPT
+{
+ if (lstat (w->path, &w->attr) < 0)
+ w->attr.st_nlink = 0;
+ else if (!w->attr.st_nlink)
+ w->attr.st_nlink = 1;
+}
+
+ecb_noinline
+static void
+stat_timer_cb (EV_P_ ev_timer *w_, int revents)
+{
+ ev_stat *w = (ev_stat *)(((char *)w_) - offsetof (ev_stat, timer));
+
+ ev_statdata prev = w->attr;
+ ev_stat_stat (EV_A_ w);
+
+ /* memcmp doesn't work on netbsd, they.... do stuff to their struct stat */
+ if (
+ prev.st_dev != w->attr.st_dev
+ || prev.st_ino != w->attr.st_ino
+ || prev.st_mode != w->attr.st_mode
+ || prev.st_nlink != w->attr.st_nlink
+ || prev.st_uid != w->attr.st_uid
+ || prev.st_gid != w->attr.st_gid
+ || prev.st_rdev != w->attr.st_rdev
+ || prev.st_size != w->attr.st_size
+ /* || prev.st_atime != w->attr.st_atime */ /* Rspamd: to avoid constant maps reload */
+ || prev.st_mtime != w->attr.st_mtime
+ || prev.st_ctime != w->attr.st_ctime
+ ) {
+ /* we only update w->prev on actual differences */
+ /* in case we test more often than invoke the callback, */
+ /* to ensure that prev is always different to attr */
+ w->prev = prev;
+
+ #if EV_USE_INOTIFY
+ if (fs_fd >= 0)
+ {
+ infy_del (EV_A_ w);
+ infy_add (EV_A_ w);
+ ev_stat_stat (EV_A_ w); /* avoid race... */
+ }
+ #endif
+
+ ev_feed_event (EV_A_ w, EV_STAT);
+ }
+}
+
+void
+ev_stat_start (EV_P_ ev_stat *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ ev_stat_stat (EV_A_ w);
+
+ if (w->interval < MIN_STAT_INTERVAL && w->interval)
+ w->interval = MIN_STAT_INTERVAL;
+
+ ev_timer_init (&w->timer, stat_timer_cb, 0., w->interval ? w->interval : DEF_STAT_INTERVAL);
+ ev_set_priority (&w->timer, ev_priority (w));
+
+#if EV_USE_INOTIFY
+ infy_init (EV_A);
+
+ if (fs_fd >= 0)
+ infy_add (EV_A_ w);
+ else
+#endif
+ {
+ ev_timer_again (EV_A_ &w->timer);
+ ev_unref (EV_A);
+ }
+
+ ev_start (EV_A_ (W)w, 1);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_stat_stop (EV_P_ ev_stat *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+#if EV_USE_INOTIFY
+ infy_del (EV_A_ w);
+#endif
+
+ if (ev_is_active (&w->timer))
+ {
+ ev_ref (EV_A);
+ ev_timer_stop (EV_A_ &w->timer);
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_IDLE_ENABLE
+void
+ev_idle_start (EV_P_ ev_idle *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ pri_adjust (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ++idlecnt [ABSPRI (w)];
+
+ ++idleall;
+ ev_start (EV_A_ (W)w, active);
+
+ array_needsize (ev_idle *, idles [ABSPRI (w)], idlemax [ABSPRI (w)], active, array_needsize_noinit);
+ idles [ABSPRI (w)][active - 1] = w;
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_idle_stop (EV_P_ ev_idle *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ idles [ABSPRI (w)][active - 1] = idles [ABSPRI (w)][--idlecnt [ABSPRI (w)]];
+ ev_active (idles [ABSPRI (w)][active - 1]) = active;
+
+ ev_stop (EV_A_ (W)w);
+ --idleall;
+ }
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_PREPARE_ENABLE
+void
+ev_prepare_start (EV_P_ ev_prepare *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++preparecnt);
+ array_needsize (ev_prepare *, prepares, preparemax, preparecnt, array_needsize_noinit);
+ prepares [preparecnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_prepare_stop (EV_P_ ev_prepare *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ prepares [active - 1] = prepares [--preparecnt];
+ ev_active (prepares [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_CHECK_ENABLE
+void
+ev_check_start (EV_P_ ev_check *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++checkcnt);
+ array_needsize (ev_check *, checks, checkmax, checkcnt, array_needsize_noinit);
+ checks [checkcnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_check_stop (EV_P_ ev_check *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ checks [active - 1] = checks [--checkcnt];
+ ev_active (checks [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_EMBED_ENABLE
+ecb_noinline
+void
+ev_embed_sweep (EV_P_ ev_embed *w) EV_NOEXCEPT
+{
+ ev_run (w->other, EVRUN_NOWAIT);
+}
+
+static void
+embed_io_cb (EV_P_ ev_io *io, int revents)
+{
+ ev_embed *w = (ev_embed *)(((char *)io) - offsetof (ev_embed, io));
+
+ if (ev_cb (w))
+ ev_feed_event (EV_A_ (W)w, EV_EMBED);
+ else
+ ev_run (w->other, EVRUN_NOWAIT);
+}
+
+static void
+embed_prepare_cb (EV_P_ ev_prepare *prepare, int revents)
+{
+ ev_embed *w = (ev_embed *)(((char *)prepare) - offsetof (ev_embed, prepare));
+
+ {
+ EV_P = w->other;
+
+ while (fdchangecnt)
+ {
+ fd_reify (EV_A);
+ ev_run (EV_A_ EVRUN_NOWAIT);
+ }
+ }
+}
+
+#if EV_FORK_ENABLE
+static void
+embed_fork_cb (EV_P_ ev_fork *fork_w, int revents)
+{
+ ev_embed *w = (ev_embed *)(((char *)fork_w) - offsetof (ev_embed, fork));
+
+ ev_embed_stop (EV_A_ w);
+
+ {
+ EV_P = w->other;
+
+ ev_loop_fork (EV_A);
+ ev_run (EV_A_ EVRUN_NOWAIT);
+ }
+
+ ev_embed_start (EV_A_ w);
+}
+#endif
+
+#if 0
+static void
+embed_idle_cb (EV_P_ ev_idle *idle, int revents)
+{
+ ev_idle_stop (EV_A_ idle);
+}
+#endif
+
+void
+ev_embed_start (EV_P_ ev_embed *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ {
+ EV_P = w->other;
+ assert (("libev: loop to be embedded is not embeddable", backend & ev_embeddable_backends ()));
+ ev_io_init (&w->io, embed_io_cb, backend_fd, EV_READ);
+ }
+
+ EV_FREQUENT_CHECK;
+
+ ev_set_priority (&w->io, ev_priority (w));
+ ev_io_start (EV_A_ &w->io);
+
+ ev_prepare_init (&w->prepare, embed_prepare_cb);
+ ev_set_priority (&w->prepare, EV_MINPRI);
+ ev_prepare_start (EV_A_ &w->prepare);
+
+#if EV_FORK_ENABLE
+ ev_fork_init (&w->fork, embed_fork_cb);
+ ev_fork_start (EV_A_ &w->fork);
+#endif
+
+ /*ev_idle_init (&w->idle, e,bed_idle_cb);*/
+
+ ev_start (EV_A_ (W)w, 1);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_embed_stop (EV_P_ ev_embed *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_io_stop (EV_A_ &w->io);
+ ev_prepare_stop (EV_A_ &w->prepare);
+#if EV_FORK_ENABLE
+ ev_fork_stop (EV_A_ &w->fork);
+#endif
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_FORK_ENABLE
+void
+ev_fork_start (EV_P_ ev_fork *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++forkcnt);
+ array_needsize (ev_fork *, forks, forkmax, forkcnt, array_needsize_noinit);
+ forks [forkcnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_fork_stop (EV_P_ ev_fork *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ forks [active - 1] = forks [--forkcnt];
+ ev_active (forks [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_CLEANUP_ENABLE
+void
+ev_cleanup_start (EV_P_ ev_cleanup *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++cleanupcnt);
+ array_needsize (ev_cleanup *, cleanups, cleanupmax, cleanupcnt, array_needsize_noinit);
+ cleanups [cleanupcnt - 1] = w;
+
+ /* cleanup watchers should never keep a refcount on the loop */
+ ev_unref (EV_A);
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_cleanup_stop (EV_P_ ev_cleanup *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+ ev_ref (EV_A);
+
+ {
+ int active = ev_active (w);
+
+ cleanups [active - 1] = cleanups [--cleanupcnt];
+ ev_active (cleanups [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_ASYNC_ENABLE
+void
+ev_async_start (EV_P_ ev_async *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ w->sent = 0;
+
+ evpipe_init (EV_A);
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++asynccnt);
+ array_needsize (ev_async *, asyncs, asyncmax, asynccnt, array_needsize_noinit);
+ asyncs [asynccnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_async_stop (EV_P_ ev_async *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ asyncs [active - 1] = asyncs [--asynccnt];
+ ev_active (asyncs [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_async_send (EV_P_ ev_async *w) EV_NOEXCEPT
+{
+ w->sent = 1;
+ evpipe_write (EV_A_ &async_pending);
+}
+#endif
+
+/*****************************************************************************/
+
+struct ev_once
+{
+ ev_io io;
+ ev_timer to;
+ void (*cb)(int revents, void *arg);
+ void *arg;
+};
+
+static void
+once_cb (EV_P_ struct ev_once *once, int revents)
+{
+ void (*cb)(int revents, void *arg) = once->cb;
+ void *arg = once->arg;
+
+ ev_io_stop (EV_A_ &once->io);
+ ev_timer_stop (EV_A_ &once->to);
+ ev_free (once);
+
+ cb (revents, arg);
+}
+
+static void
+once_cb_io (EV_P_ ev_io *w, int revents)
+{
+ struct ev_once *once = (struct ev_once *)(((char *)w) - offsetof (struct ev_once, io));
+
+ once_cb (EV_A_ once, revents | ev_clear_pending (EV_A_ &once->to));
+}
+
+static void
+once_cb_to (EV_P_ ev_timer *w, int revents)
+{
+ struct ev_once *once = (struct ev_once *)(((char *)w) - offsetof (struct ev_once, to));
+
+ once_cb (EV_A_ once, revents | ev_clear_pending (EV_A_ &once->io));
+}
+
+void
+ev_once (EV_P_ int fd, int events, ev_tstamp timeout, void (*cb)(int revents, void *arg), void *arg) EV_NOEXCEPT
+{
+ struct ev_once *once = (struct ev_once *)ev_malloc (sizeof (struct ev_once));
+
+ once->cb = cb;
+ once->arg = arg;
+
+ ev_init (&once->io, once_cb_io);
+ if (fd >= 0)
+ {
+ ev_io_set (&once->io, fd, events);
+ ev_io_start (EV_A_ &once->io);
+ }
+
+ ev_init (&once->to, once_cb_to);
+ if (timeout >= 0.)
+ {
+ ev_timer_set (&once->to, timeout, 0.);
+ ev_timer_start (EV_A_ &once->to);
+ }
+}
+/*****************************************************************************/
+
+#if EV_WALK_ENABLE
+ecb_cold
+void
+ev_walk (EV_P_ int types, void (*cb)(EV_P_ int type, void *w)) EV_NOEXCEPT
+{
+ int i, j;
+ ev_watcher_list *wl, *wn;
+
+ if (types & (EV_IO | EV_EMBED))
+ for (i = 0; i < anfdmax; ++i)
+ for (wl = anfds [i].head; wl; )
+ {
+ wn = wl->next;
+
+#if EV_EMBED_ENABLE
+ if (ev_cb ((ev_io *)wl) == embed_io_cb)
+ {
+ if (types & EV_EMBED)
+ cb (EV_A_ EV_EMBED, ((char *)wl) - offsetof (struct ev_embed, io));
+ }
+ else
+#endif
+#if EV_USE_INOTIFY
+ if (ev_cb ((ev_io *)wl) == infy_cb)
+ ;
+ else
+#endif
+ if ((ev_io *)wl != &pipe_w)
+ if (types & EV_IO)
+ cb (EV_A_ EV_IO, wl);
+
+ wl = wn;
+ }
+
+ if (types & (EV_TIMER | EV_STAT))
+ for (i = timercnt + HEAP0; i-- > HEAP0; )
+#if EV_STAT_ENABLE
+ /*TODO: timer is not always active*/
+ if (ev_cb ((ev_timer *)ANHE_w (timers [i])) == stat_timer_cb)
+ {
+ if (types & EV_STAT)
+ cb (EV_A_ EV_STAT, ((char *)ANHE_w (timers [i])) - offsetof (struct ev_stat, timer));
+ }
+ else
+#endif
+ if (types & EV_TIMER)
+ cb (EV_A_ EV_TIMER, ANHE_w (timers [i]));
+
+#if EV_PERIODIC_ENABLE
+ if (types & EV_PERIODIC)
+ for (i = periodiccnt + HEAP0; i-- > HEAP0; )
+ cb (EV_A_ EV_PERIODIC, ANHE_w (periodics [i]));
+#endif
+
+#if EV_IDLE_ENABLE
+ if (types & EV_IDLE)
+ for (j = NUMPRI; j--; )
+ for (i = idlecnt [j]; i--; )
+ cb (EV_A_ EV_IDLE, idles [j][i]);
+#endif
+
+#if EV_FORK_ENABLE
+ if (types & EV_FORK)
+ for (i = forkcnt; i--; )
+ if (ev_cb (forks [i]) != embed_fork_cb)
+ cb (EV_A_ EV_FORK, forks [i]);
+#endif
+
+#if EV_ASYNC_ENABLE
+ if (types & EV_ASYNC)
+ for (i = asynccnt; i--; )
+ cb (EV_A_ EV_ASYNC, asyncs [i]);
+#endif
+
+#if EV_PREPARE_ENABLE
+ if (types & EV_PREPARE)
+ for (i = preparecnt; i--; )
+# if EV_EMBED_ENABLE
+ if (ev_cb (prepares [i]) != embed_prepare_cb)
+# endif
+ cb (EV_A_ EV_PREPARE, prepares [i]);
+#endif
+
+#if EV_CHECK_ENABLE
+ if (types & EV_CHECK)
+ for (i = checkcnt; i--; )
+ cb (EV_A_ EV_CHECK, checks [i]);
+#endif
+
+#if EV_SIGNAL_ENABLE
+ if (types & EV_SIGNAL)
+ for (i = 0; i < EV_NSIG - 1; ++i)
+ for (wl = signals [i].head; wl; )
+ {
+ wn = wl->next;
+ cb (EV_A_ EV_SIGNAL, wl);
+ wl = wn;
+ }
+#endif
+
+#if EV_CHILD_ENABLE
+ if (types & EV_CHILD)
+ for (i = (EV_PID_HASHSIZE); i--; )
+ for (wl = childs [i]; wl; )
+ {
+ wn = wl->next;
+ cb (EV_A_ EV_CHILD, wl);
+ wl = wn;
+ }
+#endif
+// EV_STAT 0x00001000 /* stat data changed */
+// EV_EMBED 0x00010000 /* embedded event loop needs sweep */
+}
+#endif
+void
+ev_now_update_if_cheap (EV_P) EV_NOEXCEPT
+{
+ if (have_cheap_timer) time_update (EV_A_ 1e100);
+}
+
+int
+ev_active_cnt (EV_P) EV_NOEXCEPT
+{
+ return activecnt;
+}
+
+#if EV_MULTIPLICITY
+ #include "ev_wrap.h"
+#endif
+
diff --git a/contrib/libev/ev.h b/contrib/libev/ev.h
new file mode 100644
index 0000000..7135a08
--- /dev/null
+++ b/contrib/libev/ev.h
@@ -0,0 +1,849 @@
+/*
+ * libev native API header
+ *
+ * Copyright (c) 2007-2020 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef EV_H_
+#define EV_H_
+
+#ifdef __cplusplus
+# define EV_CPP(x) x
+# if __cplusplus >= 201103L
+# define EV_NOEXCEPT noexcept
+# else
+# define EV_NOEXCEPT
+# endif
+#else
+# define EV_CPP(x)
+# define EV_NOEXCEPT
+#endif
+#define EV_THROW EV_NOEXCEPT /* pre-4.25, do not use in new code */
+
+EV_CPP(extern "C" {)
+
+/*****************************************************************************/
+
+/* pre-4.0 compatibility */
+#ifndef EV_COMPAT3
+# define EV_COMPAT3 1
+#endif
+
+#ifndef EV_FEATURES
+# if defined __OPTIMIZE_SIZE__
+# define EV_FEATURES 0x7c
+# else
+# define EV_FEATURES 0x7f
+# endif
+#endif
+
+#define EV_FEATURE_CODE ((EV_FEATURES) & 1)
+#define EV_FEATURE_DATA ((EV_FEATURES) & 2)
+#define EV_FEATURE_CONFIG ((EV_FEATURES) & 4)
+#define EV_FEATURE_API ((EV_FEATURES) & 8)
+#define EV_FEATURE_WATCHERS ((EV_FEATURES) & 16)
+#define EV_FEATURE_BACKENDS ((EV_FEATURES) & 32)
+#define EV_FEATURE_OS ((EV_FEATURES) & 64)
+
+/* these priorities are inclusive, higher priorities will be invoked earlier */
+#ifndef EV_MINPRI
+# define EV_MINPRI (EV_FEATURE_CONFIG ? -2 : 0)
+#endif
+#ifndef EV_MAXPRI
+# define EV_MAXPRI (EV_FEATURE_CONFIG ? +2 : 0)
+#endif
+
+#ifndef EV_MULTIPLICITY
+# define EV_MULTIPLICITY EV_FEATURE_CONFIG
+#endif
+
+#ifndef EV_PERIODIC_ENABLE
+# define EV_PERIODIC_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_STAT_ENABLE
+# define EV_STAT_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_PREPARE_ENABLE
+# define EV_PREPARE_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_CHECK_ENABLE
+# define EV_CHECK_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_IDLE_ENABLE
+# define EV_IDLE_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_FORK_ENABLE
+# define EV_FORK_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_CLEANUP_ENABLE
+# define EV_CLEANUP_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_SIGNAL_ENABLE
+# define EV_SIGNAL_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_CHILD_ENABLE
+# ifdef _WIN32
+# define EV_CHILD_ENABLE 0
+# else
+# define EV_CHILD_ENABLE EV_FEATURE_WATCHERS
+#endif
+#endif
+
+#ifndef EV_ASYNC_ENABLE
+# define EV_ASYNC_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_EMBED_ENABLE
+# define EV_EMBED_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_WALK_ENABLE
+# define EV_WALK_ENABLE 0 /* not yet */
+#endif
+
+/*****************************************************************************/
+
+#if EV_CHILD_ENABLE && !EV_SIGNAL_ENABLE
+# undef EV_SIGNAL_ENABLE
+# define EV_SIGNAL_ENABLE 1
+#endif
+
+/*****************************************************************************/
+
+#ifndef EV_TSTAMP_T
+# define EV_TSTAMP_T double
+#endif
+typedef EV_TSTAMP_T ev_tstamp;
+
+#include <string.h> /* for memmove */
+
+#ifndef EV_ATOMIC_T
+# include <signal.h>
+# define EV_ATOMIC_T sig_atomic_t volatile
+#endif
+
+#if EV_STAT_ENABLE
+# ifdef _WIN32
+# include <time.h>
+# include <sys/types.h>
+# endif
+# include <sys/stat.h>
+#endif
+
+/* support multiple event loops? */
+#if EV_MULTIPLICITY
+struct ev_loop;
+# define EV_P struct ev_loop *loop /* a loop as sole parameter in a declaration */
+# define EV_P_ EV_P, /* a loop as first of multiple parameters */
+# define EV_A loop /* a loop as sole argument to a function call */
+# define EV_A_ EV_A, /* a loop as first of multiple arguments */
+# define EV_DEFAULT_UC ev_default_loop_uc_ () /* the default loop, if initialised, as sole arg */
+# define EV_DEFAULT_UC_ EV_DEFAULT_UC, /* the default loop as first of multiple arguments */
+# define EV_DEFAULT ev_default_loop (0) /* the default loop as sole arg */
+# define EV_DEFAULT_ EV_DEFAULT, /* the default loop as first of multiple arguments */
+#else
+# define EV_P void
+# define EV_P_
+# define EV_A
+# define EV_A_
+# define EV_DEFAULT
+# define EV_DEFAULT_
+# define EV_DEFAULT_UC
+# define EV_DEFAULT_UC_
+# undef EV_EMBED_ENABLE
+#endif
+
+/* EV_INLINE is used for functions in header files */
+#if __STDC_VERSION__ >= 199901L || __GNUC__ >= 3
+# define EV_INLINE static inline
+#else
+# define EV_INLINE static
+#endif
+
+#ifdef EV_API_STATIC
+# define EV_API_DECL static
+#else
+# define EV_API_DECL extern
+#endif
+
+/* EV_PROTOTYPES can be used to switch of prototype declarations */
+#ifndef EV_PROTOTYPES
+# define EV_PROTOTYPES 1
+#endif
+
+/*****************************************************************************/
+
+#define EV_VERSION_MAJOR 4
+#define EV_VERSION_MINOR 33
+
+/* eventmask, revents, events... */
+enum {
+ EV_UNDEF = (int)0xFFFFFFFF, /* guaranteed to be invalid */
+ EV_NONE = 0x00, /* no events */
+ EV_READ = 0x01, /* ev_io detected read will not block */
+ EV_WRITE = 0x02, /* ev_io detected write will not block */
+ EV__IOFDSET = 0x80, /* internal use only */
+ EV_IO = EV_READ, /* alias for type-detection */
+ EV_TIMER = 0x00000100, /* timer timed out */
+#if EV_COMPAT3
+ EV_TIMEOUT = EV_TIMER, /* pre 4.0 API compatibility */
+#endif
+ EV_PERIODIC = 0x00000200, /* periodic timer timed out */
+ EV_SIGNAL = 0x00000400, /* signal was received */
+ EV_CHILD = 0x00000800, /* child/pid had status change */
+ EV_STAT = 0x00001000, /* stat data changed */
+ EV_IDLE = 0x00002000, /* event loop is idling */
+ EV_PREPARE = 0x00004000, /* event loop about to poll */
+ EV_CHECK = 0x00008000, /* event loop finished poll */
+ EV_EMBED = 0x00010000, /* embedded event loop needs sweep */
+ EV_FORK = 0x00020000, /* event loop resumed in child */
+ EV_CLEANUP = 0x00040000, /* event loop resumed in child */
+ EV_ASYNC = 0x00080000, /* async intra-loop signal */
+ EV_CUSTOM = 0x01000000, /* for use by user code */
+ EV_ERROR = (int)0x80000000 /* sent when an error occurs */
+};
+
+/* can be used to add custom fields to all watchers, while losing binary compatibility */
+#ifndef EV_COMMON
+# define EV_COMMON void *data;
+#endif
+
+#ifndef EV_CB_DECLARE
+# define EV_CB_DECLARE(type) void (*cb)(EV_P_ struct type *w, int revents);
+#endif
+#ifndef EV_CB_INVOKE
+# define EV_CB_INVOKE(watcher,revents) (watcher)->cb (EV_A_ (watcher), (revents))
+#endif
+
+/* not official, do not use */
+#define EV_CB(type,name) void name (EV_P_ struct ev_ ## type *w, int revents)
+
+/*
+ * struct member types:
+ * private: you may look at them, but not change them,
+ * and they might not mean anything to you.
+ * ro: can be read anytime, but only changed when the watcher isn't active.
+ * rw: can be read and modified anytime, even when the watcher is active.
+ *
+ * some internal details that might be helpful for debugging:
+ *
+ * active is either 0, which means the watcher is not active,
+ * or the array index of the watcher (periodics, timers)
+ * or the array index + 1 (most other watchers)
+ * or simply 1 for watchers that aren't in some array.
+ * pending is either 0, in which case the watcher isn't,
+ * or the array index + 1 in the pendings array.
+ */
+
+#if EV_MINPRI == EV_MAXPRI
+# define EV_DECL_PRIORITY
+#elif !defined (EV_DECL_PRIORITY)
+# define EV_DECL_PRIORITY int priority;
+#endif
+
+/* shared by all watchers */
+#define EV_WATCHER(type) \
+ int active; /* private */ \
+ int pending; /* private */ \
+ EV_DECL_PRIORITY /* private */ \
+ EV_COMMON /* rw */ \
+ EV_CB_DECLARE (type) /* private */
+
+#define EV_WATCHER_LIST(type) \
+ EV_WATCHER (type) \
+ struct ev_watcher_list *next; /* private */
+
+#define EV_WATCHER_TIME(type) \
+ EV_WATCHER (type) \
+ ev_tstamp at; /* private */
+
+/* base class, nothing to see here unless you subclass */
+typedef struct ev_watcher
+{
+ EV_WATCHER (ev_watcher)
+} ev_watcher;
+
+/* base class, nothing to see here unless you subclass */
+typedef struct ev_watcher_list
+{
+ EV_WATCHER_LIST (ev_watcher_list)
+} ev_watcher_list;
+
+/* base class, nothing to see here unless you subclass */
+typedef struct ev_watcher_time
+{
+ EV_WATCHER_TIME (ev_watcher_time)
+} ev_watcher_time;
+
+/* invoked when fd is either EV_READable or EV_WRITEable */
+/* revent EV_READ, EV_WRITE */
+typedef struct ev_io
+{
+ EV_WATCHER_LIST (ev_io)
+
+ int fd; /* ro */
+ int events; /* ro */
+} ev_io;
+
+/* invoked after a specific time, repeatable (based on monotonic clock) */
+/* revent EV_TIMEOUT */
+typedef struct ev_timer
+{
+ EV_WATCHER_TIME (ev_timer)
+
+ ev_tstamp repeat; /* rw */
+} ev_timer;
+
+/* invoked at some specific time, possibly repeating at regular intervals (based on UTC) */
+/* revent EV_PERIODIC */
+typedef struct ev_periodic
+{
+ EV_WATCHER_TIME (ev_periodic)
+
+ ev_tstamp offset; /* rw */
+ ev_tstamp interval; /* rw */
+ ev_tstamp (*reschedule_cb)(struct ev_periodic *w, ev_tstamp now) EV_NOEXCEPT; /* rw */
+} ev_periodic;
+
+/* invoked when the given signal has been received */
+/* revent EV_SIGNAL */
+typedef struct ev_signal
+{
+ EV_WATCHER_LIST (ev_signal)
+
+ int signum; /* ro */
+} ev_signal;
+
+/* invoked when sigchld is received and waitpid indicates the given pid */
+/* revent EV_CHILD */
+/* does not support priorities */
+typedef struct ev_child
+{
+ EV_WATCHER_LIST (ev_child)
+
+ int flags; /* private */
+ int pid; /* ro */
+ int rpid; /* rw, holds the received pid */
+ int rstatus; /* rw, holds the exit status, use the macros from sys/wait.h */
+} ev_child;
+
+#if EV_STAT_ENABLE
+/* st_nlink = 0 means missing file or other error */
+# ifdef _WIN32
+typedef struct _stati64 ev_statdata;
+# else
+typedef struct stat ev_statdata;
+# endif
+
+/* invoked each time the stat data changes for a given path */
+/* revent EV_STAT */
+typedef struct ev_stat
+{
+ EV_WATCHER_LIST (ev_stat)
+
+ ev_timer timer; /* private */
+ ev_tstamp interval; /* ro */
+ const char *path; /* ro */
+ ev_statdata prev; /* ro */
+ ev_statdata attr; /* ro */
+
+ int wd; /* wd for inotify, fd for kqueue */
+} ev_stat;
+#endif
+
+/* invoked when the nothing else needs to be done, keeps the process from blocking */
+/* revent EV_IDLE */
+typedef struct ev_idle
+{
+ EV_WATCHER (ev_idle)
+} ev_idle;
+
+/* invoked for each run of the mainloop, just before the blocking call */
+/* you can still change events in any way you like */
+/* revent EV_PREPARE */
+typedef struct ev_prepare
+{
+ EV_WATCHER (ev_prepare)
+} ev_prepare;
+
+/* invoked for each run of the mainloop, just after the blocking call */
+/* revent EV_CHECK */
+typedef struct ev_check
+{
+ EV_WATCHER (ev_check)
+} ev_check;
+
+/* the callback gets invoked before check in the child process when a fork was detected */
+/* revent EV_FORK */
+typedef struct ev_fork
+{
+ EV_WATCHER (ev_fork)
+} ev_fork;
+
+/* is invoked just before the loop gets destroyed */
+/* revent EV_CLEANUP */
+typedef struct ev_cleanup
+{
+ EV_WATCHER (ev_cleanup)
+} ev_cleanup;
+
+#if EV_EMBED_ENABLE
+/* used to embed an event loop inside another */
+/* the callback gets invoked when the event loop has handled events, and can be 0 */
+typedef struct ev_embed
+{
+ EV_WATCHER (ev_embed)
+
+ struct ev_loop *other; /* ro */
+#undef EV_IO_ENABLE
+#define EV_IO_ENABLE 1
+ ev_io io; /* private */
+#undef EV_PREPARE_ENABLE
+#define EV_PREPARE_ENABLE 1
+ ev_prepare prepare; /* private */
+ ev_check check; /* unused */
+ ev_timer timer; /* unused */
+ ev_periodic periodic; /* unused */
+ ev_idle idle; /* unused */
+ ev_fork fork; /* private */
+ ev_cleanup cleanup; /* unused */
+} ev_embed;
+#endif
+
+#if EV_ASYNC_ENABLE
+/* invoked when somebody calls ev_async_send on the watcher */
+/* revent EV_ASYNC */
+typedef struct ev_async
+{
+ EV_WATCHER (ev_async)
+
+ EV_ATOMIC_T sent; /* private */
+} ev_async;
+
+# define ev_async_pending(w) (+(w)->sent)
+#endif
+
+/* the presence of this union forces similar struct layout */
+union ev_any_watcher
+{
+ struct ev_watcher w;
+ struct ev_watcher_list wl;
+
+ struct ev_io io;
+ struct ev_timer timer;
+ struct ev_periodic periodic;
+ struct ev_signal signal;
+ struct ev_child child;
+#if EV_STAT_ENABLE
+ struct ev_stat stat;
+#endif
+#if EV_IDLE_ENABLE
+ struct ev_idle idle;
+#endif
+ struct ev_prepare prepare;
+ struct ev_check check;
+#if EV_FORK_ENABLE
+ struct ev_fork fork;
+#endif
+#if EV_CLEANUP_ENABLE
+ struct ev_cleanup cleanup;
+#endif
+#if EV_EMBED_ENABLE
+ struct ev_embed embed;
+#endif
+#if EV_ASYNC_ENABLE
+ struct ev_async async;
+#endif
+};
+
+/* flag bits for ev_default_loop and ev_loop_new */
+enum {
+ /* the default */
+ EVFLAG_AUTO = 0x00000000U, /* not quite a mask */
+ /* flag bits */
+ EVFLAG_NOENV = 0x01000000U, /* do NOT consult environment */
+ EVFLAG_FORKCHECK = 0x02000000U, /* check for a fork in each iteration */
+ /* debugging/feature disable */
+ EVFLAG_NOINOTIFY = 0x00100000U, /* do not attempt to use inotify */
+#if EV_COMPAT3
+ EVFLAG_NOSIGFD = 0, /* compatibility to pre-3.9 */
+#endif
+ EVFLAG_SIGNALFD = 0x00200000U, /* attempt to use signalfd */
+ EVFLAG_NOSIGMASK = 0x00400000U, /* avoid modifying the signal mask */
+ EVFLAG_NOTIMERFD = 0x00800000U /* avoid creating a timerfd */
+};
+
+/* method bits to be ored together */
+enum {
+ EVBACKEND_SELECT = 0x00000001U, /* available just about anywhere */
+ EVBACKEND_POLL = 0x00000002U, /* !win, !aix, broken on osx */
+ EVBACKEND_EPOLL = 0x00000004U, /* linux */
+ EVBACKEND_KQUEUE = 0x00000008U, /* bsd, broken on osx */
+ EVBACKEND_DEVPOLL = 0x00000010U, /* solaris 8 */ /* NYI */
+ EVBACKEND_PORT = 0x00000020U, /* solaris 10 */
+ EVBACKEND_LINUXAIO = 0x00000040U, /* linux AIO, 4.19+ */
+ EVBACKEND_IOURING = 0x00000080U, /* linux io_uring, 5.1+ */
+ EVBACKEND_ALL = 0x000000FFU, /* all known backends */
+ EVBACKEND_MASK = 0x0000FFFFU /* all future backends */
+};
+
+#if EV_PROTOTYPES
+EV_API_DECL int ev_version_major (void) EV_NOEXCEPT;
+EV_API_DECL int ev_version_minor (void) EV_NOEXCEPT;
+
+EV_API_DECL unsigned int ev_supported_backends (void) EV_NOEXCEPT;
+EV_API_DECL unsigned int ev_recommended_backends (void) EV_NOEXCEPT;
+EV_API_DECL unsigned int ev_embeddable_backends (void) EV_NOEXCEPT;
+
+EV_API_DECL ev_tstamp ev_time (void) EV_NOEXCEPT;
+EV_API_DECL void ev_sleep (ev_tstamp delay) EV_NOEXCEPT; /* sleep for a while */
+
+/* Sets the allocation function to use, works like realloc.
+ * It is used to allocate and free memory.
+ * If it returns zero when memory needs to be allocated, the library might abort
+ * or take some potentially destructive action.
+ * The default is your system realloc function.
+ */
+EV_API_DECL void ev_set_allocator (void *(*cb)(void *ptr, long size) EV_NOEXCEPT) EV_NOEXCEPT;
+
+/* set the callback function to call on a
+ * retryable syscall error
+ * (such as failed select, poll, epoll_wait)
+ */
+EV_API_DECL void ev_set_syserr_cb (void (*cb)(const char *msg) EV_NOEXCEPT) EV_NOEXCEPT;
+
+#if EV_MULTIPLICITY
+
+/* the default loop is the only one that handles signals and child watchers */
+/* you can call this as often as you like */
+EV_API_DECL struct ev_loop *ev_default_loop (unsigned int flags EV_CPP (= 0)) EV_NOEXCEPT;
+
+/* create and destroy alternative loops that don't handle signals */
+EV_API_DECL struct ev_loop *ev_loop_new (unsigned int flags EV_CPP (= 0)) EV_NOEXCEPT;
+
+EV_API_DECL ev_tstamp ev_now (EV_P) EV_NOEXCEPT; /* time w.r.t. timers and the eventloop, updated after each poll */
+
+#else
+
+EV_API_DECL int ev_default_loop (unsigned int flags EV_CPP (= 0)) EV_NOEXCEPT; /* returns true when successful */
+
+EV_API_DECL ev_tstamp ev_rt_now;
+
+EV_INLINE ev_tstamp
+ev_now (void) EV_NOEXCEPT
+{
+ return ev_rt_now;
+}
+
+/* looks weird, but ev_is_default_loop (EV_A) still works if this exists */
+EV_INLINE int
+ev_is_default_loop (void) EV_NOEXCEPT
+{
+ return 1;
+}
+
+#endif /* multiplicity */
+
+/* destroy event loops, also works for the default loop */
+EV_API_DECL void ev_loop_destroy (EV_P);
+
+/* this needs to be called after fork, to duplicate the loop */
+/* when you want to re-use it in the child */
+/* you can call it in either the parent or the child */
+/* you can actually call it at any time, anywhere :) */
+EV_API_DECL void ev_loop_fork (EV_P) EV_NOEXCEPT;
+
+EV_API_DECL unsigned int ev_backend (EV_P) EV_NOEXCEPT; /* backend in use by loop */
+
+EV_API_DECL void ev_now_update (EV_P) EV_NOEXCEPT; /* update event loop time */
+/*
+ * Same as ev_now_update, but will update time merely if cheap (coarse) timers
+ * are used in system.
+ */
+EV_API_DECL void ev_now_update_if_cheap (EV_P) EV_NOEXCEPT;
+
+#if EV_WALK_ENABLE
+/* walk (almost) all watchers in the loop of a given type, invoking the */
+/* callback on every such watcher. The callback might stop the watcher, */
+/* but do nothing else with the loop */
+EV_API_DECL void ev_walk (EV_P_ int types, void (*cb)(EV_P_ int type, void *w)) EV_NOEXCEPT;
+#endif
+
+#endif /* prototypes */
+
+/* ev_run flags values */
+enum {
+ EVRUN_NOWAIT = 1, /* do not block/wait */
+ EVRUN_ONCE = 2 /* block *once* only */
+};
+
+/* ev_break how values */
+enum {
+ EVBREAK_CANCEL = 0, /* undo unloop */
+ EVBREAK_ONE = 1, /* unloop once */
+ EVBREAK_ALL = 2 /* unloop all loops */
+};
+
+#if EV_PROTOTYPES
+EV_API_DECL int ev_run (EV_P_ int flags EV_CPP (= 0));
+EV_API_DECL void ev_break (EV_P_ int how EV_CPP (= EVBREAK_ONE)) EV_NOEXCEPT; /* break out of the loop */
+
+/*
+ * ref/unref can be used to add or remove a refcount on the mainloop. every watcher
+ * keeps one reference. if you have a long-running watcher you never unregister that
+ * should not keep ev_loop from running, unref() after starting, and ref() before stopping.
+ */
+EV_API_DECL void ev_ref (EV_P) EV_NOEXCEPT;
+EV_API_DECL void ev_unref (EV_P) EV_NOEXCEPT;
+EV_API_DECL int ev_active_cnt (EV_P) EV_NOEXCEPT;
+
+/*
+ * convenience function, wait for a single event, without registering an event watcher
+ * if timeout is < 0, do wait indefinitely
+ */
+EV_API_DECL void ev_once (EV_P_ int fd, int events, ev_tstamp timeout, void (*cb)(int revents, void *arg), void *arg) EV_NOEXCEPT;
+
+EV_API_DECL void ev_invoke_pending (EV_P); /* invoke all pending watchers */
+
+# if EV_FEATURE_API
+EV_API_DECL unsigned int ev_iteration (EV_P) EV_NOEXCEPT; /* number of loop iterations */
+EV_API_DECL unsigned int ev_depth (EV_P) EV_NOEXCEPT; /* #ev_loop enters - #ev_loop leaves */
+EV_API_DECL void ev_verify (EV_P) EV_NOEXCEPT; /* abort if loop data corrupted */
+
+EV_API_DECL void ev_set_io_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT; /* sleep at least this time, default 0 */
+EV_API_DECL void ev_set_timeout_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT; /* sleep at least this time, default 0 */
+
+/* advanced stuff for threading etc. support, see docs */
+EV_API_DECL void ev_set_userdata (EV_P_ void *data) EV_NOEXCEPT;
+EV_API_DECL void *ev_userdata (EV_P) EV_NOEXCEPT;
+typedef void (*ev_loop_callback)(EV_P);
+EV_API_DECL void ev_set_invoke_pending_cb (EV_P_ ev_loop_callback invoke_pending_cb) EV_NOEXCEPT;
+/* C++ doesn't allow the use of the ev_loop_callback typedef here, so we need to spell it out */
+EV_API_DECL void ev_set_loop_release_cb (EV_P_ void (*release)(EV_P) EV_NOEXCEPT, void (*acquire)(EV_P) EV_NOEXCEPT) EV_NOEXCEPT;
+
+EV_API_DECL unsigned int ev_pending_count (EV_P) EV_NOEXCEPT; /* number of pending events, if any */
+
+/*
+ * stop/start the timer handling.
+ */
+EV_API_DECL void ev_suspend (EV_P) EV_NOEXCEPT;
+EV_API_DECL void ev_resume (EV_P) EV_NOEXCEPT;
+#endif
+
+#endif
+
+/* these may evaluate ev multiple times, and the other arguments at most once */
+/* either use ev_init + ev_TYPE_set, or the ev_TYPE_init macro, below, to first initialise a watcher */
+#define ev_init(ev,cb_) do { \
+ ((ev_watcher *)(void *)(ev))->active = \
+ ((ev_watcher *)(void *)(ev))->pending = 0; \
+ ev_set_priority ((ev), 0); \
+ ev_set_cb ((ev), cb_); \
+} while (0)
+
+#define ev_io_modify(ev,events_) do { (ev)->events = (ev)->events & EV__IOFDSET | (events_); } while (0)
+#define ev_io_set(ev,fd_,events_) do { (ev)->fd = (fd_); (ev)->events = (events_) | EV__IOFDSET; } while (0)
+#define ev_timer_set(ev,after_,repeat_) do { ((ev_watcher_time *)(ev))->at = (after_); (ev)->repeat = (repeat_); } while (0)
+#define ev_periodic_set(ev,ofs_,ival_,rcb_) do { (ev)->offset = (ofs_); (ev)->interval = (ival_); (ev)->reschedule_cb = (rcb_); } while (0)
+#define ev_signal_set(ev,signum_) do { (ev)->signum = (signum_); } while (0)
+#define ev_child_set(ev,pid_,trace_) do { (ev)->pid = (pid_); (ev)->flags = !!(trace_); } while (0)
+#define ev_stat_set(ev,path_,interval_) do { (ev)->path = (path_); (ev)->interval = (interval_); (ev)->wd = -2; } while (0)
+#define ev_idle_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_prepare_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_check_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_embed_set(ev,other_) do { (ev)->other = (other_); } while (0)
+#define ev_fork_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_cleanup_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_async_set(ev) /* nop, yes, this is a serious in-joke */
+
+#define ev_io_init(ev,cb,fd,events) do { ev_init ((ev), (cb)); ev_io_set ((ev),(fd),(events)); } while (0)
+#define ev_timer_init(ev,cb,after,repeat) do { ev_init ((ev), (cb)); ev_timer_set ((ev),(after),(repeat)); } while (0)
+#define ev_periodic_init(ev,cb,ofs,ival,rcb) do { ev_init ((ev), (cb)); ev_periodic_set ((ev),(ofs),(ival),(rcb)); } while (0)
+#define ev_signal_init(ev,cb,signum) do { ev_init ((ev), (cb)); ev_signal_set ((ev), (signum)); } while (0)
+#define ev_child_init(ev,cb,pid,trace) do { ev_init ((ev), (cb)); ev_child_set ((ev),(pid),(trace)); } while (0)
+#define ev_stat_init(ev,cb,path,interval) do { ev_init ((ev), (cb)); ev_stat_set ((ev),(path),(interval)); } while (0)
+#define ev_idle_init(ev,cb) do { ev_init ((ev), (cb)); ev_idle_set ((ev)); } while (0)
+#define ev_prepare_init(ev,cb) do { ev_init ((ev), (cb)); ev_prepare_set ((ev)); } while (0)
+#define ev_check_init(ev,cb) do { ev_init ((ev), (cb)); ev_check_set ((ev)); } while (0)
+#define ev_embed_init(ev,cb,other) do { ev_init ((ev), (cb)); ev_embed_set ((ev),(other)); } while (0)
+#define ev_fork_init(ev,cb) do { ev_init ((ev), (cb)); ev_fork_set ((ev)); } while (0)
+#define ev_cleanup_init(ev,cb) do { ev_init ((ev), (cb)); ev_cleanup_set ((ev)); } while (0)
+#define ev_async_init(ev,cb) do { ev_init ((ev), (cb)); ev_async_set ((ev)); } while (0)
+
+#define ev_is_pending(ev) (0 + ((ev_watcher *)(void *)(ev))->pending) /* ro, true when watcher is waiting for callback invocation */
+#define ev_is_active(ev) (0 + ((ev_watcher *)(void *)(ev))->active) /* ro, true when the watcher has been started */
+#define ev_can_stop(ev) (ev_is_pending(ev) || ev_is_active(ev)) /* ro, true when the watcher has been started */
+
+#define ev_cb_(ev) (ev)->cb /* rw */
+#define ev_cb(ev) (memmove (&ev_cb_ (ev), &((ev_watcher *)(ev))->cb, sizeof (ev_cb_ (ev))), (ev)->cb)
+
+#if EV_MINPRI == EV_MAXPRI
+# define ev_priority(ev) ((ev), EV_MINPRI)
+# define ev_set_priority(ev,pri) ((ev), (pri))
+#else
+# define ev_priority(ev) (+(((ev_watcher *)(void *)(ev))->priority))
+# define ev_set_priority(ev,pri) ( (ev_watcher *)(void *)(ev))->priority = (pri)
+#endif
+
+#define ev_periodic_at(ev) (+((ev_watcher_time *)(ev))->at)
+
+#ifndef ev_set_cb
+/* memmove is used here to avoid strict aliasing violations, and hopefully is optimized out by any reasonable compiler */
+# define ev_set_cb(ev,cb_) (ev_cb_ (ev) = (cb_), memmove (&((ev_watcher *)(ev))->cb, &ev_cb_ (ev), sizeof (ev_cb_ (ev))))
+#endif
+
+/* stopping (enabling, adding) a watcher does nothing if it is already running */
+/* stopping (disabling, deleting) a watcher does nothing unless it's already running */
+#if EV_PROTOTYPES
+
+/* feeds an event into a watcher as if the event actually occurred */
+/* accepts any ev_watcher type */
+EV_API_DECL void ev_feed_event (EV_P_ void *w, int revents) EV_NOEXCEPT;
+EV_API_DECL void ev_feed_fd_event (EV_P_ int fd, int revents) EV_NOEXCEPT;
+#if EV_SIGNAL_ENABLE
+EV_API_DECL void ev_feed_signal (int signum) EV_NOEXCEPT;
+EV_API_DECL void ev_feed_signal_event (EV_P_ int signum) EV_NOEXCEPT;
+#endif
+EV_API_DECL void ev_invoke (EV_P_ void *w, int revents);
+EV_API_DECL int ev_clear_pending (EV_P_ void *w) EV_NOEXCEPT;
+
+EV_API_DECL void ev_io_start (EV_P_ ev_io *w) EV_NOEXCEPT;
+EV_API_DECL void ev_io_stop (EV_P_ ev_io *w) EV_NOEXCEPT;
+
+EV_API_DECL void ev_timer_start (EV_P_ ev_timer *w) EV_NOEXCEPT;
+EV_API_DECL void ev_timer_stop (EV_P_ ev_timer *w) EV_NOEXCEPT;
+/* stops if active and no repeat, restarts if active and repeating, starts if inactive and repeating */
+EV_API_DECL void ev_timer_again (EV_P_ ev_timer *w) EV_NOEXCEPT;
+/* return remaining time */
+EV_API_DECL ev_tstamp ev_timer_remaining (EV_P_ ev_timer *w) EV_NOEXCEPT;
+
+#if EV_PERIODIC_ENABLE
+EV_API_DECL void ev_periodic_start (EV_P_ ev_periodic *w) EV_NOEXCEPT;
+EV_API_DECL void ev_periodic_stop (EV_P_ ev_periodic *w) EV_NOEXCEPT;
+EV_API_DECL void ev_periodic_again (EV_P_ ev_periodic *w) EV_NOEXCEPT;
+#endif
+
+/* only supported in the default loop */
+#if EV_SIGNAL_ENABLE
+EV_API_DECL void ev_signal_start (EV_P_ ev_signal *w) EV_NOEXCEPT;
+EV_API_DECL void ev_signal_stop (EV_P_ ev_signal *w) EV_NOEXCEPT;
+#endif
+
+/* only supported in the default loop */
+# if EV_CHILD_ENABLE
+EV_API_DECL void ev_child_start (EV_P_ ev_child *w) EV_NOEXCEPT;
+EV_API_DECL void ev_child_stop (EV_P_ ev_child *w) EV_NOEXCEPT;
+# endif
+
+# if EV_STAT_ENABLE
+EV_API_DECL void ev_stat_start (EV_P_ ev_stat *w) EV_NOEXCEPT;
+EV_API_DECL void ev_stat_stop (EV_P_ ev_stat *w) EV_NOEXCEPT;
+EV_API_DECL void ev_stat_stat (EV_P_ ev_stat *w) EV_NOEXCEPT;
+# endif
+
+# if EV_IDLE_ENABLE
+EV_API_DECL void ev_idle_start (EV_P_ ev_idle *w) EV_NOEXCEPT;
+EV_API_DECL void ev_idle_stop (EV_P_ ev_idle *w) EV_NOEXCEPT;
+# endif
+
+#if EV_PREPARE_ENABLE
+EV_API_DECL void ev_prepare_start (EV_P_ ev_prepare *w) EV_NOEXCEPT;
+EV_API_DECL void ev_prepare_stop (EV_P_ ev_prepare *w) EV_NOEXCEPT;
+#endif
+
+#if EV_CHECK_ENABLE
+EV_API_DECL void ev_check_start (EV_P_ ev_check *w) EV_NOEXCEPT;
+EV_API_DECL void ev_check_stop (EV_P_ ev_check *w) EV_NOEXCEPT;
+#endif
+
+# if EV_FORK_ENABLE
+EV_API_DECL void ev_fork_start (EV_P_ ev_fork *w) EV_NOEXCEPT;
+EV_API_DECL void ev_fork_stop (EV_P_ ev_fork *w) EV_NOEXCEPT;
+# endif
+
+# if EV_CLEANUP_ENABLE
+EV_API_DECL void ev_cleanup_start (EV_P_ ev_cleanup *w) EV_NOEXCEPT;
+EV_API_DECL void ev_cleanup_stop (EV_P_ ev_cleanup *w) EV_NOEXCEPT;
+# endif
+
+# if EV_EMBED_ENABLE
+/* only supported when loop to be embedded is in fact embeddable */
+EV_API_DECL void ev_embed_start (EV_P_ ev_embed *w) EV_NOEXCEPT;
+EV_API_DECL void ev_embed_stop (EV_P_ ev_embed *w) EV_NOEXCEPT;
+EV_API_DECL void ev_embed_sweep (EV_P_ ev_embed *w) EV_NOEXCEPT;
+# endif
+
+# if EV_ASYNC_ENABLE
+EV_API_DECL void ev_async_start (EV_P_ ev_async *w) EV_NOEXCEPT;
+EV_API_DECL void ev_async_stop (EV_P_ ev_async *w) EV_NOEXCEPT;
+EV_API_DECL void ev_async_send (EV_P_ ev_async *w) EV_NOEXCEPT;
+# endif
+
+#if EV_COMPAT3
+ #define EVLOOP_NONBLOCK EVRUN_NOWAIT
+ #define EVLOOP_ONESHOT EVRUN_ONCE
+ #define EVUNLOOP_CANCEL EVBREAK_CANCEL
+ #define EVUNLOOP_ONE EVBREAK_ONE
+ #define EVUNLOOP_ALL EVBREAK_ALL
+ #if EV_PROTOTYPES
+ EV_INLINE void ev_loop (EV_P_ int flags) { ev_run (EV_A_ flags); }
+ EV_INLINE void ev_unloop (EV_P_ int how ) { ev_break (EV_A_ how ); }
+ EV_INLINE void ev_default_destroy (void) { ev_loop_destroy (EV_DEFAULT); }
+ EV_INLINE void ev_default_fork (void) { ev_loop_fork (EV_DEFAULT); }
+ #if EV_FEATURE_API
+ EV_INLINE unsigned int ev_loop_count (EV_P) { return ev_iteration (EV_A); }
+ EV_INLINE unsigned int ev_loop_depth (EV_P) { return ev_depth (EV_A); }
+ EV_INLINE void ev_loop_verify (EV_P) { ev_verify (EV_A); }
+ #endif
+ #endif
+#else
+ typedef struct ev_loop ev_loop;
+#endif
+
+#endif
+
+EV_CPP(})
+
+#endif
+
diff --git a/contrib/libev/ev_epoll.c b/contrib/libev/ev_epoll.c
new file mode 100644
index 0000000..58cfa68
--- /dev/null
+++ b/contrib/libev/ev_epoll.c
@@ -0,0 +1,298 @@
+/*
+ * libev epoll fd activity backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2016,2017,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/*
+ * general notes about epoll:
+ *
+ * a) epoll silently removes fds from the fd set. as nothing tells us
+ * that an fd has been removed otherwise, we have to continually
+ * "rearm" fds that we suspect *might* have changed (same
+ * problem with kqueue, but much less costly there).
+ * b) the fact that ADD != MOD creates a lot of extra syscalls due to a)
+ * and seems not to have any advantage.
+ * c) the inability to handle fork or file descriptors (think dup)
+ * limits the applicability over poll, so this is not a generic
+ * poll replacement.
+ * d) epoll doesn't work the same as select with many file descriptors
+ * (such as files). while not critical, no other advanced interface
+ * seems to share this (rather non-unixy) limitation.
+ * e) epoll claims to be embeddable, but in practise you never get
+ * a ready event for the epoll fd (broken: <=2.6.26, working: >=2.6.32).
+ * f) epoll_ctl returning EPERM means the fd is always ready.
+ *
+ * lots of "weird code" and complication handling in this file is due
+ * to these design problems with epoll, as we try very hard to avoid
+ * epoll_ctl syscalls for common usage patterns and handle the breakage
+ * ensuing from receiving events for closed and otherwise long gone
+ * file descriptors.
+ */
+
+#include <sys/epoll.h>
+
+#define EV_EMASK_EPERM 0x80
+
+static void
+epoll_modify (EV_P_ int fd, int oev, int nev)
+{
+ struct epoll_event ev;
+ unsigned char oldmask;
+
+ /*
+ * we handle EPOLL_CTL_DEL by ignoring it here
+ * on the assumption that the fd is gone anyways
+ * if that is wrong, we have to handle the spurious
+ * event in epoll_poll.
+ * if the fd is added again, we try to ADD it, and, if that
+ * fails, we assume it still has the same eventmask.
+ */
+ if (!nev)
+ return;
+
+ oldmask = anfds [fd].emask;
+ anfds [fd].emask = nev;
+
+ /* store the generation counter in the upper 32 bits, the fd in the lower 32 bits */
+ ev.data.u64 = (uint64_t)(uint32_t)fd
+ | ((uint64_t)(uint32_t)++anfds [fd].egen << 32);
+ ev.events = (nev & EV_READ ? EPOLLIN : 0)
+ | (nev & EV_WRITE ? EPOLLOUT : 0);
+
+ if (ecb_expect_true (!epoll_ctl (backend_fd, oev && oldmask != nev ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, fd, &ev)))
+ return;
+
+ if (ecb_expect_true (errno == ENOENT))
+ {
+ /* if ENOENT then the fd went away, so try to do the right thing */
+ if (!nev)
+ goto dec_egen;
+
+ if (!epoll_ctl (backend_fd, EPOLL_CTL_ADD, fd, &ev))
+ return;
+ }
+ else if (ecb_expect_true (errno == EEXIST))
+ {
+ /* EEXIST means we ignored a previous DEL, but the fd is still active */
+ /* if the kernel mask is the same as the new mask, we assume it hasn't changed */
+ if (oldmask == nev)
+ goto dec_egen;
+
+ if (!epoll_ctl (backend_fd, EPOLL_CTL_MOD, fd, &ev))
+ return;
+ }
+ else if (ecb_expect_true (errno == EPERM))
+ {
+ /* EPERM means the fd is always ready, but epoll is too snobbish */
+ /* to handle it, unlike select or poll. */
+ anfds [fd].emask = EV_EMASK_EPERM;
+
+ /* add fd to epoll_eperms, if not already inside */
+ if (!(oldmask & EV_EMASK_EPERM))
+ {
+ array_needsize (int, epoll_eperms, epoll_epermmax, epoll_epermcnt + 1, array_needsize_noinit);
+ epoll_eperms [epoll_epermcnt++] = fd;
+ }
+
+ return;
+ }
+ else
+ assert (("libev: I/O watcher with invalid fd found in epoll_ctl", errno != EBADF && errno != ELOOP && errno != EINVAL));
+
+ fd_kill (EV_A_ fd);
+
+dec_egen:
+ /* we didn't successfully call epoll_ctl, so decrement the generation counter again */
+ --anfds [fd].egen;
+}
+
+static void
+epoll_poll (EV_P_ ev_tstamp timeout)
+{
+ int i;
+ int eventcnt;
+
+ if (ecb_expect_false (epoll_epermcnt))
+ timeout = EV_TS_CONST (0.);
+
+ /* epoll wait times cannot be larger than (LONG_MAX - 999UL) / HZ msecs, which is below */
+ /* the default libev max wait time, however. */
+ EV_RELEASE_CB;
+ eventcnt = epoll_wait (backend_fd, epoll_events, epoll_eventmax, EV_TS_TO_MSEC (timeout));
+ EV_ACQUIRE_CB;
+
+ if (ecb_expect_false (eventcnt < 0))
+ {
+ if (errno != EINTR)
+ ev_syserr ("(libev) epoll_wait");
+
+ return;
+ }
+
+ for (i = 0; i < eventcnt; ++i)
+ {
+ struct epoll_event *ev = epoll_events + i;
+
+ int fd = (uint32_t)ev->data.u64; /* mask out the lower 32 bits */
+ int want = anfds [fd].events;
+ int got = (ev->events & (EPOLLOUT | EPOLLERR | EPOLLHUP) ? EV_WRITE : 0)
+ | (ev->events & (EPOLLIN | EPOLLERR | EPOLLHUP) ? EV_READ : 0);
+
+ /*
+ * check for spurious notification.
+ * this only finds spurious notifications on egen updates
+ * other spurious notifications will be found by epoll_ctl, below
+ * we assume that fd is always in range, as we never shrink the anfds array
+ */
+ if (ecb_expect_false ((uint32_t)anfds [fd].egen != (uint32_t)(ev->data.u64 >> 32)))
+ {
+ /* recreate kernel state */
+ postfork |= 2;
+ continue;
+ }
+
+ if (ecb_expect_false (got & ~want))
+ {
+ anfds [fd].emask = want;
+
+ /*
+ * we received an event but are not interested in it, try mod or del
+ * this often happens because we optimistically do not unregister fds
+ * when we are no longer interested in them, but also when we get spurious
+ * notifications for fds from another process. this is partially handled
+ * above with the gencounter check (== our fd is not the event fd), and
+ * partially here, when epoll_ctl returns an error (== a child has the fd
+ * but we closed it).
+ * note: for events such as POLLHUP, where we can't know whether it refers
+ * to EV_READ or EV_WRITE, we might issue redundant EPOLL_CTL_MOD calls.
+ */
+ ev->events = (want & EV_READ ? EPOLLIN : 0)
+ | (want & EV_WRITE ? EPOLLOUT : 0);
+
+ /* pre-2.6.9 kernels require a non-null pointer with EPOLL_CTL_DEL, */
+ /* which is fortunately easy to do for us. */
+ if (epoll_ctl (backend_fd, want ? EPOLL_CTL_MOD : EPOLL_CTL_DEL, fd, ev))
+ {
+ postfork |= 2; /* an error occurred, recreate kernel state */
+ continue;
+ }
+ }
+
+ fd_event (EV_A_ fd, got);
+ }
+
+ /* if the receive array was full, increase its size */
+ if (ecb_expect_false (eventcnt == epoll_eventmax))
+ {
+ ev_free (epoll_events);
+ epoll_eventmax = array_nextsize (sizeof (struct epoll_event), epoll_eventmax, epoll_eventmax + 1);
+ epoll_events = (struct epoll_event *)ev_malloc (sizeof (struct epoll_event) * epoll_eventmax);
+ }
+
+ /* now synthesize events for all fds where epoll fails, while select works... */
+ for (i = epoll_epermcnt; i--; )
+ {
+ int fd = epoll_eperms [i];
+ unsigned char events = anfds [fd].events & (EV_READ | EV_WRITE);
+
+ if (anfds [fd].emask & EV_EMASK_EPERM && events)
+ fd_event (EV_A_ fd, events);
+ else
+ {
+ epoll_eperms [i] = epoll_eperms [--epoll_epermcnt];
+ anfds [fd].emask = 0;
+ }
+ }
+}
+
+static int
+epoll_epoll_create (void)
+{
+ int fd;
+
+#if defined EPOLL_CLOEXEC && !defined __ANDROID__
+ fd = epoll_create1 (EPOLL_CLOEXEC);
+
+ if (fd < 0 && (errno == EINVAL || errno == ENOSYS))
+#endif
+ {
+ fd = epoll_create (256);
+
+ if (fd >= 0)
+ fcntl (fd, F_SETFD, FD_CLOEXEC);
+ }
+
+ return fd;
+}
+
+inline_size
+int
+epoll_init (EV_P_ int flags)
+{
+ if ((backend_fd = epoll_epoll_create ()) < 0)
+ return 0;
+
+ backend_mintime = EV_TS_CONST (1e-3); /* epoll does sometimes return early, this is just to avoid the worst */
+ backend_modify = epoll_modify;
+ backend_poll = epoll_poll;
+
+ epoll_eventmax = 64; /* initial number of events receivable per poll */
+ epoll_events = (struct epoll_event *)ev_malloc (sizeof (struct epoll_event) * epoll_eventmax);
+
+ return EVBACKEND_EPOLL;
+}
+
+inline_size
+void
+epoll_destroy (EV_P)
+{
+ ev_free (epoll_events);
+ array_free (epoll_eperm, EMPTY);
+}
+
+ecb_cold
+static void
+epoll_fork (EV_P)
+{
+ close (backend_fd);
+
+ while ((backend_fd = epoll_epoll_create ()) < 0)
+ ev_syserr ("(libev) epoll_create");
+
+ fd_rearm_all (EV_A);
+}
+
diff --git a/contrib/libev/ev_iouring.c b/contrib/libev/ev_iouring.c
new file mode 100644
index 0000000..612391b
--- /dev/null
+++ b/contrib/libev/ev_iouring.c
@@ -0,0 +1,697 @@
+/*
+ * libev linux io_uring fd activity backend
+ *
+ * Copyright (c) 2019-2020 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/*
+ * general notes about linux io_uring:
+ *
+ * a) it's the best interface I have seen so far. on linux.
+ * b) best is not necessarily very good.
+ * c) it's better than the aio mess, doesn't suffer from the fork problems
+ * of linux aio or epoll and so on and so on. and you could do event stuff
+ * without any syscalls. what's not to like?
+ * d) ok, it's vastly more complex, but that's ok, really.
+ * e) why two mmaps instead of one? one would be more space-efficient,
+ * and I can't see what benefit two would have (other than being
+ * somehow resizable/relocatable, but that's apparently not possible).
+ * f) hmm, it's practically undebuggable (gdb can't access the memory, and
+ * the bizarre way structure offsets are communicated makes it hard to
+ * just print the ring buffer heads, even *iff* the memory were visible
+ * in gdb. but then, that's also ok, really.
+ * g) well, you cannot specify a timeout when waiting for events. no,
+ * seriously, the interface doesn't support a timeout. never seen _that_
+ * before. sure, you can use a timerfd, but that's another syscall
+ * you could have avoided. overall, this bizarre omission smells
+ * like a µ-optimisation by the io_uring author for his personal
+ * applications, to the detriment of everybody else who just wants
+ * an event loop. but, umm, ok, if that's all, it could be worse.
+ * (from what I gather from the author Jens Axboe, it simply didn't
+ * occur to him, and he made good on it by adding an unlimited nuber
+ * of timeouts later :).
+ * h) initially there was a hardcoded limit of 4096 outstanding events.
+ * later versions not only bump this to 32k, but also can handle
+ * an unlimited amount of events, so this only affects the batch size.
+ * i) unlike linux aio, you *can* register more then the limit
+ * of fd events. while early verisons of io_uring signalled an overflow
+ * and you ended up getting wet. 5.5+ does not do this anymore.
+ * j) but, oh my! it had exactly the same bugs as the linux aio backend,
+ * where some undocumented poll combinations just fail. fortunately,
+ * after finally reaching the author, he was more than willing to fix
+ * this probably in 5.6+.
+ * k) overall, the *API* itself is, I dare to say, not a total trainwreck.
+ * once the bugs ae fixed (probably in 5.6+), it will be without
+ * competition.
+ */
+
+/* TODO: use internal TIMEOUT */
+/* TODO: take advantage of single mmap, NODROP etc. */
+/* TODO: resize cq/sq size independently */
+
+#include <sys/timerfd.h>
+#include <sys/mman.h>
+#include <poll.h>
+#include <stdint.h>
+
+#define IOURING_INIT_ENTRIES 32
+
+/*****************************************************************************/
+/* syscall wrapdadoop - this section has the raw api/abi definitions */
+
+#include <linux/fs.h>
+#include <linux/types.h>
+
+/* mostly directly taken from the kernel or documentation */
+
+struct io_uring_sqe
+{
+ __u8 opcode;
+ __u8 flags;
+ __u16 ioprio;
+ __s32 fd;
+ union {
+ __u64 off;
+ __u64 addr2;
+ };
+ __u64 addr;
+ __u32 len;
+ union {
+ __kernel_rwf_t rw_flags;
+ __u32 fsync_flags;
+ __u16 poll_events;
+ __u32 sync_range_flags;
+ __u32 msg_flags;
+ __u32 timeout_flags;
+ __u32 accept_flags;
+ __u32 cancel_flags;
+ __u32 open_flags;
+ __u32 statx_flags;
+ };
+ __u64 user_data;
+ union {
+ __u16 buf_index;
+ __u64 __pad2[3];
+ };
+};
+
+struct io_uring_cqe
+{
+ __u64 user_data;
+ __s32 res;
+ __u32 flags;
+};
+
+struct io_sqring_offsets
+{
+ __u32 head;
+ __u32 tail;
+ __u32 ring_mask;
+ __u32 ring_entries;
+ __u32 flags;
+ __u32 dropped;
+ __u32 array;
+ __u32 resv1;
+ __u64 resv2;
+};
+
+struct io_cqring_offsets
+{
+ __u32 head;
+ __u32 tail;
+ __u32 ring_mask;
+ __u32 ring_entries;
+ __u32 overflow;
+ __u32 cqes;
+ __u64 resv[2];
+};
+
+struct io_uring_params
+{
+ __u32 sq_entries;
+ __u32 cq_entries;
+ __u32 flags;
+ __u32 sq_thread_cpu;
+ __u32 sq_thread_idle;
+ __u32 features;
+ __u32 resv[4];
+ struct io_sqring_offsets sq_off;
+ struct io_cqring_offsets cq_off;
+};
+
+#define IORING_SETUP_CQSIZE 0x00000008
+
+#define IORING_OP_POLL_ADD 6
+#define IORING_OP_POLL_REMOVE 7
+#define IORING_OP_TIMEOUT 11
+#define IORING_OP_TIMEOUT_REMOVE 12
+
+/* relative or absolute, reference clock is CLOCK_MONOTONIC */
+struct iouring_kernel_timespec
+{
+ int64_t tv_sec;
+ long long tv_nsec;
+};
+
+#define IORING_TIMEOUT_ABS 0x00000001
+
+#define IORING_ENTER_GETEVENTS 0x01
+
+#define IORING_OFF_SQ_RING 0x00000000ULL
+#define IORING_OFF_CQ_RING 0x08000000ULL
+#define IORING_OFF_SQES 0x10000000ULL
+
+#define IORING_FEAT_SINGLE_MMAP 0x00000001
+#define IORING_FEAT_NODROP 0x00000002
+#define IORING_FEAT_SUBMIT_STABLE 0x00000004
+
+inline_size
+int
+evsys_io_uring_setup (unsigned entries, struct io_uring_params *params)
+{
+ return ev_syscall2 (SYS_io_uring_setup, entries, params);
+}
+
+inline_size
+int
+evsys_io_uring_enter (int fd, unsigned to_submit, unsigned min_complete, unsigned flags, const sigset_t *sig, size_t sigsz)
+{
+ return ev_syscall6 (SYS_io_uring_enter, fd, to_submit, min_complete, flags, sig, sigsz);
+}
+
+/*****************************************************************************/
+/* actual backed implementation */
+
+/* we hope that volatile will make the compiler access this variables only once */
+#define EV_SQ_VAR(name) *(volatile unsigned *)((char *)iouring_sq_ring + iouring_sq_ ## name)
+#define EV_CQ_VAR(name) *(volatile unsigned *)((char *)iouring_cq_ring + iouring_cq_ ## name)
+
+/* the index array */
+#define EV_SQ_ARRAY ((unsigned *)((char *)iouring_sq_ring + iouring_sq_array))
+
+/* the submit/completion queue entries */
+#define EV_SQES ((struct io_uring_sqe *) iouring_sqes)
+#define EV_CQES ((struct io_uring_cqe *)((char *)iouring_cq_ring + iouring_cq_cqes))
+
+inline_speed
+int
+iouring_enter (EV_P_ ev_tstamp timeout)
+{
+ int res;
+
+ EV_RELEASE_CB;
+
+ res = evsys_io_uring_enter (iouring_fd, iouring_to_submit, 1,
+ timeout > EV_TS_CONST (0.) ? IORING_ENTER_GETEVENTS : 0, 0, 0);
+
+ assert (("libev: io_uring_enter did not consume all sqes", (res < 0 || res == iouring_to_submit)));
+
+ iouring_to_submit = 0;
+
+ EV_ACQUIRE_CB;
+
+ return res;
+}
+
+/* TODO: can we move things around so we don't need this forward-reference? */
+static void
+iouring_poll (EV_P_ ev_tstamp timeout);
+
+static
+struct io_uring_sqe *
+iouring_sqe_get (EV_P)
+{
+ unsigned tail;
+
+ for (;;)
+ {
+ tail = EV_SQ_VAR (tail);
+
+ if (ecb_expect_true (tail + 1 - EV_SQ_VAR (head) <= EV_SQ_VAR (ring_entries)))
+ break; /* whats the problem, we have free sqes */
+
+ /* queue full, need to flush and possibly handle some events */
+
+#if EV_FEATURE_CODE
+ /* first we ask the kernel nicely, most often this frees up some sqes */
+ int res = iouring_enter (EV_A_ EV_TS_CONST (0.));
+
+ ECB_MEMORY_FENCE_ACQUIRE; /* better safe than sorry */
+
+ if (res >= 0)
+ continue; /* yes, it worked, try again */
+#endif
+
+ /* some problem, possibly EBUSY - do the full poll and let it handle any issues */
+
+ iouring_poll (EV_A_ EV_TS_CONST (0.));
+ /* iouring_poll should have done ECB_MEMORY_FENCE_ACQUIRE for us */
+ }
+
+ /*assert (("libev: io_uring queue full after flush", tail + 1 - EV_SQ_VAR (head) <= EV_SQ_VAR (ring_entries)));*/
+
+ return EV_SQES + (tail & EV_SQ_VAR (ring_mask));
+}
+
+inline_size
+struct io_uring_sqe *
+iouring_sqe_submit (EV_P_ struct io_uring_sqe *sqe)
+{
+ unsigned idx = sqe - EV_SQES;
+
+ EV_SQ_ARRAY [idx] = idx;
+ ECB_MEMORY_FENCE_RELEASE;
+ ++EV_SQ_VAR (tail);
+ // ECB_MEMORY_FENCE_RELEASE; /* for the time being we assume this is not needed */
+ ++iouring_to_submit;
+ return sqe;
+}
+
+/*****************************************************************************/
+
+/* when the timerfd expires we simply note the fact,
+ * as the purpose of the timerfd is to wake us up, nothing else.
+ * the next iteration should re-set it.
+ */
+static void
+iouring_tfd_cb (EV_P_ struct ev_io *w, int revents)
+{
+ iouring_tfd_to = EV_TSTAMP_HUGE;
+}
+
+/* called for full and partial cleanup */
+ecb_cold
+static int
+iouring_internal_destroy (EV_P)
+{
+ close (iouring_tfd);
+ close (iouring_fd);
+
+ if (iouring_sq_ring != MAP_FAILED) munmap (iouring_sq_ring, iouring_sq_ring_size);
+ if (iouring_cq_ring != MAP_FAILED) munmap (iouring_cq_ring, iouring_cq_ring_size);
+ if (iouring_sqes != MAP_FAILED) munmap (iouring_sqes , iouring_sqes_size );
+
+ if (ev_is_active (&iouring_tfd_w))
+ {
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &iouring_tfd_w);
+ }
+
+ return 0;
+}
+
+ecb_cold
+static int
+iouring_internal_init (EV_P)
+{
+ struct io_uring_params params = { 0 };
+
+ iouring_to_submit = 0;
+
+ iouring_tfd = -1;
+ iouring_sq_ring = MAP_FAILED;
+ iouring_cq_ring = MAP_FAILED;
+ iouring_sqes = MAP_FAILED;
+
+ if (!have_monotonic) /* cannot really happen, but what if11 */
+ return -1;
+
+ for (;;)
+ {
+ iouring_fd = evsys_io_uring_setup (iouring_entries, &params);
+
+ if (iouring_fd >= 0)
+ break; /* yippie */
+
+ if (errno != EINVAL)
+ return -1; /* we failed */
+
+#if TODO
+ if ((~params.features) & (IORING_FEAT_NODROP | IORING_FEATURE_SINGLE_MMAP | IORING_FEAT_SUBMIT_STABLE))
+ return -1; /* we require the above features */
+#endif
+
+ /* EINVAL: lots of possible reasons, but maybe
+ * it is because we hit the unqueryable hardcoded size limit
+ */
+
+ /* we hit the limit already, give up */
+ if (iouring_max_entries)
+ return -1;
+
+ /* first time we hit EINVAL? assume we hit the limit, so go back and retry */
+ iouring_entries >>= 1;
+ iouring_max_entries = iouring_entries;
+ }
+
+ iouring_sq_ring_size = params.sq_off.array + params.sq_entries * sizeof (unsigned);
+ iouring_cq_ring_size = params.cq_off.cqes + params.cq_entries * sizeof (struct io_uring_cqe);
+ iouring_sqes_size = params.sq_entries * sizeof (struct io_uring_sqe);
+
+ iouring_sq_ring = mmap (0, iouring_sq_ring_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_SQ_RING);
+ iouring_cq_ring = mmap (0, iouring_cq_ring_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_CQ_RING);
+ iouring_sqes = mmap (0, iouring_sqes_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_SQES);
+
+ if (iouring_sq_ring == MAP_FAILED || iouring_cq_ring == MAP_FAILED || iouring_sqes == MAP_FAILED)
+ return -1;
+
+ iouring_sq_head = params.sq_off.head;
+ iouring_sq_tail = params.sq_off.tail;
+ iouring_sq_ring_mask = params.sq_off.ring_mask;
+ iouring_sq_ring_entries = params.sq_off.ring_entries;
+ iouring_sq_flags = params.sq_off.flags;
+ iouring_sq_dropped = params.sq_off.dropped;
+ iouring_sq_array = params.sq_off.array;
+
+ iouring_cq_head = params.cq_off.head;
+ iouring_cq_tail = params.cq_off.tail;
+ iouring_cq_ring_mask = params.cq_off.ring_mask;
+ iouring_cq_ring_entries = params.cq_off.ring_entries;
+ iouring_cq_overflow = params.cq_off.overflow;
+ iouring_cq_cqes = params.cq_off.cqes;
+
+ iouring_tfd = timerfd_create (CLOCK_MONOTONIC, TFD_CLOEXEC);
+
+ if (iouring_tfd < 0)
+ return iouring_tfd;
+
+ iouring_tfd_to = EV_TSTAMP_HUGE;
+
+ return 0;
+}
+
+ecb_cold
+static void
+iouring_fork (EV_P)
+{
+ iouring_internal_destroy (EV_A);
+
+ while (iouring_internal_init (EV_A) < 0)
+ ev_syserr ("(libev) io_uring_setup");
+
+ fd_rearm_all (EV_A);
+
+ ev_io_stop (EV_A_ &iouring_tfd_w);
+ ev_io_set (EV_A_ &iouring_tfd_w, iouring_tfd, EV_READ);
+ ev_io_start (EV_A_ &iouring_tfd_w);
+}
+
+/*****************************************************************************/
+
+static void
+iouring_modify (EV_P_ int fd, int oev, int nev)
+{
+ if (oev)
+ {
+ /* we assume the sqe's are all "properly" initialised */
+ struct io_uring_sqe *sqe = iouring_sqe_get (EV_A);
+ sqe->opcode = IORING_OP_POLL_REMOVE;
+ sqe->fd = fd;
+ /* Jens Axboe notified me that user_data is not what is documented, but is
+ * some kind of unique ID that has to match, otherwise the request cannot
+ * be removed. Since we don't *really* have that, we pass in the old
+ * generation counter - if that fails, too bad, it will hopefully be removed
+ * at close time and then be ignored. */
+ sqe->addr = (uint32_t)fd | ((__u64)(uint32_t)anfds [fd].egen << 32);
+ sqe->user_data = (uint64_t)-1;
+ iouring_sqe_submit (EV_A_ sqe);
+
+ /* increment generation counter to avoid handling old events */
+ ++anfds [fd].egen;
+ }
+
+ if (nev)
+ {
+ struct io_uring_sqe *sqe = iouring_sqe_get (EV_A);
+ sqe->opcode = IORING_OP_POLL_ADD;
+ sqe->fd = fd;
+ sqe->addr = 0;
+ sqe->user_data = (uint32_t)fd | ((__u64)(uint32_t)anfds [fd].egen << 32);
+ sqe->poll_events =
+ (nev & EV_READ ? POLLIN : 0)
+ | (nev & EV_WRITE ? POLLOUT : 0);
+ iouring_sqe_submit (EV_A_ sqe);
+ }
+}
+
+inline_size
+void
+iouring_tfd_update (EV_P_ ev_tstamp timeout)
+{
+ ev_tstamp tfd_to = mn_now + timeout;
+
+ /* we assume there will be many iterations per timer change, so
+ * we only re-set the timerfd when we have to because its expiry
+ * is too late.
+ */
+ if (ecb_expect_false (tfd_to < iouring_tfd_to))
+ {
+ struct itimerspec its;
+
+ iouring_tfd_to = tfd_to;
+ EV_TS_SET (its.it_interval, 0.);
+ EV_TS_SET (its.it_value, tfd_to);
+
+ if (timerfd_settime (iouring_tfd, TFD_TIMER_ABSTIME, &its, 0) < 0)
+ assert (("libev: iouring timerfd_settime failed", 0));
+ }
+}
+
+inline_size
+void
+iouring_process_cqe (EV_P_ struct io_uring_cqe *cqe)
+{
+ int fd = cqe->user_data & 0xffffffffU;
+ uint32_t gen = cqe->user_data >> 32;
+ int res = cqe->res;
+
+ /* user_data -1 is a remove that we are not atm. interested in */
+ if (cqe->user_data == (uint64_t)-1)
+ return;
+
+ assert (("libev: io_uring fd must be in-bounds", fd >= 0 && fd < anfdmax));
+
+ /* documentation lies, of course. the result value is NOT like
+ * normal syscalls, but like linux raw syscalls, i.e. negative
+ * error numbers. fortunate, as otherwise there would be no way
+ * to get error codes at all. still, why not document this?
+ */
+
+ /* ignore event if generation doesn't match */
+ /* other than skipping removal events, */
+ /* this should actually be very rare */
+ if (ecb_expect_false (gen != (uint32_t)anfds [fd].egen))
+ return;
+
+ if (ecb_expect_false (res < 0))
+ {
+ /*TODO: EINVAL handling (was something failed with this fd)*/
+
+ if (res == -EBADF)
+ {
+ assert (("libev: event loop rejected bad fd", res != -EBADF));
+ fd_kill (EV_A_ fd);
+ }
+ else
+ {
+ errno = -res;
+ ev_syserr ("(libev) IORING_OP_POLL_ADD");
+ }
+
+ return;
+ }
+
+ /* feed events, we do not expect or handle POLLNVAL */
+ fd_event (
+ EV_A_
+ fd,
+ (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+
+ /* io_uring is oneshot, so we need to re-arm the fd next iteration */
+ /* this also means we usually have to do at least one syscall per iteration */
+ anfds [fd].events = 0;
+ fd_change (EV_A_ fd, EV_ANFD_REIFY);
+}
+
+/* called when the event queue overflows */
+ecb_cold
+static void
+iouring_overflow (EV_P)
+{
+ /* we have two options, resize the queue (by tearing down
+ * everything and recreating it, or living with it
+ * and polling.
+ * we implement this by resizing the queue, and, if that fails,
+ * we just recreate the state on every failure, which
+ * kind of is a very inefficient poll.
+ * one danger is, due to the bios toward lower fds,
+ * we will only really get events for those, so
+ * maybe we need a poll() fallback, after all.
+ */
+ /*EV_CQ_VAR (overflow) = 0;*/ /* need to do this if we keep the state and poll manually */
+
+ fd_rearm_all (EV_A);
+
+ /* we double the size until we hit the hard-to-probe maximum */
+ if (!iouring_max_entries)
+ {
+ iouring_entries <<= 1;
+ iouring_fork (EV_A);
+ }
+ else
+ {
+ /* we hit the kernel limit, we should fall back to something else.
+ * we can either poll() a few times and hope for the best,
+ * poll always, or switch to epoll.
+ * TODO: is this necessary with newer kernels?
+ */
+
+ iouring_internal_destroy (EV_A);
+
+ /* this should make it so that on return, we don't call any uring functions */
+ iouring_to_submit = 0;
+
+ for (;;)
+ {
+ backend = epoll_init (EV_A_ 0);
+
+ if (backend)
+ break;
+
+ ev_syserr ("(libev) iouring switch to epoll");
+ }
+ }
+}
+
+/* handle any events in the completion queue, return true if there were any */
+static int
+iouring_handle_cq (EV_P)
+{
+ unsigned head, tail, mask;
+
+ head = EV_CQ_VAR (head);
+ ECB_MEMORY_FENCE_ACQUIRE;
+ tail = EV_CQ_VAR (tail);
+
+ if (head == tail)
+ return 0;
+
+ /* it can only overflow if we have events, yes, yes? */
+ if (ecb_expect_false (EV_CQ_VAR (overflow)))
+ {
+ iouring_overflow (EV_A);
+ return 1;
+ }
+
+ mask = EV_CQ_VAR (ring_mask);
+
+ do
+ iouring_process_cqe (EV_A_ &EV_CQES [head++ & mask]);
+ while (head != tail);
+
+ EV_CQ_VAR (head) = head;
+ ECB_MEMORY_FENCE_RELEASE;
+
+ return 1;
+}
+
+static void
+iouring_poll (EV_P_ ev_tstamp timeout)
+{
+ /* if we have events, no need for extra syscalls, but we might have to queue events */
+ /* we also clar the timeout if there are outstanding fdchanges */
+ /* the latter should only happen if both the sq and cq are full, most likely */
+ /* because we have a lot of event sources that immediately complete */
+ /* TODO: fdchacngecnt is always 0 because fd_reify does not have two buffers yet */
+ if (iouring_handle_cq (EV_A) || fdchangecnt)
+ timeout = EV_TS_CONST (0.);
+ else
+ /* no events, so maybe wait for some */
+ iouring_tfd_update (EV_A_ timeout);
+
+ /* only enter the kernel if we have something to submit, or we need to wait */
+ if (timeout || iouring_to_submit)
+ {
+ int res = iouring_enter (EV_A_ timeout);
+
+ if (ecb_expect_false (res < 0))
+ if (errno == EINTR)
+ /* ignore */;
+ else if (errno == EBUSY)
+ /* cq full, cannot submit - should be rare because we flush the cq first, so simply ignore */;
+ else
+ ev_syserr ("(libev) iouring setup");
+ else
+ iouring_handle_cq (EV_A);
+ }
+}
+
+inline_size
+int
+iouring_init (EV_P_ int flags)
+{
+ iouring_entries = IOURING_INIT_ENTRIES;
+ iouring_max_entries = 0;
+
+ if (iouring_internal_init (EV_A) < 0)
+ {
+ iouring_internal_destroy (EV_A);
+ return 0;
+ }
+
+ ev_io_init (&iouring_tfd_w, iouring_tfd_cb, iouring_tfd, EV_READ);
+ ev_set_priority (&iouring_tfd_w, EV_MINPRI);
+ ev_io_start (EV_A_ &iouring_tfd_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+
+ backend_modify = iouring_modify;
+ backend_poll = iouring_poll;
+
+ return EVBACKEND_IOURING;
+}
+
+inline_size
+void
+iouring_destroy (EV_P)
+{
+ iouring_internal_destroy (EV_A);
+}
+
diff --git a/contrib/libev/ev_kqueue.c b/contrib/libev/ev_kqueue.c
new file mode 100644
index 0000000..69c5147
--- /dev/null
+++ b/contrib/libev/ev_kqueue.c
@@ -0,0 +1,224 @@
+/*
+ * libev kqueue backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2012,2013,2016,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/event.h>
+#include <string.h>
+#include <errno.h>
+
+inline_speed
+void
+kqueue_change (EV_P_ int fd, int filter, int flags, int fflags)
+{
+ ++kqueue_changecnt;
+ array_needsize (struct kevent, kqueue_changes, kqueue_changemax, kqueue_changecnt, array_needsize_noinit);
+
+ EV_SET (&kqueue_changes [kqueue_changecnt - 1], fd, filter, flags, fflags, 0, 0);
+}
+
+/* OS X at least needs this */
+#ifndef EV_ENABLE
+# define EV_ENABLE 0
+#endif
+#ifndef NOTE_EOF
+# define NOTE_EOF 0
+#endif
+
+static void
+kqueue_modify (EV_P_ int fd, int oev, int nev)
+{
+ if (oev != nev)
+ {
+ if (oev & EV_READ)
+ kqueue_change (EV_A_ fd, EVFILT_READ , EV_DELETE, 0);
+
+ if (oev & EV_WRITE)
+ kqueue_change (EV_A_ fd, EVFILT_WRITE, EV_DELETE, 0);
+ }
+
+ /* to detect close/reopen reliably, we have to re-add */
+ /* event requests even when oev == nev */
+
+ if (nev & EV_READ)
+ kqueue_change (EV_A_ fd, EVFILT_READ , EV_ADD | EV_ENABLE, NOTE_EOF);
+
+ if (nev & EV_WRITE)
+ kqueue_change (EV_A_ fd, EVFILT_WRITE, EV_ADD | EV_ENABLE, NOTE_EOF);
+}
+
+static void
+kqueue_poll (EV_P_ ev_tstamp timeout)
+{
+ int res, i;
+ struct timespec ts;
+
+ /* need to resize so there is enough space for errors */
+ if (kqueue_changecnt > kqueue_eventmax)
+ {
+ ev_free (kqueue_events);
+ kqueue_eventmax = array_nextsize (sizeof (struct kevent), kqueue_eventmax, kqueue_changecnt);
+ kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+ }
+
+ EV_RELEASE_CB;
+ EV_TS_SET (ts, timeout);
+ res = kevent (backend_fd, kqueue_changes, kqueue_changecnt, kqueue_events, kqueue_eventmax, &ts);
+ EV_ACQUIRE_CB;
+ kqueue_changecnt = 0;
+
+ if (ecb_expect_false (res < 0))
+ {
+ if (errno != EINTR)
+ ev_syserr ("(libev) kqueue kevent");
+
+ return;
+ }
+
+ for (i = 0; i < res; ++i)
+ {
+ int fd = kqueue_events [i].ident;
+
+ if (ecb_expect_false (kqueue_events [i].flags & EV_ERROR))
+ {
+ int err = kqueue_events [i].data;
+
+ /* we are only interested in errors for fds that we are interested in :) */
+ if (anfds [fd].events)
+ {
+ if (err == ENOENT) /* resubmit changes on ENOENT */
+ kqueue_modify (EV_A_ fd, 0, anfds [fd].events);
+ else if (err == EBADF) /* on EBADF, we re-check the fd */
+ {
+ if (fd_valid (fd))
+ kqueue_modify (EV_A_ fd, 0, anfds [fd].events);
+ else
+ {
+ assert (("libev: kqueue found invalid fd", 0));
+ fd_kill (EV_A_ fd);
+ }
+ }
+ else /* on all other errors, we error out on the fd */
+ {
+ assert (("libev: kqueue found invalid fd", 0));
+ fd_kill (EV_A_ fd);
+ }
+ }
+ }
+ else
+ fd_event (
+ EV_A_
+ fd,
+ kqueue_events [i].filter == EVFILT_READ ? EV_READ
+ : kqueue_events [i].filter == EVFILT_WRITE ? EV_WRITE
+ : 0
+ );
+ }
+
+ if (ecb_expect_false (res == kqueue_eventmax))
+ {
+ ev_free (kqueue_events);
+ kqueue_eventmax = array_nextsize (sizeof (struct kevent), kqueue_eventmax, kqueue_eventmax + 1);
+ kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+ }
+}
+
+inline_size
+int
+kqueue_init (EV_P_ int flags)
+{
+ /* initialize the kernel queue */
+ kqueue_fd_pid = getpid ();
+ if ((backend_fd = kqueue ()) < 0)
+ return 0;
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC); /* not sure if necessary, hopefully doesn't hurt */
+
+ backend_mintime = EV_TS_CONST (1e-9); /* apparently, they did the right thing in freebsd */
+ backend_modify = kqueue_modify;
+ backend_poll = kqueue_poll;
+
+ kqueue_eventmax = 64; /* initial number of events receivable per poll */
+ kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+
+ kqueue_changes = 0;
+ kqueue_changemax = 0;
+ kqueue_changecnt = 0;
+
+ return EVBACKEND_KQUEUE;
+}
+
+inline_size
+void
+kqueue_destroy (EV_P)
+{
+ ev_free (kqueue_events);
+ ev_free (kqueue_changes);
+}
+
+inline_size
+void
+kqueue_fork (EV_P)
+{
+ /* some BSD kernels don't just destroy the kqueue itself,
+ * but also close the fd, which isn't documented, and
+ * impossible to support properly.
+ * we remember the pid of the kqueue call and only close
+ * the fd if the pid is still the same.
+ * this leaks fds on sane kernels, but BSD interfaces are
+ * notoriously buggy and rarely get fixed.
+ */
+ pid_t newpid = getpid ();
+
+ if (newpid == kqueue_fd_pid)
+ close (backend_fd);
+
+ kqueue_fd_pid = newpid;
+ while ((backend_fd = kqueue ()) < 0)
+ ev_syserr ("(libev) kqueue");
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC);
+
+ /* re-register interest in fds */
+ fd_rearm_all (EV_A);
+}
+
+/* sys/event.h defines EV_ERROR */
+#undef EV_ERROR
+
diff --git a/contrib/libev/ev_linuxaio.c b/contrib/libev/ev_linuxaio.c
new file mode 100644
index 0000000..4687a70
--- /dev/null
+++ b/contrib/libev/ev_linuxaio.c
@@ -0,0 +1,620 @@
+/*
+ * libev linux aio fd activity backend
+ *
+ * Copyright (c) 2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/*
+ * general notes about linux aio:
+ *
+ * a) at first, the linux aio IOCB_CMD_POLL functionality introduced in
+ * 4.18 looks too good to be true: both watchers and events can be
+ * batched, and events can even be handled in userspace using
+ * a ring buffer shared with the kernel. watchers can be canceled
+ * regardless of whether the fd has been closed. no problems with fork.
+ * ok, the ring buffer is 200% undocumented (there isn't even a
+ * header file), but otherwise, it's pure bliss!
+ * b) ok, watchers are one-shot, so you have to re-arm active ones
+ * on every iteration. so much for syscall-less event handling,
+ * but at least these re-arms can be batched, no big deal, right?
+ * c) well, linux as usual: the documentation lies to you: io_submit
+ * sometimes returns EINVAL because the kernel doesn't feel like
+ * handling your poll mask - ttys can be polled for POLLOUT,
+ * POLLOUT|POLLIN, but polling for POLLIN fails. just great,
+ * so we have to fall back to something else (hello, epoll),
+ * but at least the fallback can be slow, because these are
+ * exceptional cases, right?
+ * d) hmm, you have to tell the kernel the maximum number of watchers
+ * you want to queue when initialising the aio context. but of
+ * course the real limit is magically calculated in the kernel, and
+ * is often higher then we asked for. so we just have to destroy
+ * the aio context and re-create it a bit larger if we hit the limit.
+ * (starts to remind you of epoll? well, it's a bit more deterministic
+ * and less gambling, but still ugly as hell).
+ * e) that's when you find out you can also hit an arbitrary system-wide
+ * limit. or the kernel simply doesn't want to handle your watchers.
+ * what the fuck do we do then? you guessed it, in the middle
+ * of event handling we have to switch to 100% epoll polling. and
+ * that better is as fast as normal epoll polling, so you practically
+ * have to use the normal epoll backend with all its quirks.
+ * f) end result of this train wreck: it inherits all the disadvantages
+ * from epoll, while adding a number on its own. why even bother to use
+ * it? because if conditions are right and your fds are supported and you
+ * don't hit a limit, this backend is actually faster, doesn't gamble with
+ * your fds, batches watchers and events and doesn't require costly state
+ * recreates. well, until it does.
+ * g) all of this makes this backend use almost twice as much code as epoll.
+ * which in turn uses twice as much code as poll. and that#s not counting
+ * the fact that this backend also depends on the epoll backend, making
+ * it three times as much code as poll, or kqueue.
+ * h) bleah. why can't linux just do kqueue. sure kqueue is ugly, but by now
+ * it's clear that whatever linux comes up with is far, far, far worse.
+ */
+
+#include <sys/time.h> /* actually linux/time.h, but we must assume they are compatible */
+#include <poll.h>
+#include <linux/aio_abi.h>
+
+/*****************************************************************************/
+/* syscall wrapdadoop - this section has the raw api/abi definitions */
+
+#include <sys/syscall.h> /* no glibc wrappers */
+
+/* aio_abi.h is not versioned in any way, so we cannot test for its existance */
+#define IOCB_CMD_POLL 5
+
+/* taken from linux/fs/aio.c. yup, that's a .c file.
+ * not only is this totally undocumented, not even the source code
+ * can tell you what the future semantics of compat_features and
+ * incompat_features are, or what header_length actually is for.
+ */
+#define AIO_RING_MAGIC 0xa10a10a1
+#define EV_AIO_RING_INCOMPAT_FEATURES 0
+struct aio_ring
+{
+ unsigned id; /* kernel internal index number */
+ unsigned nr; /* number of io_events */
+ unsigned head; /* Written to by userland or by kernel. */
+ unsigned tail;
+
+ unsigned magic;
+ unsigned compat_features;
+ unsigned incompat_features;
+ unsigned header_length; /* size of aio_ring */
+
+ struct io_event io_events[0];
+};
+
+inline_size
+int
+evsys_io_setup (unsigned nr_events, aio_context_t *ctx_idp)
+{
+ return ev_syscall2 (SYS_io_setup, nr_events, ctx_idp);
+}
+
+inline_size
+int
+evsys_io_destroy (aio_context_t ctx_id)
+{
+ return ev_syscall1 (SYS_io_destroy, ctx_id);
+}
+
+inline_size
+int
+evsys_io_submit (aio_context_t ctx_id, long nr, struct iocb *cbp[])
+{
+ return ev_syscall3 (SYS_io_submit, ctx_id, nr, cbp);
+}
+
+inline_size
+int
+evsys_io_cancel (aio_context_t ctx_id, struct iocb *cbp, struct io_event *result)
+{
+ return ev_syscall3 (SYS_io_cancel, ctx_id, cbp, result);
+}
+
+inline_size
+int
+evsys_io_getevents (aio_context_t ctx_id, long min_nr, long nr, struct io_event *events, struct timespec *timeout)
+{
+ return ev_syscall5 (SYS_io_getevents, ctx_id, min_nr, nr, events, timeout);
+}
+
+/*****************************************************************************/
+/* actual backed implementation */
+
+ecb_cold
+static int
+linuxaio_nr_events (EV_P)
+{
+ /* we start with 16 iocbs and incraese from there
+ * that's tiny, but the kernel has a rather low system-wide
+ * limit that can be reached quickly, so let's be parsimonious
+ * with this resource.
+ * Rest assured, the kernel generously rounds up small and big numbers
+ * in different ways (but doesn't seem to charge you for it).
+ * The 15 here is because the kernel usually has a power of two as aio-max-nr,
+ * and this helps to take advantage of that limit.
+ */
+
+ /* we try to fill 4kB pages exactly.
+ * the ring buffer header is 32 bytes, every io event is 32 bytes.
+ * the kernel takes the io requests number, doubles it, adds 2
+ * and adds the ring buffer.
+ * the way we use this is by starting low, and then roughly doubling the
+ * size each time we hit a limit.
+ */
+
+ int requests = 15 << linuxaio_iteration;
+ int one_page = (4096
+ / sizeof (struct io_event) ) / 2; /* how many fit into one page */
+ int first_page = ((4096 - sizeof (struct aio_ring))
+ / sizeof (struct io_event) - 2) / 2; /* how many fit into the first page */
+
+ /* if everything fits into one page, use count exactly */
+ if (requests > first_page)
+ /* otherwise, round down to full pages and add the first page */
+ requests = requests / one_page * one_page + first_page;
+
+ return requests;
+}
+
+/* we use out own wrapper structure in case we ever want to do something "clever" */
+typedef struct aniocb
+{
+ struct iocb io;
+ /*int inuse;*/
+} *ANIOCBP;
+
+inline_size
+void
+linuxaio_array_needsize_iocbp (ANIOCBP *base, int offset, int count)
+{
+ while (count--)
+ {
+ /* TODO: quite the overhead to allocate every iocb separately, maybe use our own allocator? */
+ ANIOCBP iocb = (ANIOCBP)ev_malloc (sizeof (*iocb));
+
+ /* full zero initialise is probably not required at the moment, but
+ * this is not well documented, so we better do it.
+ */
+ memset (iocb, 0, sizeof (*iocb));
+
+ iocb->io.aio_lio_opcode = IOCB_CMD_POLL;
+ iocb->io.aio_fildes = offset;
+
+ base [offset++] = iocb;
+ }
+}
+
+ecb_cold
+static void
+linuxaio_free_iocbp (EV_P)
+{
+ while (linuxaio_iocbpmax--)
+ ev_free (linuxaio_iocbps [linuxaio_iocbpmax]);
+
+ linuxaio_iocbpmax = 0; /* next resize will completely reallocate the array, at some overhead */
+}
+
+static void
+linuxaio_modify (EV_P_ int fd, int oev, int nev)
+{
+ array_needsize (ANIOCBP, linuxaio_iocbps, linuxaio_iocbpmax, fd + 1, linuxaio_array_needsize_iocbp);
+ ANIOCBP iocb = linuxaio_iocbps [fd];
+ ANFD *anfd = &anfds [fd];
+
+ if (ecb_expect_false (iocb->io.aio_reqprio < 0))
+ {
+ /* we handed this fd over to epoll, so undo this first */
+ /* we do it manually because the optimisations on epoll_modify won't do us any good */
+ epoll_ctl (backend_fd, EPOLL_CTL_DEL, fd, 0);
+ anfd->emask = 0;
+ iocb->io.aio_reqprio = 0;
+ }
+ else if (ecb_expect_false (iocb->io.aio_buf))
+ {
+ /* iocb active, so cancel it first before resubmit */
+ /* this assumes we only ever get one call per fd per loop iteration */
+ for (;;)
+ {
+ /* on all relevant kernels, io_cancel fails with EINPROGRESS on "success" */
+ if (ecb_expect_false (evsys_io_cancel (linuxaio_ctx, &iocb->io, (struct io_event *)0) == 0))
+ break;
+
+ if (ecb_expect_true (errno == EINPROGRESS))
+ break;
+
+ /* the EINPROGRESS test is for nicer error message. clumsy. */
+ if (errno != EINTR)
+ {
+ assert (("libev: linuxaio unexpected io_cancel failed", errno != EINTR && errno != EINPROGRESS));
+ break;
+ }
+ }
+
+ /* increment generation counter to avoid handling old events */
+ ++anfd->egen;
+ }
+
+ iocb->io.aio_buf = (nev & EV_READ ? POLLIN : 0)
+ | (nev & EV_WRITE ? POLLOUT : 0);
+
+ if (nev)
+ {
+ iocb->io.aio_data = (uint32_t)fd | ((__u64)(uint32_t)anfd->egen << 32);
+
+ /* queue iocb up for io_submit */
+ /* this assumes we only ever get one call per fd per loop iteration */
+ ++linuxaio_submitcnt;
+ array_needsize (struct iocb *, linuxaio_submits, linuxaio_submitmax, linuxaio_submitcnt, array_needsize_noinit);
+ linuxaio_submits [linuxaio_submitcnt - 1] = &iocb->io;
+ }
+}
+
+static void
+linuxaio_epoll_cb (EV_P_ struct ev_io *w, int revents)
+{
+ epoll_poll (EV_A_ 0);
+}
+
+inline_speed
+void
+linuxaio_fd_rearm (EV_P_ int fd)
+{
+ anfds [fd].events = 0;
+ linuxaio_iocbps [fd]->io.aio_buf = 0;
+ fd_change (EV_A_ fd, EV_ANFD_REIFY);
+}
+
+static void
+linuxaio_parse_events (EV_P_ struct io_event *ev, int nr)
+{
+ while (nr)
+ {
+ int fd = ev->data & 0xffffffff;
+ uint32_t gen = ev->data >> 32;
+ int res = ev->res;
+
+ assert (("libev: iocb fd must be in-bounds", fd >= 0 && fd < anfdmax));
+
+ /* only accept events if generation counter matches */
+ if (ecb_expect_true (gen == (uint32_t)anfds [fd].egen))
+ {
+ /* feed events, we do not expect or handle POLLNVAL */
+ fd_event (
+ EV_A_
+ fd,
+ (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+
+ /* linux aio is oneshot: rearm fd. TODO: this does more work than strictly needed */
+ linuxaio_fd_rearm (EV_A_ fd);
+ }
+
+ --nr;
+ ++ev;
+ }
+}
+
+/* get any events from ring buffer, return true if any were handled */
+static int
+linuxaio_get_events_from_ring (EV_P)
+{
+ struct aio_ring *ring = (struct aio_ring *)linuxaio_ctx;
+ unsigned head, tail;
+
+ /* the kernel reads and writes both of these variables, */
+ /* as a C extension, we assume that volatile use here */
+ /* both makes reads atomic and once-only */
+ head = *(volatile unsigned *)&ring->head;
+ ECB_MEMORY_FENCE_ACQUIRE;
+ tail = *(volatile unsigned *)&ring->tail;
+
+ if (head == tail)
+ return 0;
+
+ /* parse all available events, but only once, to avoid starvation */
+ if (ecb_expect_true (tail > head)) /* normal case around */
+ linuxaio_parse_events (EV_A_ ring->io_events + head, tail - head);
+ else /* wrapped around */
+ {
+ linuxaio_parse_events (EV_A_ ring->io_events + head, ring->nr - head);
+ linuxaio_parse_events (EV_A_ ring->io_events, tail);
+ }
+
+ ECB_MEMORY_FENCE_RELEASE;
+ /* as an extension to C, we hope that the volatile will make this atomic and once-only */
+ *(volatile unsigned *)&ring->head = tail;
+
+ return 1;
+}
+
+inline_size
+int
+linuxaio_ringbuf_valid (EV_P)
+{
+ struct aio_ring *ring = (struct aio_ring *)linuxaio_ctx;
+
+ return ecb_expect_true (ring->magic == AIO_RING_MAGIC)
+ && ring->incompat_features == EV_AIO_RING_INCOMPAT_FEATURES
+ && ring->header_length == sizeof (struct aio_ring); /* TODO: or use it to find io_event[0]? */
+}
+
+/* read at least one event from kernel, or timeout */
+inline_size
+void
+linuxaio_get_events (EV_P_ ev_tstamp timeout)
+{
+ struct timespec ts;
+ struct io_event ioev[8]; /* 256 octet stack space */
+ int want = 1; /* how many events to request */
+ int ringbuf_valid = linuxaio_ringbuf_valid (EV_A);
+
+ if (ecb_expect_true (ringbuf_valid))
+ {
+ /* if the ring buffer has any events, we don't wait or call the kernel at all */
+ if (linuxaio_get_events_from_ring (EV_A))
+ return;
+
+ /* if the ring buffer is empty, and we don't have a timeout, then don't call the kernel */
+ if (!timeout)
+ return;
+ }
+ else
+ /* no ringbuffer, request slightly larger batch */
+ want = sizeof (ioev) / sizeof (ioev [0]);
+
+ /* no events, so wait for some
+ * for fairness reasons, we do this in a loop, to fetch all events
+ */
+ for (;;)
+ {
+ int res;
+
+ EV_RELEASE_CB;
+
+ EV_TS_SET (ts, timeout);
+ res = evsys_io_getevents (linuxaio_ctx, 1, want, ioev, &ts);
+
+ EV_ACQUIRE_CB;
+
+ if (res < 0)
+ if (errno == EINTR)
+ /* ignored, retry */;
+ else
+ ev_syserr ("(libev) linuxaio io_getevents");
+ else if (res)
+ {
+ /* at least one event available, handle them */
+ linuxaio_parse_events (EV_A_ ioev, res);
+
+ if (ecb_expect_true (ringbuf_valid))
+ {
+ /* if we have a ring buffer, handle any remaining events in it */
+ linuxaio_get_events_from_ring (EV_A);
+
+ /* at this point, we should have handled all outstanding events */
+ break;
+ }
+ else if (res < want)
+ /* otherwise, if there were fewere events than we wanted, we assume there are no more */
+ break;
+ }
+ else
+ break; /* no events from the kernel, we are done */
+
+ timeout = EV_TS_CONST (0.); /* only wait in the first iteration */
+ }
+}
+
+inline_size
+int
+linuxaio_io_setup (EV_P)
+{
+ linuxaio_ctx = 0;
+ return evsys_io_setup (linuxaio_nr_events (EV_A), &linuxaio_ctx);
+}
+
+static void
+linuxaio_poll (EV_P_ ev_tstamp timeout)
+{
+ int submitted;
+
+ /* first phase: submit new iocbs */
+
+ /* io_submit might return less than the requested number of iocbs */
+ /* this is, afaics, only because of errors, but we go by the book and use a loop, */
+ /* which allows us to pinpoint the erroneous iocb */
+ for (submitted = 0; submitted < linuxaio_submitcnt; )
+ {
+ int res = evsys_io_submit (linuxaio_ctx, linuxaio_submitcnt - submitted, linuxaio_submits + submitted);
+
+ if (ecb_expect_false (res < 0))
+ if (errno == EINVAL)
+ {
+ /* This happens for unsupported fds, officially, but in my testing,
+ * also randomly happens for supported fds. We fall back to good old
+ * poll() here, under the assumption that this is a very rare case.
+ * See https://lore.kernel.org/patchwork/patch/1047453/ to see
+ * discussion about such a case (ttys) where polling for POLLIN
+ * fails but POLLIN|POLLOUT works.
+ */
+ struct iocb *iocb = linuxaio_submits [submitted];
+ epoll_modify (EV_A_ iocb->aio_fildes, 0, anfds [iocb->aio_fildes].events);
+ iocb->aio_reqprio = -1; /* mark iocb as epoll */
+
+ res = 1; /* skip this iocb - another iocb, another chance */
+ }
+ else if (errno == EAGAIN)
+ {
+ /* This happens when the ring buffer is full, or some other shit we
+ * don't know and isn't documented. Most likely because we have too
+ * many requests and linux aio can't be assed to handle them.
+ * In this case, we try to allocate a larger ring buffer, freeing
+ * ours first. This might fail, in which case we have to fall back to 100%
+ * epoll.
+ * God, how I hate linux not getting its act together. Ever.
+ */
+ evsys_io_destroy (linuxaio_ctx);
+ linuxaio_submitcnt = 0;
+
+ /* rearm all fds with active iocbs */
+ {
+ int fd;
+ for (fd = 0; fd < linuxaio_iocbpmax; ++fd)
+ if (linuxaio_iocbps [fd]->io.aio_buf)
+ linuxaio_fd_rearm (EV_A_ fd);
+ }
+
+ ++linuxaio_iteration;
+ if (linuxaio_io_setup (EV_A) < 0)
+ {
+ /* TODO: rearm all and recreate epoll backend from scratch */
+ /* TODO: might be more prudent? */
+
+ /* to bad, we can't get a new aio context, go 100% epoll */
+ linuxaio_free_iocbp (EV_A);
+ ev_io_stop (EV_A_ &linuxaio_epoll_w);
+ ev_ref (EV_A);
+ linuxaio_ctx = 0;
+
+ backend = EVBACKEND_EPOLL;
+ backend_modify = epoll_modify;
+ backend_poll = epoll_poll;
+ }
+
+ timeout = EV_TS_CONST (0.);
+ /* it's easiest to handle this mess in another iteration */
+ return;
+ }
+ else if (errno == EBADF)
+ {
+ assert (("libev: event loop rejected bad fd", errno != EBADF));
+ fd_kill (EV_A_ linuxaio_submits [submitted]->aio_fildes);
+
+ res = 1; /* skip this iocb */
+ }
+ else if (errno == EINTR) /* not seen in reality, not documented */
+ res = 0; /* silently ignore and retry */
+ else
+ {
+ ev_syserr ("(libev) linuxaio io_submit");
+ res = 0;
+ }
+
+ submitted += res;
+ }
+
+ linuxaio_submitcnt = 0;
+
+ /* second phase: fetch and parse events */
+
+ linuxaio_get_events (EV_A_ timeout);
+}
+
+inline_size
+int
+linuxaio_init (EV_P_ int flags)
+{
+ /* would be great to have a nice test for IOCB_CMD_POLL instead */
+ /* also: test some semi-common fd types, such as files and ttys in recommended_backends */
+ /* 4.18 introduced IOCB_CMD_POLL, 4.19 made epoll work, and we need that */
+ if (ev_linux_version () < 0x041300)
+ return 0;
+
+ if (!epoll_init (EV_A_ 0))
+ return 0;
+
+ linuxaio_iteration = 0;
+
+ if (linuxaio_io_setup (EV_A) < 0)
+ {
+ epoll_destroy (EV_A);
+ return 0;
+ }
+
+ ev_io_init (&linuxaio_epoll_w, linuxaio_epoll_cb, backend_fd, EV_READ);
+ ev_set_priority (&linuxaio_epoll_w, EV_MAXPRI);
+ ev_io_start (EV_A_ &linuxaio_epoll_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+
+ backend_modify = linuxaio_modify;
+ backend_poll = linuxaio_poll;
+
+ linuxaio_iocbpmax = 0;
+ linuxaio_iocbps = 0;
+
+ linuxaio_submits = 0;
+ linuxaio_submitmax = 0;
+ linuxaio_submitcnt = 0;
+
+ return EVBACKEND_LINUXAIO;
+}
+
+inline_size
+void
+linuxaio_destroy (EV_P)
+{
+ epoll_destroy (EV_A);
+ linuxaio_free_iocbp (EV_A);
+ evsys_io_destroy (linuxaio_ctx); /* fails in child, aio context is destroyed */
+}
+
+ecb_cold
+static void
+linuxaio_fork (EV_P)
+{
+ linuxaio_submitcnt = 0; /* all pointers were invalidated */
+ linuxaio_free_iocbp (EV_A); /* this frees all iocbs, which is very heavy-handed */
+ evsys_io_destroy (linuxaio_ctx); /* fails in child, aio context is destroyed */
+
+ linuxaio_iteration = 0; /* we start over in the child */
+
+ while (linuxaio_io_setup (EV_A) < 0)
+ ev_syserr ("(libev) linuxaio io_setup");
+
+ /* forking epoll should also effectively unregister all fds from the backend */
+ epoll_fork (EV_A);
+ /* epoll_fork already did this. hopefully */
+ /*fd_rearm_all (EV_A);*/
+
+ ev_io_stop (EV_A_ &linuxaio_epoll_w);
+ ev_io_set (EV_A_ &linuxaio_epoll_w, backend_fd, EV_READ);
+ ev_io_start (EV_A_ &linuxaio_epoll_w);
+}
+
diff --git a/contrib/libev/ev_poll.c b/contrib/libev/ev_poll.c
new file mode 100644
index 0000000..e5508dd
--- /dev/null
+++ b/contrib/libev/ev_poll.c
@@ -0,0 +1,156 @@
+/*
+ * libev poll fd activity backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2016,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#include <poll.h>
+
+inline_size
+void
+array_needsize_pollidx (int *base, int offset, int count)
+{
+ /* using memset (.., -1, ...) is tempting, we we try
+ * to be ultraportable
+ */
+ base += offset;
+ while (count--)
+ *base++ = -1;
+}
+
+static void
+poll_modify (EV_P_ int fd, int oev, int nev)
+{
+ int idx;
+
+ if (oev == nev)
+ return;
+
+ array_needsize (int, pollidxs, pollidxmax, fd + 1, array_needsize_pollidx);
+
+ idx = pollidxs [fd];
+
+ if (idx < 0) /* need to allocate a new pollfd */
+ {
+ pollidxs [fd] = idx = pollcnt++;
+ array_needsize (struct pollfd, polls, pollmax, pollcnt, array_needsize_noinit);
+ polls [idx].fd = fd;
+ }
+
+ assert (polls [idx].fd == fd);
+
+ if (nev)
+ polls [idx].events =
+ (nev & EV_READ ? POLLIN : 0)
+ | (nev & EV_WRITE ? POLLOUT : 0);
+ else /* remove pollfd */
+ {
+ pollidxs [fd] = -1;
+
+ if (ecb_expect_true (idx < --pollcnt))
+ {
+ polls [idx] = polls [pollcnt];
+ pollidxs [polls [idx].fd] = idx;
+ }
+ }
+}
+
+static void
+poll_poll (EV_P_ ev_tstamp timeout)
+{
+ struct pollfd *p;
+ int res;
+
+ EV_RELEASE_CB;
+ res = poll (polls, pollcnt, EV_TS_TO_MSEC (timeout));
+ EV_ACQUIRE_CB;
+
+ if (ecb_expect_false (res < 0))
+ {
+ if (errno == EBADF)
+ fd_ebadf (EV_A);
+ else if (errno == ENOMEM && !syserr_cb)
+ fd_enomem (EV_A);
+ else if (errno != EINTR)
+ ev_syserr ("(libev) poll");
+ }
+ else
+ for (p = polls; res; ++p)
+ {
+ assert (("libev: poll returned illegal result, broken BSD kernel?", p < polls + pollcnt));
+
+ if (ecb_expect_false (p->revents)) /* this expect is debatable */
+ {
+ --res;
+
+ if (ecb_expect_false (p->revents & POLLNVAL))
+ {
+ assert (("libev: poll found invalid fd in poll set", 0));
+ fd_kill (EV_A_ p->fd);
+ }
+ else
+ fd_event (
+ EV_A_
+ p->fd,
+ (p->revents & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (p->revents & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+ }
+ }
+}
+
+inline_size
+int
+poll_init (EV_P_ int flags)
+{
+ backend_mintime = EV_TS_CONST (1e-3);
+ backend_modify = poll_modify;
+ backend_poll = poll_poll;
+
+ pollidxs = 0; pollidxmax = 0;
+ polls = 0; pollmax = 0; pollcnt = 0;
+
+ return EVBACKEND_POLL;
+}
+
+inline_size
+void
+poll_destroy (EV_P)
+{
+ ev_free (pollidxs);
+ ev_free (polls);
+}
+
diff --git a/contrib/libev/ev_port.c b/contrib/libev/ev_port.c
new file mode 100644
index 0000000..f4cd9d9
--- /dev/null
+++ b/contrib/libev/ev_port.c
@@ -0,0 +1,192 @@
+/*
+ * libev solaris event port backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/* useful reading:
+ *
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6268715 (random results)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6455223 (just totally broken)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6873782 (manpage ETIME)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6874410 (implementation ETIME)
+ * http://www.mail-archive.com/networking-discuss@opensolaris.org/msg11898.html ETIME vs. nget
+ * http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/port/gen/event_port.c (libc)
+ * http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/fs/portfs/port.c#1325 (kernel)
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <poll.h>
+#include <port.h>
+#include <string.h>
+#include <errno.h>
+
+inline_speed
+void
+port_associate_and_check (EV_P_ int fd, int ev)
+{
+ if (0 >
+ port_associate (
+ backend_fd, PORT_SOURCE_FD, fd,
+ (ev & EV_READ ? POLLIN : 0)
+ | (ev & EV_WRITE ? POLLOUT : 0),
+ 0
+ )
+ )
+ {
+ if (errno == EBADFD)
+ {
+ assert (("libev: port_associate found invalid fd", errno != EBADFD));
+ fd_kill (EV_A_ fd);
+ }
+ else
+ ev_syserr ("(libev) port_associate");
+ }
+}
+
+static void
+port_modify (EV_P_ int fd, int oev, int nev)
+{
+ /* we need to reassociate no matter what, as closes are
+ * once more silently being discarded.
+ */
+ if (!nev)
+ {
+ if (oev)
+ port_dissociate (backend_fd, PORT_SOURCE_FD, fd);
+ }
+ else
+ port_associate_and_check (EV_A_ fd, nev);
+}
+
+static void
+port_poll (EV_P_ ev_tstamp timeout)
+{
+ int res, i;
+ struct timespec ts;
+ uint_t nget = 1;
+
+ /* we initialise this to something we will skip in the loop, as */
+ /* port_getn can return with nget unchanged, but no indication */
+ /* whether it was the original value or has been updated :/ */
+ port_events [0].portev_source = 0;
+
+ EV_RELEASE_CB;
+ EV_TS_SET (ts, timeout);
+ res = port_getn (backend_fd, port_events, port_eventmax, &nget, &ts);
+ EV_ACQUIRE_CB;
+
+ /* port_getn may or may not set nget on error */
+ /* so we rely on port_events [0].portev_source not being updated */
+ if (res == -1 && errno != ETIME && errno != EINTR)
+ ev_syserr ("(libev) port_getn (see http://bugs.opensolaris.org/view_bug.do?bug_id=6268715, try LIBEV_FLAGS=3 env variable)");
+
+ for (i = 0; i < nget; ++i)
+ {
+ if (port_events [i].portev_source == PORT_SOURCE_FD)
+ {
+ int fd = port_events [i].portev_object;
+
+ fd_event (
+ EV_A_
+ fd,
+ (port_events [i].portev_events & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (port_events [i].portev_events & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+
+ fd_change (EV_A_ fd, EV__IOFDSET);
+ }
+ }
+
+ if (ecb_expect_false (nget == port_eventmax))
+ {
+ ev_free (port_events);
+ port_eventmax = array_nextsize (sizeof (port_event_t), port_eventmax, port_eventmax + 1);
+ port_events = (port_event_t *)ev_malloc (sizeof (port_event_t) * port_eventmax);
+ }
+}
+
+inline_size
+int
+port_init (EV_P_ int flags)
+{
+ /* Initialize the kernel queue */
+ if ((backend_fd = port_create ()) < 0)
+ return 0;
+
+ assert (("libev: PORT_SOURCE_FD must not be zero", PORT_SOURCE_FD));
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC); /* not sure if necessary, hopefully doesn't hurt */
+
+ /* if my reading of the opensolaris kernel sources are correct, then
+ * opensolaris does something very stupid: it checks if the time has already
+ * elapsed and doesn't round up if that is the case, otherwise it DOES round
+ * up. Since we can't know what the case is, we need to guess by using a
+ * "large enough" timeout. Normally, 1e-9 would be correct.
+ */
+ backend_mintime = EV_TS_CONST (1e-3); /* needed to compensate for port_getn returning early */
+ backend_modify = port_modify;
+ backend_poll = port_poll;
+
+ port_eventmax = 64; /* initial number of events receivable per poll */
+ port_events = (port_event_t *)ev_malloc (sizeof (port_event_t) * port_eventmax);
+
+ return EVBACKEND_PORT;
+}
+
+inline_size
+void
+port_destroy (EV_P)
+{
+ ev_free (port_events);
+}
+
+inline_size
+void
+port_fork (EV_P)
+{
+ close (backend_fd);
+
+ while ((backend_fd = port_create ()) < 0)
+ ev_syserr ("(libev) port");
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC);
+
+ /* re-register interest in fds */
+ fd_rearm_all (EV_A);
+}
+
diff --git a/contrib/libev/ev_select.c b/contrib/libev/ev_select.c
new file mode 100644
index 0000000..b862c81
--- /dev/null
+++ b/contrib/libev/ev_select.c
@@ -0,0 +1,316 @@
+/*
+ * libev select fd activity backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef _WIN32
+/* for unix systems */
+# include <inttypes.h>
+# ifndef __hpux
+/* for REAL unix systems */
+# include <sys/select.h>
+# endif
+#endif
+
+#ifndef EV_SELECT_USE_FD_SET
+# ifdef NFDBITS
+# define EV_SELECT_USE_FD_SET 0
+# else
+# define EV_SELECT_USE_FD_SET 1
+# endif
+#endif
+
+#if EV_SELECT_IS_WINSOCKET
+# undef EV_SELECT_USE_FD_SET
+# define EV_SELECT_USE_FD_SET 1
+# undef NFDBITS
+# define NFDBITS 0
+#endif
+
+#if !EV_SELECT_USE_FD_SET
+# define NFDBYTES (NFDBITS / 8)
+#endif
+
+#include <string.h>
+
+static void
+select_modify (EV_P_ int fd, int oev, int nev)
+{
+ if (oev == nev)
+ return;
+
+ {
+#if EV_SELECT_USE_FD_SET
+
+ #if EV_SELECT_IS_WINSOCKET
+ SOCKET handle = anfds [fd].handle;
+ #else
+ int handle = fd;
+ #endif
+
+ assert (("libev: fd >= FD_SETSIZE passed to fd_set-based select backend", fd < FD_SETSIZE));
+
+ /* FD_SET is broken on windows (it adds the fd to a set twice or more,
+ * which eventually leads to overflows). Need to call it only on changes.
+ */
+ #if EV_SELECT_IS_WINSOCKET
+ if ((oev ^ nev) & EV_READ)
+ #endif
+ if (nev & EV_READ)
+ FD_SET (handle, (fd_set *)vec_ri);
+ else
+ FD_CLR (handle, (fd_set *)vec_ri);
+
+ #if EV_SELECT_IS_WINSOCKET
+ if ((oev ^ nev) & EV_WRITE)
+ #endif
+ if (nev & EV_WRITE)
+ FD_SET (handle, (fd_set *)vec_wi);
+ else
+ FD_CLR (handle, (fd_set *)vec_wi);
+
+#else
+
+ int word = fd / NFDBITS;
+ fd_mask mask = 1UL << (fd % NFDBITS);
+
+ if (ecb_expect_false (vec_max <= word))
+ {
+ int new_max = word + 1;
+
+ vec_ri = ev_realloc (vec_ri, new_max * NFDBYTES);
+ vec_ro = ev_realloc (vec_ro, new_max * NFDBYTES); /* could free/malloc */
+ vec_wi = ev_realloc (vec_wi, new_max * NFDBYTES);
+ vec_wo = ev_realloc (vec_wo, new_max * NFDBYTES); /* could free/malloc */
+ #ifdef _WIN32
+ vec_eo = ev_realloc (vec_eo, new_max * NFDBYTES); /* could free/malloc */
+ #endif
+
+ for (; vec_max < new_max; ++vec_max)
+ ((fd_mask *)vec_ri) [vec_max] =
+ ((fd_mask *)vec_wi) [vec_max] = 0;
+ }
+
+ ((fd_mask *)vec_ri) [word] |= mask;
+ if (!(nev & EV_READ))
+ ((fd_mask *)vec_ri) [word] &= ~mask;
+
+ ((fd_mask *)vec_wi) [word] |= mask;
+ if (!(nev & EV_WRITE))
+ ((fd_mask *)vec_wi) [word] &= ~mask;
+#endif
+ }
+}
+
+static void
+select_poll (EV_P_ ev_tstamp timeout)
+{
+ struct timeval tv;
+ int res;
+ int fd_setsize;
+
+ EV_RELEASE_CB;
+ EV_TV_SET (tv, timeout);
+
+#if EV_SELECT_USE_FD_SET
+ fd_setsize = sizeof (fd_set);
+#else
+ fd_setsize = vec_max * NFDBYTES;
+#endif
+
+ memcpy (vec_ro, vec_ri, fd_setsize);
+ memcpy (vec_wo, vec_wi, fd_setsize);
+
+#ifdef _WIN32
+ /* pass in the write set as except set.
+ * the idea behind this is to work around a windows bug that causes
+ * errors to be reported as an exception and not by setting
+ * the writable bit. this is so uncontrollably lame.
+ */
+ memcpy (vec_eo, vec_wi, fd_setsize);
+ res = select (vec_max * NFDBITS, (fd_set *)vec_ro, (fd_set *)vec_wo, (fd_set *)vec_eo, &tv);
+#elif EV_SELECT_USE_FD_SET
+ fd_setsize = anfdmax < FD_SETSIZE ? anfdmax : FD_SETSIZE;
+ res = select (fd_setsize, (fd_set *)vec_ro, (fd_set *)vec_wo, 0, &tv);
+#else
+ res = select (vec_max * NFDBITS, (fd_set *)vec_ro, (fd_set *)vec_wo, 0, &tv);
+#endif
+ EV_ACQUIRE_CB;
+
+ if (ecb_expect_false (res < 0))
+ {
+ #if EV_SELECT_IS_WINSOCKET
+ errno = WSAGetLastError ();
+ #endif
+ #ifdef WSABASEERR
+ /* on windows, select returns incompatible error codes, fix this */
+ if (errno >= WSABASEERR && errno < WSABASEERR + 1000)
+ if (errno == WSAENOTSOCK)
+ errno = EBADF;
+ else
+ errno -= WSABASEERR;
+ #endif
+
+ #ifdef _WIN32
+ /* select on windows erroneously returns EINVAL when no fd sets have been
+ * provided (this is documented). what microsoft doesn't tell you that this bug
+ * exists even when the fd sets _are_ provided, so we have to check for this bug
+ * here and emulate by sleeping manually.
+ * we also get EINVAL when the timeout is invalid, but we ignore this case here
+ * and assume that EINVAL always means: you have to wait manually.
+ */
+ if (errno == EINVAL)
+ {
+ if (timeout)
+ {
+ unsigned long ms = EV_TS_TO_MSEC (timeout);
+ Sleep (ms ? ms : 1);
+ }
+
+ return;
+ }
+ #endif
+
+ if (errno == EBADF)
+ fd_ebadf (EV_A);
+ else if (errno == ENOMEM && !syserr_cb)
+ fd_enomem (EV_A);
+ else if (errno != EINTR)
+ ev_syserr ("(libev) select");
+
+ return;
+ }
+
+#if EV_SELECT_USE_FD_SET
+
+ {
+ int fd;
+
+ for (fd = 0; fd < anfdmax; ++fd)
+ if (anfds [fd].events)
+ {
+ int events = 0;
+ #if EV_SELECT_IS_WINSOCKET
+ SOCKET handle = anfds [fd].handle;
+ #else
+ int handle = fd;
+ #endif
+
+ if (FD_ISSET (handle, (fd_set *)vec_ro)) events |= EV_READ;
+ if (FD_ISSET (handle, (fd_set *)vec_wo)) events |= EV_WRITE;
+ #ifdef _WIN32
+ if (FD_ISSET (handle, (fd_set *)vec_eo)) events |= EV_WRITE;
+ #endif
+
+ if (ecb_expect_true (events))
+ fd_event (EV_A_ fd, events);
+ }
+ }
+
+#else
+
+ {
+ int word, bit;
+ for (word = vec_max; word--; )
+ {
+ fd_mask word_r = ((fd_mask *)vec_ro) [word];
+ fd_mask word_w = ((fd_mask *)vec_wo) [word];
+ #ifdef _WIN32
+ word_w |= ((fd_mask *)vec_eo) [word];
+ #endif
+
+ if (word_r || word_w)
+ for (bit = NFDBITS; bit--; )
+ {
+ fd_mask mask = 1UL << bit;
+ int events = 0;
+
+ events |= word_r & mask ? EV_READ : 0;
+ events |= word_w & mask ? EV_WRITE : 0;
+
+ if (ecb_expect_true (events))
+ fd_event (EV_A_ word * NFDBITS + bit, events);
+ }
+ }
+ }
+
+#endif
+}
+
+inline_size
+int
+select_init (EV_P_ int flags)
+{
+ backend_mintime = EV_TS_CONST (1e-6);
+ backend_modify = select_modify;
+ backend_poll = select_poll;
+
+#if EV_SELECT_USE_FD_SET
+ vec_ri = ev_malloc (sizeof (fd_set)); FD_ZERO ((fd_set *)vec_ri);
+ vec_ro = ev_malloc (sizeof (fd_set));
+ vec_wi = ev_malloc (sizeof (fd_set)); FD_ZERO ((fd_set *)vec_wi);
+ vec_wo = ev_malloc (sizeof (fd_set));
+ #ifdef _WIN32
+ vec_eo = ev_malloc (sizeof (fd_set));
+ #endif
+#else
+ vec_max = 0;
+ vec_ri = 0;
+ vec_ro = 0;
+ vec_wi = 0;
+ vec_wo = 0;
+ #ifdef _WIN32
+ vec_eo = 0;
+ #endif
+#endif
+
+ return EVBACKEND_SELECT;
+}
+
+inline_size
+void
+select_destroy (EV_P)
+{
+ ev_free (vec_ri);
+ ev_free (vec_ro);
+ ev_free (vec_wi);
+ ev_free (vec_wo);
+ #ifdef _WIN32
+ ev_free (vec_eo);
+ #endif
+}
+
diff --git a/contrib/libev/ev_vars.h b/contrib/libev/ev_vars.h
new file mode 100644
index 0000000..fb0c583
--- /dev/null
+++ b/contrib/libev/ev_vars.h
@@ -0,0 +1,249 @@
+/*
+ * loop member variable declarations
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2012,2013,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#define VARx(type,name) VAR(name, type name)
+
+VARx(ev_tstamp, now_floor) /* last time we refreshed rt_time */
+VARx(ev_tstamp, mn_now) /* monotonic clock "now" */
+VARx(ev_tstamp, rtmn_diff) /* difference realtime - monotonic time */
+
+/* for reverse feeding of events */
+VARx(W *, rfeeds)
+VARx(int, rfeedmax)
+VARx(int, rfeedcnt)
+
+VAR (pendings, ANPENDING *pendings [NUMPRI])
+VAR (pendingmax, int pendingmax [NUMPRI])
+VAR (pendingcnt, int pendingcnt [NUMPRI])
+VARx(int, pendingpri) /* highest priority currently pending */
+VARx(ev_prepare, pending_w) /* dummy pending watcher */
+
+VARx(ev_tstamp, io_blocktime)
+VARx(ev_tstamp, timeout_blocktime)
+
+VARx(int, backend)
+VARx(int, activecnt) /* total number of active events ("refcount") */
+VARx(EV_ATOMIC_T, loop_done) /* signal by ev_break */
+
+VARx(int, backend_fd)
+VARx(ev_tstamp, backend_mintime) /* assumed typical timer resolution */
+VAR (backend_modify, void (*backend_modify)(EV_P_ int fd, int oev, int nev))
+VAR (backend_poll , void (*backend_poll)(EV_P_ ev_tstamp timeout))
+
+VARx(ANFD *, anfds)
+VARx(int, anfdmax)
+
+VAR (evpipe, int evpipe [2])
+VARx(ev_io, pipe_w)
+VARx(EV_ATOMIC_T, pipe_write_wanted)
+VARx(EV_ATOMIC_T, pipe_write_skipped)
+
+#if !defined(_WIN32) || EV_GENWRAP
+VARx(pid_t, curpid)
+#endif
+
+VARx(char, postfork) /* true if we need to recreate kernel state after fork */
+
+#if EV_USE_SELECT || EV_GENWRAP
+VARx(void *, vec_ri)
+VARx(void *, vec_ro)
+VARx(void *, vec_wi)
+VARx(void *, vec_wo)
+#if defined(_WIN32) || EV_GENWRAP
+VARx(void *, vec_eo)
+#endif
+VARx(int, vec_max)
+#endif
+
+#if EV_USE_POLL || EV_GENWRAP
+VARx(struct pollfd *, polls)
+VARx(int, pollmax)
+VARx(int, pollcnt)
+VARx(int *, pollidxs) /* maps fds into structure indices */
+VARx(int, pollidxmax)
+#endif
+
+#if EV_USE_EPOLL || EV_GENWRAP
+VARx(struct epoll_event *, epoll_events)
+VARx(int, epoll_eventmax)
+VARx(int *, epoll_eperms)
+VARx(int, epoll_epermcnt)
+VARx(int, epoll_epermmax)
+#endif
+
+#if EV_USE_LINUXAIO || EV_GENWRAP
+VARx(aio_context_t, linuxaio_ctx)
+VARx(int, linuxaio_iteration)
+VARx(struct aniocb **, linuxaio_iocbps)
+VARx(int, linuxaio_iocbpmax)
+VARx(struct iocb **, linuxaio_submits)
+VARx(int, linuxaio_submitcnt)
+VARx(int, linuxaio_submitmax)
+VARx(ev_io, linuxaio_epoll_w)
+#endif
+
+#if EV_USE_IOURING || EV_GENWRAP
+VARx(int, iouring_fd)
+VARx(unsigned, iouring_to_submit);
+VARx(int, iouring_entries)
+VARx(int, iouring_max_entries)
+VARx(void *, iouring_sq_ring)
+VARx(void *, iouring_cq_ring)
+VARx(void *, iouring_sqes)
+VARx(uint32_t, iouring_sq_ring_size)
+VARx(uint32_t, iouring_cq_ring_size)
+VARx(uint32_t, iouring_sqes_size)
+VARx(uint32_t, iouring_sq_head)
+VARx(uint32_t, iouring_sq_tail)
+VARx(uint32_t, iouring_sq_ring_mask)
+VARx(uint32_t, iouring_sq_ring_entries)
+VARx(uint32_t, iouring_sq_flags)
+VARx(uint32_t, iouring_sq_dropped)
+VARx(uint32_t, iouring_sq_array)
+VARx(uint32_t, iouring_cq_head)
+VARx(uint32_t, iouring_cq_tail)
+VARx(uint32_t, iouring_cq_ring_mask)
+VARx(uint32_t, iouring_cq_ring_entries)
+VARx(uint32_t, iouring_cq_overflow)
+VARx(uint32_t, iouring_cq_cqes)
+VARx(ev_tstamp, iouring_tfd_to)
+VARx(int, iouring_tfd)
+VARx(ev_io, iouring_tfd_w)
+#endif
+
+#if EV_USE_KQUEUE || EV_GENWRAP
+VARx(pid_t, kqueue_fd_pid)
+VARx(struct kevent *, kqueue_changes)
+VARx(int, kqueue_changemax)
+VARx(int, kqueue_changecnt)
+VARx(struct kevent *, kqueue_events)
+VARx(int, kqueue_eventmax)
+#endif
+
+#if EV_USE_PORT || EV_GENWRAP
+VARx(struct port_event *, port_events)
+VARx(int, port_eventmax)
+#endif
+
+#if EV_USE_IOCP || EV_GENWRAP
+VARx(HANDLE, iocp)
+#endif
+
+VARx(int *, fdchanges)
+VARx(int, fdchangemax)
+VARx(int, fdchangecnt)
+
+VARx(ANHE *, timers)
+VARx(int, timermax)
+VARx(int, timercnt)
+
+#if EV_PERIODIC_ENABLE || EV_GENWRAP
+VARx(ANHE *, periodics)
+VARx(int, periodicmax)
+VARx(int, periodiccnt)
+#endif
+
+#if EV_IDLE_ENABLE || EV_GENWRAP
+VAR (idles, ev_idle **idles [NUMPRI])
+VAR (idlemax, int idlemax [NUMPRI])
+VAR (idlecnt, int idlecnt [NUMPRI])
+#endif
+VARx(int, idleall) /* total number */
+
+VARx(struct ev_prepare **, prepares)
+VARx(int, preparemax)
+VARx(int, preparecnt)
+
+VARx(struct ev_check **, checks)
+VARx(int, checkmax)
+VARx(int, checkcnt)
+
+#if EV_FORK_ENABLE || EV_GENWRAP
+VARx(struct ev_fork **, forks)
+VARx(int, forkmax)
+VARx(int, forkcnt)
+#endif
+
+#if EV_CLEANUP_ENABLE || EV_GENWRAP
+VARx(struct ev_cleanup **, cleanups)
+VARx(int, cleanupmax)
+VARx(int, cleanupcnt)
+#endif
+
+#if EV_ASYNC_ENABLE || EV_GENWRAP
+VARx(EV_ATOMIC_T, async_pending)
+VARx(struct ev_async **, asyncs)
+VARx(int, asyncmax)
+VARx(int, asynccnt)
+#endif
+
+#if EV_USE_INOTIFY || EV_GENWRAP
+VARx(int, fs_fd)
+VARx(ev_io, fs_w)
+VARx(char, fs_2625) /* whether we are running in linux 2.6.25 or newer */
+VAR (fs_hash, ANFS fs_hash [EV_INOTIFY_HASHSIZE])
+#endif
+
+VARx(EV_ATOMIC_T, sig_pending)
+#if EV_USE_SIGNALFD || EV_GENWRAP
+VARx(int, sigfd)
+VARx(ev_io, sigfd_w)
+VARx(sigset_t, sigfd_set)
+#endif
+
+#if EV_USE_TIMERFD || EV_GENWRAP
+VARx(int, timerfd) /* timerfd for time jump detection */
+VARx(ev_io, timerfd_w)
+#endif
+
+VARx(unsigned int, origflags) /* original loop flags */
+
+#if EV_FEATURE_API || EV_GENWRAP
+VARx(unsigned int, loop_count) /* total number of loop iterations/blocks */
+VARx(unsigned int, loop_depth) /* #ev_run enters - #ev_run leaves */
+
+VARx(void *, userdata)
+/* C++ doesn't support the ev_loop_callback typedef here. stinks. */
+VAR (release_cb, void (*release_cb)(EV_P) EV_NOEXCEPT)
+VAR (acquire_cb, void (*acquire_cb)(EV_P) EV_NOEXCEPT)
+VAR (invoke_cb , ev_loop_callback invoke_cb)
+#endif
+
+#undef VARx
+
diff --git a/contrib/libev/ev_win32.c b/contrib/libev/ev_win32.c
new file mode 100644
index 0000000..97344c3
--- /dev/null
+++ b/contrib/libev/ev_win32.c
@@ -0,0 +1,162 @@
+/*
+ * libev win32 compatibility cruft (_not_ a backend)
+ *
+ * Copyright (c) 2007,2008,2009 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifdef _WIN32
+
+/* note: the comment below could not be substantiated, but what would I care */
+/* MSDN says this is required to handle SIGFPE */
+/* my wild guess would be that using something floating-pointy is required */
+/* for the crt to do something about it */
+volatile double SIGFPE_REQ = 0.0f;
+
+static SOCKET
+ev_tcp_socket (void)
+{
+#if EV_USE_WSASOCKET
+ return WSASocket (AF_INET, SOCK_STREAM, 0, 0, 0, 0);
+#else
+ return socket (AF_INET, SOCK_STREAM, 0);
+#endif
+}
+
+/* oh, the humanity! */
+static int
+ev_pipe (int filedes [2])
+{
+ struct sockaddr_in addr = { 0 };
+ int addr_size = sizeof (addr);
+ struct sockaddr_in adr2;
+ int adr2_size = sizeof (adr2);
+ SOCKET listener;
+ SOCKET sock [2] = { -1, -1 };
+
+ if ((listener = ev_tcp_socket ()) == INVALID_SOCKET)
+ return -1;
+
+ addr.sin_family = AF_INET;
+ addr.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
+ addr.sin_port = 0;
+
+ if (bind (listener, (struct sockaddr *)&addr, addr_size))
+ goto fail;
+
+ if (getsockname (listener, (struct sockaddr *)&addr, &addr_size))
+ goto fail;
+
+ if (listen (listener, 1))
+ goto fail;
+
+ if ((sock [0] = ev_tcp_socket ()) == INVALID_SOCKET)
+ goto fail;
+
+ if (connect (sock [0], (struct sockaddr *)&addr, addr_size))
+ goto fail;
+
+ /* TODO: returns INVALID_SOCKET on winsock accept, not < 0. fix it */
+ /* when convenient, probably by just removing error checking altogether? */
+ if ((sock [1] = accept (listener, 0, 0)) < 0)
+ goto fail;
+
+ /* windows vista returns fantasy port numbers for sockets:
+ * example for two interconnected tcp sockets:
+ *
+ * (Socket::unpack_sockaddr_in getsockname $sock0)[0] == 53364
+ * (Socket::unpack_sockaddr_in getpeername $sock0)[0] == 53363
+ * (Socket::unpack_sockaddr_in getsockname $sock1)[0] == 53363
+ * (Socket::unpack_sockaddr_in getpeername $sock1)[0] == 53365
+ *
+ * wow! tridirectional sockets!
+ *
+ * this way of checking ports seems to work:
+ */
+ if (getpeername (sock [0], (struct sockaddr *)&addr, &addr_size))
+ goto fail;
+
+ if (getsockname (sock [1], (struct sockaddr *)&adr2, &adr2_size))
+ goto fail;
+
+ errno = WSAEINVAL;
+ if (addr_size != adr2_size
+ || addr.sin_addr.s_addr != adr2.sin_addr.s_addr /* just to be sure, I mean, it's windows */
+ || addr.sin_port != adr2.sin_port)
+ goto fail;
+
+ closesocket (listener);
+
+#if EV_SELECT_IS_WINSOCKET
+ filedes [0] = EV_WIN32_HANDLE_TO_FD (sock [0]);
+ filedes [1] = EV_WIN32_HANDLE_TO_FD (sock [1]);
+#else
+ /* when select isn't winsocket, we also expect socket, connect, accept etc.
+ * to work on fds */
+ filedes [0] = sock [0];
+ filedes [1] = sock [1];
+#endif
+
+ return 0;
+
+fail:
+ closesocket (listener);
+
+ if (sock [0] != INVALID_SOCKET) closesocket (sock [0]);
+ if (sock [1] != INVALID_SOCKET) closesocket (sock [1]);
+
+ return -1;
+}
+
+#undef pipe
+#define pipe(filedes) ev_pipe (filedes)
+
+#define EV_HAVE_EV_TIME 1
+ev_tstamp
+ev_time (void)
+{
+ FILETIME ft;
+ ULARGE_INTEGER ui;
+
+ GetSystemTimeAsFileTime (&ft);
+ ui.u.LowPart = ft.dwLowDateTime;
+ ui.u.HighPart = ft.dwHighDateTime;
+
+ /* also, msvc cannot convert ulonglong to double... yes, it is that sucky */
+ return EV_TS_FROM_USEC (((LONGLONG)(ui.QuadPart - 116444736000000000) * 1e-1));
+}
+
+#endif
+
diff --git a/contrib/libev/ev_wrap.h b/contrib/libev/ev_wrap.h
new file mode 100644
index 0000000..45d793c
--- /dev/null
+++ b/contrib/libev/ev_wrap.h
@@ -0,0 +1,272 @@
+/* DO NOT EDIT, automatically generated by update_ev_wrap */
+#ifndef EV_WRAP_H
+#define EV_WRAP_H
+#define acquire_cb ((loop)->acquire_cb)
+#define activecnt ((loop)->activecnt)
+#define anfdmax ((loop)->anfdmax)
+#define anfds ((loop)->anfds)
+#define async_pending ((loop)->async_pending)
+#define asynccnt ((loop)->asynccnt)
+#define asyncmax ((loop)->asyncmax)
+#define asyncs ((loop)->asyncs)
+#define backend ((loop)->backend)
+#define backend_fd ((loop)->backend_fd)
+#define backend_mintime ((loop)->backend_mintime)
+#define backend_modify ((loop)->backend_modify)
+#define backend_poll ((loop)->backend_poll)
+#define checkcnt ((loop)->checkcnt)
+#define checkmax ((loop)->checkmax)
+#define checks ((loop)->checks)
+#define cleanupcnt ((loop)->cleanupcnt)
+#define cleanupmax ((loop)->cleanupmax)
+#define cleanups ((loop)->cleanups)
+#define curpid ((loop)->curpid)
+#define epoll_epermcnt ((loop)->epoll_epermcnt)
+#define epoll_epermmax ((loop)->epoll_epermmax)
+#define epoll_eperms ((loop)->epoll_eperms)
+#define epoll_eventmax ((loop)->epoll_eventmax)
+#define epoll_events ((loop)->epoll_events)
+#define evpipe ((loop)->evpipe)
+#define fdchangecnt ((loop)->fdchangecnt)
+#define fdchangemax ((loop)->fdchangemax)
+#define fdchanges ((loop)->fdchanges)
+#define forkcnt ((loop)->forkcnt)
+#define forkmax ((loop)->forkmax)
+#define forks ((loop)->forks)
+#define fs_2625 ((loop)->fs_2625)
+#define fs_fd ((loop)->fs_fd)
+#define fs_hash ((loop)->fs_hash)
+#define fs_w ((loop)->fs_w)
+#define idleall ((loop)->idleall)
+#define idlecnt ((loop)->idlecnt)
+#define idlemax ((loop)->idlemax)
+#define idles ((loop)->idles)
+#define invoke_cb ((loop)->invoke_cb)
+#define io_blocktime ((loop)->io_blocktime)
+#define iocp ((loop)->iocp)
+#define iouring_cq_cqes ((loop)->iouring_cq_cqes)
+#define iouring_cq_head ((loop)->iouring_cq_head)
+#define iouring_cq_overflow ((loop)->iouring_cq_overflow)
+#define iouring_cq_ring ((loop)->iouring_cq_ring)
+#define iouring_cq_ring_entries ((loop)->iouring_cq_ring_entries)
+#define iouring_cq_ring_mask ((loop)->iouring_cq_ring_mask)
+#define iouring_cq_ring_size ((loop)->iouring_cq_ring_size)
+#define iouring_cq_tail ((loop)->iouring_cq_tail)
+#define iouring_entries ((loop)->iouring_entries)
+#define iouring_fd ((loop)->iouring_fd)
+#define iouring_max_entries ((loop)->iouring_max_entries)
+#define iouring_sq_array ((loop)->iouring_sq_array)
+#define iouring_sq_dropped ((loop)->iouring_sq_dropped)
+#define iouring_sq_flags ((loop)->iouring_sq_flags)
+#define iouring_sq_head ((loop)->iouring_sq_head)
+#define iouring_sq_ring ((loop)->iouring_sq_ring)
+#define iouring_sq_ring_entries ((loop)->iouring_sq_ring_entries)
+#define iouring_sq_ring_mask ((loop)->iouring_sq_ring_mask)
+#define iouring_sq_ring_size ((loop)->iouring_sq_ring_size)
+#define iouring_sq_tail ((loop)->iouring_sq_tail)
+#define iouring_sqes ((loop)->iouring_sqes)
+#define iouring_sqes_size ((loop)->iouring_sqes_size)
+#define iouring_tfd ((loop)->iouring_tfd)
+#define iouring_tfd_to ((loop)->iouring_tfd_to)
+#define iouring_tfd_w ((loop)->iouring_tfd_w)
+#define iouring_to_submit ((loop)->iouring_to_submit)
+#define kqueue_changecnt ((loop)->kqueue_changecnt)
+#define kqueue_changemax ((loop)->kqueue_changemax)
+#define kqueue_changes ((loop)->kqueue_changes)
+#define kqueue_eventmax ((loop)->kqueue_eventmax)
+#define kqueue_events ((loop)->kqueue_events)
+#define kqueue_fd_pid ((loop)->kqueue_fd_pid)
+#define linuxaio_ctx ((loop)->linuxaio_ctx)
+#define linuxaio_epoll_w ((loop)->linuxaio_epoll_w)
+#define linuxaio_iocbpmax ((loop)->linuxaio_iocbpmax)
+#define linuxaio_iocbps ((loop)->linuxaio_iocbps)
+#define linuxaio_iteration ((loop)->linuxaio_iteration)
+#define linuxaio_submitcnt ((loop)->linuxaio_submitcnt)
+#define linuxaio_submitmax ((loop)->linuxaio_submitmax)
+#define linuxaio_submits ((loop)->linuxaio_submits)
+#define loop_count ((loop)->loop_count)
+#define loop_depth ((loop)->loop_depth)
+#define loop_done ((loop)->loop_done)
+#define mn_now ((loop)->mn_now)
+#define now_floor ((loop)->now_floor)
+#define origflags ((loop)->origflags)
+#define pending_w ((loop)->pending_w)
+#define pendingcnt ((loop)->pendingcnt)
+#define pendingmax ((loop)->pendingmax)
+#define pendingpri ((loop)->pendingpri)
+#define pendings ((loop)->pendings)
+#define periodiccnt ((loop)->periodiccnt)
+#define periodicmax ((loop)->periodicmax)
+#define periodics ((loop)->periodics)
+#define pipe_w ((loop)->pipe_w)
+#define pipe_write_skipped ((loop)->pipe_write_skipped)
+#define pipe_write_wanted ((loop)->pipe_write_wanted)
+#define pollcnt ((loop)->pollcnt)
+#define pollidxmax ((loop)->pollidxmax)
+#define pollidxs ((loop)->pollidxs)
+#define pollmax ((loop)->pollmax)
+#define polls ((loop)->polls)
+#define port_eventmax ((loop)->port_eventmax)
+#define port_events ((loop)->port_events)
+#define postfork ((loop)->postfork)
+#define preparecnt ((loop)->preparecnt)
+#define preparemax ((loop)->preparemax)
+#define prepares ((loop)->prepares)
+#define release_cb ((loop)->release_cb)
+#define rfeedcnt ((loop)->rfeedcnt)
+#define rfeedmax ((loop)->rfeedmax)
+#define rfeeds ((loop)->rfeeds)
+#define rtmn_diff ((loop)->rtmn_diff)
+#define sig_pending ((loop)->sig_pending)
+#define sigfd ((loop)->sigfd)
+#define sigfd_set ((loop)->sigfd_set)
+#define sigfd_w ((loop)->sigfd_w)
+#define timeout_blocktime ((loop)->timeout_blocktime)
+#define timercnt ((loop)->timercnt)
+#define timerfd ((loop)->timerfd)
+#define timerfd_w ((loop)->timerfd_w)
+#define timermax ((loop)->timermax)
+#define timers ((loop)->timers)
+#define userdata ((loop)->userdata)
+#define vec_eo ((loop)->vec_eo)
+#define vec_max ((loop)->vec_max)
+#define vec_ri ((loop)->vec_ri)
+#define vec_ro ((loop)->vec_ro)
+#define vec_wi ((loop)->vec_wi)
+#define vec_wo ((loop)->vec_wo)
+#else
+#undef EV_WRAP_H
+#undef acquire_cb
+#undef activecnt
+#undef anfdmax
+#undef anfds
+#undef async_pending
+#undef asynccnt
+#undef asyncmax
+#undef asyncs
+#undef backend
+#undef backend_fd
+#undef backend_mintime
+#undef backend_modify
+#undef backend_poll
+#undef checkcnt
+#undef checkmax
+#undef checks
+#undef cleanupcnt
+#undef cleanupmax
+#undef cleanups
+#undef curpid
+#undef epoll_epermcnt
+#undef epoll_epermmax
+#undef epoll_eperms
+#undef epoll_eventmax
+#undef epoll_events
+#undef evpipe
+#undef fdchangecnt
+#undef fdchangemax
+#undef fdchanges
+#undef forkcnt
+#undef forkmax
+#undef forks
+#undef fs_2625
+#undef fs_fd
+#undef fs_hash
+#undef fs_w
+#undef idleall
+#undef idlecnt
+#undef idlemax
+#undef idles
+#undef invoke_cb
+#undef io_blocktime
+#undef iocp
+#undef iouring_cq_cqes
+#undef iouring_cq_head
+#undef iouring_cq_overflow
+#undef iouring_cq_ring
+#undef iouring_cq_ring_entries
+#undef iouring_cq_ring_mask
+#undef iouring_cq_ring_size
+#undef iouring_cq_tail
+#undef iouring_entries
+#undef iouring_fd
+#undef iouring_max_entries
+#undef iouring_sq_array
+#undef iouring_sq_dropped
+#undef iouring_sq_flags
+#undef iouring_sq_head
+#undef iouring_sq_ring
+#undef iouring_sq_ring_entries
+#undef iouring_sq_ring_mask
+#undef iouring_sq_ring_size
+#undef iouring_sq_tail
+#undef iouring_sqes
+#undef iouring_sqes_size
+#undef iouring_tfd
+#undef iouring_tfd_to
+#undef iouring_tfd_w
+#undef iouring_to_submit
+#undef kqueue_changecnt
+#undef kqueue_changemax
+#undef kqueue_changes
+#undef kqueue_eventmax
+#undef kqueue_events
+#undef kqueue_fd_pid
+#undef linuxaio_ctx
+#undef linuxaio_epoll_w
+#undef linuxaio_iocbpmax
+#undef linuxaio_iocbps
+#undef linuxaio_iteration
+#undef linuxaio_submitcnt
+#undef linuxaio_submitmax
+#undef linuxaio_submits
+#undef loop_count
+#undef loop_depth
+#undef loop_done
+#undef mn_now
+#undef now_floor
+#undef origflags
+#undef pending_w
+#undef pendingcnt
+#undef pendingmax
+#undef pendingpri
+#undef pendings
+#undef periodiccnt
+#undef periodicmax
+#undef periodics
+#undef pipe_w
+#undef pipe_write_skipped
+#undef pipe_write_wanted
+#undef pollcnt
+#undef pollidxmax
+#undef pollidxs
+#undef pollmax
+#undef polls
+#undef port_eventmax
+#undef port_events
+#undef postfork
+#undef preparecnt
+#undef preparemax
+#undef prepares
+#undef release_cb
+#undef rfeedcnt
+#undef rfeedmax
+#undef rfeeds
+#undef rtmn_diff
+#undef sig_pending
+#undef sigfd
+#undef sigfd_set
+#undef sigfd_w
+#undef timeout_blocktime
+#undef timercnt
+#undef timerfd
+#undef timerfd_w
+#undef timermax
+#undef timers
+#undef userdata
+#undef vec_eo
+#undef vec_max
+#undef vec_ri
+#undef vec_ro
+#undef vec_wi
+#undef vec_wo
+#endif
diff --git a/contrib/libottery/CMakeLists.txt b/contrib/libottery/CMakeLists.txt
new file mode 100644
index 0000000..b8536f2
--- /dev/null
+++ b/contrib/libottery/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(OTTERYSRC chacha_merged.c
+ ottery.c
+ ottery_cpuinfo.c
+ ottery_entropy.c
+ ottery_global.c
+ chacha_cryptobox.c
+ aes_cryptobox.c)
+ADD_LIBRARY(ottery STATIC ${OTTERYSRC})
+
+SET(OTTERY_CFLAGS "-DBUILD_RSPAMD -DOTTERY_NO_PID_CHECK -DOTTERY_NO_INIT_CHECK -DOTTERY_NO_WIPE_STACK")
+set_target_properties(ottery PROPERTIES COMPILE_FLAGS "${OTTERY_CFLAGS}") \ No newline at end of file
diff --git a/contrib/libottery/aes_cryptobox.c b/contrib/libottery/aes_cryptobox.c
new file mode 100644
index 0000000..ea86dc7
--- /dev/null
+++ b/contrib/libottery/aes_cryptobox.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2017, Vsevolod Stakhov
+ * Copyright (c) 2017, Frank Denis
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "ottery-internal.h"
+#include "cryptobox.h"
+
+#if defined(__x86_64__) && defined(RSPAMD_HAS_TARGET_ATTR)
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC push_options
+#pragma GCC target("aes")
+#endif
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __AES__
+#define __AES__
+#endif
+#include <immintrin.h>
+#define ROUNDS 10
+
+typedef struct RSPAMD_ALIGNED(16) aes_rng_state {
+ __m128i round_keys[ROUNDS + 1];
+ __m128i counter;
+} aes_stream_state;
+
+
+#define STATE_LEN sizeof(aes_stream_state)
+#define STATE_BYTES 16
+
+#define OUTPUT_LEN 1024
+
+static void
+aes_key_expand (__m128i round_keys[ROUNDS + 1], __m128i t) __attribute__((target("aes")));
+
+static void
+aes_key_expand (__m128i round_keys[ROUNDS + 1], __m128i t)
+{
+ __m128i t1;
+
+#define DO_ROUND_KEY(ROUND, RC) \
+ do { \
+ t1 = _mm_aeskeygenassist_si128(t, (RC)); \
+ round_keys[ROUND] = t; \
+ t = _mm_xor_si128(t, _mm_slli_si128(t, 4)); \
+ t = _mm_xor_si128(t, _mm_slli_si128(t, 8)); \
+ t = _mm_xor_si128(t, _mm_shuffle_epi32(t1, 0xff)); \
+ } while (0)
+
+ DO_ROUND_KEY(0, 1);
+ DO_ROUND_KEY(1, 2);
+ DO_ROUND_KEY(2, 4);
+ DO_ROUND_KEY(3, 8);
+ DO_ROUND_KEY(4, 16);
+ DO_ROUND_KEY(5, 32);
+ DO_ROUND_KEY(6, 64);
+ DO_ROUND_KEY(7, 128);
+ DO_ROUND_KEY(8, 27);
+ DO_ROUND_KEY(9, 54);
+ round_keys[10] = t;
+}
+
+/*
+ * Computes one 128 bytes block and refresh keys
+ */
+static void
+aes_round(unsigned char *buf, struct aes_rng_state *st) __attribute__((target("aes")));
+static void
+aes_round(unsigned char *buf, struct aes_rng_state *st)
+{
+ const __m128i one = _mm_set_epi64x(0, 1);
+ __m128i *round_keys = st->round_keys;
+ __m128i c0, c1, c2, c3, c4, c5, c6, c7;
+ __m128i r0, r1, r2, r3, r4, r5, r6, r7;
+ __m128i s0, s1, s2, s3, s4, s5, s6, s7;
+ size_t i;
+
+#define COMPUTE_ROUNDS(N) \
+ do { \
+ r##N = _mm_aesenc_si128( _mm_xor_si128(c##N, round_keys[0]), round_keys[1]); \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[2]), round_keys[3]); \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[4]), round_keys[5]); \
+ s##N = r##N; \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[6]), round_keys[7]); \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[8]), round_keys[9]); \
+ r##N = _mm_xor_si128(s##N, _mm_aesenclast_si128(r##N, round_keys[10])); \
+ } while (0)
+
+ c0 = st->counter;
+
+ for (i = 0; i < OUTPUT_LEN / 128; i ++) {
+ c1 = _mm_add_epi64 (c0, one);
+ c2 = _mm_add_epi64 (c1, one);
+ c3 = _mm_add_epi64 (c2, one);
+ c4 = _mm_add_epi64 (c3, one);
+ c5 = _mm_add_epi64 (c4, one);
+ c6 = _mm_add_epi64 (c5, one);
+ c7 = _mm_add_epi64 (c6, one);
+ COMPUTE_ROUNDS(0);
+ COMPUTE_ROUNDS(1);
+ COMPUTE_ROUNDS(2);
+ COMPUTE_ROUNDS(3);
+ COMPUTE_ROUNDS(4);
+ COMPUTE_ROUNDS(5);
+ COMPUTE_ROUNDS(6);
+ COMPUTE_ROUNDS(7);
+ c0 = _mm_add_epi64 (c7, one);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 0), r0);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 16), r1);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 32), r2);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 48), r3);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 64), r4);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 80), r5);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 96), r6);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 112), r7);
+ buf += 128;
+ }
+
+ st->counter = c0;
+ c0 = _mm_setzero_si128();
+ COMPUTE_ROUNDS(0);
+ aes_key_expand(round_keys, r0);
+}
+
+
+static void
+aes_cryptobox_state_setup (void *state_, const uint8_t *bytes)
+{
+ struct aes_rng_state *x = state_;
+
+ aes_key_expand (x->round_keys,
+ _mm_loadu_si128((const __m128i *) (const void *)bytes));
+}
+
+static void
+aes_cryptobox_generate (void *state_, uint8_t *output, uint32_t idx)
+{
+ struct aes_rng_state *x = state_;
+
+ aes_round(output, x);
+}
+
+#define PRF_AES(r) { \
+ "AES-" #r, \
+ "AES-" #r "-NOSIMD", \
+ "AES-" #r "-NOSIMD-DEFAULT", \
+ STATE_LEN, \
+ STATE_BYTES, \
+ OUTPUT_LEN, \
+ OTTERY_CPUCAP_AES, \
+ aes_cryptobox_state_setup, \
+ aes_cryptobox_generate \
+}
+
+const struct ottery_prf ottery_prf_aes_cryptobox_ = PRF_AES(128);
+#endif /* x86_64 */
diff --git a/contrib/libottery/chacha_cryptobox.c b/contrib/libottery/chacha_cryptobox.c
new file mode 100644
index 0000000..4e9cdae
--- /dev/null
+++ b/contrib/libottery/chacha_cryptobox.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "ottery-internal.h"
+#include "libcryptobox/chacha20/chacha.h"
+
+#define STATE_LEN (sizeof(chacha_state))
+#define STATE_BYTES 40
+
+#define IDX_STEP 16
+#define OUTPUT_LEN (IDX_STEP * 64)
+
+static void
+chacha20_cryptobox_state_setup (void *state_, const uint8_t *bytes)
+{
+ chacha_state *x = state_;
+ chacha_init (x, (chacha_key *)bytes, (chacha_iv *)(bytes + 32), 20);
+}
+
+static void
+chacha20_cryptobox_generate (void *state_, uint8_t *output, uint32_t idx)
+{
+ chacha_state *x = state_;
+
+ memset (output, 0, OUTPUT_LEN);
+ memcpy (output, &idx, sizeof (idx));
+ chacha_update (x, output, output, OUTPUT_LEN);
+}
+
+#define PRF_CHACHA(r) { \
+ "CHACHA" #r "-CRYPTOBOX", \
+ "CHACHA" #r "-CRYPTOBOX", \
+ "CHACHA" #r "-CRYPTOBOX", \
+ STATE_LEN, \
+ STATE_BYTES, \
+ OUTPUT_LEN, \
+ 0, \
+ chacha ## r ## _cryptobox_state_setup, \
+ chacha ## r ## _cryptobox_generate \
+}
+
+const struct ottery_prf ottery_prf_chacha20_cryptobox_ = PRF_CHACHA(20);
diff --git a/contrib/libottery/chacha_merged.c b/contrib/libottery/chacha_merged.c
new file mode 100644
index 0000000..c31a8bb
--- /dev/null
+++ b/contrib/libottery/chacha_merged.c
@@ -0,0 +1,218 @@
+/*
+ * This code is based on Dan Bernstein's pure C "merged" ChaCha
+ * implementation; details below.
+ *
+ * Note that I've ripped out all of the code that wasn't suitable for doing
+ * block-oriented operation, all (residual) support for 128-bit ChaCha keys,
+ * all support for counter values over 32 bits, the ability to xor the stream
+ * with a plaintext, and so on.
+ *
+ * Future versions of this might remove bigendian conversions too. DO NOT use
+ * this code for your stream cipher: go back to the original source. (I got
+ * this copy from SUPERCOP).
+ */
+
+/*
+chacha-merged.c version 20080118
+D. J. Bernstein
+Public domain.
+*/
+#include <string.h>
+#include "ottery-internal.h"
+#define u8 uint8_t
+#define u32 uint32_t
+#include "chacha_merged_ecrypt.h"
+
+#define ROTATE(v,c) (ROTL32(v,c))
+#define XOR(v,w) ((v) ^ (w))
+#define PLUS(v,w) (U32V((v) + (w)))
+#define PLUSONE(v) (PLUS((v),1))
+
+#define QUARTERROUND(a,b,c,d) \
+ a = PLUS(a,b); d = ROTATE(XOR(d,a),16); \
+ c = PLUS(c,d); b = ROTATE(XOR(b,c),12); \
+ a = PLUS(a,b); d = ROTATE(XOR(d,a), 8); \
+ c = PLUS(c,d); b = ROTATE(XOR(b,c), 7);
+
+static const char sigma[16] = "expand 32-byte k";
+
+static void ECRYPT_keysetup(ECRYPT_ctx *x,const u8 *k,u32 ivbits)
+{
+ const char *constants;
+ (void)ivbits;
+
+ x->input[4] = U8TO32_LITTLE(k + 0);
+ x->input[5] = U8TO32_LITTLE(k + 4);
+ x->input[6] = U8TO32_LITTLE(k + 8);
+ x->input[7] = U8TO32_LITTLE(k + 12);
+ k += 16;
+ constants = sigma;
+ x->input[8] = U8TO32_LITTLE(k + 0);
+ x->input[9] = U8TO32_LITTLE(k + 4);
+ x->input[10] = U8TO32_LITTLE(k + 8);
+ x->input[11] = U8TO32_LITTLE(k + 12);
+ x->input[0] = U8TO32_LITTLE(constants + 0);
+ x->input[1] = U8TO32_LITTLE(constants + 4);
+ x->input[2] = U8TO32_LITTLE(constants + 8);
+ x->input[3] = U8TO32_LITTLE(constants + 12);
+}
+
+static void ECRYPT_ivsetup(ECRYPT_ctx *x,const u8 *iv)
+{
+ x->input[12] = 0;
+ x->input[13] = 0;
+ x->input[14] = U8TO32_LITTLE(iv + 0);
+ x->input[15] = U8TO32_LITTLE(iv + 4);
+}
+
+#define IDX_STEP 16
+#define OUTPUT_LEN (IDX_STEP * 64)
+
+static inline void chacha_merged_getblocks(const int chacha_rounds, ECRYPT_ctx *x,u8 *c) __attribute__((always_inline));
+
+/** Generate OUTPUT_LEN bytes of output using the key, nonce, and counter in x,
+ * and store them in c.
+ */
+static void chacha_merged_getblocks(const int chacha_rounds, ECRYPT_ctx *x,u8 *c)
+{
+ u32 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
+ u32 j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
+ unsigned i, block;
+
+ j0 = x->input[0];
+ j1 = x->input[1];
+ j2 = x->input[2];
+ j3 = x->input[3];
+ j4 = x->input[4];
+ j5 = x->input[5];
+ j6 = x->input[6];
+ j7 = x->input[7];
+ j8 = x->input[8];
+ j9 = x->input[9];
+ j10 = x->input[10];
+ j11 = x->input[11];
+ j12 = x->input[12];
+ j13 = x->input[13];
+ j14 = x->input[14];
+ j15 = x->input[15];
+
+ for (block = 0; block < IDX_STEP; ++block) {
+ x0 = j0;
+ x1 = j1;
+ x2 = j2;
+ x3 = j3;
+ x4 = j4;
+ x5 = j5;
+ x6 = j6;
+ x7 = j7;
+ x8 = j8;
+ x9 = j9;
+ x10 = j10;
+ x11 = j11;
+ x12 = j12;
+ x13 = j13;
+ x14 = j14;
+ x15 = j15;
+ for (i = chacha_rounds;i > 0;i -= 2) {
+ QUARTERROUND( x0, x4, x8,x12)
+ QUARTERROUND( x1, x5, x9,x13)
+ QUARTERROUND( x2, x6,x10,x14)
+ QUARTERROUND( x3, x7,x11,x15)
+ QUARTERROUND( x0, x5,x10,x15)
+ QUARTERROUND( x1, x6,x11,x12)
+ QUARTERROUND( x2, x7, x8,x13)
+ QUARTERROUND( x3, x4, x9,x14)
+ }
+ x0 = PLUS(x0,j0);
+ x1 = PLUS(x1,j1);
+ x2 = PLUS(x2,j2);
+ x3 = PLUS(x3,j3);
+ x4 = PLUS(x4,j4);
+ x5 = PLUS(x5,j5);
+ x6 = PLUS(x6,j6);
+ x7 = PLUS(x7,j7);
+ x8 = PLUS(x8,j8);
+ x9 = PLUS(x9,j9);
+ x10 = PLUS(x10,j10);
+ x11 = PLUS(x11,j11);
+ x12 = PLUS(x12,j12);
+ x13 = PLUS(x13,j13);
+ x14 = PLUS(x14,j14);
+ x15 = PLUS(x15,j15);
+
+ j12 = PLUSONE(j12);
+ /* Ottery: j13 can never need to be incremented. */
+
+ U32TO8_LITTLE(c + 0,x0);
+ U32TO8_LITTLE(c + 4,x1);
+ U32TO8_LITTLE(c + 8,x2);
+ U32TO8_LITTLE(c + 12,x3);
+ U32TO8_LITTLE(c + 16,x4);
+ U32TO8_LITTLE(c + 20,x5);
+ U32TO8_LITTLE(c + 24,x6);
+
+ U32TO8_LITTLE(c + 28,x7);
+ U32TO8_LITTLE(c + 32,x8);
+ U32TO8_LITTLE(c + 36,x9);
+ U32TO8_LITTLE(c + 40,x10);
+ U32TO8_LITTLE(c + 44,x11);
+ U32TO8_LITTLE(c + 48,x12);
+ U32TO8_LITTLE(c + 52,x13);
+ U32TO8_LITTLE(c + 56,x14);
+ U32TO8_LITTLE(c + 60,x15);
+
+ c += 64;
+ }
+}
+
+#define STATE_LEN (sizeof(ECRYPT_ctx))
+#define STATE_BYTES 40
+
+static void
+chacha_merged_state_setup(void *state_, const uint8_t *bytes)
+{
+ ECRYPT_ctx *x = state_;
+ ECRYPT_keysetup(x, bytes, 0);
+ ECRYPT_ivsetup(x, bytes+32);
+}
+
+static void
+chacha8_merged_generate(void *state_, uint8_t *output, uint32_t idx)
+{
+ ECRYPT_ctx *x = state_;
+ x->input[12] = idx * IDX_STEP;
+ chacha_merged_getblocks(8, x, output);
+}
+
+static void
+chacha12_merged_generate(void *state_, uint8_t *output, uint32_t idx)
+{
+ ECRYPT_ctx *x = state_;
+ x->input[12] = idx * IDX_STEP;
+ chacha_merged_getblocks(12, x, output);
+}
+
+static void
+chacha20_merged_generate(void *state_, uint8_t *output, uint32_t idx)
+{
+ ECRYPT_ctx *x = state_;
+ x->input[12] = idx * IDX_STEP;
+ chacha_merged_getblocks(20, x, output);
+}
+
+#define PRF_CHACHA(r) { \
+ "CHACHA" #r, \
+ "CHACHA" #r "-NOSIMD", \
+ "CHACHA" #r "-NOSIMD-DEFAULT", \
+ STATE_LEN, \
+ STATE_BYTES, \
+ OUTPUT_LEN, \
+ 0, \
+ chacha_merged_state_setup, \
+ chacha ## r ## _merged_generate \
+}
+
+const struct ottery_prf ottery_prf_chacha8_merged_ = PRF_CHACHA(8);
+const struct ottery_prf ottery_prf_chacha12_merged_ = PRF_CHACHA(12);
+const struct ottery_prf ottery_prf_chacha20_merged_ = PRF_CHACHA(20);
+
diff --git a/contrib/libottery/chacha_merged_ecrypt.h b/contrib/libottery/chacha_merged_ecrypt.h
new file mode 100644
index 0000000..5cc94a9
--- /dev/null
+++ b/contrib/libottery/chacha_merged_ecrypt.h
@@ -0,0 +1,133 @@
+/* Definitions for types and macros used in chacha_merged.c. Taken from
+ * supercop.
+ */
+
+#include <limits.h>
+
+typedef struct
+{
+ u32 input[16]; /* could be compressed */
+ /*
+ * [edit]
+ *
+ * Put here all state variable needed during the encryption process.
+ */
+} ECRYPT_ctx;
+#if (UCHAR_MAX / 0xFFFFU > 0xFFFFU)
+#ifndef I32T
+#define I32T char
+#define U32C(v) (v##U)
+#endif
+#endif
+
+#if (USHRT_MAX / 0xFFFFU > 0xFFFFU)
+#ifndef I32T
+#define I32T short
+#define U32C(v) (v##U)
+#endif
+#endif
+
+#if (UINT_MAX / 0xFFFFU > 0xFFFFU)
+#ifndef I32T
+#define I32T int
+#define U32C(v) (v##U)
+#endif
+#endif
+
+#if (ULONG_MAX / 0xFFFFUL > 0xFFFFUL)
+#ifndef I32T
+#define I32T long
+#define U32C(v) (v##UL)
+#endif
+#endif
+
+#define U8C(v) (v ## U)
+#define U32V(v) ((u32)(v) & U32C(0xFFFFFFFF))
+#define U8V(v) ((u8)(v) & U8C(0xFF))
+
+#if (defined(WIN32) && defined(_MSC_VER))
+#include <stdlib.h>
+#pragma intrinsic(_lrotl) /* compile rotations "inline" */
+#define ROTL32(v, n) _lrotl(v, n)
+#else
+#define ROTL32(v, n) \
+ (U32V((v) << (n)) | ((v) >> (32 - (n))))
+#endif
+
+
+
+#if ECRYPT_LITTLE_ENDIAN
+#define U32TO32_LITTLE(v) (v)
+#endif
+#ifdef ECRYPT_BIG_ENDIAN
+#define SWAP32(v) \
+ ((ROTL32(v, 8) & U32C(0x00FF00FF)) | \
+ (ROTL32(v, 24) & U32C(0xFF00FF00)))
+
+#define U32TO32_LITTLE(v) SWAP32(v)
+#endif
+
+#ifdef U32TO32_LITTLE
+#define U8TO32_LITTLE(p) U32TO32_LITTLE(((u32*)(p))[0])
+#define U32TO8_LITTLE(p, v) (((u32*)(p))[0] = U32TO32_LITTLE(v))
+#else
+#define U8TO32_LITTLE(p) \
+ (((u32)((p)[0]) ) | \
+ ((u32)((p)[1]) << 8) | \
+ ((u32)((p)[2]) << 16) | \
+ ((u32)((p)[3]) << 24))
+#define U32TO8_LITTLE(p, v) \
+ do { \
+ (p)[0] = U8V((v) ); \
+ (p)[1] = U8V((v) >> 8); \
+ (p)[2] = U8V((v) >> 16); \
+ (p)[3] = U8V((v) >> 24); \
+ } while (0)
+#endif
+
+/*
+ * The LITTLE endian machines:
+ */
+#if defined(__ultrix) /* Older MIPS */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__alpha) /* Alpha */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(i386) /* x86 (gcc) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__i386) /* x86 (gcc) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__x86_64) /* x86_64 (gcc) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(_M_IX86) /* x86 (MSC, Borland) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(_MSC_VER) /* x86 (surely MSC) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__INTEL_COMPILER) /* x86 (surely Intel compiler icl.exe) */
+#define ECRYPT_LITTLE_ENDIAN
+
+/*
+ * The BIG endian machines:
+ */
+#elif defined(__sparc) /* Newer Sparc's */
+#define ECRYPT_BIG_ENDIAN
+#elif defined(__powerpc__) /* PowerPC */
+#define ECRYPT_BIG_ENDIAN
+#elif defined(__ppc__) /* PowerPC */
+#define ECRYPT_BIG_ENDIAN
+#elif defined(__hppa) /* HP-PA */
+#define ECRYPT_BIG_ENDIAN
+
+/*
+ * Finally machines with UNKNOWN endianness:
+ */
+#elif defined (_AIX) /* RS6000 */
+#define ECRYPT_UNKNOWN
+#elif defined(__aux) /* 68K */
+#define ECRYPT_UNKNOWN
+#elif defined(__dgux) /* 88K (but P6 in latest boxes) */
+#define ECRYPT_UNKNOWN
+#elif defined(__sgi) /* Newer MIPS */
+#define ECRYPT_UNKNOWN
+#else /* Any other processor */
+#define ECRYPT_UNKNOWN
+#endif
diff --git a/contrib/libottery/ottery-internal.h b/contrib/libottery/ottery-internal.h
new file mode 100644
index 0000000..cc047f8
--- /dev/null
+++ b/contrib/libottery/ottery-internal.h
@@ -0,0 +1,335 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_INTERNAL_H_HEADER_INCLUDED_
+#define OTTERY_INTERNAL_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef BUILD_RSPAMD
+#include "config.h"
+#endif
+
+#include "ottery-threading.h"
+
+
+/**
+ * Version number for Libottery. The first three bytes are the major number,
+ * minor number, and patch-level respectively. The final byte is 0 for a
+ * released version, and nonzero otherwise.
+ */
+#define OTTERY_VERSION 0x00000001
+/**
+ * Human-readable string representing the Libottery version.
+ */
+#define OTTERY_VERSION_STRING "0.0.0"
+
+/** Largest possible state_bytes value. */
+#define MAX_STATE_BYTES 64
+/** Largest possible state_len value. */
+#define MAX_STATE_LEN 256
+/** Largest possible output_len value. */
+#define MAX_OUTPUT_LEN 1024
+
+/**
+ * @brief Flags for external entropy sources.
+ *
+ * @{ */
+/** An RNG that probably provides strong entropy. */
+#define OTTERY_ENTROPY_FL_STRONG 0x000001
+/** An RNG that runs very quickly. */
+#define OTTERY_ENTROPY_FL_FAST 0x000002
+/** @} */
+
+/**
+ * @brief Identifying external entropy domains.
+ */
+/** An RNG provided by the operating system. */
+#define OTTERY_ENTROPY_DOM_OS 0x000100
+/** An RNG provided by the CPU. */
+#define OTTERY_ENTROPY_DOM_CPU 0x000200
+/** An EGD-style entropy source */
+#define OTTERY_ENTROPY_DOM_EGD 0x000400
+/** @} */
+
+#define OTTERY_ENTROPY_FLAG_MASK 0x000000ff
+#define OTTERY_ENTROPY_DOM_MASK 0x0000ff00
+#define OTTERY_ENTROPY_ALL_SOURCES 0x0fff0000
+
+struct sockaddr;
+
+/** Configuration for the strong RNG the we use for entropy. */
+struct ottery_entropy_config {
+ /** The filename to use as /dev/urandom. Ignored if this
+ * is not a unix-like operating system. If this is NULL, we use
+ * the default value. */
+ const char *urandom_fname;
+ /** An fd to use to access /dev/urandom. -1 if not set. Overrides
+ * urandom_fname. */
+ int urandom_fd;
+ /** True if urandom_fd has been set. */
+ unsigned urandom_fd_is_set;
+ /** Socket for egd */
+ const struct sockaddr *egd_sockaddr;
+ /** Socklen for egd_sockaddr. */
+ int egd_socklen;
+ /** Bitmask of sources to disable. */
+ uint32_t disabled_sources;
+ /** Bitmask of sources to consider weak. */
+ uint32_t weak_sources;
+
+ /** If true, we don't enforce that urandom_fname must be a device file.
+ * This is for testing, and is not exposed to user code.
+ */
+ unsigned allow_nondev_urandom;
+};
+
+struct ottery_entropy_state {
+ /* Cached value for the inode of the urandom device. If this value changes,
+ * we assume that somebody messed with the fd by accident. */
+ uint64_t urandom_fd_inode;
+};
+
+/**
+ * Return the buffer size to allocate when getting at least n bytes from each
+ * entropy source. We might not actually need so many. */
+size_t ottery_get_entropy_bufsize_(size_t n);
+
+/**
+ * Interface to underlying strong RNGs. If this were fast, we'd just use it
+ * for everything, and forget about having a userspace PRNG. Unfortunately,
+ * it typically isn't.
+ *
+ * @param config A correctly set-up ottery_entropy_config.
+ * @param state A correctly set-up ottery_entropy_state.
+ * @param require_flags Only run entropy sources with *all* of these
+ * OTTERY_ENTROPY_* flags set. Set this to 0 to use all the sources
+ * that work.
+ * @param bytes A buffer to receive random bytes.
+ * @param n The number of bytes to try to get from each entropy source.
+ * @param bufsize The number of bytes available in the buffer; modified
+ * to hold the number of bytes actually written.
+ * @param flags_out Set to a bitwise OR of all of the OTTERY_ENTROPY_* flags
+ * for sources in the result.
+ * @return Zero on success, or an error code on failure. On failure, it is not
+ * safe to treat the contents of the buffer as random at all.
+ */
+int ottery_get_entropy_(const struct ottery_entropy_config *config,
+ struct ottery_entropy_state *state,
+ uint32_t require_flags,
+ uint8_t *bytes, size_t n, size_t *bufsize,
+ uint32_t *flags_out);
+
+/**
+ * Clear all bytes stored in a structure. Unlike memset, the compiler is not
+ * going to optimize this out of existence because the target is about to go
+ * out of scope.
+ *
+ * @param mem Pointer to the memory to erase.
+ * @param len The number of bytes to erase.
+ */
+void ottery_memclear_(void *mem, size_t len);
+
+/**
+ * Information on a single pseudorandom function that we can use to generate
+ * a bytestream which (we hope) an observer can't distinguish from random
+ * bytes.
+ *
+ * Broadly speaking, every ottery_prf has an underlying function from an
+ * (state_bytes)-byte state and a 4 byte counter to an output_len-byte
+ * output block.
+ **/
+struct ottery_prf {
+ /** The name of this algorithm. */
+ const char *name;
+ /** The name of the implementation of this algorithm*/
+ const char *impl;
+ /** The name of the flavor of the implementation of this algorithm*/
+ const char *flav;
+ /** The length of the object that's used to hold the state (keys, nonces,
+ * subkeys as needed, etc) for this PRF. This can be longer than
+ * state_bytes because of key expansion or structure padding. It must be
+ * no greater than MAX_STATE_LEN. */
+ unsigned state_len;
+ /** The number of bytes used to generate a state object. It must be no
+ * greater than MAX_STATE_BYTES. It must be no grater than output_len. */
+ unsigned state_bytes;
+ /** The number of bytes generated by a single call to the generate
+ * function. It must be no larger than MAX_OUTPUT_LEN.
+ */
+ unsigned output_len;
+ /** Bitmask of CPU flags required to run this PRF. */
+ uint32_t required_cpucap;
+ /** Pointer to a function to initialize a state structure for the PRF.
+ *
+ * @param state An object of size at least (state_len) that will
+ * hold the state and any derived values. It must be aligned to
+ * a 16-byte boundary.
+ * @param bytes An array of (state_bytes) random bytes.
+ */
+ void (*setup)(void *state, const uint8_t *bytes);
+ /** Pointer to a function that calculates the PRF.
+ *
+ * @param state A state object previously initialized by the setup
+ * function.
+ * @param output An array of (output_len) bytes in which to store the
+ * result of the function
+ * @param idx A counter value for the function.
+ */
+ void (*generate)(void *state, uint8_t *output, uint32_t idx);
+};
+
+/**
+ * Evaluate the condition 'x', while hinting to the compiler that it is
+ * likely to be false.
+ */
+#ifdef __GNUC__
+#define UNLIKELY(x) __builtin_expect((x), 0)
+#else
+#define UNLIKELY(x) (x)
+#endif
+
+#ifdef OTTERY_INTERNAL
+struct ottery_config {
+ /** The PRF that we should use. If NULL, we use the default. */
+ const struct ottery_prf *impl;
+
+ /** Configuration for how we will set up our entropy sources. */
+ struct ottery_entropy_config entropy_config;
+};
+
+#define ottery_state_nolock ottery_state
+
+struct RSPAMD_ALIGNED(16) ottery_state {
+ /**
+ * Holds up to prf.output_len bytes that have been generated by the
+ * pseudorandom function. */
+ uint8_t buffer[MAX_OUTPUT_LEN] RSPAMD_ALIGNED(16);
+ /**
+ * Holds the state information (typically nonces and keys) used by the
+ * pseudorandom function. */
+
+ uint8_t state[MAX_STATE_LEN] RSPAMD_ALIGNED(16);
+ /**
+ * Parameters and function pointers for the cryptographic pseudorandom
+ * function that we're using. */
+ struct ottery_prf prf;
+ /**
+ * Index of the *next* block counter to use when generating random bytes
+ * with prf. When this equals or exceeds prf.stir_after, we should stir
+ * the PRNG. */
+ uint32_t block_counter;
+ /**
+ * Magic number; used to tell whether this state is initialized.
+ */
+ uint32_t magic;
+ /**
+ * Index of the next byte in (buffer) to yield to the user.
+ *
+ * Invariant: this is less than prf.output_len. */
+ uint16_t pos;
+ /**
+ * The pid of the process in which this PRF was most recently seeded
+ * from the OS. We use this to avoid use-after-fork problems; see
+ * ottery_st_rand_lock_and_check(). */
+ pid_t pid;
+ /**
+ * Combined flags_out results from all calls to the entropy source that
+ * have influenced our current state.
+ */
+ uint32_t entropy_src_flags;
+ /**
+ * flags_out result from our last call to the entropy source.
+ */
+ uint32_t last_entropy_flags;
+ /**
+ * Configuration for the entropy source.
+ */
+ struct ottery_entropy_config entropy_config;
+ /** State for the entropy source.
+ */
+ struct ottery_entropy_state entropy_state;
+ /**
+ * @brief Locks for this structure.
+ *
+ * This lock will not necessarily be recursive. It's probably a
+ * spinlock.
+ *
+ * @{
+ */
+DECL_LOCK(mutex)
+ /**@}*/
+};
+#endif
+
+struct ottery_config;
+/**
+ * For testing: manually supply a PRF.
+ */
+void ottery_config_set_manual_prf_(struct ottery_config *cfg,
+ const struct ottery_prf *prf);
+
+
+/** Called when a fatal error has occurred: Die horribly, or invoke
+ * ottery_fatal_handler. */
+void ottery_fatal_error_(int error);
+
+#define OTTERY_CPUCAP_SIMD (1<<0)
+#define OTTERY_CPUCAP_SSSE3 (1<<1)
+#define OTTERY_CPUCAP_AES (1<<2)
+#define OTTERY_CPUCAP_RAND (1<<3)
+
+/** Return a mask of OTTERY_CPUCAP_* for what the CPU will offer us. */
+uint32_t ottery_get_cpu_capabilities_(void);
+
+/** Tell ottery_get_cpu_capabilities to never report certain capabilities as
+ * present. */
+void ottery_disable_cpu_capabilities_(uint32_t disable);
+
+/**
+ * @brief pure-C portable ChaCha implementations.
+ *
+ * @{
+ */
+extern const struct ottery_prf ottery_prf_chacha8_merged_;
+extern const struct ottery_prf ottery_prf_chacha12_merged_;
+extern const struct ottery_prf ottery_prf_chacha20_merged_;
+
+#ifdef BUILD_RSPAMD
+#ifdef __x86_64__
+extern const struct ottery_prf ottery_prf_aes_cryptobox_;
+#endif
+extern const struct ottery_prf ottery_prf_chacha20_cryptobox_;
+#endif
+/**@}*/
+
+/**
+ * @brief SIMD-basd ChaCha implementations.
+ *
+ * These are much, much faster.
+ *
+ * @{ */
+#ifdef HAVE_SIMD_CHACHA
+extern const struct ottery_prf ottery_prf_chacha8_krovetz_1_;
+extern const struct ottery_prf ottery_prf_chacha12_krovetz_1_;
+extern const struct ottery_prf ottery_prf_chacha20_krovetz_1_;
+#endif
+
+#ifdef HAVE_SIMD_CHACHA_2
+extern const struct ottery_prf ottery_prf_chacha8_krovetz_2_;
+extern const struct ottery_prf ottery_prf_chacha12_krovetz_2_;
+extern const struct ottery_prf ottery_prf_chacha20_krovetz_2_;
+#endif
+/** @} */
+
+#endif
diff --git a/contrib/libottery/ottery-threading.h b/contrib/libottery/ottery-threading.h
new file mode 100644
index 0000000..c5427ad
--- /dev/null
+++ b/contrib/libottery/ottery-threading.h
@@ -0,0 +1,96 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_LOCKING_H_HEADER_INCLUDED_
+#define OTTERY_LOCKING_H_HEADER_INCLUDED_
+
+/* We don't need locks when building rspamd */
+#ifdef BUILD_RSPAMD
+#define OTTERY_NO_LOCKS
+#endif
+
+/* Locks */
+#ifdef OTTERY_NO_LOCKS
+/* Nothing here. */
+#elif defined(__APPLE__) && !defined(OTTERY_NO_SPINLOCKS)
+#define OTTERY_OSATOMIC_LOCKS
+#include <libkern/OSAtomic.h>
+#elif defined(_WIN32)
+#define OTTERY_CRITICAL_SECTION
+#include <windows.h>
+#elif defined(HAVE_PTHREAD)
+#define OTTERY_PTHREADS
+#include <pthread.h>
+#else
+#define OTTERY_NO_LOCKS
+#endif
+
+#ifdef OTTERY_NO_LOCKS
+#define DECL_LOCK(mutex)
+#elif defined(OTTERY_OSATOMIC_LOCKS)
+#define DECL_LOCK(mutex) OSSpinLock mutex;
+#elif defined(OTTERY_CRITICAL_SECTION)
+#define DECL_LOCK(mutex) CRITICAL_SECTION mutex;
+#elif defined(OTTERY_PTHREADS)
+#define DECL_LOCK(mutex) pthread_mutex_t mutex;
+#endif
+
+#if defined(OTTERY_PTHREADS)
+#define INIT_LOCK(mutex) \
+ (pthread_mutex_init((mutex), NULL) != 0)
+/** Acquire the lock for the state "st". */
+#define ACQUIRE_LOCK(mutex) do { \
+ pthread_mutex_lock(mutex); \
+ } while (0)
+/** Release the lock for the state "st". */
+#define RELEASE_LOCK(mutex) do { \
+ pthread_mutex_unlock(mutex); \
+ } while (0)
+#define DESTROY_LOCK(mutex) do { \
+ pthread_mutex_destroy(mutex); \
+ } while (0)
+
+#elif defined(OTTERY_CRITICAL_SECTION)
+#define INIT_LOCK(mutex) \
+ (InitializeCriticalSectionAndSpinCount((mutex), 3000) == 0)
+#define ACQUIRE_LOCK(mutex) do { \
+ EnterCriticalSection(mutex); \
+ } while (0)
+#define RELEASE_LOCK(mutex) do { \
+ LeaveCriticalSection(mutex); \
+ } while (0)
+#define DESTROY_LOCK(mutex) do { \
+ DeleteCriticalSection(mutex); \
+ } while (0)
+
+#elif defined(OTTERY_OSATOMIC_LOCKS)
+#define INIT_LOCK(mutex) \
+ ((*(mutex) = 0), 0)
+#define ACQUIRE_LOCK(mutex) do { \
+ OSSpinLockLock(mutex); \
+ } while (0)
+#define RELEASE_LOCK(mutex) do { \
+ OSSpinLockUnlock(mutex); \
+ } while (0)
+#define DESTROY_LOCK(mutex) ((void)0)
+
+#elif defined(OTTERY_NO_LOCKS)
+#define INIT_LOCK(mutex) (0)
+#define DESTROY_LOCK(mutex) ((void)0)
+#define ACQUIRE_LOCK(mutex) ((void)0)
+#define RELEASE_LOCK(mutex) ((void)0)
+#else
+#error How do I lock?
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery.c b/contrib/libottery/ottery.c
new file mode 100644
index 0000000..c58a901
--- /dev/null
+++ b/contrib/libottery/ottery.c
@@ -0,0 +1,847 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include "ottery-internal.h"
+#include "ottery.h"
+#include "ottery_st.h"
+#include "ottery_nolock.h"
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include <stdio.h>
+
+/* I've added a few assertions to sanity-check for debugging, but they should
+ * never ever ever trigger. It's fine to build this code with NDEBUG. */
+#include <assert.h>
+
+#ifdef _WIN32
+/* On Windows, there is no fork(), so we don't need to worry about forking. */
+#define OTTERY_NO_PID_CHECK
+#endif
+
+#ifdef BUILD_RSPAMD
+#include "cryptobox.h"
+#endif
+
+/** Magic number for deciding whether an ottery_state is initialized. */
+#define MAGIC_BASIS 0x11b07734
+
+/** Macro: yield the correct magic number for an ottery_state, based on
+ * its position in RAM. */
+#define MAGIC(ptr) (((uint32_t)(uintptr_t)(ptr)) ^ MAGIC_BASIS)
+
+static inline int ottery_st_rand_lock_and_check(struct ottery_state *st)
+__attribute__((always_inline));
+static int ottery_st_reseed(struct ottery_state *state);
+static int ottery_st_add_seed_impl(struct ottery_state *st, const uint8_t *seed, size_t n, int locking, int check_magic);
+
+#ifndef OTTERY_NO_WIPE_STACK
+static void ottery_wipe_stack_(void) __attribute__((noinline));
+#endif
+
+#define LOCK(st) ACQUIRE_LOCK(&(st)->mutex)
+#define UNLOCK(st) RELEASE_LOCK(&(st)->mutex)
+
+size_t
+ottery_get_sizeof_config(void)
+{
+ return sizeof(struct ottery_config);
+}
+
+size_t
+ottery_get_sizeof_state(void)
+{
+ return sizeof(struct ottery_state);
+}
+
+size_t
+ottery_get_sizeof_state_nolock(void)
+{
+ return sizeof(struct ottery_state_nolock);
+}
+
+const char *
+ottery_get_version_string(void)
+{
+ return OTTERY_VERSION_STRING;
+}
+
+uint32_t
+ottery_get_version(void)
+{
+ return OTTERY_VERSION;
+}
+
+uint32_t
+ottery_get_build_flags(void)
+{
+ uint32_t result = 0;
+#ifdef OTTERY_NO_PID_CHECK
+ result |= OTTERY_BLDFLG_NO_PID_CHECK;
+#endif
+#ifdef OTTERY_NO_INIT_CHECK
+ result |= OTTERY_BLDFLG_NO_INIT_CHECK;
+#endif
+#ifdef OTTERY_NO_LOCKS
+ result |= OTTERY_BLDFLG_NO_LOCKING;
+#endif
+#ifdef OTTERY_NO_CLEAR_AFTER_YIELD
+ result |= OTTERY_BLDFLG_NO_CLEAR_AFTER_YIELD;
+#endif
+#ifdef OTTERY_NO_WIPE_STACK
+ result |= OTTERY_BLDFLG_NO_WIPE_STACK;
+#endif
+#ifdef OTTERY_NO_SIMD
+ result |= OTTERY_BLDFLG_NO_SIMD;
+#endif
+ return result;
+}
+
+#ifndef OTTERY_NO_CLEAR_AFTER_YIELD
+/** Used to zero out the contents of our buffer after we've just given a few
+ * to the user. */
+#define CLEARBUF(ptr,n) do { memset((ptr), 0, (n)); } while (0)
+#else
+#define CLEARBUF(ptr,n) ((void)0)
+#endif
+
+/**
+ * Volatile pointer to memset: we use this to keep the compiler from
+ * eliminating our call to memset. (Don't make this static.)
+ */
+void * (*volatile ottery_memset_volatile_)(void *, int, size_t) = memset;
+
+
+void
+ottery_memclear_(void *mem, size_t len)
+{
+ /* NOTE: whenever we change this, change test/test_memclear.c accordingly */
+ ottery_memset_volatile_(mem, 0, len);
+}
+
+#ifndef OTTERY_NO_WIPE_STACK
+
+/* Chosen more or less arbitrarily */
+#define WIPE_STACK_LEN 512
+
+/**
+ * Try to clear memory on the stack to clean up after our PRF. This can't
+ * easily be done in standard C, so we're doing an ugly hack in hopes that it
+ * actually helps.
+ *
+ * This should never be necessary in a correct program, but if your program is
+ * doing something stupid like leaking uninitialized stack, it might keep an
+ * attacker from exploiting that.
+ **/
+static void
+ottery_wipe_stack_(void)
+{
+ char buf[WIPE_STACK_LEN];
+ ottery_memset_volatile_(buf, 0, sizeof(buf));
+}
+#else
+#define ottery_wipe_stack_() ((void)0)
+#endif
+
+int
+ottery_config_init(struct ottery_config *cfg)
+{
+ cfg->impl = NULL;
+ cfg->entropy_config.urandom_fname = NULL;
+ cfg->entropy_config.urandom_fd = -1;
+ cfg->entropy_config.urandom_fd_is_set = 0;
+ cfg->entropy_config.disabled_sources = 0;
+ cfg->entropy_config.weak_sources = 0;
+ cfg->entropy_config.egd_sockaddr = NULL;
+ cfg->entropy_config.egd_socklen = 0;
+ cfg->entropy_config.allow_nondev_urandom = 0;
+ return 0;
+}
+
+static const struct ottery_prf *
+ottery_get_impl(const char *impl)
+{
+ int i;
+ const struct ottery_prf *ALL_PRFS[] = {
+#ifdef HAVE_SIMD_CHACHA_2
+ &ottery_prf_chacha20_krovetz_2_,
+ &ottery_prf_chacha12_krovetz_2_,
+ &ottery_prf_chacha8_krovetz_2_,
+#endif
+#ifdef HAVE_SIMD_CHACHA
+ &ottery_prf_chacha20_krovetz_1_,
+ &ottery_prf_chacha12_krovetz_1_,
+ &ottery_prf_chacha8_krovetz_1_,
+#endif
+
+#ifdef BUILD_RSPAMD
+#if defined(__x86_64__) && defined(RSPAMD_HAS_TARGET_ATTR)
+ &ottery_prf_aes_cryptobox_,
+#endif
+ &ottery_prf_chacha20_cryptobox_,
+#endif
+ &ottery_prf_chacha20_merged_,
+ &ottery_prf_chacha12_merged_,
+ &ottery_prf_chacha8_merged_,
+
+ NULL,
+ };
+ const uint32_t cap = ottery_get_cpu_capabilities_();
+
+ for (i = 0; ALL_PRFS[i]; ++i) {
+ const struct ottery_prf *prf = ALL_PRFS[i];
+ if ((prf->required_cpucap & cap) != prf->required_cpucap)
+ continue;
+ if (impl == NULL)
+ return prf;
+ if (!strcmp(impl, prf->name))
+ return prf;
+ if (!strcmp(impl, prf->impl))
+ return prf;
+ if (!strcmp(impl, prf->flav))
+ return prf;
+ }
+ return NULL;
+}
+
+int
+ottery_config_force_implementation(struct ottery_config *cfg,
+ const char *impl)
+{
+ const struct ottery_prf *prf = ottery_get_impl(impl);
+ if (prf) {
+ cfg->impl = prf;
+ return 0;
+ }
+ return OTTERY_ERR_INVALID_ARGUMENT;
+}
+
+void
+ottery_config_set_manual_prf_(struct ottery_config *cfg,
+ const struct ottery_prf *prf)
+{
+ cfg->impl = prf;
+}
+
+void
+ottery_config_set_urandom_device(struct ottery_config *cfg,
+ const char *fname)
+{
+ cfg->entropy_config.urandom_fname = fname;
+}
+
+void
+ottery_config_set_urandom_fd(struct ottery_config *cfg,
+ int fd)
+{
+ cfg->entropy_config.urandom_fd = fd;
+ cfg->entropy_config.urandom_fd_is_set = (fd >= 0);
+}
+
+void
+ottery_config_set_egd_socket(struct ottery_config *cfg,
+ const struct sockaddr *addr,
+ int len)
+{
+ cfg->entropy_config.egd_sockaddr = addr;
+ cfg->entropy_config.egd_socklen = len;
+}
+
+void
+ottery_config_disable_entropy_sources(struct ottery_config *cfg,
+ uint32_t disabled_sources)
+{
+ cfg->entropy_config.disabled_sources =
+ (disabled_sources & OTTERY_ENTROPY_ALL_SOURCES);
+}
+
+void
+ottery_config_mark_entropy_sources_weak(struct ottery_config *cfg,
+ uint32_t disabled_sources)
+{
+ cfg->entropy_config.weak_sources =
+ (disabled_sources & OTTERY_ENTROPY_ALL_SOURCES);
+}
+
+/**
+ * As ottery_st_nextblock_nolock(), but fill the entire block with
+ * entropy, and don't try to rekey the state.
+ */
+static void
+ottery_st_nextblock_nolock_norekey(struct ottery_state *st)
+{
+ st->prf.generate(st->state, st->buffer, st->block_counter);
+ ottery_wipe_stack_();
+ ++st->block_counter;
+}
+
+/**
+ * Generate (st->output_len) bytes of pseudorandom data from the PRF into
+ * (st->buffer). Use the first st->prf.state_bytes of those bytes to replace
+ * the PRF state and advance (st->pos) to point after them.
+ *
+ * This function does not acquire the lock on the state; use it within
+ * another function that does.
+ *
+ * @param st The state to use when generating the block.
+ */
+static void
+ottery_st_nextblock_nolock(struct ottery_state_nolock *st)
+{
+ ottery_st_nextblock_nolock_norekey(st);
+ st->prf.setup(st->state, st->buffer);
+ CLEARBUF(st->buffer, st->prf.state_bytes);
+ st->block_counter = 0;
+ st->pos = st->prf.state_bytes;
+}
+
+/**
+ * Initialize or reinitialize a PRNG state.
+ *
+ * @param st The state to initialize or reinitialize.
+ * @param prf The configuration to use. (Ignored for reinit)
+ * @return An OTTERY_ERR_* value (zero on success, nonzero on failure).
+ */
+static int
+ottery_st_initialize(struct ottery_state *st,
+ const struct ottery_config *config,
+ int locked)
+{
+ const struct ottery_prf *prf = NULL;
+ struct ottery_config cfg_tmp;
+ int err;
+ /* We really need our state to be aligned. If it isn't, let's give an
+ * error now, and not a crash when the SIMD instructions start to fail.
+ */
+ if (((uintptr_t)st) & 0xf)
+ return OTTERY_ERR_STATE_ALIGNMENT;
+
+ if (!config) {
+ ottery_config_init(&cfg_tmp);
+ config = &cfg_tmp;
+ }
+
+ prf = config->impl;
+
+ if (!prf)
+ prf = ottery_get_impl(NULL);
+
+ memset(st, 0, sizeof(*st));
+
+ if (locked) {
+ /* Now set up the spinlock or mutex or hybrid thing. */
+ if (INIT_LOCK(&st->mutex))
+ return OTTERY_ERR_LOCK_INIT;
+ }
+
+ /* Check invariants for PRF, in case we wrote some bad code. */
+ if ((prf->state_len > MAX_STATE_LEN) ||
+ (prf->state_bytes > MAX_STATE_BYTES) ||
+ (prf->state_bytes > prf->output_len) ||
+ (prf->output_len > MAX_OUTPUT_LEN))
+ return OTTERY_ERR_INTERNAL;
+
+ /* Check whether some of our structure size assumptions are right. */
+ if ((sizeof(struct ottery_state) > OTTERY_STATE_DUMMY_SIZE_) ||
+ (sizeof(struct ottery_config) > OTTERY_CONFIG_DUMMY_SIZE_))
+ return OTTERY_ERR_INTERNAL;
+
+ memcpy(&st->entropy_config, &config->entropy_config,
+ sizeof(struct ottery_entropy_config));
+
+ /* Copy the PRF into place. */
+ memcpy(&st->prf, prf, sizeof(*prf));
+
+ if ((err = ottery_st_reseed(st)))
+ return err;
+
+ /* Set the magic number last, or else we might look like we succeeded
+ * when we didn't */
+ st->magic = MAGIC(st);
+
+ st->pid = getpid();
+
+ return 0;
+}
+
+static int
+ottery_st_reseed(struct ottery_state *st)
+{
+ /* Now seed the PRF: Generate some random bytes from the OS, and use them
+ * as whatever keys/nonces/whatever the PRF wants to have. */
+ /* XXXX Add seed rather than starting from scratch? */
+ int err;
+ uint32_t flags=0;
+ size_t buflen = ottery_get_entropy_bufsize_(st->prf.state_bytes);
+ uint8_t *buf = alloca(buflen);
+ if (!buf)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+
+ if ((err = ottery_get_entropy_(&st->entropy_config, &st->entropy_state, 0,
+ buf, st->prf.state_bytes,
+ &buflen,
+ &flags)))
+ return err;
+ if (buflen < st->prf.state_bytes)
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+ /* The first state_bytes bytes become the initial key. */
+ st->prf.setup(st->state, buf);
+ /* If there are more bytes, we mix them into the key with add_seed */
+ if (buflen > st->prf.state_bytes)
+ ottery_st_add_seed_impl(st,
+ buf + st->prf.state_bytes,
+ buflen - st->prf.state_bytes,
+ 0,
+ 0);
+ ottery_memclear_(buf, buflen);
+ st->last_entropy_flags = flags;
+ st->entropy_src_flags = flags;
+
+ /* Generate the first block of output. */
+ st->block_counter = 0;
+ ottery_st_nextblock_nolock(st);
+
+ return 0;
+}
+
+int
+ottery_st_init(struct ottery_state *st, const struct ottery_config *cfg)
+{
+ return ottery_st_initialize(st, cfg, 1);
+}
+
+int
+ottery_st_init_nolock(struct ottery_state_nolock *st,
+ const struct ottery_config *cfg)
+{
+ return ottery_st_initialize(st, cfg, 0);
+}
+
+static int
+ottery_st_add_seed_impl(struct ottery_state *st, const uint8_t *seed, size_t n, int locking, int check_magic)
+{
+#ifndef OTTERY_NO_INIT_CHECK
+ if (check_magic && UNLIKELY(st->magic != MAGIC(st))) {
+ ottery_fatal_error_(OTTERY_ERR_STATE_INIT);
+ return OTTERY_ERR_STATE_INIT;
+ }
+#endif
+
+ /* If the user passed NULL, then we should reseed from the operating
+ * system. */
+ uint8_t *tmp_seed = NULL;
+ size_t tmp_seed_len = 0;
+ uint32_t flags = 0;
+
+ if (!seed || !n) {
+ int err;
+ tmp_seed_len = ottery_get_entropy_bufsize_(st->prf.state_bytes);
+ tmp_seed = alloca(tmp_seed_len);
+ if (!tmp_seed)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ n = tmp_seed_len;
+ if ((err = ottery_get_entropy_(&st->entropy_config, &st->entropy_state, 0,
+ tmp_seed, st->prf.state_bytes,
+ &n,
+ &flags)))
+ return err;
+ if (n < st->prf.state_bytes)
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+ seed = tmp_seed;
+ }
+
+ if (locking)
+ LOCK(st);
+ /* The algorithm here is really easy. We grab a block of output from the
+ * PRNG, that the first (state_bytes) bytes of that, XOR it with up to
+ * (state_bytes) bytes of our new seed data, and use that to set our new
+ * state. We do this over and over until we have no more seed data to add.
+ */
+ while (n) {
+ unsigned i;
+ size_t m = n > st->prf.state_bytes/2 ? st->prf.state_bytes/2 : n;
+ ottery_st_nextblock_nolock_norekey(st);
+ for (i = 0; i < m; ++i) {
+ st->buffer[i] ^= seed[i];
+ }
+ st->prf.setup(st->state, st->buffer);
+ st->block_counter = 0;
+ n -= m;
+ seed += m;
+ }
+
+ /* Now make sure that st->buffer is set up with the new state. */
+ ottery_st_nextblock_nolock(st);
+
+ st->entropy_src_flags |= flags;
+ st->last_entropy_flags = flags;
+
+ if (locking)
+ UNLOCK(st);
+
+ /* If we used stack-allocated seed material, wipe it. */
+ if (tmp_seed)
+ ottery_memclear_(tmp_seed, tmp_seed_len);
+
+ return 0;
+}
+
+int
+ottery_st_add_seed(struct ottery_state *st, const uint8_t *seed, size_t n)
+{
+ return ottery_st_add_seed_impl(st, seed, n, 1, 1);
+}
+int
+ottery_st_add_seed_nolock(struct ottery_state_nolock *st, const uint8_t *seed, size_t n)
+{
+ return ottery_st_add_seed_impl(st, seed, n, 0, 1);
+}
+
+
+void
+ottery_st_wipe(struct ottery_state *st)
+{
+ DESTROY_LOCK(&st->mutex);
+
+ ottery_st_wipe_nolock(st);
+}
+
+void
+ottery_st_wipe_nolock(struct ottery_state_nolock *st)
+{
+ ottery_memclear_(st, sizeof(struct ottery_state));
+}
+
+void
+ottery_st_prevent_backtracking_nolock(struct ottery_state_nolock *st)
+{
+#ifdef OTTERY_NO_CLEAR_AFTER_YIELD
+ memset(st->buffer, 0, st->pos);
+#else
+ (void)st;
+#endif
+}
+
+void
+ottery_st_prevent_backtracking(struct ottery_state *st)
+{
+ LOCK(st);
+ ottery_st_prevent_backtracking_nolock(st);
+ UNLOCK(st);
+}
+
+/** Function that's invoked on a fatal error. See
+ * ottery_set_fatal_handler() for more information. */
+static void (*ottery_fatal_handler)(int) = NULL;
+
+void
+ottery_fatal_error_(int error)
+{
+ if (ottery_fatal_handler)
+ ottery_fatal_handler(error);
+ else
+ abort();
+}
+
+void
+ottery_set_fatal_handler(void (*fn)(int))
+{
+ ottery_fatal_handler = fn;
+}
+
+/**
+ * Shared prologue for functions generating random bytes from an ottery_state.
+ * Make sure that the state is initialized.
+ */
+static inline int
+ottery_st_rand_check_init(struct ottery_state *st)
+{
+#ifndef OTTERY_NO_INIT_CHECK
+ if (UNLIKELY(st->magic != MAGIC(st))) {
+ ottery_fatal_error_(OTTERY_ERR_STATE_INIT);
+ return -1;
+ }
+#else
+ (void)st;
+#endif
+ return 0;
+}
+
+/* XXXX */
+static inline int
+ottery_st_rand_check_pid(struct ottery_state *st)
+{
+#ifndef OTTERY_NO_PID_CHECK
+ if (UNLIKELY(st->pid != getpid())) {
+ int err;
+ if ((err = ottery_st_reseed(st))) {
+ ottery_fatal_error_(OTTERY_ERR_FLAG_POSTFORK_RESEED|err);
+ return -1;
+ }
+ st->pid = getpid();
+ }
+#else
+ (void) st;
+#endif
+ return 0;
+}
+
+static inline int
+ottery_st_rand_lock_and_check(struct ottery_state *st)
+{
+ if (ottery_st_rand_check_init(st))
+ return -1;
+ LOCK(st);
+ if (ottery_st_rand_check_pid(st)) {
+ UNLOCK(st);
+ return -1;
+ }
+ return 0;
+}
+
+static inline int
+ottery_st_rand_check_nolock(struct ottery_state_nolock *st)
+{
+ if (ottery_st_rand_check_init(st))
+ return -1;
+ if (ottery_st_rand_check_pid(st))
+ return -1;
+ return 0;
+}
+
+/**
+ * Generate a small-ish number of bytes from an ottery_state, using
+ * buffered data. If there is insufficient data in the buffer right now,
+ * use what we have, and generate more.
+ *
+ * @param st The state to use.
+ * @param out A location to write to.
+ * @param n The number of bytes to write. Must not be greater than
+ * st->prf.output_len*2 - st->prf.state_bytes - st->pos - 1.
+ */
+static inline void
+ottery_st_rand_bytes_from_buf(struct ottery_state *st, uint8_t *out,
+ size_t n)
+{
+ if (n + st->pos < st->prf.output_len) {
+ memcpy(out, st->buffer+st->pos, n);
+ CLEARBUF(st->buffer+st->pos, n);
+ st->pos += n;
+ } else {
+ unsigned cpy = st->prf.output_len - st->pos;
+ memcpy(out, st->buffer+st->pos, cpy);
+ n -= cpy;
+ out += cpy;
+ ottery_st_nextblock_nolock(st);
+ memcpy(out, st->buffer+st->pos, n);
+ CLEARBUF(st->buffer, n);
+ st->pos += n;
+ assert(st->pos < st->prf.output_len);
+ }
+}
+
+static void
+ottery_st_rand_bytes_impl(struct ottery_state *st, void *out_,
+ size_t n)
+{
+ uint8_t *out = out_;
+ size_t cpy;
+
+ if (n + st->pos < st->prf.output_len * 2 - st->prf.state_bytes - 1) {
+ /* Fulfill it all from the buffer simply if possible. */
+ ottery_st_rand_bytes_from_buf(st, out, n);
+ return;
+ }
+
+ /* Okay. That's not going to happen. Well, take what we can... */
+ cpy = st->prf.output_len - st->pos;
+ memcpy(out, st->buffer + st->pos, cpy);
+ out += cpy;
+ n -= cpy;
+
+ /* Then take whole blocks so long as we need them, without stirring... */
+ while (n >= st->prf.output_len) {
+ /* (We could save a memcpy here if we generated the block directly at out
+ * rather than doing the memcpy here. First we'd need to make sure that we
+ * had gotten the block aligned to a 16-byte boundary, though, and we'd
+ * have some other tricky bookkeeping to do. Let's call this good enough
+ * for now.) */
+ ottery_st_nextblock_nolock_norekey(st);
+ memcpy(out, st->buffer, st->prf.output_len);
+ out += st->prf.output_len;
+ n -= st->prf.output_len;
+ }
+
+ /* Then stir for the last part. */
+ ottery_st_nextblock_nolock(st);
+ ottery_st_rand_bytes_from_buf(st, out, n);
+}
+
+void
+ottery_st_rand_bytes(struct ottery_state *st, void *out_, size_t n)
+{
+ if (ottery_st_rand_lock_and_check(st))
+ return;
+ ottery_st_rand_bytes_impl(st, out_, n);
+ UNLOCK(st);
+}
+
+void
+ottery_st_rand_bytes_nolock(struct ottery_state_nolock *st, void *out_, size_t n)
+{
+ if (ottery_st_rand_check_nolock(st))
+ return;
+ ottery_st_rand_bytes_impl(st, out_, n);
+}
+
+/**
+ * Assign an integer type from bytes at a possibly unaligned pointer.
+ *
+ * @param type the type of integer to assign.
+ * @param r the integer lvalue to write to.
+ * @param p a pointer to the bytes to read from.
+ **/
+#define INT_ASSIGN_PTR(type, r, p) do { \
+ memcpy(&r, p, sizeof(type)); \
+} while (0)
+
+/**
+ * Shared code for implementing rand_unsigned() and rand_uint64().
+ *
+ * @param st The state to use.
+ * @param inttype The type of integer to generate.
+ **/
+#define OTTERY_RETURN_RAND_INTTYPE_IMPL(st, inttype, unlock) do { \
+ inttype result; \
+ if (sizeof(inttype) + (st)->pos <= (st)->prf.output_len) { \
+ INT_ASSIGN_PTR(inttype, result, (st)->buffer + (st)->pos); \
+ CLEARBUF((st)->buffer + (st)->pos, sizeof(inttype)); \
+ (st)->pos += sizeof(inttype); \
+ if (st->pos == (st)->prf.output_len) { \
+ ottery_st_nextblock_nolock(st); \
+ } \
+ } else { \
+ /* Our handling of this case here is significantly simpler */ \
+ /* than that of ottery_st_rand_bytes_from_buf, at the expense */ \
+ /* of wasting up to sizeof(inttype)-1 bytes. Since inttype */ \
+ /* is at most 8 bytes long, that's not such a big deal. */ \
+ ottery_st_nextblock_nolock(st); \
+ INT_ASSIGN_PTR(inttype, result, (st)->buffer + (st)->pos); \
+ CLEARBUF((st)->buffer, sizeof(inttype)); \
+ (st)->pos += sizeof(inttype); \
+ } \
+ unlock; \
+ return result; \
+} while (0)
+
+#define OTTERY_RETURN_RAND_INTTYPE(st, inttype) do { \
+ if (ottery_st_rand_lock_and_check(st)) \
+ return (inttype)0; \
+ OTTERY_RETURN_RAND_INTTYPE_IMPL(st, inttype, UNLOCK(st)); \
+} while (0)
+
+#define OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, inttype) do { \
+ if (ottery_st_rand_check_nolock(st)) \
+ return (inttype)0; \
+ OTTERY_RETURN_RAND_INTTYPE_IMPL(st, inttype, ); \
+} while (0)
+
+unsigned
+ottery_st_rand_unsigned(struct ottery_state *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE(st, unsigned);
+}
+
+unsigned
+ottery_st_rand_unsigned_nolock(struct ottery_state_nolock *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, unsigned);
+}
+
+uint32_t
+ottery_st_rand_uint32(struct ottery_state *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE(st, uint32_t);
+}
+
+uint32_t
+ottery_st_rand_uint32_nolock(struct ottery_state_nolock *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, uint32_t);
+}
+
+uint64_t
+ottery_st_rand_uint64(struct ottery_state *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE(st, uint64_t);
+}
+
+uint64_t
+ottery_st_rand_uint64_nolock(struct ottery_state_nolock *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, uint64_t);
+}
+
+unsigned
+ottery_st_rand_range_nolock(struct ottery_state_nolock *st, unsigned upper)
+{
+ unsigned lim = upper+1;
+ unsigned divisor = lim ? (UINT_MAX / lim) : 1;
+ unsigned n;
+ do {
+ n = (ottery_st_rand_unsigned_nolock(st) / divisor);
+ } while (n > upper);
+
+ return n;
+}
+
+uint64_t
+ottery_st_rand_range64_nolock(struct ottery_state_nolock *st, uint64_t upper)
+{
+ uint64_t lim = upper+1;
+ uint64_t divisor = lim ? (UINT64_MAX / lim) : 1;
+ uint64_t n;
+ do {
+ n = (ottery_st_rand_uint64_nolock(st) / divisor);
+ } while (n > upper);
+
+ return n;
+}
+
+unsigned
+ottery_st_rand_range(struct ottery_state *state, unsigned upper)
+{
+ unsigned n;
+ if (ottery_st_rand_check_init(state))
+ return 0;
+ LOCK(state);
+ n = ottery_st_rand_range_nolock(state, upper);
+ UNLOCK(state);
+ return n;
+}
+
+uint64_t
+ottery_st_rand_range64(struct ottery_state *state, uint64_t upper)
+{
+ uint64_t n;
+ if (ottery_st_rand_check_init(state))
+ return 0;
+ LOCK(state);
+ n = ottery_st_rand_range64_nolock(state, upper);
+ UNLOCK(state);
+ return n;
+}
diff --git a/contrib/libottery/ottery.h b/contrib/libottery/ottery.h
new file mode 100644
index 0000000..e2caac2
--- /dev/null
+++ b/contrib/libottery/ottery.h
@@ -0,0 +1,143 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_H_HEADER_INCLUDED_
+#define OTTERY_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ottery_common.h"
+
+/** @file */
+
+struct ottery_config;
+
+/* Functions that use an implicit global state */
+
+/**
+ * Fill a buffer with random bytes.
+ *
+ * @param buf The buffer to fill.
+ * @param n The number of bytes to write.
+ */
+void ottery_rand_bytes(void *buf, size_t n);
+/**
+ * Generate a random number of type unsigned.
+ *
+ * @return A random number between 0 and UINT_MAX included,
+ * chosen uniformly.
+ */
+unsigned ottery_rand_unsigned(void);
+/**
+ * Generate a random number of type uint32_t.
+ *
+ * @return A random number between 0 and UINT32_MAX included,
+ * chosen uniformly.
+ */
+uint32_t ottery_rand_uint32(void);
+/**
+ * Generate a random number of type uint64_t.
+ *
+ * @return A random number between 0 and UINT64_MAX included,
+ * chosen uniformly.
+ */
+uint64_t ottery_rand_uint64(void);
+/**
+ * Generate a random number of type unsigned in a given range.
+ *
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+unsigned ottery_rand_range(unsigned top);
+/**
+ * Generate a random number of type uint64_t in a given range.
+ *
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+uint64_t ottery_rand_range64(uint64_t top);
+
+/**
+ * Initialize the libottery global state.
+ *
+ * Most users should not need to use this function. If you use it, you must
+ * call it before any of: ottery_rand_bytes, ottery_rand_unsigned,
+ * ottery_rand_uint64, ottery_rand_range, ottery_rand_uint64_range,
+ * ottery_add_seed, ottery_wipe, ottery_stir.
+ *
+ * You would want to use this function if you want to select some non-default
+ * behavior using an ottery_config structure.
+ *
+ * @param cfg Either NULL, or an ottery_config structure that has been
+ * initialized with ottery_config_init().
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_init(const struct ottery_config *cfg);
+
+/**
+ * Add more entropy to the libottery global state.
+ *
+ * Calling this function should be needless, if you trust your operating
+ * system's random number generator and entropy extraction features. You
+ * would want to use this function if you think the operating system's random
+ * number generator might be inadequate, and you want to add more entropy from
+ * EGD or something.
+ *
+ * You might also want to call this function if your belief system says that
+ * it's useful to periodically add more raw entropy to a well-seeded
+ * cryptographically strong PRNG.
+ *
+ * @param seed Bytes to add to the state. If this value is NULL, we take
+ * more random bytes from the OS.
+ * @param n The number of bytes to add. If this value is 0, we take more
+ * random bytes from the OS, regardless of the value of seed.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_add_seed(const uint8_t *seed, size_t n);
+
+/**
+ * Destroy the libottery global state and release any resources that it might
+ * hold.
+ *
+ * Ordinarily, you would only want to call this at exit, if at all.
+ */
+void ottery_wipe(void);
+
+/**
+ * Explicitly tell libottery to prevent backtracking attacks. (Usually
+ * needless.)
+ *
+ * Once this function has been called, an attacker who compromises the state
+ * later on will not be able to recover bytes that have previously been
+ * returned by any of the ottery_rand_* functions.
+ *
+ * You should not usually need to call this function: Libottery provides
+ * backtracking resistance by default, so unless you have manually recompiled
+ * with the OTTERY_NO_CLEAR_AFTER_YIELD option, this function isn't
+ * necessary and has no effect. Even *with* OTTERY_NO_CLEAR_AFTER_YIELD,
+ * this function isn't necessary in ordinary operation: the libottery state is
+ * implicitly "stirred" every 1k or so.
+ */
+void ottery_prevent_backtracking(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery_common.h b/contrib/libottery/ottery_common.h
new file mode 100644
index 0000000..bac6f04
--- /dev/null
+++ b/contrib/libottery/ottery_common.h
@@ -0,0 +1,351 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_COMMON_H_HEADER_INCLUDED_
+#define OTTERY_COMMON_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+/** @file */
+
+struct ottery_config;
+
+/* Error codes */
+
+/**
+ * @name libottery error codes and flags
+ *
+ * @{
+ */
+/** No error has occurred. */
+#define OTTERY_ERR_NONE 0x0000
+/** We failed to allocate or initialize a lock. */
+#define OTTERY_ERR_LOCK_INIT 0x0001
+/** An internal error occurrred. This is probably a programming mistake
+ * in libottery. */
+#define OTTERY_ERR_INTERNAL 0x0002
+/** We were unable to connect to the operating system's strong RNG. */
+#define OTTERY_ERR_INIT_STRONG_RNG 0x0003
+/** We were unable to retrieve sufficient random bytes from the
+ * operating system's strong RNG. */
+#define OTTERY_ERR_ACCESS_STRONG_RNG 0x0004
+/** At least one argument to the function was invalid. */
+#define OTTERY_ERR_INVALID_ARGUMENT 0x0005
+/** An ottery_state structure was not aligned to a 16-byte boundary. */
+#define OTTERY_ERR_STATE_ALIGNMENT 0x0006
+
+/** FATAL ERROR: An ottery_st function other than ottery_st_init() was
+ * called on and uninitialized state. */
+#define OTTERY_ERR_STATE_INIT 0x1000
+/** FLAG; FATAL ERROR: The error occurred while initializing the global
+ * state during the first call to an ottery_rand_* function. */
+#define OTTERY_ERR_FLAG_GLOBAL_PRNG_INIT 0x2000
+/** FLAG; FATAL ERROR: The error occurred while reinitializing a state
+ * after a fork(). (We need to do this, or else both processes would
+ * generate the same values, which could give dire results.)
+ */
+#define OTTERY_ERR_FLAG_POSTFORK_RESEED 0x4000
+
+/**
+ * Checks whether an OTTERY_ERR value is a fatal error.
+ *
+ * @param err an OTTERY_ERR_* valuer
+ * @return True if err is fatal; false if it is not fatal.
+ */
+#define OTTERY_ERR_IS_FATAL(err) \
+ (((err) & ~0xfff) != 0)
+
+/* Functions to interact with the library on a global level */
+
+/**
+ * Override the behavior of libottery on a fatal error.
+ *
+ * By default, libottery will call abort() in a few circumstances, in
+ * order to keep the program from operating insecurely. If you want,
+ * you can provide another function to call instead.
+ *
+ * If your function does not itself abort() or exit() the process, or throw an
+ * exception (assuming some C family that has exceptions), libottery will
+ * continue running insecurely -- it might return predictable random numbers,
+ * leak secrets, or just return 0 for everything -- so you should really be
+ * very careful here.
+ *
+ * (The alternative to fatal errors would have been having all the
+ * ottery_rand_* functions able to return an error, and requiring users
+ * to check those codes. But experience suggests that C programmers
+ * frequently do not check error codes.)
+ *
+ * @param fn A function to call in place of abort(). It will receive as
+ * its argument one of the OTTERY_ERR_* error codes.
+ */
+void ottery_set_fatal_handler(void (*fn)(int errorcode));
+
+/* Functions to manipulate parameters. */
+
+/**
+ * @name Names of prfs for use with ottery_config_force_implementation
+ *
+ * @{ */
+#define OTTERY_PRF_CHACHA "CHACHA"
+#define OTTERY_PRF_CHACHA8 "CHACHA8"
+#define OTTERY_PRF_CHACHA12 "CHACHA12"
+#define OTTERY_PRF_CHACHA20 "CHACHA20"
+#define OTTERY_PRF_CHACHA_SIMD "CHACHA-SIMD"
+#define OTTERY_PRF_CHACHA8_SIMD "CHACHA8-SIMD"
+#define OTTERY_PRF_CHACHA12_SIMD "CHACHA12-SIMD"
+#define OTTERY_PRF_CHACHA20_SIMD "CHACHA20-SIMD"
+#define OTTERY_PRF_CHACHA_NO_SIMD "CHACHA-NOSIMD"
+#define OTTERY_PRF_CHACHA8_NO_SIMD "CHACHA8-NOSIMD"
+#define OTTERY_PRF_CHACHA12_NO_SIMD "CHACHA12-NOSIMD"
+#define OTTERY_PRF_CHACHA20_NO_SIMD "CHACHA20-NOSIMD"
+/** @} */
+
+/**
+ * Initialize an ottery_config structure.
+ *
+ * You must call this function on any ottery_config structure before it
+ * can be passed to ottery_init() or ottery_st_init().
+ *
+ * @param cfg The configuration object to initialize.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on
+ * failure.
+ */
+int ottery_config_init(struct ottery_config *cfg);
+
+/**
+ * Try to force the use of a particular pseudorandom function for a given
+ * libottery instance.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * @param cfg The configuration structure to configure.
+ * @param impl The name of a pseudorandom function. One of the
+ * OTTERY_PRF_* values.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on
+ * failure.
+ */
+int ottery_config_force_implementation(struct ottery_config *cfg,
+ const char *impl);
+
+/**
+ * Set a device file to use as a source of strong entropy.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * By default, libottery will try /dev/urandom on Unix-like systems.
+ *
+ * @param cfg The configuration structure to configure.
+ * @param fname The name of the device to use instead of /dev/urandom. This
+ * pointer is copied around, and must not be freed while any libottery state
+ * configured using this structure is still in use.
+ *
+ */
+void ottery_config_set_urandom_device(struct ottery_config *cfg,
+ const char *fname);
+
+/**
+ * Set a device file to use as a source of strong entropy from the operating
+ * system.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * This function overrides the default behavior, and overrides any
+ * setting in ottery_config_set_urandom_device.
+ *
+ * You MUST NOT change the the file descriptor while any libottery PRNG
+ * configured with it is still running. For example, don't close it, or use
+ * dup2 to make it refer to a different file, or anything like that.
+ *
+ * It is probably a good idea to open the file with the CLOEXEC flag set.
+ *
+ * @param cfg The configuration structure to configure.
+ * @param fd A file descriptor to use as an OS rng source.
+ */
+void ottery_config_set_urandom_fd(struct ottery_config *cfg,
+ int fd);
+
+struct sockaddr;
+
+/**
+ * Configure a socket at which to find a local copy of some service
+ * implementing the EGD (entropy-gathering daemon) protocol.
+ *
+ * Unless this function is called, EGD is not used by default.
+
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * TODO: This is not implemented for Windows yet.
+ *
+ * @param cfg The configuration structure to configure.
+ * @param addr The address of the daemon. Obviously, this should be
+ * some port on localhost, or a unix socket. This pointer is copied
+ * around, and must not be freed while any libottery state configured
+ * using this structure is still in use.
+ * @param len the length of the address.
+ *
+ */
+void ottery_config_set_egd_socket(struct ottery_config *cfg,
+ const struct sockaddr *addr,
+ int len);
+
+/**
+ * @brief External entropy sources.
+ *
+ * These can be passed as a bitmask to ottery_config_disable_entropy_sources.
+ *
+ * @{ */
+/** A unix-style /dev/urandom device. */
+#define OTTERY_ENTROPY_SRC_RANDOMDEV 0x0010000
+/** The Windows CryptGenRandom call. */
+#define OTTERY_ENTROPY_SRC_CRYPTGENRANDOM 0x0020000
+/** The Intel RDRAND instruction. */
+#define OTTERY_ENTROPY_SRC_RDRAND 0x0040000
+/** Some local server obeying the EGD protocol. Has no effect unless
+ * ottery_config_set_egd_socket was called. */
+#define OTTERY_ENTROPY_SRC_EGD 0x0080000
+/** @} */
+
+/**
+ * Disable the use of one or more entropy sources.
+ *
+ * Note that if enough entropy sources are disabled, the state will
+ * not be able to get initialized, and libottery might not work.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * @param cfg A configuration in which to disable one or more entropy sources.
+ * @param disabled_sources a bitwise combination of one or more
+ * OTTERY_ENTROPY_SRC_* values to disable. This will replace
+ * any previous bitmask of disabled sources.
+ *
+ */
+void ottery_config_disable_entropy_sources(struct ottery_config *cfg,
+ uint32_t disabled_sources);
+
+/**
+ * Mark one or more entropy sources as "weak".
+ *
+ * Unlike a disabled source, we will still try to read entropy from
+ * a weak source -- but we will fail if _only_ weak sources are available.
+ *
+ * Note that if enough entropy sources are disabled and/or weak sources are
+ * failing, the state will not be able to get initialized, and libottery might
+ * not work.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * @param cfg A configuration in which to disable one or more entropy sources.
+ * @param weak_sources a bitwise combination of one or more
+ * OTTERY_ENTROPY_SRC_* values to mark as weak. This will replace
+ * any previous bitmask of weak sources.
+ */
+void ottery_config_mark_entropy_sources_weak(struct ottery_config *cfg,
+ uint32_t weak_source);
+
+/** Size reserved for struct ottery_config */
+#define OTTERY_CONFIG_DUMMY_SIZE_ 1024
+
+#ifndef OTTERY_INTERNAL
+/**
+ * A configuration object for setting up a libottery instance.
+ *
+ * An ottery_config structure is initialized with ottery_config_init,
+ * and passed to ottery_init() or ottery_st_init().
+ *
+ * The contents of this structure are opaque; The definition here is
+ * defined to be large enough so that programs that allocate it will get
+ * more than enough room.
+ */
+struct ottery_config {
+ /** Nothing to see here */
+ uint8_t dummy_[OTTERY_CONFIG_DUMMY_SIZE_];
+};
+#endif
+
+/**
+ * Get the minimal size for allocating an ottery_config.
+ *
+ * sizeof(ottery_config) will give an overestimate to allow binary
+ * compatibility with future versions of libottery. Use this function instead
+ * to get the minimal number of bytes to allocate.
+ *
+ * @return The minimal number of bytes to use when allocating an
+ * ottery_config structure.
+ */
+size_t ottery_get_sizeof_config(void);
+
+/**
+ * @name libottery build flag
+ *
+ * @see ottery_Get_build_flags()
+ *
+ * @{
+ */
+/** Set if libottery was built with PID checking disabled. If this option is
+ * present, fork()ing can be dangerous. */
+#define OTTERY_BLDFLG_NO_PID_CHECK 0x00000001
+/** Set if libottery was built with initialization checking disabled. If this
+ * option is present, libottery might use an uninitialized, unseeded PRNGs.
+ */
+#define OTTERY_BLDFLG_NO_INIT_CHECK 0x00000002
+/** Set if locking was disabled. If this option is present, no libottery
+ * state, including the global state, is thread-safe. */
+#define OTTERY_BLDFLG_NO_LOCKING 0x00000004
+/** Set if the clear-after-yield feature was disabled. If this option is
+ * present, backtracking-resistance is somewhat compromised. */
+#define OTTERY_BLDFLG_NO_CLEAR_AFTER_YIELD 0x00000008
+/** Set if the stack-wiping feature was disabled. If this option is
+ * present, programs which accidentally read uninitialized data from the
+ * stack may leak some cryptographic state. */
+#define OTTERY_BLDFLG_NO_WIPE_STACK 0x00000010
+/** Set if SIMD support was disabled. This will make libottery slower. */
+#define OTTERY_BLDFLG_NO_SIMD 0x00010000
+/** @} */
+
+/** A bitmask of any flags that might affect safe and secure program
+ * operation. */
+#define OTTERY_BLDFLG_MASK_SAFETY 0x0000ffff
+
+/**
+ * Return a bitmask of flags describing the compile-time options that this
+ * libottery instance was built with. Some of these flags might make the
+ * library less safe to use!
+ */
+uint32_t ottery_get_build_flags(void);
+
+/**
+ * Return a run-time version number for Libottery. The first three bytes are
+ * the major number, minor number, and patch-level respectively. The final
+ * byte is 0 for a released version, and nonzero otherwise.
+ */
+uint32_t ottery_get_version(void);
+/**
+ * Return a human-readable string representing the run-time Libottery version.
+ */
+const char *ottery_get_version_string(void);
+
+const char *ottery_get_impl_name(void);
+
+#endif
diff --git a/contrib/libottery/ottery_cpuinfo.c b/contrib/libottery/ottery_cpuinfo.c
new file mode 100644
index 0000000..2e8bdaa
--- /dev/null
+++ b/contrib/libottery/ottery_cpuinfo.c
@@ -0,0 +1,88 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#include "ottery-internal.h"
+#include <stdint.h>
+
+#if defined(i386) || \
+ defined(__i386) || \
+ defined(__M_IX86) || \
+ defined(_M_IX86)
+#define X86
+#elif defined(__x86_64) || \
+ defined(_M_AMD64)
+#define X86
+#define X86_64
+#endif
+
+#if defined(__arm__) || \
+ defined(_M_ARM)
+#define ARM
+#endif
+
+#if defined(X86)
+#ifdef _MSC_VER
+#include <intrin.h>
+#define cpuid(a,b) __cpuid((b), (a))
+#else
+static void
+cpuid(int index, int regs[4])
+{
+ unsigned int eax, ebx, ecx, edx;
+#ifdef X86_64
+ __asm("cpuid" : "=a"(eax), "=b" (ebx), "=c"(ecx), "=d"(edx)
+ : "0"(index));
+#else
+ __asm volatile(
+ "xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
+ : "=a" (eax), "=r" (ebx), "=c" (ecx), "=d" (edx)
+ : "0" (index)
+ : "cc" );
+#endif
+
+ regs[0] = eax;
+ regs[1] = ebx;
+ regs[2] = ecx;
+ regs[3] = edx;
+}
+#endif
+#endif
+
+static uint32_t disabled_cpu_capabilities = 0;
+
+void
+ottery_disable_cpu_capabilities_(uint32_t disable)
+{
+ disabled_cpu_capabilities |= disable;
+}
+
+uint32_t
+ottery_get_cpu_capabilities_(void)
+{
+#ifdef X86
+ uint32_t cap = 0;
+ int res[4];
+ cpuid(1, res);
+ if (res[3] & (1<<26))
+ cap |= OTTERY_CPUCAP_SIMD;
+ if (res[2] & (1<<9))
+ cap |= OTTERY_CPUCAP_SSSE3;
+ if (res[2] & (1<<25))
+ cap |= OTTERY_CPUCAP_AES;
+ if (res[2] & (1<<30))
+ cap |= OTTERY_CPUCAP_RAND;
+#else
+ uint32_t cap = OTTERY_CPUCAP_SIMD;
+#endif
+ return cap & ~disabled_cpu_capabilities;
+}
diff --git a/contrib/libottery/ottery_entropy.c b/contrib/libottery/ottery_entropy.c
new file mode 100644
index 0000000..819a380
--- /dev/null
+++ b/contrib/libottery/ottery_entropy.c
@@ -0,0 +1,112 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include "ottery-internal.h"
+#include "ottery.h"
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+#define SRC(x) OTTERY_ENTROPY_SRC_ ## x
+#define DOM(x) OTTERY_ENTROPY_DOM_ ## x
+#define FL(x) OTTERY_ENTROPY_FL_ ## x
+
+#include "ottery_entropy_cryptgenrandom.c"
+#include "ottery_entropy_urandom.c"
+#include "ottery_entropy_rdrand.c"
+#include "ottery_entropy_egd.c"
+
+/** Table of RNG functions and their properties. */
+static struct ottery_randbytes_source {
+ int (*fn)(const struct ottery_entropy_config *,
+ struct ottery_entropy_state *,
+ uint8_t *, size_t);
+ uint32_t flags;
+} RAND_SOURCES[] = {
+#ifdef ENTROPY_SOURCE_CRYPTGENRANDOM
+ ENTROPY_SOURCE_CRYPTGENRANDOM,
+#endif
+#ifdef ENTROPY_SOURCE_URANDOM
+ ENTROPY_SOURCE_URANDOM,
+#endif
+#ifdef ENTROPY_SOURCE_EGD
+ ENTROPY_SOURCE_EGD,
+#endif
+#ifdef ENTROPY_SOURCE_RDRAND
+ ENTROPY_SOURCE_RDRAND,
+#endif
+ { NULL, 0 }
+};
+
+size_t
+ottery_get_entropy_bufsize_(size_t n)
+{
+ return n * (sizeof(RAND_SOURCES)/sizeof(RAND_SOURCES[0]) - 1);
+}
+
+int
+ottery_get_entropy_(const struct ottery_entropy_config *config,
+ struct ottery_entropy_state *state,
+ uint32_t select_sources,
+ uint8_t *bytes, size_t n, size_t *buflen,
+ uint32_t *flags_out)
+{
+ ssize_t err = OTTERY_ERR_INIT_STRONG_RNG, last_err = 0;
+ int i;
+ uint32_t got = 0;
+ uint8_t *next;
+ const uint32_t disabled_sources = config ? config->disabled_sources : 0;
+
+ memset(bytes, 0, *buflen);
+ next = bytes;
+
+ *flags_out = 0;
+
+ for (i=0; RAND_SOURCES[i].fn; ++i) {
+ uint32_t flags = RAND_SOURCES[i].flags;
+ /* Don't use a disabled source. */
+ if (0 != (flags & disabled_sources))
+ continue;
+ /* If some flags must be set, only use those. */
+ if ((flags & select_sources) != select_sources)
+ continue;
+ /* If we already have input from a certain domain, we don't need more */
+ if ((flags & (got & OTTERY_ENTROPY_DOM_MASK)) != 0)
+ continue;
+ /* If we can't write these bytes, don't try. */
+ if (next + n > bytes + *buflen)
+ break;
+ err = RAND_SOURCES[i].fn(config, state, next, n);
+ if (err == 0) {
+ uint32_t flags = RAND_SOURCES[i].flags;
+ if (config && (flags & config->weak_sources))
+ flags &= ~OTTERY_ENTROPY_FL_STRONG;
+
+ got |= flags;
+ next += n;
+ } else {
+ last_err = err;
+ }
+ }
+
+ /* Do not report success unless at least one source was strong. */
+ if (0 == (got & OTTERY_ENTROPY_FL_STRONG))
+ return last_err ? last_err : OTTERY_ERR_INIT_STRONG_RNG;
+
+ *flags_out = got;
+ *buflen = next - bytes;
+
+ return 0;
+}
diff --git a/contrib/libottery/ottery_entropy_cryptgenrandom.c b/contrib/libottery/ottery_entropy_cryptgenrandom.c
new file mode 100644
index 0000000..d29d9d1
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_cryptgenrandom.c
@@ -0,0 +1,51 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include "ottery-internal.h"
+#include "ottery.h"
+
+#ifdef _WIN32
+
+/** Generate random bytes using the Windows CryptGenRandom operating-system
+ * RNG. */
+static int
+ottery_get_entropy_cryptgenrandom(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ /* On Windows, CryptGenRandom is supposed to be a well-seeded
+ * cryptographically strong random number generator. */
+ HCRYPTPROV provider;
+ int retval = 0;
+ (void) cfg;
+ (void) state;
+
+ if (0 == CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL,
+ CRYPT_VERIFYCONTEXT))
+ return OTTERY_ERR_INIT_STRONG_RNG;
+
+ if (0 == CryptGenRandom(provider, outlen, out))
+ retval = OTTERY_ERR_ACCESS_STRONG_RNG;
+
+ CryptReleaseContext(provider, 0);
+ return retval;
+}
+
+#define ENTROPY_SOURCE_CRYPTGENRANDOM \
+ { ottery_get_entropy_cryptgenrandom, \
+ SRC(CRYPTGENRANDOM)|DOM(OS)|FL(STRONG) }
+
+#endif
+
+
diff --git a/contrib/libottery/ottery_entropy_egd.c b/contrib/libottery/ottery_entropy_egd.c
new file mode 100644
index 0000000..7e9cb07
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_egd.c
@@ -0,0 +1,75 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef _WIN32
+/* TODO: Support win32. */
+#include <sys/socket.h>
+
+/** Implement an entropy-source that uses the EGD protocol. The
+ * Entropy-Gathering Daemon is program (actually, one of several programs)
+ * that watches system events, periodically runs commands whose outputs have
+ * high variance, and so on. It communicates over a simple socket-based
+ * protocol, of which we use only a tiny piece. */
+static int
+ottery_get_entropy_egd(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ int sock, n, result;
+ unsigned char msg[2];
+ (void) state;
+
+ if (! cfg || ! cfg->egd_sockaddr || ! cfg->egd_socklen)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ if (outlen > 255)
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+
+ sock = socket(cfg->egd_sockaddr->sa_family, SOCK_STREAM, 0);
+ if (sock < 0)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+
+ if (connect(sock, cfg->egd_sockaddr, cfg->egd_socklen) < 0) {
+ result = OTTERY_ERR_INIT_STRONG_RNG;
+ goto out;
+ }
+
+ msg[0] = 1; /* nonblocking request */
+ msg[1] = (unsigned char) outlen; /* for outlen bytes */
+
+ if (write(sock, msg, 2) != 2 ||
+ read(sock, msg, 1) != 1) {
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+ goto out;
+ }
+
+ if (msg[0] != outlen) {
+ /* TODO Use any bytes we get, even if they aren't as many as we wanted. */
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+ goto out;
+ }
+
+ n = ottery_read_n_bytes_from_file_(sock, out, outlen);
+ if (n < 0 || (size_t)n != outlen) {
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+ goto out;
+ }
+ result = 0;
+ out:
+ close(sock);
+ return result;
+}
+
+#define ENTROPY_SOURCE_EGD \
+ { ottery_get_entropy_egd, SRC(EGD)|DOM(EGD)|FL(STRONG) }
+
+#endif
diff --git a/contrib/libottery/ottery_entropy_rdrand.c b/contrib/libottery/ottery_entropy_rdrand.c
new file mode 100644
index 0000000..4f227d3
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_rdrand.c
@@ -0,0 +1,58 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+
+#if defined(i386) || \
+ defined(__i386) || \
+ defined(__x86_64) || \
+ defined(__M_IX86) || \
+ defined(_M_IX86) || \
+ defined(__INTEL_COMPILER)
+
+extern int ottery_valgrind_;
+/** Helper: invoke the RDRAND instruction to get 4 random bytes in the output
+ * value. Return 1 on success, and 0 on failure. */
+#define rdrand32(x) ({ unsigned char err = 0; __asm volatile(".byte 0x0f; .byte 0xc7; .byte 0xf0; setc %1":"=a"(x), "=qm"(err) :"a"(0) :"cc"); err; })
+
+/** Generate bytes using the Intel RDRAND instruction. */
+static int
+ottery_get_entropy_rdrand(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ uint32_t up;
+ (void) cfg;
+ (void) state;
+ if (! (ottery_get_cpu_capabilities_() & OTTERY_CPUCAP_RAND) || ottery_valgrind_)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ while (outlen >= 4) {
+ if (rdrand32(up) != 1)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ memcpy (out, &up, sizeof (up));
+ out += sizeof (up);
+ outlen -= 4;
+ }
+
+ if (outlen) {
+ if (rdrand32(up) != 1)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ memcpy(out, &up, outlen);
+ }
+ return 0;
+}
+
+#define ENTROPY_SOURCE_RDRAND \
+ { ottery_get_entropy_rdrand, SRC(RDRAND)|DOM(CPU)|FL(FAST)|FL(STRONG) }
+
+#endif
+
diff --git a/contrib/libottery/ottery_entropy_urandom.c b/contrib/libottery/ottery_entropy_urandom.c
new file mode 100644
index 0000000..9eb761b
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_urandom.c
@@ -0,0 +1,117 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+
+#ifndef _WIN32
+
+/**
+ * Read from a file into an n-byte buffer until the buffer is full or until
+ * we reach an error. Returns the number of bytes read. If the return
+ * value is less than n, an error occurred.
+ */
+static int
+ottery_read_n_bytes_from_file_(int fd, uint8_t *out, size_t n)
+{
+ ssize_t r;
+ uint8_t *outp = out;
+ while (n) {
+ r = read(fd, outp, n);
+ if (r <= 0 || (size_t)r > n)
+ return outp - out;
+ outp += r;
+ n -= r;
+ }
+ return outp - out;
+}
+
+
+/** Generate random bytes using the unix-style /dev/urandom RNG, or another
+ * such device as configured in the configuration. */
+static int
+ottery_get_entropy_urandom(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ /* On most unixes these days, you can get strong random numbers from
+ * /dev/urandom.
+ *
+ * That's assuming that /dev/urandom is seeded. For most applications,
+ * that won't be a problem. But for stuff that starts close to system
+ * startup, before the operating system has added any entropy to the pool,
+ * it can be pretty bad.
+ *
+ * You could use /dev/random instead, if you want, but that has another
+ * problem. It will block if the OS PRNG has received less entropy than
+ * it has emitted. If we assume that the OS PRNG isn't cryptographically
+ * weak, blocking in that case is simple overkill.
+ *
+ * It would be best if there were an alternative that blocked if the PRNG
+ * had _never_ been seeded. But most operating systems don't have that.
+ */
+ int fd;
+ ssize_t n;
+ int result = 0;
+ const char *urandom_fname;
+ struct stat st;
+ int own_fd = 0;
+ int check_device = !cfg || !cfg->allow_nondev_urandom;
+#ifndef O_CLOEXEC
+#define O_CLOEXEC 0
+#endif
+ if (cfg && cfg->urandom_fd_is_set && cfg->urandom_fd >= 0) {
+ fd = cfg->urandom_fd;
+ } else {
+ if (cfg && cfg->urandom_fname)
+ urandom_fname = cfg->urandom_fname;
+ else
+ urandom_fname = "/dev/urandom";
+
+ fd = open(urandom_fname, O_RDONLY|O_CLOEXEC);
+ own_fd = 1;
+ if (fd < 0)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ }
+ if (fstat(fd, &st) < 0) {
+ result = OTTERY_ERR_INIT_STRONG_RNG;
+ goto end;
+ }
+ if (check_device) {
+ if (0 == (st.st_mode & S_IFCHR)) {
+ result = OTTERY_ERR_INIT_STRONG_RNG;
+ goto end;
+ }
+
+ if (state) {
+ if (0 == state->urandom_fd_inode) {
+ state->urandom_fd_inode = (uint64_t) st.st_ino;
+ } else if ((uint64_t)st.st_ino != state->urandom_fd_inode) {
+ close(fd);
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+ }
+ }
+ }
+
+ n = ottery_read_n_bytes_from_file_(fd, out, outlen);
+ if (n < 0 || (size_t)n != outlen)
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+
+ end:
+ if (own_fd)
+ close(fd);
+ return result;
+}
+
+#define ENTROPY_SOURCE_URANDOM \
+ { ottery_get_entropy_urandom, SRC(RANDOMDEV)|DOM(OS)|FL(STRONG) }
+
+#endif
diff --git a/contrib/libottery/ottery_global.c b/contrib/libottery/ottery_global.c
new file mode 100644
index 0000000..dd1efc5
--- /dev/null
+++ b/contrib/libottery/ottery_global.c
@@ -0,0 +1,116 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include <stdlib.h>
+#include "ottery-internal.h"
+#include "ottery.h"
+#include "ottery_st.h"
+
+
+/** Flag: true iff ottery_global_state_ is initialized. */
+static int ottery_global_state_initialized_ = 0;
+int ottery_valgrind_ = 0;
+/** A global state to use for the ottery_* functions that don't take a
+ * state. */
+static struct ottery_state ottery_global_state_;
+
+/** Initialize ottery_global_state_ if it has not been initialize. */
+#define CHECK_INIT(rv) do { \
+ if (UNLIKELY(!ottery_global_state_initialized_)) { \
+ int err; \
+ if ((err = ottery_init(NULL))) { \
+ ottery_fatal_error_(OTTERY_ERR_FLAG_GLOBAL_PRNG_INIT|err); \
+ return rv; \
+ } \
+ } \
+} while (0)
+
+int
+ottery_init(const struct ottery_config *cfg)
+{
+ if (getenv("VALGRIND")) {
+ ottery_valgrind_ = 1;
+ }
+ int n = ottery_st_init(&ottery_global_state_, cfg);
+ if (n == 0)
+ ottery_global_state_initialized_ = 1;
+ return n;
+}
+
+int
+ottery_add_seed(const uint8_t *seed, size_t n)
+{
+ CHECK_INIT(0);
+ return ottery_st_add_seed(&ottery_global_state_, seed, n);
+}
+
+void
+ottery_wipe(void)
+{
+ if (ottery_global_state_initialized_) {
+ ottery_global_state_initialized_ = 0;
+ ottery_st_wipe(&ottery_global_state_);
+ }
+}
+
+void
+ottery_prevent_backtracking(void)
+{
+ CHECK_INIT();
+ ottery_st_prevent_backtracking(&ottery_global_state_);
+}
+
+void
+ottery_rand_bytes(void *out, size_t n)
+{
+ CHECK_INIT();
+ ottery_st_rand_bytes(&ottery_global_state_, out, n);
+}
+
+unsigned
+ottery_rand_unsigned(void)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_unsigned(&ottery_global_state_);
+}
+uint32_t
+ottery_rand_uint32(void)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_uint32(&ottery_global_state_);
+}
+uint64_t
+ottery_rand_uint64(void)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_uint64(&ottery_global_state_);
+}
+unsigned
+ottery_rand_range(unsigned top)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_range(&ottery_global_state_, top);
+}
+uint64_t
+ottery_rand_range64(uint64_t top)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_range64(&ottery_global_state_, top);
+}
+
+const char *ottery_get_impl_name(void)
+{
+ CHECK_INIT(0);
+ return ottery_global_state_.prf.name;
+} \ No newline at end of file
diff --git a/contrib/libottery/ottery_nolock.h b/contrib/libottery/ottery_nolock.h
new file mode 100644
index 0000000..b504e11
--- /dev/null
+++ b/contrib/libottery/ottery_nolock.h
@@ -0,0 +1,190 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_NOLOCK_H_HEADER_INCLUDED_
+#define OTTERY_NOLOCK_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ottery_common.h"
+
+/** @file */
+
+struct ottery_config;
+struct ottery_state_nolock;
+
+/** Size reserved for struct ottery_state_nolock */
+#define OTTERY_STATE_NOLOCK_DUMMY_SIZE_ 1536
+
+#ifndef OTTERY_INTERNAL
+/**
+ * The state for a non-thread-safe libottery PRNG
+ *
+ * Like struct ottery_state, but this structure (and its associated functions)
+ * are not thread safe. If you try to use this structure in more than one
+ * thread at a time, your program's behavior will be undefined. It might
+ * crash. It might insecurely give the same random sequence to multiple
+ * threads. It might fail in strange ways that you'd never predict.
+ *
+ * An ottery_state_nolock structure is constucted with ottery_st_init(). It
+ * MUST be aligned on a 16-byte boundary.
+ *
+ * You may not use an ottery_state_nolock structure with any other function
+ * before you have first initialized it with ottery_st_init_nolock().
+ *
+ * The contents of this structure are opaque; The definition here is
+ * defined to be large enough so that programs that allocate it will get
+ * more than enough room.
+ */
+struct __attribute__((aligned(16))) ottery_state_nolock {
+ /** Nothing to see here */
+ uint8_t dummy_[OTTERY_STATE_NOLOCK_DUMMY_SIZE_];
+};
+#endif
+
+/**
+ * Get the minimal size for allocating an ottery_state_nolock.
+ *
+ * sizeof(ottery_state_nolock) will give an overestimate to allow binary
+ * compatibility with future versions of libottery. Use this function instead
+ * to get the minimal number of bytes to allocate.
+ *
+ * @return The minimal number of bytes to use when allocating an
+ * ottery_state_nolock structure.
+ */
+size_t ottery_get_sizeof_state_nolock(void);
+
+/**
+ * Initialize an ottery_state_nolock structure.
+ *
+ * You must call this function on any ottery_state_nolock structure before
+ * calling any other functions on it.
+ *
+ * @param st The ottery_state_nolock to initialize.
+ * @param cfg Either NULL, or an ottery_config structure that has been
+ * initialized with ottery_config_init().
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_init_nolock(struct ottery_state_nolock *st, const struct ottery_config *cfg);
+
+/**
+ * Add more entropy to an ottery_state_nolock structure.
+ *
+ * Calling this function should be needless, if you trust your operating
+ * system's random number generator and entropy extraction features. You
+ * would want to use this function if you think the operating system's random
+ * number generator might be inadequate, and you want to add more entropy from
+ * EGD or something.
+ *
+ * You might also want to call this function if your belief system says that
+ * it's useful to periodically add more raw entropy to a well-seeded
+ * cryptographically strong PRNG.
+ *
+ * @param st The state which will receive more entropy.
+ * @param seed Bytes to add to the state.
+ * @param n The number of bytes to add.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_add_seed_nolock(struct ottery_state_nolock *st, const uint8_t *seed, size_t n);
+
+/**
+ * Destroy an ottery_state_nolock structure and release any resources that it
+ * might hold.
+ *
+ * Ordinarily, you would want to call this at exit, or before freeing an
+ * ottery_state_nolock
+ *
+ * @param st The state to wipe.
+ */
+void ottery_st_wipe_nolock(struct ottery_state_nolock *st);
+
+/**
+ * Explicitly prevent backtracking attacks. (Usually needless).
+ *
+ * Once this function has been called, an attacker who compromises the state
+ * later on will not be able to recover bytes that have previously been
+ * returned by any of the ottery_st_rand_*_nolock functions.
+ *
+ * You should not usually need to call this function: Libottery provides
+ * backtracking resistance by default, so unless you have manually recompiled
+ * with the OTTERY_NO_CLEAR_AFTER_YIELD option, this function isn't
+ * necessary and has no effect. Even *with* OTTERY_NO_CLEAR_AFTER_YIELD,
+ * this function isn't necessary in ordinary operation: the libottery state is
+ * implicitly "stirred" every 1k or so.
+ *
+ * @param st The state to stir.
+ */
+void ottery_st_prevent_backtracking_nolock(struct ottery_state_nolock *st);
+
+/**
+ * Use an ottery_state_nolock structure to fill a buffer with random bytes.
+ *
+ * @param st The state structure to use.
+ * @param buf The buffer to fill.
+ * @param n The number of bytes to write.
+ */
+void ottery_st_rand_bytes_nolock(struct ottery_state_nolock *st, void *buf, size_t n);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type unsigned.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT_MAX included,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_unsigned_nolock(struct ottery_state_nolock *st);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type uint32_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT32_MAX included,
+ * chosen uniformly.
+ */
+uint32_t ottery_st_rand_uint32_nolock(struct ottery_state_nolock *st);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type uint64_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT64_MAX included,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_uint64_nolock(struct ottery_state_nolock *st);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type unsigned
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_range_nolock(struct ottery_state_nolock *st, unsigned top);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type uint64_t
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_range64_nolock(struct ottery_state_nolock *st, uint64_t top);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery_st.h b/contrib/libottery/ottery_st.h
new file mode 100644
index 0000000..38955cb
--- /dev/null
+++ b/contrib/libottery/ottery_st.h
@@ -0,0 +1,184 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_ST_H_HEADER_INCLUDED_
+#define OTTERY_ST_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ottery_common.h"
+
+/** @file */
+
+struct ottery_config;
+struct ottery_state;
+
+/** Size reserved for struct ottery_state */
+#define OTTERY_STATE_DUMMY_SIZE_ 1536
+
+#ifndef OTTERY_INTERNAL
+/**
+ * The state for a libottery PRNG.
+ *
+ * An ottery_state structure is constucted with ottery_st_init(). It MUST be
+ * aligned on a 16-byte boundary.
+ *
+ * You may not use an ottery_state structure with any other function before
+ * you have first initialized it with ottery_st_init().
+ *
+ * The contents of this structure are opaque; The definition here is
+ * defined to be large enough so that programs that allocate it will get
+ * more than enough room.
+ */
+struct __attribute__((aligned(16))) ottery_state {
+ /** Nothing to see here */
+ uint8_t dummy_[OTTERY_STATE_DUMMY_SIZE_];
+};
+#endif
+
+/**
+ * Get the minimal size for allocating an ottery_state.
+ *
+ * sizeof(ottery_state) will give an overestimate to allow binary
+ * compatibility with future versions of libottery. Use this function instead
+ * to get the minimal number of bytes to allocate.
+ *
+ * @return The minimal number of bytes to use when allocating an
+ * ottery_state structure.
+ */
+size_t ottery_get_sizeof_state(void);
+
+/**
+ * Initialize an ottery_state structure.
+ *
+ * You must call this function on any ottery_state structure before
+ * calling any other functions on it.
+ *
+ * @param st The ottery_state to initialize.
+ * @param cfg Either NULL, or an ottery_config structure that has been
+ * initialized with ottery_config_init().
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_init(struct ottery_state *st, const struct ottery_config *cfg);
+
+/**
+ * Add more entropy to an ottery_state structure.
+ *
+ * Calling this function should be needless, if you trust your operating
+ * system's random number generator and entropy extraction features. You
+ * would want to use this function if you think the operating system's random
+ * number generator might be inadequate, and you want to add more entropy from
+ * EGD or something.
+ *
+ * You might also want to call this function if your belief system says that
+ * it's useful to periodically add more raw entropy to a well-seeded
+ * cryptographically strong PRNG.
+ *
+ * @param st The state which will receive more entropy.
+ * @param seed Bytes to add to the state.
+ * @param n The number of bytes to add.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_add_seed(struct ottery_state *st, const uint8_t *seed, size_t n);
+
+/**
+ * Destroy an ottery_state structure and release any resources that it might
+ * hold.
+ *
+ * Ordinarily, you would want to call this at exit, or before freeing an
+ * ottery_state
+ *
+ * @param st The state to wipe.
+ */
+void ottery_st_wipe(struct ottery_state *st);
+
+/**
+ * Explicitly prevent backtracking attacks. (Usually needless).
+ *
+ * Once this function has been called, an attacker who compromises the state
+ * later on will not be able to recover bytes that have previously been
+ * returned by any of the ottery_st_rand_* functions.
+ *
+ * You should not usually need to call this function: Libottery provides
+ * backtracking resistance by default, so unless you have manually recompiled
+ * with the OTTERY_NO_CLEAR_AFTER_YIELD option, this function isn't
+ * necessary and has no effect. Even *with* OTTERY_NO_CLEAR_AFTER_YIELD,
+ * this function isn't necessary in ordinary operation: the libottery state is
+ * implicitly "stirred" every 1k or so.
+ *
+ * @param st The state to stir.
+ */
+void ottery_st_prevent_backtracking(struct ottery_state *st);
+
+/**
+ * Use an ottery_state structure to fill a buffer with random bytes.
+ *
+ * @param st The state structure to use.
+ * @param buf The buffer to fill.
+ * @param n The number of bytes to write.
+ */
+void ottery_st_rand_bytes(struct ottery_state *st, void *buf, size_t n);
+/**
+ * Use an ottery_state structure to generate a random number of type unsigned.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT_MAX included,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_unsigned(struct ottery_state *st);
+/**
+ * Use an ottery_state structure to generate a random number of type uint32_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT32_MAX included,
+ * chosen uniformly.
+ */
+uint32_t ottery_st_rand_uint32(struct ottery_state *st);
+/**
+ * Use an ottery_state structure to generate a random number of type uint64_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT64_MAX included,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_uint64(struct ottery_state *st);
+/**
+ * Use an ottery_state structure to generate a random number of type unsigned
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_range(struct ottery_state *st, unsigned top);
+/**
+ * Use an ottery_state structure to generate a random number of type uint64_t
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_range64(struct ottery_state *st, uint64_t top);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery_version.h.in b/contrib/libottery/ottery_version.h.in
new file mode 100644
index 0000000..1b6a244
--- /dev/null
+++ b/contrib/libottery/ottery_version.h.in
@@ -0,0 +1,28 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_VERSION_H_HEADER_INCLUDED_
+#define OTTERY_VERSION_H_HEADER_INCLUDED_
+
+/**
+ * Version number for Libottery. The first three bytes are the major number,
+ * minor number, and patch-level respectively. The final byte is 0 for a
+ * released version, and nonzero otherwise.
+ */
+#undef OTTERY_VERSION
+/**
+ * Human-readable string representing the Libottery version.
+ */
+#undef OTTERY_VERSION_STRING
+
+#endif
diff --git a/contrib/librdns/CMakeLists.txt b/contrib/librdns/CMakeLists.txt
new file mode 100644
index 0000000..a5733e6
--- /dev/null
+++ b/contrib/librdns/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(LIBRDNSSRC util.c
+ logger.c
+ compression.c
+ punycode.c
+ curve.c
+ parse.c
+ packet.c
+ resolver.c)
+
+ADD_LIBRARY(rdns STATIC ${LIBRDNSSRC})
+SET_TARGET_PROPERTIES(rdns PROPERTIES COMPILE_FLAGS "-DUSE_RSPAMD_CRYPTOBOX") \ No newline at end of file
diff --git a/contrib/librdns/compression.c b/contrib/librdns/compression.c
new file mode 100644
index 0000000..895178e
--- /dev/null
+++ b/contrib/librdns/compression.c
@@ -0,0 +1,168 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "compression.h"
+#include "logger.h"
+#include "contrib/mumhash/mum.h"
+
+#define rdns_compression_hash(n) (mum_hash(n.suffix, n.suffix_len, 0xdeadbeef))
+#define rdns_compression_equal(n1, n2) ((n1).suffix_len == (n2).suffix_len && \
+ (memcmp((n1).suffix, (n2).suffix, (n1).suffix_len) == 0))
+__KHASH_IMPL(rdns_compression_hash, kh_inline, struct rdns_compression_name, char, 0, rdns_compression_hash,
+ rdns_compression_equal);
+
+static struct rdns_compression_name *
+rdns_can_compress (const char *pos, unsigned int len, khash_t(rdns_compression_hash) *comp)
+{
+ struct rdns_compression_name check;
+ khiter_t k;
+
+ if (comp == NULL) {
+ return NULL;
+ }
+
+ check.suffix_len = len;
+ check.suffix = pos;
+ k = kh_get(rdns_compression_hash, comp, check);
+
+ if (k != kh_end(comp)) {
+ return &kh_key(comp, k);
+ }
+
+ return NULL;
+}
+
+static unsigned int
+rdns_calculate_label_len (const char *pos, const char *end)
+{
+ const char *p = pos;
+ unsigned int res = 0;
+
+ while (p != end) {
+ if (*p == '.') {
+ break;
+ }
+ res ++;
+ p ++;
+ }
+ return res;
+}
+
+static void
+rdns_add_compressed (const char *pos, const char *end,
+ khash_t(rdns_compression_hash) *comp,
+ int offset)
+{
+ struct rdns_compression_name new_name;
+ int r;
+
+ if (comp != NULL) {
+
+ assert (offset >= 0);
+ new_name.suffix_len = end - pos;
+ new_name.suffix = pos;
+ new_name.offset = offset;
+
+ kh_put(rdns_compression_hash, comp, new_name, &r);
+ }
+}
+
+void
+rdns_compression_free (khash_t(rdns_compression_hash) *comp)
+{
+ if (comp != NULL) {
+ kh_destroy(rdns_compression_hash, comp);
+ }
+}
+
+bool
+rdns_write_name_compressed (struct rdns_request *req,
+ const char *name, unsigned int namelen,
+ khash_t(rdns_compression_hash) **comp)
+{
+ uint8_t *target = req->packet + req->pos;
+ const char *pos = name, *end = name + namelen;
+ unsigned int remain = req->packet_len - req->pos - 5, label_len;
+ struct rdns_resolver *resolver = req->resolver;
+ uint16_t pointer;
+
+ if (comp != NULL && *comp == NULL) {
+ *comp = kh_init(rdns_compression_hash);
+ }
+
+ while (pos < end && remain > 0) {
+ if (comp) {
+ struct rdns_compression_name *test = rdns_can_compress(pos, end - pos, *comp);
+ if (test != NULL) {
+ /* Can compress name */
+ if (remain < 2) {
+ rdns_info ("no buffer remain for constructing query");
+ return false;
+ }
+
+ pointer = htons ((uint16_t) test->offset) | DNS_COMPRESSION_BITS;
+ memcpy(target, &pointer, sizeof(pointer));
+ req->pos += 2;
+
+ return true;
+ }
+ }
+
+ label_len = rdns_calculate_label_len (pos, end);
+ if (label_len == 0) {
+ /* We have empty label it is allowed only if pos == end - 1 */
+ if (pos == end - 1) {
+ break;
+ }
+ else {
+ rdns_err ("double dots in the name requested");
+ return false;
+ }
+ }
+ else if (label_len > DNS_D_MAXLABEL) {
+ rdns_err ("too large label: %d", (int)label_len);
+ return false;
+ }
+
+ if (label_len + 1 > remain) {
+ rdns_info ("no buffer remain for constructing query, strip %d to %d",
+ (int)label_len, (int)remain);
+ label_len = remain - 1;
+ }
+
+ if (comp) {
+ rdns_add_compressed(pos, end, *comp, target - req->packet);
+ }
+ /* Write label as is */
+ *target++ = (uint8_t)label_len;
+ memcpy (target, pos, label_len);
+ target += label_len;
+ pos += label_len + 1;
+ }
+
+ /* Termination label */
+ *target++ = '\0';
+ req->pos = target - req->packet;
+
+ return true;
+}
diff --git a/contrib/librdns/compression.h b/contrib/librdns/compression.h
new file mode 100644
index 0000000..6056117
--- /dev/null
+++ b/contrib/librdns/compression.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef COMPRESSION_H_
+#define COMPRESSION_H_
+
+#include "dns_private.h"
+#include "khash.h"
+
+struct rdns_compression_name {
+ const char *suffix; /**< suffix packed */
+ unsigned int suffix_len; /**< length of the suffix */
+ unsigned int offset; /**< offset in the packet */
+};
+
+
+KHASH_DECLARE(rdns_compression_hash, struct rdns_compression_name, char);
+
+/**
+ * Try to compress name passed or write it 'as is'
+ * @return
+ */
+bool rdns_write_name_compressed (struct rdns_request *req,
+ const char *name, unsigned int namelen,
+ khash_t(rdns_compression_hash) **comp);
+
+void rdns_compression_free (khash_t(rdns_compression_hash) *comp);
+
+#endif /* COMPRESSION_H_ */
diff --git a/contrib/librdns/curve.c b/contrib/librdns/curve.c
new file mode 100644
index 0000000..19ec250
--- /dev/null
+++ b/contrib/librdns/curve.c
@@ -0,0 +1,890 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "rdns_curve.h"
+#include "ottery.h"
+#include "ref.h"
+#include "logger.h"
+
+#ifdef TWEETNACL
+
+#include <tweetnacl.h>
+
+void
+randombytes(uint8_t *data, uint64_t len)
+{
+ ottery_rand_bytes (data, len);
+}
+void sodium_memzero (uint8_t *data, uint64_t len)
+{
+ volatile uint8_t *p = data;
+
+ while (len--) {
+ *p = '\0';
+ }
+}
+void sodium_init(void)
+{
+
+}
+
+ssize_t rdns_curve_send (struct rdns_request *req, void *plugin_data);
+ssize_t rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len,
+ void *plugin_data, struct rdns_request **req_out);
+void rdns_curve_finish_request (struct rdns_request *req, void *plugin_data);
+void rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data);
+
+struct rdns_curve_entry {
+ char *name;
+ unsigned char pk[crypto_box_PUBLICKEYBYTES];
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_nm_entry {
+ unsigned char k[crypto_box_BEFORENMBYTES];
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *prev, *next;
+};
+
+struct rdns_curve_client_key {
+ unsigned char pk[crypto_box_PUBLICKEYBYTES];
+ unsigned char sk[crypto_box_SECRETKEYBYTES];
+ struct rdns_curve_nm_entry *nms;
+ uint64_t counter;
+ unsigned int uses;
+ ref_entry_t ref;
+};
+
+struct rdns_curve_request {
+ struct rdns_request *req;
+ struct rdns_curve_client_key *key;
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *nm;
+ unsigned char nonce[crypto_box_NONCEBYTES];
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_ctx {
+ struct rdns_curve_entry *entries;
+ struct rdns_curve_client_key *cur_key;
+ struct rdns_curve_request *requests;
+ double key_refresh_interval;
+ void *key_refresh_event;
+ struct rdns_resolver *resolver;
+};
+
+static struct rdns_curve_client_key *
+rdns_curve_client_key_new (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_client_key *new;
+ struct rdns_curve_nm_entry *nm;
+ struct rdns_curve_entry *entry, *tmp;
+
+ new = calloc (1, sizeof (struct rdns_curve_client_key));
+ crypto_box_keypair (new->pk, new->sk);
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ nm = calloc (1, sizeof (struct rdns_curve_nm_entry));
+ nm->entry = entry;
+ crypto_box_beforenm (nm->k, entry->pk, new->sk);
+
+ DL_APPEND (new->nms, nm);
+ }
+
+ new->counter = ottery_rand_uint64 ();
+
+ return new;
+}
+
+static struct rdns_curve_nm_entry *
+rdns_curve_find_nm (struct rdns_curve_client_key *key, struct rdns_curve_entry *entry)
+{
+ struct rdns_curve_nm_entry *nm;
+
+ DL_FOREACH (key->nms, nm) {
+ if (nm->entry == entry) {
+ return nm;
+ }
+ }
+
+ return NULL;
+}
+
+static void
+rdns_curve_client_key_free (struct rdns_curve_client_key *key)
+{
+ struct rdns_curve_nm_entry *nm, *tmp;
+
+ DL_FOREACH_SAFE (key->nms, nm, tmp) {
+ sodium_memzero (nm->k, sizeof (nm->k));
+ free (nm);
+ }
+ sodium_memzero (key->sk, sizeof (key->sk));
+ free (key);
+}
+
+struct rdns_curve_ctx*
+rdns_curve_ctx_new (double key_refresh_interval)
+{
+ struct rdns_curve_ctx *new;
+
+ new = calloc (1, sizeof (struct rdns_curve_ctx));
+ new->key_refresh_interval = key_refresh_interval;
+
+ return new;
+}
+
+void
+rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey)
+{
+ struct rdns_curve_entry *entry;
+ bool success = true;
+
+ entry = malloc (sizeof (struct rdns_curve_entry));
+ if (entry != NULL) {
+ entry->name = strdup (name);
+ if (entry->name == NULL) {
+ success = false;
+ }
+ memcpy (entry->pk, pubkey, sizeof (entry->pk));
+ if (success) {
+ HASH_ADD_KEYPTR (hh, ctx->entries, entry->name, strlen (entry->name), entry);
+ }
+ }
+}
+
+#define rdns_curve_write_hex(in, out, offset, base) do { \
+ *(out) |= ((in)[(offset)] - (base)) << ((1 - offset) * 4); \
+} while (0)
+
+static bool
+rdns_curve_hex_to_byte (const char *in, unsigned char *out)
+{
+ int i;
+
+ for (i = 0; i <= 1; i ++) {
+ if (in[i] >= '0' && in[i] <= '9') {
+ rdns_curve_write_hex (in, out, i, '0');
+ }
+ else if (in[i] >= 'a' && in[i] <= 'f') {
+ rdns_curve_write_hex (in, out, i, 'a' - 10);
+ }
+ else if (in[i] >= 'A' && in[i] <= 'F') {
+ rdns_curve_write_hex (in, out, i, 'A' - 10);
+ }
+ else {
+ return false;
+ }
+ }
+ return true;
+}
+
+#undef rdns_curve_write_hex
+
+unsigned char *
+rdns_curve_key_from_hex (const char *hex)
+{
+ unsigned int len = strlen (hex), i;
+ unsigned char *res = NULL;
+
+ if (len == crypto_box_PUBLICKEYBYTES * 2) {
+ res = calloc (1, crypto_box_PUBLICKEYBYTES);
+ for (i = 0; i < crypto_box_PUBLICKEYBYTES; i ++) {
+ if (!rdns_curve_hex_to_byte (&hex[i * 2], &res[i])) {
+ free (res);
+ return NULL;
+ }
+ }
+ }
+
+ return res;
+}
+
+void
+rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_entry *entry, *tmp;
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ free (entry->name);
+ free (entry);
+ }
+
+ free (ctx);
+}
+
+static void
+rdns_curve_refresh_key_callback (void *user_data)
+{
+ struct rdns_curve_ctx *ctx = user_data;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ rdns_info ("refresh dnscurve keys");
+ REF_RELEASE (ctx->cur_key);
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+}
+
+void
+rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx)
+{
+ struct rdns_plugin *plugin;
+
+ if (!resolver->async_binded) {
+ return;
+ }
+
+ plugin = calloc (1, sizeof (struct rdns_plugin));
+ if (plugin != NULL) {
+ plugin->data = ctx;
+ plugin->type = RDNS_PLUGIN_CURVE;
+ plugin->cb.curve_plugin.send_cb = rdns_curve_send;
+ plugin->cb.curve_plugin.recv_cb = rdns_curve_recv;
+ plugin->cb.curve_plugin.finish_cb = rdns_curve_finish_request;
+ plugin->dtor = rdns_curve_dtor;
+ sodium_init ();
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+
+ if (ctx->key_refresh_interval > 0) {
+ ctx->key_refresh_event = resolver->async->add_periodic (
+ resolver->async->data, ctx->key_refresh_interval,
+ rdns_curve_refresh_key_callback, ctx);
+ }
+ ctx->resolver = resolver;
+ rdns_resolver_register_plugin (resolver, plugin);
+ }
+}
+
+ssize_t
+rdns_curve_send (struct rdns_request *req, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_entry *entry;
+ struct iovec iov[4];
+ unsigned char *m;
+ static const char qmagic[] = "Q6fnvWj8";
+ struct rdns_curve_request *creq;
+ struct rdns_curve_nm_entry *nm;
+ ssize_t ret, boxed_len;
+
+ /* Check for key */
+ HASH_FIND_STR (ctx->entries, req->io->srv->name, entry);
+ if (entry != NULL) {
+ nm = rdns_curve_find_nm (ctx->cur_key, entry);
+ creq = malloc (sizeof (struct rdns_curve_request));
+ if (creq == NULL) {
+ return -1;
+ }
+
+ boxed_len = req->pos + crypto_box_ZEROBYTES;
+ m = malloc (boxed_len);
+ if (m == NULL) {
+ return -1;
+ }
+
+ /* Ottery is faster than sodium native PRG that uses /dev/random only */
+ memcpy (creq->nonce, &ctx->cur_key->counter, sizeof (uint64_t));
+ ottery_rand_bytes (creq->nonce + sizeof (uint64_t), 12 - sizeof (uint64_t));
+ sodium_memzero (creq->nonce + 12, crypto_box_NONCEBYTES - 12);
+
+ sodium_memzero (m, crypto_box_ZEROBYTES);
+ memcpy (m + crypto_box_ZEROBYTES, req->packet, req->pos);
+
+ if (crypto_box_afternm (m, m, boxed_len,
+ creq->nonce, nm->k) == -1) {
+ sodium_memzero (m, boxed_len);
+ free (m);
+ return -1;
+ }
+
+ creq->key = ctx->cur_key;
+ REF_RETAIN (ctx->cur_key);
+ creq->entry = entry;
+ creq->req = req;
+ creq->nm = nm;
+ HASH_ADD_KEYPTR (hh, ctx->requests, creq->nonce, 12, creq);
+ req->curve_plugin_data = creq;
+
+ ctx->cur_key->counter ++;
+ ctx->cur_key->uses ++;
+
+ /* Now form a dnscurve packet */
+ iov[0].iov_base = (void *)qmagic;
+ iov[0].iov_len = sizeof (qmagic) - 1;
+ iov[1].iov_base = ctx->cur_key->pk;
+ iov[1].iov_len = sizeof (ctx->cur_key->pk);
+ iov[2].iov_base = creq->nonce;
+ iov[2].iov_len = 12;
+ iov[3].iov_base = m + crypto_box_BOXZEROBYTES;
+ iov[3].iov_len = boxed_len - crypto_box_BOXZEROBYTES;
+
+ ret = writev (req->io->sock, iov, sizeof (iov) / sizeof (iov[0]));
+ sodium_memzero (m, boxed_len);
+ free (m);
+ }
+ else {
+ ret = write (req->io->sock, req->packet, req->pos);
+ req->curve_plugin_data = NULL;
+ }
+
+ return ret;
+}
+
+ssize_t
+rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len, void *plugin_data,
+ struct rdns_request **req_out)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ ssize_t ret, boxlen;
+ static const char rmagic[] = "R6fnvWJ8";
+ unsigned char *p, *box;
+ unsigned char enonce[crypto_box_NONCEBYTES];
+ struct rdns_curve_request *creq;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ ret = read (ioc->sock, buf, len);
+
+ if (ret <= 0 || ret < 64) {
+ /* Definitely not a DNSCurve packet */
+ return ret;
+ }
+
+ if (memcmp (buf, rmagic, sizeof (rmagic) - 1) == 0) {
+ /* Likely DNSCurve packet */
+ p = ((unsigned char *)buf) + 8;
+ HASH_FIND (hh, ctx->requests, p, 12, creq);
+ if (creq == NULL) {
+ rdns_info ("unable to find nonce in the internal hash");
+ return ret;
+ }
+ memcpy (enonce, p, crypto_box_NONCEBYTES);
+ p += crypto_box_NONCEBYTES;
+ boxlen = ret - crypto_box_NONCEBYTES +
+ crypto_box_BOXZEROBYTES -
+ sizeof (rmagic) + 1;
+ if (boxlen < 0) {
+ return ret;
+ }
+ box = malloc (boxlen);
+ sodium_memzero (box, crypto_box_BOXZEROBYTES);
+ memcpy (box + crypto_box_BOXZEROBYTES, p,
+ boxlen - crypto_box_BOXZEROBYTES);
+
+ if (crypto_box_open_afternm (box, box, boxlen, enonce, creq->nm->k) != -1) {
+ memcpy (buf, box + crypto_box_ZEROBYTES,
+ boxlen - crypto_box_ZEROBYTES);
+ ret = boxlen - crypto_box_ZEROBYTES;
+ *req_out = creq->req;
+ }
+ else {
+ rdns_info ("unable open cryptobox of size %d", (int)boxlen);
+ }
+ free (box);
+ }
+
+ return ret;
+}
+
+void
+rdns_curve_finish_request (struct rdns_request *req, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_request *creq = req->curve_plugin_data;
+
+ if (creq != NULL) {
+ REF_RELEASE (creq->key);
+ HASH_DELETE (hh, ctx->requests, creq);
+ }
+}
+
+void
+rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+
+ if (ctx->key_refresh_event != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ ctx->key_refresh_event);
+ }
+ REF_RELEASE (ctx->cur_key);
+}
+#elif defined(USE_RSPAMD_CRYPTOBOX)
+
+#include "cryptobox.h"
+
+
+#ifndef crypto_box_ZEROBYTES
+#define crypto_box_ZEROBYTES 32
+#endif
+#ifndef crypto_box_BOXZEROBYTES
+#define crypto_box_BOXZEROBYTES 16
+#endif
+
+ssize_t rdns_curve_send (struct rdns_request *req, void *plugin_data,
+ struct sockaddr *saddr, socklen_t slen);
+ssize_t rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len,
+ void *plugin_data, struct rdns_request **req_out,
+ struct sockaddr *saddr, socklen_t slen);
+void rdns_curve_finish_request (struct rdns_request *req, void *plugin_data);
+void rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data);
+
+struct rdns_curve_entry {
+ char *name;
+ rspamd_pk_t pk;
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_nm_entry {
+ rspamd_nm_t k;
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *prev, *next;
+};
+
+struct rdns_curve_client_key {
+ rspamd_pk_t pk;
+ rspamd_sk_t sk;
+ struct rdns_curve_nm_entry *nms;
+ uint64_t counter;
+ unsigned int uses;
+ ref_entry_t ref;
+};
+
+struct rdns_curve_request {
+ struct rdns_request *req;
+ struct rdns_curve_client_key *key;
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *nm;
+ rspamd_nonce_t nonce;
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_ctx {
+ struct rdns_curve_entry *entries;
+ struct rdns_curve_client_key *cur_key;
+ struct rdns_curve_request *requests;
+ double key_refresh_interval;
+ void *key_refresh_event;
+ struct rdns_resolver *resolver;
+};
+
+static struct rdns_curve_client_key *
+rdns_curve_client_key_new (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_client_key *new;
+ struct rdns_curve_nm_entry *nm;
+ struct rdns_curve_entry *entry, *tmp;
+
+ new = calloc (1, sizeof (struct rdns_curve_client_key));
+ rspamd_cryptobox_keypair (new->pk, new->sk, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ nm = calloc (1, sizeof (struct rdns_curve_nm_entry));
+ nm->entry = entry;
+ rspamd_cryptobox_nm (nm->k, entry->pk, new->sk,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+
+ DL_APPEND (new->nms, nm);
+ }
+
+ new->counter = ottery_rand_uint64 ();
+
+ return new;
+}
+
+static struct rdns_curve_nm_entry *
+rdns_curve_find_nm (struct rdns_curve_client_key *key, struct rdns_curve_entry *entry)
+{
+ struct rdns_curve_nm_entry *nm;
+
+ DL_FOREACH (key->nms, nm) {
+ if (nm->entry == entry) {
+ return nm;
+ }
+ }
+
+ return NULL;
+}
+
+static void
+rdns_curve_client_key_free (struct rdns_curve_client_key *key)
+{
+ struct rdns_curve_nm_entry *nm, *tmp;
+
+ DL_FOREACH_SAFE (key->nms, nm, tmp) {
+ rspamd_explicit_memzero (nm->k, sizeof (nm->k));
+ free (nm);
+ }
+
+ rspamd_explicit_memzero (key->sk, sizeof (key->sk));
+ free (key);
+}
+
+struct rdns_curve_ctx*
+rdns_curve_ctx_new (double key_refresh_interval)
+{
+ struct rdns_curve_ctx *new;
+
+ new = calloc (1, sizeof (struct rdns_curve_ctx));
+ new->key_refresh_interval = key_refresh_interval;
+
+ return new;
+}
+
+void
+rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey)
+{
+ struct rdns_curve_entry *entry;
+ bool success = true;
+
+ entry = malloc (sizeof (struct rdns_curve_entry));
+ if (entry != NULL) {
+ entry->name = strdup (name);
+ if (entry->name == NULL) {
+ success = false;
+ }
+ memcpy (entry->pk, pubkey, sizeof (entry->pk));
+ if (success) {
+ HASH_ADD_KEYPTR (hh, ctx->entries, entry->name, strlen (entry->name), entry);
+ }
+ }
+}
+
+#define rdns_curve_write_hex(in, out, offset, base) do { \
+ *(out) |= ((in)[(offset)] - (base)) << ((1 - offset) * 4); \
+} while (0)
+
+static bool
+rdns_curve_hex_to_byte (const char *in, unsigned char *out)
+{
+ int i;
+
+ for (i = 0; i <= 1; i ++) {
+ if (in[i] >= '0' && in[i] <= '9') {
+ rdns_curve_write_hex (in, out, i, '0');
+ }
+ else if (in[i] >= 'a' && in[i] <= 'f') {
+ rdns_curve_write_hex (in, out, i, 'a' - 10);
+ }
+ else if (in[i] >= 'A' && in[i] <= 'F') {
+ rdns_curve_write_hex (in, out, i, 'A' - 10);
+ }
+ else {
+ return false;
+ }
+ }
+ return true;
+}
+
+#undef rdns_curve_write_hex
+
+unsigned char *
+rdns_curve_key_from_hex (const char *hex)
+{
+ unsigned int len = strlen (hex), i;
+ unsigned char *res = NULL;
+
+ if (len == rspamd_cryptobox_pk_bytes (RSPAMD_CRYPTOBOX_MODE_25519) * 2) {
+ res = calloc (1, rspamd_cryptobox_pk_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
+ for (i = 0;
+ i < rspamd_cryptobox_pk_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+ i ++) {
+ if (!rdns_curve_hex_to_byte (&hex[i * 2], &res[i])) {
+ free (res);
+ return NULL;
+ }
+ }
+ }
+
+ return res;
+}
+
+void
+rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_entry *entry, *tmp;
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ free (entry->name);
+ free (entry);
+ }
+
+ free (ctx);
+}
+
+static void
+rdns_curve_refresh_key_callback (void *user_data)
+{
+ struct rdns_curve_ctx *ctx = user_data;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ rdns_info ("refresh dnscurve keys");
+ REF_RELEASE (ctx->cur_key);
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+}
+
+void
+rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx)
+{
+ struct rdns_plugin *plugin;
+
+ if (!resolver->async_binded) {
+ return;
+ }
+
+ plugin = calloc (1, sizeof (struct rdns_plugin));
+ if (plugin != NULL) {
+ plugin->data = ctx;
+ plugin->type = RDNS_PLUGIN_CURVE;
+ plugin->cb.curve_plugin.send_cb = rdns_curve_send;
+ plugin->cb.curve_plugin.recv_cb = rdns_curve_recv;
+ plugin->cb.curve_plugin.finish_cb = rdns_curve_finish_request;
+ plugin->dtor = rdns_curve_dtor;
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+
+ if (ctx->key_refresh_interval > 0) {
+ ctx->key_refresh_event = resolver->async->add_periodic (
+ resolver->async->data, ctx->key_refresh_interval,
+ rdns_curve_refresh_key_callback, ctx);
+ }
+ ctx->resolver = resolver;
+ rdns_resolver_register_plugin (resolver, plugin);
+ }
+}
+
+ssize_t
+rdns_curve_send (struct rdns_request *req, void *plugin_data,
+ struct sockaddr *saddr, socklen_t slen)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_entry *entry;
+ struct iovec iov[4];
+ unsigned char *m;
+ static const char qmagic[] = "Q6fnvWj8";
+ struct rdns_curve_request *creq;
+ struct rdns_curve_nm_entry *nm;
+ ssize_t ret, boxed_len;
+
+ /* Check for key */
+ HASH_FIND_STR (ctx->entries, req->io->srv->name, entry);
+ if (entry != NULL) {
+ nm = rdns_curve_find_nm (ctx->cur_key, entry);
+ creq = malloc (sizeof (struct rdns_curve_request));
+ if (creq == NULL) {
+ return -1;
+ }
+
+ boxed_len = req->pos + crypto_box_ZEROBYTES;
+ m = malloc (boxed_len);
+ if (m == NULL) {
+ free(creq);
+ return -1;
+ }
+
+ /* Ottery is faster than sodium native PRG that uses /dev/random only */
+ memcpy (creq->nonce, &ctx->cur_key->counter, sizeof (uint64_t));
+ ottery_rand_bytes (creq->nonce + sizeof (uint64_t), 12 - sizeof (uint64_t));
+ rspamd_explicit_memzero (creq->nonce + 12,
+ rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519) - 12);
+
+ rspamd_explicit_memzero (m, crypto_box_ZEROBYTES);
+ memcpy (m + crypto_box_ZEROBYTES, req->packet, req->pos);
+
+ rspamd_cryptobox_encrypt_nm_inplace (m + crypto_box_ZEROBYTES,
+ boxed_len,
+ creq->nonce,
+ nm->k,
+ m,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+
+ creq->key = ctx->cur_key;
+ REF_RETAIN (ctx->cur_key);
+ creq->entry = entry;
+ creq->req = req;
+ creq->nm = nm;
+ HASH_ADD_KEYPTR (hh, ctx->requests, creq->nonce, 12, creq);
+ req->curve_plugin_data = creq;
+
+ ctx->cur_key->counter ++;
+ ctx->cur_key->uses ++;
+
+ /* Now form a dnscurve packet */
+ iov[0].iov_base = (void *)qmagic;
+ iov[0].iov_len = sizeof (qmagic) - 1;
+ iov[1].iov_base = ctx->cur_key->pk;
+ iov[1].iov_len = sizeof (ctx->cur_key->pk);
+ iov[2].iov_base = creq->nonce;
+ iov[2].iov_len = 12;
+ iov[3].iov_base = m + crypto_box_BOXZEROBYTES;
+ iov[3].iov_len = boxed_len - crypto_box_BOXZEROBYTES;
+
+ struct msghdr msg;
+
+ memset (&msg, 0, sizeof (msg));
+ msg.msg_namelen = slen;
+ msg.msg_name = saddr;
+ msg.msg_iov = iov;
+ msg.msg_iovlen = sizeof (iov) / sizeof (iov[0]);
+ ret = sendmsg (req->io->sock, &msg, 0);
+ rspamd_explicit_memzero (m, boxed_len);
+ free (m);
+ }
+ else {
+ ret = sendto (req->io->sock, req->packet, req->pos, 0, saddr, slen);
+ req->curve_plugin_data = NULL;
+ }
+
+ return ret;
+}
+
+ssize_t
+rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len, void *plugin_data,
+ struct rdns_request **req_out, struct sockaddr *saddr, socklen_t slen)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ ssize_t ret, boxlen;
+ static const char rmagic[] = "R6fnvWJ8";
+ unsigned char *p, *box;
+ unsigned char enonce[24];
+ struct rdns_curve_request *creq;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ ret = recv (ioc->sock, buf, len, 0);
+
+ if (ret <= 0 || ret < 64) {
+ /* Definitely not a DNSCurve packet */
+ return ret;
+ }
+
+ if (memcmp (buf, rmagic, sizeof (rmagic) - 1) == 0) {
+ /* Likely DNSCurve packet */
+ p = ((unsigned char *)buf) + 8;
+ HASH_FIND (hh, ctx->requests, p, 12, creq);
+ if (creq == NULL) {
+ rdns_info ("unable to find nonce in the internal hash");
+ return ret;
+ }
+
+ memcpy (enonce, p, rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
+ p += rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+ boxlen = ret - rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519) +
+ crypto_box_BOXZEROBYTES -
+ sizeof (rmagic) + 1;
+ if (boxlen < 0) {
+ return ret;
+ }
+
+ box = malloc (boxlen);
+ rspamd_explicit_memzero (box, crypto_box_BOXZEROBYTES);
+ memcpy (box + crypto_box_BOXZEROBYTES, p,
+ boxlen - crypto_box_BOXZEROBYTES);
+
+ if (!rspamd_cryptobox_decrypt_nm_inplace (
+ box + rspamd_cryptobox_mac_bytes (RSPAMD_CRYPTOBOX_MODE_25519),
+ boxlen - rspamd_cryptobox_mac_bytes (RSPAMD_CRYPTOBOX_MODE_25519),
+ enonce, creq->nm->k, box, RSPAMD_CRYPTOBOX_MODE_25519)) {
+ memcpy (buf, box + crypto_box_ZEROBYTES,
+ boxlen - crypto_box_ZEROBYTES);
+ ret = boxlen - crypto_box_ZEROBYTES;
+ *req_out = creq->req;
+ }
+ else {
+ rdns_info ("unable open cryptobox of size %d", (int)boxlen);
+ }
+
+ free (box);
+ }
+
+ return ret;
+}
+
+void
+rdns_curve_finish_request (struct rdns_request *req, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_request *creq = req->curve_plugin_data;
+
+ if (creq != NULL) {
+ REF_RELEASE (creq->key);
+ HASH_DELETE (hh, ctx->requests, creq);
+ }
+}
+
+void
+rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+
+ if (ctx->key_refresh_event != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ ctx->key_refresh_event);
+ }
+ REF_RELEASE (ctx->cur_key);
+}
+#else
+
+/* Fake functions */
+struct rdns_curve_ctx* rdns_curve_ctx_new (double rekey_interval)
+{
+ return NULL;
+}
+void rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey)
+{
+
+}
+void rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx)
+{
+
+}
+void rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx)
+{
+
+}
+
+unsigned char *
+rdns_curve_key_from_hex (const char *hex)
+{
+ return NULL;
+}
+#endif
diff --git a/contrib/librdns/dns_private.h b/contrib/librdns/dns_private.h
new file mode 100644
index 0000000..c240deb
--- /dev/null
+++ b/contrib/librdns/dns_private.h
@@ -0,0 +1,338 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DNS_PRIVATE_H_
+#define DNS_PRIVATE_H_
+
+#include "config.h"
+#include "uthash.h"
+#include "utlist.h"
+#include "khash.h"
+#include "rdns.h"
+#include "upstream.h"
+#include "ref.h"
+
+static const int dns_port = 53;
+static const int default_io_cnt = 8;
+static const int default_tcp_io_cnt = 1;
+
+#define UDP_PACKET_SIZE (4096)
+
+#define DNS_COMPRESSION_BITS 0xC0
+
+#define DNS_D_MAXLABEL 63 /* + 1 '\0' */
+#define DNS_D_MAXNAME 255 /* + 1 '\0' */
+
+#define RESOLV_CONF "/etc/resolv.conf"
+
+struct dns_header {
+ unsigned int qid :16;
+
+#if BYTE_ORDER == BIG_ENDIAN
+ unsigned int qr:1;
+ unsigned int opcode:4;
+ unsigned int aa:1;
+ unsigned int tc:1;
+ unsigned int rd:1;
+
+ unsigned int ra:1;
+ unsigned int cd : 1;
+ unsigned int ad : 1;
+ unsigned int z : 1;
+ unsigned int rcode:4;
+#else
+ unsigned int rd :1;
+ unsigned int tc :1;
+ unsigned int aa :1;
+ unsigned int opcode :4;
+ unsigned int qr :1;
+
+ unsigned int rcode :4;
+ unsigned int z : 1;
+ unsigned int ad : 1;
+ unsigned int cd : 1;
+ unsigned int ra :1;
+#endif
+
+ unsigned int qdcount :16;
+ unsigned int ancount :16;
+ unsigned int nscount :16;
+ unsigned int arcount :16;
+};
+
+/**
+ * Represents DNS server
+ */
+struct rdns_server {
+ char *name;
+ unsigned int port;
+ unsigned int io_cnt;
+ unsigned int tcp_io_cnt;
+
+ struct rdns_io_channel **io_channels;
+ struct rdns_io_channel **tcp_io_channels;
+ void *ups_elt;
+ upstream_entry_t up;
+};
+
+enum rdns_request_state {
+ RDNS_REQUEST_NEW = 0,
+ RDNS_REQUEST_REGISTERED = 1,
+ RDNS_REQUEST_WAIT_SEND,
+ RDNS_REQUEST_WAIT_REPLY,
+ RDNS_REQUEST_REPLIED,
+ RDNS_REQUEST_FAKE,
+ RDNS_REQUEST_ERROR,
+ RDNS_REQUEST_TCP,
+};
+
+struct rdns_request {
+ struct rdns_resolver *resolver;
+ struct rdns_async_context *async;
+ struct rdns_io_channel *io;
+ struct rdns_reply *reply;
+ enum rdns_request_type type;
+
+ double timeout;
+ unsigned int retransmits;
+
+ int id;
+ struct rdns_request_name *requested_names;
+ unsigned int qcount;
+ enum rdns_request_state state;
+
+ uint8_t *packet;
+ off_t pos;
+ unsigned int packet_len;
+
+ dns_callback_type func;
+ void *arg;
+
+ void *async_event;
+
+#if defined(TWEETNACL) || defined(USE_RSPAMD_CRYPTOBOX)
+ void *curve_plugin_data;
+#endif
+
+ ref_entry_t ref;
+};
+
+
+enum rdns_io_channel_flags {
+ RDNS_CHANNEL_CONNECTED = 1u << 0u,
+ RDNS_CHANNEL_ACTIVE = 1u << 1u,
+ RDNS_CHANNEL_TCP = 1u << 2u,
+ RDNS_CHANNEL_TCP_CONNECTING = 1u << 3u,
+};
+
+#define IS_CHANNEL_CONNECTED(ioc) (((ioc)->flags & RDNS_CHANNEL_CONNECTED) != 0)
+#define IS_CHANNEL_ACTIVE(ioc) (((ioc)->flags & RDNS_CHANNEL_ACTIVE) != 0)
+#define IS_CHANNEL_TCP(ioc) (((ioc)->flags & RDNS_CHANNEL_TCP) != 0)
+
+/**
+ * Used to chain output DNS requests for a TCP connection
+ */
+struct rdns_tcp_output_chain {
+ uint16_t next_write_size; /* Network byte order! */
+ uint16_t cur_write; /* Cur bytes written including `next_write_size` */
+ unsigned char *write_buf;
+ struct rdns_tcp_output_chain *prev, *next;
+};
+
+/**
+ * Specific stuff for a TCP IO chain
+ */
+struct rdns_tcp_channel {
+ uint16_t next_read_size; /* Network byte order on read, then host byte order */
+ uint16_t cur_read; /* Cur bytes read including `next_read_size` */
+ unsigned char *cur_read_buf;
+ unsigned read_buf_allocated;
+
+ /* Chained set of the planned writes */
+ struct rdns_tcp_output_chain *output_chain;
+ unsigned cur_output_chains;
+
+ void *async_read; /** async read event */
+ void *async_write; /** async write event */
+};
+
+KHASH_DECLARE(rdns_requests_hash, int, struct rdns_request *);
+#define RDNS_IO_CHANNEL_TAG UINT64_C(0xe190a5ba12f094c8)
+/**
+ * IO channel for a specific DNS server
+ */
+struct rdns_io_channel {
+ uint64_t struct_magic; /**< tag for this structure */
+ struct rdns_server *srv;
+ struct rdns_resolver *resolver;
+ struct sockaddr *saddr;
+ socklen_t slen;
+ int sock; /**< persistent socket */
+ int flags; /**< see enum rdns_io_channel_flags */
+ void *async_io; /** async opaque ptr */
+ khash_t(rdns_requests_hash) *requests;
+ /*
+ * For DNS replies parsing we use per-channel structure
+ * which is used for two purposes:
+ * 1) We read the next DNS header
+ * 2) We find the corresponding request (if any)
+ * 3) We read the remaining packet (associated with a request or dangling)
+ * This structure is filled on each read-readiness for an IO channel
+ */
+ struct rdns_tcp_channel *tcp;
+ uint64_t uses;
+ ref_entry_t ref;
+};
+
+struct rdns_fake_reply_idx {
+ enum rdns_request_type type;
+ unsigned len;
+ char request[0];
+};
+
+struct rdns_fake_reply {
+ enum dns_rcode rcode;
+ struct rdns_reply_entry *result;
+ UT_hash_handle hh;
+ struct rdns_fake_reply_idx key;
+};
+
+
+struct rdns_resolver {
+ struct rdns_server *servers;
+ struct rdns_async_context *async; /** async callbacks */
+ void *periodic; /** periodic event for resolver */
+ struct rdns_upstream_context *ups;
+ struct rdns_plugin *curve_plugin;
+ struct rdns_fake_reply *fake_elts;
+
+#ifdef __GNUC__
+ __attribute__((format(printf, 4, 0)))
+#endif
+ rdns_log_function logger;
+ void *log_data;
+ enum rdns_log_level log_level;
+
+ uint64_t max_ioc_uses;
+ void *refresh_ioc_periodic;
+
+ bool async_binded;
+ bool initialized;
+ bool enable_dnssec;
+ int flags;
+ ref_entry_t ref;
+};
+
+struct dns_query;
+
+/* Internal DNS structs */
+
+enum dns_section {
+ DNS_S_QD = 0x01,
+#define DNS_S_QUESTION DNS_S_QD
+
+ DNS_S_AN = 0x02,
+#define DNS_S_ANSWER DNS_S_AN
+
+ DNS_S_NS = 0x04,
+#define DNS_S_AUTHORITY DNS_S_NS
+
+ DNS_S_AR = 0x08,
+#define DNS_S_ADDITIONAL DNS_S_AR
+
+ DNS_S_ALL = 0x0f
+};
+/* enum dns_section */
+
+enum dns_opcode {
+ DNS_OP_QUERY = 0,
+ DNS_OP_IQUERY = 1,
+ DNS_OP_STATUS = 2,
+ DNS_OP_NOTIFY = 4,
+ DNS_OP_UPDATE = 5,
+};
+/* dns_opcode */
+
+enum dns_class {
+ DNS_C_IN = 1,
+
+ DNS_C_ANY = 255
+};
+/* enum dns_class */
+
+struct dns_query {
+ char *qname;
+ unsigned int qtype :16;
+ unsigned int qclass :16;
+};
+
+enum dns_type {
+ DNS_T_A = RDNS_REQUEST_A,
+ DNS_T_NS = RDNS_REQUEST_NS,
+ DNS_T_CNAME = 5,
+ DNS_T_SOA = RDNS_REQUEST_SOA,
+ DNS_T_PTR = RDNS_REQUEST_PTR,
+ DNS_T_MX = RDNS_REQUEST_MX,
+ DNS_T_TXT = RDNS_REQUEST_TXT,
+ DNS_T_AAAA = RDNS_REQUEST_AAAA,
+ DNS_T_SRV = RDNS_REQUEST_SRV,
+ DNS_T_OPT = 41,
+ DNS_T_SSHFP = 44,
+ DNS_T_TLSA = RDNS_REQUEST_TLSA,
+ DNS_T_SPF = RDNS_REQUEST_SPF,
+ DNS_T_ALL = RDNS_REQUEST_ANY
+};
+/* enum dns_type */
+
+static const char dns_rcodes[][32] = {
+ [RDNS_RC_NOERROR] = "no error",
+ [RDNS_RC_FORMERR] = "query format error",
+ [RDNS_RC_SERVFAIL] = "server fail",
+ [RDNS_RC_NXDOMAIN] = "no records with this name",
+ [RDNS_RC_NOTIMP] = "not implemented",
+ [RDNS_RC_REFUSED] = "query refused",
+ [RDNS_RC_YXDOMAIN] = "YXDOMAIN",
+ [RDNS_RC_YXRRSET] = "YXRRSET",
+ [RDNS_RC_NXRRSET] = "NXRRSET",
+ [RDNS_RC_NOTAUTH] = "not authorized",
+ [RDNS_RC_NOTZONE] = "no such zone",
+ [RDNS_RC_TIMEOUT] = "query timed out",
+ [RDNS_RC_NETERR] = "network error",
+ [RDNS_RC_NOREC] = "requested record is not found"
+};
+
+static const char dns_types[][16] = {
+ [RDNS_REQUEST_A] = "A request",
+ [RDNS_REQUEST_NS] = "NS request",
+ [RDNS_REQUEST_PTR] = "PTR request",
+ [RDNS_REQUEST_MX] = "MX request",
+ [RDNS_REQUEST_TXT] = "TXT request",
+ [RDNS_REQUEST_SRV] = "SRV request",
+ [RDNS_REQUEST_SPF] = "SPF request",
+ [RDNS_REQUEST_AAAA] = "AAAA request",
+ [RDNS_REQUEST_TLSA] = "TLSA request",
+ [RDNS_REQUEST_ANY] = "ANY request"
+};
+
+
+#endif /* DNS_PRIVATE_H_ */
diff --git a/contrib/librdns/logger.c b/contrib/librdns/logger.c
new file mode 100644
index 0000000..c9ed2d9
--- /dev/null
+++ b/contrib/librdns/logger.c
@@ -0,0 +1,53 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include "dns_private.h"
+#include "logger.h"
+
+void
+rdns_logger_internal (void *log_data, enum rdns_log_level level,
+ const char *function, const char *format,
+ va_list args)
+{
+ struct rdns_resolver *resolver = log_data;
+
+ if (level <= resolver->log_level) {
+ fprintf (stderr, "rdns: %s: ", function);
+ vfprintf (stderr, format, args);
+ fprintf (stderr, "\n");
+ }
+}
+
+void rdns_logger_helper (struct rdns_resolver *resolver,
+ enum rdns_log_level level,
+ const char *function, const char *format, ...)
+{
+ va_list va;
+
+ if (resolver->logger != NULL) {
+ va_start (va, format);
+ resolver->logger (resolver->log_data, level, function, format, va);
+ va_end (va);
+ }
+}
diff --git a/contrib/librdns/logger.h b/contrib/librdns/logger.h
new file mode 100644
index 0000000..8072876
--- /dev/null
+++ b/contrib/librdns/logger.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef LOGGER_H_
+#define LOGGER_H_
+
+#include <stdarg.h>
+#include "dns_private.h"
+
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 0)))
+#endif
+void rdns_logger_internal (void *log_data, enum rdns_log_level level,
+ const char *function, const char *format,
+ va_list args);
+
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 5)))
+#endif
+void rdns_logger_helper (struct rdns_resolver *resolver,
+ enum rdns_log_level level,
+ const char *function, const char *format, ...);
+
+#define rdns_err(...) do { rdns_logger_helper (resolver, RDNS_LOG_ERROR, __func__, __VA_ARGS__); } while (0)
+#define rdns_warn(...) do { rdns_logger_helper (resolver, RDNS_LOG_WARNING, __func__, __VA_ARGS__); } while (0)
+#define rdns_info(...) do { rdns_logger_helper (resolver, RDNS_LOG_INFO, __func__, __VA_ARGS__); } while (0)
+#define rdns_debug(...) do { rdns_logger_helper (resolver, RDNS_LOG_DEBUG, __func__, __VA_ARGS__); } while (0)
+
+#endif /* LOGGER_H_ */
diff --git a/contrib/librdns/packet.c b/contrib/librdns/packet.c
new file mode 100644
index 0000000..59a919e
--- /dev/null
+++ b/contrib/librdns/packet.c
@@ -0,0 +1,288 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "punycode.h"
+#include "packet.h"
+#include "util.h"
+#include "logger.h"
+#include "compression.h"
+
+void
+rdns_allocate_packet (struct rdns_request* req, unsigned int namelen)
+{
+ namelen += 96 + 2 + 4 + 11; /* EDNS0 RR */
+ req->packet = malloc (namelen);
+ req->pos = 0;
+ req->packet_len = namelen;
+}
+
+
+void
+rdns_make_dns_header (struct rdns_request *req, unsigned int qcount)
+{
+ struct dns_header *header;
+
+ /* Set DNS header values */
+ header = (struct dns_header *)req->packet;
+ memset (header, 0 , sizeof (struct dns_header));
+ header->qid = rdns_permutor_generate_id ();
+ header->rd = 1;
+ header->qdcount = htons (qcount);
+ header->arcount = htons (1);
+ req->pos += sizeof (struct dns_header);
+ req->id = header->qid;
+}
+
+static bool
+rdns_maybe_punycode_label (const uint8_t *begin,
+ uint8_t const **dot, size_t *label_len)
+{
+ bool ret = false;
+ const uint8_t *p = begin;
+
+ *dot = NULL;
+
+ while (*p) {
+ if (*p == '.') {
+ *dot = p;
+ break;
+ }
+ else if ((*p) & 0x80) {
+ ret = true;
+ }
+ p ++;
+ }
+
+ if (label_len) {
+ *label_len = p - begin;
+ }
+
+ return ret;
+}
+
+bool
+rdns_format_dns_name (struct rdns_resolver *resolver, const char *in,
+ size_t inlen,
+ char **out, size_t *outlen)
+{
+ const uint8_t *dot;
+ const uint8_t *p = in, *end = in + inlen;
+ char *o;
+ int labels = 0;
+ size_t label_len, olen, remain;
+ uint32_t *uclabel = NULL;
+ size_t punylabel_len, uclabel_len;
+ char tmp_label[DNS_D_MAXLABEL];
+ bool need_encode = false;
+
+ if (inlen == 0) {
+ inlen = strlen (in);
+ }
+
+ /* Check for non-ascii characters */
+ if (!(resolver->flags & RDNS_RESOLVER_NOIDN)) {
+ while (p != end) {
+ if (*p >= 0x80) {
+ need_encode = true;
+ }
+ else if (*p == '.') {
+ labels++;
+ }
+ p++;
+ }
+ }
+
+ if (!need_encode) {
+ *out = malloc (inlen + 1);
+
+ if (*out == NULL) {
+ return false;
+ }
+
+ o = *out;
+ memcpy (o, in, inlen);
+ o[inlen] = '\0';
+ *outlen = inlen;
+
+ return true;
+ }
+
+ /* We need to encode */
+ p = in;
+ /* We allocate 4 times more memory as we cannot guarantee encoding bounds */
+ olen = inlen * sizeof (int32_t) + 1 + sizeof ("xn--") * labels;
+ *out = malloc (olen + 1);
+
+ if (*out == NULL) {
+ return false;
+ }
+
+ o = *out;
+ remain = olen;
+
+ while (p != end) {
+ /* Check label for unicode characters */
+ if (rdns_maybe_punycode_label (p, &dot, &label_len)) {
+ /* Convert to ucs4 */
+ if (rdns_utf8_to_ucs4 (p, label_len, &uclabel, &uclabel_len) == 0) {
+ punylabel_len = DNS_D_MAXLABEL;
+
+ rdns_punycode_label_toascii (uclabel, uclabel_len,
+ tmp_label, &punylabel_len);
+ if (remain >= punylabel_len + 1) {
+ memcpy (o, tmp_label, punylabel_len);
+ o += punylabel_len;
+ *o++ = '.';
+ remain -= punylabel_len + 1;
+ }
+ else {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+
+ free (uclabel);
+ uclabel = NULL;
+
+ if (dot) {
+ p = dot + 1;
+ }
+ else {
+ break;
+ }
+ }
+ else {
+ break;
+ }
+ }
+ else {
+ if (dot) {
+ if (label_len > DNS_D_MAXLABEL) {
+ rdns_info ("dns name component is longer than 63 bytes, should be stripped");
+ label_len = DNS_D_MAXLABEL;
+ }
+ if (remain < label_len + 1) {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+ if (label_len == 0) {
+ /* Two dots in order, skip this */
+ rdns_info ("name contains two or more dots in a row, replace it with one dot");
+ p = dot + 1;
+ continue;
+ }
+ memcpy (o, p, label_len);
+ o += label_len;
+ *o++ = '.';
+ remain -= label_len + 1;
+ p = dot + 1;
+ }
+ else {
+ if (label_len == 0) {
+ /* If name is ended with dot */
+ break;
+ }
+ if (label_len > DNS_D_MAXLABEL) {
+ rdns_info ("dns name component is longer than 63 bytes, should be stripped");
+ label_len = DNS_D_MAXLABEL;
+ }
+ if (remain < label_len + 1) {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+ memcpy (o, p, label_len);
+ o += label_len;
+ *o++ = '.';
+ remain -= label_len + 1;
+ p = dot + 1;
+ break;
+ }
+ }
+ if (remain == 0) {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+ }
+
+ *o = '\0';
+
+ *outlen = o - *out;
+
+ return true;
+
+err:
+ free (*out);
+ *out = NULL;
+ free (uclabel);
+
+ return false;
+}
+
+#define U16_TO_WIRE_ADVANCE(val, p8) \
+ *p8++ = (uint8_t)(((uint16_t)(val)) >> 8); \
+ *p8++ = (uint8_t)(((uint16_t)(val)) & 0xFF);
+
+bool
+rdns_add_rr (struct rdns_request *req, const char *name, unsigned int len,
+ enum dns_type type, struct kh_rdns_compression_hash_s **comp)
+{
+ uint8_t *p8;
+
+ if (!rdns_write_name_compressed (req, name, len, comp)) {
+ return false;
+ }
+
+ p8 = (req->packet + req->pos);
+ U16_TO_WIRE_ADVANCE (type, p8);
+ U16_TO_WIRE_ADVANCE (DNS_C_IN, p8);
+ req->pos += sizeof (uint16_t) * 2;
+
+ return true;
+}
+
+bool
+rdns_add_edns0 (struct rdns_request *req)
+{
+ uint8_t *p8;
+
+ p8 = (req->packet + req->pos);
+ *p8++ = '\0'; /* Name is root */
+ U16_TO_WIRE_ADVANCE (DNS_T_OPT, p8);
+ U16_TO_WIRE_ADVANCE (UDP_PACKET_SIZE, p8);
+ U16_TO_WIRE_ADVANCE (0, p8);
+
+ if (req->resolver->enable_dnssec) {
+ *p8++ = 0x80;
+ }
+ else {
+ *p8++ = 0x00;
+ }
+ *p8++ = 0;
+ /* Length */
+ U16_TO_WIRE_ADVANCE (0, p8);
+
+ req->pos += sizeof (uint8_t) + sizeof (uint16_t) * 5;
+
+ return true;
+}
diff --git a/contrib/librdns/packet.h b/contrib/librdns/packet.h
new file mode 100644
index 0000000..44e16a5
--- /dev/null
+++ b/contrib/librdns/packet.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef PACKET_H_
+#define PACKET_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "dns_private.h"
+
+struct kh_rdns_compression_hash_s;
+
+/**
+ * Allocate dns packet suitable to handle up to `namelen` name
+ * @param req request
+ * @param namelen requested name
+ */
+void rdns_allocate_packet (struct rdns_request* req, unsigned int namelen);
+
+/**
+ * Add basic header to the dns packet
+ * @param req
+ */
+void rdns_make_dns_header (struct rdns_request *req, unsigned int qcount);
+
+
+/**
+ * Add a resource record to the DNS packet
+ * @param req request
+ * @param name requested name
+ * @param type type of resource record
+ */
+bool rdns_add_rr (struct rdns_request *req, const char *name, unsigned int len,
+ enum dns_type type, struct kh_rdns_compression_hash_s **comp);
+
+/**
+ * Add EDNS0 section
+ * @param req
+ */
+bool rdns_add_edns0 (struct rdns_request *req);
+
+#endif /* PACKET_H_ */
diff --git a/contrib/librdns/parse.c b/contrib/librdns/parse.c
new file mode 100644
index 0000000..f9025e1
--- /dev/null
+++ b/contrib/librdns/parse.c
@@ -0,0 +1,461 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "parse.h"
+#include "logger.h"
+
+static uint8_t *
+rdns_decompress_label (uint8_t *begin, uint16_t *len, uint16_t max)
+{
+ uint16_t offset = (*len);
+
+ if (offset > max) {
+ return NULL;
+ }
+ *len = *(begin + offset);
+ return begin + offset;
+}
+
+#define UNCOMPRESS_DNS_OFFSET(p) (((*(p)) ^ DNS_COMPRESSION_BITS) << 8) + *((p) + 1)
+
+uint8_t *
+rdns_request_reply_cmp (struct rdns_request *req, uint8_t *in, int len)
+{
+ uint8_t *p, *c, *l1, *l2;
+ uint16_t len1, len2;
+ int decompressed = 0;
+ struct rdns_resolver *resolver = req->resolver;
+
+ /* QR format:
+ * labels - len:octets
+ * null label - 0
+ * class - 2 octets
+ * type - 2 octets
+ */
+
+ /* In p we would store current position in reply and in c - position in request */
+ p = in;
+ c = req->packet + req->pos;
+
+ for (;;) {
+ /* Get current label */
+ len1 = *p;
+ len2 = *c;
+ if (p - in > len) {
+ rdns_info ("invalid dns reply");
+ return NULL;
+ }
+ /* This may be compressed, so we need to decompress it */
+ if (len1 & DNS_COMPRESSION_BITS) {
+ len1 = UNCOMPRESS_DNS_OFFSET(p);
+ l1 = rdns_decompress_label (in, &len1, len);
+ if (l1 == NULL) {
+ return NULL;
+ }
+ decompressed ++;
+ l1 ++;
+ p += 2;
+ }
+ else {
+ l1 = ++p;
+ p += len1;
+ }
+ if (len2 & DNS_COMPRESSION_BITS) {
+ len2 = UNCOMPRESS_DNS_OFFSET(c);
+ l2 = rdns_decompress_label (c, &len2, len);
+ if (l2 == NULL) {
+ rdns_info ("invalid DNS pointer, cannot decompress");
+ return NULL;
+ }
+ decompressed ++;
+ l2 ++;
+ c += 2;
+ }
+ else {
+ l2 = ++c;
+ c += len2;
+ }
+ if (len1 != len2) {
+ return NULL;
+ }
+ if (len1 == 0) {
+ break;
+ }
+
+ if (memcmp (l1, l2, len1) != 0) {
+ return NULL;
+ }
+ if (decompressed == 2) {
+ break;
+ }
+ }
+
+ /* p now points to the end of QR section */
+ /* Compare class and type */
+ if (memcmp (p, c, sizeof (uint16_t) * 2) == 0) {
+ req->pos = c - req->packet + sizeof (uint16_t) * 2;
+ return p + sizeof (uint16_t) * 2;
+ }
+ return NULL;
+}
+
+#define MAX_RECURSION_LEVEL 10
+
+bool
+rdns_parse_labels (struct rdns_resolver *resolver,
+ uint8_t *in, char **target, uint8_t **pos, struct rdns_reply *rep,
+ int *remain, bool make_name)
+{
+ uint16_t namelen = 0;
+ uint8_t *p = *pos, *begin = *pos, *l, *t, *end = *pos + *remain, *new_pos = *pos;
+ uint16_t llen;
+ int length = *remain, new_remain = *remain;
+ int ptrs = 0, labels = 0;
+ bool got_compression = false;
+
+ /* First go through labels and calculate name length */
+ while (p - begin < length) {
+ if (ptrs > MAX_RECURSION_LEVEL) {
+ rdns_info ("dns pointers are nested too much");
+ return false;
+ }
+ llen = *p;
+ if (llen == 0) {
+ if (!got_compression) {
+ /* In case of compression we have already decremented the processing position */
+ new_remain -= sizeof (uint8_t);
+ new_pos += sizeof (uint8_t);
+ }
+ break;
+ }
+ else if ((llen & DNS_COMPRESSION_BITS)) {
+ if (end - p > 1) {
+ ptrs ++;
+ llen = UNCOMPRESS_DNS_OFFSET(p);
+ l = rdns_decompress_label (in, &llen, end - in);
+ if (l == NULL) {
+ rdns_info ("invalid DNS pointer");
+ return false;
+ }
+ if (!got_compression) {
+ /* Our label processing is finished actually */
+ new_remain -= sizeof (uint16_t);
+ new_pos += sizeof (uint16_t);
+ got_compression = true;
+ }
+ if (l < in || l > begin + length) {
+ rdns_info ("invalid pointer in DNS packet");
+ return false;
+ }
+ begin = l;
+ length = end - begin;
+ p = l + *l + 1;
+ namelen += *l;
+ labels ++;
+ }
+ else {
+ rdns_info ("DNS packet has incomplete compressed label, input length: %d bytes, remain: %d",
+ *remain, new_remain);
+ return false;
+ }
+ }
+ else {
+ namelen += llen;
+ p += llen + 1;
+ labels ++;
+ if (!got_compression) {
+ new_remain -= llen + 1;
+ new_pos += llen + 1;
+ }
+ }
+ }
+
+ if (!make_name) {
+ goto end;
+ }
+ *target = malloc (namelen + labels + 3);
+ t = (uint8_t *)*target;
+ p = *pos;
+ begin = *pos;
+ length = *remain;
+ /* Now copy labels to name */
+ while (p - begin < length) {
+ llen = *p;
+ if (llen == 0) {
+ break;
+ }
+ else if (llen & DNS_COMPRESSION_BITS) {
+ llen = UNCOMPRESS_DNS_OFFSET(p);
+ l = rdns_decompress_label (in, &llen, end - in);
+
+ if (l == NULL) {
+ goto end;
+ }
+
+ begin = l;
+ length = end - begin;
+ p = l + *l + 1;
+ memcpy (t, l + 1, *l);
+ t += *l;
+ *t ++ = '.';
+ }
+ else {
+ memcpy (t, p + 1, *p);
+ t += *p;
+ *t ++ = '.';
+ p += *p + 1;
+ }
+ }
+ if (t > (uint8_t *)*target) {
+ *(t - 1) = '\0';
+ }
+ else {
+ /* Handle empty labels */
+ **target = '\0';
+ }
+end:
+ *remain = new_remain;
+ *pos = new_pos;
+
+ return true;
+}
+
+#define GET8(x) do {(x) = ((*p)); p += sizeof (uint8_t); *remain -= sizeof (uint8_t); } while(0)
+#define GET16(x) do {(x) = ((*p) << 8) + *(p + 1); p += sizeof (uint16_t); *remain -= sizeof (uint16_t); } while(0)
+#define GET32(x) do {(x) = ((*p) << 24) + ((*(p + 1)) << 16) + ((*(p + 2)) << 8) + *(p + 3); p += sizeof (uint32_t); *remain -= sizeof (uint32_t); } while(0)
+#define SKIP(type) do { p += sizeof(type); *remain -= sizeof(type); } while (0)
+
+int
+rdns_parse_rr (struct rdns_resolver *resolver,
+ uint8_t *in, struct rdns_reply_entry *elt, uint8_t **pos,
+ struct rdns_reply *rep, int *remain)
+{
+ uint8_t *p = *pos, parts;
+ uint16_t type, datalen, txtlen, copied;
+ int32_t ttl;
+ bool parsed = false;
+
+ /* Skip the whole name */
+ if (!rdns_parse_labels (resolver, in, NULL, &p, rep, remain, false)) {
+ rdns_info ("bad RR name");
+ return -1;
+ }
+ if (*remain < (int)sizeof (uint16_t) * 6) {
+ rdns_info ("stripped dns reply: %d bytes remain; domain %s", *remain,
+ rep->requested_name);
+ return -1;
+ }
+ GET16 (type);
+ /* Skip class */
+ SKIP (uint16_t);
+ GET32 (ttl);
+ GET16 (datalen);
+ elt->type = type;
+ /* Now p points to RR data */
+ switch (type) {
+ case DNS_T_A:
+ if (!(datalen & 0x3) && datalen <= *remain) {
+ memcpy (&elt->content.a.addr, p, sizeof (struct in_addr));
+ p += datalen;
+ *remain -= datalen;
+ parsed = true;
+ }
+ else {
+ rdns_info ("corrupted A record; domain: %s", rep->requested_name);
+ return -1;
+ }
+ break;
+ case DNS_T_AAAA:
+ if (datalen == sizeof (struct in6_addr) && datalen <= *remain) {
+ memcpy (&elt->content.aaa.addr, p, sizeof (struct in6_addr));
+ p += datalen;
+ *remain -= datalen;
+ parsed = true;
+ }
+ else {
+ rdns_info ("corrupted AAAA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ break;
+ case DNS_T_PTR:
+ if (! rdns_parse_labels (resolver, in, &elt->content.ptr.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in PTR record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_NS:
+ if (! rdns_parse_labels (resolver, in, &elt->content.ns.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in NS record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_SOA:
+ if (! rdns_parse_labels (resolver, in, &elt->content.soa.mname, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in SOA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ if (! rdns_parse_labels (resolver, in, &elt->content.soa.admin, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in SOA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ if (*remain >= sizeof(int32_t) * 5) {
+ GET32 (elt->content.soa.serial);
+ GET32 (elt->content.soa.refresh);
+ GET32 (elt->content.soa.retry);
+ GET32 (elt->content.soa.expire);
+ GET32 (elt->content.soa.minimum);
+ }
+ else {
+ rdns_info ("invalid data in SOA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_MX:
+ GET16 (elt->content.mx.priority);
+ if (! rdns_parse_labels (resolver, in, &elt->content.mx.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in MX record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_TXT:
+ case DNS_T_SPF:
+ if (datalen <= *remain) {
+ elt->content.txt.data = malloc(datalen + 1);
+ if (elt->content.txt.data == NULL) {
+ rdns_err ("failed to allocate %d bytes for TXT record; domain %s",
+ (int) datalen + 1, rep->requested_name);
+ return -1;
+ }
+ /* Now we should compose data from parts */
+ copied = 0;
+ parts = 0;
+ while (copied + parts < datalen && *remain > 0) {
+ txtlen = *p;
+ if (txtlen + copied + parts <= datalen && *remain >= txtlen + 1) {
+ parts++;
+ memcpy (elt->content.txt.data + copied, p + 1, txtlen);
+ copied += txtlen;
+ p += txtlen + 1;
+ *remain -= txtlen + 1;
+ }
+ else {
+
+ if (txtlen + copied + parts > datalen) {
+ /* Incorrect datalen reported ! */
+ rdns_err ("incorrect txtlen (%d) > datalen (%d) reported; domain %s",
+ (txtlen + copied + parts), datalen,
+ rep->requested_name);
+ return -1;
+ }
+
+ /* Reported equal to the actual data copied */
+ break;
+ }
+ }
+ *(elt->content.txt.data + copied) = '\0';
+ parsed = true;
+ elt->type = RDNS_REQUEST_TXT;
+ }
+ else {
+ rdns_info ("stripped data in TXT record (%d bytes available, %d requested); "
+ "domain %s", (int)*remain, (int)datalen, rep->requested_name);
+ return -1;
+ }
+ break;
+ case DNS_T_SRV:
+ if (p - *pos > (int)(*remain - sizeof (uint16_t) * 3)) {
+ rdns_info ("stripped dns reply while reading SRV record; domain %s", rep->requested_name);
+ return -1;
+ }
+ GET16 (elt->content.srv.priority);
+ GET16 (elt->content.srv.weight);
+ GET16 (elt->content.srv.port);
+ if (! rdns_parse_labels (resolver, in, &elt->content.srv.target,
+ &p, rep, remain, true)) {
+ rdns_info ("invalid labels in SRV record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_TLSA:
+ if (p - *pos > (int)(*remain - sizeof (uint8_t) * 3) || datalen <= 3) {
+ rdns_info ("stripped dns reply while reading TLSA record; domain %s", rep->requested_name);
+ return -1;
+ }
+
+ if (datalen > *remain) {
+ rdns_info ("too large datalen; domain %s", rep->requested_name);
+ return -1;
+ }
+
+ GET8 (elt->content.tlsa.usage);
+ GET8 (elt->content.tlsa.selector);
+ GET8 (elt->content.tlsa.match_type);
+ datalen -= 3;
+
+ elt->content.tlsa.data = malloc (datalen);
+ if (elt->content.tlsa.data == NULL) {
+ rdns_err ("failed to allocate %d bytes for TLSA record; domain %s",
+ (int)datalen + 1, rep->requested_name);
+ return -1;
+ }
+
+ elt->content.tlsa.datalen = datalen;
+ memcpy (elt->content.tlsa.data, p, datalen);
+ p += datalen;
+ *remain -= datalen;
+ parsed = true;
+ break;
+ case DNS_T_CNAME:
+ if (! rdns_parse_labels (resolver, in, &elt->content.cname.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in CNAME record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ default:
+ rdns_info ("unexpected RR type: %d; domain %s", type, rep->requested_name);
+ p += datalen;
+ *remain -= datalen;
+ break;
+ }
+ *pos = p;
+
+ if (parsed) {
+ elt->ttl = ttl;
+ return 1;
+ }
+ return 0;
+}
diff --git a/contrib/librdns/parse.h b/contrib/librdns/parse.h
new file mode 100644
index 0000000..ed8cd70
--- /dev/null
+++ b/contrib/librdns/parse.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef PARSE_H_
+#define PARSE_H_
+
+#include "dns_private.h"
+
+/**
+ * Compare request and reply checking names
+ * @param req request object
+ * @param in incoming packet
+ * @param len length of the incoming packet
+ * @return new position in the incoming packet or NULL if request is not equal to reply
+ */
+uint8_t * rdns_request_reply_cmp (struct rdns_request *req, uint8_t *in, int len);
+
+/**
+ * Parse labels in the packet
+ * @param in incoming packet
+ * @param target target to write the parsed label (out)
+ * @param pos output position in the packet (it/out)
+ * @param rep dns reply
+ * @param remain remaining bytes (in/out)
+ * @param make_name create name or just skip to the next label
+ * @return true if a label has been successfully parsed
+ */
+bool rdns_parse_labels (struct rdns_resolver *resolver,
+ uint8_t *in, char **target,
+ uint8_t **pos, struct rdns_reply *rep,
+ int *remain, bool make_name);
+
+/**
+ * Parse resource record
+ * @param in incoming packet
+ * @param elt new reply entry
+ * @param pos output position in the packet (it/out)
+ * @param rep dns reply
+ * @param remain remaining bytes (in/out)
+ * @return 1 if rr has been parsed, 0 if rr has been skipped and -1 if there was a parsing error
+ */
+int rdns_parse_rr (struct rdns_resolver *resolver,
+ uint8_t *in, struct rdns_reply_entry *elt, uint8_t **pos,
+ struct rdns_reply *rep, int *remain);
+
+#endif /* PARSE_H_ */
diff --git a/contrib/librdns/punycode.c b/contrib/librdns/punycode.c
new file mode 100644
index 0000000..61091b2
--- /dev/null
+++ b/contrib/librdns/punycode.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ * Copyright (c) 2004, 2006, 2007, 2008 Kungliga Tekniska Högskolan
+ * (Royal Institute of Technology, Stockholm, Sweden).
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the Institute nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "dns_private.h"
+static const unsigned event_loop = 36;
+static const unsigned t_min = 1;
+static const unsigned t_max = 26;
+static const unsigned skew = 38;
+static const unsigned damp = 700;
+static const unsigned initial_n = 128;
+static const unsigned initial_bias = 72;
+/* Punycode utility */
+static unsigned int
+digit (unsigned n)
+{
+ static const char ascii[] = "abcdefghijklmnopqrstuvwxyz0123456789";
+ return ascii[n];
+}
+
+static unsigned int
+adapt (unsigned int delta, unsigned int numpoints, int first)
+{
+ unsigned int k;
+
+ if (first) {
+ delta = delta / damp;
+ }
+ else {
+ delta /= 2;
+ }
+ delta += delta / numpoints;
+ k = 0;
+ while (delta > ((event_loop - t_min) * t_max) / 2) {
+ delta /= event_loop - t_min;
+ k += event_loop;
+ }
+ return k + (((event_loop - t_min + 1) * delta) / (delta + skew));
+}
+
+/**
+ * Convert an UCS4 string to a puny-coded DNS label string suitable
+ * when combined with delimiters and other labels for DNS lookup.
+ *
+ * @param in an UCS4 string to convert
+ * @param in_len the length of in.
+ * @param out the resulting puny-coded string. The string is not NULL
+ * terminated.
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an wind error code otherwise
+ */
+
+bool
+rdns_punycode_label_toascii (const uint32_t *in, size_t in_len, char *out,
+ size_t *out_len)
+{
+ unsigned int n = initial_n;
+ unsigned int delta = 0;
+ unsigned int bias = initial_bias;
+ unsigned int h = 0;
+ unsigned int b;
+ unsigned int i;
+ unsigned int o = 0;
+ unsigned int m;
+
+ for (i = 0; i < in_len; ++i) {
+ if (in[i] < 0x80) {
+ ++h;
+ if (o >= *out_len) {
+ return false;
+ }
+ out[o++] = in[i];
+ }
+ }
+ b = h;
+ if (b > 0) {
+ if (o >= *out_len) {
+ return false;
+ }
+ out[o++] = 0x2D;
+ }
+ /* is this string punycoded */
+ if (h < in_len) {
+ if (o + 4 >= *out_len) {
+ return false;
+ }
+ memmove (out + 4, out, o);
+ memcpy (out, "xn--", 4);
+ o += 4;
+ }
+
+ while (h < in_len) {
+ m = (unsigned int) -1;
+ for (i = 0; i < in_len; ++i) {
+
+ if (in[i] < m && in[i] >= n) {
+ m = in[i];
+ }
+ }
+ delta += (m - n) * (h + 1);
+ n = m;
+ for (i = 0; i < in_len; ++i) {
+ if (in[i] < n) {
+ ++delta;
+ }
+ else if (in[i] == n) {
+ unsigned int q = delta;
+ unsigned int k;
+ for (k = event_loop;; k += event_loop) {
+ unsigned int t;
+ if (k <= bias) {
+ t = t_min;
+ }
+ else if (k >= bias + t_max) {
+ t = t_max;
+ }
+ else {
+ t = k - bias;
+ }
+ if (q < t) {
+ break;
+ }
+ if (o >= *out_len) {
+ return -1;
+ }
+ out[o++] = digit (t + ((q - t) % (event_loop - t)));
+ q = (q - t) / (event_loop - t);
+ }
+ if (o >= *out_len) {
+ return -1;
+ }
+ out[o++] = digit (q);
+ /* output */
+ bias = adapt (delta, h + 1, h == b);
+ delta = 0;
+ ++h;
+ }
+ }
+ ++delta;
+ ++n;
+ }
+
+ *out_len = o;
+ return true;
+}
+
+static int
+utf8toutf32 (const unsigned char **pp, uint32_t *out, size_t *remain)
+{
+ const unsigned char *p = *pp;
+ unsigned c = *p;
+ size_t reduce;
+
+ if (c & 0x80) {
+ if ((c & 0xE0) == 0xC0 && *remain >= 2) {
+ const unsigned c2 = *++p;
+ reduce = 2;
+ if ((c2 & 0xC0) == 0x80) {
+ *out = ((c & 0x1F) << 6) | (c2 & 0x3F);
+ }
+ else {
+ return -1;
+ }
+ }
+ else if ((c & 0xF0) == 0xE0 && *remain >= 3) {
+ const unsigned c2 = *++p;
+ if ((c2 & 0xC0) == 0x80) {
+ const unsigned c3 = *++p;
+ reduce = 3;
+ if ((c3 & 0xC0) == 0x80) {
+ *out = ((c & 0x0F) << 12) | ((c2 & 0x3F) << 6)
+ | (c3 & 0x3F);
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else if ((c & 0xF8) == 0xF0 && *remain >= 4) {
+ const unsigned c2 = *++p;
+ if ((c2 & 0xC0) == 0x80) {
+ const unsigned c3 = *++p;
+ if ((c3 & 0xC0) == 0x80) {
+ const unsigned c4 = *++p;
+ reduce = 4;
+ if ((c4 & 0xC0) == 0x80) {
+ *out = ((c & 0x07) << 18) | ((c2 & 0x3F) << 12)
+ | ((c3 & 0x3F) << 6) | (c4 & 0x3F);
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ *out = c;
+ reduce = 1;
+ }
+
+ *pp = ++p;
+ *remain -= reduce;
+
+ return 0;
+}
+
+/**
+ * Convert an UTF-8 string to an UCS4 string.
+ *
+ * @param in an UTF-8 string to convert.
+ * @param out the resulting UCS4 string
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an -1 otherwise
+ * @ingroup wind
+ */
+
+int
+rdns_utf8_to_ucs4 (const char *in, size_t in_len, uint32_t **out, size_t *out_len)
+{
+ const unsigned char *p;
+ size_t remain = in_len, olen = 0;
+ int ret;
+ uint32_t *res;
+
+ p = (const unsigned char *)in;
+ while (remain > 0) {
+ uint32_t u;
+
+ ret = utf8toutf32 (&p, &u, &remain);
+ if (ret != 0) {
+ return ret;
+ }
+
+ olen ++;
+ }
+ res = malloc (olen * sizeof (uint32_t));
+ if (res == NULL) {
+ return -1;
+ }
+
+ p = (const unsigned char *)in;
+ remain = in_len;
+ olen = 0;
+ while (remain > 0) {
+ uint32_t u;
+
+ (void)utf8toutf32 (&p, &u, &remain);
+ res[olen++] = u;
+ }
+
+ *out_len = olen;
+ *out = res;
+ return 0;
+}
diff --git a/contrib/librdns/punycode.h b/contrib/librdns/punycode.h
new file mode 100644
index 0000000..300fb92
--- /dev/null
+++ b/contrib/librdns/punycode.h
@@ -0,0 +1,59 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef PUNYCODE_H_
+#define PUNYCODE_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/**
+ * Convert an UCS4 string to a puny-coded DNS label string suitable
+ * when combined with delimiters and other labels for DNS lookup.
+ *
+ * @param in an UCS4 string to convert
+ * @param in_len the length of in.
+ * @param out the resulting puny-coded string. The string is not NULL
+ * terminated.
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an wind error code otherwise
+ */
+bool rdns_punycode_label_toascii (const uint32_t *in, size_t in_len, char *out, size_t *out_len);
+/**
+ * Convert an UTF-8 string to an UCS4 string.
+ *
+ * @param in an UTF-8 string to convert.
+ * @param out the resulting UCS4 string
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an -1 otherwise
+ * @ingroup wind
+ */
+
+int rdns_utf8_to_ucs4 (const char *in, size_t in_len, uint32_t **out, size_t *out_len);
+
+#endif /* PUNYCODE_H_ */
diff --git a/contrib/librdns/rdns.h b/contrib/librdns/rdns.h
new file mode 100644
index 0000000..c0da5ed
--- /dev/null
+++ b/contrib/librdns/rdns.h
@@ -0,0 +1,493 @@
+/*
+ * Copyright (c) 2013-2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RDNS_H
+#define RDNS_H
+
+#include <sys/types.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rdns_reply;
+struct rdns_request;
+struct rdns_io_channel;
+
+typedef void (*dns_callback_type) (struct rdns_reply *reply, void *arg);
+
+enum rdns_request_type {
+ RDNS_REQUEST_INVALID = -1,
+ RDNS_REQUEST_A = 1,
+ RDNS_REQUEST_NS = 2,
+ RDNS_REQUEST_CNAME = 5,
+ RDNS_REQUEST_SOA = 6,
+ RDNS_REQUEST_PTR = 12,
+ RDNS_REQUEST_MX = 15,
+ RDNS_REQUEST_TXT = 16,
+ RDNS_REQUEST_SRV = 33,
+ RDNS_REQUEST_SPF = 99,
+ RDNS_REQUEST_AAAA = 28,
+ RDNS_REQUEST_TLSA = 52,
+ RDNS_REQUEST_ANY = 255
+};
+
+union rdns_reply_element_un {
+ struct {
+ struct in_addr addr;
+ } a;
+ struct {
+ struct in6_addr addr;
+ } aaa;
+ struct {
+ char *name;
+ } ptr;
+ struct {
+ char *name;
+ } ns;
+ struct {
+ char *name;
+ uint16_t priority;
+ } mx;
+ struct {
+ char *data;
+ } txt;
+ struct {
+ uint16_t priority;
+ uint16_t weight;
+ uint16_t port;
+ char *target;
+ } srv;
+ struct {
+ char *mname;
+ char *admin;
+ uint32_t serial;
+ int32_t refresh;
+ int32_t retry;
+ int32_t expire;
+ uint32_t minimum;
+ } soa;
+ struct {
+ uint8_t usage;
+ uint8_t selector;
+ uint8_t match_type;
+ uint16_t datalen;
+ uint8_t *data;
+ } tlsa;
+ struct {
+ char *name;
+ } cname;
+};
+
+struct rdns_reply_entry {
+ union rdns_reply_element_un content;
+ enum rdns_request_type type;
+ int32_t ttl;
+ struct rdns_reply_entry *prev, *next;
+};
+
+
+enum dns_rcode {
+ RDNS_RC_INVALID = -1,
+ RDNS_RC_NOERROR = 0,
+ RDNS_RC_FORMERR = 1,
+ RDNS_RC_SERVFAIL = 2,
+ RDNS_RC_NXDOMAIN = 3,
+ RDNS_RC_NOTIMP = 4,
+ RDNS_RC_REFUSED = 5,
+ RDNS_RC_YXDOMAIN = 6,
+ RDNS_RC_YXRRSET = 7,
+ RDNS_RC_NXRRSET = 8,
+ RDNS_RC_NOTAUTH = 9,
+ RDNS_RC_NOTZONE = 10,
+ RDNS_RC_TIMEOUT = 11,
+ RDNS_RC_NETERR = 12,
+ RDNS_RC_NOREC = 13
+};
+
+enum dns_reply_flags {
+ RDNS_AUTH = (1u << 0u),
+ RDNS_TRUNCATED = (1u << 1u)
+};
+
+struct rdns_reply {
+ struct rdns_request *request;
+ struct rdns_resolver *resolver;
+ struct rdns_reply_entry *entries;
+ const char *requested_name;
+ enum dns_rcode code;
+ uint8_t flags; /* see enum dns_reply_flags */
+};
+
+typedef void (*rdns_periodic_callback)(void *user_data);
+
+struct rdns_async_context {
+ void *data;
+ void* (*add_read)(void *priv_data, int fd, void *user_data);
+ void (*del_read)(void *priv_data, void *ev_data);
+ void* (*add_write)(void *priv_data, int fd, void *user_data);
+ void (*del_write)(void *priv_data, void *ev_data);
+ void* (*add_timer)(void *priv_data, double after, void *user_data);
+ void (*repeat_timer)(void *priv_data, void *ev_data);
+ void (*del_timer)(void *priv_data, void *ev_data);
+ void* (*add_periodic)(void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data);
+ void (*del_periodic)(void *priv_data, void *ev_data);
+ void (*cleanup)(void *priv_data);
+};
+
+struct rdns_upstream_elt {
+ void *server;
+ void *lib_data;
+};
+
+struct rdns_upstream_context {
+ void *data;
+ struct rdns_upstream_elt* (*select)(const char *name,
+ size_t len, void *ups_data);
+ struct rdns_upstream_elt* (*select_retransmit)(const char *name, size_t len,
+ struct rdns_upstream_elt* prev_elt,
+ void *ups_data);
+ unsigned int (*count)(void *ups_data);
+ void (*ok)(struct rdns_upstream_elt *elt, void *ups_data);
+ void (*fail)(struct rdns_upstream_elt *elt, void *ups_data, const char *reason);
+};
+
+/**
+ * Type of rdns plugin
+ */
+enum rdns_plugin_type {
+ RDNS_PLUGIN_CURVE = 0
+};
+
+typedef ssize_t (*rdns_network_send_callback) (struct rdns_request *req, void *plugin_data,
+ struct sockaddr *saddr, socklen_t slen);
+typedef ssize_t (*rdns_network_recv_callback) (struct rdns_io_channel *ioc, void *buf,
+ size_t len, void *plugin_data,
+ struct rdns_request **req_out,
+ struct sockaddr *saddr, socklen_t slen);
+typedef void (*rdns_network_finish_callback) (struct rdns_request *req, void *plugin_data);
+typedef void (*rdns_plugin_dtor_callback) (struct rdns_resolver *resolver, void *plugin_data);
+
+struct rdns_plugin {
+ enum rdns_plugin_type type;
+ union {
+ struct {
+ rdns_network_send_callback send_cb;
+ rdns_network_recv_callback recv_cb;
+ rdns_network_finish_callback finish_cb;
+ } curve_plugin;
+ } cb;
+ rdns_plugin_dtor_callback dtor;
+ void *data;
+};
+
+/*
+ * RDNS logger types
+ */
+/*
+ * These types are somehow compatible with glib
+ */
+enum rdns_log_level {
+ RDNS_LOG_ERROR = 1 << 3,
+ RDNS_LOG_WARNING = 1 << 4,
+ RDNS_LOG_INFO = 1 << 6,
+ RDNS_LOG_DEBUG = 1 << 7
+};
+typedef void (*rdns_log_function) (
+ void *log_data, //!< opaque data pointer
+ enum rdns_log_level level, //!< level of message
+ const char *function, //!< calling function
+ const char *format, //!< format
+ va_list args //!< set of arguments
+ );
+
+struct rdns_request_name {
+ char *name;
+ enum rdns_request_type type;
+ unsigned int len;
+};
+
+#define MAX_FAKE_NAME 1000
+
+/*
+ * RDNS API
+ */
+
+enum rdns_resolver_flags {
+ RDNS_RESOLVER_DEFAULT,
+ RDNS_RESOLVER_NOIDN = (1u << 0u),
+};
+
+/**
+ * Create DNS resolver structure
+ */
+struct rdns_resolver *rdns_resolver_new (int flags);
+
+/**
+ * Bind resolver to specified async context
+ * @param ctx
+ */
+void rdns_resolver_async_bind (struct rdns_resolver *resolver,
+ struct rdns_async_context *ctx);
+
+/**
+ * Enable stub dnssec resolver
+ * @param resolver
+ */
+void rdns_resolver_set_dnssec (struct rdns_resolver *resolver, bool enabled);
+
+/**
+ * Add new DNS server definition to the resolver
+ * @param resolver resolver object
+ * @param name name of DNS server (should be ipv4 or ipv6 address)
+ * @param priority priority (can be 0 for fair round-robin)
+ * @param io_cnt a number of sockets that are simultaneously opened to this server
+ * @return opaque pointer that could be used to select upstream
+ */
+void* rdns_resolver_add_server (struct rdns_resolver *resolver,
+ const char *name, unsigned int port,
+ int priority, unsigned int io_cnt);
+
+
+/**
+ * Load nameservers definition from resolv.conf file
+ * @param resolver resolver object
+ * @param path path to resolv.conf file (/etc/resolv.conf typically)
+ * @return true if resolv.conf has been parsed
+ */
+bool rdns_resolver_parse_resolv_conf (struct rdns_resolver *resolver,
+ const char *path);
+
+typedef bool (*rdns_resolv_conf_cb) (struct rdns_resolver *resolver,
+ const char *name, unsigned int port,
+ int priority, unsigned int io_cnt, void *ud);
+/**
+ * Parse nameservers calling the specified callback for each nameserver
+ * @param resolve resolver object
+ * @param path path to resolv.conf file (/etc/resolv.conf typically)
+ * @param cb callback to call
+ * @param ud userdata for callback
+ * @return true if resolv.conf has been parsed
+ */
+bool rdns_resolver_parse_resolv_conf_cb (struct rdns_resolver *resolver,
+ const char *path, rdns_resolv_conf_cb cb, void *ud);
+
+/**
+ * Set an external logger function to log messages from the resolver
+ * @param resolver resolver object
+ * @param logger logger callback
+ * @param log_data opaque data
+ */
+void rdns_resolver_set_logger (struct rdns_resolver *resolver,
+ rdns_log_function logger, void *log_data);
+
+/**
+ * Set log level for an internal logger (stderr one)
+ * @param resolver resolver object
+ * @param level desired log level
+ */
+void rdns_resolver_set_log_level (struct rdns_resolver *resolver,
+ enum rdns_log_level level);
+
+/**
+ * Set upstream library for selecting DNS upstreams
+ * @param resolver resolver object
+ * @param ups_ctx upstream functions
+ * @param ups_data opaque data
+ */
+void rdns_resolver_set_upstream_lib (struct rdns_resolver *resolver,
+ struct rdns_upstream_context *ups_ctx,
+ void *ups_data);
+
+/**
+ * Set maximum number of dns requests to be sent to a socket to be refreshed
+ * @param resolver resolver object
+ * @param max_ioc_uses unsigned count of socket usage limit
+ * @param check_time specifies how often to check for sockets and refresh them
+ */
+void rdns_resolver_set_max_io_uses (struct rdns_resolver *resolver,
+ uint64_t max_ioc_uses, double check_time);
+
+/**
+ * Register new plugin for rdns resolver
+ * @param resolver
+ * @param plugin
+ */
+void rdns_resolver_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_plugin *plugin);
+
+/**
+ * Add a fake reply for a specified name
+ * @param resolver
+ * @param type
+ * @param name (must not be larger than MAX_FAKE_NAME)
+ * @param reply
+ */
+void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver,
+ const char *name,
+ enum rdns_request_type type,
+ enum dns_rcode rcode,
+ struct rdns_reply_entry *reply);
+
+/**
+ * Init DNS resolver
+ * @param resolver
+ * @return
+ */
+bool rdns_resolver_init (struct rdns_resolver *resolver);
+
+/**
+ * Decrease refcount for a resolver and free it if refcount is 0
+ * @param resolver
+ */
+void rdns_resolver_release (struct rdns_resolver *resolver);
+
+/**
+ * Make a DNS request
+ * @param resolver resolver object
+ * @param cb callback to call on resolve completing
+ * @param ud user data for callback
+ * @param timeout timeout in seconds
+ * @param repeats how much time to retransmit query
+ * @param queries how much RR queries to send
+ * @param ... -> queries in format: <query_type>[,type_argument[,type_argument...]]
+ * @return opaque request object or NULL
+ */
+struct rdns_request* rdns_make_request_full (
+ struct rdns_resolver *resolver,
+ dns_callback_type cb,
+ void *cbdata,
+ double timeout,
+ unsigned int repeats,
+ unsigned int queries,
+ ...
+ );
+
+/**
+ * Get textual presentation of DNS error code
+ */
+const char *rdns_strerror (enum dns_rcode rcode);
+
+/**
+ * Get textual presentation of DNS request type
+ */
+const char *rdns_strtype (enum rdns_request_type type);
+
+/**
+ * Parse string and return request type
+ * @param str
+ * @return
+ */
+enum rdns_request_type rdns_type_fromstr (const char *str);
+
+/**
+ * Returns string representing request type
+ * @param rcode
+ * @return
+ */
+const char *
+rdns_str_from_type (enum rdns_request_type rcode);
+
+/**
+ * Parse string and return error code
+ * @param str
+ * @return
+ */
+enum dns_rcode rdns_rcode_fromstr (const char *str);
+
+/**
+ * Increase refcount for a request
+ * @param req
+ * @return
+ */
+struct rdns_request* rdns_request_retain (struct rdns_request *req);
+
+/**
+ * Decrease refcount for a request and free it if refcount is 0
+ * @param req
+ */
+void rdns_request_release (struct rdns_request *req);
+
+/**
+ * Check whether a request contains `type` request
+ * @param req request object
+ * @param type check for a specified type
+ * @return true if `type` has been requested
+ */
+bool rdns_request_has_type (struct rdns_request *req, enum rdns_request_type type);
+
+/**
+ * Return requested name for a request
+ * @param req request object
+ * @return requested name as it was passed to `rdns_make_request`
+ */
+const struct rdns_request_name* rdns_request_get_name (struct rdns_request *req,
+ unsigned int *count);
+
+/**
+ * Return a DNS server name associated with the request
+ * @param req request object
+ * @return name of a DNS server
+ */
+const char* rdns_request_get_server (struct rdns_request *req);
+
+
+/**
+ * Return PTR string for a request (ipv4 or ipv6) addresses
+ * @param str string representation of IP address
+ * @return name to resolve or NULL if `str` is not an IP address; caller must free result when it is unused
+ */
+char * rdns_generate_ptr_from_str (const char *str);
+
+/**
+ * Format DNS name of the packet punycoding if needed
+ * @param req request
+ * @param name name string
+ * @param namelen length of name
+ */
+bool rdns_format_dns_name (struct rdns_resolver *resolver,
+ const char *name, size_t namelen,
+ char **out, size_t *outlen);
+
+/*
+ * Private functions used by async libraries as callbacks
+ */
+
+void rdns_process_read (int fd, void *arg);
+void rdns_process_timer (void *arg);
+void rdns_process_write (int fd, void *arg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/librdns/rdns_curve.h b/contrib/librdns/rdns_curve.h
new file mode 100644
index 0000000..365e91b
--- /dev/null
+++ b/contrib/librdns/rdns_curve.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RDNS_CURVE_H_
+#define RDNS_CURVE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rdns_curve_ctx;
+
+/**
+ * Create new dnscurve ctx
+ * @return
+ */
+struct rdns_curve_ctx* rdns_curve_ctx_new (double rekey_interval);
+
+/**
+ * Add key for server `name`
+ * @param ctx curve context
+ * @param name name of server (ip address)
+ * @param pubkey pubkey bytes (must be `RDSN_CURVE_PUBKEY_LEN`)
+ */
+void rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey);
+
+/**
+ * Destroy curve context
+ * @param ctx
+ */
+void rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx);
+
+
+/**
+ * Register DNSCurve plugin (libsodium should be enabled for this)
+ * @param resolver
+ * @param ctx
+ */
+void rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx);
+
+/**
+ * Create DNSCurve key from the base16 encoded string
+ * @param hex input hex (must be NULL terminated)
+ * @return a key or NULL (not NULL terminated)
+ */
+unsigned char * rdns_curve_key_from_hex (const char *hex);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDNS_CURVE_H_ */
diff --git a/contrib/librdns/rdns_ev.h b/contrib/librdns/rdns_ev.h
new file mode 100644
index 0000000..35f532a
--- /dev/null
+++ b/contrib/librdns/rdns_ev.h
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RDNS_EV_H_
+#define RDNS_EV_H_
+
+#include "contrib/libev/ev.h"
+#include <stdlib.h>
+#include <string.h>
+#include "rdns.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static void* rdns_libev_add_read (void *priv_data, int fd, void *user_data);
+static void rdns_libev_del_read(void *priv_data, void *ev_data);
+static void* rdns_libev_add_write (void *priv_data, int fd, void *user_data);
+static void rdns_libev_del_write (void *priv_data, void *ev_data);
+static void* rdns_libev_add_timer (void *priv_data, double after, void *user_data);
+static void* rdns_libev_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data);
+static void rdns_libev_del_periodic (void *priv_data, void *ev_data);
+static void rdns_libev_repeat_timer (void *priv_data, void *ev_data);
+static void rdns_libev_del_timer (void *priv_data, void *ev_data);
+
+struct rdns_ev_periodic_cbdata {
+ ev_timer *ev;
+ rdns_periodic_callback cb;
+ void *cbdata;
+};
+
+static void
+rdns_bind_libev (struct rdns_resolver *resolver, struct ev_loop *loop)
+{
+ static struct rdns_async_context ev_ctx = {
+ .data = NULL,
+ .add_read = rdns_libev_add_read,
+ .del_read = rdns_libev_del_read,
+ .add_write = rdns_libev_add_write,
+ .del_write = rdns_libev_del_write,
+ .add_timer = rdns_libev_add_timer,
+ .repeat_timer = rdns_libev_repeat_timer,
+ .del_timer = rdns_libev_del_timer,
+ .add_periodic = rdns_libev_add_periodic,
+ .del_periodic = rdns_libev_del_periodic,
+ .cleanup = NULL
+ }, *nctx;
+ void *ptr;
+
+ /* XXX: never got freed */
+ ptr = malloc (sizeof (struct rdns_async_context));
+ if (ptr != NULL) {
+ nctx = (struct rdns_async_context *)ptr;
+ memcpy (ptr, (void *)&ev_ctx, sizeof (struct rdns_async_context));
+ nctx->data = (void*)loop;
+ rdns_resolver_async_bind (resolver, nctx);
+ }
+}
+
+static void
+rdns_libev_read_event (struct ev_loop *loop, ev_io *ev, int revents)
+{
+ rdns_process_read (ev->fd, ev->data);
+}
+
+static void
+rdns_libev_write_event (struct ev_loop *loop, ev_io *ev, int revents)
+{
+ rdns_process_write(ev->fd, ev->data);
+}
+
+static void
+rdns_libev_timer_event (struct ev_loop *loop, ev_timer *ev, int revents)
+{
+ rdns_process_timer (ev->data);
+}
+
+static void
+rdns_libev_periodic_event (struct ev_loop *loop, ev_timer *ev, int revents)
+{
+ struct rdns_ev_periodic_cbdata *cbdata = (struct rdns_ev_periodic_cbdata *)
+ ev->data;
+ cbdata->cb (cbdata->cbdata);
+}
+
+static void*
+rdns_libev_add_read (void *priv_data, int fd, void *user_data)
+{
+ ev_io *ev;
+ void *ptr;
+
+ ptr = malloc (sizeof (ev_io));
+ if (ptr != NULL) {
+ ev = (ev_io *)ptr;
+ ev_io_init (ev, rdns_libev_read_event, fd, EV_READ);
+ ev->data = user_data;
+ ev_io_start ((struct ev_loop *)priv_data, ev);
+ }
+ return ptr;
+}
+
+static void
+rdns_libev_del_read (void *priv_data, void *ev_data)
+{
+ ev_io *ev = (ev_io*)ev_data;
+ if (ev != NULL) {
+ ev_io_stop ((struct ev_loop *)priv_data, ev);
+ free ((void *)ev);
+ }
+}
+static void*
+rdns_libev_add_write (void *priv_data, int fd, void *user_data)
+{
+ ev_io *ev;
+
+ ev = (ev_io *)malloc (sizeof (ev_io));
+ if (ev != NULL) {
+ ev_io_init (ev, rdns_libev_write_event, fd, EV_WRITE);
+ ev->data = user_data;
+ ev_io_start ((struct ev_loop *)priv_data, ev);
+ }
+ return (void *)ev;
+}
+
+static void
+rdns_libev_del_write (void *priv_data, void *ev_data)
+{
+ ev_io *ev = (ev_io *)ev_data;
+ if (ev != NULL) {
+ ev_io_stop ((struct ev_loop *)priv_data, ev);
+ free ((void *)ev);
+ }
+}
+
+static void*
+rdns_libev_add_timer (void *priv_data, double after, void *user_data)
+{
+ ev_timer *ev;
+ ev = (ev_timer *)malloc (sizeof (ev_timer));
+ if (ev != NULL) {
+ ev_timer_init (ev, rdns_libev_timer_event, after, after);
+ ev->data = user_data;
+ ev_now_update_if_cheap ((struct ev_loop *)priv_data);
+ ev_timer_start ((struct ev_loop *)priv_data, ev);
+ }
+ return (void *)ev;
+}
+
+static void*
+rdns_libev_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data)
+{
+ ev_timer *ev;
+ struct rdns_ev_periodic_cbdata *cbdata = NULL;
+
+ ev = (ev_timer *)malloc (sizeof (ev_timer));
+ if (ev != NULL) {
+ cbdata = (struct rdns_ev_periodic_cbdata *)
+ malloc (sizeof (struct rdns_ev_periodic_cbdata));
+ if (cbdata != NULL) {
+ cbdata->cb = cb;
+ cbdata->cbdata = user_data;
+ cbdata->ev = ev;
+ ev_timer_init (ev, rdns_libev_periodic_event, after, after);
+ ev->data = cbdata;
+ ev_now_update_if_cheap ((struct ev_loop *)priv_data);
+ ev_timer_start ((struct ev_loop *)priv_data, ev);
+ }
+ else {
+ free ((void *)ev);
+ return NULL;
+ }
+ }
+ return (void *)cbdata;
+}
+
+static void
+rdns_libev_del_periodic (void *priv_data, void *ev_data)
+{
+ struct rdns_ev_periodic_cbdata *cbdata = (struct rdns_ev_periodic_cbdata *)
+ ev_data;
+ if (cbdata != NULL) {
+ ev_timer_stop ((struct ev_loop *)priv_data, (ev_timer *)cbdata->ev);
+ free ((void *)cbdata->ev);
+ free ((void *)cbdata);
+ }
+}
+
+static void
+rdns_libev_repeat_timer (void *priv_data, void *ev_data)
+{
+ ev_timer *ev = (ev_timer *)ev_data;
+ if (ev != NULL) {
+ ev_now_update_if_cheap ((struct ev_loop *)priv_data);
+ ev_timer_again ((struct ev_loop *)priv_data, ev);
+ }
+}
+
+static void
+rdns_libev_del_timer (void *priv_data, void *ev_data)
+{
+ ev_timer *ev = (ev_timer *)ev_data;
+ if (ev != NULL) {
+ ev_timer_stop ((struct ev_loop *)priv_data, ev);
+ free ((void *)ev);
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDNS_EV_H_ */
diff --git a/contrib/librdns/rdns_event.h b/contrib/librdns/rdns_event.h
new file mode 100644
index 0000000..027181a
--- /dev/null
+++ b/contrib/librdns/rdns_event.h
@@ -0,0 +1,231 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RDNS_EVENT_H_
+#define RDNS_EVENT_H_
+
+#include <event.h>
+#include <stdlib.h>
+#include <string.h>
+#include "rdns.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static void* rdns_libevent_add_read (void *priv_data, int fd, void *user_data);
+static void rdns_libevent_del_read(void *priv_data, void *ev_data);
+static void* rdns_libevent_add_write (void *priv_data, int fd, void *user_data);
+static void rdns_libevent_del_write (void *priv_data, void *ev_data);
+static void* rdns_libevent_add_timer (void *priv_data, double after, void *user_data);
+static void* rdns_libevent_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data);
+static void rdns_libevent_del_periodic (void *priv_data, void *ev_data);
+static void rdns_libevent_repeat_timer (void *priv_data, void *ev_data);
+static void rdns_libevent_del_timer (void *priv_data, void *ev_data);
+
+struct rdns_event_periodic_cbdata {
+ struct event *ev;
+ rdns_periodic_callback cb;
+ void *cbdata;
+};
+
+static void
+rdns_bind_libevent (struct rdns_resolver *resolver, struct event_base *ev_base)
+{
+ struct rdns_async_context ev_ctx = {
+ .add_read = rdns_libevent_add_read,
+ .del_read = rdns_libevent_del_read,
+ .add_write = rdns_libevent_add_write,
+ .del_write = rdns_libevent_del_write,
+ .add_timer = rdns_libevent_add_timer,
+ .add_periodic = rdns_libevent_add_periodic,
+ .del_periodic = rdns_libevent_del_periodic,
+ .repeat_timer = rdns_libevent_repeat_timer,
+ .del_timer = rdns_libevent_del_timer,
+ .cleanup = NULL
+ }, *nctx;
+
+ /* XXX: never got freed */
+ nctx = malloc (sizeof (struct rdns_async_context));
+ if (nctx != NULL) {
+ memcpy (nctx, &ev_ctx, sizeof (struct rdns_async_context));
+ nctx->data = ev_base;
+ }
+ rdns_resolver_async_bind (resolver, nctx);
+}
+
+static void
+rdns_libevent_read_event (int fd, short what, void *ud)
+{
+ rdns_process_read (fd, ud);
+}
+
+static void
+rdns_libevent_write_event (int fd, short what, void *ud)
+{
+ rdns_process_write (fd, ud);
+}
+
+static void
+rdns_libevent_timer_event (int fd, short what, void *ud)
+{
+ rdns_process_timer (ud);
+}
+
+static void
+rdns_libevent_periodic_event (int fd, short what, void *ud)
+{
+ struct rdns_event_periodic_cbdata *cbdata = ud;
+ cbdata->cb (cbdata->cbdata);
+}
+
+static void*
+rdns_libevent_add_read (void *priv_data, int fd, void *user_data)
+{
+ struct event *ev;
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ event_set (ev, fd, EV_READ | EV_PERSIST, rdns_libevent_read_event, user_data);
+ event_base_set (priv_data, ev);
+ event_add (ev, NULL);
+ }
+ return ev;
+}
+
+static void
+rdns_libevent_del_read(void *priv_data, void *ev_data)
+{
+ struct event *ev = ev_data;
+ if (ev != NULL) {
+ event_del (ev);
+ free (ev);
+ }
+}
+static void*
+rdns_libevent_add_write (void *priv_data, int fd, void *user_data)
+{
+ struct event *ev;
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ event_set (ev, fd, EV_WRITE | EV_PERSIST,
+ rdns_libevent_write_event, user_data);
+ event_base_set (priv_data, ev);
+ event_add (ev, NULL);
+ }
+ return ev;
+}
+
+static void
+rdns_libevent_del_write (void *priv_data, void *ev_data)
+{
+ struct event *ev = ev_data;
+ if (ev != NULL) {
+ event_del (ev);
+ free (ev);
+ }
+}
+
+#define rdns_event_double_to_tv(dbl, tv) do { \
+ (tv)->tv_sec = (int)(dbl); \
+ (tv)->tv_usec = ((dbl) - (int)(dbl))*1000*1000; \
+} while(0)
+
+static void*
+rdns_libevent_add_timer (void *priv_data, double after, void *user_data)
+{
+ struct event *ev;
+ struct timeval tv;
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ rdns_event_double_to_tv (after, &tv);
+ event_set (ev, -1, EV_TIMEOUT|EV_PERSIST, rdns_libevent_timer_event, user_data);
+ event_base_set (priv_data, ev);
+ event_add (ev, &tv);
+ }
+ return ev;
+}
+
+static void*
+rdns_libevent_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data)
+{
+ struct event *ev;
+ struct timeval tv;
+ struct rdns_event_periodic_cbdata *cbdata = NULL;
+
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ cbdata = malloc (sizeof (struct rdns_event_periodic_cbdata));
+ if (cbdata != NULL) {
+ rdns_event_double_to_tv (after, &tv);
+ cbdata->cb = cb;
+ cbdata->cbdata = user_data;
+ cbdata->ev = ev;
+ event_set (ev, -1, EV_TIMEOUT|EV_PERSIST, rdns_libevent_periodic_event, cbdata);
+ event_base_set (priv_data, ev);
+ event_add (ev, &tv);
+ }
+ else {
+ free (ev);
+ return NULL;
+ }
+ }
+ return cbdata;
+}
+
+static void
+rdns_libevent_del_periodic (void *priv_data, void *ev_data)
+{
+ struct rdns_event_periodic_cbdata *cbdata = ev_data;
+ if (cbdata != NULL) {
+ event_del (cbdata->ev);
+ free (cbdata->ev);
+ free (cbdata);
+ }
+}
+
+static void
+rdns_libevent_repeat_timer (void *priv_data, void *ev_data)
+{
+ /* XXX: libevent hides timeval, so timeouts are persistent here */
+}
+
+#undef rdns_event_double_to_tv
+
+static void
+rdns_libevent_del_timer (void *priv_data, void *ev_data)
+{
+ struct event *ev = ev_data;
+ if (ev != NULL) {
+ event_del (ev);
+ free (ev);
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDNS_EV_H_ */
diff --git a/contrib/librdns/ref.h b/contrib/librdns/ref.h
new file mode 100644
index 0000000..a8016b1
--- /dev/null
+++ b/contrib/librdns/ref.h
@@ -0,0 +1,71 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef REF_H_
+#define REF_H_
+
+/**
+ * @file ref.h
+ * A set of macros to handle refcounts
+ */
+
+typedef void (*ref_dtor_cb_t)(void *data);
+
+typedef struct ref_entry_s {
+ unsigned int refcount;
+ ref_dtor_cb_t dtor;
+} ref_entry_t;
+
+#define REF_INIT(obj, dtor_cb) do { \
+ (obj)->ref.refcount = 0; \
+ (obj)->ref.dtor = (ref_dtor_cb_t)(dtor_cb); \
+} while (0)
+
+#define REF_INIT_RETAIN(obj, dtor_cb) do { \
+ (obj)->ref.refcount = 1; \
+ (obj)->ref.dtor = (ref_dtor_cb_t)(dtor_cb); \
+} while (0)
+
+#ifdef HAVE_ATOMIC_BUILTINS
+#define REF_RETAIN(obj) do { \
+ __sync_add_and_fetch (&(obj)->ref.refcount, 1); \
+} while (0)
+
+#define REF_RELEASE(obj) do { \
+ unsigned int rc = __sync_sub_and_fetch (&(obj)->ref.refcount, 1); \
+ if (rc == 0 && (obj)->ref.dtor) { \
+ (obj)->ref.dtor (obj); \
+ } \
+} while (0)
+#else
+#define REF_RETAIN(obj) do { \
+ (obj)->ref.refcount ++; \
+} while (0)
+
+#define REF_RELEASE(obj) do { \
+ if (--(obj)->ref.refcount == 0 && (obj)->ref.dtor) { \
+ (obj)->ref.dtor (obj); \
+ } \
+} while (0)
+#endif
+
+#endif /* REF_H_ */
diff --git a/contrib/librdns/resolver.c b/contrib/librdns/resolver.c
new file mode 100644
index 0000000..bfcfd0a
--- /dev/null
+++ b/contrib/librdns/resolver.c
@@ -0,0 +1,1577 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <sys/uio.h>
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "ottery.h"
+#include "util.h"
+#include "packet.h"
+#include "parse.h"
+#include "logger.h"
+#include "compression.h"
+
+__KHASH_IMPL(rdns_requests_hash, kh_inline, int, struct rdns_request *, true,
+ kh_int_hash_func, kh_int_hash_equal);
+
+static int
+rdns_send_request (struct rdns_request *req, int fd, bool new_req)
+{
+ ssize_t r;
+ struct rdns_server *serv = req->io->srv;
+ struct rdns_resolver *resolver = req->resolver;
+ struct dns_header *header;
+ const int max_id_cycles = 32;
+ khiter_t k;
+
+ /* Find ID collision */
+ if (new_req) {
+ r = 0;
+
+ for (;;) {
+ k = kh_get(rdns_requests_hash, req->io->requests, req->id);
+ if (k != kh_end(req->io->requests)) {
+ /* Check for unique id */
+ header = (struct dns_header *) req->packet;
+ header->qid = rdns_permutor_generate_id();
+ req->id = header->qid;
+ if (++r > max_id_cycles) {
+ return -1;
+ }
+ }
+ else {
+ break;
+ }
+ }
+ }
+
+ if (resolver->curve_plugin == NULL) {
+ if (!IS_CHANNEL_CONNECTED(req->io)) {
+ r = sendto (fd, req->packet, req->pos, 0,
+ req->io->saddr,
+ req->io->slen);
+ }
+ else {
+ r = send (fd, req->packet, req->pos, 0);
+ }
+ }
+ else {
+ if (!IS_CHANNEL_CONNECTED(req->io)) {
+ r = resolver->curve_plugin->cb.curve_plugin.send_cb (req,
+ resolver->curve_plugin->data,
+ req->io->saddr,
+ req->io->slen);
+ }
+ else {
+ r = resolver->curve_plugin->cb.curve_plugin.send_cb (req,
+ resolver->curve_plugin->data,
+ NULL,
+ 0);
+ }
+ }
+ if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ if (new_req) {
+ /* Write when socket is ready */
+ int pr;
+
+ k = kh_put(rdns_requests_hash, req->io->requests, req->id, &pr);
+ kh_value(req->io->requests, k) = req;
+ req->async_event = resolver->async->add_write (resolver->async->data,
+ fd, req);
+ req->state = RDNS_REQUEST_WAIT_SEND;
+ }
+ /*
+ * If request is already processed then the calling function
+ * should take care about events processing
+ */
+ return 0;
+ }
+ else {
+ rdns_debug ("send failed: %s for server %s", strerror (errno), serv->name);
+ return -1;
+ }
+ }
+ else if (!IS_CHANNEL_CONNECTED(req->io)) {
+ /* Connect socket */
+ r = connect (fd, req->io->saddr, req->io->slen);
+
+ if (r == -1) {
+ rdns_err ("cannot connect after sending request: %s for server %s",
+ strerror (errno), serv->name);
+ }
+ else {
+ req->io->flags |= RDNS_CHANNEL_CONNECTED;
+ }
+ }
+
+ if (new_req) {
+ /* Add request to hash table */
+ int pr;
+ k = kh_put(rdns_requests_hash, req->io->requests, req->id, &pr);
+ kh_value(req->io->requests, k) = req;
+ /* Fill timeout */
+ req->async_event = resolver->async->add_timer (resolver->async->data,
+ req->timeout, req);
+ req->state = RDNS_REQUEST_WAIT_REPLY;
+ }
+
+ return 1;
+}
+
+
+static struct rdns_request *
+rdns_find_dns_request (uint8_t *in, struct rdns_io_channel *ioc)
+{
+ struct dns_header header;
+ int id;
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ memcpy (&header, in, sizeof(header));
+ id = header.qid;
+ khiter_t k = kh_get(rdns_requests_hash, ioc->requests, id);
+
+ if (k == kh_end(ioc->requests)) {
+ /* No such requests found */
+ rdns_debug ("DNS request with id %d has not been found for IO channel", id);
+
+ return NULL;
+ }
+
+ return kh_value(ioc->requests, k);
+}
+
+static bool
+rdns_parse_reply (uint8_t *in, int r, struct rdns_request *req,
+ struct rdns_reply **_rep)
+{
+ struct dns_header *header = (struct dns_header *)in;
+ struct rdns_reply *rep;
+ struct rdns_reply_entry *elt;
+ uint8_t *pos, *npos;
+ struct rdns_resolver *resolver = req->resolver;
+ uint16_t qdcount;
+ int type;
+ bool found = false;
+
+ int i, t;
+
+ /* First check header fields */
+ if (header->qr == 0) {
+ rdns_info ("got request while waiting for reply");
+ return false;
+ }
+
+ qdcount = ntohs (header->qdcount);
+
+ if (qdcount != req->qcount) {
+ rdns_info ("request has %d queries, reply has %d queries", (int)req->qcount, (int)header->qdcount);
+ return false;
+ }
+
+ /*
+ * Now we have request and query data is now at the end of header, so compare
+ * request QR section and reply QR section
+ */
+ req->pos = sizeof (struct dns_header);
+ pos = in + sizeof (struct dns_header);
+ t = r - sizeof (struct dns_header);
+ for (i = 0; i < (int)qdcount; i ++) {
+ if ((npos = rdns_request_reply_cmp (req, pos,t)) == NULL) {
+ rdns_info ("DNS request with id %d is for different query, ignoring", (int)req->id);
+ return false;
+ }
+ t -= npos - pos;
+ pos = npos;
+ }
+ /*
+ * Now pos is in answer section, so we should extract data and form reply
+ */
+ rep = rdns_make_reply (req, header->rcode);
+
+ if (header->ad) {
+ rep->flags |= RDNS_AUTH;
+ }
+
+ if (header->tc) {
+ rep->flags |= RDNS_TRUNCATED;
+ }
+
+ if (rep == NULL) {
+ rdns_warn ("Cannot allocate memory for reply");
+ return false;
+ }
+
+ type = req->requested_names[0].type;
+
+ if (rep->code == RDNS_RC_NOERROR) {
+ r -= pos - in;
+ /* Extract RR records */
+ for (i = 0; i < ntohs (header->ancount); i ++) {
+ elt = malloc (sizeof (struct rdns_reply_entry));
+ t = rdns_parse_rr (resolver, in, elt, &pos, rep, &r);
+ if (t == -1) {
+ free (elt);
+ rdns_debug ("incomplete reply");
+ break;
+ }
+ else if (t == 1) {
+ DL_APPEND (rep->entries, elt);
+ if (elt->type == type) {
+ found = true;
+ }
+ }
+ else {
+ rdns_debug ("no matching reply for %s",
+ req->requested_names[0].name);
+ free (elt);
+ }
+ }
+ }
+
+ if (!found && type != RDNS_REQUEST_ANY) {
+ /* We have not found the requested RR type */
+ if (rep->code == RDNS_RC_NOERROR) {
+ rep->code = RDNS_RC_NOREC;
+ }
+ }
+
+ *_rep = rep;
+ return true;
+}
+
+static bool
+rdns_tcp_maybe_realloc_read_buf (struct rdns_io_channel *ioc)
+{
+ if (ioc->tcp->read_buf_allocated == 0 && ioc->tcp->next_read_size > 0) {
+ ioc->tcp->cur_read_buf = malloc(ioc->tcp->next_read_size);
+
+ if (ioc->tcp->cur_read_buf == NULL) {
+ return false;
+ }
+ ioc->tcp->read_buf_allocated = ioc->tcp->next_read_size;
+ }
+ else if (ioc->tcp->read_buf_allocated < ioc->tcp->next_read_size) {
+ /* Need to realloc */
+ unsigned next_shift = ioc->tcp->next_read_size;
+
+ if (next_shift < ioc->tcp->read_buf_allocated * 2) {
+ if (next_shift < UINT16_MAX && ioc->tcp->read_buf_allocated * 2 <= UINT16_MAX) {
+ next_shift = ioc->tcp->read_buf_allocated * 2;
+ }
+ }
+ void *next_buf = realloc(ioc->tcp->cur_read_buf, next_shift);
+
+ if (next_buf == NULL) {
+ free (ioc->tcp->cur_read_buf);
+ ioc->tcp->cur_read_buf = NULL;
+ return false;
+ }
+
+ ioc->tcp->cur_read_buf = next_buf;
+ }
+
+ return true;
+}
+
+static void
+rdns_process_tcp_read (int fd, struct rdns_io_channel *ioc)
+{
+ ssize_t r;
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ if (ioc->tcp->cur_read == 0) {
+ /* We have to read size first */
+ r = read(fd, &ioc->tcp->next_read_size, sizeof(ioc->tcp->next_read_size));
+
+ if (r == -1 || r == 0) {
+ goto err;
+ }
+
+ ioc->tcp->cur_read += r;
+
+ if (r == sizeof(ioc->tcp->next_read_size)) {
+ ioc->tcp->next_read_size = ntohs(ioc->tcp->next_read_size);
+
+ /* We have read the size, so we can try read one more time */
+ if (!rdns_tcp_maybe_realloc_read_buf(ioc)) {
+ rdns_err("failed to allocate %d bytes: %s",
+ (int)ioc->tcp->next_read_size, strerror(errno));
+ r = -1;
+ goto err;
+ }
+ }
+ else {
+ /* We have read one byte, need to retry... */
+ return;
+ }
+ }
+ else if (ioc->tcp->cur_read == 1) {
+ r = read(fd, ((unsigned char *)&ioc->tcp->next_read_size) + 1, 1);
+
+ if (r == -1 || r == 0) {
+ goto err;
+ }
+
+ ioc->tcp->cur_read += r;
+ ioc->tcp->next_read_size = ntohs(ioc->tcp->next_read_size);
+
+ /* We have read the size, so we can try read one more time */
+ if (!rdns_tcp_maybe_realloc_read_buf(ioc)) {
+ rdns_err("failed to allocate %d bytes: %s",
+ (int)ioc->tcp->next_read_size, strerror(errno));
+ r = -1;
+ goto err;
+ }
+ }
+
+ if (ioc->tcp->next_read_size < sizeof(struct dns_header)) {
+ /* Truncated reply, reset channel */
+ rdns_err("got truncated size: %d on TCP read", ioc->tcp->next_read_size);
+ r = -1;
+ errno = EINVAL;
+ goto err;
+ }
+
+ /* Try to read the full packet if we can */
+ int to_read = ioc->tcp->next_read_size - (ioc->tcp->cur_read - 2);
+
+ if (to_read <= 0) {
+ /* Internal error */
+ rdns_err("internal buffer error on reading!");
+ r = -1;
+ errno = EINVAL;
+ goto err;
+ }
+
+ r = read(fd, ioc->tcp->cur_read_buf + (ioc->tcp->cur_read - 2), to_read);
+ ioc->tcp->cur_read += r;
+
+ if ((ioc->tcp->cur_read - 2) == ioc->tcp->next_read_size) {
+ /* We have a full packet ready, process it */
+ struct rdns_request *req = rdns_find_dns_request (ioc->tcp->cur_read_buf, ioc);
+
+ if (req != NULL) {
+ struct rdns_reply *rep;
+
+ if (rdns_parse_reply (ioc->tcp->cur_read_buf,
+ ioc->tcp->next_read_size, req, &rep)) {
+ UPSTREAM_OK (req->io->srv);
+
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->ok (req->io->srv->ups_elt,
+ req->resolver->ups->data);
+ }
+
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ }
+ }
+ else {
+ rdns_warn("unwanted DNS id received over TCP");
+ }
+
+ ioc->tcp->next_read_size = 0;
+ ioc->tcp->cur_read = 0;
+
+ /* Retry read the next packet to avoid unnecessary polling */
+ rdns_process_tcp_read (fd, ioc);
+ }
+
+ return;
+
+err:
+ if (r == 0) {
+ /* Got EOF, just close the socket */
+ rdns_debug ("closing TCP channel due to EOF");
+ rdns_ioc_tcp_reset (ioc);
+ }
+ else if (errno == EINTR || errno == EAGAIN) {
+ /* We just retry later as there is no real error */
+ return;
+ }
+ else {
+ rdns_debug ("closing TCP channel due to IO error: %s", strerror(errno));
+ rdns_ioc_tcp_reset (ioc);
+ }
+}
+
+static void
+rdns_process_tcp_connect (int fd, struct rdns_io_channel *ioc)
+{
+ ioc->flags |= RDNS_CHANNEL_CONNECTED|RDNS_CHANNEL_ACTIVE;
+ ioc->flags &= ~RDNS_CHANNEL_TCP_CONNECTING;
+
+ if (ioc->tcp->async_read == NULL) {
+ ioc->tcp->async_read = ioc->resolver->async->add_read(ioc->resolver->async->data,
+ ioc->sock, ioc);
+ }
+}
+
+static bool
+rdns_reschedule_req_over_tcp (struct rdns_request *req, struct rdns_server *serv)
+{
+ struct rdns_resolver *resolver;
+ struct rdns_io_channel *old_ioc = req->io,
+ *ioc = serv->tcp_io_channels[ottery_rand_uint32 () % serv->tcp_io_cnt];
+
+ resolver = req->resolver;
+
+ if (ioc != NULL) {
+ if (!IS_CHANNEL_CONNECTED(ioc)) {
+ if (!rdns_ioc_tcp_connect(ioc)) {
+ return false;
+ }
+ }
+
+ struct rdns_tcp_output_chain *oc;
+
+ oc = calloc(1, sizeof(*oc) + req->packet_len);
+
+ if (oc == NULL) {
+ rdns_err("failed to allocate output buffer for TCP ioc: %s",
+ strerror(errno));
+ return false;
+ }
+
+ oc->write_buf = ((unsigned char *)oc) + sizeof(*oc);
+ memcpy(oc->write_buf, req->packet, req->packet_len);
+ oc->next_write_size = htons(req->packet_len);
+
+ DL_APPEND(ioc->tcp->output_chain, oc);
+
+ if (ioc->tcp->async_write == NULL) {
+ ioc->tcp->async_write = resolver->async->add_write (
+ resolver->async->data,
+ ioc->sock, ioc);
+ }
+
+ req->state = RDNS_REQUEST_TCP;
+ /* Switch IO channel from UDP to TCP */
+ rdns_request_remove_from_hash (req);
+ req->io = ioc;
+
+ khiter_t k;
+ for (;;) {
+ int pr;
+ k = kh_put(rdns_requests_hash, ioc->requests, req->id, &pr);
+
+ if (pr == 0) {
+ /* We have already a request with this id, so we have to regenerate ID */
+ req->id = rdns_permutor_generate_id ();
+ /* Update packet as well */
+ uint16_t raw_id = req->id;
+ memcpy(req->packet, &raw_id, sizeof(raw_id));
+ }
+ else {
+ break;
+ }
+ }
+
+ req->async_event = resolver->async->add_timer (resolver->async->data,
+ req->timeout, req);
+
+ kh_value(req->io->requests, k) = req;
+ REF_RELEASE(old_ioc);
+ REF_RETAIN(ioc);
+
+ return true;
+ }
+
+ return false;
+}
+
+static void
+rdns_process_udp_read (int fd, struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver;
+ struct rdns_request *req = NULL;
+ ssize_t r;
+ struct rdns_reply *rep;
+ uint8_t in[UDP_PACKET_SIZE];
+
+ resolver = ioc->resolver;
+
+ /* First read packet from socket */
+ if (resolver->curve_plugin == NULL) {
+ r = recv (fd, in, sizeof (in), 0);
+ if (r > (int)(sizeof (struct dns_header) + sizeof (struct dns_query))) {
+ req = rdns_find_dns_request (in, ioc);
+ }
+ }
+ else {
+ r = resolver->curve_plugin->cb.curve_plugin.recv_cb (ioc, in,
+ sizeof (in), resolver->curve_plugin->data, &req,
+ ioc->saddr, ioc->slen);
+ if (req == NULL &&
+ r > (int)(sizeof (struct dns_header) + sizeof (struct dns_query))) {
+ req = rdns_find_dns_request (in, ioc);
+ }
+ }
+
+ if (req != NULL) {
+ if (rdns_parse_reply (in, r, req, &rep)) {
+ UPSTREAM_OK (req->io->srv);
+
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->ok (req->io->srv->ups_elt,
+ req->resolver->ups->data);
+ }
+
+ rdns_request_unschedule (req, true);
+
+ if (!(rep->flags & RDNS_TRUNCATED)) {
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func(rep, req->arg);
+ /* This will free reply as well */
+ REF_RELEASE (req);
+ }
+ else {
+ if (req->io->srv->tcp_io_cnt > 0) {
+ rdns_debug("truncated UDP reply for %s; schedule over TCP", req->requested_names[0].name);
+ /* Reschedule via TCP */
+ if (!rdns_reschedule_req_over_tcp (req, req->io->srv)) {
+ /* Use truncated reply as we have no other options */
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func(rep, req->arg);
+ REF_RELEASE (req);
+ }
+ else {
+ /* Remove and free the truncated reply, as we have rescheduled the reply */
+ req->reply = NULL;
+ rdns_reply_free(rep);
+ }
+ }
+ else {
+ /* No TCP channels available */
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func(rep, req->arg);
+ /* This will free reply as well */
+ REF_RELEASE (req);
+ }
+ }
+ }
+ }
+ else {
+ /* Still want to increase uses */
+ ioc->uses ++;
+ }
+}
+
+void
+rdns_process_read (int fd, void *arg)
+{
+ struct rdns_io_channel *ioc = (struct rdns_io_channel *)arg;
+ struct rdns_resolver *resolver;
+
+ resolver = ioc->resolver;
+
+ if (IS_CHANNEL_TCP(ioc)) {
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ rdns_process_tcp_read (fd, ioc);
+ }
+ else {
+ rdns_err ("read readiness on non connected TCP channel!");
+ }
+ }
+ else {
+ rdns_process_udp_read (fd, ioc);
+ }
+}
+
+void
+rdns_process_timer (void *arg)
+{
+ struct rdns_request *req = (struct rdns_request *)arg;
+ struct rdns_reply *rep;
+ int r;
+ bool renew = false;
+ struct rdns_resolver *resolver;
+ struct rdns_server *serv = NULL;
+ unsigned cnt;
+
+ req->retransmits --;
+ resolver = req->resolver;
+
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->fail (req->io->srv->ups_elt,
+ req->resolver->ups->data, "timeout waiting reply");
+ }
+ else {
+ UPSTREAM_FAIL (req->io->srv, time (NULL));
+ }
+
+ if (req->state == RDNS_REQUEST_TCP) {
+ rep = rdns_make_reply (req, RDNS_RC_TIMEOUT);
+ rdns_request_unschedule (req, true);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ if (req->retransmits == 0) {
+
+ rep = rdns_make_reply (req, RDNS_RC_TIMEOUT);
+ rdns_request_unschedule (req, true);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ if (!IS_CHANNEL_ACTIVE(req->io) || req->retransmits == 1) {
+
+ if (resolver->ups) {
+ cnt = resolver->ups->count (resolver->ups->data);
+ }
+ else {
+ cnt = 0;
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ cnt ++;
+ }
+ }
+
+ if (!IS_CHANNEL_ACTIVE(req->io) || cnt > 1) {
+ /* Do not reschedule IO requests on inactive sockets */
+ rdns_debug ("reschedule request with id: %d", (int)req->id);
+ rdns_request_unschedule (req, true);
+ REF_RELEASE (req->io);
+
+ if (resolver->ups) {
+ struct rdns_upstream_elt *elt;
+
+ elt = resolver->ups->select_retransmit (
+ req->requested_names[0].name,
+ req->requested_names[0].len,
+ req->io->srv->ups_elt,
+ resolver->ups->data);
+
+ if (elt) {
+ serv = elt->server;
+ serv->ups_elt = elt;
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+
+ if (serv == NULL) {
+ rdns_warn ("cannot find suitable server for request");
+ rep = rdns_make_reply (req, RDNS_RC_SERVFAIL);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ /* Select random IO channel */
+ req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt];
+ req->io->uses ++;
+ REF_RETAIN (req->io);
+ renew = true;
+ }
+ }
+
+ /*
+ * Note: when `renew` is true, then send_request deals with the
+ * timers and events itself
+ */
+ r = rdns_send_request (req, req->io->sock, renew);
+ if (r == 0) {
+ /* Retransmit one more time */
+ if (!renew) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ req->async_event = req->async->add_write (req->async->data,
+ req->io->sock, req);
+ }
+
+ req->state = RDNS_REQUEST_WAIT_SEND;
+ }
+ else if (r == -1) {
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->fail (req->io->srv->ups_elt,
+ req->resolver->ups->data, "cannot send retransmit after timeout");
+ }
+ else {
+ UPSTREAM_FAIL (req->io->srv, time (NULL));
+ }
+
+ if (!renew) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ req->async_event = NULL;
+ rdns_request_remove_from_hash(req);
+ }
+
+ /* We have not scheduled timeout actually due to send error */
+ rep = rdns_make_reply (req, RDNS_RC_NETERR);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ }
+ else {
+ req->async->repeat_timer (req->async->data, req->async_event);
+ req->state = RDNS_REQUEST_WAIT_REPLY;
+ }
+}
+
+static void
+rdns_process_periodic (void *arg)
+{
+ struct rdns_resolver *resolver = (struct rdns_resolver*)arg;
+ struct rdns_server *serv;
+
+ UPSTREAM_RESCAN (resolver->servers, time (NULL));
+
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ for (int i = 0; i < serv->tcp_io_cnt; i ++) {
+ if (IS_CHANNEL_CONNECTED(serv->tcp_io_channels[i])) {
+ /* Disconnect channels with no requests in flight */
+ if (kh_size(serv->tcp_io_channels[i]->requests) == 0) {
+ rdns_debug ("reset inactive TCP connection to %s", serv->name);
+ rdns_ioc_tcp_reset (serv->tcp_io_channels[i]);
+ }
+ }
+ }
+ }
+}
+
+static void
+rdns_process_ioc_refresh (void *arg)
+{
+ struct rdns_resolver *resolver = (struct rdns_resolver*)arg;
+ struct rdns_server *serv;
+ struct rdns_io_channel *ioc, *nioc;
+ unsigned int i;
+
+ if (resolver->max_ioc_uses > 0) {
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ for (i = 0; i < serv->io_cnt; i ++) {
+ ioc = serv->io_channels[i];
+ if (ioc->uses > resolver->max_ioc_uses) {
+ /* Schedule IOC removing */
+ nioc = rdns_ioc_new (serv, resolver, false);
+
+ if (nioc == NULL) {
+ rdns_err ("calloc fails to allocate rdns_io_channel");
+ continue;
+ }
+
+ serv->io_channels[i] = nioc;
+ rdns_debug ("scheduled io channel for server %s to be refreshed after "
+ "%lu usages", serv->name, (unsigned long)ioc->uses);
+ ioc->flags &= ~RDNS_CHANNEL_ACTIVE;
+ REF_RELEASE (ioc);
+ }
+ }
+ }
+ }
+}
+
+static void
+rdns_process_udp_retransmit (int fd, struct rdns_request *req)
+{
+ struct rdns_resolver *resolver;
+ struct rdns_reply *rep;
+ int r;
+
+ resolver = req->resolver;
+
+ resolver->async->del_write (resolver->async->data,
+ req->async_event);
+ req->async_event = NULL;
+
+ if (req->state == RDNS_REQUEST_FAKE) {
+ /* Reply is ready */
+ req->func (req->reply, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ r = rdns_send_request (req, fd, false);
+
+ if (r == 0) {
+ /* Retransmit one more time */
+ req->async_event = req->async->add_write (req->async->data,
+ fd, req);
+ req->state = RDNS_REQUEST_WAIT_SEND;
+ }
+ else if (r == -1) {
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->fail (req->io->srv->ups_elt,
+ req->resolver->ups->data, "retransmit send failed");
+ }
+ else {
+ UPSTREAM_FAIL (req->io->srv, time (NULL));
+ }
+
+ rep = rdns_make_reply (req, RDNS_RC_NETERR);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ }
+ else {
+ req->async_event = req->async->add_timer (req->async->data,
+ req->timeout, req);
+ req->state = RDNS_REQUEST_WAIT_REPLY;
+ }
+}
+
+static ssize_t
+rdns_write_output_chain (struct rdns_io_channel *ioc, struct rdns_tcp_output_chain *oc)
+{
+ ssize_t r;
+ struct iovec iov[2];
+ int niov, already_written;
+ int packet_len = ntohs (oc->next_write_size);
+
+ switch (oc->cur_write) {
+ case 0:
+ /* Size + DNS request in full */
+ iov[0].iov_base = &oc->next_write_size;
+ iov[0].iov_len = sizeof (oc->next_write_size);
+ iov[1].iov_base = oc->write_buf;
+ iov[1].iov_len = packet_len;
+ niov = 2;
+ break;
+ case 1:
+ /* Partial Size + DNS request in full */
+ iov[0].iov_base = ((unsigned char *)&oc->next_write_size) + 1;
+ iov[0].iov_len = 1;
+ iov[1].iov_base = oc->write_buf;
+ iov[1].iov_len = packet_len;
+ niov = 2;
+ break;
+ default:
+ /* Merely DNS packet */
+ already_written = oc->cur_write - 2;
+ if (packet_len <= already_written) {
+ errno = EINVAL;
+ return -1;
+ }
+ iov[0].iov_base = oc->write_buf + already_written;
+ iov[0].iov_len = packet_len - already_written;
+ niov = 1;
+ break;
+ }
+
+ r = writev(ioc->sock, iov, niov);
+
+ if (r > 0) {
+ oc->cur_write += r;
+ }
+
+ return r;
+}
+
+static void
+rdns_process_tcp_write (int fd, struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver = ioc->resolver;
+
+
+ /* Try to write as much as we can */
+ struct rdns_tcp_output_chain *oc, *tmp;
+ DL_FOREACH_SAFE(ioc->tcp->output_chain, oc, tmp) {
+ ssize_t r = rdns_write_output_chain (ioc, oc);
+
+ if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ /* Write even is persistent */
+ return;
+ }
+ else {
+ rdns_err ("error when trying to write request to %s: %s",
+ ioc->srv->name, strerror (errno));
+ rdns_ioc_tcp_reset (ioc);
+ return;
+ }
+ }
+ else if (ntohs(oc->next_write_size) < oc->cur_write) {
+ /* Packet has been fully written, remove it */
+ DL_DELETE(ioc->tcp->output_chain, oc);
+ free (oc); /* It also frees write buf */
+ ioc->tcp->cur_output_chains --;
+ }
+ else {
+ /* Buffer is not yet processed, stop unless we can continue */
+ break;
+ }
+ }
+
+ if (ioc->tcp->cur_output_chains == 0) {
+ /* Unregister write event */
+ ioc->resolver->async->del_write (ioc->resolver->async->data,
+ ioc->tcp->async_write);
+ ioc->tcp->async_write = NULL;
+ }
+}
+
+void
+rdns_process_write (int fd, void *arg)
+{
+ /*
+ * We first need to dispatch *arg to understand what has caused the write
+ * readiness event.
+ * The one possibility is that it was a UDP retransmit request, so our
+ * arg will be struct rdns_request *
+ * Another possibility is that write event was triggered by some TCP related
+ * stuff. In this case the only possibility is that our arg is struct rdns_io_channel *
+ * To distinguish these two cases (due to flaws in the rdns architecture in the first
+ * place) we compare the first 8 bytes with RDNS_IO_CHANNEL_TAG
+ */
+ uint64_t tag;
+
+ memcpy (&tag, arg, sizeof(tag));
+
+ if (tag == RDNS_IO_CHANNEL_TAG) {
+ struct rdns_io_channel *ioc = (struct rdns_io_channel *) arg;
+
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ rdns_process_tcp_write(fd, ioc);
+ }
+ else {
+ rdns_process_tcp_connect(fd, ioc);
+ rdns_process_tcp_write(fd, ioc);
+ }
+ }
+ else {
+ struct rdns_request *req = (struct rdns_request *) arg;
+ rdns_process_udp_retransmit(fd, req);
+ }
+}
+
+struct rdns_server *
+rdns_select_request_upstream (struct rdns_resolver *resolver,
+ struct rdns_request *req,
+ bool is_retransmit,
+ struct rdns_server *prev_serv)
+{
+ struct rdns_server *serv = NULL;
+
+ if (resolver->ups) {
+ struct rdns_upstream_elt *elt;
+
+ if (is_retransmit && prev_serv) {
+ elt = resolver->ups->select_retransmit (req->requested_names[0].name,
+ req->requested_names[0].len,
+ prev_serv->ups_elt,
+ resolver->ups->data);
+ }
+ else {
+ elt = resolver->ups->select (req->requested_names[0].name,
+ req->requested_names[0].len, resolver->ups->data);
+ }
+
+ if (elt) {
+ serv = elt->server;
+ serv->ups_elt = elt;
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+
+ return serv;
+}
+
+#define align_ptr(p, a) \
+ (guint8 *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1))
+
+struct rdns_request*
+rdns_make_request_full (
+ struct rdns_resolver *resolver,
+ dns_callback_type cb,
+ void *cbdata,
+ double timeout,
+ unsigned int repeats,
+ unsigned int queries,
+ ...
+ )
+{
+ va_list args;
+ struct rdns_request *req;
+ struct rdns_server *serv;
+ int r, type;
+ unsigned int i, tlen = 0, clen = 0, cur;
+ size_t olen;
+ const char *cur_name, *last_name = NULL;
+ khash_t(rdns_compression_hash) *comp = NULL;
+ struct rdns_fake_reply *fake_rep = NULL;
+ char fake_buf[MAX_FAKE_NAME + sizeof (struct rdns_fake_reply_idx) + 16];
+ struct rdns_fake_reply_idx *idx;
+
+ if (resolver == NULL || !resolver->initialized) {
+ if (resolver == NULL) {
+ return NULL;
+ }
+
+ rdns_err ("resolver is uninitialized");
+
+ return NULL;
+ }
+
+ req = malloc (sizeof (struct rdns_request));
+ if (req == NULL) {
+ rdns_err ("failed to allocate memory for request: %s",
+ strerror (errno));
+ return NULL;
+ }
+
+ req->resolver = resolver;
+ req->func = cb;
+ req->arg = cbdata;
+ req->reply = NULL;
+ req->qcount = queries;
+ req->io = NULL;
+ req->state = RDNS_REQUEST_NEW;
+ req->packet = NULL;
+ req->requested_names = calloc (queries, sizeof (struct rdns_request_name));
+ req->async_event = NULL;
+
+ if (req->requested_names == NULL) {
+ free (req);
+ rdns_err ("failed to allocate memory for request data: %s",
+ strerror (errno));
+
+ return NULL;
+ }
+
+ req->type = 0;
+#ifdef TWEETNACL
+ req->curve_plugin_data = NULL;
+#endif
+ REF_INIT_RETAIN (req, rdns_request_free);
+
+ /* Calculate packet's total length based on records count */
+ va_start (args, queries);
+ for (i = 0; i < queries * 2; i += 2) {
+ cur = i / 2;
+ cur_name = va_arg (args, const char *);
+ type = va_arg (args, int);
+
+ if (cur_name != NULL) {
+ clen = strlen (cur_name);
+
+ if (clen == 0) {
+ rdns_warn ("got empty name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+
+ if (cur_name[0] == '.') {
+ /* Skip dots at the begin */
+ unsigned int ndots = strspn (cur_name, ".");
+
+ cur_name += ndots;
+ clen -= ndots;
+
+ if (clen == 0) {
+ rdns_warn ("got empty name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+ }
+
+ if (cur_name[clen - 1] == '.') {
+ /* Skip trailing dots */
+ while (clen >= 1 && cur_name[clen - 1] == '.') {
+ clen --;
+ }
+
+ if (clen == 0) {
+ rdns_warn ("got empty name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+ }
+
+ if (last_name == NULL && queries == 1 && clen < MAX_FAKE_NAME) {
+ /* We allocate structure in the static space */
+ idx = (struct rdns_fake_reply_idx *)align_ptr (fake_buf, 16);
+ idx->type = type;
+ idx->len = clen;
+ memcpy (idx->request, cur_name, clen);
+ HASH_FIND (hh, resolver->fake_elts, idx, sizeof (*idx) + clen,
+ fake_rep);
+
+ if (fake_rep) {
+ /* We actually treat it as a short-circuit */
+ req->reply = rdns_make_reply (req, fake_rep->rcode);
+ req->reply->entries = fake_rep->result;
+ req->state = RDNS_REQUEST_FAKE;
+ }
+ }
+
+ last_name = cur_name;
+ tlen += clen;
+ }
+ else if (last_name == NULL) {
+ rdns_err ("got NULL as the first name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+
+ if (req->state != RDNS_REQUEST_FAKE) {
+ if (!rdns_format_dns_name (resolver, last_name, clen,
+ &req->requested_names[cur].name, &olen)) {
+ rdns_err ("cannot format %s", last_name);
+ rdns_request_free (req);
+ return NULL;
+ }
+
+ req->requested_names[cur].len = olen;
+ }
+ else {
+ req->requested_names[cur].len = clen;
+ }
+
+ req->requested_names[cur].type = type;
+ }
+
+ va_end (args);
+
+ if (req->state != RDNS_REQUEST_FAKE) {
+ rdns_allocate_packet (req, tlen);
+ rdns_make_dns_header (req, queries);
+
+ for (i = 0; i < queries; i++) {
+ cur_name = req->requested_names[i].name;
+ clen = req->requested_names[i].len;
+ type = req->requested_names[i].type;
+ if (queries > 1) {
+ if (!rdns_add_rr (req, cur_name, clen, type, &comp)) {
+ rdns_err ("cannot add rr");
+ REF_RELEASE (req);
+ rdns_compression_free(comp);
+ return NULL;
+ }
+ } else {
+ if (!rdns_add_rr (req, cur_name, clen, type, NULL)) {
+ rdns_err ("cannot add rr");
+ REF_RELEASE (req);
+ rdns_compression_free(comp);
+ return NULL;
+ }
+ }
+ }
+
+ rdns_compression_free(comp);
+
+ /* Add EDNS RR */
+ rdns_add_edns0 (req);
+
+ req->retransmits = repeats ? repeats : 1;
+ req->timeout = timeout;
+ req->state = RDNS_REQUEST_NEW;
+ }
+
+ req->async = resolver->async;
+
+ serv = rdns_select_request_upstream (resolver, req, false, NULL);
+
+ if (serv == NULL) {
+ rdns_warn ("cannot find suitable server for request");
+ REF_RELEASE (req);
+ return NULL;
+ }
+
+ /* Select random IO channel */
+ req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt];
+
+ if (req->state == RDNS_REQUEST_FAKE) {
+ req->async_event = resolver->async->add_write (resolver->async->data,
+ req->io->sock, req);
+ }
+ else {
+ /* Now send request to server */
+ do {
+ r = rdns_send_request (req, req->io->sock, true);
+
+ if (r == -1) {
+ req->retransmits --; /* It must be > 0 */
+
+ if (req->retransmits > 0) {
+ if (resolver->ups && serv->ups_elt) {
+ resolver->ups->fail (serv->ups_elt, resolver->ups->data,
+ "send IO error");
+ }
+ else {
+ UPSTREAM_FAIL (serv, time (NULL));
+ }
+
+ serv = rdns_select_request_upstream (resolver, req,
+ true, serv);
+
+ if (serv == NULL) {
+ rdns_warn ("cannot find suitable server for request");
+ REF_RELEASE (req);
+ return NULL;
+ }
+
+ req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt];
+ }
+ else {
+ rdns_info ("cannot send DNS request: %s", strerror (errno));
+ REF_RELEASE (req);
+
+ if (resolver->ups && serv->ups_elt) {
+ resolver->ups->fail (serv->ups_elt, resolver->ups->data,
+ "send IO error");
+ }
+ else {
+ UPSTREAM_FAIL (serv, time (NULL));
+ }
+
+ return NULL;
+ }
+ }
+ else {
+ /* All good */
+ req->io->uses++;
+ break;
+ }
+ } while (req->retransmits > 0);
+ }
+
+ REF_RETAIN (req->io);
+ REF_RETAIN (req->resolver);
+
+ return req;
+}
+
+bool
+rdns_resolver_init (struct rdns_resolver *resolver)
+{
+ unsigned int i;
+ struct rdns_server *serv;
+ struct rdns_io_channel *ioc;
+
+ if (!resolver->async_binded) {
+ rdns_err ("no async backend specified");
+ return false;
+ }
+
+ if (resolver->servers == NULL) {
+ rdns_err ("no DNS servers defined");
+ return false;
+ }
+
+ /* Now init io channels to all servers */
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ serv->io_channels = calloc (serv->io_cnt, sizeof (struct rdns_io_channel *));
+
+ if (serv->io_channels == NULL) {
+ rdns_err ("cannot allocate memory for the resolver IO channels");
+ return false;
+ }
+
+ for (i = 0; i < serv->io_cnt; i ++) {
+ ioc = rdns_ioc_new(serv, resolver, false);
+
+ if (ioc == NULL) {
+ rdns_err ("cannot allocate memory or init the IO channel");
+ return false;
+ }
+
+ serv->io_channels[i] = ioc;
+ }
+
+ int ntcp_channels = 0;
+
+ /*
+ * We are more forgiving for TCP IO channels: we can have zero of them
+ * if DNS is misconfigured and still be able to resolve stuff
+ */
+ serv->tcp_io_channels = calloc (serv->tcp_io_cnt, sizeof (struct rdns_io_channel *));
+ if (serv->tcp_io_channels == NULL) {
+ rdns_err ("cannot allocate memory for the resolver TCP IO channels");
+ return false;
+ }
+ for (i = 0; i < serv->tcp_io_cnt; i ++) {
+ ioc = rdns_ioc_new (serv, resolver, true);
+
+ if (ioc == NULL) {
+ rdns_err ("cannot allocate memory or init the TCP IO channel");
+ continue;
+ }
+
+ serv->tcp_io_channels[ntcp_channels++] = ioc;
+ }
+
+ serv->tcp_io_cnt = ntcp_channels;
+ }
+
+ if (resolver->async->add_periodic) {
+ resolver->periodic = resolver->async->add_periodic (resolver->async->data,
+ UPSTREAM_REVIVE_TIME, rdns_process_periodic, resolver);
+ }
+
+ resolver->initialized = true;
+
+ return true;
+}
+
+void
+rdns_resolver_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_plugin *plugin)
+{
+ if (resolver != NULL && plugin != NULL) {
+ /* XXX: support only network plugin now, and only a single one */
+ if (plugin->type == RDNS_PLUGIN_CURVE) {
+ resolver->curve_plugin = plugin;
+ }
+ }
+}
+
+void *
+rdns_resolver_add_server (struct rdns_resolver *resolver,
+ const char *name, unsigned int port,
+ int priority, unsigned int io_cnt)
+{
+ struct rdns_server *serv;
+ union {
+ struct in_addr v4;
+ struct in6_addr v6;
+ } addr;
+
+ if (inet_pton (AF_INET, name, &addr) == 0 &&
+ inet_pton (AF_INET6, name, &addr) == 0) {
+ /* Invalid IP */
+ return NULL;
+ }
+
+ if (io_cnt == 0) {
+ return NULL;
+ }
+ if (port == 0 || port > UINT16_MAX) {
+ return NULL;
+ }
+
+ serv = calloc (1, sizeof (struct rdns_server));
+ if (serv == NULL) {
+ return NULL;
+ }
+ serv->name = strdup (name);
+ if (serv->name == NULL) {
+ free (serv);
+ return NULL;
+ }
+
+ serv->io_cnt = io_cnt;
+ /* TODO: make it configurable maybe? */
+ serv->tcp_io_cnt = default_tcp_io_cnt;
+ serv->port = port;
+
+ UPSTREAM_ADD (resolver->servers, serv, priority);
+
+ return serv;
+}
+
+void
+rdns_resolver_set_logger (struct rdns_resolver *resolver,
+ rdns_log_function logger, void *log_data)
+{
+ resolver->logger = logger;
+ resolver->log_data = log_data;
+}
+
+void
+rdns_resolver_set_log_level (struct rdns_resolver *resolver,
+ enum rdns_log_level level)
+{
+ resolver->log_level = level;
+}
+
+void
+rdns_resolver_set_upstream_lib (struct rdns_resolver *resolver,
+ struct rdns_upstream_context *ups_ctx,
+ void *ups_data)
+{
+ resolver->ups = ups_ctx;
+ resolver->ups->data = ups_data;
+}
+
+
+void
+rdns_resolver_set_max_io_uses (struct rdns_resolver *resolver,
+ uint64_t max_ioc_uses, double check_time)
+{
+ if (resolver->refresh_ioc_periodic != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ resolver->refresh_ioc_periodic);
+ resolver->refresh_ioc_periodic = NULL;
+ }
+
+ resolver->max_ioc_uses = max_ioc_uses;
+ if (check_time > 0.0 && resolver->async->add_periodic) {
+ resolver->refresh_ioc_periodic =
+ resolver->async->add_periodic (resolver->async->data,
+ check_time, rdns_process_ioc_refresh, resolver);
+ }
+}
+
+static void
+rdns_resolver_free (struct rdns_resolver *resolver)
+{
+ struct rdns_server *serv, *stmp;
+ struct rdns_io_channel *ioc;
+ unsigned int i;
+
+ if (resolver->initialized) {
+ if (resolver->periodic != NULL) {
+ resolver->async->del_periodic (resolver->async->data, resolver->periodic);
+ }
+ if (resolver->refresh_ioc_periodic != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ resolver->refresh_ioc_periodic);
+ }
+ if (resolver->curve_plugin != NULL && resolver->curve_plugin->dtor != NULL) {
+ resolver->curve_plugin->dtor (resolver, resolver->curve_plugin->data);
+ }
+ /* Stop IO watch on all IO channels */
+ UPSTREAM_FOREACH_SAFE (resolver->servers, serv, stmp) {
+ for (i = 0; i < serv->io_cnt; i ++) {
+ ioc = serv->io_channels[i];
+ REF_RELEASE (ioc);
+ }
+ for (i = 0; i < serv->tcp_io_cnt; i ++) {
+ ioc = serv->tcp_io_channels[i];
+ REF_RELEASE (ioc);
+ }
+ UPSTREAM_DEL (resolver->servers, serv);
+ free (serv->io_channels);
+ free (serv->tcp_io_channels);
+ free (serv->name);
+ free (serv);
+ }
+ }
+ free (resolver->async);
+ free (resolver);
+}
+
+
+struct rdns_resolver *
+rdns_resolver_new (int flags)
+{
+ struct rdns_resolver *new_resolver;
+
+ new_resolver = calloc (1, sizeof (struct rdns_resolver));
+
+ REF_INIT_RETAIN (new_resolver, rdns_resolver_free);
+
+ new_resolver->logger = rdns_logger_internal;
+ new_resolver->log_data = new_resolver;
+ new_resolver->flags = flags;
+
+ return new_resolver;
+}
+
+void
+rdns_resolver_async_bind (struct rdns_resolver *resolver,
+ struct rdns_async_context *ctx)
+{
+ if (resolver != NULL && ctx != NULL) {
+ resolver->async = ctx;
+ resolver->async_binded = true;
+ }
+}
+
+void
+rdns_resolver_set_dnssec (struct rdns_resolver *resolver, bool enabled)
+{
+ if (resolver) {
+ resolver->enable_dnssec = enabled;
+ }
+}
+
+
+void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver,
+ const char *name,
+ enum rdns_request_type type,
+ enum dns_rcode rcode,
+ struct rdns_reply_entry *reply)
+{
+ struct rdns_fake_reply *fake_rep;
+ struct rdns_fake_reply_idx *srch;
+ unsigned len = strlen (name);
+
+ assert (len < MAX_FAKE_NAME);
+ srch = malloc (sizeof (*srch) + len);
+ srch->len = len;
+ srch->type = type;
+ memcpy (srch->request, name, len);
+
+ HASH_FIND (hh, resolver->fake_elts, srch, len + sizeof (*srch), fake_rep);
+
+ if (fake_rep) {
+ /* Append reply to the existing list */
+ fake_rep->rcode = rcode;
+
+ if (reply) {
+ DL_CONCAT (fake_rep->result, reply);
+ }
+ }
+ else {
+ fake_rep = calloc (1, sizeof (*fake_rep) + len);
+
+ if (fake_rep == NULL) {
+ abort ();
+ }
+
+ fake_rep->rcode = rcode;
+
+ memcpy (&fake_rep->key, srch, sizeof (*srch) + len);
+
+ if (reply) {
+ DL_CONCAT (fake_rep->result, reply);
+ }
+
+ HASH_ADD (hh, resolver->fake_elts, key, sizeof (*srch) + len, fake_rep);
+ }
+
+ free (srch);
+}
diff --git a/contrib/librdns/upstream.h b/contrib/librdns/upstream.h
new file mode 100644
index 0000000..a384155
--- /dev/null
+++ b/contrib/librdns/upstream.h
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UPSTREAM_H_
+#define UPSTREAM_H_
+
+#include <time.h>
+#include <stdio.h>
+
+/**
+ * @file upstream.h
+ * The basic macros to define upstream objects
+ */
+
+#ifndef upstream_fatal
+#define upstream_fatal(msg) \
+ do { \
+ perror(msg); \
+ exit(-1); \
+ } while (0)
+#endif
+
+#ifndef upstream_malloc
+#define upstream_malloc(size) malloc(size)
+#endif
+
+#ifndef upstream_free
+#define upstream_free(size, ptr) free(ptr)
+#endif
+
+struct upstream_entry_s;
+struct upstream_common_data {
+ void **upstreams;
+ unsigned int allocated_nelts;
+ unsigned int nelts;
+ unsigned int alive;
+};
+
+typedef struct upstream_entry_s {
+ unsigned short errors; /**< errors for this upstream */
+ unsigned short dead;
+ unsigned short priority;
+ unsigned short weight;
+ time_t time; /**< time of marking */
+ void *parent; /**< parent object */
+ struct upstream_common_data *common; /**< common data */
+ void *next; /**< link to the next */
+} upstream_entry_t;
+
+/*
+ * Here we define some reasonable defaults:
+ * if an upstream has more than `UPSTREAM_MAX_ERRORS` in the period of time
+ * of `UPSTREAM_ERROR_TIME` then we shut it down for `UPSTREAM_REVIVE_TIME`.
+ * In this particular case times are 10 seconds for 10 errors and revive in
+ * 30 seconds.
+ */
+#ifndef UPSTREAM_REVIVE_TIME
+#define UPSTREAM_REVIVE_TIME 30
+#endif
+#ifndef UPSTREAM_ERROR_TIME
+#define UPSTREAM_ERROR_TIME 10
+#endif
+#ifndef UPSTREAM_MAX_ERRORS
+#define UPSTREAM_MAX_ERRORS 10
+#endif
+
+#define UPSTREAM_FAIL(u, now) \
+ do { \
+ if ((u)->up.time != 0) { \
+ if ((now) - (u)->up.time >= UPSTREAM_ERROR_TIME) { \
+ if ((u)->up.errors >= UPSTREAM_MAX_ERRORS) { \
+ (u)->up.dead = 1; \
+ (u)->up.time = now; \
+ (u)->up.common->alive--; \
+ } \
+ else { \
+ (u)->up.errors = 1; \
+ (u)->up.time = (now); \
+ } \
+ } \
+ else { \
+ (u)->up.errors++; \
+ } \
+ } \
+ else { \
+ (u)->up.errors++; \
+ (u)->up.time = (now); \
+ } \
+ } while (0)
+
+#define UPSTREAM_OK(u) \
+ do { \
+ (u)->up.errors = 0; \
+ (u)->up.time = 0; \
+ } while (0)
+
+#define UPSTREAM_ADD(head, u, priority) \
+ do { \
+ if (head == NULL) { \
+ struct upstream_common_data *cd; \
+ cd = upstream_malloc(sizeof(struct upstream_common_data)); \
+ if (cd == NULL) { \
+ upstream_fatal("malloc failed"); \
+ } \
+ cd->upstreams = upstream_malloc(sizeof(void *) * 8); \
+ if (cd == NULL) { \
+ upstream_fatal("malloc failed"); \
+ } \
+ cd->allocated_nelts = 8; \
+ cd->nelts = 1; \
+ cd->alive = 1; \
+ cd->upstreams[0] = (u); \
+ (u)->up.common = cd; \
+ } \
+ else { \
+ struct upstream_common_data *cd = (head)->up.common; \
+ (u)->up.common = cd; \
+ if (cd->nelts == cd->allocated_nelts) { \
+ void **nup; \
+ nup = upstream_malloc(sizeof(void *) * cd->nelts * 2); \
+ if (nup == NULL) { \
+ upstream_fatal("malloc failed"); \
+ } \
+ memcpy(nup, cd->upstreams, cd->nelts * sizeof(void *)); \
+ upstream_free(cd->nelts * sizeof(void *), cd->upstreams); \
+ cd->upstreams = nup; \
+ cd->allocated_nelts *= 2; \
+ } \
+ cd->upstreams[cd->nelts++] = (u); \
+ cd->alive++; \
+ } \
+ (u)->up.next = (head); \
+ (head) = (u); \
+ if (priority > 0) { \
+ (u)->up.priority = (u)->up.weight = (priority); \
+ } \
+ else { \
+ (u)->up.priority = (u)->up.weight = 65535; \
+ } \
+ (u)->up.time = 0; \
+ (u)->up.errors = 0; \
+ (u)->up.dead = 0; \
+ (u)->up.parent = (u); \
+ } while (0)
+
+#define UPSTREAM_DEL(head, u) \
+ do { \
+ if (head != NULL) { \
+ struct upstream_common_data *cd = (head)->up.common; \
+ if ((u)->up.next != NULL) { \
+ (head) = (u)->up.next; \
+ cd->nelts--; \
+ cd->alive--; \
+ } \
+ else { \
+ upstream_free(cd->allocated_nelts * sizeof(void *), \
+ cd->upstreams); \
+ upstream_free(sizeof(struct upstream_common_data), cd); \
+ (head) = NULL; \
+ } \
+ } \
+ } while (0)
+
+#define UPSTREAM_FOREACH(head, u) for ((u) = (head); (u) != NULL; (u) = (u)->up.next)
+#define UPSTREAM_FOREACH_SAFE(head, u, tmp) \
+ for ((u) = (head); \
+ (u) != NULL && ((tmp = (u)->up.next) || true); \
+ (u) = (tmp))
+
+#define UPSTREAM_REVIVE_ALL(head) \
+ do { \
+ __typeof(head) elt = (head); \
+ while (elt != NULL) { \
+ elt->up.dead = 0; \
+ elt->up.errors = 0; \
+ elt->up.time = 0; \
+ elt = elt->up.next; \
+ } \
+ (head)->up.common->alive = (head)->up.common->nelts; \
+ } while (0)
+
+#define UPSTREAM_RESCAN(head, now) \
+ do { \
+ __typeof(head) elt = (head); \
+ if ((head)->up.common->alive == 0) { \
+ UPSTREAM_REVIVE_ALL((head)); \
+ } \
+ else { \
+ while (elt != NULL) { \
+ if (elt->up.dead) { \
+ if ((now) -elt->up.time >= UPSTREAM_REVIVE_TIME) { \
+ elt->up.dead = 0; \
+ elt->up.errors = 0; \
+ elt->up.weight = elt->up.priority; \
+ (head)->up.common->alive++; \
+ } \
+ } \
+ else { \
+ if ((now) -elt->up.time >= UPSTREAM_ERROR_TIME && \
+ elt->up.errors >= UPSTREAM_MAX_ERRORS) { \
+ elt->up.dead = 1; \
+ elt->up.time = now; \
+ (head)->up.common->alive--; \
+ } \
+ } \
+ elt = elt->up.next; \
+ } \
+ } \
+ } while (0)
+
+#define UPSTREAM_SELECT_ROUND_ROBIN(head, selected) \
+ do { \
+ __typeof(head) elt = (head); \
+ (selected) = NULL; \
+ unsigned max_weight = 0; \
+ if ((head)->up.common->alive == 0) { \
+ UPSTREAM_REVIVE_ALL(head); \
+ } \
+ while (elt != NULL) { \
+ if (!elt->up.dead) { \
+ if (elt->up.weight > max_weight) { \
+ max_weight = elt->up.weight; \
+ (selected) = elt; \
+ } \
+ } \
+ elt = elt->up.next; \
+ } \
+ if (max_weight == 0) { \
+ elt = (head); \
+ while (elt != NULL) { \
+ elt->up.weight = elt->up.priority; \
+ if (!elt->up.dead) { \
+ if (elt->up.priority > max_weight) { \
+ max_weight = elt->up.priority; \
+ (selected) = elt; \
+ } \
+ } \
+ elt = elt->up.next; \
+ } \
+ } \
+ (selected)->up.weight--; \
+ } while (0)
+
+#endif /* UPSTREAM_H_ */
diff --git a/contrib/librdns/util.c b/contrib/librdns/util.c
new file mode 100644
index 0000000..0172ce0
--- /dev/null
+++ b/contrib/librdns/util.c
@@ -0,0 +1,1035 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/un.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <netdb.h>
+#include <fcntl.h>
+#include <ctype.h>
+
+#include "ottery.h"
+#include "util.h"
+#include "logger.h"
+#include "rdns.h"
+
+inline void
+rdns_request_remove_from_hash (struct rdns_request *req)
+{
+ /* Remove from id hashes */
+ if (req->io) {
+ khiter_t k;
+
+ k = kh_get(rdns_requests_hash, req->io->requests, req->id);
+
+ if (k != kh_end(req->io->requests)) {
+ kh_del(rdns_requests_hash, req->io->requests, k);
+ }
+ }
+}
+
+static int
+rdns_make_socket_nonblocking (int fd)
+{
+ int ofl;
+
+ ofl = fcntl (fd, F_GETFL, 0);
+
+ if (fcntl (fd, F_SETFL, ofl | O_NONBLOCK) == -1) {
+ return -1;
+ }
+ return 0;
+}
+
+static int
+rdns_make_inet_socket (int type, struct addrinfo *addr, struct sockaddr **psockaddr,
+ socklen_t *psocklen)
+{
+ int fd = -1;
+ struct addrinfo *cur;
+
+ cur = addr;
+ while (cur) {
+ /* Create socket */
+ fd = socket (cur->ai_family, type, 0);
+ if (fd == -1) {
+ goto out;
+ }
+
+ if (rdns_make_socket_nonblocking (fd) < 0) {
+ goto out;
+ }
+
+ /* Set close on exec */
+ if (fcntl (fd, F_SETFD, FD_CLOEXEC) == -1) {
+ goto out;
+ }
+
+ if (psockaddr) {
+ *psockaddr = cur->ai_addr;
+ *psocklen = cur->ai_addrlen;
+ }
+ break;
+out:
+ if (fd != -1) {
+ close (fd);
+ }
+ fd = -1;
+ cur = cur->ai_next;
+ }
+
+ return (fd);
+}
+
+static int
+rdns_make_unix_socket (const char *path, struct sockaddr_un *addr, int type)
+{
+ int fd = -1, serrno;
+
+ if (path == NULL) {
+ return -1;
+ }
+
+ addr->sun_family = AF_UNIX;
+
+ memset (addr->sun_path, 0, sizeof (addr->sun_path));
+ memccpy (addr->sun_path, path, 0, sizeof (addr->sun_path) - 1);
+#ifdef FREEBSD
+ addr->sun_len = SUN_LEN (addr);
+#endif
+
+ fd = socket (PF_LOCAL, type, 0);
+
+ if (fd == -1) {
+ return -1;
+ }
+
+ if (rdns_make_socket_nonblocking (fd) < 0) {
+ goto out;
+ }
+
+ /* Set close on exec */
+ if (fcntl (fd, F_SETFD, FD_CLOEXEC) == -1) {
+ goto out;
+ }
+
+ return (fd);
+
+ out:
+ serrno = errno;
+ if (fd != -1) {
+ close (fd);
+ }
+ errno = serrno;
+ return (-1);
+}
+
+/**
+ * Make a universal socket
+ * @param credits host, ip or path to unix socket
+ * @param port port (used for network sockets)
+ * @param async make this socket asynced
+ * @param is_server make this socket as server socket
+ * @param try_resolve try name resolution for a socket (BLOCKING)
+ */
+int
+rdns_make_client_socket (const char *credits,
+ uint16_t port,
+ int type,
+ struct sockaddr **psockaddr,
+ socklen_t *psocklen)
+{
+ struct sockaddr_un un;
+ struct stat st;
+ struct addrinfo hints, *res;
+ int r;
+ char portbuf[8];
+
+ if (*credits == '/') {
+ r = stat (credits, &st);
+ if (r == -1) {
+ /* Unix socket doesn't exists it must be created first */
+ errno = ENOENT;
+ return -1;
+ }
+ else {
+ if ((st.st_mode & S_IFSOCK) == 0) {
+ /* Path is not valid socket */
+ errno = EINVAL;
+ return -1;
+ }
+ else {
+ r = rdns_make_unix_socket (credits, &un, type);
+
+ if (r != -1 && psockaddr) {
+ struct sockaddr *cpy;
+
+ cpy = calloc (1, sizeof (un));
+ *psocklen = sizeof (un);
+
+ if (cpy == NULL) {
+ close (r);
+
+ return -1;
+ }
+
+ memcpy (cpy, &un, *psocklen);
+ *psockaddr = cpy;
+ }
+
+ return r;
+ }
+ }
+ }
+ else {
+ /* TCP related part */
+ memset (&hints, 0, sizeof (hints));
+ hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
+ hints.ai_socktype = type; /* Type of the socket */
+ hints.ai_flags = 0;
+ hints.ai_protocol = 0; /* Any protocol */
+ hints.ai_canonname = NULL;
+ hints.ai_addr = NULL;
+ hints.ai_next = NULL;
+
+ hints.ai_flags |= AI_NUMERICHOST | AI_NUMERICSERV;
+
+ snprintf (portbuf, sizeof (portbuf), "%d", (int)port);
+ if (getaddrinfo (credits, portbuf, &hints, &res) == 0) {
+ r = rdns_make_inet_socket (type, res, psockaddr, psocklen);
+
+ if (r != -1 && psockaddr) {
+ struct sockaddr *cpy;
+
+ cpy = calloc (1, *psocklen);
+
+ if (cpy == NULL) {
+ close (r);
+ freeaddrinfo (res);
+
+ return -1;
+ }
+
+ memcpy (cpy, *psockaddr, *psocklen);
+ *psockaddr = cpy;
+ }
+
+ freeaddrinfo (res);
+ return r;
+ }
+ else {
+ return -1;
+ }
+ }
+
+ /* Not reached */
+ return -1;
+}
+
+const char *
+rdns_strerror (enum dns_rcode rcode)
+{
+ rcode &= 0xf;
+ static char numbuf[16];
+
+ if ('\0' == dns_rcodes[rcode][0]) {
+ snprintf (numbuf, sizeof (numbuf), "UNKNOWN: %d", (int)rcode);
+ return numbuf;
+ }
+ return dns_rcodes[rcode];
+}
+
+const char *
+rdns_strtype (enum rdns_request_type type)
+{
+ return dns_types[type];
+}
+
+enum rdns_request_type
+rdns_type_fromstr (const char *str)
+{
+ if (str) {
+ if (strcmp (str, "a") == 0) {
+ return RDNS_REQUEST_A;
+ }
+ else if (strcmp (str, "ns") == 0) {
+ return RDNS_REQUEST_NS;
+ }
+ else if (strcmp (str, "soa") == 0) {
+ return RDNS_REQUEST_SOA;
+ }
+ else if (strcmp (str, "ptr") == 0) {
+ return RDNS_REQUEST_PTR;
+ }
+ else if (strcmp (str, "mx") == 0) {
+ return RDNS_REQUEST_MX;
+ }
+ else if (strcmp (str, "srv") == 0) {
+ return RDNS_REQUEST_SRV;
+ }
+ else if (strcmp (str, "txt") == 0) {
+ return RDNS_REQUEST_TXT;
+ }
+ else if (strcmp (str, "spf") == 0) {
+ return RDNS_REQUEST_SPF;
+ }
+ else if (strcmp (str, "aaaa") == 0) {
+ return RDNS_REQUEST_AAAA;
+ }
+ else if (strcmp (str, "tlsa") == 0) {
+ return RDNS_REQUEST_TLSA;
+ }
+ else if (strcmp (str, "cname") == 0) {
+ return RDNS_REQUEST_CNAME;
+ }
+ else if (strcmp (str, "any") == 0) {
+ return RDNS_REQUEST_ANY;
+ }
+ }
+
+ return RDNS_REQUEST_INVALID;
+}
+
+const char *
+rdns_str_from_type (enum rdns_request_type rcode)
+{
+ switch (rcode) {
+ case RDNS_REQUEST_INVALID:
+ return "(invalid)";
+ case RDNS_REQUEST_A:
+ return "a";
+ case RDNS_REQUEST_NS:
+ return "ns";
+ case RDNS_REQUEST_SOA:
+ return "soa";
+ case RDNS_REQUEST_PTR:
+ return "ptr";
+ case RDNS_REQUEST_MX:
+ return "mx";
+ case RDNS_REQUEST_TXT:
+ return "txt";
+ case RDNS_REQUEST_SRV:
+ return "srv";
+ case RDNS_REQUEST_SPF:
+ return "spf";
+ case RDNS_REQUEST_AAAA:
+ return "aaaa";
+ case RDNS_REQUEST_TLSA:
+ return "tlsa";
+ case RDNS_REQUEST_CNAME:
+ return "cname";
+ case RDNS_REQUEST_ANY:
+ return "any";
+ default:
+ return "(unknown)";
+ }
+
+}
+
+enum dns_rcode
+rdns_rcode_fromstr (const char *str)
+{
+ if (str) {
+ if (strcmp (str, "noerror") == 0) {
+ return RDNS_RC_NOERROR;
+ }
+ else if (strcmp (str, "formerr") == 0) {
+ return RDNS_RC_FORMERR;
+ }
+ else if (strcmp (str, "servfail") == 0) {
+ return RDNS_RC_SERVFAIL;
+ }
+ else if (strcmp (str, "nxdomain") == 0) {
+ return RDNS_RC_NXDOMAIN;
+ }
+ else if (strcmp (str, "notimp") == 0) {
+ return RDNS_RC_NOTIMP;
+ }
+ else if (strcmp (str, "yxdomain") == 0) {
+ return RDNS_RC_YXDOMAIN;
+ }
+ else if (strcmp (str, "yxrrset") == 0) {
+ return RDNS_RC_YXRRSET;
+ }
+ else if (strcmp (str, "nxrrset") == 0) {
+ return RDNS_RC_NXRRSET;
+ }
+ else if (strcmp (str, "notauth") == 0) {
+ return RDNS_RC_NOTAUTH;
+ }
+ else if (strcmp (str, "notzone") == 0) {
+ return RDNS_RC_NOTZONE;
+ }
+ else if (strcmp (str, "timeout") == 0) {
+ return RDNS_RC_TIMEOUT;
+ }
+ else if (strcmp (str, "neterr") == 0) {
+ return RDNS_RC_NETERR;
+ }
+ else if (strcmp (str, "norec") == 0) {
+ return RDNS_RC_NOREC;
+ }
+ }
+
+ return RDNS_RC_INVALID;
+}
+
+uint16_t
+rdns_permutor_generate_id (void)
+{
+ uint16_t id;
+
+ id = ottery_rand_unsigned ();
+
+ return id;
+}
+
+struct rdns_reply *
+rdns_make_reply (struct rdns_request *req, enum dns_rcode rcode)
+{
+ struct rdns_reply *rep;
+
+ rep = malloc (sizeof (struct rdns_reply));
+ if (rep != NULL) {
+ rep->request = req;
+ rep->resolver = req->resolver;
+ rep->entries = NULL;
+ rep->code = rcode;
+ req->reply = rep;
+ rep->flags = 0;
+ rep->requested_name = req->requested_names[0].name;
+ }
+
+ return rep;
+}
+
+void
+rdns_reply_free (struct rdns_reply *rep)
+{
+ struct rdns_reply_entry *entry, *tmp;
+
+ /* We don't need to free data for faked replies */
+ if (!rep->request || rep->request->state != RDNS_REQUEST_FAKE) {
+ LL_FOREACH_SAFE (rep->entries, entry, tmp) {
+ switch (entry->type) {
+ case RDNS_REQUEST_PTR:
+ free (entry->content.ptr.name);
+ break;
+ case RDNS_REQUEST_NS:
+ free (entry->content.ns.name);
+ break;
+ case RDNS_REQUEST_MX:
+ free (entry->content.mx.name);
+ break;
+ case RDNS_REQUEST_TXT:
+ case RDNS_REQUEST_SPF:
+ free (entry->content.txt.data);
+ break;
+ case RDNS_REQUEST_SRV:
+ free (entry->content.srv.target);
+ break;
+ case RDNS_REQUEST_TLSA:
+ free (entry->content.tlsa.data);
+ break;
+ case RDNS_REQUEST_SOA:
+ free (entry->content.soa.mname);
+ free (entry->content.soa.admin);
+ break;
+ case RDNS_REQUEST_CNAME:
+ free(entry->content.cname.name);
+ break;
+ default:
+ break;
+ }
+ free (entry);
+ }
+ }
+
+ free (rep);
+}
+
+void
+rdns_request_free (struct rdns_request *req)
+{
+ unsigned int i;
+
+ if (req != NULL) {
+ if (req->packet != NULL) {
+ free (req->packet);
+ }
+ for (i = 0; i < req->qcount; i ++) {
+ free (req->requested_names[i].name);
+ }
+ if (req->requested_names != NULL) {
+ free (req->requested_names);
+ }
+ if (req->reply != NULL) {
+ rdns_reply_free (req->reply);
+ }
+ if (req->async_event) {
+ if (req->state == RDNS_REQUEST_WAIT_REPLY) {
+ /* Remove timer */
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ rdns_request_remove_from_hash(req);
+ req->async_event = NULL;
+ }
+ else if (req->state == RDNS_REQUEST_WAIT_SEND) {
+ /* Remove retransmit event */
+ req->async->del_write (req->async->data,
+ req->async_event);
+ rdns_request_remove_from_hash(req);
+ req->async_event = NULL;
+ }
+ else if (req->state == RDNS_REQUEST_FAKE) {
+ req->async->del_write (req->async->data,
+ req->async_event);
+ req->async_event = NULL;
+ }
+ }
+ if (req->state == RDNS_REQUEST_TCP) {
+ if (req->async_event) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ }
+
+ rdns_request_remove_from_hash(req);
+ }
+#ifdef TWEETNACL
+ if (req->curve_plugin_data != NULL) {
+ req->resolver->curve_plugin->cb.curve_plugin.finish_cb (
+ req, req->resolver->curve_plugin->data);
+ }
+#endif
+ if (req->io != NULL && req->state > RDNS_REQUEST_NEW) {
+ REF_RELEASE (req->io);
+ REF_RELEASE (req->resolver);
+ }
+
+ free (req);
+ }
+}
+
+void
+rdns_ioc_free (struct rdns_io_channel *ioc)
+{
+ struct rdns_request *req;
+
+ if (IS_CHANNEL_TCP(ioc)) {
+ rdns_ioc_tcp_reset(ioc);
+ }
+
+ kh_foreach_value(ioc->requests, req, {
+ REF_RELEASE (req);
+ });
+
+ if (ioc->async_io) {
+ ioc->resolver->async->del_read(ioc->resolver->async->data,
+ ioc->async_io);
+ }
+ kh_destroy(rdns_requests_hash, ioc->requests);
+
+ if (ioc->sock != -1) {
+ close(ioc->sock);
+ }
+
+ if (ioc->saddr != NULL) {
+ free(ioc->saddr);
+ }
+
+ free (ioc);
+}
+
+struct rdns_io_channel *
+rdns_ioc_new (struct rdns_server *serv,
+ struct rdns_resolver *resolver,
+ bool is_tcp)
+{
+ struct rdns_io_channel *nioc;
+
+ if (is_tcp) {
+ nioc = calloc (1, sizeof (struct rdns_io_channel)
+ + sizeof (struct rdns_tcp_channel));
+ }
+ else {
+ nioc = calloc (1, sizeof (struct rdns_io_channel));
+ }
+
+ if (nioc == NULL) {
+ rdns_err ("calloc fails to allocate rdns_io_channel");
+ return NULL;
+ }
+
+ nioc->struct_magic = RDNS_IO_CHANNEL_TAG;
+ nioc->srv = serv;
+ nioc->resolver = resolver;
+
+ nioc->sock = rdns_make_client_socket (serv->name, serv->port,
+ is_tcp ? SOCK_STREAM : SOCK_DGRAM, &nioc->saddr, &nioc->slen);
+ if (nioc->sock == -1) {
+ rdns_err ("cannot open socket to %s: %s", serv->name,
+ strerror (errno));
+ free (nioc);
+ return NULL;
+ }
+
+ if (is_tcp) {
+ /* We also need to connect a TCP channel and set a TCP buffer */
+ nioc->tcp = (struct rdns_tcp_channel *)(((unsigned char *)nioc) + sizeof(*nioc));
+
+ if (!rdns_ioc_tcp_connect(nioc)) {
+ rdns_err ("cannot connect TCP socket to %s: %s", serv->name,
+ strerror (errno));
+ close (nioc->sock);
+ free (nioc);
+
+ return NULL;
+ }
+
+ nioc->flags |= RDNS_CHANNEL_TCP;
+ }
+ else {
+ nioc->flags |= RDNS_CHANNEL_ACTIVE;
+ nioc->async_io = resolver->async->add_read(resolver->async->data,
+ nioc->sock, nioc);
+ }
+
+ nioc->requests = kh_init(rdns_requests_hash);
+ REF_INIT_RETAIN (nioc, rdns_ioc_free);
+
+ return nioc;
+}
+
+void
+rdns_resolver_release (struct rdns_resolver *resolver)
+{
+ REF_RELEASE (resolver);
+}
+
+struct rdns_request*
+rdns_request_retain (struct rdns_request *req)
+{
+ REF_RETAIN (req);
+ return req;
+}
+
+void
+rdns_request_unschedule (struct rdns_request *req, bool remove_from_hash)
+{
+ struct rdns_resolver *resolver = req->resolver;
+
+ switch (req->state) {
+ case RDNS_REQUEST_WAIT_REPLY:
+ /* We have a timer pending */
+ if (req->async_event) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ if (remove_from_hash) {
+ rdns_request_remove_from_hash(req);
+ }
+ req->async_event = NULL;
+ }
+ break;
+ case RDNS_REQUEST_WAIT_SEND:
+ /* We have write request pending */
+ if (req->async_event) {
+ req->async->del_write (req->async->data,
+ req->async_event);
+ /* Remove from id hashes */
+ if (remove_from_hash) {
+ rdns_request_remove_from_hash(req);
+ }
+ req->async_event = NULL;
+ }
+ break;
+ case RDNS_REQUEST_TCP:
+ /* We also have a timer */
+ if (req->async_event) {
+ if (remove_from_hash) {
+ rdns_request_remove_from_hash(req);
+ }
+
+ req->async->del_timer(req->async->data,
+ req->async_event);
+
+ req->async_event = NULL;
+ }
+ default:
+ /* Nothing to unschedule, so blame if we have any event pending */
+ if (req->async_event) {
+ rdns_err("internal error: have unexpected pending async state on stage %d",
+ req->state);
+ }
+ break;
+ }
+}
+
+void
+rdns_request_release (struct rdns_request *req)
+{
+ rdns_request_unschedule (req, true);
+ REF_RELEASE (req);
+}
+
+void
+rdns_ioc_tcp_reset (struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ if (ioc->tcp->async_write) {
+ resolver->async->del_write (resolver->async->data, ioc->tcp->async_write);
+ ioc->tcp->async_write = NULL;
+ }
+ if (ioc->tcp->async_read) {
+ resolver->async->del_read (resolver->async->data, ioc->tcp->async_read);
+ ioc->tcp->async_read = NULL;
+ }
+
+ /* Clean all buffers and temporaries */
+ if (ioc->tcp->cur_read_buf) {
+ free (ioc->tcp->cur_read_buf);
+ ioc->tcp->read_buf_allocated = 0;
+ ioc->tcp->next_read_size = 0;
+ ioc->tcp->cur_read = 0;
+ ioc->tcp->cur_read_buf = NULL;
+ }
+
+ struct rdns_tcp_output_chain *oc, *tmp;
+ DL_FOREACH_SAFE(ioc->tcp->output_chain, oc, tmp) {
+ DL_DELETE (ioc->tcp->output_chain, oc);
+ free (oc);
+ }
+
+ ioc->tcp->cur_output_chains = 0;
+ ioc->tcp->output_chain = NULL;
+
+ ioc->flags &= ~RDNS_CHANNEL_CONNECTED;
+ }
+
+ /* Remove all requests pending as we are unable to complete them */
+ struct rdns_request *req;
+ kh_foreach_value(ioc->requests, req, {
+ struct rdns_reply *rep = rdns_make_reply (req, RDNS_RC_NETERR);
+ /*
+ * Unschedule request explicitly as we set state to RDNS_REQUEST_REPLIED
+ * that will prevent timer from being removed on req dtor.
+ *
+ * We skip hash removal here, as the hash will be cleared as a single
+ * operation afterwards.
+ */
+ rdns_request_unschedule(req, false);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ });
+
+ if (ioc->sock != -1) {
+ close (ioc->sock);
+ ioc->sock = -1;
+ }
+ if (ioc->saddr) {
+ free (ioc->saddr);
+ ioc->saddr = NULL;
+ }
+
+ kh_clear(rdns_requests_hash, ioc->requests);
+}
+
+bool
+rdns_ioc_tcp_connect (struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ rdns_err ("trying to connect already connected IO channel!");
+ return false;
+ }
+
+ if (ioc->flags & RDNS_CHANNEL_TCP_CONNECTING) {
+ /* Already connecting channel, ignore connect request */
+
+ return true;
+ }
+
+ if (ioc->sock == -1) {
+ ioc->sock = rdns_make_client_socket (ioc->srv->name, ioc->srv->port,
+ SOCK_STREAM, &ioc->saddr, &ioc->slen);
+ if (ioc->sock == -1) {
+ rdns_err ("cannot open socket to %s: %s", ioc->srv->name,
+ strerror (errno));
+
+ if (ioc->saddr) {
+ free (ioc->saddr);
+ ioc->saddr = NULL;
+ }
+
+ return false;
+ }
+ }
+
+ int r = connect (ioc->sock, ioc->saddr, ioc->slen);
+
+ if (r == -1) {
+ if (errno != EAGAIN && errno != EINTR && errno != EINPROGRESS) {
+ rdns_err ("cannot connect a TCP socket: %s for server %s",
+ strerror(errno), ioc->srv->name);
+ close (ioc->sock);
+
+ if (ioc->saddr) {
+ free (ioc->saddr);
+ ioc->saddr = NULL;
+ }
+
+ ioc->sock = -1;
+
+ return false;
+ }
+ else {
+ /* We need to wait for write readiness here */
+ if (ioc->tcp->async_write != NULL) {
+ rdns_err("internal rdns error: write event is already registered on connect");
+ }
+ else {
+ ioc->tcp->async_write = resolver->async->add_write(resolver->async->data,
+ ioc->sock, ioc);
+ }
+ /* Prevent double connect attempts */
+ ioc->flags |= RDNS_CHANNEL_TCP_CONNECTING;
+ }
+ }
+ else {
+ /* Always be ready to read from a TCP socket */
+ ioc->flags |= RDNS_CHANNEL_CONNECTED|RDNS_CHANNEL_ACTIVE;
+ ioc->flags &= ~RDNS_CHANNEL_TCP_CONNECTING;
+ ioc->tcp->async_read = resolver->async->add_read(resolver->async->data,
+ ioc->sock, ioc);
+ }
+
+ return true;
+}
+
+static bool
+rdns_resolver_conf_process_line (struct rdns_resolver *resolver,
+ const char *line, rdns_resolv_conf_cb cb, void *ud)
+{
+ const char *p, *c, *end;
+ bool has_obrace = false, ret;
+ unsigned int port = dns_port;
+ char *cpy_buf;
+
+ end = line + strlen (line);
+
+ if (end - line > sizeof ("nameserver") - 1 &&
+ strncmp (line, "nameserver", sizeof ("nameserver") - 1) == 0) {
+ p = line + sizeof ("nameserver") - 1;
+ /* Skip spaces */
+ while (isspace (*p)) {
+ p ++;
+ }
+
+ if (*p == '[') {
+ has_obrace = true;
+ p ++;
+ }
+
+ if (isxdigit (*p) || *p == ':') {
+ c = p;
+ while (isxdigit (*p) || *p == ':' || *p == '.') {
+ p ++;
+ }
+ if (has_obrace && *p != ']') {
+ return false;
+ }
+ else if (*p != '\0' && !isspace (*p) && *p != '#') {
+ return false;
+ }
+
+ if (has_obrace) {
+ p ++;
+ if (*p == ':') {
+ /* Maybe we have a port definition */
+ port = strtoul (p + 1, NULL, 10);
+ if (port == 0 || port > UINT16_MAX) {
+ return false;
+ }
+ }
+ }
+
+ cpy_buf = malloc (p - c + 1);
+ assert (cpy_buf != NULL);
+ memcpy (cpy_buf, c, p - c);
+ cpy_buf[p - c] = '\0';
+
+ if (cb == NULL) {
+ ret = rdns_resolver_add_server (resolver, cpy_buf, port, 0,
+ default_io_cnt) != NULL;
+ }
+ else {
+ ret = cb (resolver, cpy_buf, port, 0,
+ default_io_cnt, ud);
+ }
+
+ free (cpy_buf);
+
+ return ret;
+ }
+ else {
+ return false;
+ }
+ }
+ /* XXX: skip unknown resolv.conf lines */
+
+ return false;
+}
+
+bool
+rdns_resolver_parse_resolv_conf_cb (struct rdns_resolver *resolver,
+ const char *path, rdns_resolv_conf_cb cb, void *ud)
+{
+ FILE *in;
+ char buf[BUFSIZ];
+ char *p;
+ bool processed = false;
+
+ in = fopen (path, "r");
+
+ if (in == NULL) {
+ return false;
+ }
+
+ while (!feof (in)) {
+ if (fgets (buf, sizeof (buf) - 1, in) == NULL) {
+ break;
+ }
+
+ /* Strip trailing spaces */
+ p = buf + strlen (buf) - 1;
+ while (p > buf &&
+ (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n')) {
+ *p-- = '\0';
+ }
+
+ if (rdns_resolver_conf_process_line (resolver, buf, cb, ud)) {
+ processed = true;
+ }
+ }
+
+ fclose (in);
+
+ return processed;
+}
+
+bool
+rdns_resolver_parse_resolv_conf (struct rdns_resolver *resolver, const char *path)
+{
+ return rdns_resolver_parse_resolv_conf_cb (resolver, path, NULL, NULL);
+}
+
+bool
+rdns_request_has_type (struct rdns_request *req, enum rdns_request_type type)
+{
+ unsigned int i;
+
+ for (i = 0; i < req->qcount; i ++) {
+ if (req->requested_names[i].type == type) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+const struct rdns_request_name *
+rdns_request_get_name (struct rdns_request *req, unsigned int *count)
+{
+
+ if (count != NULL) {
+ *count = req->qcount;
+ }
+ return req->requested_names;
+}
+
+const char*
+rdns_request_get_server (struct rdns_request *req)
+{
+ if (req && req->io) {
+ return req->io->srv->name;
+ }
+
+ return NULL;
+}
+
+char *
+rdns_generate_ptr_from_str (const char *str)
+{
+ union {
+ struct in_addr v4;
+ struct in6_addr v6;
+ } addr;
+ char *res = NULL;
+ unsigned char *bytes;
+ size_t len;
+
+ if (inet_pton (AF_INET, str, &addr.v4) == 1) {
+ bytes = (unsigned char *)&addr.v4;
+
+ len = 4 * 4 + sizeof ("in-addr.arpa");
+ res = malloc (len);
+ if (res) {
+ snprintf (res, len, "%u.%u.%u.%u.in-addr.arpa",
+ (unsigned)bytes[3]&0xFF,
+ (unsigned)bytes[2]&0xFF,
+ (unsigned)bytes[1]&0xFF,
+ (unsigned)bytes[0]&0xFF);
+ }
+ }
+ else if (inet_pton (AF_INET6, str, &addr.v6) == 1) {
+ bytes = (unsigned char *)&addr.v6;
+
+ len = 2*32 + sizeof ("ip6.arpa");
+ res = malloc (len);
+ if (res) {
+ snprintf(res, len,
+ "%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x."
+ "%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.ip6.arpa",
+ bytes[15]&0xF, bytes[15] >> 4, bytes[14]&0xF, bytes[14] >> 4,
+ bytes[13]&0xF, bytes[13] >> 4, bytes[12]&0xF, bytes[12] >> 4,
+ bytes[11]&0xF, bytes[11] >> 4, bytes[10]&0xF, bytes[10] >> 4,
+ bytes[9]&0xF, bytes[9] >> 4, bytes[8]&0xF, bytes[8] >> 4,
+ bytes[7]&0xF, bytes[7] >> 4, bytes[6]&0xF, bytes[6] >> 4,
+ bytes[5]&0xF, bytes[5] >> 4, bytes[4]&0xF, bytes[4] >> 4,
+ bytes[3]&0xF, bytes[3] >> 4, bytes[2]&0xF, bytes[2] >> 4,
+ bytes[1]&0xF, bytes[1] >> 4, bytes[0]&0xF, bytes[0] >> 4);
+ }
+ }
+
+ return res;
+}
diff --git a/contrib/librdns/util.h b/contrib/librdns/util.h
new file mode 100644
index 0000000..6f74d8b
--- /dev/null
+++ b/contrib/librdns/util.h
@@ -0,0 +1,99 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef UTIL_H_
+#define UTIL_H_
+
+#include "dns_private.h"
+
+/**
+ * Make a universal socket
+ * @param credits host, ip or path to unix socket
+ * @param port port (used for network sockets)
+ * @param type of socket (SOCK_STREAM or SOCK_DGRAM)
+ */
+int
+rdns_make_client_socket (const char *credits,
+ uint16_t port,
+ int type,
+ struct sockaddr **psockaddr,
+ socklen_t *psocklen);
+
+/**
+ * Generate new random DNS id
+ * @return dns id
+ */
+uint16_t rdns_permutor_generate_id (void);
+
+
+/**
+ * Free IO channel
+ */
+void rdns_ioc_free (struct rdns_io_channel *ioc);
+
+/**
+ * Creates a new IO channel
+ */
+struct rdns_io_channel * rdns_ioc_new (struct rdns_server *srv,
+ struct rdns_resolver *resolver,
+ bool is_tcp);
+
+/**
+ * Resets inactive/errored TCP chain as recommended by RFC
+ * @param ioc
+ */
+void rdns_ioc_tcp_reset (struct rdns_io_channel *ioc);
+
+/**
+ * Connect TCP IO channel to a server
+ * @param ioc
+ */
+bool rdns_ioc_tcp_connect (struct rdns_io_channel *ioc);
+
+/**
+ * Free request
+ * @param req
+ */
+void rdns_request_free (struct rdns_request *req);
+
+/**
+ * Removes request from a channel's hash (e.g. if needed to migrate to another channel)
+ * @param req
+ */
+void rdns_request_remove_from_hash (struct rdns_request *req);
+
+/**
+ * Creates a new reply
+ * @param req
+ * @param rcode
+ * @return
+ */
+struct rdns_reply * rdns_make_reply (struct rdns_request *req, enum dns_rcode rcode);
+/**
+ * Free reply
+ * @param rep
+ */
+void rdns_reply_free (struct rdns_reply *rep);
+
+void rdns_request_unschedule (struct rdns_request *req, bool remove_from_hash);
+
+#endif /* UTIL_H_ */
diff --git a/contrib/libucl/CMakeLists.txt b/contrib/libucl/CMakeLists.txt
new file mode 100644
index 0000000..eb78701
--- /dev/null
+++ b/contrib/libucl/CMakeLists.txt
@@ -0,0 +1,19 @@
+SET(UCLSRC ucl_util.c
+ ucl_parser.c
+ ucl_emitter.c
+ ucl_emitter_streamline.c
+ ucl_hash.c
+ ucl_schema.c
+ lua_ucl.c
+ ucl_msgpack.c
+ ucl_sexp.c)
+
+
+SET (LIB_TYPE STATIC)
+ADD_LIBRARY(ucl ${LIB_TYPE} ${UCLSRC})
+
+IF(ENABLE_URL_SIGN MATCHES "ON")
+ IF(OPENSSL_FOUND)
+ TARGET_LINK_LIBRARIES(ucl ${OPENSSL_LIBRARIES})
+ ENDIF(OPENSSL_FOUND)
+ENDIF(ENABLE_URL_SIGN MATCHES "ON")
diff --git a/contrib/libucl/README.md b/contrib/libucl/README.md
new file mode 100644
index 0000000..146143d
--- /dev/null
+++ b/contrib/libucl/README.md
@@ -0,0 +1,397 @@
+# LIBUCL
+
+[![CircleCI](https://circleci.com/gh/vstakhov/libucl.svg?style=svg)](https://circleci.com/gh/vstakhov/libucl)
+[![Coverity](https://scan.coverity.com/projects/4138/badge.svg)](https://scan.coverity.com/projects/4138)
+[![Coverage Status](https://coveralls.io/repos/github/vstakhov/libucl/badge.svg?branch=master)](https://coveralls.io/github/vstakhov/libucl?branch=master)
+
+**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)*
+
+- [Introduction](#introduction)
+- [Basic structure](#basic-structure)
+- [Improvements to the json notation](#improvements-to-the-json-notation)
+ - [General syntax sugar](#general-syntax-sugar)
+ - [Automatic arrays creation](#automatic-arrays-creation)
+ - [Named keys hierarchy](#named-keys-hierarchy)
+ - [Convenient numbers and booleans](#convenient-numbers-and-booleans)
+- [General improvements](#general-improvements)
+ - [Comments](#comments)
+ - [Macros support](#macros-support)
+ - [Variables support](#variables-support)
+ - [Multiline strings](#multiline-strings)
+ - [Single quoted strings](#single-quoted-strings)
+- [Emitter](#emitter)
+- [Validation](#validation)
+- [Performance](#performance)
+- [Conclusion](#conclusion)
+
+## Introduction
+
+This document describes the main features and principles of the configuration
+language called `UCL` - universal configuration language.
+
+If you are looking for the libucl API documentation you can find it at [this page](doc/api.md).
+
+## Basic structure
+
+UCL is heavily infused by `nginx` configuration as the example of a convenient configuration
+system. However, UCL is fully compatible with `JSON` format and is able to parse json files.
+For example, you can write the same configuration in the following ways:
+
+* in nginx like:
+
+```nginx
+param = value;
+section {
+ param = value;
+ param1 = value1;
+ flag = true;
+ number = 10k;
+ time = 0.2s;
+ string = "something";
+ subsection {
+ host = {
+ host = "hostname";
+ port = 900;
+ }
+ host = {
+ host = "hostname";
+ port = 901;
+ }
+ }
+}
+```
+
+* or in JSON:
+
+```json
+{
+ "param": "value",
+ "param1": "value1",
+ "flag": true,
+ "subsection": {
+ "host": [
+ {
+ "host": "hostname",
+ "port": 900
+ },
+ {
+ "host": "hostname",
+ "port": 901
+ }
+ ]
+ }
+}
+```
+
+## Improvements to the json notation.
+
+There are various things that make ucl configuration more convenient for editing than strict json:
+
+### General syntax sugar
+
+* Braces are not necessary to enclose a top object: it is automatically treated as an object:
+
+```json
+"key": "value"
+```
+is equal to:
+```json
+{"key": "value"}
+```
+
+* There is no requirement of quotes for strings and keys, moreover, `:` may be replaced `=` or even be skipped for objects:
+
+```nginx
+key = value;
+section {
+ key = value;
+}
+```
+is equal to:
+```json
+{
+ "key": "value",
+ "section": {
+ "key": "value"
+ }
+}
+```
+
+* No commas mess: you can safely place a comma or semicolon for the last element in an array or an object:
+
+```json
+{
+ "key1": "value",
+ "key2": "value",
+}
+```
+### Automatic arrays creation
+
+* Non-unique keys in an object are allowed and are automatically converted to the arrays internally:
+
+```json
+{
+ "key": "value1",
+ "key": "value2"
+}
+```
+is converted to:
+```json
+{
+ "key": ["value1", "value2"]
+}
+```
+
+### Named keys hierarchy
+
+UCL accepts named keys and organize them into objects hierarchy internally. Here is an example of this process:
+```nginx
+section "blah" {
+ key = value;
+}
+section foo {
+ key = value;
+}
+```
+
+is converted to the following object:
+
+```nginx
+section {
+ blah {
+ key = value;
+ }
+ foo {
+ key = value;
+ }
+}
+```
+
+Plain definitions may be more complex and contain more than a single level of nested objects:
+
+```nginx
+section "blah" "foo" {
+ key = value;
+}
+```
+
+is presented as:
+
+```nginx
+section {
+ blah {
+ foo {
+ key = value;
+ }
+ }
+}
+```
+
+### Convenient numbers and booleans
+
+* Numbers can have suffixes to specify standard multipliers:
+ + `[kKmMgG]` - standard 10 base multipliers (so `1k` is translated to 1000)
+ + `[kKmMgG]b` - 2 power multipliers (so `1kb` is translated to 1024)
+ + `[s|min|d|w|y]` - time multipliers, all time values are translated to float number of seconds, for example `10min` is translated to 600.0 and `10ms` is translated to 0.01
+* Hexadecimal integers can be used by `0x` prefix, for example `key = 0xff`. However, floating point values can use decimal base only.
+* Booleans can be specified as `true` or `yes` or `on` and `false` or `no` or `off`.
+* It is still possible to treat numbers and booleans as strings by enclosing them in double quotes.
+
+## General improvements
+
+### Comments
+
+UCL supports different style of comments:
+
+* single line: `#`
+* multiline: `/* ... */`
+
+Multiline comments may be nested:
+```c
+# Sample single line comment
+/*
+ some comment
+ /* nested comment */
+ end of comment
+*/
+```
+
+### Macros support
+
+UCL supports external macros both multiline and single line ones:
+```nginx
+.macro_name "sometext";
+.macro_name {
+ Some long text
+ ....
+};
+```
+
+Moreover, each macro can accept an optional list of arguments in braces. These
+arguments themselves are the UCL object that is parsed and passed to a macro as
+options:
+
+```nginx
+.macro_name(param=value) "something";
+.macro_name(param={key=value}) "something";
+.macro_name(.include "params.conf") "something";
+.macro_name(#this is multiline macro
+param = [value1, value2]) "something";
+.macro_name(key="()") "something";
+```
+
+UCL also provide a convenient `include` macro to load content from another files
+to the current UCL object. This macro accepts either path to file:
+
+```nginx
+.include "/full/path.conf"
+.include "./relative/path.conf"
+.include "${CURDIR}/path.conf"
+```
+
+or URL (if ucl is built with url support provided by either `libcurl` or `libfetch`):
+
+ .include "http://example.com/file.conf"
+
+`.include` macro supports a set of options:
+
+* `try` (default: **false**) - if this option is `true` than UCL treats errors on loading of
+this file as non-fatal. For example, such a file can be absent but it won't stop the parsing
+of the top-level document.
+* `sign` (default: **false**) - if this option is `true` UCL loads and checks the signature for
+a file from path named `<FILEPATH>.sig`. Trusted public keys should be provided for UCL API after
+parser is created but before any configurations are parsed.
+* `glob` (default: **false**) - if this option is `true` UCL treats the filename as GLOB pattern and load
+all files that matches the specified pattern (normally the format of patterns is defined in `glob` manual page
+for your operating system). This option is meaningless for URL includes.
+* `url` (default: **true**) - allow URL includes.
+* `path` (default: empty) - A UCL_ARRAY of directories to search for the include file.
+Search ends after the first match, unless `glob` is true, then all matches are included.
+* `prefix` (default false) - Put included contents inside an object, instead
+of loading them into the root. If no `key` is provided, one is automatically generated based on each files basename()
+* `key` (default: <empty string>) - Key to load contents of include into. If
+the key already exists, it must be the correct type
+* `target` (default: object) - Specify if the `prefix` `key` should be an
+object or an array.
+* `priority` (default: 0) - specify priority for the include (see below).
+* `duplicate` (default: 'append') - specify policy of duplicates resolving:
+ - `append` - default strategy, if we have new object of higher priority then it replaces old one, if we have new object with less priority it is ignored completely, and if we have two duplicate objects with the same priority then we have a multi-value key (implicit array)
+ - `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardless of priorities)
+ - `error` - create error on duplicate keys and stop parsing
+ - `rewrite` - always rewrite an old value with new one (ignoring priorities)
+
+Priorities are used by UCL parser to manage the policy of objects rewriting during including other files
+as following:
+
+* If we have two objects with the same priority then we form an implicit array
+* If a new object has bigger priority then we overwrite an old one
+* If a new object has lower priority then we ignore it
+
+By default, the priority of top-level object is set to zero (lowest priority). Currently,
+you can define up to 16 priorities (from 0 to 15). Includes with bigger priorities will
+rewrite keys from the objects with lower priorities as specified by the policy.
+
+### Variables support
+
+UCL supports variables in input. Variables are registered by a user of the UCL parser and can be presented in the following forms:
+
+* `${VARIABLE}`
+* `$VARIABLE`
+
+UCL currently does not support nested variables. To escape variables one could use double dollar signs:
+
+* `$${VARIABLE}` is converted to `${VARIABLE}`
+* `$$VARIABLE` is converted to `$VARIABLE`
+
+However, if no valid variables are found in a string, no expansion will be performed (and `$$` thus remains unchanged). This may be a subject
+to change in future libucl releases.
+
+### Multiline strings
+
+UCL can handle multiline strings as well as single line ones. It uses shell/perl like notation for such objects:
+```
+key = <<EOD
+some text
+splitted to
+lines
+EOD
+```
+
+In this example `key` will be interpreted as the following string: `some text\nsplitted to\nlines`.
+Here are some rules for this syntax:
+
+* Multiline terminator must start just after `<<` symbols and it must consist of capital letters only (e.g. `<<eof` or `<< EOF` won't work);
+* Terminator must end with a single newline character (and no spaces are allowed between terminator and newline character);
+* To finish multiline string you need to include a terminator string just after newline and followed by a newline (no spaces or other characters are allowed as well);
+* The initial and the final newlines are not inserted to the resulting string, but you can still specify newlines at the beginning and at the end of a value, for example:
+
+```
+key <<EOD
+
+some
+text
+
+EOD
+```
+
+### Single quoted strings
+
+It is possible to use single quoted strings to simplify escaping rules. All values passed in single quoted strings are *NOT* escaped, with two exceptions: a single `'` character just before `\` character, and a newline character just after `\` character that is ignored.
+
+```
+key = 'value'; # Read as value
+key = 'value\n\'; # Read as value\n\
+key = 'value\''; # Read as value'
+key = 'value\
+bla'; # Read as valuebla
+```
+
+## Emitter
+
+Each UCL object can be serialized to one of the three supported formats:
+
+* `JSON` - canonic json notation (with spaces indented structure);
+* `Compacted JSON` - compact json notation (without spaces or newlines);
+* `Configuration` - nginx like notation;
+* `YAML` - yaml inlined notation.
+
+## Validation
+
+UCL allows validation of objects. It uses the same schema that is used for json: [json schema v4](http://json-schema.org). UCL supports the full set of json schema with the exception of remote references. This feature is unlikely useful for configuration objects. Of course, a schema definition can be in UCL format instead of JSON that simplifies schemas writing. Moreover, since UCL supports multiple values for keys in an object it is possible to specify generic integer constraints `maxValues` and `minValues` to define the limits of values count in a single key. UCL currently is not absolutely strict about validation schemas themselves, therefore UCL users should supply valid schemas (as it is defined in json-schema draft v4) to ensure that the input objects are validated properly.
+
+## Performance
+
+Are UCL parser and emitter fast enough? Well, there are some numbers.
+I got a 19Mb file that consist of ~700 thousand lines of json (obtained via
+http://www.json-generator.com/). Then I checked jansson library that performs json
+parsing and emitting and compared it with UCL. Here are results:
+
+```
+jansson: parsed json in 1.3899 seconds
+jansson: emitted object in 0.2609 seconds
+
+ucl: parsed input in 0.6649 seconds
+ucl: emitted config in 0.2423 seconds
+ucl: emitted json in 0.2329 seconds
+ucl: emitted compact json in 0.1811 seconds
+ucl: emitted yaml in 0.2489 seconds
+```
+
+So far, UCL seems to be significantly faster than jansson on parsing and slightly faster on emitting. Moreover,
+UCL compiled with optimizations (-O3) performs significantly faster:
+```
+ucl: parsed input in 0.3002 seconds
+ucl: emitted config in 0.1174 seconds
+ucl: emitted json in 0.1174 seconds
+ucl: emitted compact json in 0.0991 seconds
+ucl: emitted yaml in 0.1354 seconds
+```
+
+You can do your own benchmarks by running `make check` in libucl top directory.
+
+## Conclusion
+
+UCL has clear design that should be very convenient for reading and writing. At the same time it is compatible with
+JSON language and therefore can be used as a simple JSON parser. Macro logic provides an ability to extend configuration
+language (for example by including some lua code) and comments allow to disable or enable the parts of a configuration
+quickly.
diff --git a/contrib/libucl/khash.h b/contrib/libucl/khash.h
new file mode 100644
index 0000000..1499d75
--- /dev/null
+++ b/contrib/libucl/khash.h
@@ -0,0 +1,682 @@
+/* The MIT License
+
+ Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/*
+ An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+ int ret, is_missing;
+ khiter_t k;
+ khash_t(32) *h = kh_init(32);
+ k = kh_put(32, h, 5, &ret);
+ kh_value(h, k) = 10;
+ k = kh_get(32, h, 10);
+ is_missing = (k == kh_end(h));
+ k = kh_get(32, h, 5);
+ kh_del(32, h, k);
+ for (k = kh_begin(h); k != kh_end(h); ++k)
+ if (kh_exist(h, k)) kh_value(h, k) = 1;
+ kh_destroy(32, h);
+ return 0;
+}
+*/
+
+/*
+ 2013-05-02 (0.2.8):
+
+ * Use quadratic probing. When the capacity is power of 2, stepping function
+ i*(i+1)/2 guarantees to traverse each bucket. It is better than double
+ hashing on cache performance and is more robust than linear probing.
+
+ In theory, double hashing should be more robust than quadratic probing.
+ However, my implementation is probably not for large hash tables, because
+ the second hash function is closely tied to the first hash function,
+ which reduce the effectiveness of double hashing.
+
+ Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php
+
+ 2011-12-29 (0.2.7):
+
+ * Minor code clean up; no actual effect.
+
+ 2011-09-16 (0.2.6):
+
+ * The capacity is a power of 2. This seems to dramatically improve the
+ speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
+
+ - http://code.google.com/p/ulib/
+ - http://nothings.org/computer/judy/
+
+ * Allow to optionally use linear probing which usually has better
+ performance for random input. Double hashing is still the default as it
+ is more robust to certain non-random input.
+
+ * Added Wang's integer hash function (not used by default). This hash
+ function is more robust to certain non-random input.
+
+ 2011-02-14 (0.2.5):
+
+ * Allow to declare global functions.
+
+ 2009-09-26 (0.2.4):
+
+ * Improve portability
+
+ 2008-09-19 (0.2.3):
+
+ * Corrected the example
+ * Improved interfaces
+
+ 2008-09-11 (0.2.2):
+
+ * Improved speed a little in kh_put()
+
+ 2008-09-10 (0.2.1):
+
+ * Added kh_clear()
+ * Fixed a compiling error
+
+ 2008-09-02 (0.2.0):
+
+ * Changed to token concatenation which increases flexibility.
+
+ 2008-08-31 (0.1.2):
+
+ * Fixed a bug in kh_get(), which has not been tested previously.
+
+ 2008-08-31 (0.1.1):
+
+ * Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+/*!
+ @header
+
+ Generic hash table library.
+ */
+
+#define AC_VERSION_KHASH_H "0.2.8"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+/* compiler specific configuration */
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifndef kh_inline
+#ifdef _MSC_VER
+#define kh_inline __inline
+#else
+#define kh_inline inline
+#endif
+#endif /* kh_inline */
+
+#ifndef kh_unused
+# ifdef __GNUC__
+# define kh_unused(x) __attribute__((__unused__)) x
+# else
+# define kh_unused(x) x
+# endif
+#endif
+
+typedef khint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef kcalloc
+#define kcalloc(N,Z) calloc(N,Z)
+#endif
+#ifndef kmalloc
+#define kmalloc(Z) malloc(Z)
+#endif
+#ifndef krealloc
+#define krealloc(P,Z) realloc(P,Z)
+#endif
+#ifndef kfree
+#define kfree(P) free(P)
+#endif
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define __KHASH_TYPE(name, khkey_t, khval_t) \
+ typedef struct kh_##name##_s { \
+ khint_t n_buckets, size, n_occupied, upper_bound; \
+ khint32_t *flags; \
+ khkey_t *keys; \
+ khval_t *vals; \
+ } kh_##name##_t
+
+#define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \
+ extern kh_##name##_t * kh_init_##name(void); \
+ extern void kh_static_init_##name(kh_##name##_t *); \
+ extern void kh_destroy_##name(kh_##name##_t *h); \
+ extern void kh_static_destroy_##name(kh_##name##_t *h); \
+ extern void kh_clear_##name(kh_##name##_t *h); \
+ extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
+ extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+ extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
+ extern void kh_del_##name(kh_##name##_t *h, khint_t x);
+
+#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ SCOPE kh_##name##_t *kh_init_##name(void) { \
+ return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \
+ } \
+ SCOPE void kh_unused(kh_static_init_##name)(kh_##name##_t *target) {\
+ memset(target, 0, sizeof(*target)); \
+ } \
+ SCOPE void kh_destroy_##name(kh_##name##_t *h) \
+ { \
+ if (h) { \
+ kfree((void *)h->keys); kfree(h->flags); \
+ kfree((void *)h->vals); \
+ kfree(h); \
+ } \
+ } \
+ SCOPE void kh_unused(kh_static_destroy_##name)(kh_##name##_t *h) { \
+ kfree((void *)h->keys); kfree(h->flags); \
+ kfree((void *)h->vals); \
+ } \
+ SCOPE void kh_unused(kh_clear_##name)(kh_##name##_t *h) \
+ { \
+ if (h && h->flags) { \
+ memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
+ h->size = h->n_occupied = 0; \
+ } \
+ } \
+ SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \
+ { \
+ if (h->n_buckets) { \
+ khint_t k, i, last, mask, step = 0; \
+ mask = h->n_buckets - 1; \
+ k = __hash_func(key); i = k & mask; \
+ last = i; \
+ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+ i = (i + (++step)) & mask; \
+ if (i == last) return h->n_buckets; \
+ } \
+ return __ac_iseither(h->flags, i)? h->n_buckets : i; \
+ } else return 0; \
+ } \
+ SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+ { /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
+ khint32_t *new_flags = 0; \
+ khint_t j = 1; \
+ { \
+ kroundup32(new_n_buckets); \
+ if (new_n_buckets < 4) new_n_buckets = 4; \
+ if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \
+ else { /* hash table size to be changed (shrink or expand); rehash */ \
+ new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+ if (!new_flags) return -1; \
+ memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+ if (h->n_buckets < new_n_buckets) { /* expand */ \
+ khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+ if (!new_keys) { kfree(new_flags); return -1; } \
+ h->keys = new_keys; \
+ if (kh_is_map) { \
+ khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+ if (!new_vals) { kfree(new_flags); return -1; } \
+ h->vals = new_vals; \
+ } \
+ } /* otherwise shrink */ \
+ } \
+ } \
+ if (j) { /* rehashing is needed */ \
+ for (j = 0; j != h->n_buckets; ++j) { \
+ if (__ac_iseither(h->flags, j) == 0) { \
+ khkey_t key = h->keys[j]; \
+ khval_t val; \
+ khint_t new_mask; \
+ new_mask = new_n_buckets - 1; \
+ if (kh_is_map) val = h->vals[j]; \
+ __ac_set_isdel_true(h->flags, j); \
+ while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+ khint_t k, i, step = 0; \
+ k = __hash_func(key); \
+ i = k & new_mask; \
+ while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \
+ __ac_set_isempty_false(new_flags, i); \
+ if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
+ { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+ if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+ __ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
+ } else { /* write the element and jump out of the loop */ \
+ h->keys[i] = key; \
+ if (kh_is_map) h->vals[i] = val; \
+ break; \
+ } \
+ } \
+ } \
+ } \
+ if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
+ h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+ if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+ } \
+ kfree(h->flags); /* free the working space */ \
+ h->flags = new_flags; \
+ h->n_buckets = new_n_buckets; \
+ h->n_occupied = h->size; \
+ h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+ } \
+ return 0; \
+ } \
+ SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+ { \
+ khint_t x; \
+ if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
+ if (h->n_buckets > (h->size<<1)) { \
+ if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
+ *ret = -1; return h->n_buckets; \
+ } \
+ } else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
+ *ret = -1; return h->n_buckets; \
+ } \
+ } /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+ { \
+ khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \
+ x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
+ if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \
+ else { \
+ last = i; \
+ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+ if (__ac_isdel(h->flags, i)) site = i; \
+ i = (i + (++step)) & mask; \
+ if (i == last) { x = site; break; } \
+ } \
+ if (x == h->n_buckets) { \
+ if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+ else x = i; \
+ } \
+ } \
+ } \
+ if (__ac_isempty(h->flags, x)) { /* not present at all */ \
+ h->keys[x] = key; \
+ __ac_set_isboth_false(h->flags, x); \
+ ++h->size; ++h->n_occupied; \
+ *ret = 1; \
+ } else if (__ac_isdel(h->flags, x)) { /* deleted */ \
+ h->keys[x] = key; \
+ __ac_set_isboth_false(h->flags, x); \
+ ++h->size; \
+ *ret = 2; \
+ } else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+ return x; \
+ } \
+ SCOPE void kh_unused(kh_del_##name)(kh_##name##_t *h, khint_t x) \
+ { \
+ if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \
+ __ac_set_isdel_true(h->flags, x); \
+ --h->size; \
+ } \
+ }
+
+#define KHASH_DECLARE(name, khkey_t, khval_t) \
+ __KHASH_TYPE(name, khkey_t, khval_t); \
+ __KHASH_PROTOTYPES(name, khkey_t, khval_t)
+
+#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ __KHASH_TYPE(name, khkey_t, khval_t); \
+ __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ KHASH_INIT2(name, static kh_inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+/*! @function
+ @abstract Integer hash function
+ @param key The integer [khint32_t]
+ @return The hash value [khint_t]
+ */
+#define kh_int_hash_func(key) (khint32_t)(key)
+/*! @function
+ @abstract Integer comparison function
+ */
+#define kh_int_hash_equal(a, b) ((a) == (b))
+/*! @function
+ @abstract 64-bit integer hash function
+ @param key The integer [khint64_t]
+ @return The hash value [khint_t]
+ */
+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
+/*! @function
+ @abstract 64-bit integer comparison function
+ */
+#define kh_int64_hash_equal(a, b) ((a) == (b))
+/*! @function
+ @abstract const char* hash function
+ @param s Pointer to a null terminated string
+ @return The hash value
+ */
+static kh_inline khint_t __ac_X31_hash_string(const char *s)
+{
+ khint_t h = (khint_t)*s;
+ if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
+ return h;
+}
+
+/**
+ * Wyhash implementation from https://github.com/wangyi-fudan/wyhash
+ */
+static inline unsigned _wyr32(const uint8_t *p) { unsigned v; memcpy(&v, p, 4); return v;}
+static inline unsigned _wyr24(const uint8_t *p, unsigned k) { return (((unsigned)p[0])<<16)|(((unsigned)p[k>>1])<<8)|p[k-1];}
+static inline void _wymix32(unsigned *A, unsigned *B){
+ uint64_t c=*A^0x53c5ca59u; c*=*B^0x74743c1bu;
+ *A=(unsigned)c;
+ *B=(unsigned)(c>>32);
+}
+// This version is vulnerable when used with a few bad seeds, which should be skipped beforehand:
+// 0x429dacdd, 0xd637dbf3
+static inline unsigned _wyhash32(const void *key, uint64_t len, unsigned seed) {
+ const uint8_t *p=(const uint8_t *)key; uint64_t i=len;
+ unsigned see1=(unsigned)len; seed^=(unsigned)(len>>32); _wymix32(&seed, &see1);
+ for(;i>8;i-=8,p+=8){ seed^=_wyr32(p); see1^=_wyr32(p+4); _wymix32(&seed, &see1); }
+ if(i>=4){ seed^=_wyr32(p); see1^=_wyr32(p+i-4); } else if (i) seed^=_wyr24(p,(unsigned)i);
+ _wymix32(&seed, &see1); _wymix32(&seed, &see1); return seed^see1;
+}
+
+/*! @function
+ @abstract Another interface to const char* hash function
+ @param key Pointer to a null terminated string [const char*]
+ @return The hash value [khint_t]
+ */
+#define kh_str_hash_func(key) _wyhash32(key, strlen(key), 0)
+/*! @function
+ @abstract Const char* comparison function
+ */
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+static kh_inline khint_t __ac_Wang_hash(khint_t key)
+{
+ key += ~(key << 15);
+ key ^= (key >> 10);
+ key += (key << 3);
+ key ^= (key >> 6);
+ key += ~(key << 11);
+ key ^= (key >> 16);
+ return key;
+}
+#define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key)
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other convenient macros... */
+
+/*!
+ @abstract Type of the hash table.
+ @param name Name of the hash table [symbol]
+ */
+#define khash_t(name) kh_##name##_t
+
+/*! @function
+ @abstract Initiate a hash table.
+ @param name Name of the hash table [symbol]
+ @return Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_init(name) kh_init_##name()
+#define kh_static_init(name, h) kh_static_init_##name(h)
+
+/*! @function
+ @abstract Destroy a hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_destroy(name, h) kh_destroy_##name(h)
+#define kh_static_destroy(name, h) kh_static_destroy_##name(h)
+
+/*! @function
+ @abstract Reset a hash table without deallocating memory.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_clear(name, h) kh_clear_##name(h)
+
+/*! @function
+ @abstract Resize a hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param s New size [khint_t]
+ */
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+
+/*! @function
+ @abstract Insert a key to the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Key [type of keys]
+ @param r Extra return code: -1 if the operation failed;
+ 0 if the key is present in the hash table;
+ 1 if the bucket is empty (never used); 2 if the element in
+ the bucket has been deleted [int*]
+ @return Iterator to the inserted element [khint_t]
+ */
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+
+/*! @function
+ @abstract Retrieve a key from the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Key [type of keys]
+ @return Iterator to the found element, or kh_end(h) if the element is absent [khint_t]
+ */
+#define kh_get(name, h, k) kh_get_##name(h, k)
+
+/*! @function
+ @abstract Remove a key from the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Iterator to the element to be deleted [khint_t]
+ */
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+/*! @function
+ @abstract Test whether a bucket contains data.
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return 1 if containing data; 0 otherwise [int]
+ */
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+
+/*! @function
+ @abstract Get key given an iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return Key [type of keys]
+ */
+#define kh_key(h, x) ((h)->keys[x])
+
+/*! @function
+ @abstract Get value given an iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return Value [type of values]
+ @discussion For hash sets, calling this results in segfault.
+ */
+#define kh_val(h, x) ((h)->vals[x])
+
+/*! @function
+ @abstract Alias of kh_val()
+ */
+#define kh_value(h, x) ((h)->vals[x])
+
+/*! @function
+ @abstract Get the start iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return The start iterator [khint_t]
+ */
+#define kh_begin(h) (khint_t)(0)
+
+/*! @function
+ @abstract Get the end iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return The end iterator [khint_t]
+ */
+#define kh_end(h) ((h)->n_buckets)
+
+/*! @function
+ @abstract Get the number of elements in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return Number of elements in the hash table [khint_t]
+ */
+#define kh_size(h) ((h)->size)
+
+/*! @function
+ @abstract Get the number of buckets in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return Number of buckets in the hash table [khint_t]
+ */
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/*! @function
+ @abstract Iterate over the entries in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param kvar Variable to which key will be assigned
+ @param vvar Variable to which value will be assigned
+ @param code Block of code to execute
+ */
+#define kh_foreach(h, kvar, vvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (kvar) = kh_key(h,__i); \
+ (vvar) = kh_val(h,__i); \
+ code; \
+ } }
+
+/*! @function
+ @abstract Iterate over the values in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param vvar Variable to which value will be assigned
+ @param code Block of code to execute
+ */
+#define kh_foreach_value(h, vvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (vvar) = kh_val(h,__i); \
+ code; \
+ } }
+
+#define kh_foreach_value_ptr(h, pvvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (pvvar) = &kh_val(h,__i); \
+ code; \
+ } }
+
+#define kh_foreach_key_value_ptr(h, kvar, pvvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (kvar) = kh_key(h,__i); \
+ (pvvar) = &kh_val(h,__i); \
+ code; \
+ } }
+
+#define kh_foreach_key(h, kvar, code) { \
+ khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (kvar) = kh_key(h,__i); \
+ code; \
+ } }
+
+/* More conenient interfaces */
+
+/*! @function
+ @abstract Instantiate a hash set containing integer keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT(name) \
+ KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing integer keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT(name, khval_t) \
+ KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing 64-bit integer keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT64(name) \
+ KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing 64-bit integer keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT64(name, khval_t) \
+ KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+/*! @function
+ @abstract Instantiate a hash map containing const char* keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_STR(name) \
+ KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing const char* keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_STR(name, khval_t) \
+ KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#endif /* __AC_KHASH_H */
diff --git a/contrib/libucl/kvec.h b/contrib/libucl/kvec.h
new file mode 100644
index 0000000..ce6a536
--- /dev/null
+++ b/contrib/libucl/kvec.h
@@ -0,0 +1,161 @@
+/* The MIT License
+
+ Copyright (c) 2008, by Attractive Chaos <attractor@live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/*
+ An example:
+
+#include "kvec.h"
+int main() {
+ kvec_t(int) array;
+ kv_init(array);
+ kv_push_safe(int, array, 10, e0); // append
+ kv_a(int, array, 20) = 5; // dynamic
+ kv_A(array, 20) = 4; // static
+ kv_destroy(array);
+ return 0;
+e0:
+ return 1;
+}
+*/
+
+/*
+ 2008-09-22 (0.1.0):
+
+ * The initial version.
+
+*/
+
+#ifndef AC_KVEC_H
+#define AC_KVEC_H
+
+#include <stdlib.h>
+
+#define kv_roundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+
+#define kvec_t(type) struct { size_t n, m; type *a; }
+#define kv_init(v) ((v).n = (v).m = 0, (v).a = 0)
+#define kv_destroy(v) free((v).a)
+#define kv_A(v, i) ((v).a[(i)])
+#define kv_pop(v) ((v).a[--(v).n])
+#define kv_size(v) ((v).n)
+#define kv_max(v) ((v).m)
+
+#define kv_resize_safe(type, v, s, el) do { \
+ type *_tp = (type*)realloc((v).a, sizeof(type) * (s)); \
+ if (_tp == NULL) { \
+ goto el; \
+ } else { \
+ (v).a = _tp; \
+ (v).m = (s); \
+ } \
+ } while (0)
+
+#define kv_grow_factor 1.5
+#define kv_grow_safe(type, v, el) do { \
+ size_t _ts = ((v).m > 1 ? (v).m * kv_grow_factor : 2); \
+ type *_tp = (type*)realloc((v).a, sizeof(type) * _ts); \
+ if (_tp == NULL) { \
+ goto el; \
+ } else { \
+ (v).a = _tp; \
+ (v).m = _ts; \
+ } \
+ } while (0)
+
+#define kv_copy_safe(type, v1, v0, el) do { \
+ if ((v1).m < (v0).n) kv_resize_safe(type, v1, (v0).n, el); \
+ (v1).n = (v0).n; \
+ memcpy((v1).a, (v0).a, sizeof(type) * (v0).n); \
+ } while (0)
+
+#define kv_push_safe(type, v, x, el) do { \
+ if ((v).n == (v).m) { \
+ kv_grow_safe(type, v, el); \
+ } \
+ (v).a[(v).n++] = (x); \
+ } while (0)
+
+#define kv_prepend_safe(type, v, x, el) do { \
+ if ((v).n == (v).m) { \
+ kv_grow_safe(type, v, el); \
+ } \
+ memmove((v).a + 1, (v).a, sizeof(type) * (v).n); \
+ (v).a[0] = (x); \
+ (v).n ++; \
+ } while (0)
+
+#define kv_concat_safe(type, v1, v0, el) do { \
+ if ((v1).m < (v0).n + (v1).n) \
+ kv_resize_safe(type, v1, (v0).n + (v1).n, el); \
+ memcpy((v1).a + (v1).n, (v0).a, sizeof(type) * (v0).n); \
+ (v1).n = (v0).n + (v1).n; \
+ } while (0)
+
+#define kv_del(type, v, i) do { \
+ if ((i) < (v).n) { \
+ memmove((v).a + (i), (v).a + ((i) + 1), sizeof(type) * ((v).n - (i) - 1)); \
+ (v).n --; \
+ } \
+} while (0)
+
+/*
+ * Old (ENOMEM-unsafe) version of kv_xxx macros. Compat-only, not for use in
+ * the new library code.
+ */
+
+#define kv_resize(type, v, s) ((v).m = (s), (v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+
+#define kv_grow(type, v) ((v).m = ((v).m > 1 ? (v).m * kv_grow_factor : 2), \
+ (v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+
+#define kv_copy(type, v1, v0) do { \
+ if ((v1).m < (v0).n) kv_resize(type, v1, (v0).n); \
+ (v1).n = (v0).n; \
+ memcpy((v1).a, (v0).a, sizeof(type) * (v0).n); \
+ } while (0) \
+
+#define kv_push(type, v, x) do { \
+ if ((v).n == (v).m) { \
+ kv_grow(type, v); \
+ } \
+ (v).a[(v).n++] = (x); \
+ } while (0)
+
+#define kv_prepend(type, v, x) do { \
+ if ((v).n == (v).m) { \
+ kv_grow(type, v); \
+ } \
+ memmove((v).a + 1, (v).a, sizeof(type) * (v).n); \
+ (v).a[0] = (x); \
+ (v).n ++; \
+} while (0)
+
+#define kv_concat(type, v1, v0) do { \
+ if ((v1).m < (v0).n + (v1).n) kv_resize(type, v1, (v0).n + (v1).n); \
+ memcpy((v1).a + (v1).n, (v0).a, sizeof(type) * (v0).n); \
+ (v1).n = (v0).n + (v1).n; \
+ } while (0)
+
+#endif /* AC_KVEC_H */
diff --git a/contrib/libucl/lua_ucl.c b/contrib/libucl/lua_ucl.c
new file mode 100644
index 0000000..c2e39c4
--- /dev/null
+++ b/contrib/libucl/lua_ucl.c
@@ -0,0 +1,1574 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @file lua ucl bindings
+ */
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "lua_ucl.h"
+#include <strings.h>
+
+/***
+ * @module ucl
+ * This lua module allows to parse objects from strings and to store data into
+ * ucl objects. It uses `libucl` C library to parse and manipulate with ucl objects.
+ * @example
+local ucl = require("ucl")
+
+local parser = ucl.parser()
+local res,err = parser:parse_string('{key=value}')
+
+if not res then
+ print('parser error: ' .. err)
+else
+ local obj = parser:get_object()
+ local got = ucl.to_format(obj, 'json')
+endif
+
+local table = {
+ str = 'value',
+ num = 100500,
+ null = ucl.null,
+ func = function ()
+ return 'huh'
+ end
+}
+
+print(ucl.to_format(table, 'ucl'))
+-- Output:
+--[[
+num = 100500;
+str = "value";
+null = null;
+func = "huh";
+--]]
+ */
+
+#define PARSER_META "ucl.parser.meta"
+#define EMITTER_META "ucl.emitter.meta"
+#define NULL_META "ucl.null.meta"
+#define OBJECT_META "ucl.object.meta"
+#define UCL_OBJECT_TYPE_META "ucl.type.object"
+#define UCL_ARRAY_TYPE_META "ucl.type.array"
+#define UCL_IMPL_ARRAY_TYPE_META "ucl.type.impl_array"
+
+static int ucl_object_lua_push_array (lua_State *L, const ucl_object_t *obj, int flags);
+static int ucl_object_lua_push_scalar (lua_State *L, const ucl_object_t *obj, int flags);
+static int ucl_object_push_lua_common (lua_State *L, const ucl_object_t *obj, int flags);
+static ucl_object_t* ucl_object_lua_fromtable (lua_State *L, int idx, ucl_string_flags_t flags);
+static ucl_object_t* ucl_object_lua_fromelt (lua_State *L, int idx, ucl_string_flags_t flags);
+
+static void *ucl_null;
+
+struct _rspamd_lua_text {
+ const char *start;
+ unsigned int len;
+ unsigned int flags;
+};
+
+enum lua_ucl_push_flags {
+ LUA_UCL_DEFAULT_FLAGS = 0,
+ LUA_UCL_ALLOW_ARRAY = (1u << 0u),
+ LUA_UCL_CONVERT_NIL = (1u << 1u),
+};
+
+/**
+ * Push a single element of an object to lua
+ * @param L
+ * @param key
+ * @param obj
+ */
+static void
+ucl_object_lua_push_element (lua_State *L, const char *key,
+ const ucl_object_t *obj, int flags)
+{
+ lua_pushstring (L, key);
+ ucl_object_push_lua_common (L, obj, flags|LUA_UCL_ALLOW_ARRAY);
+ lua_settable (L, -3);
+}
+
+static void
+lua_ucl_userdata_dtor (void *ud)
+{
+ struct ucl_lua_funcdata *fd = (struct ucl_lua_funcdata *)ud;
+
+ luaL_unref (fd->L, LUA_REGISTRYINDEX, fd->idx);
+ if (fd->ret != NULL) {
+ free (fd->ret);
+ }
+ free (fd);
+}
+
+static const char *
+lua_ucl_userdata_emitter (void *ud)
+{
+ struct ucl_lua_funcdata *fd = (struct ucl_lua_funcdata *)ud;
+ const char *out = "";
+
+ lua_rawgeti (fd->L, LUA_REGISTRYINDEX, fd->idx);
+
+ lua_pcall (fd->L, 0, 1, 0);
+ out = lua_tostring (fd->L, -1);
+
+ if (out != NULL) {
+ /* We need to store temporary string in a more appropriate place */
+ if (fd->ret) {
+ free (fd->ret);
+ }
+ fd->ret = strdup (out);
+ }
+
+ lua_settop (fd->L, 0);
+
+ return fd->ret;
+}
+
+/**
+ * Push a single object to lua
+ * @param L
+ * @param obj
+ * @return
+ */
+static int
+ucl_object_lua_push_object (lua_State *L, const ucl_object_t *obj,
+ int flags)
+{
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+
+ if ((flags & LUA_UCL_ALLOW_ARRAY) && obj->next != NULL) {
+ /* Actually we need to push this as an array */
+ return ucl_object_lua_push_array (L, obj, flags);
+ }
+
+ lua_createtable (L, 0, obj->len);
+ it = NULL;
+
+ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
+ ucl_object_lua_push_element (L, ucl_object_key (cur), cur, flags);
+ }
+
+ luaL_getmetatable (L, UCL_OBJECT_TYPE_META);
+ lua_setmetatable (L, -2);
+
+ return 1;
+}
+
+/**
+ * Push an array to lua as table indexed by integers
+ * @param L
+ * @param obj
+ * @return
+ */
+static int
+ucl_object_lua_push_array (lua_State *L, const ucl_object_t *obj, int flags)
+{
+ const ucl_object_t *cur;
+ ucl_object_iter_t it;
+ int i = 1, nelt = 0;
+
+ if (obj->type == UCL_ARRAY) {
+ nelt = obj->len;
+ it = ucl_object_iterate_new (obj);
+ lua_createtable (L, nelt, 0);
+
+ while ((cur = ucl_object_iterate_safe (it, true))) {
+ ucl_object_push_lua (L, cur, (flags & ~LUA_UCL_ALLOW_ARRAY));
+ lua_rawseti (L, -2, i);
+ i ++;
+ }
+
+ luaL_getmetatable (L, UCL_ARRAY_TYPE_META);
+ lua_setmetatable (L, -2);
+
+ ucl_object_iterate_free (it);
+ }
+ else {
+ /* Optimize allocation by preallocation of table */
+ LL_FOREACH (obj, cur) {
+ nelt ++;
+ }
+
+ lua_createtable (L, nelt, 0);
+
+ LL_FOREACH (obj, cur) {
+ ucl_object_push_lua (L, cur, (flags & ~LUA_UCL_ALLOW_ARRAY));
+ lua_rawseti (L, -2, i);
+ i ++;
+ }
+
+ luaL_getmetatable (L, UCL_IMPL_ARRAY_TYPE_META);
+ lua_setmetatable (L, -2);
+ }
+
+ return 1;
+}
+
+/**
+ * Push a simple object to lua depending on its actual type
+ */
+static int
+ucl_object_lua_push_scalar (lua_State *L, const ucl_object_t *obj,
+ int flags)
+{
+ struct ucl_lua_funcdata *fd;
+
+ if ((flags & LUA_UCL_ALLOW_ARRAY) && obj->next != NULL) {
+ /* Actually we need to push this as an array */
+ return ucl_object_lua_push_array (L, obj, flags);
+ }
+
+ switch (obj->type) {
+ case UCL_BOOLEAN:
+ lua_pushboolean (L, ucl_obj_toboolean (obj));
+ break;
+ case UCL_STRING:
+ lua_pushlstring (L, ucl_obj_tostring (obj), obj->len);
+ break;
+ case UCL_INT:
+#if LUA_VERSION_NUM >= 501
+ lua_pushinteger (L, ucl_obj_toint (obj));
+#else
+ lua_pushnumber (L, ucl_obj_toint (obj));
+#endif
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ lua_pushnumber (L, ucl_obj_todouble (obj));
+ break;
+ case UCL_NULL:
+ if (flags & LUA_UCL_CONVERT_NIL) {
+ lua_pushboolean (L, false);
+ }
+ else {
+ lua_getfield (L, LUA_REGISTRYINDEX, "ucl.null");
+ }
+ break;
+ case UCL_USERDATA:
+ fd = (struct ucl_lua_funcdata *)obj->value.ud;
+ lua_rawgeti (L, LUA_REGISTRYINDEX, fd->idx);
+ break;
+ default:
+ lua_pushnil (L);
+ break;
+ }
+
+ return 1;
+}
+
+static int
+ucl_object_push_lua_common (lua_State *L, const ucl_object_t *obj, int flags)
+{
+ switch (obj->type) {
+ case UCL_OBJECT:
+ return ucl_object_lua_push_object (L, obj, flags);
+ case UCL_ARRAY:
+ return ucl_object_lua_push_array (L, obj, flags);
+ default:
+ return ucl_object_lua_push_scalar (L, obj, flags);
+ }
+}
+
+/***
+ * @function ucl_object_push_lua(L, obj, allow_array)
+ * This is a `C` function to push `UCL` object as lua variable. This function
+ * converts `obj` to lua representation using the following conversions:
+ *
+ * - *scalar* values are directly presented by lua objects
+ * - *userdata* values are converted to lua function objects using `LUA_REGISTRYINDEX`,
+ * this can be used to pass functions from lua to c and vice-versa
+ * - *arrays* are converted to lua tables with numeric indicies suitable for `ipairs` iterations
+ * - *objects* are converted to lua tables with string indicies
+ * @param {lua_State} L lua state pointer
+ * @param {ucl_object_t} obj object to push
+ * @param {bool} allow_array expand implicit arrays (should be true for all but partial arrays)
+ * @return {int} `1` if an object is pushed to lua
+ */
+int
+ucl_object_push_lua (lua_State *L, const ucl_object_t *obj, bool allow_array)
+{
+ return ucl_object_push_lua_common (L, obj,
+ allow_array ? LUA_UCL_ALLOW_ARRAY : LUA_UCL_DEFAULT_FLAGS);
+}
+
+int
+ucl_object_push_lua_filter_nil (lua_State *L, const ucl_object_t *obj, bool allow_array)
+{
+ return ucl_object_push_lua_common (L, obj,
+ allow_array ? (LUA_UCL_ALLOW_ARRAY|LUA_UCL_CONVERT_NIL) :
+ (LUA_UCL_DEFAULT_FLAGS|LUA_UCL_CONVERT_NIL));
+}
+
+/**
+ * Parse lua table into object top
+ * @param L
+ * @param top
+ * @param idx
+ */
+static ucl_object_t *
+ucl_object_lua_fromtable (lua_State *L, int idx, ucl_string_flags_t flags)
+{
+ ucl_object_t *obj, *top = NULL, *cur;
+ size_t keylen;
+ const char *k;
+ bool is_array = true, is_implicit = false, found_mt = false;
+ size_t max = 0, nelts = 0;
+
+ if (idx < 0) {
+ /* For negative indicies we want to invert them */
+ idx = lua_gettop (L) + idx + 1;
+ }
+
+ /* First, we check from metatable */
+ if (luaL_getmetafield (L, idx, "class") != 0) {
+
+ if (lua_type (L, -1) == LUA_TSTRING) {
+ const char *classname = lua_tostring (L, -1);
+
+ if (strcmp (classname, UCL_OBJECT_TYPE_META) == 0) {
+ is_array = false;
+ found_mt = true;
+ } else if (strcmp (classname, UCL_ARRAY_TYPE_META) == 0) {
+ is_array = true;
+ found_mt = true;
+#if LUA_VERSION_NUM >= 502
+ max = lua_rawlen (L, idx);
+#else
+ max = lua_objlen (L, idx);
+#endif
+ nelts = max;
+ } else if (strcmp (classname, UCL_IMPL_ARRAY_TYPE_META) == 0) {
+ is_array = true;
+ is_implicit = true;
+ found_mt = true;
+#if LUA_VERSION_NUM >= 502
+ max = lua_rawlen (L, idx);
+#else
+ max = lua_objlen (L, idx);
+#endif
+ nelts = max;
+ }
+ }
+
+ lua_pop (L, 1);
+ }
+
+ if (!found_mt) {
+ /* Check for array (it is all inefficient) */
+ lua_pushnil (L);
+
+ while (lua_next (L, idx) != 0) {
+ lua_pushvalue (L, -2);
+
+ if (lua_type (L, -1) == LUA_TNUMBER) {
+ double num = lua_tonumber (L, -1);
+ if (num == (int) num) {
+ if (num > max) {
+ max = num;
+ }
+ }
+ else {
+ /* Keys are not integer */
+ is_array = false;
+ }
+ }
+ else {
+ /* Keys are not numeric */
+ is_array = false;
+ }
+
+ lua_pop (L, 2);
+ nelts ++;
+ }
+ }
+
+ /* Table iterate */
+ if (is_array) {
+ int i;
+
+ if (!is_implicit) {
+ top = ucl_object_typed_new (UCL_ARRAY);
+ ucl_object_reserve (top, nelts);
+ }
+ else {
+ top = NULL;
+ }
+
+ for (i = 1; i <= max; i ++) {
+ lua_pushinteger (L, i);
+ lua_gettable (L, idx);
+
+ obj = ucl_object_lua_fromelt (L, lua_gettop (L), flags);
+
+ if (obj != NULL) {
+ if (is_implicit) {
+ DL_APPEND (top, obj);
+ }
+ else {
+ ucl_array_append (top, obj);
+ }
+ }
+ lua_pop (L, 1);
+ }
+ }
+ else {
+ lua_pushnil (L);
+ top = ucl_object_typed_new (UCL_OBJECT);
+ ucl_object_reserve (top, nelts);
+
+ while (lua_next (L, idx) != 0) {
+ /* copy key to avoid modifications */
+ lua_pushvalue (L, -2);
+ k = lua_tolstring (L, -1, &keylen);
+ obj = ucl_object_lua_fromelt (L, lua_gettop (L) - 1, flags);
+
+ if (obj != NULL) {
+ ucl_object_insert_key (top, obj, k, keylen, true);
+
+ DL_FOREACH (obj, cur) {
+ if (cur->keylen == 0) {
+ cur->keylen = obj->keylen;
+ cur->key = obj->key;
+ }
+ }
+ }
+ lua_pop (L, 2);
+ }
+ }
+
+ return top;
+}
+
+/**
+ * Get a single element from lua to object obj
+ * @param L
+ * @param obj
+ * @param idx
+ */
+static ucl_object_t *
+ucl_object_lua_fromelt (lua_State *L, int idx, ucl_string_flags_t flags)
+{
+ int type;
+ double num;
+ ucl_object_t *obj = NULL;
+ struct ucl_lua_funcdata *fd;
+ const char *str;
+ size_t sz;
+
+ type = lua_type (L, idx);
+
+ switch (type) {
+ case LUA_TSTRING:
+ str = lua_tolstring (L, idx, &sz);
+
+ if (str) {
+ /*
+ * ucl_object_fromstring_common has a `logic` to use strlen if sz is zero
+ * which is totally broken...
+ */
+ if (sz > 0) {
+ obj = ucl_object_fromstring_common(str, sz, flags);
+ }
+ else {
+ obj = ucl_object_fromstring_common("", sz, flags);
+ }
+ }
+ else {
+ obj = ucl_object_typed_new (UCL_NULL);
+ }
+ break;
+ case LUA_TNUMBER:
+ num = lua_tonumber (L, idx);
+ if (num == (int64_t)num) {
+ obj = ucl_object_fromint (num);
+ }
+ else {
+ obj = ucl_object_fromdouble (num);
+ }
+ break;
+ case LUA_TBOOLEAN:
+ obj = ucl_object_frombool (lua_toboolean (L, idx));
+ break;
+ case LUA_TUSERDATA:
+ if (lua_topointer (L, idx) == ucl_null) {
+ obj = ucl_object_typed_new (UCL_NULL);
+ }
+ else {
+ /* Assume it is a text like object */
+ struct _rspamd_lua_text *t = lua_touserdata (L, idx);
+
+ if (t) {
+ if (t->len >0) {
+ obj = ucl_object_fromstring_common(t->start, t->len, 0);
+ }
+ else {
+ obj = ucl_object_fromstring_common("", 0, 0);
+ }
+
+ /* Binary text */
+ if (t->flags & (1u << 5u)) {
+ obj->flags |= UCL_OBJECT_BINARY;
+ }
+ }
+ }
+ break;
+ case LUA_TTABLE:
+ case LUA_TFUNCTION:
+ case LUA_TTHREAD:
+ if (luaL_getmetafield (L, idx, "__gen_ucl")) {
+ if (lua_isfunction (L, -1)) {
+ lua_settop (L, 3); /* gen, obj, func */
+ lua_insert (L, 1); /* func, gen, obj */
+ lua_insert (L, 2); /* func, obj, gen */
+ lua_call(L, 2, 1);
+ obj = ucl_object_lua_fromelt (L, 1, flags);
+ }
+ lua_pop (L, 2);
+ }
+ else {
+ if (type == LUA_TTABLE) {
+ obj = ucl_object_lua_fromtable (L, idx, flags);
+ }
+ else if (type == LUA_TFUNCTION) {
+ fd = malloc (sizeof (*fd));
+ if (fd != NULL) {
+ lua_pushvalue (L, idx);
+ fd->L = L;
+ fd->ret = NULL;
+ fd->idx = luaL_ref (L, LUA_REGISTRYINDEX);
+
+ obj = ucl_object_new_userdata (lua_ucl_userdata_dtor,
+ lua_ucl_userdata_emitter, (void *)fd);
+ }
+ }
+ }
+ break;
+ }
+
+ return obj;
+}
+
+/**
+ * @function ucl_object_lua_import(L, idx)
+ * Extracts ucl object from lua variable at `idx` position,
+ * @see ucl_object_push_lua for conversion definitions
+ * @param {lua_state} L lua state machine pointer
+ * @param {int} idx index where the source variable is placed
+ * @return {ucl_object_t} new ucl object extracted from lua variable. Reference count of this object is 1,
+ * this object thus needs to be unref'ed after usage.
+ */
+ucl_object_t *
+ucl_object_lua_import (lua_State *L, int idx)
+{
+ ucl_object_t *obj;
+ int t;
+
+ t = lua_type (L, idx);
+ switch (t) {
+ case LUA_TTABLE:
+ obj = ucl_object_lua_fromtable (L, idx, UCL_STRING_RAW);
+ break;
+ default:
+ obj = ucl_object_lua_fromelt (L, idx, UCL_STRING_RAW);
+ break;
+ }
+
+ return obj;
+}
+
+/**
+ * @function ucl_object_lua_import_escape(L, idx)
+ * Extracts ucl object from lua variable at `idx` position escaping JSON strings
+ * @see ucl_object_push_lua for conversion definitions
+ * @param {lua_state} L lua state machine pointer
+ * @param {int} idx index where the source variable is placed
+ * @return {ucl_object_t} new ucl object extracted from lua variable. Reference count of this object is 1,
+ * this object thus needs to be unref'ed after usage.
+ */
+ucl_object_t *
+ucl_object_lua_import_escape (lua_State *L, int idx)
+{
+ ucl_object_t *obj;
+ int t;
+
+ t = lua_type (L, idx);
+ switch (t) {
+ case LUA_TTABLE:
+ obj = ucl_object_lua_fromtable (L, idx, UCL_STRING_ESCAPE);
+ break;
+ default:
+ obj = ucl_object_lua_fromelt (L, idx, UCL_STRING_ESCAPE);
+ break;
+ }
+
+ return obj;
+}
+
+static int
+lua_ucl_to_string (lua_State *L, const ucl_object_t *obj, enum ucl_emitter type)
+{
+ unsigned char *result;
+ size_t outlen;
+
+ result = ucl_object_emit_len (obj, type, &outlen);
+
+ if (result != NULL) {
+ lua_pushlstring (L, (const char *)result, outlen);
+ free (result);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static int
+lua_ucl_parser_init (lua_State *L)
+{
+ struct ucl_parser *parser, **pparser;
+ int flags = UCL_PARSER_NO_FILEVARS;
+
+ if (lua_gettop (L) >= 1) {
+ flags = lua_tonumber (L, 1);
+ }
+
+ parser = ucl_parser_new (flags);
+ if (parser == NULL) {
+ lua_pushnil (L);
+ }
+
+ pparser = lua_newuserdata (L, sizeof (parser));
+ *pparser = parser;
+ luaL_getmetatable (L, PARSER_META);
+ lua_setmetatable (L, -2);
+
+ return 1;
+}
+
+static struct ucl_parser *
+lua_ucl_parser_get (lua_State *L, int index)
+{
+ return *((struct ucl_parser **) luaL_checkudata(L, index, PARSER_META));
+}
+
+static ucl_object_t *
+lua_ucl_object_get (lua_State *L, int index)
+{
+ return *((ucl_object_t **) luaL_checkudata(L, index, OBJECT_META));
+}
+
+static void
+lua_ucl_push_opaque (lua_State *L, ucl_object_t *obj)
+{
+ ucl_object_t **pobj;
+
+ pobj = lua_newuserdata (L, sizeof (*pobj));
+ *pobj = obj;
+ luaL_getmetatable (L, OBJECT_META);
+ lua_setmetatable (L, -2);
+}
+
+static inline enum ucl_parse_type
+lua_ucl_str_to_parse_type (const char *str)
+{
+ enum ucl_parse_type type = UCL_PARSE_UCL;
+
+ if (str != NULL) {
+ if (strcasecmp (str, "msgpack") == 0) {
+ type = UCL_PARSE_MSGPACK;
+ }
+ else if (strcasecmp (str, "sexp") == 0 ||
+ strcasecmp (str, "csexp") == 0) {
+ type = UCL_PARSE_CSEXP;
+ }
+ else if (strcasecmp (str, "auto") == 0) {
+ type = UCL_PARSE_AUTO;
+ }
+ }
+
+ return type;
+}
+
+/***
+ * @method parser:parse_file(name)
+ * Parse UCL object from file.
+ * @param {string} name filename to parse
+ * @return {bool[, string]} if res is `true` then file has been parsed successfully, otherwise an error string is also returned
+@example
+local parser = ucl.parser()
+local res,err = parser:parse_file('/some/file.conf')
+
+if not res then
+ print('parser error: ' .. err)
+else
+ -- Do something with object
+end
+ */
+static int
+lua_ucl_parser_parse_file (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *file;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+ file = luaL_checkstring (L, 2);
+
+ if (parser != NULL && file != NULL) {
+ if (ucl_parser_add_file (parser, file)) {
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, ucl_parser_get_error (parser));
+ }
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:register_variable(name, value)
+ * Register parser variable
+ * @param {string} name name of variable
+ * @param {string} value value of variable
+ * @return {bool} success
+@example
+local parser = ucl.parser()
+local res = parser:register_variable('CONFDIR', '/etc/foo')
+ */
+static int
+lua_ucl_parser_register_variable (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *name, *value;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+ name = luaL_checkstring (L, 2);
+ value = luaL_checkstring (L, 3);
+
+ if (parser != NULL && name != NULL && value != NULL) {
+ ucl_parser_register_variable (parser, name, value);
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:register_variables(vars)
+ * Register parser variables
+ * @param {table} vars names/values of variables
+ * @return {bool} success
+@example
+local parser = ucl.parser()
+local res = parser:register_variables({CONFDIR = '/etc/foo', VARDIR = '/var'})
+ */
+static int
+lua_ucl_parser_register_variables (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *name, *value;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+
+ if (parser != NULL && lua_type (L, 2) == LUA_TTABLE) {
+ for (lua_pushnil (L); lua_next (L, 2); lua_pop (L, 1)) {
+ lua_pushvalue (L, -2);
+ name = luaL_checkstring (L, -1);
+ value = luaL_checkstring (L, -2);
+ ucl_parser_register_variable (parser, name, value);
+ lua_pop (L, 1);
+ }
+
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:parse_string(input)
+ * Parse UCL object from file.
+ * @param {string} input string to parse
+ * @return {bool[, string]} if res is `true` then file has been parsed successfully, otherwise an error string is also returned
+ */
+static int
+lua_ucl_parser_parse_string (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *string;
+ size_t llen;
+ enum ucl_parse_type type = UCL_PARSE_UCL;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+ string = luaL_checklstring (L, 2, &llen);
+
+ if (lua_type (L, 3) == LUA_TSTRING) {
+ type = lua_ucl_str_to_parse_type (lua_tostring (L, 3));
+ }
+
+ if (parser != NULL && string != NULL) {
+ if (ucl_parser_add_chunk_full (parser, (const unsigned char *)string,
+ llen, 0, UCL_DUPLICATE_APPEND, type)) {
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, ucl_parser_get_error (parser));
+ }
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:parse_text(input)
+ * Parse UCL object from text object (Rspamd specific).
+ * @param {rspamd_text} input text to parse
+ * @return {bool[, string]} if res is `true` then file has been parsed successfully, otherwise an error string is also returned
+ */
+static int
+lua_ucl_parser_parse_text (lua_State *L)
+{
+ struct ucl_parser *parser;
+ struct _rspamd_lua_text *t;
+ enum ucl_parse_type type = UCL_PARSE_UCL;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+
+ if (lua_type (L, 2) == LUA_TUSERDATA) {
+ t = lua_touserdata (L, 2);
+ }
+ else if (lua_type (L, 2) == LUA_TSTRING) {
+ const gchar *s;
+ gsize len;
+ static struct _rspamd_lua_text st_t;
+
+ s = lua_tolstring (L, 2, &len);
+ st_t.start = s;
+ st_t.len = len;
+
+ t = &st_t;
+ }
+ else {
+ return luaL_error(L, "invalid argument as input, expected userdata or a string");
+ }
+
+ if (lua_type (L, 3) == LUA_TSTRING) {
+ type = lua_ucl_str_to_parse_type (lua_tostring (L, 3));
+ }
+
+ if (parser != NULL && t != NULL) {
+ if (ucl_parser_add_chunk_full (parser, (const unsigned char *)t->start,
+ t->len, 0, UCL_DUPLICATE_APPEND, type)) {
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, ucl_parser_get_error (parser));
+ }
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:get_object()
+ * Get top object from parser and export it to lua representation.
+ * @return {variant or nil} ucl object as lua native variable
+ */
+static int
+lua_ucl_parser_get_object (lua_State *L)
+{
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ int ret = 1;
+
+ parser = lua_ucl_parser_get (L, 1);
+ obj = ucl_parser_get_object (parser);
+
+ if (obj != NULL) {
+ ret = ucl_object_push_lua (L, obj, false);
+ /* no need to keep reference */
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:get_object_wrapped()
+ * Get top object from parser and export it to userdata object without
+ * unwrapping to lua.
+ * @return {ucl.object or nil} ucl object wrapped variable
+ */
+static int
+lua_ucl_parser_get_object_wrapped (lua_State *L)
+{
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ int ret = 1;
+
+ parser = lua_ucl_parser_get (L, 1);
+ obj = ucl_parser_get_object (parser);
+
+ if (obj != NULL) {
+ lua_ucl_push_opaque (L, obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:validate(schema)
+ * Validates the top object in the parser against schema. Schema might be
+ * another object or a string that represents file to load schema from.
+ *
+ * @param {string/table} schema input schema
+ * @return {result,err} two values: boolean result and the corresponding error
+ *
+ */
+static int
+lua_ucl_parser_validate (lua_State *L)
+{
+ struct ucl_parser *parser, *schema_parser;
+ ucl_object_t *schema;
+ const char *schema_file;
+ struct ucl_schema_error err;
+
+ parser = lua_ucl_parser_get (L, 1);
+
+ if (parser && parser->top_obj) {
+ if (lua_type (L, 2) == LUA_TTABLE) {
+ schema = ucl_object_lua_import (L, 2);
+
+ if (schema == NULL) {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "cannot load schema from lua table");
+
+ return 2;
+ }
+ }
+ else if (lua_type (L, 2) == LUA_TSTRING) {
+ schema_parser = ucl_parser_new (0);
+ schema_file = luaL_checkstring (L, 2);
+
+ if (!ucl_parser_add_file (schema_parser, schema_file)) {
+ lua_pushboolean (L, false);
+ lua_pushfstring (L, "cannot parse schema file \"%s\": "
+ "%s", schema_file, ucl_parser_get_error (parser));
+ ucl_parser_free (schema_parser);
+
+ return 2;
+ }
+
+ schema = ucl_parser_get_object (schema_parser);
+ ucl_parser_free (schema_parser);
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid schema argument");
+
+ return 2;
+ }
+
+ if (!ucl_object_validate (schema, parser->top_obj, &err)) {
+ lua_pushboolean (L, false);
+ lua_pushfstring (L, "validation error: "
+ "%s", err.msg);
+ }
+ else {
+ lua_pushboolean (L, true);
+ lua_pushnil (L);
+ }
+
+ ucl_object_unref (schema);
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid parser or empty top object");
+ }
+
+ return 2;
+}
+
+static int
+lua_ucl_parser_gc (lua_State *L)
+{
+ struct ucl_parser *parser;
+
+ parser = lua_ucl_parser_get (L, 1);
+ ucl_parser_free (parser);
+
+ return 0;
+}
+
+/***
+ * @method object:unwrap()
+ * Unwraps opaque ucl object to the native lua object (performing copying)
+ * @return {variant} any lua object
+ */
+static int
+lua_ucl_object_unwrap (lua_State *L)
+{
+ ucl_object_t *obj;
+
+ obj = lua_ucl_object_get (L, 1);
+
+ if (obj) {
+ ucl_object_push_lua (L, obj, true);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static inline enum ucl_emitter
+lua_ucl_str_to_emit_type (const char *strtype)
+{
+ enum ucl_emitter format = UCL_EMIT_JSON_COMPACT;
+
+ if (strcasecmp (strtype, "json") == 0) {
+ format = UCL_EMIT_JSON;
+ }
+ else if (strcasecmp (strtype, "json-compact") == 0) {
+ format = UCL_EMIT_JSON_COMPACT;
+ }
+ else if (strcasecmp (strtype, "yaml") == 0) {
+ format = UCL_EMIT_YAML;
+ }
+ else if (strcasecmp (strtype, "config") == 0 ||
+ strcasecmp (strtype, "ucl") == 0) {
+ format = UCL_EMIT_CONFIG;
+ }
+
+ return format;
+}
+
+/***
+ * @method object:tostring(type)
+ * Unwraps opaque ucl object to string (json by default). Optionally you can
+ * specify output format:
+ *
+ * - `json` - fine printed json
+ * - `json-compact` - compacted json
+ * - `config` - fine printed configuration
+ * - `ucl` - same as `config`
+ * - `yaml` - embedded yaml
+ * @param {string} type optional
+ * @return {string} string representation of the opaque ucl object
+ */
+static int
+lua_ucl_object_tostring (lua_State *L)
+{
+ ucl_object_t *obj;
+ enum ucl_emitter format = UCL_EMIT_JSON_COMPACT;
+
+ obj = lua_ucl_object_get (L, 1);
+
+ if (obj) {
+ if (lua_gettop (L) > 1) {
+ if (lua_type (L, 2) == LUA_TSTRING) {
+ const char *strtype = lua_tostring (L, 2);
+
+ format = lua_ucl_str_to_emit_type (strtype);
+ }
+ }
+
+ return lua_ucl_to_string (L, obj, format);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method object:validate(schema[, path[, ext_refs]])
+ * Validates the given ucl object using schema object represented as another
+ * opaque ucl object. You can also specify path in the form `#/path/def` to
+ * specify the specific schema element to perform validation.
+ *
+ * @param {ucl.object} schema schema object
+ * @param {string} path optional path for validation procedure
+ * @return {result,err} two values: boolean result and the corresponding
+ * error, if `ext_refs` are also specified, then they are returned as opaque
+ * ucl object as {result,err,ext_refs}
+ */
+static int
+lua_ucl_object_validate (lua_State *L)
+{
+ ucl_object_t *obj, *schema, *ext_refs = NULL;
+ const ucl_object_t *schema_elt;
+ bool res = false;
+ struct ucl_schema_error err;
+ const char *path = NULL;
+
+ obj = lua_ucl_object_get (L, 1);
+ schema = lua_ucl_object_get (L, 2);
+
+ if (schema && obj && ucl_object_type (schema) == UCL_OBJECT) {
+ if (lua_gettop (L) > 2) {
+ if (lua_type (L, 3) == LUA_TSTRING) {
+ path = lua_tostring (L, 3);
+ if (path[0] == '#') {
+ path++;
+ }
+ }
+ else if (lua_type (L, 3) == LUA_TUSERDATA || lua_type (L, 3) ==
+ LUA_TTABLE) {
+ /* External refs */
+ ext_refs = lua_ucl_object_get (L, 3);
+ }
+
+ if (lua_gettop (L) > 3) {
+ if (lua_type (L, 4) == LUA_TUSERDATA || lua_type (L, 4) ==
+ LUA_TTABLE) {
+ /* External refs */
+ ext_refs = lua_ucl_object_get (L, 4);
+ }
+ }
+ }
+
+ if (path) {
+ schema_elt = ucl_object_lookup_path_char (schema, path, '/');
+ }
+ else {
+ /* Use the top object */
+ schema_elt = schema;
+ }
+
+ if (schema_elt) {
+ res = ucl_object_validate_root_ext (schema_elt, obj, schema,
+ ext_refs, &err);
+
+ if (res) {
+ lua_pushboolean (L, res);
+ lua_pushnil (L);
+
+ if (ext_refs) {
+ lua_ucl_push_opaque (L, ext_refs);
+ }
+ }
+ else {
+ lua_pushboolean (L, res);
+ lua_pushfstring (L, "validation error: %s", err.msg);
+
+ if (ext_refs) {
+ lua_ucl_push_opaque (L, ext_refs);
+ }
+ }
+ }
+ else {
+ lua_pushboolean (L, res);
+
+ lua_pushfstring (L, "cannot find the requested path: %s", path);
+
+ if (ext_refs) {
+ lua_ucl_push_opaque (L, ext_refs);
+ }
+ }
+ }
+ else {
+ lua_pushboolean (L, res);
+ lua_pushstring (L, "invalid object or schema");
+ }
+
+ if (ext_refs) {
+ return 3;
+ }
+
+ return 2;
+}
+
+static int
+lua_ucl_object_gc (lua_State *L)
+{
+ ucl_object_t *obj;
+
+ obj = lua_ucl_object_get (L, 1);
+
+ ucl_object_unref (obj);
+
+ return 0;
+}
+
+static void
+lua_ucl_parser_mt (lua_State *L)
+{
+ luaL_newmetatable (L, PARSER_META);
+
+ lua_pushvalue(L, -1);
+ lua_setfield(L, -2, "__index");
+
+ lua_pushcfunction (L, lua_ucl_parser_gc);
+ lua_setfield (L, -2, "__gc");
+
+ lua_pushcfunction (L, lua_ucl_parser_parse_file);
+ lua_setfield (L, -2, "parse_file");
+
+ lua_pushcfunction (L, lua_ucl_parser_parse_string);
+ lua_setfield (L, -2, "parse_string");
+
+ lua_pushcfunction (L, lua_ucl_parser_parse_text);
+ lua_setfield (L, -2, "parse_text");
+
+ lua_pushcfunction (L, lua_ucl_parser_register_variable);
+ lua_setfield (L, -2, "register_variable");
+
+ lua_pushcfunction (L, lua_ucl_parser_register_variables);
+ lua_setfield (L, -2, "register_variables");
+
+ lua_pushcfunction (L, lua_ucl_parser_get_object);
+ lua_setfield (L, -2, "get_object");
+
+ lua_pushcfunction (L, lua_ucl_parser_get_object_wrapped);
+ lua_setfield (L, -2, "get_object_wrapped");
+
+ lua_pushcfunction (L, lua_ucl_parser_validate);
+ lua_setfield (L, -2, "validate");
+
+ lua_pop (L, 1);
+}
+
+static void
+lua_ucl_object_mt (lua_State *L)
+{
+ luaL_newmetatable (L, OBJECT_META);
+
+ lua_pushvalue(L, -1);
+ lua_setfield(L, -2, "__index");
+
+ lua_pushcfunction (L, lua_ucl_object_gc);
+ lua_setfield (L, -2, "__gc");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_unwrap);
+ lua_setfield (L, -2, "unwrap");
+
+ lua_pushcfunction (L, lua_ucl_object_unwrap);
+ lua_setfield (L, -2, "tolua");
+
+ lua_pushcfunction (L, lua_ucl_object_validate);
+ lua_setfield (L, -2, "validate");
+
+ lua_pushstring (L, OBJECT_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+}
+
+static void
+lua_ucl_types_mt (lua_State *L)
+{
+ luaL_newmetatable (L, UCL_OBJECT_TYPE_META);
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushstring (L, UCL_OBJECT_TYPE_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+
+ luaL_newmetatable (L, UCL_ARRAY_TYPE_META);
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushstring (L, UCL_ARRAY_TYPE_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+
+ luaL_newmetatable (L, UCL_IMPL_ARRAY_TYPE_META);
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushstring (L, UCL_IMPL_ARRAY_TYPE_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+}
+
+static int
+lua_ucl_to_json (lua_State *L)
+{
+ ucl_object_t *obj;
+ int format = UCL_EMIT_JSON;
+
+ if (lua_gettop (L) > 1) {
+ if (lua_toboolean (L, 2)) {
+ format = UCL_EMIT_JSON_COMPACT;
+ }
+ }
+
+ obj = ucl_object_lua_import (L, 1);
+ if (obj != NULL) {
+ lua_ucl_to_string (L, obj, format);
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static int
+lua_ucl_to_config (lua_State *L)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_lua_import (L, 1);
+ if (obj != NULL) {
+ lua_ucl_to_string (L, obj, UCL_EMIT_CONFIG);
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+/***
+ * @function ucl.to_format(var, format)
+ * Converts lua variable `var` to the specified `format`. Formats supported are:
+ *
+ * - `json` - fine printed json
+ * - `json-compact` - compacted json
+ * - `config` - fine printed configuration
+ * - `ucl` - same as `config`
+ * - `yaml` - embedded yaml
+ *
+ * If `var` contains function, they are called during output formatting and if
+ * they return string value, then this value is used for output.
+ * @param {variant} var any sort of lua variable (if userdata then metafield `__to_ucl` is searched for output)
+ * @param {string} format any available format
+ * @return {string} string representation of `var` in the specific `format`.
+ * @example
+local table = {
+ str = 'value',
+ num = 100500,
+ null = ucl.null,
+ func = function ()
+ return 'huh'
+ end
+}
+
+print(ucl.to_format(table, 'ucl'))
+-- Output:
+--[[
+num = 100500;
+str = "value";
+null = null;
+func = "huh";
+--]]
+ */
+static int
+lua_ucl_to_format (lua_State *L)
+{
+ ucl_object_t *obj;
+ int format = UCL_EMIT_JSON;
+ bool sort = false;
+
+ if (lua_gettop (L) > 1) {
+ if (lua_type (L, 2) == LUA_TNUMBER) {
+ format = lua_tonumber (L, 2);
+ if (format < 0 || format >= UCL_EMIT_YAML) {
+ lua_pushnil (L);
+ return 1;
+ }
+ }
+ else if (lua_type (L, 2) == LUA_TSTRING) {
+ const char *strtype = lua_tostring (L, 2);
+
+ if (strcasecmp (strtype, "json") == 0) {
+ format = UCL_EMIT_JSON;
+ }
+ else if (strcasecmp (strtype, "json-compact") == 0) {
+ format = UCL_EMIT_JSON_COMPACT;
+ }
+ else if (strcasecmp (strtype, "yaml") == 0) {
+ format = UCL_EMIT_YAML;
+ }
+ else if (strcasecmp (strtype, "config") == 0 ||
+ strcasecmp (strtype, "ucl") == 0) {
+ format = UCL_EMIT_CONFIG;
+ }
+ else if (strcasecmp (strtype, "msgpack") == 0 ||
+ strcasecmp (strtype, "messagepack") == 0) {
+ format = UCL_EMIT_MSGPACK;
+ }
+ }
+
+ if (lua_isboolean (L, 3)) {
+ sort = lua_toboolean (L, 3);
+ }
+ }
+
+ obj = ucl_object_lua_import (L, 1);
+
+ if (obj != NULL) {
+
+ if (sort) {
+ if (ucl_object_type (obj) == UCL_OBJECT) {
+ ucl_object_sort_keys (obj, UCL_SORT_KEYS_RECURSIVE);
+ }
+ }
+
+ lua_ucl_to_string (L, obj, format);
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static int
+lua_ucl_null_tostring (lua_State* L)
+{
+ lua_pushstring (L, "null");
+ return 1;
+}
+
+static void
+lua_ucl_null_mt (lua_State *L)
+{
+ luaL_newmetatable (L, NULL_META);
+
+ lua_pushcfunction (L, lua_ucl_null_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pop (L, 1);
+}
+
+int
+luaopen_ucl (lua_State *L)
+{
+ lua_ucl_parser_mt (L);
+ lua_ucl_null_mt (L);
+ lua_ucl_object_mt (L);
+ lua_ucl_types_mt (L);
+
+ /* Create the refs weak table: */
+ lua_createtable (L, 0, 2);
+ lua_pushliteral (L, "v"); /* tbl, "v" */
+ lua_setfield (L, -2, "__mode");
+ lua_pushvalue (L, -1); /* tbl, tbl */
+ lua_setmetatable (L, -2); /* tbl */
+ lua_setfield (L, LUA_REGISTRYINDEX, "ucl.refs");
+
+ lua_newtable (L);
+
+ lua_pushcfunction (L, lua_ucl_parser_init);
+ lua_setfield (L, -2, "parser");
+
+ lua_pushcfunction (L, lua_ucl_to_json);
+ lua_setfield (L, -2, "to_json");
+
+ lua_pushcfunction (L, lua_ucl_to_config);
+ lua_setfield (L, -2, "to_config");
+
+ lua_pushcfunction (L, lua_ucl_to_format);
+ lua_setfield (L, -2, "to_format");
+
+ ucl_null = lua_newuserdata (L, 0);
+ luaL_getmetatable (L, NULL_META);
+ lua_setmetatable (L, -2);
+
+ lua_pushvalue (L, -1);
+ lua_setfield (L, LUA_REGISTRYINDEX, "ucl.null");
+
+ lua_setfield (L, -2, "null");
+
+ return 1;
+}
+
+struct ucl_lua_funcdata*
+ucl_object_toclosure (const ucl_object_t *obj)
+{
+ if (obj == NULL || obj->type != UCL_USERDATA) {
+ return NULL;
+ }
+
+ return (struct ucl_lua_funcdata*)obj->value.ud;
+}
diff --git a/contrib/libucl/lua_ucl.h b/contrib/libucl/lua_ucl.h
new file mode 100644
index 0000000..4a759e3
--- /dev/null
+++ b/contrib/libucl/lua_ucl.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef LUA_UCL_H_
+#define LUA_UCL_H_
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Include C++ guard as Lua headers miss one */
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
+
+/**
+ * Closure structure for lua function storing inside UCL
+ */
+struct ucl_lua_funcdata {
+ lua_State *L;
+ int idx;
+ char *ret;
+};
+
+/**
+ * Initialize lua UCL API
+ */
+UCL_EXTERN int luaopen_ucl(lua_State *L);
+
+/**
+ * Import UCL object from lua state
+ * @param L lua state
+ * @param idx index of object at the lua stack to convert to UCL
+ * @return new UCL object or NULL, the caller should unref object after using
+ */
+UCL_EXTERN ucl_object_t *ucl_object_lua_import(lua_State *L, int idx);
+
+/**
+ * Import UCL object from lua state, escaping JSON strings
+ * @param L lua state
+ * @param idx index of object at the lua stack to convert to UCL
+ * @return new UCL object or NULL, the caller should unref object after using
+ */
+UCL_EXTERN ucl_object_t *ucl_object_lua_import_escape(lua_State *L, int idx);
+
+/**
+ * Push an object to lua
+ * @param L lua state
+ * @param obj object to push
+ * @param allow_array traverse over implicit arrays
+ */
+UCL_EXTERN int ucl_object_push_lua(lua_State *L,
+ const ucl_object_t *obj, bool allow_array);
+/**
+ * Push an object to lua replacing all ucl.null with `false`
+ * @param L lua state
+ * @param obj object to push
+ * @param allow_array traverse over implicit arrays
+ */
+UCL_EXTERN int ucl_object_push_lua_filter_nil(lua_State *L,
+ const ucl_object_t *obj,
+ bool allow_array);
+
+UCL_EXTERN struct ucl_lua_funcdata *ucl_object_toclosure(const ucl_object_t *obj);
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* LUA_UCL_H_ */
diff --git a/contrib/libucl/tree.h b/contrib/libucl/tree.h
new file mode 100644
index 0000000..404b4a8
--- /dev/null
+++ b/contrib/libucl/tree.h
@@ -0,0 +1,219 @@
+/* tree.h -- AVL trees (in the spirit of BSD's 'queue.h') -*- C -*- */
+
+/* Copyright (c) 2005 Ian Piumarta
+ *
+ * All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the 'Software'), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * provided that the above copyright notice(s) and this permission notice appear
+ * in all copies of the Software and that both the above copyright notice(s) and
+ * this permission notice appear in supporting documentation.
+ *
+ * THE SOFTWARE IS PROVIDED 'AS IS'. USE ENTIRELY AT YOUR OWN RISK.
+ */
+
+/* This file defines an AVL balanced binary tree [Georgii M. Adelson-Velskii and
+ * Evgenii M. Landis, 'An algorithm for the organization of information',
+ * Doklady Akademii Nauk SSSR, 146:263-266, 1962 (Russian). Also in Myron
+ * J. Ricci (trans.), Soviet Math, 3:1259-1263, 1962 (English)].
+ *
+ * An AVL tree is headed by pointers to the root node and to a function defining
+ * the ordering relation between nodes. Each node contains an arbitrary payload
+ * plus three fields per tree entry: the depth of the subtree for which it forms
+ * the root and two pointers to child nodes (singly-linked for minimum space, at
+ * the expense of direct access to the parent node given a pointer to one of the
+ * children). The tree is rebalanced after every insertion or removal. The
+ * tree may be traversed in two directions: forward (in-order left-to-right) and
+ * reverse (in-order, right-to-left).
+ *
+ * Because of the recursive nature of many of the operations on trees it is
+ * necessary to define a number of helper functions for each type of tree node.
+ * The macro TREE_DEFINE(node_tag, entry_name) defines these functions with
+ * unique names according to the node_tag. This macro should be invoked,
+ * thereby defining the necessary functions, once per node tag in the program.
+ *
+ * For details on the use of these macros, see the tree(3) manual page.
+ */
+
+#ifndef __tree_h
+#define __tree_h
+
+
+#define TREE_DELTA_MAX 1
+#ifndef _HU_FUNCTION
+# if defined(__GNUC__) || defined(__clang__)
+# define _HU_FUNCTION(x) __attribute__((__unused__)) x
+# else
+# define _HU_FUNCTION(x) x
+# endif
+#endif
+
+#define TREE_ENTRY(type) \
+ struct { \
+ struct type *avl_left; \
+ struct type *avl_right; \
+ int avl_height; \
+ }
+
+#define TREE_HEAD(name, type) \
+ struct name { \
+ struct type *th_root; \
+ int (*th_cmp)(struct type *lhs, struct type *rhs); \
+ }
+
+#define TREE_INITIALIZER(cmp) { 0, cmp }
+
+#define TREE_DELTA(self, field) \
+ (( (((self)->field.avl_left) ? (self)->field.avl_left->field.avl_height : 0)) \
+ - (((self)->field.avl_right) ? (self)->field.avl_right->field.avl_height : 0))
+
+/* Recursion prevents the following from being defined as macros. */
+
+#define TREE_DEFINE(node, field) \
+ \
+ static struct node *_HU_FUNCTION(TREE_BALANCE_##node##_##field)(struct node *); \
+ \
+ static struct node *_HU_FUNCTION(TREE_ROTL_##node##_##field)(struct node *self) \
+ { \
+ struct node *r= self->field.avl_right; \
+ self->field.avl_right= r->field.avl_left; \
+ r->field.avl_left= TREE_BALANCE_##node##_##field(self); \
+ return TREE_BALANCE_##node##_##field(r); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_ROTR_##node##_##field)(struct node *self) \
+ { \
+ struct node *l= self->field.avl_left; \
+ self->field.avl_left= l->field.avl_right; \
+ l->field.avl_right= TREE_BALANCE_##node##_##field(self); \
+ return TREE_BALANCE_##node##_##field(l); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_BALANCE_##node##_##field)(struct node *self) \
+ { \
+ int delta= TREE_DELTA(self, field); \
+ \
+ if (delta < -TREE_DELTA_MAX) \
+ { \
+ if (TREE_DELTA(self->field.avl_right, field) > 0) \
+ self->field.avl_right= TREE_ROTR_##node##_##field(self->field.avl_right); \
+ return TREE_ROTL_##node##_##field(self); \
+ } \
+ else if (delta > TREE_DELTA_MAX) \
+ { \
+ if (TREE_DELTA(self->field.avl_left, field) < 0) \
+ self->field.avl_left= TREE_ROTL_##node##_##field(self->field.avl_left); \
+ return TREE_ROTR_##node##_##field(self); \
+ } \
+ self->field.avl_height= 0; \
+ if (self->field.avl_left && (self->field.avl_left->field.avl_height > self->field.avl_height)) \
+ self->field.avl_height= self->field.avl_left->field.avl_height; \
+ if (self->field.avl_right && (self->field.avl_right->field.avl_height > self->field.avl_height)) \
+ self->field.avl_height= self->field.avl_right->field.avl_height; \
+ self->field.avl_height += 1; \
+ return self; \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_INSERT_##node##_##field) \
+ (struct node *self, struct node *elm, int (*compare)(struct node *lhs, struct node *rhs)) \
+ { \
+ if (!self) \
+ return elm; \
+ if (compare(elm, self) < 0) \
+ self->field.avl_left= TREE_INSERT_##node##_##field(self->field.avl_left, elm, compare); \
+ else \
+ self->field.avl_right= TREE_INSERT_##node##_##field(self->field.avl_right, elm, compare); \
+ return TREE_BALANCE_##node##_##field(self); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_FIND_##node##_##field) \
+ (struct node *self, struct node *elm, int (*compare)(struct node *lhs, struct node *rhs)) \
+ { \
+ if (!self) \
+ return 0; \
+ if (compare(elm, self) == 0) \
+ return self; \
+ if (compare(elm, self) < 0) \
+ return TREE_FIND_##node##_##field(self->field.avl_left, elm, compare); \
+ else \
+ return TREE_FIND_##node##_##field(self->field.avl_right, elm, compare); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_MOVE_RIGHT)(struct node *self, struct node *rhs) \
+ { \
+ if (!self) \
+ return rhs; \
+ self->field.avl_right= TREE_MOVE_RIGHT(self->field.avl_right, rhs); \
+ return TREE_BALANCE_##node##_##field(self); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_REMOVE_##node##_##field) \
+ (struct node *self, struct node *elm, int (*compare)(struct node *lhs, struct node *rhs)) \
+ { \
+ if (!self) return 0; \
+ \
+ if (compare(elm, self) == 0) \
+ { \
+ struct node *tmp= TREE_MOVE_RIGHT(self->field.avl_left, self->field.avl_right); \
+ self->field.avl_left= 0; \
+ self->field.avl_right= 0; \
+ return tmp; \
+ } \
+ if (compare(elm, self) < 0) \
+ self->field.avl_left= TREE_REMOVE_##node##_##field(self->field.avl_left, elm, compare); \
+ else \
+ self->field.avl_right= TREE_REMOVE_##node##_##field(self->field.avl_right, elm, compare); \
+ return TREE_BALANCE_##node##_##field(self); \
+ } \
+ \
+ static void _HU_FUNCTION(TREE_FORWARD_APPLY_ALL_##node##_##field) \
+ (struct node *self, void (*function)(struct node *node, void *data), void *data) \
+ { \
+ if (self) \
+ { \
+ TREE_FORWARD_APPLY_ALL_##node##_##field(self->field.avl_left, function, data); \
+ function(self, data); \
+ TREE_FORWARD_APPLY_ALL_##node##_##field(self->field.avl_right, function, data); \
+ } \
+ } \
+ \
+ static void _HU_FUNCTION(TREE_REVERSE_APPLY_ALL_##node##_##field) \
+ (struct node *self, void (*function)(struct node *node, void *data), void *data) \
+ { \
+ if (self) \
+ { \
+ TREE_REVERSE_APPLY_ALL_##node##_##field(self->field.avl_right, function, data); \
+ function(self, data); \
+ TREE_REVERSE_APPLY_ALL_##node##_##field(self->field.avl_left, function, data); \
+ } \
+ }
+
+#define TREE_INSERT(head, node, field, elm) \
+ ((head)->th_root= TREE_INSERT_##node##_##field((head)->th_root, (elm), (head)->th_cmp))
+
+#define TREE_FIND(head, node, field, elm) \
+ (TREE_FIND_##node##_##field((head)->th_root, (elm), (head)->th_cmp))
+
+#define TREE_REMOVE(head, node, field, elm) \
+ ((head)->th_root= TREE_REMOVE_##node##_##field((head)->th_root, (elm), (head)->th_cmp))
+
+#define TREE_DEPTH(head, field) \
+ ((head)->th_root->field.avl_height)
+
+#define TREE_FORWARD_APPLY(head, node, field, function, data) \
+ TREE_FORWARD_APPLY_ALL_##node##_##field((head)->th_root, function, data)
+
+#define TREE_REVERSE_APPLY(head, node, field, function, data) \
+ TREE_REVERSE_APPLY_ALL_##node##_##field((head)->th_root, function, data)
+
+#define TREE_INIT(head, cmp) do { \
+ (head)->th_root= 0; \
+ (head)->th_cmp= (cmp); \
+ } while (0)
+
+
+#endif /* __tree_h */
diff --git a/contrib/libucl/ucl.h b/contrib/libucl/ucl.h
new file mode 100644
index 0000000..b6b9f44
--- /dev/null
+++ b/contrib/libucl/ucl.h
@@ -0,0 +1,1650 @@
+/* Copyright (c) 2013-2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UCL_H_
+#define UCL_H_
+
+#include <string.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#ifdef _WIN32
+# define UCL_EXTERN __declspec(dllexport)
+#else
+# define UCL_EXTERN
+#endif
+
+/**
+ * @mainpage
+ * This is a reference manual for UCL API. You may find the description of UCL format by following this
+ * [github repository](https://github.com/vstakhov/libucl).
+ *
+ * This manual has several main sections:
+ * - @ref structures
+ * - @ref utils
+ * - @ref parser
+ * - @ref emitter
+ */
+
+/**
+ * @file ucl.h
+ * @brief UCL parsing and emitting functions
+ *
+ * UCL is universal configuration language, which is a form of
+ * JSON with less strict rules that make it more comfortable for
+ * using as a configuration language
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Memory allocation utilities
+ * UCL_ALLOC(size) - allocate memory for UCL
+ * UCL_FREE(size, ptr) - free memory of specified size at ptr
+ * Default: malloc and free
+ */
+#ifndef UCL_ALLOC
+#define UCL_ALLOC(size) malloc(size)
+#endif
+#ifndef UCL_FREE
+#define UCL_FREE(size, ptr) free(ptr)
+#endif
+
+#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)
+#define UCL_WARN_UNUSED_RESULT \
+ __attribute__((warn_unused_result))
+#else
+#define UCL_WARN_UNUSED_RESULT
+#endif
+
+#ifdef __GNUC__
+#define UCL_DEPRECATED(func) func __attribute__ ((deprecated))
+#elif defined(_MSC_VER)
+#define UCL_DEPRECATED(func) __declspec(deprecated) func
+#else
+#define UCL_DEPRECATED(func) func
+#endif
+
+/**
+ * @defgroup structures Structures and types
+ * UCL defines several enumeration types used for error reporting or specifying flags and attributes.
+ *
+ * @{
+ */
+
+/**
+ * The common error codes returned by ucl parser
+ */
+typedef enum ucl_error {
+ UCL_EOK = 0, /**< No error */
+ UCL_ESYNTAX, /**< Syntax error occurred during parsing */
+ UCL_EIO, /**< IO error occurred during parsing */
+ UCL_ESTATE, /**< Invalid state machine state */
+ UCL_ENESTED, /**< Input has too many recursion levels */
+ UCL_EUNPAIRED, /**< Input has too many recursion levels */
+ UCL_EMACRO, /**< Error processing a macro */
+ UCL_EINTERNAL, /**< Internal unclassified error */
+ UCL_ESSL, /**< SSL error */
+ UCL_EMERGE /**< A merge error occurred */
+} ucl_error_t;
+
+/**
+ * #ucl_object_t may have one of specified types, some types are compatible with each other and some are not.
+ * For example, you can always convert #UCL_TIME to #UCL_FLOAT. Also you can convert #UCL_FLOAT to #UCL_INTEGER
+ * by loosing floating point. Every object may be converted to a string by #ucl_object_tostring_forced() function.
+ *
+ */
+typedef enum ucl_type {
+ UCL_OBJECT = 0, /**< UCL object - key/value pairs */
+ UCL_ARRAY, /**< UCL array */
+ UCL_INT, /**< Integer number */
+ UCL_FLOAT, /**< Floating point number */
+ UCL_STRING, /**< Null terminated string */
+ UCL_BOOLEAN, /**< Boolean value */
+ UCL_TIME, /**< Time value (floating point number of seconds) */
+ UCL_USERDATA, /**< Opaque userdata pointer (may be used in macros) */
+ UCL_NULL /**< Null value */
+} ucl_type_t;
+
+/**
+ * You can use one of these types to serialise #ucl_object_t by using ucl_object_emit().
+ */
+typedef enum ucl_emitter {
+ UCL_EMIT_JSON = 0, /**< Emit fine formatted JSON */
+ UCL_EMIT_JSON_COMPACT, /**< Emit compacted JSON */
+ UCL_EMIT_CONFIG, /**< Emit human readable config format */
+ UCL_EMIT_YAML, /**< Emit embedded YAML format */
+ UCL_EMIT_MSGPACK, /**< Emit msgpack output */
+ UCL_EMIT_MAX /**< Unsupported emitter type */
+} ucl_emitter_t;
+
+/**
+ * These flags defines parser behaviour. If you specify #UCL_PARSER_ZEROCOPY you must ensure
+ * that the input memory is not freed if an object is in use. Moreover, if you want to use
+ * zero-terminated keys and string values then you should not use zero-copy mode, as in this case
+ * UCL still has to perform copying implicitly.
+ */
+typedef enum ucl_parser_flags {
+ UCL_PARSER_DEFAULT = 0, /**< No special flags */
+ UCL_PARSER_KEY_LOWERCASE = (1 << 0), /**< Convert all keys to lower case */
+ UCL_PARSER_ZEROCOPY = (1 << 1), /**< Parse input in zero-copy mode if possible */
+ UCL_PARSER_NO_TIME = (1 << 2), /**< Do not parse time and treat time values as strings */
+ UCL_PARSER_NO_IMPLICIT_ARRAYS = (1 << 3), /** Create explicit arrays instead of implicit ones */
+ UCL_PARSER_SAVE_COMMENTS = (1 << 4), /** Save comments in the parser context */
+ UCL_PARSER_DISABLE_MACRO = (1 << 5), /** Treat macros as comments */
+ UCL_PARSER_NO_FILEVARS = (1 << 6) /** Do not set file vars */
+} ucl_parser_flags_t;
+
+/**
+ * String conversion flags, that are used in #ucl_object_fromstring_common function.
+ */
+typedef enum ucl_string_flags {
+ UCL_STRING_RAW = 0x0, /**< Treat string as is */
+ UCL_STRING_ESCAPE = (1 << 0), /**< Perform JSON escape */
+ UCL_STRING_TRIM = (1 << 1), /**< Trim leading and trailing whitespaces */
+ UCL_STRING_PARSE_BOOLEAN = (1 << 2), /**< Parse passed string and detect boolean */
+ UCL_STRING_PARSE_INT = (1 << 3), /**< Parse passed string and detect integer number */
+ UCL_STRING_PARSE_DOUBLE = (1 << 4), /**< Parse passed string and detect integer or float number */
+ UCL_STRING_PARSE_TIME = (1 << 5), /**< Parse time strings */
+ UCL_STRING_PARSE_NUMBER = UCL_STRING_PARSE_INT|UCL_STRING_PARSE_DOUBLE|UCL_STRING_PARSE_TIME, /**<
+ Parse passed string and detect number */
+ UCL_STRING_PARSE = UCL_STRING_PARSE_BOOLEAN|UCL_STRING_PARSE_NUMBER, /**<
+ Parse passed string (and detect booleans and numbers) */
+ UCL_STRING_PARSE_BYTES = (1 << 6) /**< Treat numbers as bytes */
+} ucl_string_flags_t;
+
+/**
+ * Basic flags for an object (can use up to 12 bits as higher 4 bits are used
+ * for priorities)
+ */
+typedef enum ucl_object_flags {
+ UCL_OBJECT_ALLOCATED_KEY = (1 << 0), /**< An object has key allocated internally */
+ UCL_OBJECT_ALLOCATED_VALUE = (1 << 1), /**< An object has a string value allocated internally */
+ UCL_OBJECT_NEED_KEY_ESCAPE = (1 << 2), /**< The key of an object need to be escaped on output */
+ UCL_OBJECT_EPHEMERAL = (1 << 3), /**< Temporary object that does not need to be freed really */
+ UCL_OBJECT_MULTILINE = (1 << 4), /**< String should be displayed as multiline string */
+ UCL_OBJECT_MULTIVALUE = (1 << 5), /**< Object is a key with multiple values */
+ UCL_OBJECT_INHERITED = (1 << 6), /**< Object has been inherited from another */
+ UCL_OBJECT_BINARY = (1 << 7), /**< Object contains raw binary data */
+ UCL_OBJECT_SQUOTED = (1 << 8) /**< Object has been enclosed in single quotes */
+} ucl_object_flags_t;
+
+/**
+ * Duplicate policy types
+ */
+enum ucl_duplicate_strategy {
+ UCL_DUPLICATE_APPEND = 0, /**< Default policy to merge based on priorities */
+ UCL_DUPLICATE_MERGE, /**< Merge new object with old one */
+ UCL_DUPLICATE_REWRITE, /**< Rewrite old keys */
+ UCL_DUPLICATE_ERROR /**< Stop parsing on duplicate found */
+};
+
+/**
+ * Input format type
+ */
+enum ucl_parse_type {
+ UCL_PARSE_UCL = 0, /**< Default ucl format */
+ UCL_PARSE_MSGPACK, /**< Message pack input format */
+ UCL_PARSE_CSEXP, /**< Canonical S-expressions */
+ UCL_PARSE_AUTO /**< Try to detect parse type */
+};
+
+/**
+ * UCL object structure. Please mention that the most of fields should not be touched by
+ * UCL users. In future, this structure may be converted to private one.
+ */
+typedef struct ucl_object_s {
+ /**
+ * Variant value type
+ */
+ union {
+ int64_t iv; /**< Int value of an object */
+ const char *sv; /**< String value of an object */
+ double dv; /**< Double value of an object */
+ void *av; /**< Array */
+ void *ov; /**< Object */
+ void* ud; /**< Opaque user data */
+ } value;
+ const char *key; /**< Key of an object */
+ struct ucl_object_s *next; /**< Array handle */
+ struct ucl_object_s *prev; /**< Array handle */
+ uint32_t keylen; /**< Length of a key */
+ uint32_t len; /**< Size of an object */
+ uint32_t ref; /**< Reference count */
+ uint16_t flags; /**< Object flags */
+ uint16_t type; /**< Real type */
+ unsigned char* trash_stack[2]; /**< Pointer to allocated chunks */
+} ucl_object_t;
+
+/**
+ * Destructor type for userdata objects
+ * @param ud user specified data pointer
+ */
+typedef void (*ucl_userdata_dtor)(void *ud);
+typedef const char* (*ucl_userdata_emitter)(void *ud);
+
+/** @} */
+
+/**
+ * @defgroup utils Utility functions
+ * A number of utility functions simplify handling of UCL objects
+ *
+ * @{
+ */
+/**
+ * Copy and return a key of an object, returned key is zero-terminated
+ * @param obj CL object
+ * @return zero terminated key
+ */
+UCL_EXTERN char* ucl_copy_key_trash (const ucl_object_t *obj);
+
+/**
+ * Copy and return a string value of an object, returned key is zero-terminated
+ * @param obj CL object
+ * @return zero terminated string representation of object value
+ */
+UCL_EXTERN char* ucl_copy_value_trash (const ucl_object_t *obj);
+
+/**
+ * Creates a new object
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_new (void) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create new object with type specified
+ * @param type type of a new object
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_typed_new (ucl_type_t type) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create new object with type and priority specified
+ * @param type type of a new object
+ * @param priority priority of an object
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_new_full (ucl_type_t type, unsigned priority)
+ UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create new object with userdata dtor
+ * @param dtor destructor function
+ * @param emitter emitter for userdata
+ * @param ptr opaque pointer
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_new_userdata (ucl_userdata_dtor dtor,
+ ucl_userdata_emitter emitter, void *ptr) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Perform deep copy of an object copying everything
+ * @param other object to copy
+ * @return new object with refcount equal to 1
+ */
+UCL_EXTERN ucl_object_t * ucl_object_copy (const ucl_object_t *other)
+ UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Return the type of an object
+ * @return the object type
+ */
+UCL_EXTERN ucl_type_t ucl_object_type (const ucl_object_t *obj);
+
+/**
+ * Converts ucl object type to its string representation
+ * @param type type of object
+ * @return constant string describing type
+ */
+UCL_EXTERN const char * ucl_object_type_to_string (ucl_type_t type);
+
+/**
+ * Converts string that represents ucl type to real ucl type enum
+ * @param input C string with name of type
+ * @param res resulting target
+ * @return true if `input` is a name of type stored in `res`
+ */
+UCL_EXTERN bool ucl_object_string_to_type (const char *input, ucl_type_t *res);
+
+/**
+ * Convert any string to an ucl object making the specified transformations
+ * @param str fixed size or NULL terminated string
+ * @param len length (if len is zero, than str is treated as NULL terminated)
+ * @param flags conversion flags
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t * ucl_object_fromstring_common (const char *str, size_t len,
+ enum ucl_string_flags flags) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create a UCL object from the specified string
+ * @param str NULL terminated string, will be json escaped
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t *ucl_object_fromstring (const char *str) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create a UCL object from the specified string
+ * @param str fixed size string, will be json escaped
+ * @param len length of a string
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t *ucl_object_fromlstring (const char *str,
+ size_t len) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create an object from an integer number
+ * @param iv number
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_fromint (int64_t iv) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create an object from a float number
+ * @param dv number
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_fromdouble (double dv) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create an object from a boolean
+ * @param bv bool value
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_frombool (bool bv) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Insert a object 'elt' to the hash 'top' and associate it with key 'key'
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must NOT be NULL)
+ * @param key key to associate with this object (either const or preallocated)
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @param copy_key make an internal copy of key
+ * @return true if key has been inserted
+ */
+UCL_EXTERN bool ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key);
+
+/**
+ * Replace a object 'elt' to the hash 'top' and associate it with key 'key', old object will be unrefed,
+ * if no object has been found this function works like ucl_object_insert_key()
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must NOT be NULL)
+ * @param key key to associate with this object (either const or preallocated)
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @param copy_key make an internal copy of key
+ * @return true if key has been inserted
+ */
+UCL_EXTERN bool ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key);
+
+/**
+ * Merge the keys from one object to another object. Overwrite on conflict
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must be of type UCL_OBJECT)
+ * @param copy copy rather than reference the elements
+ * @return true if all keys have been merged
+ */
+UCL_EXTERN bool ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy);
+
+/**
+ * Delete a object associated with key 'key', old object will be unrefered,
+ * @param top object
+ * @param key key associated to the object to remove
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ */
+UCL_EXTERN bool ucl_object_delete_keyl (ucl_object_t *top,
+ const char *key, size_t keylen);
+
+/**
+ * Delete a object associated with key 'key', old object will be unrefered,
+ * @param top object
+ * @param key key associated to the object to remove
+ */
+UCL_EXTERN bool ucl_object_delete_key (ucl_object_t *top,
+ const char *key);
+
+
+/**
+ * Removes `key` from `top` object, returning the object that was removed. This
+ * object is not released, caller must unref the returned object when it is no
+ * longer needed.
+ * @param top object
+ * @param key key to remove
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @return removed object or NULL if object has not been found
+ */
+UCL_EXTERN ucl_object_t* ucl_object_pop_keyl (ucl_object_t *top, const char *key,
+ size_t keylen) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Removes `key` from `top` object returning the object that was removed. This
+ * object is not released, caller must unref the returned object when it is no
+ * longer needed.
+ * @param top object
+ * @param key key to remove
+ * @return removed object or NULL if object has not been found
+ */
+UCL_EXTERN ucl_object_t* ucl_object_pop_key (ucl_object_t *top, const char *key)
+ UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Insert a object 'elt' to the hash 'top' and associate it with key 'key', if
+ * the specified key exist, try to merge its content
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must NOT be NULL)
+ * @param key key to associate with this object (either const or preallocated)
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @param copy_key make an internal copy of key
+ * @return true if key has been inserted
+ */
+UCL_EXTERN bool ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key);
+
+/**
+ * Reserve space in ucl array or object for `elt` elements
+ * @param obj object to reserve
+ * @param reserved size to reserve in an object
+ * @return 0 on success, -1 on failure (i.e. ENOMEM)
+ */
+UCL_EXTERN bool ucl_object_reserve (ucl_object_t *obj, size_t reserved);
+
+/**
+ * Append an element to the end of array object
+ * @param top destination object (must NOT be NULL)
+ * @param elt element to append (must NOT be NULL)
+ * @return true if value has been inserted
+ */
+UCL_EXTERN bool ucl_array_append (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Append an element to the start of array object
+ * @param top destination object (must NOT be NULL)
+ * @param elt element to append (must NOT be NULL)
+ * @return true if value has been inserted
+ */
+UCL_EXTERN bool ucl_array_prepend (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Merge all elements of second array into the first array
+ * @param top destination array (must be of type UCL_ARRAY)
+ * @param elt array to copy elements from (must be of type UCL_ARRAY)
+ * @param copy copy elements instead of referencing them
+ * @return true if arrays were merged
+ */
+UCL_EXTERN bool ucl_array_merge (ucl_object_t *top, ucl_object_t *elt,
+ bool copy);
+
+/**
+ * Removes an element `elt` from the array `top`, returning the object that was
+ * removed. This object is not released, caller must unref the returned object
+ * when it is no longer needed.
+ * @param top array ucl object
+ * @param elt element to remove
+ * @return removed element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN ucl_object_t* ucl_array_delete (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Returns the first element of the array `top`
+ * @param top array ucl object
+ * @return element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN const ucl_object_t* ucl_array_head (const ucl_object_t *top);
+
+/**
+ * Returns the last element of the array `top`
+ * @param top array ucl object
+ * @return element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN const ucl_object_t* ucl_array_tail (const ucl_object_t *top);
+
+/**
+ * Removes the last element from the array `top`, returning the object that was
+ * removed. This object is not released, caller must unref the returned object
+ * when it is no longer needed.
+ * @param top array ucl object
+ * @return removed element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN ucl_object_t* ucl_array_pop_last (ucl_object_t *top);
+
+/**
+ * Removes the first element from the array `top`, returning the object that was
+ * removed. This object is not released, caller must unref the returned object
+ * when it is no longer needed.
+ * @param top array ucl object
+ * @return removed element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN ucl_object_t* ucl_array_pop_first (ucl_object_t *top);
+
+/**
+ * Return size of the array `top`
+ * @param top object to get size from (must be of type UCL_ARRAY)
+ * @return size of the array
+ */
+UCL_EXTERN unsigned int ucl_array_size (const ucl_object_t *top);
+
+/**
+ * Return object identified by index of the array `top`
+ * @param top object to get a key from (must be of type UCL_ARRAY)
+ * @param index array index to return
+ * @return object at the specified index or NULL if index is not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_array_find_index (const ucl_object_t *top,
+ unsigned int index);
+
+/**
+ * Return the index of `elt` in the array `top`
+ * @param top object to get a key from (must be of type UCL_ARRAY)
+ * @param elt element to find index of (must NOT be NULL)
+ * @return index of `elt` in the array `top or (unsigned int)-1 if `elt` is not found
+ */
+UCL_EXTERN unsigned int ucl_array_index_of (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Replace an element in an array with a different element, returning the object
+ * that was replaced. This object is not released, caller must unref the
+ * returned object when it is no longer needed.
+ * @param top destination object (must be of type UCL_ARRAY)
+ * @param elt element to append (must NOT be NULL)
+ * @param index array index in destination to overwrite with elt
+ * @return object that was replaced or NULL if index is not found
+ */
+ucl_object_t *
+ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt,
+ unsigned int index);
+
+/**
+ * Append a element to another element forming an implicit array
+ * @param head head to append (may be NULL)
+ * @param elt new element
+ * @return the new implicit array
+ */
+UCL_EXTERN ucl_object_t * ucl_elt_append (ucl_object_t *head,
+ ucl_object_t *elt);
+
+/**
+ * Converts an object to double value
+ * @param obj CL object
+ * @param target target double variable
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_todouble_safe (const ucl_object_t *obj, double *target);
+
+/**
+ * Unsafe version of \ref ucl_obj_todouble_safe
+ * @param obj CL object
+ * @return double value
+ */
+UCL_EXTERN double ucl_object_todouble (const ucl_object_t *obj);
+
+/**
+ * Converts an object to integer value
+ * @param obj CL object
+ * @param target target integer variable
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target);
+
+/**
+ * Unsafe version of \ref ucl_obj_toint_safe
+ * @param obj CL object
+ * @return int value
+ */
+UCL_EXTERN int64_t ucl_object_toint (const ucl_object_t *obj);
+
+/**
+ * Converts an object to boolean value
+ * @param obj CL object
+ * @param target target boolean variable
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target);
+
+/**
+ * Unsafe version of \ref ucl_obj_toboolean_safe
+ * @param obj CL object
+ * @return boolean value
+ */
+UCL_EXTERN bool ucl_object_toboolean (const ucl_object_t *obj);
+
+/**
+ * Converts an object to string value
+ * @param obj CL object
+ * @param target target string variable, no need to free value
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_tostring_safe (const ucl_object_t *obj, const char **target);
+
+/**
+ * Unsafe version of \ref ucl_obj_tostring_safe
+ * @param obj CL object
+ * @return string value
+ */
+UCL_EXTERN const char* ucl_object_tostring (const ucl_object_t *obj);
+
+/**
+ * Convert any object to a string in JSON notation if needed
+ * @param obj CL object
+ * @return string value
+ */
+UCL_EXTERN const char* ucl_object_tostring_forced (const ucl_object_t *obj);
+
+/**
+ * Return string as char * and len, string may be not zero terminated, more efficient that \ref ucl_obj_tostring as it
+ * allows zero-copy (if #UCL_PARSER_ZEROCOPY has been used during parsing)
+ * @param obj CL object
+ * @param target target string variable, no need to free value
+ * @param tlen target length
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_tolstring_safe (const ucl_object_t *obj,
+ const char **target, size_t *tlen);
+
+/**
+ * Unsafe version of \ref ucl_obj_tolstring_safe
+ * @param obj CL object
+ * @return string value
+ */
+UCL_EXTERN const char* ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen);
+
+/**
+ * Return object identified by a key in the specified object
+ * @param obj object to get a key from (must be of type UCL_OBJECT)
+ * @param key key to search
+ * @return object matching the specified key or NULL if key was not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_lookup (const ucl_object_t *obj,
+ const char *key);
+#define ucl_object_find_key ucl_object_lookup
+
+/**
+ * Return object identified by a key in the specified object, if the first key is
+ * not found then look for the next one. This process is repeated unless
+ * the next argument in the list is not NULL. So, `ucl_object_find_any_key(obj, key, NULL)`
+ * is equal to `ucl_object_find_key(obj, key)`
+ * @param obj object to get a key from (must be of type UCL_OBJECT)
+ * @param key key to search
+ * @param ... list of alternative keys to search (NULL terminated)
+ * @return object matching the specified key or NULL if key was not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_lookup_any (const ucl_object_t *obj,
+ const char *key, ...);
+#define ucl_object_find_any_key ucl_object_lookup_any
+
+/**
+ * Return object identified by a fixed size key in the specified object
+ * @param obj object to get a key from (must be of type UCL_OBJECT)
+ * @param key key to search
+ * @param klen length of a key
+ * @return object matching the specified key or NULL if key was not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_lookup_len (const ucl_object_t *obj,
+ const char *key, size_t klen);
+#define ucl_object_find_keyl ucl_object_lookup_len
+
+/**
+ * Return object identified by dot notation string
+ * @param obj object to search in
+ * @param path dot.notation.path to the path to lookup. May use numeric .index on arrays
+ * @return object matched the specified path or NULL if path is not found
+ */
+UCL_EXTERN const ucl_object_t *ucl_object_lookup_path (const ucl_object_t *obj,
+ const char *path);
+#define ucl_lookup_path ucl_object_lookup_path
+
+/**
+ * Return object identified by object notation string using arbitrary delimiter
+ * @param obj object to search in
+ * @param path dot.notation.path to the path to lookup. May use numeric .index on arrays
+ * @param sep the sepatorator to use in place of . (incase keys have . in them)
+ * @return object matched the specified path or NULL if path is not found
+ */
+UCL_EXTERN const ucl_object_t *ucl_object_lookup_path_char (const ucl_object_t *obj,
+ const char *path, char sep);
+#define ucl_lookup_path_char ucl_object_lookup_path_char
+
+/**
+ * Returns a key of an object as a NULL terminated string
+ * @param obj CL object
+ * @return key or NULL if there is no key
+ */
+UCL_EXTERN const char* ucl_object_key (const ucl_object_t *obj);
+
+/**
+ * Returns a key of an object as a fixed size string (may be more efficient)
+ * @param obj CL object
+ * @param len target key length
+ * @return key pointer
+ */
+UCL_EXTERN const char* ucl_object_keyl (const ucl_object_t *obj, size_t *len);
+
+/**
+ * Increase reference count for an object
+ * @param obj object to ref
+ * @return the referenced object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_ref (const ucl_object_t *obj);
+
+/**
+ * Free ucl object
+ * @param obj ucl object to free
+ */
+UCL_DEPRECATED(UCL_EXTERN void ucl_object_free (ucl_object_t *obj));
+
+/**
+ * Decrease reference count for an object
+ * @param obj object to unref
+ */
+UCL_EXTERN void ucl_object_unref (ucl_object_t *obj);
+
+/**
+ * Compare objects `o1` and `o2`
+ * @param o1 the first object
+ * @param o2 the second object
+ * @return values >0, 0 and <0 if `o1` is more than, equal and less than `o2`.
+ * The order of comparison:
+ * 1) Type of objects
+ * 2) Size of objects
+ * 3) Content of objects
+ */
+UCL_EXTERN int ucl_object_compare (const ucl_object_t *o1,
+ const ucl_object_t *o2);
+
+/**
+ * Compare objects `o1` and `o2` useful for sorting
+ * @param o1 the first object
+ * @param o2 the second object
+ * @return values >0, 0 and <0 if `o1` is more than, equal and less than `o2`.
+ * The order of comparison:
+ * 1) Type of objects
+ * 2) Size of objects
+ * 3) Content of objects
+ */
+UCL_EXTERN int ucl_object_compare_qsort (const ucl_object_t **o1,
+ const ucl_object_t **o2);
+
+/**
+ * Sort UCL array using `cmp` compare function
+ * @param ar
+ * @param cmp
+ */
+UCL_EXTERN void ucl_object_array_sort (ucl_object_t *ar,
+ int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2));
+
+enum ucl_object_keys_sort_flags {
+ UCL_SORT_KEYS_DEFAULT = 0,
+ UCL_SORT_KEYS_ICASE = (1u << 0u),
+ UCL_SORT_KEYS_RECURSIVE = (1u << 1u),
+};
+/***
+ * Sorts keys in object in place
+ * @param obj
+ * @param how
+ */
+UCL_EXTERN void ucl_object_sort_keys (ucl_object_t *obj,
+ enum ucl_object_keys_sort_flags how);
+
+/**
+ * Get the priority for specific UCL object
+ * @param obj any ucl object
+ * @return priority of an object
+ */
+UCL_EXTERN unsigned int ucl_object_get_priority (const ucl_object_t *obj);
+
+/**
+ * Set explicit priority of an object.
+ * @param obj any ucl object
+ * @param priority new priroity value (only 4 least significant bits are considred)
+ */
+UCL_EXTERN void ucl_object_set_priority (ucl_object_t *obj,
+ unsigned int priority);
+
+/**
+ * Opaque iterator object
+ */
+typedef void* ucl_object_iter_t;
+
+/**
+ * Get next key from an object
+ * @param obj object to iterate
+ * @param iter opaque iterator, must be set to NULL on the first call:
+ * ucl_object_iter_t it = NULL;
+ * while ((cur = ucl_iterate_object (obj, &it)) != NULL) ...
+ * @param ep pointer record exception (such as ENOMEM), could be NULL
+ * @return the next object or NULL
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_iterate_with_error (const ucl_object_t *obj,
+ ucl_object_iter_t *iter, bool expand_values, int *ep);
+
+#define ucl_iterate_object ucl_object_iterate
+#define ucl_object_iterate(ob, it, ev) ucl_object_iterate_with_error((ob), (it), (ev), NULL)
+
+/**
+ * Create new safe iterator for the specified object
+ * @param obj object to iterate
+ * @return new iterator object that should be used with safe iterators API only
+ */
+UCL_EXTERN ucl_object_iter_t ucl_object_iterate_new (const ucl_object_t *obj)
+ UCL_WARN_UNUSED_RESULT;
+/**
+ * Check safe iterator object after performing some operations on it
+ * (such as ucl_object_iterate_safe()) to see if operation has encountered
+ * fatal exception while performing that operation (e.g. ENOMEM).
+ * @param iter opaque iterator
+ * @return true if exception has occured, false otherwise
+ */
+UCL_EXTERN bool ucl_object_iter_chk_excpn(ucl_object_iter_t *it);
+
+/**
+ * Reset initialized iterator to a new object
+ * @param obj new object to iterate
+ * @return modified iterator object
+ */
+UCL_EXTERN ucl_object_iter_t ucl_object_iterate_reset (ucl_object_iter_t it,
+ const ucl_object_t *obj);
+
+/**
+ * Get the next object from the `obj`. This function iterates over arrays, objects
+ * and implicit arrays
+ * @param iter safe iterator
+ * @param expand_values expand explicit arrays and objects
+ * @return the next object in sequence
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_iterate_safe (ucl_object_iter_t iter,
+ bool expand_values);
+/**
+ * Iteration type enumerator
+ */
+enum ucl_iterate_type {
+ UCL_ITERATE_EXPLICIT = 1 << 0, /**< Iterate just explicit arrays and objects */
+ UCL_ITERATE_IMPLICIT = 1 << 1, /**< Iterate just implicit arrays */
+ UCL_ITERATE_BOTH = (1 << 0) | (1 << 1), /**< Iterate both explicit and implicit arrays*/
+};
+
+/**
+ * Get the next object from the `obj`. This function iterates over arrays, objects
+ * and implicit arrays if needed
+ * @param iter safe iterator
+ * @param
+ * @return the next object in sequence
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_iterate_full (ucl_object_iter_t iter,
+ enum ucl_iterate_type type);
+
+/**
+ * Free memory associated with the safe iterator
+ * @param it safe iterator object
+ */
+UCL_EXTERN void ucl_object_iterate_free (ucl_object_iter_t it);
+
+/** @} */
+
+
+/**
+ * @defgroup parser Parsing functions
+ * These functions are used to parse UCL objects
+ *
+ * @{
+ */
+
+/**
+ * Macro handler for a parser
+ * @param data the content of macro
+ * @param len the length of content
+ * @param arguments arguments object
+ * @param ud opaque user data
+ * @param err error pointer
+ * @return true if macro has been parsed
+ */
+typedef bool (*ucl_macro_handler) (const unsigned char *data, size_t len,
+ const ucl_object_t *arguments,
+ void* ud);
+
+/**
+ * Context dependent macro handler for a parser
+ * @param data the content of macro
+ * @param len the length of content
+ * @param arguments arguments object
+ * @param context previously parsed context
+ * @param ud opaque user data
+ * @param err error pointer
+ * @return true if macro has been parsed
+ */
+typedef bool (*ucl_context_macro_handler) (const unsigned char *data, size_t len,
+ const ucl_object_t *arguments,
+ const ucl_object_t *context,
+ void* ud);
+
+/* Opaque parser */
+struct ucl_parser;
+
+/**
+ * Creates new parser object
+ * @param pool pool to allocate memory from
+ * @return new parser object
+ */
+UCL_EXTERN struct ucl_parser* ucl_parser_new (int flags);
+
+/**
+ * Sets the default priority for the parser applied to chunks that do not
+ * specify priority explicitly
+ * @param parser parser object
+ * @param prio default priority (0 .. 16)
+ * @return true if parser's default priority was set
+ */
+UCL_EXTERN bool ucl_parser_set_default_priority (struct ucl_parser *parser,
+ unsigned prio);
+/**
+ * Gets the default priority for the parser applied to chunks that do not
+ * specify priority explicitly
+ * @param parser parser object
+ * @return true default priority (0 .. 16), -1 for failure
+ */
+UCL_EXTERN int ucl_parser_get_default_priority (struct ucl_parser *parser);
+
+/**
+ * Register new handler for a macro
+ * @param parser parser object
+ * @param macro macro name (without leading dot)
+ * @param handler handler (it is called immediately after macro is parsed)
+ * @param ud opaque user data for a handler
+ * @return true on success, false on failure (i.e. ENOMEM)
+ */
+UCL_EXTERN bool ucl_parser_register_macro (struct ucl_parser *parser,
+ const char *macro,
+ ucl_macro_handler handler, void* ud);
+
+/**
+ * Register new context dependent handler for a macro
+ * @param parser parser object
+ * @param macro macro name (without leading dot)
+ * @param handler handler (it is called immediately after macro is parsed)
+ * @param ud opaque user data for a handler
+ * @return true on success, false on failure (i.e. ENOMEM)
+ */
+UCL_EXTERN bool ucl_parser_register_context_macro (struct ucl_parser *parser,
+ const char *macro,
+ ucl_context_macro_handler handler,
+ void* ud);
+
+/**
+ * Handler to detect unregistered variables
+ * @param data variable data
+ * @param len length of variable
+ * @param replace (out) replace value for variable
+ * @param replace_len (out) replace length for variable
+ * @param need_free (out) UCL will free `dest` after usage
+ * @param ud opaque userdata
+ * @return true if variable
+ */
+typedef bool (*ucl_variable_handler) (const unsigned char *data, size_t len,
+ unsigned char **replace, size_t *replace_len, bool *need_free, void* ud);
+
+/**
+ * Register new parser variable
+ * @param parser parser object
+ * @param var variable name
+ * @param value variable value
+ */
+UCL_EXTERN void ucl_parser_register_variable (struct ucl_parser *parser, const char *var,
+ const char *value);
+
+/**
+ * Set handler for unknown variables
+ * @param parser parser structure
+ * @param handler desired handler
+ * @param ud opaque data for the handler
+ */
+UCL_EXTERN void ucl_parser_set_variables_handler (struct ucl_parser *parser,
+ ucl_variable_handler handler, void *ud);
+
+/**
+ * Load new chunk to a parser
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_chunk (struct ucl_parser *parser,
+ const unsigned char *data, size_t len);
+
+/**
+ * Load new chunk to a parser with the specified priority
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_chunk_priority (struct ucl_parser *parser,
+ const unsigned char *data, size_t len, unsigned priority);
+
+/**
+ * Insert new chunk to a parser (must have previously processed data with an existing top object)
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_insert_chunk (struct ucl_parser *parser,
+ const unsigned char *data, size_t len);
+
+/**
+ * Full version of ucl_add_chunk with priority and duplicate strategy
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @param strat duplicates merging strategy
+ * @param parse_type input format
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_chunk_full (struct ucl_parser *parser,
+ const unsigned char *data, size_t len, unsigned priority,
+ enum ucl_duplicate_strategy strat, enum ucl_parse_type parse_type);
+
+/**
+ * Load ucl object from a string
+ * @param parser parser structure
+ * @param data the pointer to the string
+ * @param len the length of the string, if `len` is 0 then `data` must be zero-terminated string
+ * @return true if string has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_string (struct ucl_parser *parser,
+ const char *data, size_t len);
+
+/**
+ * Load ucl object from a string
+ * @param parser parser structure
+ * @param data the pointer to the string
+ * @param len the length of the string, if `len` is 0 then `data` must be zero-terminated string
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if string has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_string_priority (struct ucl_parser *parser,
+ const char *data, size_t len, unsigned priority);
+
+/**
+ * Load and add data from a file
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser,
+ const char *filename);
+
+/**
+ * Load and add data from a file
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_file_priority (struct ucl_parser *parser,
+ const char *filename, unsigned priority);
+
+/**
+ * Load and add data from a file
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @param strat Merge strategy to use while parsing this file
+ * @param parse_type Parser type to use while parsing this file
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type);
+
+/**
+ * Load and add data from a file descriptor
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_fd (struct ucl_parser *parser,
+ int fd);
+
+/**
+ * Load and add data from a file descriptor
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_fd_priority (struct ucl_parser *parser,
+ int fd, unsigned priority);
+
+/**
+ * Load and add data from a file descriptor
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @param strat Merge strategy to use while parsing this file
+ * @param parse_type Parser type to use while parsing this file
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_fd_full (struct ucl_parser *parser, int fd,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type);
+
+/**
+ * Provide a UCL_ARRAY of paths to search for include files. The object is
+ * copied so caller must unref the object.
+ * @param parser parser structure
+ * @param paths UCL_ARRAY of paths to search
+ * @return true if the path search array was replaced in the parser
+ */
+UCL_EXTERN bool ucl_set_include_path (struct ucl_parser *parser,
+ ucl_object_t *paths);
+
+/**
+ * Get a top object for a parser (refcount is increased)
+ * @param parser parser structure
+ * @param err if *err is NULL it is set to parser error
+ * @return top parser object or NULL
+ */
+UCL_EXTERN ucl_object_t* ucl_parser_get_object (struct ucl_parser *parser);
+
+/**
+ * Get the current stack object as stack accessor function for use in macro
+ * functions (refcount is increased)
+ * @param parser parser object
+ * @param depth depth of stack to retrieve (top is 0)
+ * @return current stack object or NULL
+ */
+UCL_EXTERN ucl_object_t* ucl_parser_get_current_stack_object (struct ucl_parser *parser, unsigned int depth);
+
+/**
+ * Peek at the character at the current chunk position
+ * @param parser parser structure
+ * @return current chunk position character
+ */
+UCL_EXTERN unsigned char ucl_parser_chunk_peek (struct ucl_parser *parser);
+
+/**
+ * Skip the character at the current chunk position
+ * @param parser parser structure
+ * @return success boolean
+ */
+UCL_EXTERN bool ucl_parser_chunk_skip (struct ucl_parser *parser);
+
+/**
+ * Get the error string if parsing has been failed
+ * @param parser parser object
+ * @return error description
+ */
+UCL_EXTERN const char *ucl_parser_get_error (struct ucl_parser *parser);
+
+/**
+ * Get the code of the last error
+ * @param parser parser object
+ * @return error code
+ */
+UCL_EXTERN int ucl_parser_get_error_code (struct ucl_parser *parser);
+
+/**
+ * Get the current column number within parser
+ * @param parser parser object
+ * @return current column number
+ */
+UCL_EXTERN unsigned ucl_parser_get_column (struct ucl_parser *parser);
+
+/**
+ * Get the current line number within parser
+ * @param parser parser object
+ * @return current line number
+ */
+UCL_EXTERN unsigned ucl_parser_get_linenum (struct ucl_parser *parser);
+
+/**
+ * Clear the error in the parser
+ * @param parser parser object
+ */
+UCL_EXTERN void ucl_parser_clear_error (struct ucl_parser *parser);
+
+/**
+ * Free ucl parser object
+ * @param parser parser object
+ */
+UCL_EXTERN void ucl_parser_free (struct ucl_parser *parser);
+
+/**
+ * Get constant opaque pointer to comments structure for this parser. Increase
+ * refcount to prevent this object to be destroyed on parser's destruction
+ * @param parser parser structure
+ * @return ucl comments pointer or NULL
+ */
+UCL_EXTERN const ucl_object_t * ucl_parser_get_comments (struct ucl_parser *parser);
+
+/**
+ * Utility function to find a comment object for the specified object in the input
+ * @param comments comments object
+ * @param srch search object
+ * @return string comment enclosed in ucl_object_t
+ */
+UCL_EXTERN const ucl_object_t * ucl_comments_find (const ucl_object_t *comments,
+ const ucl_object_t *srch);
+
+/**
+ * Move comment from `from` object to `to` object
+ * @param comments comments object
+ * @param what source object
+ * @param with destination object
+ * @return `true` if `from` has comment and it has been moved to `to`
+ */
+UCL_EXTERN bool ucl_comments_move (ucl_object_t *comments,
+ const ucl_object_t *from, const ucl_object_t *to);
+
+/**
+ * Adds a new comment for an object
+ * @param comments comments object
+ * @param obj object to add comment to
+ * @param comment string representation of a comment
+ */
+UCL_EXTERN void ucl_comments_add (ucl_object_t *comments,
+ const ucl_object_t *obj, const char *comment);
+
+/**
+ * Add new public key to parser for signatures check
+ * @param parser parser object
+ * @param key PEM representation of a key
+ * @param len length of the key
+ * @param err if *err is NULL it is set to parser error
+ * @return true if a key has been successfully added
+ */
+UCL_EXTERN bool ucl_parser_pubkey_add (struct ucl_parser *parser,
+ const unsigned char *key, size_t len);
+
+/**
+ * Set FILENAME and CURDIR variables in parser
+ * @param parser parser object
+ * @param filename filename to set or NULL to set FILENAME to "undef" and CURDIR to getcwd()
+ * @param need_expand perform realpath() if this variable is true and filename is not NULL
+ * @return true if variables has been set
+ */
+UCL_EXTERN bool ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename,
+ bool need_expand);
+
+/**
+ * Returns current file for the parser
+ * @param parser parser object
+ * @return current file or NULL if parsing memory
+ */
+UCL_EXTERN const char *ucl_parser_get_cur_file (struct ucl_parser *parser);
+
+/**
+ * Defines special handler for certain types of data (identified by magic)
+ */
+typedef bool (*ucl_parser_special_handler_t) (struct ucl_parser *parser,
+ const unsigned char *source, size_t source_len,
+ unsigned char **destination, size_t *dest_len,
+ void *user_data);
+
+/**
+ * Special handler flags
+ */
+enum ucl_special_handler_flags {
+ UCL_SPECIAL_HANDLER_DEFAULT = 0,
+ UCL_SPECIAL_HANDLER_PREPROCESS_ALL = (1u << 0),
+};
+
+/**
+ * Special handler structure
+ */
+struct ucl_parser_special_handler {
+ const unsigned char *magic;
+ size_t magic_len;
+ enum ucl_special_handler_flags flags;
+ ucl_parser_special_handler_t handler;
+ void (*free_function) (unsigned char *data, size_t len, void *user_data);
+ void *user_data;
+ struct ucl_parser_special_handler *next; /* Used internally */
+};
+
+/**
+ * Add special handler for a parser, handles special sequences identified by magic
+ * @param parser parser structure
+ * @param handler handler structure
+ */
+UCL_EXTERN void ucl_parser_add_special_handler (struct ucl_parser *parser,
+ struct ucl_parser_special_handler *handler);
+
+/**
+ * Handler for include traces:
+ * @param parser parser object
+ * @param parent where include is done from
+ * @param args arguments to an include
+ * @param path path of the include
+ * @param pathlen length of the path
+ * @param user_data opaque userdata
+ */
+typedef void (ucl_include_trace_func_t) (struct ucl_parser *parser,
+ const ucl_object_t *parent,
+ const ucl_object_t *args,
+ const char *path,
+ size_t pathlen,
+ void *user_data);
+
+/**
+ * Register trace function for an include handler
+ * @param parser parser object
+ * @param func function to trace includes
+ * @param user_data opaque data
+ */
+UCL_EXTERN void ucl_parser_set_include_tracer (struct ucl_parser *parser,
+ ucl_include_trace_func_t func,
+ void *user_data);
+
+/** @} */
+
+/**
+ * @defgroup emitter Emitting functions
+ * These functions are used to serialise UCL objects to some string representation.
+ *
+ * @{
+ */
+
+struct ucl_emitter_context;
+/**
+ * Structure using for emitter callbacks
+ */
+struct ucl_emitter_functions {
+ /** Append a single character */
+ int (*ucl_emitter_append_character) (unsigned char c, size_t nchars, void *ud);
+ /** Append a string of a specified length */
+ int (*ucl_emitter_append_len) (unsigned const char *str, size_t len, void *ud);
+ /** Append a 64 bit integer */
+ int (*ucl_emitter_append_int) (int64_t elt, void *ud);
+ /** Append floating point element */
+ int (*ucl_emitter_append_double) (double elt, void *ud);
+ /** Free userdata */
+ void (*ucl_emitter_free_func)(void *ud);
+ /** Opaque userdata pointer */
+ void *ud;
+};
+
+struct ucl_emitter_operations {
+ /** Write a primitive element */
+ void (*ucl_emitter_write_elt) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key);
+ /** Start ucl object */
+ void (*ucl_emitter_start_object) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key);
+ /** End ucl object */
+ void (*ucl_emitter_end_object) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj);
+ /** Start ucl array */
+ void (*ucl_emitter_start_array) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key);
+ void (*ucl_emitter_end_array) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj);
+};
+
+/**
+ * Structure that defines emitter functions
+ */
+struct ucl_emitter_context {
+ /** Name of emitter (e.g. json, compact_json) */
+ const char *name;
+ /** Unique id (e.g. UCL_EMIT_JSON for standard emitters */
+ int id;
+ /** A set of output functions */
+ const struct ucl_emitter_functions *func;
+ /** A set of output operations */
+ const struct ucl_emitter_operations *ops;
+ /** Current amount of indent tabs */
+ unsigned int indent;
+ /** Top level object */
+ const ucl_object_t *top;
+ /** Optional comments */
+ const ucl_object_t *comments;
+};
+
+/**
+ * Emit object to a string
+ * @param obj object
+ * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is
+ * #UCL_EMIT_CONFIG then emit config like object
+ * @return dump of an object (must be freed after using) or NULL in case of error
+ */
+UCL_EXTERN unsigned char *ucl_object_emit (const ucl_object_t *obj,
+ enum ucl_emitter emit_type);
+
+/**
+ * Emit object to a string that can contain `\0` inside
+ * @param obj object
+ * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is
+ * #UCL_EMIT_CONFIG then emit config like object
+ * @param len the resulting length
+ * @return dump of an object (must be freed after using) or NULL in case of error
+ */
+UCL_EXTERN unsigned char *ucl_object_emit_len (const ucl_object_t *obj,
+ enum ucl_emitter emit_type, size_t *len);
+
+/**
+ * Emit object to a string
+ * @param obj object
+ * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is
+ * #UCL_EMIT_CONFIG then emit config like object
+ * @param emitter a set of emitter functions
+ * @param comments optional comments for the parser
+ * @return dump of an object (must be freed after using) or NULL in case of error
+ */
+UCL_EXTERN bool ucl_object_emit_full (const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter,
+ const ucl_object_t *comments);
+
+/**
+ * Start streamlined UCL object emitter
+ * @param obj top UCL object
+ * @param emit_type emit type
+ * @param emitter a set of emitter functions
+ * @return new streamlined context that should be freed by
+ * `ucl_object_emit_streamline_finish`
+ */
+UCL_EXTERN struct ucl_emitter_context* ucl_object_emit_streamline_new (
+ const ucl_object_t *obj, enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter);
+
+/**
+ * Start object or array container for the streamlined output
+ * @param ctx streamlined context
+ * @param obj container object
+ */
+UCL_EXTERN void ucl_object_emit_streamline_start_container (
+ struct ucl_emitter_context *ctx, const ucl_object_t *obj);
+/**
+ * Add a complete UCL object to streamlined output
+ * @param ctx streamlined context
+ * @param obj object to output
+ */
+UCL_EXTERN void ucl_object_emit_streamline_add_object (
+ struct ucl_emitter_context *ctx, const ucl_object_t *obj);
+/**
+ * End previously added container
+ * @param ctx streamlined context
+ */
+UCL_EXTERN void ucl_object_emit_streamline_end_container (
+ struct ucl_emitter_context *ctx);
+/**
+ * Terminate streamlined container finishing all containers in it
+ * @param ctx streamlined context
+ */
+UCL_EXTERN void ucl_object_emit_streamline_finish (
+ struct ucl_emitter_context *ctx);
+
+/**
+ * Returns functions to emit object to memory
+ * @param pmem target pointer (should be freed by caller)
+ * @return emitter functions structure
+ */
+UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_memory_funcs (
+ void **pmem);
+
+/**
+ * Returns functions to emit object to FILE *
+ * @param fp FILE * object
+ * @return emitter functions structure
+ */
+UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_file_funcs (
+ FILE *fp);
+/**
+ * Returns functions to emit object to a file descriptor
+ * @param fd file descriptor
+ * @return emitter functions structure
+ */
+UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_fd_funcs (
+ int fd);
+
+/**
+ * Free emitter functions
+ * @param f pointer to functions
+ */
+UCL_EXTERN void ucl_object_emit_funcs_free (struct ucl_emitter_functions *f);
+
+/** @} */
+
+/**
+ * @defgroup schema Schema functions
+ * These functions are used to validate UCL objects using json schema format
+ *
+ * @{
+ */
+
+/**
+ * Used to define UCL schema error
+ */
+enum ucl_schema_error_code {
+ UCL_SCHEMA_OK = 0, /**< no error */
+ UCL_SCHEMA_TYPE_MISMATCH, /**< type of object is incorrect */
+ UCL_SCHEMA_INVALID_SCHEMA, /**< schema is invalid */
+ UCL_SCHEMA_MISSING_PROPERTY,/**< one or more missing properties */
+ UCL_SCHEMA_CONSTRAINT, /**< constraint found */
+ UCL_SCHEMA_MISSING_DEPENDENCY, /**< missing dependency */
+ UCL_SCHEMA_EXTERNAL_REF_MISSING, /**< cannot fetch external ref */
+ UCL_SCHEMA_EXTERNAL_REF_INVALID, /**< invalid external ref */
+ UCL_SCHEMA_INTERNAL_ERROR, /**< something bad happened */
+ UCL_SCHEMA_UNKNOWN /**< generic error */
+};
+
+/**
+ * Generic ucl schema error
+ */
+struct ucl_schema_error {
+ enum ucl_schema_error_code code; /**< error code */
+ char msg[128]; /**< error message */
+ const ucl_object_t *obj; /**< object where error occurred */
+};
+
+/**
+ * Validate object `obj` using schema object `schema`.
+ * @param schema schema object
+ * @param obj object to validate
+ * @param err error pointer, if this parameter is not NULL and error has been
+ * occurred, then `err` is filled with the exact error definition.
+ * @return true if `obj` is valid using `schema`
+ */
+UCL_EXTERN bool ucl_object_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err);
+
+/**
+ * Validate object `obj` using schema object `schema` and root schema at `root`.
+ * @param schema schema object
+ * @param obj object to validate
+ * @param root root schema object
+ * @param err error pointer, if this parameter is not NULL and error has been
+ * occurred, then `err` is filled with the exact error definition.
+ * @return true if `obj` is valid using `schema`
+ */
+UCL_EXTERN bool ucl_object_validate_root (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ struct ucl_schema_error *err);
+
+/**
+ * Validate object `obj` using schema object `schema` and root schema at `root`
+ * using some external references provided.
+ * @param schema schema object
+ * @param obj object to validate
+ * @param root root schema object
+ * @param ext_refs external references (might be modified during validation)
+ * @param err error pointer, if this parameter is not NULL and error has been
+ * occurred, then `err` is filled with the exact error definition.
+ * @return true if `obj` is valid using `schema`
+ */
+UCL_EXTERN bool ucl_object_validate_root_ext (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ ucl_object_t *ext_refs,
+ struct ucl_schema_error *err);
+
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif
+/*
+ * XXX: Poorly named API functions, need to replace them with the appropriate
+ * named function. All API functions *must* use naming ucl_object_*. Usage of
+ * ucl_obj* should be avoided.
+ */
+#define ucl_obj_todouble_safe ucl_object_todouble_safe
+#define ucl_obj_todouble ucl_object_todouble
+#define ucl_obj_tostring ucl_object_tostring
+#define ucl_obj_tostring_safe ucl_object_tostring_safe
+#define ucl_obj_tolstring ucl_object_tolstring
+#define ucl_obj_tolstring_safe ucl_object_tolstring_safe
+#define ucl_obj_toint ucl_object_toint
+#define ucl_obj_toint_safe ucl_object_toint_safe
+#define ucl_obj_toboolean ucl_object_toboolean
+#define ucl_obj_toboolean_safe ucl_object_toboolean_safe
+#define ucl_obj_get_key ucl_object_find_key
+#define ucl_obj_get_keyl ucl_object_find_keyl
+#define ucl_obj_unref ucl_object_unref
+#define ucl_obj_ref ucl_object_ref
+#define ucl_obj_free ucl_object_free
+
+#endif /* UCL_H_ */
diff --git a/contrib/libucl/ucl_chartable.h b/contrib/libucl/ucl_chartable.h
new file mode 100644
index 0000000..7571a1d
--- /dev/null
+++ b/contrib/libucl/ucl_chartable.h
@@ -0,0 +1,268 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UCL_CHARTABLE_H_
+#define UCL_CHARTABLE_H_
+
+#include "ucl_internal.h"
+
+static const unsigned int ucl_chartable[256] = {
+UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_VALUE_END|UCL_CHARACTER_UCL_UNSAFE, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE|UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_VALUE_END|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_VALUE_END|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_WHITESPACE|UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_UCL_UNSAFE /* */,
+UCL_CHARACTER_VALUE_STR /* ! */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_ESCAPE|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE /* " */,
+UCL_CHARACTER_VALUE_END /* # */, UCL_CHARACTER_VALUE_STR /* $ */,
+UCL_CHARACTER_VALUE_STR /* % */, UCL_CHARACTER_VALUE_STR /* & */,
+UCL_CHARACTER_VALUE_STR /* ' */, UCL_CHARACTER_VALUE_STR /* ( */,
+UCL_CHARACTER_VALUE_STR /* ) */, UCL_CHARACTER_VALUE_STR /* * */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_UCL_UNSAFE /* + */,
+UCL_CHARACTER_VALUE_END /* , */,
+UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* - */,
+UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* . */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_ESCAPE /* / */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 0 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 1 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 2 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 3 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 4 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 5 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 6 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 7 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 8 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 9 */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_UCL_UNSAFE /* : */,
+UCL_CHARACTER_VALUE_END /* ; */, UCL_CHARACTER_VALUE_STR /* < */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_UCL_UNSAFE /* = */,
+UCL_CHARACTER_VALUE_STR /* > */, UCL_CHARACTER_VALUE_STR /* ? */,
+UCL_CHARACTER_VALUE_STR /* @ */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* A */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* B */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* C */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* D */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* E */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* F */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* G */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* H */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* I */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* J */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* K */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* L */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* M */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* N */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* O */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* P */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* Q */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* R */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* S */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* T */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* U */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* V */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* W */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* X */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* Y */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* Z */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_UCL_UNSAFE /* [ */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_ESCAPE|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE /* \ */,
+UCL_CHARACTER_VALUE_END /* ] */, UCL_CHARACTER_VALUE_STR /* ^ */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR /* _ */,
+UCL_CHARACTER_VALUE_STR /* ` */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* a */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* b */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* c */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* d */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* e */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* f */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* g */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* h */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* i */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* j */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* k */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* l */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* m */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* n */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* o */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* p */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* q */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* r */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* s */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* t */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* u */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* v */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* w */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* x */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* y */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* z */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_UCL_UNSAFE /* { */,
+UCL_CHARACTER_VALUE_STR /* | */, UCL_CHARACTER_VALUE_END /* } */,
+UCL_CHARACTER_VALUE_STR /* ~ */, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR
+};
+
+static inline bool
+ucl_test_character (unsigned char c, int type_flags)
+{
+ return (ucl_chartable[c] & type_flags) != 0;
+}
+
+#endif /* UCL_CHARTABLE_H_ */
diff --git a/contrib/libucl/ucl_emitter.c b/contrib/libucl/ucl_emitter.c
new file mode 100644
index 0000000..777093a
--- /dev/null
+++ b/contrib/libucl/ucl_emitter.c
@@ -0,0 +1,1189 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+#ifdef HAVE_FLOAT_H
+#include <float.h>
+#endif
+#ifdef HAVE_MATH_H
+#include <math.h>
+#endif
+
+/**
+ * @file ucl_emitter.c
+ * Serialise UCL object to various of output formats
+ */
+
+static void ucl_emitter_common_elt (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key, bool compact);
+
+#define UCL_EMIT_TYPE_OPS(type) \
+ static void ucl_emit_ ## type ## _elt (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool first, bool print_key); \
+ static void ucl_emit_ ## type ## _start_obj (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key); \
+ static void ucl_emit_ ## type## _start_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key); \
+ static void ucl_emit_ ##type## _end_object (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj); \
+ static void ucl_emit_ ##type## _end_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj)
+
+/*
+ * JSON format operations
+ */
+UCL_EMIT_TYPE_OPS(json);
+UCL_EMIT_TYPE_OPS(json_compact);
+UCL_EMIT_TYPE_OPS(config);
+UCL_EMIT_TYPE_OPS(yaml);
+UCL_EMIT_TYPE_OPS(msgpack);
+
+#define UCL_EMIT_TYPE_CONTENT(type) { \
+ .ucl_emitter_write_elt = ucl_emit_ ## type ## _elt, \
+ .ucl_emitter_start_object = ucl_emit_ ## type ##_start_obj, \
+ .ucl_emitter_start_array = ucl_emit_ ## type ##_start_array, \
+ .ucl_emitter_end_object = ucl_emit_ ## type ##_end_object, \
+ .ucl_emitter_end_array = ucl_emit_ ## type ##_end_array \
+}
+
+static const struct ucl_emitter_operations ucl_standart_emitter_ops[] = {
+ [UCL_EMIT_JSON] = UCL_EMIT_TYPE_CONTENT(json),
+ [UCL_EMIT_JSON_COMPACT] = UCL_EMIT_TYPE_CONTENT(json_compact),
+ [UCL_EMIT_CONFIG] = UCL_EMIT_TYPE_CONTENT(config),
+ [UCL_EMIT_YAML] = UCL_EMIT_TYPE_CONTENT(yaml),
+ [UCL_EMIT_MSGPACK] = UCL_EMIT_TYPE_CONTENT(msgpack)
+};
+
+/*
+ * Utility to check whether we need a top object
+ */
+#define UCL_EMIT_IDENT_TOP_OBJ(ctx, obj) ((ctx)->top != (obj) || \
+ ((ctx)->id == UCL_EMIT_JSON_COMPACT || (ctx)->id == UCL_EMIT_JSON))
+
+
+/**
+ * Add tabulation to the output buffer
+ * @param buf target buffer
+ * @param tabs number of tabs to add
+ */
+static inline void
+ucl_add_tabs (const struct ucl_emitter_functions *func, unsigned int tabs,
+ bool compact)
+{
+ if (!compact && tabs > 0) {
+ func->ucl_emitter_append_character (' ', tabs * 4, func->ud);
+ }
+}
+
+/**
+ * Print key for the element
+ * @param ctx
+ * @param obj
+ */
+static void
+ucl_emitter_print_key (bool print_key, struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ if (!print_key) {
+ return;
+ }
+
+ if (ctx->id == UCL_EMIT_CONFIG) {
+ if (obj->flags & UCL_OBJECT_NEED_KEY_ESCAPE) {
+ ucl_elt_string_write_json (obj->key, obj->keylen, ctx);
+ }
+ else {
+ func->ucl_emitter_append_len (obj->key, obj->keylen, func->ud);
+ }
+
+ if (obj->type != UCL_OBJECT && obj->type != UCL_ARRAY) {
+ func->ucl_emitter_append_len (" = ", 3, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_character (' ', 1, func->ud);
+ }
+ }
+ else if (ctx->id == UCL_EMIT_YAML) {
+ if (obj->keylen > 0 && (obj->flags & UCL_OBJECT_NEED_KEY_ESCAPE)) {
+ ucl_elt_string_write_json (obj->key, obj->keylen, ctx);
+ }
+ else if (obj->keylen > 0) {
+ func->ucl_emitter_append_len (obj->key, obj->keylen, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("null", 4, func->ud);
+ }
+
+ func->ucl_emitter_append_len (": ", 2, func->ud);
+ }
+ else {
+ if (obj->keylen > 0) {
+ ucl_elt_string_write_json (obj->key, obj->keylen, ctx);
+ }
+ else {
+ func->ucl_emitter_append_len ("null", 4, func->ud);
+ }
+
+ if (compact) {
+ func->ucl_emitter_append_character (':', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len (": ", 2, func->ud);
+ }
+ }
+}
+
+static void
+ucl_emitter_finish_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact, bool is_array)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ if (ctx->id == UCL_EMIT_CONFIG && obj != ctx->top) {
+ if (obj->type != UCL_OBJECT && obj->type != UCL_ARRAY) {
+ if (!is_array) {
+ /* Objects are split by ';' */
+ func->ucl_emitter_append_len (";\n", 2, func->ud);
+ }
+ else {
+ /* Use commas for arrays */
+ func->ucl_emitter_append_len (",\n", 2, func->ud);
+ }
+ }
+ else {
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ }
+ }
+}
+
+/**
+ * End standard ucl object
+ * @param ctx emitter context
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_end_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ if (UCL_EMIT_IDENT_TOP_OBJ(ctx, obj)) {
+ ctx->indent --;
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character ('}', 1, func->ud);
+ }
+ else {
+ if (ctx->id != UCL_EMIT_CONFIG) {
+ /* newline is already added for this format */
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ }
+ ucl_add_tabs (func, ctx->indent, compact);
+ func->ucl_emitter_append_character ('}', 1, func->ud);
+ }
+ }
+
+ ucl_emitter_finish_object (ctx, obj, compact, false);
+}
+
+/**
+ * End standard ucl array
+ * @param ctx emitter context
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_end_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ ctx->indent --;
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character (']', 1, func->ud);
+ }
+ else {
+ if (ctx->id != UCL_EMIT_CONFIG) {
+ /* newline is already added for this format */
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ }
+ ucl_add_tabs (func, ctx->indent, compact);
+ func->ucl_emitter_append_character (']', 1, func->ud);
+ }
+
+ ucl_emitter_finish_object (ctx, obj, compact, true);
+}
+
+/**
+ * Start emit standard UCL array
+ * @param ctx emitter context
+ * @param obj object to write
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_start_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key, bool compact)
+{
+ const ucl_object_t *cur;
+ ucl_object_iter_t iter = NULL;
+ const struct ucl_emitter_functions *func = ctx->func;
+ bool first = true;
+
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character ('[', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("[\n", 2, func->ud);
+ }
+
+ ctx->indent ++;
+
+ if (obj->type == UCL_ARRAY) {
+ /* explicit array */
+ while ((cur = ucl_object_iterate (obj, &iter, true)) != NULL) {
+ ucl_emitter_common_elt (ctx, cur, first, false, compact);
+ first = false;
+ }
+ }
+ else {
+ /* implicit array */
+ cur = obj;
+ while (cur) {
+ ucl_emitter_common_elt (ctx, cur, first, false, compact);
+ first = false;
+ cur = cur->next;
+ }
+ }
+}
+
+/**
+ * Start emit standard UCL object
+ * @param ctx emitter context
+ * @param obj object to write
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_start_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key, bool compact)
+{
+ ucl_hash_iter_t it = NULL;
+ const ucl_object_t *cur, *elt;
+ const struct ucl_emitter_functions *func = ctx->func;
+ bool first = true;
+
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ /*
+ * Print <ident_level>{
+ * <ident_level + 1><object content>
+ */
+ if (UCL_EMIT_IDENT_TOP_OBJ(ctx, obj)) {
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character ('{', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("{\n", 2, func->ud);
+ }
+ ctx->indent ++;
+ }
+
+ while ((cur = ucl_hash_iterate (obj->value.ov, &it))) {
+
+ if (ctx->id == UCL_EMIT_CONFIG) {
+ LL_FOREACH (cur, elt) {
+ ucl_emitter_common_elt (ctx, elt, first, true, compact);
+ }
+ }
+ else {
+ /* Expand implicit arrays */
+ if (cur->next != NULL) {
+ if (!first) {
+ if (compact) {
+ func->ucl_emitter_append_character (',', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len (",\n", 2, func->ud);
+ }
+ }
+ ucl_add_tabs (func, ctx->indent, compact);
+ ucl_emitter_common_start_array (ctx, cur, true, compact);
+ ucl_emitter_common_end_array (ctx, cur, compact);
+ }
+ else {
+ ucl_emitter_common_elt (ctx, cur, first, true, compact);
+ }
+ }
+
+ first = false;
+ }
+}
+
+/**
+ * Common choice of object emitting
+ * @param ctx emitter context
+ * @param obj object to print
+ * @param first flag to mark the first element
+ * @param print_key print key of an object
+ * @param compact compact output
+ */
+static void
+ucl_emitter_common_elt (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ bool flag;
+ struct ucl_object_userdata *ud;
+ const ucl_object_t *comment = NULL, *cur_comment;
+ const char *ud_out = "";
+
+ if (ctx->id != UCL_EMIT_CONFIG && !first) {
+ if (compact) {
+ func->ucl_emitter_append_character (',', 1, func->ud);
+ }
+ else {
+ if (ctx->id == UCL_EMIT_YAML && ctx->indent == 0) {
+ func->ucl_emitter_append_len ("\n", 1, func->ud);
+ } else {
+ func->ucl_emitter_append_len (",\n", 2, func->ud);
+ }
+ }
+ }
+
+ ucl_add_tabs (func, ctx->indent, compact);
+
+ if (ctx->comments && ctx->id == UCL_EMIT_CONFIG) {
+ comment = ucl_object_lookup_len (ctx->comments, (const char *)&obj,
+ sizeof (void *));
+
+ if (comment) {
+ if (!(comment->flags & UCL_OBJECT_INHERITED)) {
+ DL_FOREACH (comment, cur_comment) {
+ func->ucl_emitter_append_len (cur_comment->value.sv,
+ cur_comment->len,
+ func->ud);
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ ucl_add_tabs (func, ctx->indent, compact);
+ }
+
+ comment = NULL;
+ }
+ }
+ }
+
+ switch (obj->type) {
+ case UCL_INT:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ func->ucl_emitter_append_int (ucl_object_toint (obj), func->ud);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ func->ucl_emitter_append_double (ucl_object_todouble (obj), func->ud);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_BOOLEAN:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ flag = ucl_object_toboolean (obj);
+ if (flag) {
+ func->ucl_emitter_append_len ("true", 4, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("false", 5, func->ud);
+ }
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_STRING:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ if (ctx->id == UCL_EMIT_CONFIG) {
+ if (ucl_maybe_long_string (obj)) {
+ ucl_elt_string_write_multiline (obj->value.sv, obj->len, ctx);
+ } else {
+ if (obj->flags & UCL_OBJECT_SQUOTED) {
+ ucl_elt_string_write_squoted (obj->value.sv, obj->len, ctx);
+ } else {
+ ucl_elt_string_write_json (obj->value.sv, obj->len, ctx);
+ }
+ }
+ }
+ else {
+ ucl_elt_string_write_json (obj->value.sv, obj->len, ctx);
+ }
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_NULL:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ func->ucl_emitter_append_len ("null", 4, func->ud);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_OBJECT:
+ ucl_emitter_common_start_object (ctx, obj, print_key, compact);
+ ucl_emitter_common_end_object (ctx, obj, compact);
+ break;
+ case UCL_ARRAY:
+ ucl_emitter_common_start_array (ctx, obj, print_key, compact);
+ ucl_emitter_common_end_array (ctx, obj, compact);
+ break;
+ case UCL_USERDATA:
+ ud = (struct ucl_object_userdata *)obj;
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ if (ud->emitter) {
+ ud_out = ud->emitter (obj->value.ud);
+ if (ud_out == NULL) {
+ ud_out = "null";
+ }
+ }
+ ucl_elt_string_write_json (ud_out, strlen (ud_out), ctx);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ }
+
+ if (comment) {
+ DL_FOREACH (comment, cur_comment) {
+ func->ucl_emitter_append_len (cur_comment->value.sv,
+ cur_comment->len,
+ func->ud);
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+
+ if (cur_comment->next) {
+ ucl_add_tabs (func, ctx->indent, compact);
+ }
+ }
+ }
+}
+
+/*
+ * Specific standard implementations of the emitter functions
+ */
+#define UCL_EMIT_TYPE_IMPL(type, compact) \
+ static void ucl_emit_ ## type ## _elt (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool first, bool print_key) { \
+ ucl_emitter_common_elt (ctx, obj, first, print_key, (compact)); \
+ } \
+ static void ucl_emit_ ## type ## _start_obj (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key) { \
+ ucl_emitter_common_start_object (ctx, obj, print_key, (compact)); \
+ } \
+ static void ucl_emit_ ## type## _start_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key) { \
+ ucl_emitter_common_start_array (ctx, obj, print_key, (compact)); \
+ } \
+ static void ucl_emit_ ##type## _end_object (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj) { \
+ ucl_emitter_common_end_object (ctx, obj, (compact)); \
+ } \
+ static void ucl_emit_ ##type## _end_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj) { \
+ ucl_emitter_common_end_array (ctx, obj, (compact)); \
+ }
+
+UCL_EMIT_TYPE_IMPL(json, false)
+UCL_EMIT_TYPE_IMPL(json_compact, true)
+UCL_EMIT_TYPE_IMPL(config, false)
+UCL_EMIT_TYPE_IMPL(yaml, false)
+
+static void
+ucl_emit_msgpack_elt (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key)
+{
+ ucl_object_iter_t it;
+ struct ucl_object_userdata *ud;
+ const char *ud_out;
+ const ucl_object_t *cur, *celt;
+
+ switch (obj->type) {
+ case UCL_INT:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_int_msgpack (ctx, ucl_object_toint (obj));
+ break;
+
+ case UCL_FLOAT:
+ case UCL_TIME:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_double_msgpack (ctx, ucl_object_todouble (obj));
+ break;
+
+ case UCL_BOOLEAN:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_bool_msgpack (ctx, ucl_object_toboolean (obj));
+ break;
+
+ case UCL_STRING:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+
+ if (obj->flags & UCL_OBJECT_BINARY) {
+ ucl_emitter_print_binary_string_msgpack (ctx, obj->value.sv,
+ obj->len);
+ }
+ else {
+ ucl_emitter_print_string_msgpack (ctx, obj->value.sv, obj->len);
+ }
+ break;
+
+ case UCL_NULL:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_null_msgpack (ctx);
+ break;
+
+ case UCL_OBJECT:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emit_msgpack_start_obj (ctx, obj, print_key);
+ it = NULL;
+
+ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
+ LL_FOREACH (cur, celt) {
+ ucl_emit_msgpack_elt (ctx, celt, false, true);
+ /* XXX:
+ * in msgpack the length of objects is encoded within a single elt
+ * so in case of multi-value keys we are using merely the first
+ * element ignoring others
+ */
+ break;
+ }
+ }
+
+ break;
+
+ case UCL_ARRAY:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emit_msgpack_start_array (ctx, obj, print_key);
+ it = NULL;
+
+ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
+ ucl_emit_msgpack_elt (ctx, cur, false, false);
+ }
+
+ break;
+
+ case UCL_USERDATA:
+ ud = (struct ucl_object_userdata *)obj;
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+
+ if (ud->emitter) {
+ ud_out = ud->emitter (obj->value.ud);
+ if (ud_out == NULL) {
+ ud_out = "null";
+ }
+ }
+ ucl_emitter_print_string_msgpack (ctx, obj->value.sv, obj->len);
+ break;
+ }
+}
+
+static void
+ucl_emit_msgpack_start_obj (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key)
+{
+ ucl_emitter_print_object_msgpack (ctx, obj->len);
+}
+
+static void
+ucl_emit_msgpack_start_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key)
+{
+ ucl_emitter_print_array_msgpack (ctx, obj->len);
+}
+
+static void
+ucl_emit_msgpack_end_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+
+}
+
+static void
+ucl_emit_msgpack_end_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+
+}
+
+unsigned char *
+ucl_object_emit (const ucl_object_t *obj, enum ucl_emitter emit_type)
+{
+ return ucl_object_emit_len (obj, emit_type, NULL);
+}
+
+unsigned char *
+ucl_object_emit_len (const ucl_object_t *obj, enum ucl_emitter emit_type,
+ size_t *outlen)
+{
+ unsigned char *res = NULL;
+ struct ucl_emitter_functions *func;
+ UT_string *s;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+
+ func = ucl_object_emit_memory_funcs ((void **)&res);
+
+ if (func != NULL) {
+ s = func->ud;
+ ucl_object_emit_full (obj, emit_type, func, NULL);
+
+ if (outlen != NULL) {
+ *outlen = s->i;
+ }
+
+ ucl_object_emit_funcs_free (func);
+ }
+
+ return res;
+}
+
+bool
+ucl_object_emit_full (const ucl_object_t *obj, enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter,
+ const ucl_object_t *comments)
+{
+ const struct ucl_emitter_context *ctx;
+ struct ucl_emitter_context my_ctx;
+ bool res = false;
+
+ ctx = ucl_emit_get_standard_context (emit_type);
+ if (ctx != NULL) {
+ memcpy (&my_ctx, ctx, sizeof (my_ctx));
+ my_ctx.func = emitter;
+ my_ctx.indent = 0;
+ my_ctx.top = obj;
+ my_ctx.comments = comments;
+
+ my_ctx.ops->ucl_emitter_write_elt (&my_ctx, obj, true, false);
+ res = true;
+ }
+
+ return res;
+}
+
+static const struct ucl_emitter_context ucl_standard_emitters[] = {
+ [UCL_EMIT_JSON] = {
+ .name = "json",
+ .id = UCL_EMIT_JSON,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_JSON]
+ },
+ [UCL_EMIT_JSON_COMPACT] = {
+ .name = "json_compact",
+ .id = UCL_EMIT_JSON_COMPACT,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_JSON_COMPACT]
+ },
+ [UCL_EMIT_CONFIG] = {
+ .name = "config",
+ .id = UCL_EMIT_CONFIG,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_CONFIG]
+ },
+ [UCL_EMIT_YAML] = {
+ .name = "yaml",
+ .id = UCL_EMIT_YAML,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_YAML]
+ },
+ [UCL_EMIT_MSGPACK] = {
+ .name = "msgpack",
+ .id = UCL_EMIT_MSGPACK,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_MSGPACK]
+ }
+};
+
+/**
+ * Get standard emitter context for a specified emit_type
+ * @param emit_type type of emitter
+ * @return context or NULL if input is invalid
+ */
+const struct ucl_emitter_context *
+ucl_emit_get_standard_context (enum ucl_emitter emit_type)
+{
+ if (emit_type >= UCL_EMIT_JSON && emit_type < UCL_EMIT_MAX) {
+ return &ucl_standard_emitters[emit_type];
+ }
+
+ return NULL;
+}
+
+/**
+ * Serialise string
+ * @param str string to emit
+ * @param buf target buffer
+ */
+void
+ucl_elt_string_write_json (const char *str, size_t size,
+ struct ucl_emitter_context *ctx)
+{
+ const char *p = str, *c = str;
+ size_t len = 0;
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ func->ucl_emitter_append_character ('"', 1, func->ud);
+
+ while (size) {
+ if (ucl_test_character (*p, (UCL_CHARACTER_JSON_UNSAFE|
+ UCL_CHARACTER_DENIED|
+ UCL_CHARACTER_WHITESPACE_UNSAFE))) {
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+ switch (*p) {
+ case '\0':
+ func->ucl_emitter_append_len ("\\u0000", 6, func->ud);
+ break;
+ case '\n':
+ func->ucl_emitter_append_len ("\\n", 2, func->ud);
+ break;
+ case '\r':
+ func->ucl_emitter_append_len ("\\r", 2, func->ud);
+ break;
+ case '\b':
+ func->ucl_emitter_append_len ("\\b", 2, func->ud);
+ break;
+ case '\t':
+ func->ucl_emitter_append_len ("\\t", 2, func->ud);
+ break;
+ case '\f':
+ func->ucl_emitter_append_len ("\\f", 2, func->ud);
+ break;
+ case '\v':
+ func->ucl_emitter_append_len ("\\u000B", 6, func->ud);
+ break;
+ case '\\':
+ func->ucl_emitter_append_len ("\\\\", 2, func->ud);
+ break;
+ case ' ':
+ func->ucl_emitter_append_character (' ', 1, func->ud);
+ break;
+ case '"':
+ func->ucl_emitter_append_len ("\\\"", 2, func->ud);
+ break;
+ default:
+ /* Emit unicode unknown character */
+ func->ucl_emitter_append_len ("\\uFFFD", 6, func->ud);
+ break;
+ }
+ len = 0;
+ c = ++p;
+ }
+ else {
+ p ++;
+ len ++;
+ }
+ size --;
+ }
+
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+
+ func->ucl_emitter_append_character ('"', 1, func->ud);
+}
+
+void
+ucl_elt_string_write_squoted (const char *str, size_t size,
+ struct ucl_emitter_context *ctx)
+{
+ const char *p = str, *c = str;
+ size_t len = 0;
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ func->ucl_emitter_append_character ('\'', 1, func->ud);
+
+ while (size) {
+ if (*p == '\'') {
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+
+ len = 0;
+ c = ++p;
+ func->ucl_emitter_append_len ("\\\'", 2, func->ud);
+ }
+ else {
+ p ++;
+ len ++;
+ }
+ size --;
+ }
+
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+
+ func->ucl_emitter_append_character ('\'', 1, func->ud);
+}
+
+void
+ucl_elt_string_write_multiline (const char *str, size_t size,
+ struct ucl_emitter_context *ctx)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ func->ucl_emitter_append_len ("<<EOD\n", sizeof ("<<EOD\n") - 1, func->ud);
+ func->ucl_emitter_append_len (str, size, func->ud);
+ func->ucl_emitter_append_len ("\nEOD", sizeof ("\nEOD") - 1, func->ud);
+}
+
+/*
+ * Generic utstring output
+ */
+static int
+ucl_utstring_append_character (unsigned char c, size_t len, void *ud)
+{
+ UT_string *buf = ud;
+
+ if (len == 1) {
+ utstring_append_c (buf, c);
+ }
+ else {
+ utstring_reserve (buf, len + 1);
+ memset (&buf->d[buf->i], c, len);
+ buf->i += len;
+ buf->d[buf->i] = '\0';
+ }
+
+ return 0;
+}
+
+static int
+ucl_utstring_append_len (const unsigned char *str, size_t len, void *ud)
+{
+ UT_string *buf = ud;
+
+ utstring_append_len (buf, str, len);
+
+ return 0;
+}
+
+static int
+ucl_utstring_append_int (int64_t val, void *ud)
+{
+ UT_string *buf = ud;
+
+ utstring_printf (buf, "%jd", (intmax_t)val);
+ return 0;
+}
+
+static int
+ucl_utstring_append_double (double val, void *ud)
+{
+ UT_string *buf = ud;
+ const double delta = 0.0000001;
+
+ if (val == (double)(int)val) {
+ utstring_printf (buf, "%.1lf", val);
+ }
+ else if (fabs (val - (double)(int)val) < delta) {
+ /* Write at maximum precision */
+ utstring_printf (buf, "%.*lg", DBL_DIG, val);
+ }
+ else {
+ utstring_printf (buf, "%lf", val);
+ }
+
+ return 0;
+}
+
+/*
+ * Generic file output
+ */
+static int
+ucl_file_append_character (unsigned char c, size_t len, void *ud)
+{
+ FILE *fp = ud;
+
+ while (len --) {
+ fputc (c, fp);
+ }
+
+ return 0;
+}
+
+static int
+ucl_file_append_len (const unsigned char *str, size_t len, void *ud)
+{
+ FILE *fp = ud;
+
+ fwrite (str, len, 1, fp);
+
+ return 0;
+}
+
+static int
+ucl_file_append_int (int64_t val, void *ud)
+{
+ FILE *fp = ud;
+
+ fprintf (fp, "%jd", (intmax_t)val);
+
+ return 0;
+}
+
+static int
+ucl_file_append_double (double val, void *ud)
+{
+ FILE *fp = ud;
+ const double delta = 0.0000001;
+
+ if (val == (double)(int)val) {
+ fprintf (fp, "%.1lf", val);
+ }
+ else if (fabs (val - (double)(int)val) < delta) {
+ /* Write at maximum precision */
+ fprintf (fp, "%.*lg", DBL_DIG, val);
+ }
+ else {
+ fprintf (fp, "%lf", val);
+ }
+
+ return 0;
+}
+
+/*
+ * Generic file descriptor writing functions
+ */
+static int
+ucl_fd_append_character (unsigned char c, size_t len, void *ud)
+{
+ int fd = *(int *)ud;
+ unsigned char *buf;
+
+ if (len == 1) {
+ return write (fd, &c, 1);
+ }
+ else {
+ buf = malloc (len);
+ if (buf == NULL) {
+ /* Fallback */
+ while (len --) {
+ if (write (fd, &c, 1) == -1) {
+ return -1;
+ }
+ }
+ }
+ else {
+ memset (buf, c, len);
+ if (write (fd, buf, len) == -1) {
+ free(buf);
+ return -1;
+ }
+ free (buf);
+ }
+ }
+
+ return 0;
+}
+
+static int
+ucl_fd_append_len (const unsigned char *str, size_t len, void *ud)
+{
+ int fd = *(int *)ud;
+
+ return write (fd, str, len);
+}
+
+static int
+ucl_fd_append_int (int64_t val, void *ud)
+{
+ int fd = *(int *)ud;
+ char intbuf[64];
+
+ snprintf (intbuf, sizeof (intbuf), "%jd", (intmax_t)val);
+ return write (fd, intbuf, strlen (intbuf));
+}
+
+static int
+ucl_fd_append_double (double val, void *ud)
+{
+ int fd = *(int *)ud;
+ const double delta = 0.0000001;
+ char nbuf[64];
+
+ if (val == (double)(int)val) {
+ snprintf (nbuf, sizeof (nbuf), "%.1lf", val);
+ }
+ else if (fabs (val - (double)(int)val) < delta) {
+ /* Write at maximum precision */
+ snprintf (nbuf, sizeof (nbuf), "%.*lg", DBL_DIG, val);
+ }
+ else {
+ snprintf (nbuf, sizeof (nbuf), "%lf", val);
+ }
+
+ return write (fd, nbuf, strlen (nbuf));
+}
+
+struct ucl_emitter_functions*
+ucl_object_emit_memory_funcs (void **pmem)
+{
+ struct ucl_emitter_functions *f;
+ UT_string *s;
+
+ f = calloc (1, sizeof (*f));
+
+ if (f != NULL) {
+ f->ucl_emitter_append_character = ucl_utstring_append_character;
+ f->ucl_emitter_append_double = ucl_utstring_append_double;
+ f->ucl_emitter_append_int = ucl_utstring_append_int;
+ f->ucl_emitter_append_len = ucl_utstring_append_len;
+ f->ucl_emitter_free_func = free;
+ utstring_new (s);
+ f->ud = s;
+ *pmem = s->d;
+ s->pd = pmem;
+ }
+
+ return f;
+}
+
+struct ucl_emitter_functions*
+ucl_object_emit_file_funcs (FILE *fp)
+{
+ struct ucl_emitter_functions *f;
+
+ f = calloc (1, sizeof (*f));
+
+ if (f != NULL) {
+ f->ucl_emitter_append_character = ucl_file_append_character;
+ f->ucl_emitter_append_double = ucl_file_append_double;
+ f->ucl_emitter_append_int = ucl_file_append_int;
+ f->ucl_emitter_append_len = ucl_file_append_len;
+ f->ucl_emitter_free_func = NULL;
+ f->ud = fp;
+ }
+
+ return f;
+}
+
+struct ucl_emitter_functions*
+ucl_object_emit_fd_funcs (int fd)
+{
+ struct ucl_emitter_functions *f;
+ int *ip;
+
+ f = calloc (1, sizeof (*f));
+
+ if (f != NULL) {
+ ip = malloc (sizeof (fd));
+ if (ip == NULL) {
+ free (f);
+ return NULL;
+ }
+
+ memcpy (ip, &fd, sizeof (fd));
+ f->ucl_emitter_append_character = ucl_fd_append_character;
+ f->ucl_emitter_append_double = ucl_fd_append_double;
+ f->ucl_emitter_append_int = ucl_fd_append_int;
+ f->ucl_emitter_append_len = ucl_fd_append_len;
+ f->ucl_emitter_free_func = free;
+ f->ud = ip;
+ }
+
+ return f;
+}
+
+void
+ucl_object_emit_funcs_free (struct ucl_emitter_functions *f)
+{
+ if (f != NULL) {
+ if (f->ucl_emitter_free_func != NULL) {
+ f->ucl_emitter_free_func (f->ud);
+ }
+ free (f);
+ }
+}
+
+
+unsigned char *
+ucl_object_emit_single_json (const ucl_object_t *obj)
+{
+ UT_string *buf = NULL;
+ unsigned char *res = NULL;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+
+ utstring_new (buf);
+
+ if (buf != NULL) {
+ switch (obj->type) {
+ case UCL_OBJECT:
+ ucl_utstring_append_len ("object", 6, buf);
+ break;
+ case UCL_ARRAY:
+ ucl_utstring_append_len ("array", 5, buf);
+ break;
+ case UCL_INT:
+ ucl_utstring_append_int (obj->value.iv, buf);
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ ucl_utstring_append_double (obj->value.dv, buf);
+ break;
+ case UCL_NULL:
+ ucl_utstring_append_len ("null", 4, buf);
+ break;
+ case UCL_BOOLEAN:
+ if (obj->value.iv) {
+ ucl_utstring_append_len ("true", 4, buf);
+ }
+ else {
+ ucl_utstring_append_len ("false", 5, buf);
+ }
+ break;
+ case UCL_STRING:
+ ucl_utstring_append_len (obj->value.sv, obj->len, buf);
+ break;
+ case UCL_USERDATA:
+ ucl_utstring_append_len ("userdata", 8, buf);
+ break;
+ }
+ res = utstring_body (buf);
+ free (buf);
+ }
+
+ return res;
+}
+
+#define LONG_STRING_LIMIT 80
+
+bool
+ucl_maybe_long_string (const ucl_object_t *obj)
+{
+ if (obj->len > LONG_STRING_LIMIT || (obj->flags & UCL_OBJECT_MULTILINE)) {
+ /* String is long enough, so search for newline characters in it */
+ if (memchr (obj->value.sv, '\n', obj->len) != NULL) {
+ return true;
+ }
+ }
+
+ return false;
+}
diff --git a/contrib/libucl/ucl_emitter_streamline.c b/contrib/libucl/ucl_emitter_streamline.c
new file mode 100644
index 0000000..a7178c5
--- /dev/null
+++ b/contrib/libucl/ucl_emitter_streamline.c
@@ -0,0 +1,173 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+
+struct ucl_emitter_streamline_stack {
+ bool is_array;
+ bool empty;
+ const ucl_object_t *obj;
+ struct ucl_emitter_streamline_stack *next;
+};
+
+struct ucl_emitter_context_streamline {
+ /* Inherited from the main context */
+ /** Name of emitter (e.g. json, compact_json) */
+ const char *name;
+ /** Unique id (e.g. UCL_EMIT_JSON for standard emitters */
+ int id;
+ /** A set of output functions */
+ const struct ucl_emitter_functions *func;
+ /** A set of output operations */
+ const struct ucl_emitter_operations *ops;
+ /** Current amount of indent tabs */
+ unsigned int indent;
+ /** Top level object */
+ const ucl_object_t *top;
+ /** Optional comments */
+ const ucl_object_t *comments;
+
+ /* Streamline specific fields */
+ struct ucl_emitter_streamline_stack *containers;
+};
+
+#define TO_STREAMLINE(ctx) (struct ucl_emitter_context_streamline *)(ctx)
+
+struct ucl_emitter_context*
+ucl_object_emit_streamline_new (const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter)
+{
+ const struct ucl_emitter_context *ctx;
+ struct ucl_emitter_context_streamline *sctx;
+
+ ctx = ucl_emit_get_standard_context (emit_type);
+ if (ctx == NULL) {
+ return NULL;
+ }
+
+ sctx = calloc (1, sizeof (*sctx));
+ if (sctx == NULL) {
+ return NULL;
+ }
+
+ memcpy (sctx, ctx, sizeof (*ctx));
+ sctx->func = emitter;
+ sctx->top = obj;
+
+ ucl_object_emit_streamline_start_container ((struct ucl_emitter_context *)sctx,
+ obj);
+
+ return (struct ucl_emitter_context *)sctx;
+}
+
+void
+ucl_object_emit_streamline_start_container (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+ struct ucl_emitter_streamline_stack *st, *top;
+ bool print_key = false;
+
+ /* Check top object presence */
+ if (sctx->top == NULL) {
+ sctx->top = obj;
+ }
+
+ top = sctx->containers;
+ st = malloc (sizeof (*st));
+ if (st != NULL) {
+ if (top != NULL && !top->is_array) {
+ print_key = true;
+ }
+ st->empty = true;
+ st->obj = obj;
+ if (obj != NULL && obj->type == UCL_ARRAY) {
+ st->is_array = true;
+ sctx->ops->ucl_emitter_start_array (ctx, obj, print_key);
+ }
+ else {
+ st->is_array = false;
+ sctx->ops->ucl_emitter_start_object (ctx, obj, print_key);
+ }
+ LL_PREPEND (sctx->containers, st);
+ }
+}
+
+void
+ucl_object_emit_streamline_add_object (
+ struct ucl_emitter_context *ctx, const ucl_object_t *obj)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+ bool is_array = false, is_first = false;
+
+ if (sctx->containers != NULL) {
+ if (sctx->containers->is_array) {
+ is_array = true;
+ }
+ if (sctx->containers->empty) {
+ is_first = true;
+ sctx->containers->empty = false;
+ }
+ }
+
+ sctx->ops->ucl_emitter_write_elt (ctx, obj, is_first, !is_array);
+}
+
+void
+ucl_object_emit_streamline_end_container (struct ucl_emitter_context *ctx)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+ struct ucl_emitter_streamline_stack *st;
+
+ if (sctx->containers != NULL) {
+ st = sctx->containers;
+
+ if (st->is_array) {
+ sctx->ops->ucl_emitter_end_array (ctx, st->obj);
+ }
+ else {
+ sctx->ops->ucl_emitter_end_object (ctx, st->obj);
+ }
+ sctx->containers = st->next;
+ free (st);
+ }
+}
+
+void
+ucl_object_emit_streamline_finish (struct ucl_emitter_context *ctx)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+
+ while (sctx->containers != NULL) {
+ ucl_object_emit_streamline_end_container (ctx);
+ }
+
+ free (sctx);
+}
diff --git a/contrib/libucl/ucl_hash.c b/contrib/libucl/ucl_hash.c
new file mode 100644
index 0000000..a26c26f
--- /dev/null
+++ b/contrib/libucl/ucl_hash.c
@@ -0,0 +1,498 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ucl_internal.h"
+#include "ucl_hash.h"
+#include "khash.h"
+#include "utlist.h"
+
+#include "cryptobox.h"
+#include "libutil/str_util.h"
+#include "ucl.h"
+
+#include <time.h>
+#include <limits.h>
+
+struct ucl_hash_elt {
+ const ucl_object_t *obj;
+ struct ucl_hash_elt *prev, *next;
+};
+
+struct ucl_hash_struct {
+ void *hash;
+ struct ucl_hash_elt *head;
+ bool caseless;
+};
+
+static uint64_t
+ucl_hash_seed (void)
+{
+ static uint64_t seed;
+ if (seed == 0) {
+#ifdef UCL_RANDOM_FUNCTION
+ seed = UCL_RANDOM_FUNCTION;
+#else
+ /* Not very random but can be useful for our purposes */
+ seed = time (NULL);
+#endif
+ }
+
+ return seed;
+}
+
+extern const guchar lc_map[256];
+
+static inline uint32_t
+ucl_hash_func (const ucl_object_t *o)
+{
+ return (uint32_t)rspamd_cryptobox_fast_hash (o->key, o->keylen, 0xb9a1ef83c4561c95ULL);
+}
+
+static inline int
+ucl_hash_equal (const ucl_object_t *k1, const ucl_object_t *k2)
+{
+ if (k1->keylen == k2->keylen) {
+ return memcmp (k1->key, k2->key, k1->keylen) == 0;
+ }
+
+ return 0;
+}
+
+KHASH_INIT (ucl_hash_node, const ucl_object_t *, struct ucl_hash_elt *, 1,
+ ucl_hash_func, ucl_hash_equal)
+
+static inline uint32_t
+ucl_hash_caseless_func (const ucl_object_t *o)
+{
+ unsigned len = o->keylen;
+ unsigned leftover = o->keylen % 4;
+ unsigned fp, i;
+ const uint8_t* s = (const uint8_t*)o->key;
+ union {
+ struct {
+ unsigned char c1, c2, c3, c4;
+ } c;
+ uint32_t pp;
+ } u;
+ uint64_t h = 0xe5ae6ab1ef9f3b54ULL;
+ rspamd_cryptobox_fast_hash_state_t hst;
+
+ fp = len - leftover;
+ rspamd_cryptobox_fast_hash_init (&hst, h);
+
+ for (i = 0; i != fp; i += 4) {
+ u.c.c1 = s[i], u.c.c2 = s[i + 1], u.c.c3 = s[i + 2], u.c.c4 = s[i + 3];
+ u.c.c1 = lc_map[u.c.c1];
+ u.c.c2 = lc_map[u.c.c2];
+ u.c.c3 = lc_map[u.c.c3];
+ u.c.c4 = lc_map[u.c.c4];
+ rspamd_cryptobox_fast_hash_update (&hst, &u, sizeof (u));
+ }
+
+ u.pp = 0;
+ switch (leftover) {
+ case 3:
+ u.c.c3 = lc_map[(unsigned char)s[i++]];
+ case 2:
+ /* fallthrough */
+ u.c.c2 = lc_map[(unsigned char)s[i++]];
+ case 1:
+ /* fallthrough */
+ u.c.c1 = lc_map[(unsigned char)s[i]];
+ rspamd_cryptobox_fast_hash_update (&hst, &u, sizeof (u));
+ break;
+ }
+
+ return (uint32_t)rspamd_cryptobox_fast_hash_final (&hst);
+}
+
+
+static inline bool
+ucl_hash_caseless_equal (const ucl_object_t *k1, const ucl_object_t *k2)
+{
+ if (k1->keylen == k2->keylen) {
+ return rspamd_lc_cmp (k1->key, k2->key, k1->keylen) == 0;
+ }
+
+ return false;
+}
+
+KHASH_INIT (ucl_hash_caseless_node, const ucl_object_t *, struct ucl_hash_elt *, 1,
+ ucl_hash_caseless_func, ucl_hash_caseless_equal)
+
+ucl_hash_t*
+ucl_hash_create (bool ignore_case)
+{
+ ucl_hash_t *new;
+
+ new = UCL_ALLOC (sizeof (ucl_hash_t));
+ if (new != NULL) {
+ void *h;
+ new->head = NULL;
+ new->caseless = ignore_case;
+ if (ignore_case) {
+ h = (void *)kh_init (ucl_hash_caseless_node);
+ }
+ else {
+ h = (void *)kh_init (ucl_hash_node);
+ }
+ if (h == NULL) {
+ UCL_FREE (sizeof (ucl_hash_t), new);
+ return NULL;
+ }
+ new->hash = h;
+ }
+ return new;
+}
+
+void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func func)
+{
+
+ if (hashlin == NULL) {
+ return;
+ }
+
+ if (func != NULL) {
+ /* Iterate over the hash first */
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ khiter_t k;
+ const ucl_object_t *cur, *tmp;
+
+ for (k = kh_begin (h); k != kh_end (h); ++k) {
+ if (kh_exist (h, k)) {
+ cur = (kh_value (h, k))->obj;
+ while (cur != NULL) {
+ tmp = cur->next;
+ func (__DECONST (ucl_object_t *, cur));
+ cur = tmp;
+ }
+ }
+ }
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+ kh_destroy (ucl_hash_caseless_node, h);
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ kh_destroy (ucl_hash_node, h);
+ }
+
+ struct ucl_hash_elt *cur, *tmp;
+
+ DL_FOREACH_SAFE(hashlin->head, cur, tmp) {
+ UCL_FREE(sizeof(*cur), cur);
+ }
+
+ UCL_FREE (sizeof (*hashlin), hashlin);
+}
+
+bool
+ucl_hash_insert (ucl_hash_t* hashlin, const ucl_object_t *obj,
+ const char *key, unsigned keylen)
+{
+ khiter_t k;
+ int ret;
+ struct ucl_hash_elt **pelt, *elt;
+
+ if (hashlin == NULL) {
+ return false;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_caseless_node, h, obj, &ret);
+ if (ret > 0) {
+ elt = UCL_ALLOC(sizeof(*elt));
+ pelt = &kh_value (h, k);
+ *pelt = elt;
+ DL_APPEND(hashlin->head, elt);
+ elt->obj = obj;
+ }
+ else if (ret < 0) {
+ goto e0;
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_node, h, obj, &ret);
+ if (ret > 0) {
+ elt = UCL_ALLOC(sizeof(*elt));
+ pelt = &kh_value (h, k);
+ *pelt = elt;
+ DL_APPEND(hashlin->head, elt);
+ elt->obj = obj;
+ } else if (ret < 0) {
+ goto e0;
+ }
+ }
+ return true;
+ e0:
+ return false;
+}
+
+void ucl_hash_replace (ucl_hash_t* hashlin, const ucl_object_t *old,
+ const ucl_object_t *new)
+{
+ khiter_t k;
+ int ret;
+ struct ucl_hash_elt *elt, *nelt;
+
+ if (hashlin == NULL) {
+ return;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_caseless_node, h, old, &ret);
+ if (ret == 0) {
+ elt = kh_value(h, k);
+ kh_del (ucl_hash_caseless_node, h, k);
+ k = kh_put (ucl_hash_caseless_node, h, new, &ret);
+ nelt = UCL_ALLOC(sizeof(*nelt));
+ nelt->obj = new;
+ kh_value(h, k) = nelt;
+ DL_REPLACE_ELEM(hashlin->head, elt, nelt);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_node, h, old, &ret);
+ if (ret == 0) {
+ elt = kh_value (h, k);
+ kh_del (ucl_hash_node, h, k);
+ k = kh_put (ucl_hash_node, h, new, &ret);
+ nelt = UCL_ALLOC(sizeof(*nelt));
+ nelt->obj = new;
+ kh_value(h, k) = nelt;
+ DL_REPLACE_ELEM(hashlin->head, elt, nelt);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+}
+
+struct ucl_hash_real_iter {
+ const struct ucl_hash_elt *cur;
+};
+
+#define UHI_SETERR(ep, ern) {if (ep != NULL) *ep = (ern);}
+
+const void*
+ucl_hash_iterate2 (ucl_hash_t *hashlin, ucl_hash_iter_t *iter, int *ep)
+{
+ struct ucl_hash_real_iter *it = (struct ucl_hash_real_iter *)(*iter);
+ const ucl_object_t *ret = NULL;
+
+ if (hashlin == NULL) {
+ UHI_SETERR(ep, EINVAL);
+ return NULL;
+ }
+
+ if (it == NULL) {
+ it = UCL_ALLOC (sizeof (*it));
+
+ if (it == NULL) {
+ UHI_SETERR(ep, ENOMEM);
+ return NULL;
+ }
+
+ it->cur = hashlin->head;
+ }
+
+ UHI_SETERR(ep, 0);
+ if (it->cur) {
+ ret = it->cur->obj;
+ it->cur = it->cur->next;
+ }
+ else {
+ UCL_FREE (sizeof (*it), it);
+ *iter = NULL;
+ return NULL;
+ }
+
+ *iter = it;
+
+ return ret;
+}
+
+bool
+ucl_hash_iter_has_next (ucl_hash_t *hashlin, ucl_hash_iter_t iter)
+{
+ struct ucl_hash_real_iter *it = (struct ucl_hash_real_iter *)(iter);
+
+ return it->cur != NULL;
+}
+
+
+const ucl_object_t*
+ucl_hash_search (ucl_hash_t* hashlin, const char *key, unsigned keylen)
+{
+ khiter_t k;
+ const ucl_object_t *ret = NULL;
+ ucl_object_t search;
+ struct ucl_hash_elt *elt;
+
+ search.key = key;
+ search.keylen = keylen;
+
+ if (hashlin == NULL) {
+ return NULL;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+
+ k = kh_get (ucl_hash_caseless_node, h, &search);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ ret = elt->obj;
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_get (ucl_hash_node, h, &search);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ ret = elt->obj;
+ }
+ }
+
+ return ret;
+}
+
+void
+ucl_hash_delete (ucl_hash_t* hashlin, const ucl_object_t *obj)
+{
+ khiter_t k;
+ struct ucl_hash_elt *elt;
+
+ if (hashlin == NULL) {
+ return;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+
+ k = kh_get (ucl_hash_caseless_node, h, obj);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ DL_DELETE(hashlin->head, elt);
+ kh_del (ucl_hash_caseless_node, h, k);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_get (ucl_hash_node, h, obj);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ DL_DELETE(hashlin->head, elt);
+ kh_del (ucl_hash_node, h, k);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+}
+
+bool
+ucl_hash_reserve (ucl_hash_t *hashlin, size_t sz)
+{
+ if (hashlin == NULL) {
+ return false;
+ }
+
+ if (sz > kh_size((khash_t(ucl_hash_node) *)hashlin->hash)) {
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(
+ ucl_hash_caseless_node) *)
+ hashlin->hash;
+ kh_resize (ucl_hash_caseless_node, h, sz * 2);
+ } else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ kh_resize (ucl_hash_node, h, sz * 2);
+ }
+ }
+
+ return true;
+}
+
+static int
+ucl_hash_cmp_icase (const void *a, const void *b)
+{
+ const struct ucl_hash_elt *oa = (const struct ucl_hash_elt *)a,
+ *ob = (const struct ucl_hash_elt *)b;
+
+ if (oa->obj->keylen == ob->obj->keylen) {
+ return rspamd_lc_cmp (oa->obj->key, ob->obj->key, oa->obj->keylen);
+ }
+
+ return ((int)(oa->obj->keylen)) - ob->obj->keylen;
+}
+
+static int
+ucl_hash_cmp_case_sens (const void *a, const void *b)
+{
+ const struct ucl_hash_elt *oa = (const struct ucl_hash_elt *)a,
+ *ob = (const struct ucl_hash_elt *)b;
+
+ if (oa->obj->keylen == ob->obj->keylen) {
+ return memcmp (oa->obj->key, ob->obj->key, oa->obj->keylen);
+ }
+
+ return ((int)(oa->obj->keylen)) - ob->obj->keylen;
+}
+
+void
+ucl_hash_sort (ucl_hash_t *hashlin, enum ucl_object_keys_sort_flags fl)
+{
+
+ if (fl & UCL_SORT_KEYS_ICASE) {
+ DL_SORT(hashlin->head, ucl_hash_cmp_icase);
+ }
+ else {
+ DL_SORT(hashlin->head, ucl_hash_cmp_case_sens);
+ }
+
+ if (fl & UCL_SORT_KEYS_RECURSIVE) {
+ struct ucl_hash_elt *elt;
+
+ DL_FOREACH(hashlin->head, elt) {
+ if (ucl_object_type (elt->obj) == UCL_OBJECT) {
+ ucl_hash_sort (elt->obj->value.ov, fl);
+ }
+ }
+ }
+} \ No newline at end of file
diff --git a/contrib/libucl/ucl_hash.h b/contrib/libucl/ucl_hash.h
new file mode 100644
index 0000000..c2d5517
--- /dev/null
+++ b/contrib/libucl/ucl_hash.h
@@ -0,0 +1,114 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __UCL_HASH_H
+#define __UCL_HASH_H
+
+#include "ucl.h"
+
+/******************************************************************************/
+
+struct ucl_hash_node_s;
+typedef struct ucl_hash_node_s ucl_hash_node_t;
+
+typedef int (*ucl_hash_cmp_func) (const void* void_a, const void* void_b);
+typedef void (*ucl_hash_free_func) (void *ptr);
+typedef void* ucl_hash_iter_t;
+
+
+/**
+ * Linear chained hashtable.
+ */
+struct ucl_hash_struct;
+typedef struct ucl_hash_struct ucl_hash_t;
+
+
+/**
+ * Initializes the hashtable.
+ */
+ucl_hash_t* ucl_hash_create (bool ignore_case);
+
+/**
+ * Deinitializes the hashtable.
+ */
+void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func func);
+
+/**
+ * Inserts an element in the the hashtable.
+ * @return true on success, false on failure (i.e. ENOMEM)
+ */
+bool ucl_hash_insert (ucl_hash_t* hashlin, const ucl_object_t *obj, const char *key,
+ unsigned keylen);
+
+/**
+ * Replace element in the hash
+ */
+void ucl_hash_replace (ucl_hash_t* hashlin, const ucl_object_t *old,
+ const ucl_object_t *new);
+
+/**
+ * Delete an element from the the hashtable.
+ */
+void ucl_hash_delete (ucl_hash_t* hashlin, const ucl_object_t *obj);
+
+/**
+ * Searches an element in the hashtable.
+ */
+const ucl_object_t* ucl_hash_search (ucl_hash_t* hashlin, const char *key,
+ unsigned keylen);
+
+
+/**
+ * Iterate over hash table
+ * @param hashlin hash
+ * @param iter iterator (must be NULL on first iteration)
+ * @param ep pointer record exception (such as ENOMEM), could be NULL
+ * @return the next object
+ */
+const void* ucl_hash_iterate2 (ucl_hash_t *hashlin, ucl_hash_iter_t *iter, int *ep);
+
+/**
+ * Helper macro to support older code
+ */
+#define ucl_hash_iterate(hl, ip) ucl_hash_iterate2((hl), (ip), NULL)
+
+/**
+ * Check whether an iterator has next element
+ */
+bool ucl_hash_iter_has_next (ucl_hash_t *hashlin, ucl_hash_iter_t iter);
+
+/**
+ * Reserves space in hash
+ * @return true on sucess, false on failure (e.g. ENOMEM)
+ * @param hashlin
+ */
+bool ucl_hash_reserve (ucl_hash_t *hashlin, size_t sz);
+
+/**
+ * Sorts keys in a hash
+ * @param hashlin
+ * @param fl
+ */
+void ucl_hash_sort (ucl_hash_t *hashlin, enum ucl_object_keys_sort_flags fl);
+
+#endif
diff --git a/contrib/libucl/ucl_internal.h b/contrib/libucl/ucl_internal.h
new file mode 100644
index 0000000..a7dd6ee
--- /dev/null
+++ b/contrib/libucl/ucl_internal.h
@@ -0,0 +1,666 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UCL_INTERNAL_H_
+#define UCL_INTERNAL_H_
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#else
+/* Help embedded builds */
+#define HAVE_SYS_TYPES_H
+#define HAVE_SYS_MMAN_H
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_PARAM_H
+#define HAVE_LIMITS_H
+#define HAVE_FCNTL_H
+#define HAVE_ERRNO_H
+#define HAVE_UNISTD_H
+#define HAVE_CTYPE_H
+#define HAVE_STDIO_H
+#define HAVE_STRING_H
+#define HAVE_FLOAT_H
+#define HAVE_LIBGEN_H
+#define HAVE_MATH_H
+#define HAVE_STDBOOL_H
+#define HAVE_STDINT_H
+#define HAVE_STDARG_H
+#ifndef _WIN32
+# define HAVE_REGEX_H
+#endif
+#endif
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+#ifdef HAVE_SYS_MMAN_H
+# ifndef _WIN32
+# include <sys/mman.h>
+# endif
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+# ifndef _WIN32
+# include <sys/param.h>
+# endif
+#endif
+
+#ifdef HAVE_LIMITS_H
+#include <limits.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+#ifdef HAVE_UNISTD_H
+# ifndef _WIN32
+# include <unistd.h>
+# endif
+#endif
+#ifdef HAVE_CTYPE_H
+#include <ctype.h>
+#endif
+#ifdef HAVE_STDIO_H
+#include <stdio.h>
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#ifdef HAVE_STRINGS_H
+#include <strings.h>
+#endif
+
+#if defined(_MSC_VER)
+/* Windows hacks */
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#define strdup _strdup
+#define snprintf _snprintf
+#define vsnprintf _vsnprintf
+#define strcasecmp _stricmp
+#define strncasecmp _strnicmp
+#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#if _MSC_VER >= 1900
+#include <../ucrt/stdlib.h>
+#else
+#include <../include/stdlib.h>
+#endif
+#ifndef PATH_MAX
+#define PATH_MAX _MAX_PATH
+#endif
+
+/* Dirname, basename implementations */
+
+
+#endif
+
+#include "utlist.h"
+#include "utstring.h"
+#include "uthash.h"
+#include "ucl.h"
+#include "ucl_hash.h"
+
+#ifdef HAVE_OPENSSL
+#include <openssl/evp.h>
+#endif
+
+#ifndef __DECONST
+#define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
+#endif
+
+/**
+ * @file rcl_internal.h
+ * Internal structures and functions of UCL library
+ */
+
+#define UCL_MAX_RECURSION 16
+#define UCL_TRASH_KEY 0
+#define UCL_TRASH_VALUE 1
+
+enum ucl_parser_state {
+ UCL_STATE_INIT = 0,
+ UCL_STATE_OBJECT,
+ UCL_STATE_ARRAY,
+ UCL_STATE_KEY,
+ UCL_STATE_KEY_OBRACE,
+ UCL_STATE_VALUE,
+ UCL_STATE_AFTER_VALUE,
+ UCL_STATE_ARRAY_VALUE,
+ UCL_STATE_SCOMMENT,
+ UCL_STATE_MCOMMENT,
+ UCL_STATE_MACRO_NAME,
+ UCL_STATE_MACRO,
+ UCL_STATE_ERROR
+};
+
+enum ucl_character_type {
+ UCL_CHARACTER_DENIED = (1 << 0),
+ UCL_CHARACTER_KEY = (1 << 1),
+ UCL_CHARACTER_KEY_START = (1 << 2),
+ UCL_CHARACTER_WHITESPACE = (1 << 3),
+ UCL_CHARACTER_WHITESPACE_UNSAFE = (1 << 4),
+ UCL_CHARACTER_VALUE_END = (1 << 5),
+ UCL_CHARACTER_VALUE_STR = (1 << 6),
+ UCL_CHARACTER_VALUE_DIGIT = (1 << 7),
+ UCL_CHARACTER_VALUE_DIGIT_START = (1 << 8),
+ UCL_CHARACTER_ESCAPE = (1 << 9),
+ UCL_CHARACTER_KEY_SEP = (1 << 10),
+ UCL_CHARACTER_JSON_UNSAFE = (1 << 11),
+ UCL_CHARACTER_UCL_UNSAFE = (1 << 12)
+};
+
+struct ucl_macro {
+ char *name;
+ union _ucl_macro {
+ ucl_macro_handler handler;
+ ucl_context_macro_handler context_handler;
+ } h;
+ void* ud;
+ bool is_context;
+ UT_hash_handle hh;
+};
+
+enum ucl_stack_flags {
+ UCL_STACK_HAS_OBRACE = (1u << 0),
+ UCL_STACK_MAX = (1u << 1),
+};
+
+struct ucl_stack {
+ ucl_object_t *obj;
+ struct ucl_stack *next;
+ union {
+ struct {
+ uint16_t level;
+ uint16_t flags;
+ uint32_t line;
+ } params;
+ uint64_t len;
+ } e;
+ struct ucl_chunk *chunk;
+};
+
+struct ucl_parser_special_handler_chain {
+ unsigned char *begin;
+ size_t len;
+ struct ucl_parser_special_handler *special_handler;
+ struct ucl_parser_special_handler_chain *next;
+};
+
+struct ucl_chunk {
+ const unsigned char *begin;
+ const unsigned char *end;
+ const unsigned char *pos;
+ char *fname;
+ size_t remain;
+ unsigned int line;
+ unsigned int column;
+ unsigned priority;
+ enum ucl_duplicate_strategy strategy;
+ enum ucl_parse_type parse_type;
+ struct ucl_parser_special_handler_chain *special_handlers;
+ struct ucl_chunk *next;
+};
+
+#ifdef HAVE_OPENSSL
+struct ucl_pubkey {
+ EVP_PKEY *key;
+ struct ucl_pubkey *next;
+};
+#else
+struct ucl_pubkey {
+ struct ucl_pubkey *next;
+};
+#endif
+
+struct ucl_variable {
+ char *var;
+ char *value;
+ size_t var_len;
+ size_t value_len;
+ struct ucl_variable *prev, *next;
+};
+
+struct ucl_parser {
+ enum ucl_parser_state state;
+ enum ucl_parser_state prev_state;
+ unsigned int recursion;
+ int flags;
+ unsigned default_priority;
+ int err_code;
+ ucl_object_t *top_obj;
+ ucl_object_t *cur_obj;
+ ucl_object_t *trash_objs;
+ ucl_object_t *includepaths;
+ char *cur_file;
+ struct ucl_macro *macroes;
+ struct ucl_stack *stack;
+ struct ucl_chunk *chunks;
+ struct ucl_pubkey *keys;
+ struct ucl_parser_special_handler *special_handlers;
+ ucl_include_trace_func_t *include_trace_func;
+ void *include_trace_ud;
+ struct ucl_variable *variables;
+ ucl_variable_handler var_handler;
+ void *var_data;
+ ucl_object_t *comments;
+ ucl_object_t *last_comment;
+ UT_string *err;
+};
+
+struct ucl_object_userdata {
+ ucl_object_t obj;
+ ucl_userdata_dtor dtor;
+ ucl_userdata_emitter emitter;
+};
+
+/**
+ * Unescape json string inplace
+ * @param str
+ */
+size_t ucl_unescape_json_string (char *str, size_t len);
+
+
+/**
+ * Unescape single quoted string inplace
+ * @param str
+ */
+size_t ucl_unescape_squoted_string (char *str, size_t len);
+
+/**
+ * Handle include macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle tryinclude macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_try_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle includes macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_includes_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle priority macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_priority_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle load macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_load_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+/**
+ * Handle inherit macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ctx the current context object
+ * @param ud user data
+ * @return
+ */
+bool ucl_inherit_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, const ucl_object_t *ctx, void* ud);
+
+size_t ucl_strlcpy (char *dst, const char *src, size_t siz);
+size_t ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz);
+size_t ucl_strlcpy_tolower (char *dst, const char *src, size_t siz);
+
+char *ucl_strnstr (const char *s, const char *find, int len);
+char *ucl_strncasestr (const char *s, const char *find, int len);
+
+#ifdef __GNUC__
+static inline void
+ucl_create_err (UT_string **err, const char *fmt, ...)
+__attribute__ (( format( printf, 2, 3) ));
+#endif
+
+#undef UCL_FATAL_ERRORS
+
+static inline void
+ucl_create_err (UT_string **err, const char *fmt, ...)
+{
+ if (*err == NULL) {
+ utstring_new (*err);
+ va_list ap;
+ va_start (ap, fmt);
+ utstring_printf_va (*err, fmt, ap);
+ va_end (ap);
+ }
+
+#ifdef UCL_FATAL_ERRORS
+ assert (0);
+#endif
+}
+
+/**
+ * Check whether a given string contains a boolean value
+ * @param obj object to set
+ * @param start start of a string
+ * @param len length of a string
+ * @return true if a string is a boolean value
+ */
+static inline bool
+ucl_maybe_parse_boolean (ucl_object_t *obj, const unsigned char *start, size_t len)
+{
+ const char *p = (const char *)start;
+ bool ret = false, val = false;
+
+ if (len == 5) {
+ if ((p[0] == 'f' || p[0] == 'F') && strncasecmp (p, "false", 5) == 0) {
+ ret = true;
+ val = false;
+ }
+ }
+ else if (len == 4) {
+ if ((p[0] == 't' || p[0] == 'T') && strncasecmp (p, "true", 4) == 0) {
+ ret = true;
+ val = true;
+ }
+ }
+ else if (len == 3) {
+ if ((p[0] == 'y' || p[0] == 'Y') && strncasecmp (p, "yes", 3) == 0) {
+ ret = true;
+ val = true;
+ }
+ else if ((p[0] == 'o' || p[0] == 'O') && strncasecmp (p, "off", 3) == 0) {
+ ret = true;
+ val = false;
+ }
+ }
+ else if (len == 2) {
+ if ((p[0] == 'n' || p[0] == 'N') && strncasecmp (p, "no", 2) == 0) {
+ ret = true;
+ val = false;
+ }
+ else if ((p[0] == 'o' || p[0] == 'O') && strncasecmp (p, "on", 2) == 0) {
+ ret = true;
+ val = true;
+ }
+ }
+
+ if (ret && obj != NULL) {
+ obj->type = UCL_BOOLEAN;
+ obj->value.iv = val;
+ }
+
+ return ret;
+}
+
+/**
+ * Check numeric string
+ * @param obj object to set if a string is numeric
+ * @param start start of string
+ * @param end end of string
+ * @param pos position where parsing has stopped
+ * @param allow_double allow parsing of floating point values
+ * @return 0 if string is numeric and error code (EINVAL or ERANGE) in case of conversion error
+ */
+int ucl_maybe_parse_number (ucl_object_t *obj,
+ const char *start, const char *end, const char **pos,
+ bool allow_double, bool number_bytes, bool allow_time);
+
+
+static inline const ucl_object_t *
+ucl_hash_search_obj (ucl_hash_t* hashlin, ucl_object_t *obj)
+{
+ return (const ucl_object_t *)ucl_hash_search (hashlin, obj->key, obj->keylen);
+}
+
+static inline ucl_hash_t * ucl_hash_insert_object (ucl_hash_t *hashlin,
+ const ucl_object_t *obj,
+ bool ignore_case) UCL_WARN_UNUSED_RESULT;
+
+static inline ucl_hash_t *
+ucl_hash_insert_object (ucl_hash_t *hashlin,
+ const ucl_object_t *obj,
+ bool ignore_case)
+{
+ ucl_hash_t *nhp;
+
+ if (hashlin == NULL) {
+ nhp = ucl_hash_create (ignore_case);
+ if (nhp == NULL) {
+ return NULL;
+ }
+ } else {
+ nhp = hashlin;
+ }
+ if (!ucl_hash_insert (nhp, obj, obj->key, obj->keylen)) {
+ if (nhp != hashlin) {
+ ucl_hash_destroy(nhp, NULL);
+ }
+ return NULL;
+ }
+
+ return nhp;
+}
+
+/**
+ * Get standard emitter context for a specified emit_type
+ * @param emit_type type of emitter
+ * @return context or NULL if input is invalid
+ */
+const struct ucl_emitter_context *
+ucl_emit_get_standard_context (enum ucl_emitter emit_type);
+
+/**
+ * Serialize string as JSON string
+ * @param str string to emit
+ * @param buf target buffer
+ */
+void ucl_elt_string_write_json (const char *str, size_t size,
+ struct ucl_emitter_context *ctx);
+
+
+/**
+ * Serialize string as single quoted string
+ * @param str string to emit
+ * @param buf target buffer
+ */
+void
+ucl_elt_string_write_squoted (const char *str, size_t size,
+ struct ucl_emitter_context *ctx);
+
+/**
+ * Write multiline string using `EOD` as string terminator
+ * @param str
+ * @param size
+ * @param ctx
+ */
+void ucl_elt_string_write_multiline (const char *str, size_t size,
+ struct ucl_emitter_context *ctx);
+
+/**
+ * Emit a single object to string
+ * @param obj
+ * @return
+ */
+unsigned char * ucl_object_emit_single_json (const ucl_object_t *obj);
+
+/**
+ * Check whether a specified string is long and should be likely printed in
+ * multiline mode
+ * @param obj
+ * @return
+ */
+bool ucl_maybe_long_string (const ucl_object_t *obj);
+
+/**
+ * Print integer to the msgpack output
+ * @param ctx
+ * @param val
+ */
+void ucl_emitter_print_int_msgpack (struct ucl_emitter_context *ctx,
+ int64_t val);
+/**
+ * Print integer to the msgpack output
+ * @param ctx
+ * @param val
+ */
+void ucl_emitter_print_double_msgpack (struct ucl_emitter_context *ctx,
+ double val);
+/**
+ * Print double to the msgpack output
+ * @param ctx
+ * @param val
+ */
+void ucl_emitter_print_bool_msgpack (struct ucl_emitter_context *ctx,
+ bool val);
+/**
+ * Print string to the msgpack output
+ * @param ctx
+ * @param s
+ * @param len
+ */
+void ucl_emitter_print_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len);
+
+/**
+ * Print binary string to the msgpack output
+ * @param ctx
+ * @param s
+ * @param len
+ */
+void ucl_emitter_print_binary_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len);
+
+/**
+ * Print array preamble for msgpack
+ * @param ctx
+ * @param len
+ */
+void ucl_emitter_print_array_msgpack (struct ucl_emitter_context *ctx,
+ size_t len);
+
+/**
+ * Print object preamble for msgpack
+ * @param ctx
+ * @param len
+ */
+void ucl_emitter_print_object_msgpack (struct ucl_emitter_context *ctx,
+ size_t len);
+/**
+ * Print NULL to the msgpack output
+ * @param ctx
+ */
+void ucl_emitter_print_null_msgpack (struct ucl_emitter_context *ctx);
+/**
+ * Print object's key if needed to the msgpack output
+ * @param print_key
+ * @param ctx
+ * @param obj
+ */
+void ucl_emitter_print_key_msgpack (bool print_key,
+ struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj);
+
+/**
+ * Fetch URL into a buffer
+ * @param url url to fetch
+ * @param buf pointer to buffer (must be freed by callee)
+ * @param buflen pointer to buffer length
+ * @param err pointer to error argument
+ * @param must_exist fail if cannot find a url
+ */
+bool ucl_fetch_url (const unsigned char *url,
+ unsigned char **buf,
+ size_t *buflen,
+ UT_string **err,
+ bool must_exist);
+
+/**
+ * Fetch a file and save results to the memory buffer
+ * @param filename filename to fetch
+ * @param len length of filename
+ * @param buf target buffer
+ * @param buflen target length
+ * @return
+ */
+bool ucl_fetch_file (const unsigned char *filename,
+ unsigned char **buf,
+ size_t *buflen,
+ UT_string **err,
+ bool must_exist);
+
+/**
+ * Add new element to an object using the current merge strategy and priority
+ * @param parser
+ * @param nobj
+ * @return
+ */
+bool ucl_parser_process_object_element (struct ucl_parser *parser,
+ ucl_object_t *nobj);
+
+/**
+ * Parse msgpack chunk
+ * @param parser
+ * @return
+ */
+bool ucl_parse_msgpack (struct ucl_parser *parser);
+
+bool ucl_parse_csexp (struct ucl_parser *parser);
+
+/**
+ * Free ucl chunk
+ * @param chunk
+ */
+void ucl_chunk_free (struct ucl_chunk *chunk);
+
+#endif /* UCL_INTERNAL_H_ */
diff --git a/contrib/libucl/ucl_msgpack.c b/contrib/libucl/ucl_msgpack.c
new file mode 100644
index 0000000..1fcdcc8
--- /dev/null
+++ b/contrib/libucl/ucl_msgpack.c
@@ -0,0 +1,1615 @@
+/*
+ * Copyright (c) 2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ucl.h"
+#include "ucl_internal.h"
+
+#ifdef HAVE_ENDIAN_H
+#include <endian.h>
+#elif defined(HAVE_SYS_ENDIAN_H)
+#include <sys/endian.h>
+#elif defined(HAVE_MACHINE_ENDIAN_H)
+#include <machine/endian.h>
+#endif
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+ #if __BYTE_ORDER == __LITTLE_ENDIAN
+ #define __LITTLE_ENDIAN__
+ #elif __BYTE_ORDER == __BIG_ENDIAN
+ #define __BIG_ENDIAN__
+ #elif _WIN32
+ #define __LITTLE_ENDIAN__
+ #endif
+#endif
+
+#define SWAP_LE_BE16(val) ((uint16_t) ( \
+ (uint16_t) ((uint16_t) (val) >> 8) | \
+ (uint16_t) ((uint16_t) (val) << 8)))
+
+#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined (__GNUC_MINOR__) && __GNUC_MINOR__ >= 3)
+# define SWAP_LE_BE32(val) ((uint32_t)__builtin_bswap32 ((uint32_t)(val)))
+# define SWAP_LE_BE64(val) ((uint64_t)__builtin_bswap64 ((uint64_t)(val)))
+#else
+ #define SWAP_LE_BE32(val) ((uint32_t)( \
+ (((uint32_t)(val) & (uint32_t)0x000000ffU) << 24) | \
+ (((uint32_t)(val) & (uint32_t)0x0000ff00U) << 8) | \
+ (((uint32_t)(val) & (uint32_t)0x00ff0000U) >> 8) | \
+ (((uint32_t)(val) & (uint32_t)0xff000000U) >> 24)))
+
+ #define SWAP_LE_BE64(val) ((uint64_t)( \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x00000000000000ffULL)) << 56) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x000000000000ff00ULL)) << 40) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x0000000000ff0000ULL)) << 24) | \
+ (((uint64_t)(val) & \
+ (uint64_t) (0x00000000ff000000ULL)) << 8) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x000000ff00000000ULL)) >> 8) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x0000ff0000000000ULL)) >> 24) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x00ff000000000000ULL)) >> 40) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0xff00000000000000ULL)) >> 56)))
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define TO_BE16 SWAP_LE_BE16
+#define TO_BE32 SWAP_LE_BE32
+#define TO_BE64 SWAP_LE_BE64
+#define FROM_BE16 SWAP_LE_BE16
+#define FROM_BE32 SWAP_LE_BE32
+#define FROM_BE64 SWAP_LE_BE64
+#else
+#define TO_BE16(val) (uint16_t)(val)
+#define TO_BE32(val) (uint32_t)(val)
+#define TO_BE64(val) (uint64_t)(val)
+#define FROM_BE16(val) (uint16_t)(val)
+#define FROM_BE32(val) (uint32_t)(val)
+#define FROM_BE64(val) (uint64_t)(val)
+#endif
+
+void
+ucl_emitter_print_int_msgpack (struct ucl_emitter_context *ctx, int64_t val)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ unsigned char buf[sizeof(uint64_t) + 1];
+ const unsigned char mask_positive = 0x7f, mask_negative = 0xe0,
+ uint8_ch = 0xcc, uint16_ch = 0xcd, uint32_ch = 0xce, uint64_ch = 0xcf,
+ int8_ch = 0xd0, int16_ch = 0xd1, int32_ch = 0xd2, int64_ch = 0xd3;
+ unsigned len;
+
+ if (val >= 0) {
+ if (val <= 0x7f) {
+ /* Fixed num 7 bits */
+ len = 1;
+ buf[0] = mask_positive & val;
+ }
+ else if (val <= UINT8_MAX) {
+ len = 2;
+ buf[0] = uint8_ch;
+ buf[1] = val & 0xff;
+ }
+ else if (val <= UINT16_MAX) {
+ uint16_t v = TO_BE16 (val);
+
+ len = 3;
+ buf[0] = uint16_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else if (val <= UINT32_MAX) {
+ uint32_t v = TO_BE32 (val);
+
+ len = 5;
+ buf[0] = uint32_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else {
+ uint64_t v = TO_BE64 (val);
+
+ len = 9;
+ buf[0] = uint64_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ }
+ else {
+ uint64_t uval;
+ /* Bithack abs */
+ uval = ((val ^ (val >> 63)) - (val >> 63));
+
+ if (val > -(1 << 5)) {
+ len = 1;
+ buf[0] = (mask_negative | uval) & 0xff;
+ }
+ else if (uval <= INT8_MAX) {
+ uint8_t v = (uint8_t)val;
+ len = 2;
+ buf[0] = int8_ch;
+ buf[1] = v;
+ }
+ else if (uval <= INT16_MAX) {
+ uint16_t v = TO_BE16 (val);
+
+ len = 3;
+ buf[0] = int16_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else if (uval <= INT32_MAX) {
+ uint32_t v = TO_BE32 (val);
+
+ len = 5;
+ buf[0] = int32_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else {
+ uint64_t v = TO_BE64 (val);
+
+ len = 9;
+ buf[0] = int64_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ }
+
+ func->ucl_emitter_append_len (buf, len, func->ud);
+}
+
+void
+ucl_emitter_print_double_msgpack (struct ucl_emitter_context *ctx, double val)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ union {
+ double d;
+ uint64_t i;
+ } u;
+ const unsigned char dbl_ch = 0xcb;
+ unsigned char buf[sizeof(double) + 1];
+
+ /* Convert to big endian */
+ u.d = val;
+ u.i = TO_BE64 (u.i);
+
+ buf[0] = dbl_ch;
+ memcpy (&buf[1], &u.d, sizeof (double));
+ func->ucl_emitter_append_len (buf, sizeof (buf), func->ud);
+}
+
+void
+ucl_emitter_print_bool_msgpack (struct ucl_emitter_context *ctx, bool val)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char true_ch = 0xc3, false_ch = 0xc2;
+
+ func->ucl_emitter_append_character (val ? true_ch : false_ch, 1, func->ud);
+}
+
+void
+ucl_emitter_print_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char fix_mask = 0xA0, l8_ch = 0xd9, l16_ch = 0xda, l32_ch = 0xdb;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0x1F) {
+ blen = 1;
+ buf[0] = (len | fix_mask) & 0xff;
+ }
+ else if (len <= 0xff) {
+ blen = 2;
+ buf[0] = l8_ch;
+ buf[1] = len & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+ func->ucl_emitter_append_len (s, len, func->ud);
+}
+
+void
+ucl_emitter_print_binary_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char l8_ch = 0xc4, l16_ch = 0xc5, l32_ch = 0xc6;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0xff) {
+ blen = 2;
+ buf[0] = l8_ch;
+ buf[1] = len & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+ func->ucl_emitter_append_len (s, len, func->ud);
+}
+
+void
+ucl_emitter_print_null_msgpack (struct ucl_emitter_context *ctx)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char nil = 0xc0;
+
+ func->ucl_emitter_append_character (nil, 1, func->ud);
+}
+
+void
+ucl_emitter_print_key_msgpack (bool print_key, struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+ if (print_key) {
+ ucl_emitter_print_string_msgpack (ctx, obj->key, obj->keylen);
+ }
+}
+
+void
+ucl_emitter_print_array_msgpack (struct ucl_emitter_context *ctx, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char fix_mask = 0x90, l16_ch = 0xdc, l32_ch = 0xdd;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0xF) {
+ blen = 1;
+ buf[0] = (len | fix_mask) & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+}
+
+void
+ucl_emitter_print_object_msgpack (struct ucl_emitter_context *ctx, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char fix_mask = 0x80, l16_ch = 0xde, l32_ch = 0xdf;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0xF) {
+ blen = 1;
+ buf[0] = (len | fix_mask) & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+}
+
+
+enum ucl_msgpack_format {
+ msgpack_positive_fixint = 0,
+ msgpack_fixmap,
+ msgpack_fixarray,
+ msgpack_fixstr,
+ msgpack_nil,
+ msgpack_false,
+ msgpack_true,
+ msgpack_bin8,
+ msgpack_bin16,
+ msgpack_bin32,
+ msgpack_ext8,
+ msgpack_ext16,
+ msgpack_ext32,
+ msgpack_float32,
+ msgpack_float64,
+ msgpack_uint8,
+ msgpack_uint16,
+ msgpack_uint32,
+ msgpack_uint64,
+ msgpack_int8,
+ msgpack_int16,
+ msgpack_int32,
+ msgpack_int64,
+ msgpack_fixext1,
+ msgpack_fixext2,
+ msgpack_fixext4,
+ msgpack_fixext8,
+ msgpack_fixext16,
+ msgpack_str8,
+ msgpack_str16,
+ msgpack_str32,
+ msgpack_array16,
+ msgpack_array32,
+ msgpack_map16,
+ msgpack_map32,
+ msgpack_negative_fixint,
+ msgpack_invalid
+};
+
+typedef ssize_t (*ucl_msgpack_parse_function)(struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+
+static ssize_t ucl_msgpack_parse_map (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_array (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_string (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_int (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_float (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_bool (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_null (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_ignore (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+
+#define MSGPACK_FLAG_FIXED (1 << 0)
+#define MSGPACK_FLAG_CONTAINER (1 << 1)
+#define MSGPACK_FLAG_TYPEVALUE (1 << 2)
+#define MSGPACK_FLAG_EXT (1 << 3)
+#define MSGPACK_FLAG_ASSOC (1 << 4)
+#define MSGPACK_FLAG_KEY (1 << 5)
+
+/*
+ * Search tree packed in array
+ */
+static struct ucl_msgpack_parser {
+ uint8_t prefix; /* Prefix byte */
+ uint8_t prefixlen; /* Length of prefix in bits */
+ uint8_t fmt; /* The desired format */
+ uint8_t len; /* Length of the object
+ (either length bytes
+ or length of value in case
+ of fixed objects */
+ uint8_t flags; /* Flags of the specified type */
+ ucl_msgpack_parse_function func; /* Parser function */
+} parsers[] = {
+ {
+ 0xa0,
+ 3,
+ msgpack_fixstr,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0x0,
+ 1,
+ msgpack_positive_fixint,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xe0,
+ 3,
+ msgpack_negative_fixint,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0x80,
+ 4,
+ msgpack_fixmap,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_CONTAINER|MSGPACK_FLAG_ASSOC,
+ ucl_msgpack_parse_map
+ },
+ {
+ 0x90,
+ 4,
+ msgpack_fixarray,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_CONTAINER,
+ ucl_msgpack_parse_array
+ },
+ {
+ 0xd9,
+ 8,
+ msgpack_str8,
+ 1,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xc4,
+ 8,
+ msgpack_bin8,
+ 1,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xcf,
+ 8,
+ msgpack_uint64,
+ 8,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd3,
+ 8,
+ msgpack_int64,
+ 8,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xce,
+ 8,
+ msgpack_uint32,
+ 4,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd2,
+ 8,
+ msgpack_int32,
+ 4,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xcb,
+ 8,
+ msgpack_float64,
+ 8,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_float
+ },
+ {
+ 0xca,
+ 8,
+ msgpack_float32,
+ 4,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_float
+ },
+ {
+ 0xc2,
+ 8,
+ msgpack_false,
+ 1,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_bool
+ },
+ {
+ 0xc3,
+ 8,
+ msgpack_true,
+ 1,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_bool
+ },
+ {
+ 0xcc,
+ 8,
+ msgpack_uint8,
+ 1,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xcd,
+ 8,
+ msgpack_uint16,
+ 2,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd0,
+ 8,
+ msgpack_int8,
+ 1,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd1,
+ 8,
+ msgpack_int16,
+ 2,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xc0,
+ 8,
+ msgpack_nil,
+ 0,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_null
+ },
+ {
+ 0xda,
+ 8,
+ msgpack_str16,
+ 2,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xdb,
+ 8,
+ msgpack_str32,
+ 4,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xc5,
+ 8,
+ msgpack_bin16,
+ 2,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xc6,
+ 8,
+ msgpack_bin32,
+ 4,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xdc,
+ 8,
+ msgpack_array16,
+ 2,
+ MSGPACK_FLAG_CONTAINER,
+ ucl_msgpack_parse_array
+ },
+ {
+ 0xdd,
+ 8,
+ msgpack_array32,
+ 4,
+ MSGPACK_FLAG_CONTAINER,
+ ucl_msgpack_parse_array
+ },
+ {
+ 0xde,
+ 8,
+ msgpack_map16,
+ 2,
+ MSGPACK_FLAG_CONTAINER|MSGPACK_FLAG_ASSOC,
+ ucl_msgpack_parse_map
+ },
+ {
+ 0xdf,
+ 8,
+ msgpack_map32,
+ 4,
+ MSGPACK_FLAG_CONTAINER|MSGPACK_FLAG_ASSOC,
+ ucl_msgpack_parse_map
+ },
+ {
+ 0xc7,
+ 8,
+ msgpack_ext8,
+ 1,
+ MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xc8,
+ 8,
+ msgpack_ext16,
+ 2,
+ MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xc9,
+ 8,
+ msgpack_ext32,
+ 4,
+ MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd4,
+ 8,
+ msgpack_fixext1,
+ 1,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd5,
+ 8,
+ msgpack_fixext2,
+ 2,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd6,
+ 8,
+ msgpack_fixext4,
+ 4,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd7,
+ 8,
+ msgpack_fixext8,
+ 8,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd8,
+ 8,
+ msgpack_fixext16,
+ 16,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ }
+};
+
+#undef MSGPACK_DEBUG_PARSER
+
+static inline struct ucl_msgpack_parser *
+ucl_msgpack_get_parser_from_type (unsigned char t)
+{
+ unsigned int i, shift, mask;
+
+ for (i = 0; i < sizeof (parsers) / sizeof (parsers[0]); i ++) {
+ shift = CHAR_BIT - parsers[i].prefixlen;
+ mask = parsers[i].prefix >> shift;
+
+ if (mask == (((unsigned int)t) >> shift)) {
+ return &parsers[i];
+ }
+ }
+
+ return NULL;
+}
+
+static inline struct ucl_stack *
+ucl_msgpack_get_container (struct ucl_parser *parser,
+ struct ucl_msgpack_parser *obj_parser, uint64_t len)
+{
+ struct ucl_stack *stack;
+
+ assert (obj_parser != NULL);
+
+ if (obj_parser->flags & MSGPACK_FLAG_CONTAINER) {
+ /*
+ * Insert new container to the stack
+ */
+ if (parser->stack == NULL) {
+ parser->stack = calloc (1, sizeof (struct ucl_stack));
+
+ if (parser->stack == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ return NULL;
+ }
+
+ parser->stack->chunk = parser->chunks;
+ }
+ else {
+ stack = calloc (1, sizeof (struct ucl_stack));
+
+ if (stack == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ return NULL;
+ }
+
+ stack->chunk = parser->chunks;
+ stack->next = parser->stack;
+ parser->stack = stack;
+ }
+
+ parser->stack->e.len = len;
+
+#ifdef MSGPACK_DEBUG_PARSER
+ stack = parser->stack;
+ while (stack) {
+ fprintf(stderr, "+");
+ stack = stack->next;
+ }
+
+ fprintf(stderr, "%s -> %d\n", obj_parser->flags & MSGPACK_FLAG_ASSOC ? "object" : "array", (int)len);
+#endif
+ }
+ else {
+ /*
+ * Get the current stack top
+ */
+ if (parser->stack) {
+ return parser->stack;
+ }
+ else {
+ ucl_create_err (&parser->err, "bad top level object for msgpack");
+ return NULL;
+ }
+ }
+
+ return parser->stack;
+}
+
+static bool
+ucl_msgpack_is_container_finished (struct ucl_stack *container)
+{
+ assert (container != NULL);
+
+
+ if (container->e.len == 0) {
+ return true;
+ }
+
+ return false;
+}
+
+static bool
+ucl_msgpack_insert_object (struct ucl_parser *parser,
+ const unsigned char *key,
+ size_t keylen, ucl_object_t *obj)
+{
+ struct ucl_stack *container;
+
+ container = parser->stack;
+ assert (container != NULL);
+ assert (container->e.len > 0);
+ assert (obj != NULL);
+ assert (container->obj != NULL);
+
+ if (container->obj->type == UCL_ARRAY) {
+ ucl_array_append (container->obj, obj);
+ }
+ else if (container->obj->type == UCL_OBJECT) {
+ if (key == NULL || keylen == 0) {
+ ucl_create_err (&parser->err, "cannot insert object with no key");
+ return false;
+ }
+
+ obj->key = key;
+ obj->keylen = keylen;
+
+ if (!(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ ucl_copy_key_trash (obj);
+ }
+
+ ucl_parser_process_object_element (parser, obj);
+ }
+ else {
+ ucl_create_err (&parser->err, "bad container type");
+ return false;
+ }
+
+ container->e.len--;
+
+ return true;
+}
+
+static struct ucl_stack *
+ucl_msgpack_get_next_container (struct ucl_parser *parser)
+{
+ struct ucl_stack *cur = NULL;
+ uint64_t len;
+
+ cur = parser->stack;
+
+ if (cur == NULL) {
+ return NULL;
+ }
+
+ len = cur->e.len;
+
+ if (len == 0) {
+ /* We need to switch to the previous container */
+ parser->stack = cur->next;
+ parser->cur_obj = cur->obj;
+ free (cur);
+
+#ifdef MSGPACK_DEBUG_PARSER
+ cur = parser->stack;
+ while (cur) {
+ fprintf(stderr, "-");
+ cur = cur->next;
+ }
+ fprintf(stderr, "-%s -> %d\n", parser->cur_obj->type == UCL_OBJECT ? "object" : "array", (int)parser->cur_obj->len);
+#endif
+
+ return ucl_msgpack_get_next_container (parser);
+ }
+
+ /*
+ * For UCL containers we don't know length, so we just insert the whole
+ * message pack blob into the top level container
+ */
+
+ assert (cur->obj != NULL);
+
+ return cur;
+}
+
+#define CONSUME_RET do { \
+ if (ret != -1) { \
+ p += ret; \
+ remain -= ret; \
+ obj_parser = NULL; \
+ assert (remain >= 0); \
+ } \
+ else { \
+ ucl_create_err (&parser->err, \
+ "cannot parse type %d of len %u", \
+ (int)obj_parser->fmt, \
+ (unsigned)len); \
+ return false; \
+ } \
+} while(0)
+
+#define GET_NEXT_STATE do { \
+ container = ucl_msgpack_get_next_container (parser); \
+ if (container == NULL) { \
+ ucl_create_err (&parser->err, \
+ "empty container"); \
+ return false; \
+ } \
+ next_state = container->obj->type == UCL_OBJECT ? \
+ read_assoc_key : read_array_value; \
+} while(0)
+
+static bool
+ucl_msgpack_consume (struct ucl_parser *parser)
+{
+ const unsigned char *p, *end, *key = NULL;
+ struct ucl_stack *container;
+ enum e_msgpack_parser_state {
+ read_type,
+ start_assoc,
+ start_array,
+ read_assoc_key,
+ read_assoc_value,
+ finish_assoc_value,
+ read_array_value,
+ finish_array_value,
+ error_state
+ } state = read_type, next_state = error_state;
+ struct ucl_msgpack_parser *obj_parser = NULL;
+ uint64_t len = 0;
+ ssize_t ret, remain, keylen = 0;
+#ifdef MSGPACK_DEBUG_PARSER
+ uint64_t i;
+ enum e_msgpack_parser_state hist[256];
+#endif
+
+ p = parser->chunks->begin;
+ remain = parser->chunks->remain;
+ end = p + remain;
+
+
+ while (p < end) {
+#ifdef MSGPACK_DEBUG_PARSER
+ hist[i++ % 256] = state;
+#endif
+ switch (state) {
+ case read_type:
+ obj_parser = ucl_msgpack_get_parser_from_type (*p);
+
+ if (obj_parser == NULL) {
+ ucl_create_err (&parser->err, "unknown msgpack format: %x",
+ (unsigned int)*p);
+
+ return false;
+ }
+ /* Now check length sanity */
+ if (obj_parser->flags & MSGPACK_FLAG_FIXED) {
+ if (obj_parser->len == 0) {
+ /* We have an embedded size */
+ len = *p & ~obj_parser->prefix;
+ }
+ else {
+ if (remain < obj_parser->len) {
+ ucl_create_err (&parser->err, "not enough data remain to "
+ "read object's length: %u remain, %u needed",
+ (unsigned)remain, obj_parser->len);
+
+ return false;
+ }
+
+ len = obj_parser->len;
+ }
+
+ if (!(obj_parser->flags & MSGPACK_FLAG_TYPEVALUE)) {
+ /* We must pass value as the second byte */
+ if (remain > 0) {
+ p ++;
+ remain --;
+ }
+ }
+ else {
+ /* Len is irrelevant now */
+ len = 0;
+ }
+ }
+ else {
+ /* Length is not embedded */
+ if (remain < obj_parser->len) {
+ ucl_create_err (&parser->err, "not enough data remain to "
+ "read object's length: %u remain, %u needed",
+ (unsigned)remain, obj_parser->len);
+
+ return false;
+ }
+
+ p ++;
+ remain --;
+
+ switch (obj_parser->len) {
+ case 1:
+ len = *p;
+ break;
+ case 2:
+ len = FROM_BE16 (*(uint16_t *)p);
+ break;
+ case 4:
+ len = FROM_BE32 (*(uint32_t *)p);
+ break;
+ case 8:
+ len = FROM_BE64 (*(uint64_t *)p);
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ p += obj_parser->len;
+ remain -= obj_parser->len;
+ }
+
+ if (obj_parser->flags & MSGPACK_FLAG_ASSOC) {
+ /* We have just read the new associative map */
+ state = start_assoc;
+ }
+ else if (obj_parser->flags & MSGPACK_FLAG_CONTAINER){
+ state = start_array;
+ }
+ else {
+ state = next_state;
+ }
+
+ break;
+ case start_assoc:
+ parser->cur_obj = ucl_object_new_full (UCL_OBJECT,
+ parser->chunks->priority);
+ /* Insert to the previous level container */
+ if (parser->stack && !ucl_msgpack_insert_object (parser,
+ key, keylen, parser->cur_obj)) {
+ return false;
+ }
+ /* Get new container */
+ container = ucl_msgpack_get_container (parser, obj_parser, len);
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+ key = NULL;
+ keylen = 0;
+
+ if (len > 0) {
+ state = read_type;
+ next_state = read_assoc_key;
+ }
+ else {
+ /* Empty object */
+ state = finish_assoc_value;
+ }
+ break;
+
+ case start_array:
+ parser->cur_obj = ucl_object_new_full (UCL_ARRAY,
+ parser->chunks->priority);
+ /* Insert to the previous level container */
+ if (parser->stack && !ucl_msgpack_insert_object (parser,
+ key, keylen, parser->cur_obj)) {
+ return false;
+ }
+ /* Get new container */
+ container = ucl_msgpack_get_container (parser, obj_parser, len);
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+ if (len > 0) {
+ state = read_type;
+ next_state = read_array_value;
+ }
+ else {
+ /* Empty array */
+ state = finish_array_value;
+ }
+ break;
+
+ case read_array_value:
+ /*
+ * p is now at the value start, len now contains length read and
+ * obj_parser contains the corresponding specific parser
+ */
+ container = parser->stack;
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+
+ /* Insert value to the container and check if we have finished array */
+ if (!ucl_msgpack_insert_object (parser, NULL, 0,
+ parser->cur_obj)) {
+ return false;
+ }
+
+ if (ucl_msgpack_is_container_finished (container)) {
+ state = finish_array_value;
+ }
+ else {
+ /* Read more elements */
+ state = read_type;
+ next_state = read_array_value;
+ }
+
+ break;
+
+ case read_assoc_key:
+ /*
+ * Keys must have string type for ucl msgpack
+ */
+ if (!(obj_parser->flags & MSGPACK_FLAG_KEY)) {
+ ucl_create_err (&parser->err, "bad type for key: %u, expected "
+ "string", (unsigned)obj_parser->fmt);
+
+ return false;
+ }
+
+ key = p;
+ keylen = len;
+
+ if (keylen > remain || keylen == 0) {
+ ucl_create_err (&parser->err, "too long or empty key");
+ return false;
+ }
+
+ p += len;
+ remain -= len;
+
+ state = read_type;
+ next_state = read_assoc_value;
+ break;
+
+ case read_assoc_value:
+ /*
+ * p is now at the value start, len now contains length read and
+ * obj_parser contains the corresponding specific parser
+ */
+ container = parser->stack;
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+ assert (key != NULL && keylen > 0);
+
+ if (!ucl_msgpack_insert_object (parser, key, keylen,
+ parser->cur_obj)) {
+ return false;
+ }
+
+ key = NULL;
+ keylen = 0;
+
+ if (ucl_msgpack_is_container_finished (container)) {
+ state = finish_assoc_value;
+ }
+ else {
+ /* Read more elements */
+ state = read_type;
+ next_state = read_assoc_key;
+ }
+ break;
+
+ case finish_array_value:
+ case finish_assoc_value:
+ GET_NEXT_STATE;
+ state = read_type;
+ break;
+
+ case error_state:
+ ucl_create_err (&parser->err, "invalid state machine state");
+
+ return false;
+ }
+ }
+
+ /* Check the finishing state */
+ switch (state) {
+ case start_array:
+ case start_assoc:
+ /* Empty container at the end */
+ if (len != 0) {
+ ucl_create_err (&parser->err, "invalid non-empty container at the end");
+
+ return false;
+ }
+
+ parser->cur_obj = ucl_object_new_full (
+ state == start_array ? UCL_ARRAY : UCL_OBJECT,
+ parser->chunks->priority);
+ /* Insert to the previous level container */
+ if (!ucl_msgpack_insert_object (parser,
+ key, keylen, parser->cur_obj)) {
+ return false;
+ }
+ /* Get new container */
+ container = ucl_msgpack_get_container (parser, obj_parser, len);
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ break;
+
+ case read_array_value:
+ case read_assoc_value:
+ if (len != 0) {
+ ucl_create_err (&parser->err, "unfinished value at the end");
+
+ return false;
+ }
+
+ container = parser->stack;
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+
+ /* Insert value to the container and check if we have finished array */
+ if (!ucl_msgpack_insert_object (parser, NULL, 0,
+ parser->cur_obj)) {
+ return false;
+ }
+ break;
+ case finish_array_value:
+ case finish_assoc_value:
+ case read_type:
+ /* Valid finishing state */
+ break;
+ default:
+ /* Invalid finishing state */
+ ucl_create_err (&parser->err, "invalid state machine finishing state: %d",
+ state);
+
+ return false;
+ }
+
+ /* Rewind to the top level container */
+ ucl_msgpack_get_next_container (parser);
+ assert (parser->stack == NULL);
+
+ return true;
+}
+
+bool
+ucl_parse_msgpack (struct ucl_parser *parser)
+{
+ ucl_object_t *container = NULL;
+ const unsigned char *p;
+ bool ret;
+
+ assert (parser != NULL);
+ assert (parser->chunks != NULL);
+ assert (parser->chunks->begin != NULL);
+ assert (parser->chunks->remain != 0);
+
+ p = parser->chunks->begin;
+
+ if (parser->stack) {
+ container = parser->stack->obj;
+ }
+
+ /*
+ * When we start parsing message pack chunk, we must ensure that we
+ * have either a valid container or the top object inside message pack is
+ * of container type
+ */
+ if (container == NULL) {
+ if ((*p & 0x80) != 0x80 && !(*p >= 0xdc && *p <= 0xdf)) {
+ ucl_create_err (&parser->err, "bad top level object for msgpack");
+ return false;
+ }
+ }
+
+ ret = ucl_msgpack_consume (parser);
+
+ if (ret && parser->top_obj == NULL) {
+ parser->top_obj = parser->cur_obj;
+ }
+
+ return ret;
+}
+
+static ssize_t
+ucl_msgpack_parse_map (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ container->obj = parser->cur_obj;
+
+ return 0;
+}
+
+static ssize_t
+ucl_msgpack_parse_array (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ container->obj = parser->cur_obj;
+
+ return 0;
+}
+
+static ssize_t
+ucl_msgpack_parse_string (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_STRING, parser->chunks->priority);
+ obj->value.sv = pos;
+ obj->len = len;
+
+ if (fmt >= msgpack_bin8 && fmt <= msgpack_bin32) {
+ obj->flags |= UCL_OBJECT_BINARY;
+ }
+
+ if (!(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ if (obj->flags & UCL_OBJECT_BINARY) {
+ obj->trash_stack[UCL_TRASH_VALUE] = malloc (len);
+
+ if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ memcpy (obj->trash_stack[UCL_TRASH_VALUE], pos, len);
+ }
+ }
+ else {
+ ucl_copy_value_trash (obj);
+ }
+ }
+
+ parser->cur_obj = obj;
+
+ return len;
+}
+
+static ssize_t
+ucl_msgpack_parse_int (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+ int8_t iv8;
+ int16_t iv16;
+ int32_t iv32;
+ int64_t iv64;
+ uint16_t uiv16;
+ uint32_t uiv32;
+ uint64_t uiv64;
+
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_INT, parser->chunks->priority);
+
+ switch (fmt) {
+ case msgpack_positive_fixint:
+ obj->value.iv = (*pos & 0x7f);
+ len = 1;
+ break;
+ case msgpack_negative_fixint:
+ obj->value.iv = - (*pos & 0x1f);
+ len = 1;
+ break;
+ case msgpack_uint8:
+ obj->value.iv = (unsigned char)*pos;
+ len = 1;
+ break;
+ case msgpack_int8:
+ memcpy (&iv8, pos, sizeof (iv8));
+ obj->value.iv = iv8;
+ len = 1;
+ break;
+ case msgpack_int16:
+ memcpy (&iv16, pos, sizeof (iv16));
+ iv16 = FROM_BE16 (iv16);
+ obj->value.iv = iv16;
+ len = 2;
+ break;
+ case msgpack_uint16:
+ memcpy (&uiv16, pos, sizeof (uiv16));
+ uiv16 = FROM_BE16 (uiv16);
+ obj->value.iv = uiv16;
+ len = 2;
+ break;
+ case msgpack_int32:
+ memcpy (&iv32, pos, sizeof (iv32));
+ iv32 = FROM_BE32 (iv32);
+ obj->value.iv = iv32;
+ len = 4;
+ break;
+ case msgpack_uint32:
+ memcpy(&uiv32, pos, sizeof(uiv32));
+ uiv32 = FROM_BE32(uiv32);
+ obj->value.iv = uiv32;
+ len = 4;
+ break;
+ case msgpack_int64:
+ memcpy (&iv64, pos, sizeof (iv64));
+ iv64 = FROM_BE64 (iv64);
+ obj->value.iv = iv64;
+ len = 8;
+ break;
+ case msgpack_uint64:
+ memcpy(&uiv64, pos, sizeof(uiv64));
+ uiv64 = FROM_BE64(uiv64);
+ obj->value.iv = uiv64;
+ len = 8;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ parser->cur_obj = obj;
+
+ return len;
+}
+
+static ssize_t
+ucl_msgpack_parse_float (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+ union {
+ uint32_t i;
+ float f;
+ } d;
+ uint64_t uiv64;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_FLOAT, parser->chunks->priority);
+
+ switch (fmt) {
+ case msgpack_float32:
+ memcpy(&d.i, pos, sizeof(d.i));
+ d.i = FROM_BE32(d.i);
+ /* XXX: can be slow */
+ obj->value.dv = d.f;
+ len = 4;
+ break;
+ case msgpack_float64:
+ memcpy(&uiv64, pos, sizeof(uiv64));
+ uiv64 = FROM_BE64(uiv64);
+ obj->value.iv = uiv64;
+ len = 8;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ parser->cur_obj = obj;
+
+ return len;
+}
+
+static ssize_t
+ucl_msgpack_parse_bool (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_BOOLEAN, parser->chunks->priority);
+
+ switch (fmt) {
+ case msgpack_true:
+ obj->value.iv = true;
+ break;
+ case msgpack_false:
+ obj->value.iv = false;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ parser->cur_obj = obj;
+
+ return 1;
+}
+
+static ssize_t
+ucl_msgpack_parse_null (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
+ parser->cur_obj = obj;
+
+ return 1;
+}
+
+static ssize_t
+ucl_msgpack_parse_ignore (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ if (len > remain) {
+ return -1;
+ }
+
+ switch (fmt) {
+ case msgpack_fixext1:
+ len = 2;
+ break;
+ case msgpack_fixext2:
+ len = 3;
+ break;
+ case msgpack_fixext4:
+ len = 5;
+ break;
+ case msgpack_fixext8:
+ len = 9;
+ break;
+ case msgpack_fixext16:
+ len = 17;
+ break;
+ case msgpack_ext8:
+ case msgpack_ext16:
+ case msgpack_ext32:
+ len = len + 1;
+ break;
+ default:
+ ucl_create_err (&parser->err, "bad type: %x", (unsigned)fmt);
+ return -1;
+ }
+
+ return len;
+}
diff --git a/contrib/libucl/ucl_parser.c b/contrib/libucl/ucl_parser.c
new file mode 100644
index 0000000..354bfe8
--- /dev/null
+++ b/contrib/libucl/ucl_parser.c
@@ -0,0 +1,3212 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <math.h>
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+
+/**
+ * @file ucl_parser.c
+ * The implementation of ucl parser
+ */
+
+struct ucl_parser_saved_state {
+ unsigned int line;
+ unsigned int column;
+ size_t remain;
+ const unsigned char *pos;
+};
+
+/**
+ * Move up to len characters
+ * @param parser
+ * @param begin
+ * @param len
+ * @return new position in chunk
+ */
+#define ucl_chunk_skipc(chunk, p) \
+do { \
+ if (p == chunk->end) { \
+ break; \
+ } \
+ if (*(p) == '\n') { \
+ (chunk)->line ++; \
+ (chunk)->column = 0; \
+ } \
+ else (chunk)->column ++; \
+ (p++); \
+ (chunk)->pos ++; \
+ (chunk)->remain --; \
+} while (0)
+
+static inline void
+ucl_set_err (struct ucl_parser *parser, int code, const char *str, UT_string **err)
+{
+ const char *fmt_string, *filename;
+ struct ucl_chunk *chunk = parser->chunks;
+
+ if (parser->cur_file) {
+ filename = parser->cur_file;
+ }
+ else {
+ filename = "<unknown>";
+ }
+
+ if (chunk->pos < chunk->end) {
+ if (isgraph (*chunk->pos)) {
+ fmt_string = "error while parsing %s: "
+ "line: %d, column: %d - '%s', character: '%c'";
+ }
+ else {
+ fmt_string = "error while parsing %s: "
+ "line: %d, column: %d - '%s', character: '0x%02x'";
+ }
+ ucl_create_err (err, fmt_string,
+ filename, chunk->line, chunk->column,
+ str, *chunk->pos);
+ }
+ else {
+ ucl_create_err (err, "error while parsing %s: at the end of chunk: %s",
+ filename, str);
+ }
+
+ parser->err_code = code;
+ parser->state = UCL_STATE_ERROR;
+}
+
+static void
+ucl_save_comment (struct ucl_parser *parser, const char *begin, size_t len)
+{
+ ucl_object_t *nobj;
+
+ if (len > 0 && begin != NULL) {
+ nobj = ucl_object_fromstring_common (begin, len, 0);
+
+ if (parser->last_comment) {
+ /* We need to append data to an existing object */
+ DL_APPEND (parser->last_comment, nobj);
+ }
+ else {
+ parser->last_comment = nobj;
+ }
+ }
+}
+
+static void
+ucl_attach_comment (struct ucl_parser *parser, ucl_object_t *obj, bool before)
+{
+ if (parser->last_comment) {
+ ucl_object_insert_key (parser->comments, parser->last_comment,
+ (const char *)&obj, sizeof (void *), true);
+
+ if (before) {
+ parser->last_comment->flags |= UCL_OBJECT_INHERITED;
+ }
+
+ parser->last_comment = NULL;
+ }
+}
+
+/**
+ * Skip all comments from the current pos resolving nested and multiline comments
+ * @param parser
+ * @return
+ */
+static bool
+ucl_skip_comments (struct ucl_parser *parser)
+{
+ struct ucl_chunk *chunk = parser->chunks;
+ const unsigned char *p, *beg = NULL;
+ int comments_nested = 0;
+ bool quoted = false;
+
+ p = chunk->pos;
+
+start:
+ if (chunk->remain > 0 && *p == '#') {
+ if (parser->state != UCL_STATE_SCOMMENT &&
+ parser->state != UCL_STATE_MCOMMENT) {
+ beg = p;
+
+ while (p < chunk->end) {
+ if (*p == '\n') {
+ if (parser->flags & UCL_PARSER_SAVE_COMMENTS) {
+ ucl_save_comment (parser, beg, p - beg);
+ beg = NULL;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+
+ goto start;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ }
+ }
+ else if (chunk->remain >= 2 && *p == '/') {
+ if (p[1] == '*') {
+ beg = p;
+ ucl_chunk_skipc (chunk, p);
+ comments_nested ++;
+ ucl_chunk_skipc (chunk, p);
+
+ while (p < chunk->end) {
+ if (*p == '"' && *(p - 1) != '\\') {
+ quoted = !quoted;
+ }
+
+ if (!quoted) {
+ if (*p == '*') {
+ ucl_chunk_skipc (chunk, p);
+ if (chunk->remain > 0 && *p == '/') {
+ comments_nested --;
+ if (comments_nested == 0) {
+ if (parser->flags & UCL_PARSER_SAVE_COMMENTS) {
+ ucl_save_comment (parser, beg, p - beg + 1);
+ beg = NULL;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ goto start;
+ }
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (p[0] == '/' && chunk->remain >= 2 && p[1] == '*') {
+ comments_nested ++;
+ ucl_chunk_skipc (chunk, p);
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (comments_nested != 0) {
+ ucl_set_err (parser, UCL_ENESTED,
+ "unfinished multiline comment", &parser->err);
+ return false;
+ }
+ }
+ }
+
+ if (beg && p > beg && (parser->flags & UCL_PARSER_SAVE_COMMENTS)) {
+ ucl_save_comment (parser, beg, p - beg);
+ }
+
+ return true;
+}
+
+/**
+ * Return multiplier for a character
+ * @param c multiplier character
+ * @param is_bytes if true use 1024 multiplier
+ * @return multiplier
+ */
+static inline unsigned long
+ucl_lex_num_multiplier (const unsigned char c, bool is_bytes) {
+ const struct {
+ char c;
+ long mult_normal;
+ long mult_bytes;
+ } multipliers[] = {
+ {'m', 1000 * 1000, 1024 * 1024},
+ {'k', 1000, 1024},
+ {'g', 1000 * 1000 * 1000, 1024 * 1024 * 1024}
+ };
+ int i;
+
+ for (i = 0; i < 3; i ++) {
+ if (tolower (c) == multipliers[i].c) {
+ if (is_bytes) {
+ return multipliers[i].mult_bytes;
+ }
+ return multipliers[i].mult_normal;
+ }
+ }
+
+ return 1;
+}
+
+
+/**
+ * Return multiplier for time scaling
+ * @param c
+ * @return
+ */
+static inline double
+ucl_lex_time_multiplier (const unsigned char c) {
+ const struct {
+ char c;
+ double mult;
+ } multipliers[] = {
+ {'m', 60},
+ {'h', 60 * 60},
+ {'d', 60 * 60 * 24},
+ {'w', 60 * 60 * 24 * 7},
+ {'y', 60 * 60 * 24 * 365}
+ };
+ int i;
+
+ for (i = 0; i < 5; i ++) {
+ if (tolower (c) == multipliers[i].c) {
+ return multipliers[i].mult;
+ }
+ }
+
+ return 1;
+}
+
+/**
+ * Return true if a character is a end of an atom
+ * @param c
+ * @return
+ */
+static inline bool
+ucl_lex_is_atom_end (const unsigned char c)
+{
+ return ucl_test_character (c, UCL_CHARACTER_VALUE_END);
+}
+
+static inline bool
+ucl_lex_is_comment (const unsigned char c1, const unsigned char c2)
+{
+ if (c1 == '/') {
+ if (c2 == '*') {
+ return true;
+ }
+ }
+ else if (c1 == '#') {
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Check variable found
+ * @param parser
+ * @param ptr
+ * @param remain
+ * @param out_len
+ * @param strict
+ * @param found
+ * @return
+ */
+static inline const char *
+ucl_check_variable_safe (struct ucl_parser *parser, const char *ptr, size_t remain,
+ size_t *out_len, bool strict, bool *found)
+{
+ struct ucl_variable *var;
+ unsigned char *dst;
+ size_t dstlen;
+ bool need_free = false;
+
+ LL_FOREACH (parser->variables, var) {
+ if (strict) {
+ if (remain == var->var_len) {
+ if (memcmp (ptr, var->var, var->var_len) == 0) {
+ *out_len += var->value_len;
+ *found = true;
+ return (ptr + var->var_len);
+ }
+ }
+ }
+ else {
+ if (remain >= var->var_len) {
+ if (memcmp (ptr, var->var, var->var_len) == 0) {
+ *out_len += var->value_len;
+ *found = true;
+ return (ptr + var->var_len);
+ }
+ }
+ }
+ }
+
+ /* XXX: can only handle ${VAR} */
+ if (!(*found) && parser->var_handler != NULL && strict) {
+ /* Call generic handler */
+ if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free,
+ parser->var_data)) {
+ *found = true;
+ if (need_free) {
+ free (dst);
+ }
+ return (ptr + remain);
+ }
+ }
+
+ return ptr;
+}
+
+/**
+ * Check for a variable in a given string
+ * @param parser
+ * @param ptr
+ * @param remain
+ * @param out_len
+ * @param vars_found
+ * @return
+ */
+static const char *
+ucl_check_variable (struct ucl_parser *parser, const char *ptr,
+ size_t remain, size_t *out_len, bool *vars_found)
+{
+ const char *p, *end, *ret = ptr;
+ bool found = false;
+
+ if (*ptr == '{') {
+ /* We need to match the variable enclosed in braces */
+ p = ptr + 1;
+ end = ptr + remain;
+ while (p < end) {
+ if (*p == '}') {
+ ret = ucl_check_variable_safe (parser, ptr + 1, p - ptr - 1,
+ out_len, true, &found);
+ if (found) {
+ /* {} must be excluded actually */
+ ret ++;
+ if (!*vars_found) {
+ *vars_found = true;
+ }
+ }
+ else {
+ *out_len += 2;
+ }
+ break;
+ }
+ p ++;
+ }
+ if(p == end) {
+ (*out_len) ++;
+ }
+ }
+ else if (*ptr != '$') {
+ /* Not count escaped dollar sign */
+ ret = ucl_check_variable_safe (parser, ptr, remain, out_len, false, &found);
+ if (found && !*vars_found) {
+ *vars_found = true;
+ }
+ if (!found) {
+ (*out_len) ++;
+ }
+ }
+ else {
+ ret ++;
+ (*out_len) ++;
+ }
+
+ return ret;
+}
+
+/**
+ * Expand a single variable
+ * @param parser
+ * @param ptr
+ * @param in_len
+ * @param dest
+ * @param out_len
+ * @return
+ */
+static const char *
+ucl_expand_single_variable (struct ucl_parser *parser, const char *ptr,
+ size_t in_len, unsigned char **dest, size_t out_len)
+{
+ unsigned char *d = *dest, *dst;
+ const char *p = ptr + 1, *ret;
+ struct ucl_variable *var;
+ size_t dstlen;
+ bool need_free = false;
+ bool found = false;
+ bool strict = false;
+
+ ret = ptr + 1;
+ /* For the $ sign */
+ in_len --;
+
+ if (*p == '$') {
+ *d++ = *p++;
+ *dest = d;
+ return p;
+ }
+ else if (*p == '{') {
+ p ++;
+ in_len --;
+ strict = true;
+ ret += 2;
+ }
+
+ LL_FOREACH (parser->variables, var) {
+ if (out_len >= var->value_len && in_len >= (var->var_len + (strict ? 1 : 0))) {
+ if (memcmp (p, var->var, var->var_len) == 0) {
+ if (!strict || p[var->var_len] == '}') {
+ memcpy (d, var->value, var->value_len);
+ ret += var->var_len;
+ d += var->value_len;
+ found = true;
+ break;
+ }
+ }
+ }
+ }
+
+ if (!found) {
+ if (strict && parser->var_handler != NULL) {
+ dstlen = out_len;
+
+ if (parser->var_handler (p, in_len, &dst, &dstlen, &need_free,
+ parser->var_data)) {
+ if (dstlen > out_len) {
+ /* We do not have enough space! */
+ if (need_free) {
+ free (dst);
+ }
+ }
+ else {
+ memcpy(d, dst, dstlen);
+ ret += in_len;
+ d += dstlen;
+ found = true;
+
+ if (need_free) {
+ free(dst);
+ }
+ }
+ }
+ }
+
+ /* Leave variable as is, in this case we use dest */
+ if (!found) {
+ if (strict && out_len >= 2) {
+ /* Copy '${' */
+ memcpy (d, ptr, 2);
+ d += 2;
+ ret --;
+ }
+ else {
+ memcpy (d, ptr, 1);
+ d ++;
+ }
+ }
+ }
+
+ *dest = d;
+ return ret;
+}
+
+/**
+ * Expand variables in string
+ * @param parser
+ * @param dst
+ * @param src
+ * @param in_len
+ * @return
+ */
+static ssize_t
+ucl_expand_variable (struct ucl_parser *parser, unsigned char **dst,
+ const char *src, size_t in_len)
+{
+ const char *p, *end = src + in_len;
+ unsigned char *d, *d_end;
+ size_t out_len = 0;
+ bool vars_found = false;
+
+ if (parser->flags & UCL_PARSER_DISABLE_MACRO) {
+ *dst = NULL;
+ return in_len;
+ }
+
+ p = src;
+ while (p != end) {
+ if (*p == '$' && p + 1 != end) {
+ p = ucl_check_variable (parser, p + 1, end - p - 1, &out_len, &vars_found);
+ }
+ else {
+ p ++;
+ out_len ++;
+ }
+ }
+
+ if (!vars_found) {
+ /* Trivial case */
+ *dst = NULL;
+ return in_len;
+ }
+
+ *dst = UCL_ALLOC (out_len + 1);
+ if (*dst == NULL) {
+ return in_len;
+ }
+
+ d = *dst;
+ d_end = d + out_len;
+ p = src;
+ while (p != end && d != d_end) {
+ if (*p == '$' && p + 1 != end) {
+ p = ucl_expand_single_variable (parser, p, end - p, &d, d_end - d);
+ }
+ else {
+ *d++ = *p++;
+ }
+ }
+
+ *d = '\0';
+
+ return out_len;
+}
+
+/**
+ * Store or copy pointer to the trash stack
+ * @param parser parser object
+ * @param src src string
+ * @param dst destination buffer (trash stack pointer)
+ * @param dst_const const destination pointer (e.g. value of object)
+ * @param in_len input length
+ * @param need_unescape need to unescape source (and copy it)
+ * @param need_lowercase need to lowercase value (and copy)
+ * @param need_expand need to expand variables (and copy as well)
+ * @param unescape_squote unescape single quoted string
+ * @return output length (excluding \0 symbol)
+ */
+static inline ssize_t
+ucl_copy_or_store_ptr (struct ucl_parser *parser,
+ const unsigned char *src, unsigned char **dst,
+ const char **dst_const, size_t in_len,
+ bool need_unescape, bool need_lowercase, bool need_expand,
+ bool unescape_squote)
+{
+ ssize_t ret = -1, tret;
+ unsigned char *tmp;
+
+ if (need_unescape || need_lowercase ||
+ (need_expand && parser->variables != NULL) ||
+ !(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ /* Copy string */
+ *dst = UCL_ALLOC (in_len + 1);
+ if (*dst == NULL) {
+ ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for a string",
+ &parser->err);
+ return false;
+ }
+ if (need_lowercase) {
+ ret = ucl_strlcpy_tolower (*dst, src, in_len + 1);
+ }
+ else {
+ ret = ucl_strlcpy_unsafe (*dst, src, in_len + 1);
+ }
+
+ if (need_unescape) {
+ if (!unescape_squote) {
+ ret = ucl_unescape_json_string (*dst, ret);
+ }
+ else {
+ ret = ucl_unescape_squoted_string (*dst, ret);
+ }
+ }
+
+ if (need_expand) {
+ tmp = *dst;
+ tret = ret;
+ ret = ucl_expand_variable (parser, dst, tmp, ret);
+ if (*dst == NULL) {
+ /* Nothing to expand */
+ *dst = tmp;
+ ret = tret;
+ }
+ else {
+ /* Free unexpanded value */
+ UCL_FREE (in_len + 1, tmp);
+ }
+ }
+ *dst_const = *dst;
+ }
+ else {
+ *dst_const = src;
+ ret = in_len;
+ }
+
+ return ret;
+}
+
+/**
+ * Create and append an object at the specified level
+ * @param parser
+ * @param is_array
+ * @param level
+ * @return
+ */
+static inline ucl_object_t *
+ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser,
+ bool is_array, uint32_t level, bool has_obrace)
+{
+ struct ucl_stack *st;
+ ucl_object_t *nobj;
+
+ if (obj == NULL) {
+ nobj = ucl_object_new_full (is_array ? UCL_ARRAY : UCL_OBJECT, parser->chunks->priority);
+ if (nobj == NULL) {
+ goto enomem0;
+ }
+ } else {
+ if (obj->type == (is_array ? UCL_OBJECT : UCL_ARRAY)) {
+ /* Bad combination for merge: array and object */
+ ucl_set_err (parser, UCL_EMERGE,
+ "cannot merge an object with an array",
+ &parser->err);
+
+ return NULL;
+ }
+ nobj = obj;
+ nobj->type = is_array ? UCL_ARRAY : UCL_OBJECT;
+ }
+
+ if (!is_array) {
+ if (nobj->value.ov == NULL) {
+ nobj->value.ov = ucl_hash_create (parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ if (nobj->value.ov == NULL) {
+ goto enomem1;
+ }
+ }
+ parser->state = UCL_STATE_KEY;
+ } else {
+ parser->state = UCL_STATE_VALUE;
+ }
+
+ st = UCL_ALLOC (sizeof (struct ucl_stack));
+
+ if (st == NULL) {
+ goto enomem1;
+ }
+
+ st->obj = nobj;
+
+ if (level >= UINT16_MAX) {
+ ucl_set_err (parser, UCL_ENESTED,
+ "objects are nesting too deep (over 65535 limit)",
+ &parser->err);
+ if (nobj != obj) {
+ ucl_object_unref (obj);
+ }
+
+ UCL_FREE(sizeof (struct ucl_stack), st);
+
+ return NULL;
+ }
+
+
+ st->e.params.level = level;
+ st->e.params.line = parser->chunks->line;
+ st->chunk = parser->chunks;
+
+ if (has_obrace) {
+ st->e.params.flags = UCL_STACK_HAS_OBRACE;
+ }
+ else {
+ st->e.params.flags = 0;
+ }
+
+ LL_PREPEND (parser->stack, st);
+ parser->cur_obj = nobj;
+
+ return nobj;
+enomem1:
+ if (nobj != obj)
+ ucl_object_unref (nobj);
+enomem0:
+ ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for an object",
+ &parser->err);
+ return NULL;
+}
+
+int
+ucl_maybe_parse_number (ucl_object_t *obj,
+ const char *start, const char *end, const char **pos,
+ bool allow_double, bool number_bytes, bool allow_time)
+{
+ const char *p = start, *c = start;
+ char *endptr;
+ bool got_dot = false, got_exp = false, need_double = false,
+ is_time = false, valid_start = false, is_hex = false;
+ int is_neg = 0;
+ double dv = 0;
+ int64_t lv = 0;
+
+ if (*p == '-') {
+ is_neg = 1;
+ c ++;
+ p ++;
+ }
+ while (p < end) {
+ if (is_hex && isxdigit (*p)) {
+ p ++;
+ }
+ else if (isdigit (*p)) {
+ valid_start = true;
+ p ++;
+ }
+ else if (!is_hex && (*p == 'x' || *p == 'X')) {
+ is_hex = true;
+ allow_double = false;
+ c = p + 1;
+ p ++;
+ }
+ else if (allow_double) {
+ if (p == c) {
+ /* Empty digits sequence, not a number */
+ *pos = start;
+ return EINVAL;
+ }
+ else if (*p == '.') {
+ if (got_dot) {
+ /* Double dots, not a number */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ got_dot = true;
+ need_double = true;
+ p ++;
+ }
+ }
+ else if (*p == 'e' || *p == 'E') {
+ if (got_exp) {
+ /* Double exp, not a number */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ got_exp = true;
+ need_double = true;
+ p ++;
+ if (p >= end) {
+ *pos = start;
+ return EINVAL;
+ }
+ if (!isdigit (*p) && *p != '+' && *p != '-') {
+ /* Wrong exponent sign */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ p ++;
+ }
+ }
+ }
+ else {
+ /* Got the end of the number, need to check */
+ break;
+ }
+ }
+ else if (!allow_double && *p == '.') {
+ /* Unexpected dot */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ break;
+ }
+ }
+
+ if (!valid_start || p == c) {
+ *pos = start;
+ return EINVAL;
+ }
+
+ char numbuf[128];
+
+ if ((size_t)(p - c + 1) >= sizeof(numbuf)) {
+ *pos = start;
+ return EINVAL;
+ }
+
+ if (is_neg) {
+ numbuf[0] = '-';
+ ucl_strlcpy (&numbuf[1], c, p - c + 1);
+ }
+ else {
+ ucl_strlcpy (numbuf, c, p - c + 1);
+ }
+
+ errno = 0;
+ if (need_double) {
+ dv = strtod (numbuf, &endptr);
+ }
+ else {
+ if (is_hex) {
+ lv = strtoimax (numbuf, &endptr, 16);
+ }
+ else {
+ lv = strtoimax (numbuf, &endptr, 10);
+ }
+ }
+ if (errno == ERANGE) {
+ *pos = start;
+ return ERANGE;
+ }
+
+ /* Now check endptr and move it from numbuf to the real ending */
+ if (endptr != NULL) {
+ long shift = endptr - numbuf - is_neg;
+ endptr = (char *)c + shift;
+ }
+ if (endptr >= end) {
+ p = end;
+ goto set_obj;
+ }
+ if (endptr == NULL || ucl_lex_is_atom_end (*endptr) || *endptr == '\0') {
+ p = endptr;
+ goto set_obj;
+ }
+
+ if (endptr < end && endptr != start) {
+ switch (*p) {
+ case 'm':
+ case 'M':
+ case 'g':
+ case 'G':
+ case 'k':
+ case 'K':
+ if (end - p >= 2) {
+ if (p[1] == 's' || p[1] == 'S') {
+ /* Milliseconds */
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ is_time = true;
+ if (p[0] == 'm' || p[0] == 'M') {
+ dv /= 1000.;
+ }
+ else {
+ dv *= ucl_lex_num_multiplier (*p, false);
+ }
+ p += 2;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ else if (number_bytes || (p[1] == 'b' || p[1] == 'B')) {
+ /* Bytes */
+ if (need_double) {
+ need_double = false;
+ lv = dv;
+ }
+ lv *= ucl_lex_num_multiplier (*p, true);
+ p += 2;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ else if (ucl_lex_is_atom_end (p[1])) {
+ if (need_double) {
+ dv *= ucl_lex_num_multiplier (*p, false);
+ }
+ else {
+ lv *= ucl_lex_num_multiplier (*p, number_bytes);
+ }
+ p ++;
+ goto set_obj;
+ }
+ else if (allow_time && end - p >= 3) {
+ if (tolower (p[0]) == 'm' &&
+ tolower (p[1]) == 'i' &&
+ tolower (p[2]) == 'n') {
+ /* Minutes */
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ is_time = true;
+ dv *= 60.;
+ p += 3;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ }
+ }
+ else {
+ if (need_double) {
+ dv *= ucl_lex_num_multiplier (*p, false);
+ }
+ else {
+ lv *= ucl_lex_num_multiplier (*p, number_bytes);
+ }
+ p ++;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ break;
+ case 'S':
+ case 's':
+ if (allow_time &&
+ (p == end - 1 || ucl_lex_is_atom_end (p[1]))) {
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ p ++;
+ is_time = true;
+ goto set_obj;
+ }
+ break;
+ case 'h':
+ case 'H':
+ case 'd':
+ case 'D':
+ case 'w':
+ case 'W':
+ case 'Y':
+ case 'y':
+ if (allow_time &&
+ (p == end - 1 || ucl_lex_is_atom_end (p[1]))) {
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ is_time = true;
+ dv *= ucl_lex_time_multiplier (*p);
+ p ++;
+ goto set_obj;
+ }
+ break;
+ case '\t':
+ case ' ':
+ while (p < end && ucl_test_character(*p, UCL_CHARACTER_WHITESPACE)) {
+ p++;
+ }
+ if (ucl_lex_is_atom_end(*p))
+ goto set_obj;
+ break;
+ }
+ }
+ else if (endptr == end) {
+ /* Just a number at the end of chunk */
+ p = end;
+ goto set_obj;
+ }
+
+ *pos = c;
+ return EINVAL;
+
+set_obj:
+ if (obj != NULL) {
+ if (allow_double && (need_double || is_time)) {
+ if (!is_time) {
+ obj->type = UCL_FLOAT;
+ }
+ else {
+ obj->type = UCL_TIME;
+ }
+ obj->value.dv = dv;
+ }
+ else {
+ obj->type = UCL_INT;
+ obj->value.iv = lv;
+ }
+ }
+ *pos = p;
+ return 0;
+}
+
+/**
+ * Parse possible number
+ * @param parser
+ * @param chunk
+ * @param obj
+ * @return true if a number has been parsed
+ */
+static bool
+ucl_lex_number (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, ucl_object_t *obj)
+{
+ const unsigned char *pos;
+ int ret;
+
+ ret = ucl_maybe_parse_number (obj, chunk->pos, chunk->end, (const char **)&pos,
+ true, false, ((parser->flags & UCL_PARSER_NO_TIME) == 0));
+
+ if (ret == 0) {
+ chunk->remain -= pos - chunk->pos;
+ chunk->column += pos - chunk->pos;
+ chunk->pos = pos;
+ return true;
+ }
+ else if (ret == ERANGE) {
+ ucl_set_err (parser, UCL_ESYNTAX, "numeric value out of range",
+ &parser->err);
+ }
+
+ return false;
+}
+
+/**
+ * Parse quoted string with possible escapes
+ * @param parser
+ * @param chunk
+ * @param need_unescape
+ * @param ucl_escape
+ * @param var_expand
+ * @return true if a string has been parsed
+ */
+static bool
+ucl_lex_json_string (struct ucl_parser *parser,
+ struct ucl_chunk *chunk,
+ bool *need_unescape,
+ bool *ucl_escape,
+ bool *var_expand)
+{
+ const unsigned char *p = chunk->pos;
+ unsigned char c;
+ int i;
+
+ while (p < chunk->end) {
+ c = *p;
+ if (c < 0x1F) {
+ /* Unmasked control character */
+ if (c == '\n') {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected newline",
+ &parser->err);
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected control character",
+ &parser->err);
+ }
+ return false;
+ }
+ else if (c == '\\') {
+ ucl_chunk_skipc (chunk, p);
+ if (p >= chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character",
+ &parser->err);
+ return false;
+ }
+ c = *p;
+ if (ucl_test_character (c, UCL_CHARACTER_ESCAPE)) {
+ if (c == 'u') {
+ ucl_chunk_skipc (chunk, p);
+ for (i = 0; i < 4 && p < chunk->end; i ++) {
+ if (!isxdigit (*p)) {
+ ucl_set_err (parser, UCL_ESYNTAX, "invalid utf escape",
+ &parser->err);
+ return false;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (p >= chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unfinished escape character",
+ &parser->err);
+ return false;
+ }
+ }
+ else {
+ ucl_chunk_skipc (chunk, p);
+ }
+ }
+ *need_unescape = true;
+ *ucl_escape = true;
+ continue;
+ }
+ else if (c == '"') {
+ ucl_chunk_skipc (chunk, p);
+ return true;
+ }
+ else if (ucl_test_character (c, UCL_CHARACTER_UCL_UNSAFE)) {
+ *ucl_escape = true;
+ }
+ else if (c == '$') {
+ *var_expand = true;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "no quote at the end of json string",
+ &parser->err);
+ return false;
+}
+
+/**
+ * Process single quoted string
+ * @param parser
+ * @param chunk
+ * @param need_unescape
+ * @return
+ */
+static bool
+ucl_lex_squoted_string (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, bool *need_unescape)
+{
+ const unsigned char *p = chunk->pos;
+ unsigned char c;
+
+ while (p < chunk->end) {
+ c = *p;
+ if (c == '\\') {
+ ucl_chunk_skipc (chunk, p);
+
+ if (p >= chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unfinished escape character",
+ &parser->err);
+ return false;
+ }
+ else {
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ *need_unescape = true;
+ continue;
+ }
+ else if (c == '\'') {
+ ucl_chunk_skipc (chunk, p);
+ return true;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "no quote at the end of single quoted string",
+ &parser->err);
+ return false;
+}
+
+static void
+ucl_parser_append_elt (struct ucl_parser *parser, ucl_hash_t *cont,
+ ucl_object_t *top,
+ ucl_object_t *elt)
+{
+ ucl_object_t *nobj;
+
+ if ((parser->flags & UCL_PARSER_NO_IMPLICIT_ARRAYS) == 0) {
+ /* Implicit array */
+ top->flags |= UCL_OBJECT_MULTIVALUE;
+ DL_APPEND (top, elt);
+ parser->stack->obj->len ++;
+ }
+ else {
+ if ((top->flags & UCL_OBJECT_MULTIVALUE) != 0) {
+ /* Just add to the explicit array */
+ ucl_array_append (top, elt);
+ }
+ else {
+ /* Convert to an array */
+ nobj = ucl_object_typed_new (UCL_ARRAY);
+ nobj->key = top->key;
+ nobj->keylen = top->keylen;
+ nobj->flags |= UCL_OBJECT_MULTIVALUE;
+ ucl_array_append (nobj, top);
+ ucl_array_append (nobj, elt);
+ ucl_hash_replace (cont, top, nobj);
+ }
+ }
+}
+
+bool
+ucl_parser_process_object_element (struct ucl_parser *parser, ucl_object_t *nobj)
+{
+ ucl_hash_t *container;
+ ucl_object_t *tobj = NULL, *cur;
+ char errmsg[256];
+
+ container = parser->stack->obj->value.ov;
+
+ DL_FOREACH (parser->stack->obj, cur) {
+ tobj = __DECONST (ucl_object_t *, ucl_hash_search_obj (cur->value.ov, nobj));
+
+ if (tobj != NULL) {
+ break;
+ }
+ }
+
+
+ if (tobj == NULL) {
+ container = ucl_hash_insert_object (container, nobj,
+ parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ if (container == NULL) {
+ return false;
+ }
+ nobj->prev = nobj;
+ nobj->next = NULL;
+ parser->stack->obj->len ++;
+ }
+ else {
+ unsigned priold = ucl_object_get_priority (tobj),
+ prinew = ucl_object_get_priority (nobj);
+ switch (parser->chunks->strategy) {
+
+ case UCL_DUPLICATE_APPEND:
+ /*
+ * The logic here is the following:
+ *
+ * - if we have two objects with the same priority, then we form an
+ * implicit or explicit array
+ * - if a new object has bigger priority, then we overwrite an old one
+ * - if a new object has lower priority, then we ignore it
+ */
+ /* Special case for inherited objects */
+ if (tobj->flags & UCL_OBJECT_INHERITED) {
+ prinew = priold + 1;
+ }
+
+ if (priold == prinew) {
+ ucl_parser_append_elt (parser, container, tobj, nobj);
+ }
+ else if (priold > prinew) {
+ /*
+ * We add this new object to a list of trash objects just to ensure
+ * that it won't come to any real object
+ * XXX: rather inefficient approach
+ */
+ DL_APPEND (parser->trash_objs, nobj);
+ }
+ else {
+ ucl_hash_replace (container, tobj, nobj);
+ ucl_object_unref (tobj);
+ }
+
+ break;
+
+ case UCL_DUPLICATE_REWRITE:
+ /* We just rewrite old values regardless of priority */
+ ucl_hash_replace (container, tobj, nobj);
+ ucl_object_unref (tobj);
+
+ break;
+
+ case UCL_DUPLICATE_ERROR:
+ snprintf(errmsg, sizeof(errmsg),
+ "duplicate element for key '%s' found",
+ nobj->key);
+ ucl_set_err (parser, UCL_EMERGE, errmsg, &parser->err);
+ return false;
+
+ case UCL_DUPLICATE_MERGE:
+ /*
+ * Here we do have some old object so we just push it on top of objects stack
+ * Check priority and then perform the merge on the remaining objects
+ */
+ if (tobj->type == UCL_OBJECT || tobj->type == UCL_ARRAY) {
+ ucl_object_unref (nobj);
+ nobj = tobj;
+ }
+ else if (priold == prinew) {
+ ucl_parser_append_elt (parser, container, tobj, nobj);
+ }
+ else if (priold > prinew) {
+ /*
+ * We add this new object to a list of trash objects just to ensure
+ * that it won't come to any real object
+ * XXX: rather inefficient approach
+ */
+ DL_APPEND (parser->trash_objs, nobj);
+ }
+ else {
+ ucl_hash_replace (container, tobj, nobj);
+ ucl_object_unref (tobj);
+ }
+ break;
+ }
+ }
+
+ parser->stack->obj->value.ov = container;
+ parser->cur_obj = nobj;
+ ucl_attach_comment (parser, nobj, false);
+
+ return true;
+}
+
+/**
+ * Parse a key in an object
+ * @param parser
+ * @param chunk
+ * @param next_key
+ * @param end_of_object
+ * @return true if a key has been parsed
+ */
+static bool
+ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk,
+ bool *next_key, bool *end_of_object)
+{
+ const unsigned char *p, *c = NULL, *end, *t;
+ const char *key = NULL;
+ bool got_quote = false, got_eq = false, got_semicolon = false,
+ need_unescape = false, ucl_escape = false, var_expand = false,
+ got_content = false, got_sep = false;
+ ucl_object_t *nobj;
+ ssize_t keylen;
+
+ p = chunk->pos;
+
+ if (*p == '.') {
+ /* It is macro actually */
+ if (!(parser->flags & UCL_PARSER_DISABLE_MACRO)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_MACRO_NAME;
+ *end_of_object = false;
+ return true;
+ }
+ while (p < chunk->end) {
+ /*
+ * A key must start with alpha, number, '/' or '_' and end with space character
+ */
+ if (c == NULL) {
+ if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ p = chunk->pos;
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_KEY_START)) {
+ /* The first symbol */
+ c = p;
+ ucl_chunk_skipc (chunk, p);
+ got_content = true;
+ }
+ else if (*p == '"') {
+ /* JSON style key */
+ c = p + 1;
+ got_quote = true;
+ got_content = true;
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (*p == '}') {
+ /* We have actually end of an object */
+ *end_of_object = true;
+ return true;
+ }
+ else if (*p == '.') {
+ ucl_chunk_skipc (chunk, p);
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_MACRO_NAME;
+ return true;
+ }
+ else {
+ /* Invalid identifier */
+ ucl_set_err (parser, UCL_ESYNTAX, "key must begin with a letter",
+ &parser->err);
+ return false;
+ }
+ }
+ else {
+ /* Parse the body of a key */
+ if (!got_quote) {
+ if (ucl_test_character (*p, UCL_CHARACTER_KEY)) {
+ got_content = true;
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_KEY_SEP)) {
+ end = p;
+ break;
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "invalid character in a key",
+ &parser->err);
+ return false;
+ }
+ }
+ else {
+ /* We need to parse json like quoted string */
+ if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) {
+ return false;
+ }
+ /* Always escape keys obtained via json */
+ end = chunk->pos - 1;
+ p = chunk->pos;
+ break;
+ }
+ }
+ }
+
+ if (p >= chunk->end && got_content) {
+ ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err);
+ return false;
+ }
+ else if (!got_content) {
+ return true;
+ }
+ *end_of_object = false;
+ /* We are now at the end of the key, need to parse the rest */
+ while (p < chunk->end) {
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (*p == '=') {
+ if (!got_eq && !got_semicolon) {
+ ucl_chunk_skipc (chunk, p);
+ got_eq = true;
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected '=' character",
+ &parser->err);
+ return false;
+ }
+ }
+ else if (*p == ':') {
+ if (!got_eq && !got_semicolon) {
+ ucl_chunk_skipc (chunk, p);
+ got_semicolon = true;
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected ':' character",
+ &parser->err);
+ return false;
+ }
+ }
+ else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
+ /* Check for comment */
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ p = chunk->pos;
+ }
+ else {
+ /* Start value */
+ break;
+ }
+ }
+
+ if (p >= chunk->end && got_content) {
+ ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err);
+ return false;
+ }
+
+ got_sep = got_semicolon || got_eq;
+
+ if (!got_sep) {
+ /*
+ * Maybe we have more keys nested, so search for termination character.
+ * Possible choices:
+ * 1) key1 key2 ... keyN [:=] value <- we treat that as error
+ * 2) key1 ... keyN {} or [] <- we treat that as nested objects
+ * 3) key1 value[;,\n] <- we treat that as linear object
+ */
+ t = p;
+ *next_key = false;
+ while (ucl_test_character (*t, UCL_CHARACTER_WHITESPACE)) {
+ t ++;
+ }
+ /* Check first non-space character after a key */
+ if (*t != '{' && *t != '[') {
+ while (t < chunk->end) {
+ if (*t == ',' || *t == ';' || *t == '\n' || *t == '\r') {
+ break;
+ }
+ else if (*t == '{' || *t == '[') {
+ *next_key = true;
+ break;
+ }
+ t ++;
+ }
+ }
+ }
+
+ /* Create a new object */
+ nobj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
+ if (nobj == NULL) {
+ return false;
+ }
+ keylen = ucl_copy_or_store_ptr (parser, c, &nobj->trash_stack[UCL_TRASH_KEY],
+ &key, end - c, need_unescape, parser->flags & UCL_PARSER_KEY_LOWERCASE,
+ false, false);
+ if (keylen == -1) {
+ ucl_object_unref (nobj);
+ return false;
+ }
+ else if (keylen == 0) {
+ ucl_set_err (parser, UCL_ESYNTAX, "empty keys are not allowed", &parser->err);
+ ucl_object_unref (nobj);
+ return false;
+ }
+
+ nobj->key = key;
+ nobj->keylen = keylen;
+
+ if (!ucl_parser_process_object_element (parser, nobj)) {
+ return false;
+ }
+
+ if (ucl_escape) {
+ nobj->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
+ }
+
+
+ return true;
+}
+
+/**
+ * Parse a cl string
+ * @param parser
+ * @param chunk
+ * @param var_expand
+ * @param need_unescape
+ * @return true if a key has been parsed
+ */
+static bool
+ucl_parse_string_value (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, bool *var_expand, bool *need_unescape)
+{
+ const unsigned char *p;
+ enum {
+ UCL_BRACE_ROUND = 0,
+ UCL_BRACE_SQUARE,
+ UCL_BRACE_FIGURE
+ };
+ int braces[3][2] = {{0, 0}, {0, 0}, {0, 0}};
+
+ p = chunk->pos;
+
+ while (p < chunk->end) {
+
+ /* Skip pairs of figure braces */
+ if (*p == '{') {
+ braces[UCL_BRACE_FIGURE][0] ++;
+ }
+ else if (*p == '}') {
+ braces[UCL_BRACE_FIGURE][1] ++;
+ if (braces[UCL_BRACE_FIGURE][1] <= braces[UCL_BRACE_FIGURE][0]) {
+ /* This is not a termination symbol, continue */
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ }
+ /* Skip pairs of square braces */
+ else if (*p == '[') {
+ braces[UCL_BRACE_SQUARE][0] ++;
+ }
+ else if (*p == ']') {
+ braces[UCL_BRACE_SQUARE][1] ++;
+ if (braces[UCL_BRACE_SQUARE][1] <= braces[UCL_BRACE_SQUARE][0]) {
+ /* This is not a termination symbol, continue */
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ }
+ else if (*p == '$') {
+ *var_expand = true;
+ }
+ else if (*p == '\\') {
+ *need_unescape = true;
+ ucl_chunk_skipc (chunk, p);
+ if (p < chunk->end) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ continue;
+ }
+
+ if (ucl_lex_is_atom_end (*p) || (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ return true;
+}
+
+/**
+ * Parse multiline string ending with \n{term}\n
+ * @param parser
+ * @param chunk
+ * @param term
+ * @param term_len
+ * @param beg
+ * @param var_expand
+ * @return size of multiline string or 0 in case of error
+ */
+static int
+ucl_parse_multiline_string (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, const unsigned char *term,
+ int term_len, unsigned char const **beg,
+ bool *var_expand)
+{
+ const unsigned char *p, *c, *tend;
+ bool newline = false;
+ int len = 0;
+
+ p = chunk->pos;
+
+ c = p;
+
+ while (p < chunk->end) {
+ if (newline) {
+ if (chunk->end - p < term_len) {
+ return 0;
+ }
+ else if (memcmp (p, term, term_len) == 0) {
+ tend = p + term_len;
+ if (*tend != '\n' && *tend != ';' && *tend != ',') {
+ /* Incomplete terminator */
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ len = p - c;
+ chunk->remain -= term_len;
+ chunk->pos = p + term_len;
+ chunk->column = term_len;
+ *beg = c;
+ break;
+ }
+ }
+ if (*p == '\n') {
+ newline = true;
+ }
+ else {
+ if (*p == '$') {
+ *var_expand = true;
+ }
+ newline = false;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ return len;
+}
+
+static inline ucl_object_t*
+ucl_parser_get_container (struct ucl_parser *parser)
+{
+ ucl_object_t *t, *obj = NULL;
+
+ if (parser == NULL || parser->stack == NULL || parser->stack->obj == NULL) {
+ return NULL;
+ }
+
+ if (parser->stack->obj->type == UCL_ARRAY) {
+ /* Object must be allocated */
+ obj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
+ t = parser->stack->obj;
+
+ if (!ucl_array_append (t, obj)) {
+ ucl_object_unref (obj);
+ return NULL;
+ }
+
+ parser->cur_obj = obj;
+ ucl_attach_comment (parser, obj, false);
+ }
+ else {
+ /* Object has been already allocated */
+ obj = parser->cur_obj;
+ }
+
+ return obj;
+}
+
+/**
+ * Handle value data
+ * @param parser
+ * @param chunk
+ * @return
+ */
+static bool
+ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
+{
+ const unsigned char *p, *c;
+ ucl_object_t *obj = NULL;
+ unsigned int stripped_spaces;
+ ssize_t str_len;
+ bool need_unescape = false, ucl_escape = false, var_expand = false;
+
+ p = chunk->pos;
+
+ /* Skip any spaces and comments */
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) ||
+ (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) {
+ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ p = chunk->pos;
+ }
+
+ while (p < chunk->end) {
+ c = p;
+ switch (*p) {
+ case '"':
+ ucl_chunk_skipc (chunk, p);
+
+ if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape,
+ &var_expand)) {
+ return false;
+ }
+
+ obj = ucl_parser_get_container (parser);
+ if (!obj) {
+ return false;
+ }
+
+ str_len = chunk->pos - c - 2;
+ obj->type = UCL_STRING;
+ if ((str_len = ucl_copy_or_store_ptr (parser, c + 1,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len, need_unescape, false,
+ var_expand, false)) == -1) {
+ return false;
+ }
+
+ obj->len = str_len;
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ break;
+ case '\'':
+ ucl_chunk_skipc (chunk, p);
+
+ if (!ucl_lex_squoted_string (parser, chunk, &need_unescape)) {
+ return false;
+ }
+
+ obj = ucl_parser_get_container (parser);
+ if (!obj) {
+ return false;
+ }
+
+ str_len = chunk->pos - c - 2;
+ obj->type = UCL_STRING;
+ obj->flags |= UCL_OBJECT_SQUOTED;
+
+ if ((str_len = ucl_copy_or_store_ptr (parser, c + 1,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len, need_unescape, false,
+ var_expand, true)) == -1) {
+ return false;
+ }
+
+ obj->len = str_len;
+
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ break;
+ case '{':
+ obj = ucl_parser_get_container (parser);
+ if (obj == NULL) {
+ return false;
+ }
+ /* We have a new object */
+ if (parser->stack) {
+ obj = ucl_parser_add_container (obj, parser, false,
+ parser->stack->e.params.level, true);
+ }
+ else {
+ return false;
+ }
+ if (obj == NULL) {
+ return false;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+
+ return true;
+ break;
+ case '[':
+ obj = ucl_parser_get_container (parser);
+ if (obj == NULL) {
+ return false;
+ }
+ /* We have a new array */
+ if (parser->stack) {
+ obj = ucl_parser_add_container (obj, parser, true,
+ parser->stack->e.params.level, true);
+ }
+ else {
+ return false;
+ }
+
+ if (obj == NULL) {
+ return false;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+
+ return true;
+ break;
+ case ']':
+ /* We have the array ending */
+ if (parser->stack && parser->stack->obj->type == UCL_ARRAY) {
+ parser->state = UCL_STATE_AFTER_VALUE;
+ return true;
+ }
+ else {
+ goto parse_string;
+ }
+ break;
+ case '<':
+ obj = ucl_parser_get_container (parser);
+ /* We have something like multiline value, which must be <<[A-Z]+\n */
+ if (chunk->end - p > 3) {
+ if (memcmp (p, "<<", 2) == 0) {
+ p += 2;
+ /* We allow only uppercase characters in multiline definitions */
+ while (p < chunk->end && *p >= 'A' && *p <= 'Z') {
+ p ++;
+ }
+ if(p == chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unterminated multiline value", &parser->err);
+ return false;
+ }
+ if (*p =='\n') {
+ /* Set chunk positions and start multiline parsing */
+ chunk->remain -= p - c + 1;
+ c += 2;
+ chunk->pos = p + 1;
+ chunk->column = 0;
+ chunk->line ++;
+ if ((str_len = ucl_parse_multiline_string (parser, chunk, c,
+ p - c, &c, &var_expand)) == 0) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unterminated multiline value", &parser->err);
+ return false;
+ }
+
+ obj->type = UCL_STRING;
+ obj->flags |= UCL_OBJECT_MULTILINE;
+ if ((str_len = ucl_copy_or_store_ptr (parser, c,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len - 1, false,
+ false, var_expand, false)) == -1) {
+ return false;
+ }
+ obj->len = str_len;
+
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ }
+ }
+ }
+ /* Fallback to ordinary strings */
+ /* FALLTHRU */
+ default:
+parse_string:
+ if (obj == NULL) {
+ obj = ucl_parser_get_container (parser);
+ }
+
+ /* Parse atom */
+ if (ucl_test_character (*p, UCL_CHARACTER_VALUE_DIGIT_START)) {
+ if (!ucl_lex_number (parser, chunk, obj)) {
+ if (parser->state == UCL_STATE_ERROR) {
+ return false;
+ }
+ }
+ else {
+ parser->state = UCL_STATE_AFTER_VALUE;
+ return true;
+ }
+ /* Fallback to normal string */
+ }
+
+ if (!ucl_parse_string_value (parser, chunk, &var_expand,
+ &need_unescape)) {
+ return false;
+ }
+ /* Cut trailing spaces */
+ stripped_spaces = 0;
+ while (ucl_test_character (*(chunk->pos - 1 - stripped_spaces),
+ UCL_CHARACTER_WHITESPACE)) {
+ stripped_spaces ++;
+ }
+ str_len = chunk->pos - c - stripped_spaces;
+ if (str_len <= 0) {
+ ucl_set_err (parser, UCL_ESYNTAX, "string value must not be empty",
+ &parser->err);
+ return false;
+ }
+ else if (str_len == 4 && memcmp (c, "null", 4) == 0) {
+ obj->len = 0;
+ obj->type = UCL_NULL;
+ }
+ else if (str_len == 3 && memcmp (c, "nan", 3) == 0) {
+ obj->len = 0;
+ obj->type = UCL_FLOAT;
+ obj->value.dv = NAN;
+ }
+ else if (str_len == 3 && memcmp (c, "inf", 3) == 0) {
+ obj->len = 0;
+ obj->type = UCL_FLOAT;
+ obj->value.dv = INFINITY;
+ }
+ else if (!ucl_maybe_parse_boolean (obj, c, str_len)) {
+ obj->type = UCL_STRING;
+ if ((str_len = ucl_copy_or_store_ptr (parser, c,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len, need_unescape,
+ false, var_expand, false)) == -1) {
+ return false;
+ }
+ obj->len = str_len;
+ }
+
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ break;
+ }
+ }
+
+ return true;
+}
+
+/**
+ * Handle after value data
+ * @param parser
+ * @param chunk
+ * @return
+ */
+static bool
+ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
+{
+ const unsigned char *p;
+ bool got_sep = false;
+ struct ucl_stack *st;
+
+ p = chunk->pos;
+
+ while (p < chunk->end) {
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
+ /* Skip whitespaces */
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
+ /* Skip comment */
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ /* Treat comment as a separator */
+ got_sep = true;
+ p = chunk->pos;
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_VALUE_END)) {
+ if (*p == '}' || *p == ']') {
+ if (parser->stack == NULL) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "end of array or object detected without corresponding start",
+ &parser->err);
+ return false;
+ }
+ if ((*p == '}' && parser->stack->obj->type == UCL_OBJECT) ||
+ (*p == ']' && parser->stack->obj->type == UCL_ARRAY)) {
+
+ /* Pop all nested objects from a stack */
+ st = parser->stack;
+
+ if (!(st->e.params.flags & UCL_STACK_HAS_OBRACE)) {
+ parser->err_code = UCL_EUNPAIRED;
+ ucl_create_err (&parser->err,
+ "%s:%d object closed with } is not opened with { at line %d",
+ chunk->fname ? chunk->fname : "memory",
+ parser->chunks->line, st->e.params.line);
+
+ return false;
+ }
+
+ parser->stack = st->next;
+ UCL_FREE (sizeof (struct ucl_stack), st);
+
+ if (parser->cur_obj) {
+ ucl_attach_comment (parser, parser->cur_obj, true);
+ }
+
+ while (parser->stack != NULL) {
+ st = parser->stack;
+
+ if (st->next == NULL) {
+ break;
+ }
+ else if (st->next->e.params.level == st->e.params.level) {
+ break;
+ }
+
+
+ parser->stack = st->next;
+ parser->cur_obj = st->obj;
+ UCL_FREE (sizeof (struct ucl_stack), st);
+ }
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unexpected terminating symbol detected",
+ &parser->err);
+ return false;
+ }
+
+ if (parser->stack == NULL) {
+ /* Ignore everything after a top object */
+ return true;
+ }
+ else {
+ ucl_chunk_skipc (chunk, p);
+ }
+ got_sep = true;
+ }
+ else {
+ /* Got a separator */
+ got_sep = true;
+ ucl_chunk_skipc (chunk, p);
+ }
+ }
+ else {
+ /* Anything else */
+ if (!got_sep) {
+ ucl_set_err (parser, UCL_ESYNTAX, "delimiter is missing",
+ &parser->err);
+ return false;
+ }
+ return true;
+ }
+ }
+
+ return true;
+}
+
+static bool
+ucl_skip_macro_as_comment (struct ucl_parser *parser,
+ struct ucl_chunk *chunk)
+{
+ const unsigned char *p, *c;
+ enum {
+ macro_skip_start = 0,
+ macro_has_symbols,
+ macro_has_obrace,
+ macro_has_quote,
+ macro_has_backslash,
+ macro_has_sqbrace,
+ macro_save
+ } state = macro_skip_start, prev_state = macro_skip_start;
+
+ p = chunk->pos;
+ c = chunk->pos;
+
+ while (p < chunk->end) {
+ switch (state) {
+ case macro_skip_start:
+ if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
+ state = macro_has_symbols;
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ state = macro_save;
+ continue;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_symbols:
+ if (*p == '{') {
+ state = macro_has_sqbrace;
+ }
+ else if (*p == '(') {
+ state = macro_has_obrace;
+ }
+ else if (*p == '"') {
+ state = macro_has_quote;
+ }
+ else if (*p == '\n') {
+ state = macro_save;
+ continue;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_obrace:
+ if (*p == '\\') {
+ prev_state = state;
+ state = macro_has_backslash;
+ }
+ else if (*p == ')') {
+ state = macro_has_symbols;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_sqbrace:
+ if (*p == '\\') {
+ prev_state = state;
+ state = macro_has_backslash;
+ }
+ else if (*p == '}') {
+ state = macro_save;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_quote:
+ if (*p == '\\') {
+ prev_state = state;
+ state = macro_has_backslash;
+ }
+ else if (*p == '"') {
+ state = macro_save;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_backslash:
+ state = prev_state;
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_save:
+ if (parser->flags & UCL_PARSER_SAVE_COMMENTS) {
+ ucl_save_comment (parser, c, p - c);
+ }
+
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * Handle macro data
+ * @param parser
+ * @param chunk
+ * @param marco
+ * @param macro_start
+ * @param macro_len
+ * @return
+ */
+static bool
+ucl_parse_macro_value (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, struct ucl_macro *macro,
+ unsigned char const **macro_start, size_t *macro_len)
+{
+ const unsigned char *p, *c;
+ bool need_unescape = false, ucl_escape = false, var_expand = false;
+
+ p = chunk->pos;
+
+ switch (*p) {
+ case '"':
+ /* We have macro value encoded in quotes */
+ c = p;
+ ucl_chunk_skipc (chunk, p);
+ if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) {
+ return false;
+ }
+
+ *macro_start = c + 1;
+ *macro_len = chunk->pos - c - 2;
+ p = chunk->pos;
+ break;
+ case '{':
+ /* We got a multiline macro body */
+ ucl_chunk_skipc (chunk, p);
+ /* Skip spaces at the beginning */
+ while (p < chunk->end) {
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else {
+ break;
+ }
+ }
+ c = p;
+ while (p < chunk->end) {
+ if (*p == '}') {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ *macro_start = c;
+ *macro_len = p - c;
+ ucl_chunk_skipc (chunk, p);
+ break;
+ default:
+ /* Macro is not enclosed in quotes or braces */
+ c = p;
+ while (p < chunk->end) {
+ if (ucl_lex_is_atom_end (*p)) {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ *macro_start = c;
+ *macro_len = p - c;
+ break;
+ }
+
+ /* We are at the end of a macro */
+ /* Skip ';' and space characters and return to previous state */
+ while (p < chunk->end) {
+ if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) && *p != ';') {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ return true;
+}
+
+/**
+ * Parse macro arguments as UCL object
+ * @param parser parser structure
+ * @param chunk the current data chunk
+ * @return
+ */
+static ucl_object_t *
+ucl_parse_macro_arguments (struct ucl_parser *parser,
+ struct ucl_chunk *chunk)
+{
+ ucl_object_t *res = NULL;
+ struct ucl_parser *params_parser;
+ int obraces = 1, ebraces = 0, state = 0;
+ const unsigned char *p, *c;
+ size_t args_len = 0;
+ struct ucl_parser_saved_state saved;
+
+ saved.column = chunk->column;
+ saved.line = chunk->line;
+ saved.pos = chunk->pos;
+ saved.remain = chunk->remain;
+ p = chunk->pos;
+
+ if (*p != '(' || chunk->remain < 2) {
+ return NULL;
+ }
+
+ /* Set begin and start */
+ ucl_chunk_skipc (chunk, p);
+ c = p;
+
+ while ((p) < (chunk)->end) {
+ switch (state) {
+ case 0:
+ /* Parse symbols and check for '(', ')' and '"' */
+ if (*p == '(') {
+ obraces ++;
+ }
+ else if (*p == ')') {
+ ebraces ++;
+ }
+ else if (*p == '"') {
+ state = 1;
+ }
+ /* Check pairing */
+ if (obraces == ebraces) {
+ state = 99;
+ }
+ else {
+ args_len ++;
+ }
+ /* Check overflow */
+ if (chunk->remain == 0) {
+ goto restore_chunk;
+ }
+ ucl_chunk_skipc (chunk, p);
+ break;
+ case 1:
+ /* We have quote character, so skip all but quotes */
+ if (*p == '"' && *(p - 1) != '\\') {
+ state = 0;
+ }
+ if (chunk->remain == 0) {
+ goto restore_chunk;
+ }
+ args_len ++;
+ ucl_chunk_skipc (chunk, p);
+ break;
+ case 99:
+ /*
+ * We have read the full body of arguments, so we need to parse and set
+ * object from that
+ */
+ params_parser = ucl_parser_new (parser->flags);
+ if (!ucl_parser_add_chunk (params_parser, c, args_len)) {
+ ucl_set_err (parser, UCL_ESYNTAX, "macro arguments parsing error",
+ &parser->err);
+ }
+ else {
+ res = ucl_parser_get_object (params_parser);
+ }
+ ucl_parser_free (params_parser);
+
+ return res;
+
+ break;
+ }
+ }
+
+ return res;
+
+restore_chunk:
+ chunk->column = saved.column;
+ chunk->line = saved.line;
+ chunk->pos = saved.pos;
+ chunk->remain = saved.remain;
+
+ return NULL;
+}
+
+#define SKIP_SPACES_COMMENTS(parser, chunk, p) do { \
+ while ((p) < (chunk)->end) { \
+ if (!ucl_test_character (*(p), UCL_CHARACTER_WHITESPACE_UNSAFE)) { \
+ if ((chunk)->remain >= 2 && ucl_lex_is_comment ((p)[0], (p)[1])) { \
+ if (!ucl_skip_comments (parser)) { \
+ return false; \
+ } \
+ p = (chunk)->pos; \
+ } \
+ break; \
+ } \
+ ucl_chunk_skipc (chunk, p); \
+ } \
+} while(0)
+
+/**
+ * Handle the main states of rcl parser
+ * @param parser parser structure
+ * @return true if chunk has been parsed and false in case of error
+ */
+static bool
+ucl_state_machine (struct ucl_parser *parser)
+{
+ ucl_object_t *obj, *macro_args;
+ struct ucl_chunk *chunk = parser->chunks;
+ const unsigned char *p, *c = NULL, *macro_start = NULL;
+ unsigned char *macro_escaped;
+ size_t macro_len = 0;
+ struct ucl_macro *macro = NULL;
+ bool next_key = false, end_of_object = false, ret;
+
+ if (parser->top_obj == NULL) {
+ parser->state = UCL_STATE_INIT;
+ }
+
+ p = chunk->pos;
+ while (chunk->pos < chunk->end) {
+ switch (parser->state) {
+ case UCL_STATE_INIT:
+ /*
+ * At the init state we can either go to the parse array or object
+ * if we got [ or { correspondingly or can just treat new data as
+ * a key of newly created object
+ */
+ if (!ucl_skip_comments (parser)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ else {
+ bool seen_obrace = false;
+
+ /* Skip any spaces */
+ while (p < chunk->end && ucl_test_character (*p,
+ UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ p = chunk->pos;
+
+ if (p < chunk->end) {
+ if (*p == '[') {
+ parser->state = UCL_STATE_VALUE;
+ ucl_chunk_skipc (chunk, p);
+ seen_obrace = true;
+ }
+ else {
+
+ if (*p == '{') {
+ ucl_chunk_skipc (chunk, p);
+ parser->state = UCL_STATE_KEY_OBRACE;
+ seen_obrace = true;
+ }
+ else {
+ parser->state = UCL_STATE_KEY;
+ }
+ }
+ }
+
+ if (parser->top_obj == NULL) {
+ if (parser->state == UCL_STATE_VALUE) {
+ obj = ucl_parser_add_container (NULL, parser, true, 0,
+ seen_obrace);
+ }
+ else {
+ obj = ucl_parser_add_container (NULL, parser, false, 0,
+ seen_obrace);
+ }
+
+ if (obj == NULL) {
+ return false;
+ }
+
+ parser->top_obj = obj;
+ parser->cur_obj = obj;
+ }
+
+ }
+ break;
+ case UCL_STATE_KEY:
+ case UCL_STATE_KEY_OBRACE:
+ /* Skip any spaces */
+ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (p == chunk->end || *p == '}') {
+ /* We have the end of an object */
+ parser->state = UCL_STATE_AFTER_VALUE;
+ continue;
+ }
+ if (parser->stack == NULL) {
+ /* No objects are on stack, but we want to parse a key */
+ ucl_set_err (parser, UCL_ESYNTAX, "top object is finished but the parser "
+ "expects a key", &parser->err);
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ if (!ucl_parse_key (parser, chunk, &next_key, &end_of_object)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+
+ if (end_of_object) {
+ p = chunk->pos;
+ parser->state = UCL_STATE_AFTER_VALUE;
+ continue;
+ }
+ else if (parser->state != UCL_STATE_MACRO_NAME) {
+ if (next_key && parser->stack->obj->type == UCL_OBJECT) {
+ /* Parse more keys and nest objects accordingly */
+ obj = ucl_parser_add_container (parser->cur_obj,
+ parser,
+ false,
+ parser->stack->e.params.level + 1,
+ parser->state == UCL_STATE_KEY_OBRACE);
+ if (obj == NULL) {
+ return false;
+ }
+ }
+ else {
+ parser->state = UCL_STATE_VALUE;
+ }
+ }
+ else {
+ c = chunk->pos;
+ }
+ p = chunk->pos;
+ break;
+ case UCL_STATE_VALUE:
+ /* We need to check what we do have */
+ if (!parser->cur_obj || !ucl_parse_value (parser, chunk)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ /* State is set in ucl_parse_value call */
+ p = chunk->pos;
+ break;
+ case UCL_STATE_AFTER_VALUE:
+ if (!ucl_parse_after_value (parser, chunk)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+
+ if (parser->stack != NULL) {
+ if (parser->stack->obj->type == UCL_OBJECT) {
+ parser->state = UCL_STATE_KEY;
+ }
+ else {
+ /* Array */
+ parser->state = UCL_STATE_VALUE;
+ }
+ }
+ else {
+ /* Skip everything at the end */
+ return true;
+ }
+
+ p = chunk->pos;
+ break;
+ case UCL_STATE_MACRO_NAME:
+ if (parser->flags & UCL_PARSER_DISABLE_MACRO) {
+ if (!ucl_skip_macro_as_comment (parser, chunk)) {
+ /* We have invalid macro */
+ ucl_create_err (&parser->err,
+ "error at %s:%d at column %d: invalid macro",
+ chunk->fname ? chunk->fname : "memory",
+ chunk->line,
+ chunk->column);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ else {
+ p = chunk->pos;
+ parser->state = parser->prev_state;
+ }
+ }
+ else {
+ if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) &&
+ *p != '(') {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else {
+ if (c != NULL && p - c > 0) {
+ /* We got macro name */
+ macro_len = (size_t) (p - c);
+ HASH_FIND (hh, parser->macroes, c, macro_len, macro);
+ if (macro == NULL) {
+ ucl_create_err (&parser->err,
+ "error at %s:%d at column %d: "
+ "unknown macro: '%.*s', character: '%c'",
+ chunk->fname ? chunk->fname : "memory",
+ chunk->line,
+ chunk->column,
+ (int) (p - c),
+ c,
+ *chunk->pos);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ /* Now we need to skip all spaces */
+ SKIP_SPACES_COMMENTS(parser, chunk, p);
+ parser->state = UCL_STATE_MACRO;
+ }
+ else {
+ /* We have invalid macro name */
+ ucl_create_err (&parser->err,
+ "error at %s:%d at column %d: invalid macro name",
+ chunk->fname ? chunk->fname : "memory",
+ chunk->line,
+ chunk->column);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ }
+ }
+ break;
+ case UCL_STATE_MACRO:
+ if (*chunk->pos == '(') {
+ macro_args = ucl_parse_macro_arguments (parser, chunk);
+ p = chunk->pos;
+ if (macro_args) {
+ SKIP_SPACES_COMMENTS(parser, chunk, p);
+ }
+ }
+ else {
+ macro_args = NULL;
+ }
+ if (!ucl_parse_macro_value (parser, chunk, macro,
+ &macro_start, &macro_len)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ macro_len = ucl_expand_variable (parser, &macro_escaped,
+ macro_start, macro_len);
+ parser->state = parser->prev_state;
+
+ if (macro_escaped == NULL && macro != NULL) {
+ if (macro->is_context) {
+ ret = macro->h.context_handler (macro_start, macro_len,
+ macro_args,
+ parser->top_obj,
+ macro->ud);
+ }
+ else {
+ ret = macro->h.handler (macro_start, macro_len, macro_args,
+ macro->ud);
+ }
+ }
+ else if (macro != NULL) {
+ if (macro->is_context) {
+ ret = macro->h.context_handler (macro_escaped, macro_len,
+ macro_args,
+ parser->top_obj,
+ macro->ud);
+ }
+ else {
+ ret = macro->h.handler (macro_escaped, macro_len, macro_args,
+ macro->ud);
+ }
+
+ UCL_FREE (macro_len + 1, macro_escaped);
+ }
+ else {
+ ret = false;
+ ucl_set_err (parser, UCL_EINTERNAL,
+ "internal error: parser has macro undefined", &parser->err);
+ }
+
+ /*
+ * Chunk can be modified within macro handler
+ */
+ chunk = parser->chunks;
+ p = chunk->pos;
+
+ if (macro_args) {
+ ucl_object_unref (macro_args);
+ }
+
+ if (!ret) {
+ return false;
+ }
+ break;
+ default:
+ ucl_set_err (parser, UCL_EINTERNAL,
+ "internal error: parser is in an unknown state", &parser->err);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ }
+
+ if (parser->last_comment) {
+ if (parser->cur_obj) {
+ ucl_attach_comment (parser, parser->cur_obj, true);
+ }
+ else if (parser->stack && parser->stack->obj) {
+ ucl_attach_comment (parser, parser->stack->obj, true);
+ }
+ else if (parser->top_obj) {
+ ucl_attach_comment (parser, parser->top_obj, true);
+ }
+ else {
+ ucl_object_unref (parser->last_comment);
+ }
+ }
+
+ if (parser->stack != NULL && parser->state != UCL_STATE_ERROR) {
+ struct ucl_stack *st;
+ bool has_error = false;
+
+ LL_FOREACH (parser->stack, st) {
+ if (st->chunk != parser->chunks) {
+ break; /* Not our chunk, give up */
+ }
+ if (st->e.params.flags & UCL_STACK_HAS_OBRACE) {
+ if (parser->err == NULL) {
+ utstring_new (parser->err);
+ }
+
+ utstring_printf (parser->err, "%s:%d unmatched open brace at %d; ",
+ chunk->fname ? chunk->fname : "memory",
+ parser->chunks->line,
+ st->e.params.line);
+
+ has_error = true;
+ }
+ }
+
+ if (has_error) {
+ parser->err_code = UCL_EUNPAIRED;
+
+ return false;
+ }
+ }
+
+ return true;
+}
+
+#define UPRM_SAFE(fn, a, b, c, el) do { \
+ if (!fn(a, b, c, a)) \
+ goto el; \
+ } while (0)
+
+struct ucl_parser*
+ucl_parser_new (int flags)
+{
+ struct ucl_parser *parser;
+
+ parser = UCL_ALLOC (sizeof (struct ucl_parser));
+ if (parser == NULL) {
+ return NULL;
+ }
+
+ memset (parser, 0, sizeof (struct ucl_parser));
+
+ UPRM_SAFE(ucl_parser_register_macro, parser, "include", ucl_include_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "try_include", ucl_try_include_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "includes", ucl_includes_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "priority", ucl_priority_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "load", ucl_load_handler, e0);
+ UPRM_SAFE(ucl_parser_register_context_macro, parser, "inherit", ucl_inherit_handler, e0);
+
+ parser->flags = flags;
+ parser->includepaths = NULL;
+
+ if (flags & UCL_PARSER_SAVE_COMMENTS) {
+ parser->comments = ucl_object_typed_new (UCL_OBJECT);
+ }
+
+ if (!(flags & UCL_PARSER_NO_FILEVARS)) {
+ /* Initial assumption about filevars */
+ ucl_parser_set_filevars (parser, NULL, false);
+ }
+
+ return parser;
+e0:
+ ucl_parser_free(parser);
+ return NULL;
+}
+
+bool
+ucl_parser_set_default_priority (struct ucl_parser *parser, unsigned prio)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ parser->default_priority = prio;
+
+ return true;
+}
+
+int
+ucl_parser_get_default_priority (struct ucl_parser *parser)
+{
+ if (parser == NULL) {
+ return -1;
+ }
+
+ return parser->default_priority;
+}
+
+bool
+ucl_parser_register_macro (struct ucl_parser *parser, const char *macro,
+ ucl_macro_handler handler, void* ud)
+{
+ struct ucl_macro *new;
+
+ if (macro == NULL || handler == NULL) {
+ return false;
+ }
+
+ new = UCL_ALLOC (sizeof (struct ucl_macro));
+ if (new == NULL) {
+ return false;
+ }
+
+ memset (new, 0, sizeof (struct ucl_macro));
+ new->h.handler = handler;
+ new->name = strdup (macro);
+ if (new->name == NULL) {
+ UCL_FREE (sizeof (struct ucl_macro), new);
+ return false;
+ }
+ new->ud = ud;
+ HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new);
+ return true;
+}
+
+bool
+ucl_parser_register_context_macro (struct ucl_parser *parser, const char *macro,
+ ucl_context_macro_handler handler, void* ud)
+{
+ struct ucl_macro *new;
+
+ if (macro == NULL || handler == NULL) {
+ return false;
+ }
+
+ new = UCL_ALLOC (sizeof (struct ucl_macro));
+ if (new == NULL) {
+ return false;
+ }
+
+ memset (new, 0, sizeof (struct ucl_macro));
+ new->h.context_handler = handler;
+ new->name = strdup (macro);
+ if (new->name == NULL) {
+ UCL_FREE (sizeof (struct ucl_macro), new);
+ return false;
+ }
+ new->ud = ud;
+ new->is_context = true;
+ HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new);
+ return true;
+}
+
+void
+ucl_parser_register_variable (struct ucl_parser *parser, const char *var,
+ const char *value)
+{
+ struct ucl_variable *new = NULL, *cur;
+
+ if (var == NULL) {
+ return;
+ }
+
+ /* Find whether a variable already exists */
+ LL_FOREACH (parser->variables, cur) {
+ if (strcmp (cur->var, var) == 0) {
+ new = cur;
+ break;
+ }
+ }
+
+ if (value == NULL) {
+
+ if (new != NULL) {
+ /* Remove variable */
+ DL_DELETE (parser->variables, new);
+ free (new->var);
+ free (new->value);
+ UCL_FREE (sizeof (struct ucl_variable), new);
+ }
+ else {
+ /* Do nothing */
+ return;
+ }
+ }
+ else {
+ if (new == NULL) {
+ new = UCL_ALLOC (sizeof (struct ucl_variable));
+ if (new == NULL) {
+ return;
+ }
+ memset (new, 0, sizeof (struct ucl_variable));
+ new->var = strdup (var);
+ new->var_len = strlen (var);
+ new->value = strdup (value);
+ new->value_len = strlen (value);
+
+ DL_APPEND (parser->variables, new);
+ }
+ else {
+ free (new->value);
+ new->value = strdup (value);
+ new->value_len = strlen (value);
+ }
+ }
+}
+
+void
+ucl_parser_set_variables_handler (struct ucl_parser *parser,
+ ucl_variable_handler handler, void *ud)
+{
+ parser->var_handler = handler;
+ parser->var_data = ud;
+}
+
+bool
+ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data,
+ size_t len, unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type)
+{
+ struct ucl_chunk *chunk;
+ struct ucl_parser_special_handler *special_handler;
+
+ if (parser == NULL) {
+ return false;
+ }
+
+ if (data == NULL && len != 0) {
+ ucl_create_err (&parser->err, "invalid chunk added");
+ return false;
+ }
+
+ if (parser->state != UCL_STATE_ERROR) {
+ chunk = UCL_ALLOC (sizeof (struct ucl_chunk));
+ if (chunk == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate chunk structure");
+ return false;
+ }
+
+ memset (chunk, 0, sizeof (*chunk));
+
+ /* Apply all matching handlers from the first to the last */
+ LL_FOREACH (parser->special_handlers, special_handler) {
+ if ((special_handler->flags & UCL_SPECIAL_HANDLER_PREPROCESS_ALL) ||
+ (len >= special_handler->magic_len &&
+ memcmp (data, special_handler->magic, special_handler->magic_len) == 0)) {
+ unsigned char *ndata = NULL;
+ size_t nlen = 0;
+
+ if (!special_handler->handler (parser, data, len, &ndata, &nlen,
+ special_handler->user_data)) {
+ UCL_FREE(sizeof (struct ucl_chunk), chunk);
+ ucl_create_err (&parser->err, "call for external handler failed");
+
+ return false;
+ }
+
+ struct ucl_parser_special_handler_chain *nchain;
+ nchain = UCL_ALLOC (sizeof (*nchain));
+ nchain->begin = ndata;
+ nchain->len = nlen;
+ nchain->special_handler = special_handler;
+
+ /* Free order is reversed */
+ LL_PREPEND (chunk->special_handlers, nchain);
+
+ data = ndata;
+ len = nlen;
+ }
+ }
+
+ if (parse_type == UCL_PARSE_AUTO && len > 0) {
+ /* We need to detect parse type by the first symbol */
+ if ((*data & 0x80) == 0x80 && (*data >= 0xdc && *data <= 0xdf)) {
+ parse_type = UCL_PARSE_MSGPACK;
+ }
+ else if (*data == '(') {
+ parse_type = UCL_PARSE_CSEXP;
+ }
+ else {
+ parse_type = UCL_PARSE_UCL;
+ }
+ }
+
+ chunk->begin = data;
+ chunk->remain = len;
+ chunk->pos = chunk->begin;
+ chunk->end = chunk->begin + len;
+ chunk->line = 1;
+ chunk->column = 0;
+ chunk->priority = priority;
+ chunk->strategy = strat;
+ chunk->parse_type = parse_type;
+
+ if (parser->cur_file) {
+ chunk->fname = strdup (parser->cur_file);
+ }
+
+ LL_PREPEND (parser->chunks, chunk);
+ parser->recursion ++;
+
+ if (parser->recursion > UCL_MAX_RECURSION) {
+ ucl_create_err (&parser->err, "maximum include nesting limit is reached: %d",
+ parser->recursion);
+ return false;
+ }
+
+ if (len > 0) {
+ /* Need to parse something */
+ switch (parse_type) {
+ default:
+ case UCL_PARSE_UCL:
+ return ucl_state_machine (parser);
+ case UCL_PARSE_MSGPACK:
+ return ucl_parse_msgpack (parser);
+ case UCL_PARSE_CSEXP:
+ return ucl_parse_csexp (parser);
+ }
+ }
+ else {
+ /* Just add empty chunk and go forward */
+ if (parser->top_obj == NULL) {
+ /*
+ * In case of empty object, create one to indicate that we've
+ * read something
+ */
+ parser->top_obj = ucl_object_new_full (UCL_OBJECT, priority);
+ }
+
+ return true;
+ }
+ }
+
+ ucl_create_err (&parser->err, "a parser is in an invalid state");
+
+ return false;
+}
+
+bool
+ucl_parser_add_chunk_priority (struct ucl_parser *parser,
+ const unsigned char *data, size_t len, unsigned priority)
+{
+ /* We dereference parser, so this check is essential */
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_chunk_full (parser, data, len,
+ priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_add_chunk (struct ucl_parser *parser, const unsigned char *data,
+ size_t len)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_chunk_full (parser, data, len,
+ parser->default_priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_insert_chunk (struct ucl_parser *parser, const unsigned char *data,
+ size_t len)
+{
+ if (parser == NULL || parser->top_obj == NULL) {
+ return false;
+ }
+
+ bool res;
+ struct ucl_chunk *chunk;
+
+ int state = parser->state;
+ parser->state = UCL_STATE_INIT;
+
+ /* Prevent inserted chunks from unintentionally closing the current object */
+ if (parser->stack != NULL && parser->stack->next != NULL) {
+ parser->stack->e.params.level = parser->stack->next->e.params.level;
+ }
+
+ res = ucl_parser_add_chunk_full (parser, data, len, parser->chunks->priority,
+ parser->chunks->strategy, parser->chunks->parse_type);
+
+ /* Remove chunk from the stack */
+ chunk = parser->chunks;
+ if (chunk != NULL) {
+ parser->chunks = chunk->next;
+ ucl_chunk_free (chunk);
+ parser->recursion --;
+ }
+
+ parser->state = state;
+
+ return res;
+}
+
+bool
+ucl_parser_add_string_priority (struct ucl_parser *parser, const char *data,
+ size_t len, unsigned priority)
+{
+ if (data == NULL) {
+ ucl_create_err (&parser->err, "invalid string added");
+ return false;
+ }
+ if (len == 0) {
+ len = strlen (data);
+ }
+
+ return ucl_parser_add_chunk_priority (parser,
+ (const unsigned char *)data, len, priority);
+}
+
+bool
+ucl_parser_add_string (struct ucl_parser *parser, const char *data,
+ size_t len)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_string_priority (parser,
+ (const unsigned char *)data, len, parser->default_priority);
+}
+
+bool
+ucl_set_include_path (struct ucl_parser *parser, ucl_object_t *paths)
+{
+ if (parser == NULL || paths == NULL) {
+ return false;
+ }
+
+ if (parser->includepaths == NULL) {
+ parser->includepaths = ucl_object_copy (paths);
+ }
+ else {
+ ucl_object_unref (parser->includepaths);
+ parser->includepaths = ucl_object_copy (paths);
+ }
+
+ if (parser->includepaths == NULL) {
+ return false;
+ }
+
+ return true;
+}
+
+unsigned char ucl_parser_chunk_peek (struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL || parser->chunks->pos == NULL || parser->chunks->end == NULL ||
+ parser->chunks->pos == parser->chunks->end) {
+ return 0;
+ }
+
+ return( *parser->chunks->pos );
+}
+
+bool ucl_parser_chunk_skip (struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL || parser->chunks->pos == NULL || parser->chunks->end == NULL ||
+ parser->chunks->pos == parser->chunks->end) {
+ return false;
+ }
+
+ const unsigned char *p = parser->chunks->pos;
+ ucl_chunk_skipc( parser->chunks, p );
+ if( parser->chunks->pos != NULL ) return true;
+ return false;
+}
+
+ucl_object_t*
+ucl_parser_get_current_stack_object (struct ucl_parser *parser, unsigned int depth)
+{
+ ucl_object_t *obj;
+
+ if (parser == NULL || parser->stack == NULL) {
+ return NULL;
+ }
+
+ struct ucl_stack *stack = parser->stack;
+ if(stack == NULL || stack->obj == NULL || ucl_object_type (stack->obj) != UCL_OBJECT)
+ {
+ return NULL;
+ }
+
+ for( unsigned int i = 0; i < depth; ++i )
+ {
+ stack = stack->next;
+ if(stack == NULL || stack->obj == NULL || ucl_object_type (stack->obj) != UCL_OBJECT)
+ {
+ return NULL;
+ }
+ }
+
+ obj = ucl_object_ref (stack->obj);
+ return obj;
+}
+
diff --git a/contrib/libucl/ucl_schema.c b/contrib/libucl/ucl_schema.c
new file mode 100644
index 0000000..68f0118
--- /dev/null
+++ b/contrib/libucl/ucl_schema.c
@@ -0,0 +1,1104 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "tree.h"
+#include "utlist.h"
+#ifdef HAVE_STDARG_H
+#include <stdarg.h>
+#endif
+#ifdef HAVE_STDIO_H
+#include <stdio.h>
+#endif
+#ifdef HAVE_REGEX_H
+#include <regex.h>
+#endif
+#ifdef HAVE_MATH_H
+#include <math.h>
+#endif
+
+static bool ucl_schema_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, bool try_array,
+ struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref);
+
+/*
+ * Create validation error
+ */
+
+#ifdef __GNUC__
+static inline void
+ucl_schema_create_error (struct ucl_schema_error *err,
+ enum ucl_schema_error_code code, const ucl_object_t *obj,
+ const char *fmt, ...)
+__attribute__ (( format( printf, 4, 5) ));
+#endif
+
+static inline void
+ucl_schema_create_error (struct ucl_schema_error *err,
+ enum ucl_schema_error_code code, const ucl_object_t *obj,
+ const char *fmt, ...)
+{
+ va_list va;
+
+ if (err != NULL) {
+ err->code = code;
+ err->obj = obj;
+ va_start (va, fmt);
+ vsnprintf (err->msg, sizeof (err->msg), fmt, va);
+ va_end (va);
+ }
+}
+
+/*
+ * Check whether we have a pattern specified
+ */
+static const ucl_object_t *
+ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern, bool recursive)
+{
+ const ucl_object_t *res = NULL;
+#ifdef HAVE_REGEX_H
+ regex_t reg;
+ const ucl_object_t *elt;
+ ucl_object_iter_t iter = NULL;
+
+ if (regcomp (&reg, pattern, REG_EXTENDED | REG_NOSUB) == 0) {
+ if (recursive) {
+ while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) {
+ if (regexec (&reg, ucl_object_key (elt), 0, NULL, 0) == 0) {
+ res = elt;
+ break;
+ }
+ }
+ } else {
+ if (regexec (&reg, ucl_object_key (obj), 0, NULL, 0) == 0)
+ res = obj;
+ }
+ regfree (&reg);
+ }
+#endif
+ return res;
+}
+
+/*
+ * Check dependencies for an object
+ */
+static bool
+ucl_schema_validate_dependencies (const ucl_object_t *deps,
+ const ucl_object_t *obj, struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref)
+{
+ const ucl_object_t *elt, *cur, *cur_dep;
+ ucl_object_iter_t iter = NULL, piter;
+ bool ret = true;
+
+ while (ret && (cur = ucl_object_iterate (deps, &iter, true)) != NULL) {
+ elt = ucl_object_lookup (obj, ucl_object_key (cur));
+ if (elt != NULL) {
+ /* Need to check dependencies */
+ if (cur->type == UCL_ARRAY) {
+ piter = NULL;
+ while (ret && (cur_dep = ucl_object_iterate (cur, &piter, true)) != NULL) {
+ if (ucl_object_lookup (obj, ucl_object_tostring (cur_dep)) == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_MISSING_DEPENDENCY, elt,
+ "dependency %s is missing for key %s",
+ ucl_object_tostring (cur_dep), ucl_object_key (cur));
+ ret = false;
+ break;
+ }
+ }
+ }
+ else if (cur->type == UCL_OBJECT) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, ext_ref);
+ }
+ }
+ }
+
+ return ret;
+}
+
+/*
+ * Validate object
+ */
+static bool
+ucl_schema_validate_object (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref)
+{
+ const ucl_object_t *elt, *prop, *found, *additional_schema = NULL,
+ *required = NULL, *pat, *pelt;
+ ucl_object_iter_t iter = NULL, piter = NULL;
+ bool ret = true, allow_additional = true;
+ int64_t minmax;
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if (elt->type == UCL_OBJECT &&
+ strcmp (ucl_object_key (elt), "properties") == 0) {
+ piter = NULL;
+ while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) {
+ found = ucl_object_lookup (obj, ucl_object_key (prop));
+ if (found) {
+ ret = ucl_schema_validate (prop, found, true, err, root,
+ ext_ref);
+ }
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "additionalProperties") == 0) {
+ if (elt->type == UCL_BOOLEAN) {
+ if (!ucl_object_toboolean (elt)) {
+ /* Deny additional fields completely */
+ allow_additional = false;
+ }
+ }
+ else if (elt->type == UCL_OBJECT) {
+ /* Define validator for additional fields */
+ additional_schema = elt;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "additionalProperties attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "required") == 0) {
+ if (elt->type == UCL_ARRAY) {
+ required = elt;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "required attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "minProperties") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len < minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has not enough properties: %u, minimum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "maxProperties") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len > minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has too many properties: %u, maximum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) {
+ const ucl_object_t *vobj;
+ ucl_object_iter_t viter;
+ piter = NULL;
+ while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) {
+ viter = NULL;
+ while (ret && (vobj = ucl_object_iterate (obj, &viter, true)) != NULL) {
+ found = ucl_schema_test_pattern (vobj, ucl_object_key (prop), false);
+ if (found) {
+ ret = ucl_schema_validate (prop, found, true, err, root,
+ ext_ref);
+ }
+ }
+ }
+ }
+ else if (elt->type == UCL_OBJECT &&
+ strcmp (ucl_object_key (elt), "dependencies") == 0) {
+ ret = ucl_schema_validate_dependencies (elt, obj, err, root,
+ ext_ref);
+ }
+ }
+
+ if (ret) {
+ /* Additional properties */
+ if (!allow_additional || additional_schema != NULL) {
+ /* Check if we have exactly the same properties in schema and object */
+ iter = ucl_object_iterate_new (obj);
+ prop = ucl_object_lookup (schema, "properties");
+ while ((elt = ucl_object_iterate_safe (iter, true)) != NULL) {
+ found = ucl_object_lookup (prop, ucl_object_key (elt));
+ if (found == NULL) {
+ /* Try patternProperties */
+ pat = ucl_object_lookup (schema, "patternProperties");
+ piter = ucl_object_iterate_new (pat);
+ while ((pelt = ucl_object_iterate_safe (piter, true)) != NULL) {
+ found = ucl_schema_test_pattern (obj, ucl_object_key (pelt), true);
+ if (found != NULL) {
+ break;
+ }
+ }
+ ucl_object_iterate_free (piter);
+ piter = NULL;
+ }
+ if (found == NULL) {
+ if (!allow_additional) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has non-allowed property %s",
+ ucl_object_key (elt));
+ ret = false;
+ break;
+ }
+ else if (additional_schema != NULL) {
+ if (!ucl_schema_validate (additional_schema, elt,
+ true, err, root, ext_ref)) {
+ ret = false;
+ break;
+ }
+ }
+ }
+ }
+ ucl_object_iterate_free (iter);
+ iter = NULL;
+ }
+ /* Required properties */
+ if (required != NULL) {
+ iter = NULL;
+ while ((elt = ucl_object_iterate (required, &iter, true)) != NULL) {
+ if (ucl_object_lookup (obj, ucl_object_tostring (elt)) == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_MISSING_PROPERTY, obj,
+ "object has missing property %s",
+ ucl_object_tostring (elt));
+ ret = false;
+ break;
+ }
+ }
+ }
+ }
+
+
+ return ret;
+}
+
+static bool
+ucl_schema_validate_number (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ const ucl_object_t *elt, *test;
+ ucl_object_iter_t iter = NULL;
+ bool ret = true, exclusive = false;
+ double constraint, val;
+ const double alpha = 1e-16;
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
+ strcmp (ucl_object_key (elt), "multipleOf") == 0) {
+ constraint = ucl_object_todouble (elt);
+ if (constraint <= 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "multipleOf must be greater than zero");
+ ret = false;
+ break;
+ }
+ val = ucl_object_todouble (obj);
+ if (fabs (remainder (val, constraint)) > alpha) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "number %.4f is not multiple of %.4f, remainder is %.7f",
+ val, constraint, remainder (val, constraint));
+ ret = false;
+ break;
+ }
+ }
+ else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
+ strcmp (ucl_object_key (elt), "maximum") == 0) {
+ constraint = ucl_object_todouble (elt);
+ test = ucl_object_lookup (schema, "exclusiveMaximum");
+ if (test && test->type == UCL_BOOLEAN) {
+ exclusive = ucl_object_toboolean (test);
+ }
+ val = ucl_object_todouble (obj);
+ if (val > constraint || (exclusive && val >= constraint)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "number is too big: %.3f, maximum is: %.3f",
+ val, constraint);
+ ret = false;
+ break;
+ }
+ }
+ else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
+ strcmp (ucl_object_key (elt), "minimum") == 0) {
+ constraint = ucl_object_todouble (elt);
+ test = ucl_object_lookup (schema, "exclusiveMinimum");
+ if (test && test->type == UCL_BOOLEAN) {
+ exclusive = ucl_object_toboolean (test);
+ }
+ val = ucl_object_todouble (obj);
+ if (val < constraint || (exclusive && val <= constraint)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "number is too small: %.3f, minimum is: %.3f",
+ val, constraint);
+ ret = false;
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+static bool
+ucl_schema_validate_string (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ const ucl_object_t *elt;
+ ucl_object_iter_t iter = NULL;
+ bool ret = true;
+ int64_t constraint;
+#ifdef HAVE_REGEX_H
+ regex_t re;
+#endif
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if (elt->type == UCL_INT &&
+ strcmp (ucl_object_key (elt), "maxLength") == 0) {
+ constraint = ucl_object_toint (elt);
+ if (obj->len > constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "string is too big: %u, maximum is: %" PRId64,
+ obj->len, constraint);
+ ret = false;
+ break;
+ }
+ }
+ else if (elt->type == UCL_INT &&
+ strcmp (ucl_object_key (elt), "minLength") == 0) {
+ constraint = ucl_object_toint (elt);
+ if (obj->len < constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "string is too short: %u, minimum is: %" PRId64,
+ obj->len, constraint);
+ ret = false;
+ break;
+ }
+ }
+#ifdef HAVE_REGEX_H
+ else if (elt->type == UCL_STRING &&
+ strcmp (ucl_object_key (elt), "pattern") == 0) {
+ if (regcomp (&re, ucl_object_tostring (elt),
+ REG_EXTENDED | REG_NOSUB) != 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "cannot compile pattern %s", ucl_object_tostring (elt));
+ ret = false;
+ break;
+ }
+ if (regexec (&re, ucl_object_tostring (obj), 0, NULL, 0) != 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "string doesn't match regexp %s",
+ ucl_object_tostring (elt));
+ ret = false;
+ }
+ regfree (&re);
+ }
+#endif
+ }
+
+ return ret;
+}
+
+struct ucl_compare_node {
+ const ucl_object_t *obj;
+ TREE_ENTRY(ucl_compare_node) link;
+ struct ucl_compare_node *next;
+};
+
+typedef TREE_HEAD(_tree, ucl_compare_node) ucl_compare_tree_t;
+
+TREE_DEFINE(ucl_compare_node, link)
+
+static int
+ucl_schema_elt_compare (struct ucl_compare_node *n1, struct ucl_compare_node *n2)
+{
+ const ucl_object_t *o1 = n1->obj, *o2 = n2->obj;
+
+ return ucl_object_compare (o1, o2);
+}
+
+static bool
+ucl_schema_array_is_unique (const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ ucl_compare_tree_t tree = TREE_INITIALIZER (ucl_schema_elt_compare);
+ ucl_object_iter_t iter = NULL;
+ const ucl_object_t *elt;
+ struct ucl_compare_node *node, test, *nodes = NULL, *tmp;
+ bool ret = true;
+
+ while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) {
+ test.obj = elt;
+ node = TREE_FIND (&tree, ucl_compare_node, link, &test);
+ if (node != NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, elt,
+ "duplicate values detected while uniqueItems is true");
+ ret = false;
+ break;
+ }
+ node = calloc (1, sizeof (*node));
+ if (node == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_UNKNOWN, elt,
+ "cannot allocate tree node");
+ ret = false;
+ break;
+ }
+ node->obj = elt;
+ TREE_INSERT (&tree, ucl_compare_node, link, node);
+ LL_PREPEND (nodes, node);
+ }
+
+ LL_FOREACH_SAFE (nodes, node, tmp) {
+ free (node);
+ }
+
+ return ret;
+}
+
+static bool
+ucl_schema_validate_array (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref)
+{
+ const ucl_object_t *elt, *it, *found, *additional_schema = NULL,
+ *first_unvalidated = NULL;
+ ucl_object_iter_t iter = NULL, piter = NULL;
+ bool ret = true, allow_additional = true, need_unique = false;
+ int64_t minmax;
+ unsigned int idx = 0;
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if (strcmp (ucl_object_key (elt), "items") == 0) {
+ if (elt->type == UCL_ARRAY) {
+ found = ucl_array_head (obj);
+ while (ret && (it = ucl_object_iterate (elt, &piter, true)) != NULL) {
+ if (found) {
+ ret = ucl_schema_validate (it, found, false, err,
+ root, ext_ref);
+ found = ucl_array_find_index (obj, ++idx);
+ }
+ }
+ if (found != NULL) {
+ /* The first element that is not validated */
+ first_unvalidated = found;
+ }
+ }
+ else if (elt->type == UCL_OBJECT) {
+ /* Validate all items using the specified schema */
+ while (ret && (it = ucl_object_iterate (obj, &piter, true)) != NULL) {
+ ret = ucl_schema_validate (elt, it, false, err, root,
+ ext_ref);
+ }
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "items attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "additionalItems") == 0) {
+ if (elt->type == UCL_BOOLEAN) {
+ if (!ucl_object_toboolean (elt)) {
+ /* Deny additional fields completely */
+ allow_additional = false;
+ }
+ }
+ else if (elt->type == UCL_OBJECT) {
+ /* Define validator for additional fields */
+ additional_schema = elt;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "additionalItems attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (elt->type == UCL_BOOLEAN &&
+ strcmp (ucl_object_key (elt), "uniqueItems") == 0) {
+ need_unique = ucl_object_toboolean (elt);
+ }
+ else if (strcmp (ucl_object_key (elt), "minItems") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len < minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "array has not enough items: %u, minimum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "maxItems") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len > minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "array has too many items: %u, maximum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ }
+
+ if (ret) {
+ /* Additional properties */
+ if (!allow_additional || additional_schema != NULL) {
+ if (first_unvalidated != NULL) {
+ if (!allow_additional) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "array has undefined item");
+ ret = false;
+ }
+ else if (additional_schema != NULL) {
+ elt = ucl_array_find_index (obj, idx);
+ while (elt) {
+ if (!ucl_schema_validate (additional_schema, elt, false,
+ err, root, ext_ref)) {
+ ret = false;
+ break;
+ }
+ elt = ucl_array_find_index (obj, idx ++);
+ }
+ }
+ }
+ }
+ /* Required properties */
+ if (ret && need_unique) {
+ ret = ucl_schema_array_is_unique (obj, err);
+ }
+ }
+
+ return ret;
+}
+
+/*
+ * Returns whether this object is allowed for this type
+ */
+static bool
+ucl_schema_type_is_allowed (const ucl_object_t *type, const ucl_object_t *obj,
+ struct ucl_schema_error *err)
+{
+ ucl_object_iter_t iter = NULL;
+ const ucl_object_t *elt;
+ const char *type_str;
+ ucl_type_t t;
+
+ if (type == NULL) {
+ /* Any type is allowed */
+ return true;
+ }
+
+ if (type->type == UCL_ARRAY) {
+ /* One of allowed types */
+ while ((elt = ucl_object_iterate (type, &iter, true)) != NULL) {
+ if (ucl_schema_type_is_allowed (elt, obj, err)) {
+ return true;
+ }
+ }
+ }
+ else if (type->type == UCL_STRING) {
+ type_str = ucl_object_tostring (type);
+ if (!ucl_object_string_to_type (type_str, &t)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, type,
+ "Type attribute is invalid in schema");
+ return false;
+ }
+ if (obj->type != t) {
+ /* Some types are actually compatible */
+ if (obj->type == UCL_TIME && t == UCL_FLOAT) {
+ return true;
+ }
+ else if (obj->type == UCL_INT && t == UCL_FLOAT) {
+ return true;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_TYPE_MISMATCH, obj,
+ "Invalid type of %s, expected %s",
+ ucl_object_type_to_string (obj->type),
+ ucl_object_type_to_string (t));
+ }
+ }
+ else {
+ /* Types are equal */
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/*
+ * Check if object is equal to one of elements of enum
+ */
+static bool
+ucl_schema_validate_enum (const ucl_object_t *en, const ucl_object_t *obj,
+ struct ucl_schema_error *err)
+{
+ ucl_object_iter_t iter = NULL;
+ const ucl_object_t *elt;
+ bool ret = false;
+
+ while ((elt = ucl_object_iterate (en, &iter, true)) != NULL) {
+ if (ucl_object_compare (elt, obj) == 0) {
+ ret = true;
+ break;
+ }
+ }
+
+ if (!ret) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object is not one of enumerated patterns");
+ }
+
+ return ret;
+}
+
+
+/*
+ * Check a single ref component
+ */
+static const ucl_object_t *
+ucl_schema_resolve_ref_component (const ucl_object_t *cur,
+ const char *refc, int len,
+ struct ucl_schema_error *err)
+{
+ const ucl_object_t *res = NULL;
+ char *err_str;
+ int num, i;
+
+ if (cur->type == UCL_OBJECT) {
+ /* Find a key inside an object */
+ res = ucl_object_lookup_len (cur, refc, len);
+ if (res == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur,
+ "reference %s is invalid, missing path component", refc);
+ return NULL;
+ }
+ }
+ else if (cur->type == UCL_ARRAY) {
+ /* We must figure out a number inside array */
+ num = strtoul (refc, &err_str, 10);
+ if (err_str != NULL && *err_str != '/' && *err_str != '\0') {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur,
+ "reference %s is invalid, invalid item number", refc);
+ return NULL;
+ }
+ res = ucl_array_head (cur);
+ i = 0;
+ while (res != NULL) {
+ if (i == num) {
+ break;
+ }
+ res = res->next;
+ }
+ if (res == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur,
+ "reference %s is invalid, item number %d does not exist",
+ refc, num);
+ return NULL;
+ }
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res,
+ "reference %s is invalid, contains primitive object in the path",
+ refc);
+ return NULL;
+ }
+
+ return res;
+}
+/*
+ * Find reference schema
+ */
+static const ucl_object_t *
+ucl_schema_resolve_ref (const ucl_object_t *root, const char *ref,
+ struct ucl_schema_error *err, ucl_object_t *ext_ref,
+ ucl_object_t const ** nroot)
+{
+ UT_string *url_err = NULL;
+ struct ucl_parser *parser;
+ const ucl_object_t *res = NULL, *ext_obj = NULL;
+ ucl_object_t *url_obj;
+ const char *p, *c, *hash_ptr = NULL;
+ char *url_copy = NULL;
+ unsigned char *url_buf;
+ size_t url_buflen;
+
+ if (ref[0] != '#') {
+ hash_ptr = strrchr (ref, '#');
+
+ if (hash_ptr) {
+ url_copy = malloc (hash_ptr - ref + 1);
+
+ if (url_copy == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INTERNAL_ERROR, root,
+ "cannot allocate memory");
+ return NULL;
+ }
+
+ ucl_strlcpy (url_copy, ref, hash_ptr - ref + 1);
+ p = url_copy;
+ }
+ else {
+ /* Full URL */
+ p = ref;
+ }
+
+ ext_obj = ucl_object_lookup (ext_ref, p);
+
+ if (ext_obj == NULL) {
+ if (ucl_strnstr (p, "://", strlen (p)) != NULL) {
+ if (!ucl_fetch_url (p, &url_buf, &url_buflen, &url_err, true)) {
+
+ ucl_schema_create_error (err,
+ UCL_SCHEMA_INVALID_SCHEMA,
+ root,
+ "cannot fetch reference %s: %s",
+ p,
+ url_err != NULL ? utstring_body (url_err)
+ : "unknown");
+ free (url_copy);
+
+ return NULL;
+ }
+ }
+ else {
+ if (!ucl_fetch_file (p, &url_buf, &url_buflen, &url_err,
+ true)) {
+ ucl_schema_create_error (err,
+ UCL_SCHEMA_INVALID_SCHEMA,
+ root,
+ "cannot fetch reference %s: %s",
+ p,
+ url_err != NULL ? utstring_body (url_err)
+ : "unknown");
+ free (url_copy);
+
+ return NULL;
+ }
+ }
+
+ parser = ucl_parser_new (0);
+
+ if (!ucl_parser_add_chunk (parser, url_buf, url_buflen)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root,
+ "cannot fetch reference %s: %s", p,
+ ucl_parser_get_error (parser));
+ ucl_parser_free (parser);
+ free (url_copy);
+
+ return NULL;
+ }
+
+ url_obj = ucl_parser_get_object (parser);
+ ext_obj = url_obj;
+ ucl_object_insert_key (ext_ref, url_obj, p, 0, true);
+ free (url_buf);
+ }
+
+ free (url_copy);
+
+ if (hash_ptr) {
+ p = hash_ptr + 1;
+ }
+ else {
+ p = "";
+ }
+ }
+ else {
+ p = ref + 1;
+ }
+
+ res = ext_obj != NULL ? ext_obj : root;
+ *nroot = res;
+
+ if (*p == '/') {
+ p++;
+ }
+ else if (*p == '\0') {
+ return res;
+ }
+
+ c = p;
+
+ while (*p != '\0') {
+ if (*p == '/') {
+ if (p - c == 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res,
+ "reference %s is invalid, empty path component", ref);
+ return NULL;
+ }
+ /* Now we have some url part, so we need to figure out where we are */
+ res = ucl_schema_resolve_ref_component (res, c, p - c, err);
+ if (res == NULL) {
+ return NULL;
+ }
+ c = p + 1;
+ }
+ p ++;
+ }
+
+ if (p - c != 0) {
+ res = ucl_schema_resolve_ref_component (res, c, p - c, err);
+ }
+
+ if (res == NULL || res->type != UCL_OBJECT) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res,
+ "reference %s is invalid, cannot find specified object",
+ ref);
+ return NULL;
+ }
+
+ return res;
+}
+
+static bool
+ucl_schema_validate_values (const ucl_object_t *schema, const ucl_object_t *obj,
+ struct ucl_schema_error *err)
+{
+ const ucl_object_t *elt, *cur;
+ int64_t constraint, i;
+
+ elt = ucl_object_lookup (schema, "maxValues");
+ if (elt != NULL && elt->type == UCL_INT) {
+ constraint = ucl_object_toint (elt);
+ cur = obj;
+ i = 0;
+ while (cur) {
+ if (i > constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has more values than defined: %ld",
+ (long int)constraint);
+ return false;
+ }
+ i ++;
+ cur = cur->next;
+ }
+ }
+ elt = ucl_object_lookup (schema, "minValues");
+ if (elt != NULL && elt->type == UCL_INT) {
+ constraint = ucl_object_toint (elt);
+ cur = obj;
+ i = 0;
+ while (cur) {
+ if (i >= constraint) {
+ break;
+ }
+ i ++;
+ cur = cur->next;
+ }
+ if (i < constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has less values than defined: %ld",
+ (long int)constraint);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool
+ucl_schema_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, bool try_array,
+ struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *external_refs)
+{
+ const ucl_object_t *elt, *cur, *ref_root;
+ ucl_object_iter_t iter = NULL;
+ bool ret;
+
+ if (schema->type != UCL_OBJECT) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, schema,
+ "schema is %s instead of object",
+ ucl_object_type_to_string (schema->type));
+ return false;
+ }
+
+ if (try_array) {
+ /*
+ * Special case for multiple values
+ */
+ if (!ucl_schema_validate_values (schema, obj, err)) {
+ return false;
+ }
+ LL_FOREACH (obj, cur) {
+ if (!ucl_schema_validate (schema, cur, false, err, root, external_refs)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ elt = ucl_object_lookup (schema, "enum");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ if (!ucl_schema_validate_enum (elt, obj, err)) {
+ return false;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "allOf");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ iter = NULL;
+ while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, external_refs);
+ if (!ret) {
+ return false;
+ }
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "anyOf");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ iter = NULL;
+ while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, external_refs);
+ if (ret) {
+ break;
+ }
+ }
+ if (!ret) {
+ return false;
+ }
+ else {
+ /* Reset error */
+ err->code = UCL_SCHEMA_OK;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "oneOf");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ iter = NULL;
+ ret = false;
+ while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) {
+ if (!ret) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, external_refs);
+ }
+ else if (ucl_schema_validate (cur, obj, true, err, root, external_refs)) {
+ ret = false;
+ break;
+ }
+ }
+ if (!ret) {
+ return false;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "not");
+ if (elt != NULL && elt->type == UCL_OBJECT) {
+ if (ucl_schema_validate (elt, obj, true, err, root, external_refs)) {
+ return false;
+ }
+ else {
+ /* Reset error */
+ err->code = UCL_SCHEMA_OK;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "$ref");
+ if (elt != NULL) {
+ ref_root = root;
+ cur = ucl_schema_resolve_ref (root, ucl_object_tostring (elt),
+ err, external_refs, &ref_root);
+
+ if (cur == NULL) {
+ return false;
+ }
+ if (!ucl_schema_validate (cur, obj, try_array, err, ref_root,
+ external_refs)) {
+ return false;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "type");
+ if (!ucl_schema_type_is_allowed (elt, obj, err)) {
+ return false;
+ }
+
+ switch (obj->type) {
+ case UCL_OBJECT:
+ return ucl_schema_validate_object (schema, obj, err, root, external_refs);
+ break;
+ case UCL_ARRAY:
+ return ucl_schema_validate_array (schema, obj, err, root, external_refs);
+ break;
+ case UCL_INT:
+ case UCL_FLOAT:
+ return ucl_schema_validate_number (schema, obj, err);
+ break;
+ case UCL_STRING:
+ return ucl_schema_validate_string (schema, obj, err);
+ break;
+ default:
+ break;
+ }
+
+ return true;
+}
+
+bool
+ucl_object_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ return ucl_object_validate_root_ext (schema, obj, schema, NULL, err);
+}
+
+bool
+ucl_object_validate_root (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ struct ucl_schema_error *err)
+{
+ return ucl_object_validate_root_ext (schema, obj, root, NULL, err);
+}
+
+bool
+ucl_object_validate_root_ext (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ ucl_object_t *ext_refs,
+ struct ucl_schema_error *err)
+{
+ bool ret, need_unref = false;
+
+ if (ext_refs == NULL) {
+ ext_refs = ucl_object_typed_new (UCL_OBJECT);
+ need_unref = true;
+ }
+
+ ret = ucl_schema_validate (schema, obj, true, err, root, ext_refs);
+
+ if (need_unref) {
+ ucl_object_unref (ext_refs);
+ }
+
+ return ret;
+}
diff --git a/contrib/libucl/ucl_sexp.c b/contrib/libucl/ucl_sexp.c
new file mode 100644
index 0000000..1ad93d2
--- /dev/null
+++ b/contrib/libucl/ucl_sexp.c
@@ -0,0 +1,226 @@
+/*
+ * Copyright (c) 2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <ucl.h>
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "utlist.h"
+
+#define NEXT_STATE do { \
+if (p >= end) { \
+ if (state != read_ebrace) { \
+ ucl_create_err (&parser->err,\
+ "extra data");\
+ state = parse_err; \
+ } \
+} \
+else { \
+switch (*p) { \
+ case '(': \
+ state = read_obrace; \
+ break; \
+ case ')': \
+ state = read_ebrace; \
+ break; \
+ default: \
+ len = 0; \
+ mult = 1; \
+ state = read_length; \
+ break; \
+ } \
+} \
+} while(0)
+
+bool
+ucl_parse_csexp (struct ucl_parser *parser)
+{
+ const unsigned char *p, *end;
+ ucl_object_t *obj;
+ struct ucl_stack *st;
+ uint64_t len = 0, mult = 1;
+ enum {
+ start_parse,
+ read_obrace,
+ read_length,
+ read_value,
+ read_ebrace,
+ parse_err
+ } state = start_parse;
+
+ assert (parser != NULL);
+ assert (parser->chunks != NULL);
+ assert (parser->chunks->begin != NULL);
+ assert (parser->chunks->remain != 0);
+
+ p = parser->chunks->begin;
+ end = p + parser->chunks->remain;
+
+ while (p < end) {
+ switch (state) {
+ case start_parse:
+ /* At this point we expect open brace */
+ if (*p == '(') {
+ state = read_obrace;
+ }
+ else {
+ ucl_create_err (&parser->err, "bad starting character for "
+ "sexp block: %x", (int)*p);
+ state = parse_err;
+ }
+ break;
+
+ case read_obrace:
+ st = calloc (1, sizeof (*st));
+
+ if (st == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ state = parse_err;
+ continue;
+ }
+
+ st->obj = ucl_object_typed_new (UCL_ARRAY);
+
+ if (st->obj == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ state = parse_err;
+ free (st);
+ continue;
+ }
+
+ if (parser->stack == NULL) {
+ /* We have no stack */
+ parser->stack = st;
+
+ if (parser->top_obj == NULL) {
+ parser->top_obj = st->obj;
+ }
+ }
+ else {
+ /* Prepend new element to the stack */
+ LL_PREPEND (parser->stack, st);
+ }
+
+ p ++;
+ NEXT_STATE;
+
+ break;
+
+ case read_length:
+ if (*p == ':') {
+ if (len == 0) {
+ ucl_create_err (&parser->err, "zero length element");
+ state = parse_err;
+ continue;
+ }
+
+ state = read_value;
+ }
+ else if (*p >= '0' && *p <= '9') {
+ len += (*p - '0') * mult;
+ mult *= 10;
+
+ if (len > UINT32_MAX) {
+ ucl_create_err (&parser->err, "too big length of an "
+ "element");
+ state = parse_err;
+ continue;
+ }
+ }
+ else {
+ ucl_create_err (&parser->err, "bad length character: %x",
+ (int)*p);
+ state = parse_err;
+ continue;
+ }
+
+ p ++;
+ break;
+
+ case read_value:
+ if ((uint64_t)(end - p) > len || len == 0) {
+ ucl_create_err (&parser->err, "invalid length: %llu, %ld "
+ "remain", (long long unsigned)len, (long)(end - p));
+ state = parse_err;
+ continue;
+ }
+ obj = ucl_object_typed_new (UCL_STRING);
+
+ obj->value.sv = (const char*)p;
+ obj->len = len;
+ obj->flags |= UCL_OBJECT_BINARY;
+
+ if (!(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ ucl_copy_value_trash (obj);
+ }
+
+ ucl_array_append (parser->stack->obj, obj);
+ p += len;
+ NEXT_STATE;
+ break;
+
+ case read_ebrace:
+ if (parser->stack == NULL) {
+ /* We have an extra end brace */
+ ucl_create_err (&parser->err, "invalid length: %llu, %ld "
+ "remain", (long long unsigned)len, (long)(end - p));
+ state = parse_err;
+ continue;
+ }
+ /* Pop the container */
+ st = parser->stack;
+ parser->stack = st->next;
+
+ if (parser->stack->obj->type == UCL_ARRAY) {
+ ucl_array_append (parser->stack->obj, st->obj);
+ }
+ else {
+ ucl_create_err (&parser->err, "bad container object, array "
+ "expected");
+ state = parse_err;
+ continue;
+ }
+
+ free (st);
+ st = NULL;
+ p++;
+ NEXT_STATE;
+ break;
+
+ case parse_err:
+ default:
+ return false;
+ }
+ }
+
+ if (state != read_ebrace) {
+ ucl_create_err (&parser->err, "invalid finishing state: %d", state);
+ return false;
+ }
+
+ return true;
+}
diff --git a/contrib/libucl/ucl_util.c b/contrib/libucl/ucl_util.c
new file mode 100644
index 0000000..3f2483c
--- /dev/null
+++ b/contrib/libucl/ucl_util.c
@@ -0,0 +1,3952 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * Copyright (c) 2015 Allan Jude <allanjude@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+#include "kvec.h"
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h> /* for snprintf */
+
+#ifndef _WIN32
+#include <glob.h>
+#include <sys/param.h>
+#else
+#ifndef NBBY
+#define NBBY 8
+#endif
+#endif
+
+#ifdef HAVE_LIBGEN_H
+#ifndef _WIN32
+# include <libgen.h> /* For dirname */
+#endif
+#endif
+
+typedef kvec_t(ucl_object_t *) ucl_array_t;
+
+#define UCL_ARRAY_GET(ar, obj) ucl_array_t *ar = \
+ (ucl_array_t *)((obj) != NULL ? (obj)->value.av : NULL)
+
+#ifdef HAVE_OPENSSL
+#include <openssl/err.h>
+#include <openssl/sha.h>
+#include <openssl/rsa.h>
+#include <openssl/ssl.h>
+#include <openssl/evp.h>
+#endif
+
+#ifdef CURL_FOUND
+/* Seems to be broken */
+#define CURL_DISABLE_TYPECHECK 1
+#include <curl/curl.h>
+#endif
+#ifdef HAVE_FETCH_H
+#include <fetch.h>
+#endif
+
+#if defined(_MSC_VER)
+#include <windows.h>
+#include <io.h>
+#include <direct.h>
+
+#ifndef PROT_READ
+#define PROT_READ 1
+#endif
+#ifndef PROT_WRITE
+#define PROT_WRITE 2
+#endif
+#ifndef PROT_READWRITE
+#define PROT_READWRITE 3
+#endif
+#ifndef MAP_SHARED
+#define MAP_SHARED 1
+#endif
+#ifndef MAP_PRIVATE
+#define MAP_PRIVATE 2
+#endif
+#ifndef MAP_FAILED
+#define MAP_FAILED ((void *) -1)
+#endif
+
+#define getcwd _getcwd
+#define open _open
+#define close _close
+
+static void *ucl_mmap(char *addr, size_t length, int prot, int access, int fd, off_t offset)
+{
+ void *map = NULL;
+ HANDLE handle = INVALID_HANDLE_VALUE;
+
+ switch (prot) {
+ default:
+ case PROT_READ:
+ {
+ handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READONLY, 0, length, 0);
+ if (!handle) break;
+ map = (void *) MapViewOfFile(handle, FILE_MAP_READ, 0, 0, length);
+ CloseHandle(handle);
+ break;
+ }
+ case PROT_WRITE:
+ {
+ handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
+ if (!handle) break;
+ map = (void *) MapViewOfFile(handle, FILE_MAP_WRITE, 0, 0, length);
+ CloseHandle(handle);
+ break;
+ }
+ case PROT_READWRITE:
+ {
+ handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
+ if (!handle) break;
+ map = (void *) MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, length);
+ CloseHandle(handle);
+ break;
+ }
+ }
+ if (map == (void *) NULL) {
+ return (void *) MAP_FAILED;
+ }
+ return (void *) ((char *) map + offset);
+}
+
+static int ucl_munmap(void *map,size_t length)
+{
+ if (!UnmapViewOfFile(map)) {
+ return(-1);
+ }
+ return(0);
+}
+
+static char* ucl_realpath(const char *path, char *resolved_path)
+{
+ char *p;
+ char tmp[MAX_PATH + 1];
+ strncpy(tmp, path, sizeof(tmp)-1);
+ p = tmp;
+ while(*p) {
+ if (*p == '/') *p = '\\';
+ p++;
+ }
+ return _fullpath(resolved_path, tmp, MAX_PATH);
+}
+
+
+char *dirname(char *path)
+{
+ static char path_buffer[_MAX_PATH];
+ char drive[_MAX_DRIVE];
+ char dir[_MAX_DIR];
+ char fname[_MAX_FNAME];
+ char ext[_MAX_EXT];
+
+ _splitpath (path, drive, dir, fname, ext);
+ _makepath(path_buffer, drive, dir, NULL, NULL);
+
+ return path_buffer;
+}
+
+char *basename(char *path)
+{
+ static char path_buffer[_MAX_PATH];
+ char drive[_MAX_DRIVE];
+ char dir[_MAX_DIR];
+ char fname[_MAX_FNAME];
+ char ext[_MAX_EXT];
+
+ _splitpath(path, drive, dir, fname, ext);
+ _makepath(path_buffer, NULL, NULL, fname, ext);
+
+ return path_buffer;
+}
+#else
+#define ucl_mmap mmap
+#define ucl_munmap munmap
+#define ucl_realpath realpath
+#endif
+
+typedef void (*ucl_object_dtor) (ucl_object_t *obj);
+static void ucl_object_free_internal (ucl_object_t *obj, bool allow_rec,
+ ucl_object_dtor dtor);
+static void ucl_object_dtor_unref (ucl_object_t *obj);
+
+static void
+ucl_object_dtor_free (ucl_object_t *obj)
+{
+ if (obj->trash_stack[UCL_TRASH_KEY] != NULL) {
+ UCL_FREE (obj->hh.keylen, obj->trash_stack[UCL_TRASH_KEY]);
+ }
+ if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ UCL_FREE (obj->len, obj->trash_stack[UCL_TRASH_VALUE]);
+ }
+ /* Do not free ephemeral objects */
+ if ((obj->flags & UCL_OBJECT_EPHEMERAL) == 0) {
+ if (obj->type != UCL_USERDATA) {
+ UCL_FREE (sizeof (ucl_object_t), obj);
+ }
+ else {
+ struct ucl_object_userdata *ud = (struct ucl_object_userdata *)obj;
+ if (ud->dtor) {
+ ud->dtor (obj->value.ud);
+ }
+ UCL_FREE (sizeof (*ud), obj);
+ }
+ }
+}
+
+/*
+ * This is a helper function that performs exactly the same as
+ * `ucl_object_unref` but it doesn't iterate over elements allowing
+ * to use it for individual elements of arrays and multiple values
+ */
+static void
+ucl_object_dtor_unref_single (ucl_object_t *obj)
+{
+ if (obj != NULL) {
+#ifdef HAVE_ATOMIC_BUILTINS
+ unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
+ if (rc == 0) {
+#else
+ if (--obj->ref == 0) {
+#endif
+ ucl_object_free_internal (obj, false, ucl_object_dtor_unref);
+ }
+ }
+}
+
+static void
+ucl_object_dtor_unref (ucl_object_t *obj)
+{
+ if (obj->ref == 0) {
+ ucl_object_dtor_free (obj);
+ }
+ else {
+ /* This may cause dtor unref being called one more time */
+ ucl_object_dtor_unref_single (obj);
+ }
+}
+
+static void
+ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dtor)
+{
+ ucl_object_t *tmp, *sub;
+
+ while (obj != NULL) {
+ if (obj->type == UCL_ARRAY) {
+ UCL_ARRAY_GET (vec, obj);
+ unsigned int i;
+
+ if (vec != NULL) {
+ for (i = 0; i < vec->n; i ++) {
+ sub = kv_A (*vec, i);
+ if (sub != NULL) {
+ tmp = sub;
+ while (sub) {
+ tmp = sub->next;
+ dtor (sub);
+ sub = tmp;
+ }
+ }
+ }
+ kv_destroy (*vec);
+ UCL_FREE (sizeof (*vec), vec);
+ }
+ obj->value.av = NULL;
+ }
+ else if (obj->type == UCL_OBJECT) {
+ if (obj->value.ov != NULL) {
+ ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func)dtor);
+ }
+ obj->value.ov = NULL;
+ }
+ tmp = obj->next;
+ dtor (obj);
+ obj = tmp;
+
+ if (!allow_rec) {
+ break;
+ }
+ }
+}
+
+void
+ucl_object_free (ucl_object_t *obj)
+{
+ ucl_object_free_internal (obj, true, ucl_object_dtor_free);
+}
+
+size_t
+ucl_unescape_json_string (char *str, size_t len)
+{
+ char *t = str, *h = str;
+ int i, uval;
+
+ if (len <= 1) {
+ return len;
+ }
+ /* t is target (tortoise), h is source (hare) */
+
+ while (len) {
+ if (*h == '\\') {
+ h ++;
+
+ if (len == 1) {
+ /*
+ * If \ is last, then do not try to go further
+ * Issue: #74
+ */
+ len --;
+ *t++ = '\\';
+ continue;
+ }
+
+ switch (*h) {
+ case 'n':
+ *t++ = '\n';
+ break;
+ case 'r':
+ *t++ = '\r';
+ break;
+ case 'b':
+ *t++ = '\b';
+ break;
+ case 't':
+ *t++ = '\t';
+ break;
+ case 'f':
+ *t++ = '\f';
+ break;
+ case '\\':
+ *t++ = '\\';
+ break;
+ case '"':
+ *t++ = '"';
+ break;
+ case 'u':
+ /* Unicode escape */
+ uval = 0;
+ h ++; /* u character */
+ len --;
+
+ if (len > 3) {
+ for (i = 0; i < 4; i++) {
+ uval <<= 4;
+ if (isdigit (h[i])) {
+ uval += h[i] - '0';
+ }
+ else if (h[i] >= 'a' && h[i] <= 'f') {
+ uval += h[i] - 'a' + 10;
+ }
+ else if (h[i] >= 'A' && h[i] <= 'F') {
+ uval += h[i] - 'A' + 10;
+ }
+ else {
+ break;
+ }
+ }
+
+ /* Encode */
+ if(uval < 0x80) {
+ t[0] = (char)uval;
+ t ++;
+ }
+ else if(uval < 0x800) {
+ t[0] = 0xC0 + ((uval & 0x7C0) >> 6);
+ t[1] = 0x80 + ((uval & 0x03F));
+ t += 2;
+ }
+ else if(uval < 0x10000) {
+ t[0] = 0xE0 + ((uval & 0xF000) >> 12);
+ t[1] = 0x80 + ((uval & 0x0FC0) >> 6);
+ t[2] = 0x80 + ((uval & 0x003F));
+ t += 3;
+ }
+#if 0
+ /* It's not actually supported now */
+ else if(uval <= 0x10FFFF) {
+ t[0] = 0xF0 + ((uval & 0x1C0000) >> 18);
+ t[1] = 0x80 + ((uval & 0x03F000) >> 12);
+ t[2] = 0x80 + ((uval & 0x000FC0) >> 6);
+ t[3] = 0x80 + ((uval & 0x00003F));
+ t += 4;
+ }
+#endif
+ else {
+ *t++ = '?';
+ }
+
+ /* Consume 4 characters of source */
+ h += 4;
+ len -= 4;
+
+ if (len > 0) {
+ len --; /* for '\' character */
+ }
+ continue;
+ }
+ else {
+ *t++ = 'u';
+ }
+ break;
+ default:
+ *t++ = *h;
+ break;
+ }
+ h ++;
+ len --;
+ }
+ else {
+ *t++ = *h++;
+ }
+
+ if (len > 0) {
+ len --;
+ }
+ }
+ *t = '\0';
+
+ return (t - str);
+}
+
+size_t
+ucl_unescape_squoted_string (char *str, size_t len)
+{
+ char *t = str, *h = str;
+
+ if (len <= 1) {
+ return len;
+ }
+
+ /* t is target (tortoise), h is source (hare) */
+
+ while (len) {
+ if (*h == '\\') {
+ h ++;
+
+ if (len == 1) {
+ /*
+ * If \ is last, then do not try to go further
+ * Issue: #74
+ */
+ len --;
+ *t++ = '\\';
+ continue;
+ }
+
+ switch (*h) {
+ case '\'':
+ *t++ = '\'';
+ break;
+ case '\n':
+ /* Ignore \<newline> style stuff */
+ break;
+ case '\r':
+ /* Ignore \r and the following \n if needed */
+ if (len > 1 && h[1] == '\n') {
+ h ++;
+ len --;
+ }
+ break;
+ default:
+ /* Ignore \ */
+ *t++ = '\\';
+ *t++ = *h;
+ break;
+ }
+
+ h ++;
+ len --;
+ }
+ else {
+ *t++ = *h++;
+ }
+
+ if (len > 0) {
+ len --;
+ }
+ }
+
+ *t = '\0';
+
+ return (t - str);
+}
+
+char *
+ucl_copy_key_trash (const ucl_object_t *obj)
+{
+ ucl_object_t *deconst;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+ if (obj->trash_stack[UCL_TRASH_KEY] == NULL && obj->key != NULL) {
+ deconst = __DECONST (ucl_object_t *, obj);
+ deconst->trash_stack[UCL_TRASH_KEY] = malloc (obj->keylen + 1);
+ if (deconst->trash_stack[UCL_TRASH_KEY] != NULL) {
+ memcpy (deconst->trash_stack[UCL_TRASH_KEY], obj->key, obj->keylen);
+ deconst->trash_stack[UCL_TRASH_KEY][obj->keylen] = '\0';
+ }
+ deconst->key = obj->trash_stack[UCL_TRASH_KEY];
+ deconst->flags |= UCL_OBJECT_ALLOCATED_KEY;
+ }
+
+ return obj->trash_stack[UCL_TRASH_KEY];
+}
+
+void
+ucl_chunk_free (struct ucl_chunk *chunk)
+{
+ if (chunk) {
+ struct ucl_parser_special_handler_chain *chain, *tmp;
+
+ LL_FOREACH_SAFE (chunk->special_handlers, chain, tmp) {
+ if (chain->special_handler->free_function) {
+ chain->special_handler->free_function (
+ chain->begin,
+ chain->len,
+ chain->special_handler->user_data);
+ } else {
+ UCL_FREE (chain->len, chain->begin);
+ }
+
+ UCL_FREE (sizeof (*chain), chain);
+ }
+
+ chunk->special_handlers = NULL;
+
+ if (chunk->fname) {
+ free (chunk->fname);
+ }
+
+ UCL_FREE (sizeof (*chunk), chunk);
+ }
+}
+
+char *
+ucl_copy_value_trash (const ucl_object_t *obj)
+{
+ ucl_object_t *deconst;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+ if (obj->trash_stack[UCL_TRASH_VALUE] == NULL) {
+ deconst = __DECONST (ucl_object_t *, obj);
+ if (obj->type == UCL_STRING) {
+
+ /* Special case for strings */
+ if (obj->flags & UCL_OBJECT_BINARY) {
+ deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len);
+ if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
+ obj->value.sv,
+ obj->len);
+ deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
+ }
+ }
+ else {
+ deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1);
+ if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
+ obj->value.sv,
+ obj->len);
+ deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0';
+ deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
+ }
+ }
+ }
+ else {
+ /* Just emit value in json notation */
+ deconst->trash_stack[UCL_TRASH_VALUE] = ucl_object_emit_single_json (obj);
+ deconst->len = strlen (obj->trash_stack[UCL_TRASH_VALUE]);
+ }
+ deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE;
+ }
+
+ return obj->trash_stack[UCL_TRASH_VALUE];
+}
+
+ucl_object_t*
+ucl_parser_get_object (struct ucl_parser *parser)
+{
+ if (parser->state != UCL_STATE_ERROR && parser->top_obj != NULL) {
+ return ucl_object_ref (parser->top_obj);
+ }
+
+ return NULL;
+}
+
+void
+ucl_parser_free (struct ucl_parser *parser)
+{
+ struct ucl_stack *stack, *stmp;
+ struct ucl_macro *macro, *mtmp;
+ struct ucl_chunk *chunk, *ctmp;
+ struct ucl_pubkey *key, *ktmp;
+ struct ucl_variable *var, *vtmp;
+ ucl_object_t *tr, *trtmp;
+
+ if (parser == NULL) {
+ return;
+ }
+
+ if (parser->top_obj != NULL) {
+ ucl_object_unref (parser->top_obj);
+ }
+
+ if (parser->includepaths != NULL) {
+ ucl_object_unref (parser->includepaths);
+ }
+
+ LL_FOREACH_SAFE (parser->stack, stack, stmp) {
+ free (stack);
+ }
+ HASH_ITER (hh, parser->macroes, macro, mtmp) {
+ free (macro->name);
+ HASH_DEL (parser->macroes, macro);
+ UCL_FREE (sizeof (struct ucl_macro), macro);
+ }
+ LL_FOREACH_SAFE (parser->chunks, chunk, ctmp) {
+ ucl_chunk_free (chunk);
+ }
+ LL_FOREACH_SAFE (parser->keys, key, ktmp) {
+ UCL_FREE (sizeof (struct ucl_pubkey), key);
+ }
+ LL_FOREACH_SAFE (parser->variables, var, vtmp) {
+ free (var->value);
+ free (var->var);
+ UCL_FREE (sizeof (struct ucl_variable), var);
+ }
+ LL_FOREACH_SAFE (parser->trash_objs, tr, trtmp) {
+ ucl_object_free_internal (tr, false, ucl_object_dtor_free);
+ }
+
+ if (parser->err != NULL) {
+ utstring_free (parser->err);
+ }
+
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+
+ if (parser->comments) {
+ ucl_object_unref (parser->comments);
+ }
+
+ UCL_FREE (sizeof (struct ucl_parser), parser);
+}
+
+const char *
+ucl_parser_get_error(struct ucl_parser *parser)
+{
+ if (parser == NULL) {
+ return NULL;
+ }
+
+ if (parser->err == NULL) {
+ return NULL;
+ }
+
+ return utstring_body (parser->err);
+}
+
+int
+ucl_parser_get_error_code(struct ucl_parser *parser)
+{
+ if (parser == NULL) {
+ return 0;
+ }
+
+ return parser->err_code;
+}
+
+unsigned
+ucl_parser_get_column(struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL) {
+ return 0;
+ }
+
+ return parser->chunks->column;
+}
+
+unsigned
+ucl_parser_get_linenum(struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL) {
+ return 0;
+ }
+
+ return parser->chunks->line;
+}
+
+void
+ucl_parser_clear_error(struct ucl_parser *parser)
+{
+ if (parser != NULL && parser->err != NULL) {
+ utstring_free(parser->err);
+ parser->err = NULL;
+ parser->err_code = 0;
+ }
+}
+
+bool
+ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len)
+{
+#ifndef HAVE_OPENSSL
+ ucl_create_err (&parser->err, "cannot check signatures without openssl");
+ return false;
+#else
+# if (OPENSSL_VERSION_NUMBER < 0x10000000L)
+ ucl_create_err (&parser->err, "cannot check signatures, openssl version is unsupported");
+ return EXIT_FAILURE;
+# else
+ struct ucl_pubkey *nkey;
+ BIO *mem;
+
+ mem = BIO_new_mem_buf ((void *)key, len);
+ nkey = UCL_ALLOC (sizeof (struct ucl_pubkey));
+ if (nkey == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate memory for key");
+ return false;
+ }
+ nkey->key = PEM_read_bio_PUBKEY (mem, &nkey->key, NULL, NULL);
+ BIO_free (mem);
+ if (nkey->key == NULL) {
+ UCL_FREE (sizeof (struct ucl_pubkey), nkey);
+ ucl_create_err (&parser->err, "%s",
+ ERR_error_string (ERR_get_error (), NULL));
+ return false;
+ }
+ LL_PREPEND (parser->keys, nkey);
+# endif
+#endif
+ return true;
+}
+
+void ucl_parser_add_special_handler (struct ucl_parser *parser,
+ struct ucl_parser_special_handler *handler)
+{
+ LL_APPEND (parser->special_handlers, handler);
+}
+
+#ifdef CURL_FOUND
+struct ucl_curl_cbdata {
+ unsigned char *buf;
+ size_t buflen;
+};
+
+static size_t
+ucl_curl_write_callback (void* contents, size_t size, size_t nmemb, void* ud)
+{
+ struct ucl_curl_cbdata *cbdata = ud;
+ size_t realsize = size * nmemb;
+
+ cbdata->buf = realloc (cbdata->buf, cbdata->buflen + realsize + 1);
+ if (cbdata->buf == NULL) {
+ return 0;
+ }
+
+ memcpy (&(cbdata->buf[cbdata->buflen]), contents, realsize);
+ cbdata->buflen += realsize;
+ cbdata->buf[cbdata->buflen] = 0;
+
+ return realsize;
+}
+#endif
+
+/**
+ * Fetch a url and save results to the memory buffer
+ * @param url url to fetch
+ * @param len length of url
+ * @param buf target buffer
+ * @param buflen target length
+ * @return
+ */
+bool
+ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen,
+ UT_string **err, bool must_exist)
+{
+
+#ifdef HAVE_FETCH_H
+ struct url *fetch_url;
+ struct url_stat us;
+ FILE *in;
+
+ fetch_url = fetchParseURL (url);
+ if (fetch_url == NULL) {
+ ucl_create_err (err, "invalid URL %s: %s",
+ url, strerror (errno));
+ return false;
+ }
+ if ((in = fetchXGet (fetch_url, &us, "")) == NULL) {
+ if (!must_exist) {
+ ucl_create_err (err, "cannot fetch URL %s: %s",
+ url, strerror (errno));
+ }
+ fetchFreeURL (fetch_url);
+ return false;
+ }
+
+ *buflen = us.size;
+ *buf = malloc (*buflen);
+ if (*buf == NULL) {
+ ucl_create_err (err, "cannot allocate buffer for URL %s: %s",
+ url, strerror (errno));
+ fclose (in);
+ fetchFreeURL (fetch_url);
+ return false;
+ }
+
+ if (fread (*buf, *buflen, 1, in) != 1) {
+ ucl_create_err (err, "cannot read URL %s: %s",
+ url, strerror (errno));
+ fclose (in);
+ fetchFreeURL (fetch_url);
+ return false;
+ }
+
+ fetchFreeURL (fetch_url);
+ return true;
+#elif defined(CURL_FOUND)
+ CURL *curl;
+ int r;
+ struct ucl_curl_cbdata cbdata;
+
+ curl = curl_easy_init ();
+ if (curl == NULL) {
+ ucl_create_err (err, "CURL interface is broken");
+ return false;
+ }
+ if ((r = curl_easy_setopt (curl, CURLOPT_URL, url)) != CURLE_OK) {
+ ucl_create_err (err, "invalid URL %s: %s",
+ url, curl_easy_strerror (r));
+ curl_easy_cleanup (curl);
+ return false;
+ }
+ curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ucl_curl_write_callback);
+ cbdata.buf = NULL;
+ cbdata.buflen = 0;
+ curl_easy_setopt (curl, CURLOPT_WRITEDATA, &cbdata);
+
+ if ((r = curl_easy_perform (curl)) != CURLE_OK) {
+ if (!must_exist) {
+ ucl_create_err (err, "error fetching URL %s: %s",
+ url, curl_easy_strerror (r));
+ }
+ curl_easy_cleanup (curl);
+ if (cbdata.buf) {
+ free (cbdata.buf);
+ }
+ return false;
+ }
+ *buf = cbdata.buf;
+ *buflen = cbdata.buflen;
+
+ return true;
+#else
+ ucl_create_err (err, "URL support is disabled");
+ return false;
+#endif
+}
+
+/**
+ * Fetch a file and save results to the memory buffer
+ * @param filename filename to fetch
+ * @param len length of filename
+ * @param buf target buffer
+ * @param buflen target length
+ * @return
+ */
+bool
+ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *buflen,
+ UT_string **err, bool must_exist)
+{
+ int fd;
+ struct stat st;
+ if ((fd = open (filename, O_RDONLY)) == -1) {
+ ucl_create_err (err, "cannot open file %s: %s",
+ filename, strerror (errno));
+ return false;
+ }
+
+ if (fstat (fd, &st) == -1) {
+ if (must_exist || errno == EPERM) {
+ ucl_create_err (err, "cannot stat file %s: %s",
+ filename, strerror (errno));
+ }
+ close (fd);
+
+ return false;
+ }
+ if (!S_ISREG (st.st_mode)) {
+ if (must_exist) {
+ ucl_create_err (err, "file %s is not a regular file", filename);
+ }
+ close (fd);
+
+ return false;
+ }
+
+ if (st.st_size == 0) {
+ /* Do not map empty files */
+ *buf = NULL;
+ *buflen = 0;
+ }
+ else {
+ if ((*buf = ucl_mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ close(fd);
+ ucl_create_err(err, "cannot mmap file %s: %s",
+ filename, strerror(errno));
+ *buf = NULL;
+
+ return false;
+ }
+ *buflen = st.st_size;
+ }
+
+ close (fd);
+
+ return true;
+}
+
+
+#if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
+static inline bool
+ucl_sig_check (const unsigned char *data, size_t datalen,
+ const unsigned char *sig, size_t siglen, struct ucl_parser *parser)
+{
+ struct ucl_pubkey *key;
+ char dig[EVP_MAX_MD_SIZE];
+ unsigned int diglen;
+ EVP_PKEY_CTX *key_ctx;
+ EVP_MD_CTX *sign_ctx = NULL;
+
+ sign_ctx = EVP_MD_CTX_create ();
+
+ LL_FOREACH (parser->keys, key) {
+ key_ctx = EVP_PKEY_CTX_new (key->key, NULL);
+ if (key_ctx != NULL) {
+ if (EVP_PKEY_verify_init (key_ctx) <= 0) {
+ EVP_PKEY_CTX_free (key_ctx);
+ continue;
+ }
+ if (EVP_PKEY_CTX_set_rsa_padding (key_ctx, RSA_PKCS1_PADDING) <= 0) {
+ EVP_PKEY_CTX_free (key_ctx);
+ continue;
+ }
+ if (EVP_PKEY_CTX_set_signature_md (key_ctx, EVP_sha256 ()) <= 0) {
+ EVP_PKEY_CTX_free (key_ctx);
+ continue;
+ }
+ EVP_DigestInit (sign_ctx, EVP_sha256 ());
+ EVP_DigestUpdate (sign_ctx, data, datalen);
+ EVP_DigestFinal (sign_ctx, dig, &diglen);
+
+ if (EVP_PKEY_verify (key_ctx, sig, siglen, dig, diglen) == 1) {
+ EVP_MD_CTX_destroy (sign_ctx);
+ EVP_PKEY_CTX_free (key_ctx);
+ return true;
+ }
+
+ EVP_PKEY_CTX_free (key_ctx);
+ }
+ }
+
+ EVP_MD_CTX_destroy (sign_ctx);
+
+ return false;
+}
+#endif
+
+struct ucl_include_params {
+ bool check_signature;
+ bool must_exist;
+ bool use_glob;
+ bool use_prefix;
+ bool soft_fail;
+ bool allow_glob;
+ unsigned priority;
+ enum ucl_duplicate_strategy strat;
+ enum ucl_parse_type parse_type;
+ const char *prefix;
+ const char *target;
+};
+
+/**
+ * Include an url to configuration
+ * @param data
+ * @param len
+ * @param parser
+ * @param err
+ * @return
+ */
+static bool
+ucl_include_url (const unsigned char *data, size_t len,
+ struct ucl_parser *parser,
+ struct ucl_include_params *params)
+{
+
+ bool res;
+ unsigned char *buf = NULL;
+ size_t buflen = 0;
+ struct ucl_chunk *chunk;
+ char urlbuf[PATH_MAX];
+ int prev_state;
+
+ snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data);
+
+ if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) {
+ return !params->must_exist;
+ }
+
+ if (params->check_signature) {
+#if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
+ unsigned char *sigbuf = NULL;
+ size_t siglen = 0;
+ /* We need to check signature first */
+ snprintf (urlbuf, sizeof (urlbuf), "%.*s.sig", (int)len, data);
+ if (!ucl_fetch_url (urlbuf, &sigbuf, &siglen, &parser->err, true)) {
+ return false;
+ }
+ if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
+ ucl_create_err (&parser->err, "cannot verify url %s: %s",
+ urlbuf,
+ ERR_error_string (ERR_get_error (), NULL));
+ if (siglen > 0) {
+ ucl_munmap (sigbuf, siglen);
+ }
+ return false;
+ }
+ if (siglen > 0) {
+ ucl_munmap (sigbuf, siglen);
+ }
+#endif
+ }
+
+ prev_state = parser->state;
+ parser->state = UCL_STATE_INIT;
+
+ res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
+ params->strat, params->parse_type);
+ if (res == true) {
+ /* Remove chunk from the stack */
+ chunk = parser->chunks;
+ if (chunk != NULL) {
+ parser->chunks = chunk->next;
+ ucl_chunk_free (chunk);
+ }
+ }
+
+ parser->state = prev_state;
+ free (buf);
+
+ return res;
+}
+
+/**
+ * Include a single file to the parser
+ * @param data
+ * @param len
+ * @param parser
+ * @param check_signature
+ * @param must_exist
+ * @param allow_glob
+ * @param priority
+ * @return
+ */
+static bool
+ucl_include_file_single (const unsigned char *data, size_t len,
+ struct ucl_parser *parser, struct ucl_include_params *params)
+{
+ bool res;
+ struct ucl_chunk *chunk;
+ unsigned char *buf = NULL;
+ char *old_curfile, *ext;
+ size_t buflen = 0;
+ char filebuf[PATH_MAX], realbuf[PATH_MAX];
+ int prev_state;
+ struct ucl_variable *cur_var, *tmp_var, *old_curdir = NULL,
+ *old_filename = NULL;
+ ucl_object_t *nest_obj = NULL, *old_obj = NULL, *new_obj = NULL;
+ ucl_hash_t *container = NULL;
+ struct ucl_stack *st = NULL;
+
+ snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data);
+ if (ucl_realpath (filebuf, realbuf) == NULL) {
+ if (params->soft_fail) {
+ return false;
+ }
+ if (!params->must_exist && errno != EPERM) {
+ return true;
+ }
+
+ ucl_create_err (&parser->err, "cannot open file %s: %s",
+ filebuf,
+ strerror (errno));
+ return false;
+ }
+
+ if (parser->cur_file && strcmp (realbuf, parser->cur_file) == 0) {
+ /* We are likely including the file itself */
+ if (params->soft_fail) {
+ return false;
+ }
+
+ ucl_create_err (&parser->err, "trying to include the file %s from itself",
+ realbuf);
+ return false;
+ }
+
+ if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, params->must_exist)) {
+ if (params->soft_fail) {
+ return false;
+ }
+
+ if (params->must_exist || parser->err != NULL) {
+ /* The case of fatal errors */
+ return false;
+ }
+
+ return true;
+ }
+
+ if (params->check_signature) {
+#if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
+ unsigned char *sigbuf = NULL;
+ size_t siglen = 0;
+ /* We need to check signature first */
+ snprintf (filebuf, sizeof (filebuf), "%s.sig", realbuf);
+ if (!ucl_fetch_file (filebuf, &sigbuf, &siglen, &parser->err, true)) {
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
+ ucl_create_err (&parser->err, "cannot verify file %s: %s",
+ filebuf,
+ ERR_error_string (ERR_get_error (), NULL));
+ if (sigbuf) {
+ ucl_munmap (sigbuf, siglen);
+ }
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ if (sigbuf) {
+ ucl_munmap (sigbuf, siglen);
+ }
+#endif
+ }
+
+ old_curfile = parser->cur_file;
+ parser->cur_file = NULL;
+
+ /* Store old file vars */
+ DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
+ if (strcmp (cur_var->var, "CURDIR") == 0) {
+ old_curdir = cur_var;
+ DL_DELETE (parser->variables, cur_var);
+ }
+ else if (strcmp (cur_var->var, "FILENAME") == 0) {
+ old_filename = cur_var;
+ DL_DELETE (parser->variables, cur_var);
+ }
+ }
+
+ ucl_parser_set_filevars (parser, realbuf, false);
+
+ prev_state = parser->state;
+ parser->state = UCL_STATE_INIT;
+
+ if (params->use_prefix && params->prefix == NULL) {
+ /* Auto generate a key name based on the included filename */
+ params->prefix = basename (realbuf);
+ ext = strrchr (params->prefix, '.');
+ if (ext != NULL && (strcmp (ext, ".conf") == 0 || strcmp (ext, ".ucl") == 0)) {
+ /* Strip off .conf or .ucl */
+ *ext = '\0';
+ }
+ }
+ if (params->prefix != NULL) {
+ /* This is a prefixed include */
+ container = parser->stack->obj->value.ov;
+
+ old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container,
+ params->prefix, strlen (params->prefix)));
+
+ if (strcasecmp (params->target, "array") == 0) {
+ if (old_obj == NULL) {
+ /* Create an array with key: prefix */
+ old_obj = ucl_object_new_full (UCL_ARRAY, params->priority);
+ old_obj->key = params->prefix;
+ old_obj->keylen = strlen (params->prefix);
+ ucl_copy_key_trash (old_obj);
+ old_obj->prev = old_obj;
+ old_obj->next = NULL;
+
+ container = ucl_hash_insert_object (container, old_obj,
+ parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ parser->stack->obj->len++;
+
+ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ ucl_array_append (old_obj, nest_obj);
+ }
+ else {
+ if (ucl_object_type (old_obj) == UCL_ARRAY) {
+ /* Append to the existing array */
+ nest_obj = ucl_object_new_full (UCL_OBJECT,
+ params->priority);
+ if (nest_obj == NULL) {
+ ucl_create_err (&parser->err,
+ "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ ucl_array_append (old_obj, nest_obj);
+ }
+ else {
+ /* Convert the object to an array */
+ new_obj = ucl_object_typed_new (UCL_ARRAY);
+ if (new_obj == NULL) {
+ ucl_create_err (&parser->err,
+ "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ new_obj->key = old_obj->key;
+ new_obj->keylen = old_obj->keylen;
+ new_obj->flags |= UCL_OBJECT_MULTIVALUE;
+ new_obj->prev = new_obj;
+ new_obj->next = NULL;
+
+ nest_obj = ucl_object_new_full (UCL_OBJECT,
+ params->priority);
+ if (nest_obj == NULL) {
+ ucl_create_err (&parser->err,
+ "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ ucl_object_unref (new_obj);
+
+ return false;
+ }
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ ucl_array_append (new_obj, old_obj);
+ ucl_array_append (new_obj, nest_obj);
+ ucl_hash_replace (container, old_obj, new_obj);
+ }
+ }
+ }
+ else {
+ /* Case of object */
+ if (old_obj == NULL) {
+ /* Create an object with key: prefix */
+ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
+
+ if (nest_obj == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ nest_obj->key = params->prefix;
+ nest_obj->keylen = strlen (params->prefix);
+ ucl_copy_key_trash(nest_obj);
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ container = ucl_hash_insert_object (container, nest_obj,
+ parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ parser->stack->obj->len ++;
+ }
+ else {
+ if (ucl_object_type (old_obj) == UCL_OBJECT) {
+ /* Append to existing Object*/
+ nest_obj = old_obj;
+ }
+ else {
+ /* The key is not an object */
+ ucl_create_err (&parser->err,
+ "Conflicting type for key: %s, asked %s, has %s",
+ params->prefix, params->target,
+ ucl_object_type_to_string (ucl_object_type (old_obj)));
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ }
+ }
+
+
+ /* Put all of the content of the include inside that object */
+ parser->stack->obj->value.ov = container;
+
+ st = UCL_ALLOC (sizeof (struct ucl_stack));
+ if (st == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate memory for an object");
+ ucl_object_unref (nest_obj);
+
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ st->obj = nest_obj;
+ st->e.params.level = parser->stack->e.params.level;
+ st->e.params.flags = parser->stack->e.params.flags;
+ st->e.params.line = parser->stack->e.params.line;
+ st->chunk = parser->chunks;
+ LL_PREPEND (parser->stack, st);
+ parser->cur_obj = nest_obj;
+ }
+
+ res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
+ params->strat, params->parse_type);
+
+ if (res) {
+ /* Stop nesting the include, take 1 level off the stack */
+ if (params->prefix != NULL && nest_obj != NULL) {
+ parser->stack = st->next;
+ UCL_FREE (sizeof (struct ucl_stack), st);
+ }
+
+ /* Remove chunk from the stack */
+ chunk = parser->chunks;
+ if (chunk != NULL) {
+ parser->chunks = chunk->next;
+ ucl_chunk_free (chunk);
+ parser->recursion--;
+ }
+
+ /* Restore old file vars */
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+
+ parser->cur_file = old_curfile;
+ DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
+ if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) {
+ DL_DELETE (parser->variables, cur_var);
+ free (cur_var->var);
+ free (cur_var->value);
+ UCL_FREE (sizeof (struct ucl_variable), cur_var);
+ } else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) {
+ DL_DELETE (parser->variables, cur_var);
+ free (cur_var->var);
+ free (cur_var->value);
+ UCL_FREE (sizeof (struct ucl_variable), cur_var);
+ }
+ }
+ if (old_filename) {
+ DL_APPEND (parser->variables, old_filename);
+ }
+ if (old_curdir) {
+ DL_APPEND (parser->variables, old_curdir);
+ }
+
+ parser->state = prev_state;
+ }
+
+ if (buflen > 0) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return res;
+}
+
+/**
+ * Include a file to configuration
+ * @param data
+ * @param len
+ * @param parser
+ * @param err
+ * @return
+ */
+static bool
+ucl_include_file (const unsigned char *data, size_t len,
+ struct ucl_parser *parser,
+ struct ucl_include_params *params,
+ const ucl_object_t *args)
+{
+ const unsigned char *p = data, *end = data + len;
+ bool need_glob = false;
+ int cnt = 0;
+ char glob_pattern[PATH_MAX];
+ size_t i;
+
+#ifndef _WIN32
+ if (!params->allow_glob) {
+ return ucl_include_file_single (data, len, parser, params);
+ }
+ else {
+ /* Check for special symbols in a filename */
+ while (p != end) {
+ if (*p == '*' || *p == '?') {
+ need_glob = true;
+ break;
+ }
+ p ++;
+ }
+ if (need_glob) {
+ glob_t globbuf;
+ memset (&globbuf, 0, sizeof (globbuf));
+ ucl_strlcpy (glob_pattern, (const char *)data,
+ (len + 1 < sizeof (glob_pattern) ? len + 1 : sizeof (glob_pattern)));
+ if (glob (glob_pattern, 0, NULL, &globbuf) != 0) {
+ return (!params->must_exist || false);
+ }
+ for (i = 0; i < globbuf.gl_pathc; i ++) {
+
+ if (parser->include_trace_func) {
+ const ucl_object_t *parent = NULL;
+
+ if (parser->stack) {
+ parent = parser->stack->obj;
+ }
+
+ parser->include_trace_func (parser, parent, NULL,
+ globbuf.gl_pathv[i],
+ strlen (globbuf.gl_pathv[i]),
+ parser->include_trace_ud);
+ }
+
+ if (!ucl_include_file_single ((unsigned char *)globbuf.gl_pathv[i],
+ strlen (globbuf.gl_pathv[i]), parser, params)) {
+ if (params->soft_fail) {
+ continue;
+ }
+ globfree (&globbuf);
+ return false;
+ }
+ cnt ++;
+ }
+ globfree (&globbuf);
+
+ if (cnt == 0 && params->must_exist) {
+ ucl_create_err (&parser->err, "cannot match any files for pattern %s",
+ glob_pattern);
+ return false;
+ }
+ }
+ else {
+ return ucl_include_file_single (data, len, parser, params);
+ }
+ }
+#else
+ /* Win32 compilers do not support globbing. Therefore, for Win32,
+ treat allow_glob/need_glob as a NOOP and just return */
+ return ucl_include_file_single (data, len, parser, params);
+#endif
+
+ return true;
+}
+
+/**
+ * Common function to handle .*include* macros
+ * @param data
+ * @param len
+ * @param args
+ * @param parser
+ * @param default_try
+ * @param default_sign
+ * @return
+ */
+static bool
+ucl_include_common (const unsigned char *data, size_t len,
+ const ucl_object_t *args, struct ucl_parser *parser,
+ bool default_try,
+ bool default_sign)
+{
+ bool allow_url = false, search = false;
+ const char *duplicate;
+ const ucl_object_t *param;
+ ucl_object_iter_t it = NULL, ip = NULL;
+ char ipath[PATH_MAX];
+ struct ucl_include_params params;
+
+ /* Default values */
+ params.soft_fail = default_try;
+ params.allow_glob = false;
+ params.check_signature = default_sign;
+ params.use_prefix = false;
+ params.target = "object";
+ params.prefix = NULL;
+ params.priority = 0;
+ params.parse_type = UCL_PARSE_UCL;
+ params.strat = UCL_DUPLICATE_APPEND;
+ params.must_exist = !default_try;
+
+ if (parser->include_trace_func) {
+ const ucl_object_t *parent = NULL;
+
+ if (parser->stack) {
+ parent = parser->stack->obj;
+ }
+
+ parser->include_trace_func (parser, parent, args,
+ data, len, parser->include_trace_ud);
+ }
+
+ /* Process arguments */
+ if (args != NULL && args->type == UCL_OBJECT) {
+ while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
+ if (param->type == UCL_BOOLEAN) {
+ if (strncmp (param->key, "try", param->keylen) == 0) {
+ params.must_exist = !ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "sign", param->keylen) == 0) {
+ params.check_signature = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "glob", param->keylen) == 0) {
+ params.allow_glob = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "url", param->keylen) == 0) {
+ allow_url = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "prefix", param->keylen) == 0) {
+ params.use_prefix = ucl_object_toboolean (param);
+ }
+ }
+ else if (param->type == UCL_STRING) {
+ if (strncmp (param->key, "key", param->keylen) == 0) {
+ params.prefix = ucl_object_tostring (param);
+ }
+ else if (strncmp (param->key, "target", param->keylen) == 0) {
+ params.target = ucl_object_tostring (param);
+ }
+ else if (strncmp (param->key, "duplicate", param->keylen) == 0) {
+ duplicate = ucl_object_tostring (param);
+
+ if (strcmp (duplicate, "append") == 0) {
+ params.strat = UCL_DUPLICATE_APPEND;
+ }
+ else if (strcmp (duplicate, "merge") == 0) {
+ params.strat = UCL_DUPLICATE_MERGE;
+ }
+ else if (strcmp (duplicate, "rewrite") == 0) {
+ params.strat = UCL_DUPLICATE_REWRITE;
+ }
+ else if (strcmp (duplicate, "error") == 0) {
+ params.strat = UCL_DUPLICATE_ERROR;
+ }
+ }
+ }
+ else if (param->type == UCL_ARRAY) {
+ if (strncmp (param->key, "path", param->keylen) == 0) {
+ ucl_set_include_path (parser, __DECONST(ucl_object_t *, param));
+ }
+ }
+ else if (param->type == UCL_INT) {
+ if (strncmp (param->key, "priority", param->keylen) == 0) {
+ params.priority = ucl_object_toint (param);
+ }
+ }
+ }
+ }
+
+ if (parser->includepaths == NULL) {
+ if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
+ /* Globbing is not used for URL's */
+ return ucl_include_url (data, len, parser, &params);
+ }
+ else if (data != NULL) {
+ /* Try to load a file */
+ return ucl_include_file (data, len, parser, &params, args);
+ }
+ }
+ else {
+ if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
+ /* Globbing is not used for URL's */
+ return ucl_include_url (data, len, parser, &params);
+ }
+
+ ip = ucl_object_iterate_new (parser->includepaths);
+ while ((param = ucl_object_iterate_safe (ip, true)) != NULL) {
+ if (ucl_object_type(param) == UCL_STRING) {
+ snprintf (ipath, sizeof (ipath), "%s/%.*s", ucl_object_tostring(param),
+ (int)len, data);
+ if ((search = ucl_include_file (ipath, strlen (ipath),
+ parser, &params, args))) {
+ if (!params.allow_glob) {
+ break;
+ }
+ }
+ }
+ }
+ ucl_object_iterate_free (ip);
+ if (search == true) {
+ return true;
+ }
+ else {
+ ucl_create_err (&parser->err,
+ "cannot find file: %.*s in search path",
+ (int)len, data);
+ return false;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * Handle include macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+
+ return ucl_include_common (data, len, args, parser, false, false);
+}
+
+/**
+ * Handle includes macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_includes_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+
+ return ucl_include_common (data, len, args, parser, false, true);
+}
+
+/**
+ * Handle tryinclude macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_try_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+
+ return ucl_include_common (data, len, args, parser, true, false);
+}
+
+/**
+ * Handle priority macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_priority_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+ unsigned priority = 255;
+ const ucl_object_t *param;
+ bool found = false;
+ char *value = NULL, *leftover = NULL;
+ ucl_object_iter_t it = NULL;
+
+ if (parser == NULL) {
+ return false;
+ }
+
+ /* Process arguments */
+ if (args != NULL && args->type == UCL_OBJECT) {
+ while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
+ if (param->type == UCL_INT) {
+ if (strncmp (param->key, "priority", param->keylen) == 0) {
+ priority = ucl_object_toint (param);
+ found = true;
+ }
+ }
+ }
+ }
+
+ if (len > 0) {
+ value = malloc(len + 1);
+ ucl_strlcpy(value, (const char *)data, len + 1);
+ priority = strtol(value, &leftover, 10);
+ if (*leftover != '\0') {
+ ucl_create_err (&parser->err, "Invalid priority value in macro: %s",
+ value);
+ free(value);
+ return false;
+ }
+ free(value);
+ found = true;
+ }
+
+ if (found == true) {
+ parser->chunks->priority = priority;
+ return true;
+ }
+
+ ucl_create_err (&parser->err, "Unable to parse priority macro");
+ return false;
+}
+
+/**
+ * Handle load macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_load_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+ const ucl_object_t *param;
+ ucl_object_t *obj, *old_obj;
+ ucl_object_iter_t it = NULL;
+ bool try_load, multiline, test;
+ const char *target, *prefix;
+ char *load_file, *tmp;
+ unsigned char *buf;
+ size_t buflen;
+ unsigned priority;
+ int64_t iv;
+ ucl_object_t *container = NULL;
+ enum ucl_string_flags flags;
+
+ /* Default values */
+ try_load = false;
+ multiline = false;
+ test = false;
+ target = "string";
+ prefix = NULL;
+ load_file = NULL;
+ buf = NULL;
+ buflen = 0;
+ priority = 0;
+ obj = NULL;
+ old_obj = NULL;
+ flags = 0;
+
+ if (parser == NULL) {
+ return false;
+ }
+
+ /* Process arguments */
+ if (args != NULL && args->type == UCL_OBJECT) {
+ while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
+ if (param->type == UCL_BOOLEAN) {
+ if (strncmp (param->key, "try", param->keylen) == 0) {
+ try_load = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "multiline", param->keylen) == 0) {
+ multiline = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "escape", param->keylen) == 0) {
+ test = ucl_object_toboolean (param);
+ if (test) {
+ flags |= UCL_STRING_ESCAPE;
+ }
+ }
+ else if (strncmp (param->key, "trim", param->keylen) == 0) {
+ test = ucl_object_toboolean (param);
+ if (test) {
+ flags |= UCL_STRING_TRIM;
+ }
+ }
+ }
+ else if (param->type == UCL_STRING) {
+ if (strncmp (param->key, "key", param->keylen) == 0) {
+ prefix = ucl_object_tostring (param);
+ }
+ else if (strncmp (param->key, "target", param->keylen) == 0) {
+ target = ucl_object_tostring (param);
+ }
+ }
+ else if (param->type == UCL_INT) {
+ if (strncmp (param->key, "priority", param->keylen) == 0) {
+ priority = ucl_object_toint (param);
+ }
+ }
+ }
+ }
+
+ if (prefix == NULL || strlen (prefix) == 0) {
+ ucl_create_err (&parser->err, "No Key specified in load macro");
+ return false;
+ }
+
+ if (len > 0) {
+ load_file = malloc (len + 1);
+ if (!load_file) {
+ ucl_create_err (&parser->err, "cannot allocate memory for suffix");
+
+ return false;
+ }
+
+ snprintf (load_file, len + 1, "%.*s", (int)len, data);
+
+ if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err,
+ !try_load)) {
+ free (load_file);
+
+ return (try_load || false);
+ }
+
+ free (load_file);
+ container = parser->stack->obj;
+ old_obj = __DECONST (ucl_object_t *, ucl_object_lookup (container,
+ prefix));
+
+ if (old_obj != NULL) {
+ ucl_create_err (&parser->err, "Key %s already exists", prefix);
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ if (strcasecmp (target, "string") == 0) {
+ obj = ucl_object_fromstring_common (buf, buflen, flags);
+ ucl_copy_value_trash (obj);
+ if (multiline) {
+ obj->flags |= UCL_OBJECT_MULTILINE;
+ }
+ }
+ else if (strcasecmp (target, "int") == 0) {
+ tmp = malloc (buflen + 1);
+
+ if (tmp == NULL) {
+ ucl_create_err (&parser->err, "Memory allocation failed");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ snprintf (tmp, buflen + 1, "%.*s", (int)buflen, buf);
+ iv = strtoll (tmp, NULL, 10);
+ obj = ucl_object_fromint (iv);
+ free (tmp);
+ }
+
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ if (obj != NULL) {
+ obj->key = prefix;
+ obj->keylen = strlen (prefix);
+ ucl_copy_key_trash (obj);
+ obj->prev = obj;
+ obj->next = NULL;
+ ucl_object_set_priority (obj, priority);
+ ucl_object_insert_key (container, obj, obj->key, obj->keylen, false);
+ }
+
+ return true;
+ }
+
+ ucl_create_err (&parser->err, "Unable to parse load macro");
+ return false;
+}
+
+bool
+ucl_inherit_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, const ucl_object_t *ctx, void* ud)
+{
+ const ucl_object_t *parent, *cur;
+ ucl_object_t *target, *copy;
+ ucl_object_iter_t it = NULL;
+ bool replace = false;
+ struct ucl_parser *parser = ud;
+
+ parent = ucl_object_lookup_len (ctx, data, len);
+
+ /* Some sanity checks */
+ if (parent == NULL || ucl_object_type (parent) != UCL_OBJECT) {
+ ucl_create_err (&parser->err, "Unable to find inherited object %.*s",
+ (int)len, data);
+ return false;
+ }
+
+ if (parser->stack == NULL || parser->stack->obj == NULL ||
+ ucl_object_type (parser->stack->obj) != UCL_OBJECT) {
+ ucl_create_err (&parser->err, "Invalid inherit context");
+ return false;
+ }
+
+ target = parser->stack->obj;
+
+ if (args && (cur = ucl_object_lookup (args, "replace")) != NULL) {
+ replace = ucl_object_toboolean (cur);
+ }
+
+ while ((cur = ucl_object_iterate (parent, &it, true))) {
+ /* We do not replace existing keys */
+ if (!replace && ucl_object_lookup_len (target, cur->key, cur->keylen)) {
+ continue;
+ }
+
+ copy = ucl_object_copy (cur);
+
+ if (!replace) {
+ copy->flags |= UCL_OBJECT_INHERITED;
+ }
+
+ ucl_object_insert_key (target, copy, copy->key,
+ copy->keylen, false);
+ }
+
+ return true;
+}
+
+bool
+ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand)
+{
+ char realbuf[PATH_MAX], *curdir;
+
+ if (filename != NULL) {
+ if (need_expand) {
+ if (ucl_realpath (filename, realbuf) == NULL) {
+ return false;
+ }
+ }
+ else {
+ ucl_strlcpy (realbuf, filename, sizeof (realbuf));
+ }
+
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+
+ parser->cur_file = strdup (realbuf);
+
+ /* Define variables */
+ ucl_parser_register_variable (parser, "FILENAME", realbuf);
+ curdir = dirname (realbuf);
+ ucl_parser_register_variable (parser, "CURDIR", curdir);
+ }
+ else {
+ /* Set everything from the current dir */
+ curdir = getcwd (realbuf, sizeof (realbuf));
+ ucl_parser_register_variable (parser, "FILENAME", "undef");
+ ucl_parser_register_variable (parser, "CURDIR", curdir);
+ }
+
+ return true;
+}
+
+bool
+ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type)
+{
+ unsigned char *buf;
+ size_t len;
+ bool ret;
+ char realbuf[PATH_MAX];
+
+ if (ucl_realpath (filename, realbuf) == NULL) {
+ ucl_create_err (&parser->err, "cannot open file %s: %s",
+ filename,
+ strerror (errno));
+ return false;
+ }
+
+ if (!ucl_fetch_file (realbuf, &buf, &len, &parser->err, true)) {
+ return false;
+ }
+
+ ucl_parser_set_filevars (parser, realbuf, false);
+ ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat,
+ parse_type);
+
+ if (len > 0) {
+ ucl_munmap (buf, len);
+ }
+
+ return ret;
+}
+
+bool
+ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename,
+ unsigned priority)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_file_full(parser, filename, priority,
+ UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_add_file (struct ucl_parser *parser, const char *filename)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_file_full(parser, filename,
+ parser->default_priority, UCL_DUPLICATE_APPEND,
+ UCL_PARSE_UCL);
+}
+
+
+bool
+ucl_parser_add_fd_full (struct ucl_parser *parser, int fd,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type)
+{
+ unsigned char *buf;
+ size_t len;
+ bool ret;
+ struct stat st;
+
+ if (fstat (fd, &st) == -1) {
+ ucl_create_err (&parser->err, "cannot stat fd %d: %s",
+ fd, strerror (errno));
+ return false;
+ }
+ if (st.st_size == 0) {
+ return true;
+ }
+ if ((buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ ucl_create_err (&parser->err, "cannot mmap fd %d: %s",
+ fd, strerror (errno));
+ return false;
+ }
+
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+ parser->cur_file = NULL;
+ len = st.st_size;
+ ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat,
+ parse_type);
+
+ if (len > 0) {
+ ucl_munmap (buf, len);
+ }
+
+ return ret;
+}
+
+bool
+ucl_parser_add_fd_priority (struct ucl_parser *parser, int fd,
+ unsigned priority)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_fd_full(parser, fd, parser->default_priority,
+ UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_add_fd (struct ucl_parser *parser, int fd)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_fd_priority(parser, fd, parser->default_priority);
+}
+
+size_t
+ucl_strlcpy (char *dst, const char *src, size_t siz)
+{
+ char *d = dst;
+ const char *s = src;
+ size_t n = siz;
+
+ /* Copy as many bytes as will fit */
+ if (n != 0) {
+ while (--n != 0) {
+ if ((*d++ = *s++) == '\0') {
+ break;
+ }
+ }
+ }
+
+ if (n == 0 && siz != 0) {
+ *d = '\0';
+ }
+
+ return (s - src - 1); /* count does not include NUL */
+}
+
+size_t
+ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz)
+{
+ memcpy (dst, src, siz - 1);
+ dst[siz - 1] = '\0';
+
+ return siz - 1;
+}
+
+size_t
+ucl_strlcpy_tolower (char *dst, const char *src, size_t siz)
+{
+ char *d = dst;
+ const char *s = src;
+ size_t n = siz;
+
+ /* Copy as many bytes as will fit */
+ if (n != 0) {
+ while (--n != 0) {
+ if ((*d++ = tolower (*s++)) == '\0') {
+ break;
+ }
+ }
+ }
+
+ if (n == 0 && siz != 0) {
+ *d = '\0';
+ }
+
+ return (s - src); /* count does not include NUL */
+}
+
+/*
+ * Find the first occurrence of find in s
+ */
+char *
+ucl_strnstr (const char *s, const char *find, int len)
+{
+ char c, sc;
+ int mlen;
+
+ if ((c = *find++) != 0) {
+ mlen = strlen (find);
+ do {
+ do {
+ if ((sc = *s++) == 0 || len-- < mlen)
+ return (NULL);
+ } while (sc != c);
+ } while (strncmp (s, find, mlen) != 0);
+ s--;
+ }
+ return ((char *)s);
+}
+
+/*
+ * Find the first occurrence of find in s, ignore case.
+ */
+char *
+ucl_strncasestr (const char *s, const char *find, int len)
+{
+ char c, sc;
+ int mlen;
+
+ if ((c = *find++) != 0) {
+ c = tolower (c);
+ mlen = strlen (find);
+ do {
+ do {
+ if ((sc = *s++) == 0 || len-- == 0)
+ return (NULL);
+ } while (tolower (sc) != c);
+ } while (strncasecmp (s, find, mlen) != 0);
+ s--;
+ }
+ return ((char *)s);
+}
+
+ucl_object_t *
+ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags flags)
+{
+ ucl_object_t *obj;
+ const char *start, *end, *p, *pos;
+ char *dst, *d;
+ size_t escaped_len;
+
+ if (str == NULL) {
+ return NULL;
+ }
+
+ obj = ucl_object_new ();
+ if (obj) {
+ if (len == 0) {
+ len = strlen (str);
+ }
+ if (flags & UCL_STRING_TRIM) {
+ /* Skip leading spaces */
+ for (start = str; (size_t)(start - str) < len; start ++) {
+ if (!ucl_test_character (*start, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ break;
+ }
+ }
+ /* Skip trailing spaces */
+ for (end = str + len - 1; end > start; end --) {
+ if (!ucl_test_character (*end, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ break;
+ }
+ }
+ end ++;
+ }
+ else {
+ start = str;
+ end = str + len;
+ }
+
+ obj->type = UCL_STRING;
+ if (flags & UCL_STRING_ESCAPE) {
+ for (p = start, escaped_len = 0; p < end; p ++, escaped_len ++) {
+ if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE | UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ switch (*p) {
+ case '\v':
+ case '\0':
+ escaped_len += 5;
+ break;
+ case ' ':
+ break;
+ default:
+ escaped_len ++;
+ break;
+ }
+ }
+ }
+ dst = malloc (escaped_len + 1);
+ if (dst != NULL) {
+ for (p = start, d = dst; p < end; p ++, d ++) {
+ if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE | UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ switch (*p) {
+ case '\n':
+ *d++ = '\\';
+ *d = 'n';
+ break;
+ case '\r':
+ *d++ = '\\';
+ *d = 'r';
+ break;
+ case '\b':
+ *d++ = '\\';
+ *d = 'b';
+ break;
+ case '\t':
+ *d++ = '\\';
+ *d = 't';
+ break;
+ case '\f':
+ *d++ = '\\';
+ *d = 'f';
+ break;
+ case '\0':
+ *d++ = '\\';
+ *d++ = 'u';
+ *d++ = '0';
+ *d++ = '0';
+ *d++ = '0';
+ *d = '0';
+ break;
+ case '\v':
+ *d++ = '\\';
+ *d++ = 'u';
+ *d++ = '0';
+ *d++ = '0';
+ *d++ = '0';
+ *d = 'B';
+ break;
+ case '\\':
+ *d++ = '\\';
+ *d = '\\';
+ break;
+ case ' ':
+ *d = ' ';
+ break;
+ case '"':
+ *d++ = '\\';
+ *d = '"';
+ break;
+ }
+ }
+ else {
+ *d = *p;
+ }
+ }
+ *d = '\0';
+ obj->value.sv = dst;
+ obj->trash_stack[UCL_TRASH_VALUE] = dst;
+ obj->len = escaped_len;
+ }
+ }
+ else {
+ dst = malloc (end - start + 1);
+ if (dst != NULL) {
+ ucl_strlcpy_unsafe (dst, start, end - start + 1);
+ obj->value.sv = dst;
+ obj->trash_stack[UCL_TRASH_VALUE] = dst;
+ obj->len = end - start;
+ }
+ }
+ if ((flags & UCL_STRING_PARSE) && dst != NULL) {
+ /* Parse what we have */
+ if (flags & UCL_STRING_PARSE_BOOLEAN) {
+ if (!ucl_maybe_parse_boolean (obj, dst, obj->len) && (flags & UCL_STRING_PARSE_NUMBER)) {
+ ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
+ flags & UCL_STRING_PARSE_DOUBLE,
+ flags & UCL_STRING_PARSE_BYTES,
+ flags & UCL_STRING_PARSE_TIME);
+ }
+ }
+ else {
+ ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
+ flags & UCL_STRING_PARSE_DOUBLE,
+ flags & UCL_STRING_PARSE_BYTES,
+ flags & UCL_STRING_PARSE_TIME);
+ }
+ }
+ }
+
+ return obj;
+}
+
+static bool
+ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key, bool merge, bool replace)
+{
+ ucl_object_t *found, *tmp;
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+ const char *p;
+ int ret = true;
+
+ if (elt == NULL || key == NULL) {
+ return false;
+ }
+
+ if (top == NULL) {
+ return false;
+ }
+
+ if (top->type != UCL_OBJECT) {
+ /* It is possible to convert NULL type to an object */
+ if (top->type == UCL_NULL) {
+ top->type = UCL_OBJECT;
+ }
+ else {
+ /* Refuse converting of other object types */
+ return false;
+ }
+ }
+
+ if (top->value.ov == NULL) {
+ top->value.ov = ucl_hash_create (false);
+ }
+
+ if (keylen == 0) {
+ keylen = strlen (key);
+ }
+
+ for (p = key; p < key + keylen; p ++) {
+ if (ucl_test_character (*p, UCL_CHARACTER_UCL_UNSAFE)) {
+ elt->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
+ break;
+ }
+ }
+
+ /* workaround for some use cases */
+ if (elt->trash_stack[UCL_TRASH_KEY] != NULL &&
+ key != (const char *)elt->trash_stack[UCL_TRASH_KEY]) {
+ /* Remove copied key */
+ free (elt->trash_stack[UCL_TRASH_KEY]);
+ elt->trash_stack[UCL_TRASH_KEY] = NULL;
+ elt->flags &= ~UCL_OBJECT_ALLOCATED_KEY;
+ }
+
+ elt->key = key;
+ elt->keylen = keylen;
+
+ if (copy_key) {
+ ucl_copy_key_trash (elt);
+ }
+
+ found = __DECONST (ucl_object_t *, ucl_hash_search_obj (top->value.ov, elt));
+
+ if (found == NULL) {
+ top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
+ top->len ++;
+ if (replace) {
+ ret = false;
+ }
+ }
+ else {
+ if (replace) {
+ ucl_hash_replace (top->value.ov, found, elt);
+ ucl_object_unref (found);
+ }
+ else if (merge) {
+ if (found->type != UCL_OBJECT && elt->type == UCL_OBJECT) {
+ /* Insert old elt to new one */
+ ucl_object_insert_key_common (elt, found, found->key,
+ found->keylen, copy_key, false, false);
+ ucl_hash_delete (top->value.ov, found);
+ top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
+ }
+ else if (found->type == UCL_OBJECT && elt->type != UCL_OBJECT) {
+ /* Insert new to old */
+ ucl_object_insert_key_common (found, elt, elt->key,
+ elt->keylen, copy_key, false, false);
+ }
+ else if (found->type == UCL_OBJECT && elt->type == UCL_OBJECT) {
+ /* Mix two hashes */
+ while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
+ tmp = ucl_object_ref (cur);
+ ucl_object_insert_key_common (found, tmp, cur->key,
+ cur->keylen, copy_key, true, false);
+ }
+ ucl_object_unref (elt);
+ }
+ else {
+ /* Just make a list of scalars */
+ DL_CONCAT (found, elt);
+ }
+ }
+ else {
+ DL_CONCAT (found, elt);
+ }
+ }
+
+ return ret;
+}
+
+bool
+ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen)
+{
+ ucl_object_t *found;
+
+ if (top == NULL || key == NULL) {
+ return false;
+ }
+
+ found = __DECONST (ucl_object_t *, ucl_object_lookup_len (top, key, keylen));
+
+ if (found == NULL) {
+ return false;
+ }
+
+ ucl_hash_delete (top->value.ov, found);
+ ucl_object_unref (found);
+ top->len --;
+
+ return true;
+}
+
+bool
+ucl_object_delete_key (ucl_object_t *top, const char *key)
+{
+ return ucl_object_delete_keyl (top, key, strlen (key));
+}
+
+ucl_object_t*
+ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen)
+{
+ const ucl_object_t *found;
+
+ if (top == NULL || key == NULL) {
+ return false;
+ }
+ found = ucl_object_lookup_len (top, key, keylen);
+
+ if (found == NULL) {
+ return NULL;
+ }
+ ucl_hash_delete (top->value.ov, found);
+ top->len --;
+
+ return __DECONST (ucl_object_t *, found);
+}
+
+ucl_object_t*
+ucl_object_pop_key (ucl_object_t *top, const char *key)
+{
+ return ucl_object_pop_keyl (top, key, strlen (key));
+}
+
+bool
+ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key)
+{
+ return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, false);
+}
+
+bool
+ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key)
+{
+ return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, true, false);
+}
+
+bool
+ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key)
+{
+ return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, true);
+}
+
+bool
+ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
+{
+ ucl_object_t *cur = NULL, *cp = NULL, *found = NULL;
+ ucl_object_iter_t iter = NULL;
+
+ if (top == NULL || elt == NULL) {
+ return false;
+ }
+
+ if (top->type == UCL_ARRAY) {
+ if (elt->type == UCL_ARRAY) {
+ /* Merge two arrays */
+ return ucl_array_merge (top, elt, copy);
+ }
+ else {
+ if (copy) {
+ ucl_array_append (top, ucl_object_copy (elt));
+
+ return true;
+ }
+ else {
+ ucl_array_append (top, ucl_object_ref (elt));
+
+ return true;
+ }
+ }
+ }
+ else if (top->type == UCL_OBJECT) {
+ if (elt->type == UCL_OBJECT) {
+ /* Mix two hashes */
+ while ((cur = (ucl_object_t *) ucl_hash_iterate (elt->value.ov,
+ &iter))) {
+
+ if (copy) {
+ cp = ucl_object_copy (cur);
+ } else {
+ cp = ucl_object_ref (cur);
+ }
+
+ found = __DECONST(ucl_object_t *,
+ ucl_hash_search (top->value.ov, cp->key, cp->keylen));
+
+ if (found == NULL) {
+ /* The key does not exist */
+ top->value.ov = ucl_hash_insert_object (top->value.ov, cp,
+ false);
+ top->len++;
+ }
+ else {
+ /* The key already exists, merge it recursively */
+ if (found->type == UCL_OBJECT || found->type == UCL_ARRAY) {
+ if (!ucl_object_merge (found, cp, copy)) {
+ return false;
+ }
+ ucl_object_unref (cp);
+ }
+ else {
+ ucl_hash_replace (top->value.ov, found, cp);
+ ucl_object_unref (found);
+ }
+ }
+ }
+ }
+ else {
+ if (copy) {
+ cp = ucl_object_copy (elt);
+ }
+ else {
+ cp = ucl_object_ref (elt);
+ }
+
+ found = __DECONST(ucl_object_t *,
+ ucl_hash_search (top->value.ov, cp->key, cp->keylen));
+
+ if (found == NULL) {
+ /* The key does not exist */
+ top->value.ov = ucl_hash_insert_object (top->value.ov, cp,
+ false);
+ top->len++;
+ }
+ else {
+ /* The key already exists, merge it recursively */
+ if (found->type == UCL_OBJECT || found->type == UCL_ARRAY) {
+ if (!ucl_object_merge (found, cp, copy)) {
+ return false;
+ }
+ ucl_object_unref (cp);
+ }
+ else {
+ ucl_hash_replace (top->value.ov, found, cp);
+ ucl_object_unref (found);
+ }
+ }
+ }
+ }
+ else {
+ /* Cannot merge trivial objects */
+ return false;
+ }
+
+ return true;
+}
+
+const ucl_object_t *
+ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen)
+{
+ const ucl_object_t *ret;
+ ucl_object_t srch;
+
+ if (obj == NULL || obj->type != UCL_OBJECT || key == NULL) {
+ return NULL;
+ }
+
+ srch.key = key;
+ srch.keylen = klen;
+ ret = ucl_hash_search_obj (obj->value.ov, &srch);
+
+ return ret;
+}
+
+const ucl_object_t *
+ucl_object_lookup (const ucl_object_t *obj, const char *key)
+{
+ if (key == NULL) {
+ return NULL;
+ }
+
+ return ucl_object_lookup_len (obj, key, strlen (key));
+}
+
+const ucl_object_t*
+ucl_object_lookup_any (const ucl_object_t *obj,
+ const char *key, ...)
+{
+ va_list ap;
+ const ucl_object_t *ret = NULL;
+ const char *nk = NULL;
+
+ if (obj == NULL || key == NULL) {
+ return NULL;
+ }
+
+ ret = ucl_object_lookup_len (obj, key, strlen (key));
+
+ if (ret == NULL) {
+ va_start (ap, key);
+
+ while (ret == NULL) {
+ nk = va_arg (ap, const char *);
+
+ if (nk == NULL) {
+ break;
+ }
+ else {
+ ret = ucl_object_lookup_len (obj, nk, strlen (nk));
+ }
+ }
+
+ va_end (ap);
+ }
+
+ return ret;
+}
+
+const ucl_object_t*
+ucl_object_iterate_with_error (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values,
+ int *ep)
+{
+ const ucl_object_t *elt = NULL;
+
+ if (obj == NULL || iter == NULL) {
+ return NULL;
+ }
+
+ if (expand_values) {
+ switch (obj->type) {
+ case UCL_OBJECT:
+ return (const ucl_object_t*)ucl_hash_iterate2 (obj->value.ov, iter, ep);
+ break;
+ case UCL_ARRAY: {
+ unsigned int idx;
+ UCL_ARRAY_GET (vec, obj);
+ idx = (unsigned int)(uintptr_t)(*iter);
+
+ if (vec != NULL) {
+ while (idx < kv_size (*vec)) {
+ if ((elt = kv_A (*vec, idx)) != NULL) {
+ idx ++;
+ break;
+ }
+ idx ++;
+ }
+ *iter = (void *)(uintptr_t)idx;
+ }
+
+ return elt;
+ break;
+ }
+ default:
+ /* Go to linear iteration */
+ break;
+ }
+ }
+ /* Treat everything as a linear list */
+ elt = *iter;
+ if (elt == NULL) {
+ elt = obj;
+ }
+ else if (elt == obj) {
+ return NULL;
+ }
+ *iter = __DECONST (void *, elt->next ? elt->next : obj);
+ return elt;
+
+ /* Not reached */
+ return NULL;
+}
+
+enum ucl_safe_iter_flags {
+ UCL_ITERATE_FLAG_UNDEFINED = 0,
+ UCL_ITERATE_FLAG_INSIDE_ARRAY,
+ UCL_ITERATE_FLAG_INSIDE_OBJECT,
+ UCL_ITERATE_FLAG_IMPLICIT,
+ UCL_ITERATE_FLAG_EXCEPTION
+};
+
+static const char safe_iter_magic[4] = {'u', 'i', 't', 'e'};
+struct ucl_object_safe_iter {
+ char magic[4]; /* safety check */
+ uint32_t flags;
+ const ucl_object_t *impl_it; /* implicit object iteration */
+ ucl_object_iter_t expl_it; /* explicit iteration */
+};
+
+#define UCL_SAFE_ITER(ptr) (struct ucl_object_safe_iter *)(ptr)
+#define UCL_SAFE_ITER_CHECK(it) do { \
+ assert (it != NULL); \
+ assert (memcmp (it->magic, safe_iter_magic, sizeof (it->magic)) == 0); \
+ } while (0)
+
+ucl_object_iter_t
+ucl_object_iterate_new (const ucl_object_t *obj)
+{
+ struct ucl_object_safe_iter *it;
+
+ it = UCL_ALLOC (sizeof (*it));
+ if (it != NULL) {
+ memcpy (it->magic, safe_iter_magic, sizeof (it->magic));
+ it->flags = UCL_ITERATE_FLAG_UNDEFINED;
+ it->expl_it = NULL;
+ it->impl_it = obj;
+ }
+
+ return (ucl_object_iter_t)it;
+}
+
+bool
+ucl_object_iter_chk_excpn(ucl_object_iter_t *it)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ return (rit->flags == UCL_ITERATE_FLAG_EXCEPTION);
+}
+
+ucl_object_iter_t
+ucl_object_iterate_reset (ucl_object_iter_t it, const ucl_object_t *obj)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ if (rit->expl_it != NULL) {
+ if (rit->flags == UCL_ITERATE_FLAG_INSIDE_OBJECT) {
+ UCL_FREE (sizeof (*rit->expl_it), rit->expl_it);
+ }
+ }
+
+ rit->impl_it = obj;
+ rit->expl_it = NULL;
+ rit->flags = UCL_ITERATE_FLAG_UNDEFINED;
+
+ return it;
+}
+
+const ucl_object_t*
+ucl_object_iterate_safe (ucl_object_iter_t it, bool expand_values)
+{
+ return ucl_object_iterate_full (it, expand_values ? UCL_ITERATE_BOTH :
+ UCL_ITERATE_IMPLICIT);
+}
+
+const ucl_object_t*
+ucl_object_iterate_full (ucl_object_iter_t it, enum ucl_iterate_type type)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+ const ucl_object_t *ret = NULL;
+ int ern;
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ if (rit->impl_it == NULL) {
+ return NULL;
+ }
+
+ if (rit->impl_it->type == UCL_OBJECT) {
+ rit->flags = UCL_ITERATE_FLAG_INSIDE_OBJECT;
+ ret = ucl_object_iterate_with_error (rit->impl_it, &rit->expl_it, true, &ern);
+
+ if (ret == NULL && ern != 0) {
+ rit->flags = UCL_ITERATE_FLAG_EXCEPTION;
+ return NULL;
+ }
+
+ if (ret == NULL && (type & UCL_ITERATE_IMPLICIT)) {
+ /* Need to switch to another implicit object in chain */
+ rit->impl_it = rit->impl_it->next;
+ rit->expl_it = NULL;
+
+ return ucl_object_iterate_safe (it, type);
+ }
+ }
+ else if (rit->impl_it->type == UCL_ARRAY) {
+ rit->flags = UCL_ITERATE_FLAG_INSIDE_ARRAY;
+ ret = ucl_object_iterate (rit->impl_it, &rit->expl_it, true);
+
+ if (ret == NULL && (type & UCL_ITERATE_IMPLICIT)) {
+ /* Need to switch to another implicit object in chain */
+ rit->impl_it = rit->impl_it->next;
+ rit->expl_it = NULL;
+
+ return ucl_object_iterate_safe (it, type);
+ }
+ }
+ else {
+ /* Just iterate over the implicit array */
+ rit->flags = UCL_ITERATE_FLAG_IMPLICIT;
+ ret = rit->impl_it;
+ rit->impl_it = rit->impl_it->next;
+
+ if (type & UCL_ITERATE_EXPLICIT) {
+ /* We flatten objects if need to expand values */
+ if (ret->type == UCL_OBJECT || ret->type == UCL_ARRAY) {
+ return ucl_object_iterate_safe (it, type);
+ }
+ }
+ }
+
+ return ret;
+}
+
+void
+ucl_object_iterate_free (ucl_object_iter_t it)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ if (rit->expl_it != NULL) {
+ if (rit->flags == UCL_ITERATE_FLAG_INSIDE_OBJECT) {
+ UCL_FREE (sizeof (*rit->expl_it), rit->expl_it);
+ }
+ }
+
+ UCL_FREE (sizeof (*rit), it);
+}
+
+const ucl_object_t *
+ucl_object_lookup_path (const ucl_object_t *top, const char *path_in) {
+ return ucl_object_lookup_path_char (top, path_in, '.');
+}
+
+
+const ucl_object_t *
+ucl_object_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) {
+ const ucl_object_t *o = NULL, *found;
+ const char *p, *c;
+ char *err_str;
+ unsigned index;
+
+ if (path_in == NULL || top == NULL) {
+ return NULL;
+ }
+
+ found = NULL;
+ p = path_in;
+
+ /* Skip leading dots */
+ while (*p == sep) {
+ p ++;
+ }
+
+ c = p;
+ while (*p != '\0') {
+ p ++;
+ if (*p == sep || *p == '\0') {
+ if (p > c) {
+ switch (top->type) {
+ case UCL_ARRAY:
+ /* Key should be an int */
+ index = strtoul (c, &err_str, 10);
+ if (err_str != NULL && (*err_str != sep && *err_str != '\0')) {
+ return NULL;
+ }
+ o = ucl_array_find_index (top, index);
+ break;
+ default:
+ o = ucl_object_lookup_len (top, c, p - c);
+ break;
+ }
+ if (o == NULL) {
+ return NULL;
+ }
+ top = o;
+ }
+ if (*p != '\0') {
+ c = p + 1;
+ }
+ }
+ }
+ found = o;
+
+ return found;
+}
+
+
+ucl_object_t *
+ucl_object_new (void)
+{
+ return ucl_object_typed_new (UCL_NULL);
+}
+
+ucl_object_t *
+ucl_object_typed_new (ucl_type_t type)
+{
+ return ucl_object_new_full (type, 0);
+}
+
+ucl_object_t *
+ucl_object_new_full (ucl_type_t type, unsigned priority)
+{
+ ucl_object_t *new;
+
+ if (type != UCL_USERDATA) {
+ new = UCL_ALLOC (sizeof (ucl_object_t));
+ if (new != NULL) {
+ memset (new, 0, sizeof (ucl_object_t));
+ new->ref = 1;
+ new->type = (type <= UCL_NULL ? type : UCL_NULL);
+ new->next = NULL;
+ new->prev = new;
+ ucl_object_set_priority (new, priority);
+
+ if (type == UCL_ARRAY) {
+ new->value.av = UCL_ALLOC (sizeof (ucl_array_t));
+ if (new->value.av) {
+ memset (new->value.av, 0, sizeof (ucl_array_t));
+ UCL_ARRAY_GET (vec, new);
+
+ /* Preallocate some space for arrays */
+ kv_resize_safe (ucl_object_t *, *vec, 8, enomem);
+ }
+ }
+ }
+ }
+ else {
+ new = ucl_object_new_userdata (NULL, NULL, NULL);
+ ucl_object_set_priority (new, priority);
+ }
+enomem:
+ return new;
+}
+
+bool ucl_object_reserve (ucl_object_t *obj, size_t reserved)
+{
+ if (obj->type == UCL_ARRAY) {
+ UCL_ARRAY_GET (vec, obj);
+
+ if (vec->m < reserved) {
+ /* Preallocate some space for arrays */
+ kv_resize_safe (ucl_object_t *, *vec, reserved, e0);
+ }
+ }
+ else if (obj->type == UCL_OBJECT) {
+ ucl_hash_reserve (obj->value.ov, reserved);
+ }
+ return true;
+e0:
+ return false;
+}
+
+ucl_object_t*
+ucl_object_new_userdata (ucl_userdata_dtor dtor,
+ ucl_userdata_emitter emitter,
+ void *ptr)
+{
+ struct ucl_object_userdata *new;
+ size_t nsize = sizeof (*new);
+
+ new = UCL_ALLOC (nsize);
+ if (new != NULL) {
+ memset (new, 0, nsize);
+ new->obj.ref = 1;
+ new->obj.type = UCL_USERDATA;
+ new->obj.next = NULL;
+ new->obj.prev = (ucl_object_t *)new;
+ new->dtor = dtor;
+ new->emitter = emitter;
+ new->obj.value.ud = ptr;
+ }
+
+ return (ucl_object_t *)new;
+}
+
+ucl_type_t
+ucl_object_type (const ucl_object_t *obj)
+{
+ if (obj == NULL) {
+ return UCL_NULL;
+ }
+
+ return obj->type;
+}
+
+ucl_object_t*
+ucl_object_fromstring (const char *str)
+{
+ return ucl_object_fromstring_common (str, 0, UCL_STRING_ESCAPE);
+}
+
+ucl_object_t *
+ucl_object_fromlstring (const char *str, size_t len)
+{
+ return ucl_object_fromstring_common (str, len, UCL_STRING_ESCAPE);
+}
+
+ucl_object_t *
+ucl_object_fromint (int64_t iv)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_new ();
+ if (obj != NULL) {
+ obj->type = UCL_INT;
+ obj->value.iv = iv;
+ }
+
+ return obj;
+}
+
+ucl_object_t *
+ucl_object_fromdouble (double dv)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_new ();
+ if (obj != NULL) {
+ obj->type = UCL_FLOAT;
+ obj->value.dv = dv;
+ }
+
+ return obj;
+}
+
+ucl_object_t*
+ucl_object_frombool (bool bv)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_new ();
+ if (obj != NULL) {
+ obj->type = UCL_BOOLEAN;
+ obj->value.iv = bv;
+ }
+
+ return obj;
+}
+
+bool
+ucl_array_append (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (elt == NULL || top == NULL) {
+ return false;
+ }
+
+ if (vec == NULL) {
+ vec = UCL_ALLOC (sizeof (*vec));
+
+ if (vec == NULL) {
+ return false;
+ }
+
+ kv_init (*vec);
+ top->value.av = (void *)vec;
+ }
+
+ kv_push_safe (ucl_object_t *, *vec, elt, e0);
+
+ top->len ++;
+
+ return true;
+e0:
+ return false;
+}
+
+bool
+ucl_array_prepend (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (elt == NULL || top == NULL) {
+ return false;
+ }
+
+ if (vec == NULL) {
+ vec = UCL_ALLOC (sizeof (*vec));
+ kv_init (*vec);
+ top->value.av = (void *)vec;
+ kv_push_safe (ucl_object_t *, *vec, elt, e0);
+ }
+ else {
+ /* Slow O(n) algorithm */
+ kv_prepend_safe (ucl_object_t *, *vec, elt, e0);
+ }
+
+ top->len ++;
+
+ return true;
+e0:
+ return false;
+}
+
+bool
+ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
+{
+ unsigned i;
+ ucl_object_t *cp = NULL;
+ ucl_object_t **obj;
+
+ if (elt == NULL || top == NULL || top->type != UCL_ARRAY || elt->type != UCL_ARRAY) {
+ return false;
+ }
+
+ if (copy) {
+ cp = ucl_object_copy (elt);
+ }
+ else {
+ cp = ucl_object_ref (elt);
+ }
+
+ UCL_ARRAY_GET (v1, top);
+ UCL_ARRAY_GET (v2, cp);
+
+ if (v1 && v2) {
+ kv_concat_safe (ucl_object_t *, *v1, *v2, e0);
+
+ for (i = v2->n; i < v1->n; i ++) {
+ obj = &kv_A (*v1, i);
+ if (*obj == NULL) {
+ continue;
+ }
+ top->len ++;
+ }
+ }
+
+ return true;
+e0:
+ return false;
+}
+
+ucl_object_t *
+ucl_array_delete (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t *ret = NULL;
+ unsigned i;
+
+ if (vec == NULL) {
+ return NULL;
+ }
+
+ for (i = 0; i < vec->n; i ++) {
+ if (kv_A (*vec, i) == elt) {
+ kv_del (ucl_object_t *, *vec, i);
+ ret = elt;
+ top->len --;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+const ucl_object_t *
+ucl_array_head (const ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (vec == NULL || top == NULL || top->type != UCL_ARRAY ||
+ top->value.av == NULL) {
+ return NULL;
+ }
+
+ return (vec->n > 0 ? vec->a[0] : NULL);
+}
+
+const ucl_object_t *
+ucl_array_tail (const ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (top == NULL || top->type != UCL_ARRAY || top->value.av == NULL) {
+ return NULL;
+ }
+
+ return (vec->n > 0 ? vec->a[vec->n - 1] : NULL);
+}
+
+ucl_object_t *
+ucl_array_pop_last (ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t **obj, *ret = NULL;
+
+ if (vec != NULL && vec->n > 0) {
+ obj = &kv_A (*vec, vec->n - 1);
+ ret = *obj;
+ kv_del (ucl_object_t *, *vec, vec->n - 1);
+ top->len --;
+ }
+
+ return ret;
+}
+
+ucl_object_t *
+ucl_array_pop_first (ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t **obj, *ret = NULL;
+
+ if (vec != NULL && vec->n > 0) {
+ obj = &kv_A (*vec, 0);
+ ret = *obj;
+ kv_del (ucl_object_t *, *vec, 0);
+ top->len --;
+ }
+
+ return ret;
+}
+
+unsigned int
+ucl_array_size (const ucl_object_t *top)
+{
+ if (top == NULL || top->type != UCL_ARRAY) {
+ return 0;
+ }
+
+ UCL_ARRAY_GET (vec, top);
+
+ if (vec != NULL) {
+ return kv_size(*vec);
+ }
+
+ return 0;
+}
+
+const ucl_object_t *
+ucl_array_find_index (const ucl_object_t *top, unsigned int index)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (vec != NULL && vec->n > 0 && index < vec->n) {
+ return kv_A (*vec, index);
+ }
+
+ return NULL;
+}
+
+unsigned int
+ucl_array_index_of (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+ unsigned i;
+
+ if (vec == NULL) {
+ return (unsigned int)(-1);
+ }
+
+ for (i = 0; i < vec->n; i ++) {
+ if (kv_A (*vec, i) == elt) {
+ return i;
+ }
+ }
+
+ return (unsigned int)(-1);
+}
+
+ucl_object_t *
+ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt,
+ unsigned int index)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t *ret = NULL;
+
+ if (vec != NULL && vec->n > 0 && index < vec->n) {
+ ret = kv_A (*vec, index);
+ kv_A (*vec, index) = elt;
+ }
+
+ return ret;
+}
+
+ucl_object_t *
+ucl_elt_append (ucl_object_t *head, ucl_object_t *elt)
+{
+
+ if (head == NULL) {
+ elt->next = NULL;
+ elt->prev = elt;
+ head = elt;
+ }
+ else {
+ elt->prev = head->prev;
+ head->prev->next = elt;
+ head->prev = elt;
+ elt->next = NULL;
+ }
+
+ return head;
+}
+
+bool
+ucl_object_todouble_safe (const ucl_object_t *obj, double *target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_INT:
+ *target = obj->value.iv; /* Probably could cause overflow */
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ *target = obj->value.dv;
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+double
+ucl_object_todouble (const ucl_object_t *obj)
+{
+ double result = 0.;
+
+ ucl_object_todouble_safe (obj, &result);
+ return result;
+}
+
+bool
+ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_INT:
+ *target = obj->value.iv;
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ *target = obj->value.dv; /* Losing of decimal points */
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+int64_t
+ucl_object_toint (const ucl_object_t *obj)
+{
+ int64_t result = 0;
+
+ ucl_object_toint_safe (obj, &result);
+ return result;
+}
+
+bool
+ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_BOOLEAN:
+ *target = (obj->value.iv == true);
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+bool
+ucl_object_toboolean (const ucl_object_t *obj)
+{
+ bool result = false;
+
+ ucl_object_toboolean_safe (obj, &result);
+ return result;
+}
+
+bool
+ucl_object_tostring_safe (const ucl_object_t *obj, const char **target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+
+ switch (obj->type) {
+ case UCL_STRING:
+ if (!(obj->flags & UCL_OBJECT_BINARY)) {
+ *target = ucl_copy_value_trash (obj);
+ }
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+const char *
+ucl_object_tostring (const ucl_object_t *obj)
+{
+ const char *result = NULL;
+
+ ucl_object_tostring_safe (obj, &result);
+ return result;
+}
+
+const char *
+ucl_object_tostring_forced (const ucl_object_t *obj)
+{
+ /* TODO: For binary strings we might encode string here */
+ if (!(obj->flags & UCL_OBJECT_BINARY)) {
+ return ucl_copy_value_trash (obj);
+ }
+
+ return NULL;
+}
+
+bool
+ucl_object_tolstring_safe (const ucl_object_t *obj, const char **target, size_t *tlen)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_STRING:
+ *target = obj->value.sv;
+ if (tlen != NULL) {
+ *tlen = obj->len;
+ }
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+const char *
+ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen)
+{
+ const char *result = NULL;
+
+ ucl_object_tolstring_safe (obj, &result, tlen);
+ return result;
+}
+
+const char *
+ucl_object_key (const ucl_object_t *obj)
+{
+ return ucl_copy_key_trash (obj);
+}
+
+const char *
+ucl_object_keyl (const ucl_object_t *obj, size_t *len)
+{
+ if (len == NULL || obj == NULL) {
+ return NULL;
+ }
+ *len = obj->keylen;
+ return obj->key;
+}
+
+ucl_object_t *
+ucl_object_ref (const ucl_object_t *obj)
+{
+ ucl_object_t *res = NULL;
+
+ if (obj != NULL) {
+ if (obj->flags & UCL_OBJECT_EPHEMERAL) {
+ /*
+ * Use deep copy for ephemeral objects, note that its refcount
+ * is NOT increased, since ephemeral objects does not need refcount
+ * at all
+ */
+ res = ucl_object_copy (obj);
+ }
+ else {
+ res = __DECONST (ucl_object_t *, obj);
+#ifdef HAVE_ATOMIC_BUILTINS
+ (void)__sync_add_and_fetch (&res->ref, 1);
+#else
+ res->ref ++;
+#endif
+ }
+ }
+ return res;
+}
+
+static ucl_object_t *
+ucl_object_copy_internal (const ucl_object_t *other, bool allow_array)
+{
+
+ ucl_object_t *new;
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+
+ new = malloc (sizeof (*new));
+
+ if (new != NULL) {
+ memcpy (new, other, sizeof (*new));
+ if (other->flags & UCL_OBJECT_EPHEMERAL) {
+ /* Copied object is always non ephemeral */
+ new->flags &= ~UCL_OBJECT_EPHEMERAL;
+ }
+ new->ref = 1;
+ /* Unlink from others */
+ new->next = NULL;
+ new->prev = new;
+
+ /* deep copy of values stored */
+ if (other->trash_stack[UCL_TRASH_KEY] != NULL) {
+ new->trash_stack[UCL_TRASH_KEY] = NULL;
+ if (other->key == (const char *)other->trash_stack[UCL_TRASH_KEY]) {
+ new->trash_stack[UCL_TRASH_KEY] = malloc(other->keylen + 1);
+ memcpy(new->trash_stack[UCL_TRASH_KEY], other->trash_stack[UCL_TRASH_KEY], other->keylen);
+ new->trash_stack[UCL_TRASH_KEY][other->keylen] = '\0';
+ new->key = new->trash_stack[UCL_TRASH_KEY];
+ }
+ }
+ if (other->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ new->trash_stack[UCL_TRASH_VALUE] =
+ strdup (other->trash_stack[UCL_TRASH_VALUE]);
+ if (new->type == UCL_STRING) {
+ new->value.sv = new->trash_stack[UCL_TRASH_VALUE];
+ }
+ }
+
+ if (other->type == UCL_ARRAY || other->type == UCL_OBJECT) {
+ /* reset old value */
+ memset (&new->value, 0, sizeof (new->value));
+
+ while ((cur = ucl_object_iterate (other, &it, true)) != NULL) {
+ if (other->type == UCL_ARRAY) {
+ ucl_array_append (new, ucl_object_copy_internal (cur, false));
+ }
+ else {
+ ucl_object_t *cp = ucl_object_copy_internal (cur, true);
+ if (cp != NULL) {
+ ucl_object_insert_key (new, cp, cp->key, cp->keylen,
+ false);
+ }
+ }
+ }
+ }
+ else if (allow_array && other->next != NULL) {
+ LL_FOREACH (other->next, cur) {
+ ucl_object_t *cp = ucl_object_copy_internal (cur, false);
+ if (cp != NULL) {
+ DL_APPEND (new, cp);
+ }
+ }
+ }
+ }
+
+ return new;
+}
+
+ucl_object_t *
+ucl_object_copy (const ucl_object_t *other)
+{
+ return ucl_object_copy_internal (other, true);
+}
+
+void
+ucl_object_unref (ucl_object_t *obj)
+{
+ if (obj != NULL) {
+#ifdef HAVE_ATOMIC_BUILTINS
+ unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
+ if (rc == 0) {
+#else
+ if (--obj->ref == 0) {
+#endif
+ ucl_object_free_internal (obj, true, ucl_object_dtor_unref);
+ }
+ }
+}
+
+int
+ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2)
+{
+ const ucl_object_t *it1, *it2;
+ ucl_object_iter_t iter = NULL;
+ int ret = 0;
+
+ if (o1->type != o2->type) {
+ return (o1->type) - (o2->type);
+ }
+
+ switch (o1->type) {
+ case UCL_STRING:
+ if (o1->len == o2->len && o1->len > 0) {
+ ret = strcmp (ucl_object_tostring(o1), ucl_object_tostring(o2));
+ }
+ else {
+ ret = o1->len - o2->len;
+ }
+ break;
+ case UCL_FLOAT:
+ case UCL_INT:
+ case UCL_TIME:
+ ret = ucl_object_todouble (o1) - ucl_object_todouble (o2);
+ break;
+ case UCL_BOOLEAN:
+ ret = ucl_object_toboolean (o1) - ucl_object_toboolean (o2);
+ break;
+ case UCL_ARRAY:
+ if (o1->len == o2->len && o1->len > 0) {
+ UCL_ARRAY_GET (vec1, o1);
+ UCL_ARRAY_GET (vec2, o2);
+ unsigned i;
+
+ /* Compare all elements in both arrays */
+ for (i = 0; i < vec1->n; i ++) {
+ it1 = kv_A (*vec1, i);
+ it2 = kv_A (*vec2, i);
+
+ if (it1 == NULL && it2 != NULL) {
+ return -1;
+ }
+ else if (it2 == NULL && it1 != NULL) {
+ return 1;
+ }
+ else if (it1 != NULL && it2 != NULL) {
+ ret = ucl_object_compare (it1, it2);
+ if (ret != 0) {
+ break;
+ }
+ }
+ }
+ }
+ else {
+ ret = o1->len - o2->len;
+ }
+ break;
+ case UCL_OBJECT:
+ if (o1->len == o2->len && o1->len > 0) {
+ while ((it1 = ucl_object_iterate (o1, &iter, true)) != NULL) {
+ it2 = ucl_object_lookup (o2, ucl_object_key (it1));
+ if (it2 == NULL) {
+ ret = 1;
+ break;
+ }
+ ret = ucl_object_compare (it1, it2);
+ if (ret != 0) {
+ break;
+ }
+ }
+ }
+ else {
+ ret = o1->len - o2->len;
+ }
+ break;
+ default:
+ ret = 0;
+ break;
+ }
+
+ return ret;
+}
+
+int
+ucl_object_compare_qsort (const ucl_object_t **o1,
+ const ucl_object_t **o2)
+{
+ return ucl_object_compare (*o1, *o2);
+}
+
+void
+ucl_object_array_sort (ucl_object_t *ar,
+ int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2))
+{
+ UCL_ARRAY_GET (vec, ar);
+
+ if (cmp == NULL || ar == NULL || ar->type != UCL_ARRAY) {
+ return;
+ }
+
+ qsort (vec->a, vec->n, sizeof (ucl_object_t *),
+ (int (*)(const void *, const void *))cmp);
+}
+
+void ucl_object_sort_keys (ucl_object_t *obj,
+ enum ucl_object_keys_sort_flags how)
+{
+ if (obj != NULL && obj->type == UCL_OBJECT) {
+ ucl_hash_sort (obj->value.ov, how);
+ }
+}
+
+#define PRIOBITS 4
+
+unsigned int
+ucl_object_get_priority (const ucl_object_t *obj)
+{
+ if (obj == NULL) {
+ return 0;
+ }
+
+ return (obj->flags >> ((sizeof (obj->flags) * NBBY) - PRIOBITS));
+}
+
+void
+ucl_object_set_priority (ucl_object_t *obj,
+ unsigned int priority)
+{
+ if (obj != NULL) {
+ priority &= (0x1 << PRIOBITS) - 1;
+ priority <<= ((sizeof (obj->flags) * NBBY) - PRIOBITS);
+ priority |= obj->flags & ((1 << ((sizeof (obj->flags) * NBBY) -
+ PRIOBITS)) - 1);
+ obj->flags = priority;
+ }
+}
+
+bool
+ucl_object_string_to_type (const char *input, ucl_type_t *res)
+{
+ if (strcasecmp (input, "object") == 0) {
+ *res = UCL_OBJECT;
+ }
+ else if (strcasecmp (input, "array") == 0) {
+ *res = UCL_ARRAY;
+ }
+ else if (strcasecmp (input, "integer") == 0) {
+ *res = UCL_INT;
+ }
+ else if (strcasecmp (input, "number") == 0) {
+ *res = UCL_FLOAT;
+ }
+ else if (strcasecmp (input, "string") == 0) {
+ *res = UCL_STRING;
+ }
+ else if (strcasecmp (input, "boolean") == 0) {
+ *res = UCL_BOOLEAN;
+ }
+ else if (strcasecmp (input, "null") == 0) {
+ *res = UCL_NULL;
+ }
+ else if (strcasecmp (input, "userdata") == 0) {
+ *res = UCL_USERDATA;
+ }
+ else {
+ return false;
+ }
+
+ return true;
+}
+
+const char *
+ucl_object_type_to_string (ucl_type_t type)
+{
+ const char *res = "unknown";
+
+ switch (type) {
+ case UCL_OBJECT:
+ res = "object";
+ break;
+ case UCL_ARRAY:
+ res = "array";
+ break;
+ case UCL_INT:
+ res = "integer";
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ res = "number";
+ break;
+ case UCL_STRING:
+ res = "string";
+ break;
+ case UCL_BOOLEAN:
+ res = "boolean";
+ break;
+ case UCL_USERDATA:
+ res = "userdata";
+ break;
+ case UCL_NULL:
+ res = "null";
+ break;
+ }
+
+ return res;
+}
+
+const ucl_object_t *
+ucl_parser_get_comments (struct ucl_parser *parser)
+{
+ if (parser && parser->comments) {
+ return parser->comments;
+ }
+
+ return NULL;
+}
+
+const ucl_object_t *
+ucl_comments_find (const ucl_object_t *comments,
+ const ucl_object_t *srch)
+{
+ if (comments && srch) {
+ return ucl_object_lookup_len (comments, (const char *)&srch,
+ sizeof (void *));
+ }
+
+ return NULL;
+}
+
+bool
+ucl_comments_move (ucl_object_t *comments,
+ const ucl_object_t *from, const ucl_object_t *to)
+{
+ const ucl_object_t *found;
+ ucl_object_t *obj;
+
+ if (comments && from && to) {
+ found = ucl_object_lookup_len (comments,
+ (const char *)&from, sizeof (void *));
+
+ if (found) {
+ /* Replace key */
+ obj = ucl_object_ref (found);
+ ucl_object_delete_keyl (comments, (const char *)&from,
+ sizeof (void *));
+ ucl_object_insert_key (comments, obj, (const char *)&to,
+ sizeof (void *), true);
+
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void
+ucl_comments_add (ucl_object_t *comments, const ucl_object_t *obj,
+ const char *comment)
+{
+ if (comments && obj && comment) {
+ ucl_object_insert_key (comments, ucl_object_fromstring (comment),
+ (const char *)&obj, sizeof (void *), true);
+ }
+}
+
+void
+ucl_parser_set_include_tracer (struct ucl_parser *parser,
+ ucl_include_trace_func_t func,
+ void *user_data)
+{
+ parser->include_trace_func = func;
+ parser->include_trace_ud = user_data;
+}
+
+const char *
+ucl_parser_get_cur_file (struct ucl_parser *parser)
+{
+ return parser->cur_file;
+} \ No newline at end of file
diff --git a/contrib/lua-argparse/LICENSE b/contrib/lua-argparse/LICENSE
new file mode 100644
index 0000000..87579ac
--- /dev/null
+++ b/contrib/lua-argparse/LICENSE
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 - 2018 Peter Melnichenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/lua-argparse/argparse.lua b/contrib/lua-argparse/argparse.lua
new file mode 100644
index 0000000..6b52962
--- /dev/null
+++ b/contrib/lua-argparse/argparse.lua
@@ -0,0 +1,2100 @@
+-- The MIT License (MIT)
+
+-- Copyright (c) 2013 - 2018 Peter Melnichenko
+-- 2019 Paul Ouellette
+
+-- Permission is hereby granted, free of charge, to any person obtaining a copy of
+-- this software and associated documentation files (the "Software"), to deal in
+-- the Software without restriction, including without limitation the rights to
+-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+-- the Software, and to permit persons to whom the Software is furnished to do so,
+-- subject to the following conditions:
+
+-- The above copyright notice and this permission notice shall be included in all
+-- copies or substantial portions of the Software.
+
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+-- FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+-- COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+-- IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+-- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+local function deep_update(t1, t2)
+ for k, v in pairs(t2) do
+ if type(v) == "table" then
+ v = deep_update({}, v)
+ end
+
+ t1[k] = v
+ end
+
+ return t1
+end
+
+-- A property is a tuple {name, callback}.
+-- properties.args is number of properties that can be set as arguments
+-- when calling an object.
+local function class(prototype, properties, parent)
+ -- Class is the metatable of its instances.
+ local cl = {}
+ cl.__index = cl
+
+ if parent then
+ cl.__prototype = deep_update(deep_update({}, parent.__prototype), prototype)
+ else
+ cl.__prototype = prototype
+ end
+
+ if properties then
+ local names = {}
+
+ -- Create setter methods and fill set of property names.
+ for _, property in ipairs(properties) do
+ local name, callback = property[1], property[2]
+
+ cl[name] = function(self, value)
+ if not callback(self, value) then
+ self["_" .. name] = value
+ end
+
+ return self
+ end
+
+ names[name] = true
+ end
+
+ function cl.__call(self, ...)
+ -- When calling an object, if the first argument is a table,
+ -- interpret keys as property names, else delegate arguments
+ -- to corresponding setters in order.
+ if type((...)) == "table" then
+ for name, value in pairs((...)) do
+ if names[name] then
+ self[name](self, value)
+ end
+ end
+ else
+ local nargs = select("#", ...)
+
+ for i, property in ipairs(properties) do
+ if i > nargs or i > properties.args then
+ break
+ end
+
+ local arg = select(i, ...)
+
+ if arg ~= nil then
+ self[property[1]](self, arg)
+ end
+ end
+ end
+
+ return self
+ end
+ end
+
+ -- If indexing class fails, fallback to its parent.
+ local class_metatable = {}
+ class_metatable.__index = parent
+
+ function class_metatable.__call(self, ...)
+ -- Calling a class returns its instance.
+ -- Arguments are delegated to the instance.
+ local object = deep_update({}, self.__prototype)
+ setmetatable(object, self)
+ return object(...)
+ end
+
+ return setmetatable(cl, class_metatable)
+end
+
+local function typecheck(name, types, value)
+ for _, type_ in ipairs(types) do
+ if type(value) == type_ then
+ return true
+ end
+ end
+
+ error(("bad property '%s' (%s expected, got %s)"):format(name, table.concat(types, " or "), type(value)))
+end
+
+local function typechecked(name, ...)
+ local types = {...}
+ return {name, function(_, value) typecheck(name, types, value) end}
+end
+
+local multiname = {"name", function(self, value)
+ typecheck("name", {"string"}, value)
+
+ for alias in value:gmatch("%S+") do
+ self._name = self._name or alias
+ table.insert(self._aliases, alias)
+ table.insert(self._public_aliases, alias)
+ -- If alias contains '_', accept '-' also.
+ if alias:find("_", 1, true) then
+ table.insert(self._aliases, (alias:gsub("_", "-")))
+ end
+ end
+
+ -- Do not set _name as with other properties.
+ return true
+end}
+
+local multiname_hidden = {"hidden_name", function(self, value)
+ typecheck("hidden_name", {"string"}, value)
+
+ for alias in value:gmatch("%S+") do
+ table.insert(self._aliases, alias)
+ if alias:find("_", 1, true) then
+ table.insert(self._aliases, (alias:gsub("_", "-")))
+ end
+ end
+
+ return true
+end}
+
+local function parse_boundaries(str)
+ if tonumber(str) then
+ return tonumber(str), tonumber(str)
+ end
+
+ if str == "*" then
+ return 0, math.huge
+ end
+
+ if str == "+" then
+ return 1, math.huge
+ end
+
+ if str == "?" then
+ return 0, 1
+ end
+
+ if str:match "^%d+%-%d+$" then
+ local min, max = str:match "^(%d+)%-(%d+)$"
+ return tonumber(min), tonumber(max)
+ end
+
+ if str:match "^%d+%+$" then
+ local min = str:match "^(%d+)%+$"
+ return tonumber(min), math.huge
+ end
+end
+
+local function boundaries(name)
+ return {name, function(self, value)
+ typecheck(name, {"number", "string"}, value)
+
+ local min, max = parse_boundaries(value)
+
+ if not min then
+ error(("bad property '%s'"):format(name))
+ end
+
+ self["_min" .. name], self["_max" .. name] = min, max
+ end}
+end
+
+local actions = {}
+
+local option_action = {"action", function(_, value)
+ typecheck("action", {"function", "string"}, value)
+
+ if type(value) == "string" and not actions[value] then
+ error(("unknown action '%s'"):format(value))
+ end
+end}
+
+local option_init = {"init", function(self)
+ self._has_init = true
+end}
+
+local option_default = {"default", function(self, value)
+ if type(value) ~= "string" then
+ self._init = value
+ self._has_init = true
+ return true
+ end
+end}
+
+local add_help = {"add_help", function(self, value)
+ typecheck("add_help", {"boolean", "string", "table"}, value)
+
+ if self._help_option_idx then
+ table.remove(self._options, self._help_option_idx)
+ self._help_option_idx = nil
+ end
+
+ if value then
+ local help = self:flag()
+ :description "Show this help message and exit."
+ :action(function()
+ print(self:get_help())
+ os.exit(0)
+ end)
+
+ if value ~= true then
+ help = help(value)
+ end
+
+ if not help._name then
+ help "-h" "--help"
+ end
+
+ self._help_option_idx = #self._options
+ end
+end}
+
+local Parser = class({
+ _arguments = {},
+ _options = {},
+ _commands = {},
+ _mutexes = {},
+ _groups = {},
+ _require_command = true,
+ _handle_options = true
+}, {
+ args = 3,
+ typechecked("name", "string"),
+ typechecked("description", "string"),
+ typechecked("epilog", "string"),
+ typechecked("usage", "string"),
+ typechecked("help", "string"),
+ typechecked("require_command", "boolean"),
+ typechecked("handle_options", "boolean"),
+ typechecked("action", "function"),
+ typechecked("command_target", "string"),
+ typechecked("help_vertical_space", "number"),
+ typechecked("usage_margin", "number"),
+ typechecked("usage_max_width", "number"),
+ typechecked("help_usage_margin", "number"),
+ typechecked("help_description_margin", "number"),
+ typechecked("help_max_width", "number"),
+ add_help
+})
+
+local Command = class({
+ _aliases = {},
+ _public_aliases = {}
+}, {
+ args = 3,
+ multiname,
+ typechecked("description", "string"),
+ typechecked("epilog", "string"),
+ multiname_hidden,
+ typechecked("summary", "string"),
+ typechecked("target", "string"),
+ typechecked("usage", "string"),
+ typechecked("help", "string"),
+ typechecked("require_command", "boolean"),
+ typechecked("handle_options", "boolean"),
+ typechecked("action", "function"),
+ typechecked("command_target", "string"),
+ typechecked("help_vertical_space", "number"),
+ typechecked("usage_margin", "number"),
+ typechecked("usage_max_width", "number"),
+ typechecked("help_usage_margin", "number"),
+ typechecked("help_description_margin", "number"),
+ typechecked("help_max_width", "number"),
+ typechecked("hidden", "boolean"),
+ add_help
+}, Parser)
+
+local Argument = class({
+ _minargs = 1,
+ _maxargs = 1,
+ _mincount = 1,
+ _maxcount = 1,
+ _defmode = "unused",
+ _show_default = true
+}, {
+ args = 5,
+ typechecked("name", "string"),
+ typechecked("description", "string"),
+ option_default,
+ typechecked("convert", "function", "table"),
+ boundaries("args"),
+ typechecked("target", "string"),
+ typechecked("defmode", "string"),
+ typechecked("show_default", "boolean"),
+ typechecked("argname", "string", "table"),
+ typechecked("choices", "table"),
+ typechecked("hidden", "boolean"),
+ option_action,
+ option_init
+})
+
+local Option = class({
+ _aliases = {},
+ _public_aliases = {},
+ _mincount = 0,
+ _overwrite = true
+}, {
+ args = 6,
+ multiname,
+ typechecked("description", "string"),
+ option_default,
+ typechecked("convert", "function", "table"),
+ boundaries("args"),
+ boundaries("count"),
+ multiname_hidden,
+ typechecked("target", "string"),
+ typechecked("defmode", "string"),
+ typechecked("show_default", "boolean"),
+ typechecked("overwrite", "boolean"),
+ typechecked("argname", "string", "table"),
+ typechecked("choices", "table"),
+ typechecked("hidden", "boolean"),
+ option_action,
+ option_init
+}, Argument)
+
+function Parser:_inherit_property(name, default)
+ local element = self
+
+ while true do
+ local value = element["_" .. name]
+
+ if value ~= nil then
+ return value
+ end
+
+ if not element._parent then
+ return default
+ end
+
+ element = element._parent
+ end
+end
+
+function Argument:_get_argument_list()
+ local buf = {}
+ local i = 1
+
+ while i <= math.min(self._minargs, 3) do
+ local argname = self:_get_argname(i)
+
+ if self._default and self._defmode:find "a" then
+ argname = "[" .. argname .. "]"
+ end
+
+ table.insert(buf, argname)
+ i = i+1
+ end
+
+ while i <= math.min(self._maxargs, 3) do
+ table.insert(buf, "[" .. self:_get_argname(i) .. "]")
+ i = i+1
+
+ if self._maxargs == math.huge then
+ break
+ end
+ end
+
+ if i < self._maxargs then
+ table.insert(buf, "...")
+ end
+
+ return buf
+end
+
+function Argument:_get_usage()
+ local usage = table.concat(self:_get_argument_list(), " ")
+
+ if self._default and self._defmode:find "u" then
+ if self._maxargs > 1 or (self._minargs == 1 and not self._defmode:find "a") then
+ usage = "[" .. usage .. "]"
+ end
+ end
+
+ return usage
+end
+
+function actions.store_true(result, target)
+ result[target] = true
+end
+
+function actions.store_false(result, target)
+ result[target] = false
+end
+
+function actions.store(result, target, argument)
+ result[target] = argument
+end
+
+function actions.count(result, target, _, overwrite)
+ if not overwrite then
+ result[target] = result[target] + 1
+ end
+end
+
+function actions.append(result, target, argument, overwrite)
+ result[target] = result[target] or {}
+ table.insert(result[target], argument)
+
+ if overwrite then
+ table.remove(result[target], 1)
+ end
+end
+
+function actions.concat(result, target, arguments, overwrite)
+ if overwrite then
+ error("'concat' action can't handle too many invocations")
+ end
+
+ result[target] = result[target] or {}
+
+ for _, argument in ipairs(arguments) do
+ table.insert(result[target], argument)
+ end
+end
+
+function Argument:_get_action()
+ local action, init
+
+ if self._maxcount == 1 then
+ if self._maxargs == 0 then
+ action, init = "store_true", nil
+ else
+ action, init = "store", nil
+ end
+ else
+ if self._maxargs == 0 then
+ action, init = "count", 0
+ else
+ action, init = "append", {}
+ end
+ end
+
+ if self._action then
+ action = self._action
+ end
+
+ if self._has_init then
+ init = self._init
+ end
+
+ if type(action) == "string" then
+ action = actions[action]
+ end
+
+ return action, init
+end
+
+-- Returns placeholder for `narg`-th argument.
+function Argument:_get_argname(narg)
+ local argname = self._argname or self:_get_default_argname()
+
+ if type(argname) == "table" then
+ return argname[narg]
+ else
+ return argname
+ end
+end
+
+function Argument:_get_choices_list()
+ return "{" .. table.concat(self._choices, ",") .. "}"
+end
+
+function Argument:_get_default_argname()
+ if self._choices then
+ return self:_get_choices_list()
+ else
+ return "<" .. self._name .. ">"
+ end
+end
+
+function Option:_get_default_argname()
+ if self._choices then
+ return self:_get_choices_list()
+ else
+ return "<" .. self:_get_default_target() .. ">"
+ end
+end
+
+-- Returns labels to be shown in the help message.
+function Argument:_get_label_lines()
+ if self._choices then
+ return {self:_get_choices_list()}
+ else
+ return {self._name}
+ end
+end
+
+function Option:_get_label_lines()
+ local argument_list = self:_get_argument_list()
+
+ if #argument_list == 0 then
+ -- Don't put aliases for simple flags like `-h` on different lines.
+ return {table.concat(self._public_aliases, ", ")}
+ end
+
+ local longest_alias_length = -1
+
+ for _, alias in ipairs(self._public_aliases) do
+ longest_alias_length = math.max(longest_alias_length, #alias)
+ end
+
+ local argument_list_repr = table.concat(argument_list, " ")
+ local lines = {}
+
+ for i, alias in ipairs(self._public_aliases) do
+ local line = (" "):rep(longest_alias_length - #alias) .. alias .. " " .. argument_list_repr
+
+ if i ~= #self._public_aliases then
+ line = line .. ","
+ end
+
+ table.insert(lines, line)
+ end
+
+ return lines
+end
+
+function Command:_get_label_lines()
+ return {table.concat(self._public_aliases, ", ")}
+end
+
+function Argument:_get_description()
+ if self._default and self._show_default then
+ if self._description then
+ return ("%s (default: %s)"):format(self._description, self._default)
+ else
+ return ("default: %s"):format(self._default)
+ end
+ else
+ return self._description or ""
+ end
+end
+
+function Command:_get_description()
+ return self._summary or self._description or ""
+end
+
+function Option:_get_usage()
+ local usage = self:_get_argument_list()
+ table.insert(usage, 1, self._name)
+ usage = table.concat(usage, " ")
+
+ if self._mincount == 0 or self._default then
+ usage = "[" .. usage .. "]"
+ end
+
+ return usage
+end
+
+function Argument:_get_default_target()
+ return self._name
+end
+
+function Option:_get_default_target()
+ local res
+
+ for _, alias in ipairs(self._public_aliases) do
+ if alias:sub(1, 1) == alias:sub(2, 2) then
+ res = alias:sub(3)
+ break
+ end
+ end
+
+ res = res or self._name:sub(2)
+ return (res:gsub("-", "_"))
+end
+
+function Option:_is_vararg()
+ return self._maxargs ~= self._minargs
+end
+
+function Parser:_get_fullname(exclude_root)
+ local parent = self._parent
+ if exclude_root and not parent then
+ return ""
+ end
+ local buf = {self._name}
+
+ while parent do
+ if not exclude_root or parent._parent then
+ table.insert(buf, 1, parent._name)
+ end
+ parent = parent._parent
+ end
+
+ return table.concat(buf, " ")
+end
+
+function Parser:_update_charset(charset)
+ charset = charset or {}
+
+ for _, command in ipairs(self._commands) do
+ command:_update_charset(charset)
+ end
+
+ for _, option in ipairs(self._options) do
+ for _, alias in ipairs(option._aliases) do
+ charset[alias:sub(1, 1)] = true
+ end
+ end
+
+ return charset
+end
+
+function Parser:argument(...)
+ local argument = Argument(...)
+ table.insert(self._arguments, argument)
+ return argument
+end
+
+function Parser:option(...)
+ local option = Option(...)
+ table.insert(self._options, option)
+ return option
+end
+
+function Parser:flag(...)
+ return self:option():args(0)(...)
+end
+
+function Parser:command(...)
+ local command = Command():add_help(true)(...)
+ command._parent = self
+ table.insert(self._commands, command)
+ return command
+end
+
+function Parser:mutex(...)
+ local elements = {...}
+
+ for i, element in ipairs(elements) do
+ local mt = getmetatable(element)
+ assert(mt == Option or mt == Argument, ("bad argument #%d to 'mutex' (Option or Argument expected)"):format(i))
+ end
+
+ table.insert(self._mutexes, elements)
+ return self
+end
+
+function Parser:group(name, ...)
+ assert(type(name) == "string", ("bad argument #1 to 'group' (string expected, got %s)"):format(type(name)))
+
+ local group = {name = name, ...}
+
+ for i, element in ipairs(group) do
+ local mt = getmetatable(element)
+ assert(mt == Option or mt == Argument or mt == Command,
+ ("bad argument #%d to 'group' (Option or Argument or Command expected)"):format(i + 1))
+ end
+
+ table.insert(self._groups, group)
+ return self
+end
+
+local usage_welcome = "Usage: "
+
+function Parser:get_usage()
+ if self._usage then
+ return self._usage
+ end
+
+ local usage_margin = self:_inherit_property("usage_margin", #usage_welcome)
+ local max_usage_width = self:_inherit_property("usage_max_width", 70)
+ local lines = {usage_welcome .. self:_get_fullname()}
+
+ local function add(s)
+ if #lines[#lines]+1+#s <= max_usage_width then
+ lines[#lines] = lines[#lines] .. " " .. s
+ else
+ lines[#lines+1] = (" "):rep(usage_margin) .. s
+ end
+ end
+
+ -- Normally options are before positional arguments in usage messages.
+ -- However, vararg options should be after, because they can't be reliable used
+ -- before a positional argument.
+ -- Mutexes come into play, too, and are shown as soon as possible.
+ -- Overall, output usages in the following order:
+ -- 1. Mutexes that don't have positional arguments or vararg options.
+ -- 2. Options that are not in any mutexes and are not vararg.
+ -- 3. Positional arguments - on their own or as a part of a mutex.
+ -- 4. Remaining mutexes.
+ -- 5. Remaining options.
+
+ local elements_in_mutexes = {}
+ local added_elements = {}
+ local added_mutexes = {}
+ local argument_to_mutexes = {}
+
+ local function add_mutex(mutex, main_argument)
+ if added_mutexes[mutex] then
+ return
+ end
+
+ added_mutexes[mutex] = true
+ local buf = {}
+
+ for _, element in ipairs(mutex) do
+ if not element._hidden and not added_elements[element] then
+ if getmetatable(element) == Option or element == main_argument then
+ table.insert(buf, element:_get_usage())
+ added_elements[element] = true
+ end
+ end
+ end
+
+ if #buf == 1 then
+ add(buf[1])
+ elseif #buf > 1 then
+ add("(" .. table.concat(buf, " | ") .. ")")
+ end
+ end
+
+ local function add_element(element)
+ if not element._hidden and not added_elements[element] then
+ add(element:_get_usage())
+ added_elements[element] = true
+ end
+ end
+
+ for _, mutex in ipairs(self._mutexes) do
+ local is_vararg = false
+ local has_argument = false
+
+ for _, element in ipairs(mutex) do
+ if getmetatable(element) == Option then
+ if element:_is_vararg() then
+ is_vararg = true
+ end
+ else
+ has_argument = true
+ argument_to_mutexes[element] = argument_to_mutexes[element] or {}
+ table.insert(argument_to_mutexes[element], mutex)
+ end
+
+ elements_in_mutexes[element] = true
+ end
+
+ if not is_vararg and not has_argument then
+ add_mutex(mutex)
+ end
+ end
+
+ for _, option in ipairs(self._options) do
+ if not elements_in_mutexes[option] and not option:_is_vararg() then
+ add_element(option)
+ end
+ end
+
+ -- Add usages for positional arguments, together with one mutex containing them, if they are in a mutex.
+ for _, argument in ipairs(self._arguments) do
+ -- Pick a mutex as a part of which to show this argument, take the first one that's still available.
+ local mutex
+
+ if elements_in_mutexes[argument] then
+ for _, argument_mutex in ipairs(argument_to_mutexes[argument]) do
+ if not added_mutexes[argument_mutex] then
+ mutex = argument_mutex
+ end
+ end
+ end
+
+ if mutex then
+ add_mutex(mutex, argument)
+ else
+ add_element(argument)
+ end
+ end
+
+ for _, mutex in ipairs(self._mutexes) do
+ add_mutex(mutex)
+ end
+
+ for _, option in ipairs(self._options) do
+ add_element(option)
+ end
+
+ if #self._commands > 0 then
+ if self._require_command then
+ add("<command>")
+ else
+ add("[<command>]")
+ end
+
+ add("...")
+ end
+
+ return table.concat(lines, "\n")
+end
+
+local function split_lines(s)
+ if s == "" then
+ return {}
+ end
+
+ local lines = {}
+
+ if s:sub(-1) ~= "\n" then
+ s = s .. "\n"
+ end
+
+ for line in s:gmatch("([^\n]*)\n") do
+ table.insert(lines, line)
+ end
+
+ return lines
+end
+
+local function autowrap_line(line, max_length)
+ -- Algorithm for splitting lines is simple and greedy.
+ local result_lines = {}
+
+ -- Preserve original indentation of the line, put this at the beginning of each result line.
+ -- If the first word looks like a list marker ('*', '+', or '-'), add spaces so that starts
+ -- of the second and the following lines vertically align with the start of the second word.
+ local indentation = line:match("^ *")
+
+ if line:find("^ *[%*%+%-]") then
+ indentation = indentation .. " " .. line:match("^ *[%*%+%-]( *)")
+ end
+
+ -- Parts of the last line being assembled.
+ local line_parts = {}
+
+ -- Length of the current line.
+ local line_length = 0
+
+ -- Index of the next character to consider.
+ local index = 1
+
+ while true do
+ local word_start, word_finish, word = line:find("([^ ]+)", index)
+
+ if not word_start then
+ -- Ignore trailing spaces, if any.
+ break
+ end
+
+ local preceding_spaces = line:sub(index, word_start - 1)
+ index = word_finish + 1
+
+ if (#line_parts == 0) or (line_length + #preceding_spaces + #word <= max_length) then
+ -- Either this is the very first word or it fits as an addition to the current line, add it.
+ table.insert(line_parts, preceding_spaces) -- For the very first word this adds the indentation.
+ table.insert(line_parts, word)
+ line_length = line_length + #preceding_spaces + #word
+ else
+ -- Does not fit, finish current line and put the word into a new one.
+ table.insert(result_lines, table.concat(line_parts))
+ line_parts = {indentation, word}
+ line_length = #indentation + #word
+ end
+ end
+
+ if #line_parts > 0 then
+ table.insert(result_lines, table.concat(line_parts))
+ end
+
+ if #result_lines == 0 then
+ -- Preserve empty lines.
+ result_lines[1] = ""
+ end
+
+ return result_lines
+end
+
+-- Automatically wraps lines within given array,
+-- attempting to limit line length to `max_length`.
+-- Existing line splits are preserved.
+local function autowrap(lines, max_length)
+ local result_lines = {}
+
+ for _, line in ipairs(lines) do
+ local autowrapped_lines = autowrap_line(line, max_length)
+
+ for _, autowrapped_line in ipairs(autowrapped_lines) do
+ table.insert(result_lines, autowrapped_line)
+ end
+ end
+
+ return result_lines
+end
+
+function Parser:_get_element_help(element)
+ local label_lines = element:_get_label_lines()
+ local description_lines = split_lines(element:_get_description())
+
+ local result_lines = {}
+
+ -- All label lines should have the same length (except the last one, it has no comma).
+ -- If too long, start description after all the label lines.
+ -- Otherwise, combine label and description lines.
+
+ local usage_margin_len = self:_inherit_property("help_usage_margin", 3)
+ local usage_margin = (" "):rep(usage_margin_len)
+ local description_margin_len = self:_inherit_property("help_description_margin", 25)
+ local description_margin = (" "):rep(description_margin_len)
+
+ local help_max_width = self:_inherit_property("help_max_width")
+
+ if help_max_width then
+ local description_max_width = math.max(help_max_width - description_margin_len, 10)
+ description_lines = autowrap(description_lines, description_max_width)
+ end
+
+ if #label_lines[1] >= (description_margin_len - usage_margin_len) then
+ for _, label_line in ipairs(label_lines) do
+ table.insert(result_lines, usage_margin .. label_line)
+ end
+
+ for _, description_line in ipairs(description_lines) do
+ table.insert(result_lines, description_margin .. description_line)
+ end
+ else
+ for i = 1, math.max(#label_lines, #description_lines) do
+ local label_line = label_lines[i]
+ local description_line = description_lines[i]
+
+ local line = ""
+
+ if label_line then
+ line = usage_margin .. label_line
+ end
+
+ if description_line and description_line ~= "" then
+ line = line .. (" "):rep(description_margin_len - #line) .. description_line
+ end
+
+ table.insert(result_lines, line)
+ end
+ end
+
+ return table.concat(result_lines, "\n")
+end
+
+local function get_group_types(group)
+ local types = {}
+
+ for _, element in ipairs(group) do
+ types[getmetatable(element)] = true
+ end
+
+ return types
+end
+
+function Parser:_add_group_help(blocks, added_elements, label, elements)
+ local buf = {label}
+
+ for _, element in ipairs(elements) do
+ if not element._hidden and not added_elements[element] then
+ added_elements[element] = true
+ table.insert(buf, self:_get_element_help(element))
+ end
+ end
+
+ if #buf > 1 then
+ table.insert(blocks, table.concat(buf, ("\n"):rep(self:_inherit_property("help_vertical_space", 0) + 1)))
+ end
+end
+
+function Parser:get_help()
+ if self._help then
+ return self._help
+ end
+
+ local blocks = {self:get_usage()}
+
+ local help_max_width = self:_inherit_property("help_max_width")
+
+ if self._description then
+ local description = self._description
+
+ if help_max_width then
+ description = table.concat(autowrap(split_lines(description), help_max_width), "\n")
+ end
+
+ table.insert(blocks, description)
+ end
+
+ -- 1. Put groups containing arguments first, then other arguments.
+ -- 2. Put remaining groups containing options, then other options.
+ -- 3. Put remaining groups containing commands, then other commands.
+ -- Assume that an element can't be in several groups.
+ local groups_by_type = {
+ [Argument] = {},
+ [Option] = {},
+ [Command] = {}
+ }
+
+ for _, group in ipairs(self._groups) do
+ local group_types = get_group_types(group)
+
+ for _, mt in ipairs({Argument, Option, Command}) do
+ if group_types[mt] then
+ table.insert(groups_by_type[mt], group)
+ break
+ end
+ end
+ end
+
+ local default_groups = {
+ {name = "Arguments", type = Argument, elements = self._arguments},
+ {name = "Options", type = Option, elements = self._options},
+ {name = "Commands", type = Command, elements = self._commands}
+ }
+
+ local added_elements = {}
+
+ for _, default_group in ipairs(default_groups) do
+ local type_groups = groups_by_type[default_group.type]
+
+ for _, group in ipairs(type_groups) do
+ self:_add_group_help(blocks, added_elements, group.name .. ":", group)
+ end
+
+ local default_label = default_group.name .. ":"
+
+ if #type_groups > 0 then
+ default_label = "Other " .. default_label:gsub("^.", string.lower)
+ end
+
+ self:_add_group_help(blocks, added_elements, default_label, default_group.elements)
+ end
+
+ if self._epilog then
+ local epilog = self._epilog
+
+ if help_max_width then
+ epilog = table.concat(autowrap(split_lines(epilog), help_max_width), "\n")
+ end
+
+ table.insert(blocks, epilog)
+ end
+
+ return table.concat(blocks, "\n\n")
+end
+
+function Parser:add_help_command(value)
+ if value then
+ assert(type(value) == "string" or type(value) == "table",
+ ("bad argument #1 to 'add_help_command' (string or table expected, got %s)"):format(type(value)))
+ end
+
+ local help = self:command()
+ :description "Show help for commands."
+ help:argument "command"
+ :description "The command to show help for."
+ :args "?"
+ :action(function(_, _, cmd)
+ if not cmd then
+ print(self:get_help())
+ os.exit(0)
+ else
+ for _, command in ipairs(self._commands) do
+ for _, alias in ipairs(command._aliases) do
+ if alias == cmd then
+ print(command:get_help())
+ os.exit(0)
+ end
+ end
+ end
+ end
+ help:error(("unknown command '%s'"):format(cmd))
+ end)
+
+ if value then
+ help = help(value)
+ end
+
+ if not help._name then
+ help "help"
+ end
+
+ help._is_help_command = true
+ return self
+end
+
+function Parser:_is_shell_safe()
+ if self._basename then
+ if self._basename:find("[^%w_%-%+%.]") then
+ return false
+ end
+ else
+ for _, alias in ipairs(self._aliases) do
+ if alias:find("[^%w_%-%+%.]") then
+ return false
+ end
+ end
+ end
+ for _, option in ipairs(self._options) do
+ for _, alias in ipairs(option._aliases) do
+ if alias:find("[^%w_%-%+%.]") then
+ return false
+ end
+ end
+ if option._choices then
+ for _, choice in ipairs(option._choices) do
+ if choice:find("[%s'\"]") then
+ return false
+ end
+ end
+ end
+ end
+ for _, argument in ipairs(self._arguments) do
+ if argument._choices then
+ for _, choice in ipairs(argument._choices) do
+ if choice:find("[%s'\"]") then
+ return false
+ end
+ end
+ end
+ end
+ for _, command in ipairs(self._commands) do
+ if not command:_is_shell_safe() then
+ return false
+ end
+ end
+ return true
+end
+
+function Parser:add_complete(value)
+ if value then
+ assert(type(value) == "string" or type(value) == "table",
+ ("bad argument #1 to 'add_complete' (string or table expected, got %s)"):format(type(value)))
+ end
+
+ local complete = self:option()
+ :description "Output a shell completion script for the specified shell."
+ :args(1)
+ :choices {"bash", "zsh", "fish"}
+ :action(function(_, _, shell)
+ io.write(self["get_" .. shell .. "_complete"](self))
+ os.exit(0)
+ end)
+
+ if value then
+ complete = complete(value)
+ end
+
+ if not complete._name then
+ complete "--completion"
+ end
+
+ return self
+end
+
+function Parser:add_complete_command(value)
+ if value then
+ assert(type(value) == "string" or type(value) == "table",
+ ("bad argument #1 to 'add_complete_command' (string or table expected, got %s)"):format(type(value)))
+ end
+
+ local complete = self:command()
+ :description "Output a shell completion script."
+ complete:argument "shell"
+ :description "The shell to output a completion script for."
+ :choices {"bash", "zsh", "fish"}
+ :action(function(_, _, shell)
+ io.write(self["get_" .. shell .. "_complete"](self))
+ os.exit(0)
+ end)
+
+ if value then
+ complete = complete(value)
+ end
+
+ if not complete._name then
+ complete "completion"
+ end
+
+ return self
+end
+
+local function base_name(pathname)
+ return pathname:gsub("[/\\]*$", ""):match(".*[/\\]([^/\\]*)") or pathname
+end
+
+local function get_short_description(element)
+ local short = element:_get_description():match("^(.-)%.%s")
+ return short or element:_get_description():match("^(.-)%.?$")
+end
+
+function Parser:_get_options()
+ local options = {}
+ for _, option in ipairs(self._options) do
+ for _, alias in ipairs(option._aliases) do
+ table.insert(options, alias)
+ end
+ end
+ return table.concat(options, " ")
+end
+
+function Parser:_get_commands()
+ local commands = {}
+ for _, command in ipairs(self._commands) do
+ for _, alias in ipairs(command._aliases) do
+ table.insert(commands, alias)
+ end
+ end
+ return table.concat(commands, " ")
+end
+
+function Parser:_bash_option_args(buf, indent)
+ local opts = {}
+ for _, option in ipairs(self._options) do
+ if option._choices or option._minargs > 0 then
+ local compreply
+ if option._choices then
+ compreply = 'COMPREPLY=($(compgen -W "' .. table.concat(option._choices, " ") .. '" -- "$cur"))'
+ else
+ compreply = 'COMPREPLY=($(compgen -f -- "$cur"))'
+ end
+ table.insert(opts, (" "):rep(indent + 4) .. table.concat(option._aliases, "|") .. ")")
+ table.insert(opts, (" "):rep(indent + 8) .. compreply)
+ table.insert(opts, (" "):rep(indent + 8) .. "return 0")
+ table.insert(opts, (" "):rep(indent + 8) .. ";;")
+ end
+ end
+
+ if #opts > 0 then
+ table.insert(buf, (" "):rep(indent) .. 'case "$prev" in')
+ table.insert(buf, table.concat(opts, "\n"))
+ table.insert(buf, (" "):rep(indent) .. "esac\n")
+ end
+end
+
+function Parser:_bash_get_cmd(buf, indent)
+ if #self._commands == 0 then
+ return
+ end
+
+ table.insert(buf, (" "):rep(indent) .. 'args=("${args[@]:1}")')
+ table.insert(buf, (" "):rep(indent) .. 'for arg in "${args[@]}"; do')
+ table.insert(buf, (" "):rep(indent + 4) .. 'case "$arg" in')
+
+ for _, command in ipairs(self._commands) do
+ table.insert(buf, (" "):rep(indent + 8) .. table.concat(command._aliases, "|") .. ")")
+ if self._parent then
+ table.insert(buf, (" "):rep(indent + 12) .. 'cmd="$cmd ' .. command._name .. '"')
+ else
+ table.insert(buf, (" "):rep(indent + 12) .. 'cmd="' .. command._name .. '"')
+ end
+ table.insert(buf, (" "):rep(indent + 12) .. 'opts="$opts ' .. command:_get_options() .. '"')
+ command:_bash_get_cmd(buf, indent + 12)
+ table.insert(buf, (" "):rep(indent + 12) .. "break")
+ table.insert(buf, (" "):rep(indent + 12) .. ";;")
+ end
+
+ table.insert(buf, (" "):rep(indent + 4) .. "esac")
+ table.insert(buf, (" "):rep(indent) .. "done")
+end
+
+function Parser:_bash_cmd_completions(buf)
+ local cmd_buf = {}
+ if self._parent then
+ self:_bash_option_args(cmd_buf, 12)
+ end
+ if #self._commands > 0 then
+ table.insert(cmd_buf, (" "):rep(12) .. 'COMPREPLY=($(compgen -W "' .. self:_get_commands() .. '" -- "$cur"))')
+ elseif self._is_help_command then
+ table.insert(cmd_buf, (" "):rep(12)
+ .. 'COMPREPLY=($(compgen -W "'
+ .. self._parent:_get_commands()
+ .. '" -- "$cur"))')
+ end
+ if #cmd_buf > 0 then
+ table.insert(buf, (" "):rep(8) .. "'" .. self:_get_fullname(true) .. "')")
+ table.insert(buf, table.concat(cmd_buf, "\n"))
+ table.insert(buf, (" "):rep(12) .. ";;")
+ end
+
+ for _, command in ipairs(self._commands) do
+ command:_bash_cmd_completions(buf)
+ end
+end
+
+function Parser:get_bash_complete()
+ self._basename = base_name(self._name)
+ assert(self:_is_shell_safe())
+ local buf = {([[
+_%s() {
+ local IFS=$' \t\n'
+ local args cur prev cmd opts arg
+ args=("${COMP_WORDS[@]}")
+ cur="${COMP_WORDS[COMP_CWORD]}"
+ prev="${COMP_WORDS[COMP_CWORD-1]}"
+ opts="%s"
+]]):format(self._basename, self:_get_options())}
+
+ self:_bash_option_args(buf, 4)
+ self:_bash_get_cmd(buf, 4)
+ if #self._commands > 0 then
+ table.insert(buf, "")
+ table.insert(buf, (" "):rep(4) .. 'case "$cmd" in')
+ self:_bash_cmd_completions(buf)
+ table.insert(buf, (" "):rep(4) .. "esac\n")
+ end
+
+ table.insert(buf, ([=[
+ if [[ "$cur" = -* ]]; then
+ COMPREPLY=($(compgen -W "$opts" -- "$cur"))
+ fi
+}
+
+complete -F _%s -o bashdefault -o default %s
+]=]):format(self._basename, self._basename))
+
+ return table.concat(buf, "\n")
+end
+
+function Parser:_zsh_arguments(buf, cmd_name, indent)
+ if self._parent then
+ table.insert(buf, (" "):rep(indent) .. "options=(")
+ table.insert(buf, (" "):rep(indent + 2) .. "$options")
+ else
+ table.insert(buf, (" "):rep(indent) .. "local -a options=(")
+ end
+
+ for _, option in ipairs(self._options) do
+ local line = {}
+ if #option._aliases > 1 then
+ if option._maxcount > 1 then
+ table.insert(line, '"*"')
+ end
+ table.insert(line, "{" .. table.concat(option._aliases, ",") .. '}"')
+ else
+ table.insert(line, '"')
+ if option._maxcount > 1 then
+ table.insert(line, "*")
+ end
+ table.insert(line, option._name)
+ end
+ if option._description then
+ local description = get_short_description(option):gsub('["%]:`$]', "\\%0")
+ table.insert(line, "[" .. description .. "]")
+ end
+ if option._maxargs == math.huge then
+ table.insert(line, ":*")
+ end
+ if option._choices then
+ table.insert(line, ": :(" .. table.concat(option._choices, " ") .. ")")
+ elseif option._maxargs > 0 then
+ table.insert(line, ": :_files")
+ end
+ table.insert(line, '"')
+ table.insert(buf, (" "):rep(indent + 2) .. table.concat(line))
+ end
+
+ table.insert(buf, (" "):rep(indent) .. ")")
+ table.insert(buf, (" "):rep(indent) .. "_arguments -s -S \\")
+ table.insert(buf, (" "):rep(indent + 2) .. "$options \\")
+
+ if self._is_help_command then
+ table.insert(buf, (" "):rep(indent + 2) .. '": :(' .. self._parent:_get_commands() .. ')" \\')
+ else
+ for _, argument in ipairs(self._arguments) do
+ local spec
+ if argument._choices then
+ spec = ": :(" .. table.concat(argument._choices, " ") .. ")"
+ else
+ spec = ": :_files"
+ end
+ if argument._maxargs == math.huge then
+ table.insert(buf, (" "):rep(indent + 2) .. '"*' .. spec .. '" \\')
+ break
+ end
+ for _ = 1, argument._maxargs do
+ table.insert(buf, (" "):rep(indent + 2) .. '"' .. spec .. '" \\')
+ end
+ end
+
+ if #self._commands > 0 then
+ table.insert(buf, (" "):rep(indent + 2) .. '": :_' .. cmd_name .. '_cmds" \\')
+ table.insert(buf, (" "):rep(indent + 2) .. '"*:: :->args" \\')
+ end
+ end
+
+ table.insert(buf, (" "):rep(indent + 2) .. "&& return 0")
+end
+
+function Parser:_zsh_cmds(buf, cmd_name)
+ table.insert(buf, "\n_" .. cmd_name .. "_cmds() {")
+ table.insert(buf, " local -a commands=(")
+
+ for _, command in ipairs(self._commands) do
+ local line = {}
+ if #command._aliases > 1 then
+ table.insert(line, "{" .. table.concat(command._aliases, ",") .. '}"')
+ else
+ table.insert(line, '"' .. command._name)
+ end
+ if command._description then
+ table.insert(line, ":" .. get_short_description(command):gsub('["`$]', "\\%0"))
+ end
+ table.insert(buf, " " .. table.concat(line) .. '"')
+ end
+
+ table.insert(buf, ' )\n _describe "command" commands\n}')
+end
+
+function Parser:_zsh_complete_help(buf, cmds_buf, cmd_name, indent)
+ if #self._commands == 0 then
+ return
+ end
+
+ self:_zsh_cmds(cmds_buf, cmd_name)
+ table.insert(buf, "\n" .. (" "):rep(indent) .. "case $words[1] in")
+
+ for _, command in ipairs(self._commands) do
+ local name = cmd_name .. "_" .. command._name
+ table.insert(buf, (" "):rep(indent + 2) .. table.concat(command._aliases, "|") .. ")")
+ command:_zsh_arguments(buf, name, indent + 4)
+ command:_zsh_complete_help(buf, cmds_buf, name, indent + 4)
+ table.insert(buf, (" "):rep(indent + 4) .. ";;\n")
+ end
+
+ table.insert(buf, (" "):rep(indent) .. "esac")
+end
+
+function Parser:get_zsh_complete()
+ self._basename = base_name(self._name)
+ assert(self:_is_shell_safe())
+ local buf = {("#compdef %s\n"):format(self._basename)}
+ local cmds_buf = {}
+ table.insert(buf, "_" .. self._basename .. "() {")
+ if #self._commands > 0 then
+ table.insert(buf, " local context state state_descr line")
+ table.insert(buf, " typeset -A opt_args\n")
+ end
+ self:_zsh_arguments(buf, self._basename, 2)
+ self:_zsh_complete_help(buf, cmds_buf, self._basename, 2)
+ table.insert(buf, "\n return 1")
+ table.insert(buf, "}")
+
+ local result = table.concat(buf, "\n")
+ if #cmds_buf > 0 then
+ result = result .. "\n" .. table.concat(cmds_buf, "\n")
+ end
+ return result .. "\n\n_" .. self._basename .. "\n"
+end
+
+local function fish_escape(string)
+ return string:gsub("[\\']", "\\%0")
+end
+
+function Parser:_fish_get_cmd(buf, indent)
+ if #self._commands == 0 then
+ return
+ end
+
+ table.insert(buf, (" "):rep(indent) .. "set -e cmdline[1]")
+ table.insert(buf, (" "):rep(indent) .. "for arg in $cmdline")
+ table.insert(buf, (" "):rep(indent + 4) .. "switch $arg")
+
+ for _, command in ipairs(self._commands) do
+ table.insert(buf, (" "):rep(indent + 8) .. "case " .. table.concat(command._aliases, " "))
+ table.insert(buf, (" "):rep(indent + 12) .. "set cmd $cmd " .. command._name)
+ command:_fish_get_cmd(buf, indent + 12)
+ table.insert(buf, (" "):rep(indent + 12) .. "break")
+ end
+
+ table.insert(buf, (" "):rep(indent + 4) .. "end")
+ table.insert(buf, (" "):rep(indent) .. "end")
+end
+
+function Parser:_fish_complete_help(buf, basename)
+ local prefix = "complete -c " .. basename
+ table.insert(buf, "")
+
+ for _, command in ipairs(self._commands) do
+ local aliases = table.concat(command._aliases, " ")
+ local line
+ if self._parent then
+ line = ("%s -n '__fish_%s_using_command %s' -xa '%s'")
+ :format(prefix, basename, self:_get_fullname(true), aliases)
+ else
+ line = ("%s -n '__fish_%s_using_command' -xa '%s'"):format(prefix, basename, aliases)
+ end
+ if command._description then
+ line = ("%s -d '%s'"):format(line, fish_escape(get_short_description(command)))
+ end
+ table.insert(buf, line)
+ end
+
+ if self._is_help_command then
+ local line = ("%s -n '__fish_%s_using_command %s' -xa '%s'")
+ :format(prefix, basename, self:_get_fullname(true), self._parent:_get_commands())
+ table.insert(buf, line)
+ end
+
+ for _, option in ipairs(self._options) do
+ local parts = {prefix}
+
+ if self._parent then
+ table.insert(parts, "-n '__fish_" .. basename .. "_seen_command " .. self:_get_fullname(true) .. "'")
+ end
+
+ for _, alias in ipairs(option._aliases) do
+ if alias:match("^%-.$") then
+ table.insert(parts, "-s " .. alias:sub(2))
+ elseif alias:match("^%-%-.+") then
+ table.insert(parts, "-l " .. alias:sub(3))
+ end
+ end
+
+ if option._choices then
+ table.insert(parts, "-xa '" .. table.concat(option._choices, " ") .. "'")
+ elseif option._minargs > 0 then
+ table.insert(parts, "-r")
+ end
+
+ if option._description then
+ table.insert(parts, "-d '" .. fish_escape(get_short_description(option)) .. "'")
+ end
+
+ table.insert(buf, table.concat(parts, " "))
+ end
+
+ for _, command in ipairs(self._commands) do
+ command:_fish_complete_help(buf, basename)
+ end
+end
+
+function Parser:get_fish_complete()
+ self._basename = base_name(self._name)
+ assert(self:_is_shell_safe())
+ local buf = {}
+
+ if #self._commands > 0 then
+ table.insert(buf, ([[
+function __fish_%s_print_command
+ set -l cmdline (commandline -poc)
+ set -l cmd]]):format(self._basename))
+ self:_fish_get_cmd(buf, 4)
+ table.insert(buf, ([[
+ echo "$cmd"
+end
+
+function __fish_%s_using_command
+ test (__fish_%s_print_command) = "$argv"
+ and return 0
+ or return 1
+end
+
+function __fish_%s_seen_command
+ string match -q "$argv*" (__fish_%s_print_command)
+ and return 0
+ or return 1
+end]]):format(self._basename, self._basename, self._basename, self._basename))
+ end
+
+ self:_fish_complete_help(buf, self._basename)
+ return table.concat(buf, "\n") .. "\n"
+end
+
+local function get_tip(context, wrong_name)
+ local context_pool = {}
+ local possible_name
+ local possible_names = {}
+
+ for name in pairs(context) do
+ if type(name) == "string" then
+ for i = 1, #name do
+ possible_name = name:sub(1, i - 1) .. name:sub(i + 1)
+
+ if not context_pool[possible_name] then
+ context_pool[possible_name] = {}
+ end
+
+ table.insert(context_pool[possible_name], name)
+ end
+ end
+ end
+
+ for i = 1, #wrong_name + 1 do
+ possible_name = wrong_name:sub(1, i - 1) .. wrong_name:sub(i + 1)
+
+ if context[possible_name] then
+ possible_names[possible_name] = true
+ elseif context_pool[possible_name] then
+ for _, name in ipairs(context_pool[possible_name]) do
+ possible_names[name] = true
+ end
+ end
+ end
+
+ local first = next(possible_names)
+
+ if first then
+ if next(possible_names, first) then
+ local possible_names_arr = {}
+
+ for name in pairs(possible_names) do
+ table.insert(possible_names_arr, "'" .. name .. "'")
+ end
+
+ table.sort(possible_names_arr)
+ return "\nDid you mean one of these: " .. table.concat(possible_names_arr, " ") .. "?"
+ else
+ return "\nDid you mean '" .. first .. "'?"
+ end
+ else
+ return ""
+ end
+end
+
+local ElementState = class({
+ invocations = 0
+})
+
+function ElementState:__call(state, element)
+ self.state = state
+ self.result = state.result
+ self.element = element
+ self.target = element._target or element:_get_default_target()
+ self.action, self.result[self.target] = element:_get_action()
+ return self
+end
+
+function ElementState:error(fmt, ...)
+ self.state:error(fmt, ...)
+end
+
+function ElementState:convert(argument, index)
+ local converter = self.element._convert
+
+ if converter then
+ local ok, err
+
+ if type(converter) == "function" then
+ ok, err = converter(argument)
+ elseif type(converter[index]) == "function" then
+ ok, err = converter[index](argument)
+ else
+ ok = converter[argument]
+ end
+
+ if ok == nil then
+ self:error(err and "%s" or "malformed argument '%s'", err or argument)
+ end
+
+ argument = ok
+ end
+
+ return argument
+end
+
+function ElementState:default(mode)
+ return self.element._defmode:find(mode) and self.element._default
+end
+
+local function bound(noun, min, max, is_max)
+ local res = ""
+
+ if min ~= max then
+ res = "at " .. (is_max and "most" or "least") .. " "
+ end
+
+ local number = is_max and max or min
+ return res .. tostring(number) .. " " .. noun .. (number == 1 and "" or "s")
+end
+
+function ElementState:set_name(alias)
+ self.name = ("%s '%s'"):format(alias and "option" or "argument", alias or self.element._name)
+end
+
+function ElementState:invoke()
+ self.open = true
+ self.overwrite = false
+
+ if self.invocations >= self.element._maxcount then
+ if self.element._overwrite then
+ self.overwrite = true
+ else
+ local num_times_repr = bound("time", self.element._mincount, self.element._maxcount, true)
+ self:error("%s must be used %s", self.name, num_times_repr)
+ end
+ else
+ self.invocations = self.invocations + 1
+ end
+
+ self.args = {}
+
+ if self.element._maxargs <= 0 then
+ self:close()
+ end
+
+ return self.open
+end
+
+function ElementState:check_choices(argument)
+ if self.element._choices then
+ for _, choice in ipairs(self.element._choices) do
+ if argument == choice then
+ return
+ end
+ end
+ local choices_list = "'" .. table.concat(self.element._choices, "', '") .. "'"
+ local is_option = getmetatable(self.element) == Option
+ self:error("%s%s must be one of %s", is_option and "argument for " or "", self.name, choices_list)
+ end
+end
+
+function ElementState:pass(argument)
+ self:check_choices(argument)
+ argument = self:convert(argument, #self.args + 1)
+ table.insert(self.args, argument)
+
+ if #self.args >= self.element._maxargs then
+ self:close()
+ end
+
+ return self.open
+end
+
+function ElementState:complete_invocation()
+ while #self.args < self.element._minargs do
+ self:pass(self.element._default)
+ end
+end
+
+function ElementState:close()
+ if self.open then
+ self.open = false
+
+ if #self.args < self.element._minargs then
+ if self:default("a") then
+ self:complete_invocation()
+ else
+ if #self.args == 0 then
+ if getmetatable(self.element) == Argument then
+ self:error("missing %s", self.name)
+ elseif self.element._maxargs == 1 then
+ self:error("%s requires an argument", self.name)
+ end
+ end
+
+ self:error("%s requires %s", self.name, bound("argument", self.element._minargs, self.element._maxargs))
+ end
+ end
+
+ local args
+
+ if self.element._maxargs == 0 then
+ args = self.args[1]
+ elseif self.element._maxargs == 1 then
+ if self.element._minargs == 0 and self.element._mincount ~= self.element._maxcount then
+ args = self.args
+ else
+ args = self.args[1]
+ end
+ else
+ args = self.args
+ end
+
+ self.action(self.result, self.target, args, self.overwrite)
+ end
+end
+
+local ParseState = class({
+ result = {},
+ options = {},
+ arguments = {},
+ argument_i = 1,
+ element_to_mutexes = {},
+ mutex_to_element_state = {},
+ command_actions = {}
+})
+
+function ParseState:__call(parser, error_handler)
+ self.parser = parser
+ self.error_handler = error_handler
+ self.charset = parser:_update_charset()
+ self:switch(parser)
+ return self
+end
+
+function ParseState:error(fmt, ...)
+ self.error_handler(self.parser, fmt:format(...))
+end
+
+function ParseState:switch(parser)
+ self.parser = parser
+
+ if parser._action then
+ table.insert(self.command_actions, {action = parser._action, name = parser._name})
+ end
+
+ for _, option in ipairs(parser._options) do
+ option = ElementState(self, option)
+ table.insert(self.options, option)
+
+ for _, alias in ipairs(option.element._aliases) do
+ self.options[alias] = option
+ end
+ end
+
+ for _, mutex in ipairs(parser._mutexes) do
+ for _, element in ipairs(mutex) do
+ if not self.element_to_mutexes[element] then
+ self.element_to_mutexes[element] = {}
+ end
+
+ table.insert(self.element_to_mutexes[element], mutex)
+ end
+ end
+
+ for _, argument in ipairs(parser._arguments) do
+ argument = ElementState(self, argument)
+ table.insert(self.arguments, argument)
+ argument:set_name()
+ argument:invoke()
+ end
+
+ self.handle_options = parser._handle_options
+ self.argument = self.arguments[self.argument_i]
+ self.commands = parser._commands
+
+ for _, command in ipairs(self.commands) do
+ for _, alias in ipairs(command._aliases) do
+ self.commands[alias] = command
+ end
+ end
+end
+
+function ParseState:get_option(name)
+ local option = self.options[name]
+
+ if not option then
+ self:error("unknown option '%s'%s", name, get_tip(self.options, name))
+ else
+ return option
+ end
+end
+
+function ParseState:get_command(name)
+ local command = self.commands[name]
+
+ if not command then
+ if #self.commands > 0 then
+ self:error("unknown command '%s'%s", name, get_tip(self.commands, name))
+ else
+ self:error("too many arguments")
+ end
+ else
+ return command
+ end
+end
+
+function ParseState:check_mutexes(element_state)
+ if self.element_to_mutexes[element_state.element] then
+ for _, mutex in ipairs(self.element_to_mutexes[element_state.element]) do
+ local used_element_state = self.mutex_to_element_state[mutex]
+
+ if used_element_state and used_element_state ~= element_state then
+ self:error("%s can not be used together with %s", element_state.name, used_element_state.name)
+ else
+ self.mutex_to_element_state[mutex] = element_state
+ end
+ end
+ end
+end
+
+function ParseState:invoke(option, name)
+ self:close()
+ option:set_name(name)
+ self:check_mutexes(option, name)
+
+ if option:invoke() then
+ self.option = option
+ end
+end
+
+function ParseState:pass(arg)
+ if self.option then
+ if not self.option:pass(arg) then
+ self.option = nil
+ end
+ elseif self.argument then
+ self:check_mutexes(self.argument)
+
+ if not self.argument:pass(arg) then
+ self.argument_i = self.argument_i + 1
+ self.argument = self.arguments[self.argument_i]
+ end
+ else
+ local command = self:get_command(arg)
+ self.result[command._target or command._name] = true
+
+ if self.parser._command_target then
+ self.result[self.parser._command_target] = command._name
+ end
+
+ self:switch(command)
+ end
+end
+
+function ParseState:close()
+ if self.option then
+ self.option:close()
+ self.option = nil
+ end
+end
+
+function ParseState:finalize()
+ self:close()
+
+ for i = self.argument_i, #self.arguments do
+ local argument = self.arguments[i]
+ if #argument.args == 0 and argument:default("u") then
+ argument:complete_invocation()
+ else
+ argument:close()
+ end
+ end
+
+ if self.parser._require_command and #self.commands > 0 then
+ self:error("a command is required")
+ end
+
+ for _, option in ipairs(self.options) do
+ option.name = option.name or ("option '%s'"):format(option.element._name)
+
+ if option.invocations == 0 then
+ if option:default("u") then
+ option:invoke()
+ option:complete_invocation()
+ option:close()
+ end
+ end
+
+ local mincount = option.element._mincount
+
+ if option.invocations < mincount then
+ if option:default("a") then
+ while option.invocations < mincount do
+ option:invoke()
+ option:close()
+ end
+ elseif option.invocations == 0 then
+ self:error("missing %s", option.name)
+ else
+ self:error("%s must be used %s", option.name, bound("time", mincount, option.element._maxcount))
+ end
+ end
+ end
+
+ for i = #self.command_actions, 1, -1 do
+ self.command_actions[i].action(self.result, self.command_actions[i].name)
+ end
+end
+
+function ParseState:parse(args)
+ for _, arg in ipairs(args) do
+ local plain = true
+
+ if self.handle_options then
+ local first = arg:sub(1, 1)
+
+ if self.charset[first] then
+ if #arg > 1 then
+ plain = false
+
+ if arg:sub(2, 2) == first then
+ if #arg == 2 then
+ if self.options[arg] then
+ local option = self:get_option(arg)
+ self:invoke(option, arg)
+ else
+ self:close()
+ end
+
+ self.handle_options = false
+ else
+ local equals = arg:find "="
+ if equals then
+ local name = arg:sub(1, equals - 1)
+ local option = self:get_option(name)
+
+ if option.element._maxargs <= 0 then
+ self:error("option '%s' does not take arguments", name)
+ end
+
+ self:invoke(option, name)
+ self:pass(arg:sub(equals + 1))
+ else
+ local option = self:get_option(arg)
+ self:invoke(option, arg)
+ end
+ end
+ else
+ for i = 2, #arg do
+ local name = first .. arg:sub(i, i)
+ local option = self:get_option(name)
+ self:invoke(option, name)
+
+ if i ~= #arg and option.element._maxargs > 0 then
+ self:pass(arg:sub(i + 1))
+ break
+ end
+ end
+ end
+ end
+ end
+ end
+
+ if plain then
+ self:pass(arg)
+ end
+ end
+
+ self:finalize()
+ return self.result
+end
+
+function Parser:error(msg)
+ io.stderr:write(("%s\n\nError: %s\n"):format(self:get_usage(), msg))
+ os.exit(1)
+end
+
+-- Compatibility with strict.lua and other checkers:
+local default_cmdline = rawget(_G, "arg") or {}
+
+function Parser:_parse(args, error_handler)
+ return ParseState(self, error_handler):parse(args or default_cmdline)
+end
+
+function Parser:parse(args)
+ return self:_parse(args, self.error)
+end
+
+local function xpcall_error_handler(err)
+ return tostring(err) .. "\noriginal " .. debug.traceback("", 2):sub(2)
+end
+
+function Parser:pparse(args)
+ local parse_error
+
+ local ok, result = xpcall(function()
+ return self:_parse(args, function(_, err)
+ parse_error = err
+ error(err, 0)
+ end)
+ end, xpcall_error_handler)
+
+ if ok then
+ return true, result
+ elseif not parse_error then
+ error(result, 0)
+ else
+ return false, parse_error
+ end
+end
+
+local argparse = {}
+
+argparse.version = "0.7.1"
+
+setmetatable(argparse, {__call = function(_, ...)
+ return Parser(default_cmdline[0]):add_help(true)(...)
+end})
+
+return argparse
diff --git a/contrib/lua-bit/CMakeLists.txt b/contrib/lua-bit/CMakeLists.txt
new file mode 100644
index 0000000..8fac490
--- /dev/null
+++ b/contrib/lua-bit/CMakeLists.txt
@@ -0,0 +1,4 @@
+SET(BITSRC bit.c)
+
+SET(LIB_TYPE STATIC)
+ADD_LIBRARY(rspamd-bit ${LIB_TYPE} ${BITSRC})
diff --git a/contrib/lua-bit/bit.c b/contrib/lua-bit/bit.c
new file mode 100644
index 0000000..01326c9
--- /dev/null
+++ b/contrib/lua-bit/bit.c
@@ -0,0 +1,198 @@
+/*
+** Lua BitOp -- a bit operations library for Lua 5.1/5.2.
+** http://bitop.luajit.org/
+**
+** Copyright (C) 2008-2012 Mike Pall. All rights reserved.
+**
+** Permission is hereby granted, free of charge, to any person obtaining
+** a copy of this software and associated documentation files (the
+** "Software"), to deal in the Software without restriction, including
+** without limitation the rights to use, copy, modify, merge, publish,
+** distribute, sublicense, and/or sell copies of the Software, and to
+** permit persons to whom the Software is furnished to do so, subject to
+** the following conditions:
+**
+** The above copyright notice and this permission notice shall be
+** included in all copies or substantial portions of the Software.
+**
+** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+**
+** [ MIT license: http://www.opensource.org/licenses/mit-license.php ]
+*/
+
+#define LUA_BITOP_VERSION "1.0.2"
+
+#define LUA_LIB
+#include "lua.h"
+#include "lauxlib.h"
+
+#ifdef _MSC_VER
+/* MSVC is stuck in the last century and doesn't have C99's stdint.h. */
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+typedef int32_t SBits;
+typedef uint32_t UBits;
+
+typedef union {
+ lua_Number n;
+#if defined(LUA_NUMBER_DOUBLE) || defined(LUA_FLOAT_DOUBLE)
+ uint64_t b;
+#else
+ UBits b;
+#endif
+} BitNum;
+
+/* Convert argument to bit type. */
+static UBits barg(lua_State *L, int idx)
+{
+ BitNum bn;
+ UBits b;
+#if LUA_VERSION_NUM < 502
+ bn.n = lua_tonumber(L, idx);
+#else
+ bn.n = luaL_checknumber(L, idx);
+#endif
+#if defined(LUA_NUMBER_DOUBLE) || defined(LUA_FLOAT_DOUBLE)
+ bn.n += 6755399441055744.0; /* 2^52+2^51 */
+#ifdef SWAPPED_DOUBLE
+ b = (UBits)(bn.b >> 32);
+#else
+ b = (UBits)bn.b;
+#endif
+#elif defined(LUA_NUMBER_INT) || defined(LUA_INT_INT) || \
+ defined(LUA_NUMBER_LONG) || defined(LUA_INT_LONG) || \
+ defined(LUA_NUMBER_LONGLONG) || defined(LUA_INT_LONGLONG) || \
+ defined(LUA_NUMBER_LONG_LONG) || defined(LUA_NUMBER_LLONG)
+ if (sizeof(UBits) == sizeof(lua_Number))
+ b = bn.b;
+ else
+ b = (UBits)(SBits)bn.n;
+#elif defined(LUA_NUMBER_FLOAT) || defined(LUA_FLOAT_FLOAT)
+#error "A 'float' lua_Number type is incompatible with this library"
+#else
+#error "Unknown number type, check LUA_NUMBER_*, LUA_FLOAT_*, LUA_INT_* in luaconf.h"
+#endif
+#if LUA_VERSION_NUM < 502
+ if (b == 0 && !lua_isnumber(L, idx)) {
+ luaL_typerror(L, idx, "number");
+ }
+#endif
+ return b;
+}
+
+/* Return bit type. */
+#if LUA_VERSION_NUM < 503
+#define BRET(b) lua_pushnumber(L, (lua_Number)(SBits)(b)); return 1;
+#else
+#define BRET(b) lua_pushinteger(L, (lua_Integer)(SBits)(b)); return 1;
+#endif
+
+static int bit_tobit(lua_State *L) { BRET(barg(L, 1)) }
+static int bit_bnot(lua_State *L) { BRET(~barg(L, 1)) }
+
+#define BIT_OP(func, opr) \
+ static int func(lua_State *L) { int i; UBits b = barg(L, 1); \
+ for (i = lua_gettop(L); i > 1; i--) b opr barg(L, i); BRET(b) }
+BIT_OP(bit_band, &=)
+BIT_OP(bit_bor, |=)
+BIT_OP(bit_bxor, ^=)
+
+#define bshl(b, n) (b << n)
+#define bshr(b, n) (b >> n)
+#define bsar(b, n) ((SBits)b >> n)
+#define brol(b, n) ((b << n) | (b >> (32-n)))
+#define bror(b, n) ((b << (32-n)) | (b >> n))
+#define BIT_SH(func, fn) \
+ static int func(lua_State *L) { \
+ UBits b = barg(L, 1); UBits n = barg(L, 2) & 31; BRET(fn(b, n)) }
+BIT_SH(bit_lshift, bshl)
+BIT_SH(bit_rshift, bshr)
+BIT_SH(bit_arshift, bsar)
+BIT_SH(bit_rol, brol)
+BIT_SH(bit_ror, bror)
+
+static int bit_bswap(lua_State *L)
+{
+ UBits b = barg(L, 1);
+ b = (b >> 24) | ((b >> 8) & 0xff00) | ((b & 0xff00) << 8) | (b << 24);
+ BRET(b)
+}
+
+static int bit_tohex(lua_State *L)
+{
+ UBits b = barg(L, 1);
+ SBits n = lua_isnone(L, 2) ? 8 : (SBits)barg(L, 2);
+ const char *hexdigits = "0123456789abcdef";
+ char buf[8];
+ int i;
+ if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; }
+ if (n > 8) n = 8;
+ for (i = (int)n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; }
+ lua_pushlstring(L, buf, (size_t)n);
+ return 1;
+}
+
+static const struct luaL_Reg bit_funcs[] = {
+ { "tobit", bit_tobit },
+ { "bnot", bit_bnot },
+ { "band", bit_band },
+ { "bor", bit_bor },
+ { "bxor", bit_bxor },
+ { "lshift", bit_lshift },
+ { "rshift", bit_rshift },
+ { "arshift", bit_arshift },
+ { "rol", bit_rol },
+ { "ror", bit_ror },
+ { "bswap", bit_bswap },
+ { "tohex", bit_tohex },
+ { NULL, NULL }
+};
+
+/* Signed right-shifts are implementation-defined per C89/C99.
+** But the de facto standard are arithmetic right-shifts on two's
+** complement CPUs. This behaviour is required here, so test for it.
+*/
+#define BAD_SAR (bsar(-8, 2) != (SBits)-2)
+
+LUALIB_API int luaopen_bit(lua_State *L)
+{
+ UBits b;
+#if LUA_VERSION_NUM < 503
+ lua_pushnumber(L, (lua_Number)1437217655L);
+#else
+ lua_pushinteger(L, (lua_Integer)1437217655L);
+#endif
+ b = barg(L, -1);
+ if (b != (UBits)1437217655L || BAD_SAR) { /* Perform a simple self-test. */
+ const char *msg = "compiled with incompatible luaconf.h";
+#if defined(LUA_NUMBER_DOUBLE) || defined(LUA_FLOAT_DOUBLE)
+#ifdef _WIN32
+ if (b == (UBits)1610612736L)
+ msg = "use D3DCREATE_FPU_PRESERVE with DirectX";
+#endif
+ if (b == (UBits)1127743488L)
+ msg = "not compiled with SWAPPED_DOUBLE";
+#endif
+ if (BAD_SAR)
+ msg = "arithmetic right-shift broken";
+ luaL_error(L, "bit library self-test failed (%s)", msg);
+ }
+#if LUA_VERSION_NUM < 502
+ luaL_register(L, "bit", bit_funcs);
+#else
+ luaL_newlib(L, bit_funcs);
+#endif
+ return 1;
+}
+
diff --git a/contrib/lua-fun/COPYING.md b/contrib/lua-fun/COPYING.md
new file mode 100644
index 0000000..42ee231
--- /dev/null
+++ b/contrib/lua-fun/COPYING.md
@@ -0,0 +1,27 @@
+Copying
+=======
+
+**Lua Fun** source codes, logo and documentation are distributed under the
+**[MIT/X11 License]** - same as Lua and LuaJIT.
+
+Copyright (c) 2013 Roman Tsisyk <roman@tsisyk.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+[MIT/X11 License]: http://www.opensource.org/licenses/mit-license.php
diff --git a/contrib/lua-fun/fun.lua b/contrib/lua-fun/fun.lua
new file mode 100644
index 0000000..2c712d3
--- /dev/null
+++ b/contrib/lua-fun/fun.lua
@@ -0,0 +1,1076 @@
+---
+--- Lua Fun - a high-performance functional programming library for LuaJIT
+---
+--- Copyright (c) 2013-2017 Roman Tsisyk <roman@tsisyk.com>
+---
+--- Distributed under the MIT/X11 License. See COPYING.md for more details.
+---
+
+local exports = {}
+local methods = {}
+
+-- Rspamd specific
+local rspamd_text = require "rspamd_text"
+local text_cookie = rspamd_text.cookie
+
+-- compatibility with Lua 5.1/5.2
+local unpack = rawget(table, "unpack") or unpack
+
+--------------------------------------------------------------------------------
+-- Tools
+--------------------------------------------------------------------------------
+
+local return_if_not_empty = function(state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return ...
+end
+
+local call_if_not_empty = function(fun, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return state_x, fun(...)
+end
+
+local function deepcopy(orig) -- used by cycle()
+ local orig_type = type(orig)
+ local copy
+ if orig_type == 'table' then
+ copy = {}
+ for orig_key, orig_value in next, orig, nil do
+ copy[deepcopy(orig_key)] = deepcopy(orig_value)
+ end
+ else
+ copy = orig
+ end
+ return copy
+end
+
+local iterator_mt = {
+ -- usually called by for-in loop
+ __call = function(self, param, state)
+ return self.gen(param, state)
+ end;
+ __tostring = function(self)
+ return '<generator>'
+ end;
+ -- add all exported methods
+ __index = methods;
+}
+
+local wrap = function(gen, param, state)
+ return setmetatable({
+ gen = gen,
+ param = param,
+ state = state
+ }, iterator_mt), param, state
+end
+exports.wrap = wrap
+
+local unwrap = function(self)
+ return self.gen, self.param, self.state
+end
+methods.unwrap = unwrap
+
+--------------------------------------------------------------------------------
+-- Basic Functions
+--------------------------------------------------------------------------------
+
+local nil_gen = function(_param, _state)
+ return nil
+end
+
+local string_gen = function(param, state)
+ local state = state + 1
+ if state > #param then
+ return nil
+ end
+ local r = string.sub(param, state, state)
+ return state, r
+end
+
+local text_gen = function(param, state)
+ local state = state + 1
+ if state > #param then
+ return nil
+ end
+ local r = string.char(param:byte(state))
+ return state, r
+end
+
+local ipairs_gen = ipairs({}) -- get the generating function from ipairs
+
+local pairs_gen = pairs({ a = 0 }) -- get the generating function from pairs
+local map_gen = function(tab, key)
+ local value
+ local key, value = pairs_gen(tab, key)
+ return key, key, value
+end
+
+local rawiter = function(obj, param, state)
+ assert(obj ~= nil, "invalid iterator")
+ if type(obj) == "table" then
+ local mt = getmetatable(obj);
+ if mt ~= nil then
+ if mt == iterator_mt then
+ return obj.gen, obj.param, obj.state
+ elseif mt.__ipairs ~= nil then
+ return mt.__ipairs(obj)
+ elseif mt.__pairs ~= nil then
+ return mt.__pairs(obj)
+ end
+ end
+ if #obj > 0 then
+ -- array
+ return ipairs(obj)
+ else
+ -- hash
+ return map_gen, obj, nil
+ end
+ elseif (type(obj) == "function") then
+ return obj, param, state
+ elseif (type(obj) == "string") then
+ if #obj == 0 then
+ return nil_gen, nil, nil
+ end
+ return string_gen, obj, 0
+ elseif (type(obj) == "userdata" and obj.cookie == text_cookie) then
+ if #obj == 0 then
+ return nil_gen, nil, nil
+ end
+ return text_gen, obj, 0
+ end
+ error(string.format('object %s of type "%s" is not iterable',
+ obj, type(obj)))
+end
+
+local iter = function(obj, param, state)
+ return wrap(rawiter(obj, param, state))
+end
+exports.iter = iter
+
+local method0 = function(fun)
+ return function(self)
+ return fun(self.gen, self.param, self.state)
+ end
+end
+
+local method1 = function(fun)
+ return function(self, arg1)
+ return fun(arg1, self.gen, self.param, self.state)
+ end
+end
+
+local method2 = function(fun)
+ return function(self, arg1, arg2)
+ return fun(arg1, arg2, self.gen, self.param, self.state)
+ end
+end
+
+local export0 = function(fun)
+ return function(gen, param, state)
+ return fun(rawiter(gen, param, state))
+ end
+end
+
+local export1 = function(fun)
+ return function(arg1, gen, param, state)
+ return fun(arg1, rawiter(gen, param, state))
+ end
+end
+
+local export2 = function(fun)
+ return function(arg1, arg2, gen, param, state)
+ return fun(arg1, arg2, rawiter(gen, param, state))
+ end
+end
+
+local each = function(fun, gen, param, state)
+ repeat
+ state = call_if_not_empty(fun, gen(param, state))
+ until state == nil
+end
+methods.each = method1(each)
+exports.each = export1(each)
+methods.for_each = methods.each
+exports.for_each = exports.each
+methods.foreach = methods.each
+exports.foreach = exports.each
+
+--------------------------------------------------------------------------------
+-- Generators
+--------------------------------------------------------------------------------
+
+local range_gen = function(param, state)
+ local stop, step = param[1], param[2]
+ local state = state + step
+ if state > stop then
+ return nil
+ end
+ return state, state
+end
+
+local range_rev_gen = function(param, state)
+ local stop, step = param[1], param[2]
+ local state = state + step
+ if state < stop then
+ return nil
+ end
+ return state, state
+end
+
+local range = function(start, stop, step)
+ if step == nil then
+ if stop == nil then
+ if start == 0 then
+ return nil_gen, nil, nil
+ end
+ stop = start
+ start = stop > 0 and 1 or -1
+ end
+ step = start <= stop and 1 or -1
+ end
+
+ assert(type(start) == "number", "start must be a number")
+ assert(type(stop) == "number", "stop must be a number")
+ assert(type(step) == "number", "step must be a number")
+ assert(step ~= 0, "step must not be zero")
+
+ if (step > 0) then
+ return wrap(range_gen, {stop, step}, start - step)
+ elseif (step < 0) then
+ return wrap(range_rev_gen, {stop, step}, start - step)
+ end
+end
+exports.range = range
+
+local duplicate_table_gen = function(param_x, state_x)
+ return state_x + 1, unpack(param_x)
+end
+
+local duplicate_fun_gen = function(param_x, state_x)
+ return state_x + 1, param_x(state_x)
+end
+
+local duplicate_gen = function(param_x, state_x)
+ return state_x + 1, param_x
+end
+
+local duplicate = function(...)
+ if select('#', ...) <= 1 then
+ return wrap(duplicate_gen, select(1, ...), 0)
+ else
+ return wrap(duplicate_table_gen, {...}, 0)
+ end
+end
+exports.duplicate = duplicate
+exports.replicate = duplicate
+exports.xrepeat = duplicate
+
+local tabulate = function(fun)
+ assert(type(fun) == "function")
+ return wrap(duplicate_fun_gen, fun, 0)
+end
+exports.tabulate = tabulate
+
+local zeros = function()
+ return wrap(duplicate_gen, 0, 0)
+end
+exports.zeros = zeros
+
+local ones = function()
+ return wrap(duplicate_gen, 1, 0)
+end
+exports.ones = ones
+
+local rands_gen = function(param_x, _state_x)
+ return 0, math.random(param_x[1], param_x[2])
+end
+
+local rands_nil_gen = function(_param_x, _state_x)
+ return 0, math.random()
+end
+
+local rands = function(n, m)
+ if n == nil and m == nil then
+ return wrap(rands_nil_gen, 0, 0)
+ end
+ assert(type(n) == "number", "invalid first arg to rands")
+ if m == nil then
+ m = n
+ n = 0
+ else
+ assert(type(m) == "number", "invalid second arg to rands")
+ end
+ assert(n < m, "empty interval")
+ return wrap(rands_gen, {n, m - 1}, 0)
+end
+exports.rands = rands
+
+--------------------------------------------------------------------------------
+-- Slicing
+--------------------------------------------------------------------------------
+
+local nth = function(n, gen_x, param_x, state_x)
+ assert(n > 0, "invalid first argument to nth")
+ -- An optimization for arrays and strings
+ if gen_x == ipairs_gen then
+ return param_x[n]
+ elseif gen_x == string_gen then
+ if n <= #param_x then
+ return string.sub(param_x, n, n)
+ else
+ return nil
+ end
+ end
+ for i=1,n-1,1 do
+ state_x = gen_x(param_x, state_x)
+ if state_x == nil then
+ return nil
+ end
+ end
+ return return_if_not_empty(gen_x(param_x, state_x))
+end
+methods.nth = method1(nth)
+exports.nth = export1(nth)
+
+local head_call = function(state, ...)
+ if state == nil then
+ error("head: iterator is empty")
+ end
+ return ...
+end
+
+local head = function(gen, param, state)
+ return head_call(gen(param, state))
+end
+methods.head = method0(head)
+exports.head = export0(head)
+exports.car = exports.head
+methods.car = methods.head
+
+local tail = function(gen, param, state)
+ state = gen(param, state)
+ if state == nil then
+ return wrap(nil_gen, nil, nil)
+ end
+ return wrap(gen, param, state)
+end
+methods.tail = method0(tail)
+exports.tail = export0(tail)
+exports.cdr = exports.tail
+methods.cdr = methods.tail
+
+local take_n_gen_x = function(i, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return {i, state_x}, ...
+end
+
+local take_n_gen = function(param, state)
+ local n, gen_x, param_x = param[1], param[2], param[3]
+ local i, state_x = state[1], state[2]
+ if i >= n then
+ return nil
+ end
+ return take_n_gen_x(i + 1, gen_x(param_x, state_x))
+end
+
+local take_n = function(n, gen, param, state)
+ assert(n >= 0, "invalid first argument to take_n")
+ return wrap(take_n_gen, {n, gen, param}, {0, state})
+end
+methods.take_n = method1(take_n)
+exports.take_n = export1(take_n)
+
+local take_while_gen_x = function(fun, state_x, ...)
+ if state_x == nil or not fun(...) then
+ return nil
+ end
+ return state_x, ...
+end
+
+local take_while_gen = function(param, state_x)
+ local fun, gen_x, param_x = param[1], param[2], param[3]
+ return take_while_gen_x(fun, gen_x(param_x, state_x))
+end
+
+local take_while = function(fun, gen, param, state)
+ assert(type(fun) == "function", "invalid first argument to take_while")
+ return wrap(take_while_gen, {fun, gen, param}, state)
+end
+methods.take_while = method1(take_while)
+exports.take_while = export1(take_while)
+
+local take = function(n_or_fun, gen, param, state)
+ if type(n_or_fun) == "number" then
+ return take_n(n_or_fun, gen, param, state)
+ else
+ return take_while(n_or_fun, gen, param, state)
+ end
+end
+methods.take = method1(take)
+exports.take = export1(take)
+
+local drop_n = function(n, gen, param, state)
+ assert(n >= 0, "invalid first argument to drop_n")
+ local i
+ for i=1,n,1 do
+ state = gen(param, state)
+ if state == nil then
+ return wrap(nil_gen, nil, nil)
+ end
+ end
+ return wrap(gen, param, state)
+end
+methods.drop_n = method1(drop_n)
+exports.drop_n = export1(drop_n)
+
+local drop_while_x = function(fun, state_x, ...)
+ if state_x == nil or not fun(...) then
+ return state_x, false
+ end
+ return state_x, true, ...
+end
+
+local drop_while = function(fun, gen_x, param_x, state_x)
+ assert(type(fun) == "function", "invalid first argument to drop_while")
+ local cont, state_x_prev
+ repeat
+ state_x_prev = deepcopy(state_x)
+ state_x, cont = drop_while_x(fun, gen_x(param_x, state_x))
+ until not cont
+ if state_x == nil then
+ return wrap(nil_gen, nil, nil)
+ end
+ return wrap(gen_x, param_x, state_x_prev)
+end
+methods.drop_while = method1(drop_while)
+exports.drop_while = export1(drop_while)
+
+local drop = function(n_or_fun, gen_x, param_x, state_x)
+ if type(n_or_fun) == "number" then
+ return drop_n(n_or_fun, gen_x, param_x, state_x)
+ else
+ return drop_while(n_or_fun, gen_x, param_x, state_x)
+ end
+end
+methods.drop = method1(drop)
+exports.drop = export1(drop)
+
+local split = function(n_or_fun, gen_x, param_x, state_x)
+ return take(n_or_fun, gen_x, param_x, state_x),
+ drop(n_or_fun, gen_x, param_x, state_x)
+end
+methods.split = method1(split)
+exports.split = export1(split)
+methods.split_at = methods.split
+exports.split_at = exports.split
+methods.span = methods.split
+exports.span = exports.split
+
+--------------------------------------------------------------------------------
+-- Indexing
+--------------------------------------------------------------------------------
+
+local index = function(x, gen, param, state)
+ local i = 1
+ for _k, r in gen, param, state do
+ if r == x then
+ return i
+ end
+ i = i + 1
+ end
+ return nil
+end
+methods.index = method1(index)
+exports.index = export1(index)
+methods.index_of = methods.index
+exports.index_of = exports.index
+methods.elem_index = methods.index
+exports.elem_index = exports.index
+
+local indexes_gen = function(param, state)
+ local x, gen_x, param_x = param[1], param[2], param[3]
+ local i, state_x = state[1], state[2]
+ local r
+ while true do
+ state_x, r = gen_x(param_x, state_x)
+ if state_x == nil then
+ return nil
+ end
+ i = i + 1
+ if r == x then
+ return {i, state_x}, i
+ end
+ end
+end
+
+local indexes = function(x, gen, param, state)
+ return wrap(indexes_gen, {x, gen, param}, {0, state})
+end
+methods.indexes = method1(indexes)
+exports.indexes = export1(indexes)
+methods.elem_indexes = methods.indexes
+exports.elem_indexes = exports.indexes
+methods.indices = methods.indexes
+exports.indices = exports.indexes
+methods.elem_indices = methods.indexes
+exports.elem_indices = exports.indexes
+
+--------------------------------------------------------------------------------
+-- Filtering
+--------------------------------------------------------------------------------
+
+local filter1_gen = function(fun, gen_x, param_x, state_x, a)
+ while true do
+ if state_x == nil or fun(a) then break; end
+ state_x, a = gen_x(param_x, state_x)
+ end
+ return state_x, a
+end
+
+-- call each other
+local filterm_gen
+local filterm_gen_shrink = function(fun, gen_x, param_x, state_x)
+ return filterm_gen(fun, gen_x, param_x, gen_x(param_x, state_x))
+end
+
+filterm_gen = function(fun, gen_x, param_x, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ if fun(...) then
+ return state_x, ...
+ end
+ return filterm_gen_shrink(fun, gen_x, param_x, state_x)
+end
+
+local filter_detect = function(fun, gen_x, param_x, state_x, ...)
+ if select('#', ...) < 2 then
+ return filter1_gen(fun, gen_x, param_x, state_x, ...)
+ else
+ return filterm_gen(fun, gen_x, param_x, state_x, ...)
+ end
+end
+
+local filter_gen = function(param, state_x)
+ local fun, gen_x, param_x = param[1], param[2], param[3]
+ return filter_detect(fun, gen_x, param_x, gen_x(param_x, state_x))
+end
+
+local filter = function(fun, gen, param, state)
+ return wrap(filter_gen, {fun, gen, param}, state)
+end
+methods.filter = method1(filter)
+exports.filter = export1(filter)
+methods.remove_if = methods.filter
+exports.remove_if = exports.filter
+
+local grep = function(fun_or_regexp, gen, param, state)
+ local fun = fun_or_regexp
+ if type(fun_or_regexp) == "string" then
+ fun = function(x) return string.find(x, fun_or_regexp) ~= nil end
+ end
+ return filter(fun, gen, param, state)
+end
+methods.grep = method1(grep)
+exports.grep = export1(grep)
+
+local partition = function(fun, gen, param, state)
+ local neg_fun = function(...)
+ return not fun(...)
+ end
+ return filter(fun, gen, param, state),
+ filter(neg_fun, gen, param, state)
+end
+methods.partition = method1(partition)
+exports.partition = export1(partition)
+
+--------------------------------------------------------------------------------
+-- Reducing
+--------------------------------------------------------------------------------
+
+local foldl_call = function(fun, start, state, ...)
+ if state == nil then
+ return nil, start
+ end
+ return state, fun(start, ...)
+end
+
+local foldl = function(fun, start, gen_x, param_x, state_x)
+ while true do
+ state_x, start = foldl_call(fun, start, gen_x(param_x, state_x))
+ if state_x == nil then
+ break;
+ end
+ end
+ return start
+end
+methods.foldl = method2(foldl)
+exports.foldl = export2(foldl)
+methods.reduce = methods.foldl
+exports.reduce = exports.foldl
+
+local length = function(gen, param, state)
+ if gen == ipairs_gen or gen == string_gen then
+ return #param
+ end
+ local len = 0
+ repeat
+ state = gen(param, state)
+ len = len + 1
+ until state == nil
+ return len - 1
+end
+methods.length = method0(length)
+exports.length = export0(length)
+
+local is_null = function(gen, param, state)
+ return gen(param, deepcopy(state)) == nil
+end
+methods.is_null = method0(is_null)
+exports.is_null = export0(is_null)
+
+local is_prefix_of = function(iter_x, iter_y)
+ local gen_x, param_x, state_x = iter(iter_x)
+ local gen_y, param_y, state_y = iter(iter_y)
+
+ local r_x, r_y
+ for i=1,10,1 do
+ state_x, r_x = gen_x(param_x, state_x)
+ state_y, r_y = gen_y(param_y, state_y)
+ if state_x == nil then
+ return true
+ end
+ if state_y == nil or r_x ~= r_y then
+ return false
+ end
+ end
+end
+methods.is_prefix_of = is_prefix_of
+exports.is_prefix_of = is_prefix_of
+
+local all = function(fun, gen_x, param_x, state_x)
+ local r
+ repeat
+ state_x, r = call_if_not_empty(fun, gen_x(param_x, state_x))
+ until state_x == nil or not r
+ return state_x == nil
+end
+methods.all = method1(all)
+exports.all = export1(all)
+methods.every = methods.all
+exports.every = exports.all
+
+local any = function(fun, gen_x, param_x, state_x)
+ local r
+ repeat
+ state_x, r = call_if_not_empty(fun, gen_x(param_x, state_x))
+ until state_x == nil or r
+ return not not r
+end
+methods.any = method1(any)
+exports.any = export1(any)
+methods.some = methods.any
+exports.some = exports.any
+
+local sum = function(gen, param, state)
+ local s = 0
+ local r = 0
+ repeat
+ s = s + r
+ state, r = gen(param, state)
+ until state == nil
+ return s
+end
+methods.sum = method0(sum)
+exports.sum = export0(sum)
+
+local product = function(gen, param, state)
+ local p = 1
+ local r = 1
+ repeat
+ p = p * r
+ state, r = gen(param, state)
+ until state == nil
+ return p
+end
+methods.product = method0(product)
+exports.product = export0(product)
+
+local min_cmp = function(m, n)
+ if n < m then return n else return m end
+end
+
+local max_cmp = function(m, n)
+ if n > m then return n else return m end
+end
+
+local min = function(gen, param, state)
+ local state, m = gen(param, state)
+ if state == nil then
+ error("min: iterator is empty")
+ end
+
+ local cmp
+ if type(m) == "number" then
+ -- An optimization: use math.min for numbers
+ cmp = math.min
+ else
+ cmp = min_cmp
+ end
+
+ for _, r in gen, param, state do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.min = method0(min)
+exports.min = export0(min)
+methods.minimum = methods.min
+exports.minimum = exports.min
+
+local min_by = function(cmp, gen_x, param_x, state_x)
+ local state_x, m = gen_x(param_x, state_x)
+ if state_x == nil then
+ error("min: iterator is empty")
+ end
+
+ for _, r in gen_x, param_x, state_x do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.min_by = method1(min_by)
+exports.min_by = export1(min_by)
+methods.minimum_by = methods.min_by
+exports.minimum_by = exports.min_by
+
+local max = function(gen_x, param_x, state_x)
+ local state_x, m = gen_x(param_x, state_x)
+ if state_x == nil then
+ error("max: iterator is empty")
+ end
+
+ local cmp
+ if type(m) == "number" then
+ -- An optimization: use math.max for numbers
+ cmp = math.max
+ else
+ cmp = max_cmp
+ end
+
+ for _, r in gen_x, param_x, state_x do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.max = method0(max)
+exports.max = export0(max)
+methods.maximum = methods.max
+exports.maximum = exports.max
+
+local max_by = function(cmp, gen_x, param_x, state_x)
+ local state_x, m = gen_x(param_x, state_x)
+ if state_x == nil then
+ error("max: iterator is empty")
+ end
+
+ for _, r in gen_x, param_x, state_x do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.max_by = method1(max_by)
+exports.max_by = export1(max_by)
+methods.maximum_by = methods.maximum_by
+exports.maximum_by = exports.maximum_by
+
+local totable = function(gen_x, param_x, state_x)
+ local tab, key, val = {}
+ while true do
+ state_x, val = gen_x(param_x, state_x)
+ if state_x == nil then
+ break
+ end
+ table.insert(tab, val)
+ end
+ return tab
+end
+methods.totable = method0(totable)
+exports.totable = export0(totable)
+
+local tomap = function(gen_x, param_x, state_x)
+ local tab, key, val = {}
+ while true do
+ state_x, key, val = gen_x(param_x, state_x)
+ if state_x == nil then
+ break
+ end
+ tab[key] = val
+ end
+ return tab
+end
+methods.tomap = method0(tomap)
+exports.tomap = export0(tomap)
+
+--------------------------------------------------------------------------------
+-- Transformations
+--------------------------------------------------------------------------------
+
+local map_gen = function(param, state)
+ local gen_x, param_x, fun = param[1], param[2], param[3]
+ return call_if_not_empty(fun, gen_x(param_x, state))
+end
+
+local map = function(fun, gen, param, state)
+ return wrap(map_gen, {gen, param, fun}, state)
+end
+methods.map = method1(map)
+exports.map = export1(map)
+
+local enumerate_gen_call = function(state, i, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return {i + 1, state_x}, i, ...
+end
+
+local enumerate_gen = function(param, state)
+ local gen_x, param_x = param[1], param[2]
+ local i, state_x = state[1], state[2]
+ return enumerate_gen_call(state, i, gen_x(param_x, state_x))
+end
+
+local enumerate = function(gen, param, state)
+ return wrap(enumerate_gen, {gen, param}, {1, state})
+end
+methods.enumerate = method0(enumerate)
+exports.enumerate = export0(enumerate)
+
+local intersperse_call = function(i, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return {i + 1, state_x}, ...
+end
+
+local intersperse_gen = function(param, state)
+ local x, gen_x, param_x = param[1], param[2], param[3]
+ local i, state_x = state[1], state[2]
+ if i % 2 == 1 then
+ return {i + 1, state_x}, x
+ else
+ return intersperse_call(i, gen_x(param_x, state_x))
+ end
+end
+
+-- TODO: interperse must not add x to the tail
+local intersperse = function(x, gen, param, state)
+ return wrap(intersperse_gen, {x, gen, param}, {0, state})
+end
+methods.intersperse = method1(intersperse)
+exports.intersperse = export1(intersperse)
+
+--------------------------------------------------------------------------------
+-- Compositions
+--------------------------------------------------------------------------------
+
+local function zip_gen_r(param, state, state_new, ...)
+ if #state_new == #param / 2 then
+ return state_new, ...
+ end
+
+ local i = #state_new + 1
+ local gen_x, param_x = param[2 * i - 1], param[2 * i]
+ local state_x, r = gen_x(param_x, state[i])
+ if state_x == nil then
+ return nil
+ end
+ table.insert(state_new, state_x)
+ return zip_gen_r(param, state, state_new, r, ...)
+end
+
+local zip_gen = function(param, state)
+ return zip_gen_r(param, state, {})
+end
+
+-- A special hack for zip/chain to skip last two state, if a wrapped iterator
+-- has been passed
+local numargs = function(...)
+ local n = select('#', ...)
+ if n >= 3 then
+ -- Fix last argument
+ local it = select(n - 2, ...)
+ if type(it) == 'table' and getmetatable(it) == iterator_mt and
+ it.param == select(n - 1, ...) and it.state == select(n, ...) then
+ return n - 2
+ end
+ end
+ return n
+end
+
+local zip = function(...)
+ local n = numargs(...)
+ if n == 0 then
+ return wrap(nil_gen, nil, nil)
+ end
+ local param = { [2 * n] = 0 }
+ local state = { [n] = 0 }
+
+ local i, gen_x, param_x, state_x
+ for i=1,n,1 do
+ local it = select(n - i + 1, ...)
+ gen_x, param_x, state_x = rawiter(it)
+ param[2 * i - 1] = gen_x
+ param[2 * i] = param_x
+ state[i] = state_x
+ end
+
+ return wrap(zip_gen, param, state)
+end
+methods.zip = zip
+exports.zip = zip
+
+local cycle_gen_call = function(param, state_x, ...)
+ if state_x == nil then
+ local gen_x, param_x, state_x0 = param[1], param[2], param[3]
+ return gen_x(param_x, deepcopy(state_x0))
+ end
+ return state_x, ...
+end
+
+local cycle_gen = function(param, state_x)
+ local gen_x, param_x, state_x0 = param[1], param[2], param[3]
+ return cycle_gen_call(param, gen_x(param_x, state_x))
+end
+
+local cycle = function(gen, param, state)
+ return wrap(cycle_gen, {gen, param, state}, deepcopy(state))
+end
+methods.cycle = method0(cycle)
+exports.cycle = export0(cycle)
+
+-- call each other
+local chain_gen_r1
+local chain_gen_r2 = function(param, state, state_x, ...)
+ if state_x == nil then
+ local i = state[1]
+ i = i + 1
+ if param[3 * i - 1] == nil then
+ return nil
+ end
+ local state_x = param[3 * i]
+ return chain_gen_r1(param, {i, state_x})
+ end
+ return {state[1], state_x}, ...
+end
+
+chain_gen_r1 = function(param, state)
+ local i, state_x = state[1], state[2]
+ local gen_x, param_x = param[3 * i - 2], param[3 * i - 1]
+ return chain_gen_r2(param, state, gen_x(param_x, state[2]))
+end
+
+local chain = function(...)
+ local n = numargs(...)
+ if n == 0 then
+ return wrap(nil_gen, nil, nil)
+ end
+
+ local param = { [3 * n] = 0 }
+ local i, gen_x, param_x, state_x
+ for i=1,n,1 do
+ local elem = select(i, ...)
+ gen_x, param_x, state_x = iter(elem)
+ param[3 * i - 2] = gen_x
+ param[3 * i - 1] = param_x
+ param[3 * i] = state_x
+ end
+
+ return wrap(chain_gen_r1, param, {1, param[3]})
+end
+methods.chain = chain
+exports.chain = chain
+
+--------------------------------------------------------------------------------
+-- Operators
+--------------------------------------------------------------------------------
+
+local operator = {
+ ----------------------------------------------------------------------------
+ -- Comparison operators
+ ----------------------------------------------------------------------------
+ lt = function(a, b) return a < b end,
+ le = function(a, b) return a <= b end,
+ eq = function(a, b) return a == b end,
+ ne = function(a, b) return a ~= b end,
+ ge = function(a, b) return a >= b end,
+ gt = function(a, b) return a > b end,
+
+ ----------------------------------------------------------------------------
+ -- Arithmetic operators
+ ----------------------------------------------------------------------------
+ add = function(a, b) return a + b end,
+ div = function(a, b) return a / b end,
+ floordiv = function(a, b) return math.floor(a/b) end,
+ intdiv = function(a, b)
+ local q = a / b
+ if a >= 0 then return math.floor(q) else return math.ceil(q) end
+ end,
+ mod = function(a, b) return a % b end,
+ mul = function(a, b) return a * b end,
+ neq = function(a) return -a end,
+ unm = function(a) return -a end, -- an alias
+ pow = function(a, b) return a ^ b end,
+ sub = function(a, b) return a - b end,
+ truediv = function(a, b) return a / b end,
+
+ ----------------------------------------------------------------------------
+ -- String operators
+ ----------------------------------------------------------------------------
+ concat = function(a, b) return a..b end,
+ len = function(a) return #a end,
+ length = function(a) return #a end, -- an alias
+
+ ----------------------------------------------------------------------------
+ -- Logical operators
+ ----------------------------------------------------------------------------
+ land = function(a, b) return a and b end,
+ lor = function(a, b) return a or b end,
+ lnot = function(a) return not a end,
+ truth = function(a) return not not a end,
+}
+exports.operator = operator
+methods.operator = operator
+exports.op = operator
+methods.op = operator
+
+--------------------------------------------------------------------------------
+-- module definitions
+--------------------------------------------------------------------------------
+
+-- a special syntax sugar to export all functions to the global table
+setmetatable(exports, {
+ __call = function(t, override)
+ for k, v in pairs(t) do
+ if _G[k] ~= nil then
+ local msg = 'function ' .. k .. ' already exists in global scope.'
+ if override then
+ _G[k] = v
+ print('WARNING: ' .. msg .. ' Overwritten.')
+ else
+ print('NOTICE: ' .. msg .. ' Skipped.')
+ end
+ else
+ _G[k] = v
+ end
+ end
+ end,
+})
+
+return exports
diff --git a/contrib/lua-lpeg/CMakeLists.txt b/contrib/lua-lpeg/CMakeLists.txt
new file mode 100644
index 0000000..92dd018
--- /dev/null
+++ b/contrib/lua-lpeg/CMakeLists.txt
@@ -0,0 +1,9 @@
+SET(LPEGSRC lpcap.c
+ lpcode.c
+ lpprint.c
+ lptree.c
+ lpvm.c)
+
+SET(LIB_TYPE STATIC)
+ADD_LIBRARY(rspamd-lpeg ${LIB_TYPE} ${LPEGSRC})
+set_target_properties(rspamd-lpeg PROPERTIES COMPILE_FLAGS "${LPEG_CFLAGS}")
diff --git a/contrib/lua-lpeg/LICENSE b/contrib/lua-lpeg/LICENSE
new file mode 100644
index 0000000..cea2d8b
--- /dev/null
+++ b/contrib/lua-lpeg/LICENSE
@@ -0,0 +1,19 @@
+Copyright © 2007-2015 Lua.org, PUC-Rio.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/lua-lpeg/lpcap.c b/contrib/lua-lpeg/lpcap.c
new file mode 100644
index 0000000..b332fde
--- /dev/null
+++ b/contrib/lua-lpeg/lpcap.c
@@ -0,0 +1,555 @@
+/*
+** $Id: lpcap.c $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lpcap.h"
+#include "lptypes.h"
+
+
+#define captype(cap) ((cap)->kind)
+
+#define isclosecap(cap) (captype(cap) == Cclose)
+
+#define closeaddr(c) ((c)->s + (c)->siz - 1)
+
+#define isfullcap(cap) ((cap)->siz != 0)
+
+#define getfromktable(cs,v) lua_rawgeti((cs)->L, ktableidx((cs)->ptop), v)
+
+#define pushluaval(cs) getfromktable(cs, (cs)->cap->idx)
+
+
+
+/*
+** Put at the cache for Lua values the value indexed by 'v' in ktable
+** of the running pattern (if it is not there yet); returns its index.
+*/
+static int updatecache (CapState *cs, int v) {
+ int idx = cs->ptop + 1; /* stack index of cache for Lua values */
+ if (v != cs->valuecached) { /* not there? */
+ getfromktable(cs, v); /* get value from 'ktable' */
+ lua_replace(cs->L, idx); /* put it at reserved stack position */
+ cs->valuecached = v; /* keep track of what is there */
+ }
+ return idx;
+}
+
+
+static int pushcapture (CapState *cs);
+
+
+/*
+** Goes back in a list of captures looking for an open capture
+** corresponding to a close
+*/
+static Capture *findopen (Capture *cap) {
+ int n = 0; /* number of closes waiting an open */
+ for (;;) {
+ cap--;
+ if (isclosecap(cap)) n++; /* one more open to skip */
+ else if (!isfullcap(cap))
+ if (n-- == 0) return cap;
+ }
+}
+
+
+/*
+** Go to the next capture
+*/
+static void nextcap (CapState *cs) {
+ Capture *cap = cs->cap;
+ if (!isfullcap(cap)) { /* not a single capture? */
+ int n = 0; /* number of opens waiting a close */
+ for (;;) { /* look for corresponding close */
+ cap++;
+ if (isclosecap(cap)) {
+ if (n-- == 0) break;
+ }
+ else if (!isfullcap(cap)) n++;
+ }
+ }
+ cs->cap = cap + 1; /* + 1 to skip last close (or entire single capture) */
+}
+
+
+/*
+** Push on the Lua stack all values generated by nested captures inside
+** the current capture. Returns number of values pushed. 'addextra'
+** makes it push the entire match after all captured values. The
+** entire match is pushed also if there are no other nested values,
+** so the function never returns zero.
+*/
+static int pushnestedvalues (CapState *cs, int addextra) {
+ Capture *co = cs->cap;
+ if (isfullcap(cs->cap++)) { /* no nested captures? */
+ lua_pushlstring(cs->L, co->s, co->siz - 1); /* push whole match */
+ return 1; /* that is it */
+ }
+ else {
+ int n = 0;
+ while (!isclosecap(cs->cap)) /* repeat for all nested patterns */
+ n += pushcapture(cs);
+ if (addextra || n == 0) { /* need extra? */
+ lua_pushlstring(cs->L, co->s, cs->cap->s - co->s); /* push whole match */
+ n++;
+ }
+ cs->cap++; /* skip close entry */
+ return n;
+ }
+}
+
+
+/*
+** Push only the first value generated by nested captures
+*/
+static void pushonenestedvalue (CapState *cs) {
+ int n = pushnestedvalues(cs, 0);
+ if (n > 1)
+ lua_pop(cs->L, n - 1); /* pop extra values */
+}
+
+
+/*
+** Try to find a named group capture with the name given at the top of
+** the stack; goes backward from 'cap'.
+*/
+static Capture *findback (CapState *cs, Capture *cap) {
+ lua_State *L = cs->L;
+ while (cap-- > cs->ocap) { /* repeat until end of list */
+ if (isclosecap(cap))
+ cap = findopen(cap); /* skip nested captures */
+ else if (!isfullcap(cap))
+ continue; /* opening an enclosing capture: skip and get previous */
+ if (captype(cap) == Cgroup) {
+ getfromktable(cs, cap->idx); /* get group name */
+ if (lp_equal(L, -2, -1)) { /* right group? */
+ lua_pop(L, 2); /* remove reference name and group name */
+ return cap;
+ }
+ else lua_pop(L, 1); /* remove group name */
+ }
+ }
+ luaL_error(L, "back reference '%s' not found", lua_tostring(L, -1));
+ return NULL; /* to avoid warnings */
+}
+
+
+/*
+** Back-reference capture. Return number of values pushed.
+*/
+static int backrefcap (CapState *cs) {
+ int n;
+ Capture *curr = cs->cap;
+ pushluaval(cs); /* reference name */
+ cs->cap = findback(cs, curr); /* find corresponding group */
+ n = pushnestedvalues(cs, 0); /* push group's values */
+ cs->cap = curr + 1;
+ return n;
+}
+
+
+/*
+** Table capture: creates a new table and populates it with nested
+** captures.
+*/
+static int tablecap (CapState *cs) {
+ lua_State *L = cs->L;
+ int n = 0;
+ lua_newtable(L);
+ if (isfullcap(cs->cap++))
+ return 1; /* table is empty */
+ while (!isclosecap(cs->cap)) {
+ if (captype(cs->cap) == Cgroup && cs->cap->idx != 0) { /* named group? */
+ pushluaval(cs); /* push group name */
+ pushonenestedvalue(cs);
+ lua_settable(L, -3);
+ }
+ else { /* not a named group */
+ int i;
+ int k = pushcapture(cs);
+ for (i = k; i > 0; i--) /* store all values into table */
+ lua_rawseti(L, -(i + 1), n + i);
+ n += k;
+ }
+ }
+ cs->cap++; /* skip close entry */
+ return 1; /* number of values pushed (only the table) */
+}
+
+
+/*
+** Table-query capture
+*/
+static int querycap (CapState *cs) {
+ int idx = cs->cap->idx;
+ pushonenestedvalue(cs); /* get nested capture */
+ lua_gettable(cs->L, updatecache(cs, idx)); /* query cap. value at table */
+ if (!lua_isnil(cs->L, -1))
+ return 1;
+ else { /* no value */
+ lua_pop(cs->L, 1); /* remove nil */
+ return 0;
+ }
+}
+
+
+/*
+** Fold capture
+*/
+static int foldcap (CapState *cs) {
+ int n;
+ lua_State *L = cs->L;
+ int idx = cs->cap->idx;
+ if (isfullcap(cs->cap++) || /* no nested captures? */
+ isclosecap(cs->cap) || /* no nested captures (large subject)? */
+ (n = pushcapture(cs)) == 0) /* nested captures with no values? */
+ return luaL_error(L, "no initial value for fold capture");
+ if (n > 1)
+ lua_pop(L, n - 1); /* leave only one result for accumulator */
+ while (!isclosecap(cs->cap)) {
+ lua_pushvalue(L, updatecache(cs, idx)); /* get folding function */
+ lua_insert(L, -2); /* put it before accumulator */
+ n = pushcapture(cs); /* get next capture's values */
+ lua_call(L, n + 1, 1); /* call folding function */
+ }
+ cs->cap++; /* skip close entry */
+ return 1; /* only accumulator left on the stack */
+}
+
+
+/*
+** Function capture
+*/
+static int functioncap (CapState *cs) {
+ int n;
+ int top = lua_gettop(cs->L);
+ pushluaval(cs); /* push function */
+ n = pushnestedvalues(cs, 0); /* push nested captures */
+ lua_call(cs->L, n, LUA_MULTRET); /* call function */
+ return lua_gettop(cs->L) - top; /* return function's results */
+}
+
+
+/*
+** Select capture
+*/
+static int numcap (CapState *cs) {
+ int idx = cs->cap->idx; /* value to select */
+ if (idx == 0) { /* no values? */
+ nextcap(cs); /* skip entire capture */
+ return 0; /* no value produced */
+ }
+ else {
+ int n = pushnestedvalues(cs, 0);
+ if (n < idx) /* invalid index? */
+ return luaL_error(cs->L, "no capture '%d'", idx);
+ else {
+ lua_pushvalue(cs->L, -(n - idx + 1)); /* get selected capture */
+ lua_replace(cs->L, -(n + 1)); /* put it in place of 1st capture */
+ lua_pop(cs->L, n - 1); /* remove other captures */
+ return 1;
+ }
+ }
+}
+
+
+/*
+** Return the stack index of the first runtime capture in the given
+** list of captures (or zero if no runtime captures)
+*/
+int finddyncap (Capture *cap, Capture *last) {
+ for (; cap < last; cap++) {
+ if (cap->kind == Cruntime)
+ return cap->idx; /* stack position of first capture */
+ }
+ return 0; /* no dynamic captures in this segment */
+}
+
+
+/*
+** Calls a runtime capture. Returns number of captures "removed" by the
+** call, that is, those inside the group capture. Captures to be added
+** are on the Lua stack.
+*/
+int runtimecap (CapState *cs, Capture *close, const char *s, int *rem) {
+ int n, id;
+ lua_State *L = cs->L;
+ int otop = lua_gettop(L);
+ Capture *open = findopen(close); /* get open group capture */
+ assert(captype(open) == Cgroup);
+ id = finddyncap(open, close); /* get first dynamic capture argument */
+ close->kind = Cclose; /* closes the group */
+ close->s = s;
+ cs->cap = open; cs->valuecached = 0; /* prepare capture state */
+ luaL_checkstack(L, 4, "too many runtime captures");
+ pushluaval(cs); /* push function to be called */
+ lua_pushvalue(L, SUBJIDX); /* push original subject */
+ lua_pushinteger(L, s - cs->s + 1); /* push current position */
+ n = pushnestedvalues(cs, 0); /* push nested captures */
+ lua_call(L, n + 2, LUA_MULTRET); /* call dynamic function */
+ if (id > 0) { /* are there old dynamic captures to be removed? */
+ int i;
+ for (i = id; i <= otop; i++)
+ lua_remove(L, id); /* remove old dynamic captures */
+ *rem = otop - id + 1; /* total number of dynamic captures removed */
+ }
+ else
+ *rem = 0; /* no dynamic captures removed */
+ return close - open - 1; /* number of captures to be removed */
+}
+
+
+/*
+** Auxiliary structure for substitution and string captures: keep
+** information about nested captures for future use, avoiding to push
+** string results into Lua
+*/
+typedef struct StrAux {
+ int isstring; /* whether capture is a string */
+ union {
+ Capture *cp; /* if not a string, respective capture */
+ struct { /* if it is a string... */
+ const char *s; /* ... starts here */
+ const char *e; /* ... ends here */
+ } s;
+ } u;
+} StrAux;
+
+#define MAXSTRCAPS 10
+
+/*
+** Collect values from current capture into array 'cps'. Current
+** capture must be Cstring (first call) or Csimple (recursive calls).
+** (In first call, fills %0 with whole match for Cstring.)
+** Returns number of elements in the array that were filled.
+*/
+static int getstrcaps (CapState *cs, StrAux *cps, int n) {
+ int k = n++;
+ cps[k].isstring = 1; /* get string value */
+ cps[k].u.s.s = cs->cap->s; /* starts here */
+ if (!isfullcap(cs->cap++)) { /* nested captures? */
+ while (!isclosecap(cs->cap)) { /* traverse them */
+ if (n >= MAXSTRCAPS) /* too many captures? */
+ nextcap(cs); /* skip extra captures (will not need them) */
+ else if (captype(cs->cap) == Csimple) /* string? */
+ n = getstrcaps(cs, cps, n); /* put info. into array */
+ else {
+ cps[n].isstring = 0; /* not a string */
+ cps[n].u.cp = cs->cap; /* keep original capture */
+ nextcap(cs);
+ n++;
+ }
+ }
+ cs->cap++; /* skip close */
+ }
+ cps[k].u.s.e = closeaddr(cs->cap - 1); /* ends here */
+ return n;
+}
+
+
+/*
+** add next capture value (which should be a string) to buffer 'b'
+*/
+static int addonestring (luaL_Buffer *b, CapState *cs, const char *what);
+
+
+/*
+** String capture: add result to buffer 'b' (instead of pushing
+** it into the stack)
+*/
+static void stringcap (luaL_Buffer *b, CapState *cs) {
+ StrAux cps[MAXSTRCAPS];
+ int n;
+ size_t len, i;
+ const char *fmt; /* format string */
+ fmt = lua_tolstring(cs->L, updatecache(cs, cs->cap->idx), &len);
+ n = getstrcaps(cs, cps, 0) - 1; /* collect nested captures */
+ for (i = 0; i < len; i++) { /* traverse them */
+ if (fmt[i] != '%') /* not an escape? */
+ luaL_addchar(b, fmt[i]); /* add it to buffer */
+ else if (fmt[++i] < '0' || fmt[i] > '9') /* not followed by a digit? */
+ luaL_addchar(b, fmt[i]); /* add to buffer */
+ else {
+ int l = fmt[i] - '0'; /* capture index */
+ if (l > n)
+ luaL_error(cs->L, "invalid capture index (%d)", l);
+ else if (cps[l].isstring)
+ luaL_addlstring(b, cps[l].u.s.s, cps[l].u.s.e - cps[l].u.s.s);
+ else {
+ Capture *curr = cs->cap;
+ cs->cap = cps[l].u.cp; /* go back to evaluate that nested capture */
+ if (!addonestring(b, cs, "capture"))
+ luaL_error(cs->L, "no values in capture index %d", l);
+ cs->cap = curr; /* continue from where it stopped */
+ }
+ }
+ }
+}
+
+
+/*
+** Substitution capture: add result to buffer 'b'
+*/
+static void substcap (luaL_Buffer *b, CapState *cs) {
+ const char *curr = cs->cap->s;
+ if (isfullcap(cs->cap)) /* no nested captures? */
+ luaL_addlstring(b, curr, cs->cap->siz - 1); /* keep original text */
+ else {
+ cs->cap++; /* skip open entry */
+ while (!isclosecap(cs->cap)) { /* traverse nested captures */
+ const char *next = cs->cap->s;
+ luaL_addlstring(b, curr, next - curr); /* add text up to capture */
+ if (addonestring(b, cs, "replacement"))
+ curr = closeaddr(cs->cap - 1); /* continue after match */
+ else /* no capture value */
+ curr = next; /* keep original text in final result */
+ }
+ luaL_addlstring(b, curr, cs->cap->s - curr); /* add last piece of text */
+ }
+ cs->cap++; /* go to next capture */
+}
+
+
+/*
+** Evaluates a capture and adds its first value to buffer 'b'; returns
+** whether there was a value
+*/
+static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) {
+ switch (captype(cs->cap)) {
+ case Cstring:
+ stringcap(b, cs); /* add capture directly to buffer */
+ return 1;
+ case Csubst:
+ substcap(b, cs); /* add capture directly to buffer */
+ return 1;
+ default: {
+ lua_State *L = cs->L;
+ int n = pushcapture(cs);
+ if (n > 0) {
+ if (n > 1) lua_pop(L, n - 1); /* only one result */
+ if (!lua_isstring(L, -1))
+ luaL_error(L, "invalid %s value (a %s)", what, luaL_typename(L, -1));
+ luaL_addvalue(b);
+ }
+ return n;
+ }
+ }
+}
+
+
+#if !defined(MAXRECLEVEL)
+#define MAXRECLEVEL 200
+#endif
+
+
+/*
+** Push all values of the current capture into the stack; returns
+** number of values pushed
+*/
+static int pushcapture (CapState *cs) {
+ lua_State *L = cs->L;
+ int res;
+ luaL_checkstack(L, 4, "too many captures");
+ if (cs->reclevel++ > MAXRECLEVEL)
+ return luaL_error(L, "subcapture nesting too deep");
+ switch (captype(cs->cap)) {
+ case Cposition: {
+ lua_pushinteger(L, cs->cap->s - cs->s + 1);
+ cs->cap++;
+ res = 1;
+ break;
+ }
+ case Cconst: {
+ pushluaval(cs);
+ cs->cap++;
+ res = 1;
+ break;
+ }
+ case Carg: {
+ int arg = (cs->cap++)->idx;
+ if (arg + FIXEDARGS > cs->ptop)
+ return luaL_error(L, "reference to absent extra argument #%d", arg);
+ lua_pushvalue(L, arg + FIXEDARGS);
+ res = 1;
+ break;
+ }
+ case Csimple: {
+ int k = pushnestedvalues(cs, 1);
+ lua_insert(L, -k); /* make whole match be first result */
+ res = k;
+ break;
+ }
+ case Cruntime: {
+ lua_pushvalue(L, (cs->cap++)->idx); /* value is in the stack */
+ res = 1;
+ break;
+ }
+ case Cstring: {
+ luaL_Buffer b;
+ luaL_buffinit(L, &b);
+ stringcap(&b, cs);
+ luaL_pushresult(&b);
+ res = 1;
+ break;
+ }
+ case Csubst: {
+ luaL_Buffer b;
+ luaL_buffinit(L, &b);
+ substcap(&b, cs);
+ luaL_pushresult(&b);
+ res = 1;
+ break;
+ }
+ case Cgroup: {
+ if (cs->cap->idx == 0) /* anonymous group? */
+ res = pushnestedvalues(cs, 0); /* add all nested values */
+ else { /* named group: add no values */
+ nextcap(cs); /* skip capture */
+ res = 0;
+ }
+ break;
+ }
+ case Cbackref: res = backrefcap(cs); break;
+ case Ctable: res = tablecap(cs); break;
+ case Cfunction: res = functioncap(cs); break;
+ case Cnum: res = numcap(cs); break;
+ case Cquery: res = querycap(cs); break;
+ case Cfold: res = foldcap(cs); break;
+ default: assert(0); res = 0;
+ }
+ cs->reclevel--;
+ return res;
+}
+
+
+/*
+** Prepare a CapState structure and traverse the entire list of
+** captures in the stack pushing its results. 's' is the subject
+** string, 'r' is the final position of the match, and 'ptop'
+** the index in the stack where some useful values were pushed.
+** Returns the number of results pushed. (If the list produces no
+** results, push the final position of the match.)
+*/
+int getcaptures (lua_State *L, const char *s, const char *r, int ptop) {
+ Capture *capture = (Capture *)lua_touserdata(L, caplistidx(ptop));
+ int n = 0;
+ if (!isclosecap(capture)) { /* is there any capture? */
+ CapState cs;
+ cs.ocap = cs.cap = capture; cs.L = L; cs.reclevel = 0;
+ cs.s = s; cs.valuecached = 0; cs.ptop = ptop;
+ do { /* collect their values */
+ n += pushcapture(&cs);
+ } while (!isclosecap(cs.cap));
+ }
+ if (n == 0) { /* no capture values? */
+ lua_pushinteger(L, r - s + 1); /* return only end position */
+ n = 1;
+ }
+ return n;
+}
+
+
diff --git a/contrib/lua-lpeg/lpcap.h b/contrib/lua-lpeg/lpcap.h
new file mode 100644
index 0000000..dc10d69
--- /dev/null
+++ b/contrib/lua-lpeg/lpcap.h
@@ -0,0 +1,57 @@
+/*
+** $Id: lpcap.h $
+*/
+
+#if !defined(lpcap_h)
+#define lpcap_h
+
+
+#include "lptypes.h"
+
+
+/* kinds of captures */
+typedef enum CapKind {
+ Cclose, /* not used in trees */
+ Cposition,
+ Cconst, /* ktable[key] is Lua constant */
+ Cbackref, /* ktable[key] is "name" of group to get capture */
+ Carg, /* 'key' is arg's number */
+ Csimple, /* next node is pattern */
+ Ctable, /* next node is pattern */
+ Cfunction, /* ktable[key] is function; next node is pattern */
+ Cquery, /* ktable[key] is table; next node is pattern */
+ Cstring, /* ktable[key] is string; next node is pattern */
+ Cnum, /* numbered capture; 'key' is number of value to return */
+ Csubst, /* substitution capture; next node is pattern */
+ Cfold, /* ktable[key] is function; next node is pattern */
+ Cruntime, /* not used in trees (is uses another type for tree) */
+ Cgroup /* ktable[key] is group's "name" */
+} CapKind;
+
+
+typedef struct Capture {
+ const char *s; /* subject position */
+ unsigned short idx; /* extra info (group name, arg index, etc.) */
+ byte kind; /* kind of capture */
+ byte siz; /* size of full capture + 1 (0 = not a full capture) */
+} Capture;
+
+
+typedef struct CapState {
+ Capture *cap; /* current capture */
+ Capture *ocap; /* (original) capture list */
+ lua_State *L;
+ int ptop; /* index of last argument to 'match' */
+ const char *s; /* original string */
+ int valuecached; /* value stored in cache slot */
+ int reclevel; /* recursion level */
+} CapState;
+
+
+int runtimecap (CapState *cs, Capture *close, const char *s, int *rem);
+int getcaptures (lua_State *L, const char *s, const char *r, int ptop);
+int finddyncap (Capture *cap, Capture *last);
+
+#endif
+
+
diff --git a/contrib/lua-lpeg/lpcode.c b/contrib/lua-lpeg/lpcode.c
new file mode 100644
index 0000000..3923459
--- /dev/null
+++ b/contrib/lua-lpeg/lpcode.c
@@ -0,0 +1,1014 @@
+/*
+** $Id: lpcode.c $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include <limits.h>
+
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lptypes.h"
+#include "lpcode.h"
+
+
+/* signals a "no-instruction */
+#define NOINST -1
+
+
+
+static const Charset fullset_ =
+ {{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}};
+
+static const Charset *fullset = &fullset_;
+
+/*
+** {======================================================
+** Analysis and some optimizations
+** =======================================================
+*/
+
+/*
+** Check whether a charset is empty (returns IFail), singleton (IChar),
+** full (IAny), or none of those (ISet). When singleton, '*c' returns
+** which character it is. (When generic set, the set was the input,
+** so there is no need to return it.)
+*/
+static Opcode charsettype (const byte *cs, int *c) {
+ int count = 0; /* number of characters in the set */
+ int i;
+ int candidate = -1; /* candidate position for the singleton char */
+ for (i = 0; i < CHARSETSIZE; i++) { /* for each byte */
+ int b = cs[i];
+ if (b == 0) { /* is byte empty? */
+ if (count > 1) /* was set neither empty nor singleton? */
+ return ISet; /* neither full nor empty nor singleton */
+ /* else set is still empty or singleton */
+ }
+ else if (b == 0xFF) { /* is byte full? */
+ if (count < (i * BITSPERCHAR)) /* was set not full? */
+ return ISet; /* neither full nor empty nor singleton */
+ else count += BITSPERCHAR; /* set is still full */
+ }
+ else if ((b & (b - 1)) == 0) { /* has byte only one bit? */
+ if (count > 0) /* was set not empty? */
+ return ISet; /* neither full nor empty nor singleton */
+ else { /* set has only one char till now; track it */
+ count++;
+ candidate = i;
+ }
+ }
+ else return ISet; /* byte is neither empty, full, nor singleton */
+ }
+ switch (count) {
+ case 0: return IFail; /* empty set */
+ case 1: { /* singleton; find character bit inside byte */
+ int b = cs[candidate];
+ *c = candidate * BITSPERCHAR;
+ if ((b & 0xF0) != 0) { *c += 4; b >>= 4; }
+ if ((b & 0x0C) != 0) { *c += 2; b >>= 2; }
+ if ((b & 0x02) != 0) { *c += 1; }
+ return IChar;
+ }
+ default: {
+ assert(count == CHARSETSIZE * BITSPERCHAR); /* full set */
+ return IAny;
+ }
+ }
+}
+
+
+/*
+** A few basic operations on Charsets
+*/
+static void cs_complement (Charset *cs) {
+ loopset(i, cs->cs[i] = ~cs->cs[i]);
+}
+
+static int cs_equal (const byte *cs1, const byte *cs2) {
+ loopset(i, if (cs1[i] != cs2[i]) return 0);
+ return 1;
+}
+
+static int cs_disjoint (const Charset *cs1, const Charset *cs2) {
+ loopset(i, if ((cs1->cs[i] & cs2->cs[i]) != 0) return 0;)
+ return 1;
+}
+
+
+/*
+** If 'tree' is a 'char' pattern (TSet, TChar, TAny), convert it into a
+** charset and return 1; else return 0.
+*/
+int tocharset (TTree *tree, Charset *cs) {
+ switch (tree->tag) {
+ case TSet: { /* copy set */
+ loopset(i, cs->cs[i] = treebuffer(tree)[i]);
+ return 1;
+ }
+ case TChar: { /* only one char */
+ assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX);
+ loopset(i, cs->cs[i] = 0); /* erase all chars */
+ setchar(cs->cs, tree->u.n); /* add that one */
+ return 1;
+ }
+ case TAny: {
+ loopset(i, cs->cs[i] = 0xFF); /* add all characters to the set */
+ return 1;
+ }
+ default: return 0;
+ }
+}
+
+
+/*
+** Visit a TCall node taking care to stop recursion. If node not yet
+** visited, return 'f(sib2(tree))', otherwise return 'def' (default
+** value)
+*/
+static int callrecursive (TTree *tree, int f (TTree *t), int def) {
+ int key = tree->key;
+ assert(tree->tag == TCall);
+ assert(sib2(tree)->tag == TRule);
+ if (key == 0) /* node already visited? */
+ return def; /* return default value */
+ else { /* first visit */
+ int result;
+ tree->key = 0; /* mark call as already visited */
+ result = f(sib2(tree)); /* go to called rule */
+ tree->key = key; /* restore tree */
+ return result;
+ }
+}
+
+
+/*
+** Check whether a pattern tree has captures
+*/
+int hascaptures (TTree *tree) {
+ tailcall:
+ switch (tree->tag) {
+ case TCapture: case TRunTime:
+ return 1;
+ case TCall:
+ return callrecursive(tree, hascaptures, 0);
+ case TRule: /* do not follow siblings */
+ tree = sib1(tree); goto tailcall;
+ case TOpenCall: assert(0);
+ default: {
+ switch (numsiblings[tree->tag]) {
+ case 1: /* return hascaptures(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case 2:
+ if (hascaptures(sib1(tree)))
+ return 1;
+ /* else return hascaptures(sib2(tree)); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(numsiblings[tree->tag] == 0); return 0;
+ }
+ }
+ }
+}
+
+
+/*
+** Checks how a pattern behaves regarding the empty string,
+** in one of two different ways:
+** A pattern is *nullable* if it can match without consuming any character;
+** A pattern is *nofail* if it never fails for any string
+** (including the empty string).
+** The difference is only for predicates and run-time captures;
+** for other patterns, the two properties are equivalent.
+** (With predicates, &'a' is nullable but not nofail. Of course,
+** nofail => nullable.)
+** These functions are all convervative in the following way:
+** p is nullable => nullable(p)
+** nofail(p) => p cannot fail
+** The function assumes that TOpenCall is not nullable;
+** this will be checked again when the grammar is fixed.
+** Run-time captures can do whatever they want, so the result
+** is conservative.
+*/
+int checkaux (TTree *tree, int pred) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ case TFalse: case TOpenCall:
+ return 0; /* not nullable */
+ case TRep: case TTrue:
+ return 1; /* no fail */
+ case TNot: case TBehind: /* can match empty, but can fail */
+ if (pred == PEnofail) return 0;
+ else return 1; /* PEnullable */
+ case TAnd: /* can match empty; fail iff body does */
+ if (pred == PEnullable) return 1;
+ /* else return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TRunTime: /* can fail; match empty iff body does */
+ if (pred == PEnofail) return 0;
+ /* else return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TSeq:
+ if (!checkaux(sib1(tree), pred)) return 0;
+ /* else return checkaux(sib2(tree), pred); */
+ tree = sib2(tree); goto tailcall;
+ case TChoice:
+ if (checkaux(sib2(tree), pred)) return 1;
+ /* else return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TCapture: case TGrammar: case TRule:
+ /* return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TCall: /* return checkaux(sib2(tree), pred); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(0); return 0;
+ }
+}
+
+
+/*
+** number of characters to match a pattern (or -1 if variable)
+*/
+int fixedlen (TTree *tree) {
+ int len = 0; /* to accumulate in tail calls */
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ return len + 1;
+ case TFalse: case TTrue: case TNot: case TAnd: case TBehind:
+ return len;
+ case TRep: case TRunTime: case TOpenCall:
+ return -1;
+ case TCapture: case TRule: case TGrammar:
+ /* return fixedlen(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case TCall: {
+ int n1 = callrecursive(tree, fixedlen, -1);
+ if (n1 < 0)
+ return -1;
+ else
+ return len + n1;
+ }
+ case TSeq: {
+ int n1 = fixedlen(sib1(tree));
+ if (n1 < 0)
+ return -1;
+ /* else return fixedlen(sib2(tree)) + len; */
+ len += n1; tree = sib2(tree); goto tailcall;
+ }
+ case TChoice: {
+ int n1 = fixedlen(sib1(tree));
+ int n2 = fixedlen(sib2(tree));
+ if (n1 != n2 || n1 < 0)
+ return -1;
+ else
+ return len + n1;
+ }
+ default: assert(0); return 0;
+ };
+}
+
+
+/*
+** Computes the 'first set' of a pattern.
+** The result is a conservative aproximation:
+** match p ax -> x (for some x) ==> a belongs to first(p)
+** or
+** a not in first(p) ==> match p ax -> fail (for all x)
+**
+** The set 'follow' is the first set of what follows the
+** pattern (full set if nothing follows it).
+**
+** The function returns 0 when this resulting set can be used for
+** test instructions that avoid the pattern altogether.
+** A non-zero return can happen for two reasons:
+** 1) match p '' -> '' ==> return has bit 1 set
+** (tests cannot be used because they would always fail for an empty input);
+** 2) there is a match-time capture ==> return has bit 2 set
+** (optimizations should not bypass match-time captures).
+*/
+static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny: {
+ tocharset(tree, firstset);
+ return 0;
+ }
+ case TTrue: {
+ loopset(i, firstset->cs[i] = follow->cs[i]);
+ return 1; /* accepts the empty string */
+ }
+ case TFalse: {
+ loopset(i, firstset->cs[i] = 0);
+ return 0;
+ }
+ case TChoice: {
+ Charset csaux;
+ int e1 = getfirst(sib1(tree), follow, firstset);
+ int e2 = getfirst(sib2(tree), follow, &csaux);
+ loopset(i, firstset->cs[i] |= csaux.cs[i]);
+ return e1 | e2;
+ }
+ case TSeq: {
+ if (!nullable(sib1(tree))) {
+ /* when p1 is not nullable, p2 has nothing to contribute;
+ return getfirst(sib1(tree), fullset, firstset); */
+ tree = sib1(tree); follow = fullset; goto tailcall;
+ }
+ else { /* FIRST(p1 p2, fl) = FIRST(p1, FIRST(p2, fl)) */
+ Charset csaux;
+ int e2 = getfirst(sib2(tree), follow, &csaux);
+ int e1 = getfirst(sib1(tree), &csaux, firstset);
+ if (e1 == 0) return 0; /* 'e1' ensures that first can be used */
+ else if ((e1 | e2) & 2) /* one of the children has a matchtime? */
+ return 2; /* pattern has a matchtime capture */
+ else return e2; /* else depends on 'e2' */
+ }
+ }
+ case TRep: {
+ getfirst(sib1(tree), follow, firstset);
+ loopset(i, firstset->cs[i] |= follow->cs[i]);
+ return 1; /* accept the empty string */
+ }
+ case TCapture: case TGrammar: case TRule: {
+ /* return getfirst(sib1(tree), follow, firstset); */
+ tree = sib1(tree); goto tailcall;
+ }
+ case TRunTime: { /* function invalidates any follow info. */
+ int e = getfirst(sib1(tree), fullset, firstset);
+ if (e) return 2; /* function is not "protected"? */
+ else return 0; /* pattern inside capture ensures first can be used */
+ }
+ case TCall: {
+ /* return getfirst(sib2(tree), follow, firstset); */
+ tree = sib2(tree); goto tailcall;
+ }
+ case TAnd: {
+ int e = getfirst(sib1(tree), follow, firstset);
+ loopset(i, firstset->cs[i] &= follow->cs[i]);
+ return e;
+ }
+ case TNot: {
+ if (tocharset(sib1(tree), firstset)) {
+ cs_complement(firstset);
+ return 1;
+ }
+ /* else go through */
+ }
+ case TBehind: { /* instruction gives no new information */
+ /* call 'getfirst' only to check for math-time captures */
+ int e = getfirst(sib1(tree), follow, firstset);
+ loopset(i, firstset->cs[i] = follow->cs[i]); /* uses follow */
+ return e | 1; /* always can accept the empty string */
+ }
+ default: assert(0); return 0;
+ }
+}
+
+
+/*
+** If 'headfail(tree)' true, then 'tree' can fail only depending on the
+** next character of the subject.
+*/
+static int headfail (TTree *tree) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny: case TFalse:
+ return 1;
+ case TTrue: case TRep: case TRunTime: case TNot:
+ case TBehind:
+ return 0;
+ case TCapture: case TGrammar: case TRule: case TAnd:
+ tree = sib1(tree); goto tailcall; /* return headfail(sib1(tree)); */
+ case TCall:
+ tree = sib2(tree); goto tailcall; /* return headfail(sib2(tree)); */
+ case TSeq:
+ if (!nofail(sib2(tree))) return 0;
+ /* else return headfail(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case TChoice:
+ if (!headfail(sib1(tree))) return 0;
+ /* else return headfail(sib2(tree)); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(0); return 0;
+ }
+}
+
+
+/*
+** Check whether the code generation for the given tree can benefit
+** from a follow set (to avoid computing the follow set when it is
+** not needed)
+*/
+static int needfollow (TTree *tree) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ case TFalse: case TTrue: case TAnd: case TNot:
+ case TRunTime: case TGrammar: case TCall: case TBehind:
+ return 0;
+ case TChoice: case TRep:
+ return 1;
+ case TCapture:
+ tree = sib1(tree); goto tailcall;
+ case TSeq:
+ tree = sib2(tree); goto tailcall;
+ default: assert(0); return 0;
+ }
+}
+
+/* }====================================================== */
+
+
+
+/*
+** {======================================================
+** Code generation
+** =======================================================
+*/
+
+
+/*
+** size of an instruction
+*/
+int sizei (const Instruction *i) {
+ switch((Opcode)i->i.code) {
+ case ISet: case ISpan: return CHARSETINSTSIZE;
+ case ITestSet: return CHARSETINSTSIZE + 1;
+ case ITestChar: case ITestAny: case IChoice: case IJmp: case ICall:
+ case IOpenCall: case ICommit: case IPartialCommit: case IBackCommit:
+ return 2;
+ default: return 1;
+ }
+}
+
+
+/*
+** state for the compiler
+*/
+typedef struct CompileState {
+ Pattern *p; /* pattern being compiled */
+ int ncode; /* next position in p->code to be filled */
+ lua_State *L;
+} CompileState;
+
+
+/*
+** code generation is recursive; 'opt' indicates that the code is being
+** generated as the last thing inside an optional pattern (so, if that
+** code is optional too, it can reuse the 'IChoice' already in place for
+** the outer pattern). 'tt' points to a previous test protecting this
+** code (or NOINST). 'fl' is the follow set of the pattern.
+*/
+static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
+ const Charset *fl);
+
+
+void realloccode (lua_State *L, Pattern *p, int nsize) {
+ void *ud;
+ lua_Alloc f = lua_getallocf(L, &ud);
+ void *newblock = f(ud, p->code, p->codesize * sizeof(Instruction),
+ nsize * sizeof(Instruction));
+ if (newblock == NULL && nsize > 0)
+ luaL_error(L, "not enough memory");
+ p->code = (Instruction *)newblock;
+ p->codesize = nsize;
+}
+
+
+static int nextinstruction (CompileState *compst) {
+ int size = compst->p->codesize;
+ if (compst->ncode >= size)
+ realloccode(compst->L, compst->p, size * 2);
+ return compst->ncode++;
+}
+
+
+#define getinstr(cs,i) ((cs)->p->code[i])
+
+
+static int addinstruction (CompileState *compst, Opcode op, int aux) {
+ int i = nextinstruction(compst);
+ getinstr(compst, i).i.code = op;
+ getinstr(compst, i).i.aux = aux;
+ return i;
+}
+
+
+/*
+** Add an instruction followed by space for an offset (to be set later)
+*/
+static int addoffsetinst (CompileState *compst, Opcode op) {
+ int i = addinstruction(compst, op, 0); /* instruction */
+ addinstruction(compst, (Opcode)0, 0); /* open space for offset */
+ assert(op == ITestSet || sizei(&getinstr(compst, i)) == 2);
+ return i;
+}
+
+
+/*
+** Set the offset of an instruction
+*/
+static void setoffset (CompileState *compst, int instruction, int offset) {
+ getinstr(compst, instruction + 1).offset = offset;
+}
+
+
+/*
+** Add a capture instruction:
+** 'op' is the capture instruction; 'cap' the capture kind;
+** 'key' the key into ktable; 'aux' is the optional capture offset
+**
+*/
+static int addinstcap (CompileState *compst, Opcode op, int cap, int key,
+ int aux) {
+ int i = addinstruction(compst, op, joinkindoff(cap, aux));
+ getinstr(compst, i).i.key = key;
+ return i;
+}
+
+
+#define gethere(compst) ((compst)->ncode)
+
+#define target(code,i) ((i) + code[i + 1].offset)
+
+
+/*
+** Patch 'instruction' to jump to 'target'
+*/
+static void jumptothere (CompileState *compst, int instruction, int target) {
+ if (instruction >= 0)
+ setoffset(compst, instruction, target - instruction);
+}
+
+
+/*
+** Patch 'instruction' to jump to current position
+*/
+static void jumptohere (CompileState *compst, int instruction) {
+ jumptothere(compst, instruction, gethere(compst));
+}
+
+
+/*
+** Code an IChar instruction, or IAny if there is an equivalent
+** test dominating it
+*/
+static void codechar (CompileState *compst, int c, int tt) {
+ if (tt >= 0 && getinstr(compst, tt).i.code == ITestChar &&
+ getinstr(compst, tt).i.aux == c)
+ addinstruction(compst, IAny, 0);
+ else
+ addinstruction(compst, IChar, c);
+}
+
+
+/*
+** Add a charset posfix to an instruction
+*/
+static void addcharset (CompileState *compst, const byte *cs) {
+ int p = gethere(compst);
+ int i;
+ for (i = 0; i < (int)CHARSETINSTSIZE - 1; i++)
+ nextinstruction(compst); /* space for buffer */
+ /* fill buffer with charset */
+ loopset(j, getinstr(compst, p).buff[j] = cs[j]);
+}
+
+
+/*
+** code a char set, optimizing unit sets for IChar, "complete"
+** sets for IAny, and empty sets for IFail; also use an IAny
+** when instruction is dominated by an equivalent test.
+*/
+static void codecharset (CompileState *compst, const byte *cs, int tt) {
+ int c = 0; /* (=) to avoid warnings */
+ Opcode op = charsettype(cs, &c);
+ switch (op) {
+ case IChar: codechar(compst, c, tt); break;
+ case ISet: { /* non-trivial set? */
+ if (tt >= 0 && getinstr(compst, tt).i.code == ITestSet &&
+ cs_equal(cs, getinstr(compst, tt + 2).buff))
+ addinstruction(compst, IAny, 0);
+ else {
+ addinstruction(compst, ISet, 0);
+ addcharset(compst, cs);
+ }
+ break;
+ }
+ default: addinstruction(compst, op, c); break;
+ }
+}
+
+
+/*
+** code a test set, optimizing unit sets for ITestChar, "complete"
+** sets for ITestAny, and empty sets for IJmp (always fails).
+** 'e' is true iff test should accept the empty string. (Test
+** instructions in the current VM never accept the empty string.)
+*/
+static int codetestset (CompileState *compst, Charset *cs, int e) {
+ if (e) return NOINST; /* no test */
+ else {
+ int c = 0;
+ Opcode op = charsettype(cs->cs, &c);
+ switch (op) {
+ case IFail: return addoffsetinst(compst, IJmp); /* always jump */
+ case IAny: return addoffsetinst(compst, ITestAny);
+ case IChar: {
+ int i = addoffsetinst(compst, ITestChar);
+ getinstr(compst, i).i.aux = c;
+ return i;
+ }
+ case ISet: {
+ int i = addoffsetinst(compst, ITestSet);
+ addcharset(compst, cs->cs);
+ return i;
+ }
+ default: assert(0); return 0;
+ }
+ }
+}
+
+
+/*
+** Find the final destination of a sequence of jumps
+*/
+static int finaltarget (Instruction *code, int i) {
+ while (code[i].i.code == IJmp)
+ i = target(code, i);
+ return i;
+}
+
+
+/*
+** final label (after traversing any jumps)
+*/
+static int finallabel (Instruction *code, int i) {
+ return finaltarget(code, target(code, i));
+}
+
+
+/*
+** <behind(p)> == behind n; <p> (where n = fixedlen(p))
+*/
+static void codebehind (CompileState *compst, TTree *tree) {
+ if (tree->u.n > 0)
+ addinstruction(compst, IBehind, tree->u.n);
+ codegen(compst, sib1(tree), 0, NOINST, fullset);
+}
+
+
+/*
+** Choice; optimizations:
+** - when p1 is headfail or
+** when first(p1) and first(p2) are disjoint, than
+** a character not in first(p1) cannot go to p1, and a character
+** in first(p1) cannot go to p2 (at it is not in first(p2)).
+** (The optimization is not valid if p1 accepts the empty string,
+** as then there is no character at all...)
+** - when p2 is empty and opt is true; a IPartialCommit can reuse
+** the Choice already active in the stack.
+*/
+static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
+ const Charset *fl) {
+ int emptyp2 = (p2->tag == TTrue);
+ Charset cs1, cs2;
+ int e1 = getfirst(p1, fullset, &cs1);
+ if (headfail(p1) ||
+ (!e1 && (getfirst(p2, fl, &cs2), cs_disjoint(&cs1, &cs2)))) {
+ /* <p1 / p2> == test (fail(p1)) -> L1 ; p1 ; jmp L2; L1: p2; L2: */
+ int test = codetestset(compst, &cs1, 0);
+ int jmp = NOINST;
+ codegen(compst, p1, 0, test, fl);
+ if (!emptyp2)
+ jmp = addoffsetinst(compst, IJmp);
+ jumptohere(compst, test);
+ codegen(compst, p2, opt, NOINST, fl);
+ jumptohere(compst, jmp);
+ }
+ else if (opt && emptyp2) {
+ /* p1? == IPartialCommit; p1 */
+ jumptohere(compst, addoffsetinst(compst, IPartialCommit));
+ codegen(compst, p1, 1, NOINST, fullset);
+ }
+ else {
+ /* <p1 / p2> ==
+ test(first(p1)) -> L1; choice L1; <p1>; commit L2; L1: <p2>; L2: */
+ int pcommit;
+ int test = codetestset(compst, &cs1, e1);
+ int pchoice = addoffsetinst(compst, IChoice);
+ codegen(compst, p1, emptyp2, test, fullset);
+ pcommit = addoffsetinst(compst, ICommit);
+ jumptohere(compst, pchoice);
+ jumptohere(compst, test);
+ codegen(compst, p2, opt, NOINST, fl);
+ jumptohere(compst, pcommit);
+ }
+}
+
+
+/*
+** And predicate
+** optimization: fixedlen(p) = n ==> <&p> == <p>; behind n
+** (valid only when 'p' has no captures)
+*/
+static void codeand (CompileState *compst, TTree *tree, int tt) {
+ int n = fixedlen(tree);
+ if (n >= 0 && n <= MAXBEHIND && !hascaptures(tree)) {
+ codegen(compst, tree, 0, tt, fullset);
+ if (n > 0)
+ addinstruction(compst, IBehind, n);
+ }
+ else { /* default: Choice L1; p1; BackCommit L2; L1: Fail; L2: */
+ int pcommit;
+ int pchoice = addoffsetinst(compst, IChoice);
+ codegen(compst, tree, 0, tt, fullset);
+ pcommit = addoffsetinst(compst, IBackCommit);
+ jumptohere(compst, pchoice);
+ addinstruction(compst, IFail, 0);
+ jumptohere(compst, pcommit);
+ }
+}
+
+
+/*
+** Captures: if pattern has fixed (and not too big) length, and it
+** has no nested captures, use a single IFullCapture instruction
+** after the match; otherwise, enclose the pattern with OpenCapture -
+** CloseCapture.
+*/
+static void codecapture (CompileState *compst, TTree *tree, int tt,
+ const Charset *fl) {
+ int len = fixedlen(sib1(tree));
+ if (len >= 0 && len <= MAXOFF && !hascaptures(sib1(tree))) {
+ codegen(compst, sib1(tree), 0, tt, fl);
+ addinstcap(compst, IFullCapture, tree->cap, tree->key, len);
+ }
+ else {
+ addinstcap(compst, IOpenCapture, tree->cap, tree->key, 0);
+ codegen(compst, sib1(tree), 0, tt, fl);
+ addinstcap(compst, ICloseCapture, Cclose, 0, 0);
+ }
+}
+
+
+static void coderuntime (CompileState *compst, TTree *tree, int tt) {
+ addinstcap(compst, IOpenCapture, Cgroup, tree->key, 0);
+ codegen(compst, sib1(tree), 0, tt, fullset);
+ addinstcap(compst, ICloseRunTime, Cclose, 0, 0);
+}
+
+
+/*
+** Repetion; optimizations:
+** When pattern is a charset, can use special instruction ISpan.
+** When pattern is head fail, or if it starts with characters that
+** are disjoint from what follows the repetions, a simple test
+** is enough (a fail inside the repetition would backtrack to fail
+** again in the following pattern, so there is no need for a choice).
+** When 'opt' is true, the repetion can reuse the Choice already
+** active in the stack.
+*/
+static void coderep (CompileState *compst, TTree *tree, int opt,
+ const Charset *fl) {
+ Charset st;
+ if (tocharset(tree, &st)) {
+ addinstruction(compst, ISpan, 0);
+ addcharset(compst, st.cs);
+ }
+ else {
+ int e1 = getfirst(tree, fullset, &st);
+ if (headfail(tree) || (!e1 && cs_disjoint(&st, fl))) {
+ /* L1: test (fail(p1)) -> L2; <p>; jmp L1; L2: */
+ int jmp;
+ int test = codetestset(compst, &st, 0);
+ codegen(compst, tree, 0, test, fullset);
+ jmp = addoffsetinst(compst, IJmp);
+ jumptohere(compst, test);
+ jumptothere(compst, jmp, test);
+ }
+ else {
+ /* test(fail(p1)) -> L2; choice L2; L1: <p>; partialcommit L1; L2: */
+ /* or (if 'opt'): partialcommit L1; L1: <p>; partialcommit L1; */
+ int commit, l2;
+ int test = codetestset(compst, &st, e1);
+ int pchoice = NOINST;
+ if (opt)
+ jumptohere(compst, addoffsetinst(compst, IPartialCommit));
+ else
+ pchoice = addoffsetinst(compst, IChoice);
+ l2 = gethere(compst);
+ codegen(compst, tree, 0, NOINST, fullset);
+ commit = addoffsetinst(compst, IPartialCommit);
+ jumptothere(compst, commit, l2);
+ jumptohere(compst, pchoice);
+ jumptohere(compst, test);
+ }
+ }
+}
+
+
+/*
+** Not predicate; optimizations:
+** In any case, if first test fails, 'not' succeeds, so it can jump to
+** the end. If pattern is headfail, that is all (it cannot fail
+** in other parts); this case includes 'not' of simple sets. Otherwise,
+** use the default code (a choice plus a failtwice).
+*/
+static void codenot (CompileState *compst, TTree *tree) {
+ Charset st;
+ int e = getfirst(tree, fullset, &st);
+ int test = codetestset(compst, &st, e);
+ if (headfail(tree)) /* test (fail(p1)) -> L1; fail; L1: */
+ addinstruction(compst, IFail, 0);
+ else {
+ /* test(fail(p))-> L1; choice L1; <p>; failtwice; L1: */
+ int pchoice = addoffsetinst(compst, IChoice);
+ codegen(compst, tree, 0, NOINST, fullset);
+ addinstruction(compst, IFailTwice, 0);
+ jumptohere(compst, pchoice);
+ }
+ jumptohere(compst, test);
+}
+
+
+/*
+** change open calls to calls, using list 'positions' to find
+** correct offsets; also optimize tail calls
+*/
+static void correctcalls (CompileState *compst, int *positions,
+ int from, int to) {
+ int i;
+ Instruction *code = compst->p->code;
+ for (i = from; i < to; i += sizei(&code[i])) {
+ if (code[i].i.code == IOpenCall) {
+ int n = code[i].i.key; /* rule number */
+ int rule = positions[n]; /* rule position */
+ assert(rule == from || code[rule - 1].i.code == IRet);
+ if (code[finaltarget(code, i + 2)].i.code == IRet) /* call; ret ? */
+ code[i].i.code = IJmp; /* tail call */
+ else
+ code[i].i.code = ICall;
+ jumptothere(compst, i, rule); /* call jumps to respective rule */
+ }
+ }
+ assert(i == to);
+}
+
+
+/*
+** Code for a grammar:
+** call L1; jmp L2; L1: rule 1; ret; rule 2; ret; ...; L2:
+*/
+static void codegrammar (CompileState *compst, TTree *grammar) {
+ int positions[MAXRULES];
+ int rulenumber = 0;
+ TTree *rule;
+ int firstcall = addoffsetinst(compst, ICall); /* call initial rule */
+ int jumptoend = addoffsetinst(compst, IJmp); /* jump to the end */
+ int start = gethere(compst); /* here starts the initial rule */
+ jumptohere(compst, firstcall);
+ for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
+ positions[rulenumber++] = gethere(compst); /* save rule position */
+ codegen(compst, sib1(rule), 0, NOINST, fullset); /* code rule */
+ addinstruction(compst, IRet, 0);
+ }
+ assert(rule->tag == TTrue);
+ jumptohere(compst, jumptoend);
+ correctcalls(compst, positions, start, gethere(compst));
+}
+
+
+static void codecall (CompileState *compst, TTree *call) {
+ int c = addoffsetinst(compst, IOpenCall); /* to be corrected later */
+ getinstr(compst, c).i.key = sib2(call)->cap; /* rule number */
+ assert(sib2(call)->tag == TRule);
+}
+
+
+/*
+** Code first child of a sequence
+** (second child is called in-place to allow tail call)
+** Return 'tt' for second child
+*/
+static int codeseq1 (CompileState *compst, TTree *p1, TTree *p2,
+ int tt, const Charset *fl) {
+ if (needfollow(p1)) {
+ Charset fl1;
+ getfirst(p2, fl, &fl1); /* p1 follow is p2 first */
+ codegen(compst, p1, 0, tt, &fl1);
+ }
+ else /* use 'fullset' as follow */
+ codegen(compst, p1, 0, tt, fullset);
+ if (fixedlen(p1) != 0) /* can 'p1' consume anything? */
+ return NOINST; /* invalidate test */
+ else return tt; /* else 'tt' still protects sib2 */
+}
+
+
+/*
+** Main code-generation function: dispatch to auxiliar functions
+** according to kind of tree. ('needfollow' should return true
+** only for consructions that use 'fl'.)
+*/
+static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
+ const Charset *fl) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: codechar(compst, tree->u.n, tt); break;
+ case TAny: addinstruction(compst, IAny, 0); break;
+ case TSet: codecharset(compst, treebuffer(tree), tt); break;
+ case TTrue: break;
+ case TFalse: addinstruction(compst, IFail, 0); break;
+ case TChoice: codechoice(compst, sib1(tree), sib2(tree), opt, fl); break;
+ case TRep: coderep(compst, sib1(tree), opt, fl); break;
+ case TBehind: codebehind(compst, tree); break;
+ case TNot: codenot(compst, sib1(tree)); break;
+ case TAnd: codeand(compst, sib1(tree), tt); break;
+ case TCapture: codecapture(compst, tree, tt, fl); break;
+ case TRunTime: coderuntime(compst, tree, tt); break;
+ case TGrammar: codegrammar(compst, tree); break;
+ case TCall: codecall(compst, tree); break;
+ case TSeq: {
+ tt = codeseq1(compst, sib1(tree), sib2(tree), tt, fl); /* code 'p1' */
+ /* codegen(compst, p2, opt, tt, fl); */
+ tree = sib2(tree); goto tailcall;
+ }
+ default: assert(0);
+ }
+}
+
+
+/*
+** Optimize jumps and other jump-like instructions.
+** * Update labels of instructions with labels to their final
+** destinations (e.g., choice L1; ... L1: jmp L2: becomes
+** choice L2)
+** * Jumps to other instructions that do jumps become those
+** instructions (e.g., jump to return becomes a return; jump
+** to commit becomes a commit)
+*/
+static void peephole (CompileState *compst) {
+ Instruction *code = compst->p->code;
+ int i;
+ for (i = 0; i < compst->ncode; i += sizei(&code[i])) {
+ redo:
+ switch (code[i].i.code) {
+ case IChoice: case ICall: case ICommit: case IPartialCommit:
+ case IBackCommit: case ITestChar: case ITestSet:
+ case ITestAny: { /* instructions with labels */
+ jumptothere(compst, i, finallabel(code, i)); /* optimize label */
+ break;
+ }
+ case IJmp: {
+ int ft = finaltarget(code, i);
+ switch (code[ft].i.code) { /* jumping to what? */
+ case IRet: case IFail: case IFailTwice:
+ case IEnd: { /* instructions with unconditional implicit jumps */
+ code[i] = code[ft]; /* jump becomes that instruction */
+ code[i + 1].i.code = IAny; /* 'no-op' for target position */
+ break;
+ }
+ case ICommit: case IPartialCommit:
+ case IBackCommit: { /* inst. with unconditional explicit jumps */
+ int fft = finallabel(code, ft);
+ code[i] = code[ft]; /* jump becomes that instruction... */
+ jumptothere(compst, i, fft); /* but must correct its offset */
+ goto redo; /* reoptimize its label */
+ }
+ default: {
+ jumptothere(compst, i, ft); /* optimize label */
+ break;
+ }
+ }
+ break;
+ }
+ default: break;
+ }
+ }
+ assert(code[i - 1].i.code == IEnd);
+}
+
+
+/*
+** Compile a pattern
+*/
+Instruction *compile (lua_State *L, Pattern *p) {
+ CompileState compst;
+ compst.p = p; compst.ncode = 0; compst.L = L;
+ realloccode(L, p, 2); /* minimum initial size */
+ codegen(&compst, p->tree, 0, NOINST, fullset);
+ addinstruction(&compst, IEnd, 0);
+ realloccode(L, p, compst.ncode); /* set final size */
+ peephole(&compst);
+ return p->code;
+}
+
+
+/* }====================================================== */
+
diff --git a/contrib/lua-lpeg/lpcode.h b/contrib/lua-lpeg/lpcode.h
new file mode 100644
index 0000000..34ee276
--- /dev/null
+++ b/contrib/lua-lpeg/lpcode.h
@@ -0,0 +1,40 @@
+/*
+** $Id: lpcode.h $
+*/
+
+#if !defined(lpcode_h)
+#define lpcode_h
+
+#include "lua.h"
+
+#include "lptypes.h"
+#include "lptree.h"
+#include "lpvm.h"
+
+int tocharset (TTree *tree, Charset *cs);
+int checkaux (TTree *tree, int pred);
+int fixedlen (TTree *tree);
+int hascaptures (TTree *tree);
+int lp_gc (lua_State *L);
+Instruction *compile (lua_State *L, Pattern *p);
+void realloccode (lua_State *L, Pattern *p, int nsize);
+int sizei (const Instruction *i);
+
+
+#define PEnullable 0
+#define PEnofail 1
+
+/*
+** nofail(t) implies that 't' cannot fail with any input
+*/
+#define nofail(t) checkaux(t, PEnofail)
+
+/*
+** (not nullable(t)) implies 't' cannot match without consuming
+** something
+*/
+#define nullable(t) checkaux(t, PEnullable)
+
+
+
+#endif
diff --git a/contrib/lua-lpeg/lpegre.lua b/contrib/lua-lpeg/lpegre.lua
new file mode 100644
index 0000000..3bb8af7
--- /dev/null
+++ b/contrib/lua-lpeg/lpegre.lua
@@ -0,0 +1,267 @@
+-- $Id: re.lua $
+
+-- imported functions and modules
+local tonumber, type, print, error = tonumber, type, print, error
+local setmetatable = setmetatable
+local m = require"lpeg"
+
+-- 'm' will be used to parse expressions, and 'mm' will be used to
+-- create expressions; that is, 're' runs on 'm', creating patterns
+-- on 'mm'
+local mm = m
+
+-- pattern's metatable
+local mt = getmetatable(mm.P(0))
+
+
+
+-- No more global accesses after this point
+local version = _VERSION
+if version == "Lua 5.2" then _ENV = nil end
+
+
+local any = m.P(1)
+
+
+-- Pre-defined names
+local Predef = { nl = m.P"\n" }
+
+
+local mem
+local fmem
+local gmem
+
+
+local function updatelocale ()
+ mm.locale(Predef)
+ Predef.a = Predef.alpha
+ Predef.c = Predef.cntrl
+ Predef.d = Predef.digit
+ Predef.g = Predef.graph
+ Predef.l = Predef.lower
+ Predef.p = Predef.punct
+ Predef.s = Predef.space
+ Predef.u = Predef.upper
+ Predef.w = Predef.alnum
+ Predef.x = Predef.xdigit
+ Predef.A = any - Predef.a
+ Predef.C = any - Predef.c
+ Predef.D = any - Predef.d
+ Predef.G = any - Predef.g
+ Predef.L = any - Predef.l
+ Predef.P = any - Predef.p
+ Predef.S = any - Predef.s
+ Predef.U = any - Predef.u
+ Predef.W = any - Predef.w
+ Predef.X = any - Predef.x
+ mem = {} -- restart memoization
+ fmem = {}
+ gmem = {}
+ local mt = {__mode = "v"}
+ setmetatable(mem, mt)
+ setmetatable(fmem, mt)
+ setmetatable(gmem, mt)
+end
+
+
+updatelocale()
+
+
+
+local I = m.P(function (s,i) print(i, s:sub(1, i-1)); return i end)
+
+
+local function patt_error (s, i)
+ local msg = (#s < i + 20) and s:sub(i)
+ or s:sub(i,i+20) .. "..."
+ msg = ("pattern error near '%s'"):format(msg)
+ error(msg, 2)
+end
+
+local function mult (p, n)
+ local np = mm.P(true)
+ while n >= 1 do
+ if n%2 >= 1 then np = np * p end
+ p = p * p
+ n = n/2
+ end
+ return np
+end
+
+local function equalcap (s, i, c)
+ if type(c) ~= "string" then return nil end
+ local e = #c + i
+ if s:sub(i, e - 1) == c then return e else return nil end
+end
+
+
+local S = (Predef.space + "--" * (any - Predef.nl)^0)^0
+
+local name = m.R("AZ", "az", "__") * m.R("AZ", "az", "__", "09")^0
+
+local arrow = S * "<-"
+
+local seq_follow = m.P"/" + ")" + "}" + ":}" + "~}" + "|}" + (name * arrow) + -1
+
+name = m.C(name)
+
+
+-- a defined name only have meaning in a given environment
+local Def = name * m.Carg(1)
+
+
+local function getdef (id, defs)
+ local c = defs and defs[id]
+ if not c then error("undefined name: " .. id) end
+ return c
+end
+
+-- match a name and return a group of its corresponding definition
+-- and 'f' (to be folded in 'Suffix')
+local function defwithfunc (f)
+ return m.Cg(Def / getdef * m.Cc(f))
+end
+
+
+local num = m.C(m.R"09"^1) * S / tonumber
+
+local String = "'" * m.C((any - "'")^0) * "'" +
+ '"' * m.C((any - '"')^0) * '"'
+
+
+local defined = "%" * Def / function (c,Defs)
+ local cat = Defs and Defs[c] or Predef[c]
+ if not cat then error ("name '" .. c .. "' undefined") end
+ return cat
+end
+
+local Range = m.Cs(any * (m.P"-"/"") * (any - "]")) / mm.R
+
+local item = (defined + Range + m.C(any)) / m.P
+
+local Class =
+ "["
+ * (m.C(m.P"^"^-1)) -- optional complement symbol
+ * m.Cf(item * (item - "]")^0, mt.__add) /
+ function (c, p) return c == "^" and any - p or p end
+ * "]"
+
+local function adddef (t, k, exp)
+ if t[k] then
+ error("'"..k.."' already defined as a rule")
+ else
+ t[k] = exp
+ end
+ return t
+end
+
+local function firstdef (n, r) return adddef({n}, n, r) end
+
+
+local function NT (n, b)
+ if not b then
+ error("rule '"..n.."' used outside a grammar")
+ else return mm.V(n)
+ end
+end
+
+
+local exp = m.P{ "Exp",
+ Exp = S * ( m.V"Grammar"
+ + m.Cf(m.V"Seq" * ("/" * S * m.V"Seq")^0, mt.__add) );
+ Seq = m.Cf(m.Cc(m.P"") * m.V"Prefix"^0 , mt.__mul)
+ * (#seq_follow + patt_error);
+ Prefix = "&" * S * m.V"Prefix" / mt.__len
+ + "!" * S * m.V"Prefix" / mt.__unm
+ + m.V"Suffix";
+ Suffix = m.Cf(m.V"Primary" * S *
+ ( ( m.P"+" * m.Cc(1, mt.__pow)
+ + m.P"*" * m.Cc(0, mt.__pow)
+ + m.P"?" * m.Cc(-1, mt.__pow)
+ + "^" * ( m.Cg(num * m.Cc(mult))
+ + m.Cg(m.C(m.S"+-" * m.R"09"^1) * m.Cc(mt.__pow))
+ )
+ + "->" * S * ( m.Cg((String + num) * m.Cc(mt.__div))
+ + m.P"{}" * m.Cc(nil, m.Ct)
+ + defwithfunc(mt.__div)
+ )
+ + "=>" * S * defwithfunc(m.Cmt)
+ + "~>" * S * defwithfunc(m.Cf)
+ ) * S
+ )^0, function (a,b,f) return f(a,b) end );
+ Primary = "(" * m.V"Exp" * ")"
+ + String / mm.P
+ + Class
+ + defined
+ + "{:" * (name * ":" + m.Cc(nil)) * m.V"Exp" * ":}" /
+ function (n, p) return mm.Cg(p, n) end
+ + "=" * name / function (n) return mm.Cmt(mm.Cb(n), equalcap) end
+ + m.P"{}" / mm.Cp
+ + "{~" * m.V"Exp" * "~}" / mm.Cs
+ + "{|" * m.V"Exp" * "|}" / mm.Ct
+ + "{" * m.V"Exp" * "}" / mm.C
+ + m.P"." * m.Cc(any)
+ + (name * -arrow + "<" * name * ">") * m.Cb("G") / NT;
+ Definition = name * arrow * m.V"Exp";
+ Grammar = m.Cg(m.Cc(true), "G") *
+ m.Cf(m.V"Definition" / firstdef * m.Cg(m.V"Definition")^0,
+ adddef) / mm.P
+}
+
+local pattern = S * m.Cg(m.Cc(false), "G") * exp / mm.P * (-any + patt_error)
+
+
+local function compile (p, defs)
+ if mm.type(p) == "pattern" then return p end -- already compiled
+ local cp = pattern:match(p, 1, defs)
+ if not cp then error("incorrect pattern", 3) end
+ return cp
+end
+
+local function match (s, p, i)
+ local cp = mem[p]
+ if not cp then
+ cp = compile(p)
+ mem[p] = cp
+ end
+ return cp:match(s, i or 1)
+end
+
+local function find (s, p, i)
+ local cp = fmem[p]
+ if not cp then
+ cp = compile(p) / 0
+ cp = mm.P{ mm.Cp() * cp * mm.Cp() + 1 * mm.V(1) }
+ fmem[p] = cp
+ end
+ local i, e = cp:match(s, i or 1)
+ if i then return i, e - 1
+ else return i
+ end
+end
+
+local function gsub (s, p, rep)
+ local g = gmem[p] or {} -- ensure gmem[p] is not collected while here
+ gmem[p] = g
+ local cp = g[rep]
+ if not cp then
+ cp = compile(p)
+ cp = mm.Cs((cp / rep + 1)^0)
+ g[rep] = cp
+ end
+ return cp:match(s)
+end
+
+
+-- exported names
+local re = {
+ compile = compile,
+ match = match,
+ find = find,
+ gsub = gsub,
+ updatelocale = updatelocale,
+}
+
+if version == "Lua 5.1" then _G.re = re end
+
+return re
diff --git a/contrib/lua-lpeg/lpprint.c b/contrib/lua-lpeg/lpprint.c
new file mode 100644
index 0000000..174d168
--- /dev/null
+++ b/contrib/lua-lpeg/lpprint.c
@@ -0,0 +1,244 @@
+/*
+** $Id: lpprint.c,v 1.9 2015/06/15 16:09:57 roberto Exp $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include <ctype.h>
+#include <limits.h>
+#include <stdio.h>
+
+
+#include "lptypes.h"
+#include "lpprint.h"
+#include "lpcode.h"
+
+
+#if defined(LPEG_DEBUG)
+
+/*
+** {======================================================
+** Printing patterns (for debugging)
+** =======================================================
+*/
+
+
+void printcharset (const byte *st) {
+ int i;
+ printf("[");
+ for (i = 0; i <= UCHAR_MAX; i++) {
+ int first = i;
+ while (testchar(st, i) && i <= UCHAR_MAX) i++;
+ if (i - 1 == first) /* unary range? */
+ printf("(%02x)", first);
+ else if (i - 1 > first) /* non-empty range? */
+ printf("(%02x-%02x)", first, i - 1);
+ }
+ printf("]");
+}
+
+
+static void printcapkind (int kind) {
+ const char *const modes[] = {
+ "close", "position", "constant", "backref",
+ "argument", "simple", "table", "function",
+ "query", "string", "num", "substitution", "fold",
+ "runtime", "group"};
+ printf("%s", modes[kind]);
+}
+
+
+static void printjmp (const Instruction *op, const Instruction *p) {
+ printf("-> %d", (int)(p + (p + 1)->offset - op));
+}
+
+
+void printinst (const Instruction *op, const Instruction *p) {
+ const char *const names[] = {
+ "any", "char", "set",
+ "testany", "testchar", "testset",
+ "span", "behind",
+ "ret", "end",
+ "choice", "jmp", "call", "open_call",
+ "commit", "partial_commit", "back_commit", "failtwice", "fail", "giveup",
+ "fullcapture", "opencapture", "closecapture", "closeruntime"
+ };
+ printf("%02ld: %s ", (long)(p - op), names[p->i.code]);
+ switch ((Opcode)p->i.code) {
+ case IChar: {
+ printf("'%c'", p->i.aux);
+ break;
+ }
+ case ITestChar: {
+ printf("'%c'", p->i.aux); printjmp(op, p);
+ break;
+ }
+ case IFullCapture: {
+ printcapkind(getkind(p));
+ printf(" (size = %d) (idx = %d)", getoff(p), p->i.key);
+ break;
+ }
+ case IOpenCapture: {
+ printcapkind(getkind(p));
+ printf(" (idx = %d)", p->i.key);
+ break;
+ }
+ case ISet: {
+ printcharset((p+1)->buff);
+ break;
+ }
+ case ITestSet: {
+ printcharset((p+2)->buff); printjmp(op, p);
+ break;
+ }
+ case ISpan: {
+ printcharset((p+1)->buff);
+ break;
+ }
+ case IOpenCall: {
+ printf("-> %d", (p + 1)->offset);
+ break;
+ }
+ case IBehind: {
+ printf("%d", p->i.aux);
+ break;
+ }
+ case IJmp: case ICall: case ICommit: case IChoice:
+ case IPartialCommit: case IBackCommit: case ITestAny: {
+ printjmp(op, p);
+ break;
+ }
+ default: break;
+ }
+ printf("\n");
+}
+
+
+void printpatt (Instruction *p, int n) {
+ Instruction *op = p;
+ while (p < op + n) {
+ printinst(op, p);
+ p += sizei(p);
+ }
+}
+
+
+#if defined(LPEG_DEBUG)
+static void printcap (Capture *cap) {
+ printcapkind(cap->kind);
+ printf(" (idx: %d - size: %d) -> %p\n", cap->idx, cap->siz, cap->s);
+}
+
+
+void printcaplist (Capture *cap, Capture *limit) {
+ printf(">======\n");
+ for (; cap->s && (limit == NULL || cap < limit); cap++)
+ printcap(cap);
+ printf("=======\n");
+}
+#endif
+
+/* }====================================================== */
+
+
+/*
+** {======================================================
+** Printing trees (for debugging)
+** =======================================================
+*/
+
+static const char *tagnames[] = {
+ "char", "set", "any",
+ "true", "false",
+ "rep",
+ "seq", "choice",
+ "not", "and",
+ "call", "opencall", "rule", "grammar",
+ "behind",
+ "capture", "run-time"
+};
+
+
+void printtree (TTree *tree, int ident) {
+ int i;
+ for (i = 0; i < ident; i++) printf(" ");
+ printf("%s", tagnames[tree->tag]);
+ switch (tree->tag) {
+ case TChar: {
+ int c = tree->u.n;
+ if (isprint(c))
+ printf(" '%c'\n", c);
+ else
+ printf(" (%02X)\n", c);
+ break;
+ }
+ case TSet: {
+ printcharset(treebuffer(tree));
+ printf("\n");
+ break;
+ }
+ case TOpenCall: case TCall: {
+ printf(" key: %d\n", tree->key);
+ break;
+ }
+ case TBehind: {
+ printf(" %d\n", tree->u.n);
+ printtree(sib1(tree), ident + 2);
+ break;
+ }
+ case TCapture: {
+ printf(" cap: %d key: %d n: %d\n", tree->cap, tree->key, tree->u.n);
+ printtree(sib1(tree), ident + 2);
+ break;
+ }
+ case TRule: {
+ printf(" n: %d key: %d\n", tree->cap, tree->key);
+ printtree(sib1(tree), ident + 2);
+ break; /* do not print next rule as a sibling */
+ }
+ case TGrammar: {
+ TTree *rule = sib1(tree);
+ printf(" %d\n", tree->u.n); /* number of rules */
+ for (i = 0; i < tree->u.n; i++) {
+ printtree(rule, ident + 2);
+ rule = sib2(rule);
+ }
+ assert(rule->tag == TTrue); /* sentinel */
+ break;
+ }
+ default: {
+ int sibs = numsiblings[tree->tag];
+ printf("\n");
+ if (sibs >= 1) {
+ printtree(sib1(tree), ident + 2);
+ if (sibs >= 2)
+ printtree(sib2(tree), ident + 2);
+ }
+ break;
+ }
+ }
+}
+
+
+void printktable (lua_State *L, int idx) {
+ int n, i;
+ lua_getuservalue(L, idx);
+ if (lua_isnil(L, -1)) /* no ktable? */
+ return;
+ n = lua_rawlen(L, -1);
+ printf("[");
+ for (i = 1; i <= n; i++) {
+ printf("%d = ", i);
+ lua_rawgeti(L, -1, i);
+ if (lua_isstring(L, -1))
+ printf("%s ", lua_tostring(L, -1));
+ else
+ printf("%s ", lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 1);
+ }
+ printf("]\n");
+ /* leave ktable at the stack */
+}
+
+/* }====================================================== */
+
+#endif
diff --git a/contrib/lua-lpeg/lpprint.h b/contrib/lua-lpeg/lpprint.h
new file mode 100644
index 0000000..6329760
--- /dev/null
+++ b/contrib/lua-lpeg/lpprint.h
@@ -0,0 +1,36 @@
+/*
+** $Id: lpprint.h,v 1.2 2015/06/12 18:18:08 roberto Exp $
+*/
+
+
+#if !defined(lpprint_h)
+#define lpprint_h
+
+
+#include "lptree.h"
+#include "lpvm.h"
+
+
+#if defined(LPEG_DEBUG)
+
+void printpatt (Instruction *p, int n);
+void printtree (TTree *tree, int ident);
+void printktable (lua_State *L, int idx);
+void printcharset (const byte *st);
+void printcaplist (Capture *cap, Capture *limit);
+void printinst (const Instruction *op, const Instruction *p);
+
+#else
+
+#define printktable(L,idx) \
+ luaL_error(L, "function only implemented in debug mode")
+#define printtree(tree,i) \
+ luaL_error(L, "function only implemented in debug mode")
+#define printpatt(p,n) \
+ luaL_error(L, "function only implemented in debug mode")
+
+#endif
+
+
+#endif
+
diff --git a/contrib/lua-lpeg/lptree.c b/contrib/lua-lpeg/lptree.c
new file mode 100644
index 0000000..df24e3c
--- /dev/null
+++ b/contrib/lua-lpeg/lptree.c
@@ -0,0 +1,1319 @@
+/*
+** $Id: lptree.c,v 1.21 2015/09/28 17:01:25 roberto Exp $
+** Copyright 2013, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include <ctype.h>
+#include <limits.h>
+#include <string.h>
+#include <src/lua/lua_common.h>
+
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lptypes.h"
+#include "lpcap.h"
+#include "lpcode.h"
+#include "lpprint.h"
+#include "lptree.h"
+
+
+/* number of siblings for each tree */
+const byte numsiblings[] = {
+ 0, 0, 0, /* char, set, any */
+ 0, 0, /* true, false */
+ 1, /* rep */
+ 2, 2, /* seq, choice */
+ 1, 1, /* not, and */
+ 0, 0, 2, 1, /* call, opencall, rule, grammar */
+ 1, /* behind */
+ 1, 1 /* capture, runtime capture */
+};
+
+
+static TTree *newgrammar (lua_State *L, int arg);
+
+
+/*
+** returns a reasonable name for value at index 'idx' on the stack
+*/
+static const char *val2str (lua_State *L, int idx) {
+ const char *k = lua_tostring(L, idx);
+ if (k != NULL)
+ return lua_pushfstring(L, "%s", k);
+ else
+ return lua_pushfstring(L, "(a %s)", luaL_typename(L, idx));
+}
+
+
+/*
+** Fix a TOpenCall into a TCall node, using table 'postable' to
+** translate a key to its rule address in the tree. Raises an
+** error if key does not exist.
+*/
+static void fixonecall (lua_State *L, int postable, TTree *g, TTree *t) {
+ int n;
+ lua_rawgeti(L, -1, t->key); /* get rule's name */
+ lua_gettable(L, postable); /* query name in position table */
+ n = lua_tonumber(L, -1); /* get (absolute) position */
+ lua_pop(L, 1); /* remove position */
+ if (n == 0) { /* no position? */
+ lua_rawgeti(L, -1, t->key); /* get rule's name again */
+ luaL_error(L, "rule '%s' undefined in given grammar", val2str(L, -1));
+ }
+ t->tag = TCall;
+ t->u.ps = n - (t - g); /* position relative to node */
+ assert(sib2(t)->tag == TRule);
+ sib2(t)->key = t->key;
+}
+
+
+/*
+** Transform left associative constructions into right
+** associative ones, for sequence and choice; that is:
+** (t11 + t12) + t2 => t11 + (t12 + t2)
+** (t11 * t12) * t2 => t11 * (t12 * t2)
+** (that is, Op (Op t11 t12) t2 => Op t11 (Op t12 t2))
+*/
+static void correctassociativity (TTree *tree) {
+ TTree *t1 = sib1(tree);
+ assert(tree->tag == TChoice || tree->tag == TSeq);
+ while (t1->tag == tree->tag) {
+ int n1size = tree->u.ps - 1; /* t1 == Op t11 t12 */
+ int n11size = t1->u.ps - 1;
+ int n12size = n1size - n11size - 1;
+ memmove(sib1(tree), sib1(t1), n11size * sizeof(TTree)); /* move t11 */
+ tree->u.ps = n11size + 1;
+ sib2(tree)->tag = tree->tag;
+ sib2(tree)->u.ps = n12size + 1;
+ }
+}
+
+
+/*
+** Make final adjustments in a tree. Fix open calls in tree 't',
+** making them refer to their respective rules or raising appropriate
+** errors (if not inside a grammar). Correct associativity of associative
+** constructions (making them right associative). Assume that tree's
+** ktable is at the top of the stack (for error messages).
+*/
+static void finalfix (lua_State *L, int postable, TTree *g, TTree *t) {
+ tailcall:
+ switch (t->tag) {
+ case TGrammar: /* subgrammars were already fixed */
+ return;
+ case TOpenCall: {
+ if (g != NULL) /* inside a grammar? */
+ fixonecall(L, postable, g, t);
+ else { /* open call outside grammar */
+ lua_rawgeti(L, -1, t->key);
+ luaL_error(L, "rule '%s' used outside a grammar", val2str(L, -1));
+ }
+ break;
+ }
+ case TSeq: case TChoice:
+ correctassociativity(t);
+ break;
+ }
+ switch (numsiblings[t->tag]) {
+ case 1: /* finalfix(L, postable, g, sib1(t)); */
+ t = sib1(t); goto tailcall;
+ case 2:
+ finalfix(L, postable, g, sib1(t));
+ t = sib2(t); goto tailcall; /* finalfix(L, postable, g, sib2(t)); */
+ default: assert(numsiblings[t->tag] == 0); break;
+ }
+}
+
+
+
+/*
+** {===================================================================
+** KTable manipulation
+**
+** - The ktable of a pattern 'p' can be shared by other patterns that
+** contain 'p' and no other constants. Because of this sharing, we
+** should not add elements to a 'ktable' unless it was freshly created
+** for the new pattern.
+**
+** - The maximum index in a ktable is USHRT_MAX, because trees and
+** patterns use unsigned shorts to store those indices.
+** ====================================================================
+*/
+
+/*
+** Create a new 'ktable' to the pattern at the top of the stack.
+*/
+static void newktable (lua_State *L, int n) {
+ lua_createtable(L, n, 0); /* create a fresh table */
+ lua_setuservalue(L, -2); /* set it as 'ktable' for pattern */
+}
+
+
+/*
+** Add element 'idx' to 'ktable' of pattern at the top of the stack;
+** Return index of new element.
+** If new element is nil, does not add it to table (as it would be
+** useless) and returns 0, as ktable[0] is always nil.
+*/
+static int addtoktable (lua_State *L, int idx) {
+ if (lua_isnil(L, idx)) /* nil value? */
+ return 0;
+ else {
+ int n;
+ lua_getuservalue(L, -1); /* get ktable from pattern */
+ n = lua_rawlen(L, -1);
+ if (n >= USHRT_MAX)
+ luaL_error(L, "too many Lua values in pattern");
+ lua_pushvalue(L, idx); /* element to be added */
+ lua_rawseti(L, -2, ++n);
+ lua_pop(L, 1); /* remove 'ktable' */
+ return n;
+ }
+}
+
+
+/*
+** Return the number of elements in the ktable at 'idx'.
+** In Lua 5.2/5.3, default "environment" for patterns is nil, not
+** a table. Treat it as an empty table. In Lua 5.1, assumes that
+** the environment has no numeric indices (len == 0)
+*/
+static int ktablelen (lua_State *L, int idx) {
+ if (!lua_istable(L, idx)) return 0;
+ else return lua_rawlen(L, idx);
+}
+
+
+/*
+** Concatenate the contents of table 'idx1' into table 'idx2'.
+** (Assume that both indices are negative.)
+** Return the original length of table 'idx2' (or 0, if no
+** element was added, as there is no need to correct any index).
+*/
+static int concattable (lua_State *L, int idx1, int idx2) {
+ int i;
+ int n1 = ktablelen(L, idx1);
+ int n2 = ktablelen(L, idx2);
+ if (n1 + n2 > USHRT_MAX)
+ luaL_error(L, "too many Lua values in pattern");
+ if (n1 == 0) return 0; /* nothing to correct */
+ for (i = 1; i <= n1; i++) {
+ lua_rawgeti(L, idx1, i);
+ lua_rawseti(L, idx2 - 1, n2 + i); /* correct 'idx2' */
+ }
+ return n2;
+}
+
+
+/*
+** When joining 'ktables', constants from one of the subpatterns must
+** be renumbered; 'correctkeys' corrects their indices (adding 'n'
+** to each of them)
+*/
+static void correctkeys (TTree *tree, int n) {
+ if (n == 0) return; /* no correction? */
+ tailcall:
+ switch (tree->tag) {
+ case TOpenCall: case TCall: case TRunTime: case TRule: {
+ if (tree->key > 0)
+ tree->key += n;
+ break;
+ }
+ case TCapture: {
+ if (tree->key > 0 && tree->cap != Carg && tree->cap != Cnum)
+ tree->key += n;
+ break;
+ }
+ default: break;
+ }
+ switch (numsiblings[tree->tag]) {
+ case 1: /* correctkeys(sib1(tree), n); */
+ tree = sib1(tree); goto tailcall;
+ case 2:
+ correctkeys(sib1(tree), n);
+ tree = sib2(tree); goto tailcall; /* correctkeys(sib2(tree), n); */
+ default: assert(numsiblings[tree->tag] == 0); break;
+ }
+}
+
+
+/*
+** Join the ktables from p1 and p2 the ktable for the new pattern at the
+** top of the stack, reusing them when possible.
+*/
+static void joinktables (lua_State *L, int p1, TTree *t2, int p2) {
+ int n1, n2;
+ lua_getuservalue(L, p1); /* get ktables */
+ lua_getuservalue(L, p2);
+ n1 = ktablelen(L, -2);
+ n2 = ktablelen(L, -1);
+ if (n1 == 0 && n2 == 0) /* are both tables empty? */
+ lua_pop(L, 2); /* nothing to be done; pop tables */
+ else if (n2 == 0 || lp_equal(L, -2, -1)) { /* 2nd table empty or equal? */
+ lua_pop(L, 1); /* pop 2nd table */
+ lua_setuservalue(L, -2); /* set 1st ktable into new pattern */
+ }
+ else if (n1 == 0) { /* first table is empty? */
+ lua_setuservalue(L, -3); /* set 2nd table into new pattern */
+ lua_pop(L, 1); /* pop 1st table */
+ }
+ else {
+ lua_createtable(L, n1 + n2, 0); /* create ktable for new pattern */
+ /* stack: new p; ktable p1; ktable p2; new ktable */
+ concattable(L, -3, -1); /* from p1 into new ktable */
+ concattable(L, -2, -1); /* from p2 into new ktable */
+ lua_setuservalue(L, -4); /* new ktable becomes 'p' environment */
+ lua_pop(L, 2); /* pop other ktables */
+ correctkeys(t2, n1); /* correction for indices from p2 */
+ }
+}
+
+
+/*
+** copy 'ktable' of element 'idx' to new tree (on top of stack)
+*/
+static void copyktable (lua_State *L, int idx) {
+ lua_getuservalue(L, idx);
+ lua_setuservalue(L, -2);
+}
+
+
+/*
+** merge 'ktable' from 'stree' at stack index 'idx' into 'ktable'
+** from tree at the top of the stack, and correct corresponding
+** tree.
+*/
+static void mergektable (lua_State *L, int idx, TTree *stree) {
+ int n;
+ lua_getuservalue(L, -1); /* get ktables */
+ lua_getuservalue(L, idx);
+ n = concattable(L, -1, -2);
+ lua_pop(L, 2); /* remove both ktables */
+ correctkeys(stree, n);
+}
+
+
+/*
+** Create a new 'ktable' to the pattern at the top of the stack, adding
+** all elements from pattern 'p' (if not 0) plus element 'idx' to it.
+** Return index of new element.
+*/
+static int addtonewktable (lua_State *L, int p, int idx) {
+ newktable(L, 1);
+ if (p)
+ mergektable(L, p, NULL);
+ return addtoktable(L, idx);
+}
+
+/* }====================================================== */
+
+
+/*
+** {======================================================
+** Tree generation
+** =======================================================
+*/
+
+/*
+** In 5.2, could use 'luaL_testudata'...
+*/
+static int testpattern (lua_State *L, int idx) {
+ if (lua_touserdata(L, idx)) { /* value is a userdata? */
+ if (lua_getmetatable(L, idx)) { /* does it have a metatable? */
+ luaL_getmetatable(L, PATTERN_T);
+ if (lua_rawequal(L, -1, -2)) { /* does it have the correct mt? */
+ lua_pop(L, 2); /* remove both metatables */
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
+
+static Pattern *getpattern (lua_State *L, int idx) {
+ return (Pattern *)luaL_checkudata(L, idx, PATTERN_T);
+}
+
+
+static int getsize (lua_State *L, int idx) {
+ return (lua_rawlen(L, idx) - sizeof(Pattern)) / sizeof(TTree) + 1;
+}
+
+
+static TTree *gettree (lua_State *L, int idx, int *len) {
+ Pattern *p = getpattern(L, idx);
+ if (len)
+ *len = getsize(L, idx);
+ return p->tree;
+}
+
+
+/*
+** create a pattern. Set its uservalue (the 'ktable') equal to its
+** metatable. (It could be any empty sequence; the metatable is at
+** hand here, so we use it.)
+*/
+static TTree *newtree (lua_State *L, int len) {
+ size_t size = (len - 1) * sizeof(TTree) + sizeof(Pattern);
+ Pattern *p = (Pattern *)lua_newuserdata(L, size);
+ memset(p, 0, size);
+ luaL_getmetatable(L, PATTERN_T);
+ lua_pushvalue(L, -1);
+ lua_setuservalue(L, -3);
+ lua_setmetatable(L, -2);
+ p->code = NULL; p->codesize = 0;
+ return p->tree;
+}
+
+
+static TTree *newleaf (lua_State *L, int tag) {
+ TTree *tree = newtree(L, 1);
+ tree->tag = tag;
+ return tree;
+}
+
+
+static TTree *newcharset (lua_State *L) {
+ TTree *tree = newtree(L, bytes2slots(CHARSETSIZE) + 1);
+ tree->tag = TSet;
+ loopset(i, treebuffer(tree)[i] = 0);
+ return tree;
+}
+
+
+/*
+** add to tree a sequence where first sibling is 'sib' (with size
+** 'sibsize'); returns position for second sibling
+*/
+static TTree *seqaux (TTree *tree, TTree *sib, int sibsize) {
+ tree->tag = TSeq; tree->u.ps = sibsize + 1;
+ memcpy(sib1(tree), sib, sibsize * sizeof(TTree));
+ return sib2(tree);
+}
+
+
+/*
+** Build a sequence of 'n' nodes, each with tag 'tag' and 'u.n' got
+** from the array 's' (or 0 if array is NULL). (TSeq is binary, so it
+** must build a sequence of sequence of sequence...)
+*/
+static void fillseq (TTree *tree, int tag, int n, const char *s) {
+ int i;
+ for (i = 0; i < n - 1; i++) { /* initial n-1 copies of Seq tag; Seq ... */
+ tree->tag = TSeq; tree->u.ps = 2;
+ sib1(tree)->tag = tag;
+ sib1(tree)->u.n = s ? (byte)s[i] : 0;
+ tree = sib2(tree);
+ }
+ tree->tag = tag; /* last one does not need TSeq */
+ tree->u.n = s ? (byte)s[i] : 0;
+}
+
+
+/*
+** Numbers as patterns:
+** 0 == true (always match); n == TAny repeated 'n' times;
+** -n == not (TAny repeated 'n' times)
+*/
+static TTree *numtree (lua_State *L, int n) {
+ if (n == 0)
+ return newleaf(L, TTrue);
+ else {
+ TTree *tree, *nd;
+ if (n > 0)
+ tree = nd = newtree(L, 2 * n - 1);
+ else { /* negative: code it as !(-n) */
+ n = -n;
+ tree = newtree(L, 2 * n);
+ tree->tag = TNot;
+ nd = sib1(tree);
+ }
+ fillseq(nd, TAny, n, NULL); /* sequence of 'n' any's */
+ return tree;
+ }
+}
+
+
+/*
+** Convert value at index 'idx' to a pattern
+*/
+static TTree *getpatt (lua_State *L, int idx, int *len) {
+ TTree *tree;
+ switch (lua_type(L, idx)) {
+ case LUA_TSTRING: {
+ size_t slen;
+ const char *s = lua_tolstring(L, idx, &slen); /* get string */
+ if (slen == 0) /* empty? */
+ tree = newleaf(L, TTrue); /* always match */
+ else {
+ tree = newtree(L, 2 * (slen - 1) + 1);
+ fillseq(tree, TChar, slen, s); /* sequence of 'slen' chars */
+ }
+ break;
+ }
+ case LUA_TNUMBER: {
+ int n = lua_tointeger(L, idx);
+ tree = numtree(L, n);
+ break;
+ }
+ case LUA_TBOOLEAN: {
+ tree = (lua_toboolean(L, idx) ? newleaf(L, TTrue) : newleaf(L, TFalse));
+ break;
+ }
+ case LUA_TTABLE: {
+ tree = newgrammar(L, idx);
+ break;
+ }
+ case LUA_TFUNCTION: {
+ tree = newtree(L, 2);
+ tree->tag = TRunTime;
+ tree->key = addtonewktable(L, 0, idx);
+ sib1(tree)->tag = TTrue;
+ break;
+ }
+ default: {
+ return gettree(L, idx, len);
+ }
+ }
+ lua_replace(L, idx); /* put new tree into 'idx' slot */
+ if (len)
+ *len = getsize(L, idx);
+ return tree;
+}
+
+
+/*
+** create a new tree, with a new root and one sibling.
+** Sibling must be on the Lua stack, at index 1.
+*/
+static TTree *newroot1sib (lua_State *L, int tag) {
+ int s1;
+ TTree *tree1 = getpatt(L, 1, &s1);
+ TTree *tree = newtree(L, 1 + s1); /* create new tree */
+ tree->tag = tag;
+ memcpy(sib1(tree), tree1, s1 * sizeof(TTree));
+ copyktable(L, 1);
+ return tree;
+}
+
+
+/*
+** create a new tree, with a new root and 2 siblings.
+** Siblings must be on the Lua stack, first one at index 1.
+*/
+static TTree *newroot2sib (lua_State *L, int tag) {
+ int s1, s2;
+ TTree *tree1 = getpatt(L, 1, &s1);
+ TTree *tree2 = getpatt(L, 2, &s2);
+ TTree *tree = newtree(L, 1 + s1 + s2); /* create new tree */
+ tree->tag = tag;
+ tree->u.ps = 1 + s1;
+ memcpy(sib1(tree), tree1, s1 * sizeof(TTree));
+ memcpy(sib2(tree), tree2, s2 * sizeof(TTree));
+ joinktables(L, 1, sib2(tree), 2);
+ return tree;
+}
+
+
+static int lp_P (lua_State *L) {
+ luaL_checkany(L, 1);
+ getpatt(L, 1, NULL);
+ lua_settop(L, 1);
+ return 1;
+}
+
+
+/*
+** sequence operator; optimizations:
+** false x => false, x true => x, true x => x
+** (cannot do x . false => false because x may have runtime captures)
+*/
+static int lp_seq (lua_State *L) {
+ TTree *tree1 = getpatt(L, 1, NULL);
+ TTree *tree2 = getpatt(L, 2, NULL);
+ if (tree1->tag == TFalse || tree2->tag == TTrue)
+ lua_pushvalue(L, 1); /* false . x == false, x . true = x */
+ else if (tree1->tag == TTrue)
+ lua_pushvalue(L, 2); /* true . x = x */
+ else
+ newroot2sib(L, TSeq);
+ return 1;
+}
+
+
+/*
+** choice operator; optimizations:
+** charset / charset => charset
+** true / x => true, x / false => x, false / x => x
+** (x / true is not equivalent to true)
+*/
+static int lp_choice (lua_State *L) {
+ Charset st1, st2;
+ TTree *t1 = getpatt(L, 1, NULL);
+ TTree *t2 = getpatt(L, 2, NULL);
+ if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
+ TTree *t = newcharset(L);
+ loopset(i, treebuffer(t)[i] = st1.cs[i] | st2.cs[i]);
+ }
+ else if (nofail(t1) || t2->tag == TFalse)
+ lua_pushvalue(L, 1); /* true / x => true, x / false => x */
+ else if (t1->tag == TFalse)
+ lua_pushvalue(L, 2); /* false / x => x */
+ else
+ newroot2sib(L, TChoice);
+ return 1;
+}
+
+
+/*
+** p^n
+*/
+static int lp_star (lua_State *L) {
+ int size1;
+ int n = (int)luaL_checkinteger(L, 2);
+ TTree *tree1 = getpatt(L, 1, &size1);
+ if (n >= 0) { /* seq tree1 (seq tree1 ... (seq tree1 (rep tree1))) */
+ TTree *tree = newtree(L, (n + 1) * (size1 + 1));
+ if (nullable(tree1))
+ luaL_error(L, "loop body may accept empty string");
+ while (n--) /* repeat 'n' times */
+ tree = seqaux(tree, tree1, size1);
+ tree->tag = TRep;
+ memcpy(sib1(tree), tree1, size1 * sizeof(TTree));
+ }
+ else { /* choice (seq tree1 ... choice tree1 true ...) true */
+ TTree *tree;
+ n = -n;
+ /* size = (choice + seq + tree1 + true) * n, but the last has no seq */
+ tree = newtree(L, n * (size1 + 3) - 1);
+ for (; n > 1; n--) { /* repeat (n - 1) times */
+ tree->tag = TChoice; tree->u.ps = n * (size1 + 3) - 2;
+ sib2(tree)->tag = TTrue;
+ tree = sib1(tree);
+ tree = seqaux(tree, tree1, size1);
+ }
+ tree->tag = TChoice; tree->u.ps = size1 + 1;
+ sib2(tree)->tag = TTrue;
+ memcpy(sib1(tree), tree1, size1 * sizeof(TTree));
+ }
+ copyktable(L, 1);
+ return 1;
+}
+
+
+/*
+** #p == &p
+*/
+static int lp_and (lua_State *L) {
+ newroot1sib(L, TAnd);
+ return 1;
+}
+
+
+/*
+** -p == !p
+*/
+static int lp_not (lua_State *L) {
+ newroot1sib(L, TNot);
+ return 1;
+}
+
+
+/*
+** [t1 - t2] == Seq (Not t2) t1
+** If t1 and t2 are charsets, make their difference.
+*/
+static int lp_sub (lua_State *L) {
+ Charset st1, st2;
+ int s1, s2;
+ TTree *t1 = getpatt(L, 1, &s1);
+ TTree *t2 = getpatt(L, 2, &s2);
+ if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
+ TTree *t = newcharset(L);
+ loopset(i, treebuffer(t)[i] = st1.cs[i] & ~st2.cs[i]);
+ }
+ else {
+ TTree *tree = newtree(L, 2 + s1 + s2);
+ tree->tag = TSeq; /* sequence of... */
+ tree->u.ps = 2 + s2;
+ sib1(tree)->tag = TNot; /* ...not... */
+ memcpy(sib1(sib1(tree)), t2, s2 * sizeof(TTree)); /* ...t2 */
+ memcpy(sib2(tree), t1, s1 * sizeof(TTree)); /* ... and t1 */
+ joinktables(L, 1, sib1(tree), 2);
+ }
+ return 1;
+}
+
+
+static int lp_set (lua_State *L) {
+ size_t l;
+ const char *s = luaL_checklstring(L, 1, &l);
+ TTree *tree = newcharset(L);
+ while (l--) {
+ setchar(treebuffer(tree), (byte)(*s));
+ s++;
+ }
+ return 1;
+}
+
+
+static int lp_range (lua_State *L) {
+ int arg;
+ int top = lua_gettop(L);
+ TTree *tree = newcharset(L);
+ for (arg = 1; arg <= top; arg++) {
+ int c;
+ size_t l;
+ const char *r = luaL_checklstring(L, arg, &l);
+ luaL_argcheck(L, l == 2, arg, "range must have two characters");
+ for (c = (byte)r[0]; c <= (byte)r[1]; c++)
+ setchar(treebuffer(tree), c);
+ }
+ return 1;
+}
+
+
+/*
+** Look-behind predicate
+*/
+static int lp_behind (lua_State *L) {
+ TTree *tree;
+ TTree *tree1 = getpatt(L, 1, NULL);
+ int n = fixedlen(tree1);
+ luaL_argcheck(L, n >= 0, 1, "pattern may not have fixed length");
+ luaL_argcheck(L, !hascaptures(tree1), 1, "pattern have captures");
+ luaL_argcheck(L, n <= MAXBEHIND, 1, "pattern too long to look behind");
+ tree = newroot1sib(L, TBehind);
+ tree->u.n = n;
+ return 1;
+}
+
+
+/*
+** Create a non-terminal
+*/
+static int lp_V (lua_State *L) {
+ TTree *tree = newleaf(L, TOpenCall);
+ luaL_argcheck(L, !lua_isnoneornil(L, 1), 1, "non-nil value expected");
+ tree->key = addtonewktable(L, 0, 1);
+ return 1;
+}
+
+
+/*
+** Create a tree for a non-empty capture, with a body and
+** optionally with an associated Lua value (at index 'labelidx' in the
+** stack)
+*/
+static int capture_aux (lua_State *L, int cap, int labelidx) {
+ TTree *tree = newroot1sib(L, TCapture);
+ tree->cap = cap;
+ tree->key = (labelidx == 0) ? 0 : addtonewktable(L, 1, labelidx);
+ return 1;
+}
+
+
+/*
+** Fill a tree with an empty capture, using an empty (TTrue) sibling.
+*/
+static TTree *auxemptycap (TTree *tree, int cap) {
+ tree->tag = TCapture;
+ tree->cap = cap;
+ sib1(tree)->tag = TTrue;
+ return tree;
+}
+
+
+/*
+** Create a tree for an empty capture
+*/
+static TTree *newemptycap (lua_State *L, int cap) {
+ return auxemptycap(newtree(L, 2), cap);
+}
+
+
+/*
+** Create a tree for an empty capture with an associated Lua value
+*/
+static TTree *newemptycapkey (lua_State *L, int cap, int idx) {
+ TTree *tree = auxemptycap(newtree(L, 2), cap);
+ tree->key = addtonewktable(L, 0, idx);
+ return tree;
+}
+
+
+/*
+** Captures with syntax p / v
+** (function capture, query capture, string capture, or number capture)
+*/
+static int lp_divcapture (lua_State *L) {
+ switch (lua_type(L, 2)) {
+ case LUA_TFUNCTION: return capture_aux(L, Cfunction, 2);
+ case LUA_TTABLE: return capture_aux(L, Cquery, 2);
+ case LUA_TSTRING: return capture_aux(L, Cstring, 2);
+ case LUA_TNUMBER: {
+ int n = lua_tointeger(L, 2);
+ TTree *tree = newroot1sib(L, TCapture);
+ luaL_argcheck(L, 0 <= n && n <= SHRT_MAX, 1, "invalid number");
+ tree->cap = Cnum;
+ tree->key = n;
+ return 1;
+ }
+ default: return luaL_argerror(L, 2, "invalid replacement value");
+ }
+}
+
+
+static int lp_substcapture (lua_State *L) {
+ return capture_aux(L, Csubst, 0);
+}
+
+
+static int lp_tablecapture (lua_State *L) {
+ return capture_aux(L, Ctable, 0);
+}
+
+
+static int lp_groupcapture (lua_State *L) {
+ if (lua_isnoneornil(L, 2))
+ return capture_aux(L, Cgroup, 0);
+ else
+ return capture_aux(L, Cgroup, 2);
+}
+
+
+static int lp_foldcapture (lua_State *L) {
+ luaL_checktype(L, 2, LUA_TFUNCTION);
+ return capture_aux(L, Cfold, 2);
+}
+
+
+static int lp_simplecapture (lua_State *L) {
+ return capture_aux(L, Csimple, 0);
+}
+
+
+static int lp_poscapture (lua_State *L) {
+ newemptycap(L, Cposition);
+ return 1;
+}
+
+
+static int lp_argcapture (lua_State *L) {
+ int n = (int)luaL_checkinteger(L, 1);
+ TTree *tree = newemptycap(L, Carg);
+ tree->key = n;
+ luaL_argcheck(L, 0 < n && n <= SHRT_MAX, 1, "invalid argument index");
+ return 1;
+}
+
+
+static int lp_backref (lua_State *L) {
+ luaL_checkany(L, 1);
+ newemptycapkey(L, Cbackref, 1);
+ return 1;
+}
+
+
+/*
+** Constant capture
+*/
+static int lp_constcapture (lua_State *L) {
+ int i;
+ int n = lua_gettop(L); /* number of values */
+ if (n == 0) /* no values? */
+ newleaf(L, TTrue); /* no capture */
+ else if (n == 1)
+ newemptycapkey(L, Cconst, 1); /* single constant capture */
+ else { /* create a group capture with all values */
+ TTree *tree = newtree(L, 1 + 3 * (n - 1) + 2);
+ newktable(L, n); /* create a 'ktable' for new tree */
+ tree->tag = TCapture;
+ tree->cap = Cgroup;
+ tree->key = 0;
+ tree = sib1(tree);
+ for (i = 1; i <= n - 1; i++) {
+ tree->tag = TSeq;
+ tree->u.ps = 3; /* skip TCapture and its sibling */
+ auxemptycap(sib1(tree), Cconst);
+ sib1(tree)->key = addtoktable(L, i);
+ tree = sib2(tree);
+ }
+ auxemptycap(tree, Cconst);
+ tree->key = addtoktable(L, i);
+ }
+ return 1;
+}
+
+
+static int lp_matchtime (lua_State *L) {
+ TTree *tree;
+ luaL_checktype(L, 2, LUA_TFUNCTION);
+ tree = newroot1sib(L, TRunTime);
+ tree->key = addtonewktable(L, 1, 2);
+ return 1;
+}
+
+/* }====================================================== */
+
+
+/*
+** {======================================================
+** Grammar - Tree generation
+** =======================================================
+*/
+
+/*
+** push on the stack the index and the pattern for the
+** initial rule of grammar at index 'arg' in the stack;
+** also add that index into position table.
+*/
+static void getfirstrule (lua_State *L, int arg, int postab) {
+ lua_rawgeti(L, arg, 1); /* access first element */
+ if (lua_isstring(L, -1)) { /* is it the name of initial rule? */
+ lua_pushvalue(L, -1); /* duplicate it to use as key */
+ lua_gettable(L, arg); /* get associated rule */
+ }
+ else {
+ lua_pushinteger(L, 1); /* key for initial rule */
+ lua_insert(L, -2); /* put it before rule */
+ }
+ if (!testpattern(L, -1)) { /* initial rule not a pattern? */
+ if (lua_isnil(L, -1))
+ luaL_error(L, "grammar has no initial rule");
+ else
+ luaL_error(L, "initial rule '%s' is not a pattern", lua_tostring(L, -2));
+ }
+ lua_pushvalue(L, -2); /* push key */
+ lua_pushinteger(L, 1); /* push rule position (after TGrammar) */
+ lua_settable(L, postab); /* insert pair at position table */
+}
+
+/*
+** traverse grammar at index 'arg', pushing all its keys and patterns
+** into the stack. Create a new table (before all pairs key-pattern) to
+** collect all keys and their associated positions in the final tree
+** (the "position table").
+** Return the number of rules and (in 'totalsize') the total size
+** for the new tree.
+*/
+static int collectrules (lua_State *L, int arg, int *totalsize) {
+ int n = 1; /* to count number of rules */
+ int postab = lua_gettop(L) + 1; /* index of position table */
+ int size; /* accumulator for total size */
+ lua_newtable(L); /* create position table */
+ getfirstrule(L, arg, postab);
+ size = 2 + getsize(L, postab + 2); /* TGrammar + TRule + rule */
+ lua_pushnil(L); /* prepare to traverse grammar table */
+ while (lua_next(L, arg) != 0) {
+ if (lua_tonumber(L, -2) == 1 ||
+ lp_equal(L, -2, postab + 1)) { /* initial rule? */
+ lua_pop(L, 1); /* remove value (keep key for lua_next) */
+ continue;
+ }
+ if (!testpattern(L, -1)) /* value is not a pattern? */
+ luaL_error(L, "rule '%s' is not a pattern", val2str(L, -2));
+ luaL_checkstack(L, LUA_MINSTACK, "grammar has too many rules");
+ lua_pushvalue(L, -2); /* push key (to insert into position table) */
+ lua_pushinteger(L, size);
+ lua_settable(L, postab);
+ size += 1 + getsize(L, -1); /* update size */
+ lua_pushvalue(L, -2); /* push key (for next lua_next) */
+ n++;
+ }
+ *totalsize = size + 1; /* TTrue to finish list of rules */
+ return n;
+}
+
+
+static void buildgrammar (lua_State *L, TTree *grammar, int frule, int n) {
+ int i;
+ TTree *nd = sib1(grammar); /* auxiliary pointer to traverse the tree */
+ for (i = 0; i < n; i++) { /* add each rule into new tree */
+ int ridx = frule + 2*i + 1; /* index of i-th rule */
+ int rulesize;
+ TTree *rn = gettree(L, ridx, &rulesize);
+ nd->tag = TRule;
+ nd->key = 0;
+ nd->cap = i; /* rule number */
+ nd->u.ps = rulesize + 1; /* point to next rule */
+ memcpy(sib1(nd), rn, rulesize * sizeof(TTree)); /* copy rule */
+ mergektable(L, ridx, sib1(nd)); /* merge its ktable into new one */
+ nd = sib2(nd); /* move to next rule */
+ }
+ nd->tag = TTrue; /* finish list of rules */
+}
+
+
+/*
+** Check whether a tree has potential infinite loops
+*/
+static int checkloops (TTree *tree) {
+ tailcall:
+ if (tree->tag == TRep && nullable(sib1(tree)))
+ return 1;
+ else if (tree->tag == TGrammar)
+ return 0; /* sub-grammars already checked */
+ else {
+ switch (numsiblings[tree->tag]) {
+ case 1: /* return checkloops(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case 2:
+ if (checkloops(sib1(tree))) return 1;
+ /* else return checkloops(sib2(tree)); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(numsiblings[tree->tag] == 0); return 0;
+ }
+ }
+}
+
+
+static int verifyerror (lua_State *L, int *passed, int npassed) {
+ int i, j;
+ for (i = npassed - 1; i >= 0; i--) { /* search for a repetition */
+ for (j = i - 1; j >= 0; j--) {
+ if (passed[i] == passed[j]) {
+ lua_rawgeti(L, -1, passed[i]); /* get rule's key */
+ return luaL_error(L, "rule '%s' may be left recursive", val2str(L, -1));
+ }
+ }
+ }
+ return luaL_error(L, "too many left calls in grammar");
+}
+
+
+/*
+** Check whether a rule can be left recursive; raise an error in that
+** case; otherwise return 1 iff pattern is nullable.
+** The return value is used to check sequences, where the second pattern
+** is only relevant if the first is nullable.
+** Parameter 'nb' works as an accumulator, to allow tail calls in
+** choices. ('nb' true makes function returns true.)
+** Assume ktable at the top of the stack.
+*/
+static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
+ int nb) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ case TFalse:
+ return nb; /* cannot pass from here */
+ case TTrue:
+ case TBehind: /* look-behind cannot have calls */
+ return 1;
+ case TNot: case TAnd: case TRep:
+ /* return verifyrule(L, sib1(tree), passed, npassed, 1); */
+ tree = sib1(tree); nb = 1; goto tailcall;
+ case TCapture: case TRunTime:
+ /* return verifyrule(L, sib1(tree), passed, npassed, nb); */
+ tree = sib1(tree); goto tailcall;
+ case TCall:
+ /* return verifyrule(L, sib2(tree), passed, npassed, nb); */
+ tree = sib2(tree); goto tailcall;
+ case TSeq: /* only check 2nd child if first is nb */
+ if (!verifyrule(L, sib1(tree), passed, npassed, 0))
+ return nb;
+ /* else return verifyrule(L, sib2(tree), passed, npassed, nb); */
+ tree = sib2(tree); goto tailcall;
+ case TChoice: /* must check both children */
+ nb = verifyrule(L, sib1(tree), passed, npassed, nb);
+ /* return verifyrule(L, sib2(tree), passed, npassed, nb); */
+ tree = sib2(tree); goto tailcall;
+ case TRule:
+ if (npassed >= MAXRULES)
+ return verifyerror(L, passed, npassed);
+ else {
+ passed[npassed++] = tree->key;
+ /* return verifyrule(L, sib1(tree), passed, npassed); */
+ tree = sib1(tree); goto tailcall;
+ }
+ case TGrammar:
+ return nullable(tree); /* sub-grammar cannot be left recursive */
+ default: assert(0); return 0;
+ }
+}
+
+
+static void verifygrammar (lua_State *L, TTree *grammar) {
+ int passed[MAXRULES];
+ TTree *rule;
+ /* check left-recursive rules */
+ for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
+ if (rule->key == 0) continue; /* unused rule */
+ verifyrule(L, sib1(rule), passed, 0, 0);
+ }
+ assert(rule->tag == TTrue);
+ /* check infinite loops inside rules */
+ for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
+ if (rule->key == 0) continue; /* unused rule */
+ if (checkloops(sib1(rule))) {
+ lua_rawgeti(L, -1, rule->key); /* get rule's key */
+ luaL_error(L, "empty loop in rule '%s'", val2str(L, -1));
+ }
+ }
+ assert(rule->tag == TTrue);
+}
+
+
+/*
+** Give a name for the initial rule if it is not referenced
+*/
+static void initialrulename (lua_State *L, TTree *grammar, int frule) {
+ if (sib1(grammar)->key == 0) { /* initial rule is not referenced? */
+ int n = lua_rawlen(L, -1) + 1; /* index for name */
+ lua_pushvalue(L, frule); /* rule's name */
+ lua_rawseti(L, -2, n); /* ktable was on the top of the stack */
+ sib1(grammar)->key = n;
+ }
+}
+
+
+static TTree *newgrammar (lua_State *L, int arg) {
+ int treesize;
+ int frule = lua_gettop(L) + 2; /* position of first rule's key */
+ int n = collectrules(L, arg, &treesize);
+ TTree *g = newtree(L, treesize);
+ luaL_argcheck(L, n <= MAXRULES, arg, "grammar has too many rules");
+ g->tag = TGrammar; g->u.n = n;
+ lua_newtable(L); /* create 'ktable' */
+ lua_setuservalue(L, -2);
+ buildgrammar(L, g, frule, n);
+ lua_getuservalue(L, -1); /* get 'ktable' for new tree */
+ finalfix(L, frule - 1, g, sib1(g));
+ initialrulename(L, g, frule);
+ verifygrammar(L, g);
+ lua_pop(L, 1); /* remove 'ktable' */
+ lua_insert(L, -(n * 2 + 2)); /* move new table to proper position */
+ lua_pop(L, n * 2 + 1); /* remove position table + rule pairs */
+ return g; /* new table at the top of the stack */
+}
+
+/* }====================================================== */
+
+
+static Instruction *prepcompile (lua_State *L, Pattern *p, int idx) {
+ lua_getuservalue(L, idx); /* push 'ktable' (may be used by 'finalfix') */
+ finalfix(L, 0, NULL, p->tree);
+ lua_pop(L, 1); /* remove 'ktable' */
+ return compile(L, p);
+}
+
+
+static int lp_printtree (lua_State *L) {
+ TTree *tree = getpatt(L, 1, NULL);
+ int c = lua_toboolean(L, 2);
+ if (c) {
+ lua_getuservalue(L, 1); /* push 'ktable' (may be used by 'finalfix') */
+ finalfix(L, 0, NULL, tree);
+ lua_pop(L, 1); /* remove 'ktable' */
+ }
+ printktable(L, 1);
+ printtree(tree, 0);
+ return 0;
+}
+
+
+static int lp_printcode (lua_State *L) {
+ Pattern *p = getpattern(L, 1);
+ printktable(L, 1);
+ if (p->code == NULL) /* not compiled yet? */
+ prepcompile(L, p, 1);
+ printpatt(p->code, p->codesize);
+ return 0;
+}
+
+
+/*
+** Get the initial position for the match, interpreting negative
+** values from the end of the subject
+*/
+static size_t initposition (lua_State *L, size_t len) {
+ lua_Integer ii = luaL_optinteger(L, 3, 1);
+ if (ii > 0) { /* positive index? */
+ if ((size_t)ii <= len) /* inside the string? */
+ return (size_t)ii - 1; /* return it (corrected to 0-base) */
+ else return len; /* crop at the end */
+ }
+ else { /* negative index */
+ if ((size_t)(-ii) <= len) /* inside the string? */
+ return len - ((size_t)(-ii)); /* return position from the end */
+ else return 0; /* crop at the beginning */
+ }
+}
+
+
+/*
+** Main match function
+*/
+static int lp_match (lua_State *L) {
+ Capture capture[INITCAPSIZE];
+ const char *r;
+ size_t l;
+ const char *s;
+
+ Pattern *p = (getpatt(L, 1, NULL), getpattern(L, 1));
+ Instruction *code = (p->code != NULL) ? p->code : prepcompile(L, p, 1);
+
+ if (lua_type (L, SUBJIDX) == LUA_TSTRING) {
+ s = luaL_checklstring (L, SUBJIDX, &l);
+ }
+ else if (lua_type (L, SUBJIDX) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text (L, SUBJIDX);
+ if (!t) {
+ return luaL_error (L, "invalid argument (not a text)");
+ }
+ s = t->start;
+ l = t->len;
+
+ if (s == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+ }
+ else {
+ return luaL_error (L, "invalid argument: %s",
+ lua_typename (L, lua_type (L, SUBJIDX)));
+ }
+ size_t i = initposition(L, l);
+ int ptop = lua_gettop(L), rs;
+ lua_pushnil(L); /* initialize subscache */
+ lua_pushlightuserdata(L, capture); /* initialize caplistidx */
+ lua_getuservalue(L, 1); /* initialize penvidx */
+ r = match(L, s, s + i, s + l, code, capture, ptop);
+ if (r == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+ rs = getcaptures(L, s, r, ptop);
+ return rs;
+}
+
+
+
+/*
+** {======================================================
+** Library creation and functions not related to matching
+** =======================================================
+*/
+
+/* maximum limit for stack size */
+#define MAXLIM (INT_MAX / 100)
+
+static int lp_setmax (lua_State *L) {
+ lua_Integer lim = luaL_checkinteger(L, 1);
+ luaL_argcheck(L, 0 < lim && lim <= MAXLIM, 1, "out of range");
+ lua_settop(L, 1);
+ lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
+ return 0;
+}
+
+
+static int lp_version (lua_State *L) {
+ lua_pushstring(L, VERSION);
+ return 1;
+}
+
+
+static int lp_type (lua_State *L) {
+ if (testpattern(L, 1))
+ lua_pushliteral(L, "pattern");
+ else
+ lua_pushnil(L);
+ return 1;
+}
+
+
+int lp_gc (lua_State *L) {
+ Pattern *p = getpattern(L, 1);
+ realloccode(L, p, 0); /* delete code block */
+ return 0;
+}
+
+
+static void createcat (lua_State *L, const char *catname, int (catf) (int)) {
+ TTree *t = newcharset(L);
+ int i;
+ for (i = 0; i <= UCHAR_MAX; i++)
+ if (catf(i)) setchar(treebuffer(t), i);
+ lua_setfield(L, -2, catname);
+}
+
+
+static int lp_locale (lua_State *L) {
+ if (lua_isnoneornil(L, 1)) {
+ lua_settop(L, 0);
+ lua_createtable(L, 0, 12);
+ }
+ else {
+ luaL_checktype(L, 1, LUA_TTABLE);
+ lua_settop(L, 1);
+ }
+ createcat(L, "alnum", isalnum);
+ createcat(L, "alpha", isalpha);
+ createcat(L, "cntrl", iscntrl);
+ createcat(L, "digit", isdigit);
+ createcat(L, "graph", isgraph);
+ createcat(L, "lower", islower);
+ createcat(L, "print", isprint);
+ createcat(L, "punct", ispunct);
+ createcat(L, "space", isspace);
+ createcat(L, "upper", isupper);
+ createcat(L, "xdigit", isxdigit);
+ return 1;
+}
+
+
+static struct luaL_Reg pattreg[] = {
+ {"ptree", lp_printtree},
+ {"pcode", lp_printcode},
+ {"match", lp_match},
+ {"B", lp_behind},
+ {"V", lp_V},
+ {"C", lp_simplecapture},
+ {"Cc", lp_constcapture},
+ {"Cmt", lp_matchtime},
+ {"Cb", lp_backref},
+ {"Carg", lp_argcapture},
+ {"Cp", lp_poscapture},
+ {"Cs", lp_substcapture},
+ {"Ct", lp_tablecapture},
+ {"Cf", lp_foldcapture},
+ {"Cg", lp_groupcapture},
+ {"P", lp_P},
+ {"S", lp_set},
+ {"R", lp_range},
+ {"locale", lp_locale},
+ {"version", lp_version},
+ {"setmaxstack", lp_setmax},
+ {"type", lp_type},
+ {NULL, NULL}
+};
+
+
+static struct luaL_Reg metareg[] = {
+ {"__mul", lp_seq},
+ {"__add", lp_choice},
+ {"__pow", lp_star},
+ {"__gc", lp_gc},
+ {"__len", lp_and},
+ {"__div", lp_divcapture},
+ {"__unm", lp_not},
+ {"__sub", lp_sub},
+ {NULL, NULL}
+};
+
+int luaopen_lpeg (lua_State *L) {
+ luaL_newmetatable(L, PATTERN_T);
+ lua_pushnumber(L, MAXBACK); /* initialize maximum backtracking */
+ lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
+ luaL_setfuncs(L, metareg, 0);
+ luaL_newlib(L, pattreg);
+ lua_pushvalue(L, -1);
+ lua_setfield(L, -3, "__index");
+ return 1;
+}
+
+/* }====================================================== */
diff --git a/contrib/lua-lpeg/lptree.h b/contrib/lua-lpeg/lptree.h
new file mode 100644
index 0000000..38a668e
--- /dev/null
+++ b/contrib/lua-lpeg/lptree.h
@@ -0,0 +1,77 @@
+/*
+** $Id: lptree.h,v 1.2 2013/03/24 13:51:12 roberto Exp $
+*/
+
+#if !defined(lptree_h)
+#define lptree_h
+
+
+#include "lptypes.h"
+
+
+/*
+** types of trees
+*/
+typedef enum TTag {
+ TChar = 0, TSet, TAny, /* standard PEG elements */
+ TTrue, TFalse,
+ TRep,
+ TSeq, TChoice,
+ TNot, TAnd,
+ TCall,
+ TOpenCall,
+ TRule, /* sib1 is rule's pattern, sib2 is 'next' rule */
+ TGrammar, /* sib1 is initial (and first) rule */
+ TBehind, /* match behind */
+ TCapture, /* regular capture */
+ TRunTime /* run-time capture */
+} TTag;
+
+/* number of siblings for each tree */
+extern const byte numsiblings[];
+
+
+/*
+** Tree trees
+** The first sibling of a tree (if there is one) is immediately after
+** the tree. A reference to a second sibling (ps) is its position
+** relative to the position of the tree itself. A key in ktable
+** uses the (unique) address of the original tree that created that
+** entry. NULL means no data.
+*/
+typedef struct TTree {
+ byte tag;
+ byte cap; /* kind of capture (if it is a capture) */
+ unsigned short key; /* key in ktable for Lua data (0 if no key) */
+ union {
+ int ps; /* occasional second sibling */
+ int n; /* occasional counter */
+ } u;
+} TTree;
+
+
+/*
+** A complete pattern has its tree plus, if already compiled,
+** its corresponding code
+*/
+typedef struct Pattern {
+ union Instruction *code;
+ int codesize;
+ TTree tree[1];
+} Pattern;
+
+
+/* number of siblings for each tree */
+extern const byte numsiblings[];
+
+/* access to siblings */
+#define sib1(t) ((t) + 1)
+#define sib2(t) ((t) + (t)->u.ps)
+
+
+int luaopen_lpeg (lua_State *L);
+
+
+
+#endif
+
diff --git a/contrib/lua-lpeg/lptypes.h b/contrib/lua-lpeg/lptypes.h
new file mode 100644
index 0000000..f541c7a
--- /dev/null
+++ b/contrib/lua-lpeg/lptypes.h
@@ -0,0 +1,154 @@
+/*
+** $Id: lptypes.h,v 1.14 2015/09/28 17:17:41 roberto Exp $
+** LPeg - PEG pattern matching for Lua
+** Copyright 2007-2015, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+** written by Roberto Ierusalimschy
+*/
+
+#if !defined(lptypes_h)
+#define lptypes_h
+
+
+#if !defined(LPEG_DEBUG) && !defined(NDEBUG)
+#define NDEBUG
+#endif
+
+#include <assert.h>
+#include <limits.h>
+
+#include "lua.h"
+
+
+#define VERSION "1.0.0"
+
+
+#define PATTERN_T "lpeg-pattern"
+#define MAXSTACKIDX "lpeg-maxstack"
+
+
+/*
+** compatibility with Lua 5.1
+*/
+#if (LUA_VERSION_NUM == 501)
+
+#define lp_equal lua_equal
+
+#define lua_getuservalue lua_getfenv
+#define lua_setuservalue lua_setfenv
+
+#ifndef lua_rawlen
+#define lua_rawlen lua_objlen
+#endif
+
+#ifndef luaL_setfuncs
+#define luaL_setfuncs(L,f,n) luaL_register(L,NULL,f)
+#endif
+#ifndef luaL_newlib
+#define luaL_newlib(L,f) luaL_register(L,"lpeg",f)
+#endif
+#endif
+
+
+#if !defined(lp_equal)
+#define lp_equal(L,idx1,idx2) lua_compare(L,(idx1),(idx2),LUA_OPEQ)
+#endif
+
+
+/* default maximum size for call/backtrack stack */
+#if !defined(MAXBACK)
+#define MAXBACK 400
+#endif
+
+
+/* maximum number of rules in a grammar */
+#if !defined(MAXRULES)
+#define MAXRULES 1000
+#endif
+
+
+
+/* initial size for capture's list */
+#define INITCAPSIZE 32
+
+
+/* index, on Lua stack, for subject */
+#define SUBJIDX 2
+
+/* number of fixed arguments to 'match' (before capture arguments) */
+#define FIXEDARGS 3
+
+/* index, on Lua stack, for capture list */
+#define caplistidx(ptop) ((ptop) + 2)
+
+/* index, on Lua stack, for pattern's ktable */
+#define ktableidx(ptop) ((ptop) + 3)
+
+/* index, on Lua stack, for backtracking stack */
+#define stackidx(ptop) ((ptop) + 4)
+
+
+
+typedef unsigned char byte;
+
+
+#define BITSPERCHAR 8
+
+#define CHARSETSIZE ((UCHAR_MAX/BITSPERCHAR) + 1)
+
+
+
+typedef struct Charset {
+ byte cs[CHARSETSIZE];
+} Charset;
+
+
+
+#define loopset(v,b) { int v; for (v = 0; v < CHARSETSIZE; v++) {b;} }
+
+/* access to charset */
+#define treebuffer(t) ((byte *)((t) + 1))
+
+/* number of slots needed for 'n' bytes */
+#define bytes2slots(n) (((n) - 1) / sizeof(TTree) + 1)
+
+/* set 'b' bit in charset 'cs' */
+#define setchar(cs,b) ((cs)[(b) >> 3] |= (1 << ((b) & 7)))
+
+
+/*
+** in capture instructions, 'kind' of capture and its offset are
+** packed in field 'aux', 4 bits for each
+*/
+#define getkind(op) ((op)->i.aux & 0xF)
+#define getoff(op) (((op)->i.aux >> 4) & 0xF)
+#define joinkindoff(k,o) ((k) | ((o) << 4))
+
+#define MAXOFF 0xF
+#define MAXAUX 0xFF
+
+
+/* maximum number of bytes to look behind */
+#define MAXBEHIND MAXAUX
+
+
+/* maximum size (in elements) for a pattern */
+#define MAXPATTSIZE (SHRT_MAX - 10)
+
+
+/* size (in elements) for an instruction plus extra l bytes */
+#define instsize(l) (((l) + sizeof(Instruction) - 1)/sizeof(Instruction) + 1)
+
+
+/* size (in elements) for a ISet instruction */
+#define CHARSETINSTSIZE instsize(CHARSETSIZE)
+
+/* size (in elements) for a IFunc instruction */
+#define funcinstsize(p) ((p)->i.aux + 2)
+
+
+
+#define testchar(st,c) (((int)(st)[((c) >> 3)] & (1 << ((c) & 7))))
+
+
+#endif
+
diff --git a/contrib/lua-lpeg/lpvm.c b/contrib/lua-lpeg/lpvm.c
new file mode 100644
index 0000000..0dfca18
--- /dev/null
+++ b/contrib/lua-lpeg/lpvm.c
@@ -0,0 +1,383 @@
+/*
+** $Id: lpvm.c,v 1.6 2015/09/28 17:01:25 roberto Exp $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include "config.h"
+
+#include <limits.h>
+#include <string.h>
+
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lpcap.h"
+#include "lptypes.h"
+#include "lpvm.h"
+#include "lpprint.h"
+
+/* initial size for call/backtrack stack */
+#if !defined(INITBACK)
+#define INITBACK MAXBACK
+#endif
+
+
+#define getoffset(p) (((p) + 1)->offset)
+
+static const Instruction giveup = {{IGiveup, 0, 0}};
+
+
+/*
+** {======================================================
+** Virtual Machine
+** =======================================================
+*/
+
+
+typedef struct Stack {
+ const char *s; /* saved position (or NULL for calls) */
+ const Instruction *p; /* next instruction */
+ int caplevel;
+} Stack;
+
+
+#define getstackbase(L, ptop) ((Stack *)lua_touserdata(L, stackidx(ptop)))
+
+
+/*
+** Ensures the size of array 'capture' (with size '*capsize' and
+** 'captop' elements being used) is enough to accomodate 'n' extra
+** elements plus one. (Because several opcodes add stuff to the capture
+** array, it is simpler to ensure the array always has at least one free
+** slot upfront and check its size later.)
+*/
+static Capture *growcap (lua_State *L, Capture *capture, int *capsize,
+ int captop, int n, int ptop) {
+ if (*capsize - captop > n)
+ return capture; /* no need to grow array */
+ else { /* must grow */
+ Capture *newc;
+ int newsize = captop + n + 1; /* minimum size needed */
+ if (newsize < INT_MAX/((int)sizeof(Capture) * 2))
+ newsize *= 2; /* twice that size, if not too big */
+ else if (newsize >= INT_MAX/((int)sizeof(Capture)))
+ luaL_error(L, "too many captures");
+ newc = (Capture *)lua_newuserdata(L, newsize * sizeof(Capture));
+ memcpy(newc, capture, captop * sizeof(Capture));
+ *capsize = newsize;
+ lua_replace(L, caplistidx(ptop));
+ return newc;
+ }
+}
+
+
+/*
+** Double the size of the stack
+*/
+static Stack *doublestack (lua_State *L, Stack **stacklimit, int ptop) {
+ Stack *stack = getstackbase(L, ptop);
+ Stack *newstack;
+ int n = *stacklimit - stack; /* current stack size */
+ int max, newn;
+ lua_getfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
+ max = lua_tointeger(L, -1); /* maximum allowed size */
+ lua_pop(L, 1);
+ if (n >= max) /* already at maximum size? */
+ luaL_error(L, "backtrack stack overflow (current limit is %d)", max);
+ newn = 2 * n; /* new size */
+ if (newn > max) newn = max;
+ newstack = (Stack *)lua_newuserdata(L, newn * sizeof(Stack));
+ memcpy(newstack, stack, n * sizeof(Stack));
+ lua_replace(L, stackidx(ptop));
+ *stacklimit = newstack + newn;
+ return newstack + n; /* return next position */
+}
+
+
+/*
+** Interpret the result of a dynamic capture: false -> fail;
+** true -> keep current position; number -> next position.
+** Return new subject position. 'fr' is stack index where
+** is the result; 'curr' is current subject position; 'limit'
+** is subject's size.
+*/
+static int resdyncaptures (lua_State *L, int fr, int curr, int limit) {
+ lua_Integer res;
+ if (!lua_toboolean(L, fr)) { /* false value? */
+ lua_settop(L, fr - 1); /* remove results */
+ return -1; /* and fail */
+ }
+ else if (lua_isboolean(L, fr)) /* true? */
+ res = curr; /* keep current position */
+ else {
+ res = lua_tointeger(L, fr) - 1; /* new position */
+ if (res < curr || res > limit)
+ luaL_error(L, "invalid position returned by match-time capture");
+ }
+ lua_remove(L, fr); /* remove first result (offset) */
+ return res;
+}
+
+
+/*
+** Add capture values returned by a dynamic capture to the list
+** 'capture', nested inside a group. 'fd' indexes the first capture
+** value, 'n' is the number of values (at least 1). The open group
+** capture is already in 'capture', before the place for the new entries.
+*/
+static void adddyncaptures (const char *s, Capture *capture, int n, int fd) {
+ int i;
+ assert(capture[-1].kind == Cgroup && capture[-1].siz == 0);
+ capture[-1].idx = 0; /* make group capture an anonymous group */
+ for (i = 0; i < n; i++) { /* add runtime captures */
+ capture[i].kind = Cruntime;
+ capture[i].siz = 1; /* mark it as closed */
+ capture[i].idx = fd + i; /* stack index of capture value */
+ capture[i].s = s;
+ }
+ capture[n].kind = Cclose; /* close group */
+ capture[n].siz = 1;
+ capture[n].s = s;
+}
+
+
+/*
+** Remove dynamic captures from the Lua stack (called in case of failure)
+*/
+static int removedyncap (lua_State *L, Capture *capture,
+ int level, int last) {
+ int id = finddyncap(capture + level, capture + last); /* index of 1st cap. */
+ int top = lua_gettop(L);
+ if (id == 0) return 0; /* no dynamic captures? */
+ lua_settop(L, id - 1); /* remove captures */
+ return top - id + 1; /* number of values removed */
+}
+
+
+/*
+** Opcode interpreter
+*/
+const char *match (lua_State *L, const char *o, const char *s, const char *e,
+ Instruction *op, Capture *capture, int ptop) {
+ Stack stackbase[INITBACK];
+ Stack *stacklimit = stackbase + INITBACK;
+ Stack *stack = stackbase; /* point to first empty slot in stack */
+ int capsize = INITCAPSIZE;
+ int captop = 0; /* point to first empty slot in captures */
+ int ndyncap = 0; /* number of dynamic captures (in Lua stack) */
+ const Instruction *p = op; /* current instruction */
+ stack->p = &giveup; stack->s = s; stack->caplevel = 0; stack++;
+ lua_pushlightuserdata(L, stackbase);
+ for (;;) {
+#if defined(DEBUG)
+ printf("s: |%s| stck:%d, dyncaps:%d, caps:%d ",
+ s, (int)(stack - getstackbase(L, ptop)), ndyncap, captop);
+ printinst(op, p);
+#endif
+ assert(stackidx(ptop) + ndyncap == lua_gettop(L) && ndyncap <= captop);
+ switch ((Opcode)p->i.code) {
+ case IEnd: {
+ assert(stack == getstackbase(L, ptop) + 1);
+ capture[captop].kind = Cclose;
+ capture[captop].s = NULL;
+ return s;
+ }
+ case IGiveup: {
+ assert(stack == getstackbase(L, ptop));
+ return NULL;
+ }
+ case IRet: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s == NULL);
+ p = (--stack)->p;
+ continue;
+ }
+ case IAny: {
+ if (s < e) { p++; s++; }
+ else goto fail;
+ continue;
+ }
+ case ITestAny: {
+ if (s < e) p += 2;
+ else p += getoffset(p);
+ continue;
+ }
+ case IChar: {
+ if (s < e && (byte)*s == p->i.aux) { p++; s++; }
+ else goto fail;
+ continue;
+ }
+ case ITestChar: {
+ if (s < e && (byte)*s == p->i.aux) p += 2;
+ else p += getoffset(p);
+ continue;
+ }
+ case ISet: {
+ if (s < e) {
+ int c = (byte) *s;
+ if (testchar((p + 1)->buff, c)) {
+ p += CHARSETINSTSIZE;
+ s++;
+ }
+ else goto fail;
+ }
+ else {
+ goto fail;
+ }
+ continue;
+ }
+ case ITestSet: {
+ if (s < e) {
+ int c = (byte) *s;
+ if (testchar((p + 2)->buff, c))
+ p += 1 + CHARSETINSTSIZE;
+ else p += getoffset(p);
+ }
+ else {
+ p += getoffset(p);
+ }
+ continue;
+ }
+ case IBehind: {
+ int n = p->i.aux;
+ if (n > s - o) goto fail;
+ s -= n; p++;
+ continue;
+ }
+ case ISpan: {
+ for (; s < e; s++) {
+ int c = (byte)*s;
+ if (!testchar((p+1)->buff, c)) break;
+ }
+ p += CHARSETINSTSIZE;
+ continue;
+ }
+ case IJmp: {
+ p += getoffset(p);
+ continue;
+ }
+ case IChoice: {
+ if (stack == stacklimit)
+ stack = doublestack(L, &stacklimit, ptop);
+ stack->p = p + getoffset(p);
+ stack->s = s;
+ stack->caplevel = captop;
+ stack++;
+ p += 2;
+ continue;
+ }
+ case ICall: {
+ if (stack == stacklimit)
+ stack = doublestack(L, &stacklimit, ptop);
+ stack->s = NULL;
+ stack->p = p + 2; /* save return address */
+ stack++;
+ p += getoffset(p);
+ continue;
+ }
+ case ICommit: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
+ stack--;
+ p += getoffset(p);
+ continue;
+ }
+ case IPartialCommit: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
+ (stack - 1)->s = s;
+ (stack - 1)->caplevel = captop;
+ p += getoffset(p);
+ continue;
+ }
+ case IBackCommit: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
+ s = (--stack)->s;
+ captop = stack->caplevel;
+ p += getoffset(p);
+ continue;
+ }
+ case IFailTwice:
+ assert(stack > getstackbase(L, ptop));
+ stack--;
+ /* go through */
+ case IFail:
+ fail: { /* pattern failed: try to backtrack */
+ do { /* remove pending calls */
+ assert(stack > getstackbase(L, ptop));
+ s = (--stack)->s;
+ } while (s == NULL);
+ if (ndyncap > 0) /* is there matchtime captures? */
+ ndyncap -= removedyncap(L, capture, stack->caplevel, captop);
+ captop = stack->caplevel;
+ p = stack->p;
+ continue;
+ }
+ case ICloseRunTime: {
+ CapState cs;
+ int rem, res, n;
+ int fr = lua_gettop(L) + 1; /* stack index of first result */
+ cs.reclevel = 0; cs.L = L;
+ cs.s = o; cs.ocap = capture; cs.ptop = ptop;
+ n = runtimecap(&cs, capture + captop, s, &rem); /* call function */
+ captop -= n; /* remove nested captures */
+ ndyncap -= rem; /* update number of dynamic captures */
+ fr -= rem; /* 'rem' items were popped from Lua stack */
+ res = resdyncaptures(L, fr, s - o, e - o); /* get result */
+ if (res == -1) /* fail? */
+ goto fail;
+ s = o + res; /* else update current position */
+ n = lua_gettop(L) - fr + 1; /* number of new captures */
+ ndyncap += n; /* update number of dynamic captures */
+ if (n == 0) /* no new captures? */
+ captop--; /* remove open group */
+ else { /* new captures; keep original open group */
+ if (fr + n >= SHRT_MAX)
+ luaL_error(L, "too many results in match-time capture");
+ /* add new captures + close group to 'capture' list */
+ capture = growcap(L, capture, &capsize, captop, n + 1, ptop);
+ adddyncaptures(s, capture + captop, n, fr);
+ captop += n + 1; /* new captures + close group */
+ }
+ p++;
+ continue;
+ }
+ case ICloseCapture: {
+ const char *s1 = s;
+ assert(captop > 0);
+ /* if possible, turn capture into a full capture */
+ if (capture[captop - 1].siz == 0 &&
+ s1 - capture[captop - 1].s < UCHAR_MAX) {
+ capture[captop - 1].siz = s1 - capture[captop - 1].s + 1;
+ p++;
+ continue;
+ }
+ else {
+ capture[captop].siz = 1; /* mark entry as closed */
+ capture[captop].s = s;
+ goto pushcapture;
+ }
+ }
+ case IOpenCapture:
+ capture[captop].siz = 0; /* mark entry as open */
+ capture[captop].s = s;
+ goto pushcapture;
+ case IFullCapture:
+ capture[captop].siz = getoff(p) + 1; /* save capture size */
+ capture[captop].s = s - getoff(p);
+ /* goto pushcapture; */
+ pushcapture: {
+ capture[captop].idx = p->i.key;
+ capture[captop].kind = getkind(p);
+ captop++;
+ capture = growcap(L, capture, &capsize, captop, 0, ptop);
+ p++;
+ continue;
+ }
+ default: assert(0); return NULL;
+ }
+ }
+
+}
+
+/* }====================================================== */
+
+
diff --git a/contrib/lua-lpeg/lpvm.h b/contrib/lua-lpeg/lpvm.h
new file mode 100644
index 0000000..757b9e1
--- /dev/null
+++ b/contrib/lua-lpeg/lpvm.h
@@ -0,0 +1,58 @@
+/*
+** $Id: lpvm.h,v 1.3 2014/02/21 13:06:41 roberto Exp $
+*/
+
+#if !defined(lpvm_h)
+#define lpvm_h
+
+#include "lpcap.h"
+
+
+/* Virtual Machine's instructions */
+typedef enum Opcode {
+ IAny, /* if no char, fail */
+ IChar, /* if char != aux, fail */
+ ISet, /* if char not in buff, fail */
+ ITestAny, /* in no char, jump to 'offset' */
+ ITestChar, /* if char != aux, jump to 'offset' */
+ ITestSet, /* if char not in buff, jump to 'offset' */
+ ISpan, /* read a span of chars in buff */
+ IBehind, /* walk back 'aux' characters (fail if not possible) */
+ IRet, /* return from a rule */
+ IEnd, /* end of pattern */
+ IChoice, /* stack a choice; next fail will jump to 'offset' */
+ IJmp, /* jump to 'offset' */
+ ICall, /* call rule at 'offset' */
+ IOpenCall, /* call rule number 'key' (must be closed to a ICall) */
+ ICommit, /* pop choice and jump to 'offset' */
+ IPartialCommit, /* update top choice to current position and jump */
+ IBackCommit, /* "fails" but jump to its own 'offset' */
+ IFailTwice, /* pop one choice and then fail */
+ IFail, /* go back to saved state on choice and jump to saved offset */
+ IGiveup, /* internal use */
+ IFullCapture, /* complete capture of last 'off' chars */
+ IOpenCapture, /* start a capture */
+ ICloseCapture,
+ ICloseRunTime
+} Opcode;
+
+
+
+typedef union Instruction {
+ struct Inst {
+ byte code;
+ byte aux;
+ short key;
+ } i;
+ int offset;
+ byte buff[1];
+} Instruction;
+
+
+void printpatt (Instruction *p, int n);
+const char *match (lua_State *L, const char *o, const char *s, const char *e,
+ Instruction *op, Capture *capture, int ptop);
+
+
+#endif
+
diff --git a/contrib/lua-lupa/LICENSE b/contrib/lua-lupa/LICENSE
new file mode 100644
index 0000000..66c1141
--- /dev/null
+++ b/contrib/lua-lupa/LICENSE
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2015-2018 Mitchell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/contrib/lua-lupa/README.md b/contrib/lua-lupa/README.md
new file mode 100644
index 0000000..edf6dce
--- /dev/null
+++ b/contrib/lua-lupa/README.md
@@ -0,0 +1,179 @@
+# Lupa
+
+## Introduction
+
+Lupa is a [Jinja2][] template engine implementation written in Lua and supports
+Lua syntax within tags and variables.
+
+Lupa was sponsored by the [Library of the University of Antwerp][].
+
+[Jinja2]: http://jinja.pocoo.org
+[Library of the University of Antwerp]: http://www.uantwerpen.be/
+
+## Requirements
+
+Lupa has the following requirements:
+
+* [Lua][] 5.1, 5.2, or 5.3.
+* The [LPeg][] library.
+
+[Lua]: http://www.lua.org
+[LPeg]: http://www.inf.puc-rio.br/~roberto/lpeg/
+
+## Download
+
+Download Lupa from the project’s [download page][].
+
+[download page]: download
+
+## Installation
+
+Unzip Lupa and place the "lupa.lua" file in your Lua installation's
+`package.path`. This location depends on your version of Lua. Typical locations
+are listed below.
+
+* Lua 5.1: */usr/local/share/lua/5.1/* or */usr/local/share/lua/5.1/*
+* Lua 5.2: */usr/local/share/lua/5.2/* or */usr/local/share/lua/5.2/*
+* Lua 5.3: */usr/local/share/lua/5.3/* or */usr/local/share/lua/5.3/*
+
+You can also place the "lupa.lua" file wherever you'd like and add it to Lua's
+`package.path` manually in your program. For example, if Lupa was placed in a
+*/home/user/lua/* directory, it can be used as follows:
+
+ package.path = package.path..';/home/user/lua/?.lua'
+
+## Usage
+
+Lupa is simply a Lua library. Its `lupa.expand()` and `lupa.expand_file()`
+functions may called to process templates. For example:
+
+ lupa = require('lupa')
+ lupa.expand("hello {{ s }}!", {s = "world"}) --> "hello world!"
+ lupa.expand("{% for i in {1, 2, 3} %}{{ i }}{% endfor %}") --> 123
+
+By default, Lupa loads templates relative to the current working directory. This
+can be changed by reconfiguring Lupa:
+
+ lupa.expand_file('name') --> expands template "./name"
+ lupa.configure{loader = lupa.loaders.filesystem('path/to/templates')}
+ lupa.expand_file('name') --> expands template "path/to/templates/name"
+
+See Lupa's [API documentation][] for more information.
+
+[API documentation]: api.html
+
+## Syntax
+
+Please refer to Jinja2's extensive [template documentation][]. Any
+incompatibilities are listed in the sections below.
+
+[template documentation]: http://jinja.pocoo.org/docs/dev/templates/
+
+## Comparison with Jinja2
+
+While Lua and Python (Jinja2's implementation language) share some similarities,
+the languages themselves are fundamentally different. Nevertheless, a
+significant effort was made to support a vast majority of Jinja2's Python-style
+syntax. As a result, Lupa passes Jinja2's test suite with only a handful of
+modifications. The comprehensive list of differences between Lupa and Jinja2 is
+described in the following sections.
+
+### Fundamental Differences
+
+* Expressions use Lua's syntax instead of Python's, so many of Python's
+ syntactic constructs are not valid. However, the following constructs
+ *are valid*, despite being invalid in pure Lua:
+
+ + Iterating over table literals or table variables directly in a "for" loop:
+
+ {% for i in {1, 2, 3} %}...{% endfor %}
+
+ + Conditional loops via an "if" expression suffix:
+
+ {% for x in range(10) if is_odd(x) %}...{% endfor %}
+
+ + Table unpacking for list elements when iterating through a list of lists:
+
+ {% for a, b, c in {{1, 2, 3}, {4, 5, 6}} %}...{% endfor %}
+
+ + Default values for macro arguments:
+
+ {% macro m(a, b, c='c', d='d') %}...{% endmacro %}
+
+* Strings do not have unicode escapes nor is unicode interpreted in any way.
+
+### Syntactic Differences
+
+* Line statements are not supported due to parsing complexity.
+* In `{% for ... %}` loops, the `loop.length`, `loop.revindex`,
+ `loop.revindex0`, and `loop.last` variables only apply to sequences, where
+ Lua's `'#'` operator applies.
+* The `{% continue %}` and `{% break %}` loop controls are not supported due to
+ complexity.
+* Loops may be used recursively by default, so the `recursive` loop modifier is
+ not supported.
+* The `is` operator is not supported by Lua, so tests of the form `{{ x is y }}`
+ should be written `{{ is_y(x) }}` (e.g. `{{ is_number(42) }}`).
+* Filters cannot occur after tokens within an expression (e.g.
+ `{{ "foo"|upper .. "bar"|upper }}`), but can only occur at the end of an
+ expression (e.g. `{{ "foo".."bar"|upper }}`).
+* Blocks always have access to scoped variables, so the `scoped` block modifier
+ is not supported.
+* Named block end tags are not supported since the parser cannot easily keep
+ track of that state information.
+* Any `{% block ... %}` tags within a "false" block (e.g. `{% if a %}` where `a`
+ evaluates to `false`) are never read and stored due to the parser
+ implementation.
+* Inline "if" expressions (e.g. `{% extends b if a else c %}`) are not
+ supported. Instead, use a Lua conditional expression
+ (e.g. `{% extends a and b or c %}`).
+* Any `{% extends ... %}` tags within a sub-scope are not effective outside that
+ scope (e.g. `{% if a %}{% extends a %}{% else %}{% extends b %}{% endif %}`).
+ Instead, use a Lua conditional expression (e.g. `{% extends a or b %}`).
+* Macros are simply Lua functions and have no metadata attributes.
+* Macros do not have access to a `kwargs` variable since Lua does not support
+ keyword arguments.
+* `{% from x import y %}` tags are not supported. Instead, you must use either
+ `{% import x %}`, which imports all globals in `x` into the current
+ environment, or use `{% import x as z %}`, which imports all globals in `x`
+ into the variable `z`.
+* `{% set ... %}` does not support multiple assignment. Use `{% do ...%}`
+ instead. The catch is that `{% do ... %}` does not support filters.
+* The `{% trans %}` and `{% endtrans %}` tags, `{% with %}` and `{% endwith %}`
+ tags, and `{% autoescape %}` and `{% endautoescape %}` tags are not supported
+ since they are outside the scope of this implementation.
+
+### Filter Differences
+
+* Only the `batch`, `groupby`, and `slice` filters return generators which
+ produce one item at a time when looping. All other filters that produce
+ iterable results generate all items at once.
+* The `float` filter only works in Lua 5.3 since that version of Lua has a
+ distinction between floats and integers.
+* The `safe` filter must appear at the end of a filter chain since its output
+ cannot be passed to any other filter.
+
+### Function Differences
+
+* The global `range(n)` function returns a sequence from 1 to `n`, inclusive,
+ since lists start at 1 in Lua.
+* No `lipsum()`, `dict()`, or `joiner()` functions for the sake of simplicity.
+
+### API Differences
+
+* Lupa has a much simpler API consisting of just four functions and three
+ fields:
+
+ + `lupa.expand()`: Expands a string template subject to an environment.
+ + `lupa.expand_file()`: Expands a file template subject to an environment.
+ + `lupa.configure()` Configures delimiters and template options.
+ + `lupa.reset()`: Resets delimiters and options to their defaults.
+ + `lupa.env`: The default environment for templates.
+ + `lupa.filters`: The set of available filters (`escape`, `join`, etc.).
+ + `lupa.tests`: The set of available tests (`is_odd`, `is_defined`, etc.).
+
+* There is no bytecode caching.
+* Lupa has no extension mechanism. Instead, modify `lupa.env`, `lupa.filters`,
+ and `lupa.tests` directly. However, the parser cannot be extended.
+* Sandboxing is not supported, although `lupa.env` is safe by default (`io`,
+ `os.execute`, `os.remove`, etc. are not available).
diff --git a/contrib/lua-lupa/lupa.lua b/contrib/lua-lupa/lupa.lua
new file mode 100644
index 0000000..adaf419
--- /dev/null
+++ b/contrib/lua-lupa/lupa.lua
@@ -0,0 +1,1810 @@
+-- Copyright 2015-2020 Mitchell. See LICENSE.
+-- Sponsored by the Library of the University of Antwerp.
+-- Contributions from Ana Balan.
+-- Lupa templating engine.
+
+--[[ This comment is for LuaDoc.
+---
+-- Lupa is a Jinja2 template engine implementation written in Lua and supports
+-- Lua syntax within tags and variables.
+module('lupa')]]
+local M = {}
+
+local lpeg = require('lpeg')
+lpeg.locale(lpeg)
+local space, newline = lpeg.space, lpeg.P('\r')^-1 * '\n'
+local P, S, V = lpeg.P, lpeg.S, lpeg.V
+local C, Cc, Cg, Cp, Ct = lpeg.C, lpeg.Cc, lpeg.Cg, lpeg.Cp, lpeg.Ct
+
+---
+-- Lupa's expression filters.
+-- @class table
+-- @name filters
+M.filters = {}
+
+---
+-- Lupa's value tests.
+-- @class table
+-- @name tests
+M.tests = {}
+
+---
+-- Lupa's template loaders.
+-- @class table
+-- @name loaders
+M.loaders = {}
+
+-- Lua version compatibility.
+if _VERSION == 'Lua 5.1' then
+ function load(ld, source, mode, env)
+ local f, err = loadstring(ld)
+ if f and env then return setfenv(f, env) end
+ return f, err
+ end
+ table.unpack = unpack
+end
+
+local newline_sequence, keep_trailing_newline, autoescape = '\n', false, false
+local loader
+
+-- Creates and returns a token pattern with token name *name* and pattern
+-- *patt*.
+-- The returned pattern captures three values: the token's position and name,
+-- and either a string value or table of capture values.
+-- Tokens are used to construct an Abstract Syntax Tree (AST) for a template.
+-- @param name The name of the token.
+-- @param patt The pattern to match. It must contain only one capture: either a
+-- string or table of captures.
+-- @see evaluate
+local function token(name, patt) return Cp() * Cc(name) * patt end
+
+-- Returns an LPeg pattern that immediately raises an error with message
+-- *errmsg* for invalid syntax when parsing a template.
+-- @param errmsg The error message to raise an error with.
+local function lpeg_error(errmsg)
+ return P(function(input, index)
+ input = input:sub(1, index)
+ local _, line_num = input:gsub('\n', '')
+ local col_num = #input:match('[^\n]*$')
+ error(string.format('Parse Error in file "%s" on line %d, column %d: %s',
+ M._FILENAME, line_num + 1, col_num, errmsg), 0)
+ end)
+end
+
+---
+-- Configures the basic delimiters and options for templates.
+-- This function then regenerates the grammar for parsing templates.
+-- Note: this function cannot be used iteratively to configure Lupa options.
+-- Any options not provided are reset to their default values.
+-- @param ts The tag start delimiter. The default value is '{%'.
+-- @param te The tag end delimiter. The default value is '%}'.
+-- @param vs The variable start delimiter. The default value is '{{'.
+-- @param ve The variable end delimiter. The default value is '}}'.
+-- @param cs The comment start delimiter. The default value is '{#'.
+-- @param ce The comment end delimiter. The default value is '#}'.
+-- @param options Optional set of options for templates:
+--
+-- * `trim_blocks`: Trim the first newline after blocks.
+-- * `lstrip_blocks`: Strip line-leading whitespace in front of tags.
+-- * `newline_sequence`: The end-of-line character to use.
+-- * `keep_trailing_newline`: Whether or not to keep a newline at the end of
+-- a template.
+-- * `autoescape`: Whether or not to autoescape HTML entities. May be a
+-- function that accepts the template's filename as an argument and returns
+-- a boolean.
+-- * `loader`: Function that receives a template name to load and returns the
+-- path to that template.
+-- @name configure
+function M.configure(ts, te, vs, ve, cs, ce, options)
+ if type(ts) == 'table' then options, ts = ts, nil end
+ if not ts then ts = '{%' end
+ if not te then te = '%}' end
+ if not vs then vs = '{{' end
+ if not ve then ve = '}}' end
+ if not cs then cs = '{#' end
+ if not ce then ce = '#}' end
+
+ -- Tokens for whitespace control.
+ local lstrip = token('lstrip', C('-')) + '+' -- '+' is handled by grammar
+ local rstrip = token('rstrip', -(P(te) + ve + ce) * C('-'))
+
+ -- Configure delimiters, including whitespace control.
+ local tag_start = P(ts) * lstrip^-1 * space^0
+ local tag_end = space^0 * rstrip^-1 * P(te)
+ local variable_start = P(vs) * lstrip^-1 * space^0
+ local variable_end = space^0 * rstrip^-1 * P(ve)
+ local comment_start = P(cs) * lstrip^-1 * space^0
+ local comment_end = space^0 * rstrip^-1 * P(ce)
+ if options and options.trim_blocks then
+ -- Consider whitespace, including a newline, immediately following a tag as
+ -- part of that tag so it is not captured as plain text. Basically, strip
+ -- the trailing newline from tags.
+ tag_end = tag_end * S(' \t')^0 * newline^-1
+ comment_end = comment_end * S(' \t')^0 * newline^-1
+ end
+
+ -- Error messages.
+ local variable_end_error = lpeg_error('"'..ve..'" expected')
+ local comment_end_error = lpeg_error('"'..ce..'" expected')
+ local tag_end_error = lpeg_error('"'..te..'" expected')
+ local endraw_error = lpeg_error('additional tag or "'..ts..' endraw '..te..
+ '" expected')
+ local expr_error = lpeg_error('expression expected')
+ local endblock_error = lpeg_error('additional tag or "'..ts..' endblock '..
+ te..'" expected')
+ local endfor_error = lpeg_error('additional tag or "'..ts..' endfor '..te..
+ '" expected')
+ local endif_error = lpeg_error('additional tag or "'..ts..' endif '..te..
+ '" expected')
+ local endmacro_error = lpeg_error('additional tag or "'..ts..' endmacro '..
+ te..'" expected')
+ local endcall_error = lpeg_error('additional tag or "'..ts..' endcall '..te..
+ '" expected')
+ local endfilter_error = lpeg_error('additional tag or "'..ts..' endfilter '..
+ te..'" expected')
+ local tag_error = lpeg_error('unknown or unexpected tag')
+ local main_error = lpeg_error('unexpected character; text or tag expected')
+
+ -- Grammar.
+ M.grammar = Ct(P{
+ -- Utility patterns used by tokens.
+ entity_start = tag_start + variable_start + comment_start,
+ any_text = (1 - V('entity_start'))^1,
+ -- Allow '{{' by default in expression text since it is valid in Lua.
+ expr_text = (1 - tag_end - tag_start - comment_start)^1,
+ -- When `options.lstrip_blocks` is enabled, ignore leading whitespace
+ -- immediately followed by a tag (as long as '+' is not present) so that
+ -- whitespace not captured as plain text. Basically, strip leading spaces
+ -- from tags.
+ line_text = (1 - newline - V('entity_start'))^1,
+ lstrip_entity_start = -P(vs) * (P(ts) + cs) * -P('+'),
+ lstrip_space = S(' \t')^1 * #V('lstrip_entity_start'),
+ text_lines = V('line_text') * (newline * -(S(' \t')^0 * V('lstrip_entity_start')) * V('line_text'))^0 * newline^-1 + newline,
+
+ -- Plain text.
+ text = (not options or not options.lstrip_blocks) and
+ token('text', C(V('any_text'))) or
+ V('lstrip_space') + token('text', C(V('text_lines'))),
+
+ -- Variables: {{ expr }}.
+ lua_table = '{' * ((1 - S('{}')) + V('lua_table'))^0 * '}',
+ variable = variable_start *
+ token('variable', C((V('lua_table') + (1 - variable_end))^0)) *
+ (variable_end + variable_end_error),
+
+ -- Filters: handled in variable evaluation.
+
+ -- Tests: handled in control structure expression evaluation.
+
+ -- Comments: {# comment #}.
+ comment = comment_start * (1 - comment_end)^0 * (comment_end + comment_end_error),
+
+ -- Whitespace control: handled in tag/variable/comment start/end.
+
+ -- Escaping: {% raw %} body {% endraw %}.
+ raw_block = tag_start * 'raw' * (tag_end + tag_end_error) *
+ token('text', C((1 - (tag_start * 'endraw' * tag_end))^0)) *
+ (tag_start * 'endraw' * tag_end + endraw_error),
+
+ -- Note: line statements are not supported since this grammer cannot parse
+ -- Lua itself.
+
+ -- Template inheritence.
+ -- {% block ... %} body {% endblock %}
+ block_block = tag_start * 'block' * space^1 * token('block', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endblock' * tag_end + endblock_error),
+ -- {% extends ... %}
+ extends_tag = tag_start * 'extends' * space^1 * token('extends', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+ -- Super blocks are handled in variables.
+ -- Note: named block end tags are not supported since keeping track of that
+ -- state information is difficult.
+ -- Note: block nesting and scope is not applicable since blocks always have
+ -- access to scoped variables in this implementation.
+
+ -- Control Structures.
+ -- {% for expr %} body {% else %} body {% endfor %}
+ for_block = tag_start * 'for' * space^1 * token('for', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1 *
+ Cg(Ct(tag_start * 'else' * tag_end *
+ V('body')^-1), 'else')^-1)) *
+ (tag_start * 'endfor' * tag_end + endfor_error),
+ -- {% if expr %} body {% elseif expr %} body {% else %} body {% endif %}
+ if_block = tag_start * 'if' * space^1 * token('if', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1 *
+ Cg(Ct(Ct(tag_start * 'elseif' * space^1 * (Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)^1), 'elseif')^-1 *
+ Cg(Ct(tag_start * 'else' * tag_end *
+ V('body')^-1), 'else')^-1)) *
+ (tag_start * 'endif' * tag_end + endif_error),
+ -- {% macro expr %} body {% endmacro %}
+ macro_block = tag_start * 'macro' * space^1 * token('macro', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endmacro' * tag_end + endmacro_error),
+ -- {% call expr %} body {% endcall %}
+ call_block = tag_start * 'call' * (space^1 + #P('(')) * token('call', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endcall' * tag_end + endcall_error),
+ -- {% filter expr %} body {% endfilter %}
+ filter_block = tag_start * 'filter' * space^1 * token('filter', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endfilter' * tag_end + endfilter_error),
+ -- {% set ... %}
+ set_tag = tag_start * 'set' * space^1 * token('set', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+ -- {% include ... %}
+ include_tag = tag_start * 'include' * space^1 * token('include', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+ -- {% import ... %}
+ import_tag = tag_start * 'import' * space^1 * token('import', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+
+ -- Note: i18n is not supported since it is out of scope for this
+ -- implementation.
+
+ -- Expression statement: {% do ... %}.
+ do_tag = tag_start * 'do' * space^1 * token('do', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+
+ -- Note: loop controls are not supported since that would require jumping
+ -- between "scopes" (e.g. from within an "if" block to outside that "if"
+ -- block's parent "for" block when coming across a {% break %} tag).
+
+ -- Note: with statement is not supported since it is out of scope for this
+ -- implementation.
+
+ -- Note: autoescape is not supported since it is out of scope for this
+ -- implementation.
+
+ -- Any valid blocks of text or tags.
+ body = (V('text') + V('variable') + V('comment') + V('raw_block') +
+ V('block_block') + V('extends_tag') + V('for_block') +
+ V('if_block') + V('macro_block') + V('call_block') +
+ V('filter_block') + V('set_tag') + V('include_tag') +
+ V('import_tag') + V('do_tag'))^0,
+
+ -- Main pattern.
+ V('body') * (-1 + tag_start * tag_error + main_error),
+ })
+
+ -- Other options.
+ if options and options.newline_sequence then
+ assert(options.newline_sequence:find('^\r?\n$'),
+ 'options.newline_sequence must be "\r\n" or "\n"')
+ newline_sequence = options.newline_sequence
+ else
+ newline_sequence = '\n'
+ end
+ if options and options.keep_trailing_newline then
+ keep_trailing_newline = options.keep_trailing_newline
+ else
+ keep_trailing_newline = false
+ end
+ if options and options.autoescape then
+ autoescape = options.autoescape
+ else
+ autoescape = false
+ end
+ if options and options.loader then
+ assert(type(options.loader) == 'function',
+ 'options.loader must be a function that returns a filename')
+ loader = options.loader
+ else
+ loader = M.loaders.filesystem()
+ end
+end
+
+-- Wraps Lua's `assert()` in template environment *env* such that, when called
+-- in conjunction with another Lua function that produces an error message (e.g.
+-- `load()` and `pcall()`), that error message's context (source and line
+-- number) is replaced by the template's context.
+-- This results in Lua's error messages pointing to a template position rather
+-- than this library's source code.
+-- @param env The environment for the currently running template. It must have
+-- a `_SOURCE` field with the template's source text and a `_POSITION` field
+-- with the current position of expansion.
+-- @param ... Arguments to Lua's `assert()`.
+local function env_assert(env, ...)
+ if not select(1, ...) then
+ local input = env._LUPASOURCE:sub(1, env._LUPAPOSITION)
+ local _, line_num = input:gsub('\n', '')
+ local col_num = #input:match('[^\n]*$')
+ local errmsg = select(2, ...)
+ errmsg = errmsg:match(':%d+: (.*)$') or errmsg -- reformat if necessary
+ error(string.format('Runtime Error in file "%s" on line %d, column %d: %s',
+ env._LUPAFILENAME, line_num + 1, col_num, errmsg), 0)
+ end
+ return ...
+end
+
+-- Returns a generator that returns the position and filter in a list of
+-- filters, taking into account '|'s that may be within filter arguments.
+-- @usage for pos, filter in each_filter('foo|join("|")|bar') do ... end
+local function each_filter(s)
+ local init = 1
+ return function(s)
+ local pos, filter, e = s:match('^%s*()([^|(]+%b()[^|]*)|?()', init)
+ if not pos then pos, filter, e = s:match('()([^|]+)|?()', init) end
+ init = e
+ return pos, filter
+ end, s
+end
+
+-- Evaluates template variable *expression* subject to template environment
+-- *env*, applying any filters given in *expression*.
+-- @param expression The string expression to evaluate.
+-- @param env The environment to evaluate the expression in.
+local function eval(expression, env)
+ local expr, pos, filters = expression:match('^([^|]*)|?()(.-)$')
+ -- Evaluate base expression.
+ local f = env_assert(env, load('return '..expr, nil, nil, env))
+ local result = select(2, env_assert(env, pcall(f)))
+ -- Apply any filters.
+ local results, multiple_results = nil, false
+ local p = env._LUPAPOSITION + pos - 1 -- mark position at first filter
+ for pos, filter in each_filter(filters) do
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ local name, params = filter:match('^%s*([%w_]+)%(?(.-)%)?%s*$')
+ f = M.filters[name]
+ env_assert(env, f, 'unknown filter "'..name..'"')
+ local args = env_assert(env, load('return {'..params..'}', nil, nil, env),
+ 'invalid filter parameter(s) for "'..name..'"')()
+ if not multiple_results then
+ results = {select(2,
+ env_assert(env, pcall(f, result, table.unpack(args))))}
+ else
+ for i = 1, #results do table.insert(args, i, results[i]) end
+ results = {select(2, env_assert(env, pcall(f, table.unpack(args))))}
+ end
+ result, multiple_results = results[1], #results > 1
+ end
+ if multiple_results then return table.unpack(results) end
+ return result
+end
+
+local iterate
+
+-- Iterates over *ast*, a collection of tokens from a portion of a template's
+-- Abstract Syntax Tree (AST), evaluating any expressions in template
+-- environment *env*, and returns a concatenation of the results.
+-- @param ast A template's AST or portion of its AST (e.g. portion inside a
+-- 'for' control structure).
+-- @param env Environment to evaluate any expressions in.
+local function evaluate(ast, env)
+ local chunks = {}
+ local extends -- text of a parent template
+ local rstrip -- flag for stripping leading whitespace of next token
+ for i = 1, #ast, 3 do
+ local pos, token, block = ast[i], ast[i + 1], ast[i + 2]
+ env._LUPAPOSITION = pos
+ if token == 'text' then
+ chunks[#chunks + 1] = block
+ elseif token == 'variable' then
+ local value = eval(block, env)
+ if autoescape then
+ local escape = autoescape
+ if type(autoescape) == 'function' then
+ escape = autoescape(env._LUPAFILENAME) -- TODO: test
+ end
+ if escape and type(value) == 'string' then
+ value = M.filters.escape(value)
+ end
+ end
+ chunks[#chunks + 1] = value ~= nil and tostring(value) or ''
+ elseif token == 'extends' then
+ env_assert(env, not extends,
+ 'cannot have multiple "extends" in the same scope')
+ local file = eval(block, env) -- covers strings and variables
+ extends = file
+ env._LUPAEXTENDED = true -- used by parent templates
+ elseif token == 'block' then
+ local name = block.expression:match('^[%w_]+$')
+ env_assert(env, name, 'invalid block name')
+ -- Store the block for potential use by the parent template if this
+ -- template is a child template, or for use by `self`.
+ if not env._LUPABLOCKS then env._LUPABLOCKS = {} end
+ if not env._LUPABLOCKS[name] then env._LUPABLOCKS[name] = {} end
+ table.insert(env._LUPABLOCKS[name], 1, block)
+ -- Handle the block properly.
+ if not extends then
+ if not env._LUPAEXTENDED then
+ -- Evaluate the block normally.
+ chunks[#chunks + 1] = evaluate(block, env)
+ else
+ -- A child template is overriding this parent's named block. Evaluate
+ -- the child's block and use it instead of the parent's.
+ local blocks = env._LUPABLOCKS[name]
+ local super_env = setmetatable({super = function()
+ -- Loop through the chain of defined blocks, evaluating from top to
+ -- bottom, and return the bottom block. In each sub-block, the
+ -- 'super' variable needs to point to the next-highest block's
+ -- evaluated result.
+ local super = evaluate(block, env) -- start with parent block
+ local sub_env = setmetatable({super = function() return super end},
+ {__index = env})
+ for i = 1, #blocks - 1 do super = evaluate(blocks[i], sub_env) end
+ return super
+ end}, {__index = env})
+ chunks[#chunks + 1] = evaluate(blocks[#blocks], super_env)
+ end
+ end
+ elseif token == 'for' then
+ local expr = block.expression
+ local p = env._LUPAPOSITION -- mark position at beginning of expression
+ -- Extract variable list and generator.
+ local patt = '^([%w_,%s]+)%s+in%s+()(.+)%s+if%s+(.+)$'
+ local var_list, pos, generator, if_expr = expr:match(patt)
+ if not var_list then
+ var_list, pos, generator = expr:match('^([%w_,%s]+)%s+in%s+()(.+)$')
+ end
+ env_assert(env, var_list and generator, 'invalid for expression')
+ -- Store variable names in a list for loop assignment.
+ local variables = {}
+ for variable, pos in var_list:gmatch('([^,%s]+)()') do
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ env_assert(env, variable:find('^[%a_]') and variable ~= 'loop',
+ 'invalid variable name')
+ variables[#variables + 1] = variable
+ end
+ -- Evaluate the generator and perform the iteration.
+ env._LUPAPOSITION = p + pos - 1 -- update position to generator
+ if not generator:find('|') then
+ generator = env_assert(env, load('return '..generator, nil, nil, env))
+ else
+ local generator_expr = generator
+ generator = function() return eval(generator_expr, env) end
+ end
+ local new_env = setmetatable({}, {__index = env})
+ chunks[#chunks + 1] = iterate(generator, variables, if_expr, block,
+ new_env, 1, ast[i + 4] == 'lstrip')
+ elseif token == 'if' then
+ if eval(block.expression, env) then
+ chunks[#chunks + 1] = evaluate(block, env)
+ else
+ local evaluate_else = true
+ local elseifs = block['elseif']
+ if elseifs then
+ for j = 1, #elseifs do
+ if eval(elseifs[j].expression, env) then
+ chunks[#chunks + 1] = evaluate(elseifs[j], env)
+ evaluate_else = false
+ break
+ end
+ end
+ end
+ if evaluate_else and block['else'] then
+ chunks[#chunks + 1] = evaluate(block['else'], env)
+ end
+ end
+ elseif token == 'macro' then
+ -- Parse the macro's name and parameter list.
+ local signature = block.expression
+ local name, param_list = signature:match('^([%w_]+)(%b())')
+ env_assert(env, name and param_list, 'invalid macro expression')
+ param_list = param_list:sub(2, -2)
+ local p = env._LUPAPOSITION + #name + 1 -- mark pos at beginning of args
+ local params, defaults = {}, {}
+ for param, pos, default in param_list:gmatch('([%w_]+)=?()([^,]*)') do
+ params[#params + 1] = param
+ if default ~= '' then
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ local f = env_assert(env, load('return '..default))
+ defaults[param] = select(2, env_assert(env, pcall(f)))
+ end
+ end
+ -- Create the function associated with the macro such that when the
+ -- function is called (from within {{ ... }}), the macro's body is
+ -- evaluated subject to an environment where parameter names are variables
+ -- whose values are the ones passed to the macro itself.
+ env[name] = function(...)
+ local new_env = setmetatable({}, {__index = function(_, k)
+ if k == 'caller' and type(env[k]) ~= 'function' then return nil end
+ return env[k]
+ end})
+ local args = {...}
+ -- Assign the given parameter values.
+ for i = 1, #args do
+ if i > #params then break end
+ new_env[params[i]] = args[i]
+ end
+ -- Clear all other unspecified parameter values or set them to their
+ -- defined defaults.
+ for i = #args + 1, #params do
+ new_env[params[i]] = defaults[params[i]]
+ end
+ -- Store extra parameters in "varargs" variable.
+ new_env.varargs = {}
+ for i = #params + 1, #args do
+ new_env.varargs[#new_env.varargs + 1] = args[i]
+ end
+ local chunk = evaluate(block, new_env)
+ if ast[i + 4] == 'lstrip' then chunk = chunk:gsub('%s*$', '') end
+ return chunk
+ end
+ elseif token == 'call' then
+ -- Parse the call block's parameter list (if any) and determine the macro
+ -- to call.
+ local param_list = block.expression:match('^(%b())')
+ local params = {}
+ if param_list then
+ for param in param_list:gmatch('[%w_]+') do
+ params[#params + 1] = param
+ end
+ end
+ local macro = block.expression:match('^%b()(.+)$') or block.expression
+ -- Evaluate the given macro, subject to a "caller" function that returns
+ -- the contents of this call block. Any arguments passed to the caller
+ -- function are used as values of this parameters parsed earlier.
+ local old_caller = M.env.caller -- save
+ M.env.caller = function(...)
+ local new_env = setmetatable({}, {__index = env})
+ local args = {...}
+ -- Assign the given parameter values (if any).
+ for i = 1, #args do new_env[params[i]] = args[i] end
+ local chunk = evaluate(block, new_env)
+ if ast[i + 4] == 'lstrip' then chunk = chunk:gsub('%s*$', '') end
+ return chunk
+ end
+ chunks[#chunks + 1] = eval(macro, env)
+ M.env.caller = old_caller -- restore
+ elseif token == 'filter' then
+ local text = evaluate(block, env)
+ local p = env._LUPAPOSITION -- mark position at beginning of expression
+ for pos, filter in each_filter(block.expression) do
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ local name, params = filter:match('^%s*([%w_]+)%(?(.-)%)?%s*$')
+ local f = M.filters[name]
+ env_assert(env, f, 'unknown filter "'..name..'"')
+ local args = env_assert(env, load('return {'..params..'}'),
+ 'invalid filter parameter(s) for "'..name..
+ '"')()
+ text = select(2, env_assert(env, pcall(f, text, table.unpack(args))))
+ end
+ chunks[#chunks + 1] = text
+ elseif token == 'set' then
+ local var, expr = block:match('^([%a_][%w_]*)%s*=%s*(.+)$')
+ env_assert(env, var and expr, 'invalid variable name or expression')
+ env[var] = eval(expr, env)
+ elseif token == 'do' then
+ env_assert(env, pcall(env_assert(env, load(block, nil, nil, env))))
+ elseif token == 'include' then
+ -- Parse the include block for flags.
+ local without_context = block:find('without%s+context%s*')
+ local ignore_missing = block:find('ignore%s+missing%s*')
+ block = block:gsub('witho?u?t?%s+context%s*', '')
+ :gsub('ignore%s+missing%s*', '')
+ -- Evaluate the include expression in order to determine the file to
+ -- include. If the result is a table, use the first file that exists.
+ local file = eval(block, env) -- covers strings and variables
+ if type(file) == 'table' then
+ local files = file
+ for i = 1, #files do
+ file = loader(files[i], env)
+ if file then break end
+ end
+ if type(file) == 'table' then file = nil end
+ elseif type(file) == 'string' then
+ file = loader(file, env)
+ else
+ error('"include" requires a string or table of files')
+ end
+ -- If the file exists, include it. Otherwise throw an error unless the
+ -- "ignore missing" flag was given.
+ env_assert(env, file or ignore_missing, 'no file(s) found to include')
+ if file then
+ chunks[#chunks + 1] = M.expand_file(file, not without_context and env or
+ M.env)
+ end
+ elseif token == 'import' then
+ local file, global = block:match('^%s*(.+)%s+as%s+([%a][%w_]*)%s*')
+ local new_env = setmetatable({}, {
+ __index = block:find('with%s+context%s*$') and env or M.env
+ })
+ M.expand_file(eval(file or block, env), new_env)
+ -- Copy any defined macros and variables over into the proper namespace.
+ if global then env[global] = {} end
+ local namespace = global and env[global] or env
+ for k, v in pairs(new_env) do if not env[k] then namespace[k] = v end end
+ elseif token == 'lstrip' and chunks[#chunks] then
+ chunks[#chunks] = chunks[#chunks]:gsub('%s*$', '')
+ elseif token == 'rstrip' then
+ rstrip = true -- can only strip after determining the next chunk
+ end
+ if rstrip and token ~= 'rstrip' then
+ chunks[#chunks] = chunks[#chunks]:gsub('^%s*', '')
+ rstrip = false
+ end
+ end
+ return not extends and table.concat(chunks) or M.expand_file(extends, env)
+end
+
+local pairs_gen, ipairs_gen = pairs({}), ipairs({})
+
+-- Iterates over the generator *generator* subject to string "if" expression
+-- *if_expr*, assigns that generator's returned values to the variable names
+-- listed in *variables* within template environment *env*, evaluates any
+-- expressions in *block* (a portion of a template's AST), and returns a
+-- concatenation of the results.
+-- @param generator Either a function that returns a generator function, or a
+-- table to iterate over. In the latter case, `ipairs()` is used as the
+-- generator function.
+-- @param variables List of variable names to assign values returned by
+-- *generator* to.
+-- @param if_expr A conditional expression that when `false`, skips the current
+-- loop item.
+-- @param block The portion inside the 'for' structure of a template's AST to
+-- iterate with.
+-- @param env The environment iteration variables are defined in and where
+-- expressions are evaluated in.
+-- @param depth The current recursion depth. Recursion is performed by calling
+-- `loop(t)` with a table to iterate over.
+-- @param lstrip Whether or not the "endfor" block strips whitespace on the
+-- left. When `true`, all blocks produced by iteration are left-stripped.
+iterate = function(generator, variables, if_expr, block, env, depth, lstrip)
+ local chunks = {}
+ local orig_variables = {} -- used to store original loop variables' values
+ for i = 1, #variables do orig_variables[variables[i]] = env[variables[i]] end
+ local i, n = 1 -- used for loop variables
+ local _, s, v -- state variables
+ if type(generator) == 'function' then
+ _, generator, s, v = env_assert(env, pcall(generator))
+ -- In practice, a generator's state variable is normally unused and hidden.
+ -- This is not the case for 'pairs()' and 'ipairs', though.
+ if variables[1] ~= '_index' and generator ~= pairs_gen and
+ generator ~= ipairs_gen then
+ table.insert(variables, 1, '_index')
+ end
+ end
+ if type(generator) == 'table' then
+ n = #generator
+ generator, s, v = ipairs(generator)
+ -- "for x in y" translates to "for _, x in ipairs(y)"; hide _ state variable
+ if variables[1] ~= '_index' then table.insert(variables, 1, '_index') end
+ end
+ if generator then
+ local first_results -- for preventing infinite loop from invalid generator
+ while true do
+ local results = {generator(s, v)}
+ if results[1] == nil then break end
+ -- If the results from the generator look like results returned by a
+ -- generator itself (function, state, initial variable), verify last two
+ -- results are different. If they are the same, then the original
+ -- generator is invalid and will loop infinitely.
+ if first_results == nil then
+ first_results = #results == 3 and type(results[1]) == 'function' and
+ results
+ elseif first_results then
+ env_assert(env, results[3] ~= first_results[3] or
+ results[2] ~= first_results[2],
+ 'invalid generator (infinite loop)')
+ end
+ -- Assign context variables and evaluate the body of the loop.
+ -- As long as the result (ignoring the _index variable) is not a single
+ -- table and there is only one loop variable defined (again, ignoring
+ -- _index variable), assignment occurs as normal in Lua. Otherwise,
+ -- unpacking on the table is done (like assignment to ...).
+ if not (type(results[2]) == 'table' and #results == 2 and
+ #variables > 2) then
+ for j = 1, #variables do env[variables[j]] = results[j] end
+ else
+ for j = 2, #variables do env[variables[j]] = results[2][j - 1] end
+ end
+ if not if_expr or eval(if_expr, env) then
+ env.loop = setmetatable({
+ index = i, index0 = i - 1,
+ revindex = n and n - (i - 1), revindex0 = n and n - i,
+ first = i == 1, last = i == n, length = n,
+ cycle = function(...)
+ return select((i - 1) % select('#', ...) + 1, ...)
+ end,
+ depth = depth, depth0 = depth - 1
+ }, {__call = function(_, t)
+ return iterate(t, variables, if_expr, block, env, depth + 1, lstrip)
+ end})
+ chunks[#chunks + 1] = evaluate(block, env)
+ if lstrip then chunks[#chunks] = chunks[#chunks]:gsub('%s*$', '') end
+ i = i + 1
+ end
+ -- Prepare for next iteration.
+ v = results[1]
+ end
+ end
+ if i == 1 and block['else'] then
+ chunks[#chunks + 1] = evaluate(block['else'], env)
+ end
+ for i = 1, #variables do env[variables[i]] = orig_variables[variables[i]] end
+ return table.concat(chunks)
+end
+
+-- Expands string template *template* from source *source*, subject to template
+-- environment *env*, and returns the result.
+-- @param template String template to expand.
+-- @param env Environment for the given template.
+-- @param source Filename or identifier the template comes from for error
+-- messages and debugging.
+local function expand(template, env, source)
+ template = template:gsub('\r?\n', newline_sequence) -- normalize
+ if not keep_trailing_newline then template = template:gsub('\r?\n$', '') end
+ -- Set up environment.
+ if not env then env = {} end
+ if not getmetatable(env) then env = setmetatable(env, {__index = M.env}) end
+ env.self = setmetatable({}, {__index = function(_, k)
+ env_assert(env, env._LUPABLOCKS and env._LUPABLOCKS[k],
+ 'undefined block "'..k..'"')
+ return function() return evaluate(env._LUPABLOCKS[k][1], env) end
+ end})
+ -- Set context variables and expand the template.
+ env._LUPASOURCE, env._LUPAFILENAME = template, source
+ M._FILENAME = source -- for lpeg errors only
+ local ast = assert(lpeg.match(M.grammar, template), "internal error")
+ local result = evaluate(ast, env)
+ return result
+end
+
+---
+-- Expands the string template *template*, subject to template environment
+-- *env*, and returns the result.
+-- @param template String template to expand.
+-- @param env Optional environment for the given template.
+-- @name expand
+function M.expand(template, env) return expand(template, env, '<string>') end
+
+---
+-- Expands the template within file *filename*, subject to template environment
+-- *env*, and returns the result.
+-- @param filename Filename containing the template to expand.
+-- @param env Optional environment for the template to expand.
+-- @name expand_file
+function M.expand_file(filename, env)
+ filename = loader(filename, env) or filename
+ local f = (not env or not env._LUPASOURCE) and assert(io.open(filename)) or
+ env_assert(env, io.open(filename))
+ local template = f:read('*a')
+ f:close()
+ return expand(template, env, filename)
+end
+
+---
+-- Returns a loader for templates that uses the filesystem starting at directory
+-- *directory*.
+-- When looking up the template for a given filename, the loader considers the
+-- following: if no template is being expanded, the loader assumes the given
+-- filename is relative to *directory* and returns the full path; otherwise the
+-- loader assumes the given filename is relative to the current template's
+-- directory and returns the full path.
+-- The returned path may be passed to `io.open()`.
+-- @param directory Optional the template root directory. The default value is
+-- ".", which is the current working directory.
+-- @name loaders.filesystem
+-- @see configure
+function M.loaders.filesystem(directory)
+ return function(filename, env)
+ if not filename then return nil end
+ local current_dir = env and env._LUPAFILENAME and
+ env._LUPAFILENAME:match('^(.+)[/\\]')
+ if not filename:find('^/') and not filename:find('^%a:[/\\]') then
+ filename = (current_dir or directory or '.')..'/'..filename
+ end
+ local f = io.open(filename)
+ if not f then return nil end
+ f:close()
+ return filename
+ end
+end
+
+-- Globally defined functions.
+
+---
+-- Returns a sequence of integers from *start* to *stop*, inclusive, in
+-- increments of *step*.
+-- The complete sequence is generated at once -- no generator is returned.
+-- @param start Optional number to start at. The default value is `1`.
+-- @param stop Number to stop at.
+-- @param step Optional increment between sequence elements. The default value
+-- is `1`.
+-- @name _G.range
+function range(start, stop, step)
+ if not stop and not step then stop, start = start, 1 end
+ if not step then step = 1 end
+ local t = {}
+ for i = start, stop, step do t[#t + 1] = i end
+ return t
+end
+
+---
+-- Returns an object that cycles through the given values by calls to its
+-- `next()` function.
+-- A `current` field contains the cycler's current value and a `reset()`
+-- function resets the cycler to its beginning.
+-- @param ... Values to cycle through.
+-- @usage c = cycler(1, 2, 3)
+-- @usage c:next(), c:next() --> 1, 2
+-- @usage c:reset() --> c.current == 1
+-- @name _G.cycler
+function cycler(...)
+ local c = {...}
+ c.n, c.i, c.current = #c, 1, c[1]
+ function c:next()
+ local current = self.current
+ self.i = self.i + 1
+ if self.i > self.n then self.i = 1 end
+ self.current = self[self.i]
+ return current
+ end
+ function c:reset() self.i, self.current = 1, self[1] end
+ return c
+end
+
+-- Create the default sandbox environment for templates.
+local safe = {
+ -- Lua globals.
+ '_VERSION', 'ipairs', 'math', 'pairs', 'select', 'tonumber', 'tostring',
+ 'type', 'bit32', 'os.date', 'os.time', 'string', 'table', 'utf8',
+ -- Lupa globals.
+ 'range', 'cycler'
+}
+local sandbox_env = setmetatable({}, {__index = M.tests})
+for i = 1, #safe do
+ local v = safe[i]
+ if not v:find('%.') then
+ sandbox_env[v] = _G[v]
+ else
+ local mod, func = v:match('^([^.]+)%.(.+)$')
+ if not sandbox_env[mod] then sandbox_env[mod] = {} end
+ sandbox_env[mod][func] = _G[mod][func]
+ end
+end
+sandbox_env._G = sandbox_env
+
+---
+-- Resets Lupa's default delimiters, options, and environments to their
+-- original default values.
+-- @name reset
+function M.reset()
+ M.configure('{%', '%}', '{{', '}}', '{#', '#}')
+ M.env = setmetatable({}, {__index = sandbox_env})
+end
+M.reset()
+
+---
+-- The default template environment.
+-- @class table
+-- @name env
+local env
+
+-- Lupa filters.
+
+---
+-- Returns the absolute value of number *n*.
+-- @param n The number to compute the absolute value of.
+-- @name filters.abs
+M.filters.abs = math.abs
+
+-- Returns a table that, when indexed with an integer, indexes table *t* with
+-- that integer along with string *attribute*.
+-- This is used by filters that operate on particular attributes of table
+-- elements.
+-- @param t The table to index.
+-- @param attribute The additional attribute to index with.
+local function attr_accessor(t, attribute)
+ return setmetatable({}, {__index = function(_, i)
+ local value = t[i]
+ attribute = tonumber(attribute) or attribute
+ if type(attribute) == 'number' then return value[attribute] end
+ for k in attribute:gmatch('[^.]+') do value = value[k] end
+ return value
+ end})
+end
+
+---
+-- Returns a generator that produces all of the items in table *t* in batches
+-- of size *size*, filling any empty spaces with value *fill*.
+-- Combine this with the "list" filter to produce a list.
+-- @param t The table to split into batches.
+-- @param size The batch size.
+-- @param fill The value to use when filling in any empty space in the last
+-- batch.
+-- @usage expand('{% for i in {1, 2, 3}|batch(2, 0) %}{{ i|string }}
+-- {% endfor %}') --> {1, 2} {3, 0}
+-- @see filters.list
+-- @name filters.batch
+function M.filters.batch(t, size, fill)
+ assert(t, 'input to filter "batch" was nil instead of a table')
+ local n = #t
+ return function(t, i)
+ if i > n then return nil end
+ local batch = {}
+ for j = i, i + size - 1 do batch[j - i + 1] = t[j] end
+ if i + size > n and fill then
+ for j = n + 1, i + size - 1 do batch[#batch + 1] = fill end
+ end
+ return i + size, batch
+ end, t, 1
+end
+
+---
+-- Capitalizes string *s*.
+-- The first character will be uppercased, the others lowercased.
+-- @param s The string to capitalize.
+-- @usage expand('{{ "foo bar"|capitalize }}') --> Foo bar
+-- @name filters.capitalize
+function M.filters.capitalize(s)
+ assert(s, 'input to filter "capitalize" was nil instead of a string')
+ local first, rest = s:match('^(.)(.*)$')
+ return first and first:upper()..rest:lower() or s
+end
+
+---
+-- Centers string *s* within a string of length *width*.
+-- @param s The string to center.
+-- @param width The length of the centered string.
+-- @usage expand('{{ "foo"|center(9) }}') --> " foo "
+-- @name filters.center
+function M.filters.center(s, width)
+ assert(s, 'input to filter "center" was nil instead of a string')
+ local padding = (width or 80) - #s
+ local left, right = math.ceil(padding / 2), math.floor(padding / 2)
+ return ("%s%s%s"):format((' '):rep(left), s, (' '):rep(right))
+end
+
+---
+-- Returns value *value* or value *default*, depending on whether or not *value*
+-- is "true" and whether or not boolean *false_defaults* is `true`.
+-- @param value The value return if "true" or if `false` and *false_defaults*
+-- is `true`.
+-- @param default The value to return if *value* is `nil` or `false` (the latter
+-- applies only if *false_defaults* is `true`).
+-- @param false_defaults Optional flag indicating whether or not to return
+-- *default* if *value* is `false`. The default value is `false`.
+-- @usage expand('{{ false|default("no") }}') --> false
+-- @usage expand('{{ false|default("no", true) }') --> no
+-- @name filters.default
+function M.filters.default(value, default, false_defaults)
+ if value == nil or false_defaults and not value then return default end
+ return value
+end
+
+---
+-- Returns a table constructed from table *t* such that each element is a list
+-- that contains a single key-value pair and all elements are sorted according
+-- to string *by* (which is either "key" or "value") and boolean
+-- *case_sensitive*.
+-- @param value The table to sort.
+-- @param case_sensitive Optional flag indicating whether or not to consider
+-- case when sorting string values. The default value is `false`.
+-- @param by Optional string that specifies which of the key-value to sort by,
+-- either "key" or "value". The default value is `"key"`.
+-- @usage expand('{{ {b = 1, a = 2}|dictsort|string }}') --> {{"a", 2},
+-- {"b", 1}}
+-- @name filters.dictsort
+function M.filters.dictsort(t, case_sensitive, by)
+ assert(t, 'input to filter "dictsort" was nil instead of a table')
+ assert(not by or by == 'key' or by == 'value',
+ 'filter "dictsort" can only sort tables by "key" or "value"')
+ local i = (not by or by == 'key') and 1 or 2
+ local items = {}
+ for k, v in pairs(t) do items[#items + 1] = {k, v} end
+ table.sort(items, function(a, b)
+ a, b = a[i], b[i]
+ if not case_sensitive then
+ if type(a) == 'string' then a = a:lower() end
+ if type(b) == 'string' then b = b:lower() end
+ end
+ return a < b
+ end)
+ return items
+end
+
+---
+-- Returns an HTML-safe copy of string *s*.
+-- @param s String to ensure is HTML-safe.
+-- @usage expand([[{{ '<">&'|e}}]]) --> &lt;&#34;&gt;&amp;
+-- @name filters.escape
+function M.filters.escape(s)
+ assert(s, 'input to filter "escape" was nil instead of a string')
+ return s:gsub('[<>"\'&]', {
+ ['<'] = '&lt;', ['>'] = '&gt;', ['"'] = '&#34;', ["'"] = '&#39;',
+ ['&'] = '&amp;'
+ })
+end
+
+---
+-- Returns an HTML-safe copy of string *s*.
+-- @param s String to ensure is HTML-safe.
+-- @usage expand([[{{ '<">&'|escape}}]]) --> &lt;&#34;&gt;&amp;
+-- @name filters.e
+function M.filters.e(s)
+ assert(s, 'input to filter "e" was nil instead of a string')
+ return M.filters.escape(s)
+end
+
+---
+-- Returns a human-readable, decimal (or binary, depending on boolean *binary*)
+-- file size for *bytes* number of bytes.
+-- @param bytes The number of bytes to return the size for.
+-- @param binary Flag indicating whether or not to report binary file size
+-- as opposed to decimal file size. The default value is `false`.
+-- @usage expand('{{ 1000|filesizeformat }}') --> 1.0 kB
+-- @name filters.filesizeformat
+function M.filters.filesizeformat(bytes, binary)
+ assert(bytes, 'input to filter "filesizeformat" was nil instead of a number')
+ local base = binary and 1024 or 1000
+ local units = {
+ binary and 'KiB' or 'kB', binary and 'MiB' or 'MB',
+ binary and 'GiB' or 'GB', binary and 'TiB' or 'TB',
+ binary and 'PiB' or 'PB', binary and 'EiB' or 'EB',
+ binary and 'ZiB' or 'ZB', binary and 'YiB' or 'YB'
+ }
+ if bytes < base then
+ return string.format('%d Byte%s', bytes, bytes > 1 and 's' or '')
+ else
+ local limit, unit
+ for i = 1, #units do
+ limit, unit = base^(i + 1), units[i]
+ if bytes < limit then break end
+ end
+ return string.format('%.1f %s', (base * bytes / limit), unit)
+ end
+end
+
+---
+-- Returns the first element in table *t*.
+-- @param t The table to get the first element of.
+-- @usage expand('{{ range(10)|first }}') --> 1
+-- @name filters.first
+function M.filters.first(t)
+ assert(t, 'input to filter "first" was nil instead of a table')
+ return t[1]
+end
+
+---
+-- Returns value *value* as a float.
+-- This filter only works in Lua 5.3, which has a distinction between floats and
+-- integers.
+-- @param value The value to interpret as a float.
+-- @usage expand('{{ 42|float }}') --> 42.0
+-- @name filters.float
+function M.filters.float(value)
+ assert(value, 'input to filter "float" was nil instead of a number')
+ return (tonumber(value) or 0) * 1.0
+end
+
+---
+-- Returns an HTML-safe copy of value *value*, even if *value* was returned by
+-- the "safe" filter.
+-- @param value Value to ensure is HTML-safe.
+-- @usage expand('{% set x = "<div />"|safe %}{{ x|forceescape }}') -->
+-- &lt;div /&gt;
+-- @name filters.forceescape
+function M.filters.forceescape(value)
+ assert(value, 'input to filter "forceescape" was nil instead of a string')
+ return M.filters.escape(tostring(value))
+end
+
+---
+-- Returns the given arguments formatted according to string *s*.
+-- See Lua's `string.format()` for more information.
+-- @param s The string to format subsequent arguments according to.
+-- @param ... Arguments to format.
+-- @usage expand('{{ "%s,%s"|format("a", "b") }}') --> a,b
+-- @name filters.format
+function M.filters.format(s, ...)
+ assert(s, 'input to filter "format" was nil instead of a string')
+ return string.format(s, ...)
+end
+
+---
+-- Returns a generator that produces lists of items in table *t* grouped by
+-- string attribute *attribute*.
+-- @param t The table to group items from.
+-- @param attribute The attribute of items in the table to group by. This may
+-- be nested (e.g. "foo.bar" groups by t[i].foo.bar for all i).
+-- @usage expand('{% for age, group in people|groupby("age") %}...{% endfor %}')
+-- @name filters.groupby
+function M.filters.groupby(t, attribute)
+ assert(t, 'input to filter "groupby" was nil instead of a table')
+ local n = #t
+ local seen = {} -- keep track of groupers in order to avoid duplicates
+ return function(t, i)
+ if i > n then return nil end
+ local ta = attr_accessor(t, attribute)
+ -- Determine the next grouper.
+ local grouper = ta[i]
+ while seen[grouper] do
+ i = i + 1
+ if i > n then return nil end
+ grouper = ta[i]
+ end
+ seen[grouper] = true
+ -- Create and return the group.
+ local group = {}
+ for j = i, #t do if ta[j] == grouper then group[#group + 1] = t[j] end end
+ return i + 1, grouper, group
+ end, t, 1
+end
+
+---
+-- Returns a copy of string *s* with all lines after the first indented by
+-- *width* number of spaces.
+-- If boolean *first_line* is `true`, indents the first line as well.
+-- @param s The string to indent lines of.
+-- @param width The number of spaces to indent lines with.
+-- @param first_line Optional flag indicating whether or not to indent the
+-- first line of text. The default value is `false`.
+-- @usage expand('{{ "foo\nbar"|indent(2) }}') --> "foo\n bar"
+-- @name filters.indent
+function M.filters.indent(s, width, first_line)
+ assert(s, 'input to filter "indent" was nil instead of a string')
+ local indent = (' '):rep(width)
+ return (first_line and indent or '')..s:gsub('([\r\n]+)', '%1'..indent)
+end
+
+---
+-- Returns value *value* as an integer.
+-- @param value The value to interpret as an integer.
+-- @usage expand('{{ 32.32|int }}') --> 32
+-- @name filters.int
+function M.filters.int(value)
+ assert(value, 'input to filter "int" was nil instead of a number')
+ return math.floor(tonumber(value) or 0)
+end
+
+---
+-- Returns a string that contains all the elements in table *t* (or all the
+-- attributes named *attribute* in *t*) separated by string *sep*.
+-- @param t The table to join.
+-- @param sep The string to separate table elements with.
+-- @param attribute Optional attribute of elements to use for joining instead
+-- of the elements themselves. This may be nested (e.g. "foo.bar" joins
+-- `t[i].foo.bar` for all i).
+-- @usage expand('{{ {1, 2, 3}|join("|") }}') --> 1|2|3
+-- @name filters.join
+function M.filters.join(t, sep, attribute)
+ assert(t, 'input to filter "join" was nil instead of a table')
+ if not attribute then
+ local strings = {}
+ for i = 1, #t do strings[#strings + 1] = tostring(t[i]) end
+ return table.concat(strings, sep)
+ end
+ local ta = attr_accessor(t, attribute)
+ local attributes = {}
+ for i = 1, #t do attributes[#attributes + 1] = ta[i] end
+ return table.concat(attributes, sep)
+end
+
+---
+-- Returns the last element in table *t*.
+-- @param t The table to get the last element of.
+-- @usage expand('{{ range(10)|last }}') --> 10
+-- @name filters.last
+function M.filters.last(t)
+ assert(t, 'input to filter "last" was nil instead of a table')
+ return t[#t]
+end
+
+---
+-- Returns the length of string or table *value*.
+-- @param value The value to get the length of.
+-- @usage expand('{{ "hello world"|length }}') --> 11
+-- @name filters.length
+function M.filters.length(value)
+ assert(value, 'input to filter "length" was nil instead of a table or string')
+ return #value
+end
+
+---
+-- Returns the list of items produced by generator *generator*, subject to
+-- initial state *s* and initial iterator variable *i*.
+-- This filter should only be used after a filter that returns a generator.
+-- @param generator Generator function that produces an item.
+-- @param s Initial state for the generator.
+-- @param i Initial iterator variable for the generator.
+-- @usage expand('{{ range(4)|batch(2)|list|string }}') --> {{1, 2}, {3, 4}}
+-- @see filters.batch
+-- @see filters.groupby
+-- @see filters.slice
+-- @name filters.list
+function M.filters.list(generator, s, i)
+ assert(type(generator) == 'function',
+ 'input to filter "list" must be a generator')
+ local list = {}
+ for _, v in generator, s, i do list[#list + 1] = v end
+ return list
+end
+
+---
+-- Returns a copy of string *s* with all lowercase characters.
+-- @param s The string to lowercase.
+-- @usage expand('{{ "FOO"|lower }}') --> foo
+-- @name filters.lower
+function M.filters.lower(s)
+ assert(s, 'input to filter "lower" was nil instead of a string')
+ return string.lower(s)
+end
+
+---
+-- Maps each element of table *t* to a value produced by filter name *filter*
+-- and returns the resultant table.
+-- @param t The table of elements to map.
+-- @param filter The name of the filter to pass table elements through.
+-- @param ... Any arguments for the filter.
+-- @usage expand('{{ {"1", "2", "3"}|map("int")|sum }}') --> 6
+-- @name filters.map
+function M.filters.map(t, filter, ...)
+ assert(t, 'input to filter "map" was nil instead of a table')
+ local f = M.filters[filter]
+ assert(f, 'unknown filter "'..filter..'"')
+ local map = {}
+ for i = 1, #t do map[i] = f(t[i], ...) end
+ return map
+end
+
+---
+-- Maps the value of each element's string *attribute* in table *t* to the
+-- value produced by filter name *filter* and returns the resultant table.
+-- @param t The table of elements with attributes to map.
+-- @param attribute The attribute of elements in the table to filter. This may
+-- be nested (e.g. "foo.bar" maps t[i].foo.bar for all i).
+-- @param filter The name of the filter to pass table elements through.
+-- @param ... Any arguments for the filter.
+-- @usage expand('{{ users|mapattr("name")|join("|") }}')
+-- @name filters.mapattr
+function M.filters.mapattr(t, attribute, filter, ...)
+ assert(t, 'input to filter "mapattr" was nil instead of a table')
+ local ta = attr_accessor(t, attribute)
+ local f = M.filters[filter]
+ if filter then
+ assert(f, 'unknown filter "'..filter..'" given to filter "mapattr"')
+ end
+ local map = {}
+ for i = 1, #t do map[i] = filter and f(ta[i], ...) or ta[i] end
+ return map
+end
+
+---
+-- Returns a random element from table *t*.
+-- @param t The table to get a random element from.
+-- @usage expand('{{ range(100)|random }}')
+-- @name filters.random
+function M.filters.random(t)
+ assert(t, 'input to filter "random" was nil instead of a table')
+ math.randomseed(os.time())
+ return t[math.random(#t)]
+end
+
+---
+-- Returns a list of elements in table *t* that fail test name *test*.
+-- @param t The table of elements to reject from.
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ range(5)|reject(is_odd)|join("|") }}') --> 2|4
+-- @name filters.reject
+function M.filters.reject(t, test, ...)
+ assert(t, 'input to filter "reject" was nil instead of a table')
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if not f(t[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a list of elements in table *t* whose string attribute *attribute*
+-- fails test name *test*.
+-- @param t The table of elements to reject from.
+-- @param attribute The attribute of items in the table to reject from. This
+-- may be nested (e.g. "foo.bar" tests t[i].foo.bar for all i).
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ users|rejectattr("offline")|mapattr("name")|join(",") }}')
+-- @name filters.rejectattr
+function M.filters.rejectattr(t, attribute, test, ...)
+ assert(t, 'input to filter "rejectattr" was nil instead of a table')
+ local ta = attr_accessor(t, attribute)
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if not f(ta[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a copy of string *s* with all (or up to *n*) occurrences of string
+-- *old* replaced by string *new*.
+-- Identical to Lua's `string.gsub()` and handles Lua patterns.
+-- @param s The subject string.
+-- @param pattern The string or Lua pattern to replace.
+-- @param repl The replacement text (may contain Lua captures).
+-- @param n Optional number indicating the maximum number of replacements to
+-- make. The default value is `nil`, which is unlimited.
+-- @usage expand('{% filter upper|replace("FOO", "foo") %}foobar
+-- {% endfilter %}') --> fooBAR
+-- @name filters.replace
+function M.filters.replace(s, pattern, repl, n)
+ assert(s, 'input to filter "replace" was nil instead of a string')
+ return string.gsub(s, pattern, repl, n)
+end
+
+---
+-- Returns a copy of the given string or table *value* in reverse order.
+-- @param value The value to reverse.
+-- @usage expand('{{ {1, 2, 3}|reverse|string }}') --> {3, 2, 1}
+-- @name filters.reverse
+function M.filters.reverse(value)
+ assert(type(value) == 'table' or type(value) == 'string',
+ 'input to filter "reverse" was nil instead of a table or string')
+ if type(value) == 'string' then return value:reverse() end
+ local t = {}
+ for i = 1, #value do t[i] = value[#value - i + 1] end
+ return t
+end
+
+---
+-- Returns number *value* rounded to *precision* decimal places based on string
+-- *method* (if given).
+-- @param value The number to round.
+-- @param precision Optional precision to round the number to. The default
+-- value is `0`.
+-- @param method Optional string rounding method, either `"ceil"` or
+-- `"floor"`. The default value is `nil`, which uses the common rounding
+-- method (if a number's fractional part is 0.5 or greater, rounds up;
+-- otherwise rounds down).
+-- @usage expand('{{ 2.1236|round(3, "floor") }}') --> 2.123
+-- @name filters.round
+function M.filters.round(value, precision, method)
+ assert(value, 'input to filter "round" was nil instead of a number')
+ assert(not method or method == 'ceil' or method == 'floor',
+ 'rounding method given to filter "round" must be "ceil" or "floor"')
+ precision = precision or 0
+ method = method or (select(2, math.modf(value)) >= 0.5 and 'ceil' or 'floor')
+ local s = string.format('%.'..(precision >= 0 and precision or 0)..'f',
+ math[method](value * 10^precision) / 10^precision)
+ return tonumber(s)
+end
+
+---
+-- Marks string *s* as HTML-safe, preventing Lupa from modifying it when
+-- configured to autoescape HTML entities.
+-- This filter must be used at the end of a filter chain unless it is
+-- immediately proceeded by the "forceescape" filter.
+-- @param s The string to mark as HTML-safe.
+-- @usage lupa.configure{autoescape = true}
+-- @usage expand('{{ "<div>foo</div>"|safe }}') --> <div>foo</div>
+-- @name filters.safe
+function M.filters.safe(s)
+ assert(s, 'input to filter "safe" was nil instead of a string')
+ return setmetatable({}, {__tostring = function() return s end})
+end
+
+---
+-- Returns a list of the elements in table *t* that pass test name *test*.
+-- @param t The table of elements to select from.
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ range(5)|select(is_odd)|join("|") }}') --> 1|3|5
+-- @name filters.select
+function M.filters.select(t, test, ...)
+ assert(t, 'input to filter "select" was nil instead of a table')
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if f(t[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a list of elements in table *t* whose string attribute *attribute*
+-- passes test name *test*.
+-- @param t The table of elements to select from.
+-- @param attribute The attribute of items in the table to select from. This
+-- may be nested (e.g. "foo.bar" tests t[i].foo.bar for all i).
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ users|selectattr("online")|mapattr("name")|join("|") }}')
+-- @name filters.selectattr
+function M.filters.selectattr(t, attribute, test, ...)
+ assert(t, 'input to filter "selectattr" was nil instead of a table')
+ local ta = attr_accessor(t, attribute)
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if f(ta[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a generator that produces all of the items in table *t* in *slices*
+-- number of iterations, filling any empty spaces with value *fill*.
+-- Combine this with the "list" filter to produce a list.
+-- @param t The table to slice.
+-- @param slices The number of slices to produce.
+-- @param fill The value to use when filling in any empty space in the last
+-- slice.
+-- @usage expand('{% for i in {1, 2, 3}|slice(2, 0) %}{{ i|string }}
+-- {% endfor %}') --> {1, 2} {3, 0}
+-- @see filters.list
+-- @name filters.slice
+function M.filters.slice(t, slices, fill)
+ assert(t, 'input to filter "slice" was nil instead of a table')
+ local size, slices_with_extra = math.floor(#t / slices), #t % slices
+ return function(t, i)
+ if i > slices then return nil end
+ local slice = {}
+ local s = (i - 1) * size + math.min(i, slices_with_extra + 1)
+ local e = i * size + math.min(i, slices_with_extra)
+ for j = s, e do slice[j - s + 1] = t[j] end
+ if slices_with_extra > 0 and i > slices_with_extra and fill then
+ slice[#slice + 1] = fill
+ end
+ return i + 1, slice
+ end, t, 1
+end
+
+---
+-- Returns a copy of table or string *value* in sorted order by value (or by
+-- an attribute named *attribute*), depending on booleans *reverse* and
+-- *case_sensitive*.
+-- @param value The table or string to sort.
+-- @param reverse Optional flag indicating whether or not to sort in reverse
+-- (descending) order. The default value is `false`, which sorts in ascending
+-- order.
+-- @param case_sensitive Optional flag indicating whether or not to consider
+-- case when sorting string values. The default value is `false`.
+-- @param attribute Optional attribute of elements to sort by instead of the
+-- elements themselves.
+-- @usage expand('{{ {2, 3, 1}|sort|string }}') --> {1, 2, 3}
+-- @name filters.sort
+function M.filters.sort(value, reverse, case_sensitive, attribute)
+ assert(value, 'input to filter "sort" was nil instead of a table or string')
+ assert(not attribute or type(attribute) == 'string' or
+ type(attribute) == 'number',
+ 'attribute to filter "sort" must be a string or number')
+ local t = {}
+ local sort_string = type(value) == 'string'
+ if not sort_string then
+ for i = 1, #value do t[#t + 1] = value[i] end
+ else
+ for char in value:gmatch('.') do t[#t + 1] = char end -- chars in string
+ end
+ table.sort(t, function(a, b)
+ if attribute then
+ if type(attribute) == 'number' then
+ a, b = a[attribute], b[attribute]
+ else
+ for k in attribute:gmatch('[^.]+') do a, b = a[k], b[k] end
+ end
+ end
+ if not case_sensitive then
+ if type(a) == 'string' then a = a:lower() end
+ if type(b) == 'string' then b = b:lower() end
+ end
+ if not reverse then
+ return a < b
+ else
+ return a > b
+ end
+ end)
+ return not sort_string and t or table.concat(t)
+end
+
+---
+-- Returns the string representation of value *value*, handling lists properly.
+-- @param value Value to return the string representation of.
+-- @usage expand('{{ {1 * 1, 2 * 2, 3 * 3}|string }}') --> {1, 4, 9}
+-- @name filters.string
+function M.filters.string(value)
+ if type(value) ~= 'table' then return tostring(value) end
+ local t = {}
+ for i = 1, #value do
+ local item = value[i]
+ t[i] = type(item) == 'string' and '"'..item..'"' or M.filters.string(item)
+ end
+ return '{'..table.concat(t, ', ')..'}'
+end
+
+---
+-- Returns a copy of string *s* with any HTML tags stripped.
+-- Also cleans up whitespace.
+-- @param s String to strip HTML tags from.
+-- @usage expand('{{ "<div>foo</div>"|striptags }}') --> foo
+-- @name filters.striptags
+function M.filters.striptags(s)
+ assert(s, 'input to filter "striptags" was nil instead of a string')
+ return s:gsub('%b<>', ''):gsub('%s+', ' '):match('^%s*(.-)%s*$')
+end
+
+---
+-- Returns the numeric sum of the elements in table *t* or the sum of all
+-- attributes named *attribute* in *t*.
+-- @param t The table to calculate the sum of.
+-- @param attribute Optional attribute of elements to use for summing instead
+-- of the elements themselves. This may be nested (e.g. "foo.bar" sums
+-- `t[i].foo.bar` for all i).
+-- @usage expand('{{ range(6)|sum }}') --> 21
+-- @name filters.sum
+function M.filters.sum(t, attribute)
+ assert(t, 'input to filter "sum" was nil instead of a table')
+ local ta = attribute and attr_accessor(t, attribute) or t
+ local sum = 0
+ for i = 1, #t do sum = sum + ta[i] end
+ return sum
+end
+
+---
+-- Returns a copy of all words in string *s* in titlecase.
+-- @param s The string to titlecase.
+-- @usage expand('{{ "foo bar"|title }}') --> Foo Bar
+-- @name filters.title
+function M.filters.title(s)
+ assert(s, 'input to filter "title" was nil instead of a string')
+ return s:gsub('[^-%s]+', M.filters.capitalize)
+end
+
+---
+-- Returns a copy of string *s* truncated to *length* number of characters.
+-- Truncated strings end with '...' or string *delimiter*. If boolean
+-- *partial_words* is `false`, truncation will only happen at word boundaries.
+-- @param s The string to truncate.
+-- @param length The length to truncate the string to.
+-- @param partial_words Optional flag indicating whether or not to allow
+-- truncation within word boundaries. The default value is `false`.
+-- @param delimiter Optional delimiter text. The default value is '...'.
+-- @usage expand('{{ "foo bar"|truncate(4) }}') --> "foo ..."
+-- @name filters.truncate
+function M.filters.truncate(s, length, partial_words, delimiter)
+ assert(s, 'input to filter "truncate" was nil instead of a string')
+ if #s <= length then return s end
+ local truncated = s:sub(1, length)
+ if s:find('[%w_]', length) and not partial_words then
+ truncated = truncated:match('^(.-)[%w_]*$') -- drop partial word
+ end
+ return truncated..(delimiter or '...')
+end
+
+---
+-- Returns a copy of string *s* with all uppercase characters.
+-- @param s The string to uppercase.
+-- @usage expand('{{ "foo"|upper }}') --> FOO
+-- @name filters.upper
+function M.filters.upper(s)
+ assert(s, 'input to filter "upper" was nil instead of a string')
+ return string.upper(s)
+end
+
+---
+-- Returns a string suitably encoded to be used in a URL from value *value*.
+-- *value* may be a string, table of key-value query parameters, or table of
+-- lists of key-value query parameters (for order).
+-- @param value Value to URL-encode.
+-- @usage expand('{{ {{'f', 1}, {'z', 2}}|urlencode }}') --> f=1&z=2
+-- @name filters.urlencode
+function M.filters.urlencode(value)
+ assert(value,
+ 'input to filter "urlencode" was nil instead of a string or table')
+ if type(value) ~= 'table' then
+ return tostring(value):gsub('[^%w.-]', function(c)
+ return string.format('%%%X', string.byte(c))
+ end)
+ end
+ local params = {}
+ if #value > 0 then
+ for i = 1, #value do
+ local k = M.filters.urlencode(value[i][1])
+ local v = M.filters.urlencode(value[i][2])
+ params[#params + 1] = k..'='..v
+ end
+ else
+ for k, v in pairs(value) do
+ params[#params + 1] = M.filters.urlencode(k)..'='..M.filters.urlencode(v)
+ end
+ end
+ return table.concat(params, '&')
+end
+
+---
+-- Replaces any URLs in string *s* with HTML links, limiting link text to
+-- *length* characters.
+-- @param s The string to replace URLs with HTML links in.
+-- @param length Optional maximum number of characters to include in link text.
+-- The default value is `nil`, which imposes no limit.
+-- @param nofollow Optional flag indicating whether or not HTML links will get a
+-- "nofollow" attribute.
+-- @usage expand('{{ "example.com"|urlize }}') -->
+-- <a href="http://example.com">example.com</a>
+-- @name filters.urlize
+function M.filters.urlize(s, length, nofollow)
+ assert(s, 'input to filter "urlize" was nil instead of a string')
+ -- Trims the given url.
+ local function trim_url(url)
+ return length and s:sub(1, length)..(#s > length and '...' or '') or url
+ end
+ local nofollow_attr = nofollow and ' rel="nofollow"' or ''
+ local lead, trail = C((S('(<') + '&lt;')^0), C((S('.,)>\n') + '&gt;')^0) * -1
+ local middle = C((1 - trail)^0)
+ local patt = lpeg.Cs(lead * middle * trail / function(lead, middle, trail)
+ local linked
+ if middle:find('^www%.') or (not middle:find('@') and
+ not middle:find('^https?://') and
+ #middle > 0 and middle:find('^%w') and (
+ middle:find('%.com$') or
+ middle:find('%.net$') or
+ middle:find('%.org$')
+ )) then
+ middle, linked = string.format('<a href="http://%s"%s>%s</a>', middle,
+ nofollow_attr, trim_url(middle)), true
+ end
+ if middle:find('^https?://') then
+ middle, linked = string.format('<a href="%s"%s>%s</a>', middle,
+ nofollow_attr, trim_url(middle)), true
+ end
+ if middle:find('@') and not middle:find('^www%.') and
+ not middle:find(':') and middle:find('^%S+@[%w._-]+%.[%w._-]+$') then
+ middle, linked = string.format('<a href="mailto:%s">%s</a>', middle,
+ middle), true
+ end
+ if linked then return lead..middle..trail end
+ end)
+ return M.filters.escape(s):gsub('%S+', function(word)
+ return lpeg.match(patt, word)
+ end)
+end
+
+---
+-- Returns the number of words in string *s*.
+-- A word is a sequence of non-space characters.
+-- @param s The string to count words in.
+-- @usage expand('{{ "foo bar baz"|wordcount }}') --> 3
+-- @name filters.wordcount
+function M.filters.wordcount(s)
+ assert(s, 'input to filter "wordcount" was nil instead of a string')
+ return select(2, s:gsub('%S+', ''))
+end
+
+---
+-- Interprets table *t* as a list of XML attribute-value pairs, returning them
+-- as a properly formatted, space-separated string.
+-- @param t The table of XML attribute-value pairs.
+-- @usage expand('<data {{ {foo = 42, bar = 23}|xmlattr }} />')
+-- @name filters.xmlattr
+function M.filters.xmlattr(t)
+ assert(t, 'input to filter "xmlattr" was nil instead of a table')
+ local attributes = {}
+ for k, v in pairs(t) do
+ attributes[#attributes + 1] = string.format('%s="%s"', k,
+ M.filters.escape(tostring(v)))
+ end
+ return table.concat(attributes, ' ')
+end
+
+-- Lupa tests.
+
+---
+-- Returns whether or not number *n* is odd.
+-- @param n The number to test.
+-- @usage expand('{% for x in range(10) if is_odd(x) %}...{% endif %}')
+-- @name tests.is_odd
+function M.tests.is_odd(n) return n % 2 == 1 end
+
+---
+-- Returns whether or not number *n* is even.
+-- @param n The number to test.
+-- @usage expand('{% for x in range(10) if is_even(x) %}...{% endif %}')
+-- @name tests.is_even
+function M.tests.is_even(n) return n % 2 == 0 end
+
+---
+-- Returns whether or not number *n* is evenly divisible by number *num*.
+-- @param n The dividend to test.
+-- @param num The divisor to use.
+-- @usage expand('{% if is_divisibleby(x, y) %}...{% endif %}')
+-- @name tests.is_divisibleby
+function M.tests.is_divisibleby(n, num) return n % num == 0 end
+
+---
+-- Returns whether or not value *value* is non-nil, and thus defined.
+-- @param value The value to test.
+-- @usage expand('{% if is_defined(x) %}...{% endif %}')
+-- @name tests.is_defined
+function M.tests.is_defined(value) return value ~= nil end
+
+---
+-- Returns whether or not value *value* is nil, and thus effectively undefined.
+-- @param value The value to test.
+-- @usage expand('{% if is_undefined(x) %}...{% endif %}')
+-- @name tests.is_undefined
+function M.tests.is_undefined(value) return value == nil end
+
+---
+-- Returns whether or not value *value* is nil.
+-- @param value The value to test.
+-- @usage expand('{% if is_none(x) %}...{% endif %}')
+-- @name tests.is_none
+function M.tests.is_none(value) return value == nil end
+
+---
+-- Returns whether or not value *value* is nil.
+-- @param value The value to test.
+-- @usage expand('{% if is_nil(x) %}...{% endif %}')
+-- @name tests.is_nil
+function M.tests.is_nil(value) return value == nil end
+
+---
+-- Returns whether or not string *s* is in all lower-case characters.
+-- @param s The string to test.
+-- @usage expand('{% if is_lower(s) %}...{% endif %}')
+-- @name tests.is_lower
+function M.tests.is_lower(s) return s:lower() == s end
+
+---
+-- Returns whether or not string *s* is in all upper-case characters.
+-- @param s The string to test.
+-- @usage expand('{% if is_upper(s) %}...{% endif %}')
+-- @name tests.is_upper
+function M.tests.is_upper(s) return s:upper() == s end
+
+---
+-- Returns whether or not value *value* is a string.
+-- @param value The value to test.
+-- @usage expand('{% if is_string(x) %}...{% endif %}')
+-- @name tests.is_string
+function M.tests.is_string(value) return type(value) == 'string' end
+
+---
+-- Returns whether or not value *value* is a table.
+-- @param value The value to test.
+-- @usage expand('{% if is_mapping(x) %}...{% endif %}')
+-- @name tests.is_mapping
+function M.tests.is_mapping(value) return type(value) == 'table' end
+
+---
+-- Returns whether or not value *value* is a table.
+-- @param value The value to test.
+-- @usage expand('{% if is_table(x) %}...{% endif %}')
+-- @name tests.is_table
+function M.tests.is_table(value) return type(value) == 'table' end
+
+---
+-- Returns whether or not value *value* is a number.
+-- @param value The value to test.
+-- @usage expand('{% if is_number(x) %}...{% endif %}')
+-- @name tests.is_number
+function M.tests.is_number(value) return type(value) == 'number' end
+
+---
+-- Returns whether or not value *value* is a sequence, namely a table with
+-- non-zero length.
+-- @param value The value to test.
+-- @usage expand('{% if is_sequence(x) %}...{% endif %}')
+-- @name tests.is_sequence
+function M.tests.is_sequence(value)
+ return type(value) == 'table' and #value > 0
+end
+
+---
+-- Returns whether or not value *value* is a sequence (a table with non-zero
+-- length) or a generator.
+-- At the moment, all functions are considered generators.
+-- @param value The value to test.
+-- @usage expand('{% if is_iterable(x) %}...{% endif %}')
+-- @name tests.is_iterable
+function M.tests.is_iterable(value)
+ return M.tests.is_sequence(value) or type(value) == 'function'
+end
+
+---
+-- Returns whether or not value *value* is a function.
+-- @param value The value to test.
+-- @usage expand('{% if is_callable(x) %}...{% endif %}')
+-- @name tests.is_callable
+function M.tests.is_callable(value) return type(value) == 'function' end
+
+---
+-- Returns whether or not value *value* is the same as value *other*.
+-- @param value The value to test.
+-- @param other The value to compare with.
+-- @usage expand('{% if is_sameas(x, y) %}...{% endif %}')
+-- @name tests.is_sameas
+function M.tests.is_sameas(value, other) return value == other end
+
+---
+-- Returns whether or not value *value* is HTML-safe.
+-- @param value The value to test.
+-- @usage expand('{% if is_escaped(x) %}...{% endif %}')
+-- @name tests.is_escaped
+function M.tests.is_escaped(value)
+ return getmetatable(value) and getmetatable(value).__tostring ~= nil
+end
+
+return M
diff --git a/contrib/lua-tableshape/LICENSE b/contrib/lua-tableshape/LICENSE
new file mode 100644
index 0000000..38ba5fc
--- /dev/null
+++ b/contrib/lua-tableshape/LICENSE
@@ -0,0 +1,19 @@
+Copyright (C) 2018 by Leaf Corcoran
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/lua-tableshape/tableshape.lua b/contrib/lua-tableshape/tableshape.lua
new file mode 100644
index 0000000..ccc7355
--- /dev/null
+++ b/contrib/lua-tableshape/tableshape.lua
@@ -0,0 +1,2324 @@
+local OptionalType, TaggedType, types, is_type
+local BaseType, TransformNode, SequenceNode, FirstOfNode, DescribeNode, NotType, Literal
+local FailedTransform = { }
+local unpack = unpack or table.unpack
+local clone_state
+clone_state = function(state_obj)
+ if type(state_obj) ~= "table" then
+ return { }
+ end
+ local out
+ do
+ local _tbl_0 = { }
+ for k, v in pairs(state_obj) do
+ _tbl_0[k] = v
+ end
+ out = _tbl_0
+ end
+ do
+ local mt = getmetatable(state_obj)
+ if mt then
+ setmetatable(out, mt)
+ end
+ end
+ return out
+end
+local describe_type
+describe_type = function(val)
+ if type(val) == "string" then
+ if not val:match('"') then
+ return "\"" .. tostring(val) .. "\""
+ elseif not val:match("'") then
+ return "'" .. tostring(val) .. "'"
+ else
+ return "`" .. tostring(val) .. "`"
+ end
+ elseif BaseType:is_base_type(val) then
+ return val:_describe()
+ else
+ return tostring(val)
+ end
+end
+local coerce_literal
+coerce_literal = function(value)
+ local _exp_0 = type(value)
+ if "string" == _exp_0 or "number" == _exp_0 or "boolean" == _exp_0 then
+ return Literal(value)
+ elseif "table" == _exp_0 then
+ if BaseType:is_base_type(value) then
+ return value
+ end
+ end
+ return nil, "failed to coerce literal into type, use types.literal() to test for literal value"
+end
+local join_names
+join_names = function(items, sep, last_sep)
+ if sep == nil then
+ sep = ", "
+ end
+ local count = #items
+ local chunks = { }
+ for idx, name in ipairs(items) do
+ if idx > 1 then
+ local current_sep
+ if idx == count then
+ current_sep = last_sep or sep
+ else
+ current_sep = sep
+ end
+ table.insert(chunks, current_sep)
+ end
+ table.insert(chunks, name)
+ end
+ return table.concat(chunks)
+end
+do
+ local _class_0
+ local _base_0 = {
+ __div = function(self, fn)
+ return TransformNode(self, fn)
+ end,
+ __mod = function(self, fn)
+ do
+ local _with_0 = TransformNode(self, fn)
+ _with_0.with_state = true
+ return _with_0
+ end
+ end,
+ __mul = function(_left, _right)
+ local left, err = coerce_literal(_left)
+ if not (left) then
+ error("left hand side of multiplication: " .. tostring(_left) .. ": " .. tostring(err))
+ end
+ local right
+ right, err = coerce_literal(_right)
+ if not (right) then
+ error("right hand side of multiplication: " .. tostring(_right) .. ": " .. tostring(err))
+ end
+ return SequenceNode(left, right)
+ end,
+ __add = function(_left, _right)
+ local left, err = coerce_literal(_left)
+ if not (left) then
+ error("left hand side of addition: " .. tostring(_left) .. ": " .. tostring(err))
+ end
+ local right
+ right, err = coerce_literal(_right)
+ if not (right) then
+ error("right hand side of addition: " .. tostring(_right) .. ": " .. tostring(err))
+ end
+ if left.__class == FirstOfNode then
+ local options = {
+ unpack(left.options)
+ }
+ table.insert(options, right)
+ return FirstOfNode(unpack(options))
+ elseif right.__class == FirstOfNode then
+ return FirstOfNode(left, unpack(right.options))
+ else
+ return FirstOfNode(left, right)
+ end
+ end,
+ __unm = function(self, right)
+ return NotType(right)
+ end,
+ __tostring = function(self)
+ return self:_describe()
+ end,
+ _describe = function(self)
+ return error("Node missing _describe: " .. tostring(self.__class.__name))
+ end,
+ check_value = function(self, ...)
+ local value, state_or_err = self:_transform(...)
+ if value == FailedTransform then
+ return nil, state_or_err
+ end
+ if type(state_or_err) == "table" then
+ return state_or_err
+ else
+ return true
+ end
+ end,
+ transform = function(self, ...)
+ local value, state_or_err = self:_transform(...)
+ if value == FailedTransform then
+ return nil, state_or_err
+ end
+ if type(state_or_err) == "table" then
+ return value, state_or_err
+ else
+ return value
+ end
+ end,
+ repair = function(self, ...)
+ return self:transform(...)
+ end,
+ on_repair = function(self, fn)
+ return (self + types.any / fn * self):describe(function()
+ return self:_describe()
+ end)
+ end,
+ is_optional = function(self)
+ return OptionalType(self)
+ end,
+ describe = function(self, ...)
+ return DescribeNode(self, ...)
+ end,
+ tag = function(self, name)
+ return TaggedType(self, {
+ tag = name
+ })
+ end,
+ clone_opts = function(self)
+ return error("clone_opts is not longer supported")
+ end,
+ __call = function(self, ...)
+ return self:check_value(...)
+ end
+ }
+ _base_0.__index = _base_0
+ _class_0 = setmetatable({
+ __init = function(self, opts) end,
+ __base = _base_0,
+ __name = "BaseType"
+ }, {
+ __index = _base_0,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.is_base_type = function(self, val)
+ do
+ local mt = type(val) == "table" and getmetatable(val)
+ if mt then
+ if mt.__class then
+ return mt.__class.is_base_type == BaseType.is_base_type
+ end
+ end
+ end
+ return false
+ end
+ self.__inherited = function(self, cls)
+ cls.__base.__call = cls.__call
+ cls.__base.__div = self.__div
+ cls.__base.__mod = self.__mod
+ cls.__base.__mul = self.__mul
+ cls.__base.__add = self.__add
+ cls.__base.__unm = self.__unm
+ cls.__base.__tostring = self.__tostring
+ end
+ BaseType = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return self.node:_describe()
+ end,
+ _transform = function(self, value, state)
+ local state_or_err
+ value, state_or_err = self.node:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state_or_err
+ else
+ local out
+ local _exp_0 = type(self.t_fn)
+ if "function" == _exp_0 then
+ if self.with_state then
+ out = self.t_fn(value, state_or_err)
+ else
+ out = self.t_fn(value)
+ end
+ else
+ out = self.t_fn
+ end
+ return out, state_or_err
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, node, t_fn)
+ self.node, self.t_fn = node, t_fn
+ return assert(self.node, "missing node for transform")
+ end,
+ __base = _base_0,
+ __name = "TransformNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ TransformNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.sequence
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ _accum_0[_len_0] = describe_type(i)
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return join_names(item_names, " then ")
+ end,
+ _transform = function(self, value, state)
+ local _list_0 = self.sequence
+ for _index_0 = 1, #_list_0 do
+ local node = _list_0[_index_0]
+ value, state = node:_transform(value, state)
+ if value == FailedTransform then
+ break
+ end
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ self.sequence = {
+ ...
+ }
+ end,
+ __base = _base_0,
+ __name = "SequenceNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ SequenceNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ _accum_0[_len_0] = describe_type(i)
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return join_names(item_names, ", ", ", or ")
+ end,
+ _transform = function(self, value, state)
+ if not (self.options[1]) then
+ return FailedTransform, "no options for node"
+ end
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local node = _list_0[_index_0]
+ local new_val, new_state = node:_transform(value, state)
+ if not (new_val == FailedTransform) then
+ return new_val, new_state
+ end
+ end
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ self.options = {
+ ...
+ }
+ end,
+ __base = _base_0,
+ __name = "FirstOfNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ FirstOfNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, input, ...)
+ local value, state = self.node:_transform(input, ...)
+ if value == FailedTransform then
+ local err
+ if self.err_handler then
+ err = self.err_handler(input, state)
+ else
+ err = "expected " .. tostring(self:_describe())
+ end
+ return FailedTransform, err
+ end
+ return value, state
+ end,
+ describe = function(self, ...)
+ return DescribeNode(self.node, ...)
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, node, describe)
+ self.node = node
+ local err_message
+ if type(describe) == "table" then
+ describe, err_message = describe.type, describe.error
+ end
+ if type(describe) == "string" then
+ self._describe = function()
+ return describe
+ end
+ else
+ self._describe = describe
+ end
+ if err_message then
+ if type(err_message) == "string" then
+ self.err_handler = function()
+ return err_message
+ end
+ else
+ self.err_handler = err_message
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "DescribeNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ DescribeNode = _class_0
+end
+local AnnotateNode
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ format_error = function(self, value, err)
+ return tostring(tostring(value)) .. ": " .. tostring(err)
+ end,
+ _transform = function(self, value, state)
+ local new_value, state_or_err = self.base_type:_transform(value, state)
+ if new_value == FailedTransform then
+ return FailedTransform, self:format_error(value, state_or_err)
+ else
+ return new_value, state_or_err
+ end
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ return self.base_type:_describe()
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type, opts)
+ self.base_type = assert(coerce_literal(base_type))
+ if opts then
+ if opts.format_error then
+ self.format_error = assert(types.func:transform(opts.format_error))
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "AnnotateNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AnnotateNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ update_state = function(self, state, value, ...)
+ local out = clone_state(state)
+ if self.tag_type == "function" then
+ if select("#", ...) > 0 then
+ self.tag_name(out, ..., value)
+ else
+ self.tag_name(out, value)
+ end
+ else
+ if self.tag_array then
+ local existing = out[self.tag_name]
+ if type(existing) == "table" then
+ local copy
+ do
+ local _tbl_0 = { }
+ for k, v in pairs(existing) do
+ _tbl_0[k] = v
+ end
+ copy = _tbl_0
+ end
+ table.insert(copy, value)
+ out[self.tag_name] = copy
+ else
+ out[self.tag_name] = {
+ value
+ }
+ end
+ else
+ out[self.tag_name] = value
+ end
+ end
+ return out
+ end,
+ _transform = function(self, value, state)
+ value, state = self.base_type:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state
+ end
+ state = self:update_state(state, value)
+ return value, state
+ end,
+ _describe = function(self)
+ local base_description = self.base_type:_describe()
+ return tostring(base_description) .. " tagged " .. tostring(describe_type(self.tag_name))
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type, opts)
+ if opts == nil then
+ opts = { }
+ end
+ self.base_type = base_type
+ self.tag_name = assert(opts.tag, "tagged type missing tag")
+ self.tag_type = type(self.tag_name)
+ if self.tag_type == "string" then
+ if self.tag_name:match("%[%]$") then
+ self.tag_name = self.tag_name:sub(1, -3)
+ self.tag_array = true
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "TaggedType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ TaggedType = _class_0
+end
+local TagScopeType
+do
+ local _class_0
+ local _parent_0 = TaggedType
+ local _base_0 = {
+ create_scope_state = function(self, state)
+ return nil
+ end,
+ _transform = function(self, value, state)
+ local scope
+ value, scope = self.base_type:_transform(value, self:create_scope_state(state))
+ if value == FailedTransform then
+ return FailedTransform, scope
+ end
+ if self.tag_name then
+ state = self:update_state(state, scope, value)
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type, opts)
+ if opts then
+ return _class_0.__parent.__init(self, base_type, opts)
+ else
+ self.base_type = base_type
+ end
+ end,
+ __base = _base_0,
+ __name = "TagScopeType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ TagScopeType = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ if value == nil then
+ return value, state
+ end
+ return self.base_type:_transform(value, state)
+ end,
+ is_optional = function(self)
+ return self
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ local base_description = self.base_type:_describe()
+ return "optional " .. tostring(base_description)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type)
+ self.base_type = base_type
+ return assert(BaseType:is_base_type(self.base_type), "expected a type checker")
+ end,
+ __base = _base_0,
+ __name = "OptionalType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ OptionalType = _class_0
+end
+local AnyType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, v, state)
+ return v, state
+ end,
+ _describe = function(self)
+ return "anything"
+ end,
+ is_optional = function(self)
+ return self
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "AnyType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AnyType = _class_0
+end
+local Type
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local got = type(value)
+ if self.t ~= got then
+ return FailedTransform, "expected type " .. tostring(describe_type(self.t)) .. ", got " .. tostring(describe_type(got))
+ end
+ if self.length_type then
+ local len = #value
+ local res
+ res, state = self.length_type:_transform(len, state)
+ if res == FailedTransform then
+ return FailedTransform, tostring(self.t) .. " length " .. tostring(state) .. ", got " .. tostring(len)
+ end
+ end
+ return value, state
+ end,
+ length = function(self, left, right)
+ local l
+ if BaseType:is_base_type(left) then
+ l = left
+ else
+ l = types.range(left, right)
+ end
+ return Type(self.t, {
+ length = l
+ })
+ end,
+ _describe = function(self)
+ local t = "type " .. tostring(describe_type(self.t))
+ if self.length_type then
+ t = t .. " length_type " .. tostring(self.length_type:_describe())
+ end
+ return t
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, t, opts)
+ self.t = t
+ if opts then
+ if opts.length then
+ self.length_type = assert(coerce_literal(opts.length))
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "Type",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Type = _class_0
+end
+local ArrayType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "an array"
+ end,
+ _transform = function(self, value, state)
+ if not (type(value) == "table") then
+ return FailedTransform, "expecting table"
+ end
+ local k = 1
+ for i, v in pairs(value) do
+ if not (type(i) == "number") then
+ return FailedTransform, "non number field: " .. tostring(i)
+ end
+ if not (i == k) then
+ return FailedTransform, "non array index, got " .. tostring(describe_type(i)) .. " but expected " .. tostring(describe_type(k))
+ end
+ k = k + 1
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "ArrayType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ ArrayType = _class_0
+end
+local OneOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ if type(i) == "table" and i._describe then
+ _accum_0[_len_0] = i:_describe()
+ else
+ _accum_0[_len_0] = describe_type(i)
+ end
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return tostring(join_names(item_names, ", ", ", or "))
+ end,
+ _transform = function(self, value, state)
+ if self.options_hash then
+ if self.options_hash[value] then
+ return value, state
+ end
+ else
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local _continue_0 = false
+ repeat
+ local item = _list_0[_index_0]
+ if item == value then
+ return value, state
+ end
+ if BaseType:is_base_type(item) then
+ local new_value, new_state = item:_transform(value, state)
+ if new_value == FailedTransform then
+ _continue_0 = true
+ break
+ end
+ return new_value, new_state
+ end
+ _continue_0 = true
+ until true
+ if not _continue_0 then
+ break
+ end
+ end
+ end
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, options)
+ self.options = options
+ assert(type(self.options) == "table", "expected table for options in one_of")
+ local fast_opts = types.array_of(types.number + types.string)
+ if fast_opts(self.options) then
+ do
+ local _tbl_0 = { }
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local v = _list_0[_index_0]
+ _tbl_0[v] = true
+ end
+ self.options_hash = _tbl_0
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "OneOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ OneOf = _class_0
+end
+local AllOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.types
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ _accum_0[_len_0] = describe_type(i)
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return join_names(item_names, " and ")
+ end,
+ _transform = function(self, value, state)
+ local _list_0 = self.types
+ for _index_0 = 1, #_list_0 do
+ local t = _list_0[_index_0]
+ value, state = t:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state
+ end
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, types)
+ self.types = types
+ assert(type(self.types) == "table", "expected table for first argument")
+ local _list_0 = self.types
+ for _index_0 = 1, #_list_0 do
+ local checker = _list_0[_index_0]
+ assert(BaseType:is_base_type(checker), "all_of expects all type checkers")
+ end
+ end,
+ __base = _base_0,
+ __name = "AllOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AllOf = _class_0
+end
+local ArrayOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "array of " .. tostring(describe_type(self.expected))
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ if self.length_type then
+ local len = #value
+ local res
+ res, state = self.length_type:_transform(len, state)
+ if res == FailedTransform then
+ return FailedTransform, "array length " .. tostring(state) .. ", got " .. tostring(len)
+ end
+ end
+ local is_literal = not BaseType:is_base_type(self.expected)
+ local copy, k
+ for idx, item in ipairs(value) do
+ local skip_item = false
+ local transformed_item
+ if is_literal then
+ if self.expected ~= item then
+ return FailedTransform, "array item " .. tostring(idx) .. ": expected " .. tostring(describe_type(self.expected))
+ else
+ transformed_item = item
+ end
+ else
+ local item_val
+ item_val, state = self.expected:_transform(item, state)
+ if item_val == FailedTransform then
+ return FailedTransform, "array item " .. tostring(idx) .. ": " .. tostring(state)
+ end
+ if item_val == nil and not self.keep_nils then
+ skip_item = true
+ else
+ transformed_item = item_val
+ end
+ end
+ if transformed_item ~= item or skip_item then
+ if not (copy) then
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _max_0 = idx - 1
+ for _index_0 = 1, _max_0 < 0 and #value + _max_0 or _max_0 do
+ local i = value[_index_0]
+ _accum_0[_len_0] = i
+ _len_0 = _len_0 + 1
+ end
+ copy = _accum_0
+ end
+ k = idx
+ end
+ end
+ if copy and not skip_item then
+ copy[k] = transformed_item
+ k = k + 1
+ end
+ end
+ return copy or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, expected, opts)
+ self.expected = expected
+ if opts then
+ self.keep_nils = opts.keep_nils and true
+ if opts.length then
+ self.length_type = assert(coerce_literal(opts.length))
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "ArrayOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.type_err_message = "expecting table"
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ ArrayOf = _class_0
+end
+local ArrayContains
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ short_circuit = true,
+ keep_nils = false,
+ _describe = function(self)
+ return "array containing " .. tostring(describe_type(self.contains))
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ local is_literal = not BaseType:is_base_type(self.contains)
+ local contains = false
+ local copy, k
+ for idx, item in ipairs(value) do
+ local skip_item = false
+ local transformed_item
+ if is_literal then
+ if self.contains == item then
+ contains = true
+ end
+ transformed_item = item
+ else
+ local item_val, new_state = self.contains:_transform(item, state)
+ if item_val == FailedTransform then
+ transformed_item = item
+ else
+ state = new_state
+ contains = true
+ if item_val == nil and not self.keep_nils then
+ skip_item = true
+ else
+ transformed_item = item_val
+ end
+ end
+ end
+ if transformed_item ~= item or skip_item then
+ if not (copy) then
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _max_0 = idx - 1
+ for _index_0 = 1, _max_0 < 0 and #value + _max_0 or _max_0 do
+ local i = value[_index_0]
+ _accum_0[_len_0] = i
+ _len_0 = _len_0 + 1
+ end
+ copy = _accum_0
+ end
+ k = idx
+ end
+ end
+ if copy and not skip_item then
+ copy[k] = transformed_item
+ k = k + 1
+ end
+ if contains and self.short_circuit then
+ if copy then
+ for kdx = idx + 1, #value do
+ copy[k] = value[kdx]
+ k = k + 1
+ end
+ end
+ break
+ end
+ end
+ if not (contains) then
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ return copy or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, contains, opts)
+ self.contains = contains
+ assert(self.contains, "missing contains")
+ if opts then
+ self.short_circuit = opts.short_circuit and true
+ self.keep_nils = opts.keep_nils and true
+ end
+ end,
+ __base = _base_0,
+ __name = "ArrayContains",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.type_err_message = "expecting table"
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ ArrayContains = _class_0
+end
+local MapOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "map of " .. tostring(self.expected_key:_describe()) .. " -> " .. tostring(self.expected_value:_describe())
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ local key_literal = not BaseType:is_base_type(self.expected_key)
+ local value_literal = not BaseType:is_base_type(self.expected_value)
+ local transformed = false
+ local out = { }
+ for k, v in pairs(value) do
+ local _continue_0 = false
+ repeat
+ local new_k = k
+ local new_v = v
+ if key_literal then
+ if k ~= self.expected_key then
+ return FailedTransform, "map key expected " .. tostring(describe_type(self.expected_key))
+ end
+ else
+ new_k, state = self.expected_key:_transform(k, state)
+ if new_k == FailedTransform then
+ return FailedTransform, "map key " .. tostring(state)
+ end
+ end
+ if value_literal then
+ if v ~= self.expected_value then
+ return FailedTransform, "map value expected " .. tostring(describe_type(self.expected_value))
+ end
+ else
+ new_v, state = self.expected_value:_transform(v, state)
+ if new_v == FailedTransform then
+ return FailedTransform, "map value " .. tostring(state)
+ end
+ end
+ if new_k ~= k or new_v ~= v then
+ transformed = true
+ end
+ if new_k == nil then
+ _continue_0 = true
+ break
+ end
+ out[new_k] = new_v
+ _continue_0 = true
+ until true
+ if not _continue_0 then
+ break
+ end
+ end
+ return transformed and out or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, expected_key, expected_value)
+ self.expected_key = coerce_literal(expected_key)
+ self.expected_value = coerce_literal(expected_value)
+ end,
+ __base = _base_0,
+ __name = "MapOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ MapOf = _class_0
+end
+local Shape
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ open = false,
+ check_all = false,
+ is_open = function(self)
+ return Shape(self.shape, {
+ open = true,
+ check_all = self.check_all or nil
+ })
+ end,
+ _describe = function(self)
+ local parts
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ for k, v in pairs(self.shape) do
+ _accum_0[_len_0] = tostring(describe_type(k)) .. " = " .. tostring(describe_type(v))
+ _len_0 = _len_0 + 1
+ end
+ parts = _accum_0
+ end
+ return "{ " .. tostring(table.concat(parts, ", ")) .. " }"
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ local check_all = self.check_all
+ local remaining_keys
+ do
+ local _tbl_0 = { }
+ for key in pairs(value) do
+ _tbl_0[key] = true
+ end
+ remaining_keys = _tbl_0
+ end
+ local errors
+ local dirty = false
+ local out = { }
+ for shape_key, shape_val in pairs(self.shape) do
+ local item_value = value[shape_key]
+ if remaining_keys then
+ remaining_keys[shape_key] = nil
+ end
+ local new_val
+ if BaseType:is_base_type(shape_val) then
+ new_val, state = shape_val:_transform(item_value, state)
+ else
+ if shape_val == item_value then
+ new_val, state = item_value, state
+ else
+ new_val, state = FailedTransform, "expected " .. tostring(describe_type(shape_val))
+ end
+ end
+ if new_val == FailedTransform then
+ err = "field " .. tostring(describe_type(shape_key)) .. ": " .. tostring(state)
+ if check_all then
+ if errors then
+ table.insert(errors, err)
+ else
+ errors = {
+ err
+ }
+ end
+ else
+ return FailedTransform, err
+ end
+ else
+ if new_val ~= item_value then
+ dirty = true
+ end
+ out[shape_key] = new_val
+ end
+ end
+ if remaining_keys and next(remaining_keys) then
+ if self.open then
+ for k in pairs(remaining_keys) do
+ out[k] = value[k]
+ end
+ elseif self.extra_fields_type then
+ for k in pairs(remaining_keys) do
+ local item_value = value[k]
+ local tuple
+ tuple, state = self.extra_fields_type:_transform({
+ [k] = item_value
+ }, state)
+ if tuple == FailedTransform then
+ err = "field " .. tostring(describe_type(k)) .. ": " .. tostring(state)
+ if check_all then
+ if errors then
+ table.insert(errors, err)
+ else
+ errors = {
+ err
+ }
+ end
+ else
+ return FailedTransform, err
+ end
+ else
+ do
+ local nk = tuple and next(tuple)
+ if nk then
+ if nk ~= k then
+ dirty = true
+ elseif tuple[nk] ~= item_value then
+ dirty = true
+ end
+ out[nk] = tuple[nk]
+ else
+ dirty = true
+ end
+ end
+ end
+ end
+ else
+ local names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ for key in pairs(remaining_keys) do
+ _accum_0[_len_0] = describe_type(key)
+ _len_0 = _len_0 + 1
+ end
+ names = _accum_0
+ end
+ err = "extra fields: " .. tostring(table.concat(names, ", "))
+ if check_all then
+ if errors then
+ table.insert(errors, err)
+ else
+ errors = {
+ err
+ }
+ end
+ else
+ return FailedTransform, err
+ end
+ end
+ end
+ if errors and next(errors) then
+ return FailedTransform, table.concat(errors, "; ")
+ end
+ return dirty and out or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, shape, opts)
+ self.shape = shape
+ assert(type(self.shape) == "table", "expected table for shape")
+ if opts then
+ if opts.extra_fields then
+ assert(BaseType:is_base_type(opts.extra_fields), "extra_fields_type must be type checker")
+ self.extra_fields_type = opts.extra_fields
+ end
+ self.open = opts.open and true
+ self.check_all = opts.check_all and true
+ if self.open then
+ assert(not self.extra_fields_type, "open can not be combined with extra_fields")
+ end
+ if self.extra_fields_type then
+ return assert(not self.open, "extra_fields can not be combined with open")
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "Shape",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.type_err_message = "expecting table"
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Shape = _class_0
+end
+local Partial
+do
+ local _class_0
+ local _parent_0 = Shape
+ local _base_0 = {
+ open = true,
+ is_open = function(self)
+ return error("is_open has no effect on Partial")
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "Partial",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Partial = _class_0
+end
+local Pattern
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "pattern " .. tostring(describe_type(self.pattern))
+ end,
+ _transform = function(self, value, state)
+ local test_value
+ if self.coerce then
+ if BaseType:is_base_type(self.coerce) then
+ local c_res, err = self.coerce:_transform(value)
+ if c_res == FailedTransform then
+ return FailedTransform, err
+ end
+ test_value = c_res
+ else
+ test_value = tostring(value)
+ end
+ else
+ test_value = value
+ end
+ local t_res, err = types.string(test_value)
+ if not (t_res) then
+ return FailedTransform, err
+ end
+ if test_value:match(self.pattern) then
+ return value, state
+ else
+ return FailedTransform, "doesn't match " .. tostring(self:_describe())
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, pattern, opts)
+ self.pattern = pattern
+ assert(type(self.pattern) == "string", "Pattern must be a string")
+ if opts then
+ self.coerce = opts.coerce
+ return assert(opts.initial_type == nil, "initial_type has been removed from types.pattern (got: " .. tostring(opts.initial_type) .. ")")
+ end
+ end,
+ __base = _base_0,
+ __name = "Pattern",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Pattern = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return describe_type(self.value)
+ end,
+ _transform = function(self, value, state)
+ if self.value ~= value then
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, value)
+ self.value = value
+ end,
+ __base = _base_0,
+ __name = "Literal",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Literal = _class_0
+end
+local Custom
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "custom checker " .. tostring(self.fn)
+ end,
+ _transform = function(self, value, state)
+ local pass, err = self.fn(value, state)
+ if not (pass) then
+ return FailedTransform, err or "failed custom check"
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, fn)
+ self.fn = fn
+ return assert(type(self.fn) == "function", "custom checker must be a function")
+ end,
+ __base = _base_0,
+ __name = "Custom",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Custom = _class_0
+end
+local Equivalent
+do
+ local _class_0
+ local values_equivalent
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "equivalent to " .. tostring(describe_type(self.val))
+ end,
+ _transform = function(self, value, state)
+ if values_equivalent(self.val, value) then
+ return value, state
+ else
+ return FailedTransform, "not equivalent to " .. tostring(self.val)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, val)
+ self.val = val
+ end,
+ __base = _base_0,
+ __name = "Equivalent",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ values_equivalent = function(a, b)
+ if a == b then
+ return true
+ end
+ if type(a) == "table" and type(b) == "table" then
+ local seen_keys = { }
+ for k, v in pairs(a) do
+ seen_keys[k] = true
+ if not (values_equivalent(v, b[k])) then
+ return false
+ end
+ end
+ for k, v in pairs(b) do
+ local _continue_0 = false
+ repeat
+ if seen_keys[k] then
+ _continue_0 = true
+ break
+ end
+ if not (values_equivalent(v, a[k])) then
+ return false
+ end
+ _continue_0 = true
+ until true
+ if not _continue_0 then
+ break
+ end
+ end
+ return true
+ else
+ return false
+ end
+ end
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Equivalent = _class_0
+end
+local Range
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local res
+ res, state = self.value_type:_transform(value, state)
+ if res == FailedTransform then
+ return FailedTransform, "range " .. tostring(state)
+ end
+ if value < self.left then
+ return FailedTransform, "not in " .. tostring(self:_describe())
+ end
+ if value > self.right then
+ return FailedTransform, "not in " .. tostring(self:_describe())
+ end
+ return value, state
+ end,
+ _describe = function(self)
+ return "range from " .. tostring(self.left) .. " to " .. tostring(self.right)
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, left, right)
+ self.left, self.right = left, right
+ assert(self.left <= self.right, "left range value should be less than right range value")
+ self.value_type = assert(types[type(self.left)], "couldn't figure out type of range boundary")
+ end,
+ __base = _base_0,
+ __name = "Range",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Range = _class_0
+end
+local Proxy
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, ...)
+ return assert(self.fn(), "proxy missing transformer"):_transform(...)
+ end,
+ _describe = function(self, ...)
+ return assert(self.fn(), "proxy missing transformer"):_describe(...)
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, fn)
+ self.fn = fn
+ end,
+ __base = _base_0,
+ __name = "Proxy",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Proxy = _class_0
+end
+local AssertType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ assert = assert,
+ _transform = function(self, value, state)
+ local state_or_err
+ value, state_or_err = self.base_type:_transform(value, state)
+ self.assert(value ~= FailedTransform, state_or_err)
+ return value, state_or_err
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ local base_description = self.base_type:_describe()
+ return "assert " .. tostring(base_description)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type)
+ self.base_type = base_type
+ return assert(BaseType:is_base_type(self.base_type), "expected a type checker")
+ end,
+ __base = _base_0,
+ __name = "AssertType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AssertType = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local out, _ = self.base_type:_transform(value, state)
+ if out == FailedTransform then
+ return value, state
+ else
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ local base_description = self.base_type:_describe()
+ return "not " .. tostring(base_description)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type)
+ self.base_type = base_type
+ return assert(BaseType:is_base_type(self.base_type), "expected a type checker")
+ end,
+ __base = _base_0,
+ __name = "NotType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ NotType = _class_0
+end
+local CloneType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local _exp_0 = type(value)
+ if "nil" == _exp_0 or "string" == _exp_0 or "number" == _exp_0 or "boolean" == _exp_0 then
+ return value, state
+ elseif "table" == _exp_0 then
+ local clone_value
+ do
+ local _tbl_0 = { }
+ for k, v in pairs(value) do
+ _tbl_0[k] = v
+ end
+ clone_value = _tbl_0
+ end
+ do
+ local mt = getmetatable(value)
+ if mt then
+ setmetatable(clone_value, mt)
+ end
+ end
+ return clone_value, state
+ else
+ return FailedTransform, tostring(describe_type(value)) .. " is not cloneable"
+ end
+ end,
+ _describe = function(self)
+ return "cloneable value"
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "CloneType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ CloneType = _class_0
+end
+local MetatableIsType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ allow_metatable_update = false,
+ _transform = function(self, value, state)
+ local state_or_err
+ value, state_or_err = types.table:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state_or_err
+ end
+ local mt = getmetatable(value)
+ local new_mt
+ new_mt, state_or_err = self.metatable_type:_transform(mt, state_or_err)
+ if new_mt == FailedTransform then
+ return FailedTransform, "metatable expected: " .. tostring(state_or_err)
+ end
+ if new_mt ~= mt then
+ if self.allow_metatable_update then
+ setmetatable(value, new_mt)
+ else
+ return FailedTransform, "metatable was modified by a type but { allow_metatable_update = true } is not enabled"
+ end
+ end
+ return value, state_or_err
+ end,
+ _describe = function(self)
+ return "has metatable " .. tostring(describe_type(self.metatable_type))
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, metatable_type, opts)
+ if BaseType:is_base_type(metatable_type) then
+ self.metatable_type = metatable_type
+ else
+ self.metatable_type = Literal(metatable_type)
+ end
+ if opts then
+ self.allow_metatable_update = opts.allow_metatable_update and true
+ end
+ end,
+ __base = _base_0,
+ __name = "MetatableIsType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ MetatableIsType = _class_0
+end
+local type_nil = Type("nil")
+local type_function = Type("function")
+local type_number = Type("number")
+types = setmetatable({
+ any = AnyType(),
+ string = Type("string"),
+ number = type_number,
+ ["function"] = type_function,
+ func = type_function,
+ boolean = Type("boolean"),
+ userdata = Type("userdata"),
+ ["nil"] = type_nil,
+ null = type_nil,
+ table = Type("table"),
+ array = ArrayType(),
+ clone = CloneType(),
+ integer = Pattern("^%d+$", {
+ coerce = type_number / tostring
+ }),
+ one_of = OneOf,
+ all_of = AllOf,
+ shape = Shape,
+ partial = Partial,
+ pattern = Pattern,
+ array_of = ArrayOf,
+ array_contains = ArrayContains,
+ map_of = MapOf,
+ literal = Literal,
+ range = Range,
+ equivalent = Equivalent,
+ custom = Custom,
+ scope = TagScopeType,
+ proxy = Proxy,
+ assert = AssertType,
+ annotate = AnnotateNode,
+ metatable_is = MetatableIsType
+}, {
+ __index = function(self, fn_name)
+ return error("Type checker does not exist: `" .. tostring(fn_name) .. "`")
+ end
+})
+local check_shape
+check_shape = function(value, shape)
+ assert(shape.check_value, "missing check_value method from shape")
+ return shape:check_value(value)
+end
+is_type = function(val)
+ return BaseType:is_base_type(val)
+end
+return {
+ check_shape = check_shape,
+ types = types,
+ is_type = is_type,
+ BaseType = BaseType,
+ FailedTransform = FailedTransform,
+ VERSION = "2.6.0"
+}
diff --git a/contrib/mumhash/mum.h b/contrib/mumhash/mum.h
new file mode 100644
index 0000000..52b7845
--- /dev/null
+++ b/contrib/mumhash/mum.h
@@ -0,0 +1,392 @@
+/* Copyright (c) 2016 Vladimir Makarov <vmakarov@gcc.gnu.org>
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use, copy,
+ modify, merge, publish, distribute, sublicense, and/or sell copies
+ of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/* This file implements MUM (MUltiply and Mix) hashing. We randomize
+ input data by 64x64-bit multiplication and mixing hi- and low-parts
+ of the multiplication result by using an addition and then mix it
+ into the current state. We use prime numbers randomly generated
+ with the equal probability of their bit values for the
+ multiplication. When all primes are used once, the state is
+ randomized and the same prime numbers are used again for data
+ randomization.
+
+ The MUM hashing passes all SMHasher tests. Pseudo Random Number
+ Generator based on MUM also passes NIST Statistical Test Suite for
+ Random and Pseudorandom Number Generators for Cryptographic
+ Applications (version 2.2.1) with 1000 bitstreams each containing
+ 1M bits. MUM hashing is also faster Spooky64 and City64 on small
+ strings (at least up to 512-bit) on Haswell and Power7. The MUM bulk
+ speed (speed on very long data) is bigger than Spooky and City on
+ Power7. On Haswell the bulk speed is bigger than Spooky one and
+ close to City speed. */
+
+#ifndef __MUM_HASH__
+#define __MUM_HASH__
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#ifdef _MSC_VER
+typedef unsigned __int16 uint16_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+/* Macro saying to use 128-bit integers implemented by GCC for some
+ targets. */
+#ifndef _MUM_USE_INT128
+/* In GCC uint128_t is defined if HOST_BITS_PER_WIDE_INT >= 64.
+ HOST_WIDE_INT is long if HOST_BITS_PER_LONG > HOST_BITS_PER_INT,
+ otherwise int. */
+#ifdef __SIZEOF_INT128__
+#define _MUM_USE_INT128 1
+#else
+#define _MUM_USE_INT128 0
+#endif
+#endif
+
+#if defined(__GNUC__) && ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 9) || (__GNUC__ > 4))
+#define _MUM_FRESH_GCC
+#endif
+
+#if !defined(__llvm__) && defined(_MUM_FRESH_GCC)
+#define _MUM_ATTRIBUTE_UNUSED __attribute__((unused))
+#define _MUM_OPTIMIZE(opts) __attribute__((__optimize__ (opts)))
+#define _MUM_TARGET(opts) __attribute__((__target__ (opts)))
+#define _MUM_INLINE __attribute__((always_inline))
+#else
+#define _MUM_ATTRIBUTE_UNUSED
+#define _MUM_OPTIMIZE(opts)
+#define _MUM_TARGET(opts)
+#define _MUM_INLINE
+#endif
+
+
+/* Here are different primes randomly generated with the equal
+ probability of their bit values. They are used to randomize input
+ values. */
+static uint64_t _mum_hash_step_prime = 0x2e0bb864e9ea7df5ULL;
+static uint64_t _mum_key_step_prime = 0xcdb32970830fcaa1ULL;
+static uint64_t _mum_block_start_prime = 0xc42b5e2e6480b23bULL;
+static uint64_t _mum_unroll_prime = 0x7b51ec3d22f7096fULL;
+static uint64_t _mum_tail_prime = 0xaf47d47c99b1461bULL;
+static uint64_t _mum_finish_prime1 = 0xa9a7ae7ceff79f3fULL;
+static uint64_t _mum_finish_prime2 = 0xaf47d47c99b1461bULL;
+
+static uint64_t _mum_primes [] = {
+ 0X9ebdcae10d981691, 0X32b9b9b97a27ac7d, 0X29b5584d83d35bbd, 0X4b04e0e61401255f,
+ 0X25e8f7b1f1c9d027, 0X80d4c8c000f3e881, 0Xbd1255431904b9dd, 0X8a3bd4485eee6d81,
+ 0X3bc721b2aad05197, 0X71b1a19b907d6e33, 0X525e6c1084a8534b, 0X9e4c2cd340c1299f,
+ 0Xde3add92e94caa37, 0X7e14eadb1f65311d, 0X3f5aa40f89812853, 0X33b15a3b587d15c9,
+};
+
+/* Multiply 64-bit V and P and return sum of high and low parts of the
+ result. */
+static inline uint64_t _MUM_INLINE
+_mum (uint64_t v, uint64_t p) {
+ uint64_t hi, lo;
+#if _MUM_USE_INT128
+#if defined(__aarch64__)
+ /* AARCH64 needs 2 insns to calculate 128-bit result of the
+ multiplication. If we use a generic code we actually call a
+ function doing 128x128->128 bit multiplication. The function is
+ very slow. */
+ lo = v * p;
+ __asm__ ("umulh %0, %1, %2" : "=r" (hi) : "r" (v), "r" (p));
+#else
+ __uint128_t r = (__uint128_t) v * (__uint128_t) p;
+ hi = (uint64_t) (r >> 64);
+ lo = (uint64_t) r;
+#endif
+#else
+ /* Implementation of 64x64->128-bit multiplication by four 32x32->64
+ bit multiplication. */
+ uint64_t hv = v >> 32, hp = p >> 32;
+ uint64_t lv = (uint32_t) v, lp = (uint32_t) p;
+ uint64_t rh = hv * hp;
+ uint64_t rm_0 = hv * lp;
+ uint64_t rm_1 = hp * lv;
+ uint64_t rl = lv * lp;
+ uint64_t t, carry = 0;
+
+ /* We could ignore a carry bit here if we did not care about the
+ same hash for 32-bit and 64-bit targets. */
+ t = rl + (rm_0 << 32);
+#ifdef MUM_TARGET_INDEPENDENT_HASH
+ carry = t < rl;
+#endif
+ lo = t + (rm_1 << 32);
+#ifdef MUM_TARGET_INDEPENDENT_HASH
+ carry += lo < t;
+#endif
+ hi = rh + (rm_0 >> 32) + (rm_1 >> 32) + carry;
+#endif
+ /* We could use XOR here too but, for some reasons, on Haswell and
+ Power7 using an addition improves hashing performance by 10% for
+ small strings. */
+ return hi + lo;
+}
+
+#if defined(_MSC_VER)
+#define _mum_bswap_32(x) _byteswap_uint32_t (x)
+#define _mum_bswap_64(x) _byteswap_uint64_t (x)
+#elif defined(__APPLE__)
+#include <libkern/OSByteOrder.h>
+#define _mum_bswap_32(x) OSSwapInt32 (x)
+#define _mum_bswap_64(x) OSSwapInt64 (x)
+#elif defined(__GNUC__)
+#define _mum_bswap32(x) __builtin_bswap32 (x)
+#define _mum_bswap64(x) __builtin_bswap64 (x)
+#else
+#include <byteswap.h>
+#define _mum_bswap32(x) bswap32 (x)
+#define _mum_bswap64(x) bswap64 (x)
+#endif
+
+static inline uint64_t _MUM_INLINE
+_mum_le (uint64_t v) {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ || !defined(MUM_TARGET_INDEPENDENT_HASH)
+ return v;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return _mum_bswap64 (v);
+#else
+#error "Unknown endianness"
+#endif
+}
+
+static inline uint32_t _MUM_INLINE
+_mum_le32 (uint32_t v) {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ || !defined(MUM_TARGET_INDEPENDENT_HASH)
+ return v;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return _mum_bswap32 (v);
+#else
+#error "Unknown endianness"
+#endif
+}
+
+/* Macro defining how many times the most nested loop in
+ _mum_hash_aligned will be unrolled by the compiler (although it can
+ make an own decision:). Use only a constant here to help a
+ compiler to unroll a major loop.
+
+ The macro value affects the result hash for strings > 128 bit. The
+ unroll factor greatly affects the hashing speed. We prefer the
+ speed. */
+#ifndef _MUM_UNROLL_FACTOR_POWER
+#if defined(__PPC64__) && !defined(MUM_TARGET_INDEPENDENT_HASH)
+#define _MUM_UNROLL_FACTOR_POWER 3
+#elif defined(__aarch64__) && !defined(MUM_TARGET_INDEPENDENT_HASH)
+#define _MUM_UNROLL_FACTOR_POWER 4
+#else
+#define _MUM_UNROLL_FACTOR_POWER 2
+#endif
+#endif
+
+#if _MUM_UNROLL_FACTOR_POWER < 1
+#error "too small unroll factor"
+#elif _MUM_UNROLL_FACTOR_POWER > 4
+#error "We have not enough primes for such unroll factor"
+#endif
+
+#define _MUM_UNROLL_FACTOR (1 << _MUM_UNROLL_FACTOR_POWER)
+
+static inline uint64_t _MUM_OPTIMIZE("unroll-loops") _MUM_INLINE
+_mum_hash_aligned (uint64_t start, const void *key, size_t len) {
+ uint64_t result = start;
+ const unsigned char *str = (const unsigned char *) key;
+ uint64_t u64;
+ int i;
+ size_t n;
+
+ result = _mum (result, _mum_block_start_prime);
+ while (len > _MUM_UNROLL_FACTOR * sizeof (uint64_t)) {
+ /* This loop could be vectorized when we have vector insns for
+ 64x64->128-bit multiplication. AVX2 currently only have a
+ vector insn for 4 32x32->64-bit multiplication. */
+ for (i = 0; i < _MUM_UNROLL_FACTOR; i++)
+ result ^= _mum (_mum_le (((uint64_t *) str)[i]), _mum_primes[i]);
+ len -= _MUM_UNROLL_FACTOR * sizeof (uint64_t);
+ str += _MUM_UNROLL_FACTOR * sizeof (uint64_t);
+ /* We will use the same prime numbers on the next iterations --
+ randomize the state. */
+ result = _mum (result, _mum_unroll_prime);
+ }
+ n = len / sizeof (uint64_t);
+ for (i = 0; i < (int)n; i++)
+ result ^= _mum (_mum_le (((uint64_t *) str)[i]), _mum_primes[i]);
+ len -= n * sizeof (uint64_t); str += n * sizeof (uint64_t);
+ switch (len) {
+ case 7:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ u64 |= (uint64_t) str[4] << 32;
+ u64 |= (uint64_t) str[5] << 40;
+ u64 |= (uint64_t) str[6] << 48;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 6:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ u64 |= (uint64_t) str[4] << 32;
+ u64 |= (uint64_t) str[5] << 40;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 5:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ u64 |= (uint64_t) str[4] << 32;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 4:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 3:
+ u64 = str[0];
+ u64 |= (uint64_t) str[1] << 8;
+ u64 |= (uint64_t) str[2] << 16;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 2:
+ u64 = str[0];
+ u64 |= (uint64_t) str[1] << 8;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 1:
+ u64 = str[0];
+ return result ^ _mum (u64, _mum_tail_prime);
+ }
+ return result;
+}
+
+/* Final randomization of H. */
+static inline uint64_t
+_mum_final (uint64_t h) {
+ h ^= _mum (h, _mum_finish_prime1);
+ h ^= _mum (h, _mum_finish_prime2);
+ return h;
+}
+
+#ifndef _MUM_UNALIGNED_ACCESS
+#if defined(__x86_64__) || defined(__i386__) || defined(__PPC64__) \
+ || defined(__s390__) || defined(__m32c__) || defined(cris) \
+ || defined(__CR16__) || defined(__vax__) || defined(__m68k__) \
+ || defined(__aarch64__)
+#define _MUM_UNALIGNED_ACCESS 1
+#else
+#define _MUM_UNALIGNED_ACCESS 0
+#endif
+#endif
+
+/* When we need an aligned access to data being hashed we move part of
+ the unaligned data to an aligned block of given size and then
+ process it, repeating processing the data by the block. */
+#ifndef _MUM_BLOCK_LEN
+#define _MUM_BLOCK_LEN 1024
+#endif
+
+#if _MUM_BLOCK_LEN < 8
+#error "too small block length"
+#endif
+
+static inline uint64_t _MUM_INLINE
+_mum_hash_default (const void *key, size_t len, uint64_t seed) {
+ uint64_t result;
+ const unsigned char *str = (const unsigned char *) key;
+ size_t block_len;
+ uint64_t buf[_MUM_BLOCK_LEN / sizeof (uint64_t)];
+
+ result = seed + len;
+ if (_MUM_UNALIGNED_ACCESS || ((size_t) str & 0x7) == 0)
+ result = _mum_hash_aligned (result, key, len);
+ else {
+ while (len != 0) {
+ block_len = len < _MUM_BLOCK_LEN ? len : _MUM_BLOCK_LEN;
+ memmove (buf, str, block_len);
+ result = _mum_hash_aligned (result, buf, block_len);
+ len -= block_len;
+ str += block_len;
+ }
+ }
+ return _mum_final (result);
+}
+
+static inline uint64_t _MUM_INLINE
+_mum_next_factor (void) {
+ uint64_t start = 0;
+ int i;
+
+ for (i = 0; i < 8; i++)
+ start = (start << 8) | rand() % 256;
+ return start;
+}
+
+/* ++++++++++++++++++++++++++ Interface functions: +++++++++++++++++++ */
+
+/* Set random multiplicators depending on SEED. */
+static inline void
+mum_hash_randomize (uint64_t seed) {
+ int i;
+
+ srand (seed);
+ _mum_hash_step_prime = _mum_next_factor ();
+ _mum_key_step_prime = _mum_next_factor ();
+ _mum_finish_prime1 = _mum_next_factor ();
+ _mum_finish_prime2 = _mum_next_factor ();
+ _mum_block_start_prime = _mum_next_factor ();
+ _mum_unroll_prime = _mum_next_factor ();
+ _mum_tail_prime = _mum_next_factor ();
+ for (i = 0; i < (int)(sizeof (_mum_primes) / sizeof (uint64_t)); i++)
+ _mum_primes[i] = _mum_next_factor ();
+}
+
+/* Start hashing data with SEED. Return the state. */
+static inline uint64_t
+mum_hash_init (uint64_t seed) {
+ return seed;
+}
+
+/* Process data KEY with the state H and return the updated state. */
+static inline uint64_t
+mum_hash_step (uint64_t h, uint64_t key)
+{
+ return _mum (h, _mum_hash_step_prime) ^ _mum (key, _mum_key_step_prime);
+}
+
+/* Return the result of hashing using the current state H. */
+static inline uint64_t
+mum_hash_finish (uint64_t h) {
+ return _mum_final (h);
+}
+
+/* Fast hashing of KEY with SEED. The hash is always the same for the
+ same key on any target. */
+static inline size_t
+mum_hash64 (uint64_t key, uint64_t seed) {
+ return mum_hash_finish (mum_hash_step (mum_hash_init (seed), key));
+}
+
+/* Hash data KEY of length LEN and SEED. The hash depends on the
+ target endianness and the unroll factor. */
+static inline uint64_t _MUM_INLINE
+mum_hash (const void *key, size_t len, uint64_t seed) {
+ return _mum_hash_default (key, len, seed);
+}
+
+#endif
diff --git a/contrib/publicsuffix/effective_tld_names.dat b/contrib/publicsuffix/effective_tld_names.dat
new file mode 100644
index 0000000..c4b3c98
--- /dev/null
+++ b/contrib/publicsuffix/effective_tld_names.dat
@@ -0,0 +1,14206 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Please pull this list from, and only from https://publicsuffix.org/list/public_suffix_list.dat,
+// rather than any other VCS sites. Pulling from any other URL is not guaranteed to be supported.
+
+// Instructions on pulling and using this list can be found at https://publicsuffix.org/list/.
+
+// ===BEGIN ICANN DOMAINS===
+
+// ac : https://en.wikipedia.org/wiki/.ac
+ac
+com.ac
+edu.ac
+gov.ac
+net.ac
+mil.ac
+org.ac
+
+// ad : https://en.wikipedia.org/wiki/.ad
+ad
+nom.ad
+
+// ae : https://en.wikipedia.org/wiki/.ae
+// see also: "Domain Name Eligibility Policy" at http://www.aeda.ae/eng/aepolicy.php
+ae
+co.ae
+net.ae
+org.ae
+sch.ae
+ac.ae
+gov.ae
+mil.ae
+
+// aero : see https://www.information.aero/index.php?id=66
+aero
+accident-investigation.aero
+accident-prevention.aero
+aerobatic.aero
+aeroclub.aero
+aerodrome.aero
+agents.aero
+aircraft.aero
+airline.aero
+airport.aero
+air-surveillance.aero
+airtraffic.aero
+air-traffic-control.aero
+ambulance.aero
+amusement.aero
+association.aero
+author.aero
+ballooning.aero
+broker.aero
+caa.aero
+cargo.aero
+catering.aero
+certification.aero
+championship.aero
+charter.aero
+civilaviation.aero
+club.aero
+conference.aero
+consultant.aero
+consulting.aero
+control.aero
+council.aero
+crew.aero
+design.aero
+dgca.aero
+educator.aero
+emergency.aero
+engine.aero
+engineer.aero
+entertainment.aero
+equipment.aero
+exchange.aero
+express.aero
+federation.aero
+flight.aero
+fuel.aero
+gliding.aero
+government.aero
+groundhandling.aero
+group.aero
+hanggliding.aero
+homebuilt.aero
+insurance.aero
+journal.aero
+journalist.aero
+leasing.aero
+logistics.aero
+magazine.aero
+maintenance.aero
+media.aero
+microlight.aero
+modelling.aero
+navigation.aero
+parachuting.aero
+paragliding.aero
+passenger-association.aero
+pilot.aero
+press.aero
+production.aero
+recreation.aero
+repbody.aero
+res.aero
+research.aero
+rotorcraft.aero
+safety.aero
+scientist.aero
+services.aero
+show.aero
+skydiving.aero
+software.aero
+student.aero
+trader.aero
+trading.aero
+trainer.aero
+union.aero
+workinggroup.aero
+works.aero
+
+// af : http://www.nic.af/help.jsp
+af
+gov.af
+com.af
+org.af
+net.af
+edu.af
+
+// ag : http://www.nic.ag/prices.htm
+ag
+com.ag
+org.ag
+net.ag
+co.ag
+nom.ag
+
+// ai : http://nic.com.ai/
+ai
+off.ai
+com.ai
+net.ai
+org.ai
+
+// al : http://www.ert.gov.al/ert_alb/faq_det.html?Id=31
+al
+com.al
+edu.al
+gov.al
+mil.al
+net.al
+org.al
+
+// am : https://www.amnic.net/policy/en/Policy_EN.pdf
+am
+co.am
+com.am
+commune.am
+net.am
+org.am
+
+// ao : https://en.wikipedia.org/wiki/.ao
+// http://www.dns.ao/REGISTR.DOC
+ao
+ed.ao
+gv.ao
+og.ao
+co.ao
+pb.ao
+it.ao
+
+// aq : https://en.wikipedia.org/wiki/.aq
+aq
+
+// ar : https://nic.ar/nic-argentina/normativa-vigente
+ar
+com.ar
+edu.ar
+gob.ar
+gov.ar
+int.ar
+mil.ar
+musica.ar
+net.ar
+org.ar
+tur.ar
+
+// arpa : https://en.wikipedia.org/wiki/.arpa
+// Confirmed by registry <iana-questions@icann.org> 2008-06-18
+arpa
+e164.arpa
+in-addr.arpa
+ip6.arpa
+iris.arpa
+uri.arpa
+urn.arpa
+
+// as : https://en.wikipedia.org/wiki/.as
+as
+gov.as
+
+// asia : https://en.wikipedia.org/wiki/.asia
+asia
+
+// at : https://en.wikipedia.org/wiki/.at
+// Confirmed by registry <it@nic.at> 2008-06-17
+at
+ac.at
+co.at
+gv.at
+or.at
+sth.ac.at
+
+// au : https://en.wikipedia.org/wiki/.au
+// http://www.auda.org.au/
+au
+// 2LDs
+com.au
+net.au
+org.au
+edu.au
+gov.au
+asn.au
+id.au
+// Historic 2LDs (closed to new registration, but sites still exist)
+info.au
+conf.au
+oz.au
+// CGDNs - http://www.cgdn.org.au/
+act.au
+nsw.au
+nt.au
+qld.au
+sa.au
+tas.au
+vic.au
+wa.au
+// 3LDs
+act.edu.au
+catholic.edu.au
+// eq.edu.au - Removed at the request of the Queensland Department of Education
+nsw.edu.au
+nt.edu.au
+qld.edu.au
+sa.edu.au
+tas.edu.au
+vic.edu.au
+wa.edu.au
+// act.gov.au Bug 984824 - Removed at request of Greg Tankard
+// nsw.gov.au Bug 547985 - Removed at request of <Shae.Donelan@services.nsw.gov.au>
+// nt.gov.au Bug 940478 - Removed at request of Greg Connors <Greg.Connors@nt.gov.au>
+qld.gov.au
+sa.gov.au
+tas.gov.au
+vic.gov.au
+wa.gov.au
+// 4LDs
+// education.tas.edu.au - Removed at the request of the Department of Education Tasmania
+schools.nsw.edu.au
+
+// aw : https://en.wikipedia.org/wiki/.aw
+aw
+com.aw
+
+// ax : https://en.wikipedia.org/wiki/.ax
+ax
+
+// az : https://en.wikipedia.org/wiki/.az
+az
+com.az
+net.az
+int.az
+gov.az
+org.az
+edu.az
+info.az
+pp.az
+mil.az
+name.az
+pro.az
+biz.az
+
+// ba : http://nic.ba/users_data/files/pravilnik_o_registraciji.pdf
+ba
+com.ba
+edu.ba
+gov.ba
+mil.ba
+net.ba
+org.ba
+
+// bb : https://en.wikipedia.org/wiki/.bb
+bb
+biz.bb
+co.bb
+com.bb
+edu.bb
+gov.bb
+info.bb
+net.bb
+org.bb
+store.bb
+tv.bb
+
+// bd : https://en.wikipedia.org/wiki/.bd
+*.bd
+
+// be : https://en.wikipedia.org/wiki/.be
+// Confirmed by registry <tech@dns.be> 2008-06-08
+be
+ac.be
+
+// bf : https://en.wikipedia.org/wiki/.bf
+bf
+gov.bf
+
+// bg : https://en.wikipedia.org/wiki/.bg
+// https://www.register.bg/user/static/rules/en/index.html
+bg
+a.bg
+b.bg
+c.bg
+d.bg
+e.bg
+f.bg
+g.bg
+h.bg
+i.bg
+j.bg
+k.bg
+l.bg
+m.bg
+n.bg
+o.bg
+p.bg
+q.bg
+r.bg
+s.bg
+t.bg
+u.bg
+v.bg
+w.bg
+x.bg
+y.bg
+z.bg
+0.bg
+1.bg
+2.bg
+3.bg
+4.bg
+5.bg
+6.bg
+7.bg
+8.bg
+9.bg
+
+// bh : https://en.wikipedia.org/wiki/.bh
+bh
+com.bh
+edu.bh
+net.bh
+org.bh
+gov.bh
+
+// bi : https://en.wikipedia.org/wiki/.bi
+// http://whois.nic.bi/
+bi
+co.bi
+com.bi
+edu.bi
+or.bi
+org.bi
+
+// biz : https://en.wikipedia.org/wiki/.biz
+biz
+
+// bj : https://en.wikipedia.org/wiki/.bj
+bj
+asso.bj
+barreau.bj
+gouv.bj
+
+// bm : http://www.bermudanic.bm/dnr-text.txt
+bm
+com.bm
+edu.bm
+gov.bm
+net.bm
+org.bm
+
+// bn : http://www.bnnic.bn/faqs
+bn
+com.bn
+edu.bn
+gov.bn
+net.bn
+org.bn
+
+// bo : https://nic.bo/delegacion2015.php#h-1.10
+bo
+com.bo
+edu.bo
+gob.bo
+int.bo
+org.bo
+net.bo
+mil.bo
+tv.bo
+web.bo
+// Social Domains
+academia.bo
+agro.bo
+arte.bo
+blog.bo
+bolivia.bo
+ciencia.bo
+cooperativa.bo
+democracia.bo
+deporte.bo
+ecologia.bo
+economia.bo
+empresa.bo
+indigena.bo
+industria.bo
+info.bo
+medicina.bo
+movimiento.bo
+musica.bo
+natural.bo
+nombre.bo
+noticias.bo
+patria.bo
+politica.bo
+profesional.bo
+plurinacional.bo
+pueblo.bo
+revista.bo
+salud.bo
+tecnologia.bo
+tksat.bo
+transporte.bo
+wiki.bo
+
+// br : http://registro.br/dominio/categoria.html
+// Submitted by registry <fneves@registro.br>
+br
+9guacu.br
+abc.br
+adm.br
+adv.br
+agr.br
+aju.br
+am.br
+anani.br
+aparecida.br
+app.br
+arq.br
+art.br
+ato.br
+b.br
+barueri.br
+belem.br
+bhz.br
+bib.br
+bio.br
+blog.br
+bmd.br
+boavista.br
+bsb.br
+campinagrande.br
+campinas.br
+caxias.br
+cim.br
+cng.br
+cnt.br
+com.br
+contagem.br
+coop.br
+coz.br
+cri.br
+cuiaba.br
+curitiba.br
+def.br
+des.br
+det.br
+dev.br
+ecn.br
+eco.br
+edu.br
+emp.br
+enf.br
+eng.br
+esp.br
+etc.br
+eti.br
+far.br
+feira.br
+flog.br
+floripa.br
+fm.br
+fnd.br
+fortal.br
+fot.br
+foz.br
+fst.br
+g12.br
+geo.br
+ggf.br
+goiania.br
+gov.br
+// gov.br 26 states + df https://en.wikipedia.org/wiki/States_of_Brazil
+ac.gov.br
+al.gov.br
+am.gov.br
+ap.gov.br
+ba.gov.br
+ce.gov.br
+df.gov.br
+es.gov.br
+go.gov.br
+ma.gov.br
+mg.gov.br
+ms.gov.br
+mt.gov.br
+pa.gov.br
+pb.gov.br
+pe.gov.br
+pi.gov.br
+pr.gov.br
+rj.gov.br
+rn.gov.br
+ro.gov.br
+rr.gov.br
+rs.gov.br
+sc.gov.br
+se.gov.br
+sp.gov.br
+to.gov.br
+gru.br
+imb.br
+ind.br
+inf.br
+jab.br
+jampa.br
+jdf.br
+joinville.br
+jor.br
+jus.br
+leg.br
+lel.br
+log.br
+londrina.br
+macapa.br
+maceio.br
+manaus.br
+maringa.br
+mat.br
+med.br
+mil.br
+morena.br
+mp.br
+mus.br
+natal.br
+net.br
+niteroi.br
+*.nom.br
+not.br
+ntr.br
+odo.br
+ong.br
+org.br
+osasco.br
+palmas.br
+poa.br
+ppg.br
+pro.br
+psc.br
+psi.br
+pvh.br
+qsl.br
+radio.br
+rec.br
+recife.br
+rep.br
+ribeirao.br
+rio.br
+riobranco.br
+riopreto.br
+salvador.br
+sampa.br
+santamaria.br
+santoandre.br
+saobernardo.br
+saogonca.br
+seg.br
+sjc.br
+slg.br
+slz.br
+sorocaba.br
+srv.br
+taxi.br
+tc.br
+tec.br
+teo.br
+the.br
+tmp.br
+trd.br
+tur.br
+tv.br
+udi.br
+vet.br
+vix.br
+vlog.br
+wiki.br
+zlg.br
+
+// bs : http://www.nic.bs/rules.html
+bs
+com.bs
+net.bs
+org.bs
+edu.bs
+gov.bs
+
+// bt : https://en.wikipedia.org/wiki/.bt
+bt
+com.bt
+edu.bt
+gov.bt
+net.bt
+org.bt
+
+// bv : No registrations at this time.
+// Submitted by registry <jarle@uninett.no>
+bv
+
+// bw : https://en.wikipedia.org/wiki/.bw
+// http://www.gobin.info/domainname/bw.doc
+// list of other 2nd level tlds ?
+bw
+co.bw
+org.bw
+
+// by : https://en.wikipedia.org/wiki/.by
+// http://tld.by/rules_2006_en.html
+// list of other 2nd level tlds ?
+by
+gov.by
+mil.by
+// Official information does not indicate that com.by is a reserved
+// second-level domain, but it's being used as one (see www.google.com.by and
+// www.yahoo.com.by, for example), so we list it here for safety's sake.
+com.by
+
+// http://hoster.by/
+of.by
+
+// bz : https://en.wikipedia.org/wiki/.bz
+// http://www.belizenic.bz/
+bz
+com.bz
+net.bz
+org.bz
+edu.bz
+gov.bz
+
+// ca : https://en.wikipedia.org/wiki/.ca
+ca
+// ca geographical names
+ab.ca
+bc.ca
+mb.ca
+nb.ca
+nf.ca
+nl.ca
+ns.ca
+nt.ca
+nu.ca
+on.ca
+pe.ca
+qc.ca
+sk.ca
+yk.ca
+// gc.ca: https://en.wikipedia.org/wiki/.gc.ca
+// see also: http://registry.gc.ca/en/SubdomainFAQ
+gc.ca
+
+// cat : https://en.wikipedia.org/wiki/.cat
+cat
+
+// cc : https://en.wikipedia.org/wiki/.cc
+cc
+
+// cd : https://en.wikipedia.org/wiki/.cd
+// see also: https://www.nic.cd/domain/insertDomain_2.jsp?act=1
+cd
+gov.cd
+
+// cf : https://en.wikipedia.org/wiki/.cf
+cf
+
+// cg : https://en.wikipedia.org/wiki/.cg
+cg
+
+// ch : https://en.wikipedia.org/wiki/.ch
+ch
+
+// ci : https://en.wikipedia.org/wiki/.ci
+// http://www.nic.ci/index.php?page=charte
+ci
+org.ci
+or.ci
+com.ci
+co.ci
+edu.ci
+ed.ci
+ac.ci
+net.ci
+go.ci
+asso.ci
+xn--aroport-bya.ci
+aéroport.ci
+int.ci
+presse.ci
+md.ci
+gouv.ci
+
+// ck : https://en.wikipedia.org/wiki/.ck
+*.ck
+!www.ck
+
+// cl : https://www.nic.cl
+// Confirmed by .CL registry <hsalgado@nic.cl>
+cl
+co.cl
+gob.cl
+gov.cl
+mil.cl
+
+// cm : https://en.wikipedia.org/wiki/.cm plus bug 981927
+cm
+co.cm
+com.cm
+gov.cm
+net.cm
+
+// cn : https://en.wikipedia.org/wiki/.cn
+// Submitted by registry <tanyaling@cnnic.cn>
+cn
+ac.cn
+com.cn
+edu.cn
+gov.cn
+net.cn
+org.cn
+mil.cn
+xn--55qx5d.cn
+公司.cn
+xn--io0a7i.cn
+网络.cn
+xn--od0alg.cn
+網絡.cn
+// cn geographic names
+ah.cn
+bj.cn
+cq.cn
+fj.cn
+gd.cn
+gs.cn
+gz.cn
+gx.cn
+ha.cn
+hb.cn
+he.cn
+hi.cn
+hl.cn
+hn.cn
+jl.cn
+js.cn
+jx.cn
+ln.cn
+nm.cn
+nx.cn
+qh.cn
+sc.cn
+sd.cn
+sh.cn
+sn.cn
+sx.cn
+tj.cn
+xj.cn
+xz.cn
+yn.cn
+zj.cn
+hk.cn
+mo.cn
+tw.cn
+
+// co : https://en.wikipedia.org/wiki/.co
+// Submitted by registry <tecnico@uniandes.edu.co>
+co
+arts.co
+com.co
+edu.co
+firm.co
+gov.co
+info.co
+int.co
+mil.co
+net.co
+nom.co
+org.co
+rec.co
+web.co
+
+// com : https://en.wikipedia.org/wiki/.com
+com
+
+// coop : https://en.wikipedia.org/wiki/.coop
+coop
+
+// cr : http://www.nic.cr/niccr_publico/showRegistroDominiosScreen.do
+cr
+ac.cr
+co.cr
+ed.cr
+fi.cr
+go.cr
+or.cr
+sa.cr
+
+// cu : https://en.wikipedia.org/wiki/.cu
+cu
+com.cu
+edu.cu
+org.cu
+net.cu
+gov.cu
+inf.cu
+
+// cv : https://en.wikipedia.org/wiki/.cv
+cv
+
+// cw : http://www.una.cw/cw_registry/
+// Confirmed by registry <registry@una.net> 2013-03-26
+cw
+com.cw
+edu.cw
+net.cw
+org.cw
+
+// cx : https://en.wikipedia.org/wiki/.cx
+// list of other 2nd level tlds ?
+cx
+gov.cx
+
+// cy : http://www.nic.cy/
+// Submitted by registry Panayiotou Fotia <cydns@ucy.ac.cy>
+cy
+ac.cy
+biz.cy
+com.cy
+ekloges.cy
+gov.cy
+ltd.cy
+name.cy
+net.cy
+org.cy
+parliament.cy
+press.cy
+pro.cy
+tm.cy
+
+// cz : https://en.wikipedia.org/wiki/.cz
+cz
+
+// de : https://en.wikipedia.org/wiki/.de
+// Confirmed by registry <ops@denic.de> (with technical
+// reservations) 2008-07-01
+de
+
+// dj : https://en.wikipedia.org/wiki/.dj
+dj
+
+// dk : https://en.wikipedia.org/wiki/.dk
+// Confirmed by registry <robert@dk-hostmaster.dk> 2008-06-17
+dk
+
+// dm : https://en.wikipedia.org/wiki/.dm
+dm
+com.dm
+net.dm
+org.dm
+edu.dm
+gov.dm
+
+// do : https://en.wikipedia.org/wiki/.do
+do
+art.do
+com.do
+edu.do
+gob.do
+gov.do
+mil.do
+net.do
+org.do
+sld.do
+web.do
+
+// dz : http://www.nic.dz/images/pdf_nic/charte.pdf
+dz
+art.dz
+asso.dz
+com.dz
+edu.dz
+gov.dz
+org.dz
+net.dz
+pol.dz
+soc.dz
+tm.dz
+
+// ec : http://www.nic.ec/reg/paso1.asp
+// Submitted by registry <vabboud@nic.ec>
+ec
+com.ec
+info.ec
+net.ec
+fin.ec
+k12.ec
+med.ec
+pro.ec
+org.ec
+edu.ec
+gov.ec
+gob.ec
+mil.ec
+
+// edu : https://en.wikipedia.org/wiki/.edu
+edu
+
+// ee : http://www.eenet.ee/EENet/dom_reeglid.html#lisa_B
+ee
+edu.ee
+gov.ee
+riik.ee
+lib.ee
+med.ee
+com.ee
+pri.ee
+aip.ee
+org.ee
+fie.ee
+
+// eg : https://en.wikipedia.org/wiki/.eg
+eg
+com.eg
+edu.eg
+eun.eg
+gov.eg
+mil.eg
+name.eg
+net.eg
+org.eg
+sci.eg
+
+// er : https://en.wikipedia.org/wiki/.er
+*.er
+
+// es : https://www.nic.es/site_ingles/ingles/dominios/index.html
+es
+com.es
+nom.es
+org.es
+gob.es
+edu.es
+
+// et : https://en.wikipedia.org/wiki/.et
+et
+com.et
+gov.et
+org.et
+edu.et
+biz.et
+name.et
+info.et
+net.et
+
+// eu : https://en.wikipedia.org/wiki/.eu
+eu
+
+// fi : https://en.wikipedia.org/wiki/.fi
+fi
+// aland.fi : https://en.wikipedia.org/wiki/.ax
+// This domain is being phased out in favor of .ax. As there are still many
+// domains under aland.fi, we still keep it on the list until aland.fi is
+// completely removed.
+// TODO: Check for updates (expected to be phased out around Q1/2009)
+aland.fi
+
+// fj : http://domains.fj/
+// Submitted by registry <garth.miller@cocca.org.nz> 2020-02-11
+fj
+ac.fj
+biz.fj
+com.fj
+gov.fj
+info.fj
+mil.fj
+name.fj
+net.fj
+org.fj
+pro.fj
+
+// fk : https://en.wikipedia.org/wiki/.fk
+*.fk
+
+// fm : https://en.wikipedia.org/wiki/.fm
+com.fm
+edu.fm
+net.fm
+org.fm
+fm
+
+// fo : https://en.wikipedia.org/wiki/.fo
+fo
+
+// fr : http://www.afnic.fr/
+// domaines descriptifs : https://www.afnic.fr/medias/documents/Cadre_legal/Afnic_Naming_Policy_12122016_VEN.pdf
+fr
+asso.fr
+com.fr
+gouv.fr
+nom.fr
+prd.fr
+tm.fr
+// domaines sectoriels : https://www.afnic.fr/en/products-and-services/the-fr-tld/sector-based-fr-domains-4.html
+aeroport.fr
+avocat.fr
+avoues.fr
+cci.fr
+chambagri.fr
+chirurgiens-dentistes.fr
+experts-comptables.fr
+geometre-expert.fr
+greta.fr
+huissier-justice.fr
+medecin.fr
+notaires.fr
+pharmacien.fr
+port.fr
+veterinaire.fr
+
+// ga : https://en.wikipedia.org/wiki/.ga
+ga
+
+// gb : This registry is effectively dormant
+// Submitted by registry <Damien.Shaw@ja.net>
+gb
+
+// gd : https://en.wikipedia.org/wiki/.gd
+edu.gd
+gov.gd
+gd
+
+// ge : http://www.nic.net.ge/policy_en.pdf
+ge
+com.ge
+edu.ge
+gov.ge
+org.ge
+mil.ge
+net.ge
+pvt.ge
+
+// gf : https://en.wikipedia.org/wiki/.gf
+gf
+
+// gg : http://www.channelisles.net/register-domains/
+// Confirmed by registry <nigel@channelisles.net> 2013-11-28
+gg
+co.gg
+net.gg
+org.gg
+
+// gh : https://en.wikipedia.org/wiki/.gh
+// see also: http://www.nic.gh/reg_now.php
+// Although domains directly at second level are not possible at the moment,
+// they have been possible for some time and may come back.
+gh
+com.gh
+edu.gh
+gov.gh
+org.gh
+mil.gh
+
+// gi : http://www.nic.gi/rules.html
+gi
+com.gi
+ltd.gi
+gov.gi
+mod.gi
+edu.gi
+org.gi
+
+// gl : https://en.wikipedia.org/wiki/.gl
+// http://nic.gl
+gl
+co.gl
+com.gl
+edu.gl
+net.gl
+org.gl
+
+// gm : http://www.nic.gm/htmlpages%5Cgm-policy.htm
+gm
+
+// gn : http://psg.com/dns/gn/gn.txt
+// Submitted by registry <randy@psg.com>
+gn
+ac.gn
+com.gn
+edu.gn
+gov.gn
+org.gn
+net.gn
+
+// gov : https://en.wikipedia.org/wiki/.gov
+gov
+
+// gp : http://www.nic.gp/index.php?lang=en
+gp
+com.gp
+net.gp
+mobi.gp
+edu.gp
+org.gp
+asso.gp
+
+// gq : https://en.wikipedia.org/wiki/.gq
+gq
+
+// gr : https://grweb.ics.forth.gr/english/1617-B-2005.html
+// Submitted by registry <segred@ics.forth.gr>
+gr
+com.gr
+edu.gr
+net.gr
+org.gr
+gov.gr
+
+// gs : https://en.wikipedia.org/wiki/.gs
+gs
+
+// gt : https://www.gt/sitio/registration_policy.php?lang=en
+gt
+com.gt
+edu.gt
+gob.gt
+ind.gt
+mil.gt
+net.gt
+org.gt
+
+// gu : http://gadao.gov.gu/register.html
+// University of Guam : https://www.uog.edu
+// Submitted by uognoc@triton.uog.edu
+gu
+com.gu
+edu.gu
+gov.gu
+guam.gu
+info.gu
+net.gu
+org.gu
+web.gu
+
+// gw : https://en.wikipedia.org/wiki/.gw
+gw
+
+// gy : https://en.wikipedia.org/wiki/.gy
+// http://registry.gy/
+gy
+co.gy
+com.gy
+edu.gy
+gov.gy
+net.gy
+org.gy
+
+// hk : https://www.hkirc.hk
+// Submitted by registry <hk.tech@hkirc.hk>
+hk
+com.hk
+edu.hk
+gov.hk
+idv.hk
+net.hk
+org.hk
+xn--55qx5d.hk
+公司.hk
+xn--wcvs22d.hk
+教育.hk
+xn--lcvr32d.hk
+敎育.hk
+xn--mxtq1m.hk
+政府.hk
+xn--gmqw5a.hk
+個人.hk
+xn--ciqpn.hk
+个人.hk
+xn--gmq050i.hk
+箇人.hk
+xn--zf0avx.hk
+網络.hk
+xn--io0a7i.hk
+网络.hk
+xn--mk0axi.hk
+组織.hk
+xn--od0alg.hk
+網絡.hk
+xn--od0aq3b.hk
+网絡.hk
+xn--tn0ag.hk
+组织.hk
+xn--uc0atv.hk
+組織.hk
+xn--uc0ay4a.hk
+組织.hk
+
+// hm : https://en.wikipedia.org/wiki/.hm
+hm
+
+// hn : http://www.nic.hn/politicas/ps02,,05.html
+hn
+com.hn
+edu.hn
+org.hn
+net.hn
+mil.hn
+gob.hn
+
+// hr : http://www.dns.hr/documents/pdf/HRTLD-regulations.pdf
+hr
+iz.hr
+from.hr
+name.hr
+com.hr
+
+// ht : http://www.nic.ht/info/charte.cfm
+ht
+com.ht
+shop.ht
+firm.ht
+info.ht
+adult.ht
+net.ht
+pro.ht
+org.ht
+med.ht
+art.ht
+coop.ht
+pol.ht
+asso.ht
+edu.ht
+rel.ht
+gouv.ht
+perso.ht
+
+// hu : http://www.domain.hu/domain/English/sld.html
+// Confirmed by registry <pasztor@iszt.hu> 2008-06-12
+hu
+co.hu
+info.hu
+org.hu
+priv.hu
+sport.hu
+tm.hu
+2000.hu
+agrar.hu
+bolt.hu
+casino.hu
+city.hu
+erotica.hu
+erotika.hu
+film.hu
+forum.hu
+games.hu
+hotel.hu
+ingatlan.hu
+jogasz.hu
+konyvelo.hu
+lakas.hu
+media.hu
+news.hu
+reklam.hu
+sex.hu
+shop.hu
+suli.hu
+szex.hu
+tozsde.hu
+utazas.hu
+video.hu
+
+// id : https://pandi.id/en/domain/registration-requirements/
+id
+ac.id
+biz.id
+co.id
+desa.id
+go.id
+mil.id
+my.id
+net.id
+or.id
+ponpes.id
+sch.id
+web.id
+
+// ie : https://en.wikipedia.org/wiki/.ie
+ie
+gov.ie
+
+// il : http://www.isoc.org.il/domains/
+il
+ac.il
+co.il
+gov.il
+idf.il
+k12.il
+muni.il
+net.il
+org.il
+
+// im : https://www.nic.im/
+// Submitted by registry <info@nic.im>
+im
+ac.im
+co.im
+com.im
+ltd.co.im
+net.im
+org.im
+plc.co.im
+tt.im
+tv.im
+
+// in : https://en.wikipedia.org/wiki/.in
+// see also: https://registry.in/Policies
+// Please note, that nic.in is not an official eTLD, but used by most
+// government institutions.
+in
+co.in
+firm.in
+net.in
+org.in
+gen.in
+ind.in
+nic.in
+ac.in
+edu.in
+res.in
+gov.in
+mil.in
+
+// info : https://en.wikipedia.org/wiki/.info
+info
+
+// int : https://en.wikipedia.org/wiki/.int
+// Confirmed by registry <iana-questions@icann.org> 2008-06-18
+int
+eu.int
+
+// io : http://www.nic.io/rules.html
+// list of other 2nd level tlds ?
+io
+com.io
+
+// iq : http://www.cmc.iq/english/iq/iqregister1.htm
+iq
+gov.iq
+edu.iq
+mil.iq
+com.iq
+org.iq
+net.iq
+
+// ir : http://www.nic.ir/Terms_and_Conditions_ir,_Appendix_1_Domain_Rules
+// Also see http://www.nic.ir/Internationalized_Domain_Names
+// Two <iran>.ir entries added at request of <tech-team@nic.ir>, 2010-04-16
+ir
+ac.ir
+co.ir
+gov.ir
+id.ir
+net.ir
+org.ir
+sch.ir
+// xn--mgba3a4f16a.ir (<iran>.ir, Persian YEH)
+xn--mgba3a4f16a.ir
+ایران.ir
+// xn--mgba3a4fra.ir (<iran>.ir, Arabic YEH)
+xn--mgba3a4fra.ir
+ايران.ir
+
+// is : http://www.isnic.is/domain/rules.php
+// Confirmed by registry <marius@isgate.is> 2008-12-06
+is
+net.is
+com.is
+edu.is
+gov.is
+org.is
+int.is
+
+// it : https://en.wikipedia.org/wiki/.it
+it
+gov.it
+edu.it
+// Reserved geo-names (regions and provinces):
+// https://www.nic.it/sites/default/files/archivio/docs/Regulation_assignation_v7.1.pdf
+// Regions
+abr.it
+abruzzo.it
+aosta-valley.it
+aostavalley.it
+bas.it
+basilicata.it
+cal.it
+calabria.it
+cam.it
+campania.it
+emilia-romagna.it
+emiliaromagna.it
+emr.it
+friuli-v-giulia.it
+friuli-ve-giulia.it
+friuli-vegiulia.it
+friuli-venezia-giulia.it
+friuli-veneziagiulia.it
+friuli-vgiulia.it
+friuliv-giulia.it
+friulive-giulia.it
+friulivegiulia.it
+friulivenezia-giulia.it
+friuliveneziagiulia.it
+friulivgiulia.it
+fvg.it
+laz.it
+lazio.it
+lig.it
+liguria.it
+lom.it
+lombardia.it
+lombardy.it
+lucania.it
+mar.it
+marche.it
+mol.it
+molise.it
+piedmont.it
+piemonte.it
+pmn.it
+pug.it
+puglia.it
+sar.it
+sardegna.it
+sardinia.it
+sic.it
+sicilia.it
+sicily.it
+taa.it
+tos.it
+toscana.it
+trentin-sud-tirol.it
+xn--trentin-sd-tirol-rzb.it
+trentin-süd-tirol.it
+trentin-sudtirol.it
+xn--trentin-sdtirol-7vb.it
+trentin-südtirol.it
+trentin-sued-tirol.it
+trentin-suedtirol.it
+trentino-a-adige.it
+trentino-aadige.it
+trentino-alto-adige.it
+trentino-altoadige.it
+trentino-s-tirol.it
+trentino-stirol.it
+trentino-sud-tirol.it
+xn--trentino-sd-tirol-c3b.it
+trentino-süd-tirol.it
+trentino-sudtirol.it
+xn--trentino-sdtirol-szb.it
+trentino-südtirol.it
+trentino-sued-tirol.it
+trentino-suedtirol.it
+trentino.it
+trentinoa-adige.it
+trentinoaadige.it
+trentinoalto-adige.it
+trentinoaltoadige.it
+trentinos-tirol.it
+trentinostirol.it
+trentinosud-tirol.it
+xn--trentinosd-tirol-rzb.it
+trentinosüd-tirol.it
+trentinosudtirol.it
+xn--trentinosdtirol-7vb.it
+trentinosüdtirol.it
+trentinosued-tirol.it
+trentinosuedtirol.it
+trentinsud-tirol.it
+xn--trentinsd-tirol-6vb.it
+trentinsüd-tirol.it
+trentinsudtirol.it
+xn--trentinsdtirol-nsb.it
+trentinsüdtirol.it
+trentinsued-tirol.it
+trentinsuedtirol.it
+tuscany.it
+umb.it
+umbria.it
+val-d-aosta.it
+val-daosta.it
+vald-aosta.it
+valdaosta.it
+valle-aosta.it
+valle-d-aosta.it
+valle-daosta.it
+valleaosta.it
+valled-aosta.it
+valledaosta.it
+vallee-aoste.it
+xn--valle-aoste-ebb.it
+vallée-aoste.it
+vallee-d-aoste.it
+xn--valle-d-aoste-ehb.it
+vallée-d-aoste.it
+valleeaoste.it
+xn--valleaoste-e7a.it
+valléeaoste.it
+valleedaoste.it
+xn--valledaoste-ebb.it
+valléedaoste.it
+vao.it
+vda.it
+ven.it
+veneto.it
+// Provinces
+ag.it
+agrigento.it
+al.it
+alessandria.it
+alto-adige.it
+altoadige.it
+an.it
+ancona.it
+andria-barletta-trani.it
+andria-trani-barletta.it
+andriabarlettatrani.it
+andriatranibarletta.it
+ao.it
+aosta.it
+aoste.it
+ap.it
+aq.it
+aquila.it
+ar.it
+arezzo.it
+ascoli-piceno.it
+ascolipiceno.it
+asti.it
+at.it
+av.it
+avellino.it
+ba.it
+balsan-sudtirol.it
+xn--balsan-sdtirol-nsb.it
+balsan-südtirol.it
+balsan-suedtirol.it
+balsan.it
+bari.it
+barletta-trani-andria.it
+barlettatraniandria.it
+belluno.it
+benevento.it
+bergamo.it
+bg.it
+bi.it
+biella.it
+bl.it
+bn.it
+bo.it
+bologna.it
+bolzano-altoadige.it
+bolzano.it
+bozen-sudtirol.it
+xn--bozen-sdtirol-2ob.it
+bozen-südtirol.it
+bozen-suedtirol.it
+bozen.it
+br.it
+brescia.it
+brindisi.it
+bs.it
+bt.it
+bulsan-sudtirol.it
+xn--bulsan-sdtirol-nsb.it
+bulsan-südtirol.it
+bulsan-suedtirol.it
+bulsan.it
+bz.it
+ca.it
+cagliari.it
+caltanissetta.it
+campidano-medio.it
+campidanomedio.it
+campobasso.it
+carbonia-iglesias.it
+carboniaiglesias.it
+carrara-massa.it
+carraramassa.it
+caserta.it
+catania.it
+catanzaro.it
+cb.it
+ce.it
+cesena-forli.it
+xn--cesena-forl-mcb.it
+cesena-forlì.it
+cesenaforli.it
+xn--cesenaforl-i8a.it
+cesenaforlì.it
+ch.it
+chieti.it
+ci.it
+cl.it
+cn.it
+co.it
+como.it
+cosenza.it
+cr.it
+cremona.it
+crotone.it
+cs.it
+ct.it
+cuneo.it
+cz.it
+dell-ogliastra.it
+dellogliastra.it
+en.it
+enna.it
+fc.it
+fe.it
+fermo.it
+ferrara.it
+fg.it
+fi.it
+firenze.it
+florence.it
+fm.it
+foggia.it
+forli-cesena.it
+xn--forl-cesena-fcb.it
+forlì-cesena.it
+forlicesena.it
+xn--forlcesena-c8a.it
+forlìcesena.it
+fr.it
+frosinone.it
+ge.it
+genoa.it
+genova.it
+go.it
+gorizia.it
+gr.it
+grosseto.it
+iglesias-carbonia.it
+iglesiascarbonia.it
+im.it
+imperia.it
+is.it
+isernia.it
+kr.it
+la-spezia.it
+laquila.it
+laspezia.it
+latina.it
+lc.it
+le.it
+lecce.it
+lecco.it
+li.it
+livorno.it
+lo.it
+lodi.it
+lt.it
+lu.it
+lucca.it
+macerata.it
+mantova.it
+massa-carrara.it
+massacarrara.it
+matera.it
+mb.it
+mc.it
+me.it
+medio-campidano.it
+mediocampidano.it
+messina.it
+mi.it
+milan.it
+milano.it
+mn.it
+mo.it
+modena.it
+monza-brianza.it
+monza-e-della-brianza.it
+monza.it
+monzabrianza.it
+monzaebrianza.it
+monzaedellabrianza.it
+ms.it
+mt.it
+na.it
+naples.it
+napoli.it
+no.it
+novara.it
+nu.it
+nuoro.it
+og.it
+ogliastra.it
+olbia-tempio.it
+olbiatempio.it
+or.it
+oristano.it
+ot.it
+pa.it
+padova.it
+padua.it
+palermo.it
+parma.it
+pavia.it
+pc.it
+pd.it
+pe.it
+perugia.it
+pesaro-urbino.it
+pesarourbino.it
+pescara.it
+pg.it
+pi.it
+piacenza.it
+pisa.it
+pistoia.it
+pn.it
+po.it
+pordenone.it
+potenza.it
+pr.it
+prato.it
+pt.it
+pu.it
+pv.it
+pz.it
+ra.it
+ragusa.it
+ravenna.it
+rc.it
+re.it
+reggio-calabria.it
+reggio-emilia.it
+reggiocalabria.it
+reggioemilia.it
+rg.it
+ri.it
+rieti.it
+rimini.it
+rm.it
+rn.it
+ro.it
+roma.it
+rome.it
+rovigo.it
+sa.it
+salerno.it
+sassari.it
+savona.it
+si.it
+siena.it
+siracusa.it
+so.it
+sondrio.it
+sp.it
+sr.it
+ss.it
+suedtirol.it
+xn--sdtirol-n2a.it
+südtirol.it
+sv.it
+ta.it
+taranto.it
+te.it
+tempio-olbia.it
+tempioolbia.it
+teramo.it
+terni.it
+tn.it
+to.it
+torino.it
+tp.it
+tr.it
+trani-andria-barletta.it
+trani-barletta-andria.it
+traniandriabarletta.it
+tranibarlettaandria.it
+trapani.it
+trento.it
+treviso.it
+trieste.it
+ts.it
+turin.it
+tv.it
+ud.it
+udine.it
+urbino-pesaro.it
+urbinopesaro.it
+va.it
+varese.it
+vb.it
+vc.it
+ve.it
+venezia.it
+venice.it
+verbania.it
+vercelli.it
+verona.it
+vi.it
+vibo-valentia.it
+vibovalentia.it
+vicenza.it
+viterbo.it
+vr.it
+vs.it
+vt.it
+vv.it
+
+// je : http://www.channelisles.net/register-domains/
+// Confirmed by registry <nigel@channelisles.net> 2013-11-28
+je
+co.je
+net.je
+org.je
+
+// jm : http://www.com.jm/register.html
+*.jm
+
+// jo : http://www.dns.jo/Registration_policy.aspx
+jo
+com.jo
+org.jo
+net.jo
+edu.jo
+sch.jo
+gov.jo
+mil.jo
+name.jo
+
+// jobs : https://en.wikipedia.org/wiki/.jobs
+jobs
+
+// jp : https://en.wikipedia.org/wiki/.jp
+// http://jprs.co.jp/en/jpdomain.html
+// Submitted by registry <info@jprs.jp>
+jp
+// jp organizational type names
+ac.jp
+ad.jp
+co.jp
+ed.jp
+go.jp
+gr.jp
+lg.jp
+ne.jp
+or.jp
+// jp prefecture type names
+aichi.jp
+akita.jp
+aomori.jp
+chiba.jp
+ehime.jp
+fukui.jp
+fukuoka.jp
+fukushima.jp
+gifu.jp
+gunma.jp
+hiroshima.jp
+hokkaido.jp
+hyogo.jp
+ibaraki.jp
+ishikawa.jp
+iwate.jp
+kagawa.jp
+kagoshima.jp
+kanagawa.jp
+kochi.jp
+kumamoto.jp
+kyoto.jp
+mie.jp
+miyagi.jp
+miyazaki.jp
+nagano.jp
+nagasaki.jp
+nara.jp
+niigata.jp
+oita.jp
+okayama.jp
+okinawa.jp
+osaka.jp
+saga.jp
+saitama.jp
+shiga.jp
+shimane.jp
+shizuoka.jp
+tochigi.jp
+tokushima.jp
+tokyo.jp
+tottori.jp
+toyama.jp
+wakayama.jp
+yamagata.jp
+yamaguchi.jp
+yamanashi.jp
+xn--4pvxs.jp
+栃木.jp
+xn--vgu402c.jp
+愛知.jp
+xn--c3s14m.jp
+愛媛.jp
+xn--f6qx53a.jp
+兵庫.jp
+xn--8pvr4u.jp
+熊本.jp
+xn--uist22h.jp
+茨城.jp
+xn--djrs72d6uy.jp
+北海道.jp
+xn--mkru45i.jp
+千葉.jp
+xn--0trq7p7nn.jp
+和歌山.jp
+xn--8ltr62k.jp
+長崎.jp
+xn--2m4a15e.jp
+長野.jp
+xn--efvn9s.jp
+新潟.jp
+xn--32vp30h.jp
+青森.jp
+xn--4it797k.jp
+静岡.jp
+xn--1lqs71d.jp
+東京.jp
+xn--5rtp49c.jp
+石川.jp
+xn--5js045d.jp
+埼玉.jp
+xn--ehqz56n.jp
+三重.jp
+xn--1lqs03n.jp
+京都.jp
+xn--qqqt11m.jp
+佐賀.jp
+xn--kbrq7o.jp
+大分.jp
+xn--pssu33l.jp
+大阪.jp
+xn--ntsq17g.jp
+奈良.jp
+xn--uisz3g.jp
+宮城.jp
+xn--6btw5a.jp
+宮崎.jp
+xn--1ctwo.jp
+富山.jp
+xn--6orx2r.jp
+山口.jp
+xn--rht61e.jp
+山形.jp
+xn--rht27z.jp
+山梨.jp
+xn--djty4k.jp
+岩手.jp
+xn--nit225k.jp
+岐阜.jp
+xn--rht3d.jp
+岡山.jp
+xn--klty5x.jp
+島根.jp
+xn--kltx9a.jp
+広島.jp
+xn--kltp7d.jp
+徳島.jp
+xn--uuwu58a.jp
+沖縄.jp
+xn--zbx025d.jp
+滋賀.jp
+xn--ntso0iqx3a.jp
+神奈川.jp
+xn--elqq16h.jp
+福井.jp
+xn--4it168d.jp
+福岡.jp
+xn--klt787d.jp
+福島.jp
+xn--rny31h.jp
+秋田.jp
+xn--7t0a264c.jp
+群馬.jp
+xn--5rtq34k.jp
+香川.jp
+xn--k7yn95e.jp
+高知.jp
+xn--tor131o.jp
+鳥取.jp
+xn--d5qv7z876c.jp
+鹿児島.jp
+// jp geographic type names
+// http://jprs.jp/doc/rule/saisoku-1.html
+*.kawasaki.jp
+*.kitakyushu.jp
+*.kobe.jp
+*.nagoya.jp
+*.sapporo.jp
+*.sendai.jp
+*.yokohama.jp
+!city.kawasaki.jp
+!city.kitakyushu.jp
+!city.kobe.jp
+!city.nagoya.jp
+!city.sapporo.jp
+!city.sendai.jp
+!city.yokohama.jp
+// 4th level registration
+aisai.aichi.jp
+ama.aichi.jp
+anjo.aichi.jp
+asuke.aichi.jp
+chiryu.aichi.jp
+chita.aichi.jp
+fuso.aichi.jp
+gamagori.aichi.jp
+handa.aichi.jp
+hazu.aichi.jp
+hekinan.aichi.jp
+higashiura.aichi.jp
+ichinomiya.aichi.jp
+inazawa.aichi.jp
+inuyama.aichi.jp
+isshiki.aichi.jp
+iwakura.aichi.jp
+kanie.aichi.jp
+kariya.aichi.jp
+kasugai.aichi.jp
+kira.aichi.jp
+kiyosu.aichi.jp
+komaki.aichi.jp
+konan.aichi.jp
+kota.aichi.jp
+mihama.aichi.jp
+miyoshi.aichi.jp
+nishio.aichi.jp
+nisshin.aichi.jp
+obu.aichi.jp
+oguchi.aichi.jp
+oharu.aichi.jp
+okazaki.aichi.jp
+owariasahi.aichi.jp
+seto.aichi.jp
+shikatsu.aichi.jp
+shinshiro.aichi.jp
+shitara.aichi.jp
+tahara.aichi.jp
+takahama.aichi.jp
+tobishima.aichi.jp
+toei.aichi.jp
+togo.aichi.jp
+tokai.aichi.jp
+tokoname.aichi.jp
+toyoake.aichi.jp
+toyohashi.aichi.jp
+toyokawa.aichi.jp
+toyone.aichi.jp
+toyota.aichi.jp
+tsushima.aichi.jp
+yatomi.aichi.jp
+akita.akita.jp
+daisen.akita.jp
+fujisato.akita.jp
+gojome.akita.jp
+hachirogata.akita.jp
+happou.akita.jp
+higashinaruse.akita.jp
+honjo.akita.jp
+honjyo.akita.jp
+ikawa.akita.jp
+kamikoani.akita.jp
+kamioka.akita.jp
+katagami.akita.jp
+kazuno.akita.jp
+kitaakita.akita.jp
+kosaka.akita.jp
+kyowa.akita.jp
+misato.akita.jp
+mitane.akita.jp
+moriyoshi.akita.jp
+nikaho.akita.jp
+noshiro.akita.jp
+odate.akita.jp
+oga.akita.jp
+ogata.akita.jp
+semboku.akita.jp
+yokote.akita.jp
+yurihonjo.akita.jp
+aomori.aomori.jp
+gonohe.aomori.jp
+hachinohe.aomori.jp
+hashikami.aomori.jp
+hiranai.aomori.jp
+hirosaki.aomori.jp
+itayanagi.aomori.jp
+kuroishi.aomori.jp
+misawa.aomori.jp
+mutsu.aomori.jp
+nakadomari.aomori.jp
+noheji.aomori.jp
+oirase.aomori.jp
+owani.aomori.jp
+rokunohe.aomori.jp
+sannohe.aomori.jp
+shichinohe.aomori.jp
+shingo.aomori.jp
+takko.aomori.jp
+towada.aomori.jp
+tsugaru.aomori.jp
+tsuruta.aomori.jp
+abiko.chiba.jp
+asahi.chiba.jp
+chonan.chiba.jp
+chosei.chiba.jp
+choshi.chiba.jp
+chuo.chiba.jp
+funabashi.chiba.jp
+futtsu.chiba.jp
+hanamigawa.chiba.jp
+ichihara.chiba.jp
+ichikawa.chiba.jp
+ichinomiya.chiba.jp
+inzai.chiba.jp
+isumi.chiba.jp
+kamagaya.chiba.jp
+kamogawa.chiba.jp
+kashiwa.chiba.jp
+katori.chiba.jp
+katsuura.chiba.jp
+kimitsu.chiba.jp
+kisarazu.chiba.jp
+kozaki.chiba.jp
+kujukuri.chiba.jp
+kyonan.chiba.jp
+matsudo.chiba.jp
+midori.chiba.jp
+mihama.chiba.jp
+minamiboso.chiba.jp
+mobara.chiba.jp
+mutsuzawa.chiba.jp
+nagara.chiba.jp
+nagareyama.chiba.jp
+narashino.chiba.jp
+narita.chiba.jp
+noda.chiba.jp
+oamishirasato.chiba.jp
+omigawa.chiba.jp
+onjuku.chiba.jp
+otaki.chiba.jp
+sakae.chiba.jp
+sakura.chiba.jp
+shimofusa.chiba.jp
+shirako.chiba.jp
+shiroi.chiba.jp
+shisui.chiba.jp
+sodegaura.chiba.jp
+sosa.chiba.jp
+tako.chiba.jp
+tateyama.chiba.jp
+togane.chiba.jp
+tohnosho.chiba.jp
+tomisato.chiba.jp
+urayasu.chiba.jp
+yachimata.chiba.jp
+yachiyo.chiba.jp
+yokaichiba.chiba.jp
+yokoshibahikari.chiba.jp
+yotsukaido.chiba.jp
+ainan.ehime.jp
+honai.ehime.jp
+ikata.ehime.jp
+imabari.ehime.jp
+iyo.ehime.jp
+kamijima.ehime.jp
+kihoku.ehime.jp
+kumakogen.ehime.jp
+masaki.ehime.jp
+matsuno.ehime.jp
+matsuyama.ehime.jp
+namikata.ehime.jp
+niihama.ehime.jp
+ozu.ehime.jp
+saijo.ehime.jp
+seiyo.ehime.jp
+shikokuchuo.ehime.jp
+tobe.ehime.jp
+toon.ehime.jp
+uchiko.ehime.jp
+uwajima.ehime.jp
+yawatahama.ehime.jp
+echizen.fukui.jp
+eiheiji.fukui.jp
+fukui.fukui.jp
+ikeda.fukui.jp
+katsuyama.fukui.jp
+mihama.fukui.jp
+minamiechizen.fukui.jp
+obama.fukui.jp
+ohi.fukui.jp
+ono.fukui.jp
+sabae.fukui.jp
+sakai.fukui.jp
+takahama.fukui.jp
+tsuruga.fukui.jp
+wakasa.fukui.jp
+ashiya.fukuoka.jp
+buzen.fukuoka.jp
+chikugo.fukuoka.jp
+chikuho.fukuoka.jp
+chikujo.fukuoka.jp
+chikushino.fukuoka.jp
+chikuzen.fukuoka.jp
+chuo.fukuoka.jp
+dazaifu.fukuoka.jp
+fukuchi.fukuoka.jp
+hakata.fukuoka.jp
+higashi.fukuoka.jp
+hirokawa.fukuoka.jp
+hisayama.fukuoka.jp
+iizuka.fukuoka.jp
+inatsuki.fukuoka.jp
+kaho.fukuoka.jp
+kasuga.fukuoka.jp
+kasuya.fukuoka.jp
+kawara.fukuoka.jp
+keisen.fukuoka.jp
+koga.fukuoka.jp
+kurate.fukuoka.jp
+kurogi.fukuoka.jp
+kurume.fukuoka.jp
+minami.fukuoka.jp
+miyako.fukuoka.jp
+miyama.fukuoka.jp
+miyawaka.fukuoka.jp
+mizumaki.fukuoka.jp
+munakata.fukuoka.jp
+nakagawa.fukuoka.jp
+nakama.fukuoka.jp
+nishi.fukuoka.jp
+nogata.fukuoka.jp
+ogori.fukuoka.jp
+okagaki.fukuoka.jp
+okawa.fukuoka.jp
+oki.fukuoka.jp
+omuta.fukuoka.jp
+onga.fukuoka.jp
+onojo.fukuoka.jp
+oto.fukuoka.jp
+saigawa.fukuoka.jp
+sasaguri.fukuoka.jp
+shingu.fukuoka.jp
+shinyoshitomi.fukuoka.jp
+shonai.fukuoka.jp
+soeda.fukuoka.jp
+sue.fukuoka.jp
+tachiarai.fukuoka.jp
+tagawa.fukuoka.jp
+takata.fukuoka.jp
+toho.fukuoka.jp
+toyotsu.fukuoka.jp
+tsuiki.fukuoka.jp
+ukiha.fukuoka.jp
+umi.fukuoka.jp
+usui.fukuoka.jp
+yamada.fukuoka.jp
+yame.fukuoka.jp
+yanagawa.fukuoka.jp
+yukuhashi.fukuoka.jp
+aizubange.fukushima.jp
+aizumisato.fukushima.jp
+aizuwakamatsu.fukushima.jp
+asakawa.fukushima.jp
+bandai.fukushima.jp
+date.fukushima.jp
+fukushima.fukushima.jp
+furudono.fukushima.jp
+futaba.fukushima.jp
+hanawa.fukushima.jp
+higashi.fukushima.jp
+hirata.fukushima.jp
+hirono.fukushima.jp
+iitate.fukushima.jp
+inawashiro.fukushima.jp
+ishikawa.fukushima.jp
+iwaki.fukushima.jp
+izumizaki.fukushima.jp
+kagamiishi.fukushima.jp
+kaneyama.fukushima.jp
+kawamata.fukushima.jp
+kitakata.fukushima.jp
+kitashiobara.fukushima.jp
+koori.fukushima.jp
+koriyama.fukushima.jp
+kunimi.fukushima.jp
+miharu.fukushima.jp
+mishima.fukushima.jp
+namie.fukushima.jp
+nango.fukushima.jp
+nishiaizu.fukushima.jp
+nishigo.fukushima.jp
+okuma.fukushima.jp
+omotego.fukushima.jp
+ono.fukushima.jp
+otama.fukushima.jp
+samegawa.fukushima.jp
+shimogo.fukushima.jp
+shirakawa.fukushima.jp
+showa.fukushima.jp
+soma.fukushima.jp
+sukagawa.fukushima.jp
+taishin.fukushima.jp
+tamakawa.fukushima.jp
+tanagura.fukushima.jp
+tenei.fukushima.jp
+yabuki.fukushima.jp
+yamato.fukushima.jp
+yamatsuri.fukushima.jp
+yanaizu.fukushima.jp
+yugawa.fukushima.jp
+anpachi.gifu.jp
+ena.gifu.jp
+gifu.gifu.jp
+ginan.gifu.jp
+godo.gifu.jp
+gujo.gifu.jp
+hashima.gifu.jp
+hichiso.gifu.jp
+hida.gifu.jp
+higashishirakawa.gifu.jp
+ibigawa.gifu.jp
+ikeda.gifu.jp
+kakamigahara.gifu.jp
+kani.gifu.jp
+kasahara.gifu.jp
+kasamatsu.gifu.jp
+kawaue.gifu.jp
+kitagata.gifu.jp
+mino.gifu.jp
+minokamo.gifu.jp
+mitake.gifu.jp
+mizunami.gifu.jp
+motosu.gifu.jp
+nakatsugawa.gifu.jp
+ogaki.gifu.jp
+sakahogi.gifu.jp
+seki.gifu.jp
+sekigahara.gifu.jp
+shirakawa.gifu.jp
+tajimi.gifu.jp
+takayama.gifu.jp
+tarui.gifu.jp
+toki.gifu.jp
+tomika.gifu.jp
+wanouchi.gifu.jp
+yamagata.gifu.jp
+yaotsu.gifu.jp
+yoro.gifu.jp
+annaka.gunma.jp
+chiyoda.gunma.jp
+fujioka.gunma.jp
+higashiagatsuma.gunma.jp
+isesaki.gunma.jp
+itakura.gunma.jp
+kanna.gunma.jp
+kanra.gunma.jp
+katashina.gunma.jp
+kawaba.gunma.jp
+kiryu.gunma.jp
+kusatsu.gunma.jp
+maebashi.gunma.jp
+meiwa.gunma.jp
+midori.gunma.jp
+minakami.gunma.jp
+naganohara.gunma.jp
+nakanojo.gunma.jp
+nanmoku.gunma.jp
+numata.gunma.jp
+oizumi.gunma.jp
+ora.gunma.jp
+ota.gunma.jp
+shibukawa.gunma.jp
+shimonita.gunma.jp
+shinto.gunma.jp
+showa.gunma.jp
+takasaki.gunma.jp
+takayama.gunma.jp
+tamamura.gunma.jp
+tatebayashi.gunma.jp
+tomioka.gunma.jp
+tsukiyono.gunma.jp
+tsumagoi.gunma.jp
+ueno.gunma.jp
+yoshioka.gunma.jp
+asaminami.hiroshima.jp
+daiwa.hiroshima.jp
+etajima.hiroshima.jp
+fuchu.hiroshima.jp
+fukuyama.hiroshima.jp
+hatsukaichi.hiroshima.jp
+higashihiroshima.hiroshima.jp
+hongo.hiroshima.jp
+jinsekikogen.hiroshima.jp
+kaita.hiroshima.jp
+kui.hiroshima.jp
+kumano.hiroshima.jp
+kure.hiroshima.jp
+mihara.hiroshima.jp
+miyoshi.hiroshima.jp
+naka.hiroshima.jp
+onomichi.hiroshima.jp
+osakikamijima.hiroshima.jp
+otake.hiroshima.jp
+saka.hiroshima.jp
+sera.hiroshima.jp
+seranishi.hiroshima.jp
+shinichi.hiroshima.jp
+shobara.hiroshima.jp
+takehara.hiroshima.jp
+abashiri.hokkaido.jp
+abira.hokkaido.jp
+aibetsu.hokkaido.jp
+akabira.hokkaido.jp
+akkeshi.hokkaido.jp
+asahikawa.hokkaido.jp
+ashibetsu.hokkaido.jp
+ashoro.hokkaido.jp
+assabu.hokkaido.jp
+atsuma.hokkaido.jp
+bibai.hokkaido.jp
+biei.hokkaido.jp
+bifuka.hokkaido.jp
+bihoro.hokkaido.jp
+biratori.hokkaido.jp
+chippubetsu.hokkaido.jp
+chitose.hokkaido.jp
+date.hokkaido.jp
+ebetsu.hokkaido.jp
+embetsu.hokkaido.jp
+eniwa.hokkaido.jp
+erimo.hokkaido.jp
+esan.hokkaido.jp
+esashi.hokkaido.jp
+fukagawa.hokkaido.jp
+fukushima.hokkaido.jp
+furano.hokkaido.jp
+furubira.hokkaido.jp
+haboro.hokkaido.jp
+hakodate.hokkaido.jp
+hamatonbetsu.hokkaido.jp
+hidaka.hokkaido.jp
+higashikagura.hokkaido.jp
+higashikawa.hokkaido.jp
+hiroo.hokkaido.jp
+hokuryu.hokkaido.jp
+hokuto.hokkaido.jp
+honbetsu.hokkaido.jp
+horokanai.hokkaido.jp
+horonobe.hokkaido.jp
+ikeda.hokkaido.jp
+imakane.hokkaido.jp
+ishikari.hokkaido.jp
+iwamizawa.hokkaido.jp
+iwanai.hokkaido.jp
+kamifurano.hokkaido.jp
+kamikawa.hokkaido.jp
+kamishihoro.hokkaido.jp
+kamisunagawa.hokkaido.jp
+kamoenai.hokkaido.jp
+kayabe.hokkaido.jp
+kembuchi.hokkaido.jp
+kikonai.hokkaido.jp
+kimobetsu.hokkaido.jp
+kitahiroshima.hokkaido.jp
+kitami.hokkaido.jp
+kiyosato.hokkaido.jp
+koshimizu.hokkaido.jp
+kunneppu.hokkaido.jp
+kuriyama.hokkaido.jp
+kuromatsunai.hokkaido.jp
+kushiro.hokkaido.jp
+kutchan.hokkaido.jp
+kyowa.hokkaido.jp
+mashike.hokkaido.jp
+matsumae.hokkaido.jp
+mikasa.hokkaido.jp
+minamifurano.hokkaido.jp
+mombetsu.hokkaido.jp
+moseushi.hokkaido.jp
+mukawa.hokkaido.jp
+muroran.hokkaido.jp
+naie.hokkaido.jp
+nakagawa.hokkaido.jp
+nakasatsunai.hokkaido.jp
+nakatombetsu.hokkaido.jp
+nanae.hokkaido.jp
+nanporo.hokkaido.jp
+nayoro.hokkaido.jp
+nemuro.hokkaido.jp
+niikappu.hokkaido.jp
+niki.hokkaido.jp
+nishiokoppe.hokkaido.jp
+noboribetsu.hokkaido.jp
+numata.hokkaido.jp
+obihiro.hokkaido.jp
+obira.hokkaido.jp
+oketo.hokkaido.jp
+okoppe.hokkaido.jp
+otaru.hokkaido.jp
+otobe.hokkaido.jp
+otofuke.hokkaido.jp
+otoineppu.hokkaido.jp
+oumu.hokkaido.jp
+ozora.hokkaido.jp
+pippu.hokkaido.jp
+rankoshi.hokkaido.jp
+rebun.hokkaido.jp
+rikubetsu.hokkaido.jp
+rishiri.hokkaido.jp
+rishirifuji.hokkaido.jp
+saroma.hokkaido.jp
+sarufutsu.hokkaido.jp
+shakotan.hokkaido.jp
+shari.hokkaido.jp
+shibecha.hokkaido.jp
+shibetsu.hokkaido.jp
+shikabe.hokkaido.jp
+shikaoi.hokkaido.jp
+shimamaki.hokkaido.jp
+shimizu.hokkaido.jp
+shimokawa.hokkaido.jp
+shinshinotsu.hokkaido.jp
+shintoku.hokkaido.jp
+shiranuka.hokkaido.jp
+shiraoi.hokkaido.jp
+shiriuchi.hokkaido.jp
+sobetsu.hokkaido.jp
+sunagawa.hokkaido.jp
+taiki.hokkaido.jp
+takasu.hokkaido.jp
+takikawa.hokkaido.jp
+takinoue.hokkaido.jp
+teshikaga.hokkaido.jp
+tobetsu.hokkaido.jp
+tohma.hokkaido.jp
+tomakomai.hokkaido.jp
+tomari.hokkaido.jp
+toya.hokkaido.jp
+toyako.hokkaido.jp
+toyotomi.hokkaido.jp
+toyoura.hokkaido.jp
+tsubetsu.hokkaido.jp
+tsukigata.hokkaido.jp
+urakawa.hokkaido.jp
+urausu.hokkaido.jp
+uryu.hokkaido.jp
+utashinai.hokkaido.jp
+wakkanai.hokkaido.jp
+wassamu.hokkaido.jp
+yakumo.hokkaido.jp
+yoichi.hokkaido.jp
+aioi.hyogo.jp
+akashi.hyogo.jp
+ako.hyogo.jp
+amagasaki.hyogo.jp
+aogaki.hyogo.jp
+asago.hyogo.jp
+ashiya.hyogo.jp
+awaji.hyogo.jp
+fukusaki.hyogo.jp
+goshiki.hyogo.jp
+harima.hyogo.jp
+himeji.hyogo.jp
+ichikawa.hyogo.jp
+inagawa.hyogo.jp
+itami.hyogo.jp
+kakogawa.hyogo.jp
+kamigori.hyogo.jp
+kamikawa.hyogo.jp
+kasai.hyogo.jp
+kasuga.hyogo.jp
+kawanishi.hyogo.jp
+miki.hyogo.jp
+minamiawaji.hyogo.jp
+nishinomiya.hyogo.jp
+nishiwaki.hyogo.jp
+ono.hyogo.jp
+sanda.hyogo.jp
+sannan.hyogo.jp
+sasayama.hyogo.jp
+sayo.hyogo.jp
+shingu.hyogo.jp
+shinonsen.hyogo.jp
+shiso.hyogo.jp
+sumoto.hyogo.jp
+taishi.hyogo.jp
+taka.hyogo.jp
+takarazuka.hyogo.jp
+takasago.hyogo.jp
+takino.hyogo.jp
+tamba.hyogo.jp
+tatsuno.hyogo.jp
+toyooka.hyogo.jp
+yabu.hyogo.jp
+yashiro.hyogo.jp
+yoka.hyogo.jp
+yokawa.hyogo.jp
+ami.ibaraki.jp
+asahi.ibaraki.jp
+bando.ibaraki.jp
+chikusei.ibaraki.jp
+daigo.ibaraki.jp
+fujishiro.ibaraki.jp
+hitachi.ibaraki.jp
+hitachinaka.ibaraki.jp
+hitachiomiya.ibaraki.jp
+hitachiota.ibaraki.jp
+ibaraki.ibaraki.jp
+ina.ibaraki.jp
+inashiki.ibaraki.jp
+itako.ibaraki.jp
+iwama.ibaraki.jp
+joso.ibaraki.jp
+kamisu.ibaraki.jp
+kasama.ibaraki.jp
+kashima.ibaraki.jp
+kasumigaura.ibaraki.jp
+koga.ibaraki.jp
+miho.ibaraki.jp
+mito.ibaraki.jp
+moriya.ibaraki.jp
+naka.ibaraki.jp
+namegata.ibaraki.jp
+oarai.ibaraki.jp
+ogawa.ibaraki.jp
+omitama.ibaraki.jp
+ryugasaki.ibaraki.jp
+sakai.ibaraki.jp
+sakuragawa.ibaraki.jp
+shimodate.ibaraki.jp
+shimotsuma.ibaraki.jp
+shirosato.ibaraki.jp
+sowa.ibaraki.jp
+suifu.ibaraki.jp
+takahagi.ibaraki.jp
+tamatsukuri.ibaraki.jp
+tokai.ibaraki.jp
+tomobe.ibaraki.jp
+tone.ibaraki.jp
+toride.ibaraki.jp
+tsuchiura.ibaraki.jp
+tsukuba.ibaraki.jp
+uchihara.ibaraki.jp
+ushiku.ibaraki.jp
+yachiyo.ibaraki.jp
+yamagata.ibaraki.jp
+yawara.ibaraki.jp
+yuki.ibaraki.jp
+anamizu.ishikawa.jp
+hakui.ishikawa.jp
+hakusan.ishikawa.jp
+kaga.ishikawa.jp
+kahoku.ishikawa.jp
+kanazawa.ishikawa.jp
+kawakita.ishikawa.jp
+komatsu.ishikawa.jp
+nakanoto.ishikawa.jp
+nanao.ishikawa.jp
+nomi.ishikawa.jp
+nonoichi.ishikawa.jp
+noto.ishikawa.jp
+shika.ishikawa.jp
+suzu.ishikawa.jp
+tsubata.ishikawa.jp
+tsurugi.ishikawa.jp
+uchinada.ishikawa.jp
+wajima.ishikawa.jp
+fudai.iwate.jp
+fujisawa.iwate.jp
+hanamaki.iwate.jp
+hiraizumi.iwate.jp
+hirono.iwate.jp
+ichinohe.iwate.jp
+ichinoseki.iwate.jp
+iwaizumi.iwate.jp
+iwate.iwate.jp
+joboji.iwate.jp
+kamaishi.iwate.jp
+kanegasaki.iwate.jp
+karumai.iwate.jp
+kawai.iwate.jp
+kitakami.iwate.jp
+kuji.iwate.jp
+kunohe.iwate.jp
+kuzumaki.iwate.jp
+miyako.iwate.jp
+mizusawa.iwate.jp
+morioka.iwate.jp
+ninohe.iwate.jp
+noda.iwate.jp
+ofunato.iwate.jp
+oshu.iwate.jp
+otsuchi.iwate.jp
+rikuzentakata.iwate.jp
+shiwa.iwate.jp
+shizukuishi.iwate.jp
+sumita.iwate.jp
+tanohata.iwate.jp
+tono.iwate.jp
+yahaba.iwate.jp
+yamada.iwate.jp
+ayagawa.kagawa.jp
+higashikagawa.kagawa.jp
+kanonji.kagawa.jp
+kotohira.kagawa.jp
+manno.kagawa.jp
+marugame.kagawa.jp
+mitoyo.kagawa.jp
+naoshima.kagawa.jp
+sanuki.kagawa.jp
+tadotsu.kagawa.jp
+takamatsu.kagawa.jp
+tonosho.kagawa.jp
+uchinomi.kagawa.jp
+utazu.kagawa.jp
+zentsuji.kagawa.jp
+akune.kagoshima.jp
+amami.kagoshima.jp
+hioki.kagoshima.jp
+isa.kagoshima.jp
+isen.kagoshima.jp
+izumi.kagoshima.jp
+kagoshima.kagoshima.jp
+kanoya.kagoshima.jp
+kawanabe.kagoshima.jp
+kinko.kagoshima.jp
+kouyama.kagoshima.jp
+makurazaki.kagoshima.jp
+matsumoto.kagoshima.jp
+minamitane.kagoshima.jp
+nakatane.kagoshima.jp
+nishinoomote.kagoshima.jp
+satsumasendai.kagoshima.jp
+soo.kagoshima.jp
+tarumizu.kagoshima.jp
+yusui.kagoshima.jp
+aikawa.kanagawa.jp
+atsugi.kanagawa.jp
+ayase.kanagawa.jp
+chigasaki.kanagawa.jp
+ebina.kanagawa.jp
+fujisawa.kanagawa.jp
+hadano.kanagawa.jp
+hakone.kanagawa.jp
+hiratsuka.kanagawa.jp
+isehara.kanagawa.jp
+kaisei.kanagawa.jp
+kamakura.kanagawa.jp
+kiyokawa.kanagawa.jp
+matsuda.kanagawa.jp
+minamiashigara.kanagawa.jp
+miura.kanagawa.jp
+nakai.kanagawa.jp
+ninomiya.kanagawa.jp
+odawara.kanagawa.jp
+oi.kanagawa.jp
+oiso.kanagawa.jp
+sagamihara.kanagawa.jp
+samukawa.kanagawa.jp
+tsukui.kanagawa.jp
+yamakita.kanagawa.jp
+yamato.kanagawa.jp
+yokosuka.kanagawa.jp
+yugawara.kanagawa.jp
+zama.kanagawa.jp
+zushi.kanagawa.jp
+aki.kochi.jp
+geisei.kochi.jp
+hidaka.kochi.jp
+higashitsuno.kochi.jp
+ino.kochi.jp
+kagami.kochi.jp
+kami.kochi.jp
+kitagawa.kochi.jp
+kochi.kochi.jp
+mihara.kochi.jp
+motoyama.kochi.jp
+muroto.kochi.jp
+nahari.kochi.jp
+nakamura.kochi.jp
+nankoku.kochi.jp
+nishitosa.kochi.jp
+niyodogawa.kochi.jp
+ochi.kochi.jp
+okawa.kochi.jp
+otoyo.kochi.jp
+otsuki.kochi.jp
+sakawa.kochi.jp
+sukumo.kochi.jp
+susaki.kochi.jp
+tosa.kochi.jp
+tosashimizu.kochi.jp
+toyo.kochi.jp
+tsuno.kochi.jp
+umaji.kochi.jp
+yasuda.kochi.jp
+yusuhara.kochi.jp
+amakusa.kumamoto.jp
+arao.kumamoto.jp
+aso.kumamoto.jp
+choyo.kumamoto.jp
+gyokuto.kumamoto.jp
+kamiamakusa.kumamoto.jp
+kikuchi.kumamoto.jp
+kumamoto.kumamoto.jp
+mashiki.kumamoto.jp
+mifune.kumamoto.jp
+minamata.kumamoto.jp
+minamioguni.kumamoto.jp
+nagasu.kumamoto.jp
+nishihara.kumamoto.jp
+oguni.kumamoto.jp
+ozu.kumamoto.jp
+sumoto.kumamoto.jp
+takamori.kumamoto.jp
+uki.kumamoto.jp
+uto.kumamoto.jp
+yamaga.kumamoto.jp
+yamato.kumamoto.jp
+yatsushiro.kumamoto.jp
+ayabe.kyoto.jp
+fukuchiyama.kyoto.jp
+higashiyama.kyoto.jp
+ide.kyoto.jp
+ine.kyoto.jp
+joyo.kyoto.jp
+kameoka.kyoto.jp
+kamo.kyoto.jp
+kita.kyoto.jp
+kizu.kyoto.jp
+kumiyama.kyoto.jp
+kyotamba.kyoto.jp
+kyotanabe.kyoto.jp
+kyotango.kyoto.jp
+maizuru.kyoto.jp
+minami.kyoto.jp
+minamiyamashiro.kyoto.jp
+miyazu.kyoto.jp
+muko.kyoto.jp
+nagaokakyo.kyoto.jp
+nakagyo.kyoto.jp
+nantan.kyoto.jp
+oyamazaki.kyoto.jp
+sakyo.kyoto.jp
+seika.kyoto.jp
+tanabe.kyoto.jp
+uji.kyoto.jp
+ujitawara.kyoto.jp
+wazuka.kyoto.jp
+yamashina.kyoto.jp
+yawata.kyoto.jp
+asahi.mie.jp
+inabe.mie.jp
+ise.mie.jp
+kameyama.mie.jp
+kawagoe.mie.jp
+kiho.mie.jp
+kisosaki.mie.jp
+kiwa.mie.jp
+komono.mie.jp
+kumano.mie.jp
+kuwana.mie.jp
+matsusaka.mie.jp
+meiwa.mie.jp
+mihama.mie.jp
+minamiise.mie.jp
+misugi.mie.jp
+miyama.mie.jp
+nabari.mie.jp
+shima.mie.jp
+suzuka.mie.jp
+tado.mie.jp
+taiki.mie.jp
+taki.mie.jp
+tamaki.mie.jp
+toba.mie.jp
+tsu.mie.jp
+udono.mie.jp
+ureshino.mie.jp
+watarai.mie.jp
+yokkaichi.mie.jp
+furukawa.miyagi.jp
+higashimatsushima.miyagi.jp
+ishinomaki.miyagi.jp
+iwanuma.miyagi.jp
+kakuda.miyagi.jp
+kami.miyagi.jp
+kawasaki.miyagi.jp
+marumori.miyagi.jp
+matsushima.miyagi.jp
+minamisanriku.miyagi.jp
+misato.miyagi.jp
+murata.miyagi.jp
+natori.miyagi.jp
+ogawara.miyagi.jp
+ohira.miyagi.jp
+onagawa.miyagi.jp
+osaki.miyagi.jp
+rifu.miyagi.jp
+semine.miyagi.jp
+shibata.miyagi.jp
+shichikashuku.miyagi.jp
+shikama.miyagi.jp
+shiogama.miyagi.jp
+shiroishi.miyagi.jp
+tagajo.miyagi.jp
+taiwa.miyagi.jp
+tome.miyagi.jp
+tomiya.miyagi.jp
+wakuya.miyagi.jp
+watari.miyagi.jp
+yamamoto.miyagi.jp
+zao.miyagi.jp
+aya.miyazaki.jp
+ebino.miyazaki.jp
+gokase.miyazaki.jp
+hyuga.miyazaki.jp
+kadogawa.miyazaki.jp
+kawaminami.miyazaki.jp
+kijo.miyazaki.jp
+kitagawa.miyazaki.jp
+kitakata.miyazaki.jp
+kitaura.miyazaki.jp
+kobayashi.miyazaki.jp
+kunitomi.miyazaki.jp
+kushima.miyazaki.jp
+mimata.miyazaki.jp
+miyakonojo.miyazaki.jp
+miyazaki.miyazaki.jp
+morotsuka.miyazaki.jp
+nichinan.miyazaki.jp
+nishimera.miyazaki.jp
+nobeoka.miyazaki.jp
+saito.miyazaki.jp
+shiiba.miyazaki.jp
+shintomi.miyazaki.jp
+takaharu.miyazaki.jp
+takanabe.miyazaki.jp
+takazaki.miyazaki.jp
+tsuno.miyazaki.jp
+achi.nagano.jp
+agematsu.nagano.jp
+anan.nagano.jp
+aoki.nagano.jp
+asahi.nagano.jp
+azumino.nagano.jp
+chikuhoku.nagano.jp
+chikuma.nagano.jp
+chino.nagano.jp
+fujimi.nagano.jp
+hakuba.nagano.jp
+hara.nagano.jp
+hiraya.nagano.jp
+iida.nagano.jp
+iijima.nagano.jp
+iiyama.nagano.jp
+iizuna.nagano.jp
+ikeda.nagano.jp
+ikusaka.nagano.jp
+ina.nagano.jp
+karuizawa.nagano.jp
+kawakami.nagano.jp
+kiso.nagano.jp
+kisofukushima.nagano.jp
+kitaaiki.nagano.jp
+komagane.nagano.jp
+komoro.nagano.jp
+matsukawa.nagano.jp
+matsumoto.nagano.jp
+miasa.nagano.jp
+minamiaiki.nagano.jp
+minamimaki.nagano.jp
+minamiminowa.nagano.jp
+minowa.nagano.jp
+miyada.nagano.jp
+miyota.nagano.jp
+mochizuki.nagano.jp
+nagano.nagano.jp
+nagawa.nagano.jp
+nagiso.nagano.jp
+nakagawa.nagano.jp
+nakano.nagano.jp
+nozawaonsen.nagano.jp
+obuse.nagano.jp
+ogawa.nagano.jp
+okaya.nagano.jp
+omachi.nagano.jp
+omi.nagano.jp
+ookuwa.nagano.jp
+ooshika.nagano.jp
+otaki.nagano.jp
+otari.nagano.jp
+sakae.nagano.jp
+sakaki.nagano.jp
+saku.nagano.jp
+sakuho.nagano.jp
+shimosuwa.nagano.jp
+shinanomachi.nagano.jp
+shiojiri.nagano.jp
+suwa.nagano.jp
+suzaka.nagano.jp
+takagi.nagano.jp
+takamori.nagano.jp
+takayama.nagano.jp
+tateshina.nagano.jp
+tatsuno.nagano.jp
+togakushi.nagano.jp
+togura.nagano.jp
+tomi.nagano.jp
+ueda.nagano.jp
+wada.nagano.jp
+yamagata.nagano.jp
+yamanouchi.nagano.jp
+yasaka.nagano.jp
+yasuoka.nagano.jp
+chijiwa.nagasaki.jp
+futsu.nagasaki.jp
+goto.nagasaki.jp
+hasami.nagasaki.jp
+hirado.nagasaki.jp
+iki.nagasaki.jp
+isahaya.nagasaki.jp
+kawatana.nagasaki.jp
+kuchinotsu.nagasaki.jp
+matsuura.nagasaki.jp
+nagasaki.nagasaki.jp
+obama.nagasaki.jp
+omura.nagasaki.jp
+oseto.nagasaki.jp
+saikai.nagasaki.jp
+sasebo.nagasaki.jp
+seihi.nagasaki.jp
+shimabara.nagasaki.jp
+shinkamigoto.nagasaki.jp
+togitsu.nagasaki.jp
+tsushima.nagasaki.jp
+unzen.nagasaki.jp
+ando.nara.jp
+gose.nara.jp
+heguri.nara.jp
+higashiyoshino.nara.jp
+ikaruga.nara.jp
+ikoma.nara.jp
+kamikitayama.nara.jp
+kanmaki.nara.jp
+kashiba.nara.jp
+kashihara.nara.jp
+katsuragi.nara.jp
+kawai.nara.jp
+kawakami.nara.jp
+kawanishi.nara.jp
+koryo.nara.jp
+kurotaki.nara.jp
+mitsue.nara.jp
+miyake.nara.jp
+nara.nara.jp
+nosegawa.nara.jp
+oji.nara.jp
+ouda.nara.jp
+oyodo.nara.jp
+sakurai.nara.jp
+sango.nara.jp
+shimoichi.nara.jp
+shimokitayama.nara.jp
+shinjo.nara.jp
+soni.nara.jp
+takatori.nara.jp
+tawaramoto.nara.jp
+tenkawa.nara.jp
+tenri.nara.jp
+uda.nara.jp
+yamatokoriyama.nara.jp
+yamatotakada.nara.jp
+yamazoe.nara.jp
+yoshino.nara.jp
+aga.niigata.jp
+agano.niigata.jp
+gosen.niigata.jp
+itoigawa.niigata.jp
+izumozaki.niigata.jp
+joetsu.niigata.jp
+kamo.niigata.jp
+kariwa.niigata.jp
+kashiwazaki.niigata.jp
+minamiuonuma.niigata.jp
+mitsuke.niigata.jp
+muika.niigata.jp
+murakami.niigata.jp
+myoko.niigata.jp
+nagaoka.niigata.jp
+niigata.niigata.jp
+ojiya.niigata.jp
+omi.niigata.jp
+sado.niigata.jp
+sanjo.niigata.jp
+seiro.niigata.jp
+seirou.niigata.jp
+sekikawa.niigata.jp
+shibata.niigata.jp
+tagami.niigata.jp
+tainai.niigata.jp
+tochio.niigata.jp
+tokamachi.niigata.jp
+tsubame.niigata.jp
+tsunan.niigata.jp
+uonuma.niigata.jp
+yahiko.niigata.jp
+yoita.niigata.jp
+yuzawa.niigata.jp
+beppu.oita.jp
+bungoono.oita.jp
+bungotakada.oita.jp
+hasama.oita.jp
+hiji.oita.jp
+himeshima.oita.jp
+hita.oita.jp
+kamitsue.oita.jp
+kokonoe.oita.jp
+kuju.oita.jp
+kunisaki.oita.jp
+kusu.oita.jp
+oita.oita.jp
+saiki.oita.jp
+taketa.oita.jp
+tsukumi.oita.jp
+usa.oita.jp
+usuki.oita.jp
+yufu.oita.jp
+akaiwa.okayama.jp
+asakuchi.okayama.jp
+bizen.okayama.jp
+hayashima.okayama.jp
+ibara.okayama.jp
+kagamino.okayama.jp
+kasaoka.okayama.jp
+kibichuo.okayama.jp
+kumenan.okayama.jp
+kurashiki.okayama.jp
+maniwa.okayama.jp
+misaki.okayama.jp
+nagi.okayama.jp
+niimi.okayama.jp
+nishiawakura.okayama.jp
+okayama.okayama.jp
+satosho.okayama.jp
+setouchi.okayama.jp
+shinjo.okayama.jp
+shoo.okayama.jp
+soja.okayama.jp
+takahashi.okayama.jp
+tamano.okayama.jp
+tsuyama.okayama.jp
+wake.okayama.jp
+yakage.okayama.jp
+aguni.okinawa.jp
+ginowan.okinawa.jp
+ginoza.okinawa.jp
+gushikami.okinawa.jp
+haebaru.okinawa.jp
+higashi.okinawa.jp
+hirara.okinawa.jp
+iheya.okinawa.jp
+ishigaki.okinawa.jp
+ishikawa.okinawa.jp
+itoman.okinawa.jp
+izena.okinawa.jp
+kadena.okinawa.jp
+kin.okinawa.jp
+kitadaito.okinawa.jp
+kitanakagusuku.okinawa.jp
+kumejima.okinawa.jp
+kunigami.okinawa.jp
+minamidaito.okinawa.jp
+motobu.okinawa.jp
+nago.okinawa.jp
+naha.okinawa.jp
+nakagusuku.okinawa.jp
+nakijin.okinawa.jp
+nanjo.okinawa.jp
+nishihara.okinawa.jp
+ogimi.okinawa.jp
+okinawa.okinawa.jp
+onna.okinawa.jp
+shimoji.okinawa.jp
+taketomi.okinawa.jp
+tarama.okinawa.jp
+tokashiki.okinawa.jp
+tomigusuku.okinawa.jp
+tonaki.okinawa.jp
+urasoe.okinawa.jp
+uruma.okinawa.jp
+yaese.okinawa.jp
+yomitan.okinawa.jp
+yonabaru.okinawa.jp
+yonaguni.okinawa.jp
+zamami.okinawa.jp
+abeno.osaka.jp
+chihayaakasaka.osaka.jp
+chuo.osaka.jp
+daito.osaka.jp
+fujiidera.osaka.jp
+habikino.osaka.jp
+hannan.osaka.jp
+higashiosaka.osaka.jp
+higashisumiyoshi.osaka.jp
+higashiyodogawa.osaka.jp
+hirakata.osaka.jp
+ibaraki.osaka.jp
+ikeda.osaka.jp
+izumi.osaka.jp
+izumiotsu.osaka.jp
+izumisano.osaka.jp
+kadoma.osaka.jp
+kaizuka.osaka.jp
+kanan.osaka.jp
+kashiwara.osaka.jp
+katano.osaka.jp
+kawachinagano.osaka.jp
+kishiwada.osaka.jp
+kita.osaka.jp
+kumatori.osaka.jp
+matsubara.osaka.jp
+minato.osaka.jp
+minoh.osaka.jp
+misaki.osaka.jp
+moriguchi.osaka.jp
+neyagawa.osaka.jp
+nishi.osaka.jp
+nose.osaka.jp
+osakasayama.osaka.jp
+sakai.osaka.jp
+sayama.osaka.jp
+sennan.osaka.jp
+settsu.osaka.jp
+shijonawate.osaka.jp
+shimamoto.osaka.jp
+suita.osaka.jp
+tadaoka.osaka.jp
+taishi.osaka.jp
+tajiri.osaka.jp
+takaishi.osaka.jp
+takatsuki.osaka.jp
+tondabayashi.osaka.jp
+toyonaka.osaka.jp
+toyono.osaka.jp
+yao.osaka.jp
+ariake.saga.jp
+arita.saga.jp
+fukudomi.saga.jp
+genkai.saga.jp
+hamatama.saga.jp
+hizen.saga.jp
+imari.saga.jp
+kamimine.saga.jp
+kanzaki.saga.jp
+karatsu.saga.jp
+kashima.saga.jp
+kitagata.saga.jp
+kitahata.saga.jp
+kiyama.saga.jp
+kouhoku.saga.jp
+kyuragi.saga.jp
+nishiarita.saga.jp
+ogi.saga.jp
+omachi.saga.jp
+ouchi.saga.jp
+saga.saga.jp
+shiroishi.saga.jp
+taku.saga.jp
+tara.saga.jp
+tosu.saga.jp
+yoshinogari.saga.jp
+arakawa.saitama.jp
+asaka.saitama.jp
+chichibu.saitama.jp
+fujimi.saitama.jp
+fujimino.saitama.jp
+fukaya.saitama.jp
+hanno.saitama.jp
+hanyu.saitama.jp
+hasuda.saitama.jp
+hatogaya.saitama.jp
+hatoyama.saitama.jp
+hidaka.saitama.jp
+higashichichibu.saitama.jp
+higashimatsuyama.saitama.jp
+honjo.saitama.jp
+ina.saitama.jp
+iruma.saitama.jp
+iwatsuki.saitama.jp
+kamiizumi.saitama.jp
+kamikawa.saitama.jp
+kamisato.saitama.jp
+kasukabe.saitama.jp
+kawagoe.saitama.jp
+kawaguchi.saitama.jp
+kawajima.saitama.jp
+kazo.saitama.jp
+kitamoto.saitama.jp
+koshigaya.saitama.jp
+kounosu.saitama.jp
+kuki.saitama.jp
+kumagaya.saitama.jp
+matsubushi.saitama.jp
+minano.saitama.jp
+misato.saitama.jp
+miyashiro.saitama.jp
+miyoshi.saitama.jp
+moroyama.saitama.jp
+nagatoro.saitama.jp
+namegawa.saitama.jp
+niiza.saitama.jp
+ogano.saitama.jp
+ogawa.saitama.jp
+ogose.saitama.jp
+okegawa.saitama.jp
+omiya.saitama.jp
+otaki.saitama.jp
+ranzan.saitama.jp
+ryokami.saitama.jp
+saitama.saitama.jp
+sakado.saitama.jp
+satte.saitama.jp
+sayama.saitama.jp
+shiki.saitama.jp
+shiraoka.saitama.jp
+soka.saitama.jp
+sugito.saitama.jp
+toda.saitama.jp
+tokigawa.saitama.jp
+tokorozawa.saitama.jp
+tsurugashima.saitama.jp
+urawa.saitama.jp
+warabi.saitama.jp
+yashio.saitama.jp
+yokoze.saitama.jp
+yono.saitama.jp
+yorii.saitama.jp
+yoshida.saitama.jp
+yoshikawa.saitama.jp
+yoshimi.saitama.jp
+aisho.shiga.jp
+gamo.shiga.jp
+higashiomi.shiga.jp
+hikone.shiga.jp
+koka.shiga.jp
+konan.shiga.jp
+kosei.shiga.jp
+koto.shiga.jp
+kusatsu.shiga.jp
+maibara.shiga.jp
+moriyama.shiga.jp
+nagahama.shiga.jp
+nishiazai.shiga.jp
+notogawa.shiga.jp
+omihachiman.shiga.jp
+otsu.shiga.jp
+ritto.shiga.jp
+ryuoh.shiga.jp
+takashima.shiga.jp
+takatsuki.shiga.jp
+torahime.shiga.jp
+toyosato.shiga.jp
+yasu.shiga.jp
+akagi.shimane.jp
+ama.shimane.jp
+gotsu.shimane.jp
+hamada.shimane.jp
+higashiizumo.shimane.jp
+hikawa.shimane.jp
+hikimi.shimane.jp
+izumo.shimane.jp
+kakinoki.shimane.jp
+masuda.shimane.jp
+matsue.shimane.jp
+misato.shimane.jp
+nishinoshima.shimane.jp
+ohda.shimane.jp
+okinoshima.shimane.jp
+okuizumo.shimane.jp
+shimane.shimane.jp
+tamayu.shimane.jp
+tsuwano.shimane.jp
+unnan.shimane.jp
+yakumo.shimane.jp
+yasugi.shimane.jp
+yatsuka.shimane.jp
+arai.shizuoka.jp
+atami.shizuoka.jp
+fuji.shizuoka.jp
+fujieda.shizuoka.jp
+fujikawa.shizuoka.jp
+fujinomiya.shizuoka.jp
+fukuroi.shizuoka.jp
+gotemba.shizuoka.jp
+haibara.shizuoka.jp
+hamamatsu.shizuoka.jp
+higashiizu.shizuoka.jp
+ito.shizuoka.jp
+iwata.shizuoka.jp
+izu.shizuoka.jp
+izunokuni.shizuoka.jp
+kakegawa.shizuoka.jp
+kannami.shizuoka.jp
+kawanehon.shizuoka.jp
+kawazu.shizuoka.jp
+kikugawa.shizuoka.jp
+kosai.shizuoka.jp
+makinohara.shizuoka.jp
+matsuzaki.shizuoka.jp
+minamiizu.shizuoka.jp
+mishima.shizuoka.jp
+morimachi.shizuoka.jp
+nishiizu.shizuoka.jp
+numazu.shizuoka.jp
+omaezaki.shizuoka.jp
+shimada.shizuoka.jp
+shimizu.shizuoka.jp
+shimoda.shizuoka.jp
+shizuoka.shizuoka.jp
+susono.shizuoka.jp
+yaizu.shizuoka.jp
+yoshida.shizuoka.jp
+ashikaga.tochigi.jp
+bato.tochigi.jp
+haga.tochigi.jp
+ichikai.tochigi.jp
+iwafune.tochigi.jp
+kaminokawa.tochigi.jp
+kanuma.tochigi.jp
+karasuyama.tochigi.jp
+kuroiso.tochigi.jp
+mashiko.tochigi.jp
+mibu.tochigi.jp
+moka.tochigi.jp
+motegi.tochigi.jp
+nasu.tochigi.jp
+nasushiobara.tochigi.jp
+nikko.tochigi.jp
+nishikata.tochigi.jp
+nogi.tochigi.jp
+ohira.tochigi.jp
+ohtawara.tochigi.jp
+oyama.tochigi.jp
+sakura.tochigi.jp
+sano.tochigi.jp
+shimotsuke.tochigi.jp
+shioya.tochigi.jp
+takanezawa.tochigi.jp
+tochigi.tochigi.jp
+tsuga.tochigi.jp
+ujiie.tochigi.jp
+utsunomiya.tochigi.jp
+yaita.tochigi.jp
+aizumi.tokushima.jp
+anan.tokushima.jp
+ichiba.tokushima.jp
+itano.tokushima.jp
+kainan.tokushima.jp
+komatsushima.tokushima.jp
+matsushige.tokushima.jp
+mima.tokushima.jp
+minami.tokushima.jp
+miyoshi.tokushima.jp
+mugi.tokushima.jp
+nakagawa.tokushima.jp
+naruto.tokushima.jp
+sanagochi.tokushima.jp
+shishikui.tokushima.jp
+tokushima.tokushima.jp
+wajiki.tokushima.jp
+adachi.tokyo.jp
+akiruno.tokyo.jp
+akishima.tokyo.jp
+aogashima.tokyo.jp
+arakawa.tokyo.jp
+bunkyo.tokyo.jp
+chiyoda.tokyo.jp
+chofu.tokyo.jp
+chuo.tokyo.jp
+edogawa.tokyo.jp
+fuchu.tokyo.jp
+fussa.tokyo.jp
+hachijo.tokyo.jp
+hachioji.tokyo.jp
+hamura.tokyo.jp
+higashikurume.tokyo.jp
+higashimurayama.tokyo.jp
+higashiyamato.tokyo.jp
+hino.tokyo.jp
+hinode.tokyo.jp
+hinohara.tokyo.jp
+inagi.tokyo.jp
+itabashi.tokyo.jp
+katsushika.tokyo.jp
+kita.tokyo.jp
+kiyose.tokyo.jp
+kodaira.tokyo.jp
+koganei.tokyo.jp
+kokubunji.tokyo.jp
+komae.tokyo.jp
+koto.tokyo.jp
+kouzushima.tokyo.jp
+kunitachi.tokyo.jp
+machida.tokyo.jp
+meguro.tokyo.jp
+minato.tokyo.jp
+mitaka.tokyo.jp
+mizuho.tokyo.jp
+musashimurayama.tokyo.jp
+musashino.tokyo.jp
+nakano.tokyo.jp
+nerima.tokyo.jp
+ogasawara.tokyo.jp
+okutama.tokyo.jp
+ome.tokyo.jp
+oshima.tokyo.jp
+ota.tokyo.jp
+setagaya.tokyo.jp
+shibuya.tokyo.jp
+shinagawa.tokyo.jp
+shinjuku.tokyo.jp
+suginami.tokyo.jp
+sumida.tokyo.jp
+tachikawa.tokyo.jp
+taito.tokyo.jp
+tama.tokyo.jp
+toshima.tokyo.jp
+chizu.tottori.jp
+hino.tottori.jp
+kawahara.tottori.jp
+koge.tottori.jp
+kotoura.tottori.jp
+misasa.tottori.jp
+nanbu.tottori.jp
+nichinan.tottori.jp
+sakaiminato.tottori.jp
+tottori.tottori.jp
+wakasa.tottori.jp
+yazu.tottori.jp
+yonago.tottori.jp
+asahi.toyama.jp
+fuchu.toyama.jp
+fukumitsu.toyama.jp
+funahashi.toyama.jp
+himi.toyama.jp
+imizu.toyama.jp
+inami.toyama.jp
+johana.toyama.jp
+kamiichi.toyama.jp
+kurobe.toyama.jp
+nakaniikawa.toyama.jp
+namerikawa.toyama.jp
+nanto.toyama.jp
+nyuzen.toyama.jp
+oyabe.toyama.jp
+taira.toyama.jp
+takaoka.toyama.jp
+tateyama.toyama.jp
+toga.toyama.jp
+tonami.toyama.jp
+toyama.toyama.jp
+unazuki.toyama.jp
+uozu.toyama.jp
+yamada.toyama.jp
+arida.wakayama.jp
+aridagawa.wakayama.jp
+gobo.wakayama.jp
+hashimoto.wakayama.jp
+hidaka.wakayama.jp
+hirogawa.wakayama.jp
+inami.wakayama.jp
+iwade.wakayama.jp
+kainan.wakayama.jp
+kamitonda.wakayama.jp
+katsuragi.wakayama.jp
+kimino.wakayama.jp
+kinokawa.wakayama.jp
+kitayama.wakayama.jp
+koya.wakayama.jp
+koza.wakayama.jp
+kozagawa.wakayama.jp
+kudoyama.wakayama.jp
+kushimoto.wakayama.jp
+mihama.wakayama.jp
+misato.wakayama.jp
+nachikatsuura.wakayama.jp
+shingu.wakayama.jp
+shirahama.wakayama.jp
+taiji.wakayama.jp
+tanabe.wakayama.jp
+wakayama.wakayama.jp
+yuasa.wakayama.jp
+yura.wakayama.jp
+asahi.yamagata.jp
+funagata.yamagata.jp
+higashine.yamagata.jp
+iide.yamagata.jp
+kahoku.yamagata.jp
+kaminoyama.yamagata.jp
+kaneyama.yamagata.jp
+kawanishi.yamagata.jp
+mamurogawa.yamagata.jp
+mikawa.yamagata.jp
+murayama.yamagata.jp
+nagai.yamagata.jp
+nakayama.yamagata.jp
+nanyo.yamagata.jp
+nishikawa.yamagata.jp
+obanazawa.yamagata.jp
+oe.yamagata.jp
+oguni.yamagata.jp
+ohkura.yamagata.jp
+oishida.yamagata.jp
+sagae.yamagata.jp
+sakata.yamagata.jp
+sakegawa.yamagata.jp
+shinjo.yamagata.jp
+shirataka.yamagata.jp
+shonai.yamagata.jp
+takahata.yamagata.jp
+tendo.yamagata.jp
+tozawa.yamagata.jp
+tsuruoka.yamagata.jp
+yamagata.yamagata.jp
+yamanobe.yamagata.jp
+yonezawa.yamagata.jp
+yuza.yamagata.jp
+abu.yamaguchi.jp
+hagi.yamaguchi.jp
+hikari.yamaguchi.jp
+hofu.yamaguchi.jp
+iwakuni.yamaguchi.jp
+kudamatsu.yamaguchi.jp
+mitou.yamaguchi.jp
+nagato.yamaguchi.jp
+oshima.yamaguchi.jp
+shimonoseki.yamaguchi.jp
+shunan.yamaguchi.jp
+tabuse.yamaguchi.jp
+tokuyama.yamaguchi.jp
+toyota.yamaguchi.jp
+ube.yamaguchi.jp
+yuu.yamaguchi.jp
+chuo.yamanashi.jp
+doshi.yamanashi.jp
+fuefuki.yamanashi.jp
+fujikawa.yamanashi.jp
+fujikawaguchiko.yamanashi.jp
+fujiyoshida.yamanashi.jp
+hayakawa.yamanashi.jp
+hokuto.yamanashi.jp
+ichikawamisato.yamanashi.jp
+kai.yamanashi.jp
+kofu.yamanashi.jp
+koshu.yamanashi.jp
+kosuge.yamanashi.jp
+minami-alps.yamanashi.jp
+minobu.yamanashi.jp
+nakamichi.yamanashi.jp
+nanbu.yamanashi.jp
+narusawa.yamanashi.jp
+nirasaki.yamanashi.jp
+nishikatsura.yamanashi.jp
+oshino.yamanashi.jp
+otsuki.yamanashi.jp
+showa.yamanashi.jp
+tabayama.yamanashi.jp
+tsuru.yamanashi.jp
+uenohara.yamanashi.jp
+yamanakako.yamanashi.jp
+yamanashi.yamanashi.jp
+
+// ke : http://www.kenic.or.ke/index.php/en/ke-domains/ke-domains
+ke
+ac.ke
+co.ke
+go.ke
+info.ke
+me.ke
+mobi.ke
+ne.ke
+or.ke
+sc.ke
+
+// kg : http://www.domain.kg/dmn_n.html
+kg
+org.kg
+net.kg
+com.kg
+edu.kg
+gov.kg
+mil.kg
+
+// kh : http://www.mptc.gov.kh/dns_registration.htm
+*.kh
+
+// ki : http://www.ki/dns/index.html
+ki
+edu.ki
+biz.ki
+net.ki
+org.ki
+gov.ki
+info.ki
+com.ki
+
+// km : https://en.wikipedia.org/wiki/.km
+// http://www.domaine.km/documents/charte.doc
+km
+org.km
+nom.km
+gov.km
+prd.km
+tm.km
+edu.km
+mil.km
+ass.km
+com.km
+// These are only mentioned as proposed suggestions at domaine.km, but
+// https://en.wikipedia.org/wiki/.km says they're available for registration:
+coop.km
+asso.km
+presse.km
+medecin.km
+notaires.km
+pharmaciens.km
+veterinaire.km
+gouv.km
+
+// kn : https://en.wikipedia.org/wiki/.kn
+// http://www.dot.kn/domainRules.html
+kn
+net.kn
+org.kn
+edu.kn
+gov.kn
+
+// kp : http://www.kcce.kp/en_index.php
+kp
+com.kp
+edu.kp
+gov.kp
+org.kp
+rep.kp
+tra.kp
+
+// kr : https://en.wikipedia.org/wiki/.kr
+// see also: http://domain.nida.or.kr/eng/registration.jsp
+kr
+ac.kr
+co.kr
+es.kr
+go.kr
+hs.kr
+kg.kr
+mil.kr
+ms.kr
+ne.kr
+or.kr
+pe.kr
+re.kr
+sc.kr
+// kr geographical names
+busan.kr
+chungbuk.kr
+chungnam.kr
+daegu.kr
+daejeon.kr
+gangwon.kr
+gwangju.kr
+gyeongbuk.kr
+gyeonggi.kr
+gyeongnam.kr
+incheon.kr
+jeju.kr
+jeonbuk.kr
+jeonnam.kr
+seoul.kr
+ulsan.kr
+
+// kw : https://www.nic.kw/policies/
+// Confirmed by registry <nic.tech@citra.gov.kw>
+kw
+com.kw
+edu.kw
+emb.kw
+gov.kw
+ind.kw
+net.kw
+org.kw
+
+// ky : http://www.icta.ky/da_ky_reg_dom.php
+// Confirmed by registry <kysupport@perimeterusa.com> 2008-06-17
+ky
+edu.ky
+gov.ky
+com.ky
+org.ky
+net.ky
+
+// kz : https://en.wikipedia.org/wiki/.kz
+// see also: http://www.nic.kz/rules/index.jsp
+kz
+org.kz
+edu.kz
+net.kz
+gov.kz
+mil.kz
+com.kz
+
+// la : https://en.wikipedia.org/wiki/.la
+// Submitted by registry <gavin.brown@nic.la>
+la
+int.la
+net.la
+info.la
+edu.la
+gov.la
+per.la
+com.la
+org.la
+
+// lb : https://en.wikipedia.org/wiki/.lb
+// Submitted by registry <randy@psg.com>
+lb
+com.lb
+edu.lb
+gov.lb
+net.lb
+org.lb
+
+// lc : https://en.wikipedia.org/wiki/.lc
+// see also: http://www.nic.lc/rules.htm
+lc
+com.lc
+net.lc
+co.lc
+org.lc
+edu.lc
+gov.lc
+
+// li : https://en.wikipedia.org/wiki/.li
+li
+
+// lk : https://www.nic.lk/index.php/domain-registration/lk-domain-naming-structure
+lk
+gov.lk
+sch.lk
+net.lk
+int.lk
+com.lk
+org.lk
+edu.lk
+ngo.lk
+soc.lk
+web.lk
+ltd.lk
+assn.lk
+grp.lk
+hotel.lk
+ac.lk
+
+// lr : http://psg.com/dns/lr/lr.txt
+// Submitted by registry <randy@psg.com>
+lr
+com.lr
+edu.lr
+gov.lr
+org.lr
+net.lr
+
+// ls : http://www.nic.ls/
+// Confirmed by registry <lsadmin@nic.ls>
+ls
+ac.ls
+biz.ls
+co.ls
+edu.ls
+gov.ls
+info.ls
+net.ls
+org.ls
+sc.ls
+
+// lt : https://en.wikipedia.org/wiki/.lt
+lt
+// gov.lt : http://www.gov.lt/index_en.php
+gov.lt
+
+// lu : http://www.dns.lu/en/
+lu
+
+// lv : http://www.nic.lv/DNS/En/generic.php
+lv
+com.lv
+edu.lv
+gov.lv
+org.lv
+mil.lv
+id.lv
+net.lv
+asn.lv
+conf.lv
+
+// ly : http://www.nic.ly/regulations.php
+ly
+com.ly
+net.ly
+gov.ly
+plc.ly
+edu.ly
+sch.ly
+med.ly
+org.ly
+id.ly
+
+// ma : https://en.wikipedia.org/wiki/.ma
+// http://www.anrt.ma/fr/admin/download/upload/file_fr782.pdf
+ma
+co.ma
+net.ma
+gov.ma
+org.ma
+ac.ma
+press.ma
+
+// mc : http://www.nic.mc/
+mc
+tm.mc
+asso.mc
+
+// md : https://en.wikipedia.org/wiki/.md
+md
+
+// me : https://en.wikipedia.org/wiki/.me
+me
+co.me
+net.me
+org.me
+edu.me
+ac.me
+gov.me
+its.me
+priv.me
+
+// mg : http://nic.mg/nicmg/?page_id=39
+mg
+org.mg
+nom.mg
+gov.mg
+prd.mg
+tm.mg
+edu.mg
+mil.mg
+com.mg
+co.mg
+
+// mh : https://en.wikipedia.org/wiki/.mh
+mh
+
+// mil : https://en.wikipedia.org/wiki/.mil
+mil
+
+// mk : https://en.wikipedia.org/wiki/.mk
+// see also: http://dns.marnet.net.mk/postapka.php
+mk
+com.mk
+org.mk
+net.mk
+edu.mk
+gov.mk
+inf.mk
+name.mk
+
+// ml : http://www.gobin.info/domainname/ml-template.doc
+// see also: https://en.wikipedia.org/wiki/.ml
+ml
+com.ml
+edu.ml
+gouv.ml
+gov.ml
+net.ml
+org.ml
+presse.ml
+
+// mm : https://en.wikipedia.org/wiki/.mm
+*.mm
+
+// mn : https://en.wikipedia.org/wiki/.mn
+mn
+gov.mn
+edu.mn
+org.mn
+
+// mo : http://www.monic.net.mo/
+mo
+com.mo
+net.mo
+org.mo
+edu.mo
+gov.mo
+
+// mobi : https://en.wikipedia.org/wiki/.mobi
+mobi
+
+// mp : http://www.dot.mp/
+// Confirmed by registry <dcamacho@saipan.com> 2008-06-17
+mp
+
+// mq : https://en.wikipedia.org/wiki/.mq
+mq
+
+// mr : https://en.wikipedia.org/wiki/.mr
+mr
+gov.mr
+
+// ms : http://www.nic.ms/pdf/MS_Domain_Name_Rules.pdf
+ms
+com.ms
+edu.ms
+gov.ms
+net.ms
+org.ms
+
+// mt : https://www.nic.org.mt/go/policy
+// Submitted by registry <help@nic.org.mt>
+mt
+com.mt
+edu.mt
+net.mt
+org.mt
+
+// mu : https://en.wikipedia.org/wiki/.mu
+mu
+com.mu
+net.mu
+org.mu
+gov.mu
+ac.mu
+co.mu
+or.mu
+
+// museum : http://about.museum/naming/
+// http://index.museum/
+museum
+academy.museum
+agriculture.museum
+air.museum
+airguard.museum
+alabama.museum
+alaska.museum
+amber.museum
+ambulance.museum
+american.museum
+americana.museum
+americanantiques.museum
+americanart.museum
+amsterdam.museum
+and.museum
+annefrank.museum
+anthro.museum
+anthropology.museum
+antiques.museum
+aquarium.museum
+arboretum.museum
+archaeological.museum
+archaeology.museum
+architecture.museum
+art.museum
+artanddesign.museum
+artcenter.museum
+artdeco.museum
+arteducation.museum
+artgallery.museum
+arts.museum
+artsandcrafts.museum
+asmatart.museum
+assassination.museum
+assisi.museum
+association.museum
+astronomy.museum
+atlanta.museum
+austin.museum
+australia.museum
+automotive.museum
+aviation.museum
+axis.museum
+badajoz.museum
+baghdad.museum
+bahn.museum
+bale.museum
+baltimore.museum
+barcelona.museum
+baseball.museum
+basel.museum
+baths.museum
+bauern.museum
+beauxarts.museum
+beeldengeluid.museum
+bellevue.museum
+bergbau.museum
+berkeley.museum
+berlin.museum
+bern.museum
+bible.museum
+bilbao.museum
+bill.museum
+birdart.museum
+birthplace.museum
+bonn.museum
+boston.museum
+botanical.museum
+botanicalgarden.museum
+botanicgarden.museum
+botany.museum
+brandywinevalley.museum
+brasil.museum
+bristol.museum
+british.museum
+britishcolumbia.museum
+broadcast.museum
+brunel.museum
+brussel.museum
+brussels.museum
+bruxelles.museum
+building.museum
+burghof.museum
+bus.museum
+bushey.museum
+cadaques.museum
+california.museum
+cambridge.museum
+can.museum
+canada.museum
+capebreton.museum
+carrier.museum
+cartoonart.museum
+casadelamoneda.museum
+castle.museum
+castres.museum
+celtic.museum
+center.museum
+chattanooga.museum
+cheltenham.museum
+chesapeakebay.museum
+chicago.museum
+children.museum
+childrens.museum
+childrensgarden.museum
+chiropractic.museum
+chocolate.museum
+christiansburg.museum
+cincinnati.museum
+cinema.museum
+circus.museum
+civilisation.museum
+civilization.museum
+civilwar.museum
+clinton.museum
+clock.museum
+coal.museum
+coastaldefence.museum
+cody.museum
+coldwar.museum
+collection.museum
+colonialwilliamsburg.museum
+coloradoplateau.museum
+columbia.museum
+columbus.museum
+communication.museum
+communications.museum
+community.museum
+computer.museum
+computerhistory.museum
+xn--comunicaes-v6a2o.museum
+comunicações.museum
+contemporary.museum
+contemporaryart.museum
+convent.museum
+copenhagen.museum
+corporation.museum
+xn--correios-e-telecomunicaes-ghc29a.museum
+correios-e-telecomunicações.museum
+corvette.museum
+costume.museum
+countryestate.museum
+county.museum
+crafts.museum
+cranbrook.museum
+creation.museum
+cultural.museum
+culturalcenter.museum
+culture.museum
+cyber.museum
+cymru.museum
+dali.museum
+dallas.museum
+database.museum
+ddr.museum
+decorativearts.museum
+delaware.museum
+delmenhorst.museum
+denmark.museum
+depot.museum
+design.museum
+detroit.museum
+dinosaur.museum
+discovery.museum
+dolls.museum
+donostia.museum
+durham.museum
+eastafrica.museum
+eastcoast.museum
+education.museum
+educational.museum
+egyptian.museum
+eisenbahn.museum
+elburg.museum
+elvendrell.museum
+embroidery.museum
+encyclopedic.museum
+england.museum
+entomology.museum
+environment.museum
+environmentalconservation.museum
+epilepsy.museum
+essex.museum
+estate.museum
+ethnology.museum
+exeter.museum
+exhibition.museum
+family.museum
+farm.museum
+farmequipment.museum
+farmers.museum
+farmstead.museum
+field.museum
+figueres.museum
+filatelia.museum
+film.museum
+fineart.museum
+finearts.museum
+finland.museum
+flanders.museum
+florida.museum
+force.museum
+fortmissoula.museum
+fortworth.museum
+foundation.museum
+francaise.museum
+frankfurt.museum
+franziskaner.museum
+freemasonry.museum
+freiburg.museum
+fribourg.museum
+frog.museum
+fundacio.museum
+furniture.museum
+gallery.museum
+garden.museum
+gateway.museum
+geelvinck.museum
+gemological.museum
+geology.museum
+georgia.museum
+giessen.museum
+glas.museum
+glass.museum
+gorge.museum
+grandrapids.museum
+graz.museum
+guernsey.museum
+halloffame.museum
+hamburg.museum
+handson.museum
+harvestcelebration.museum
+hawaii.museum
+health.museum
+heimatunduhren.museum
+hellas.museum
+helsinki.museum
+hembygdsforbund.museum
+heritage.museum
+histoire.museum
+historical.museum
+historicalsociety.museum
+historichouses.museum
+historisch.museum
+historisches.museum
+history.museum
+historyofscience.museum
+horology.museum
+house.museum
+humanities.museum
+illustration.museum
+imageandsound.museum
+indian.museum
+indiana.museum
+indianapolis.museum
+indianmarket.museum
+intelligence.museum
+interactive.museum
+iraq.museum
+iron.museum
+isleofman.museum
+jamison.museum
+jefferson.museum
+jerusalem.museum
+jewelry.museum
+jewish.museum
+jewishart.museum
+jfk.museum
+journalism.museum
+judaica.museum
+judygarland.museum
+juedisches.museum
+juif.museum
+karate.museum
+karikatur.museum
+kids.museum
+koebenhavn.museum
+koeln.museum
+kunst.museum
+kunstsammlung.museum
+kunstunddesign.museum
+labor.museum
+labour.museum
+lajolla.museum
+lancashire.museum
+landes.museum
+lans.museum
+xn--lns-qla.museum
+läns.museum
+larsson.museum
+lewismiller.museum
+lincoln.museum
+linz.museum
+living.museum
+livinghistory.museum
+localhistory.museum
+london.museum
+losangeles.museum
+louvre.museum
+loyalist.museum
+lucerne.museum
+luxembourg.museum
+luzern.museum
+mad.museum
+madrid.museum
+mallorca.museum
+manchester.museum
+mansion.museum
+mansions.museum
+manx.museum
+marburg.museum
+maritime.museum
+maritimo.museum
+maryland.museum
+marylhurst.museum
+media.museum
+medical.museum
+medizinhistorisches.museum
+meeres.museum
+memorial.museum
+mesaverde.museum
+michigan.museum
+midatlantic.museum
+military.museum
+mill.museum
+miners.museum
+mining.museum
+minnesota.museum
+missile.museum
+missoula.museum
+modern.museum
+moma.museum
+money.museum
+monmouth.museum
+monticello.museum
+montreal.museum
+moscow.museum
+motorcycle.museum
+muenchen.museum
+muenster.museum
+mulhouse.museum
+muncie.museum
+museet.museum
+museumcenter.museum
+museumvereniging.museum
+music.museum
+national.museum
+nationalfirearms.museum
+nationalheritage.museum
+nativeamerican.museum
+naturalhistory.museum
+naturalhistorymuseum.museum
+naturalsciences.museum
+nature.museum
+naturhistorisches.museum
+natuurwetenschappen.museum
+naumburg.museum
+naval.museum
+nebraska.museum
+neues.museum
+newhampshire.museum
+newjersey.museum
+newmexico.museum
+newport.museum
+newspaper.museum
+newyork.museum
+niepce.museum
+norfolk.museum
+north.museum
+nrw.museum
+nyc.museum
+nyny.museum
+oceanographic.museum
+oceanographique.museum
+omaha.museum
+online.museum
+ontario.museum
+openair.museum
+oregon.museum
+oregontrail.museum
+otago.museum
+oxford.museum
+pacific.museum
+paderborn.museum
+palace.museum
+paleo.museum
+palmsprings.museum
+panama.museum
+paris.museum
+pasadena.museum
+pharmacy.museum
+philadelphia.museum
+philadelphiaarea.museum
+philately.museum
+phoenix.museum
+photography.museum
+pilots.museum
+pittsburgh.museum
+planetarium.museum
+plantation.museum
+plants.museum
+plaza.museum
+portal.museum
+portland.museum
+portlligat.museum
+posts-and-telecommunications.museum
+preservation.museum
+presidio.museum
+press.museum
+project.museum
+public.museum
+pubol.museum
+quebec.museum
+railroad.museum
+railway.museum
+research.museum
+resistance.museum
+riodejaneiro.museum
+rochester.museum
+rockart.museum
+roma.museum
+russia.museum
+saintlouis.museum
+salem.museum
+salvadordali.museum
+salzburg.museum
+sandiego.museum
+sanfrancisco.museum
+santabarbara.museum
+santacruz.museum
+santafe.museum
+saskatchewan.museum
+satx.museum
+savannahga.museum
+schlesisches.museum
+schoenbrunn.museum
+schokoladen.museum
+school.museum
+schweiz.museum
+science.museum
+scienceandhistory.museum
+scienceandindustry.museum
+sciencecenter.museum
+sciencecenters.museum
+science-fiction.museum
+sciencehistory.museum
+sciences.museum
+sciencesnaturelles.museum
+scotland.museum
+seaport.museum
+settlement.museum
+settlers.museum
+shell.museum
+sherbrooke.museum
+sibenik.museum
+silk.museum
+ski.museum
+skole.museum
+society.museum
+sologne.museum
+soundandvision.museum
+southcarolina.museum
+southwest.museum
+space.museum
+spy.museum
+square.museum
+stadt.museum
+stalbans.museum
+starnberg.museum
+state.museum
+stateofdelaware.museum
+station.museum
+steam.museum
+steiermark.museum
+stjohn.museum
+stockholm.museum
+stpetersburg.museum
+stuttgart.museum
+suisse.museum
+surgeonshall.museum
+surrey.museum
+svizzera.museum
+sweden.museum
+sydney.museum
+tank.museum
+tcm.museum
+technology.museum
+telekommunikation.museum
+television.museum
+texas.museum
+textile.museum
+theater.museum
+time.museum
+timekeeping.museum
+topology.museum
+torino.museum
+touch.museum
+town.museum
+transport.museum
+tree.museum
+trolley.museum
+trust.museum
+trustee.museum
+uhren.museum
+ulm.museum
+undersea.museum
+university.museum
+usa.museum
+usantiques.museum
+usarts.museum
+uscountryestate.museum
+usculture.museum
+usdecorativearts.museum
+usgarden.museum
+ushistory.museum
+ushuaia.museum
+uslivinghistory.museum
+utah.museum
+uvic.museum
+valley.museum
+vantaa.museum
+versailles.museum
+viking.museum
+village.museum
+virginia.museum
+virtual.museum
+virtuel.museum
+vlaanderen.museum
+volkenkunde.museum
+wales.museum
+wallonie.museum
+war.museum
+washingtondc.museum
+watchandclock.museum
+watch-and-clock.museum
+western.museum
+westfalen.museum
+whaling.museum
+wildlife.museum
+williamsburg.museum
+windmill.museum
+workshop.museum
+york.museum
+yorkshire.museum
+yosemite.museum
+youth.museum
+zoological.museum
+zoology.museum
+xn--9dbhblg6di.museum
+ירושלים.museum
+xn--h1aegh.museum
+иком.museum
+
+// mv : https://en.wikipedia.org/wiki/.mv
+// "mv" included because, contra Wikipedia, google.mv exists.
+mv
+aero.mv
+biz.mv
+com.mv
+coop.mv
+edu.mv
+gov.mv
+info.mv
+int.mv
+mil.mv
+museum.mv
+name.mv
+net.mv
+org.mv
+pro.mv
+
+// mw : http://www.registrar.mw/
+mw
+ac.mw
+biz.mw
+co.mw
+com.mw
+coop.mw
+edu.mw
+gov.mw
+int.mw
+museum.mw
+net.mw
+org.mw
+
+// mx : http://www.nic.mx/
+// Submitted by registry <farias@nic.mx>
+mx
+com.mx
+org.mx
+gob.mx
+edu.mx
+net.mx
+
+// my : http://www.mynic.my/
+// Available strings: https://mynic.my/resources/domains/buying-a-domain/
+my
+biz.my
+com.my
+edu.my
+gov.my
+mil.my
+name.my
+net.my
+org.my
+
+// mz : http://www.uem.mz/
+// Submitted by registry <antonio@uem.mz>
+mz
+ac.mz
+adv.mz
+co.mz
+edu.mz
+gov.mz
+mil.mz
+net.mz
+org.mz
+
+// na : http://www.na-nic.com.na/
+// http://www.info.na/domain/
+na
+info.na
+pro.na
+name.na
+school.na
+or.na
+dr.na
+us.na
+mx.na
+ca.na
+in.na
+cc.na
+tv.na
+ws.na
+mobi.na
+co.na
+com.na
+org.na
+
+// name : has 2nd-level tlds, but there's no list of them
+name
+
+// nc : http://www.cctld.nc/
+nc
+asso.nc
+nom.nc
+
+// ne : https://en.wikipedia.org/wiki/.ne
+ne
+
+// net : https://en.wikipedia.org/wiki/.net
+net
+
+// nf : https://en.wikipedia.org/wiki/.nf
+nf
+com.nf
+net.nf
+per.nf
+rec.nf
+web.nf
+arts.nf
+firm.nf
+info.nf
+other.nf
+store.nf
+
+// ng : http://www.nira.org.ng/index.php/join-us/register-ng-domain/189-nira-slds
+ng
+com.ng
+edu.ng
+gov.ng
+i.ng
+mil.ng
+mobi.ng
+name.ng
+net.ng
+org.ng
+sch.ng
+
+// ni : http://www.nic.ni/
+ni
+ac.ni
+biz.ni
+co.ni
+com.ni
+edu.ni
+gob.ni
+in.ni
+info.ni
+int.ni
+mil.ni
+net.ni
+nom.ni
+org.ni
+web.ni
+
+// nl : https://en.wikipedia.org/wiki/.nl
+// https://www.sidn.nl/
+// ccTLD for the Netherlands
+nl
+
+// no : https://www.norid.no/en/om-domenenavn/regelverk-for-no/
+// Norid geographical second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-b/
+// Norid category second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-c/
+// Norid category second-level domains managed by parties other than Norid : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-d/
+// RSS feed: https://teknisk.norid.no/en/feed/
+no
+// Norid category second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-c/
+fhs.no
+vgs.no
+fylkesbibl.no
+folkebibl.no
+museum.no
+idrett.no
+priv.no
+// Norid category second-level domains managed by parties other than Norid : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-d/
+mil.no
+stat.no
+dep.no
+kommune.no
+herad.no
+// Norid geographical second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-b/
+// counties
+aa.no
+ah.no
+bu.no
+fm.no
+hl.no
+hm.no
+jan-mayen.no
+mr.no
+nl.no
+nt.no
+of.no
+ol.no
+oslo.no
+rl.no
+sf.no
+st.no
+svalbard.no
+tm.no
+tr.no
+va.no
+vf.no
+// primary and lower secondary schools per county
+gs.aa.no
+gs.ah.no
+gs.bu.no
+gs.fm.no
+gs.hl.no
+gs.hm.no
+gs.jan-mayen.no
+gs.mr.no
+gs.nl.no
+gs.nt.no
+gs.of.no
+gs.ol.no
+gs.oslo.no
+gs.rl.no
+gs.sf.no
+gs.st.no
+gs.svalbard.no
+gs.tm.no
+gs.tr.no
+gs.va.no
+gs.vf.no
+// cities
+akrehamn.no
+åkrehamn.no
+algard.no
+xn--lgrd-poac.no
+ålgård.no
+arna.no
+brumunddal.no
+bryne.no
+bronnoysund.no
+xn--brnnysund-m8ac.no
+brønnøysund.no
+drobak.no
+xn--drbak-wua.no
+drøbak.no
+egersund.no
+fetsund.no
+floro.no
+xn--flor-jra.no
+florø.no
+fredrikstad.no
+hokksund.no
+honefoss.no
+xn--hnefoss-q1a.no
+hønefoss.no
+jessheim.no
+jorpeland.no
+xn--jrpeland-54a.no
+jørpeland.no
+kirkenes.no
+kopervik.no
+krokstadelva.no
+langevag.no
+xn--langevg-jxa.no
+langevåg.no
+leirvik.no
+mjondalen.no
+xn--mjndalen-64a.no
+mjøndalen.no
+mo-i-rana.no
+mosjoen.no
+xn--mosjen-eya.no
+mosjøen.no
+nesoddtangen.no
+orkanger.no
+osoyro.no
+xn--osyro-wua.no
+osøyro.no
+raholt.no
+xn--rholt-mra.no
+råholt.no
+sandnessjoen.no
+xn--sandnessjen-ogb.no
+sandnessjøen.no
+skedsmokorset.no
+slattum.no
+spjelkavik.no
+stathelle.no
+stavern.no
+stjordalshalsen.no
+xn--stjrdalshalsen-sqb.no
+stjørdalshalsen.no
+tananger.no
+tranby.no
+vossevangen.no
+// communities
+afjord.no
+åfjord.no
+agdenes.no
+al.no
+ål.no
+alesund.no
+ålesund.no
+alstahaug.no
+alta.no
+xn--lt-liac.no
+áltá.no
+alaheadju.no
+álaheadju.no
+alvdal.no
+amli.no
+åmli.no
+amot.no
+åmot.no
+andebu.no
+andoy.no
+xn--andy-ira.no
+andøy.no
+andasuolo.no
+ardal.no
+årdal.no
+aremark.no
+arendal.no
+ås.no
+aseral.no
+åseral.no
+asker.no
+askim.no
+askvoll.no
+askoy.no
+xn--asky-ira.no
+askøy.no
+asnes.no
+åsnes.no
+audnedaln.no
+aukra.no
+aure.no
+aurland.no
+aurskog-holand.no
+xn--aurskog-hland-jnb.no
+aurskog-høland.no
+austevoll.no
+austrheim.no
+averoy.no
+xn--avery-yua.no
+averøy.no
+balestrand.no
+ballangen.no
+balat.no
+xn--blt-elab.no
+bálát.no
+balsfjord.no
+bahccavuotna.no
+xn--bhccavuotna-k7a.no
+báhccavuotna.no
+bamble.no
+bardu.no
+beardu.no
+beiarn.no
+bajddar.no
+xn--bjddar-pta.no
+bájddar.no
+baidar.no
+xn--bidr-5nac.no
+báidár.no
+berg.no
+bergen.no
+berlevag.no
+xn--berlevg-jxa.no
+berlevåg.no
+bearalvahki.no
+xn--bearalvhki-y4a.no
+bearalváhki.no
+bindal.no
+birkenes.no
+bjarkoy.no
+xn--bjarky-fya.no
+bjarkøy.no
+bjerkreim.no
+bjugn.no
+bodo.no
+xn--bod-2na.no
+bodø.no
+badaddja.no
+xn--bdddj-mrabd.no
+bådåddjå.no
+budejju.no
+bokn.no
+bremanger.no
+bronnoy.no
+xn--brnny-wuac.no
+brønnøy.no
+bygland.no
+bykle.no
+barum.no
+xn--brum-voa.no
+bærum.no
+bo.telemark.no
+xn--b-5ga.telemark.no
+bø.telemark.no
+bo.nordland.no
+xn--b-5ga.nordland.no
+bø.nordland.no
+bievat.no
+xn--bievt-0qa.no
+bievát.no
+bomlo.no
+xn--bmlo-gra.no
+bømlo.no
+batsfjord.no
+xn--btsfjord-9za.no
+båtsfjord.no
+bahcavuotna.no
+xn--bhcavuotna-s4a.no
+báhcavuotna.no
+dovre.no
+drammen.no
+drangedal.no
+dyroy.no
+xn--dyry-ira.no
+dyrøy.no
+donna.no
+xn--dnna-gra.no
+dønna.no
+eid.no
+eidfjord.no
+eidsberg.no
+eidskog.no
+eidsvoll.no
+eigersund.no
+elverum.no
+enebakk.no
+engerdal.no
+etne.no
+etnedal.no
+evenes.no
+evenassi.no
+xn--eveni-0qa01ga.no
+evenášši.no
+evje-og-hornnes.no
+farsund.no
+fauske.no
+fuossko.no
+fuoisku.no
+fedje.no
+fet.no
+finnoy.no
+xn--finny-yua.no
+finnøy.no
+fitjar.no
+fjaler.no
+fjell.no
+flakstad.no
+flatanger.no
+flekkefjord.no
+flesberg.no
+flora.no
+fla.no
+xn--fl-zia.no
+flå.no
+folldal.no
+forsand.no
+fosnes.no
+frei.no
+frogn.no
+froland.no
+frosta.no
+frana.no
+xn--frna-woa.no
+fræna.no
+froya.no
+xn--frya-hra.no
+frøya.no
+fusa.no
+fyresdal.no
+forde.no
+xn--frde-gra.no
+førde.no
+gamvik.no
+gangaviika.no
+xn--ggaviika-8ya47h.no
+gáŋgaviika.no
+gaular.no
+gausdal.no
+gildeskal.no
+xn--gildeskl-g0a.no
+gildeskål.no
+giske.no
+gjemnes.no
+gjerdrum.no
+gjerstad.no
+gjesdal.no
+gjovik.no
+xn--gjvik-wua.no
+gjøvik.no
+gloppen.no
+gol.no
+gran.no
+grane.no
+granvin.no
+gratangen.no
+grimstad.no
+grong.no
+kraanghke.no
+xn--kranghke-b0a.no
+kråanghke.no
+grue.no
+gulen.no
+hadsel.no
+halden.no
+halsa.no
+hamar.no
+hamaroy.no
+habmer.no
+xn--hbmer-xqa.no
+hábmer.no
+hapmir.no
+xn--hpmir-xqa.no
+hápmir.no
+hammerfest.no
+hammarfeasta.no
+xn--hmmrfeasta-s4ac.no
+hámmárfeasta.no
+haram.no
+hareid.no
+harstad.no
+hasvik.no
+aknoluokta.no
+xn--koluokta-7ya57h.no
+ákŋoluokta.no
+hattfjelldal.no
+aarborte.no
+haugesund.no
+hemne.no
+hemnes.no
+hemsedal.no
+heroy.more-og-romsdal.no
+xn--hery-ira.xn--mre-og-romsdal-qqb.no
+herøy.møre-og-romsdal.no
+heroy.nordland.no
+xn--hery-ira.nordland.no
+herøy.nordland.no
+hitra.no
+hjartdal.no
+hjelmeland.no
+hobol.no
+xn--hobl-ira.no
+hobøl.no
+hof.no
+hol.no
+hole.no
+holmestrand.no
+holtalen.no
+xn--holtlen-hxa.no
+holtålen.no
+hornindal.no
+horten.no
+hurdal.no
+hurum.no
+hvaler.no
+hyllestad.no
+hagebostad.no
+xn--hgebostad-g3a.no
+hægebostad.no
+hoyanger.no
+xn--hyanger-q1a.no
+høyanger.no
+hoylandet.no
+xn--hylandet-54a.no
+høylandet.no
+ha.no
+xn--h-2fa.no
+hå.no
+ibestad.no
+inderoy.no
+xn--indery-fya.no
+inderøy.no
+iveland.no
+jevnaker.no
+jondal.no
+jolster.no
+xn--jlster-bya.no
+jølster.no
+karasjok.no
+karasjohka.no
+xn--krjohka-hwab49j.no
+kárášjohka.no
+karlsoy.no
+galsa.no
+xn--gls-elac.no
+gálsá.no
+karmoy.no
+xn--karmy-yua.no
+karmøy.no
+kautokeino.no
+guovdageaidnu.no
+klepp.no
+klabu.no
+xn--klbu-woa.no
+klæbu.no
+kongsberg.no
+kongsvinger.no
+kragero.no
+xn--krager-gya.no
+kragerø.no
+kristiansand.no
+kristiansund.no
+krodsherad.no
+xn--krdsherad-m8a.no
+krødsherad.no
+kvalsund.no
+rahkkeravju.no
+xn--rhkkervju-01af.no
+ráhkkerávju.no
+kvam.no
+kvinesdal.no
+kvinnherad.no
+kviteseid.no
+kvitsoy.no
+xn--kvitsy-fya.no
+kvitsøy.no
+kvafjord.no
+xn--kvfjord-nxa.no
+kvæfjord.no
+giehtavuoatna.no
+kvanangen.no
+xn--kvnangen-k0a.no
+kvænangen.no
+navuotna.no
+xn--nvuotna-hwa.no
+návuotna.no
+kafjord.no
+xn--kfjord-iua.no
+kåfjord.no
+gaivuotna.no
+xn--givuotna-8ya.no
+gáivuotna.no
+larvik.no
+lavangen.no
+lavagis.no
+loabat.no
+xn--loabt-0qa.no
+loabát.no
+lebesby.no
+davvesiida.no
+leikanger.no
+leirfjord.no
+leka.no
+leksvik.no
+lenvik.no
+leangaviika.no
+xn--leagaviika-52b.no
+leaŋgaviika.no
+lesja.no
+levanger.no
+lier.no
+lierne.no
+lillehammer.no
+lillesand.no
+lindesnes.no
+lindas.no
+xn--linds-pra.no
+lindås.no
+lom.no
+loppa.no
+lahppi.no
+xn--lhppi-xqa.no
+láhppi.no
+lund.no
+lunner.no
+luroy.no
+xn--lury-ira.no
+lurøy.no
+luster.no
+lyngdal.no
+lyngen.no
+ivgu.no
+lardal.no
+lerdal.no
+xn--lrdal-sra.no
+lærdal.no
+lodingen.no
+xn--ldingen-q1a.no
+lødingen.no
+lorenskog.no
+xn--lrenskog-54a.no
+lørenskog.no
+loten.no
+xn--lten-gra.no
+løten.no
+malvik.no
+masoy.no
+xn--msy-ula0h.no
+måsøy.no
+muosat.no
+xn--muost-0qa.no
+muosát.no
+mandal.no
+marker.no
+marnardal.no
+masfjorden.no
+meland.no
+meldal.no
+melhus.no
+meloy.no
+xn--mely-ira.no
+meløy.no
+meraker.no
+xn--merker-kua.no
+meråker.no
+moareke.no
+xn--moreke-jua.no
+moåreke.no
+midsund.no
+midtre-gauldal.no
+modalen.no
+modum.no
+molde.no
+moskenes.no
+moss.no
+mosvik.no
+malselv.no
+xn--mlselv-iua.no
+målselv.no
+malatvuopmi.no
+xn--mlatvuopmi-s4a.no
+málatvuopmi.no
+namdalseid.no
+aejrie.no
+namsos.no
+namsskogan.no
+naamesjevuemie.no
+xn--nmesjevuemie-tcba.no
+nååmesjevuemie.no
+laakesvuemie.no
+nannestad.no
+narvik.no
+narviika.no
+naustdal.no
+nedre-eiker.no
+nes.akershus.no
+nes.buskerud.no
+nesna.no
+nesodden.no
+nesseby.no
+unjarga.no
+xn--unjrga-rta.no
+unjárga.no
+nesset.no
+nissedal.no
+nittedal.no
+nord-aurdal.no
+nord-fron.no
+nord-odal.no
+norddal.no
+nordkapp.no
+davvenjarga.no
+xn--davvenjrga-y4a.no
+davvenjárga.no
+nordre-land.no
+nordreisa.no
+raisa.no
+xn--risa-5na.no
+ráisa.no
+nore-og-uvdal.no
+notodden.no
+naroy.no
+xn--nry-yla5g.no
+nærøy.no
+notteroy.no
+xn--nttery-byae.no
+nøtterøy.no
+odda.no
+oksnes.no
+øksnes.no
+oppdal.no
+oppegard.no
+xn--oppegrd-ixa.no
+oppegård.no
+orkdal.no
+orland.no
+ørland.no
+orskog.no
+ørskog.no
+orsta.no
+ørsta.no
+os.hedmark.no
+os.hordaland.no
+osen.no
+osteroy.no
+xn--ostery-fya.no
+osterøy.no
+ostre-toten.no
+østre-toten.no
+overhalla.no
+ovre-eiker.no
+øvre-eiker.no
+oyer.no
+øyer.no
+oygarden.no
+øygarden.no
+oystre-slidre.no
+øystre-slidre.no
+porsanger.no
+porsangu.no
+xn--porsgu-sta26f.no
+porsáŋgu.no
+porsgrunn.no
+radoy.no
+xn--rady-ira.no
+radøy.no
+rakkestad.no
+rana.no
+ruovat.no
+randaberg.no
+rauma.no
+rendalen.no
+rennebu.no
+rennesoy.no
+xn--rennesy-v1a.no
+rennesøy.no
+rindal.no
+ringebu.no
+ringerike.no
+ringsaker.no
+rissa.no
+risor.no
+xn--risr-ira.no
+risør.no
+roan.no
+rollag.no
+rygge.no
+ralingen.no
+xn--rlingen-mxa.no
+rælingen.no
+rodoy.no
+xn--rdy-0nab.no
+rødøy.no
+romskog.no
+xn--rmskog-bya.no
+rømskog.no
+roros.no
+xn--rros-gra.no
+røros.no
+rost.no
+xn--rst-0na.no
+røst.no
+royken.no
+xn--ryken-vua.no
+røyken.no
+royrvik.no
+xn--ryrvik-bya.no
+røyrvik.no
+rade.no
+xn--rde-ula.no
+råde.no
+salangen.no
+siellak.no
+saltdal.no
+salat.no
+xn--slt-elab.no
+sálát.no
+xn--slat-5na.no
+sálat.no
+samnanger.no
+sande.more-og-romsdal.no
+sande.xn--mre-og-romsdal-qqb.no
+sande.møre-og-romsdal.no
+sande.vestfold.no
+sandefjord.no
+sandnes.no
+sandoy.no
+xn--sandy-yua.no
+sandøy.no
+sarpsborg.no
+sauda.no
+sauherad.no
+sel.no
+selbu.no
+selje.no
+seljord.no
+sigdal.no
+siljan.no
+sirdal.no
+skaun.no
+skedsmo.no
+ski.no
+skien.no
+skiptvet.no
+skjervoy.no
+xn--skjervy-v1a.no
+skjervøy.no
+skierva.no
+xn--skierv-uta.no
+skiervá.no
+skjak.no
+xn--skjk-soa.no
+skjåk.no
+skodje.no
+skanland.no
+xn--sknland-fxa.no
+skånland.no
+skanit.no
+xn--sknit-yqa.no
+skánit.no
+smola.no
+xn--smla-hra.no
+smøla.no
+snillfjord.no
+snasa.no
+xn--snsa-roa.no
+snåsa.no
+snoasa.no
+snaase.no
+xn--snase-nra.no
+snåase.no
+sogndal.no
+sokndal.no
+sola.no
+solund.no
+songdalen.no
+sortland.no
+spydeberg.no
+stange.no
+stavanger.no
+steigen.no
+steinkjer.no
+stjordal.no
+xn--stjrdal-s1a.no
+stjørdal.no
+stokke.no
+stor-elvdal.no
+stord.no
+stordal.no
+storfjord.no
+omasvuotna.no
+strand.no
+stranda.no
+stryn.no
+sula.no
+suldal.no
+sund.no
+sunndal.no
+surnadal.no
+sveio.no
+svelvik.no
+sykkylven.no
+sogne.no
+xn--sgne-gra.no
+søgne.no
+somna.no
+xn--smna-gra.no
+sømna.no
+sondre-land.no
+xn--sndre-land-0cb.no
+søndre-land.no
+sor-aurdal.no
+xn--sr-aurdal-l8a.no
+sør-aurdal.no
+sor-fron.no
+xn--sr-fron-q1a.no
+sør-fron.no
+sor-odal.no
+xn--sr-odal-q1a.no
+sør-odal.no
+sor-varanger.no
+xn--sr-varanger-ggb.no
+sør-varanger.no
+matta-varjjat.no
+xn--mtta-vrjjat-k7af.no
+mátta-várjjat.no
+sorfold.no
+xn--srfold-bya.no
+sørfold.no
+sorreisa.no
+xn--srreisa-q1a.no
+sørreisa.no
+sorum.no
+xn--srum-gra.no
+sørum.no
+tana.no
+deatnu.no
+time.no
+tingvoll.no
+tinn.no
+tjeldsund.no
+dielddanuorri.no
+tjome.no
+xn--tjme-hra.no
+tjøme.no
+tokke.no
+tolga.no
+torsken.no
+tranoy.no
+xn--trany-yua.no
+tranøy.no
+tromso.no
+xn--troms-zua.no
+tromsø.no
+tromsa.no
+romsa.no
+trondheim.no
+troandin.no
+trysil.no
+trana.no
+xn--trna-woa.no
+træna.no
+trogstad.no
+xn--trgstad-r1a.no
+trøgstad.no
+tvedestrand.no
+tydal.no
+tynset.no
+tysfjord.no
+divtasvuodna.no
+divttasvuotna.no
+tysnes.no
+tysvar.no
+xn--tysvr-vra.no
+tysvær.no
+tonsberg.no
+xn--tnsberg-q1a.no
+tønsberg.no
+ullensaker.no
+ullensvang.no
+ulvik.no
+utsira.no
+vadso.no
+xn--vads-jra.no
+vadsø.no
+cahcesuolo.no
+xn--hcesuolo-7ya35b.no
+čáhcesuolo.no
+vaksdal.no
+valle.no
+vang.no
+vanylven.no
+vardo.no
+xn--vard-jra.no
+vardø.no
+varggat.no
+xn--vrggt-xqad.no
+várggát.no
+vefsn.no
+vaapste.no
+vega.no
+vegarshei.no
+xn--vegrshei-c0a.no
+vegårshei.no
+vennesla.no
+verdal.no
+verran.no
+vestby.no
+vestnes.no
+vestre-slidre.no
+vestre-toten.no
+vestvagoy.no
+xn--vestvgy-ixa6o.no
+vestvågøy.no
+vevelstad.no
+vik.no
+vikna.no
+vindafjord.no
+volda.no
+voss.no
+varoy.no
+xn--vry-yla5g.no
+værøy.no
+vagan.no
+xn--vgan-qoa.no
+vågan.no
+voagat.no
+vagsoy.no
+xn--vgsy-qoa0j.no
+vågsøy.no
+vaga.no
+xn--vg-yiab.no
+vågå.no
+valer.ostfold.no
+xn--vler-qoa.xn--stfold-9xa.no
+våler.østfold.no
+valer.hedmark.no
+xn--vler-qoa.hedmark.no
+våler.hedmark.no
+
+// np : http://www.mos.com.np/register.html
+*.np
+
+// nr : http://cenpac.net.nr/dns/index.html
+// Submitted by registry <technician@cenpac.net.nr>
+nr
+biz.nr
+info.nr
+gov.nr
+edu.nr
+org.nr
+net.nr
+com.nr
+
+// nu : https://en.wikipedia.org/wiki/.nu
+nu
+
+// nz : https://en.wikipedia.org/wiki/.nz
+// Submitted by registry <jay@nzrs.net.nz>
+nz
+ac.nz
+co.nz
+cri.nz
+geek.nz
+gen.nz
+govt.nz
+health.nz
+iwi.nz
+kiwi.nz
+maori.nz
+mil.nz
+xn--mori-qsa.nz
+māori.nz
+net.nz
+org.nz
+parliament.nz
+school.nz
+
+// om : https://en.wikipedia.org/wiki/.om
+om
+co.om
+com.om
+edu.om
+gov.om
+med.om
+museum.om
+net.om
+org.om
+pro.om
+
+// onion : https://tools.ietf.org/html/rfc7686
+onion
+
+// org : https://en.wikipedia.org/wiki/.org
+org
+
+// pa : http://www.nic.pa/
+// Some additional second level "domains" resolve directly as hostnames, such as
+// pannet.pa, so we add a rule for "pa".
+pa
+ac.pa
+gob.pa
+com.pa
+org.pa
+sld.pa
+edu.pa
+net.pa
+ing.pa
+abo.pa
+med.pa
+nom.pa
+
+// pe : https://www.nic.pe/InformeFinalComision.pdf
+pe
+edu.pe
+gob.pe
+nom.pe
+mil.pe
+org.pe
+com.pe
+net.pe
+
+// pf : http://www.gobin.info/domainname/formulaire-pf.pdf
+pf
+com.pf
+org.pf
+edu.pf
+
+// pg : https://en.wikipedia.org/wiki/.pg
+*.pg
+
+// ph : http://www.domains.ph/FAQ2.asp
+// Submitted by registry <jed@email.com.ph>
+ph
+com.ph
+net.ph
+org.ph
+gov.ph
+edu.ph
+ngo.ph
+mil.ph
+i.ph
+
+// pk : http://pk5.pknic.net.pk/pk5/msgNamepk.PK
+pk
+com.pk
+net.pk
+edu.pk
+org.pk
+fam.pk
+biz.pk
+web.pk
+gov.pk
+gob.pk
+gok.pk
+gon.pk
+gop.pk
+gos.pk
+info.pk
+
+// pl http://www.dns.pl/english/index.html
+// Submitted by registry
+pl
+com.pl
+net.pl
+org.pl
+// pl functional domains (http://www.dns.pl/english/index.html)
+aid.pl
+agro.pl
+atm.pl
+auto.pl
+biz.pl
+edu.pl
+gmina.pl
+gsm.pl
+info.pl
+mail.pl
+miasta.pl
+media.pl
+mil.pl
+nieruchomosci.pl
+nom.pl
+pc.pl
+powiat.pl
+priv.pl
+realestate.pl
+rel.pl
+sex.pl
+shop.pl
+sklep.pl
+sos.pl
+szkola.pl
+targi.pl
+tm.pl
+tourism.pl
+travel.pl
+turystyka.pl
+// Government domains
+gov.pl
+ap.gov.pl
+ic.gov.pl
+is.gov.pl
+us.gov.pl
+kmpsp.gov.pl
+kppsp.gov.pl
+kwpsp.gov.pl
+psp.gov.pl
+wskr.gov.pl
+kwp.gov.pl
+mw.gov.pl
+ug.gov.pl
+um.gov.pl
+umig.gov.pl
+ugim.gov.pl
+upow.gov.pl
+uw.gov.pl
+starostwo.gov.pl
+pa.gov.pl
+po.gov.pl
+psse.gov.pl
+pup.gov.pl
+rzgw.gov.pl
+sa.gov.pl
+so.gov.pl
+sr.gov.pl
+wsa.gov.pl
+sko.gov.pl
+uzs.gov.pl
+wiih.gov.pl
+winb.gov.pl
+pinb.gov.pl
+wios.gov.pl
+witd.gov.pl
+wzmiuw.gov.pl
+piw.gov.pl
+wiw.gov.pl
+griw.gov.pl
+wif.gov.pl
+oum.gov.pl
+sdn.gov.pl
+zp.gov.pl
+uppo.gov.pl
+mup.gov.pl
+wuoz.gov.pl
+konsulat.gov.pl
+oirm.gov.pl
+// pl regional domains (http://www.dns.pl/english/index.html)
+augustow.pl
+babia-gora.pl
+bedzin.pl
+beskidy.pl
+bialowieza.pl
+bialystok.pl
+bielawa.pl
+bieszczady.pl
+boleslawiec.pl
+bydgoszcz.pl
+bytom.pl
+cieszyn.pl
+czeladz.pl
+czest.pl
+dlugoleka.pl
+elblag.pl
+elk.pl
+glogow.pl
+gniezno.pl
+gorlice.pl
+grajewo.pl
+ilawa.pl
+jaworzno.pl
+jelenia-gora.pl
+jgora.pl
+kalisz.pl
+kazimierz-dolny.pl
+karpacz.pl
+kartuzy.pl
+kaszuby.pl
+katowice.pl
+kepno.pl
+ketrzyn.pl
+klodzko.pl
+kobierzyce.pl
+kolobrzeg.pl
+konin.pl
+konskowola.pl
+kutno.pl
+lapy.pl
+lebork.pl
+legnica.pl
+lezajsk.pl
+limanowa.pl
+lomza.pl
+lowicz.pl
+lubin.pl
+lukow.pl
+malbork.pl
+malopolska.pl
+mazowsze.pl
+mazury.pl
+mielec.pl
+mielno.pl
+mragowo.pl
+naklo.pl
+nowaruda.pl
+nysa.pl
+olawa.pl
+olecko.pl
+olkusz.pl
+olsztyn.pl
+opoczno.pl
+opole.pl
+ostroda.pl
+ostroleka.pl
+ostrowiec.pl
+ostrowwlkp.pl
+pila.pl
+pisz.pl
+podhale.pl
+podlasie.pl
+polkowice.pl
+pomorze.pl
+pomorskie.pl
+prochowice.pl
+pruszkow.pl
+przeworsk.pl
+pulawy.pl
+radom.pl
+rawa-maz.pl
+rybnik.pl
+rzeszow.pl
+sanok.pl
+sejny.pl
+slask.pl
+slupsk.pl
+sosnowiec.pl
+stalowa-wola.pl
+skoczow.pl
+starachowice.pl
+stargard.pl
+suwalki.pl
+swidnica.pl
+swiebodzin.pl
+swinoujscie.pl
+szczecin.pl
+szczytno.pl
+tarnobrzeg.pl
+tgory.pl
+turek.pl
+tychy.pl
+ustka.pl
+walbrzych.pl
+warmia.pl
+warszawa.pl
+waw.pl
+wegrow.pl
+wielun.pl
+wlocl.pl
+wloclawek.pl
+wodzislaw.pl
+wolomin.pl
+wroclaw.pl
+zachpomor.pl
+zagan.pl
+zarow.pl
+zgora.pl
+zgorzelec.pl
+
+// pm : http://www.afnic.fr/medias/documents/AFNIC-naming-policy2012.pdf
+pm
+
+// pn : http://www.government.pn/PnRegistry/policies.htm
+pn
+gov.pn
+co.pn
+org.pn
+edu.pn
+net.pn
+
+// post : https://en.wikipedia.org/wiki/.post
+post
+
+// pr : http://www.nic.pr/index.asp?f=1
+pr
+com.pr
+net.pr
+org.pr
+gov.pr
+edu.pr
+isla.pr
+pro.pr
+biz.pr
+info.pr
+name.pr
+// these aren't mentioned on nic.pr, but on https://en.wikipedia.org/wiki/.pr
+est.pr
+prof.pr
+ac.pr
+
+// pro : http://registry.pro/get-pro
+pro
+aaa.pro
+aca.pro
+acct.pro
+avocat.pro
+bar.pro
+cpa.pro
+eng.pro
+jur.pro
+law.pro
+med.pro
+recht.pro
+
+// ps : https://en.wikipedia.org/wiki/.ps
+// http://www.nic.ps/registration/policy.html#reg
+ps
+edu.ps
+gov.ps
+sec.ps
+plo.ps
+com.ps
+org.ps
+net.ps
+
+// pt : http://online.dns.pt/dns/start_dns
+pt
+net.pt
+gov.pt
+org.pt
+edu.pt
+int.pt
+publ.pt
+com.pt
+nome.pt
+
+// pw : https://en.wikipedia.org/wiki/.pw
+pw
+co.pw
+ne.pw
+or.pw
+ed.pw
+go.pw
+belau.pw
+
+// py : http://www.nic.py/pautas.html#seccion_9
+// Submitted by registry
+py
+com.py
+coop.py
+edu.py
+gov.py
+mil.py
+net.py
+org.py
+
+// qa : http://domains.qa/en/
+qa
+com.qa
+edu.qa
+gov.qa
+mil.qa
+name.qa
+net.qa
+org.qa
+sch.qa
+
+// re : http://www.afnic.re/obtenir/chartes/nommage-re/annexe-descriptifs
+re
+asso.re
+com.re
+nom.re
+
+// ro : http://www.rotld.ro/
+ro
+arts.ro
+com.ro
+firm.ro
+info.ro
+nom.ro
+nt.ro
+org.ro
+rec.ro
+store.ro
+tm.ro
+www.ro
+
+// rs : https://www.rnids.rs/en/domains/national-domains
+rs
+ac.rs
+co.rs
+edu.rs
+gov.rs
+in.rs
+org.rs
+
+// ru : https://cctld.ru/files/pdf/docs/en/rules_ru-rf.pdf
+// Submitted by George Georgievsky <gug@cctld.ru>
+ru
+
+// rw : https://www.ricta.org.rw/sites/default/files/resources/registry_registrar_contract_0.pdf
+rw
+ac.rw
+co.rw
+coop.rw
+gov.rw
+mil.rw
+net.rw
+org.rw
+
+// sa : http://www.nic.net.sa/
+sa
+com.sa
+net.sa
+org.sa
+gov.sa
+med.sa
+pub.sa
+edu.sa
+sch.sa
+
+// sb : http://www.sbnic.net.sb/
+// Submitted by registry <lee.humphries@telekom.com.sb>
+sb
+com.sb
+edu.sb
+gov.sb
+net.sb
+org.sb
+
+// sc : http://www.nic.sc/
+sc
+com.sc
+gov.sc
+net.sc
+org.sc
+edu.sc
+
+// sd : http://www.isoc.sd/sudanic.isoc.sd/billing_pricing.htm
+// Submitted by registry <admin@isoc.sd>
+sd
+com.sd
+net.sd
+org.sd
+edu.sd
+med.sd
+tv.sd
+gov.sd
+info.sd
+
+// se : https://en.wikipedia.org/wiki/.se
+// Submitted by registry <patrik.wallstrom@iis.se>
+se
+a.se
+ac.se
+b.se
+bd.se
+brand.se
+c.se
+d.se
+e.se
+f.se
+fh.se
+fhsk.se
+fhv.se
+g.se
+h.se
+i.se
+k.se
+komforb.se
+kommunalforbund.se
+komvux.se
+l.se
+lanbib.se
+m.se
+n.se
+naturbruksgymn.se
+o.se
+org.se
+p.se
+parti.se
+pp.se
+press.se
+r.se
+s.se
+t.se
+tm.se
+u.se
+w.se
+x.se
+y.se
+z.se
+
+// sg : http://www.nic.net.sg/page/registration-policies-procedures-and-guidelines
+sg
+com.sg
+net.sg
+org.sg
+gov.sg
+edu.sg
+per.sg
+
+// sh : http://www.nic.sh/registrar.html
+sh
+com.sh
+net.sh
+gov.sh
+org.sh
+mil.sh
+
+// si : https://en.wikipedia.org/wiki/.si
+si
+
+// sj : No registrations at this time.
+// Submitted by registry <jarle@uninett.no>
+sj
+
+// sk : https://en.wikipedia.org/wiki/.sk
+// list of 2nd level domains ?
+sk
+
+// sl : http://www.nic.sl
+// Submitted by registry <adam@neoip.com>
+sl
+com.sl
+net.sl
+edu.sl
+gov.sl
+org.sl
+
+// sm : https://en.wikipedia.org/wiki/.sm
+sm
+
+// sn : https://en.wikipedia.org/wiki/.sn
+sn
+art.sn
+com.sn
+edu.sn
+gouv.sn
+org.sn
+perso.sn
+univ.sn
+
+// so : http://sonic.so/policies/
+so
+com.so
+edu.so
+gov.so
+me.so
+net.so
+org.so
+
+// sr : https://en.wikipedia.org/wiki/.sr
+sr
+
+// ss : https://registry.nic.ss/
+// Submitted by registry <technical@nic.ss>
+ss
+biz.ss
+com.ss
+edu.ss
+gov.ss
+me.ss
+net.ss
+org.ss
+sch.ss
+
+// st : http://www.nic.st/html/policyrules/
+st
+co.st
+com.st
+consulado.st
+edu.st
+embaixada.st
+mil.st
+net.st
+org.st
+principe.st
+saotome.st
+store.st
+
+// su : https://en.wikipedia.org/wiki/.su
+su
+
+// sv : http://www.svnet.org.sv/niveldos.pdf
+sv
+com.sv
+edu.sv
+gob.sv
+org.sv
+red.sv
+
+// sx : https://en.wikipedia.org/wiki/.sx
+// Submitted by registry <jcvignes@openregistry.com>
+sx
+gov.sx
+
+// sy : https://en.wikipedia.org/wiki/.sy
+// see also: http://www.gobin.info/domainname/sy.doc
+sy
+edu.sy
+gov.sy
+net.sy
+mil.sy
+com.sy
+org.sy
+
+// sz : https://en.wikipedia.org/wiki/.sz
+// http://www.sispa.org.sz/
+sz
+co.sz
+ac.sz
+org.sz
+
+// tc : https://en.wikipedia.org/wiki/.tc
+tc
+
+// td : https://en.wikipedia.org/wiki/.td
+td
+
+// tel: https://en.wikipedia.org/wiki/.tel
+// http://www.telnic.org/
+tel
+
+// tf : https://en.wikipedia.org/wiki/.tf
+tf
+
+// tg : https://en.wikipedia.org/wiki/.tg
+// http://www.nic.tg/
+tg
+
+// th : https://en.wikipedia.org/wiki/.th
+// Submitted by registry <krit@thains.co.th>
+th
+ac.th
+co.th
+go.th
+in.th
+mi.th
+net.th
+or.th
+
+// tj : http://www.nic.tj/policy.html
+tj
+ac.tj
+biz.tj
+co.tj
+com.tj
+edu.tj
+go.tj
+gov.tj
+int.tj
+mil.tj
+name.tj
+net.tj
+nic.tj
+org.tj
+test.tj
+web.tj
+
+// tk : https://en.wikipedia.org/wiki/.tk
+tk
+
+// tl : https://en.wikipedia.org/wiki/.tl
+tl
+gov.tl
+
+// tm : http://www.nic.tm/local.html
+tm
+com.tm
+co.tm
+org.tm
+net.tm
+nom.tm
+gov.tm
+mil.tm
+edu.tm
+
+// tn : https://en.wikipedia.org/wiki/.tn
+// http://whois.ati.tn/
+tn
+com.tn
+ens.tn
+fin.tn
+gov.tn
+ind.tn
+intl.tn
+nat.tn
+net.tn
+org.tn
+info.tn
+perso.tn
+tourism.tn
+edunet.tn
+rnrt.tn
+rns.tn
+rnu.tn
+mincom.tn
+agrinet.tn
+defense.tn
+turen.tn
+
+// to : https://en.wikipedia.org/wiki/.to
+// Submitted by registry <egullich@colo.to>
+to
+com.to
+gov.to
+net.to
+org.to
+edu.to
+mil.to
+
+// tr : https://nic.tr/
+// https://nic.tr/forms/eng/policies.pdf
+// https://nic.tr/index.php?USRACTN=PRICELST
+tr
+av.tr
+bbs.tr
+bel.tr
+biz.tr
+com.tr
+dr.tr
+edu.tr
+gen.tr
+gov.tr
+info.tr
+mil.tr
+k12.tr
+kep.tr
+name.tr
+net.tr
+org.tr
+pol.tr
+tel.tr
+tsk.tr
+tv.tr
+web.tr
+// Used by Northern Cyprus
+nc.tr
+// Used by government agencies of Northern Cyprus
+gov.nc.tr
+
+// tt : http://www.nic.tt/
+tt
+co.tt
+com.tt
+org.tt
+net.tt
+biz.tt
+info.tt
+pro.tt
+int.tt
+coop.tt
+jobs.tt
+mobi.tt
+travel.tt
+museum.tt
+aero.tt
+name.tt
+gov.tt
+edu.tt
+
+// tv : https://en.wikipedia.org/wiki/.tv
+// Not listing any 2LDs as reserved since none seem to exist in practice,
+// Wikipedia notwithstanding.
+tv
+
+// tw : https://en.wikipedia.org/wiki/.tw
+tw
+edu.tw
+gov.tw
+mil.tw
+com.tw
+net.tw
+org.tw
+idv.tw
+game.tw
+ebiz.tw
+club.tw
+xn--zf0ao64a.tw
+網路.tw
+xn--uc0atv.tw
+組織.tw
+xn--czrw28b.tw
+商業.tw
+
+// tz : http://www.tznic.or.tz/index.php/domains
+// Submitted by registry <manager@tznic.or.tz>
+tz
+ac.tz
+co.tz
+go.tz
+hotel.tz
+info.tz
+me.tz
+mil.tz
+mobi.tz
+ne.tz
+or.tz
+sc.tz
+tv.tz
+
+// ua : https://hostmaster.ua/policy/?ua
+// Submitted by registry <dk@cctld.ua>
+ua
+// ua 2LD
+com.ua
+edu.ua
+gov.ua
+in.ua
+net.ua
+org.ua
+// ua geographic names
+// https://hostmaster.ua/2ld/
+cherkassy.ua
+cherkasy.ua
+chernigov.ua
+chernihiv.ua
+chernivtsi.ua
+chernovtsy.ua
+ck.ua
+cn.ua
+cr.ua
+crimea.ua
+cv.ua
+dn.ua
+dnepropetrovsk.ua
+dnipropetrovsk.ua
+donetsk.ua
+dp.ua
+if.ua
+ivano-frankivsk.ua
+kh.ua
+kharkiv.ua
+kharkov.ua
+kherson.ua
+khmelnitskiy.ua
+khmelnytskyi.ua
+kiev.ua
+kirovograd.ua
+km.ua
+kr.ua
+krym.ua
+ks.ua
+kv.ua
+kyiv.ua
+lg.ua
+lt.ua
+lugansk.ua
+lutsk.ua
+lv.ua
+lviv.ua
+mk.ua
+mykolaiv.ua
+nikolaev.ua
+od.ua
+odesa.ua
+odessa.ua
+pl.ua
+poltava.ua
+rivne.ua
+rovno.ua
+rv.ua
+sb.ua
+sebastopol.ua
+sevastopol.ua
+sm.ua
+sumy.ua
+te.ua
+ternopil.ua
+uz.ua
+uzhgorod.ua
+vinnica.ua
+vinnytsia.ua
+vn.ua
+volyn.ua
+yalta.ua
+zaporizhzhe.ua
+zaporizhzhia.ua
+zhitomir.ua
+zhytomyr.ua
+zp.ua
+zt.ua
+
+// ug : https://www.registry.co.ug/
+ug
+co.ug
+or.ug
+ac.ug
+sc.ug
+go.ug
+ne.ug
+com.ug
+org.ug
+
+// uk : https://en.wikipedia.org/wiki/.uk
+// Submitted by registry <Michael.Daly@nominet.org.uk>
+uk
+ac.uk
+co.uk
+gov.uk
+ltd.uk
+me.uk
+net.uk
+nhs.uk
+org.uk
+plc.uk
+police.uk
+*.sch.uk
+
+// us : https://en.wikipedia.org/wiki/.us
+us
+dni.us
+fed.us
+isa.us
+kids.us
+nsn.us
+// us geographic names
+ak.us
+al.us
+ar.us
+as.us
+az.us
+ca.us
+co.us
+ct.us
+dc.us
+de.us
+fl.us
+ga.us
+gu.us
+hi.us
+ia.us
+id.us
+il.us
+in.us
+ks.us
+ky.us
+la.us
+ma.us
+md.us
+me.us
+mi.us
+mn.us
+mo.us
+ms.us
+mt.us
+nc.us
+nd.us
+ne.us
+nh.us
+nj.us
+nm.us
+nv.us
+ny.us
+oh.us
+ok.us
+or.us
+pa.us
+pr.us
+ri.us
+sc.us
+sd.us
+tn.us
+tx.us
+ut.us
+vi.us
+vt.us
+va.us
+wa.us
+wi.us
+wv.us
+wy.us
+// The registrar notes several more specific domains available in each state,
+// such as state.*.us, dst.*.us, etc., but resolution of these is somewhat
+// haphazard; in some states these domains resolve as addresses, while in others
+// only subdomains are available, or even nothing at all. We include the
+// most common ones where it's clear that different sites are different
+// entities.
+k12.ak.us
+k12.al.us
+k12.ar.us
+k12.as.us
+k12.az.us
+k12.ca.us
+k12.co.us
+k12.ct.us
+k12.dc.us
+k12.de.us
+k12.fl.us
+k12.ga.us
+k12.gu.us
+// k12.hi.us Bug 614565 - Hawaii has a state-wide DOE login
+k12.ia.us
+k12.id.us
+k12.il.us
+k12.in.us
+k12.ks.us
+k12.ky.us
+k12.la.us
+k12.ma.us
+k12.md.us
+k12.me.us
+k12.mi.us
+k12.mn.us
+k12.mo.us
+k12.ms.us
+k12.mt.us
+k12.nc.us
+// k12.nd.us Bug 1028347 - Removed at request of Travis Rosso <trossow@nd.gov>
+k12.ne.us
+k12.nh.us
+k12.nj.us
+k12.nm.us
+k12.nv.us
+k12.ny.us
+k12.oh.us
+k12.ok.us
+k12.or.us
+k12.pa.us
+k12.pr.us
+// k12.ri.us Removed at request of Kim Cournoyer <netsupport@staff.ri.net>
+k12.sc.us
+// k12.sd.us Bug 934131 - Removed at request of James Booze <James.Booze@k12.sd.us>
+k12.tn.us
+k12.tx.us
+k12.ut.us
+k12.vi.us
+k12.vt.us
+k12.va.us
+k12.wa.us
+k12.wi.us
+// k12.wv.us Bug 947705 - Removed at request of Verne Britton <verne@wvnet.edu>
+k12.wy.us
+cc.ak.us
+cc.al.us
+cc.ar.us
+cc.as.us
+cc.az.us
+cc.ca.us
+cc.co.us
+cc.ct.us
+cc.dc.us
+cc.de.us
+cc.fl.us
+cc.ga.us
+cc.gu.us
+cc.hi.us
+cc.ia.us
+cc.id.us
+cc.il.us
+cc.in.us
+cc.ks.us
+cc.ky.us
+cc.la.us
+cc.ma.us
+cc.md.us
+cc.me.us
+cc.mi.us
+cc.mn.us
+cc.mo.us
+cc.ms.us
+cc.mt.us
+cc.nc.us
+cc.nd.us
+cc.ne.us
+cc.nh.us
+cc.nj.us
+cc.nm.us
+cc.nv.us
+cc.ny.us
+cc.oh.us
+cc.ok.us
+cc.or.us
+cc.pa.us
+cc.pr.us
+cc.ri.us
+cc.sc.us
+cc.sd.us
+cc.tn.us
+cc.tx.us
+cc.ut.us
+cc.vi.us
+cc.vt.us
+cc.va.us
+cc.wa.us
+cc.wi.us
+cc.wv.us
+cc.wy.us
+lib.ak.us
+lib.al.us
+lib.ar.us
+lib.as.us
+lib.az.us
+lib.ca.us
+lib.co.us
+lib.ct.us
+lib.dc.us
+// lib.de.us Issue #243 - Moved to Private section at request of Ed Moore <Ed.Moore@lib.de.us>
+lib.fl.us
+lib.ga.us
+lib.gu.us
+lib.hi.us
+lib.ia.us
+lib.id.us
+lib.il.us
+lib.in.us
+lib.ks.us
+lib.ky.us
+lib.la.us
+lib.ma.us
+lib.md.us
+lib.me.us
+lib.mi.us
+lib.mn.us
+lib.mo.us
+lib.ms.us
+lib.mt.us
+lib.nc.us
+lib.nd.us
+lib.ne.us
+lib.nh.us
+lib.nj.us
+lib.nm.us
+lib.nv.us
+lib.ny.us
+lib.oh.us
+lib.ok.us
+lib.or.us
+lib.pa.us
+lib.pr.us
+lib.ri.us
+lib.sc.us
+lib.sd.us
+lib.tn.us
+lib.tx.us
+lib.ut.us
+lib.vi.us
+lib.vt.us
+lib.va.us
+lib.wa.us
+lib.wi.us
+// lib.wv.us Bug 941670 - Removed at request of Larry W Arnold <arnold@wvlc.lib.wv.us>
+lib.wy.us
+// k12.ma.us contains school districts in Massachusetts. The 4LDs are
+// managed independently except for private (PVT), charter (CHTR) and
+// parochial (PAROCH) schools. Those are delegated directly to the
+// 5LD operators. <k12-ma-hostmaster _ at _ rsuc.gweep.net>
+pvt.k12.ma.us
+chtr.k12.ma.us
+paroch.k12.ma.us
+// Merit Network, Inc. maintains the registry for =~ /(k12|cc|lib).mi.us/ and the following
+// see also: http://domreg.merit.edu
+// see also: whois -h whois.domreg.merit.edu help
+ann-arbor.mi.us
+cog.mi.us
+dst.mi.us
+eaton.mi.us
+gen.mi.us
+mus.mi.us
+tec.mi.us
+washtenaw.mi.us
+
+// uy : http://www.nic.org.uy/
+uy
+com.uy
+edu.uy
+gub.uy
+mil.uy
+net.uy
+org.uy
+
+// uz : http://www.reg.uz/
+uz
+co.uz
+com.uz
+net.uz
+org.uz
+
+// va : https://en.wikipedia.org/wiki/.va
+va
+
+// vc : https://en.wikipedia.org/wiki/.vc
+// Submitted by registry <kshah@ca.afilias.info>
+vc
+com.vc
+net.vc
+org.vc
+gov.vc
+mil.vc
+edu.vc
+
+// ve : https://registro.nic.ve/
+// Submitted by registry
+ve
+arts.ve
+co.ve
+com.ve
+e12.ve
+edu.ve
+firm.ve
+gob.ve
+gov.ve
+info.ve
+int.ve
+mil.ve
+net.ve
+org.ve
+rec.ve
+store.ve
+tec.ve
+web.ve
+
+// vg : https://en.wikipedia.org/wiki/.vg
+vg
+
+// vi : http://www.nic.vi/newdomainform.htm
+// http://www.nic.vi/Domain_Rules/body_domain_rules.html indicates some other
+// TLDs are "reserved", such as edu.vi and gov.vi, but doesn't actually say they
+// are available for registration (which they do not seem to be).
+vi
+co.vi
+com.vi
+k12.vi
+net.vi
+org.vi
+
+// vn : https://www.dot.vn/vnnic/vnnic/domainregistration.jsp
+vn
+com.vn
+net.vn
+org.vn
+edu.vn
+gov.vn
+int.vn
+ac.vn
+biz.vn
+info.vn
+name.vn
+pro.vn
+health.vn
+
+// vu : https://en.wikipedia.org/wiki/.vu
+// http://www.vunic.vu/
+vu
+com.vu
+edu.vu
+net.vu
+org.vu
+
+// wf : http://www.afnic.fr/medias/documents/AFNIC-naming-policy2012.pdf
+wf
+
+// ws : https://en.wikipedia.org/wiki/.ws
+// http://samoanic.ws/index.dhtml
+ws
+com.ws
+net.ws
+org.ws
+gov.ws
+edu.ws
+
+// yt : http://www.afnic.fr/medias/documents/AFNIC-naming-policy2012.pdf
+yt
+
+// IDN ccTLDs
+// When submitting patches, please maintain a sort by ISO 3166 ccTLD, then
+// U-label, and follow this format:
+// // A-Label ("<Latin renderings>", <language name>[, variant info]) : <ISO 3166 ccTLD>
+// // [sponsoring org]
+// U-Label
+
+// xn--mgbaam7a8h ("Emerat", Arabic) : AE
+// http://nic.ae/english/arabicdomain/rules.jsp
+xn--mgbaam7a8h
+امارات
+
+// xn--y9a3aq ("hye", Armenian) : AM
+// ISOC AM (operated by .am Registry)
+xn--y9a3aq
+հայ
+
+// xn--54b7fta0cc ("Bangla", Bangla) : BD
+xn--54b7fta0cc
+বাংলা
+
+// xn--90ae ("bg", Bulgarian) : BG
+xn--90ae
+бг
+
+// xn--mgbcpq6gpa1a ("albahrain", Arabic) : BH
+xn--mgbcpq6gpa1a
+البحرين
+
+// xn--90ais ("bel", Belarusian/Russian Cyrillic) : BY
+// Operated by .by registry
+xn--90ais
+бел
+
+// xn--fiqs8s ("Zhongguo/China", Chinese, Simplified) : CN
+// CNNIC
+// http://cnnic.cn/html/Dir/2005/10/11/3218.htm
+xn--fiqs8s
+中国
+
+// xn--fiqz9s ("Zhongguo/China", Chinese, Traditional) : CN
+// CNNIC
+// http://cnnic.cn/html/Dir/2005/10/11/3218.htm
+xn--fiqz9s
+中國
+
+// xn--lgbbat1ad8j ("Algeria/Al Jazair", Arabic) : DZ
+xn--lgbbat1ad8j
+الجزائر
+
+// xn--wgbh1c ("Egypt/Masr", Arabic) : EG
+// http://www.dotmasr.eg/
+xn--wgbh1c
+مصر
+
+// xn--e1a4c ("eu", Cyrillic) : EU
+// https://eurid.eu
+xn--e1a4c
+ею
+
+// xn--qxa6a ("eu", Greek) : EU
+// https://eurid.eu
+xn--qxa6a
+ευ
+
+// xn--mgbah1a3hjkrd ("Mauritania", Arabic) : MR
+xn--mgbah1a3hjkrd
+موريتانيا
+
+// xn--node ("ge", Georgian Mkhedruli) : GE
+xn--node
+გე
+
+// xn--qxam ("el", Greek) : GR
+// Hellenic Ministry of Infrastructure, Transport, and Networks
+xn--qxam
+ελ
+
+// xn--j6w193g ("Hong Kong", Chinese) : HK
+// https://www.hkirc.hk
+// Submitted by registry <hk.tech@hkirc.hk>
+// https://www.hkirc.hk/content.jsp?id=30#!/34
+xn--j6w193g
+香港
+xn--55qx5d.xn--j6w193g
+公司.香港
+xn--wcvs22d.xn--j6w193g
+教育.香港
+xn--mxtq1m.xn--j6w193g
+政府.香港
+xn--gmqw5a.xn--j6w193g
+個人.香港
+xn--od0alg.xn--j6w193g
+網絡.香港
+xn--uc0atv.xn--j6w193g
+組織.香港
+
+// xn--2scrj9c ("Bharat", Kannada) : IN
+// India
+xn--2scrj9c
+ಭಾರತ
+
+// xn--3hcrj9c ("Bharat", Oriya) : IN
+// India
+xn--3hcrj9c
+ଭାରତ
+
+// xn--45br5cyl ("Bharatam", Assamese) : IN
+// India
+xn--45br5cyl
+ভাৰত
+
+// xn--h2breg3eve ("Bharatam", Sanskrit) : IN
+// India
+xn--h2breg3eve
+भारतम्
+
+// xn--h2brj9c8c ("Bharot", Santali) : IN
+// India
+xn--h2brj9c8c
+भारोत
+
+// xn--mgbgu82a ("Bharat", Sindhi) : IN
+// India
+xn--mgbgu82a
+ڀارت
+
+// xn--rvc1e0am3e ("Bharatam", Malayalam) : IN
+// India
+xn--rvc1e0am3e
+ഭാരതം
+
+// xn--h2brj9c ("Bharat", Devanagari) : IN
+// India
+xn--h2brj9c
+भारत
+
+// xn--mgbbh1a ("Bharat", Kashmiri) : IN
+// India
+xn--mgbbh1a
+بارت
+
+// xn--mgbbh1a71e ("Bharat", Arabic) : IN
+// India
+xn--mgbbh1a71e
+بھارت
+
+// xn--fpcrj9c3d ("Bharat", Telugu) : IN
+// India
+xn--fpcrj9c3d
+భారత్
+
+// xn--gecrj9c ("Bharat", Gujarati) : IN
+// India
+xn--gecrj9c
+ભારત
+
+// xn--s9brj9c ("Bharat", Gurmukhi) : IN
+// India
+xn--s9brj9c
+ਭਾਰਤ
+
+// xn--45brj9c ("Bharat", Bengali) : IN
+// India
+xn--45brj9c
+ভারত
+
+// xn--xkc2dl3a5ee0h ("India", Tamil) : IN
+// India
+xn--xkc2dl3a5ee0h
+இந்தியா
+
+// xn--mgba3a4f16a ("Iran", Persian) : IR
+xn--mgba3a4f16a
+ایران
+
+// xn--mgba3a4fra ("Iran", Arabic) : IR
+xn--mgba3a4fra
+ايران
+
+// xn--mgbtx2b ("Iraq", Arabic) : IQ
+// Communications and Media Commission
+xn--mgbtx2b
+عراق
+
+// xn--mgbayh7gpa ("al-Ordon", Arabic) : JO
+// National Information Technology Center (NITC)
+// Royal Scientific Society, Al-Jubeiha
+xn--mgbayh7gpa
+الاردن
+
+// xn--3e0b707e ("Republic of Korea", Hangul) : KR
+xn--3e0b707e
+한국
+
+// xn--80ao21a ("Kaz", Kazakh) : KZ
+xn--80ao21a
+қаз
+
+// xn--q7ce6a ("Lao", Lao) : LA
+xn--q7ce6a
+ລາວ
+
+// xn--fzc2c9e2c ("Lanka", Sinhalese-Sinhala) : LK
+// https://nic.lk
+xn--fzc2c9e2c
+ලංකා
+
+// xn--xkc2al3hye2a ("Ilangai", Tamil) : LK
+// https://nic.lk
+xn--xkc2al3hye2a
+இலங்கை
+
+// xn--mgbc0a9azcg ("Morocco/al-Maghrib", Arabic) : MA
+xn--mgbc0a9azcg
+المغرب
+
+// xn--d1alf ("mkd", Macedonian) : MK
+// MARnet
+xn--d1alf
+мкд
+
+// xn--l1acc ("mon", Mongolian) : MN
+xn--l1acc
+мон
+
+// xn--mix891f ("Macao", Chinese, Traditional) : MO
+// MONIC / HNET Asia (Registry Operator for .mo)
+xn--mix891f
+澳門
+
+// xn--mix082f ("Macao", Chinese, Simplified) : MO
+xn--mix082f
+澳门
+
+// xn--mgbx4cd0ab ("Malaysia", Malay) : MY
+xn--mgbx4cd0ab
+مليسيا
+
+// xn--mgb9awbf ("Oman", Arabic) : OM
+xn--mgb9awbf
+عمان
+
+// xn--mgbai9azgqp6j ("Pakistan", Urdu/Arabic) : PK
+xn--mgbai9azgqp6j
+پاکستان
+
+// xn--mgbai9a5eva00b ("Pakistan", Urdu/Arabic, variant) : PK
+xn--mgbai9a5eva00b
+پاكستان
+
+// xn--ygbi2ammx ("Falasteen", Arabic) : PS
+// The Palestinian National Internet Naming Authority (PNINA)
+// http://www.pnina.ps
+xn--ygbi2ammx
+فلسطين
+
+// xn--90a3ac ("srb", Cyrillic) : RS
+// https://www.rnids.rs/en/domains/national-domains
+xn--90a3ac
+срб
+xn--o1ac.xn--90a3ac
+пр.срб
+xn--c1avg.xn--90a3ac
+орг.срб
+xn--90azh.xn--90a3ac
+обр.срб
+xn--d1at.xn--90a3ac
+од.срб
+xn--o1ach.xn--90a3ac
+упр.срб
+xn--80au.xn--90a3ac
+ак.срб
+
+// xn--p1ai ("rf", Russian-Cyrillic) : RU
+// https://cctld.ru/files/pdf/docs/en/rules_ru-rf.pdf
+// Submitted by George Georgievsky <gug@cctld.ru>
+xn--p1ai
+рф
+
+// xn--wgbl6a ("Qatar", Arabic) : QA
+// http://www.ict.gov.qa/
+xn--wgbl6a
+قطر
+
+// xn--mgberp4a5d4ar ("AlSaudiah", Arabic) : SA
+// http://www.nic.net.sa/
+xn--mgberp4a5d4ar
+السعودية
+
+// xn--mgberp4a5d4a87g ("AlSaudiah", Arabic, variant) : SA
+xn--mgberp4a5d4a87g
+السعودیة
+
+// xn--mgbqly7c0a67fbc ("AlSaudiah", Arabic, variant) : SA
+xn--mgbqly7c0a67fbc
+السعودیۃ
+
+// xn--mgbqly7cvafr ("AlSaudiah", Arabic, variant) : SA
+xn--mgbqly7cvafr
+السعوديه
+
+// xn--mgbpl2fh ("sudan", Arabic) : SD
+// Operated by .sd registry
+xn--mgbpl2fh
+سودان
+
+// xn--yfro4i67o Singapore ("Singapore", Chinese) : SG
+xn--yfro4i67o
+新加坡
+
+// xn--clchc0ea0b2g2a9gcd ("Singapore", Tamil) : SG
+xn--clchc0ea0b2g2a9gcd
+சிங்கப்பூர்
+
+// xn--ogbpf8fl ("Syria", Arabic) : SY
+xn--ogbpf8fl
+سورية
+
+// xn--mgbtf8fl ("Syria", Arabic, variant) : SY
+xn--mgbtf8fl
+سوريا
+
+// xn--o3cw4h ("Thai", Thai) : TH
+// http://www.thnic.co.th
+xn--o3cw4h
+ไทย
+xn--12c1fe0br.xn--o3cw4h
+ศึกษา.ไทย
+xn--12co0c3b4eva.xn--o3cw4h
+ธุรกิจ.ไทย
+xn--h3cuzk1di.xn--o3cw4h
+รัฐบาล.ไทย
+xn--o3cyx2a.xn--o3cw4h
+ทหาร.ไทย
+xn--m3ch0j3a.xn--o3cw4h
+เน็ต.ไทย
+xn--12cfi8ixb8l.xn--o3cw4h
+องค์กร.ไทย
+
+// xn--pgbs0dh ("Tunisia", Arabic) : TN
+// http://nic.tn
+xn--pgbs0dh
+تونس
+
+// xn--kpry57d ("Taiwan", Chinese, Traditional) : TW
+// http://www.twnic.net/english/dn/dn_07a.htm
+xn--kpry57d
+台灣
+
+// xn--kprw13d ("Taiwan", Chinese, Simplified) : TW
+// http://www.twnic.net/english/dn/dn_07a.htm
+xn--kprw13d
+台湾
+
+// xn--nnx388a ("Taiwan", Chinese, variant) : TW
+xn--nnx388a
+臺灣
+
+// xn--j1amh ("ukr", Cyrillic) : UA
+xn--j1amh
+укр
+
+// xn--mgb2ddes ("AlYemen", Arabic) : YE
+xn--mgb2ddes
+اليمن
+
+// xxx : http://icmregistry.com
+xxx
+
+// ye : http://www.y.net.ye/services/domain_name.htm
+ye
+com.ye
+edu.ye
+gov.ye
+net.ye
+mil.ye
+org.ye
+
+// za : https://www.zadna.org.za/content/page/domain-information/
+ac.za
+agric.za
+alt.za
+co.za
+edu.za
+gov.za
+grondar.za
+law.za
+mil.za
+net.za
+ngo.za
+nic.za
+nis.za
+nom.za
+org.za
+school.za
+tm.za
+web.za
+
+// zm : https://zicta.zm/
+// Submitted by registry <info@zicta.zm>
+zm
+ac.zm
+biz.zm
+co.zm
+com.zm
+edu.zm
+gov.zm
+info.zm
+mil.zm
+net.zm
+org.zm
+sch.zm
+
+// zw : https://www.potraz.gov.zw/
+// Confirmed by registry <bmtengwa@potraz.gov.zw> 2017-01-25
+zw
+ac.zw
+co.zw
+gov.zw
+mil.zw
+org.zw
+
+
+// newGTLDs
+
+// List of new gTLDs imported from https://www.icann.org/resources/registries/gtlds/v2/gtlds.json on 2021-08-05T15:14:20Z
+// This list is auto-generated, don't edit it manually.
+// aaa : 2015-02-26 American Automobile Association, Inc.
+aaa
+
+// aarp : 2015-05-21 AARP
+aarp
+
+// abarth : 2015-07-30 Fiat Chrysler Automobiles N.V.
+abarth
+
+// abb : 2014-10-24 ABB Ltd
+abb
+
+// abbott : 2014-07-24 Abbott Laboratories, Inc.
+abbott
+
+// abbvie : 2015-07-30 AbbVie Inc.
+abbvie
+
+// abc : 2015-07-30 Disney Enterprises, Inc.
+abc
+
+// able : 2015-06-25 Able Inc.
+able
+
+// abogado : 2014-04-24 Registry Services, LLC
+abogado
+
+// abudhabi : 2015-07-30 Abu Dhabi Systems and Information Centre
+abudhabi
+
+// academy : 2013-11-07 Binky Moon, LLC
+academy
+
+// accenture : 2014-08-15 Accenture plc
+accenture
+
+// accountant : 2014-11-20 dot Accountant Limited
+accountant
+
+// accountants : 2014-03-20 Binky Moon, LLC
+accountants
+
+// aco : 2015-01-08 ACO Severin Ahlmann GmbH & Co. KG
+aco
+
+// actor : 2013-12-12 Dog Beach, LLC
+actor
+
+// adac : 2015-07-16 Allgemeiner Deutscher Automobil-Club e.V. (ADAC)
+adac
+
+// ads : 2014-12-04 Charleston Road Registry Inc.
+ads
+
+// adult : 2014-10-16 ICM Registry AD LLC
+adult
+
+// aeg : 2015-03-19 Aktiebolaget Electrolux
+aeg
+
+// aetna : 2015-05-21 Aetna Life Insurance Company
+aetna
+
+// afamilycompany : 2015-07-23 Johnson Shareholdings, Inc.
+afamilycompany
+
+// afl : 2014-10-02 Australian Football League
+afl
+
+// africa : 2014-03-24 ZA Central Registry NPC trading as Registry.Africa
+africa
+
+// agakhan : 2015-04-23 Fondation Aga Khan (Aga Khan Foundation)
+agakhan
+
+// agency : 2013-11-14 Binky Moon, LLC
+agency
+
+// aig : 2014-12-18 American International Group, Inc.
+aig
+
+// airbus : 2015-07-30 Airbus S.A.S.
+airbus
+
+// airforce : 2014-03-06 Dog Beach, LLC
+airforce
+
+// airtel : 2014-10-24 Bharti Airtel Limited
+airtel
+
+// akdn : 2015-04-23 Fondation Aga Khan (Aga Khan Foundation)
+akdn
+
+// alfaromeo : 2015-07-31 Fiat Chrysler Automobiles N.V.
+alfaromeo
+
+// alibaba : 2015-01-15 Alibaba Group Holding Limited
+alibaba
+
+// alipay : 2015-01-15 Alibaba Group Holding Limited
+alipay
+
+// allfinanz : 2014-07-03 Allfinanz Deutsche Vermögensberatung Aktiengesellschaft
+allfinanz
+
+// allstate : 2015-07-31 Allstate Fire and Casualty Insurance Company
+allstate
+
+// ally : 2015-06-18 Ally Financial Inc.
+ally
+
+// alsace : 2014-07-02 Region Grand Est
+alsace
+
+// alstom : 2015-07-30 ALSTOM
+alstom
+
+// amazon : 2019-12-19 Amazon Registry Services, Inc.
+amazon
+
+// americanexpress : 2015-07-31 American Express Travel Related Services Company, Inc.
+americanexpress
+
+// americanfamily : 2015-07-23 AmFam, Inc.
+americanfamily
+
+// amex : 2015-07-31 American Express Travel Related Services Company, Inc.
+amex
+
+// amfam : 2015-07-23 AmFam, Inc.
+amfam
+
+// amica : 2015-05-28 Amica Mutual Insurance Company
+amica
+
+// amsterdam : 2014-07-24 Gemeente Amsterdam
+amsterdam
+
+// analytics : 2014-12-18 Campus IP LLC
+analytics
+
+// android : 2014-08-07 Charleston Road Registry Inc.
+android
+
+// anquan : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+anquan
+
+// anz : 2015-07-31 Australia and New Zealand Banking Group Limited
+anz
+
+// aol : 2015-09-17 Oath Inc.
+aol
+
+// apartments : 2014-12-11 Binky Moon, LLC
+apartments
+
+// app : 2015-05-14 Charleston Road Registry Inc.
+app
+
+// apple : 2015-05-14 Apple Inc.
+apple
+
+// aquarelle : 2014-07-24 Aquarelle.com
+aquarelle
+
+// arab : 2015-11-12 League of Arab States
+arab
+
+// aramco : 2014-11-20 Aramco Services Company
+aramco
+
+// archi : 2014-02-06 Afilias Limited
+archi
+
+// army : 2014-03-06 Dog Beach, LLC
+army
+
+// art : 2016-03-24 UK Creative Ideas Limited
+art
+
+// arte : 2014-12-11 Association Relative à la Télévision Européenne G.E.I.E.
+arte
+
+// asda : 2015-07-31 Wal-Mart Stores, Inc.
+asda
+
+// associates : 2014-03-06 Binky Moon, LLC
+associates
+
+// athleta : 2015-07-30 The Gap, Inc.
+athleta
+
+// attorney : 2014-03-20 Dog Beach, LLC
+attorney
+
+// auction : 2014-03-20 Dog Beach, LLC
+auction
+
+// audi : 2015-05-21 AUDI Aktiengesellschaft
+audi
+
+// audible : 2015-06-25 Amazon Registry Services, Inc.
+audible
+
+// audio : 2014-03-20 UNR Corp.
+audio
+
+// auspost : 2015-08-13 Australian Postal Corporation
+auspost
+
+// author : 2014-12-18 Amazon Registry Services, Inc.
+author
+
+// auto : 2014-11-13 XYZ.COM LLC
+auto
+
+// autos : 2014-01-09 XYZ.COM LLC
+autos
+
+// avianca : 2015-01-08 Avianca Holdings S.A.
+avianca
+
+// aws : 2015-06-25 AWS Registry LLC
+aws
+
+// axa : 2013-12-19 AXA Group Operations SAS
+axa
+
+// azure : 2014-12-18 Microsoft Corporation
+azure
+
+// baby : 2015-04-09 XYZ.COM LLC
+baby
+
+// baidu : 2015-01-08 Baidu, Inc.
+baidu
+
+// banamex : 2015-07-30 Citigroup Inc.
+banamex
+
+// bananarepublic : 2015-07-31 The Gap, Inc.
+bananarepublic
+
+// band : 2014-06-12 Dog Beach, LLC
+band
+
+// bank : 2014-09-25 fTLD Registry Services LLC
+bank
+
+// bar : 2013-12-12 Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable
+bar
+
+// barcelona : 2014-07-24 Municipi de Barcelona
+barcelona
+
+// barclaycard : 2014-11-20 Barclays Bank PLC
+barclaycard
+
+// barclays : 2014-11-20 Barclays Bank PLC
+barclays
+
+// barefoot : 2015-06-11 Gallo Vineyards, Inc.
+barefoot
+
+// bargains : 2013-11-14 Binky Moon, LLC
+bargains
+
+// baseball : 2015-10-29 MLB Advanced Media DH, LLC
+baseball
+
+// basketball : 2015-08-20 Fédération Internationale de Basketball (FIBA)
+basketball
+
+// bauhaus : 2014-04-17 Werkhaus GmbH
+bauhaus
+
+// bayern : 2014-01-23 Bayern Connect GmbH
+bayern
+
+// bbc : 2014-12-18 British Broadcasting Corporation
+bbc
+
+// bbt : 2015-07-23 BB&T Corporation
+bbt
+
+// bbva : 2014-10-02 BANCO BILBAO VIZCAYA ARGENTARIA, S.A.
+bbva
+
+// bcg : 2015-04-02 The Boston Consulting Group, Inc.
+bcg
+
+// bcn : 2014-07-24 Municipi de Barcelona
+bcn
+
+// beats : 2015-05-14 Beats Electronics, LLC
+beats
+
+// beauty : 2015-12-03 XYZ.COM LLC
+beauty
+
+// beer : 2014-01-09 Registry Services, LLC
+beer
+
+// bentley : 2014-12-18 Bentley Motors Limited
+bentley
+
+// berlin : 2013-10-31 dotBERLIN GmbH & Co. KG
+berlin
+
+// best : 2013-12-19 BestTLD Pty Ltd
+best
+
+// bestbuy : 2015-07-31 BBY Solutions, Inc.
+bestbuy
+
+// bet : 2015-05-07 Afilias Limited
+bet
+
+// bharti : 2014-01-09 Bharti Enterprises (Holding) Private Limited
+bharti
+
+// bible : 2014-06-19 American Bible Society
+bible
+
+// bid : 2013-12-19 dot Bid Limited
+bid
+
+// bike : 2013-08-27 Binky Moon, LLC
+bike
+
+// bing : 2014-12-18 Microsoft Corporation
+bing
+
+// bingo : 2014-12-04 Binky Moon, LLC
+bingo
+
+// bio : 2014-03-06 Afilias Limited
+bio
+
+// black : 2014-01-16 Afilias Limited
+black
+
+// blackfriday : 2014-01-16 UNR Corp.
+blackfriday
+
+// blockbuster : 2015-07-30 Dish DBS Corporation
+blockbuster
+
+// blog : 2015-05-14 Knock Knock WHOIS There, LLC
+blog
+
+// bloomberg : 2014-07-17 Bloomberg IP Holdings LLC
+bloomberg
+
+// blue : 2013-11-07 Afilias Limited
+blue
+
+// bms : 2014-10-30 Bristol-Myers Squibb Company
+bms
+
+// bmw : 2014-01-09 Bayerische Motoren Werke Aktiengesellschaft
+bmw
+
+// bnpparibas : 2014-05-29 BNP Paribas
+bnpparibas
+
+// boats : 2014-12-04 XYZ.COM LLC
+boats
+
+// boehringer : 2015-07-09 Boehringer Ingelheim International GmbH
+boehringer
+
+// bofa : 2015-07-31 Bank of America Corporation
+bofa
+
+// bom : 2014-10-16 Núcleo de Informação e Coordenação do Ponto BR - NIC.br
+bom
+
+// bond : 2014-06-05 ShortDot SA
+bond
+
+// boo : 2014-01-30 Charleston Road Registry Inc.
+boo
+
+// book : 2015-08-27 Amazon Registry Services, Inc.
+book
+
+// booking : 2015-07-16 Booking.com B.V.
+booking
+
+// bosch : 2015-06-18 Robert Bosch GMBH
+bosch
+
+// bostik : 2015-05-28 Bostik SA
+bostik
+
+// boston : 2015-12-10 Boston TLD Management, LLC
+boston
+
+// bot : 2014-12-18 Amazon Registry Services, Inc.
+bot
+
+// boutique : 2013-11-14 Binky Moon, LLC
+boutique
+
+// box : 2015-11-12 Intercap Registry Inc.
+box
+
+// bradesco : 2014-12-18 Banco Bradesco S.A.
+bradesco
+
+// bridgestone : 2014-12-18 Bridgestone Corporation
+bridgestone
+
+// broadway : 2014-12-22 Celebrate Broadway, Inc.
+broadway
+
+// broker : 2014-12-11 Dog Beach, LLC
+broker
+
+// brother : 2015-01-29 Brother Industries, Ltd.
+brother
+
+// brussels : 2014-02-06 DNS.be vzw
+brussels
+
+// budapest : 2013-11-21 Minds + Machines Group Limited
+budapest
+
+// bugatti : 2015-07-23 Bugatti International SA
+bugatti
+
+// build : 2013-11-07 Plan Bee LLC
+build
+
+// builders : 2013-11-07 Binky Moon, LLC
+builders
+
+// business : 2013-11-07 Binky Moon, LLC
+business
+
+// buy : 2014-12-18 Amazon Registry Services, Inc.
+buy
+
+// buzz : 2013-10-02 DOTSTRATEGY CO.
+buzz
+
+// bzh : 2014-02-27 Association www.bzh
+bzh
+
+// cab : 2013-10-24 Binky Moon, LLC
+cab
+
+// cafe : 2015-02-11 Binky Moon, LLC
+cafe
+
+// cal : 2014-07-24 Charleston Road Registry Inc.
+cal
+
+// call : 2014-12-18 Amazon Registry Services, Inc.
+call
+
+// calvinklein : 2015-07-30 PVH gTLD Holdings LLC
+calvinklein
+
+// cam : 2016-04-21 AC Webconnecting Holding B.V.
+cam
+
+// camera : 2013-08-27 Binky Moon, LLC
+camera
+
+// camp : 2013-11-07 Binky Moon, LLC
+camp
+
+// cancerresearch : 2014-05-15 Australian Cancer Research Foundation
+cancerresearch
+
+// canon : 2014-09-12 Canon Inc.
+canon
+
+// capetown : 2014-03-24 ZA Central Registry NPC trading as ZA Central Registry
+capetown
+
+// capital : 2014-03-06 Binky Moon, LLC
+capital
+
+// capitalone : 2015-08-06 Capital One Financial Corporation
+capitalone
+
+// car : 2015-01-22 XYZ.COM LLC
+car
+
+// caravan : 2013-12-12 Caravan International, Inc.
+caravan
+
+// cards : 2013-12-05 Binky Moon, LLC
+cards
+
+// care : 2014-03-06 Binky Moon, LLC
+care
+
+// career : 2013-10-09 dotCareer LLC
+career
+
+// careers : 2013-10-02 Binky Moon, LLC
+careers
+
+// cars : 2014-11-13 XYZ.COM LLC
+cars
+
+// casa : 2013-11-21 Registry Services, LLC
+casa
+
+// case : 2015-09-03 CNH Industrial N.V.
+case
+
+// cash : 2014-03-06 Binky Moon, LLC
+cash
+
+// casino : 2014-12-18 Binky Moon, LLC
+casino
+
+// catering : 2013-12-05 Binky Moon, LLC
+catering
+
+// catholic : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+catholic
+
+// cba : 2014-06-26 COMMONWEALTH BANK OF AUSTRALIA
+cba
+
+// cbn : 2014-08-22 The Christian Broadcasting Network, Inc.
+cbn
+
+// cbre : 2015-07-02 CBRE, Inc.
+cbre
+
+// cbs : 2015-08-06 CBS Domains Inc.
+cbs
+
+// center : 2013-11-07 Binky Moon, LLC
+center
+
+// ceo : 2013-11-07 CEOTLD Pty Ltd
+ceo
+
+// cern : 2014-06-05 European Organization for Nuclear Research ("CERN")
+cern
+
+// cfa : 2014-08-28 CFA Institute
+cfa
+
+// cfd : 2014-12-11 ShortDot SA
+cfd
+
+// chanel : 2015-04-09 Chanel International B.V.
+chanel
+
+// channel : 2014-05-08 Charleston Road Registry Inc.
+channel
+
+// charity : 2018-04-11 Binky Moon, LLC
+charity
+
+// chase : 2015-04-30 JPMorgan Chase Bank, National Association
+chase
+
+// chat : 2014-12-04 Binky Moon, LLC
+chat
+
+// cheap : 2013-11-14 Binky Moon, LLC
+cheap
+
+// chintai : 2015-06-11 CHINTAI Corporation
+chintai
+
+// christmas : 2013-11-21 UNR Corp.
+christmas
+
+// chrome : 2014-07-24 Charleston Road Registry Inc.
+chrome
+
+// church : 2014-02-06 Binky Moon, LLC
+church
+
+// cipriani : 2015-02-19 Hotel Cipriani Srl
+cipriani
+
+// circle : 2014-12-18 Amazon Registry Services, Inc.
+circle
+
+// cisco : 2014-12-22 Cisco Technology, Inc.
+cisco
+
+// citadel : 2015-07-23 Citadel Domain LLC
+citadel
+
+// citi : 2015-07-30 Citigroup Inc.
+citi
+
+// citic : 2014-01-09 CITIC Group Corporation
+citic
+
+// city : 2014-05-29 Binky Moon, LLC
+city
+
+// cityeats : 2014-12-11 Lifestyle Domain Holdings, Inc.
+cityeats
+
+// claims : 2014-03-20 Binky Moon, LLC
+claims
+
+// cleaning : 2013-12-05 Binky Moon, LLC
+cleaning
+
+// click : 2014-06-05 UNR Corp.
+click
+
+// clinic : 2014-03-20 Binky Moon, LLC
+clinic
+
+// clinique : 2015-10-01 The Estée Lauder Companies Inc.
+clinique
+
+// clothing : 2013-08-27 Binky Moon, LLC
+clothing
+
+// cloud : 2015-04-16 Aruba PEC S.p.A.
+cloud
+
+// club : 2013-11-08 Registry Services, LLC
+club
+
+// clubmed : 2015-06-25 Club Méditerranée S.A.
+clubmed
+
+// coach : 2014-10-09 Binky Moon, LLC
+coach
+
+// codes : 2013-10-31 Binky Moon, LLC
+codes
+
+// coffee : 2013-10-17 Binky Moon, LLC
+coffee
+
+// college : 2014-01-16 XYZ.COM LLC
+college
+
+// cologne : 2014-02-05 dotKoeln GmbH
+cologne
+
+// comcast : 2015-07-23 Comcast IP Holdings I, LLC
+comcast
+
+// commbank : 2014-06-26 COMMONWEALTH BANK OF AUSTRALIA
+commbank
+
+// community : 2013-12-05 Binky Moon, LLC
+community
+
+// company : 2013-11-07 Binky Moon, LLC
+company
+
+// compare : 2015-10-08 Registry Services, LLC
+compare
+
+// computer : 2013-10-24 Binky Moon, LLC
+computer
+
+// comsec : 2015-01-08 VeriSign, Inc.
+comsec
+
+// condos : 2013-12-05 Binky Moon, LLC
+condos
+
+// construction : 2013-09-16 Binky Moon, LLC
+construction
+
+// consulting : 2013-12-05 Dog Beach, LLC
+consulting
+
+// contact : 2015-01-08 Dog Beach, LLC
+contact
+
+// contractors : 2013-09-10 Binky Moon, LLC
+contractors
+
+// cooking : 2013-11-21 Registry Services, LLC
+cooking
+
+// cookingchannel : 2015-07-02 Lifestyle Domain Holdings, Inc.
+cookingchannel
+
+// cool : 2013-11-14 Binky Moon, LLC
+cool
+
+// corsica : 2014-09-25 Collectivité de Corse
+corsica
+
+// country : 2013-12-19 DotCountry LLC
+country
+
+// coupon : 2015-02-26 Amazon Registry Services, Inc.
+coupon
+
+// coupons : 2015-03-26 Binky Moon, LLC
+coupons
+
+// courses : 2014-12-04 OPEN UNIVERSITIES AUSTRALIA PTY LTD
+courses
+
+// cpa : 2019-06-10 American Institute of Certified Public Accountants
+cpa
+
+// credit : 2014-03-20 Binky Moon, LLC
+credit
+
+// creditcard : 2014-03-20 Binky Moon, LLC
+creditcard
+
+// creditunion : 2015-01-22 DotCooperation LLC
+creditunion
+
+// cricket : 2014-10-09 dot Cricket Limited
+cricket
+
+// crown : 2014-10-24 Crown Equipment Corporation
+crown
+
+// crs : 2014-04-03 Federated Co-operatives Limited
+crs
+
+// cruise : 2015-12-10 Viking River Cruises (Bermuda) Ltd.
+cruise
+
+// cruises : 2013-12-05 Binky Moon, LLC
+cruises
+
+// csc : 2014-09-25 Alliance-One Services, Inc.
+csc
+
+// cuisinella : 2014-04-03 SCHMIDT GROUPE S.A.S.
+cuisinella
+
+// cymru : 2014-05-08 Nominet UK
+cymru
+
+// cyou : 2015-01-22 ShortDot SA
+cyou
+
+// dabur : 2014-02-06 Dabur India Limited
+dabur
+
+// dad : 2014-01-23 Charleston Road Registry Inc.
+dad
+
+// dance : 2013-10-24 Dog Beach, LLC
+dance
+
+// data : 2016-06-02 Dish DBS Corporation
+data
+
+// date : 2014-11-20 dot Date Limited
+date
+
+// dating : 2013-12-05 Binky Moon, LLC
+dating
+
+// datsun : 2014-03-27 NISSAN MOTOR CO., LTD.
+datsun
+
+// day : 2014-01-30 Charleston Road Registry Inc.
+day
+
+// dclk : 2014-11-20 Charleston Road Registry Inc.
+dclk
+
+// dds : 2015-05-07 Registry Services, LLC
+dds
+
+// deal : 2015-06-25 Amazon Registry Services, Inc.
+deal
+
+// dealer : 2014-12-22 Intercap Registry Inc.
+dealer
+
+// deals : 2014-05-22 Binky Moon, LLC
+deals
+
+// degree : 2014-03-06 Dog Beach, LLC
+degree
+
+// delivery : 2014-09-11 Binky Moon, LLC
+delivery
+
+// dell : 2014-10-24 Dell Inc.
+dell
+
+// deloitte : 2015-07-31 Deloitte Touche Tohmatsu
+deloitte
+
+// delta : 2015-02-19 Delta Air Lines, Inc.
+delta
+
+// democrat : 2013-10-24 Dog Beach, LLC
+democrat
+
+// dental : 2014-03-20 Binky Moon, LLC
+dental
+
+// dentist : 2014-03-20 Dog Beach, LLC
+dentist
+
+// desi : 2013-11-14 Desi Networks LLC
+desi
+
+// design : 2014-11-07 Registry Services, LLC
+design
+
+// dev : 2014-10-16 Charleston Road Registry Inc.
+dev
+
+// dhl : 2015-07-23 Deutsche Post AG
+dhl
+
+// diamonds : 2013-09-22 Binky Moon, LLC
+diamonds
+
+// diet : 2014-06-26 UNR Corp.
+diet
+
+// digital : 2014-03-06 Binky Moon, LLC
+digital
+
+// direct : 2014-04-10 Binky Moon, LLC
+direct
+
+// directory : 2013-09-20 Binky Moon, LLC
+directory
+
+// discount : 2014-03-06 Binky Moon, LLC
+discount
+
+// discover : 2015-07-23 Discover Financial Services
+discover
+
+// dish : 2015-07-30 Dish DBS Corporation
+dish
+
+// diy : 2015-11-05 Lifestyle Domain Holdings, Inc.
+diy
+
+// dnp : 2013-12-13 Dai Nippon Printing Co., Ltd.
+dnp
+
+// docs : 2014-10-16 Charleston Road Registry Inc.
+docs
+
+// doctor : 2016-06-02 Binky Moon, LLC
+doctor
+
+// dog : 2014-12-04 Binky Moon, LLC
+dog
+
+// domains : 2013-10-17 Binky Moon, LLC
+domains
+
+// dot : 2015-05-21 Dish DBS Corporation
+dot
+
+// download : 2014-11-20 dot Support Limited
+download
+
+// drive : 2015-03-05 Charleston Road Registry Inc.
+drive
+
+// dtv : 2015-06-04 Dish DBS Corporation
+dtv
+
+// dubai : 2015-01-01 Dubai Smart Government Department
+dubai
+
+// duck : 2015-07-23 Johnson Shareholdings, Inc.
+duck
+
+// dunlop : 2015-07-02 The Goodyear Tire & Rubber Company
+dunlop
+
+// dupont : 2015-06-25 E. I. du Pont de Nemours and Company
+dupont
+
+// durban : 2014-03-24 ZA Central Registry NPC trading as ZA Central Registry
+durban
+
+// dvag : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+dvag
+
+// dvr : 2016-05-26 DISH Technologies L.L.C.
+dvr
+
+// earth : 2014-12-04 Interlink Co., Ltd.
+earth
+
+// eat : 2014-01-23 Charleston Road Registry Inc.
+eat
+
+// eco : 2016-07-08 Big Room Inc.
+eco
+
+// edeka : 2014-12-18 EDEKA Verband kaufmännischer Genossenschaften e.V.
+edeka
+
+// education : 2013-11-07 Binky Moon, LLC
+education
+
+// email : 2013-10-31 Binky Moon, LLC
+email
+
+// emerck : 2014-04-03 Merck KGaA
+emerck
+
+// energy : 2014-09-11 Binky Moon, LLC
+energy
+
+// engineer : 2014-03-06 Dog Beach, LLC
+engineer
+
+// engineering : 2014-03-06 Binky Moon, LLC
+engineering
+
+// enterprises : 2013-09-20 Binky Moon, LLC
+enterprises
+
+// epson : 2014-12-04 Seiko Epson Corporation
+epson
+
+// equipment : 2013-08-27 Binky Moon, LLC
+equipment
+
+// ericsson : 2015-07-09 Telefonaktiebolaget L M Ericsson
+ericsson
+
+// erni : 2014-04-03 ERNI Group Holding AG
+erni
+
+// esq : 2014-05-08 Charleston Road Registry Inc.
+esq
+
+// estate : 2013-08-27 Binky Moon, LLC
+estate
+
+// etisalat : 2015-09-03 Emirates Telecommunications Corporation (trading as Etisalat)
+etisalat
+
+// eurovision : 2014-04-24 European Broadcasting Union (EBU)
+eurovision
+
+// eus : 2013-12-12 Puntueus Fundazioa
+eus
+
+// events : 2013-12-05 Binky Moon, LLC
+events
+
+// exchange : 2014-03-06 Binky Moon, LLC
+exchange
+
+// expert : 2013-11-21 Binky Moon, LLC
+expert
+
+// exposed : 2013-12-05 Binky Moon, LLC
+exposed
+
+// express : 2015-02-11 Binky Moon, LLC
+express
+
+// extraspace : 2015-05-14 Extra Space Storage LLC
+extraspace
+
+// fage : 2014-12-18 Fage International S.A.
+fage
+
+// fail : 2014-03-06 Binky Moon, LLC
+fail
+
+// fairwinds : 2014-11-13 FairWinds Partners, LLC
+fairwinds
+
+// faith : 2014-11-20 dot Faith Limited
+faith
+
+// family : 2015-04-02 Dog Beach, LLC
+family
+
+// fan : 2014-03-06 Dog Beach, LLC
+fan
+
+// fans : 2014-11-07 ZDNS International Limited
+fans
+
+// farm : 2013-11-07 Binky Moon, LLC
+farm
+
+// farmers : 2015-07-09 Farmers Insurance Exchange
+farmers
+
+// fashion : 2014-07-03 Registry Services, LLC
+fashion
+
+// fast : 2014-12-18 Amazon Registry Services, Inc.
+fast
+
+// fedex : 2015-08-06 Federal Express Corporation
+fedex
+
+// feedback : 2013-12-19 Top Level Spectrum, Inc.
+feedback
+
+// ferrari : 2015-07-31 Fiat Chrysler Automobiles N.V.
+ferrari
+
+// ferrero : 2014-12-18 Ferrero Trading Lux S.A.
+ferrero
+
+// fiat : 2015-07-31 Fiat Chrysler Automobiles N.V.
+fiat
+
+// fidelity : 2015-07-30 Fidelity Brokerage Services LLC
+fidelity
+
+// fido : 2015-08-06 Rogers Communications Canada Inc.
+fido
+
+// film : 2015-01-08 Motion Picture Domain Registry Pty Ltd
+film
+
+// final : 2014-10-16 Núcleo de Informação e Coordenação do Ponto BR - NIC.br
+final
+
+// finance : 2014-03-20 Binky Moon, LLC
+finance
+
+// financial : 2014-03-06 Binky Moon, LLC
+financial
+
+// fire : 2015-06-25 Amazon Registry Services, Inc.
+fire
+
+// firestone : 2014-12-18 Bridgestone Licensing Services, Inc
+firestone
+
+// firmdale : 2014-03-27 Firmdale Holdings Limited
+firmdale
+
+// fish : 2013-12-12 Binky Moon, LLC
+fish
+
+// fishing : 2013-11-21 Registry Services, LLC
+fishing
+
+// fit : 2014-11-07 Registry Services, LLC
+fit
+
+// fitness : 2014-03-06 Binky Moon, LLC
+fitness
+
+// flickr : 2015-04-02 Flickr, Inc.
+flickr
+
+// flights : 2013-12-05 Binky Moon, LLC
+flights
+
+// flir : 2015-07-23 FLIR Systems, Inc.
+flir
+
+// florist : 2013-11-07 Binky Moon, LLC
+florist
+
+// flowers : 2014-10-09 UNR Corp.
+flowers
+
+// fly : 2014-05-08 Charleston Road Registry Inc.
+fly
+
+// foo : 2014-01-23 Charleston Road Registry Inc.
+foo
+
+// food : 2016-04-21 Lifestyle Domain Holdings, Inc.
+food
+
+// foodnetwork : 2015-07-02 Lifestyle Domain Holdings, Inc.
+foodnetwork
+
+// football : 2014-12-18 Binky Moon, LLC
+football
+
+// ford : 2014-11-13 Ford Motor Company
+ford
+
+// forex : 2014-12-11 Dog Beach, LLC
+forex
+
+// forsale : 2014-05-22 Dog Beach, LLC
+forsale
+
+// forum : 2015-04-02 Fegistry, LLC
+forum
+
+// foundation : 2013-12-05 Binky Moon, LLC
+foundation
+
+// fox : 2015-09-11 FOX Registry, LLC
+fox
+
+// free : 2015-12-10 Amazon Registry Services, Inc.
+free
+
+// fresenius : 2015-07-30 Fresenius Immobilien-Verwaltungs-GmbH
+fresenius
+
+// frl : 2014-05-15 FRLregistry B.V.
+frl
+
+// frogans : 2013-12-19 OP3FT
+frogans
+
+// frontdoor : 2015-07-02 Lifestyle Domain Holdings, Inc.
+frontdoor
+
+// frontier : 2015-02-05 Frontier Communications Corporation
+frontier
+
+// ftr : 2015-07-16 Frontier Communications Corporation
+ftr
+
+// fujitsu : 2015-07-30 Fujitsu Limited
+fujitsu
+
+// fun : 2016-01-14 Radix FZC
+fun
+
+// fund : 2014-03-20 Binky Moon, LLC
+fund
+
+// furniture : 2014-03-20 Binky Moon, LLC
+furniture
+
+// futbol : 2013-09-20 Dog Beach, LLC
+futbol
+
+// fyi : 2015-04-02 Binky Moon, LLC
+fyi
+
+// gal : 2013-11-07 Asociación puntoGAL
+gal
+
+// gallery : 2013-09-13 Binky Moon, LLC
+gallery
+
+// gallo : 2015-06-11 Gallo Vineyards, Inc.
+gallo
+
+// gallup : 2015-02-19 Gallup, Inc.
+gallup
+
+// game : 2015-05-28 UNR Corp.
+game
+
+// games : 2015-05-28 Dog Beach, LLC
+games
+
+// gap : 2015-07-31 The Gap, Inc.
+gap
+
+// garden : 2014-06-26 Registry Services, LLC
+garden
+
+// gay : 2019-05-23 Top Level Design, LLC
+gay
+
+// gbiz : 2014-07-17 Charleston Road Registry Inc.
+gbiz
+
+// gdn : 2014-07-31 Joint Stock Company "Navigation-information systems"
+gdn
+
+// gea : 2014-12-04 GEA Group Aktiengesellschaft
+gea
+
+// gent : 2014-01-23 COMBELL NV
+gent
+
+// genting : 2015-03-12 Resorts World Inc Pte. Ltd.
+genting
+
+// george : 2015-07-31 Wal-Mart Stores, Inc.
+george
+
+// ggee : 2014-01-09 GMO Internet, Inc.
+ggee
+
+// gift : 2013-10-17 DotGift, LLC
+gift
+
+// gifts : 2014-07-03 Binky Moon, LLC
+gifts
+
+// gives : 2014-03-06 Dog Beach, LLC
+gives
+
+// giving : 2014-11-13 Giving Limited
+giving
+
+// glade : 2015-07-23 Johnson Shareholdings, Inc.
+glade
+
+// glass : 2013-11-07 Binky Moon, LLC
+glass
+
+// gle : 2014-07-24 Charleston Road Registry Inc.
+gle
+
+// global : 2014-04-17 Dot Global Domain Registry Limited
+global
+
+// globo : 2013-12-19 Globo Comunicação e Participações S.A
+globo
+
+// gmail : 2014-05-01 Charleston Road Registry Inc.
+gmail
+
+// gmbh : 2016-01-29 Binky Moon, LLC
+gmbh
+
+// gmo : 2014-01-09 GMO Internet, Inc.
+gmo
+
+// gmx : 2014-04-24 1&1 Mail & Media GmbH
+gmx
+
+// godaddy : 2015-07-23 Go Daddy East, LLC
+godaddy
+
+// gold : 2015-01-22 Binky Moon, LLC
+gold
+
+// goldpoint : 2014-11-20 YODOBASHI CAMERA CO.,LTD.
+goldpoint
+
+// golf : 2014-12-18 Binky Moon, LLC
+golf
+
+// goo : 2014-12-18 NTT Resonant Inc.
+goo
+
+// goodyear : 2015-07-02 The Goodyear Tire & Rubber Company
+goodyear
+
+// goog : 2014-11-20 Charleston Road Registry Inc.
+goog
+
+// google : 2014-07-24 Charleston Road Registry Inc.
+google
+
+// gop : 2014-01-16 Republican State Leadership Committee, Inc.
+gop
+
+// got : 2014-12-18 Amazon Registry Services, Inc.
+got
+
+// grainger : 2015-05-07 Grainger Registry Services, LLC
+grainger
+
+// graphics : 2013-09-13 Binky Moon, LLC
+graphics
+
+// gratis : 2014-03-20 Binky Moon, LLC
+gratis
+
+// green : 2014-05-08 Afilias Limited
+green
+
+// gripe : 2014-03-06 Binky Moon, LLC
+gripe
+
+// grocery : 2016-06-16 Wal-Mart Stores, Inc.
+grocery
+
+// group : 2014-08-15 Binky Moon, LLC
+group
+
+// guardian : 2015-07-30 The Guardian Life Insurance Company of America
+guardian
+
+// gucci : 2014-11-13 Guccio Gucci S.p.a.
+gucci
+
+// guge : 2014-08-28 Charleston Road Registry Inc.
+guge
+
+// guide : 2013-09-13 Binky Moon, LLC
+guide
+
+// guitars : 2013-11-14 UNR Corp.
+guitars
+
+// guru : 2013-08-27 Binky Moon, LLC
+guru
+
+// hair : 2015-12-03 XYZ.COM LLC
+hair
+
+// hamburg : 2014-02-20 Hamburg Top-Level-Domain GmbH
+hamburg
+
+// hangout : 2014-11-13 Charleston Road Registry Inc.
+hangout
+
+// haus : 2013-12-05 Dog Beach, LLC
+haus
+
+// hbo : 2015-07-30 HBO Registry Services, Inc.
+hbo
+
+// hdfc : 2015-07-30 HOUSING DEVELOPMENT FINANCE CORPORATION LIMITED
+hdfc
+
+// hdfcbank : 2015-02-12 HDFC Bank Limited
+hdfcbank
+
+// health : 2015-02-11 DotHealth, LLC
+health
+
+// healthcare : 2014-06-12 Binky Moon, LLC
+healthcare
+
+// help : 2014-06-26 UNR Corp.
+help
+
+// helsinki : 2015-02-05 City of Helsinki
+helsinki
+
+// here : 2014-02-06 Charleston Road Registry Inc.
+here
+
+// hermes : 2014-07-10 HERMES INTERNATIONAL
+hermes
+
+// hgtv : 2015-07-02 Lifestyle Domain Holdings, Inc.
+hgtv
+
+// hiphop : 2014-03-06 UNR Corp.
+hiphop
+
+// hisamitsu : 2015-07-16 Hisamitsu Pharmaceutical Co.,Inc.
+hisamitsu
+
+// hitachi : 2014-10-31 Hitachi, Ltd.
+hitachi
+
+// hiv : 2014-03-13 UNR Corp.
+hiv
+
+// hkt : 2015-05-14 PCCW-HKT DataCom Services Limited
+hkt
+
+// hockey : 2015-03-19 Binky Moon, LLC
+hockey
+
+// holdings : 2013-08-27 Binky Moon, LLC
+holdings
+
+// holiday : 2013-11-07 Binky Moon, LLC
+holiday
+
+// homedepot : 2015-04-02 Home Depot Product Authority, LLC
+homedepot
+
+// homegoods : 2015-07-16 The TJX Companies, Inc.
+homegoods
+
+// homes : 2014-01-09 XYZ.COM LLC
+homes
+
+// homesense : 2015-07-16 The TJX Companies, Inc.
+homesense
+
+// honda : 2014-12-18 Honda Motor Co., Ltd.
+honda
+
+// horse : 2013-11-21 Registry Services, LLC
+horse
+
+// hospital : 2016-10-20 Binky Moon, LLC
+hospital
+
+// host : 2014-04-17 Radix FZC
+host
+
+// hosting : 2014-05-29 UNR Corp.
+hosting
+
+// hot : 2015-08-27 Amazon Registry Services, Inc.
+hot
+
+// hoteles : 2015-03-05 Travel Reservations SRL
+hoteles
+
+// hotels : 2016-04-07 Booking.com B.V.
+hotels
+
+// hotmail : 2014-12-18 Microsoft Corporation
+hotmail
+
+// house : 2013-11-07 Binky Moon, LLC
+house
+
+// how : 2014-01-23 Charleston Road Registry Inc.
+how
+
+// hsbc : 2014-10-24 HSBC Global Services (UK) Limited
+hsbc
+
+// hughes : 2015-07-30 Hughes Satellite Systems Corporation
+hughes
+
+// hyatt : 2015-07-30 Hyatt GTLD, L.L.C.
+hyatt
+
+// hyundai : 2015-07-09 Hyundai Motor Company
+hyundai
+
+// ibm : 2014-07-31 International Business Machines Corporation
+ibm
+
+// icbc : 2015-02-19 Industrial and Commercial Bank of China Limited
+icbc
+
+// ice : 2014-10-30 IntercontinentalExchange, Inc.
+ice
+
+// icu : 2015-01-08 ShortDot SA
+icu
+
+// ieee : 2015-07-23 IEEE Global LLC
+ieee
+
+// ifm : 2014-01-30 ifm electronic gmbh
+ifm
+
+// ikano : 2015-07-09 Ikano S.A.
+ikano
+
+// imamat : 2015-08-06 Fondation Aga Khan (Aga Khan Foundation)
+imamat
+
+// imdb : 2015-06-25 Amazon Registry Services, Inc.
+imdb
+
+// immo : 2014-07-10 Binky Moon, LLC
+immo
+
+// immobilien : 2013-11-07 Dog Beach, LLC
+immobilien
+
+// inc : 2018-03-10 Intercap Registry Inc.
+inc
+
+// industries : 2013-12-05 Binky Moon, LLC
+industries
+
+// infiniti : 2014-03-27 NISSAN MOTOR CO., LTD.
+infiniti
+
+// ing : 2014-01-23 Charleston Road Registry Inc.
+ing
+
+// ink : 2013-12-05 Top Level Design, LLC
+ink
+
+// institute : 2013-11-07 Binky Moon, LLC
+institute
+
+// insurance : 2015-02-19 fTLD Registry Services LLC
+insurance
+
+// insure : 2014-03-20 Binky Moon, LLC
+insure
+
+// international : 2013-11-07 Binky Moon, LLC
+international
+
+// intuit : 2015-07-30 Intuit Administrative Services, Inc.
+intuit
+
+// investments : 2014-03-20 Binky Moon, LLC
+investments
+
+// ipiranga : 2014-08-28 Ipiranga Produtos de Petroleo S.A.
+ipiranga
+
+// irish : 2014-08-07 Binky Moon, LLC
+irish
+
+// ismaili : 2015-08-06 Fondation Aga Khan (Aga Khan Foundation)
+ismaili
+
+// ist : 2014-08-28 Istanbul Metropolitan Municipality
+ist
+
+// istanbul : 2014-08-28 Istanbul Metropolitan Municipality
+istanbul
+
+// itau : 2014-10-02 Itau Unibanco Holding S.A.
+itau
+
+// itv : 2015-07-09 ITV Services Limited
+itv
+
+// jaguar : 2014-11-13 Jaguar Land Rover Ltd
+jaguar
+
+// java : 2014-06-19 Oracle Corporation
+java
+
+// jcb : 2014-11-20 JCB Co., Ltd.
+jcb
+
+// jeep : 2015-07-30 FCA US LLC.
+jeep
+
+// jetzt : 2014-01-09 Binky Moon, LLC
+jetzt
+
+// jewelry : 2015-03-05 Binky Moon, LLC
+jewelry
+
+// jio : 2015-04-02 Reliance Industries Limited
+jio
+
+// jll : 2015-04-02 Jones Lang LaSalle Incorporated
+jll
+
+// jmp : 2015-03-26 Matrix IP LLC
+jmp
+
+// jnj : 2015-06-18 Johnson & Johnson Services, Inc.
+jnj
+
+// joburg : 2014-03-24 ZA Central Registry NPC trading as ZA Central Registry
+joburg
+
+// jot : 2014-12-18 Amazon Registry Services, Inc.
+jot
+
+// joy : 2014-12-18 Amazon Registry Services, Inc.
+joy
+
+// jpmorgan : 2015-04-30 JPMorgan Chase Bank, National Association
+jpmorgan
+
+// jprs : 2014-09-18 Japan Registry Services Co., Ltd.
+jprs
+
+// juegos : 2014-03-20 UNR Corp.
+juegos
+
+// juniper : 2015-07-30 JUNIPER NETWORKS, INC.
+juniper
+
+// kaufen : 2013-11-07 Dog Beach, LLC
+kaufen
+
+// kddi : 2014-09-12 KDDI CORPORATION
+kddi
+
+// kerryhotels : 2015-04-30 Kerry Trading Co. Limited
+kerryhotels
+
+// kerrylogistics : 2015-04-09 Kerry Trading Co. Limited
+kerrylogistics
+
+// kerryproperties : 2015-04-09 Kerry Trading Co. Limited
+kerryproperties
+
+// kfh : 2014-12-04 Kuwait Finance House
+kfh
+
+// kia : 2015-07-09 KIA MOTORS CORPORATION
+kia
+
+// kim : 2013-09-23 Afilias Limited
+kim
+
+// kinder : 2014-11-07 Ferrero Trading Lux S.A.
+kinder
+
+// kindle : 2015-06-25 Amazon Registry Services, Inc.
+kindle
+
+// kitchen : 2013-09-20 Binky Moon, LLC
+kitchen
+
+// kiwi : 2013-09-20 DOT KIWI LIMITED
+kiwi
+
+// koeln : 2014-01-09 dotKoeln GmbH
+koeln
+
+// komatsu : 2015-01-08 Komatsu Ltd.
+komatsu
+
+// kosher : 2015-08-20 Kosher Marketing Assets LLC
+kosher
+
+// kpmg : 2015-04-23 KPMG International Cooperative (KPMG International Genossenschaft)
+kpmg
+
+// kpn : 2015-01-08 Koninklijke KPN N.V.
+kpn
+
+// krd : 2013-12-05 KRG Department of Information Technology
+krd
+
+// kred : 2013-12-19 KredTLD Pty Ltd
+kred
+
+// kuokgroup : 2015-04-09 Kerry Trading Co. Limited
+kuokgroup
+
+// kyoto : 2014-11-07 Academic Institution: Kyoto Jyoho Gakuen
+kyoto
+
+// lacaixa : 2014-01-09 Fundación Bancaria Caixa d’Estalvis i Pensions de Barcelona, “la Caixa”
+lacaixa
+
+// lamborghini : 2015-06-04 Automobili Lamborghini S.p.A.
+lamborghini
+
+// lamer : 2015-10-01 The Estée Lauder Companies Inc.
+lamer
+
+// lancaster : 2015-02-12 LANCASTER
+lancaster
+
+// lancia : 2015-07-31 Fiat Chrysler Automobiles N.V.
+lancia
+
+// land : 2013-09-10 Binky Moon, LLC
+land
+
+// landrover : 2014-11-13 Jaguar Land Rover Ltd
+landrover
+
+// lanxess : 2015-07-30 LANXESS Corporation
+lanxess
+
+// lasalle : 2015-04-02 Jones Lang LaSalle Incorporated
+lasalle
+
+// lat : 2014-10-16 ECOM-LAC Federaciòn de Latinoamèrica y el Caribe para Internet y el Comercio Electrònico
+lat
+
+// latino : 2015-07-30 Dish DBS Corporation
+latino
+
+// latrobe : 2014-06-16 La Trobe University
+latrobe
+
+// law : 2015-01-22 Registry Services, LLC
+law
+
+// lawyer : 2014-03-20 Dog Beach, LLC
+lawyer
+
+// lds : 2014-03-20 IRI Domain Management, LLC
+lds
+
+// lease : 2014-03-06 Binky Moon, LLC
+lease
+
+// leclerc : 2014-08-07 A.C.D. LEC Association des Centres Distributeurs Edouard Leclerc
+leclerc
+
+// lefrak : 2015-07-16 LeFrak Organization, Inc.
+lefrak
+
+// legal : 2014-10-16 Binky Moon, LLC
+legal
+
+// lego : 2015-07-16 LEGO Juris A/S
+lego
+
+// lexus : 2015-04-23 TOYOTA MOTOR CORPORATION
+lexus
+
+// lgbt : 2014-05-08 Afilias Limited
+lgbt
+
+// lidl : 2014-09-18 Schwarz Domains und Services GmbH & Co. KG
+lidl
+
+// life : 2014-02-06 Binky Moon, LLC
+life
+
+// lifeinsurance : 2015-01-15 American Council of Life Insurers
+lifeinsurance
+
+// lifestyle : 2014-12-11 Lifestyle Domain Holdings, Inc.
+lifestyle
+
+// lighting : 2013-08-27 Binky Moon, LLC
+lighting
+
+// like : 2014-12-18 Amazon Registry Services, Inc.
+like
+
+// lilly : 2015-07-31 Eli Lilly and Company
+lilly
+
+// limited : 2014-03-06 Binky Moon, LLC
+limited
+
+// limo : 2013-10-17 Binky Moon, LLC
+limo
+
+// lincoln : 2014-11-13 Ford Motor Company
+lincoln
+
+// linde : 2014-12-04 Linde Aktiengesellschaft
+linde
+
+// link : 2013-11-14 UNR Corp.
+link
+
+// lipsy : 2015-06-25 Lipsy Ltd
+lipsy
+
+// live : 2014-12-04 Dog Beach, LLC
+live
+
+// living : 2015-07-30 Lifestyle Domain Holdings, Inc.
+living
+
+// lixil : 2015-03-19 LIXIL Group Corporation
+lixil
+
+// llc : 2017-12-14 Afilias Limited
+llc
+
+// llp : 2019-08-26 UNR Corp.
+llp
+
+// loan : 2014-11-20 dot Loan Limited
+loan
+
+// loans : 2014-03-20 Binky Moon, LLC
+loans
+
+// locker : 2015-06-04 Dish DBS Corporation
+locker
+
+// locus : 2015-06-25 Locus Analytics LLC
+locus
+
+// loft : 2015-07-30 Annco, Inc.
+loft
+
+// lol : 2015-01-30 UNR Corp.
+lol
+
+// london : 2013-11-14 Dot London Domains Limited
+london
+
+// lotte : 2014-11-07 Lotte Holdings Co., Ltd.
+lotte
+
+// lotto : 2014-04-10 Afilias Limited
+lotto
+
+// love : 2014-12-22 Merchant Law Group LLP
+love
+
+// lpl : 2015-07-30 LPL Holdings, Inc.
+lpl
+
+// lplfinancial : 2015-07-30 LPL Holdings, Inc.
+lplfinancial
+
+// ltd : 2014-09-25 Binky Moon, LLC
+ltd
+
+// ltda : 2014-04-17 InterNetX, Corp
+ltda
+
+// lundbeck : 2015-08-06 H. Lundbeck A/S
+lundbeck
+
+// luxe : 2014-01-09 Registry Services, LLC
+luxe
+
+// luxury : 2013-10-17 Luxury Partners, LLC
+luxury
+
+// macys : 2015-07-31 Macys, Inc.
+macys
+
+// madrid : 2014-05-01 Comunidad de Madrid
+madrid
+
+// maif : 2014-10-02 Mutuelle Assurance Instituteur France (MAIF)
+maif
+
+// maison : 2013-12-05 Binky Moon, LLC
+maison
+
+// makeup : 2015-01-15 XYZ.COM LLC
+makeup
+
+// man : 2014-12-04 MAN SE
+man
+
+// management : 2013-11-07 Binky Moon, LLC
+management
+
+// mango : 2013-10-24 PUNTO FA S.L.
+mango
+
+// map : 2016-06-09 Charleston Road Registry Inc.
+map
+
+// market : 2014-03-06 Dog Beach, LLC
+market
+
+// marketing : 2013-11-07 Binky Moon, LLC
+marketing
+
+// markets : 2014-12-11 Dog Beach, LLC
+markets
+
+// marriott : 2014-10-09 Marriott Worldwide Corporation
+marriott
+
+// marshalls : 2015-07-16 The TJX Companies, Inc.
+marshalls
+
+// maserati : 2015-07-31 Fiat Chrysler Automobiles N.V.
+maserati
+
+// mattel : 2015-08-06 Mattel Sites, Inc.
+mattel
+
+// mba : 2015-04-02 Binky Moon, LLC
+mba
+
+// mckinsey : 2015-07-31 McKinsey Holdings, Inc.
+mckinsey
+
+// med : 2015-08-06 Medistry LLC
+med
+
+// media : 2014-03-06 Binky Moon, LLC
+media
+
+// meet : 2014-01-16 Charleston Road Registry Inc.
+meet
+
+// melbourne : 2014-05-29 The Crown in right of the State of Victoria, represented by its Department of State Development, Business and Innovation
+melbourne
+
+// meme : 2014-01-30 Charleston Road Registry Inc.
+meme
+
+// memorial : 2014-10-16 Dog Beach, LLC
+memorial
+
+// men : 2015-02-26 Exclusive Registry Limited
+men
+
+// menu : 2013-09-11 Dot Menu Registry, LLC
+menu
+
+// merckmsd : 2016-07-14 MSD Registry Holdings, Inc.
+merckmsd
+
+// miami : 2013-12-19 Minds + Machines Group Limited
+miami
+
+// microsoft : 2014-12-18 Microsoft Corporation
+microsoft
+
+// mini : 2014-01-09 Bayerische Motoren Werke Aktiengesellschaft
+mini
+
+// mint : 2015-07-30 Intuit Administrative Services, Inc.
+mint
+
+// mit : 2015-07-02 Massachusetts Institute of Technology
+mit
+
+// mitsubishi : 2015-07-23 Mitsubishi Corporation
+mitsubishi
+
+// mlb : 2015-05-21 MLB Advanced Media DH, LLC
+mlb
+
+// mls : 2015-04-23 The Canadian Real Estate Association
+mls
+
+// mma : 2014-11-07 MMA IARD
+mma
+
+// mobile : 2016-06-02 Dish DBS Corporation
+mobile
+
+// moda : 2013-11-07 Dog Beach, LLC
+moda
+
+// moe : 2013-11-13 Interlink Co., Ltd.
+moe
+
+// moi : 2014-12-18 Amazon Registry Services, Inc.
+moi
+
+// mom : 2015-04-16 UNR Corp.
+mom
+
+// monash : 2013-09-30 Monash University
+monash
+
+// money : 2014-10-16 Binky Moon, LLC
+money
+
+// monster : 2015-09-11 XYZ.COM LLC
+monster
+
+// mormon : 2013-12-05 IRI Domain Management, LLC
+mormon
+
+// mortgage : 2014-03-20 Dog Beach, LLC
+mortgage
+
+// moscow : 2013-12-19 Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID)
+moscow
+
+// moto : 2015-06-04 Motorola Trademark Holdings, LLC
+moto
+
+// motorcycles : 2014-01-09 XYZ.COM LLC
+motorcycles
+
+// mov : 2014-01-30 Charleston Road Registry Inc.
+mov
+
+// movie : 2015-02-05 Binky Moon, LLC
+movie
+
+// msd : 2015-07-23 MSD Registry Holdings, Inc.
+msd
+
+// mtn : 2014-12-04 MTN Dubai Limited
+mtn
+
+// mtr : 2015-03-12 MTR Corporation Limited
+mtr
+
+// music : 2021-05-04 DotMusic Limited
+music
+
+// mutual : 2015-04-02 Northwestern Mutual MU TLD Registry, LLC
+mutual
+
+// nab : 2015-08-20 National Australia Bank Limited
+nab
+
+// nagoya : 2013-10-24 GMO Registry, Inc.
+nagoya
+
+// natura : 2015-03-12 NATURA COSMÉTICOS S.A.
+natura
+
+// navy : 2014-03-06 Dog Beach, LLC
+navy
+
+// nba : 2015-07-31 NBA REGISTRY, LLC
+nba
+
+// nec : 2015-01-08 NEC Corporation
+nec
+
+// netbank : 2014-06-26 COMMONWEALTH BANK OF AUSTRALIA
+netbank
+
+// netflix : 2015-06-18 Netflix, Inc.
+netflix
+
+// network : 2013-11-14 Binky Moon, LLC
+network
+
+// neustar : 2013-12-05 NeuStar, Inc.
+neustar
+
+// new : 2014-01-30 Charleston Road Registry Inc.
+new
+
+// news : 2014-12-18 Dog Beach, LLC
+news
+
+// next : 2015-06-18 Next plc
+next
+
+// nextdirect : 2015-06-18 Next plc
+nextdirect
+
+// nexus : 2014-07-24 Charleston Road Registry Inc.
+nexus
+
+// nfl : 2015-07-23 NFL Reg Ops LLC
+nfl
+
+// ngo : 2014-03-06 Public Interest Registry
+ngo
+
+// nhk : 2014-02-13 Japan Broadcasting Corporation (NHK)
+nhk
+
+// nico : 2014-12-04 DWANGO Co., Ltd.
+nico
+
+// nike : 2015-07-23 NIKE, Inc.
+nike
+
+// nikon : 2015-05-21 NIKON CORPORATION
+nikon
+
+// ninja : 2013-11-07 Dog Beach, LLC
+ninja
+
+// nissan : 2014-03-27 NISSAN MOTOR CO., LTD.
+nissan
+
+// nissay : 2015-10-29 Nippon Life Insurance Company
+nissay
+
+// nokia : 2015-01-08 Nokia Corporation
+nokia
+
+// northwesternmutual : 2015-06-18 Northwestern Mutual Registry, LLC
+northwesternmutual
+
+// norton : 2014-12-04 NortonLifeLock Inc.
+norton
+
+// now : 2015-06-25 Amazon Registry Services, Inc.
+now
+
+// nowruz : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+nowruz
+
+// nowtv : 2015-05-14 Starbucks (HK) Limited
+nowtv
+
+// nra : 2014-05-22 NRA Holdings Company, INC.
+nra
+
+// nrw : 2013-11-21 Minds + Machines GmbH
+nrw
+
+// ntt : 2014-10-31 NIPPON TELEGRAPH AND TELEPHONE CORPORATION
+ntt
+
+// nyc : 2014-01-23 The City of New York by and through the New York City Department of Information Technology & Telecommunications
+nyc
+
+// obi : 2014-09-25 OBI Group Holding SE & Co. KGaA
+obi
+
+// observer : 2015-04-30 Dog Beach, LLC
+observer
+
+// off : 2015-07-23 Johnson Shareholdings, Inc.
+off
+
+// office : 2015-03-12 Microsoft Corporation
+office
+
+// okinawa : 2013-12-05 BRregistry, Inc.
+okinawa
+
+// olayan : 2015-05-14 Crescent Holding GmbH
+olayan
+
+// olayangroup : 2015-05-14 Crescent Holding GmbH
+olayangroup
+
+// oldnavy : 2015-07-31 The Gap, Inc.
+oldnavy
+
+// ollo : 2015-06-04 Dish DBS Corporation
+ollo
+
+// omega : 2015-01-08 The Swatch Group Ltd
+omega
+
+// one : 2014-11-07 One.com A/S
+one
+
+// ong : 2014-03-06 Public Interest Registry
+ong
+
+// onl : 2013-09-16 iRegistry GmbH
+onl
+
+// online : 2015-01-15 Radix FZC
+online
+
+// ooo : 2014-01-09 INFIBEAM AVENUES LIMITED
+ooo
+
+// open : 2015-07-31 American Express Travel Related Services Company, Inc.
+open
+
+// oracle : 2014-06-19 Oracle Corporation
+oracle
+
+// orange : 2015-03-12 Orange Brand Services Limited
+orange
+
+// organic : 2014-03-27 Afilias Limited
+organic
+
+// origins : 2015-10-01 The Estée Lauder Companies Inc.
+origins
+
+// osaka : 2014-09-04 Osaka Registry Co., Ltd.
+osaka
+
+// otsuka : 2013-10-11 Otsuka Holdings Co., Ltd.
+otsuka
+
+// ott : 2015-06-04 Dish DBS Corporation
+ott
+
+// ovh : 2014-01-16 MédiaBC
+ovh
+
+// page : 2014-12-04 Charleston Road Registry Inc.
+page
+
+// panasonic : 2015-07-30 Panasonic Corporation
+panasonic
+
+// paris : 2014-01-30 City of Paris
+paris
+
+// pars : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+pars
+
+// partners : 2013-12-05 Binky Moon, LLC
+partners
+
+// parts : 2013-12-05 Binky Moon, LLC
+parts
+
+// party : 2014-09-11 Blue Sky Registry Limited
+party
+
+// passagens : 2015-03-05 Travel Reservations SRL
+passagens
+
+// pay : 2015-08-27 Amazon Registry Services, Inc.
+pay
+
+// pccw : 2015-05-14 PCCW Enterprises Limited
+pccw
+
+// pet : 2015-05-07 Afilias Limited
+pet
+
+// pfizer : 2015-09-11 Pfizer Inc.
+pfizer
+
+// pharmacy : 2014-06-19 National Association of Boards of Pharmacy
+pharmacy
+
+// phd : 2016-07-28 Charleston Road Registry Inc.
+phd
+
+// philips : 2014-11-07 Koninklijke Philips N.V.
+philips
+
+// phone : 2016-06-02 Dish DBS Corporation
+phone
+
+// photo : 2013-11-14 UNR Corp.
+photo
+
+// photography : 2013-09-20 Binky Moon, LLC
+photography
+
+// photos : 2013-10-17 Binky Moon, LLC
+photos
+
+// physio : 2014-05-01 PhysBiz Pty Ltd
+physio
+
+// pics : 2013-11-14 UNR Corp.
+pics
+
+// pictet : 2014-06-26 Pictet Europe S.A.
+pictet
+
+// pictures : 2014-03-06 Binky Moon, LLC
+pictures
+
+// pid : 2015-01-08 Top Level Spectrum, Inc.
+pid
+
+// pin : 2014-12-18 Amazon Registry Services, Inc.
+pin
+
+// ping : 2015-06-11 Ping Registry Provider, Inc.
+ping
+
+// pink : 2013-10-01 Afilias Limited
+pink
+
+// pioneer : 2015-07-16 Pioneer Corporation
+pioneer
+
+// pizza : 2014-06-26 Binky Moon, LLC
+pizza
+
+// place : 2014-04-24 Binky Moon, LLC
+place
+
+// play : 2015-03-05 Charleston Road Registry Inc.
+play
+
+// playstation : 2015-07-02 Sony Interactive Entertainment Inc.
+playstation
+
+// plumbing : 2013-09-10 Binky Moon, LLC
+plumbing
+
+// plus : 2015-02-05 Binky Moon, LLC
+plus
+
+// pnc : 2015-07-02 PNC Domain Co., LLC
+pnc
+
+// pohl : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+pohl
+
+// poker : 2014-07-03 Afilias Limited
+poker
+
+// politie : 2015-08-20 Politie Nederland
+politie
+
+// porn : 2014-10-16 ICM Registry PN LLC
+porn
+
+// pramerica : 2015-07-30 Prudential Financial, Inc.
+pramerica
+
+// praxi : 2013-12-05 Praxi S.p.A.
+praxi
+
+// press : 2014-04-03 Radix FZC
+press
+
+// prime : 2015-06-25 Amazon Registry Services, Inc.
+prime
+
+// prod : 2014-01-23 Charleston Road Registry Inc.
+prod
+
+// productions : 2013-12-05 Binky Moon, LLC
+productions
+
+// prof : 2014-07-24 Charleston Road Registry Inc.
+prof
+
+// progressive : 2015-07-23 Progressive Casualty Insurance Company
+progressive
+
+// promo : 2014-12-18 Afilias Limited
+promo
+
+// properties : 2013-12-05 Binky Moon, LLC
+properties
+
+// property : 2014-05-22 UNR Corp.
+property
+
+// protection : 2015-04-23 XYZ.COM LLC
+protection
+
+// pru : 2015-07-30 Prudential Financial, Inc.
+pru
+
+// prudential : 2015-07-30 Prudential Financial, Inc.
+prudential
+
+// pub : 2013-12-12 Dog Beach, LLC
+pub
+
+// pwc : 2015-10-29 PricewaterhouseCoopers LLP
+pwc
+
+// qpon : 2013-11-14 dotCOOL, Inc.
+qpon
+
+// quebec : 2013-12-19 PointQuébec Inc
+quebec
+
+// quest : 2015-03-26 XYZ.COM LLC
+quest
+
+// qvc : 2015-07-30 QVC, Inc.
+qvc
+
+// racing : 2014-12-04 Premier Registry Limited
+racing
+
+// radio : 2016-07-21 European Broadcasting Union (EBU)
+radio
+
+// raid : 2015-07-23 Johnson Shareholdings, Inc.
+raid
+
+// read : 2014-12-18 Amazon Registry Services, Inc.
+read
+
+// realestate : 2015-09-11 dotRealEstate LLC
+realestate
+
+// realtor : 2014-05-29 Real Estate Domains LLC
+realtor
+
+// realty : 2015-03-19 Dog Beach, LLC
+realty
+
+// recipes : 2013-10-17 Binky Moon, LLC
+recipes
+
+// red : 2013-11-07 Afilias Limited
+red
+
+// redstone : 2014-10-31 Redstone Haute Couture Co., Ltd.
+redstone
+
+// redumbrella : 2015-03-26 Travelers TLD, LLC
+redumbrella
+
+// rehab : 2014-03-06 Dog Beach, LLC
+rehab
+
+// reise : 2014-03-13 Binky Moon, LLC
+reise
+
+// reisen : 2014-03-06 Binky Moon, LLC
+reisen
+
+// reit : 2014-09-04 National Association of Real Estate Investment Trusts, Inc.
+reit
+
+// reliance : 2015-04-02 Reliance Industries Limited
+reliance
+
+// ren : 2013-12-12 ZDNS International Limited
+ren
+
+// rent : 2014-12-04 XYZ.COM LLC
+rent
+
+// rentals : 2013-12-05 Binky Moon, LLC
+rentals
+
+// repair : 2013-11-07 Binky Moon, LLC
+repair
+
+// report : 2013-12-05 Binky Moon, LLC
+report
+
+// republican : 2014-03-20 Dog Beach, LLC
+republican
+
+// rest : 2013-12-19 Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable
+rest
+
+// restaurant : 2014-07-03 Binky Moon, LLC
+restaurant
+
+// review : 2014-11-20 dot Review Limited
+review
+
+// reviews : 2013-09-13 Dog Beach, LLC
+reviews
+
+// rexroth : 2015-06-18 Robert Bosch GMBH
+rexroth
+
+// rich : 2013-11-21 iRegistry GmbH
+rich
+
+// richardli : 2015-05-14 Pacific Century Asset Management (HK) Limited
+richardli
+
+// ricoh : 2014-11-20 Ricoh Company, Ltd.
+ricoh
+
+// ril : 2015-04-02 Reliance Industries Limited
+ril
+
+// rio : 2014-02-27 Empresa Municipal de Informática SA - IPLANRIO
+rio
+
+// rip : 2014-07-10 Dog Beach, LLC
+rip
+
+// rmit : 2015-11-19 Royal Melbourne Institute of Technology
+rmit
+
+// rocher : 2014-12-18 Ferrero Trading Lux S.A.
+rocher
+
+// rocks : 2013-11-14 Dog Beach, LLC
+rocks
+
+// rodeo : 2013-12-19 Registry Services, LLC
+rodeo
+
+// rogers : 2015-08-06 Rogers Communications Canada Inc.
+rogers
+
+// room : 2014-12-18 Amazon Registry Services, Inc.
+room
+
+// rsvp : 2014-05-08 Charleston Road Registry Inc.
+rsvp
+
+// rugby : 2016-12-15 World Rugby Strategic Developments Limited
+rugby
+
+// ruhr : 2013-10-02 regiodot GmbH & Co. KG
+ruhr
+
+// run : 2015-03-19 Binky Moon, LLC
+run
+
+// rwe : 2015-04-02 RWE AG
+rwe
+
+// ryukyu : 2014-01-09 BRregistry, Inc.
+ryukyu
+
+// saarland : 2013-12-12 dotSaarland GmbH
+saarland
+
+// safe : 2014-12-18 Amazon Registry Services, Inc.
+safe
+
+// safety : 2015-01-08 Safety Registry Services, LLC.
+safety
+
+// sakura : 2014-12-18 SAKURA Internet Inc.
+sakura
+
+// sale : 2014-10-16 Dog Beach, LLC
+sale
+
+// salon : 2014-12-11 Binky Moon, LLC
+salon
+
+// samsclub : 2015-07-31 Wal-Mart Stores, Inc.
+samsclub
+
+// samsung : 2014-04-03 SAMSUNG SDS CO., LTD
+samsung
+
+// sandvik : 2014-11-13 Sandvik AB
+sandvik
+
+// sandvikcoromant : 2014-11-07 Sandvik AB
+sandvikcoromant
+
+// sanofi : 2014-10-09 Sanofi
+sanofi
+
+// sap : 2014-03-27 SAP AG
+sap
+
+// sarl : 2014-07-03 Binky Moon, LLC
+sarl
+
+// sas : 2015-04-02 Research IP LLC
+sas
+
+// save : 2015-06-25 Amazon Registry Services, Inc.
+save
+
+// saxo : 2014-10-31 Saxo Bank A/S
+saxo
+
+// sbi : 2015-03-12 STATE BANK OF INDIA
+sbi
+
+// sbs : 2014-11-07 ShortDot SA
+sbs
+
+// sca : 2014-03-13 SVENSKA CELLULOSA AKTIEBOLAGET SCA (publ)
+sca
+
+// scb : 2014-02-20 The Siam Commercial Bank Public Company Limited ("SCB")
+scb
+
+// schaeffler : 2015-08-06 Schaeffler Technologies AG & Co. KG
+schaeffler
+
+// schmidt : 2014-04-03 SCHMIDT GROUPE S.A.S.
+schmidt
+
+// scholarships : 2014-04-24 Scholarships.com, LLC
+scholarships
+
+// school : 2014-12-18 Binky Moon, LLC
+school
+
+// schule : 2014-03-06 Binky Moon, LLC
+schule
+
+// schwarz : 2014-09-18 Schwarz Domains und Services GmbH & Co. KG
+schwarz
+
+// science : 2014-09-11 dot Science Limited
+science
+
+// scjohnson : 2015-07-23 Johnson Shareholdings, Inc.
+scjohnson
+
+// scot : 2014-01-23 Dot Scot Registry Limited
+scot
+
+// search : 2016-06-09 Charleston Road Registry Inc.
+search
+
+// seat : 2014-05-22 SEAT, S.A. (Sociedad Unipersonal)
+seat
+
+// secure : 2015-08-27 Amazon Registry Services, Inc.
+secure
+
+// security : 2015-05-14 XYZ.COM LLC
+security
+
+// seek : 2014-12-04 Seek Limited
+seek
+
+// select : 2015-10-08 Registry Services, LLC
+select
+
+// sener : 2014-10-24 Sener Ingeniería y Sistemas, S.A.
+sener
+
+// services : 2014-02-27 Binky Moon, LLC
+services
+
+// ses : 2015-07-23 SES
+ses
+
+// seven : 2015-08-06 Seven West Media Ltd
+seven
+
+// sew : 2014-07-17 SEW-EURODRIVE GmbH & Co KG
+sew
+
+// sex : 2014-11-13 ICM Registry SX LLC
+sex
+
+// sexy : 2013-09-11 UNR Corp.
+sexy
+
+// sfr : 2015-08-13 Societe Francaise du Radiotelephone - SFR
+sfr
+
+// shangrila : 2015-09-03 Shangri‐La International Hotel Management Limited
+shangrila
+
+// sharp : 2014-05-01 Sharp Corporation
+sharp
+
+// shaw : 2015-04-23 Shaw Cablesystems G.P.
+shaw
+
+// shell : 2015-07-30 Shell Information Technology International Inc
+shell
+
+// shia : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+shia
+
+// shiksha : 2013-11-14 Afilias Limited
+shiksha
+
+// shoes : 2013-10-02 Binky Moon, LLC
+shoes
+
+// shop : 2016-04-08 GMO Registry, Inc.
+shop
+
+// shopping : 2016-03-31 Binky Moon, LLC
+shopping
+
+// shouji : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+shouji
+
+// show : 2015-03-05 Binky Moon, LLC
+show
+
+// showtime : 2015-08-06 CBS Domains Inc.
+showtime
+
+// silk : 2015-06-25 Amazon Registry Services, Inc.
+silk
+
+// sina : 2015-03-12 Sina Corporation
+sina
+
+// singles : 2013-08-27 Binky Moon, LLC
+singles
+
+// site : 2015-01-15 Radix FZC
+site
+
+// ski : 2015-04-09 Afilias Limited
+ski
+
+// skin : 2015-01-15 XYZ.COM LLC
+skin
+
+// sky : 2014-06-19 Sky International AG
+sky
+
+// skype : 2014-12-18 Microsoft Corporation
+skype
+
+// sling : 2015-07-30 DISH Technologies L.L.C.
+sling
+
+// smart : 2015-07-09 Smart Communications, Inc. (SMART)
+smart
+
+// smile : 2014-12-18 Amazon Registry Services, Inc.
+smile
+
+// sncf : 2015-02-19 Société Nationale des Chemins de fer Francais S N C F
+sncf
+
+// soccer : 2015-03-26 Binky Moon, LLC
+soccer
+
+// social : 2013-11-07 Dog Beach, LLC
+social
+
+// softbank : 2015-07-02 SoftBank Group Corp.
+softbank
+
+// software : 2014-03-20 Dog Beach, LLC
+software
+
+// sohu : 2013-12-19 Sohu.com Limited
+sohu
+
+// solar : 2013-11-07 Binky Moon, LLC
+solar
+
+// solutions : 2013-11-07 Binky Moon, LLC
+solutions
+
+// song : 2015-02-26 Amazon Registry Services, Inc.
+song
+
+// sony : 2015-01-08 Sony Corporation
+sony
+
+// soy : 2014-01-23 Charleston Road Registry Inc.
+soy
+
+// spa : 2019-09-19 Asia Spa and Wellness Promotion Council Limited
+spa
+
+// space : 2014-04-03 Radix FZC
+space
+
+// sport : 2017-11-16 Global Association of International Sports Federations (GAISF)
+sport
+
+// spot : 2015-02-26 Amazon Registry Services, Inc.
+spot
+
+// srl : 2015-05-07 InterNetX, Corp
+srl
+
+// stada : 2014-11-13 STADA Arzneimittel AG
+stada
+
+// staples : 2015-07-30 Staples, Inc.
+staples
+
+// star : 2015-01-08 Star India Private Limited
+star
+
+// statebank : 2015-03-12 STATE BANK OF INDIA
+statebank
+
+// statefarm : 2015-07-30 State Farm Mutual Automobile Insurance Company
+statefarm
+
+// stc : 2014-10-09 Saudi Telecom Company
+stc
+
+// stcgroup : 2014-10-09 Saudi Telecom Company
+stcgroup
+
+// stockholm : 2014-12-18 Stockholms kommun
+stockholm
+
+// storage : 2014-12-22 XYZ.COM LLC
+storage
+
+// store : 2015-04-09 Radix FZC
+store
+
+// stream : 2016-01-08 dot Stream Limited
+stream
+
+// studio : 2015-02-11 Dog Beach, LLC
+studio
+
+// study : 2014-12-11 OPEN UNIVERSITIES AUSTRALIA PTY LTD
+study
+
+// style : 2014-12-04 Binky Moon, LLC
+style
+
+// sucks : 2014-12-22 Vox Populi Registry Ltd.
+sucks
+
+// supplies : 2013-12-19 Binky Moon, LLC
+supplies
+
+// supply : 2013-12-19 Binky Moon, LLC
+supply
+
+// support : 2013-10-24 Binky Moon, LLC
+support
+
+// surf : 2014-01-09 Registry Services, LLC
+surf
+
+// surgery : 2014-03-20 Binky Moon, LLC
+surgery
+
+// suzuki : 2014-02-20 SUZUKI MOTOR CORPORATION
+suzuki
+
+// swatch : 2015-01-08 The Swatch Group Ltd
+swatch
+
+// swiftcover : 2015-07-23 Swiftcover Insurance Services Limited
+swiftcover
+
+// swiss : 2014-10-16 Swiss Confederation
+swiss
+
+// sydney : 2014-09-18 State of New South Wales, Department of Premier and Cabinet
+sydney
+
+// systems : 2013-11-07 Binky Moon, LLC
+systems
+
+// tab : 2014-12-04 Tabcorp Holdings Limited
+tab
+
+// taipei : 2014-07-10 Taipei City Government
+taipei
+
+// talk : 2015-04-09 Amazon Registry Services, Inc.
+talk
+
+// taobao : 2015-01-15 Alibaba Group Holding Limited
+taobao
+
+// target : 2015-07-31 Target Domain Holdings, LLC
+target
+
+// tatamotors : 2015-03-12 Tata Motors Ltd
+tatamotors
+
+// tatar : 2014-04-24 Limited Liability Company "Coordination Center of Regional Domain of Tatarstan Republic"
+tatar
+
+// tattoo : 2013-08-30 UNR Corp.
+tattoo
+
+// tax : 2014-03-20 Binky Moon, LLC
+tax
+
+// taxi : 2015-03-19 Binky Moon, LLC
+taxi
+
+// tci : 2014-09-12 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+tci
+
+// tdk : 2015-06-11 TDK Corporation
+tdk
+
+// team : 2015-03-05 Binky Moon, LLC
+team
+
+// tech : 2015-01-30 Radix FZC
+tech
+
+// technology : 2013-09-13 Binky Moon, LLC
+technology
+
+// temasek : 2014-08-07 Temasek Holdings (Private) Limited
+temasek
+
+// tennis : 2014-12-04 Binky Moon, LLC
+tennis
+
+// teva : 2015-07-02 Teva Pharmaceutical Industries Limited
+teva
+
+// thd : 2015-04-02 Home Depot Product Authority, LLC
+thd
+
+// theater : 2015-03-19 Binky Moon, LLC
+theater
+
+// theatre : 2015-05-07 XYZ.COM LLC
+theatre
+
+// tiaa : 2015-07-23 Teachers Insurance and Annuity Association of America
+tiaa
+
+// tickets : 2015-02-05 XYZ.COM LLC
+tickets
+
+// tienda : 2013-11-14 Binky Moon, LLC
+tienda
+
+// tiffany : 2015-01-30 Tiffany and Company
+tiffany
+
+// tips : 2013-09-20 Binky Moon, LLC
+tips
+
+// tires : 2014-11-07 Binky Moon, LLC
+tires
+
+// tirol : 2014-04-24 punkt Tirol GmbH
+tirol
+
+// tjmaxx : 2015-07-16 The TJX Companies, Inc.
+tjmaxx
+
+// tjx : 2015-07-16 The TJX Companies, Inc.
+tjx
+
+// tkmaxx : 2015-07-16 The TJX Companies, Inc.
+tkmaxx
+
+// tmall : 2015-01-15 Alibaba Group Holding Limited
+tmall
+
+// today : 2013-09-20 Binky Moon, LLC
+today
+
+// tokyo : 2013-11-13 GMO Registry, Inc.
+tokyo
+
+// tools : 2013-11-21 Binky Moon, LLC
+tools
+
+// top : 2014-03-20 .TOP Registry
+top
+
+// toray : 2014-12-18 Toray Industries, Inc.
+toray
+
+// toshiba : 2014-04-10 TOSHIBA Corporation
+toshiba
+
+// total : 2015-08-06 Total SA
+total
+
+// tours : 2015-01-22 Binky Moon, LLC
+tours
+
+// town : 2014-03-06 Binky Moon, LLC
+town
+
+// toyota : 2015-04-23 TOYOTA MOTOR CORPORATION
+toyota
+
+// toys : 2014-03-06 Binky Moon, LLC
+toys
+
+// trade : 2014-01-23 Elite Registry Limited
+trade
+
+// trading : 2014-12-11 Dog Beach, LLC
+trading
+
+// training : 2013-11-07 Binky Moon, LLC
+training
+
+// travel : 2015-10-09 Dog Beach, LLC
+travel
+
+// travelchannel : 2015-07-02 Lifestyle Domain Holdings, Inc.
+travelchannel
+
+// travelers : 2015-03-26 Travelers TLD, LLC
+travelers
+
+// travelersinsurance : 2015-03-26 Travelers TLD, LLC
+travelersinsurance
+
+// trust : 2014-10-16 UNR Corp.
+trust
+
+// trv : 2015-03-26 Travelers TLD, LLC
+trv
+
+// tube : 2015-06-11 Latin American Telecom LLC
+tube
+
+// tui : 2014-07-03 TUI AG
+tui
+
+// tunes : 2015-02-26 Amazon Registry Services, Inc.
+tunes
+
+// tushu : 2014-12-18 Amazon Registry Services, Inc.
+tushu
+
+// tvs : 2015-02-19 T V SUNDRAM IYENGAR & SONS LIMITED
+tvs
+
+// ubank : 2015-08-20 National Australia Bank Limited
+ubank
+
+// ubs : 2014-12-11 UBS AG
+ubs
+
+// unicom : 2015-10-15 China United Network Communications Corporation Limited
+unicom
+
+// university : 2014-03-06 Binky Moon, LLC
+university
+
+// uno : 2013-09-11 Radix FZC
+uno
+
+// uol : 2014-05-01 UBN INTERNET LTDA.
+uol
+
+// ups : 2015-06-25 UPS Market Driver, Inc.
+ups
+
+// vacations : 2013-12-05 Binky Moon, LLC
+vacations
+
+// vana : 2014-12-11 Lifestyle Domain Holdings, Inc.
+vana
+
+// vanguard : 2015-09-03 The Vanguard Group, Inc.
+vanguard
+
+// vegas : 2014-01-16 Dot Vegas, Inc.
+vegas
+
+// ventures : 2013-08-27 Binky Moon, LLC
+ventures
+
+// verisign : 2015-08-13 VeriSign, Inc.
+verisign
+
+// versicherung : 2014-03-20 tldbox GmbH
+versicherung
+
+// vet : 2014-03-06 Dog Beach, LLC
+vet
+
+// viajes : 2013-10-17 Binky Moon, LLC
+viajes
+
+// video : 2014-10-16 Dog Beach, LLC
+video
+
+// vig : 2015-05-14 VIENNA INSURANCE GROUP AG Wiener Versicherung Gruppe
+vig
+
+// viking : 2015-04-02 Viking River Cruises (Bermuda) Ltd.
+viking
+
+// villas : 2013-12-05 Binky Moon, LLC
+villas
+
+// vin : 2015-06-18 Binky Moon, LLC
+vin
+
+// vip : 2015-01-22 Registry Services, LLC
+vip
+
+// virgin : 2014-09-25 Virgin Enterprises Limited
+virgin
+
+// visa : 2015-07-30 Visa Worldwide Pte. Limited
+visa
+
+// vision : 2013-12-05 Binky Moon, LLC
+vision
+
+// viva : 2014-11-07 Saudi Telecom Company
+viva
+
+// vivo : 2015-07-31 Telefonica Brasil S.A.
+vivo
+
+// vlaanderen : 2014-02-06 DNS.be vzw
+vlaanderen
+
+// vodka : 2013-12-19 Registry Services, LLC
+vodka
+
+// volkswagen : 2015-05-14 Volkswagen Group of America Inc.
+volkswagen
+
+// volvo : 2015-11-12 Volvo Holding Sverige Aktiebolag
+volvo
+
+// vote : 2013-11-21 Monolith Registry LLC
+vote
+
+// voting : 2013-11-13 Valuetainment Corp.
+voting
+
+// voto : 2013-11-21 Monolith Registry LLC
+voto
+
+// voyage : 2013-08-27 Binky Moon, LLC
+voyage
+
+// vuelos : 2015-03-05 Travel Reservations SRL
+vuelos
+
+// wales : 2014-05-08 Nominet UK
+wales
+
+// walmart : 2015-07-31 Wal-Mart Stores, Inc.
+walmart
+
+// walter : 2014-11-13 Sandvik AB
+walter
+
+// wang : 2013-10-24 Zodiac Wang Limited
+wang
+
+// wanggou : 2014-12-18 Amazon Registry Services, Inc.
+wanggou
+
+// watch : 2013-11-14 Binky Moon, LLC
+watch
+
+// watches : 2014-12-22 Afilias Limited
+watches
+
+// weather : 2015-01-08 International Business Machines Corporation
+weather
+
+// weatherchannel : 2015-03-12 International Business Machines Corporation
+weatherchannel
+
+// webcam : 2014-01-23 dot Webcam Limited
+webcam
+
+// weber : 2015-06-04 Saint-Gobain Weber SA
+weber
+
+// website : 2014-04-03 Radix FZC
+website
+
+// wedding : 2014-04-24 Registry Services, LLC
+wedding
+
+// weibo : 2015-03-05 Sina Corporation
+weibo
+
+// weir : 2015-01-29 Weir Group IP Limited
+weir
+
+// whoswho : 2014-02-20 Who's Who Registry
+whoswho
+
+// wien : 2013-10-28 punkt.wien GmbH
+wien
+
+// wiki : 2013-11-07 Top Level Design, LLC
+wiki
+
+// williamhill : 2014-03-13 William Hill Organization Limited
+williamhill
+
+// win : 2014-11-20 First Registry Limited
+win
+
+// windows : 2014-12-18 Microsoft Corporation
+windows
+
+// wine : 2015-06-18 Binky Moon, LLC
+wine
+
+// winners : 2015-07-16 The TJX Companies, Inc.
+winners
+
+// wme : 2014-02-13 William Morris Endeavor Entertainment, LLC
+wme
+
+// wolterskluwer : 2015-08-06 Wolters Kluwer N.V.
+wolterskluwer
+
+// woodside : 2015-07-09 Woodside Petroleum Limited
+woodside
+
+// work : 2013-12-19 Registry Services, LLC
+work
+
+// works : 2013-11-14 Binky Moon, LLC
+works
+
+// world : 2014-06-12 Binky Moon, LLC
+world
+
+// wow : 2015-10-08 Amazon Registry Services, Inc.
+wow
+
+// wtc : 2013-12-19 World Trade Centers Association, Inc.
+wtc
+
+// wtf : 2014-03-06 Binky Moon, LLC
+wtf
+
+// xbox : 2014-12-18 Microsoft Corporation
+xbox
+
+// xerox : 2014-10-24 Xerox DNHC LLC
+xerox
+
+// xfinity : 2015-07-09 Comcast IP Holdings I, LLC
+xfinity
+
+// xihuan : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+xihuan
+
+// xin : 2014-12-11 Elegant Leader Limited
+xin
+
+// xn--11b4c3d : 2015-01-15 VeriSign Sarl
+xn--11b4c3d
+कॉम
+
+// xn--1ck2e1b : 2015-02-26 Amazon Registry Services, Inc.
+xn--1ck2e1b
+セール
+
+// xn--1qqw23a : 2014-01-09 Guangzhou YU Wei Information Technology Co., Ltd.
+xn--1qqw23a
+佛山
+
+// xn--30rr7y : 2014-06-12 Excellent First Limited
+xn--30rr7y
+慈善
+
+// xn--3bst00m : 2013-09-13 Eagle Horizon Limited
+xn--3bst00m
+集团
+
+// xn--3ds443g : 2013-09-08 TLD REGISTRY LIMITED OY
+xn--3ds443g
+在线
+
+// xn--3oq18vl8pn36a : 2015-07-02 Volkswagen (China) Investment Co., Ltd.
+xn--3oq18vl8pn36a
+大众汽车
+
+// xn--3pxu8k : 2015-01-15 VeriSign Sarl
+xn--3pxu8k
+点看
+
+// xn--42c2d9a : 2015-01-15 VeriSign Sarl
+xn--42c2d9a
+คอม
+
+// xn--45q11c : 2013-11-21 Zodiac Gemini Ltd
+xn--45q11c
+八卦
+
+// xn--4gbrim : 2013-10-04 Fans TLD Limited
+xn--4gbrim
+موقع
+
+// xn--55qw42g : 2013-11-08 China Organizational Name Administration Center
+xn--55qw42g
+公益
+
+// xn--55qx5d : 2013-11-14 China Internet Network Information Center (CNNIC)
+xn--55qx5d
+公司
+
+// xn--5su34j936bgsg : 2015-09-03 Shangri‐La International Hotel Management Limited
+xn--5su34j936bgsg
+香格里拉
+
+// xn--5tzm5g : 2014-12-22 Global Website TLD Asia Limited
+xn--5tzm5g
+网站
+
+// xn--6frz82g : 2013-09-23 Afilias Limited
+xn--6frz82g
+移动
+
+// xn--6qq986b3xl : 2013-09-13 Tycoon Treasure Limited
+xn--6qq986b3xl
+我爱你
+
+// xn--80adxhks : 2013-12-19 Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID)
+xn--80adxhks
+москва
+
+// xn--80aqecdr1a : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+xn--80aqecdr1a
+католик
+
+// xn--80asehdb : 2013-07-14 CORE Association
+xn--80asehdb
+онлайн
+
+// xn--80aswg : 2013-07-14 CORE Association
+xn--80aswg
+сайт
+
+// xn--8y0a063a : 2015-03-26 China United Network Communications Corporation Limited
+xn--8y0a063a
+联通
+
+// xn--9dbq2a : 2015-01-15 VeriSign Sarl
+xn--9dbq2a
+קום
+
+// xn--9et52u : 2014-06-12 RISE VICTORY LIMITED
+xn--9et52u
+时尚
+
+// xn--9krt00a : 2015-03-12 Sina Corporation
+xn--9krt00a
+微博
+
+// xn--b4w605ferd : 2014-08-07 Temasek Holdings (Private) Limited
+xn--b4w605ferd
+淡马锡
+
+// xn--bck1b9a5dre4c : 2015-02-26 Amazon Registry Services, Inc.
+xn--bck1b9a5dre4c
+ファッション
+
+// xn--c1avg : 2013-11-14 Public Interest Registry
+xn--c1avg
+орг
+
+// xn--c2br7g : 2015-01-15 VeriSign Sarl
+xn--c2br7g
+नेट
+
+// xn--cck2b3b : 2015-02-26 Amazon Registry Services, Inc.
+xn--cck2b3b
+ストア
+
+// xn--cckwcxetd : 2019-12-19 Amazon Registry Services, Inc.
+xn--cckwcxetd
+アマゾン
+
+// xn--cg4bki : 2013-09-27 SAMSUNG SDS CO., LTD
+xn--cg4bki
+삼성
+
+// xn--czr694b : 2014-01-16 Internet DotTrademark Organisation Limited
+xn--czr694b
+商标
+
+// xn--czrs0t : 2013-12-19 Binky Moon, LLC
+xn--czrs0t
+商店
+
+// xn--czru2d : 2013-11-21 Zodiac Aquarius Limited
+xn--czru2d
+商城
+
+// xn--d1acj3b : 2013-11-20 The Foundation for Network Initiatives “The Smart Internet”
+xn--d1acj3b
+дети
+
+// xn--eckvdtc9d : 2014-12-18 Amazon Registry Services, Inc.
+xn--eckvdtc9d
+ポイント
+
+// xn--efvy88h : 2014-08-22 Guangzhou YU Wei Information Technology Co., Ltd.
+xn--efvy88h
+新闻
+
+// xn--fct429k : 2015-04-09 Amazon Registry Services, Inc.
+xn--fct429k
+家電
+
+// xn--fhbei : 2015-01-15 VeriSign Sarl
+xn--fhbei
+كوم
+
+// xn--fiq228c5hs : 2013-09-08 TLD REGISTRY LIMITED OY
+xn--fiq228c5hs
+中文网
+
+// xn--fiq64b : 2013-10-14 CITIC Group Corporation
+xn--fiq64b
+中信
+
+// xn--fjq720a : 2014-05-22 Binky Moon, LLC
+xn--fjq720a
+娱乐
+
+// xn--flw351e : 2014-07-31 Charleston Road Registry Inc.
+xn--flw351e
+谷歌
+
+// xn--fzys8d69uvgm : 2015-05-14 PCCW Enterprises Limited
+xn--fzys8d69uvgm
+電訊盈科
+
+// xn--g2xx48c : 2015-01-30 Nawang Heli(Xiamen) Network Service Co., LTD.
+xn--g2xx48c
+购物
+
+// xn--gckr3f0f : 2015-02-26 Amazon Registry Services, Inc.
+xn--gckr3f0f
+クラウド
+
+// xn--gk3at1e : 2015-10-08 Amazon Registry Services, Inc.
+xn--gk3at1e
+通販
+
+// xn--hxt814e : 2014-05-15 Zodiac Taurus Limited
+xn--hxt814e
+网店
+
+// xn--i1b6b1a6a2e : 2013-11-14 Public Interest Registry
+xn--i1b6b1a6a2e
+संगठन
+
+// xn--imr513n : 2014-12-11 Internet DotTrademark Organisation Limited
+xn--imr513n
+餐厅
+
+// xn--io0a7i : 2013-11-14 China Internet Network Information Center (CNNIC)
+xn--io0a7i
+网络
+
+// xn--j1aef : 2015-01-15 VeriSign Sarl
+xn--j1aef
+ком
+
+// xn--jlq480n2rg : 2019-12-19 Amazon Registry Services, Inc.
+xn--jlq480n2rg
+亚马逊
+
+// xn--jlq61u9w7b : 2015-01-08 Nokia Corporation
+xn--jlq61u9w7b
+诺基亚
+
+// xn--jvr189m : 2015-02-26 Amazon Registry Services, Inc.
+xn--jvr189m
+食品
+
+// xn--kcrx77d1x4a : 2014-11-07 Koninklijke Philips N.V.
+xn--kcrx77d1x4a
+飞利浦
+
+// xn--kput3i : 2014-02-13 Beijing RITT-Net Technology Development Co., Ltd
+xn--kput3i
+手机
+
+// xn--mgba3a3ejt : 2014-11-20 Aramco Services Company
+xn--mgba3a3ejt
+ارامكو
+
+// xn--mgba7c0bbn0a : 2015-05-14 Crescent Holding GmbH
+xn--mgba7c0bbn0a
+العليان
+
+// xn--mgbaakc7dvf : 2015-09-03 Emirates Telecommunications Corporation (trading as Etisalat)
+xn--mgbaakc7dvf
+اتصالات
+
+// xn--mgbab2bd : 2013-10-31 CORE Association
+xn--mgbab2bd
+بازار
+
+// xn--mgbca7dzdo : 2015-07-30 Abu Dhabi Systems and Information Centre
+xn--mgbca7dzdo
+ابوظبي
+
+// xn--mgbi4ecexp : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+xn--mgbi4ecexp
+كاثوليك
+
+// xn--mgbt3dhd : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+xn--mgbt3dhd
+همراه
+
+// xn--mk1bu44c : 2015-01-15 VeriSign Sarl
+xn--mk1bu44c
+닷컴
+
+// xn--mxtq1m : 2014-03-06 Net-Chinese Co., Ltd.
+xn--mxtq1m
+政府
+
+// xn--ngbc5azd : 2013-07-13 International Domain Registry Pty. Ltd.
+xn--ngbc5azd
+شبكة
+
+// xn--ngbe9e0a : 2014-12-04 Kuwait Finance House
+xn--ngbe9e0a
+بيتك
+
+// xn--ngbrx : 2015-11-12 League of Arab States
+xn--ngbrx
+عرب
+
+// xn--nqv7f : 2013-11-14 Public Interest Registry
+xn--nqv7f
+机构
+
+// xn--nqv7fs00ema : 2013-11-14 Public Interest Registry
+xn--nqv7fs00ema
+组织机构
+
+// xn--nyqy26a : 2014-11-07 Stable Tone Limited
+xn--nyqy26a
+健康
+
+// xn--otu796d : 2017-08-06 Jiang Yu Liang Cai Technology Company Limited
+xn--otu796d
+招聘
+
+// xn--p1acf : 2013-12-12 Rusnames Limited
+xn--p1acf
+рус
+
+// xn--pssy2u : 2015-01-15 VeriSign Sarl
+xn--pssy2u
+大拿
+
+// xn--q9jyb4c : 2013-09-17 Charleston Road Registry Inc.
+xn--q9jyb4c
+みんな
+
+// xn--qcka1pmc : 2014-07-31 Charleston Road Registry Inc.
+xn--qcka1pmc
+グーグル
+
+// xn--rhqv96g : 2013-09-11 Stable Tone Limited
+xn--rhqv96g
+世界
+
+// xn--rovu88b : 2015-02-26 Amazon Registry Services, Inc.
+xn--rovu88b
+書籍
+
+// xn--ses554g : 2014-01-16 KNET Co., Ltd.
+xn--ses554g
+网址
+
+// xn--t60b56a : 2015-01-15 VeriSign Sarl
+xn--t60b56a
+닷넷
+
+// xn--tckwe : 2015-01-15 VeriSign Sarl
+xn--tckwe
+コム
+
+// xn--tiq49xqyj : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+xn--tiq49xqyj
+天主教
+
+// xn--unup4y : 2013-07-14 Binky Moon, LLC
+xn--unup4y
+游戏
+
+// xn--vermgensberater-ctb : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+xn--vermgensberater-ctb
+vermögensberater
+
+// xn--vermgensberatung-pwb : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+xn--vermgensberatung-pwb
+vermögensberatung
+
+// xn--vhquv : 2013-08-27 Binky Moon, LLC
+xn--vhquv
+企业
+
+// xn--vuq861b : 2014-10-16 Beijing Tele-info Network Technology Co., Ltd.
+xn--vuq861b
+信息
+
+// xn--w4r85el8fhu5dnra : 2015-04-30 Kerry Trading Co. Limited
+xn--w4r85el8fhu5dnra
+嘉里大酒店
+
+// xn--w4rs40l : 2015-07-30 Kerry Trading Co. Limited
+xn--w4rs40l
+嘉里
+
+// xn--xhq521b : 2013-11-14 Guangzhou YU Wei Information Technology Co., Ltd.
+xn--xhq521b
+广东
+
+// xn--zfr164b : 2013-11-08 China Organizational Name Administration Center
+xn--zfr164b
+政务
+
+// xyz : 2013-12-05 XYZ.COM LLC
+xyz
+
+// yachts : 2014-01-09 XYZ.COM LLC
+yachts
+
+// yahoo : 2015-04-02 Oath Inc.
+yahoo
+
+// yamaxun : 2014-12-18 Amazon Registry Services, Inc.
+yamaxun
+
+// yandex : 2014-04-10 Yandex Europe B.V.
+yandex
+
+// yodobashi : 2014-11-20 YODOBASHI CAMERA CO.,LTD.
+yodobashi
+
+// yoga : 2014-05-29 Registry Services, LLC
+yoga
+
+// yokohama : 2013-12-12 GMO Registry, Inc.
+yokohama
+
+// you : 2015-04-09 Amazon Registry Services, Inc.
+you
+
+// youtube : 2014-05-01 Charleston Road Registry Inc.
+youtube
+
+// yun : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+yun
+
+// zappos : 2015-06-25 Amazon Registry Services, Inc.
+zappos
+
+// zara : 2014-11-07 Industria de Diseño Textil, S.A. (INDITEX, S.A.)
+zara
+
+// zero : 2014-12-18 Amazon Registry Services, Inc.
+zero
+
+// zip : 2014-05-08 Charleston Road Registry Inc.
+zip
+
+// zone : 2013-11-14 Binky Moon, LLC
+zone
+
+// zuerich : 2014-11-07 Kanton Zürich (Canton of Zurich)
+zuerich
+
+
+// ===END ICANN DOMAINS===
+// ===BEGIN PRIVATE DOMAINS===
+// (Note: these are in alphabetical order by company name)
+
+// 1GB LLC : https://www.1gb.ua/
+// Submitted by 1GB LLC <noc@1gb.com.ua>
+cc.ua
+inf.ua
+ltd.ua
+
+// 611coin : https://611project.org/
+611.to
+
+// Aaron Marais' Gitlab pages: https://lab.aaronleem.co.za
+// Submitted by Aaron Marais <its_me@aaronleem.co.za>
+graphox.us
+
+// accesso Technology Group, plc. : https://accesso.com/
+// Submitted by accesso Team <accessoecommerce@accesso.com>
+*.devcdnaccesso.com
+
+// Adobe : https://www.adobe.com/
+// Submitted by Ian Boston <boston@adobe.com> and Lars Trieloff <trieloff@adobe.com>
+adobeaemcloud.com
+*.dev.adobeaemcloud.com
+hlx.live
+adobeaemcloud.net
+hlx.page
+hlx3.page
+
+// Agnat sp. z o.o. : https://domena.pl
+// Submitted by Przemyslaw Plewa <it-admin@domena.pl>
+beep.pl
+
+// alboto.ca : http://alboto.ca
+// Submitted by Anton Avramov <avramov@alboto.ca>
+barsy.ca
+
+// Alces Software Ltd : http://alces-software.com
+// Submitted by Mark J. Titorenko <mark.titorenko@alces-software.com>
+*.compute.estate
+*.alces.network
+
+// all-inkl.com : https://all-inkl.com
+// Submitted by Werner Kaltofen <wk@all-inkl.com>
+kasserver.com
+
+// Altervista: https://www.altervista.org
+// Submitted by Carlo Cannas <tech_staff@altervista.it>
+altervista.org
+
+// alwaysdata : https://www.alwaysdata.com
+// Submitted by Cyril <admin@alwaysdata.com>
+alwaysdata.net
+
+// Amazon CloudFront : https://aws.amazon.com/cloudfront/
+// Submitted by Donavan Miller <donavanm@amazon.com>
+cloudfront.net
+
+// Amazon Elastic Compute Cloud : https://aws.amazon.com/ec2/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+*.compute.amazonaws.com
+*.compute-1.amazonaws.com
+*.compute.amazonaws.com.cn
+us-east-1.amazonaws.com
+
+// Amazon Elastic Beanstalk : https://aws.amazon.com/elasticbeanstalk/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+cn-north-1.eb.amazonaws.com.cn
+cn-northwest-1.eb.amazonaws.com.cn
+elasticbeanstalk.com
+ap-northeast-1.elasticbeanstalk.com
+ap-northeast-2.elasticbeanstalk.com
+ap-northeast-3.elasticbeanstalk.com
+ap-south-1.elasticbeanstalk.com
+ap-southeast-1.elasticbeanstalk.com
+ap-southeast-2.elasticbeanstalk.com
+ca-central-1.elasticbeanstalk.com
+eu-central-1.elasticbeanstalk.com
+eu-west-1.elasticbeanstalk.com
+eu-west-2.elasticbeanstalk.com
+eu-west-3.elasticbeanstalk.com
+sa-east-1.elasticbeanstalk.com
+us-east-1.elasticbeanstalk.com
+us-east-2.elasticbeanstalk.com
+us-gov-west-1.elasticbeanstalk.com
+us-west-1.elasticbeanstalk.com
+us-west-2.elasticbeanstalk.com
+
+// Amazon Elastic Load Balancing : https://aws.amazon.com/elasticloadbalancing/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+*.elb.amazonaws.com
+*.elb.amazonaws.com.cn
+
+// Amazon Global Accelerator : https://aws.amazon.com/global-accelerator/
+// Submitted by Daniel Massaguer <psl-maintainers@amazon.com>
+awsglobalaccelerator.com
+
+// Amazon S3 : https://aws.amazon.com/s3/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+s3.amazonaws.com
+s3-ap-northeast-1.amazonaws.com
+s3-ap-northeast-2.amazonaws.com
+s3-ap-south-1.amazonaws.com
+s3-ap-southeast-1.amazonaws.com
+s3-ap-southeast-2.amazonaws.com
+s3-ca-central-1.amazonaws.com
+s3-eu-central-1.amazonaws.com
+s3-eu-west-1.amazonaws.com
+s3-eu-west-2.amazonaws.com
+s3-eu-west-3.amazonaws.com
+s3-external-1.amazonaws.com
+s3-fips-us-gov-west-1.amazonaws.com
+s3-sa-east-1.amazonaws.com
+s3-us-gov-west-1.amazonaws.com
+s3-us-east-2.amazonaws.com
+s3-us-west-1.amazonaws.com
+s3-us-west-2.amazonaws.com
+s3.ap-northeast-2.amazonaws.com
+s3.ap-south-1.amazonaws.com
+s3.cn-north-1.amazonaws.com.cn
+s3.ca-central-1.amazonaws.com
+s3.eu-central-1.amazonaws.com
+s3.eu-west-2.amazonaws.com
+s3.eu-west-3.amazonaws.com
+s3.us-east-2.amazonaws.com
+s3.dualstack.ap-northeast-1.amazonaws.com
+s3.dualstack.ap-northeast-2.amazonaws.com
+s3.dualstack.ap-south-1.amazonaws.com
+s3.dualstack.ap-southeast-1.amazonaws.com
+s3.dualstack.ap-southeast-2.amazonaws.com
+s3.dualstack.ca-central-1.amazonaws.com
+s3.dualstack.eu-central-1.amazonaws.com
+s3.dualstack.eu-west-1.amazonaws.com
+s3.dualstack.eu-west-2.amazonaws.com
+s3.dualstack.eu-west-3.amazonaws.com
+s3.dualstack.sa-east-1.amazonaws.com
+s3.dualstack.us-east-1.amazonaws.com
+s3.dualstack.us-east-2.amazonaws.com
+s3-website-us-east-1.amazonaws.com
+s3-website-us-west-1.amazonaws.com
+s3-website-us-west-2.amazonaws.com
+s3-website-ap-northeast-1.amazonaws.com
+s3-website-ap-southeast-1.amazonaws.com
+s3-website-ap-southeast-2.amazonaws.com
+s3-website-eu-west-1.amazonaws.com
+s3-website-sa-east-1.amazonaws.com
+s3-website.ap-northeast-2.amazonaws.com
+s3-website.ap-south-1.amazonaws.com
+s3-website.ca-central-1.amazonaws.com
+s3-website.eu-central-1.amazonaws.com
+s3-website.eu-west-2.amazonaws.com
+s3-website.eu-west-3.amazonaws.com
+s3-website.us-east-2.amazonaws.com
+
+// Amsterdam Wireless: https://www.amsterdamwireless.nl/
+// Submitted by Imre Jonk <hostmaster@amsterdamwireless.nl>
+amsw.nl
+
+// Amune : https://amune.org/
+// Submitted by Team Amune <cert@amune.org>
+t3l3p0rt.net
+tele.amune.org
+
+// Apigee : https://apigee.com/
+// Submitted by Apigee Security Team <security@apigee.com>
+apigee.io
+
+// Appspace : https://www.appspace.com
+// Submitted by Appspace Security Team <security@appspace.com>
+appspacehosted.com
+appspaceusercontent.com
+
+// Appudo UG (haftungsbeschränkt) : https://www.appudo.com
+// Submitted by Alexander Hochbaum <admin@appudo.com>
+appudo.net
+
+// Aptible : https://www.aptible.com/
+// Submitted by Thomas Orozco <thomas@aptible.com>
+on-aptible.com
+
+// ASEINet : https://www.aseinet.com/
+// Submitted by Asei SEKIGUCHI <mail@aseinet.com>
+user.aseinet.ne.jp
+gv.vc
+d.gv.vc
+
+// Asociación Amigos de la Informática "Euskalamiga" : http://encounter.eus/
+// Submitted by Hector Martin <marcan@euskalencounter.org>
+user.party.eus
+
+// Association potager.org : https://potager.org/
+// Submitted by Lunar <jardiniers@potager.org>
+pimienta.org
+poivron.org
+potager.org
+sweetpepper.org
+
+// ASUSTOR Inc. : http://www.asustor.com
+// Submitted by Vincent Tseng <vincenttseng@asustor.com>
+myasustor.com
+
+// Atlassian : https://atlassian.com
+// Submitted by Sam Smyth <devloop@atlassian.com>
+cdn.prod.atlassian-dev.net
+
+// AVM : https://avm.de
+// Submitted by Andreas Weise <a.weise@avm.de>
+myfritz.net
+
+// AVStack Pte. Ltd. : https://avstack.io
+// Submitted by Jasper Hugo <jasper@avstack.io>
+onavstack.net
+
+// AW AdvisorWebsites.com Software Inc : https://advisorwebsites.com
+// Submitted by James Kennedy <domains@advisorwebsites.com>
+*.awdev.ca
+*.advisor.ws
+
+// b-data GmbH : https://www.b-data.io
+// Submitted by Olivier Benz <olivier.benz@b-data.ch>
+b-data.io
+
+// backplane : https://www.backplane.io
+// Submitted by Anthony Voutas <anthony@backplane.io>
+backplaneapp.io
+
+// Balena : https://www.balena.io
+// Submitted by Petros Angelatos <petrosagg@balena.io>
+balena-devices.com
+
+// University of Banja Luka : https://unibl.org
+// Domains for Republic of Srpska administrative entity.
+// Submitted by Marko Ivanovic <kormang@hotmail.rs>
+rs.ba
+
+// Banzai Cloud
+// Submitted by Janos Matyas <info@banzaicloud.com>
+*.banzai.cloud
+app.banzaicloud.io
+*.backyards.banzaicloud.io
+
+
+// BetaInABox
+// Submitted by Adrian <adrian@betainabox.com>
+betainabox.com
+
+// BinaryLane : http://www.binarylane.com
+// Submitted by Nathan O'Sullivan <nathan@mammoth.com.au>
+bnr.la
+
+// Bitbucket : http://bitbucket.org
+// Submitted by Andy Ortlieb <aortlieb@atlassian.com>
+bitbucket.io
+
+// Blackbaud, Inc. : https://www.blackbaud.com
+// Submitted by Paul Crowder <paul.crowder@blackbaud.com>
+blackbaudcdn.net
+
+// Blatech : http://www.blatech.net
+// Submitted by Luke Bratch <luke@bratch.co.uk>
+of.je
+
+// Blue Bite, LLC : https://bluebite.com
+// Submitted by Joshua Weiss <admin.engineering@bluebite.com>
+bluebite.io
+
+// Boomla : https://boomla.com
+// Submitted by Tibor Halter <thalter@boomla.com>
+boomla.net
+
+// Boutir : https://www.boutir.com
+// Submitted by Eric Ng Ka Ka <ngkaka@boutir.com>
+boutir.com
+
+// Boxfuse : https://boxfuse.com
+// Submitted by Axel Fontaine <axel@boxfuse.com>
+boxfuse.io
+
+// bplaced : https://www.bplaced.net/
+// Submitted by Miroslav Bozic <security@bplaced.net>
+square7.ch
+bplaced.com
+bplaced.de
+square7.de
+bplaced.net
+square7.net
+
+// Brendly : https://brendly.rs
+// Submitted by Dusan Radovanovic <dusan.radovanovic@brendly.rs>
+shop.brendly.rs
+
+// BrowserSafetyMark
+// Submitted by Dave Tharp <browsersafetymark.io@quicinc.com>
+browsersafetymark.io
+
+// Bytemark Hosting : https://www.bytemark.co.uk
+// Submitted by Paul Cammish <paul.cammish@bytemark.co.uk>
+uk0.bigv.io
+dh.bytemark.co.uk
+vm.bytemark.co.uk
+
+// Caf.js Labs LLC : https://www.cafjs.com
+// Submitted by Antonio Lain <antlai@cafjs.com>
+cafjs.com
+
+// callidomus : https://www.callidomus.com/
+// Submitted by Marcus Popp <admin@callidomus.com>
+mycd.eu
+
+// Carrd : https://carrd.co
+// Submitted by AJ <aj@carrd.co>
+drr.ac
+uwu.ai
+carrd.co
+crd.co
+ju.mp
+
+// CentralNic : http://www.centralnic.com/names/domains
+// Submitted by registry <gavin.brown@centralnic.com>
+ae.org
+br.com
+cn.com
+com.de
+com.se
+de.com
+eu.com
+gb.net
+hu.net
+jp.net
+jpn.com
+mex.com
+ru.com
+sa.com
+se.net
+uk.com
+uk.net
+us.com
+za.bz
+za.com
+
+// No longer operated by CentralNic, these entries should be adopted and/or removed by current operators
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+ar.com
+gb.com
+hu.com
+kr.com
+no.com
+qc.com
+uy.com
+
+// Africa.com Web Solutions Ltd : https://registry.africa.com
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+africa.com
+
+// iDOT Services Limited : http://www.domain.gr.com
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+gr.com
+
+// Radix FZC : http://domains.in.net
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+in.net
+web.in
+
+// US REGISTRY LLC : http://us.org
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+us.org
+
+// co.com Registry, LLC : https://registry.co.com
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+co.com
+
+// Roar Domains LLC : https://roar.basketball/
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+aus.basketball
+nz.basketball
+
+// BRS Media : https://brsmedia.com/
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+radio.am
+radio.fm
+
+// c.la : http://www.c.la/
+c.la
+
+// certmgr.org : https://certmgr.org
+// Submitted by B. Blechschmidt <hostmaster@certmgr.org>
+certmgr.org
+
+// Cityhost LLC : https://cityhost.ua
+// Submitted by Maksym Rivtin <support@cityhost.net.ua>
+cx.ua
+
+// Civilized Discourse Construction Kit, Inc. : https://www.discourse.org/
+// Submitted by Rishabh Nambiar & Michael Brown <team@discourse.org>
+discourse.group
+discourse.team
+
+// ClearVox : http://www.clearvox.nl/
+// Submitted by Leon Rowland <leon@clearvox.nl>
+virtueeldomein.nl
+
+// Clever Cloud : https://www.clever-cloud.com/
+// Submitted by Quentin Adam <noc@clever-cloud.com>
+cleverapps.io
+
+// Clerk : https://www.clerk.dev
+// Submitted by Colin Sidoti <systems@clerk.dev>
+*.lcl.dev
+*.lclstage.dev
+*.stg.dev
+*.stgstage.dev
+
+// Clic2000 : https://clic2000.fr
+// Submitted by Mathilde Blanchemanche <mathilde@clic2000.fr>
+clic2000.net
+
+// ClickRising : https://clickrising.com/
+// Submitted by Umut Gumeli <infrastructure-publicsuffixlist@clickrising.com>
+clickrising.net
+
+// Cloud66 : https://www.cloud66.com/
+// Submitted by Khash Sajadi <khash@cloud66.com>
+c66.me
+cloud66.ws
+cloud66.zone
+
+// CloudAccess.net : https://www.cloudaccess.net/
+// Submitted by Pawel Panek <noc@cloudaccess.net>
+jdevcloud.com
+wpdevcloud.com
+cloudaccess.host
+freesite.host
+cloudaccess.net
+
+// cloudControl : https://www.cloudcontrol.com/
+// Submitted by Tobias Wilken <tw@cloudcontrol.com>
+cloudcontrolled.com
+cloudcontrolapp.com
+
+// Cloudera, Inc. : https://www.cloudera.com/
+// Submitted by Philip Langdale <security@cloudera.com>
+cloudera.site
+
+// Cloudflare, Inc. : https://www.cloudflare.com/
+// Submitted by Cloudflare Team <publicsuffixlist@cloudflare.com>
+pages.dev
+trycloudflare.com
+workers.dev
+
+// Clovyr : https://clovyr.io
+// Submitted by Patrick Nielsen <patrick@clovyr.io>
+wnext.app
+
+// co.ca : http://registry.co.ca/
+co.ca
+
+// Co & Co : https://co-co.nl/
+// Submitted by Govert Versluis <govert@co-co.nl>
+*.otap.co
+
+// i-registry s.r.o. : http://www.i-registry.cz/
+// Submitted by Martin Semrad <semrad@i-registry.cz>
+co.cz
+
+// CDN77.com : http://www.cdn77.com
+// Submitted by Jan Krpes <jan.krpes@cdn77.com>
+c.cdn77.org
+cdn77-ssl.net
+r.cdn77.net
+rsc.cdn77.org
+ssl.origin.cdn77-secure.org
+
+// Cloud DNS Ltd : http://www.cloudns.net
+// Submitted by Aleksander Hristov <noc@cloudns.net>
+cloudns.asia
+cloudns.biz
+cloudns.club
+cloudns.cc
+cloudns.eu
+cloudns.in
+cloudns.info
+cloudns.org
+cloudns.pro
+cloudns.pw
+cloudns.us
+
+// CNPY : https://cnpy.gdn
+// Submitted by Angelo Gladding <angelo@lahacker.net>
+cnpy.gdn
+
+// CoDNS B.V.
+co.nl
+co.no
+
+// Combell.com : https://www.combell.com
+// Submitted by Thomas Wouters <thomas.wouters@combellgroup.com>
+webhosting.be
+hosting-cluster.nl
+
+// Coordination Center for TLD RU and XN--P1AI : https://cctld.ru/en/domains/domens_ru/reserved/
+// Submitted by George Georgievsky <gug@cctld.ru>
+ac.ru
+edu.ru
+gov.ru
+int.ru
+mil.ru
+test.ru
+
+// COSIMO GmbH : http://www.cosimo.de
+// Submitted by Rene Marticke <rmarticke@cosimo.de>
+dyn.cosidns.de
+dynamisches-dns.de
+dnsupdater.de
+internet-dns.de
+l-o-g-i-n.de
+dynamic-dns.info
+feste-ip.net
+knx-server.net
+static-access.net
+
+// Craynic, s.r.o. : http://www.craynic.com/
+// Submitted by Ales Krajnik <ales.krajnik@craynic.com>
+realm.cz
+
+// Cryptonomic : https://cryptonomic.net/
+// Submitted by Andrew Cady <public-suffix-list@cryptonomic.net>
+*.cryptonomic.net
+
+// Cupcake : https://cupcake.io/
+// Submitted by Jonathan Rudenberg <jonathan@cupcake.io>
+cupcake.is
+
+// Curv UG : https://curv-labs.de/
+// Submitted by Marvin Wiesner <Marvin@curv-labs.de>
+curv.dev
+
+// Customer OCI - Oracle Dyn https://cloud.oracle.com/home https://dyn.com/dns/
+// Submitted by Gregory Drake <support@dyn.com>
+// Note: This is intended to also include customer-oci.com due to wildcards implicitly including the current label
+*.customer-oci.com
+*.oci.customer-oci.com
+*.ocp.customer-oci.com
+*.ocs.customer-oci.com
+
+// cyon GmbH : https://www.cyon.ch/
+// Submitted by Dominic Luechinger <dol@cyon.ch>
+cyon.link
+cyon.site
+
+// Danger Science Group: https://dangerscience.com/
+// Submitted by Skylar MacDonald <skylar@dangerscience.com>
+fnwk.site
+folionetwork.site
+platform0.app
+
+// Daplie, Inc : https://daplie.com
+// Submitted by AJ ONeal <aj@daplie.com>
+daplie.me
+localhost.daplie.me
+
+// Datto, Inc. : https://www.datto.com/
+// Submitted by Philipp Heckel <ph@datto.com>
+dattolocal.com
+dattorelay.com
+dattoweb.com
+mydatto.com
+dattolocal.net
+mydatto.net
+
+// Dansk.net : http://www.dansk.net/
+// Submitted by Anani Voule <digital@digital.co.dk>
+biz.dk
+co.dk
+firm.dk
+reg.dk
+store.dk
+
+// dappnode.io : https://dappnode.io/
+// Submitted by Abel Boldu / DAppNode Team <community@dappnode.io>
+dyndns.dappnode.io
+
+// dapps.earth : https://dapps.earth/
+// Submitted by Daniil Burdakov <icqkill@gmail.com>
+*.dapps.earth
+*.bzz.dapps.earth
+
+// Dark, Inc. : https://darklang.com
+// Submitted by Paul Biggar <ops@darklang.com>
+builtwithdark.com
+
+// DataDetect, LLC. : https://datadetect.com
+// Submitted by Andrew Banchich <abanchich@sceven.com>
+demo.datadetect.com
+instance.datadetect.com
+
+// Datawire, Inc : https://www.datawire.io
+// Submitted by Richard Li <secalert@datawire.io>
+edgestack.me
+
+// DDNS5 : https://ddns5.com
+// Submitted by Cameron Elliott <cameron@cameronelliott.com>
+ddns5.com
+
+// Debian : https://www.debian.org/
+// Submitted by Peter Palfrader / Debian Sysadmin Team <dsa-publicsuffixlist@debian.org>
+debian.net
+
+// Deno Land Inc : https://deno.com/
+// Submitted by Luca Casonato <hostmaster@deno.com>
+deno.dev
+deno-staging.dev
+
+// deSEC : https://desec.io/
+// Submitted by Peter Thomassen <peter@desec.io>
+dedyn.io
+
+// DNS Africa Ltd https://dns.business
+// Submitted by Calvin Browne <calvin@dns.business>
+jozi.biz
+
+// DNShome : https://www.dnshome.de/
+// Submitted by Norbert Auler <mail@dnshome.de>
+dnshome.de
+
+// DotArai : https://www.dotarai.com/
+// Submitted by Atsadawat Netcharadsang <atsadawat@dotarai.co.th>
+online.th
+shop.th
+
+// DrayTek Corp. : https://www.draytek.com/
+// Submitted by Paul Fang <mis@draytek.com>
+drayddns.com
+
+// DreamCommerce : https://shoper.pl/
+// Submitted by Konrad Kotarba <konrad.kotarba@dreamcommerce.com>
+shoparena.pl
+
+// DreamHost : http://www.dreamhost.com/
+// Submitted by Andrew Farmer <andrew.farmer@dreamhost.com>
+dreamhosters.com
+
+// Drobo : http://www.drobo.com/
+// Submitted by Ricardo Padilha <rpadilha@drobo.com>
+mydrobo.com
+
+// Drud Holdings, LLC. : https://www.drud.com/
+// Submitted by Kevin Bridges <kevin@drud.com>
+drud.io
+drud.us
+
+// DuckDNS : http://www.duckdns.org/
+// Submitted by Richard Harper <richard@duckdns.org>
+duckdns.org
+
+// Bip : https://bip.sh
+// Submitted by Joel Kennedy <joel@bip.sh>
+bip.sh
+
+// bitbridge.net : Submitted by Craig Welch, abeliidev@gmail.com
+bitbridge.net
+
+// dy.fi : http://dy.fi/
+// Submitted by Heikki Hannikainen <hessu@hes.iki.fi>
+dy.fi
+tunk.org
+
+// DynDNS.com : http://www.dyndns.com/services/dns/dyndns/
+dyndns-at-home.com
+dyndns-at-work.com
+dyndns-blog.com
+dyndns-free.com
+dyndns-home.com
+dyndns-ip.com
+dyndns-mail.com
+dyndns-office.com
+dyndns-pics.com
+dyndns-remote.com
+dyndns-server.com
+dyndns-web.com
+dyndns-wiki.com
+dyndns-work.com
+dyndns.biz
+dyndns.info
+dyndns.org
+dyndns.tv
+at-band-camp.net
+ath.cx
+barrel-of-knowledge.info
+barrell-of-knowledge.info
+better-than.tv
+blogdns.com
+blogdns.net
+blogdns.org
+blogsite.org
+boldlygoingnowhere.org
+broke-it.net
+buyshouses.net
+cechire.com
+dnsalias.com
+dnsalias.net
+dnsalias.org
+dnsdojo.com
+dnsdojo.net
+dnsdojo.org
+does-it.net
+doesntexist.com
+doesntexist.org
+dontexist.com
+dontexist.net
+dontexist.org
+doomdns.com
+doomdns.org
+dvrdns.org
+dyn-o-saur.com
+dynalias.com
+dynalias.net
+dynalias.org
+dynathome.net
+dyndns.ws
+endofinternet.net
+endofinternet.org
+endoftheinternet.org
+est-a-la-maison.com
+est-a-la-masion.com
+est-le-patron.com
+est-mon-blogueur.com
+for-better.biz
+for-more.biz
+for-our.info
+for-some.biz
+for-the.biz
+forgot.her.name
+forgot.his.name
+from-ak.com
+from-al.com
+from-ar.com
+from-az.net
+from-ca.com
+from-co.net
+from-ct.com
+from-dc.com
+from-de.com
+from-fl.com
+from-ga.com
+from-hi.com
+from-ia.com
+from-id.com
+from-il.com
+from-in.com
+from-ks.com
+from-ky.com
+from-la.net
+from-ma.com
+from-md.com
+from-me.org
+from-mi.com
+from-mn.com
+from-mo.com
+from-ms.com
+from-mt.com
+from-nc.com
+from-nd.com
+from-ne.com
+from-nh.com
+from-nj.com
+from-nm.com
+from-nv.com
+from-ny.net
+from-oh.com
+from-ok.com
+from-or.com
+from-pa.com
+from-pr.com
+from-ri.com
+from-sc.com
+from-sd.com
+from-tn.com
+from-tx.com
+from-ut.com
+from-va.com
+from-vt.com
+from-wa.com
+from-wi.com
+from-wv.com
+from-wy.com
+ftpaccess.cc
+fuettertdasnetz.de
+game-host.org
+game-server.cc
+getmyip.com
+gets-it.net
+go.dyndns.org
+gotdns.com
+gotdns.org
+groks-the.info
+groks-this.info
+ham-radio-op.net
+here-for-more.info
+hobby-site.com
+hobby-site.org
+home.dyndns.org
+homedns.org
+homeftp.net
+homeftp.org
+homeip.net
+homelinux.com
+homelinux.net
+homelinux.org
+homeunix.com
+homeunix.net
+homeunix.org
+iamallama.com
+in-the-band.net
+is-a-anarchist.com
+is-a-blogger.com
+is-a-bookkeeper.com
+is-a-bruinsfan.org
+is-a-bulls-fan.com
+is-a-candidate.org
+is-a-caterer.com
+is-a-celticsfan.org
+is-a-chef.com
+is-a-chef.net
+is-a-chef.org
+is-a-conservative.com
+is-a-cpa.com
+is-a-cubicle-slave.com
+is-a-democrat.com
+is-a-designer.com
+is-a-doctor.com
+is-a-financialadvisor.com
+is-a-geek.com
+is-a-geek.net
+is-a-geek.org
+is-a-green.com
+is-a-guru.com
+is-a-hard-worker.com
+is-a-hunter.com
+is-a-knight.org
+is-a-landscaper.com
+is-a-lawyer.com
+is-a-liberal.com
+is-a-libertarian.com
+is-a-linux-user.org
+is-a-llama.com
+is-a-musician.com
+is-a-nascarfan.com
+is-a-nurse.com
+is-a-painter.com
+is-a-patsfan.org
+is-a-personaltrainer.com
+is-a-photographer.com
+is-a-player.com
+is-a-republican.com
+is-a-rockstar.com
+is-a-socialist.com
+is-a-soxfan.org
+is-a-student.com
+is-a-teacher.com
+is-a-techie.com
+is-a-therapist.com
+is-an-accountant.com
+is-an-actor.com
+is-an-actress.com
+is-an-anarchist.com
+is-an-artist.com
+is-an-engineer.com
+is-an-entertainer.com
+is-by.us
+is-certified.com
+is-found.org
+is-gone.com
+is-into-anime.com
+is-into-cars.com
+is-into-cartoons.com
+is-into-games.com
+is-leet.com
+is-lost.org
+is-not-certified.com
+is-saved.org
+is-slick.com
+is-uberleet.com
+is-very-bad.org
+is-very-evil.org
+is-very-good.org
+is-very-nice.org
+is-very-sweet.org
+is-with-theband.com
+isa-geek.com
+isa-geek.net
+isa-geek.org
+isa-hockeynut.com
+issmarterthanyou.com
+isteingeek.de
+istmein.de
+kicks-ass.net
+kicks-ass.org
+knowsitall.info
+land-4-sale.us
+lebtimnetz.de
+leitungsen.de
+likes-pie.com
+likescandy.com
+merseine.nu
+mine.nu
+misconfused.org
+mypets.ws
+myphotos.cc
+neat-url.com
+office-on-the.net
+on-the-web.tv
+podzone.net
+podzone.org
+readmyblog.org
+saves-the-whales.com
+scrapper-site.net
+scrapping.cc
+selfip.biz
+selfip.com
+selfip.info
+selfip.net
+selfip.org
+sells-for-less.com
+sells-for-u.com
+sells-it.net
+sellsyourhome.org
+servebbs.com
+servebbs.net
+servebbs.org
+serveftp.net
+serveftp.org
+servegame.org
+shacknet.nu
+simple-url.com
+space-to-rent.com
+stuff-4-sale.org
+stuff-4-sale.us
+teaches-yoga.com
+thruhere.net
+traeumtgerade.de
+webhop.biz
+webhop.info
+webhop.net
+webhop.org
+worse-than.tv
+writesthisblog.com
+
+// ddnss.de : https://www.ddnss.de/
+// Submitted by Robert Niedziela <webmaster@ddnss.de>
+ddnss.de
+dyn.ddnss.de
+dyndns.ddnss.de
+dyndns1.de
+dyn-ip24.de
+home-webserver.de
+dyn.home-webserver.de
+myhome-server.de
+ddnss.org
+
+// Definima : http://www.definima.com/
+// Submitted by Maxence Bitterli <maxence@definima.com>
+definima.net
+definima.io
+
+// DigitalOcean : https://digitalocean.com/
+// Submitted by Braxton Huggins <bhuggins@digitalocean.com>
+ondigitalocean.app
+
+// dnstrace.pro : https://dnstrace.pro/
+// Submitted by Chris Partridge <chris@partridge.tech>
+bci.dnstrace.pro
+
+// Dynu.com : https://www.dynu.com/
+// Submitted by Sue Ye <sue@dynu.com>
+ddnsfree.com
+ddnsgeek.com
+giize.com
+gleeze.com
+kozow.com
+loseyourip.com
+ooguy.com
+theworkpc.com
+casacam.net
+dynu.net
+accesscam.org
+camdvr.org
+freeddns.org
+mywire.org
+webredirect.org
+myddns.rocks
+blogsite.xyz
+
+// dynv6 : https://dynv6.com
+// Submitted by Dominik Menke <dom@digineo.de>
+dynv6.net
+
+// E4YOU spol. s.r.o. : https://e4you.cz/
+// Submitted by Vladimir Dudr <info@e4you.cz>
+e4.cz
+
+// eero : https://eero.com/
+// Submitted by Yue Kang <eero-dynamic-dns@amazon.com>
+eero.online
+eero-stage.online
+
+// Elementor : Elementor Ltd.
+// Submitted by Anton Barkan <antonb@elementor.com>
+elementor.cloud
+elementor.cool
+
+// En root‽ : https://en-root.org
+// Submitted by Emmanuel Raviart <emmanuel@raviart.com>
+en-root.fr
+
+// Enalean SAS: https://www.enalean.com
+// Submitted by Thomas Cottier <thomas.cottier@enalean.com>
+mytuleap.com
+tuleap-partners.com
+
+// ECG Robotics, Inc: https://ecgrobotics.org
+// Submitted by <frc1533@ecgrobotics.org>
+onred.one
+staging.onred.one
+
+// One.com: https://www.one.com/
+// Submitted by Jacob Bunk Nielsen <jbn@one.com>
+service.one
+
+// Enonic : http://enonic.com/
+// Submitted by Erik Kaareng-Sunde <esu@enonic.com>
+enonic.io
+customer.enonic.io
+
+// EU.org https://eu.org/
+// Submitted by Pierre Beyssac <hostmaster@eu.org>
+eu.org
+al.eu.org
+asso.eu.org
+at.eu.org
+au.eu.org
+be.eu.org
+bg.eu.org
+ca.eu.org
+cd.eu.org
+ch.eu.org
+cn.eu.org
+cy.eu.org
+cz.eu.org
+de.eu.org
+dk.eu.org
+edu.eu.org
+ee.eu.org
+es.eu.org
+fi.eu.org
+fr.eu.org
+gr.eu.org
+hr.eu.org
+hu.eu.org
+ie.eu.org
+il.eu.org
+in.eu.org
+int.eu.org
+is.eu.org
+it.eu.org
+jp.eu.org
+kr.eu.org
+lt.eu.org
+lu.eu.org
+lv.eu.org
+mc.eu.org
+me.eu.org
+mk.eu.org
+mt.eu.org
+my.eu.org
+net.eu.org
+ng.eu.org
+nl.eu.org
+no.eu.org
+nz.eu.org
+paris.eu.org
+pl.eu.org
+pt.eu.org
+q-a.eu.org
+ro.eu.org
+ru.eu.org
+se.eu.org
+si.eu.org
+sk.eu.org
+tr.eu.org
+uk.eu.org
+us.eu.org
+
+// Eurobyte : https://eurobyte.ru
+// Submitted by Evgeniy Subbotin <e.subbotin@eurobyte.ru>
+eurodir.ru
+
+// Evennode : http://www.evennode.com/
+// Submitted by Michal Kralik <support@evennode.com>
+eu-1.evennode.com
+eu-2.evennode.com
+eu-3.evennode.com
+eu-4.evennode.com
+us-1.evennode.com
+us-2.evennode.com
+us-3.evennode.com
+us-4.evennode.com
+
+// eDirect Corp. : https://hosting.url.com.tw/
+// Submitted by C.S. chang <cschang@corp.url.com.tw>
+twmail.cc
+twmail.net
+twmail.org
+mymailer.com.tw
+url.tw
+
+// Fabrica Technologies, Inc. : https://www.fabrica.dev/
+// Submitted by Eric Jiang <eric@fabrica.dev>
+onfabrica.com
+
+// Facebook, Inc.
+// Submitted by Peter Ruibal <public-suffix@fb.com>
+apps.fbsbx.com
+
+// FAITID : https://faitid.org/
+// Submitted by Maxim Alzoba <tech.contact@faitid.org>
+// https://www.flexireg.net/stat_info
+ru.net
+adygeya.ru
+bashkiria.ru
+bir.ru
+cbg.ru
+com.ru
+dagestan.ru
+grozny.ru
+kalmykia.ru
+kustanai.ru
+marine.ru
+mordovia.ru
+msk.ru
+mytis.ru
+nalchik.ru
+nov.ru
+pyatigorsk.ru
+spb.ru
+vladikavkaz.ru
+vladimir.ru
+abkhazia.su
+adygeya.su
+aktyubinsk.su
+arkhangelsk.su
+armenia.su
+ashgabad.su
+azerbaijan.su
+balashov.su
+bashkiria.su
+bryansk.su
+bukhara.su
+chimkent.su
+dagestan.su
+east-kazakhstan.su
+exnet.su
+georgia.su
+grozny.su
+ivanovo.su
+jambyl.su
+kalmykia.su
+kaluga.su
+karacol.su
+karaganda.su
+karelia.su
+khakassia.su
+krasnodar.su
+kurgan.su
+kustanai.su
+lenug.su
+mangyshlak.su
+mordovia.su
+msk.su
+murmansk.su
+nalchik.su
+navoi.su
+north-kazakhstan.su
+nov.su
+obninsk.su
+penza.su
+pokrovsk.su
+sochi.su
+spb.su
+tashkent.su
+termez.su
+togliatti.su
+troitsk.su
+tselinograd.su
+tula.su
+tuva.su
+vladikavkaz.su
+vladimir.su
+vologda.su
+
+// Fancy Bits, LLC : http://getchannels.com
+// Submitted by Aman Gupta <aman@getchannels.com>
+channelsdvr.net
+u.channelsdvr.net
+
+// Fastly Inc. : http://www.fastly.com/
+// Submitted by Fastly Security <security@fastly.com>
+edgecompute.app
+fastly-terrarium.com
+fastlylb.net
+map.fastlylb.net
+freetls.fastly.net
+map.fastly.net
+a.prod.fastly.net
+global.prod.fastly.net
+a.ssl.fastly.net
+b.ssl.fastly.net
+global.ssl.fastly.net
+
+// FASTVPS EESTI OU : https://fastvps.ru/
+// Submitted by Likhachev Vasiliy <lihachev@fastvps.ru>
+fastvps-server.com
+fastvps.host
+myfast.host
+fastvps.site
+myfast.space
+
+// Fedora : https://fedoraproject.org/
+// submitted by Patrick Uiterwijk <puiterwijk@fedoraproject.org>
+fedorainfracloud.org
+fedorapeople.org
+cloud.fedoraproject.org
+app.os.fedoraproject.org
+app.os.stg.fedoraproject.org
+
+// FearWorks Media Ltd. : https://fearworksmedia.co.uk
+// submitted by Keith Fairley <domains@fearworksmedia.co.uk>
+couk.me
+ukco.me
+conn.uk
+copro.uk
+hosp.uk
+
+// Fermax : https://fermax.com/
+// submitted by Koen Van Isterdael <k.vanisterdael@fermax.be>
+mydobiss.com
+
+// FH Muenster : https://www.fh-muenster.de
+// Submitted by Robin Naundorf <r.naundorf@fh-muenster.de>
+fh-muenster.io
+
+// Filegear Inc. : https://www.filegear.com
+// Submitted by Jason Zhu <jason@owtware.com>
+filegear.me
+filegear-au.me
+filegear-de.me
+filegear-gb.me
+filegear-ie.me
+filegear-jp.me
+filegear-sg.me
+
+// Firebase, Inc.
+// Submitted by Chris Raynor <chris@firebase.com>
+firebaseapp.com
+
+// Firewebkit : https://www.firewebkit.com
+// Submitted by Majid Qureshi <mqureshi@amrayn.com>
+fireweb.app
+
+// FLAP : https://www.flap.cloud
+// Submitted by Louis Chemineau <louis@chmn.me>
+flap.id
+
+// fly.io: https://fly.io
+// Submitted by Kurt Mackey <kurt@fly.io>
+fly.dev
+edgeapp.net
+shw.io
+
+// Flynn : https://flynn.io
+// Submitted by Jonathan Rudenberg <jonathan@flynn.io>
+flynnhosting.net
+
+// Forgerock : https://www.forgerock.com
+// Submitted by Roderick Parr <roderick.parr@forgerock.com>
+forgeblocks.com
+id.forgerock.io
+
+// Framer : https://www.framer.com
+// Submitted by Koen Rouwhorst <koenrh@framer.com>
+framer.app
+framercanvas.com
+
+// Frusky MEDIA&PR : https://www.frusky.de
+// Submitted by Victor Pupynin <hallo@frusky.de>
+*.frusky.de
+
+// RavPage : https://www.ravpage.co.il
+// Submitted by Roni Horowitz <roni@responder.co.il>
+ravpage.co.il
+
+// Frederik Braun https://frederik-braun.com
+// Submitted by Frederik Braun <fb@frederik-braun.com>
+0e.vc
+
+// Freebox : http://www.freebox.fr
+// Submitted by Romain Fliedel <rfliedel@freebox.fr>
+freebox-os.com
+freeboxos.com
+fbx-os.fr
+fbxos.fr
+freebox-os.fr
+freeboxos.fr
+
+// freedesktop.org : https://www.freedesktop.org
+// Submitted by Daniel Stone <daniel@fooishbar.org>
+freedesktop.org
+
+// freemyip.com : https://freemyip.com
+// Submitted by Cadence <contact@freemyip.com>
+freemyip.com
+
+// FunkFeuer - Verein zur Förderung freier Netze : https://www.funkfeuer.at
+// Submitted by Daniel A. Maierhofer <vorstand@funkfeuer.at>
+wien.funkfeuer.at
+
+// Futureweb OG : http://www.futureweb.at
+// Submitted by Andreas Schnederle-Wagner <schnederle@futureweb.at>
+*.futurecms.at
+*.ex.futurecms.at
+*.in.futurecms.at
+futurehosting.at
+futuremailing.at
+*.ex.ortsinfo.at
+*.kunden.ortsinfo.at
+*.statics.cloud
+
+// GDS : https://www.gov.uk/service-manual/operations/operating-servicegovuk-subdomains
+// Submitted by David Illsley <david.illsley@digital.cabinet-office.gov.uk>
+service.gov.uk
+
+// Gehirn Inc. : https://www.gehirn.co.jp/
+// Submitted by Kohei YOSHIDA <tech@gehirn.co.jp>
+gehirn.ne.jp
+usercontent.jp
+
+// Gentlent, Inc. : https://www.gentlent.com
+// Submitted by Tom Klein <tom@gentlent.com>
+gentapps.com
+gentlentapis.com
+lab.ms
+cdn-edges.net
+
+// Ghost Foundation : https://ghost.org
+// Submitted by Matt Hanley <security@ghost.org>
+ghost.io
+
+// GignoSystemJapan: http://gsj.bz
+// Submitted by GignoSystemJapan <kakutou-ec@gsj.bz>
+gsj.bz
+
+// GitHub, Inc.
+// Submitted by Patrick Toomey <security@github.com>
+githubusercontent.com
+githubpreview.dev
+github.io
+
+// GitLab, Inc.
+// Submitted by Alex Hanselka <alex@gitlab.com>
+gitlab.io
+
+// Gitplac.si - https://gitplac.si
+// Submitted by Aljaž Starc <me@aljaxus.eu>
+gitapp.si
+gitpage.si
+
+// Glitch, Inc : https://glitch.com
+// Submitted by Mads Hartmann <mads@glitch.com>
+glitch.me
+
+// Global NOG Alliance : https://nogalliance.org/
+// Submitted by Sander Steffann <sander@nogalliance.org>
+nog.community
+
+// Globe Hosting SRL : https://www.globehosting.com/
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+co.ro
+shop.ro
+
+// GMO Pepabo, Inc. : https://pepabo.com/
+// Submitted by dojineko <admin@pepabo.com>
+lolipop.io
+
+// GOV.UK Platform as a Service : https://www.cloud.service.gov.uk/
+// Submitted by Tom Whitwell <gov-uk-paas-support@digital.cabinet-office.gov.uk>
+cloudapps.digital
+london.cloudapps.digital
+
+// GOV.UK Pay : https://www.payments.service.gov.uk/
+// Submitted by Richard Baker <richard.baker@digital.cabinet-office.gov.uk>
+pymnt.uk
+
+// UKHomeOffice : https://www.gov.uk/government/organisations/home-office
+// Submitted by Jon Shanks <jon.shanks@digital.homeoffice.gov.uk>
+homeoffice.gov.uk
+
+// GlobeHosting, Inc.
+// Submitted by Zoltan Egresi <egresi@globehosting.com>
+ro.im
+
+// GoIP DNS Services : http://www.goip.de
+// Submitted by Christian Poulter <milchstrasse@goip.de>
+goip.de
+
+// Google, Inc.
+// Submitted by Eduardo Vela <evn@google.com>
+run.app
+a.run.app
+web.app
+*.0emm.com
+appspot.com
+*.r.appspot.com
+codespot.com
+googleapis.com
+googlecode.com
+pagespeedmobilizer.com
+publishproxy.com
+withgoogle.com
+withyoutube.com
+*.gateway.dev
+cloud.goog
+translate.goog
+cloudfunctions.net
+blogspot.ae
+blogspot.al
+blogspot.am
+blogspot.ba
+blogspot.be
+blogspot.bg
+blogspot.bj
+blogspot.ca
+blogspot.cf
+blogspot.ch
+blogspot.cl
+blogspot.co.at
+blogspot.co.id
+blogspot.co.il
+blogspot.co.ke
+blogspot.co.nz
+blogspot.co.uk
+blogspot.co.za
+blogspot.com
+blogspot.com.ar
+blogspot.com.au
+blogspot.com.br
+blogspot.com.by
+blogspot.com.co
+blogspot.com.cy
+blogspot.com.ee
+blogspot.com.eg
+blogspot.com.es
+blogspot.com.mt
+blogspot.com.ng
+blogspot.com.tr
+blogspot.com.uy
+blogspot.cv
+blogspot.cz
+blogspot.de
+blogspot.dk
+blogspot.fi
+blogspot.fr
+blogspot.gr
+blogspot.hk
+blogspot.hr
+blogspot.hu
+blogspot.ie
+blogspot.in
+blogspot.is
+blogspot.it
+blogspot.jp
+blogspot.kr
+blogspot.li
+blogspot.lt
+blogspot.lu
+blogspot.md
+blogspot.mk
+blogspot.mr
+blogspot.mx
+blogspot.my
+blogspot.nl
+blogspot.no
+blogspot.pe
+blogspot.pt
+blogspot.qa
+blogspot.re
+blogspot.ro
+blogspot.rs
+blogspot.ru
+blogspot.se
+blogspot.sg
+blogspot.si
+blogspot.sk
+blogspot.sn
+blogspot.td
+blogspot.tw
+blogspot.ug
+blogspot.vn
+
+// Goupile : https://goupile.fr
+// Submitted by Niels Martignene <hello@goupile.fr>
+goupile.fr
+
+// Group 53, LLC : https://www.group53.com
+// Submitted by Tyler Todd <noc@nova53.net>
+awsmppl.com
+
+// GünstigBestellen : https://günstigbestellen.de
+// Submitted by Furkan Akkoc <info@hendelzon.de>
+xn--gnstigbestellen-zvb.de
+günstigbestellen.de
+xn--gnstigliefern-wob.de
+günstigliefern.de
+
+// Hakaran group: http://hakaran.cz
+// Submited by Arseniy Sokolov <security@hakaran.cz>
+fin.ci
+free.hr
+caa.li
+ua.rs
+conf.se
+
+// Handshake : https://handshake.org
+// Submitted by Mike Damm <md@md.vc>
+hs.zone
+hs.run
+
+// Hashbang : https://hashbang.sh
+hashbang.sh
+
+// Hasura : https://hasura.io
+// Submitted by Shahidh K Muhammed <shahidh@hasura.io>
+hasura.app
+hasura-app.io
+
+// Hepforge : https://www.hepforge.org
+// Submitted by David Grellscheid <admin@hepforge.org>
+hepforge.org
+
+// Heroku : https://www.heroku.com/
+// Submitted by Tom Maher <tmaher@heroku.com>
+herokuapp.com
+herokussl.com
+
+// Hibernating Rhinos
+// Submitted by Oren Eini <oren@ravendb.net>
+myravendb.com
+ravendb.community
+ravendb.me
+development.run
+ravendb.run
+
+// Hong Kong Productivity Council: https://www.hkpc.org/
+// Submitted by SECaaS Team <summchan@hkpc.org>
+secaas.hk
+
+// HOSTBIP REGISTRY : https://www.hostbip.com/
+// Submitted by Atanunu Igbunuroghene <publicsuffixlist@hostbip.com>
+orx.biz
+biz.gl
+col.ng
+firm.ng
+gen.ng
+ltd.ng
+ngo.ng
+edu.scot
+sch.so
+org.yt
+
+// HostyHosting (hostyhosting.com)
+hostyhosting.io
+
+// Häkkinen.fi
+// Submitted by Eero Häkkinen <Eero+psl@Häkkinen.fi>
+xn--hkkinen-5wa.fi
+häkkinen.fi
+
+// Ici la Lune : http://www.icilalune.com/
+// Submitted by Simon Morvan <simon@icilalune.com>
+*.moonscale.io
+moonscale.net
+
+// iki.fi
+// Submitted by Hannu Aronsson <haa@iki.fi>
+iki.fi
+
+// Impertrix Solutions : <https://impertrixcdn.com>
+// Submitted by Zhixiang Zhao <csuite@impertrix.com>
+impertrixcdn.com
+impertrix.com
+
+// Incsub, LLC: https://incsub.com/
+// Submitted by Aaron Edwards <sysadmins@incsub.com>
+smushcdn.com
+wphostedmail.com
+wpmucdn.com
+tempurl.host
+wpmudev.host
+
+// Individual Network Berlin e.V. : https://www.in-berlin.de/
+// Submitted by Christian Seitz <chris@in-berlin.de>
+dyn-berlin.de
+in-berlin.de
+in-brb.de
+in-butter.de
+in-dsl.de
+in-dsl.net
+in-dsl.org
+in-vpn.de
+in-vpn.net
+in-vpn.org
+
+// info.at : http://www.info.at/
+biz.at
+info.at
+
+// info.cx : http://info.cx
+// Submitted by Jacob Slater <whois@igloo.to>
+info.cx
+
+// Interlegis : http://www.interlegis.leg.br
+// Submitted by Gabriel Ferreira <registrobr@interlegis.leg.br>
+ac.leg.br
+al.leg.br
+am.leg.br
+ap.leg.br
+ba.leg.br
+ce.leg.br
+df.leg.br
+es.leg.br
+go.leg.br
+ma.leg.br
+mg.leg.br
+ms.leg.br
+mt.leg.br
+pa.leg.br
+pb.leg.br
+pe.leg.br
+pi.leg.br
+pr.leg.br
+rj.leg.br
+rn.leg.br
+ro.leg.br
+rr.leg.br
+rs.leg.br
+sc.leg.br
+se.leg.br
+sp.leg.br
+to.leg.br
+
+// intermetrics GmbH : https://pixolino.com/
+// Submitted by Wolfgang Schwarz <admin@intermetrics.de>
+pixolino.com
+
+// Internet-Pro, LLP: https://netangels.ru/
+// Submited by Vasiliy Sheredeko <piphon@gmail.com>
+na4u.ru
+
+// iopsys software solutions AB : https://iopsys.eu/
+// Submitted by Roman Azarenko <roman.azarenko@iopsys.eu>
+iopsys.se
+
+// IPiFony Systems, Inc. : https://www.ipifony.com/
+// Submitted by Matthew Hardeman <mhardeman@ipifony.com>
+ipifony.net
+
+// IServ GmbH : https://iserv.eu
+// Submitted by Kim-Alexander Brodowski <info@iserv.eu>
+mein-iserv.de
+schulserver.de
+test-iserv.de
+iserv.dev
+
+// I-O DATA DEVICE, INC. : http://www.iodata.com/
+// Submitted by Yuji Minagawa <domains-admin@iodata.jp>
+iobb.net
+
+// Jelastic, Inc. : https://jelastic.com/
+// Submited by Ihor Kolodyuk <ik@jelastic.com>
+mel.cloudlets.com.au
+cloud.interhostsolutions.be
+users.scale.virtualcloud.com.br
+mycloud.by
+alp1.ae.flow.ch
+appengine.flow.ch
+es-1.axarnet.cloud
+diadem.cloud
+vip.jelastic.cloud
+jele.cloud
+it1.eur.aruba.jenv-aruba.cloud
+it1.jenv-aruba.cloud
+keliweb.cloud
+cs.keliweb.cloud
+oxa.cloud
+tn.oxa.cloud
+uk.oxa.cloud
+primetel.cloud
+uk.primetel.cloud
+ca.reclaim.cloud
+uk.reclaim.cloud
+us.reclaim.cloud
+ch.trendhosting.cloud
+de.trendhosting.cloud
+jele.club
+amscompute.com
+clicketcloud.com
+dopaas.com
+hidora.com
+paas.hosted-by-previder.com
+rag-cloud.hosteur.com
+rag-cloud-ch.hosteur.com
+jcloud.ik-server.com
+jcloud-ver-jpc.ik-server.com
+demo.jelastic.com
+kilatiron.com
+paas.massivegrid.com
+jed.wafaicloud.com
+lon.wafaicloud.com
+ryd.wafaicloud.com
+j.scaleforce.com.cy
+jelastic.dogado.eu
+fi.cloudplatform.fi
+demo.datacenter.fi
+paas.datacenter.fi
+jele.host
+mircloud.host
+paas.beebyte.io
+sekd1.beebyteapp.io
+jele.io
+cloud-fr1.unispace.io
+jc.neen.it
+cloud.jelastic.open.tim.it
+jcloud.kz
+upaas.kazteleport.kz
+cloudjiffy.net
+fra1-de.cloudjiffy.net
+west1-us.cloudjiffy.net
+jls-sto1.elastx.net
+jls-sto2.elastx.net
+jls-sto3.elastx.net
+faststacks.net
+fr-1.paas.massivegrid.net
+lon-1.paas.massivegrid.net
+lon-2.paas.massivegrid.net
+ny-1.paas.massivegrid.net
+ny-2.paas.massivegrid.net
+sg-1.paas.massivegrid.net
+jelastic.saveincloud.net
+nordeste-idc.saveincloud.net
+j.scaleforce.net
+jelastic.tsukaeru.net
+sdscloud.pl
+unicloud.pl
+mircloud.ru
+jelastic.regruhosting.ru
+enscaled.sg
+jele.site
+jelastic.team
+orangecloud.tn
+j.layershift.co.uk
+phx.enscaled.us
+mircloud.us
+
+// Jino : https://www.jino.ru
+// Submitted by Sergey Ulyashin <ulyashin@jino.ru>
+myjino.ru
+*.hosting.myjino.ru
+*.landing.myjino.ru
+*.spectrum.myjino.ru
+*.vps.myjino.ru
+
+// Jotelulu S.L. : https://jotelulu.com
+// Submitted by Daniel Fariña <ingenieria@jotelulu.com>
+jotelulu.cloud
+
+// Joyent : https://www.joyent.com/
+// Submitted by Brian Bennett <brian.bennett@joyent.com>
+*.triton.zone
+*.cns.joyent.com
+
+// JS.ORG : http://dns.js.org
+// Submitted by Stefan Keim <admin@js.org>
+js.org
+
+// KaasHosting : http://www.kaashosting.nl/
+// Submitted by Wouter Bakker <hostmaster@kaashosting.nl>
+kaas.gg
+khplay.nl
+
+// Keyweb AG : https://www.keyweb.de
+// Submitted by Martin Dannehl <postmaster@keymachine.de>
+keymachine.de
+
+// KingHost : https://king.host
+// Submitted by Felipe Keller Braz <felipebraz@kinghost.com.br>
+kinghost.net
+uni5.net
+
+// KnightPoint Systems, LLC : http://www.knightpoint.com/
+// Submitted by Roy Keene <rkeene@knightpoint.com>
+knightpoint.systems
+
+// KUROKU LTD : https://kuroku.ltd/
+// Submitted by DisposaBoy <security@oya.to>
+oya.to
+
+// Katholieke Universiteit Leuven: https://www.kuleuven.be
+// Submitted by Abuse KU Leuven <abuse@kuleuven.be>
+kuleuven.cloud
+ezproxy.kuleuven.be
+
+// .KRD : http://nic.krd/data/krd/Registration%20Policy.pdf
+co.krd
+edu.krd
+
+// Krellian Ltd. : https://krellian.com
+// Submitted by Ben Francis <ben@krellian.com>
+krellian.net
+webthings.io
+
+// LCube - Professional hosting e.K. : https://www.lcube-webhosting.de
+// Submitted by Lars Laehn <info@lcube.de>
+git-repos.de
+lcube-server.de
+svn-repos.de
+
+// Leadpages : https://www.leadpages.net
+// Submitted by Greg Dallavalle <domains@leadpages.net>
+leadpages.co
+lpages.co
+lpusercontent.com
+
+// Lelux.fi : https://lelux.fi/
+// Submitted by Lelux Admin <publisuffix@lelux.site>
+lelux.site
+
+// Lifetime Hosting : https://Lifetime.Hosting/
+// Submitted by Mike Fillator <support@lifetime.hosting>
+co.business
+co.education
+co.events
+co.financial
+co.network
+co.place
+co.technology
+
+// Lightmaker Property Manager, Inc. : https://app.lmpm.com/
+// Submitted by Greg Holland <greg.holland@lmpm.com>
+app.lmpm.com
+
+// linkyard ldt: https://www.linkyard.ch/
+// Submitted by Mario Siegenthaler <mario.siegenthaler@linkyard.ch>
+linkyard.cloud
+linkyard-cloud.ch
+
+// Linode : https://linode.com
+// Submitted by <security@linode.com>
+members.linode.com
+*.nodebalancer.linode.com
+*.linodeobjects.com
+
+// LiquidNet Ltd : http://www.liquidnetlimited.com/
+// Submitted by Victor Velchev <admin@liquidnetlimited.com>
+we.bs
+
+// localzone.xyz
+// Submitted by Kenny Niehage <hello@yahe.sh>
+localzone.xyz
+
+// Log'in Line : https://www.loginline.com/
+// Submitted by Rémi Mach <remi.mach@loginline.com>
+loginline.app
+loginline.dev
+loginline.io
+loginline.services
+loginline.site
+
+// Lõhmus Family, The
+// Submitted by Heiki Lõhmus <hostmaster at lohmus dot me>
+lohmus.me
+
+// LubMAN UMCS Sp. z o.o : https://lubman.pl/
+// Submitted by Ireneusz Maliszewski <ireneusz.maliszewski@lubman.pl>
+krasnik.pl
+leczna.pl
+lubartow.pl
+lublin.pl
+poniatowa.pl
+swidnik.pl
+
+// Lug.org.uk : https://lug.org.uk
+// Submitted by Jon Spriggs <admin@lug.org.uk>
+glug.org.uk
+lug.org.uk
+lugs.org.uk
+
+// Lukanet Ltd : https://lukanet.com
+// Submitted by Anton Avramov <register@lukanet.com>
+barsy.bg
+barsy.co.uk
+barsyonline.co.uk
+barsycenter.com
+barsyonline.com
+barsy.club
+barsy.de
+barsy.eu
+barsy.in
+barsy.info
+barsy.io
+barsy.me
+barsy.menu
+barsy.mobi
+barsy.net
+barsy.online
+barsy.org
+barsy.pro
+barsy.pub
+barsy.shop
+barsy.site
+barsy.support
+barsy.uk
+
+// Magento Commerce
+// Submitted by Damien Tournoud <dtournoud@magento.cloud>
+*.magentosite.cloud
+
+// May First - People Link : https://mayfirst.org/
+// Submitted by Jamie McClelland <info@mayfirst.org>
+mayfirst.info
+mayfirst.org
+
+// Mail.Ru Group : https://hb.cldmail.ru
+// Submitted by Ilya Zaretskiy <zaretskiy@corp.mail.ru>
+hb.cldmail.ru
+
+// Mail Transfer Platform : https://www.neupeer.com
+// Submitted by Li Hui <lihui@neupeer.com>
+cn.vu
+
+// Maze Play: https://www.mazeplay.com
+// Submitted by Adam Humpherys <adam@mws.dev>
+mazeplay.com
+
+// mcpe.me : https://mcpe.me
+// Submitted by Noa Heyl <hi@noa.dev>
+mcpe.me
+
+// McHost : https://mchost.ru
+// Submitted by Evgeniy Subbotin <e.subbotin@mchost.ru>
+mcdir.me
+mcdir.ru
+mcpre.ru
+vps.mcdir.ru
+
+// Mediatech : https://mediatech.by
+// Submitted by Evgeniy Kozhuhovskiy <ugenk@mediatech.by>
+mediatech.by
+mediatech.dev
+
+// Medicom Health : https://medicomhealth.com
+// Submitted by Michael Olson <molson@medicomhealth.com>
+hra.health
+
+// Memset hosting : https://www.memset.com
+// Submitted by Tom Whitwell <domains@memset.com>
+miniserver.com
+memset.net
+
+// MetaCentrum, CESNET z.s.p.o. : https://www.metacentrum.cz/en/
+// Submitted by Zdeněk Šustr <zdenek.sustr@cesnet.cz>
+*.cloud.metacentrum.cz
+custom.metacentrum.cz
+
+// MetaCentrum, CESNET z.s.p.o. : https://www.metacentrum.cz/en/
+// Submitted by Radim Janča <janca@cesnet.cz>
+flt.cloud.muni.cz
+usr.cloud.muni.cz
+
+// Meteor Development Group : https://www.meteor.com/hosting
+// Submitted by Pierre Carrier <pierre@meteor.com>
+meteorapp.com
+eu.meteorapp.com
+
+// Michau Enterprises Limited : http://www.co.pl/
+co.pl
+
+// Microsoft Corporation : http://microsoft.com
+// Submitted by Mitch Webster <miwebst@microsoft.com>
+*.azurecontainer.io
+azurewebsites.net
+azure-mobile.net
+cloudapp.net
+azurestaticapps.net
+centralus.azurestaticapps.net
+eastasia.azurestaticapps.net
+eastus2.azurestaticapps.net
+westeurope.azurestaticapps.net
+westus2.azurestaticapps.net
+
+// minion.systems : http://minion.systems
+// Submitted by Robert Böttinger <r@minion.systems>
+csx.cc
+
+// Mintere : https://mintere.com/
+// Submitted by Ben Aubin <security@mintere.com>
+mintere.site
+
+// MobileEducation, LLC : https://joinforte.com
+// Submitted by Grayson Martin <grayson.martin@mobileeducation.us>
+forte.id
+
+// Mozilla Corporation : https://mozilla.com
+// Submitted by Ben Francis <bfrancis@mozilla.com>
+mozilla-iot.org
+
+// Mozilla Foundation : https://mozilla.org/
+// Submitted by glob <glob@mozilla.com>
+bmoattachments.org
+
+// MSK-IX : https://www.msk-ix.ru/
+// Submitted by Khannanov Roman <r.khannanov@msk-ix.ru>
+net.ru
+org.ru
+pp.ru
+
+// Mythic Beasts : https://www.mythic-beasts.com
+// Submitted by Paul Cammish <kelduum@mythic-beasts.com>
+hostedpi.com
+customer.mythic-beasts.com
+caracal.mythic-beasts.com
+fentiger.mythic-beasts.com
+lynx.mythic-beasts.com
+ocelot.mythic-beasts.com
+oncilla.mythic-beasts.com
+onza.mythic-beasts.com
+sphinx.mythic-beasts.com
+vs.mythic-beasts.com
+x.mythic-beasts.com
+yali.mythic-beasts.com
+cust.retrosnub.co.uk
+
+// Nabu Casa : https://www.nabucasa.com
+// Submitted by Paulus Schoutsen <infra@nabucasa.com>
+ui.nabu.casa
+
+// Names.of.London : https://names.of.london/
+// Submitted by James Stevens <registry[at]names.of.london> or <publiclist[at]jrcs.net>
+pony.club
+of.fashion
+in.london
+of.london
+from.marketing
+with.marketing
+for.men
+repair.men
+and.mom
+for.mom
+for.one
+under.one
+for.sale
+that.win
+from.work
+to.work
+
+// NCTU.ME : https://nctu.me/
+// Submitted by Tocknicsu <admin@nctu.me>
+nctu.me
+
+// Netlify : https://www.netlify.com
+// Submitted by Jessica Parsons <jessica@netlify.com>
+netlify.app
+
+// Neustar Inc.
+// Submitted by Trung Tran <Trung.Tran@neustar.biz>
+4u.com
+
+// ngrok : https://ngrok.com/
+// Submitted by Alan Shreve <alan@ngrok.com>
+ngrok.io
+
+// Nimbus Hosting Ltd. : https://www.nimbushosting.co.uk/
+// Submitted by Nicholas Ford <nick@nimbushosting.co.uk>
+nh-serv.co.uk
+
+// NFSN, Inc. : https://www.NearlyFreeSpeech.NET/
+// Submitted by Jeff Wheelhouse <support@nearlyfreespeech.net>
+nfshost.com
+
+// Noop : https://noop.app
+// Submitted by Nathaniel Schweinberg <noop@rearc.io>
+*.developer.app
+noop.app
+
+// Northflank Ltd. : https://northflank.com/
+// Submitted by Marco Suter <marco@northflank.com>
+*.northflank.app
+*.code.run
+
+// Noticeable : https://noticeable.io
+// Submitted by Laurent Pellegrino <security@noticeable.io>
+noticeable.news
+
+// Now-DNS : https://now-dns.com
+// Submitted by Steve Russell <steve@now-dns.com>
+dnsking.ch
+mypi.co
+n4t.co
+001www.com
+ddnslive.com
+myiphost.com
+forumz.info
+16-b.it
+32-b.it
+64-b.it
+soundcast.me
+tcp4.me
+dnsup.net
+hicam.net
+now-dns.net
+ownip.net
+vpndns.net
+dynserv.org
+now-dns.org
+x443.pw
+now-dns.top
+ntdll.top
+freeddns.us
+crafting.xyz
+zapto.xyz
+
+// nsupdate.info : https://www.nsupdate.info/
+// Submitted by Thomas Waldmann <info@nsupdate.info>
+nsupdate.info
+nerdpol.ovh
+
+// No-IP.com : https://noip.com/
+// Submitted by Deven Reza <publicsuffixlist@noip.com>
+blogsyte.com
+brasilia.me
+cable-modem.org
+ciscofreak.com
+collegefan.org
+couchpotatofries.org
+damnserver.com
+ddns.me
+ditchyourip.com
+dnsfor.me
+dnsiskinky.com
+dvrcam.info
+dynns.com
+eating-organic.net
+fantasyleague.cc
+geekgalaxy.com
+golffan.us
+health-carereform.com
+homesecuritymac.com
+homesecuritypc.com
+hopto.me
+ilovecollege.info
+loginto.me
+mlbfan.org
+mmafan.biz
+myactivedirectory.com
+mydissent.net
+myeffect.net
+mymediapc.net
+mypsx.net
+mysecuritycamera.com
+mysecuritycamera.net
+mysecuritycamera.org
+net-freaks.com
+nflfan.org
+nhlfan.net
+no-ip.ca
+no-ip.co.uk
+no-ip.net
+noip.us
+onthewifi.com
+pgafan.net
+point2this.com
+pointto.us
+privatizehealthinsurance.net
+quicksytes.com
+read-books.org
+securitytactics.com
+serveexchange.com
+servehumour.com
+servep2p.com
+servesarcasm.com
+stufftoread.com
+ufcfan.org
+unusualperson.com
+workisboring.com
+3utilities.com
+bounceme.net
+ddns.net
+ddnsking.com
+gotdns.ch
+hopto.org
+myftp.biz
+myftp.org
+myvnc.com
+no-ip.biz
+no-ip.info
+no-ip.org
+noip.me
+redirectme.net
+servebeer.com
+serveblog.net
+servecounterstrike.com
+serveftp.com
+servegame.com
+servehalflife.com
+servehttp.com
+serveirc.com
+serveminecraft.net
+servemp3.com
+servepics.com
+servequake.com
+sytes.net
+webhop.me
+zapto.org
+
+// NodeArt : https://nodeart.io
+// Submitted by Konstantin Nosov <Nosov@nodeart.io>
+stage.nodeart.io
+
+// Nodum B.V. : https://nodum.io/
+// Submitted by Wietse Wind <hello+publicsuffixlist@nodum.io>
+nodum.co
+nodum.io
+
+// Nucleos Inc. : https://nucleos.com
+// Submitted by Piotr Zduniak <piotr@nucleos.com>
+pcloud.host
+
+// NYC.mn : http://www.information.nyc.mn
+// Submitted by Matthew Brown <mattbrown@nyc.mn>
+nyc.mn
+
+// Observable, Inc. : https://observablehq.com
+// Submitted by Mike Bostock <dns@observablehq.com>
+static.observableusercontent.com
+
+// Octopodal Solutions, LLC. : https://ulterius.io/
+// Submitted by Andrew Sampson <andrew@ulterius.io>
+cya.gg
+
+// OMG.LOL : <https://omg.lol>
+// Submitted by Adam Newbold <adam@omg.lol>
+omg.lol
+
+// Omnibond Systems, LLC. : https://www.omnibond.com
+// Submitted by Cole Estep <cole@omnibond.com>
+cloudycluster.net
+
+// OmniWe Limited: https://omniwe.com
+// Submitted by Vicary Archangel <vicary@omniwe.com>
+omniwe.site
+
+// One Fold Media : http://www.onefoldmedia.com/
+// Submitted by Eddie Jones <eddie@onefoldmedia.com>
+nid.io
+
+// Open Social : https://www.getopensocial.com/
+// Submitted by Alexander Varwijk <security@getopensocial.com>
+opensocial.site
+
+// OpenCraft GmbH : http://opencraft.com/
+// Submitted by Sven Marnach <sven@opencraft.com>
+opencraft.hosting
+
+// OpenResearch GmbH: https://openresearch.com/
+// Submitted by Philipp Schmid <ops@openresearch.com>
+orsites.com
+
+// Opera Software, A.S.A.
+// Submitted by Yngve Pettersen <yngve@opera.com>
+operaunite.com
+
+// Oursky Limited : https://authgear.com/, https://skygear.io/
+// Submited by Authgear Team <hello@authgear.com>, Skygear Developer <hello@skygear.io>
+authgear-staging.com
+authgearapps.com
+skygearapp.com
+
+// OutSystems
+// Submitted by Duarte Santos <domain-admin@outsystemscloud.com>
+outsystemscloud.com
+
+// OVHcloud: https://ovhcloud.com
+// Submitted by Vincent Cassé <vincent.casse@ovhcloud.com>
+*.webpaas.ovh.net
+*.hosting.ovh.net
+
+// OwnProvider GmbH: http://www.ownprovider.com
+// Submitted by Jan Moennich <jan.moennich@ownprovider.com>
+ownprovider.com
+own.pm
+
+// OwO : https://whats-th.is/
+// Submitted by Dean Sheather <dean@deansheather.com>
+*.owo.codes
+
+// OX : http://www.ox.rs
+// Submitted by Adam Grand <webmaster@mail.ox.rs>
+ox.rs
+
+// oy.lc
+// Submitted by Charly Coste <changaco@changaco.oy.lc>
+oy.lc
+
+// Pagefog : https://pagefog.com/
+// Submitted by Derek Myers <derek@pagefog.com>
+pgfog.com
+
+// Pagefront : https://www.pagefronthq.com/
+// Submitted by Jason Kriss <jason@pagefronthq.com>
+pagefrontapp.com
+
+// PageXL : https://pagexl.com
+// Submitted by Yann Guichard <yann@pagexl.com>
+pagexl.com
+
+// Paywhirl, Inc : https://paywhirl.com/
+// Submitted by Daniel Netzer <dan@paywhirl.com>
+*.paywhirl.com
+
+// pcarrier.ca Software Inc: https://pcarrier.ca/
+// Submitted by Pierre Carrier <pc@rrier.ca>
+bar0.net
+bar1.net
+bar2.net
+rdv.to
+
+// .pl domains (grandfathered)
+art.pl
+gliwice.pl
+krakow.pl
+poznan.pl
+wroc.pl
+zakopane.pl
+
+// Pantheon Systems, Inc. : https://pantheon.io/
+// Submitted by Gary Dylina <gary@pantheon.io>
+pantheonsite.io
+gotpantheon.com
+
+// Peplink | Pepwave : http://peplink.com/
+// Submitted by Steve Leung <steveleung@peplink.com>
+mypep.link
+
+// Perspecta : https://perspecta.com/
+// Submitted by Kenneth Van Alstyne <kvanalstyne@perspecta.com>
+perspecta.cloud
+
+// PE Ulyanov Kirill Sergeevich : https://airy.host
+// Submitted by Kirill Ulyanov <k.ulyanov@airy.host>
+lk3.ru
+
+// Planet-Work : https://www.planet-work.com/
+// Submitted by Frédéric VANNIÈRE <f.vanniere@planet-work.com>
+on-web.fr
+
+// Platform.sh : https://platform.sh
+// Submitted by Nikola Kotur <nikola@platform.sh>
+bc.platform.sh
+ent.platform.sh
+eu.platform.sh
+us.platform.sh
+*.platformsh.site
+*.tst.site
+
+// Platter: https://platter.dev
+// Submitted by Patrick Flor <patrick@platter.dev>
+platter-app.com
+platter-app.dev
+platterp.us
+
+// Plesk : https://www.plesk.com/
+// Submitted by Anton Akhtyamov <program-managers@plesk.com>
+pdns.page
+plesk.page
+pleskns.com
+
+// Port53 : https://port53.io/
+// Submitted by Maximilian Schieder <maxi@zeug.co>
+dyn53.io
+
+// Positive Codes Technology Company : http://co.bn/faq.html
+// Submitted by Zulfais <pc@co.bn>
+co.bn
+
+// Postman, Inc : https://postman.com
+// Submitted by Rahul Dhawan <security@postman.com>
+postman-echo.com
+pstmn.io
+mock.pstmn.io
+httpbin.org
+
+// prgmr.com : https://prgmr.com/
+// Submitted by Sarah Newman <owner@prgmr.com>
+xen.prgmr.com
+
+// priv.at : http://www.nic.priv.at/
+// Submitted by registry <lendl@nic.at>
+priv.at
+
+// privacytools.io : https://www.privacytools.io/
+// Submitted by Jonah Aragon <jonah@privacytools.io>
+prvcy.page
+
+// Protocol Labs : https://protocol.ai/
+// Submitted by Michael Burns <noc@protocol.ai>
+*.dweb.link
+
+// Protonet GmbH : http://protonet.io
+// Submitted by Martin Meier <admin@protonet.io>
+protonet.io
+
+// Publication Presse Communication SARL : https://ppcom.fr
+// Submitted by Yaacov Akiba Slama <admin@chirurgiens-dentistes-en-france.fr>
+chirurgiens-dentistes-en-france.fr
+byen.site
+
+// pubtls.org: https://www.pubtls.org
+// Submitted by Kor Nielsen <kor@pubtls.org>
+pubtls.org
+
+// PythonAnywhere LLP: https://www.pythonanywhere.com
+// Submitted by Giles Thomas <giles@pythonanywhere.com>
+pythonanywhere.com
+eu.pythonanywhere.com
+
+// QOTO, Org.
+// Submitted by Jeffrey Phillips Freeman <jeffrey.freeman@qoto.org>
+qoto.io
+
+// Qualifio : https://qualifio.com/
+// Submitted by Xavier De Cock <xdecock@gmail.com>
+qualifioapp.com
+
+// QuickBackend: https://www.quickbackend.com
+// Submitted by Dani Biro <dani@pymet.com>
+qbuser.com
+
+// Rad Web Hosting: https://radwebhosting.com
+// Submitted by Scott Claeys <s.claeys@radwebhosting.com>
+cloudsite.builders
+
+// Redstar Consultants : https://www.redstarconsultants.com/
+// Submitted by Jons Slemmer <jons@redstarconsultants.com>
+instantcloud.cn
+
+// Russian Academy of Sciences
+// Submitted by Tech Support <support@rasnet.ru>
+ras.ru
+
+// QA2
+// Submitted by Daniel Dent (https://www.danieldent.com/)
+qa2.com
+
+// QCX
+// Submitted by Cassandra Beelen <cassandra@beelen.one>
+qcx.io
+*.sys.qcx.io
+
+// QNAP System Inc : https://www.qnap.com
+// Submitted by Nick Chang <nickchang@qnap.com>
+dev-myqnapcloud.com
+alpha-myqnapcloud.com
+myqnapcloud.com
+
+// Quip : https://quip.com
+// Submitted by Patrick Linehan <plinehan@quip.com>
+*.quipelements.com
+
+// Qutheory LLC : http://qutheory.io
+// Submitted by Jonas Schwartz <jonas@qutheory.io>
+vapor.cloud
+vaporcloud.io
+
+// Rackmaze LLC : https://www.rackmaze.com
+// Submitted by Kirill Pertsev <kika@rackmaze.com>
+rackmaze.com
+rackmaze.net
+
+// Rakuten Games, Inc : https://dev.viberplay.io
+// Submitted by Joshua Zhang <public-suffix@rgames.jp>
+g.vbrplsbx.io
+
+// Rancher Labs, Inc : https://rancher.com
+// Submitted by Vincent Fiduccia <domains@rancher.com>
+*.on-k3s.io
+*.on-rancher.cloud
+*.on-rio.io
+
+// Read The Docs, Inc : https://www.readthedocs.org
+// Submitted by David Fischer <team@readthedocs.org>
+readthedocs.io
+
+// Red Hat, Inc. OpenShift : https://openshift.redhat.com/
+// Submitted by Tim Kramer <tkramer@rhcloud.com>
+rhcloud.com
+
+// Render : https://render.com
+// Submitted by Anurag Goel <dev@render.com>
+app.render.com
+onrender.com
+
+// Repl.it : https://repl.it
+// Submitted by Mason Clayton <mason@repl.it>
+repl.co
+id.repl.co
+repl.run
+
+// Resin.io : https://resin.io
+// Submitted by Tim Perry <tim@resin.io>
+resindevice.io
+devices.resinstaging.io
+
+// RethinkDB : https://www.rethinkdb.com/
+// Submitted by Chris Kastorff <info@rethinkdb.com>
+hzc.io
+
+// Revitalised Limited : http://www.revitalised.co.uk
+// Submitted by Jack Price <jack@revitalised.co.uk>
+wellbeingzone.eu
+wellbeingzone.co.uk
+
+// Rico Developments Limited : https://adimo.co
+// Submitted by Colin Brown <hello@adimo.co>
+adimo.co.uk
+
+// Riseup Networks : https://riseup.net
+// Submitted by Micah Anderson <micah@riseup.net>
+itcouldbewor.se
+
+// Rochester Institute of Technology : http://www.rit.edu/
+// Submitted by Jennifer Herting <jchits@rit.edu>
+git-pages.rit.edu
+
+// Rusnames Limited: http://rusnames.ru/
+// Submitted by Sergey Zotov <admin@rusnames.ru>
+xn--90amc.xn--p1acf
+биз.рус
+xn--j1aef.xn--p1acf
+ком.рус
+xn--j1ael8b.xn--p1acf
+крым.рус
+xn--h1ahn.xn--p1acf
+мир.рус
+xn--j1adp.xn--p1acf
+мск.рус
+xn--c1avg.xn--p1acf
+орг.рус
+xn--80aaa0cvac.xn--p1acf
+самара.рус
+xn--h1aliz.xn--p1acf
+сочи.рус
+xn--90a1af.xn--p1acf
+спб.рус
+xn--41a.xn--p1acf
+я.рус
+
+// Sandstorm Development Group, Inc. : https://sandcats.io/
+// Submitted by Asheesh Laroia <asheesh@sandstorm.io>
+sandcats.io
+
+// SBE network solutions GmbH : https://www.sbe.de/
+// Submitted by Norman Meilick <nm@sbe.de>
+logoip.de
+logoip.com
+
+// schokokeks.org GbR : https://schokokeks.org/
+// Submitted by Hanno Böck <hanno@schokokeks.org>
+schokokeks.net
+
+// Scottish Government: https://www.gov.scot
+// Submitted by Martin Ellis <martin.ellis@gov.scot>
+gov.scot
+service.gov.scot
+
+// Scry Security : http://www.scrysec.com
+// Submitted by Shante Adam <shante@skyhat.io>
+scrysec.com
+
+// Securepoint GmbH : https://www.securepoint.de
+// Submitted by Erik Anders <erik.anders@securepoint.de>
+firewall-gateway.com
+firewall-gateway.de
+my-gateway.de
+my-router.de
+spdns.de
+spdns.eu
+firewall-gateway.net
+my-firewall.org
+myfirewall.org
+spdns.org
+
+// Seidat : https://www.seidat.com
+// Submitted by Artem Kondratev <accounts@seidat.com>
+seidat.net
+
+// Sellfy : https://sellfy.com
+// Submitted by Yuriy Romadin <contact@sellfy.com>
+sellfy.store
+
+// Senseering GmbH : https://www.senseering.de
+// Submitted by Felix Mönckemeyer <f.moenckemeyer@senseering.de>
+senseering.net
+
+// Sendmsg: https://www.sendmsg.co.il
+// Submitted by Assaf Stern <domains@comstar.co.il>
+minisite.ms
+
+// Service Magnet : https://myservicemagnet.com
+// Submitted by Dave Sanders <dave@myservicemagnet.com>
+magnet.page
+
+// Service Online LLC : http://drs.ua/
+// Submitted by Serhii Bulakh <support@drs.ua>
+biz.ua
+co.ua
+pp.ua
+
+// Shift Crypto AG : https://shiftcrypto.ch
+// Submitted by alex <alex@shiftcrypto.ch>
+shiftcrypto.dev
+shiftcrypto.io
+
+// ShiftEdit : https://shiftedit.net/
+// Submitted by Adam Jimenez <adam@shiftcreate.com>
+shiftedit.io
+
+// Shopblocks : http://www.shopblocks.com/
+// Submitted by Alex Bowers <alex@shopblocks.com>
+myshopblocks.com
+
+// Shopify : https://www.shopify.com
+// Submitted by Alex Richter <alex.richter@shopify.com>
+myshopify.com
+
+// Shopit : https://www.shopitcommerce.com/
+// Submitted by Craig McMahon <craig@shopitcommerce.com>
+shopitsite.com
+
+// shopware AG : https://shopware.com
+// Submitted by Jens Küper <cloud@shopware.com>
+shopware.store
+
+// Siemens Mobility GmbH
+// Submitted by Oliver Graebner <security@mo-siemens.io>
+mo-siemens.io
+
+// SinaAppEngine : http://sae.sina.com.cn/
+// Submitted by SinaAppEngine <saesupport@sinacloud.com>
+1kapp.com
+appchizi.com
+applinzi.com
+sinaapp.com
+vipsinaapp.com
+
+// Siteleaf : https://www.siteleaf.com/
+// Submitted by Skylar Challand <support@siteleaf.com>
+siteleaf.net
+
+// Skyhat : http://www.skyhat.io
+// Submitted by Shante Adam <shante@skyhat.io>
+bounty-full.com
+alpha.bounty-full.com
+beta.bounty-full.com
+
+// Small Technology Foundation : https://small-tech.org
+// Submitted by Aral Balkan <aral@small-tech.org>
+small-web.org
+
+// Snowplow Analytics : https://snowplowanalytics.com/
+// Submitted by Ian Streeter <ian@snowplowanalytics.com>
+try-snowplow.com
+
+// SourceHut : https://sourcehut.org
+// Submitted by Drew DeVault <sir@cmpwn.com>
+srht.site
+
+// Stackhero : https://www.stackhero.io
+// Submitted by Adrien Gillon <adrien+public-suffix-list@stackhero.io>
+stackhero-network.com
+
+// Staclar : https://staclar.com
+// Submitted by Matthias Merkel <matthias.merkel@staclar.com>
+novecore.site
+
+// staticland : https://static.land
+// Submitted by Seth Vincent <sethvincent@gmail.com>
+static.land
+dev.static.land
+sites.static.land
+
+// Storebase : https://www.storebase.io
+// Submitted by Tony Schirmer <tony@storebase.io>
+storebase.store
+
+// Strategic System Consulting (eApps Hosting): https://www.eapps.com/
+// Submitted by Alex Oancea <aoancea@cloudscale365.com>
+vps-host.net
+atl.jelastic.vps-host.net
+njs.jelastic.vps-host.net
+ric.jelastic.vps-host.net
+
+// Sony Interactive Entertainment LLC : https://sie.com/
+// Submitted by David Coles <david.coles@sony.com>
+playstation-cloud.com
+
+// SourceLair PC : https://www.sourcelair.com
+// Submitted by Antonis Kalipetis <akalipetis@sourcelair.com>
+apps.lair.io
+*.stolos.io
+
+// SpaceKit : https://www.spacekit.io/
+// Submitted by Reza Akhavan <spacekit.io@gmail.com>
+spacekit.io
+
+// SpeedPartner GmbH: https://www.speedpartner.de/
+// Submitted by Stefan Neufeind <info@speedpartner.de>
+customer.speedpartner.de
+
+// Spreadshop (sprd.net AG) : https://www.spreadshop.com/
+// Submitted by Martin Breest <security@spreadshop.com>
+myspreadshop.at
+myspreadshop.com.au
+myspreadshop.be
+myspreadshop.ca
+myspreadshop.ch
+myspreadshop.com
+myspreadshop.de
+myspreadshop.dk
+myspreadshop.es
+myspreadshop.fi
+myspreadshop.fr
+myspreadshop.ie
+myspreadshop.it
+myspreadshop.net
+myspreadshop.nl
+myspreadshop.no
+myspreadshop.pl
+myspreadshop.se
+myspreadshop.co.uk
+
+// Standard Library : https://stdlib.com
+// Submitted by Jacob Lee <jacob@stdlib.com>
+api.stdlib.com
+
+// Storj Labs Inc. : https://storj.io/
+// Submitted by Philip Hutchins <hostmaster@storj.io>
+storj.farm
+
+// Studenten Net Twente : http://www.snt.utwente.nl/
+// Submitted by Silke Hofstra <syscom@snt.utwente.nl>
+utwente.io
+
+// Student-Run Computing Facility : https://www.srcf.net/
+// Submitted by Edwin Balani <sysadmins@srcf.net>
+soc.srcf.net
+user.srcf.net
+
+// Sub 6 Limited: http://www.sub6.com
+// Submitted by Dan Miller <dm@sub6.com>
+temp-dns.com
+
+// Supabase : https://supabase.io
+// Submitted by Inian Parameshwaran <security@supabase.io>
+supabase.co
+supabase.in
+supabase.net
+su.paba.se
+
+// Symfony, SAS : https://symfony.com/
+// Submitted by Fabien Potencier <fabien@symfony.com>
+*.s5y.io
+*.sensiosite.cloud
+
+// Syncloud : https://syncloud.org
+// Submitted by Boris Rybalkin <syncloud@syncloud.it>
+syncloud.it
+
+// Synology, Inc. : https://www.synology.com/
+// Submitted by Rony Weng <ronyweng@synology.com>
+diskstation.me
+dscloud.biz
+dscloud.me
+dscloud.mobi
+dsmynas.com
+dsmynas.net
+dsmynas.org
+familyds.com
+familyds.net
+familyds.org
+i234.me
+myds.me
+synology.me
+vpnplus.to
+direct.quickconnect.to
+
+// TAIFUN Software AG : http://taifun-software.de
+// Submitted by Bjoern Henke <dev-server@taifun-software.de>
+taifun-dns.de
+
+// TASK geographical domains (www.task.gda.pl/uslugi/dns)
+gda.pl
+gdansk.pl
+gdynia.pl
+med.pl
+sopot.pl
+
+// Teckids e.V. : https://www.teckids.org
+// Submitted by Dominik George <dominik.george@teckids.org>
+edugit.org
+
+// Telebit : https://telebit.cloud
+// Submitted by AJ ONeal <aj@telebit.cloud>
+telebit.app
+telebit.io
+*.telebit.xyz
+
+// The Gwiddle Foundation : https://gwiddlefoundation.org.uk
+// Submitted by Joshua Bayfield <joshua.bayfield@gwiddlefoundation.org.uk>
+gwiddle.co.uk
+
+// Thingdust AG : https://thingdust.com/
+// Submitted by Adrian Imboden <adi@thingdust.com>
+*.firenet.ch
+*.svc.firenet.ch
+reservd.com
+thingdustdata.com
+cust.dev.thingdust.io
+cust.disrec.thingdust.io
+cust.prod.thingdust.io
+cust.testing.thingdust.io
+reservd.dev.thingdust.io
+reservd.disrec.thingdust.io
+reservd.testing.thingdust.io
+
+// Tlon.io : https://tlon.io
+// Submitted by Mark Staarink <mark@tlon.io>
+arvo.network
+azimuth.network
+tlon.network
+
+// Tor Project, Inc. : https://torproject.org
+// Submitted by Antoine Beaupré <anarcat@torproject.org
+torproject.net
+pages.torproject.net
+
+// TownNews.com : http://www.townnews.com
+// Submitted by Dustin Ward <dward@townnews.com>
+bloxcms.com
+townnews-staging.com
+
+// TradableBits: https://tradablebits.com
+// Submitted by Dmitry Khrisanov dmitry@tradablebits.com
+tbits.me
+
+// TrafficPlex GmbH : https://www.trafficplex.de/
+// Submitted by Phillipp Röll <phillipp.roell@trafficplex.de>
+12hp.at
+2ix.at
+4lima.at
+lima-city.at
+12hp.ch
+2ix.ch
+4lima.ch
+lima-city.ch
+trafficplex.cloud
+de.cool
+12hp.de
+2ix.de
+4lima.de
+lima-city.de
+1337.pictures
+clan.rip
+lima-city.rocks
+webspace.rocks
+lima.zone
+
+// TransIP : https://www.transip.nl
+// Submitted by Rory Breuk <rbreuk@transip.nl>
+*.transurl.be
+*.transurl.eu
+*.transurl.nl
+
+// TuxFamily : http://tuxfamily.org
+// Submitted by TuxFamily administrators <adm@staff.tuxfamily.org>
+tuxfamily.org
+
+// TwoDNS : https://www.twodns.de/
+// Submitted by TwoDNS-Support <support@two-dns.de>
+dd-dns.de
+diskstation.eu
+diskstation.org
+dray-dns.de
+draydns.de
+dyn-vpn.de
+dynvpn.de
+mein-vigor.de
+my-vigor.de
+my-wan.de
+syno-ds.de
+synology-diskstation.de
+synology-ds.de
+
+// Uberspace : https://uberspace.de
+// Submitted by Moritz Werner <mwerner@jonaspasche.com>
+uber.space
+*.uberspace.de
+
+// UDR Limited : http://www.udr.hk.com
+// Submitted by registry <hostmaster@udr.hk.com>
+hk.com
+hk.org
+ltd.hk
+inc.hk
+
+// United Gameserver GmbH : https://united-gameserver.de
+// Submitted by Stefan Schwarz <sysadm@united-gameserver.de>
+virtualuser.de
+virtual-user.de
+
+// urown.net : https://urown.net
+// Submitted by Hostmaster <hostmaster@urown.net>
+urown.cloud
+dnsupdate.info
+
+// .US
+// Submitted by Ed Moore <Ed.Moore@lib.de.us>
+lib.de.us
+
+// VeryPositive SIA : http://very.lv
+// Submitted by Danko Aleksejevs <danko@very.lv>
+2038.io
+
+// Vercel, Inc : https://vercel.com/
+// Submitted by Connor Davis <security@vercel.com>
+vercel.app
+vercel.dev
+now.sh
+
+// Viprinet Europe GmbH : http://www.viprinet.com
+// Submitted by Simon Kissel <hostmaster@viprinet.com>
+router.management
+
+// Virtual-Info : https://www.virtual-info.info/
+// Submitted by Adnan RIHAN <hostmaster@v-info.info>
+v-info.info
+
+// Voorloper.com: https://voorloper.com
+// Submitted by Nathan van Bakel <info@voorloper.com>
+voorloper.cloud
+
+// Voxel.sh DNS : https://voxel.sh/dns/
+// Submitted by Mia Rehlinger <dns@voxel.sh>
+neko.am
+nyaa.am
+be.ax
+cat.ax
+es.ax
+eu.ax
+gg.ax
+mc.ax
+us.ax
+xy.ax
+nl.ci
+xx.gl
+app.gp
+blog.gt
+de.gt
+to.gt
+be.gy
+cc.hn
+blog.kg
+io.kg
+jp.kg
+tv.kg
+uk.kg
+us.kg
+de.ls
+at.md
+de.md
+jp.md
+to.md
+indie.porn
+vxl.sh
+ch.tc
+me.tc
+we.tc
+nyan.to
+at.vg
+blog.vu
+dev.vu
+me.vu
+
+// V.UA Domain Administrator : https://domain.v.ua/
+// Submitted by Serhii Rostilo <sergey@rostilo.kiev.ua>
+v.ua
+
+// Waffle Computer Inc., Ltd. : https://docs.waffleinfo.com
+// Submitted by Masayuki Note <masa@blade.wafflecell.com>
+wafflecell.com
+
+// WapBlog.ID : https://www.wapblog.id
+// Submitted by Fajar Sodik <official@wapblog.id>
+idnblogger.com
+indowapblog.com
+bloger.id
+wblog.id
+wbq.me
+fastblog.net
+
+// WebHare bv: https://www.webhare.com/
+// Submitted by Arnold Hendriks <info@webhare.com>
+*.webhare.dev
+
+// WebHotelier Technologies Ltd: https://www.webhotelier.net/
+// Submitted by Apostolos Tsakpinis <apostolos.tsakpinis@gmail.com>
+reserve-online.net
+reserve-online.com
+bookonline.app
+hotelwithflight.com
+
+// WeDeploy by Liferay, Inc. : https://www.wedeploy.com
+// Submitted by Henrique Vicente <security@wedeploy.com>
+wedeploy.io
+wedeploy.me
+wedeploy.sh
+
+// Western Digital Technologies, Inc : https://www.wdc.com
+// Submitted by Jung Jin <jungseok.jin@wdc.com>
+remotewd.com
+
+// WIARD Enterprises : https://wiardweb.com
+// Submitted by Kidd Hustle <kiddhustle@wiardweb.com>
+pages.wiardweb.com
+
+// Wikimedia Labs : https://wikitech.wikimedia.org
+// Submitted by Arturo Borrero Gonzalez <aborrero@wikimedia.org>
+wmflabs.org
+toolforge.org
+wmcloud.org
+
+// WISP : https://wisp.gg
+// Submitted by Stepan Fedotov <stepan@wisp.gg>
+panel.gg
+daemon.panel.gg
+
+// WoltLab GmbH : https://www.woltlab.com
+// Submitted by Tim Düsterhus <security@woltlab.cloud>
+woltlab-demo.com
+myforum.community
+community-pro.de
+diskussionsbereich.de
+community-pro.net
+meinforum.net
+
+// WP Engine : https://wpengine.com/
+// Submitted by Michael Smith <michael.smith@wpengine.com>
+// Submitted by Brandon DuRette <brandon.durette@wpengine.com>
+wpenginepowered.com
+js.wpenginepowered.com
+
+// Wix.com, Inc. : https://www.wix.com
+// Submitted by Shahar Talmi <shahart@wix.com>
+wixsite.com
+editorx.io
+
+// XenonCloud GbR: https://xenoncloud.net
+// Submitted by Julian Uphoff <publicsuffixlist@xenoncloud.net>
+half.host
+
+// XnBay Technology : http://www.xnbay.com/
+// Submitted by XnBay Developer <developer.xncloud@gmail.com>
+xnbay.com
+u2.xnbay.com
+u2-local.xnbay.com
+
+// XS4ALL Internet bv : https://www.xs4all.nl/
+// Submitted by Daniel Mostertman <unixbeheer+publicsuffix@xs4all.net>
+cistron.nl
+demon.nl
+xs4all.space
+
+// Yandex.Cloud LLC: https://cloud.yandex.com
+// Submitted by Alexander Lodin <security+psl@yandex-team.ru>
+yandexcloud.net
+storage.yandexcloud.net
+website.yandexcloud.net
+
+// YesCourse Pty Ltd : https://yescourse.com
+// Submitted by Atul Bhouraskar <atul@yescourse.com>
+official.academy
+
+// Yola : https://www.yola.com/
+// Submitted by Stefano Rivera <stefano@yola.com>
+yolasite.com
+
+// Yombo : https://yombo.net
+// Submitted by Mitch Schwenk <mitch@yombo.net>
+ybo.faith
+yombo.me
+homelink.one
+ybo.party
+ybo.review
+ybo.science
+ybo.trade
+
+// Yunohost : https://yunohost.org
+// Submitted by Valentin Grimaud <security@yunohost.org>
+ynh.fr
+nohost.me
+noho.st
+
+// ZaNiC : http://www.za.net/
+// Submitted by registry <hostmaster@nic.za.net>
+za.net
+za.org
+
+// Zine EOOD : https://zine.bg/
+// Submitted by Martin Angelov <martin@zine.bg>
+bss.design
+
+// Zitcom A/S : https://www.zitcom.dk
+// Submitted by Emil Stahl <esp@zitcom.dk>
+basicserver.io
+virtualserver.io
+enterprisecloud.nu
+
+// ===END PRIVATE DOMAINS===
diff --git a/contrib/publicsuffix/idn.pl b/contrib/publicsuffix/idn.pl
new file mode 100644
index 0000000..dd46d65
--- /dev/null
+++ b/contrib/publicsuffix/idn.pl
@@ -0,0 +1,20 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use Net::IDN::Encode ':all';
+use Unicode::Normalize;
+
+binmode(STDOUT, ":utf8");
+binmode(STDIN, ":utf8");
+
+while (<>) {
+ $_ = NFC($_);
+ if (/^[^\/].*[^\x00-\x7F]+.*/) {
+ chomp;
+ printf "%s\n", domain_to_ascii($_);
+ $_ .= "\n";
+ }
+} continue {
+ print $_;
+} \ No newline at end of file
diff --git a/contrib/replxx/CMakeLists.txt b/contrib/replxx/CMakeLists.txt
new file mode 100644
index 0000000..f48a068
--- /dev/null
+++ b/contrib/replxx/CMakeLists.txt
@@ -0,0 +1,82 @@
+# -*- mode: CMAKE; -*-
+project( replxx VERSION 0.0.2 LANGUAGES CXX C )
+
+# INFO
+set(REPLXX_DISPLAY_NAME "replxx")
+set(REPLXX_URL_INFO_ABOUT "https://github.com/AmokHuginnsson/replxx")
+set(REPLXX_CONTACT "amok@codestation.org")
+set(REPLXX_FRIENDLY_STRING "replxx - Read Evaluate Print Loop library")
+
+# compiler options
+if(CMAKE_COMPILER_IS_GNUCXX)
+ message(STATUS "Compiler type GNU: ${CMAKE_CXX_COMPILER}")
+ set(BASE_COMPILER_OPTIONS "-std=c++11 -Wall -D_GNU_SOURCE -pthread")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS}")
+ set(CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS} -O0 --coverage -fno-inline -fno-default-inline -fno-inline-small-functions")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${BASE_COMPILER_OPTIONS} -O0 -g -ggdb -g3 -ggdb3")
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${BASE_COMPILER_OPTIONS} -Os")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${BASE_COMPILER_OPTIONS} -O3 -fomit-frame-pointer")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${BASE_COMPILER_OPTIONS} -O3 -g")
+ set(CMAKE_C_FLAGS "-std=c99")
+elseif(CMAKE_COMPILER_IS_CLANGCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+ # using regular Clang or AppleClang
+ message(STATUS "Compiler type CLANG: ${CMAKE_CXX_COMPILER}")
+ set(BASE_COMPILER_OPTIONS "-std=c++11 -Wall -Wextra -D_GNU_SOURCE -pthread")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS}")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${BASE_COMPILER_OPTIONS} -O0 -g")
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${BASE_COMPILER_OPTIONS} -Os")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${BASE_COMPILER_OPTIONS} -O3 -fomit-frame-pointer")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${BASE_COMPILER_OPTIONS} -O3 -g")
+ set(CMAKE_C_FLAGS "-std=c99")
+elseif(MSVC)
+ message(STATUS "Compiler type MSVC: ${CMAKE_CXX_COMPILER}")
+ add_definitions("-D_CRT_SECURE_NO_WARNINGS=1")
+
+ set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /INCREMENTAL:NO /SUBSYSTEM:CONSOLE /LTCG /ignore:4099")
+ set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /SUBSYSTEM:CONSOLE /ignore:4099")
+ set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /SUBSYSTEM:CONSOLE /ignore:4099")
+ set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /SUBSYSTEM:CONSOLE /ignore:4099")
+else()
+ # unknown compiler
+ message(STATUS "Compiler type UNKNOWN: ${CMAKE_CXX_COMPILER}")
+ set(BASE_COMPILER_OPTIONS "-std=c++11 -Wall -pthread")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS}")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${BASE_COMPILER_OPTIONS} -O0 -g")
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${BASE_COMPILER_OPTIONS} -Os")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${BASE_COMPILER_OPTIONS} -O3 -fomit-frame-pointer")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${BASE_COMPILER_OPTIONS} -O3 -g")
+ set(CMAKE_C_FLAGS "-std=c99")
+endif()
+
+# build libreplxx
+set(
+ REPLXX_SOURCES
+ src/conversion.cxx
+ src/escape.cxx
+ src/history.cxx
+ src/replxx_impl.cxx
+ src/prompt.cxx
+ src/replxx.cxx
+ src/util.cxx
+ src/wcwidth.cpp
+ src/terminal.cxx
+ src/windows.cxx
+)
+
+set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+if(ENABLE_STATIC MATCHES "ON")
+ add_library(rspamd-replxx STATIC ${REPLXX_SOURCES})
+else()
+ add_library(rspamd-replxx SHARED ${REPLXX_SOURCES})
+endif()
+
+target_include_directories(
+ rspamd-replxx
+ PUBLIC ${PROJECT_SOURCE_DIR}/include
+ PRIVATE ${PROJECT_SOURCE_DIR}/src
+)
+set( TARGETS ${TARGETS} rspamd-replxx )
+target_compile_definitions(rspamd-replxx PRIVATE REPLXX_BUILDING_DLL)
+target_link_libraries(rspamd-replxx "${RSPAMD_REQUIRED_LIBRARIES}")
+
+install( TARGETS ${TARGETS} LIBRARY DESTINATION ${RSPAMD_LIBDIR}) \ No newline at end of file
diff --git a/contrib/replxx/LICENSE.md b/contrib/replxx/LICENSE.md
new file mode 100644
index 0000000..c73c3f2
--- /dev/null
+++ b/contrib/replxx/LICENSE.md
@@ -0,0 +1,63 @@
+Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+Copyright (c) 2010, Salvatore Sanfilippo (antirez at gmail dot com)
+Copyright (c) 2010, Pieter Noordhuis (pcnoordhuis at gmail dot com)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of Redis nor the names of its contributors may be used
+ to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+wcwidth.cpp
+===========
+
+Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+
+Permission to use, copy, modify, and distribute this software
+for any purpose and without fee is hereby granted. The author
+disclaims all warranties with regard to this software.
+
+
+ConvertUTF.cpp
+==============
+
+Copyright 2001-2004 Unicode, Inc.
+
+Disclaimer
+
+This source code is provided as is by Unicode, Inc. No claims are
+made as to fitness for any particular purpose. No warranties of any
+kind are expressed or implied. The recipient agrees to determine
+applicability of information provided. If this file has been
+purchased on magnetic or optical media from Unicode, Inc., the
+sole remedy for any claim will be exchange of defective media
+within 90 days of receipt.
+
+Limitations on Rights to Redistribute This Code
+
+Unicode, Inc. hereby grants the right to freely use the information
+supplied in this file in the creation of products supporting the
+Unicode Standard, and to make copies of this file in any form
+for internal or external distribution as long as this notice
+remains attached.
diff --git a/contrib/replxx/README.md b/contrib/replxx/README.md
new file mode 100644
index 0000000..21c907a
--- /dev/null
+++ b/contrib/replxx/README.md
@@ -0,0 +1,119 @@
+# Read Evaluate Print Loop ++
+
+![demo](https://drive.google.com/uc?export=download&id=0B53g2Y3z7rWNT2dCRGVVNldaRnc)
+
+[![Build Status](https://travis-ci.org/AmokHuginnsson/replxx.svg?branch=master)](https://travis-ci.org/AmokHuginnsson/replxx)
+
+A small, portable GNU readline replacement for Linux, Windows and
+MacOS which is capable of handling UTF-8 characters. Unlike GNU
+readline, which is GPL, this library uses a BSD license and can be
+used in any kind of program.
+
+## Origin
+
+This replxx implementation is based on the work by
+[ArangoDB Team](https://github.com/arangodb/linenoise-ng) and
+[Salvatore Sanfilippo](https://github.com/antirez/linenoise) and
+10gen Inc. The goal is to create a zero-config, BSD
+licensed, readline replacement usable in Apache2 or BSD licensed
+programs.
+
+## Features
+
+* single-line and multi-line editing mode with the usual key bindings implemented
+* history handling
+* completion
+* syntax highlighting
+* hints
+* BSD license source code
+* Only uses a subset of VT100 escapes (ANSI.SYS compatible)
+* UTF8 aware
+* support for Linux, MacOS and Windows
+
+It deviates from Salvatore's original goal to have a minimal readline
+replacement for the sake of supporting UTF8 and Windows. It deviates
+from 10gen Inc.'s goal to create a C++ interface to linenoise. This
+library uses C++ internally, but to the user it provides a pure C
+interface that is compatible with the original linenoise API.
+C interface.
+
+## Requirements
+
+To build this library, you will need a C++11-enabled compiler and
+some recent version of CMake.
+
+## Build instructions
+
+### *nix
+
+1. Create a build directory
+
+```bash
+mkdir -p build && cd build
+```
+
+2. Build the library
+
+```bash
+cmake -DCMAKE_BUILD_TYPE=Release .. && make
+```
+
+3. Install the library at the default target location
+
+```bash
+sudo make install
+```
+
+The default installation location can be adjusted by setting the `DESTDIR`
+variable when invoking `make install`:
+
+```bash
+make DESTDIR=/tmp install
+```
+
+### Windows
+
+1. Create a build directory in MS-DOS command prompt
+
+```
+md build
+cd build
+```
+
+2. Generate Visual Studio solution file with cmake
+
+* 32 bit:
+```bash
+cmake -G "Visual Studio 12 2013" -DCMAKE_BUILD_TYPE=Release ..`
+```
+* 64 bit:
+```bash
+`cmake -G "Visual Studio 12 2013 Win64" -DCMAKE_BUILD_TYPE=Release ..`
+```
+
+3. Open the generated file `replxx.sln` in the `build` subdirectory with Visual Studio.
+
+## Tested with...
+
+ * Linux text only console ($TERM = linux)
+ * Linux KDE terminal application ($TERM = xterm)
+ * Linux xterm ($TERM = xterm)
+ * Linux Buildroot ($TERM = vt100)
+ * Mac OS X iTerm ($TERM = xterm)
+ * Mac OS X default Terminal.app ($TERM = xterm)
+ * OpenBSD 4.5 through an OSX Terminal.app ($TERM = screen)
+ * IBM AIX 6.1
+ * FreeBSD xterm ($TERM = xterm)
+ * ANSI.SYS
+ * Emacs comint mode ($TERM = dumb)
+ * Windows
+
+Please test it everywhere you can and report back!
+
+## Let's push this forward!
+
+Patches should be provided in the respect of linenoise sensibility for
+small and easy to understand code that and the license
+restrictions. Extensions must be submitted under a BSD license-style.
+A contributor license is required for contributions.
+
diff --git a/contrib/replxx/include/replxx.h b/contrib/replxx/include/replxx.h
new file mode 100644
index 0000000..5127ac2
--- /dev/null
+++ b/contrib/replxx/include/replxx.h
@@ -0,0 +1,575 @@
+/* linenoise.h -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __REPLXX_H
+#define __REPLXX_H
+
+#define REPLXX_VERSION "0.0.2"
+#define REPLXX_VERSION_MAJOR 0
+#define REPLXX_VERSION_MINOR 0
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * For use in Windows DLLs:
+ *
+ * If you are building replxx into a DLL,
+ * unless you are using supplied CMake based build,
+ * ensure that 'REPLXX_BUILDING_DLL' is defined when
+ * building the DLL so that proper symbols are exported.
+ */
+#if defined( _WIN32 ) && ! defined( REPLXX_STATIC )
+# ifdef REPLXX_BUILDING_DLL
+# define REPLXX_IMPEXP __declspec( dllexport )
+# else
+# define REPLXX_IMPEXP __declspec( dllimport )
+# endif
+#else
+# define REPLXX_IMPEXP /**/
+#endif
+
+/*! \brief Color definitions to use in highlighter callbacks.
+ */
+typedef enum {
+ REPLXX_COLOR_BLACK = 0,
+ REPLXX_COLOR_RED = 1,
+ REPLXX_COLOR_GREEN = 2,
+ REPLXX_COLOR_BROWN = 3,
+ REPLXX_COLOR_BLUE = 4,
+ REPLXX_COLOR_MAGENTA = 5,
+ REPLXX_COLOR_CYAN = 6,
+ REPLXX_COLOR_LIGHTGRAY = 7,
+ REPLXX_COLOR_GRAY = 8,
+ REPLXX_COLOR_BRIGHTRED = 9,
+ REPLXX_COLOR_BRIGHTGREEN = 10,
+ REPLXX_COLOR_YELLOW = 11,
+ REPLXX_COLOR_BRIGHTBLUE = 12,
+ REPLXX_COLOR_BRIGHTMAGENTA = 13,
+ REPLXX_COLOR_BRIGHTCYAN = 14,
+ REPLXX_COLOR_WHITE = 15,
+ REPLXX_COLOR_NORMAL = REPLXX_COLOR_LIGHTGRAY,
+ REPLXX_COLOR_DEFAULT = -1,
+ REPLXX_COLOR_ERROR = -2
+} ReplxxColor;
+
+enum { REPLXX_KEY_BASE = 0x0010ffff + 1 };
+enum { REPLXX_KEY_BASE_SHIFT = 0x01000000 };
+enum { REPLXX_KEY_BASE_CONTROL = 0x02000000 };
+enum { REPLXX_KEY_BASE_META = 0x04000000 };
+enum { REPLXX_KEY_ESCAPE = 27 };
+enum { REPLXX_KEY_PAGE_UP = REPLXX_KEY_BASE + 1 };
+enum { REPLXX_KEY_PAGE_DOWN = REPLXX_KEY_PAGE_UP + 1 };
+enum { REPLXX_KEY_DOWN = REPLXX_KEY_PAGE_DOWN + 1 };
+enum { REPLXX_KEY_UP = REPLXX_KEY_DOWN + 1 };
+enum { REPLXX_KEY_LEFT = REPLXX_KEY_UP + 1 };
+enum { REPLXX_KEY_RIGHT = REPLXX_KEY_LEFT + 1 };
+enum { REPLXX_KEY_HOME = REPLXX_KEY_RIGHT + 1 };
+enum { REPLXX_KEY_END = REPLXX_KEY_HOME + 1 };
+enum { REPLXX_KEY_DELETE = REPLXX_KEY_END + 1 };
+enum { REPLXX_KEY_INSERT = REPLXX_KEY_DELETE + 1 };
+enum { REPLXX_KEY_F1 = REPLXX_KEY_INSERT + 1 };
+enum { REPLXX_KEY_F2 = REPLXX_KEY_F1 + 1 };
+enum { REPLXX_KEY_F3 = REPLXX_KEY_F2 + 1 };
+enum { REPLXX_KEY_F4 = REPLXX_KEY_F3 + 1 };
+enum { REPLXX_KEY_F5 = REPLXX_KEY_F4 + 1 };
+enum { REPLXX_KEY_F6 = REPLXX_KEY_F5 + 1 };
+enum { REPLXX_KEY_F7 = REPLXX_KEY_F6 + 1 };
+enum { REPLXX_KEY_F8 = REPLXX_KEY_F7 + 1 };
+enum { REPLXX_KEY_F9 = REPLXX_KEY_F8 + 1 };
+enum { REPLXX_KEY_F10 = REPLXX_KEY_F9 + 1 };
+enum { REPLXX_KEY_F11 = REPLXX_KEY_F10 + 1 };
+enum { REPLXX_KEY_F12 = REPLXX_KEY_F11 + 1 };
+enum { REPLXX_KEY_F13 = REPLXX_KEY_F12 + 1 };
+enum { REPLXX_KEY_F14 = REPLXX_KEY_F13 + 1 };
+enum { REPLXX_KEY_F15 = REPLXX_KEY_F14 + 1 };
+enum { REPLXX_KEY_F16 = REPLXX_KEY_F15 + 1 };
+enum { REPLXX_KEY_F17 = REPLXX_KEY_F16 + 1 };
+enum { REPLXX_KEY_F18 = REPLXX_KEY_F17 + 1 };
+enum { REPLXX_KEY_F19 = REPLXX_KEY_F18 + 1 };
+enum { REPLXX_KEY_F20 = REPLXX_KEY_F19 + 1 };
+enum { REPLXX_KEY_F21 = REPLXX_KEY_F20 + 1 };
+enum { REPLXX_KEY_F22 = REPLXX_KEY_F21 + 1 };
+enum { REPLXX_KEY_F23 = REPLXX_KEY_F22 + 1 };
+enum { REPLXX_KEY_F24 = REPLXX_KEY_F23 + 1 };
+enum { REPLXX_KEY_MOUSE = REPLXX_KEY_F24 + 1 };
+enum { REPLXX_KEY_PASTE_START = REPLXX_KEY_MOUSE + 1 };
+enum { REPLXX_KEY_PASTE_FINISH = REPLXX_KEY_PASTE_START + 1 };
+
+#define REPLXX_KEY_SHIFT( key ) ( ( key ) | REPLXX_KEY_BASE_SHIFT )
+#define REPLXX_KEY_CONTROL( key ) ( ( key ) | REPLXX_KEY_BASE_CONTROL )
+#define REPLXX_KEY_META( key ) ( ( key ) | REPLXX_KEY_BASE_META )
+
+enum { REPLXX_KEY_BACKSPACE = REPLXX_KEY_CONTROL( 'H' ) };
+enum { REPLXX_KEY_TAB = REPLXX_KEY_CONTROL( 'I' ) };
+enum { REPLXX_KEY_ENTER = REPLXX_KEY_CONTROL( 'M' ) };
+
+/*! \brief List of built-in actions that act upon user input.
+ */
+typedef enum {
+ REPLXX_ACTION_INSERT_CHARACTER,
+ REPLXX_ACTION_NEW_LINE,
+ REPLXX_ACTION_DELETE_CHARACTER_UNDER_CURSOR,
+ REPLXX_ACTION_DELETE_CHARACTER_LEFT_OF_CURSOR,
+ REPLXX_ACTION_KILL_TO_END_OF_LINE,
+ REPLXX_ACTION_KILL_TO_BEGINING_OF_LINE,
+ REPLXX_ACTION_KILL_TO_END_OF_WORD,
+ REPLXX_ACTION_KILL_TO_BEGINING_OF_WORD,
+ REPLXX_ACTION_KILL_TO_END_OF_SUBWORD,
+ REPLXX_ACTION_KILL_TO_BEGINING_OF_SUBWORD,
+ REPLXX_ACTION_KILL_TO_WHITESPACE_ON_LEFT,
+ REPLXX_ACTION_YANK,
+ REPLXX_ACTION_YANK_CYCLE,
+ REPLXX_ACTION_YANK_LAST_ARG,
+ REPLXX_ACTION_MOVE_CURSOR_TO_BEGINING_OF_LINE,
+ REPLXX_ACTION_MOVE_CURSOR_TO_END_OF_LINE,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_WORD_LEFT,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_WORD_RIGHT,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_SUBWORD_LEFT,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_SUBWORD_RIGHT,
+ REPLXX_ACTION_MOVE_CURSOR_LEFT,
+ REPLXX_ACTION_MOVE_CURSOR_RIGHT,
+ REPLXX_ACTION_HISTORY_NEXT,
+ REPLXX_ACTION_HISTORY_PREVIOUS,
+ REPLXX_ACTION_HISTORY_FIRST,
+ REPLXX_ACTION_HISTORY_LAST,
+ REPLXX_ACTION_HISTORY_INCREMENTAL_SEARCH,
+ REPLXX_ACTION_HISTORY_COMMON_PREFIX_SEARCH,
+ REPLXX_ACTION_HINT_NEXT,
+ REPLXX_ACTION_HINT_PREVIOUS,
+ REPLXX_ACTION_CAPITALIZE_WORD,
+ REPLXX_ACTION_LOWERCASE_WORD,
+ REPLXX_ACTION_UPPERCASE_WORD,
+ REPLXX_ACTION_CAPITALIZE_SUBWORD,
+ REPLXX_ACTION_LOWERCASE_SUBWORD,
+ REPLXX_ACTION_UPPERCASE_SUBWORD,
+ REPLXX_ACTION_TRANSPOSE_CHARACTERS,
+ REPLXX_ACTION_TOGGLE_OVERWRITE_MODE,
+#ifndef _WIN32
+ REPLXX_ACTION_VERBATIM_INSERT,
+ REPLXX_ACTION_SUSPEND,
+#endif
+ REPLXX_ACTION_BRACKETED_PASTE,
+ REPLXX_ACTION_CLEAR_SCREEN,
+ REPLXX_ACTION_CLEAR_SELF,
+ REPLXX_ACTION_REPAINT,
+ REPLXX_ACTION_COMPLETE_LINE,
+ REPLXX_ACTION_COMPLETE_NEXT,
+ REPLXX_ACTION_COMPLETE_PREVIOUS,
+ REPLXX_ACTION_COMMIT_LINE,
+ REPLXX_ACTION_ABORT_LINE,
+ REPLXX_ACTION_SEND_EOF
+} ReplxxAction;
+
+/*! \brief Possible results of key-press handler actions.
+ */
+typedef enum {
+ REPLXX_ACTION_RESULT_CONTINUE, /*!< Continue processing user input. */
+ REPLXX_ACTION_RESULT_RETURN, /*!< Return user input entered so far. */
+ REPLXX_ACTION_RESULT_BAIL /*!< Stop processing user input, returns nullptr from the \e input() call. */
+} ReplxxActionResult;
+
+typedef struct ReplxxStateTag {
+ char const* text;
+ int cursorPosition;
+} ReplxxState;
+
+typedef struct Replxx Replxx;
+typedef struct ReplxxHistoryScan ReplxxHistoryScan;
+typedef struct ReplxxHistoryEntryTag {
+ char const* timestamp;
+ char const* text;
+} ReplxxHistoryEntry;
+
+/*! \brief Create Replxx library resource holder.
+ *
+ * Use replxx_end() to free resources acquired with this function.
+ *
+ * \return Replxx library resource holder.
+ */
+REPLXX_IMPEXP Replxx* replxx_init( void );
+
+/*! \brief Cleanup resources used by Replxx library.
+ *
+ * \param replxx - a Replxx library resource holder.
+ */
+REPLXX_IMPEXP void replxx_end( Replxx* replxx );
+
+/*! \brief Line modification callback type definition.
+ *
+ * User can observe and modify line contents (and cursor position)
+ * in response to changes to both introduced by the user through
+ * normal interactions.
+ *
+ * When callback returns Replxx updates current line content
+ * and current cursor position to the ones updated by the callback.
+ *
+ * \param line[in,out] - a R/W reference to an UTF-8 encoded input entered by the user so far.
+ * \param cursorPosition[in,out] - a R/W reference to current cursor position.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void (replxx_modify_callback_t)(char** input, int* contextLen, void* userData);
+
+/*! \brief Register modify callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_modify_callback( Replxx*, replxx_modify_callback_t* fn, void* userData );
+
+/*! \brief Highlighter callback type definition.
+ *
+ * If user want to have colorful input she must simply install highlighter callback.
+ * The callback would be invoked by the library after each change to the input done by
+ * the user. After callback returns library uses data from colors buffer to colorize
+ * displayed user input.
+ *
+ * \e size of \e colors buffer is equal to number of code points in user \e input
+ * which will be different from simple `strlen( input )`!
+ *
+ * \param input - an UTF-8 encoded input entered by the user so far.
+ * \param colors - output buffer for color information.
+ * \param size - size of output buffer for color information.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void (replxx_highlighter_callback_t)(char const* input, ReplxxColor* colors, int size, void* userData);
+
+/*! \brief Register highlighter callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_highlighter_callback( Replxx*, replxx_highlighter_callback_t* fn, void* userData );
+
+typedef struct replxx_completions replxx_completions;
+
+/*! \brief Completions callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e replxx_set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param completions - pointer to opaque list of user completions.
+ * \param contextLen[in,out] - length of the additional context to provide while displaying completions.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void(replxx_completion_callback_t)(const char* input, replxx_completions* completions, int* contextLen, void* userData);
+
+/*! \brief Register completion callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_completion_callback( Replxx*, replxx_completion_callback_t* fn, void* userData );
+
+/*! \brief Add another possible completion for current user input.
+ *
+ * \param completions - pointer to opaque list of user completions.
+ * \param str - UTF-8 encoded completion string.
+ */
+REPLXX_IMPEXP void replxx_add_completion( replxx_completions* completions, const char* str );
+
+/*! \brief Add another possible completion for current user input.
+ *
+ * \param completions - pointer to opaque list of user completions.
+ * \param str - UTF-8 encoded completion string.
+ * \param color - a color for the completion.
+ */
+REPLXX_IMPEXP void replxx_add_color_completion( replxx_completions* completions, const char* str, ReplxxColor color );
+
+typedef struct replxx_hints replxx_hints;
+
+/*! \brief Hints callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e replxx_set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param hints - pointer to opaque list of possible hints.
+ * \param contextLen[in,out] - length of the additional context to provide while displaying hints.
+ * \param color - a color used for displaying hints.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void(replxx_hint_callback_t)(const char* input, replxx_hints* hints, int* contextLen, ReplxxColor* color, void* userData);
+
+/*! \brief Register hints callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_hint_callback( Replxx*, replxx_hint_callback_t* fn, void* userData );
+
+/*! \brief Key press handler type definition.
+ *
+ * \param code - the key code replxx got from terminal.
+ * \return Decision on how should input() behave after this key handler returns.
+ */
+typedef ReplxxActionResult (key_press_handler_t)( int code, void* userData );
+
+/*! \brief Add another possible hint for current user input.
+ *
+ * \param hints - pointer to opaque list of hints.
+ * \param str - UTF-8 encoded hint string.
+ */
+REPLXX_IMPEXP void replxx_add_hint( replxx_hints* hints, const char* str );
+
+/*! \brief Read line of user input.
+ *
+ * Returned pointer is managed by the library and is not to be freed in the client.
+ *
+ * \param prompt - prompt to be displayed before getting user input.
+ * \return An UTF-8 encoded input given by the user (or nullptr on EOF).
+ */
+REPLXX_IMPEXP char const* replxx_input( Replxx*, const char* prompt );
+
+/*! \brief Get current state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \param state - buffer for current state of the model.
+ */
+REPLXX_IMPEXP void replxx_get_state( Replxx*, ReplxxState* state );
+
+/*! \brief Set new state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \param state - new state of the model.
+ */
+REPLXX_IMPEXP void replxx_set_state( Replxx*, ReplxxState* state );
+
+/*! \brief Print formatted string to standard output.
+ *
+ * This function ensures proper handling of ANSI escape sequences
+ * contained in printed data, which is especially useful on Windows
+ * since Unixes handle them correctly out of the box.
+ *
+ * \param fmt - printf style format.
+ */
+REPLXX_IMPEXP int replxx_print( Replxx*, char const* fmt, ... );
+
+/*! \brief Prints a char array with the given length to standard output.
+ *
+ * \copydetails print
+ *
+ * \param str - The char array to print.
+ * \param length - The length of the array.
+ */
+REPLXX_IMPEXP int replxx_write( Replxx*, char const* str, int length );
+
+/*! \brief Schedule an emulated key press event.
+ *
+ * \param code - key press code to be emulated.
+ */
+REPLXX_IMPEXP void replxx_emulate_key_press( Replxx*, int unsigned code );
+
+/*! \brief Invoke built-in action handler.
+ *
+ * \pre This function can be called only from key-press handler.
+ *
+ * \param action - a built-in action to invoke.
+ * \param code - a supplementary key-code to consume by built-in action handler.
+ * \return The action result informing the replxx what shall happen next.
+ */
+REPLXX_IMPEXP ReplxxActionResult replxx_invoke( Replxx*, ReplxxAction action, int unsigned code );
+
+/*! \brief Bind user defined action to handle given key-press event.
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param handler - use this handler to handle key-press event.
+ * \param userData - supplementary user data passed to invoked handlers.
+ */
+REPLXX_IMPEXP void replxx_bind_key( Replxx*, int code, key_press_handler_t handler, void* userData );
+
+/*! \brief Bind internal `replxx` action (by name) to handle given key-press event.
+ *
+ * Action names are the same as unique part of names of ReplxxAction enumerations
+ * but in lower case, e.g.: an action for recalling previous history line
+ * is \e REPLXX_ACTION_HISTORY_PREVIOUS so action name to be used in this
+ * interface for the same effect is "history_previous".
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param actionName - name of internal action to be invoked on key press.
+ * \return -1 if invalid action name was used, 0 otherwise.
+ */
+int replxx_bind_key_internal( Replxx*, int code, char const* actionName );
+
+REPLXX_IMPEXP void replxx_set_preload_buffer( Replxx*, const char* preloadText );
+
+REPLXX_IMPEXP void replxx_history_add( Replxx*, const char* line );
+REPLXX_IMPEXP int replxx_history_size( Replxx* );
+
+/*! \brief Set set of word break characters.
+ *
+ * This setting influences word based cursor movement and line editing capabilities.
+ *
+ * \param wordBreakers - 7-bit ASCII set of word breaking characters.
+ */
+REPLXX_IMPEXP void replxx_set_word_break_characters( Replxx*, char const* wordBreakers );
+
+/*! \brief How many completions should trigger pagination.
+ */
+REPLXX_IMPEXP void replxx_set_completion_count_cutoff( Replxx*, int count );
+
+/*! \brief Set maximum number of displayed hint rows.
+ */
+REPLXX_IMPEXP void replxx_set_max_hint_rows( Replxx*, int count );
+
+/*! \brief Set a delay before hint are shown after user stopped typing..
+ *
+ * \param milliseconds - a number of milliseconds to wait before showing hints.
+ */
+REPLXX_IMPEXP void replxx_set_hint_delay( Replxx*, int milliseconds );
+
+/*! \brief Set tab completion behavior.
+ *
+ * \param val - use double tab to invoke completions (if != 0).
+ */
+REPLXX_IMPEXP void replxx_set_double_tab_completion( Replxx*, int val );
+
+/*! \brief Set tab completion behavior.
+ *
+ * \param val - invoke completion even if user input is empty (if != 0).
+ */
+REPLXX_IMPEXP void replxx_set_complete_on_empty( Replxx*, int val );
+
+/*! \brief Set tab completion behavior.
+ *
+ * \param val - beep if completion is ambiguous (if != 0).
+ */
+REPLXX_IMPEXP void replxx_set_beep_on_ambiguous_completion( Replxx*, int val );
+
+/*! \brief Set complete next/complete previous behavior.
+ *
+ * COMPLETE_NEXT/COMPLETE_PREVIOUS actions have two modes of operations,
+ * in case when a partial completion is possible complete only partial part (`false` setting)
+ * or complete first proposed completion fully (`true` setting).
+ * The default is to complete fully (a `true` setting - complete immediately).
+ *
+ * \param val - complete immediately.
+ */
+REPLXX_IMPEXP void replxx_set_immediate_completion( Replxx*, int val );
+
+/*! \brief Set history duplicate entries behaviour.
+ *
+ * \param val - should history contain only unique entries?
+ */
+REPLXX_IMPEXP void replxx_set_unique_history( Replxx*, int val );
+
+/*! \brief Disable output coloring.
+ *
+ * \param val - if set to non-zero disable output colors.
+ */
+REPLXX_IMPEXP void replxx_set_no_color( Replxx*, int val );
+
+/*! \brief Set maximum number of entries in history list.
+ */
+REPLXX_IMPEXP void replxx_set_max_history_size( Replxx*, int len );
+REPLXX_IMPEXP ReplxxHistoryScan* replxx_history_scan_start( Replxx* );
+REPLXX_IMPEXP void replxx_history_scan_stop( Replxx*, ReplxxHistoryScan* );
+REPLXX_IMPEXP int replxx_history_scan_next( Replxx*, ReplxxHistoryScan*, ReplxxHistoryEntry* );
+
+/*! \brief Synchronize REPL's history with given file.
+ *
+ * Synchronizing means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping merged version as current REPL's history.
+ *
+ * This call is an equivalent of calling:
+ * replxx_history_save( rx, "some-file" );
+ * replxx_history_load( rx, "some-file" );
+ *
+ * \param filename - a path to the file with which REPL's current history should be synchronized.
+ * \return 0 iff history file was successfully created, -1 otherwise.
+ */
+REPLXX_IMPEXP int replxx_history_sync( Replxx*, const char* filename );
+
+/*! \brief Save REPL's history into given file.
+ *
+ * Saving means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping original (NOT merged) version as current REPL's history.
+ *
+ * \param filename - a path to the file where REPL's history should be saved.
+ * \return 0 iff history file was successfully created, -1 otherwise.
+ */
+REPLXX_IMPEXP int replxx_history_save( Replxx*, const char* filename );
+
+/*! \brief Load REPL's history from given file.
+ *
+ * \param filename - a path to the file which contains REPL's history that should be loaded.
+ * \return 0 iff history file was successfully opened, -1 otherwise.
+ */
+REPLXX_IMPEXP int replxx_history_load( Replxx*, const char* filename );
+
+/*! \brief Clear REPL's in-memory history.
+ */
+REPLXX_IMPEXP void replxx_history_clear( Replxx* );
+REPLXX_IMPEXP void replxx_clear_screen( Replxx* );
+#ifdef __REPLXX_DEBUG__
+void replxx_debug_dump_print_codes(void);
+#endif
+/* the following is extension to the original linenoise API */
+REPLXX_IMPEXP int replxx_install_window_change_handler( Replxx* );
+REPLXX_IMPEXP void replxx_enable_bracketed_paste( Replxx* );
+REPLXX_IMPEXP void replxx_disable_bracketed_paste( Replxx* );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __REPLXX_H */
+
diff --git a/contrib/replxx/include/replxx.hxx b/contrib/replxx/include/replxx.hxx
new file mode 100644
index 0000000..5362312
--- /dev/null
+++ b/contrib/replxx/include/replxx.hxx
@@ -0,0 +1,615 @@
+/*
+ * Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HAVE_REPLXX_HXX_INCLUDED
+#define HAVE_REPLXX_HXX_INCLUDED 1
+
+#include <memory>
+#include <vector>
+#include <string>
+#include <functional>
+
+/*
+ * For use in Windows DLLs:
+ *
+ * If you are building replxx into a DLL,
+ * unless you are using supplied CMake based build,
+ * ensure that 'REPLXX_BUILDING_DLL' is defined when
+ * building the DLL so that proper symbols are exported.
+ */
+#if defined( _WIN32 ) && ! defined( REPLXX_STATIC )
+# ifdef REPLXX_BUILDING_DLL
+# define REPLXX_IMPEXP __declspec( dllexport )
+# else
+# define REPLXX_IMPEXP __declspec( dllimport )
+# endif
+#else
+# define REPLXX_IMPEXP /**/
+#endif
+
+#ifdef ERROR
+enum { ERROR_BB1CA97EC761FC37101737BA0AA2E7C5 = ERROR };
+#undef ERROR
+enum { ERROR = ERROR_BB1CA97EC761FC37101737BA0AA2E7C5 };
+#endif
+#ifdef DELETE
+enum { DELETE_32F68A60CEF40FAEDBC6AF20298C1A1E = DELETE };
+#undef DELETE
+enum { DELETE = DELETE_32F68A60CEF40FAEDBC6AF20298C1A1E };
+#endif
+
+namespace replxx {
+
+class REPLXX_IMPEXP Replxx {
+public:
+ enum class Color {
+ BLACK = 0,
+ RED = 1,
+ GREEN = 2,
+ BROWN = 3,
+ BLUE = 4,
+ MAGENTA = 5,
+ CYAN = 6,
+ LIGHTGRAY = 7,
+ GRAY = 8,
+ BRIGHTRED = 9,
+ BRIGHTGREEN = 10,
+ YELLOW = 11,
+ BRIGHTBLUE = 12,
+ BRIGHTMAGENTA = 13,
+ BRIGHTCYAN = 14,
+ WHITE = 15,
+ NORMAL = LIGHTGRAY,
+ DEFAULT = -1,
+ ERROR = -2
+ };
+ struct KEY {
+ static char32_t const BASE = 0x0010ffff + 1;
+ static char32_t const BASE_SHIFT = 0x01000000;
+ static char32_t const BASE_CONTROL = 0x02000000;
+ static char32_t const BASE_META = 0x04000000;
+ static char32_t const ESCAPE = 27;
+ static char32_t const PAGE_UP = BASE + 1;
+ static char32_t const PAGE_DOWN = PAGE_UP + 1;
+ static char32_t const DOWN = PAGE_DOWN + 1;
+ static char32_t const UP = DOWN + 1;
+ static char32_t const LEFT = UP + 1;
+ static char32_t const RIGHT = LEFT + 1;
+ static char32_t const HOME = RIGHT + 1;
+ static char32_t const END = HOME + 1;
+ static char32_t const DELETE = END + 1;
+ static char32_t const INSERT = DELETE + 1;
+ static char32_t const F1 = INSERT + 1;
+ static char32_t const F2 = F1 + 1;
+ static char32_t const F3 = F2 + 1;
+ static char32_t const F4 = F3 + 1;
+ static char32_t const F5 = F4 + 1;
+ static char32_t const F6 = F5 + 1;
+ static char32_t const F7 = F6 + 1;
+ static char32_t const F8 = F7 + 1;
+ static char32_t const F9 = F8 + 1;
+ static char32_t const F10 = F9 + 1;
+ static char32_t const F11 = F10 + 1;
+ static char32_t const F12 = F11 + 1;
+ static char32_t const F13 = F12 + 1;
+ static char32_t const F14 = F13 + 1;
+ static char32_t const F15 = F14 + 1;
+ static char32_t const F16 = F15 + 1;
+ static char32_t const F17 = F16 + 1;
+ static char32_t const F18 = F17 + 1;
+ static char32_t const F19 = F18 + 1;
+ static char32_t const F20 = F19 + 1;
+ static char32_t const F21 = F20 + 1;
+ static char32_t const F22 = F21 + 1;
+ static char32_t const F23 = F22 + 1;
+ static char32_t const F24 = F23 + 1;
+ static char32_t const MOUSE = F24 + 1;
+ static char32_t const PASTE_START = MOUSE + 1;
+ static char32_t const PASTE_FINISH = PASTE_START + 1;
+ static constexpr char32_t shift( char32_t key_ ) {
+ return ( key_ | BASE_SHIFT );
+ }
+ static constexpr char32_t control( char32_t key_ ) {
+ return ( key_ | BASE_CONTROL );
+ }
+ static constexpr char32_t meta( char32_t key_ ) {
+ return ( key_ | BASE_META );
+ }
+ static char32_t const BACKSPACE = 'H' | BASE_CONTROL;
+ static char32_t const TAB = 'I' | BASE_CONTROL;
+ static char32_t const ENTER = 'M' | BASE_CONTROL;
+ };
+ /*! \brief List of built-in actions that act upon user input.
+ */
+ enum class ACTION {
+ INSERT_CHARACTER,
+ NEW_LINE,
+ DELETE_CHARACTER_UNDER_CURSOR,
+ DELETE_CHARACTER_LEFT_OF_CURSOR,
+ KILL_TO_END_OF_LINE,
+ KILL_TO_BEGINING_OF_LINE,
+ KILL_TO_END_OF_WORD,
+ KILL_TO_BEGINING_OF_WORD,
+ KILL_TO_END_OF_SUBWORD,
+ KILL_TO_BEGINING_OF_SUBWORD,
+ KILL_TO_WHITESPACE_ON_LEFT,
+ YANK,
+ YANK_CYCLE,
+ YANK_LAST_ARG,
+ MOVE_CURSOR_TO_BEGINING_OF_LINE,
+ MOVE_CURSOR_TO_END_OF_LINE,
+ MOVE_CURSOR_ONE_WORD_LEFT,
+ MOVE_CURSOR_ONE_WORD_RIGHT,
+ MOVE_CURSOR_ONE_SUBWORD_LEFT,
+ MOVE_CURSOR_ONE_SUBWORD_RIGHT,
+ MOVE_CURSOR_LEFT,
+ MOVE_CURSOR_RIGHT,
+ HISTORY_NEXT,
+ HISTORY_PREVIOUS,
+ HISTORY_FIRST,
+ HISTORY_LAST,
+ HISTORY_INCREMENTAL_SEARCH,
+ HISTORY_COMMON_PREFIX_SEARCH,
+ HINT_NEXT,
+ HINT_PREVIOUS,
+ CAPITALIZE_WORD,
+ LOWERCASE_WORD,
+ UPPERCASE_WORD,
+ CAPITALIZE_SUBWORD,
+ LOWERCASE_SUBWORD,
+ UPPERCASE_SUBWORD,
+ TRANSPOSE_CHARACTERS,
+ TOGGLE_OVERWRITE_MODE,
+#ifndef _WIN32
+ VERBATIM_INSERT,
+ SUSPEND,
+#endif
+ BRACKETED_PASTE,
+ CLEAR_SCREEN,
+ CLEAR_SELF,
+ REPAINT,
+ COMPLETE_LINE,
+ COMPLETE_NEXT,
+ COMPLETE_PREVIOUS,
+ COMMIT_LINE,
+ ABORT_LINE,
+ SEND_EOF
+ };
+ /*! \brief Possible results of key-press handler actions.
+ */
+ enum class ACTION_RESULT {
+ CONTINUE, /*!< Continue processing user input. */
+ RETURN, /*!< Return user input entered so far. */
+ BAIL /*!< Stop processing user input, returns nullptr from the \e input() call. */
+ };
+ typedef std::vector<Color> colors_t;
+ class Completion {
+ std::string _text;
+ Color _color;
+ public:
+ Completion( char const* text_ )
+ : _text( text_ )
+ , _color( Color::DEFAULT ) {
+ }
+ Completion( std::string const& text_ )
+ : _text( text_ )
+ , _color( Color::DEFAULT ) {
+ }
+ Completion( std::string const& text_, Color color_ )
+ : _text( text_ )
+ , _color( color_ ) {
+ }
+ std::string const& text( void ) const {
+ return ( _text );
+ }
+ Color color( void ) const {
+ return ( _color );
+ }
+ };
+ typedef std::vector<Completion> completions_t;
+ class HistoryEntry {
+ std::string _timestamp;
+ std::string _text;
+ public:
+ HistoryEntry( std::string const& timestamp_, std::string const& text_ )
+ : _timestamp( timestamp_ )
+ , _text( text_ ) {
+ }
+ std::string const& timestamp( void ) const {
+ return ( _timestamp );
+ }
+ std::string const& text( void ) const {
+ return ( _text );
+ }
+ };
+ class HistoryScanImpl;
+ class HistoryScan {
+ public:
+ typedef std::unique_ptr<HistoryScanImpl, void (*)( HistoryScanImpl* )> impl_t;
+ private:
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4251)
+#endif
+ impl_t _impl;
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+ public:
+ HistoryScan( impl_t );
+ HistoryScan( HistoryScan&& ) = default;
+ HistoryScan& operator = ( HistoryScan&& ) = default;
+ bool next( void );
+ HistoryEntry const& get( void ) const;
+ private:
+ HistoryScan( HistoryScan const& ) = delete;
+ HistoryScan& operator = ( HistoryScan const& ) = delete;
+ };
+ typedef std::vector<std::string> hints_t;
+
+ /*! \brief Line modification callback type definition.
+ *
+ * User can observe and modify line contents (and cursor position)
+ * in response to changes to both introduced by the user through
+ * normal interactions.
+ *
+ * When callback returns Replxx updates current line content
+ * and current cursor position to the ones updated by the callback.
+ *
+ * \param line[in,out] - a R/W reference to an UTF-8 encoded input entered by the user so far.
+ * \param cursorPosition[in,out] - a R/W reference to current cursor position.
+ */
+ typedef std::function<void ( std::string& line, int& cursorPosition )> modify_callback_t;
+
+ /*! \brief Completions callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param[in,out] contextLen - length of the additional context to provide while displaying completions.
+ * \return A list of user completions.
+ */
+ typedef std::function<completions_t ( std::string const& input, int& contextLen )> completion_callback_t;
+
+ /*! \brief Highlighter callback type definition.
+ *
+ * If user want to have colorful input she must simply install highlighter callback.
+ * The callback would be invoked by the library after each change to the input done by
+ * the user. After callback returns library uses data from colors buffer to colorize
+ * displayed user input.
+ *
+ * Size of \e colors buffer is equal to number of code points in user \e input
+ * which will be different from simple `input.length()`!
+ *
+ * \param input - an UTF-8 encoded input entered by the user so far.
+ * \param colors - output buffer for color information.
+ */
+ typedef std::function<void ( std::string const& input, colors_t& colors )> highlighter_callback_t;
+
+ /*! \brief Hints callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param contextLen[in,out] - length of the additional context to provide while displaying hints.
+ * \param color - a color used for displaying hints.
+ * \return A list of possible hints.
+ */
+ typedef std::function<hints_t ( std::string const& input, int& contextLen, Color& color )> hint_callback_t;
+
+ /*! \brief Key press handler type definition.
+ *
+ * \param code - the key code replxx got from terminal.
+ * \return Decision on how should input() behave after this key handler returns.
+ */
+ typedef std::function<ACTION_RESULT ( char32_t code )> key_press_handler_t;
+
+ struct State {
+ char const* _text;
+ int _cursorPosition;
+ State( char const* text_, int cursorPosition_ = -1 )
+ : _text( text_ )
+ , _cursorPosition( cursorPosition_ ) {
+ }
+ State( State const& ) = default;
+ State& operator = ( State const& ) = default;
+ char const* text( void ) const {
+ return ( _text );
+ }
+ int cursor_position( void ) const {
+ return ( _cursorPosition );
+ }
+ };
+
+ class ReplxxImpl;
+private:
+ typedef std::unique_ptr<ReplxxImpl, void (*)( ReplxxImpl* )> impl_t;
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4251)
+#endif
+ impl_t _impl;
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+public:
+ Replxx( void );
+ Replxx( Replxx&& ) = default;
+ Replxx& operator = ( Replxx&& ) = default;
+
+ /*! \brief Register modify callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_modify_callback( modify_callback_t const& fn );
+
+ /*! \brief Register completion callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_completion_callback( completion_callback_t const& fn );
+
+ /*! \brief Register highlighter callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_highlighter_callback( highlighter_callback_t const& fn );
+
+ /*! \brief Register hints callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_hint_callback( hint_callback_t const& fn );
+
+ /*! \brief Read line of user input.
+ *
+ * Returned pointer is managed by the library and is not to be freed in the client.
+ *
+ * \param prompt - prompt to be displayed before getting user input.
+ * \return An UTF-8 encoded input given by the user (or nullptr on EOF).
+ */
+ char const* input( std::string const& prompt );
+
+ /*! \brief Get current state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \return Current state of the model.
+ */
+ State get_state( void ) const;
+
+ /*! \brief Set new state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \param state - new state of the model.
+ */
+ void set_state( State const& state );
+
+ /*! \brief Print formatted string to standard output.
+ *
+ * This function ensures proper handling of ANSI escape sequences
+ * contained in printed data, which is especially useful on Windows
+ * since Unixes handle them correctly out of the box.
+ *
+ * \param fmt - printf style format.
+ */
+ void print( char const* fmt, ... );
+
+ /*! \brief Prints a char array with the given length to standard output.
+ *
+ * \copydetails print
+ *
+ * \param str - The char array to print.
+ * \param length - The length of the array.
+ */
+ void write( char const* str, int length );
+
+ /*! \brief Schedule an emulated key press event.
+ *
+ * \param code - key press code to be emulated.
+ */
+ void emulate_key_press( char32_t code );
+
+ /*! \brief Invoke built-in action handler.
+ *
+ * \pre This method can be called only from key-press handler.
+ *
+ * \param action - a built-in action to invoke.
+ * \param code - a supplementary key-code to consume by built-in action handler.
+ * \return The action result informing the replxx what shall happen next.
+ */
+ ACTION_RESULT invoke( ACTION action, char32_t code );
+
+ /*! \brief Bind user defined action to handle given key-press event.
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param handle - use this handler to handle key-press event.
+ */
+ void bind_key( char32_t code, key_press_handler_t handler );
+
+ /*! \brief Bind internal `replxx` action (by name) to handle given key-press event.
+ *
+ * Action names are the same as names of Replxx::ACTION enumerations
+ * but in lower case, e.g.: an action for recalling previous history line
+ * is \e Replxx::ACTION::HISTORY_PREVIOUS so action name to be used in this
+ * interface for the same effect is "history_previous".
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param actionName - name of internal action to be invoked on key press.
+ */
+ void bind_key_internal( char32_t code, char const* actionName );
+
+ void history_add( std::string const& line );
+
+ /*! \brief Synchronize REPL's history with given file.
+ *
+ * Synchronizing means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping merged version as current REPL's history.
+ *
+ * This call is an equivalent of calling:
+ * history_save( "some-file" );
+ * history_load( "some-file" );
+ *
+ * \param filename - a path to the file with which REPL's current history should be synchronized.
+ * \return True iff history file was successfully created.
+ */
+ bool history_sync( std::string const& filename );
+
+ /*! \brief Save REPL's history into given file.
+ *
+ * Saving means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping original (NOT merged) version as current REPL's history.
+ *
+ * \param filename - a path to the file where REPL's history should be saved.
+ * \return True iff history file was successfully created.
+ */
+ bool history_save( std::string const& filename );
+
+ /*! \brief Load REPL's history from given file.
+ *
+ * \param filename - a path to the file which contains REPL's history that should be loaded.
+ * \return True iff history file was successfully opened.
+ */
+ bool history_load( std::string const& filename );
+
+ /*! \brief Clear REPL's in-memory history.
+ */
+ void history_clear( void );
+ int history_size( void ) const;
+ HistoryScan history_scan( void ) const;
+
+ void set_preload_buffer( std::string const& preloadText );
+
+ /*! \brief Set set of word break characters.
+ *
+ * This setting influences word based cursor movement and line editing capabilities.
+ *
+ * \param wordBreakers - 7-bit ASCII set of word breaking characters.
+ */
+ void set_word_break_characters( char const* wordBreakers );
+
+ /*! \brief How many completions should trigger pagination.
+ */
+ void set_completion_count_cutoff( int count );
+
+ /*! \brief Set maximum number of displayed hint rows.
+ */
+ void set_max_hint_rows( int count );
+
+ /*! \brief Set a delay before hint are shown after user stopped typing..
+ *
+ * \param milliseconds - a number of milliseconds to wait before showing hints.
+ */
+ void set_hint_delay( int milliseconds );
+
+ /*! \brief Set tab completion behavior.
+ *
+ * \param val - use double tab to invoke completions.
+ */
+ void set_double_tab_completion( bool val );
+
+ /*! \brief Set tab completion behavior.
+ *
+ * \param val - invoke completion even if user input is empty.
+ */
+ void set_complete_on_empty( bool val );
+
+ /*! \brief Set tab completion behavior.
+ *
+ * \param val - beep if completion is ambiguous.
+ */
+ void set_beep_on_ambiguous_completion( bool val );
+
+ /*! \brief Set complete next/complete previous behavior.
+ *
+ * COMPLETE_NEXT/COMPLETE_PREVIOUS actions have two modes of operations,
+ * in case when a partial completion is possible complete only partial part (`false` setting)
+ * or complete first proposed completion fully (`true` setting).
+ * The default is to complete fully (a `true` setting - complete immediately).
+ *
+ * \param val - complete immediately.
+ */
+ void set_immediate_completion( bool val );
+
+ /*! \brief Set history duplicate entries behaviour.
+ *
+ * \param val - should history contain only unique entries?
+ */
+ void set_unique_history( bool val );
+
+ /*! \brief Disable output coloring.
+ *
+ * \param val - if set to non-zero disable output colors.
+ */
+ void set_no_color( bool val );
+
+ /*! \brief Set maximum number of entries in history list.
+ */
+ void set_max_history_size( int len );
+ void clear_screen( void );
+ int install_window_change_handler( void );
+ void enable_bracketed_paste( void );
+ void disable_bracketed_paste( void );
+
+private:
+ Replxx( Replxx const& ) = delete;
+ Replxx& operator = ( Replxx const& ) = delete;
+};
+
+}
+
+#endif /* HAVE_REPLXX_HXX_INCLUDED */
+
diff --git a/contrib/replxx/src/conversion.cxx b/contrib/replxx/src/conversion.cxx
new file mode 100644
index 0000000..f629f91
--- /dev/null
+++ b/contrib/replxx/src/conversion.cxx
@@ -0,0 +1,134 @@
+#include <algorithm>
+#include <string>
+#include <cstring>
+#include <cctype>
+#include <clocale>
+
+#include "unicode/utf8.h"
+#include "conversion.hxx"
+
+#ifdef _WIN32
+#define strdup _strdup
+#endif
+
+using namespace std;
+
+namespace replxx {
+
+namespace locale {
+
+void to_lower( std::string& s_ ) {
+ transform( s_.begin(), s_.end(), s_.begin(), static_cast<int(*)(int)>( &tolower ) );
+}
+
+bool is_8bit_encoding( void ) {
+ bool is8BitEncoding( false );
+ string origLC( setlocale( LC_CTYPE, nullptr ) );
+ string lc( origLC );
+ to_lower( lc );
+ if ( lc == "c" ) {
+ setlocale( LC_CTYPE, "" );
+ }
+ lc = setlocale( LC_CTYPE, nullptr );
+ setlocale( LC_CTYPE, origLC.c_str() );
+ to_lower( lc );
+ if ( lc.find( "8859" ) != std::string::npos ) {
+ is8BitEncoding = true;
+ }
+ return ( is8BitEncoding );
+}
+
+bool is8BitEncoding( is_8bit_encoding() );
+
+}
+
+ConversionResult copyString8to32(char32_t* dst, int dstSize, int& dstCount, const char* src) {
+ ConversionResult res = ConversionResult::conversionOK;
+ if ( ! locale::is8BitEncoding ) {
+ auto sourceStart = reinterpret_cast<const unsigned char*>(src);
+ auto slen = strlen(src);
+ auto targetStart = reinterpret_cast<UChar32*>(dst);
+ int i = 0, j = 0;
+
+ while (i < slen && j < dstSize) {
+ UChar32 uc;
+ auto prev_i = i;
+ U8_NEXT (sourceStart, i, slen, uc);
+
+ if (uc <= 0) {
+ if (U8_IS_LEAD (sourceStart[prev_i])) {
+ auto lead_byte = sourceStart[prev_i];
+ auto trailing_bytes = (((uint8_t)(lead_byte)>=0xc2)+
+ ((uint8_t)(lead_byte)>=0xe0)+
+ ((uint8_t)(lead_byte)>=0xf0));
+
+ if (trailing_bytes + i > slen) {
+ return ConversionResult::sourceExhausted;
+ }
+ }
+
+ /* Replace with 0xFFFD */
+ uc = 0x0000FFFD;
+ }
+ targetStart[j++] = uc;
+ }
+
+ dstCount = j;
+
+ if (j < dstSize) {
+ targetStart[j] = 0;
+ }
+ } else {
+ for ( dstCount = 0; ( dstCount < dstSize ) && src[dstCount]; ++ dstCount ) {
+ dst[dstCount] = src[dstCount];
+ }
+ }
+ return res;
+}
+
+ConversionResult copyString8to32(char32_t* dst, int dstSize, int& dstCount, const char8_t* src) {
+ return copyString8to32(
+ dst, dstSize, dstCount, reinterpret_cast<const char*>(src)
+ );
+}
+
+int copyString32to8(
+ char* dst, int dstSize, const char32_t* src, int srcSize
+) {
+ int resCount = 0;
+
+ if ( ! locale::is8BitEncoding ) {
+ int j = 0;
+ UBool is_error = 0;
+
+ for (auto i = 0; i < srcSize; i ++) {
+ U8_APPEND ((uint8_t *)dst, j, dstSize, src[i], is_error);
+
+ if (is_error) {
+ break;
+ }
+ }
+
+ if (!is_error) {
+ resCount = j;
+
+ if (j < dstSize) {
+ dst[j] = '\0';
+ }
+ }
+ } else {
+ int i( 0 );
+ for ( i = 0; ( i < dstSize ) && ( i < srcSize ) && src[i]; ++ i ) {
+ dst[i] = static_cast<char>( src[i] );
+ }
+ resCount = i;
+ if ( i < dstSize ) {
+ dst[i] = 0;
+ }
+ }
+
+ return resCount;
+}
+
+}
+
diff --git a/contrib/replxx/src/conversion.hxx b/contrib/replxx/src/conversion.hxx
new file mode 100644
index 0000000..05ea64f
--- /dev/null
+++ b/contrib/replxx/src/conversion.hxx
@@ -0,0 +1,35 @@
+#ifndef REPLXX_CONVERSION_HXX_INCLUDED
+#define REPLXX_CONVERSION_HXX_INCLUDED 1
+
+#ifdef __has_include
+#if __has_include( <version> )
+#include <version>
+#endif
+#endif
+
+typedef enum {
+ conversionOK, /* conversion successful */
+ sourceExhausted, /* partial character in source, but hit end */
+ targetExhausted, /* insuff. room in target for conversion */
+ sourceIllegal /* source sequence is illegal/malformed */
+} ConversionResult;
+
+#if ! ( defined( __cpp_lib_char8_t ) || ( defined( __clang_major__ ) && ( __clang_major__ >= 8 ) && ( __cplusplus > 201703L ) ) )
+namespace replxx {
+typedef unsigned char char8_t;
+}
+#endif
+
+namespace replxx {
+
+ConversionResult copyString8to32( char32_t* dst, int dstSize, int& dstCount, char const* src );
+ConversionResult copyString8to32( char32_t* dst, int dstSize, int& dstCount, char8_t const* src );
+int copyString32to8( char* dst, int dstSize, char32_t const* src, int srcSize );
+
+namespace locale {
+extern bool is8BitEncoding;
+}
+
+}
+
+#endif
diff --git a/contrib/replxx/src/escape.cxx b/contrib/replxx/src/escape.cxx
new file mode 100644
index 0000000..dda1ab0
--- /dev/null
+++ b/contrib/replxx/src/escape.cxx
@@ -0,0 +1,890 @@
+#include "escape.hxx"
+#include "terminal.hxx"
+#include "replxx.hxx"
+
+#ifndef _WIN32
+
+namespace replxx {
+
+namespace EscapeSequenceProcessing { // move these out of global namespace
+
+// This chunk of code does parsing of the escape sequences sent by various Linux
+// terminals.
+//
+// It handles arrow keys, Home, End and Delete keys by interpreting the
+// sequences sent by
+// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and
+// Ctrl key
+// combinations that are understood by replxx.
+//
+// The parsing uses tables, a bunch of intermediate dispatch routines and a
+// doDispatch
+// loop that reads the tables and sends control to "deeper" routines to continue
+// the
+// parsing. The starting call to doDispatch( c, initialDispatch ) will
+// eventually return
+// either a character (with optional CTRL and META bits set), or -1 if parsing
+// fails, or
+// zero if an attempt to read from the keyboard fails.
+//
+// This is rather sloppy escape sequence processing, since we're not paying
+// attention to what the
+// actual TERM is set to and are processing all key sequences for all terminals,
+// but it works with
+// the most common keystrokes on the most common terminals. It's intricate, but
+// the nested 'if'
+// statements required to do it directly would be worse. This way has the
+// advantage of allowing
+// changes and extensions without having to touch a lot of code.
+
+
+static char32_t thisKeyMetaCtrl = 0; // holds pre-set Meta and/or Ctrl modifiers
+
+// This dispatch routine is given a dispatch table and then farms work out to
+// routines
+// listed in the table based on the character it is called with. The dispatch
+// routines can
+// read more input characters to decide what should eventually be returned.
+// Eventually,
+// a called routine returns either a character or -1 to indicate parsing
+// failure.
+//
+char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) {
+ for (unsigned int i = 0; i < dispatchTable.len; ++i) {
+ if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) {
+ return dispatchTable.dispatch[i](c);
+ }
+ }
+ return dispatchTable.dispatch[dispatchTable.len](c);
+}
+
+// Final dispatch routines -- return something
+//
+static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; }
+static char32_t upArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::UP;;
+}
+static char32_t downArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::DOWN;
+}
+static char32_t rightArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::RIGHT;
+}
+static char32_t leftArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::LEFT;
+}
+static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::HOME; }
+static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::END; }
+static char32_t shiftTabRoutine(char32_t) { return Replxx::KEY::BASE_SHIFT | Replxx::KEY::TAB; }
+static char32_t f1KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F1; }
+static char32_t f2KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F2; }
+static char32_t f3KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F3; }
+static char32_t f4KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F4; }
+static char32_t f5KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F5; }
+static char32_t f6KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F6; }
+static char32_t f7KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F7; }
+static char32_t f8KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F8; }
+static char32_t f9KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F9; }
+static char32_t f10KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F10; }
+static char32_t f11KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F11; }
+static char32_t f12KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F12; }
+static char32_t pageUpKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PAGE_UP;
+}
+static char32_t pageDownKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PAGE_DOWN;
+}
+static char32_t deleteCharRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BACKSPACE;
+} // key labeled Backspace
+static char32_t insertKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::INSERT;
+} // key labeled Delete
+static char32_t deleteKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::DELETE;
+} // key labeled Delete
+static char32_t ctrlUpArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::UP;
+}
+static char32_t ctrlDownArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::DOWN;
+}
+static char32_t ctrlRightArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::RIGHT;
+}
+static char32_t ctrlLeftArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::LEFT;
+}
+static char32_t bracketPasteStartKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PASTE_START;
+}
+static char32_t bracketPasteFinishKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PASTE_FINISH;
+}
+static char32_t escFailureRoutine(char32_t) {
+ beep();
+ return -1;
+}
+
+// Handle ESC [ 1 ; 2 or 3 (or 5) <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket1Semicolon2or3or5Routines[] = {
+ upArrowKeyRoutine,
+ downArrowKeyRoutine,
+ rightArrowKeyRoutine,
+ leftArrowKeyRoutine,
+ homeKeyRoutine,
+ endKeyRoutine,
+ f1KeyRoutine,
+ f2KeyRoutine,
+ f3KeyRoutine,
+ f4KeyRoutine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket1Semicolon2or3or5Dispatch = {
+ 10, "ABCDHFPQRS", escLeftBracket1Semicolon2or3or5Routines
+};
+
+// Handle ESC [ 1 ; <more stuff> escape sequences
+//
+static char32_t escLeftBracket1Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket1Semicolon2or3or5Dispatch);
+}
+static char32_t escLeftBracket1Semicolon3Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_META;
+ return doDispatch(c, escLeftBracket1Semicolon2or3or5Dispatch);
+}
+static char32_t escLeftBracket1Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket1Semicolon2or3or5Dispatch);
+}
+static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = {
+ escLeftBracket1Semicolon2Routine,
+ escLeftBracket1Semicolon3Routine,
+ escLeftBracket1Semicolon5Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket1SemicolonDispatch = {
+ 3, "235", escLeftBracket1SemicolonRoutines
+};
+
+// Handle ESC [ 1 ; <more stuff> escape sequences
+//
+static char32_t escLeftBracket1SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket1SemicolonDispatch);
+}
+
+// (S)-F5
+static CharacterDispatchRoutine escLeftBracket15Semicolon2Routines[] = {
+ f5KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15Semicolon2Dispatch = {
+ 1, "~", escLeftBracket15Semicolon2Routines
+};
+static char32_t escLeftBracket15Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket15Semicolon2Dispatch);
+}
+
+// (C)-F5
+static CharacterDispatchRoutine escLeftBracket15Semicolon5Routines[] = {
+ f5KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15Semicolon5Dispatch = {
+ 1, "~", escLeftBracket15Semicolon5Routines
+};
+static char32_t escLeftBracket15Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket15Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket15SemicolonRoutines[] = {
+ escLeftBracket15Semicolon2Routine, escLeftBracket15Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15SemicolonDispatch = {
+ 2, "25", escLeftBracket15SemicolonRoutines
+};
+static char32_t escLeftBracket15SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket15SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket15Routines[] = {
+ f5KeyRoutine, escLeftBracket15SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15Dispatch = {
+ 2, "~;", escLeftBracket15Routines
+};
+static char32_t escLeftBracket15Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket15Dispatch);
+}
+
+// (S)-F6
+static CharacterDispatchRoutine escLeftBracket17Semicolon2Routines[] = {
+ f6KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17Semicolon2Dispatch = {
+ 1, "~", escLeftBracket17Semicolon2Routines
+};
+static char32_t escLeftBracket17Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket17Semicolon2Dispatch);
+}
+
+// (C)-F6
+static CharacterDispatchRoutine escLeftBracket17Semicolon5Routines[] = {
+ f6KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17Semicolon5Dispatch = {
+ 1, "~", escLeftBracket17Semicolon5Routines
+};
+static char32_t escLeftBracket17Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket17Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket17SemicolonRoutines[] = {
+ escLeftBracket17Semicolon2Routine, escLeftBracket17Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17SemicolonDispatch = {
+ 2, "25", escLeftBracket17SemicolonRoutines
+};
+static char32_t escLeftBracket17SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket17SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket17Routines[] = {
+ f6KeyRoutine, escLeftBracket17SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17Dispatch = {
+ 2, "~;", escLeftBracket17Routines
+};
+static char32_t escLeftBracket17Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket17Dispatch);
+}
+
+// (S)-F7
+static CharacterDispatchRoutine escLeftBracket18Semicolon2Routines[] = {
+ f7KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18Semicolon2Dispatch = {
+ 1, "~", escLeftBracket18Semicolon2Routines
+};
+static char32_t escLeftBracket18Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket18Semicolon2Dispatch);
+}
+
+// (C)-F7
+static CharacterDispatchRoutine escLeftBracket18Semicolon5Routines[] = {
+ f7KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18Semicolon5Dispatch = {
+ 1, "~", escLeftBracket18Semicolon5Routines
+};
+static char32_t escLeftBracket18Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket18Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket18SemicolonRoutines[] = {
+ escLeftBracket18Semicolon2Routine, escLeftBracket18Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18SemicolonDispatch = {
+ 2, "25", escLeftBracket18SemicolonRoutines
+};
+static char32_t escLeftBracket18SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket18SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket18Routines[] = {
+ f7KeyRoutine, escLeftBracket18SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18Dispatch = {
+ 2, "~;", escLeftBracket18Routines
+};
+static char32_t escLeftBracket18Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket18Dispatch);
+}
+
+// (S)-F8
+static CharacterDispatchRoutine escLeftBracket19Semicolon2Routines[] = {
+ f8KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19Semicolon2Dispatch = {
+ 1, "~", escLeftBracket19Semicolon2Routines
+};
+static char32_t escLeftBracket19Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket19Semicolon2Dispatch);
+}
+
+// (C)-F8
+static CharacterDispatchRoutine escLeftBracket19Semicolon5Routines[] = {
+ f8KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19Semicolon5Dispatch = {
+ 1, "~", escLeftBracket19Semicolon5Routines
+};
+static char32_t escLeftBracket19Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket19Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket19SemicolonRoutines[] = {
+ escLeftBracket19Semicolon2Routine, escLeftBracket19Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19SemicolonDispatch = {
+ 2, "25", escLeftBracket19SemicolonRoutines
+};
+static char32_t escLeftBracket19SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket19SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket19Routines[] = {
+ f8KeyRoutine, escLeftBracket19SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19Dispatch = {
+ 2, "~;", escLeftBracket19Routines
+};
+static char32_t escLeftBracket19Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket19Dispatch);
+}
+
+// Handle ESC [ 1 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket1Routines[] = {
+ homeKeyRoutine, escLeftBracket1SemicolonRoutine,
+ escLeftBracket15Routine,
+ escLeftBracket17Routine,
+ escLeftBracket18Routine,
+ escLeftBracket19Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket1Dispatch = {
+ 6, "~;5789", escLeftBracket1Routines
+};
+
+// Handle ESC [ 2 <more stuff> escape sequences
+//
+
+// (S)-F9
+static CharacterDispatchRoutine escLeftBracket20Semicolon2Routines[] = {
+ f9KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20Semicolon2Dispatch = {
+ 1, "~", escLeftBracket20Semicolon2Routines
+};
+static char32_t escLeftBracket20Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket20Semicolon2Dispatch);
+}
+
+// (C)-F9
+static CharacterDispatchRoutine escLeftBracket20Semicolon5Routines[] = {
+ f9KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20Semicolon5Dispatch = {
+ 1, "~", escLeftBracket20Semicolon5Routines
+};
+static char32_t escLeftBracket20Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket20Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket20SemicolonRoutines[] = {
+ escLeftBracket20Semicolon2Routine, escLeftBracket20Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20SemicolonDispatch = {
+ 2, "25", escLeftBracket20SemicolonRoutines
+};
+static char32_t escLeftBracket20SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket20SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket200Routines[] = {
+ bracketPasteStartKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket200Dispatch = {
+ 1, "~", escLeftBracket200Routines
+};
+static char32_t escLeftBracket200Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket200Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket201Routines[] = {
+ bracketPasteFinishKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket201Dispatch = {
+ 1, "~", escLeftBracket201Routines
+};
+static char32_t escLeftBracket201Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket201Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket20Routines[] = {
+ f9KeyRoutine, escLeftBracket20SemicolonRoutine, escLeftBracket200Routine, escLeftBracket201Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20Dispatch = {
+ 4, "~;01", escLeftBracket20Routines
+};
+static char32_t escLeftBracket20Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket20Dispatch);
+}
+
+// (S)-F10
+static CharacterDispatchRoutine escLeftBracket21Semicolon2Routines[] = {
+ f10KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21Semicolon2Dispatch = {
+ 1, "~", escLeftBracket21Semicolon2Routines
+};
+static char32_t escLeftBracket21Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket21Semicolon2Dispatch);
+}
+
+// (C)-F10
+static CharacterDispatchRoutine escLeftBracket21Semicolon5Routines[] = {
+ f10KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21Semicolon5Dispatch = {
+ 1, "~", escLeftBracket21Semicolon5Routines
+};
+static char32_t escLeftBracket21Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket21Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket21SemicolonRoutines[] = {
+ escLeftBracket21Semicolon2Routine, escLeftBracket21Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21SemicolonDispatch = {
+ 2, "25", escLeftBracket21SemicolonRoutines
+};
+static char32_t escLeftBracket21SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket21SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket21Routines[] = {
+ f10KeyRoutine, escLeftBracket21SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21Dispatch = {
+ 2, "~;", escLeftBracket21Routines
+};
+static char32_t escLeftBracket21Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket21Dispatch);
+}
+
+// (S)-F11
+static CharacterDispatchRoutine escLeftBracket23Semicolon2Routines[] = {
+ f11KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23Semicolon2Dispatch = {
+ 1, "~", escLeftBracket23Semicolon2Routines
+};
+static char32_t escLeftBracket23Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket23Semicolon2Dispatch);
+}
+
+// (C)-F11
+static CharacterDispatchRoutine escLeftBracket23Semicolon5Routines[] = {
+ f11KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23Semicolon5Dispatch = {
+ 1, "~", escLeftBracket23Semicolon5Routines
+};
+static char32_t escLeftBracket23Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket23Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket23SemicolonRoutines[] = {
+ escLeftBracket23Semicolon2Routine, escLeftBracket23Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23SemicolonDispatch = {
+ 2, "25", escLeftBracket23SemicolonRoutines
+};
+static char32_t escLeftBracket23SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket23SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket23Routines[] = {
+ f11KeyRoutine, escLeftBracket23SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23Dispatch = {
+ 2, "~;", escLeftBracket23Routines
+};
+static char32_t escLeftBracket23Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket23Dispatch);
+}
+
+// (S)-F12
+static CharacterDispatchRoutine escLeftBracket24Semicolon2Routines[] = {
+ f12KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24Semicolon2Dispatch = {
+ 1, "~", escLeftBracket24Semicolon2Routines
+};
+static char32_t escLeftBracket24Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket24Semicolon2Dispatch);
+}
+
+// (C)-F12
+static CharacterDispatchRoutine escLeftBracket24Semicolon5Routines[] = {
+ f12KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24Semicolon5Dispatch = {
+ 1, "~", escLeftBracket24Semicolon5Routines
+};
+static char32_t escLeftBracket24Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket24Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket24SemicolonRoutines[] = {
+ escLeftBracket24Semicolon2Routine, escLeftBracket24Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24SemicolonDispatch = {
+ 2, "25", escLeftBracket24SemicolonRoutines
+};
+static char32_t escLeftBracket24SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket24SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket24Routines[] = {
+ f12KeyRoutine, escLeftBracket24SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24Dispatch = {
+ 2, "~;", escLeftBracket24Routines
+};
+static char32_t escLeftBracket24Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket24Dispatch);
+}
+
+// Handle ESC [ 2 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket2Routines[] = {
+ insertKeyRoutine,
+ escLeftBracket20Routine,
+ escLeftBracket21Routine,
+ escLeftBracket23Routine,
+ escLeftBracket24Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket2Dispatch = {
+ 5, "~0134", escLeftBracket2Routines
+};
+
+// Handle ESC [ 3 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket3Routines[] = {
+ deleteKeyRoutine, escFailureRoutine
+};
+
+static CharacterDispatch escLeftBracket3Dispatch = {
+ 1, "~", escLeftBracket3Routines
+};
+
+// Handle ESC [ 4 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket4Routines[] = {
+ endKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket4Dispatch = {
+ 1, "~", escLeftBracket4Routines
+};
+
+// Handle ESC [ 5 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket5Semicolon5Routines[] = {
+ pageUpKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket5Semicolon5Dispatch = {
+ 1, "~", escLeftBracket5Semicolon5Routines
+};
+static char32_t escLeftBracket5Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket5Semicolon5Dispatch);
+}
+static CharacterDispatchRoutine escLeftBracket5SemicolonRoutines[] = {
+ escLeftBracket5Semicolon5Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket5SemicolonDispatch = {
+ 1, "5", escLeftBracket5SemicolonRoutines
+};
+static char32_t escLeftBracket5SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket5SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket5Routines[] = {
+ pageUpKeyRoutine, escLeftBracket5SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket5Dispatch = {
+ 2, "~;", escLeftBracket5Routines
+};
+
+// Handle ESC [ 6 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket6Semicolon5Routines[] = {
+ pageDownKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket6Semicolon5Dispatch = {
+ 1, "~", escLeftBracket6Semicolon5Routines
+};
+static char32_t escLeftBracket6Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket6Semicolon5Dispatch);
+}
+static CharacterDispatchRoutine escLeftBracket6SemicolonRoutines[] = {
+ escLeftBracket6Semicolon5Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket6SemicolonDispatch = {
+ 1, "5", escLeftBracket6SemicolonRoutines
+};
+static char32_t escLeftBracket6SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket6SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket6Routines[] = {
+ pageDownKeyRoutine, escLeftBracket6SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket6Dispatch = {
+ 2, "~;", escLeftBracket6Routines
+};
+
+// Handle ESC [ 7 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket7Routines[] = {
+ homeKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket7Dispatch = {
+ 1, "~", escLeftBracket7Routines
+};
+
+// Handle ESC [ 8 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket8Routines[] = {
+ endKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket8Dispatch = {
+ 1, "~", escLeftBracket8Routines
+};
+
+// Handle ESC [ <digit> escape sequences
+//
+static char32_t escLeftBracket0Routine(char32_t c) {
+ return escFailureRoutine(c);
+}
+static char32_t escLeftBracket1Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket1Dispatch);
+}
+static char32_t escLeftBracket2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket2Dispatch);
+}
+static char32_t escLeftBracket3Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket3Dispatch);
+}
+static char32_t escLeftBracket4Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket4Dispatch);
+}
+static char32_t escLeftBracket5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket5Dispatch);
+}
+static char32_t escLeftBracket6Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket6Dispatch);
+}
+static char32_t escLeftBracket7Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket7Dispatch);
+}
+static char32_t escLeftBracket8Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket8Dispatch);
+}
+static char32_t escLeftBracket9Routine(char32_t c) {
+ return escFailureRoutine(c);
+}
+
+// Handle ESC [ <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracketRoutines[] = {
+ upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine,
+ leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine,
+ shiftTabRoutine,
+ escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine,
+ escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine,
+ escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine,
+ escLeftBracket9Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracketDispatch = {17, "ABCDHFZ0123456789",
+ escLeftBracketRoutines};
+
+// Handle ESC O <char> escape sequences
+//
+static CharacterDispatchRoutine escORoutines[] = {
+ upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine,
+ leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine,
+ f1KeyRoutine, f2KeyRoutine, f3KeyRoutine,
+ f4KeyRoutine,
+ ctrlUpArrowKeyRoutine, ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine,
+ ctrlLeftArrowKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escODispatch = {14, "ABCDHFPQRSabcd", escORoutines};
+
+// Initial ESC dispatch -- could be a Meta prefix or the start of an escape
+// sequence
+//
+static char32_t escLeftBracketRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracketDispatch);
+}
+static char32_t escORoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escODispatch);
+}
+static char32_t setMetaRoutine(char32_t c); // need forward reference
+static CharacterDispatchRoutine escRoutines[] = {
+ escLeftBracketRoutine, escORoutine, setMetaRoutine
+};
+static CharacterDispatch escDispatch = {2, "[O", escRoutines};
+
+// Initial dispatch -- we are not in the middle of anything yet
+//
+static char32_t escRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escDispatch);
+}
+static CharacterDispatchRoutine initialRoutines[] = {
+ escRoutine, deleteCharRoutine, normalKeyRoutine
+};
+static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines};
+
+// Special handling for the ESC key because it does double duty
+//
+static char32_t setMetaRoutine(char32_t c) {
+ thisKeyMetaCtrl = Replxx::KEY::BASE_META;
+ if (c == 0x1B) { // another ESC, stay in ESC processing mode
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escDispatch);
+ }
+ return doDispatch(c, initialDispatch);
+}
+
+char32_t doDispatch(char32_t c) {
+ EscapeSequenceProcessing::thisKeyMetaCtrl = 0; // no modifiers yet at initialDispatch
+ return doDispatch(c, initialDispatch);
+}
+
+} // namespace EscapeSequenceProcessing // move these out of global namespace
+
+}
+
+#endif /* #ifndef _WIN32 */
+
diff --git a/contrib/replxx/src/escape.hxx b/contrib/replxx/src/escape.hxx
new file mode 100644
index 0000000..6597395
--- /dev/null
+++ b/contrib/replxx/src/escape.hxx
@@ -0,0 +1,37 @@
+#ifndef REPLXX_ESCAPE_HXX_INCLUDED
+#define REPLXX_ESCAPE_HXX_INCLUDED 1
+
+namespace replxx {
+
+namespace EscapeSequenceProcessing {
+
+// This is a typedef for the routine called by doDispatch(). It takes the
+// current character
+// as input, does any required processing including reading more characters and
+// calling other
+// dispatch routines, then eventually returns the final (possibly extended or
+// special) character.
+//
+typedef char32_t (*CharacterDispatchRoutine)(char32_t);
+
+// This structure is used by doDispatch() to hold a list of characters to test
+// for and
+// a list of routines to call if the character matches. The dispatch routine
+// list is one
+// longer than the character list; the final entry is used if no character
+// matches.
+//
+struct CharacterDispatch {
+ unsigned int len; // length of the chars list
+ const char* chars; // chars to test
+ CharacterDispatchRoutine* dispatch; // array of routines to call
+};
+
+char32_t doDispatch(char32_t c);
+
+}
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/history.cxx b/contrib/replxx/src/history.cxx
new file mode 100644
index 0000000..fe691df
--- /dev/null
+++ b/contrib/replxx/src/history.cxx
@@ -0,0 +1,402 @@
+#include <algorithm>
+#include <memory>
+#include <fstream>
+#include <cstring>
+
+#ifndef _WIN32
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+#endif /* _WIN32 */
+
+#include "replxx.hxx"
+#include "history.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+namespace {
+void delete_ReplxxHistoryScanImpl( Replxx::HistoryScanImpl* impl_ ) {
+ delete impl_;
+}
+}
+
+static int const REPLXX_DEFAULT_HISTORY_MAX_LEN( 1000 );
+
+Replxx::HistoryScan::HistoryScan( impl_t impl_ )
+ : _impl( std::move( impl_ ) ) {
+}
+
+bool Replxx::HistoryScan::next( void ) {
+ return ( _impl->next() );
+}
+
+Replxx::HistoryScanImpl::HistoryScanImpl( History::entries_t const& entries_ )
+ : _entries( entries_ )
+ , _it( _entries.end() )
+ , _utf8Cache()
+ , _entryCache( std::string(), std::string() )
+ , _cacheValid( false ) {
+}
+
+Replxx::HistoryEntry const& Replxx::HistoryScan::get( void ) const {
+ return ( _impl->get() );
+}
+
+bool Replxx::HistoryScanImpl::next( void ) {
+ if ( _it == _entries.end() ) {
+ _it = _entries.begin();
+ } else {
+ ++ _it;
+ }
+ _cacheValid = false;
+ return ( _it != _entries.end() );
+}
+
+Replxx::HistoryEntry const& Replxx::HistoryScanImpl::get( void ) const {
+ if ( _cacheValid ) {
+ return ( _entryCache );
+ }
+ _utf8Cache.assign( _it->text() );
+ _entryCache = Replxx::HistoryEntry( _it->timestamp(), _utf8Cache.get() );
+ _cacheValid = true;
+ return ( _entryCache );
+}
+
+Replxx::HistoryScan::impl_t History::scan( void ) const {
+ return ( Replxx::HistoryScan::impl_t( new Replxx::HistoryScanImpl( _entries ), delete_ReplxxHistoryScanImpl ) );
+}
+
+History::History( void )
+ : _entries()
+ , _maxSize( REPLXX_DEFAULT_HISTORY_MAX_LEN )
+ , _current( _entries.begin() )
+ , _yankPos( _entries.end() )
+ , _previous( _entries.begin() )
+ , _recallMostRecent( false )
+ , _unique( true ) {
+}
+
+void History::add( UnicodeString const& line, std::string const& when ) {
+ if ( _maxSize <= 0 ) {
+ return;
+ }
+ if ( ! _entries.empty() && ( line == _entries.back().text() ) ) {
+ _entries.back() = Entry( now_ms_str(), line );
+ return;
+ }
+ remove_duplicate( line );
+ trim_to_max_size();
+ _entries.emplace_back( when, line );
+ _locations.insert( make_pair( line, last() ) );
+ if ( _current == _entries.end() ) {
+ _current = last();
+ }
+ _yankPos = _entries.end();
+}
+
+#ifndef _WIN32
+class FileLock {
+ std::string _path;
+ int _lockFd;
+public:
+ FileLock( std::string const& name_ )
+ : _path( name_ + ".lock" )
+ , _lockFd( ::open( _path.c_str(), O_CREAT | O_RDWR, 0600 ) ) {
+ static_cast<void>( ::lockf( _lockFd, F_LOCK, 0 ) == 0 );
+ }
+ ~FileLock( void ) {
+ static_cast<void>( ::lockf( _lockFd, F_ULOCK, 0 ) == 0 );
+ ::close( _lockFd );
+ ::unlink( _path.c_str() );
+ return;
+ }
+};
+#endif
+
+bool History::save( std::string const& filename, bool sync_ ) {
+#ifndef _WIN32
+ mode_t old_umask = umask( S_IXUSR | S_IRWXG | S_IRWXO );
+ FileLock fileLock( filename );
+#endif
+ entries_t entries;
+ locations_t locations;
+ if ( ! sync_ ) {
+ entries.swap( _entries );
+ locations.swap( _locations );
+ _entries = entries;
+ reset_iters();
+ }
+ do_load( filename );
+ sort();
+ remove_duplicates();
+ trim_to_max_size();
+ ofstream histFile( filename );
+ if ( ! histFile ) {
+ return ( false );
+ }
+#ifndef _WIN32
+ umask( old_umask );
+ chmod( filename.c_str(), S_IRUSR | S_IWUSR );
+#endif
+ Utf8String utf8;
+ for ( Entry const& h : _entries ) {
+ if ( ! h.text().is_empty() ) {
+ utf8.assign( h.text() );
+ histFile << "### " << h.timestamp() << "\n" << utf8.get() << endl;
+ }
+ }
+ if ( ! sync_ ) {
+ _entries = std::move( entries );
+ _locations = std::move( locations );
+ }
+ reset_iters();
+ return ( true );
+}
+
+namespace {
+
+bool is_timestamp( std::string const& s ) {
+ static char const TIMESTAMP_PATTERN[] = "### dddd-dd-dd dd:dd:dd.ddd";
+ static int const TIMESTAMP_LENGTH( sizeof ( TIMESTAMP_PATTERN ) - 1 );
+ if ( s.length() != TIMESTAMP_LENGTH ) {
+ return ( false );
+ }
+ for ( int i( 0 ); i < TIMESTAMP_LENGTH; ++ i ) {
+ if ( TIMESTAMP_PATTERN[i] == 'd' ) {
+ if ( ! isdigit( s[i] ) ) {
+ return ( false );
+ }
+ } else if ( s[i] != TIMESTAMP_PATTERN[i] ) {
+ return ( false );
+ }
+ }
+ return ( true );
+}
+
+}
+
+bool History::do_load( std::string const& filename ) {
+ ifstream histFile( filename );
+ if ( ! histFile ) {
+ return ( false );
+ }
+ string line;
+ string when( "0000-00-00 00:00:00.000" );
+ while ( getline( histFile, line ).good() ) {
+ string::size_type eol( line.find_first_of( "\r\n" ) );
+ if ( eol != string::npos ) {
+ line.erase( eol );
+ }
+ if ( is_timestamp( line ) ) {
+ when.assign( line, 4, std::string::npos );
+ continue;
+ }
+ if ( ! line.empty() ) {
+ _entries.emplace_back( when, UnicodeString( line ) );
+ }
+ }
+ return ( true );
+}
+
+bool History::load( std::string const& filename ) {
+ clear();
+ bool success( do_load( filename ) );
+ sort();
+ remove_duplicates();
+ trim_to_max_size();
+ _previous = _current = last();
+ _yankPos = _entries.end();
+ return ( success );
+}
+
+void History::sort( void ) {
+ typedef std::vector<Entry> sortable_entries_t;
+ _locations.clear();
+ sortable_entries_t sortableEntries( _entries.begin(), _entries.end() );
+ std::stable_sort( sortableEntries.begin(), sortableEntries.end() );
+ _entries.clear();
+ _entries.insert( _entries.begin(), sortableEntries.begin(), sortableEntries.end() );
+}
+
+void History::clear( void ) {
+ _locations.clear();
+ _entries.clear();
+ _current = _entries.begin();
+ _recallMostRecent = false;
+}
+
+void History::set_max_size( int size_ ) {
+ if ( size_ >= 0 ) {
+ _maxSize = size_;
+ trim_to_max_size();
+ }
+}
+
+void History::reset_yank_iterator( void ) {
+ _yankPos = _entries.end();
+}
+
+bool History::next_yank_position( void ) {
+ bool resetYankSize( false );
+ if ( _yankPos == _entries.end() ) {
+ resetYankSize = true;
+ }
+ if ( ( _yankPos != _entries.begin() ) && ( _yankPos != _entries.end() ) ) {
+ -- _yankPos;
+ } else {
+ _yankPos = moved( _entries.end(), -2 );
+ }
+ return ( resetYankSize );
+}
+
+bool History::move( bool up_ ) {
+ bool doRecall( _recallMostRecent && ! up_ );
+ if ( doRecall ) {
+ _current = _previous; // emulate Windows down-arrow
+ }
+ _recallMostRecent = false;
+ return ( doRecall || move( _current, up_ ? -1 : 1 ) );
+}
+
+void History::jump( bool start_, bool reset_ ) {
+ if ( start_ ) {
+ _current = _entries.begin();
+ } else {
+ _current = last();
+ }
+ if ( reset_ ) {
+ _recallMostRecent = false;
+ }
+}
+
+void History::save_pos( void ) {
+ _previous = _current;
+}
+
+void History::restore_pos( void ) {
+ _current = _previous;
+}
+
+bool History::common_prefix_search( UnicodeString const& prefix_, int prefixSize_, bool back_ ) {
+ int step( back_ ? -1 : 1 );
+ entries_t::const_iterator it( moved( _current, step, true ) );
+ while ( it != _current ) {
+ if ( it->text().starts_with( prefix_.begin(), prefix_.begin() + prefixSize_ ) ) {
+ _current = it;
+ commit_index();
+ return ( true );
+ }
+ move( it, step, true );
+ }
+ return ( false );
+}
+
+bool History::move( entries_t::const_iterator& it_, int by_, bool wrapped_ ) const {
+ if ( by_ > 0 ) {
+ for ( int i( 0 ); i < by_; ++ i ) {
+ ++ it_;
+ if ( it_ != _entries.end() ) {
+ } else if ( wrapped_ ) {
+ it_ = _entries.begin();
+ } else {
+ -- it_;
+ return ( false );
+ }
+ }
+ } else {
+ for ( int i( 0 ); i > by_; -- i ) {
+ if ( it_ != _entries.begin() ) {
+ -- it_;
+ } else if ( wrapped_ ) {
+ it_ = last();
+ } else {
+ return ( false );
+ }
+ }
+ }
+ return ( true );
+}
+
+History::entries_t::const_iterator History::moved( entries_t::const_iterator it_, int by_, bool wrapped_ ) const {
+ move( it_, by_, wrapped_ );
+ return ( it_ );
+}
+
+void History::erase( entries_t::const_iterator it_ ) {
+ bool invalidated( it_ == _current );
+ _locations.erase( it_->text() );
+ it_ = _entries.erase( it_ );
+ if ( invalidated ) {
+ _current = it_;
+ }
+ if ( ( _current == _entries.end() ) && ! _entries.empty() ) {
+ -- _current;
+ }
+ _yankPos = _entries.end();
+ _previous = _current;
+}
+
+void History::trim_to_max_size( void ) {
+ while ( size() > _maxSize ) {
+ erase( _entries.begin() );
+ }
+}
+
+void History::remove_duplicate( UnicodeString const& line_ ) {
+ if ( ! _unique ) {
+ return;
+ }
+ locations_t::iterator it( _locations.find( line_ ) );
+ if ( it == _locations.end() ) {
+ return;
+ }
+ erase( it->second );
+}
+
+void History::remove_duplicates( void ) {
+ if ( ! _unique ) {
+ return;
+ }
+ _locations.clear();
+ typedef std::pair<locations_t::iterator, bool> locations_insertion_result_t;
+ for ( entries_t::iterator it( _entries.begin() ), end( _entries.end() ); it != end; ++ it ) {
+ locations_insertion_result_t locationsInsertionResult( _locations.insert( make_pair( it->text(), it ) ) );
+ if ( ! locationsInsertionResult.second ) {
+ _entries.erase( locationsInsertionResult.first->second );
+ locationsInsertionResult.first->second = it;
+ }
+ }
+}
+
+void History::update_last( UnicodeString const& line_ ) {
+ if ( _unique ) {
+ _locations.erase( _entries.back().text() );
+ remove_duplicate( line_ );
+ _locations.insert( make_pair( line_, last() ) );
+ }
+ _entries.back() = Entry( now_ms_str(), line_ );
+}
+
+void History::drop_last( void ) {
+ erase( last() );
+}
+
+bool History::is_last( void ) const {
+ return ( _current == last() );
+}
+
+History::entries_t::const_iterator History::last( void ) const {
+ return ( moved( _entries.end(), -1 ) );
+}
+
+void History::reset_iters( void ) {
+ _previous = _current = last();
+ _yankPos = _entries.end();
+}
+
+}
+
diff --git a/contrib/replxx/src/history.hxx b/contrib/replxx/src/history.hxx
new file mode 100644
index 0000000..4e72c03
--- /dev/null
+++ b/contrib/replxx/src/history.hxx
@@ -0,0 +1,141 @@
+#ifndef REPLXX_HISTORY_HXX_INCLUDED
+#define REPLXX_HISTORY_HXX_INCLUDED 1
+
+#include <list>
+#include <unordered_map>
+
+#include "unicodestring.hxx"
+#include "utf8string.hxx"
+#include "conversion.hxx"
+#include "util.hxx"
+
+namespace std {
+template<>
+struct hash<replxx::UnicodeString> {
+ std::size_t operator()( replxx::UnicodeString const& us_ ) const {
+ std::size_t h( 0 );
+ char32_t const* p( us_.get() );
+ char32_t const* e( p + us_.length() );
+ while ( p != e ) {
+ h *= 31;
+ h += *p;
+ ++ p;
+ }
+ return ( h );
+ }
+};
+}
+
+namespace replxx {
+
+class History {
+public:
+ class Entry {
+ std::string _timestamp;
+ UnicodeString _text;
+ public:
+ Entry( std::string const& timestamp_, UnicodeString const& text_ )
+ : _timestamp( timestamp_ )
+ , _text( text_ ) {
+ }
+ std::string const& timestamp( void ) const {
+ return ( _timestamp );
+ }
+ UnicodeString const& text( void ) const {
+ return ( _text );
+ }
+ bool operator < ( Entry const& other_ ) const {
+ return ( _timestamp < other_._timestamp );
+ }
+ };
+ typedef std::list<Entry> entries_t;
+ typedef std::unordered_map<UnicodeString, entries_t::const_iterator> locations_t;
+private:
+ entries_t _entries;
+ locations_t _locations;
+ int _maxSize;
+ entries_t::const_iterator _current;
+ entries_t::const_iterator _yankPos;
+ /*
+ * _previous and _recallMostRecent are used to allow
+ * HISTORY_NEXT action (a down-arrow key) to have a special meaning
+ * if invoked after a line from history was accepted without
+ * any modification.
+ * Special meaning is: a down arrow shall jump to the line one
+ * after previously accepted from history.
+ */
+ entries_t::const_iterator _previous;
+ bool _recallMostRecent;
+ bool _unique;
+public:
+ History( void );
+ void add( UnicodeString const& line, std::string const& when = now_ms_str() );
+ bool save( std::string const& filename, bool );
+ bool load( std::string const& filename );
+ void clear( void );
+ void set_max_size( int len );
+ void set_unique( bool unique_ ) {
+ _unique = unique_;
+ remove_duplicates();
+ }
+ void reset_yank_iterator();
+ bool next_yank_position( void );
+ void reset_recall_most_recent( void ) {
+ _recallMostRecent = false;
+ }
+ void commit_index( void ) {
+ _previous = _current;
+ _recallMostRecent = true;
+ }
+ bool is_empty( void ) const {
+ return ( _entries.empty() );
+ }
+ void update_last( UnicodeString const& );
+ void drop_last( void );
+ bool is_last( void ) const;
+ bool move( bool );
+ UnicodeString const& current( void ) const {
+ return ( _current->text() );
+ }
+ UnicodeString const& yank_line( void ) const {
+ return ( _yankPos->text() );
+ }
+ void jump( bool, bool = true );
+ bool common_prefix_search( UnicodeString const&, int, bool );
+ int size( void ) const {
+ return ( static_cast<int>( _entries.size() ) );
+ }
+ Replxx::HistoryScan::impl_t scan( void ) const;
+ void save_pos( void );
+ void restore_pos( void );
+private:
+ History( History const& ) = delete;
+ History& operator = ( History const& ) = delete;
+ bool move( entries_t::const_iterator&, int, bool = false ) const;
+ entries_t::const_iterator moved( entries_t::const_iterator, int, bool = false ) const;
+ void erase( entries_t::const_iterator );
+ void trim_to_max_size( void );
+ void remove_duplicate( UnicodeString const& );
+ void remove_duplicates( void );
+ bool do_load( std::string const& );
+ entries_t::const_iterator last( void ) const;
+ void sort( void );
+ void reset_iters( void );
+};
+
+class Replxx::HistoryScanImpl {
+ History::entries_t const& _entries;
+ History::entries_t::const_iterator _it;
+ mutable Utf8String _utf8Cache;
+ mutable Replxx::HistoryEntry _entryCache;
+ mutable bool _cacheValid;
+public:
+ HistoryScanImpl( History::entries_t const& );
+ bool next( void );
+ Replxx::HistoryEntry const& get( void ) const;
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/killring.hxx b/contrib/replxx/src/killring.hxx
new file mode 100644
index 0000000..0baf108
--- /dev/null
+++ b/contrib/replxx/src/killring.hxx
@@ -0,0 +1,78 @@
+#ifndef REPLXX_KILLRING_HXX_INCLUDED
+#define REPLXX_KILLRING_HXX_INCLUDED 1
+
+#include <vector>
+
+#include "unicodestring.hxx"
+
+namespace replxx {
+
+class KillRing {
+ static const int capacity = 10;
+ int size;
+ int index;
+ char indexToSlot[10];
+ std::vector<UnicodeString> theRing;
+
+public:
+ enum action { actionOther, actionKill, actionYank };
+ action lastAction;
+
+ KillRing()
+ : size(0)
+ , index(0)
+ , lastAction(actionOther) {
+ theRing.reserve(capacity);
+ }
+
+ void kill(const char32_t* text, int textLen, bool forward) {
+ if (textLen == 0) {
+ return;
+ }
+ UnicodeString killedText(text, textLen);
+ if (lastAction == actionKill && size > 0) {
+ int slot = indexToSlot[0];
+ int currentLen = static_cast<int>(theRing[slot].length());
+ UnicodeString temp;
+ if ( forward ) {
+ temp.append( theRing[slot].get(), currentLen ).append( killedText.get(), textLen );
+ } else {
+ temp.append( killedText.get(), textLen ).append( theRing[slot].get(), currentLen );
+ }
+ theRing[slot] = temp;
+ } else {
+ if (size < capacity) {
+ if (size > 0) {
+ memmove(&indexToSlot[1], &indexToSlot[0], size);
+ }
+ indexToSlot[0] = size;
+ size++;
+ theRing.push_back(killedText);
+ } else {
+ int slot = indexToSlot[capacity - 1];
+ theRing[slot] = killedText;
+ memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1);
+ indexToSlot[0] = slot;
+ }
+ index = 0;
+ }
+ }
+
+ UnicodeString* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; }
+
+ UnicodeString* yankPop() {
+ if (size == 0) {
+ return 0;
+ }
+ ++index;
+ if (index == size) {
+ index = 0;
+ }
+ return &theRing[indexToSlot[index]];
+ }
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/prompt.cxx b/contrib/replxx/src/prompt.cxx
new file mode 100644
index 0000000..c13ea80
--- /dev/null
+++ b/contrib/replxx/src/prompt.cxx
@@ -0,0 +1,144 @@
+#ifdef _WIN32
+
+#include <conio.h>
+#include <windows.h>
+#include <io.h>
+#if _MSC_VER < 1900 && defined (_MSC_VER)
+#define snprintf _snprintf // Microsoft headers use underscores in some names
+#endif
+#define strcasecmp _stricmp
+#define strdup _strdup
+#define write _write
+#define STDIN_FILENO 0
+
+#else /* _WIN32 */
+
+#include <unistd.h>
+
+#endif /* _WIN32 */
+
+#include "prompt.hxx"
+#include "util.hxx"
+
+namespace replxx {
+
+Prompt::Prompt( Terminal& terminal_ )
+ : _extraLines( 0 )
+ , _lastLinePosition( 0 )
+ , _cursorRowOffset( 0 )
+ , _screenColumns( 0 )
+ , _terminal( terminal_ ) {
+}
+
+void Prompt::write() {
+ _terminal.write32( _text.get(), _text.length() );
+}
+
+void Prompt::update_screen_columns( void ) {
+ _screenColumns = _terminal.get_screen_columns();
+}
+
+void Prompt::set_text( UnicodeString const& text_ ) {
+ _text = text_;
+ update_state();
+}
+
+void Prompt::update_state() {
+ _cursorRowOffset -= _extraLines;
+ _extraLines = 0;
+ _lastLinePosition = 0;
+ _screenColumns = 0;
+ update_screen_columns();
+ // strip control characters from the prompt -- we do allow newline
+ UnicodeString::const_iterator in( _text.begin() );
+ UnicodeString::iterator out( _text.begin() );
+
+ int visibleCount = 0;
+ int x = 0;
+
+ bool const strip = !tty::out;
+
+ while (in != _text.end()) {
+ char32_t c = *in;
+ if ('\n' == c || !is_control_code(c)) {
+ *out = c;
+ ++out;
+ ++in;
+ ++visibleCount;
+ if ('\n' == c || ++x >= _screenColumns) {
+ x = 0;
+ ++_extraLines;
+ _lastLinePosition = visibleCount;
+ }
+ } else if (c == '\x1b') {
+ if ( strip ) {
+ // jump over control chars
+ ++in;
+ if (*in == '[') {
+ ++in;
+ while ( ( in != _text.end() ) && ( ( *in == ';' ) || ( ( ( *in >= '0' ) && ( *in <= '9' ) ) ) ) ) {
+ ++in;
+ }
+ if (*in == 'm') {
+ ++in;
+ }
+ }
+ } else {
+ // copy control chars
+ *out = *in;
+ ++out;
+ ++in;
+ if (*in == '[') {
+ *out = *in;
+ ++out;
+ ++in;
+ while ( ( in != _text.end() ) && ( ( *in == ';' ) || ( ( ( *in >= '0' ) && ( *in <= '9' ) ) ) ) ) {
+ *out = *in;
+ ++out;
+ ++in;
+ }
+ if (*in == 'm') {
+ *out = *in;
+ ++out;
+ ++in;
+ }
+ }
+ }
+ } else {
+ ++in;
+ }
+ }
+ _characterCount = visibleCount;
+ int charCount( static_cast<int>( out - _text.begin() ) );
+ _text.erase( charCount, _text.length() - charCount );
+
+ _cursorRowOffset += _extraLines;
+}
+
+int Prompt::indentation() const {
+ return _characterCount - _lastLinePosition;
+}
+
+// Used with DynamicPrompt (history search)
+//
+const UnicodeString forwardSearchBasePrompt("(i-search)`");
+const UnicodeString reverseSearchBasePrompt("(reverse-i-search)`");
+const UnicodeString endSearchBasePrompt("': ");
+
+DynamicPrompt::DynamicPrompt( Terminal& terminal_, int initialDirection )
+ : Prompt( terminal_ )
+ , _searchText()
+ , _direction( initialDirection ) {
+ updateSearchPrompt();
+}
+
+void DynamicPrompt::updateSearchPrompt(void) {
+ update_screen_columns();
+ const UnicodeString* basePrompt =
+ (_direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
+ _text.assign( *basePrompt ).append( _searchText ).append( endSearchBasePrompt );
+ update_state();
+}
+
+}
+
diff --git a/contrib/replxx/src/prompt.hxx b/contrib/replxx/src/prompt.hxx
new file mode 100644
index 0000000..9ed3f5f
--- /dev/null
+++ b/contrib/replxx/src/prompt.hxx
@@ -0,0 +1,45 @@
+#ifndef REPLXX_PROMPT_HXX_INCLUDED
+#define REPLXX_PROMPT_HXX_INCLUDED 1
+
+#include <cstdlib>
+
+#include "unicodestring.hxx"
+#include "terminal.hxx"
+
+namespace replxx {
+
+class Prompt { // a convenience struct for grouping prompt info
+public:
+ UnicodeString _text; // our copy of the prompt text, edited
+ int _characterCount; // visible characters in _text
+ int _extraLines; // extra lines (beyond 1) occupied by prompt
+ int _lastLinePosition; // index into _text where last line begins
+ int _cursorRowOffset; // where the cursor is relative to the start of the prompt
+private:
+ int _screenColumns; // width of screen in columns [cache]
+ Terminal& _terminal;
+public:
+ Prompt( Terminal& );
+ void set_text( UnicodeString const& textPtr );
+ void update_state();
+ void update_screen_columns( void );
+ int screen_columns() const {
+ return ( _screenColumns );
+ }
+ void write();
+ int indentation() const;
+};
+
+// changing prompt for "(reverse-i-search)`text':" etc.
+//
+struct DynamicPrompt : public Prompt {
+ UnicodeString _searchText; // text we are searching for
+ int _direction; // current search _direction, 1=forward, -1=reverse
+
+ DynamicPrompt( Terminal&, int initialDirection );
+ void updateSearchPrompt(void);
+};
+
+}
+
+#endif
diff --git a/contrib/replxx/src/replxx.cxx b/contrib/replxx/src/replxx.cxx
new file mode 100644
index 0000000..29d35a2
--- /dev/null
+++ b/contrib/replxx/src/replxx.cxx
@@ -0,0 +1,648 @@
+/*
+ * Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * You can find the latest source code at:
+ *
+ * http://github.com/antirez/linenoise
+ *
+ * Does a number of crazy assumptions that happen to be true in 99.9999% of
+ * the 2010 UNIX computers around.
+ *
+ * References:
+ * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
+ * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
+ *
+ * Todo list:
+ * - Switch to gets() if $TERM is something we can't support.
+ * - Filter bogus Ctrl+<char> combinations.
+ * - Win32 support
+ *
+ * Bloat:
+ * - Completion?
+ * - History search like Ctrl+r in readline?
+ *
+ * List of escape sequences used by this program, we do everything just
+ * with three sequences. In order to be so cheap we may have some
+ * flickering effect with some slow terminal, but the lesser sequences
+ * the more compatible.
+ *
+ * CHA (Cursor Horizontal Absolute)
+ * Sequence: ESC [ n G
+ * Effect: moves cursor to column n (1 based)
+ *
+ * EL (Erase Line)
+ * Sequence: ESC [ n K
+ * Effect: if n is 0 or missing, clear from cursor to end of line
+ * Effect: if n is 1, clear from beginning of line to cursor
+ * Effect: if n is 2, clear entire line
+ *
+ * CUF (Cursor Forward)
+ * Sequence: ESC [ n C
+ * Effect: moves cursor forward of n chars
+ *
+ * The following are used to clear the screen: ESC [ H ESC [ 2 J
+ * This is actually composed of two sequences:
+ *
+ * cursorhome
+ * Sequence: ESC [ H
+ * Effect: moves the cursor to upper left corner
+ *
+ * ED2 (Clear entire screen)
+ * Sequence: ESC [ 2 J
+ * Effect: clear the whole screen
+ *
+ */
+
+#include <algorithm>
+#include <cstdarg>
+
+#ifdef _WIN32
+
+#include <io.h>
+#define STDIN_FILENO 0
+
+#else /* _WIN32 */
+
+#include <signal.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#endif /* _WIN32 */
+
+#include "replxx.h"
+#include "replxx.hxx"
+#include "replxx_impl.hxx"
+#include "history.hxx"
+
+static_assert(
+ static_cast<int>( replxx::Replxx::ACTION::SEND_EOF ) == static_cast<int>( REPLXX_ACTION_SEND_EOF ),
+ "C and C++ `ACTION` APIs are missaligned!"
+);
+
+static_assert(
+ static_cast<int>( replxx::Replxx::KEY::PASTE_FINISH ) == static_cast<int>( REPLXX_KEY_PASTE_FINISH ),
+ "C and C++ `KEY` APIs are missaligned!"
+);
+
+using namespace std;
+using namespace std::placeholders;
+using namespace replxx;
+
+namespace replxx {
+
+namespace {
+void delete_ReplxxImpl( Replxx::ReplxxImpl* impl_ ) {
+ delete impl_;
+}
+}
+
+Replxx::Replxx( void )
+ : _impl( new Replxx::ReplxxImpl( nullptr, nullptr, nullptr ), delete_ReplxxImpl ) {
+}
+
+void Replxx::set_completion_callback( completion_callback_t const& fn ) {
+ _impl->set_completion_callback( fn );
+}
+
+void Replxx::set_modify_callback( modify_callback_t const& fn ) {
+ _impl->set_modify_callback( fn );
+}
+
+void Replxx::set_highlighter_callback( highlighter_callback_t const& fn ) {
+ _impl->set_highlighter_callback( fn );
+}
+
+void Replxx::set_hint_callback( hint_callback_t const& fn ) {
+ _impl->set_hint_callback( fn );
+}
+
+char const* Replxx::input( std::string const& prompt ) {
+ return ( _impl->input( prompt ) );
+}
+
+void Replxx::history_add( std::string const& line ) {
+ _impl->history_add( line );
+}
+
+bool Replxx::history_sync( std::string const& filename ) {
+ return ( _impl->history_sync( filename ) );
+}
+
+bool Replxx::history_save( std::string const& filename ) {
+ return ( _impl->history_save( filename ) );
+}
+
+bool Replxx::history_load( std::string const& filename ) {
+ return ( _impl->history_load( filename ) );
+}
+
+void Replxx::history_clear( void ) {
+ _impl->history_clear();
+}
+
+int Replxx::history_size( void ) const {
+ return ( _impl->history_size() );
+}
+
+Replxx::HistoryScan Replxx::history_scan( void ) const {
+ return ( _impl->history_scan() );
+}
+
+void Replxx::set_preload_buffer( std::string const& preloadText ) {
+ _impl->set_preload_buffer( preloadText );
+}
+
+void Replxx::set_word_break_characters( char const* wordBreakers ) {
+ _impl->set_word_break_characters( wordBreakers );
+}
+
+void Replxx::set_max_hint_rows( int count ) {
+ _impl->set_max_hint_rows( count );
+}
+
+void Replxx::set_hint_delay( int milliseconds ) {
+ _impl->set_hint_delay( milliseconds );
+}
+
+void Replxx::set_completion_count_cutoff( int count ) {
+ _impl->set_completion_count_cutoff( count );
+}
+
+void Replxx::set_double_tab_completion( bool val ) {
+ _impl->set_double_tab_completion( val );
+}
+
+void Replxx::set_complete_on_empty( bool val ) {
+ _impl->set_complete_on_empty( val );
+}
+
+void Replxx::set_beep_on_ambiguous_completion( bool val ) {
+ _impl->set_beep_on_ambiguous_completion( val );
+}
+
+void Replxx::set_immediate_completion( bool val ) {
+ _impl->set_immediate_completion( val );
+}
+
+void Replxx::set_unique_history( bool val ) {
+ _impl->set_unique_history( val );
+}
+
+void Replxx::set_no_color( bool val ) {
+ _impl->set_no_color( val );
+}
+
+void Replxx::set_max_history_size( int len ) {
+ _impl->set_max_history_size( len );
+}
+
+void Replxx::clear_screen( void ) {
+ _impl->clear_screen( 0 );
+}
+
+void Replxx::emulate_key_press( char32_t keyPress_ ) {
+ _impl->emulate_key_press( keyPress_ );
+}
+
+Replxx::ACTION_RESULT Replxx::invoke( ACTION action_, char32_t keyPress_ ) {
+ return ( _impl->invoke( action_, keyPress_ ) );
+}
+
+void Replxx::bind_key( char32_t keyPress_, key_press_handler_t handler_ ) {
+ _impl->bind_key( keyPress_, handler_ );
+}
+
+void Replxx::bind_key_internal( char32_t keyPress_, char const* actionName_ ) {
+ _impl->bind_key_internal( keyPress_, actionName_ );
+}
+
+Replxx::State Replxx::get_state( void ) const {
+ return ( _impl->get_state() );
+}
+
+void Replxx::set_state( Replxx::State const& state_ ) {
+ _impl->set_state( state_ );
+}
+
+int Replxx::install_window_change_handler( void ) {
+ return ( _impl->install_window_change_handler() );
+}
+
+void Replxx::enable_bracketed_paste( void ) {
+ _impl->enable_bracketed_paste();
+}
+
+void Replxx::disable_bracketed_paste( void ) {
+ _impl->disable_bracketed_paste();
+}
+
+void Replxx::print( char const* format_, ... ) {
+ ::std::va_list ap;
+ va_start( ap, format_ );
+ int size = static_cast<int>( vsnprintf( nullptr, 0, format_, ap ) );
+ va_end( ap );
+ va_start( ap, format_ );
+ unique_ptr<char[]> buf( new char[size + 1] );
+ vsnprintf( buf.get(), static_cast<size_t>( size + 1 ), format_, ap );
+ va_end( ap );
+ return ( _impl->print( buf.get(), size ) );
+}
+
+void Replxx::write( char const* str, int length ) {
+ return ( _impl->print( str, length ) );
+}
+
+}
+
+::Replxx* replxx_init() {
+ typedef ::Replxx* replxx_data_t;
+ return ( reinterpret_cast<replxx_data_t>( new replxx::Replxx::ReplxxImpl( nullptr, nullptr, nullptr ) ) );
+}
+
+void replxx_end( ::Replxx* replxx_ ) {
+ delete reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ );
+}
+
+void replxx_clear_screen( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->clear_screen( 0 );
+}
+
+void replxx_emulate_key_press( ::Replxx* replxx_, int unsigned keyPress_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->emulate_key_press( keyPress_ );
+}
+
+ReplxxActionResult replxx_invoke( ::Replxx* replxx_, ReplxxAction action_, int unsigned keyPress_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( static_cast<ReplxxActionResult>( replxx->invoke( static_cast<replxx::Replxx::ACTION>( action_ ), keyPress_ ) ) );
+}
+
+replxx::Replxx::ACTION_RESULT key_press_handler_forwarder( key_press_handler_t handler_, char32_t code_, void* userData_ ) {
+ return ( static_cast<replxx::Replxx::ACTION_RESULT>( handler_( code_, userData_ ) ) );
+}
+
+void replxx_bind_key( ::Replxx* replxx_, int code_, key_press_handler_t handler_, void* userData_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->bind_key( code_, std::bind( key_press_handler_forwarder, handler_, _1, userData_ ) );
+}
+
+int replxx_bind_key_internal( ::Replxx* replxx_, int code_, char const* actionName_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ try {
+ replxx->bind_key_internal( code_, actionName_ );
+ } catch ( ... ) {
+ return ( -1 );
+ }
+ return ( 0 );
+}
+
+void replxx_get_state( ::Replxx* replxx_, ReplxxState* state ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx::Replxx::State s( replxx->get_state() );
+ state->text = s.text();
+ state->cursorPosition = s.cursor_position();
+}
+
+void replxx_set_state( ::Replxx* replxx_, ReplxxState* state ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_state( replxx::Replxx::State( state->text, state->cursorPosition ) );
+}
+
+/**
+ * replxx_set_preload_buffer provides text to be inserted into the command buffer
+ *
+ * the provided text will be processed to be usable and will be used to preload
+ * the input buffer on the next call to replxx_input()
+ *
+ * @param preloadText text to begin with on the next call to replxx_input()
+ */
+void replxx_set_preload_buffer(::Replxx* replxx_, const char* preloadText) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_preload_buffer( preloadText ? preloadText : "" );
+}
+
+/**
+ * replxx_input is a readline replacement.
+ *
+ * call it with a prompt to display and it will return a line of input from the
+ * user
+ *
+ * @param prompt text of prompt to display to the user
+ * @return the returned string is managed by replxx library
+ * and it must NOT be freed in the client.
+ */
+char const* replxx_input( ::Replxx* replxx_, const char* prompt ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->input( prompt ) );
+}
+
+int replxx_print( ::Replxx* replxx_, char const* format_, ... ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ ::std::va_list ap;
+ va_start( ap, format_ );
+ int size = static_cast<int>( vsnprintf( nullptr, 0, format_, ap ) );
+ va_end( ap );
+ va_start( ap, format_ );
+ unique_ptr<char[]> buf( new char[size + 1] );
+ vsnprintf( buf.get(), static_cast<size_t>( size + 1 ), format_, ap );
+ va_end( ap );
+ try {
+ replxx->print( buf.get(), size );
+ } catch ( ... ) {
+ return ( -1 );
+ }
+ return ( size );
+}
+
+int replxx_write( ::Replxx* replxx_, char const* str, int length ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ try {
+ replxx->print( str, length );
+ } catch ( ... ) {
+ return ( -1 );
+ }
+ return static_cast<int>( length );
+}
+
+struct replxx_completions {
+ replxx::Replxx::completions_t data;
+};
+
+struct replxx_hints {
+ replxx::Replxx::hints_t data;
+};
+
+void modify_fwd( replxx_modify_callback_t fn, std::string& line_, int& cursorPosition_, void* userData_ ) {
+#ifdef _WIN32
+#define strdup _strdup
+#endif
+ char* s( strdup( line_.c_str() ) );
+#undef strdup
+ fn( &s, &cursorPosition_, userData_ );
+ line_ = s;
+ free( s );
+ return;
+}
+
+void replxx_set_modify_callback(::Replxx* replxx_, replxx_modify_callback_t* fn, void* userData) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_modify_callback( std::bind( &modify_fwd, fn, _1, _2, userData ) );
+}
+
+replxx::Replxx::completions_t completions_fwd( replxx_completion_callback_t fn, std::string const& input_, int& contextLen_, void* userData ) {
+ replxx_completions completions;
+ fn( input_.c_str(), &completions, &contextLen_, userData );
+ return ( completions.data );
+}
+
+/* Register a callback function to be called for tab-completion. */
+void replxx_set_completion_callback(::Replxx* replxx_, replxx_completion_callback_t* fn, void* userData) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_completion_callback( std::bind( &completions_fwd, fn, _1, _2, userData ) );
+}
+
+void highlighter_fwd( replxx_highlighter_callback_t fn, std::string const& input, replxx::Replxx::colors_t& colors, void* userData ) {
+ std::vector<ReplxxColor> colorsTmp( colors.size() );
+ std::transform(
+ colors.begin(),
+ colors.end(),
+ colorsTmp.begin(),
+ []( replxx::Replxx::Color c ) {
+ return ( static_cast<ReplxxColor>( c ) );
+ }
+ );
+ fn( input.c_str(), colorsTmp.data(), static_cast<int>( colors.size() ), userData );
+ std::transform(
+ colorsTmp.begin(),
+ colorsTmp.end(),
+ colors.begin(),
+ []( ReplxxColor c ) {
+ return ( static_cast<replxx::Replxx::Color>( c ) );
+ }
+ );
+}
+
+void replxx_set_highlighter_callback( ::Replxx* replxx_, replxx_highlighter_callback_t* fn, void* userData ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_highlighter_callback( std::bind( &highlighter_fwd, fn, _1, _2, userData ) );
+}
+
+replxx::Replxx::hints_t hints_fwd( replxx_hint_callback_t fn, std::string const& input_, int& contextLen_, replxx::Replxx::Color& color_, void* userData ) {
+ replxx_hints hints;
+ ReplxxColor c( static_cast<ReplxxColor>( color_ ) );
+ fn( input_.c_str(), &hints, &contextLen_, &c, userData );
+ return ( hints.data );
+}
+
+void replxx_set_hint_callback( ::Replxx* replxx_, replxx_hint_callback_t* fn, void* userData ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_hint_callback( std::bind( &hints_fwd, fn, _1, _2, _3, userData ) );
+}
+
+void replxx_add_hint(replxx_hints* lh, const char* str) {
+ lh->data.emplace_back(str);
+}
+
+void replxx_add_completion( replxx_completions* lc, const char* str ) {
+ lc->data.emplace_back( str );
+}
+
+void replxx_add_color_completion( replxx_completions* lc, const char* str, ReplxxColor color ) {
+ lc->data.emplace_back( str, static_cast<replxx::Replxx::Color>( color ) );
+}
+
+void replxx_history_add( ::Replxx* replxx_, const char* line ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->history_add( line );
+}
+
+void replxx_set_max_history_size( ::Replxx* replxx_, int len ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_max_history_size( len );
+}
+
+void replxx_set_max_hint_rows( ::Replxx* replxx_, int count ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_max_hint_rows( count );
+}
+
+void replxx_set_hint_delay( ::Replxx* replxx_, int milliseconds ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_hint_delay( milliseconds );
+}
+
+void replxx_set_completion_count_cutoff( ::Replxx* replxx_, int count ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_completion_count_cutoff( count );
+}
+
+void replxx_set_word_break_characters( ::Replxx* replxx_, char const* breakChars_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_word_break_characters( breakChars_ );
+}
+
+void replxx_set_double_tab_completion( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_double_tab_completion( val ? true : false );
+}
+
+void replxx_set_complete_on_empty( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_complete_on_empty( val ? true : false );
+}
+
+void replxx_set_no_color( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_no_color( val ? true : false );
+}
+
+void replxx_set_beep_on_ambiguous_completion( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_beep_on_ambiguous_completion( val ? true : false );
+}
+
+void replxx_set_immediate_completion( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_immediate_completion( val ? true : false );
+}
+
+void replxx_set_unique_history( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_unique_history( val ? true : false );
+}
+
+void replxx_enable_bracketed_paste( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->enable_bracketed_paste();
+}
+
+void replxx_disable_bracketed_paste( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->disable_bracketed_paste();
+}
+
+ReplxxHistoryScan* replxx_history_scan_start( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( reinterpret_cast<ReplxxHistoryScan*>( replxx->history_scan().release() ) );
+}
+
+void replxx_history_scan_stop( ::Replxx*, ReplxxHistoryScan* historyScan_ ) {
+ delete reinterpret_cast<replxx::Replxx::HistoryScanImpl*>( historyScan_ );
+}
+
+int replxx_history_scan_next( ::Replxx*, ReplxxHistoryScan* historyScan_, ReplxxHistoryEntry* historyEntry_ ) {
+ replxx::Replxx::HistoryScanImpl* historyScan( reinterpret_cast<replxx::Replxx::HistoryScanImpl*>( historyScan_ ) );
+ bool hasNext( historyScan->next() );
+ if ( hasNext ) {
+ replxx::Replxx::HistoryEntry const& historyEntry( historyScan->get() );
+ historyEntry_->timestamp = historyEntry.timestamp().c_str();
+ historyEntry_->text = historyEntry.text().c_str();
+ }
+ return ( hasNext ? 0 : -1 );
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int replxx_history_sync( ::Replxx* replxx_, const char* filename ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_sync( filename ) ? 0 : -1 );
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int replxx_history_save( ::Replxx* replxx_, const char* filename ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_save( filename ) ? 0 : -1 );
+}
+
+/* Load the history from the specified file. If the file does not exist
+ * zero is returned and no operation is performed.
+ *
+ * If the file exists and the operation succeeded 0 is returned, otherwise
+ * on error -1 is returned. */
+int replxx_history_load( ::Replxx* replxx_, const char* filename ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_load( filename ) ? 0 : -1 );
+}
+
+void replxx_history_clear( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->history_clear();
+}
+
+int replxx_history_size( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_size() );
+}
+
+/* This special mode is used by replxx in order to print scan codes
+ * on screen for debugging / development purposes. It is implemented
+ * by the replxx-c-api-example program using the --keycodes option. */
+#ifdef __REPLXX_DEBUG__
+void replxx_debug_dump_print_codes(void) {
+ char quit[4];
+
+ printf(
+ "replxx key codes debugging mode.\n"
+ "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
+ if (enableRawMode() == -1) return;
+ memset(quit, ' ', 4);
+ while (1) {
+ char c;
+ int nread;
+
+#if _WIN32
+ nread = _read(STDIN_FILENO, &c, 1);
+#else
+ nread = read(STDIN_FILENO, &c, 1);
+#endif
+ if (nread <= 0) continue;
+ memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */
+ quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */
+ if (memcmp(quit, "quit", sizeof(quit)) == 0) break;
+
+ printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c,
+ (int)c);
+ printf("\r"); /* Go left edge manually, we are in raw mode. */
+ fflush(stdout);
+ }
+ disableRawMode();
+}
+#endif // __REPLXX_DEBUG__
+
+int replxx_install_window_change_handler( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->install_window_change_handler() );
+}
+
diff --git a/contrib/replxx/src/replxx_impl.cxx b/contrib/replxx/src/replxx_impl.cxx
new file mode 100644
index 0000000..28152d5
--- /dev/null
+++ b/contrib/replxx/src/replxx_impl.cxx
@@ -0,0 +1,2248 @@
+#include <algorithm>
+#include <memory>
+#include <cerrno>
+#include <iostream>
+#include <chrono>
+
+#ifdef _WIN32
+
+#include <windows.h>
+#include <io.h>
+#if _MSC_VER < 1900
+#define snprintf _snprintf // Microsoft headers use underscores in some names
+#endif
+#define strcasecmp _stricmp
+#define write _write
+#define STDIN_FILENO 0
+
+#else /* _WIN32 */
+
+#include <unistd.h>
+#include <signal.h>
+
+#endif /* _WIN32 */
+
+#ifdef _WIN32
+#include "windows.hxx"
+#endif
+
+#include "replxx_impl.hxx"
+#include "utf8string.hxx"
+#include "prompt.hxx"
+#include "util.hxx"
+#include "terminal.hxx"
+#include "history.hxx"
+#include "replxx.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+namespace {
+
+namespace action_names {
+
+char const INSERT_CHARACTER[] = "insert_character";
+char const NEW_LINE[] = "new_line";
+char const MOVE_CURSOR_TO_BEGINING_OF_LINE[] = "move_cursor_to_begining_of_line";
+char const MOVE_CURSOR_TO_END_OF_LINE[] = "move_cursor_to_end_of_line";
+char const MOVE_CURSOR_LEFT[] = "move_cursor_left";
+char const MOVE_CURSOR_RIGHT[] = "move_cursor_right";
+char const MOVE_CURSOR_ONE_WORD_LEFT[] = "move_cursor_one_word_left";
+char const MOVE_CURSOR_ONE_WORD_RIGHT[] = "move_cursor_one_word_right";
+char const MOVE_CURSOR_ONE_SUBWORD_LEFT[] = "move_cursor_one_subword_left";
+char const MOVE_CURSOR_ONE_SUBWORD_RIGHT[] = "move_cursor_one_subword_right";
+char const KILL_TO_WHITESPACE_ON_LEFT[] = "kill_to_whitespace_on_left";
+char const KILL_TO_END_OF_WORD[] = "kill_to_end_of_word";
+char const KILL_TO_END_OF_SUBWORD[] = "kill_to_end_of_subword";
+char const KILL_TO_BEGINING_OF_WORD[] = "kill_to_begining_of_word";
+char const KILL_TO_BEGINING_OF_SUBWORD[] = "kill_to_begining_of_subword";
+char const KILL_TO_BEGINING_OF_LINE[] = "kill_to_begining_of_line";
+char const KILL_TO_END_OF_LINE[] = "kill_to_end_of_line";
+char const YANK[] = "yank";
+char const YANK_CYCLE[] = "yank_cycle";
+char const YANK_LAST_ARG[] = "yank_last_arg";
+char const CAPITALIZE_WORD[] = "capitalize_word";
+char const LOWERCASE_WORD[] = "lowercase_word";
+char const UPPERCASE_WORD[] = "uppercase_word";
+char const CAPITALIZE_SUBWORD[] = "capitalize_subword";
+char const LOWERCASE_SUBWORD[] = "lowercase_subword";
+char const UPPERCASE_SUBWORD[] = "uppercase_subword";
+char const TRANSPOSE_CHARACTERS[] = "transpose_characters";
+char const ABORT_LINE[] = "abort_line";
+char const SEND_EOF[] = "send_eof";
+char const TOGGLE_OVERWRITE_MODE[] = "toggle_overwrite_mode";
+char const DELETE_CHARACTER_UNDER_CURSOR[] = "delete_character_under_cursor";
+char const DELETE_CHARACTER_LEFT_OF_CURSOR[] = "delete_character_left_of_cursor";
+char const COMMIT_LINE[] = "commit_line";
+char const CLEAR_SCREEN[] = "clear_screen";
+char const COMPLETE_NEXT[] = "complete_next";
+char const COMPLETE_PREVIOUS[] = "complete_previous";
+char const HISTORY_NEXT[] = "history_next";
+char const HISTORY_PREVIOUS[] = "history_previous";
+char const HISTORY_LAST[] = "history_last";
+char const HISTORY_FIRST[] = "history_first";
+char const HINT_PREVIOUS[] = "hint_previous";
+char const HINT_NEXT[] = "hint_next";
+char const VERBATIM_INSERT[] = "verbatim_insert";
+char const SUSPEND[] = "suspend";
+char const COMPLETE_LINE[] = "complete_line";
+char const HISTORY_INCREMENTAL_SEARCH[] = "history_incremental_search";
+char const HISTORY_COMMON_PREFIX_SEARCH[] = "history_common_prefix_search";
+}
+
+static int const REPLXX_MAX_HINT_ROWS( 4 );
+/*
+ * All whitespaces and all non-alphanumerical characters from ASCII range
+ * with an exception of an underscore ('_').
+ */
+char const defaultWordBreakChars[] = " \t\v\f\a\b\r\n`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?";
+/*
+ * All whitespaces and all non-alphanumerical characters from ASCII range
+ */
+char const defaultSubwordBreakChars[] = " \t\v\f\a\b\r\n`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?_";
+static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL};
+
+static bool isUnsupportedTerm(void) {
+ char* term = getenv("TERM");
+ if (term == NULL) {
+ return false;
+ }
+ for (int j = 0; unsupported_term[j]; ++j) {
+ if (!strcasecmp(term, unsupported_term[j])) {
+ return true;
+ }
+ }
+ return false;
+}
+
+int long long RAPID_REFRESH_MS = 1;
+int long long RAPID_REFRESH_US = RAPID_REFRESH_MS * 1000;
+
+inline int long long now_us( void ) {
+ return ( std::chrono::duration_cast<std::chrono::microseconds>( std::chrono::high_resolution_clock::now().time_since_epoch() ).count() );
+}
+
+class IOModeGuard {
+ Terminal& _terminal;
+public:
+ IOModeGuard( Terminal& terminal_ )
+ : _terminal( terminal_ ) {
+ _terminal.disable_raw_mode();
+ }
+ ~IOModeGuard( void ) {
+ try {
+ _terminal.enable_raw_mode();
+ } catch ( ... ) {
+ }
+ }
+};
+
+}
+
+Replxx::ReplxxImpl::ReplxxImpl( FILE*, FILE*, FILE* )
+ : _utf8Buffer()
+ , _data()
+ , _pos( 0 )
+ , _display()
+ , _displayInputLength( 0 )
+ , _hint()
+ , _prefix( 0 )
+ , _hintSelection( -1 )
+ , _history()
+ , _killRing()
+ , _lastRefreshTime( now_us() )
+ , _refreshSkipped( false )
+ , _lastYankSize( 0 )
+ , _maxHintRows( REPLXX_MAX_HINT_ROWS )
+ , _hintDelay( 0 )
+ , _wordBreakChars( defaultWordBreakChars )
+ , _subwordBreakChars( defaultSubwordBreakChars )
+ , _completionCountCutoff( 100 )
+ , _overwrite( false )
+ , _doubleTabCompletion( false )
+ , _completeOnEmpty( true )
+ , _beepOnAmbiguousCompletion( false )
+ , _immediateCompletion( true )
+ , _bracketedPaste( false )
+ , _noColor( false )
+ , _namedActions()
+ , _keyPressHandlers()
+ , _terminal()
+ , _currentThread()
+ , _prompt( _terminal )
+ , _completionCallback( nullptr )
+ , _highlighterCallback( nullptr )
+ , _hintCallback( nullptr )
+ , _keyPresses()
+ , _messages()
+ , _completions()
+ , _completionContextLength( 0 )
+ , _completionSelection( -1 )
+ , _preloadedBuffer()
+ , _errorMessage()
+ , _previousSearchText()
+ , _modifiedState( false )
+ , _hintColor( Replxx::Color::GRAY )
+ , _hintsCache()
+ , _hintContextLenght( -1 )
+ , _hintSeed()
+ , _mutex() {
+ using namespace std::placeholders;
+ _namedActions[action_names::INSERT_CHARACTER] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::INSERT_CHARACTER, _1 );
+ _namedActions[action_names::NEW_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::NEW_LINE, _1 );
+ _namedActions[action_names::MOVE_CURSOR_TO_BEGINING_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_TO_BEGINING_OF_LINE, _1 );
+ _namedActions[action_names::MOVE_CURSOR_TO_END_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_TO_END_OF_LINE, _1 );
+ _namedActions[action_names::MOVE_CURSOR_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_LEFT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_RIGHT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_RIGHT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_WORD_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_WORD_LEFT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_WORD_RIGHT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_WORD_RIGHT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_SUBWORD_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_LEFT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_SUBWORD_RIGHT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_RIGHT, _1 );
+ _namedActions[action_names::KILL_TO_WHITESPACE_ON_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT, _1 );
+ _namedActions[action_names::KILL_TO_END_OF_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_END_OF_WORD, _1 );
+ _namedActions[action_names::KILL_TO_BEGINING_OF_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_BEGINING_OF_WORD, _1 );
+ _namedActions[action_names::KILL_TO_END_OF_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_END_OF_SUBWORD, _1 );
+ _namedActions[action_names::KILL_TO_BEGINING_OF_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_BEGINING_OF_SUBWORD, _1 );
+ _namedActions[action_names::KILL_TO_BEGINING_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_BEGINING_OF_LINE, _1 );
+ _namedActions[action_names::KILL_TO_END_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_END_OF_LINE, _1 );
+ _namedActions[action_names::YANK] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::YANK, _1 );
+ _namedActions[action_names::YANK_CYCLE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::YANK_CYCLE, _1 );
+ _namedActions[action_names::YANK_LAST_ARG] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::YANK_LAST_ARG, _1 );
+ _namedActions[action_names::CAPITALIZE_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::CAPITALIZE_WORD, _1 );
+ _namedActions[action_names::LOWERCASE_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::LOWERCASE_WORD, _1 );
+ _namedActions[action_names::UPPERCASE_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::UPPERCASE_WORD, _1 );
+ _namedActions[action_names::CAPITALIZE_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::CAPITALIZE_SUBWORD, _1 );
+ _namedActions[action_names::LOWERCASE_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::LOWERCASE_SUBWORD, _1 );
+ _namedActions[action_names::UPPERCASE_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::UPPERCASE_SUBWORD, _1 );
+ _namedActions[action_names::TRANSPOSE_CHARACTERS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::TRANSPOSE_CHARACTERS, _1 );
+ _namedActions[action_names::ABORT_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::ABORT_LINE, _1 );
+ _namedActions[action_names::SEND_EOF] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::SEND_EOF, _1 );
+ _namedActions[action_names::TOGGLE_OVERWRITE_MODE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::TOGGLE_OVERWRITE_MODE, _1 );
+ _namedActions[action_names::DELETE_CHARACTER_UNDER_CURSOR] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::DELETE_CHARACTER_UNDER_CURSOR, _1 );
+ _namedActions[action_names::DELETE_CHARACTER_LEFT_OF_CURSOR] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::DELETE_CHARACTER_LEFT_OF_CURSOR, _1 );
+ _namedActions[action_names::COMMIT_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMMIT_LINE, _1 );
+ _namedActions[action_names::CLEAR_SCREEN] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::CLEAR_SCREEN, _1 );
+ _namedActions[action_names::COMPLETE_NEXT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMPLETE_NEXT, _1 );
+ _namedActions[action_names::COMPLETE_PREVIOUS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMPLETE_PREVIOUS, _1 );
+ _namedActions[action_names::HISTORY_NEXT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_NEXT, _1 );
+ _namedActions[action_names::HISTORY_PREVIOUS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_PREVIOUS, _1 );
+ _namedActions[action_names::HISTORY_LAST] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_LAST, _1 );
+ _namedActions[action_names::HISTORY_FIRST] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_FIRST, _1 );
+ _namedActions[action_names::HINT_PREVIOUS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HINT_PREVIOUS, _1 );
+ _namedActions[action_names::HINT_NEXT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HINT_NEXT, _1 );
+#ifndef _WIN32
+ _namedActions[action_names::VERBATIM_INSERT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::VERBATIM_INSERT, _1 );
+ _namedActions[action_names::SUSPEND] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::SUSPEND, _1 );
+#else
+ _namedActions[action_names::VERBATIM_INSERT] = _namedActions[action_names::SUSPEND] = Replxx::key_press_handler_t();
+#endif
+ _namedActions[action_names::COMPLETE_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMPLETE_LINE, _1 );
+ _namedActions[action_names::HISTORY_INCREMENTAL_SEARCH] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, _1 );
+ _namedActions[action_names::HISTORY_COMMON_PREFIX_SEARCH] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH, _1 );
+
+ bind_key( Replxx::KEY::control( 'A' ), _namedActions.at( action_names::MOVE_CURSOR_TO_BEGINING_OF_LINE ) );
+ bind_key( Replxx::KEY::HOME + 0, _namedActions.at( action_names::MOVE_CURSOR_TO_BEGINING_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'E' ), _namedActions.at( action_names::MOVE_CURSOR_TO_END_OF_LINE ) );
+ bind_key( Replxx::KEY::END + 0, _namedActions.at( action_names::MOVE_CURSOR_TO_END_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'B' ), _namedActions.at( action_names::MOVE_CURSOR_LEFT ) );
+ bind_key( Replxx::KEY::LEFT + 0, _namedActions.at( action_names::MOVE_CURSOR_LEFT ) );
+ bind_key( Replxx::KEY::control( 'F' ), _namedActions.at( action_names::MOVE_CURSOR_RIGHT ) );
+ bind_key( Replxx::KEY::RIGHT + 0, _namedActions.at( action_names::MOVE_CURSOR_RIGHT ) );
+ bind_key( Replxx::KEY::meta( 'b' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_LEFT ) );
+ bind_key( Replxx::KEY::meta( 'B' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_SUBWORD_LEFT ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::LEFT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_LEFT ) );
+ bind_key( Replxx::KEY::meta( Replxx::KEY::LEFT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_LEFT ) ); // Emacs allows Meta, readline don't
+ bind_key( Replxx::KEY::meta( 'f' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_RIGHT ) );
+ bind_key( Replxx::KEY::meta( 'F' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_SUBWORD_RIGHT ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::RIGHT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_RIGHT ) );
+ bind_key( Replxx::KEY::meta( Replxx::KEY::RIGHT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_RIGHT ) ); // Emacs allows Meta, readline don't
+ bind_key( Replxx::KEY::meta( Replxx::KEY::BACKSPACE ), _namedActions.at( action_names::KILL_TO_WHITESPACE_ON_LEFT ) );
+ bind_key( Replxx::KEY::meta( 'd' ), _namedActions.at( action_names::KILL_TO_END_OF_WORD ) );
+ bind_key( Replxx::KEY::meta( 'D' ), _namedActions.at( action_names::KILL_TO_END_OF_SUBWORD ) );
+ bind_key( Replxx::KEY::control( 'W' ), _namedActions.at( action_names::KILL_TO_BEGINING_OF_WORD ) );
+ bind_key( Replxx::KEY::meta( 'W' ), _namedActions.at( action_names::KILL_TO_BEGINING_OF_SUBWORD ) );
+ bind_key( Replxx::KEY::control( 'U' ), _namedActions.at( action_names::KILL_TO_BEGINING_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'K' ), _namedActions.at( action_names::KILL_TO_END_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'Y' ), _namedActions.at( action_names::YANK ) );
+ bind_key( Replxx::KEY::meta( 'y' ), _namedActions.at( action_names::YANK_CYCLE ) );
+ bind_key( Replxx::KEY::meta( 'Y' ), _namedActions.at( action_names::YANK_CYCLE ) );
+ bind_key( Replxx::KEY::meta( '.' ), _namedActions.at( action_names::YANK_LAST_ARG ) );
+ bind_key( Replxx::KEY::meta( 'c' ), _namedActions.at( action_names::CAPITALIZE_WORD ) );
+ bind_key( Replxx::KEY::meta( 'C' ), _namedActions.at( action_names::CAPITALIZE_SUBWORD ) );
+ bind_key( Replxx::KEY::meta( 'l' ), _namedActions.at( action_names::LOWERCASE_WORD ) );
+ bind_key( Replxx::KEY::meta( 'L' ), _namedActions.at( action_names::LOWERCASE_SUBWORD ) );
+ bind_key( Replxx::KEY::meta( 'u' ), _namedActions.at( action_names::UPPERCASE_WORD ) );
+ bind_key( Replxx::KEY::meta( 'U' ), _namedActions.at( action_names::UPPERCASE_SUBWORD ) );
+ bind_key( Replxx::KEY::control( 'T' ), _namedActions.at( action_names::TRANSPOSE_CHARACTERS ) );
+ bind_key( Replxx::KEY::control( 'C' ), _namedActions.at( action_names::ABORT_LINE ) );
+ bind_key( Replxx::KEY::control( 'D' ), _namedActions.at( action_names::SEND_EOF ) );
+ bind_key( Replxx::KEY::INSERT + 0, _namedActions.at( action_names::TOGGLE_OVERWRITE_MODE ) );
+ bind_key( 127, _namedActions.at( action_names::DELETE_CHARACTER_UNDER_CURSOR ) );
+ bind_key( Replxx::KEY::DELETE + 0, _namedActions.at( action_names::DELETE_CHARACTER_UNDER_CURSOR ) );
+ bind_key( Replxx::KEY::BACKSPACE + 0, _namedActions.at( action_names::DELETE_CHARACTER_LEFT_OF_CURSOR ) );
+ bind_key( Replxx::KEY::control( 'J' ), _namedActions.at( action_names::NEW_LINE ) );
+ bind_key( Replxx::KEY::ENTER + 0, _namedActions.at( action_names::COMMIT_LINE ) );
+ bind_key( Replxx::KEY::control( 'L' ), _namedActions.at( action_names::CLEAR_SCREEN ) );
+ bind_key( Replxx::KEY::control( 'N' ), _namedActions.at( action_names::COMPLETE_NEXT ) );
+ bind_key( Replxx::KEY::control( 'P' ), _namedActions.at( action_names::COMPLETE_PREVIOUS ) );
+ bind_key( Replxx::KEY::DOWN + 0, _namedActions.at( action_names::HISTORY_NEXT ) );
+ bind_key( Replxx::KEY::UP + 0, _namedActions.at( action_names::HISTORY_PREVIOUS ) );
+ bind_key( Replxx::KEY::meta( '<' ), _namedActions.at( action_names::HISTORY_FIRST ) );
+ bind_key( Replxx::KEY::PAGE_UP + 0, _namedActions.at( action_names::HISTORY_FIRST ) );
+ bind_key( Replxx::KEY::meta( '>' ), _namedActions.at( action_names::HISTORY_LAST ) );
+ bind_key( Replxx::KEY::PAGE_DOWN + 0, _namedActions.at( action_names::HISTORY_LAST ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::UP ), _namedActions.at( action_names::HINT_PREVIOUS ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::DOWN ), _namedActions.at( action_names::HINT_NEXT ) );
+#ifndef _WIN32
+ bind_key( Replxx::KEY::control( 'V' ), _namedActions.at( action_names::VERBATIM_INSERT ) );
+ bind_key( Replxx::KEY::control( 'Z' ), _namedActions.at( action_names::SUSPEND ) );
+#endif
+ bind_key( Replxx::KEY::TAB + 0, _namedActions.at( action_names::COMPLETE_LINE ) );
+ bind_key( Replxx::KEY::control( 'R' ), _namedActions.at( action_names::HISTORY_INCREMENTAL_SEARCH ) );
+ bind_key( Replxx::KEY::control( 'S' ), _namedActions.at( action_names::HISTORY_INCREMENTAL_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'p' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'P' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'n' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'N' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::PASTE_START, std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::BRACKETED_PASTE, _1 ) );
+}
+
+Replxx::ReplxxImpl::~ReplxxImpl( void ) {
+ disable_bracketed_paste();
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::invoke( Replxx::ACTION action_, char32_t code ) {
+ switch ( action_ ) {
+ case ( Replxx::ACTION::INSERT_CHARACTER ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::insert_character, code ) );
+ case ( Replxx::ACTION::NEW_LINE ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::new_line, code ) );
+ case ( Replxx::ACTION::DELETE_CHARACTER_UNDER_CURSOR ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::delete_character, code ) );
+ case ( Replxx::ACTION::DELETE_CHARACTER_LEFT_OF_CURSOR ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::backspace_character, code ) );
+ case ( Replxx::ACTION::KILL_TO_END_OF_LINE ): return ( action( WANT_REFRESH | SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_to_end_of_line, code ) );
+ case ( Replxx::ACTION::KILL_TO_BEGINING_OF_LINE ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_to_begining_of_line, code ) );
+ case ( Replxx::ACTION::KILL_TO_END_OF_WORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_right<false>, code ) );
+ case ( Replxx::ACTION::KILL_TO_BEGINING_OF_WORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_left<false>, code ) );
+ case ( Replxx::ACTION::KILL_TO_END_OF_SUBWORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_right<true>, code ) );
+ case ( Replxx::ACTION::KILL_TO_BEGINING_OF_SUBWORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_left<true>, code ) );
+ case ( Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_to_whitespace_to_left, code ) );
+ case ( Replxx::ACTION::YANK ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::yank, code ) );
+ case ( Replxx::ACTION::YANK_CYCLE ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::yank_cycle, code ) );
+ case ( Replxx::ACTION::YANK_LAST_ARG ): return ( action( HISTORY_RECALL_MOST_RECENT | DONT_RESET_HIST_YANK_INDEX, &Replxx::ReplxxImpl::yank_last_arg, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_TO_BEGINING_OF_LINE ): return ( action( WANT_REFRESH, &Replxx::ReplxxImpl::go_to_begining_of_line, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_TO_END_OF_LINE ): return ( action( WANT_REFRESH, &Replxx::ReplxxImpl::go_to_end_of_line, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_WORD_LEFT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_left<false>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_WORD_RIGHT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_right<false>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_LEFT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_left<true>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_RIGHT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_right<true>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_LEFT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_char_left, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_RIGHT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_char_right, code ) );
+ case ( Replxx::ACTION::HISTORY_NEXT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_next, code ) );
+ case ( Replxx::ACTION::HISTORY_PREVIOUS ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_previous, code ) );
+ case ( Replxx::ACTION::HISTORY_FIRST ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_first, code ) );
+ case ( Replxx::ACTION::HISTORY_LAST ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_last, code ) );
+ case ( Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH ): return ( action( NOOP, &Replxx::ReplxxImpl::incremental_history_search, code ) );
+ case ( Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH ): return ( action( RESET_KILL_ACTION | DONT_RESET_PREFIX, &Replxx::ReplxxImpl::common_prefix_search, code ) );
+ case ( Replxx::ACTION::HINT_NEXT ): return ( action( NOOP, &Replxx::ReplxxImpl::hint_next, code ) );
+ case ( Replxx::ACTION::HINT_PREVIOUS ): return ( action( NOOP, &Replxx::ReplxxImpl::hint_previous, code ) );
+ case ( Replxx::ACTION::CAPITALIZE_WORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::capitalize_word<false>, code ) );
+ case ( Replxx::ACTION::LOWERCASE_WORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::lowercase_word<false>, code ) );
+ case ( Replxx::ACTION::UPPERCASE_WORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::uppercase_word<false>, code ) );
+ case ( Replxx::ACTION::CAPITALIZE_SUBWORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::capitalize_word<true>, code ) );
+ case ( Replxx::ACTION::LOWERCASE_SUBWORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::lowercase_word<true>, code ) );
+ case ( Replxx::ACTION::UPPERCASE_SUBWORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::uppercase_word<true>, code ) );
+ case ( Replxx::ACTION::TRANSPOSE_CHARACTERS ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::transpose_characters, code ) );
+ case ( Replxx::ACTION::TOGGLE_OVERWRITE_MODE ): return ( action( NOOP, &Replxx::ReplxxImpl::toggle_overwrite_mode, code ) );
+#ifndef _WIN32
+ case ( Replxx::ACTION::VERBATIM_INSERT ): return ( action( WANT_REFRESH | RESET_KILL_ACTION, &Replxx::ReplxxImpl::verbatim_insert, code ) );
+ case ( Replxx::ACTION::SUSPEND ): return ( action( WANT_REFRESH, &Replxx::ReplxxImpl::suspend, code ) );
+#endif
+ case ( Replxx::ACTION::CLEAR_SCREEN ): return ( action( NOOP, &Replxx::ReplxxImpl::clear_screen, code ) );
+ case ( Replxx::ACTION::CLEAR_SELF ): clear_self_to_end_of_screen(); return ( Replxx::ACTION_RESULT::CONTINUE );
+ case ( Replxx::ACTION::REPAINT ): repaint(); return ( Replxx::ACTION_RESULT::CONTINUE );
+ case ( Replxx::ACTION::COMPLETE_LINE ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::complete_line, code ) );
+ case ( Replxx::ACTION::COMPLETE_NEXT ): return ( action( RESET_KILL_ACTION | DONT_RESET_COMPLETIONS | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::complete_next, code ) );
+ case ( Replxx::ACTION::COMPLETE_PREVIOUS ): return ( action( RESET_KILL_ACTION | DONT_RESET_COMPLETIONS | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::complete_previous, code ) );
+ case ( Replxx::ACTION::COMMIT_LINE ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::commit_line, code ) );
+ case ( Replxx::ACTION::ABORT_LINE ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::abort_line, code ) );
+ case ( Replxx::ACTION::SEND_EOF ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::send_eof, code ) );
+ case ( Replxx::ACTION::BRACKETED_PASTE ): return ( action( WANT_REFRESH | RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::bracketed_paste, code ) );
+ }
+ return ( Replxx::ACTION_RESULT::BAIL );
+}
+
+void Replxx::ReplxxImpl::bind_key( char32_t code_, Replxx::key_press_handler_t handler_ ) {
+ _keyPressHandlers[code_] = handler_;
+}
+
+void Replxx::ReplxxImpl::bind_key_internal( char32_t code_, char const* actionName_ ) {
+ named_actions_t::const_iterator it( _namedActions.find( actionName_ ) );
+ if ( it == _namedActions.end() ) {
+ throw std::runtime_error( std::string( "replxx: Unknown action name: " ).append( actionName_ ) );
+ }
+ if ( !! it->second ) {
+ bind_key( code_, it->second );
+ }
+}
+
+Replxx::State Replxx::ReplxxImpl::get_state( void ) const {
+ _utf8Buffer.assign( _data );
+ return ( Replxx::State( _utf8Buffer.get(), _pos ) );
+}
+
+void Replxx::ReplxxImpl::set_state( Replxx::State const& state_ ) {
+ _data.assign( state_.text() );
+ if ( state_.cursor_position() >= 0 ) {
+ _pos = min( state_.cursor_position(), _data.length() );
+ }
+ _modifiedState = true;
+}
+
+char32_t Replxx::ReplxxImpl::read_char( HINT_ACTION hintAction_ ) {
+ /* try scheduled key presses */ {
+ std::lock_guard<std::mutex> l( _mutex );
+ if ( !_keyPresses.empty() ) {
+ char32_t keyPress( _keyPresses.front() );
+ _keyPresses.pop_front();
+ return ( keyPress );
+ }
+ }
+ int hintDelay(
+ _refreshSkipped
+ ? static_cast<int>( RAPID_REFRESH_MS * 2 )
+ : ( hintAction_ != HINT_ACTION::SKIP ? _hintDelay : 0 )
+ );
+ while ( true ) {
+ Terminal::EVENT_TYPE eventType( _terminal.wait_for_input( hintDelay ) );
+ if ( eventType == Terminal::EVENT_TYPE::TIMEOUT ) {
+ refresh_line( _refreshSkipped ? HINT_ACTION::REGENERATE : HINT_ACTION::REPAINT );
+ hintDelay = 0;
+ _refreshSkipped = false;
+ continue;
+ }
+ if ( eventType == Terminal::EVENT_TYPE::KEY_PRESS ) {
+ break;
+ }
+ if ( eventType == Terminal::EVENT_TYPE::RESIZE ) {
+ // caught a window resize event
+ // now redraw the prompt and line
+ _prompt.update_screen_columns();
+ // redraw the original prompt with current input
+ refresh_line( HINT_ACTION::REPAINT );
+ continue;
+ }
+ std::lock_guard<std::mutex> l( _mutex );
+ clear_self_to_end_of_screen();
+ while ( ! _messages.empty() ) {
+ string const& message( _messages.front() );
+ _terminal.write8( message.data(), static_cast<int>( message.length() ) );
+ _messages.pop_front();
+ }
+ repaint();
+ }
+ /* try scheduled key presses */ {
+ std::lock_guard<std::mutex> l( _mutex );
+ if ( !_keyPresses.empty() ) {
+ char32_t keyPress( _keyPresses.front() );
+ _keyPresses.pop_front();
+ return ( keyPress );
+ }
+ }
+ return ( _terminal.read_char() );
+}
+
+void Replxx::ReplxxImpl::clear( void ) {
+ _pos = 0;
+ _prefix = 0;
+ _completions.clear();
+ _completionContextLength = 0;
+ _completionSelection = -1;
+ _data.clear();
+ _hintSelection = -1;
+ _hint = UnicodeString();
+ _display.clear();
+ _displayInputLength = 0;
+}
+
+void Replxx::ReplxxImpl::call_modify_callback( void ) {
+ if ( ! _modifyCallback ) {
+ return;
+ }
+ _utf8Buffer.assign( _data );
+ std::string origLine( _utf8Buffer.get() );
+ int pos( _pos );
+ std::string line( origLine );
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ _modifyCallback( line, pos );
+ }
+ if ( ( pos != _pos ) || ( line != origLine ) ) {
+ _data.assign( line.c_str() );
+ _pos = min( pos, _data.length() );
+ _modifiedState = true;
+ }
+}
+
+Replxx::ReplxxImpl::completions_t Replxx::ReplxxImpl::call_completer( std::string const& input, int& contextLen_ ) const {
+ Replxx::completions_t completionsIntermediary(
+ !! _completionCallback
+ ? _completionCallback( input, contextLen_ )
+ : Replxx::completions_t()
+ );
+ completions_t completions;
+ completions.reserve( completionsIntermediary.size() );
+ for ( Replxx::Completion const& c : completionsIntermediary ) {
+ completions.emplace_back( c );
+ }
+ return ( completions );
+}
+
+Replxx::ReplxxImpl::hints_t Replxx::ReplxxImpl::call_hinter( std::string const& input, int& contextLen, Replxx::Color& color ) const {
+ Replxx::hints_t hintsIntermediary(
+ !! _hintCallback
+ ? _hintCallback( input, contextLen, color )
+ : Replxx::hints_t()
+ );
+ hints_t hints;
+ hints.reserve( hintsIntermediary.size() );
+ for ( std::string const& h : hintsIntermediary ) {
+ hints.emplace_back( h.c_str() );
+ }
+ return ( hints );
+}
+
+void Replxx::ReplxxImpl::set_preload_buffer( std::string const& preloadText ) {
+ _preloadedBuffer = preloadText;
+ // remove characters that won't display correctly
+ bool controlsStripped = false;
+ int whitespaceSeen( 0 );
+ for ( std::string::iterator it( _preloadedBuffer.begin() ); it != _preloadedBuffer.end(); ) {
+ unsigned char c = *it;
+ if ( '\r' == c ) { // silently skip CR
+ _preloadedBuffer.erase( it, it + 1 );
+ continue;
+ }
+ if ( ( '\n' == c ) || ( '\t' == c ) ) { // note newline or tab
+ ++ whitespaceSeen;
+ ++ it;
+ continue;
+ }
+ if ( whitespaceSeen > 0 ) {
+ it -= whitespaceSeen;
+ *it = ' ';
+ _preloadedBuffer.erase( it + 1, it + whitespaceSeen - 1 );
+ }
+ if ( is_control_code( c ) ) { // remove other control characters, flag for message
+ controlsStripped = true;
+ if ( whitespaceSeen > 0 ) {
+ _preloadedBuffer.erase( it, it + 1 );
+ -- it;
+ } else {
+ *it = ' ';
+ }
+ }
+ whitespaceSeen = 0;
+ ++ it;
+ }
+ if ( whitespaceSeen > 0 ) {
+ std::string::iterator it = _preloadedBuffer.end() - whitespaceSeen;
+ *it = ' ';
+ if ( whitespaceSeen > 1 ) {
+ _preloadedBuffer.erase( it + 1, _preloadedBuffer.end() );
+ }
+ }
+ _errorMessage.clear();
+ if ( controlsStripped ) {
+ _errorMessage.assign( " [Edited line: control characters were converted to spaces]\n" );
+ }
+}
+
+char const* Replxx::ReplxxImpl::read_from_stdin( void ) {
+ if ( _preloadedBuffer.empty() ) {
+ getline( cin, _preloadedBuffer );
+ if ( ! cin.good() ) {
+ return nullptr;
+ }
+ }
+ while ( ! _preloadedBuffer.empty() && ( ( _preloadedBuffer.back() == '\r' ) || ( _preloadedBuffer.back() == '\n' ) ) ) {
+ _preloadedBuffer.pop_back();
+ }
+ _utf8Buffer.assign( _preloadedBuffer );
+ _preloadedBuffer.clear();
+ return _utf8Buffer.get();
+}
+
+void Replxx::ReplxxImpl::emulate_key_press( char32_t keyCode_ ) {
+ std::lock_guard<std::mutex> l( _mutex );
+ _keyPresses.push_back( keyCode_ );
+ if ( ( _currentThread != std::thread::id() ) && ( _currentThread != std::this_thread::get_id() ) ) {
+ _terminal.notify_event( Terminal::EVENT_TYPE::KEY_PRESS );
+ }
+}
+
+char const* Replxx::ReplxxImpl::input( std::string const& prompt ) {
+ try {
+ errno = 0;
+ if ( ! tty::in ) { // input not from a terminal, we should work with piped input, i.e. redirected stdin
+ return ( read_from_stdin() );
+ }
+ if (!_errorMessage.empty()) {
+ printf("%s", _errorMessage.c_str());
+ fflush(stdout);
+ _errorMessage.clear();
+ }
+ if ( isUnsupportedTerm() ) {
+ cout << prompt << flush;
+ fflush(stdout);
+ return ( read_from_stdin() );
+ }
+ if (_terminal.enable_raw_mode() == -1) {
+ return nullptr;
+ }
+ _prompt.set_text( UnicodeString( prompt ) );
+ _currentThread = std::this_thread::get_id();
+ clear();
+ if (!_preloadedBuffer.empty()) {
+ preload_puffer(_preloadedBuffer.c_str());
+ _preloadedBuffer.clear();
+ }
+ if ( get_input_line() == -1 ) {
+ return ( finalize_input( nullptr ) );
+ }
+ _terminal.write8( "\n", 1 );
+ _utf8Buffer.assign( _data );
+ return ( finalize_input( _utf8Buffer.get() ) );
+ } catch ( std::exception const& ) {
+ return ( finalize_input( nullptr ) );
+ }
+}
+
+char const* Replxx::ReplxxImpl::finalize_input( char const* retVal_ ) {
+ _currentThread = std::thread::id();
+ _terminal.disable_raw_mode();
+ return ( retVal_ );
+}
+
+int Replxx::ReplxxImpl::install_window_change_handler( void ) {
+#ifndef _WIN32
+ return ( _terminal.install_window_change_handler() );
+#else
+ return 0;
+#endif
+}
+
+void Replxx::ReplxxImpl::enable_bracketed_paste( void ) {
+ if ( _bracketedPaste ) {
+ return;
+ }
+ _terminal.enable_bracketed_paste();
+ _bracketedPaste = true;
+}
+
+void Replxx::ReplxxImpl::disable_bracketed_paste( void ) {
+ if ( ! _bracketedPaste ) {
+ return;
+ }
+ _terminal.disable_bracketed_paste();
+ _bracketedPaste = false;
+}
+
+void Replxx::ReplxxImpl::print( char const* str_, int size_ ) {
+ if ( ( _currentThread == std::thread::id() ) || ( _currentThread == std::this_thread::get_id() ) ) {
+ _terminal.write8( str_, size_ );
+ } else {
+ std::lock_guard<std::mutex> l( _mutex );
+ _messages.emplace_back( str_, size_ );
+ _terminal.notify_event( Terminal::EVENT_TYPE::MESSAGE );
+ }
+ return;
+}
+
+void Replxx::ReplxxImpl::preload_puffer(const char* preloadText) {
+ _data.assign( preloadText );
+ _prefix = _pos = _data.length();
+}
+
+void Replxx::ReplxxImpl::set_color( Replxx::Color color_ ) {
+ char const* code( ansi_color( color_ ) );
+ while ( *code ) {
+ _display.push_back( *code );
+ ++ code;
+ }
+}
+
+void Replxx::ReplxxImpl::render( char32_t ch ) {
+ if ( ch == Replxx::KEY::ESCAPE ) {
+ _display.push_back( '^' );
+ _display.push_back( '[' );
+ } else if ( is_control_code( ch ) && ( ch != '\n' ) ) {
+ _display.push_back( '^' );
+ _display.push_back( control_to_human( ch ) );
+ } else {
+ _display.push_back( ch );
+ }
+ return;
+}
+
+void Replxx::ReplxxImpl::render( HINT_ACTION hintAction_ ) {
+ if ( hintAction_ == HINT_ACTION::TRIM ) {
+ _display.erase( _display.begin() + _displayInputLength, _display.end() );
+ _modifiedState = false;
+ return;
+ }
+ if ( hintAction_ == HINT_ACTION::SKIP ) {
+ return;
+ }
+ _display.clear();
+ if ( _noColor ) {
+ for ( char32_t ch : _data ) {
+ render( ch );
+ }
+ _displayInputLength = static_cast<int>( _display.size() );
+ _modifiedState = false;
+ return;
+ }
+ Replxx::colors_t colors( _data.length(), Replxx::Color::DEFAULT );
+ _utf8Buffer.assign( _data );
+ if ( !! _highlighterCallback ) {
+ IOModeGuard ioModeGuard( _terminal );
+ _highlighterCallback( _utf8Buffer.get(), colors );
+ }
+ paren_info_t pi( matching_paren() );
+ if ( pi.index != -1 ) {
+ colors[pi.index] = pi.error ? Replxx::Color::ERROR : Replxx::Color::BRIGHTRED;
+ }
+ Replxx::Color c( Replxx::Color::DEFAULT );
+ for ( int i( 0 ); i < _data.length(); ++ i ) {
+ if ( colors[i] != c ) {
+ c = colors[i];
+ set_color( c );
+ }
+ render( _data[i] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ _displayInputLength = static_cast<int>( _display.size() );
+ _modifiedState = false;
+ return;
+}
+
+int Replxx::ReplxxImpl::handle_hints( HINT_ACTION hintAction_ ) {
+ if ( _noColor ) {
+ return ( 0 );
+ }
+ if ( ! _hintCallback ) {
+ return ( 0 );
+ }
+ if ( ( _hintDelay > 0 ) && ( hintAction_ != HINT_ACTION::REPAINT ) ) {
+ _hintSelection = -1;
+ return ( 0 );
+ }
+ if ( ( hintAction_ == HINT_ACTION::SKIP ) || ( hintAction_ == HINT_ACTION::TRIM ) ) {
+ return ( 0 );
+ }
+ if ( _pos != _data.length() ) {
+ return ( 0 );
+ }
+ _hint = UnicodeString();
+ int len( 0 );
+ if ( hintAction_ == HINT_ACTION::REGENERATE ) {
+ _hintSelection = -1;
+ }
+ _utf8Buffer.assign( _data, _pos );
+ if ( ( _utf8Buffer != _hintSeed ) || ( _hintContextLenght < 0 ) ) {
+ _hintSeed.assign( _utf8Buffer );
+ _hintContextLenght = context_length();
+ _hintColor = Replxx::Color::GRAY;
+ IOModeGuard ioModeGuard( _terminal );
+ _hintsCache = call_hinter( _utf8Buffer.get(), _hintContextLenght, _hintColor );
+ }
+ int hintCount( static_cast<int>( _hintsCache.size() ) );
+ if ( hintCount == 1 ) {
+ _hint = _hintsCache.front();
+ len = _hint.length() - _hintContextLenght;
+ if ( len > 0 ) {
+ set_color( _hintColor );
+ for ( int i( 0 ); i < len; ++ i ) {
+ _display.push_back( _hint[i + _hintContextLenght] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ }
+ } else if ( ( _maxHintRows > 0 ) && ( hintCount > 0 ) ) {
+ int startCol( _prompt.indentation() + _pos );
+ int maxCol( _prompt.screen_columns() );
+#ifdef _WIN32
+ -- maxCol;
+#endif
+ if ( _hintSelection < -1 ) {
+ _hintSelection = hintCount - 1;
+ } else if ( _hintSelection >= hintCount ) {
+ _hintSelection = -1;
+ }
+ if ( _hintSelection != -1 ) {
+ _hint = _hintsCache[_hintSelection];
+ len = min<int>( _hint.length(), maxCol - startCol );
+ if ( _hintContextLenght < len ) {
+ set_color( _hintColor );
+ for ( int i( _hintContextLenght ); i < len; ++ i ) {
+ _display.push_back( _hint[i] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ }
+ }
+ startCol -= _hintContextLenght;
+ for ( int hintRow( 0 ); hintRow < min( hintCount, _maxHintRows ); ++ hintRow ) {
+#ifdef _WIN32
+ _display.push_back( '\r' );
+#endif
+ _display.push_back( '\n' );
+ int col( 0 );
+ for ( int i( 0 ); ( i < startCol ) && ( col < maxCol ); ++ i, ++ col ) {
+ _display.push_back( ' ' );
+ }
+ set_color( _hintColor );
+ for ( int i( _pos - _hintContextLenght ); ( i < _pos ) && ( col < maxCol ); ++ i, ++ col ) {
+ _display.push_back( _data[i] );
+ }
+ int hintNo( hintRow + _hintSelection + 1 );
+ if ( hintNo == hintCount ) {
+ continue;
+ } else if ( hintNo > hintCount ) {
+ -- hintNo;
+ }
+ UnicodeString const& h( _hintsCache[hintNo % hintCount] );
+ for ( int i( _hintContextLenght ); ( i < h.length() ) && ( col < maxCol ); ++ i, ++ col ) {
+ _display.push_back( h[i] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ }
+ }
+ return ( len );
+}
+
+Replxx::ReplxxImpl::paren_info_t Replxx::ReplxxImpl::matching_paren( void ) {
+ if (_pos >= _data.length()) {
+ return ( paren_info_t{ -1, false } );
+ }
+ /* this scans for a brace matching _data[_pos] to highlight */
+ unsigned char part1, part2;
+ int scanDirection = 0;
+ if ( strchr("}])", _data[_pos]) ) {
+ scanDirection = -1; /* backwards */
+ if (_data[_pos] == '}') {
+ part1 = '}'; part2 = '{';
+ } else if (_data[_pos] == ']') {
+ part1 = ']'; part2 = '[';
+ } else {
+ part1 = ')'; part2 = '(';
+ }
+ } else if ( strchr("{[(", _data[_pos]) ) {
+ scanDirection = 1; /* forwards */
+ if (_data[_pos] == '{') {
+ //part1 = '{'; part2 = '}';
+ part1 = '}'; part2 = '{';
+ } else if (_data[_pos] == '[') {
+ //part1 = '['; part2 = ']';
+ part1 = ']'; part2 = '[';
+ } else {
+ //part1 = '('; part2 = ')';
+ part1 = ')'; part2 = '(';
+ }
+ } else {
+ return ( paren_info_t{ -1, false } );
+ }
+ int highlightIdx = -1;
+ bool indicateError = false;
+ int unmatched = scanDirection;
+ int unmatchedOther = 0;
+ for (int i = _pos + scanDirection; i >= 0 && i < _data.length(); i += scanDirection) {
+ /* TODO: the right thing when inside a string */
+ if (strchr("}])", _data[i])) {
+ if (_data[i] == part1) {
+ --unmatched;
+ } else {
+ --unmatchedOther;
+ }
+ } else if (strchr("{[(", _data[i])) {
+ if (_data[i] == part2) {
+ ++unmatched;
+ } else {
+ ++unmatchedOther;
+ }
+ }
+
+ if (unmatched == 0) {
+ highlightIdx = i;
+ indicateError = (unmatchedOther != 0);
+ break;
+ }
+ }
+ return ( paren_info_t{ highlightIdx, indicateError } );
+}
+
+/**
+ * Refresh the user's input line: the prompt is already onscreen and is not
+ * redrawn here screen position
+ */
+void Replxx::ReplxxImpl::refresh_line( HINT_ACTION hintAction_ ) {
+ int long long now( now_us() );
+ int long long duration( now - _lastRefreshTime );
+ if ( duration < RAPID_REFRESH_US ) {
+ _lastRefreshTime = now;
+ _refreshSkipped = true;
+ return;
+ }
+ _refreshSkipped = false;
+ // check for a matching brace/bracket/paren, remember its position if found
+ render( hintAction_ );
+ int hintLen( handle_hints( hintAction_ ) );
+ // calculate the position of the end of the input line
+ int xEndOfInput( 0 ), yEndOfInput( 0 );
+ calculate_screen_position(
+ _prompt.indentation(), 0, _prompt.screen_columns(),
+ calculate_displayed_length( _data.get(), _data.length() ) + hintLen,
+ xEndOfInput, yEndOfInput
+ );
+ yEndOfInput += static_cast<int>( count( _display.begin(), _display.end(), '\n' ) );
+
+ // calculate the desired position of the cursor
+ int xCursorPos( 0 ), yCursorPos( 0 );
+ calculate_screen_position(
+ _prompt.indentation(), 0, _prompt.screen_columns(),
+ calculate_displayed_length( _data.get(), _pos ),
+ xCursorPos, yCursorPos
+ );
+
+ // position at the end of the prompt, clear to end of previous input
+ _terminal.set_cursor_visible( false );
+ _terminal.jump_cursor(
+ _prompt.indentation(), // 0-based on Win32
+ -( _prompt._cursorRowOffset - _prompt._extraLines )
+ );
+ // display the input line
+ _terminal.write32( _display.data(), _displayInputLength );
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::TO_END );
+ _terminal.write32( _display.data() + _displayInputLength, static_cast<int>( _display.size() ) - _displayInputLength );
+#ifndef _WIN32
+ // we have to generate our own newline on line wrap
+ if ( ( xEndOfInput == 0 ) && ( yEndOfInput > 0 ) ) {
+ _terminal.write8( "\n", 1 );
+ }
+#endif
+ // position the cursor
+ _terminal.jump_cursor( xCursorPos, -( yEndOfInput - yCursorPos ) );
+ _terminal.set_cursor_visible( true );
+ _prompt._cursorRowOffset = _prompt._extraLines + yCursorPos; // remember row for next pass
+ _lastRefreshTime = now_us();
+}
+
+int Replxx::ReplxxImpl::context_length() {
+ int prefixLength = _pos;
+ while ( prefixLength > 0 ) {
+ if ( is_word_break_character<false>( _data[prefixLength - 1] ) ) {
+ break;
+ }
+ -- prefixLength;
+ }
+ return ( _pos - prefixLength );
+}
+
+void Replxx::ReplxxImpl::repaint( void ) {
+ _prompt.write();
+ for ( int i( _prompt._extraLines ); i < _prompt._cursorRowOffset; ++ i ) {
+ _terminal.write8( "\n", 1 );
+ }
+ refresh_line( HINT_ACTION::SKIP );
+}
+
+void Replxx::ReplxxImpl::clear_self_to_end_of_screen( Prompt const* prompt_ ) {
+ // position at the start of the prompt, clear to end of previous input
+ _terminal.jump_cursor( 0, prompt_ ? -prompt_->_cursorRowOffset : -_prompt._cursorRowOffset );
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::TO_END );
+ return;
+}
+
+namespace {
+int longest_common_prefix( Replxx::ReplxxImpl::completions_t const& completions ) {
+ int completionsCount( static_cast<int>( completions.size() ) );
+ if ( completionsCount < 1 ) {
+ return ( 0 );
+ }
+ int longestCommonPrefix( 0 );
+ UnicodeString const& sample( completions.front().text() );
+ while ( true ) {
+ if ( longestCommonPrefix >= sample.length() ) {
+ return ( longestCommonPrefix );
+ }
+ char32_t sc( sample[longestCommonPrefix] );
+ for ( int i( 1 ); i < completionsCount; ++ i ) {
+ UnicodeString const& candidate( completions[i].text() );
+ if ( longestCommonPrefix >= candidate.length() ) {
+ return ( longestCommonPrefix );
+ }
+ char32_t cc( candidate[longestCommonPrefix] );
+ if ( cc != sc ) {
+ return ( longestCommonPrefix );
+ }
+ }
+ ++ longestCommonPrefix;
+ }
+}
+}
+
+/**
+ * Handle command completion, using a completionCallback() routine to provide
+ * possible substitutions
+ * This routine handles the mechanics of updating the user's input buffer with
+ * possible replacement of text as the user selects a proposed completion string,
+ * or cancels the completion attempt.
+ * @param pi - Prompt struct holding information about the prompt and our
+ * screen position
+ */
+char32_t Replxx::ReplxxImpl::do_complete_line( bool showCompletions_ ) {
+ char32_t c = 0;
+
+ // completionCallback() expects a parsable entity, so find the previous break
+ // character and
+ // extract a copy to parse. we also handle the case where tab is hit while
+ // not at end-of-line.
+
+ _utf8Buffer.assign( _data, _pos );
+ // get a list of completions
+ _completionSelection = -1;
+ _completionContextLength = context_length();
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ _completions = call_completer( _utf8Buffer.get(), _completionContextLength );
+ }
+
+ // if no completions, we are done
+ if ( _completions.empty() ) {
+ beep();
+ return 0;
+ }
+
+ // at least one completion
+ int longestCommonPrefix = 0;
+ int completionsCount( static_cast<int>( _completions.size() ) );
+ int selectedCompletion( 0 );
+ if ( _hintSelection != -1 ) {
+ selectedCompletion = _hintSelection;
+ completionsCount = 1;
+ }
+ if ( completionsCount == 1 ) {
+ longestCommonPrefix = static_cast<int>( _completions[selectedCompletion].text().length() );
+ } else {
+ longestCommonPrefix = longest_common_prefix( _completions );
+ }
+ if ( _beepOnAmbiguousCompletion && ( completionsCount != 1 ) ) { // beep if ambiguous
+ beep();
+ }
+
+ // if we can extend the item, extend it and return to main loop
+ if ( ( longestCommonPrefix > _completionContextLength ) || ( completionsCount == 1 ) ) {
+ _pos -= _completionContextLength;
+ _data.erase( _pos, _completionContextLength );
+ _data.insert( _pos, _completions[selectedCompletion].text(), 0, longestCommonPrefix );
+ _pos = _pos + longestCommonPrefix;
+ _completionContextLength = longestCommonPrefix;
+ refresh_line();
+ return 0;
+ }
+
+ if ( ! showCompletions_ ) {
+ return ( 0 );
+ }
+
+ if ( _doubleTabCompletion ) {
+ // we can't complete any further, wait for second tab
+ do {
+ c = read_char();
+ } while ( c == static_cast<char32_t>( -1 ) );
+
+ // if any character other than tab, pass it to the main loop
+ if ( c != Replxx::KEY::TAB ) {
+ return c;
+ }
+ }
+
+ // we got a second tab, maybe show list of possible completions
+ bool showCompletions = true;
+ bool onNewLine = false;
+ if ( static_cast<int>( _completions.size() ) > _completionCountCutoff ) {
+ int savePos = _pos; // move cursor to EOL to avoid overwriting the command line
+ _pos = _data.length();
+ refresh_line();
+ _pos = savePos;
+ printf( "\nDisplay all %u possibilities? (y or n)", static_cast<unsigned int>( _completions.size() ) );
+ fflush(stdout);
+ onNewLine = true;
+ while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != Replxx::KEY::control('C')) {
+ do {
+ c = read_char();
+ } while (c == static_cast<char32_t>(-1));
+ }
+ switch (c) {
+ case 'n':
+ case 'N':
+ showCompletions = false;
+ break;
+ case Replxx::KEY::control('C'):
+ showCompletions = false;
+ // Display the ^C we got
+ _terminal.write8( "^C", 2 );
+ c = 0;
+ break;
+ }
+ }
+
+ // if showing the list, do it the way readline does it
+ bool stopList( false );
+ if ( showCompletions ) {
+ int longestCompletion( 0 );
+ for ( size_t j( 0 ); j < _completions.size(); ++ j ) {
+ int itemLength( static_cast<int>( _completions[j].text().length() ) );
+ if ( itemLength > longestCompletion ) {
+ longestCompletion = itemLength;
+ }
+ }
+ longestCompletion += 2;
+ int columnCount = _prompt.screen_columns() / longestCompletion;
+ if ( columnCount < 1 ) {
+ columnCount = 1;
+ }
+ if ( ! onNewLine ) { // skip this if we showed "Display all %d possibilities?"
+ int savePos = _pos; // move cursor to EOL to avoid overwriting the command line
+ _pos = _data.length();
+ refresh_line( HINT_ACTION::TRIM );
+ _pos = savePos;
+ } else {
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::TO_END );
+ }
+ size_t pauseRow = _terminal.get_screen_rows() - 1;
+ size_t rowCount = (_completions.size() + columnCount - 1) / columnCount;
+ for (size_t row = 0; row < rowCount; ++row) {
+ if (row == pauseRow) {
+ printf("\n--More--");
+ fflush(stdout);
+ c = 0;
+ bool doBeep = false;
+ while (c != ' ' && c != Replxx::KEY::ENTER && c != 'y' && c != 'Y' &&
+ c != 'n' && c != 'N' && c != 'q' && c != 'Q' &&
+ c != Replxx::KEY::control('C')) {
+ if (doBeep) {
+ beep();
+ }
+ doBeep = true;
+ do {
+ c = read_char();
+ } while (c == static_cast<char32_t>(-1));
+ }
+ switch (c) {
+ case ' ':
+ case 'y':
+ case 'Y':
+ printf("\r \r");
+ pauseRow += _terminal.get_screen_rows() - 1;
+ break;
+ case Replxx::KEY::ENTER:
+ printf("\r \r");
+ ++pauseRow;
+ break;
+ case 'n':
+ case 'N':
+ case 'q':
+ case 'Q':
+ printf("\r \r");
+ stopList = true;
+ break;
+ case Replxx::KEY::control('C'):
+ // Display the ^C we got
+ _terminal.write8( "^C", 2 );
+ stopList = true;
+ break;
+ }
+ } else {
+ _terminal.write8( "\n", 1 );
+ }
+ if (stopList) {
+ break;
+ }
+ static UnicodeString const res( ansi_color( Replxx::Color::DEFAULT ) );
+ for (int column = 0; column < columnCount; ++column) {
+ size_t index = (column * rowCount) + row;
+ if ( index < _completions.size() ) {
+ Completion const& c( _completions[index] );
+ int itemLength = static_cast<int>(c.text().length());
+ fflush(stdout);
+
+ if ( longestCommonPrefix > 0 ) {
+ static UnicodeString const col( ansi_color( Replxx::Color::BRIGHTMAGENTA ) );
+ if (!_noColor) {
+ _terminal.write32(col.get(), col.length());
+ }
+ _terminal.write32(&_data[_pos - _completionContextLength], longestCommonPrefix);
+ if (!_noColor) {
+ _terminal.write32(res.get(), res.length());
+ }
+ }
+
+ if ( !_noColor && ( c.color() != Replxx::Color::DEFAULT ) ) {
+ UnicodeString ac( ansi_color( c.color() ) );
+ _terminal.write32( ac.get(), ac.length() );
+ }
+ _terminal.write32( c.text().get() + longestCommonPrefix, itemLength - longestCommonPrefix );
+ if ( !_noColor && ( c.color() != Replxx::Color::DEFAULT ) ) {
+ _terminal.write32( res.get(), res.length() );
+ }
+
+ if ( ((column + 1) * rowCount) + row < _completions.size() ) {
+ for ( int k( itemLength ); k < longestCompletion; ++k ) {
+ printf( " " );
+ }
+ }
+ }
+ }
+ }
+ fflush(stdout);
+ }
+
+ // display the prompt on a new line, then redisplay the input buffer
+ if (!stopList || c == Replxx::KEY::control('C')) {
+ _terminal.write8( "\n", 1 );
+ }
+ _prompt.write();
+ _prompt._cursorRowOffset = _prompt._extraLines;
+ refresh_line();
+ return 0;
+}
+
+int Replxx::ReplxxImpl::get_input_line( void ) {
+ // The latest history entry is always our current buffer
+ if ( _data.length() > 0 ) {
+ _history.add( _data );
+ } else {
+ _history.add( UnicodeString() );
+ }
+ _history.jump( false, false );
+
+ // display the prompt
+ _prompt.write();
+
+ // the cursor starts out at the end of the prompt
+ _prompt._cursorRowOffset = _prompt._extraLines;
+
+ // kill and yank start in "other" mode
+ _killRing.lastAction = KillRing::actionOther;
+
+ // if there is already text in the buffer, display it first
+ if (_data.length() > 0) {
+ refresh_line();
+ }
+
+ // loop collecting characters, respond to line editing characters
+ Replxx::ACTION_RESULT next( Replxx::ACTION_RESULT::CONTINUE );
+ while ( next == Replxx::ACTION_RESULT::CONTINUE ) {
+ int c( read_char( HINT_ACTION::REPAINT ) ); // get a new keystroke
+
+ if (c == 0) {
+ return _data.length();
+ }
+
+ if (c == -1) {
+ refresh_line();
+ continue;
+ }
+
+ if (c == -2) {
+ _prompt.write();
+ refresh_line();
+ continue;
+ }
+
+ key_press_handlers_t::iterator it( _keyPressHandlers.find( c ) );
+ if ( it != _keyPressHandlers.end() ) {
+ next = it->second( c );
+ if ( _modifiedState ) {
+ refresh_line();
+ }
+ } else {
+ next = action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::insert_character, c );
+ }
+ }
+ return ( next == Replxx::ACTION_RESULT::RETURN ? _data.length() : -1 );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::action( action_trait_t actionTrait_, key_press_handler_raw_t const& handler_, char32_t code_ ) {
+ Replxx::ACTION_RESULT res( ( this->*handler_ )( code_ ) );
+ call_modify_callback();
+ if ( actionTrait_ & HISTORY_RECALL_MOST_RECENT ) {
+ _history.reset_recall_most_recent();
+ }
+ if ( actionTrait_ & RESET_KILL_ACTION ) {
+ _killRing.lastAction = KillRing::actionOther;
+ }
+ if ( actionTrait_ & SET_KILL_ACTION ) {
+ _killRing.lastAction = KillRing::actionKill;
+ }
+ if ( ! ( actionTrait_ & DONT_RESET_PREFIX ) ) {
+ _prefix = _pos;
+ }
+ if ( ! ( actionTrait_ & DONT_RESET_COMPLETIONS ) ) {
+ _completions.clear();
+ _completionSelection = -1;
+ _completionContextLength = 0;
+ }
+ if ( ! ( actionTrait_ & DONT_RESET_HIST_YANK_INDEX ) ) {
+ _history.reset_yank_iterator();
+ }
+ if ( actionTrait_ & WANT_REFRESH ) {
+ _modifiedState = true;
+ }
+ return ( res );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::insert_character( char32_t c ) {
+ /*
+ * beep on unknown Ctrl and/or Meta keys
+ * don't insert control characters
+ */
+ if ( ( c >= static_cast<int>( Replxx::KEY::BASE ) ) || ( is_control_code( c ) && ( c != '\n' ) ) ) {
+ beep();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ if ( ! _overwrite || ( _pos >= _data.length() ) ) {
+ _data.insert( _pos, c );
+ } else {
+ _data[_pos] = c;
+ }
+ ++ _pos;
+ call_modify_callback();
+ int long long now( now_us() );
+ int long long duration( now - _lastRefreshTime );
+ if ( duration < RAPID_REFRESH_US ) {
+ _lastRefreshTime = now;
+ _refreshSkipped = true;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ int inputLen = calculate_displayed_length( _data.get(), _data.length() );
+ if (
+ ( _pos == _data.length() )
+ && ! _modifiedState
+ && ( _noColor || ! ( !! _highlighterCallback || !! _hintCallback ) )
+ && ( _prompt.indentation() + inputLen < _prompt.screen_columns() )
+ ) {
+ /* Avoid a full assign of the line in the
+ * trivial case. */
+ render( c );
+ _displayInputLength = static_cast<int>( _display.size() );
+ _terminal.write32( reinterpret_cast<char32_t*>( &c ), 1 );
+ } else {
+ refresh_line();
+ }
+ _lastRefreshTime = now_us();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-J/linefeed/newline
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::new_line( char32_t ) {
+ return ( insert_character( '\n' ) );
+}
+
+// ctrl-A, HOME: move cursor to start of line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::go_to_begining_of_line( char32_t ) {
+ _pos = 0;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::go_to_end_of_line( char32_t ) {
+ _pos = _data.length();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-B, move cursor left by one character
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_char_left( char32_t ) {
+ if (_pos > 0) {
+ --_pos;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-F, move cursor right by one character
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_char_right( char32_t ) {
+ if ( _pos < _data.length() ) {
+ ++_pos;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-B, move cursor left by one word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_word_left( char32_t ) {
+ if (_pos > 0) {
+ while (_pos > 0 && is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ --_pos;
+ }
+ while (_pos > 0 && !is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ --_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-f, move cursor right by one word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_word_right( char32_t ) {
+ if ( _pos < _data.length() ) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++_pos;
+ }
+ while ( _pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ ++_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-Backspace, kill word to left of cursor
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_word_to_left( char32_t ) {
+ if ( _pos > 0 ) {
+ int startingPos = _pos;
+ while ( _pos > 0 && is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ -- _pos;
+ }
+ while ( _pos > 0 && !is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ -- _pos;
+ }
+ _killRing.kill( _data.get() + _pos, startingPos - _pos, false);
+ _data.erase( _pos, startingPos - _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-D, kill word to right of cursor
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_word_to_right( char32_t ) {
+ if ( _pos < _data.length() ) {
+ int endingPos = _pos;
+ while ( endingPos < _data.length() && is_word_break_character<subword>( _data[endingPos] ) ) {
+ ++ endingPos;
+ }
+ while ( endingPos < _data.length() && !is_word_break_character<subword>( _data[endingPos] ) ) {
+ ++ endingPos;
+ }
+ _killRing.kill( _data.get() + _pos, endingPos - _pos, true );
+ _data.erase( _pos, endingPos - _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-W, kill to whitespace (not word) to left of cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_to_whitespace_to_left( char32_t ) {
+ if ( _pos > 0 ) {
+ int startingPos = _pos;
+ while ( ( _pos > 0 ) && isspace( _data[_pos - 1] ) ) {
+ --_pos;
+ }
+ while ( ( _pos > 0 ) && ! isspace( _data[_pos - 1] ) ) {
+ -- _pos;
+ }
+ _killRing.kill( _data.get() + _pos, startingPos - _pos, false );
+ _data.erase( _pos, startingPos - _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-K, kill from cursor to end of line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_to_end_of_line( char32_t ) {
+ _killRing.kill( _data.get() + _pos, _data.length() - _pos, true );
+ _data.erase( _pos, _data.length() - _pos );
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-U, kill all characters to the left of the cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_to_begining_of_line( char32_t ) {
+ if (_pos > 0) {
+ _killRing.kill( _data.get(), _pos, false );
+ _data.erase( 0, _pos );
+ _pos = 0;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-Y, yank killed text
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::yank( char32_t ) {
+ UnicodeString* restoredText( _killRing.yank() );
+ if ( restoredText ) {
+ _data.insert( _pos, *restoredText, 0, restoredText->length() );
+ _pos += restoredText->length();
+ refresh_line();
+ _killRing.lastAction = KillRing::actionYank;
+ _lastYankSize = restoredText->length();
+ } else {
+ beep();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-Y, "yank-pop", rotate popped text
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::yank_cycle( char32_t ) {
+ if ( _killRing.lastAction != KillRing::actionYank ) {
+ beep();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ UnicodeString* restoredText = _killRing.yankPop();
+ if ( !restoredText ) {
+ beep();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ _pos -= _lastYankSize;
+ _data.erase( _pos, _lastYankSize );
+ _data.insert( _pos, *restoredText, 0, restoredText->length() );
+ _pos += restoredText->length();
+ _lastYankSize = restoredText->length();
+ refresh_line();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-., "yank-last-arg", on consecutive uses move back in history for popped text
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::yank_last_arg( char32_t ) {
+ if ( _history.size() < 2 ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ if ( _history.next_yank_position() ) {
+ _lastYankSize = 0;
+ }
+ UnicodeString const& histLine( _history.yank_line() );
+ int endPos( histLine.length() );
+ while ( ( endPos > 0 ) && isspace( histLine[endPos - 1] ) ) {
+ -- endPos;
+ }
+ int startPos( endPos );
+ while ( ( startPos > 0 ) && ! isspace( histLine[startPos - 1] ) ) {
+ -- startPos;
+ }
+ _pos -= _lastYankSize;
+ _data.erase( _pos, _lastYankSize );
+ _lastYankSize = endPos - startPos;
+ _data.insert( _pos, histLine, startPos, _lastYankSize );
+ _pos += _lastYankSize;
+ refresh_line();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-C, give word initial Cap
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::capitalize_word( char32_t ) {
+ if (_pos < _data.length()) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++_pos;
+ }
+ if (_pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'a' && _data[_pos] <= 'z' ) {
+ _data[_pos] += 'A' - 'a';
+ }
+ ++_pos;
+ }
+ while (_pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'A' && _data[_pos] <= 'Z' ) {
+ _data[_pos] += 'a' - 'A';
+ }
+ ++_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-L, lowercase word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::lowercase_word( char32_t ) {
+ if (_pos < _data.length()) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++ _pos;
+ }
+ while (_pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'A' && _data[_pos] <= 'Z' ) {
+ _data[_pos] += 'a' - 'A';
+ }
+ ++ _pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-U, uppercase word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::uppercase_word( char32_t ) {
+ if (_pos < _data.length()) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++ _pos;
+ }
+ while ( _pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'a' && _data[_pos] <= 'z') {
+ _data[_pos] += 'A' - 'a';
+ }
+ ++ _pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-T, transpose characters
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::transpose_characters( char32_t ) {
+ if ( _pos > 0 && _data.length() > 1 ) {
+ size_t leftCharPos = ( _pos == _data.length() ) ? _pos - 2 : _pos - 1;
+ char32_t aux = _data[leftCharPos];
+ _data[leftCharPos] = _data[leftCharPos + 1];
+ _data[leftCharPos + 1] = aux;
+ if ( _pos != _data.length() ) {
+ ++_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-C, abort this line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::abort_line( char32_t ) {
+ errno = EAGAIN;
+ _history.drop_last();
+ // we need one last refresh with the cursor at the end of the line
+ // so we don't display the next prompt over the previous input line
+ _pos = _data.length(); // pass _data.length() as _pos for EOL
+ _lastRefreshTime = 0;
+ refresh_line( _refreshSkipped ? HINT_ACTION::REGENERATE : HINT_ACTION::TRIM );
+ _terminal.write8( "^C\r\n", 4 );
+ return ( Replxx::ACTION_RESULT::BAIL );
+}
+
+// DEL, delete the character under the cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::delete_character( char32_t ) {
+ if ( ( _data.length() > 0 ) && ( _pos < _data.length() ) ) {
+ _data.erase( _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-D, delete the character under the cursor
+// on an empty line, exit the shell
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::send_eof( char32_t key_ ) {
+ if ( _data.length() == 0 ) {
+ _history.drop_last();
+ return ( Replxx::ACTION_RESULT::BAIL );
+ }
+ return ( delete_character( key_ ) );
+}
+
+// backspace/ctrl-H, delete char to left of cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::backspace_character( char32_t ) {
+ if ( _pos > 0 ) {
+ -- _pos;
+ _data.erase( _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-M/return/enter, accept line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::commit_line( char32_t ) {
+ // we need one last refresh with the cursor at the end of the line
+ // so we don't display the next prompt over the previous input line
+ _pos = _data.length(); // pass _data.length() as _pos for EOL
+ _lastRefreshTime = 0;
+ refresh_line( _refreshSkipped ? HINT_ACTION::REGENERATE : HINT_ACTION::TRIM );
+ _history.commit_index();
+ _history.drop_last();
+ return ( Replxx::ACTION_RESULT::RETURN );
+}
+
+// Down, recall next line in history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_next( char32_t ) {
+ return ( history_move( false ) );
+}
+
+// Up, recall previous line in history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_previous( char32_t ) {
+ return ( history_move( true ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_move( bool previous_ ) {
+ // if not already recalling, add the current line to the history list so
+ // we don't
+ // have to special case it
+ if ( _history.is_last() ) {
+ _history.update_last( _data );
+ }
+ if ( _history.is_empty() ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ if ( ! _history.move( previous_ ) ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ _data.assign( _history.current() );
+ _pos = _data.length();
+ refresh_line();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-<, beginning of history
+// Page Up, beginning of history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_first( char32_t ) {
+ return ( history_jump( true ) );
+}
+
+// meta->, end of history
+// Page Down, end of history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_last( char32_t ) {
+ return ( history_jump( false ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_jump( bool back_ ) {
+ // if not already recalling, add the current line to the history list so
+ // we don't
+ // have to special case it
+ if ( _history.is_last() ) {
+ _history.update_last( _data );
+ }
+ if ( ! _history.is_empty() ) {
+ _history.jump( back_ );
+ _data.assign( _history.current() );
+ _pos = _data.length();
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::hint_next( char32_t ) {
+ return ( hint_move( false ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::hint_previous( char32_t ) {
+ return ( hint_move( true ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::hint_move( bool previous_ ) {
+ if ( ! _noColor ) {
+ _killRing.lastAction = KillRing::actionOther;
+ if ( previous_ ) {
+ -- _hintSelection;
+ } else {
+ ++ _hintSelection;
+ }
+ refresh_line( HINT_ACTION::REPAINT );
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::toggle_overwrite_mode( char32_t ) {
+ _overwrite = ! _overwrite;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+#ifndef _WIN32
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::verbatim_insert( char32_t ) {
+ static int const MAX_ESC_SEQ( 32 );
+ char32_t buf[MAX_ESC_SEQ];
+ int len( _terminal.read_verbatim( buf, MAX_ESC_SEQ ) );
+ _data.insert( _pos, UnicodeString( buf, len ), 0, len );
+ _pos += len;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-Z, job control
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::suspend( char32_t ) {
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ raise( SIGSTOP ); // Break out in mid-line
+ }
+ // Redraw prompt
+ _prompt.write();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+#endif
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete_line( char32_t c ) {
+ if ( !! _completionCallback && ( _completeOnEmpty || ( _pos > 0 ) ) ) {
+ // complete_line does the actual completion and replacement
+ c = do_complete_line( c != 0 );
+
+ if ( static_cast<int>( c ) < 0 ) {
+ return ( Replxx::ACTION_RESULT::BAIL );
+ }
+ if ( c != 0 ) {
+ emulate_key_press( c );
+ }
+ } else {
+ insert_character( c );
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete( bool previous_ ) {
+ if ( _completions.empty() ) {
+ bool first( _completions.empty() );
+ int dataLen( _data.length() );
+ complete_line( 0 );
+ if ( ! _immediateCompletion && first && ( _data.length() > dataLen ) ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ }
+ int newSelection( _completionSelection + ( previous_ ? -1 : 1 ) );
+ if ( newSelection >= static_cast<int>( _completions.size() ) ) {
+ newSelection = -1;
+ } else if ( newSelection == -2 ) {
+ newSelection = static_cast<int>( _completions.size() ) - 1;
+ }
+ if ( _completionSelection != -1 ) {
+ int oldCompletionLength( max( _completions[_completionSelection].text().length() - _completionContextLength, 0 ) );
+ _pos -= oldCompletionLength;
+ _data.erase( _pos, oldCompletionLength );
+ }
+ if ( newSelection != -1 ) {
+ int newCompletionLength( max( _completions[newSelection].text().length() - _completionContextLength, 0 ) );
+ _data.insert( _pos, _completions[newSelection].text(), _completionContextLength, newCompletionLength );
+ _pos += newCompletionLength;
+ }
+ _completionSelection = newSelection;
+ refresh_line(); // Refresh the line
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete_next( char32_t ) {
+ return ( complete( false ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete_previous( char32_t ) {
+ return ( complete( true ) );
+}
+
+// Alt-P, reverse history search for prefix
+// Alt-P, reverse history search for prefix
+// Alt-N, forward history search for prefix
+// Alt-N, forward history search for prefix
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::common_prefix_search( char32_t startChar ) {
+ int prefixSize( calculate_displayed_length( _data.get(), _prefix ) );
+ if (
+ _history.common_prefix_search(
+ _data, prefixSize, ( startChar == ( Replxx::KEY::meta( 'p' ) ) ) || ( startChar == ( Replxx::KEY::meta( 'P' ) ) )
+ )
+ ) {
+ _data.assign( _history.current() );
+ _pos = _data.length();
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-R, reverse history search
+// ctrl-S, forward history search
+/**
+ * Incremental history search -- take over the prompt and keyboard as the user
+ * types a search string, deletes characters from it, changes _direction,
+ * and either accepts the found line (for execution orediting) or cancels.
+ * @param startChar - the character that began the search, used to set the initial
+ * _direction
+ */
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::incremental_history_search( char32_t startChar ) {
+ // if not already recalling, add the current line to the history list so we
+ // don't have to special case it
+ if ( _history.is_last() ) {
+ _history.update_last( _data );
+ }
+ _history.save_pos();
+ int historyLinePosition( _pos );
+ clear_self_to_end_of_screen();
+
+ DynamicPrompt dp( _terminal, (startChar == Replxx::KEY::control('R')) ? -1 : 1 );
+
+ // draw user's text with our prompt
+ dynamicRefresh(_prompt, dp, _data.get(), _data.length(), historyLinePosition);
+
+ // loop until we get an exit character
+ char32_t c( 0 );
+ bool keepLooping = true;
+ bool useSearchedLine = true;
+ bool searchAgain = false;
+ UnicodeString activeHistoryLine;
+ while ( keepLooping ) {
+ c = read_char();
+
+ switch (c) {
+ // these characters keep the selected text but do not execute it
+ case Replxx::KEY::control('A'): // ctrl-A, move cursor to start of line
+ case Replxx::KEY::HOME:
+ case Replxx::KEY::control('B'): // ctrl-B, move cursor left by one character
+ case Replxx::KEY::LEFT:
+ case Replxx::KEY::meta( 'b' ): // meta-B, move cursor left by one word
+ case Replxx::KEY::meta( 'B' ):
+ case Replxx::KEY::control( Replxx::KEY::LEFT ):
+ case Replxx::KEY::meta( Replxx::KEY::LEFT ): // Emacs allows Meta, bash & readline don't
+ case Replxx::KEY::control('D'):
+ case Replxx::KEY::meta( 'd' ): // meta-D, kill word to right of cursor
+ case Replxx::KEY::meta( 'D' ):
+ case Replxx::KEY::control('E'): // ctrl-E, move cursor to end of line
+ case Replxx::KEY::END:
+ case Replxx::KEY::control('F'): // ctrl-F, move cursor right by one character
+ case Replxx::KEY::RIGHT:
+ case Replxx::KEY::meta( 'f' ): // meta-F, move cursor right by one word
+ case Replxx::KEY::meta( 'F' ):
+ case Replxx::KEY::control( Replxx::KEY::RIGHT ):
+ case Replxx::KEY::meta( Replxx::KEY::RIGHT ): // Emacs allows Meta, bash & readline don't
+ case Replxx::KEY::meta( Replxx::KEY::BACKSPACE ):
+ case Replxx::KEY::control('J'):
+ case Replxx::KEY::control('K'): // ctrl-K, kill from cursor to end of line
+ case Replxx::KEY::ENTER:
+ case Replxx::KEY::control('N'): // ctrl-N, recall next line in history
+ case Replxx::KEY::control('P'): // ctrl-P, recall previous line in history
+ case Replxx::KEY::DOWN:
+ case Replxx::KEY::UP:
+ case Replxx::KEY::control('T'): // ctrl-T, transpose characters
+ case Replxx::KEY::control('U'): // ctrl-U, kill all characters to the left of the cursor
+ case Replxx::KEY::control('W'):
+ case Replxx::KEY::meta( 'y' ): // meta-Y, "yank-pop", rotate popped text
+ case Replxx::KEY::meta( 'Y' ):
+ case 127:
+ case Replxx::KEY::DELETE:
+ case Replxx::KEY::meta( '<' ): // start of history
+ case Replxx::KEY::PAGE_UP:
+ case Replxx::KEY::meta( '>' ): // end of history
+ case Replxx::KEY::PAGE_DOWN: {
+ keepLooping = false;
+ } break;
+
+ // these characters revert the input line to its previous state
+ case Replxx::KEY::control('C'): // ctrl-C, abort this line
+ case Replxx::KEY::control('G'):
+ case Replxx::KEY::control('L'): { // ctrl-L, clear screen and redisplay line
+ keepLooping = false;
+ useSearchedLine = false;
+ if (c != Replxx::KEY::control('L')) {
+ c = -1; // ctrl-C and ctrl-G just abort the search and do nothing else
+ }
+ } break;
+
+ // these characters stay in search mode and assign the display
+ case Replxx::KEY::control('S'):
+ case Replxx::KEY::control('R'): {
+ if ( dp._searchText.length() == 0 ) { // if no current search text, recall previous text
+ if ( _previousSearchText.length() > 0 ) {
+ dp._searchText = _previousSearchText;
+ }
+ }
+ if (
+ ( ( dp._direction == 1 ) && ( c == Replxx::KEY::control( 'R' ) ) )
+ || ( ( dp._direction == -1 ) && ( c == Replxx::KEY::control( 'S' ) ) )
+ ) {
+ dp._direction = 0 - dp._direction; // reverse direction
+ dp.updateSearchPrompt(); // change the prompt
+ } else {
+ searchAgain = true; // same direction, search again
+ }
+ } break;
+
+// job control is its own thing
+#ifndef _WIN32
+ case Replxx::KEY::control('Z'): { // ctrl-Z, job control
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ // Returning to Linux (whatever) shell, leave raw mode
+ // Break out in mid-line
+ // Back from Linux shell, re-enter raw mode
+ raise( SIGSTOP );
+ }
+ dynamicRefresh( dp, dp, activeHistoryLine.get(), activeHistoryLine.length(), historyLinePosition );
+ continue;
+ } break;
+#endif
+
+ // these characters assign the search string, and hence the selected input line
+ case Replxx::KEY::BACKSPACE: { // backspace/ctrl-H, delete char to left of cursor
+ if ( dp._searchText.length() > 0 ) {
+ dp._searchText.erase( dp._searchText.length() - 1 );
+ dp.updateSearchPrompt();
+ _history.restore_pos();
+ historyLinePosition = _pos;
+ } else {
+ beep();
+ }
+ } break;
+
+ case Replxx::KEY::control('Y'): { // ctrl-Y, yank killed text
+ } break;
+
+ default: {
+ if ( ! is_control_code( c ) && ( c < static_cast<int>( Replxx::KEY::BASE ) ) ) { // not an action character
+ dp._searchText.insert( dp._searchText.length(), c );
+ dp.updateSearchPrompt();
+ } else {
+ beep();
+ }
+ }
+ } // switch
+
+ // if we are staying in search mode, search now
+ if ( ! keepLooping ) {
+ break;
+ }
+ activeHistoryLine.assign( _history.current() );
+ if ( dp._searchText.length() > 0 ) {
+ bool found = false;
+ int lineSearchPos = historyLinePosition;
+ if ( searchAgain ) {
+ lineSearchPos += dp._direction;
+ }
+ searchAgain = false;
+ while ( true ) {
+ while (
+ dp._direction < 0
+ ? ( lineSearchPos >= 0 )
+ : ( ( lineSearchPos + dp._searchText.length() ) <= activeHistoryLine.length() )
+ ) {
+ if (
+ ( lineSearchPos >= 0 )
+ && ( ( lineSearchPos + dp._searchText.length() ) <= activeHistoryLine.length() )
+ && std::equal( dp._searchText.begin(), dp._searchText.end(), activeHistoryLine.begin() + lineSearchPos )
+ ) {
+ found = true;
+ break;
+ }
+ lineSearchPos += dp._direction;
+ }
+ if ( found ) {
+ historyLinePosition = lineSearchPos;
+ break;
+ } else if ( _history.move( dp._direction < 0 ) ) {
+ activeHistoryLine.assign( _history.current() );
+ lineSearchPos = ( dp._direction > 0 ) ? 0 : ( activeHistoryLine.length() - dp._searchText.length() );
+ } else {
+ historyLinePosition = _pos;
+ beep();
+ break;
+ }
+ } // while
+ if ( ! found ) {
+ _history.restore_pos();
+ }
+ } else {
+ _history.restore_pos();
+ historyLinePosition = _pos;
+ }
+ activeHistoryLine.assign( _history.current() );
+ dynamicRefresh( dp, dp, activeHistoryLine.get(), activeHistoryLine.length(), historyLinePosition ); // draw user's text with our prompt
+ } // while
+
+ // leaving history search, restore previous prompt, maybe make searched line
+ // current
+ Prompt pb( _terminal );
+ UnicodeString tempUnicode( &_prompt._text[_prompt._lastLinePosition], _prompt._text.length() - _prompt._lastLinePosition );
+ pb.set_text( tempUnicode );
+ pb.update_screen_columns();
+ if ( useSearchedLine && ( activeHistoryLine.length() > 0 ) ) {
+ _history.commit_index();
+ _data.assign( activeHistoryLine );
+ _pos = historyLinePosition;
+ _modifiedState = true;
+ } else if ( ! useSearchedLine ) {
+ _history.restore_pos();
+ }
+ dynamicRefresh(pb, _prompt, _data.get(), _data.length(), _pos); // redraw the original prompt with current input
+ _previousSearchText = dp._searchText; // save search text for possible reuse on ctrl-R ctrl-R
+ emulate_key_press( c ); // pass a character or -1 back to main loop
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-L, clear screen and redisplay line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::clear_screen( char32_t c ) {
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::WHOLE );
+ if ( c ) {
+ _prompt.write();
+ _prompt._cursorRowOffset = _prompt._extraLines;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::bracketed_paste( char32_t ) {
+ UnicodeString buf;
+ while ( char32_t c = _terminal.read_char() ) {
+ if ( c == KEY::PASTE_FINISH ) {
+ break;
+ }
+ if ( ( c == '\r' ) || ( c == KEY::control( 'M' ) ) ) {
+ c = '\n';
+ }
+ buf.push_back( c );
+ }
+ _data.insert( _pos, buf, 0, buf.length() );
+ _pos += buf.length();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+template <bool subword>
+bool Replxx::ReplxxImpl::is_word_break_character( char32_t char_ ) const {
+ bool wbc( false );
+ if ( char_ < 128 ) {
+ wbc = strchr( subword ? _subwordBreakChars.c_str() : _wordBreakChars.c_str(), static_cast<char>( char_ ) ) != nullptr;
+ }
+ return ( wbc );
+}
+
+void Replxx::ReplxxImpl::history_add( std::string const& line ) {
+ _history.add( UnicodeString( line ) );
+}
+
+bool Replxx::ReplxxImpl::history_save( std::string const& filename ) {
+ return ( _history.save( filename, false ) );
+}
+
+bool Replxx::ReplxxImpl::history_sync( std::string const& filename ) {
+ return ( _history.save( filename, true ) );
+}
+
+bool Replxx::ReplxxImpl::history_load( std::string const& filename ) {
+ return ( _history.load( filename ) );
+}
+
+void Replxx::ReplxxImpl::history_clear( void ) {
+ _history.clear();
+}
+
+int Replxx::ReplxxImpl::history_size( void ) const {
+ return ( _history.size() );
+}
+
+Replxx::HistoryScan::impl_t Replxx::ReplxxImpl::history_scan( void ) const {
+ return ( _history.scan() );
+}
+
+void Replxx::ReplxxImpl::set_modify_callback( Replxx::modify_callback_t const& fn ) {
+ _modifyCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_completion_callback( Replxx::completion_callback_t const& fn ) {
+ _completionCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_highlighter_callback( Replxx::highlighter_callback_t const& fn ) {
+ _highlighterCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_hint_callback( Replxx::hint_callback_t const& fn ) {
+ _hintCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_max_history_size( int len ) {
+ _history.set_max_size( len );
+}
+
+void Replxx::ReplxxImpl::set_completion_count_cutoff( int count ) {
+ _completionCountCutoff = count;
+}
+
+void Replxx::ReplxxImpl::set_max_hint_rows( int count ) {
+ _maxHintRows = count;
+}
+
+void Replxx::ReplxxImpl::set_hint_delay( int hintDelay_ ) {
+ _hintDelay = hintDelay_;
+}
+
+void Replxx::ReplxxImpl::set_word_break_characters( char const* wordBreakers ) {
+ _wordBreakChars = wordBreakers;
+}
+
+void Replxx::ReplxxImpl::set_subword_break_characters( char const* subwordBreakers ) {
+ _subwordBreakChars = subwordBreakers;
+}
+
+void Replxx::ReplxxImpl::set_double_tab_completion( bool val ) {
+ _doubleTabCompletion = val;
+}
+
+void Replxx::ReplxxImpl::set_complete_on_empty( bool val ) {
+ _completeOnEmpty = val;
+}
+
+void Replxx::ReplxxImpl::set_beep_on_ambiguous_completion( bool val ) {
+ _beepOnAmbiguousCompletion = val;
+}
+
+void Replxx::ReplxxImpl::set_immediate_completion( bool val ) {
+ _immediateCompletion = val;
+}
+
+void Replxx::ReplxxImpl::set_unique_history( bool val ) {
+ _history.set_unique( val );
+}
+
+void Replxx::ReplxxImpl::set_no_color( bool val ) {
+ _noColor = val;
+}
+
+/**
+ * Display the dynamic incremental search prompt and the current user input
+ * line.
+ * @param pi Prompt struct holding information about the prompt and our
+ * screen position
+ * @param buf32 input buffer to be displayed
+ * @param len count of characters in the buffer
+ * @param pos current cursor position within the buffer (0 <= pos <= len)
+ */
+void Replxx::ReplxxImpl::dynamicRefresh(Prompt& oldPrompt, Prompt& newPrompt, char32_t* buf32, int len, int pos) {
+ clear_self_to_end_of_screen( &oldPrompt );
+ // calculate the position of the end of the prompt
+ int xEndOfPrompt, yEndOfPrompt;
+ calculate_screen_position(
+ 0, 0, newPrompt.screen_columns(), newPrompt._characterCount,
+ xEndOfPrompt, yEndOfPrompt
+ );
+
+ // calculate the position of the end of the input line
+ int xEndOfInput, yEndOfInput;
+ calculate_screen_position(
+ xEndOfPrompt, yEndOfPrompt, newPrompt.screen_columns(),
+ calculate_displayed_length(buf32, len), xEndOfInput,
+ yEndOfInput
+ );
+
+ // calculate the desired position of the cursor
+ int xCursorPos, yCursorPos;
+ calculate_screen_position(
+ xEndOfPrompt, yEndOfPrompt, newPrompt.screen_columns(),
+ calculate_displayed_length(buf32, pos), xCursorPos,
+ yCursorPos
+ );
+
+ // display the prompt
+ newPrompt.write();
+
+ // display the input line
+ _terminal.write32( buf32, len );
+
+#ifndef _WIN32
+ // we have to generate our own newline on line wrap
+ if (xEndOfInput == 0 && yEndOfInput > 0) {
+ _terminal.write8( "\n", 1 );
+ }
+#endif
+ // position the cursor
+ _terminal.jump_cursor(
+ xCursorPos, // 0-based on Win32
+ -( yEndOfInput - yCursorPos )
+ );
+ newPrompt._cursorRowOffset = newPrompt._extraLines + yCursorPos; // remember row for next pass
+}
+
+}
+
diff --git a/contrib/replxx/src/replxx_impl.hxx b/contrib/replxx/src/replxx_impl.hxx
new file mode 100644
index 0000000..bec9383
--- /dev/null
+++ b/contrib/replxx/src/replxx_impl.hxx
@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HAVE_REPLXX_REPLXX_IMPL_HXX_INCLUDED
+#define HAVE_REPLXX_REPLXX_IMPL_HXX_INCLUDED 1
+
+#include <vector>
+#include <deque>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <thread>
+#include <mutex>
+#include <chrono>
+
+#include "replxx.hxx"
+#include "history.hxx"
+#include "killring.hxx"
+#include "utf8string.hxx"
+#include "prompt.hxx"
+
+namespace replxx {
+
+class Replxx::ReplxxImpl {
+public:
+ class Completion {
+ UnicodeString _text;
+ Replxx::Color _color;
+ public:
+ Completion( UnicodeString const& text_, Replxx::Color color_ )
+ : _text( text_ )
+ , _color( color_ ) {
+ }
+ Completion( Replxx::Completion const& completion_ )
+ : _text( completion_.text() )
+ , _color( completion_.color() ) {
+ }
+ Completion( Completion const& ) = default;
+ Completion& operator = ( Completion const& ) = default;
+ Completion( Completion&& ) = default;
+ Completion& operator = ( Completion&& ) = default;
+ UnicodeString const& text( void ) const {
+ return ( _text );
+ }
+ Replxx::Color color( void ) const {
+ return ( _color );
+ }
+ };
+ typedef std::vector<Completion> completions_t;
+ typedef std::vector<UnicodeString> data_t;
+ typedef std::vector<UnicodeString> hints_t;
+ typedef std::unique_ptr<char[]> utf8_buffer_t;
+ typedef std::unique_ptr<char32_t[]> input_buffer_t;
+ typedef std::vector<char32_t> display_t;
+ typedef std::deque<char32_t> key_presses_t;
+ typedef std::deque<std::string> messages_t;
+ enum class HINT_ACTION {
+ REGENERATE,
+ REPAINT,
+ TRIM,
+ SKIP
+ };
+ typedef std::unordered_map<std::string, Replxx::key_press_handler_t> named_actions_t;
+ typedef Replxx::ACTION_RESULT ( ReplxxImpl::* key_press_handler_raw_t )( char32_t );
+ typedef std::unordered_map<int, Replxx::key_press_handler_t> key_press_handlers_t;
+private:
+ typedef int long long unsigned action_trait_t;
+ static action_trait_t const NOOP = 0;
+ static action_trait_t const WANT_REFRESH = 1;
+ static action_trait_t const RESET_KILL_ACTION = 2;
+ static action_trait_t const SET_KILL_ACTION = 4;
+ static action_trait_t const DONT_RESET_PREFIX = 8;
+ static action_trait_t const DONT_RESET_COMPLETIONS = 16;
+ static action_trait_t const HISTORY_RECALL_MOST_RECENT = 32;
+ static action_trait_t const DONT_RESET_HIST_YANK_INDEX = 64;
+private:
+ mutable Utf8String _utf8Buffer;
+ UnicodeString _data;
+ int _pos; // character position in buffer ( 0 <= _pos <= _data[_line].length() )
+ display_t _display;
+ int _displayInputLength;
+ UnicodeString _hint;
+ int _prefix; // prefix length used in common prefix search
+ int _hintSelection; // Currently selected hint.
+ History _history;
+ KillRing _killRing;
+ int long long _lastRefreshTime;
+ bool _refreshSkipped;
+ int _lastYankSize;
+ int _maxHintRows;
+ int _hintDelay;
+ std::string _wordBreakChars;
+ std::string _subwordBreakChars;
+ int _completionCountCutoff;
+ bool _overwrite;
+ bool _doubleTabCompletion;
+ bool _completeOnEmpty;
+ bool _beepOnAmbiguousCompletion;
+ bool _immediateCompletion;
+ bool _bracketedPaste;
+ bool _noColor;
+ named_actions_t _namedActions;
+ key_press_handlers_t _keyPressHandlers;
+ Terminal _terminal;
+ std::thread::id _currentThread;
+ Prompt _prompt;
+ Replxx::modify_callback_t _modifyCallback;
+ Replxx::completion_callback_t _completionCallback;
+ Replxx::highlighter_callback_t _highlighterCallback;
+ Replxx::hint_callback_t _hintCallback;
+ key_presses_t _keyPresses;
+ messages_t _messages;
+ completions_t _completions;
+ int _completionContextLength;
+ int _completionSelection;
+ std::string _preloadedBuffer; // used with set_preload_buffer
+ std::string _errorMessage;
+ UnicodeString _previousSearchText; // remembered across invocations of replxx_input()
+ bool _modifiedState;
+ Replxx::Color _hintColor;
+ hints_t _hintsCache;
+ int _hintContextLenght;
+ Utf8String _hintSeed;
+ mutable std::mutex _mutex;
+public:
+ ReplxxImpl( FILE*, FILE*, FILE* );
+ virtual ~ReplxxImpl( void );
+ void set_modify_callback( Replxx::modify_callback_t const& fn );
+ void set_completion_callback( Replxx::completion_callback_t const& fn );
+ void set_highlighter_callback( Replxx::highlighter_callback_t const& fn );
+ void set_hint_callback( Replxx::hint_callback_t const& fn );
+ char const* input( std::string const& prompt );
+ void history_add( std::string const& line );
+ bool history_sync( std::string const& filename );
+ bool history_save( std::string const& filename );
+ bool history_load( std::string const& filename );
+ void history_clear( void );
+ Replxx::HistoryScan::impl_t history_scan( void ) const;
+ int history_size( void ) const;
+ void set_preload_buffer(std::string const& preloadText);
+ void set_word_break_characters( char const* wordBreakers );
+ void set_subword_break_characters( char const* subwordBreakers );
+ void set_max_hint_rows( int count );
+ void set_hint_delay( int milliseconds );
+ void set_double_tab_completion( bool val );
+ void set_complete_on_empty( bool val );
+ void set_beep_on_ambiguous_completion( bool val );
+ void set_immediate_completion( bool val );
+ void set_unique_history( bool );
+ void set_no_color( bool val );
+ void set_max_history_size( int len );
+ void set_completion_count_cutoff( int len );
+ int install_window_change_handler( void );
+ void enable_bracketed_paste( void );
+ void disable_bracketed_paste( void );
+ void print( char const*, int );
+ Replxx::ACTION_RESULT clear_screen( char32_t );
+ void emulate_key_press( char32_t );
+ Replxx::ACTION_RESULT invoke( Replxx::ACTION, char32_t );
+ void bind_key( char32_t, Replxx::key_press_handler_t );
+ void bind_key_internal( char32_t, char const* );
+ Replxx::State get_state( void ) const;
+ void set_state( Replxx::State const& );
+private:
+ ReplxxImpl( ReplxxImpl const& ) = delete;
+ ReplxxImpl& operator = ( ReplxxImpl const& ) = delete;
+private:
+ void preload_puffer( char const* preloadText );
+ int get_input_line( void );
+ Replxx::ACTION_RESULT action( action_trait_t, key_press_handler_raw_t const&, char32_t );
+ Replxx::ACTION_RESULT insert_character( char32_t );
+ Replxx::ACTION_RESULT new_line( char32_t );
+ Replxx::ACTION_RESULT go_to_begining_of_line( char32_t );
+ Replxx::ACTION_RESULT go_to_end_of_line( char32_t );
+ Replxx::ACTION_RESULT move_one_char_left( char32_t );
+ Replxx::ACTION_RESULT move_one_char_right( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT move_one_word_left( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT move_one_word_right( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT kill_word_to_left( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT kill_word_to_right( char32_t );
+ Replxx::ACTION_RESULT kill_to_whitespace_to_left( char32_t );
+ Replxx::ACTION_RESULT kill_to_begining_of_line( char32_t );
+ Replxx::ACTION_RESULT kill_to_end_of_line( char32_t );
+ Replxx::ACTION_RESULT yank( char32_t );
+ Replxx::ACTION_RESULT yank_cycle( char32_t );
+ Replxx::ACTION_RESULT yank_last_arg( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT capitalize_word( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT lowercase_word( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT uppercase_word( char32_t );
+ Replxx::ACTION_RESULT transpose_characters( char32_t );
+ Replxx::ACTION_RESULT abort_line( char32_t );
+ Replxx::ACTION_RESULT send_eof( char32_t );
+ Replxx::ACTION_RESULT delete_character( char32_t );
+ Replxx::ACTION_RESULT backspace_character( char32_t );
+ Replxx::ACTION_RESULT commit_line( char32_t );
+ Replxx::ACTION_RESULT history_next( char32_t );
+ Replxx::ACTION_RESULT history_previous( char32_t );
+ Replxx::ACTION_RESULT history_move( bool );
+ Replxx::ACTION_RESULT history_first( char32_t );
+ Replxx::ACTION_RESULT history_last( char32_t );
+ Replxx::ACTION_RESULT history_jump( bool );
+ Replxx::ACTION_RESULT hint_next( char32_t );
+ Replxx::ACTION_RESULT hint_previous( char32_t );
+ Replxx::ACTION_RESULT hint_move( bool );
+ Replxx::ACTION_RESULT toggle_overwrite_mode( char32_t );
+#ifndef _WIN32
+ Replxx::ACTION_RESULT verbatim_insert( char32_t );
+ Replxx::ACTION_RESULT suspend( char32_t );
+#endif
+ Replxx::ACTION_RESULT complete_line( char32_t );
+ Replxx::ACTION_RESULT complete_next( char32_t );
+ Replxx::ACTION_RESULT complete_previous( char32_t );
+ Replxx::ACTION_RESULT complete( bool );
+ Replxx::ACTION_RESULT incremental_history_search( char32_t startChar );
+ Replxx::ACTION_RESULT common_prefix_search( char32_t startChar );
+ Replxx::ACTION_RESULT bracketed_paste( char32_t startChar );
+ char32_t read_char( HINT_ACTION = HINT_ACTION::SKIP );
+ char const* read_from_stdin( void );
+ char32_t do_complete_line( bool );
+ void call_modify_callback( void );
+ completions_t call_completer( std::string const& input, int& ) const;
+ hints_t call_hinter( std::string const& input, int&, Replxx::Color& color ) const;
+ void refresh_line( HINT_ACTION = HINT_ACTION::REGENERATE );
+ void render( char32_t );
+ void render( HINT_ACTION );
+ int handle_hints( HINT_ACTION );
+ void set_color( Replxx::Color );
+ int context_length( void );
+ void clear( void );
+ void repaint( void );
+ template <bool subword>
+ bool is_word_break_character( char32_t ) const;
+ void dynamicRefresh(Prompt& oldPrompt, Prompt& newPrompt, char32_t* buf32, int len, int pos);
+ char const* finalize_input( char const* );
+ void clear_self_to_end_of_screen( Prompt const* = nullptr );
+ typedef struct {
+ int index;
+ bool error;
+ } paren_info_t;
+ paren_info_t matching_paren( void );
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/terminal.cxx b/contrib/replxx/src/terminal.cxx
new file mode 100644
index 0000000..e618219
--- /dev/null
+++ b/contrib/replxx/src/terminal.cxx
@@ -0,0 +1,742 @@
+#include <memory>
+#include <cerrno>
+#include <cstdlib>
+#include <cstring>
+#include <array>
+#include <stdexcept>
+
+#ifdef _WIN32
+
+#include <conio.h>
+#include <windows.h>
+#include <io.h>
+#define isatty _isatty
+#define strcasecmp _stricmp
+#define strdup _strdup
+#define write _write
+#define STDIN_FILENO 0
+
+#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
+static DWORD const ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4;
+#endif
+
+#include "windows.hxx"
+
+#else /* _WIN32 */
+
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/select.h>
+#include <fcntl.h>
+#include <signal.h>
+
+#endif /* _WIN32 */
+
+#include "terminal.hxx"
+#include "conversion.hxx"
+#include "escape.hxx"
+#include "replxx.hxx"
+#include "util.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+namespace tty {
+
+bool is_a_tty( int fd_ ) {
+ bool aTTY( isatty( fd_ ) != 0 );
+#ifdef _WIN32
+ do {
+ if ( aTTY ) {
+ break;
+ }
+ HANDLE h( (HANDLE)_get_osfhandle( fd_ ) );
+ if ( h == INVALID_HANDLE_VALUE ) {
+ break;
+ }
+ DWORD st( 0 );
+ if ( ! GetConsoleMode( h, &st ) ) {
+ break;
+ }
+ aTTY = true;
+ } while ( false );
+#endif
+ return ( aTTY );
+}
+
+bool in( is_a_tty( 0 ) );
+bool out( is_a_tty( 1 ) );
+
+}
+
+#ifndef _WIN32
+Terminal* _terminal_ = nullptr;
+static void WindowSizeChanged( int ) {
+ if ( ! _terminal_ ) {
+ return;
+ }
+ _terminal_->notify_event( Terminal::EVENT_TYPE::RESIZE );
+}
+#endif
+
+
+Terminal::Terminal( void )
+#ifdef _WIN32
+ : _consoleOut( INVALID_HANDLE_VALUE )
+ , _consoleIn( INVALID_HANDLE_VALUE )
+ , _origOutMode()
+ , _origInMode()
+ , _oldDisplayAttribute()
+ , _inputCodePage( GetConsoleCP() )
+ , _outputCodePage( GetConsoleOutputCP() )
+ , _interrupt( INVALID_HANDLE_VALUE )
+ , _events()
+ , _empty()
+#else
+ : _origTermios()
+ , _interrupt()
+#endif
+ , _rawMode( false )
+ , _utf8() {
+#ifdef _WIN32
+ _interrupt = CreateEvent( nullptr, true, false, TEXT( "replxx_interrupt_event" ) );
+#else
+ static_cast<void>( ::pipe( _interrupt ) == 0 );
+#endif
+}
+
+Terminal::~Terminal( void ) {
+ if ( _rawMode ) {
+ disable_raw_mode();
+ }
+#ifdef _WIN32
+ CloseHandle( _interrupt );
+#else
+ static_cast<void>( ::close( _interrupt[0] ) == 0 );
+ static_cast<void>( ::close( _interrupt[1] ) == 0 );
+#endif
+}
+
+void Terminal::write32( char32_t const* text32, int len32 ) {
+ _utf8.assign( text32, len32 );
+ write8( _utf8.get(), _utf8.size() );
+ return;
+}
+
+void Terminal::write8( char const* data_, int size_ ) {
+#ifdef _WIN32
+ if ( ! _rawMode ) {
+ enable_out();
+ }
+ int nWritten( win_write( _consoleOut, _autoEscape, data_, size_ ) );
+ if ( ! _rawMode ) {
+ disable_out();
+ }
+#else
+ int nWritten( write( 1, data_, size_ ) );
+#endif
+ if ( nWritten != size_ ) {
+ throw std::runtime_error( "write failed" );
+ }
+ return;
+}
+
+int Terminal::get_screen_columns( void ) {
+ int cols( 0 );
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ GetConsoleScreenBufferInfo( _consoleOut, &inf );
+ cols = inf.dwSize.X;
+#else
+ struct winsize ws;
+ cols = ( ioctl( 1, TIOCGWINSZ, &ws ) == -1 ) ? 80 : ws.ws_col;
+#endif
+ // cols is 0 in certain circumstances like inside debugger, which creates
+ // further issues
+ return ( cols > 0 ) ? cols : 80;
+}
+
+int Terminal::get_screen_rows( void ) {
+ int rows;
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ GetConsoleScreenBufferInfo( _consoleOut, &inf );
+ rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top;
+#else
+ struct winsize ws;
+ rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row;
+#endif
+ return (rows > 0) ? rows : 24;
+}
+
+namespace {
+inline int notty( void ) {
+ errno = ENOTTY;
+ return ( -1 );
+}
+}
+
+void Terminal::enable_out( void ) {
+#ifdef _WIN32
+ _consoleOut = GetStdHandle( STD_OUTPUT_HANDLE );
+ SetConsoleOutputCP( 65001 );
+ GetConsoleMode( _consoleOut, &_origOutMode );
+ _autoEscape = SetConsoleMode( _consoleOut, _origOutMode | ENABLE_VIRTUAL_TERMINAL_PROCESSING ) != 0;
+#endif
+}
+
+void Terminal::disable_out( void ) {
+#ifdef _WIN32
+ SetConsoleMode( _consoleOut, _origOutMode );
+ SetConsoleOutputCP( _outputCodePage );
+ _consoleOut = INVALID_HANDLE_VALUE;
+ _autoEscape = false;
+#endif
+}
+
+void Terminal::enable_bracketed_paste( void ) {
+ static char const BRACK_PASTE_INIT[] = "\033[?2004h";
+ write8( BRACK_PASTE_INIT, sizeof ( BRACK_PASTE_INIT ) - 1 );
+}
+
+void Terminal::disable_bracketed_paste( void ) {
+ static char const BRACK_PASTE_DISABLE[] = "\033[?2004l";
+ write8( BRACK_PASTE_DISABLE, sizeof ( BRACK_PASTE_DISABLE ) - 1 );
+}
+
+int Terminal::enable_raw_mode( void ) {
+ if ( ! _rawMode ) {
+#ifdef _WIN32
+ _consoleIn = GetStdHandle( STD_INPUT_HANDLE );
+ SetConsoleCP( 65001 );
+ GetConsoleMode( _consoleIn, &_origInMode );
+ SetConsoleMode(
+ _consoleIn,
+ _origInMode & ~( ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT )
+ );
+ enable_out();
+#else
+ struct termios raw;
+
+ if ( ! tty::in ) {
+ return ( notty() );
+ }
+ if ( tcgetattr( 0, &_origTermios ) == -1 ) {
+ return ( notty() );
+ }
+
+ raw = _origTermios; /* modify the original mode */
+ /* input modes: no break, no CR to NL, no parity check, no strip char,
+ * no start/stop output control. */
+ raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
+ /* output modes - disable post processing */
+ // this is wrong, we don't want raw output, it turns newlines into straight
+ // linefeeds
+ // raw.c_oflag &= ~(OPOST);
+ /* control modes - set 8 bit chars */
+ raw.c_cflag |= (CS8);
+ /* local modes - echoing off, canonical off, no extended functions,
+ * no signal chars (^Z,^C) */
+ raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
+ /* control chars - set return condition: min number of bytes and timer.
+ * We want read to return every single byte, without timeout. */
+ raw.c_cc[VMIN] = 1;
+ raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
+
+ /* put terminal in raw mode after flushing */
+ if ( tcsetattr(0, TCSADRAIN, &raw) < 0 ) {
+ return ( notty() );
+ }
+ _terminal_ = this;
+#endif
+ _rawMode = true;
+ }
+ return ( 0 );
+}
+
+void Terminal::disable_raw_mode(void) {
+ if ( _rawMode ) {
+#ifdef _WIN32
+ disable_out();
+ SetConsoleMode( _consoleIn, _origInMode );
+ SetConsoleCP( _inputCodePage );
+ _consoleIn = INVALID_HANDLE_VALUE;
+#else
+ _terminal_ = nullptr;
+ if ( tcsetattr( 0, TCSADRAIN, &_origTermios ) == -1 ) {
+ return;
+ }
+#endif
+ _rawMode = false;
+ }
+}
+
+#ifndef _WIN32
+
+/**
+ * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode
+ * (char32_t) character it encodes
+ *
+ * @return char32_t Unicode character
+ */
+char32_t read_unicode_character(void) {
+ static char8_t utf8String[5];
+ static size_t utf8Count = 0;
+ while (true) {
+ char8_t c;
+
+ /* Continue reading if interrupted by signal. */
+ ssize_t nread;
+ do {
+ nread = read( STDIN_FILENO, &c, 1 );
+ } while ((nread == -1) && (errno == EINTR));
+
+ if (nread <= 0) return 0;
+ if (c <= 0x7F || locale::is8BitEncoding) { // short circuit ASCII
+ utf8Count = 0;
+ return c;
+ } else if (utf8Count < sizeof(utf8String) - 1) {
+ utf8String[utf8Count++] = c;
+ utf8String[utf8Count] = 0;
+ char32_t unicodeChar[2];
+ int ucharCount( 0 );
+ ConversionResult res = copyString8to32(unicodeChar, 2, ucharCount, utf8String);
+ if (res == conversionOK && ucharCount) {
+ utf8Count = 0;
+ return unicodeChar[0];
+ }
+ } else {
+ utf8Count = 0; // this shouldn't happen: got four bytes but no UTF-8 character
+ }
+ }
+}
+
+#endif // #ifndef _WIN32
+
+void beep() {
+ fprintf(stderr, "\x7"); // ctrl-G == bell/beep
+ fflush(stderr);
+}
+
+// replxx_read_char -- read a keystroke or keychord from the keyboard, and translate it
+// into an encoded "keystroke". When convenient, extended keys are translated into their
+// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B.
+//
+// A return value of zero means "no input available", and a return value of -1
+// means "invalid key".
+//
+char32_t Terminal::read_char( void ) {
+ char32_t c( 0 );
+#ifdef _WIN32
+ INPUT_RECORD rec;
+ DWORD count;
+ char32_t modifierKeys = 0;
+ bool escSeen = false;
+ int highSurrogate( 0 );
+ while (true) {
+ ReadConsoleInputW( _consoleIn, &rec, 1, &count );
+#if __REPLXX_DEBUG__ // helper for debugging keystrokes, display info in the debug "Output"
+ // window in the debugger
+ {
+ if ( rec.EventType == KEY_EVENT ) {
+ //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) {
+ char buf[1024];
+ sprintf(
+ buf,
+ "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, "
+ "virtual scancode 0x%04X, key %s%s%s%s%s\n",
+ rec.Event.KeyEvent.uChar.UnicodeChar,
+ rec.Event.KeyEvent.wRepeatCount,
+ rec.Event.KeyEvent.wVirtualKeyCode,
+ rec.Event.KeyEvent.wVirtualScanCode,
+ rec.Event.KeyEvent.bKeyDown ? "down" : "up",
+ (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED) ? " L-Ctrl" : "",
+ (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ? " R-Ctrl" : "",
+ (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED) ? " L-Alt" : "",
+ (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED) ? " R-Alt" : ""
+ );
+ OutputDebugStringA( buf );
+ //}
+ }
+ }
+#endif
+ if ( rec.EventType != KEY_EVENT ) {
+ continue;
+ }
+ // Windows provides for entry of characters that are not on your keyboard by sending the
+ // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU == Alt key) ...
+ // accept these characters, otherwise only process characters on "key down"
+ if ( !rec.Event.KeyEvent.bKeyDown && ( rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU ) ) {
+ continue;
+ }
+ modifierKeys = 0;
+ // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't treat this
+ // combination as either CTRL or META we just turn off those two bits, so it is still
+ // possible to combine CTRL and/or META with an AltGr key by using right-Ctrl and/or
+ // left-Alt
+ DWORD const AltGr( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED );
+ if ( ( rec.Event.KeyEvent.dwControlKeyState & AltGr ) == AltGr ) {
+ rec.Event.KeyEvent.dwControlKeyState &= ~( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED );
+ }
+ if ( rec.Event.KeyEvent.dwControlKeyState & ( RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED ) ) {
+ modifierKeys |= Replxx::KEY::BASE_CONTROL;
+ }
+ if ( rec.Event.KeyEvent.dwControlKeyState & ( RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED ) ) {
+ modifierKeys |= Replxx::KEY::BASE_META;
+ }
+ if ( escSeen ) {
+ modifierKeys |= Replxx::KEY::BASE_META;
+ }
+ int key( rec.Event.KeyEvent.uChar.UnicodeChar );
+ if ( key == 0 ) {
+ switch (rec.Event.KeyEvent.wVirtualKeyCode) {
+ case VK_LEFT:
+ return modifierKeys | Replxx::KEY::LEFT;
+ case VK_RIGHT:
+ return modifierKeys | Replxx::KEY::RIGHT;
+ case VK_UP:
+ return modifierKeys | Replxx::KEY::UP;
+ case VK_DOWN:
+ return modifierKeys | Replxx::KEY::DOWN;
+ case VK_DELETE:
+ return modifierKeys | Replxx::KEY::DELETE;
+ case VK_HOME:
+ return modifierKeys | Replxx::KEY::HOME;
+ case VK_END:
+ return modifierKeys | Replxx::KEY::END;
+ case VK_PRIOR:
+ return modifierKeys | Replxx::KEY::PAGE_UP;
+ case VK_NEXT:
+ return modifierKeys | Replxx::KEY::PAGE_DOWN;
+ case VK_F1:
+ return modifierKeys | Replxx::KEY::F1;
+ case VK_F2:
+ return modifierKeys | Replxx::KEY::F2;
+ case VK_F3:
+ return modifierKeys | Replxx::KEY::F3;
+ case VK_F4:
+ return modifierKeys | Replxx::KEY::F4;
+ case VK_F5:
+ return modifierKeys | Replxx::KEY::F5;
+ case VK_F6:
+ return modifierKeys | Replxx::KEY::F6;
+ case VK_F7:
+ return modifierKeys | Replxx::KEY::F7;
+ case VK_F8:
+ return modifierKeys | Replxx::KEY::F8;
+ case VK_F9:
+ return modifierKeys | Replxx::KEY::F9;
+ case VK_F10:
+ return modifierKeys | Replxx::KEY::F10;
+ case VK_F11:
+ return modifierKeys | Replxx::KEY::F11;
+ case VK_F12:
+ return modifierKeys | Replxx::KEY::F12;
+ default:
+ continue; // in raw mode, ReadConsoleInput shows shift, ctrl - ignore them
+ }
+ } else if ( key == Replxx::KEY::ESCAPE ) { // ESC, set flag for later
+ escSeen = true;
+ continue;
+ } else if ( ( key >= 0xD800 ) && ( key <= 0xDBFF ) ) {
+ highSurrogate = key - 0xD800;
+ continue;
+ } else {
+ // we got a real character, return it
+ if ( ( key >= 0xDC00 ) && ( key <= 0xDFFF ) ) {
+ key -= 0xDC00;
+ key |= ( highSurrogate << 10 );
+ key += 0x10000;
+ }
+ if ( is_control_code( key ) ) {
+ key = control_to_human( key );
+ modifierKeys |= Replxx::KEY::BASE_CONTROL;
+ }
+ key |= modifierKeys;
+ highSurrogate = 0;
+ c = key;
+ break;
+ }
+ }
+
+#else
+ c = read_unicode_character();
+ if (c == 0) {
+ return 0;
+ }
+
+// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard
+// debugging mode
+// where we print out decimal and decoded values for whatever the "terminal"
+// program
+// gives us on different keystrokes. Hit ctrl-C to exit this mode.
+//
+#ifdef __REPLXX_DEBUG__
+ if (c == ctrlChar('^')) { // ctrl-^, special debug mode, prints all keys hit,
+ // ctrl-C to get out
+ printf(
+ "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit "
+ "this mode\n");
+ while (true) {
+ unsigned char keys[10];
+ int ret = read(0, keys, 10);
+
+ if (ret <= 0) {
+ printf("\nret: %d\n", ret);
+ }
+ for (int i = 0; i < ret; ++i) {
+ char32_t key = static_cast<char32_t>(keys[i]);
+ char* friendlyTextPtr;
+ char friendlyTextBuf[10];
+ const char* prefixText = (key < 0x80) ? "" : "0x80+";
+ char32_t keyCopy = (key < 0x80) ? key : key - 0x80;
+ if (keyCopy >= '!' && keyCopy <= '~') { // printable
+ friendlyTextBuf[0] = '\'';
+ friendlyTextBuf[1] = keyCopy;
+ friendlyTextBuf[2] = '\'';
+ friendlyTextBuf[3] = 0;
+ friendlyTextPtr = friendlyTextBuf;
+ } else if (keyCopy == ' ') {
+ friendlyTextPtr = const_cast<char*>("space");
+ } else if (keyCopy == 27) {
+ friendlyTextPtr = const_cast<char*>("ESC");
+ } else if (keyCopy == 0) {
+ friendlyTextPtr = const_cast<char*>("NUL");
+ } else if (keyCopy == 127) {
+ friendlyTextPtr = const_cast<char*>("DEL");
+ } else {
+ friendlyTextBuf[0] = '^';
+ friendlyTextBuf[1] = control_to_human( keyCopy );
+ friendlyTextBuf[2] = 0;
+ friendlyTextPtr = friendlyTextBuf;
+ }
+ printf("%d x%02X (%s%s) ", key, key, prefixText, friendlyTextPtr);
+ }
+ printf("\x1b[1G\n"); // go to first column of new line
+
+ // drop out of this loop on ctrl-C
+ if (keys[0] == ctrlChar('C')) {
+ printf("Leaving keyboard debugging mode (on ctrl-C)\n");
+ fflush(stdout);
+ return -2;
+ }
+ }
+ }
+#endif // __REPLXX_DEBUG__
+
+ c = EscapeSequenceProcessing::doDispatch(c);
+ if ( is_control_code( c ) ) {
+ c = Replxx::KEY::control( control_to_human( c ) );
+ }
+#endif // #_WIN32
+ return ( c );
+}
+
+Terminal::EVENT_TYPE Terminal::wait_for_input( int long timeout_ ) {
+#ifdef _WIN32
+ std::array<HANDLE,2> handles = { _consoleIn, _interrupt };
+ while ( true ) {
+ DWORD event( WaitForMultipleObjects( static_cast<DWORD>( handles.size() ), handles.data(), false, timeout_ > 0 ? timeout_ : INFINITE ) );
+ switch ( event ) {
+ case ( WAIT_OBJECT_0 + 0 ): {
+ // peek events that will be skipped
+ INPUT_RECORD rec;
+ DWORD count;
+ PeekConsoleInputW( _consoleIn, &rec, 1, &count );
+
+ if (
+ ( rec.EventType != KEY_EVENT )
+ || ( !rec.Event.KeyEvent.bKeyDown && ( rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU ) )
+ ) {
+ // read the event to unsignal the handle
+ ReadConsoleInputW( _consoleIn, &rec, 1, &count );
+ continue;
+ } else if (rec.EventType == KEY_EVENT) {
+ int key(rec.Event.KeyEvent.uChar.UnicodeChar);
+ if (key == 0) {
+ switch (rec.Event.KeyEvent.wVirtualKeyCode) {
+ case VK_LEFT:
+ case VK_RIGHT:
+ case VK_UP:
+ case VK_DOWN:
+ case VK_DELETE:
+ case VK_HOME:
+ case VK_END:
+ case VK_PRIOR:
+ case VK_NEXT:
+ break;
+ default:
+ ReadConsoleInputW(_consoleIn, &rec, 1, &count);
+ continue; // in raw mode, ReadConsoleInput shows shift, ctrl - ignore them
+ }
+ }
+ }
+
+ return ( EVENT_TYPE::KEY_PRESS );
+ }
+ case ( WAIT_OBJECT_0 + 1 ): {
+ ResetEvent( _interrupt );
+ if ( _events.empty() ) {
+ continue;
+ }
+ EVENT_TYPE eventType( _events.front() );
+ _events.pop_front();
+ return ( eventType );
+ }
+ case ( WAIT_TIMEOUT ): {
+ return ( EVENT_TYPE::TIMEOUT );
+ }
+ }
+ }
+#else
+ fd_set fdSet;
+ int nfds( max( _interrupt[0], _interrupt[1] ) + 1 );
+ while ( true ) {
+ FD_ZERO( &fdSet );
+ FD_SET( 0, &fdSet );
+ FD_SET( _interrupt[0], &fdSet );
+ timeval tv{ timeout_ / 1000, static_cast<suseconds_t>( ( timeout_ % 1000 ) * 1000 ) };
+ int err( select( nfds, &fdSet, nullptr, nullptr, timeout_ > 0 ? &tv : nullptr ) );
+ if ( ( err == -1 ) && ( errno == EINTR ) ) {
+ continue;
+ }
+ if ( err == 0 ) {
+ return ( EVENT_TYPE::TIMEOUT );
+ }
+ if ( FD_ISSET( _interrupt[0], &fdSet ) ) {
+ char data( 0 );
+ static_cast<void>( read( _interrupt[0], &data, 1 ) == 1 );
+ if ( data == 'k' ) {
+ return ( EVENT_TYPE::KEY_PRESS );
+ }
+ if ( data == 'm' ) {
+ return ( EVENT_TYPE::MESSAGE );
+ }
+ if ( data == 'r' ) {
+ return ( EVENT_TYPE::RESIZE );
+ }
+ }
+ if ( FD_ISSET( 0, &fdSet ) ) {
+ return ( EVENT_TYPE::KEY_PRESS );
+ }
+ }
+#endif
+}
+
+void Terminal::notify_event( EVENT_TYPE eventType_ ) {
+#ifdef _WIN32
+ _events.push_back( eventType_ );
+ SetEvent( _interrupt );
+#else
+ char data( ( eventType_ == EVENT_TYPE::KEY_PRESS ) ? 'k' : ( eventType_ == EVENT_TYPE::MESSAGE ? 'm' : 'r' ) );
+ static_cast<void>( write( _interrupt[1], &data, 1 ) == 1 );
+#endif
+}
+
+/**
+ * Clear the screen ONLY (no redisplay of anything)
+ */
+void Terminal::clear_screen( CLEAR_SCREEN clearScreen_ ) {
+#ifdef _WIN32
+ if ( _autoEscape ) {
+#endif
+ if ( clearScreen_ == CLEAR_SCREEN::WHOLE ) {
+ char const clearCode[] = "\033c\033[H\033[2J\033[0m";
+ static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+ } else {
+ char const clearCode[] = "\033[J";
+ static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+ }
+ return;
+#ifdef _WIN32
+ }
+ COORD coord = { 0, 0 };
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ HANDLE consoleOut( _consoleOut != INVALID_HANDLE_VALUE ? _consoleOut : GetStdHandle( STD_OUTPUT_HANDLE ) );
+ GetConsoleScreenBufferInfo( consoleOut, &inf );
+ if ( clearScreen_ == CLEAR_SCREEN::TO_END ) {
+ coord = inf.dwCursorPosition;
+ DWORD nWritten( 0 );
+ SHORT height( inf.srWindow.Bottom - inf.srWindow.Top );
+ DWORD yPos( inf.dwCursorPosition.Y - inf.srWindow.Top );
+ DWORD toWrite( ( height + 1 - yPos ) * inf.dwSize.X - inf.dwCursorPosition.X );
+// FillConsoleOutputCharacterA( consoleOut, ' ', toWrite, coord, &nWritten );
+ _empty.resize( toWrite - 1, ' ' );
+ WriteConsoleA( consoleOut, _empty.data(), toWrite - 1, &nWritten, nullptr );
+ } else {
+ COORD scrollTarget = { 0, -inf.dwSize.Y };
+ CHAR_INFO fill{ TEXT( ' ' ), inf.wAttributes };
+ SMALL_RECT scrollRect = { 0, 0, inf.dwSize.X, inf.dwSize.Y };
+ ScrollConsoleScreenBuffer( consoleOut, &scrollRect, nullptr, scrollTarget, &fill );
+ }
+ SetConsoleCursorPosition( consoleOut, coord );
+#endif
+}
+
+void Terminal::jump_cursor( int xPos_, int yOffset_ ) {
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ GetConsoleScreenBufferInfo( _consoleOut, &inf );
+ inf.dwCursorPosition.X = xPos_;
+ inf.dwCursorPosition.Y += yOffset_;
+ SetConsoleCursorPosition( _consoleOut, inf.dwCursorPosition );
+#else
+ char seq[64];
+ if ( yOffset_ != 0 ) { // move the cursor up as required
+ snprintf( seq, sizeof seq, "\033[%d%c", abs( yOffset_ ), yOffset_ > 0 ? 'B' : 'A' );
+ write8( seq, strlen( seq ) );
+ }
+ // position at the end of the prompt, clear to end of screen
+ snprintf(
+ seq, sizeof seq, "\033[%dG",
+ xPos_ + 1 /* 1-based on VT100 */
+ );
+ write8( seq, strlen( seq ) );
+#endif
+}
+
+#ifdef _WIN32
+void Terminal::set_cursor_visible( bool visible_ ) {
+ CONSOLE_CURSOR_INFO cursorInfo;
+ GetConsoleCursorInfo( _consoleOut, &cursorInfo );
+ cursorInfo.bVisible = visible_;
+ SetConsoleCursorInfo( _consoleOut, &cursorInfo );
+ return;
+}
+#else
+void Terminal::set_cursor_visible( bool ) {}
+#endif
+
+#ifndef _WIN32
+int Terminal::read_verbatim( char32_t* buffer_, int size_ ) {
+ int len( 0 );
+ buffer_[len ++] = read_unicode_character();
+ int statusFlags( ::fcntl( STDIN_FILENO, F_GETFL, 0 ) );
+ ::fcntl( STDIN_FILENO, F_SETFL, statusFlags | O_NONBLOCK );
+ while ( len < size_ ) {
+ char32_t c( read_unicode_character() );
+ if ( c == 0 ) {
+ break;
+ }
+ buffer_[len ++] = c;
+ }
+ ::fcntl( STDIN_FILENO, F_SETFL, statusFlags );
+ return ( len );
+}
+
+int Terminal::install_window_change_handler( void ) {
+ struct sigaction sa;
+ sigemptyset(&sa.sa_mask);
+ sa.sa_flags = 0;
+ sa.sa_handler = &WindowSizeChanged;
+
+ if (sigaction(SIGWINCH, &sa, nullptr) == -1) {
+ return errno;
+ }
+ return 0;
+}
+#endif
+
+}
+
diff --git a/contrib/replxx/src/terminal.hxx b/contrib/replxx/src/terminal.hxx
new file mode 100644
index 0000000..e6a2578
--- /dev/null
+++ b/contrib/replxx/src/terminal.hxx
@@ -0,0 +1,94 @@
+#ifndef REPLXX_IO_HXX_INCLUDED
+#define REPLXX_IO_HXX_INCLUDED 1
+
+#include <deque>
+
+#ifdef _WIN32
+#include <vector>
+#include <windows.h>
+#else
+#include <termios.h>
+#endif
+
+#include "utf8string.hxx"
+
+namespace replxx {
+
+class Terminal {
+public:
+ enum class EVENT_TYPE {
+ KEY_PRESS,
+ MESSAGE,
+ TIMEOUT,
+ RESIZE
+ };
+private:
+#ifdef _WIN32
+ HANDLE _consoleOut;
+ HANDLE _consoleIn;
+ DWORD _origOutMode;
+ DWORD _origInMode;
+ bool _autoEscape;
+ WORD _oldDisplayAttribute;
+ UINT const _inputCodePage;
+ UINT const _outputCodePage;
+ HANDLE _interrupt;
+ typedef std::deque<EVENT_TYPE> events_t;
+ events_t _events;
+ std::vector<char> _empty;
+#else
+ struct termios _origTermios; /* in order to restore at exit */
+ int _interrupt[2];
+#endif
+ bool _rawMode; /* for destructor to check if restore is needed */
+ Utf8String _utf8;
+public:
+ enum class CLEAR_SCREEN {
+ WHOLE,
+ TO_END
+ };
+public:
+ Terminal( void );
+ ~Terminal( void );
+ void write32( char32_t const*, int );
+ void write8( char const*, int );
+ int get_screen_columns(void);
+ int get_screen_rows(void);
+ void enable_bracketed_paste( void );
+ void disable_bracketed_paste( void );
+ int enable_raw_mode(void);
+ void disable_raw_mode(void);
+ char32_t read_char(void);
+ void clear_screen( CLEAR_SCREEN );
+ EVENT_TYPE wait_for_input( int long = 0 );
+ void notify_event( EVENT_TYPE );
+ void jump_cursor( int, int );
+ void set_cursor_visible( bool );
+#ifndef _WIN32
+ int read_verbatim( char32_t*, int );
+ int install_window_change_handler( void );
+#endif
+private:
+ void enable_out( void );
+ void disable_out( void );
+private:
+ Terminal( Terminal const& ) = delete;
+ Terminal& operator = ( Terminal const& ) = delete;
+ Terminal( Terminal&& ) = delete;
+ Terminal& operator = ( Terminal&& ) = delete;
+};
+
+void beep();
+char32_t read_unicode_character(void);
+
+namespace tty {
+
+extern bool in;
+extern bool out;
+
+}
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/unicodestring.hxx b/contrib/replxx/src/unicodestring.hxx
new file mode 100644
index 0000000..8ff98a7
--- /dev/null
+++ b/contrib/replxx/src/unicodestring.hxx
@@ -0,0 +1,201 @@
+#ifndef REPLXX_UNICODESTRING_HXX_INCLUDED
+#define REPLXX_UNICODESTRING_HXX_INCLUDED
+
+#include <vector>
+#include <cstring>
+#include <string>
+
+#include "conversion.hxx"
+
+namespace replxx {
+
+class UnicodeString {
+public:
+ typedef std::vector<char32_t> data_buffer_t;
+ typedef data_buffer_t::const_iterator const_iterator;
+ typedef data_buffer_t::iterator iterator;
+private:
+ data_buffer_t _data;
+public:
+ UnicodeString()
+ : _data() {
+ }
+
+ explicit UnicodeString( std::string const& src )
+ : _data() {
+ assign( src );
+ }
+
+ explicit UnicodeString( UnicodeString const& other, int offset, int len = -1 )
+ : _data() {
+ _data.insert(
+ _data.end(),
+ other._data.begin() + offset,
+ len > 0 ? other._data.begin() + offset + len : other._data.end()
+ );
+ }
+
+ explicit UnicodeString( char const* src )
+ : _data() {
+ assign( src );
+ }
+
+ explicit UnicodeString( char8_t const* src )
+ : UnicodeString( reinterpret_cast<const char*>( src ) ) {
+ }
+
+ explicit UnicodeString( char32_t const* src )
+ : _data() {
+ int len( 0 );
+ while ( src[len] != 0 ) {
+ ++ len;
+ }
+ _data.assign( src, src + len );
+ }
+
+ explicit UnicodeString( char32_t const* src, int len )
+ : _data() {
+ _data.assign( src, src + len );
+ }
+
+ explicit UnicodeString( int len )
+ : _data() {
+ _data.resize( len );
+ }
+
+ UnicodeString& assign( std::string const& str_ ) {
+ _data.resize( static_cast<int>( str_.length() ) );
+ int len( 0 );
+ copyString8to32( _data.data(), static_cast<int>( str_.length() ), len, str_.c_str() );
+ _data.resize( len );
+ return *this;
+ }
+
+ UnicodeString& assign( char const* str_ ) {
+ int byteCount( static_cast<int>( strlen( str_ ) ) );
+ _data.resize( byteCount );
+ int len( 0 );
+ copyString8to32( _data.data(), byteCount, len, str_ );
+ _data.resize( len );
+ return *this;
+ }
+
+ UnicodeString& assign( UnicodeString const& other_ ) {
+ _data = other_._data;
+ return *this;
+ }
+
+ explicit UnicodeString( UnicodeString const& ) = default;
+ UnicodeString& operator = ( UnicodeString const& ) = default;
+ UnicodeString( UnicodeString&& ) = default;
+ UnicodeString& operator = ( UnicodeString&& ) = default;
+ bool operator == ( UnicodeString const& other_ ) const {
+ return ( _data == other_._data );
+ }
+
+ bool operator != ( UnicodeString const& other_ ) const {
+ return ( _data != other_._data );
+ }
+
+ UnicodeString& append( UnicodeString const& other ) {
+ _data.insert( _data.end(), other._data.begin(), other._data.end() );
+ return *this;
+ }
+
+ void push_back( char32_t c_ ) {
+ _data.push_back( c_ );
+ }
+
+ UnicodeString& append( char32_t const* src, int len ) {
+ _data.insert( _data.end(), src, src + len );
+ return *this;
+ }
+
+ UnicodeString& insert( int pos_, UnicodeString const& str_, int offset_, int len_ ) {
+ _data.insert( _data.begin() + pos_, str_._data.begin() + offset_, str_._data.begin() + offset_ + len_ );
+ return *this;
+ }
+
+ UnicodeString& insert( int pos_, char32_t c_ ) {
+ _data.insert( _data.begin() + pos_, c_ );
+ return *this;
+ }
+
+ UnicodeString& erase( int pos_ ) {
+ _data.erase( _data.begin() + pos_ );
+ return *this;
+ }
+
+ UnicodeString& erase( int pos_, int len_ ) {
+ _data.erase( _data.begin() + pos_, _data.begin() + pos_ + len_ );
+ return *this;
+ }
+
+ char32_t const* get() const {
+ return _data.data();
+ }
+
+ char32_t* get() {
+ return _data.data();
+ }
+
+ int length() const {
+ return static_cast<int>( _data.size() );
+ }
+
+ void clear( void ) {
+ _data.clear();
+ }
+
+ const char32_t& operator[]( size_t pos ) const {
+ return _data[pos];
+ }
+
+ char32_t& operator[]( size_t pos ) {
+ return _data[pos];
+ }
+
+ bool starts_with( data_buffer_t::const_iterator first_, data_buffer_t::const_iterator last_ ) const {
+ return (
+ ( std::distance( first_, last_ ) <= length() )
+ && ( std::equal( first_, last_, _data.begin() ) )
+ );
+ }
+
+ bool ends_with( data_buffer_t::const_iterator first_, data_buffer_t::const_iterator last_ ) const {
+ int len( static_cast<int>( std::distance( first_, last_ ) ) );
+ return (
+ ( len <= length() )
+ && ( std::equal( first_, last_, _data.end() - len ) )
+ );
+ }
+
+ bool is_empty( void ) const {
+ return ( _data.size() == 0 );
+ }
+
+ void swap( UnicodeString& other_ ) {
+ _data.swap( other_._data );
+ }
+
+ const_iterator begin( void ) const {
+ return ( _data.begin() );
+ }
+
+ const_iterator end( void ) const {
+ return ( _data.end() );
+ }
+
+ iterator begin( void ) {
+ return ( _data.begin() );
+ }
+
+ iterator end( void ) {
+ return ( _data.end() );
+ }
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/utf8string.hxx b/contrib/replxx/src/utf8string.hxx
new file mode 100644
index 0000000..29effa2
--- /dev/null
+++ b/contrib/replxx/src/utf8string.hxx
@@ -0,0 +1,94 @@
+#ifndef REPLXX_UTF8STRING_HXX_INCLUDED
+#define REPLXX_UTF8STRING_HXX_INCLUDED
+
+#include <memory>
+
+#include "unicodestring.hxx"
+
+namespace replxx {
+
+class Utf8String {
+private:
+ typedef std::unique_ptr<char[]> buffer_t;
+ buffer_t _data;
+ int _bufSize;
+ int _len;
+public:
+ Utf8String( void )
+ : _data()
+ , _bufSize( 0 )
+ , _len( 0 ) {
+ }
+ explicit Utf8String( UnicodeString const& src )
+ : _data()
+ , _bufSize( 0 )
+ , _len( 0 ) {
+ assign( src, src.length() );
+ }
+
+ Utf8String( UnicodeString const& src_, int len_ )
+ : _data()
+ , _bufSize( 0 )
+ , _len( 0 ) {
+ assign( src_, len_ );
+ }
+
+ void assign( UnicodeString const& str_ ) {
+ assign( str_, str_.length() );
+ }
+
+ void assign( UnicodeString const& str_, int len_ ) {
+ assign( str_.get(), len_ );
+ }
+
+ void assign( char32_t const* str_, int len_ ) {
+ int len( len_ * 4 );
+ realloc( len );
+ _len = copyString32to8( _data.get(), len, str_, len_ );
+ }
+
+ void assign( std::string const& str_ ) {
+ realloc( static_cast<int>( str_.length() ) );
+ strncpy( _data.get(), str_.c_str(), str_.length() );
+ _len = static_cast<int>( str_.length() );
+ }
+
+ void assign( Utf8String const& other_ ) {
+ realloc( other_._len );
+ strncpy( _data.get(), other_._data.get(), other_._len );
+ _len = other_._len;
+ }
+
+ char const* get() const {
+ return _data.get();
+ }
+
+ int size( void ) const {
+ return ( _len );
+ }
+
+ bool operator != ( Utf8String const& other_ ) {
+ return ( ( other_._len != _len ) || ( memcmp( other_._data.get(), _data.get(), _len ) != 0 ) );
+ }
+
+private:
+ void realloc( int reqLen ) {
+ if ( ( reqLen + 1 ) > _bufSize ) {
+ _bufSize = 1;
+ while ( ( reqLen + 1 ) > _bufSize ) {
+ _bufSize *= 2;
+ }
+ _data.reset( new char[_bufSize] );
+ memset( _data.get(), 0, _bufSize );
+ }
+ _data[reqLen] = 0;
+ return;
+ }
+ Utf8String(const Utf8String&) = delete;
+ Utf8String& operator=(const Utf8String&) = delete;
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/util.cxx b/contrib/replxx/src/util.cxx
new file mode 100644
index 0000000..719d707
--- /dev/null
+++ b/contrib/replxx/src/util.cxx
@@ -0,0 +1,158 @@
+#include <chrono>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <wctype.h>
+
+#include "util.hxx"
+
+namespace replxx {
+
+int mk_wcwidth( char32_t );
+
+/**
+ * Calculate a new screen position given a starting position, screen width and
+ * character count
+ * @param x - initial x position (zero-based)
+ * @param y - initial y position (zero-based)
+ * @param screenColumns - screen column count
+ * @param charCount - character positions to advance
+ * @param xOut - returned x position (zero-based)
+ * @param yOut - returned y position (zero-based)
+ */
+void calculate_screen_position(
+ int x, int y, int screenColumns,
+ int charCount, int& xOut, int& yOut
+) {
+ xOut = x;
+ yOut = y;
+ int charsRemaining = charCount;
+ while ( charsRemaining > 0 ) {
+ int charsThisRow = ( ( x + charsRemaining ) < screenColumns )
+ ? charsRemaining
+ : screenColumns - x;
+ xOut = x + charsThisRow;
+ yOut = y;
+ charsRemaining -= charsThisRow;
+ x = 0;
+ ++ y;
+ }
+ if ( xOut == screenColumns ) { // we have to special-case line wrap
+ xOut = 0;
+ ++ yOut;
+ }
+}
+
+/**
+ * Calculate a column width using mk_wcswidth()
+ * @param buf32 - text to calculate
+ * @param len - length of text to calculate
+ */
+int calculate_displayed_length( char32_t const* buf32_, int size_ ) {
+ int len( 0 );
+ for ( int i( 0 ); i < size_; ++ i ) {
+ char32_t c( buf32_[i] );
+ if ( c == '\033' ) {
+ int escStart( i );
+ ++ i;
+ if ( ( i < size_ ) && ( buf32_[i] != '[' ) ) {
+ i = escStart;
+ ++ len;
+ continue;
+ }
+ ++ i;
+ for ( ; i < size_; ++ i ) {
+ c = buf32_[i];
+ if ( ( c != ';' ) && ( ( c < '0' ) || ( c > '9' ) ) ) {
+ break;
+ }
+ }
+ if ( ( i < size_ ) && ( buf32_[i] == 'm' ) ) {
+ continue;
+ }
+ i = escStart;
+ len += 2;
+ } else if ( is_control_code( c ) ) {
+ len += 2;
+ } else {
+ int wcw( mk_wcwidth( c ) );
+ if ( wcw < 0 ) {
+ len = -1;
+ break;
+ }
+ len += wcw;
+ }
+ }
+ return ( len );
+}
+
+char const* ansi_color( Replxx::Color color_ ) {
+ static char const reset[] = "\033[0m";
+ static char const black[] = "\033[0;22;30m";
+ static char const red[] = "\033[0;22;31m";
+ static char const green[] = "\033[0;22;32m";
+ static char const brown[] = "\033[0;22;33m";
+ static char const blue[] = "\033[0;22;34m";
+ static char const magenta[] = "\033[0;22;35m";
+ static char const cyan[] = "\033[0;22;36m";
+ static char const lightgray[] = "\033[0;22;37m";
+
+#ifdef _WIN32
+ static bool const has256colorDefault( true );
+#else
+ static bool const has256colorDefault( false );
+#endif
+ static char const* TERM( getenv( "TERM" ) );
+ static bool const has256color( TERM ? ( strstr( TERM, "256" ) != nullptr ) : has256colorDefault );
+ static char const* gray = has256color ? "\033[0;1;90m" : "\033[0;1;30m";
+ static char const* brightred = has256color ? "\033[0;1;91m" : "\033[0;1;31m";
+ static char const* brightgreen = has256color ? "\033[0;1;92m" : "\033[0;1;32m";
+ static char const* yellow = has256color ? "\033[0;1;93m" : "\033[0;1;33m";
+ static char const* brightblue = has256color ? "\033[0;1;94m" : "\033[0;1;34m";
+ static char const* brightmagenta = has256color ? "\033[0;1;95m" : "\033[0;1;35m";
+ static char const* brightcyan = has256color ? "\033[0;1;96m" : "\033[0;1;36m";
+ static char const* white = has256color ? "\033[0;1;97m" : "\033[0;1;37m";
+ static char const error[] = "\033[101;1;33m";
+
+ char const* code( reset );
+ switch ( color_ ) {
+ case Replxx::Color::BLACK: code = black; break;
+ case Replxx::Color::RED: code = red; break;
+ case Replxx::Color::GREEN: code = green; break;
+ case Replxx::Color::BROWN: code = brown; break;
+ case Replxx::Color::BLUE: code = blue; break;
+ case Replxx::Color::MAGENTA: code = magenta; break;
+ case Replxx::Color::CYAN: code = cyan; break;
+ case Replxx::Color::LIGHTGRAY: code = lightgray; break;
+ case Replxx::Color::GRAY: code = gray; break;
+ case Replxx::Color::BRIGHTRED: code = brightred; break;
+ case Replxx::Color::BRIGHTGREEN: code = brightgreen; break;
+ case Replxx::Color::YELLOW: code = yellow; break;
+ case Replxx::Color::BRIGHTBLUE: code = brightblue; break;
+ case Replxx::Color::BRIGHTMAGENTA: code = brightmagenta; break;
+ case Replxx::Color::BRIGHTCYAN: code = brightcyan; break;
+ case Replxx::Color::WHITE: code = white; break;
+ case Replxx::Color::ERROR: code = error; break;
+ case Replxx::Color::DEFAULT: code = reset; break;
+ }
+ return ( code );
+}
+
+std::string now_ms_str( void ) {
+ std::chrono::milliseconds ms( std::chrono::duration_cast<std::chrono::milliseconds>( std::chrono::system_clock::now().time_since_epoch() ) );
+ time_t t( ms.count() / 1000 );
+ tm broken;
+#ifdef _WIN32
+#define localtime_r( t, b ) localtime_s( ( b ), ( t ) )
+#endif
+ localtime_r( &t, &broken );
+#undef localtime_r
+ static int const BUFF_SIZE( 32 );
+ char str[BUFF_SIZE];
+ strftime( str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken );
+ snprintf( str + sizeof ( "YYYY-mm-dd HH:MM:SS" ), 5, "%03d", static_cast<int>( ms.count() % 1000 ) );
+ return ( str );
+}
+
+}
+
diff --git a/contrib/replxx/src/util.hxx b/contrib/replxx/src/util.hxx
new file mode 100644
index 0000000..17c1086
--- /dev/null
+++ b/contrib/replxx/src/util.hxx
@@ -0,0 +1,25 @@
+#ifndef REPLXX_UTIL_HXX_INCLUDED
+#define REPLXX_UTIL_HXX_INCLUDED 1
+
+#include "replxx.hxx"
+
+namespace replxx {
+
+inline bool is_control_code(char32_t testChar) {
+ return (testChar < ' ') || // C0 controls
+ (testChar >= 0x7F && testChar <= 0x9F); // DEL and C1 controls
+}
+
+inline char32_t control_to_human( char32_t key ) {
+ return ( key < 27 ? ( key + 0x40 ) : ( key + 0x18 ) );
+}
+
+void calculate_screen_position( int x, int y, int screenColumns, int charCount, int& xOut, int& yOut );
+int calculate_displayed_length( char32_t const* buf32, int size );
+char const* ansi_color( Replxx::Color );
+std::string now_ms_str( void );
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/wcwidth.cpp b/contrib/replxx/src/wcwidth.cpp
new file mode 100644
index 0000000..c6c05fa
--- /dev/null
+++ b/contrib/replxx/src/wcwidth.cpp
@@ -0,0 +1,296 @@
+/*
+ * This is an implementation of wcwidth() and wcswidth() (defined in
+ * IEEE Std 1002.1-2001) for Unicode.
+ *
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
+ *
+ * In fixed-width output devices, Latin characters all occupy a single
+ * "cell" position of equal width, whereas ideographic CJK characters
+ * occupy two such cells. Interoperability between terminal-line
+ * applications and (teletype-style) character terminals using the
+ * UTF-8 encoding requires agreement on which character should advance
+ * the cursor by how many cell positions. No established formal
+ * standards exist at present on which Unicode character shall occupy
+ * how many cell positions on character terminals. These routines are
+ * a first attempt of defining such behavior based on simple rules
+ * applied to data provided by the Unicode Consortium.
+ *
+ * For some graphical characters, the Unicode standard explicitly
+ * defines a character-cell width via the definition of the East Asian
+ * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
+ * In all these cases, there is no ambiguity about which width a
+ * terminal shall use. For characters in the East Asian Ambiguous (A)
+ * class, the width choice depends purely on a preference of backward
+ * compatibility with either historic CJK or Western practice.
+ * Choosing single-width for these characters is easy to justify as
+ * the appropriate long-term solution, as the CJK practice of
+ * displaying these characters as double-width comes from historic
+ * implementation simplicity (8-bit encoded characters were displayed
+ * single-width and 16-bit ones double-width, even for Greek,
+ * Cyrillic, etc.) and not any typographic considerations.
+ *
+ * Much less clear is the choice of width for the Not East Asian
+ * (Neutral) class. Existing practice does not dictate a width for any
+ * of these characters. It would nevertheless make sense
+ * typographically to allocate two character cells to characters such
+ * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
+ * represented adequately with a single-width glyph. The following
+ * routines at present merely assign a single-cell width to all
+ * neutral characters, in the interest of simplicity. This is not
+ * entirely satisfactory and should be reconsidered before
+ * establishing a formal standard in this area. At the moment, the
+ * decision which Not East Asian (Neutral) characters should be
+ * represented by double-width glyphs cannot yet be answered by
+ * applying a simple rule from the Unicode database content. Setting
+ * up a proper standard for the behavior of UTF-8 character terminals
+ * will require a careful analysis not only of each Unicode character,
+ * but also of each presentation form, something the author of these
+ * routines has avoided to do so far.
+ *
+ * http://www.unicode.org/unicode/reports/tr11/
+ *
+ * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+ *
+ * Permission to use, copy, modify, and distribute this software
+ * for any purpose and without fee is hereby granted. The author
+ * disclaims all warranties with regard to this software.
+ *
+ * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ */
+
+#include <wchar.h>
+#include <string>
+#include <memory>
+
+namespace replxx {
+
+struct interval {
+ char32_t first;
+ char32_t last;
+};
+
+/* auxiliary function for binary search in interval table */
+static int bisearch(char32_t ucs, const struct interval *table, int max) {
+ int min = 0;
+ int mid;
+
+ if (ucs < table[0].first || ucs > table[max].last)
+ return 0;
+ while (max >= min) {
+ mid = (min + max) / 2;
+ if (ucs > table[mid].last)
+ min = mid + 1;
+ else if (ucs < table[mid].first)
+ max = mid - 1;
+ else
+ return 1;
+ }
+
+ return 0;
+}
+
+
+/* The following two functions define the column width of an ISO 10646
+ * character as follows:
+ *
+ * - The null character (U+0000) has a column width of 0.
+ *
+ * - Other C0/C1 control characters and DEL will lead to a return
+ * value of -1.
+ *
+ * - Non-spacing and enclosing combining characters (general
+ * category code Mn or Me in the Unicode database) have a
+ * column width of 0.
+ *
+ * - SOFT HYPHEN (U+00AD) has a column width of 1.
+ *
+ * - Other format characters (general category code Cf in the Unicode
+ * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
+ *
+ * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
+ * have a column width of 0.
+ *
+ * - Spacing characters in the East Asian Wide (W) or East Asian
+ * Full-width (F) category as defined in Unicode Technical
+ * Report #11 have a column width of 2.
+ *
+ * - All remaining characters (including all printable
+ * ISO 8859-1 and WGL4 characters, Unicode control characters,
+ * etc.) have a column width of 1.
+ *
+ * This implementation assumes that wchar_t characters are encoded
+ * in ISO 10646.
+ */
+
+int mk_is_wide_char(char32_t ucs) {
+ static const struct interval wide[] = {
+ {0x1100, 0x115f}, {0x231a, 0x231b}, {0x2329, 0x232a},
+ {0x23e9, 0x23ec}, {0x23f0, 0x23f0}, {0x23f3, 0x23f3},
+ {0x25fd, 0x25fe}, {0x2614, 0x2615}, {0x2648, 0x2653},
+ {0x267f, 0x267f}, {0x2693, 0x2693}, {0x26a1, 0x26a1},
+ {0x26aa, 0x26ab}, {0x26bd, 0x26be}, {0x26c4, 0x26c5},
+ {0x26ce, 0x26ce}, {0x26d4, 0x26d4}, {0x26ea, 0x26ea},
+ {0x26f2, 0x26f3}, {0x26f5, 0x26f5}, {0x26fa, 0x26fa},
+ {0x26fd, 0x26fd}, {0x2705, 0x2705}, {0x270a, 0x270b},
+ {0x2728, 0x2728}, {0x274c, 0x274c}, {0x274e, 0x274e},
+ {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797},
+ {0x27b0, 0x27b0}, {0x27bf, 0x27bf}, {0x2b1b, 0x2b1c},
+ {0x2b50, 0x2b50}, {0x2b55, 0x2b55}, {0x2e80, 0x2fdf},
+ {0x2ff0, 0x303e}, {0x3040, 0x3247}, {0x3250, 0x4dbf},
+ {0x4e00, 0xa4cf}, {0xa960, 0xa97f}, {0xac00, 0xd7a3},
+ {0xf900, 0xfaff}, {0xfe10, 0xfe19}, {0xfe30, 0xfe6f},
+ {0xff01, 0xff60}, {0xffe0, 0xffe6}, {0x16fe0, 0x16fe1},
+ {0x17000, 0x18aff}, {0x1b000, 0x1b12f}, {0x1b170, 0x1b2ff},
+ {0x1f004, 0x1f004}, {0x1f0cf, 0x1f0cf}, {0x1f18e, 0x1f18e},
+ {0x1f191, 0x1f19a}, {0x1f200, 0x1f202}, {0x1f210, 0x1f23b},
+ {0x1f240, 0x1f248}, {0x1f250, 0x1f251}, {0x1f260, 0x1f265},
+ {0x1f300, 0x1f320}, {0x1f32d, 0x1f335}, {0x1f337, 0x1f37c},
+ {0x1f37e, 0x1f393}, {0x1f3a0, 0x1f3ca}, {0x1f3cf, 0x1f3d3},
+ {0x1f3e0, 0x1f3f0}, {0x1f3f4, 0x1f3f4}, {0x1f3f8, 0x1f43e},
+ {0x1f440, 0x1f440}, {0x1f442, 0x1f4fc}, {0x1f4ff, 0x1f53d},
+ {0x1f54b, 0x1f54e}, {0x1f550, 0x1f567}, {0x1f57a, 0x1f57a},
+ {0x1f595, 0x1f596}, {0x1f5a4, 0x1f5a4}, {0x1f5fb, 0x1f64f},
+ {0x1f680, 0x1f6c5}, {0x1f6cc, 0x1f6cc}, {0x1f6d0, 0x1f6d2},
+ {0x1f6eb, 0x1f6ec}, {0x1f6f4, 0x1f6f8}, {0x1f910, 0x1f93e},
+ {0x1f940, 0x1f94c}, {0x1f950, 0x1f96b}, {0x1f980, 0x1f997},
+ {0x1f9c0, 0x1f9c0}, {0x1f9d0, 0x1f9e6}, {0x20000, 0x2fffd},
+ {0x30000, 0x3fffd},
+ };
+
+ if ( bisearch(ucs, wide, sizeof(wide) / sizeof(struct interval) - 1) ) {
+ return 1;
+ }
+
+ return 0;
+}
+
+int mk_wcwidth(char32_t ucs) {
+ /* sorted list of non-overlapping intervals of non-spacing characters */
+ /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
+ static const struct interval combining[] = {
+ {0x00ad, 0x00ad}, {0x0300, 0x036f}, {0x0483, 0x0489},
+ {0x0591, 0x05bd}, {0x05bf, 0x05bf}, {0x05c1, 0x05c2},
+ {0x05c4, 0x05c5}, {0x05c7, 0x05c7}, {0x0610, 0x061a},
+ {0x061c, 0x061c}, {0x064b, 0x065f}, {0x0670, 0x0670},
+ {0x06d6, 0x06dc}, {0x06df, 0x06e4}, {0x06e7, 0x06e8},
+ {0x06ea, 0x06ed}, {0x0711, 0x0711}, {0x0730, 0x074a},
+ {0x07a6, 0x07b0}, {0x07eb, 0x07f3}, {0x0816, 0x0819},
+ {0x081b, 0x0823}, {0x0825, 0x0827}, {0x0829, 0x082d},
+ {0x0859, 0x085b}, {0x08d4, 0x08e1}, {0x08e3, 0x0902},
+ {0x093a, 0x093a}, {0x093c, 0x093c}, {0x0941, 0x0948},
+ {0x094d, 0x094d}, {0x0951, 0x0957}, {0x0962, 0x0963},
+ {0x0981, 0x0981}, {0x09bc, 0x09bc}, {0x09c1, 0x09c4},
+ {0x09cd, 0x09cd}, {0x09e2, 0x09e3}, {0x0a01, 0x0a02},
+ {0x0a3c, 0x0a3c}, {0x0a41, 0x0a42}, {0x0a47, 0x0a48},
+ {0x0a4b, 0x0a4d}, {0x0a51, 0x0a51}, {0x0a70, 0x0a71},
+ {0x0a75, 0x0a75}, {0x0a81, 0x0a82}, {0x0abc, 0x0abc},
+ {0x0ac1, 0x0ac5}, {0x0ac7, 0x0ac8}, {0x0acd, 0x0acd},
+ {0x0ae2, 0x0ae3}, {0x0afa, 0x0aff}, {0x0b01, 0x0b01},
+ {0x0b3c, 0x0b3c}, {0x0b3f, 0x0b3f}, {0x0b41, 0x0b44},
+ {0x0b4d, 0x0b4d}, {0x0b56, 0x0b56}, {0x0b62, 0x0b63},
+ {0x0b82, 0x0b82}, {0x0bc0, 0x0bc0}, {0x0bcd, 0x0bcd},
+ {0x0c00, 0x0c00}, {0x0c3e, 0x0c40}, {0x0c46, 0x0c48},
+ {0x0c4a, 0x0c4d}, {0x0c55, 0x0c56}, {0x0c62, 0x0c63},
+ {0x0c81, 0x0c81}, {0x0cbc, 0x0cbc}, {0x0cbf, 0x0cbf},
+ {0x0cc6, 0x0cc6}, {0x0ccc, 0x0ccd}, {0x0ce2, 0x0ce3},
+ {0x0d00, 0x0d01}, {0x0d3b, 0x0d3c}, {0x0d41, 0x0d44},
+ {0x0d4d, 0x0d4d}, {0x0d62, 0x0d63}, {0x0dca, 0x0dca},
+ {0x0dd2, 0x0dd4}, {0x0dd6, 0x0dd6}, {0x0e31, 0x0e31},
+ {0x0e34, 0x0e3a}, {0x0e47, 0x0e4e}, {0x0eb1, 0x0eb1},
+ {0x0eb4, 0x0eb9}, {0x0ebb, 0x0ebc}, {0x0ec8, 0x0ecd},
+ {0x0f18, 0x0f19}, {0x0f35, 0x0f35}, {0x0f37, 0x0f37},
+ {0x0f39, 0x0f39}, {0x0f71, 0x0f7e}, {0x0f80, 0x0f84},
+ {0x0f86, 0x0f87}, {0x0f8d, 0x0f97}, {0x0f99, 0x0fbc},
+ {0x0fc6, 0x0fc6}, {0x102d, 0x1030}, {0x1032, 0x1037},
+ {0x1039, 0x103a}, {0x103d, 0x103e}, {0x1058, 0x1059},
+ {0x105e, 0x1060}, {0x1071, 0x1074}, {0x1082, 0x1082},
+ {0x1085, 0x1086}, {0x108d, 0x108d}, {0x109d, 0x109d},
+ {0x1160, 0x11ff}, {0x135d, 0x135f}, {0x1712, 0x1714},
+ {0x1732, 0x1734}, {0x1752, 0x1753}, {0x1772, 0x1773},
+ {0x17b4, 0x17b5}, {0x17b7, 0x17bd}, {0x17c6, 0x17c6},
+ {0x17c9, 0x17d3}, {0x17dd, 0x17dd}, {0x180b, 0x180e},
+ {0x1885, 0x1886}, {0x18a9, 0x18a9}, {0x1920, 0x1922},
+ {0x1927, 0x1928}, {0x1932, 0x1932}, {0x1939, 0x193b},
+ {0x1a17, 0x1a18}, {0x1a1b, 0x1a1b}, {0x1a56, 0x1a56},
+ {0x1a58, 0x1a5e}, {0x1a60, 0x1a60}, {0x1a62, 0x1a62},
+ {0x1a65, 0x1a6c}, {0x1a73, 0x1a7c}, {0x1a7f, 0x1a7f},
+ {0x1ab0, 0x1abe}, {0x1b00, 0x1b03}, {0x1b34, 0x1b34},
+ {0x1b36, 0x1b3a}, {0x1b3c, 0x1b3c}, {0x1b42, 0x1b42},
+ {0x1b6b, 0x1b73}, {0x1b80, 0x1b81}, {0x1ba2, 0x1ba5},
+ {0x1ba8, 0x1ba9}, {0x1bab, 0x1bad}, {0x1be6, 0x1be6},
+ {0x1be8, 0x1be9}, {0x1bed, 0x1bed}, {0x1bef, 0x1bf1},
+ {0x1c2c, 0x1c33}, {0x1c36, 0x1c37}, {0x1cd0, 0x1cd2},
+ {0x1cd4, 0x1ce0}, {0x1ce2, 0x1ce8}, {0x1ced, 0x1ced},
+ {0x1cf4, 0x1cf4}, {0x1cf8, 0x1cf9}, {0x1dc0, 0x1df9},
+ {0x1dfb, 0x1dff}, {0x200b, 0x200f}, {0x202a, 0x202e},
+ {0x2060, 0x2064}, {0x2066, 0x206f}, {0x20d0, 0x20f0},
+ {0x2cef, 0x2cf1}, {0x2d7f, 0x2d7f}, {0x2de0, 0x2dff},
+ {0x302a, 0x302d}, {0x3099, 0x309a}, {0xa66f, 0xa672},
+ {0xa674, 0xa67d}, {0xa69e, 0xa69f}, {0xa6f0, 0xa6f1},
+ {0xa802, 0xa802}, {0xa806, 0xa806}, {0xa80b, 0xa80b},
+ {0xa825, 0xa826}, {0xa8c4, 0xa8c5}, {0xa8e0, 0xa8f1},
+ {0xa926, 0xa92d}, {0xa947, 0xa951}, {0xa980, 0xa982},
+ {0xa9b3, 0xa9b3}, {0xa9b6, 0xa9b9}, {0xa9bc, 0xa9bc},
+ {0xa9e5, 0xa9e5}, {0xaa29, 0xaa2e}, {0xaa31, 0xaa32},
+ {0xaa35, 0xaa36}, {0xaa43, 0xaa43}, {0xaa4c, 0xaa4c},
+ {0xaa7c, 0xaa7c}, {0xaab0, 0xaab0}, {0xaab2, 0xaab4},
+ {0xaab7, 0xaab8}, {0xaabe, 0xaabf}, {0xaac1, 0xaac1},
+ {0xaaec, 0xaaed}, {0xaaf6, 0xaaf6}, {0xabe5, 0xabe5},
+ {0xabe8, 0xabe8}, {0xabed, 0xabed}, {0xfb1e, 0xfb1e},
+ {0xfe00, 0xfe0f}, {0xfe20, 0xfe2f}, {0xfeff, 0xfeff},
+ {0xfff9, 0xfffb}, {0x101fd, 0x101fd}, {0x102e0, 0x102e0},
+ {0x10376, 0x1037a}, {0x10a01, 0x10a03}, {0x10a05, 0x10a06},
+ {0x10a0c, 0x10a0f}, {0x10a38, 0x10a3a}, {0x10a3f, 0x10a3f},
+ {0x10ae5, 0x10ae6}, {0x11001, 0x11001}, {0x11038, 0x11046},
+ {0x1107f, 0x11081}, {0x110b3, 0x110b6}, {0x110b9, 0x110ba},
+ {0x11100, 0x11102}, {0x11127, 0x1112b}, {0x1112d, 0x11134},
+ {0x11173, 0x11173}, {0x11180, 0x11181}, {0x111b6, 0x111be},
+ {0x111ca, 0x111cc}, {0x1122f, 0x11231}, {0x11234, 0x11234},
+ {0x11236, 0x11237}, {0x1123e, 0x1123e}, {0x112df, 0x112df},
+ {0x112e3, 0x112ea}, {0x11300, 0x11301}, {0x1133c, 0x1133c},
+ {0x11340, 0x11340}, {0x11366, 0x1136c}, {0x11370, 0x11374},
+ {0x11438, 0x1143f}, {0x11442, 0x11444}, {0x11446, 0x11446},
+ {0x114b3, 0x114b8}, {0x114ba, 0x114ba}, {0x114bf, 0x114c0},
+ {0x114c2, 0x114c3}, {0x115b2, 0x115b5}, {0x115bc, 0x115bd},
+ {0x115bf, 0x115c0}, {0x115dc, 0x115dd}, {0x11633, 0x1163a},
+ {0x1163d, 0x1163d}, {0x1163f, 0x11640}, {0x116ab, 0x116ab},
+ {0x116ad, 0x116ad}, {0x116b0, 0x116b5}, {0x116b7, 0x116b7},
+ {0x1171d, 0x1171f}, {0x11722, 0x11725}, {0x11727, 0x1172b},
+ {0x11a01, 0x11a06}, {0x11a09, 0x11a0a}, {0x11a33, 0x11a38},
+ {0x11a3b, 0x11a3e}, {0x11a47, 0x11a47}, {0x11a51, 0x11a56},
+ {0x11a59, 0x11a5b}, {0x11a8a, 0x11a96}, {0x11a98, 0x11a99},
+ {0x11c30, 0x11c36}, {0x11c38, 0x11c3d}, {0x11c3f, 0x11c3f},
+ {0x11c92, 0x11ca7}, {0x11caa, 0x11cb0}, {0x11cb2, 0x11cb3},
+ {0x11cb5, 0x11cb6}, {0x11d31, 0x11d36}, {0x11d3a, 0x11d3a},
+ {0x11d3c, 0x11d3d}, {0x11d3f, 0x11d45}, {0x11d47, 0x11d47},
+ {0x16af0, 0x16af4}, {0x16b30, 0x16b36}, {0x16f8f, 0x16f92},
+ {0x1bc9d, 0x1bc9e}, {0x1bca0, 0x1bca3}, {0x1d167, 0x1d169},
+ {0x1d173, 0x1d182}, {0x1d185, 0x1d18b}, {0x1d1aa, 0x1d1ad},
+ {0x1d242, 0x1d244}, {0x1da00, 0x1da36}, {0x1da3b, 0x1da6c},
+ {0x1da75, 0x1da75}, {0x1da84, 0x1da84}, {0x1da9b, 0x1da9f},
+ {0x1daa1, 0x1daaf}, {0x1e000, 0x1e006}, {0x1e008, 0x1e018},
+ {0x1e01b, 0x1e021}, {0x1e023, 0x1e024}, {0x1e026, 0x1e02a},
+ {0x1e8d0, 0x1e8d6}, {0x1e944, 0x1e94a}, {0xe0001, 0xe0001},
+ {0xe0020, 0xe007f}, {0xe0100, 0xe01ef},
+ };
+
+ /* test for 8-bit control characters */
+ if ( ucs == 0 ) {
+ return 0;
+ }
+ if ( ( ucs < 32 ) || ( ( ucs >= 0x7f ) && ( ucs < 0xa0 ) ) ) {
+ return -1;
+ }
+
+ /* binary search in table of non-spacing characters */
+ if ( bisearch( ucs, combining, sizeof( combining ) / sizeof( struct interval ) - 1 ) ) {
+ return 0;
+ }
+
+ /* if we arrive here, ucs is not a combining or C0/C1 control character */
+ return ( mk_is_wide_char( ucs ) ? 2 : 1 );
+}
+
+}
+
diff --git a/contrib/replxx/src/windows.cxx b/contrib/replxx/src/windows.cxx
new file mode 100644
index 0000000..715292c
--- /dev/null
+++ b/contrib/replxx/src/windows.cxx
@@ -0,0 +1,144 @@
+#ifdef _WIN32
+
+#include <iostream>
+
+#include "windows.hxx"
+#include "conversion.hxx"
+#include "terminal.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+WinAttributes WIN_ATTR;
+
+template<typename T>
+T* HandleEsc(HANDLE out_, T* p, T* end) {
+ if (*p == '[') {
+ int code = 0;
+
+ int thisBackground( WIN_ATTR._defaultBackground );
+ for (++p; p < end; ++p) {
+ char32_t c = *p;
+
+ if ('0' <= c && c <= '9') {
+ code = code * 10 + (c - '0');
+ } else if (c == 'm' || c == ';') {
+ switch (code) {
+ case 0:
+ WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute;
+ WIN_ATTR._consoleColor = WIN_ATTR._defaultColor | thisBackground;
+ break;
+ case 1: // BOLD
+ case 5: // BLINK
+ WIN_ATTR._consoleAttribute = (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY;
+ break;
+ case 22:
+ WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute;
+ break;
+ case 30:
+ case 90:
+ WIN_ATTR._consoleColor = thisBackground;
+ break;
+ case 31:
+ case 91:
+ WIN_ATTR._consoleColor = FOREGROUND_RED | thisBackground;
+ break;
+ case 32:
+ case 92:
+ WIN_ATTR._consoleColor = FOREGROUND_GREEN | thisBackground;
+ break;
+ case 33:
+ case 93:
+ WIN_ATTR._consoleColor = FOREGROUND_RED | FOREGROUND_GREEN | thisBackground;
+ break;
+ case 34:
+ case 94:
+ WIN_ATTR._consoleColor = FOREGROUND_BLUE | thisBackground;
+ break;
+ case 35:
+ case 95:
+ WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_RED | thisBackground;
+ break;
+ case 36:
+ case 96:
+ WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN | thisBackground;
+ break;
+ case 37:
+ case 97:
+ WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE | thisBackground;
+ break;
+ case 101:
+ thisBackground = BACKGROUND_RED;
+ break;
+ }
+
+ if ( ( code >= 90 ) && ( code <= 97 ) ) {
+ WIN_ATTR._consoleAttribute = (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY;
+ }
+
+ code = 0;
+ }
+
+ if (*p == 'm') {
+ ++p;
+ break;
+ }
+ }
+ } else {
+ ++p;
+ }
+
+ SetConsoleTextAttribute(
+ out_,
+ WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor
+ );
+
+ return p;
+}
+
+int win_write( HANDLE out_, bool autoEscape_, char const* str_, int size_ ) {
+ int count( 0 );
+ if ( tty::out ) {
+ DWORD nWritten( 0 );
+ if ( autoEscape_ ) {
+ WriteConsoleA( out_, str_, size_, &nWritten, nullptr );
+ count = nWritten;
+ } else {
+ char const* s( str_ );
+ char const* e( str_ + size_ );
+ while ( str_ < e ) {
+ if ( *str_ == 27 ) {
+ if ( s < str_ ) {
+ int toWrite( static_cast<int>( str_ - s ) );
+ WriteConsoleA( out_, s, static_cast<DWORD>( toWrite ), &nWritten, nullptr );
+ count += nWritten;
+ if ( nWritten != toWrite ) {
+ s = str_ = nullptr;
+ break;
+ }
+ }
+ s = HandleEsc( out_, str_ + 1, e );
+ int escaped( static_cast<int>( s - str_ ) );
+ count += escaped;
+ str_ = s;
+ } else {
+ ++ str_;
+ }
+ }
+
+ if ( s < str_ ) {
+ WriteConsoleA( out_, s, static_cast<DWORD>( str_ - s ), &nWritten, nullptr );
+ count += nWritten;
+ }
+ }
+ } else {
+ count = _write( 1, str_, size_ );
+ }
+ return ( count );
+}
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/windows.hxx b/contrib/replxx/src/windows.hxx
new file mode 100644
index 0000000..243f41c
--- /dev/null
+++ b/contrib/replxx/src/windows.hxx
@@ -0,0 +1,44 @@
+#ifndef REPLXX_WINDOWS_HXX_INCLUDED
+#define REPLXX_WINDOWS_HXX_INCLUDED 1
+
+#include <conio.h>
+#include <windows.h>
+#include <io.h>
+
+namespace replxx {
+
+static const int FOREGROUND_WHITE =
+ FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;
+static const int BACKGROUND_WHITE =
+ BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE;
+static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY;
+
+class WinAttributes {
+ public:
+ WinAttributes() {
+ CONSOLE_SCREEN_BUFFER_INFO info;
+ GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info);
+ _defaultAttribute = info.wAttributes & INTENSITY;
+ _defaultColor = info.wAttributes & FOREGROUND_WHITE;
+ _defaultBackground = info.wAttributes & BACKGROUND_WHITE;
+
+ _consoleAttribute = _defaultAttribute;
+ _consoleColor = _defaultColor | _defaultBackground;
+ }
+
+ public:
+ int _defaultAttribute;
+ int _defaultColor;
+ int _defaultBackground;
+
+ int _consoleAttribute;
+ int _consoleColor;
+};
+
+int win_write( HANDLE, bool, char const*, int );
+
+extern WinAttributes WIN_ATTR;
+
+}
+
+#endif
diff --git a/contrib/snowball/.gitignore b/contrib/snowball/.gitignore
new file mode 100644
index 0000000..2147da8
--- /dev/null
+++ b/contrib/snowball/.gitignore
@@ -0,0 +1,5 @@
+*.o
+/libstemmer
+/snowball
+/src_c
+/stemwords
diff --git a/contrib/snowball/.travis.yml b/contrib/snowball/.travis.yml
new file mode 100644
index 0000000..e576233
--- /dev/null
+++ b/contrib/snowball/.travis.yml
@@ -0,0 +1,4 @@
+language: c
+compiler: gcc
+before_script: git clone https://github.com/snowballstem/snowball-data ../data
+script: make check
diff --git a/contrib/snowball/AUTHORS b/contrib/snowball/AUTHORS
new file mode 100644
index 0000000..60eae6f
--- /dev/null
+++ b/contrib/snowball/AUTHORS
@@ -0,0 +1,27 @@
+Authors
+=======
+
+Martin Porter
+-------------
+
+ - Designed the snowball language.
+ - Implemented the snowball to C compiler.
+ - Implemented the stemming algorithms in C.
+ - Wrote the documentation.
+
+Richard Boulton
+---------------
+
+ - Implemented Java backend of the snowball compiler.
+ - Developed build system.
+ - Assisted with website maintenance.
+
+
+Assistance from
+---------------
+
+Olivier Bornet - fixes to java packaging and build system.
+Andreas Jung - useful bug reports on the libstemmer library.
+Olly Betts - several patches, bug reports, and performance improvements.
+Sebastiano Vigna and Oerd Cukalla - patches for the Java stemming algorithms.
+Ralf Junker - fix a potential memory leak in sb_stemmer_new().
diff --git a/contrib/snowball/CMakeLists.txt b/contrib/snowball/CMakeLists.txt
new file mode 100644
index 0000000..7ee961e
--- /dev/null
+++ b/contrib/snowball/CMakeLists.txt
@@ -0,0 +1,70 @@
+# End of configuration
+SET(LIBSTEM_ALGORITHMS arabic danish dutch english finnish french german greek hindi hungarian
+ indonesian italian lithuanian nepali norwegian porter portuguese romanian
+ russian serbian spanish swedish tamil turkish)
+SET(ALL_ALGORITHMS ${LIBSTEM_ALGORITHMS})
+
+SET(COMPILER_SOURCES compiler/space.c
+ compiler/tokeniser.c
+ compiler/analyser.c
+ compiler/generator.c
+ compiler/driver.c)
+
+SET(SNOWBALL_RUNTIME runtime/api.c
+ runtime/utilities.c)
+SET(LIBSTEMMER_SOURCES libstemmer/libstemmer.c)
+SET(LIBSTEMMER_UTF8_SOURCES libstemmer/libstemmer_utf8.c)
+#LIBSTEMMER_UTF8_SOURCES = libstemmer/libstemmer_utf8.c
+#LIBSTEMMER_HEADERS = include/libstemmer.h libstemmer/modules.h libstemmer/modules_utf8.h
+#LIBSTEMMER_EXTRA = libstemmer/modules.txt libstemmer/modules_utf8.txt libstemmer/libstemmer_c.in
+
+SET(MODULES_H "modules.h")
+CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/libstemmer/libstemmer_c.in ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/libstemmer.c @ONLY)
+ADD_DEFINITIONS("-DDISABLE_JS")
+ADD_DEFINITIONS("-DDISABLE_GO")
+ADD_DEFINITIONS("-DDISABLE_JAVA")
+ADD_DEFINITIONS("-DDISABLE_PYTHON")
+ADD_DEFINITIONS("-DDISABLE_CSHARP")
+ADD_DEFINITIONS("-DDISABLE_PASCAL")
+ADD_DEFINITIONS("-DDISABLE_RUST")
+
+MACRO(gen_stem IN ENCODING)
+ FOREACH(_it ${IN})
+ SET(_base "${CMAKE_CURRENT_BINARY_DIR}/libstemmer/stem_${ENCODING}_${_it}")
+ SET(_header "${_base}.h")
+ SET(_source "${_base}.c")
+ STRING(REPLACE "UTF_8" "Unicode" _in_enc "${ENCODING}")
+ SET(_input "${CMAKE_CURRENT_SOURCE_DIR}/algorithms/${_it}.sbl")
+ IF(${_in_enc} STREQUAL "Unicode" AND NOT EXISTS ${_input})
+ ADD_CUSTOM_COMMAND(OUTPUT ${_source}
+ COMMAND env "ASAN_OPTIONS=detect_leaks=0" ${CMAKE_CURRENT_BINARY_DIR}/snowball "${CMAKE_CURRENT_SOURCE_DIR}/algorithms/${_it}/stem_ISO_8859_1.sbl" -o ${_base} -eprefix ${_it}_${ENCODING}_ -r ../runtime -u
+ DEPENDS snowball)
+ LIST(APPEND STEMMER_SOURCES ${_source})
+
+ ELSE()
+ IF(EXISTS "${_input}")
+ ADD_CUSTOM_COMMAND(OUTPUT ${_source}
+ COMMAND env "ASAN_OPTIONS=detect_leaks=0" ${CMAKE_CURRENT_BINARY_DIR}/snowball ${_input} -o ${_base} -eprefix ${_it}_${ENCODING}_ -r ../runtime -u
+ DEPENDS snowball)
+ LIST(APPEND STEMMER_SOURCES ${_source})
+ ENDIF()
+ ENDIF()
+ ENDFOREACH()
+ENDMACRO()
+
+INCLUDE_DIRECTORIES("include")
+
+ADD_EXECUTABLE(snowball ${COMPILER_SOURCES})
+
+ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/modules.h
+ COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/libstemmer/mkmodules.pl ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/modules.h libstemmer ${CMAKE_CURRENT_SOURCE_DIR}/libstemmer/modules.txt ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/mkinc.mak)
+ADD_CUSTOM_TARGET(modules DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/libstemmer/modules.h")
+
+SET(STEMMER_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/libstemmer/libstemmer.c")
+ADD_CUSTOM_TARGET(stemmer_deps ALL)
+ADD_DEPENDENCIES(stemmer_deps modules)
+
+gen_stem("${LIBSTEM_ALGORITHMS}" "UTF_8")
+
+ADD_LIBRARY(stemmer ${LINK_TYPE} ${SNOWBALL_RUNTIME} ${STEMMER_SOURCES})
+ADD_DEPENDENCIES(stemmer stemmer_deps)
diff --git a/contrib/snowball/NEWS b/contrib/snowball/NEWS
new file mode 100644
index 0000000..c71c12d
--- /dev/null
+++ b/contrib/snowball/NEWS
@@ -0,0 +1,407 @@
+Snowball 2.0.0 (2019-10-02)
+===========================
+
+C/C++
+-----
+
+* Fully handle 4-byte UTF-8 sequences. Previously `hop` and `next` handled
+ sequences of any length, but commands which look at the character value only
+ handled sequences up to length 3. Fixes #89.
+
+* Fix handling of a 3-byte UTF-8 sequence in a grouping in `backwardmode`.
+
+Java
+----
+
+* TestApp.java:
+
+ - Always use UTF-8 for I/O. Patch from David Corbett (#80).
+
+ - Allow reading input from stdin.
+
+ - Remove rather pointless "stem n times" feature.
+
+ - Only lower case ASCII to match stemwords.c.
+
+ - Stem empty lines too to match stemwords.c.
+
+Code Quality Improvements
+-------------------------
+
+* Fix various warnings from newer compilers.
+
+* Improve use of `const`.
+
+* Share common functions between compiler backends rather than having multiple
+ copies of the same code.
+
+* Assorted code clean-up.
+
+* Initialise line_labelled member of struct generator to 0. Previously we were
+ invoking undefined behaviour, though in practice it'll be zero initialised on
+ most platforms.
+
+New Code Generators
+-------------------
+
+* Add Python generator (#24). Originally written by Yoshiki Shibukawa, with
+ additional updates by Dmitry Shachnev.
+
+* Add Javascript generator. Based on JSX generator (#26) written by Yoshiki
+ Shibukawa.
+
+* Add Rust generator from Jakob Demler (#51).
+
+* Add Go generator from Marty Schoch (#57).
+
+* Add C# generator. Based on patch from Cesar Souza (#16, #17).
+
+* Add Pascal generator. Based on Delphi backend from stemming.zip file on old
+ website (#75).
+
+New Language Features
+---------------------
+
+* Add `len` and `lenof` to measure Unicode length. These are similar to `size`
+ and `sizeof` (respectively), but `size` and `sizeof` return the length in
+ bytes under `-utf8`, whereas these new commands give the same result whether
+ using `-utf8`, `-widechars` or neither (but under `-utf8` they are O(n) in
+ the length of the string). For compatibility with existing code which might
+ use these as variable or function names, they stop being treated as tokens if
+ declared to be a variable or function.
+
+* New `{U+1234}` stringdef notation for Unicode codepoints.
+
+* More versatile integer tests. Now you can compare any two arithmetic
+ expressions with a relational operator in parentheses after the `$`, so for
+ example `$(len > 3)` can now be used when previously a temporary variable was
+ required: `$tmp = len $tmp > 3`
+
+Code generation improvements
+----------------------------
+
+* General:
+
+ + Avoid unnecessarily saving and restoring of the cursor for more commands -
+ `atlimit`, `do`, `set` and `unset` all leave the cursor alone or always
+ restore its value, and for C `booltest` (which other languages already
+ handled).
+
+ + Special case handling for `setlimit tomark AE`. All uses of setlimit in
+ the current stemmers we ship follow this pattern, and by special-casing we
+ can avoid having to save and restore the cursor (#74).
+
+ + Merge duplicate actions in the same `among`. This reduces the size of the
+ switch/if-chain in the generated code which dispatch the among for many of
+ the stemmers.
+
+ + Generate simpler code for `among`. We always check for a zero return value
+ when we call the among, so there's no point also checking for that in the
+ switch/if-chain. We can also avoid the switch/if-chain entirely when
+ there's only one possible outcome (besides the zero return).
+
+ + Optimise code generated for `do <function call>`. This speeds up "make
+ check_python" by about 2%, and should speed up other interpreted languages
+ too (#110).
+
+ + Generate more and better comments referencing snowball source.
+
+ + Add homepage URL and compiler version as comments in generated files.
+
+* C/C++:
+
+ + Fix `size` and `sizeof` to not report one too high (reported by Assem
+ Chelli in #32).
+
+ + If signal `f` from a function call would lead to return from the current
+ function then handle this and bailing out on an error together with a
+ simple `if (ret <= 0) return ret;`
+
+ + Inline testing for a single character literals.
+
+ + Avoiding generating `|| 0` in corner case - this can result in a compiler
+ warning when building the generated code.
+
+ + Implement `insert_v()` in terms of `insert_s()`.
+
+ + Add conditional `extern "C"` so `runtime/api.h` can be included from C++
+ code. Closes #90, reported by vvarma.
+
+* Java:
+
+ + Fix functions in `among` to work in Java. We seem to need to make the
+ methods called from among `public` instead of `private`, and to call them
+ on `this` instead of the `methodObject` (which is cleaner anyway). No
+ revision in version control seems to generate working code for this case,
+ but Richard says it definitely used to work - possibly older JVMs failed to
+ correctly enforce the access controls when methods were invoked by
+ reflection.
+
+ + Code after handling `f` by returning from the current function is
+ unreachable too.
+
+ + Previously we incorrectly decided that code after an `or` was
+ unreachable in certain cases. None of the current stemmers in the
+ distribution triggered this, but Martin Porter's snowball version
+ of the Schinke Latin stemmer does. Fixes #58, reported by Alexander
+ Myltsev.
+
+ + The reachability logic was failing to consider reachability from
+ the final command in an `or`. Fixes #82, reported by David Corbett.
+
+ + Fix `maxint` and `minint`. Patch from David Corbett in #31.
+
+ + Fix `$` on strings. The previous generated code was just wrong. This
+ doesn't affect any of the included algorithms, but for example breaks
+ Martin Porter's snowball implementation of Schinke's Latin Stemmer.
+ Issue noted by Jakob Demler while working on the Rust backend in #51,
+ and reported in the Schinke's Latin Stemmer by Alexander Myltsev
+ in #58.
+
+ + Make SnowballProgram objects serializable. Patch from Oleg Smirnov in #43.
+
+ + Eliminate range-check implementation for groupings. This was removed from
+ the C generator 10 years earlier, isn't used for any of the existing
+ algorithms, and it doesn't seem likely it would be - the grouping would
+ have to consist entirely of a contiguous block of Unicode code-points.
+
+ + Simplify code generated for `repeat` and `atleast`.
+
+ + Eliminate unused return values and variables from runtime functions.
+
+ + Only import the `among` and `SnowballProgram` classes if they're actually
+ used.
+
+ + Only generate `copy_from()` method if it's used.
+
+ + Merge runtime functions `eq_s` and `eq_v` functions.
+
+ + Java arrays know their own length so stop storing it separately.
+
+ + Escape char 127 (DEL) in generated Java code. It's unlikely that this
+ character would actually be used in a real stemmer, so this was more of a
+ theoretical bug.
+
+ + Drop unused import of InvocationTargetException from SnowballStemmer.
+ Reported by GerritDeMeulder in #72.
+
+ + Fix lint check issues in generated Java code. The stemmer classes are only
+ referenced in the example app via reflection, so add
+ @SuppressWarnings("unused") for them. The stemmer classes override
+ equals() and hashCode() methods from the standard java Object class, so
+ mark these with @Override. Both suggested by GerritDeMeulder in #72.
+
+ + Declare Java variables at point of use in generated code. Putting all
+ declarations at the top of the function was adding unnecessary complexity
+ to the Java generator code for no benefit.
+
+ + Improve formatting of generated code.
+
+New stemming algorithms
+-----------------------
+
+* Add Tamil stemmer from Damodharan Rajalingam (#2, #3).
+
+* Add Arabic stemmer from Assem Chelli (#32, #50).
+
+* Add Irish stemmer Jim O'Regan (#48).
+
+* Add Nepali stemmer from Arthur Zakirov (#70).
+
+* Add Indonesian stemmer from Olly Betts (#71).
+
+* Add Hindi stemmer from Olly Betts (#73). Thanks to David Corbett for review.
+
+* Add Lithuanian stemmer from Dainius Jocas (#22, #76).
+
+* Add Greek stemmer from Oleg Smirnov (#44).
+
+* Add Catalan and Basque stemmers from Israel Olalla (#104).
+
+Behavioural changes to existing algorithms
+------------------------------------------
+
+* Portuguese:
+
+ + Replace incorrect Spanish suffixes by Portuguese suffixes (#1).
+
+* French:
+
+ + The MSDOS CP850 version of the French algorithm was missing changes present
+ in the ISO8859-1 and Unicode versions. There's now a single version of
+ each algorithm which was based on the Unicode version.
+
+ + Recognize French suffixes even when they begin with diaereses. Patch from
+ David Corbett in #78.
+
+* Russian:
+
+ + We now normalise 'ё' to 'е' before stemming. The documentation has long
+ said "we assume ['ё'] is mapped into ['е']" but it's more convenient for
+ the stemmer to actually perform this normalisation. This change has no
+ effect if the caller is already normalising as we recommend. It's a change
+ in behaviour they aren't, but 'ё' occurs rarely (there are currently no
+ instances in our test vocabulary) and this improves behaviour when it does
+ occur. Patch from Eugene Mirotin (#65, #68).
+
+* Finish:
+
+ + Adjust the Finnish algorithm not to mangle numbers. This change also
+ means it tends to leave foreign words alone. Fixes #66.
+
+* Danish:
+
+ + Adjust Danish algorithm not to mangle alphanumeric codes. In particular
+ alphanumeric codes ending in a double digit (e.g. 0x0e00, hal9000,
+ space1999) are no longer mangled. See #81.
+
+Optimisations to existing algorithms
+------------------------------------
+
+* Turkish:
+
+ + Simplify uses of `test` in stemmer code.
+
+ + Check for 'ad' or 'soyad' more efficiently, and without needing the
+ strlen variable. This speeds up "make check_utf8_turkish" by 11%
+ on x86 Linux.
+
+* Kraaij-Pohlmann:
+
+ + Eliminate variable x `$p1 <= cursor` is simpler and a little more efficient
+ than `setmark x $x >= p1`.
+
+Code clarity improvements to existing algorithms
+------------------------------------------------
+
+* Turkish:
+
+ + Use , for cedilla to match the conventions used in other stemmers.
+
+* Kraaij-Pohlmann:
+
+ + Avoid cryptic `[among ( (])` ... `)` construct - instead use the same
+ `[substring] among (` ... `)` construct we do in other stemmers.
+
+Compiler
+--------
+
+* Support conventional --help and --version options.
+
+* Warn if -r or -ep used with backend other than C/C++.
+
+* Warn if encoding command line options are specified when generating code in a
+ language with a fixed encoding.
+
+* The default classname is now set based on the output filename, so `-n` is now
+ often no longer needed. Fixes #64.
+
+* Avoid potential one byte buffer over-read when parsing snowball code.
+
+* Avoid comparing with uninitialised array element during compilation.
+
+* Improve `-syntax` output for `setlimit L for C`.
+
+* Optimise away double negation so generators don't have to worry about
+ generating `--` (decrement operator in many languages). Fixes #52, reported
+ by David Corbett.
+
+* Improved compiler error and warning messages:
+
+ - We now report FILE:LINE: before each diagnostic message.
+
+ - Improve warnings for unused declarations/definitions.
+
+ - Warn for variables which are used, but either never initialised
+ or never read.
+
+ - Flag non-ASCII literal strings. This is an error for wide Unicode, but
+ only a warning for single-byte and UTF-8 which work so long as the source
+ encoding matches the encoding used in the generated stemmer code.
+
+ - Improve error recovery after an undeclared `define`. We now sniff the
+ token after the identifier and if it is `as` we parse as a routine,
+ otherwise we parse as a grouping. Previously we always just assumed it was
+ a routine, which gave a confusing second error if it was a grouping.
+
+ - Improve error recovery after an unexpected token in `among`. Previously
+ we acted as if the unexpected token closed the `among` (this probably
+ wasn't intended but just a missing `break;` in a switch statement). Now we
+ issue an error and try the next token.
+
+* Report error instead of silently truncating character values (e.g. `hex 123`
+ previously silently became byte 0x23 which is `#` rather than a
+ g-with-cedilla).
+
+* Enlarge the initial input buffer size to 8192 bytes and double each time we
+ hit the end. Snowball programs are typically a few KB in size (with the
+ current largest we ship being the Greek stemmer at 27KB) so the previous
+ approach of starting with a 10 byte input buffer and increasing its size by
+ 50% plus 40 bytes each time it filled was inefficient, needing up to 15
+ reallocations to load greek.sbl.
+
+* Identify variables only used by one `routine`/`external`. This information
+ isn't yet used, but such variables which are also always written to before
+ being read can be emitted as local variables in most target languages.
+
+* We now allow multiple source files on command line, and allow them to be
+ after (or even interspersed) with options to better match modern Unix
+ conventions. Support for multiple source files allows specifying a single
+ byte character set mapping via a source file of `stringdef`.
+
+* Avoid infinite recursion in compiler when optimising a recursive snowball
+ function. Recursive functions aren't typical in snowball programs, but
+ the compiler shouldn't crash for any input, especially not a valid one.
+ We now simply limit on how deep the compiler will recurse and make the
+ pessimistic assumption in the unlikely event we hit this limit.
+
+Build system:
+
+* `make clean` in C libstemmer_c distribution now removes `examples/*.o`.
+ (#59)
+
+* Fix all the places which previously had to have a list of stemmers to work
+ dynamically or be generated, so now only modules.txt needs updating to add
+ a new stemmer.
+
+* Add check_java make target which runs tests for java.
+
+* Support gzipped test data (the uncompressed arabic test data is too big for
+ github).
+
+* GNUmakefile: Drop useless `-eprefix` and `-r` options from snowball
+ invocations for Java - these are only meaningful when generating C code.
+
+* Pass CFLAGS when linking which matches convention (e.g. automake does it) and
+ facilitates use of tools such as ASan. Fixes #84, reported by Thomas
+ Pointhuber.
+
+* Add CI builds with -std=c90 to check compiler and generated code are C90
+ (#54)
+
+libstemmer stuff:
+
+* Split out CPPFLAGS from CFLAGS and use CFLAGS when linking stemwords.
+
+* Add -O2 to CFLAGS.
+
+* Make generated tables of encodings and modules const.
+
+* Fix clang static analyzer memory leak warning (in practice this code path
+ can never actually be taken). Patch from Patrick O. Perry (#56)
+
+documentation
+
+* Added copyright and licensing details (#10).
+
+* Document that libstemmer supports ISO_8859_2 encoding. Currently hungarian
+ and romanian are available in ISO_8859_2.
+
+* Remove documentation falsely claiming that libstemmer supports CP850
+ encoding.
+
+* CONTRIBUTING.rst: Add guidance for contributing new stemming algorithms and
+ new language backends.
+
+* Overhaul libstemmer_python_README. Most notably, replace the benchmark data
+ which was very out of date.
diff --git a/contrib/snowball/README b/contrib/snowball/README
new file mode 100644
index 0000000..afb51b3
--- /dev/null
+++ b/contrib/snowball/README
@@ -0,0 +1,5 @@
+This contains the source code for the snowball compiler and the stemming
+algorithms on the website.
+
+See http://snowball.tartarus.org/ for more details.
+
diff --git a/contrib/snowball/algorithms/arabic.sbl b/contrib/snowball/algorithms/arabic.sbl
new file mode 100644
index 0000000..d827ee7
--- /dev/null
+++ b/contrib/snowball/algorithms/arabic.sbl
@@ -0,0 +1,561 @@
+/*
+ * Authors:
+ * - Assem Chelli, < assem [dot] ch [at] gmail >
+ * - Abdelkrim Aries <ab [underscore] aries [at] esi [dot] dz>
+ *
+*/
+
+stringescapes { }
+
+/* the Arabic letters in Unicode */
+// Hamza
+stringdef o '{U+0621}' // Hamza
+stringdef ao '{U+0623}' // Hamza above Alef
+stringdef ao_ '{U+0625}' // Hamza below Alef
+stringdef a~ '{U+0622}' // Alef madda
+stringdef wo '{U+0624}' // Hamza above waw
+stringdef yo '{U+0626}' // Hamza above yeh
+
+// Letters
+stringdef a '{U+0627}' // Alef
+stringdef a_ '{U+0649}' // Alef Maksura
+stringdef b '{U+0628}' // Beh
+stringdef t_ '{U+0629}' // Teh_Marbuta
+stringdef t '{U+062A}' // Teh
+stringdef th '{U+062B}' // Theh
+stringdef j '{U+062C}' // Jeem
+stringdef h '{U+062D}' // Hah
+stringdef x '{U+062E}' // Khah
+stringdef d '{U+062F}' // Dal
+stringdef dz '{U+0630}' // Thal
+stringdef r '{U+0631}' // Reh
+stringdef z '{U+0632}' // Zain
+stringdef s '{U+0633}' // Seen
+stringdef sh '{U+0634}' // Sheen
+stringdef c '{U+0635}' // Sad
+stringdef dh '{U+0636}' // Dad
+stringdef tt '{U+0637}' // Tah
+stringdef zh '{U+0638}' // Zah
+stringdef i '{U+0639}' // Ain
+stringdef gh '{U+063A}' // Ghain
+stringdef f '{U+0641}' // Feh
+stringdef q '{U+0642}' // Qaf
+stringdef k '{U+0643}' // Kaf
+stringdef l '{U+0644}' // Lam
+stringdef m '{U+0645}' // Meem
+stringdef n '{U+0646}' // Noon
+stringdef e '{U+0647}' // Heh
+stringdef w '{U+0648}' // Waw
+stringdef y '{U+064A}' // Yeh
+
+// Diacritics
+stringdef aan '{U+064B}' // FatHatan
+stringdef uun '{U+064C}' // Dammatan
+stringdef iin '{U+064D}' // Kasratan
+stringdef aa '{U+064E}' // FatHa
+stringdef uu '{U+064F}' // Damma
+stringdef ii '{U+0650}' // Kasra
+stringdef oo '{U+0652}' // Sukun
+stringdef ~ '{U+0651}' // Shadda
+
+// Hindu–Arabic numerals
+stringdef 0 '{U+0660}'
+stringdef 1 '{U+0661}'
+stringdef 2 '{U+0662}'
+stringdef 3 '{U+0663}'
+stringdef 4 '{U+0664}'
+stringdef 5 '{U+0665}'
+stringdef 6 '{U+0666}'
+stringdef 7 '{U+0667}'
+stringdef 8 '{U+0668}'
+stringdef 9 '{U+0669}'
+
+
+// Kasheeda
+stringdef _ '{U+0640}' // Kasheeda, Tatweel
+
+// Shaped forms
+stringdef o1 '{U+FE80}' // HAMZA
+stringdef ao1 '{U+FE83}' // ALEF_HAMZA_ABOVE
+stringdef ao2 '{U+FE84}' // ALEF_HAMZA_ABOVE
+stringdef ao_1 '{U+FE87}' // ALEF_HAMZA_BELOW
+stringdef ao_2 '{U+FE88}' // ALEF_HAMZA_BELOW
+stringdef yo1 '{U+FE8B}' // YEH_HAMZA
+stringdef yo2 '{U+FE8C}' // YEH_HAMZA
+stringdef yo3 '{U+FE89}' // YEH_HAMZA
+stringdef yo4 '{U+FE8A}' // YEH_HAMZA
+stringdef a~1 '{U+FE81}' // ALEF_MADDA
+stringdef a~2 '{U+FE82}' // ALEF_MADDA
+stringdef wo1 '{U+FE85}' // WAW_HAMZA
+stringdef wo2 '{U+FE86}' // WAW_HAMZA
+stringdef a1 '{U+FE8D}' // ALEF
+stringdef a2 '{U+FE8E}' // ALEF
+stringdef b1 '{U+FE8F}' // BEH
+stringdef b2 '{U+FE90}' // BEH
+stringdef b3 '{U+FE91}' // BEH
+stringdef b4 '{U+FE92}' // BEH
+stringdef t_1 '{U+FE93}' // TEH_MARBUTA
+stringdef t_2 '{U+FE94}' // TEH_MARBUTA
+stringdef t1 '{U+FE97}' // TEH
+stringdef t2 '{U+FE98}' // TEH
+stringdef t3 '{U+FE95}' // TEH
+stringdef t4 '{U+FE96}' // TEH
+stringdef th1 '{U+FE9B}' // THEH
+stringdef th2 '{U+FE9C}' // THEH
+stringdef th3 '{U+FE9A}' // THEH
+stringdef th4 '{U+FE99}' // THEH
+stringdef j1 '{U+FE9F}' // JEEM
+stringdef j2 '{U+FEA0}' // JEEM
+stringdef j3 '{U+FE9D}' // JEEM
+stringdef j4 '{U+FE9E}' // JEEM
+stringdef h1 '{U+FEA3}' // HAH
+stringdef h2 '{U+FEA4}' // HAH
+stringdef h3 '{U+FEA1}' // HAH
+stringdef h4 '{U+FEA2}' // HAH
+stringdef x1 '{U+FEA7}' // KHAH
+stringdef x2 '{U+FEA8}' // KHAH
+stringdef x3 '{U+FEA5}' // KHAH
+stringdef x4 '{U+FEA6}' // KHAH
+stringdef d1 '{U+FEA9}' // DAL
+stringdef d2 '{U+FEAA}' // DAL
+stringdef dz1 '{U+FEAB}' // THAL
+stringdef dz2 '{U+FEAC}' // THAL
+stringdef r1 '{U+FEAD}' // REH
+stringdef r2 '{U+FEAE}' // REH
+stringdef z1 '{U+FEAF}' // ZAIN
+stringdef z2 '{U+FEB0}' // ZAIN
+stringdef s1 '{U+FEB3}' // SEEN
+stringdef s2 '{U+FEB4}' // SEEN
+stringdef s3 '{U+FEB1}' // SEEN
+stringdef s4 '{U+FEB2}' // SEEN
+stringdef sh1 '{U+FEB7}' // SHEEN
+stringdef sh2 '{U+FEB8}' // SHEEN
+stringdef sh3 '{U+FEB5}' // SHEEN
+stringdef sh4 '{U+FEB6}' // SHEEN
+stringdef c1 '{U+FEBB}' // SAD
+stringdef c2 '{U+FEBC}' // SAD
+stringdef c3 '{U+FEB9}' // SAD
+stringdef c4 '{U+FEBA}' // SAD
+stringdef dh1 '{U+FEBF}' // DAD
+stringdef dh2 '{U+FEC0}' // DAD
+stringdef dh3 '{U+FEBD}' // DAD
+stringdef dh4 '{U+FEBE}' // DAD
+stringdef tt1 '{U+FEC3}' // TAH
+stringdef tt2 '{U+FEC4}' // TAH
+stringdef tt3 '{U+FEC1}' // TAH
+stringdef tt4 '{U+FEC2}' // TAH
+stringdef zh1 '{U+FEC7}' // ZAH
+stringdef zh2 '{U+FEC8}' // ZAH
+stringdef zh3 '{U+FEC5}' // ZAH
+stringdef zh4 '{U+FEC6}' // ZAH
+stringdef i1 '{U+FECB}' // AIN
+stringdef i2 '{U+FECC}' // AIN
+stringdef i3 '{U+FEC9}' // AIN
+stringdef i4 '{U+FECA}' // AIN
+stringdef gh1 '{U+FECF}' // GHAIN
+stringdef gh2 '{U+FED0}' // GHAIN
+stringdef gh3 '{U+FECD}' // GHAIN
+stringdef gh4 '{U+FECE}' // GHAIN
+stringdef f1 '{U+FED3}' // FEH
+stringdef f2 '{U+FED4}' // FEH
+stringdef f3 '{U+FED1}' // FEH
+stringdef f4 '{U+FED2}' // FEH
+stringdef q1 '{U+FED7}' // QAF
+stringdef q2 '{U+FED8}' // QAF
+stringdef q3 '{U+FED5}' // QAF
+stringdef q4 '{U+FED6}' // QAF
+stringdef k1 '{U+FEDB}' // KAF
+stringdef k2 '{U+FEDC}' // KAF
+stringdef k3 '{U+FED9}' // KAF
+stringdef k4 '{U+FEDA}' // KAF
+stringdef l1 '{U+FEDF}' // LAM
+stringdef l2 '{U+FEE0}' // LAM
+stringdef l3 '{U+FEDD}' // LAM
+stringdef l4 '{U+FEDE}' // LAM
+stringdef m1 '{U+FEE3}' // MEEM
+stringdef m2 '{U+FEE4}' // MEEM
+stringdef m3 '{U+FEE1}' // MEEM
+stringdef m4 '{U+FEE2}' // MEEM
+stringdef n1 '{U+FEE7}' // NOON
+stringdef n2 '{U+FEE8}' // NOON
+stringdef n3 '{U+FEE5}' // NOON
+stringdef n4 '{U+FEE6}' // NOON
+stringdef e1 '{U+FEEB}' // HEH
+stringdef e2 '{U+FEEC}' // HEH
+stringdef e3 '{U+FEE9}' // HEH
+stringdef e4 '{U+FEEA}' // HEH
+stringdef w1 '{U+FEED}' // WAW
+stringdef w2 '{U+FEEE}' // WAW
+stringdef a_1 '{U+FEEF}' // ALEF_MAKSURA
+stringdef a_2 '{U+FEF0}' // ALEF_MAKSURA
+stringdef y1 '{U+FEF3}' // YEH
+stringdef y2 '{U+FEF4}' // YEH
+stringdef y3 '{U+FEF1}' // YEH
+stringdef y4 '{U+FEF2}' // YEH
+
+// Ligatures Lam-Alef
+stringdef la '{U+FEFB}' // LAM_ALEF
+stringdef la2 '{U+FEFC}' // LAM_ALEF
+stringdef lao '{U+FEF7}' // LAM_ALEF_HAMZA_ABOVE
+stringdef lao2 '{U+FEF8}' // LAM_ALEF_HAMZA_ABOVE
+stringdef lao_ '{U+FEF9}' // LAM_ALEF_HAMZA_BELOW
+stringdef lao_2 '{U+FEFA}' // LAM_ALEF_HAMZA_BELOW
+stringdef la~ '{U+FEF5}' // LAM_ALEF_MADDA_ABOVE
+stringdef la~2 '{U+FEF6}' // LAM_ALEF_MADDA_ABOVE
+
+
+booleans (
+ is_noun
+ is_verb
+ is_defined
+ )
+
+routines (
+ Prefix_Step1
+ Prefix_Step2
+ Prefix_Step3a_Noun
+ Prefix_Step3b_Noun
+ Prefix_Step3_Verb
+ Prefix_Step4_Verb
+
+ Suffix_All_alef_maqsura
+ Suffix_Noun_Step1a
+ Suffix_Noun_Step1b
+ Suffix_Noun_Step2a
+ Suffix_Noun_Step2b
+ Suffix_Noun_Step2c1
+ Suffix_Noun_Step2c2
+ Suffix_Noun_Step3
+ Suffix_Verb_Step1
+ Suffix_Verb_Step2a
+ Suffix_Verb_Step2b
+ Suffix_Verb_Step2c
+
+ Normalize_post
+ Normalize_pre
+
+ Checks1
+)
+
+externals ( stem )
+
+groupings ( )
+
+
+// Normalizations
+define Normalize_pre as (
+ do repeat (
+ (
+ [substring] among (
+ '{aan}' '{uun}' '{iin}' '{aa}' '{uu}' '{ii}' '{oo}' '{~}'( delete ) // strip vocalization
+ '{_}' ( delete ) // strip kasheeda
+
+ // Hindu–Arabic numerals
+ '{0}' ( <- '0')
+ '{1}' ( <- '1')
+ '{2}' ( <- '2')
+ '{3}' ( <- '3')
+ '{4}' ( <- '4')
+ '{5}' ( <- '5')
+ '{6}' ( <- '6')
+ '{7}' ( <- '7')
+ '{8}' ( <- '8')
+ '{9}' ( <- '9')
+
+ // Shaped forms
+ '{o1}' ( <- '{o}' ) // HAMZA
+ '{ao1}' '{ao2}' ( <- '{ao}' ) // ALEF_HAMZA_ABOVE
+ '{ao_1}' '{ao_2}' ( <- '{ao_}' ) // ALEF_HAMZA_BELOW
+ '{yo1}' '{yo2}' '{yo3}' '{yo4}' ( <- '{yo}' ) // YEH_HAMZA
+ '{a~1}' '{a~2}'( <- '{a~}' ) // ALEF_MADDA
+ '{wo1}' '{wo2}'( <- '{wo}' ) // WAW_HAMZA
+ '{a1}' '{a2}' ( <- '{a}' ) // ALEF
+ '{b1}' '{b2}' '{b3}' '{b4}' ( <- '{b}' ) // BEH
+ '{t_1}' '{t_2}' ( <- '{t_}' ) // TEH_MARBUTA
+ '{t1}' '{t2}' '{t3}' '{t4}' ( <- '{t}' ) // TEH
+ '{th1}' '{th2}' '{th3}' '{th4}' ( <- '{th}' ) // THEH
+ '{j1}' '{j2}' '{j3}' '{j4}'( <- '{j}' ) // JEEM
+ '{h1}' '{h2}' '{h3}' '{h4}' ( <- '{h}' ) // HAH
+ '{x1}' '{x2}' '{x3}' '{x4}'( <- '{x}' ) // KHAH
+ '{d1}' '{d2}' ( <- '{d}' ) // DAL
+ '{dz1}''{dz2}' ( <- '{dz}' ) // THAL
+ '{r1}' '{r2}'( <- '{r}' ) // REH
+ '{z1}' '{z2}' ( <- '{z}' ) // ZAIN
+ '{s1}' '{s2}' '{s3}' '{s4}'( <- '{s}' ) // SEEN
+ '{sh1}' '{sh2}' '{sh3}' '{sh4}' ( <- '{sh}' ) // SHEEN
+ '{c1}' '{c2}' '{c3}' '{c4}'( <- '{c}' ) // SAD
+ '{dh1}' '{dh2}' '{dh3}' '{dh4}'( <- '{dh}' ) // DAD
+ '{tt1}' '{tt2}' '{tt3}' '{tt4}' ( <- '{tt}' ) // TAH
+ '{zh1}' '{zh2}' '{zh3}' '{zh4}'( <- '{zh}' ) // ZAH
+ '{i1}' '{i2}' '{i3}' '{i4}'( <- '{i}' ) // AIN
+ '{gh1}' '{gh2}' '{gh3}' '{gh4}'( <- '{gh}' ) // GHAIN
+ '{f1}' '{f2}' '{f3}' '{f4}' ( <- '{f}' ) // FEH
+ '{q1}' '{q2}' '{q3}' '{q4}' ( <- '{q}' ) // QAF
+ '{k1}' '{k2}' '{k3}' '{k4}'( <- '{k}' ) // KAF
+ '{l1}' '{l2}' '{l3}' '{l4}'( <- '{l}' ) // LAM
+ '{m1}' '{m2}' '{m3}' '{m4}' ( <- '{m}' ) // MEEM
+ '{n1}' '{n2}' '{n3}' '{n4}'( <- '{n}' ) // NOON
+ '{e1}' '{e2}' '{e3}' '{e4}' ( <- '{e}' ) // HEH
+ '{w1}' '{w2}' ( <- '{w}' ) // WAW
+ '{a_1}' '{a_2}' ( <- '{a_}' ) // ALEF_MAKSURA
+ '{y1}' '{y2}' '{y3}' '{y4}' ( <- '{y}' ) // YEH
+
+ // Ligatures Lam-Alef
+ '{la}' '{la2}' (<- '{l}{a}')
+ '{lao}' '{lao2}' (<- '{l}{ao}')
+ '{lao_}' '{lao_2}' (<- '{l}{ao_}')
+ '{la~}' '{la~2}' (<- '{l}{a~}')
+
+ )
+ )
+ or
+ next
+ )
+)
+
+define Normalize_post as (
+
+ do (
+ // normalize last hamza
+ backwards (
+ [substring] among (
+ '{ao}''{ao_}' '{a~}' ( <- '{o}')
+ '{wo}' ( <- '{o}')
+ '{yo}' ( <- '{o}')
+ )
+ )
+ )
+
+ do repeat (
+ (
+ // normalize other hamza's
+ [substring] among (
+ '{ao}''{ao_}' '{a~}' ( <- '{a}')
+ '{wo}' ( <- '{w}')
+ '{yo}' ( <- '{y}')
+ )
+ )
+ or
+ next
+ )
+)
+
+// Checks
+define Checks1 as (
+ [substring] among (
+ '{b}{a}{l}' '{k}{a}{l}' ($(len > 4) set is_noun unset is_verb set is_defined)
+ '{l}{l}' '{a}{l}' ($(len > 3) set is_noun unset is_verb set is_defined)
+ )
+)
+
+
+//prefixes
+define Prefix_Step1 as (
+ [substring] among (
+ '{ao}{ao}' ($(len > 3) <- '{ao}' )
+ '{ao}{a~}' ($(len > 3) <- '{a~}' )
+ '{ao}{wo}' ($(len > 3) <- '{ao}' )
+ '{ao}{a}' ($(len > 3) <- '{a}' )
+ '{ao}{ao_}' ($(len > 3) <- '{ao_}' )
+ // '{ao}' ($(len > 3) delete) //rare case
+ )
+)
+
+define Prefix_Step2 as (
+ not '{f}{a}'
+ not '{w}{a}'
+ [substring] among (
+ '{f}' ($(len > 3) delete)
+ '{w}' ($(len > 3) delete)
+ )
+)
+
+define Prefix_Step3a_Noun as ( // it is noun and defined
+ [substring] among (
+ '{b}{a}{l}' '{k}{a}{l}' ($(len > 5) delete)
+ '{l}{l}' '{a}{l}' ($(len > 4) delete)
+ )
+)
+
+define Prefix_Step3b_Noun as ( // probably noun and defined
+ not '{b}{a}' // exception
+ [substring] among (
+ '{b}' ($(len > 3) delete)
+ // '{k}' '{l}' ($(len > 3) delete) // BUG: cause confusion
+ '{b}{b}' ($(len > 3) <- '{b}' )
+ '{k}{k}' ($(len > 3) <- '{k}' )
+ )
+
+)
+
+define Prefix_Step3_Verb as (
+ [substring] among (
+ //'{s}' ($(len > 4) delete)// BUG: cause confusion
+ '{s}{y}' ($(len > 4) <- '{y}' )
+ '{s}{t}' ($(len > 4) <- '{t}')
+ '{s}{n}' ($(len > 4) <- '{n}')
+ '{s}{ao}' ($(len > 4) <- '{ao}')
+ )
+)
+
+define Prefix_Step4_Verb as (
+ [substring] among (
+ '{y}{s}{t}' '{n}{s}{t}' '{t}{s}{t}' ($(len > 4) set is_verb unset is_noun <- '{a}{s}{t}' )
+ )
+)
+
+// suffixes
+backwardmode (
+
+ define Suffix_Noun_Step1a as (
+ [substring] among (
+ '{y}' '{k}' '{e}' ($(len >= 4) delete)
+ '{n}{a}' '{k}{m}' '{e}{a}' '{e}{n}' '{e}{m}' ($(len >= 5) delete)
+ '{k}{m}{a}' '{e}{m}{a}' ($(len >= 6) delete)
+ )
+ )
+ define Suffix_Noun_Step1b as (
+ [substring] among (
+ '{n}' ($(len > 5) delete)
+ )
+ )
+
+ define Suffix_Noun_Step2a as (
+ [substring] among (
+ '{a}' '{y}' '{w}' ($(len > 4) delete)
+ )
+ )
+
+ define Suffix_Noun_Step2b as (
+ [substring] among (
+ '{a}{t}' ($(len >= 5) delete)
+ )
+ )
+
+ define Suffix_Noun_Step2c1 as (
+ [substring] among (
+ '{t}' ($(len >= 4) delete)
+ )
+ )
+ define Suffix_Noun_Step2c2 as ( // feminine t_
+ [substring] among (
+ '{t_}' ($(len >= 4) delete)
+ )
+ )
+ define Suffix_Noun_Step3 as ( // ya' nisbiya
+ [substring] among (
+ '{y}' ($(len >= 3) delete)
+ )
+ )
+
+ define Suffix_Verb_Step1 as (
+ [substring] among (
+ '{e}' '{k}' ($(len >= 4) delete)
+ '{n}{y}' '{n}{a}' '{e}{a}' '{e}{m}' '{e}{n}' '{k}{m}' '{k}{n}' ($(len >= 5) delete)
+ '{e}{m}{a}' '{k}{m}{a}' '{k}{m}{w}'($(len >= 6) delete)
+ )
+ )
+ define Suffix_Verb_Step2a as (
+ [substring] among (
+ '{t}' ($(len >= 4) delete)
+ '{a}' '{n}' '{y}' ($(len >= 4) delete)
+ '{n}{a}' '{t}{a}' '{t}{n}' ($(len >= 5) delete)// past
+ '{a}{n}' '{w}{n}' '{y}{n}' ($(len > 5) delete) // present
+ '{t}{m}{a}' ($(len >= 6) delete)
+ )
+ )
+
+ define Suffix_Verb_Step2b as (
+ [substring] among (
+ '{w}{a}' '{t}{m}' ($(len >= 5) delete)
+ )
+ )
+
+
+ define Suffix_Verb_Step2c as (
+ [substring] among (
+ '{w}' ($(len >= 4) delete)
+ '{t}{m}{w}' ($(len >= 6) delete)
+ )
+ )
+
+ define Suffix_All_alef_maqsura as (
+ [substring] among (
+ '{a_}' ( <- '{y}' ) // spell error
+ // '{a_}' ( delete ) // if noun > 3
+ // '{a_}' ( <- '{a}') // if verb
+ )
+ )
+)
+
+define stem as (
+ // set initial values
+ set is_noun
+ set is_verb
+ unset is_defined
+
+ // guess type and properties
+ do Checks1
+
+ // normalization pre-stemming
+ do Normalize_pre
+
+
+ backwards (
+
+ do (
+ //Suffixes for verbs
+ (
+ is_verb
+ (
+ (
+ (atleast 1 Suffix_Verb_Step1)
+ ( Suffix_Verb_Step2a or Suffix_Verb_Step2c or next)
+ )
+ or Suffix_Verb_Step2b
+ or Suffix_Verb_Step2a
+ )
+ )
+ //Suffixes for nouns
+ or (
+ is_noun
+ (
+
+ try (
+ Suffix_Noun_Step2c2
+ or (not is_defined Suffix_Noun_Step1a (
+ Suffix_Noun_Step2a
+ or Suffix_Noun_Step2b
+ or Suffix_Noun_Step2c1
+ or next))
+ or (Suffix_Noun_Step1b (
+ Suffix_Noun_Step2a
+ or Suffix_Noun_Step2b
+ or Suffix_Noun_Step2c1))
+ or (not is_defined Suffix_Noun_Step2a)
+ or (Suffix_Noun_Step2b)
+ )
+ Suffix_Noun_Step3
+ )
+
+ )
+
+ // Suffixes for alef maqsura
+ or Suffix_All_alef_maqsura
+ )
+ )
+
+ //Prefixes
+ do (
+ try Prefix_Step1
+ try Prefix_Step2
+ ( Prefix_Step3a_Noun
+ or (is_noun Prefix_Step3b_Noun)
+ or (is_verb try Prefix_Step3_Verb Prefix_Step4_Verb)
+ )
+ )
+
+ // normalization post-stemming
+ do Normalize_post
+
+)
diff --git a/contrib/snowball/algorithms/basque.sbl b/contrib/snowball/algorithms/basque.sbl
new file mode 100644
index 0000000..267abc7
--- /dev/null
+++ b/contrib/snowball/algorithms/basque.sbl
@@ -0,0 +1,149 @@
+routines (
+ aditzak
+ izenak
+ adjetiboak
+ mark_regions
+ RV R2 R1
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef n~ '{U+00F1}'
+
+define v 'aeiou'
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R2 as $p2 <= cursor
+ define R1 as $p1 <= cursor
+
+ define aditzak as (
+ [substring] among(
+ 'le' 'la' 'tzaile' 'aldatu' 'atu' 'tzailea' 'taile' 'tailea' 'pera' 'gale' 'galea'
+ 'gura' 'kura' 'kor' 'korra' 'or' 'orra' 'tun' 'tuna' 'gaitz' 'gaitza'
+ 'kaitz' 'kaitza' 'ezin' 'ezina' 'tezin' 'tezina' 'errez' 'erreza'
+ 'karri' 'karria' 'tzaga' 'tzaka' 'tzake' 'tzeke' 'ez' 'eza' 'tzez'
+ 'keta' 'eta' 'etan' 'pen' 'pena' 'tze' 'atze' 'kuntza' 'kunde' 'kundea'
+ 'kune' 'kunea' 'kuna' 'kera' 'era' 'kizun' 'kizuna' 'dura' 'tura' 'men' 'mena'
+ 'go' 'ago' 'tio' 'taldi' 'taldia' 'aldi' 'aldia' 'gune' 'gunea' 'bide' 'bidea'
+ 'pide' 'pidea' 'gai' 'gaia' 'ki' 'kin' 'rekin' 'kina' 'kari' 'karia' 'ari' 'tari' 'etari'
+ 'gailu' 'gailua' 'kide' 'kidea' 'ide' 'idea' 'du' 'ka' 'kan' 'an' 'ean' 'tu' 'lari' 'tatu'
+ 'rean' 'tarazi' 'arazi' 'tzat' 'bera' 'dako'
+ ( RV delete )
+ 'garri' 'garria' 'tza'
+ (R2 delete)
+ 'atseden'
+ (<- 'atseden')
+ 'arabera'
+ (<- 'arabera')
+ 'baditu'
+ (<- 'baditu')
+
+ )
+ )
+
+ define izenak as (
+ [substring] among(
+ 'ari' 'aria' 'bizia' 'kari' 'karia' 'lari' 'laria' 'tari' 'taria' 'zain' 'zaina'
+ 'tzain' 'tzaina' 'zale' 'zalea' 'tzale' 'tzalea' 'aizun' 'orde' 'ordea'
+ 'burua' 'ohi' 'ohia' 'kintza' 'gintzo' 'gintzu' 'tzu' 'tzua'
+ 'tzo' 'tzoa' 'kuntza' 'talde' 'taldea' 'eria' 'keria' 'teria' 'di'
+ 'za' 'ada' 'tara' 'etara' 'tra' 'ta' 'tegi' 'tegia' 'keta' 'z' 'zko' 'zkoa'
+ 'ti' 'tia' 'tsu' 'tsua' 'zu' 'zua' 'bera' 'pera' 'zto' 'ztoa' 'asi' 'asia'
+ 'gile' 'gilea' 'estu' 'estua' 'larri' 'larria' 'nahi' 'nahia'
+ 'koi' 'koia' 'oi' 'oia' 'goi' 'min' 'mina' 'dun' 'duna' 'duru' 'durua'
+ 'duri' 'duria' 'os' 'osa' 'oso' 'osoa' 'ar' 'ara' 'tar' 'dar' 'dara'
+ 'tiar' 'tiara' 'liar' 'liara' 'gabe' 'gabea' 'kabe' 'kabea' 'ga' 'ge'
+ 'kada' 'tasun' 'tasuna' 'asun' 'asuna' 'go' 'mendu' 'mendua' 'mentu' 'mentua'
+ 'mendi' 'mendia' 'zio' 'zioa' 'zino' 'zinoa' 'zione' 'zionea' 'ezia'
+ 'degi' 'degia' 'egi' 'egia' 'toki' 'tokia' 'leku' 'lekua' 'gintza' 'alde'
+ 'aldea' 'kalde' 'kaldea' 'gune' 'gunea' 'une' 'unea' 'una' 'pe' 'pea'
+ 'gibel' 'gibela' 'ondo' 'ondoa' 'arte' 'artea' 'aurre' 'aurrea'
+ 'etxe' 'etxea' 'ola' 'ontzi' 'ontzia' 'gela' 'denda' 'taldi' 'taldia'
+ 'aldi' 'aldia' 'te' 'tea' 'zaro' 'zaroa' 'taro' 'taroa' 'oro' 'oroa'
+ 'aro' 'aroa' 'ero' 'eroa' 'eroz' 'eroza' 'ka' 'kan' 'kana' 'tako' 'etako' 'takoa'
+ 'kote' 'kotea' 'tzar' 'tzarra' 'handi' 'handia' 'kondo' 'kondoa' 'skila'
+ 'no' 'noa' '{n~}o' '{n~}oa' 'ska' 'xka' 'zka' 'tila' 'to' 'toa' 'tto' 'ttoa'
+ 'txo' 'txoa' 'txu' 'txua' 'anda' 'anga' 'urren' 'urrena' 'gai' 'gaia'
+ 'gei' 'geia' 'eme' 'emea' 'kume' 'kumea' 'sa' 'ko' 'eko' 'koa' 'ena'
+ 'enea' 'ne' 'nea' 'kor' 'korra' 'ez' 'eza' 'eta' 'etan'
+ 'ki' 'kia' 'kin' 'kina' 'tu' 'tua' 'du' 'dua' 'ek'
+ 'tarik' 'tariko' 'tan' 'ordu' 'ordua' 'oste' 'ostea' 'tzara'
+ 'ra' 'antza' 'behar' 'ro' 'giro' 'ak' 'zp' 'ket'
+ 'kail' 'kaila' 'ail' 'kirri' 'kirria' 'ngo' 'ngoa' '{n~}i' 'sko'
+ 'sta' 'koitz' 'koitza' 'na' 'garren' 'garrena' 'kera'
+ 'gerren' 'gerrena' 'garna' 'kide' 'tz' 'tuko'
+ ( RV delete )
+ 'ora' 'garri' 'garria' 'or' 'buru' 'ren' 'tza'
+ ( R2 delete )
+ 'joka'
+ (<- 'jok')
+ 'tzen' 'ten' 'en' 'tatu'
+ (R1 delete)
+ 'trako'
+ (<- 'tra')
+ 'minutuko'
+ (<- 'minutu')
+ 'zehar'
+ (<- 'zehar')
+ 'geldi'
+ (<- 'geldi')
+ 'igaro'
+ (<- 'igaro')
+ 'aurka'
+ (<- 'aurka')
+ )
+ )
+
+ define adjetiboak as (
+ [substring] among(
+ 'era' 'ero' 'go' 'tate' 'tade' 'date' 'dade' 'keria'
+ 'ki' 'to' 'ro' 'la' 'gi' 'larik' 'lanik' 'ik' 'ztik' 'rik'
+ ( RV delete )
+ 'zlea'
+ (<- 'z')
+ )
+ )
+
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ repeat aditzak
+ repeat izenak
+ do adjetiboak
+ )
+
+)
+
+/*
+ Note 1: additions of 21 Jul 2010
+*/
diff --git a/contrib/snowball/algorithms/catalan.sbl b/contrib/snowball/algorithms/catalan.sbl
new file mode 100644
index 0000000..0a1e3a5
--- /dev/null
+++ b/contrib/snowball/algorithms/catalan.sbl
@@ -0,0 +1,202 @@
+routines (
+ cleaning mark_regions
+ R1 R2
+ attached_pronoun
+ standard_suffix
+ verb_suffix
+ residual_suffix
+)
+
+externals ( stem )
+
+integers ( p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef a` '{U+00E0}' // a-grave
+stringdef c, '{U+00E7}' // c-cedilla
+stringdef e' '{U+00E9}' // e-acute
+stringdef e` '{U+00E8}' // e-grave
+stringdef i' '{U+00ED}' // i-acute
+stringdef i` '{U+00EC}' // i-grave
+stringdef i" '{U+00EF}' // i-diaeresis
+stringdef o' '{U+00F3}' // o-acute
+stringdef o` '{U+00F2}' // o-grave
+stringdef u' '{U+00FA}' // u-acute
+stringdef u" '{U+00FC}' // u-diaeresis
+stringdef . '{U+00B7}' // - per l aggeminades
+
+define v 'aeiou{a'}{a`}{e'}{e`}{i'}{i"}{o'}{o`}{u'}{u"}'
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define cleaning as repeat (
+ [substring] among(
+ '{a'}' (<- 'a')
+ '{a`}' (<- 'a')
+ '{e'}' (<- 'e')
+ '{e`}' (<- 'e')
+ '{i'}' (<- 'i')
+ '{i`}' (<- 'i')
+ '{o'}' (<- 'o')
+ '{o`}' (<- 'o')
+ '{u'}' (<- 'u')
+ '{u"}' (<- 'u')
+ '{i"}' (<- 'i')
+ '{.}' (<- '.')
+ '' (next)
+ )
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define attached_pronoun as (
+ [substring] among (
+ '{'}s' '{'}hi' '{'}ho' '{'}l' '{'}ls'
+ '-ls' '-la' '-les' '-li'
+ 'vos' 'se' 'nos' '-nos' '-us' 'us'
+ '{'}n' '{'}ns' '-n' '-ns'
+ '{'}m' '-me' '-m'
+ '-te' '{'}t'
+ 'li' 'lo' 'los'
+ 'me' 'sela' 'selo' 'selas' 'selos' 'le'
+ 'la' 'las' 'les' 'ens' 'ho' 'hi'
+ (R1 delete)
+ )
+ )
+
+ define standard_suffix as (
+ [substring] among(
+ 'ar' 'atge' 'formes' 'icte' 'ictes'
+ 'ell' 'ells' 'ella' '{e'}s' '{e`}s' 'esc' 'essa' 'et' 'ets' 'eta'
+ 'eres' 'eries' 'ers' 'ina' 'ines' 'able' 'ls'
+ 'i{o'}' 'itat' 'itats' 'itzar' 'iva' 'ives' 'ivisme' 'ius'
+ 'fer' 'ment' 'amen' 'ament' 'aments' 'ments' 'ot' 'sfera' 'al' 'als' 'era' 'ana' 'iste'
+ 'aire' 'eria' 'esa' 'eses' 'esos' 'or' '{i'}cia' '{i'}cies' 'icis' 'ici' '{i'}ci' '{i'}cis'
+ '{a`}ria' '{a`}ries' 'alla' 'ci{o'}' 'cions' 'n{c,}a' 'nces' '{o'}' 'dor' 'all'
+ 'il' '{i'}stic' 'enc' 'enca' '{i'}s' 'issa' 'issos' '{i'}ssem' '{i'}ssiu' 'issem' 'isseu' '{i'}sseu'
+ '{o'}s' 'osa' 'dora' 'dores' 'dors' 'adura' 'ble' 'bles' '{i'}vol' '{i'}vola' 'd{i'}s' 'egar' 'ejar' 'ificar'
+ 'itar' 'ables' 'adors' 'idores' 'idors'
+ 'adora' 'aci{o'}' 'doras' 'dur' 'dures' 'alleng{u"}es'
+ 'ant' 'ants' 'ancia' 'ancies' 'at{o`}ria' 'at{o`}ries' 'tori' 'toris'
+ 'ats' 'ions' 'ota' 'isam' 'ors' 'ora' 'ores' 'isament'
+ 'bilitat' 'bilitats' 'ivitat' 'ivitats' 'ari' 'aris' 'ionisme' 'ionista' 'ionistes'
+ 'ialista' 'ialistes' 'ialisme' 'ialismes' 'ud' 'uts' 'uds' 'encia' 'encies' '{e`}ncia' '{e`}ncies'
+ '{i"}tat' '{i"}tats' 'atiu' 'atius' 'atives' 'ativa' 'ativitat' 'ativitats' 'ible' 'ibles'
+ 'assa' 'asses' 'assos'
+ 'ent' 'ents'
+ '{i'}ssim' '{i'}ssima' '{i'}ssims' '{i'}ssimes' '{i`}ssem' '{i`}sseu' '{i`}ssin'
+ 'ims' 'ima' 'imes'
+ 'isme' 'ista' 'ismes' 'istes'
+ 'inia' 'inies' '{i'}inia' '{i'}nies' 'ita' 'ites' 'triu' 'trius'
+ 'oses' 'osos' 'ient' 'otes' 'ots'
+ (R1 delete)
+ 'acions' 'ada' 'ades'
+ (R2 delete)
+ 'log{i'}a' 'log{i'}es''logia' 'logies' 'logi' 'logis' 'l{o'}gica' 'l{o'}gics' 'l{o'}giques'
+ (R2 <- 'log')
+ 'ic' 'ica' 'ics' 'iques'
+ (R2 <- 'ic')
+ 'qu{i'}ssim' 'qu{i'}ssims' 'qu{i'}ssimes' 'qu{i'}ssima'
+ (R1 <- 'c')
+ )
+ )
+
+ define verb_suffix as (
+ [substring] among(
+ 'ador' 'adora' 'adors' 'adores' 're' 'ie'
+ 'ent' 'ents' 'udes' 'ar{a`}' 'eren'
+ 'ar{a'}' 'ar{i'}an' 'ar{i'}as' 'ar{a'}n' 'ar{a'}s' 'ar{i'}ais'
+ 'aria' 'arian' 'arien' 'aries' 'ar{a`}s'
+ 'ar{i'}a' 'ar{e'}is' 'ar{i'}amos' 'aremos' 'ara'
+ 'ar{e'}' 'ar{e'}s'
+ 'er{i'}an' 'er{i'}as' 'er{a'}n' 'er{a'}s' 'er{i'}ais'
+ 'er{i'}a' 'er{e'}is' 'er{i'}amos' 'eremos' 'er{a'}'
+ 'er{e'}' 'er' 'erau' 'erass'
+ 'ir{i'}an' 'ir{i'}as' 'ir{a'}n' 'ir{a'}s' 'ir{i'}ais'
+ 'ir{i'}a' 'ir{e'}is' 'ir{i'}amos' 'iremos' 'ir{a'}'
+ 'ir{e'}' '{i'}rem' '{i'}reu' '{i'}eu'
+ 'ia' 'ies' '{i'}em' '{i`}eu' 'ien'
+ 'at' 'ut' 'uda' 'ava' 'aves' 'avem' '{a'}vem' '{a`}vem' '{a`}veu' '{a'}veu' 'aven' 'au' 'ats'
+ 'asseu' 'esseu' 'eresseu' '{a`}sseu' '{a`}ssem' '{a`}ssim' '{a`}ssiu'
+ 'essen' 'esses' 'assen' 'asses' 'assim' 'assiu'
+ '{e'}ssen' '{e'}sseu' '{e'}ssim' '{e'}ssiu' '{e'}ssem'
+ '{i'}' 'ares' '{a`}rem' '{a`}reu' '{a`}ren'
+ 'ar{i'}em' 'ar{i'}eu'
+ 'areu' 'aren' 'ant' '{i"}m' '{i"}u'
+ '{e'}s' '{i"}en' 'en' 'es' 'em' 'am' 'ams' '{i"}a' '{i"}es'
+ 'dre' 'eix' 'eixer' 'tzar' 'eixes' 'ides' '{i"}des' 'it' '{i"}t' '{i"}da'
+ 'aba' 'ada' 'ades' 'ida' '{i'}a' 'iera' 'ad' 'ed' 'its'
+ 'id' 'ids' 'ase' 'iese' 'aste' 'iste' 'an' 'aban' '{i'}an'
+ 'aran' 'ieran' 'asen' 'iesen' 'aron' 'ieron' 'ado'
+ 'ido' 'iendo' 'i{o'}' 'ar' 'ir' 'as'
+ 'ieu' 'ii' 'io' 'i{a`}'
+ 'ess' 'essin' 'essis' 'ass' 'assin' 'assis' 'essim' '{e`}ssim' '{e`}ssiu'
+ 'abas' 'adas' 'idas' '{i'}as' 'aras' 'ieras' 'ases'
+ 'ieses' '{i'}s' '{a'}is' 'abais' '{i'}ais' 'arais'
+ 'ierais' 'aseis' 'ieseis' 'asteis' 'isteis' 'ados'
+ 'idos' 'amos' '{a'}bamos' '{i'}amos' 'imos' 'ques'
+ '{a'}ramos' 'i{e'}ramos' 'i{e'}semos' '{a'}semos'
+ 'ira' 'iran' 'irem' 'iren' 'ires' 'ireu' 'iria' 'irien'
+ 'iries' 'ir{a`}' 'ir{a`}s' 'ir{e`}' 'ir{i`}em' 'ir{i`}eu'
+ 'isquen' 'iguem' 'igueu' 'esqui' 'esquin' 'esquis' 'eixi' 'eixin' 'eixis'
+ 'eixen' 'eixo' 'isin' 'isis' 'esques' 'sis' 'sin'
+ 'int' 'ir{i'}em' 'ir{i'}eu' 'isc' 'atges' 'esca' 'esquen'
+ 'issen' 'isses' 'issin' 'issis' 'isca' 'issiu' 'issim'
+ '{i"}sc' '{i"}sca' '{i"}ssin' '{i'}ssiu' '{i'}ssim' '{i"}ssis' '{i"}guem' '{i"}gueu'
+ '{i"}ra' '{i"}ren' '{i"}res'
+ '{i"}squen' '{i"}sques' '{i"}ssen' '{i"}sses' '{i"}xo' '{i"}xen' '{i"}xes' '{i"}x'
+ 'ixo' 'ixen' 'ixes' 'ix' 'ixa' 'inin' 'inis' 'ini' 'ineu' 'itza' 'itzi' 'itzeu' 'itzis'
+ 'itzo' 'itz' 'itz{a`}' 'arem' 'in' '{a`}s' 'i{i"}' 'i{i"}n' 'i{i"}s'
+ (R1 delete)
+ 'ando'
+ (R2 delete)
+ )
+ )
+
+ define residual_suffix as (
+ [substring] among(
+ 'os' 'a' 'o' '{a'}' '{a`}' '{i'}' '{o'}' 'e' '{e'}' 'eu' 'iu'
+ 'is' 'i' 'ir' 's' '{i`}' 'itz' '{i"}' '{i"}n' '{i"}s' 'it'
+ (R1 delete)
+ 'iqu'
+ (R1 <- 'ic')
+ )
+ )
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ do attached_pronoun
+ do ( standard_suffix or
+ verb_suffix
+ )
+ do residual_suffix
+ )
+ do cleaning
+)
+
+/*
+ First works 2010/07/19
+ First Grammatical Reviews: https://ca.wikipedia.org/wiki/Gram%C3%A0tica_del_catal%C3%A0
+ Suffix list: https://ca.wikipedia.org/wiki/Llista_de_sufixos
+ Irregular Verbs: https://ca.wikipedia.org/wiki/Flexi%C3%B3_verbal_del_catal%C3%A0
+*/
diff --git a/contrib/snowball/algorithms/danish.sbl b/contrib/snowball/algorithms/danish.sbl
new file mode 100644
index 0000000..761270f
--- /dev/null
+++ b/contrib/snowball/algorithms/danish.sbl
@@ -0,0 +1,93 @@
+routines (
+ mark_regions
+ main_suffix
+ consonant_pair
+ other_suffix
+ undouble
+)
+
+externals ( stem )
+
+strings ( ch )
+
+integers ( p1 x )
+
+groupings ( c v s_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef ae '{U+00E6}'
+stringdef ao '{U+00E5}'
+stringdef o/ '{U+00F8}'
+
+define c 'bcdfghjklmnpqrstvwxz'
+
+define v 'aeiouy{ae}{ao}{o/}'
+
+define s_ending 'abcdfghjklmnoprtvyz{ao}'
+
+define mark_regions as (
+
+ $p1 = limit
+
+ test ( hop 3 setmark x )
+ goto v gopast non-v setmark p1
+ try ( $p1 < x $p1 = x )
+)
+
+backwardmode (
+
+ define main_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+
+ 'hed' 'ethed' 'ered' 'e' 'erede' 'ende' 'erende' 'ene' 'erne' 'ere'
+ 'en' 'heden' 'eren' 'er' 'heder' 'erer' 'heds' 'es' 'endes'
+ 'erendes' 'enes' 'ernes' 'eres' 'ens' 'hedens' 'erens' 'ers' 'ets'
+ 'erets' 'et' 'eret'
+ (delete)
+ 's'
+ (s_ending delete)
+ )
+ )
+
+ define consonant_pair as (
+ test (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'gd' // significant in the call from other_suffix
+ 'dt' 'gt' 'kt'
+ )
+ )
+ next] delete
+ )
+
+ define other_suffix as (
+ do ( ['st'] 'ig' delete )
+ setlimit tomark p1 for ([substring])
+ among(
+ 'ig' 'lig' 'elig' 'els'
+ (delete do consonant_pair)
+ 'l{o/}st'
+ (<-'l{o/}s')
+ )
+ )
+ define undouble as (
+ setlimit tomark p1 for ([c] ->ch)
+ ch
+ delete
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ backwards (
+ do main_suffix
+ do consonant_pair
+ do other_suffix
+ do undouble
+ )
+)
diff --git a/contrib/snowball/algorithms/dutch.sbl b/contrib/snowball/algorithms/dutch.sbl
new file mode 100644
index 0000000..f24c82d
--- /dev/null
+++ b/contrib/snowball/algorithms/dutch.sbl
@@ -0,0 +1,164 @@
+routines (
+ prelude postlude
+ e_ending
+ en_ending
+ mark_regions
+ R1 R2
+ undouble
+ standard_suffix
+)
+
+externals ( stem )
+
+booleans ( e_found )
+
+integers ( p1 p2 )
+
+groupings ( v v_I v_j )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef e" '{U+00EB}'
+stringdef i" '{U+00EF}'
+stringdef o" '{U+00F6}'
+stringdef u" '{U+00FC}'
+
+stringdef a' '{U+00E1}'
+stringdef e' '{U+00E9}'
+stringdef i' '{U+00ED}'
+stringdef o' '{U+00F3}'
+stringdef u' '{U+00FA}'
+
+stringdef e` '{U+00E8}'
+
+define v 'aeiouy{e`}'
+define v_I v + 'I'
+define v_j v + 'j'
+
+define prelude as (
+ test repeat (
+ [substring] among(
+ '{a"}' '{a'}'
+ (<- 'a')
+ '{e"}' '{e'}'
+ (<- 'e')
+ '{i"}' '{i'}'
+ (<- 'i')
+ '{o"}' '{o'}'
+ (<- 'o')
+ '{u"}' '{u'}'
+ (<- 'u')
+ '' (next)
+ ) //or next
+ )
+ try(['y'] <- 'Y')
+ repeat goto (
+ v [('i'] v <- 'I') or
+ ('y'] <- 'Y')
+ )
+)
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ gopast v gopast non-v setmark p1
+ try($p1 < 3 $p1 = 3) // at least 3
+ gopast v gopast non-v setmark p2
+
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'Y' (<- 'y')
+ 'I' (<- 'i')
+ '' (next)
+ ) //or next
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define undouble as (
+ test among('kk' 'dd' 'tt') [next] delete
+ )
+
+ define e_ending as (
+ unset e_found
+ ['e'] R1 test non-v delete
+ set e_found
+ undouble
+ )
+
+ define en_ending as (
+ R1 non-v and not 'gem' delete
+ undouble
+ )
+
+ define standard_suffix as (
+ do (
+ [substring] among(
+ 'heden'
+ ( R1 <- 'heid'
+ )
+ 'en' 'ene'
+ ( en_ending
+ )
+ 's' 'se'
+ ( R1 non-v_j delete
+ )
+ )
+ )
+ do e_ending
+
+ do ( ['heid'] R2 not 'c' delete
+ ['en'] en_ending
+ )
+
+ do (
+ [substring] among(
+ 'end' 'ing'
+ ( R2 delete
+ (['ig'] R2 not 'e' delete) or undouble
+ )
+ 'ig'
+ ( R2 not 'e' delete
+ )
+ 'lijk'
+ ( R2 delete e_ending
+ )
+ 'baar'
+ ( R2 delete
+ )
+ 'bar'
+ ( R2 e_found delete
+ )
+ )
+ )
+ do (
+ non-v_I
+ test (
+ among ('aa' 'ee' 'oo' 'uu')
+ non-v
+ )
+ [next] delete
+ )
+ )
+)
+
+define stem as (
+
+ do prelude
+ do mark_regions
+ backwards
+ do standard_suffix
+ do postlude
+)
diff --git a/contrib/snowball/algorithms/english.sbl b/contrib/snowball/algorithms/english.sbl
new file mode 100644
index 0000000..fe18d7a
--- /dev/null
+++ b/contrib/snowball/algorithms/english.sbl
@@ -0,0 +1,229 @@
+integers ( p1 p2 )
+booleans ( Y_found )
+
+routines (
+ prelude postlude
+ mark_regions
+ shortv
+ R1 R2
+ Step_1a Step_1b Step_1c Step_2 Step_3 Step_4 Step_5
+ exception1
+ exception2
+)
+
+externals ( stem )
+
+groupings ( v v_WXY valid_LI )
+
+stringescapes {}
+
+define v 'aeiouy'
+define v_WXY v + 'wxY'
+
+define valid_LI 'cdeghkmnrt'
+
+define prelude as (
+ unset Y_found
+ do ( ['{'}'] delete)
+ do ( ['y'] <-'Y' set Y_found)
+ do repeat(goto (v ['y']) <-'Y' set Y_found)
+)
+
+define mark_regions as (
+ $p1 = limit
+ $p2 = limit
+ do(
+ among (
+ 'gener'
+ 'commun' // added May 2005
+ 'arsen' // added Nov 2006 (arsenic/arsenal)
+ // ... extensions possible here ...
+ ) or (gopast v gopast non-v)
+ setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+backwardmode (
+
+ define shortv as (
+ ( non-v_WXY v non-v )
+ or
+ ( non-v v atlimit )
+ )
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define Step_1a as (
+ try (
+ [substring] among (
+ '{'}' '{'}s' '{'}s{'}'
+ (delete)
+ )
+ )
+ [substring] among (
+ 'sses' (<-'ss')
+ 'ied' 'ies'
+ ((hop 2 <-'i') or <-'ie')
+ 's' (next gopast v delete)
+ 'us' 'ss'
+ )
+ )
+
+ define Step_1b as (
+ [substring] among (
+ 'eed' 'eedly'
+ (R1 <-'ee')
+ 'ed' 'edly' 'ing' 'ingly'
+ (
+ test gopast v delete
+ test substring among(
+ 'at' 'bl' 'iz'
+ (<+ 'e')
+ 'bb' 'dd' 'ff' 'gg' 'mm' 'nn' 'pp' 'rr' 'tt'
+ // ignoring double c, h, j, k, q, v, w, and x
+ ([next] delete)
+ '' (atmark p1 test shortv <+ 'e')
+ )
+ )
+ )
+ )
+
+ define Step_1c as (
+ ['y' or 'Y']
+ non-v not atlimit
+ <-'i'
+ )
+
+ define Step_2 as (
+ [substring] R1 among (
+ 'tional' (<-'tion')
+ 'enci' (<-'ence')
+ 'anci' (<-'ance')
+ 'abli' (<-'able')
+ 'entli' (<-'ent')
+ 'izer' 'ization'
+ (<-'ize')
+ 'ational' 'ation' 'ator'
+ (<-'ate')
+ 'alism' 'aliti' 'alli'
+ (<-'al')
+ 'fulness' (<-'ful')
+ 'ousli' 'ousness'
+ (<-'ous')
+ 'iveness' 'iviti'
+ (<-'ive')
+ 'biliti' 'bli'
+ (<-'ble')
+ 'ogi' ('l' <-'og')
+ 'fulli' (<-'ful')
+ 'lessli' (<-'less')
+ 'li' (valid_LI delete)
+ )
+ )
+
+ define Step_3 as (
+ [substring] R1 among (
+ 'tional' (<- 'tion')
+ 'ational' (<- 'ate')
+ 'alize' (<-'al')
+ 'icate' 'iciti' 'ical'
+ (<-'ic')
+ 'ful' 'ness'
+ (delete)
+ 'ative'
+ (R2 delete) // 'R2' added Dec 2001
+ )
+ )
+
+ define Step_4 as (
+ [substring] R2 among (
+ 'al' 'ance' 'ence' 'er' 'ic' 'able' 'ible' 'ant' 'ement'
+ 'ment' 'ent' 'ism' 'ate' 'iti' 'ous' 'ive' 'ize'
+ (delete)
+ 'ion' ('s' or 't' delete)
+ )
+ )
+
+ define Step_5 as (
+ [substring] among (
+ 'e' (R2 or (R1 not shortv) delete)
+ 'l' (R2 'l' delete)
+ )
+ )
+
+ define exception2 as (
+
+ [substring] atlimit among(
+ 'inning' 'outing' 'canning' 'herring' 'earring'
+ 'proceed' 'exceed' 'succeed'
+
+ // ... extensions possible here ...
+
+ )
+ )
+)
+
+define exception1 as (
+
+ [substring] atlimit among(
+
+ /* special changes: */
+
+ 'skis' (<-'ski')
+ 'skies' (<-'sky')
+ 'dying' (<-'die')
+ 'lying' (<-'lie')
+ 'tying' (<-'tie')
+
+ /* special -LY cases */
+
+ 'idly' (<-'idl')
+ 'gently' (<-'gentl')
+ 'ugly' (<-'ugli')
+ 'early' (<-'earli')
+ 'only' (<-'onli')
+ 'singly' (<-'singl')
+
+ // ... extensions possible here ...
+
+ /* invariant forms: */
+
+ 'sky'
+ 'news'
+ 'howe'
+
+ 'atlas' 'cosmos' 'bias' 'andes' // not plural forms
+
+ // ... extensions possible here ...
+ )
+)
+
+define postlude as (Y_found repeat(goto (['Y']) <-'y'))
+
+define stem as (
+
+ exception1 or
+ not hop 3 or (
+ do prelude
+ do mark_regions
+ backwards (
+
+ do Step_1a
+
+ exception2 or (
+
+ do Step_1b
+ do Step_1c
+
+ do Step_2
+ do Step_3
+ do Step_4
+
+ do Step_5
+ )
+ )
+ do postlude
+ )
+)
diff --git a/contrib/snowball/algorithms/finnish.sbl b/contrib/snowball/algorithms/finnish.sbl
new file mode 100644
index 0000000..3891d22
--- /dev/null
+++ b/contrib/snowball/algorithms/finnish.sbl
@@ -0,0 +1,197 @@
+
+/* Finnish stemmer.
+
+ Numbers in square brackets refer to the sections in
+ Fred Karlsson, Finnish: An Essential Grammar. Routledge, 1999
+ ISBN 0-415-20705-3
+
+*/
+
+routines (
+ mark_regions
+ R2
+ particle_etc possessive
+ LONG VI
+ case_ending
+ i_plural
+ t_plural
+ other_endings
+ tidy
+)
+
+externals ( stem )
+
+integers ( p1 p2 )
+strings ( x )
+booleans ( ending_removed )
+groupings ( AEI C V1 V2 particle_end )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef o" '{U+00F6}'
+
+define AEI 'a{a"}ei'
+define C 'bcdfghjklmnpqrstvwxz'
+define V1 'aeiouy{a"}{o"}'
+define V2 'aeiou{a"}{o"}'
+define particle_end V1 + 'nt'
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ goto V1 gopast non-V1 setmark p1
+ goto V1 gopast non-V1 setmark p2
+)
+
+backwardmode (
+
+ define R2 as $p2 <= cursor
+
+ define particle_etc as (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'kin'
+ 'kaan' 'k{a"}{a"}n'
+ 'ko' 'k{o"}'
+ 'han' 'h{a"}n'
+ 'pa' 'p{a"}' // Particles [91]
+ (particle_end)
+ 'sti' // Adverb [87]
+ (R2)
+ )
+ delete
+ )
+ define possessive as ( // [36]
+ setlimit tomark p1 for ([substring])
+ among(
+ 'si'
+ (not 'k' delete) // take 'ksi' as the Comitative case
+ 'ni'
+ (delete ['kse'] <- 'ksi') // kseni = ksi + ni
+ 'nsa' 'ns{a"}'
+ 'mme'
+ 'nne'
+ (delete)
+ /* Now for Vn possessives after case endings: [36] */
+ 'an'
+ (among('ta' 'ssa' 'sta' 'lla' 'lta' 'na') delete)
+ '{a"}n'
+ (among('t{a"}' 'ss{a"}' 'st{a"}'
+ 'll{a"}' 'lt{a"}' 'n{a"}') delete)
+ 'en'
+ (among('lle' 'ine') delete)
+ )
+ )
+
+ define LONG as
+ among('aa' 'ee' 'ii' 'oo' 'uu' '{a"}{a"}' '{o"}{o"}')
+
+ define VI as ('i' V2)
+
+ define case_ending as (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'han' ('a') //-.
+ 'hen' ('e') // |
+ 'hin' ('i') // |
+ 'hon' ('o') // |
+ 'h{a"}n' ('{a"}') // Illative [43]
+ 'h{o"}n' ('{o"}') // |
+ 'siin' VI // |
+ 'seen' LONG //-'
+
+ 'den' VI
+ 'tten' VI // Genitive plurals [34]
+ ()
+ 'n' // Genitive or Illative
+ ( try ( LONG // Illative
+ or 'ie' // Genitive
+ and next ]
+ )
+ /* otherwise Genitive */
+ )
+
+ 'a' '{a"}' //-.
+ (V1 C) // |
+ 'tta' 'tt{a"}' // Partitive [32]
+ ('e') // |
+ 'ta' 't{a"}' //-'
+
+ 'ssa' 'ss{a"}' // Inessive [41]
+ 'sta' 'st{a"}' // Elative [42]
+
+ 'lla' 'll{a"}' // Adessive [44]
+ 'lta' 'lt{a"}' // Ablative [51]
+ 'lle' // Allative [46]
+ 'na' 'n{a"}' // Essive [49]
+ 'ksi' // Translative[50]
+ 'ine' // Comitative [51]
+
+ /* Abessive and Instructive are too rare for
+ inclusion [51] */
+
+ )
+ delete
+ set ending_removed
+ )
+ define other_endings as (
+ setlimit tomark p2 for ([substring])
+ among(
+ 'mpi' 'mpa' 'mp{a"}'
+ 'mmi' 'mma' 'mm{a"}' // Comparative forms [85]
+ (not 'po') //-improves things
+ 'impi' 'impa' 'imp{a"}'
+ 'immi' 'imma' 'imm{a"}' // Superlative forms [86]
+ 'eja' 'ej{a"}' // indicates agent [93.1B]
+ )
+ delete
+ )
+ define i_plural as ( // [26]
+ setlimit tomark p1 for ([substring])
+ among(
+ 'i' 'j'
+ )
+ delete
+ )
+ define t_plural as ( // [26]
+ setlimit tomark p1 for (
+ ['t'] test V1
+ delete
+ )
+ setlimit tomark p2 for ([substring])
+ among(
+ 'mma' (not 'po') //-mmat endings
+ 'imma' //-immat endings
+ )
+ delete
+ )
+ define tidy as (
+ setlimit tomark p1 for (
+ do ( LONG and ([next] delete ) ) // undouble vowel
+ do ( [AEI] C delete ) // remove trailing a, a", e, i
+ do ( ['j'] 'o' or 'u' delete )
+ do ( ['o'] 'j' delete )
+ )
+ goto non-V1 [C] -> x x delete // undouble consonant
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ unset ending_removed
+ backwards (
+ do particle_etc
+ do possessive
+ do case_ending
+ do other_endings
+ (ending_removed do i_plural) or do t_plural
+ do tidy
+ )
+)
+
diff --git a/contrib/snowball/algorithms/french.sbl b/contrib/snowball/algorithms/french.sbl
new file mode 100644
index 0000000..5c4f32d
--- /dev/null
+++ b/contrib/snowball/algorithms/french.sbl
@@ -0,0 +1,254 @@
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ standard_suffix
+ i_verb_suffix
+ verb_suffix
+ residual_suffix
+ un_double
+ un_accent
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v keep_with_s )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a^ '{U+00E2}' // a-circumflex
+stringdef a` '{U+00E0}' // a-grave
+stringdef c, '{U+00E7}' // c-cedilla
+
+stringdef e" '{U+00EB}' // e-diaeresis (rare)
+stringdef e' '{U+00E9}' // e-acute
+stringdef e^ '{U+00EA}' // e-circumflex
+stringdef e` '{U+00E8}' // e-grave
+stringdef i" '{U+00EF}' // i-diaeresis
+stringdef i^ '{U+00EE}' // i-circumflex
+stringdef o^ '{U+00F4}' // o-circumflex
+stringdef u^ '{U+00FB}' // u-circumflex
+stringdef u` '{U+00F9}' // u-grave
+
+define v 'aeiouy{a^}{a`}{e"}{e'}{e^}{e`}{i"}{i^}{o^}{u^}{u`}'
+
+define prelude as repeat goto (
+
+ ( v [ ('u' ] v <- 'U') or
+ ('i' ] v <- 'I') or
+ ('y' ] <- 'Y')
+ )
+ or
+ ( [ '{e"}' ] <- 'He' )
+ or
+ ( [ '{i"}' ] <- 'Hi' )
+ or
+ ( ['y'] v <- 'Y' )
+ or
+ ( 'q' ['u'] <- 'U' )
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v v next )
+ or
+ among ( // this exception list begun Nov 2006
+ 'par' // paris, parie, pari
+ 'col' // colis
+ 'tap' // tapis
+ // extensions possible here
+ )
+ or
+ ( next gopast v )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'I' (<- 'i')
+ 'U' (<- 'u')
+ 'Y' (<- 'y')
+ 'He' (<- '{e"}')
+ 'Hi' (<- '{i"}')
+ 'H' (delete)
+ '' (next)
+ )
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'ance' 'iqUe' 'isme' 'able' 'iste' 'eux'
+ 'ances' 'iqUes' 'ismes' 'ables' 'istes'
+ ( R2 delete )
+ 'atrice' 'ateur' 'ation'
+ 'atrices' 'ateurs' 'ations'
+ ( R2 delete
+ try ( ['ic'] (R2 delete) or <-'iqU' )
+ )
+ 'logie'
+ 'logies'
+ ( R2 <- 'log' )
+ 'usion' 'ution'
+ 'usions' 'utions'
+ ( R2 <- 'u' )
+ 'ence'
+ 'ences'
+ ( R2 <- 'ent' )
+ 'ement'
+ 'ements'
+ (
+ RV delete
+ try (
+ [substring] among(
+ 'iv' (R2 delete ['at'] R2 delete)
+ 'eus' ((R2 delete) or (R1<-'eux'))
+ 'abl' 'iqU'
+ (R2 delete)
+ 'i{e`}r' 'I{e`}r' //)
+ (RV <-'i') //)--new 2 Sept 02
+ )
+ )
+ )
+ 'it{e'}'
+ 'it{e'}s'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil' ((R2 delete) or <-'abl')
+ 'ic' ((R2 delete) or <-'iqU')
+ 'iv' (R2 delete)
+ )
+ )
+ )
+ 'if' 'ive'
+ 'ifs' 'ives'
+ (
+ R2 delete
+ try ( ['at'] R2 delete ['ic'] (R2 delete) or <-'iqU' )
+ )
+ 'eaux' (<- 'eau')
+ 'aux' (R1 <- 'al')
+ 'euse'
+ 'euses'((R2 delete) or (R1<-'eux'))
+
+ 'issement'
+ 'issements'(R1 non-v delete) // verbal
+
+ // fail(...) below forces entry to verb_suffix. -ment typically
+ // follows the p.p., e.g 'confus{e'}ment'.
+
+ 'amment' (RV fail(<- 'ant'))
+ 'emment' (RV fail(<- 'ent'))
+ 'ment'
+ 'ments' (test(v RV) fail(delete))
+ // v is e,i,u,{e'},I or U
+ )
+ )
+
+ define i_verb_suffix as setlimit tomark pV for (
+ [substring] among (
+ '{i^}mes' '{i^}t' '{i^}tes' 'i' 'ie' 'ies' 'ir' 'ira' 'irai'
+ 'iraIent' 'irais' 'irait' 'iras' 'irent' 'irez' 'iriez'
+ 'irions' 'irons' 'iront' 'is' 'issaIent' 'issais' 'issait'
+ 'issant' 'issante' 'issantes' 'issants' 'isse' 'issent' 'isses'
+ 'issez' 'issiez' 'issions' 'issons' 'it'
+ (not 'H' non-v delete)
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among (
+ 'ions'
+ (R2 delete)
+
+ '{e'}' '{e'}e' '{e'}es' '{e'}s' '{e`}rent' 'er' 'era' 'erai'
+ 'eraIent' 'erais' 'erait' 'eras' 'erez' 'eriez' 'erions'
+ 'erons' 'eront' 'ez' 'iez'
+
+ // 'ons' //-best omitted
+
+ (delete)
+
+ '{a^}mes' '{a^}t' '{a^}tes' 'a' 'ai' 'aIent' 'ais' 'ait' 'ant'
+ 'ante' 'antes' 'ants' 'as' 'asse' 'assent' 'asses' 'assiez'
+ 'assions'
+ (delete
+ try(['e'] delete)
+ )
+ )
+ )
+
+ define keep_with_s 'aiou{e`}s'
+
+ define residual_suffix as (
+ try(['s'] test ('Hi' or non-keep_with_s) delete)
+ setlimit tomark pV for (
+ [substring] among(
+ 'ion' (R2 's' or 't' delete)
+ 'ier' 'i{e`}re'
+ 'Ier' 'I{e`}re' (<-'i')
+ 'e' (delete)
+ )
+ )
+ )
+
+ define un_double as (
+ test among('enn' 'onn' 'ett' 'ell' 'eill') [next] delete
+ )
+
+ define un_accent as (
+ atleast 1 non-v
+ [ '{e'}' or '{e`}' ] <-'e'
+ )
+)
+
+define stem as (
+
+ do prelude
+ do mark_regions
+ backwards (
+
+ do (
+ (
+ ( standard_suffix or
+ i_verb_suffix or
+ verb_suffix
+ )
+ and
+ try( [ ('Y' ] <- 'i' ) or
+ ('{c,}'] <- 'c' )
+ )
+ ) or
+ residual_suffix
+ )
+
+ // try(['ent'] RV delete) // is best omitted
+
+ do un_double
+ do un_accent
+ )
+ do postlude
+)
+
diff --git a/contrib/snowball/algorithms/german.sbl b/contrib/snowball/algorithms/german.sbl
new file mode 100644
index 0000000..61f24ef
--- /dev/null
+++ b/contrib/snowball/algorithms/german.sbl
@@ -0,0 +1,139 @@
+
+/*
+ Extra rule for -nisse ending added 11 Dec 2009
+*/
+
+routines (
+ prelude postlude
+ mark_regions
+ R1 R2
+ standard_suffix
+)
+
+externals ( stem )
+
+integers ( p1 p2 x )
+
+groupings ( v s_ending st_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef o" '{U+00F6}'
+stringdef u" '{U+00FC}'
+stringdef ss '{U+00DF}'
+
+define v 'aeiouy{a"}{o"}{u"}'
+
+define s_ending 'bdfghklmnrt'
+define st_ending s_ending - 'r'
+
+define prelude as (
+
+ test repeat (
+ (
+ ['{ss}'] <- 'ss'
+ ) or next
+ )
+
+ repeat goto (
+ v [('u'] v <- 'U') or
+ ('y'] v <- 'Y')
+ )
+)
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ test(hop 3 setmark x)
+
+ gopast v gopast non-v setmark p1
+ try($p1 < x $p1 = x) // at least 3
+ gopast v gopast non-v setmark p2
+
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'Y' (<- 'y')
+ 'U' (<- 'u')
+ '{a"}' (<- 'a')
+ '{o"}' (<- 'o')
+ '{u"}' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ do (
+ [substring] R1 among(
+ 'em' 'ern' 'er'
+ ( delete
+ )
+ 'e' 'en' 'es'
+ ( delete
+ try (['s'] 'nis' delete)
+ )
+ 's'
+ ( s_ending delete
+ )
+ )
+ )
+ do (
+ [substring] R1 among(
+ 'en' 'er' 'est'
+ ( delete
+ )
+ 'st'
+ ( st_ending hop 3 delete
+ )
+ )
+ )
+ do (
+ [substring] R2 among(
+ 'end' 'ung'
+ ( delete
+ try (['ig'] not 'e' R2 delete)
+ )
+ 'ig' 'ik' 'isch'
+ ( not 'e' delete
+ )
+ 'lich' 'heit'
+ ( delete
+ try (
+ ['er' or 'en'] R1 delete
+ )
+ )
+ 'keit'
+ ( delete
+ try (
+ [substring] R2 among(
+ 'lich' 'ig'
+ ( delete
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards
+ do standard_suffix
+ do postlude
+)
diff --git a/contrib/snowball/algorithms/german2.sbl b/contrib/snowball/algorithms/german2.sbl
new file mode 100644
index 0000000..47ff61e
--- /dev/null
+++ b/contrib/snowball/algorithms/german2.sbl
@@ -0,0 +1,145 @@
+
+/*
+ Extra rule for -nisse ending added 11 Dec 2009
+*/
+
+routines (
+ prelude postlude
+ mark_regions
+ R1 R2
+ standard_suffix
+)
+
+externals ( stem )
+
+integers ( p1 p2 x )
+
+groupings ( v s_ending st_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef o" '{U+00F6}'
+stringdef u" '{U+00FC}'
+stringdef ss '{U+00DF}'
+
+define v 'aeiouy{a"}{o"}{u"}'
+
+define s_ending 'bdfghklmnrt'
+define st_ending s_ending - 'r'
+
+define prelude as (
+
+ test repeat goto (
+ v [('u'] v <- 'U') or
+ ('y'] v <- 'Y')
+ )
+
+ repeat (
+ [substring] among(
+ '{ss}' (<- 'ss')
+ 'ae' (<- '{a"}')
+ 'oe' (<- '{o"}')
+ 'ue' (<- '{u"}')
+ 'qu' (hop 2)
+ '' (next)
+ )
+ )
+
+)
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ test(hop 3 setmark x)
+
+ gopast v gopast non-v setmark p1
+ try($p1 < x $p1 = x) // at least 3
+ gopast v gopast non-v setmark p2
+
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'Y' (<- 'y')
+ 'U' (<- 'u')
+ '{a"}' (<- 'a')
+ '{o"}' (<- 'o')
+ '{u"}' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ do (
+ [substring] R1 among(
+ 'em' 'ern' 'er'
+ ( delete
+ )
+ 'e' 'en' 'es'
+ ( delete
+ try (['s'] 'nis' delete)
+ )
+ 's'
+ ( s_ending delete
+ )
+ )
+ )
+ do (
+ [substring] R1 among(
+ 'en' 'er' 'est'
+ ( delete
+ )
+ 'st'
+ ( st_ending hop 3 delete
+ )
+ )
+ )
+ do (
+ [substring] R2 among(
+ 'end' 'ung'
+ ( delete
+ try (['ig'] not 'e' R2 delete)
+ )
+ 'ig' 'ik' 'isch'
+ ( not 'e' delete
+ )
+ 'lich' 'heit'
+ ( delete
+ try (
+ ['er' or 'en'] R1 delete
+ )
+ )
+ 'keit'
+ ( delete
+ try (
+ [substring] R2 among(
+ 'lich' 'ig'
+ ( delete
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards
+ do standard_suffix
+ do postlude
+)
diff --git a/contrib/snowball/algorithms/greek.sbl b/contrib/snowball/algorithms/greek.sbl
new file mode 100644
index 0000000..02df6c3
--- /dev/null
+++ b/contrib/snowball/algorithms/greek.sbl
@@ -0,0 +1,706 @@
+// A stemmer for Modern Greek language, based on:
+//
+// Ntais, Georgios. Development of a Stemmer for the Greek
+// Language. Diss. Royal Institute of Technology, 2006.
+// https://sais.se/mthprize/2007/ntais2007.pdf
+//
+// Saroukos, Spyridon. Enhancing a Greek language stemmer.
+// University of Tampere, 2008.
+// https://tampub.uta.fi/bitstream/handle/10024/80480/gradu03463.pdf
+
+stringescapes {}
+
+stringdef a '{U+03B1}' // alpha
+stringdef v '{U+03B2}' // beta
+stringdef g '{U+03B3}' // gamma
+stringdef d '{U+03B4}' // delta
+stringdef e '{U+03B5}' // epsilon
+stringdef z '{U+03B6}' // zeta
+stringdef i '{U+03B7}' // eta
+stringdef th '{U+03B8}' // theta
+stringdef y '{U+03B9}' // iota
+stringdef k '{U+03BA}' // kappa
+stringdef l '{U+03BB}' // lamda
+stringdef m '{U+03BC}' // mu
+stringdef n '{U+03BD}' // nu
+stringdef x '{U+03BE}' // xi
+stringdef o '{U+03BF}' // omicron
+stringdef p '{U+03C0}' // pi
+stringdef r '{U+03C1}' // rho
+stringdef ss '{U+03C2}' // sigma final
+stringdef s '{U+03C3}' // sigma
+stringdef t '{U+03C4}' // tau
+stringdef u '{U+03C5}' // upsilon
+stringdef f '{U+03C6}' // phi
+stringdef ch '{U+03C7}' // chi
+stringdef ps '{U+03C8}' // psi
+stringdef oo '{U+03C9}' // omega
+
+stringdef A '{U+0391}' // Alpha
+stringdef V '{U+0392}' // Beta
+stringdef G '{U+0393}' // Gamma
+stringdef D '{U+0394}' // Delta
+stringdef E '{U+0395}' // Epsilon
+stringdef Z '{U+0396}' // Zeta
+stringdef I '{U+0397}' // Eta
+stringdef Th '{U+0398}' // Theta
+stringdef Y '{U+0399}' // Iota
+stringdef K '{U+039A}' // Kappa
+stringdef L '{U+039B}' // Lamda
+stringdef M '{U+039C}' // Mu
+stringdef N '{U+039D}' // Nu
+stringdef X '{U+039E}' // Xi
+stringdef O '{U+039F}' // Omicron
+stringdef P '{U+03A0}' // Pi
+stringdef R '{U+03A1}' // Rho
+stringdef S '{U+03A3}' // Sigma
+stringdef T '{U+03A4}' // Tau
+stringdef U '{U+03A5}' // Upsilon
+stringdef F '{U+03A6}' // Phi
+stringdef Ch '{U+03A7}' // Chi
+stringdef Ps '{U+03A8}' // Psi
+stringdef Oo '{U+03A9}' // Omega
+
+stringdef Y: '{U+03AA}' // Iota with dialytika
+stringdef U: '{U+03AB}' // Upsilon with dialytika
+
+stringdef a' '{U+03AC}' // alpha with tonos
+stringdef e' '{U+03AD}' // epsilon with tonos
+stringdef i' '{U+03AE}' // eta with tonos
+stringdef y' '{U+03AF}' // iota with tonos
+stringdef o' '{U+03CC}' // omicron with tonos
+stringdef u' '{U+03CD}' // upsilon with tonos
+stringdef oo' '{U+03CE}' // omega with tonos
+
+stringdef i:' '{U+0390}' // iota with dialytika and tonos
+stringdef u:' '{U+03B0}' // upsilon with dialytika and tonos
+
+stringdef i: '{U+03CA}' // iota with dialytika
+stringdef u: '{U+03CB}' // upsilon with dialytika
+
+stringdef A' '{U+0386}' // Alpha with tonos
+stringdef E' '{U+0388}' // Epsilon with tonos
+stringdef I' '{U+0389}' // Eta with tonos
+stringdef Y' '{U+038A}' // Iota with tonos
+stringdef O' '{U+038C}' // Omicron with tonos
+stringdef U' '{U+038E}' // Upsilon with tonos
+stringdef OO' '{U+038F}' // Omega with tonos
+
+externals ( stem )
+
+booleans ( test1 )
+
+groupings ( v v2 )
+
+routines ( tolower has_min_length
+ steps1 steps2 steps3 steps4 steps5 steps6 steps7
+ steps8 steps9 steps10
+ step1 step2a step2b step2c step2d step3 step4
+ step5a step5b step5c step5d step5e step5f
+ step5g step5h step5i
+ step5j step5k step5l step5m
+ step6 step7 )
+
+define v '{a}{e}{i}{y}{o}{u}{oo}'
+define v2 '{a}{e}{i}{y}{o}{oo}'
+
+backwardmode (
+ define has_min_length as (
+ $(len >= 3)
+ )
+
+ define tolower as (
+ repeat (
+ [substring] among (
+ '{A}' (<- '{a}')
+ '{V}' (<- '{v}')
+ '{G}' (<- '{g}')
+ '{D}' (<- '{d}')
+ '{E}' (<- '{e}')
+ '{Z}' (<- '{z}')
+ '{I}' (<- '{i}')
+ '{Th}' (<- '{th}')
+ '{Y}' (<- '{y}')
+ '{K}' (<- '{k}')
+ '{L}' (<- '{l}')
+ '{M}' (<- '{m}')
+ '{N}' (<- '{n}')
+ '{X}' (<- '{x}')
+ '{O}' (<- '{o}')
+ '{P}' (<- '{p}')
+ '{R}' (<- '{r}')
+ '{S}' (<- '{s}')
+ '{T}' (<- '{t}')
+ '{U}' (<- '{u}')
+ '{F}' (<- '{f}')
+ '{Ch}' (<- '{ch}')
+ '{Ps}' (<- '{ps}')
+ '{Oo}' (<- '{oo}')
+ '{Y:}' (<- '{y}')
+ '{U:}' (<- '{u}')
+ '{a'}' (<- '{a}')
+ '{e'}' (<- '{e}')
+ '{i'}' (<- '{i}')
+ '{y'}' (<- '{y}')
+ '{o'}' (<- '{o}')
+ '{u'}' (<- '{u}')
+ '{oo'}' (<- '{oo}')
+ '{i:'}' (<- '{i}')
+ '{u:'}' (<- '{u}')
+ '{i:}' (<- '{i}')
+ '{u:}' (<- '{u}')
+ '{A'}' (<- '{a}')
+ '{E'}' (<- '{e}')
+ '{I'}' (<- '{i}')
+ '{Y'}' (<- '{y}')
+ '{O'}' (<- '{o}')
+ '{U'}' (<- '{u}')
+ '{OO'}' (<- '{oo}')
+ '{ss}' (<- '{s}')
+ '' (next)
+ )
+ )
+ )
+
+ define step1 as (
+ [substring] among (
+ '{f}{a}{g}{y}{a}' '{f}{a}{g}{y}{o}{u}' '{f}{a}{g}{y}{oo}{n}' (<- '{f}{a}')
+ '{s}{k}{a}{g}{y}{a}' '{s}{k}{a}{g}{y}{o}{u}' '{s}{k}{a}{g}{y}{oo}{n}' (<- '{s}{k}{a}')
+ '{o}{l}{o}{g}{y}{o}{u}' '{o}{l}{o}{g}{y}{a}' '{o}{l}{o}{g}{y}{oo}{n}' (<- '{o}{l}{o}')
+ '{s}{o}{g}{y}{o}{u}' '{s}{o}{g}{y}{a}' '{s}{o}{g}{y}{oo}{n}' (<- '{s}{o}')
+ '{t}{a}{t}{o}{g}{y}{a}' '{t}{a}{t}{o}{g}{y}{o}{u}' '{t}{a}{t}{o}{g}{y}{oo}{n}' (<- '{t}{a}{t}{o}')
+ '{k}{r}{e}{a}{s}' '{k}{r}{e}{a}{t}{o}{s}' '{k}{r}{e}{a}{t}{a}' '{k}{r}{e}{a}{t}{oo}{n}' (<- '{k}{r}{e}')
+ '{p}{e}{r}{a}{s}' '{p}{e}{r}{a}{t}{o}{s}' '{p}{e}{r}{a}{t}{i}' '{p}{e}{r}{a}{t}{a}' '{p}{e}{r}{a}{t}{oo}{n}' (<- '{p}{e}{r}')
+ '{t}{e}{r}{a}{s}' '{t}{e}{r}{a}{t}{o}{s}' '{t}{e}{r}{a}{t}{a}' '{t}{e}{r}{a}{t}{oo}{n}' (<- '{t}{e}{r}')
+ '{f}{oo}{s}' '{f}{oo}{t}{o}{s}' '{f}{oo}{t}{a}' '{f}{oo}{t}{oo}{n}' (<- '{f}{oo}')
+ '{k}{a}{th}{e}{s}{t}{oo}{s}' '{k}{a}{th}{e}{s}{t}{oo}{t}{o}{s}' '{k}{a}{th}{e}{s}{t}{oo}{t}{a}' '{k}{a}{th}{e}{s}{t}{oo}{t}{oo}{n}' (<- '{k}{a}{th}{e}{s}{t}')
+ '{g}{e}{g}{o}{n}{o}{s}' '{g}{e}{g}{o}{n}{o}{t}{o}{s}' '{g}{e}{g}{o}{n}{o}{t}{a}' '{g}{e}{g}{o}{n}{o}{t}{oo}{n}' (<- '{g}{e}{g}{o}{n}')
+ )
+ unset test1
+ )
+
+ define steps1 as (
+ [substring] among (
+ '{y}{z}{a}' '{y}{z}{e}{s}' '{y}{z}{e}' '{y}{z}{a}{m}{e}' '{y}{z}{a}{t}{e}' '{y}{z}{a}{n}' '{y}{z}{a}{n}{e}' '{y}{z}{oo}' '{y}{z}{e}{y}{s}' '{y}{z}{e}{y}'
+ '{y}{z}{o}{u}{m}{e}' '{y}{z}{e}{t}{e}' '{y}{z}{o}{u}{n}' '{y}{z}{o}{u}{n}{e}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{a}{n}{a}{m}{p}{a}' '{e}{m}{p}{a}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{p}{a}' '{p}{e}{r}{y}{p}{a}' '{a}{th}{r}{o}' '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}'
+ (<- '{y}')
+ )) or
+ ([] substring atlimit among (
+ '{m}{a}{r}{k}' '{k}{o}{r}{n}' '{a}{m}{p}{a}{r}' '{a}{r}{r}' '{v}{a}{th}{u}{r}{y}' '{v}{a}{r}{k}' '{v}' '{v}{o}{l}{v}{o}{r}' '{g}{k}{r}'
+ '{g}{l}{u}{k}{o}{r}' '{g}{l}{u}{k}{u}{r}' '{y}{m}{p}' '{l}' '{l}{o}{u}' '{m}{a}{r}' '{m}' '{p}{r}' '{m}{p}{r}' '{p}{o}{l}{u}{r}' '{p}'
+ '{r}' '{p}{y}{p}{e}{r}{o}{r}'
+ (<- '{y}{z}')
+ ))
+ )
+ )
+ )
+
+ define steps2 as (
+ [substring] among (
+ '{oo}{th}{i}{k}{a}' '{oo}{th}{i}{k}{e}{s}' '{oo}{th}{i}{k}{e}' '{oo}{th}{i}{k}{a}{m}{e}' '{oo}{th}{i}{k}{a}{t}{e}' '{oo}{th}{i}{k}{a}{n}' '{oo}{th}{i}{k}{a}{n}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{l}' '{v}{y}' '{e}{n}' '{u}{ps}' '{l}{y}' '{z}{oo}' '{s}' '{ch}' (<- '{oo}{n}')
+ )
+ )
+ )
+ )
+
+ define steps3 as (
+ [substring] among (
+ '{y}{s}{a}' '{y}{s}{e}{s}' '{y}{s}{e}' '{y}{s}{a}{m}{e}' '{y}{s}{a}{t}{e}' '{y}{s}{a}{n}' '{y}{s}{a}{n}{e}' (
+ delete
+ unset test1
+ ('{y}{s}{a}' atlimit <- '{y}{s}') or
+ ([] substring atlimit among (
+ '{a}{n}{a}{m}{p}{a}' '{a}{th}{r}{o}' '{e}{m}{p}{a}' '{e}{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{e}{p}{e}' '{p}{e}{r}{y}{p}{a}'
+ '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}' '{k}{l}{e}' '{ch}{a}{r}{t}{o}{p}{a}' '{e}{x}{a}{r}{ch}{a}' '{m}{e}{t}{e}{p}{e}' '{a}{p}{o}{k}{l}{e}'
+ '{a}{p}{e}{k}{l}{e}' '{e}{k}{l}{e}' '{p}{e}'
+ (<- '{y}')
+ )) or
+ ([] substring atlimit among (
+ '{a}{n}' '{a}{f}' '{g}{e}' '{g}{y}{g}{a}{n}{t}{o}{a}{f}' '{g}{k}{e}' '{d}{i}{m}{o}{k}{r}{a}{t}' '{k}{o}{m}' '{g}{k}' '{m}' '{p}'
+ '{p}{o}{u}{k}{a}{m}' '{o}{l}{o}' '{l}{a}{r}'
+ (<- '{y}{s}')
+ ))
+ )
+ )
+ )
+
+ define steps4 as (
+ [substring] among (
+ '{y}{s}{oo}' '{y}{s}{e}{y}{s}' '{y}{s}{e}{y}' '{y}{s}{o}{u}{m}{e}' '{y}{s}{e}{t}{e}' '{y}{s}{o}{u}{n}' '{y}{s}{o}{u}{n}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{n}{a}{m}{p}{a}' '{e}{m}{p}{a}' '{e}{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{e}{p}{e}' '{p}{e}{r}{y}{p}{a}' '{a}{th}{r}{o}'
+ '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}' '{k}{l}{e}' '{ch}{a}{r}{t}{o}{p}{a}' '{e}{x}{a}{r}{ch}{a}' '{m}{e}{t}{e}{p}{e}' '{a}{p}{o}{k}{l}{e}' '{a}{p}{e}{k}{l}{e}'
+ '{e}{k}{l}{e}' '{p}{e}'
+ (<- '{y}')
+ )
+ )
+ )
+ )
+
+ define steps5 as (
+ [substring] among (
+ '{y}{s}{t}{o}{s}' '{y}{s}{t}{o}{u}' '{y}{s}{t}{o}' '{y}{s}{t}{e}' '{y}{s}{t}{o}{y}' '{y}{s}{t}{oo}{n}' '{y}{s}{t}{o}{u}{s}' '{y}{s}{t}{i}' '{y}{s}{t}{i}{s}'
+ '{y}{s}{t}{a}' '{y}{s}{t}{e}{s}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{d}{a}{n}{e}' '{s}{u}{n}{a}{th}{r}{o}' '{k}{l}{e}' '{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{a}{s}{e}' '{p}{l}{e}'
+ (<- '{y}')
+ )) or
+ ([] substring atlimit among (
+ '{m}' '{p}' '{a}{p}' '{a}{r}' '{i}{d}' '{k}{t}' '{s}{k}' '{s}{ch}' '{u}{ps}' '{f}{a}' '{ch}{r}' '{ch}{t}' '{a}{k}{t}'
+ '{a}{o}{r}' '{a}{s}{ch}' '{a}{t}{a}' '{a}{ch}{n}' '{a}{ch}{t}' '{g}{e}{m}' '{g}{u}{r}' '{e}{m}{p}' '{e}{u}{p}' '{e}{ch}{th}' '{i}{f}{a}'
+ '{k}{a}{th}' '{k}{a}{k}' '{k}{u}{l}' '{l}{u}{g}' '{m}{a}{k}' '{m}{e}{g}' '{t}{a}{ch}' '{f}{y}{l}' '{ch}{oo}{r}'
+ (<- '{y}{s}{t}')
+ ))
+ )
+ )
+ )
+
+ define steps6 as (
+ [substring] among (
+ '{y}{s}{m}{o}' '{y}{s}{m}{o}{y}' '{y}{s}{m}{o}{s}' '{y}{s}{m}{o}{u}' '{y}{s}{m}{o}{u}{s}' '{y}{s}{m}{oo}{n}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{s}{e}' '{m}{e}{t}{a}{s}{e}' '{m}{y}{k}{r}{o}{s}{e}' '{e}{g}{k}{l}{e}' '{a}{p}{o}{k}{l}{e}'
+ (<- '{y}{s}{m}')
+ )) or
+ ([] substring atlimit among (
+ '{d}{a}{n}{e}' '{a}{n}{t}{y}{d}{a}{n}{e}'
+ (<- '{y}')
+ )) or
+ ([substring] among (
+ '{a}{g}{n}{oo}{s}{t}{y}{k}' (<- '{a}{g}{n}{oo}{s}{t}')
+ '{a}{t}{o}{m}{y}{k}' (<- '{a}{t}{o}{m}')
+ '{g}{n}{oo}{s}{t}{y}{k}' (<- '{g}{n}{oo}{s}{t}')
+ '{e}{th}{n}{y}{k}' (<- '{e}{th}{n}')
+ '{e}{k}{l}{e}{k}{t}{y}{k}' (<- '{e}{k}{l}{e}{k}{t}')
+ '{s}{k}{e}{p}{t}{y}{k}' (<- '{s}{k}{e}{p}{t}')
+ '{t}{o}{p}{y}{k}' (<- '{t}{o}{p}')
+ '{a}{l}{e}{x}{a}{n}{d}{r}{y}{n}' (<- '{a}{l}{e}{x}{a}{n}{d}{r}')
+ '{v}{u}{z}{a}{n}{t}{y}{n}' (<- '{v}{u}{z}{a}{n}{t}')
+ '{th}{e}{a}{t}{r}{y}{n}' (<- '{th}{e}{a}{t}{r}')
+ ))
+ )
+ )
+ )
+
+ define steps7 as (
+ [substring] among (
+ '{a}{r}{a}{k}{y}' '{a}{r}{a}{k}{y}{a}' '{o}{u}{d}{a}{k}{y}' '{o}{u}{d}{a}{k}{y}{a}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{s}' '{ch}'
+ (<- '{a}{r}{a}{k}')
+ )
+ )
+ )
+ )
+
+ define steps8 as (
+ [substring] among (
+ '{a}{k}{y}' '{a}{k}{y}{a}' '{y}{t}{s}{a}' '{y}{t}{s}{a}{s}' '{y}{t}{s}{e}{s}' '{y}{t}{s}{oo}{n}' '{a}{r}{a}{k}{y}' '{a}{r}{a}{k}{y}{a}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{v}{a}{m}{v}' '{v}{r}' '{k}{a}{y}{m}' '{k}{o}{n}' '{k}{o}{r}' '{l}{a}{v}{r}' '{l}{o}{u}{l}' '{m}{e}{r}' '{m}{o}{u}{s}{t}'
+ '{n}{a}{g}{k}{a}{s}' '{p}{l}' '{r}' '{r}{u}' '{s}' '{s}{k}' '{s}{o}{k}' '{s}{p}{a}{n}' '{t}{z}' '{f}{a}{r}{m}' '{ch}' '{k}{a}{p}{a}{k}'
+ '{a}{l}{y}{s}{f}' '{a}{m}{v}{r}' '{a}{n}{th}{r}' '{k}' '{f}{u}{l}' '{k}{a}{t}{r}{a}{p}' '{k}{l}{y}{m}' '{m}{a}{l}' '{s}{l}{o}{v}' '{f}'
+ '{s}{f}' '{t}{s}{e}{ch}{o}{s}{l}{o}{v}'
+ (<- '{a}{k}')
+ )) or
+ ([] substring atlimit among (
+ '{v}' '{v}{a}{l}' '{g}{y}{a}{n}' '{g}{l}' '{z}' '{i}{g}{o}{u}{m}{e}{n}' '{k}{a}{r}{d}' '{k}{o}{n}' '{m}{a}{k}{r}{u}{n}' '{n}{u}{f}'
+ '{p}{a}{t}{e}{r}' '{p}' '{s}{k}' '{t}{o}{s}' '{t}{r}{y}{p}{o}{l}'
+ (<- '{y}{t}{s}')
+ )) or
+ ([] '{k}{o}{r}' <- '{y}{t}{s}')
+ )
+ )
+ )
+
+ define steps9 as (
+ [substring] among (
+ '{y}{d}{y}{o}' '{y}{d}{y}{a}' '{y}{d}{y}{oo}{n}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{a}{y}{f}{n}' '{y}{r}' '{o}{l}{o}' '{ps}{a}{l}' (<- '{y}{d}')
+ )) or
+ ([] substring among (
+ '{e}' '{p}{a}{y}{ch}{n}' (<- '{y}{d}')
+ ))
+ )
+ )
+ )
+
+ define steps10 as (
+ [substring] among (
+ '{y}{s}{k}{o}{s}' '{y}{s}{k}{o}{u}' '{y}{s}{k}{o}' '{y}{s}{k}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{d}' '{y}{v}' '{m}{i}{n}' '{r}' '{f}{r}{a}{g}{k}' '{l}{u}{k}' '{o}{v}{e}{l}'
+ (<- '{y}{s}{k}')
+ )
+ )
+ )
+ )
+
+ define step2a as (
+ [substring] among (
+ '{a}{d}{e}{s}' '{a}{d}{oo}{n}' (delete)
+ )
+ not ([substring] among (
+ '{o}{k}' '{m}{a}{m}' '{m}{a}{n}' '{m}{p}{a}{m}{p}' '{p}{a}{t}{e}{r}' '{g}{y}{a}{g}{y}' '{n}{t}{a}{n}{t}' '{k}{u}{r}' '{th}{e}{y}' '{p}{e}{th}{e}{r}'
+ ))
+ insert '{a}{d}'
+ )
+
+ define step2b as (
+ [substring] among (
+ '{e}{d}{e}{s}' '{e}{d}{oo}{n}' (delete)
+ )
+ [] substring among (
+ '{o}{p}' '{y}{p}' '{e}{m}{p}' '{u}{p}' '{g}{i}{p}' '{d}{a}{p}' '{k}{r}{a}{s}{p}' '{m}{y}{l}' (<- '{e}{d}')
+ )
+ )
+
+ define step2c as (
+ [substring] among (
+ '{o}{u}{d}{e}{s}' '{o}{u}{d}{oo}{n}' (delete)
+ )
+ [] substring among (
+ '{a}{r}{k}' '{k}{a}{l}{y}{a}{k}' '{p}{e}{t}{a}{l}' '{l}{y}{ch}' '{p}{l}{e}{x}' '{s}{k}' '{s}' '{f}{l}' '{f}{r}' '{v}{e}{l}' '{l}{o}{u}{l}' '{ch}{n}'
+ '{s}{p}' '{t}{r}{a}{g}' '{f}{e}' (<- '{o}{u}{d}')
+ )
+ )
+
+ define step2d as (
+ [substring] among (
+ '{e}{oo}{s}' '{e}{oo}{n}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{th}' '{d}' '{e}{l}' '{g}{a}{l}' '{n}' '{p}' '{y}{d}' '{p}{a}{r}' (<- '{e}')
+ )
+ )
+
+ define step3 as (
+ [substring] among (
+ '{y}{a}' '{y}{o}{u}' '{y}{oo}{n}' (delete unset test1)
+ )
+ ([] v <- '{y}')
+ )
+
+ define step4 as (
+ [substring] among (
+ '{y}{k}{a}' '{y}{k}{o}' '{y}{k}{o}{u}' '{y}{k}{oo}{n}' (delete unset test1)
+ )
+ ([] v <- '{y}{k}') or
+ [] substring atlimit among (
+ '{a}{l}' '{a}{d}' '{e}{n}{d}' '{a}{m}{a}{n}' '{a}{m}{m}{o}{ch}{a}{l}' '{i}{th}' '{a}{n}{i}{th}' '{a}{n}{t}{y}{d}' '{f}{u}{s}' '{v}{r}{oo}{m}' '{g}{e}{r}'
+ '{e}{x}{oo}{d}' '{k}{a}{l}{p}' '{k}{a}{l}{l}{y}{n}' '{k}{a}{t}{a}{d}' '{m}{o}{u}{l}' '{m}{p}{a}{n}' '{m}{p}{a}{g}{y}{a}{t}' '{m}{p}{o}{l}' '{m}{p}{o}{s}'
+ '{n}{y}{t}' '{x}{y}{k}' '{s}{u}{n}{o}{m}{i}{l}' '{p}{e}{t}{s}' '{p}{y}{t}{s}' '{p}{y}{k}{a}{n}{t}' '{p}{l}{y}{a}{t}{s}' '{p}{o}{s}{t}{e}{l}{n}' '{p}{r}{oo}{t}{o}{d}'
+ '{s}{e}{r}{t}' '{s}{u}{n}{a}{d}' '{t}{s}{a}{m}' '{u}{p}{o}{d}' '{f}{y}{l}{o}{n}' '{f}{u}{l}{o}{d}' '{ch}{a}{s}'
+ (<- '{y}{k}')
+ )
+ )
+
+ define step5a as (
+ do ('{a}{g}{a}{m}{e}' atlimit <- '{a}{g}{a}{m}')
+ do (
+ [substring] among (
+ '{a}{g}{a}{m}{e}' '{i}{s}{a}{m}{e}' '{o}{u}{s}{a}{m}{e}' '{i}{k}{a}{m}{e}' '{i}{th}{i}{k}{a}{m}{e}' (delete unset test1)
+ )
+ )
+ ['{a}{m}{e}']
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{n}{a}{p}' '{a}{p}{o}{th}' '{a}{p}{o}{k}' '{a}{p}{o}{s}{t}' '{v}{o}{u}{v}' '{x}{e}{th}' '{o}{u}{l}' '{p}{e}{th}' '{p}{y}{k}{r}' '{p}{o}{t}' '{s}{y}{ch}' '{ch}'
+ (<- '{a}{m}')
+ )
+ )
+
+ define step5b as (
+ do (
+ [substring] among (
+ '{a}{g}{a}{n}{e}' '{i}{s}{a}{n}{e}' '{o}{u}{s}{a}{n}{e}' '{y}{o}{n}{t}{a}{n}{e}' '{y}{o}{t}{a}{n}{e}' '{y}{o}{u}{n}{t}{a}{n}{e}' '{o}{n}{t}{a}{n}{e}' '{o}{t}{a}{n}{e}'
+ '{o}{u}{n}{t}{a}{n}{e}' '{i}{k}{a}{n}{e}' '{i}{th}{i}{k}{a}{n}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{t}{r}' '{t}{s}' (<- '{a}{g}{a}{n}')
+ )
+ )
+ )
+ )
+ ['{a}{n}{e}']
+ delete
+ unset test1
+ ([] v2 <- '{a}{n}') or
+ [] substring atlimit among (
+ '{v}{e}{t}{e}{r}' '{v}{o}{u}{l}{k}' '{v}{r}{a}{ch}{m}' '{g}' '{d}{r}{a}{d}{o}{u}{m}'
+ '{th}' '{k}{a}{l}{p}{o}{u}{z}' '{k}{a}{s}{t}{e}{l}' '{k}{o}{r}{m}{o}{r}' '{l}{a}{o}{p}{l}' '{m}{oo}{a}{m}{e}{th}'
+ '{m}' '{m}{o}{u}{s}{o}{u}{l}{m}' '{n}' '{o}{u}{l}' '{p}' '{p}{e}{l}{e}{k}' '{p}{l}' '{p}{o}{l}{y}{s}'
+ '{p}{o}{r}{t}{o}{l}' '{s}{a}{r}{a}{k}{a}{t}{s}' '{s}{o}{u}{l}{t}' '{t}{s}{a}{r}{l}{a}{t}' '{o}{r}{f}'
+ '{t}{s}{y}{g}{g}' '{t}{s}{o}{p}' '{f}{oo}{t}{o}{s}{t}{e}{f}' '{ch}' '{ps}{u}{ch}{o}{p}{l}' '{a}{g}'
+ '{g}{a}{l}' '{g}{e}{r}' '{d}{e}{k}' '{d}{y}{p}{l}' '{a}{m}{e}{r}{y}{k}{a}{n}' '{o}{u}{r}' '{p}{y}{th}'
+ '{p}{o}{u}{r}{y}{t}' '{s}' '{z}{oo}{n}{t}' '{y}{k}' '{k}{a}{s}{t}' '{k}{o}{p}' '{l}{y}{ch}'
+ '{l}{o}{u}{th}{i}{r}' '{m}{a}{y}{n}{t}' '{m}{e}{l}' '{s}{y}{g}' '{s}{p}' '{s}{t}{e}{g}' '{t}{r}{a}{g}'
+ '{t}{s}{a}{g}' '{f}' '{e}{r}' '{a}{d}{a}{p}' '{a}{th}{y}{g}{g}' '{a}{m}{i}{ch}' '{a}{n}{y}{k}'
+ '{a}{n}{o}{r}{g}' '{a}{p}{i}{g}' '{a}{p}{y}{th}' '{a}{t}{s}{y}{g}{g}' '{v}{a}{s}' '{v}{a}{s}{k}'
+ '{v}{a}{th}{u}{g}{a}{l}' '{v}{y}{o}{m}{i}{ch}' '{v}{r}{a}{ch}{u}{k}' '{d}{y}{a}{t}' '{d}{y}{a}{f}' '{e}{n}{o}{r}{g}'
+ '{th}{u}{s}' '{k}{a}{p}{n}{o}{v}{y}{o}{m}{i}{ch}' '{k}{a}{t}{a}{g}{a}{l}' '{k}{l}{y}{v}' '{k}{o}{y}{l}{a}{r}{f}'
+ '{l}{y}{v}' '{m}{e}{g}{l}{o}{v}{y}{o}{m}{i}{ch}' '{m}{y}{k}{r}{o}{v}{y}{o}{m}{i}{ch}' '{n}{t}{a}{v}'
+ '{x}{i}{r}{o}{k}{l}{y}{v}' '{o}{l}{y}{g}{o}{d}{a}{m}' '{o}{l}{o}{g}{a}{l}' '{p}{e}{n}{t}{a}{r}{f}' '{p}{e}{r}{i}{f}'
+ '{p}{e}{r}{y}{t}{r}' '{p}{l}{a}{t}' '{p}{o}{l}{u}{d}{a}{p}' '{p}{o}{l}{u}{m}{i}{ch}' '{s}{t}{e}{f}' '{t}{a}{v}'
+ '{t}{e}{t}' '{u}{p}{e}{r}{i}{f}' '{u}{p}{o}{k}{o}{p}' '{ch}{a}{m}{i}{l}{o}{d}{a}{p}' '{ps}{i}{l}{o}{t}{a}{v}'
+ (<- '{a}{n}')
+ )
+ )
+
+ define step5c as (
+ do (
+ [substring] among (
+ '{i}{s}{e}{t}{e}' (delete unset test1)
+ )
+ )
+ ['{e}{t}{e}']
+ delete
+ unset test1
+ ([] v2 <- '{e}{t}') or
+ ([] substring among (
+ '{o}{d}' '{a}{y}{r}' '{f}{o}{r}' '{t}{a}{th}' '{d}{y}{a}{th}' '{s}{ch}' '{e}{n}{d}' '{e}{u}{r}' '{t}{y}{th}' '{u}{p}{e}{r}{th}'
+ '{r}{a}{th}' '{e}{n}{th}' '{r}{o}{th}' '{s}{th}' '{p}{u}{r}' '{a}{y}{n}' '{s}{u}{n}{d}' '{s}{u}{n}' '{s}{u}{n}{th}' '{ch}{oo}{r}'
+ '{p}{o}{n}' '{v}{r}' '{k}{a}{th}' '{e}{u}{th}' '{e}{k}{th}' '{n}{e}{t}' '{r}{o}{n}' '{a}{r}{k}' '{v}{a}{r}' '{v}{o}{l}' '{oo}{f}{e}{l}'
+ (<- '{e}{t}')
+ )) or
+ [] substring atlimit among (
+ '{a}{v}{a}{r}' '{v}{e}{n}' '{e}{n}{a}{r}' '{a}{v}{r}' '{a}{d}' '{a}{th}' '{a}{n}' '{a}{p}{l}' '{v}{a}{r}{o}{n}' '{n}{t}{r}' '{s}{k}' '{k}{o}{p}'
+ '{m}{p}{o}{r}' '{n}{y}{f}' '{p}{a}{g}' '{p}{a}{r}{a}{k}{a}{l}' '{s}{e}{r}{p}' '{s}{k}{e}{l}' '{s}{u}{r}{f}' '{t}{o}{k}' '{u}' '{d}' '{e}{m}'
+ '{th}{a}{r}{r}' '{th}'
+ (<- '{e}{t}')
+ )
+ )
+
+ define step5d as (
+ [substring] among (
+ '{o}{n}{t}{a}{s}' '{oo}{n}{t}{a}{s}' (
+ delete
+ unset test1
+ ([] '{a}{r}{ch}' atlimit <- '{o}{n}{t}') or
+ ([] '{k}{r}{e}' <- '{oo}{n}{t}')
+ )
+ )
+ )
+
+ define step5e as (
+ [substring] among (
+ '{o}{m}{a}{s}{t}{e}' '{y}{o}{m}{a}{s}{t}{e}' (
+ delete
+ unset test1
+ ([] '{o}{n}' atlimit <- '{o}{m}{a}{s}{t}')
+ )
+ )
+ )
+
+ define step5f as (
+ do (
+ ['{y}{e}{s}{t}{e}']
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{p}' '{a}{p}' '{s}{u}{m}{p}' '{a}{s}{u}{m}{p}' '{a}{k}{a}{t}{a}{p}' '{a}{m}{e}{t}{a}{m}{f}' (<- '{y}{e}{s}{t}')
+ )
+ )
+ ['{e}{s}{t}{e}']
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{l}' '{a}{r}' '{e}{k}{t}{e}{l}' '{z}' '{m}' '{x}' '{p}{a}{r}{a}{k}{a}{l}' '{p}{r}{o}' '{n}{y}{s}'
+ (<- '{y}{e}{s}{t}')
+ )
+ )
+
+ define step5g as (
+ do (
+ [substring] among (
+ '{i}{th}{i}{k}{a}' '{i}{th}{i}{k}{e}{s}' '{i}{th}{i}{k}{e}' (delete unset test1)
+ )
+ )
+ [substring] among (
+ '{i}{k}{a}' '{i}{k}{e}{s}' '{i}{k}{e}' (
+ delete
+ unset test1
+ ([] substring among (
+ '{s}{k}{oo}{l}' '{s}{k}{o}{u}{l}' '{n}{a}{r}{th}' '{s}{f}' '{o}{th}' '{p}{y}{th}' (<- '{i}{k}')
+ )) or
+ ([] substring atlimit among (
+ '{d}{y}{a}{th}' '{th}' '{p}{a}{r}{a}{k}{a}{t}{a}{th}' '{p}{r}{o}{s}{th}' '{s}{u}{n}{th}' (<- '{i}{k}')
+ ))
+ )
+ )
+ )
+
+ define step5h as (
+ [substring] among (
+ '{o}{u}{s}{a}' '{o}{u}{s}{e}{s}' '{o}{u}{s}{e}' (
+ delete
+ unset test1
+ ([] substring among (
+ '{p}{o}{d}{a}{r}' '{v}{l}{e}{p}' '{p}{a}{n}{t}{a}{ch}' '{f}{r}{u}{d}' '{m}{a}{n}{t}{y}{l}' '{m}{a}{l}{l}' '{k}{u}{m}{a}{t}' '{l}{a}{ch}' '{l}{i}{g}'
+ '{f}{a}{g}' '{o}{m}' '{p}{r}{oo}{t}' (<- '{o}{u}{s}')
+
+ )) or
+ ([] substring atlimit among (
+ '{f}{a}{r}{m}{a}{k}' '{ch}{a}{d}' '{a}{g}{k}' '{a}{n}{a}{r}{r}' '{v}{r}{o}{m}' '{e}{k}{l}{y}{p}' '{l}{a}{m}{p}{y}{d}' '{l}{e}{ch}' '{m}' '{p}{a}{t}'
+ '{r}' '{l}' '{m}{e}{d}' '{m}{e}{s}{a}{z}' '{u}{p}{o}{t}{e}{y}{n}' '{a}{m}' '{a}{y}{th}' '{a}{n}{i}{k}' '{d}{e}{s}{p}{o}{z}'
+ '{e}{n}{d}{y}{a}{f}{e}{r}' '{d}{e}' '{d}{e}{u}{t}{e}{r}{e}{u}' '{k}{a}{th}{a}{r}{e}{u}' '{p}{l}{e}' '{t}{s}{a}'
+ (<- '{o}{u}{s}')
+ ))
+ )
+ )
+ )
+
+ define step5i as (
+ [substring] among (
+ '{a}{g}{a}' '{a}{g}{e}{s}' '{a}{g}{e}' (
+ delete
+ unset test1
+ ([] '{k}{o}{l}{l}' <- '{a}{g}') or (
+ not ([substring] among ('{ps}{o}{f}' '{n}{a}{u}{l}{o}{ch}'))
+ ([] substring among (
+ '{o}{f}' '{p}{e}{l}' '{ch}{o}{r}{t}' '{l}{l}' '{s}{f}' '{r}{p}' '{f}{r}' '{p}{r}' '{l}{o}{ch}' '{s}{m}{i}{n}'
+ (<- '{a}{g}')
+ )) or
+ ([] substring atlimit among (
+ '{a}{v}{a}{s}{t}' '{p}{o}{l}{u}{f}' '{a}{d}{i}{f}' '{p}{a}{m}{f}' '{r}' '{a}{s}{p}' '{a}{f}' '{a}{m}{a}{l}' '{a}{m}{a}{l}{l}{y}'
+ '{a}{n}{u}{s}{t}' '{a}{p}{e}{r}' '{a}{s}{p}{a}{r}' '{a}{ch}{a}{r}' '{d}{e}{r}{v}{e}{n}' '{d}{r}{o}{s}{o}{p}' '{x}{e}{f}' '{n}{e}{o}{p}'
+ '{n}{o}{m}{o}{t}' '{o}{l}{o}{p}' '{o}{m}{o}{t}' '{p}{r}{o}{s}{t}' '{p}{r}{o}{s}{oo}{p}{o}{p}' '{s}{u}{m}{p}' '{s}{u}{n}{t}' '{t}' '{u}{p}{o}{t}'
+ '{ch}{a}{r}' '{a}{e}{y}{p}' '{a}{y}{m}{o}{s}{t}' '{a}{n}{u}{p}' '{a}{p}{o}{t}' '{a}{r}{t}{y}{p}' '{d}{y}{a}{t}' '{e}{n}' '{e}{p}{y}{t}'
+ '{k}{r}{o}{k}{a}{l}{o}{p}' '{s}{y}{d}{i}{r}{o}{p}' '{l}' '{n}{a}{u}' '{o}{u}{l}{a}{m}' '{o}{u}{r}' '{p}' '{t}{r}' '{m}'
+ (<- '{a}{g}')
+ ))
+ )
+ )
+ )
+ )
+
+ define step5j as (
+ [substring] among (
+ '{i}{s}{e}' '{i}{s}{o}{u}' '{i}{s}{a}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{n}' '{ch}{e}{r}{s}{o}{n}' '{d}{oo}{d}{e}{k}{a}{n}' '{e}{r}{i}{m}{o}{n}' '{m}{e}{g}{a}{l}{o}{n}' '{e}{p}{t}{a}{n}' (<- '{i}{s}')
+ )
+ )
+
+ define step5k as (
+ [substring] among (
+ '{i}{s}{t}{e}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{a}{s}{v}' '{s}{v}' '{a}{ch}{r}' '{ch}{r}' '{a}{p}{l}' '{a}{e}{y}{m}{n}' '{d}{u}{s}{ch}{r}' '{e}{u}{ch}{r}' '{k}{o}{y}{n}{o}{ch}{r}' '{p}{a}{l}{y}{m}{ps}'
+ (<- '{i}{s}{t}')
+ )
+ )
+
+ define step5l as (
+ [substring] among (
+ '{o}{u}{n}{e}' '{i}{s}{o}{u}{n}{e}' '{i}{th}{o}{u}{n}{e}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{n}' '{r}' '{s}{p}{y}' '{s}{t}{r}{a}{v}{o}{m}{o}{u}{t}{s}' '{k}{a}{k}{o}{m}{o}{u}{t}{s}' '{e}{x}{oo}{n}' (<- '{o}{u}{n}')
+ )
+ )
+
+ define step5m as (
+ [substring] among (
+ '{o}{u}{m}{e}' '{i}{s}{o}{u}{m}{e}' '{i}{th}{o}{u}{m}{e}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{p}{a}{r}{a}{s}{o}{u}{s}' '{f}' '{ch}' '{oo}{r}{y}{o}{p}{l}' '{a}{z}' '{a}{l}{l}{o}{s}{o}{u}{s}' '{a}{s}{o}{u}{s}'
+ (<- '{o}{u}{m}')
+ )
+ )
+
+ define step6 as (
+ do (
+ [substring] among (
+ '{m}{a}{t}{a}' '{m}{a}{t}{oo}{n}' '{m}{a}{t}{o}{s}' (<- '{m}{a}')
+ )
+ )
+ test1
+ [substring] among (
+ '{a}' '{a}{g}{a}{t}{e}' '{a}{g}{a}{n}' '{a}{e}{y}' '{a}{m}{a}{y}' '{a}{n}' '{a}{s}' '{a}{s}{a}{y}' '{a}{t}{a}{y}' '{a}{oo}' '{e}' '{e}{y}'
+ '{e}{y}{s}' '{e}{y}{t}{e}' '{e}{s}{a}{y}' '{e}{s}' '{e}{t}{a}{y}' '{y}' '{y}{e}{m}{a}{y}' '{y}{e}{m}{a}{s}{t}{e}' '{y}{e}{t}{a}{y}' '{y}{e}{s}{a}{y}'
+ '{y}{e}{s}{a}{s}{t}{e}' '{y}{o}{m}{a}{s}{t}{a}{n}' '{y}{o}{m}{o}{u}{n}' '{y}{o}{m}{o}{u}{n}{a}' '{y}{o}{n}{t}{a}{n}' '{y}{o}{n}{t}{o}{u}{s}{a}{n}' '{y}{o}{s}{a}{s}{t}{a}{n}'
+ '{y}{o}{s}{a}{s}{t}{e}' '{y}{o}{s}{o}{u}{n}' '{y}{o}{s}{o}{u}{n}{a}' '{y}{o}{t}{a}{n}' '{y}{o}{u}{m}{a}' '{y}{o}{u}{m}{a}{s}{t}{e}' '{y}{o}{u}{n}{t}{a}{y}'
+ '{y}{o}{u}{n}{t}{a}{n}' '{i}' '{i}{d}{e}{s}' '{i}{d}{oo}{n}' '{i}{th}{e}{y}' '{i}{th}{e}{y}{s}' '{i}{th}{e}{y}{t}{e}' '{i}{th}{i}{k}{a}{t}{e}' '{i}{th}{i}{k}{a}{n}'
+ '{i}{th}{o}{u}{n}' '{i}{th}{oo}' '{i}{k}{a}{t}{e}' '{i}{k}{a}{n}' '{i}{s}' '{i}{s}{a}{n}' '{i}{s}{a}{t}{e}' '{i}{s}{e}{y}' '{i}{s}{e}{s}' '{i}{s}{o}{u}{n}'
+ '{i}{s}{oo}' '{o}' '{o}{y}' '{o}{m}{a}{y}' '{o}{m}{a}{s}{t}{a}{n}' '{o}{m}{o}{u}{n}' '{o}{m}{o}{u}{n}{a}' '{o}{n}{t}{a}{y}' '{o}{n}{t}{a}{n}'
+ '{o}{n}{t}{o}{u}{s}{a}{n}' '{o}{s}' '{o}{s}{a}{s}{t}{a}{n}' '{o}{s}{a}{s}{t}{e}' '{o}{s}{o}{u}{n}' '{o}{s}{o}{u}{n}{a}' '{o}{t}{a}{n}' '{o}{u}' '{o}{u}{m}{a}{y}'
+ '{o}{u}{m}{a}{s}{t}{e}' '{o}{u}{n}' '{o}{u}{n}{t}{a}{y}' '{o}{u}{n}{t}{a}{n}' '{o}{u}{s}' '{o}{u}{s}{a}{n}' '{o}{u}{s}{a}{t}{e}' '{u}' '{u}{s}' '{oo}'
+ '{oo}{n}' (delete)
+ )
+ )
+
+ define step7 as (
+ [substring] among (
+ '{e}{s}{t}{e}{r}' '{e}{s}{t}{a}{t}' '{o}{t}{e}{r}' '{o}{t}{a}{t}' '{u}{t}{e}{r}' '{u}{t}{a}{t}' '{oo}{t}{e}{r}' '{oo}{t}{a}{t}' (delete)
+ )
+ )
+)
+
+define stem as (
+ backwards (
+ do tolower
+ has_min_length
+ set test1
+ do step1
+ do steps1
+ do steps2
+ do steps3
+ do steps4
+ do steps5
+ do steps6
+ do steps7
+ do steps8
+ do steps9
+ do steps10
+ do step2a
+ do step2b
+ do step2c
+ do step2d
+ do step3
+ do step4
+ do step5a
+ do step5b
+ do step5c
+ do step5d
+ do step5e
+ do step5f
+ do step5g
+ do step5h
+ do step5j
+ do step5i
+ do step5k
+ do step5l
+ do step5m
+ do step6
+ do step7
+ )
+)
diff --git a/contrib/snowball/algorithms/hindi.sbl b/contrib/snowball/algorithms/hindi.sbl
new file mode 100644
index 0000000..bfdfac0
--- /dev/null
+++ b/contrib/snowball/algorithms/hindi.sbl
@@ -0,0 +1,323 @@
+// An implementation of "A Lightweight Stemmer for Hindi":
+// http://www.kbcs.in/downloads/papers/StmmerHindi.pdf
+
+externals ( stem )
+
+stringescapes {}
+
+// The transliteration scheme used for our stringdefs matches that used in the
+// paper, as documented in the appendix. It appears to match the WX notation
+// (https://en.wikipedia.org/wiki/WX_notation) except that WX apparently
+// uses 'z' for Anunasika whereas the paper uses Mh.
+//
+// We discriminate dependent vowels by adding a leading "_" to their stringdef
+// names (mnemonic: the _ signifies removing the implicit a from the preceding
+// character).
+
+// Vowels and sonorants:
+stringdef a '{U+0905}'
+stringdef A '{U+0906}'
+stringdef i '{U+0907}'
+stringdef I '{U+0908}'
+stringdef u '{U+0909}'
+stringdef U '{U+090A}'
+stringdef q '{U+090B}'
+stringdef e '{U+090F}'
+stringdef E '{U+0910}'
+stringdef o '{U+0913}'
+stringdef O '{U+0914}'
+
+// Vowel signs:
+stringdef _A '{U+093E}'
+stringdef _i '{U+093F}'
+stringdef _I '{U+0940}'
+stringdef _u '{U+0941}'
+stringdef _U '{U+0942}'
+stringdef _q '{U+0943}'
+stringdef _e '{U+0947}'
+stringdef _E '{U+0948}'
+stringdef _o '{U+094B}'
+stringdef _O '{U+094C}'
+
+// Diacritics:
+stringdef M '{U+0902}'
+stringdef H '{U+0903}'
+stringdef Mh '{U+0901}'
+stringdef Z '{U+093C}' // Nukta
+stringdef virama '{U+094D}'
+
+// Velar consonants:
+stringdef k '{U+0915}'
+stringdef K '{U+0916}'
+stringdef g '{U+0917}'
+stringdef G '{U+0918}'
+stringdef f '{U+0919}'
+
+// Palatal consonants:
+stringdef c '{U+091A}'
+stringdef C '{U+091B}'
+stringdef j '{U+091C}'
+stringdef J '{U+091D}'
+stringdef F '{U+091E}'
+
+// Retroflex consonants:
+stringdef t '{U+091F}'
+stringdef T '{U+0920}'
+stringdef d '{U+0921}'
+stringdef D '{U+0922}'
+stringdef N '{U+0923}'
+
+// Dental consonants:
+stringdef w '{U+0924}'
+stringdef W '{U+0925}'
+stringdef x '{U+0926}'
+stringdef X '{U+0927}'
+stringdef n '{U+0928}'
+
+// Labial consonants:
+stringdef p '{U+092A}'
+stringdef P '{U+092B}'
+stringdef b '{U+092C}'
+stringdef B '{U+092D}'
+stringdef m '{U+092E}'
+
+// Semi-vowels:
+stringdef y '{U+092F}'
+stringdef r '{U+0930}'
+stringdef l '{U+0932}'
+stringdef v '{U+0935}'
+
+// Fricatives:
+stringdef S '{U+0936}'
+stringdef R '{U+0937}'
+stringdef s '{U+0938}'
+stringdef h '{U+0939}'
+
+stringdef lY '{U+0933}'
+
+// Precomposed characters - letters + nukta:
+stringdef nZ '{U+0929}' // ≡ {n}{Z}
+stringdef rZ '{U+0931}' // ≡ {r}{Z}
+stringdef lYZ '{U+0934}' // ≡ {lY}{Z}
+stringdef kZ '{U+0958}' // ≡ {k}{Z}
+stringdef KZ '{U+0959}' // ≡ {K}{Z}
+stringdef gZ '{U+095A}' // ≡ {g}{Z}
+stringdef jZ '{U+095B}' // ≡ {j}{Z}
+stringdef dZ '{U+095C}' // ≡ {d}{Z}
+stringdef DZ '{U+095D}' // ≡ {D}{Z}
+stringdef PZ '{U+095E}' // ≡ {P}{Z}
+stringdef yZ '{U+095F}' // ≡ {y}{Z}
+
+integers ( p )
+
+groupings ( consonant )
+
+routines ( CONSONANT )
+
+define consonant '{k}{K}{g}{G}{f}' +
+ '{c}{C}{j}{J}{F}' +
+ '{t}{T}{d}{D}{N}' +
+ '{w}{W}{x}{X}{n}' +
+ '{p}{P}{b}{B}{m}' +
+ '{y}{r}{l}{v}' +
+ '{S}{R}{s}{h}' +
+ '{lY}' +
+ '{Z}' + // Nukta
+ // Precomposed characters - letter and nukta:
+ '{nZ}{rZ}{lYZ}{kZ}{KZ}{gZ}{jZ}{dZ}{DZ}{PZ}{yZ}'
+
+backwardmode ( define CONSONANT as ( consonant ) )
+
+define stem as (
+ test ( next setmark p )
+ backwards (
+ // We assume in this implementation that the whole word doesn't count
+ // as a valid suffix to remove, so we remove the longest suffix from
+ // the list which leaves at least one character. This change affects
+ // 47 words out of the 65,140 in the sample vocabulary from Hindi
+ // wikipedia.
+ setlimit tomark p for ([substring])
+ among (
+ // The list below is derived from figure 3 in the paper.
+ //
+ // We perform the stemming on the Devanagari characters rather than
+ // transliterating to Latin, so we have adapted the list below to
+ // reflect this by converting suffixes back to Devanagari as
+ // follows:
+ //
+ // * within the suffixes, "a" after a consonant is dropped since
+ // consonants have an implicit "a".
+ //
+ // * within the suffixes, a vowel other than "a" after a consonant
+ // is a dependent vowel (vowel sign); a vowel (including "a")
+ // after a non-consonant is an independent vowel.
+ //
+ // * to allow the vowel at the start of each suffix being dependent
+ // or independent, we include each suffix twice. For the
+ // dependent version, a leading "a" is dropped and we check that
+ // the suffix is preceded by a consonant (which will have an
+ // implicit "a").
+ //
+ // * we add '{a}', which is needed for the example given right at
+ // the end of section 5 to work (conflating BarawIya and
+ // BarawIyawA), and which 3.1 a.v strongly suggests should be in
+ // the list:
+ //
+ // Thus, the following suffix deletions (longest possible
+ // match) are required to reduce inflected forms of masculine
+ // nouns to a common stem:
+ // a A i [...]
+ //
+ // Adding '{a}' only affect 2 words out of the 65,140 in the
+ // sample vocabulary.
+ //
+ // * The transliterations of our stems would end with "a" when our
+ // stems end in a consonant, so we also include {virama} in the
+ // list of suffixes to remove (this affects 222 words from the
+ // sample vocabulary).
+ //
+ // We've also assumed that Mh in the suffix list always means {Mh}
+ // and never {M}{h}{virama}. Only one of the 65,140 words in the
+ // sample vocabulary stems differently due to this (and that word
+ // seems to be a typo).
+
+ '{virama}'
+
+ '{a}'
+ '{A}'
+ '{i}'
+ '{I}'
+ '{u}'
+ '{U}'
+ '{e}'
+ '{o}'
+ '{e}{M}'
+ '{o}{M}'
+ '{A}{M}'
+ '{u}{A}{M}'
+ '{u}{e}{M}'
+ '{u}{o}{M}'
+ '{A}{e}{M}'
+ '{A}{o}{M}'
+ '{i}{y}{_A}{M}'
+ '{i}{y}{_o}{M}'
+ '{A}{i}{y}{_A}{M}'
+ '{A}{i}{y}{_o}{M}'
+ '{A}{Mh}'
+ '{i}{y}{_A}{Mh}'
+ '{A}{i}{y}{_A}{Mh}'
+ '{a}{w}{_A}{e}{M}'
+ '{a}{w}{_A}{o}{M}'
+ '{a}{n}{_A}{e}{M}'
+ '{a}{n}{_A}{o}{M}'
+ '{a}{w}{_A}'
+ '{a}{w}{_I}'
+ '{I}{M}'
+ '{a}{w}{_I}{M}'
+ '{a}{w}{_e}'
+ '{A}{w}{_A}'
+ '{A}{w}{_I}'
+ '{A}{w}{_I}{M}'
+ '{A}{w}{_e}'
+ '{a}{n}{_A}'
+ '{a}{n}{_I}'
+ '{a}{n}{_e}'
+ '{A}{n}{_A}'
+ '{A}{n}{_e}'
+ '{U}{M}{g}{_A}'
+ '{U}{M}{g}{_I}'
+ '{A}{U}{M}{g}{_A}'
+ '{A}{U}{M}{g}{_I}'
+ '{e}{M}{g}{_e}'
+ '{e}{M}{g}{_I}'
+ '{A}{e}{M}{g}{_e}'
+ '{A}{e}{M}{g}{_I}'
+ '{o}{g}{_e}'
+ '{o}{g}{_I}'
+ '{A}{o}{g}{_e}'
+ '{A}{o}{g}{_I}'
+ '{e}{g}{_A}'
+ '{e}{g}{_I}'
+ '{A}{e}{g}{_A}'
+ '{A}{e}{g}{_I}'
+ '{A}{y}{_A}'
+ '{A}{e}'
+ '{A}{I}'
+ '{A}{I}{M}'
+ '{i}{e}'
+ '{A}{o}'
+ '{A}{i}{e}'
+ '{a}{k}{r}'
+ '{A}{k}{r}'
+
+ '{_A}'
+ '{_i}'
+ '{_I}'
+ '{_u}'
+ '{_U}'
+ '{_e}'
+ '{_o}'
+ '{_e}{M}'
+ '{_o}{M}'
+ '{_A}{M}'
+ '{_u}{A}{M}'
+ '{_u}{e}{M}'
+ '{_u}{o}{M}'
+ '{_A}{e}{M}'
+ '{_A}{o}{M}'
+ '{_i}{y}{_A}{M}'
+ '{_i}{y}{_o}{M}'
+ '{_A}{i}{y}{_A}{M}'
+ '{_A}{i}{y}{_o}{M}'
+ '{_A}{Mh}'
+ '{_i}{y}{_A}{Mh}'
+ '{_A}{i}{y}{_A}{Mh}'
+ '{_I}{M}'
+ '{_A}{w}{_A}'
+ '{_A}{w}{_I}'
+ '{_A}{w}{_I}{M}'
+ '{_A}{w}{_e}'
+ '{_A}{n}{_A}'
+ '{_A}{n}{_e}'
+ '{_U}{M}{g}{_A}'
+ '{_U}{M}{g}{_I}'
+ '{_A}{U}{M}{g}{_A}'
+ '{_A}{U}{M}{g}{_I}'
+ '{_e}{M}{g}{_e}'
+ '{_e}{M}{g}{_I}'
+ '{_A}{e}{M}{g}{_e}'
+ '{_A}{e}{M}{g}{_I}'
+ '{_o}{g}{_e}'
+ '{_o}{g}{_I}'
+ '{_A}{o}{g}{_e}'
+ '{_A}{o}{g}{_I}'
+ '{_e}{g}{_A}'
+ '{_e}{g}{_I}'
+ '{_A}{e}{g}{_A}'
+ '{_A}{e}{g}{_I}'
+ '{_A}{y}{_A}'
+ '{_A}{e}'
+ '{_A}{I}'
+ '{_A}{I}{M}'
+ '{_i}{e}'
+ '{_A}{o}'
+ '{_A}{i}{e}'
+ '{_A}{k}{r}'
+
+ /* Suffixes with a leading implicit a: */
+ '{w}{_A}{e}{M}' CONSONANT
+ '{w}{_A}{o}{M}' CONSONANT
+ '{n}{_A}{e}{M}' CONSONANT
+ '{n}{_A}{o}{M}' CONSONANT
+ '{w}{_A}' CONSONANT
+ '{w}{_I}' CONSONANT
+ '{w}{_I}{M}' CONSONANT
+ '{w}{_e}' CONSONANT
+ '{n}{_A}' CONSONANT
+ '{n}{_I}' CONSONANT
+ '{n}{_e}' CONSONANT
+ '{k}{r}' CONSONANT
+ )
+ delete
+ )
+)
diff --git a/contrib/snowball/algorithms/hungarian.sbl b/contrib/snowball/algorithms/hungarian.sbl
new file mode 100644
index 0000000..2d7885c
--- /dev/null
+++ b/contrib/snowball/algorithms/hungarian.sbl
@@ -0,0 +1,241 @@
+/*
+Hungarian Stemmer
+Removes noun inflections
+*/
+
+routines (
+ mark_regions
+ R1
+ v_ending
+ case
+ case_special
+ case_other
+ plural
+ owned
+ sing_owner
+ plur_owner
+ instrum
+ factive
+ undouble
+ double
+)
+
+externals ( stem )
+
+integers ( p1 )
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' //a-acute
+stringdef e' '{U+00E9}' //e-acute
+stringdef i' '{U+00ED}' //i-acute
+stringdef o' '{U+00F3}' //o-acute
+stringdef o" '{U+00F6}' //o-umlaut
+stringdef oq '{U+0151}' //o-double acute
+stringdef u' '{U+00FA}' //u-acute
+stringdef u" '{U+00FC}' //u-umlaut
+stringdef uq '{U+0171}' //u-double acute
+
+define v 'aeiou{a'}{e'}{i'}{o'}{o"}{oq}{u'}{u"}{uq}'
+
+define mark_regions as (
+
+ $p1 = limit
+
+ (v goto non-v
+ among('cs' 'gy' 'ly' 'ny' 'sz' 'ty' 'zs' 'dzs') or next
+ setmark p1)
+ or
+
+ (non-v gopast v setmark p1)
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+
+ define v_ending as (
+ [substring] R1 among(
+ '{a'}' (<- 'a')
+ '{e'}' (<- 'e')
+ )
+ )
+
+ define double as (
+ test among('bb' 'cc' 'ccs' 'dd' 'ff' 'gg' 'ggy' 'jj' 'kk' 'll' 'lly' 'mm'
+ 'nn' 'nny' 'pp' 'rr' 'ss' 'ssz' 'tt' 'tty' 'vv' 'zz' 'zzs')
+ )
+
+ define undouble as (
+ next [hop 1] delete
+ )
+
+ define instrum as(
+ [substring] R1 among(
+ 'al' (double)
+ 'el' (double)
+ )
+ delete
+ undouble
+ )
+
+
+ define case as (
+ [substring] R1 among(
+ 'ban' 'ben'
+ 'ba' 'be'
+ 'ra' 're'
+ 'nak' 'nek'
+ 'val' 'vel'
+ 't{o'}l' 't{oq}l'
+ 'r{o'}l' 'r{oq}l'
+ 'b{o'}l' 'b{oq}l'
+ 'hoz' 'hez' 'h{o"}z'
+ 'n{a'}l' 'n{e'}l'
+ 'ig'
+ 'at' 'et' 'ot' '{o"}t'
+ '{e'}rt'
+ 'k{e'}pp' 'k{e'}ppen'
+ 'kor'
+ 'ul' '{u"}l'
+ 'v{a'}' 'v{e'}'
+ 'onk{e'}nt' 'enk{e'}nt' 'ank{e'}nt'
+ 'k{e'}nt'
+ 'en' 'on' 'an' '{o"}n'
+ 'n'
+ 't'
+ )
+ delete
+ v_ending
+ )
+
+ define case_special as(
+ [substring] R1 among(
+ '{e'}n' (<- 'e')
+ '{a'}n' (<- 'a')
+ '{a'}nk{e'}nt' (<- 'a')
+ )
+ )
+
+ define case_other as(
+ [substring] R1 among(
+ 'astul' 'est{u"}l' (delete)
+ 'stul' 'st{u"}l' (delete)
+ '{a'}stul' (<- 'a')
+ '{e'}st{u"}l' (<- 'e')
+ )
+ )
+
+ define factive as(
+ [substring] R1 among(
+ '{a'}' (double)
+ '{e'}' (double)
+ )
+ delete
+ undouble
+ )
+
+ define plural as (
+ [substring] R1 among(
+ '{a'}k' (<- 'a')
+ '{e'}k' (<- 'e')
+ '{o"}k' (delete)
+ 'ak' (delete)
+ 'ok' (delete)
+ 'ek' (delete)
+ 'k' (delete)
+ )
+ )
+
+ define owned as (
+ [substring] R1 among (
+ 'ok{e'}' '{o"}k{e'}' 'ak{e'}' 'ek{e'}' (delete)
+ '{e'}k{e'}' (<- 'e')
+ '{a'}k{e'}' (<- 'a')
+ 'k{e'}' (delete)
+ '{e'}{e'}i' (<- 'e')
+ '{a'}{e'}i' (<- 'a')
+ '{e'}i' (delete)
+ '{e'}{e'}' (<- 'e')
+ '{e'}' (delete)
+ )
+ )
+
+ define sing_owner as (
+ [substring] R1 among(
+ '{u"}nk' 'unk' (delete)
+ '{a'}nk' (<- 'a')
+ '{e'}nk' (<- 'e')
+ 'nk' (delete)
+ '{a'}juk' (<- 'a')
+ '{e'}j{u"}k' (<- 'e')
+ 'juk' 'j{u"}k' (delete)
+ 'uk' '{u"}k' (delete)
+ 'em' 'om' 'am' (delete)
+ '{a'}m' (<- 'a')
+ '{e'}m' (<- 'e')
+ 'm' (delete)
+ 'od' 'ed' 'ad' '{o"}d' (delete)
+ '{a'}d' (<- 'a')
+ '{e'}d' (<- 'e')
+ 'd' (delete)
+ 'ja' 'je' (delete)
+ 'a' 'e' 'o' (delete)
+ '{a'}' (<- 'a')
+ '{e'}' (<- 'e')
+ )
+ )
+
+ define plur_owner as (
+ [substring] R1 among(
+ 'jaim' 'jeim' (delete)
+ '{a'}im' (<- 'a')
+ '{e'}im' (<- 'e')
+ 'aim' 'eim' (delete)
+ 'im' (delete)
+ 'jaid' 'jeid' (delete)
+ '{a'}id' (<- 'a')
+ '{e'}id' (<- 'e')
+ 'aid' 'eid' (delete)
+ 'id' (delete)
+ 'jai' 'jei' (delete)
+ '{a'}i' (<- 'a')
+ '{e'}i' (<- 'e')
+ 'ai' 'ei' (delete)
+ 'i' (delete)
+ 'jaink' 'jeink' (delete)
+ 'eink' 'aink' (delete)
+ '{a'}ink' (<- 'a')
+ '{e'}ink' (<- 'e')
+ 'ink'
+ 'jaitok' 'jeitek' (delete)
+ 'aitok' 'eitek' (delete)
+ '{a'}itok' (<- 'a')
+ '{e'}itek' (<- 'e')
+ 'itek' (delete)
+ 'jeik' 'jaik' (delete)
+ 'aik' 'eik' (delete)
+ '{a'}ik' (<- 'a')
+ '{e'}ik' (<- 'e')
+ 'ik' (delete)
+ )
+ )
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ do instrum
+ do case
+ do case_special
+ do case_other
+ do factive
+ do owned
+ do sing_owner
+ do plur_owner
+ do plural
+ )
+)
diff --git a/contrib/snowball/algorithms/indonesian.sbl b/contrib/snowball/algorithms/indonesian.sbl
new file mode 100644
index 0000000..ac0ee36
--- /dev/null
+++ b/contrib/snowball/algorithms/indonesian.sbl
@@ -0,0 +1,192 @@
+// An implementation of the "Porter Stemmer for Bahasa Indonesia" from:
+// http://www.illc.uva.nl/Research/Publications/Reports/MoL-2003-02.text.pdf
+
+integers (
+ // The paper defines measure as the number of vowels in the word. We
+ // count this initially, then adjust the count each time we remove a
+ // prefix or suffix.
+ measure
+
+ // Numeric code for the type of prefix removed:
+ //
+ // 0 other/none
+ // 1 'di' or 'meng' or 'ter'
+ // 2 'per'
+ // 3 'ke' or 'peng'
+ // 4 'ber'
+ //
+ // Some of these have variant forms, so e.g. "meng" includes "men", "me",
+ // "meny", "mem".
+ //
+ // Note that the value of prefix is only used in remove_suffix (and
+ // routines it calls) so we don't need to worry about
+ // remove_second_order_prefix overwriting a value of prefix set by
+ // remove_first_order_prefix since remove_suffix gets called between
+ // the two.
+ prefix
+)
+
+groupings ( vowel )
+
+routines (
+ remove_particle
+ remove_possessive_pronoun
+ remove_first_order_prefix
+ remove_second_order_prefix
+ remove_suffix
+ KER
+ SUFFIX_KAN_OK
+ SUFFIX_AN_OK
+ SUFFIX_I_OK
+ VOWEL
+)
+
+externals ( stem )
+
+stringescapes {}
+
+backwardmode (
+
+ define remove_particle as (
+ [substring] among (
+ 'kah' 'lah' 'pun' (delete $measure-=1)
+ )
+ )
+
+ define remove_possessive_pronoun as (
+ [substring] among (
+ 'ku' 'mu' 'nya' (delete $measure-=1)
+ )
+ )
+
+ // prefix not in {ke, peng, per}
+ define SUFFIX_KAN_OK as (
+ // On page 29, the example "kompas Q.31" says "Both Nazief and Porter
+ // stemmer converted the word peledakan (blast, explotion) to ledak (to
+ // blast, to explode)". However, the algorithm as described doesn't
+ // behave in this way - grammatically the prefix pe- occurs as a
+ // variation of both the first-order derivational prefix peng- and the
+ // second-order derivational prefix per-, but table 2.5 doesn't include
+ // "pe", only table 2.6 does, so "peledakan" is handled (incorrectly)
+ // as having prefix "per" not "peng", and so we remove derivational
+ // suffix "kan" rather than "an" to give stem leda. (Porter-style
+ // stemmers remove the longest suffix they can amongst those available,
+ // which this paper notes in the last paragraph on page 15).
+ //
+ // We resolve this by amending the condition on suffix "kan" to
+ // "prefix ∉ {ke, peng, per}", which seems to make the stemmer's
+ // behaviour match all the examples in the paper except for one:
+ // "perbaikan" is shown in table 3.4 as stemming to "bai", but with
+ // this change it now stems to "baik". The table notes that "baik" is
+ // the actual root so this deviation is an improvement. In a sample
+ // vocabulary derived from the most common words in id.wikipedia.org,
+ // this change only affects 0.12% of words (76 out of 64,587, including
+ // "peledakan" and "perbaikan").
+ $prefix != 3 and $prefix != 2
+ )
+
+ // prefix not in {di, meng, ter}
+ define SUFFIX_AN_OK as ( $prefix != 1 )
+
+ define SUFFIX_I_OK as (
+ // prefix not in {ke, peng, ber}
+ $prefix <= 2
+
+ // The rest of the condition from the paper is:
+ // V|K...c₁c₁, c₁ ≠ s, c₂ ≠ i
+ //
+ // The meaning of this is unclear in several ways, and none of the
+ // examples given of the stemmer's behaviour in the paper help to
+ // resolve these issues.
+ //
+ // Notice that c₂ isn't actually used - the most obvious explanation
+ // seems to be that "c₁c₁" should read "c₁c₂", or maybe "c₂c₁".
+ //
+ // Elsewhere the paper defines V... as meaning "the stem starts with
+ // a vowel" and K... as meaning "the stem starts with a consonant".
+ //
+ // In other places where it says X|Y... it seems the | binds more
+ // tightly, so it's (V|K)...cᵢcⱼ not V|(K...cᵢcⱼ). That seems a bit
+ // odd as the first letter must be either a vowel or a consonant, so
+ // that really just means "ends cᵢcⱼ". However, nowhere in the paper
+ // uses or defines a notation such as ...X, which may explain this
+ // seemingly redundant way of specifying this.
+ //
+ // The conditions elsewhere on prefix removal (e.g. V...) are clearly
+ // on the stem left after the prefix is removed. None of the other
+ // rules for suffix removal have conditions on the stem, but for
+ // consistency with the prefix rules we might expect that the cᵢcⱼ
+ // test is on what's left *after* removing the "i" suffix.
+ //
+ // However, studying Indonesian wordlists and discussion with a native
+ // speaker leads us to conclude that the purpose of this check is to
+ // protect words of foreign origin (e.g. "televisi", "organisasi",
+ // "komunikasi") from stemming, and the common feature of these is
+ // that the word ends "-si", so we conclude that the condition here
+ // should be read as "word does not end -si", and this is what we
+ // have implemented.
+ not 's'
+ )
+
+ define remove_suffix as (
+ [substring] among (
+ 'kan' SUFFIX_KAN_OK 'an' SUFFIX_AN_OK 'i' SUFFIX_I_OK
+ (delete $measure-=1)
+ )
+ )
+)
+
+define vowel 'aeiou'
+
+define VOWEL as ( vowel )
+
+define KER as ( non-vowel 'er' )
+
+define remove_first_order_prefix as (
+ [substring] among (
+ 'di' 'meng' 'men' 'me' 'ter' (delete $prefix=1 $measure-=1)
+ 'ke' 'peng' 'pen' (delete $prefix=3 $measure-=1)
+ 'meny' VOWEL ($prefix=1 <-'s' $measure-=1)
+ 'peny' VOWEL ($prefix=3 <-'s' $measure-=1)
+ 'mem' ($prefix=1 $measure-=1 vowel and <-'p' or delete)
+ 'pem' ($prefix=3 $measure-=1 vowel and <-'p' or delete)
+ )
+)
+
+define remove_second_order_prefix as (
+ // The paper has the condition on removal of prefix "bel" and "pel" as
+ // just "ajar" not "ajar..." but it seems that the latter must be what
+ // is intended so that e.g. "pelajaran" stems to "ajar" not "lajar".
+ // This change only affects a very small number of words (11 out of
+ // 64,587) and only for the better.
+ [substring] among (
+ 'per' 'pe' (delete $prefix=2 $measure-=1)
+ 'pelajar' (<-'ajar' $measure-=1)
+ 'ber' (delete $prefix=4 $measure-=1)
+ 'belajar' (<-'ajar' $prefix=4 $measure-=1)
+ 'be' KER (delete $prefix=4 $measure-=1)
+ )
+)
+
+define stem as (
+ $measure = 0
+ do ( repeat ( gopast vowel $measure+=1 ) )
+ $measure > 2
+ $prefix = 0
+ backwards (
+ do remove_particle
+ $measure > 2
+ do remove_possessive_pronoun
+ )
+ $measure > 2
+ test (
+ remove_first_order_prefix
+ do (
+ test ($measure > 2 backwards remove_suffix)
+ $measure > 2 remove_second_order_prefix
+ )
+ ) or (
+ do remove_second_order_prefix
+ do ($measure > 2 backwards remove_suffix)
+ )
+)
diff --git a/contrib/snowball/algorithms/irish.sbl b/contrib/snowball/algorithms/irish.sbl
new file mode 100644
index 0000000..0b1288a
--- /dev/null
+++ b/contrib/snowball/algorithms/irish.sbl
@@ -0,0 +1,151 @@
+routines (
+ R1 R2 RV
+ initial_morph
+ mark_regions
+ noun_sfx
+ deriv
+ verb_sfx
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* Accented characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef e' '{U+00E9}' // e-acute
+stringdef i' '{U+00ED}' // i-acute
+stringdef o' '{U+00F3}' // o-acute
+stringdef u' '{U+00FA}' // u-acute
+
+define v 'aeiou{a'}{e'}{i'}{o'}{u'}'
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ gopast v setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define initial_morph as (
+ [substring] among (
+ 'h-' 'n-' 't-' //nAthair -> n-athair, but alone are problematic
+ (delete)
+
+ // verbs
+ 'd{'}'
+ (delete)
+ 'd{'}fh'
+ (<- 'f')
+ // other contractions
+ 'm{'}' 'b{'}'
+ (delete)
+
+ 'sh'
+ (<- 's')
+
+ 'mb'
+ (<- 'b')
+ 'gc'
+ (<- 'c')
+ 'nd'
+ (<- 'd')
+ 'bhf'
+ (<- 'f')
+ 'ng'
+ (<- 'g')
+ 'bp'
+ (<- 'p')
+ 'ts'
+ (<- 's')
+ 'dt'
+ (<- 't')
+
+ // Lenition
+ 'bh'
+ (<- 'b')
+ 'ch'
+ (<- 'c')
+ 'dh'
+ (<- 'd')
+ 'fh'
+ (<- 'f')
+ 'gh'
+ (<- 'g')
+ 'mh'
+ (<- 'm')
+ 'ph'
+ (<- 'p')
+ 'th'
+ (<- 't')
+ )
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define noun_sfx as (
+ [substring] among (
+ 'amh' 'eamh' 'abh' 'eabh'
+ 'aibh' 'ibh' 'aimh' 'imh'
+ 'a{i'}ocht' '{i'}ocht' 'a{i'}ochta' '{i'}ochta'
+ (R1 delete)
+ 'ire' 'ir{i'}' 'aire' 'air{i'}'
+ (R2 delete)
+ )
+ )
+ define deriv as (
+ [substring] among (
+ 'acht' 'eacht' 'ach' 'each' 'eacht{u'}il' 'eachta' 'acht{u'}il' 'achta'
+ (R2 delete) //siopadóireacht -> siopadóir but not poblacht -> pobl
+ 'arcacht' 'arcachta{i'}' 'arcachta'
+ (<- 'arc') // monarcacht -> monarc
+ 'gineach' 'gineas' 'ginis'
+ (<- 'gin')
+ 'grafa{i'}och' 'grafa{i'}ocht' 'grafa{i'}ochta' 'grafa{i'}ochta{i'}'
+ (<- 'graf')
+ 'paite' 'patach' 'pataigh' 'patacha'
+ (<- 'paite')
+ '{o'}ideach' '{o'}ideacha' '{o'}idigh'
+ (<- '{o'}id')
+ )
+ )
+ define verb_sfx as (
+ [substring] among (
+ 'imid' 'aimid' '{i'}mid' 'a{i'}mid'
+ 'faidh' 'fidh'
+ (RV delete)
+ 'ain'
+ 'eadh' 'adh'
+ '{a'}il'
+ 'tear' 'tar'
+ (R1 delete)
+ )
+ )
+)
+
+define stem as (
+ do initial_morph
+ do mark_regions
+ backwards (
+ do noun_sfx
+ do deriv
+ do verb_sfx
+ )
+)
diff --git a/contrib/snowball/algorithms/italian.sbl b/contrib/snowball/algorithms/italian.sbl
new file mode 100644
index 0000000..bf0c161
--- /dev/null
+++ b/contrib/snowball/algorithms/italian.sbl
@@ -0,0 +1,195 @@
+
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ attached_pronoun
+ standard_suffix
+ verb_suffix
+ vowel_suffix
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v AEIO CG )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}'
+stringdef a` '{U+00E0}'
+stringdef e' '{U+00E9}'
+stringdef e` '{U+00E8}'
+stringdef i' '{U+00ED}'
+stringdef i` '{U+00EC}'
+stringdef o' '{U+00F3}'
+stringdef o` '{U+00F2}'
+stringdef u' '{U+00FA}'
+stringdef u` '{U+00F9}'
+
+define v 'aeiou{a`}{e`}{i`}{o`}{u`}'
+
+define prelude as (
+ test repeat (
+ [substring] among(
+ '{a'}' (<- '{a`}')
+ '{e'}' (<- '{e`}')
+ '{i'}' (<- '{i`}')
+ '{o'}' (<- '{o`}')
+ '{u'}' (<- '{u`}')
+ 'qu' (<- 'qU')
+ '' (next)
+ )
+ )
+ repeat goto (
+ v [ ('u' ] v <- 'U') or
+ ('i' ] v <- 'I')
+ )
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'I' (<- 'i')
+ 'U' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define attached_pronoun as (
+ [substring] among(
+ 'ci' 'gli' 'la' 'le' 'li' 'lo'
+ 'mi' 'ne' 'si' 'ti' 'vi'
+ // the compound forms are:
+ 'sene' 'gliela' 'gliele' 'glieli' 'glielo' 'gliene'
+ 'mela' 'mele' 'meli' 'melo' 'mene'
+ 'tela' 'tele' 'teli' 'telo' 'tene'
+ 'cela' 'cele' 'celi' 'celo' 'cene'
+ 'vela' 'vele' 'veli' 'velo' 'vene'
+ )
+ among( (RV)
+ 'ando' 'endo' (delete)
+ 'ar' 'er' 'ir' (<- 'e')
+ )
+ )
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'anza' 'anze' 'ico' 'ici' 'ica' 'ice' 'iche' 'ichi' 'ismo'
+ 'ismi' 'abile' 'abili' 'ibile' 'ibili' 'ista' 'iste' 'isti'
+ 'ist{a`}' 'ist{e`}' 'ist{i`}' 'oso' 'osi' 'osa' 'ose' 'mente'
+ 'atrice' 'atrici'
+ 'ante' 'anti' // Note 1
+ ( R2 delete )
+ 'azione' 'azioni' 'atore' 'atori'
+ ( R2 delete
+ try ( ['ic'] R2 delete )
+ )
+ 'logia' 'logie'
+ ( R2 <- 'log' )
+ 'uzione' 'uzioni' 'usione' 'usioni'
+ ( R2 <- 'u' )
+ 'enza' 'enze'
+ ( R2 <- 'ente' )
+ 'amento' 'amenti' 'imento' 'imenti'
+ ( RV delete )
+ 'amente' (
+ R1 delete
+ try (
+ [substring] R2 delete among(
+ 'iv' ( ['at'] R2 delete )
+ 'os' 'ic' 'abil'
+ )
+ )
+ )
+ 'it{a`}' (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil' 'ic' 'iv' (R2 delete)
+ )
+ )
+ )
+ 'ivo' 'ivi' 'iva' 'ive' (
+ R2 delete
+ try ( ['at'] R2 delete ['ic'] R2 delete )
+ )
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among(
+ 'ammo' 'ando' 'ano' 'are' 'arono' 'asse' 'assero' 'assi'
+ 'assimo' 'ata' 'ate' 'ati' 'ato' 'ava' 'avamo' 'avano' 'avate'
+ 'avi' 'avo' 'emmo' 'enda' 'ende' 'endi' 'endo' 'er{a`}' 'erai'
+ 'eranno' 'ere' 'erebbe' 'erebbero' 'erei' 'eremmo' 'eremo'
+ 'ereste' 'eresti' 'erete' 'er{o`}' 'erono' 'essero' 'ete'
+ 'eva' 'evamo' 'evano' 'evate' 'evi' 'evo' 'Yamo' 'iamo' 'immo'
+ 'ir{a`}' 'irai' 'iranno' 'ire' 'irebbe' 'irebbero' 'irei'
+ 'iremmo' 'iremo' 'ireste' 'iresti' 'irete' 'ir{o`}' 'irono'
+ 'isca' 'iscano' 'isce' 'isci' 'isco' 'iscono' 'issero' 'ita'
+ 'ite' 'iti' 'ito' 'iva' 'ivamo' 'ivano' 'ivate' 'ivi' 'ivo'
+ 'ono' 'uta' 'ute' 'uti' 'uto'
+
+ 'ar' 'ir' // but 'er' is problematical
+ (delete)
+ )
+ )
+
+ define AEIO 'aeio{a`}{e`}{i`}{o`}'
+ define CG 'cg'
+
+ define vowel_suffix as (
+ try (
+ [AEIO] RV delete
+ ['i'] RV delete
+ )
+ try (
+ ['h'] CG RV delete
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards (
+ do attached_pronoun
+ do (standard_suffix or verb_suffix)
+ do vowel_suffix
+ )
+ do postlude
+)
+
+/*
+ Note 1: additions of 15 Jun 2005
+*/
+
diff --git a/contrib/snowball/algorithms/kraaij_pohlmann.sbl b/contrib/snowball/algorithms/kraaij_pohlmann.sbl
new file mode 100644
index 0000000..409bf0f
--- /dev/null
+++ b/contrib/snowball/algorithms/kraaij_pohlmann.sbl
@@ -0,0 +1,240 @@
+strings ( ch )
+integers ( p1 p2 )
+booleans ( Y_found stemmed GE_removed )
+
+routines (
+
+ R1 R2
+ C V VX
+ lengthen_V
+ Step_1 Step_2 Step_3 Step_4 Step_7
+ Step_6 Step_1c
+ Lose_prefix
+ Lose_infix
+ measure
+)
+
+externals ( stem )
+
+groupings ( v v_WX AOU AIOU )
+
+stringescapes {}
+
+define v 'aeiouy'
+define v_WX v + 'wx'
+define AOU 'aou'
+define AIOU 'aiou'
+
+backwardmode (
+
+ define R1 as ($p1 <= cursor)
+ define R2 as ($p2 <= cursor)
+
+ define V as test (v or 'ij')
+ define VX as test (next v or 'ij')
+ define C as test (not 'ij' non-v)
+
+ define lengthen_V as do (
+ non-v_WX [ (AOU] test (non-v or atlimit)) or
+ ('e'] test (non-v or atlimit
+ not AIOU
+ not (next AIOU non-v)))
+ ->ch insert ch
+ )
+
+ define Step_1 as
+ (
+ [substring] among (
+
+ '{'}s' (delete)
+ 's' (R1 not ('t' R1) C delete)
+ 'ies' (R1 <-'ie')
+ 'es'
+ (('ar' R1 C ] delete lengthen_V) or
+ ('er' R1 C ] delete) or
+ (R1 C <-'e'))
+
+ 'aus' (R1 V <-'au')
+ 'en' (('hed' R1 ] <-'heid') or
+ ('nd' delete) or
+ ('d' R1 C ] delete) or
+ ('i' or 'j' V delete) or
+ (R1 C delete lengthen_V))
+ 'nde' (<-'nd')
+ )
+ )
+
+ define Step_2 as
+ (
+ [substring] among (
+ 'je' (('{'}t' ] delete) or
+ ('et' ] R1 C delete) or
+ ('rnt' ] <-'rn') or
+ ('t' ] R1 VX delete) or
+ ('ink' ] <-'ing') or
+ ('mp' ] <-'m') or
+ ('{'}' ] R1 delete) or
+ (] R1 C delete))
+ 'ge' (R1 <-'g')
+ 'lijke'(R1 <-'lijk')
+ 'ische'(R1 <-'isch')
+ 'de' (R1 C delete)
+ 'te' (R1 <-'t')
+ 'se' (R1 <-'s')
+ 're' (R1 <-'r')
+ 'le' (R1 delete attach 'l' lengthen_V)
+ 'ene' (R1 C delete attach 'en' lengthen_V)
+ 'ieve' (R1 C <-'ief')
+ )
+ )
+
+ define Step_3 as
+ (
+ [substring] among (
+ 'atie' (R1 <-'eer')
+ 'iteit' (R1 delete lengthen_V)
+ 'heid'
+ 'sel'
+ 'ster' (R1 delete)
+ 'rder' (<-'r')
+ 'ing'
+ 'isme'
+ 'erij' (R1 delete lengthen_V)
+ 'arij' (R1 C <-'aar')
+ 'fie' (R2 delete attach 'f' lengthen_V)
+ 'gie' (R2 delete attach 'g' lengthen_V)
+ 'tst' (R1 C <-'t')
+ 'dst' (R1 C <-'d')
+ )
+ )
+
+ define Step_4 as
+ (
+ ( [substring] among (
+ 'ioneel' (R1 <-'ie')
+ 'atief' (R1 <-'eer')
+ 'baar' (R1 delete)
+ 'naar' (R1 V <-'n')
+ 'laar' (R1 V <-'l')
+ 'raar' (R1 V <-'r')
+ 'tant' (R1 <-'teer')
+ 'lijker'
+ 'lijkst' (R1 <-'lijk')
+ 'achtig'
+ 'achtiger'
+ 'achtigst'(R1 delete)
+ 'eriger'
+ 'erigst'
+ 'erig'
+ 'end' (R1 C delete lengthen_V)
+ )
+ )
+ or
+ ( [substring] among (
+ 'iger'
+ 'igst'
+ 'ig' (R1 C delete lengthen_V)
+ )
+ )
+ )
+
+ define Step_7 as
+ (
+ [substring] among (
+ 'kt' (<-'k')
+ 'ft' (<-'f')
+ 'pt' (<-'p')
+ )
+ )
+
+ define Step_6 as
+ (
+ [substring] among (
+ 'bb' (<-'b')
+ 'cc' (<-'c')
+ 'dd' (<-'d')
+ 'ff' (<-'f')
+ 'gg' (<-'g')
+ 'hh' (<-'h')
+ 'jj' (<-'j')
+ 'kk' (<-'k')
+ 'll' (<-'l')
+ 'mm' (<-'m')
+ 'nn' (<-'n')
+ 'pp' (<-'p')
+ 'qq' (<-'q')
+ 'rr' (<-'r')
+ 'ss' (<-'s')
+ 'tt' (<-'t')
+ 'vv' (<-'v')
+ 'ww' (<-'w')
+ 'xx' (<-'x')
+ 'zz' (<-'z')
+ 'v' (<-'f')
+ 'z' (<-'s')
+ )
+ )
+
+ define Step_1c as
+ (
+ [substring] among ( (R1 C)
+ 'd' (not ('n' R1) delete)
+ 't' (not ('h' R1) delete)
+ )
+ )
+)
+
+define Lose_prefix as (
+ ['ge'] test hop 3 (goto v goto non-v)
+ set GE_removed
+ delete
+)
+
+define Lose_infix as (
+ next
+ gopast (['ge']) test hop 3 (goto v goto non-v)
+ set GE_removed
+ delete
+)
+
+define measure as (
+ $p1 = limit
+ $p2 = limit
+ do(
+ repeat non-v atleast 1 ('ij' or v) non-v setmark p1
+ repeat non-v atleast 1 ('ij' or v) non-v setmark p2
+ )
+
+)
+define stem as (
+
+ unset Y_found
+ unset stemmed
+ do ( ['y'] <-'Y' set Y_found )
+ do repeat(goto (v ['y'])<-'Y' set Y_found )
+
+ measure
+
+ backwards (
+ do (Step_1 set stemmed )
+ do (Step_2 set stemmed )
+ do (Step_3 set stemmed )
+ do (Step_4 set stemmed )
+ )
+ unset GE_removed
+ do (Lose_prefix and measure)
+ backwards (
+ do (GE_removed Step_1c)
+ )
+ unset GE_removed
+ do (Lose_infix and measure)
+ backwards (
+ do (GE_removed Step_1c)
+ )
+ backwards (
+ do (Step_7 set stemmed )
+ do (stemmed or GE_removed Step_6)
+ )
+ do(Y_found repeat(goto (['Y']) <-'y'))
+)
+
diff --git a/contrib/snowball/algorithms/lithuanian.sbl b/contrib/snowball/algorithms/lithuanian.sbl
new file mode 100644
index 0000000..ff7fdb9
--- /dev/null
+++ b/contrib/snowball/algorithms/lithuanian.sbl
@@ -0,0 +1,373 @@
+externals ( stem )
+
+// escape symbols for substituting lithuanian characters
+stringescapes { }
+
+/* Special characters in Unicode Latin Extended-A */
+// ' nosine
+stringdef a' '{U+0105}' // ą a + ogonek
+stringdef e' '{U+0119}' // ę e + ogonek
+stringdef i' '{U+012F}' // į i + ogonek
+stringdef u' '{U+0173}' // ų u + ogonek
+
+// . taskas
+stringdef e. '{U+0117}' // ė e + dot
+
+// - ilgoji
+stringdef u- '{U+016B}' // ū u + macron
+
+// * varnele
+stringdef c* '{U+010D}' // č c + caron (haček)
+stringdef s* '{U+0161}' // š s + caron (haček)
+stringdef z* '{U+017E}' // ž z + caron (haček)
+
+// [C](VC)^m[V|C]
+// definitions of variables for
+// p1 - position of m = 0
+integers ( p1 )
+
+// groupings
+// v - lithuanian vowels
+groupings ( v )
+
+// v - all lithuanian vowels
+define v 'aeiyou{a'}{e'}{i'}{u'}{e.}{u-}'
+
+// all lithuanian stemmer routines: 4 steps
+routines (
+ step2 R1 step1 fix_chdz fix_gd fix_conflicts
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define step1 as (
+ setlimit tomark p1 for ([substring]) R1 among (
+ // Daiktavardžiai (Nouns)
+ // I linksniuotė (declension I)
+ 'as' 'ias' 'is' 'ys' // vyras, kelias, brolis, gaidys
+ 'o' 'io' // vyro, kelio
+ 'ui' 'iui' // vyrui, keliui
+ '{a'}' 'i{a'}' '{i'}' // vyrą, kelią, brolį
+ 'u' 'iu' // vyru, keliu
+ 'e' 'yje' // vyre, kelyje
+ 'y' 'au' 'i' // kely, brolau, broli,
+ 'an' // nusižengiman
+
+ 'ai' 'iai' // vyrai, keliai
+ '{u'}' 'i{u'}' // vyrų, kelių
+ 'ams' 'am' // vyrams, vyram
+ 'iams' 'iam' // broliams, broliam
+ 'us' 'ius' // vyrus, brolius
+ 'ais' 'iais' // vyrais, keliais
+ 'uose' 'iuose' 'uos' 'iuos' // vyruose, keliuose, vyruos, keliuos
+ 'uosna' 'iuosna' // vyruosna, keliuosna
+ 'ysna' // žutysna
+
+ 'asis' 'aisi' // sukimasis, sukimaisi
+ 'osi' '{u'}si' // sukimosi, sukimųsi
+ 'uisi' // sukimuisi
+ '{a'}si' // sukimąsi
+ 'usi' // sukimusi
+ 'esi' // sukimesi
+
+ 'uo' // mėnuo
+
+
+ // II linksniuote (declension II)
+ 'a' 'ia' // galva, vysnios
+ 'os' 'ios' // galvos, vysnios
+ 'oj' 'oje' 'ioje' // galvoje, vysnioje
+ 'osna' 'iosna' // galvosna, vyšniosna
+ 'om' 'oms' 'ioms' // galvoms, vysnioms
+ 'omis' 'iomis' // galvomis, vysniomis
+ 'ose' 'iose' // galvose, vysniose
+ 'on' 'ion' // galvon, vyšnion
+
+
+ // III linksniuote (declension III)
+ '{e.}' // gervė
+ '{e.}s' // gervės
+ 'ei' // gervei
+ '{e'}' // gervę
+ '{e.}j' '{e.}je' // gervėj, gervėje
+ '{e.}ms' // gervėms
+ 'es' // gerves
+ '{e.}mis' // gervėmis
+ '{e.}se' // gervėse
+ '{e.}sna' // gervėsna
+ '{e.}n' // žydaitėn
+
+
+ // IV linksniuote (declension IV)
+ 'aus' 'iaus' // sūnaus, skaičiaus
+ 'umi' 'iumi' // sūnumi, skaičiumi
+ 'uje' 'iuje' // sūnuje, skaičiuje
+ 'iau' // skaičiau
+
+ '{u-}s' // sūnūs
+ 'ums' // sūnums
+ 'umis' // sūnumis
+ 'un' 'iun' // sūnun, administratoriun
+
+
+ // V linksniuote (declension V)
+ 'ies' 'ens' 'enio' 'ers' // avies, vandens, sesers
+ 'eniui' 'eriai' // vandeniui, eriai
+ 'en{i'}' 'er{i'}' // vandenį, seserį
+ 'imi' 'eniu' 'erimi' 'eria' // avimi, vandeniu, seserimi, seseria
+ 'enyje' 'eryje' // vandenyje, seseryje
+ 'ie' 'enie' 'erie' // avie, vandenie, seserie
+
+ 'enys' 'erys' // vandenys, seserys
+ // 'en{u'}' konfliktas su 'žandenų' 'antenų'
+ 'er{u'}' // seserų
+ 'ims' 'enims' 'erims' // avims, vandemins, seserims
+ 'enis' // vandenis
+ 'imis' // žebenkštimis
+ 'enimis' // vandenimis
+ 'yse' 'enyse' 'eryse' // avyse, vandenyse, seseryse
+
+
+ // Būdvardžiai (Adjectives)
+ // (i)a linksniuotė
+ 'iem' 'iems' // geriem, geriems
+ 'ame' 'iame' // naujame, mediniame
+
+
+ // Veiksmažodžiai (Verbs)
+ // Tiesioginė nuosaka (indicative mood)
+ // esamasis laikas (present tense)
+ // (i)a asmenuotė (declension (i)a)
+ 'uosi' 'iuosi' // dirbuosi, traukiuosi
+ 'iesi' // dirbiesi
+ 'asi' 'iasi' // dirbasi, traukiasi
+ 'am{e.}s' 'iam{e.}s' // dirbamės, traukiamės
+ 'at' 'ate' 'iat' 'iate' // dirbat, dirbate, ariat, traukiate
+ 'at{e.}s' 'iat{e.}s' // dirbatės, traukiatės
+
+ // i asmenuotė (declension i)
+ 'isi' // tikisi
+ 'im' // mylim
+ // 'ime' konfliktassu daiktavardžiu vietininku, pvz. 'gėrime'
+ 'im{e.}s' // tikimės
+ 'it' 'ite' // mylit, mylite, tikitės
+ // 'it{e.}s' konfliktas su priesaga ir dgs. vardininko galūne -ait-ės pvz. žydaitės
+
+ // o asmenuotė (declension o)
+ 'ome' // mokome
+ 'ot' 'ote' // mokot, mokote
+
+ // būtasis laikas
+ // o asmenuotė (declension o)
+ '{e.}jo' '{e.}josi' // tikėjo, tikėjosi
+ 'ot{e.}s' // tikėjotės/bijotės
+
+ // ė asmenuotė (declension ė)
+ 'eisi' // mokeisi
+ '{e.}si' // mokėsi
+ '{e.}m' '{e.}me' // mokėm, mokėme
+ '{e.}m{e.}s' // mokėmės
+ '{e.}t' '{e.}te' // mokėt, mokėte
+ '{e.}t{e.}s' // mokėtės
+
+ // būtasis dažninis laikas (frequentative past tense)
+ 'ausi' // mokydavausi
+ 'om{e.}s' // mokydavomės/bijomės
+
+
+ // būsimasis laikas (future tense)
+ 'siu' 'siuosi' // dirbsiu, mokysiuosi
+ 'si' 'siesi' // dirbsi, dirbsiesi
+ 's' 'ysis' // dirbs, mokysis
+ 'sim' 'sime' // dirbsim, dirbsime
+ 'sit' 'site' // gersit, gersite
+
+ // tariamoji nuosaka (subjunctive mood)
+ '{c*}iau' '{c*}iausi' // dirbčiau
+ 'tum' 'tumei' // dirbtum, dirbtumei
+ 'tumeis' 'tumeisi' // mokytumeis, mokytumeisi
+ // 't{u'}' nes blogai batutų -> batų
+ 't{u'}si' // mokytųsi
+ // 'tume' konfliktas su 'šventume'
+ 'tum{e.}m' // dirbtumėm
+ 'tum{e.}me' // dirbtumėme
+ 'tum{e.}m{e.}s' // mokytumėmės
+ 'tute' 'tum{e.}t' // dirbtute, dirbtumėt
+ 'tum{e.}te' // dirbtumėte
+ 'tum{e.}t{e.}s' // mokytumėtės
+
+ // liepiamoji nuosaka (imperative mood)
+ 'k' 'ki' // dirbk, dirbki, mokykis
+ // 'kis' konfliktas viln-išk-is
+ // 'kime' konfliktas, nes pirkime
+ 'kim{e.}s' // mokykimės
+
+ // bendratis (infinitive)
+ 'uoti' 'iuoti' // meluoti, dygsniuoti
+ 'auti' 'iauti' // draugauti, girtuokliauti
+ 'oti' 'ioti' // dovanoti, meškerioti
+ '{e.}ti' // auklėti
+ 'yti' // akyti
+ 'inti' // auginti
+ 'in{e.}ti' // blusinėti
+ 'enti' // gyventi
+ 'tel{e.}ti' // bumbtelėti
+ 'ter{e.}ti' // bumbterėti
+
+ 'ti' // skalbti
+ // 'tis' konfliktas, nes rytme-tis -> rytme
+
+ // dalyviai (participles)
+ '{a'}s' 'i{a'}s' '{i'}s' // dirbąs, žaidžiąs, gulįs
+ 't{u'}s' // suktųs -> suk
+ 'sim{e.}s' // suksimės
+ 'sit{e.}s' // suksitės
+ 'kite' // supkite
+ )
+
+ delete
+ )
+
+ define step2 as repeat (
+ setlimit tomark p1 for ([substring]) among (
+ // daiktavardziu priesagos (Noun suffixes)
+
+ // budvardziu priesagos (Adjective suffixes)
+ // 'in' // konfliktas su 'augintinis' ir 'akiniais' // lauk-in-is
+ 'ing' // tvark-ing-as
+ 'i{s*}k' // lenk-išk-as
+ '{e.}t' // dem-ėt-as
+ 'ot' // garban-ot-as
+ 'uot' 'iuot' // lang-uot-as, akin-iuot-as
+ // 'tin', nes augintinis // dirb-tin-is
+ // 'ut', nes batutas, degutas etc. // maž-ut-is
+ 'yt' // maž-yt-is
+ 'iuk' // maž-iuk-as
+ 'iul' // maž-ul-is
+ '{e.}l' // maž-ėl-is
+ 'yl' // maž-yl-is
+ 'u{c*}iuk' // maž-učiuk-as
+ 'uliuk' // maž-uliuk-as
+ 'ut{e.}ait' // maž-utėlait-is
+ 'ok' // did-ok-as
+ 'iok' // višč-iok-as
+ 'sv' '{s*}v' 'zgan' // sal-sv-as, pilk-šv-as, bal-zgan-as
+ 'op' 'iop' // dvej-op-as, viener-iop-as
+ 'ain' // apval-ain-as
+ 'yk{s*}t' 'yk{s*}{c*}' // ten-ykšt-is, vakar-ykšč-ias
+
+ // laisniai
+ 'esn' // did-esn-is
+ 'aus' 'iaus' // nauj-aus-ias, ger-iaus-ias
+
+ // ivardziuotiniai budvardziai (Pronominal adjectives)
+ // vyriska gimine (Male gender)
+ 'ias' // žaliasis
+ 'oj' 'ioj' // gerojo, žaliojo
+ 'aj' 'iaj' // gerajam, žaliajam
+ '{a'}j' 'i{a'}j' // garąjį, žaliąjį
+ 'uoj' 'iuoj' // geruoju, žaliuoju
+ 'iej' // gerieji
+ '{u'}j' 'i{u'}j' // gerųjų, žaliųjų
+ 'ies' // geriesiems
+ 'uos' 'iuos' // geruosius, žaliuosius
+ 'ais' 'iais' // geraisiais, žaliaisiais
+
+ // moteriska gimine (Female gender)
+ 'os' 'ios' // gerosios, žaliosios
+ '{a'}s' 'i{a'}s' // gerąsios, žaliąsias
+
+ // būtasis dažninis laikas (frequentative past tense)
+ 'dav' // ei-dav-o
+
+ // dalyvių priesagos (particple suffix)
+ 'ant' 'iant'
+ 'int' // tur-int-is
+ '{e.}j' // tur-ėj-o
+ '{e'}' //
+ '{e.}j{e'}'
+ '{e'}s' // dirb-ęs-is
+
+ 'siant' // dirb-siant
+
+ // pusdalyviai (participle)
+ 'dam' // bėg-dam-as
+
+ 'auj' // ūkinink-auj-a
+ 'jam'
+ 'iau'
+ 'am' // baiminim-ams-i
+ )
+
+ delete
+ )
+
+ define fix_conflicts as (
+ [substring] among (
+ // 'lietuvaite' -> 'lietuvaitė', konfliktas su 'myl-ite'
+ 'aite' (<-'ait{e.}')
+ // 'lietuvaitės' -> 'lietuvaitė', konfliktas su 'myl-itės'
+ 'ait{e.}s' (<-'ait{e.}')
+
+ // ''ūs-uotės' -> 'ūs-uotė', konfliktas 'mokotės'
+ 'uot{e.}s' (<-'uot{e.}')
+ // ''ūs-uote' -> 'ūs-uotė', konfliktas 'mokote'
+ 'uote' (<-'uot{e.}')
+
+ // 'žerėjime' -> 'žėrėjimas', konfliktas su 'žais-ime'
+ '{e.}jime' (<-'{e.}jimas')
+
+ // 'žvilgesiu' -> 'žvilgesys', konfliktas su 'dirb-siu'
+ 'esiu' (<-'esys')
+ // 'duobkasiu' -> 'duobkasys', konfliktas su 'pakasiu'
+ 'asius' (<-'asys')
+
+ // 'žioravime' -> 'žioravimas', konfliktas su 'myl-ime'
+ 'avime' (<-'avimas')
+ 'ojime' (<-'ojimas')
+
+ // 'advokatės' -> 'advokatė', konfliktas su 'dirb-atės'
+ 'okat{e.}s' (<-'okat{e.}')
+ // 'advokate' -> 'advokatė', konfliktas su 'dirb-ate'
+ 'okate' (<-'okat{e.}')
+ )
+ )
+
+ define fix_chdz as (
+ [substring] among (
+ '{c*}' (<-'t')
+ 'd{z*}' (<-'d')
+ )
+ )
+
+ define fix_gd as (
+ [substring] among (
+ 'gd' (<-'g')
+ // '{e.}k' (<-'{e.}g')
+ )
+ )
+
+)
+
+define stem as (
+
+ $p1 = limit
+
+ do (
+ // priešdėlis 'a' ilgeniuose nei 6 raidės žodžiuose, pvz. 'a-liejus'.
+ try (test 'a' $(len > 6) hop 1)
+
+ gopast v gopast non-v setmark p1
+ )
+
+ backwards (
+ do fix_conflicts
+ do step1
+ do fix_chdz
+ do step2
+ do fix_chdz
+ do fix_gd
+ )
+
+)
diff --git a/contrib/snowball/algorithms/lovins.sbl b/contrib/snowball/algorithms/lovins.sbl
new file mode 100644
index 0000000..3f69f15
--- /dev/null
+++ b/contrib/snowball/algorithms/lovins.sbl
@@ -0,0 +1,208 @@
+
+stringescapes {}
+
+routines (
+ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z AA BB CC
+
+ endings
+
+ undouble respell
+)
+
+externals ( stem )
+
+backwardmode (
+
+ /* Lovins' conditions A, B ... CC, as given in her Appendix B, where
+ a test for a two letter prefix ('test hop 2') is implicitly
+ assumed. Note that 'e' next 'u' corresponds to her u*e because
+ Snowball is scanning backwards. */
+
+ define A as ( hop 2 )
+ define B as ( hop 3 )
+ define C as ( hop 4 )
+ define D as ( hop 5 )
+ define E as ( test hop 2 not 'e' )
+ define F as ( test hop 3 not 'e' )
+ define G as ( test hop 3 'f' )
+ define H as ( test hop 2 't' or 'll' )
+ define I as ( test hop 2 not 'o' not 'e' )
+ define J as ( test hop 2 not 'a' not 'e' )
+ define K as ( test hop 3 'l' or 'i' or ('e' next 'u') )
+ define L as ( test hop 2 not 'u' not 'x' not ('s' not 'o') )
+ define M as ( test hop 2 not 'a' not 'c' not 'e' not 'm' )
+ define N as ( test hop 3 ( hop 2 not 's' or hop 2 ) )
+ define O as ( test hop 2 'l' or 'i' )
+ define P as ( test hop 2 not 'c' )
+ define Q as ( test hop 2 test hop 3 not 'l' not 'n' )
+ define R as ( test hop 2 'n' or 'r' )
+ define S as ( test hop 2 'dr' or ('t' not 't') )
+ define T as ( test hop 2 's' or ('t' not 'o') )
+ define U as ( test hop 2 'l' or 'm' or 'n' or 'r' )
+ define V as ( test hop 2 'c' )
+ define W as ( test hop 2 not 's' not 'u' )
+ define X as ( test hop 2 'l' or 'i' or ('e' next 'u') )
+ define Y as ( test hop 2 'in' )
+ define Z as ( test hop 2 not 'f' )
+ define AA as ( test hop 2 among ( 'd' 'f' 'ph' 'th' 'l' 'er' 'or'
+ 'es' 't' ) )
+ define BB as ( test hop 3 not 'met' not 'ryst' )
+ define CC as ( test hop 2 'l' )
+
+
+ /* The system of endings, as given in Appendix A. */
+
+ define endings as (
+ [substring] among(
+ 'alistically' B 'arizability' A 'izationally' B
+
+ 'antialness' A 'arisations' A 'arizations' A 'entialness' A
+
+ 'allically' C 'antaneous' A 'antiality' A 'arisation' A
+ 'arization' A 'ationally' B 'ativeness' A 'eableness' E
+ 'entations' A 'entiality' A 'entialize' A 'entiation' A
+ 'ionalness' A 'istically' A 'itousness' A 'izability' A
+ 'izational' A
+
+ 'ableness' A 'arizable' A 'entation' A 'entially' A
+ 'eousness' A 'ibleness' A 'icalness' A 'ionalism' A
+ 'ionality' A 'ionalize' A 'iousness' A 'izations' A
+ 'lessness' A
+
+ 'ability' A 'aically' A 'alistic' B 'alities' A
+ 'ariness' E 'aristic' A 'arizing' A 'ateness' A
+ 'atingly' A 'ational' B 'atively' A 'ativism' A
+ 'elihood' E 'encible' A 'entally' A 'entials' A
+ 'entiate' A 'entness' A 'fulness' A 'ibility' A
+ 'icalism' A 'icalist' A 'icality' A 'icalize' A
+ 'ication' G 'icianry' A 'ination' A 'ingness' A
+ 'ionally' A 'isation' A 'ishness' A 'istical' A
+ 'iteness' A 'iveness' A 'ivistic' A 'ivities' A
+ 'ization' F 'izement' A 'oidally' A 'ousness' A
+
+ 'aceous' A 'acious' B 'action' G 'alness' A
+ 'ancial' A 'ancies' A 'ancing' B 'ariser' A
+ 'arized' A 'arizer' A 'atable' A 'ations' B
+ 'atives' A 'eature' Z 'efully' A 'encies' A
+ 'encing' A 'ential' A 'enting' C 'entist' A
+ 'eously' A 'ialist' A 'iality' A 'ialize' A
+ 'ically' A 'icance' A 'icians' A 'icists' A
+ 'ifully' A 'ionals' A 'ionate' D 'ioning' A
+ 'ionist' A 'iously' A 'istics' A 'izable' E
+ 'lessly' A 'nesses' A 'oidism' A
+
+ 'acies' A 'acity' A 'aging' B 'aical' A
+ 'alist' A 'alism' B 'ality' A 'alize' A
+ 'allic'BB 'anced' B 'ances' B 'antic' C
+ 'arial' A 'aries' A 'arily' A 'arity' B
+ 'arize' A 'aroid' A 'ately' A 'ating' I
+ 'ation' B 'ative' A 'ators' A 'atory' A
+ 'ature' E 'early' Y 'ehood' A 'eless' A
+ 'elity' A 'ement' A 'enced' A 'ences' A
+ 'eness' E 'ening' E 'ental' A 'ented' C
+ 'ently' A 'fully' A 'ially' A 'icant' A
+ 'ician' A 'icide' A 'icism' A 'icist' A
+ 'icity' A 'idine' I 'iedly' A 'ihood' A
+ 'inate' A 'iness' A 'ingly' B 'inism' J
+ 'inity'CC 'ional' A 'ioned' A 'ished' A
+ 'istic' A 'ities' A 'itous' A 'ively' A
+ 'ivity' A 'izers' F 'izing' F 'oidal' A
+ 'oides' A 'otide' A 'ously' A
+
+ 'able' A 'ably' A 'ages' B 'ally' B
+ 'ance' B 'ancy' B 'ants' B 'aric' A
+ 'arly' K 'ated' I 'ates' A 'atic' B
+ 'ator' A 'ealy' Y 'edly' E 'eful' A
+ 'eity' A 'ence' A 'ency' A 'ened' E
+ 'enly' E 'eous' A 'hood' A 'ials' A
+ 'ians' A 'ible' A 'ibly' A 'ical' A
+ 'ides' L 'iers' A 'iful' A 'ines' M
+ 'ings' N 'ions' B 'ious' A 'isms' B
+ 'ists' A 'itic' H 'ized' F 'izer' F
+ 'less' A 'lily' A 'ness' A 'ogen' A
+ 'ward' A 'wise' A 'ying' B 'yish' A
+
+ 'acy' A 'age' B 'aic' A 'als'BB
+ 'ant' B 'ars' O 'ary' F 'ata' A
+ 'ate' A 'eal' Y 'ear' Y 'ely' E
+ 'ene' E 'ent' C 'ery' E 'ese' A
+ 'ful' A 'ial' A 'ian' A 'ics' A
+ 'ide' L 'ied' A 'ier' A 'ies' P
+ 'ily' A 'ine' M 'ing' N 'ion' Q
+ 'ish' C 'ism' B 'ist' A 'ite'AA
+ 'ity' A 'ium' A 'ive' A 'ize' F
+ 'oid' A 'one' R 'ous' A
+
+ 'ae' A 'al'BB 'ar' X 'as' B
+ 'ed' E 'en' F 'es' E 'ia' A
+ 'ic' A 'is' A 'ly' B 'on' S
+ 'or' T 'um' U 'us' V 'yl' R
+ '{'}s' A 's{'}' A
+
+ 'a' A 'e' A 'i' A 'o' A
+ 's' W 'y' B
+
+ (delete)
+ )
+ )
+
+ /* Undoubling is rule 1 of appendix C. */
+
+ define undouble as (
+ test substring among ('bb' 'dd' 'gg' 'll' 'mm' 'nn' 'pp' 'rr' 'ss'
+ 'tt')
+ [next] delete
+ )
+
+ /* The other appendix C rules can be done together. */
+
+ define respell as (
+ [substring] among (
+ 'iev' (<-'ief')
+ 'uct' (<-'uc')
+ 'umpt' (<-'um')
+ 'rpt' (<-'rb')
+ 'urs' (<-'ur')
+ 'istr' (<-'ister')
+ 'metr' (<-'meter')
+ 'olv' (<-'olut')
+ 'ul' (not 'a' not 'i' not 'o' <-'l')
+ 'bex' (<-'bic')
+ 'dex' (<-'dic')
+ 'pex' (<-'pic')
+ 'tex' (<-'tic')
+ 'ax' (<-'ac')
+ 'ex' (<-'ec')
+ 'ix' (<-'ic')
+ 'lux' (<-'luc')
+ 'uad' (<-'uas')
+ 'vad' (<-'vas')
+ 'cid' (<-'cis')
+ 'lid' (<-'lis')
+ 'erid' (<-'eris')
+ 'pand' (<-'pans')
+ 'end' (not 's' <-'ens')
+ 'ond' (<-'ons')
+ 'lud' (<-'lus')
+ 'rud' (<-'rus')
+ 'her' (not 'p' not 't' <-'hes')
+ 'mit' (<-'mis')
+ 'ent' (not 'm' <-'ens')
+ /* 'ent' was 'end' in the 1968 paper - a typo. */
+ 'ert' (<-'ers')
+ 'et' (not 'n' <-'es')
+ 'yt' (<-'ys')
+ 'yz' (<-'ys')
+ )
+ )
+)
+
+define stem as (
+
+ backwards (
+ do endings
+ do undouble
+ do respell
+ )
+)
+
diff --git a/contrib/snowball/algorithms/nepali.sbl b/contrib/snowball/algorithms/nepali.sbl
new file mode 100644
index 0000000..d388748
--- /dev/null
+++ b/contrib/snowball/algorithms/nepali.sbl
@@ -0,0 +1,92 @@
+/*
+ * Authors:
+ * - Ingroj Shrestha <ing.stha@gmail.com>, Nepali NLP Group
+ * - Oleg Bartunov <obartunov@gmail.com>, Postgres Professional Ltd.
+ * - Shreeya Singh Dhakal, Nepali NLP Group
+ */
+
+routines (
+ remove_category_1
+ check_category_2
+ remove_category_2
+ remove_category_3
+)
+
+stringescapes {}
+
+stringdef dsc '{U+0901}' // DEVANAGARI_SIGN_CANDRABINDU
+stringdef dsa '{U+0902}' // DEVANAGARI_SIGN_ANUSVARA
+stringdef dli '{U+0907}' // DEVANAGARI_LETTER_I
+stringdef dlii '{U+0908}' // DEVANAGARI_LETTER_II
+stringdef dle '{U+090F}' // DEVANAGARI_LETTER_E
+stringdef dlka '{U+0915}' // DEVANAGARI_LETTER_KA
+stringdef dlkha '{U+0916}' // DEVANAGARI_LETTER_KHA
+stringdef dlg '{U+0917}' // DEVANAGARI_LETTER_GA
+stringdef dlc '{U+091B}' // DEVANAGARI_LETTER_CHA
+stringdef dlta '{U+0924}' // DEVANAGARI_LETTER_TA
+stringdef dltha '{U+0925}' // DEVANAGARI_LETTER_THA
+stringdef dld '{U+0926}' // DEVANAGARI_LETTER_DA
+stringdef dln '{U+0928}' // DEVANAGARI_LETTER_NA
+stringdef dlpa '{U+092A}' // DEVANAGARI_LETTER_PA
+stringdef dlpha '{U+092B}' // DEVANAGARI_LETTER_PHA
+stringdef dlb '{U+092D}' // DEVANAGARI_LETTER_BHA
+stringdef dlm '{U+092E}' // DEVANAGARI_LETTER_MA
+stringdef dly '{U+092F}' // DEVANAGARI_LETTER_YA
+stringdef dlr '{U+0930}' // DEVANAGARI_LETTER_RA
+stringdef dll '{U+0932}' // DEVANAGARI_LETTER_LA
+stringdef dlv '{U+0935}' // DEVANAGARI_LETTER_VA
+stringdef dls '{U+0938}' // DEVANAGARI_LETTER_SA
+stringdef dlh '{U+0939}' // DEVANAGARI_LETTER_HA
+stringdef dvsaa '{U+093E}' // DEVANAGARI_VOWEL_SIGN_AA
+stringdef dvsi '{U+093F}' // DEVANAGARI_VOWEL_SIGN_I
+stringdef dvsii '{U+0940}' // DEVANAGARI_VOWEL_SIGN_II
+stringdef dvsu '{U+0941}' // DEVANAGARI_VOWEL_SIGN_U
+stringdef dvsuu '{U+0942}' // DEVANAGARI_VOWEL_SIGN_UU
+stringdef dvse '{U+0947}' // DEVANAGARI_VOWEL_SIGN_E
+stringdef dvsai '{U+0948}' // DEVANAGARI_VOWEL_SIGN_AI
+stringdef dvso '{U+094B}' // DEVANAGARI_VOWEL_SIGN_O
+stringdef dvsau '{U+094C}' // DEVANAGARI_VOWEL_SIGN_AU
+stringdef dsv '{U+094D}' // DEVANAGARI_SIGN_VIRAMA
+
+externals ( stem )
+backwardmode (
+ define remove_category_1 as(
+ [substring] among (
+ '{dlm}{dvsaa}{dlr}{dsv}{dlpha}{dlta}' '{dld}{dsv}{dlv}{dvsaa}{dlr}{dvsaa}' '{dls}{dsc}{dlg}{dvsai}' '{dls}{dsa}{dlg}'
+ '{dls}{dsc}{dlg}' '{dll}{dvsaa}{dli}' '{dll}{dvsaa}{dlii}' '{dlpa}{dlc}{dvsi}'
+ '{dll}{dvse}' '{dlr}{dlta}' '{dlm}{dvsai}' '{dlm}{dvsaa}'
+ (delete)
+ '{dlka}{dvso}' '{dlka}{dvsaa}' '{dlka}{dvsi}' '{dlka}{dvsii}' '{dlka}{dvsai}'(('{dle}' or '{dvse}' ()) or delete)
+ )
+ )
+
+ define check_category_2 as(
+ [substring] among(
+ '{dsc}' '{dsa}' '{dvsai}'
+ )
+ )
+
+ define remove_category_2 as (
+ [substring] among(
+ '{dsc}' '{dsa}' ('{dly}{dvsau}' or '{dlc}{dvsau}' or '{dln}{dvsau}' or '{dltha}{dvse}' delete)
+ '{dvsai}' ('{dlta}{dsv}{dlr}' delete)
+ )
+ )
+
+ define remove_category_3 as(
+ [substring] among(
+ '{dltha}{dvsi}{dli}{dls}{dsv}' '{dlh}{dvsu}{dln}{dvse}{dlc}' '{dlh}{dvsu}{dln}{dsv}{dlc}' '{dln}{dvse}{dlc}{dls}{dsv}' '{dln}{dvse}{dlc}{dln}{dsv}' '{dli}{dle}{dlka}{dvsii}' '{dli}{dle}{dlka}{dvsaa}' '{dli}{dle}{dlka}{dvso}' '{dvsi}{dle}{dlka}{dvsii}' '{dvsi}{dle}{dlka}{dvsaa}' '{dvsi}{dle}{dlka}{dvso}' '{dli}{dlc}{dln}{dsv}' '{dvsi}{dlc}{dln}{dsv}' '{dli}{dlc}{dls}{dsv}' '{dvsi}{dlc}{dls}{dsv}' '{dle}{dlc}{dln}{dsv}' '{dvse}{dlc}{dln}{dsv}' '{dle}{dlc}{dls}{dsv}' '{dvse}{dlc}{dls}{dsv}' '{dlc}{dvsi}{dln}{dsv}' '{dlc}{dvse}{dls}{dsv}' '{dlc}{dsv}{dly}{dvsau}' '{dltha}{dvsi}{dln}{dsv}' '{dltha}{dvsi}{dly}{dvso}' '{dltha}{dvsi}{dly}{dvsau}' '{dltha}{dvsi}{dls}{dsv}' '{dltha}{dsv}{dly}{dvso}' '{dltha}{dsv}{dly}{dvsau}' '{dld}{dvsi}{dly}{dvso}' '{dld}{dvse}{dlkha}{dvsi}' '{dld}{dvse}{dlkha}{dvsii}' '{dll}{dvsaa}{dln}{dsv}' '{dlm}{dvsaa}{dltha}{dvsi}' '{dln}{dvse}{dlka}{dvsai}' '{dln}{dvse}{dlka}{dvsaa}' '{dln}{dvse}{dlka}{dvso}' '{dln}{dvse}{dlc}{dvsau}' '{dlh}{dvso}{dls}{dsv}' '{dli}{dln}{dsv}{dlc}' '{dvsi}{dln}{dsv}{dlc}' '{dln}{dvse}{dlc}{dvsu}' '{dli}{dlc}{dvsau}' '{dvsi}{dlc}{dvsau}' '{dli}{dls}{dsv}' '{dvsi}{dls}{dsv}' '{dvsi}{dly}{dvso}' '{dli}{dly}{dvso}' '{dle}{dlka}{dvsaa}' '{dvse}{dlka}{dvsaa}' '{dle}{dlka}{dvsii}' '{dvse}{dlka}{dvsii}' '{dle}{dlka}{dvsai}' '{dvse}{dlka}{dvsai}' '{dle}{dlka}{dvso}' '{dvse}{dlka}{dvso}' '{dle}{dlc}{dvsu}' '{dvse}{dlc}{dvsu}' '{dle}{dlc}{dvsau}' '{dvse}{dlc}{dvsau}' '{dlc}{dln}{dsv}' '{dlc}{dls}{dsv}' '{dltha}{dvsi}{dle}' '{dlpa}{dlr}{dsv}' '{dlb}{dly}{dvso}' '{dlh}{dlr}{dvsu}' '{dlh}{dlr}{dvsuu}' '{dvsi}{dld}{dvsaa}' '{dli}{dld}{dvsaa}' '{dvsi}{dld}{dvso}' '{dli}{dld}{dvso}' '{dvsi}{dld}{dvsai}' '{dli}{dld}{dvsai}' '{dln}{dvse}{dlc}' '{dli}{dlc}' '{dvsi}{dlc}' '{dle}{dlc}' '{dvse}{dlc}' '{dlc}{dvsu}' '{dlc}{dvse}' '{dlc}{dvsau}' '{dltha}{dvsii}' '{dltha}{dvse}' '{dld}{dvsaa}' '{dld}{dvsii}' '{dld}{dvsai}' '{dld}{dvso}' '{dln}{dvsu}' '{dln}{dvse}' '{dly}{dvso}' '{dly}{dvsau}' '{dlc}'
+ (delete)
+ )
+ )
+
+)
+
+define stem as (
+ backwards (
+ do remove_category_1
+ do (
+ repeat (do (check_category_2 and remove_category_2) remove_category_3)
+ )
+ )
+)
diff --git a/contrib/snowball/algorithms/norwegian.sbl b/contrib/snowball/algorithms/norwegian.sbl
new file mode 100644
index 0000000..39f4aff
--- /dev/null
+++ b/contrib/snowball/algorithms/norwegian.sbl
@@ -0,0 +1,80 @@
+routines (
+ mark_regions
+ main_suffix
+ consonant_pair
+ other_suffix
+)
+
+externals ( stem )
+
+integers ( p1 x )
+
+groupings ( v s_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef ae '{U+00E6}'
+stringdef ao '{U+00E5}'
+stringdef o/ '{U+00F8}'
+
+define v 'aeiouy{ae}{ao}{o/}'
+
+define s_ending 'bcdfghjlmnoprtvyz'
+
+define mark_regions as (
+
+ $p1 = limit
+
+ test ( hop 3 setmark x )
+ goto v gopast non-v setmark p1
+ try ( $p1 < x $p1 = x )
+)
+
+backwardmode (
+
+ define main_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+
+ 'a' 'e' 'ede' 'ande' 'ende' 'ane' 'ene' 'hetene' 'en' 'heten' 'ar'
+ 'er' 'heter' 'as' 'es' 'edes' 'endes' 'enes' 'hetenes' 'ens'
+ 'hetens' 'ers' 'ets' 'et' 'het' 'ast'
+ (delete)
+ 's'
+ (s_ending or ('k' non-v) delete)
+ 'erte' 'ert'
+ (<-'er')
+ )
+ )
+
+ define consonant_pair as (
+ test (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'dt' 'vt'
+ )
+ )
+ next] delete
+ )
+
+ define other_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'leg' 'eleg' 'ig' 'eig' 'lig' 'elig' 'els' 'lov' 'elov' 'slov'
+ 'hetslov'
+ (delete)
+ )
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ backwards (
+ do main_suffix
+ do consonant_pair
+ do other_suffix
+ )
+)
diff --git a/contrib/snowball/algorithms/porter.sbl b/contrib/snowball/algorithms/porter.sbl
new file mode 100644
index 0000000..9533b79
--- /dev/null
+++ b/contrib/snowball/algorithms/porter.sbl
@@ -0,0 +1,139 @@
+integers ( p1 p2 )
+booleans ( Y_found )
+
+routines (
+ shortv
+ R1 R2
+ Step_1a Step_1b Step_1c Step_2 Step_3 Step_4 Step_5a Step_5b
+)
+
+externals ( stem )
+
+groupings ( v v_WXY )
+
+define v 'aeiouy'
+define v_WXY v + 'wxY'
+
+backwardmode (
+
+ define shortv as ( non-v_WXY v non-v )
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define Step_1a as (
+ [substring] among (
+ 'sses' (<-'ss')
+ 'ies' (<-'i')
+ 'ss' ()
+ 's' (delete)
+ )
+ )
+
+ define Step_1b as (
+ [substring] among (
+ 'eed' (R1 <-'ee')
+ 'ed'
+ 'ing' (
+ test gopast v delete
+ test substring among(
+ 'at' 'bl' 'iz'
+ (<+ 'e')
+ 'bb' 'dd' 'ff' 'gg' 'mm' 'nn' 'pp' 'rr' 'tt'
+ // ignoring double c, h, j, k, q, v, w, and x
+ ([next] delete)
+ '' (atmark p1 test shortv <+ 'e')
+ )
+ )
+ )
+ )
+
+ define Step_1c as (
+ ['y' or 'Y']
+ gopast v
+ <-'i'
+ )
+
+ define Step_2 as (
+ [substring] R1 among (
+ 'tional' (<-'tion')
+ 'enci' (<-'ence')
+ 'anci' (<-'ance')
+ 'abli' (<-'able')
+ 'entli' (<-'ent')
+ 'eli' (<-'e')
+ 'izer' 'ization'
+ (<-'ize')
+ 'ational' 'ation' 'ator'
+ (<-'ate')
+ 'alli' (<-'al')
+ 'alism' 'aliti'
+ (<-'al')
+ 'fulness' (<-'ful')
+ 'ousli' 'ousness'
+ (<-'ous')
+ 'iveness' 'iviti'
+ (<-'ive')
+ 'biliti' (<-'ble')
+ )
+ )
+
+ define Step_3 as (
+ [substring] R1 among (
+ 'alize' (<-'al')
+ 'icate' 'iciti' 'ical'
+ (<-'ic')
+ 'ative' 'ful' 'ness'
+ (delete)
+ )
+ )
+
+ define Step_4 as (
+ [substring] R2 among (
+ 'al' 'ance' 'ence' 'er' 'ic' 'able' 'ible' 'ant' 'ement'
+ 'ment' 'ent' 'ou' 'ism' 'ate' 'iti' 'ous' 'ive' 'ize'
+ (delete)
+ 'ion' ('s' or 't' delete)
+ )
+ )
+
+ define Step_5a as (
+ ['e']
+ R2 or (R1 not shortv)
+ delete
+ )
+
+ define Step_5b as (
+ ['l']
+ R2 'l'
+ delete
+ )
+)
+
+define stem as (
+
+ unset Y_found
+ do ( ['y'] <-'Y' set Y_found)
+ do repeat(goto (v ['y']) <-'Y' set Y_found)
+
+ $p1 = limit
+ $p2 = limit
+ do(
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+
+ backwards (
+ do Step_1a
+ do Step_1b
+ do Step_1c
+ do Step_2
+ do Step_3
+ do Step_4
+ do Step_5a
+ do Step_5b
+ )
+
+ do(Y_found repeat(goto (['Y']) <-'y'))
+
+)
diff --git a/contrib/snowball/algorithms/portuguese.sbl b/contrib/snowball/algorithms/portuguese.sbl
new file mode 100644
index 0000000..3fb14f1
--- /dev/null
+++ b/contrib/snowball/algorithms/portuguese.sbl
@@ -0,0 +1,218 @@
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ standard_suffix
+ verb_suffix
+ residual_suffix
+ residual_form
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef a^ '{U+00E2}' // a-circumflex e.g. 'bota^nico
+stringdef e' '{U+00E9}' // e-acute
+stringdef e^ '{U+00EA}' // e-circumflex
+stringdef i' '{U+00ED}' // i-acute
+stringdef o^ '{U+00F4}' // o-circumflex
+stringdef o' '{U+00F3}' // o-acute
+stringdef u' '{U+00FA}' // u-acute
+stringdef c, '{U+00E7}' // c-cedilla
+
+stringdef a~ '{U+00E3}' // a-tilde
+stringdef o~ '{U+00F5}' // o-tilde
+
+
+define v 'aeiou{a'}{e'}{i'}{o'}{u'}{a^}{e^}{o^}'
+
+define prelude as repeat (
+ [substring] among(
+ '{a~}' (<- 'a~')
+ '{o~}' (<- 'o~')
+ '' (next)
+ ) //or next
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+ [substring] among(
+ 'a~' (<- '{a~}')
+ 'o~' (<- '{o~}')
+ '' (next)
+ ) //or next
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'eza' 'ezas'
+ 'ico' 'ica' 'icos' 'icas'
+ 'ismo' 'ismos'
+ '{a'}vel'
+ '{i'}vel'
+ 'ista' 'istas'
+ 'oso' 'osa' 'osos' 'osas'
+ 'amento' 'amentos'
+ 'imento' 'imentos'
+
+ 'adora' 'ador' 'a{c,}a~o'
+ 'adoras' 'adores' 'a{c,}o~es' // no -ic test
+ 'ante' 'antes' '{a^}ncia' // Note 1
+ (
+ R2 delete
+ )
+ 'logia'
+ 'logias'
+ (
+ R2 <- 'log'
+ )
+ 'u{c,}a~o' 'u{c,}o~es'
+ (
+ R2 <- 'u'
+ )
+ '{e^}ncia' '{e^}ncias'
+ (
+ R2 <- 'ente'
+ )
+ 'amente'
+ (
+ R1 delete
+ try (
+ [substring] R2 delete among(
+ 'iv' (['at'] R2 delete)
+ 'os'
+ 'ic'
+ 'ad'
+ )
+ )
+ )
+ 'mente'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'ante' // Note 1
+ 'avel'
+ '{i'}vel' (R2 delete)
+ )
+ )
+ )
+ 'idade'
+ 'idades'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil'
+ 'ic'
+ 'iv' (R2 delete)
+ )
+ )
+ )
+ 'iva' 'ivo'
+ 'ivas' 'ivos'
+ (
+ R2 delete
+ try (
+ ['at'] R2 delete // but not a further ['ic'] R2 delete
+ )
+ )
+ 'ira' 'iras'
+ (
+ RV 'e' // -eira -eiras usually non-verbal
+ <- 'ir'
+ )
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among(
+ 'ada' 'ida' 'ia' 'aria' 'eria' 'iria' 'ar{a'}' 'ara' 'er{a'}'
+ 'era' 'ir{a'}' 'ava' 'asse' 'esse' 'isse' 'aste' 'este' 'iste'
+ 'ei' 'arei' 'erei' 'irei' 'am' 'iam' 'ariam' 'eriam' 'iriam'
+ 'aram' 'eram' 'iram' 'avam' 'em' 'arem' 'erem' 'irem' 'assem'
+ 'essem' 'issem' 'ado' 'ido' 'ando' 'endo' 'indo' 'ara~o'
+ 'era~o' 'ira~o' 'ar' 'er' 'ir' 'as' 'adas' 'idas' 'ias'
+ 'arias' 'erias' 'irias' 'ar{a'}s' 'aras' 'er{a'}s' 'eras'
+ 'ir{a'}s' 'avas' 'es' 'ardes' 'erdes' 'irdes' 'ares' 'eres'
+ 'ires' 'asses' 'esses' 'isses' 'astes' 'estes' 'istes' 'is'
+ 'ais' 'eis' '{i'}eis' 'ar{i'}eis' 'er{i'}eis' 'ir{i'}eis'
+ '{a'}reis' 'areis' '{e'}reis' 'ereis' '{i'}reis' 'ireis'
+ '{a'}sseis' '{e'}sseis' '{i'}sseis' '{a'}veis' 'ados' 'idos'
+ '{a'}mos' 'amos' '{i'}amos' 'ar{i'}amos' 'er{i'}amos'
+ 'ir{i'}amos' '{a'}ramos' '{e'}ramos' '{i'}ramos' '{a'}vamos'
+ 'emos' 'aremos' 'eremos' 'iremos' '{a'}ssemos' '{e^}ssemos'
+ '{i'}ssemos' 'imos' 'armos' 'ermos' 'irmos' 'eu' 'iu' 'ou'
+
+ 'ira' 'iras'
+ (delete)
+ )
+ )
+
+ define residual_suffix as (
+ [substring] among(
+ 'os'
+ 'a' 'i' 'o' '{a'}' '{i'}' '{o'}'
+ ( RV delete )
+ )
+ )
+
+ define residual_form as (
+ [substring] among(
+ 'e' '{e'}' '{e^}'
+ ( RV delete [('u'] test 'g') or
+ ('i'] test 'c') RV delete )
+ '{c,}' (<-'c')
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards (
+ do (
+ ( ( standard_suffix or verb_suffix )
+ and do ( ['i'] test 'c' RV delete )
+ )
+ or residual_suffix
+ )
+ do residual_form
+ )
+ do postlude
+)
+
+/*
+ Note 1: additions of 15 Jun 2005
+*/
diff --git a/contrib/snowball/algorithms/romanian.sbl b/contrib/snowball/algorithms/romanian.sbl
new file mode 100644
index 0000000..7db9e0a
--- /dev/null
+++ b/contrib/snowball/algorithms/romanian.sbl
@@ -0,0 +1,236 @@
+
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ step_0
+ standard_suffix combo_suffix
+ verb_suffix
+ vowel_suffix
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+booleans ( standard_suffix_removed )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a^ '{U+00E2}' // a circumflex
+stringdef i^ '{U+00EE}' // i circumflex
+stringdef a+ '{U+0103}' // a breve
+stringdef s, '{U+015F}' // s cedilla
+stringdef t, '{U+0163}' // t cedilla
+
+define v 'aeiou{a^}{i^}{a+}'
+
+define prelude as (
+ repeat goto (
+ v [ ('u' ] v <- 'U') or
+ ('i' ] v <- 'I')
+ )
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'I' (<- 'i')
+ 'U' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define step_0 as (
+ [substring] R1 among(
+ 'ul' 'ului'
+ ( delete )
+ 'aua'
+ ( <-'a' )
+ 'ea' 'ele' 'elor'
+ ( <-'e' )
+ 'ii' 'iua' 'iei' 'iile' 'iilor' 'ilor'
+ ( <-'i')
+ 'ile'
+ ( not 'ab' <- 'i' )
+ 'atei'
+ ( <- 'at' )
+ 'a{t,}ie' 'a{t,}ia'
+ ( <- 'a{t,}i' )
+ )
+ )
+
+ define combo_suffix as test (
+ [substring] R1 (
+ among(
+ /* 'IST'. alternative: include the following
+ 'alism' 'alisme'
+ 'alist' 'alista' 'aliste' 'alisti' 'alist{a+}' 'ali{s,}ti' (
+ <- 'al'
+ )
+ */
+ 'abilitate' 'abilitati' 'abilit{a+}i' 'abilit{a+}{t,}i' (
+ <- 'abil'
+ )
+ 'ibilitate' (
+ <- 'ibil'
+ )
+ 'ivitate' 'ivitati' 'ivit{a+}i' 'ivit{a+}{t,}i' (
+ <- 'iv'
+ )
+ 'icitate' 'icitati' 'icit{a+}i' 'icit{a+}{t,}i'
+ 'icator' 'icatori'
+ 'iciv' 'iciva' 'icive' 'icivi' 'iciv{a+}'
+ 'ical' 'icala' 'icale' 'icali' 'ical{a+}' (
+ <- 'ic'
+ )
+ 'ativ' 'ativa' 'ative' 'ativi' 'ativ{a+}' 'a{t,}iune'
+ 'atoare' 'ator' 'atori'
+ '{a+}toare' '{a+}tor' '{a+}tori' (
+ <- 'at'
+ )
+ 'itiv' 'itiva' 'itive' 'itivi' 'itiv{a+}' 'i{t,}iune'
+ 'itoare' 'itor' 'itori' (
+ <- 'it'
+ )
+ )
+ set standard_suffix_removed
+ )
+ )
+
+ define standard_suffix as (
+ unset standard_suffix_removed
+ repeat combo_suffix
+ [substring] R2 (
+ among(
+
+ // past participle is treated here, rather than
+ // as a verb ending:
+ 'at' 'ata' 'at{a+}' 'ati' 'ate'
+ 'ut' 'uta' 'ut{a+}' 'uti' 'ute'
+ 'it' 'ita' 'it{a+}' 'iti' 'ite'
+
+ 'ic' 'ica' 'ice' 'ici' 'ic{a+}'
+ 'abil' 'abila' 'abile' 'abili' 'abil{a+}'
+ 'ibil' 'ibila' 'ibile' 'ibili' 'ibil{a+}'
+ 'oasa' 'oas{a+}' 'oase' 'os' 'osi' 'o{s,}i'
+ 'ant' 'anta' 'ante' 'anti' 'ant{a+}'
+ 'ator' 'atori'
+ 'itate' 'itati' 'it{a+}i' 'it{a+}{t,}i'
+ 'iv' 'iva' 'ive' 'ivi' 'iv{a+}' (
+ delete
+ )
+ 'iune' 'iuni' (
+ '{t,}'] <- 't'
+ )
+ 'ism' 'isme'
+ 'ist' 'ista' 'iste' 'isti' 'ist{a+}' 'i{s,}ti' (
+ <- 'ist'
+ /* 'IST'. alternative: remove with <- '' */
+ )
+ )
+ set standard_suffix_removed
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among(
+ // 'long' infinitive:
+ 'are' 'ere' 'ire' '{a^}re'
+
+ // gerund:
+ 'ind' '{a^}nd'
+ 'indu' '{a^}ndu'
+
+ 'eze'
+ 'easc{a+}'
+ // present:
+ 'ez' 'ezi' 'eaz{a+}' 'esc' 'e{s,}ti'
+ 'e{s,}te'
+ '{a+}sc' '{a+}{s,}ti'
+ '{a+}{s,}te'
+
+ // imperfect:
+ 'am' 'ai' 'au'
+ 'eam' 'eai' 'ea' 'ea{t,}i' 'eau'
+ 'iam' 'iai' 'ia' 'ia{t,}i' 'iau'
+
+ // past: // (not 'ii')
+ 'ui'
+ 'a{s,}i' 'ar{a+}m' 'ar{a+}{t,}i' 'ar{a+}'
+ 'u{s,}i' 'ur{a+}m' 'ur{a+}{t,}i' 'ur{a+}'
+ 'i{s,}i' 'ir{a+}m' 'ir{a+}{t,}i' 'ir{a+}'
+ '{a^}i' '{a^}{s,}i' '{a^}r{a+}m' '{a^}r{a+}{t,}i' '{a^}r{a+}'
+
+ // pluferfect:
+ 'asem' 'ase{s,}i' 'ase' 'aser{a+}m' 'aser{a+}{t,}i' 'aser{a+}'
+ 'isem' 'ise{s,}i' 'ise' 'iser{a+}m' 'iser{a+}{t,}i' 'iser{a+}'
+ '{a^}sem' '{a^}se{s,}i' '{a^}se' '{a^}ser{a+}m' '{a^}ser{a+}{t,}i'
+ '{a^}ser{a+}'
+ 'usem' 'use{s,}i' 'use' 'user{a+}m' 'user{a+}{t,}i' 'user{a+}'
+
+ ( non-v or 'u' delete )
+
+ // present:
+ '{a+}m' 'a{t,}i'
+ 'em' 'e{t,}i'
+ 'im' 'i{t,}i'
+ '{a^}m' '{a^}{t,}i'
+
+ // past:
+ 'se{s,}i' 'ser{a+}m' 'ser{a+}{t,}i' 'ser{a+}'
+ 'sei' 'se'
+
+ // pluperfect:
+ 'sesem' 'sese{s,}i' 'sese' 'seser{a+}m' 'seser{a+}{t,}i' 'seser{a+}'
+ (delete)
+ )
+ )
+
+ define vowel_suffix as (
+ [substring] RV among (
+ 'a' 'e' 'i' 'ie' '{a+}' ( delete )
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards (
+ do step_0
+ do standard_suffix
+ do ( standard_suffix_removed or verb_suffix )
+ do vowel_suffix
+ )
+ do postlude
+)
+
diff --git a/contrib/snowball/algorithms/russian.sbl b/contrib/snowball/algorithms/russian.sbl
new file mode 100644
index 0000000..20de639
--- /dev/null
+++ b/contrib/snowball/algorithms/russian.sbl
@@ -0,0 +1,221 @@
+stringescapes {}
+
+/* the 33 Cyrillic letters represented in ASCII characters following the
+ * conventions of the standard Library of Congress transliteration: */
+
+stringdef a '{U+0430}'
+stringdef b '{U+0431}'
+stringdef v '{U+0432}'
+stringdef g '{U+0433}'
+stringdef d '{U+0434}'
+stringdef e '{U+0435}'
+stringdef e" '{U+0451}'
+stringdef zh '{U+0436}'
+stringdef z '{U+0437}'
+stringdef i '{U+0438}'
+stringdef i` '{U+0439}'
+stringdef k '{U+043A}'
+stringdef l '{U+043B}'
+stringdef m '{U+043C}'
+stringdef n '{U+043D}'
+stringdef o '{U+043E}'
+stringdef p '{U+043F}'
+stringdef r '{U+0440}'
+stringdef s '{U+0441}'
+stringdef t '{U+0442}'
+stringdef u '{U+0443}'
+stringdef f '{U+0444}'
+stringdef kh '{U+0445}'
+stringdef ts '{U+0446}'
+stringdef ch '{U+0447}'
+stringdef sh '{U+0448}'
+stringdef shch '{U+0449}'
+stringdef " '{U+044A}'
+stringdef y '{U+044B}'
+stringdef ' '{U+044C}'
+stringdef e` '{U+044D}'
+stringdef iu '{U+044E}'
+stringdef ia '{U+044F}'
+
+routines ( mark_regions R2
+ perfective_gerund
+ adjective
+ adjectival
+ reflexive
+ verb
+ noun
+ derivational
+ tidy_up
+)
+
+externals ( stem )
+
+integers ( pV p2 )
+
+groupings ( v )
+
+define v '{a}{e}{i}{o}{u}{y}{e`}{iu}{ia}'
+
+define mark_regions as (
+
+ $pV = limit
+ $p2 = limit
+ do (
+ gopast v setmark pV gopast non-v
+ gopast v gopast non-v setmark p2
+ )
+)
+
+backwardmode (
+
+ define R2 as $p2 <= cursor
+
+ define perfective_gerund as (
+ [substring] among (
+ '{v}'
+ '{v}{sh}{i}'
+ '{v}{sh}{i}{s}{'}'
+ ('{a}' or '{ia}' delete)
+ '{i}{v}'
+ '{i}{v}{sh}{i}'
+ '{i}{v}{sh}{i}{s}{'}'
+ '{y}{v}'
+ '{y}{v}{sh}{i}'
+ '{y}{v}{sh}{i}{s}{'}'
+ (delete)
+ )
+ )
+
+ define adjective as (
+ [substring] among (
+ '{e}{e}' '{i}{e}' '{y}{e}' '{o}{e}' '{i}{m}{i}' '{y}{m}{i}'
+ '{e}{i`}' '{i}{i`}' '{y}{i`}' '{o}{i`}' '{e}{m}' '{i}{m}'
+ '{y}{m}' '{o}{m}' '{e}{g}{o}' '{o}{g}{o}' '{e}{m}{u}'
+ '{o}{m}{u}' '{i}{kh}' '{y}{kh}' '{u}{iu}' '{iu}{iu}' '{a}{ia}'
+ '{ia}{ia}'
+ // and -
+ '{o}{iu}' // - which is somewhat archaic
+ '{e}{iu}' // - soft form of {o}{iu}
+ (delete)
+ )
+ )
+
+ define adjectival as (
+ adjective
+
+ /* of the participle forms, em, vsh, ivsh, yvsh are readily removable.
+ nn, {iu}shch, shch, u{iu}shch can be removed, with a small proportion of
+ errors. Removing im, uem, enn creates too many errors.
+ */
+
+ try (
+ [substring] among (
+ '{e}{m}' // present passive participle
+ '{n}{n}' // adjective from past passive participle
+ '{v}{sh}' // past active participle
+ '{iu}{shch}' '{shch}' // present active participle
+ ('{a}' or '{ia}' delete)
+
+ //but not '{i}{m}' '{u}{e}{m}' // present passive participle
+ //or '{e}{n}{n}' // adjective from past passive participle
+
+ '{i}{v}{sh}' '{y}{v}{sh}'// past active participle
+ '{u}{iu}{shch}' // present active participle
+ (delete)
+ )
+ )
+
+ )
+
+ define reflexive as (
+ [substring] among (
+ '{s}{ia}'
+ '{s}{'}'
+ (delete)
+ )
+ )
+
+ define verb as (
+ [substring] among (
+ '{l}{a}' '{n}{a}' '{e}{t}{e}' '{i`}{t}{e}' '{l}{i}' '{i`}'
+ '{l}' '{e}{m}' '{n}' '{l}{o}' '{n}{o}' '{e}{t}' '{iu}{t}'
+ '{n}{y}' '{t}{'}' '{e}{sh}{'}'
+
+ '{n}{n}{o}'
+ ('{a}' or '{ia}' delete)
+
+ '{i}{l}{a}' '{y}{l}{a}' '{e}{n}{a}' '{e}{i`}{t}{e}'
+ '{u}{i`}{t}{e}' '{i}{t}{e}' '{i}{l}{i}' '{y}{l}{i}' '{e}{i`}'
+ '{u}{i`}' '{i}{l}' '{y}{l}' '{i}{m}' '{y}{m}' '{e}{n}'
+ '{i}{l}{o}' '{y}{l}{o}' '{e}{n}{o}' '{ia}{t}' '{u}{e}{t}'
+ '{u}{iu}{t}' '{i}{t}' '{y}{t}' '{e}{n}{y}' '{i}{t}{'}'
+ '{y}{t}{'}' '{i}{sh}{'}' '{u}{iu}' '{iu}'
+ (delete)
+ /* note the short passive participle tests:
+ '{n}{a}' '{n}' '{n}{o}' '{n}{y}'
+ '{e}{n}{a}' '{e}{n}' '{e}{n}{o}' '{e}{n}{y}'
+ */
+ )
+ )
+
+ define noun as (
+ [substring] among (
+ '{a}' '{e}{v}' '{o}{v}' '{i}{e}' '{'}{e}' '{e}'
+ '{i}{ia}{m}{i}' '{ia}{m}{i}' '{a}{m}{i}' '{e}{i}' '{i}{i}'
+ '{i}' '{i}{e}{i`}' '{e}{i`}' '{o}{i`}' '{i}{i`}' '{i`}'
+ '{i}{ia}{m}' '{ia}{m}' '{i}{e}{m}' '{e}{m}' '{a}{m}' '{o}{m}'
+ '{o}' '{u}' '{a}{kh}' '{i}{ia}{kh}' '{ia}{kh}' '{y}' '{'}'
+ '{i}{iu}' '{'}{iu}' '{iu}' '{i}{ia}' '{'}{ia}' '{ia}'
+ (delete)
+ /* the small class of neuter forms '{e}{n}{i}' '{e}{n}{e}{m}'
+ '{e}{n}{a}' '{e}{n}' '{e}{n}{a}{m}' '{e}{n}{a}{m}{i}' '{e}{n}{a}{x}'
+ omitted - they only occur on 12 words.
+ */
+ )
+ )
+
+ define derivational as (
+ [substring] R2 among (
+ '{o}{s}{t}'
+ '{o}{s}{t}{'}'
+ (delete)
+ )
+ )
+
+ define tidy_up as (
+ [substring] among (
+
+ '{e}{i`}{sh}'
+ '{e}{i`}{sh}{e}' // superlative forms
+ (delete
+ ['{n}'] '{n}' delete
+ )
+ '{n}'
+ ('{n}' delete) // e.g. -nno endings
+ '{'}'
+ (delete) // with some slight false conflations
+ )
+ )
+)
+
+define stem as (
+
+ // Normalise {e"} to {e}. The documentation has long suggested the user
+ // should do this before calling the stemmer - we now do it for them.
+ do repeat ( goto (['{e"}']) <- '{e}' )
+
+ do mark_regions
+ backwards setlimit tomark pV for (
+ do (
+ perfective_gerund or
+ ( try reflexive
+ adjectival or verb or noun
+ )
+ )
+ try([ '{i}' ] delete)
+ // because noun ending -i{iu} is being treated as verb ending -{iu}
+
+ do derivational
+ do tidy_up
+ )
+)
diff --git a/contrib/snowball/algorithms/serbian.sbl b/contrib/snowball/algorithms/serbian.sbl
new file mode 100644
index 0000000..bddf76b
--- /dev/null
+++ b/contrib/snowball/algorithms/serbian.sbl
@@ -0,0 +1,2378 @@
+/* Stemmer for Serbian language, based on:
+ *
+ * Ljubesic, Nikola. Pandzic, Ivan. Stemmer for Croatian
+ * http://nlp.ffzg.hr/resources/tools/stemmer-for-croatian/
+ *
+ * authors: Stefan Petkovic and Dragan Ivanovic
+ * emails: petkovic8 at gmail.com and dragan.ivanovic at uns.ac.rs
+ * version: 1.0 (20.04.2019)
+*/
+
+routines (
+ cyr_to_lat
+ prelude
+ mark_regions
+ R1 R2
+ Step_1
+ Step_2
+ Step_3
+)
+
+externals ( stem )
+
+integers ( p1 p2 p3 )
+
+groupings ( v ca sa rg )
+
+stringescapes {}
+
+/* special characters - Unicode codepoints */
+
+/* serbian cyrillic */
+
+stringdef cyrA '{U+0430}'
+stringdef cyrB '{U+0431}'
+stringdef cyrV '{U+0432}'
+stringdef cyrG '{U+0433}'
+stringdef cyrD '{U+0434}'
+stringdef cyrDx '{U+0452}'
+stringdef cyrE '{U+0435}'
+stringdef cyrZh '{U+0436}'
+stringdef cyrZ '{U+0437}'
+stringdef cyrI '{U+0438}'
+stringdef cyrJ '{U+0458}'
+stringdef cyrK '{U+043A}'
+stringdef cyrL '{U+043B}'
+stringdef cyrLJ '{U+0459}'
+stringdef cyrM '{U+043C}'
+stringdef cyrN '{U+043D}'
+stringdef cyrNJ '{U+045A}'
+stringdef cyrO '{U+043E}'
+stringdef cyrP '{U+043F}'
+stringdef cyrR '{U+0440}'
+stringdef cyrS '{U+0441}'
+stringdef cyrT '{U+0442}'
+stringdef cyrCy '{U+045B}'
+stringdef cyrU '{U+0443}'
+stringdef cyrF '{U+0444}'
+stringdef cyrH '{U+0445}'
+stringdef cyrC '{U+0446}'
+stringdef cyrCx '{U+0447}'
+stringdef cyrDzx '{U+045F}'
+stringdef cyrSx '{U+0448}'
+
+/* serbian latin with diacritics */
+
+stringdef cx '{U+010D}' // small c with caron
+stringdef cy '{U+0107}' // small c with acute
+stringdef zx '{U+017E}' // small z with caron
+stringdef sx '{U+0161}' // small s with caron
+stringdef dx '{U+0111}' // small d with stroke
+
+define v 'aeiou'
+define sa '{cx}{cy}{zx}{sx}{dx}'
+define ca 'bvgdzjklmnprstfhc' + sa
+define rg 'r'
+
+
+define cyr_to_lat as (
+
+ do repeat goto (
+ [substring] among (
+ '{cyrA}' (<- 'a')
+ '{cyrB}' (<- 'b')
+ '{cyrV}' (<- 'v')
+ '{cyrG}' (<- 'g')
+ '{cyrD}' (<- 'd')
+ '{cyrDx}' (<- '{dx}')
+ '{cyrE}' (<- 'e')
+ '{cyrZh}' (<- '{zx}')
+ '{cyrZ}' (<- 'z')
+ '{cyrI}' (<- 'i')
+ '{cyrJ}' (<- 'j')
+ '{cyrK}' (<- 'k')
+ '{cyrL}' (<- 'l')
+ '{cyrLJ}' (<- 'lj')
+ '{cyrM}' (<- 'm')
+ '{cyrN}' (<- 'n')
+ '{cyrNJ}' (<- 'nj')
+ '{cyrO}' (<- 'o')
+ '{cyrP}' (<- 'p')
+ '{cyrR}' (<- 'r')
+ '{cyrS}' (<- 's')
+ '{cyrT}' (<- 't')
+ '{cyrCy}' (<- '{cy}')
+ '{cyrU}' (<- 'u')
+ '{cyrF}' (<- 'f')
+ '{cyrH}' (<- 'h')
+ '{cyrC}' (<- 'c')
+ '{cyrCx}' (<- '{cx}')
+ '{cyrDzx}' (<- 'd{zx}')
+ '{cyrSx}' (<- '{sx}')
+ )
+ )
+
+)
+
+define prelude as (
+
+ do repeat goto (
+ ca ['ije'] ca <- 'e'
+ )
+
+ do repeat goto (
+ ca ['je'] ca <- 'e'
+ )
+
+ do repeat goto (
+ ['dj'] <- '{dx}'
+ )
+
+)
+
+define mark_regions as (
+
+ $p3 = 0
+
+ do (
+ gopast sa setmark p3
+ )
+
+ $p1 = limit
+ $p2 = 0
+
+ do (
+ gopast v setmark p1
+ )
+ do (
+ gopast 'r' setmark p2
+ $(p1 - p2 > 1) ($p1 = p2)
+ )
+ ($p1 < 2) (
+ ($p1 == p2 gopast 'r' gopast non-rg) or ($p1 != p2 gopast v gopast non-v)
+ setmark p1
+ )
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p3 == 0
+
+ define Step_1 as (
+ [substring] among (
+ 'lozi'
+ 'lozima' (<-'loga')
+ 'pesi'
+ 'pesima' (<-'peh')
+ 'vojci' (<-'vojka')
+ 'bojci' (<-'bojka')
+ 'jaci'
+ 'jacima' (<-'jak')
+ '{cx}ajan' (<-'{cx}ajni')
+ 'cajan' (R2 <-'cajni')
+ 'eran' (<-'erni')
+ 'laran' (<-'larni')
+ 'esan' (<-'esni')
+ 'anjac' (<-'anjca')
+ 'ajac'
+ 'ajaca' (<-'ajca')
+ 'ljaca'
+ 'ljac' (<-'ljca')
+ 'ejac'
+ 'ejaca' (<-'ejca')
+ 'ojac'
+ 'ojaca' (<-'ojca')
+ 'ajaka' (<-'ajka')
+ 'ojaka' (<-'ojka')
+ '{sx}aca'
+ '{sx}ac' (<-'{sx}ca')
+ 'inzima'
+ 'inzi' (<-'ing')
+ 'tvenici' (<-'tvenik')
+ 'tetici'
+ 'teticima' (<-'tetika')
+ 'nstava' (<-'nstva')
+ 'nicima' (<-'nik')
+ 'ticima' (<-'tik')
+ 'zicima' (<-'zik')
+ 'snici' (<-'snik')
+ 'kuse' (<-'kusi')
+ 'kusan' (<-'kusni')
+ 'kustava' (<-'kustva')
+ 'du{sx}an' (<-'du{sx}ni')
+ 'dusan' (R2 <-'dusni')
+ 'antan' (<-'antni')
+ 'bilan' (<-'bilni')
+ 'tilan' (<-'tilni')
+ 'avilan' (<-'avilni')
+ 'silan' (<-'silni')
+ 'gilan' (<-'gilni')
+ 'rilan' (<-'rilni')
+ 'nilan' (<-'nilni')
+ 'alan' (<-'alni')
+ 'ozan' (<-'ozni')
+ 'rave' (<-'ravi')
+ 'stavan' (<-'stavni')
+ 'pravan' (<-'pravni')
+ 'tivan' (<-'tivni')
+ 'sivan' (<-'sivni')
+ 'atan' (<-'atni')
+ 'enat' (<-'enta')
+ 'tetan' (<-'tetni')
+ 'pletan' (<-'pletni')
+ '{sx}ave' (<-'{sx}avi')
+ 'save' (R2 <-'savi')
+ 'anata' (<-'anta')
+ 'a{cx}ak'
+ 'a{cx}aka' (<-'a{cx}ka')
+ 'acak'
+ 'acaka' (R2 <-'acka')
+ 'u{sx}ak' (<-'u{sx}ka')
+ 'usak' (R2 <-'uska')
+ 'atak'
+ 'ataka'
+ 'atci'
+ 'atcima' (<-'atka')
+ 'etak'
+ 'etaka' (<-'etka')
+ 'itak'
+ 'itaka'
+ 'itci' (<-'itka')
+ 'otak'
+ 'otaka' (<-'otka')
+ 'utak'
+ 'utaka'
+ 'utci'
+ 'utcima' (<-'utka')
+ 'eskan' (<-'eskna')
+ 'ti{cx}an' (<-'ti{cx}ni')
+ 'tican' (R2 <-'ticni')
+ 'ojsci' (<-'ojska')
+ 'esama' (<-'esma')
+ 'metar'
+ 'metara' (<-'metra')
+ 'centar'
+ 'centara' (<-'centra')
+ 'istar'
+ 'istara' (<-'istra')
+ 'o{sx}{cy}u' (<-'osti')
+ 'oscu' (R2 <-'osti')
+ 'daba' (<-'dba')
+ '{cx}cima'
+ '{cx}ci' (<-'{cx}ka')
+ 'mac'
+ 'maca' (<-'mca')
+ 'naca'
+ 'nac' (<-'nca')
+ 'voljan' (<-'voljni')
+ 'anaka' (<-'anki')
+ 'vac'
+ 'vaca' (<-'vca')
+ 'saca'
+ 'sac' (<-'sca')
+ 'raca'
+ 'rac' (<-'rca')
+ 'aoca'
+ 'alaca'
+ 'alac' (<-'alca')
+ 'elaca'
+ 'elac' (<-'elca')
+ 'olaca'
+ 'olac'
+ 'olce' (<-'olca')
+ 'njac'
+ 'njaca' (<-'njca')
+ 'ekata'
+ 'ekat' (<-'ekta')
+ 'izam'
+ 'izama' (<-'izma')
+ 'jebe' (<-'jebi')
+ 'baci' (<-'baci')
+ 'a{sx}an' (<-'a{sx}ni')
+ 'asan' (R2 <-'asni')
+ )
+ )
+
+ define Step_2 as (
+ [substring] R1 among (
+ 'skijima'
+ 'skijega'
+ 'skijemu'
+ 'skijem'
+ 'skega'
+ 'skemu'
+ 'skem'
+ 'skijim'
+ 'skijih'
+ 'skijoj'
+ 'skijeg'
+ 'skiji'
+ 'skije'
+ 'skija'
+ 'skoga'
+ 'skome'
+ 'skomu'
+ 'skima'
+ 'skog'
+ 'skom'
+ 'skim'
+ 'skih'
+ 'skoj'
+ 'ski'
+ 'ske'
+ 'sko'
+ 'ska'
+ 'sku' (<-'sk')
+ '{sx}kijima'
+ '{sx}kijega'
+ '{sx}kijemu'
+ '{sx}kijem'
+ '{sx}kega'
+ '{sx}kemu'
+ '{sx}kem'
+ '{sx}kijim'
+ '{sx}kijih'
+ '{sx}kijoj'
+ '{sx}kijeg'
+ '{sx}kiji'
+ '{sx}kije'
+ '{sx}kija'
+ '{sx}koga'
+ '{sx}kome'
+ '{sx}komu'
+ '{sx}kima'
+ '{sx}kog'
+ '{sx}kom'
+ '{sx}kim'
+ '{sx}kih'
+ '{sx}koj'
+ '{sx}ki'
+ '{sx}ke'
+ '{sx}ko'
+ '{sx}ka'
+ '{sx}ku' (<-'{sx}k')
+ 'stvima'
+ 'stvom'
+ 'stvo'
+ 'stva'
+ 'stvu' (<-'stv')
+ '{sx}tvima'
+ '{sx}tvom'
+ '{sx}tvo'
+ '{sx}tva'
+ '{sx}tvu' (<-'{sx}tv')
+ 'tanijama'
+ 'tanijima'
+ 'tanijom'
+ 'tanija'
+ 'taniju'
+ 'tanije'
+ 'taniji' (<-'tanij')
+ 'manijama'
+ 'manijima'
+ 'manijom'
+ 'manija'
+ 'maniju'
+ 'manije'
+ 'maniji' (<-'manij')
+ 'panijama'
+ 'panijima'
+ 'panijom'
+ 'panija'
+ 'paniju'
+ 'panije'
+ 'paniji' (<-'panij')
+ 'ranijama'
+ 'ranijima'
+ 'ranijom'
+ 'ranija'
+ 'raniju'
+ 'ranije'
+ 'raniji' (<-'ranij')
+ 'ganijama'
+ 'ganijima'
+ 'ganijom'
+ 'ganija'
+ 'ganiju'
+ 'ganije'
+ 'ganiji' (<-'ganij')
+ 'aninom'
+ 'anina'
+ 'aninu'
+ 'anine'
+ 'anima'
+ 'anin'
+ 'anom'
+ 'anu'
+ 'ani'
+ 'ana'
+ 'ane' (<-'an')
+ 'inima'
+ 'inama'
+ 'inom'
+ 'ina'
+ 'ine'
+ 'ini'
+ 'inu'
+ 'ino' (<-'in')
+ 'onovima'
+ 'onova'
+ 'onove'
+ 'onovi'
+ 'onima'
+ 'onom'
+ 'ona'
+ 'one'
+ 'oni'
+ 'onu' (<-'on')
+ 'nijima'
+ 'nijega'
+ 'nijemu'
+ 'nijeg'
+ 'nijem'
+ 'nega'
+ 'nemu'
+ 'neg'
+ 'nem'
+ 'nijim'
+ 'nijih'
+ 'nijoj'
+ 'niji'
+ 'nije'
+ 'nija'
+ 'niju'
+ 'nima'
+ 'nome'
+ 'nomu'
+ 'noga'
+ 'noj'
+ 'nom'
+ 'nih'
+ 'nim'
+ 'nog'
+ 'no'
+ 'ne'
+ 'na'
+ 'nu'
+ 'ni' (<-'n')
+ 'a{cy}oga'
+ 'a{cy}ome'
+ 'a{cy}omu'
+ 'a{cy}ega'
+ 'a{cy}emu'
+ 'a{cy}ima'
+ 'a{cy}oj'
+ 'a{cy}ih'
+ 'a{cy}om'
+ 'a{cy}eg'
+ 'a{cy}em'
+ 'a{cy}og'
+ 'a{cy}uh'
+ 'a{cy}im'
+ 'a{cy}e'
+ 'a{cy}a' (<-'a{cy}')
+ 'e{cy}oga'
+ 'e{cy}ome'
+ 'e{cy}omu'
+ 'e{cy}ega'
+ 'e{cy}emu'
+ 'e{cy}ima'
+ 'e{cy}oj'
+ 'e{cy}ih'
+ 'e{cy}om'
+ 'e{cy}eg'
+ 'e{cy}em'
+ 'e{cy}og'
+ 'e{cy}uh'
+ 'e{cy}im'
+ 'e{cy}e'
+ 'e{cy}a' (<-'e{cy}')
+ 'u{cy}oga'
+ 'u{cy}ome'
+ 'u{cy}omu'
+ 'u{cy}ega'
+ 'u{cy}emu'
+ 'u{cy}ima'
+ 'u{cy}oj'
+ 'u{cy}ih'
+ 'u{cy}om'
+ 'u{cy}eg'
+ 'u{cy}em'
+ 'u{cy}og'
+ 'u{cy}uh'
+ 'u{cy}im'
+ 'u{cy}e'
+ 'u{cy}a' (<-'u{cy}')
+ 'ugovima'
+ 'ugovi'
+ 'ugove'
+ 'ugova' (<-'ugov')
+ 'ugama'
+ 'ugom'
+ 'uga'
+ 'uge'
+ 'ugi'
+ 'ugu'
+ 'ugo' (<-'ug')
+ 'logama'
+ 'logom'
+ 'loga'
+ 'logu'
+ 'loge' (<-'log')
+ 'govima'
+ 'gama'
+ 'govi'
+ 'gove'
+ 'gova'
+ 'gom'
+ 'ga'
+ 'ge'
+ 'gi'
+ 'gu'
+ 'go' (<-'g')
+ 'rarijem'
+ 'rarija'
+ 'rariju'
+ 'rario' (<-'rari')
+ 'otijem'
+ 'otija'
+ 'otiju'
+ 'otio' (<-'oti')
+ 'sijem'
+ 'sija'
+ 'siju'
+ 'sio' (<-'si')
+ 'lijem'
+ 'lija'
+ 'liju'
+ 'lio' (<-'li')
+ 'uju{cy}i'
+ 'ujemo'
+ 'ujete'
+ 'ujmo'
+ 'ujem'
+ 'uje{sx}'
+ 'uje'
+ 'uju' (<-'uj')
+ 'cajevima'
+ 'cajevi'
+ 'cajeva'
+ 'cajeve'
+ 'cajama'
+ 'cajima'
+ 'cajem'
+ 'caja'
+ 'caje'
+ 'caji'
+ 'caju' (<-'caj')
+ '{cx}ajevima'
+ '{cx}ajevi'
+ '{cx}ajeva'
+ '{cx}ajeve'
+ '{cx}ajama'
+ '{cx}ajima'
+ '{cx}ajem'
+ '{cx}aja'
+ '{cx}aje'
+ '{cx}aji'
+ '{cx}aju' (<-'{cx}aj')
+ '{cy}ajevima'
+ '{cy}ajevi'
+ '{cy}ajeva'
+ '{cy}ajeve'
+ '{cy}ajama'
+ '{cy}ajima'
+ '{cy}ajem'
+ '{cy}aja'
+ '{cy}aje'
+ '{cy}aji'
+ '{cy}aju' (<-'{cy}aj')
+ '{dx}ajevima'
+ '{dx}ajevi'
+ '{dx}ajeva'
+ '{dx}ajeve'
+ '{dx}ajama'
+ '{dx}ajima'
+ '{dx}ajem'
+ '{dx}aja'
+ '{dx}aje'
+ '{dx}aji'
+ '{dx}aju' (<-'{dx}aj')
+ 'lajevima'
+ 'lajevi'
+ 'lajeva'
+ 'lajeve'
+ 'lajama'
+ 'lajima'
+ 'lajem'
+ 'laja'
+ 'laje'
+ 'laji'
+ 'laju' (<-'laj')
+ 'rajevima'
+ 'rajevi'
+ 'rajeva'
+ 'rajeve'
+ 'rajama'
+ 'rajima'
+ 'rajem'
+ 'raja'
+ 'raje'
+ 'raji'
+ 'raju' (<-'raj')
+ 'bijima'
+ 'bijama'
+ 'bijom'
+ 'bija'
+ 'bije'
+ 'biji'
+ 'biju'
+ 'bijo' (<-'bij')
+ 'cijima'
+ 'cijama'
+ 'cijom'
+ 'cija'
+ 'cije'
+ 'ciji'
+ 'ciju'
+ 'cijo' (<-'cij')
+ 'dijima'
+ 'dijama'
+ 'dijom'
+ 'dija'
+ 'dije'
+ 'diji'
+ 'diju'
+ 'dijo' (<-'dij')
+ 'lijima'
+ 'lijama'
+ 'lijom'
+ 'lije'
+ 'liji'
+ 'lijo' (<-'lij')
+ 'nijama'
+ 'nijom'
+ 'nijo' (<-'nij')
+ 'mijima'
+ 'mijama'
+ 'mijom'
+ 'mija'
+ 'mije'
+ 'miji'
+ 'miju'
+ 'mijo' (<-'mij')
+ '{zx}ijima'
+ '{zx}ijama'
+ '{zx}ijom'
+ '{zx}ija'
+ '{zx}ije'
+ '{zx}iji'
+ '{zx}iju'
+ '{zx}ijo' (<-'{zx}ij')
+ 'gijima'
+ 'gijama'
+ 'gijom'
+ 'gija'
+ 'gije'
+ 'giji'
+ 'giju'
+ 'gijo' (<-'gij')
+ 'fijima'
+ 'fijama'
+ 'fijom'
+ 'fija'
+ 'fije'
+ 'fiji'
+ 'fiju'
+ 'fijo' (<-'fij')
+ 'pijima'
+ 'pijama'
+ 'pijom'
+ 'pija'
+ 'pije'
+ 'piji'
+ 'piju'
+ 'pijo' (<-'pij')
+ 'rijima'
+ 'rijama'
+ 'rijom'
+ 'rija'
+ 'rije'
+ 'riji'
+ 'riju'
+ 'rijo' (<-'rij')
+ 'sijima'
+ 'sijama'
+ 'sijom'
+ 'sije'
+ 'siji'
+ 'sijo' (<-'sij')
+ 'tijima'
+ 'tijama'
+ 'tijom'
+ 'tija'
+ 'tije'
+ 'tiji'
+ 'tiju'
+ 'tijo' (<-'tij')
+ 'zijima'
+ 'zijama'
+ 'zijom'
+ 'zija'
+ 'zije'
+ 'ziji'
+ 'ziju'
+ 'zijo' (<-'zij')
+ 'nalima'
+ 'nalama'
+ 'nalom'
+ 'nala'
+ 'nale'
+ 'nali'
+ 'nalu'
+ 'nalo' (<-'nal')
+ 'ijalima'
+ 'ijalama'
+ 'ijalom'
+ 'ijala'
+ 'ijale'
+ 'ijali'
+ 'ijalu'
+ 'ijalo' (<-'ijal')
+ 'ozilima'
+ 'ozilom'
+ 'ozila'
+ 'ozile'
+ 'ozilu'
+ 'ozili' (<-'ozil')
+ 'olovima'
+ 'olovi'
+ 'olova'
+ 'olove' (<-'olov')
+ 'olima'
+ 'olom'
+ 'ola'
+ 'olu'
+ 'ole'
+ 'oli' (<-'ol')
+ 'lemama'
+ 'lemima'
+ 'lemom'
+ 'lema'
+ 'leme'
+ 'lemi'
+ 'lemu'
+ 'lemo' (<-'lem')
+ 'ramama'
+ 'ramom'
+ 'rama'
+ 'rame'
+ 'rami'
+ 'ramu'
+ 'ramo' (<-'ram')
+ 'arama'
+ 'arima'
+ 'arom'
+ 'aru'
+ 'ara'
+ 'are'
+ 'ari' (<-'ar')
+ 'drama'
+ 'drima'
+ 'drom'
+ 'dru'
+ 'dra'
+ 'dre'
+ 'dri' (<-'dr')
+ 'erama'
+ 'erima'
+ 'erom'
+ 'eru'
+ 'era'
+ 'ere'
+ 'eri' (<-'er')
+ 'orama'
+ 'orima'
+ 'orom'
+ 'oru'
+ 'ora'
+ 'ore'
+ 'ori' (<-'or')
+ 'esima'
+ 'esom'
+ 'ese'
+ 'esa'
+ 'esu' (<-'es')
+ 'isima'
+ 'isom'
+ 'ise'
+ 'isa'
+ 'isu' (<-'is')
+ 'ta{sx}ama'
+ 'ta{sx}ima'
+ 'ta{sx}om'
+ 'ta{sx}em'
+ 'ta{sx}a'
+ 'ta{sx}u'
+ 'ta{sx}i'
+ 'ta{sx}e' (<-'ta{sx}')
+ 'na{sx}ama'
+ 'na{sx}ima'
+ 'na{sx}om'
+ 'na{sx}em'
+ 'na{sx}a'
+ 'na{sx}u'
+ 'na{sx}i'
+ 'na{sx}e' (<-'na{sx}')
+ 'ja{sx}ama'
+ 'ja{sx}ima'
+ 'ja{sx}om'
+ 'ja{sx}em'
+ 'ja{sx}a'
+ 'ja{sx}u'
+ 'ja{sx}i'
+ 'ja{sx}e' (<-'ja{sx}')
+ 'ka{sx}ama'
+ 'ka{sx}ima'
+ 'ka{sx}om'
+ 'ka{sx}em'
+ 'ka{sx}a'
+ 'ka{sx}u'
+ 'ka{sx}i'
+ 'ka{sx}e' (<-'ka{sx}')
+ 'ba{sx}ama'
+ 'ba{sx}ima'
+ 'ba{sx}om'
+ 'ba{sx}em'
+ 'ba{sx}a'
+ 'ba{sx}u'
+ 'ba{sx}i'
+ 'ba{sx}e' (<-'ba{sx}')
+ 'ga{sx}ama'
+ 'ga{sx}ima'
+ 'ga{sx}om'
+ 'ga{sx}em'
+ 'ga{sx}a'
+ 'ga{sx}u'
+ 'ga{sx}i'
+ 'ga{sx}e' (<-'ga{sx}')
+ 'va{sx}ama'
+ 'va{sx}ima'
+ 'va{sx}om'
+ 'va{sx}em'
+ 'va{sx}a'
+ 'va{sx}u'
+ 'va{sx}i'
+ 'va{sx}e' (<-'va{sx}')
+ 'e{sx}ima'
+ 'e{sx}ama'
+ 'e{sx}om'
+ 'e{sx}em'
+ 'e{sx}i'
+ 'e{sx}e'
+ 'e{sx}a'
+ 'e{sx}u' (<-'e{sx}')
+ 'i{sx}ima'
+ 'i{sx}ama'
+ 'i{sx}om'
+ 'i{sx}em'
+ 'i{sx}i'
+ 'i{sx}e'
+ 'i{sx}a'
+ 'i{sx}u' (<-'i{sx}')
+ 'ikatima'
+ 'ikatom'
+ 'ikata'
+ 'ikate'
+ 'ikati'
+ 'ikatu'
+ 'ikato' (<-'ikat')
+ 'latima'
+ 'latom'
+ 'lata'
+ 'late'
+ 'lati'
+ 'latu'
+ 'lato' (<-'lat')
+ 'etama'
+ 'etima'
+ 'etom'
+ 'eta'
+ 'ete'
+ 'eti'
+ 'etu'
+ 'eto' (<-'et')
+ 'estima'
+ 'estama'
+ 'estom'
+ 'esta'
+ 'este'
+ 'esti'
+ 'estu'
+ 'esto' (<-'est')
+ 'istima'
+ 'istama'
+ 'istom'
+ 'ista'
+ 'iste'
+ 'isti'
+ 'istu'
+ 'isto' (<-'ist')
+ 'kstima'
+ 'kstama'
+ 'kstom'
+ 'ksta'
+ 'kste'
+ 'ksti'
+ 'kstu'
+ 'ksto' (<-'kst')
+ 'ostima'
+ 'ostama'
+ 'ostom'
+ 'osta'
+ 'oste'
+ 'osti'
+ 'ostu'
+ 'osto' (<-'ost')
+ 'i{sx}tima'
+ 'i{sx}tem'
+ 'i{sx}ta'
+ 'i{sx}te'
+ 'i{sx}tu' (<-'i{sx}t')
+ 'ovasmo'
+ 'ovaste'
+ 'ovahu'
+ 'ovati'
+ 'ova{sx}e'
+ 'ovali'
+ 'ovala'
+ 'ovale'
+ 'ovalo'
+ 'ovat'
+ 'ovah'
+ 'ovao' (<-'ova')
+ 'avijemu'
+ 'avijima'
+ 'avijega'
+ 'avijeg'
+ 'avijem'
+ 'avemu'
+ 'avega'
+ 'aveg'
+ 'avem'
+ 'avijim'
+ 'avijih'
+ 'avijoj'
+ 'avoga'
+ 'avome'
+ 'avomu'
+ 'avima'
+ 'avama'
+ 'aviji'
+ 'avije'
+ 'avija'
+ 'aviju'
+ 'avim'
+ 'avih'
+ 'avoj'
+ 'avom'
+ 'avog'
+ 'avi'
+ 'ava'
+ 'avu'
+ 'ave'
+ 'avo' (<-'av')
+ 'evijemu'
+ 'evijima'
+ 'evijega'
+ 'evijeg'
+ 'evijem'
+ 'evemu'
+ 'evega'
+ 'eveg'
+ 'evem'
+ 'evijim'
+ 'evijih'
+ 'evijoj'
+ 'evoga'
+ 'evome'
+ 'evomu'
+ 'evima'
+ 'evama'
+ 'eviji'
+ 'evije'
+ 'evija'
+ 'eviju'
+ 'evim'
+ 'evih'
+ 'evoj'
+ 'evom'
+ 'evog'
+ 'evi'
+ 'eva'
+ 'evu'
+ 'eve'
+ 'evo' (<-'ev')
+ 'ivijemu'
+ 'ivijima'
+ 'ivijega'
+ 'ivijeg'
+ 'ivijem'
+ 'ivemu'
+ 'ivega'
+ 'iveg'
+ 'ivem'
+ 'ivijim'
+ 'ivijih'
+ 'ivijoj'
+ 'ivoga'
+ 'ivome'
+ 'ivomu'
+ 'ivima'
+ 'ivama'
+ 'iviji'
+ 'ivije'
+ 'ivija'
+ 'iviju'
+ 'ivim'
+ 'ivih'
+ 'ivoj'
+ 'ivom'
+ 'ivog'
+ 'ivi'
+ 'iva'
+ 'ivu'
+ 'ive'
+ 'ivo' (<-'iv')
+ 'ovijemu'
+ 'ovijima'
+ 'ovijega'
+ 'ovijeg'
+ 'ovijem'
+ 'ovemu'
+ 'ovega'
+ 'oveg'
+ 'ovijim'
+ 'ovijih'
+ 'ovijoj'
+ 'ovoga'
+ 'ovome'
+ 'ovomu'
+ 'ovima'
+ 'oviji'
+ 'ovije'
+ 'ovija'
+ 'oviju'
+ 'ovim'
+ 'ovih'
+ 'ovoj'
+ 'ovom'
+ 'ovog'
+ 'ovi'
+ 'ova'
+ 'ovu'
+ 'ove'
+ 'ovo' (<-'ov')
+ 'movima'
+ 'movom'
+ 'mova'
+ 'movu'
+ 'move'
+ 'movi' (<-'mov')
+ 'lovima'
+ 'lovom'
+ 'lova'
+ 'lovu'
+ 'love'
+ 'lovi' (<-'lov')
+ 'elijemu'
+ 'elijima'
+ 'elijega'
+ 'elijeg'
+ 'elijem'
+ 'elemu'
+ 'elega'
+ 'eleg'
+ 'elem'
+ 'elijim'
+ 'elijih'
+ 'elijoj'
+ 'eloga'
+ 'elome'
+ 'elomu'
+ 'elima'
+ 'eliji'
+ 'elije'
+ 'elija'
+ 'eliju'
+ 'elim'
+ 'elih'
+ 'eloj'
+ 'elom'
+ 'elog'
+ 'eli'
+ 'ela'
+ 'elu'
+ 'ele'
+ 'elo' (<-'el')
+ 'anjijemu'
+ 'anjijima'
+ 'anjijega'
+ 'anjijeg'
+ 'anjijem'
+ 'anjemu'
+ 'anjega'
+ 'anjeg'
+ 'anjem'
+ 'anjijim'
+ 'anjijih'
+ 'anjijoj'
+ 'anjoga'
+ 'anjome'
+ 'anjomu'
+ 'anjima'
+ 'anjiji'
+ 'anjije'
+ 'anjija'
+ 'anjiju'
+ 'anjim'
+ 'anjih'
+ 'anjoj'
+ 'anjom'
+ 'anjog'
+ 'anja'
+ 'anje'
+ 'anji'
+ 'anjo'
+ 'anju' (<-'anj')
+ 'enjijemu'
+ 'enjijima'
+ 'enjijega'
+ 'enjijeg'
+ 'enjijem'
+ 'enjemu'
+ 'enjega'
+ 'enjeg'
+ 'enjem'
+ 'enjijim'
+ 'enjijih'
+ 'enjijoj'
+ 'enjoga'
+ 'enjome'
+ 'enjomu'
+ 'enjima'
+ 'enjiji'
+ 'enjije'
+ 'enjija'
+ 'enjiju'
+ 'enjim'
+ 'enjih'
+ 'enjoj'
+ 'enjom'
+ 'enjog'
+ 'enja'
+ 'enje'
+ 'enji'
+ 'enjo'
+ 'enju' (<-'enj')
+ '{sx}njijemu'
+ '{sx}njijima'
+ '{sx}njijega'
+ '{sx}njijeg'
+ '{sx}njijem'
+ '{sx}njemu'
+ '{sx}njega'
+ '{sx}njeg'
+ '{sx}njem'
+ '{sx}njijim'
+ '{sx}njijih'
+ '{sx}njijoj'
+ '{sx}njoga'
+ '{sx}njome'
+ '{sx}njomu'
+ '{sx}njima'
+ '{sx}njiji'
+ '{sx}njije'
+ '{sx}njija'
+ '{sx}njiju'
+ '{sx}njim'
+ '{sx}njih'
+ '{sx}njoj'
+ '{sx}njom'
+ '{sx}njog'
+ '{sx}nja'
+ '{sx}nje'
+ '{sx}nji'
+ '{sx}njo'
+ '{sx}nju' (<-'{sx}nj')
+ 'anemu'
+ 'anega'
+ 'aneg'
+ 'anem' (<-'an')
+ 'enemu'
+ 'enega'
+ 'eneg'
+ 'enem' (<-'en')
+ '{sx}nemu'
+ '{sx}nega'
+ '{sx}neg'
+ '{sx}nem' (<-'{sx}n')
+ '{cx}inama'
+ '{cx}inome'
+ '{cx}inomu'
+ '{cx}inoga'
+ '{cx}inima'
+ '{cx}inog'
+ '{cx}inom'
+ '{cx}inim'
+ '{cx}inih'
+ '{cx}inoj'
+ '{cx}ina'
+ '{cx}inu'
+ '{cx}ini'
+ '{cx}ino'
+ '{cx}ine' (<-'{cx}in')
+ 'ro{sx}iv{sx}i'
+ 'ro{sx}ismo'
+ 'ro{sx}iste'
+ 'ro{sx}i{sx}e'
+ 'ro{sx}imo'
+ 'ro{sx}ite'
+ 'ro{sx}iti'
+ 'ro{sx}ili'
+ 'ro{sx}ila'
+ 'ro{sx}ilo'
+ 'ro{sx}ile'
+ 'ro{sx}im'
+ 'ro{sx}i{sx}'
+ 'ro{sx}it'
+ 'ro{sx}ih'
+ 'ro{sx}io' (<-'ro{sx}i')
+ 'o{sx}ijemu'
+ 'o{sx}ijima'
+ 'o{sx}ijega'
+ 'o{sx}ijeg'
+ 'o{sx}ijem'
+ 'o{sx}emu'
+ 'o{sx}ega'
+ 'o{sx}eg'
+ 'o{sx}em'
+ 'o{sx}ijim'
+ 'o{sx}ijih'
+ 'o{sx}ijoj'
+ 'o{sx}oga'
+ 'o{sx}ome'
+ 'o{sx}omu'
+ 'o{sx}ima'
+ 'o{sx}iji'
+ 'o{sx}ije'
+ 'o{sx}ija'
+ 'o{sx}iju'
+ 'o{sx}im'
+ 'o{sx}ih'
+ 'o{sx}oj'
+ 'o{sx}om'
+ 'o{sx}og'
+ 'o{sx}i'
+ 'o{sx}a'
+ 'o{sx}u'
+ 'o{sx}e' (<-'o{sx}')
+ 'evitijima'
+ 'evitijega'
+ 'evitijemu'
+ 'evitijem'
+ 'evitega'
+ 'evitemu'
+ 'evitem'
+ 'evitijim'
+ 'evitijih'
+ 'evitijoj'
+ 'evitijeg'
+ 'evitiji'
+ 'evitije'
+ 'evitija'
+ 'evitoga'
+ 'evitome'
+ 'evitomu'
+ 'evitima'
+ 'evitog'
+ 'evitom'
+ 'evitim'
+ 'evitih'
+ 'evitoj'
+ 'eviti'
+ 'evite'
+ 'evito'
+ 'evita'
+ 'evitu' (<-'evit')
+ 'ovitijima'
+ 'ovitijega'
+ 'ovitijemu'
+ 'ovitijem'
+ 'ovitega'
+ 'ovitemu'
+ 'ovitem'
+ 'ovitijim'
+ 'ovitijih'
+ 'ovitijoj'
+ 'ovitijeg'
+ 'ovitiji'
+ 'ovitije'
+ 'ovitija'
+ 'ovitoga'
+ 'ovitome'
+ 'ovitomu'
+ 'ovitima'
+ 'ovitog'
+ 'ovitom'
+ 'ovitim'
+ 'ovitih'
+ 'ovitoj'
+ 'oviti'
+ 'ovite'
+ 'ovito'
+ 'ovita'
+ 'ovitu' (<-'ovit')
+ 'astijima'
+ 'astijega'
+ 'astijemu'
+ 'astijem'
+ 'astega'
+ 'astemu'
+ 'astem'
+ 'astijim'
+ 'astijih'
+ 'astijoj'
+ 'astijeg'
+ 'astiji'
+ 'astije'
+ 'astija'
+ 'astoga'
+ 'astome'
+ 'astomu'
+ 'astima'
+ 'astog'
+ 'astom'
+ 'astim'
+ 'astih'
+ 'astoj'
+ 'asti'
+ 'aste'
+ 'asto'
+ 'asta'
+ 'astu' (<-'ast')
+ 'kijemu'
+ 'kijima'
+ 'kijega'
+ 'kijeg'
+ 'kijem'
+ 'kemu'
+ 'kega'
+ 'keg'
+ 'kem'
+ 'kijim'
+ 'kijih'
+ 'kijoj'
+ 'koga'
+ 'kome'
+ 'komu'
+ 'kima'
+ 'kiji'
+ 'kije'
+ 'kija'
+ 'kiju'
+ 'kim'
+ 'kih'
+ 'koj'
+ 'kom'
+ 'kog'
+ 'kov'
+ 'ki'
+ 'ka'
+ 'ku'
+ 'ke'
+ 'ko' (<-'k')
+ 'evaju{cy}i'
+ 'evasmo'
+ 'evaste'
+ 'evajmo'
+ 'evajte'
+ 'evaju'
+ 'evala'
+ 'evale'
+ 'evali'
+ 'evalo'
+ 'evamo'
+ 'evana'
+ 'evane'
+ 'evani'
+ 'evano'
+ 'evate'
+ 'evati'
+ 'eva{sx}e'
+ 'evahu'
+ 'evah'
+ 'evaj'
+ 'evam'
+ 'evan'
+ 'evao'
+ 'evat'
+ 'evav'
+ 'eva{sx}' (<-'eva')
+ 'avaju{cy}i'
+ 'avasmo'
+ 'avaste'
+ 'avajmo'
+ 'avajte'
+ 'avaju'
+ 'avala'
+ 'avale'
+ 'avali'
+ 'avalo'
+ 'avamo'
+ 'avana'
+ 'avane'
+ 'avani'
+ 'avano'
+ 'avate'
+ 'avati'
+ 'ava{sx}e'
+ 'avahu'
+ 'avah'
+ 'avaj'
+ 'avam'
+ 'avan'
+ 'avao'
+ 'avat'
+ 'avav'
+ 'ava{sx}' (<-'ava')
+ 'ivaju{cy}i'
+ 'ivasmo'
+ 'ivaste'
+ 'ivajmo'
+ 'ivajte'
+ 'ivaju'
+ 'ivala'
+ 'ivale'
+ 'ivali'
+ 'ivalo'
+ 'ivamo'
+ 'ivana'
+ 'ivane'
+ 'ivani'
+ 'ivano'
+ 'ivate'
+ 'ivati'
+ 'iva{sx}e'
+ 'ivahu'
+ 'ivah'
+ 'ivaj'
+ 'ivam'
+ 'ivan'
+ 'ivao'
+ 'ivat'
+ 'ivav'
+ 'iva{sx}' (<-'iva')
+ 'uvaju{cy}i'
+ 'uvasmo'
+ 'uvaste'
+ 'uvajmo'
+ 'uvajte'
+ 'uvaju'
+ 'uvala'
+ 'uvale'
+ 'uvali'
+ 'uvalo'
+ 'uvamo'
+ 'uvana'
+ 'uvane'
+ 'uvani'
+ 'uvano'
+ 'uvate'
+ 'uvati'
+ 'uva{sx}e'
+ 'uvahu'
+ 'uvah'
+ 'uvaj'
+ 'uvam'
+ 'uvan'
+ 'uvao'
+ 'uvat'
+ 'uvav'
+ 'uva{sx}' (<-'uva')
+ 'irujemo'
+ 'irujete'
+ 'iruju{cy}i'
+ 'iraju{cy}i'
+ 'irivat'
+ 'irujem'
+ 'iruje{sx}'
+ 'irujmo'
+ 'irujte'
+ 'irav{sx}i'
+ 'irasmo'
+ 'iraste'
+ 'irati'
+ 'iramo'
+ 'irate'
+ 'iraju'
+ 'ira{sx}e'
+ 'irahu'
+ 'irala'
+ 'iralo'
+ 'irali'
+ 'irale'
+ 'iruje'
+ 'iruju'
+ 'iruj'
+ 'iral'
+ 'iran'
+ 'iram'
+ 'ira{sx}'
+ 'irat'
+ 'irah'
+ 'irao' (<-'ir')
+ 'a{cx}ismo'
+ 'a{cx}iste'
+ 'a{cx}iti'
+ 'a{cx}imo'
+ 'a{cx}ite'
+ 'a{cx}i{sx}e'
+ 'a{cx}e{cy}i'
+ 'a{cx}ila'
+ 'a{cx}ilo'
+ 'a{cx}ili'
+ 'a{cx}ile'
+ 'a{cx}ena'
+ 'a{cx}eno'
+ 'a{cx}eni'
+ 'a{cx}ene'
+ 'a{cx}io'
+ 'a{cx}im'
+ 'a{cx}i{sx}'
+ 'a{cx}it'
+ 'a{cx}ih'
+ 'a{cx}en'
+ 'a{cx}i'
+ 'a{cx}e' (<-'a{cx}')
+ 'a{cx}av{sx}i'
+ 'a{cx}asmo'
+ 'a{cx}aste'
+ 'a{cx}ahu'
+ 'a{cx}ati'
+ 'a{cx}amo'
+ 'a{cx}ate'
+ 'a{cx}a{sx}e'
+ 'a{cx}ala'
+ 'a{cx}alo'
+ 'a{cx}ali'
+ 'a{cx}ale'
+ 'a{cx}aju'
+ 'a{cx}ana'
+ 'a{cx}ano'
+ 'a{cx}ani'
+ 'a{cx}ane'
+ 'a{cx}ao'
+ 'a{cx}am'
+ 'a{cx}a{sx}'
+ 'a{cx}at'
+ 'a{cx}ah'
+ 'a{cx}an' (<-'a{cx}a')
+ 'nuv{sx}i'
+ 'nusmo'
+ 'nuste'
+ 'nu{cy}i'
+ 'nimo'
+ 'nite'
+ 'nemo'
+ 'nete'
+ 'nula'
+ 'nulo'
+ 'nule'
+ 'nuli'
+ 'nuto'
+ 'nuti'
+ 'nuta'
+ 'ne{sx}'
+ 'nuo'
+ 'nut' (<-'n')
+ 'niv{sx}i'
+ 'nismo'
+ 'niste'
+ 'niti'
+ 'nila'
+ 'nilo'
+ 'nile'
+ 'nili'
+ 'ni{sx}'
+ 'nio' (<-'ni')
+ 'aju{cy}i'
+ 'av{sx}i'
+ 'asmo'
+ 'ajmo'
+ 'ajte'
+ 'ajem'
+ 'aloj'
+ 'amo'
+ 'ate'
+ 'aje'
+ 'aju'
+ 'ati'
+ 'a{sx}e'
+ 'ahu'
+ 'ala'
+ 'ali'
+ 'ale'
+ 'alo'
+ 'ano'
+ 'at'
+ 'ah'
+ 'ao'
+ 'aj'
+ 'an'
+ 'am'
+ 'a{sx}' (<-'a')
+ 'uraju{cy}i'
+ 'urasmo'
+ 'uraste'
+ 'urajmo'
+ 'urajte'
+ 'uramo'
+ 'urate'
+ 'uraju'
+ 'urati'
+ 'ura{sx}e'
+ 'urahu'
+ 'urala'
+ 'urali'
+ 'urale'
+ 'uralo'
+ 'urana'
+ 'urano'
+ 'urani'
+ 'urane'
+ 'ural'
+ 'urat'
+ 'urah'
+ 'urao'
+ 'uraj'
+ 'uran'
+ 'uram'
+ 'ura{sx}' (<-'ur')
+ 'astajasmo'
+ 'astajaste'
+ 'astajahu'
+ 'astajati'
+ 'astajemo'
+ 'astajete'
+ 'astaja{sx}e'
+ 'astajali'
+ 'astaju{cy}i'
+ 'astajala'
+ 'astajalo'
+ 'astajale'
+ 'astajmo'
+ 'astajao'
+ 'astajem'
+ 'astaje{sx}'
+ 'astajat'
+ 'astajah'
+ 'astajte'
+ 'astaje'
+ 'astaju' (<-'astaj')
+ 'istajasmo'
+ 'istajaste'
+ 'istajahu'
+ 'istajati'
+ 'istajemo'
+ 'istajete'
+ 'istaja{sx}e'
+ 'istajali'
+ 'istaju{cy}i'
+ 'istajala'
+ 'istajalo'
+ 'istajale'
+ 'istajmo'
+ 'istajao'
+ 'istajem'
+ 'istaje{sx}'
+ 'istajat'
+ 'istajah'
+ 'istajte'
+ 'istaje'
+ 'istaju' (<-'istaj')
+ 'ostajasmo'
+ 'ostajaste'
+ 'ostajahu'
+ 'ostajati'
+ 'ostajemo'
+ 'ostajete'
+ 'ostaja{sx}e'
+ 'ostajali'
+ 'ostaju{cy}i'
+ 'ostajala'
+ 'ostajalo'
+ 'ostajale'
+ 'ostajmo'
+ 'ostajao'
+ 'ostajem'
+ 'ostaje{sx}'
+ 'ostajat'
+ 'ostajah'
+ 'ostajte'
+ 'ostaje'
+ 'ostaju' (<-'ostaj')
+ 'alama'
+ 'alima'
+ 'alom'
+ 'alu'
+ 'al' (<-'a')
+ 'ajevima'
+ 'ajevi'
+ 'ajeva'
+ 'ajeve'
+ 'ajama'
+ 'ajima'
+ 'aja'
+ 'aji' (<-'aj')
+ 'astadosmo'
+ 'astadoste'
+ 'astado{sx}e'
+ 'astanemo'
+ 'astademo'
+ 'astanete'
+ 'astadete'
+ 'astanimo'
+ 'astanite'
+ 'astanila'
+ 'astav{sx}i'
+ 'astanem'
+ 'astadem'
+ 'astane{sx}'
+ 'astade{sx}'
+ 'astadoh'
+ 'astade'
+ 'astati'
+ 'astane'
+ 'astanu'
+ 'astadu'
+ 'astala'
+ 'astali'
+ 'astalo'
+ 'astale'
+ 'astat'
+ 'astao' (<-'asta')
+ 'istadosmo'
+ 'istadoste'
+ 'istado{sx}e'
+ 'istanemo'
+ 'istademo'
+ 'istanete'
+ 'istadete'
+ 'istanimo'
+ 'istanite'
+ 'istanila'
+ 'istav{sx}i'
+ 'istanem'
+ 'istadem'
+ 'istane{sx}'
+ 'istade{sx}'
+ 'istadoh'
+ 'istade'
+ 'istati'
+ 'istane'
+ 'istanu'
+ 'istadu'
+ 'istala'
+ 'istali'
+ 'istalo'
+ 'istale'
+ 'istat'
+ 'istao' (<-'ista')
+ 'ostadosmo'
+ 'ostadoste'
+ 'ostado{sx}e'
+ 'ostanemo'
+ 'ostademo'
+ 'ostanete'
+ 'ostadete'
+ 'ostanimo'
+ 'ostanite'
+ 'ostanila'
+ 'ostav{sx}i'
+ 'ostanem'
+ 'ostadem'
+ 'ostane{sx}'
+ 'ostade{sx}'
+ 'ostadoh'
+ 'ostade'
+ 'ostati'
+ 'ostane'
+ 'ostanu'
+ 'ostadu'
+ 'ostala'
+ 'ostali'
+ 'ostalo'
+ 'ostale'
+ 'ostat'
+ 'ostao' (<-'osta')
+ 'tasmo'
+ 'taste'
+ 'tajmo'
+ 'tajte'
+ 'tav{sx}i'
+ 'tati'
+ 'tamo'
+ 'tate'
+ 'taju'
+ 'tala'
+ 'talo'
+ 'tale'
+ 'tali'
+ 'tana'
+ 'tano'
+ 'tani'
+ 'tane'
+ 'tan'
+ 'taj'
+ 'tao'
+ 'tam'
+ 'ta{sx}'
+ 'tat'
+ 'tah' (<-'ta')
+ 'injasmo'
+ 'injaste'
+ 'injati'
+ 'injemo'
+ 'injete'
+ 'injali'
+ 'injala'
+ 'injalo'
+ 'injale'
+ 'inja{sx}e'
+ 'injahu'
+ 'injem'
+ 'inje{sx}'
+ 'injat'
+ 'injah'
+ 'injao' (<-'inj')
+ 'astemo'
+ 'astete'
+ 'astimo'
+ 'astite'
+ 'astu{cy}i'
+ 'aste{sx}'
+ 'asli'
+ 'asla'
+ 'aslo'
+ 'asle' (<-'as')
+ 'iv{sx}i'
+ 'ie{cy}i'
+ 'ismo'
+ 'imo'
+ 'ite'
+ 'iti'
+ 'ili'
+ 'ila'
+ 'ilo'
+ 'ile'
+ 'im'
+ 'i{sx}'
+ 'it'
+ 'ih'
+ 'io' (<-'i')
+ 'ijemo'
+ 'ijete'
+ 'ijem'
+ 'ije{sx}'
+ 'ijmo'
+ 'ijte'
+ 'iju'
+ 'ije'
+ 'ij'
+ 'ilu' (<-'i')
+ 'lu{cx}ujete'
+ 'lu{cx}uju{cy}i'
+ 'lu{cx}ujemo'
+ 'lu{cx}ujem'
+ 'lu{cx}uje{sx}'
+ 'lu{cx}ismo'
+ 'lu{cx}iste'
+ 'lu{cx}ujmo'
+ 'lu{cx}ujte'
+ 'lu{cx}uje'
+ 'lu{cx}uju'
+ 'lu{cx}i{sx}e'
+ 'lu{cx}iti'
+ 'lu{cx}imo'
+ 'lu{cx}ite'
+ 'lu{cx}ila'
+ 'lu{cx}ilo'
+ 'lu{cx}ili'
+ 'lu{cx}ile'
+ 'lu{cx}ena'
+ 'lu{cx}eno'
+ 'lu{cx}eni'
+ 'lu{cx}ene'
+ 'lu{cx}uj'
+ 'lu{cx}io'
+ 'lu{cx}en'
+ 'lu{cx}im'
+ 'lu{cx}i{sx}'
+ 'lu{cx}it'
+ 'lu{cx}ih'
+ 'lu{cx}e'
+ 'lu{cx}i' (<-'lu{cx}')
+ 'jetismo'
+ 'jetiste'
+ 'jeti{sx}e'
+ 'jetimo'
+ 'jetite'
+ 'jetiti'
+ 'jetili'
+ 'jetila'
+ 'jetilo'
+ 'jetile'
+ 'jetim'
+ 'jeti{sx}'
+ 'jetit'
+ 'jetih'
+ 'jetio' (<-'jeti')
+ 'emo'
+ 'em'
+ 'e{sx}'
+ 'elama'
+ 'el' (<-'e')
+ 'ilama'
+ 'ilima'
+ 'ilom'
+ 'il' (<-'i')
+ 'atijega'
+ 'atijemu'
+ 'atijima'
+ 'atijeg'
+ 'atijem'
+ 'atega'
+ 'atemu'
+ 'ateg'
+ 'atem'
+ 'atijih'
+ 'atijim'
+ 'atima'
+ 'atoga'
+ 'atome'
+ 'atomu'
+ 'atiji'
+ 'atije'
+ 'atija'
+ 'atiju'
+ 'atoj'
+ 'atog'
+ 'atom'
+ 'atim'
+ 'atih'
+ 'ata'
+ 'atu'
+ 'ato' (<-'at')
+ 'etav{sx}i'
+ 'etu{cy}i'
+ 'etemo'
+ 'etimo'
+ 'etem'
+ 'ete{sx}' (<-'et')
+ 'lucujuci'
+ 'lucujemo'
+ 'lucujete'
+ 'lucujem'
+ 'lucujes'
+ 'lucujmo'
+ 'lucujte'
+ 'lucismo'
+ 'luciste'
+ 'luciti'
+ 'lucite'
+ 'lucise'
+ 'lucuje'
+ 'lucuju'
+ 'lucila'
+ 'lucile'
+ 'lucili'
+ 'lucilo'
+ 'lucena'
+ 'luceni'
+ 'lucene'
+ 'luceno'
+ 'lucimo'
+ 'lucim'
+ 'lucis'
+ 'lucih'
+ 'lucit'
+ 'lucio'
+ 'lucuj'
+ 'lucen'
+ 'luce'
+ 'luci' (R2 <-'luc')
+ 'snjijima'
+ 'snjijemu'
+ 'snjijega'
+ 'snjijim'
+ 'snjijih'
+ 'snjijeg'
+ 'snjijoj'
+ 'snjiji'
+ 'snjija'
+ 'snjije'
+ 'snjiju'
+ 'snjima'
+ 'snjemu'
+ 'snjomu'
+ 'snjome'
+ 'snjega'
+ 'snjoga'
+ 'snjih'
+ 'snjim'
+ 'snjem'
+ 'snjom'
+ 'snjeg'
+ 'snjog'
+ 'snjoj'
+ 'snja'
+ 'snje'
+ 'snji'
+ 'snjo'
+ 'snju' (R2 <-'snj')
+ 'osijima'
+ 'osijemu'
+ 'osijega'
+ 'snjijem'
+ 'osijih'
+ 'osijim'
+ 'osijem'
+ 'osijeg'
+ 'osijoj'
+ 'osima'
+ 'osemu'
+ 'osomu'
+ 'osome'
+ 'osega'
+ 'osoga'
+ 'osija'
+ 'osije'
+ 'osiji'
+ 'osiju'
+ 'osih'
+ 'osim'
+ 'osem'
+ 'osom'
+ 'oseg'
+ 'osog'
+ 'osoj'
+ 'osa'
+ 'ose'
+ 'osi'
+ 'osu' (R2 <-'os')
+ 'acismo'
+ 'aciste'
+ 'acima'
+ 'acimo'
+ 'acome'
+ 'acomu'
+ 'acite'
+ 'aciti'
+ 'acise'
+ 'acila'
+ 'acile'
+ 'acili'
+ 'acilo'
+ 'acega'
+ 'acene'
+ 'aceci'
+ 'aceni'
+ 'acemu'
+ 'acena'
+ 'aceno'
+ 'acoga'
+ 'acoj'
+ 'acih'
+ 'acem'
+ 'acom'
+ 'acen'
+ 'acog'
+ 'acit'
+ 'acio'
+ 'aceg'
+ 'acim'
+ 'acuh'
+ 'acis'
+ 'ace'
+ 'aca'
+ 'aci' (R2 <-'ac')
+ 'ecome'
+ 'ecoga'
+ 'ecemu'
+ 'ecima'
+ 'ecega'
+ 'ecomu'
+ 'ecoj'
+ 'ecuh'
+ 'ecom'
+ 'ecog'
+ 'eceg'
+ 'ecih'
+ 'ecem'
+ 'ecim'
+ 'eca'
+ 'ece' (R2 <-'ec')
+ 'ucomu'
+ 'ucome'
+ 'ucima'
+ 'ucoga'
+ 'ucega'
+ 'ucemu'
+ 'ucih'
+ 'ucog'
+ 'uceg'
+ 'ucom'
+ 'ucem'
+ 'ucim'
+ 'ucuh'
+ 'ucoj'
+ 'uca'
+ 'uce' (R2 <-'uc')
+ 'rosismo'
+ 'rosivsi'
+ 'rosiste'
+ 'rositi'
+ 'rosili'
+ 'rosise'
+ 'rosite'
+ 'rosilo'
+ 'rosimo'
+ 'rosile'
+ 'rosila'
+ 'rosit'
+ 'rosis'
+ 'rosio'
+ 'rosim'
+ 'rosih' (R2 <-'rosi')
+ 'acavsi'
+ 'acaste'
+ 'acasmo'
+ 'acaju'
+ 'acane'
+ 'acate'
+ 'acali'
+ 'acani'
+ 'acati'
+ 'acale'
+ 'acahu'
+ 'acase'
+ 'acano'
+ 'acamo'
+ 'acalo'
+ 'acana'
+ 'acala'
+ 'acam'
+ 'acan'
+ 'acao'
+ 'acas'
+ 'acat'
+ 'acah' (R2 <-'aca')
+ 'jasima'
+ 'jasama'
+ 'jasem'
+ 'jasom'
+ 'jase'
+ 'jasi'
+ 'jasa'
+ 'jasu' (R2 <-'jas')
+ 'tasima'
+ 'tasama'
+ 'tasem'
+ 'tasom'
+ 'tase'
+ 'tasa'
+ 'tasu'
+ 'tasi' (R2 <-'tas')
+ 'gasima'
+ 'gasama'
+ 'gasem'
+ 'gasom'
+ 'gasi'
+ 'gasu'
+ 'gase'
+ 'gasa' (R2 <-'gas')
+ 'nasama'
+ 'nasima'
+ 'nasem'
+ 'nasom'
+ 'nasu'
+ 'nasi'
+ 'nase'
+ 'nasa' (R2 <-'nas')
+ 'kasama'
+ 'kasima'
+ 'kasom'
+ 'kasem'
+ 'kasi'
+ 'kasu'
+ 'kase'
+ 'kasa' (R2 <-'kas')
+ 'vasama'
+ 'vasima'
+ 'vasom'
+ 'vasem'
+ 'vasi'
+ 'vase'
+ 'vasa'
+ 'vasu' (R2 <-'vas')
+ 'basama'
+ 'basima'
+ 'basom'
+ 'basem'
+ 'basi'
+ 'base'
+ 'basu'
+ 'basa' (R2 <-'bas')
+ 'astuci'
+ 'astes' (R2 <-'as')
+ 'cinima'
+ 'cinome'
+ 'cinama'
+ 'cinomu'
+ 'cinoga'
+ 'cinom'
+ 'cinih'
+ 'cinim'
+ 'cinog'
+ 'cinoj'
+ 'cino'
+ 'cini'
+ 'cinu'
+ 'cine'
+ 'cina' (R2 <-'cin')
+ 'astajase'
+ 'astajuci'
+ 'astajes' (R2 <-'astaj')
+ 'istajase'
+ 'istajuci'
+ 'istajes' (R2 <-'istaj')
+ 'ostajase'
+ 'ostajuci'
+ 'ostajes' (R2 <-'ostaj')
+ 'astadose'
+ 'astades'
+ 'astanes'
+ 'astavsi' (R2 <-'asta')
+ 'istadose'
+ 'istades'
+ 'istanes'
+ 'istavsi' (R2 <-'ista')
+ 'ostadose'
+ 'ostades'
+ 'ostanes'
+ 'ostavsi' (R2 <-'osta')
+ 'avajuci'
+ 'avase'
+ 'avas' (R2 <-'ava')
+ 'evajuci'
+ 'evase'
+ 'evas' (R2 <-'eva')
+ 'ivajuci'
+ 'ivase'
+ 'ivas' (R2 <-'iva')
+ 'uvajuci'
+ 'uvase'
+ 'uvas' (R2 <-'uva')
+ 'ovase' (R2 <-'ova')
+ 'jetise'
+ 'jetis' (R2 <-'jeti')
+ 'injase'
+ 'injes' (R2 <-'inj')
+ 'istem' (R2 <-'ist')
+ 'esama'
+ 'esem'
+ 'esi' (R2 <-'es')
+ 'etavsi'
+ 'etuci'
+ 'etes' (R2 <-'et')
+ 'isama'
+ 'isem'
+ 'isi' (R2 <-'is')
+ 'irajuci'
+ 'irujuci'
+ 'irujes'
+ 'iravsi'
+ 'irase'
+ 'iras' (R2 <-'ir')
+ 'urajuci'
+ 'urase'
+ 'uras' (R2 <-'ur')
+ 'ujuci'
+ 'ujes' (R2 <-'uj')
+ 'nivsi'
+ 'nis' (R2 <-'ni')
+ 'snega'
+ 'snemu'
+ 'snem'
+ 'sneg' (R2 <-'sn')
+ 'tavsi'
+ 'tas' (R2 <-'ta')
+ 'ajuci'
+ 'avsi'
+ 'ase'
+ 'as' (R2 <-'a')
+ 'ijes'
+ 'ivsi'
+ 'ieci'
+ 'is' (R2 <-'i')
+ 'es' (R2 <-'e')
+ 'nuvsi'
+ 'nuci'
+ 'nes' (R2 <-'n')
+ )
+ )
+
+ define Step_3 as (
+ [substring] R1 among (
+ 'enom'
+ 'enoj'
+ 'enog'
+ 'enim'
+ 'enih'
+ 'anoj'
+ 'anog'
+ 'anim'
+ 'anih'
+ 'ost'
+ 'eno'
+ 'eni'
+ 'oga'
+ 'ima'
+ 'enu'
+ 'ena'
+ 'ama'
+ 'ano'
+ 'ani'
+ 'om'
+ 'og'
+ 'u'
+ 'o'
+ 'i'
+ 'e'
+ 'a' (<-'')
+ )
+ )
+)
+
+define stem as (
+ do cyr_to_lat
+ do prelude
+ do mark_regions
+ backwards (
+ do Step_1
+ do (Step_2 or Step_3)
+ )
+)
diff --git a/contrib/snowball/algorithms/spanish.sbl b/contrib/snowball/algorithms/spanish.sbl
new file mode 100644
index 0000000..6638f5f
--- /dev/null
+++ b/contrib/snowball/algorithms/spanish.sbl
@@ -0,0 +1,230 @@
+routines (
+ postlude mark_regions
+ RV R1 R2
+ attached_pronoun
+ standard_suffix
+ y_verb_suffix
+ verb_suffix
+ residual_suffix
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef e' '{U+00E9}' // e-acute
+stringdef i' '{U+00ED}' // i-acute
+stringdef o' '{U+00F3}' // o-acute
+stringdef u' '{U+00FA}' // u-acute
+stringdef u" '{U+00FC}' // u-diaeresis
+stringdef n~ '{U+00F1}' // n-tilde
+
+define v 'aeiou{a'}{e'}{i'}{o'}{u'}{u"}'
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+ [substring] among(
+ '{a'}' (<- 'a')
+ '{e'}' (<- 'e')
+ '{i'}' (<- 'i')
+ '{o'}' (<- 'o')
+ '{u'}' (<- 'u')
+ // and possibly {u"}->u here, or in prelude
+ '' (next)
+ ) //or next
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define attached_pronoun as (
+ [substring] among(
+ 'me' 'se' 'sela' 'selo' 'selas' 'selos' 'la' 'le' 'lo'
+ 'las' 'les' 'los' 'nos'
+ )
+ substring RV among(
+ 'i{e'}ndo' (] <- 'iendo')
+ '{a'}ndo' (] <- 'ando')
+ '{a'}r' (] <- 'ar')
+ '{e'}r' (] <- 'er')
+ '{i'}r' (] <- 'ir')
+ 'ando'
+ 'iendo'
+ 'ar' 'er' 'ir'
+ (delete)
+ 'yendo' ('u' delete)
+ )
+ )
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'anza' 'anzas'
+ 'ico' 'ica' 'icos' 'icas'
+ 'ismo' 'ismos'
+ 'able' 'ables'
+ 'ible' 'ibles'
+ 'ista' 'istas'
+ 'oso' 'osa' 'osos' 'osas'
+ 'amiento' 'amientos'
+ 'imiento' 'imientos'
+ (
+ R2 delete
+ )
+ 'adora' 'ador' 'aci{o'}n'
+ 'adoras' 'adores' 'aciones'
+ 'ante' 'antes' 'ancia' 'ancias'// Note 1
+ (
+ R2 delete
+ try ( ['ic'] R2 delete )
+ )
+ 'log{i'}a'
+ 'log{i'}as'
+ (
+ R2 <- 'log'
+ )
+ 'uci{o'}n' 'uciones'
+ (
+ R2 <- 'u'
+ )
+ 'encia' 'encias'
+ (
+ R2 <- 'ente'
+ )
+ 'amente'
+ (
+ R1 delete
+ try (
+ [substring] R2 delete among(
+ 'iv' (['at'] R2 delete)
+ 'os'
+ 'ic'
+ 'ad'
+ )
+ )
+ )
+ 'mente'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'ante' // Note 1
+ 'able'
+ 'ible' (R2 delete)
+ )
+ )
+ )
+ 'idad'
+ 'idades'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil'
+ 'ic'
+ 'iv' (R2 delete)
+ )
+ )
+ )
+ 'iva' 'ivo'
+ 'ivas' 'ivos'
+ (
+ R2 delete
+ try (
+ ['at'] R2 delete // but not a further ['ic'] R2 delete
+ )
+ )
+ )
+ )
+
+ define y_verb_suffix as (
+ setlimit tomark pV for ([substring]) among(
+ 'ya' 'ye' 'yan' 'yen' 'yeron' 'yendo' 'yo' 'y{o'}'
+ 'yas' 'yes' 'yais' 'yamos'
+ ('u' delete)
+ )
+ )
+
+ define verb_suffix as (
+ setlimit tomark pV for ([substring]) among(
+
+ 'en' 'es' '{e'}is' 'emos'
+ (try ('u' test 'g') ] delete)
+
+ 'ar{i'}an' 'ar{i'}as' 'ar{a'}n' 'ar{a'}s' 'ar{i'}ais'
+ 'ar{i'}a' 'ar{e'}is' 'ar{i'}amos' 'aremos' 'ar{a'}'
+ 'ar{e'}'
+ 'er{i'}an' 'er{i'}as' 'er{a'}n' 'er{a'}s' 'er{i'}ais'
+ 'er{i'}a' 'er{e'}is' 'er{i'}amos' 'eremos' 'er{a'}'
+ 'er{e'}'
+ 'ir{i'}an' 'ir{i'}as' 'ir{a'}n' 'ir{a'}s' 'ir{i'}ais'
+ 'ir{i'}a' 'ir{e'}is' 'ir{i'}amos' 'iremos' 'ir{a'}'
+ 'ir{e'}'
+
+ 'aba' 'ada' 'ida' '{i'}a' 'ara' 'iera' 'ad' 'ed'
+ 'id' 'ase' 'iese' 'aste' 'iste' 'an' 'aban' '{i'}an'
+ 'aran' 'ieran' 'asen' 'iesen' 'aron' 'ieron' 'ado'
+ 'ido' 'ando' 'iendo' 'i{o'}' 'ar' 'er' 'ir' 'as'
+ 'abas' 'adas' 'idas' '{i'}as' 'aras' 'ieras' 'ases'
+ 'ieses' '{i'}s' '{a'}is' 'abais' '{i'}ais' 'arais'
+ 'ierais' 'aseis' 'ieseis' 'asteis' 'isteis' 'ados'
+ 'idos' 'amos' '{a'}bamos' '{i'}amos' 'imos'
+ '{a'}ramos' 'i{e'}ramos' 'i{e'}semos' '{a'}semos'
+ (delete)
+ )
+ )
+
+ define residual_suffix as (
+ [substring] among(
+ 'os'
+ 'a' 'o' '{a'}' '{i'}' '{o'}'
+ ( RV delete )
+ 'e' '{e'}'
+ ( RV delete try( ['u'] test 'g' RV delete ) )
+ )
+ )
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ do attached_pronoun
+ do ( standard_suffix or
+ y_verb_suffix or
+ verb_suffix
+ )
+ do residual_suffix
+ )
+ do postlude
+)
+
+/*
+ Note 1: additions of 15 Jun 2005
+*/
diff --git a/contrib/snowball/algorithms/swedish.sbl b/contrib/snowball/algorithms/swedish.sbl
new file mode 100644
index 0000000..2cbb885
--- /dev/null
+++ b/contrib/snowball/algorithms/swedish.sbl
@@ -0,0 +1,72 @@
+routines (
+ mark_regions
+ main_suffix
+ consonant_pair
+ other_suffix
+)
+
+externals ( stem )
+
+integers ( p1 x )
+
+groupings ( v s_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef ao '{U+00E5}'
+stringdef o" '{U+00F6}'
+
+define v 'aeiouy{a"}{ao}{o"}'
+
+define s_ending 'bcdfghjklmnoprtvy'
+
+define mark_regions as (
+
+ $p1 = limit
+ test ( hop 3 setmark x )
+ goto v gopast non-v setmark p1
+ try ( $p1 < x $p1 = x )
+)
+
+backwardmode (
+
+ define main_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+
+ 'a' 'arna' 'erna' 'heterna' 'orna' 'ad' 'e' 'ade' 'ande' 'arne'
+ 'are' 'aste' 'en' 'anden' 'aren' 'heten' 'ern' 'ar' 'er' 'heter'
+ 'or' 'as' 'arnas' 'ernas' 'ornas' 'es' 'ades' 'andes' 'ens' 'arens'
+ 'hetens' 'erns' 'at' 'andet' 'het' 'ast'
+ (delete)
+ 's'
+ (s_ending delete)
+ )
+ )
+
+ define consonant_pair as setlimit tomark p1 for (
+ among('dd' 'gd' 'nn' 'dt' 'gt' 'kt' 'tt')
+ and ([next] delete)
+ )
+
+ define other_suffix as setlimit tomark p1 for (
+ [substring] among(
+ 'lig' 'ig' 'els' (delete)
+ 'l{o"}st' (<-'l{o"}s')
+ 'fullt' (<-'full')
+ )
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ backwards (
+ do main_suffix
+ do consonant_pair
+ do other_suffix
+ )
+)
diff --git a/contrib/snowball/algorithms/tamil.sbl b/contrib/snowball/algorithms/tamil.sbl
new file mode 100644
index 0000000..9635777
--- /dev/null
+++ b/contrib/snowball/algorithms/tamil.sbl
@@ -0,0 +1,405 @@
+/*
+* Affix stripping stemming algorithm for Tamil
+* By Damodharan Rajalingam
+*/
+
+stringescapes {}
+
+/* Aytham */
+stringdef aytham '{U+0B83}'
+
+/* Uyir - independent vowels */
+stringdef a '{U+0B85}'
+stringdef aa '{U+0B86}'
+stringdef i '{U+0B87}'
+stringdef ii '{U+0B88}'
+stringdef u '{U+0B89}'
+stringdef uu '{U+0B8A}'
+stringdef e '{U+0B8E}'
+stringdef ee '{U+0B8F}'
+stringdef ai '{U+0B90}'
+stringdef o '{U+0B92}'
+stringdef oo '{U+0B93}'
+stringdef au '{U+0B94}'
+
+/* Consonants */
+stringdef ka '{U+0B95}'
+stringdef nga '{U+0B99}'
+stringdef ca '{U+0B9A}'
+stringdef ja '{U+0B9C}'
+stringdef nya '{U+0B9E}'
+stringdef tta '{U+0B9F}'
+stringdef nna '{U+0BA3}'
+stringdef ta '{U+0BA4}'
+stringdef tha '{U+0BA4}'
+stringdef na '{U+0BA8}'
+stringdef nnna '{U+0BA9}'
+stringdef pa '{U+0BAA}'
+stringdef ma '{U+0BAE}'
+stringdef ya '{U+0BAF}'
+stringdef ra '{U+0BB0}'
+stringdef rra '{U+0BB1}'
+stringdef la '{U+0BB2}'
+stringdef lla '{U+0BB3}'
+stringdef llla '{U+0BB4}'
+stringdef zha '{U+0BB4}'
+stringdef va '{U+0BB5}'
+
+/* Vatamozi - borrowed */
+stringdef sha '{U+0BB6}'
+stringdef ssa '{U+0BB7}'
+stringdef sa '{U+0BB8}'
+stringdef ha '{U+0BB9}'
+
+
+/* Dependent vowel signs (kombu etc.) */
+stringdef vs_aa '{U+0BBE}'
+stringdef vs_i '{U+0BBF}'
+stringdef vs_ii '{U+0BC0}'
+stringdef vs_u '{U+0BC1}'
+stringdef vs_uu '{U+0BC2}'
+stringdef vs_e '{U+0BC6}'
+stringdef vs_ee '{U+0BC7}'
+stringdef vs_ai '{U+0BC8}'
+stringdef vs_o '{U+0BCA}'
+stringdef vs_oo '{U+0BCB}'
+stringdef vs_au '{U+0BCC}'
+
+/* Pulli */
+stringdef pulli '{U+0BCD}'
+
+/* AU length markk */
+stringdef au_lmark '{U+0BD7}'
+
+
+routines (
+ remove_plural_suffix
+ remove_question_suffixes
+ remove_question_prefixes
+ remove_pronoun_prefixes
+ remove_command_suffixes
+ remove_um
+ remove_vetrumai_urupukal
+ fix_va_start
+ fix_ending
+ fix_endings
+ remove_tense_suffix
+ remove_tense_suffixes
+ remove_common_word_endings
+ has_min_length
+)
+
+externals ( stem )
+
+booleans (
+ found_a_match
+ found_vetrumai_urupu
+)
+
+define has_min_length as (
+ $(len > 4)
+)
+
+define fix_va_start as (
+ (try '{va}{vs_oo}' and [ '{va}{vs_oo}' ] <- '{oo}' ) or
+ (try '{va}{vs_o}' and [ '{va}{vs_o}' ] <- '{o}' ) or
+ (try '{va}{vs_u}' and [ '{va}{vs_u}' ] <- '{u}' ) or
+ (try '{va}{vs_uu}' and [ '{va}{vs_uu}' ] <- '{uu}' )
+)
+
+define fix_endings as (
+ do repeat fix_ending
+)
+
+define remove_question_prefixes as (
+ [ ('{e}' ) among('{ka}' '{ca}' '{tha}' '{va}' '{na}' '{pa}' '{ma}' '{ya}' '{nga}' '{nya}') '{pulli}' ] delete
+ do fix_va_start
+)
+
+// Gives signal t if an ending was fixed, signal f otherwise.
+define fix_ending as (
+ $(len > 3)
+ backwards (
+ ( [among('{na}{pulli}' '{na}{pulli}{ta}' '{na}{pulli}{ta}{pulli}') ] delete )
+ or
+ ( ['{ya}{pulli}' test among('{vs_ai}' '{vs_i}' '{vs_ii}') ] delete )
+ or
+ ( [ '{tta}{pulli}{pa}{pulli}' or '{tta}{pulli}{ka}{pulli}' ] <- '{lla}{pulli}' )
+ or
+ ( [ '{nnna}{pulli}{rra}{pulli}' ] <- '{la}{pulli}' )
+ or
+// ( [ '{rra}{pulli}{ka}{pulli}' or '{nnna}{pulli}{nnna}{pulli}' ] <- '{la}{pulli}' )
+ ( [ '{rra}{pulli}{ka}{pulli}' ] <- '{la}{pulli}' )
+ or
+ ( [ '{tta}{pulli}{tta}{pulli}' ] <- '{tta}{vs_u}' )
+ or
+ ( found_vetrumai_urupu [ '{ta}{pulli}{ta}{pulli}' (test not '{vs_ai}') ] <- '{ma}{pulli}' ] )
+ or
+ ( [ '{vs_u}{ka}{pulli}' or '{vs_u}{ka}{pulli}{ka}{pulli}' ] <- '{pulli}' )
+ or
+ ( [ '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') ] delete )
+ or
+ ( [ '{vs_u}{ka}{pulli}' ] <- '{pulli}' )
+ or
+ ( [ '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') ] delete )
+ or
+ ( [ '{pulli}' (among('{ya}' '{ra}' '{la}' '{va}' '{zha}' '{lla}') or among('{nga}' '{nya}' '{nna}' '{na}' '{ma}' '{nnna}')) '{pulli}' ] <- '{pulli}' )
+ or
+ ( [ among('{va}' '{ya}' '{va}{pulli}') ] delete )
+ or
+ ( [ '{nnna}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}')) ] delete )
+ or
+ ( [ '{nga}{pulli}' (test not '{vs_ai}')] <- '{ma}{pulli}' )
+ or
+ ( [ '{nga}{pulli}' ] delete )
+ or
+ ( [ '{pulli}' (test (among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}') or '{pulli}')) ] delete )
+ )
+)
+
+define remove_pronoun_prefixes as (
+ unset found_a_match
+ [ among('{a}' '{i}' '{u}') among('{ka}' '{ca}' '{tha}' '{va}' '{na}' '{pa}' '{ma}' '{ya}' '{nga}' '{nya}') '{pulli}' ] delete
+ (set found_a_match)
+ do fix_va_start
+)
+
+define remove_plural_suffix as (
+ unset found_a_match
+ backwards (
+ ( [ '{vs_u}{nga}{pulli}{ka}{lla}{pulli}' (test not among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}')) ] <- '{pulli}' ) or
+ ( [ '{rra}{pulli}{ka}{lla}{pulli}' ] <- '{la}{pulli}' ) or
+ ( [ '{tta}{pulli}{ka}{lla}{pulli}' ] <- '{lla}{pulli}' ) or
+ ( [ '{ka}{lla}{pulli}' ] delete )
+ (set found_a_match)
+ )
+)
+
+define remove_question_suffixes as (
+ has_min_length
+ unset found_a_match
+ backwards (
+ do (
+ [ among('{vs_oo}' '{vs_ee}' '{vs_aa}') ] <- '{pulli}'
+ (set found_a_match)
+ )
+ )
+ do fix_endings
+)
+
+define remove_command_suffixes as (
+ has_min_length
+ unset found_a_match
+ backwards (
+ [ among('{pa}{vs_i}' '{va}{vs_i}') ] delete
+ (set found_a_match)
+ )
+)
+
+define remove_um as (
+ unset found_a_match
+ has_min_length
+ backwards ( [ '{vs_u}{ma}{pulli}' ] <- '{pulli}'
+ (set found_a_match)
+ )
+ do fix_ending
+)
+
+define remove_common_word_endings as (
+ // These are not suffixes actually but are
+ // some words that are attached to other words
+ // but can be removed for stemming
+ unset found_a_match
+ has_min_length
+ backwards (
+ test ( [ '{vs_u}{tta}{nnna}{pulli}' or
+ '{vs_i}{la}{pulli}{la}{vs_ai}' or
+ '{vs_i}{tta}{ma}{pulli}' or
+ '{vs_i}{nnna}{pulli}{rra}{vs_i}' or
+ '{vs_aa}{ka}{vs_i}' or
+ '{vs_aa}{ka}{vs_i}{ya}' or
+ '{vs_e}{nnna}{pulli}{rra}{vs_u}' or
+ '{vs_u}{lla}{pulli}{lla}' or
+ '{vs_u}{tta}{vs_ai}{ya}' or
+ '{vs_u}{tta}{vs_ai}' or
+ '{vs_e}{nnna}{vs_u}{ma}{pulli}' or
+ ('{la}{pulli}{la}' test (not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or
+ '{vs_e}{nnna}' or
+ '{vs_aa}{ka}{vs_i}' ] <- '{pulli}'
+ (set found_a_match)
+ )
+ or
+ test ( [ among('{pa}{tta}{vs_u}'
+ '{pa}{tta}{pulli}{tta}'
+ '{pa}{tta}{pulli}{tta}{vs_u}'
+ '{pa}{tta}{pulli}{tta}{ta}{vs_u}'
+ '{pa}{tta}{pulli}{tta}{nna}'
+ '{ka}{vs_u}{ra}{vs_i}{ya}'
+ '{pa}{rra}{pulli}{rra}{vs_i}'
+ '{va}{vs_i}{tta}{vs_u}'
+ '{va}{vs_i}{tta}{pulli}{tta}{vs_u}'
+ '{pa}{tta}{vs_i}{ta}{vs_aa}{nnna}'
+ '{pa}{tta}{vs_i}'
+ '{ta}{vs_aa}{nnna}'
+ '{vs_e}{la}{pulli}{la}{vs_aa}{ma}{pulli}')
+ ] delete
+ (set found_a_match)
+ )
+ )
+ do fix_endings
+)
+
+define remove_vetrumai_urupukal as (
+ unset found_a_match
+ unset found_vetrumai_urupu
+ has_min_length
+ backwards (
+ (
+ test ( ['{nnna}{vs_ai}'] delete )
+ or
+ test ([ ( '{vs_i}{nnna}{vs_ai}' or
+ '{vs_ai}' (test not among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}'))) or
+ ( '{vs_ai}' (test (among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') '{pulli}')))
+ ] <- '{pulli}'
+ )
+ or
+ test ( [
+ '{vs_o}{tta}{vs_u}' or
+ '{vs_oo}{tta}{vs_u}' or
+ '{vs_i}{la}{pulli}' or
+ '{vs_i}{rra}{pulli}' or
+ ('{vs_i}{nnna}{pulli}' (test not '{ma}')) or
+ '{vs_i}{nnna}{pulli}{rra}{vs_u}' or
+ '{vs_i}{ra}{vs_u}{na}{pulli}{ta}{vs_u}' or
+ '{va}{vs_i}{tta}' or
+ ($(len >= 7) '{vs_i}{tta}{ma}{pulli}') or
+ '{vs_aa}{la}{pulli}' or
+ '{vs_u}{tta}{vs_ai}' or
+ '{vs_aa}{ma}{la}{pulli}' or
+ ('{la}{pulli}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or
+ '{vs_u}{lla}{pulli}'
+ ] <- '{pulli}'
+ )
+ or
+ test ( [
+ '{ka}{nna}{pulli}' or
+ '{ma}{vs_u}{nnna}{pulli}' or
+ '{ma}{vs_ee}{la}{pulli}' or
+ '{ma}{vs_ee}{rra}{pulli}' or
+ '{ka}{vs_ii}{llla}{pulli}' or
+ '{pa}{vs_i}{nnna}{pulli}' or
+ ('{ta}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}')))
+ ] delete
+ )
+ or
+ test ([ '{vs_ii}' ] <- '{vs_i}')
+ )
+ (set found_a_match)
+ (set found_vetrumai_urupu)
+ do ( [ '{vs_i}{nnna}{pulli}' ] <- '{pulli}' )
+ )
+ do fix_endings
+)
+
+define remove_tense_suffixes as (
+ set found_a_match
+ repeat ( found_a_match (do remove_tense_suffix) )
+)
+
+define remove_tense_suffix as (
+ unset found_a_match
+ has_min_length
+ backwards (
+ do (
+ test ( [among(
+ '{ka}{vs_o}{nna}{pulli}{tta}{vs_i}{ra}{pulli}'
+ '{pa}{tta}{vs_u}'
+ )] delete
+ (set found_a_match)
+ )
+ or
+ test ( [
+ '{ma}{vs_aa}{ra}{pulli}' or
+ '{ma}{vs_i}{nnna}{pulli}' or
+ '{nnna}{nnna}{pulli}' or
+ '{nnna}{vs_aa}{nnna}{pulli}' or
+ '{nnna}{vs_aa}{lla}{pulli}' or
+ '{nnna}{vs_aa}{ra}{pulli}' or
+ ('{va}{nnna}{pulli}' test (not among('{a}' '{aa}' '{i}' '{ii}' '{u}' '{uu}' '{e}' '{ee}' '{ai}' '{o}' '{oo}' '{au}')) ) or
+ '{nnna}{lla}{pulli}' or
+ '{va}{lla}{pulli}' or
+ '{nnna}{ra}{pulli}' or
+ '{va}{ra}{pulli}' or
+ '{nnna}' or '{pa}' or '{ka}' or '{ta}' or '{ya}' or
+ '{pa}{nnna}{pulli}' or
+ '{pa}{lla}{pulli}' or
+ '{pa}{ra}{pulli}' or
+ ('{ta}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or
+ '{vs_i}{rra}{pulli}{rra}{vs_u}' or
+ '{pa}{ma}{pulli}' or
+ '{nnna}{ma}{pulli}' or
+ '{ta}{vs_u}{ma}{pulli}' or
+ '{rra}{vs_u}{ma}{pulli}' or
+ '{ka}{vs_u}{ma}{pulli}' or
+ '{nnna}{vs_e}{nnna}{pulli}' or
+ '{nnna}{vs_ai}' or
+ '{va}{vs_ai}'
+ ] delete
+ (set found_a_match)
+ )
+ or
+ test ( [
+ ('{vs_aa}{nnna}{pulli}' test (not '{ca}')) or
+ '{vs_aa}{lla}{pulli}' or
+ '{vs_aa}{ra}{pulli}' or
+ '{vs_ee}{nnna}{pulli}' or
+ '{vs_aa}' or
+ '{vs_aa}{ma}{pulli}' or
+ '{vs_e}{ma}{pulli}' or
+ '{vs_ee}{ma}{pulli}' or
+ '{vs_oo}{ma}{pulli}' or
+ '{ka}{vs_u}{ma}{pulli}' or
+ '{ta}{vs_u}{ma}{pulli}' or
+ '{tta}{vs_u}{ma}{pulli}' or
+ '{rra}{vs_u}{ma}{pulli}' or
+ '{vs_aa}{ya}{pulli}' or
+ '{nnna}{vs_e}{nnna}{pulli}' or
+ '{nnna}{vs_i}{ra}{pulli}' or
+ '{vs_ii}{ra}{pulli}' or
+ '{vs_ii}{ya}{ra}{pulli}'
+ ] <- '{pulli}'
+ (set found_a_match)
+ )
+ or
+ test ( ([ '{ka}{vs_u}' or '{ta}{vs_u}' ) (test '{pulli}') ] delete
+ (set found_a_match)
+ )
+ )
+ do ([among(
+ '{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}'
+ '{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}{pulli}'
+ '{ka}{vs_i}{nnna}{pulli}{rra}'
+ '{ka}{vs_i}{nnna}{pulli}{rra}{pulli}'
+ '{ka}{vs_i}{rra}'
+ '{ka}{vs_i}{rra}{pulli}'
+ )] delete
+ (set found_a_match)
+ )
+ )
+ do fix_endings
+)
+
+define stem as (
+ unset found_vetrumai_urupu
+ do fix_ending
+ has_min_length
+ do remove_question_prefixes
+ do remove_pronoun_prefixes
+ do remove_question_suffixes
+ do remove_um
+ do remove_common_word_endings
+ do remove_vetrumai_urupukal
+ do remove_plural_suffix
+ do remove_command_suffixes
+ do remove_tense_suffixes
+)
diff --git a/contrib/snowball/algorithms/turkish.sbl b/contrib/snowball/algorithms/turkish.sbl
new file mode 100644
index 0000000..eadd61d
--- /dev/null
+++ b/contrib/snowball/algorithms/turkish.sbl
@@ -0,0 +1,470 @@
+/* Stemmer for Turkish
+ * author: Evren (Kapusuz) Çilden
+ * email: evren.kapusuz at gmail.com
+ * version: 1.0 (15.01.2007)
+
+
+ * stems nominal verb suffixes
+ * stems nominal inflections
+ * more than one syllable word check
+ * (y,n,s,U) context check
+ * vowel harmony check
+ * last consonant check and conversion (b, c, d, ğ to p, ç, t, k)
+
+ * The stemming algorithm is based on the paper "An Affix Stripping
+ * Morphological Analyzer for Turkish" by Gülşen Eryiğit and
+ * Eşref Adalı (Proceedings of the IAESTED International Conference
+ * ARTIFICIAL INTELLIGENCE AND APPLICATIONS, February 16-18,2004,
+ * Innsbruck, Austria
+
+ * Turkish is an agglutinative language and has a very rich morphological
+ * structure. In Turkish, you can form many different words from a single stem
+ * by appending a sequence of suffixes. Eg. The word "doktoruymuşsunuz" means
+ * "You had been the doctor of him". The stem of the word is "doktor" and it
+ * takes three different suffixes -sU, -ymUs, and -sUnUz. The rules about
+ * the append order of suffixes can be clearly described as FSMs.
+ * The paper referenced above defines some FSMs for right to left
+ * morphological analysis. I generated a method for constructing snowball
+ * expressions from right to left FSMs for stemming suffixes.
+*/
+
+routines (
+ append_U_to_stems_ending_with_d_or_g // for preventing some overstemmings
+ check_vowel_harmony // tests vowel harmony for suffixes
+ is_reserved_word // tests whether current string is a reserved word ('ad','soyad')
+ mark_cAsInA // nominal verb suffix
+ mark_DA // noun suffix
+ mark_DAn // noun suffix
+ mark_DUr // nominal verb suffix
+ mark_ki // noun suffix
+ mark_lAr // noun suffix, nominal verb suffix
+ mark_lArI // noun suffix
+ mark_nA // noun suffix
+ mark_ncA // noun suffix
+ mark_ndA // noun suffix
+ mark_ndAn // noun suffix
+ mark_nU // noun suffix
+ mark_nUn // noun suffix
+ mark_nUz // nominal verb suffix
+ mark_sU // noun suffix
+ mark_sUn // nominal verb suffix
+ mark_sUnUz // nominal verb suffix
+ mark_possessives // -(U)m,-(U)n,-(U)mUz,-(U)nUz,
+ mark_yA // noun suffix
+ mark_ylA // noun suffix
+ mark_yU // noun suffix
+ mark_yUm // nominal verb suffix
+ mark_yUz // nominal verb suffix
+ mark_yDU // nominal verb suffix
+ mark_yken // nominal verb suffix
+ mark_ymUs_ // nominal verb suffix
+ mark_ysA // nominal verb suffix
+
+ mark_suffix_with_optional_y_consonant
+ mark_suffix_with_optional_U_vowel
+ mark_suffix_with_optional_n_consonant
+ mark_suffix_with_optional_s_consonant
+
+ more_than_one_syllable_word
+
+ post_process_last_consonants
+ postlude
+
+ stem_nominal_verb_suffixes
+ stem_noun_suffixes
+ stem_suffix_chain_before_ki
+)
+
+stringescapes { }
+
+/* Special characters in Unicode Latin-1 and Latin Extended-A */
+stringdef c, '{U+00E7}' // LATIN SMALL LETTER C WITH CEDILLA
+stringdef g~ '{U+011F}' // LATIN SMALL LETTER G WITH BREVE
+stringdef i' '{U+0131}' // LATIN SMALL LETTER I WITHOUT DOT
+stringdef o" '{U+00F6}' // LATIN SMALL LETTER O WITH DIAERESIS
+stringdef s, '{U+015F}' // LATIN SMALL LETTER S WITH CEDILLA
+stringdef u" '{U+00FC}' // LATIN SMALL LETTER U WITH DIAERESIS
+
+booleans ( continue_stemming_noun_suffixes )
+
+groupings ( vowel U vowel1 vowel2 vowel3 vowel4 vowel5 vowel6)
+
+define vowel 'ae{i'}io{o"}u{u"}'
+define U '{i'}iu{u"}'
+
+// the vowel grouping definitions below are used for checking vowel harmony
+define vowel1 'a{i'}ou' // vowels that can end with suffixes containing 'a'
+define vowel2 'ei{o"}{u"}' // vowels that can end with suffixes containing 'e'
+define vowel3 'a{i'}' // vowels that can end with suffixes containing 'i''
+define vowel4 'ei' // vowels that can end with suffixes containing 'i'
+define vowel5 'ou' // vowels that can end with suffixes containing 'o' or 'u'
+define vowel6 '{o"}{u"}' // vowels that can end with suffixes containing 'o"' or 'u"'
+
+externals ( stem )
+
+backwardmode (
+ // checks vowel harmony for possible suffixes,
+ // helps to detect whether the candidate for suffix applies to vowel harmony
+ // this rule is added to prevent over stemming
+ define check_vowel_harmony as (
+ test
+ (
+ (goto vowel) // if there is a vowel
+ (
+ ('a' goto vowel1) or
+ ('e' goto vowel2) or
+ ('{i'}' goto vowel3) or
+ ('i' goto vowel4) or
+ ('o' goto vowel5) or
+ ('{o"}' goto vowel6) or
+ ('u' goto vowel5) or
+ ('{u"}' goto vowel6)
+ )
+ )
+ )
+
+ // if the last consonant before suffix is vowel and n then advance and delete
+ // if the last consonant before suffix is non vowel and n do nothing
+ // if the last consonant before suffix is not n then only delete the suffix
+ // assumption: slice beginning is set correctly
+ define mark_suffix_with_optional_n_consonant as (
+ ('n' (test vowel))
+ or
+ ((not(test 'n')) test(next vowel))
+
+ )
+
+ // if the last consonant before suffix is vowel and s then advance and delete
+ // if the last consonant before suffix is non vowel and s do nothing
+ // if the last consonant before suffix is not s then only delete the suffix
+ // assumption: slice beginning is set correctly
+ define mark_suffix_with_optional_s_consonant as (
+ ('s' (test vowel))
+ or
+ ((not(test 's')) test(next vowel))
+ )
+
+ // if the last consonant before suffix is vowel and y then advance and delete
+ // if the last consonant before suffix is non vowel and y do nothing
+ // if the last consonant before suffix is not y then only delete the suffix
+ // assumption: slice beginning is set correctly
+ define mark_suffix_with_optional_y_consonant as (
+ ('y' (test vowel))
+ or
+ ((not(test 'y')) test(next vowel))
+ )
+
+ define mark_suffix_with_optional_U_vowel as (
+ (U (test non-vowel))
+ or
+ ((not(test U)) test(next non-vowel))
+
+ )
+
+ define mark_possessives as (
+ among ('m{i'}z' 'miz' 'muz' 'm{u"}z'
+ 'n{i'}z' 'niz' 'nuz' 'n{u"}z' 'm' 'n')
+ (mark_suffix_with_optional_U_vowel)
+ )
+
+ define mark_sU as (
+ check_vowel_harmony
+ U
+ (mark_suffix_with_optional_s_consonant)
+ )
+
+ define mark_lArI as (
+ among ('leri' 'lar{i'}')
+ )
+
+ define mark_yU as (
+ check_vowel_harmony
+ U
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_nU as (
+ check_vowel_harmony
+ among ('n{i'}' 'ni' 'nu' 'n{u"}')
+ )
+
+ define mark_nUn as (
+ check_vowel_harmony
+ among ('{i'}n' 'in' 'un' '{u"}n')
+ (mark_suffix_with_optional_n_consonant)
+ )
+
+ define mark_yA as (
+ check_vowel_harmony
+ among('a' 'e')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_nA as (
+ check_vowel_harmony
+ among('na' 'ne')
+ )
+
+ define mark_DA as (
+ check_vowel_harmony
+ among('da' 'de' 'ta' 'te')
+ )
+
+ define mark_ndA as (
+ check_vowel_harmony
+ among('nda' 'nde')
+ )
+
+ define mark_DAn as (
+ check_vowel_harmony
+ among('dan' 'den' 'tan' 'ten')
+ )
+
+ define mark_ndAn as (
+ check_vowel_harmony
+ among('ndan' 'nden')
+ )
+
+ define mark_ylA as (
+ check_vowel_harmony
+ among('la' 'le')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_ki as (
+ 'ki'
+ )
+
+ define mark_ncA as (
+ check_vowel_harmony
+ among('ca' 'ce')
+ (mark_suffix_with_optional_n_consonant)
+ )
+
+ define mark_yUm as (
+ check_vowel_harmony
+ among ('{i'}m' 'im' 'um' '{u"}m')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_sUn as (
+ check_vowel_harmony
+ among ('s{i'}n' 'sin' 'sun' 's{u"}n' )
+ )
+
+ define mark_yUz as (
+ check_vowel_harmony
+ among ('{i'}z' 'iz' 'uz' '{u"}z')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_sUnUz as (
+ among ('s{i'}n{i'}z' 'siniz' 'sunuz' 's{u"}n{u"}z')
+ )
+
+ define mark_lAr as (
+ check_vowel_harmony
+ among ('ler' 'lar')
+ )
+
+ define mark_nUz as (
+ check_vowel_harmony
+ among ('n{i'}z' 'niz' 'nuz' 'n{u"}z')
+ )
+
+ define mark_DUr as (
+ check_vowel_harmony
+ among ('t{i'}r' 'tir' 'tur' 't{u"}r' 'd{i'}r' 'dir' 'dur' 'd{u"}r')
+ )
+
+ define mark_cAsInA as (
+ among ('cas{i'}na' 'cesine')
+ )
+
+ define mark_yDU as (
+ check_vowel_harmony
+ among ('t{i'}m' 'tim' 'tum' 't{u"}m' 'd{i'}m' 'dim' 'dum' 'd{u"}m'
+ 't{i'}n' 'tin' 'tun' 't{u"}n' 'd{i'}n' 'din' 'dun' 'd{u"}n'
+ 't{i'}k' 'tik' 'tuk' 't{u"}k' 'd{i'}k' 'dik' 'duk' 'd{u"}k'
+ 't{i'}' 'ti' 'tu' 't{u"}' 'd{i'}' 'di' 'du' 'd{u"}')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ // does not fully obey vowel harmony
+ define mark_ysA as (
+ among ('sam' 'san' 'sak' 'sem' 'sen' 'sek' 'sa' 'se')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_ymUs_ as (
+ check_vowel_harmony
+ among ('m{i'}{s,}' 'mi{s,}' 'mu{s,}' 'm{u"}{s,}')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_yken as (
+ 'ken' (mark_suffix_with_optional_y_consonant)
+ )
+
+ define stem_nominal_verb_suffixes as (
+ [
+ set continue_stemming_noun_suffixes
+ (mark_ymUs_ or mark_yDU or mark_ysA or mark_yken)
+ or
+ (mark_cAsInA (mark_sUnUz or mark_lAr or mark_yUm or mark_sUn or mark_yUz or true) mark_ymUs_)
+ or
+ (
+ mark_lAr ] delete try([(mark_DUr or mark_yDU or mark_ysA or mark_ymUs_))
+ unset continue_stemming_noun_suffixes
+ )
+ or
+ (mark_nUz (mark_yDU or mark_ysA))
+ or
+ ((mark_sUnUz or mark_yUz or mark_sUn or mark_yUm) ] delete try([ mark_ymUs_))
+ or
+ (mark_DUr ] delete try([ (mark_sUnUz or mark_lAr or mark_yUm or mark_sUn or mark_yUz or true) mark_ymUs_))
+ ]delete
+ )
+
+ // stems noun suffix chains ending with -ki
+ define stem_suffix_chain_before_ki as (
+ [
+ mark_ki
+ (
+ (mark_DA] delete try([
+ (mark_lAr] delete try(stem_suffix_chain_before_ki))
+ or
+ (mark_possessives] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+
+ ))
+ or
+ (mark_nUn] delete try([
+ (mark_lArI] delete)
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ (stem_suffix_chain_before_ki)
+ ))
+ or
+ (mark_ndA (
+ (mark_lArI] delete)
+ or
+ ((mark_sU] delete try([mark_lAr]delete stem_suffix_chain_before_ki)))
+ or
+ (stem_suffix_chain_before_ki)
+ ))
+ )
+ )
+
+ define stem_noun_suffixes as (
+ ([mark_lAr] delete try(stem_suffix_chain_before_ki))
+ or
+ ([mark_ncA] delete
+ try(
+ ([mark_lArI] delete)
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ ([mark_lAr] delete stem_suffix_chain_before_ki)
+ )
+ )
+ or
+ ([(mark_ndA or mark_nA)
+ (
+ (mark_lArI] delete)
+ or
+ (mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ (stem_suffix_chain_before_ki)
+ )
+ )
+ or
+ ([(mark_ndAn or mark_nU) ((mark_sU ] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) or (mark_lArI)))
+ or
+ ( [mark_DAn] delete try ([
+ (
+ (mark_possessives ] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ (mark_lAr] delete try(stem_suffix_chain_before_ki))
+ or
+ (stem_suffix_chain_before_ki)
+ ))
+ )
+ or
+ ([mark_nUn or mark_ylA] delete
+ try(
+ ([mark_lAr] delete stem_suffix_chain_before_ki)
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ stem_suffix_chain_before_ki
+ )
+ )
+ or
+ ([mark_lArI] delete)
+ or
+ (stem_suffix_chain_before_ki)
+ or
+ ([mark_DA or mark_yU or mark_yA] delete try([((mark_possessives] delete try([mark_lAr)) or mark_lAr) ] delete [ stem_suffix_chain_before_ki))
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ )
+
+ define post_process_last_consonants as (
+ [substring] among (
+ 'b' (<- 'p')
+ 'c' (<- '{c,}')
+ 'd' (<- 't')
+ '{g~}' (<- 'k')
+ )
+ )
+
+ // after stemming if the word ends with 'd' or 'g' most probably last U is overstemmed
+ // like in 'kedim' -> 'ked'
+ // Turkish words don't usually end with 'd' or 'g'
+ // some very well known words are ignored (like 'ad' 'soyad'
+ // appends U to stems ending with d or g, decides which vowel to add
+ // based on the last vowel in the stem
+ define append_U_to_stems_ending_with_d_or_g as (
+ test('d' or 'g')
+ (test((goto vowel) 'a' or '{i'}') <+ '{i'}')
+ or
+ (test((goto vowel) 'e' or 'i') <+ 'i')
+ or
+ (test((goto vowel) 'o' or 'u') <+ 'u')
+ or
+ (test((goto vowel) '{o"}' or '{u"}') <+ '{u"}')
+ )
+
+ define is_reserved_word as (
+ 'ad' try 'soy' atlimit
+ )
+)
+
+// Tests if there are more than one syllables
+// In Turkish each vowel indicates a distinct syllable
+define more_than_one_syllable_word as (
+ test (atleast 2 (gopast vowel))
+)
+
+define postlude as (
+ backwards (
+ not(is_reserved_word)
+ do append_U_to_stems_ending_with_d_or_g
+ do post_process_last_consonants
+
+ )
+)
+
+define stem as (
+ (more_than_one_syllable_word)
+ (
+ backwards (
+ do stem_nominal_verb_suffixes
+ continue_stemming_noun_suffixes
+ do stem_noun_suffixes
+ )
+
+ postlude
+ )
+)
diff --git a/contrib/snowball/charsets/ISO-8859-2.sbl b/contrib/snowball/charsets/ISO-8859-2.sbl
new file mode 100644
index 0000000..5829ea8
--- /dev/null
+++ b/contrib/snowball/charsets/ISO-8859-2.sbl
@@ -0,0 +1,98 @@
+// ISO-8859-2 character mappings.
+
+stringdef U+00A0 hex 'A0'
+stringdef U+0104 hex 'A1'
+stringdef U+02D8 hex 'A2'
+stringdef U+0141 hex 'A3'
+stringdef U+00A4 hex 'A4'
+stringdef U+013D hex 'A5'
+stringdef U+015A hex 'A6'
+stringdef U+00A7 hex 'A7'
+stringdef U+00A8 hex 'A8'
+stringdef U+0160 hex 'A9'
+stringdef U+015E hex 'AA'
+stringdef U+0164 hex 'AB'
+stringdef U+0179 hex 'AC'
+stringdef U+00AD hex 'AD'
+stringdef U+017D hex 'AE'
+stringdef U+017B hex 'AF'
+stringdef U+00B0 hex 'B0'
+stringdef U+0105 hex 'B1'
+stringdef U+02DB hex 'B2'
+stringdef U+0142 hex 'B3'
+stringdef U+00B4 hex 'B4'
+stringdef U+013E hex 'B5'
+stringdef U+015B hex 'B6'
+stringdef U+02C7 hex 'B7'
+stringdef U+00B8 hex 'B8'
+stringdef U+0161 hex 'B9'
+stringdef U+015F hex 'BA'
+stringdef U+0165 hex 'BB'
+stringdef U+017A hex 'BC'
+stringdef U+02DD hex 'BD'
+stringdef U+017E hex 'BE'
+stringdef U+017C hex 'BF'
+stringdef U+0154 hex 'C0'
+stringdef U+00C1 hex 'C1'
+stringdef U+00C2 hex 'C2'
+stringdef U+0102 hex 'C3'
+stringdef U+00C4 hex 'C4'
+stringdef U+0139 hex 'C5'
+stringdef U+0106 hex 'C6'
+stringdef U+00C7 hex 'C7'
+stringdef U+010C hex 'C8'
+stringdef U+00C9 hex 'C9'
+stringdef U+0118 hex 'CA'
+stringdef U+00CB hex 'CB'
+stringdef U+011A hex 'CC'
+stringdef U+00CD hex 'CD'
+stringdef U+00CE hex 'CE'
+stringdef U+010E hex 'CF'
+stringdef U+0110 hex 'D0'
+stringdef U+0143 hex 'D1'
+stringdef U+0147 hex 'D2'
+stringdef U+00D3 hex 'D3'
+stringdef U+00D4 hex 'D4'
+stringdef U+0150 hex 'D5'
+stringdef U+00D6 hex 'D6'
+stringdef U+00D7 hex 'D7'
+stringdef U+0158 hex 'D8'
+stringdef U+016E hex 'D9'
+stringdef U+00DA hex 'DA'
+stringdef U+0170 hex 'DB'
+stringdef U+00DC hex 'DC'
+stringdef U+00DD hex 'DD'
+stringdef U+0162 hex 'DE'
+stringdef U+00DF hex 'DF'
+stringdef U+0155 hex 'E0'
+stringdef U+00E1 hex 'E1'
+stringdef U+00E2 hex 'E2'
+stringdef U+0103 hex 'E3'
+stringdef U+00E4 hex 'E4'
+stringdef U+013A hex 'E5'
+stringdef U+0107 hex 'E6'
+stringdef U+00E7 hex 'E7'
+stringdef U+010D hex 'E8'
+stringdef U+00E9 hex 'E9'
+stringdef U+0119 hex 'EA'
+stringdef U+00EB hex 'EB'
+stringdef U+011B hex 'EC'
+stringdef U+00ED hex 'ED'
+stringdef U+00EE hex 'EE'
+stringdef U+010F hex 'EF'
+stringdef U+0111 hex 'F0'
+stringdef U+0144 hex 'F1'
+stringdef U+0148 hex 'F2'
+stringdef U+00F3 hex 'F3'
+stringdef U+00F4 hex 'F4'
+stringdef U+0151 hex 'F5'
+stringdef U+00F6 hex 'F6'
+stringdef U+00F7 hex 'F7'
+stringdef U+0159 hex 'F8'
+stringdef U+016F hex 'F9'
+stringdef U+00FA hex 'FA'
+stringdef U+0171 hex 'FB'
+stringdef U+00FC hex 'FC'
+stringdef U+00FD hex 'FD'
+stringdef U+0163 hex 'FE'
+stringdef U+02D9 hex 'FF'
diff --git a/contrib/snowball/charsets/KOI8-R.sbl b/contrib/snowball/charsets/KOI8-R.sbl
new file mode 100644
index 0000000..46854e8
--- /dev/null
+++ b/contrib/snowball/charsets/KOI8-R.sbl
@@ -0,0 +1,74 @@
+// KOI8-R character mappings.
+
+stringdef U+00A0 hex '9A'
+stringdef U+00A9 hex 'BF'
+stringdef U+00B0 hex '9C'
+stringdef U+00B2 hex '9D'
+stringdef U+00B7 hex '9E'
+stringdef U+00F7 hex '9F'
+stringdef U+0401 hex 'B3'
+stringdef U+0410 hex 'E1'
+stringdef U+0411 hex 'E2'
+stringdef U+0412 hex 'F7'
+stringdef U+0413 hex 'E7'
+stringdef U+0414 hex 'E4'
+stringdef U+0415 hex 'E5'
+stringdef U+0416 hex 'F6'
+stringdef U+0417 hex 'FA'
+stringdef U+0418 hex 'E9'
+stringdef U+0419 hex 'EA'
+stringdef U+041A hex 'EB'
+stringdef U+041B hex 'EC'
+stringdef U+041C hex 'ED'
+stringdef U+041D hex 'EE'
+stringdef U+041E hex 'EF'
+stringdef U+041F hex 'F0'
+stringdef U+0420 hex 'F2'
+stringdef U+0421 hex 'F3'
+stringdef U+0422 hex 'F4'
+stringdef U+0423 hex 'F5'
+stringdef U+0424 hex 'E6'
+stringdef U+0425 hex 'E8'
+stringdef U+0426 hex 'E3'
+stringdef U+0427 hex 'FE'
+stringdef U+0428 hex 'FB'
+stringdef U+0429 hex 'FD'
+stringdef U+042A hex 'FF'
+stringdef U+042B hex 'F9'
+stringdef U+042C hex 'F8'
+stringdef U+042D hex 'FC'
+stringdef U+042E hex 'E0'
+stringdef U+042F hex 'F1'
+stringdef U+0430 hex 'C1'
+stringdef U+0431 hex 'C2'
+stringdef U+0432 hex 'D7'
+stringdef U+0433 hex 'C7'
+stringdef U+0434 hex 'C4'
+stringdef U+0435 hex 'C5'
+stringdef U+0436 hex 'D6'
+stringdef U+0437 hex 'DA'
+stringdef U+0438 hex 'C9'
+stringdef U+0439 hex 'CA'
+stringdef U+043A hex 'CB'
+stringdef U+043B hex 'CC'
+stringdef U+043C hex 'CD'
+stringdef U+043D hex 'CE'
+stringdef U+043E hex 'CF'
+stringdef U+043F hex 'D0'
+stringdef U+0440 hex 'D2'
+stringdef U+0441 hex 'D3'
+stringdef U+0442 hex 'D4'
+stringdef U+0443 hex 'D5'
+stringdef U+0444 hex 'C6'
+stringdef U+0445 hex 'C8'
+stringdef U+0446 hex 'C3'
+stringdef U+0447 hex 'DE'
+stringdef U+0448 hex 'DB'
+stringdef U+0449 hex 'DD'
+stringdef U+044A hex 'DF'
+stringdef U+044B hex 'D9'
+stringdef U+044C hex 'D8'
+stringdef U+044D hex 'DC'
+stringdef U+044E hex 'C0'
+stringdef U+044F hex 'D1'
+stringdef U+0451 hex 'A3'
diff --git a/contrib/snowball/charsets/cp850.sbl b/contrib/snowball/charsets/cp850.sbl
new file mode 100644
index 0000000..b780220
--- /dev/null
+++ b/contrib/snowball/charsets/cp850.sbl
@@ -0,0 +1,130 @@
+// Code page 850 (MSDOS Latin 1) character mappings.
+
+stringdef U+00A0 hex 'FF'
+stringdef U+00A1 hex 'AD'
+stringdef U+00A2 hex 'BD'
+stringdef U+00A3 hex '9C'
+stringdef U+00A4 hex 'CF'
+stringdef U+00A5 hex 'BE'
+stringdef U+00A6 hex 'DD'
+stringdef U+00A7 hex 'F5'
+stringdef U+00A8 hex 'F9'
+stringdef U+00A9 hex 'B8'
+stringdef U+00AA hex 'A6'
+stringdef U+00AB hex 'AE'
+stringdef U+00AC hex 'AA'
+stringdef U+00AD hex 'F0'
+stringdef U+00AE hex 'A9'
+stringdef U+00AF hex 'EE'
+stringdef U+00B0 hex 'F8'
+stringdef U+00B1 hex 'F1'
+stringdef U+00B2 hex 'FD'
+stringdef U+00B3 hex 'FC'
+stringdef U+00B4 hex 'EF'
+stringdef U+00B5 hex 'E6'
+stringdef U+00B6 hex 'F4'
+stringdef U+00B7 hex 'FA'
+stringdef U+00B8 hex 'F7'
+stringdef U+00B9 hex 'FB'
+stringdef U+00BA hex 'A7'
+stringdef U+00BB hex 'AF'
+stringdef U+00BC hex 'AC'
+stringdef U+00BD hex 'AB'
+stringdef U+00BE hex 'F3'
+stringdef U+00BF hex 'A8'
+stringdef U+00C0 hex 'B7'
+stringdef U+00C1 hex 'B5'
+stringdef U+00C2 hex 'B6'
+stringdef U+00C3 hex 'C7'
+stringdef U+00C4 hex '8E'
+stringdef U+00C5 hex '8F'
+stringdef U+00C6 hex '92'
+stringdef U+00C7 hex '80'
+stringdef U+00C8 hex 'D4'
+stringdef U+00C9 hex '90'
+stringdef U+00CA hex 'D2'
+stringdef U+00CB hex 'D3'
+stringdef U+00CC hex 'DE'
+stringdef U+00CD hex 'D6'
+stringdef U+00CE hex 'D7'
+stringdef U+00CF hex 'D8'
+stringdef U+00D0 hex 'D1'
+stringdef U+00D1 hex 'A5'
+stringdef U+00D2 hex 'E3'
+stringdef U+00D3 hex 'E0'
+stringdef U+00D4 hex 'E2'
+stringdef U+00D5 hex 'E5'
+stringdef U+00D6 hex '99'
+stringdef U+00D7 hex '9E'
+stringdef U+00D8 hex '9D'
+stringdef U+00D9 hex 'EB'
+stringdef U+00DA hex 'E9'
+stringdef U+00DB hex 'EA'
+stringdef U+00DC hex '9A'
+stringdef U+00DD hex 'ED'
+stringdef U+00DE hex 'E8'
+stringdef U+00DF hex 'E1'
+stringdef U+00E0 hex '85'
+stringdef U+00E1 hex 'A0'
+stringdef U+00E2 hex '83'
+stringdef U+00E3 hex 'C6'
+stringdef U+00E4 hex '84'
+stringdef U+00E5 hex '86'
+stringdef U+00E6 hex '91'
+stringdef U+00E7 hex '87'
+stringdef U+00E8 hex '8A'
+stringdef U+00E9 hex '82'
+stringdef U+00EA hex '88'
+stringdef U+00EB hex '89'
+stringdef U+00EC hex '8D'
+stringdef U+00ED hex 'A1'
+stringdef U+00EE hex '8C'
+stringdef U+00EF hex '8B'
+stringdef U+00F0 hex 'D0'
+stringdef U+00F1 hex 'A4'
+stringdef U+00F2 hex '95'
+stringdef U+00F3 hex 'A2'
+stringdef U+00F4 hex '93'
+stringdef U+00F5 hex 'E4'
+stringdef U+00F6 hex '94'
+stringdef U+00F7 hex 'F6'
+stringdef U+00F8 hex '9B'
+stringdef U+00F9 hex '97'
+stringdef U+00FA hex 'A3'
+stringdef U+00FB hex '96'
+stringdef U+00FC hex '81'
+stringdef U+00FD hex 'EC'
+stringdef U+00FE hex 'E7'
+stringdef U+00FF hex '98'
+stringdef U+0131 hex 'D5'
+stringdef U+0192 hex '9F'
+stringdef U+2017 hex 'F2'
+stringdef U+2500 hex 'C4'
+stringdef U+2502 hex 'B3'
+stringdef U+250C hex 'DA'
+stringdef U+2510 hex 'BF'
+stringdef U+2514 hex 'C0'
+stringdef U+2518 hex 'D9'
+stringdef U+251C hex 'C3'
+stringdef U+2524 hex 'B4'
+stringdef U+252C hex 'C2'
+stringdef U+2534 hex 'C1'
+stringdef U+253C hex 'C5'
+stringdef U+2550 hex 'CD'
+stringdef U+2551 hex 'BA'
+stringdef U+2554 hex 'C9'
+stringdef U+2557 hex 'BB'
+stringdef U+255A hex 'C8'
+stringdef U+255D hex 'BC'
+stringdef U+2560 hex 'CC'
+stringdef U+2563 hex 'B9'
+stringdef U+2566 hex 'CB'
+stringdef U+2569 hex 'CA'
+stringdef U+256C hex 'CE'
+stringdef U+2580 hex 'DF'
+stringdef U+2584 hex 'DC'
+stringdef U+2588 hex 'DB'
+stringdef U+2591 hex 'B0'
+stringdef U+2592 hex 'B1'
+stringdef U+2593 hex 'B2'
+stringdef U+25A0 hex 'FE'
diff --git a/contrib/snowball/compiler/analyser.c b/contrib/snowball/compiler/analyser.c
new file mode 100644
index 0000000..dffa555
--- /dev/null
+++ b/contrib/snowball/compiler/analyser.c
@@ -0,0 +1,1380 @@
+
+#include <stdio.h> /* printf etc */
+#include <stdlib.h> /* exit */
+#include <string.h> /* memmove */
+#include "header.h"
+
+typedef enum {
+ e_token_omitted = 0,
+ e_unexpected_token = 1,
+ e_string_omitted = 2,
+ e_unexpected_token_in_among = 3,
+ /* For codes above here, report "after " t->previous_token after the error. */
+ e_unresolved_substring = 14,
+ e_not_allowed_inside_reverse = 15,
+ e_empty_grouping = 16,
+ e_already_backwards = 17,
+ e_empty_among = 18,
+ e_adjacent_bracketed_in_among = 19,
+ e_substring_preceded_by_substring = 20,
+ /* For codes below here, tokeniser->b is printed before the error. */
+ e_redeclared = 30,
+ e_undeclared = 31,
+ e_declared_as_different_mode = 32,
+ e_not_of_type_x = 33,
+ e_not_of_type_string_or_integer = 34,
+ e_misplaced = 35,
+ e_redefined = 36,
+ e_misused = 37
+} error_code;
+
+/* recursive usage: */
+
+static void read_program_(struct analyser * a, int terminator);
+static struct node * read_C(struct analyser * a);
+static struct node * C_style(struct analyser * a, const char * s, int token);
+
+
+static void print_node_(struct node * p, int n, const char * s) {
+
+ int i;
+ for (i = 0; i < n; i++) fputs(i == n - 1 ? s : " ", stdout);
+ printf("%s ", name_of_token(p->type));
+ if (p->name) report_b(stdout, p->name->b);
+ if (p->literalstring) {
+ printf("'");
+ report_b(stdout, p->literalstring);
+ printf("'");
+ }
+ printf("\n");
+ if (p->AE) print_node_(p->AE, n+1, "# ");
+ if (p->left) print_node_(p->left, n+1, " ");
+ if (p->aux) print_node_(p->aux, n+1, "@ ");
+ if (p->right) print_node_(p->right, n, " ");
+}
+
+extern void print_program(struct analyser * a) {
+ print_node_(a->program, 0, " ");
+}
+
+static struct node * new_node(struct analyser * a, int type) {
+ NEW(node, p);
+ p->next = a->nodes; a->nodes = p;
+ p->left = 0;
+ p->right = 0;
+ p->aux = 0;
+ p->AE = 0;
+ p->name = 0;
+ p->literalstring = 0;
+ p->mode = a->mode;
+ p->line_number = a->tokeniser->line_number;
+ p->type = type;
+ return p;
+}
+
+static const char * name_of_mode(int n) {
+ switch (n) {
+ case m_backward: return "string backward";
+ case m_forward: return "string forward";
+ /* case m_integer: return "integer"; */
+ }
+ fprintf(stderr, "Invalid mode %d in name_of_mode()\n", n);
+ exit(1);
+}
+
+static const char * name_of_type(int n) {
+ switch (n) {
+ case 's': return "string";
+ case 'i': return "integer";
+ case 'r': return "routine";
+ case 'R': return "routine or grouping";
+ case 'g': return "grouping";
+ }
+ fprintf(stderr, "Invalid type %d in name_of_type()\n", n);
+ exit(1);
+}
+
+static const char * name_of_name_type(int code) {
+ switch (code) {
+ case t_string: return "string";
+ case t_boolean: return "boolean";
+ case t_integer: return "integer";
+ case t_routine: return "routine";
+ case t_external: return "external";
+ case t_grouping: return "grouping";
+ }
+ fprintf(stderr, "Invalid type code %d in name_of_name_type()\n", code);
+ exit(1);
+}
+
+static void count_error(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ if (t->error_count >= 20) { fprintf(stderr, "... etc\n"); exit(1); }
+ t->error_count++;
+}
+
+static void error2(struct analyser * a, error_code n, int x) {
+ struct tokeniser * t = a->tokeniser;
+ count_error(a);
+ fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+ if ((int)n >= (int)e_redeclared) report_b(stderr, t->b);
+ switch (n) {
+ case e_token_omitted:
+ fprintf(stderr, "%s omitted", name_of_token(t->omission)); break;
+ case e_unexpected_token_in_among:
+ fprintf(stderr, "in among(...), ");
+ /* fall through */
+ case e_unexpected_token:
+ fprintf(stderr, "unexpected %s", name_of_token(t->token));
+ if (t->token == c_number) fprintf(stderr, " %d", t->number);
+ if (t->token == c_name) {
+ fprintf(stderr, " ");
+ report_b(stderr, t->b);
+ } break;
+ case e_string_omitted:
+ fprintf(stderr, "string omitted"); break;
+
+ case e_unresolved_substring:
+ fprintf(stderr, "unresolved substring on line %d", x); break;
+ case e_not_allowed_inside_reverse:
+ fprintf(stderr, "%s not allowed inside reverse(...)", name_of_token(t->token)); break;
+ case e_empty_grouping:
+ fprintf(stderr, "empty grouping"); break;
+ case e_already_backwards:
+ fprintf(stderr, "backwards used when already in this mode"); break;
+ case e_empty_among:
+ fprintf(stderr, "empty among(...)"); break;
+ case e_adjacent_bracketed_in_among:
+ fprintf(stderr, "two adjacent bracketed expressions in among(...)"); break;
+ case e_substring_preceded_by_substring:
+ fprintf(stderr, "substring preceded by another substring on line %d", x); break;
+
+ case e_redeclared:
+ fprintf(stderr, " re-declared"); break;
+ case e_undeclared:
+ fprintf(stderr, " undeclared"); break;
+ case e_declared_as_different_mode:
+ fprintf(stderr, " declared as %s mode; used as %s mode",
+ name_of_mode(a->mode), name_of_mode(x)); break;
+ case e_not_of_type_x:
+ fprintf(stderr, " not of type %s", name_of_type(x)); break;
+ case e_not_of_type_string_or_integer:
+ fprintf(stderr, " not of type string or integer"); break;
+ case e_misplaced:
+ fprintf(stderr, " misplaced"); break;
+ case e_redefined:
+ fprintf(stderr, " redefined"); break;
+ case e_misused:
+ fprintf(stderr, " mis-used as %s mode",
+ name_of_mode(x)); break;
+ }
+ if ((int)n < (int)e_unresolved_substring && t->previous_token > 0)
+ fprintf(stderr, " after %s", name_of_token(t->previous_token));
+ fprintf(stderr, "\n");
+}
+
+static void error(struct analyser * a, error_code n) { error2(a, n, 0); }
+
+static void error4(struct analyser * a, struct name * q) {
+ count_error(a);
+ fprintf(stderr, "%s:%d: ", a->tokeniser->file, q->used->line_number);
+ report_b(stderr, q->b);
+ fprintf(stderr, " undefined\n");
+}
+
+static void omission_error(struct analyser * a, int n) {
+ a->tokeniser->omission = n;
+ error(a, e_token_omitted);
+}
+
+static int check_token(struct analyser * a, int code) {
+ struct tokeniser * t = a->tokeniser;
+ if (t->token != code) { omission_error(a, code); return false; }
+ return true;
+}
+
+static int get_token(struct analyser * a, int code) {
+ struct tokeniser * t = a->tokeniser;
+ read_token(t);
+ {
+ int x = check_token(a, code);
+ if (!x) t->token_held = true;
+ return x;
+ }
+}
+
+static struct name * look_for_name(struct analyser * a) {
+ symbol * q = a->tokeniser->b;
+ struct name * p;
+ for (p = a->names; p; p = p->next) {
+ symbol * b = p->b;
+ int n = SIZE(b);
+ if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) {
+ p->referenced = true;
+ return p;
+ }
+ }
+ return 0;
+}
+
+static struct name * find_name(struct analyser * a) {
+ struct name * p = look_for_name(a);
+ if (p == 0) error(a, e_undeclared);
+ return p;
+}
+
+static void check_routine_mode(struct analyser * a, struct name * p, int mode) {
+ if (p->mode < 0) p->mode = mode; else
+ if (p->mode != mode) error2(a, e_misused, mode);
+}
+
+static void check_name_type(struct analyser * a, struct name * p, int type) {
+ switch (type) {
+ case 's':
+ if (p->type == t_string) return;
+ break;
+ case 'i':
+ if (p->type == t_integer) return;
+ break;
+ case 'b':
+ if (p->type == t_boolean) return;
+ break;
+ case 'R':
+ if (p->type == t_grouping) return;
+ /* FALLTHRU */
+ case 'r':
+ if (p->type == t_routine || p->type == t_external) return;
+ break;
+ case 'g':
+ if (p->type == t_grouping) return;
+ break;
+ }
+ error2(a, e_not_of_type_x, type);
+}
+
+static void read_names(struct analyser * a, int type) {
+ struct tokeniser * t = a->tokeniser;
+ if (!get_token(a, c_bra)) return;
+ while (true) {
+ int token = read_token(t);
+ switch (token) {
+ case c_len: {
+ /* Context-sensitive token - once declared as a name, it loses
+ * its special meaning, for compatibility with older versions
+ * of snowball.
+ */
+ static const symbol c_len_lit[] = {
+ 'l', 'e', 'n'
+ };
+ MOVE_TO_B(t->b, c_len_lit);
+ goto handle_as_name;
+ }
+ case c_lenof: {
+ /* Context-sensitive token - once declared as a name, it loses
+ * its special meaning, for compatibility with older versions
+ * of snowball.
+ */
+ static const symbol c_lenof_lit[] = {
+ 'l', 'e', 'n', 'o', 'f'
+ };
+ MOVE_TO_B(t->b, c_lenof_lit);
+ goto handle_as_name;
+ }
+ case c_name:
+handle_as_name:
+ if (look_for_name(a) != 0) error(a, e_redeclared); else {
+ NEW(name, p);
+ p->b = copy_b(t->b);
+ p->type = type;
+ p->mode = -1; /* routines, externals */
+ /* We defer assigning counts until after we've eliminated
+ * variables whose values are never used. */
+ p->count = -1;
+ p->referenced = false;
+ p->used_in_among = false;
+ p->used = 0;
+ p->value_used = false;
+ p->initialised = false;
+ p->used_in_definition = false;
+ p->local_to = 0;
+ p->grouping = 0;
+ p->definition = 0;
+ p->declaration_line_number = t->line_number;
+ p->next = a->names;
+ a->names = p;
+ if (token != c_name) {
+ disable_token(t, token);
+ }
+ }
+ break;
+ default:
+ if (!check_token(a, c_ket)) t->token_held = true;
+ return;
+ }
+ }
+}
+
+static symbol * new_literalstring(struct analyser * a) {
+ NEW(literalstring, p);
+ p->b = copy_b(a->tokeniser->b);
+ p->next = a->literalstrings;
+ a->literalstrings = p;
+ return p->b;
+}
+
+static int read_AE_test(struct analyser * a) {
+
+ struct tokeniser * t = a->tokeniser;
+ switch (read_token(t)) {
+ case c_assign: return c_mathassign;
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le: return t->token;
+ default: error(a, e_unexpected_token); t->token_held = true; return c_eq;
+ }
+}
+
+static int binding(int t) {
+ switch (t) {
+ case c_plus: case c_minus: return 1;
+ case c_multiply: case c_divide: return 2;
+ default: return -2;
+ }
+}
+
+static void mark_used_in(struct analyser * a, struct name * q, struct node * p) {
+ if (!q->used) {
+ q->used = p;
+ q->local_to = a->program_end->name;
+ } else if (q->local_to) {
+ if (q->local_to != a->program_end->name) {
+ /* Used in more than one routine/external. */
+ q->local_to = NULL;
+ }
+ }
+}
+
+static void name_to_node(struct analyser * a, struct node * p, int type) {
+ struct name * q = find_name(a);
+ if (q) {
+ check_name_type(a, q, type);
+ mark_used_in(a, q, p);
+ }
+ p->name = q;
+}
+
+static struct node * read_AE(struct analyser * a, int B) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p;
+ struct node * q;
+ switch (read_token(t)) {
+ case c_minus: /* monadic */
+ q = read_AE(a, 100);
+ if (q->type == c_neg) {
+ /* Optimise away double negation, which avoids generators
+ * having to worry about generating "--" (decrement operator
+ * in many languages).
+ */
+ p = q->right;
+ /* Don't free q, it's in the linked list a->nodes. */
+ break;
+ }
+ p = new_node(a, c_neg);
+ p->right = q;
+ break;
+ case c_bra:
+ p = read_AE(a, 0);
+ get_token(a, c_ket);
+ break;
+ case c_name:
+ p = new_node(a, c_name);
+ name_to_node(a, p, 'i');
+ if (p->name) p->name->value_used = true;
+ break;
+ case c_maxint:
+ case c_minint:
+ a->int_limits_used = true;
+ /* fall through */
+ case c_cursor:
+ case c_limit:
+ case c_len:
+ case c_size:
+ p = new_node(a, t->token);
+ break;
+ case c_number:
+ p = new_node(a, c_number);
+ p->number = t->number;
+ break;
+ case c_lenof:
+ case c_sizeof:
+ p = C_style(a, "s", t->token);
+ break;
+ default:
+ error(a, e_unexpected_token);
+ t->token_held = true;
+ return 0;
+ }
+ while (true) {
+ int token = read_token(t);
+ int b = binding(token);
+ if (binding(token) <= B) {
+ t->token_held = true;
+ return p;
+ }
+ q = new_node(a, token);
+ q->left = p;
+ q->right = read_AE(a, b);
+ p = q;
+ }
+}
+
+static struct node * read_C_connection(struct analyser * a, struct node * q, int op) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, op);
+ struct node * p_end = q;
+ p->left = q;
+ do {
+ q = read_C(a);
+ p_end->right = q; p_end = q;
+ } while (read_token(t) == op);
+ t->token_held = true;
+ return p;
+}
+
+static struct node * read_C_list(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, c_bra);
+ struct node * p_end = 0;
+ while (true) {
+ int token = read_token(t);
+ if (token == c_ket) return p;
+ if (token < 0) { omission_error(a, c_ket); return p; }
+ t->token_held = true;
+ {
+ struct node * q = read_C(a);
+ while (true) {
+ token = read_token(t);
+ if (token != c_and && token != c_or) {
+ t->token_held = true;
+ break;
+ }
+ q = read_C_connection(a, q, token);
+ }
+ if (p_end == 0) p->left = q; else p_end->right = q;
+ p_end = q;
+ }
+ }
+}
+
+static struct node * C_style(struct analyser * a, const char * s, int token) {
+ int i;
+ struct node * p = new_node(a, token);
+ for (i = 0; s[i] != 0; i++) switch (s[i]) {
+ case 'C':
+ p->left = read_C(a); continue;
+ case 'D':
+ p->aux = read_C(a); continue;
+ case 'A':
+ p->AE = read_AE(a, 0); continue;
+ case 'f':
+ get_token(a, c_for); continue;
+ case 'S':
+ {
+ int str_token = read_token(a->tokeniser);
+ if (str_token == c_name) name_to_node(a, p, 's'); else
+ if (str_token == c_literalstring) p->literalstring = new_literalstring(a);
+ else error(a, e_string_omitted);
+ }
+ continue;
+ case 'b':
+ case 's':
+ case 'i':
+ if (get_token(a, c_name)) name_to_node(a, p, s[i]);
+ continue;
+ }
+ return p;
+}
+
+static struct node * read_literalstring(struct analyser * a) {
+ struct node * p = new_node(a, c_literalstring);
+ p->literalstring = new_literalstring(a);
+ return p;
+}
+
+static void reverse_b(symbol * b) {
+ int i = 0; int j = SIZE(b) - 1;
+ while (i < j) {
+ int ch1 = b[i]; int ch2 = b[j];
+ b[i++] = ch2; b[j--] = ch1;
+ }
+}
+
+static int compare_amongvec(const void *pv, const void *qv) {
+ const struct amongvec * p = (const struct amongvec*)pv;
+ const struct amongvec * q = (const struct amongvec*)qv;
+ symbol * b_p = p->b; int p_size = p->size;
+ symbol * b_q = q->b; int q_size = q->size;
+ int smaller_size = p_size < q_size ? p_size : q_size;
+ int i;
+ for (i = 0; i < smaller_size; i++)
+ if (b_p[i] != b_q[i]) return b_p[i] - b_q[i];
+ if (p_size - q_size)
+ return p_size - q_size;
+ return p->line_number - q->line_number;
+}
+
+#define PTR_NULL_CHECK(P, Q) do {\
+ if ((Q) == NULL) {\
+ if ((P) != NULL) return 1;\
+ } else {\
+ if ((P) == NULL) return -1;\
+ }\
+ } while (0)
+
+static int compare_node(const struct node *p, const struct node *q) {
+ PTR_NULL_CHECK(p, q);
+ if (q == NULL) {
+ /* p must be NULL too. */
+ return 0;
+ }
+
+ if (p->type != q->type) return p->type > q->type ? 1 : -1;
+ if (p->mode != q->mode) return p->mode > q->mode ? 1 : -1;
+ if (p->type == c_number) {
+ if (p->number != q->number)
+ return p->number > q->number ? 1 : -1;
+ }
+
+ PTR_NULL_CHECK(p->left, q->left);
+ if (p->left) {
+ int r = compare_node(p->left, q->left);
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->AE, q->AE);
+ if (p->AE) {
+ int r = compare_node(p->AE, q->AE);
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->aux, q->aux);
+ if (p->aux) {
+ int r = compare_node(p->aux, q->aux);
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->name, q->name);
+ if (p->name) {
+ int r;
+ if (SIZE(p->name->b) != SIZE(q->name->b)) {
+ return SIZE(p->name->b) - SIZE(q->name->b);
+ }
+ r = memcmp(p->name->b, q->name->b,
+ SIZE(p->name->b) * sizeof(symbol));
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->literalstring, q->literalstring);
+ if (p->literalstring) {
+ int r;
+ if (SIZE(p->literalstring) != SIZE(q->literalstring)) {
+ return SIZE(p->literalstring) - SIZE(q->literalstring);
+ }
+ r = memcmp(p->literalstring, q->literalstring,
+ SIZE(p->literalstring) * sizeof(symbol));
+ if (r != 0) return r;
+ }
+
+ return compare_node(p->right, q->right);
+}
+
+static void make_among(struct analyser * a, struct node * p, struct node * substring) {
+
+ NEW(among, x);
+ NEWVEC(amongvec, v, p->number);
+ struct node * q = p->left;
+ struct amongvec * w0 = v;
+ struct amongvec * w1 = v;
+ int result = 1;
+
+ int direction = substring != 0 ? substring->mode : p->mode;
+ int backward = direction == m_backward;
+
+ if (a->amongs == 0) a->amongs = x; else a->amongs_end->next = x;
+ a->amongs_end = x;
+ x->next = 0;
+ x->b = v;
+ x->number = a->among_count++;
+ x->function_count = 0;
+ x->starter = 0;
+ x->nocommand_count = 0;
+ x->amongvar_needed = false;
+
+ if (q->type == c_bra) { x->starter = q; q = q->right; }
+
+ while (q) {
+ if (q->type == c_literalstring) {
+ symbol * b = q->literalstring;
+ w1->b = b; /* pointer to case string */
+ w1->action = NULL; /* action gets filled in below */
+ w1->line_number = q->line_number;
+ w1->size = SIZE(b); /* number of characters in string */
+ w1->i = -1; /* index of longest substring */
+ w1->result = -1; /* number of corresponding case expression */
+ if (q->left) {
+ struct name * function = q->left->name;
+ w1->function = function;
+ function->used_in_among = true;
+ check_routine_mode(a, function, direction);
+ x->function_count++;
+ } else {
+ w1->function = 0;
+ }
+ w1++;
+ } else if (q->left == 0) {
+ /* empty command: () */
+ w0 = w1;
+ } else {
+ /* Check for previous action which is the same as this one and use
+ * the same action code if we find one.
+ */
+ int among_result = -1;
+ struct amongvec * w;
+ for (w = v; w < w0; ++w) {
+ if (w->action && compare_node(w->action->left, q->left) == 0) {
+ if (w->result <= 0) {
+ printf("Among code %d isn't positive\n", w->result);
+ exit(1);
+ }
+ among_result = w->result;
+ break;
+ }
+ }
+ if (among_result < 0) {
+ among_result = result++;
+ }
+
+ while (w0 != w1) {
+ w0->action = q;
+ w0->result = among_result;
+ w0++;
+ }
+ }
+ q = q->right;
+ }
+ if (w1-v != p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); }
+ x->command_count = result - 1;
+ {
+ NEWVEC(node*, commands, x->command_count);
+ memset(commands, 0, x->command_count * sizeof(struct node*));
+ for (w0 = v; w0 < w1; w0++) {
+ if (w0->result > 0) {
+ /* result == -1 when there's no command. */
+ if (w0->result > x->command_count) {
+ fprintf(stderr, "More among codes than expected\n");
+ exit(1);
+ }
+ if (!commands[w0->result - 1])
+ commands[w0->result - 1] = w0->action;
+ } else {
+ ++x->nocommand_count;
+ }
+ if (backward) reverse_b(w0->b);
+ }
+ x->commands = commands;
+ }
+ qsort(v, w1 - v, sizeof(struct amongvec), compare_amongvec);
+
+ /* the following loop is O(n squared) */
+ for (w0 = w1 - 1; w0 >= v; w0--) {
+ symbol * b = w0->b;
+ int size = w0->size;
+ struct amongvec * w;
+
+ for (w = w0 - 1; w >= v; w--) {
+ if (w->size < size && memcmp(w->b, b, w->size * sizeof(symbol)) == 0) {
+ w0->i = w - v; /* fill in index of longest substring */
+ break;
+ }
+ }
+ }
+ if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b);
+
+ for (w0 = v; w0 < w1 - 1; w0++)
+ if (w0->size == (w0 + 1)->size &&
+ memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) {
+ count_error(a);
+ fprintf(stderr, "%s:%d: among(...) has repeated string '",
+ a->tokeniser->file, (w0 + 1)->line_number);
+ report_b(stderr, (w0 + 1)->b);
+ fprintf(stderr, "'\n");
+ count_error(a);
+ fprintf(stderr, "%s:%d: previously seen here\n",
+ a->tokeniser->file, w0->line_number);
+ }
+
+ x->literalstring_count = p->number;
+ p->among = x;
+
+ x->substring = substring;
+ if (substring != 0) substring->among = x;
+ if (x->command_count > 1 ||
+ (x->command_count == 1 && x->nocommand_count > 0) ||
+ x->starter != 0) {
+ /* We need to set among_var rather than just checking if find_among*()
+ * returns zero or not.
+ */
+ x->amongvar_needed = a->amongvar_needed = true;
+ }
+}
+
+static struct node * read_among(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, c_among);
+ struct node * p_end = 0;
+ int previous_token = -1;
+ struct node * substring = a->substring;
+
+ a->substring = 0;
+ p->number = 0; /* counts the number of literals */
+ if (!get_token(a, c_bra)) return p;
+ while (true) {
+ struct node * q;
+ int token = read_token(t);
+ switch (token) {
+ case c_literalstring:
+ q = read_literalstring(a);
+ if (read_token(t) == c_name) {
+ struct node * r = new_node(a, c_name);
+ name_to_node(a, r, 'r');
+ q->left = r;
+ }
+ else t->token_held = true;
+ p->number++; break;
+ case c_bra:
+ if (previous_token == c_bra) error(a, e_adjacent_bracketed_in_among);
+ q = read_C_list(a); break;
+ default:
+ error(a, e_unexpected_token_in_among);
+ previous_token = token;
+ continue;
+ case c_ket:
+ if (p->number == 0) error(a, e_empty_among);
+ if (t->error_count == 0) make_among(a, p, substring);
+ return p;
+ }
+ previous_token = token;
+ if (p_end == 0) p->left = q; else p_end->right = q;
+ p_end = q;
+ }
+}
+
+static struct node * read_substring(struct analyser * a) {
+
+ struct node * p = new_node(a, c_substring);
+ if (a->substring != 0) error2(a, e_substring_preceded_by_substring, a->substring->line_number);
+ a->substring = p;
+ return p;
+}
+
+static void check_modifyable(struct analyser * a) {
+ if (!a->modifyable) error(a, e_not_allowed_inside_reverse);
+}
+
+static struct node * read_C(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ int token = read_token(t);
+ switch (token) {
+ case c_bra:
+ return read_C_list(a);
+ case c_backwards:
+ {
+ int mode = a->mode;
+ if (a->mode == m_backward) error(a, e_already_backwards); else a->mode = m_backward;
+ { struct node * p = C_style(a, "C", token);
+ a->mode = mode;
+ return p;
+ }
+ }
+ case c_reverse:
+ {
+ int mode = a->mode;
+ int modifyable = a->modifyable;
+ a->modifyable = false;
+ a->mode = mode == m_forward ? m_backward : m_forward;
+ {
+ struct node * p = C_style(a, "C", token);
+ a->mode = mode;
+ a->modifyable = modifyable;
+ return p;
+ }
+ }
+ case c_not:
+ case c_try:
+ case c_fail:
+ case c_test:
+ case c_do:
+ case c_goto:
+ case c_gopast:
+ case c_repeat:
+ return C_style(a, "C", token);
+ case c_loop:
+ case c_atleast:
+ return C_style(a, "AC", token);
+ case c_setmark: {
+ struct node * n = C_style(a, "i", token);
+ if (n->name) n->name->initialised = true;
+ return n;
+ }
+ case c_tomark:
+ case c_atmark:
+ case c_hop:
+ return C_style(a, "A", token);
+ case c_delete:
+ check_modifyable(a);
+ /* fall through */
+ case c_next:
+ case c_tolimit:
+ case c_atlimit:
+ case c_leftslice:
+ case c_rightslice:
+ case c_true:
+ case c_false:
+ case c_debug:
+ return new_node(a, token);
+ case c_assignto:
+ case c_sliceto: {
+ struct node *n;
+ check_modifyable(a);
+ n = C_style(a, "s", token);
+ if (n->name) n->name->initialised = true;
+ return n;
+ }
+ case c_assign:
+ case c_insert:
+ case c_attach:
+ case c_slicefrom: {
+ struct node *n;
+ check_modifyable(a);
+ n = C_style(a, "S", token);
+ if (n->name) n->name->value_used = true;
+ return n;
+ }
+ case c_setlimit:
+ return C_style(a, "CfD", token);
+ case c_set:
+ case c_unset: {
+ struct node * n = C_style(a, "b", token);
+ if (n->name) n->name->initialised = true;
+ return n;
+ }
+ case c_dollar: {
+ struct tokeniser * t = a->tokeniser;
+ read_token(t);
+ if (t->token == c_bra) {
+ /* Handle newer $(AE REL_OP AE) syntax. */
+ struct node * n = read_AE(a, 0);
+ read_token(t);
+ switch (t->token) {
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le: {
+ struct node * lhs = n;
+ n = new_node(a, t->token);
+ n->left = lhs;
+ n->AE = read_AE(a, 0);
+ get_token(a, c_ket);
+ break;
+ }
+ default:
+ error(a, e_unexpected_token);
+ t->token_held = true;
+ break;
+ }
+ return n;
+ }
+
+ if (t->token == c_name) {
+ struct node * p;
+ struct name * q = find_name(a);
+ int mode = a->mode;
+ int modifyable = a->modifyable;
+ if (q && q->type == t_string) {
+ /* Assume for now that $ on string both initialises and
+ * uses the string variable. FIXME: Can we do better?
+ */
+ q->initialised = true;
+ q->value_used = true;
+ a->mode = m_forward;
+ a->modifyable = true;
+ p = new_node(a, c_dollar);
+ p->left = read_C(a);
+ p->name = q;
+ } else {
+ if (q && q->type != t_integer) {
+ /* If $ is used on an unknown name or a name which
+ * isn't a string or an integer then we assume the
+ * unknown name is an integer as $ is used more often
+ * on integers than strings, so hopefully this it less
+ * likely to cause an error avalanche.
+ *
+ * For an unknown name, we'll already have reported an
+ * error.
+ */
+ error(a, e_not_of_type_string_or_integer);
+ q = NULL;
+ }
+ p = new_node(a, read_AE_test(a));
+ p->AE = read_AE(a, 0);
+
+ if (q) {
+ switch (p->type) {
+ case c_mathassign:
+ q->initialised = true;
+ p->name = q;
+ break;
+ default:
+ /* +=, etc don't "initialise" as they only
+ * amend an existing value. Similarly, they
+ * don't count as using the value.
+ */
+ p->name = q;
+ break;
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ p->left = new_node(a, c_name);
+ p->left->name = q;
+ q->value_used = true;
+ break;
+ }
+ }
+ }
+ if (q) mark_used_in(a, q, p);
+ a->mode = mode;
+ a->modifyable = modifyable;
+ return p;
+ }
+
+ error(a, e_unexpected_token);
+ t->token_held = true;
+ return new_node(a, c_dollar);
+ }
+ case c_name:
+ {
+ struct name * q = find_name(a);
+ struct node * p = new_node(a, c_name);
+ if (q) {
+ mark_used_in(a, q, p);
+ switch (q->type) {
+ case t_boolean:
+ p->type = c_booltest;
+ q->value_used = true;
+ break;
+ case t_integer:
+ error(a, e_misplaced); /* integer name misplaced */
+ break;
+ case t_string:
+ q->value_used = true;
+ break;
+ case t_routine:
+ case t_external:
+ p->type = c_call;
+ check_routine_mode(a, q, a->mode);
+ break;
+ case t_grouping:
+ p->type = c_grouping; break;
+ }
+ }
+ p->name = q;
+ return p;
+ }
+ case c_non:
+ {
+ struct node * p = new_node(a, token);
+ read_token(t);
+ if (t->token == c_minus) read_token(t);
+ if (!check_token(a, c_name)) { omission_error(a, c_name); return p; }
+ name_to_node(a, p, 'g');
+ return p;
+ }
+ case c_literalstring:
+ return read_literalstring(a);
+ case c_among: return read_among(a);
+ case c_substring: return read_substring(a);
+ default: error(a, e_unexpected_token); return 0;
+ }
+}
+
+static int next_symbol(symbol * p, symbol * W, int utf8) {
+ if (utf8) {
+ int ch;
+ int j = get_utf8(p, & ch);
+ W[0] = ch; return j;
+ } else {
+ W[0] = p[0]; return 1;
+ }
+}
+
+static symbol * alter_grouping(symbol * p, symbol * q, int style, int utf8) {
+ int j = 0;
+ symbol W[1];
+ int width;
+ if (style == c_plus) {
+ while (j < SIZE(q)) {
+ width = next_symbol(q + j, W, utf8);
+ p = add_to_b(p, 1, W);
+ j += width;
+ }
+ } else {
+ while (j < SIZE(q)) {
+ int i;
+ width = next_symbol(q + j, W, utf8);
+ for (i = 0; i < SIZE(p); i++) {
+ if (p[i] == W[0]) {
+ memmove(p + i, p + i + 1, (SIZE(p) - i - 1) * sizeof(symbol));
+ SIZE(p)--;
+ }
+ }
+ j += width;
+ }
+ }
+ return p;
+}
+
+static void read_define_grouping(struct analyser * a, struct name * q) {
+ struct tokeniser * t = a->tokeniser;
+ int style = c_plus;
+ {
+ NEW(grouping, p);
+ if (a->groupings == 0) a->groupings = p; else a->groupings_end->next = p;
+ a->groupings_end = p;
+ if (q) q->grouping = p;
+ p->next = 0;
+ p->name = q;
+ p->line_number = a->tokeniser->line_number;
+ p->b = create_b(0);
+ while (true) {
+ switch (read_token(t)) {
+ case c_name:
+ {
+ struct name * r = find_name(a);
+ if (r) {
+ check_name_type(a, r, 'g');
+ p->b = alter_grouping(p->b, r->grouping->b, style, false);
+ r->used_in_definition = true;
+ }
+ }
+ break;
+ case c_literalstring:
+ p->b = alter_grouping(p->b, t->b, style, (a->encoding == ENC_UTF8));
+ break;
+ default: error(a, e_unexpected_token); return;
+ }
+ switch (read_token(t)) {
+ case c_plus:
+ case c_minus: style = t->token; break;
+ default: goto label0;
+ }
+ }
+ label0:
+ {
+ int i;
+ int max = 0;
+ int min = 1<<16;
+ for (i = 0; i < SIZE(p->b); i++) {
+ if (p->b[i] > max) max = p->b[i];
+ if (p->b[i] < min) min = p->b[i];
+ }
+ p->largest_ch = max;
+ p->smallest_ch = min;
+ if (min == 1<<16) error(a, e_empty_grouping);
+ }
+ t->token_held = true; return;
+ }
+}
+
+static void read_define_routine(struct analyser * a, struct name * q) {
+ struct node * p = new_node(a, c_define);
+ a->amongvar_needed = false;
+ if (q) {
+ check_name_type(a, q, 'R');
+ if (q->definition != 0) error(a, e_redefined);
+ if (q->mode < 0) q->mode = a->mode; else
+ if (q->mode != a->mode) error2(a, e_declared_as_different_mode, q->mode);
+ }
+ p->name = q;
+ if (a->program == 0) a->program = p; else a->program_end->right = p;
+ a->program_end = p;
+ get_token(a, c_as);
+ p->left = read_C(a);
+ if (q) q->definition = p->left;
+
+ if (a->substring != 0) {
+ error2(a, e_unresolved_substring, a->substring->line_number);
+ a->substring = 0;
+ }
+ p->amongvar_needed = a->amongvar_needed;
+}
+
+static void read_define(struct analyser * a) {
+ if (get_token(a, c_name)) {
+ struct name * q = find_name(a);
+ int type;
+ if (q) {
+ type = q->type;
+ } else {
+ /* No declaration, so sniff next token - if it is 'as' then parse
+ * as a routine, otherwise as a grouping.
+ */
+ if (read_token(a->tokeniser) == c_as) {
+ type = t_routine;
+ } else {
+ type = t_grouping;
+ }
+ a->tokeniser->token_held = true;
+ }
+
+ if (type == t_grouping) {
+ read_define_grouping(a, q);
+ } else {
+ read_define_routine(a, q);
+ }
+ }
+}
+
+static void read_backwardmode(struct analyser * a) {
+ int mode = a->mode;
+ a->mode = m_backward;
+ if (get_token(a, c_bra)) {
+ read_program_(a, c_ket);
+ check_token(a, c_ket);
+ }
+ a->mode = mode;
+}
+
+static void read_program_(struct analyser * a, int terminator) {
+ struct tokeniser * t = a->tokeniser;
+ while (true) {
+ switch (read_token(t)) {
+ case c_strings: read_names(a, t_string); break;
+ case c_booleans: read_names(a, t_boolean); break;
+ case c_integers: read_names(a, t_integer); break;
+ case c_routines: read_names(a, t_routine); break;
+ case c_externals: read_names(a, t_external); break;
+ case c_groupings: read_names(a, t_grouping); break;
+ case c_define: read_define(a); break;
+ case c_backwardmode:read_backwardmode(a); break;
+ case c_ket:
+ if (terminator == c_ket) return;
+ /* fall through */
+ default:
+ error(a, e_unexpected_token); break;
+ case -1:
+ if (terminator >= 0) omission_error(a, c_ket);
+ return;
+ }
+ }
+}
+
+static void remove_dead_assignments(struct node * p, struct name * q) {
+ if (p->name == q) {
+ switch (p->type) {
+ case c_assignto:
+ case c_sliceto:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_setmark:
+ case c_set:
+ case c_unset:
+ case c_dollar:
+ /* c_true is a no-op. */
+ p->type = c_true;
+ break;
+ default:
+ /* There are no read accesses to this variable, so any
+ * references must be assignments.
+ */
+ fprintf(stderr, "Unhandled type of dead assignment via %s\n",
+ name_of_token(p->type));
+ exit(1);
+ }
+ }
+ if (p->AE) remove_dead_assignments(p->AE, q);
+ if (p->left) remove_dead_assignments(p->left, q);
+ if (p->aux) remove_dead_assignments(p->aux, q);
+ if (p->right) remove_dead_assignments(p->right, q);
+}
+
+extern void read_program(struct analyser * a) {
+ read_program_(a, -1);
+ {
+ struct name * q = a->names;
+ while (q) {
+ switch (q->type) {
+ case t_external: case t_routine:
+ if (q->used && q->definition == 0) error4(a, q);
+ break;
+ case t_grouping:
+ if (q->used && q->grouping == 0) error4(a, q);
+ break;
+ }
+ q = q->next;
+ }
+ }
+
+ if (a->tokeniser->error_count == 0) {
+ struct name * q = a->names;
+ struct name ** ptr = &(a->names);
+ while (q) {
+ if (!q->referenced) {
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ q->declaration_line_number,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ if (q->type == t_routine ||
+ q->type == t_external ||
+ q->type == t_grouping) {
+ fprintf(stderr, "' declared but not defined\n");
+ } else {
+ fprintf(stderr, "' defined but not used\n");
+ q = q->next;
+ *ptr = q;
+ continue;
+ }
+ } else if (q->type == t_routine || q->type == t_grouping) {
+ /* It's OK to define a grouping but only use it to define other
+ * groupings.
+ */
+ if (!q->used && !q->used_in_definition) {
+ int line_num;
+ if (q->type == t_routine) {
+ line_num = q->definition->line_number;
+ } else {
+ line_num = q->grouping->line_number;
+ }
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ line_num,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ fprintf(stderr, "' defined but not used\n");
+ }
+ } else if (q->type == t_external) {
+ /* Unused is OK. */
+ } else if (!q->initialised) {
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ q->declaration_line_number,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ fprintf(stderr, "' is never initialised\n");
+ } else if (!q->value_used) {
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ q->declaration_line_number,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ fprintf(stderr, "' is set but never used\n");
+ remove_dead_assignments(a->program, q);
+ q = q->next;
+ *ptr = q;
+ continue;
+ }
+ ptr = &(q->next);
+ q = q->next;
+ }
+
+ {
+ /* Now we've eliminated variables whose values are never used we
+ * can number the variables, which is used by some generators.
+ */
+ int * name_count = a->name_count;
+ struct name * n;
+ for (n = a->names; n; n = n->next) {
+ n->count = name_count[n->type]++;
+ }
+ }
+ }
+}
+
+extern struct analyser * create_analyser(struct tokeniser * t) {
+ NEW(analyser, a);
+ a->tokeniser = t;
+ a->nodes = 0;
+ a->names = 0;
+ a->literalstrings = 0;
+ a->program = 0;
+ a->amongs = 0;
+ a->among_count = 0;
+ a->groupings = 0;
+ a->mode = m_forward;
+ a->modifyable = true;
+ { int i; for (i = 0; i < t_size; i++) a->name_count[i] = 0; }
+ a->substring = 0;
+ a->int_limits_used = false;
+ return a;
+}
+
+extern void close_analyser(struct analyser * a) {
+ {
+ struct node * q = a->nodes;
+ while (q) {
+ struct node * q_next = q->next;
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct name * q = a->names;
+ while (q) {
+ struct name * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct literalstring * q = a->literalstrings;
+ while (q) {
+ struct literalstring * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct among * q = a->amongs;
+ while (q) {
+ struct among * q_next = q->next;
+ FREE(q->b);
+ FREE(q->commands);
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct grouping * q = a->groupings;
+ while (q) {
+ struct grouping * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ FREE(a);
+}
diff --git a/contrib/snowball/compiler/driver.c b/contrib/snowball/compiler/driver.c
new file mode 100644
index 0000000..587028f
--- /dev/null
+++ b/contrib/snowball/compiler/driver.c
@@ -0,0 +1,574 @@
+#include <ctype.h> /* for toupper etc */
+#include <stdio.h> /* for fprintf etc */
+#include <stdlib.h> /* for free etc */
+#include <string.h> /* for strcmp */
+#include "header.h"
+
+#define DEFAULT_JAVA_PACKAGE "org.tartarus.snowball.ext"
+#define DEFAULT_JAVA_BASE_CLASS "org.tartarus.snowball.SnowballProgram"
+#define DEFAULT_JAVA_AMONG_CLASS "org.tartarus.snowball.Among"
+#define DEFAULT_JAVA_STRING_CLASS "java.lang.StringBuilder"
+
+#define DEFAULT_GO_PACKAGE "snowball"
+#define DEFAULT_GO_SNOWBALL_RUNTIME "github.com/snowballstem/snowball/go"
+
+#define DEFAULT_CS_NAMESPACE "Snowball"
+#define DEFAULT_CS_BASE_CLASS "Stemmer"
+#define DEFAULT_CS_AMONG_CLASS "Among"
+#define DEFAULT_CS_STRING_CLASS "StringBuilder"
+
+#define DEFAULT_JS_BASE_CLASS "BaseStemmer"
+
+#define DEFAULT_PYTHON_BASE_CLASS "BaseStemmer"
+
+static int eq(const char * s1, const char * s2) {
+ return strcmp(s1, s2) == 0;
+}
+
+__attribute__((noreturn))
+static void print_arglist(int exit_code) {
+ FILE * f = exit_code ? stderr : stdout;
+ fprintf(f, "Usage: snowball SOURCE_FILE... [OPTIONS]\n\n"
+ "Supported options:\n"
+ " -o[utput] file\n"
+ " -s[yntax]\n"
+ " -comments\n"
+#ifndef DISABLE_JAVA
+ " -j[ava]\n"
+#endif
+#ifndef DISABLE_CSHARP
+ " -cs[harp]\n"
+#endif
+ " -c++\n"
+#ifndef DISABLE_PASCAL
+ " -pascal\n"
+#endif
+#ifndef DISABLE_PYTHON
+ " -py[thon]\n"
+#endif
+#ifndef DISABLE_JS
+ " -js\n"
+#endif
+#ifndef DISABLE_RUST
+ " -rust\n"
+#endif
+#ifndef DISABLE_GO
+ " -go\n"
+#endif
+ " -w[idechars]\n"
+ " -u[tf8]\n"
+ " -n[ame] class name\n"
+ " -ep[refix] string\n"
+ " -vp[refix] string\n"
+ " -i[nclude] directory\n"
+ " -r[untime] path to runtime headers\n"
+ " -p[arentclassname] fully qualified parent class name\n"
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
+ " -P[ackage] package name for stemmers\n"
+ " -S[tringclass] StringBuffer-compatible class\n"
+ " -a[mongclass] fully qualified name of the Among class\n"
+#endif
+#ifndef DISABLE_GO
+ " -gop[ackage] Go package name for stemmers\n"
+ " -gor[untime] Go snowball runtime package\n"
+#endif
+ " --help display this help and exit\n"
+ " --version output version information and exit\n"
+ );
+ exit(exit_code);
+}
+
+static void check_lim(int i, int argc) {
+ if (i >= argc) {
+ fprintf(stderr, "argument list is one short\n");
+ print_arglist(1);
+ }
+}
+
+static FILE * get_output(symbol * b) {
+ char * s = b_to_s(b);
+ FILE * output = fopen(s, "w");
+ if (output == 0) {
+ fprintf(stderr, "Can't open output %s\n", s);
+ exit(1);
+ }
+ free(s);
+ return output;
+}
+
+static int read_options(struct options * o, int argc, char * argv[]) {
+ char * s;
+ int i = 1;
+ int new_argc = 1;
+ /* Note down the last option used to specify an explicit encoding so
+ * we can warn we ignored it for languages with a fixed encoding.
+ */
+ const char * encoding_opt = NULL;
+
+ /* set defaults: */
+
+ o->output_file = 0;
+ o->syntax_tree = false;
+ o->comments = false;
+ o->externals_prefix = NULL;
+ o->variables_prefix = 0;
+ o->runtime_path = 0;
+ o->parent_class_name = NULL;
+ o->string_class = NULL;
+ o->among_class = NULL;
+ o->package = NULL;
+ o->go_snowball_runtime = DEFAULT_GO_SNOWBALL_RUNTIME;
+ o->name = NULL;
+ o->make_lang = LANG_C;
+ o->includes = 0;
+ o->includes_end = 0;
+ o->encoding = ENC_SINGLEBYTE;
+
+ /* read options: */
+
+ while (i < argc) {
+ s = argv[i++];
+ if (s[0] != '-') {
+ /* Non-option argument - shuffle down. */
+ argv[new_argc++] = s;
+ continue;
+ }
+
+ {
+ if (eq(s, "-o") || eq(s, "-output")) {
+ check_lim(i, argc);
+ o->output_file = argv[i++];
+ continue;
+ }
+ if (eq(s, "-n") || eq(s, "-name")) {
+ check_lim(i, argc);
+ o->name = argv[i++];
+ continue;
+ }
+#ifndef DISABLE_JS
+ if (eq(s, "-js")) {
+ o->make_lang = LANG_JAVASCRIPT;
+ continue;
+ }
+#endif
+#ifndef DISABLE_RUST
+ if (eq(s, "-rust")) {
+ o->make_lang = LANG_RUST;
+ continue;
+ }
+#endif
+#ifndef DISABLE_GO
+ if (eq(s, "-go")) {
+ o->make_lang = LANG_GO;
+ continue;
+ }
+#endif
+#ifndef DISABLE_JAVA
+ if (eq(s, "-j") || eq(s, "-java")) {
+ o->make_lang = LANG_JAVA;
+ continue;
+ }
+#endif
+#ifndef DISABLE_CSHARP
+ if (eq(s, "-cs") || eq(s, "-csharp")) {
+ o->make_lang = LANG_CSHARP;
+ continue;
+ }
+#endif
+ if (eq(s, "-c++")) {
+ o->make_lang = LANG_CPLUSPLUS;
+ continue;
+ }
+#ifndef DISABLE_PASCAL
+ if (eq(s, "-pascal")) {
+ o->make_lang = LANG_PASCAL;
+ continue;
+ }
+#endif
+#ifndef DISABLE_PYTHON
+ if (eq(s, "-py") || eq(s, "-python")) {
+ o->make_lang = LANG_PYTHON;
+ continue;
+ }
+#endif
+ if (eq(s, "-w") || eq(s, "-widechars")) {
+ encoding_opt = s;
+ o->encoding = ENC_WIDECHARS;
+ continue;
+ }
+ if (eq(s, "-s") || eq(s, "-syntax")) {
+ o->syntax_tree = true;
+ continue;
+ }
+ if (eq(s, "-comments")) {
+ o->comments = true;
+ continue;
+ }
+ if (eq(s, "-ep") || eq(s, "-eprefix")) {
+ check_lim(i, argc);
+ o->externals_prefix = argv[i++];
+ continue;
+ }
+ if (eq(s, "-vp") || eq(s, "-vprefix")) {
+ check_lim(i, argc);
+ o->variables_prefix = argv[i++];
+ continue;
+ }
+ if (eq(s, "-i") || eq(s, "-include")) {
+ check_lim(i, argc);
+
+ {
+ NEW(include, p);
+ symbol * b = add_s_to_b(0, argv[i++]);
+ b = add_s_to_b(b, "/");
+ p->next = 0; p->b = b;
+
+ if (o->includes == 0) o->includes = p; else
+ o->includes_end->next = p;
+ o->includes_end = p;
+ }
+ continue;
+ }
+ if (eq(s, "-r") || eq(s, "-runtime")) {
+ check_lim(i, argc);
+ o->runtime_path = argv[i++];
+ continue;
+ }
+ if (eq(s, "-u") || eq(s, "-utf8")) {
+ encoding_opt = s;
+ o->encoding = ENC_UTF8;
+ continue;
+ }
+ if (eq(s, "-p") || eq(s, "-parentclassname")) {
+ check_lim(i, argc);
+ o->parent_class_name = argv[i++];
+ continue;
+ }
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
+ if (eq(s, "-P") || eq(s, "-Package")) {
+ check_lim(i, argc);
+ o->package = argv[i++];
+ continue;
+ }
+ if (eq(s, "-S") || eq(s, "-stringclass")) {
+ check_lim(i, argc);
+ o->string_class = argv[i++];
+ continue;
+ }
+ if (eq(s, "-a") || eq(s, "-amongclass")) {
+ check_lim(i, argc);
+ o->among_class = argv[i++];
+ continue;
+ }
+#endif
+#ifndef DISABLE_GO
+ if (eq(s, "-gop") || eq(s, "-gopackage")) {
+ check_lim(i, argc);
+ o->package = argv[i++];
+ continue;
+ }
+ if (eq(s, "-gor") || eq(s, "-goruntime")) {
+ check_lim(i, argc);
+ o->go_snowball_runtime = argv[i++];
+ continue;
+ }
+#endif
+ if (eq(s, "--help")) {
+ print_arglist(0);
+ }
+
+ if (eq(s, "--version")) {
+ printf("Snowball compiler version " SNOWBALL_VERSION "\n");
+ exit(0);
+ }
+
+ fprintf(stderr, "'%s' misplaced\n", s);
+ print_arglist(1);
+ }
+ }
+ if (new_argc == 1) {
+ fprintf(stderr, "no source files specified\n");
+ print_arglist(1);
+ }
+ argv[new_argc] = NULL;
+
+ /* Set language-dependent defaults. */
+ switch (o->make_lang) {
+ case LANG_C:
+ case LANG_CPLUSPLUS:
+ encoding_opt = NULL;
+ break;
+ case LANG_CSHARP:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_CS_BASE_CLASS;
+ if (!o->string_class)
+ o->string_class = DEFAULT_CS_STRING_CLASS;
+ if (!o->among_class)
+ o->among_class = DEFAULT_CS_AMONG_CLASS;
+ if (!o->package)
+ o->package = DEFAULT_CS_NAMESPACE;
+ break;
+ case LANG_GO:
+ o->encoding = ENC_UTF8;
+ if (!o->package)
+ o->package = DEFAULT_GO_PACKAGE;
+ break;
+ case LANG_JAVA:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_JAVA_BASE_CLASS;
+ if (!o->string_class)
+ o->string_class = DEFAULT_JAVA_STRING_CLASS;
+ if (!o->among_class)
+ o->among_class = DEFAULT_JAVA_AMONG_CLASS;
+ if (!o->package)
+ o->package = DEFAULT_JAVA_PACKAGE;
+ break;
+ case LANG_JAVASCRIPT:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_JS_BASE_CLASS;
+ break;
+ case LANG_PYTHON:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_PYTHON_BASE_CLASS;
+ break;
+ case LANG_RUST:
+ o->encoding = ENC_UTF8;
+ break;
+ default:
+ break;
+ }
+
+ if (encoding_opt) {
+ fprintf(stderr, "warning: %s only meaningful for C and C++\n",
+ encoding_opt);
+ }
+
+ if (o->make_lang != LANG_C && o->make_lang != LANG_CPLUSPLUS) {
+ if (o->runtime_path) {
+ fprintf(stderr, "warning: -r/-runtime only meaningful for C and C++\n");
+ }
+ if (o->externals_prefix) {
+ fprintf(stderr, "warning: -ep/-eprefix only meaningful for C and C++\n");
+ }
+ }
+ if (!o->externals_prefix) o->externals_prefix = "";
+
+ if (!o->name && o->output_file) {
+ /* Default class name to basename of output_file - this is the standard
+ * convention for at least Java and C#.
+ */
+ const char * slash = strrchr(o->output_file, '/');
+ size_t len;
+ const char * leaf = (slash == NULL) ? o->output_file : slash + 1;
+
+ slash = strrchr(leaf, '\\');
+ if (slash != NULL) leaf = slash + 1;
+
+ {
+ const char * dot = strchr(leaf, '.');
+ len = (dot == NULL) ? strlen(leaf) : (size_t)(dot - leaf);
+ }
+
+ {
+ char * new_name = malloc(len + 1);
+ switch (o->make_lang) {
+ case LANG_CSHARP:
+ case LANG_PASCAL:
+ /* Upper case initial letter. */
+ memcpy(new_name, leaf, len);
+ new_name[0] = toupper(new_name[0]);
+ break;
+ case LANG_JAVASCRIPT:
+ case LANG_PYTHON: {
+ /* Upper case initial letter and change each
+ * underscore+letter or hyphen+letter to an upper case
+ * letter.
+ */
+ size_t i, j = 0;
+ int uc_next = true;
+ for (i = 0; i != len; ++i) {
+ unsigned char ch = leaf[i];
+ if (ch == '_' || ch == '-') {
+ uc_next = true;
+ } else {
+ if (uc_next) {
+ new_name[j] = toupper(ch);
+ uc_next = false;
+ } else {
+ new_name[j] = ch;
+ }
+ ++j;
+ }
+ }
+ len = j;
+ break;
+ }
+ default:
+ /* Just copy. */
+ memcpy(new_name, leaf, len);
+ break;
+ }
+ new_name[len] = '\0';
+ o->name = new_name;
+ }
+ }
+
+ return new_argc;
+}
+
+extern int main(int argc, char * argv[]) {
+
+ int i;
+ NEW(options, o);
+ argc = read_options(o, argc, argv);
+ {
+ char * file = argv[1];
+ symbol * u = get_input(file);
+ if (u == 0) {
+ fprintf(stderr, "Can't open input %s\n", file);
+ exit(1);
+ }
+ {
+ struct tokeniser * t = create_tokeniser(u, file);
+ struct analyser * a = create_analyser(t);
+ struct input ** next_input_ptr = &(t->next);
+ a->encoding = t->encoding = o->encoding;
+ t->includes = o->includes;
+ /* If multiple source files are specified, set up the others to be
+ * read after the first in order, using the same mechanism as
+ * 'get' uses. */
+ for (i = 2; i != argc; ++i) {
+ NEW(input, q);
+ file = argv[i];
+ u = get_input(file);
+ if (u == 0) {
+ fprintf(stderr, "Can't open input %s\n", file);
+ exit(1);
+ }
+ q->p = u;
+ q->c = 0;
+ q->file = file;
+ q->file_needs_freeing = false;
+ q->line_number = 1;
+ *next_input_ptr = q;
+ next_input_ptr = &(q->next);
+ }
+ *next_input_ptr = NULL;
+ read_program(a);
+ if (t->error_count > 0) exit(1);
+ if (o->syntax_tree) print_program(a);
+ close_tokeniser(t);
+ if (!o->syntax_tree) {
+ struct generator * g;
+
+ const char * s = o->output_file;
+ if (!s) {
+ fprintf(stderr, "Please include the -o option\n");
+ print_arglist(1);
+ }
+ g = create_generator(a, o);
+ if (o->make_lang == LANG_C || o->make_lang == LANG_CPLUSPLUS) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".h");
+ o->output_h = get_output(b);
+ b[SIZE(b) - 1] = 'c';
+ if (o->make_lang == LANG_CPLUSPLUS) {
+ b = add_s_to_b(b, "c");
+ }
+ o->output_src = get_output(b);
+ lose_b(b);
+
+ generate_program_c(g);
+ fclose(o->output_src);
+ fclose(o->output_h);
+ }
+#ifndef DISABLE_JAVA
+ if (o->make_lang == LANG_JAVA) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".java");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_java(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_PASCAL
+ if (o->make_lang == LANG_PASCAL) {
+ symbol *b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".pas");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_pascal(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_PYTHON
+ if (o->make_lang == LANG_PYTHON) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".py");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_python(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_JS
+ if (o->make_lang == LANG_JAVASCRIPT) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".js");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_js(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_CSHARP
+ if (o->make_lang == LANG_CSHARP) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".cs");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_csharp(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_RUST
+ if (o->make_lang == LANG_RUST) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".rs");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_rust(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_GO
+ if (o->make_lang == LANG_GO) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".go");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_go(g);
+ fclose(o->output_src);
+ }
+#endif
+ close_generator(g);
+ }
+ close_analyser(a);
+ }
+ lose_b(u);
+ }
+ { struct include * p = o->includes;
+ while (p) {
+ struct include * q = p->next;
+ lose_b(p->b); FREE(p); p = q;
+ }
+ }
+ FREE(o);
+ if (space_count) fprintf(stderr, "%d blocks unfreed\n", space_count);
+ return 0;
+}
diff --git a/contrib/snowball/compiler/generator.c b/contrib/snowball/compiler/generator.c
new file mode 100644
index 0000000..eed86c1
--- /dev/null
+++ b/contrib/snowball/compiler/generator.c
@@ -0,0 +1,1725 @@
+
+#include <limits.h> /* for INT_MAX */
+#include <stdio.h> /* for fprintf etc */
+#include <stdlib.h> /* for free etc */
+#include <string.h> /* for strlen */
+#include "header.h"
+
+/* Define this to get warning messages when optimisations can't be used. */
+/* #define OPTIMISATION_WARNINGS */
+
+/* recursive use: */
+
+static void generate(struct generator * g, struct node * p);
+
+static int new_label(struct generator * g) {
+ return g->next_label++;
+}
+
+/* Write routines for simple entities */
+
+/* Write a space if the preceding character was not whitespace */
+static void ws_opt_space(struct generator * g, const char * s) {
+ int ch = str_back(g->outbuf);
+ if (ch != ' ' && ch != '\n' && ch != '\t' && ch != -1)
+ write_char(g, ' ');
+ write_string(g, s);
+}
+
+static void wi3(struct generator * g, int i) {
+ if (i < 100) write_char(g, ' ');
+ if (i < 10) write_char(g, ' ');
+ write_int(g, i); /* integer (width 3) */
+}
+
+
+/* Write routines for items from the syntax tree */
+
+static void write_varname(struct generator * g, struct name * p) {
+
+ int ch = "SIIrxg"[p->type];
+ switch (p->type) {
+ case t_external:
+ write_string(g, g->options->externals_prefix); break;
+ case t_string:
+ case t_boolean:
+ case t_integer: {
+ int count = p->count;
+ if (count < 0) {
+ fprintf(stderr, "Reference to optimised out variable ");
+ report_b(stderr, p->b);
+ fprintf(stderr, " attempted\n");
+ exit(1);
+ }
+ if (p->type == t_boolean) {
+ /* We use a single array for booleans and integers, with the
+ * integers first.
+ */
+ count += g->analyser->name_count[t_integer];
+ }
+ write_char(g, ch);
+ write_char(g, '[');
+ write_int(g, count);
+ write_char(g, ']');
+ return;
+ }
+ default:
+ write_char(g, ch); write_char(g, '_');
+ }
+ write_b(g, p->b);
+}
+
+static void write_varref(struct generator * g, struct name * p) { /* reference to variable */
+ if (p->type < t_routine) write_string(g, "z->");
+ write_varname(g, p);
+}
+
+static void write_hexdigit(struct generator * g, int i) {
+ str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */
+}
+
+static void write_hex(struct generator * g, int i) {
+ if (i >> 4) write_hex(g, i >> 4);
+ write_hexdigit(g, i); /* hex integer */
+}
+
+/* write character literal */
+static void wlitch(struct generator * g, int ch) {
+ if (32 <= ch && ch < 127) {
+ write_char(g, '\'');
+ if (ch == '\'' || ch == '\\') {
+ write_char(g, '\\');
+ }
+ write_char(g, ch);
+ write_char(g, '\'');
+ } else {
+ write_string(g, "0x"); write_hex(g, ch);
+ }
+}
+
+static void wlitarray(struct generator * g, symbol * p) { /* write literal array */
+
+ write_string(g, "{ ");
+ {
+ int i;
+ for (i = 0; i < SIZE(p); i++) {
+ wlitch(g, p[i]);
+ if (i < SIZE(p) - 1) write_string(g, ", ");
+ }
+ }
+ write_string(g, " }");
+}
+
+static void wlitref(struct generator * g, symbol * p) { /* write ref to literal array */
+
+ if (SIZE(p) == 0) {
+ write_char(g, '0');
+ } else {
+ struct str * s = g->outbuf;
+ g->outbuf = g->declarations;
+ write_string(g, "static const symbol s_"); write_int(g, g->literalstring_count); write_string(g, "[] = ");
+ wlitarray(g, p);
+ write_string(g, ";\n");
+ g->outbuf = s;
+ write_string(g, "s_"); write_int(g, g->literalstring_count);
+ g->literalstring_count++;
+ }
+}
+
+static void write_margin(struct generator * g) {
+ int i;
+ for (i = 0; i < g->margin; i++) write_string(g, " ");
+}
+
+void write_comment_content(struct generator * g, struct node * p) {
+ switch (p->type) {
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ if (p->name) {
+ write_char(g, '$');
+ write_b(g, p->name->b);
+ write_char(g, ' ');
+ }
+ write_string(g, name_of_token(p->type));
+ write_string(g, " <integer expression>");
+ break;
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ write_string(g, "$(<integer expression> ");
+ write_string(g, name_of_token(p->type));
+ write_string(g, " <integer expression>)");
+ break;
+ default:
+ write_string(g, name_of_token(p->type));
+ if (p->name) {
+ write_char(g, ' ');
+ write_b(g, p->name->b);
+ }
+ }
+ write_string(g, ", line ");
+ write_int(g, p->line_number);
+}
+
+static void write_comment(struct generator * g, struct node * p) {
+ if (g->options->comments) {
+ ws_opt_space(g, "/* ");
+ write_comment_content(g, p);
+ write_string(g, " */");
+ }
+ write_newline(g);
+}
+
+static void wms(struct generator * g, const char * s) {
+ write_margin(g); write_string(g, s); } /* margin + string */
+
+static void write_block_start(struct generator * g) { /* block start */
+ wms(g, "{ ");
+ g->margin++;
+}
+
+static void write_block_end(struct generator * g) { /* block end */
+
+ if (g->line_labelled == g->line_count) { wms(g, ";"); write_newline(g); }
+ g->margin--;
+ wms(g, "}"); write_newline(g);
+}
+
+static void w(struct generator * g, const char * s);
+
+/* keep c */
+static void wk(struct generator * g, struct node * p, int keep_limit) {
+ ++g->keep_count;
+ if (p->mode == m_forward) {
+ write_string(g, "int c");
+ write_int(g, g->keep_count);
+ write_string(g, " = z->c");
+ if (keep_limit) {
+ write_string(g, ", mlimit");
+ write_int(g, g->keep_count);
+ }
+ write_char(g, ';');
+ } else {
+ write_string(g, "int m");
+ write_int(g, g->keep_count);
+ write_string(g, " = z->l - z->c");
+ if (keep_limit) {
+ write_string(g, ", mlimit");
+ write_int(g, g->keep_count);
+ }
+ write_string(g, "; (void)m");
+ write_int(g, g->keep_count);
+ write_char(g, ';');
+ }
+}
+
+static void wrestore(struct generator * g, struct node * p, int keep_token) { /* restore c */
+ if (p->mode == m_forward) {
+ write_string(g, "z->c = c");
+ } else {
+ write_string(g, "z->c = z->l - m");
+ }
+ write_int(g, keep_token); write_char(g, ';');
+}
+
+static void wrestorelimit(struct generator * g, struct node * p, int keep_token) { /* restore limit */
+ if (p->mode == m_forward) {
+ w(g, "z->l += mlimit");
+ } else {
+ w(g, "z->lb = mlimit");
+ }
+ write_int(g, keep_token); write_string(g, ";");
+}
+
+static void winc(struct generator * g, struct node * p) { /* increment c */
+ write_string(g, p->mode == m_forward ? "z->c++;" :
+ "z->c--;");
+}
+
+static void wsetl(struct generator * g, int n) {
+
+ g->margin--;
+ wms(g, "lab"); write_int(g, n); write_char(g, ':'); write_newline(g);
+ g->line_labelled = g->line_count;
+ g->margin++;
+}
+
+static void wgotol(struct generator * g, int n) {
+ wms(g, "goto lab"); write_int(g, n); write_char(g, ';'); write_newline(g);
+}
+
+static void write_failure(struct generator * g, struct node * p) { /* fail */
+ if (g->failure_keep_count != 0) {
+ write_string(g, "{ ");
+ if (g->failure_keep_count > 0) {
+ wrestore(g, p, g->failure_keep_count);
+ } else {
+ wrestorelimit(g, p, -g->failure_keep_count);
+ }
+ write_char(g, ' ');
+ }
+ switch (g->failure_label) {
+ case x_return:
+ write_string(g, "return 0;");
+ break;
+ default:
+ write_string(g, "goto lab");
+ write_int(g, g->failure_label);
+ write_char(g, ';');
+ g->label_used = 1;
+ }
+ if (g->failure_keep_count != 0) write_string(g, " }");
+}
+
+
+/* if at limit fail */
+static void write_check_limit(struct generator * g, struct node * p) {
+
+ write_string(g, p->mode == m_forward ? "if (z->c >= z->l) " :
+ "if (z->c <= z->lb) ");
+ write_failure(g, p);
+}
+
+static void write_data_address(struct generator * g, struct node * p) {
+ symbol * b = p->literalstring;
+ if (b != 0) {
+ write_int(g, SIZE(b)); w(g, ", ");
+ wlitref(g, b);
+ } else {
+ write_varref(g, p->name);
+ }
+}
+
+/* Formatted write. */
+static void writef(struct generator * g, const char * input, struct node * p) {
+ int i = 0;
+ int l = strlen(input);
+
+ while (i < l) {
+ int ch = input[i++];
+ if (ch != '~') {
+ write_char(g, ch);
+ continue;
+ }
+ switch (input[i++]) {
+ default: write_char(g, input[i - 1]); continue;
+ case 'C': write_comment(g, p); continue;
+ case 'k': wk(g, p, false); continue;
+ case 'K': wk(g, p, true); continue;
+ case 'i': winc(g, p); continue;
+ case 'l': write_check_limit(g, p); continue;
+ case 'f': write_failure(g, p); continue;
+ case 'M': write_margin(g); continue;
+ case 'N': write_newline(g); continue;
+ case '{': write_block_start(g); continue;
+ case '}': write_block_end(g); continue;
+ case 'S': write_string(g, g->S[input[i++] - '0']); continue;
+ case 'I': write_int(g, g->I[input[i++] - '0']); continue;
+ case 'J': wi3(g, g->I[input[i++] - '0']); continue;
+ case 'V': write_varref(g, g->V[input[i++] - '0']); continue;
+ case 'W': write_varname(g, g->V[input[i++] - '0']); continue;
+ case 'L': wlitref(g, g->L[input[i++] - '0']); continue;
+ case 'A': wlitarray(g, g->L[input[i++] - '0']); continue;
+ case 'c': wlitch(g, g->I[input[i++] - '0']); continue;
+ case 'a': write_data_address(g, p); continue;
+ case '+': g->margin++; continue;
+ case '-': g->margin--; continue;
+ case '$': /* insert_s, insert_v etc */
+ write_char(g, p->literalstring == 0 ? 'v' : 's');
+ continue;
+ case 'p': write_string(g, g->options->externals_prefix); continue;
+ }
+ }
+}
+
+static void w(struct generator * g, const char * s) {
+ writef(g, s, 0);
+}
+
+static void generate_AE(struct generator * g, struct node * p) {
+ const char * s;
+ switch (p->type) {
+ case c_name:
+ write_varref(g, p->name); break;
+ case c_number:
+ write_int(g, p->number); break;
+ case c_maxint:
+ write_string(g, "MAXINT"); break;
+ case c_minint:
+ write_string(g, "MININT"); break;
+ case c_neg:
+ write_char(g, '-'); generate_AE(g, p->right); break;
+ case c_multiply:
+ s = " * "; goto label0;
+ case c_plus:
+ s = " + "; goto label0;
+ case c_minus:
+ s = " - "; goto label0;
+ case c_divide:
+ s = " / ";
+ label0:
+ write_char(g, '('); generate_AE(g, p->left);
+ write_string(g, s); generate_AE(g, p->right); write_char(g, ')'); break;
+ case c_cursor:
+ w(g, "z->c"); break;
+ case c_limit:
+ w(g, p->mode == m_forward ? "z->l" : "z->lb"); break;
+ case c_len:
+ if (g->options->encoding == ENC_UTF8) {
+ w(g, "len_utf8(z->p)");
+ break;
+ }
+ /* FALLTHRU */
+ case c_size:
+ w(g, "SIZE(z->p)");
+ break;
+ case c_lenof:
+ if (g->options->encoding == ENC_UTF8) {
+ g->V[0] = p->name;
+ w(g, "len_utf8(~V0)");
+ break;
+ }
+ /* FALLTHRU */
+ case c_sizeof:
+ g->V[0] = p->name;
+ w(g, "SIZE(~V0)");
+ break;
+ }
+}
+
+/* K_needed() tests to see if we really need to keep c. Not true when the
+ command does not touch the cursor. This and repeat_score() could be
+ elaborated almost indefinitely.
+*/
+
+static int K_needed_(struct generator * g, struct node * p, int call_depth) {
+ while (p) {
+ switch (p->type) {
+ case c_atlimit:
+ case c_do:
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto:
+ case c_booltest:
+ case c_set:
+ case c_unset:
+ case c_true:
+ case c_false:
+ case c_debug:
+ break;
+
+ case c_call:
+ /* Recursive functions aren't typical in snowball programs, so
+ * make the pessimistic assumption that keep is needed if we
+ * hit a generous limit on recursion. It's not likely to make
+ * a difference to any real world program, but means we won't
+ * recurse until we run out of stack for pathological cases.
+ */
+ if (call_depth >= 100) return true;
+ if (K_needed_(g, p->name->definition, call_depth + 1))
+ return true;
+ break;
+
+ case c_bra:
+ if (K_needed_(g, p->left, call_depth)) return true;
+ break;
+
+ default: return true;
+ }
+ p = p->right;
+ }
+ return false;
+}
+
+extern int K_needed(struct generator * g, struct node * p) {
+ return K_needed_(g, p, 0);
+}
+
+static int repeat_score(struct generator * g, struct node * p, int call_depth) {
+ int score = 0;
+ while (p) {
+ switch (p->type) {
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto: /* case c_not: must not be included here! */
+ case c_debug:
+ break;
+
+ case c_call:
+ /* Recursive functions aren't typical in snowball programs, so
+ * make the pessimistic assumption that repeat requires cursor
+ * reinstatement if we hit a generous limit on recursion. It's
+ * not likely to make a difference to any real world program,
+ * but means we won't recurse until we run out of stack for
+ * pathological cases.
+ */
+ if (call_depth >= 100) {
+ return 2;
+ }
+ score += repeat_score(g, p->name->definition, call_depth + 1);
+ if (score >= 2)
+ return score;
+ break;
+
+ case c_bra:
+ score += repeat_score(g, p->left, call_depth);
+ if (score >= 2)
+ return score;
+ break;
+
+ case c_name:
+ case c_literalstring:
+ case c_next:
+ case c_grouping:
+ case c_non:
+ case c_hop:
+ if (++score >= 2)
+ return score;
+ break;
+
+ default:
+ return 2;
+ }
+ p = p->right;
+ }
+ return score;
+}
+
+/* tests if an expression requires cursor reinstatement in a repeat */
+
+extern int repeat_restore(struct generator * g, struct node * p) {
+ return repeat_score(g, p, 0) >= 2;
+}
+
+static void generate_bra(struct generator * g, struct node * p) {
+ p = p->left;
+ while (p) {
+ generate(g, p);
+ p = p->right;
+ }
+}
+
+static void generate_and(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+ p = p->left;
+ while (p) {
+ generate(g, p);
+ if (keep_c && p->right != 0) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ p = p->right;
+ }
+ if (keep_c) w(g, "~}");
+}
+
+static void generate_or(struct generator * g, struct node * p) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ int out_lab = new_label(g);
+
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+ p = p->left;
+ g->failure_keep_count = 0;
+ while (p->right) {
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p);
+ wgotol(g, out_lab);
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ p = p->right;
+ }
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+ generate(g, p);
+ if (keep_c) w(g, "~}");
+ wsetl(g, out_lab);
+}
+
+static void generate_backwards(struct generator * g, struct node * p) {
+
+ writef(g, "~Mz->lb = z->c; z->c = z->l;~C~N", p);
+ generate(g, p->left);
+ w(g, "~Mz->c = z->lb;~N");
+}
+
+
+static void generate_not(struct generator * g, struct node * p) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+ generate(g, p->left);
+
+ {
+ int l = g->failure_label;
+ int u = g->label_used;
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+ writef(g, "~M~f~N", p);
+ if (u)
+ wsetl(g, l);
+ }
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N~}");
+ }
+}
+
+
+static void generate_try(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+ g->failure_keep_count = keep_c;
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p->left);
+
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+
+ if (keep_c) w(g, "~}");
+}
+
+static void generate_set(struct generator * g, struct node * p) {
+ g->V[0] = p->name; writef(g, "~M~V0 = 1;~C", p);
+}
+
+static void generate_unset(struct generator * g, struct node * p) {
+ g->V[0] = p->name; writef(g, "~M~V0 = 0;~C", p);
+}
+
+static void generate_fail(struct generator * g, struct node * p) {
+ generate(g, p->left);
+ writef(g, "~M~f~C", p);
+}
+
+/* generate_test() also implements 'reverse' */
+
+static void generate_test(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ keep_c = ++g->keep_count;
+ w(g, p->mode == m_forward ? "~{int c_test" :
+ "~{int m_test");
+ write_int(g, keep_c);
+ w(g, p->mode == m_forward ? " = z->c;" :
+ " = z->l - z->c;");
+ writef(g, "~C", p);
+ } else writef(g, "~M~C", p);
+
+ generate(g, p->left);
+
+ if (keep_c) {
+ w(g, p->mode == m_forward ? "~Mz->c = c_test" :
+ "~Mz->c = z->l - m_test");
+ write_int(g, keep_c);
+ writef(g, ";~N~}", p);
+ }
+}
+
+static void generate_do(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+
+ if (p->left->type == c_call) {
+ /* Optimise do <call> */
+ g->V[0] = p->left->name;
+ writef(g, "~{int ret = ~V0(z);~C", p->left);
+ w(g, "~Mif (ret < 0) return ret;~N~}");
+ } else {
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+ generate(g, p->left);
+
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ }
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c);
+ w(g, "~N~}");
+ }
+}
+
+static void generate_next(struct generator * g, struct node * p) {
+ if (g->options->encoding == ENC_UTF8) {
+ if (p->mode == m_forward)
+ w(g, "~{int ret = skip_utf8(z->p, z->c, 0, z->l, 1");
+ else
+ w(g, "~{int ret = skip_utf8(z->p, z->c, z->lb, 0, -1");
+ writef(g, ");~N"
+ "~Mif (ret < 0) ~f~N"
+ "~Mz->c = ret;~C"
+ "~}", p);
+ } else
+ writef(g, "~M~l~N"
+ "~M~i~C", p);
+}
+
+static void generate_GO_grouping(struct generator * g, struct node * p, int is_goto, int complement) {
+
+ struct grouping * q = p->name->grouping;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->S[1] = complement ? "in" : "out";
+ g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
+ g->V[0] = p->name;
+ g->I[0] = q->smallest_ch;
+ g->I[1] = q->largest_ch;
+ if (is_goto) {
+ writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f~C", p);
+ } else {
+ writef(g, "~{~C"
+ "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N"
+ "~Mif (ret < 0) ~f~N", p);
+ if (p->mode == m_forward)
+ w(g, "~Mz->c += ret;~N");
+ else
+ w(g, "~Mz->c -= ret;~N");
+ w(g, "~}");
+ }
+}
+
+static void generate_GO(struct generator * g, struct node * p, int style) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ if (p->left->type == c_grouping || p->left->type == c_non) {
+ /* Special case for "goto" or "gopast" when used on a grouping or an
+ * inverted grouping - the movement of c by the matching action is
+ * exactly what we want! */
+#ifdef OPTIMISATION_WARNINGS
+ printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping");
+#endif
+ if (g->options->comments) {
+ writef(g, "~M~C", p);
+ }
+ generate_GO_grouping(g, p->left, style, p->left->type == c_non);
+ return;
+ }
+
+ w(g, "~Mwhile(1) {"); writef(g, "~C~+", p);
+
+ if (style == 1 || repeat_restore(g, p->left)) {
+ writef(g, "~M~k~N", p);
+ keep_c = g->keep_count;
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p->left);
+
+ if (style == 1) {
+ /* include for goto; omit for gopast */
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ w(g, "~Mbreak;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+/* writef(g, "~M~l~N"
+ "~M~i~N", p); */
+ generate_next(g, p);
+ w(g, "~}");
+}
+
+static void generate_loop(struct generator * g, struct node * p) {
+ w(g, "~{int i; for (i = "); generate_AE(g, p->AE); writef(g, "; i > 0; i--)~C"
+ "~{", p);
+
+ generate(g, p->left);
+
+ w(g, "~}"
+ "~}");
+}
+
+static void generate_repeat_or_atleast(struct generator * g, struct node * p, int atleast_case) {
+ int keep_c = 0;
+ if (atleast_case) {
+ writef(g, "~Mwhile(1) {~+~N", p);
+ } else {
+ writef(g, "~Mwhile(1) {~+~C", p);
+ }
+
+ if (repeat_restore(g, p->left)) {
+ writef(g, "~M~k~N", p);
+ keep_c = g->keep_count;
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+ generate(g, p->left);
+
+ if (atleast_case) w(g, "~Mi--;~N");
+
+ w(g, "~Mcontinue;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ w(g, "~Mbreak;~N"
+ "~}");
+}
+
+static void generate_repeat(struct generator * g, struct node * p) {
+ generate_repeat_or_atleast(g, p, false);
+}
+
+static void generate_atleast(struct generator * g, struct node * p) {
+ w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~C");
+ {
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ generate_repeat_or_atleast(g, p, true);
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+ }
+ writef(g, "~Mif (i > 0) ~f~N"
+ "~}", p);
+}
+
+static void generate_setmark(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = z->c;~C", p);
+}
+
+static void generate_tomark(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? ">" : "<";
+
+ w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); writef(g, ") ~f~N", p);
+ w(g, "~Mz->c = "); generate_AE(g, p->AE); writef(g, ";~C", p);
+}
+
+static void generate_atmark(struct generator * g, struct node * p) {
+
+ w(g, "~Mif (z->c != "); generate_AE(g, p->AE); writef(g, ") ~f~C", p);
+}
+
+static void generate_hop(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "+" : "-";
+ g->S[1] = p->mode == m_forward ? "0" : "z->lb";
+ if (g->options->encoding == ENC_UTF8) {
+ w(g, "~{int ret = skip_utf8(z->p, z->c, ~S1, z->l, ~S0 ");
+ generate_AE(g, p->AE); writef(g, ");~C", p);
+ writef(g, "~Mif (ret < 0) ~f~N", p);
+ } else {
+ w(g, "~{int ret = z->c ~S0 ");
+ generate_AE(g, p->AE); writef(g, ";~C", p);
+ writef(g, "~Mif (~S1 > ret || ret > z->l) ~f~N", p);
+ }
+ writef(g, "~Mz->c = ret;~N"
+ "~}", p);
+}
+
+static void generate_delete(struct generator * g, struct node * p) {
+ writef(g, "~{int ret = slice_del(z);~C", p);
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_tolimit(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "" : "b";
+ writef(g, "~Mz->c = z->l~S0;~C", p);
+}
+
+static void generate_atlimit(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "" : "b";
+ g->S[1] = p->mode == m_forward ? "<" : ">";
+ writef(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p);
+}
+
+static void generate_leftslice(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "bra" : "ket";
+ writef(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_rightslice(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "ket" : "bra";
+ writef(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_assignto(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = assign_to(z, ~V0);~C"
+ "~Mif (~V0 == 0) return -1;~C", p);
+}
+
+static void generate_sliceto(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = slice_to(z, ~V0);~C"
+ "~Mif (~V0 == 0) return -1;~N", p);
+}
+
+static void generate_insert(struct generator * g, struct node * p, int style) {
+
+ int keep_c = style == c_attach;
+ if (p->mode == m_backward) keep_c = !keep_c;
+ writef(g, "~{int ret;~N", p);
+ if (keep_c) w(g, "~{int saved_c = z->c;~N");
+ writef(g, "~Mret = insert_~$(z, z->c, z->c, ~a);~C", p);
+ if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_assignfrom(struct generator * g, struct node * p) {
+
+ int keep_c = p->mode == m_forward; /* like 'attach' */
+ writef(g, "~{int ret;~N", p);
+ if (keep_c) writef(g, "~{int saved_c = z->c;~N", p);
+ w(g, "~Mret = ");
+ writef(g, keep_c ? "insert_~$(z, z->c, z->l, ~a);~C" : "insert_~$(z, z->lb, z->c, ~a);~C", p);
+ if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+/* bugs marked <======= fixed 22/7/02. Similar fixes required for Java */
+
+static void generate_slicefrom(struct generator * g, struct node * p) {
+
+/* w(g, "~Mslice_from_s(z, "); <============= bug! should be: */
+ writef(g, "~{int ret = slice_from_~$(z, ~a);~C", p);
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_setlimit(struct generator * g, struct node * p) {
+ int keep_c;
+ if (p->left && p->left->type == c_tomark) {
+ /* Special case for:
+ *
+ * setlimit tomark AE for C
+ *
+ * All uses of setlimit in the current stemmers we ship follow this
+ * pattern, and by special-casing we can avoid having to save and
+ * restore c.
+ */
+ struct node * q = p->left;
+
+ ++g->keep_count;
+ writef(g, "~N~{int mlimit", p);
+ write_int(g, g->keep_count);
+ writef(g, ";~C", p);
+ keep_c = g->keep_count;
+
+ g->S[0] = q->mode == m_forward ? ">" : "<";
+
+ w(g, "~Mif (z->c ~S0 "); generate_AE(g, q->AE); writef(g, ") ~f~N", q);
+ w(g, "~Mmlimit");
+ write_int(g, keep_c);
+ if (p->mode == m_forward) {
+ w(g, " = z->l - z->c; z->l = ");
+ } else {
+ w(g, " = z->lb; z->lb = ");
+ }
+ generate_AE(g, q->AE);
+ w(g, ";~N");
+ } else {
+ writef(g, "~{~K~C", p);
+ keep_c = g->keep_count;
+ generate(g, p->left);
+
+ w(g, "~Mmlimit");
+ write_int(g, keep_c);
+ if (p->mode == m_forward)
+ w(g, " = z->l - z->c; z->l = z->c;~N");
+ else
+ w(g, " = z->lb; z->lb = z->c;~N");
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ g->failure_keep_count = -keep_c;
+ generate(g, p->aux);
+ w(g, "~M");
+ wrestorelimit(g, p, -g->failure_keep_count);
+ w(g, "~N"
+ "~}");
+}
+
+/* dollar sets snowball up to operate on a string variable as if it were the
+ * current string */
+static void generate_dollar(struct generator * g, struct node * p) {
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+ int keep_token;
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+
+ keep_token = ++g->keep_count;
+ g->I[0] = keep_token;
+ writef(g, "~{struct SN_env env~I0 = * z;~C", p);
+ g->V[0] = p->name;
+ /* Assume failure. */
+ writef(g, "~Mint failure = 1;~N"
+ "~Mz->p = ~V0;~N"
+ "~Mz->lb = z->c = 0;~N"
+ "~Mz->l = SIZE(z->p);~N", p);
+ generate(g, p->left);
+ /* Mark success. */
+ w(g, "~Mfailure = 0;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ g->V[0] = p->name; /* necessary */
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+ g->I[0] = keep_token;
+ writef(g, "~M~V0 = z->p;~N"
+ "~M* z = env~I0;~N"
+ "~Mif (failure) ~f~N~}", p);
+}
+
+static void generate_integer_assign(struct generator * g, struct node * p, char * s) {
+
+ g->V[0] = p->name;
+ g->S[0] = s;
+ w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); writef(g, ";~C", p);
+}
+
+static void generate_integer_test(struct generator * g, struct node * p, char * s) {
+
+ w(g, "~Mif (!(");
+ generate_AE(g, p->left);
+ write_char(g, ' ');
+ write_string(g, s);
+ write_char(g, ' ');
+ generate_AE(g, p->AE);
+ writef(g, ")) ~f~C", p);
+}
+
+static void generate_call(struct generator * g, struct node * p) {
+
+ g->V[0] = p->name;
+ writef(g, "~{int ret = ~V0(z);~C", p);
+ if (g->failure_keep_count == 0 && g->failure_label == x_return) {
+ /* Combine the two tests in this special case for better optimisation
+ * and clearer generated code. */
+ writef(g, "~Mif (ret <= 0) return ret;~N~}", p);
+ } else {
+ writef(g, "~Mif (ret == 0) ~f~N"
+ "~Mif (ret < 0) return ret;~N~}", p);
+ }
+}
+
+static void generate_grouping(struct generator * g, struct node * p, int complement) {
+
+ struct grouping * q = p->name->grouping;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->S[1] = complement ? "out" : "in";
+ g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
+ g->V[0] = p->name;
+ g->I[0] = q->smallest_ch;
+ g->I[1] = q->largest_ch;
+ writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p);
+}
+
+static void generate_namedstring(struct generator * g, struct node * p) {
+
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->V[0] = p->name;
+ writef(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p);
+}
+
+static void generate_literalstring(struct generator * g, struct node * p) {
+ symbol * b = p->literalstring;
+ if (SIZE(b) == 1) {
+ /* It's quite common to compare with a single character literal string,
+ * so just inline the simpler code for this case rather than making a
+ * function call. In UTF-8 mode, only do this for the ASCII subset,
+ * since multi-byte characters are more complex to test against.
+ */
+ if (g->options->encoding == ENC_UTF8 && *b >= 128) {
+ printf("single byte %d\n", *b);
+ exit(1);
+ }
+ g->I[0] = *b;
+ if (p->mode == m_forward) {
+ writef(g, "~Mif (z->c == z->l || z->p[z->c] != ~c0) ~f~C"
+ "~Mz->c++;~N", p);
+ } else {
+ writef(g, "~Mif (z->c <= z->lb || z->p[z->c - 1] != ~c0) ~f~C"
+ "~Mz->c--;~N", p);
+ }
+ } else {
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = SIZE(b);
+ g->L[0] = b;
+
+ writef(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p);
+ }
+}
+
+static void generate_define(struct generator * g, struct node * p) {
+ struct name * q = p->name;
+ g->next_label = 0;
+
+ g->S[0] = q->type == t_routine ? "static" : "extern";
+ g->V[0] = q;
+
+ w(g, "~N~S0 int ~V0(struct SN_env * z) {");
+ if (g->options->comments) {
+ write_string(g, p->mode == m_forward ? " /* forwardmode */" : " /* backwardmode */");
+ }
+ w(g, "~N~+");
+ if (p->amongvar_needed) w(g, "~Mint among_var;~N");
+ g->failure_keep_count = 0;
+ g->failure_label = x_return;
+ g->label_used = 0;
+ g->keep_count = 0;
+ generate(g, p->left);
+ w(g, "~Mreturn 1;~N~}");
+}
+
+static void generate_substring(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+ int block = -1;
+ unsigned int bitmap = 0;
+ struct amongvec * among_cases = x->b;
+ int c;
+ int empty_case = -1;
+ int n_cases = 0;
+ symbol cases[2];
+ int shortest_size = INT_MAX;
+ int shown_comment = 0;
+
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = x->number;
+ g->I[1] = x->literalstring_count;
+
+ /* In forward mode with non-ASCII UTF-8 characters, the first character
+ * of the string will often be the same, so instead look at the last
+ * common character position.
+ *
+ * In backward mode, we can't match if there are fewer characters before
+ * the current position than the minimum length.
+ */
+ for (c = 0; c < x->literalstring_count; ++c) {
+ int size = among_cases[c].size;
+ if (size != 0 && size < shortest_size) {
+ shortest_size = size;
+ }
+ }
+
+ for (c = 0; c < x->literalstring_count; ++c) {
+ symbol ch;
+ if (among_cases[c].size == 0) {
+ empty_case = c;
+ continue;
+ }
+ if (p->mode == m_forward) {
+ ch = among_cases[c].b[shortest_size - 1];
+ } else {
+ ch = among_cases[c].b[among_cases[c].size - 1];
+ }
+ if (n_cases == 0) {
+ block = ch >> 5;
+ } else if (ch >> 5 != block) {
+ block = -1;
+ if (n_cases > 2) break;
+ }
+ if (block == -1) {
+ if (n_cases > 0 && ch == cases[0]) continue;
+ if (n_cases < 2) {
+ cases[n_cases++] = ch;
+ } else if (ch != cases[1]) {
+ ++n_cases;
+ break;
+ }
+ } else {
+ if ((bitmap & (1u << (ch & 0x1f))) == 0) {
+ bitmap |= 1u << (ch & 0x1f);
+ if (n_cases < 2)
+ cases[n_cases] = ch;
+ ++n_cases;
+ }
+ }
+ }
+
+ if (block != -1 || n_cases <= 2) {
+ char buf[64];
+ g->I[2] = block;
+ g->I[3] = bitmap;
+ g->I[4] = shortest_size - 1;
+ if (p->mode == m_forward) {
+ sprintf(buf, "z->p[z->c + %d]", shortest_size - 1);
+ g->S[1] = buf;
+ if (shortest_size == 1) {
+ writef(g, "~Mif (z->c >= z->l", p);
+ } else {
+ writef(g, "~Mif (z->c + ~I4 >= z->l", p);
+ }
+ } else {
+ g->S[1] = "z->p[z->c - 1]";
+ if (shortest_size == 1) {
+ writef(g, "~Mif (z->c <= z->lb", p);
+ } else {
+ writef(g, "~Mif (z->c - ~I4 <= z->lb", p);
+ }
+ }
+ if (n_cases == 0) {
+ /* We get this for the degenerate case: among { '' }
+ * This doesn't seem to be a useful construct, but it is
+ * syntactically valid.
+ */
+ } else if (n_cases == 1) {
+ g->I[4] = cases[0];
+ writef(g, " || ~S1 != ~I4", p);
+ } else if (n_cases == 2) {
+ g->I[4] = cases[0];
+ g->I[5] = cases[1];
+ writef(g, " || (~S1 != ~I4 && ~S1 != ~I5)", p);
+ } else {
+ writef(g, " || ~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p);
+ }
+ write_string(g, ") ");
+ if (empty_case != -1) {
+ /* If the among includes the empty string, it can never fail
+ * so not matching the bitmap means we match the empty string.
+ */
+ g->I[4] = among_cases[empty_case].result;
+ writef(g, "among_var = ~I4; else~C", p);
+ } else {
+ writef(g, "~f~C", p);
+ }
+ shown_comment = 1;
+ } else {
+#ifdef OPTIMISATION_WARNINGS
+ printf("Couldn't shortcut among %d\n", x->number);
+#endif
+ }
+
+ if (!x->amongvar_needed) {
+ writef(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f", p);
+ writef(g, shown_comment ? "~N" : "~C", p);
+ } else {
+ writef(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);", p);
+ writef(g, shown_comment ? "~N" : "~C", p);
+ writef(g, "~Mif (!(among_var)) ~f~N", p);
+ }
+}
+
+static void generate_among(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+
+ if (x->substring == 0) generate_substring(g, p);
+
+ if (x->starter != 0) generate(g, x->starter);
+
+ if (x->command_count == 1 && x->nocommand_count == 0) {
+ /* Only one outcome ("no match" already handled). */
+ generate(g, x->commands[0]);
+ } else if (x->command_count > 0) {
+ int i;
+ writef(g, "~Mswitch (among_var) {~C~+", p);
+ for (i = 1; i <= x->command_count; i++) {
+ g->I[0] = i;
+ w(g, "~Mcase ~I0:~N~+");
+ generate(g, x->commands[i - 1]);
+ w(g, "~Mbreak;~N~-");
+ }
+ w(g, "~}");
+ }
+}
+
+static void generate_booltest(struct generator * g, struct node * p) {
+
+ g->V[0] = p->name;
+ writef(g, "~Mif (!(~V0)) ~f~C", p);
+}
+
+static void generate_false(struct generator * g, struct node * p) {
+
+ writef(g, "~M~f~C", p);
+}
+
+static void generate_debug(struct generator * g, struct node * p) {
+
+ g->I[0] = g->debug_count++;
+ g->I[1] = p->line_number;
+ writef(g, "~Mdebug(z, ~I0, ~I1);~C", p);
+
+}
+
+static void generate(struct generator * g, struct node * p) {
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ switch (p->type) {
+ case c_define: generate_define(g, p); break;
+ case c_bra: generate_bra(g, p); break;
+ case c_and: generate_and(g, p); break;
+ case c_or: generate_or(g, p); break;
+ case c_backwards: generate_backwards(g, p); break;
+ case c_not: generate_not(g, p); break;
+ case c_set: generate_set(g, p); break;
+ case c_unset: generate_unset(g, p); break;
+ case c_try: generate_try(g, p); break;
+ case c_fail: generate_fail(g, p); break;
+ case c_reverse:
+ case c_test: generate_test(g, p); break;
+ case c_do: generate_do(g, p); break;
+ case c_goto: generate_GO(g, p, 1); break;
+ case c_gopast: generate_GO(g, p, 0); break;
+ case c_repeat: generate_repeat(g, p); break;
+ case c_loop: generate_loop(g, p); break;
+ case c_atleast: generate_atleast(g, p); break;
+ case c_setmark: generate_setmark(g, p); break;
+ case c_tomark: generate_tomark(g, p); break;
+ case c_atmark: generate_atmark(g, p); break;
+ case c_hop: generate_hop(g, p); break;
+ case c_delete: generate_delete(g, p); break;
+ case c_next: generate_next(g, p); break;
+ case c_tolimit: generate_tolimit(g, p); break;
+ case c_atlimit: generate_atlimit(g, p); break;
+ case c_leftslice: generate_leftslice(g, p); break;
+ case c_rightslice: generate_rightslice(g, p); break;
+ case c_assignto: generate_assignto(g, p); break;
+ case c_sliceto: generate_sliceto(g, p); break;
+ case c_assign: generate_assignfrom(g, p); break;
+ case c_insert:
+ case c_attach: generate_insert(g, p, p->type); break;
+ case c_slicefrom: generate_slicefrom(g, p); break;
+ case c_setlimit: generate_setlimit(g, p); break;
+ case c_dollar: generate_dollar(g, p); break;
+ case c_mathassign: generate_integer_assign(g, p, "="); break;
+ case c_plusassign: generate_integer_assign(g, p, "+="); break;
+ case c_minusassign: generate_integer_assign(g, p, "-="); break;
+ case c_multiplyassign:generate_integer_assign(g, p, "*="); break;
+ case c_divideassign: generate_integer_assign(g, p, "/="); break;
+ case c_eq: generate_integer_test(g, p, "=="); break;
+ case c_ne: generate_integer_test(g, p, "!="); break;
+ case c_gr: generate_integer_test(g, p, ">"); break;
+ case c_ge: generate_integer_test(g, p, ">="); break;
+ case c_ls: generate_integer_test(g, p, "<"); break;
+ case c_le: generate_integer_test(g, p, "<="); break;
+ case c_call: generate_call(g, p); break;
+ case c_grouping: generate_grouping(g, p, false); break;
+ case c_non: generate_grouping(g, p, true); break;
+ case c_name: generate_namedstring(g, p); break;
+ case c_literalstring: generate_literalstring(g, p); break;
+ case c_among: generate_among(g, p); break;
+ case c_substring: generate_substring(g, p); break;
+ case c_booltest: generate_booltest(g, p); break;
+ case c_false: generate_false(g, p); break;
+ case c_true: break;
+ case c_debug: generate_debug(g, p); break;
+ default: fprintf(stderr, "%d encountered\n", p->type);
+ exit(1);
+ }
+
+ if (g->failure_label != a0)
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+}
+
+void write_generated_comment_content(struct generator * g) {
+ w(g, "Generated by Snowball " SNOWBALL_VERSION
+ " - https://snowballstem.org/");
+}
+
+void write_start_comment(struct generator * g,
+ const char * comment_start,
+ const char * comment_end) {
+ write_margin(g);
+ w(g, comment_start);
+ write_generated_comment_content(g);
+ if (comment_end) {
+ w(g, comment_end);
+ }
+ w(g, "~N~N");
+}
+
+static void generate_head(struct generator * g) {
+
+ w(g, "#include \"");
+ if (g->options->runtime_path) {
+ write_string(g, g->options->runtime_path);
+ if (g->options->runtime_path[strlen(g->options->runtime_path) - 1] != '/')
+ write_char(g, '/');
+ }
+ w(g, "header.h\"~N~N");
+}
+
+static void generate_routine_headers(struct generator * g) {
+ struct name * q;
+ for (q = g->analyser->names; q; q = q->next) {
+ g->V[0] = q;
+ switch (q->type) {
+ case t_routine:
+ w(g, "static int ~W0(struct SN_env * z);~N");
+ break;
+ case t_external:
+ w(g,
+ "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"
+ "extern int ~W0(struct SN_env * z);~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N"
+ );
+ break;
+ }
+ }
+}
+
+static void generate_among_table(struct generator * g, struct among * x) {
+
+ struct amongvec * v = x->b;
+
+ g->I[0] = x->number;
+ {
+ int i;
+ for (i = 0; i < x->literalstring_count; i++) {
+ g->I[1] = i;
+ g->I[2] = v->size;
+ g->L[0] = v->b;
+ if (v->size)
+ w(g, "static const symbol s_~I0_~I1[~I2] = ~A0;~N");
+ v++;
+ }
+ }
+
+ g->I[1] = x->literalstring_count;
+ w(g, "~N~Mstatic const struct among a_~I0[~I1] =~N{~N");
+
+ v = x->b;
+ {
+ int i;
+ for (i = 0; i < x->literalstring_count; i++) {
+ g->I[1] = i;
+ g->I[2] = v->size;
+ g->I[3] = v->i;
+ g->I[4] = v->result;
+ g->S[0] = i < x->literalstring_count - 1 ? "," : "";
+
+ if (g->options->comments) {
+ w(g, "/*~J1 */ ");
+ }
+ w(g, "{ ~I2, ");
+ if (v->size == 0) {
+ w(g, "0,");
+ } else {
+ w(g, "s_~I0_~I1,");
+ }
+ w(g, " ~I3, ~I4, ");
+ if (v->function == 0) {
+ write_char(g, '0');
+ } else {
+ write_varname(g, v->function);
+ }
+ w(g, "}~S0~N");
+ v++;
+ }
+ }
+ w(g, "};~N~N");
+}
+
+static void generate_amongs(struct generator * g) {
+ struct among * x;
+ for (x = g->analyser->amongs; x; x = x->next) {
+ generate_among_table(g, x);
+ }
+}
+
+static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; }
+
+static void generate_grouping_table(struct generator * g, struct grouping * q) {
+
+ int range = q->largest_ch - q->smallest_ch + 1;
+ int size = (range + 7)/ 8; /* assume 8 bits per symbol */
+ symbol * b = q->b;
+ symbol * map = create_b(size);
+ int i;
+ for (i = 0; i < size; i++) map[i] = 0;
+
+ for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch);
+
+ g->V[0] = q->name;
+
+ w(g, "static const unsigned char ~V0[] = { ");
+ for (i = 0; i < size; i++) {
+ write_int(g, map[i]);
+ if (i < size - 1) w(g, ", ");
+ }
+ w(g, " };~N~N");
+ lose_b(map);
+}
+
+static void generate_groupings(struct generator * g) {
+ struct grouping * q;
+ for (q = g->analyser->groupings; q; q = q->next) {
+ if (q->name->used)
+ generate_grouping_table(g, q);
+ }
+}
+
+static void generate_create(struct generator * g) {
+
+ int * p = g->analyser->name_count;
+ g->I[0] = p[t_string];
+ g->I[1] = p[t_integer] + p[t_boolean];
+ w(g, "~N"
+ "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1); }"
+ "~N");
+}
+
+static void generate_close(struct generator * g) {
+
+ int * p = g->analyser->name_count;
+ g->I[0] = p[t_string];
+ w(g, "~Nextern void ~pclose_env(struct SN_env * z) { SN_close_env(z, ~I0); }~N~N");
+}
+
+static void generate_create_and_close_templates(struct generator * g) {
+ w(g, "~N"
+ "extern struct SN_env * ~pcreate_env(void);~N"
+ "extern void ~pclose_env(struct SN_env * z);~N"
+ "~N");
+}
+
+static void generate_header_file(struct generator * g) {
+
+ struct name * q;
+ const char * vp = g->options->variables_prefix;
+ g->S[0] = vp;
+
+ w(g, "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"); /* for C++ */
+
+ generate_create_and_close_templates(g);
+ for (q = g->analyser->names; q; q = q->next) {
+ g->V[0] = q;
+ switch (q->type) {
+ case t_external:
+ w(g, "extern int ~W0(struct SN_env * z);~N");
+ break;
+ case t_string:
+ case t_integer:
+ case t_boolean:
+ if (vp) {
+ int count = q->count;
+ if (count < 0) {
+ /* Unused variables should get removed from `names`. */
+ fprintf(stderr, "Optimised out variable ");
+ report_b(stderr, q->b);
+ fprintf(stderr, " still in names list\n");
+ exit(1);
+ }
+ if (q->type == t_boolean) {
+ /* We use a single array for booleans and integers,
+ * with the integers first.
+ */
+ count += g->analyser->name_count[t_integer];
+ }
+ g->I[0] = count;
+ g->I[1] = "SIIrxg"[q->type];
+ w(g, "#define ~S0");
+ write_b(g, q->b);
+ w(g, " (~c1[~I0])~N");
+ }
+ break;
+ }
+ }
+
+ w(g, "~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N"); /* for C++ */
+
+ w(g, "~N");
+}
+
+extern void generate_program_c(struct generator * g) {
+
+ g->outbuf = str_new();
+ write_start_comment(g, "/* ", " */");
+ generate_head(g);
+ generate_routine_headers(g);
+ w(g, "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"
+ "~N");
+ generate_create_and_close_templates(g);
+ w(g, "~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N");
+ generate_amongs(g);
+ generate_groupings(g);
+ g->declarations = g->outbuf;
+ g->outbuf = str_new();
+ g->literalstring_count = 0;
+ {
+ struct node * p = g->analyser->program;
+ while (p) { generate(g, p); p = p->right; }
+ }
+ generate_create(g);
+ generate_close(g);
+ output_str(g->options->output_src, g->declarations);
+ str_delete(g->declarations);
+ output_str(g->options->output_src, g->outbuf);
+ str_clear(g->outbuf);
+
+ write_start_comment(g, "/* ", " */");
+ generate_header_file(g);
+ output_str(g->options->output_h, g->outbuf);
+ str_delete(g->outbuf);
+}
+
+/* Generator functions common to multiple languages. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o) {
+ NEW(generator, g);
+ g->analyser = a;
+ g->options = o;
+ g->margin = 0;
+ g->debug_count = 0;
+ g->copy_from_count = 0;
+ g->line_count = 0;
+ g->line_labelled = 0;
+ g->failure_label = -1;
+ g->unreachable = false;
+#ifndef DISABLE_PYTHON
+ g->max_label = 0;
+#endif
+ return g;
+}
+
+extern void close_generator(struct generator * g) {
+ FREE(g);
+}
+
+/* Write routines for simple entities */
+
+extern void write_char(struct generator * g, int ch) {
+ str_append_ch(g->outbuf, ch); /* character */
+}
+
+extern void write_newline(struct generator * g) {
+ str_append_ch(g->outbuf, '\n'); /* newline */
+ g->line_count++;
+}
+
+extern void write_string(struct generator * g, const char * s) {
+ str_append_string(g->outbuf, s);
+}
+
+extern void write_int(struct generator * g, int i) {
+ str_append_int(g->outbuf, i);
+}
+
+extern void write_b(struct generator * g, symbol * b) {
+
+ str_append_b(g->outbuf, b);
+}
+
+extern void write_str(struct generator * g, struct str * str) {
+
+ str_append(g->outbuf, str);
+}
diff --git a/contrib/snowball/compiler/header.h b/contrib/snowball/compiler/header.h
new file mode 100644
index 0000000..dadbee3
--- /dev/null
+++ b/contrib/snowball/compiler/header.h
@@ -0,0 +1,411 @@
+#include <stdio.h>
+
+#define SNOWBALL_VERSION "2.0.0"
+
+typedef unsigned char byte;
+typedef unsigned short symbol;
+
+#define true 1
+#define false 0
+
+#define MALLOC check_malloc
+#define FREE check_free
+
+#define NEW(type, p) struct type * p = (struct type *) MALLOC(sizeof(struct type))
+#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * (n))
+
+#define SIZE(p) ((int *)(p))[-1]
+#define CAPACITY(p) ((int *)(p))[-2]
+
+extern symbol * create_b(int n);
+extern void report_b(FILE * out, const symbol * p);
+extern void lose_b(symbol * p);
+extern symbol * increase_capacity(symbol * p, int n);
+extern symbol * move_to_b(symbol * p, int n, const symbol * q);
+extern symbol * add_to_b(symbol * p, int n, const symbol * q);
+extern symbol * copy_b(const symbol * p);
+extern char * b_to_s(const symbol * p);
+extern symbol * add_s_to_b(symbol * p, const char * s);
+
+#define MOVE_TO_B(B, LIT) \
+ move_to_b(B, sizeof(LIT) / sizeof(LIT[0]), LIT)
+
+struct str; /* defined in space.c */
+
+extern struct str * str_new(void);
+extern void str_delete(struct str * str);
+extern void str_append(struct str * str, const struct str * add);
+extern void str_append_ch(struct str * str, char add);
+extern void str_append_b(struct str * str, const symbol * q);
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip);
+extern void str_append_string(struct str * str, const char * s);
+extern void str_append_int(struct str * str, int i);
+extern void str_clear(struct str * str);
+extern void str_assign(struct str * str, const char * s);
+extern struct str * str_copy(const struct str * old);
+extern symbol * str_data(const struct str * str);
+extern int str_len(const struct str * str);
+extern int str_back(const struct str *str);
+extern int get_utf8(const symbol * p, int * slot);
+extern int put_utf8(int ch, symbol * p);
+extern void output_str(FILE * outfile, struct str * str);
+
+typedef enum { ENC_SINGLEBYTE, ENC_UTF8, ENC_WIDECHARS } enc;
+
+struct m_pair {
+
+ struct m_pair * next;
+ symbol * name;
+ symbol * value;
+
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct input {
+
+ struct input * next;
+ symbol * p;
+ int c;
+ char * file;
+ int file_needs_freeing;
+ int line_number;
+
+};
+
+struct include {
+
+ struct include * next;
+ symbol * b;
+
+};
+
+enum token_codes {
+
+#include "syswords2.h"
+
+ c_mathassign,
+ c_name,
+ c_number,
+ c_literalstring,
+ c_neg,
+ c_call,
+ c_grouping,
+ c_booltest,
+
+ NUM_TOKEN_CODES
+};
+
+enum uplus_modes {
+ UPLUS_NONE,
+ UPLUS_DEFINED,
+ UPLUS_UNICODE
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct tokeniser {
+
+ struct input * next;
+ symbol * p;
+ int c;
+ char * file;
+ int file_needs_freeing;
+ int line_number;
+ symbol * b;
+ symbol * b2;
+ int number;
+ int m_start;
+ int m_end;
+ struct m_pair * m_pairs;
+ int get_depth;
+ int error_count;
+ int token;
+ int previous_token;
+ byte token_held;
+ enc encoding;
+
+ int omission;
+ struct include * includes;
+
+ /* Mode in which U+ has been used:
+ * UPLUS_NONE - not used yet
+ * UPLUS_DEFINED - stringdef U+xxxx ....
+ * UPLUS_UNICODE - {U+xxxx} used with implicit meaning
+ */
+ int uplusmode;
+
+ char token_disabled[NUM_TOKEN_CODES];
+};
+
+extern symbol * get_input(const char * filename);
+extern struct tokeniser * create_tokeniser(symbol * b, char * file);
+extern int read_token(struct tokeniser * t);
+extern const char * name_of_token(int code);
+extern void disable_token(struct tokeniser * t, int code);
+extern void close_tokeniser(struct tokeniser * t);
+
+extern int space_count;
+extern void * check_malloc(int n);
+extern void check_free(void * p);
+
+struct node;
+
+struct name {
+
+ struct name * next;
+ symbol * b;
+ int type; /* t_string etc */
+ int mode; /* )_ for routines, externals */
+ struct node * definition; /* ) */
+ int count; /* 0, 1, 2 for each type */
+ struct grouping * grouping; /* for grouping names */
+ byte referenced;
+ byte used_in_among; /* Function used in among? */
+ byte value_used; /* (For variables) is its value ever used? */
+ byte initialised; /* (For variables) is it ever initialised? */
+ byte used_in_definition; /* (grouping) used in grouping definition? */
+ struct node * used; /* First use, or NULL if not used */
+ struct name * local_to; /* Local to one routine/external */
+ int declaration_line_number;/* Line number of declaration */
+
+};
+
+struct literalstring {
+
+ struct literalstring * next;
+ symbol * b;
+
+};
+
+struct amongvec {
+
+ symbol * b; /* the string giving the case */
+ int size; /* - and its size */
+ struct node * action; /* the corresponding action */
+ int i; /* the amongvec index of the longest substring of b */
+ int result; /* the numeric result for the case */
+ int line_number; /* for diagnostics and stable sorting */
+ struct name * function;
+
+};
+
+struct among {
+
+ struct among * next;
+ struct amongvec * b; /* pointer to the amongvec */
+ int number; /* amongs are numbered 0, 1, 2 ... */
+ int literalstring_count; /* in this among */
+ int command_count; /* in this among */
+ int nocommand_count; /* number of "no command" entries in this among */
+ int function_count; /* in this among */
+ int amongvar_needed; /* do we need to set among_var? */
+ struct node * starter; /* i.e. among( (starter) 'string' ... ) */
+ struct node * substring; /* i.e. substring ... among ( ... ) */
+ struct node ** commands; /* array with command_count entries */
+};
+
+struct grouping {
+
+ struct grouping * next;
+ symbol * b; /* the characters of this group */
+ int largest_ch; /* character with max code */
+ int smallest_ch; /* character with min code */
+ struct name * name; /* so g->name->grouping == g */
+ int line_number;
+};
+
+struct node {
+
+ struct node * next;
+ struct node * left;
+ struct node * aux; /* used in setlimit */
+ struct among * among; /* used in among */
+ struct node * right;
+ int type;
+ int mode;
+ struct node * AE;
+ struct name * name;
+ symbol * literalstring;
+ int number;
+ int line_number;
+ int amongvar_needed; /* used in routine definitions */
+};
+
+enum name_types {
+
+ t_size = 6,
+
+ t_string = 0, t_boolean = 1, t_integer = 2, t_routine = 3, t_external = 4,
+ t_grouping = 5
+
+/* If this list is extended, adjust wvn in generator.c */
+};
+
+/* In name_count[i] below, remember that
+ type is
+ ----+----
+ 0 | string
+ 1 | boolean
+ 2 | integer
+ 3 | routine
+ 4 | external
+ 5 | grouping
+*/
+
+struct analyser {
+
+ struct tokeniser * tokeniser;
+ struct node * nodes;
+ struct name * names;
+ struct literalstring * literalstrings;
+ int mode;
+ byte modifyable; /* false inside reverse(...) */
+ struct node * program;
+ struct node * program_end;
+ int name_count[t_size]; /* name_count[i] counts the number of names of type i */
+ struct among * amongs;
+ struct among * amongs_end;
+ int among_count;
+ int amongvar_needed; /* used in reading routine definitions */
+ struct grouping * groupings;
+ struct grouping * groupings_end;
+ struct node * substring; /* pending 'substring' in current routine definition */
+ enc encoding;
+ byte int_limits_used; /* are maxint or minint used? */
+};
+
+enum analyser_modes {
+
+ m_forward = 0, m_backward /*, m_integer */
+
+};
+
+extern void print_program(struct analyser * a);
+extern struct analyser * create_analyser(struct tokeniser * t);
+extern void close_analyser(struct analyser * a);
+
+extern void read_program(struct analyser * a);
+
+struct generator {
+
+ struct analyser * analyser;
+ struct options * options;
+ int unreachable; /* 0 if code can be reached, 1 if current code
+ * is unreachable. */
+ int var_number; /* Number of next variable to use. */
+ struct str * outbuf; /* temporary str to store output */
+ struct str * declarations; /* str storing variable declarations */
+ int next_label;
+#ifndef DISABLE_PYTHON
+ int max_label;
+#endif
+ int margin;
+
+ /* if > 0, keep_count to restore in case of a failure;
+ * if < 0, the negated keep_count for the limit to restore in case of
+ * failure. */
+ int failure_keep_count;
+#if !defined(DISABLE_JAVA) && !defined(DISABLE_JS) && !defined(DISABLE_PYTHON) && !defined(DISABLE_CSHARP)
+ struct str * failure_str; /* This is used by some generators instead of failure_keep_count */
+#endif
+
+ int label_used; /* Keep track of whether the failure label is used. */
+ int failure_label;
+ int debug_count;
+ int copy_from_count; /* count of calls to copy_from() */
+
+ const char * S[10]; /* strings */
+ symbol * B[10]; /* blocks */
+ int I[10]; /* integers */
+ struct name * V[5]; /* variables */
+ symbol * L[5]; /* literals, used in formatted write */
+
+ int line_count; /* counts number of lines output */
+ int line_labelled; /* in ISO C, will need extra ';' if it is a block end */
+ int literalstring_count;
+ int keep_count; /* used to number keep/restore pairs to avoid compiler warnings
+ about shadowed variables */
+};
+
+/* Special values for failure_label in struct generator. */
+enum special_labels {
+ x_return = -1
+};
+
+struct options {
+
+ /* for the command line: */
+
+ const char * output_file;
+ const char * name;
+ FILE * output_src;
+ FILE * output_h;
+ byte syntax_tree;
+ byte comments;
+ enc encoding;
+ enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS, LANG_CSHARP, LANG_PASCAL, LANG_PYTHON, LANG_JAVASCRIPT, LANG_RUST, LANG_GO } make_lang;
+ const char * externals_prefix;
+ const char * variables_prefix;
+ const char * runtime_path;
+ const char * parent_class_name;
+ const char * package;
+ const char * go_snowball_runtime;
+ const char * string_class;
+ const char * among_class;
+ struct include * includes;
+ struct include * includes_end;
+};
+
+/* Generator functions common to several backends. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o);
+extern void close_generator(struct generator * g);
+
+extern void write_char(struct generator * g, int ch);
+extern void write_newline(struct generator * g);
+extern void write_string(struct generator * g, const char * s);
+extern void write_int(struct generator * g, int i);
+extern void write_b(struct generator * g, symbol * b);
+extern void write_str(struct generator * g, struct str * str);
+
+extern void write_comment_content(struct generator * g, struct node * p);
+extern void write_generated_comment_content(struct generator * g);
+extern void write_start_comment(struct generator * g,
+ const char * comment_start,
+ const char * comment_end);
+
+extern int K_needed(struct generator * g, struct node * p);
+extern int repeat_restore(struct generator * g, struct node * p);
+
+/* Generator for C code. */
+extern void generate_program_c(struct generator * g);
+
+#ifndef DISABLE_JAVA
+/* Generator for Java code. */
+extern void generate_program_java(struct generator * g);
+#endif
+
+#ifndef DISABLE_CSHARP
+/* Generator for C# code. */
+extern void generate_program_csharp(struct generator * g);
+#endif
+
+#ifndef DISABLE_PASCAL
+extern void generate_program_pascal(struct generator * g);
+#endif
+
+#ifndef DISABLE_PYTHON
+/* Generator for Python code. */
+extern void generate_program_python(struct generator * g);
+#endif
+
+#ifndef DISABLE_JS
+extern void generate_program_js(struct generator * g);
+#endif
+
+#ifndef DISABLE_RUST
+extern void generate_program_rust(struct generator * g);
+#endif
+
+#ifndef DISABLE_GO
+extern void generate_program_go(struct generator * g);
+#endif
diff --git a/contrib/snowball/compiler/space.c b/contrib/snowball/compiler/space.c
new file mode 100644
index 0000000..5b05876
--- /dev/null
+++ b/contrib/snowball/compiler/space.c
@@ -0,0 +1,287 @@
+
+#include <stdio.h> /* for printf */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* memmove */
+
+#include "header.h"
+
+#define HEAD 2*sizeof(int)
+#define EXTENDER 40
+
+
+/* This modules provides a simple mechanism for arbitrary length writable
+ strings, called 'blocks'. They are 'symbol *' items rather than 'char *'
+ items however.
+
+ The calls are:
+
+ symbol * b = create_b(n);
+ - create an empty block b with room for n symbols
+ b = increase_capacity(b, n);
+ - increase the capacity of block b by n symbols (b may change)
+ b2 = copy_b(b)
+ - copy block b into b2
+ lose_b(b);
+ - lose block b
+ b = move_to_b(b, n, p);
+ - set the data in b to be the n symbols at address p
+ b = add_to_b(b, n, p);
+ - add the n symbols at address p to the end of the data in b
+ SIZE(b)
+ - is the number of symbols in b
+ For example:
+
+ symbol * b = create_b(0);
+ { int i;
+ char p[10];
+ for (i = 0; i < 100; i++) {
+ sprintf(p, " %d", i);
+ add_s_to_b(b, p);
+ }
+ }
+
+ and b contains " 0 1 2 ... 99" spaced out as symbols.
+*/
+
+/* For a block b, SIZE(b) is the number of symbols so far written into it,
+ CAPACITY(b) the total number it can contain, so SIZE(b) <= CAPACITY(b).
+ In fact blocks have 1 extra character over the promised capacity so
+ they can be zero terminated by 'b[SIZE(b)] = 0;' without fear of
+ overwriting.
+*/
+
+extern symbol * create_b(int n) {
+ symbol * p = (symbol *) (HEAD + (char *) MALLOC(HEAD + (n + 1) * sizeof(symbol)));
+ CAPACITY(p) = n;
+ SIZE(p) = 0;
+ return p;
+}
+
+extern void report_b(FILE * out, const symbol * p) {
+ int i;
+ for (i = 0; i < SIZE(p); i++) {
+ if (p[i] > 255) {
+ printf("In report_b, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+ exit(1);
+ }
+ putc(p[i], out);
+ }
+}
+
+extern void output_str(FILE * outfile, struct str * str) {
+ report_b(outfile, str_data(str));
+}
+
+extern void lose_b(symbol * p) {
+ if (p == 0) return;
+ FREE((char *) p - HEAD);
+}
+
+extern symbol * increase_capacity(symbol * p, int n) {
+ symbol * q = create_b(CAPACITY(p) + n + EXTENDER);
+ memmove(q, p, CAPACITY(p) * sizeof(symbol));
+ SIZE(q) = SIZE(p);
+ lose_b(p); return q;
+}
+
+extern symbol * move_to_b(symbol * p, int n, const symbol * q) {
+ int x = n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ memmove(p, q, n * sizeof(symbol)); SIZE(p) = n; return p;
+}
+
+extern symbol * add_to_b(symbol * p, int n, const symbol * q) {
+ int x = SIZE(p) + n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ memmove(p + SIZE(p), q, n * sizeof(symbol)); SIZE(p) += n; return p;
+}
+
+extern symbol * copy_b(const symbol * p) {
+ int n = SIZE(p);
+ symbol * q = create_b(n);
+ move_to_b(q, n, p);
+ return q;
+}
+
+int space_count = 0;
+
+extern void * check_malloc(int n) {
+ space_count++;
+ return malloc(n);
+}
+
+extern void check_free(void * p) {
+ space_count--;
+ free(p);
+}
+
+/* To convert a block to a zero terminated string: */
+
+extern char * b_to_s(const symbol * p) {
+ int n = SIZE(p);
+ char * s = (char *)malloc(n + 1);
+ {
+ int i;
+ for (i = 0; i < n; i++) {
+ if (p[i] > 255) {
+ printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+ exit(1);
+ }
+ s[i] = (char)p[i];
+ }
+ }
+ s[n] = 0;
+ return s;
+}
+
+/* To add a zero terminated string to a block. If p = 0 the
+ block is created. */
+
+extern symbol * add_s_to_b(symbol * p, const char * s) {
+ int n = strlen(s);
+ int k;
+ if (p == 0) p = create_b(n);
+ k = SIZE(p);
+ {
+ int x = k + n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ }
+ {
+ int i;
+ for (i = 0; i < n; i++) p[i + k] = s[i];
+ }
+ SIZE(p) += n;
+ return p;
+}
+
+/* The next section defines string handling capabilities in terms
+ of the lower level block handling capabilities of space.c */
+/* -------------------------------------------------------------*/
+
+struct str {
+ symbol * data;
+};
+
+/* Create a new string. */
+extern struct str * str_new(void) {
+
+ struct str * output = (struct str *) malloc(sizeof(struct str));
+ output->data = create_b(0);
+ return output;
+}
+
+/* Delete a string. */
+extern void str_delete(struct str * str) {
+
+ lose_b(str->data);
+ free(str);
+}
+
+/* Append a str to this str. */
+extern void str_append(struct str * str, const struct str * add) {
+
+ symbol * q = add->data;
+ str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append a character to this str. */
+extern void str_append_ch(struct str * str, char add) {
+
+ symbol q[1];
+ q[0] = add;
+ str->data = add_to_b(str->data, 1, q);
+}
+
+/* Append a low level block to a str. */
+extern void str_append_b(struct str * str, const symbol * q) {
+
+ str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append the tail of a low level block to a str. */
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip) {
+ if (skip < 0 || skip >= SIZE(q)) return;
+
+ str->data = add_to_b(str->data, SIZE(q) - skip, q + skip);
+}
+
+/* Append a (char *, null terminated) string to a str. */
+extern void str_append_string(struct str * str, const char * s) {
+
+ str->data = add_s_to_b(str->data, s);
+}
+
+/* Append an integer to a str. */
+extern void str_append_int(struct str * str, int i) {
+
+ char s[30];
+ sprintf(s, "%d", i);
+ str_append_string(str, s);
+}
+
+/* Clear a string */
+extern void str_clear(struct str * str) {
+
+ SIZE(str->data) = 0;
+}
+
+/* Set a string */
+extern void str_assign(struct str * str, const char * s) {
+
+ str_clear(str);
+ str_append_string(str, s);
+}
+
+/* Copy a string. */
+extern struct str * str_copy(const struct str * old) {
+
+ struct str * newstr = str_new();
+ str_append(newstr, old);
+ return newstr;
+}
+
+/* Get the data stored in this str. */
+extern symbol * str_data(const struct str * str) {
+
+ return str->data;
+}
+
+/* Get the length of the str. */
+extern int str_len(const struct str * str) {
+
+ return SIZE(str->data);
+}
+
+/* Get the last character of the str.
+ *
+ * Or -1 if the string is empty.
+ */
+extern int str_back(const struct str *str) {
+ return SIZE(str->data) ? str->data[SIZE(str->data) - 1] : -1;
+}
+
+extern int get_utf8(const symbol * p, int * slot) {
+ int b0, b1;
+ b0 = *p++;
+ if (b0 < 0xC0) { /* 1100 0000 */
+ * slot = b0; return 1;
+ }
+ b1 = *p++;
+ if (b0 < 0xE0) { /* 1110 0000 */
+ * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
+ }
+ * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (*p & 0x3F); return 3;
+}
+
+extern int put_utf8(int ch, symbol * p) {
+ if (ch < 0x80) {
+ p[0] = ch; return 1;
+ }
+ if (ch < 0x800) {
+ p[0] = (ch >> 6) | 0xC0;
+ p[1] = (ch & 0x3F) | 0x80; return 2;
+ }
+ p[0] = (ch >> 12) | 0xE0;
+ p[1] = ((ch >> 6) & 0x3F) | 0x80;
+ p[2] = (ch & 0x3F) | 0x80; return 3;
+}
diff --git a/contrib/snowball/compiler/syswords.h b/contrib/snowball/compiler/syswords.h
new file mode 100644
index 0000000..c401d3e
--- /dev/null
+++ b/contrib/snowball/compiler/syswords.h
@@ -0,0 +1,86 @@
+static const struct system_word vocab[82+1] = {
+ { 0, (const byte *)"", 82+1},
+
+ { 1, (const byte *)"$", c_dollar },
+ { 1, (const byte *)"(", c_bra },
+ { 1, (const byte *)")", c_ket },
+ { 1, (const byte *)"*", c_multiply },
+ { 1, (const byte *)"+", c_plus },
+ { 1, (const byte *)"-", c_minus },
+ { 1, (const byte *)"/", c_divide },
+ { 1, (const byte *)"<", c_ls },
+ { 1, (const byte *)"=", c_assign },
+ { 1, (const byte *)">", c_gr },
+ { 1, (const byte *)"?", c_debug },
+ { 1, (const byte *)"[", c_leftslice },
+ { 1, (const byte *)"]", c_rightslice },
+ { 2, (const byte *)"!=", c_ne },
+ { 2, (const byte *)"*=", c_multiplyassign },
+ { 2, (const byte *)"+=", c_plusassign },
+ { 2, (const byte *)"-=", c_minusassign },
+ { 2, (const byte *)"->", c_sliceto },
+ { 2, (const byte *)"/*", c_comment2 },
+ { 2, (const byte *)"//", c_comment1 },
+ { 2, (const byte *)"/=", c_divideassign },
+ { 2, (const byte *)"<+", c_insert },
+ { 2, (const byte *)"<-", c_slicefrom },
+ { 2, (const byte *)"<=", c_le },
+ { 2, (const byte *)"==", c_eq },
+ { 2, (const byte *)"=>", c_assignto },
+ { 2, (const byte *)">=", c_ge },
+ { 2, (const byte *)"as", c_as },
+ { 2, (const byte *)"do", c_do },
+ { 2, (const byte *)"or", c_or },
+ { 3, (const byte *)"and", c_and },
+ { 3, (const byte *)"for", c_for },
+ { 3, (const byte *)"get", c_get },
+ { 3, (const byte *)"hex", c_hex },
+ { 3, (const byte *)"hop", c_hop },
+ { 3, (const byte *)"len", c_len },
+ { 3, (const byte *)"non", c_non },
+ { 3, (const byte *)"not", c_not },
+ { 3, (const byte *)"set", c_set },
+ { 3, (const byte *)"try", c_try },
+ { 4, (const byte *)"fail", c_fail },
+ { 4, (const byte *)"goto", c_goto },
+ { 4, (const byte *)"loop", c_loop },
+ { 4, (const byte *)"next", c_next },
+ { 4, (const byte *)"size", c_size },
+ { 4, (const byte *)"test", c_test },
+ { 4, (const byte *)"true", c_true },
+ { 5, (const byte *)"among", c_among },
+ { 5, (const byte *)"false", c_false },
+ { 5, (const byte *)"lenof", c_lenof },
+ { 5, (const byte *)"limit", c_limit },
+ { 5, (const byte *)"unset", c_unset },
+ { 6, (const byte *)"atmark", c_atmark },
+ { 6, (const byte *)"attach", c_attach },
+ { 6, (const byte *)"cursor", c_cursor },
+ { 6, (const byte *)"define", c_define },
+ { 6, (const byte *)"delete", c_delete },
+ { 6, (const byte *)"gopast", c_gopast },
+ { 6, (const byte *)"insert", c_insert },
+ { 6, (const byte *)"maxint", c_maxint },
+ { 6, (const byte *)"minint", c_minint },
+ { 6, (const byte *)"repeat", c_repeat },
+ { 6, (const byte *)"sizeof", c_sizeof },
+ { 6, (const byte *)"tomark", c_tomark },
+ { 7, (const byte *)"atleast", c_atleast },
+ { 7, (const byte *)"atlimit", c_atlimit },
+ { 7, (const byte *)"decimal", c_decimal },
+ { 7, (const byte *)"reverse", c_reverse },
+ { 7, (const byte *)"setmark", c_setmark },
+ { 7, (const byte *)"strings", c_strings },
+ { 7, (const byte *)"tolimit", c_tolimit },
+ { 8, (const byte *)"booleans", c_booleans },
+ { 8, (const byte *)"integers", c_integers },
+ { 8, (const byte *)"routines", c_routines },
+ { 8, (const byte *)"setlimit", c_setlimit },
+ { 9, (const byte *)"backwards", c_backwards },
+ { 9, (const byte *)"externals", c_externals },
+ { 9, (const byte *)"groupings", c_groupings },
+ { 9, (const byte *)"stringdef", c_stringdef },
+ { 9, (const byte *)"substring", c_substring },
+ { 12, (const byte *)"backwardmode", c_backwardmode },
+ { 13, (const byte *)"stringescapes", c_stringescapes }
+};
diff --git a/contrib/snowball/compiler/syswords2.h b/contrib/snowball/compiler/syswords2.h
new file mode 100644
index 0000000..e853e32
--- /dev/null
+++ b/contrib/snowball/compiler/syswords2.h
@@ -0,0 +1,13 @@
+ c_among = 4, c_and, c_as, c_assign, c_assignto, c_atleast,
+ c_atlimit, c_atmark, c_attach, c_backwardmode, c_backwards,
+ c_booleans, c_bra, c_comment1, c_comment2, c_cursor, c_debug,
+ c_decimal, c_define, c_delete, c_divide, c_divideassign, c_do,
+ c_dollar, c_eq, c_externals, c_fail, c_false, c_for, c_ge, c_get,
+ c_gopast, c_goto, c_gr, c_groupings, c_hex, c_hop, c_insert,
+ c_integers, c_ket, c_le, c_leftslice, c_len, c_lenof, c_limit, c_loop,
+ c_ls, c_maxint, c_minint, c_minus, c_minusassign, c_multiply,
+ c_multiplyassign, c_ne, c_next, c_non, c_not, c_or, c_plus,
+ c_plusassign, c_repeat, c_reverse, c_rightslice, c_routines,
+ c_set, c_setlimit, c_setmark, c_size, c_sizeof, c_slicefrom,
+ c_sliceto, c_stringdef, c_stringescapes, c_strings, c_substring,
+ c_test, c_tolimit, c_tomark, c_true, c_try, c_unset,
diff --git a/contrib/snowball/compiler/tokeniser.c b/contrib/snowball/compiler/tokeniser.c
new file mode 100644
index 0000000..e6c6386
--- /dev/null
+++ b/contrib/snowball/compiler/tokeniser.c
@@ -0,0 +1,567 @@
+
+#include <stdio.h> /* stderr etc */
+#include <stdlib.h> /* malloc free */
+#include <string.h> /* strlen */
+#include <ctype.h> /* isalpha etc */
+#include "header.h"
+
+struct system_word {
+ int s_size; /* size of system word */
+ const byte * s; /* pointer to the system word */
+ int code; /* its internal code */
+};
+
+
+/* ASCII collating assumed in syswords.c */
+
+#include "syswords.h"
+
+#define INITIAL_INPUT_BUFFER_SIZE 8192
+
+static int hex_to_num(int ch);
+
+static int smaller(int a, int b) { return a < b ? a : b; }
+
+extern symbol * get_input(const char * filename) {
+ FILE * input = fopen(filename, "r");
+ if (input == 0) { return 0; }
+ {
+ symbol * u = create_b(INITIAL_INPUT_BUFFER_SIZE);
+ int size = 0;
+ while (true) {
+ int ch = getc(input);
+ if (ch == EOF) break;
+ if (size >= CAPACITY(u)) u = increase_capacity(u, size);
+ u[size++] = ch;
+ }
+ fclose(input);
+ SIZE(u) = size;
+ return u;
+ }
+}
+
+static void error(struct tokeniser * t, const char * s1, int n, symbol * p, const char * s2) {
+ if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); }
+ fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+ if (s1) fprintf(stderr, "%s", s1);
+ if (p) {
+ int i;
+ for (i = 0; i < n; i++) fprintf(stderr, "%c", p[i]);
+ }
+ if (s2) fprintf(stderr, "%s", s2);
+ fprintf(stderr, "\n");
+ t->error_count++;
+}
+
+static void error1(struct tokeniser * t, const char * s) {
+ error(t, s, 0,0, 0);
+}
+
+static void error2(struct tokeniser * t, const char * s) {
+ error(t, "unexpected end of text after ", 0,0, s);
+}
+
+static int compare_words(int m, symbol * p, int n, const byte * q) {
+ if (m != n) return m - n;
+ {
+ int i; for (i = 0; i < n; i++) {
+ int diff = p[i] - q[i];
+ if (diff) return diff;
+ }
+ }
+ return 0;
+}
+
+static int find_word(int n, symbol * p) {
+ int i = 0; int j = vocab->code;
+ do {
+ int k = i + (j - i)/2;
+ const struct system_word * w = vocab + k;
+ int diff = compare_words(n, p, w->s_size, w->s);
+ if (diff == 0) return w->code;
+ if (diff < 0) j = k; else i = k;
+ } while (j - i != 1);
+ return -1;
+}
+
+static int get_number(int n, symbol * p) {
+ int x = 0;
+ int i; for (i = 0; i < n; i++) x = 10*x + p[i] - '0';
+ return x;
+}
+
+static int eq_s(struct tokeniser * t, const char * s) {
+ int l = strlen(s);
+ if (SIZE(t->p) - t->c < l) return false;
+ {
+ int i;
+ for (i = 0; i < l; i++) if (t->p[t->c + i] != s[i]) return false;
+ }
+ t->c += l; return true;
+}
+
+static int white_space(struct tokeniser * t, int ch) {
+ switch (ch) {
+ case '\n':
+ t->line_number++;
+ /* fall through */
+ case '\r':
+ case '\t':
+ case ' ':
+ return true;
+ }
+ return false;
+}
+
+static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) {
+ struct m_pair * q;
+ for (q = t->m_pairs; q; q = q->next) {
+ symbol * name = q->name;
+ if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value;
+ }
+ return 0;
+}
+
+static int read_literal_string(struct tokeniser * t, int c) {
+ symbol * p = t->p;
+ int ch;
+ SIZE(t->b) = 0;
+ while (true) {
+ if (c >= SIZE(p)) { error2(t, "'"); return c; }
+ ch = p[c];
+ if (ch == '\n') { error1(t, "string not terminated"); return c; }
+ c++;
+ if (ch == t->m_start) {
+ /* Inside insert characters. */
+ int c0 = c;
+ int newlines = false; /* no newlines as yet */
+ int black_found = false; /* no printing chars as yet */
+ while (true) {
+ if (c >= SIZE(p)) { error2(t, "'"); return c; }
+ ch = p[c]; c++;
+ if (ch == t->m_end) break;
+ if (!white_space(t, ch)) black_found = true;
+ if (ch == '\n') newlines = true;
+ if (newlines && black_found) {
+ error1(t, "string not terminated");
+ return c;
+ }
+ }
+ if (!newlines) {
+ int n = c - c0 - 1; /* macro size */
+ int firstch = p[c0];
+ symbol * q = find_in_m(t, n, p + c0);
+ if (q == 0) {
+ if (n == 1 && (firstch == '\'' || firstch == t->m_start))
+ t->b = add_to_b(t->b, 1, p + c0);
+ else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') {
+ int codepoint = 0;
+ int x;
+ if (t->uplusmode == UPLUS_DEFINED) {
+ /* See if found with xxxx upper-cased. */
+ symbol * uc = create_b(n);
+ int i;
+ for (i = 0; i != n; ++i) {
+ uc[i] = toupper(p[c0 + i]);
+ }
+ q = find_in_m(t, n, uc);
+ lose_b(uc);
+ if (q != 0) {
+ t->b = add_to_b(t->b, SIZE(q), q);
+ continue;
+ }
+ error1(t, "Some U+xxxx stringdefs seen but not this one");
+ } else {
+ t->uplusmode = UPLUS_UNICODE;
+ }
+ for (x = c0 + 2; x != c - 1; ++x) {
+ int hex = hex_to_num(p[x]);
+ if (hex < 0) {
+ error1(t, "Bad hex digit following U+");
+ break;
+ }
+ codepoint = (codepoint << 4) | hex;
+ }
+ if (t->encoding == ENC_UTF8) {
+ if (codepoint < 0 || codepoint > 0x01ffff) {
+ error1(t, "character values exceed 0x01ffff");
+ }
+ /* Ensure there's enough space for a max length
+ * UTF-8 sequence. */
+ if (CAPACITY(t->b) < SIZE(t->b) + 3) {
+ t->b = increase_capacity(t->b, 3);
+ }
+ SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b));
+ } else {
+ symbol sym;
+ if (t->encoding == ENC_SINGLEBYTE) {
+ /* Only ISO-8859-1 is handled this way - for
+ * other single-byte character sets you need
+ * stringdef all the U+xxxx codes you use
+ * like - e.g.:
+ *
+ * stringdef U+0171 hex 'FB'
+ */
+ if (codepoint < 0 || codepoint > 0xff) {
+ error1(t, "character values exceed 256");
+ }
+ } else {
+ if (codepoint < 0 || codepoint > 0xffff) {
+ error1(t, "character values exceed 64K");
+ }
+ }
+ sym = codepoint;
+ t->b = add_to_b(t->b, 1, &sym);
+ }
+ } else
+ error(t, "string macro '", n, p + c0, "' undeclared");
+ } else
+ t->b = add_to_b(t->b, SIZE(q), q);
+ }
+ } else {
+ if (ch == '\'') return c;
+ if (ch < 0 || ch >= 0x80) {
+ if (t->encoding != ENC_WIDECHARS) {
+ /* We don't really want people using non-ASCII literal
+ * strings, but historically it's worked for single-byte
+ * and UTF-8 if the source encoding matches what the
+ * generated stemmer works in and it seems unfair to just
+ * suddenly make this a hard error.`
+ */
+ fprintf(stderr,
+ "%s:%d: warning: Non-ASCII literal strings aren't "
+ "portable - use stringdef instead\n",
+ t->file, t->line_number);
+ } else {
+ error1(t, "Non-ASCII literal strings aren't "
+ "portable - use stringdef instead");
+ }
+ }
+ t->b = add_to_b(t->b, 1, p + c - 1);
+ }
+ }
+}
+
+static int next_token(struct tokeniser * t) {
+ symbol * p = t->p;
+ int c = t->c;
+ int ch;
+ int code = -1;
+ while (true) {
+ if (c >= SIZE(p)) { t->c = c; return -1; }
+ ch = p[c];
+ if (white_space(t, ch)) { c++; continue; }
+ if (isalpha(ch)) {
+ int c0 = c;
+ while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++;
+ code = find_word(c - c0, p + c0);
+ if (code < 0 || t->token_disabled[code]) {
+ t->b = move_to_b(t->b, c - c0, p + c0);
+ code = c_name;
+ }
+ } else
+ if (isdigit(ch)) {
+ int c0 = c;
+ while (c < SIZE(p) && isdigit(p[c])) c++;
+ t->number = get_number(c - c0, p + c0);
+ code = c_number;
+ } else
+ if (ch == '\'') {
+ c = read_literal_string(t, c + 1);
+ code = c_literalstring;
+ } else
+ {
+ int lim = smaller(2, SIZE(p) - c);
+ int i;
+ for (i = lim; i > 0; i--) {
+ code = find_word(i, p + c);
+ if (code >= 0) { c += i; break; }
+ }
+ }
+ if (code >= 0) {
+ t->c = c;
+ return code;
+ }
+ error(t, "'", 1, p + c, "' unknown");
+ c++;
+ continue;
+ }
+}
+
+static int next_char(struct tokeniser * t) {
+ if (t->c >= SIZE(t->p)) return -1;
+ return t->p[t->c++];
+}
+
+static int next_real_char(struct tokeniser * t) {
+ while (true) {
+ int ch = next_char(t);
+ if (!white_space(t, ch)) return ch;
+ }
+}
+
+static void read_chars(struct tokeniser * t) {
+ int ch = next_real_char(t);
+ if (ch < 0) { error2(t, "stringdef"); return; }
+ {
+ int c0 = t->c-1;
+ while (true) {
+ ch = next_char(t);
+ if (white_space(t, ch) || ch < 0) break;
+ }
+ t->b2 = move_to_b(t->b2, t->c - c0 - 1, t->p + c0);
+ }
+}
+
+static int decimal_to_num(int ch) {
+ if ('0' <= ch && ch <= '9') return ch - '0';
+ return -1;
+}
+
+static int hex_to_num(int ch) {
+ if ('0' <= ch && ch <= '9') return ch - '0';
+ if ('a' <= ch && ch <= 'f') return ch - 'a' + 10;
+ if ('A' <= ch && ch <= 'F') return ch - 'A' + 10;
+ return -1;
+}
+
+static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
+ int c = 0; int d = 0;
+ while (true) {
+ while (c < SIZE(p) && p[c] == ' ') c++;
+ if (c == SIZE(p)) break;
+ {
+ int number = 0;
+ while (c != SIZE(p)) {
+ int ch = p[c];
+ if (ch == ' ') break;
+ if (base == 10) {
+ ch = decimal_to_num(ch);
+ if (ch < 0) {
+ error1(t, "decimal string contains non-digits");
+ return;
+ }
+ } else {
+ ch = hex_to_num(ch);
+ if (ch < 0) {
+ error1(t, "hex string contains non-hex characters");
+ return;
+ }
+ }
+ number = base * number + ch;
+ c++;
+ }
+ if (t->encoding == ENC_SINGLEBYTE) {
+ if (number < 0 || number > 0xff) {
+ error1(t, "character values exceed 256");
+ return;
+ }
+ } else {
+ if (number < 0 || number > 0xffff) {
+ error1(t, "character values exceed 64K");
+ return;
+ }
+ }
+ if (t->encoding == ENC_UTF8)
+ d += put_utf8(number, p + d);
+ else
+ p[d++] = number;
+ }
+ }
+ SIZE(p) = d;
+}
+
+extern int read_token(struct tokeniser * t) {
+ symbol * p = t->p;
+ int held = t->token_held;
+ t->token_held = false;
+ if (held) return t->token;
+ while (true) {
+ int code = next_token(t);
+ switch (code) {
+ case c_comment1: /* slash-slash comment */
+ while (t->c < SIZE(p) && p[t->c] != '\n') t->c++;
+ continue;
+ case c_comment2: /* slash-star comment */
+ while (true) {
+ if (t->c >= SIZE(p)) {
+ error1(t, "/* comment not terminated");
+ t->token = -1;
+ return -1;
+ }
+ if (p[t->c] == '\n') t->line_number++;
+ if (eq_s(t, "*/")) break;
+ t->c++;
+ }
+ continue;
+ case c_stringescapes: {
+ int ch1 = next_real_char(t);
+ int ch2 = next_real_char(t);
+ if (ch2 < 0) {
+ error2(t, "stringescapes");
+ continue;
+ }
+ if (ch1 == '\'') {
+ error1(t, "first stringescape cannot be '");
+ continue;
+ }
+ t->m_start = ch1;
+ t->m_end = ch2;
+ continue;
+ }
+ case c_stringdef: {
+ int base = 0;
+ read_chars(t);
+ code = read_token(t);
+ if (code == c_hex) { base = 16; code = read_token(t); } else
+ if (code == c_decimal) { base = 10; code = read_token(t); }
+ if (code != c_literalstring) {
+ error1(t, "string omitted after stringdef");
+ continue;
+ }
+ if (base > 0) convert_numeric_string(t, t->b, base);
+ { NEW(m_pair, q);
+ q->next = t->m_pairs;
+ q->name = copy_b(t->b2);
+ q->value = copy_b(t->b);
+ t->m_pairs = q;
+ if (t->uplusmode != UPLUS_DEFINED &&
+ (SIZE(t->b2) >= 3 && t->b2[0] == 'U' && t->b2[1] == '+')) {
+ if (t->uplusmode == UPLUS_UNICODE) {
+ error1(t, "U+xxxx already used with implicit meaning");
+ } else {
+ t->uplusmode = UPLUS_DEFINED;
+ }
+ }
+ }
+ continue;
+ }
+ case c_get:
+ code = read_token(t);
+ if (code != c_literalstring) {
+ error1(t, "string omitted after get"); continue;
+ }
+ t->get_depth++;
+ if (t->get_depth > 10) {
+ fprintf(stderr, "get directives go 10 deep. Looping?\n");
+ exit(1);
+ }
+ {
+ NEW(input, q);
+ char * file = b_to_s(t->b);
+ symbol * u = get_input(file);
+ if (u == 0) {
+ struct include * r;
+ for (r = t->includes; r; r = r->next) {
+ symbol * b = copy_b(r->b);
+ b = add_to_b(b, SIZE(t->b), t->b);
+ free(file);
+ file = b_to_s(b);
+ u = get_input(file);
+ lose_b(b);
+ if (u != 0) break;
+ }
+ }
+ if (u == 0) {
+ error(t, "Can't get '", SIZE(t->b), t->b, "'");
+ exit(1);
+ }
+ memmove(q, t, sizeof(struct input));
+ t->next = q;
+ t->p = u;
+ t->c = 0;
+ t->file = file;
+ t->file_needs_freeing = true;
+ t->line_number = 1;
+ }
+ p = t->p;
+ continue;
+ case -1:
+ if (t->next) {
+ lose_b(p);
+ {
+ struct input * q = t->next;
+ memmove(t, q, sizeof(struct input)); p = t->p;
+ FREE(q);
+ }
+ t->get_depth--;
+ continue;
+ }
+ /* fall through */
+ default:
+ t->previous_token = t->token;
+ t->token = code;
+ return code;
+ }
+ }
+}
+
+extern const char * name_of_token(int code) {
+ int i;
+ for (i = 1; i < vocab->code; i++)
+ if ((vocab + i)->code == code) return (const char *)(vocab + i)->s;
+ switch (code) {
+ case c_mathassign: return "=";
+ case c_name: return "name";
+ case c_number: return "number";
+ case c_literalstring:return "literal";
+ case c_neg: return "neg";
+ case c_grouping: return "grouping";
+ case c_call: return "call";
+ case c_booltest: return "Boolean test";
+ case -2: return "start of text";
+ case -1: return "end of text";
+ default: return "?";
+ }
+}
+
+extern void disable_token(struct tokeniser * t, int code) {
+ t->token_disabled[code] = 1;
+}
+
+extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
+ NEW(tokeniser, t);
+ t->next = 0;
+ t->p = p;
+ t->c = 0;
+ t->file = file;
+ t->file_needs_freeing = false;
+ t->line_number = 1;
+ t->b = create_b(0);
+ t->b2 = create_b(0);
+ t->m_start = -1;
+ t->m_pairs = 0;
+ t->get_depth = 0;
+ t->error_count = 0;
+ t->token_held = false;
+ t->token = -2;
+ t->previous_token = -2;
+ t->uplusmode = UPLUS_NONE;
+ memset(t->token_disabled, 0, sizeof(t->token_disabled));
+ return t;
+}
+
+extern void close_tokeniser(struct tokeniser * t) {
+ lose_b(t->b);
+ lose_b(t->b2);
+ {
+ struct m_pair * q = t->m_pairs;
+ while (q) {
+ struct m_pair * q_next = q->next;
+ lose_b(q->name);
+ lose_b(q->value);
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct input * q = t->next;
+ while (q) {
+ struct input * q_next = q->next;
+ FREE(q);
+ q = q_next;
+ }
+ }
+ if (t->file_needs_freeing) free(t->file);
+ FREE(t);
+}
diff --git a/contrib/snowball/doc/TODO b/contrib/snowball/doc/TODO
new file mode 100644
index 0000000..0cfa1b1
--- /dev/null
+++ b/contrib/snowball/doc/TODO
@@ -0,0 +1,15 @@
+Things to do:
+
+ - Write documentation for how to use libstemmer (as opposed to how stemming
+ algorithms themselves work).
+ Currently, the documentation in the include/libstemmer.h header file is
+ pretty clear and comprehensive, but an overview document wouldn't go amiss.
+
+Things that would be nice to include at some point.
+
+ - Add version numbers to each stemming algorithm, and allow the interface to
+ request a specific version of the stemming algorithms. Default to providing
+ the latest version of the algorithm.
+ - Make mkmodules.pl generate the build system, instead of being called from it.
+ This would allow it to generate the list of modules to be built, so that it's
+ not necessary to change things in more than one place to add a new algorithm.
diff --git a/contrib/snowball/doc/libstemmer_c_README b/contrib/snowball/doc/libstemmer_c_README
new file mode 100644
index 0000000..9d3af8b
--- /dev/null
+++ b/contrib/snowball/doc/libstemmer_c_README
@@ -0,0 +1,125 @@
+libstemmer_c
+============
+
+This document pertains to the C version of the libstemmer distribution,
+available for download from:
+
+http://snowball.tartarus.org/dist/libstemmer_c.tgz
+
+
+Compiling the library
+=====================
+
+A simple makefile is provided for Unix style systems. On such systems, it
+should be possible simply to run "make", and the file "libstemmer.o"
+and the example program "stemwords" will be generated.
+
+If this doesn't work on your system, you need to write your own build
+system (or call the compiler directly). The files to compile are
+all contained in the "libstemmer", "runtime" and "src_c" directories,
+and the public header file is contained in the "include" directory.
+
+The library comes in two flavours; UTF-8 only, and UTF-8 plus other character
+sets. To use the utf-8 only flavour, compile "libstemmer_utf8.c" instead of
+"libstemmer.c".
+
+For convenience "mkinc.mak" is a makefile fragment listing the source files and
+header files used to compile the standard version of the library.
+"mkinc_utf8.mak" is a comparable makefile fragment listing just the source
+files for the UTF-8 only version of the library.
+
+
+Using the library
+=================
+
+The library provides a simple C API. Essentially, a new stemmer can
+be obtained by using "sb_stemmer_new". "sb_stemmer_stem" is then
+used to stem a word, "sb_stemmer_length" returns the stemmed
+length of the last word processed, and "sb_stemmer_delete" is
+used to delete a stemmer.
+
+Creating a stemmer is a relatively expensive operation - the expected
+usage pattern is that a new stemmer is created when needed, used
+to stem many words, and deleted after some time.
+
+Stemmers are re-entrant, but not threadsafe. In other words, if
+you wish to access the same stemmer object from multiple threads,
+you must ensure that all access is protected by a mutex or similar
+device.
+
+libstemmer does not currently incorporate any mechanism for caching the results
+of stemming operations. Such caching can greatly increase the performance of a
+stemmer under certain situations, so suitable patches will be considered for
+inclusion.
+
+The standard libstemmer sources contain an algorithm for each of the supported
+languages. The algorithm may be selected using the english name of the
+language, or using the 2 or 3 letter ISO 639 language codes. In addition,
+the traditional "Porter" stemming algorithm for english is included for
+backwards compatibility purposes, but we recommend use of the "English"
+stemmer in preference for new projects.
+
+(Some minor algorithms which are included only as curiosities in the snowball
+website, such as the Lovins stemmer and the Kraaij Pohlmann stemmer, are not
+included in the standard libstemmer sources. These are not really supported by
+the snowball project, but it would be possible to compile a modified libstemmer
+library containing these if desired.)
+
+
+The stemwords example
+=====================
+
+The stemwords example program allows you to run any of the stemmers
+compiled into the libstemmer library on a sample vocabulary. For
+details on how to use it, run it with the "-h" command line option.
+
+
+Using the library in a larger system
+====================================
+
+If you are incorporating the library into the build system of a larger
+program, I recommend copying the unpacked tarball without modification into
+a subdirectory of the sources of your program. Future versions of the
+library are intended to keep the same structure, so this will keep the
+work required to move to a new version of the library to a minimum.
+
+As an additional convenience, the list of source and header files used
+in the library is detailed in mkinc.mak - a file which is in a suitable
+format for inclusion by a Makefile. By including this file in your build
+system, you can link the snowball system into your program with a few
+extra rules.
+
+Using the library in a system using GNU autotools
+=================================================
+
+The libstemmer_c library can be integrated into a larger system which uses the
+GNU autotool framework (and in particular, automake and autoconf) as follows:
+
+1) Unpack libstemmer_c.tgz in the top level project directory so that there is
+ a libstemmer_c subdirectory of the top level directory of the project.
+
+2) Add a file "Makefile.am" to the unpacked libstemmer_c folder, containing:
+
+noinst_LTLIBRARIES = libstemmer.la
+include $(srcdir)/mkinc.mak
+noinst_HEADERS = $(snowball_headers)
+libstemmer_la_SOURCES = $(snowball_sources)
+
+(You may also need to add other lines to this, for example, if you are using
+compiler options which are not compatible with compiling the libstemmer
+library.)
+
+3) Add libstemmer_c to the AC_CONFIG_FILES declaration in the project's
+ configure.ac file.
+
+4) Add to the top level makefile the following lines (or modify existing
+ assignments to these variables appropriately):
+
+AUTOMAKE_OPTIONS = subdir-objects
+AM_CPPFLAGS = -I$(top_srcdir)/libstemmer_c/include
+SUBDIRS=libstemmer_c
+<name>_LIBADD = libstemmer_c/libstemmer.la
+
+(Where <name> is the name of the library or executable which links against
+libstemmer.)
+
diff --git a/contrib/snowball/doc/libstemmer_java_README b/contrib/snowball/doc/libstemmer_java_README
new file mode 100644
index 0000000..38b1af6
--- /dev/null
+++ b/contrib/snowball/doc/libstemmer_java_README
@@ -0,0 +1,40 @@
+libstemmer_java
+===============
+
+This document pertains to the Java version of the libstemmer distribution,
+available for download from:
+
+http://snowball.tartarus.org/dist/libstemmer_java.tgz
+
+
+Compiling the library
+=====================
+
+Simply run the java compiler on all the java source files under the java
+directory. For example, this can be done under unix by changing directory into
+the java directory, and running:
+
+ javac org/tartarus/snowball/*.java org/tartarus/snowball/ext/*.java
+
+This will compile the library and also an example program "TestApp" which
+provides a command line interface to the library.
+
+
+Using the library
+=================
+
+There is currently no formal documentation on the use of the Java version
+of the library. Additionally, its interface is not guaranteed to be
+stable.
+
+The best documentation of the library is the source of the TestApp example
+program.
+
+
+The TestApp example
+===================
+
+The TestApp example program allows you to run any of the stemmers
+compiled into the libstemmer library on a sample vocabulary. For
+details on how to use it, run it with no command line parameters.
+
diff --git a/contrib/snowball/include/libstemmer.h b/contrib/snowball/include/libstemmer.h
new file mode 100644
index 0000000..98051e1
--- /dev/null
+++ b/contrib/snowball/include/libstemmer.h
@@ -0,0 +1,78 @@
+
+/* Make header file work when included from C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sb_stemmer;
+typedef unsigned char sb_symbol;
+
+/* FIXME - should be able to get a version number for each stemming
+ * algorithm (which will be incremented each time the output changes). */
+
+/** Returns an array of the names of the available stemming algorithms.
+ * Note that these are the canonical names - aliases (ie, other names for
+ * the same algorithm) will not be included in the list.
+ * The list is terminated with a null pointer.
+ *
+ * The list must not be modified in any way.
+ */
+const char ** sb_stemmer_list(void);
+
+/** Create a new stemmer object, using the specified algorithm, for the
+ * specified character encoding.
+ *
+ * All algorithms will usually be available in UTF-8, but may also be
+ * available in other character encodings.
+ *
+ * @param algorithm The algorithm name. This is either the english
+ * name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
+ * language. Note that case is significant in this parameter - the
+ * value should be supplied in lower case.
+ *
+ * @param charenc The character encoding. NULL may be passed as
+ * this value, in which case UTF-8 encoding will be assumed. Otherwise,
+ * the argument may be one of "UTF_8", "ISO_8859_1" (i.e. Latin 1),
+ * "ISO_8859_2" (i.e. Latin 2) or "KOI8_R" (Russian). Note that case is
+ * significant in this parameter.
+ *
+ * @return NULL if the specified algorithm is not recognised, or the
+ * algorithm is not available for the requested encoding. Otherwise,
+ * returns a pointer to a newly created stemmer for the requested algorithm.
+ * The returned pointer must be deleted by calling sb_stemmer_delete().
+ *
+ * @note NULL will also be returned if an out of memory error occurs.
+ */
+struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
+
+/** Delete a stemmer object.
+ *
+ * This frees all resources allocated for the stemmer. After calling
+ * this function, the supplied stemmer may no longer be used in any way.
+ *
+ * It is safe to pass a null pointer to this function - this will have
+ * no effect.
+ */
+void sb_stemmer_delete(struct sb_stemmer * stemmer);
+
+/** Stem a word.
+ *
+ * The return value is owned by the stemmer - it must not be freed or
+ * modified, and it will become invalid when the stemmer is called again,
+ * or if the stemmer is freed.
+ *
+ * The length of the return value can be obtained using sb_stemmer_length().
+ *
+ * If an out-of-memory error occurs, this will return NULL.
+ */
+const sb_symbol * sb_stemmer_stem(struct sb_stemmer * stemmer,
+ const sb_symbol * word, int size);
+
+/** Get the length of the result of the last stemmed word.
+ * This should not be called before sb_stemmer_stem() has been called.
+ */
+int sb_stemmer_length(struct sb_stemmer * stemmer);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/contrib/snowball/libstemmer/libstemmer_c.in b/contrib/snowball/libstemmer/libstemmer_c.in
new file mode 100644
index 0000000..2aa918d
--- /dev/null
+++ b/contrib/snowball/libstemmer/libstemmer_c.in
@@ -0,0 +1,96 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "@MODULES_H@"
+
+struct sb_stemmer {
+ struct SN_env * (*create)(void);
+ void (*close)(struct SN_env *);
+ int (*stem)(struct SN_env *);
+
+ struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+ return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+ const struct stemmer_encoding * encoding;
+ if (charenc == NULL) return ENC_UTF_8;
+ for (encoding = encodings; encoding->name != 0; encoding++) {
+ if (strcmp(encoding->name, charenc) == 0) break;
+ }
+ if (encoding->name == NULL) return ENC_UNKNOWN;
+ return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+ stemmer_encoding_t enc;
+ const struct stemmer_modules * module;
+ struct sb_stemmer * stemmer;
+
+ enc = sb_getenc(charenc);
+ if (enc == ENC_UNKNOWN) return NULL;
+
+ for (module = modules; module->name != 0; module++) {
+ if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+ }
+ if (module->name == NULL) return NULL;
+
+ stemmer = (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+ if (stemmer == NULL) return NULL;
+
+ stemmer->create = module->create;
+ stemmer->close = module->close;
+ stemmer->stem = module->stem;
+
+ stemmer->env = stemmer->create();
+ if (stemmer->env == NULL)
+ {
+ sb_stemmer_delete(stemmer);
+ return NULL;
+ }
+
+ return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+ if (stemmer == 0) return;
+ if (stemmer->close) {
+ stemmer->close(stemmer->env);
+ stemmer->close = 0;
+ }
+ free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+ int ret;
+ if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+ {
+ stemmer->env->l = 0;
+ return NULL;
+ }
+ ret = stemmer->stem(stemmer->env);
+ if (ret < 0) return NULL;
+ stemmer->env->p[stemmer->env->l] = 0;
+ return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+ return stemmer->env->l;
+}
diff --git a/contrib/snowball/libstemmer/mkmodules.pl b/contrib/snowball/libstemmer/mkmodules.pl
new file mode 100755
index 0000000..dd66787
--- /dev/null
+++ b/contrib/snowball/libstemmer/mkmodules.pl
@@ -0,0 +1,267 @@
+#!/usr/bin/env perl
+use strict;
+use 5.006;
+use warnings;
+
+my $progname = $0;
+
+if (scalar @ARGV < 4 || scalar @ARGV > 5) {
+ print "Usage: $progname <outfile> <C source directory> <modules description file> <source list file> [<enc>]\n";
+ exit 1;
+}
+
+my $outname = shift(@ARGV);
+my $c_src_dir = shift(@ARGV);
+my $descfile = shift(@ARGV);
+my $srclistfile = shift(@ARGV);
+my $enc_only;
+my $extn = '';
+if (@ARGV) {
+ $enc_only = shift(@ARGV);
+ $extn = '_'.$enc_only;
+}
+
+my %aliases = ();
+my %algorithms = ();
+my %algorithm_encs = ();
+
+my %encs = ();
+
+sub addalgenc($$) {
+ my $alg = shift();
+ my $enc = shift();
+
+ if (defined $enc_only) {
+ my $norm_enc = lc $enc;
+ $norm_enc =~ s/_//g;
+ if ($norm_enc ne $enc_only) {
+ return;
+ }
+ }
+
+ if (defined $algorithm_encs{$alg}) {
+ my $hashref = $algorithm_encs{$alg};
+ $$hashref{$enc}=1;
+ } else {
+ my %newhash = ($enc => 1);
+ $algorithm_encs{$alg}=\%newhash;
+ }
+
+ $encs{$enc} = 1;
+}
+
+sub readinput()
+{
+ open DESCFILE, $descfile;
+ my $line;
+ while ($line = <DESCFILE>)
+ {
+ next if $line =~ m/^\s*#/;
+ next if $line =~ m/^\s*$/;
+ my ($alg,$encstr,$aliases) = split(/\s+/, $line);
+ my $enc;
+ my $alias;
+
+ $algorithms{$alg} = 1;
+ foreach $alias (split(/,/, $aliases)) {
+ foreach $enc (split(/,/, $encstr)) {
+ # print "$alias, $enc\n";
+ $aliases{$alias} = $alg;
+ addalgenc($alg, $enc);
+ }
+ }
+ }
+}
+
+sub printoutput()
+{
+ open (OUT, ">$outname") or die "Can't open output file `$outname': $!\n";
+
+ print OUT <<EOS;
+/* $outname: List of stemming modules.
+ *
+ * This file is generated by mkmodules.pl from a list of module names.
+ * Do not edit manually.
+ *
+EOS
+
+ my $line = " * Modules included by this file are: ";
+ print OUT $line;
+ my $linelen = length($line);
+
+ my $need_sep = 0;
+ my $lang;
+ my $enc;
+ my @algorithms = sort keys(%algorithms);
+ foreach $lang (@algorithms) {
+ if ($need_sep) {
+ if (($linelen + 2 + length($lang)) > 77) {
+ print OUT ",\n * ";
+ $linelen = 3;
+ } else {
+ print OUT ', ';
+ $linelen += 2;
+ }
+ }
+ print OUT $lang;
+ $linelen += length($lang);
+ $need_sep = 1;
+ }
+ print OUT "\n */\n\n";
+
+ foreach $lang (@algorithms) {
+ my $hashref = $algorithm_encs{$lang};
+ foreach $enc (sort keys (%$hashref)) {
+ print OUT "#include \"../$c_src_dir/stem_${enc}_$lang.h\"\n";
+ }
+ }
+
+ print OUT <<EOS;
+
+typedef enum {
+ ENC_UNKNOWN=0,
+EOS
+ my $neednl = 0;
+ for $enc (sort keys %encs) {
+ print OUT ",\n" if $neednl;
+ print OUT " ENC_${enc}";
+ $neednl = 1;
+ }
+ print OUT <<EOS;
+
+} stemmer_encoding_t;
+
+struct stemmer_encoding {
+ const char * name;
+ stemmer_encoding_t enc;
+};
+static const struct stemmer_encoding encodings[] = {
+EOS
+ for $enc (sort keys %encs) {
+ print OUT " {\"${enc}\", ENC_${enc}},\n";
+ }
+ print OUT <<EOS;
+ {0,ENC_UNKNOWN}
+};
+
+struct stemmer_modules {
+ const char * name;
+ stemmer_encoding_t enc;
+ struct SN_env * (*create)(void);
+ void (*close)(struct SN_env *);
+ int (*stem)(struct SN_env *);
+};
+static const struct stemmer_modules modules[] = {
+EOS
+
+ for $lang (sort keys %aliases) {
+ my $l = $aliases{$lang};
+ my $hashref = $algorithm_encs{$l};
+ my $enc;
+ foreach $enc (sort keys (%$hashref)) {
+ my $p = "${l}_${enc}";
+ print OUT " {\"$lang\", ENC_$enc, ${p}_create_env, ${p}_close_env, ${p}_stem},\n";
+ }
+ }
+
+ print OUT <<EOS;
+ {0,ENC_UNKNOWN,0,0,0}
+};
+EOS
+
+ print OUT <<EOS;
+static const char * algorithm_names[] = {
+EOS
+
+ for $lang (@algorithms) {
+ print OUT " \"$lang\", \n";
+ }
+
+ print OUT <<EOS;
+ 0
+};
+EOS
+ close OUT or die "Can't close ${outname}: $!\n";
+}
+
+sub printsrclist()
+{
+ open (OUT, ">$srclistfile") or die "Can't open output file `$srclistfile': $!\n";
+
+ print OUT <<EOS;
+# $srclistfile: List of stemming module source files
+#
+# This file is generated by mkmodules.pl from a list of module names.
+# Do not edit manually.
+#
+EOS
+
+ my $line = "# Modules included by this file are: ";
+ print OUT $line;
+ my $linelen = length($line);
+
+ my $need_sep = 0;
+ my $lang;
+ my $srcfile;
+ my $enc;
+ my @algorithms = sort keys(%algorithms);
+ foreach $lang (@algorithms) {
+ if ($need_sep) {
+ if (($linelen + 2 + length($lang)) > 77) {
+ print OUT ",\n# ";
+ $linelen = 3;
+ } else {
+ print OUT ', ';
+ $linelen += 2;
+ }
+ }
+ print OUT $lang;
+ $linelen += length($lang);
+ $need_sep = 1;
+ }
+
+ print OUT "\n\nsnowball_sources= \\\n";
+ for $lang (sort keys %aliases) {
+ my $hashref = $algorithm_encs{$lang};
+ my $enc;
+ foreach $enc (sort keys (%$hashref)) {
+ print OUT " src_c/stem_${enc}_${lang}.c \\\n";
+ }
+ }
+
+ $need_sep = 0;
+ for $srcfile ('runtime/api.c',
+ 'runtime/utilities.c',
+ "libstemmer/libstemmer${extn}.c") {
+ print OUT " \\\n" if $need_sep;
+ print OUT " $srcfile";
+ $need_sep = 1;
+ }
+
+ print OUT "\n\nsnowball_headers= \\\n";
+ for $lang (sort keys %aliases) {
+ my $hashref = $algorithm_encs{$lang};
+ my $enc;
+ foreach $enc (sort keys (%$hashref)) {
+ my $p = "${lang}_${enc}";
+ print OUT " src_c/stem_${enc}_${lang}.h \\\n";
+ }
+ }
+
+ $need_sep = 0;
+ for $srcfile ('include/libstemmer.h',
+ "libstemmer/modules${extn}.h",
+ 'runtime/api.h',
+ 'runtime/header.h') {
+ print OUT " \\\n" if $need_sep;
+ print OUT " $srcfile";
+ $need_sep = 1;
+ }
+
+ print OUT "\n\n";
+ close OUT or die "Can't close ${srclistfile}: $!\n";
+}
+
+readinput();
+printoutput();
+printsrclist();
diff --git a/contrib/snowball/libstemmer/modules.txt b/contrib/snowball/libstemmer/modules.txt
new file mode 100644
index 0000000..f6dcc7e
--- /dev/null
+++ b/contrib/snowball/libstemmer/modules.txt
@@ -0,0 +1,58 @@
+# This file contains a list of stemmers to include in the distribution.
+# The format is a set of space separated lines - on each line:
+# First item is name of stemmer.
+# Second item is comma separated list of character sets.
+# Third item is comma separated list of names to refer to the stemmer by.
+#
+# Lines starting with a #, or blank lines, are ignored.
+
+# List all the main algorithms for each language, in UTF-8, and also with
+# the most commonly used encoding.
+
+arabic UTF_8 arabic,ar,ara
+danish UTF_8 danish,da,dan
+dutch UTF_8 dutch,nl,dut,nld
+english UTF_8 english,en,eng
+finnish UTF_8 finnish,fi,fin
+french UTF_8 french,fr,fre,fra
+german UTF_8 german,de,ger,deu
+greek UTF_8 greek,el,gre,ell
+hindi UTF_8 hindi,hi,hin
+hungarian UTF_8 hungarian,hu,hun
+indonesian UTF_8 indonesian,id,ind
+italian UTF_8 italian,it,ita
+lithuanian UTF_8 lithuanian,lt,lit
+nepali UTF_8 nepali,ne,nep
+norwegian UTF_8 norwegian,no,nor
+portuguese UTF_8 portuguese,pt,por
+romanian UTF_8 romanian,ro,rum,ron
+russian UTF_8 russian,ru,rus
+serbian UTF_8 serbian,sr,srp
+spanish UTF_8 spanish,es,esl,spa
+swedish UTF_8 swedish,sv,swe
+tamil UTF_8 tamil,ta,tam
+turkish UTF_8 turkish,tr,tur
+
+# Also include the traditional porter algorithm for english.
+# The porter algorithm is included in the libstemmer distribution to assist
+# with backwards compatibility, but for new systems the english algorithm
+# should be used in preference.
+porter UTF_8 porter english
+
+# Some other stemmers in the snowball project are not included in the standard
+# distribution. To compile a libstemmer with them in, add them to this list,
+# and regenerate the distribution. (You will need a full source checkout for
+# this.) They are included in the snowball website as curiosities, but are not
+# intended for general use, and use of them is is not fully supported. These
+# algorithms are:
+#
+# german2 - This is a slight modification of the german stemmer.
+#german2 UTF_8,ISO_8859_1 german2 german
+#
+# kraaij_pohlmann - This is a different dutch stemmer.
+#kraaij_pohlmann UTF_8,ISO_8859_1 kraaij_pohlmann dutch
+#
+# lovins - This is an english stemmer, but fairly outdated, and
+# only really applicable to a restricted type of input text
+# (keywords in academic publications).
+#lovins UTF_8,ISO_8859_1 lovins english
diff --git a/contrib/snowball/libstemmer/modules_utf8.txt b/contrib/snowball/libstemmer/modules_utf8.txt
new file mode 100644
index 0000000..60a0e1d
--- /dev/null
+++ b/contrib/snowball/libstemmer/modules_utf8.txt
@@ -0,0 +1,49 @@
+# This file contains a list of stemmers to include in the distribution.
+# The format is a set of space separated lines - on each line:
+# First item is name of stemmer.
+# Second item is comma separated list of character sets.
+# Third item is comma separated list of names to refer to the stemmer by.
+#
+# Lines starting with a #, or blank lines, are ignored.
+
+# List all the main algorithms for each language, in UTF-8.
+
+danish UTF_8 danish,da,dan
+dutch UTF_8 dutch,nl,dut,nld
+english UTF_8 english,en,eng
+finnish UTF_8 finnish,fi,fin
+french UTF_8 french,fr,fre,fra
+german UTF_8 german,de,ger,deu
+hungarian UTF_8 hungarian,hu,hun
+italian UTF_8 italian,it,ita
+norwegian UTF_8 norwegian,no,nor
+portuguese UTF_8 portuguese,pt,por
+romanian UTF_8 romanian,ro,rum,ron
+russian UTF_8 russian,ru,rus
+spanish UTF_8 spanish,es,esl,spa
+swedish UTF_8 swedish,sv,swe
+turkish UTF_8 turkish,tr,tur
+
+# Also include the traditional porter algorithm for english.
+# The porter algorithm is included in the libstemmer distribution to assist
+# with backwards compatibility, but for new systems the english algorithm
+# should be used in preference.
+porter UTF_8 porter
+
+# Some other stemmers in the snowball project are not included in the standard
+# distribution. To compile a libstemmer with them in, add them to this list,
+# and regenerate the distribution. (You will need a full source checkout for
+# this.) They are included in the snowball website as curiosities, but are not
+# intended for general use, and use of them is is not fully supported. These
+# algorithms are:
+#
+# german2 - This is a slight modification of the german stemmer.
+#german2 UTF_8 german2
+#
+# kraaij_pohlmann - This is a different dutch stemmer.
+#kraaij_pohlmann UTF_8 kraaij_pohlmann
+#
+# lovins - This is an english stemmer, but fairly outdated, and
+# only really applicable to a restricted type of input text
+# (keywords in academic publications).
+#lovins UTF_8 lovins
diff --git a/contrib/snowball/runtime/api.c b/contrib/snowball/runtime/api.c
new file mode 100644
index 0000000..21ea5f2
--- /dev/null
+++ b/contrib/snowball/runtime/api.c
@@ -0,0 +1,58 @@
+
+#include <stdlib.h> /* for calloc, free */
+#include "header.h"
+
+extern struct SN_env * SN_create_env(int S_size, int I_size)
+{
+ struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
+ if (z == NULL) return NULL;
+ z->p = create_s();
+ if (z->p == NULL) goto error;
+ if (S_size)
+ {
+ int i;
+ z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
+ if (z->S == NULL) goto error;
+
+ for (i = 0; i < S_size; i++)
+ {
+ z->S[i] = create_s();
+ if (z->S[i] == NULL) goto error;
+ }
+ }
+
+ if (I_size)
+ {
+ z->I = (int *) calloc(I_size, sizeof(int));
+ if (z->I == NULL) goto error;
+ }
+
+ return z;
+error:
+ SN_close_env(z, S_size);
+ return NULL;
+}
+
+extern void SN_close_env(struct SN_env * z, int S_size)
+{
+ if (z == NULL) return;
+ if (S_size)
+ {
+ int i;
+ for (i = 0; i < S_size; i++)
+ {
+ lose_s(z->S[i]);
+ }
+ free(z->S);
+ }
+ free(z->I);
+ if (z->p) lose_s(z->p);
+ free(z);
+}
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
+{
+ int err = replace_s(z, 0, z->l, size, s, NULL);
+ z->c = 0;
+ return err;
+}
diff --git a/contrib/snowball/runtime/api.h b/contrib/snowball/runtime/api.h
new file mode 100644
index 0000000..ba9d1c1
--- /dev/null
+++ b/contrib/snowball/runtime/api.h
@@ -0,0 +1,32 @@
+
+typedef unsigned char symbol;
+
+/* Or replace 'char' above with 'short' for 16 bit characters.
+
+ More precisely, replace 'char' with whatever type guarantees the
+ character width you need. Note however that sizeof(symbol) should divide
+ HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
+ there is an alignment problem. In the unlikely event of a problem here,
+ consult Martin Porter.
+
+*/
+
+struct SN_env {
+ symbol * p;
+ int c; int l; int lb; int bra; int ket;
+ symbol * * S;
+ int * I;
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * SN_create_env(int S_size, int I_size);
+extern void SN_close_env(struct SN_env * z, int S_size);
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/contrib/snowball/runtime/header.h b/contrib/snowball/runtime/header.h
new file mode 100644
index 0000000..85a42fd
--- /dev/null
+++ b/contrib/snowball/runtime/header.h
@@ -0,0 +1,59 @@
+
+#include <limits.h>
+
+#include "api.h"
+
+#define MAXINT INT_MAX
+#define MININT INT_MIN
+
+#define HEAD 2*sizeof(int)
+
+#define SIZE(p) ((int *)(p))[-1]
+#define SET_SIZE(p, n) ((int *)(p))[-1] = n
+#define CAPACITY(p) ((int *)(p))[-2]
+
+struct among
+{ int s_size; /* number of chars in string */
+ const symbol * s; /* search string */
+ int substring_i;/* index to longest matching substring */
+ int result; /* result of the lookup */
+ int (* function)(struct SN_env *);
+};
+
+extern symbol * create_s(void);
+extern void lose_s(symbol * p);
+
+extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_v(struct SN_env * z, const symbol * p);
+extern int eq_v_b(struct SN_env * z, const symbol * p);
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size);
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
+
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
+extern int slice_from_v(struct SN_env * z, const symbol * p);
+extern int slice_del(struct SN_env * z);
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
+
+extern symbol * slice_to(struct SN_env * z, symbol * p);
+extern symbol * assign_to(struct SN_env * z, symbol * p);
+
+extern int len_utf8(const symbol * p);
+
+extern void debug(struct SN_env * z, int number, int line_count);
diff --git a/contrib/snowball/runtime/utilities.c b/contrib/snowball/runtime/utilities.c
new file mode 100644
index 0000000..1cfd1a1
--- /dev/null
+++ b/contrib/snowball/runtime/utilities.c
@@ -0,0 +1,503 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "header.h"
+
+#define CREATE_SIZE 1
+
+extern symbol * create_s(void) {
+ symbol * p;
+ void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
+ if (mem == NULL) return NULL;
+ p = (symbol *) (HEAD + (char *) mem);
+ CAPACITY(p) = CREATE_SIZE;
+ SET_SIZE(p, 0);
+ return p;
+}
+
+extern void lose_s(symbol * p) {
+ if (p == NULL) return;
+ free((char *) p - HEAD);
+}
+
+/*
+ new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
+ if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
+ position, or -1 on failure.
+
+ -- used to implement hop and next in the utf8 case.
+*/
+
+extern int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
+ int b;
+ if (n >= 0) {
+ for (; n > 0; n--) {
+ if (c >= l) return -1;
+ b = p[c++];
+ if (b >= 0xC0) { /* 1100 0000 */
+ while (c < l) {
+ b = p[c];
+ if (b >= 0xC0 || b < 0x80) break;
+ /* break unless b is 10------ */
+ c++;
+ }
+ }
+ }
+ } else {
+ for (; n < 0; n++) {
+ if (c <= lb) return -1;
+ b = p[--c];
+ if (b >= 0x80) { /* 1000 0000 */
+ while (c > lb) {
+ b = p[c];
+ if (b >= 0xC0) break; /* 1100 0000 */
+ c--;
+ }
+ }
+ }
+ }
+ return c;
+}
+
+/* Code for character groupings: utf8 cases */
+
+static int get_utf8(const symbol * p, int c, int l, int * slot) {
+ int b0, b1, b2;
+ if (c >= l) return 0;
+ b0 = p[c++];
+ if (b0 < 0xC0 || c == l) { /* 1100 0000 */
+ *slot = b0;
+ return 1;
+ }
+ b1 = p[c++] & 0x3F;
+ if (b0 < 0xE0 || c == l) { /* 1110 0000 */
+ *slot = (b0 & 0x1F) << 6 | b1;
+ return 2;
+ }
+ b2 = p[c++] & 0x3F;
+ if (b0 < 0xF0 || c == l) { /* 1111 0000 */
+ *slot = (b0 & 0xF) << 12 | b1 << 6 | b2;
+ return 3;
+ }
+ *slot = (b0 & 0xE) << 18 | b1 << 12 | b2 << 6 | (p[c] & 0x3F);
+ return 4;
+}
+
+static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
+ int a, b;
+ if (c <= lb) return 0;
+ b = p[--c];
+ if (b < 0x80 || c == lb) { /* 1000 0000 */
+ *slot = b;
+ return 1;
+ }
+ a = b & 0x3F;
+ b = p[--c];
+ if (b >= 0xC0 || c == lb) { /* 1100 0000 */
+ *slot = (b & 0x1F) << 6 | a;
+ return 2;
+ }
+ a |= (b & 0x3F) << 6;
+ b = p[--c];
+ if (b >= 0xE0 || c == lb) { /* 1110 0000 */
+ *slot = (b & 0xF) << 12 | a;
+ return 3;
+ }
+ *slot = (p[--c] & 0xE) << 18 | (b & 0x3F) << 12 | a;
+ return 4;
+}
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_utf8(z->p, z->c, z->l, & ch);
+ if (!w) return -1;
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c += w;
+ } while (repeat);
+ return 0;
+}
+
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+ if (!w) return -1;
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c -= w;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_utf8(z->p, z->c, z->l, & ch);
+ if (!w) return -1;
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return w;
+ z->c += w;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+ if (!w) return -1;
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return w;
+ z->c -= w;
+ } while (repeat);
+ return 0;
+}
+
+/* Code for character groupings: non-utf8 cases */
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c >= z->l) return -1;
+ ch = z->p[z->c];
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c++;
+ } while (repeat);
+ return 0;
+}
+
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c <= z->lb) return -1;
+ ch = z->p[z->c - 1];
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c--;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c >= z->l) return -1;
+ ch = z->p[z->c];
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return 1;
+ z->c++;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c <= z->lb) return -1;
+ ch = z->p[z->c - 1];
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return 1;
+ z->c--;
+ } while (repeat);
+ return 0;
+}
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
+ if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
+ z->c += s_size; return 1;
+}
+
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
+ if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
+ z->c -= s_size; return 1;
+}
+
+extern int eq_v(struct SN_env * z, const symbol * p) {
+ return eq_s(z, SIZE(p), p);
+}
+
+extern int eq_v_b(struct SN_env * z, const symbol * p) {
+ return eq_s_b(z, SIZE(p), p);
+}
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
+
+ int i = 0;
+ int j = v_size;
+
+ int c = z->c; int l = z->l;
+ const symbol * q = z->p + c;
+
+ const struct among * w;
+
+ int common_i = 0;
+ int common_j = 0;
+
+ int first_key_inspected = 0;
+
+ while (1) {
+ int k = i + ((j - i) >> 1);
+ int diff = 0;
+ int common = common_i < common_j ? common_i : common_j; /* smaller */
+ w = v + k;
+ {
+ int i2; for (i2 = common; i2 < w->s_size; i2++) {
+ if (c + common == l) { diff = -1; break; }
+ diff = q[common] - w->s[i2];
+ if (diff != 0) break;
+ common++;
+ }
+ }
+ if (diff < 0) {
+ j = k;
+ common_j = common;
+ } else {
+ i = k;
+ common_i = common;
+ }
+ if (j - i <= 1) {
+ if (i > 0) break; /* v->s has been inspected */
+ if (j == i) break; /* only one item in v */
+
+ /* - but now we need to go round once more to get
+ v->s inspected. This looks messy, but is actually
+ the optimal approach. */
+
+ if (first_key_inspected) break;
+ first_key_inspected = 1;
+ }
+ }
+ while (1) {
+ w = v + i;
+ if (common_i >= w->s_size) {
+ z->c = c + w->s_size;
+ if (w->function == 0) return w->result;
+ {
+ int res = w->function(z);
+ z->c = c + w->s_size;
+ if (res) return w->result;
+ }
+ }
+ i = w->substring_i;
+ if (i < 0) return 0;
+ }
+}
+
+/* find_among_b is for backwards processing. Same comments apply */
+
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
+
+ int i = 0;
+ int j = v_size;
+
+ int c = z->c; int lb = z->lb;
+ const symbol * q = z->p + c - 1;
+
+ const struct among * w;
+
+ int common_i = 0;
+ int common_j = 0;
+
+ int first_key_inspected = 0;
+
+ while (1) {
+ int k = i + ((j - i) >> 1);
+ int diff = 0;
+ int common = common_i < common_j ? common_i : common_j;
+ w = v + k;
+ {
+ int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
+ if (c - common == lb) { diff = -1; break; }
+ diff = q[- common] - w->s[i2];
+ if (diff != 0) break;
+ common++;
+ }
+ }
+ if (diff < 0) { j = k; common_j = common; }
+ else { i = k; common_i = common; }
+ if (j - i <= 1) {
+ if (i > 0) break;
+ if (j == i) break;
+ if (first_key_inspected) break;
+ first_key_inspected = 1;
+ }
+ }
+ while (1) {
+ w = v + i;
+ if (common_i >= w->s_size) {
+ z->c = c - w->s_size;
+ if (w->function == 0) return w->result;
+ {
+ int res = w->function(z);
+ z->c = c - w->s_size;
+ if (res) return w->result;
+ }
+ }
+ i = w->substring_i;
+ if (i < 0) return 0;
+ }
+}
+
+
+/* Increase the size of the buffer pointed to by p to at least n symbols.
+ * If insufficient memory, returns NULL and frees the old buffer.
+ */
+static symbol * increase_size(symbol * p, int n) {
+ symbol * q;
+ int new_size = n + 20;
+ void * mem = realloc((char *) p - HEAD,
+ HEAD + (new_size + 1) * sizeof(symbol));
+ if (mem == NULL) {
+ lose_s(p);
+ return NULL;
+ }
+ q = (symbol *) (HEAD + (char *)mem);
+ CAPACITY(q) = new_size;
+ return q;
+}
+
+/* to replace symbols between c_bra and c_ket in z->p by the
+ s_size symbols at s.
+ Returns 0 on success, -1 on error.
+ Also, frees z->p (and sets it to NULL) on error.
+*/
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
+{
+ int adjustment;
+ int len;
+ if (z->p == NULL) {
+ z->p = create_s();
+ if (z->p == NULL) return -1;
+ }
+ adjustment = s_size - (c_ket - c_bra);
+ len = SIZE(z->p);
+ if (adjustment != 0) {
+ if (adjustment + len > CAPACITY(z->p)) {
+ z->p = increase_size(z->p, adjustment + len);
+ if (z->p == NULL) return -1;
+ }
+ memmove(z->p + c_ket + adjustment,
+ z->p + c_ket,
+ (len - c_ket) * sizeof(symbol));
+ SET_SIZE(z->p, adjustment + len);
+ z->l += adjustment;
+ if (z->c >= c_ket)
+ z->c += adjustment;
+ else if (z->c > c_bra)
+ z->c = c_bra;
+ }
+ if (s_size) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
+ if (adjptr != NULL)
+ *adjptr = adjustment;
+ return 0;
+}
+
+static int slice_check(struct SN_env * z) {
+
+ if (z->bra < 0 ||
+ z->bra > z->ket ||
+ z->ket > z->l ||
+ z->p == NULL ||
+ z->l > SIZE(z->p)) /* this line could be removed */
+ {
+#if 0
+ fprintf(stderr, "faulty slice operation:\n");
+ debug(z, -1, 0);
+#endif
+ return -1;
+ }
+ return 0;
+}
+
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
+ if (slice_check(z)) return -1;
+ return replace_s(z, z->bra, z->ket, s_size, s, NULL);
+}
+
+extern int slice_from_v(struct SN_env * z, const symbol * p) {
+ return slice_from_s(z, SIZE(p), p);
+}
+
+extern int slice_del(struct SN_env * z) {
+ return slice_from_s(z, 0, 0);
+}
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
+ int adjustment;
+ if (replace_s(z, bra, ket, s_size, s, &adjustment))
+ return -1;
+ if (bra <= z->bra) z->bra += adjustment;
+ if (bra <= z->ket) z->ket += adjustment;
+ return 0;
+}
+
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
+ return insert_s(z, bra, ket, SIZE(p), p);
+}
+
+extern symbol * slice_to(struct SN_env * z, symbol * p) {
+ if (slice_check(z)) {
+ lose_s(p);
+ return NULL;
+ }
+ {
+ int len = z->ket - z->bra;
+ if (CAPACITY(p) < len) {
+ p = increase_size(p, len);
+ if (p == NULL)
+ return NULL;
+ }
+ memmove(p, z->p + z->bra, len * sizeof(symbol));
+ SET_SIZE(p, len);
+ }
+ return p;
+}
+
+extern symbol * assign_to(struct SN_env * z, symbol * p) {
+ int len = z->l;
+ if (CAPACITY(p) < len) {
+ p = increase_size(p, len);
+ if (p == NULL)
+ return NULL;
+ }
+ memmove(p, z->p, len * sizeof(symbol));
+ SET_SIZE(p, len);
+ return p;
+}
+
+extern int len_utf8(const symbol * p) {
+ int size = SIZE(p);
+ int len = 0;
+ while (size--) {
+ symbol b = *p++;
+ if (b >= 0xC0 || b < 0x80) ++len;
+ }
+ return len;
+}
+
+#if 0
+extern void debug(struct SN_env * z, int number, int line_count) {
+ int i;
+ int limit = SIZE(z->p);
+ /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/
+ if (number >= 0) printf("%3d (line %4d): [%d]'", number, line_count,limit);
+ for (i = 0; i <= limit; i++) {
+ if (z->lb == i) printf("{");
+ if (z->bra == i) printf("[");
+ if (z->c == i) printf("|");
+ if (z->ket == i) printf("]");
+ if (z->l == i) printf("}");
+ if (i < limit)
+ { int ch = z->p[i];
+ if (ch == 0) ch = '#';
+ printf("%c", ch);
+ }
+ }
+ printf("'\n");
+}
+#endif
diff --git a/contrib/t1ha/CMakeLists.txt b/contrib/t1ha/CMakeLists.txt
new file mode 100644
index 0000000..c8de483
--- /dev/null
+++ b/contrib/t1ha/CMakeLists.txt
@@ -0,0 +1,6 @@
+SET(T1HASRC t1ha1.c
+ t1ha2.c)
+
+ADD_LIBRARY(rspamd-t1ha STATIC ${T1HASRC})
+SET_TARGET_PROPERTIES(rspamd-t1ha PROPERTIES VERSION ${RSPAMD_VERSION})
+ADD_DEFINITIONS("-DT1HA_USE_FAST_ONESHOT_READ=0")
diff --git a/contrib/t1ha/LICENSE b/contrib/t1ha/LICENSE
new file mode 100644
index 0000000..d02db65
--- /dev/null
+++ b/contrib/t1ha/LICENSE
@@ -0,0 +1,21 @@
+ Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ Fast Positive Hash.
+
+ Portions Copyright (c) 2010-2013 Leonid Yuriev <leo@yuriev.ru>,
+ The 1Hippeus project (t1h).
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+
+ 1. The origin of this software must not be misrepresented; you must not
+ claim that you wrote the original software. If you use this software
+ in a product, an acknowledgement in the product documentation would be
+ appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+ misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
diff --git a/contrib/t1ha/t1ha.h b/contrib/t1ha/t1ha.h
new file mode 100644
index 0000000..e5f2dad
--- /dev/null
+++ b/contrib/t1ha/t1ha.h
@@ -0,0 +1,321 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Всё будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#pragma once
+
+#ifndef __has_attribute
+#define __has_attribute(x) (0)
+#endif
+
+#ifndef __has_include
+#define __has_include(x) (0)
+#endif
+
+#ifndef __GNUC_PREREQ
+#if defined(__GNUC__) && defined(__GNUC_MINOR__)
+#define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+#else
+#define __GNUC_PREREQ(maj, min) 0
+#endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+#ifdef __clang__
+#define __CLANG_PREREQ(maj, min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+#else
+#define __CLANG_PREREQ(maj, min) (0)
+#endif
+#endif /* __CLANG_PREREQ */
+
+/*****************************************************************************/
+
+#ifdef _MSC_VER
+/* Avoid '16' bytes padding added after data member 't1ha_context::total'
+ * and other warnings from std-headers if warning-level > 3. */
+#pragma warning(push, 3)
+#endif
+
+#if defined(__cplusplus) && __cplusplus >= 201103L
+#include <climits>
+#include <cstddef>
+#include <cstdint>
+#else
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#endif
+
+/*****************************************************************************/
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 archs */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NETBSD__) || defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/*****************************************************************************/
+
+#ifndef __dll_export
+#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
+#if defined(__GNUC__) || __has_attribute(dllexport)
+#define __dll_export __attribute__((dllexport))
+#elif defined(_MSC_VER)
+#define __dll_export __declspec(dllexport)
+#else
+#define __dll_export
+#endif
+#elif defined(__GNUC__) || __has_attribute(visibility)
+#define __dll_export __attribute__((visibility("default")))
+#else
+#define __dll_export
+#endif
+#endif /* __dll_export */
+
+#ifndef __dll_import
+#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
+#if defined(__GNUC__) || __has_attribute(dllimport)
+#define __dll_import __attribute__((dllimport))
+#elif defined(_MSC_VER)
+#define __dll_import __declspec(dllimport)
+#else
+#define __dll_import
+#endif
+#else
+#define __dll_import
+#endif
+#endif /* __dll_import */
+
+#if defined(t1ha_EXPORTS)
+#define T1HA_API __dll_export
+#elif defined(t1ha_IMPORTS)
+#define T1HA_API __dll_import
+#else
+#define T1HA_API
+#endif /* T1HA_API */
+
+#define T1HA_ALIGN_PREFIX
+#define T1HA_ALIGN_SUFFIX
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef union T1HA_ALIGN_PREFIX t1ha_state256 {
+ uint8_t bytes[32];
+ uint32_t u32[8];
+ uint64_t u64[4];
+ struct {
+ uint64_t a, b, c, d;
+ } n;
+} t1ha_state256_t T1HA_ALIGN_SUFFIX;
+
+typedef struct t1ha_context {
+ t1ha_state256_t state;
+ t1ha_state256_t buffer;
+ size_t partial;
+ uint64_t total;
+} t1ha_context_t;
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+/******************************************************************************
+ *
+ * t1ha2 = 64 and 128-bit, SLIGHTLY MORE ATTENTION FOR QUALITY AND STRENGTH.
+ *
+ * - The recommended version of "Fast Positive Hash" with good quality
+ * for checksum, hash tables and fingerprinting.
+ * - Portable and extremely efficiency on modern 64-bit CPUs.
+ * Designed for 64-bit little-endian platforms,
+ * in other cases will runs slowly.
+ * - Great quality of hashing and still faster than other non-t1ha hashes.
+ * Provides streaming mode and 128-bit result.
+ *
+ * Note: Due performance reason 64- and 128-bit results are completely
+ * different each other, i.e. 64-bit result is NOT any part of 128-bit.
+ */
+
+/* The at-once variant with 64-bit result */
+T1HA_API uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed);
+
+/* The at-once variant with 128-bit result.
+ * Argument `extra_result` is NOT optional and MUST be valid.
+ * The high 64-bit part of 128-bit hash will be always unconditionally
+ * stored to the address given by `extra_result` argument. */
+T1HA_API uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
+ const void *__restrict data, size_t length,
+ uint64_t seed);
+
+/* The init/update/final trinity for streaming.
+ * Return 64 or 128-bit result depentently from `extra_result` argument. */
+T1HA_API void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y);
+T1HA_API void t1ha2_update(t1ha_context_t *__restrict ctx,
+ const void *__restrict data, size_t length);
+
+/* Argument `extra_result` is optional and MAY be NULL.
+ * - If `extra_result` is NOT NULL then the 128-bit hash will be calculated,
+ * and high 64-bit part of it will be stored to the address given
+ * by `extra_result` argument.
+ * - Otherwise the 64-bit hash will be calculated
+ * and returned from function directly.
+ *
+ * Note: Due performance reason 64- and 128-bit results are completely
+ * different each other, i.e. 64-bit result is NOT any part of 128-bit. */
+T1HA_API uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
+ uint64_t *__restrict extra_result /* optional */);
+
+/******************************************************************************
+ *
+ * t1ha1 = 64-bit, BASELINE FAST PORTABLE HASH:
+ *
+ * - Runs faster on 64-bit platforms in other cases may runs slowly.
+ * - Portable and stable, returns same 64-bit result
+ * on all architectures and CPUs.
+ * - Unfortunately it fails the "strict avalanche criteria",
+ * see test results at https://github.com/demerphq/smhasher.
+ *
+ * This flaw is insignificant for the t1ha1() purposes and imperceptible
+ * from a practical point of view.
+ * However, nowadays this issue has resolved in the next t1ha2(),
+ * that was initially planned to providing a bit more quality.
+ */
+
+/* The little-endian variant. */
+T1HA_API uint64_t t1ha1_le(const void *data, size_t length, uint64_t seed);
+
+/* The big-endian variant. */
+T1HA_API uint64_t t1ha1_be(const void *data, size_t length, uint64_t seed);
+
+/* The historical nicname for generic little-endian variant. */
+static __inline uint64_t t1ha(const void *data, size_t length, uint64_t seed) {
+ return t1ha1_le(data, length, seed);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/contrib/t1ha/t1ha1.c b/contrib/t1ha/t1ha1.c
new file mode 100644
index 0000000..6e25d37
--- /dev/null
+++ b/contrib/t1ha/t1ha1.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Всё будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#include "config.h"
+#include "t1ha_bits.h"
+
+/* xor-mul-xor mixer */
+static __inline uint64_t mix64(uint64_t v, uint64_t p) {
+ v *= p;
+ return v ^ rot64(v, 41);
+}
+
+static __inline uint64_t final_weak_avalanche(uint64_t a, uint64_t b) {
+ /* LY: for performance reason on a some not high-end CPUs
+ * I replaced the second mux64() operation by mix64().
+ * Unfortunately this approach fails the "strict avalanche criteria",
+ * see test results at https://github.com/demerphq/smhasher. */
+ return mux64(rot64(a + b, 17), prime_4) + mix64(a ^ b, prime_0);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA1_BODY(ENDIANNES, ALIGNESS) \
+ const uint64_t *v = (const uint64_t *)data; \
+ if (unlikely(len > 32)) { \
+ uint64_t c = rot64(len, 17) + seed; \
+ uint64_t d = len ^ rot64(seed, 17); \
+ const uint64_t *detent = \
+ (const uint64_t *)((const uint8_t *)data + len - 31); \
+ do { \
+ const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0); \
+ const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1); \
+ const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2); \
+ const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3); \
+ v += 4; \
+ prefetch(v); \
+ \
+ const uint64_t d02 = w0 ^ rot64(w2 + d, 17); \
+ const uint64_t c13 = w1 ^ rot64(w3 + c, 17); \
+ c += a ^ rot64(w0, 41); \
+ d -= b ^ rot64(w1, 31); \
+ a ^= prime_1 * (d02 + w3); \
+ b ^= prime_0 * (c13 + w2); \
+ } while (likely(v < detent)); \
+ \
+ a ^= prime_6 * (rot64(c, 17) + d); \
+ b ^= prime_5 * (c + rot64(d, 17)); \
+ len &= 31; \
+ } \
+ \
+ switch (len) { \
+ default: \
+ b += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_4); \
+ /* fall through */ \
+ case 24: \
+ case 23: \
+ case 22: \
+ case 21: \
+ case 20: \
+ case 19: \
+ case 18: \
+ case 17: \
+ a += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_3); \
+ /* fall through */ \
+ case 16: \
+ case 15: \
+ case 14: \
+ case 13: \
+ case 12: \
+ case 11: \
+ case 10: \
+ case 9: \
+ b += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_2); \
+ /* fall through */ \
+ case 8: \
+ case 7: \
+ case 6: \
+ case 5: \
+ case 4: \
+ case 3: \
+ case 2: \
+ case 1: \
+ a += mux64(tail64_##ENDIANNES##_##ALIGNESS(v, len), prime_1); \
+ /* fall through */ \
+ case 0: \
+ return final_weak_avalanche(a, b); \
+ }
+
+uint64_t t1ha1_le(const void *data, size_t len, uint64_t seed) {
+ uint64_t a = seed;
+ uint64_t b = len;
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA1_BODY(le, unaligned);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA1_BODY(le, unaligned);
+ } else {
+ T1HA1_BODY(le, aligned);
+ }
+#endif
+}
+
+uint64_t t1ha1_be(const void *data, size_t len, uint64_t seed) {
+ uint64_t a = seed;
+ uint64_t b = len;
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA1_BODY(be, unaligned);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA1_BODY(be, unaligned);
+ } else {
+ T1HA1_BODY(be, aligned);
+ }
+#endif
+}
diff --git a/contrib/t1ha/t1ha2.c b/contrib/t1ha/t1ha2.c
new file mode 100644
index 0000000..4cb5281
--- /dev/null
+++ b/contrib/t1ha/t1ha2.c
@@ -0,0 +1,326 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Всё будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#include "config.h"
+#include "t1ha_bits.h"
+
+static __always_inline void init_ab(t1ha_state256_t *s, uint64_t x,
+ uint64_t y) {
+ s->n.a = x;
+ s->n.b = y;
+}
+
+static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x,
+ uint64_t y) {
+ s->n.c = rot64(y, 23) + ~x;
+ s->n.d = ~y + rot64(x, 19);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA2_UPDATE(ENDIANNES, ALIGNESS, state, v) \
+ do { \
+ t1ha_state256_t *const s = state; \
+ const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0); \
+ const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1); \
+ const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2); \
+ const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3); \
+ \
+ const uint64_t d02 = w0 + rot64(w2 + s->n.d, 56); \
+ const uint64_t c13 = w1 + rot64(w3 + s->n.c, 19); \
+ s->n.c ^= s->n.a + rot64(w0, 57); \
+ s->n.d ^= s->n.b + rot64(w1, 38); \
+ s->n.b ^= prime_6 * (c13 + w2); \
+ s->n.a ^= prime_5 * (d02 + w3); \
+ } while (0)
+
+static __always_inline void squash(t1ha_state256_t *s) {
+ s->n.a ^= prime_6 * (s->n.c + rot64(s->n.d, 23));
+ s->n.b ^= prime_5 * (rot64(s->n.c, 19) + s->n.d);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA2_LOOP(ENDIANNES, ALIGNESS, state, data, len) \
+ do { \
+ const void *detent = (const uint8_t *)data + len - 31; \
+ do { \
+ const uint64_t *v = (const uint64_t *)data; \
+ data = (const uint64_t *)data + 4; \
+ prefetch(data); \
+ T1HA2_UPDATE(le, ALIGNESS, state, v); \
+ } while (likely(data < detent)); \
+ } while (0)
+
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, state, data, len) \
+ do { \
+ t1ha_state256_t *const s = state; \
+ const uint64_t *v = (const uint64_t *)data; \
+ switch (len) { \
+ default: \
+ mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_4); \
+ /* fall through */ \
+ case 24: \
+ case 23: \
+ case 22: \
+ case 21: \
+ case 20: \
+ case 19: \
+ case 18: \
+ case 17: \
+ mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_3); \
+ /* fall through */ \
+ case 16: \
+ case 15: \
+ case 14: \
+ case 13: \
+ case 12: \
+ case 11: \
+ case 10: \
+ case 9: \
+ mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_2); \
+ /* fall through */ \
+ case 8: \
+ case 7: \
+ case 6: \
+ case 5: \
+ case 4: \
+ case 3: \
+ case 2: \
+ case 1: \
+ mixup64(&s->n.b, &s->n.a, tail64_##ENDIANNES##_##ALIGNESS(v, len), \
+ prime_1); \
+ /* fall through */ \
+ case 0: \
+ return final64(s->n.a, s->n.b); \
+ } \
+ } while (0)
+
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, state, data, len) \
+ do { \
+ t1ha_state256_t *const s = state; \
+ const uint64_t *v = (const uint64_t *)data; \
+ switch (len) { \
+ default: \
+ mixup64(&s->n.a, &s->n.d, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_4); \
+ /* fall through */ \
+ case 24: \
+ case 23: \
+ case 22: \
+ case 21: \
+ case 20: \
+ case 19: \
+ case 18: \
+ case 17: \
+ mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_3); \
+ /* fall through */ \
+ case 16: \
+ case 15: \
+ case 14: \
+ case 13: \
+ case 12: \
+ case 11: \
+ case 10: \
+ case 9: \
+ mixup64(&s->n.c, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_2); \
+ /* fall through */ \
+ case 8: \
+ case 7: \
+ case 6: \
+ case 5: \
+ case 4: \
+ case 3: \
+ case 2: \
+ case 1: \
+ mixup64(&s->n.d, &s->n.c, tail64_##ENDIANNES##_##ALIGNESS(v, len), \
+ prime_1); \
+ /* fall through */ \
+ case 0: \
+ return final128(s->n.a, s->n.b, s->n.c, s->n.d, extra_result); \
+ } \
+ } while (0)
+
+static __always_inline uint64_t final128(uint64_t a, uint64_t b, uint64_t c,
+ uint64_t d, uint64_t *h) {
+ mixup64(&a, &b, rot64(c, 41) ^ d, prime_0);
+ mixup64(&b, &c, rot64(d, 23) ^ a, prime_6);
+ mixup64(&c, &d, rot64(a, 19) ^ b, prime_5);
+ mixup64(&d, &a, rot64(b, 31) ^ c, prime_4);
+ *h = c + d;
+ return a ^ b;
+}
+
+//------------------------------------------------------------------------------
+
+uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed) {
+ t1ha_state256_t state;
+ init_ab(&state, seed, length);
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ if (unlikely(length > 32)) {
+ init_cd(&state, seed, length);
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ squash(&state);
+ length &= 31;
+ }
+ T1HA2_TAIL_AB(le, unaligned, &state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ if (unlikely(length > 32)) {
+ init_cd(&state, seed, length);
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ squash(&state);
+ length &= 31;
+ }
+ T1HA2_TAIL_AB(le, unaligned, &state, data, length);
+ } else {
+ if (unlikely(length > 32)) {
+ init_cd(&state, seed, length);
+ T1HA2_LOOP(le, aligned, &state, data, length);
+ squash(&state);
+ length &= 31;
+ }
+ T1HA2_TAIL_AB(le, aligned, &state, data, length);
+ }
+#endif
+}
+
+uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
+ const void *__restrict data, size_t length,
+ uint64_t seed) {
+ t1ha_state256_t state;
+ init_ab(&state, seed, length);
+ init_cd(&state, seed, length);
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ if (unlikely(length > 32)) {
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ length &= 31;
+ }
+ T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ if (unlikely(length > 32)) {
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ length &= 31;
+ }
+ T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
+ } else {
+ if (unlikely(length > 32)) {
+ T1HA2_LOOP(le, aligned, &state, data, length);
+ length &= 31;
+ }
+ T1HA2_TAIL_ABCD(le, aligned, &state, data, length);
+ }
+#endif
+}
+
+//------------------------------------------------------------------------------
+
+void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y) {
+ init_ab(&ctx->state, seed_x, seed_y);
+ init_cd(&ctx->state, seed_x, seed_y);
+ ctx->partial = 0;
+ ctx->total = 0;
+}
+
+void t1ha2_update(t1ha_context_t *__restrict ctx, const void *__restrict data,
+ size_t length) {
+ ctx->total += length;
+
+ if (ctx->partial) {
+ const size_t left = 32 - ctx->partial;
+ const size_t chunk = (length >= left) ? left : length;
+ memcpy(ctx->buffer.bytes + ctx->partial, data, chunk);
+ ctx->partial += chunk;
+ if (ctx->partial < 32) {
+ assert(left >= length);
+ return;
+ }
+ ctx->partial = 0;
+ data = (const uint8_t *)data + chunk;
+ length -= chunk;
+ T1HA2_UPDATE(le, aligned, &ctx->state, ctx->buffer.u64);
+ }
+
+ if (length >= 32) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
+ } else {
+ T1HA2_LOOP(le, aligned, &ctx->state, data, length);
+ }
+#endif
+ length &= 31;
+ }
+
+ if (length)
+ memcpy(ctx->buffer.bytes, data, ctx->partial = length);
+}
+
+uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
+ uint64_t *__restrict extra_result) {
+ uint64_t bits = (ctx->total << 3) ^ (UINT64_C(1) << 63);
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
+ bits = bswap64(bits);
+#endif
+ t1ha2_update(ctx, &bits, 8);
+
+ if (likely(!extra_result)) {
+ squash(&ctx->state);
+ T1HA2_TAIL_AB(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
+ }
+
+ T1HA2_TAIL_ABCD(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
+}
diff --git a/contrib/t1ha/t1ha_bits.h b/contrib/t1ha/t1ha_bits.h
new file mode 100644
index 0000000..5710d2d
--- /dev/null
+++ b/contrib/t1ha/t1ha_bits.h
@@ -0,0 +1,1171 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Всё будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#pragma once
+
+#if defined(_MSC_VER)
+#pragma warning(disable : 4201) /* nameless struct/union */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif /* 1800 */
+#endif /* MSVC */
+
+#include "config.h"
+#include "t1ha.h"
+
+#ifndef T1HA_USE_FAST_ONESHOT_READ
+/* Define it to 1 for little bit faster code.
+ * Unfortunately this may triggering a false-positive alarms from Valgrind,
+ * AddressSanitizer and other similar tool.
+ * So, define it to 0 for calmness if doubt. */
+#define T1HA_USE_FAST_ONESHOT_READ 1
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+
+/*****************************************************************************/
+
+#include <assert.h> /* for assert() */
+#include <stdbool.h> /* for bool */
+#include <string.h> /* for memcpy() */
+
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
+ __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
+#error Unsupported byte order.
+#endif
+
+#define T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE 0
+#define T1HA_CONFIG_UNALIGNED_ACCESS__SLOW 1
+#define T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT 2
+
+#ifndef T1HA_CONFIG_UNALIGNED_ACCESS
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#elif defined(__ia32__)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#elif defined(__e2k__)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__SLOW
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#else
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+#endif
+#endif /* T1HA_CONFIG_UNALIGNED_ACCESS */
+
+#define ALIGNMENT_16 2
+#define ALIGNMENT_32 4
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define ALIGNMENT_64 8
+#else
+#define ALIGNMENT_64 4
+#endif
+
+#ifndef PAGESIZE
+#define PAGESIZE 4096
+#endif /* PAGESIZE */
+
+/***************************************************************************/
+
+#ifndef __has_builtin
+#define __has_builtin(x) (0)
+#endif
+
+#ifndef __has_warning
+#define __has_warning(x) (0)
+#endif
+
+#ifndef __has_feature
+#define __has_feature(x) (0)
+#endif
+
+#ifndef __has_extension
+#define __has_extension(x) (0)
+#endif
+
+#if __GNUC_PREREQ(4, 4) || defined(__clang__)
+
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif
+
+#if defined(__ia32__) && !defined(__cpuid_count)
+#include <cpuid.h>
+#endif
+
+#if defined(__e2k__)
+#include <e2kbuiltin.h>
+#endif
+
+#ifndef likely
+#define likely(cond) __builtin_expect(!!(cond), 1)
+#endif
+
+#ifndef unlikely
+#define unlikely(cond) __builtin_expect(!!(cond), 0)
+#endif
+
+#if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable)
+#define unreachable() __builtin_unreachable()
+#endif
+
+#define bswap64(v) __builtin_bswap64(v)
+#define bswap32(v) __builtin_bswap32(v)
+#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)
+#define bswap16(v) __builtin_bswap16(v)
+#endif
+
+#if !defined(__maybe_unused) && (__GNUC_PREREQ(4, 3) || __has_attribute(unused))
+#define __maybe_unused __attribute__((unused))
+#endif
+
+#if !defined(__always_inline) && \
+ (__GNUC_PREREQ(3, 2) || __has_attribute(always_inline))
+#define __always_inline __inline __attribute__((always_inline))
+#endif
+
+#if defined(__e2k__)
+
+#if __iset__ >= 3
+#define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b)
+#endif /* __iset__ >= 3 */
+
+#if __iset__ >= 5
+static __maybe_unused __always_inline unsigned
+e2k_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) {
+ *sum = base + addend;
+ return (unsigned)__builtin_e2k_addcd_c(base, addend, 0);
+}
+#define add64carry_first(base, addend, sum) \
+ e2k_add64carry_first(base, addend, sum)
+
+static __maybe_unused __always_inline unsigned
+e2k_add64carry_next(unsigned carry, uint64_t base, uint64_t addend,
+ uint64_t *sum) {
+ *sum = __builtin_e2k_addcd(base, addend, carry);
+ return (unsigned)__builtin_e2k_addcd_c(base, addend, carry);
+}
+#define add64carry_next(carry, base, addend, sum) \
+ e2k_add64carry_next(carry, base, addend, sum)
+
+static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry,
+ uint64_t base,
+ uint64_t addend,
+ uint64_t *sum) {
+ *sum = __builtin_e2k_addcd(base, addend, carry);
+}
+#define add64carry_last(carry, base, addend, sum) \
+ e2k_add64carry_last(carry, base, addend, sum)
+#endif /* __iset__ >= 5 */
+
+#define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr))
+#define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr))
+
+#endif /* __e2k__ Elbrus */
+
+#elif defined(_MSC_VER)
+
+#if _MSC_FULL_VER < 190024218 && defined(_M_IX86)
+#pragma message( \
+ "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24218 (Visual Studio 2015 Update 5) is required.")
+#endif
+#if _MSC_FULL_VER < 191025019
+#pragma message( \
+ "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.10.25019 (Visual Studio 2017) or newer.")
+#endif
+#if _MSC_FULL_VER < 180040629
+#error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required.
+#endif
+
+#pragma warning(push, 1)
+
+#include <intrin.h>
+#include <stdlib.h>
+#define likely(cond) (cond)
+#define unlikely(cond) (cond)
+#define unreachable() __assume(0)
+#define bswap64(v) _byteswap_uint64(v)
+#define bswap32(v) _byteswap_ulong(v)
+#define bswap16(v) _byteswap_ushort(v)
+#define rot64(v, s) _rotr64(v, s)
+#define rot32(v, s) _rotr(v, s)
+#define __always_inline __forceinline
+
+#if defined(_M_X64) || defined(_M_IA64)
+#pragma intrinsic(_umul128)
+#define mul_64x64_128(a, b, ph) _umul128(a, b, ph)
+#pragma intrinsic(_addcarry_u64)
+#define add64carry_first(base, addend, sum) _addcarry_u64(0, base, addend, sum)
+#define add64carry_next(carry, base, addend, sum) \
+ _addcarry_u64(carry, base, addend, sum)
+#define add64carry_last(carry, base, addend, sum) \
+ (void)_addcarry_u64(carry, base, addend, sum)
+#endif
+
+#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
+#pragma intrinsic(__umulh)
+#define mul_64x64_high(a, b) __umulh(a, b)
+#endif
+
+#if defined(_M_IX86)
+#pragma intrinsic(__emulu)
+#define mul_32x32_64(a, b) __emulu(a, b)
+
+#if _MSC_FULL_VER >= 190024231 /* LY: workaround for optimizer bug */
+#pragma intrinsic(_addcarry_u32)
+#define add32carry_first(base, addend, sum) _addcarry_u32(0, base, addend, sum)
+#define add32carry_next(carry, base, addend, sum) \
+ _addcarry_u32(carry, base, addend, sum)
+#define add32carry_last(carry, base, addend, sum) \
+ (void)_addcarry_u32(carry, base, addend, sum)
+
+static __forceinline char
+msvc32_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) {
+ uint32_t *const sum32 = (uint32_t *)sum;
+ const uint32_t base_32l = (uint32_t)base;
+ const uint32_t base_32h = (uint32_t)(base >> 32);
+ const uint32_t addend_32l = (uint32_t)addend;
+ const uint32_t addend_32h = (uint32_t)(addend >> 32);
+ return add32carry_next(add32carry_first(base_32l, addend_32l, sum32),
+ base_32h, addend_32h, sum32 + 1);
+}
+#define add64carry_first(base, addend, sum) \
+ msvc32_add64carry_first(base, addend, sum)
+
+static __forceinline char msvc32_add64carry_next(char carry, uint64_t base,
+ uint64_t addend,
+ uint64_t *sum) {
+ uint32_t *const sum32 = (uint32_t *)sum;
+ const uint32_t base_32l = (uint32_t)base;
+ const uint32_t base_32h = (uint32_t)(base >> 32);
+ const uint32_t addend_32l = (uint32_t)addend;
+ const uint32_t addend_32h = (uint32_t)(addend >> 32);
+ return add32carry_next(add32carry_next(carry, base_32l, addend_32l, sum32),
+ base_32h, addend_32h, sum32 + 1);
+}
+#define add64carry_next(carry, base, addend, sum) \
+ msvc32_add64carry_next(carry, base, addend, sum)
+
+static __forceinline void msvc32_add64carry_last(char carry, uint64_t base,
+ uint64_t addend,
+ uint64_t *sum) {
+ uint32_t *const sum32 = (uint32_t *)sum;
+ const uint32_t base_32l = (uint32_t)base;
+ const uint32_t base_32h = (uint32_t)(base >> 32);
+ const uint32_t addend_32l = (uint32_t)addend;
+ const uint32_t addend_32h = (uint32_t)(addend >> 32);
+ add32carry_last(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h,
+ addend_32h, sum32 + 1);
+}
+#define add64carry_last(carry, base, addend, sum) \
+ msvc32_add64carry_last(carry, base, addend, sum)
+#endif /* _MSC_FULL_VER >= 190024231 */
+
+#elif defined(_M_ARM)
+#define mul_32x32_64(a, b) _arm_umull(a, b)
+#endif
+
+#pragma warning(pop)
+#pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \
+ has been removed */
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for \
+ automatic inline expansion */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4702) /* unreachable code */
+#endif /* Compiler */
+
+#ifndef likely
+#define likely(cond) (cond)
+#endif
+#ifndef unlikely
+#define unlikely(cond) (cond)
+#endif
+#ifndef __maybe_unused
+#define __maybe_unused
+#endif
+#ifndef __always_inline
+#define __always_inline __inline
+#endif
+#ifndef unreachable
+#define unreachable() \
+ do { \
+ } while (1)
+#endif
+
+#ifndef bswap64
+#if defined(bswap_64)
+#define bswap64 bswap_64
+#elif defined(__bswap_64)
+#define bswap64 __bswap_64
+#else
+static __always_inline uint64_t bswap64(uint64_t v) {
+ return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
+ ((v << 24) & UINT64_C(0x0000ff0000000000)) |
+ ((v << 8) & UINT64_C(0x000000ff00000000)) |
+ ((v >> 8) & UINT64_C(0x00000000ff000000)) |
+ ((v >> 24) & UINT64_C(0x0000000000ff0000)) |
+ ((v >> 40) & UINT64_C(0x000000000000ff00));
+}
+#endif
+#endif /* bswap64 */
+
+#ifndef bswap32
+#if defined(bswap_32)
+#define bswap32 bswap_32
+#elif defined(__bswap_32)
+#define bswap32 __bswap_32
+#else
+static __always_inline uint32_t bswap32(uint32_t v) {
+ return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) |
+ ((v >> 8) & UINT32_C(0x0000ff00));
+}
+#endif
+#endif /* bswap32 */
+
+#ifndef bswap16
+#if defined(bswap_16)
+#define bswap16 bswap_16
+#elif defined(__bswap_16)
+#define bswap16 __bswap_16
+#else
+static __always_inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; }
+#endif
+#endif /* bswap16 */
+
+#ifndef read_unaligned
+#if defined(__GNUC__) || __has_attribute(packed)
+typedef struct {
+ uint8_t unaligned_8;
+ uint16_t unaligned_16;
+ uint32_t unaligned_32;
+ uint64_t unaligned_64;
+} __attribute__((packed)) t1ha_unaligned_proxy;
+#define read_unaligned(ptr, bits) \
+ (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \
+ t1ha_unaligned_proxy, unaligned_##bits))) \
+ ->unaligned_##bits)
+#elif defined(_MSC_VER)
+#pragma warning( \
+ disable : 4235) /* nonstandard extension used: '__unaligned' \
+ * keyword not supported on this architecture */
+#define read_unaligned(ptr, bits) (*(const __unaligned uint##bits##_t *)(ptr))
+#else
+#pragma pack(push, 1)
+typedef struct {
+ uint8_t unaligned_8;
+ uint16_t unaligned_16;
+ uint32_t unaligned_32;
+ uint64_t unaligned_64;
+} t1ha_unaligned_proxy;
+#pragma pack(pop)
+#define read_unaligned(ptr, bits) \
+ (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \
+ t1ha_unaligned_proxy, unaligned_##bits))) \
+ ->unaligned_##bits)
+#endif
+#endif /* read_unaligned */
+
+#ifndef read_aligned
+#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned)
+#define read_aligned(ptr, bits) \
+ (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits))
+#elif (__GNUC_PREREQ(3, 3) || __has_attribute(aligned)) && !defined(__clang__)
+#define read_aligned(ptr, bits) \
+ (*(const uint##bits##_t __attribute__((aligned(ALIGNMENT_##bits))) *)(ptr))
+#elif __has_attribute(assume_aligned)
+
+static __always_inline const
+ uint16_t *__attribute__((assume_aligned(ALIGNMENT_16)))
+ cast_aligned_16(const void *ptr) {
+ return (const uint16_t *)ptr;
+}
+static __always_inline const
+ uint32_t *__attribute__((assume_aligned(ALIGNMENT_32)))
+ cast_aligned_32(const void *ptr) {
+ return (const uint32_t *)ptr;
+}
+static __always_inline const
+ uint64_t *__attribute__((assume_aligned(ALIGNMENT_64)))
+ cast_aligned_64(const void *ptr) {
+ return (const uint64_t *)ptr;
+}
+
+#define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr))
+
+#elif defined(_MSC_VER)
+#define read_aligned(ptr, bits) \
+ (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr))
+#else
+#define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr))
+#endif
+#endif /* read_aligned */
+
+#ifndef prefetch
+#if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \
+ !defined(__ia32__)
+#define prefetch(ptr) __builtin_prefetch(ptr)
+#elif defined(_M_ARM64) || defined(_M_ARM)
+#define prefetch(ptr) __prefetch(ptr)
+#else
+#define prefetch(ptr) \
+ do { \
+ (void)(ptr); \
+ } while (0)
+#endif
+#endif /* prefetch */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if __has_warning("-Wtautological-pointer-compare")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wtautological-pointer-compare"
+#else
+#pragma warning disable "tautological-pointer-compare"
+#endif
+#endif /* -Wtautological-pointer-compare */
+
+/***************************************************************************/
+
+/*---------------------------------------------------------- Little Endian */
+
+#ifndef fetch16_le_aligned
+static __always_inline uint16_t fetch16_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_16 == 0);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_aligned(v, 16);
+#else
+ return bswap16(read_aligned(v, 16));
+#endif
+}
+#endif /* fetch16_le_aligned */
+
+#ifndef fetch16_le_unaligned
+static __always_inline uint16_t fetch16_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ const uint8_t *p = (const uint8_t *)v;
+ return p[0] | (uint16_t)p[1] << 8;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 16);
+#else
+ return bswap16(read_unaligned(v, 16));
+#endif
+}
+#endif /* fetch16_le_unaligned */
+
+#ifndef fetch32_le_aligned
+static __always_inline uint32_t fetch32_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_32 == 0);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_aligned(v, 32);
+#else
+ return bswap32(read_aligned(v, 32));
+#endif
+}
+#endif /* fetch32_le_aligned */
+
+#ifndef fetch32_le_unaligned
+static __always_inline uint32_t fetch32_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return fetch16_le_unaligned(v) |
+ (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 32);
+#else
+ return bswap32(read_unaligned(v, 32));
+#endif
+}
+#endif /* fetch32_le_unaligned */
+
+#ifndef fetch64_le_aligned
+static __always_inline uint64_t fetch64_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_64 == 0);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_aligned(v, 64);
+#else
+ return bswap64(read_aligned(v, 64));
+#endif
+}
+#endif /* fetch64_le_aligned */
+
+#ifndef fetch64_le_unaligned
+static __always_inline uint64_t fetch64_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return fetch32_le_unaligned(v) |
+ (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 64);
+#else
+ return bswap64(read_unaligned(v, 64));
+#endif
+}
+#endif /* fetch64_le_unaligned */
+
+static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) {
+ const uint8_t *const p = (const uint8_t *)v;
+#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__)
+ /* We can perform a 'oneshot' read, which is little bit faster. */
+ const unsigned shift = ((8 - tail) & 7) << 3;
+ return fetch64_le_aligned(p) & ((~UINT64_C(0)) >> shift);
+#else
+ uint64_t r = 0;
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ /* For most CPUs this code is better when not needed byte reordering. */
+ case 0:
+ return fetch64_le_aligned(p);
+ case 7:
+ r = (uint64_t)p[6] << 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 32;
+ /* fall through */
+ case 4:
+ return r + fetch32_le_aligned(p);
+ case 3:
+ r = (uint64_t)p[2] << 16;
+ /* fall through */
+ case 2:
+ return r + fetch16_le_aligned(p);
+ case 1:
+ return p[0];
+#else
+ case 0:
+ r = p[7] << 8;
+ /* fall through */
+ case 7:
+ r += p[6];
+ r <<= 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 8;
+ /* fall through */
+ case 4:
+ r += p[3];
+ r <<= 8;
+ /* fall through */
+ case 3:
+ r += p[2];
+ r <<= 8;
+ /* fall through */
+ case 2:
+ r += p[1];
+ r <<= 8;
+ /* fall through */
+ case 1:
+ return r + p[0];
+#endif
+ }
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+}
+
+#if T1HA_USE_FAST_ONESHOT_READ && \
+ T1HA_CONFIG_UNALIGNED_ACCESS != T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE && \
+ defined(PAGESIZE) && !defined(__sun) && !defined(__SANITIZE_ADDRESS__)
+#define can_read_underside(ptr, size) \
+ ((size) <= sizeof(uintptr_t) && ((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0)
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+
+static __always_inline uint64_t tail64_le_unaligned(const void *v,
+ size_t tail) {
+ const uint8_t *p = (const uint8_t *)v;
+#ifdef can_read_underside
+ /* On some systems (e.g. x86) we can perform a 'oneshot' read, which
+ * is little bit faster. Thanks Marcin Żukowski <marcin.zukowski@gmail.com>
+ * for the reminder. */
+ const unsigned offset = (8 - tail) & 7;
+ const unsigned shift = offset << 3;
+ if (likely(can_read_underside(p, 8))) {
+ p -= offset;
+ return fetch64_le_unaligned(p) >> shift;
+ }
+ return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift);
+#else
+ uint64_t r = 0;
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \
+ __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ /* For most CPUs this code is better when not needed
+ * copying for alignment or byte reordering. */
+ case 0:
+ return fetch64_le_unaligned(p);
+ case 7:
+ r = (uint64_t)p[6] << 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 32;
+ /* fall through */
+ case 4:
+ return r + fetch32_le_unaligned(p);
+ case 3:
+ r = (uint64_t)p[2] << 16;
+ /* fall through */
+ case 2:
+ return r + fetch16_le_unaligned(p);
+ case 1:
+ return p[0];
+#else
+ /* For most CPUs this code is better than a
+ * copying for alignment and/or byte reordering. */
+ case 0:
+ r = p[7] << 8;
+ /* fall through */
+ case 7:
+ r += p[6];
+ r <<= 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 8;
+ /* fall through */
+ case 4:
+ r += p[3];
+ r <<= 8;
+ /* fall through */
+ case 3:
+ r += p[2];
+ r <<= 8;
+ /* fall through */
+ case 2:
+ r += p[1];
+ r <<= 8;
+ /* fall through */
+ case 1:
+ return r + p[0];
+#endif
+ }
+#endif /* can_read_underside */
+}
+
+/*------------------------------------------------------------- Big Endian */
+
+#ifndef fetch16_be_aligned
+static __maybe_unused __always_inline uint16_t
+fetch16_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_16 == 0);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_aligned(v, 16);
+#else
+ return bswap16(read_aligned(v, 16));
+#endif
+}
+#endif /* fetch16_be_aligned */
+
+#ifndef fetch16_be_unaligned
+static __maybe_unused __always_inline uint16_t
+fetch16_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ const uint8_t *p = (const uint8_t *)v;
+ return (uint16_t)p[0] << 8 | p[1];
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 16);
+#else
+ return bswap16(read_unaligned(v, 16));
+#endif
+}
+#endif /* fetch16_be_unaligned */
+
+#ifndef fetch32_be_aligned
+static __maybe_unused __always_inline uint32_t
+fetch32_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_32 == 0);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_aligned(v, 32);
+#else
+ return bswap32(read_aligned(v, 32));
+#endif
+}
+#endif /* fetch32_be_aligned */
+
+#ifndef fetch32_be_unaligned
+static __maybe_unused __always_inline uint32_t
+fetch32_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return (uint32_t)fetch16_be_unaligned(v) << 16 |
+ fetch16_be_unaligned((const uint8_t *)v + 2);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 32);
+#else
+ return bswap32(read_unaligned(v, 32));
+#endif
+}
+#endif /* fetch32_be_unaligned */
+
+#ifndef fetch64_be_aligned
+static __maybe_unused __always_inline uint64_t
+fetch64_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_64 == 0);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_aligned(v, 64);
+#else
+ return bswap64(read_aligned(v, 64));
+#endif
+}
+#endif /* fetch64_be_aligned */
+
+#ifndef fetch64_be_unaligned
+static __maybe_unused __always_inline uint64_t
+fetch64_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return (uint64_t)fetch32_be_unaligned(v) << 32 |
+ fetch32_be_unaligned((const uint8_t *)v + 4);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 64);
+#else
+ return bswap64(read_unaligned(v, 64));
+#endif
+}
+#endif /* fetch64_be_unaligned */
+
+static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v,
+ size_t tail) {
+ const uint8_t *const p = (const uint8_t *)v;
+#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__)
+ /* We can perform a 'oneshot' read, which is little bit faster. */
+ const unsigned shift = ((8 - tail) & 7) << 3;
+ return fetch64_be_aligned(p) >> shift;
+#else
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ /* For most CPUs this code is better when not byte reordering. */
+ case 1:
+ return p[0];
+ case 2:
+ return fetch16_be_aligned(p);
+ case 3:
+ return (uint32_t)fetch16_be_aligned(p) << 8 | p[2];
+ case 4:
+ return fetch32_be_aligned(p);
+ case 5:
+ return (uint64_t)fetch32_be_aligned(p) << 8 | p[4];
+ case 6:
+ return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4);
+ case 7:
+ return (uint64_t)fetch32_be_aligned(p) << 24 |
+ (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6];
+ case 0:
+ return fetch64_be_aligned(p);
+#else
+ case 1:
+ return p[0];
+ case 2:
+ return p[1] | (uint32_t)p[0] << 8;
+ case 3:
+ return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16;
+ case 4:
+ return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 |
+ (uint32_t)p[0] << 24;
+ case 5:
+ return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 |
+ (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32;
+ case 6:
+ return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 |
+ (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40;
+ case 7:
+ return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 |
+ (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 |
+ (uint64_t)p[0] << 48;
+ case 0:
+ return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 |
+ (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 |
+ (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56;
+#endif
+ }
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+}
+
+static __maybe_unused __always_inline uint64_t
+tail64_be_unaligned(const void *v, size_t tail) {
+ const uint8_t *p = (const uint8_t *)v;
+#ifdef can_read_underside
+ /* On some systems we can perform a 'oneshot' read, which is little bit
+ * faster. Thanks Marcin Żukowski <marcin.zukowski@gmail.com> for the
+ * reminder. */
+ const unsigned offset = (8 - tail) & 7;
+ const unsigned shift = offset << 3;
+ if (likely(can_read_underside(p, 8))) {
+ p -= offset;
+ return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift);
+ }
+ return fetch64_be_unaligned(p) >> shift;
+#else
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \
+ __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ /* For most CPUs this code is better when not needed
+ * copying for alignment or byte reordering. */
+ case 1:
+ return p[0];
+ case 2:
+ return fetch16_be_unaligned(p);
+ case 3:
+ return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2];
+ case 4:
+ return fetch32_be(p);
+ case 5:
+ return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4];
+ case 6:
+ return (uint64_t)fetch32_be_unaligned(p) << 16 |
+ fetch16_be_unaligned(p + 4);
+ case 7:
+ return (uint64_t)fetch32_be_unaligned(p) << 24 |
+ (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6];
+ case 0:
+ return fetch64_be_unaligned(p);
+#else
+ /* For most CPUs this code is better than a
+ * copying for alignment and/or byte reordering. */
+ case 1:
+ return p[0];
+ case 2:
+ return p[1] | (uint32_t)p[0] << 8;
+ case 3:
+ return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16;
+ case 4:
+ return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 |
+ (uint32_t)p[0] << 24;
+ case 5:
+ return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 |
+ (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32;
+ case 6:
+ return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 |
+ (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40;
+ case 7:
+ return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 |
+ (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 |
+ (uint64_t)p[0] << 48;
+ case 0:
+ return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 |
+ (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 |
+ (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56;
+#endif
+ }
+#endif /* can_read_underside */
+}
+
+/***************************************************************************/
+
+#ifndef rot64
+static __always_inline uint64_t rot64(uint64_t v, unsigned s) {
+ return (v >> s) | (v << (64 - s));
+}
+#endif /* rot64 */
+
+#ifndef mul_32x32_64
+static __always_inline uint64_t mul_32x32_64(uint32_t a, uint32_t b) {
+ return a * (uint64_t)b;
+}
+#endif /* mul_32x32_64 */
+
+#ifndef add64carry_first
+static __maybe_unused __always_inline unsigned
+add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) {
+#if __has_builtin(__builtin_addcll)
+ unsigned long long carryout;
+ *sum = __builtin_addcll(base, addend, 0, &carryout);
+ return (unsigned)carryout;
+#else
+ *sum = base + addend;
+ return *sum < addend;
+#endif /* __has_builtin(__builtin_addcll) */
+}
+#endif /* add64carry_fist */
+
+#ifndef add64carry_next
+static __maybe_unused __always_inline unsigned
+add64carry_next(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) {
+#if __has_builtin(__builtin_addcll)
+ unsigned long long carryout;
+ *sum = __builtin_addcll(base, addend, carry, &carryout);
+ return (unsigned)carryout;
+#else
+ *sum = base + addend + carry;
+ return *sum < addend || (carry && *sum == addend);
+#endif /* __has_builtin(__builtin_addcll) */
+}
+#endif /* add64carry_next */
+
+#ifndef add64carry_last
+static __maybe_unused __always_inline void
+add64carry_last(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) {
+#if __has_builtin(__builtin_addcll)
+ unsigned long long carryout;
+ *sum = __builtin_addcll(base, addend, carry, &carryout);
+ (void)carryout;
+#else
+ *sum = base + addend + carry;
+#endif /* __has_builtin(__builtin_addcll) */
+}
+#endif /* add64carry_last */
+
+#ifndef mul_64x64_128
+static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a,
+ uint64_t b,
+ uint64_t *h) {
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ __uint128_t r = (__uint128_t)a * (__uint128_t)b;
+ /* modern GCC could nicely optimize this */
+ *h = (uint64_t)(r >> 64);
+ return (uint64_t)r;
+#elif defined(mul_64x64_high)
+ *h = mul_64x64_high(a, b);
+ return a * b;
+#else
+ /* performs 64x64 to 128 bit multiplication */
+ const uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b);
+ const uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b);
+ const uint64_t hl = mul_32x32_64((uint32_t)a, b >> 32);
+ const uint64_t hh = mul_32x32_64(a >> 32, b >> 32);
+
+ /* Few simplification are possible here for 32-bit architectures,
+ * but thus we would lost compatibility with the original 64-bit
+ * version. Think is very bad idea, because then 32-bit t1ha will
+ * still (relatively) very slowly and well yet not compatible. */
+ uint64_t l;
+ add64carry_last(add64carry_first(ll, lh << 32, &l), hh, lh >> 32, h);
+ add64carry_last(add64carry_first(l, hl << 32, &l), *h, hl >> 32, h);
+ return l;
+#endif
+}
+#endif /* mul_64x64_128() */
+
+#ifndef mul_64x64_high
+static __maybe_unused __always_inline uint64_t mul_64x64_high(uint64_t a,
+ uint64_t b) {
+ uint64_t h;
+ mul_64x64_128(a, b, &h);
+ return h;
+}
+#endif /* mul_64x64_high */
+
+/***************************************************************************/
+
+/* 'magic' primes */
+static const uint64_t prime_0 = UINT64_C(0xEC99BF0D8372CAAB);
+static const uint64_t prime_1 = UINT64_C(0x82434FE90EDCEF39);
+static const uint64_t prime_2 = UINT64_C(0xD4F06DB99D67BE4B);
+static const uint64_t prime_3 = UINT64_C(0xBD9CACC22C6E9571);
+static const uint64_t prime_4 = UINT64_C(0x9C06FAF4D023E3AB);
+static const uint64_t prime_5 = UINT64_C(0xC060724A8424F345);
+static const uint64_t prime_6 = UINT64_C(0xCB5AF53AE3AAAC31);
+
+/* xor high and low parts of full 128-bit product */
+static __maybe_unused __always_inline uint64_t mux64(uint64_t v,
+ uint64_t prime) {
+ uint64_t l, h;
+ l = mul_64x64_128(v, prime, &h);
+ return l ^ h;
+}
+
+static __always_inline uint64_t final64(uint64_t a, uint64_t b) {
+ uint64_t x = (a + rot64(b, 41)) * prime_0;
+ uint64_t y = (rot64(a, 23) + b) * prime_6;
+ return mux64(x ^ y, prime_5);
+}
+
+static __always_inline void mixup64(uint64_t *__restrict a,
+ uint64_t *__restrict b, uint64_t v,
+ uint64_t prime) {
+ uint64_t h;
+ *a ^= mul_64x64_128(*b + v, prime, &h);
+ *b += h;
+}
+
+/***************************************************************************/
+
+typedef union t1ha_uint128 {
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ __uint128_t v;
+#endif
+ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ uint64_t l;
+ uint64_t h;
+#else
+ uint64_t h;
+ uint64_t l;
+#endif
+ } p;
+} t1ha_uint128_t;
+
+static __always_inline t1ha_uint128_t not128(const t1ha_uint128_t v) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = ~v.v;
+#else
+ r.p.l = ~v.p.l;
+ r.p.h = ~v.p.h;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t left128(const t1ha_uint128_t v,
+ unsigned s) {
+ t1ha_uint128_t r;
+ assert(s < 128);
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = v.v << s;
+#else
+ r.p.l = (s < 64) ? v.p.l << s : 0;
+ r.p.h = (s < 64) ? (v.p.h << s) | (s ? v.p.l >> (64 - s) : 0) : v.p.l << (s - 64);
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t right128(const t1ha_uint128_t v,
+ unsigned s) {
+ t1ha_uint128_t r;
+ assert(s < 128);
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = v.v >> s;
+#else
+ r.p.l = (s < 64) ? (s ? v.p.h << (64 - s) : 0) | (v.p.l >> s) : v.p.h >> (s - 64);
+ r.p.h = (s < 64) ? v.p.h >> s : 0;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t or128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v | y.v;
+#else
+ r.p.l = x.p.l | y.p.l;
+ r.p.h = x.p.h | y.p.h;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v ^ y.v;
+#else
+ r.p.l = x.p.l ^ y.p.l;
+ r.p.h = x.p.h ^ y.p.h;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t rot128(t1ha_uint128_t v, unsigned s) {
+ s &= 127;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ v.v = (v.v << (128 - s)) | (v.v >> s);
+ return v;
+#else
+ return s ? or128(left128(v, 128 - s), right128(v, s)) : v;
+#endif
+}
+
+static __always_inline t1ha_uint128_t add128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v + y.v;
+#else
+ add64carry_last(add64carry_first(x.p.l, y.p.l, &r.p.l), x.p.h, y.p.h, &r.p.h);
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v * y.v;
+#else
+ r.p.l = mul_64x64_128(x.p.l, y.p.l, &r.p.h);
+ r.p.h += x.p.l * y.p.h + y.p.l * x.p.h;
+#endif
+ return r;
+}
diff --git a/contrib/uthash/uthash.h b/contrib/uthash/uthash.h
new file mode 100644
index 0000000..1547d30
--- /dev/null
+++ b/contrib/uthash/uthash.h
@@ -0,0 +1,951 @@
+/*
+Copyright (c) 2003-2013, Troy D. Hanson http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef UTHASH_H
+#define UTHASH_H
+
+#include <string.h> /* memcmp,strlen */
+#include <stddef.h> /* ptrdiff_t */
+#include <stdlib.h> /* exit() */
+
+/* These macros use decltype or the earlier __typeof GNU extension.
+ As decltype is only available in newer compilers (VS2010 or gcc 4.3+
+ when compiling c++ source) this code uses whatever method is needed
+ or, for VS2008 where neither is available, uses casting workarounds. */
+#ifdef _MSC_VER /* MS compiler */
+#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */
+#define DECLTYPE(x) (decltype(x))
+#else /* VS2008 or older (or VS2010 in C mode) */
+#define NO_DECLTYPE
+#define DECLTYPE(x)
+#endif
+#else /* GNU, Sun and other compilers */
+#define DECLTYPE(x) (__typeof(x))
+#endif
+
+#ifdef NO_DECLTYPE
+#define DECLTYPE_ASSIGN(dst,src) \
+do { \
+ char **_da_dst = (char**)(&(dst)); \
+ *_da_dst = (char*)(src); \
+} while(0)
+#else
+#define DECLTYPE_ASSIGN(dst,src) \
+do { \
+ (dst) = DECLTYPE(dst)(src); \
+} while(0)
+#endif
+
+/* a number of the hash function use uint32_t which isn't defined on win32 */
+#ifdef _MSC_VER
+typedef unsigned int uint32_t;
+typedef unsigned char uint8_t;
+#else
+#include <inttypes.h> /* uint32_t */
+#endif
+
+#define UTHASH_VERSION 1.9.8
+
+#ifndef uthash_fatal
+#define uthash_fatal(msg) exit(-1) /* fatal error (out of memory,etc) */
+#endif
+#ifndef uthash_malloc
+#define uthash_malloc(sz) malloc(sz) /* malloc fcn */
+#endif
+#ifndef uthash_free
+#define uthash_free(ptr,sz) free(ptr) /* free fcn */
+#endif
+
+#ifndef uthash_noexpand_fyi
+#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */
+#endif
+#ifndef uthash_expand_fyi
+#define uthash_expand_fyi(tbl) /* can be defined to log expands */
+#endif
+
+/* initial number of buckets */
+#define HASH_INITIAL_NUM_BUCKETS 32 /* initial number of buckets */
+#define HASH_INITIAL_NUM_BUCKETS_LOG2 5 /* lg2 of initial number of buckets */
+#define HASH_BKT_CAPACITY_THRESH 10 /* expand when bucket count reaches */
+
+/* calculate the element whose hash handle address is hhe */
+#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)(hhp)) - ((tbl)->hho)))
+
+#define HASH_FIND(hh,head,keyptr,keylen,out) \
+do { \
+ unsigned _hf_bkt,_hf_hashv; \
+ out=NULL; \
+ if (head) { \
+ HASH_FCN(keyptr,keylen, (head)->hh.tbl->num_buckets, _hf_hashv, _hf_bkt); \
+ if (HASH_BLOOM_TEST((head)->hh.tbl, _hf_hashv)) { \
+ HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], \
+ keyptr,keylen,out); \
+ } \
+ } \
+} while (0)
+
+#ifdef HASH_BLOOM
+#define HASH_BLOOM_BITLEN (1ULL << HASH_BLOOM)
+#define HASH_BLOOM_BYTELEN (HASH_BLOOM_BITLEN/8) + ((HASH_BLOOM_BITLEN%8) ? 1:0)
+#define HASH_BLOOM_MAKE(tbl) \
+do { \
+ (tbl)->bloom_nbits = HASH_BLOOM; \
+ (tbl)->bloom_bv = (uint8_t*)uthash_malloc(HASH_BLOOM_BYTELEN); \
+ if (!((tbl)->bloom_bv)) { uthash_fatal( "out of memory"); } \
+ memset((tbl)->bloom_bv, 0, HASH_BLOOM_BYTELEN); \
+ (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE; \
+} while (0)
+
+#define HASH_BLOOM_FREE(tbl) \
+do { \
+ uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \
+} while (0)
+
+#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8] |= (1U << ((idx)%8)))
+#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8] & (1U << ((idx)%8)))
+
+#define HASH_BLOOM_ADD(tbl,hashv) \
+ HASH_BLOOM_BITSET((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1)))
+
+#define HASH_BLOOM_TEST(tbl,hashv) \
+ HASH_BLOOM_BITTEST((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1)))
+
+#else
+#define HASH_BLOOM_MAKE(tbl)
+#define HASH_BLOOM_FREE(tbl)
+#define HASH_BLOOM_ADD(tbl,hashv)
+#define HASH_BLOOM_TEST(tbl,hashv) (1)
+#define HASH_BLOOM_BYTELEN 0
+#endif
+
+#define HASH_MAKE_TABLE(hh,head) \
+do { \
+ (head)->hh.tbl = (UT_hash_table*)uthash_malloc( \
+ sizeof(UT_hash_table)); \
+ if (!((head)->hh.tbl)) { uthash_fatal( "out of memory"); } \
+ memset((head)->hh.tbl, 0, sizeof(UT_hash_table)); \
+ (head)->hh.tbl->tail = &((head)->hh); \
+ (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \
+ (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \
+ (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head); \
+ (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_malloc( \
+ HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \
+ if (! (head)->hh.tbl->buckets) { uthash_fatal( "out of memory"); } \
+ memset((head)->hh.tbl->buckets, 0, \
+ HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \
+ HASH_BLOOM_MAKE((head)->hh.tbl); \
+ (head)->hh.tbl->signature = HASH_SIGNATURE; \
+} while(0)
+
+#define HASH_ADD(hh,head,fieldname,keylen_in,add) \
+ HASH_ADD_KEYPTR(hh,head,&((add)->fieldname),keylen_in,add)
+
+#define HASH_REPLACE(hh,head,fieldname,keylen_in,add,replaced) \
+do { \
+ replaced=NULL; \
+ HASH_FIND(hh,head,&((add)->fieldname),keylen_in,replaced); \
+ if (replaced!=NULL) { \
+ HASH_DELETE(hh,head,replaced); \
+ }; \
+ HASH_ADD(hh,head,fieldname,keylen_in,add); \
+} while(0)
+
+#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add) \
+do { \
+ unsigned _ha_bkt; \
+ (add)->hh.next = NULL; \
+ (add)->hh.key = (const char*)keyptr; \
+ (add)->hh.keylen = (unsigned)keylen_in; \
+ if (!(head)) { \
+ head = (add); \
+ (head)->hh.prev = NULL; \
+ HASH_MAKE_TABLE(hh,head); \
+ } else { \
+ (head)->hh.tbl->tail->next = (add); \
+ (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \
+ (head)->hh.tbl->tail = &((add)->hh); \
+ } \
+ (head)->hh.tbl->num_items++; \
+ (add)->hh.tbl = (head)->hh.tbl; \
+ HASH_FCN(keyptr,keylen_in, (head)->hh.tbl->num_buckets, \
+ (add)->hh.hashv, _ha_bkt); \
+ HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt],&(add)->hh); \
+ HASH_BLOOM_ADD((head)->hh.tbl,(add)->hh.hashv); \
+ HASH_EMIT_KEY(hh,head,keyptr,keylen_in); \
+ HASH_FSCK(hh,head); \
+} while(0)
+
+#define HASH_TO_BKT( hashv, num_bkts, bkt ) \
+do { \
+ bkt = ((hashv) & ((num_bkts) - 1)); \
+} while(0)
+
+/* delete "delptr" from the hash table.
+ * "the usual" patch-up process for the app-order doubly-linked-list.
+ * The use of _hd_hh_del below deserves special explanation.
+ * These used to be expressed using (delptr) but that led to a bug
+ * if someone used the same symbol for the head and deletee, like
+ * HASH_DELETE(hh,users,users);
+ * We want that to work, but by changing the head (users) below
+ * we were forfeiting our ability to further refer to the deletee (users)
+ * in the patch-up process. Solution: use scratch space to
+ * copy the deletee pointer, then the latter references are via that
+ * scratch pointer rather than through the repointed (users) symbol.
+ */
+#define HASH_DELETE(hh,head,delptr) \
+do { \
+ unsigned _hd_bkt; \
+ struct UT_hash_handle *_hd_hh_del; \
+ if ( ((delptr)->hh.prev == NULL) && ((delptr)->hh.next == NULL) ) { \
+ uthash_free((head)->hh.tbl->buckets, \
+ (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \
+ HASH_BLOOM_FREE((head)->hh.tbl); \
+ uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \
+ head = NULL; \
+ } else { \
+ _hd_hh_del = &((delptr)->hh); \
+ if ((delptr) == ELMT_FROM_HH((head)->hh.tbl,(head)->hh.tbl->tail)) { \
+ (head)->hh.tbl->tail = \
+ (UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \
+ (head)->hh.tbl->hho); \
+ } \
+ if ((delptr)->hh.prev) { \
+ ((UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \
+ (head)->hh.tbl->hho))->next = (delptr)->hh.next; \
+ } else { \
+ DECLTYPE_ASSIGN(head,(delptr)->hh.next); \
+ } \
+ if (_hd_hh_del->next) { \
+ ((UT_hash_handle*)((ptrdiff_t)_hd_hh_del->next + \
+ (head)->hh.tbl->hho))->prev = \
+ _hd_hh_del->prev; \
+ } \
+ HASH_TO_BKT( _hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \
+ HASH_DEL_IN_BKT(hh,(head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \
+ (head)->hh.tbl->num_items--; \
+ } \
+ HASH_FSCK(hh,head); \
+} while (0)
+
+
+/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */
+#define HASH_FIND_STR(head,findstr,out) \
+ HASH_FIND(hh,head,findstr,strlen(findstr),out)
+#define HASH_ADD_STR(head,strfield,add) \
+ HASH_ADD(hh,head,strfield,strlen(add->strfield),add)
+#define HASH_REPLACE_STR(head,strfield,add,replaced) \
+ HASH_REPLACE(hh,head,strfield,strlen(add->strfield),add,replaced)
+#define HASH_FIND_INT(head,findint,out) \
+ HASH_FIND(hh,head,findint,sizeof(int),out)
+#define HASH_ADD_INT(head,intfield,add) \
+ HASH_ADD(hh,head,intfield,sizeof(int),add)
+#define HASH_REPLACE_INT(head,intfield,add,replaced) \
+ HASH_REPLACE(hh,head,intfield,sizeof(int),add,replaced)
+#define HASH_FIND_PTR(head,findptr,out) \
+ HASH_FIND(hh,head,findptr,sizeof(void *),out)
+#define HASH_ADD_PTR(head,ptrfield,add) \
+ HASH_ADD(hh,head,ptrfield,sizeof(void *),add)
+#define HASH_REPLACE_PTR(head,ptrfield,add) \
+ HASH_REPLACE(hh,head,ptrfield,sizeof(void *),add,replaced)
+#define HASH_DEL(head,delptr) \
+ HASH_DELETE(hh,head,delptr)
+
+/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined.
+ * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined.
+ */
+#ifdef HASH_DEBUG
+#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0)
+#define HASH_FSCK(hh,head) \
+do { \
+ unsigned _bkt_i; \
+ unsigned _count, _bkt_count; \
+ char *_prev; \
+ struct UT_hash_handle *_thh; \
+ if (head) { \
+ _count = 0; \
+ for( _bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; _bkt_i++) { \
+ _bkt_count = 0; \
+ _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \
+ _prev = NULL; \
+ while (_thh) { \
+ if (_prev != (char*)(_thh->hh_prev)) { \
+ HASH_OOPS("invalid hh_prev %p, actual %p\n", \
+ _thh->hh_prev, _prev ); \
+ } \
+ _bkt_count++; \
+ _prev = (char*)(_thh); \
+ _thh = _thh->hh_next; \
+ } \
+ _count += _bkt_count; \
+ if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \
+ HASH_OOPS("invalid bucket count %d, actual %d\n", \
+ (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count); \
+ } \
+ } \
+ if (_count != (head)->hh.tbl->num_items) { \
+ HASH_OOPS("invalid hh item count %d, actual %d\n", \
+ (head)->hh.tbl->num_items, _count ); \
+ } \
+ /* traverse hh in app order; check next/prev integrity, count */ \
+ _count = 0; \
+ _prev = NULL; \
+ _thh = &(head)->hh; \
+ while (_thh) { \
+ _count++; \
+ if (_prev !=(char*)(_thh->prev)) { \
+ HASH_OOPS("invalid prev %p, actual %p\n", \
+ _thh->prev, _prev ); \
+ } \
+ _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh); \
+ _thh = ( _thh->next ? (UT_hash_handle*)((char*)(_thh->next) + \
+ (head)->hh.tbl->hho) : NULL ); \
+ } \
+ if (_count != (head)->hh.tbl->num_items) { \
+ HASH_OOPS("invalid app item count %d, actual %d\n", \
+ (head)->hh.tbl->num_items, _count ); \
+ } \
+ } \
+} while (0)
+#else
+#define HASH_FSCK(hh,head)
+#endif
+
+/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to
+ * the descriptor to which this macro is defined for tuning the hash function.
+ * The app can #include <unistd.h> to get the prototype for write(2). */
+#ifdef HASH_EMIT_KEYS
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) \
+do { \
+ unsigned _klen = fieldlen; \
+ write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \
+ write(HASH_EMIT_KEYS, keyptr, fieldlen); \
+} while (0)
+#else
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen)
+#endif
+
+/* default to Jenkin's hash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */
+#ifdef HASH_FUNCTION
+#define HASH_FCN HASH_FUNCTION
+#else
+#define HASH_FCN HASH_JEN
+#endif
+
+/* The Bernstein hash function, used in Perl prior to v5.6 */
+#define HASH_BER(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _hb_keylen=keylen; \
+ char *_hb_key=(char*)(key); \
+ (hashv) = 0; \
+ while (_hb_keylen--) { (hashv) = ((hashv) * 33) + *_hb_key++; } \
+ bkt = (hashv) & (num_bkts-1); \
+} while (0)
+
+
+/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at
+ * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */
+#define HASH_SAX(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _sx_i; \
+ char *_hs_key=(char*)(key); \
+ hashv = 0; \
+ for(_sx_i=0; _sx_i < keylen; _sx_i++) \
+ hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i]; \
+ bkt = hashv & (num_bkts-1); \
+} while (0)
+
+#define HASH_FNV(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _fn_i; \
+ char *_hf_key=(char*)(key); \
+ hashv = 2166136261UL; \
+ for(_fn_i=0; _fn_i < keylen; _fn_i++) \
+ hashv = (hashv * 16777619) ^ _hf_key[_fn_i]; \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+#define HASH_OAT(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _ho_i; \
+ char *_ho_key=(char*)(key); \
+ hashv = 0; \
+ for(_ho_i=0; _ho_i < keylen; _ho_i++) { \
+ hashv += _ho_key[_ho_i]; \
+ hashv += (hashv << 10); \
+ hashv ^= (hashv >> 6); \
+ } \
+ hashv += (hashv << 3); \
+ hashv ^= (hashv >> 11); \
+ hashv += (hashv << 15); \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+#define HASH_JEN_MIX(a,b,c) \
+do { \
+ a -= b; a -= c; a ^= ( c >> 13 ); \
+ b -= c; b -= a; b ^= ( a << 8 ); \
+ c -= a; c -= b; c ^= ( b >> 13 ); \
+ a -= b; a -= c; a ^= ( c >> 12 ); \
+ b -= c; b -= a; b ^= ( a << 16 ); \
+ c -= a; c -= b; c ^= ( b >> 5 ); \
+ a -= b; a -= c; a ^= ( c >> 3 ); \
+ b -= c; b -= a; b ^= ( a << 10 ); \
+ c -= a; c -= b; c ^= ( b >> 15 ); \
+} while (0)
+
+#define HASH_JEN(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _hj_i,_hj_j,_hj_k; \
+ unsigned const char *_hj_key=(unsigned const char*)(key); \
+ hashv = 0xfeedbeef; \
+ _hj_i = _hj_j = 0x9e3779b9; \
+ _hj_k = (unsigned)keylen; \
+ while (_hj_k >= 12) { \
+ _hj_i += (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 ) \
+ + ( (unsigned)_hj_key[2] << 16 ) \
+ + ( (unsigned)_hj_key[3] << 24 ) ); \
+ _hj_j += (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 ) \
+ + ( (unsigned)_hj_key[6] << 16 ) \
+ + ( (unsigned)_hj_key[7] << 24 ) ); \
+ hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 ) \
+ + ( (unsigned)_hj_key[10] << 16 ) \
+ + ( (unsigned)_hj_key[11] << 24 ) ); \
+ \
+ HASH_JEN_MIX(_hj_i, _hj_j, hashv); \
+ \
+ _hj_key += 12; \
+ _hj_k -= 12; \
+ } \
+ hashv += keylen; \
+ switch ( _hj_k ) { \
+ case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); \
+ case 10: hashv += ( (unsigned)_hj_key[9] << 16 ); \
+ case 9: hashv += ( (unsigned)_hj_key[8] << 8 ); \
+ case 8: _hj_j += ( (unsigned)_hj_key[7] << 24 ); \
+ case 7: _hj_j += ( (unsigned)_hj_key[6] << 16 ); \
+ case 6: _hj_j += ( (unsigned)_hj_key[5] << 8 ); \
+ case 5: _hj_j += _hj_key[4]; \
+ case 4: _hj_i += ( (unsigned)_hj_key[3] << 24 ); \
+ case 3: _hj_i += ( (unsigned)_hj_key[2] << 16 ); \
+ case 2: _hj_i += ( (unsigned)_hj_key[1] << 8 ); \
+ case 1: _hj_i += _hj_key[0]; \
+ } \
+ HASH_JEN_MIX(_hj_i, _hj_j, hashv); \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+/* The Paul Hsieh hash function */
+#undef get16bits
+#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \
+ || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__)
+#define get16bits(d) (*((const uint16_t *) (d)))
+#endif
+
+#if !defined (get16bits)
+#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8) \
+ +(uint32_t)(((const uint8_t *)(d))[0]) )
+#endif
+#define HASH_SFH(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned const char *_sfh_key=(unsigned const char*)(key); \
+ uint32_t _sfh_tmp, _sfh_len = keylen; \
+ \
+ int _sfh_rem = _sfh_len & 3; \
+ _sfh_len >>= 2; \
+ hashv = 0xcafebabe; \
+ \
+ /* Main loop */ \
+ for (;_sfh_len > 0; _sfh_len--) { \
+ hashv += get16bits (_sfh_key); \
+ _sfh_tmp = (uint32_t)(get16bits (_sfh_key+2)) << 11 ^ hashv; \
+ hashv = (hashv << 16) ^ _sfh_tmp; \
+ _sfh_key += 2*sizeof (uint16_t); \
+ hashv += hashv >> 11; \
+ } \
+ \
+ /* Handle end cases */ \
+ switch (_sfh_rem) { \
+ case 3: hashv += get16bits (_sfh_key); \
+ hashv ^= hashv << 16; \
+ hashv ^= (uint32_t)(_sfh_key[sizeof (uint16_t)] << 18); \
+ hashv += hashv >> 11; \
+ break; \
+ case 2: hashv += get16bits (_sfh_key); \
+ hashv ^= hashv << 11; \
+ hashv += hashv >> 17; \
+ break; \
+ case 1: hashv += *_sfh_key; \
+ hashv ^= hashv << 10; \
+ hashv += hashv >> 1; \
+ } \
+ \
+ /* Force "avalanching" of final 127 bits */ \
+ hashv ^= hashv << 3; \
+ hashv += hashv >> 5; \
+ hashv ^= hashv << 4; \
+ hashv += hashv >> 17; \
+ hashv ^= hashv << 25; \
+ hashv += hashv >> 6; \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+#ifdef HASH_USING_NO_STRICT_ALIASING
+/* The MurmurHash exploits some CPU's (x86,x86_64) tolerance for unaligned reads.
+ * For other types of CPU's (e.g. Sparc) an unaligned read causes a bus error.
+ * MurmurHash uses the faster approach only on CPU's where we know it's safe.
+ *
+ * Note the preprocessor built-in defines can be emitted using:
+ *
+ * gcc -m64 -dM -E - < /dev/null (on gcc)
+ * cc -## a.c (where a.c is a simple test file) (Sun Studio)
+ */
+#if (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86))
+#define MUR_GETBLOCK(p,i) p[i]
+#else /* non intel */
+#define MUR_PLUS0_ALIGNED(p) (((unsigned long)p & 0x3) == 0)
+#define MUR_PLUS1_ALIGNED(p) (((unsigned long)p & 0x3) == 1)
+#define MUR_PLUS2_ALIGNED(p) (((unsigned long)p & 0x3) == 2)
+#define MUR_PLUS3_ALIGNED(p) (((unsigned long)p & 0x3) == 3)
+#define WP(p) ((uint32_t*)((unsigned long)(p) & ~3UL))
+#if (defined(__BIG_ENDIAN__) || defined(SPARC) || defined(__ppc__) || defined(__ppc64__))
+#define MUR_THREE_ONE(p) ((((*WP(p))&0x00ffffff) << 8) | (((*(WP(p)+1))&0xff000000) >> 24))
+#define MUR_TWO_TWO(p) ((((*WP(p))&0x0000ffff) <<16) | (((*(WP(p)+1))&0xffff0000) >> 16))
+#define MUR_ONE_THREE(p) ((((*WP(p))&0x000000ff) <<24) | (((*(WP(p)+1))&0xffffff00) >> 8))
+#else /* assume little endian non-intel */
+#define MUR_THREE_ONE(p) ((((*WP(p))&0xffffff00) >> 8) | (((*(WP(p)+1))&0x000000ff) << 24))
+#define MUR_TWO_TWO(p) ((((*WP(p))&0xffff0000) >>16) | (((*(WP(p)+1))&0x0000ffff) << 16))
+#define MUR_ONE_THREE(p) ((((*WP(p))&0xff000000) >>24) | (((*(WP(p)+1))&0x00ffffff) << 8))
+#endif
+#define MUR_GETBLOCK(p,i) (MUR_PLUS0_ALIGNED(p) ? ((p)[i]) : \
+ (MUR_PLUS1_ALIGNED(p) ? MUR_THREE_ONE(p) : \
+ (MUR_PLUS2_ALIGNED(p) ? MUR_TWO_TWO(p) : \
+ MUR_ONE_THREE(p))))
+#endif
+#define MUR_ROTL32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
+#define MUR_FMIX(_h) \
+do { \
+ _h ^= _h >> 16; \
+ _h *= 0x85ebca6b; \
+ _h ^= _h >> 13; \
+ _h *= 0xc2b2ae35l; \
+ _h ^= _h >> 16; \
+} while(0)
+
+#define HASH_MUR(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ const uint8_t *_mur_data = (const uint8_t*)(key); \
+ const int _mur_nblocks = (keylen) / 4; \
+ uint32_t _mur_h1 = 0xf88D5353; \
+ uint32_t _mur_c1 = 0xcc9e2d51; \
+ uint32_t _mur_c2 = 0x1b873593; \
+ uint32_t _mur_k1 = 0; \
+ const uint8_t *_mur_tail; \
+ const uint32_t *_mur_blocks = (const uint32_t*)(_mur_data+_mur_nblocks*4); \
+ int _mur_i; \
+ for(_mur_i = -_mur_nblocks; _mur_i; _mur_i++) { \
+ _mur_k1 = MUR_GETBLOCK(_mur_blocks,_mur_i); \
+ _mur_k1 *= _mur_c1; \
+ _mur_k1 = MUR_ROTL32(_mur_k1,15); \
+ _mur_k1 *= _mur_c2; \
+ \
+ _mur_h1 ^= _mur_k1; \
+ _mur_h1 = MUR_ROTL32(_mur_h1,13); \
+ _mur_h1 = _mur_h1*5+0xe6546b64; \
+ } \
+ _mur_tail = (const uint8_t*)(_mur_data + _mur_nblocks*4); \
+ _mur_k1=0; \
+ switch((keylen) & 3) { \
+ case 3: _mur_k1 ^= _mur_tail[2] << 16; \
+ case 2: _mur_k1 ^= _mur_tail[1] << 8; \
+ case 1: _mur_k1 ^= _mur_tail[0]; \
+ _mur_k1 *= _mur_c1; \
+ _mur_k1 = MUR_ROTL32(_mur_k1,15); \
+ _mur_k1 *= _mur_c2; \
+ _mur_h1 ^= _mur_k1; \
+ case 0: break; \
+ } \
+ _mur_h1 ^= (keylen); \
+ MUR_FMIX(_mur_h1); \
+ hashv = _mur_h1; \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+#endif /* HASH_USING_NO_STRICT_ALIASING */
+
+/* key comparison function; return 0 if keys equal */
+#ifndef HASH_KEYCMP
+#define HASH_KEYCMP(a,b,len) memcmp(a,b,len)
+#endif
+
+/* iterate over items in a known bucket to find desired item */
+#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,out) \
+do { \
+ if (head.hh_head) DECLTYPE_ASSIGN(out,ELMT_FROM_HH(tbl,head.hh_head)); \
+ else out=NULL; \
+ while (out) { \
+ if ((out)->hh.keylen == keylen_in) { \
+ if ((HASH_KEYCMP((out)->hh.key,keyptr,keylen_in)) == 0) break; \
+ } \
+ if ((out)->hh.hh_next) DECLTYPE_ASSIGN(out,ELMT_FROM_HH(tbl,(out)->hh.hh_next)); \
+ else out = NULL; \
+ } \
+} while(0)
+
+/* add an item to a bucket */
+#define HASH_ADD_TO_BKT(head,addhh) \
+do { \
+ head.count++; \
+ (addhh)->hh_next = head.hh_head; \
+ (addhh)->hh_prev = NULL; \
+ if (head.hh_head) { (head).hh_head->hh_prev = (addhh); } \
+ (head).hh_head=addhh; \
+ if (head.count >= ((head.expand_mult+1) * HASH_BKT_CAPACITY_THRESH) \
+ && (addhh)->tbl->noexpand != 1) { \
+ HASH_EXPAND_BUCKETS((addhh)->tbl); \
+ } \
+} while(0)
+
+/* remove an item from a given bucket */
+#define HASH_DEL_IN_BKT(hh,head,hh_del) \
+ (head).count--; \
+ if ((head).hh_head == hh_del) { \
+ (head).hh_head = hh_del->hh_next; \
+ } \
+ if (hh_del->hh_prev) { \
+ hh_del->hh_prev->hh_next = hh_del->hh_next; \
+ } \
+ if (hh_del->hh_next) { \
+ hh_del->hh_next->hh_prev = hh_del->hh_prev; \
+ }
+
+/* Bucket expansion has the effect of doubling the number of buckets
+ * and redistributing the items into the new buckets. Ideally the
+ * items will distribute more or less evenly into the new buckets
+ * (the extent to which this is true is a measure of the quality of
+ * the hash function as it applies to the key domain).
+ *
+ * With the items distributed into more buckets, the chain length
+ * (item count) in each bucket is reduced. Thus by expanding buckets
+ * the hash keeps a bound on the chain length. This bounded chain
+ * length is the essence of how a hash provides constant time lookup.
+ *
+ * The calculation of tbl->ideal_chain_maxlen below deserves some
+ * explanation. First, keep in mind that we're calculating the ideal
+ * maximum chain length based on the *new* (doubled) bucket count.
+ * In fractions this is just n/b (n=number of items,b=new num buckets).
+ * Since the ideal chain length is an integer, we want to calculate
+ * ceil(n/b). We don't depend on floating point arithmetic in this
+ * hash, so to calculate ceil(n/b) with integers we could write
+ *
+ * ceil(n/b) = (n/b) + ((n%b)?1:0)
+ *
+ * and in fact a previous version of this hash did just that.
+ * But now we have improved things a bit by recognizing that b is
+ * always a power of two. We keep its base 2 log handy (call it lb),
+ * so now we can write this with a bit shift and logical AND:
+ *
+ * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0)
+ *
+ */
+#define HASH_EXPAND_BUCKETS(tbl) \
+do { \
+ unsigned _he_bkt; \
+ unsigned _he_bkt_i; \
+ struct UT_hash_handle *_he_thh, *_he_hh_nxt; \
+ UT_hash_bucket *_he_new_buckets, *_he_newbkt; \
+ _he_new_buckets = (UT_hash_bucket*)uthash_malloc( \
+ 2 * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \
+ if (!_he_new_buckets) { uthash_fatal( "out of memory"); } \
+ memset(_he_new_buckets, 0, \
+ 2 * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \
+ tbl->ideal_chain_maxlen = \
+ (tbl->num_items >> (tbl->log2_num_buckets+1)) + \
+ ((tbl->num_items & ((tbl->num_buckets*2)-1)) ? 1 : 0); \
+ tbl->nonideal_items = 0; \
+ for(_he_bkt_i = 0; _he_bkt_i < tbl->num_buckets; _he_bkt_i++) \
+ { \
+ _he_thh = tbl->buckets[ _he_bkt_i ].hh_head; \
+ while (_he_thh) { \
+ _he_hh_nxt = _he_thh->hh_next; \
+ HASH_TO_BKT( _he_thh->hashv, tbl->num_buckets*2, _he_bkt); \
+ _he_newbkt = &(_he_new_buckets[ _he_bkt ]); \
+ if (++(_he_newbkt->count) > tbl->ideal_chain_maxlen) { \
+ tbl->nonideal_items++; \
+ _he_newbkt->expand_mult = _he_newbkt->count / \
+ tbl->ideal_chain_maxlen; \
+ } \
+ _he_thh->hh_prev = NULL; \
+ _he_thh->hh_next = _he_newbkt->hh_head; \
+ if (_he_newbkt->hh_head) _he_newbkt->hh_head->hh_prev = \
+ _he_thh; \
+ _he_newbkt->hh_head = _he_thh; \
+ _he_thh = _he_hh_nxt; \
+ } \
+ } \
+ uthash_free( tbl->buckets, tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \
+ tbl->num_buckets *= 2; \
+ tbl->log2_num_buckets++; \
+ tbl->buckets = _he_new_buckets; \
+ tbl->ineff_expands = (tbl->nonideal_items > (tbl->num_items >> 1)) ? \
+ (tbl->ineff_expands+1) : 0; \
+ if (tbl->ineff_expands > 1) { \
+ tbl->noexpand=1; \
+ uthash_noexpand_fyi(tbl); \
+ } \
+ uthash_expand_fyi(tbl); \
+} while(0)
+
+
+/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */
+/* Note that HASH_SORT assumes the hash handle name to be hh.
+ * HASH_SRT was added to allow the hash handle name to be passed in. */
+#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn)
+#define HASH_SRT(hh,head,cmpfcn) \
+do { \
+ unsigned _hs_i; \
+ unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize; \
+ struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \
+ if (head) { \
+ _hs_insize = 1; \
+ _hs_looping = 1; \
+ _hs_list = &((head)->hh); \
+ while (_hs_looping) { \
+ _hs_p = _hs_list; \
+ _hs_list = NULL; \
+ _hs_tail = NULL; \
+ _hs_nmerges = 0; \
+ while (_hs_p) { \
+ _hs_nmerges++; \
+ _hs_q = _hs_p; \
+ _hs_psize = 0; \
+ for ( _hs_i = 0; _hs_i < _hs_insize; _hs_i++ ) { \
+ _hs_psize++; \
+ _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
+ ((void*)((char*)(_hs_q->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ if (! (_hs_q) ) break; \
+ } \
+ _hs_qsize = _hs_insize; \
+ while ((_hs_psize > 0) || ((_hs_qsize > 0) && _hs_q )) { \
+ if (_hs_psize == 0) { \
+ _hs_e = _hs_q; \
+ _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
+ ((void*)((char*)(_hs_q->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ _hs_qsize--; \
+ } else if ( (_hs_qsize == 0) || !(_hs_q) ) { \
+ _hs_e = _hs_p; \
+ if (_hs_p){ \
+ _hs_p = (UT_hash_handle*)((_hs_p->next) ? \
+ ((void*)((char*)(_hs_p->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ } \
+ _hs_psize--; \
+ } else if (( \
+ cmpfcn(DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_p)), \
+ DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_q))) \
+ ) <= 0) { \
+ _hs_e = _hs_p; \
+ if (_hs_p){ \
+ _hs_p = (UT_hash_handle*)((_hs_p->next) ? \
+ ((void*)((char*)(_hs_p->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ } \
+ _hs_psize--; \
+ } else { \
+ _hs_e = _hs_q; \
+ _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
+ ((void*)((char*)(_hs_q->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ _hs_qsize--; \
+ } \
+ if ( _hs_tail ) { \
+ _hs_tail->next = ((_hs_e) ? \
+ ELMT_FROM_HH((head)->hh.tbl,_hs_e) : NULL); \
+ } else { \
+ _hs_list = _hs_e; \
+ } \
+ if (_hs_e) { \
+ _hs_e->prev = ((_hs_tail) ? \
+ ELMT_FROM_HH((head)->hh.tbl,_hs_tail) : NULL); \
+ } \
+ _hs_tail = _hs_e; \
+ } \
+ _hs_p = _hs_q; \
+ } \
+ if (_hs_tail){ \
+ _hs_tail->next = NULL; \
+ } \
+ if ( _hs_nmerges <= 1 ) { \
+ _hs_looping=0; \
+ (head)->hh.tbl->tail = _hs_tail; \
+ DECLTYPE_ASSIGN(head,ELMT_FROM_HH((head)->hh.tbl, _hs_list)); \
+ } \
+ _hs_insize *= 2; \
+ } \
+ HASH_FSCK(hh,head); \
+ } \
+} while (0)
+
+/* This function selects items from one hash into another hash.
+ * The end result is that the selected items have dual presence
+ * in both hashes. There is no copy of the items made; rather
+ * they are added into the new hash through a secondary hash
+ * hash handle that must be present in the structure. */
+#define HASH_SELECT(hh_dst, dst, hh_src, src, cond) \
+do { \
+ unsigned _src_bkt, _dst_bkt; \
+ void *_last_elt=NULL, *_elt; \
+ UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL; \
+ ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst)); \
+ if (src) { \
+ for(_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) { \
+ for(_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head; \
+ _src_hh; \
+ _src_hh = _src_hh->hh_next) { \
+ _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh); \
+ if (cond(_elt)) { \
+ _dst_hh = (UT_hash_handle*)(((char*)_elt) + _dst_hho); \
+ _dst_hh->key = _src_hh->key; \
+ _dst_hh->keylen = _src_hh->keylen; \
+ _dst_hh->hashv = _src_hh->hashv; \
+ _dst_hh->prev = _last_elt; \
+ _dst_hh->next = NULL; \
+ if (_last_elt_hh) { _last_elt_hh->next = _elt; } \
+ if (!dst) { \
+ DECLTYPE_ASSIGN(dst,_elt); \
+ HASH_MAKE_TABLE(hh_dst,dst); \
+ } else { \
+ _dst_hh->tbl = (dst)->hh_dst.tbl; \
+ } \
+ HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt); \
+ HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt],_dst_hh); \
+ (dst)->hh_dst.tbl->num_items++; \
+ _last_elt = _elt; \
+ _last_elt_hh = _dst_hh; \
+ } \
+ } \
+ } \
+ } \
+ HASH_FSCK(hh_dst,dst); \
+} while (0)
+
+#define HASH_CLEAR(hh,head) \
+do { \
+ if (head) { \
+ uthash_free((head)->hh.tbl->buckets, \
+ (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket)); \
+ HASH_BLOOM_FREE((head)->hh.tbl); \
+ uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \
+ (head)=NULL; \
+ } \
+} while(0)
+
+#define HASH_OVERHEAD(hh,head) \
+ (size_t)((((head)->hh.tbl->num_items * sizeof(UT_hash_handle)) + \
+ ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket)) + \
+ (sizeof(UT_hash_table)) + \
+ (HASH_BLOOM_BYTELEN)))
+
+#ifdef NO_DECLTYPE
+#define HASH_ITER(hh,head,el,tmp) \
+for((el)=(head), (*(char**)(&(tmp)))=(char*)((head)?(head)->hh.next:NULL); \
+ el; (el)=(tmp),(*(char**)(&(tmp)))=(char*)((tmp)?(tmp)->hh.next:NULL))
+#else
+#define HASH_ITER(hh,head,el,tmp) \
+for((el)=(head),(tmp)=DECLTYPE(el)((head)?(head)->hh.next:NULL); \
+ el; (el)=(tmp),(tmp)=DECLTYPE(el)((tmp)?(tmp)->hh.next:NULL))
+#endif
+
+/* obtain a count of items in the hash */
+#define HASH_COUNT(head) HASH_CNT(hh,head)
+#define HASH_CNT(hh,head) ((head)?((head)->hh.tbl->num_items):0)
+
+typedef struct UT_hash_bucket {
+ struct UT_hash_handle *hh_head;
+ unsigned count;
+
+ /* expand_mult is normally set to 0. In this situation, the max chain length
+ * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If
+ * the bucket's chain exceeds this length, bucket expansion is triggered).
+ * However, setting expand_mult to a non-zero value delays bucket expansion
+ * (that would be triggered by additions to this particular bucket)
+ * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH.
+ * (The multiplier is simply expand_mult+1). The whole idea of this
+ * multiplier is to reduce bucket expansions, since they are expensive, in
+ * situations where we know that a particular bucket tends to be overused.
+ * It is better to let its chain length grow to a longer yet-still-bounded
+ * value, than to do an O(n) bucket expansion too often.
+ */
+ unsigned expand_mult;
+
+} UT_hash_bucket;
+
+/* random signature used only to find hash tables in external analysis */
+#define HASH_SIGNATURE 0xa0111fe1
+#define HASH_BLOOM_SIGNATURE 0xb12220f2
+
+typedef struct UT_hash_table {
+ UT_hash_bucket *buckets;
+ unsigned num_buckets, log2_num_buckets;
+ unsigned num_items;
+ struct UT_hash_handle *tail; /* tail hh in app order, for fast append */
+ ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */
+
+ /* in an ideal situation (all buckets used equally), no bucket would have
+ * more than ceil(#items/#buckets) items. that's the ideal chain length. */
+ unsigned ideal_chain_maxlen;
+
+ /* nonideal_items is the number of items in the hash whose chain position
+ * exceeds the ideal chain maxlen. these items pay the penalty for an uneven
+ * hash distribution; reaching them in a chain traversal takes >ideal steps */
+ unsigned nonideal_items;
+
+ /* ineffective expands occur when a bucket doubling was performed, but
+ * afterward, more than half the items in the hash had nonideal chain
+ * positions. If this happens on two consecutive expansions we inhibit any
+ * further expansion, as it's not helping; this happens when the hash
+ * function isn't a good fit for the key domain. When expansion is inhibited
+ * the hash will still work, albeit no longer in constant time. */
+ unsigned ineff_expands, noexpand;
+
+ uint32_t signature; /* used only to find hash tables in external analysis */
+#ifdef HASH_BLOOM
+ uint32_t bloom_sig; /* used only to test bloom exists in external analysis */
+ uint8_t *bloom_bv;
+ char bloom_nbits;
+#endif
+
+} UT_hash_table;
+
+typedef struct UT_hash_handle {
+ struct UT_hash_table *tbl;
+ void *prev; /* prev element in app order */
+ void *next; /* next element in app order */
+ struct UT_hash_handle *hh_prev; /* previous hh in bucket order */
+ struct UT_hash_handle *hh_next; /* next hh in bucket order */
+ const void *key; /* ptr to enclosing struct's key */
+ unsigned keylen; /* enclosing struct's key len */
+ unsigned hashv; /* result of hash-fcn(key) */
+} UT_hash_handle;
+
+#endif /* UTHASH_H */
diff --git a/contrib/uthash/utlist.h b/contrib/uthash/utlist.h
new file mode 100644
index 0000000..6c72b9f
--- /dev/null
+++ b/contrib/uthash/utlist.h
@@ -0,0 +1,766 @@
+/*
+Copyright (c) 2007-2013, Troy D. Hanson http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef UTLIST_H
+#define UTLIST_H
+
+#define UTLIST_VERSION 1.9.8
+
+#include <assert.h>
+
+/*
+ * This file contains macros to manipulate singly and doubly-linked lists.
+ *
+ * 1. LL_ macros: singly-linked lists.
+ * 2. DL_ macros: doubly-linked lists.
+ * 3. CDL_ macros: circular doubly-linked lists.
+ *
+ * To use singly-linked lists, your structure must have a "next" pointer.
+ * To use doubly-linked lists, your structure must "prev" and "next" pointers.
+ * Either way, the pointer to the head of the list must be initialized to NULL.
+ *
+ * ----------------.EXAMPLE -------------------------
+ * struct item {
+ * int id;
+ * struct item *prev, *next;
+ * }
+ *
+ * struct item *list = NULL:
+ *
+ * int main() {
+ * struct item *item;
+ * ... allocate and populate item ...
+ * DL_APPEND(list, item);
+ * }
+ * --------------------------------------------------
+ *
+ * For doubly-linked lists, the append and delete macros are O(1)
+ * For singly-linked lists, append and delete are O(n) but prepend is O(1)
+ * The sort macro is O(n log(n)) for all types of single/double/circular lists.
+ */
+
+/* These macros use decltype or the earlier __typeof GNU extension.
+ As decltype is only available in newer compilers (VS2010 or gcc 4.3+
+ when compiling c++ code), this code uses whatever method is needed
+ or, for VS2008 where neither is available, uses casting workarounds. */
+#ifdef _MSC_VER /* MS compiler */
+#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */
+#define LDECLTYPE(x) decltype(x)
+#else /* VS2008 or older (or VS2010 in C mode) */
+#define NO_DECLTYPE
+#define LDECLTYPE(x) char*
+#endif
+#elif defined(__ICCARM__)
+#define NO_DECLTYPE
+#define LDECLTYPE(x) char*
+#else /* GNU, Sun and other compilers */
+#define LDECLTYPE(x) __typeof(x)
+#endif
+
+/* for VS2008 we use some workarounds to get around the lack of decltype,
+ * namely, we always reassign our tmp variable to the list head if we need
+ * to dereference its prev/next pointers, and save/restore the real head.*/
+#ifdef NO_DECLTYPE
+#define _SV(elt,list) _tmp = (char*)(list); {char **_alias = (char**)&(list); *_alias = (elt); }
+#define _NEXT(elt,list,next) ((char*)((list)->next))
+#define _NEXTASGN(elt,list,to,next) { char **_alias = (char**)&((list)->next); *_alias=(char*)(to); }
+/* #define _PREV(elt,list,prev) ((char*)((list)->prev)) */
+#define _PREVASGN(elt,list,to,prev) { char **_alias = (char**)&((list)->prev); *_alias=(char*)(to); }
+#define _RS(list) { char **_alias = (char**)&(list); *_alias=_tmp; }
+#define _CASTASGN(a,b) { char **_alias = (char**)&(a); *_alias=(char*)(b); }
+#else
+#define _SV(elt,list)
+#define _NEXT(elt,list,next) ((elt)->next)
+#define _NEXTASGN(elt,list,to,next) ((elt)->next)=(to)
+/* #define _PREV(elt,list,prev) ((elt)->prev) */
+#define _PREVASGN(elt,list,to,prev) ((elt)->prev)=(to)
+#define _RS(list)
+#define _CASTASGN(a,b) (a)=(b)
+#endif
+
+/******************************************************************************
+ * The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort *
+ * Unwieldy variable names used here to avoid shadowing passed-in variables. *
+ *****************************************************************************/
+#define LL_SORT(list, cmp) \
+ LL_SORT2(list, cmp, next)
+
+#define LL_SORT2(list, cmp, next) \
+do { \
+ LDECLTYPE(list) _ls_p; \
+ LDECLTYPE(list) _ls_q; \
+ LDECLTYPE(list) _ls_e; \
+ LDECLTYPE(list) _ls_tail; \
+ int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
+ if (list) { \
+ _ls_insize = 1; \
+ _ls_looping = 1; \
+ while (_ls_looping) { \
+ _CASTASGN(_ls_p,list); \
+ list = NULL; \
+ _ls_tail = NULL; \
+ _ls_nmerges = 0; \
+ while (_ls_p) { \
+ _ls_nmerges++; \
+ _ls_q = _ls_p; \
+ _ls_psize = 0; \
+ for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
+ _ls_psize++; \
+ _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \
+ if (!_ls_q) break; \
+ } \
+ _ls_qsize = _ls_insize; \
+ while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
+ if (_ls_psize == 0) { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } else if (_ls_qsize == 0 || !_ls_q) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else if (cmp(_ls_p,_ls_q) <= 0) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
+ } else { \
+ _CASTASGN(list,_ls_e); \
+ } \
+ _ls_tail = _ls_e; \
+ } \
+ _ls_p = _ls_q; \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \
+ } \
+ if (_ls_nmerges <= 1) { \
+ _ls_looping=0; \
+ } \
+ _ls_insize *= 2; \
+ } \
+ } \
+} while (0)
+
+
+#define DL_SORT(list, cmp) \
+ DL_SORT2(list, cmp, prev, next)
+
+#define DL_SORT2(list, cmp, prev, next) \
+do { \
+ LDECLTYPE(list) _ls_p; \
+ LDECLTYPE(list) _ls_q; \
+ LDECLTYPE(list) _ls_e; \
+ LDECLTYPE(list) _ls_tail; \
+ int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
+ if (list) { \
+ _ls_insize = 1; \
+ _ls_looping = 1; \
+ while (_ls_looping) { \
+ _CASTASGN(_ls_p,list); \
+ list = NULL; \
+ _ls_tail = NULL; \
+ _ls_nmerges = 0; \
+ while (_ls_p) { \
+ _ls_nmerges++; \
+ _ls_q = _ls_p; \
+ _ls_psize = 0; \
+ for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
+ _ls_psize++; \
+ _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \
+ if (!_ls_q) break; \
+ } \
+ _ls_qsize = _ls_insize; \
+ while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
+ if (_ls_psize == 0) { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } else if (_ls_qsize == 0 || !_ls_q) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else if (cmp(_ls_p,_ls_q) <= 0) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
+ } else { \
+ _CASTASGN(list,_ls_e); \
+ } \
+ _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \
+ _ls_tail = _ls_e; \
+ } \
+ _ls_p = _ls_q; \
+ } \
+ _CASTASGN(list->prev, _ls_tail); \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \
+ if (_ls_nmerges <= 1) { \
+ _ls_looping=0; \
+ } \
+ _ls_insize *= 2; \
+ } \
+ } \
+} while (0)
+
+#define CDL_SORT(list, cmp) \
+ CDL_SORT2(list, cmp, prev, next)
+
+#define CDL_SORT2(list, cmp, prev, next) \
+do { \
+ LDECLTYPE(list) _ls_p; \
+ LDECLTYPE(list) _ls_q; \
+ LDECLTYPE(list) _ls_e; \
+ LDECLTYPE(list) _ls_tail; \
+ LDECLTYPE(list) _ls_oldhead; \
+ LDECLTYPE(list) _tmp; \
+ int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
+ if (list) { \
+ _ls_insize = 1; \
+ _ls_looping = 1; \
+ while (_ls_looping) { \
+ _CASTASGN(_ls_p,list); \
+ _CASTASGN(_ls_oldhead,list); \
+ list = NULL; \
+ _ls_tail = NULL; \
+ _ls_nmerges = 0; \
+ while (_ls_p) { \
+ _ls_nmerges++; \
+ _ls_q = _ls_p; \
+ _ls_psize = 0; \
+ for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
+ _ls_psize++; \
+ _SV(_ls_q,list); \
+ if (_NEXT(_ls_q,list,next) == _ls_oldhead) { \
+ _ls_q = NULL; \
+ } else { \
+ _ls_q = _NEXT(_ls_q,list,next); \
+ } \
+ _RS(list); \
+ if (!_ls_q) break; \
+ } \
+ _ls_qsize = _ls_insize; \
+ while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
+ if (_ls_psize == 0) { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \
+ } else if (_ls_qsize == 0 || !_ls_q) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \
+ } else if (cmp(_ls_p,_ls_q) <= 0) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \
+ } else { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
+ } else { \
+ _CASTASGN(list,_ls_e); \
+ } \
+ _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \
+ _ls_tail = _ls_e; \
+ } \
+ _ls_p = _ls_q; \
+ } \
+ _CASTASGN(list->prev,_ls_tail); \
+ _CASTASGN(_tmp,list); \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_tmp,next); _RS(list); \
+ if (_ls_nmerges <= 1) { \
+ _ls_looping=0; \
+ } \
+ _ls_insize *= 2; \
+ } \
+ } \
+} while (0)
+
+/******************************************************************************
+ * singly linked list macros (non-circular) *
+ *****************************************************************************/
+#define LL_PREPEND(head,add) \
+ LL_PREPEND2(head,add,next)
+
+#define LL_PREPEND2(head,add,next) \
+do { \
+ (add)->next = head; \
+ (head) = (add); \
+} while (0)
+
+#define LL_CONCAT(head1,head2) \
+ LL_CONCAT2(head1,head2,next)
+
+#define LL_CONCAT2(head1,head2,next) \
+do { \
+ LDECLTYPE(head1) _tmp; \
+ if (head1) { \
+ _tmp = head1; \
+ while (_tmp->next) { _tmp = _tmp->next; } \
+ _tmp->next=(head2); \
+ } else { \
+ (head1)=(head2); \
+ } \
+} while (0)
+
+#define LL_APPEND(head,add) \
+ LL_APPEND2(head,add,next)
+
+#define LL_APPEND2(head,add,next) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ (add)->next=NULL; \
+ if (head) { \
+ _tmp = head; \
+ while (_tmp->next) { _tmp = _tmp->next; } \
+ _tmp->next=(add); \
+ } else { \
+ (head)=(add); \
+ } \
+} while (0)
+
+#define LL_DELETE(head,del) \
+ LL_DELETE2(head,del,next)
+
+#define LL_DELETE2(head,del,next) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ if ((head) == (del)) { \
+ (head)=(head)->next; \
+ } else { \
+ _tmp = head; \
+ while (_tmp->next && (_tmp->next != (del))) { \
+ _tmp = _tmp->next; \
+ } \
+ if (_tmp->next) { \
+ _tmp->next = ((del)->next); \
+ } \
+ } \
+} while (0)
+
+#define LL_REVERSE2(head,next) do { \
+ LDECLTYPE(head) _cur = (head), _p = NULL, _n = NULL; \
+ while(_cur != NULL) { _n = _cur->next; _cur->next = _p; _p = _cur; _cur = _n; } \
+ (head) = _p; \
+} while (0)
+
+#define LL_REVERSE(head) \
+ LL_REVERSE2(head,next)
+
+/* Here are VS2008 replacements for LL_APPEND and LL_DELETE */
+#define LL_APPEND_VS2008(head,add) \
+ LL_APPEND2_VS2008(head,add,next)
+
+#define LL_APPEND2_VS2008(head,add,next) \
+do { \
+ if (head) { \
+ (add)->next = head; /* use add->next as a temp variable */ \
+ while ((add)->next->next) { (add)->next = (add)->next->next; } \
+ (add)->next->next=(add); \
+ } else { \
+ (head)=(add); \
+ } \
+ (add)->next=NULL; \
+} while (0)
+
+#define LL_DELETE_VS2008(head,del) \
+ LL_DELETE2_VS2008(head,del,next)
+
+#define LL_DELETE2_VS2008(head,del,next) \
+do { \
+ if ((head) == (del)) { \
+ (head)=(head)->next; \
+ } else { \
+ char *_tmp = (char*)(head); \
+ while ((head)->next && ((head)->next != (del))) { \
+ head = (head)->next; \
+ } \
+ if ((head)->next) { \
+ (head)->next = ((del)->next); \
+ } \
+ { \
+ char **_head_alias = (char**)&(head); \
+ *_head_alias = _tmp; \
+ } \
+ } \
+} while (0)
+#ifdef NO_DECLTYPE
+#undef LL_APPEND
+#define LL_APPEND LL_APPEND_VS2008
+#undef LL_DELETE
+#define LL_DELETE LL_DELETE_VS2008
+#undef LL_DELETE2
+#define LL_DELETE2 LL_DELETE2_VS2008
+#undef LL_APPEND2
+#define LL_APPEND2 LL_APPEND2_VS2008
+#undef LL_CONCAT /* no LL_CONCAT_VS2008 */
+#undef DL_CONCAT /* no DL_CONCAT_VS2008 */
+#endif
+/* end VS2008 replacements */
+
+#define LL_COUNT(head,el,counter) \
+ LL_COUNT2(head,el,counter,next) \
+
+#define LL_COUNT2(head,el,counter,next) \
+{ \
+ counter = 0; \
+ LL_FOREACH2(head,el,next){ ++counter; } \
+}
+
+#define LL_FOREACH(head,el) \
+ LL_FOREACH2(head,el,next)
+
+#define LL_FOREACH2(head,el,next) \
+ for(el=head;el;el=(el)->next)
+
+#define LL_FOREACH_SAFE(head,el,tmp) \
+ LL_FOREACH_SAFE2(head,el,tmp,next)
+
+#define LL_FOREACH_SAFE2(head,el,tmp,next) \
+ for((el)=(head);(el) && (tmp = (el)->next, 1); (el) = tmp)
+
+#define LL_SEARCH_SCALAR(head,out,field,val) \
+ LL_SEARCH_SCALAR2(head,out,field,val,next)
+
+#define LL_SEARCH_SCALAR2(head,out,field,val,next) \
+do { \
+ LL_FOREACH2(head,out,next) { \
+ if ((out)->field == (val)) break; \
+ } \
+} while(0)
+
+#define LL_SEARCH(head,out,elt,cmp) \
+ LL_SEARCH2(head,out,elt,cmp,next)
+
+#define LL_SEARCH2(head,out,elt,cmp,next) \
+do { \
+ LL_FOREACH2(head,out,next) { \
+ if ((cmp(out,elt))==0) break; \
+ } \
+} while(0)
+
+#define LL_REPLACE_ELEM(head, el, add) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el)->next; \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } else { \
+ _tmp = head; \
+ while (_tmp->next && (_tmp->next != (el))) { \
+ _tmp = _tmp->next; \
+ } \
+ if (_tmp->next) { \
+ _tmp->next = (add); \
+ } \
+ } \
+} while (0)
+
+#define LL_PREPEND_ELEM(head, el, add) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } else { \
+ _tmp = head; \
+ while (_tmp->next && (_tmp->next != (el))) { \
+ _tmp = _tmp->next; \
+ } \
+ if (_tmp->next) { \
+ _tmp->next = (add); \
+ } \
+ } \
+} while (0) \
+
+
+/******************************************************************************
+ * doubly linked list macros (non-circular) *
+ *****************************************************************************/
+#define DL_PREPEND(head,add) \
+ DL_PREPEND2(head,add,prev,next)
+
+#define DL_PREPEND2(head,add,prev,next) \
+do { \
+ (add)->next = head; \
+ if (head) { \
+ (add)->prev = (head)->prev; \
+ (head)->prev = (add); \
+ } else { \
+ (add)->prev = (add); \
+ } \
+ (head) = (add); \
+} while (0)
+
+#define DL_APPEND(head,add) \
+ DL_APPEND2(head,add,prev,next)
+
+#define DL_APPEND2(head,add,prev,next) \
+do { \
+ if (head) { \
+ (add)->prev = (head)->prev; \
+ (head)->prev->next = (add); \
+ (head)->prev = (add); \
+ (add)->next = NULL; \
+ } else { \
+ (head)=(add); \
+ (head)->prev = (head); \
+ (head)->next = NULL; \
+ } \
+} while (0)
+
+#define DL_CONCAT(head1,head2) \
+ DL_CONCAT2(head1,head2,prev,next)
+
+#define DL_CONCAT2(head1,head2,prev,next) \
+do { \
+ LDECLTYPE(head1) _tmp; \
+ if (head2) { \
+ if (head1) { \
+ _tmp = (head2)->prev; \
+ (head2)->prev = (head1)->prev; \
+ (head1)->prev->next = (head2); \
+ (head1)->prev = _tmp; \
+ } else { \
+ (head1)=(head2); \
+ } \
+ } \
+} while (0)
+
+#define DL_DELETE(head,del) \
+ DL_DELETE2(head,del,prev,next)
+
+#define DL_DELETE2(head,del,prev,next) \
+do { \
+ assert((del)->prev != NULL); \
+ if ((del)->prev == (del)) { \
+ (head)=NULL; \
+ } else if ((del)==(head)) { \
+ (del)->next->prev = (del)->prev; \
+ (head) = (del)->next; \
+ } else { \
+ (del)->prev->next = (del)->next; \
+ if ((del)->next) { \
+ (del)->next->prev = (del)->prev; \
+ } else { \
+ (head)->prev = (del)->prev; \
+ } \
+ } \
+} while (0)
+
+#define DL_COUNT(head,el,counter) \
+ DL_COUNT2(head,el,counter,next) \
+
+#define DL_COUNT2(head,el,counter,next) \
+{ \
+ counter = 0; \
+ DL_FOREACH2(head,el,next){ ++counter; } \
+}
+
+#define DL_FOREACH(head,el) \
+ DL_FOREACH2(head,el,next)
+
+#define DL_FOREACH2(head,el,next) \
+ for(el=head;el;el=(el)->next)
+
+/* this version is safe for deleting the elements during iteration */
+#define DL_FOREACH_SAFE(head,el,tmp) \
+ DL_FOREACH_SAFE2(head,el,tmp,next)
+
+#define DL_FOREACH_SAFE2(head,el,tmp,next) \
+ for((el)=(head);(el) && (tmp = (el)->next, 1); (el) = tmp)
+
+/* these are identical to their singly-linked list counterparts */
+#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR
+#define DL_SEARCH LL_SEARCH
+#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2
+#define DL_SEARCH2 LL_SEARCH2
+
+#define DL_REPLACE_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ (add)->next = (el)->next; \
+ if ((el)->next == NULL) { \
+ (add)->prev = (add); \
+ } else { \
+ (add)->prev = (el)->prev; \
+ (add)->next->prev = (add); \
+ } \
+ } else { \
+ (add)->next = (el)->next; \
+ (add)->prev = (el)->prev; \
+ (add)->prev->next = (add); \
+ if ((el)->next == NULL) { \
+ (head)->prev = (add); \
+ } else { \
+ (add)->next->prev = (add); \
+ } \
+ } \
+} while (0)
+
+#define DL_PREPEND_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el); \
+ (add)->prev = (el)->prev; \
+ (el)->prev = (add); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } else { \
+ (add)->prev->next = (add); \
+ } \
+} while (0) \
+
+
+/******************************************************************************
+ * circular doubly linked list macros *
+ *****************************************************************************/
+#define CDL_PREPEND(head,add) \
+ CDL_PREPEND2(head,add,prev,next)
+
+#define CDL_PREPEND2(head,add,prev,next) \
+do { \
+ if (head) { \
+ (add)->prev = (head)->prev; \
+ (add)->next = (head); \
+ (head)->prev = (add); \
+ (add)->prev->next = (add); \
+ } else { \
+ (add)->prev = (add); \
+ (add)->next = (add); \
+ } \
+(head)=(add); \
+} while (0)
+
+#define CDL_DELETE(head,del) \
+ CDL_DELETE2(head,del,prev,next)
+
+#define CDL_DELETE2(head,del,prev,next) \
+do { \
+ if ( ((head)==(del)) && ((head)->next == (head))) { \
+ (head) = 0L; \
+ } else { \
+ (del)->next->prev = (del)->prev; \
+ (del)->prev->next = (del)->next; \
+ if ((del) == (head)) (head)=(del)->next; \
+ } \
+} while (0)
+
+#define CDL_COUNT(head,el,counter) \
+ CDL_COUNT2(head,el,counter,next) \
+
+#define CDL_COUNT2(head, el, counter,next) \
+{ \
+ counter = 0; \
+ CDL_FOREACH2(head,el,next){ ++counter; } \
+}
+
+#define CDL_FOREACH(head,el) \
+ CDL_FOREACH2(head,el,next)
+
+#define CDL_FOREACH2(head,el,next) \
+ for(el=head;el;el=((el)->next==head ? 0L : (el)->next))
+
+#define CDL_FOREACH_SAFE(head,el,tmp1,tmp2) \
+ CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next)
+
+#define CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) \
+ for((el)=(head), ((tmp1)=(head)?((head)->prev):NULL); \
+ (el) && ((tmp2)=(el)->next, 1); \
+ ((el) = (((el)==(tmp1)) ? 0L : (tmp2))))
+
+#define CDL_SEARCH_SCALAR(head,out,field,val) \
+ CDL_SEARCH_SCALAR2(head,out,field,val,next)
+
+#define CDL_SEARCH_SCALAR2(head,out,field,val,next) \
+do { \
+ CDL_FOREACH2(head,out,next) { \
+ if ((out)->field == (val)) break; \
+ } \
+} while(0)
+
+#define CDL_SEARCH(head,out,elt,cmp) \
+ CDL_SEARCH2(head,out,elt,cmp,next)
+
+#define CDL_SEARCH2(head,out,elt,cmp,next) \
+do { \
+ CDL_FOREACH2(head,out,next) { \
+ if ((cmp(out,elt))==0) break; \
+ } \
+} while(0)
+
+#define CDL_REPLACE_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ if ((el)->next == (el)) { \
+ (add)->next = (add); \
+ (add)->prev = (add); \
+ (head) = (add); \
+ } else { \
+ (add)->next = (el)->next; \
+ (add)->prev = (el)->prev; \
+ (add)->next->prev = (add); \
+ (add)->prev->next = (add); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } \
+ } \
+} while (0)
+
+#define CDL_PREPEND_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el); \
+ (add)->prev = (el)->prev; \
+ (el)->prev = (add); \
+ (add)->prev->next = (add); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } \
+} while (0) \
+
+#endif /* UTLIST_H */
+
diff --git a/contrib/uthash/utstring.h b/contrib/uthash/utstring.h
new file mode 100644
index 0000000..6130fac
--- /dev/null
+++ b/contrib/uthash/utstring.h
@@ -0,0 +1,415 @@
+/*
+Copyright (c) 2008-2013, Troy D. Hanson http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/* a dynamic string implementation using macros
+ */
+#ifndef UTSTRING_H
+#define UTSTRING_H
+
+#define UTSTRING_VERSION 1.9.8
+
+#ifdef __GNUC__
+#define _UNUSED_ __attribute__ ((__unused__))
+#else
+#define _UNUSED_
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+
+#ifndef oom
+#define oom() exit(-1)
+#endif
+
+typedef struct {
+ char *d;
+ void **pd;
+ size_t n; /* allocd size */
+ size_t i; /* index of first unused byte */
+} UT_string;
+
+#define utstring_reserve(s,amt) \
+do { \
+ if (((s)->n - (s)->i) < (size_t)(amt)) { \
+ (s)->d = (char*)realloc((s)->d, (s)->n + amt); \
+ if ((s)->d == NULL) oom(); \
+ (s)->n += amt; \
+ if ((s)->pd) *((s)->pd) = (s)->d; \
+ } \
+} while(0)
+
+#define utstring_init(s) \
+do { \
+ (s)->n = 0; (s)->i = 0; (s)->d = NULL; \
+ utstring_reserve(s,128); \
+ (s)->d[0] = '\0'; \
+} while(0)
+
+#define utstring_done(s) \
+do { \
+ if ((s)->d != NULL) free((s)->d); \
+ (s)->n = 0; \
+} while(0)
+
+#define utstring_free(s) \
+do { \
+ utstring_done(s); \
+ free(s); \
+} while(0)
+
+#define utstring_new(s) \
+do { \
+ s = (UT_string*)calloc(1, sizeof(UT_string)); \
+ if (!s) oom(); \
+ utstring_init(s); \
+} while(0)
+
+#define utstring_renew(s) \
+do { \
+ if (s) { \
+ utstring_clear(s); \
+ } else { \
+ utstring_new(s); \
+ } \
+} while(0)
+
+#define utstring_clear(s) \
+do { \
+ (s)->i = 0; \
+ (s)->d[0] = '\0'; \
+} while(0)
+
+#define utstring_bincpy(s,b,l) \
+do { \
+ utstring_reserve((s),(l)+1); \
+ if (l) memcpy(&(s)->d[(s)->i], b, l); \
+ (s)->i += l; \
+ (s)->d[(s)->i]='\0'; \
+} while(0)
+
+#define utstring_concat(dst,src) \
+do { \
+ utstring_reserve((dst),((src)->i)+1); \
+ if ((src)->i) memcpy(&(dst)->d[(dst)->i], (src)->d, (src)->i); \
+ (dst)->i += (src)->i; \
+ (dst)->d[(dst)->i]='\0'; \
+} while(0)
+
+#define utstring_len(s) ((unsigned)((s)->i))
+
+#define utstring_body(s) ((s)->d)
+
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+_UNUSED_ static void utstring_printf_va(UT_string *s, const char *fmt, va_list ap) {
+ int n;
+ va_list cp;
+ while (1) {
+#ifdef _WIN32
+ cp = ap;
+#else
+ va_copy(cp, ap);
+#endif
+ n = vsnprintf (&s->d[s->i], s->n-s->i, fmt, cp);
+ va_end(cp);
+
+ if ((n > -1) && (n < (int)(s->n-s->i))) {
+ s->i += n;
+ return;
+ }
+
+ /* Else try again with more space. */
+ if (n > -1) utstring_reserve(s,n+1); /* exact */
+ else utstring_reserve(s,(s->n)*2); /* 2x */
+ }
+}
+#ifdef __GNUC__
+/* support printf format checking (2=the format string, 3=start of varargs) */
+static void utstring_printf(UT_string *s, const char *fmt, ...)
+ __attribute__ (( format( printf, 2, 3) ));
+#endif
+_UNUSED_ static void utstring_printf(UT_string *s, const char *fmt, ...) {
+ va_list ap;
+ va_start(ap,fmt);
+ utstring_printf_va(s,fmt,ap);
+ va_end(ap);
+}
+
+#define utstring_append_len(dst, src, len) \
+do { \
+ while ((dst)->n-(dst)->i <= (len)) utstring_reserve((dst),((dst)->n)*2); \
+ memcpy(&(dst)->d[(dst)->i], (src), (len)); \
+ (dst)->i+=(len); \
+ (dst)->d[(dst)->i]='\0'; \
+} while(0)
+
+#define utstring_append_c(dst, c) \
+do { \
+ if ((dst)->n-(dst)->i < 2) utstring_reserve((dst),((dst)->n)*2); \
+ (dst)->d[(dst)->i++] = (c); \
+ (dst)->d[(dst)->i]='\0'; \
+} while(0)
+
+/*******************************************************************************
+ * begin substring search functions *
+ ******************************************************************************/
+/* Build KMP table from left to right. */
+_UNUSED_ static void _utstring_BuildTable(
+ const char *P_Needle,
+ ssize_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+
+ i = 0;
+ j = i - 1;
+ P_KMP_Table[i] = j;
+ while (i < P_NeedleLen)
+ {
+ while ( (j > -1) && (P_Needle[i] != P_Needle[j]) )
+ {
+ j = P_KMP_Table[j];
+ }
+ i++;
+ j++;
+ if (i < P_NeedleLen)
+ {
+ if (P_Needle[i] == P_Needle[j])
+ {
+ P_KMP_Table[i] = P_KMP_Table[j];
+ }
+ else
+ {
+ P_KMP_Table[i] = j;
+ }
+ }
+ else
+ {
+ P_KMP_Table[i] = j;
+ }
+ }
+
+ return;
+}
+
+
+/* Build KMP table from right to left. */
+_UNUSED_ static void _utstring_BuildTableR(
+ const char *P_Needle,
+ ssize_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+
+ i = P_NeedleLen - 1;
+ j = i + 1;
+ P_KMP_Table[i + 1] = j;
+ while (i >= 0)
+ {
+ while ( (j < P_NeedleLen) && (P_Needle[i] != P_Needle[j]) )
+ {
+ j = P_KMP_Table[j + 1];
+ }
+ i--;
+ j--;
+ if (i >= 0)
+ {
+ if (P_Needle[i] == P_Needle[j])
+ {
+ P_KMP_Table[i + 1] = P_KMP_Table[j + 1];
+ }
+ else
+ {
+ P_KMP_Table[i + 1] = j;
+ }
+ }
+ else
+ {
+ P_KMP_Table[i + 1] = j;
+ }
+ }
+
+ return;
+}
+
+
+/* Search data from left to right. ( Multiple search mode. ) */
+_UNUSED_ static long _utstring_find(
+ const char *P_Haystack,
+ size_t P_HaystackLen,
+ const char *P_Needle,
+ size_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+ long V_FindPosition = -1;
+
+ /* Search from left to right. */
+ i = j = 0;
+ while ( (j < (int)P_HaystackLen) && (((P_HaystackLen - j) + i) >= P_NeedleLen) )
+ {
+ while ( (i > -1) && (P_Needle[i] != P_Haystack[j]) )
+ {
+ i = P_KMP_Table[i];
+ }
+ i++;
+ j++;
+ if (i >= (int)P_NeedleLen)
+ {
+ /* Found. */
+ V_FindPosition = j - i;
+ break;
+ }
+ }
+
+ return V_FindPosition;
+}
+
+
+/* Search data from right to left. ( Multiple search mode. ) */
+_UNUSED_ static long _utstring_findR(
+ const char *P_Haystack,
+ size_t P_HaystackLen,
+ const char *P_Needle,
+ size_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+ long V_FindPosition = -1;
+
+ /* Search from right to left. */
+ j = (P_HaystackLen - 1);
+ i = (P_NeedleLen - 1);
+ while ( (j >= 0) && (j >= i) )
+ {
+ while ( (i < (int)P_NeedleLen) && (P_Needle[i] != P_Haystack[j]) )
+ {
+ i = P_KMP_Table[i + 1];
+ }
+ i--;
+ j--;
+ if (i < 0)
+ {
+ /* Found. */
+ V_FindPosition = j + 1;
+ break;
+ }
+ }
+
+ return V_FindPosition;
+}
+
+
+/* Search data from left to right. ( One time search mode. ) */
+_UNUSED_ static long utstring_find(
+ UT_string *s,
+ long P_StartPosition, /* Start from 0. -1 means last position. */
+ const char *P_Needle,
+ ssize_t P_NeedleLen)
+{
+ long V_StartPosition;
+ long V_HaystackLen;
+ long *V_KMP_Table;
+ long V_FindPosition = -1;
+
+ if (P_StartPosition < 0)
+ {
+ V_StartPosition = s->i + P_StartPosition;
+ }
+ else
+ {
+ V_StartPosition = P_StartPosition;
+ }
+ V_HaystackLen = s->i - V_StartPosition;
+ if ( (V_HaystackLen >= P_NeedleLen) && (P_NeedleLen > 0) )
+ {
+ V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1));
+ if (V_KMP_Table != NULL)
+ {
+ _utstring_BuildTable(P_Needle, P_NeedleLen, V_KMP_Table);
+
+ V_FindPosition = _utstring_find(s->d + V_StartPosition,
+ V_HaystackLen,
+ P_Needle,
+ P_NeedleLen,
+ V_KMP_Table);
+ if (V_FindPosition >= 0)
+ {
+ V_FindPosition += V_StartPosition;
+ }
+
+ free(V_KMP_Table);
+ }
+ }
+
+ return V_FindPosition;
+}
+
+
+/* Search data from right to left. ( One time search mode. ) */
+_UNUSED_ static long utstring_findR(
+ UT_string *s,
+ long P_StartPosition, /* Start from 0. -1 means last position. */
+ const char *P_Needle,
+ ssize_t P_NeedleLen)
+{
+ long V_StartPosition;
+ long V_HaystackLen;
+ long *V_KMP_Table;
+ long V_FindPosition = -1;
+
+ if (P_StartPosition < 0)
+ {
+ V_StartPosition = s->i + P_StartPosition;
+ }
+ else
+ {
+ V_StartPosition = P_StartPosition;
+ }
+ V_HaystackLen = V_StartPosition + 1;
+ if ( (V_HaystackLen >= P_NeedleLen) && (P_NeedleLen > 0) )
+ {
+ V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1));
+ if (V_KMP_Table != NULL)
+ {
+ _utstring_BuildTableR(P_Needle, P_NeedleLen, V_KMP_Table);
+
+ V_FindPosition = _utstring_findR(s->d,
+ V_HaystackLen,
+ P_Needle,
+ P_NeedleLen,
+ V_KMP_Table);
+
+ free(V_KMP_Table);
+ }
+ }
+
+ return V_FindPosition;
+}
+/*******************************************************************************
+ * end substring search functions *
+ ******************************************************************************/
+
+#endif /* UTSTRING_H */
diff --git a/contrib/xxhash/CMakeLists.txt b/contrib/xxhash/CMakeLists.txt
new file mode 100644
index 0000000..fa69efb
--- /dev/null
+++ b/contrib/xxhash/CMakeLists.txt
@@ -0,0 +1,13 @@
+SET(XXHASHSRC xxhash.c)
+
+ADD_LIBRARY(xxhash STATIC ${XXHASHSRC})
+
+IF(ENABLE_FULL_DEBUG MATCHES "OFF")
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+ SET_TARGET_PROPERTIES(xxhash PROPERTIES COMPILE_FLAGS "-O3")
+endif ()
+else()
+ADD_DEFINITIONS(-DXXH_NO_INLINE_HINTS=1)
+ENDIF()
+
+set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DXXH_NO_INLINE_HINTS=1" PARENT_SCOPE)
diff --git a/contrib/xxhash/LICENSE b/contrib/xxhash/LICENSE
new file mode 100644
index 0000000..7de801e
--- /dev/null
+++ b/contrib/xxhash/LICENSE
@@ -0,0 +1,24 @@
+xxHash Library
+Copyright (c) 2012-2014, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/xxhash/xxh3.h b/contrib/xxhash/xxh3.h
new file mode 100644
index 0000000..f7dc195
--- /dev/null
+++ b/contrib/xxhash/xxh3.h
@@ -0,0 +1,55 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Development source file for `xxh3`
+ * Copyright (C) 2019-2020 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://www.xxhash.com
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+/*
+ * Note: This file used to host the source code of XXH3_* variants.
+ * during the development period.
+ * The source code is now properly integrated within xxhash.h.
+ *
+ * xxh3.h is no longer useful,
+ * but it is still provided for compatibility with source code
+ * which used to include it directly.
+ *
+ * Programs are now highly discouraged to include xxh3.h.
+ * Include `xxhash.h` instead, which is the officially supported interface.
+ *
+ * In the future, xxh3.h will start to generate warnings, then errors,
+ * then it will be removed from source package and from include directory.
+ */
+
+/* Simulate the same impact as including the old xxh3.h source file */
+
+#define XXH_INLINE_ALL
+#include "xxhash.h"
diff --git a/contrib/xxhash/xxhash.c b/contrib/xxhash/xxhash.c
new file mode 100644
index 0000000..0fae88c
--- /dev/null
+++ b/contrib/xxhash/xxhash.c
@@ -0,0 +1,43 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2020 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://www.xxhash.com
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+
+/*
+ * xxhash.c instantiates functions defined in xxhash.h
+ */
+
+#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
+#define XXH_IMPLEMENTATION /* access definitions */
+
+#include "xxhash.h"
diff --git a/contrib/xxhash/xxhash.h b/contrib/xxhash/xxhash.h
new file mode 100644
index 0000000..08ab794
--- /dev/null
+++ b/contrib/xxhash/xxhash.h
@@ -0,0 +1,5580 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Header File
+ * Copyright (C) 2012-2020 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://www.xxhash.com
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+/*!
+ * @mainpage xxHash
+ *
+ * @file xxhash.h
+ * xxHash prototypes and implementation
+ */
+/* TODO: update */
+/* Notice extracted from xxHash homepage:
+
+xxHash is an extremely fast hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name Speed Q.Score Author
+xxHash 5.4 GB/s 10
+CrapWow 3.2 GB/s 2 Andrew
+MurmurHash 3a 2.7 GB/s 10 Austin Appleby
+SpookyHash 2.0 GB/s 10 Bob Jenkins
+SBox 1.4 GB/s 9 Bret Mulvey
+Lookup3 1.2 GB/s 9 Bob Jenkins
+SuperFastHash 1.2 GB/s 1 Paul Hsieh
+CityHash64 1.05 GB/s 10 Pike & Alakuijala
+FNV 0.55 GB/s 5 Fowler, Noll, Vo
+CRC32 0.43 GB/s 9
+MD5-32 0.33 GB/s 10 Ronald L. Rivest
+SHA1-32 0.28 GB/s 10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+Note: SMHasher's CRC32 implementation is not the fastest one.
+Other speed-oriented implementations can be faster,
+especially in combination with PCLMUL instruction:
+https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735
+
+A 64-bit version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bit applications only.
+Name Speed on 64 bits Speed on 32 bits
+XXH64 13.8 GB/s 1.9 GB/s
+XXH32 6.8 GB/s 6.0 GB/s
+*/
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* ****************************
+ * INLINE mode
+ ******************************/
+/*!
+ * XXH_INLINE_ALL (and XXH_PRIVATE_API)
+ * Use these build macros to inline xxhash into the target unit.
+ * Inlining improves performance on small inputs, especially when the length is
+ * expressed as a compile-time constant:
+ *
+ * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
+ *
+ * It also keeps xxHash symbols private to the unit, so they are not exported.
+ *
+ * Usage:
+ * #define XXH_INLINE_ALL
+ * #include "xxhash.h"
+ *
+ * Do not compile and link xxhash.o as a separate object, as it is not useful.
+ */
+#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
+ && !defined(XXH_INLINE_ALL_31684351384)
+ /* this section should be traversed only once */
+# define XXH_INLINE_ALL_31684351384
+ /* give access to the advanced API, required to compile implementations */
+# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */
+# define XXH_STATIC_LINKING_ONLY
+ /* make all functions private */
+# undef XXH_PUBLIC_API
+# if defined(__GNUC__)
+# define XXH_PUBLIC_API static __inline __attribute__((unused))
+# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define XXH_PUBLIC_API static inline
+# elif defined(_MSC_VER)
+# define XXH_PUBLIC_API static __inline
+# else
+ /* note: this version may generate warnings for unused static functions */
+# define XXH_PUBLIC_API static
+# endif
+
+ /*
+ * This part deals with the special case where a unit wants to inline xxHash,
+ * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
+ * such as part of some previously included *.h header file.
+ * Without further action, the new include would just be ignored,
+ * and functions would effectively _not_ be inlined (silent failure).
+ * The following macros solve this situation by prefixing all inlined names,
+ * avoiding naming collision with previous inclusions.
+ */
+ /* Before that, we unconditionally #undef all symbols,
+ * in case they were already defined with XXH_NAMESPACE.
+ * They will then be redefined for XXH_INLINE_ALL
+ */
+# undef XXH_versionNumber
+ /* XXH32 */
+# undef XXH32
+# undef XXH32_createState
+# undef XXH32_freeState
+# undef XXH32_reset
+# undef XXH32_update
+# undef XXH32_digest
+# undef XXH32_copyState
+# undef XXH32_canonicalFromHash
+# undef XXH32_hashFromCanonical
+ /* XXH64 */
+# undef XXH64
+# undef XXH64_createState
+# undef XXH64_freeState
+# undef XXH64_reset
+# undef XXH64_update
+# undef XXH64_digest
+# undef XXH64_copyState
+# undef XXH64_canonicalFromHash
+# undef XXH64_hashFromCanonical
+ /* XXH3_64bits */
+# undef XXH3_64bits
+# undef XXH3_64bits_withSecret
+# undef XXH3_64bits_withSeed
+# undef XXH3_64bits_withSecretandSeed
+# undef XXH3_createState
+# undef XXH3_freeState
+# undef XXH3_copyState
+# undef XXH3_64bits_reset
+# undef XXH3_64bits_reset_withSeed
+# undef XXH3_64bits_reset_withSecret
+# undef XXH3_64bits_update
+# undef XXH3_64bits_digest
+# undef XXH3_generateSecret
+ /* XXH3_128bits */
+# undef XXH128
+# undef XXH3_128bits
+# undef XXH3_128bits_withSeed
+# undef XXH3_128bits_withSecret
+# undef XXH3_128bits_reset
+# undef XXH3_128bits_reset_withSeed
+# undef XXH3_128bits_reset_withSecret
+# undef XXH3_128bits_reset_withSecretandSeed
+# undef XXH3_128bits_update
+# undef XXH3_128bits_digest
+# undef XXH128_isEqual
+# undef XXH128_cmp
+# undef XXH128_canonicalFromHash
+# undef XXH128_hashFromCanonical
+ /* Finally, free the namespace itself */
+# undef XXH_NAMESPACE
+
+ /* employ the namespace for XXH_INLINE_ALL */
+# define XXH_NAMESPACE XXH_INLINE_
+ /*
+ * Some identifiers (enums, type names) are not symbols,
+ * but they must nonetheless be renamed to avoid redeclaration.
+ * Alternative solution: do not redeclare them.
+ * However, this requires some #ifdefs, and has a more dispersed impact.
+ * Meanwhile, renaming can be achieved in a single place.
+ */
+# define XXH_IPREF(Id) XXH_NAMESPACE ## Id
+# define XXH_OK XXH_IPREF(XXH_OK)
+# define XXH_ERROR XXH_IPREF(XXH_ERROR)
+# define XXH_errorcode XXH_IPREF(XXH_errorcode)
+# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t)
+# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t)
+# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
+# define XXH32_state_s XXH_IPREF(XXH32_state_s)
+# define XXH32_state_t XXH_IPREF(XXH32_state_t)
+# define XXH64_state_s XXH_IPREF(XXH64_state_s)
+# define XXH64_state_t XXH_IPREF(XXH64_state_t)
+# define XXH3_state_s XXH_IPREF(XXH3_state_s)
+# define XXH3_state_t XXH_IPREF(XXH3_state_t)
+# define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
+ /* Ensure the header is parsed again, even if it was previously included */
+# undef XXHASH_H_5627135585666179
+# undef XXHASH_H_STATIC_13879238742
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+
+
+
+/* ****************************************************************
+ * Stable API
+ *****************************************************************/
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+
+/*!
+ * @defgroup public Public API
+ * Contains details on the public xxHash functions.
+ * @{
+ */
+/* specific declaration modes for Windows */
+#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
+# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
+# ifdef XXH_EXPORT
+# define XXH_PUBLIC_API __declspec(dllexport)
+# elif XXH_IMPORT
+# define XXH_PUBLIC_API __declspec(dllimport)
+# endif
+# else
+# define XXH_PUBLIC_API /* do nothing */
+# endif
+#endif
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Emulate a namespace by transparently prefixing all symbols.
+ *
+ * If you want to include _and expose_ xxHash functions from within your own
+ * library, but also want to avoid symbol collisions with other libraries which
+ * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix
+ * any public symbol from xxhash library with the value of XXH_NAMESPACE
+ * (therefore, avoid empty or numeric values).
+ *
+ * Note that no change is required within the calling program as long as it
+ * includes `xxhash.h`: Regular symbol names will be automatically translated
+ * by this header.
+ */
+# define XXH_NAMESPACE /* YOUR NAME HERE */
+# undef XXH_NAMESPACE
+#endif
+
+#ifdef XXH_NAMESPACE
+# define XXH_CAT(A,B) A##B
+# define XXH_NAME2(A,B) XXH_CAT(A,B)
+# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+/* XXH32 */
+# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+/* XXH64 */
+# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+/* XXH3_64bits */
+# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
+# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
+# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
+# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
+# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
+# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
+# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
+# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
+# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
+# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
+# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
+# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
+# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
+# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
+# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
+/* XXH3_128bits */
+# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
+# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
+# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
+# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
+# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
+# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
+# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
+# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
+# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
+# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
+# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
+# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
+# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
+# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
+# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
+#endif
+
+
+/* *************************************
+* Version
+***************************************/
+#define XXH_VERSION_MAJOR 0
+#define XXH_VERSION_MINOR 8
+#define XXH_VERSION_RELEASE 1
+#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+
+/*!
+ * @brief Obtains the xxHash version.
+ *
+ * This is mostly useful when xxHash is compiled as a shared library,
+ * since the returned value comes from the library, as opposed to header file.
+ *
+ * @return `XXH_VERSION_NUMBER` of the invoked library.
+ */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
+
+
+/* ****************************
+* Common basic types
+******************************/
+#include <stddef.h> /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/*-**********************************************************************
+* 32-bit hash
+************************************************************************/
+#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
+/*!
+ * @brief An unsigned 32-bit integer.
+ *
+ * Not necessarily defined to `uint32_t` but functionally equivalent.
+ */
+typedef uint32_t XXH32_hash_t;
+
+#elif !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint32_t XXH32_hash_t;
+
+#else
+# include <limits.h>
+# if UINT_MAX == 0xFFFFFFFFUL
+ typedef unsigned int XXH32_hash_t;
+# else
+# if ULONG_MAX == 0xFFFFFFFFUL
+ typedef unsigned long XXH32_hash_t;
+# else
+# error "unsupported platform: need a 32-bit type"
+# endif
+# endif
+#endif
+
+/*!
+ * @}
+ *
+ * @defgroup xxh32_family XXH32 family
+ * @ingroup public
+ * Contains functions used in the classic 32-bit xxHash algorithm.
+ *
+ * @note
+ * XXH32 is useful for older platforms, with no or poor 64-bit performance.
+ * Note that @ref xxh3_family provides competitive speed
+ * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results.
+ *
+ * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families
+ * @see @ref xxh32_impl for implementation details
+ * @{
+ */
+
+/*!
+ * @brief Calculates the 32-bit hash of @p input using xxHash32.
+ *
+ * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
+ *
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed The 32-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 32-bit hash value.
+ *
+ * @see
+ * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
+ * Direct equivalents for the other variants of xxHash.
+ * @see
+ * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version.
+ */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
+
+/*!
+ * Streaming functions generate the xxHash value from an incremental input.
+ * This method is slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * An XXH state must first be allocated using `XXH*_createState()`.
+ *
+ * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
+ *
+ * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
+ *
+ * The function returns an error code, with 0 meaning OK, and any other value
+ * meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a
+ * digest, and generate new hash values later on by invoking `XXH*_digest()`.
+ *
+ * When done, release the state using `XXH*_freeState()`.
+ *
+ * Example code for incrementally hashing a file:
+ * @code{.c}
+ * #include <stdio.h>
+ * #include <xxhash.h>
+ * #define BUFFER_SIZE 256
+ *
+ * // Note: XXH64 and XXH3 use the same interface.
+ * XXH32_hash_t
+ * hashFile(FILE* stream)
+ * {
+ * XXH32_state_t* state;
+ * unsigned char buf[BUFFER_SIZE];
+ * size_t amt;
+ * XXH32_hash_t hash;
+ *
+ * state = XXH32_createState(); // Create a state
+ * assert(state != NULL); // Error check here
+ * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed
+ * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) {
+ * XXH32_update(state, buf, amt); // Hash the file in chunks
+ * }
+ * hash = XXH32_digest(state); // Finalize the hash
+ * XXH32_freeState(state); // Clean up
+ * return hash;
+ * }
+ * @endcode
+ */
+
+/*!
+ * @typedef struct XXH32_state_s XXH32_state_t
+ * @brief The opaque state struct for the XXH32 streaming API.
+ *
+ * @see XXH32_state_s for details.
+ */
+typedef struct XXH32_state_s XXH32_state_t;
+
+/*!
+ * @brief Allocates an @ref XXH32_state_t.
+ *
+ * Must be freed with XXH32_freeState().
+ * @return An allocated XXH32_state_t on success, `NULL` on failure.
+ */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+/*!
+ * @brief Frees an @ref XXH32_state_t.
+ *
+ * Must be allocated with XXH32_createState().
+ * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
+ * @return XXH_OK.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
+/*!
+ * @brief Copies one @ref XXH32_state_t to another.
+ *
+ * @param dst_state The state to copy to.
+ * @param src_state The state to copy from.
+ * @pre
+ * @p dst_state and @p src_state must not be `NULL` and must not overlap.
+ */
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+/*!
+ * @brief Resets an @ref XXH32_state_t to begin a new hash.
+ *
+ * This function resets and seeds a state. Call it before @ref XXH32_update().
+ *
+ * @param statePtr The state struct to reset.
+ * @param seed The 32-bit seed to alter the hash result predictably.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
+
+/*!
+ * @brief Consumes a block of @p input to an @ref XXH32_state_t.
+ *
+ * Call this to incrementally consume blocks of data.
+ *
+ * @param statePtr The state struct to update.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+
+/*!
+ * @brief Returns the calculated hash value from an @ref XXH32_state_t.
+ *
+ * @note
+ * Calling XXH32_digest() will not affect @p statePtr, so you can update,
+ * digest, and update again.
+ *
+ * @param statePtr The state struct to calculate the hash from.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ *
+ * @return The calculated xxHash32 value from that state.
+ */
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
+
+/******* Canonical representation *******/
+
+/*
+ * The default return values from XXH functions are unsigned 32 and 64 bit
+ * integers.
+ * This the simplest and fastest format for further post-processing.
+ *
+ * However, this leaves open the question of what is the order on the byte level,
+ * since little and big endian conventions will store the same number differently.
+ *
+ * The canonical representation settles this issue by mandating big-endian
+ * convention, the same convention as human-readable numbers (large digits first).
+ *
+ * When writing hash values to storage, sending them over a network, or printing
+ * them, it's highly recommended to use the canonical representation to ensure
+ * portability across a wider range of systems, present and future.
+ *
+ * The following functions allow transformation of hash values to and from
+ * canonical format.
+ */
+
+/*!
+ * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
+ */
+typedef struct {
+ unsigned char digest[4]; /*!< Hash bytes, big endian */
+} XXH32_canonical_t;
+
+/*!
+ * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
+ *
+ * @param dst The @ref XXH32_canonical_t pointer to be stored to.
+ * @param hash The @ref XXH32_hash_t to be converted.
+ *
+ * @pre
+ * @p dst must not be `NULL`.
+ */
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+
+/*!
+ * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
+ *
+ * @param src The @ref XXH32_canonical_t to convert.
+ *
+ * @pre
+ * @p src must not be `NULL`.
+ *
+ * @return The converted hash.
+ */
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+
+#ifdef __has_attribute
+# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
+#else
+# define XXH_HAS_ATTRIBUTE(x) 0
+#endif
+
+/* C-language Attributes are added in C23. */
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
+# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
+#else
+# define XXH_HAS_C_ATTRIBUTE(x) 0
+#endif
+
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define XXH_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+/*
+Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
+introduced in CPP17 and C23.
+CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
+C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough
+*/
+#if XXH_HAS_C_ATTRIBUTE(x)
+# define XXH_FALLTHROUGH [[fallthrough]]
+#elif XXH_HAS_CPP_ATTRIBUTE(x)
+# define XXH_FALLTHROUGH [[fallthrough]]
+#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
+# define XXH_FALLTHROUGH __attribute__ ((fallthrough))
+#else
+# define XXH_FALLTHROUGH
+#endif
+
+/*!
+ * @}
+ * @ingroup public
+ * @{
+ */
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+* 64-bit hash
+************************************************************************/
+#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
+/*!
+ * @brief An unsigned 64-bit integer.
+ *
+ * Not necessarily defined to `uint64_t` but functionally equivalent.
+ */
+typedef uint64_t XXH64_hash_t;
+#elif !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint64_t XXH64_hash_t;
+#else
+# include <limits.h>
+# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
+ /* LP64 ABI says uint64_t is unsigned long */
+ typedef unsigned long XXH64_hash_t;
+# else
+ /* the following type must have a width of 64-bit */
+ typedef unsigned long long XXH64_hash_t;
+# endif
+#endif
+
+/*!
+ * @}
+ *
+ * @defgroup xxh64_family XXH64 family
+ * @ingroup public
+ * @{
+ * Contains functions used in the classic 64-bit xxHash algorithm.
+ *
+ * @note
+ * XXH3 provides competitive speed for both 32-bit and 64-bit systems,
+ * and offers true 64/128 bit hash results.
+ * It provides better speed for systems with vector processing capabilities.
+ */
+
+
+/*!
+ * @brief Calculates the 64-bit hash of @p input using xxHash64.
+ *
+ * This function usually runs faster on 64-bit systems, but slower on 32-bit
+ * systems (see benchmark).
+ *
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed The 64-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 64-bit hash.
+ *
+ * @see
+ * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
+ * Direct equivalents for the other variants of xxHash.
+ * @see
+ * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version.
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed);
+
+/******* Streaming *******/
+/*!
+ * @brief The opaque state struct for the XXH64 streaming API.
+ *
+ * @see XXH64_state_s for details.
+ */
+typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
+
+/******* Canonical representation *******/
+typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+
+/*!
+ * @}
+ * ************************************************************************
+ * @defgroup xxh3_family XXH3 family
+ * @ingroup public
+ * @{
+ *
+ * XXH3 is a more recent hash algorithm featuring:
+ * - Improved speed for both small and large inputs
+ * - True 64-bit and 128-bit outputs
+ * - SIMD acceleration
+ * - Improved 32-bit viability
+ *
+ * Speed analysis methodology is explained here:
+ *
+ * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
+ *
+ * Compared to XXH64, expect XXH3 to run approximately
+ * ~2x faster on large inputs and >3x faster on small ones,
+ * exact differences vary depending on platform.
+ *
+ * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
+ * but does not require it.
+ * Any 32-bit and 64-bit targets that can run XXH32 smoothly
+ * can run XXH3 at competitive speeds, even without vector support.
+ * Further details are explained in the implementation.
+ *
+ * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8,
+ * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro.
+ *
+ * XXH3 implementation is portable:
+ * it has a generic C90 formulation that can be compiled on any platform,
+ * all implementations generage exactly the same hash value on all platforms.
+ * Starting from v0.8.0, it's also labelled "stable", meaning that
+ * any future version will also generate the same hash value.
+ *
+ * XXH3 offers 2 variants, _64bits and _128bits.
+ *
+ * When only 64 bits are needed, prefer invoking the _64bits variant, as it
+ * reduces the amount of mixing, resulting in faster speed on small inputs.
+ * It's also generally simpler to manipulate a scalar return type than a struct.
+ *
+ * The API supports one-shot hashing, streaming mode, and custom secrets.
+ */
+
+/*-**********************************************************************
+* XXH3 64-bit variant
+************************************************************************/
+
+/* XXH3_64bits():
+ * default 64-bit variant, using default secret and default seed of 0.
+ * It's the fastest variant. */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len);
+
+/*
+ * XXH3_64bits_withSeed():
+ * This variant generates a custom secret on the fly
+ * based on default secret altered using the `seed` value.
+ * While this operation is decently fast, note that it's not completely free.
+ * Note: seed==0 produces the same results as XXH3_64bits().
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
+
+/*!
+ * The bare minimum size for a custom secret.
+ *
+ * @see
+ * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
+ * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
+ */
+#define XXH3_SECRET_SIZE_MIN 136
+
+/*
+ * XXH3_64bits_withSecret():
+ * It's possible to provide any blob of bytes as a "secret" to generate the hash.
+ * This makes it more difficult for an external actor to prepare an intentional collision.
+ * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
+ * However, the quality of the secret impacts the dispersion of the hash algorithm.
+ * Therefore, the secret _must_ look like a bunch of random bytes.
+ * Avoid "trivial" or structured data such as repeated sequences or a text document.
+ * Whenever in doubt about the "randomness" of the blob of bytes,
+ * consider employing "XXH3_generateSecret()" instead (see below).
+ * It will generate a proper high entropy secret derived from the blob of bytes.
+ * Another advantage of using XXH3_generateSecret() is that
+ * it guarantees that all bits within the initial blob of bytes
+ * will impact every bit of the output.
+ * This is not necessarily the case when using the blob of bytes directly
+ * because, when hashing _small_ inputs, only a portion of the secret is employed.
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
+
+
+/******* Streaming *******/
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
+ * As a consequence, streaming is slower than one-shot hashing.
+ * For better performance, prefer one-shot functions whenever applicable.
+ */
+
+/*!
+ * @brief The state struct for the XXH3 streaming API.
+ *
+ * @see XXH3_state_s for details.
+ */
+typedef struct XXH3_state_s XXH3_state_t;
+XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
+XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state);
+
+/*
+ * XXH3_64bits_reset():
+ * Initialize with default parameters.
+ * digest will be equivalent to `XXH3_64bits()`.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr);
+/*
+ * XXH3_64bits_reset_withSeed():
+ * Generate a custom secret from `seed`, and store it into `statePtr`.
+ * digest will be equivalent to `XXH3_64bits_withSeed()`.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
+/*
+ * XXH3_64bits_reset_withSecret():
+ * `secret` is referenced, it _must outlive_ the hash streaming session.
+ * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
+ * and the quality of produced hash values depends on secret's entropy
+ * (secret's content should look like a bunch of random bytes).
+ * When in doubt about the randomness of a candidate `secret`,
+ * consider employing `XXH3_generateSecret()` instead (see below).
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
+
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr);
+
+/* note : canonical representation of XXH3 is the same as XXH64
+ * since they both produce XXH64_hash_t values */
+
+
+/*-**********************************************************************
+* XXH3 128-bit variant
+************************************************************************/
+
+/*!
+ * @brief The return value from 128-bit hashes.
+ *
+ * Stored in little endian order, although the fields themselves are in native
+ * endianness.
+ */
+typedef struct {
+ XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */
+ XXH64_hash_t high64; /*!< `value >> 64` */
+} XXH128_hash_t;
+
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
+
+/******* Streaming *******/
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
+ * As a consequence, streaming is slower than one-shot hashing.
+ * For better performance, prefer one-shot functions whenever applicable.
+ *
+ * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
+ * Use already declared XXH3_createState() and XXH3_freeState().
+ *
+ * All reset and streaming functions have same meaning as their 64-bit counterpart.
+ */
+
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr);
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
+
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr);
+
+/* Following helper functions make it possible to compare XXH128_hast_t values.
+ * Since XXH128_hash_t is a structure, this capability is not offered by the language.
+ * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
+
+/*!
+ * XXH128_isEqual():
+ * Return: 1 if `h1` and `h2` are equal, 0 if they are not.
+ */
+XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
+
+/*!
+ * XXH128_cmp():
+ *
+ * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
+ *
+ * return: >0 if *h128_1 > *h128_2
+ * =0 if *h128_1 == *h128_2
+ * <0 if *h128_1 < *h128_2
+ */
+XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2);
+
+
+/******* Canonical representation *******/
+typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
+XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash);
+XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src);
+
+
+#endif /* XXH_NO_LONG_LONG */
+
+/*!
+ * @}
+ */
+#endif /* XXHASH_H_5627135585666179 */
+
+
+
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
+#define XXHASH_H_STATIC_13879238742
+/* ****************************************************************************
+ * This section contains declarations which are not guaranteed to remain stable.
+ * They may change in future versions, becoming incompatible with a different
+ * version of the library.
+ * These declarations should only be used with static linking.
+ * Never use them in association with dynamic linking!
+ ***************************************************************************** */
+
+/*
+ * These definitions are only present to allow static allocation
+ * of XXH states, on stack or in a struct, for example.
+ * Never **ever** access their members directly.
+ */
+
+/*!
+ * @internal
+ * @brief Structure for XXH32 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
+ * an opaque type. This allows fields to safely be changed.
+ *
+ * Typedef'd to @ref XXH32_state_t.
+ * Do not access the members of this struct directly.
+ * @see XXH64_state_s, XXH3_state_s
+ */
+struct XXH32_state_s {
+ XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
+ XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
+ XXH32_hash_t v[4]; /*!< Accumulator lanes */
+ XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
+ XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */
+ XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may be removed. */
+}; /* typedef'd to XXH32_state_t */
+
+
+#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */
+
+/*!
+ * @internal
+ * @brief Structure for XXH64 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
+ * an opaque type. This allows fields to safely be changed.
+ *
+ * Typedef'd to @ref XXH64_state_t.
+ * Do not access the members of this struct directly.
+ * @see XXH32_state_s, XXH3_state_s
+ */
+struct XXH64_state_s {
+ XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */
+ XXH64_hash_t v[4]; /*!< Accumulator lanes */
+ XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
+ XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */
+ XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/
+ XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it may be removed. */
+}; /* typedef'd to XXH64_state_t */
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
+# include <stdalign.h>
+# define XXH_ALIGN(n) alignas(n)
+#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
+/* In C++ alignas() is a keyword */
+# define XXH_ALIGN(n) alignas(n)
+#elif defined(__GNUC__)
+# define XXH_ALIGN(n) __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+# define XXH_ALIGN(n) __declspec(align(n))
+#else
+# define XXH_ALIGN(n) /* disabled */
+#endif
+
+/* Old GCC versions only accept the attribute after the type in structures. */
+#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \
+ && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
+ && defined(__GNUC__)
+# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
+#else
+# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
+#endif
+
+/*!
+ * @brief The size of the internal XXH3 buffer.
+ *
+ * This is the optimal update size for incremental hashing.
+ *
+ * @see XXH3_64b_update(), XXH3_128b_update().
+ */
+#define XXH3_INTERNALBUFFER_SIZE 256
+
+/*!
+ * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
+ *
+ * This is the size used in @ref XXH3_kSecret and the seeded functions.
+ *
+ * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
+ */
+#define XXH3_SECRET_DEFAULT_SIZE 192
+
+/*!
+ * @internal
+ * @brief Structure for XXH3 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
+ * Otherwise it is an opaque type.
+ * Never use this definition in combination with dynamic library.
+ * This allows fields to safely be changed in the future.
+ *
+ * @note ** This structure has a strict alignment requirement of 64 bytes!! **
+ * Do not allocate this with `malloc()` or `new`,
+ * it will not be sufficiently aligned.
+ * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
+ *
+ * Typedef'd to @ref XXH3_state_t.
+ * Do never access the members of this struct directly.
+ *
+ * @see XXH3_INITSTATE() for stack initialization.
+ * @see XXH3_createState(), XXH3_freeState().
+ * @see XXH32_state_s, XXH64_state_s
+ */
+struct XXH3_state_s {
+ XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
+ /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */
+ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
+ /*!< Used to store a custom secret generated from a seed. */
+ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
+ /*!< The internal buffer. @see XXH32_state_s::mem32 */
+ XXH32_hash_t bufferedSize;
+ /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
+ XXH32_hash_t useSeed;
+ /*!< Reserved field. Needed for padding on 64-bit. */
+ size_t nbStripesSoFar;
+ /*!< Number or stripes processed. */
+ XXH64_hash_t totalLen;
+ /*!< Total length hashed. 64-bit even on 32-bit targets. */
+ size_t nbStripesPerBlock;
+ /*!< Number of stripes per block. */
+ size_t secretLimit;
+ /*!< Size of @ref customSecret or @ref extSecret */
+ XXH64_hash_t seed;
+ /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
+ XXH64_hash_t reserved64;
+ /*!< Reserved field. */
+ const unsigned char* extSecret;
+ /*!< Reference to an external secret for the _withSecret variants, NULL
+ * for other variants. */
+ /* note: there may be some padding at the end due to alignment on 64 bytes */
+}; /* typedef'd to XXH3_state_t */
+
+#undef XXH_ALIGN_MEMBER
+
+/*!
+ * @brief Initializes a stack-allocated `XXH3_state_s`.
+ *
+ * When the @ref XXH3_state_t structure is merely emplaced on stack,
+ * it should be initialized with XXH3_INITSTATE() or a memset()
+ * in case its first reset uses XXH3_NNbits_reset_withSeed().
+ * This init can be omitted if the first reset uses default or _withSecret mode.
+ * This operation isn't necessary when the state is created with XXH3_createState().
+ * Note that this doesn't prepare the state for a streaming operation,
+ * it's still necessary to use XXH3_NNbits_reset*() afterwards.
+ */
+#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; }
+
+
+/* XXH128() :
+ * simple alias to pre-selected XXH3_128bits variant
+ */
+XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed);
+
+
+/* === Experimental API === */
+/* Symbols defined below must be considered tied to a specific library version. */
+
+/*
+ * XXH3_generateSecret():
+ *
+ * Derive a high-entropy secret from any user-defined content, named customSeed.
+ * The generated secret can be used in combination with `*_withSecret()` functions.
+ * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed,
+ * as it becomes much more difficult for an external actor to guess how to impact the calculation logic.
+ *
+ * The function accepts as input a custom seed of any length and any content,
+ * and derives from it a high-entropy secret of length @secretSize
+ * into an already allocated buffer @secretBuffer.
+ * @secretSize must be >= XXH3_SECRET_SIZE_MIN
+ *
+ * The generated secret can then be used with any `*_withSecret()` variant.
+ * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`,
+ * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()`
+ * are part of this list. They all accept a `secret` parameter
+ * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN)
+ * _and_ feature very high entropy (consist of random-looking bytes).
+ * These conditions can be a high bar to meet, so
+ * XXH3_generateSecret() can be employed to ensure proper quality.
+ *
+ * customSeed can be anything. It can have any size, even small ones,
+ * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes.
+ * The resulting `secret` will nonetheless provide all required qualities.
+ *
+ * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize);
+
+
+/*
+ * XXH3_generateSecret_fromSeed():
+ *
+ * Generate the same secret as the _withSeed() variants.
+ *
+ * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily).
+ * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes.
+ *
+ * The generated secret can be used in combination with
+ *`*_withSecret()` and `_withSecretandSeed()` variants.
+ * This generator is notably useful in combination with `_withSecretandSeed()`,
+ * as a way to emulate a faster `_withSeed()` variant.
+ */
+XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed);
+
+/*
+ * *_withSecretandSeed() :
+ * These variants generate hash values using either
+ * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes)
+ * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX).
+ *
+ * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
+ * `_withSeed()` has to generate the secret on the fly for "large" keys.
+ * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
+ * `_withSecret()` has to generate the masks on the fly for "small" keys,
+ * which requires more instructions than _withSeed() variants.
+ * Therefore, _withSecretandSeed variant combines the best of both worlds.
+ *
+ * When @secret has been generated by XXH3_generateSecret_fromSeed(),
+ * this variant produces *exactly* the same results as `_withSeed()` variant,
+ * hence offering only a pure speed benefit on "large" input,
+ * by skipping the need to regenerate the secret for every large input.
+ *
+ * Another usage scenario is to hash the secret to a 64-bit hash value,
+ * for example with XXH3_64bits(), which then becomes the seed,
+ * and then employ both the seed and the secret in _withSecretandSeed().
+ * On top of speed, an added benefit is that each bit in the secret
+ * has a 50% chance to swap each bit in the output,
+ * via its impact to the seed.
+ * This is not guaranteed when using the secret directly in "small data" scenarios,
+ * because only portions of the secret are employed for small data.
+ */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecretandSeed(const void* data, size_t len,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed);
+
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecretandSeed(const void* data, size_t len,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed64);
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed64);
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed64);
+
+
+#endif /* XXH_NO_LONG_LONG */
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+# define XXH_IMPLEMENTATION
+#endif
+
+#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
+
+
+/* ======================================================================== */
+/* ======================================================================== */
+/* ======================================================================== */
+
+
+/*-**********************************************************************
+ * xxHash implementation
+ *-**********************************************************************
+ * xxHash's implementation used to be hosted inside xxhash.c.
+ *
+ * However, inlining requires implementation to be visible to the compiler,
+ * hence be included alongside the header.
+ * Previously, implementation was hosted inside xxhash.c,
+ * which was then #included when inlining was activated.
+ * This construction created issues with a few build and install systems,
+ * as it required xxhash.c to be stored in /include directory.
+ *
+ * xxHash implementation is now directly integrated within xxhash.h.
+ * As a consequence, xxhash.c is no longer needed in /include.
+ *
+ * xxhash.c is still available and is still useful.
+ * In a "normal" setup, when xxhash is not inlined,
+ * xxhash.h only exposes the prototypes and public symbols,
+ * while xxhash.c can be built into an object file xxhash.o
+ * which can then be linked into the final binary.
+ ************************************************************************/
+
+#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
+ || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
+# define XXH_IMPLEM_13a8737387
+
+/* *************************************
+* Tuning parameters
+***************************************/
+
+/*!
+ * @defgroup tuning Tuning parameters
+ * @{
+ *
+ * Various macros to control xxHash's behavior.
+ */
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Define this to disable 64-bit code.
+ *
+ * Useful if only using the @ref xxh32_family and you have a strict C90 compiler.
+ */
+# define XXH_NO_LONG_LONG
+# undef XXH_NO_LONG_LONG /* don't actually */
+/*!
+ * @brief Controls how unaligned memory is accessed.
+ *
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is
+ * safe and portable.
+ *
+ * Unfortunately, on some target/compiler combinations, the generated assembly
+ * is sub-optimal.
+ *
+ * The below switch allow selection of a different access method
+ * in the search for improved performance.
+ *
+ * @par Possible options:
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
+ * @par
+ * Use `memcpy()`. Safe and portable. Note that most modern compilers will
+ * eliminate the function call and treat it as an unaligned access.
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))`
+ * @par
+ * Depends on compiler extensions and is therefore not portable.
+ * This method is safe _if_ your compiler supports it,
+ * and *generally* as fast or faster than `memcpy`.
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
+ * @par
+ * Casts directly and dereferences. This method doesn't depend on the
+ * compiler, but it violates the C standard as it directly dereferences an
+ * unaligned pointer. It can generate buggy code on targets which do not
+ * support unaligned memory accesses, but in some circumstances, it's the
+ * only known way to get the most performance.
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
+ * @par
+ * Also portable. This can generate the best code on old compilers which don't
+ * inline small `memcpy()` calls, and it might also be faster on big-endian
+ * systems which lack a native byteswap instruction. However, some compilers
+ * will emit literal byteshifts even if the target supports unaligned access.
+ * .
+ *
+ * @warning
+ * Methods 1 and 2 rely on implementation-defined behavior. Use these with
+ * care, as what works on one compiler/platform/optimization level may cause
+ * another to read garbage data or even crash.
+ *
+ * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
+ *
+ * Prefer these methods in priority order (0 > 3 > 1 > 2)
+ */
+# define XXH_FORCE_MEMORY_ACCESS 0
+
+/*!
+ * @def XXH_FORCE_ALIGN_CHECK
+ * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
+ * and XXH64() only).
+ *
+ * This is an important performance trick for architectures without decent
+ * unaligned memory access performance.
+ *
+ * It checks for input alignment, and when conditions are met, uses a "fast
+ * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
+ * faster_ read speed.
+ *
+ * The check costs one initial branch per hash, which is generally negligible,
+ * but not zero.
+ *
+ * Moreover, it's not useful to generate an additional code path if memory
+ * access uses the same instruction for both aligned and unaligned
+ * addresses (e.g. x86 and aarch64).
+ *
+ * In these cases, the alignment check can be removed by setting this macro to 0.
+ * Then the code will always use unaligned memory access.
+ * Align check is automatically disabled on x86, x64 & arm64,
+ * which are platforms known to offer good unaligned memory accesses performance.
+ *
+ * This option does not affect XXH3 (only XXH32 and XXH64).
+ */
+# define XXH_FORCE_ALIGN_CHECK 0
+
+/*!
+ * @def XXH_NO_INLINE_HINTS
+ * @brief When non-zero, sets all functions to `static`.
+ *
+ * By default, xxHash tries to force the compiler to inline almost all internal
+ * functions.
+ *
+ * This can usually improve performance due to reduced jumping and improved
+ * constant folding, but significantly increases the size of the binary which
+ * might not be favorable.
+ *
+ * Additionally, sometimes the forced inlining can be detrimental to performance,
+ * depending on the architecture.
+ *
+ * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
+ * compiler full control on whether to inline or not.
+ *
+ * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using
+ * -fno-inline with GCC or Clang, this will automatically be defined.
+ */
+# define XXH_NO_INLINE_HINTS 0
+
+/*!
+ * @def XXH32_ENDJMP
+ * @brief Whether to use a jump for `XXH32_finalize`.
+ *
+ * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
+ * This is generally preferable for performance,
+ * but depending on exact architecture, a jmp may be preferable.
+ *
+ * This setting is only possibly making a difference for very small inputs.
+ */
+# define XXH32_ENDJMP 0
+
+/*!
+ * @internal
+ * @brief Redefines old internal names.
+ *
+ * For compatibility with code that uses xxHash's internals before the names
+ * were changed to improve namespacing. There is no other reason to use this.
+ */
+# define XXH_OLD_NAMES
+# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
+#endif /* XXH_DOXYGEN */
+/*!
+ * @}
+ */
+
+#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
+ /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */
+# if !defined(__clang__) && \
+( \
+ (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
+ ( \
+ defined(__GNUC__) && ( \
+ (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \
+ ( \
+ defined(__mips__) && \
+ (__mips <= 5 || __mips_isa_rev < 6) && \
+ (!defined(__mips16) || defined(__mips_mips16e2)) \
+ ) \
+ ) \
+ ) \
+)
+# define XXH_FORCE_MEMORY_ACCESS 1
+# endif
+#endif
+
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \
+ || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */
+# define XXH_FORCE_ALIGN_CHECK 0
+# else
+# define XXH_FORCE_ALIGN_CHECK 1
+# endif
+#endif
+
+#ifndef XXH_NO_INLINE_HINTS
+# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \
+ || defined(__NO_INLINE__) /* -O0, -fno-inline */
+# define XXH_NO_INLINE_HINTS 1
+# else
+# define XXH_NO_INLINE_HINTS 0
+# endif
+#endif
+
+#ifndef XXH32_ENDJMP
+/* generally preferable for performance */
+# define XXH32_ENDJMP 0
+#endif
+
+/*!
+ * @defgroup impl Implementation
+ * @{
+ */
+
+
+/* *************************************
+* Includes & Memory related functions
+***************************************/
+/*
+ * Modify the local functions below should you wish to use
+ * different memory routines for malloc() and free()
+ */
+#include <stdlib.h>
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than malloc().
+ */
+static void* XXH_malloc(size_t s) { return malloc(s); }
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than free().
+ */
+static void XXH_free(void* p) { free(p); }
+
+#include <string.h>
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than memcpy().
+ */
+static void* XXH_memcpy(void* dest, const void* src, size_t size)
+{
+ return memcpy(dest,src,size);
+}
+
+#include <limits.h> /* ULLONG_MAX */
+
+
+/* *************************************
+* Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER /* Visual Studio warning fix */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+#if XXH_NO_INLINE_HINTS /* disable inlining hints */
+# if defined(__GNUC__) || defined(__clang__)
+# define XXH_FORCE_INLINE static __attribute__((unused))
+# else
+# define XXH_FORCE_INLINE static
+# endif
+# define XXH_NO_INLINE static
+/* enable inlining hints */
+#elif defined(__GNUC__) || defined(__clang__)
+# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
+# define XXH_NO_INLINE static __attribute__((noinline))
+#elif defined(_MSC_VER) /* Visual Studio */
+# define XXH_FORCE_INLINE static __forceinline
+# define XXH_NO_INLINE static __declspec(noinline)
+#elif defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */
+# define XXH_FORCE_INLINE static inline
+# define XXH_NO_INLINE static
+#else
+# define XXH_FORCE_INLINE static
+# define XXH_NO_INLINE static
+#endif
+
+
+
+/* *************************************
+* Debug
+***************************************/
+/*!
+ * @ingroup tuning
+ * @def XXH_DEBUGLEVEL
+ * @brief Sets the debugging level.
+ *
+ * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
+ * compiler's command line options. The value must be a number.
+ */
+#ifndef XXH_DEBUGLEVEL
+# ifdef DEBUGLEVEL /* backwards compat */
+# define XXH_DEBUGLEVEL DEBUGLEVEL
+# else
+# define XXH_DEBUGLEVEL 0
+# endif
+#endif
+
+#if (XXH_DEBUGLEVEL>=1)
+# include <assert.h> /* note: can still be disabled with NDEBUG */
+# define XXH_ASSERT(c) assert(c)
+#else
+# define XXH_ASSERT(c) ((void)0)
+#endif
+
+/* note: use after variable declarations */
+#ifndef XXH_STATIC_ASSERT
+# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
+# include <assert.h>
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
+# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
+# else
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
+# endif
+# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
+#endif
+
+/*!
+ * @internal
+ * @def XXH_COMPILER_GUARD(var)
+ * @brief Used to prevent unwanted optimizations for @p var.
+ *
+ * It uses an empty GCC inline assembly statement with a register constraint
+ * which forces @p var into a general purpose register (eg eax, ebx, ecx
+ * on x86) and marks it as modified.
+ *
+ * This is used in a few places to avoid unwanted autovectorization (e.g.
+ * XXH32_round()). All vectorization we want is explicit via intrinsics,
+ * and _usually_ isn't wanted elsewhere.
+ *
+ * We also use it to prevent unwanted constant folding for AArch64 in
+ * XXH3_initCustomSecret_scalar().
+ */
+#if defined(__GNUC__) || defined(__clang__)
+# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var))
+#else
+# define XXH_COMPILER_GUARD(var) ((void)0)
+#endif
+
+/* *************************************
+* Basic Types
+***************************************/
+#if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint8_t xxh_u8;
+#else
+ typedef unsigned char xxh_u8;
+#endif
+typedef XXH32_hash_t xxh_u32;
+
+#ifdef XXH_OLD_NAMES
+# define BYTE xxh_u8
+# define U8 xxh_u8
+# define U32 xxh_u32
+#endif
+
+/* *** Memory access *** */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_read32(const void* ptr)
+ * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit native endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readLE32(const void* ptr)
+ * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readBE32(const void* ptr)
+ * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit big endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
+ * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
+ * always @ref XXH_alignment::XXH_unaligned.
+ *
+ * @param ptr The pointer to read from.
+ * @param align Whether @p ptr is aligned.
+ * @pre
+ * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
+ * aligned.
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
+ */
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE32 and XXH_readBE32.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/*
+ * Force direct memory access. Only works on CPU which support unaligned memory
+ * access in hardware.
+ */
+static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __pack instructions are safer but compiler specific, hence potentially
+ * problematic for some compilers.
+ *
+ * Currently only defined for GCC and ICC.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
+#endif
+static xxh_u32 XXH_read32(const void* ptr)
+{
+ typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign;
+ return ((const xxh_unalign*)ptr)->u32;
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ */
+static xxh_u32 XXH_read32(const void* memPtr)
+{
+ xxh_u32 val;
+ XXH_memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* *** Endianness *** */
+
+/*!
+ * @ingroup tuning
+ * @def XXH_CPU_LITTLE_ENDIAN
+ * @brief Whether the target is little endian.
+ *
+ * Defined to 1 if the target is little endian, or 0 if it is big endian.
+ * It can be defined externally, for example on the compiler command line.
+ *
+ * If it is not defined,
+ * a runtime check (which is usually constant folded) is used instead.
+ *
+ * @note
+ * This is not necessarily defined to an integer constant.
+ *
+ * @see XXH_isLittleEndian() for the runtime check.
+ */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+/*
+ * Try to detect endianness automatically, to avoid the nonstandard behavior
+ * in `XXH_isLittleEndian()`
+ */
+# if defined(_WIN32) /* Windows is always little endian */ \
+ || defined(__LITTLE_ENDIAN__) \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+# define XXH_CPU_LITTLE_ENDIAN 1
+# elif defined(__BIG_ENDIAN__) \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+# define XXH_CPU_LITTLE_ENDIAN 0
+# else
+/*!
+ * @internal
+ * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
+ *
+ * Most compilers will constant fold this.
+ */
+static int XXH_isLittleEndian(void)
+{
+ /*
+ * Portable and well-defined behavior.
+ * Don't use static: it is detrimental to performance.
+ */
+ const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
+ return one.c[0];
+}
+# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
+# endif
+#endif
+
+
+
+
+/* ****************************************
+* Compiler-specific Functions and Macros
+******************************************/
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#ifdef __has_builtin
+# define XXH_HAS_BUILTIN(x) __has_builtin(x)
+#else
+# define XXH_HAS_BUILTIN(x) 0
+#endif
+
+/*!
+ * @internal
+ * @def XXH_rotl32(x,r)
+ * @brief 32-bit rotate left.
+ *
+ * @param x The 32-bit integer to be rotated.
+ * @param r The number of bits to rotate.
+ * @pre
+ * @p r > 0 && @p r < 32
+ * @note
+ * @p x and @p r may be evaluated multiple times.
+ * @return The rotated result.
+ */
+#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
+ && XXH_HAS_BUILTIN(__builtin_rotateleft64)
+# define XXH_rotl32 __builtin_rotateleft32
+# define XXH_rotl64 __builtin_rotateleft64
+/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
+#elif defined(_MSC_VER)
+# define XXH_rotl32(x,r) _rotl(x,r)
+# define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
+# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
+#endif
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_swap32(xxh_u32 x)
+ * @brief A 32-bit byteswap.
+ *
+ * @param x The 32-bit integer to byteswap.
+ * @return @p x, byteswapped.
+ */
+#if defined(_MSC_VER) /* Visual Studio */
+# define XXH_swap32 _byteswap_ulong
+#elif XXH_GCC_VERSION >= 403
+# define XXH_swap32 __builtin_bswap32
+#else
+static xxh_u32 XXH_swap32 (xxh_u32 x)
+{
+ return ((x << 24) & 0xff000000 ) |
+ ((x << 8) & 0x00ff0000 ) |
+ ((x >> 8) & 0x0000ff00 ) |
+ ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* ***************************
+* Memory reads
+*****************************/
+
+/*!
+ * @internal
+ * @brief Enum to indicate whether a pointer is aligned.
+ */
+typedef enum {
+ XXH_aligned, /*!< Aligned */
+ XXH_unaligned /*!< Possibly unaligned */
+} XXH_alignment;
+
+/*
+ * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
+ *
+ * This is ideal for older compilers which don't inline memcpy.
+ */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[0]
+ | ((xxh_u32)bytePtr[1] << 8)
+ | ((xxh_u32)bytePtr[2] << 16)
+ | ((xxh_u32)bytePtr[3] << 24);
+}
+
+XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[3]
+ | ((xxh_u32)bytePtr[2] << 8)
+ | ((xxh_u32)bytePtr[1] << 16)
+ | ((xxh_u32)bytePtr[0] << 24);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+}
+
+static xxh_u32 XXH_readBE32(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u32
+XXH_readLE32_align(const void* ptr, XXH_alignment align)
+{
+ if (align==XXH_unaligned) {
+ return XXH_readLE32(ptr);
+ } else {
+ return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
+ }
+}
+
+
+/* *************************************
+* Misc
+***************************************/
+/*! @ingroup public */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+* 32-bit hash functions
+*********************************************************************/
+/*!
+ * @}
+ * @defgroup xxh32_impl XXH32 implementation
+ * @ingroup impl
+ * @{
+ */
+ /* #define instead of static const, to be used as initializers */
+#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */
+#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */
+#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */
+#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */
+#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */
+
+#ifdef XXH_OLD_NAMES
+# define PRIME32_1 XXH_PRIME32_1
+# define PRIME32_2 XXH_PRIME32_2
+# define PRIME32_3 XXH_PRIME32_3
+# define PRIME32_4 XXH_PRIME32_4
+# define PRIME32_5 XXH_PRIME32_5
+#endif
+
+/*!
+ * @internal
+ * @brief Normal stripe processing routine.
+ *
+ * This shuffles the bits so that any bit from @p input impacts several bits in
+ * @p acc.
+ *
+ * @param acc The accumulator lane.
+ * @param input The stripe of input to mix.
+ * @return The mixed accumulator lane.
+ */
+static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
+{
+ acc += input * XXH_PRIME32_2;
+ acc = XXH_rotl32(acc, 13);
+ acc *= XXH_PRIME32_1;
+#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
+ /*
+ * UGLY HACK:
+ * A compiler fence is the only thing that prevents GCC and Clang from
+ * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
+ * reason) without globally disabling SSE4.1.
+ *
+ * The reason we want to avoid vectorization is because despite working on
+ * 4 integers at a time, there are multiple factors slowing XXH32 down on
+ * SSE4:
+ * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
+ * newer chips!) making it slightly slower to multiply four integers at
+ * once compared to four integers independently. Even when pmulld was
+ * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
+ * just to multiply unless doing a long operation.
+ *
+ * - Four instructions are required to rotate,
+ * movqda tmp, v // not required with VEX encoding
+ * pslld tmp, 13 // tmp <<= 13
+ * psrld v, 19 // x >>= 19
+ * por v, tmp // x |= tmp
+ * compared to one for scalar:
+ * roll v, 13 // reliably fast across the board
+ * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason
+ *
+ * - Instruction level parallelism is actually more beneficial here because
+ * the SIMD actually serializes this operation: While v1 is rotating, v2
+ * can load data, while v3 can multiply. SSE forces them to operate
+ * together.
+ *
+ * This is also enabled on AArch64, as Clang autovectorizes it incorrectly
+ * and it is pointless writing a NEON implementation that is basically the
+ * same speed as scalar for XXH32.
+ */
+ XXH_COMPILER_GUARD(acc);
+#endif
+ return acc;
+}
+
+/*!
+ * @internal
+ * @brief Mixes all bits to finalize the hash.
+ *
+ * The final mix ensures that all input bits have a chance to impact any bit in
+ * the output digest, resulting in an unbiased distribution.
+ *
+ * @param h32 The hash to avalanche.
+ * @return The avalanched hash.
+ */
+static xxh_u32 XXH32_avalanche(xxh_u32 h32)
+{
+ h32 ^= h32 >> 15;
+ h32 *= XXH_PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= XXH_PRIME32_3;
+ h32 ^= h32 >> 16;
+ return(h32);
+}
+
+#define XXH_get32bits(p) XXH_readLE32_align(p, align)
+
+/*!
+ * @internal
+ * @brief Processes the last 0-15 bytes of @p ptr.
+ *
+ * There may be up to 15 bytes remaining to consume from the input.
+ * This final stage will digest them to ensure that all input bytes are present
+ * in the final mix.
+ *
+ * @param h32 The hash to finalize.
+ * @param ptr The pointer to the remaining input.
+ * @param len The remaining length, modulo 16.
+ * @param align Whether @p ptr is aligned.
+ * @return The finalized hash.
+ */
+static xxh_u32
+XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+#define XXH_PROCESS1 do { \
+ h32 += (*ptr++) * XXH_PRIME32_5; \
+ h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \
+} while (0)
+
+#define XXH_PROCESS4 do { \
+ h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \
+ ptr += 4; \
+ h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \
+} while (0)
+
+ if (ptr==NULL) XXH_ASSERT(len == 0);
+
+ /* Compact rerolled version; generally faster */
+ if (!XXH32_ENDJMP) {
+ len &= 15;
+ while (len >= 4) {
+ XXH_PROCESS4;
+ len -= 4;
+ }
+ while (len > 0) {
+ XXH_PROCESS1;
+ --len;
+ }
+ return XXH32_avalanche(h32);
+ } else {
+ switch(len&15) /* or switch(bEnd - p) */ {
+ case 12: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 8: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 4: XXH_PROCESS4;
+ return XXH32_avalanche(h32);
+
+ case 13: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 9: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 5: XXH_PROCESS4;
+ XXH_PROCESS1;
+ return XXH32_avalanche(h32);
+
+ case 14: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 10: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 6: XXH_PROCESS4;
+ XXH_PROCESS1;
+ XXH_PROCESS1;
+ return XXH32_avalanche(h32);
+
+ case 15: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 11: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 7: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 3: XXH_PROCESS1;
+ XXH_FALLTHROUGH;
+ case 2: XXH_PROCESS1;
+ XXH_FALLTHROUGH;
+ case 1: XXH_PROCESS1;
+ XXH_FALLTHROUGH;
+ case 0: return XXH32_avalanche(h32);
+ }
+ XXH_ASSERT(0);
+ return h32; /* reaching this point is deemed impossible */
+ }
+}
+
+#ifdef XXH_OLD_NAMES
+# define PROCESS1 XXH_PROCESS1
+# define PROCESS4 XXH_PROCESS4
+#else
+# undef XXH_PROCESS1
+# undef XXH_PROCESS4
+#endif
+
+/*!
+ * @internal
+ * @brief The implementation for @ref XXH32().
+ *
+ * @param input , len , seed Directly passed from @ref XXH32().
+ * @param align Whether @p input is aligned.
+ * @return The calculated hash.
+ */
+XXH_FORCE_INLINE xxh_u32
+XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
+{
+ xxh_u32 h32;
+
+ if (input==NULL) XXH_ASSERT(len == 0);
+
+ if (len>=16) {
+ const xxh_u8* const bEnd = input + len;
+ const xxh_u8* const limit = bEnd - 15;
+ xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+ xxh_u32 v2 = seed + XXH_PRIME32_2;
+ xxh_u32 v3 = seed + 0;
+ xxh_u32 v4 = seed - XXH_PRIME32_1;
+
+ do {
+ v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
+ v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
+ v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
+ v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
+ } while (input < limit);
+
+ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
+ + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+ } else {
+ h32 = seed + XXH_PRIME32_5;
+ }
+
+ h32 += (xxh_u32)len;
+
+ return XXH32_finalize(h32, input, len&15, align);
+}
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
+{
+#if 0
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH32_state_t state;
+ XXH32_reset(&state, seed);
+ XXH32_update(&state, (const xxh_u8*)input, len);
+ return XXH32_digest(&state);
+#else
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
+ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+ } }
+
+ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+#endif
+}
+
+
+
+/******* Hash streaming *******/
+/*!
+ * @ingroup xxh32_family
+ */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+ XXH_free(statePtr);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+ XXH_memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
+{
+ XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state));
+ state.v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+ state.v[1] = seed + XXH_PRIME32_2;
+ state.v[2] = seed + 0;
+ state.v[3] = seed - XXH_PRIME32_1;
+ /* do not write into reserved, planned to be removed in a future version */
+ XXH_memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
+ return XXH_OK;
+}
+
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH32_update(XXH32_state_t* state, const void* input, size_t len)
+{
+ if (input==NULL) {
+ XXH_ASSERT(len == 0);
+ return XXH_OK;
+ }
+
+ { const xxh_u8* p = (const xxh_u8*)input;
+ const xxh_u8* const bEnd = p + len;
+
+ state->total_len_32 += (XXH32_hash_t)len;
+ state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
+
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
+ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
+ state->memsize += (XXH32_hash_t)len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) { /* some data left from previous update */
+ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
+ { const xxh_u32* p32 = state->mem32;
+ state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
+ state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
+ state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
+ state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
+ }
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p <= bEnd-16) {
+ const xxh_u8* const limit = bEnd - 16;
+
+ do {
+ state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
+ state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
+ state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
+ state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
+ } while (p<=limit);
+
+ }
+
+ if (p < bEnd) {
+ XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
+ }
+
+ return XXH_OK;
+}
+
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
+{
+ xxh_u32 h32;
+
+ if (state->large_len) {
+ h32 = XXH_rotl32(state->v[0], 1)
+ + XXH_rotl32(state->v[1], 7)
+ + XXH_rotl32(state->v[2], 12)
+ + XXH_rotl32(state->v[3], 18);
+ } else {
+ h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
+ }
+
+ h32 += state->total_len_32;
+
+ return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
+}
+
+
+/******* Canonical representation *******/
+
+/*!
+ * @ingroup xxh32_family
+ * The default return values from XXH functions are unsigned 32 and 64 bit
+ * integers.
+ *
+ * The canonical representation uses big endian convention, the same convention
+ * as human-readable numbers (large digits first).
+ *
+ * This way, hash values can be written into a file or buffer, remaining
+ * comparable across different systems.
+ *
+ * The following functions allow transformation of hash values to and from their
+ * canonical format.
+ */
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+ XXH_memcpy(dst, &hash, sizeof(*dst));
+}
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+ return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+* 64-bit hash functions
+*********************************************************************/
+/*!
+ * @}
+ * @ingroup impl
+ * @{
+ */
+/******* Memory access *******/
+
+typedef XXH64_hash_t xxh_u64;
+
+#ifdef XXH_OLD_NAMES
+# define U64 xxh_u64
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE64 and XXH_readBE64.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static xxh_u64 XXH_read64(const void* memPtr)
+{
+ return *(const xxh_u64*) memPtr;
+}
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __pack instructions are safer, but compiler specific, hence potentially
+ * problematic for some compilers.
+ *
+ * Currently only defined for GCC and ICC.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
+#endif
+static xxh_u64 XXH_read64(const void* ptr)
+{
+ typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64;
+ return ((const xxh_unalign64*)ptr)->u64;
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ */
+static xxh_u64 XXH_read64(const void* memPtr)
+{
+ xxh_u64 val;
+ XXH_memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER) /* Visual Studio */
+# define XXH_swap64 _byteswap_uint64
+#elif XXH_GCC_VERSION >= 403
+# define XXH_swap64 __builtin_bswap64
+#else
+static xxh_u64 XXH_swap64(xxh_u64 x)
+{
+ return ((x << 56) & 0xff00000000000000ULL) |
+ ((x << 40) & 0x00ff000000000000ULL) |
+ ((x << 24) & 0x0000ff0000000000ULL) |
+ ((x << 8) & 0x000000ff00000000ULL) |
+ ((x >> 8) & 0x00000000ff000000ULL) |
+ ((x >> 24) & 0x0000000000ff0000ULL) |
+ ((x >> 40) & 0x000000000000ff00ULL) |
+ ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+
+/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[0]
+ | ((xxh_u64)bytePtr[1] << 8)
+ | ((xxh_u64)bytePtr[2] << 16)
+ | ((xxh_u64)bytePtr[3] << 24)
+ | ((xxh_u64)bytePtr[4] << 32)
+ | ((xxh_u64)bytePtr[5] << 40)
+ | ((xxh_u64)bytePtr[6] << 48)
+ | ((xxh_u64)bytePtr[7] << 56);
+}
+
+XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[7]
+ | ((xxh_u64)bytePtr[6] << 8)
+ | ((xxh_u64)bytePtr[5] << 16)
+ | ((xxh_u64)bytePtr[4] << 24)
+ | ((xxh_u64)bytePtr[3] << 32)
+ | ((xxh_u64)bytePtr[2] << 40)
+ | ((xxh_u64)bytePtr[1] << 48)
+ | ((xxh_u64)bytePtr[0] << 56);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+}
+
+static xxh_u64 XXH_readBE64(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u64
+XXH_readLE64_align(const void* ptr, XXH_alignment align)
+{
+ if (align==XXH_unaligned)
+ return XXH_readLE64(ptr);
+ else
+ return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
+}
+
+
+/******* xxh64 *******/
+/*!
+ * @}
+ * @defgroup xxh64_impl XXH64 implementation
+ * @ingroup impl
+ * @{
+ */
+/* #define rather that static const, to be used as initializers */
+#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
+#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
+#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
+#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
+#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
+
+#ifdef XXH_OLD_NAMES
+# define PRIME64_1 XXH_PRIME64_1
+# define PRIME64_2 XXH_PRIME64_2
+# define PRIME64_3 XXH_PRIME64_3
+# define PRIME64_4 XXH_PRIME64_4
+# define PRIME64_5 XXH_PRIME64_5
+#endif
+
+static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
+{
+ acc += input * XXH_PRIME64_2;
+ acc = XXH_rotl64(acc, 31);
+ acc *= XXH_PRIME64_1;
+ return acc;
+}
+
+static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
+{
+ val = XXH64_round(0, val);
+ acc ^= val;
+ acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
+ return acc;
+}
+
+static xxh_u64 XXH64_avalanche(xxh_u64 h64)
+{
+ h64 ^= h64 >> 33;
+ h64 *= XXH_PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= XXH_PRIME64_3;
+ h64 ^= h64 >> 32;
+ return h64;
+}
+
+
+#define XXH_get64bits(p) XXH_readLE64_align(p, align)
+
+static xxh_u64
+XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+ if (ptr==NULL) XXH_ASSERT(len == 0);
+ len &= 31;
+ while (len >= 8) {
+ xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
+ ptr += 8;
+ h64 ^= k1;
+ h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
+ len -= 8;
+ }
+ if (len >= 4) {
+ h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
+ ptr += 4;
+ h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
+ len -= 4;
+ }
+ while (len > 0) {
+ h64 ^= (*ptr++) * XXH_PRIME64_5;
+ h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1;
+ --len;
+ }
+ return XXH64_avalanche(h64);
+}
+
+#ifdef XXH_OLD_NAMES
+# define PROCESS1_64 XXH_PROCESS1_64
+# define PROCESS4_64 XXH_PROCESS4_64
+# define PROCESS8_64 XXH_PROCESS8_64
+#else
+# undef XXH_PROCESS1_64
+# undef XXH_PROCESS4_64
+# undef XXH_PROCESS8_64
+#endif
+
+XXH_FORCE_INLINE xxh_u64
+XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
+{
+ xxh_u64 h64;
+ if (input==NULL) XXH_ASSERT(len == 0);
+
+ if (len>=32) {
+ const xxh_u8* const bEnd = input + len;
+ const xxh_u8* const limit = bEnd - 31;
+ xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+ xxh_u64 v2 = seed + XXH_PRIME64_2;
+ xxh_u64 v3 = seed + 0;
+ xxh_u64 v4 = seed - XXH_PRIME64_1;
+
+ do {
+ v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
+ v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
+ v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
+ v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
+ } while (input<limit);
+
+ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
+
+ } else {
+ h64 = seed + XXH_PRIME64_5;
+ }
+
+ h64 += (xxh_u64) len;
+
+ return XXH64_finalize(h64, input, len, align);
+}
+
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed)
+{
+#if 0
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH64_state_t state;
+ XXH64_reset(&state, seed);
+ XXH64_update(&state, (const xxh_u8*)input, len);
+ return XXH64_digest(&state);
+#else
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
+ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+ } }
+
+ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+
+#endif
+}
+
+/******* Hash Streaming *******/
+
+/*! @ingroup xxh64_family*/
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+ XXH_free(statePtr);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+ XXH_memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed)
+{
+ XXH64_state_t state; /* use a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state));
+ state.v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+ state.v[1] = seed + XXH_PRIME64_2;
+ state.v[2] = seed + 0;
+ state.v[3] = seed - XXH_PRIME64_1;
+ /* do not write into reserved64, might be removed in a future version */
+ XXH_memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64));
+ return XXH_OK;
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH64_update (XXH64_state_t* state, const void* input, size_t len)
+{
+ if (input==NULL) {
+ XXH_ASSERT(len == 0);
+ return XXH_OK;
+ }
+
+ { const xxh_u8* p = (const xxh_u8*)input;
+ const xxh_u8* const bEnd = p + len;
+
+ state->total_len += len;
+
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
+ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
+ state->memsize += (xxh_u32)len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) { /* tmp buffer is full */
+ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
+ state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
+ state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
+ state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
+ state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
+ p += 32 - state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p+32 <= bEnd) {
+ const xxh_u8* const limit = bEnd - 32;
+
+ do {
+ state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
+ state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
+ state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
+ state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
+ } while (p<=limit);
+
+ }
+
+ if (p < bEnd) {
+ XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
+ }
+
+ return XXH_OK;
+}
+
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state)
+{
+ xxh_u64 h64;
+
+ if (state->total_len >= 32) {
+ h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
+ h64 = XXH64_mergeRound(h64, state->v[0]);
+ h64 = XXH64_mergeRound(h64, state->v[1]);
+ h64 = XXH64_mergeRound(h64, state->v[2]);
+ h64 = XXH64_mergeRound(h64, state->v[3]);
+ } else {
+ h64 = state->v[2] /*seed*/ + XXH_PRIME64_5;
+ }
+
+ h64 += (xxh_u64) state->total_len;
+
+ return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
+}
+
+
+/******* Canonical representation *******/
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+ XXH_memcpy(dst, &hash, sizeof(*dst));
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+ return XXH_readBE64(src);
+}
+
+#ifndef XXH_NO_XXH3
+
+/* *********************************************************************
+* XXH3
+* New generation hash designed for speed on small keys and vectorization
+************************************************************************ */
+/*!
+ * @}
+ * @defgroup xxh3_impl XXH3 implementation
+ * @ingroup impl
+ * @{
+ */
+
+/* === Compiler specifics === */
+
+#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
+# define XXH_RESTRICT /* disable */
+#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
+# define XXH_RESTRICT restrict
+#else
+/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */
+# define XXH_RESTRICT /* disable */
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 3)) \
+ || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
+ || defined(__clang__)
+# define XXH_likely(x) __builtin_expect(x, 1)
+# define XXH_unlikely(x) __builtin_expect(x, 0)
+#else
+# define XXH_likely(x) (x)
+# define XXH_unlikely(x) (x)
+#endif
+
+#if defined(__GNUC__)
+# if defined(__AVX2__)
+# include <immintrin.h>
+# elif defined(__SSE2__)
+# include <emmintrin.h>
+# elif defined(__ARM_NEON__) || defined(__ARM_NEON)
+# define inline __inline__ /* circumvent a clang bug */
+# include <arm_neon.h>
+# undef inline
+# endif
+#elif defined(_MSC_VER)
+# include <intrin.h>
+#endif
+
+/*
+ * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
+ * remaining a true 64-bit/128-bit hash function.
+ *
+ * This is done by prioritizing a subset of 64-bit operations that can be
+ * emulated without too many steps on the average 32-bit machine.
+ *
+ * For example, these two lines seem similar, and run equally fast on 64-bit:
+ *
+ * xxh_u64 x;
+ * x ^= (x >> 47); // good
+ * x ^= (x >> 13); // bad
+ *
+ * However, to a 32-bit machine, there is a major difference.
+ *
+ * x ^= (x >> 47) looks like this:
+ *
+ * x.lo ^= (x.hi >> (47 - 32));
+ *
+ * while x ^= (x >> 13) looks like this:
+ *
+ * // note: funnel shifts are not usually cheap.
+ * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
+ * x.hi ^= (x.hi >> 13);
+ *
+ * The first one is significantly faster than the second, simply because the
+ * shift is larger than 32. This means:
+ * - All the bits we need are in the upper 32 bits, so we can ignore the lower
+ * 32 bits in the shift.
+ * - The shift result will always fit in the lower 32 bits, and therefore,
+ * we can ignore the upper 32 bits in the xor.
+ *
+ * Thanks to this optimization, XXH3 only requires these features to be efficient:
+ *
+ * - Usable unaligned access
+ * - A 32-bit or 64-bit ALU
+ * - If 32-bit, a decent ADC instruction
+ * - A 32 or 64-bit multiply with a 64-bit result
+ * - For the 128-bit variant, a decent byteswap helps short inputs.
+ *
+ * The first two are already required by XXH32, and almost all 32-bit and 64-bit
+ * platforms which can run XXH32 can run XXH3 efficiently.
+ *
+ * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
+ * notable exception.
+ *
+ * First of all, Thumb-1 lacks support for the UMULL instruction which
+ * performs the important long multiply. This means numerous __aeabi_lmul
+ * calls.
+ *
+ * Second of all, the 8 functional registers are just not enough.
+ * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
+ * Lo registers, and this shuffling results in thousands more MOVs than A32.
+ *
+ * A32 and T32 don't have this limitation. They can access all 14 registers,
+ * do a 32->64 multiply with UMULL, and the flexible operand allowing free
+ * shifts is helpful, too.
+ *
+ * Therefore, we do a quick sanity check.
+ *
+ * If compiling Thumb-1 for a target which supports ARM instructions, we will
+ * emit a warning, as it is not a "sane" platform to compile for.
+ *
+ * Usually, if this happens, it is because of an accident and you probably need
+ * to specify -march, as you likely meant to compile for a newer architecture.
+ *
+ * Credit: large sections of the vectorial and asm source code paths
+ * have been contributed by @easyaspi314
+ */
+#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
+# warning "XXH3 is highly inefficient without ARM or Thumb-2."
+#endif
+
+/* ==========================================
+ * Vectorization detection
+ * ========================================== */
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @ingroup tuning
+ * @brief Overrides the vectorization implementation chosen for XXH3.
+ *
+ * Can be defined to 0 to disable SIMD or any of the values mentioned in
+ * @ref XXH_VECTOR_TYPE.
+ *
+ * If this is not defined, it uses predefined macros to determine the best
+ * implementation.
+ */
+# define XXH_VECTOR XXH_SCALAR
+/*!
+ * @ingroup tuning
+ * @brief Possible values for @ref XXH_VECTOR.
+ *
+ * Note that these are actually implemented as macros.
+ *
+ * If this is not defined, it is detected automatically.
+ * @ref XXH_X86DISPATCH overrides this.
+ */
+enum XXH_VECTOR_TYPE /* fake enum */ {
+ XXH_SCALAR = 0, /*!< Portable scalar version */
+ XXH_SSE2 = 1, /*!<
+ * SSE2 for Pentium 4, Opteron, all x86_64.
+ *
+ * @note SSE2 is also guaranteed on Windows 10, macOS, and
+ * Android x86.
+ */
+ XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */
+ XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */
+ XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */
+ XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */
+};
+/*!
+ * @ingroup tuning
+ * @brief Selects the minimum alignment for XXH3's accumulators.
+ *
+ * When using SIMD, this should match the alignment reqired for said vector
+ * type, so, for example, 32 for AVX2.
+ *
+ * Default: Auto detected.
+ */
+# define XXH_ACC_ALIGN 8
+#endif
+
+/* Actual definition */
+#ifndef XXH_DOXYGEN
+# define XXH_SCALAR 0
+# define XXH_SSE2 1
+# define XXH_AVX2 2
+# define XXH_AVX512 3
+# define XXH_NEON 4
+# define XXH_VSX 5
+#endif
+
+#ifndef XXH_VECTOR /* can be defined on command line */
+# if defined(__AVX512F__)
+# define XXH_VECTOR XXH_AVX512
+# elif defined(__AVX2__)
+# define XXH_VECTOR XXH_AVX2
+# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
+# define XXH_VECTOR XXH_SSE2
+# elif ( \
+ defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
+ || defined(_M_ARM64) || defined(_M_ARM_ARMV7VE) /* msvc */ \
+ ) && ( \
+ defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
+ )
+# define XXH_VECTOR XXH_NEON
+# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
+ || (defined(__s390x__) && defined(__VEC__)) \
+ && defined(__GNUC__) /* TODO: IBM XL */
+# define XXH_VECTOR XXH_VSX
+# else
+# define XXH_VECTOR XXH_SCALAR
+# endif
+#endif
+
+/*
+ * Controls the alignment of the accumulator,
+ * for compatibility with aligned vector loads, which are usually faster.
+ */
+#ifndef XXH_ACC_ALIGN
+# if defined(XXH_X86DISPATCH)
+# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */
+# elif XXH_VECTOR == XXH_SCALAR /* scalar */
+# define XXH_ACC_ALIGN 8
+# elif XXH_VECTOR == XXH_SSE2 /* sse2 */
+# define XXH_ACC_ALIGN 16
+# elif XXH_VECTOR == XXH_AVX2 /* avx2 */
+# define XXH_ACC_ALIGN 32
+# elif XXH_VECTOR == XXH_NEON /* neon */
+# define XXH_ACC_ALIGN 16
+# elif XXH_VECTOR == XXH_VSX /* vsx */
+# define XXH_ACC_ALIGN 16
+# elif XXH_VECTOR == XXH_AVX512 /* avx512 */
+# define XXH_ACC_ALIGN 64
+# endif
+#endif
+
+#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
+ || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
+# define XXH_SEC_ALIGN XXH_ACC_ALIGN
+#else
+# define XXH_SEC_ALIGN 8
+#endif
+
+/*
+ * UGLY HACK:
+ * GCC usually generates the best code with -O3 for xxHash.
+ *
+ * However, when targeting AVX2, it is overzealous in its unrolling resulting
+ * in code roughly 3/4 the speed of Clang.
+ *
+ * There are other issues, such as GCC splitting _mm256_loadu_si256 into
+ * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
+ * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
+ *
+ * That is why when compiling the AVX2 version, it is recommended to use either
+ * -O2 -mavx2 -march=haswell
+ * or
+ * -O2 -mavx2 -mno-avx256-split-unaligned-load
+ * for decent performance, or to use Clang instead.
+ *
+ * Fortunately, we can control the first one with a pragma that forces GCC into
+ * -O2, but the other one we can't control without "failed to inline always
+ * inline function due to target mismatch" warnings.
+ */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+ && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
+# pragma GCC push_options
+# pragma GCC optimize("-O2")
+#endif
+
+
+#if XXH_VECTOR == XXH_NEON
+/*
+ * NEON's setup for vmlal_u32 is a little more complicated than it is on
+ * SSE2, AVX2, and VSX.
+ *
+ * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast.
+ *
+ * To do the same operation, the 128-bit 'Q' register needs to be split into
+ * two 64-bit 'D' registers, performing this operation::
+ *
+ * [ a | b ]
+ * | '---------. .--------' |
+ * | x |
+ * | .---------' '--------. |
+ * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ]
+ *
+ * Due to significant changes in aarch64, the fastest method for aarch64 is
+ * completely different than the fastest method for ARMv7-A.
+ *
+ * ARMv7-A treats D registers as unions overlaying Q registers, so modifying
+ * D11 will modify the high half of Q5. This is similar to how modifying AH
+ * will only affect bits 8-15 of AX on x86.
+ *
+ * VZIP takes two registers, and puts even lanes in one register and odd lanes
+ * in the other.
+ *
+ * On ARMv7-A, this strangely modifies both parameters in place instead of
+ * taking the usual 3-operand form.
+ *
+ * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the
+ * lower and upper halves of the Q register to end up with the high and low
+ * halves where we want - all in one instruction.
+ *
+ * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] }
+ *
+ * Unfortunately we need inline assembly for this: Instructions modifying two
+ * registers at once is not possible in GCC or Clang's IR, and they have to
+ * create a copy.
+ *
+ * aarch64 requires a different approach.
+ *
+ * In order to make it easier to write a decent compiler for aarch64, many
+ * quirks were removed, such as conditional execution.
+ *
+ * NEON was also affected by this.
+ *
+ * aarch64 cannot access the high bits of a Q-form register, and writes to a
+ * D-form register zero the high bits, similar to how writes to W-form scalar
+ * registers (or DWORD registers on x86_64) work.
+ *
+ * The formerly free vget_high intrinsics now require a vext (with a few
+ * exceptions)
+ *
+ * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent
+ * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one
+ * operand.
+ *
+ * The equivalent of the VZIP.32 on the lower and upper halves would be this
+ * mess:
+ *
+ * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] }
+ * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] }
+ * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] }
+ *
+ * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN):
+ *
+ * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32);
+ * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF);
+ *
+ * This is available on ARMv7-A, but is less efficient than a single VZIP.32.
+ */
+
+/*!
+ * Function-like macro:
+ * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi)
+ * {
+ * outLo = (uint32x2_t)(in & 0xFFFFFFFF);
+ * outHi = (uint32x2_t)(in >> 32);
+ * in = UNDEFINED;
+ * }
+ */
+# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \
+ && defined(__GNUC__) \
+ && !defined(__aarch64__) && !defined(__arm64__) && !defined(_M_ARM64)
+# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
+ do { \
+ /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \
+ /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \
+ /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \
+ __asm__("vzip.32 %e0, %f0" : "+w" (in)); \
+ (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \
+ (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \
+ } while (0)
+# else
+# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
+ do { \
+ (outLo) = vmovn_u64 (in); \
+ (outHi) = vshrn_n_u64 ((in), 32); \
+ } while (0)
+# endif
+#endif /* XXH_VECTOR == XXH_NEON */
+
+/*
+ * VSX and Z Vector helpers.
+ *
+ * This is very messy, and any pull requests to clean this up are welcome.
+ *
+ * There are a lot of problems with supporting VSX and s390x, due to
+ * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
+ */
+#if XXH_VECTOR == XXH_VSX
+# if defined(__s390x__)
+# include <s390intrin.h>
+# else
+/* gcc's altivec.h can have the unwanted consequence to unconditionally
+ * #define bool, vector, and pixel keywords,
+ * with bad consequences for programs already using these keywords for other purposes.
+ * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined.
+ * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler,
+ * but it seems that, in some cases, it isn't.
+ * Force the build macro to be defined, so that keywords are not altered.
+ */
+# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__)
+# define __APPLE_ALTIVEC__
+# endif
+# include <altivec.h>
+# endif
+
+typedef __vector unsigned long long xxh_u64x2;
+typedef __vector unsigned char xxh_u8x16;
+typedef __vector unsigned xxh_u32x4;
+
+# ifndef XXH_VSX_BE
+# if defined(__BIG_ENDIAN__) \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+# define XXH_VSX_BE 1
+# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
+# warning "-maltivec=be is not recommended. Please use native endianness."
+# define XXH_VSX_BE 1
+# else
+# define XXH_VSX_BE 0
+# endif
+# endif /* !defined(XXH_VSX_BE) */
+
+# if XXH_VSX_BE
+# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
+# define XXH_vec_revb vec_revb
+# else
+/*!
+ * A polyfill for POWER9's vec_revb().
+ */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
+{
+ xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+ 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
+ return vec_perm(val, val, vByteSwap);
+}
+# endif
+# endif /* XXH_VSX_BE */
+
+/*!
+ * Performs an unaligned vector load and byte swaps it on big endian.
+ */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
+{
+ xxh_u64x2 ret;
+ XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
+# if XXH_VSX_BE
+ ret = XXH_vec_revb(ret);
+# endif
+ return ret;
+}
+
+/*
+ * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
+ *
+ * These intrinsics weren't added until GCC 8, despite existing for a while,
+ * and they are endian dependent. Also, their meaning swap depending on version.
+ * */
+# if defined(__s390x__)
+ /* s390x is always big endian, no issue on this platform */
+# define XXH_vec_mulo vec_mulo
+# define XXH_vec_mule vec_mule
+# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw)
+/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
+# define XXH_vec_mulo __builtin_altivec_vmulouw
+# define XXH_vec_mule __builtin_altivec_vmuleuw
+# else
+/* gcc needs inline assembly */
+/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
+{
+ xxh_u64x2 result;
+ __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+ return result;
+}
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
+{
+ xxh_u64x2 result;
+ __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+ return result;
+}
+# endif /* XXH_vec_mulo, XXH_vec_mule */
+#endif /* XXH_VECTOR == XXH_VSX */
+
+
+/* prefetch
+ * can be disabled, by declaring XXH_NO_PREFETCH build macro */
+#if defined(XXH_NO_PREFETCH)
+# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
+#else
+# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
+# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+# else
+# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
+# endif
+#endif /* XXH_NO_PREFETCH */
+
+
+/* ==========================================
+ * XXH3 default settings
+ * ========================================== */
+
+#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */
+
+#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
+# error "default keyset is not large enough"
+#endif
+
+/*! Pseudorandom secret taken directly from FARSH. */
+XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
+ 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
+ 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
+ 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
+ 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
+ 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
+ 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
+ 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
+ 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
+ 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
+ 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
+ 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
+ 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
+};
+
+
+#ifdef XXH_OLD_NAMES
+# define kSecret XXH3_kSecret
+#endif
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Calculates a 32-bit to 64-bit long multiply.
+ *
+ * Implemented as a macro.
+ *
+ * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
+ * need to (but it shouldn't need to anyways, it is about 7 instructions to do
+ * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
+ * use that instead of the normal method.
+ *
+ * If you are compiling for platforms like Thumb-1 and don't have a better option,
+ * you may also want to write your own long multiply routine here.
+ *
+ * @param x, y Numbers to be multiplied
+ * @return 64-bit product of the low 32 bits of @p x and @p y.
+ */
+XXH_FORCE_INLINE xxh_u64
+XXH_mult32to64(xxh_u64 x, xxh_u64 y)
+{
+ return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
+}
+#elif defined(_MSC_VER) && defined(_M_IX86)
+# include <intrin.h>
+# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
+#else
+/*
+ * Downcast + upcast is usually better than masking on older compilers like
+ * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
+ *
+ * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
+ * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
+ */
+# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
+#endif
+
+/*!
+ * @brief Calculates a 64->128-bit long multiply.
+ *
+ * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
+ * version.
+ *
+ * @param lhs , rhs The 64-bit integers to be multiplied
+ * @return The 128-bit result represented in an @ref XXH128_hash_t.
+ */
+static XXH128_hash_t
+XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
+{
+ /*
+ * GCC/Clang __uint128_t method.
+ *
+ * On most 64-bit targets, GCC and Clang define a __uint128_t type.
+ * This is usually the best way as it usually uses a native long 64-bit
+ * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
+ *
+ * Usually.
+ *
+ * Despite being a 32-bit platform, Clang (and emscripten) define this type
+ * despite not having the arithmetic for it. This results in a laggy
+ * compiler builtin call which calculates a full 128-bit multiply.
+ * In that case it is best to use the portable one.
+ * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
+ */
+#if defined(__GNUC__) && !defined(__wasm__) \
+ && defined(__SIZEOF_INT128__) \
+ || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+
+ __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
+ XXH128_hash_t r128;
+ r128.low64 = (xxh_u64)(product);
+ r128.high64 = (xxh_u64)(product >> 64);
+ return r128;
+
+ /*
+ * MSVC for x64's _umul128 method.
+ *
+ * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
+ *
+ * This compiles to single operand MUL on x64.
+ */
+#elif defined(_M_X64) || defined(_M_IA64)
+
+#ifndef _MSC_VER
+# pragma intrinsic(_umul128)
+#endif
+ xxh_u64 product_high;
+ xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
+ XXH128_hash_t r128;
+ r128.low64 = product_low;
+ r128.high64 = product_high;
+ return r128;
+
+ /*
+ * MSVC for ARM64's __umulh method.
+ *
+ * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
+ */
+#elif defined(_M_ARM64)
+
+#ifndef _MSC_VER
+# pragma intrinsic(__umulh)
+#endif
+ XXH128_hash_t r128;
+ r128.low64 = lhs * rhs;
+ r128.high64 = __umulh(lhs, rhs);
+ return r128;
+
+#else
+ /*
+ * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
+ *
+ * This is a fast and simple grade school multiply, which is shown below
+ * with base 10 arithmetic instead of base 0x100000000.
+ *
+ * 9 3 // D2 lhs = 93
+ * x 7 5 // D2 rhs = 75
+ * ----------
+ * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
+ * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
+ * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
+ * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
+ * ---------
+ * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
+ * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
+ * ---------
+ * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
+ *
+ * The reasons for adding the products like this are:
+ * 1. It avoids manual carry tracking. Just like how
+ * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
+ * This avoids a lot of complexity.
+ *
+ * 2. It hints for, and on Clang, compiles to, the powerful UMAAL
+ * instruction available in ARM's Digital Signal Processing extension
+ * in 32-bit ARMv6 and later, which is shown below:
+ *
+ * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
+ * {
+ * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
+ * *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
+ * *RdHi = (xxh_u32)(product >> 32);
+ * }
+ *
+ * This instruction was designed for efficient long multiplication, and
+ * allows this to be calculated in only 4 instructions at speeds
+ * comparable to some 64-bit ALUs.
+ *
+ * 3. It isn't terrible on other platforms. Usually this will be a couple
+ * of 32-bit ADD/ADCs.
+ */
+
+ /* First calculate all of the cross products. */
+ xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
+ xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
+ xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
+ xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
+
+ /* Now add the products together. These will never overflow. */
+ xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
+ xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
+ xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
+
+ XXH128_hash_t r128;
+ r128.low64 = lower;
+ r128.high64 = upper;
+ return r128;
+#endif
+}
+
+/*!
+ * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
+ *
+ * The reason for the separate function is to prevent passing too many structs
+ * around by value. This will hopefully inline the multiply, but we don't force it.
+ *
+ * @param lhs , rhs The 64-bit integers to multiply
+ * @return The low 64 bits of the product XOR'd by the high 64 bits.
+ * @see XXH_mult64to128()
+ */
+static xxh_u64
+XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
+{
+ XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
+ return product.low64 ^ product.high64;
+}
+
+/*! Seems to produce slightly better code on GCC for some reason. */
+XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
+{
+ XXH_ASSERT(0 <= shift && shift < 64);
+ return v64 ^ (v64 >> shift);
+}
+
+/*
+ * This is a fast avalanche stage,
+ * suitable when input bits are already partially mixed
+ */
+static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
+{
+ h64 = XXH_xorshift64(h64, 37);
+ h64 *= 0x165667919E3779F9ULL;
+ h64 = XXH_xorshift64(h64, 32);
+ return h64;
+}
+
+/*
+ * This is a stronger avalanche,
+ * inspired by Pelle Evensen's rrmxmx
+ * preferable when input has not been previously mixed
+ */
+static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
+{
+ /* this mix is inspired by Pelle Evensen's rrmxmx */
+ h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
+ h64 *= 0x9FB21C651E98DF25ULL;
+ h64 ^= (h64 >> 35) + len ;
+ h64 *= 0x9FB21C651E98DF25ULL;
+ return XXH_xorshift64(h64, 28);
+}
+
+
+/* ==========================================
+ * Short keys
+ * ==========================================
+ * One of the shortcomings of XXH32 and XXH64 was that their performance was
+ * sub-optimal on short lengths. It used an iterative algorithm which strongly
+ * favored lengths that were a multiple of 4 or 8.
+ *
+ * Instead of iterating over individual inputs, we use a set of single shot
+ * functions which piece together a range of lengths and operate in constant time.
+ *
+ * Additionally, the number of multiplies has been significantly reduced. This
+ * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
+ *
+ * Depending on the platform, this may or may not be faster than XXH32, but it
+ * is almost guaranteed to be faster than XXH64.
+ */
+
+/*
+ * At very short lengths, there isn't enough input to fully hide secrets, or use
+ * the entire secret.
+ *
+ * There is also only a limited amount of mixing we can do before significantly
+ * impacting performance.
+ *
+ * Therefore, we use different sections of the secret and always mix two secret
+ * samples with an XOR. This should have no effect on performance on the
+ * seedless or withSeed variants because everything _should_ be constant folded
+ * by modern compilers.
+ *
+ * The XOR mixing hides individual parts of the secret and increases entropy.
+ *
+ * This adds an extra layer of strength for custom secrets.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(1 <= len && len <= 3);
+ XXH_ASSERT(secret != NULL);
+ /*
+ * len = 1: combined = { input[0], 0x01, input[0], input[0] }
+ * len = 2: combined = { input[1], 0x02, input[0], input[1] }
+ * len = 3: combined = { input[2], 0x03, input[0], input[1] }
+ */
+ { xxh_u8 const c1 = input[0];
+ xxh_u8 const c2 = input[len >> 1];
+ xxh_u8 const c3 = input[len - 1];
+ xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24)
+ | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
+ xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+ xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
+ return XXH64_avalanche(keyed);
+ }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(4 <= len && len <= 8);
+ seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+ { xxh_u32 const input1 = XXH_readLE32(input);
+ xxh_u32 const input2 = XXH_readLE32(input + len - 4);
+ xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
+ xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
+ xxh_u64 const keyed = input64 ^ bitflip;
+ return XXH3_rrmxmx(keyed, len);
+ }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(9 <= len && len <= 16);
+ { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
+ xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
+ xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1;
+ xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
+ xxh_u64 const acc = len
+ + XXH_swap64(input_lo) + input_hi
+ + XXH3_mul128_fold64(input_lo, input_hi);
+ return XXH3_avalanche(acc);
+ }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(len <= 16);
+ { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed);
+ if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
+ if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
+ return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
+ }
+}
+
+/*
+ * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
+ * multiplication by zero, affecting hashes of lengths 17 to 240.
+ *
+ * However, they are very unlikely.
+ *
+ * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
+ * unseeded non-cryptographic hashes, it does not attempt to defend itself
+ * against specially crafted inputs, only random inputs.
+ *
+ * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
+ * cancelling out the secret is taken an arbitrary number of times (addressed
+ * in XXH3_accumulate_512), this collision is very unlikely with random inputs
+ * and/or proper seeding:
+ *
+ * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
+ * function that is only called up to 16 times per hash with up to 240 bytes of
+ * input.
+ *
+ * This is not too bad for a non-cryptographic hash function, especially with
+ * only 64 bit outputs.
+ *
+ * The 128-bit variant (which trades some speed for strength) is NOT affected
+ * by this, although it is always a good idea to use a proper seed if you care
+ * about strength.
+ */
+XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
+ const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
+{
+#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+ && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */
+ /*
+ * UGLY HACK:
+ * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
+ * slower code.
+ *
+ * By forcing seed64 into a register, we disrupt the cost model and
+ * cause it to scalarize. See `XXH32_round()`
+ *
+ * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
+ * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
+ * GCC 9.2, despite both emitting scalar code.
+ *
+ * GCC generates much better scalar code than Clang for the rest of XXH3,
+ * which is why finding a more optimal codepath is an interest.
+ */
+ XXH_COMPILER_GUARD(seed64);
+#endif
+ { xxh_u64 const input_lo = XXH_readLE64(input);
+ xxh_u64 const input_hi = XXH_readLE64(input+8);
+ return XXH3_mul128_fold64(
+ input_lo ^ (XXH_readLE64(secret) + seed64),
+ input_hi ^ (XXH_readLE64(secret+8) - seed64)
+ );
+ }
+}
+
+/* For mid range keys, XXH3 uses a Mum-hash variant. */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(16 < len && len <= 128);
+
+ { xxh_u64 acc = len * XXH_PRIME64_1;
+ if (len > 32) {
+ if (len > 64) {
+ if (len > 96) {
+ acc += XXH3_mix16B(input+48, secret+96, seed);
+ acc += XXH3_mix16B(input+len-64, secret+112, seed);
+ }
+ acc += XXH3_mix16B(input+32, secret+64, seed);
+ acc += XXH3_mix16B(input+len-48, secret+80, seed);
+ }
+ acc += XXH3_mix16B(input+16, secret+32, seed);
+ acc += XXH3_mix16B(input+len-32, secret+48, seed);
+ }
+ acc += XXH3_mix16B(input+0, secret+0, seed);
+ acc += XXH3_mix16B(input+len-16, secret+16, seed);
+
+ return XXH3_avalanche(acc);
+ }
+}
+
+#define XXH3_MIDSIZE_MAX 240
+
+XXH_NO_INLINE XXH64_hash_t
+XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+ #define XXH3_MIDSIZE_STARTOFFSET 3
+ #define XXH3_MIDSIZE_LASTOFFSET 17
+
+ { xxh_u64 acc = len * XXH_PRIME64_1;
+ int const nbRounds = (int)len / 16;
+ int i;
+ for (i=0; i<8; i++) {
+ acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
+ }
+ acc = XXH3_avalanche(acc);
+ XXH_ASSERT(nbRounds >= 8);
+#if defined(__clang__) /* Clang */ \
+ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
+ /*
+ * UGLY HACK:
+ * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
+ * In everywhere else, it uses scalar code.
+ *
+ * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
+ * would still be slower than UMAAL (see XXH_mult64to128).
+ *
+ * Unfortunately, Clang doesn't handle the long multiplies properly and
+ * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
+ * scalarized into an ugly mess of VMOV.32 instructions.
+ *
+ * This mess is difficult to avoid without turning autovectorization
+ * off completely, but they are usually relatively minor and/or not
+ * worth it to fix.
+ *
+ * This loop is the easiest to fix, as unlike XXH32, this pragma
+ * _actually works_ because it is a loop vectorization instead of an
+ * SLP vectorization.
+ */
+ #pragma clang loop vectorize(disable)
+#endif
+ for (i=8 ; i < nbRounds; i++) {
+ acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
+ }
+ /* last bytes */
+ acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
+ return XXH3_avalanche(acc);
+ }
+}
+
+
+/* ======= Long Keys ======= */
+
+#define XXH_STRIPE_LEN 64
+#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
+#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
+
+#ifdef XXH_OLD_NAMES
+# define STRIPE_LEN XXH_STRIPE_LEN
+# define ACC_NB XXH_ACC_NB
+#endif
+
+XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
+{
+ if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
+ XXH_memcpy(dst, &v64, sizeof(v64));
+}
+
+/* Several intrinsic functions below are supposed to accept __int64 as argument,
+ * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
+ * However, several environments do not define __int64 type,
+ * requiring a workaround.
+ */
+#if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+ typedef int64_t xxh_i64;
+#else
+ /* the following type must have a width of 64-bit */
+ typedef long long xxh_i64;
+#endif
+
+/*
+ * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
+ *
+ * It is a hardened version of UMAC, based off of FARSH's implementation.
+ *
+ * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
+ * implementations, and it is ridiculously fast.
+ *
+ * We harden it by mixing the original input to the accumulators as well as the product.
+ *
+ * This means that in the (relatively likely) case of a multiply by zero, the
+ * original input is preserved.
+ *
+ * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
+ * cross-pollination, as otherwise the upper and lower halves would be
+ * essentially independent.
+ *
+ * This doesn't matter on 64-bit hashes since they all get merged together in
+ * the end, so we skip the extra step.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+#if (XXH_VECTOR == XXH_AVX512) \
+ || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
+
+#ifndef XXH_TARGET_AVX512
+# define XXH_TARGET_AVX512 /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ __m512i* const xacc = (__m512i *) acc;
+ XXH_ASSERT((((size_t)acc) & 63) == 0);
+ XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+
+ {
+ /* data_vec = input[0]; */
+ __m512i const data_vec = _mm512_loadu_si512 (input);
+ /* key_vec = secret[0]; */
+ __m512i const key_vec = _mm512_loadu_si512 (secret);
+ /* data_key = data_vec ^ key_vec; */
+ __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
+ /* data_key_lo = data_key >> 32; */
+ __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+ __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo);
+ /* xacc[0] += swap(data_vec); */
+ __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
+ __m512i const sum = _mm512_add_epi64(*xacc, data_swap);
+ /* xacc[0] += product; */
+ *xacc = _mm512_add_epi64(product, sum);
+ }
+}
+
+/*
+ * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
+ *
+ * Multiplication isn't perfect, as explained by Google in HighwayHash:
+ *
+ * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
+ * // varying degrees. In descending order of goodness, bytes
+ * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
+ * // As expected, the upper and lower bytes are much worse.
+ *
+ * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
+ *
+ * Since our algorithm uses a pseudorandom secret to add some variance into the
+ * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
+ *
+ * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
+ * extraction.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 63) == 0);
+ XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+ { __m512i* const xacc = (__m512i*) acc;
+ const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
+
+ /* xacc[0] ^= (xacc[0] >> 47) */
+ __m512i const acc_vec = *xacc;
+ __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47);
+ __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted);
+ /* xacc[0] ^= secret; */
+ __m512i const key_vec = _mm512_loadu_si512 (secret);
+ __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
+
+ /* xacc[0] *= XXH_PRIME32_1; */
+ __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
+ __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32);
+ __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32);
+ *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
+ }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
+ XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
+ XXH_ASSERT(((size_t)customSecret & 63) == 0);
+ (void)(&XXH_writeLE64);
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
+ __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64));
+
+ const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret);
+ __m512i* const dest = ( __m512i*) customSecret;
+ int i;
+ XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
+ XXH_ASSERT(((size_t)dest & 63) == 0);
+ for (i=0; i < nbRounds; ++i) {
+ /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*',
+ * this will warn "discards 'const' qualifier". */
+ union {
+ const __m512i* cp;
+ void* p;
+ } remote_const_void;
+ remote_const_void.cp = src + i;
+ dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed);
+ } }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_AVX2) \
+ || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
+
+#ifndef XXH_TARGET_AVX2
+# define XXH_TARGET_AVX2 /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 31) == 0);
+ { __m256i* const xacc = (__m256i *) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+ const __m256i* const xinput = (const __m256i *) input;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+ const __m256i* const xsecret = (const __m256i *) secret;
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+ /* data_vec = xinput[i]; */
+ __m256i const data_vec = _mm256_loadu_si256 (xinput+i);
+ /* key_vec = xsecret[i]; */
+ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
+ /* data_key = data_vec ^ key_vec; */
+ __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
+ /* data_key_lo = data_key >> 32; */
+ __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+ __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo);
+ /* xacc[i] += swap(data_vec); */
+ __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
+ __m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
+ /* xacc[i] += product; */
+ xacc[i] = _mm256_add_epi64(product, sum);
+ } }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 31) == 0);
+ { __m256i* const xacc = (__m256i*) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+ const __m256i* const xsecret = (const __m256i *) secret;
+ const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47) */
+ __m256i const acc_vec = xacc[i];
+ __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
+ __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
+ /* xacc[i] ^= xsecret; */
+ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
+ __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
+
+ /* xacc[i] *= XXH_PRIME32_1; */
+ __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
+ __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
+ xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
+ }
+ }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
+ XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
+ (void)(&XXH_writeLE64);
+ XXH_PREFETCH(customSecret);
+ { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
+
+ const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret);
+ __m256i* dest = ( __m256i*) customSecret;
+
+# if defined(__GNUC__) || defined(__clang__)
+ /*
+ * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+ * - do not extract the secret from sse registers in the internal loop
+ * - use less common registers, and avoid pushing these reg into stack
+ */
+ XXH_COMPILER_GUARD(dest);
+# endif
+ XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
+ XXH_ASSERT(((size_t)dest & 31) == 0);
+
+ /* GCC -O2 need unroll loop manually */
+ dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed);
+ dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed);
+ dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed);
+ dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed);
+ dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed);
+ dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed);
+ }
+}
+
+#endif
+
+/* x86dispatch always generates SSE2 */
+#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
+
+#ifndef XXH_TARGET_SSE2
+# define XXH_TARGET_SSE2 /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ /* SSE2 is just a half-scale version of the AVX2 version. */
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ { __m128i* const xacc = (__m128i *) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+ const __m128i* const xinput = (const __m128i *) input;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+ const __m128i* const xsecret = (const __m128i *) secret;
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+ /* data_vec = xinput[i]; */
+ __m128i const data_vec = _mm_loadu_si128 (xinput+i);
+ /* key_vec = xsecret[i]; */
+ __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
+ /* data_key = data_vec ^ key_vec; */
+ __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
+ /* data_key_lo = data_key >> 32; */
+ __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+ __m128i const product = _mm_mul_epu32 (data_key, data_key_lo);
+ /* xacc[i] += swap(data_vec); */
+ __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
+ __m128i const sum = _mm_add_epi64(xacc[i], data_swap);
+ /* xacc[i] += product; */
+ xacc[i] = _mm_add_epi64(product, sum);
+ } }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ { __m128i* const xacc = (__m128i*) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+ const __m128i* const xsecret = (const __m128i *) secret;
+ const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47) */
+ __m128i const acc_vec = xacc[i];
+ __m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
+ __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
+ /* xacc[i] ^= xsecret[i]; */
+ __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
+ __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
+
+ /* xacc[i] *= XXH_PRIME32_1; */
+ __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
+ __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
+ xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
+ }
+ }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+ (void)(&XXH_writeLE64);
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
+
+# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
+ /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
+ XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
+ __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
+# else
+ __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
+# endif
+ int i;
+
+ const void* const src16 = XXH3_kSecret;
+ __m128i* dst16 = (__m128i*) customSecret;
+# if defined(__GNUC__) || defined(__clang__)
+ /*
+ * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+ * - do not extract the secret from sse registers in the internal loop
+ * - use less common registers, and avoid pushing these reg into stack
+ */
+ XXH_COMPILER_GUARD(dst16);
+# endif
+ XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
+ XXH_ASSERT(((size_t)dst16 & 15) == 0);
+
+ for (i=0; i < nbRounds; ++i) {
+ dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
+ } }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_NEON)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ {
+ uint64x2_t* const xacc = (uint64x2_t *) acc;
+ /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
+ uint8_t const* const xinput = (const uint8_t *) input;
+ uint8_t const* const xsecret = (const uint8_t *) secret;
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) {
+ /* data_vec = xinput[i]; */
+ uint8x16_t data_vec = vld1q_u8(xinput + (i * 16));
+ /* key_vec = xsecret[i]; */
+ uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16));
+ uint64x2_t data_key;
+ uint32x2_t data_key_lo, data_key_hi;
+ /* xacc[i] += swap(data_vec); */
+ uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec);
+ uint64x2_t const swapped = vextq_u64(data64, data64, 1);
+ xacc[i] = vaddq_u64 (xacc[i], swapped);
+ /* data_key = data_vec ^ key_vec; */
+ data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec));
+ /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF);
+ * data_key_hi = (uint32x2_t) (data_key >> 32);
+ * data_key = UNDEFINED; */
+ XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
+ /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */
+ xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi);
+
+ }
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+ { uint64x2_t* xacc = (uint64x2_t*) acc;
+ uint8_t const* xsecret = (uint8_t const*) secret;
+ uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1);
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(uint64x2_t); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47); */
+ uint64x2_t acc_vec = xacc[i];
+ uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47);
+ uint64x2_t data_vec = veorq_u64 (acc_vec, shifted);
+
+ /* xacc[i] ^= xsecret[i]; */
+ uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16));
+ uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec));
+
+ /* xacc[i] *= XXH_PRIME32_1 */
+ uint32x2_t data_key_lo, data_key_hi;
+ /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF);
+ * data_key_hi = (uint32x2_t) (xacc[i] >> 32);
+ * xacc[i] = UNDEFINED; */
+ XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
+ { /*
+ * prod_hi = (data_key >> 32) * XXH_PRIME32_1;
+ *
+ * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will
+ * incorrectly "optimize" this:
+ * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b));
+ * shifted = vshll_n_u32(tmp, 32);
+ * to this:
+ * tmp = "vmulq_u64"(a, b); // no such thing!
+ * shifted = vshlq_n_u64(tmp, 32);
+ *
+ * However, unlike SSE, Clang lacks a 64-bit multiply routine
+ * for NEON, and it scalarizes two 64-bit multiplies instead.
+ *
+ * vmull_u32 has the same timing as vmul_u32, and it avoids
+ * this bug completely.
+ * See https://bugs.llvm.org/show_bug.cgi?id=39967
+ */
+ uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime);
+ /* xacc[i] = prod_hi << 32; */
+ xacc[i] = vshlq_n_u64(prod_hi, 32);
+ /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */
+ xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime);
+ }
+ } }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_VSX)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ /* presumed aligned */
+ unsigned long long* const xacc = (unsigned long long*) acc;
+ xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */
+ xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */
+ xxh_u64x2 const v32 = { 32, 32 };
+ size_t i;
+ for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+ /* data_vec = xinput[i]; */
+ xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i);
+ /* key_vec = xsecret[i]; */
+ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
+ xxh_u64x2 const data_key = data_vec ^ key_vec;
+ /* shuffled = (data_key << 32) | (data_key >> 32); */
+ xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
+ /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
+ xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
+ /* acc_vec = xacc[i]; */
+ xxh_u64x2 acc_vec = vec_xl(0, xacc + 2 * i);
+ acc_vec += product;
+
+ /* swap high and low halves */
+#ifdef __s390x__
+ acc_vec += vec_permi(data_vec, data_vec, 2);
+#else
+ acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
+#endif
+ /* xacc[i] = acc_vec; */
+ vec_xst(acc_vec, 0, xacc + 2 * i);
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+ { xxh_u64x2* const xacc = (xxh_u64x2*) acc;
+ const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret;
+ /* constants */
+ xxh_u64x2 const v32 = { 32, 32 };
+ xxh_u64x2 const v47 = { 47, 47 };
+ xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
+ size_t i;
+ for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47); */
+ xxh_u64x2 const acc_vec = xacc[i];
+ xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
+
+ /* xacc[i] ^= xsecret[i]; */
+ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
+ xxh_u64x2 const data_key = data_vec ^ key_vec;
+
+ /* xacc[i] *= XXH_PRIME32_1 */
+ /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */
+ xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime);
+ /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */
+ xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime);
+ xacc[i] = prod_odd + (prod_even << v32);
+ } }
+}
+
+#endif
+
+/* scalar variants - universal */
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
+ const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */
+ const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
+ size_t i;
+ XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
+ for (i=0; i < XXH_ACC_NB; i++) {
+ xxh_u64 const data_val = XXH_readLE64(xinput + 8*i);
+ xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8);
+ xacc[i ^ 1] += data_val; /* swap adjacent lanes */
+ xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32);
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
+ const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
+ size_t i;
+ XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
+ for (i=0; i < XXH_ACC_NB; i++) {
+ xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i);
+ xxh_u64 acc64 = xacc[i];
+ acc64 = XXH_xorshift64(acc64, 47);
+ acc64 ^= key64;
+ acc64 *= XXH_PRIME32_1;
+ xacc[i] = acc64;
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ /*
+ * We need a separate pointer for the hack below,
+ * which requires a non-const pointer.
+ * Any decent compiler will optimize this out otherwise.
+ */
+ const xxh_u8* kSecretPtr = XXH3_kSecret;
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+
+#if defined(__clang__) && defined(__aarch64__)
+ /*
+ * UGLY HACK:
+ * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are
+ * placed sequentially, in order, at the top of the unrolled loop.
+ *
+ * While MOVK is great for generating constants (2 cycles for a 64-bit
+ * constant compared to 4 cycles for LDR), long MOVK chains stall the
+ * integer pipelines:
+ * I L S
+ * MOVK
+ * MOVK
+ * MOVK
+ * MOVK
+ * ADD
+ * SUB STR
+ * STR
+ * By forcing loads from memory (as the asm line causes Clang to assume
+ * that XXH3_kSecretPtr has been changed), the pipelines are used more
+ * efficiently:
+ * I L S
+ * LDR
+ * ADD LDR
+ * SUB STR
+ * STR
+ * XXH3_64bits_withSeed, len == 256, Snapdragon 835
+ * without hack: 2654.4 MB/s
+ * with hack: 3202.9 MB/s
+ */
+ XXH_COMPILER_GUARD(kSecretPtr);
+#endif
+ /*
+ * Note: in debug mode, this overrides the asm optimization
+ * and Clang will emit MOVK chains again.
+ */
+ XXH_ASSERT(kSecretPtr == XXH3_kSecret);
+
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
+ int i;
+ for (i=0; i < nbRounds; i++) {
+ /*
+ * The asm hack causes Clang to assume that kSecretPtr aliases with
+ * customSecret, and on aarch64, this prevented LDP from merging two
+ * loads together for free. Putting the loads together before the stores
+ * properly generates LDP.
+ */
+ xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64;
+ xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
+ XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo);
+ XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
+ } }
+}
+
+
+typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*);
+typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
+typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
+
+
+#if (XXH_VECTOR == XXH_AVX512)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
+#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
+
+#elif (XXH_VECTOR == XXH_AVX2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
+#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
+
+#elif (XXH_VECTOR == XXH_SSE2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
+#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
+
+#elif (XXH_VECTOR == XXH_NEON)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_neon
+#define XXH3_scrambleAcc XXH3_scrambleAcc_neon
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#elif (XXH_VECTOR == XXH_VSX)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
+#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#else /* scalar */
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
+#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#endif
+
+
+
+#ifndef XXH_PREFETCH_DIST
+# ifdef __clang__
+# define XXH_PREFETCH_DIST 320
+# else
+# if (XXH_VECTOR == XXH_AVX512)
+# define XXH_PREFETCH_DIST 512
+# else
+# define XXH_PREFETCH_DIST 384
+# endif
+# endif /* __clang__ */
+#endif /* XXH_PREFETCH_DIST */
+
+/*
+ * XXH3_accumulate()
+ * Loops over XXH3_accumulate_512().
+ * Assumption: nbStripes will not overflow the secret size
+ */
+XXH_FORCE_INLINE void
+XXH3_accumulate( xxh_u64* XXH_RESTRICT acc,
+ const xxh_u8* XXH_RESTRICT input,
+ const xxh_u8* XXH_RESTRICT secret,
+ size_t nbStripes,
+ XXH3_f_accumulate_512 f_acc512)
+{
+ size_t n;
+ for (n = 0; n < nbStripes; n++ ) {
+ const xxh_u8* const in = input + n*XXH_STRIPE_LEN;
+ XXH_PREFETCH(in + XXH_PREFETCH_DIST);
+ f_acc512(acc,
+ in,
+ secret + n*XXH_SECRET_CONSUME_RATE);
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
+ const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
+ size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
+ size_t const nb_blocks = (len - 1) / block_len;
+
+ size_t n;
+
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+
+ for (n = 0; n < nb_blocks; n++) {
+ XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512);
+ f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
+ }
+
+ /* last partial block */
+ XXH_ASSERT(len > XXH_STRIPE_LEN);
+ { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
+ XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
+ XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512);
+
+ /* last stripe */
+ { const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
+#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */
+ f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
+ } }
+}
+
+XXH_FORCE_INLINE xxh_u64
+XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
+{
+ return XXH3_mul128_fold64(
+ acc[0] ^ XXH_readLE64(secret),
+ acc[1] ^ XXH_readLE64(secret+8) );
+}
+
+static XXH64_hash_t
+XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
+{
+ xxh_u64 result64 = start;
+ size_t i = 0;
+
+ for (i = 0; i < 4; i++) {
+ result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
+#if defined(__clang__) /* Clang */ \
+ && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \
+ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
+ /*
+ * UGLY HACK:
+ * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
+ * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
+ * XXH3_64bits, len == 256, Snapdragon 835:
+ * without hack: 2063.7 MB/s
+ * with hack: 2560.7 MB/s
+ */
+ XXH_COMPILER_GUARD(result64);
+#endif
+ }
+
+ return XXH3_avalanche(result64);
+}
+
+#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
+ XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
+ const void* XXH_RESTRICT secret, size_t secretSize,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+ XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble);
+
+ /* converge into final hash */
+ XXH_STATIC_ASSERT(sizeof(acc) == 64);
+ /* do not align on 8, so that the secret is different from the accumulator */
+#define XXH_SECRET_MERGEACCS_START 11
+ XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+ return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
+}
+
+/*
+ * It's important for performance to transmit secret's size (when it's static)
+ * so that the compiler can properly optimize the vectorized loop.
+ * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64;
+ return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * It's preferable for performance that XXH3_hashLong is not inlined,
+ * as it results in a smaller function for small data, easier to the instruction cache.
+ * Note that inside this no_inline function, we do inline the internal loop,
+ * and provide a statically defined secret size to allow optimization of vector loop.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64; (void)secret; (void)secretLen;
+ return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * XXH3_hashLong_64b_withSeed():
+ * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
+ * and then use this key for long mode hashing.
+ *
+ * This operation is decently fast but nonetheless costs a little bit of time.
+ * Try to avoid it whenever possible (typically when seed==0).
+ *
+ * It's important for performance that XXH3_hashLong is not inlined. Not sure
+ * why (uop cache maybe?), but the difference is large and easily measurable.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
+ XXH64_hash_t seed,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble,
+ XXH3_f_initCustomSecret f_initSec)
+{
+ if (seed == 0)
+ return XXH3_hashLong_64b_internal(input, len,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ f_acc512, f_scramble);
+ { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+ f_initSec(secret, seed);
+ return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
+ f_acc512, f_scramble);
+ }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed(const void* input, size_t len,
+ XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen)
+{
+ (void)secret; (void)secretLen;
+ return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
+ XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+
+typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
+ XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+ XXH3_hashLong64_f f_hashLong)
+{
+ XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+ /*
+ * If an action is to be taken if `secretLen` condition is not respected,
+ * it should be done here.
+ * For now, it's a contract pre-condition.
+ * Adding a check and a branch here would cost performance at every hash.
+ * Also, note that function signature doesn't offer room to return an error.
+ */
+ if (len <= 16)
+ return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+ if (len <= 128)
+ return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
+}
+
+
+/* === Public entry point === */
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len)
+{
+ return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
+{
+ return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
+{
+ return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
+}
+
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
+ return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize);
+}
+
+
+/* === XXH3 streaming === */
+
+/*
+ * Malloc's a pointer that is always aligned to align.
+ *
+ * This must be freed with `XXH_alignedFree()`.
+ *
+ * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
+ * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
+ * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
+ *
+ * This underalignment previously caused a rather obvious crash which went
+ * completely unnoticed due to XXH3_createState() not actually being tested.
+ * Credit to RedSpah for noticing this bug.
+ *
+ * The alignment is done manually: Functions like posix_memalign or _mm_malloc
+ * are avoided: To maintain portability, we would have to write a fallback
+ * like this anyways, and besides, testing for the existence of library
+ * functions without relying on external build tools is impossible.
+ *
+ * The method is simple: Overallocate, manually align, and store the offset
+ * to the original behind the returned pointer.
+ *
+ * Align must be a power of 2 and 8 <= align <= 128.
+ */
+static void* XXH_alignedMalloc(size_t s, size_t align)
+{
+ XXH_ASSERT(align <= 128 && align >= 8); /* range check */
+ XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */
+ XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */
+ { /* Overallocate to make room for manual realignment and an offset byte */
+ xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
+ if (base != NULL) {
+ /*
+ * Get the offset needed to align this pointer.
+ *
+ * Even if the returned pointer is aligned, there will always be
+ * at least one byte to store the offset to the original pointer.
+ */
+ size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
+ /* Add the offset for the now-aligned pointer */
+ xxh_u8* ptr = base + offset;
+
+ XXH_ASSERT((size_t)ptr % align == 0);
+
+ /* Store the offset immediately before the returned pointer. */
+ ptr[-1] = (xxh_u8)offset;
+ return ptr;
+ }
+ return NULL;
+ }
+}
+/*
+ * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
+ * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
+ */
+static void XXH_alignedFree(void* p)
+{
+ if (p != NULL) {
+ xxh_u8* ptr = (xxh_u8*)p;
+ /* Get the offset byte we added in XXH_malloc. */
+ xxh_u8 offset = ptr[-1];
+ /* Free the original malloc'd pointer */
+ xxh_u8* base = ptr - offset;
+ XXH_free(base);
+ }
+}
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
+{
+ XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
+ if (state==NULL) return NULL;
+ XXH3_INITSTATE(state);
+ return state;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
+{
+ XXH_alignedFree(statePtr);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API void
+XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state)
+{
+ XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
+}
+
+static void
+XXH3_reset_internal(XXH3_state_t* statePtr,
+ XXH64_hash_t seed,
+ const void* secret, size_t secretSize)
+{
+ size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
+ size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
+ XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
+ XXH_ASSERT(statePtr != NULL);
+ /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
+ memset((char*)statePtr + initStart, 0, initLength);
+ statePtr->acc[0] = XXH_PRIME32_3;
+ statePtr->acc[1] = XXH_PRIME64_1;
+ statePtr->acc[2] = XXH_PRIME64_2;
+ statePtr->acc[3] = XXH_PRIME64_3;
+ statePtr->acc[4] = XXH_PRIME64_4;
+ statePtr->acc[5] = XXH_PRIME32_2;
+ statePtr->acc[6] = XXH_PRIME64_5;
+ statePtr->acc[7] = XXH_PRIME32_1;
+ statePtr->seed = seed;
+ statePtr->useSeed = (seed != 0);
+ statePtr->extSecret = (const unsigned char*)secret;
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+ statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
+ statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset(XXH3_state_t* statePtr)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ XXH3_reset_internal(statePtr, 0, secret, secretSize);
+ if (secret == NULL) return XXH_ERROR;
+ if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ if (seed==0) return XXH3_64bits_reset(statePtr);
+ if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
+ XXH3_initCustomSecret(statePtr->customSecret, seed);
+ XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ if (secret == NULL) return XXH_ERROR;
+ if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+ XXH3_reset_internal(statePtr, seed64, secret, secretSize);
+ statePtr->useSeed = 1; /* always, even if seed64==0 */
+ return XXH_OK;
+}
+
+/* Note : when XXH3_consumeStripes() is invoked,
+ * there must be a guarantee that at least one more byte must be consumed from input
+ * so that the function can blindly consume all stripes using the "normal" secret segment */
+XXH_FORCE_INLINE void
+XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
+ size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
+ const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */
+ XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock);
+ if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) {
+ /* need a scrambling operation */
+ size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr;
+ size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock;
+ XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512);
+ f_scramble(acc, secret + secretLimit);
+ XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512);
+ *nbStripesSoFarPtr = nbStripesAfterBlock;
+ } else {
+ XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512);
+ *nbStripesSoFarPtr += nbStripes;
+ }
+}
+
+#ifndef XXH3_STREAM_USE_STACK
+# ifndef __clang__ /* clang doesn't need additional stack space */
+# define XXH3_STREAM_USE_STACK 1
+# endif
+#endif
+/*
+ * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
+ */
+XXH_FORCE_INLINE XXH_errorcode
+XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
+ const xxh_u8* XXH_RESTRICT input, size_t len,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ if (input==NULL) {
+ XXH_ASSERT(len == 0);
+ return XXH_OK;
+ }
+
+ XXH_ASSERT(state != NULL);
+ { const xxh_u8* const bEnd = input + len;
+ const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
+ /* For some reason, gcc and MSVC seem to suffer greatly
+ * when operating accumulators directly into state.
+ * Operating into stack space seems to enable proper optimization.
+ * clang, on the other hand, doesn't seem to need this trick */
+ XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc));
+#else
+ xxh_u64* XXH_RESTRICT const acc = state->acc;
+#endif
+ state->totalLen += len;
+ XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
+
+ /* small input : just fill in tmp buffer */
+ if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) {
+ XXH_memcpy(state->buffer + state->bufferedSize, input, len);
+ state->bufferedSize += (XXH32_hash_t)len;
+ return XXH_OK;
+ }
+
+ /* total input is now > XXH3_INTERNALBUFFER_SIZE */
+ #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
+ XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */
+
+ /*
+ * Internal buffer is partially filled (always, except at beginning)
+ * Complete it, then consume it.
+ */
+ if (state->bufferedSize) {
+ size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
+ XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
+ input += loadSize;
+ XXH3_consumeStripes(acc,
+ &state->nbStripesSoFar, state->nbStripesPerBlock,
+ state->buffer, XXH3_INTERNALBUFFER_STRIPES,
+ secret, state->secretLimit,
+ f_acc512, f_scramble);
+ state->bufferedSize = 0;
+ }
+ XXH_ASSERT(input < bEnd);
+
+ /* large input to consume : ingest per full block */
+ if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) {
+ size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
+ XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar);
+ /* join to current block's end */
+ { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar;
+ XXH_ASSERT(nbStripes <= nbStripes);
+ XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512);
+ f_scramble(acc, secret + state->secretLimit);
+ state->nbStripesSoFar = 0;
+ input += nbStripesToEnd * XXH_STRIPE_LEN;
+ nbStripes -= nbStripesToEnd;
+ }
+ /* consume per entire blocks */
+ while(nbStripes >= state->nbStripesPerBlock) {
+ XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512);
+ f_scramble(acc, secret + state->secretLimit);
+ input += state->nbStripesPerBlock * XXH_STRIPE_LEN;
+ nbStripes -= state->nbStripesPerBlock;
+ }
+ /* consume last partial block */
+ XXH3_accumulate(acc, input, secret, nbStripes, f_acc512);
+ input += nbStripes * XXH_STRIPE_LEN;
+ XXH_ASSERT(input < bEnd); /* at least some bytes left */
+ state->nbStripesSoFar = nbStripes;
+ /* buffer predecessor of last partial stripe */
+ XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
+ XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN);
+ } else {
+ /* content to consume <= block size */
+ /* Consume input by a multiple of internal buffer size */
+ if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
+ const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE;
+ do {
+ XXH3_consumeStripes(acc,
+ &state->nbStripesSoFar, state->nbStripesPerBlock,
+ input, XXH3_INTERNALBUFFER_STRIPES,
+ secret, state->secretLimit,
+ f_acc512, f_scramble);
+ input += XXH3_INTERNALBUFFER_SIZE;
+ } while (input<limit);
+ /* buffer predecessor of last partial stripe */
+ XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
+ }
+ }
+
+ /* Some remaining input (always) : buffer it */
+ XXH_ASSERT(input < bEnd);
+ XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
+ XXH_ASSERT(state->bufferedSize == 0);
+ XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
+ state->bufferedSize = (XXH32_hash_t)(bEnd-input);
+#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
+ /* save stack accumulators into state */
+ memcpy(state->acc, acc, sizeof(acc));
+#endif
+ }
+
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len)
+{
+ return XXH3_update(state, (const xxh_u8*)input, len,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+
+XXH_FORCE_INLINE void
+XXH3_digest_long (XXH64_hash_t* acc,
+ const XXH3_state_t* state,
+ const unsigned char* secret)
+{
+ /*
+ * Digest on a local copy. This way, the state remains unaltered, and it can
+ * continue ingesting more input afterwards.
+ */
+ XXH_memcpy(acc, state->acc, sizeof(state->acc));
+ if (state->bufferedSize >= XXH_STRIPE_LEN) {
+ size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
+ size_t nbStripesSoFar = state->nbStripesSoFar;
+ XXH3_consumeStripes(acc,
+ &nbStripesSoFar, state->nbStripesPerBlock,
+ state->buffer, nbStripes,
+ secret, state->secretLimit,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+ /* last stripe */
+ XXH3_accumulate_512(acc,
+ state->buffer + state->bufferedSize - XXH_STRIPE_LEN,
+ secret + state->secretLimit - XXH_SECRET_LASTACC_START);
+ } else { /* bufferedSize < XXH_STRIPE_LEN */
+ xxh_u8 lastStripe[XXH_STRIPE_LEN];
+ size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
+ XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */
+ XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
+ XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
+ XXH3_accumulate_512(acc,
+ lastStripe,
+ secret + state->secretLimit - XXH_SECRET_LASTACC_START);
+ }
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state)
+{
+ const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+ if (state->totalLen > XXH3_MIDSIZE_MAX) {
+ XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+ XXH3_digest_long(acc, state, secret);
+ return XXH3_mergeAccs(acc,
+ secret + XXH_SECRET_MERGEACCS_START,
+ (xxh_u64)state->totalLen * XXH_PRIME64_1);
+ }
+ /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
+ if (state->useSeed)
+ return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+ return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
+ secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+
+
+
+/* ==========================================
+ * XXH3 128 bits (a.k.a XXH128)
+ * ==========================================
+ * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
+ * even without counting the significantly larger output size.
+ *
+ * For example, extra steps are taken to avoid the seed-dependent collisions
+ * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
+ *
+ * This strength naturally comes at the cost of some speed, especially on short
+ * lengths. Note that longer hashes are about as fast as the 64-bit version
+ * due to it using only a slight modification of the 64-bit loop.
+ *
+ * XXH128 is also more oriented towards 64-bit machines. It is still extremely
+ * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
+ */
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ /* A doubled version of 1to3_64b with different constants. */
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(1 <= len && len <= 3);
+ XXH_ASSERT(secret != NULL);
+ /*
+ * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
+ * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
+ * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
+ */
+ { xxh_u8 const c1 = input[0];
+ xxh_u8 const c2 = input[len >> 1];
+ xxh_u8 const c3 = input[len - 1];
+ xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
+ | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
+ xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
+ xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+ xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
+ xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
+ xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
+ XXH128_hash_t h128;
+ h128.low64 = XXH64_avalanche(keyed_lo);
+ h128.high64 = XXH64_avalanche(keyed_hi);
+ return h128;
+ }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(4 <= len && len <= 8);
+ seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+ { xxh_u32 const input_lo = XXH_readLE32(input);
+ xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
+ xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
+ xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
+ xxh_u64 const keyed = input_64 ^ bitflip;
+
+ /* Shift len to the left to ensure it is even, this avoids even multiplies. */
+ XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
+
+ m128.high64 += (m128.low64 << 1);
+ m128.low64 ^= (m128.high64 >> 3);
+
+ m128.low64 = XXH_xorshift64(m128.low64, 35);
+ m128.low64 *= 0x9FB21C651E98DF25ULL;
+ m128.low64 = XXH_xorshift64(m128.low64, 28);
+ m128.high64 = XXH3_avalanche(m128.high64);
+ return m128;
+ }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(9 <= len && len <= 16);
+ { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
+ xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
+ xxh_u64 const input_lo = XXH_readLE64(input);
+ xxh_u64 input_hi = XXH_readLE64(input + len - 8);
+ XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
+ /*
+ * Put len in the middle of m128 to ensure that the length gets mixed to
+ * both the low and high bits in the 128x64 multiply below.
+ */
+ m128.low64 += (xxh_u64)(len - 1) << 54;
+ input_hi ^= bitfliph;
+ /*
+ * Add the high 32 bits of input_hi to the high 32 bits of m128, then
+ * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
+ * the high 64 bits of m128.
+ *
+ * The best approach to this operation is different on 32-bit and 64-bit.
+ */
+ if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
+ /*
+ * 32-bit optimized version, which is more readable.
+ *
+ * On 32-bit, it removes an ADC and delays a dependency between the two
+ * halves of m128.high64, but it generates an extra mask on 64-bit.
+ */
+ m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
+ } else {
+ /*
+ * 64-bit optimized (albeit more confusing) version.
+ *
+ * Uses some properties of addition and multiplication to remove the mask:
+ *
+ * Let:
+ * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
+ * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
+ * c = XXH_PRIME32_2
+ *
+ * a + (b * c)
+ * Inverse Property: x + y - x == y
+ * a + (b * (1 + c - 1))
+ * Distributive Property: x * (y + z) == (x * y) + (x * z)
+ * a + (b * 1) + (b * (c - 1))
+ * Identity Property: x * 1 == x
+ * a + b + (b * (c - 1))
+ *
+ * Substitute a, b, and c:
+ * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+ *
+ * Since input_hi.hi + input_hi.lo == input_hi, we get this:
+ * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+ */
+ m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
+ }
+ /* m128 ^= XXH_swap64(m128 >> 64); */
+ m128.low64 ^= XXH_swap64(m128.high64);
+
+ { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
+ XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
+ h128.high64 += m128.high64 * XXH_PRIME64_2;
+
+ h128.low64 = XXH3_avalanche(h128.low64);
+ h128.high64 = XXH3_avalanche(h128.high64);
+ return h128;
+ } }
+}
+
+/*
+ * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(len <= 16);
+ { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
+ if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
+ if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
+ { XXH128_hash_t h128;
+ xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
+ xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
+ h128.low64 = XXH64_avalanche(seed ^ bitflipl);
+ h128.high64 = XXH64_avalanche( seed ^ bitfliph);
+ return h128;
+ } }
+}
+
+/*
+ * A bit slower than XXH3_mix16B, but handles multiply by zero better.
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
+ const xxh_u8* secret, XXH64_hash_t seed)
+{
+ acc.low64 += XXH3_mix16B (input_1, secret+0, seed);
+ acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
+ acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
+ acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
+ return acc;
+}
+
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(16 < len && len <= 128);
+
+ { XXH128_hash_t acc;
+ acc.low64 = len * XXH_PRIME64_1;
+ acc.high64 = 0;
+ if (len > 32) {
+ if (len > 64) {
+ if (len > 96) {
+ acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
+ }
+ acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
+ }
+ acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
+ }
+ acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
+ { XXH128_hash_t h128;
+ h128.low64 = acc.low64 + acc.high64;
+ h128.high64 = (acc.low64 * XXH_PRIME64_1)
+ + (acc.high64 * XXH_PRIME64_4)
+ + ((len - seed) * XXH_PRIME64_2);
+ h128.low64 = XXH3_avalanche(h128.low64);
+ h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+ return h128;
+ }
+ }
+}
+
+XXH_NO_INLINE XXH128_hash_t
+XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+ { XXH128_hash_t acc;
+ int const nbRounds = (int)len / 32;
+ int i;
+ acc.low64 = len * XXH_PRIME64_1;
+ acc.high64 = 0;
+ for (i=0; i<4; i++) {
+ acc = XXH128_mix32B(acc,
+ input + (32 * i),
+ input + (32 * i) + 16,
+ secret + (32 * i),
+ seed);
+ }
+ acc.low64 = XXH3_avalanche(acc.low64);
+ acc.high64 = XXH3_avalanche(acc.high64);
+ XXH_ASSERT(nbRounds >= 4);
+ for (i=4 ; i < nbRounds; i++) {
+ acc = XXH128_mix32B(acc,
+ input + (32 * i),
+ input + (32 * i) + 16,
+ secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)),
+ seed);
+ }
+ /* last bytes */
+ acc = XXH128_mix32B(acc,
+ input + len - 16,
+ input + len - 32,
+ secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
+ 0ULL - seed);
+
+ { XXH128_hash_t h128;
+ h128.low64 = acc.low64 + acc.high64;
+ h128.high64 = (acc.low64 * XXH_PRIME64_1)
+ + (acc.high64 * XXH_PRIME64_4)
+ + ((len - seed) * XXH_PRIME64_2);
+ h128.low64 = XXH3_avalanche(h128.low64);
+ h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+ return h128;
+ }
+ }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+ XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble);
+
+ /* converge into final hash */
+ XXH_STATIC_ASSERT(sizeof(acc) == 64);
+ XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+ { XXH128_hash_t h128;
+ h128.low64 = XXH3_mergeAccs(acc,
+ secret + XXH_SECRET_MERGEACCS_START,
+ (xxh_u64)len * XXH_PRIME64_1);
+ h128.high64 = XXH3_mergeAccs(acc,
+ secret + secretSize
+ - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+ ~((xxh_u64)len * XXH_PRIME64_2));
+ return h128;
+ }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64,
+ const void* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64; (void)secret; (void)secretLen;
+ return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * It's important for performance to pass @secretLen (when it's static)
+ * to the compiler, so that it can properly optimize the vectorized loop.
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64,
+ const void* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64;
+ return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble,
+ XXH3_f_initCustomSecret f_initSec)
+{
+ if (seed64 == 0)
+ return XXH3_hashLong_128b_internal(input, len,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ f_acc512, f_scramble);
+ { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+ f_initSec(secret, seed64);
+ return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
+ f_acc512, f_scramble);
+ }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed(const void* input, size_t len,
+ XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)secret; (void)secretLen;
+ return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
+ XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
+ XXH64_hash_t, const void* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_128bits_internal(const void* input, size_t len,
+ XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+ XXH3_hashLong128_f f_hl128)
+{
+ XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+ /*
+ * If an action is to be taken if `secret` conditions are not respected,
+ * it should be done here.
+ * For now, it's a contract pre-condition.
+ * Adding a check and a branch here would cost performance at every hash.
+ */
+ if (len <= 16)
+ return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+ if (len <= 128)
+ return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ return f_hl128(input, len, seed64, secret, secretLen);
+}
+
+
+/* === Public XXH128 API === */
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len)
+{
+ return XXH3_128bits_internal(input, len, 0,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ XXH3_hashLong_128b_default);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
+{
+ return XXH3_128bits_internal(input, len, 0,
+ (const xxh_u8*)secret, secretSize,
+ XXH3_hashLong_128b_withSecret);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
+{
+ return XXH3_128bits_internal(input, len, seed,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ XXH3_hashLong_128b_withSeed);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
+ return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH128(const void* input, size_t len, XXH64_hash_t seed)
+{
+ return XXH3_128bits_withSeed(input, len, seed);
+}
+
+
+/* === XXH3 128-bit streaming === */
+
+/*
+ * All initialization and update functions are identical to 64-bit streaming variant.
+ * The only difference is the finalization routine.
+ */
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset(XXH3_state_t* statePtr)
+{
+ return XXH3_64bits_reset(statePtr);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
+{
+ return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+ return XXH3_64bits_reset_withSeed(statePtr, seed);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+ return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len)
+{
+ return XXH3_update(state, (const xxh_u8*)input, len,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state)
+{
+ const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+ if (state->totalLen > XXH3_MIDSIZE_MAX) {
+ XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+ XXH3_digest_long(acc, state, secret);
+ XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+ { XXH128_hash_t h128;
+ h128.low64 = XXH3_mergeAccs(acc,
+ secret + XXH_SECRET_MERGEACCS_START,
+ (xxh_u64)state->totalLen * XXH_PRIME64_1);
+ h128.high64 = XXH3_mergeAccs(acc,
+ secret + state->secretLimit + XXH_STRIPE_LEN
+ - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+ ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
+ return h128;
+ }
+ }
+ /* len <= XXH3_MIDSIZE_MAX : short code */
+ if (state->seed)
+ return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+ return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
+ secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+
+/* 128-bit utility functions */
+
+#include <string.h> /* memcmp, memcpy */
+
+/* return : 1 is equal, 0 if different */
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
+{
+ /* note : XXH128_hash_t is compact, it has no padding byte */
+ return !(memcmp(&h1, &h2, sizeof(h1)));
+}
+
+/* This prototype is compatible with stdlib's qsort().
+ * return : >0 if *h128_1 > *h128_2
+ * <0 if *h128_1 < *h128_2
+ * =0 if *h128_1 == *h128_2 */
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2)
+{
+ XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
+ XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
+ int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
+ /* note : bets that, in most cases, hash values are different */
+ if (hcmp) return hcmp;
+ return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
+}
+
+
+/*====== Canonical representation ======*/
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API void
+XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) {
+ hash.high64 = XXH_swap64(hash.high64);
+ hash.low64 = XXH_swap64(hash.low64);
+ }
+ XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
+ XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH128_hashFromCanonical(const XXH128_canonical_t* src)
+{
+ XXH128_hash_t h;
+ h.high64 = XXH_readBE64(src);
+ h.low64 = XXH_readBE64(src->digest + 8);
+ return h;
+}
+
+
+
+/* ==========================================
+ * Secret generators
+ * ==========================================
+ */
+#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
+
+static void XXH3_combine16(void* dst, XXH128_hash_t h128)
+{
+ XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
+ XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize)
+{
+ XXH_ASSERT(secretBuffer != NULL);
+ if (secretBuffer == NULL) return XXH_ERROR;
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+ if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+ if (customSeedSize == 0) {
+ customSeed = XXH3_kSecret;
+ customSeedSize = XXH_SECRET_DEFAULT_SIZE;
+ }
+ XXH_ASSERT(customSeed != NULL);
+ if (customSeed == NULL) return XXH_ERROR;
+
+ /* Fill secretBuffer with a copy of customSeed - repeat as needed */
+ { size_t pos = 0;
+ while (pos < secretSize) {
+ size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
+ memcpy((char*)secretBuffer + pos, customSeed, toCopy);
+ pos += toCopy;
+ } }
+
+ { size_t const nbSeg16 = secretSize / 16;
+ size_t n;
+ XXH128_canonical_t scrambler;
+ XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
+ for (n=0; n<nbSeg16; n++) {
+ XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
+ XXH3_combine16((char*)secretBuffer + n*16, h128);
+ }
+ /* last segment */
+ XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
+ }
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API void
+XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed)
+{
+ XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+ XXH3_initCustomSecret(secret, seed);
+ XXH_ASSERT(secretBuffer != NULL);
+ memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
+}
+
+
+
+/* Pop our optimization override from above */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+ && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
+# pragma GCC pop_options
+#endif
+
+#endif /* XXH_NO_LONG_LONG */
+
+#endif /* XXH_NO_XXH3 */
+
+/*!
+ * @}
+ */
+#endif /* XXH_IMPLEMENTATION */
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/CHANGELOG b/contrib/zstd/CHANGELOG
new file mode 100644
index 0000000..0ed939a
--- /dev/null
+++ b/contrib/zstd/CHANGELOG
@@ -0,0 +1,555 @@
+v1.4.5
+fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln
+perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln
+perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta)
+perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh)
+perf: Small level 1 compression speed gains (depending on compiler)
+cli : New --patch-from command, create and apply patches from files, by @bimbashreshta
+cli : New --filelist= : Provide a list of files to operate upon from a file
+cli : -b -d command can now benchmark decompression on multiple files
+cli : New --no-content-size command
+cli : New --show-default-cparams information command
+api : ZDICT_finalizeDictionary() is promoted to stable (#2111)
+api : new experimental parameter ZSTD_d_stableOutBuffer (#2094)
+build: Generate a single-file libzstd library (#2065, by @cwoffenden)
+build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte)
+build: zstd now compiles cleanly under -pedantic (#2099)
+build: zstd now compiles with make-4.3
+build: Support mingw cross-compilation from Linux, by @Ericson2314
+build: Meson multi-thread build fix on windows
+build: Some misc icc fixes backed by new ci test on travis
+misc: bitflip analyzer tool, by @felixhandte
+misc: Extend largeNbDicts benchmark to compression
+misc: Edit-distance match finder in contrib/
+doc : Improved beginner CONTRIBUTING.md docs
+doc : New issue templates for zstd
+
+v1.4.4
+perf: Improved decompression speed, by > 10%, by @terrelln
+perf: Better compression speed when re-using a context, by @felixhandte
+perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
+perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
+perf: minor generic speed optimization, by @davidbolvansky
+api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
+api: fixed decoding of magic-less frames, by @terrelln
+api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
+cli: Named pipes support, by @bimbashrestha
+cli: short tar's extension support, by @stokito
+cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
+cli: commands --stream-size=# and --size-hint=#, by @nmagerko
+cli: command --exclude-compressed, by @shashank0791
+cli: faster `-t` test mode
+cli: improved some error messages, by @vangyzen
+cli: fix command `-D dictionary` on Windows, reported by @artyompetrov
+cli: fix rare deadlock condition within dictionary builder, by @terrelln
+build: single-file decoder with emscripten compilation script, by @cwoffenden
+build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
+build: fixed deprecation warning for certain gcc version, reported by @jasonma163
+build: fix compilation on old gcc versions, by @cemeyer
+build: improved installation directories for cmake script, by Dmitri Shubin
+pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
+misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
+misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
+
+v1.4.3
+bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
+bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
+build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
+
+v1.4.2
+bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
+bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
+misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
+misc: Restructure source files by @ephiepark (#1679)
+
+v1.4.1
+bug: Fix data corruption in niche use cases by @terrelln (#1659)
+bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
+bug: Fix out of bounds read by @terrelln (#1590)
+perf: Improve decode speed by ~7% @mgrice (#1668)
+perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
+perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
+perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
+perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
+api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
+cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
+cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
+cli: Restrict read permissions on destination files by @chungy (#1644)
+cli: zstdgrep: handle -f flag by @felixhandte (#1618)
+cli: zstdcat: follow symlinks by @vejnar (#1604)
+doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
+doc: Fix typo by @yk-tanigawa (#1633)
+doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
+build: CMake: support building with LZ4 @leeyoung624 (#1626)
+build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
+build: CMake: respect existing uninstall target by @j301scott (#1619)
+build: Make: skip multithread tests when built without support by @michaelforney (#1620)
+build: Make: Fix examples/ test target by @sjnam (#1603)
+build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
+build: Meson: fix build by @lzutao (#1602)
+build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
+build: Visual Studio: fix linking by @absotively (#1639)
+build: Fix MinGW-W64 build by @myzhang1029 (#1600)
+misc: Expand decodecorpus coverage by @ephiepark (#1664)
+
+v1.4.0
+perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
+api: Move the advanced API, including all functions in the staging section, to the stable section
+api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
+api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
+api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
+api: Don't export ZSTDMT functions from the shared library by default
+api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
+api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
+api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
+api: Move ZSTD_getDictID_*() functions to the stable section
+api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
+api: Allow compression parameters to be set when a dictionary is used
+api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
+api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
+api: Setting ZSTD_d_maxWindowLog to 0 means use the default
+cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
+cli: Add --[no-]compress-literals flag to enable or disable literal compression
+doc: Update the examples to use the advanced API
+doc: Explain how to transition from old streaming functions to the advanced API in the header
+build: Improve the Windows release packages
+build: Improve CMake build by @hjmjohnson
+build: Build fixes for FreeBSD by @lwhsu
+build: Remove redundant warnings by @thatsafunnyname
+build: Fix tests on OpenBSD by @bket
+build: Extend fuzzer build system to work with the new clang engine
+build: CMake now creates the libzstd.so.1 symlink
+build: Improve Menson build by @lzutao
+misc: Fix symbolic link detection on FreeBSD
+misc: Use physical core count for -T0 on FreeBSD by @cemeyer
+misc: Fix zstd --list on truncated files by @kostmo
+misc: Improve logging in debug mode by @felixhandte
+misc: Add CirrusCI tests by @lwhsu
+misc: Optimize dictionary memory usage in corner cases
+misc: Improve the dictionary builder on small or homogeneous data
+misc: Fix spelling across the repo by @jsoref
+
+v1.3.8
+perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
+perf: slightly better compression ratio at high compression modes
+api : finalized advanced API, last stage before "stable" status
+api : new --rsyncable mode, by @terrelln
+api : support decompression of empty frames into NULL (used to be an error) (#1385)
+build: new set of macros to build a minimal size decoder, by @felixhandte
+build: fix compilation on MIPS32, reported by @clbr (#1441)
+build: fix compilation with multiple -arch flags, by @ryandesign
+build: highly upgraded meson build, by @lzutao
+build: improved buck support, by @obelisk
+build: fix cmake script : can create debug build, by @pitrou
+build: Makefile : grep works on both colored consoles and systems without color support
+build: fixed zstd-pgo, by @bmwiedemann
+cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
+cli : --no-progress flag, preserving final summary (#1371), by @terrelln
+cli : ensure destination file is not source file (#1422)
+cli : clearer error messages, especially when input file not present
+doc : clarified zstd_compression_format.md, by @ulikunitz
+misc: fixed zstdgrep, returns 1 on failure, by @lzutao
+misc: NEWS renamed as CHANGELOG, in accordance with fboss
+
+v1.3.7
+perf: slightly better decompression speed on clang (depending on hardware target)
+fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
+build: no longer build backtrace by default in release mode; restrict further automatic mode
+build: control backtrace support through build macro BACKTRACE
+misc: added man pages for zstdless and zstdgrep, by @samrussell
+
+v1.3.6
+perf: much faster dictionary builder, by @jenniferliu
+perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
+perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
+cli : fix : does no longer overwrite destination when source does not exist (#1082)
+cli : new command --adapt, for automatic compression level adaptation
+api : fix : block api can be streamed with > 4 GB, reported by @catid
+api : reduced ZSTD_DDict size by 2 KB
+api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
+build: support Haiku target, by @korli
+build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
+doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
+misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
+
+v1.3.5
+perf: much faster dictionary compression, by @felixhandte
+perf: small quality improvement for dictionary generation, by @terrelln
+perf: slightly improved high compression levels (notably level 19)
+mem : automatic memory release for long duration contexts
+cli : fix : overlapLog can be manually set
+cli : fix : decoding invalid lz4 frames
+api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
+api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
+build: select custom libzstd scope through control macros, by @GeorgeLu97
+build: OpenBSD patch, by @bket
+build: make and make all are compatible with -j
+doc : clarify zstd_compression_format.md, updated for IETF RFC process
+misc: pzstd compatible with reproducible compilation, by @lamby
+
+v1.3.4
+perf: faster speed (especially decoding speed) on recent cpus (haswell+)
+perf: much better performance associating --long with multi-threading, by @terrelln
+perf: better compression at levels 13-15
+cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
+cli : smoother status report in multi-threading mode
+cli : added command --fast=#, for faster compression modes
+cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
+api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
+api : compression levels can be negative, for even more speed
+api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
+api : ZSTDMT can accept new compression parameters during compression
+api : implemented all advanced dictionary decompression prototypes
+build: improved meson recipe, by Shawn Landden (@shawnl)
+build: VS2017 scripts, by @HaydnTrigg
+misc: all /contrib projects fixed
+misc: added /contrib/docker script by @gyscos
+
+v1.3.3
+perf: faster zstd_opt strategy (levels 16-19)
+fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
+cli : fix : content size written in header by default
+cli : fix : improved LZ4 format support, by @felixhandte
+cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
+api : fix : support large skippable frames, by @terrelln
+api : fix : streaming interface was adding a useless 3-bytes null block to small frames
+api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
+build: fix : compilation under rhel6 and centos6, reported by @pixelb
+build: added `check` target
+
+v1.3.2
+new : long range mode, using --long command, by Stella Lau (@stellamplau)
+new : ability to generate and decode magicless frames (#591)
+changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
+fix : multi-threading compression works with custom allocators
+fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
+fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
+fix : 32-bits build can now decode large offsets (levels 21+)
+cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
+cli : improved --list output
+cli : new : can split input file for dictionary training, using command -B#
+cli : new : clean operation artefact on Ctrl-C interruption
+cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
+cli : fix : write file size in header in multiple-files mode
+api : added macro ZSTD_COMPRESSBOUND() for static allocation
+api : experimental : new advanced decompression API
+api : fix : sizeof_CCtx() used to over-estimate
+build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
+build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
+example : added streaming_memory_usage
+license : changed /examples license to BSD + GPLv2
+license : fix a few header files to reflect new license (#825)
+
+v1.3.1
+New license : BSD + GPLv2
+perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
+perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
+cli : improved and fixed --list command, by @ib (#772)
+cli : command -vV to list supported formats, by @ib (#771)
+build : fixed binary variants, reported by @svenha (#788)
+build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
+API exp : breaking change : ZSTD_getframeHeader() provides more information
+API exp : breaking change : pinned down values of error codes
+doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
+new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
+new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
+updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
+
+v1.3.0
+cli : new : `--list` command, by Paul Cruz
+cli : changed : xz/lzma support enabled by default
+cli : changed : `-t *` continue processing list after a decompression error
+API : added : ZSTD_versionString()
+API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
+API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
+API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
+API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
+API exp : clarified memory estimation / measurement functions.
+API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
+tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
+new : contrib/seekable_format, demo and API, by Sean Purcell
+changed : contrib/linux-kernel, updated version and license, by Nick Terrell
+
+v1.2.0
+cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
+cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
+cli : new : zstdmt symlink hardwired to `zstd -T0`
+cli : new : command --threads=# (#671)
+cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
+cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
+cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
+cli : fix : does not output compressed data on console
+cli : fix : ignore symbolic links unless --force specified,
+API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
+API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
+API : improved: ZSTDMT_compressCCtx() reduced memory usage
+API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
+API : fix : src size stored in frame header is controlled at end of frame
+API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
+API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
+build: improved cmake script, by @Majlen
+build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
+tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
+new : contrib/linux-kernel version, by Nick Terrell
+
+v1.1.4
+cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
+cli : new : advanced benchmark command --priority=rt
+cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
+cli : fix : --rm remains silent when input is stdin
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
+build : new: meson build system in contrib/meson, by Dima Krasner
+build : improved cmake script, by @Majlen
+build : added -Wformat-security flag, as recommended by Padraig Brady
+doc : new : educational decoder, by Sean Purcell
+
+v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
+cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
+cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
+cli : fix zstdless on Mac OS-X, by Andrew Janke
+cli : fix #232 "compress non-files"
+dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
+API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
+API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
+API : new : ZDICT_finalizeDictionary()
+API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+build : support for Solaris target, by Przemyslaw Skibinski
+doc : clarified specification, by Sean Purcell
+
+v1.1.2
+API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
+API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
+API : zbuff : changed : prototypes now generate deprecation warnings
+lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
+lib : changed : only public ZSTD_ symbols are now exposed
+lib : changed : reduced usage of stack memory
+lib : fixed : several corner case bugs, by Nick Terrell
+cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
+cli : new : preserve file attributes
+cli : new : added zstdless and zstdgrep tools
+cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
+cli : fixed : zstdcat
+zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
+install : better compatibility with FreeBSD, by Dimitry Andric
+source tree : changed : zbuff source files moved to lib/deprecated
+
+v1.1.1
+New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
+New : doc/zstd_manual.html, by Przemyslaw Skibinski
+Improved : slightly better compression ratio at --ultra levels (>= 20)
+Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
+Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
+Added : example/multiple_streaming_compression.c
+Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
+Updated man page
+Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
+
+v1.1.0
+New : contrib/pzstd, parallel version of zstd, by Nick Terrell
+added : NetBSD install target (#338)
+Improved : speed for batches of small files
+Improved : speed of zlib wrapper, by Przemyslaw Skibinski
+Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
+Fixed : CLI -d output to stdout by default when input is stdin (#322)
+Fixed : CLI correctly detects console on Mac OS-X
+Fixed : CLI supports recursive mode `-r` on Mac OS-X
+Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
+Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
+Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
+Fixed : zstd-pgo, reported by octoploid (#329)
+
+v1.0.0
+Change Licensing, all project is now BSD, Copyright Facebook
+Small decompression speed improvement
+API : Streaming API supports legacy format
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
+CLI supports legacy formats v0.4+
+Fixed : compression fails on certain huge files, reported by Jesse McGrew
+Enhanced documentation, by Przemyslaw Skibinski
+
+v0.8.1
+New streaming API
+Changed : --ultra now enables levels beyond 19
+Changed : -i# now selects benchmark time in second
+Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
+Fixed : speed regression on specific patterns (#272)
+Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
+Fixed : ICC compilation, by Przemyslaw Skibinski
+
+v0.8.0
+Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
+New : Build on FreeBSD and DragonFly, thanks to JrMarino
+Changed : modified API : ZSTD_compressEnd()
+Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
+Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
+Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
+Fixed : checksum correctly checked in single-pass mode
+Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
+Modified : minor compression level adaptations
+Updated : compression format specification to v0.2.0
+changed : zstd.h moved to /lib directory
+
+v0.7.5
+Transition version, supporting decoding of v0.8.x
+
+v0.7.4
+Added : homebrew for Mac, by Daniel Cade
+Added : more examples
+Fixed : segfault when using small dictionaries, reported by Felix Handte
+Modified : default compression level for CLI is now 3
+Updated : specification, to v0.1.1
+
+v0.7.3
+New : compression format specification
+New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
+New : `ZSTD_getDecompressedSize()`
+New : OpenBSD target, by Juan Francisco Cantero Hurtado
+New : `examples` directory
+fixed : dictBuilder using HC levels, reported by Bartosz Taudul
+fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
+fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
+modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
+modified : legacy functions no longer need magic number
+
+v0.7.2
+fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
+fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
+fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
+
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
+v0.7.0
+New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
+New : Command `--rm`, to remove source file after successful de/compression
+New : Visual build scripts, by Christophe Chevalier
+New : Support for Sparse File-systems (do not use space for zero-filled sectors)
+New : Frame checksum support
+New : Support pass-through mode (when using `-df`)
+API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
+API : create dictionary files from custom content, by Giuseppe Ottaviano
+API : support for custom malloc/free functions
+New : controllable Dictionary ID
+New : Support for skippable frames
+
+v0.6.1
+New : zlib wrapper API, thanks to Przemyslaw Skibinski
+New : Ability to compile compressor / decompressor separately
+Changed : new lib directory structure
+Fixed : Legacy codec v0.5 compatible with dictionary decompression
+Fixed : Decoder corruption error (#173)
+Fixed : null-string roundtrip (#176)
+New : benchmark mode can select directory as input
+Experimental : midipix support, VMS support
+
+v0.6.0
+Stronger high compression modes, thanks to Przemyslaw Skibinski
+API : ZSTD_getFrameParams() provides size of decompressed content
+New : highest compression modes require `--ultra` command to fully unleash their capacity
+Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
+
+v0.5.1
+New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
+Changed : Dictionary builder integrated into libzstd and zstd cli
+Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
+Fix : high compression modes for big-endian platforms
+New : zstd cli : `-t` | `--test` command
+
+v0.5.0
+New : dictionary builder utility
+Changed : streaming & dictionary API
+Improved : better compression of small data
+
+v0.4.7
+Improved : small compression speed improvement in HC mode
+Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
+fix : bt search bug
+
+v0.4.6
+fix : fast compression mode on Windows
+New : cmake configuration file, thanks to Artyom Dymchenko
+Improved : high compression mode on repetitive data
+New : block-level API
+New : ZSTD_duplicateCCtx()
+
+v0.4.5
+new : -m/--multiple : compress/decompress multiple files
+
+v0.4.4
+Fixed : high compression modes for Windows 32 bits
+new : external dictionary API extended to buffered mode and accessible through command line
+new : windows DLL project, thanks to Christophe Chevalier
+
+v0.4.3 :
+new : external dictionary API
+new : zstd-frugal
+
+v0.4.2 :
+Generic minor improvements for small blocks
+Fixed : big-endian compatibility, by Peter Harris (#85)
+
+v0.4.1
+Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
+removed `zstd.c`
+
+v0.4.0
+Command line utility compatible with high compression levels
+Removed zstdhc => merged into zstd
+Added : ZBUFF API (see zstd_buffered.h)
+Rolling buffer support
+
+v0.3.6
+small blocks params
+
+v0.3.5
+minor generic compression improvements
+
+v0.3.4
+Faster fast cLevels
+
+v0.3.3
+Small compression ratio improvement
+
+v0.3.2
+Fixed Visual Studio
+
+v0.3.1 :
+Small compression ratio improvement
+
+v0.3
+HC mode : compression levels 2-26
+
+v0.2.2
+Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
+
+v0.2.1
+Fix : Read errors, advanced fuzzer tests, by Hanno Böck
+
+v0.2.0
+**Breaking format change**
+Faster decompression speed
+Can still decode v0.1 format
+
+v0.1.3
+fix uninitialization warning, reported by Evan Nemerson
+
+v0.1.2
+frame concatenation support
+
+v0.1.1
+fix compression bug
+detects write-flush errors
+
+v0.1.0
+first release
diff --git a/contrib/zstd/CMakeLists.txt b/contrib/zstd/CMakeLists.txt
new file mode 100644
index 0000000..4601e53
--- /dev/null
+++ b/contrib/zstd/CMakeLists.txt
@@ -0,0 +1,27 @@
+SET(ZSTDSRC
+ debug.c
+ divsufsort.c
+ entropy_common.c
+ error_private.c
+ fse_compress.c
+ fse_decompress.c
+ hist.c
+ huf_compress.c
+ huf_decompress.c
+ pool.c
+ zstd_common.c
+ zstd_compress.c
+ zstd_compress_literals.c
+ zstd_compress_sequences.c
+ zstd_compress_superblock.c
+ zstd_ddict.c
+ zstd_decompress.c
+ zstd_decompress_block.c
+ zstd_double_fast.c
+ zstd_fast.c
+ zstd_lazy.c
+ zstd_ldm.c
+ zstd_opt.c)
+
+ADD_LIBRARY(rspamd-zstd STATIC ${ZSTDSRC})
+ADD_DEFINITIONS(-DZSTD_DISABLE_ASM) \ No newline at end of file
diff --git a/contrib/zstd/LICENSE b/contrib/zstd/LICENSE
new file mode 100644
index 0000000..a793a80
--- /dev/null
+++ b/contrib/zstd/LICENSE
@@ -0,0 +1,30 @@
+BSD License
+
+For Zstandard software
+
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+ endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/zstd/PATENTS b/contrib/zstd/PATENTS
new file mode 100644
index 0000000..15b4a2e
--- /dev/null
+++ b/contrib/zstd/PATENTS
@@ -0,0 +1,33 @@
+Additional Grant of Patent Rights Version 2
+
+"Software" means the Zstandard software distributed by Facebook, Inc.
+
+Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software
+("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable
+(subject to the termination provision below) license under any Necessary
+Claims, to make, have made, use, sell, offer to sell, import, and otherwise
+transfer the Software. For avoidance of doubt, no license is granted under
+Facebook’s rights in any patent claims that are infringed by (i) modifications
+to the Software made by you or any third party or (ii) the Software in
+combination with any software or other technology.
+
+The license granted hereunder will terminate, automatically and without notice,
+if you (or any of your subsidiaries, corporate affiliates or agents) initiate
+directly or indirectly, or take a direct financial interest in, any Patent
+Assertion: (i) against Facebook or any of its subsidiaries or corporate
+affiliates, (ii) against any party if such Patent Assertion arises in whole or
+in part from any software, technology, product or service of Facebook or any of
+its subsidiaries or corporate affiliates, or (iii) against any party relating
+to the Software. Notwithstanding the foregoing, if Facebook or any of its
+subsidiaries or corporate affiliates files a lawsuit alleging patent
+infringement against you in the first instance, and you respond by filing a
+patent infringement counterclaim in that lawsuit against that party that is
+unrelated to the Software, the license granted hereunder will not terminate
+under section (i) of this paragraph due to such counterclaim.
+
+A "Necessary Claim" is a claim of a patent owned by Facebook that is
+necessarily infringed by the Software standing alone.
+
+A "Patent Assertion" is any lawsuit or other action alleging direct, indirect,
+or contributory infringement or inducement to infringe any patent, including a
+cross-claim or counterclaim.
diff --git a/contrib/zstd/README.md b/contrib/zstd/README.md
new file mode 100644
index 0000000..e4af990
--- /dev/null
+++ b/contrib/zstd/README.md
@@ -0,0 +1,94 @@
+ **Zstd**, short for Zstandard, is a fast lossless compression algorithm,
+ targeting real-time compression scenarios at zlib-level and better compression ratios.
+
+It is provided as an open-source BSD-licensed **C** library.
+For other programming languages,
+you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
+
+|Branch |Status |
+|------------|---------|
+|master | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=master)](https://travis-ci.org/facebook/zstd) |
+|dev | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |
+
+As a reference, several fast compression algorithms were tested and compared on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.4.0, with the [Silesia compression corpus].
+
+[lzbench]: https://github.com/inikep/lzbench
+[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
+
+
+|Name | Ratio | C.speed | D.speed |
+|-----------------|-------|--------:|--------:|
+| | | MB/s | MB/s |
+|**zstd 0.8.2 -1**|**2.877**|**330**| **940** |
+| [zlib] 1.2.8 -1 | 2.730 | 95 | 360 |
+| brotli 0.4 -0 | 2.708 | 320 | 375 |
+| QuickLZ 1.5 | 2.237 | 510 | 605 |
+| LZO 2.09 | 2.106 | 610 | 870 |
+| [LZ4] r131 | 2.101 | 620 | 3100 |
+| Snappy 1.1.3 | 2.091 | 480 | 1600 |
+| LZF 3.6 | 2.077 | 375 | 790 |
+
+[zlib]:http://www.zlib.net/
+[LZ4]: http://www.lz4.org/
+
+Zstd can also offer stronger compression ratios at the cost of compression speed.
+Speed vs Compression trade-off is configurable by small increment. Decompression speed is preserved and remain roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
+
+The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.2.1, on the [Silesia compression corpus].
+
+Compression Speed vs Ratio | Decompression Speed
+---------------------------|--------------------
+![Compression Speed vs Ratio](images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed4.png "Decompression Speed")
+
+Several algorithms can produce higher compression ratio but at slower speed, falling outside of the graph.
+For a larger picture including very slow modes, [click on this link](images/DCspeed5.png) .
+
+
+### The case for Small Data compression
+
+Previous charts provide results applicable to typical files and streams scenarios (several MB). Small data come with different perspectives. The smaller the amount of data to compress, the more difficult it is to achieve any significant compression.
+
+This problem is common to any compression algorithm. The reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new file, there is no "past" to build upon.
+
+To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored in a file called "dictionary", which can be loaded before compression and decompression. Using this dictionary, the compression ratio achievable on small data improves dramatically :
+
+![Compressing Small Data](images/smallData.png "Compressing Small Data")
+
+These compression gains are achieved while simultaneously providing faster compression and decompression speeds.
+
+Dictionary work if there is some correlation in a family of small data (there is no _universal dictionary_).
+Hence, deploying one dictionary per type of data will provide the greater benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will rely more and more on previously decoded content to compress the rest of the file.
+
+#### Dictionary compression How To :
+
+1) Create the dictionary
+
+`zstd --train FullPathToTrainingSet/* -o dictionaryName`
+
+2) Compress with dictionary
+
+`zstd FILE -D dictionaryName`
+
+3) Decompress with dictionary
+
+`zstd --decompress FILE.zst -D dictionaryName`
+
+### Status
+
+Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amount of data in multiple formats and use cases.
+Zstandard is considered safe for production environments.
+
+### License
+
+Zstandard is [BSD-licensed](LICENSE). We also provide an [additional patent grant](PATENTS).
+
+### Contributing
+
+The "dev" branch is the one where all contributions will be merged before reaching "master".
+If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
+Direct commit to "master" are not permitted.
+For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
+
+### Miscellaneous
+
+Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy).
diff --git a/contrib/zstd/bits.h b/contrib/zstd/bits.h
new file mode 100644
index 0000000..7939f3d
--- /dev/null
+++ b/contrib/zstd/bits.h
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_BITS_H
+#define ZSTD_BITS_H
+
+#include "mem.h"
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
+{
+ assert(val != 0);
+ {
+ static const int DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
+ 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7,
+ 26, 12, 18, 6, 11, 5, 10, 9};
+ return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
+ }
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER)
+# if STATIC_BMI2 == 1
+ return _tzcnt_u32(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanForward(&r, val);
+ return (unsigned)r;
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (unsigned)__builtin_ctz(val);
+# else
+ return ZSTD_countTrailingZeros32_fallback(val);
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
+ assert(val != 0);
+ {
+ static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
+ 11, 14, 16, 18, 22, 25, 3, 30,
+ 8, 12, 20, 28, 15, 17, 24, 7,
+ 19, 27, 23, 6, 26, 5, 4, 31};
+ val |= val >> 1;
+ val |= val >> 2;
+ val |= val >> 4;
+ val |= val >> 8;
+ val |= val >> 16;
+ return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27];
+ }
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER)
+# if STATIC_BMI2 == 1
+ return _lzcnt_u32(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanReverse(&r, val);
+ return (unsigned)(31 - r);
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (unsigned)__builtin_clz(val);
+# else
+ return ZSTD_countLeadingZeros32_fallback(val);
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER) && defined(_WIN64)
+# if STATIC_BMI2 == 1
+ return _tzcnt_u64(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanForward64(&r, val);
+ return (unsigned)r;
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
+ return (unsigned)__builtin_ctzll(val);
+# else
+ {
+ U32 mostSignificantWord = (U32)(val >> 32);
+ U32 leastSignificantWord = (U32)val;
+ if (leastSignificantWord == 0) {
+ return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
+ } else {
+ return ZSTD_countTrailingZeros32(leastSignificantWord);
+ }
+ }
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER) && defined(_WIN64)
+# if STATIC_BMI2 == 1
+ return _lzcnt_u64(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanReverse64(&r, val);
+ return (unsigned)(63 - r);
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (unsigned)(__builtin_clzll(val));
+# else
+ {
+ U32 mostSignificantWord = (U32)(val >> 32);
+ U32 leastSignificantWord = (U32)val;
+ if (mostSignificantWord == 0) {
+ return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
+ } else {
+ return ZSTD_countLeadingZeros32(mostSignificantWord);
+ }
+ }
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
+{
+ if (MEM_isLittleEndian()) {
+ if (MEM_64bits()) {
+ return ZSTD_countTrailingZeros64((U64)val) >> 3;
+ } else {
+ return ZSTD_countTrailingZeros32((U32)val) >> 3;
+ }
+ } else { /* Big Endian CPU */
+ if (MEM_64bits()) {
+ return ZSTD_countLeadingZeros64((U64)val) >> 3;
+ } else {
+ return ZSTD_countLeadingZeros32((U32)val) >> 3;
+ }
+ }
+}
+
+MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
+{
+ assert(val != 0);
+ return 31 - ZSTD_countLeadingZeros32(val);
+}
+
+#endif /* ZSTD_BITS_H */
diff --git a/contrib/zstd/bitstream.h b/contrib/zstd/bitstream.h
new file mode 100644
index 0000000..db1b4cf
--- /dev/null
+++ b/contrib/zstd/bitstream.h
@@ -0,0 +1,437 @@
+/* ******************************************************************
+ * bitstream
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+#ifndef BITSTREAM_H_MODULE
+#define BITSTREAM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+/*
+* This API consists of small unitary functions, which must be inlined for best performance.
+* Since link-time-optimization is not available for all compilers,
+* these functions are defined into a .h to be included.
+*/
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include "mem.h" /* unaligned access routines */
+#include "compiler.h" /* UNLIKELY() */
+#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
+#include "error_private.h" /* error codes and messages */
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/*=========================================
+* Target specific
+=========================================*/
+#ifndef ZSTD_NO_INTRINSICS
+# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__)
+# include <immintrin.h> /* support for bextr (experimental)/bzhi */
+# elif defined(__ICCARM__)
+# include <intrinsics.h>
+# endif
+#endif
+
+#define STREAM_ACCUMULATOR_MIN_32 25
+#define STREAM_ACCUMULATOR_MIN_64 57
+#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
+
+
+/*-******************************************
+* bitStream encoding API (write forward)
+********************************************/
+/* bitStream can mix input from multiple sources.
+ * A critical property of these streams is that they encode and decode in **reverse** direction.
+ * So the first bit sequence you add will be the last to be read, like a LIFO stack.
+ */
+typedef struct {
+ size_t bitContainer;
+ unsigned bitPos;
+ char* startPtr;
+ char* ptr;
+ char* endPtr;
+} BIT_CStream_t;
+
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
+MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
+
+/* Start with initCStream, providing the size of buffer to write into.
+* bitStream will never write outside of this buffer.
+* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
+*
+* bits are first added to a local register.
+* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
+* Writing data into memory is an explicit operation, performed by the flushBits function.
+* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
+* After a flushBits, a maximum of 7 bits might still be stored into local register.
+*
+* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
+*
+* Last operation is to close the bitStream.
+* The function returns the final size of CStream in bytes.
+* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
+*/
+
+
+/*-********************************************
+* bitStream decoding API (read backward)
+**********************************************/
+typedef struct {
+ size_t bitContainer;
+ unsigned bitsConsumed;
+ const char* ptr;
+ const char* start;
+ const char* limitPtr;
+} BIT_DStream_t;
+
+typedef enum { BIT_DStream_unfinished = 0,
+ BIT_DStream_endOfBuffer = 1,
+ BIT_DStream_completed = 2,
+ BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
+ /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
+
+MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
+MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
+
+
+/* Start by invoking BIT_initDStream().
+* A chunk of the bitStream is then stored into a local register.
+* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+* You can then retrieve bitFields stored into the local register, **in reverse order**.
+* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
+* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
+* Otherwise, it can be less than that, so proceed accordingly.
+* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
+*/
+
+
+/*-****************************************
+* unsafe API
+******************************************/
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
+
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
+/* unsafe version; does not check buffer overflow */
+
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
+/* faster, but works only if nbBits >= 1 */
+
+/*===== Local Constants =====*/
+static const unsigned BIT_mask[] = {
+ 0, 1, 3, 7, 0xF, 0x1F,
+ 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,
+ 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
+ 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
+ 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
+ 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
+#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
+
+/*-**************************************************************
+* bitStream encoding
+****************************************************************/
+/*! BIT_initCStream() :
+ * `dstCapacity` must be > sizeof(size_t)
+ * @return : 0 if success,
+ * otherwise an error code (can be tested using ERR_isError()) */
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
+ void* startPtr, size_t dstCapacity)
+{
+ bitC->bitContainer = 0;
+ bitC->bitPos = 0;
+ bitC->startPtr = (char*)startPtr;
+ bitC->ptr = bitC->startPtr;
+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
+ if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
+ return 0;
+}
+
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
+{
+#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
+ return _bzhi_u64(bitContainer, nbBits);
+#else
+ assert(nbBits < BIT_MASK_SIZE);
+ return bitContainer & BIT_mask[nbBits];
+#endif
+}
+
+/*! BIT_addBits() :
+ * can add up to 31 bits into `bitC`.
+ * Note : does not check for register overflow ! */
+MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
+ size_t value, unsigned nbBits)
+{
+ DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
+ assert(nbBits < BIT_MASK_SIZE);
+ assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos;
+ bitC->bitPos += nbBits;
+}
+
+/*! BIT_addBitsFast() :
+ * works only if `value` is _clean_,
+ * meaning all high bits above nbBits are 0 */
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
+ size_t value, unsigned nbBits)
+{
+ assert((value>>nbBits) == 0);
+ assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ bitC->bitContainer |= value << bitC->bitPos;
+ bitC->bitPos += nbBits;
+}
+
+/*! BIT_flushBitsFast() :
+ * assumption : bitContainer has not overflowed
+ * unsafe version; does not check buffer overflow */
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
+{
+ size_t const nbBytes = bitC->bitPos >> 3;
+ assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ assert(bitC->ptr <= bitC->endPtr);
+ MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+ bitC->ptr += nbBytes;
+ bitC->bitPos &= 7;
+ bitC->bitContainer >>= nbBytes*8;
+}
+
+/*! BIT_flushBits() :
+ * assumption : bitContainer has not overflowed
+ * safe version; check for buffer overflow, and prevents it.
+ * note : does not signal buffer overflow.
+ * overflow will be revealed later on using BIT_closeCStream() */
+MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
+{
+ size_t const nbBytes = bitC->bitPos >> 3;
+ assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ assert(bitC->ptr <= bitC->endPtr);
+ MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+ bitC->ptr += nbBytes;
+ if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
+ bitC->bitPos &= 7;
+ bitC->bitContainer >>= nbBytes*8;
+}
+
+/*! BIT_closeCStream() :
+ * @return : size of CStream, in bytes,
+ * or 0 if it could not fit into dstBuffer */
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
+{
+ BIT_addBitsFast(bitC, 1, 1); /* endMark */
+ BIT_flushBits(bitC);
+ if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
+ return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
+}
+
+
+/*-********************************************************
+* bitStream decoding
+**********************************************************/
+/*! BIT_initDStream() :
+ * Initialize a BIT_DStream_t.
+ * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
+ * `srcSize` must be the *exact* size of the bitStream, in bytes.
+ * @return : size of stream (== srcSize), or an errorCode if a problem is detected
+ */
+MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
+{
+ if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
+
+ bitD->start = (const char*)srcBuffer;
+ bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
+
+ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
+ bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
+ bitD->bitContainer = MEM_readLEST(bitD->ptr);
+ { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+ bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
+ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
+ } else {
+ bitD->ptr = bitD->start;
+ bitD->bitContainer = *(const BYTE*)(bitD->start);
+ switch(srcSize)
+ {
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+ ZSTD_FALLTHROUGH;
+
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+ ZSTD_FALLTHROUGH;
+
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+ ZSTD_FALLTHROUGH;
+
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
+ ZSTD_FALLTHROUGH;
+
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
+ ZSTD_FALLTHROUGH;
+
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
+ ZSTD_FALLTHROUGH;
+
+ default: break;
+ }
+ { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+ bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
+ if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */
+ }
+ bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
+ }
+
+ return srcSize;
+}
+
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
+{
+ return bitContainer >> start;
+}
+
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
+{
+ U32 const regMask = sizeof(bitContainer)*8 - 1;
+ /* if start > regMask, bitstream is corrupted, and result is undefined */
+ assert(nbBits < BIT_MASK_SIZE);
+ /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
+ * than accessing memory. When bmi2 instruction is not present, we consider
+ * such cpus old (pre-Haswell, 2013) and their performance is not of that
+ * importance.
+ */
+#if defined(__x86_64__) || defined(_M_X86)
+ return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
+#else
+ return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
+#endif
+}
+
+/*! BIT_lookBits() :
+ * Provides next n bits from local register.
+ * local register is not modified.
+ * On 32-bits, maxNbBits==24.
+ * On 64-bits, maxNbBits==56.
+ * @return : value extracted */
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
+{
+ /* arbitrate between double-shift and shift+mask */
+#if 1
+ /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
+ * bitstream is likely corrupted, and result is undefined */
+ return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
+#else
+ /* this code path is slower on my os-x laptop */
+ U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+ return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
+#endif
+}
+
+/*! BIT_lookBitsFast() :
+ * unsafe version; only works if nbBits >= 1 */
+MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
+{
+ U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+ assert(nbBits >= 1);
+ return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
+}
+
+MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
+{
+ bitD->bitsConsumed += nbBits;
+}
+
+/*! BIT_readBits() :
+ * Read (consume) next n bits from local register and update.
+ * Pay attention to not read more than nbBits contained into local register.
+ * @return : extracted value. */
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
+{
+ size_t const value = BIT_lookBits(bitD, nbBits);
+ BIT_skipBits(bitD, nbBits);
+ return value;
+}
+
+/*! BIT_readBitsFast() :
+ * unsafe version; only works if nbBits >= 1 */
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
+{
+ size_t const value = BIT_lookBitsFast(bitD, nbBits);
+ assert(nbBits >= 1);
+ BIT_skipBits(bitD, nbBits);
+ return value;
+}
+
+/*! BIT_reloadDStreamFast() :
+ * Similar to BIT_reloadDStream(), but with two differences:
+ * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
+ * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
+ * point you must use BIT_reloadDStream() to reload.
+ */
+MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
+{
+ if (UNLIKELY(bitD->ptr < bitD->limitPtr))
+ return BIT_DStream_overflow;
+ assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
+ bitD->ptr -= bitD->bitsConsumed >> 3;
+ bitD->bitsConsumed &= 7;
+ bitD->bitContainer = MEM_readLEST(bitD->ptr);
+ return BIT_DStream_unfinished;
+}
+
+/*! BIT_reloadDStream() :
+ * Refill `bitD` from buffer previously set in BIT_initDStream() .
+ * This function is safe, it guarantees it will not read beyond src buffer.
+ * @return : status of `BIT_DStream_t` internal register.
+ * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
+{
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */
+ return BIT_DStream_overflow;
+
+ if (bitD->ptr >= bitD->limitPtr) {
+ return BIT_reloadDStreamFast(bitD);
+ }
+ if (bitD->ptr == bitD->start) {
+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
+ return BIT_DStream_completed;
+ }
+ /* start < ptr < limitPtr */
+ { U32 nbBytes = bitD->bitsConsumed >> 3;
+ BIT_DStream_status result = BIT_DStream_unfinished;
+ if (bitD->ptr - nbBytes < bitD->start) {
+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
+ result = BIT_DStream_endOfBuffer;
+ }
+ bitD->ptr -= nbBytes;
+ bitD->bitsConsumed -= nbBytes*8;
+ bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
+ return result;
+ }
+}
+
+/*! BIT_endOfDStream() :
+ * @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
+ */
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
+{
+ return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* BITSTREAM_H_MODULE */
diff --git a/contrib/zstd/clevels.h b/contrib/zstd/clevels.h
new file mode 100644
index 0000000..650a34f
--- /dev/null
+++ b/contrib/zstd/clevels.h
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_CLEVELS_H
+#define ZSTD_CLEVELS_H
+
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
+#include "zstd.h"
+
+/*-===== Pre-defined compression levels =====-*/
+
+#define ZSTD_MAX_CLEVEL 22
+
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+
+static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
+{ /* "default" - for any srcSize > 256 KB */
+ /* W, C, H, S, L, TL, strat */
+ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
+ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
+ { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
+ { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */
+ { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */
+ { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */
+ { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */
+ { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */
+ { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */
+ { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */
+ { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */
+ { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */
+ { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */
+ { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */
+ { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */
+ { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */
+ { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */
+ { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */
+ { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */
+ { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */
+ { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */
+ { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */
+ { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */
+},
+{ /* for srcSize <= 256 KB */
+ /* W, C, H, S, L, T, strat */
+ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
+ { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */
+ { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */
+ { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */
+ { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/
+ { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/
+ { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/
+ { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */
+ { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
+ { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
+ { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
+ { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/
+ { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/
+ { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */
+ { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
+ { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/
+ { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/
+ { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/
+ { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/
+ { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
+ { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/
+ { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/
+ { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/
+},
+{ /* for srcSize <= 128 KB */
+ /* W, C, H, S, L, T, strat */
+ { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
+ { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */
+ { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */
+ { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */
+ { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */
+ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */
+ { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */
+ { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */
+ { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
+ { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
+ { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
+ { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */
+ { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */
+ { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/
+ { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
+ { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/
+ { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/
+ { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/
+ { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/
+ { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/
+ { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/
+ { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
+ { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/
+},
+{ /* for srcSize <= 16 KB */
+ /* W, C, H, S, L, T, strat */
+ { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
+ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
+ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
+ { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */
+ { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */
+ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/
+ { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */
+ { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */
+ { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/
+ { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/
+ { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/
+ { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/
+ { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/
+ { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/
+ { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/
+ { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/
+ { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/
+ { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/
+ { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/
+ { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
+ { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/
+ { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
+ { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/
+},
+};
+
+
+
+#endif /* ZSTD_CLEVELS_H */
diff --git a/contrib/zstd/compiler.h b/contrib/zstd/compiler.h
new file mode 100644
index 0000000..d4f2f28
--- /dev/null
+++ b/contrib/zstd/compiler.h
@@ -0,0 +1,354 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPILER_H
+#define ZSTD_COMPILER_H
+
+#include "portability_macros.h"
+
+/*-*******************************************************
+* Compiler specifics
+*********************************************************/
+/* force inlining */
+
+#if !defined(ZSTD_NO_INLINE)
+#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# define INLINE_KEYWORD inline
+#else
+# define INLINE_KEYWORD
+#endif
+
+#if defined(__GNUC__) || defined(__ICCARM__)
+# define FORCE_INLINE_ATTR __attribute__((always_inline))
+#elif defined(_MSC_VER)
+# define FORCE_INLINE_ATTR __forceinline
+#else
+# define FORCE_INLINE_ATTR
+#endif
+
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
+#endif
+
+/**
+ On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).
+ This explicitly marks such functions as __cdecl so that the code will still compile
+ if a CC other than __cdecl has been made the default.
+*/
+#if defined(_MSC_VER)
+# define WIN_CDECL __cdecl
+#else
+# define WIN_CDECL
+#endif
+
+/**
+ * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
+ * parameters. They must be inlined for the compiler to eliminate the constant
+ * branches.
+ */
+#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
+/**
+ * HINT_INLINE is used to help the compiler generate better code. It is *not*
+ * used for "templates", so it can be tweaked based on the compilers
+ * performance.
+ *
+ * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
+ * always_inline attribute.
+ *
+ * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
+ * attribute.
+ */
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
+# define HINT_INLINE static INLINE_KEYWORD
+#else
+# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
+#endif
+
+/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
+#if defined(__GNUC__)
+# define UNUSED_ATTR __attribute__((unused))
+#else
+# define UNUSED_ATTR
+#endif
+
+/* force no inlining */
+#ifdef _MSC_VER
+# define FORCE_NOINLINE static __declspec(noinline)
+#else
+# if defined(__GNUC__) || defined(__ICCARM__)
+# define FORCE_NOINLINE static __attribute__((__noinline__))
+# else
+# define FORCE_NOINLINE static
+# endif
+#endif
+
+
+/* target attribute */
+#if defined(__GNUC__) || defined(__ICCARM__)
+# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
+#else
+# define TARGET_ATTRIBUTE(target)
+#endif
+
+/* Target attribute for BMI2 dynamic dispatch.
+ * Enable lzcnt, bmi, and bmi2.
+ * We test for bmi1 & bmi2. lzcnt is included in bmi1.
+ */
+#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
+
+/* prefetch
+ * can be disabled, by declaring NO_PREFETCH build macro */
+#if defined(NO_PREFETCH)
+# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
+# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
+#else
+# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
+# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+# elif defined(__aarch64__)
+# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr)))
+# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr)))
+# else
+# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
+# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
+# endif
+#endif /* NO_PREFETCH */
+
+#define CACHELINE_SIZE 64
+
+#define PREFETCH_AREA(p, s) { \
+ const char* const _ptr = (const char*)(p); \
+ size_t const _size = (size_t)(s); \
+ size_t _pos; \
+ for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
+ PREFETCH_L2(_ptr + _pos); \
+ } \
+}
+
+/* vectorization
+ * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
+ * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */
+#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__)
+# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
+# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+# else
+# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")
+# endif
+#else
+# define DONT_VECTORIZE
+#endif
+
+/* Tell the compiler that a branch is likely or unlikely.
+ * Only use these macros if it causes the compiler to generate better code.
+ * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc
+ * and clang, please do.
+ */
+#if defined(__GNUC__)
+#define LIKELY(x) (__builtin_expect((x), 1))
+#define UNLIKELY(x) (__builtin_expect((x), 0))
+#else
+#define LIKELY(x) (x)
+#define UNLIKELY(x) (x)
+#endif
+
+#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)))
+# define ZSTD_UNREACHABLE { assert(0), __builtin_unreachable(); }
+#else
+# define ZSTD_UNREACHABLE { assert(0); }
+#endif
+
+/* disable warnings */
+#ifdef _MSC_VER /* Visual Studio */
+# include <intrin.h> /* For Visual 2005 */
+# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
+# pragma warning(disable : 4324) /* disable: C4324: padded structure */
+#endif
+
+/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
+#ifndef STATIC_BMI2
+# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
+# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
+# define STATIC_BMI2 1
+# endif
+# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__)
+# define STATIC_BMI2 1
+# endif
+#endif
+
+#ifndef STATIC_BMI2
+ #define STATIC_BMI2 0
+#endif
+
+/* compile time determination of SIMD support */
+#if !defined(ZSTD_NO_INTRINSICS)
+# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
+# define ZSTD_ARCH_X86_SSE2
+# endif
+# if defined(__ARM_NEON) || defined(_M_ARM64)
+# define ZSTD_ARCH_ARM_NEON
+# endif
+#
+# if defined(ZSTD_ARCH_X86_SSE2)
+# include <emmintrin.h>
+# elif defined(ZSTD_ARCH_ARM_NEON)
+# include <arm_neon.h>
+# endif
+#endif
+
+/* C-language Attributes are added in C23. */
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
+# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
+#else
+# define ZSTD_HAS_C_ATTRIBUTE(x) 0
+#endif
+
+/* Only use C++ attributes in C++. Some compilers report support for C++
+ * attributes when compiling with C.
+ */
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute.
+ * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough
+ * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough
+ * - Else: __attribute__((__fallthrough__))
+ */
+#ifndef ZSTD_FALLTHROUGH
+# if ZSTD_HAS_C_ATTRIBUTE(fallthrough)
+# define ZSTD_FALLTHROUGH [[fallthrough]]
+# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough)
+# define ZSTD_FALLTHROUGH [[fallthrough]]
+# elif __has_attribute(__fallthrough__)
+/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon
+ * gcc complains about: a label can only be part of a statement and a declaration is not a statement.
+ */
+# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__))
+# else
+# define ZSTD_FALLTHROUGH
+# endif
+#endif
+
+/*-**************************************************************
+* Alignment check
+*****************************************************************/
+
+/* this test was initially positioned in mem.h,
+ * but this file is removed (or replaced) for linux kernel
+ * so it's now hosted in compiler.h,
+ * which remains valid for both user & kernel spaces.
+ */
+
+#ifndef ZSTD_ALIGNOF
+# if defined(__GNUC__) || defined(_MSC_VER)
+/* covers gcc, clang & MSVC */
+/* note : this section must come first, before C11,
+ * due to a limitation in the kernel source generator */
+# define ZSTD_ALIGNOF(T) __alignof(T)
+
+# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
+/* C11 support */
+# include <stdalign.h>
+# define ZSTD_ALIGNOF(T) alignof(T)
+
+# else
+/* No known support for alignof() - imperfect backup */
+# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T))
+
+# endif
+#endif /* ZSTD_ALIGNOF */
+
+/*-**************************************************************
+* Sanitizer
+*****************************************************************/
+
+/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
+ * abundance of caution, disable our custom poisoning on mingw. */
+#ifdef __MINGW32__
+#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE
+#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1
+#endif
+#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE
+#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1
+#endif
+#endif
+
+#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE)
+/* Not all platforms that support msan provide sanitizers/msan_interface.h.
+ * We therefore declare the functions we need ourselves, rather than trying to
+ * include the header file... */
+#include <stddef.h> /* size_t */
+#define ZSTD_DEPS_NEED_STDINT
+#include "zstd_deps.h" /* intptr_t */
+
+/* Make memory region fully initialized (without changing its contents). */
+void __msan_unpoison(const volatile void *a, size_t size);
+
+/* Make memory region fully uninitialized (without changing its contents).
+ This is a legacy interface that does not update origin information. Use
+ __msan_allocated_memory() instead. */
+void __msan_poison(const volatile void *a, size_t size);
+
+/* Returns the offset of the first (at least partially) poisoned byte in the
+ memory range, or -1 if the whole range is good. */
+intptr_t __msan_test_shadow(const volatile void *x, size_t size);
+#endif
+
+#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE)
+/* Not all platforms that support asan provide sanitizers/asan_interface.h.
+ * We therefore declare the functions we need ourselves, rather than trying to
+ * include the header file... */
+#include <stddef.h> /* size_t */
+
+/**
+ * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
+ *
+ * This memory must be previously allocated by your program. Instrumented
+ * code is forbidden from accessing addresses in this region until it is
+ * unpoisoned. This function is not guaranteed to poison the entire region -
+ * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
+ * alignment restrictions.
+ *
+ * \note This function is not thread-safe because no two threads can poison or
+ * unpoison memory in the same memory region simultaneously.
+ *
+ * \param addr Start of memory region.
+ * \param size Size of memory region. */
+void __asan_poison_memory_region(void const volatile *addr, size_t size);
+
+/**
+ * Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
+ *
+ * This memory must be previously allocated by your program. Accessing
+ * addresses in this region is allowed until this region is poisoned again.
+ * This function could unpoison a super-region of <c>[addr, addr+size)</c> due
+ * to ASan alignment restrictions.
+ *
+ * \note This function is not thread-safe because no two threads can
+ * poison or unpoison memory in the same memory region simultaneously.
+ *
+ * \param addr Start of memory region.
+ * \param size Size of memory region. */
+void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
+#endif
+
+#endif /* ZSTD_COMPILER_H */
diff --git a/contrib/zstd/cpu.h b/contrib/zstd/cpu.h
new file mode 100644
index 0000000..8bc34a3
--- /dev/null
+++ b/contrib/zstd/cpu.h
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMMON_CPU_H
+#define ZSTD_COMMON_CPU_H
+
+/**
+ * Implementation taken from folly/CpuId.h
+ * https://github.com/facebook/folly/blob/master/folly/CpuId.h
+ */
+
+#include "mem.h"
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+typedef struct {
+ U32 f1c;
+ U32 f1d;
+ U32 f7b;
+ U32 f7c;
+} ZSTD_cpuid_t;
+
+MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
+ U32 f1c = 0;
+ U32 f1d = 0;
+ U32 f7b = 0;
+ U32 f7c = 0;
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+ int reg[4];
+ __cpuid((int*)reg, 0);
+ {
+ int const n = reg[0];
+ if (n >= 1) {
+ __cpuid((int*)reg, 1);
+ f1c = (U32)reg[2];
+ f1d = (U32)reg[3];
+ }
+ if (n >= 7) {
+ __cpuidex((int*)reg, 7, 0);
+ f7b = (U32)reg[1];
+ f7c = (U32)reg[2];
+ }
+ }
+#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
+ /* The following block like the normal cpuid branch below, but gcc
+ * reserves ebx for use of its pic register so we must specially
+ * handle the save and restore to avoid clobbering the register
+ */
+ U32 n;
+ __asm__(
+ "pushl %%ebx\n\t"
+ "cpuid\n\t"
+ "popl %%ebx\n\t"
+ : "=a"(n)
+ : "a"(0)
+ : "ecx", "edx");
+ if (n >= 1) {
+ U32 f1a;
+ __asm__(
+ "pushl %%ebx\n\t"
+ "cpuid\n\t"
+ "popl %%ebx\n\t"
+ : "=a"(f1a), "=c"(f1c), "=d"(f1d)
+ : "a"(1));
+ }
+ if (n >= 7) {
+ __asm__(
+ "pushl %%ebx\n\t"
+ "cpuid\n\t"
+ "movl %%ebx, %%eax\n\t"
+ "popl %%ebx"
+ : "=a"(f7b), "=c"(f7c)
+ : "a"(7), "c"(0)
+ : "edx");
+ }
+#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
+ U32 n;
+ __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
+ if (n >= 1) {
+ U32 f1a;
+ __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
+ }
+ if (n >= 7) {
+ U32 f7a;
+ __asm__("cpuid"
+ : "=a"(f7a), "=b"(f7b), "=c"(f7c)
+ : "a"(7), "c"(0)
+ : "edx");
+ }
+#endif
+ {
+ ZSTD_cpuid_t cpuid;
+ cpuid.f1c = f1c;
+ cpuid.f1d = f1d;
+ cpuid.f7b = f7b;
+ cpuid.f7c = f7c;
+ return cpuid;
+ }
+}
+
+#define X(name, r, bit) \
+ MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \
+ return ((cpuid.r) & (1U << bit)) != 0; \
+ }
+
+/* cpuid(1): Processor Info and Feature Bits. */
+#define C(name, bit) X(name, f1c, bit)
+ C(sse3, 0)
+ C(pclmuldq, 1)
+ C(dtes64, 2)
+ C(monitor, 3)
+ C(dscpl, 4)
+ C(vmx, 5)
+ C(smx, 6)
+ C(eist, 7)
+ C(tm2, 8)
+ C(ssse3, 9)
+ C(cnxtid, 10)
+ C(fma, 12)
+ C(cx16, 13)
+ C(xtpr, 14)
+ C(pdcm, 15)
+ C(pcid, 17)
+ C(dca, 18)
+ C(sse41, 19)
+ C(sse42, 20)
+ C(x2apic, 21)
+ C(movbe, 22)
+ C(popcnt, 23)
+ C(tscdeadline, 24)
+ C(aes, 25)
+ C(xsave, 26)
+ C(osxsave, 27)
+ C(avx, 28)
+ C(f16c, 29)
+ C(rdrand, 30)
+#undef C
+#define D(name, bit) X(name, f1d, bit)
+ D(fpu, 0)
+ D(vme, 1)
+ D(de, 2)
+ D(pse, 3)
+ D(tsc, 4)
+ D(msr, 5)
+ D(pae, 6)
+ D(mce, 7)
+ D(cx8, 8)
+ D(apic, 9)
+ D(sep, 11)
+ D(mtrr, 12)
+ D(pge, 13)
+ D(mca, 14)
+ D(cmov, 15)
+ D(pat, 16)
+ D(pse36, 17)
+ D(psn, 18)
+ D(clfsh, 19)
+ D(ds, 21)
+ D(acpi, 22)
+ D(mmx, 23)
+ D(fxsr, 24)
+ D(sse, 25)
+ D(sse2, 26)
+ D(ss, 27)
+ D(htt, 28)
+ D(tm, 29)
+ D(pbe, 31)
+#undef D
+
+/* cpuid(7): Extended Features. */
+#define B(name, bit) X(name, f7b, bit)
+ B(bmi1, 3)
+ B(hle, 4)
+ B(avx2, 5)
+ B(smep, 7)
+ B(bmi2, 8)
+ B(erms, 9)
+ B(invpcid, 10)
+ B(rtm, 11)
+ B(mpx, 14)
+ B(avx512f, 16)
+ B(avx512dq, 17)
+ B(rdseed, 18)
+ B(adx, 19)
+ B(smap, 20)
+ B(avx512ifma, 21)
+ B(pcommit, 22)
+ B(clflushopt, 23)
+ B(clwb, 24)
+ B(avx512pf, 26)
+ B(avx512er, 27)
+ B(avx512cd, 28)
+ B(sha, 29)
+ B(avx512bw, 30)
+ B(avx512vl, 31)
+#undef B
+#define C(name, bit) X(name, f7c, bit)
+ C(prefetchwt1, 0)
+ C(avx512vbmi, 1)
+#undef C
+
+#undef X
+
+#endif /* ZSTD_COMMON_CPU_H */
diff --git a/contrib/zstd/debug.c b/contrib/zstd/debug.c
new file mode 100644
index 0000000..ebf7bfc
--- /dev/null
+++ b/contrib/zstd/debug.c
@@ -0,0 +1,24 @@
+/* ******************************************************************
+ * debug
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/*
+ * This module only hosts one global variable
+ * which can be used to dynamically influence the verbosity of traces,
+ * such as DEBUGLOG and RAWLOG
+ */
+
+#include "debug.h"
+
+int g_debuglevel = DEBUGLEVEL;
diff --git a/contrib/zstd/debug.h b/contrib/zstd/debug.h
new file mode 100644
index 0000000..0e9817e
--- /dev/null
+++ b/contrib/zstd/debug.h
@@ -0,0 +1,107 @@
+/* ******************************************************************
+ * debug
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/*
+ * The purpose of this header is to enable debug functions.
+ * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
+ * and DEBUG_STATIC_ASSERT() for compile-time.
+ *
+ * By default, DEBUGLEVEL==0, which means run-time debug is disabled.
+ *
+ * Level 1 enables assert() only.
+ * Starting level 2, traces can be generated and pushed to stderr.
+ * The higher the level, the more verbose the traces.
+ *
+ * It's possible to dynamically adjust level using variable g_debug_level,
+ * which is only declared if DEBUGLEVEL>=2,
+ * and is a global variable, not multi-thread protected (use with care)
+ */
+
+#ifndef DEBUG_H_12987983217
+#define DEBUG_H_12987983217
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* static assert is triggered at compile time, leaving no runtime artefact.
+ * static assert only works with compile-time constants.
+ * Also, this variant can only be used inside a function. */
+#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
+
+
+/* DEBUGLEVEL is expected to be defined externally,
+ * typically through compiler command line.
+ * Value must be a number. */
+#ifndef DEBUGLEVEL
+# define DEBUGLEVEL 0
+#endif
+
+
+/* recommended values for DEBUGLEVEL :
+ * 0 : release mode, no debug, all run-time checks disabled
+ * 1 : enables assert() only, no display
+ * 2 : reserved, for currently active debug path
+ * 3 : events once per object lifetime (CCtx, CDict, etc.)
+ * 4 : events once per frame
+ * 5 : events once per block
+ * 6 : events once per sequence (verbose)
+ * 7+: events at every position (*very* verbose)
+ *
+ * It's generally inconvenient to output traces > 5.
+ * In which case, it's possible to selectively trigger high verbosity levels
+ * by modifying g_debug_level.
+ */
+
+#if (DEBUGLEVEL>=1)
+# define ZSTD_DEPS_NEED_ASSERT
+# include "zstd_deps.h"
+#else
+# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
+# define assert(condition) ((void)0) /* disable assert (default) */
+# endif
+#endif
+
+#if (DEBUGLEVEL>=2)
+# define ZSTD_DEPS_NEED_IO
+# include "zstd_deps.h"
+extern int g_debuglevel; /* the variable is only declared,
+ it actually lives in debug.c,
+ and is shared by the whole process.
+ It's not thread-safe.
+ It's useful when enabling very verbose levels
+ on selective conditions (such as position in src) */
+
+# define RAWLOG(l, ...) { \
+ if (l<=g_debuglevel) { \
+ ZSTD_DEBUG_PRINT(__VA_ARGS__); \
+ } }
+# define DEBUGLOG(l, ...) { \
+ if (l<=g_debuglevel) { \
+ ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \
+ ZSTD_DEBUG_PRINT(" \n"); \
+ } }
+#else
+# define RAWLOG(l, ...) {} /* disabled */
+# define DEBUGLOG(l, ...) {} /* disabled */
+#endif
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* DEBUG_H_12987983217 */
diff --git a/contrib/zstd/divsufsort.c b/contrib/zstd/divsufsort.c
new file mode 100644
index 0000000..60cceb0
--- /dev/null
+++ b/contrib/zstd/divsufsort.c
@@ -0,0 +1,1913 @@
+/*
+ * divsufsort.c for libdivsufsort-lite
+ * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*- Compiler specifics -*/
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Wshorten-64-to-32"
+#endif
+
+#if defined(_MSC_VER)
+# pragma warning(disable : 4244)
+# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */
+#endif
+
+
+/*- Dependencies -*/
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "divsufsort.h"
+
+/*- Constants -*/
+#if defined(INLINE)
+# undef INLINE
+#endif
+#if !defined(INLINE)
+# define INLINE __inline
+#endif
+#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1)
+# undef ALPHABET_SIZE
+#endif
+#if !defined(ALPHABET_SIZE)
+# define ALPHABET_SIZE (256)
+#endif
+#define BUCKET_A_SIZE (ALPHABET_SIZE)
+#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE)
+#if defined(SS_INSERTIONSORT_THRESHOLD)
+# if SS_INSERTIONSORT_THRESHOLD < 1
+# undef SS_INSERTIONSORT_THRESHOLD
+# define SS_INSERTIONSORT_THRESHOLD (1)
+# endif
+#else
+# define SS_INSERTIONSORT_THRESHOLD (8)
+#endif
+#if defined(SS_BLOCKSIZE)
+# if SS_BLOCKSIZE < 0
+# undef SS_BLOCKSIZE
+# define SS_BLOCKSIZE (0)
+# elif 32768 <= SS_BLOCKSIZE
+# undef SS_BLOCKSIZE
+# define SS_BLOCKSIZE (32767)
+# endif
+#else
+# define SS_BLOCKSIZE (1024)
+#endif
+/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */
+#if SS_BLOCKSIZE == 0
+# define SS_MISORT_STACKSIZE (96)
+#elif SS_BLOCKSIZE <= 4096
+# define SS_MISORT_STACKSIZE (16)
+#else
+# define SS_MISORT_STACKSIZE (24)
+#endif
+#define SS_SMERGE_STACKSIZE (32)
+#define TR_INSERTIONSORT_THRESHOLD (8)
+#define TR_STACKSIZE (64)
+
+
+/*- Macros -*/
+#ifndef SWAP
+# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0)
+#endif /* SWAP */
+#ifndef MIN
+# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b))
+#endif /* MIN */
+#ifndef MAX
+# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b))
+#endif /* MAX */
+#define STACK_PUSH(_a, _b, _c, _d)\
+ do {\
+ assert(ssize < STACK_SIZE);\
+ stack[ssize].a = (_a), stack[ssize].b = (_b),\
+ stack[ssize].c = (_c), stack[ssize++].d = (_d);\
+ } while(0)
+#define STACK_PUSH5(_a, _b, _c, _d, _e)\
+ do {\
+ assert(ssize < STACK_SIZE);\
+ stack[ssize].a = (_a), stack[ssize].b = (_b),\
+ stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\
+ } while(0)
+#define STACK_POP(_a, _b, _c, _d)\
+ do {\
+ assert(0 <= ssize);\
+ if(ssize == 0) { return; }\
+ (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
+ (_c) = stack[ssize].c, (_d) = stack[ssize].d;\
+ } while(0)
+#define STACK_POP5(_a, _b, _c, _d, _e)\
+ do {\
+ assert(0 <= ssize);\
+ if(ssize == 0) { return; }\
+ (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
+ (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\
+ } while(0)
+#define BUCKET_A(_c0) bucket_A[(_c0)]
+#if ALPHABET_SIZE == 256
+#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)])
+#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)])
+#else
+#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)])
+#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)])
+#endif
+
+
+/*- Private Functions -*/
+
+static const int lg_table[256]= {
+ -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
+ 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7
+};
+
+#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
+
+static INLINE
+int
+ss_ilg(int n) {
+#if SS_BLOCKSIZE == 0
+ return (n & 0xffff0000) ?
+ ((n & 0xff000000) ?
+ 24 + lg_table[(n >> 24) & 0xff] :
+ 16 + lg_table[(n >> 16) & 0xff]) :
+ ((n & 0x0000ff00) ?
+ 8 + lg_table[(n >> 8) & 0xff] :
+ 0 + lg_table[(n >> 0) & 0xff]);
+#elif SS_BLOCKSIZE < 256
+ return lg_table[n];
+#else
+ return (n & 0xff00) ?
+ 8 + lg_table[(n >> 8) & 0xff] :
+ 0 + lg_table[(n >> 0) & 0xff];
+#endif
+}
+
+#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
+
+#if SS_BLOCKSIZE != 0
+
+static const int sqq_table[256] = {
+ 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61,
+ 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89,
+ 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109,
+110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126,
+128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
+143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155,
+156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168,
+169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180,
+181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191,
+192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201,
+202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211,
+212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221,
+221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230,
+230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238,
+239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247,
+247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255
+};
+
+static INLINE
+int
+ss_isqrt(int x) {
+ int y, e;
+
+ if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; }
+ e = (x & 0xffff0000) ?
+ ((x & 0xff000000) ?
+ 24 + lg_table[(x >> 24) & 0xff] :
+ 16 + lg_table[(x >> 16) & 0xff]) :
+ ((x & 0x0000ff00) ?
+ 8 + lg_table[(x >> 8) & 0xff] :
+ 0 + lg_table[(x >> 0) & 0xff]);
+
+ if(e >= 16) {
+ y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7);
+ if(e >= 24) { y = (y + 1 + x / y) >> 1; }
+ y = (y + 1 + x / y) >> 1;
+ } else if(e >= 8) {
+ y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1;
+ } else {
+ return sqq_table[x] >> 4;
+ }
+
+ return (x < (y * y)) ? y - 1 : y;
+}
+
+#endif /* SS_BLOCKSIZE != 0 */
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Compares two suffixes. */
+static INLINE
+int
+ss_compare(const unsigned char *T,
+ const int *p1, const int *p2,
+ int depth) {
+ const unsigned char *U1, *U2, *U1n, *U2n;
+
+ for(U1 = T + depth + *p1,
+ U2 = T + depth + *p2,
+ U1n = T + *(p1 + 1) + 2,
+ U2n = T + *(p2 + 1) + 2;
+ (U1 < U1n) && (U2 < U2n) && (*U1 == *U2);
+ ++U1, ++U2) {
+ }
+
+ return U1 < U1n ?
+ (U2 < U2n ? *U1 - *U2 : 1) :
+ (U2 < U2n ? -1 : 0);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1)
+
+/* Insertionsort for small size groups */
+static
+void
+ss_insertionsort(const unsigned char *T, const int *PA,
+ int *first, int *last, int depth) {
+ int *i, *j;
+ int t;
+ int r;
+
+ for(i = last - 2; first <= i; --i) {
+ for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) {
+ do { *(j - 1) = *j; } while((++j < last) && (*j < 0));
+ if(last <= j) { break; }
+ }
+ if(r == 0) { *j = ~*j; }
+ *(j - 1) = t;
+ }
+}
+
+#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */
+
+
+/*---------------------------------------------------------------------------*/
+
+#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
+
+static INLINE
+void
+ss_fixdown(const unsigned char *Td, const int *PA,
+ int *SA, int i, int size) {
+ int j, k;
+ int v;
+ int c, d, e;
+
+ for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
+ d = Td[PA[SA[k = j++]]];
+ if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; }
+ if(d <= c) { break; }
+ }
+ SA[i] = v;
+}
+
+/* Simple top-down heapsort. */
+static
+void
+ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) {
+ int i, m;
+ int t;
+
+ m = size;
+ if((size % 2) == 0) {
+ m--;
+ if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); }
+ }
+
+ for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); }
+ if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); }
+ for(i = m - 1; 0 < i; --i) {
+ t = SA[0], SA[0] = SA[i];
+ ss_fixdown(Td, PA, SA, 0, i);
+ SA[i] = t;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Returns the median of three elements. */
+static INLINE
+int *
+ss_median3(const unsigned char *Td, const int *PA,
+ int *v1, int *v2, int *v3) {
+ int *t;
+ if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); }
+ if(Td[PA[*v2]] > Td[PA[*v3]]) {
+ if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; }
+ else { return v3; }
+ }
+ return v2;
+}
+
+/* Returns the median of five elements. */
+static INLINE
+int *
+ss_median5(const unsigned char *Td, const int *PA,
+ int *v1, int *v2, int *v3, int *v4, int *v5) {
+ int *t;
+ if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); }
+ if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); }
+ if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); }
+ if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); }
+ if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); }
+ if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; }
+ return v3;
+}
+
+/* Returns the pivot element. */
+static INLINE
+int *
+ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) {
+ int *middle;
+ int t;
+
+ t = last - first;
+ middle = first + t / 2;
+
+ if(t <= 512) {
+ if(t <= 32) {
+ return ss_median3(Td, PA, first, middle, last - 1);
+ } else {
+ t >>= 2;
+ return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1);
+ }
+ }
+ t >>= 3;
+ first = ss_median3(Td, PA, first, first + t, first + (t << 1));
+ middle = ss_median3(Td, PA, middle - t, middle, middle + t);
+ last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1);
+ return ss_median3(Td, PA, first, middle, last);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Binary partition for substrings. */
+static INLINE
+int *
+ss_partition(const int *PA,
+ int *first, int *last, int depth) {
+ int *a, *b;
+ int t;
+ for(a = first - 1, b = last;;) {
+ for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; }
+ for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { }
+ if(b <= a) { break; }
+ t = ~*b;
+ *b = *a;
+ *a = t;
+ }
+ if(first < a) { *first = ~*first; }
+ return a;
+}
+
+/* Multikey introsort for medium size groups. */
+static
+void
+ss_mintrosort(const unsigned char *T, const int *PA,
+ int *first, int *last,
+ int depth) {
+#define STACK_SIZE SS_MISORT_STACKSIZE
+ struct { int *a, *b, c; int d; } stack[STACK_SIZE];
+ const unsigned char *Td;
+ int *a, *b, *c, *d, *e, *f;
+ int s, t;
+ int ssize;
+ int limit;
+ int v, x = 0;
+
+ for(ssize = 0, limit = ss_ilg(last - first);;) {
+
+ if((last - first) <= SS_INSERTIONSORT_THRESHOLD) {
+#if 1 < SS_INSERTIONSORT_THRESHOLD
+ if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); }
+#endif
+ STACK_POP(first, last, depth, limit);
+ continue;
+ }
+
+ Td = T + depth;
+ if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); }
+ if(limit < 0) {
+ for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) {
+ if((x = Td[PA[*a]]) != v) {
+ if(1 < (a - first)) { break; }
+ v = x;
+ first = a;
+ }
+ }
+ if(Td[PA[*first] - 1] < v) {
+ first = ss_partition(PA, first, a, depth);
+ }
+ if((a - first) <= (last - a)) {
+ if(1 < (a - first)) {
+ STACK_PUSH(a, last, depth, -1);
+ last = a, depth += 1, limit = ss_ilg(a - first);
+ } else {
+ first = a, limit = -1;
+ }
+ } else {
+ if(1 < (last - a)) {
+ STACK_PUSH(first, a, depth + 1, ss_ilg(a - first));
+ first = a, limit = -1;
+ } else {
+ last = a, depth += 1, limit = ss_ilg(a - first);
+ }
+ }
+ continue;
+ }
+
+ /* choose pivot */
+ a = ss_pivot(Td, PA, first, last);
+ v = Td[PA[*a]];
+ SWAP(*first, *a);
+
+ /* partition */
+ for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { }
+ if(((a = b) < last) && (x < v)) {
+ for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ }
+ for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { }
+ if((b < (d = c)) && (x > v)) {
+ for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+ for(; b < c;) {
+ SWAP(*b, *c);
+ for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+
+ if(a <= d) {
+ c = b - 1;
+
+ if((s = a - first) > (t = b - a)) { s = t; }
+ for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+ if((s = d - c) > (t = last - d - 1)) { s = t; }
+ for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+
+ a = first + (b - a), c = last - (d - c);
+ b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth);
+
+ if((a - first) <= (last - c)) {
+ if((last - c) <= (c - b)) {
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ STACK_PUSH(c, last, depth, limit);
+ last = a;
+ } else if((a - first) <= (c - b)) {
+ STACK_PUSH(c, last, depth, limit);
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ last = a;
+ } else {
+ STACK_PUSH(c, last, depth, limit);
+ STACK_PUSH(first, a, depth, limit);
+ first = b, last = c, depth += 1, limit = ss_ilg(c - b);
+ }
+ } else {
+ if((a - first) <= (c - b)) {
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ STACK_PUSH(first, a, depth, limit);
+ first = c;
+ } else if((last - c) <= (c - b)) {
+ STACK_PUSH(first, a, depth, limit);
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ first = c;
+ } else {
+ STACK_PUSH(first, a, depth, limit);
+ STACK_PUSH(c, last, depth, limit);
+ first = b, last = c, depth += 1, limit = ss_ilg(c - b);
+ }
+ }
+ } else {
+ limit += 1;
+ if(Td[PA[*first] - 1] < v) {
+ first = ss_partition(PA, first, last, depth);
+ limit = ss_ilg(last - first);
+ }
+ depth += 1;
+ }
+ }
+#undef STACK_SIZE
+}
+
+#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
+
+
+/*---------------------------------------------------------------------------*/
+
+#if SS_BLOCKSIZE != 0
+
+static INLINE
+void
+ss_blockswap(int *a, int *b, int n) {
+ int t;
+ for(; 0 < n; --n, ++a, ++b) {
+ t = *a, *a = *b, *b = t;
+ }
+}
+
+static INLINE
+void
+ss_rotate(int *first, int *middle, int *last) {
+ int *a, *b, t;
+ int l, r;
+ l = middle - first, r = last - middle;
+ for(; (0 < l) && (0 < r);) {
+ if(l == r) { ss_blockswap(first, middle, l); break; }
+ if(l < r) {
+ a = last - 1, b = middle - 1;
+ t = *a;
+ do {
+ *a-- = *b, *b-- = *a;
+ if(b < first) {
+ *a = t;
+ last = a;
+ if((r -= l + 1) <= l) { break; }
+ a -= 1, b = middle - 1;
+ t = *a;
+ }
+ } while(1);
+ } else {
+ a = first, b = middle;
+ t = *a;
+ do {
+ *a++ = *b, *b++ = *a;
+ if(last <= b) {
+ *a = t;
+ first = a + 1;
+ if((l -= r + 1) <= r) { break; }
+ a += 1, b = middle;
+ t = *a;
+ }
+ } while(1);
+ }
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static
+void
+ss_inplacemerge(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int depth) {
+ const int *p;
+ int *a, *b;
+ int len, half;
+ int q, r;
+ int x;
+
+ for(;;) {
+ if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); }
+ else { x = 0; p = PA + *(last - 1); }
+ for(a = first, len = middle - first, half = len >> 1, r = -1;
+ 0 < len;
+ len = half, half >>= 1) {
+ b = a + half;
+ q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth);
+ if(q < 0) {
+ a = b + 1;
+ half -= (len & 1) ^ 1;
+ } else {
+ r = q;
+ }
+ }
+ if(a < middle) {
+ if(r == 0) { *a = ~*a; }
+ ss_rotate(a, middle, last);
+ last -= middle - a;
+ middle = a;
+ if(first == middle) { break; }
+ }
+ --last;
+ if(x != 0) { while(*--last < 0) { } }
+ if(middle == last) { break; }
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Merge-forward with internal buffer. */
+static
+void
+ss_mergeforward(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int *buf, int depth) {
+ int *a, *b, *c, *bufend;
+ int t;
+ int r;
+
+ bufend = buf + (middle - first) - 1;
+ ss_blockswap(buf, first, middle - first);
+
+ for(t = *(a = first), b = buf, c = middle;;) {
+ r = ss_compare(T, PA + *b, PA + *c, depth);
+ if(r < 0) {
+ do {
+ *a++ = *b;
+ if(bufend <= b) { *bufend = t; return; }
+ *b++ = *a;
+ } while(*b < 0);
+ } else if(r > 0) {
+ do {
+ *a++ = *c, *c++ = *a;
+ if(last <= c) {
+ while(b < bufend) { *a++ = *b, *b++ = *a; }
+ *a = *b, *b = t;
+ return;
+ }
+ } while(*c < 0);
+ } else {
+ *c = ~*c;
+ do {
+ *a++ = *b;
+ if(bufend <= b) { *bufend = t; return; }
+ *b++ = *a;
+ } while(*b < 0);
+
+ do {
+ *a++ = *c, *c++ = *a;
+ if(last <= c) {
+ while(b < bufend) { *a++ = *b, *b++ = *a; }
+ *a = *b, *b = t;
+ return;
+ }
+ } while(*c < 0);
+ }
+ }
+}
+
+/* Merge-backward with internal buffer. */
+static
+void
+ss_mergebackward(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int *buf, int depth) {
+ const int *p1, *p2;
+ int *a, *b, *c, *bufend;
+ int t;
+ int r;
+ int x;
+
+ bufend = buf + (last - middle) - 1;
+ ss_blockswap(buf, middle, last - middle);
+
+ x = 0;
+ if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; }
+ else { p1 = PA + *bufend; }
+ if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; }
+ else { p2 = PA + *(middle - 1); }
+ for(t = *(a = last - 1), b = bufend, c = middle - 1;;) {
+ r = ss_compare(T, p1, p2, depth);
+ if(0 < r) {
+ if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
+ *a-- = *b;
+ if(b <= buf) { *buf = t; break; }
+ *b-- = *a;
+ if(*b < 0) { p1 = PA + ~*b; x |= 1; }
+ else { p1 = PA + *b; }
+ } else if(r < 0) {
+ if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
+ *a-- = *c, *c-- = *a;
+ if(c < first) {
+ while(buf < b) { *a-- = *b, *b-- = *a; }
+ *a = *b, *b = t;
+ break;
+ }
+ if(*c < 0) { p2 = PA + ~*c; x |= 2; }
+ else { p2 = PA + *c; }
+ } else {
+ if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
+ *a-- = ~*b;
+ if(b <= buf) { *buf = t; break; }
+ *b-- = *a;
+ if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
+ *a-- = *c, *c-- = *a;
+ if(c < first) {
+ while(buf < b) { *a-- = *b, *b-- = *a; }
+ *a = *b, *b = t;
+ break;
+ }
+ if(*b < 0) { p1 = PA + ~*b; x |= 1; }
+ else { p1 = PA + *b; }
+ if(*c < 0) { p2 = PA + ~*c; x |= 2; }
+ else { p2 = PA + *c; }
+ }
+ }
+}
+
+/* D&C based merge. */
+static
+void
+ss_swapmerge(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int *buf, int bufsize, int depth) {
+#define STACK_SIZE SS_SMERGE_STACKSIZE
+#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a)))
+#define MERGE_CHECK(a, b, c)\
+ do {\
+ if(((c) & 1) ||\
+ (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\
+ *(a) = ~*(a);\
+ }\
+ if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\
+ *(b) = ~*(b);\
+ }\
+ } while(0)
+ struct { int *a, *b, *c; int d; } stack[STACK_SIZE];
+ int *l, *r, *lm, *rm;
+ int m, len, half;
+ int ssize;
+ int check, next;
+
+ for(check = 0, ssize = 0;;) {
+ if((last - middle) <= bufsize) {
+ if((first < middle) && (middle < last)) {
+ ss_mergebackward(T, PA, first, middle, last, buf, depth);
+ }
+ MERGE_CHECK(first, last, check);
+ STACK_POP(first, middle, last, check);
+ continue;
+ }
+
+ if((middle - first) <= bufsize) {
+ if(first < middle) {
+ ss_mergeforward(T, PA, first, middle, last, buf, depth);
+ }
+ MERGE_CHECK(first, last, check);
+ STACK_POP(first, middle, last, check);
+ continue;
+ }
+
+ for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1;
+ 0 < len;
+ len = half, half >>= 1) {
+ if(ss_compare(T, PA + GETIDX(*(middle + m + half)),
+ PA + GETIDX(*(middle - m - half - 1)), depth) < 0) {
+ m += half + 1;
+ half -= (len & 1) ^ 1;
+ }
+ }
+
+ if(0 < m) {
+ lm = middle - m, rm = middle + m;
+ ss_blockswap(lm, middle, m);
+ l = r = middle, next = 0;
+ if(rm < last) {
+ if(*rm < 0) {
+ *rm = ~*rm;
+ if(first < lm) { for(; *--l < 0;) { } next |= 4; }
+ next |= 1;
+ } else if(first < lm) {
+ for(; *r < 0; ++r) { }
+ next |= 2;
+ }
+ }
+
+ if((l - first) <= (last - r)) {
+ STACK_PUSH(r, rm, last, (next & 3) | (check & 4));
+ middle = lm, last = l, check = (check & 3) | (next & 4);
+ } else {
+ if((next & 2) && (r == middle)) { next ^= 6; }
+ STACK_PUSH(first, lm, l, (check & 3) | (next & 4));
+ first = r, middle = rm, check = (next & 3) | (check & 4);
+ }
+ } else {
+ if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) {
+ *middle = ~*middle;
+ }
+ MERGE_CHECK(first, last, check);
+ STACK_POP(first, middle, last, check);
+ }
+ }
+#undef STACK_SIZE
+}
+
+#endif /* SS_BLOCKSIZE != 0 */
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Substring sort */
+static
+void
+sssort(const unsigned char *T, const int *PA,
+ int *first, int *last,
+ int *buf, int bufsize,
+ int depth, int n, int lastsuffix) {
+ int *a;
+#if SS_BLOCKSIZE != 0
+ int *b, *middle, *curbuf;
+ int j, k, curbufsize, limit;
+#endif
+ int i;
+
+ if(lastsuffix != 0) { ++first; }
+
+#if SS_BLOCKSIZE == 0
+ ss_mintrosort(T, PA, first, last, depth);
+#else
+ if((bufsize < SS_BLOCKSIZE) &&
+ (bufsize < (last - first)) &&
+ (bufsize < (limit = ss_isqrt(last - first)))) {
+ if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; }
+ buf = middle = last - limit, bufsize = limit;
+ } else {
+ middle = last, limit = 0;
+ }
+ for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) {
+#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
+ ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth);
+#elif 1 < SS_BLOCKSIZE
+ ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth);
+#endif
+ curbufsize = last - (a + SS_BLOCKSIZE);
+ curbuf = a + SS_BLOCKSIZE;
+ if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; }
+ for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) {
+ ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth);
+ }
+ }
+#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
+ ss_mintrosort(T, PA, a, middle, depth);
+#elif 1 < SS_BLOCKSIZE
+ ss_insertionsort(T, PA, a, middle, depth);
+#endif
+ for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) {
+ if(i & 1) {
+ ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth);
+ a -= k;
+ }
+ }
+ if(limit != 0) {
+#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
+ ss_mintrosort(T, PA, middle, last, depth);
+#elif 1 < SS_BLOCKSIZE
+ ss_insertionsort(T, PA, middle, last, depth);
+#endif
+ ss_inplacemerge(T, PA, first, middle, last, depth);
+ }
+#endif
+
+ if(lastsuffix != 0) {
+ /* Insert last type B* suffix. */
+ int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2;
+ for(a = first, i = *(first - 1);
+ (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth)));
+ ++a) {
+ *(a - 1) = *a;
+ }
+ *(a - 1) = i;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static INLINE
+int
+tr_ilg(int n) {
+ return (n & 0xffff0000) ?
+ ((n & 0xff000000) ?
+ 24 + lg_table[(n >> 24) & 0xff] :
+ 16 + lg_table[(n >> 16) & 0xff]) :
+ ((n & 0x0000ff00) ?
+ 8 + lg_table[(n >> 8) & 0xff] :
+ 0 + lg_table[(n >> 0) & 0xff]);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Simple insertionsort for small size groups. */
+static
+void
+tr_insertionsort(const int *ISAd, int *first, int *last) {
+ int *a, *b;
+ int t, r;
+
+ for(a = first + 1; a < last; ++a) {
+ for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) {
+ do { *(b + 1) = *b; } while((first <= --b) && (*b < 0));
+ if(b < first) { break; }
+ }
+ if(r == 0) { *b = ~*b; }
+ *(b + 1) = t;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static INLINE
+void
+tr_fixdown(const int *ISAd, int *SA, int i, int size) {
+ int j, k;
+ int v;
+ int c, d, e;
+
+ for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
+ d = ISAd[SA[k = j++]];
+ if(d < (e = ISAd[SA[j]])) { k = j; d = e; }
+ if(d <= c) { break; }
+ }
+ SA[i] = v;
+}
+
+/* Simple top-down heapsort. */
+static
+void
+tr_heapsort(const int *ISAd, int *SA, int size) {
+ int i, m;
+ int t;
+
+ m = size;
+ if((size % 2) == 0) {
+ m--;
+ if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); }
+ }
+
+ for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); }
+ if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); }
+ for(i = m - 1; 0 < i; --i) {
+ t = SA[0], SA[0] = SA[i];
+ tr_fixdown(ISAd, SA, 0, i);
+ SA[i] = t;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Returns the median of three elements. */
+static INLINE
+int *
+tr_median3(const int *ISAd, int *v1, int *v2, int *v3) {
+ int *t;
+ if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); }
+ if(ISAd[*v2] > ISAd[*v3]) {
+ if(ISAd[*v1] > ISAd[*v3]) { return v1; }
+ else { return v3; }
+ }
+ return v2;
+}
+
+/* Returns the median of five elements. */
+static INLINE
+int *
+tr_median5(const int *ISAd,
+ int *v1, int *v2, int *v3, int *v4, int *v5) {
+ int *t;
+ if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); }
+ if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); }
+ if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); }
+ if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); }
+ if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); }
+ if(ISAd[*v3] > ISAd[*v4]) { return v4; }
+ return v3;
+}
+
+/* Returns the pivot element. */
+static INLINE
+int *
+tr_pivot(const int *ISAd, int *first, int *last) {
+ int *middle;
+ int t;
+
+ t = last - first;
+ middle = first + t / 2;
+
+ if(t <= 512) {
+ if(t <= 32) {
+ return tr_median3(ISAd, first, middle, last - 1);
+ } else {
+ t >>= 2;
+ return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1);
+ }
+ }
+ t >>= 3;
+ first = tr_median3(ISAd, first, first + t, first + (t << 1));
+ middle = tr_median3(ISAd, middle - t, middle, middle + t);
+ last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1);
+ return tr_median3(ISAd, first, middle, last);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+typedef struct _trbudget_t trbudget_t;
+struct _trbudget_t {
+ int chance;
+ int remain;
+ int incval;
+ int count;
+};
+
+static INLINE
+void
+trbudget_init(trbudget_t *budget, int chance, int incval) {
+ budget->chance = chance;
+ budget->remain = budget->incval = incval;
+}
+
+static INLINE
+int
+trbudget_check(trbudget_t *budget, int size) {
+ if(size <= budget->remain) { budget->remain -= size; return 1; }
+ if(budget->chance == 0) { budget->count += size; return 0; }
+ budget->remain += budget->incval - size;
+ budget->chance -= 1;
+ return 1;
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static INLINE
+void
+tr_partition(const int *ISAd,
+ int *first, int *middle, int *last,
+ int **pa, int **pb, int v) {
+ int *a, *b, *c, *d, *e, *f;
+ int t, s;
+ int x = 0;
+
+ for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { }
+ if(((a = b) < last) && (x < v)) {
+ for(; (++b < last) && ((x = ISAd[*b]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ }
+ for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { }
+ if((b < (d = c)) && (x > v)) {
+ for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+ for(; b < c;) {
+ SWAP(*b, *c);
+ for(; (++b < c) && ((x = ISAd[*b]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+
+ if(a <= d) {
+ c = b - 1;
+ if((s = a - first) > (t = b - a)) { s = t; }
+ for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+ if((s = d - c) > (t = last - d - 1)) { s = t; }
+ for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+ first += (b - a), last -= (d - c);
+ }
+ *pa = first, *pb = last;
+}
+
+static
+void
+tr_copy(int *ISA, const int *SA,
+ int *first, int *a, int *b, int *last,
+ int depth) {
+ /* sort suffixes of middle partition
+ by using sorted order of suffixes of left and right partition. */
+ int *c, *d, *e;
+ int s, v;
+
+ v = b - SA - 1;
+ for(c = first, d = a - 1; c <= d; ++c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *++d = s;
+ ISA[s] = d - SA;
+ }
+ }
+ for(c = last - 1, e = d + 1, d = b; e < d; --c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *--d = s;
+ ISA[s] = d - SA;
+ }
+ }
+}
+
+static
+void
+tr_partialcopy(int *ISA, const int *SA,
+ int *first, int *a, int *b, int *last,
+ int depth) {
+ int *c, *d, *e;
+ int s, v;
+ int rank, lastrank, newrank = -1;
+
+ v = b - SA - 1;
+ lastrank = -1;
+ for(c = first, d = a - 1; c <= d; ++c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *++d = s;
+ rank = ISA[s + depth];
+ if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
+ ISA[s] = newrank;
+ }
+ }
+
+ lastrank = -1;
+ for(e = d; first <= e; --e) {
+ rank = ISA[*e];
+ if(lastrank != rank) { lastrank = rank; newrank = e - SA; }
+ if(newrank != rank) { ISA[*e] = newrank; }
+ }
+
+ lastrank = -1;
+ for(c = last - 1, e = d + 1, d = b; e < d; --c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *--d = s;
+ rank = ISA[s + depth];
+ if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
+ ISA[s] = newrank;
+ }
+ }
+}
+
+static
+void
+tr_introsort(int *ISA, const int *ISAd,
+ int *SA, int *first, int *last,
+ trbudget_t *budget) {
+#define STACK_SIZE TR_STACKSIZE
+ struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE];
+ int *a, *b, *c;
+ int t;
+ int v, x = 0;
+ int incr = ISAd - ISA;
+ int limit, next;
+ int ssize, trlink = -1;
+
+ for(ssize = 0, limit = tr_ilg(last - first);;) {
+
+ if(limit < 0) {
+ if(limit == -1) {
+ /* tandem repeat partition */
+ tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1);
+
+ /* update ranks */
+ if(a < last) {
+ for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
+ }
+ if(b < last) {
+ for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; }
+ }
+
+ /* push */
+ if(1 < (b - a)) {
+ STACK_PUSH5(NULL, a, b, 0, 0);
+ STACK_PUSH5(ISAd - incr, first, last, -2, trlink);
+ trlink = ssize - 2;
+ }
+ if((a - first) <= (last - b)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink);
+ last = a, limit = tr_ilg(a - first);
+ } else if(1 < (last - b)) {
+ first = b, limit = tr_ilg(last - b);
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ } else {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink);
+ first = b, limit = tr_ilg(last - b);
+ } else if(1 < (a - first)) {
+ last = a, limit = tr_ilg(a - first);
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ } else if(limit == -2) {
+ /* tandem repeat copy */
+ a = stack[--ssize].b, b = stack[ssize].c;
+ if(stack[ssize].d == 0) {
+ tr_copy(ISA, SA, first, a, b, last, ISAd - ISA);
+ } else {
+ if(0 <= trlink) { stack[trlink].d = -1; }
+ tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA);
+ }
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ } else {
+ /* sorted partition */
+ if(0 <= *first) {
+ a = first;
+ do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a));
+ first = a;
+ }
+ if(first < last) {
+ a = first; do { *a = ~*a; } while(*++a < 0);
+ next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1;
+ if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } }
+
+ /* push */
+ if(trbudget_check(budget, a - first)) {
+ if((a - first) <= (last - a)) {
+ STACK_PUSH5(ISAd, a, last, -3, trlink);
+ ISAd += incr, last = a, limit = next;
+ } else {
+ if(1 < (last - a)) {
+ STACK_PUSH5(ISAd + incr, first, a, next, trlink);
+ first = a, limit = -3;
+ } else {
+ ISAd += incr, last = a, limit = next;
+ }
+ }
+ } else {
+ if(0 <= trlink) { stack[trlink].d = -1; }
+ if(1 < (last - a)) {
+ first = a, limit = -3;
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ continue;
+ }
+
+ if((last - first) <= TR_INSERTIONSORT_THRESHOLD) {
+ tr_insertionsort(ISAd, first, last);
+ limit = -3;
+ continue;
+ }
+
+ if(limit-- == 0) {
+ tr_heapsort(ISAd, first, last - first);
+ for(a = last - 1; first < a; a = b) {
+ for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; }
+ }
+ limit = -3;
+ continue;
+ }
+
+ /* choose pivot */
+ a = tr_pivot(ISAd, first, last);
+ SWAP(*first, *a);
+ v = ISAd[*first];
+
+ /* partition */
+ tr_partition(ISAd, first, first + 1, last, &a, &b, v);
+ if((last - first) != (b - a)) {
+ next = (ISA[*a] != v) ? tr_ilg(b - a) : -1;
+
+ /* update ranks */
+ for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
+ if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } }
+
+ /* push */
+ if((1 < (b - a)) && (trbudget_check(budget, b - a))) {
+ if((a - first) <= (last - b)) {
+ if((last - b) <= (b - a)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ last = a;
+ } else if(1 < (last - b)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ first = b;
+ } else {
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else if((a - first) <= (b - a)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ last = a;
+ } else {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else {
+ if((a - first) <= (b - a)) {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ first = b;
+ } else if(1 < (a - first)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ last = a;
+ } else {
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else if((last - b) <= (b - a)) {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ first = b;
+ } else {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ }
+ } else {
+ if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; }
+ if((a - first) <= (last - b)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ last = a;
+ } else if(1 < (last - b)) {
+ first = b;
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ } else {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ first = b;
+ } else if(1 < (a - first)) {
+ last = a;
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ }
+ } else {
+ if(trbudget_check(budget, last - first)) {
+ limit = tr_ilg(last - first), ISAd += incr;
+ } else {
+ if(0 <= trlink) { stack[trlink].d = -1; }
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ }
+#undef STACK_SIZE
+}
+
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Tandem repeat sort */
+static
+void
+trsort(int *ISA, int *SA, int n, int depth) {
+ int *ISAd;
+ int *first, *last;
+ trbudget_t budget;
+ int t, skip, unsorted;
+
+ trbudget_init(&budget, tr_ilg(n) * 2 / 3, n);
+/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */
+ for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) {
+ first = SA;
+ skip = 0;
+ unsorted = 0;
+ do {
+ if((t = *first) < 0) { first -= t; skip += t; }
+ else {
+ if(skip != 0) { *(first + skip) = skip; skip = 0; }
+ last = SA + ISA[t] + 1;
+ if(1 < (last - first)) {
+ budget.count = 0;
+ tr_introsort(ISA, ISAd, SA, first, last, &budget);
+ if(budget.count != 0) { unsorted += budget.count; }
+ else { skip = first - last; }
+ } else if((last - first) == 1) {
+ skip = -1;
+ }
+ first = last;
+ }
+ } while(first < (SA + n));
+ if(skip != 0) { *(first + skip) = skip; }
+ if(unsorted == 0) { break; }
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Sorts suffixes of type B*. */
+static
+int
+sort_typeBstar(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int openMP) {
+ int *PAb, *ISAb, *buf;
+#ifdef LIBBSC_OPENMP
+ int *curbuf;
+ int l;
+#endif
+ int i, j, k, t, m, bufsize;
+ int c0, c1;
+#ifdef LIBBSC_OPENMP
+ int d0, d1;
+#endif
+ (void)openMP;
+
+ /* Initialize bucket arrays. */
+ for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; }
+ for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; }
+
+ /* Count the number of occurrences of the first one or two characters of each
+ type A, B and B* suffix. Moreover, store the beginning position of all
+ type B* suffixes into the array SA. */
+ for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) {
+ /* type A suffix. */
+ do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1));
+ if(0 <= i) {
+ /* type B* suffix. */
+ ++BUCKET_BSTAR(c0, c1);
+ SA[--m] = i;
+ /* type B suffix. */
+ for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) {
+ ++BUCKET_B(c0, c1);
+ }
+ }
+ }
+ m = n - m;
+/*
+note:
+ A type B* suffix is lexicographically smaller than a type B suffix that
+ begins with the same first two characters.
+*/
+
+ /* Calculate the index of start/end point of each bucket. */
+ for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) {
+ t = i + BUCKET_A(c0);
+ BUCKET_A(c0) = i + j; /* start point */
+ i = t + BUCKET_B(c0, c0);
+ for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) {
+ j += BUCKET_BSTAR(c0, c1);
+ BUCKET_BSTAR(c0, c1) = j; /* end point */
+ i += BUCKET_B(c0, c1);
+ }
+ }
+
+ if(0 < m) {
+ /* Sort the type B* suffixes by their first two characters. */
+ PAb = SA + n - m; ISAb = SA + m;
+ for(i = m - 2; 0 <= i; --i) {
+ t = PAb[i], c0 = T[t], c1 = T[t + 1];
+ SA[--BUCKET_BSTAR(c0, c1)] = i;
+ }
+ t = PAb[m - 1], c0 = T[t], c1 = T[t + 1];
+ SA[--BUCKET_BSTAR(c0, c1)] = m - 1;
+
+ /* Sort the type B* substrings using sssort. */
+#ifdef LIBBSC_OPENMP
+ if (openMP)
+ {
+ buf = SA + m;
+ c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m;
+#pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1)
+ {
+ bufsize = (n - (2 * m)) / omp_get_num_threads();
+ curbuf = buf + omp_get_thread_num() * bufsize;
+ k = 0;
+ for(;;) {
+ #pragma omp critical(sssort_lock)
+ {
+ if(0 < (l = j)) {
+ d0 = c0, d1 = c1;
+ do {
+ k = BUCKET_BSTAR(d0, d1);
+ if(--d1 <= d0) {
+ d1 = ALPHABET_SIZE - 1;
+ if(--d0 < 0) { break; }
+ }
+ } while(((l - k) <= 1) && (0 < (l = k)));
+ c0 = d0, c1 = d1, j = k;
+ }
+ }
+ if(l == 0) { break; }
+ sssort(T, PAb, SA + k, SA + l,
+ curbuf, bufsize, 2, n, *(SA + k) == (m - 1));
+ }
+ }
+ }
+ else
+ {
+ buf = SA + m, bufsize = n - (2 * m);
+ for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
+ for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
+ i = BUCKET_BSTAR(c0, c1);
+ if(1 < (j - i)) {
+ sssort(T, PAb, SA + i, SA + j,
+ buf, bufsize, 2, n, *(SA + i) == (m - 1));
+ }
+ }
+ }
+ }
+#else
+ buf = SA + m, bufsize = n - (2 * m);
+ for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
+ for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
+ i = BUCKET_BSTAR(c0, c1);
+ if(1 < (j - i)) {
+ sssort(T, PAb, SA + i, SA + j,
+ buf, bufsize, 2, n, *(SA + i) == (m - 1));
+ }
+ }
+ }
+#endif
+
+ /* Compute ranks of type B* substrings. */
+ for(i = m - 1; 0 <= i; --i) {
+ if(0 <= SA[i]) {
+ j = i;
+ do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i]));
+ SA[i + 1] = i - j;
+ if(i <= 0) { break; }
+ }
+ j = i;
+ do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0);
+ ISAb[SA[i]] = j;
+ }
+
+ /* Construct the inverse suffix array of type B* suffixes using trsort. */
+ trsort(ISAb, SA, m, 1);
+
+ /* Set the sorted order of tyoe B* suffixes. */
+ for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) {
+ for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { }
+ if(0 <= i) {
+ t = i;
+ for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { }
+ SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t;
+ }
+ }
+
+ /* Calculate the index of start/end point of each bucket. */
+ BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */
+ for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) {
+ i = BUCKET_A(c0 + 1) - 1;
+ for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) {
+ t = i - BUCKET_B(c0, c1);
+ BUCKET_B(c0, c1) = i; /* end point */
+
+ /* Move all type B* suffixes to the correct position. */
+ for(i = t, j = BUCKET_BSTAR(c0, c1);
+ j <= k;
+ --i, --k) { SA[i] = SA[k]; }
+ }
+ BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */
+ BUCKET_B(c0, c0) = i; /* end point */
+ }
+ }
+
+ return m;
+}
+
+/* Constructs the suffix array by using the sorted order of type B* suffixes. */
+static
+void
+construct_SA(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int m) {
+ int *i, *j, *k;
+ int s;
+ int c0, c1, c2;
+
+ if(0 < m) {
+ /* Construct the sorted order of type B suffixes by using
+ the sorted order of type B* suffixes. */
+ for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
+ /* Scan the suffix array from right to left. */
+ for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
+ j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
+ i <= j;
+ --j) {
+ if(0 < (s = *j)) {
+ assert(T[s] == c1);
+ assert(((s + 1) < n) && (T[s] <= T[s + 1]));
+ assert(T[s - 1] <= T[s]);
+ *j = ~s;
+ c0 = T[--s];
+ if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
+ if(c0 != c2) {
+ if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
+ k = SA + BUCKET_B(c2 = c0, c1);
+ }
+ assert(k < j);
+ *k-- = s;
+ } else {
+ assert(((s == 0) && (T[s] == c1)) || (s < 0));
+ *j = ~s;
+ }
+ }
+ }
+ }
+
+ /* Construct the suffix array by using
+ the sorted order of type B suffixes. */
+ k = SA + BUCKET_A(c2 = T[n - 1]);
+ *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1);
+ /* Scan the suffix array from left to right. */
+ for(i = SA, j = SA + n; i < j; ++i) {
+ if(0 < (s = *i)) {
+ assert(T[s - 1] >= T[s]);
+ c0 = T[--s];
+ if((s == 0) || (T[s - 1] < c0)) { s = ~s; }
+ if(c0 != c2) {
+ BUCKET_A(c2) = k - SA;
+ k = SA + BUCKET_A(c2 = c0);
+ }
+ assert(i < k);
+ *k++ = s;
+ } else {
+ assert(s < 0);
+ *i = ~s;
+ }
+ }
+}
+
+/* Constructs the burrows-wheeler transformed string directly
+ by using the sorted order of type B* suffixes. */
+static
+int
+construct_BWT(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int m) {
+ int *i, *j, *k, *orig;
+ int s;
+ int c0, c1, c2;
+
+ if(0 < m) {
+ /* Construct the sorted order of type B suffixes by using
+ the sorted order of type B* suffixes. */
+ for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
+ /* Scan the suffix array from right to left. */
+ for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
+ j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
+ i <= j;
+ --j) {
+ if(0 < (s = *j)) {
+ assert(T[s] == c1);
+ assert(((s + 1) < n) && (T[s] <= T[s + 1]));
+ assert(T[s - 1] <= T[s]);
+ c0 = T[--s];
+ *j = ~((int)c0);
+ if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
+ if(c0 != c2) {
+ if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
+ k = SA + BUCKET_B(c2 = c0, c1);
+ }
+ assert(k < j);
+ *k-- = s;
+ } else if(s != 0) {
+ *j = ~s;
+#ifndef NDEBUG
+ } else {
+ assert(T[s] == c1);
+#endif
+ }
+ }
+ }
+ }
+
+ /* Construct the BWTed string by using
+ the sorted order of type B suffixes. */
+ k = SA + BUCKET_A(c2 = T[n - 1]);
+ *k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1);
+ /* Scan the suffix array from left to right. */
+ for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
+ if(0 < (s = *i)) {
+ assert(T[s - 1] >= T[s]);
+ c0 = T[--s];
+ *i = c0;
+ if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); }
+ if(c0 != c2) {
+ BUCKET_A(c2) = k - SA;
+ k = SA + BUCKET_A(c2 = c0);
+ }
+ assert(i < k);
+ *k++ = s;
+ } else if(s != 0) {
+ *i = ~s;
+ } else {
+ orig = i;
+ }
+ }
+
+ return orig - SA;
+}
+
+/* Constructs the burrows-wheeler transformed string directly
+ by using the sorted order of type B* suffixes. */
+static
+int
+construct_BWT_indexes(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int m,
+ unsigned char * num_indexes, int * indexes) {
+ int *i, *j, *k, *orig;
+ int s;
+ int c0, c1, c2;
+
+ int mod = n / 8;
+ {
+ mod |= mod >> 1; mod |= mod >> 2;
+ mod |= mod >> 4; mod |= mod >> 8;
+ mod |= mod >> 16; mod >>= 1;
+
+ *num_indexes = (unsigned char)((n - 1) / (mod + 1));
+ }
+
+ if(0 < m) {
+ /* Construct the sorted order of type B suffixes by using
+ the sorted order of type B* suffixes. */
+ for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
+ /* Scan the suffix array from right to left. */
+ for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
+ j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
+ i <= j;
+ --j) {
+ if(0 < (s = *j)) {
+ assert(T[s] == c1);
+ assert(((s + 1) < n) && (T[s] <= T[s + 1]));
+ assert(T[s - 1] <= T[s]);
+
+ if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA;
+
+ c0 = T[--s];
+ *j = ~((int)c0);
+ if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
+ if(c0 != c2) {
+ if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
+ k = SA + BUCKET_B(c2 = c0, c1);
+ }
+ assert(k < j);
+ *k-- = s;
+ } else if(s != 0) {
+ *j = ~s;
+#ifndef NDEBUG
+ } else {
+ assert(T[s] == c1);
+#endif
+ }
+ }
+ }
+ }
+
+ /* Construct the BWTed string by using
+ the sorted order of type B suffixes. */
+ k = SA + BUCKET_A(c2 = T[n - 1]);
+ if (T[n - 2] < c2) {
+ if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA;
+ *k++ = ~((int)T[n - 2]);
+ }
+ else {
+ *k++ = n - 1;
+ }
+
+ /* Scan the suffix array from left to right. */
+ for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
+ if(0 < (s = *i)) {
+ assert(T[s - 1] >= T[s]);
+
+ if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA;
+
+ c0 = T[--s];
+ *i = c0;
+ if(c0 != c2) {
+ BUCKET_A(c2) = k - SA;
+ k = SA + BUCKET_A(c2 = c0);
+ }
+ assert(i < k);
+ if((0 < s) && (T[s - 1] < c0)) {
+ if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA;
+ *k++ = ~((int)T[s - 1]);
+ } else
+ *k++ = s;
+ } else if(s != 0) {
+ *i = ~s;
+ } else {
+ orig = i;
+ }
+ }
+
+ return orig - SA;
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/*- Function -*/
+
+int
+divsufsort(const unsigned char *T, int *SA, int n, int openMP) {
+ int *bucket_A, *bucket_B;
+ int m;
+ int err = 0;
+
+ /* Check arguments. */
+ if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; }
+ else if(n == 0) { return 0; }
+ else if(n == 1) { SA[0] = 0; return 0; }
+ else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; }
+
+ bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
+ bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
+
+ /* Suffixsort. */
+ if((bucket_A != NULL) && (bucket_B != NULL)) {
+ m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP);
+ construct_SA(T, SA, bucket_A, bucket_B, n, m);
+ } else {
+ err = -2;
+ }
+
+ free(bucket_B);
+ free(bucket_A);
+
+ return err;
+}
+
+int
+divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) {
+ int *B;
+ int *bucket_A, *bucket_B;
+ int m, pidx, i;
+
+ /* Check arguments. */
+ if((T == NULL) || (U == NULL) || (n < 0)) { return -1; }
+ else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; }
+
+ if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); }
+ bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
+ bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
+
+ /* Burrows-Wheeler Transform. */
+ if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) {
+ m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP);
+
+ if (num_indexes == NULL || indexes == NULL) {
+ pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m);
+ } else {
+ pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes);
+ }
+
+ /* Copy to output string. */
+ U[0] = T[n - 1];
+ for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; }
+ for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; }
+ pidx += 1;
+ } else {
+ pidx = -2;
+ }
+
+ free(bucket_B);
+ free(bucket_A);
+ if(A == NULL) { free(B); }
+
+ return pidx;
+}
diff --git a/contrib/zstd/divsufsort.h b/contrib/zstd/divsufsort.h
new file mode 100644
index 0000000..5440994
--- /dev/null
+++ b/contrib/zstd/divsufsort.h
@@ -0,0 +1,67 @@
+/*
+ * divsufsort.h for libdivsufsort-lite
+ * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _DIVSUFSORT_H
+#define _DIVSUFSORT_H 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+
+/*- Prototypes -*/
+
+/**
+ * Constructs the suffix array of a given string.
+ * @param T [0..n-1] The input string.
+ * @param SA [0..n-1] The output array of suffixes.
+ * @param n The length of the given string.
+ * @param openMP enables OpenMP optimization.
+ * @return 0 if no error occurred, -1 or -2 otherwise.
+ */
+int
+divsufsort(const unsigned char *T, int *SA, int n, int openMP);
+
+/**
+ * Constructs the burrows-wheeler transformed string of a given string.
+ * @param T [0..n-1] The input string.
+ * @param U [0..n-1] The output string. (can be T)
+ * @param A [0..n-1] The temporary array. (can be NULL)
+ * @param n The length of the given string.
+ * @param num_indexes The length of secondary indexes array. (can be NULL)
+ * @param indexes The secondary indexes array. (can be NULL)
+ * @param openMP enables OpenMP optimization.
+ * @return The primary index if no error occurred, -1 or -2 otherwise.
+ */
+int
+divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP);
+
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif /* __cplusplus */
+
+#endif /* _DIVSUFSORT_H */
diff --git a/contrib/zstd/entropy_common.c b/contrib/zstd/entropy_common.c
new file mode 100644
index 0000000..e2173af
--- /dev/null
+++ b/contrib/zstd/entropy_common.c
@@ -0,0 +1,340 @@
+/* ******************************************************************
+ * Common functions of New Generation Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* *************************************
+* Dependencies
+***************************************/
+#include "mem.h"
+#include "error_private.h" /* ERR_*, ERROR */
+#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
+#include "fse.h"
+#include "huf.h"
+#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
+
+
+/*=== Version ===*/
+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
+
+
+/*=== Error Management ===*/
+unsigned FSE_isError(size_t code) { return ERR_isError(code); }
+const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+unsigned HUF_isError(size_t code) { return ERR_isError(code); }
+const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+
+/*-**************************************************************
+* FSE NCount encoding-decoding
+****************************************************************/
+FORCE_INLINE_TEMPLATE
+size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ const BYTE* const istart = (const BYTE*) headerBuffer;
+ const BYTE* const iend = istart + hbSize;
+ const BYTE* ip = istart;
+ int nbBits;
+ int remaining;
+ int threshold;
+ U32 bitStream;
+ int bitCount;
+ unsigned charnum = 0;
+ unsigned const maxSV1 = *maxSVPtr + 1;
+ int previous0 = 0;
+
+ if (hbSize < 8) {
+ /* This function only works when hbSize >= 8 */
+ char buffer[8] = {0};
+ ZSTD_memcpy(buffer, headerBuffer, hbSize);
+ { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
+ buffer, sizeof(buffer));
+ if (FSE_isError(countSize)) return countSize;
+ if (countSize > hbSize) return ERROR(corruption_detected);
+ return countSize;
+ } }
+ assert(hbSize >= 8);
+
+ /* init */
+ ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
+ bitStream = MEM_readLE32(ip);
+ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
+ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
+ bitStream >>= 4;
+ bitCount = 4;
+ *tableLogPtr = nbBits;
+ remaining = (1<<nbBits)+1;
+ threshold = 1<<nbBits;
+ nbBits++;
+
+ for (;;) {
+ if (previous0) {
+ /* Count the number of repeats. Each time the
+ * 2-bit repeat code is 0b11 there is another
+ * repeat.
+ * Avoid UB by setting the high bit to 1.
+ */
+ int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
+ while (repeats >= 12) {
+ charnum += 3 * 12;
+ if (LIKELY(ip <= iend-7)) {
+ ip += 3;
+ } else {
+ bitCount -= (int)(8 * (iend - 7 - ip));
+ bitCount &= 31;
+ ip = iend - 4;
+ }
+ bitStream = MEM_readLE32(ip) >> bitCount;
+ repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
+ }
+ charnum += 3 * repeats;
+ bitStream >>= 2 * repeats;
+ bitCount += 2 * repeats;
+
+ /* Add the final repeat which isn't 0b11. */
+ assert((bitStream & 3) < 3);
+ charnum += bitStream & 3;
+ bitCount += 2;
+
+ /* This is an error, but break and return an error
+ * at the end, because returning out of a loop makes
+ * it harder for the compiler to optimize.
+ */
+ if (charnum >= maxSV1) break;
+
+ /* We don't need to set the normalized count to 0
+ * because we already memset the whole buffer to 0.
+ */
+
+ if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
+ assert((bitCount >> 3) <= 3); /* For first condition to work */
+ ip += bitCount>>3;
+ bitCount &= 7;
+ } else {
+ bitCount -= (int)(8 * (iend - 4 - ip));
+ bitCount &= 31;
+ ip = iend - 4;
+ }
+ bitStream = MEM_readLE32(ip) >> bitCount;
+ }
+ {
+ int const max = (2*threshold-1) - remaining;
+ int count;
+
+ if ((bitStream & (threshold-1)) < (U32)max) {
+ count = bitStream & (threshold-1);
+ bitCount += nbBits-1;
+ } else {
+ count = bitStream & (2*threshold-1);
+ if (count >= threshold) count -= max;
+ bitCount += nbBits;
+ }
+
+ count--; /* extra accuracy */
+ /* When it matters (small blocks), this is a
+ * predictable branch, because we don't use -1.
+ */
+ if (count >= 0) {
+ remaining -= count;
+ } else {
+ assert(count == -1);
+ remaining += count;
+ }
+ normalizedCounter[charnum++] = (short)count;
+ previous0 = !count;
+
+ assert(threshold > 1);
+ if (remaining < threshold) {
+ /* This branch can be folded into the
+ * threshold update condition because we
+ * know that threshold > 1.
+ */
+ if (remaining <= 1) break;
+ nbBits = ZSTD_highbit32(remaining) + 1;
+ threshold = 1 << (nbBits - 1);
+ }
+ if (charnum >= maxSV1) break;
+
+ if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
+ ip += bitCount>>3;
+ bitCount &= 7;
+ } else {
+ bitCount -= (int)(8 * (iend - 4 - ip));
+ bitCount &= 31;
+ ip = iend - 4;
+ }
+ bitStream = MEM_readLE32(ip) >> bitCount;
+ } }
+ if (remaining != 1) return ERROR(corruption_detected);
+ /* Only possible when there are too many zeros. */
+ if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall);
+ if (bitCount > 32) return ERROR(corruption_detected);
+ *maxSVPtr = charnum-1;
+
+ ip += (bitCount+7)>>3;
+ return ip-istart;
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t FSE_readNCount_body_default(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+#endif
+
+size_t FSE_readNCount_bmi2(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+ }
+#endif
+ (void)bmi2;
+ return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+
+size_t FSE_readNCount(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0);
+}
+
+
+/*! HUF_readStats() :
+ Read compact Huffman tree, saved by HUF_writeCTable().
+ `huffWeight` is destination buffer.
+ `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
+ @return : size read from `src` , or an error Code .
+ Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
+*/
+size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize)
+{
+ U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+ return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize,
+ int bmi2)
+{
+ U32 weightTotal;
+ const BYTE* ip = (const BYTE*) src;
+ size_t iSize;
+ size_t oSize;
+
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
+ /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
+
+ if (iSize >= 128) { /* special header */
+ oSize = iSize - 127;
+ iSize = ((oSize+1)/2);
+ if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
+ if (oSize >= hwSize) return ERROR(corruption_detected);
+ ip += 1;
+ { U32 n;
+ for (n=0; n<oSize; n+=2) {
+ huffWeight[n] = ip[n/2] >> 4;
+ huffWeight[n+1] = ip[n/2] & 15;
+ } } }
+ else { /* header compressed with FSE (normal case) */
+ if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
+ /* max (hwSize-1) values decoded, as last one is implied */
+ oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2);
+ if (FSE_isError(oSize)) return oSize;
+ }
+
+ /* collect weight stats */
+ ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
+ weightTotal = 0;
+ { U32 n; for (n=0; n<oSize; n++) {
+ if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
+ rankStats[huffWeight[n]]++;
+ weightTotal += (1 << huffWeight[n]) >> 1;
+ } }
+ if (weightTotal == 0) return ERROR(corruption_detected);
+
+ /* get last non-null symbol weight (implied, total must be 2^n) */
+ { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1;
+ if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
+ *tableLogPtr = tableLog;
+ /* determine last weight */
+ { U32 const total = 1 << tableLog;
+ U32 const rest = total - weightTotal;
+ U32 const verif = 1 << ZSTD_highbit32(rest);
+ U32 const lastWeight = ZSTD_highbit32(rest) + 1;
+ if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
+ huffWeight[oSize] = (BYTE)lastWeight;
+ rankStats[lastWeight]++;
+ } }
+
+ /* check tree construction validity */
+ if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
+
+ /* results */
+ *nbSymbolsPtr = (U32)(oSize+1);
+ return iSize+1;
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize)
+{
+ return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
+}
+
+#if DYNAMIC_BMI2
+static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize)
+{
+ return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
+}
+#endif
+
+size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize,
+ int flags)
+{
+#if DYNAMIC_BMI2
+ if (flags & HUF_flags_bmi2) {
+ return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
+ }
+#endif
+ (void)flags;
+ return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
+}
diff --git a/contrib/zstd/error_private.c b/contrib/zstd/error_private.c
new file mode 100644
index 0000000..075fc5e
--- /dev/null
+++ b/contrib/zstd/error_private.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* The purpose of this file is to have a single list of error strings embedded in binary */
+
+#include "error_private.h"
+
+const char* ERR_getErrorString(ERR_enum code)
+{
+#ifdef ZSTD_STRIP_ERROR_STRINGS
+ (void)code;
+ return "Error strings stripped";
+#else
+ static const char* const notErrorCode = "Unspecified error code";
+ switch( code )
+ {
+ case PREFIX(no_error): return "No error detected";
+ case PREFIX(GENERIC): return "Error (generic)";
+ case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+ case PREFIX(version_unsupported): return "Version not supported";
+ case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
+ case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
+ case PREFIX(corruption_detected): return "Data corruption detected";
+ case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+ case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
+ case PREFIX(parameter_unsupported): return "Unsupported parameter";
+ case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
+ case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
+ case PREFIX(init_missing): return "Context should be init first";
+ case PREFIX(memory_allocation): return "Allocation error : not enough memory";
+ case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
+ case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
+ case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
+ case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
+ case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
+ case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
+ case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
+ case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+ case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
+ case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+ case PREFIX(srcSize_wrong): return "Src size is incorrect";
+ case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
+ case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
+ case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
+ /* following error codes are not stable and may be removed or changed in a future version */
+ case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
+ case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
+ case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
+ case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
+ case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
+ case PREFIX(externalSequences_invalid): return "External sequences are not valid";
+ case PREFIX(maxCode):
+ default: return notErrorCode;
+ }
+#endif
+}
diff --git a/contrib/zstd/error_private.h b/contrib/zstd/error_private.h
new file mode 100644
index 0000000..d99c961
--- /dev/null
+++ b/contrib/zstd/error_private.h
@@ -0,0 +1,159 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* Note : this module is expected to remain private, do not expose it */
+
+#ifndef ERROR_H_MODULE
+#define ERROR_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************************
+* Dependencies
+******************************************/
+#include "zstd_errors.h" /* enum list */
+#include "compiler.h"
+#include "debug.h"
+#include "zstd_deps.h" /* size_t */
+
+
+/* ****************************************
+* Compiler-specific
+******************************************/
+#if defined(__GNUC__)
+# define ERR_STATIC static __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define ERR_STATIC static inline
+#elif defined(_MSC_VER)
+# define ERR_STATIC static __inline
+#else
+# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+
+/*-****************************************
+* Customization (error_public.h)
+******************************************/
+typedef ZSTD_ErrorCode ERR_enum;
+#define PREFIX(name) ZSTD_error_##name
+
+
+/*-****************************************
+* Error codes handling
+******************************************/
+#undef ERROR /* already defined on Visual Studio */
+#define ERROR(name) ZSTD_ERROR(name)
+#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
+
+ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
+
+ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
+
+/* check and forward error code */
+#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
+#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
+
+
+/*-****************************************
+* Error Strings
+******************************************/
+
+const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
+
+ERR_STATIC const char* ERR_getErrorName(size_t code)
+{
+ return ERR_getErrorString(ERR_getErrorCode(code));
+}
+
+/**
+ * Ignore: this is an internal helper.
+ *
+ * This is a helper function to help force C99-correctness during compilation.
+ * Under strict compilation modes, variadic macro arguments can't be empty.
+ * However, variadic function arguments can be. Using a function therefore lets
+ * us statically check that at least one (string) argument was passed,
+ * independent of the compilation flags.
+ */
+static INLINE_KEYWORD UNUSED_ATTR
+void _force_has_format_string(const char *format, ...) {
+ (void)format;
+}
+
+/**
+ * Ignore: this is an internal helper.
+ *
+ * We want to force this function invocation to be syntactically correct, but
+ * we don't want to force runtime evaluation of its arguments.
+ */
+#define _FORCE_HAS_FORMAT_STRING(...) \
+ if (0) { \
+ _force_has_format_string(__VA_ARGS__); \
+ }
+
+#define ERR_QUOTE(str) #str
+
+/**
+ * Return the specified error if the condition evaluates to true.
+ *
+ * In debug modes, prints additional information.
+ * In order to do that (particularly, printing the conditional that failed),
+ * this can't just wrap RETURN_ERROR().
+ */
+#define RETURN_ERROR_IF(cond, err, ...) \
+ if (cond) { \
+ RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
+ __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
+ _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
+ RAWLOG(3, ": " __VA_ARGS__); \
+ RAWLOG(3, "\n"); \
+ return ERROR(err); \
+ }
+
+/**
+ * Unconditionally return the specified error.
+ *
+ * In debug modes, prints additional information.
+ */
+#define RETURN_ERROR(err, ...) \
+ do { \
+ RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
+ __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
+ _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
+ RAWLOG(3, ": " __VA_ARGS__); \
+ RAWLOG(3, "\n"); \
+ return ERROR(err); \
+ } while(0);
+
+/**
+ * If the provided expression evaluates to an error code, returns that error code.
+ *
+ * In debug modes, prints additional information.
+ */
+#define FORWARD_IF_ERROR(err, ...) \
+ do { \
+ size_t const err_code = (err); \
+ if (ERR_isError(err_code)) { \
+ RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
+ __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
+ _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
+ RAWLOG(3, ": " __VA_ARGS__); \
+ RAWLOG(3, "\n"); \
+ return err_code; \
+ } \
+ } while(0);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ERROR_H_MODULE */
diff --git a/contrib/zstd/error_public.h b/contrib/zstd/error_public.h
new file mode 100644
index 0000000..d46abd2
--- /dev/null
+++ b/contrib/zstd/error_public.h
@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef ERROR_PUBLIC_H_MODULE
+#define ERROR_PUBLIC_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*===== dependency =====*/
+#include <stddef.h> /* size_t */
+
+
+/*-****************************************
+* error codes list
+******************************************/
+typedef enum {
+ ZSTD_error_no_error,
+ ZSTD_error_GENERIC,
+ ZSTD_error_prefix_unknown,
+ ZSTD_error_version_unsupported,
+ ZSTD_error_parameter_unknown,
+ ZSTD_error_frameParameter_unsupported,
+ ZSTD_error_frameParameter_unsupportedBy32bits,
+ ZSTD_error_compressionParameter_unsupported,
+ ZSTD_error_init_missing,
+ ZSTD_error_memory_allocation,
+ ZSTD_error_stage_wrong,
+ ZSTD_error_dstSize_tooSmall,
+ ZSTD_error_srcSize_wrong,
+ ZSTD_error_corruption_detected,
+ ZSTD_error_checksum_wrong,
+ ZSTD_error_tableLog_tooLarge,
+ ZSTD_error_maxSymbolValue_tooLarge,
+ ZSTD_error_maxSymbolValue_tooSmall,
+ ZSTD_error_dictionary_corrupted,
+ ZSTD_error_dictionary_wrong,
+ ZSTD_error_maxCode
+} ZSTD_ErrorCode;
+
+/*! ZSTD_getErrorCode() :
+ convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
+ which can be used to compare directly with enum list published into "error_public.h" */
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ERROR_PUBLIC_H_MODULE */
diff --git a/contrib/zstd/fse.h b/contrib/zstd/fse.h
new file mode 100644
index 0000000..02a1f0b
--- /dev/null
+++ b/contrib/zstd/fse.h
@@ -0,0 +1,639 @@
+/* ******************************************************************
+ * FSE : Finite State Entropy codec
+ * Public Prototypes declaration
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef FSE_H
+#define FSE_H
+
+
+/*-*****************************************
+* Dependencies
+******************************************/
+#include "zstd_deps.h" /* size_t, ptrdiff_t */
+
+
+/*-*****************************************
+* FSE_PUBLIC_API : control library symbols visibility
+******************************************/
+#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
+# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
+#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
+# define FSE_PUBLIC_API __declspec(dllexport)
+#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
+# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define FSE_PUBLIC_API
+#endif
+
+/*------ Version ------*/
+#define FSE_VERSION_MAJOR 0
+#define FSE_VERSION_MINOR 9
+#define FSE_VERSION_RELEASE 0
+
+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
+#define FSE_QUOTE(str) #str
+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
+
+#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
+FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
+
+
+/*-*****************************************
+* Tool functions
+******************************************/
+FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
+
+/* Error Management */
+FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
+FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
+
+
+/*-*****************************************
+* FSE detailed API
+******************************************/
+/*!
+FSE_compress() does the following:
+1. count symbol occurrence from source[] into table count[] (see hist.h)
+2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
+3. save normalized counters to memory buffer using writeNCount()
+4. build encoding table 'CTable' from normalized counters
+5. encode the data stream using encoding table 'CTable'
+
+FSE_decompress() does the following:
+1. read normalized counters with readNCount()
+2. build decoding table 'DTable' from normalized counters
+3. decode the data stream using decoding table 'DTable'
+
+The following API allows targeting specific sub-functions for advanced tasks.
+For example, it's possible to compress several blocks using the same 'CTable',
+or to save and provide normalized distribution using external method.
+*/
+
+/* *** COMPRESSION *** */
+
+/*! FSE_optimalTableLog():
+ dynamically downsize 'tableLog' when conditions are met.
+ It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
+ @return : recommended tableLog (necessarily <= 'maxTableLog') */
+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+
+/*! FSE_normalizeCount():
+ normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
+ 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
+ useLowProbCount is a boolean parameter which trades off compressed size for
+ faster header decoding. When it is set to 1, the compressed data will be slightly
+ smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
+ faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
+ is a good default, since header deserialization makes a big speed difference.
+ Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
+ @return : tableLog,
+ or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
+ const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount);
+
+/*! FSE_NCountWriteBound():
+ Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
+ Typically useful for allocation purpose. */
+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
+
+/*! FSE_writeNCount():
+ Compactly save 'normalizedCounter' into 'buffer'.
+ @return : size of the compressed table,
+ or an errorCode, which can be tested using FSE_isError(). */
+FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
+ const short* normalizedCounter,
+ unsigned maxSymbolValue, unsigned tableLog);
+
+/*! Constructor and Destructor of FSE_CTable.
+ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
+typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
+
+/*! FSE_buildCTable():
+ Builds `ct`, which must be already allocated, using FSE_createCTable().
+ @return : 0, or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+
+/*! FSE_compress_usingCTable():
+ Compress `src` using `ct` into `dst` which must be already allocated.
+ @return : size of compressed data (<= `dstCapacity`),
+ or 0 if compressed data could not fit into `dst`,
+ or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
+
+/*!
+Tutorial :
+----------
+The first step is to count all symbols. FSE_count() does this job very fast.
+Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
+'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
+maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
+FSE_count() will return the number of occurrence of the most frequent symbol.
+This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+The next step is to normalize the frequencies.
+FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
+It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
+You can use 'tableLog'==0 to mean "use default tableLog value".
+If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
+which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
+
+The result of FSE_normalizeCount() will be saved into a table,
+called 'normalizedCounter', which is a table of signed short.
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
+The return value is tableLog if everything proceeded as expected.
+It is 0 if there is a single symbol within distribution.
+If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
+'buffer' must be already allocated.
+For guaranteed success, buffer size must be at least FSE_headerBound().
+The result of the function is the number of bytes written into 'buffer'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
+
+'normalizedCounter' can then be used to create the compression table 'CTable'.
+The space required by 'CTable' must be already allocated, using FSE_createCTable().
+You can then use FSE_buildCTable() to fill 'CTable'.
+If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
+
+'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
+Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
+The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
+If it returns '0', compressed data could not fit into 'dst'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+*/
+
+
+/* *** DECOMPRESSION *** */
+
+/*! FSE_readNCount():
+ Read compactly saved 'normalizedCounter' from 'rBuffer'.
+ @return : size read from 'rBuffer',
+ or an errorCode, which can be tested using FSE_isError().
+ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
+FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
+ const void* rBuffer, size_t rBuffSize);
+
+/*! FSE_readNCount_bmi2():
+ * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
+ */
+FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
+ const void* rBuffer, size_t rBuffSize, int bmi2);
+
+typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
+
+/*!
+Tutorial :
+----------
+(Note : these functions only decompress FSE-compressed blocks.
+ If block is uncompressed, use memcpy() instead
+ If block is a single repeated byte, use memset() instead )
+
+The first step is to obtain the normalized frequencies of symbols.
+This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
+In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
+or size the table to handle worst case situations (typically 256).
+FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
+The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
+Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
+This is performed by the function FSE_buildDTable().
+The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
+`cSrcSize` must be strictly correct, otherwise decompression will fail.
+FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
+If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
+*/
+
+#endif /* FSE_H */
+
+#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
+#define FSE_H_FSE_STATIC_LINKING_ONLY
+
+/* *** Dependency *** */
+#include "bitstream.h"
+
+
+/* *****************************************
+* Static allocation
+*******************************************/
+/* FSE buffer bounds */
+#define FSE_NCOUNTBOUND 512
+#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)
+#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
+
+/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
+#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2))
+#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog)))
+
+/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
+#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
+#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
+
+
+/* *****************************************
+ * FSE advanced API
+ ***************************************** */
+
+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
+/**< same as FSE_optimalTableLog(), which used `minus==2` */
+
+size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
+/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
+ * See FSE_buildCTable_wksp() for breakdown of workspace usage.
+ */
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */)
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
+size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+
+#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
+#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned))
+FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
+
+#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
+#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
+size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
+ * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
+
+typedef enum {
+ FSE_repeat_none, /**< Cannot use the previous table */
+ FSE_repeat_check, /**< Can use the previous table but it must be checked */
+ FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
+ } FSE_repeat;
+
+/* *****************************************
+* FSE symbol compression API
+*******************************************/
+/*!
+ This API consists of small unitary functions, which highly benefit from being inlined.
+ Hence their body are included in next section.
+*/
+typedef struct {
+ ptrdiff_t value;
+ const void* stateTable;
+ const void* symbolTT;
+ unsigned stateLog;
+} FSE_CState_t;
+
+static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
+
+static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
+
+static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
+
+/**<
+These functions are inner components of FSE_compress_usingCTable().
+They allow the creation of custom streams, mixing multiple tables and bit sources.
+
+A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
+So the first symbol you will encode is the last you will decode, like a LIFO stack.
+
+You will need a few variables to track your CStream. They are :
+
+FSE_CTable ct; // Provided by FSE_buildCTable()
+BIT_CStream_t bitStream; // bitStream tracking structure
+FSE_CState_t state; // State tracking structure (can have several)
+
+
+The first thing to do is to init bitStream and state.
+ size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
+ FSE_initCState(&state, ct);
+
+Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
+You can then encode your input data, byte after byte.
+FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
+Remember decoding will be done in reverse direction.
+ FSE_encodeByte(&bitStream, &state, symbol);
+
+At any time, you can also add any bit sequence.
+Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
+ BIT_addBits(&bitStream, bitField, nbBits);
+
+The above methods don't commit data to memory, they just store it into local register, for speed.
+Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+Writing data to memory is a manual operation, performed by the flushBits function.
+ BIT_flushBits(&bitStream);
+
+Your last FSE encoding operation shall be to flush your last state value(s).
+ FSE_flushState(&bitStream, &state);
+
+Finally, you must close the bitStream.
+The function returns the size of CStream in bytes.
+If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
+If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
+ size_t size = BIT_closeCStream(&bitStream);
+*/
+
+
+/* *****************************************
+* FSE symbol decompression API
+*******************************************/
+typedef struct {
+ size_t state;
+ const void* table; /* precise table may vary, depending on U16 */
+} FSE_DState_t;
+
+
+static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
+
+static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
+
+static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
+
+/**<
+Let's now decompose FSE_decompress_usingDTable() into its unitary components.
+You will decode FSE-encoded symbols from the bitStream,
+and also any other bitFields you put in, **in reverse order**.
+
+You will need a few variables to track your bitStream. They are :
+
+BIT_DStream_t DStream; // Stream context
+FSE_DState_t DState; // State context. Multiple ones are possible
+FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
+
+The first thing to do is to init the bitStream.
+ errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
+
+You should then retrieve your initial state(s)
+(in reverse flushing order if you have several ones) :
+ errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
+
+You can then decode your data, symbol after symbol.
+For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
+Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
+ unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
+
+You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
+Note : maximum allowed nbBits is 25, for 32-bits compatibility
+ size_t bitField = BIT_readBits(&DStream, nbBits);
+
+All above operations only read from local register (which size depends on size_t).
+Refueling the register from memory is manually performed by the reload method.
+ endSignal = FSE_reloadDStream(&DStream);
+
+BIT_reloadDStream() result tells if there is still some more data to read from DStream.
+BIT_DStream_unfinished : there is still some data left into the DStream.
+BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
+BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
+BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
+
+When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
+to properly detect the exact end of stream.
+After each decoded symbol, check if DStream is fully consumed using this simple test :
+ BIT_reloadDStream(&DStream) >= BIT_DStream_completed
+
+When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
+Checking if DStream has reached its end is performed by :
+ BIT_endOfDStream(&DStream);
+Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
+ FSE_endOfDState(&DState);
+*/
+
+
+/* *****************************************
+* FSE unsafe API
+*******************************************/
+static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
+/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
+
+
+/* *****************************************
+* Implementation of inlined functions
+*******************************************/
+typedef struct {
+ int deltaFindState;
+ U32 deltaNbBits;
+} FSE_symbolCompressionTransform; /* total 8 bytes */
+
+MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
+{
+ const void* ptr = ct;
+ const U16* u16ptr = (const U16*) ptr;
+ const U32 tableLog = MEM_read16(ptr);
+ statePtr->value = (ptrdiff_t)1<<tableLog;
+ statePtr->stateTable = u16ptr+2;
+ statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
+ statePtr->stateLog = tableLog;
+}
+
+
+/*! FSE_initCState2() :
+* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
+* uses the smallest state value possible, saving the cost of this symbol */
+MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
+{
+ FSE_initCState(statePtr, ct);
+ { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
+ const U16* stateTable = (const U16*)(statePtr->stateTable);
+ U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
+ statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+ }
+}
+
+MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
+{
+ FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
+ const U16* const stateTable = (const U16*)(statePtr->stateTable);
+ U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
+ BIT_addBits(bitC, statePtr->value, nbBitsOut);
+ statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+}
+
+MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
+{
+ BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
+ BIT_flushBits(bitC);
+}
+
+
+/* FSE_getMaxNbBits() :
+ * Approximate maximum cost of a symbol, in bits.
+ * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
+MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
+{
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
+ return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
+}
+
+/* FSE_bitCost() :
+ * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
+MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
+{
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
+ U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
+ U32 const threshold = (minNbBits+1) << 16;
+ assert(tableLog < 16);
+ assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
+ { U32 const tableSize = 1 << tableLog;
+ U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
+ U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
+ U32 const bitMultiplier = 1 << accuracyLog;
+ assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
+ assert(normalizedDeltaFromThreshold <= bitMultiplier);
+ return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
+ }
+}
+
+
+/* ====== Decompression ====== */
+
+typedef struct {
+ U16 tableLog;
+ U16 fastMode;
+} FSE_DTableHeader; /* sizeof U32 */
+
+typedef struct
+{
+ unsigned short newState;
+ unsigned char symbol;
+ unsigned char nbBits;
+} FSE_decode_t; /* size == U32 */
+
+MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
+{
+ const void* ptr = dt;
+ const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
+ DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
+ BIT_reloadDStream(bitD);
+ DStatePtr->table = dt + 1;
+}
+
+MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ return DInfo.symbol;
+}
+
+MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+ DStatePtr->state = DInfo.newState + lowBits;
+}
+
+MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ BYTE const symbol = DInfo.symbol;
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+
+ DStatePtr->state = DInfo.newState + lowBits;
+ return symbol;
+}
+
+/*! FSE_decodeSymbolFast() :
+ unsafe, only works if no symbol has a probability > 50% */
+MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ BYTE const symbol = DInfo.symbol;
+ size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
+
+ DStatePtr->state = DInfo.newState + lowBits;
+ return symbol;
+}
+
+MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
+{
+ return DStatePtr->state == 0;
+}
+
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/* **************************************************************
+* Tuning parameters
+****************************************************************/
+/*!MEMORY_USAGE :
+* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+* Increasing memory usage improves compression ratio
+* Reduced memory usage can improve speed, due to cache effect
+* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
+#ifndef FSE_MAX_MEMORY_USAGE
+# define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+# define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
+#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE)
+# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE"
+#endif
+
+/*!FSE_MAX_SYMBOL_VALUE :
+* Maximum symbol value authorized.
+* Required for proper stack allocation */
+#ifndef FSE_MAX_SYMBOL_VALUE
+# define FSE_MAX_SYMBOL_VALUE 255
+#endif
+
+/* **************************************************************
+* template functions type & suffix
+****************************************************************/
+#define FSE_FUNCTION_TYPE BYTE
+#define FSE_FUNCTION_EXTENSION
+#define FSE_DECODE_TYPE FSE_decode_t
+
+
+#endif /* !FSE_COMMONDEFS_ONLY */
+
+
+/* ***************************************************************
+* Constants
+*****************************************************************/
+#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
+#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
+#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
+#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
+#define FSE_MIN_TABLELOG 5
+
+#define FSE_TABLELOG_ABSOLUTE_MAX 15
+#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
+# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
+#endif
+
+#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
+
+
+#endif /* FSE_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/fse_compress.c b/contrib/zstd/fse_compress.c
new file mode 100644
index 0000000..3fec4a3
--- /dev/null
+++ b/contrib/zstd/fse_compress.c
@@ -0,0 +1,624 @@
+/* ******************************************************************
+ * FSE : Finite State Entropy encoder
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "compiler.h"
+#include "mem.h" /* U32, U16, etc. */
+#include "debug.h" /* assert, DEBUGLOG */
+#include "hist.h" /* HIST_count_wksp */
+#include "bitstream.h"
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "error_private.h"
+#define ZSTD_DEPS_NEED_MALLOC
+#define ZSTD_DEPS_NEED_MATH64
+#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define FSE_isError ERR_isError
+
+
+/* **************************************************************
+* Templates
+****************************************************************/
+/*
+ designed to be included
+ for type-specific functions (template emulation in C)
+ Objective is to write these functions only once, for improved maintenance
+*/
+
+/* safety checks */
+#ifndef FSE_FUNCTION_EXTENSION
+# error "FSE_FUNCTION_EXTENSION must be defined"
+#endif
+#ifndef FSE_FUNCTION_TYPE
+# error "FSE_FUNCTION_TYPE must be defined"
+#endif
+
+/* Function names */
+#define FSE_CAT(X,Y) X##Y
+#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
+#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
+
+
+/* Function templates */
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
+ * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
+ */
+size_t FSE_buildCTable_wksp(FSE_CTable* ct,
+ const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
+ void* workSpace, size_t wkspSize)
+{
+ U32 const tableSize = 1 << tableLog;
+ U32 const tableMask = tableSize - 1;
+ void* const ptr = ct;
+ U16* const tableU16 = ( (U16*) ptr) + 2;
+ void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
+ FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 const maxSV1 = maxSymbolValue+1;
+
+ U16* cumul = (U16*)workSpace; /* size = maxSV1 */
+ FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */
+
+ U32 highThreshold = tableSize-1;
+
+ assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */
+ if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
+ /* CTable header */
+ tableU16[-2] = (U16) tableLog;
+ tableU16[-1] = (U16) maxSymbolValue;
+ assert(tableLog < 16); /* required for threshold strategy to work */
+
+ /* For explanations on how to distribute symbol values over the table :
+ * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+
+ #ifdef __clang_analyzer__
+ ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
+ #endif
+
+ /* symbol start positions */
+ { U32 u;
+ cumul[0] = 0;
+ for (u=1; u <= maxSV1; u++) {
+ if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
+ cumul[u] = cumul[u-1] + 1;
+ tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
+ } else {
+ assert(normalizedCounter[u-1] >= 0);
+ cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1];
+ assert(cumul[u] >= cumul[u-1]); /* no overflow */
+ } }
+ cumul[maxSV1] = (U16)(tableSize+1);
+ }
+
+ /* Spread symbols */
+ if (highThreshold == tableSize - 1) {
+ /* Case for no low prob count symbols. Lay down 8 bytes at a time
+ * to reduce branch misses since we are operating on a small block
+ */
+ BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */
+ { U64 const add = 0x0101010101010101ull;
+ size_t pos = 0;
+ U64 sv = 0;
+ U32 s;
+ for (s=0; s<maxSV1; ++s, sv += add) {
+ int i;
+ int const n = normalizedCounter[s];
+ MEM_write64(spread + pos, sv);
+ for (i = 8; i < n; i += 8) {
+ MEM_write64(spread + pos + i, sv);
+ }
+ assert(n>=0);
+ pos += (size_t)n;
+ }
+ }
+ /* Spread symbols across the table. Lack of lowprob symbols means that
+ * we don't need variable sized inner loop, so we can unroll the loop and
+ * reduce branch misses.
+ */
+ { size_t position = 0;
+ size_t s;
+ size_t const unroll = 2; /* Experimentally determined optimal unroll */
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
+ size_t u;
+ for (u = 0; u < unroll; ++u) {
+ size_t const uPosition = (position + (u * step)) & tableMask;
+ tableSymbol[uPosition] = spread[s + u];
+ }
+ position = (position + (unroll * step)) & tableMask;
+ }
+ assert(position == 0); /* Must have initialized all positions */
+ }
+ } else {
+ U32 position = 0;
+ U32 symbol;
+ for (symbol=0; symbol<maxSV1; symbol++) {
+ int nbOccurrences;
+ int const freq = normalizedCounter[symbol];
+ for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) {
+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
+ position = (position + step) & tableMask;
+ while (position > highThreshold)
+ position = (position + step) & tableMask; /* Low proba area */
+ } }
+ assert(position==0); /* Must have initialized all positions */
+ }
+
+ /* Build table */
+ { U32 u; for (u=0; u<tableSize; u++) {
+ FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
+ tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
+ } }
+
+ /* Build Symbol Transformation Table */
+ { unsigned total = 0;
+ unsigned s;
+ for (s=0; s<=maxSymbolValue; s++) {
+ switch (normalizedCounter[s])
+ {
+ case 0:
+ /* filling nonetheless, for compatibility with FSE_getMaxNbBits() */
+ symbolTT[s].deltaNbBits = ((tableLog+1) << 16) - (1<<tableLog);
+ break;
+
+ case -1:
+ case 1:
+ symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
+ assert(total <= INT_MAX);
+ symbolTT[s].deltaFindState = (int)(total - 1);
+ total ++;
+ break;
+ default :
+ assert(normalizedCounter[s] > 1);
+ { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1);
+ U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut;
+ symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
+ symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]);
+ total += (unsigned)normalizedCounter[s];
+ } } } }
+
+#if 0 /* debug : symbol costs */
+ DEBUGLOG(5, "\n --- table statistics : ");
+ { U32 symbol;
+ for (symbol=0; symbol<=maxSymbolValue; symbol++) {
+ DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f",
+ symbol, normalizedCounter[symbol],
+ FSE_getMaxNbBits(symbolTT, symbol),
+ (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
+ } }
+#endif
+
+ return 0;
+}
+
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/*-**************************************************************
+* FSE NCount encoding
+****************************************************************/
+size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
+{
+ size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog
+ + 4 /* bitCount initialized at 4 */
+ + 2 /* first two symbols may use one additional bit each */) / 8)
+ + 1 /* round up to whole nb bytes */
+ + 2 /* additional two bytes for bitstream flush */;
+ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
+}
+
+static size_t
+FSE_writeNCount_generic (void* header, size_t headerBufferSize,
+ const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
+ unsigned writeIsSafe)
+{
+ BYTE* const ostart = (BYTE*) header;
+ BYTE* out = ostart;
+ BYTE* const oend = ostart + headerBufferSize;
+ int nbBits;
+ const int tableSize = 1 << tableLog;
+ int remaining;
+ int threshold;
+ U32 bitStream = 0;
+ int bitCount = 0;
+ unsigned symbol = 0;
+ unsigned const alphabetSize = maxSymbolValue + 1;
+ int previousIs0 = 0;
+
+ /* Table Size */
+ bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
+ bitCount += 4;
+
+ /* Init */
+ remaining = tableSize+1; /* +1 for extra accuracy */
+ threshold = tableSize;
+ nbBits = tableLog+1;
+
+ while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */
+ if (previousIs0) {
+ unsigned start = symbol;
+ while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
+ if (symbol == alphabetSize) break; /* incorrect distribution */
+ while (symbol >= start+24) {
+ start+=24;
+ bitStream += 0xFFFFU << bitCount;
+ if ((!writeIsSafe) && (out > oend-2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE) bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out+=2;
+ bitStream>>=16;
+ }
+ while (symbol >= start+3) {
+ start+=3;
+ bitStream += 3 << bitCount;
+ bitCount += 2;
+ }
+ bitStream += (symbol-start) << bitCount;
+ bitCount += 2;
+ if (bitCount>16) {
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out += 2;
+ bitStream >>= 16;
+ bitCount -= 16;
+ } }
+ { int count = normalizedCounter[symbol++];
+ int const max = (2*threshold-1) - remaining;
+ remaining -= count < 0 ? -count : count;
+ count++; /* +1 for extra accuracy */
+ if (count>=threshold)
+ count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
+ bitStream += count << bitCount;
+ bitCount += nbBits;
+ bitCount -= (count<max);
+ previousIs0 = (count==1);
+ if (remaining<1) return ERROR(GENERIC);
+ while (remaining<threshold) { nbBits--; threshold>>=1; }
+ }
+ if (bitCount>16) {
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out += 2;
+ bitStream >>= 16;
+ bitCount -= 16;
+ } }
+
+ if (remaining != 1)
+ return ERROR(GENERIC); /* incorrect normalized distribution */
+ assert(symbol <= alphabetSize);
+
+ /* flush remaining bitStream */
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out+= (bitCount+7) /8;
+
+ return (out-ostart);
+}
+
+
+size_t FSE_writeNCount (void* buffer, size_t bufferSize,
+ const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
+{
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
+ if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
+
+ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
+
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
+}
+
+
+/*-**************************************************************
+* FSE Compression Code
+****************************************************************/
+
+/* provides the minimum logSize to safely represent a distribution */
+static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
+{
+ U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1;
+ U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2;
+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+ assert(srcSize > 1); /* Not supported, RLE should be used instead */
+ return minBits;
+}
+
+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
+{
+ U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus;
+ U32 tableLog = maxTableLog;
+ U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
+ assert(srcSize > 1); /* Not supported, RLE should be used instead */
+ if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
+ if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
+ if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
+ if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
+ if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
+ return tableLog;
+}
+
+unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
+{
+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
+}
+
+/* Secondary normalization method.
+ To be used when primary method fails. */
+
+static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount)
+{
+ short const NOT_YET_ASSIGNED = -2;
+ U32 s;
+ U32 distributed = 0;
+ U32 ToDistribute;
+
+ /* Init */
+ U32 const lowThreshold = (U32)(total >> tableLog);
+ U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
+
+ for (s=0; s<=maxSymbolValue; s++) {
+ if (count[s] == 0) {
+ norm[s]=0;
+ continue;
+ }
+ if (count[s] <= lowThreshold) {
+ norm[s] = lowProbCount;
+ distributed++;
+ total -= count[s];
+ continue;
+ }
+ if (count[s] <= lowOne) {
+ norm[s] = 1;
+ distributed++;
+ total -= count[s];
+ continue;
+ }
+
+ norm[s]=NOT_YET_ASSIGNED;
+ }
+ ToDistribute = (1 << tableLog) - distributed;
+
+ if (ToDistribute == 0)
+ return 0;
+
+ if ((total / ToDistribute) > lowOne) {
+ /* risk of rounding to zero */
+ lowOne = (U32)((total * 3) / (ToDistribute * 2));
+ for (s=0; s<=maxSymbolValue; s++) {
+ if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
+ norm[s] = 1;
+ distributed++;
+ total -= count[s];
+ continue;
+ } }
+ ToDistribute = (1 << tableLog) - distributed;
+ }
+
+ if (distributed == maxSymbolValue+1) {
+ /* all values are pretty poor;
+ probably incompressible data (should have already been detected);
+ find max, then give all remaining points to max */
+ U32 maxV = 0, maxC = 0;
+ for (s=0; s<=maxSymbolValue; s++)
+ if (count[s] > maxC) { maxV=s; maxC=count[s]; }
+ norm[maxV] += (short)ToDistribute;
+ return 0;
+ }
+
+ if (total == 0) {
+ /* all of the symbols were low enough for the lowOne or lowThreshold */
+ for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
+ if (norm[s] > 0) { ToDistribute--; norm[s]++; }
+ return 0;
+ }
+
+ { U64 const vStepLog = 62 - tableLog;
+ U64 const mid = (1ULL << (vStepLog-1)) - 1;
+ U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
+ U64 tmpTotal = mid;
+ for (s=0; s<=maxSymbolValue; s++) {
+ if (norm[s]==NOT_YET_ASSIGNED) {
+ U64 const end = tmpTotal + (count[s] * rStep);
+ U32 const sStart = (U32)(tmpTotal >> vStepLog);
+ U32 const sEnd = (U32)(end >> vStepLog);
+ U32 const weight = sEnd - sStart;
+ if (weight < 1)
+ return ERROR(GENERIC);
+ norm[s] = (short)weight;
+ tmpTotal = end;
+ } } }
+
+ return 0;
+}
+
+size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
+ const unsigned* count, size_t total,
+ unsigned maxSymbolValue, unsigned useLowProbCount)
+{
+ /* Sanity checks */
+ if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
+ if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */
+ if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
+
+ { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
+ short const lowProbCount = useLowProbCount ? -1 : 1;
+ U64 const scale = 62 - tableLog;
+ U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */
+ U64 const vStep = 1ULL<<(scale-20);
+ int stillToDistribute = 1<<tableLog;
+ unsigned s;
+ unsigned largest=0;
+ short largestP=0;
+ U32 lowThreshold = (U32)(total >> tableLog);
+
+ for (s=0; s<=maxSymbolValue; s++) {
+ if (count[s] == total) return 0; /* rle special case */
+ if (count[s] == 0) { normalizedCounter[s]=0; continue; }
+ if (count[s] <= lowThreshold) {
+ normalizedCounter[s] = lowProbCount;
+ stillToDistribute--;
+ } else {
+ short proba = (short)((count[s]*step) >> scale);
+ if (proba<8) {
+ U64 restToBeat = vStep * rtbTable[proba];
+ proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
+ }
+ if (proba > largestP) { largestP=proba; largest=s; }
+ normalizedCounter[s] = proba;
+ stillToDistribute -= proba;
+ } }
+ if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
+ /* corner case, need another normalization method */
+ size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
+ if (FSE_isError(errorCode)) return errorCode;
+ }
+ else normalizedCounter[largest] += (short)stillToDistribute;
+ }
+
+#if 0
+ { /* Print Table (debug) */
+ U32 s;
+ U32 nTotal = 0;
+ for (s=0; s<=maxSymbolValue; s++)
+ RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]);
+ for (s=0; s<=maxSymbolValue; s++)
+ nTotal += abs(normalizedCounter[s]);
+ if (nTotal != (1U<<tableLog))
+ RAWLOG(2, "Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
+ getchar();
+ }
+#endif
+
+ return tableLog;
+}
+
+/* fake FSE_CTable, for rle input (always same symbol) */
+size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
+{
+ void* ptr = ct;
+ U16* tableU16 = ( (U16*) ptr) + 2;
+ void* FSCTptr = (U32*)ptr + 2;
+ FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
+
+ /* header */
+ tableU16[-2] = (U16) 0;
+ tableU16[-1] = (U16) symbolValue;
+
+ /* Build table */
+ tableU16[0] = 0;
+ tableU16[1] = 0; /* just in case */
+
+ /* Build Symbol Transformation Table */
+ symbolTT[symbolValue].deltaNbBits = 0;
+ symbolTT[symbolValue].deltaFindState = 0;
+
+ return 0;
+}
+
+
+static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const FSE_CTable* ct, const unsigned fast)
+{
+ const BYTE* const istart = (const BYTE*) src;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* ip=iend;
+
+ BIT_CStream_t bitC;
+ FSE_CState_t CState1, CState2;
+
+ /* init */
+ if (srcSize <= 2) return 0;
+ { size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
+ if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ }
+
+#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
+
+ if (srcSize & 1) {
+ FSE_initCState2(&CState1, ct, *--ip);
+ FSE_initCState2(&CState2, ct, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ FSE_FLUSHBITS(&bitC);
+ } else {
+ FSE_initCState2(&CState2, ct, *--ip);
+ FSE_initCState2(&CState1, ct, *--ip);
+ }
+
+ /* join to mod 4 */
+ srcSize -= 2;
+ if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ FSE_FLUSHBITS(&bitC);
+ }
+
+ /* 2 or 4 encoding per loop */
+ while ( ip>istart ) {
+
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+
+ if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
+ FSE_FLUSHBITS(&bitC);
+
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+
+ if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ }
+
+ FSE_FLUSHBITS(&bitC);
+ }
+
+ FSE_flushCState(&bitC, &CState2);
+ FSE_flushCState(&bitC, &CState1);
+ return BIT_closeCStream(&bitC);
+}
+
+size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const FSE_CTable* ct)
+{
+ unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
+
+ if (fast)
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
+ else
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
+}
+
+
+size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
+
+#endif /* FSE_COMMONDEFS_ONLY */
diff --git a/contrib/zstd/fse_decompress.c b/contrib/zstd/fse_decompress.c
new file mode 100644
index 0000000..1e1c9f9
--- /dev/null
+++ b/contrib/zstd/fse_decompress.c
@@ -0,0 +1,311 @@
+/* ******************************************************************
+ * FSE : Finite State Entropy decoder
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "debug.h" /* assert */
+#include "bitstream.h"
+#include "compiler.h"
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "error_private.h"
+#define ZSTD_DEPS_NEED_MALLOC
+#include "zstd_deps.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define FSE_isError ERR_isError
+#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
+
+
+/* **************************************************************
+* Templates
+****************************************************************/
+/*
+ designed to be included
+ for type-specific functions (template emulation in C)
+ Objective is to write these functions only once, for improved maintenance
+*/
+
+/* safety checks */
+#ifndef FSE_FUNCTION_EXTENSION
+# error "FSE_FUNCTION_EXTENSION must be defined"
+#endif
+#ifndef FSE_FUNCTION_TYPE
+# error "FSE_FUNCTION_TYPE must be defined"
+#endif
+
+/* Function names */
+#define FSE_CAT(X,Y) X##Y
+#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
+#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
+
+static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
+{
+ void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
+ FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
+ U16* symbolNext = (U16*)workSpace;
+ BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1);
+
+ U32 const maxSV1 = maxSymbolValue + 1;
+ U32 const tableSize = 1 << tableLog;
+ U32 highThreshold = tableSize-1;
+
+ /* Sanity Checks */
+ if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge);
+ if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
+
+ /* Init, lay down lowprob symbols */
+ { FSE_DTableHeader DTableH;
+ DTableH.tableLog = (U16)tableLog;
+ DTableH.fastMode = 1;
+ { S16 const largeLimit= (S16)(1 << (tableLog-1));
+ U32 s;
+ for (s=0; s<maxSV1; s++) {
+ if (normalizedCounter[s]==-1) {
+ tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
+ symbolNext[s] = 1;
+ } else {
+ if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
+ symbolNext[s] = normalizedCounter[s];
+ } } }
+ ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
+ }
+
+ /* Spread symbols */
+ if (highThreshold == tableSize - 1) {
+ size_t const tableMask = tableSize-1;
+ size_t const step = FSE_TABLESTEP(tableSize);
+ /* First lay down the symbols in order.
+ * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
+ * misses since small blocks generally have small table logs, so nearly
+ * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
+ * our buffer to handle the over-write.
+ */
+ {
+ U64 const add = 0x0101010101010101ull;
+ size_t pos = 0;
+ U64 sv = 0;
+ U32 s;
+ for (s=0; s<maxSV1; ++s, sv += add) {
+ int i;
+ int const n = normalizedCounter[s];
+ MEM_write64(spread + pos, sv);
+ for (i = 8; i < n; i += 8) {
+ MEM_write64(spread + pos + i, sv);
+ }
+ pos += n;
+ }
+ }
+ /* Now we spread those positions across the table.
+ * The benefit of doing it in two stages is that we avoid the
+ * variable size inner loop, which caused lots of branch misses.
+ * Now we can run through all the positions without any branch misses.
+ * We unroll the loop twice, since that is what empirically worked best.
+ */
+ {
+ size_t position = 0;
+ size_t s;
+ size_t const unroll = 2;
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
+ size_t u;
+ for (u = 0; u < unroll; ++u) {
+ size_t const uPosition = (position + (u * step)) & tableMask;
+ tableDecode[uPosition].symbol = spread[s + u];
+ }
+ position = (position + (unroll * step)) & tableMask;
+ }
+ assert(position == 0);
+ }
+ } else {
+ U32 const tableMask = tableSize-1;
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 s, position = 0;
+ for (s=0; s<maxSV1; s++) {
+ int i;
+ for (i=0; i<normalizedCounter[s]; i++) {
+ tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
+ position = (position + step) & tableMask;
+ while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
+ } }
+ if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
+ }
+
+ /* Build Decoding table */
+ { U32 u;
+ for (u=0; u<tableSize; u++) {
+ FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
+ U32 const nextState = symbolNext[symbol]++;
+ tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
+ tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
+ } }
+
+ return 0;
+}
+
+size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
+{
+ return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
+}
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/*-*******************************************************
+* Decompression (Byte symbols)
+*********************************************************/
+
+FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
+ void* dst, size_t maxDstSize,
+ const void* cSrc, size_t cSrcSize,
+ const FSE_DTable* dt, const unsigned fast)
+{
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* op = ostart;
+ BYTE* const omax = op + maxDstSize;
+ BYTE* const olimit = omax-3;
+
+ BIT_DStream_t bitD;
+ FSE_DState_t state1;
+ FSE_DState_t state2;
+
+ /* Init */
+ CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
+
+ FSE_initDState(&state1, &bitD, dt);
+ FSE_initDState(&state2, &bitD, dt);
+
+#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
+
+ /* 4 symbols per loop */
+ for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
+ op[0] = FSE_GETSYMBOL(&state1);
+
+ if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
+ BIT_reloadDStream(&bitD);
+
+ op[1] = FSE_GETSYMBOL(&state2);
+
+ if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
+ { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
+
+ op[2] = FSE_GETSYMBOL(&state1);
+
+ if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
+ BIT_reloadDStream(&bitD);
+
+ op[3] = FSE_GETSYMBOL(&state2);
+ }
+
+ /* tail */
+ /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
+ while (1) {
+ if (op>(omax-2)) return ERROR(dstSize_tooSmall);
+ *op++ = FSE_GETSYMBOL(&state1);
+ if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
+ *op++ = FSE_GETSYMBOL(&state2);
+ break;
+ }
+
+ if (op>(omax-2)) return ERROR(dstSize_tooSmall);
+ *op++ = FSE_GETSYMBOL(&state2);
+ if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
+ *op++ = FSE_GETSYMBOL(&state1);
+ break;
+ } }
+
+ return op-ostart;
+}
+
+typedef struct {
+ short ncount[FSE_MAX_SYMBOL_VALUE + 1];
+ FSE_DTable dtable[1]; /* Dynamically sized */
+} FSE_DecompressWksp;
+
+
+FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
+ void* dst, size_t dstCapacity,
+ const void* cSrc, size_t cSrcSize,
+ unsigned maxLog, void* workSpace, size_t wkspSize,
+ int bmi2)
+{
+ const BYTE* const istart = (const BYTE*)cSrc;
+ const BYTE* ip = istart;
+ unsigned tableLog;
+ unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
+ FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
+
+ DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
+ if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
+
+ /* normal FSE decoding mode */
+ {
+ size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
+ if (FSE_isError(NCountLength)) return NCountLength;
+ if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
+ assert(NCountLength <= cSrcSize);
+ ip += NCountLength;
+ cSrcSize -= NCountLength;
+ }
+
+ if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
+ assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize);
+ workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
+ wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
+
+ CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
+
+ {
+ const void* ptr = wksp->dtable;
+ const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
+ const U32 fastMode = DTableH->fastMode;
+
+ /* select fast mode (static) */
+ if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
+ return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
+ }
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
+{
+ return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
+{
+ return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
+}
+#endif
+
+size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
+ }
+#endif
+ (void)bmi2;
+ return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
+}
+
+#endif /* FSE_COMMONDEFS_ONLY */
diff --git a/contrib/zstd/hist.c b/contrib/zstd/hist.c
new file mode 100644
index 0000000..ebb1a1a
--- /dev/null
+++ b/contrib/zstd/hist.c
@@ -0,0 +1,181 @@
+/* ******************************************************************
+ * hist : Histogram functions
+ * part of Finite State Entropy project
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* --- dependencies --- */
+#include "mem.h" /* U32, BYTE, etc. */
+#include "debug.h" /* assert, DEBUGLOG */
+#include "error_private.h" /* ERROR */
+#include "hist.h"
+
+
+/* --- Error management --- */
+unsigned HIST_isError(size_t code) { return ERR_isError(code); }
+
+/*-**************************************************************
+ * Histogram functions
+ ****************************************************************/
+unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize)
+{
+ const BYTE* ip = (const BYTE*)src;
+ const BYTE* const end = ip + srcSize;
+ unsigned maxSymbolValue = *maxSymbolValuePtr;
+ unsigned largestCount=0;
+
+ ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
+ if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
+
+ while (ip<end) {
+ assert(*ip <= maxSymbolValue);
+ count[*ip++]++;
+ }
+
+ while (!count[maxSymbolValue]) maxSymbolValue--;
+ *maxSymbolValuePtr = maxSymbolValue;
+
+ { U32 s;
+ for (s=0; s<=maxSymbolValue; s++)
+ if (count[s] > largestCount) largestCount = count[s];
+ }
+
+ return largestCount;
+}
+
+typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
+
+/* HIST_count_parallel_wksp() :
+ * store histogram into 4 intermediate tables, recombined at the end.
+ * this design makes better use of OoO cpus,
+ * and is noticeably faster when some values are heavily repeated.
+ * But it needs some additional workspace for intermediate tables.
+ * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
+ * @return : largest histogram frequency,
+ * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
+static size_t HIST_count_parallel_wksp(
+ unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize,
+ HIST_checkInput_e check,
+ U32* const workSpace)
+{
+ const BYTE* ip = (const BYTE*)source;
+ const BYTE* const iend = ip+sourceSize;
+ size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count);
+ unsigned max=0;
+ U32* const Counting1 = workSpace;
+ U32* const Counting2 = Counting1 + 256;
+ U32* const Counting3 = Counting2 + 256;
+ U32* const Counting4 = Counting3 + 256;
+
+ /* safety checks */
+ assert(*maxSymbolValuePtr <= 255);
+ if (!sourceSize) {
+ ZSTD_memset(count, 0, countSize);
+ *maxSymbolValuePtr = 0;
+ return 0;
+ }
+ ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned));
+
+ /* by stripes of 16 bytes */
+ { U32 cached = MEM_read32(ip); ip += 4;
+ while (ip < iend-15) {
+ U32 c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ }
+ ip-=4;
+ }
+
+ /* finish last symbols */
+ while (ip<iend) Counting1[*ip++]++;
+
+ { U32 s;
+ for (s=0; s<256; s++) {
+ Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
+ if (Counting1[s] > max) max = Counting1[s];
+ } }
+
+ { unsigned maxSymbolValue = 255;
+ while (!Counting1[maxSymbolValue]) maxSymbolValue--;
+ if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall);
+ *maxSymbolValuePtr = maxSymbolValue;
+ ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */
+ }
+ return (size_t)max;
+}
+
+/* HIST_countFast_wksp() :
+ * Same as HIST_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` is a writable buffer which must be 4-bytes aligned,
+ * `workSpaceSize` must be >= HIST_WKSP_SIZE
+ */
+size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize,
+ void* workSpace, size_t workSpaceSize)
+{
+ if (sourceSize < 1500) /* heuristic threshold */
+ return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
+ if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
+ if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
+ return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
+}
+
+/* HIST_count_wksp() :
+ * Same as HIST_count(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
+size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize,
+ void* workSpace, size_t workSpaceSize)
+{
+ if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
+ if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
+ if (*maxSymbolValuePtr < 255)
+ return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace);
+ *maxSymbolValuePtr = 255;
+ return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
+}
+
+#ifndef ZSTD_NO_UNUSED_FUNCTIONS
+/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
+size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize)
+{
+ unsigned tmpCounters[HIST_WKSP_SIZE_U32];
+ return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
+}
+
+size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize)
+{
+ unsigned tmpCounters[HIST_WKSP_SIZE_U32];
+ return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
+}
+#endif
diff --git a/contrib/zstd/hist.h b/contrib/zstd/hist.h
new file mode 100644
index 0000000..b8d5ba7
--- /dev/null
+++ b/contrib/zstd/hist.h
@@ -0,0 +1,75 @@
+/* ******************************************************************
+ * hist : Histogram functions
+ * part of Finite State Entropy project
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* --- dependencies --- */
+#include "zstd_deps.h" /* size_t */
+
+
+/* --- simple histogram functions --- */
+
+/*! HIST_count():
+ * Provides the precise count of each byte within a table 'count'.
+ * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
+ * Updates *maxSymbolValuePtr with actual largest symbol value detected.
+ * @return : count of the most frequent symbol (which isn't identified).
+ * or an error code, which can be tested using HIST_isError().
+ * note : if return == srcSize, there is only one symbol.
+ */
+size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize);
+
+unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */
+
+
+/* --- advanced histogram functions --- */
+
+#define HIST_WKSP_SIZE_U32 1024
+#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned))
+/** HIST_count_wksp() :
+ * Same as HIST_count(), but using an externally provided scratch buffer.
+ * Benefit is this function will use very little stack space.
+ * `workSpace` is a writable buffer which must be 4-bytes aligned,
+ * `workSpaceSize` must be >= HIST_WKSP_SIZE
+ */
+size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t workSpaceSize);
+
+/** HIST_countFast() :
+ * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr.
+ * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr`
+ */
+size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize);
+
+/** HIST_countFast_wksp() :
+ * Same as HIST_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` is a writable buffer which must be 4-bytes aligned,
+ * `workSpaceSize` must be >= HIST_WKSP_SIZE
+ */
+size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t workSpaceSize);
+
+/*! HIST_count_simple() :
+ * Same as HIST_countFast(), this function is unsafe,
+ * and will segfault if any value within `src` is `> *maxSymbolValuePtr`.
+ * It is also a bit slower for large inputs.
+ * However, it does not need any additional memory (not even on stack).
+ * @return : count of the most frequent symbol.
+ * Note this function doesn't produce any error (i.e. it must succeed).
+ */
+unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize);
diff --git a/contrib/zstd/huf.h b/contrib/zstd/huf.h
new file mode 100644
index 0000000..73d1ee5
--- /dev/null
+++ b/contrib/zstd/huf.h
@@ -0,0 +1,273 @@
+/* ******************************************************************
+ * huff0 huffman codec,
+ * part of Finite State Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef HUF_H_298734234
+#define HUF_H_298734234
+
+/* *** Dependencies *** */
+#include "zstd_deps.h" /* size_t */
+#include "mem.h" /* U32 */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+
+
+/* *** Tool functions *** */
+#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
+size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
+
+/* Error Management */
+unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
+const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
+
+
+#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
+#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
+
+/* *** Constants *** */
+#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
+#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
+#define HUF_SYMBOLVALUE_MAX 255
+
+#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
+#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
+# error "HUF_TABLELOG_MAX is too large !"
+#endif
+
+
+/* ****************************************
+* Static allocation
+******************************************/
+/* HUF buffer bounds */
+#define HUF_CTABLEBOUND 129
+#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */
+#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
+
+/* static allocation of HUF's Compression Table */
+/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
+typedef size_t HUF_CElt; /* consider it an incomplete type */
+#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
+#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
+#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
+ HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
+
+/* static allocation of HUF's DTable */
+typedef U32 HUF_DTable;
+#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
+#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
+ HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
+#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
+ HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
+
+
+/* ****************************************
+* Advanced decompression functions
+******************************************/
+
+/**
+ * Huffman flags bitset.
+ * For all flags, 0 is the default value.
+ */
+typedef enum {
+ /**
+ * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
+ * Otherwise: Ignored.
+ */
+ HUF_flags_bmi2 = (1 << 0),
+ /**
+ * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
+ * If unset: Use heuristic to find the table depth.
+ */
+ HUF_flags_optimalDepth = (1 << 1),
+ /**
+ * If set: If the previous table can encode the input, always reuse the previous table.
+ * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
+ */
+ HUF_flags_preferRepeat = (1 << 2),
+ /**
+ * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
+ * If unset: Always histogram the entire input.
+ */
+ HUF_flags_suspectUncompressible = (1 << 3),
+ /**
+ * If set: Don't use assembly implementations
+ * If unset: Allow using assembly implementations
+ */
+ HUF_flags_disableAsm = (1 << 4),
+ /**
+ * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
+ * If unset: Use the fast decoding loop when possible.
+ */
+ HUF_flags_disableFast = (1 << 5)
+} HUF_flags_e;
+
+
+/* ****************************************
+ * HUF detailed API
+ * ****************************************/
+#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
+
+/*! HUF_compress() does the following:
+ * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
+ * 2. (optional) refine tableLog using HUF_optimalTableLog()
+ * 3. build Huffman table from count using HUF_buildCTable()
+ * 4. save Huffman table to memory buffer using HUF_writeCTable()
+ * 5. encode the data stream using HUF_compress4X_usingCTable()
+ *
+ * The following API allows targeting specific sub-functions for advanced tasks.
+ * For example, it's possible to compress several blocks using the same 'CTable',
+ * or to save and regenerate 'CTable' using external methods.
+ */
+unsigned HUF_minTableLog(unsigned symbolCardinality);
+unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
+ size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
+size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
+size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
+int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
+
+typedef enum {
+ HUF_repeat_none, /**< Cannot use the previous table */
+ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
+ } HUF_repeat;
+
+/** HUF_compress4X_repeat() :
+ * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ * If it uses hufTable it does not modify hufTable or repeat.
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ * If preferRepeat then the old table will always be used if valid.
+ * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned tableLog,
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
+
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
+ */
+#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
+#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
+size_t HUF_buildCTable_wksp (HUF_CElt* tree,
+ const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
+ void* workSpace, size_t wkspSize);
+
+/*! HUF_readStats() :
+ * Read compact Huffman tree, saved by HUF_writeCTable().
+ * `huffWeight` is destination buffer.
+ * @return : size read from `src` , or an error Code .
+ * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
+size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
+ U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize);
+
+/*! HUF_readStats_wksp() :
+ * Same as HUF_readStats() but takes an external workspace which must be
+ * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
+ */
+#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
+#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
+size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
+ U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workspace, size_t wkspSize,
+ int flags);
+
+/** HUF_readCTable() :
+ * Loading a CTable saved with HUF_writeCTable() */
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
+
+/** HUF_getNbBitsFromCTable() :
+ * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
+ * Note 1 : is not inlined, as HUF_CElt definition is private */
+U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
+
+/*
+ * HUF_decompress() does the following:
+ * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
+ * 2. build Huffman table from save, using HUF_readDTableX?()
+ * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
+ */
+
+/** HUF_selectDecoder() :
+ * Tells which decoder is likely to decode faster,
+ * based on a set of pre-computed metrics.
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
+ * Assumption : 0 < dstSize <= 128 KB */
+U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
+
+/**
+ * The minimum workspace size for the `workSpace` used in
+ * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
+ *
+ * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
+ * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
+ * Buffer overflow errors may potentially occur if code modifications result in
+ * a required workspace size greater than that specified in the following
+ * macro.
+ */
+#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
+#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
+
+
+/* ====================== */
+/* single stream variants */
+/* ====================== */
+
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
+/** HUF_compress1X_repeat() :
+ * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ * If it uses hufTable it does not modify hufTable or repeat.
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ * If preferRepeat then the old table will always be used if valid.
+ * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned tableLog,
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X1
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */
+#endif
+
+/* BMI2 variants.
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
+ */
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+#ifndef HUF_FORCE_DECOMPRESS_X1
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+
+#endif /* HUF_H_298734234 */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/huf_compress.c b/contrib/zstd/huf_compress.c
new file mode 100644
index 0000000..b5f0917
--- /dev/null
+++ b/contrib/zstd/huf_compress.c
@@ -0,0 +1,1435 @@
+/* ******************************************************************
+ * Huffman encoder, part of New Generation Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* **************************************************************
+* Compiler specifics
+****************************************************************/
+#ifdef _MSC_VER /* Visual Studio */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
+#include "compiler.h"
+#include "bitstream.h"
+#include "hist.h"
+#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
+#include "fse.h" /* header compression */
+#include "huf.h"
+#include "error_private.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define HUF_isError ERR_isError
+#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
+
+
+/* **************************************************************
+* Required declarations
+****************************************************************/
+typedef struct nodeElt_s {
+ U32 count;
+ U16 parent;
+ BYTE byte;
+ BYTE nbBits;
+} nodeElt;
+
+
+/* **************************************************************
+* Debug Traces
+****************************************************************/
+
+#if DEBUGLEVEL >= 2
+
+static size_t showU32(const U32* arr, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %u", arr[u]); (void)arr;
+ }
+ RAWLOG(6, " \n");
+ return size;
+}
+
+static size_t HUF_getNbBits(HUF_CElt elt);
+
+static size_t showCTableBits(const HUF_CElt* ctable, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %zu", HUF_getNbBits(ctable[u])); (void)ctable;
+ }
+ RAWLOG(6, " \n");
+ return size;
+
+}
+
+static size_t showHNodeSymbols(const nodeElt* hnode, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %u", hnode[u].byte); (void)hnode;
+ }
+ RAWLOG(6, " \n");
+ return size;
+}
+
+static size_t showHNodeBits(const nodeElt* hnode, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %u", hnode[u].nbBits); (void)hnode;
+ }
+ RAWLOG(6, " \n");
+ return size;
+}
+
+#endif
+
+
+/* *******************************************************
+* HUF : Huffman block compression
+*********************************************************/
+#define HUF_WORKSPACE_MAX_ALIGNMENT 8
+
+static void* HUF_alignUpWorkspace(void* workspace, size_t* workspaceSizePtr, size_t align)
+{
+ size_t const mask = align - 1;
+ size_t const rem = (size_t)workspace & mask;
+ size_t const add = (align - rem) & mask;
+ BYTE* const aligned = (BYTE*)workspace + add;
+ assert((align & (align - 1)) == 0); /* pow 2 */
+ assert(align <= HUF_WORKSPACE_MAX_ALIGNMENT);
+ if (*workspaceSizePtr >= add) {
+ assert(add < align);
+ assert(((size_t)aligned & mask) == 0);
+ *workspaceSizePtr -= add;
+ return aligned;
+ } else {
+ *workspaceSizePtr = 0;
+ return NULL;
+ }
+}
+
+
+/* HUF_compressWeights() :
+ * Same as FSE_compress(), but dedicated to huff0's weights compression.
+ * The use case needs much less stack memory.
+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
+ */
+#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
+
+typedef struct {
+ FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
+ U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)];
+ unsigned count[HUF_TABLELOG_MAX+1];
+ S16 norm[HUF_TABLELOG_MAX+1];
+} HUF_CompressWeightsWksp;
+
+static size_t
+HUF_compressWeights(void* dst, size_t dstSize,
+ const void* weightTable, size_t wtSize,
+ void* workspace, size_t workspaceSize)
+{
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* op = ostart;
+ BYTE* const oend = ostart + dstSize;
+
+ unsigned maxSymbolValue = HUF_TABLELOG_MAX;
+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
+ HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
+
+ if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC);
+
+ /* init conditions */
+ if (wtSize <= 1) return 0; /* Not compressible */
+
+ /* Scan input and build symbol stats */
+ { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */
+ if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
+ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
+ }
+
+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
+ CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) );
+
+ /* Write table description header */
+ { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) );
+ op += hSize;
+ }
+
+ /* Compress */
+ CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) );
+ { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) );
+ if (cSize == 0) return 0; /* not enough space for compressed data */
+ op += cSize;
+ }
+
+ return (size_t)(op-ostart);
+}
+
+static size_t HUF_getNbBits(HUF_CElt elt)
+{
+ return elt & 0xFF;
+}
+
+static size_t HUF_getNbBitsFast(HUF_CElt elt)
+{
+ return elt;
+}
+
+static size_t HUF_getValue(HUF_CElt elt)
+{
+ return elt & ~(size_t)0xFF;
+}
+
+static size_t HUF_getValueFast(HUF_CElt elt)
+{
+ return elt;
+}
+
+static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits)
+{
+ assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX);
+ *elt = nbBits;
+}
+
+static void HUF_setValue(HUF_CElt* elt, size_t value)
+{
+ size_t const nbBits = HUF_getNbBits(*elt);
+ if (nbBits > 0) {
+ assert((value >> nbBits) == 0);
+ *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits);
+ }
+}
+
+typedef struct {
+ HUF_CompressWeightsWksp wksp;
+ BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
+} HUF_WriteCTableWksp;
+
+size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
+ const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog,
+ void* workspace, size_t workspaceSize)
+{
+ HUF_CElt const* const ct = CTable + 1;
+ BYTE* op = (BYTE*)dst;
+ U32 n;
+ HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
+
+ HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp));
+
+ /* check conditions */
+ if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
+
+ /* convert to weight */
+ wksp->bitsToWeight[0] = 0;
+ for (n=1; n<huffLog+1; n++)
+ wksp->bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
+ for (n=0; n<maxSymbolValue; n++)
+ wksp->huffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])];
+
+ /* attempt weights compression by FSE */
+ if (maxDstSize < 1) return ERROR(dstSize_tooSmall);
+ { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) );
+ if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
+ op[0] = (BYTE)hSize;
+ return hSize+1;
+ } }
+
+ /* write raw values as 4-bits (max : 15) */
+ if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
+ if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
+ op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
+ wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
+ for (n=0; n<maxSymbolValue; n+=2)
+ op[(n/2)+1] = (BYTE)((wksp->huffWeight[n] << 4) + wksp->huffWeight[n+1]);
+ return ((maxSymbolValue+1)/2) + 1;
+}
+
+
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights)
+{
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
+ U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
+ U32 tableLog = 0;
+ U32 nbSymbols = 0;
+ HUF_CElt* const ct = CTable + 1;
+
+ /* get symbol weights */
+ CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
+ *hasZeroWeights = (rankVal[0] > 0);
+
+ /* check result */
+ if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
+ if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall);
+
+ CTable[0] = tableLog;
+
+ /* Prepare base value per rank */
+ { U32 n, nextRankStart = 0;
+ for (n=1; n<=tableLog; n++) {
+ U32 curr = nextRankStart;
+ nextRankStart += (rankVal[n] << (n-1));
+ rankVal[n] = curr;
+ } }
+
+ /* fill nbBits */
+ { U32 n; for (n=0; n<nbSymbols; n++) {
+ const U32 w = huffWeight[n];
+ HUF_setNbBits(ct + n, (BYTE)(tableLog + 1 - w) & -(w != 0));
+ } }
+
+ /* fill val */
+ { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
+ U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
+ { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[HUF_getNbBits(ct[n])]++; }
+ /* determine stating value per rank */
+ valPerRank[tableLog+1] = 0; /* for w==0 */
+ { U16 min = 0;
+ U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
+ valPerRank[n] = min; /* get starting value within each rank */
+ min += nbPerRank[n];
+ min >>= 1;
+ } }
+ /* assign value within rank, symbol order */
+ { U32 n; for (n=0; n<nbSymbols; n++) HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); }
+ }
+
+ *maxSymbolValuePtr = nbSymbols - 1;
+ return readSize;
+}
+
+U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue)
+{
+ const HUF_CElt* const ct = CTable + 1;
+ assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
+ return (U32)HUF_getNbBits(ct[symbolValue]);
+}
+
+
+/**
+ * HUF_setMaxHeight():
+ * Try to enforce @targetNbBits on the Huffman tree described in @huffNode.
+ *
+ * It attempts to convert all nodes with nbBits > @targetNbBits
+ * to employ @targetNbBits instead. Then it adjusts the tree
+ * so that it remains a valid canonical Huffman tree.
+ *
+ * @pre The sum of the ranks of each symbol == 2^largestBits,
+ * where largestBits == huffNode[lastNonNull].nbBits.
+ * @post The sum of the ranks of each symbol == 2^largestBits,
+ * where largestBits is the return value (expected <= targetNbBits).
+ *
+ * @param huffNode The Huffman tree modified in place to enforce targetNbBits.
+ * It's presumed sorted, from most frequent to rarest symbol.
+ * @param lastNonNull The symbol with the lowest count in the Huffman tree.
+ * @param targetNbBits The allowed number of bits, which the Huffman tree
+ * may not respect. After this function the Huffman tree will
+ * respect targetNbBits.
+ * @return The maximum number of bits of the Huffman tree after adjustment.
+ */
+static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits)
+{
+ const U32 largestBits = huffNode[lastNonNull].nbBits;
+ /* early exit : no elt > targetNbBits, so the tree is already valid. */
+ if (largestBits <= targetNbBits) return largestBits;
+
+ DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits);
+
+ /* there are several too large elements (at least >= 2) */
+ { int totalCost = 0;
+ const U32 baseCost = 1 << (largestBits - targetNbBits);
+ int n = (int)lastNonNull;
+
+ /* Adjust any ranks > targetNbBits to targetNbBits.
+ * Compute totalCost, which is how far the sum of the ranks is
+ * we are over 2^largestBits after adjust the offending ranks.
+ */
+ while (huffNode[n].nbBits > targetNbBits) {
+ totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
+ huffNode[n].nbBits = (BYTE)targetNbBits;
+ n--;
+ }
+ /* n stops at huffNode[n].nbBits <= targetNbBits */
+ assert(huffNode[n].nbBits <= targetNbBits);
+ /* n end at index of smallest symbol using < targetNbBits */
+ while (huffNode[n].nbBits == targetNbBits) --n;
+
+ /* renorm totalCost from 2^largestBits to 2^targetNbBits
+ * note : totalCost is necessarily a multiple of baseCost */
+ assert(((U32)totalCost & (baseCost - 1)) == 0);
+ totalCost >>= (largestBits - targetNbBits);
+ assert(totalCost > 0);
+
+ /* repay normalized cost */
+ { U32 const noSymbol = 0xF0F0F0F0;
+ U32 rankLast[HUF_TABLELOG_MAX+2];
+
+ /* Get pos of last (smallest = lowest cum. count) symbol per rank */
+ ZSTD_memset(rankLast, 0xF0, sizeof(rankLast));
+ { U32 currentNbBits = targetNbBits;
+ int pos;
+ for (pos=n ; pos >= 0; pos--) {
+ if (huffNode[pos].nbBits >= currentNbBits) continue;
+ currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */
+ rankLast[targetNbBits-currentNbBits] = (U32)pos;
+ } }
+
+ while (totalCost > 0) {
+ /* Try to reduce the next power of 2 above totalCost because we
+ * gain back half the rank.
+ */
+ U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1;
+ for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
+ U32 const highPos = rankLast[nBitsToDecrease];
+ U32 const lowPos = rankLast[nBitsToDecrease-1];
+ if (highPos == noSymbol) continue;
+ /* Decrease highPos if no symbols of lowPos or if it is
+ * not cheaper to remove 2 lowPos than highPos.
+ */
+ if (lowPos == noSymbol) break;
+ { U32 const highTotal = huffNode[highPos].count;
+ U32 const lowTotal = 2 * huffNode[lowPos].count;
+ if (highTotal <= lowTotal) break;
+ } }
+ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
+ assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1);
+ /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
+ while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))
+ nBitsToDecrease++;
+ assert(rankLast[nBitsToDecrease] != noSymbol);
+ /* Increase the number of bits to gain back half the rank cost. */
+ totalCost -= 1 << (nBitsToDecrease-1);
+ huffNode[rankLast[nBitsToDecrease]].nbBits++;
+
+ /* Fix up the new rank.
+ * If the new rank was empty, this symbol is now its smallest.
+ * Otherwise, this symbol will be the largest in the new rank so no adjustment.
+ */
+ if (rankLast[nBitsToDecrease-1] == noSymbol)
+ rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease];
+ /* Fix up the old rank.
+ * If the symbol was at position 0, meaning it was the highest weight symbol in the tree,
+ * it must be the only symbol in its rank, so the old rank now has no symbols.
+ * Otherwise, since the Huffman nodes are sorted by count, the previous position is now
+ * the smallest node in the rank. If the previous position belongs to a different rank,
+ * then the rank is now empty.
+ */
+ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
+ rankLast[nBitsToDecrease] = noSymbol;
+ else {
+ rankLast[nBitsToDecrease]--;
+ if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease)
+ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
+ }
+ } /* while (totalCost > 0) */
+
+ /* If we've removed too much weight, then we have to add it back.
+ * To avoid overshooting again, we only adjust the smallest rank.
+ * We take the largest nodes from the lowest rank 0 and move them
+ * to rank 1. There's guaranteed to be enough rank 0 symbols because
+ * TODO.
+ */
+ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
+ /* special case : no rank 1 symbol (using targetNbBits-1);
+ * let's create one from largest rank 0 (using targetNbBits).
+ */
+ if (rankLast[1] == noSymbol) {
+ while (huffNode[n].nbBits == targetNbBits) n--;
+ huffNode[n+1].nbBits--;
+ assert(n >= 0);
+ rankLast[1] = (U32)(n+1);
+ totalCost++;
+ continue;
+ }
+ huffNode[ rankLast[1] + 1 ].nbBits--;
+ rankLast[1]++;
+ totalCost ++;
+ }
+ } /* repay normalized cost */
+ } /* there are several too large elements (at least >= 2) */
+
+ return targetNbBits;
+}
+
+typedef struct {
+ U16 base;
+ U16 curr;
+} rankPos;
+
+typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)];
+
+/* Number of buckets available for HUF_sort() */
+#define RANK_POSITION_TABLE_SIZE 192
+
+typedef struct {
+ huffNodeTable huffNodeTbl;
+ rankPos rankPosition[RANK_POSITION_TABLE_SIZE];
+} HUF_buildCTable_wksp_tables;
+
+/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing.
+ * Strategy is to use as many buckets as possible for representing distinct
+ * counts while using the remainder to represent all "large" counts.
+ *
+ * To satisfy this requirement for 192 buckets, we can do the following:
+ * Let buckets 0-166 represent distinct counts of [0, 166]
+ * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing.
+ */
+#define RANK_POSITION_MAX_COUNT_LOG 32
+#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */)
+#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */)
+
+/* Return the appropriate bucket index for a given count. See definition of
+ * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy.
+ */
+static U32 HUF_getIndex(U32 const count) {
+ return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF)
+ ? count
+ : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN;
+}
+
+/* Helper swap function for HUF_quickSortPartition() */
+static void HUF_swapNodes(nodeElt* a, nodeElt* b) {
+ nodeElt tmp = *a;
+ *a = *b;
+ *b = tmp;
+}
+
+/* Returns 0 if the huffNode array is not sorted by descending count */
+MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) {
+ U32 i;
+ for (i = 1; i < maxSymbolValue1; ++i) {
+ if (huffNode[i].count > huffNode[i-1].count) {
+ return 0;
+ }
+ }
+ return 1;
+}
+
+/* Insertion sort by descending order */
+HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) {
+ int i;
+ int const size = high-low+1;
+ huffNode += low;
+ for (i = 1; i < size; ++i) {
+ nodeElt const key = huffNode[i];
+ int j = i - 1;
+ while (j >= 0 && huffNode[j].count < key.count) {
+ huffNode[j + 1] = huffNode[j];
+ j--;
+ }
+ huffNode[j + 1] = key;
+ }
+}
+
+/* Pivot helper function for quicksort. */
+static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) {
+ /* Simply select rightmost element as pivot. "Better" selectors like
+ * median-of-three don't experimentally appear to have any benefit.
+ */
+ U32 const pivot = arr[high].count;
+ int i = low - 1;
+ int j = low;
+ for ( ; j < high; j++) {
+ if (arr[j].count > pivot) {
+ i++;
+ HUF_swapNodes(&arr[i], &arr[j]);
+ }
+ }
+ HUF_swapNodes(&arr[i + 1], &arr[high]);
+ return i + 1;
+}
+
+/* Classic quicksort by descending with partially iterative calls
+ * to reduce worst case callstack size.
+ */
+static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) {
+ int const kInsertionSortThreshold = 8;
+ if (high - low < kInsertionSortThreshold) {
+ HUF_insertionSort(arr, low, high);
+ return;
+ }
+ while (low < high) {
+ int const idx = HUF_quickSortPartition(arr, low, high);
+ if (idx - low < high - idx) {
+ HUF_simpleQuickSort(arr, low, idx - 1);
+ low = idx + 1;
+ } else {
+ HUF_simpleQuickSort(arr, idx + 1, high);
+ high = idx - 1;
+ }
+ }
+}
+
+/**
+ * HUF_sort():
+ * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order.
+ * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket.
+ *
+ * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled.
+ * Must have (maxSymbolValue + 1) entries.
+ * @param[in] count Histogram of the symbols.
+ * @param[in] maxSymbolValue Maximum symbol value.
+ * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries.
+ */
+static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) {
+ U32 n;
+ U32 const maxSymbolValue1 = maxSymbolValue+1;
+
+ /* Compute base and set curr to base.
+ * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1.
+ * See HUF_getIndex to see bucketing strategy.
+ * We attribute each symbol to lowerRank's base value, because we want to know where
+ * each rank begins in the output, so for rank R we want to count ranks R+1 and above.
+ */
+ ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE);
+ for (n = 0; n < maxSymbolValue1; ++n) {
+ U32 lowerRank = HUF_getIndex(count[n]);
+ assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1);
+ rankPosition[lowerRank].base++;
+ }
+
+ assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0);
+ /* Set up the rankPosition table */
+ for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) {
+ rankPosition[n-1].base += rankPosition[n].base;
+ rankPosition[n-1].curr = rankPosition[n-1].base;
+ }
+
+ /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */
+ for (n = 0; n < maxSymbolValue1; ++n) {
+ U32 const c = count[n];
+ U32 const r = HUF_getIndex(c) + 1;
+ U32 const pos = rankPosition[r].curr++;
+ assert(pos < maxSymbolValue1);
+ huffNode[pos].count = c;
+ huffNode[pos].byte = (BYTE)n;
+ }
+
+ /* Sort each bucket. */
+ for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) {
+ int const bucketSize = rankPosition[n].curr - rankPosition[n].base;
+ U32 const bucketStartIdx = rankPosition[n].base;
+ if (bucketSize > 1) {
+ assert(bucketStartIdx < maxSymbolValue1);
+ HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1);
+ }
+ }
+
+ assert(HUF_isSorted(huffNode, maxSymbolValue1));
+}
+
+
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables).
+ */
+#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
+
+/* HUF_buildTree():
+ * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree.
+ *
+ * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array.
+ * @param maxSymbolValue The maximum symbol value.
+ * @return The smallest node in the Huffman tree (by count).
+ */
+static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
+{
+ nodeElt* const huffNode0 = huffNode - 1;
+ int nonNullRank;
+ int lowS, lowN;
+ int nodeNb = STARTNODE;
+ int n, nodeRoot;
+ DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1);
+ /* init for parents */
+ nonNullRank = (int)maxSymbolValue;
+ while(huffNode[nonNullRank].count == 0) nonNullRank--;
+ lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
+ huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
+ huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb;
+ nodeNb++; lowS-=2;
+ for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
+ huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
+
+ /* create parents */
+ while (nodeNb <= nodeRoot) {
+ int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
+ int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
+ huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
+ huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb;
+ nodeNb++;
+ }
+
+ /* distribute weights (unlimited tree height) */
+ huffNode[nodeRoot].nbBits = 0;
+ for (n=nodeRoot-1; n>=STARTNODE; n--)
+ huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
+ for (n=0; n<=nonNullRank; n++)
+ huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
+
+ DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1));
+
+ return nonNullRank;
+}
+
+/**
+ * HUF_buildCTableFromTree():
+ * Build the CTable given the Huffman tree in huffNode.
+ *
+ * @param[out] CTable The output Huffman CTable.
+ * @param huffNode The Huffman tree.
+ * @param nonNullRank The last and smallest node in the Huffman tree.
+ * @param maxSymbolValue The maximum symbol value.
+ * @param maxNbBits The exact maximum number of bits used in the Huffman tree.
+ */
+static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits)
+{
+ HUF_CElt* const ct = CTable + 1;
+ /* fill result into ctable (val, nbBits) */
+ int n;
+ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
+ U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
+ int const alphabetSize = (int)(maxSymbolValue + 1);
+ for (n=0; n<=nonNullRank; n++)
+ nbPerRank[huffNode[n].nbBits]++;
+ /* determine starting value per rank */
+ { U16 min = 0;
+ for (n=(int)maxNbBits; n>0; n--) {
+ valPerRank[n] = min; /* get starting value within each rank */
+ min += nbPerRank[n];
+ min >>= 1;
+ } }
+ for (n=0; n<alphabetSize; n++)
+ HUF_setNbBits(ct + huffNode[n].byte, huffNode[n].nbBits); /* push nbBits per symbol, symbol order */
+ for (n=0; n<alphabetSize; n++)
+ HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); /* assign value within rank, symbol order */
+ CTable[0] = maxNbBits;
+}
+
+size_t
+HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
+ void* workSpace, size_t wkspSize)
+{
+ HUF_buildCTable_wksp_tables* const wksp_tables =
+ (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32));
+ nodeElt* const huffNode0 = wksp_tables->huffNodeTbl;
+ nodeElt* const huffNode = huffNode0+1;
+ int nonNullRank;
+
+ HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables));
+
+ DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1);
+
+ /* safety checks */
+ if (wkspSize < sizeof(HUF_buildCTable_wksp_tables))
+ return ERROR(workSpace_tooSmall);
+ if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
+ return ERROR(maxSymbolValue_tooLarge);
+ ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable));
+
+ /* sort, decreasing order */
+ HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition);
+ DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1));
+
+ /* build tree */
+ nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
+
+ /* determine and enforce maxTableLog */
+ maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits);
+ if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
+
+ HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits);
+
+ return maxNbBits;
+}
+
+size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
+{
+ HUF_CElt const* ct = CTable + 1;
+ size_t nbBits = 0;
+ int s;
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
+ nbBits += HUF_getNbBits(ct[s]) * count[s];
+ }
+ return nbBits >> 3;
+}
+
+int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
+ HUF_CElt const* ct = CTable + 1;
+ int bad = 0;
+ int s;
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
+ bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0);
+ }
+ return !bad;
+}
+
+size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
+
+/** HUF_CStream_t:
+ * Huffman uses its own BIT_CStream_t implementation.
+ * There are three major differences from BIT_CStream_t:
+ * 1. HUF_addBits() takes a HUF_CElt (size_t) which is
+ * the pair (nbBits, value) in the format:
+ * format:
+ * - Bits [0, 4) = nbBits
+ * - Bits [4, 64 - nbBits) = 0
+ * - Bits [64 - nbBits, 64) = value
+ * 2. The bitContainer is built from the upper bits and
+ * right shifted. E.g. to add a new value of N bits
+ * you right shift the bitContainer by N, then or in
+ * the new value into the N upper bits.
+ * 3. The bitstream has two bit containers. You can add
+ * bits to the second container and merge them into
+ * the first container.
+ */
+
+#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8)
+
+typedef struct {
+ size_t bitContainer[2];
+ size_t bitPos[2];
+
+ BYTE* startPtr;
+ BYTE* ptr;
+ BYTE* endPtr;
+} HUF_CStream_t;
+
+/**! HUF_initCStream():
+ * Initializes the bitstream.
+ * @returns 0 or an error code.
+ */
+static size_t HUF_initCStream(HUF_CStream_t* bitC,
+ void* startPtr, size_t dstCapacity)
+{
+ ZSTD_memset(bitC, 0, sizeof(*bitC));
+ bitC->startPtr = (BYTE*)startPtr;
+ bitC->ptr = bitC->startPtr;
+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]);
+ if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall);
+ return 0;
+}
+
+/*! HUF_addBits():
+ * Adds the symbol stored in HUF_CElt elt to the bitstream.
+ *
+ * @param elt The element we're adding. This is a (nbBits, value) pair.
+ * See the HUF_CStream_t docs for the format.
+ * @param idx Insert into the bitstream at this idx.
+ * @param kFast This is a template parameter. If the bitstream is guaranteed
+ * to have at least 4 unused bits after this call it may be 1,
+ * otherwise it must be 0. HUF_addBits() is faster when fast is set.
+ */
+FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast)
+{
+ assert(idx <= 1);
+ assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX);
+ /* This is efficient on x86-64 with BMI2 because shrx
+ * only reads the low 6 bits of the register. The compiler
+ * knows this and elides the mask. When fast is set,
+ * every operation can use the same value loaded from elt.
+ */
+ bitC->bitContainer[idx] >>= HUF_getNbBits(elt);
+ bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt);
+ /* We only read the low 8 bits of bitC->bitPos[idx] so it
+ * doesn't matter that the high bits have noise from the value.
+ */
+ bitC->bitPos[idx] += HUF_getNbBitsFast(elt);
+ assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER);
+ /* The last 4-bits of elt are dirty if fast is set,
+ * so we must not be overwriting bits that have already been
+ * inserted into the bit container.
+ */
+#if DEBUGLEVEL >= 1
+ {
+ size_t const nbBits = HUF_getNbBits(elt);
+ size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1;
+ (void)dirtyBits;
+ /* Middle bits are 0. */
+ assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0);
+ /* We didn't overwrite any bits in the bit container. */
+ assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER);
+ (void)dirtyBits;
+ }
+#endif
+}
+
+FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC)
+{
+ bitC->bitContainer[1] = 0;
+ bitC->bitPos[1] = 0;
+}
+
+/*! HUF_mergeIndex1() :
+ * Merges the bit container @ index 1 into the bit container @ index 0
+ * and zeros the bit container @ index 1.
+ */
+FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC)
+{
+ assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER);
+ bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF);
+ bitC->bitContainer[0] |= bitC->bitContainer[1];
+ bitC->bitPos[0] += bitC->bitPos[1];
+ assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER);
+}
+
+/*! HUF_flushBits() :
+* Flushes the bits in the bit container @ index 0.
+*
+* @post bitPos will be < 8.
+* @param kFast If kFast is set then we must know a-priori that
+* the bit container will not overflow.
+*/
+FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast)
+{
+ /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */
+ size_t const nbBits = bitC->bitPos[0] & 0xFF;
+ size_t const nbBytes = nbBits >> 3;
+ /* The top nbBits bits of bitContainer are the ones we need. */
+ size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits);
+ /* Mask bitPos to account for the bytes we consumed. */
+ bitC->bitPos[0] &= 7;
+ assert(nbBits > 0);
+ assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8);
+ assert(bitC->ptr <= bitC->endPtr);
+ MEM_writeLEST(bitC->ptr, bitContainer);
+ bitC->ptr += nbBytes;
+ assert(!kFast || bitC->ptr <= bitC->endPtr);
+ if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
+ /* bitContainer doesn't need to be modified because the leftover
+ * bits are already the top bitPos bits. And we don't care about
+ * noise in the lower values.
+ */
+}
+
+/*! HUF_endMark()
+ * @returns The Huffman stream end mark: A 1-bit value = 1.
+ */
+static HUF_CElt HUF_endMark(void)
+{
+ HUF_CElt endMark;
+ HUF_setNbBits(&endMark, 1);
+ HUF_setValue(&endMark, 1);
+ return endMark;
+}
+
+/*! HUF_closeCStream() :
+ * @return Size of CStream, in bytes,
+ * or 0 if it could not fit into dstBuffer */
+static size_t HUF_closeCStream(HUF_CStream_t* bitC)
+{
+ HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0);
+ HUF_flushBits(bitC, /* kFast */ 0);
+ {
+ size_t const nbBits = bitC->bitPos[0] & 0xFF;
+ if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
+ return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0);
+ }
+}
+
+FORCE_INLINE_TEMPLATE void
+HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast)
+{
+ HUF_addBits(bitCPtr, CTable[symbol], idx, fast);
+}
+
+FORCE_INLINE_TEMPLATE void
+HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC,
+ const BYTE* ip, size_t srcSize,
+ const HUF_CElt* ct,
+ int kUnroll, int kFastFlush, int kLastFast)
+{
+ /* Join to kUnroll */
+ int n = (int)srcSize;
+ int rem = n % kUnroll;
+ if (rem > 0) {
+ for (; rem > 0; --rem) {
+ HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0);
+ }
+ HUF_flushBits(bitC, kFastFlush);
+ }
+ assert(n % kUnroll == 0);
+
+ /* Join to 2 * kUnroll */
+ if (n % (2 * kUnroll)) {
+ int u;
+ for (u = 1; u < kUnroll; ++u) {
+ HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1);
+ }
+ HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast);
+ HUF_flushBits(bitC, kFastFlush);
+ n -= kUnroll;
+ }
+ assert(n % (2 * kUnroll) == 0);
+
+ for (; n>0; n-= 2 * kUnroll) {
+ /* Encode kUnroll symbols into the bitstream @ index 0. */
+ int u;
+ for (u = 1; u < kUnroll; ++u) {
+ HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1);
+ }
+ HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast);
+ HUF_flushBits(bitC, kFastFlush);
+ /* Encode kUnroll symbols into the bitstream @ index 1.
+ * This allows us to start filling the bit container
+ * without any data dependencies.
+ */
+ HUF_zeroIndex1(bitC);
+ for (u = 1; u < kUnroll; ++u) {
+ HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1);
+ }
+ HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast);
+ /* Merge bitstream @ index 1 into the bitstream @ index 0 */
+ HUF_mergeIndex1(bitC);
+ HUF_flushBits(bitC, kFastFlush);
+ }
+ assert(n == 0);
+
+}
+
+/**
+ * Returns a tight upper bound on the output space needed by Huffman
+ * with 8 bytes buffer to handle over-writes. If the output is at least
+ * this large we don't need to do bounds checks during Huffman encoding.
+ */
+static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog)
+{
+ return ((srcSize * tableLog) >> 3) + 8;
+}
+
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable)
+{
+ U32 const tableLog = (U32)CTable[0];
+ HUF_CElt const* ct = CTable + 1;
+ const BYTE* ip = (const BYTE*) src;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart;
+ HUF_CStream_t bitC;
+
+ /* init */
+ if (dstSize < 8) return 0; /* not enough space to compress */
+ { size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op));
+ if (HUF_isError(initErr)) return 0; }
+
+ if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11)
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0);
+ else {
+ if (MEM_32bits()) {
+ switch (tableLog) {
+ case 11:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 10: ZSTD_FALLTHROUGH;
+ case 9: ZSTD_FALLTHROUGH;
+ case 8:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ case 7: ZSTD_FALLTHROUGH;
+ default:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ }
+ } else {
+ switch (tableLog) {
+ case 11:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 10:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ case 9:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 8:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 7:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 6: ZSTD_FALLTHROUGH;
+ default:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ }
+ }
+ }
+ assert(bitC.ptr <= bitC.endPtr);
+
+ return HUF_closeCStream(&bitC);
+}
+
+#if DYNAMIC_BMI2
+
+static BMI2_TARGET_ATTRIBUTE size_t
+HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable)
+{
+ return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
+}
+
+static size_t
+HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable)
+{
+ return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
+}
+
+static size_t
+HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable, const int flags)
+{
+ if (flags & HUF_flags_bmi2) {
+ return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
+ }
+ return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
+}
+
+#else
+
+static size_t
+HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable, const int flags)
+{
+ (void)flags;
+ return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
+}
+
+#endif
+
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
+{
+ return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
+}
+
+static size_t
+HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable, int flags)
+{
+ size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
+ const BYTE* ip = (const BYTE*) src;
+ const BYTE* const iend = ip + srcSize;
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart;
+
+ if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
+ if (srcSize < 12) return 0; /* no saving possible : too small input */
+ op += 6; /* jumpTable */
+
+ assert(op <= oend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ MEM_writeLE16(ostart, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ assert(op <= oend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ MEM_writeLE16(ostart+2, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ assert(op <= oend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ MEM_writeLE16(ostart+4, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ assert(op <= oend);
+ assert(ip <= iend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ op += cSize;
+ }
+
+ return (size_t)(op-ostart);
+}
+
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
+{
+ return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
+}
+
+typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
+
+static size_t HUF_compressCTable_internal(
+ BYTE* const ostart, BYTE* op, BYTE* const oend,
+ const void* src, size_t srcSize,
+ HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags)
+{
+ size_t const cSize = (nbStreams==HUF_singleStream) ?
+ HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) :
+ HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags);
+ if (HUF_isError(cSize)) { return cSize; }
+ if (cSize==0) { return 0; } /* uncompressible */
+ op += cSize;
+ /* check compressibility */
+ assert(op >= ostart);
+ if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
+ return (size_t)(op-ostart);
+}
+
+typedef struct {
+ unsigned count[HUF_SYMBOLVALUE_MAX + 1];
+ HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)];
+ union {
+ HUF_buildCTable_wksp_tables buildCTable_wksp;
+ HUF_WriteCTableWksp writeCTable_wksp;
+ U32 hist_wksp[HIST_WKSP_SIZE_U32];
+ } wksps;
+} HUF_compress_tables_t;
+
+#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096
+#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */
+
+unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue)
+{
+ unsigned cardinality = 0;
+ unsigned i;
+
+ for (i = 0; i < maxSymbolValue + 1; i++) {
+ if (count[i] != 0) cardinality += 1;
+ }
+
+ return cardinality;
+}
+
+unsigned HUF_minTableLog(unsigned symbolCardinality)
+{
+ U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1;
+ return minBitsSymbols;
+}
+
+unsigned HUF_optimalTableLog(
+ unsigned maxTableLog,
+ size_t srcSize,
+ unsigned maxSymbolValue,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* table,
+ const unsigned* count,
+ int flags)
+{
+ assert(srcSize > 1); /* Not supported, RLE should be used instead */
+ assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
+
+ if (!(flags & HUF_flags_optimalDepth)) {
+ /* cheap evaluation, based on FSE */
+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+ }
+
+ { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+ size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
+ size_t maxBits, hSize, newSize;
+ const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
+ const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
+ size_t optSize = ((size_t) ~0) - 1;
+ unsigned optLog = maxTableLog, optLogGuess;
+
+ DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
+
+ /* Search until size increases */
+ for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+ DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
+ maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
+ if (ERR_isError(maxBits)) continue;
+
+ if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
+
+ hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
+
+ if (ERR_isError(hSize)) continue;
+
+ newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+
+ if (newSize > optSize + 1) {
+ break;
+ }
+
+ if (newSize < optSize) {
+ optSize = newSize;
+ optLog = optLogGuess;
+ }
+ }
+ assert(optLog <= HUF_TABLELOG_MAX);
+ return optLog;
+ }
+}
+
+/* HUF_compress_internal() :
+ * `workSpace_align4` must be aligned on 4-bytes boundaries,
+ * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */
+static size_t
+HUF_compress_internal (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ HUF_nbStreams_e nbStreams,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags)
+{
+ HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart;
+
+ DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize);
+ HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE);
+
+ /* checks & inits */
+ if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall);
+ if (!srcSize) return 0; /* Uncompressed */
+ if (!dstSize) return 0; /* cannot fit anything within dst budget */
+ if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
+ if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
+ if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
+ if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
+
+ /* Heuristic : If old table is valid, use it for small inputs */
+ if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) {
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, oldHufTable, flags);
+ }
+
+ /* If uncompressible data is suspected, do a smaller sampling first */
+ DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2);
+ if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
+ size_t largestTotal = 0;
+ DEBUGLOG(5, "input suspected incompressible : sampling to check");
+ { unsigned maxSymbolValueBegin = maxSymbolValue;
+ CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) );
+ largestTotal += largestBegin;
+ }
+ { unsigned maxSymbolValueEnd = maxSymbolValue;
+ CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) );
+ largestTotal += largestEnd;
+ }
+ if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */
+ }
+
+ /* Scan input and build symbol stats */
+ { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) );
+ if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
+ if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
+ }
+ DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1));
+
+ /* Check validity of previous table */
+ if ( repeat
+ && *repeat == HUF_repeat_check
+ && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) {
+ *repeat = HUF_repeat_none;
+ }
+ /* Heuristic : use existing table for small inputs */
+ if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) {
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, oldHufTable, flags);
+ }
+
+ /* Build Huffman Tree */
+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags);
+ { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
+ maxSymbolValue, huffLog,
+ &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
+ CHECK_F(maxBits);
+ huffLog = (U32)maxBits;
+ DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1));
+ }
+ /* Zero unused symbols in CTable, so we can check it for validity */
+ {
+ size_t const ctableSize = HUF_CTABLE_SIZE_ST(maxSymbolValue);
+ size_t const unusedSize = sizeof(table->CTable) - ctableSize * sizeof(HUF_CElt);
+ ZSTD_memset(table->CTable + ctableSize, 0, unusedSize);
+ }
+
+ /* Write table description header */
+ { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog,
+ &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) );
+ /* Check if using previous huffman table is beneficial */
+ if (repeat && *repeat != HUF_repeat_none) {
+ size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue);
+ size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue);
+ if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, oldHufTable, flags);
+ } }
+
+ /* Use the new huffman table */
+ if (hSize + 12ul >= srcSize) { return 0; }
+ op += hSize;
+ if (repeat) { *repeat = HUF_repeat_none; }
+ if (oldHufTable)
+ ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */
+ }
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, table->CTable, flags);
+}
+
+size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
+{
+ DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
+ return HUF_compress_internal(dst, dstSize, src, srcSize,
+ maxSymbolValue, huffLog, HUF_singleStream,
+ workSpace, wkspSize, hufTable,
+ repeat, flags);
+}
+
+/* HUF_compress4X_repeat():
+ * compress input using 4 streams.
+ * consider skipping quickly
+ * re-use an existing huffman compression table */
+size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
+{
+ DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
+ return HUF_compress_internal(dst, dstSize, src, srcSize,
+ maxSymbolValue, huffLog, HUF_fourStreams,
+ workSpace, wkspSize,
+ hufTable, repeat, flags);
+}
diff --git a/contrib/zstd/huf_decompress.c b/contrib/zstd/huf_decompress.c
new file mode 100644
index 0000000..ea72fe4
--- /dev/null
+++ b/contrib/zstd/huf_decompress.c
@@ -0,0 +1,1882 @@
+/* ******************************************************************
+ * huff0 huffman decoder,
+ * part of Finite State Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* **************************************************************
+* Dependencies
+****************************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
+#include "compiler.h"
+#include "bitstream.h" /* BIT_* */
+#include "fse.h" /* to compress headers */
+#include "huf.h"
+#include "error_private.h"
+#include "zstd_internal.h"
+#include "bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */
+
+/* **************************************************************
+* Constants
+****************************************************************/
+
+#define HUF_DECODER_FAST_TABLELOG 11
+
+/* **************************************************************
+* Macros
+****************************************************************/
+
+/* These two optional macros force the use one way or another of the two
+ * Huffman decompression implementations. You can't force in both directions
+ * at the same time.
+ */
+#if defined(HUF_FORCE_DECOMPRESS_X1) && \
+ defined(HUF_FORCE_DECOMPRESS_X2)
+#error "Cannot force the use of the X1 and X2 decoders at the same time!"
+#endif
+
+/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is
+ * supported at runtime, so we can add the BMI2 target attribute.
+ * When it is disabled, we will still get BMI2 if it is enabled statically.
+ */
+#if DYNAMIC_BMI2
+# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
+#else
+# define HUF_FAST_BMI2_ATTRS
+#endif
+
+#ifdef __cplusplus
+# define HUF_EXTERN_C extern "C"
+#else
+# define HUF_EXTERN_C
+#endif
+#define HUF_ASM_DECL HUF_EXTERN_C
+
+#if DYNAMIC_BMI2
+# define HUF_NEED_BMI2_FUNCTION 1
+#else
+# define HUF_NEED_BMI2_FUNCTION 0
+#endif
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define HUF_isError ERR_isError
+
+
+/* **************************************************************
+* Byte alignment for workSpace management
+****************************************************************/
+#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)
+#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
+
+
+/* **************************************************************
+* BMI2 Variant Wrappers
+****************************************************************/
+typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize,
+ const void *cSrc,
+ size_t cSrcSize,
+ const HUF_DTable *DTable);
+
+#if DYNAMIC_BMI2
+
+#define HUF_DGEN(fn) \
+ \
+ static size_t fn##_default( \
+ void* dst, size_t dstSize, \
+ const void* cSrc, size_t cSrcSize, \
+ const HUF_DTable* DTable) \
+ { \
+ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
+ } \
+ \
+ static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \
+ void* dst, size_t dstSize, \
+ const void* cSrc, size_t cSrcSize, \
+ const HUF_DTable* DTable) \
+ { \
+ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
+ } \
+ \
+ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
+ size_t cSrcSize, HUF_DTable const* DTable, int flags) \
+ { \
+ if (flags & HUF_flags_bmi2) { \
+ return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
+ } \
+ return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
+ }
+
+#else
+
+#define HUF_DGEN(fn) \
+ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
+ size_t cSrcSize, HUF_DTable const* DTable, int flags) \
+ { \
+ (void)flags; \
+ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
+ }
+
+#endif
+
+
+/*-***************************/
+/* generic DTableDesc */
+/*-***************************/
+typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
+
+static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
+{
+ DTableDesc dtd;
+ ZSTD_memcpy(&dtd, table, sizeof(dtd));
+ return dtd;
+}
+
+static size_t HUF_initFastDStream(BYTE const* ip) {
+ BYTE const lastByte = ip[7];
+ size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
+ size_t const value = MEM_readLEST(ip) | 1;
+ assert(bitsConsumed <= 8);
+ assert(sizeof(size_t) == 8);
+ return value << bitsConsumed;
+}
+
+
+/**
+ * The input/output arguments to the Huffman fast decoding loop:
+ *
+ * ip [in/out] - The input pointers, must be updated to reflect what is consumed.
+ * op [in/out] - The output pointers, must be updated to reflect what is written.
+ * bits [in/out] - The bitstream containers, must be updated to reflect the current state.
+ * dt [in] - The decoding table.
+ * ilimit [in] - The input limit, stop when any input pointer is below ilimit.
+ * oend [in] - The end of the output stream. op[3] must not cross oend.
+ * iend [in] - The end of each input stream. ip[i] may cross iend[i],
+ * as long as it is above ilimit, but that indicates corruption.
+ */
+typedef struct {
+ BYTE const* ip[4];
+ BYTE* op[4];
+ U64 bits[4];
+ void const* dt;
+ BYTE const* ilimit;
+ BYTE* oend;
+ BYTE const* iend[4];
+} HUF_DecompressFastArgs;
+
+typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*);
+
+/**
+ * Initializes args for the fast decoding loop.
+ * @returns 1 on success
+ * 0 if the fallback implementation should be used.
+ * Or an error code on failure.
+ */
+static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
+{
+ void const* dt = DTable + 1;
+ U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
+
+ const BYTE* const ilimit = (const BYTE*)src + 6 + 8;
+
+ BYTE* const oend = (BYTE*)dst + dstSize;
+
+ /* The fast decoding loop assumes 64-bit little-endian.
+ * This condition is false on x32.
+ */
+ if (!MEM_isLittleEndian() || MEM_32bits())
+ return 0;
+
+ /* strict minimum : jump table + 1 byte per stream */
+ if (srcSize < 10)
+ return ERROR(corruption_detected);
+
+ /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers.
+ * If table log is not correct at this point, fallback to the old decoder.
+ * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
+ */
+ if (dtLog != HUF_DECODER_FAST_TABLELOG)
+ return 0;
+
+ /* Read the jump table. */
+ {
+ const BYTE* const istart = (const BYTE*)src;
+ size_t const length1 = MEM_readLE16(istart);
+ size_t const length2 = MEM_readLE16(istart+2);
+ size_t const length3 = MEM_readLE16(istart+4);
+ size_t const length4 = srcSize - (length1 + length2 + length3 + 6);
+ args->iend[0] = istart + 6; /* jumpTable */
+ args->iend[1] = args->iend[0] + length1;
+ args->iend[2] = args->iend[1] + length2;
+ args->iend[3] = args->iend[2] + length3;
+
+ /* HUF_initFastDStream() requires this, and this small of an input
+ * won't benefit from the ASM loop anyways.
+ * length1 must be >= 16 so that ip[0] >= ilimit before the loop
+ * starts.
+ */
+ if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8)
+ return 0;
+ if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */
+ }
+ /* ip[] contains the position that is currently loaded into bits[]. */
+ args->ip[0] = args->iend[1] - sizeof(U64);
+ args->ip[1] = args->iend[2] - sizeof(U64);
+ args->ip[2] = args->iend[3] - sizeof(U64);
+ args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64);
+
+ /* op[] contains the output pointers. */
+ args->op[0] = (BYTE*)dst;
+ args->op[1] = args->op[0] + (dstSize+3)/4;
+ args->op[2] = args->op[1] + (dstSize+3)/4;
+ args->op[3] = args->op[2] + (dstSize+3)/4;
+
+ /* No point to call the ASM loop for tiny outputs. */
+ if (args->op[3] >= oend)
+ return 0;
+
+ /* bits[] is the bit container.
+ * It is read from the MSB down to the LSB.
+ * It is shifted left as it is read, and zeros are
+ * shifted in. After the lowest valid bit a 1 is
+ * set, so that CountTrailingZeros(bits[]) can be used
+ * to count how many bits we've consumed.
+ */
+ args->bits[0] = HUF_initFastDStream(args->ip[0]);
+ args->bits[1] = HUF_initFastDStream(args->ip[1]);
+ args->bits[2] = HUF_initFastDStream(args->ip[2]);
+ args->bits[3] = HUF_initFastDStream(args->ip[3]);
+
+ /* If ip[] >= ilimit, it is guaranteed to be safe to
+ * reload bits[]. It may be beyond its section, but is
+ * guaranteed to be valid (>= istart).
+ */
+ args->ilimit = ilimit;
+
+ args->oend = oend;
+ args->dt = dt;
+
+ return 1;
+}
+
+static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd)
+{
+ /* Validate that we haven't overwritten. */
+ if (args->op[stream] > segmentEnd)
+ return ERROR(corruption_detected);
+ /* Validate that we haven't read beyond iend[].
+ * Note that ip[] may be < iend[] because the MSB is
+ * the next bit to read, and we may have consumed 100%
+ * of the stream, so down to iend[i] - 8 is valid.
+ */
+ if (args->ip[stream] < args->iend[stream] - 8)
+ return ERROR(corruption_detected);
+
+ /* Construct the BIT_DStream_t. */
+ assert(sizeof(size_t) == 8);
+ bit->bitContainer = MEM_readLEST(args->ip[stream]);
+ bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]);
+ bit->start = (const char*)args->iend[0];
+ bit->limitPtr = bit->start + sizeof(size_t);
+ bit->ptr = (const char*)args->ip[stream];
+
+ return 0;
+}
+
+
+#ifndef HUF_FORCE_DECOMPRESS_X2
+
+/*-***************************/
+/* single-symbol decoding */
+/*-***************************/
+typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */
+
+/**
+ * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at
+ * a time.
+ */
+static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
+ U64 D4;
+ if (MEM_isLittleEndian()) {
+ D4 = (U64)((symbol << 8) + nbBits);
+ } else {
+ D4 = (U64)(symbol + (nbBits << 8));
+ }
+ assert(D4 < (1U << 16));
+ D4 *= 0x0001000100010001ULL;
+ return D4;
+}
+
+/**
+ * Increase the tableLog to targetTableLog and rescales the stats.
+ * If tableLog > targetTableLog this is a no-op.
+ * @returns New tableLog
+ */
+static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog)
+{
+ if (tableLog > targetTableLog)
+ return tableLog;
+ if (tableLog < targetTableLog) {
+ U32 const scale = targetTableLog - tableLog;
+ U32 s;
+ /* Increase the weight for all non-zero probability symbols by scale. */
+ for (s = 0; s < nbSymbols; ++s) {
+ huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale);
+ }
+ /* Update rankVal to reflect the new weights.
+ * All weights except 0 get moved to weight + scale.
+ * Weights [1, scale] are empty.
+ */
+ for (s = targetTableLog; s > scale; --s) {
+ rankVal[s] = rankVal[s - scale];
+ }
+ for (s = scale; s > 0; --s) {
+ rankVal[s] = 0;
+ }
+ }
+ return targetTableLog;
+}
+
+typedef struct {
+ U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
+ U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1];
+ U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+ BYTE symbols[HUF_SYMBOLVALUE_MAX + 1];
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
+} HUF_ReadDTableX1_Workspace;
+
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags)
+{
+ U32 tableLog = 0;
+ U32 nbSymbols = 0;
+ size_t iSize;
+ void* const dtPtr = DTable + 1;
+ HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr;
+ HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace;
+
+ DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp));
+ if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge);
+
+ DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
+ /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
+
+ iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags);
+ if (HUF_isError(iSize)) return iSize;
+
+
+ /* Table header */
+ { DTableDesc dtd = HUF_getDTableDesc(DTable);
+ U32 const maxTableLog = dtd.maxTableLog + 1;
+ U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG);
+ tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog);
+ if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
+ dtd.tableType = 0;
+ dtd.tableLog = (BYTE)tableLog;
+ ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
+ }
+
+ /* Compute symbols and rankStart given rankVal:
+ *
+ * rankVal already contains the number of values of each weight.
+ *
+ * symbols contains the symbols ordered by weight. First are the rankVal[0]
+ * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on.
+ * symbols[0] is filled (but unused) to avoid a branch.
+ *
+ * rankStart contains the offset where each rank belongs in the DTable.
+ * rankStart[0] is not filled because there are no entries in the table for
+ * weight 0.
+ */
+ { int n;
+ U32 nextRankStart = 0;
+ int const unroll = 4;
+ int const nLimit = (int)nbSymbols - unroll + 1;
+ for (n=0; n<(int)tableLog+1; n++) {
+ U32 const curr = nextRankStart;
+ nextRankStart += wksp->rankVal[n];
+ wksp->rankStart[n] = curr;
+ }
+ for (n=0; n < nLimit; n += unroll) {
+ int u;
+ for (u=0; u < unroll; ++u) {
+ size_t const w = wksp->huffWeight[n+u];
+ wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u);
+ }
+ }
+ for (; n < (int)nbSymbols; ++n) {
+ size_t const w = wksp->huffWeight[n];
+ wksp->symbols[wksp->rankStart[w]++] = (BYTE)n;
+ }
+ }
+
+ /* fill DTable
+ * We fill all entries of each weight in order.
+ * That way length is a constant for each iteration of the outer loop.
+ * We can switch based on the length to a different inner loop which is
+ * optimized for that particular case.
+ */
+ { U32 w;
+ int symbol = wksp->rankVal[0];
+ int rankStart = 0;
+ for (w=1; w<tableLog+1; ++w) {
+ int const symbolCount = wksp->rankVal[w];
+ int const length = (1 << w) >> 1;
+ int uStart = rankStart;
+ BYTE const nbBits = (BYTE)(tableLog + 1 - w);
+ int s;
+ int u;
+ switch (length) {
+ case 1:
+ for (s=0; s<symbolCount; ++s) {
+ HUF_DEltX1 D;
+ D.byte = wksp->symbols[symbol + s];
+ D.nbBits = nbBits;
+ dt[uStart] = D;
+ uStart += 1;
+ }
+ break;
+ case 2:
+ for (s=0; s<symbolCount; ++s) {
+ HUF_DEltX1 D;
+ D.byte = wksp->symbols[symbol + s];
+ D.nbBits = nbBits;
+ dt[uStart+0] = D;
+ dt[uStart+1] = D;
+ uStart += 2;
+ }
+ break;
+ case 4:
+ for (s=0; s<symbolCount; ++s) {
+ U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+ MEM_write64(dt + uStart, D4);
+ uStart += 4;
+ }
+ break;
+ case 8:
+ for (s=0; s<symbolCount; ++s) {
+ U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+ MEM_write64(dt + uStart, D4);
+ MEM_write64(dt + uStart + 4, D4);
+ uStart += 8;
+ }
+ break;
+ default:
+ for (s=0; s<symbolCount; ++s) {
+ U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+ for (u=0; u < length; u += 16) {
+ MEM_write64(dt + uStart + u + 0, D4);
+ MEM_write64(dt + uStart + u + 4, D4);
+ MEM_write64(dt + uStart + u + 8, D4);
+ MEM_write64(dt + uStart + u + 12, D4);
+ }
+ assert(u == length);
+ uStart += length;
+ }
+ break;
+ }
+ symbol += symbolCount;
+ rankStart += symbolCount * length;
+ }
+ }
+ return iSize;
+}
+
+FORCE_INLINE_TEMPLATE BYTE
+HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
+ BYTE const c = dt[val].byte;
+ BIT_skipBits(Dstream, dt[val].nbBits);
+ return c;
+}
+
+#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \
+ *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \
+ if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
+ HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
+
+#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \
+ if (MEM_64bits()) \
+ HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
+
+HINT_INLINE size_t
+HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog)
+{
+ BYTE* const pStart = p;
+
+ /* up to 4 symbols at a time */
+ if ((pEnd - p) > 3) {
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) {
+ HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX1_1(p, bitDPtr);
+ HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+ }
+ } else {
+ BIT_reloadDStream(bitDPtr);
+ }
+
+ /* [0-3] symbols remaining */
+ if (MEM_32bits())
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd))
+ HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+
+ /* no more data to retrieve from bitstream, no need to reload */
+ while (p < pEnd)
+ HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+
+ return (size_t)(pEnd-pStart);
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress1X1_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + dstSize;
+ const void* dtPtr = DTable + 1;
+ const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
+ BIT_DStream_t bitD;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+
+ CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
+
+ HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog);
+
+ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
+
+ return dstSize;
+}
+
+/* HUF_decompress4X1_usingDTable_internal_body():
+ * Conditions :
+ * @dstSize >= 6
+ */
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress4X1_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ /* Check */
+ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
+
+ { const BYTE* const istart = (const BYTE*) cSrc;
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* const olimit = oend - 3;
+ const void* const dtPtr = DTable + 1;
+ const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
+
+ /* Init */
+ BIT_DStream_t bitD1;
+ BIT_DStream_t bitD2;
+ BIT_DStream_t bitD3;
+ BIT_DStream_t bitD4;
+ size_t const length1 = MEM_readLE16(istart);
+ size_t const length2 = MEM_readLE16(istart+2);
+ size_t const length3 = MEM_readLE16(istart+4);
+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+ const BYTE* const istart1 = istart + 6; /* jumpTable */
+ const BYTE* const istart2 = istart1 + length1;
+ const BYTE* const istart3 = istart2 + length2;
+ const BYTE* const istart4 = istart3 + length3;
+ const size_t segmentSize = (dstSize+3) / 4;
+ BYTE* const opStart2 = ostart + segmentSize;
+ BYTE* const opStart3 = opStart2 + segmentSize;
+ BYTE* const opStart4 = opStart3 + segmentSize;
+ BYTE* op1 = ostart;
+ BYTE* op2 = opStart2;
+ BYTE* op3 = opStart3;
+ BYTE* op4 = opStart4;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+ U32 endSignal = 1;
+
+ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
+ if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
+ if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */
+ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
+ CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
+ CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
+ CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
+
+ /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */
+ if ((size_t)(oend - op4) >= sizeof(size_t)) {
+ for ( ; (endSignal) & (op4 < olimit) ; ) {
+ HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX1_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX1_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_0(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_0(op4, &bitD4);
+ endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
+ }
+ }
+
+ /* check corruption */
+ /* note : should not be necessary : op# advance in lock step, and we control op4.
+ * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */
+ if (op1 > opStart2) return ERROR(corruption_detected);
+ if (op2 > opStart3) return ERROR(corruption_detected);
+ if (op3 > opStart4) return ERROR(corruption_detected);
+ /* note : op4 supposed already verified within main loop */
+
+ /* finish bitStreams one by one */
+ HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog);
+ HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog);
+ HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog);
+ HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog);
+
+ /* check */
+ { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+ if (!endCheck) return ERROR(corruption_detected); }
+
+ /* decoded size */
+ return dstSize;
+ }
+}
+
+#if HUF_NEED_BMI2_FUNCTION
+static BMI2_TARGET_ATTRIBUTE
+size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+#endif
+
+static
+size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2
+
+HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+ U64 bits[4];
+ BYTE const* ip[4];
+ BYTE* op[4];
+ U16 const* const dtable = (U16 const*)args->dt;
+ BYTE* const oend = args->oend;
+ BYTE const* const ilimit = args->ilimit;
+
+ /* Copy the arguments to local variables */
+ ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+ ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+ ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+ assert(MEM_isLittleEndian());
+ assert(!MEM_32bits());
+
+ for (;;) {
+ BYTE* olimit;
+ int stream;
+ int symbol;
+
+ /* Assert loop preconditions */
+#ifndef NDEBUG
+ for (stream = 0; stream < 4; ++stream) {
+ assert(op[stream] <= (stream == 3 ? oend : op[stream + 1]));
+ assert(ip[stream] >= ilimit);
+ }
+#endif
+ /* Compute olimit */
+ {
+ /* Each iteration produces 5 output symbols per stream */
+ size_t const oiters = (size_t)(oend - op[3]) / 5;
+ /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes
+ * per stream.
+ */
+ size_t const iiters = (size_t)(ip[0] - ilimit) / 7;
+ /* We can safely run iters iterations before running bounds checks */
+ size_t const iters = MIN(oiters, iiters);
+ size_t const symbols = iters * 5;
+
+ /* We can simply check that op[3] < olimit, instead of checking all
+ * of our bounds, since we can't hit the other bounds until we've run
+ * iters iterations, which only happens when op[3] == olimit.
+ */
+ olimit = op[3] + symbols;
+
+ /* Exit fast decoding loop once we get close to the end. */
+ if (op[3] + 20 > olimit)
+ break;
+
+ /* Exit the decoding loop if any input pointer has crossed the
+ * previous one. This indicates corruption, and a precondition
+ * to our loop is that ip[i] >= ip[0].
+ */
+ for (stream = 1; stream < 4; ++stream) {
+ if (ip[stream] < ip[stream - 1])
+ goto _out;
+ }
+ }
+
+#ifndef NDEBUG
+ for (stream = 1; stream < 4; ++stream) {
+ assert(ip[stream] >= ip[stream - 1]);
+ }
+#endif
+
+ do {
+ /* Decode 5 symbols in each of the 4 streams */
+ for (symbol = 0; symbol < 5; ++symbol) {
+ for (stream = 0; stream < 4; ++stream) {
+ int const index = (int)(bits[stream] >> 53);
+ int const entry = (int)dtable[index];
+ bits[stream] <<= (entry & 63);
+ op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF);
+ }
+ }
+ /* Reload the bitstreams */
+ for (stream = 0; stream < 4; ++stream) {
+ int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+ int const nbBits = ctz & 7;
+ int const nbBytes = ctz >> 3;
+ op[stream] += 5;
+ ip[stream] -= nbBytes;
+ bits[stream] = MEM_read64(ip[stream]) | 1;
+ bits[stream] <<= nbBits;
+ }
+ } while (op[3] < olimit);
+ }
+
+_out:
+
+ /* Save the final values of each of the state variables back to args. */
+ ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+ ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+ ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+/**
+ * @returns @p dstSize on success (>= 6)
+ * 0 if the fallback implementation should be used
+ * An error if an error occurred
+ */
+static HUF_FAST_BMI2_ATTRS
+size_t
+HUF_decompress4X1_usingDTable_internal_fast(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable,
+ HUF_DecompressFastLoopFn loopFn)
+{
+ void const* dt = DTable + 1;
+ const BYTE* const iend = (const BYTE*)cSrc + 6;
+ BYTE* const oend = (BYTE*)dst + dstSize;
+ HUF_DecompressFastArgs args;
+ { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+ FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
+ if (ret == 0)
+ return 0;
+ }
+
+ assert(args.ip[0] >= args.ilimit);
+ loopFn(&args);
+
+ /* Our loop guarantees that ip[] >= ilimit and that we haven't
+ * overwritten any op[].
+ */
+ assert(args.ip[0] >= iend);
+ assert(args.ip[1] >= iend);
+ assert(args.ip[2] >= iend);
+ assert(args.ip[3] >= iend);
+ assert(args.op[3] <= oend);
+ (void)iend;
+
+ /* finish bit streams one by one. */
+ { size_t const segmentSize = (dstSize+3) / 4;
+ BYTE* segmentEnd = (BYTE*)dst;
+ int i;
+ for (i = 0; i < 4; ++i) {
+ BIT_DStream_t bit;
+ if (segmentSize <= (size_t)(oend - segmentEnd))
+ segmentEnd += segmentSize;
+ else
+ segmentEnd = oend;
+ FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
+ /* Decompress and validate that we've produced exactly the expected length. */
+ args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG);
+ if (args.op[i] != segmentEnd) return ERROR(corruption_detected);
+ }
+ }
+
+ /* decoded size */
+ assert(dstSize != 0);
+ return dstSize;
+}
+
+HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
+
+static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable, int flags)
+{
+ HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default;
+ HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop;
+
+#if DYNAMIC_BMI2
+ if (flags & HUF_flags_bmi2) {
+ fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2;
+# if ZSTD_ENABLE_ASM_X86_64_BMI2
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+ }
+# endif
+ } else {
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+ }
+#endif
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+ }
+#endif
+
+ if (!(flags & HUF_flags_disableFast)) {
+ size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+ if (ret != 0)
+ return ret;
+ }
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+
+#endif /* HUF_FORCE_DECOMPRESS_X2 */
+
+
+#ifndef HUF_FORCE_DECOMPRESS_X1
+
+/* *************************/
+/* double-symbols decoding */
+/* *************************/
+
+typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */
+typedef struct { BYTE symbol; } sortedSymbol_t;
+typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
+typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
+
+/**
+ * Constructs a HUF_DEltX2 in a U32.
+ */
+static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level)
+{
+ U32 seq;
+ DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0);
+ DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2);
+ DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3);
+ DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32));
+ if (MEM_isLittleEndian()) {
+ seq = level == 1 ? symbol : (baseSeq + (symbol << 8));
+ return seq + (nbBits << 16) + ((U32)level << 24);
+ } else {
+ seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol);
+ return (seq << 16) + (nbBits << 8) + (U32)level;
+ }
+}
+
+/**
+ * Constructs a HUF_DEltX2.
+ */
+static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level)
+{
+ HUF_DEltX2 DElt;
+ U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
+ DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val));
+ ZSTD_memcpy(&DElt, &val, sizeof(val));
+ return DElt;
+}
+
+/**
+ * Constructs 2 HUF_DEltX2s and packs them into a U64.
+ */
+static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level)
+{
+ U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
+ return (U64)DElt + ((U64)DElt << 32);
+}
+
+/**
+ * Fills the DTable rank with all the symbols from [begin, end) that are each
+ * nbBits long.
+ *
+ * @param DTableRank The start of the rank in the DTable.
+ * @param begin The first symbol to fill (inclusive).
+ * @param end The last symbol to fill (exclusive).
+ * @param nbBits Each symbol is nbBits long.
+ * @param tableLog The table log.
+ * @param baseSeq If level == 1 { 0 } else { the first level symbol }
+ * @param level The level in the table. Must be 1 or 2.
+ */
+static void HUF_fillDTableX2ForWeight(
+ HUF_DEltX2* DTableRank,
+ sortedSymbol_t const* begin, sortedSymbol_t const* end,
+ U32 nbBits, U32 tableLog,
+ U16 baseSeq, int const level)
+{
+ U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */);
+ const sortedSymbol_t* ptr;
+ assert(level >= 1 && level <= 2);
+ switch (length) {
+ case 1:
+ for (ptr = begin; ptr != end; ++ptr) {
+ HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
+ *DTableRank++ = DElt;
+ }
+ break;
+ case 2:
+ for (ptr = begin; ptr != end; ++ptr) {
+ HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
+ DTableRank[0] = DElt;
+ DTableRank[1] = DElt;
+ DTableRank += 2;
+ }
+ break;
+ case 4:
+ for (ptr = begin; ptr != end; ++ptr) {
+ U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+ ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+ DTableRank += 4;
+ }
+ break;
+ case 8:
+ for (ptr = begin; ptr != end; ++ptr) {
+ U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+ ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
+ DTableRank += 8;
+ }
+ break;
+ default:
+ for (ptr = begin; ptr != end; ++ptr) {
+ U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+ HUF_DEltX2* const DTableRankEnd = DTableRank + length;
+ for (; DTableRank != DTableRankEnd; DTableRank += 8) {
+ ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
+ }
+ }
+ break;
+ }
+}
+
+/* HUF_fillDTableX2Level2() :
+ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
+static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits,
+ const U32* rankVal, const int minWeight, const int maxWeight1,
+ const sortedSymbol_t* sortedSymbols, U32 const* rankStart,
+ U32 nbBitsBaseline, U16 baseSeq)
+{
+ /* Fill skipped values (all positions up to rankVal[minWeight]).
+ * These are positions only get a single symbol because the combined weight
+ * is too large.
+ */
+ if (minWeight>1) {
+ U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */);
+ U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1);
+ int const skipSize = rankVal[minWeight];
+ assert(length > 1);
+ assert((U32)skipSize < length);
+ switch (length) {
+ case 2:
+ assert(skipSize == 1);
+ ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2));
+ break;
+ case 4:
+ assert(skipSize <= 4);
+ ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2));
+ break;
+ default:
+ {
+ int i;
+ for (i = 0; i < skipSize; i += 8) {
+ ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2));
+ }
+ }
+ }
+ }
+
+ /* Fill each of the second level symbols by weight. */
+ {
+ int w;
+ for (w = minWeight; w < maxWeight1; ++w) {
+ int const begin = rankStart[w];
+ int const end = rankStart[w+1];
+ U32 const nbBits = nbBitsBaseline - w;
+ U32 const totalBits = nbBits + consumedBits;
+ HUF_fillDTableX2ForWeight(
+ DTable + rankVal[w],
+ sortedSymbols + begin, sortedSymbols + end,
+ totalBits, targetLog,
+ baseSeq, /* level */ 2);
+ }
+ }
+}
+
+static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
+ const sortedSymbol_t* sortedList,
+ const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight,
+ const U32 nbBitsBaseline)
+{
+ U32* const rankVal = rankValOrigin[0];
+ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
+ const U32 minBits = nbBitsBaseline - maxWeight;
+ int w;
+ int const wEnd = (int)maxWeight + 1;
+
+ /* Fill DTable in order of weight. */
+ for (w = 1; w < wEnd; ++w) {
+ int const begin = (int)rankStart[w];
+ int const end = (int)rankStart[w+1];
+ U32 const nbBits = nbBitsBaseline - w;
+
+ if (targetLog-nbBits >= minBits) {
+ /* Enough room for a second symbol. */
+ int start = rankVal[w];
+ U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */);
+ int minWeight = nbBits + scaleLog;
+ int s;
+ if (minWeight < 1) minWeight = 1;
+ /* Fill the DTable for every symbol of weight w.
+ * These symbols get at least 1 second symbol.
+ */
+ for (s = begin; s != end; ++s) {
+ HUF_fillDTableX2Level2(
+ DTable + start, targetLog, nbBits,
+ rankValOrigin[nbBits], minWeight, wEnd,
+ sortedList, rankStart,
+ nbBitsBaseline, sortedList[s].symbol);
+ start += length;
+ }
+ } else {
+ /* Only a single symbol. */
+ HUF_fillDTableX2ForWeight(
+ DTable + rankVal[w],
+ sortedList + begin, sortedList + end,
+ nbBits, targetLog,
+ /* baseSeq */ 0, /* level */ 1);
+ }
+ }
+}
+
+typedef struct {
+ rankValCol_t rankVal[HUF_TABLELOG_MAX];
+ U32 rankStats[HUF_TABLELOG_MAX + 1];
+ U32 rankStart0[HUF_TABLELOG_MAX + 3];
+ sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
+ BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
+ U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+} HUF_ReadDTableX2_Workspace;
+
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ U32 tableLog, maxW, nbSymbols;
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ U32 maxTableLog = dtd.maxTableLog;
+ size_t iSize;
+ void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
+ HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
+ U32 *rankStart;
+
+ HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace;
+
+ if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC);
+
+ rankStart = wksp->rankStart0 + 1;
+ ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats));
+ ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0));
+
+ DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
+ if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
+ /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
+
+ iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags);
+ if (HUF_isError(iSize)) return iSize;
+
+ /* check result */
+ if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
+ if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG;
+
+ /* find maxWeight */
+ for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
+
+ /* Get start index of each weight */
+ { U32 w, nextRankStart = 0;
+ for (w=1; w<maxW+1; w++) {
+ U32 curr = nextRankStart;
+ nextRankStart += wksp->rankStats[w];
+ rankStart[w] = curr;
+ }
+ rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
+ rankStart[maxW+1] = nextRankStart;
+ }
+
+ /* sort symbols by weight */
+ { U32 s;
+ for (s=0; s<nbSymbols; s++) {
+ U32 const w = wksp->weightList[s];
+ U32 const r = rankStart[w]++;
+ wksp->sortedSymbol[r].symbol = (BYTE)s;
+ }
+ rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
+ }
+
+ /* Build rankVal */
+ { U32* const rankVal0 = wksp->rankVal[0];
+ { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
+ U32 nextRankVal = 0;
+ U32 w;
+ for (w=1; w<maxW+1; w++) {
+ U32 curr = nextRankVal;
+ nextRankVal += wksp->rankStats[w] << (w+rescale);
+ rankVal0[w] = curr;
+ } }
+ { U32 const minBits = tableLog+1 - maxW;
+ U32 consumed;
+ for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
+ U32* const rankValPtr = wksp->rankVal[consumed];
+ U32 w;
+ for (w = 1; w < maxW+1; w++) {
+ rankValPtr[w] = rankVal0[w] >> consumed;
+ } } } }
+
+ HUF_fillDTableX2(dt, maxTableLog,
+ wksp->sortedSymbol,
+ wksp->rankStart0, wksp->rankVal, maxW,
+ tableLog+1);
+
+ dtd.tableLog = (BYTE)maxTableLog;
+ dtd.tableType = 1;
+ ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
+ return iSize;
+}
+
+
+FORCE_INLINE_TEMPLATE U32
+HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
+ ZSTD_memcpy(op, &dt[val].sequence, 2);
+ BIT_skipBits(DStream, dt[val].nbBits);
+ return dt[val].length;
+}
+
+FORCE_INLINE_TEMPLATE U32
+HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
+ ZSTD_memcpy(op, &dt[val].sequence, 1);
+ if (dt[val].length==1) {
+ BIT_skipBits(DStream, dt[val].nbBits);
+ } else {
+ if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
+ BIT_skipBits(DStream, dt[val].nbBits);
+ if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
+ /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
+ DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
+ }
+ }
+ return 1;
+}
+
+#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
+ ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
+ if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
+ ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
+ if (MEM_64bits()) \
+ ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
+
+HINT_INLINE size_t
+HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd,
+ const HUF_DEltX2* const dt, const U32 dtLog)
+{
+ BYTE* const pStart = p;
+
+ /* up to 8 symbols at a time */
+ if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) {
+ if (dtLog <= 11 && MEM_64bits()) {
+ /* up to 10 symbols at a time */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) {
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ }
+ } else {
+ /* up to 8 symbols at a time */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ }
+ }
+ } else {
+ BIT_reloadDStream(bitDPtr);
+ }
+
+ /* closer to end : up to 2 symbols at a time */
+ if ((size_t)(pEnd - p) >= 2) {
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+
+ while (p <= pEnd-2)
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
+ }
+
+ if (p < pEnd)
+ p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog);
+
+ return p-pStart;
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress1X2_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ BIT_DStream_t bitD;
+
+ /* Init */
+ CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
+
+ /* decode */
+ { BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
+ const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog);
+ }
+
+ /* check */
+ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
+
+ /* decoded size */
+ return dstSize;
+}
+
+/* HUF_decompress4X2_usingDTable_internal_body():
+ * Conditions:
+ * @dstSize >= 6
+ */
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress4X2_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
+
+ { const BYTE* const istart = (const BYTE*) cSrc;
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* const olimit = oend - (sizeof(size_t)-1);
+ const void* const dtPtr = DTable+1;
+ const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
+
+ /* Init */
+ BIT_DStream_t bitD1;
+ BIT_DStream_t bitD2;
+ BIT_DStream_t bitD3;
+ BIT_DStream_t bitD4;
+ size_t const length1 = MEM_readLE16(istart);
+ size_t const length2 = MEM_readLE16(istart+2);
+ size_t const length3 = MEM_readLE16(istart+4);
+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+ const BYTE* const istart1 = istart + 6; /* jumpTable */
+ const BYTE* const istart2 = istart1 + length1;
+ const BYTE* const istart3 = istart2 + length2;
+ const BYTE* const istart4 = istart3 + length3;
+ size_t const segmentSize = (dstSize+3) / 4;
+ BYTE* const opStart2 = ostart + segmentSize;
+ BYTE* const opStart3 = opStart2 + segmentSize;
+ BYTE* const opStart4 = opStart3 + segmentSize;
+ BYTE* op1 = ostart;
+ BYTE* op2 = opStart2;
+ BYTE* op3 = opStart3;
+ BYTE* op4 = opStart4;
+ U32 endSignal = 1;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+
+ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
+ if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
+ if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */
+ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
+ CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
+ CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
+ CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
+
+ /* 16-32 symbols per loop (4-8 symbols per stream) */
+ if ((size_t)(oend - op4) >= sizeof(size_t)) {
+ for ( ; (endSignal) & (op4 < olimit); ) {
+#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
+ endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
+ endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
+#else
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
+ endSignal = (U32)LIKELY((U32)
+ (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished)
+ & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished)
+ & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished)
+ & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished));
+#endif
+ }
+ }
+
+ /* check corruption */
+ if (op1 > opStart2) return ERROR(corruption_detected);
+ if (op2 > opStart3) return ERROR(corruption_detected);
+ if (op3 > opStart4) return ERROR(corruption_detected);
+ /* note : op4 already verified within main loop */
+
+ /* finish bitStreams one by one */
+ HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
+ HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
+ HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
+ HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
+
+ /* check */
+ { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+ if (!endCheck) return ERROR(corruption_detected); }
+
+ /* decoded size */
+ return dstSize;
+ }
+}
+
+#if HUF_NEED_BMI2_FUNCTION
+static BMI2_TARGET_ATTRIBUTE
+size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+#endif
+
+static
+size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2
+
+HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+ U64 bits[4];
+ BYTE const* ip[4];
+ BYTE* op[4];
+ BYTE* oend[4];
+ HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt;
+ BYTE const* const ilimit = args->ilimit;
+
+ /* Copy the arguments to local registers. */
+ ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+ ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+ ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+ oend[0] = op[1];
+ oend[1] = op[2];
+ oend[2] = op[3];
+ oend[3] = args->oend;
+
+ assert(MEM_isLittleEndian());
+ assert(!MEM_32bits());
+
+ for (;;) {
+ BYTE* olimit;
+ int stream;
+ int symbol;
+
+ /* Assert loop preconditions */
+#ifndef NDEBUG
+ for (stream = 0; stream < 4; ++stream) {
+ assert(op[stream] <= oend[stream]);
+ assert(ip[stream] >= ilimit);
+ }
+#endif
+ /* Compute olimit */
+ {
+ /* Each loop does 5 table lookups for each of the 4 streams.
+ * Each table lookup consumes up to 11 bits of input, and produces
+ * up to 2 bytes of output.
+ */
+ /* We can consume up to 7 bytes of input per iteration per stream.
+ * We also know that each input pointer is >= ip[0]. So we can run
+ * iters loops before running out of input.
+ */
+ size_t iters = (size_t)(ip[0] - ilimit) / 7;
+ /* Each iteration can produce up to 10 bytes of output per stream.
+ * Each output stream my advance at different rates. So take the
+ * minimum number of safe iterations among all the output streams.
+ */
+ for (stream = 0; stream < 4; ++stream) {
+ size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10;
+ iters = MIN(iters, oiters);
+ }
+
+ /* Each iteration produces at least 5 output symbols. So until
+ * op[3] crosses olimit, we know we haven't executed iters
+ * iterations yet. This saves us maintaining an iters counter,
+ * at the expense of computing the remaining # of iterations
+ * more frequently.
+ */
+ olimit = op[3] + (iters * 5);
+
+ /* Exit the fast decoding loop if we are too close to the end. */
+ if (op[3] + 10 > olimit)
+ break;
+
+ /* Exit the decoding loop if any input pointer has crossed the
+ * previous one. This indicates corruption, and a precondition
+ * to our loop is that ip[i] >= ip[0].
+ */
+ for (stream = 1; stream < 4; ++stream) {
+ if (ip[stream] < ip[stream - 1])
+ goto _out;
+ }
+ }
+
+#ifndef NDEBUG
+ for (stream = 1; stream < 4; ++stream) {
+ assert(ip[stream] >= ip[stream - 1]);
+ }
+#endif
+
+ do {
+ /* Do 5 table lookups for each of the first 3 streams */
+ for (symbol = 0; symbol < 5; ++symbol) {
+ for (stream = 0; stream < 3; ++stream) {
+ int const index = (int)(bits[stream] >> 53);
+ HUF_DEltX2 const entry = dtable[index];
+ MEM_write16(op[stream], entry.sequence);
+ bits[stream] <<= (entry.nbBits);
+ op[stream] += (entry.length);
+ }
+ }
+ /* Do 1 table lookup from the final stream */
+ {
+ int const index = (int)(bits[3] >> 53);
+ HUF_DEltX2 const entry = dtable[index];
+ MEM_write16(op[3], entry.sequence);
+ bits[3] <<= (entry.nbBits);
+ op[3] += (entry.length);
+ }
+ /* Do 4 table lookups from the final stream & reload bitstreams */
+ for (stream = 0; stream < 4; ++stream) {
+ /* Do a table lookup from the final stream.
+ * This is interleaved with the reloading to reduce register
+ * pressure. This shouldn't be necessary, but compilers can
+ * struggle with codegen with high register pressure.
+ */
+ {
+ int const index = (int)(bits[3] >> 53);
+ HUF_DEltX2 const entry = dtable[index];
+ MEM_write16(op[3], entry.sequence);
+ bits[3] <<= (entry.nbBits);
+ op[3] += (entry.length);
+ }
+ /* Reload the bistreams. The final bitstream must be reloaded
+ * after the 5th symbol was decoded.
+ */
+ {
+ int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+ int const nbBits = ctz & 7;
+ int const nbBytes = ctz >> 3;
+ ip[stream] -= nbBytes;
+ bits[stream] = MEM_read64(ip[stream]) | 1;
+ bits[stream] <<= nbBits;
+ }
+ }
+ } while (op[3] < olimit);
+ }
+
+_out:
+
+ /* Save the final values of each of the state variables back to args. */
+ ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+ ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+ ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+
+static HUF_FAST_BMI2_ATTRS size_t
+HUF_decompress4X2_usingDTable_internal_fast(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable,
+ HUF_DecompressFastLoopFn loopFn) {
+ void const* dt = DTable + 1;
+ const BYTE* const iend = (const BYTE*)cSrc + 6;
+ BYTE* const oend = (BYTE*)dst + dstSize;
+ HUF_DecompressFastArgs args;
+ {
+ size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+ FORWARD_IF_ERROR(ret, "Failed to init asm args");
+ if (ret == 0)
+ return 0;
+ }
+
+ assert(args.ip[0] >= args.ilimit);
+ loopFn(&args);
+
+ /* note : op4 already verified within main loop */
+ assert(args.ip[0] >= iend);
+ assert(args.ip[1] >= iend);
+ assert(args.ip[2] >= iend);
+ assert(args.ip[3] >= iend);
+ assert(args.op[3] <= oend);
+ (void)iend;
+
+ /* finish bitStreams one by one */
+ {
+ size_t const segmentSize = (dstSize+3) / 4;
+ BYTE* segmentEnd = (BYTE*)dst;
+ int i;
+ for (i = 0; i < 4; ++i) {
+ BIT_DStream_t bit;
+ if (segmentSize <= (size_t)(oend - segmentEnd))
+ segmentEnd += segmentSize;
+ else
+ segmentEnd = oend;
+ FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
+ args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG);
+ if (args.op[i] != segmentEnd)
+ return ERROR(corruption_detected);
+ }
+ }
+
+ /* decoded size */
+ return dstSize;
+}
+
+static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable, int flags)
+{
+ HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default;
+ HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop;
+
+#if DYNAMIC_BMI2
+ if (flags & HUF_flags_bmi2) {
+ fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2;
+# if ZSTD_ENABLE_ASM_X86_64_BMI2
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+ }
+# endif
+ } else {
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+ }
+#endif
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+ }
+#endif
+
+ if (!(flags & HUF_flags_disableFast)) {
+ size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+ if (ret != 0)
+ return ret;
+ }
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
+
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
+ workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags);
+}
+
+static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
+ workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+
+#endif /* HUF_FORCE_DECOMPRESS_X1 */
+
+
+/* ***********************************/
+/* Universal decompression selectors */
+/* ***********************************/
+
+
+#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
+typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
+static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] =
+{
+ /* single, double, quad */
+ {{0,0}, {1,1}}, /* Q==0 : impossible */
+ {{0,0}, {1,1}}, /* Q==1 : impossible */
+ {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */
+ {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */
+ {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */
+ {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */
+ {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */
+ {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */
+ {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */
+ {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */
+ {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */
+ {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */
+ {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */
+ {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */
+ {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */
+ {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */
+};
+#endif
+
+/** HUF_selectDecoder() :
+ * Tells which decoder is likely to decode faster,
+ * based on a set of pre-computed metrics.
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
+ * Assumption : 0 < dstSize <= 128 KB */
+U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
+{
+ assert(dstSize > 0);
+ assert(dstSize <= 128*1024);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)dstSize;
+ (void)cSrcSize;
+ return 0;
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)dstSize;
+ (void)cSrcSize;
+ return 1;
+#else
+ /* decoder timing evaluation */
+ { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
+ U32 const D256 = (U32)(dstSize >> 8);
+ U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
+ U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
+ DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */
+ return DTime1 < DTime0;
+ }
+#endif
+}
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ /* validation checks */
+ if (dstSize == 0) return ERROR(dstSize_tooSmall);
+ if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
+ if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
+ if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
+
+ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)algoNb;
+ assert(algoNb == 0);
+ return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)algoNb;
+ assert(algoNb == 1);
+ return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags);
+#else
+ return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags):
+ HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags);
+#endif
+ }
+}
+
+
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
+{
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)dtd;
+ assert(dtd.tableType == 0);
+ return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)dtd;
+ assert(dtd.tableType == 1);
+ return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#else
+ return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+ HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#endif
+}
+
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+#endif
+
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
+{
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)dtd;
+ assert(dtd.tableType == 0);
+ return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)dtd;
+ assert(dtd.tableType == 1);
+ return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#else
+ return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+ HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#endif
+}
+
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
+{
+ /* validation checks */
+ if (dstSize == 0) return ERROR(dstSize_tooSmall);
+ if (cSrcSize == 0) return ERROR(corruption_detected);
+
+ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)algoNb;
+ assert(algoNb == 0);
+ return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)algoNb;
+ assert(algoNb == 1);
+ return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#else
+ return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) :
+ HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#endif
+ }
+}
diff --git a/contrib/zstd/mem.h b/contrib/zstd/mem.h
new file mode 100644
index 0000000..98dd47a
--- /dev/null
+++ b/contrib/zstd/mem.h
@@ -0,0 +1,435 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MEM_H_MODULE
+#define MEM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include <stddef.h> /* size_t, ptrdiff_t */
+#include "compiler.h" /* __has_builtin */
+#include "debug.h" /* DEBUG_STATIC_ASSERT */
+#include "zstd_deps.h" /* ZSTD_memcpy */
+
+
+/*-****************************************
+* Compiler specifics
+******************************************/
+#if defined(_MSC_VER) /* Visual Studio */
+# include <stdlib.h> /* _byteswap_ulong */
+# include <intrin.h> /* _byteswap_* */
+#endif
+#if defined(__GNUC__)
+# define MEM_STATIC static __inline __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define MEM_STATIC static inline
+#elif defined(_MSC_VER)
+# define MEM_STATIC static __inline
+#else
+# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+/*-**************************************************************
+* Basic Types
+*****************************************************************/
+#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# if defined(_AIX)
+# include <inttypes.h>
+# else
+# include <stdint.h> /* intptr_t */
+# endif
+ typedef uint8_t BYTE;
+ typedef uint8_t U8;
+ typedef int8_t S8;
+ typedef uint16_t U16;
+ typedef int16_t S16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+ typedef int64_t S64;
+#else
+# include <limits.h>
+#if CHAR_BIT != 8
+# error "this implementation requires char to be exactly 8-bit type"
+#endif
+ typedef unsigned char BYTE;
+ typedef unsigned char U8;
+ typedef signed char S8;
+#if USHRT_MAX != 65535
+# error "this implementation requires short to be exactly 16-bit type"
+#endif
+ typedef unsigned short U16;
+ typedef signed short S16;
+#if UINT_MAX != 4294967295
+# error "this implementation requires int to be exactly 32-bit type"
+#endif
+ typedef unsigned int U32;
+ typedef signed int S32;
+/* note : there are no limits defined for long long type in C90.
+ * limits exist in C99, however, in such case, <stdint.h> is preferred */
+ typedef unsigned long long U64;
+ typedef signed long long S64;
+#endif
+
+
+/*-**************************************************************
+* Memory I/O API
+*****************************************************************/
+/*=== Static platform detection ===*/
+MEM_STATIC unsigned MEM_32bits(void);
+MEM_STATIC unsigned MEM_64bits(void);
+MEM_STATIC unsigned MEM_isLittleEndian(void);
+
+/*=== Native unaligned read/write ===*/
+MEM_STATIC U16 MEM_read16(const void* memPtr);
+MEM_STATIC U32 MEM_read32(const void* memPtr);
+MEM_STATIC U64 MEM_read64(const void* memPtr);
+MEM_STATIC size_t MEM_readST(const void* memPtr);
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value);
+MEM_STATIC void MEM_write32(void* memPtr, U32 value);
+MEM_STATIC void MEM_write64(void* memPtr, U64 value);
+
+/*=== Little endian unaligned read/write ===*/
+MEM_STATIC U16 MEM_readLE16(const void* memPtr);
+MEM_STATIC U32 MEM_readLE24(const void* memPtr);
+MEM_STATIC U32 MEM_readLE32(const void* memPtr);
+MEM_STATIC U64 MEM_readLE64(const void* memPtr);
+MEM_STATIC size_t MEM_readLEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val);
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val);
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val);
+
+/*=== Big endian unaligned read/write ===*/
+MEM_STATIC U32 MEM_readBE32(const void* memPtr);
+MEM_STATIC U64 MEM_readBE64(const void* memPtr);
+MEM_STATIC size_t MEM_readBEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val);
+
+/*=== Byteswap ===*/
+MEM_STATIC U32 MEM_swap32(U32 in);
+MEM_STATIC U64 MEM_swap64(U64 in);
+MEM_STATIC size_t MEM_swapST(size_t in);
+
+
+/*-**************************************************************
+* Memory I/O Implementation
+*****************************************************************/
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
+ * Method 2 : direct access. This method is portable but violate C standard.
+ * It can generate buggy code on targets depending on alignment.
+ * Default : method 1 if supported, else method 0
+ */
+#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
+# ifdef __GNUC__
+# define MEM_FORCE_MEMORY_ACCESS 1
+# endif
+#endif
+
+MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
+MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
+
+MEM_STATIC unsigned MEM_isLittleEndian(void)
+{
+#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+ return 1;
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+ return 0;
+#elif defined(__clang__) && __LITTLE_ENDIAN__
+ return 1;
+#elif defined(__clang__) && __BIG_ENDIAN__
+ return 0;
+#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
+ return 1;
+#elif defined(__DMC__) && defined(_M_IX86)
+ return 1;
+#else
+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
+ return one.c[0];
+#endif
+}
+
+#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
+
+/* violates C standard, by lying on structure alignment.
+Only use if no other choice to achieve best performance on target platform */
+MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
+MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
+MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
+MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
+
+#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
+
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
+typedef __attribute__((aligned(1))) size_t unalignArch;
+
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
+MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
+
+#else
+
+/* default method, safe and standard.
+ can sometimes prove slower */
+
+MEM_STATIC U16 MEM_read16(const void* memPtr)
+{
+ U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U32 MEM_read32(const void* memPtr)
+{
+ U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U64 MEM_read64(const void* memPtr)
+{
+ U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC size_t MEM_readST(const void* memPtr)
+{
+ size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value)
+{
+ ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write32(void* memPtr, U32 value)
+{
+ ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write64(void* memPtr, U64 value)
+{
+ ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+#endif /* MEM_FORCE_MEMORY_ACCESS */
+
+MEM_STATIC U32 MEM_swap32_fallback(U32 in)
+{
+ return ((in << 24) & 0xff000000 ) |
+ ((in << 8) & 0x00ff0000 ) |
+ ((in >> 8) & 0x0000ff00 ) |
+ ((in >> 24) & 0x000000ff );
+}
+
+MEM_STATIC U32 MEM_swap32(U32 in)
+{
+#if defined(_MSC_VER) /* Visual Studio */
+ return _byteswap_ulong(in);
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+ || (defined(__clang__) && __has_builtin(__builtin_bswap32))
+ return __builtin_bswap32(in);
+#else
+ return MEM_swap32_fallback(in);
+#endif
+}
+
+MEM_STATIC U64 MEM_swap64_fallback(U64 in)
+{
+ return ((in << 56) & 0xff00000000000000ULL) |
+ ((in << 40) & 0x00ff000000000000ULL) |
+ ((in << 24) & 0x0000ff0000000000ULL) |
+ ((in << 8) & 0x000000ff00000000ULL) |
+ ((in >> 8) & 0x00000000ff000000ULL) |
+ ((in >> 24) & 0x0000000000ff0000ULL) |
+ ((in >> 40) & 0x000000000000ff00ULL) |
+ ((in >> 56) & 0x00000000000000ffULL);
+}
+
+MEM_STATIC U64 MEM_swap64(U64 in)
+{
+#if defined(_MSC_VER) /* Visual Studio */
+ return _byteswap_uint64(in);
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+ || (defined(__clang__) && __has_builtin(__builtin_bswap64))
+ return __builtin_bswap64(in);
+#else
+ return MEM_swap64_fallback(in);
+#endif
+}
+
+MEM_STATIC size_t MEM_swapST(size_t in)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_swap32((U32)in);
+ else
+ return (size_t)MEM_swap64((U64)in);
+}
+
+/*=== Little endian r/w ===*/
+
+MEM_STATIC U16 MEM_readLE16(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_read16(memPtr);
+ else {
+ const BYTE* p = (const BYTE*)memPtr;
+ return (U16)(p[0] + (p[1]<<8));
+ }
+}
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
+{
+ if (MEM_isLittleEndian()) {
+ MEM_write16(memPtr, val);
+ } else {
+ BYTE* p = (BYTE*)memPtr;
+ p[0] = (BYTE)val;
+ p[1] = (BYTE)(val>>8);
+ }
+}
+
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+ return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
+}
+
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
+{
+ MEM_writeLE16(memPtr, (U16)val);
+ ((BYTE*)memPtr)[2] = (BYTE)(val>>16);
+}
+
+MEM_STATIC U32 MEM_readLE32(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_read32(memPtr);
+ else
+ return MEM_swap32(MEM_read32(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
+{
+ if (MEM_isLittleEndian())
+ MEM_write32(memPtr, val32);
+ else
+ MEM_write32(memPtr, MEM_swap32(val32));
+}
+
+MEM_STATIC U64 MEM_readLE64(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_read64(memPtr);
+ else
+ return MEM_swap64(MEM_read64(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
+{
+ if (MEM_isLittleEndian())
+ MEM_write64(memPtr, val64);
+ else
+ MEM_write64(memPtr, MEM_swap64(val64));
+}
+
+MEM_STATIC size_t MEM_readLEST(const void* memPtr)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_readLE32(memPtr);
+ else
+ return (size_t)MEM_readLE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
+{
+ if (MEM_32bits())
+ MEM_writeLE32(memPtr, (U32)val);
+ else
+ MEM_writeLE64(memPtr, (U64)val);
+}
+
+/*=== Big endian r/w ===*/
+
+MEM_STATIC U32 MEM_readBE32(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_swap32(MEM_read32(memPtr));
+ else
+ return MEM_read32(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
+{
+ if (MEM_isLittleEndian())
+ MEM_write32(memPtr, MEM_swap32(val32));
+ else
+ MEM_write32(memPtr, val32);
+}
+
+MEM_STATIC U64 MEM_readBE64(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_swap64(MEM_read64(memPtr));
+ else
+ return MEM_read64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
+{
+ if (MEM_isLittleEndian())
+ MEM_write64(memPtr, MEM_swap64(val64));
+ else
+ MEM_write64(memPtr, val64);
+}
+
+MEM_STATIC size_t MEM_readBEST(const void* memPtr)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_readBE32(memPtr);
+ else
+ return (size_t)MEM_readBE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
+{
+ if (MEM_32bits())
+ MEM_writeBE32(memPtr, (U32)val);
+ else
+ MEM_writeBE64(memPtr, (U64)val);
+}
+
+/* code only tested on 32 and 64 bits systems */
+MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* MEM_H_MODULE */
diff --git a/contrib/zstd/pool.c b/contrib/zstd/pool.c
new file mode 100644
index 0000000..f3d9d08
--- /dev/null
+++ b/contrib/zstd/pool.c
@@ -0,0 +1,371 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* ====== Dependencies ======= */
+#include "zstd_deps.h" /* size_t */
+#include "debug.h" /* assert */
+#include "zstd_internal.h" /* ZSTD_customCalloc, ZSTD_customFree */
+#include "pool.h"
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+#ifdef ZSTD_MULTITHREAD
+
+#include "threading.h" /* pthread adaptation */
+
+/* A job is a function and an opaque argument */
+typedef struct POOL_job_s {
+ POOL_function function;
+ void *opaque;
+} POOL_job;
+
+struct POOL_ctx_s {
+ ZSTD_customMem customMem;
+ /* Keep track of the threads */
+ ZSTD_pthread_t* threads;
+ size_t threadCapacity;
+ size_t threadLimit;
+
+ /* The queue is a circular buffer */
+ POOL_job *queue;
+ size_t queueHead;
+ size_t queueTail;
+ size_t queueSize;
+
+ /* The number of threads working on jobs */
+ size_t numThreadsBusy;
+ /* Indicates if the queue is empty */
+ int queueEmpty;
+
+ /* The mutex protects the queue */
+ ZSTD_pthread_mutex_t queueMutex;
+ /* Condition variable for pushers to wait on when the queue is full */
+ ZSTD_pthread_cond_t queuePushCond;
+ /* Condition variables for poppers to wait on when the queue is empty */
+ ZSTD_pthread_cond_t queuePopCond;
+ /* Indicates if the queue is shutting down */
+ int shutdown;
+};
+
+/* POOL_thread() :
+ * Work thread for the thread pool.
+ * Waits for jobs and executes them.
+ * @returns : NULL on failure else non-null.
+ */
+static void* POOL_thread(void* opaque) {
+ POOL_ctx* const ctx = (POOL_ctx*)opaque;
+ if (!ctx) { return NULL; }
+ for (;;) {
+ /* Lock the mutex and wait for a non-empty queue or until shutdown */
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+
+ while ( ctx->queueEmpty
+ || (ctx->numThreadsBusy >= ctx->threadLimit) ) {
+ if (ctx->shutdown) {
+ /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
+ * a few threads will be shutdown while !queueEmpty,
+ * but enough threads will remain active to finish the queue */
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return opaque;
+ }
+ ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+ }
+ /* Pop a job off the queue */
+ { POOL_job const job = ctx->queue[ctx->queueHead];
+ ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
+ ctx->numThreadsBusy++;
+ ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
+ /* Unlock the mutex, signal a pusher, and run the job */
+ ZSTD_pthread_cond_signal(&ctx->queuePushCond);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+
+ job.function(job.opaque);
+
+ /* If the intended queue size was 0, signal after finishing job */
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ ctx->numThreadsBusy--;
+ ZSTD_pthread_cond_signal(&ctx->queuePushCond);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ }
+ } /* for (;;) */
+ assert(0); /* Unreachable */
+}
+
+/* ZSTD_createThreadPool() : public access point */
+POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
+ return POOL_create (numThreads, 0);
+}
+
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
+ return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
+}
+
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
+ ZSTD_customMem customMem)
+{
+ POOL_ctx* ctx;
+ /* Check parameters */
+ if (!numThreads) { return NULL; }
+ /* Allocate the context and zero initialize */
+ ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem);
+ if (!ctx) { return NULL; }
+ /* Initialize the job queue.
+ * It needs one extra space since one space is wasted to differentiate
+ * empty and full queues.
+ */
+ ctx->queueSize = queueSize + 1;
+ ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem);
+ ctx->queueHead = 0;
+ ctx->queueTail = 0;
+ ctx->numThreadsBusy = 0;
+ ctx->queueEmpty = 1;
+ {
+ int error = 0;
+ error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
+ error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
+ error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
+ if (error) { POOL_free(ctx); return NULL; }
+ }
+ ctx->shutdown = 0;
+ /* Allocate space for the thread handles */
+ ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
+ ctx->threadCapacity = 0;
+ ctx->customMem = customMem;
+ /* Check for errors */
+ if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
+ /* Initialize the threads */
+ { size_t i;
+ for (i = 0; i < numThreads; ++i) {
+ if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+ ctx->threadCapacity = i;
+ POOL_free(ctx);
+ return NULL;
+ } }
+ ctx->threadCapacity = numThreads;
+ ctx->threadLimit = numThreads;
+ }
+ return ctx;
+}
+
+/*! POOL_join() :
+ Shutdown the queue, wake any sleeping threads, and join all of the threads.
+*/
+static void POOL_join(POOL_ctx* ctx) {
+ /* Shut down the queue */
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ ctx->shutdown = 1;
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ /* Wake up sleeping threads */
+ ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
+ ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
+ /* Join all of the threads */
+ { size_t i;
+ for (i = 0; i < ctx->threadCapacity; ++i) {
+ ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */
+ } }
+}
+
+void POOL_free(POOL_ctx *ctx) {
+ if (!ctx) { return; }
+ POOL_join(ctx);
+ ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
+ ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
+ ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
+ ZSTD_customFree(ctx->queue, ctx->customMem);
+ ZSTD_customFree(ctx->threads, ctx->customMem);
+ ZSTD_customFree(ctx, ctx->customMem);
+}
+
+/*! POOL_joinJobs() :
+ * Waits for all queued jobs to finish executing.
+ */
+void POOL_joinJobs(POOL_ctx* ctx) {
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) {
+ ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+ }
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+}
+
+void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
+ POOL_free (pool);
+}
+
+size_t POOL_sizeof(const POOL_ctx* ctx) {
+ if (ctx==NULL) return 0; /* supports sizeof NULL */
+ return sizeof(*ctx)
+ + ctx->queueSize * sizeof(POOL_job)
+ + ctx->threadCapacity * sizeof(ZSTD_pthread_t);
+}
+
+
+/* @return : 0 on success, 1 on error */
+static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
+{
+ if (numThreads <= ctx->threadCapacity) {
+ if (!numThreads) return 1;
+ ctx->threadLimit = numThreads;
+ return 0;
+ }
+ /* numThreads > threadCapacity */
+ { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
+ if (!threadPool) return 1;
+ /* replace existing thread pool */
+ ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
+ ZSTD_customFree(ctx->threads, ctx->customMem);
+ ctx->threads = threadPool;
+ /* Initialize additional threads */
+ { size_t threadId;
+ for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
+ if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
+ ctx->threadCapacity = threadId;
+ return 1;
+ } }
+ } }
+ /* successfully expanded */
+ ctx->threadCapacity = numThreads;
+ ctx->threadLimit = numThreads;
+ return 0;
+}
+
+/* @return : 0 on success, 1 on error */
+int POOL_resize(POOL_ctx* ctx, size_t numThreads)
+{
+ int result;
+ if (ctx==NULL) return 1;
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ result = POOL_resize_internal(ctx, numThreads);
+ ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return result;
+}
+
+/**
+ * Returns 1 if the queue is full and 0 otherwise.
+ *
+ * When queueSize is 1 (pool was created with an intended queueSize of 0),
+ * then a queue is empty if there is a thread free _and_ no job is waiting.
+ */
+static int isQueueFull(POOL_ctx const* ctx) {
+ if (ctx->queueSize > 1) {
+ return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
+ } else {
+ return (ctx->numThreadsBusy == ctx->threadLimit) ||
+ !ctx->queueEmpty;
+ }
+}
+
+
+static void
+POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
+{
+ POOL_job job;
+ job.function = function;
+ job.opaque = opaque;
+ assert(ctx != NULL);
+ if (ctx->shutdown) return;
+
+ ctx->queueEmpty = 0;
+ ctx->queue[ctx->queueTail] = job;
+ ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
+ ZSTD_pthread_cond_signal(&ctx->queuePopCond);
+}
+
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
+{
+ assert(ctx != NULL);
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ /* Wait until there is space in the queue for the new job */
+ while (isQueueFull(ctx) && (!ctx->shutdown)) {
+ ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+ }
+ POOL_add_internal(ctx, function, opaque);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+}
+
+
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
+{
+ assert(ctx != NULL);
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ if (isQueueFull(ctx)) {
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return 0;
+ }
+ POOL_add_internal(ctx, function, opaque);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return 1;
+}
+
+
+#else /* ZSTD_MULTITHREAD not defined */
+
+/* ========================== */
+/* No multi-threading support */
+/* ========================== */
+
+
+/* We don't need any data, but if it is empty, malloc() might return NULL. */
+struct POOL_ctx_s {
+ int dummy;
+};
+static POOL_ctx g_poolCtx;
+
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
+ return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
+}
+
+POOL_ctx*
+POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
+{
+ (void)numThreads;
+ (void)queueSize;
+ (void)customMem;
+ return &g_poolCtx;
+}
+
+void POOL_free(POOL_ctx* ctx) {
+ assert(!ctx || ctx == &g_poolCtx);
+ (void)ctx;
+}
+
+void POOL_joinJobs(POOL_ctx* ctx){
+ assert(!ctx || ctx == &g_poolCtx);
+ (void)ctx;
+}
+
+int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
+ (void)ctx; (void)numThreads;
+ return 0;
+}
+
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
+ (void)ctx;
+ function(opaque);
+}
+
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
+ (void)ctx;
+ function(opaque);
+ return 1;
+}
+
+size_t POOL_sizeof(const POOL_ctx* ctx) {
+ if (ctx==NULL) return 0; /* supports sizeof NULL */
+ assert(ctx == &g_poolCtx);
+ return sizeof(*ctx);
+}
+
+#endif /* ZSTD_MULTITHREAD */
diff --git a/contrib/zstd/pool.h b/contrib/zstd/pool.h
new file mode 100644
index 0000000..14e883f
--- /dev/null
+++ b/contrib/zstd/pool.h
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef POOL_H
+#define POOL_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+#include "zstd_deps.h"
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
+#include "zstd.h"
+
+typedef struct POOL_ctx_s POOL_ctx;
+
+/*! POOL_create() :
+ * Create a thread pool with at most `numThreads` threads.
+ * `numThreads` must be at least 1.
+ * The maximum number of queued jobs before blocking is `queueSize`.
+ * @return : POOL_ctx pointer on success, else NULL.
+*/
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
+
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
+ ZSTD_customMem customMem);
+
+/*! POOL_free() :
+ * Free a thread pool returned by POOL_create().
+ */
+void POOL_free(POOL_ctx* ctx);
+
+
+/*! POOL_joinJobs() :
+ * Waits for all queued jobs to finish executing.
+ */
+void POOL_joinJobs(POOL_ctx* ctx);
+
+/*! POOL_resize() :
+ * Expands or shrinks pool's number of threads.
+ * This is more efficient than releasing + creating a new context,
+ * since it tries to preserve and re-use existing threads.
+ * `numThreads` must be at least 1.
+ * @return : 0 when resize was successful,
+ * !0 (typically 1) if there is an error.
+ * note : only numThreads can be resized, queueSize remains unchanged.
+ */
+int POOL_resize(POOL_ctx* ctx, size_t numThreads);
+
+/*! POOL_sizeof() :
+ * @return threadpool memory usage
+ * note : compatible with NULL (returns 0 in this case)
+ */
+size_t POOL_sizeof(const POOL_ctx* ctx);
+
+/*! POOL_function :
+ * The function type that can be added to a thread pool.
+ */
+typedef void (*POOL_function)(void*);
+
+/*! POOL_add() :
+ * Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
+ * Possibly blocks until there is room in the queue.
+ * Note : The function may be executed asynchronously,
+ * therefore, `opaque` must live until function has been completed.
+ */
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
+
+
+/*! POOL_tryAdd() :
+ * Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
+ * Returns immediately even if not (does not block).
+ * @return : 1 if successful, 0 if not.
+ */
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif
diff --git a/contrib/zstd/portability_macros.h b/contrib/zstd/portability_macros.h
new file mode 100644
index 0000000..8fd6ea8
--- /dev/null
+++ b/contrib/zstd/portability_macros.h
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_PORTABILITY_MACROS_H
+#define ZSTD_PORTABILITY_MACROS_H
+
+/**
+ * This header file contains macro definitions to support portability.
+ * This header is shared between C and ASM code, so it MUST only
+ * contain macro definitions. It MUST not contain any C code.
+ *
+ * This header ONLY defines macros to detect platforms/feature support.
+ *
+ */
+
+
+/* compat. with non-clang compilers */
+#ifndef __has_attribute
+ #define __has_attribute(x) 0
+#endif
+
+/* compat. with non-clang compilers */
+#ifndef __has_builtin
+# define __has_builtin(x) 0
+#endif
+
+/* compat. with non-clang compilers */
+#ifndef __has_feature
+# define __has_feature(x) 0
+#endif
+
+/* detects whether we are being compiled under msan */
+#ifndef ZSTD_MEMORY_SANITIZER
+# if __has_feature(memory_sanitizer)
+# define ZSTD_MEMORY_SANITIZER 1
+# else
+# define ZSTD_MEMORY_SANITIZER 0
+# endif
+#endif
+
+/* detects whether we are being compiled under asan */
+#ifndef ZSTD_ADDRESS_SANITIZER
+# if __has_feature(address_sanitizer)
+# define ZSTD_ADDRESS_SANITIZER 1
+# elif defined(__SANITIZE_ADDRESS__)
+# define ZSTD_ADDRESS_SANITIZER 1
+# else
+# define ZSTD_ADDRESS_SANITIZER 0
+# endif
+#endif
+
+/* detects whether we are being compiled under dfsan */
+#ifndef ZSTD_DATAFLOW_SANITIZER
+# if __has_feature(dataflow_sanitizer)
+# define ZSTD_DATAFLOW_SANITIZER 1
+# else
+# define ZSTD_DATAFLOW_SANITIZER 0
+# endif
+#endif
+
+/* Mark the internal assembly functions as hidden */
+#ifdef __ELF__
+# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
+#else
+# define ZSTD_HIDE_ASM_FUNCTION(func)
+#endif
+
+/* Enable runtime BMI2 dispatch based on the CPU.
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
+ */
+#ifndef DYNAMIC_BMI2
+ #if ((defined(__clang__) && __has_attribute(__target__)) \
+ || (defined(__GNUC__) \
+ && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
+ && (defined(__x86_64__) || defined(_M_X64)) \
+ && !defined(__BMI2__)
+ # define DYNAMIC_BMI2 1
+ #else
+ # define DYNAMIC_BMI2 0
+ #endif
+#endif
+
+/**
+ * Only enable assembly for GNUC compatible compilers,
+ * because other platforms may not support GAS assembly syntax.
+ *
+ * Only enable assembly for Linux / MacOS, other platforms may
+ * work, but they haven't been tested. This could likely be
+ * extended to BSD systems.
+ *
+ * Disable assembly when MSAN is enabled, because MSAN requires
+ * 100% of code to be instrumented to work.
+ */
+#if defined(__GNUC__)
+# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
+# if ZSTD_MEMORY_SANITIZER
+# define ZSTD_ASM_SUPPORTED 0
+# elif ZSTD_DATAFLOW_SANITIZER
+# define ZSTD_ASM_SUPPORTED 0
+# else
+# define ZSTD_ASM_SUPPORTED 1
+# endif
+# else
+# define ZSTD_ASM_SUPPORTED 0
+# endif
+#else
+# define ZSTD_ASM_SUPPORTED 0
+#endif
+
+/**
+ * Determines whether we should enable assembly for x86-64
+ * with BMI2.
+ *
+ * Enable if all of the following conditions hold:
+ * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
+ * - Assembly is supported
+ * - We are compiling for x86-64 and either:
+ * - DYNAMIC_BMI2 is enabled
+ * - BMI2 is supported at compile time
+ */
+#if !defined(ZSTD_DISABLE_ASM) && \
+ ZSTD_ASM_SUPPORTED && \
+ defined(__x86_64__) && \
+ (DYNAMIC_BMI2 || defined(__BMI2__))
+# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
+#else
+# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
+#endif
+
+/*
+ * For x86 ELF targets, add .note.gnu.property section for Intel CET in
+ * assembly sources when CET is enabled.
+ *
+ * Additionally, any function that may be called indirectly must begin
+ * with ZSTD_CET_ENDBRANCH.
+ */
+#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
+ && defined(__has_include)
+# if __has_include(<cet.h>)
+# include <cet.h>
+# define ZSTD_CET_ENDBRANCH _CET_ENDBR
+# endif
+#endif
+
+#ifndef ZSTD_CET_ENDBRANCH
+# define ZSTD_CET_ENDBRANCH
+#endif
+
+#endif /* ZSTD_PORTABILITY_MACROS_H */
diff --git a/contrib/zstd/zdict.h b/contrib/zstd/zdict.h
new file mode 100644
index 0000000..2268f94
--- /dev/null
+++ b/contrib/zstd/zdict.h
@@ -0,0 +1,474 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef ZSTD_ZDICT_H
+#define ZSTD_ZDICT_H
+
+/*====== Dependencies ======*/
+#include <stddef.h> /* size_t */
+
+
+/* ===== ZDICTLIB_API : control library symbols visibility ===== */
+#ifndef ZDICTLIB_VISIBLE
+ /* Backwards compatibility with old macro name */
+# ifdef ZDICTLIB_VISIBILITY
+# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
+# else
+# define ZDICTLIB_VISIBLE
+# endif
+#endif
+
+#ifndef ZDICTLIB_HIDDEN
+# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+# else
+# define ZDICTLIB_HIDDEN
+# endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZDICTLIB_API ZDICTLIB_VISIBLE
+#endif
+
+/*******************************************************************************
+ * Zstd dictionary builder
+ *
+ * FAQ
+ * ===
+ * Why should I use a dictionary?
+ * ------------------------------
+ *
+ * Zstd can use dictionaries to improve compression ratio of small data.
+ * Traditionally small files don't compress well because there is very little
+ * repetition in a single sample, since it is small. But, if you are compressing
+ * many similar files, like a bunch of JSON records that share the same
+ * structure, you can train a dictionary on ahead of time on some samples of
+ * these files. Then, zstd can use the dictionary to find repetitions that are
+ * present across samples. This can vastly improve compression ratio.
+ *
+ * When is a dictionary useful?
+ * ----------------------------
+ *
+ * Dictionaries are useful when compressing many small files that are similar.
+ * The larger a file is, the less benefit a dictionary will have. Generally,
+ * we don't expect dictionary compression to be effective past 100KB. And the
+ * smaller a file is, the more we would expect the dictionary to help.
+ *
+ * How do I use a dictionary?
+ * --------------------------
+ *
+ * Simply pass the dictionary to the zstd compressor with
+ * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to
+ * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other
+ * more advanced functions that allow selecting some options, see zstd.h for
+ * complete documentation.
+ *
+ * What is a zstd dictionary?
+ * --------------------------
+ *
+ * A zstd dictionary has two pieces: Its header, and its content. The header
+ * contains a magic number, the dictionary ID, and entropy tables. These
+ * entropy tables allow zstd to save on header costs in the compressed file,
+ * which really matters for small data. The content is just bytes, which are
+ * repeated content that is common across many samples.
+ *
+ * What is a raw content dictionary?
+ * ---------------------------------
+ *
+ * A raw content dictionary is just bytes. It doesn't have a zstd dictionary
+ * header, a dictionary ID, or entropy tables. Any buffer is a valid raw
+ * content dictionary.
+ *
+ * How do I train a dictionary?
+ * ----------------------------
+ *
+ * Gather samples from your use case. These samples should be similar to each
+ * other. If you have several use cases, you could try to train one dictionary
+ * per use case.
+ *
+ * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your
+ * dictionary. There are a few advanced versions of this function, but this
+ * is a great starting point. If you want to further tune your dictionary
+ * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow
+ * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`.
+ *
+ * If the dictionary training function fails, that is likely because you
+ * either passed too few samples, or a dictionary would not be effective
+ * for your data. Look at the messages that the dictionary trainer printed,
+ * if it doesn't say too few samples, then a dictionary would not be effective.
+ *
+ * How large should my dictionary be?
+ * ----------------------------------
+ *
+ * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB.
+ * The zstd CLI defaults to a 110KB dictionary. You likely don't need a
+ * dictionary larger than that. But, most use cases can get away with a
+ * smaller dictionary. The advanced dictionary builders can automatically
+ * shrink the dictionary for you, and select the smallest size that doesn't
+ * hurt compression ratio too much. See the `shrinkDict` parameter.
+ * A smaller dictionary can save memory, and potentially speed up
+ * compression.
+ *
+ * How many samples should I provide to the dictionary builder?
+ * ------------------------------------------------------------
+ *
+ * We generally recommend passing ~100x the size of the dictionary
+ * in samples. A few thousand should suffice. Having too few samples
+ * can hurt the dictionaries effectiveness. Having more samples will
+ * only improve the dictionaries effectiveness. But having too many
+ * samples can slow down the dictionary builder.
+ *
+ * How do I determine if a dictionary will be effective?
+ * -----------------------------------------------------
+ *
+ * Simply train a dictionary and try it out. You can use zstd's built in
+ * benchmarking tool to test the dictionary effectiveness.
+ *
+ * # Benchmark levels 1-3 without a dictionary
+ * zstd -b1e3 -r /path/to/my/files
+ * # Benchmark levels 1-3 with a dictionary
+ * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary
+ *
+ * When should I retrain a dictionary?
+ * -----------------------------------
+ *
+ * You should retrain a dictionary when its effectiveness drops. Dictionary
+ * effectiveness drops as the data you are compressing changes. Generally, we do
+ * expect dictionaries to "decay" over time, as your data changes, but the rate
+ * at which they decay depends on your use case. Internally, we regularly
+ * retrain dictionaries, and if the new dictionary performs significantly
+ * better than the old dictionary, we will ship the new dictionary.
+ *
+ * I have a raw content dictionary, how do I turn it into a zstd dictionary?
+ * -------------------------------------------------------------------------
+ *
+ * If you have a raw content dictionary, e.g. by manually constructing it, or
+ * using a third-party dictionary builder, you can turn it into a zstd
+ * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to
+ * provide some samples of the data. It will add the zstd header to the
+ * raw content, which contains a dictionary ID and entropy tables, which
+ * will improve compression ratio, and allow zstd to write the dictionary ID
+ * into the frame, if you so choose.
+ *
+ * Do I have to use zstd's dictionary builder?
+ * -------------------------------------------
+ *
+ * No! You can construct dictionary content however you please, it is just
+ * bytes. It will always be valid as a raw content dictionary. If you want
+ * a zstd dictionary, which can improve compression ratio, use
+ * `ZDICT_finalizeDictionary()`.
+ *
+ * What is the attack surface of a zstd dictionary?
+ * ------------------------------------------------
+ *
+ * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so
+ * zstd should never crash, or access out-of-bounds memory no matter what
+ * the dictionary is. However, if an attacker can control the dictionary
+ * during decompression, they can cause zstd to generate arbitrary bytes,
+ * just like if they controlled the compressed data.
+ *
+ ******************************************************************************/
+
+
+/*! ZDICT_trainFromBuffer():
+ * Train a dictionary from an array of samples.
+ * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
+ * f=20, and accel=1.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * Note: Dictionary training will fail if there are not enough samples to construct a
+ * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit).
+ * If dictionary training fails, you should use zstd without a dictionary, as the dictionary
+ * would've been ineffective anyways. If you believe your samples would benefit from a dictionary
+ * please open an issue with details, and we can look into it.
+ * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ */
+ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer,
+ const size_t* samplesSizes, unsigned nbSamples);
+
+typedef struct {
+ int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */
+ unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+ unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value)
+ * NOTE: The zstd format reserves some dictionary IDs for future use.
+ * You may use them in private settings, but be warned that they
+ * may be used by zstd in a public dictionary registry in the future.
+ * These dictionary IDs are:
+ * - low range : <= 32767
+ * - high range : >= (2^31)
+ */
+} ZDICT_params_t;
+
+/*! ZDICT_finalizeDictionary():
+ * Given a custom content as a basis for dictionary, and a set of samples,
+ * finalize dictionary by adding headers and statistics according to the zstd
+ * dictionary format.
+ *
+ * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each
+ * sample in order. The samples are used to construct the statistics, so they
+ * should be representative of what you will compress with this dictionary.
+ *
+ * The compression level can be set in `parameters`. You should pass the
+ * compression level you expect to use in production. The statistics for each
+ * compression level differ, so tuning the dictionary for the compression level
+ * can help quite a bit.
+ *
+ * You can set an explicit dictionary ID in `parameters`, or allow us to pick
+ * a random dictionary ID for you, but we can't guarantee no collisions.
+ *
+ * The dstDictBuffer and the dictContent may overlap, and the content will be
+ * appended to the end of the header. If the header + the content doesn't fit in
+ * maxDictSize the beginning of the content is truncated to make room, since it
+ * is presumed that the most profitable content is at the end of the dictionary,
+ * since that is the cheapest to reference.
+ *
+ * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
+ *
+ * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
+ * or an error code, which can be tested by ZDICT_isError().
+ * Note: ZDICT_finalizeDictionary() will push notifications into stderr if
+ * instructed to, using notificationLevel>0.
+ * NOTE: This function currently may fail in several edge cases including:
+ * * Not enough samples
+ * * Samples are uncompressible
+ * * Samples are all exactly the same
+ */
+ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
+ const void* dictContent, size_t dictContentSize,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t parameters);
+
+
+/*====== Helper functions ======*/
+ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
+ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
+ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
+ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
+
+#endif /* ZSTD_ZDICT_H */
+
+#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
+#define ZSTD_ZDICT_H_STATIC
+
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZDICTLIB_STATIC_API
+# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
+# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
+# else
+# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
+# endif
+#endif
+
+/* ====================================================================================
+ * The definitions in this section are considered experimental.
+ * They should never be used with a dynamic library, as they may change in the future.
+ * They are provided for advanced usages.
+ * Use them only in association with static linking.
+ * ==================================================================================== */
+
+#define ZDICT_DICTSIZE_MIN 256
+/* Deprecated: Remove in v1.6.0 */
+#define ZDICT_CONTENTSIZE_MIN 128
+
+/*! ZDICT_cover_params_t:
+ * k and d are the only required parameters.
+ * For others, value 0 means default.
+ */
+typedef struct {
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
+ ZDICT_params_t zParams;
+} ZDICT_cover_params_t;
+
+typedef struct {
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+ unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
+ unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
+
+ ZDICT_params_t zParams;
+} ZDICT_fastCover_params_t;
+
+/*! ZDICT_trainFromBuffer_cover():
+ * Train a dictionary from an array of samples using the COVER algorithm.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
+ void *dictBuffer, size_t dictBufferCapacity,
+ const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
+ ZDICT_cover_params_t parameters);
+
+/*! ZDICT_optimizeTrainFromBuffer_cover():
+ * The same requirements as above hold for all the parameters except `parameters`.
+ * This function tries many parameter combinations and picks the best parameters.
+ * `*parameters` is filled with the best parameters found,
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
+ *
+ * All of the parameters d, k, steps are optional.
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
+ * if steps is zero it defaults to its default value.
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
+ *
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * On success `*parameters` contains the parameters selected.
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
+ void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_cover_params_t* parameters);
+
+/*! ZDICT_trainFromBuffer_fastCover():
+ * Train a dictionary from an array of samples using a modified version of COVER algorithm.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * d and k are required.
+ * All other parameters are optional, will use default values if not provided
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
+ size_t dictBufferCapacity, const void *samplesBuffer,
+ const size_t *samplesSizes, unsigned nbSamples,
+ ZDICT_fastCover_params_t parameters);
+
+/*! ZDICT_optimizeTrainFromBuffer_fastCover():
+ * The same requirements as above hold for all the parameters except `parameters`.
+ * This function tries many parameter combinations (specifically, k and d combinations)
+ * and picks the best parameters. `*parameters` is filled with the best parameters found,
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
+ * All of the parameters d, k, steps, f, and accel are optional.
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
+ * if steps is zero it defaults to its default value.
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
+ * If f is zero, default value of 20 is used.
+ * If accel is zero, default value of 1 is used.
+ *
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * On success `*parameters` contains the parameters selected.
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
+ size_t dictBufferCapacity, const void* samplesBuffer,
+ const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_fastCover_params_t* parameters);
+
+typedef struct {
+ unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
+ ZDICT_params_t zParams;
+} ZDICT_legacy_params_t;
+
+/*! ZDICT_trainFromBuffer_legacy():
+ * Train a dictionary from an array of samples.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * `parameters` is optional and can be provided with values set to 0 to mean "default".
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
+ void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_legacy_params_t parameters);
+
+
+/* Deprecation warnings */
+/* It is generally possible to disable deprecation warnings from compiler,
+ for example with -Wno-deprecated-declarations for gcc
+ or _CRT_SECURE_NO_WARNINGS in Visual.
+ Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
+#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
+# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
+#else
+# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+# define ZDICT_DEPRECATED(message) [[deprecated(message)]]
+# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
+# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
+# elif (ZDICT_GCC_VERSION >= 301)
+# define ZDICT_DEPRECATED(message) __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
+# define ZDICT_DEPRECATED(message)
+# endif
+#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
+
+ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
+ZDICTLIB_STATIC_API
+size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
+
+
+#endif /* ZSTD_ZDICT_H_STATIC */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/zstd.h b/contrib/zstd/zstd.h
new file mode 100644
index 0000000..95aac07
--- /dev/null
+++ b/contrib/zstd/zstd.h
@@ -0,0 +1,2974 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef ZSTD_H_235446
+#define ZSTD_H_235446
+
+/* ====== Dependencies ======*/
+#include <limits.h> /* INT_MAX */
+#include <stddef.h> /* size_t */
+
+
+/* ===== ZSTDLIB_API : control library symbols visibility ===== */
+#ifndef ZSTDLIB_VISIBLE
+ /* Backwards compatibility with old macro name */
+# ifdef ZSTDLIB_VISIBILITY
+# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
+# else
+# define ZSTDLIB_VISIBLE
+# endif
+#endif
+
+#ifndef ZSTDLIB_HIDDEN
+# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+# else
+# define ZSTDLIB_HIDDEN
+# endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZSTDLIB_API ZSTDLIB_VISIBLE
+#endif
+
+/* Deprecation warnings :
+ * Should these warnings be a problem, it is generally possible to disable them,
+ * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
+ * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
+ */
+#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
+# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
+#else
+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+# define ZSTD_DEPRECATED(message) [[deprecated(message)]]
+# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
+# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
+# elif defined(__GNUC__) && (__GNUC__ >= 3)
+# define ZSTD_DEPRECATED(message) __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
+# define ZSTD_DEPRECATED(message)
+# endif
+#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
+
+
+/*******************************************************************************
+ Introduction
+
+ zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
+ real-time compression scenarios at zlib-level and better compression ratios.
+ The zstd compression library provides in-memory compression and decompression
+ functions.
+
+ The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
+ which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
+ caution, as they require more memory. The library also offers negative
+ compression levels, which extend the range of speed vs. ratio preferences.
+ The lower the level, the faster the speed (at the cost of compression).
+
+ Compression can be done in:
+ - a single step (described as Simple API)
+ - a single step, reusing a context (described as Explicit context)
+ - unbounded multiple steps (described as Streaming compression)
+
+ The compression ratio achievable on small data can be highly improved using
+ a dictionary. Dictionary compression can be performed in:
+ - a single step (described as Simple dictionary API)
+ - a single step, reusing a dictionary (described as Bulk-processing
+ dictionary API)
+
+ Advanced experimental functions can be accessed using
+ `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
+
+ Advanced experimental APIs should never be used with a dynamically-linked
+ library. They are not "stable"; their definitions or signatures may change in
+ the future. Only static linking is allowed.
+*******************************************************************************/
+
+/*------ Version ------*/
+#define ZSTD_VERSION_MAJOR 1
+#define ZSTD_VERSION_MINOR 5
+#define ZSTD_VERSION_RELEASE 4
+#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+
+/*! ZSTD_versionNumber() :
+ * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
+ZSTDLIB_API unsigned ZSTD_versionNumber(void);
+
+#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
+#define ZSTD_QUOTE(str) #str
+#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
+#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
+
+/*! ZSTD_versionString() :
+ * Return runtime library version, like "1.4.5". Requires v1.3.0+. */
+ZSTDLIB_API const char* ZSTD_versionString(void);
+
+/* *************************************
+ * Default constant
+ ***************************************/
+#ifndef ZSTD_CLEVEL_DEFAULT
+# define ZSTD_CLEVEL_DEFAULT 3
+#endif
+
+/* *************************************
+ * Constants
+ ***************************************/
+
+/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
+#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
+#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
+#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
+#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
+
+#define ZSTD_BLOCKSIZELOG_MAX 17
+#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
+
+
+/***************************************
+* Simple API
+***************************************/
+/*! ZSTD_compress() :
+ * Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
+ * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ * @return : compressed size written into `dst` (<= `dstCapacity),
+ * or an error code if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+
+/*! ZSTD_decompress() :
+ * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
+ * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
+ * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ * or an errorCode if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
+
+/*! ZSTD_getFrameContentSize() : requires v1.3.0+
+ * `src` should point to the start of a ZSTD encoded frame.
+ * `srcSize` must be at least as large as the frame header.
+ * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
+ * @return : - decompressed size of `src` frame content, if known
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
+ * note 1 : a 0 return value means the frame is valid but "empty".
+ * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * Optionally, application can rely on some implicit limit,
+ * as ZSTD_decompress() only needs an upper bound of decompressed size.
+ * (For example, data could be necessarily cut into blocks <= 16 KB).
+ * note 3 : decompressed size is always present when compression is completed using single-pass functions,
+ * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
+ * note 4 : decompressed size can be very large (64-bits value),
+ * potentially larger than what local system can handle as a single memory segment.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ * Always ensure return value fits within application's authorized limits.
+ * Each application can set its own limits.
+ * note 6 : This function replaces ZSTD_getDecompressedSize() */
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
+ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+
+/*! ZSTD_getDecompressedSize() :
+ * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
+ * Both functions work the same way, but ZSTD_getDecompressedSize() blends
+ * "empty", "unknown" and "error" results to the same return value (0),
+ * while ZSTD_getFrameContentSize() gives them separate return values.
+ * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
+ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
+ * `src` should point to the start of a ZSTD frame or skippable frame.
+ * `srcSize` must be >= first frame size
+ * @return : the compressed size of the first frame starting at `src`,
+ * suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ * or an error code if input is invalid */
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
+
+/*====== Helper functions ======*/
+/* ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ * In which case, ZSTD_compressBound() will return an error code
+ * which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
+ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
+ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
+ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
+ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
+
+
+/***************************************
+* Explicit context
+***************************************/
+/*= Compression context
+ * When compressing many times,
+ * it is recommended to allocate a context just once,
+ * and re-use it for each successive compression operation.
+ * This will make workload friendlier for system's memory.
+ * Note : re-using context is just a speed / resource optimization.
+ * It doesn't change the compression ratio, which remains identical.
+ * Note 2 : In multi-threaded environments,
+ * use one different context per thread for parallel execution.
+ */
+typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
+ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
+
+/*! ZSTD_compressCCtx() :
+ * Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
+ * Important : in order to behave similarly to `ZSTD_compress()`,
+ * this function compresses at requested compression level,
+ * __ignoring any other parameter__ .
+ * If any advanced parameter was set using the advanced API,
+ * they will all be reset. Only `compressionLevel` remains.
+ */
+ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+
+/*= Decompression context
+ * When decompressing many times,
+ * it is recommended to allocate a context only once,
+ * and re-use it for each successive compression operation.
+ * This will make workload friendlier for system's memory.
+ * Use one context per thread for parallel execution. */
+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
+ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */
+
+/*! ZSTD_decompressDCtx() :
+ * Same as ZSTD_decompress(),
+ * requires an allocated ZSTD_DCtx.
+ * Compatible with sticky parameters.
+ */
+ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize);
+
+
+/*********************************************
+* Advanced compression API (Requires v1.4.0+)
+**********************************************/
+
+/* API design :
+ * Parameters are pushed one by one into an existing context,
+ * using ZSTD_CCtx_set*() functions.
+ * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
+ * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
+ * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
+ *
+ * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
+ *
+ * This API supersedes all other "advanced" API entry points in the experimental section.
+ * In the future, we expect to remove from experimental API entry points which are redundant with this API.
+ */
+
+
+/* Compression strategies, listed from fastest to strongest */
+typedef enum { ZSTD_fast=1,
+ ZSTD_dfast=2,
+ ZSTD_greedy=3,
+ ZSTD_lazy=4,
+ ZSTD_lazy2=5,
+ ZSTD_btlazy2=6,
+ ZSTD_btopt=7,
+ ZSTD_btultra=8,
+ ZSTD_btultra2=9
+ /* note : new strategies _might_ be added in the future.
+ Only the order (from fast to strong) is guaranteed */
+} ZSTD_strategy;
+
+typedef enum {
+
+ /* compression parameters
+ * Note: When compressing with a ZSTD_CDict these parameters are superseded
+ * by the parameters used to construct the ZSTD_CDict.
+ * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
+ ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
+ * Note that exact compression parameters are dynamically determined,
+ * depending on both compression level and srcSize (when known).
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
+ * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
+ * Note 1 : it's possible to pass a negative compression level.
+ * Note 2 : setting a level does not automatically set all other compression parameters
+ * to default. Setting this will however eventually dynamically impact the compression
+ * parameters which have not been manually set. The manually set
+ * ones will 'stick'. */
+ /* Advanced compression parameters :
+ * It's possible to pin down compression parameters to some specific values.
+ * In which case, these values are no longer dynamically selected by the compressor */
+ ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
+ * This will set a memory budget for streaming decompression,
+ * with larger values requiring more memory
+ * and typically compressing more.
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+ * Special: value 0 means "use default windowLog".
+ * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
+ * requires explicitly allowing such size at streaming decompression stage. */
+ ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
+ * Resulting memory usage is (1 << (hashLog+2)).
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+ * Larger tables improve compression ratio of strategies <= dFast,
+ * and improve speed of strategies > dFast.
+ * Special: value 0 means "use default hashLog". */
+ ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
+ * Resulting memory usage is (1 << (chainLog+2)).
+ * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
+ * Larger tables result in better and slower compression.
+ * This parameter is useless for "fast" strategy.
+ * It's still useful when using "dfast" strategy,
+ * in which case it defines a secondary probe table.
+ * Special: value 0 means "use default chainLog". */
+ ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
+ * More attempts result in better and slower compression.
+ * This parameter is useless for "fast" and "dFast" strategies.
+ * Special: value 0 means "use default searchLog". */
+ ZSTD_c_minMatch=105, /* Minimum size of searched matches.
+ * Note that Zstandard can still find matches of smaller size,
+ * it just tweaks its search algorithm to look for this size and larger.
+ * Larger values increase compression and decompression speed, but decrease ratio.
+ * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
+ * , for all strategies > fast, effective maximum is 6.
+ * Special: value 0 means "use default minMatchLength". */
+ ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
+ * For strategies btopt, btultra & btultra2:
+ * Length of Match considered "good enough" to stop search.
+ * Larger values make compression stronger, and slower.
+ * For strategy fast:
+ * Distance between match sampling.
+ * Larger values make compression faster, and weaker.
+ * Special: value 0 means "use default targetLength". */
+ ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
+ * The higher the value of selected strategy, the more complex it is,
+ * resulting in stronger and slower compression.
+ * Special: value 0 means "use default strategy". */
+ /* LDM mode parameters */
+ ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
+ * This parameter is designed to improve compression ratio
+ * for large inputs, by finding large matches at long distance.
+ * It increases memory usage and window size.
+ * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
+ * except when expressly set to a different value.
+ * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
+ * compression strategy >= ZSTD_btopt (== compression level 16+) */
+ ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
+ * Larger values increase memory usage and compression ratio,
+ * but decrease compression speed.
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+ * default: windowlog - 7.
+ * Special: value 0 means "automatically determine hashlog". */
+ ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
+ * Larger/too small values usually decrease compression ratio.
+ * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
+ * Special: value 0 means "use default value" (default: 64). */
+ ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
+ * Larger values improve collision resolution but decrease compression speed.
+ * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
+ * Special: value 0 means "use default value" (default: 3). */
+ ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
+ * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
+ * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
+ * Larger values improve compression speed.
+ * Deviating far from default value will likely result in a compression ratio decrease.
+ * Special: value 0 means "automatically determine hashRateLog". */
+
+ /* frame parameters */
+ ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+ * Content size must be known at the beginning of compression.
+ * This is automatically the case when using ZSTD_compress2(),
+ * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
+ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
+ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
+
+ /* multi-threading parameters */
+ /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+ * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
+ * In a situation where it's unknown if the linked library supports multi-threading or not,
+ * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
+ */
+ ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
+ * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
+ * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
+ * while compression is performed in parallel, within worker thread(s).
+ * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
+ * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
+ * More workers improve speed, but also increase memory usage.
+ * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
+ * compression is performed inside Caller's thread, and all invocations are blocking */
+ ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
+ * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
+ * 0 means default, which is dynamically determined based on compression parameters.
+ * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
+ * The minimum size is automatically and transparently enforced. */
+ ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
+ * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
+ * It helps preserve compression ratio, while each job is compressed in parallel.
+ * This value is enforced only when nbWorkers >= 1.
+ * Larger values increase compression ratio, but decrease speed.
+ * Possible values range from 0 to 9 :
+ * - 0 means "default" : value will be determined by the library, depending on strategy
+ * - 1 means "no overlap"
+ * - 9 means "full overlap", using a full window size.
+ * Each intermediate rank increases/decreases load size by a factor 2 :
+ * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
+ * default value varies between 6 and 9, depending on strategy */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_c_rsyncable
+ * ZSTD_c_format
+ * ZSTD_c_forceMaxWindow
+ * ZSTD_c_forceAttachDict
+ * ZSTD_c_literalCompressionMode
+ * ZSTD_c_targetCBlockSize
+ * ZSTD_c_srcSizeHint
+ * ZSTD_c_enableDedicatedDictSearch
+ * ZSTD_c_stableInBuffer
+ * ZSTD_c_stableOutBuffer
+ * ZSTD_c_blockDelimiters
+ * ZSTD_c_validateSequences
+ * ZSTD_c_useBlockSplitter
+ * ZSTD_c_useRowMatchFinder
+ * ZSTD_c_prefetchCDictTables
+ * ZSTD_c_enableSeqProducerFallback
+ * ZSTD_c_maxBlockSize
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly;
+ * also, the enums values themselves are unstable and can still change.
+ */
+ ZSTD_c_experimentalParam1=500,
+ ZSTD_c_experimentalParam2=10,
+ ZSTD_c_experimentalParam3=1000,
+ ZSTD_c_experimentalParam4=1001,
+ ZSTD_c_experimentalParam5=1002,
+ ZSTD_c_experimentalParam6=1003,
+ ZSTD_c_experimentalParam7=1004,
+ ZSTD_c_experimentalParam8=1005,
+ ZSTD_c_experimentalParam9=1006,
+ ZSTD_c_experimentalParam10=1007,
+ ZSTD_c_experimentalParam11=1008,
+ ZSTD_c_experimentalParam12=1009,
+ ZSTD_c_experimentalParam13=1010,
+ ZSTD_c_experimentalParam14=1011,
+ ZSTD_c_experimentalParam15=1012,
+ ZSTD_c_experimentalParam16=1013,
+ ZSTD_c_experimentalParam17=1014,
+ ZSTD_c_experimentalParam18=1015,
+ ZSTD_c_experimentalParam19=1016
+} ZSTD_cParameter;
+
+typedef struct {
+ size_t error;
+ int lowerBound;
+ int upperBound;
+} ZSTD_bounds;
+
+/*! ZSTD_cParam_getBounds() :
+ * All parameters must belong to an interval with lower and upper bounds,
+ * otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ * - an error status field, which must be tested using ZSTD_isError()
+ * - lower and upper bounds, both inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
+
+/*! ZSTD_CCtx_setParameter() :
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ * Setting a parameter is generally only possible during frame initialization (before starting compression).
+ * Exception : when using multi-threading mode (nbWorkers >= 1),
+ * the following parameters can be updated _during_ compression (within same frame):
+ * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
+ * new parameters will be active for next job only (after a flush()).
+ * @return : an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtx_setPledgedSrcSize() :
+ * Total input data size to be compressed as a single frame.
+ * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+ * This value will also be controlled at end of frame, and trigger an error if not respected.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
+ * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
+ * Note 2 : pledgedSrcSize is only valid once, for the next frame.
+ * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
+ * Note 3 : Whenever all input data is provided and consumed in a single round,
+ * for example with ZSTD_compress2(),
+ * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
+ * this value is automatically overridden by srcSize instead.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
+typedef enum {
+ ZSTD_reset_session_only = 1,
+ ZSTD_reset_parameters = 2,
+ ZSTD_reset_session_and_parameters = 3
+} ZSTD_ResetDirective;
+
+/*! ZSTD_CCtx_reset() :
+ * There are 2 different things that can be reset, independently or jointly :
+ * - The session : will stop compressing current frame, and make CCtx ready to start a new one.
+ * Useful after an error, or to interrupt any ongoing compression.
+ * Any internal data not yet flushed is cancelled.
+ * Compression parameters and dictionary remain unchanged.
+ * They will be used to compress next frame.
+ * Resetting session never fails.
+ * - The parameters : changes all parameters back to "default".
+ * This also removes any reference to any dictionary or external sequence producer.
+ * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
+ * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
+ * - Both : similar to resetting the session, followed by resetting parameters.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
+
+/*! ZSTD_compress2() :
+ * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+ * ZSTD_compress2() always starts a new frame.
+ * Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ * - The function is always blocking, returns when compression is completed.
+ * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ * @return : compressed size written into `dst` (<= `dstCapacity),
+ * or an error code if it fails (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize);
+
+
+/***********************************************
+* Advanced decompression API (Requires v1.4.0+)
+************************************************/
+
+/* The advanced API pushes parameters one by one into an existing DCtx context.
+ * Parameters are sticky, and remain valid for all following frames
+ * using the same DCtx context.
+ * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
+ * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
+ * Therefore, no new decompression function is necessary.
+ */
+
+typedef enum {
+
+ ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
+ * the streaming API will refuse to allocate memory buffer
+ * in order to protect the host from unreasonable memory requirements.
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
+ * Special: value 0 means "use default maximum windowLog". */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_d_format
+ * ZSTD_d_stableOutBuffer
+ * ZSTD_d_forceIgnoreChecksum
+ * ZSTD_d_refMultipleDDicts
+ * ZSTD_d_disableHuffmanAssembly
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly
+ */
+ ZSTD_d_experimentalParam1=1000,
+ ZSTD_d_experimentalParam2=1001,
+ ZSTD_d_experimentalParam3=1002,
+ ZSTD_d_experimentalParam4=1003,
+ ZSTD_d_experimentalParam5=1004
+
+} ZSTD_dParameter;
+
+/*! ZSTD_dParam_getBounds() :
+ * All parameters must belong to an interval with lower and upper bounds,
+ * otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ * - an error status field, which must be tested using ZSTD_isError()
+ * - both lower and upper bounds, inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
+
+/*! ZSTD_DCtx_setParameter() :
+ * Set one compression parameter, selected by enum ZSTD_dParameter.
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ * Setting a parameter is only possible during frame initialization (before starting decompression).
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
+
+/*! ZSTD_DCtx_reset() :
+ * Return a DCtx to clean state.
+ * Session and parameters can be reset jointly or separately.
+ * Parameters can only be reset when no active frame is being decompressed.
+ * @return : 0, or an error code, which can be tested with ZSTD_isError()
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
+
+
+/****************************
+* Streaming
+****************************/
+
+typedef struct ZSTD_inBuffer_s {
+ const void* src; /**< start of input buffer */
+ size_t size; /**< size of input buffer */
+ size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+ void* dst; /**< start of output buffer */
+ size_t size; /**< size of output buffer */
+ size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+
+
+/*-***********************************************************************
+* Streaming compression - HowTo
+*
+* A ZSTD_CStream object is required to track streaming operation.
+* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+*
+* For parallel execution, use one separate ZSTD_CStream per thread.
+*
+* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+*
+* Parameters are sticky : when starting a new compression on the same context,
+* it will re-use the same sticky parameters as previous compression session.
+* When in doubt, it's recommended to fully initialize the context before usage.
+* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
+* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
+* set more specific parameters, the pledged source size, or load a dictionary.
+*
+* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
+* consume input stream. The function will automatically update both `pos`
+* fields within `input` and `output`.
+* Note that the function may not consume the entire input, for example, because
+* the output buffer is already full, in which case `input.pos < input.size`.
+* The caller must check if input has been entirely consumed.
+* If not, the caller must make some room to receive more compressed data,
+* and then present again remaining input data.
+* note: ZSTD_e_continue is guaranteed to make some forward progress when called,
+* but doesn't guarantee maximal forward progress. This is especially relevant
+* when compressing with multiple threads. The call won't block if it can
+* consume some input, but if it can't it will wait for some, but not all,
+* output to be flushed.
+* @return : provides a minimum amount of data remaining to be flushed from internal buffers
+* or an error code, which can be tested using ZSTD_isError().
+*
+* At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
+* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
+* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
+* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
+* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
+* operation.
+* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
+* block until the flush is complete or the output buffer is full.
+* @return : 0 if internal buffers are entirely flushed,
+* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+* or an error code, which can be tested using ZSTD_isError().
+*
+* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
+* It will perform a flush and write frame epilogue.
+* The epilogue is required for decoders to consider a frame completed.
+* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
+* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
+* start a new frame.
+* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
+* block until the flush is complete or the output buffer is full.
+* @return : 0 if frame fully completed and fully flushed,
+* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+* or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************/
+
+typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
+ /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
+/*===== ZSTD_CStream management functions =====*/
+ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
+ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */
+
+/*===== Streaming compression functions =====*/
+typedef enum {
+ ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+ ZSTD_e_flush=1, /* flush any data provided so far,
+ * it creates (at least) one new block, that can be decoded immediately on reception;
+ * frame will continue: any future data can still reference previously compressed data, improving compression.
+ * note : multithreaded compression will block to flush as much output as possible. */
+ ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
+ * note that frame is only closed after compressed data is fully flushed (return value == 0).
+ * After that point, any additional data starts a new frame.
+ * note : each frame is independent (does not reference any content from previous frame).
+ : note : multithreaded compression will block to flush as much output as possible. */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream2() : Requires v1.4.0+
+ * Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+ * - output->pos must be <= dstCapacity, input->pos must be <= srcSize
+ * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ * - endOp must be a valid directive
+ * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
+ * and then immediately returns, just indicating that there is some data remaining to be flushed.
+ * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+ * - @return provides a minimum amount of data remaining to be flushed from internal buffers
+ * or an error code, which can be tested using ZSTD_isError().
+ * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+ * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+ * Before starting a new compression job, or changing compression parameters,
+ * it is required to fully flush internal buffers.
+ */
+ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective endOp);
+
+
+/* These buffer sizes are softly recommended.
+ * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
+ * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
+ * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
+ *
+ * However, note that these recommendations are from the perspective of a C caller program.
+ * If the streaming interface is invoked from some other language,
+ * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
+ * a major performance rule is to reduce crossing such interface to an absolute minimum.
+ * It's not rare that performance ends being spent more into the interface, rather than compression itself.
+ * In which cases, prefer using large buffers, as large as practical,
+ * for both input and output, to reduce the nb of roundtrips.
+ */
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
+
+
+/* *****************************************************************************
+ * This following is a legacy streaming API, available since v1.0+ .
+ * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
+ * It is redundant, but remains fully supported.
+ ******************************************************************************/
+
+/*!
+ * Equivalent to:
+ *
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ * to compress with a dictionary.
+ */
+ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+/*!
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
+ */
+ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
+ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
+ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+
+/*-***************************************************************************
+* Streaming decompression - HowTo
+*
+* A ZSTD_DStream object is required to track streaming operations.
+* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+* ZSTD_DStream objects can be re-used multiple times.
+*
+* Use ZSTD_initDStream() to start a new decompression operation.
+* @return : recommended first input size
+* Alternatively, use advanced API to set specific properties.
+*
+* Use ZSTD_decompressStream() repetitively to consume your input.
+* The function will update both `pos` fields.
+* If `input.pos < input.size`, some input has not been consumed.
+* It's up to the caller to present again remaining data.
+* The function tries to flush all data decoded immediately, respecting output buffer size.
+* If `output.pos < output.size`, decoder has flushed everything it could.
+* But if `output.pos == output.size`, there might be some data left within internal buffers.,
+* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+* @return : 0 when a frame is completely decoded and fully flushed,
+* or an error code, which can be tested using ZSTD_isError(),
+* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
+* the return value is a suggested next input size (just a hint for better latency)
+* that will never request more than the remaining frame size.
+* *******************************************************************************/
+
+typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+ /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
+/*===== ZSTD_DStream management functions =====*/
+ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
+ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */
+
+/*===== Streaming decompression functions =====*/
+
+/*! ZSTD_initDStream() :
+ * Initialize/reset DStream state for new decompression operation.
+ * Call before new decompression operation using same DStream.
+ *
+ * Note : This function is redundant with the advanced API and equivalent to:
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_refDDict(zds, NULL);
+ */
+ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+
+/*! ZSTD_decompressStream() :
+ * Streaming decompression function.
+ * Call repetitively to consume full input updating it as necessary.
+ * Function will update both input and output `pos` fields exposing current state via these fields:
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
+ * on the next call.
+ * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+ * call ZSTD_decompressStream() again to flush remaining data to output.
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * @return : 0 when a frame is completely decoded and fully flushed,
+ * or an error code, which can be tested using ZSTD_isError(),
+ * or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ */
+ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+
+
+/**************************
+* Simple dictionary API
+***************************/
+/*! ZSTD_compress_usingDict() :
+ * Compression at an explicit compression level using a Dictionary.
+ * A dictionary can be any arbitrary data segment (also called a prefix),
+ * or a buffer with specified information (see zdict.h).
+ * Note : This function loads the dictionary, resulting in significant startup delay.
+ * It's intended for a dictionary used only once.
+ * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ int compressionLevel);
+
+/*! ZSTD_decompress_usingDict() :
+ * Decompression using a known Dictionary.
+ * Dictionary must be identical to the one used during compression.
+ * Note : This function loads the dictionary, resulting in significant startup delay.
+ * It's intended for a dictionary used only once.
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
+
+
+/***********************************
+ * Bulk processing dictionary API
+ **********************************/
+typedef struct ZSTD_CDict_s ZSTD_CDict;
+
+/*! ZSTD_createCDict() :
+ * When compressing multiple messages or blocks using the same dictionary,
+ * it's recommended to digest the dictionary only once, since it's a costly operation.
+ * ZSTD_createCDict() will create a state from digesting a dictionary.
+ * The resulting state can be used for future compression operations with very limited startup cost.
+ * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+ * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+ * in which case the only thing that it transports is the @compressionLevel.
+ * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+ * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+ int compressionLevel);
+
+/*! ZSTD_freeCDict() :
+ * Function frees memory allocated by ZSTD_createCDict().
+ * If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
+
+/*! ZSTD_compress_usingCDict() :
+ * Compression using a digested Dictionary.
+ * Recommended when same dictionary is used multiple times.
+ * Note : compression level is _decided at dictionary creation time_,
+ * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
+ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict);
+
+
+typedef struct ZSTD_DDict_s ZSTD_DDict;
+
+/*! ZSTD_createDDict() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_freeDDict() :
+ * Function frees memory allocated with ZSTD_createDDict()
+ * If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
+
+/*! ZSTD_decompress_usingDDict() :
+ * Decompression using a digested Dictionary.
+ * Recommended when same dictionary is used multiple times. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_DDict* ddict);
+
+
+/********************************
+ * Dictionary helper functions
+ *******************************/
+
+/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
+ * Provides the dictID of the dictionary loaded into `cdict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
+
+/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
+ * Provides the dictID required to decompressed the frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary to be decoded (most common case).
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
+
+/*******************************************************************************
+ * Advanced dictionary and prefix API (Requires v1.4.0+)
+ *
+ * This API allows dictionaries to be used with ZSTD_compress2(),
+ * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
+ * only reset with the context is reset with ZSTD_reset_parameters or
+ * ZSTD_reset_session_and_parameters. Prefixes are single-use.
+ ******************************************************************************/
+
+
+/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
+ * Create an internal CDict from `dict` buffer.
+ * Decompression will have to use same dictionary.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
+ * meaning "return to no-dictionary mode".
+ * Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ * until parameters are reset, a new dictionary is loaded, or the dictionary
+ * is explicitly invalidated by loading a NULL dictionary.
+ * Note 2 : Loading a dictionary involves building tables.
+ * It's also a CPU consuming operation, with non-negligible impact on latency.
+ * Tables are dependent on compression parameters, and for this reason,
+ * compression parameters can no longer be changed after loading a dictionary.
+ * Note 3 :`dict` content will be copied internally.
+ * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
+ * In such a case, dictionary buffer must outlive its users.
+ * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
+ * to precisely select how dictionary content must be interpreted. */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
+ * Reference a prepared dictionary, to be used for all future compressed frames.
+ * Note that compression parameters are enforced from within CDict,
+ * and supersede any compression parameter previously set within CCtx.
+ * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+ * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
+ * The dictionary will remain valid for future compressed frames using same CCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special : Referencing a NULL CDict means "return to no-dictionary mode".
+ * Note 1 : Currently, only one dictionary can be managed.
+ * Referencing a new dictionary effectively "discards" any previous one.
+ * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
+ * Reference a prefix (single-usage dictionary) for next compressed frame.
+ * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
+ * Decompression will need same prefix to properly regenerate data.
+ * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ * but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
+ * Note 1 : Prefix buffer is referenced. It **must** outlive compression.
+ * Its content must remain unmodified during compression.
+ * Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ * ensure that the window size is large enough to contain the entire source.
+ * See ZSTD_c_windowLog.
+ * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ * It's a CPU consuming operation, with non-negligible impact on latency.
+ * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
+ * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
+ * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
+ const void* prefix, size_t prefixSize);
+
+/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
+ * Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ * The dictionary remains valid for all future frames, until explicitly invalidated, or
+ * a new dictionary is loaded.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ * meaning "return to no-dictionary mode".
+ * Note 1 : Loading a dictionary involves building tables,
+ * which has a non-negligible impact on CPU usage and latency.
+ * It's recommended to "load once, use many times", to amortize the cost
+ * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
+ * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
+ * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
+ * how dictionary content is loaded and interpreted.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
+ * Reference a prepared dictionary, to be used to decompress next frames.
+ * The dictionary remains active for decompression of future frames using same DCtx.
+ *
+ * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
+ * will store the DDict references in a table, and the DDict used for decompression
+ * will be determined at decompression time, as per the dict ID in the frame.
+ * The memory for the table is allocated on the first call to refDDict, and can be
+ * freed with ZSTD_freeDCtx().
+ *
+ * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ * will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: referencing a NULL DDict means "return to no-dictionary mode".
+ * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
+ * Reference a prefix (single-usage dictionary) to decompress next frame.
+ * This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ * and must use the same prefix as the one used during compression.
+ * Prefix is **only used once**. Reference is discarded at end of frame.
+ * End of frame is reached when ZSTD_decompressStream() returns 0.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
+ * Prefix buffer must remain unmodified up to the end of frame,
+ * reached when ZSTD_decompressStream() returns 0.
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
+ * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
+ * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ * A full dictionary is more costly, as it requires building tables.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
+ const void* prefix, size_t prefixSize);
+
+/* === Memory management === */
+
+/*! ZSTD_sizeof_*() : Requires v1.4.0+
+ * These functions give the _current_ memory usage of selected object.
+ * Note that object memory usage can evolve (increase or decrease) over time. */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+#endif /* ZSTD_H_235446 */
+
+
+/* **************************************************************************************
+ * ADVANCED AND EXPERIMENTAL FUNCTIONS
+ ****************************************************************************************
+ * The definitions in the following section are considered experimental.
+ * They are provided for advanced scenarios.
+ * They should never be used with a dynamic library, as prototypes may change in the future.
+ * Use them only in association with static linking.
+ * ***************************************************************************************/
+
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZSTDLIB_STATIC_API
+# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
+# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
+# else
+# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
+# endif
+#endif
+
+/****************************************************************************************
+ * experimental API (static linking only)
+ ****************************************************************************************
+ * The following symbols and constants
+ * are not planned to join "stable API" status in the near future.
+ * They can still change in future versions.
+ * Some of them are planned to remain in the static_only section indefinitely.
+ * Some of them might be removed in the future (especially when redundant with existing stable functions)
+ * ***************************************************************************************/
+
+#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
+#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
+#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
+#define ZSTD_SKIPPABLEHEADERSIZE 8
+
+/* compression parameter bounds */
+#define ZSTD_WINDOWLOG_MAX_32 30
+#define ZSTD_WINDOWLOG_MAX_64 31
+#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MIN 10
+#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
+#define ZSTD_HASHLOG_MIN 6
+#define ZSTD_CHAINLOG_MAX_32 29
+#define ZSTD_CHAINLOG_MAX_64 30
+#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
+#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
+#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN 1
+#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
+#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
+#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
+#define ZSTD_STRATEGY_MIN ZSTD_fast
+#define ZSTD_STRATEGY_MAX ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
+
+
+#define ZSTD_OVERLAPLOG_MIN 0
+#define ZSTD_OVERLAPLOG_MAX 9
+
+#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
+ * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
+ * to preserve host's memory from unreasonable requirements.
+ * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
+ * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
+
+
+/* LDM parameter bounds */
+#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
+#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
+#define ZSTD_LDM_MINMATCH_MIN 4
+#define ZSTD_LDM_MINMATCH_MAX 4096
+#define ZSTD_LDM_BUCKETSIZELOG_MIN 1
+#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
+#define ZSTD_LDM_HASHRATELOG_MIN 0
+#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
+
+/* Advanced parameter bounds */
+#define ZSTD_TARGETCBLOCKSIZE_MIN 64
+#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
+#define ZSTD_SRCSIZEHINT_MIN 0
+#define ZSTD_SRCSIZEHINT_MAX INT_MAX
+
+
+/* --- Advanced types --- */
+
+typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
+
+typedef struct {
+ unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
+ * If offset == 0 and matchLength == 0, this sequence represents the last
+ * literals in the block of litLength size.
+ */
+
+ unsigned int litLength; /* Literal length of the sequence. */
+ unsigned int matchLength; /* Match length of the sequence. */
+
+ /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
+ * In this case, we will treat the sequence as a marker for a block boundary.
+ */
+
+ unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
+ * Ranges from [0, 3].
+ *
+ * Repeat offsets are essentially previous offsets from previous sequences sorted in
+ * recency order. For more detail, see doc/zstd_compression_format.md
+ *
+ * If rep == 0, then 'offset' does not contain a repeat offset.
+ * If rep > 0:
+ * If litLength != 0:
+ * rep == 1 --> offset == repeat_offset_1
+ * rep == 2 --> offset == repeat_offset_2
+ * rep == 3 --> offset == repeat_offset_3
+ * If litLength == 0:
+ * rep == 1 --> offset == repeat_offset_2
+ * rep == 2 --> offset == repeat_offset_3
+ * rep == 3 --> offset == repeat_offset_1 - 1
+ *
+ * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
+ * 'rep', but repeat offsets do not necessarily need to be calculated from an external
+ * sequence provider's perspective. For example, ZSTD_compressSequences() does not
+ * use this 'rep' field at all (as of now).
+ */
+} ZSTD_Sequence;
+
+typedef struct {
+ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
+ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+ unsigned hashLog; /**< dispatch table : larger == faster, more memory */
+ unsigned searchLog; /**< nb of searches : larger == more compression, slower */
+ unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */
+ unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+ ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
+} ZSTD_compressionParameters;
+
+typedef struct {
+ int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+ int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
+ int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
+} ZSTD_frameParameters;
+
+typedef struct {
+ ZSTD_compressionParameters cParams;
+ ZSTD_frameParameters fParams;
+} ZSTD_parameters;
+
+typedef enum {
+ ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
+ ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
+ ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
+} ZSTD_dictContentType_e;
+
+typedef enum {
+ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
+ ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
+} ZSTD_dictLoadMethod_e;
+
+typedef enum {
+ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
+ ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
+ * Useful to save 4 bytes per generated frame.
+ * Decoder cannot recognise automatically this format, requiring this instruction. */
+} ZSTD_format_e;
+
+typedef enum {
+ /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
+ ZSTD_d_validateChecksum = 0,
+ ZSTD_d_ignoreChecksum = 1
+} ZSTD_forceIgnoreChecksum_e;
+
+typedef enum {
+ /* Note: this enum controls ZSTD_d_refMultipleDDicts */
+ ZSTD_rmd_refSingleDDict = 0,
+ ZSTD_rmd_refMultipleDDicts = 1
+} ZSTD_refMultipleDDicts_e;
+
+typedef enum {
+ /* Note: this enum and the behavior it controls are effectively internal
+ * implementation details of the compressor. They are expected to continue
+ * to evolve and should be considered only in the context of extremely
+ * advanced performance tuning.
+ *
+ * Zstd currently supports the use of a CDict in three ways:
+ *
+ * - The contents of the CDict can be copied into the working context. This
+ * means that the compression can search both the dictionary and input
+ * while operating on a single set of internal tables. This makes
+ * the compression faster per-byte of input. However, the initial copy of
+ * the CDict's tables incurs a fixed cost at the beginning of the
+ * compression. For small compressions (< 8 KB), that copy can dominate
+ * the cost of the compression.
+ *
+ * - The CDict's tables can be used in-place. In this model, compression is
+ * slower per input byte, because the compressor has to search two sets of
+ * tables. However, this model incurs no start-up cost (as long as the
+ * working context's tables can be reused). For small inputs, this can be
+ * faster than copying the CDict's tables.
+ *
+ * - The CDict's tables are not used at all, and instead we use the working
+ * context alone to reload the dictionary and use params based on the source
+ * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+ * This method is effective when the dictionary sizes are very small relative
+ * to the input size, and the input size is fairly large to begin with.
+ *
+ * Zstd has a simple internal heuristic that selects which strategy to use
+ * at the beginning of a compression. However, if experimentation shows that
+ * Zstd is making poor choices, it is possible to override that choice with
+ * this enum.
+ */
+ ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
+ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
+ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
+ ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
+} ZSTD_dictAttachPref_e;
+
+typedef enum {
+ ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
+ * Negative compression levels will be uncompressed, and positive compression
+ * levels will be compressed. */
+ ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
+ * emitted if Huffman compression is not profitable. */
+ ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
+} ZSTD_literalCompressionMode_e;
+
+typedef enum {
+ /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
+ * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
+ * or ZSTD_ps_disable allow for a force enable/disable the feature.
+ */
+ ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
+ ZSTD_ps_enable = 1, /* Force-enable the feature */
+ ZSTD_ps_disable = 2 /* Do not use the feature */
+} ZSTD_paramSwitch_e;
+
+/***************************************
+* Frame size functions
+***************************************/
+
+/*! ZSTD_findDecompressedSize() :
+ * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ * `srcSize` must be the _exact_ size of this series
+ * (i.e. there should be a frame boundary at `src + srcSize`)
+ * @return : - decompressed size of all data in all successive frames
+ * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ * note 3 : decompressed size can be very large (64-bits value),
+ * potentially larger than what local system can handle as a single memory segment.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ * Always ensure result fits within application's authorized limits.
+ * Each application can set its own limits.
+ * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+ * read each contained frame header. This is fast as most of the data is skipped,
+ * however it does mean that all frame data must be present and valid. */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_decompressBound() :
+ * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ * `srcSize` must be the _exact_ size of this series
+ * (i.e. there should be a frame boundary at `src + srcSize`)
+ * @return : - upper-bound for the decompressed size of all data in all successive frames
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
+ * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
+ * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
+ * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
+ * upper-bound = # blocks * min(128 KB, Window_Size)
+ */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
+
+/*! ZSTD_frameHeaderSize() :
+ * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
+ * @return : size of the Frame Header,
+ * or an error code (if srcSize is too small) */
+ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+
+/*! ZSTD_decompressionMargin() :
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ * and the input buffer must be at the end of the output buffer.
+ *
+ * _______________________ Output Buffer ________________________
+ * | |
+ * | ____ Input Buffer ____|
+ * | | |
+ * v v v
+ * |---------------------------------------|-----------|----------|
+ * ^ ^ ^
+ * |___________________ Output_Size ___________________|_ Margin _|
+ *
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ * ZSTD_decompressDCtx().
+ * NOTE: This function supports multi-frame input.
+ *
+ * @param src The compressed frame(s)
+ * @param srcSize The size of the compressed frame(s)
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
+/*! ZSTD_DECOMPRESS_MARGIN() :
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ * the compressed frame, compute it from the original size and the blockSizeLog.
+ * See ZSTD_decompressionMargin() for details.
+ *
+ * WARNING: This macro does not support multi-frame input, the input must be a single
+ * zstd frame. If you need that support use the function, or implement it yourself.
+ *
+ * @param originalSize The original uncompressed size of the data.
+ * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+ * Unless you explicitly set the windowLog smaller than
+ * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ */
+#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \
+ ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \
+ 4 /* checksum */ + \
+ ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+ (blockSize) /* One block of margin */ \
+ ))
+
+typedef enum {
+ ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
+ ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
+} ZSTD_sequenceFormat_e;
+
+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ * @return : upper-bound for the number of sequences that can be generated
+ * from a buffer of srcSize bytes
+ *
+ * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
+/*! ZSTD_generateSequences() :
+ * Generate sequences using ZSTD_compress2(), given a source buffer.
+ *
+ * Each block will end with a dummy sequence
+ * with offset == 0, matchLength == 0, and litLength == length of last literals.
+ * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
+ * simply acts as a block delimiter.
+ *
+ * @zc can be used to insert custom compression params.
+ * This function invokes ZSTD_compress2().
+ *
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
+ * @return : number of sequences generated
+ */
+
+ZSTDLIB_STATIC_API size_t
+ZSTD_generateSequences( ZSTD_CCtx* zc,
+ ZSTD_Sequence* outSeqs, size_t outSeqsSize,
+ const void* src, size_t srcSize);
+
+/*! ZSTD_mergeBlockDelimiters() :
+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
+ * by merging them into the literals of the next sequence.
+ *
+ * As such, the final generated result has no explicit representation of block boundaries,
+ * and the final last literals segment is not represented in the sequences.
+ *
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
+ * @return : number of sequences left after merging
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
+
+/*! ZSTD_compressSequences() :
+ * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
+ * @src contains the entire input (not just the literals).
+ * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
+ * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
+ * The entire source is compressed into a single frame.
+ *
+ * The compression behavior changes based on cctx params. In particular:
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
+ * the block size derived from the cctx, and sequences may be split. This is the default setting.
+ *
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
+ *
+ * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
+ * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
+ * specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
+ *
+ * In addition to the two adjustable experimental params, there are other important cctx params.
+ * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
+ * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
+ * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
+ * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
+ *
+ * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
+ * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
+ * and cannot emit an RLE block that disagrees with the repcode history
+ * @return : final compressed size, or a ZSTD error code.
+ */
+ZSTDLIB_STATIC_API size_t
+ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ const void* src, size_t srcSize);
+
+
+/*! ZSTD_writeSkippableFrame() :
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, if the source size is not representable
+ * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, unsigned magicVariant);
+
+/*! ZSTD_readSkippableFrame() :
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
+ * in the magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
+ const void* src, size_t srcSize);
+
+/*! ZSTD_isSkippableFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ */
+ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
+
+
+
+/***************************************
+* Memory management
+***************************************/
+
+/*! ZSTD_estimate*() :
+ * These functions make it possible to estimate memory usage
+ * of a future {D,C}Ctx, before its creation.
+ *
+ * ZSTD_estimateCCtxSize() will provide a memory budget large enough
+ * for any compression level up to selected one.
+ * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
+ * does not include space for a window buffer.
+ * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
+ * The estimate will assume the input may be arbitrarily large,
+ * which is the worst case.
+ *
+ * When srcSize can be bound by a known and rather "small" value,
+ * this fact can be used to provide a tighter estimation
+ * because the CCtx compression context will need less memory.
+ * This tighter estimation can be provided by more advanced functions
+ * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
+ * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
+ * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
+ *
+ * Note : only single-threaded compression is supported.
+ * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *
+ * Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ * Size estimates assume that no external sequence producer is registered.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
+
+/*! ZSTD_estimateCStreamSize() :
+ * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
+ * It will also consider src size to be arbitrarily "large", which is worst case.
+ * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
+ * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+ * Note : CStream size estimation is only correct for single-threaded compression.
+ * ZSTD_DStream memory budget depends on window Size.
+ * This information can be passed manually, using ZSTD_estimateDStreamSize,
+ * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
+ * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+ * an internal ?Dict will be created, which additional size is not estimated here.
+ * In this case, get total size by adding ZSTD_estimate?DictSize
+ * Note 2 : only single-threaded compression is supported.
+ * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ * Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ * Size estimates assume that no external sequence producer is registered.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
+
+/*! ZSTD_estimate?DictSize() :
+ * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+ * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
+ * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
+
+/*! ZSTD_initStatic*() :
+ * Initialize an object using a pre-allocated fixed-size buffer.
+ * workspace: The memory area to emplace the object into.
+ * Provided pointer *must be 8-bytes aligned*.
+ * Buffer must outlive object.
+ * workspaceSize: Use ZSTD_estimate*Size() to determine
+ * how large workspace must be to support target scenario.
+ * @return : pointer to object (same address as workspace, just different type),
+ * or NULL if error (size too small, incorrect alignment, etc.)
+ * Note : zstd will never resize nor malloc() when using a static buffer.
+ * If the object requires more memory than available,
+ * zstd will just error out (typically ZSTD_error_memory_allocation).
+ * Note 2 : there is no corresponding "free" function.
+ * Since workspace is allocated externally, it must be freed externally too.
+ * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
+ * into its associated cParams.
+ * Limitation 1 : currently not compatible with internal dictionary creation, triggered by
+ * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
+ * Limitation 2 : static cctx currently not compatible with multi-threading.
+ * Limitation 3 : static dctx is incompatible with legacy support.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
+
+ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
+
+ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
+ void* workspace, size_t workspaceSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams);
+
+ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
+ void* workspace, size_t workspaceSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType);
+
+
+/*! Custom memory allocation :
+ * These prototypes make it possible to pass your own allocation/free functions.
+ * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
+ * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
+ */
+typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
+
+ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams,
+ ZSTD_customMem customMem);
+
+/*! Thread pool :
+ * These prototypes make it possible to share a thread pool among multiple compression contexts.
+ * This can limit resources for applications with multiple threads where each one uses
+ * a threaded compression mode (via ZSTD_c_nbWorkers parameter).
+ * ZSTD_createThreadPool creates a new thread pool with a given number of threads.
+ * Note that the lifetime of such pool must exist while being used.
+ * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
+ * to use an internal thread pool).
+ * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
+ */
+typedef struct POOL_ctx_s ZSTD_threadPool;
+ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
+ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
+
+
+/*
+ * This API is temporary and is expected to change or disappear in the future!
+ */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_customMem customMem);
+
+
+/***************************************
+* Advanced compression functions
+***************************************/
+
+/*! ZSTD_createCDict_byReference() :
+ * Create a digested dictionary for compression
+ * Dictionary content is just referenced, not duplicated.
+ * As a consequence, `dictBuffer` **must** outlive CDict,
+ * and its content must remain unmodified throughout the lifetime of CDict.
+ * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
+/*! ZSTD_getCParams() :
+ * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+ * `estimatedSrcSize` value is optional, select 0 if not known */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_getParams() :
+ * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+ * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
+ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_checkCParams() :
+ * Ensure param values remain within authorized range.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+
+/*! ZSTD_adjustCParams() :
+ * optimize params for a given `srcSize` and `dictSize`.
+ * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
+ * `dictSize` must be `0` when there is no dictionary.
+ * cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
+ * This function never fails (wide contract) */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+
+/*! ZSTD_CCtx_setCParams() :
+ * Set all parameters provided within @cparams into the working @cctx.
+ * Note : if modifying parameters during compression (MT mode only),
+ * note that changes to the .windowLog parameter will be ignored.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
+/*! ZSTD_compress_advanced() :
+ * Note : this function is now DEPRECATED.
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
+ * This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params);
+
+/*! ZSTD_compress_usingCDict_advanced() :
+ * Note : this function is now DEPRECATED.
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
+ * This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams);
+
+
+/*! ZSTD_CCtx_loadDictionary_byReference() :
+ * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
+ * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_loadDictionary_advanced() :
+ * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
+ * how to load the dictionary (by copy ? by reference ?)
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_CCtx_refPrefix_advanced() :
+ * Same as ZSTD_CCtx_refPrefix(), but gives finer control over
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/* === experimental parameters === */
+/* these parameters can be used with ZSTD_setParameter()
+ * they are not guaranteed to remain supported in the future */
+
+ /* Enables rsyncable mode,
+ * which makes compressed files more rsync friendly
+ * by adding periodic synchronization points to the compressed data.
+ * The target average block size is ZSTD_c_jobSize / 2.
+ * It's possible to modify the job size to increase or decrease
+ * the granularity of the synchronization point.
+ * Once the jobSize is smaller than the window size,
+ * it will result in compression ratio degradation.
+ * NOTE 1: rsyncable mode only works when multithreading is enabled.
+ * NOTE 2: rsyncable performs poorly in combination with long range mode,
+ * since it will decrease the effectiveness of synchronization points,
+ * though mileage may vary.
+ * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
+ * If the selected compression level is already running significantly slower,
+ * the overall speed won't be significantly impacted.
+ */
+ #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
+
+/* Select a compression format.
+ * The value must be of type ZSTD_format_e.
+ * See ZSTD_format_e enum definition for details */
+#define ZSTD_c_format ZSTD_c_experimentalParam2
+
+/* Force back-reference distances to remain < windowSize,
+ * even when referencing into Dictionary content (default:0) */
+#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
+
+/* Controls whether the contents of a CDict
+ * are used in place, or copied into the working context.
+ * Accepts values from the ZSTD_dictAttachPref_e enum.
+ * See the comments on that enum for an explanation of the feature. */
+#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
+
+/* Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never compress literals.
+ * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
+ * may still be emitted if huffman is not beneficial to use.)
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * literals compression based on the compression parameters - specifically,
+ * negative compression levels do not use literal compression.
+ */
+#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
+
+/* Tries to fit compressed block size to be around targetCBlockSize.
+ * No target when targetCBlockSize == 0.
+ * There is no guarantee on compressed block size (default:0) */
+#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
+
+/* User's best guess of source size.
+ * Hint is not valid when srcSizeHint == 0.
+ * There is no guarantee that hint is close to actual source size,
+ * but compression ratio may regress significantly if guess considerably underestimates */
+#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
+
+/* Controls whether the new and experimental "dedicated dictionary search
+ * structure" can be used. This feature is still rough around the edges, be
+ * prepared for surprising behavior!
+ *
+ * How to use it:
+ *
+ * When using a CDict, whether to use this feature or not is controlled at
+ * CDict creation, and it must be set in a CCtxParams set passed into that
+ * construction (via ZSTD_createCDict_advanced2()). A compression will then
+ * use the feature or not based on how the CDict was constructed; the value of
+ * this param, set in the CCtx, will have no effect.
+ *
+ * However, when a dictionary buffer is passed into a CCtx, such as via
+ * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
+ * whether the CDict that is created internally can use the feature or not.
+ *
+ * What it does:
+ *
+ * Normally, the internal data structures of the CDict are analogous to what
+ * would be stored in a CCtx after compressing the contents of a dictionary.
+ * To an approximation, a compression using a dictionary can then use those
+ * data structures to simply continue what is effectively a streaming
+ * compression where the simulated compression of the dictionary left off.
+ * Which is to say, the search structures in the CDict are normally the same
+ * format as in the CCtx.
+ *
+ * It is possible to do better, since the CDict is not like a CCtx: the search
+ * structures are written once during CDict creation, and then are only read
+ * after that, while the search structures in the CCtx are both read and
+ * written as the compression goes along. This means we can choose a search
+ * structure for the dictionary that is read-optimized.
+ *
+ * This feature enables the use of that different structure.
+ *
+ * Note that some of the members of the ZSTD_compressionParameters struct have
+ * different semantics and constraints in the dedicated search structure. It is
+ * highly recommended that you simply set a compression level in the CCtxParams
+ * you pass into the CDict creation call, and avoid messing with the cParams
+ * directly.
+ *
+ * Effects:
+ *
+ * This will only have any effect when the selected ZSTD_strategy
+ * implementation supports this feature. Currently, that's limited to
+ * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
+ *
+ * Note that this means that the CDict tables can no longer be copied into the
+ * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
+ * usable. The dictionary can only be attached or reloaded.
+ *
+ * In general, you should expect compression to be faster--sometimes very much
+ * so--and CDict creation to be slightly slower. Eventually, we will probably
+ * make this mode the default.
+ */
+#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
+
+/* ZSTD_c_stableInBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the compressor that input data presented with ZSTD_inBuffer
+ * will ALWAYS be the same between calls.
+ * Technically, the @src pointer must never be changed,
+ * and the @pos field can only be updated by zstd.
+ * However, it's possible to increase the @size field,
+ * allowing scenarios where more data can be appended after compressions starts.
+ * These conditions are checked by the compressor,
+ * and compression will fail if they are not respected.
+ * Also, data in the ZSTD_inBuffer within the range [src, src + pos)
+ * MUST not be modified during compression or it will result in data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an input window buffer,
+ * because the user guarantees it can reference the ZSTD_inBuffer until
+ * the frame is complete. But, it will still allocate an output buffer
+ * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
+ * avoid the memcpy() from the input buffer to the input window buffer.
+ *
+ * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, compression WILL fail if conditions are not respected.
+ *
+ * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
+ * not be modified during compression or it will result in data corruption.
+ * This is because zstd needs to reference data in the ZSTD_inBuffer to find
+ * matches. Normally zstd maintains its own window buffer for this purpose,
+ * but passing this flag tells zstd to rely on user provided buffer instead.
+ */
+#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
+
+/* ZSTD_c_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells he compressor that the ZSTD_outBuffer will not be resized between
+ * calls. Specifically: (out.size - out.pos) will never grow. This gives the
+ * compressor the freedom to say: If the compressed data doesn't fit in the
+ * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
+ * always decompress directly into the output buffer, instead of decompressing
+ * into an internal buffer and copying to the output buffer.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer. It will still allocate the
+ * input window buffer (see ZSTD_c_stableInBuffer).
+ *
+ * Zstd will check that (out.size - out.pos) never grows and return an error
+ * if it does. While not strictly necessary, this should prevent surprises.
+ */
+#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
+
+/* ZSTD_c_blockDelimiters
+ * Default is 0 == ZSTD_sf_noBlockDelimiters.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ *
+ * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
+ * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
+ * See the definition of ZSTD_Sequence for more specifics.
+ */
+#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
+
+/* ZSTD_c_validateSequences
+ * Default is 0 == disabled. Set to 1 to enable sequence validation.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
+ * during function execution.
+ *
+ * Without validation, providing a sequence that does not conform to the zstd spec will cause
+ * undefined behavior, and may produce a corrupted block.
+ *
+ * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
+ * specifics regarding offset/matchlength requirements) then the function will bail out and
+ * return an error.
+ *
+ */
+#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
+
+/* ZSTD_c_useBlockSplitter
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use block splitter.
+ * Set to ZSTD_ps_enable to always use block splitter.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * block splitting based on the compression parameters.
+ */
+#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
+
+/* ZSTD_c_useRowMatchFinder
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use row-based matchfinder.
+ * Set to ZSTD_ps_enable to force usage of row-based matchfinder.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * the row-based matchfinder based on support for SIMD instructions and the window log.
+ * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
+ */
+#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
+
+/* ZSTD_c_deterministicRefPrefix
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Zstd produces different results for prefix compression when the prefix is
+ * directly adjacent to the data about to be compressed vs. when it isn't.
+ * This is because zstd detects that the two buffers are contiguous and it can
+ * use a more efficient match finding algorithm. However, this produces different
+ * results than when the two buffers are non-contiguous. This flag forces zstd
+ * to always load the prefix in non-contiguous mode, even if it happens to be
+ * adjacent to the data, to guarantee determinism.
+ *
+ * If you really care about determinism when using a dictionary or prefix,
+ * like when doing delta compression, you should select this option. It comes
+ * at a speed penalty of about ~2.5% if the dictionary and data happened to be
+ * contiguous, and is free if they weren't contiguous. We don't expect that
+ * intentionally making the dictionary and data contiguous will be worth the
+ * cost to memcpy() the data.
+ */
+#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
+
+/* ZSTD_c_prefetchCDictTables
+ * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
+ *
+ * In some situations, zstd uses CDict tables in-place rather than copying them
+ * into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
+ * In such situations, compression speed is seriously impacted when CDict tables are
+ * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
+ * when they are used in-place.
+ *
+ * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
+ * For sufficiently large inputs, zstd will by default memcpy() CDict tables
+ * into the working context, so there is no need to prefetch. This parameter is
+ * targeted at a middle range of input sizes, where a prefetch is cheap enough to be
+ * useful but memcpy() is too expensive. The exact range of input sizes where this
+ * makes sense is best determined by careful experimentation.
+ *
+ * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
+ * but in the future zstd may conditionally enable this feature via an auto-detection
+ * heuristic for cold CDicts.
+ * Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
+ */
+#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
+
+/* ZSTD_c_enableSeqProducerFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal sequence producer if an
+ * external sequence producer is registered and returns an error code. This fallback
+ * is block-by-block: the internal sequence producer will only be called for blocks
+ * where the external sequence producer returns an error code. Fallback parsing will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full Block-Level Sequence Producer API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
+
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() innacurate). Only currently meant to be used for testing.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
+/* ZSTD_c_searchForExternalRepcodes
+ * This parameter affects how zstd parses external sequences, such as sequences
+ * provided through the compressSequences() API or from an external block-level
+ * sequence producer.
+ *
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
+ * external sequences, even if those repcodes are not explicitly indicated in
+ * the "rep" field. Note that this is the only way to exploit repcode matches
+ * while using compressSequences() or an external sequence producer, since zstd
+ * currently ignores the "rep" field of external sequences.
+ *
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
+ * external sequences, regardless of whether the "rep" field has been set. This
+ * reduces sequence compression overhead by about 25% while sacrificing some
+ * compression ratio.
+ *
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
+ * based on compression level.
+ *
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
+ */
+#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
+
+/*! ZSTD_CCtx_getParameter() :
+ * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
+ * and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+
+
+/*! ZSTD_CCtx_params :
+ * Quick howto :
+ * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
+ * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
+ * an existing ZSTD_CCtx_params structure.
+ * This is similar to
+ * ZSTD_CCtx_setParameter().
+ * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+ * an existing CCtx.
+ * These parameters will be applied to
+ * all subsequent frames.
+ * - ZSTD_compressStream2() : Do compression using the CCtx.
+ * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
+ *
+ * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+ * for static allocation of CCtx for single-threaded compression.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */
+
+/*! ZSTD_CCtxParams_reset() :
+ * Reset params to default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+
+/*! ZSTD_CCtxParams_init() :
+ * Initializes the compression parameters of cctxParams according to
+ * compression level. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+
+/*! ZSTD_CCtxParams_init_advanced() :
+ * Initializes the compression and frame parameters of cctxParams according to
+ * params. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+
+/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
+ * Similar to ZSTD_CCtx_setParameter.
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
+ * Parameters must be applied to a ZSTD_CCtx using
+ * ZSTD_CCtx_setParametersUsingCCtxParams().
+ * @result : a code representing success or failure (which can be tested with
+ * ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtxParams_getParameter() :
+ * Similar to ZSTD_CCtx_getParameter.
+ * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+
+/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
+ * Apply a set of ZSTD_CCtx_params to the compression context.
+ * This can be done even after compression is started,
+ * if nbWorkers==0, this will have no impact until a new compression is started.
+ * if nbWorkers>=1, new parameters will be picked up at next job,
+ * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
+
+/*! ZSTD_compressStream2_simpleArgs() :
+ * Same as ZSTD_compressStream2(),
+ * but using only integral types as arguments.
+ * This variant might be helpful for binders from dynamic languages
+ * which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos,
+ ZSTD_EndDirective endOp);
+
+
+/***************************************
+* Advanced decompression functions
+***************************************/
+
+/*! ZSTD_isFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ * Note 3 : Skippable Frame Identifiers are considered valid. */
+ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
+
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * Dictionary content is referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives DDict,
+ * it must remain read accessible throughout the lifetime of DDict */
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_byReference() :
+ * Same as ZSTD_DCtx_loadDictionary(),
+ * but references `dict` content instead of copying it into `dctx`.
+ * This saves memory if `dict` remains around.,
+ * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_advanced() :
+ * Same as ZSTD_DCtx_loadDictionary(),
+ * but gives direct control over
+ * how to load the dictionary (by copy ? by reference ?)
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?). */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_refPrefix_advanced() :
+ * Same as ZSTD_DCtx_refPrefix(), but gives finer control over
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_setMaxWindowSize() :
+ * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+ * This protects a decoder context from reserving too much memory for itself (potential attack scenario).
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+
+/*! ZSTD_DCtx_getParameter() :
+ * Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
+ * and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
+
+/* ZSTD_d_format
+ * experimental parameter,
+ * allowing selection between ZSTD_format_e input compression formats
+ */
+#define ZSTD_d_format ZSTD_d_experimentalParam1
+/* ZSTD_d_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
+ * between calls, except for the modifications that zstd makes to pos (the
+ * caller must not modify pos). This is checked by the decompressor, and
+ * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
+ * MUST be large enough to fit the entire decompressed frame. This will be
+ * checked when the frame content size is known. The data in the ZSTD_outBuffer
+ * in the range [dst, dst + pos) MUST not be modified during decompression
+ * or you will get data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer, but it will still allocate
+ * an input buffer large enough to fit any compressed block. This will also
+ * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
+ * If you need to avoid the input buffer allocation use the buffer-less
+ * streaming API.
+ *
+ * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, decompression WILL fail if you violate the preconditions.
+ *
+ * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
+ * not be modified during decompression or you will get data corruption. This
+ * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
+ * matches. Normally zstd maintains its own buffer for this purpose, but passing
+ * this flag tells zstd to use the user provided buffer.
+ */
+#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
+
+/* ZSTD_d_forceIgnoreChecksum
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * Tells the decompressor to skip checksum validation during decompression, regardless
+ * of whether checksumming was specified during compression. This offers some
+ * slight performance benefits, and may be useful for debugging.
+ * Param has values of type ZSTD_forceIgnoreChecksum_e
+ */
+#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
+
+/* ZSTD_d_refMultipleDDicts
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * If enabled and dctx is allocated on the heap, then additional memory will be allocated
+ * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
+ * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
+ * store all references. At decompression time, the appropriate dictID is selected
+ * from the set of DDicts based on the dictID in the frame.
+ *
+ * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
+ *
+ * Param has values of byte ZSTD_refMultipleDDicts_e
+ *
+ * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
+ * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
+ * Memory is allocated as per ZSTD_DCtx::customMem.
+ *
+ * Although this function allocates memory for the table, the user is still responsible for
+ * memory management of the underlying ZSTD_DDict* themselves.
+ */
+#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
+
+/* ZSTD_d_disableHuffmanAssembly
+ * Set to 1 to disable the Huffman assembly implementation.
+ * The default value is 0, which allows zstd to use the Huffman assembly
+ * implementation if available.
+ *
+ * This parameter can be used to disable Huffman assembly at runtime.
+ * If you want to disable it at compile time you can define the macro
+ * ZSTD_DISABLE_ASM.
+ */
+#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
+
+
+/*! ZSTD_DCtx_setFormat() :
+ * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
+ * Instruct the decoder context about what kind of data to decode next.
+ * This instruction is mandatory to decode data without a fully-formed header,
+ * such ZSTD_f_zstd1_magicless for example.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
+ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+
+/*! ZSTD_decompressStream_simpleArgs() :
+ * Same as ZSTD_decompressStream(),
+ * but using only integral types as arguments.
+ * This can be helpful for binders from dynamic languages
+ * which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
+ ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos);
+
+
+/********************************************************************
+* Advanced streaming functions
+* Warning : most of these functions are now redundant with the Advanced API.
+* Once Advanced API reaches "stable" status,
+* redundant functions will be deprecated, and then at some point removed.
+********************************************************************/
+
+/*===== Advanced Streaming compression functions =====*/
+
+/*! ZSTD_initCStream_srcSize() :
+ * This function is DEPRECATED, and equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *
+ * pledgedSrcSize must be correct. If it is not known at init time, use
+ * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
+ * "0" also disables frame content size field. It may be enabled in the future.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+ int compressionLevel,
+ unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingDict() :
+ * This function is DEPRECATED, and is equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * Creates of an internal CDict (incompatible with static CCtx), except if
+ * dict == NULL or dictSize < 8, in which case no dict is used.
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
+ * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ int compressionLevel);
+
+/*! ZSTD_initCStream_advanced() :
+ * This function is DEPRECATED, and is approximately equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * // Pseudocode: Set each zstd parameter and leave the rest as-is.
+ * for ((param, value) : params) {
+ * ZSTD_CCtx_setParameter(zcs, param, value);
+ * }
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
+ * pledgedSrcSize must be correct.
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params,
+ unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingCDict() :
+ * This function is DEPRECATED, and equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * note : cdict will just be referenced, and must outlive compression session
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+
+/*! ZSTD_initCStream_usingCDict_advanced() :
+ * This function is DEPRECATED, and is approximately equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
+ * for ((fParam, value) : fParams) {
+ * ZSTD_CCtx_setParameter(zcs, fParam, value);
+ * }
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams,
+ unsigned long long pledgedSrcSize);
+
+/*! ZSTD_resetCStream() :
+ * This function is DEPRECATED, and is equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
+ * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
+ * explicitly specified.
+ *
+ * start a new frame, using same parameters from previous frame.
+ * This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
+ * Note that zcs must be init at least once before using ZSTD_resetCStream().
+ * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
+ * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
+ * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError())
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+
+
+typedef struct {
+ unsigned long long ingested; /* nb input bytes read and buffered */
+ unsigned long long consumed; /* nb input bytes actually compressed */
+ unsigned long long produced; /* nb of compressed bytes generated and buffered */
+ unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
+ unsigned currentJobID; /* MT only : latest started job nb */
+ unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
+} ZSTD_frameProgression;
+
+/* ZSTD_getFrameProgression() :
+ * tells how much data has been ingested (read from input)
+ * consumed (input actually compressed) and produced (output) for current frame.
+ * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
+ * Aggregates progression inside active worker threads.
+ */
+ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
+
+/*! ZSTD_toFlushNow() :
+ * Tell how many bytes are ready to be flushed immediately.
+ * Useful for multithreading scenarios (nbWorkers >= 1).
+ * Probe the oldest active job, defined as oldest job not yet entirely flushed,
+ * and check its output buffer.
+ * @return : amount of data stored in oldest job and ready to be flushed immediately.
+ * if @return == 0, it means either :
+ * + there is no active job (could be checked with ZSTD_frameProgression()), or
+ * + oldest job is still actively compressing data,
+ * but everything it has produced has also been flushed so far,
+ * therefore flush speed is limited by production speed of oldest job
+ * irrespective of the speed of concurrent (and newer) jobs.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
+
+
+/*===== Advanced Streaming decompression functions =====*/
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
+ *
+ * note: no dictionary will be used if dict == NULL or dictSize < 8
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_refDDict(zds, ddict);
+ *
+ * note : ddict is referenced, it must outlive decompression session
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *
+ * re-use decompression parameters from previous init; saves dictionary loading
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+
+
+/*********************************************************************
+* Buffer-less and synchronous inner streaming functions
+*
+* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+* But it's also a complex one, with several restrictions, documented below.
+* Prefer normal streaming API for an easier experience.
+********************************************************************* */
+
+/**
+ Buffer-less streaming compression (synchronous mode)
+
+ A ZSTD_CCtx object is required to track streaming operations.
+ Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
+ ZSTD_CCtx object can be re-used multiple times within successive compression operations.
+
+ Start by initializing a context.
+ Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
+
+ Then, consume your input using ZSTD_compressContinue().
+ There are some important considerations to keep in mind when using this advanced function :
+ - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
+ - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
+ - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+ Worst case evaluation is provided by ZSTD_compressBound().
+ ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+ - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+ It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+ - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+ In which case, it will "discard" the relevant memory section from its history.
+
+ Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
+ It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+ Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
+
+ `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
+*/
+
+/*===== Buffer-less streaming compression functions =====*/
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
+
+ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+/**
+ Buffer-less streaming decompression (synchronous mode)
+
+ A ZSTD_DCtx object is required to track streaming operations.
+ Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+ A ZSTD_DCtx object can be re-used multiple times.
+
+ First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
+ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+ Data fragment must be large enough to ensure successful decoding.
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+ result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+ >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
+ errorCode, which can be tested using ZSTD_isError().
+
+ It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+ such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
+ Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
+ As a consequence, check that values remain within valid application range.
+ For example, do not allocate memory blindly, check that `windowSize` is within expectation.
+ Each application can set its own limits, depending on local restrictions.
+ For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
+
+ ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
+ ZSTD_decompressContinue() is very sensitive to contiguity,
+ if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
+ or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
+ There are multiple ways to guarantee this condition.
+
+ The most memory efficient way is to use a round buffer of sufficient size.
+ Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
+ which can return an error code if required value is too large for current system (in 32-bits mode).
+ In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
+ up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
+ which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
+ At which point, decoding can resume from the beginning of the buffer.
+ Note that already decoded data stored in the buffer should be flushed before being overwritten.
+
+ There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
+
+ Finally, if you control the compression process, you can also ignore all buffer size rules,
+ as long as the encoder and decoder progress in "lock-step",
+ aka use exactly the same buffer sizes, break contiguity at the same place, etc.
+
+ Once buffers are setup, start decompression, with ZSTD_decompressBegin().
+ If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
+
+ Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
+ ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+ ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
+
+ result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+ It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
+ It can also be an error code, which can be tested with ZSTD_isError().
+
+ A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+ Context can then be reset to start a new decompression.
+
+ Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
+ This information is not required to properly decode a frame.
+
+ == Special case : skippable frames ==
+
+ Skippable frames allow integration of user-defined data into a flow of concatenated frames.
+ Skippable frames will be ignored (skipped) by decompressor.
+ The format of skippable frames is as follows :
+ a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
+ b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
+ c) Frame Content - any content (User Data) of length equal to Frame Size
+ For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
+ For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
+*/
+
+/*===== Buffer-less streaming decompression functions =====*/
+typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
+typedef struct {
+ unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
+ unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
+ unsigned blockSizeMax;
+ ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
+ unsigned headerSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+ unsigned _reserved1;
+ unsigned _reserved2;
+} ZSTD_frameHeader;
+
+/*! ZSTD_getFrameHeader() :
+ * decode Frame Header, or requires larger `srcSize`.
+ * @return : 0, `zfhPtr` is correctly filled,
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ * or an error code, which can be tested using ZSTD_isError() */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+/*! ZSTD_getFrameHeader_advanced() :
+ * same as ZSTD_getFrameHeader(),
+ * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* misc */
+ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+
+
+
+
+/* ============================ */
+/** Block level API */
+/* ============================ */
+
+/*!
+ Block functions produce and decode raw zstd blocks, without frame metadata.
+ Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+ But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
+
+ A few rules to respect :
+ - Compressing and decompressing require a context structure
+ + Use ZSTD_createCCtx() and ZSTD_createDCtx()
+ - It is necessary to init context before starting
+ + compression : any ZSTD_compressBegin*() variant, including with dictionary
+ + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+ - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+ + If input is larger than a block size, it's necessary to split input data into multiple blocks
+ + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
+ Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
+ - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
+ ===> In which case, nothing is produced into `dst` !
+ + User __must__ test for such outcome and deal directly with uncompressed data
+ + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
+ Doing so would mess up with statistics history, leading to potential data corruption.
+ + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
+ + In case of multiple successive blocks, should some of them be uncompressed,
+ decoder must be informed of their existence in order to follow proper history.
+ Use ZSTD_insertBlock() for such a case.
+*/
+
+/*===== Raw zstd block functions =====*/
+ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
+
+
+/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
+ *
+ * *** OVERVIEW ***
+ * The Block-Level Sequence Producer API allows users to provide their own custom
+ * sequence producer which libzstd invokes to process each block. The produced list
+ * of sequences (literals and matches) is then post-processed by libzstd to produce
+ * valid compressed blocks.
+ *
+ * This block-level offload API is a more granular complement of the existing
+ * frame-level offload API compressSequences() (introduced in v1.5.1). It offers
+ * an easier migration story for applications already integrated with libzstd: the
+ * user application continues to invoke the same compression functions
+ * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
+ * from the specific advantages of the external sequence producer. For example,
+ * the sequence producer could be tuned to take advantage of known characteristics
+ * of the input, to offer better speed / ratio, or could leverage hardware
+ * acceleration not available within libzstd itself.
+ *
+ * See contrib/externalSequenceProducer for an example program employing the
+ * Block-Level Sequence Producer API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_sequenceProducer_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ * - sequenceProducerState: a pointer to a user-managed state for the sequence
+ * producer.
+ *
+ * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
+ * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
+ * backing outSeqs is managed by the CCtx.
+ *
+ * - src, srcSize: an input buffer for the sequence producer to parse.
+ * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * - dict, dictSize: a history buffer, which may be empty, which the sequence
+ * producer may reference as it parses the src buffer. Currently, zstd will
+ * always pass dictSize == 0 into external sequence producers, but this will
+ * change in the future.
+ *
+ * - compressionLevel: a signed integer representing the zstd compression level
+ * set by the user for the current operation. The sequence producer may choose
+ * to use this information to change its compression strategy and speed/ratio
+ * tradeoff. Note: the compression level does not reflect zstd parameters set
+ * through the advanced API.
+ *
+ * - windowSize: a size_t representing the maximum allowed offset for external
+ * sequences. Note that sequence offsets are sometimes allowed to exceed the
+ * windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ * for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ * - The sum of matchLengths and literalLengths must equal srcSize.
+ * - All sequences in the parse, except for the final sequence, must have
+ * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
+ * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
+ * - All offsets must respect the windowSize parameter as specified in
+ * doc/zstd_compression_format.md.
+ * - If the final sequence has matchLength == 0, it must also have offset == 0.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal sequence producer or fail the compression operation. The user can
+ * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
+ * cParam. Fallback compression will follow any other cParam settings, such as
+ * compression level, the same as in a normal compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
+ * function by calling
+ * ZSTD_registerSequenceProducer(cctx,
+ * sequenceProducerState,
+ * sequenceProducer)
+ * This setting will persist until the next parameter reset of the CCtx.
+ *
+ * The sequenceProducerState must be initialized by the user before calling
+ * ZSTD_registerSequenceProducer(). The user is responsible for destroying the
+ * sequenceProducerState.
+ *
+ * *** LIMITATIONS ***
+ * This API is compatible with all zstd compression APIs which respect advanced parameters.
+ * However, there are three limitations:
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
+ * external sequence producer.
+ * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
+ * cases (see its documentation for details). Users must explicitly set
+ * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
+ * sequence producer is registered.
+ * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
+ * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
+ *
+ * Second, history buffers are not currently supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external sequence producer (for now). This has two implications:
+ * - Dictionaries are not currently supported. Compression will *not* fail if the user
+ * references a dictionary, but the dictionary won't have any effect.
+ * - Stream history is not currently supported. All advanced compression APIs, including
+ * streaming APIs, work with external sequence producers, but each block is treated as
+ * an independent chunk without history from previous blocks.
+ *
+ * Third, multi-threading within a single compression is not currently supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
+
+typedef size_t ZSTD_sequenceProducer_F (
+ void* sequenceProducerState,
+ ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ int compressionLevel,
+ size_t windowSize
+);
+
+/*! ZSTD_registerSequenceProducer() :
+ * Instruct zstd to use a block-level external sequence producer function.
+ *
+ * The sequenceProducerState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * Sequence producer registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on compression
+ * APIs which respect advanced parameters, such as compress2() and compressStream2().
+ * Older compression APIs such as compressCCtx(), which predate the introduction of
+ * "advanced parameters", will ignore any external sequence producer setting.
+ *
+ * The sequence producer can be "cleared" by registering a NULL function pointer. This
+ * removes all limitations described above in the "LIMITATIONS" section of the API docs.
+ *
+ * The user is strongly encouraged to read the full API documentation (above) before
+ * calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerSequenceProducer(
+ ZSTD_CCtx* cctx,
+ void* sequenceProducerState,
+ ZSTD_sequenceProducer_F* sequenceProducer
+);
+
+#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/zstd_common.c b/contrib/zstd/zstd_common.c
new file mode 100644
index 0000000..3208552
--- /dev/null
+++ b/contrib/zstd/zstd_common.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+
+/*-*************************************
+* Dependencies
+***************************************/
+#define ZSTD_DEPS_NEED_MALLOC
+#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
+#include "error_private.h"
+#include "zstd_internal.h"
+
+
+/*-****************************************
+* Version
+******************************************/
+unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
+
+const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
+
+
+/*-****************************************
+* ZSTD Error Management
+******************************************/
+#undef ZSTD_isError /* defined within zstd_internal.h */
+/*! ZSTD_isError() :
+ * tells if a return value is an error code
+ * symbol is required for external callers */
+unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
+
+/*! ZSTD_getErrorName() :
+ * provides error code string from function result (useful for debugging) */
+const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+/*! ZSTD_getError() :
+ * convert a `size_t` function result into a proper ZSTD_errorCode enum */
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
+
+/*! ZSTD_getErrorString() :
+ * provides error code string from enum */
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
+
+
+
+/*=**************************************************************
+* Custom allocator
+****************************************************************/
+void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
+{
+ if (customMem.customAlloc)
+ return customMem.customAlloc(customMem.opaque, size);
+ return ZSTD_malloc(size);
+}
+
+void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
+{
+ if (customMem.customAlloc) {
+ /* calloc implemented as malloc+memset;
+ * not as efficient as calloc, but next best guess for custom malloc */
+ void* const ptr = customMem.customAlloc(customMem.opaque, size);
+ ZSTD_memset(ptr, 0, size);
+ return ptr;
+ }
+ return ZSTD_calloc(1, size);
+}
+
+void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
+{
+ if (ptr!=NULL) {
+ if (customMem.customFree)
+ customMem.customFree(customMem.opaque, ptr);
+ else
+ ZSTD_free(ptr);
+ }
+}
diff --git a/contrib/zstd/zstd_compress.c b/contrib/zstd/zstd_compress.c
new file mode 100644
index 0000000..e11252c
--- /dev/null
+++ b/contrib/zstd/zstd_compress.c
@@ -0,0 +1,6927 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
+#include "mem.h"
+#include "hist.h" /* HIST_countFast_wksp */
+#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
+#include "fse.h"
+#include "huf.h"
+#include "zstd_compress_internal.h"
+#include "zstd_compress_sequences.h"
+#include "zstd_compress_literals.h"
+#include "zstd_fast.h"
+#include "zstd_double_fast.h"
+#include "zstd_lazy.h"
+#include "zstd_opt.h"
+#include "zstd_ldm.h"
+#include "zstd_compress_superblock.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+/* ***************************************************************
+* Tuning parameters
+*****************************************************************/
+/*!
+ * COMPRESS_HEAPMODE :
+ * Select how default decompression function ZSTD_compress() allocates its context,
+ * on stack (0, default), or into heap (1).
+ * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected.
+ */
+#ifndef ZSTD_COMPRESS_HEAPMODE
+# define ZSTD_COMPRESS_HEAPMODE 0
+#endif
+
+/*!
+ * ZSTD_HASHLOG3_MAX :
+ * Maximum size of the hash table dedicated to find 3-bytes matches,
+ * in log format, aka 17 => 1 << 17 == 128Ki positions.
+ * This structure is only used in zstd_opt.
+ * Since allocation is centralized for all strategies, it has to be known here.
+ * The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3,
+ * so that zstd_opt.c doesn't need to know about this constant.
+ */
+#ifndef ZSTD_HASHLOG3_MAX
+# define ZSTD_HASHLOG3_MAX 17
+#endif
+
+/*-*************************************
+* Helper functions
+***************************************/
+/* ZSTD_compressBound()
+ * Note that the result from this function is only valid for
+ * the one-pass compression functions.
+ * When employing the streaming mode,
+ * if flushes are frequently altering the size of blocks,
+ * the overhead from block headers can make the compressed data larger
+ * than the return value of ZSTD_compressBound().
+ */
+size_t ZSTD_compressBound(size_t srcSize) {
+ size_t const r = ZSTD_COMPRESSBOUND(srcSize);
+ if (r==0) return ERROR(srcSize_wrong);
+ return r;
+}
+
+
+/*-*************************************
+* Context memory management
+***************************************/
+struct ZSTD_CDict_s {
+ const void* dictContent;
+ size_t dictContentSize;
+ ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */
+ U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
+ ZSTD_cwksp workspace;
+ ZSTD_matchState_t matchState;
+ ZSTD_compressedBlockState_t cBlockState;
+ ZSTD_customMem customMem;
+ U32 dictID;
+ int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */
+ ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use
+ * row-based matchfinder. Unless the cdict is reloaded, we will use
+ * the same greedy/lazy matchfinder at compression time.
+ */
+}; /* typedef'd to ZSTD_CDict within "zstd.h" */
+
+ZSTD_CCtx* ZSTD_createCCtx(void)
+{
+ return ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
+}
+
+static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager)
+{
+ assert(cctx != NULL);
+ ZSTD_memset(cctx, 0, sizeof(*cctx));
+ cctx->customMem = memManager;
+ cctx->bmi2 = ZSTD_cpuSupportsBmi2();
+ { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
+ assert(!ZSTD_isError(err));
+ (void)err;
+ }
+}
+
+ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
+{
+ ZSTD_STATIC_ASSERT(zcss_init==0);
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+ { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem);
+ if (!cctx) return NULL;
+ ZSTD_initCCtx(cctx, customMem);
+ return cctx;
+ }
+}
+
+ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize)
+{
+ ZSTD_cwksp ws;
+ ZSTD_CCtx* cctx;
+ if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */
+ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */
+ ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc);
+
+ cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx));
+ if (cctx == NULL) return NULL;
+
+ ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx));
+ ZSTD_cwksp_move(&cctx->workspace, &ws);
+ cctx->staticSize = workspaceSize;
+
+ /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */
+ if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL;
+ cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
+ cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
+ cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE);
+ cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
+ return cctx;
+}
+
+/**
+ * Clears and frees all of the dictionaries in the CCtx.
+ */
+static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx)
+{
+ ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem);
+ ZSTD_freeCDict(cctx->localDict.cdict);
+ ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict));
+ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));
+ cctx->cdict = NULL;
+}
+
+static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict)
+{
+ size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0;
+ size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict);
+ return bufferSize + cdictSize;
+}
+
+static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
+{
+ assert(cctx != NULL);
+ assert(cctx->staticSize == 0);
+ ZSTD_clearAllDicts(cctx);
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
+#endif
+ ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
+}
+
+size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
+{
+ if (cctx==NULL) return 0; /* support free on NULL */
+ RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
+ "not compatible with static CCtx");
+ { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
+ ZSTD_freeCCtxContent(cctx);
+ if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem);
+ }
+ return 0;
+}
+
+
+static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ return ZSTDMT_sizeof_CCtx(cctx->mtctx);
+#else
+ (void)cctx;
+ return 0;
+#endif
+}
+
+
+size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
+{
+ if (cctx==NULL) return 0; /* support sizeof on NULL */
+ /* cctx may be in the workspace */
+ return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx))
+ + ZSTD_cwksp_sizeof(&cctx->workspace)
+ + ZSTD_sizeof_localDict(cctx->localDict)
+ + ZSTD_sizeof_mtctx(cctx);
+}
+
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
+{
+ return ZSTD_sizeof_CCtx(zcs); /* same object */
+}
+
+/* private API call, for dictBuilder only */
+const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
+
+/* Returns true if the strategy supports using a row based matchfinder */
+static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
+ return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2);
+}
+
+/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder
+ * for this compression.
+ */
+static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) {
+ assert(mode != ZSTD_ps_auto);
+ return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable);
+}
+
+/* Returns row matchfinder usage given an initial mode and cParams */
+static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode,
+ const ZSTD_compressionParameters* const cParams) {
+#if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON)
+ int const kHasSIMD128 = 1;
+#else
+ int const kHasSIMD128 = 0;
+#endif
+ if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */
+ mode = ZSTD_ps_disable;
+ if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode;
+ if (kHasSIMD128) {
+ if (cParams->windowLog > 14) mode = ZSTD_ps_enable;
+ } else {
+ if (cParams->windowLog > 17) mode = ZSTD_ps_enable;
+ }
+ return mode;
+}
+
+/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */
+static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode,
+ const ZSTD_compressionParameters* const cParams) {
+ if (mode != ZSTD_ps_auto) return mode;
+ return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable;
+}
+
+/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */
+static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const U32 forDDSDict) {
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+ /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate.
+ * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder.
+ */
+ return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
+}
+
+/* Returns ZSTD_ps_enable if compression parameters are such that we should
+ * enable long distance matching (wlog >= 27, strategy >= btopt).
+ * Returns ZSTD_ps_disable otherwise.
+ */
+static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
+ const ZSTD_compressionParameters* const cParams) {
+ if (mode != ZSTD_ps_auto) return mode;
+ return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
+}
+
+static int ZSTD_resolveExternalSequenceValidation(int mode) {
+ return mode;
+}
+
+/* Resolves maxBlockSize to the default if no value is present. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+ if (maxBlockSize == 0) {
+ return ZSTD_BLOCKSIZE_MAX;
+ } else {
+ return maxBlockSize;
+ }
+}
+
+static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) {
+ if (value != ZSTD_ps_auto) return value;
+ if (cLevel < 10) {
+ return ZSTD_ps_disable;
+ } else {
+ return ZSTD_ps_enable;
+ }
+}
+
+/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
+ * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
+static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
+ return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast;
+}
+
+static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
+ ZSTD_compressionParameters cParams)
+{
+ ZSTD_CCtx_params cctxParams;
+ /* should not matter, as all cParams are presumed properly defined */
+ ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT);
+ cctxParams.cParams = cParams;
+
+ /* Adjust advanced params according to cParams */
+ cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams);
+ if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) {
+ ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams);
+ assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog);
+ assert(cctxParams.ldmParams.hashRateLog < 32);
+ }
+ cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
+ cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+ cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+ cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
+ cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes,
+ cctxParams.compressionLevel);
+ assert(!ZSTD_checkCParams(cParams));
+ return cctxParams;
+}
+
+static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced(
+ ZSTD_customMem customMem)
+{
+ ZSTD_CCtx_params* params;
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+ params = (ZSTD_CCtx_params*)ZSTD_customCalloc(
+ sizeof(ZSTD_CCtx_params), customMem);
+ if (!params) { return NULL; }
+ ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT);
+ params->customMem = customMem;
+ return params;
+}
+
+ZSTD_CCtx_params* ZSTD_createCCtxParams(void)
+{
+ return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem);
+}
+
+size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params)
+{
+ if (params == NULL) { return 0; }
+ ZSTD_customFree(params, params->customMem);
+ return 0;
+}
+
+size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params)
+{
+ return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT);
+}
+
+size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) {
+ RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
+ ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
+ cctxParams->compressionLevel = compressionLevel;
+ cctxParams->fParams.contentSizeFlag = 1;
+ return 0;
+}
+
+#define ZSTD_NO_CLEVEL 0
+
+/**
+ * Initializes `cctxParams` from `params` and `compressionLevel`.
+ * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL.
+ */
+static void
+ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
+ const ZSTD_parameters* params,
+ int compressionLevel)
+{
+ assert(!ZSTD_checkCParams(params->cParams));
+ ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
+ cctxParams->cParams = params->cParams;
+ cctxParams->fParams = params->fParams;
+ /* Should not matter, as all cParams are presumed properly defined.
+ * But, set it for tracing anyway.
+ */
+ cctxParams->compressionLevel = compressionLevel;
+ cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
+ cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
+ cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
+ cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+ cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
+ cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel);
+ DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
+ cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
+}
+
+size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
+{
+ RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
+ FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
+ ZSTD_CCtxParams_init_internal(cctxParams, &params, ZSTD_NO_CLEVEL);
+ return 0;
+}
+
+/**
+ * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone.
+ * @param params Validated zstd parameters.
+ */
+static void ZSTD_CCtxParams_setZstdParams(
+ ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
+{
+ assert(!ZSTD_checkCParams(params->cParams));
+ cctxParams->cParams = params->cParams;
+ cctxParams->fParams = params->fParams;
+ /* Should not matter, as all cParams are presumed properly defined.
+ * But, set it for tracing anyway.
+ */
+ cctxParams->compressionLevel = ZSTD_NO_CLEVEL;
+}
+
+ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
+{
+ ZSTD_bounds bounds = { 0, 0, 0 };
+
+ switch(param)
+ {
+ case ZSTD_c_compressionLevel:
+ bounds.lowerBound = ZSTD_minCLevel();
+ bounds.upperBound = ZSTD_maxCLevel();
+ return bounds;
+
+ case ZSTD_c_windowLog:
+ bounds.lowerBound = ZSTD_WINDOWLOG_MIN;
+ bounds.upperBound = ZSTD_WINDOWLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_hashLog:
+ bounds.lowerBound = ZSTD_HASHLOG_MIN;
+ bounds.upperBound = ZSTD_HASHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_chainLog:
+ bounds.lowerBound = ZSTD_CHAINLOG_MIN;
+ bounds.upperBound = ZSTD_CHAINLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_searchLog:
+ bounds.lowerBound = ZSTD_SEARCHLOG_MIN;
+ bounds.upperBound = ZSTD_SEARCHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_minMatch:
+ bounds.lowerBound = ZSTD_MINMATCH_MIN;
+ bounds.upperBound = ZSTD_MINMATCH_MAX;
+ return bounds;
+
+ case ZSTD_c_targetLength:
+ bounds.lowerBound = ZSTD_TARGETLENGTH_MIN;
+ bounds.upperBound = ZSTD_TARGETLENGTH_MAX;
+ return bounds;
+
+ case ZSTD_c_strategy:
+ bounds.lowerBound = ZSTD_STRATEGY_MIN;
+ bounds.upperBound = ZSTD_STRATEGY_MAX;
+ return bounds;
+
+ case ZSTD_c_contentSizeFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_checksumFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_dictIDFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_nbWorkers:
+ bounds.lowerBound = 0;
+#ifdef ZSTD_MULTITHREAD
+ bounds.upperBound = ZSTDMT_NBWORKERS_MAX;
+#else
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_jobSize:
+ bounds.lowerBound = 0;
+#ifdef ZSTD_MULTITHREAD
+ bounds.upperBound = ZSTDMT_JOBSIZE_MAX;
+#else
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_overlapLog:
+#ifdef ZSTD_MULTITHREAD
+ bounds.lowerBound = ZSTD_OVERLAPLOG_MIN;
+ bounds.upperBound = ZSTD_OVERLAPLOG_MAX;
+#else
+ bounds.lowerBound = 0;
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_enableDedicatedDictSearch:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_enableLongDistanceMatching:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_ldmHashLog:
+ bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN;
+ bounds.upperBound = ZSTD_LDM_HASHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmMinMatch:
+ bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN;
+ bounds.upperBound = ZSTD_LDM_MINMATCH_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmBucketSizeLog:
+ bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN;
+ bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmHashRateLog:
+ bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN;
+ bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX;
+ return bounds;
+
+ /* experimental parameters */
+ case ZSTD_c_rsyncable:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_forceMaxWindow :
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_format:
+ ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
+ bounds.lowerBound = ZSTD_f_zstd1;
+ bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */
+ return bounds;
+
+ case ZSTD_c_forceAttachDict:
+ ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad);
+ bounds.lowerBound = ZSTD_dictDefaultAttach;
+ bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */
+ return bounds;
+
+ case ZSTD_c_literalCompressionMode:
+ ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable);
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_targetCBlockSize:
+ bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN;
+ bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX;
+ return bounds;
+
+ case ZSTD_c_srcSizeHint:
+ bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN;
+ bounds.upperBound = ZSTD_SRCSIZEHINT_MAX;
+ return bounds;
+
+ case ZSTD_c_stableInBuffer:
+ case ZSTD_c_stableOutBuffer:
+ bounds.lowerBound = (int)ZSTD_bm_buffered;
+ bounds.upperBound = (int)ZSTD_bm_stable;
+ return bounds;
+
+ case ZSTD_c_blockDelimiters:
+ bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters;
+ bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters;
+ return bounds;
+
+ case ZSTD_c_validateSequences:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_useBlockSplitter:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_useRowMatchFinder:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_deterministicRefPrefix:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_prefetchCDictTables:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_enableSeqProducerFallback:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_maxBlockSize:
+ bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+ bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+ return bounds;
+
+ case ZSTD_c_searchForExternalRepcodes:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ default:
+ bounds.error = ERROR(parameter_unsupported);
+ return bounds;
+ }
+}
+
+/* ZSTD_cParam_clampBounds:
+ * Clamps the value into the bounded range.
+ */
+static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value)
+{
+ ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
+ if (ZSTD_isError(bounds.error)) return bounds.error;
+ if (*value < bounds.lowerBound) *value = bounds.lowerBound;
+ if (*value > bounds.upperBound) *value = bounds.upperBound;
+ return 0;
+}
+
+#define BOUNDCHECK(cParam, val) { \
+ RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \
+ parameter_outOfBound, "Param out of bounds"); \
+}
+
+
+static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
+{
+ switch(param)
+ {
+ case ZSTD_c_compressionLevel:
+ case ZSTD_c_hashLog:
+ case ZSTD_c_chainLog:
+ case ZSTD_c_searchLog:
+ case ZSTD_c_minMatch:
+ case ZSTD_c_targetLength:
+ case ZSTD_c_strategy:
+ return 1;
+
+ case ZSTD_c_format:
+ case ZSTD_c_windowLog:
+ case ZSTD_c_contentSizeFlag:
+ case ZSTD_c_checksumFlag:
+ case ZSTD_c_dictIDFlag:
+ case ZSTD_c_forceMaxWindow :
+ case ZSTD_c_nbWorkers:
+ case ZSTD_c_jobSize:
+ case ZSTD_c_overlapLog:
+ case ZSTD_c_rsyncable:
+ case ZSTD_c_enableDedicatedDictSearch:
+ case ZSTD_c_enableLongDistanceMatching:
+ case ZSTD_c_ldmHashLog:
+ case ZSTD_c_ldmMinMatch:
+ case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_ldmHashRateLog:
+ case ZSTD_c_forceAttachDict:
+ case ZSTD_c_literalCompressionMode:
+ case ZSTD_c_targetCBlockSize:
+ case ZSTD_c_srcSizeHint:
+ case ZSTD_c_stableInBuffer:
+ case ZSTD_c_stableOutBuffer:
+ case ZSTD_c_blockDelimiters:
+ case ZSTD_c_validateSequences:
+ case ZSTD_c_useBlockSplitter:
+ case ZSTD_c_useRowMatchFinder:
+ case ZSTD_c_deterministicRefPrefix:
+ case ZSTD_c_prefetchCDictTables:
+ case ZSTD_c_enableSeqProducerFallback:
+ case ZSTD_c_maxBlockSize:
+ case ZSTD_c_searchForExternalRepcodes:
+ default:
+ return 0;
+ }
+}
+
+size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value);
+ if (cctx->streamStage != zcss_init) {
+ if (ZSTD_isUpdateAuthorized(param)) {
+ cctx->cParamsChanged = 1;
+ } else {
+ RETURN_ERROR(stage_wrong, "can only set params in cctx init stage");
+ } }
+
+ switch(param)
+ {
+ case ZSTD_c_nbWorkers:
+ RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
+ "MT not compatible with static alloc");
+ break;
+
+ case ZSTD_c_compressionLevel:
+ case ZSTD_c_windowLog:
+ case ZSTD_c_hashLog:
+ case ZSTD_c_chainLog:
+ case ZSTD_c_searchLog:
+ case ZSTD_c_minMatch:
+ case ZSTD_c_targetLength:
+ case ZSTD_c_strategy:
+ case ZSTD_c_ldmHashRateLog:
+ case ZSTD_c_format:
+ case ZSTD_c_contentSizeFlag:
+ case ZSTD_c_checksumFlag:
+ case ZSTD_c_dictIDFlag:
+ case ZSTD_c_forceMaxWindow:
+ case ZSTD_c_forceAttachDict:
+ case ZSTD_c_literalCompressionMode:
+ case ZSTD_c_jobSize:
+ case ZSTD_c_overlapLog:
+ case ZSTD_c_rsyncable:
+ case ZSTD_c_enableDedicatedDictSearch:
+ case ZSTD_c_enableLongDistanceMatching:
+ case ZSTD_c_ldmHashLog:
+ case ZSTD_c_ldmMinMatch:
+ case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_targetCBlockSize:
+ case ZSTD_c_srcSizeHint:
+ case ZSTD_c_stableInBuffer:
+ case ZSTD_c_stableOutBuffer:
+ case ZSTD_c_blockDelimiters:
+ case ZSTD_c_validateSequences:
+ case ZSTD_c_useBlockSplitter:
+ case ZSTD_c_useRowMatchFinder:
+ case ZSTD_c_deterministicRefPrefix:
+ case ZSTD_c_prefetchCDictTables:
+ case ZSTD_c_enableSeqProducerFallback:
+ case ZSTD_c_maxBlockSize:
+ case ZSTD_c_searchForExternalRepcodes:
+ break;
+
+ default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
+ }
+ return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value);
+}
+
+size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
+ ZSTD_cParameter param, int value)
+{
+ DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value);
+ switch(param)
+ {
+ case ZSTD_c_format :
+ BOUNDCHECK(ZSTD_c_format, value);
+ CCtxParams->format = (ZSTD_format_e)value;
+ return (size_t)CCtxParams->format;
+
+ case ZSTD_c_compressionLevel : {
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
+ if (value == 0)
+ CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */
+ else
+ CCtxParams->compressionLevel = value;
+ if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel;
+ return 0; /* return type (size_t) cannot represent negative values */
+ }
+
+ case ZSTD_c_windowLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_windowLog, value);
+ CCtxParams->cParams.windowLog = (U32)value;
+ return CCtxParams->cParams.windowLog;
+
+ case ZSTD_c_hashLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_hashLog, value);
+ CCtxParams->cParams.hashLog = (U32)value;
+ return CCtxParams->cParams.hashLog;
+
+ case ZSTD_c_chainLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_chainLog, value);
+ CCtxParams->cParams.chainLog = (U32)value;
+ return CCtxParams->cParams.chainLog;
+
+ case ZSTD_c_searchLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_searchLog, value);
+ CCtxParams->cParams.searchLog = (U32)value;
+ return (size_t)value;
+
+ case ZSTD_c_minMatch :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_minMatch, value);
+ CCtxParams->cParams.minMatch = (U32)value;
+ return CCtxParams->cParams.minMatch;
+
+ case ZSTD_c_targetLength :
+ BOUNDCHECK(ZSTD_c_targetLength, value);
+ CCtxParams->cParams.targetLength = (U32)value;
+ return CCtxParams->cParams.targetLength;
+
+ case ZSTD_c_strategy :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_strategy, value);
+ CCtxParams->cParams.strategy = (ZSTD_strategy)value;
+ return (size_t)CCtxParams->cParams.strategy;
+
+ case ZSTD_c_contentSizeFlag :
+ /* Content size written in frame header _when known_ (default:1) */
+ DEBUGLOG(4, "set content size flag = %u", (value!=0));
+ CCtxParams->fParams.contentSizeFlag = value != 0;
+ return (size_t)CCtxParams->fParams.contentSizeFlag;
+
+ case ZSTD_c_checksumFlag :
+ /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
+ CCtxParams->fParams.checksumFlag = value != 0;
+ return (size_t)CCtxParams->fParams.checksumFlag;
+
+ case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
+ DEBUGLOG(4, "set dictIDFlag = %u", (value!=0));
+ CCtxParams->fParams.noDictIDFlag = !value;
+ return !CCtxParams->fParams.noDictIDFlag;
+
+ case ZSTD_c_forceMaxWindow :
+ CCtxParams->forceWindow = (value != 0);
+ return (size_t)CCtxParams->forceWindow;
+
+ case ZSTD_c_forceAttachDict : {
+ const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
+ BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref);
+ CCtxParams->attachDictPref = pref;
+ return CCtxParams->attachDictPref;
+ }
+
+ case ZSTD_c_literalCompressionMode : {
+ const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
+ BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm);
+ CCtxParams->literalCompressionMode = lcm;
+ return CCtxParams->literalCompressionMode;
+ }
+
+ case ZSTD_c_nbWorkers :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
+ CCtxParams->nbWorkers = value;
+ return CCtxParams->nbWorkers;
+#endif
+
+ case ZSTD_c_jobSize :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ /* Adjust to the minimum non-default value. */
+ if (value != 0 && value < ZSTDMT_JOBSIZE_MIN)
+ value = ZSTDMT_JOBSIZE_MIN;
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
+ assert(value >= 0);
+ CCtxParams->jobSize = value;
+ return CCtxParams->jobSize;
+#endif
+
+ case ZSTD_c_overlapLog :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), "");
+ CCtxParams->overlapLog = value;
+ return CCtxParams->overlapLog;
+#endif
+
+ case ZSTD_c_rsyncable :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), "");
+ CCtxParams->rsyncable = value;
+ return CCtxParams->rsyncable;
+#endif
+
+ case ZSTD_c_enableDedicatedDictSearch :
+ CCtxParams->enableDedicatedDictSearch = (value!=0);
+ return (size_t)CCtxParams->enableDedicatedDictSearch;
+
+ case ZSTD_c_enableLongDistanceMatching :
+ BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
+ CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->ldmParams.enableLdm;
+
+ case ZSTD_c_ldmHashLog :
+ if (value!=0) /* 0 ==> auto */
+ BOUNDCHECK(ZSTD_c_ldmHashLog, value);
+ CCtxParams->ldmParams.hashLog = (U32)value;
+ return CCtxParams->ldmParams.hashLog;
+
+ case ZSTD_c_ldmMinMatch :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_ldmMinMatch, value);
+ CCtxParams->ldmParams.minMatchLength = (U32)value;
+ return CCtxParams->ldmParams.minMatchLength;
+
+ case ZSTD_c_ldmBucketSizeLog :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value);
+ CCtxParams->ldmParams.bucketSizeLog = (U32)value;
+ return CCtxParams->ldmParams.bucketSizeLog;
+
+ case ZSTD_c_ldmHashRateLog :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_ldmHashRateLog, value);
+ CCtxParams->ldmParams.hashRateLog = (U32)value;
+ return CCtxParams->ldmParams.hashRateLog;
+
+ case ZSTD_c_targetCBlockSize :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_targetCBlockSize, value);
+ CCtxParams->targetCBlockSize = (U32)value;
+ return CCtxParams->targetCBlockSize;
+
+ case ZSTD_c_srcSizeHint :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_srcSizeHint, value);
+ CCtxParams->srcSizeHint = value;
+ return (size_t)CCtxParams->srcSizeHint;
+
+ case ZSTD_c_stableInBuffer:
+ BOUNDCHECK(ZSTD_c_stableInBuffer, value);
+ CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value;
+ return CCtxParams->inBufferMode;
+
+ case ZSTD_c_stableOutBuffer:
+ BOUNDCHECK(ZSTD_c_stableOutBuffer, value);
+ CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value;
+ return CCtxParams->outBufferMode;
+
+ case ZSTD_c_blockDelimiters:
+ BOUNDCHECK(ZSTD_c_blockDelimiters, value);
+ CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value;
+ return CCtxParams->blockDelimiters;
+
+ case ZSTD_c_validateSequences:
+ BOUNDCHECK(ZSTD_c_validateSequences, value);
+ CCtxParams->validateSequences = value;
+ return CCtxParams->validateSequences;
+
+ case ZSTD_c_useBlockSplitter:
+ BOUNDCHECK(ZSTD_c_useBlockSplitter, value);
+ CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->useBlockSplitter;
+
+ case ZSTD_c_useRowMatchFinder:
+ BOUNDCHECK(ZSTD_c_useRowMatchFinder, value);
+ CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->useRowMatchFinder;
+
+ case ZSTD_c_deterministicRefPrefix:
+ BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value);
+ CCtxParams->deterministicRefPrefix = !!value;
+ return CCtxParams->deterministicRefPrefix;
+
+ case ZSTD_c_prefetchCDictTables:
+ BOUNDCHECK(ZSTD_c_prefetchCDictTables, value);
+ CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->prefetchCDictTables;
+
+ case ZSTD_c_enableSeqProducerFallback:
+ BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value);
+ CCtxParams->enableMatchFinderFallback = value;
+ return CCtxParams->enableMatchFinderFallback;
+
+ case ZSTD_c_maxBlockSize:
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+ CCtxParams->maxBlockSize = value;
+ return CCtxParams->maxBlockSize;
+
+ case ZSTD_c_searchForExternalRepcodes:
+ BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value);
+ CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->searchForExternalRepcodes;
+
+ default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
+ }
+}
+
+size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value)
+{
+ return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value);
+}
+
+size_t ZSTD_CCtxParams_getParameter(
+ ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value)
+{
+ switch(param)
+ {
+ case ZSTD_c_format :
+ *value = CCtxParams->format;
+ break;
+ case ZSTD_c_compressionLevel :
+ *value = CCtxParams->compressionLevel;
+ break;
+ case ZSTD_c_windowLog :
+ *value = (int)CCtxParams->cParams.windowLog;
+ break;
+ case ZSTD_c_hashLog :
+ *value = (int)CCtxParams->cParams.hashLog;
+ break;
+ case ZSTD_c_chainLog :
+ *value = (int)CCtxParams->cParams.chainLog;
+ break;
+ case ZSTD_c_searchLog :
+ *value = CCtxParams->cParams.searchLog;
+ break;
+ case ZSTD_c_minMatch :
+ *value = CCtxParams->cParams.minMatch;
+ break;
+ case ZSTD_c_targetLength :
+ *value = CCtxParams->cParams.targetLength;
+ break;
+ case ZSTD_c_strategy :
+ *value = (unsigned)CCtxParams->cParams.strategy;
+ break;
+ case ZSTD_c_contentSizeFlag :
+ *value = CCtxParams->fParams.contentSizeFlag;
+ break;
+ case ZSTD_c_checksumFlag :
+ *value = CCtxParams->fParams.checksumFlag;
+ break;
+ case ZSTD_c_dictIDFlag :
+ *value = !CCtxParams->fParams.noDictIDFlag;
+ break;
+ case ZSTD_c_forceMaxWindow :
+ *value = CCtxParams->forceWindow;
+ break;
+ case ZSTD_c_forceAttachDict :
+ *value = CCtxParams->attachDictPref;
+ break;
+ case ZSTD_c_literalCompressionMode :
+ *value = CCtxParams->literalCompressionMode;
+ break;
+ case ZSTD_c_nbWorkers :
+#ifndef ZSTD_MULTITHREAD
+ assert(CCtxParams->nbWorkers == 0);
+#endif
+ *value = CCtxParams->nbWorkers;
+ break;
+ case ZSTD_c_jobSize :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+#else
+ assert(CCtxParams->jobSize <= INT_MAX);
+ *value = (int)CCtxParams->jobSize;
+ break;
+#endif
+ case ZSTD_c_overlapLog :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+#else
+ *value = CCtxParams->overlapLog;
+ break;
+#endif
+ case ZSTD_c_rsyncable :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+#else
+ *value = CCtxParams->rsyncable;
+ break;
+#endif
+ case ZSTD_c_enableDedicatedDictSearch :
+ *value = CCtxParams->enableDedicatedDictSearch;
+ break;
+ case ZSTD_c_enableLongDistanceMatching :
+ *value = CCtxParams->ldmParams.enableLdm;
+ break;
+ case ZSTD_c_ldmHashLog :
+ *value = CCtxParams->ldmParams.hashLog;
+ break;
+ case ZSTD_c_ldmMinMatch :
+ *value = CCtxParams->ldmParams.minMatchLength;
+ break;
+ case ZSTD_c_ldmBucketSizeLog :
+ *value = CCtxParams->ldmParams.bucketSizeLog;
+ break;
+ case ZSTD_c_ldmHashRateLog :
+ *value = CCtxParams->ldmParams.hashRateLog;
+ break;
+ case ZSTD_c_targetCBlockSize :
+ *value = (int)CCtxParams->targetCBlockSize;
+ break;
+ case ZSTD_c_srcSizeHint :
+ *value = (int)CCtxParams->srcSizeHint;
+ break;
+ case ZSTD_c_stableInBuffer :
+ *value = (int)CCtxParams->inBufferMode;
+ break;
+ case ZSTD_c_stableOutBuffer :
+ *value = (int)CCtxParams->outBufferMode;
+ break;
+ case ZSTD_c_blockDelimiters :
+ *value = (int)CCtxParams->blockDelimiters;
+ break;
+ case ZSTD_c_validateSequences :
+ *value = (int)CCtxParams->validateSequences;
+ break;
+ case ZSTD_c_useBlockSplitter :
+ *value = (int)CCtxParams->useBlockSplitter;
+ break;
+ case ZSTD_c_useRowMatchFinder :
+ *value = (int)CCtxParams->useRowMatchFinder;
+ break;
+ case ZSTD_c_deterministicRefPrefix:
+ *value = (int)CCtxParams->deterministicRefPrefix;
+ break;
+ case ZSTD_c_prefetchCDictTables:
+ *value = (int)CCtxParams->prefetchCDictTables;
+ break;
+ case ZSTD_c_enableSeqProducerFallback:
+ *value = CCtxParams->enableMatchFinderFallback;
+ break;
+ case ZSTD_c_maxBlockSize:
+ *value = (int)CCtxParams->maxBlockSize;
+ break;
+ case ZSTD_c_searchForExternalRepcodes:
+ *value = (int)CCtxParams->searchForExternalRepcodes;
+ break;
+ default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
+ }
+ return 0;
+}
+
+/** ZSTD_CCtx_setParametersUsingCCtxParams() :
+ * just applies `params` into `cctx`
+ * no action is performed, parameters are merely stored.
+ * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx.
+ * This is possible even if a compression is ongoing.
+ * In which case, new parameters will be applied on the fly, starting with next compression job.
+ */
+size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams");
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "The context is in the wrong stage!");
+ RETURN_ERROR_IF(cctx->cdict, stage_wrong,
+ "Can't override parameters with cdict attached (some must "
+ "be inherited from the cdict).");
+
+ cctx->requestedParams = *params;
+ return 0;
+}
+
+size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setCParams");
+ assert(cctx != NULL);
+ if (cctx->streamStage != zcss_init) {
+ /* All parameters in @cparams are allowed to be updated during MT compression.
+ * This must be signaled, so that MT compression picks up the changes */
+ cctx->cParamsChanged = 1;
+ }
+ /* only update if parameters are valid */
+ FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
+ cctx->requestedParams.cParams = cparams;
+ return 0;
+}
+
+size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't set pledgedSrcSize when not in init stage.");
+ cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
+ return 0;
+}
+
+static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(
+ int const compressionLevel,
+ size_t const dictSize);
+static int ZSTD_dedicatedDictSearch_isSupported(
+ const ZSTD_compressionParameters* cParams);
+static void ZSTD_dedicatedDictSearch_revertCParams(
+ ZSTD_compressionParameters* cParams);
+
+/**
+ * Initializes the local dict using the requested parameters.
+ * NOTE: This does not use the pledged src size, because it may be used for more
+ * than one compression.
+ */
+static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
+{
+ ZSTD_localDict* const dl = &cctx->localDict;
+ if (dl->dict == NULL) {
+ /* No local dictionary. */
+ assert(dl->dictBuffer == NULL);
+ assert(dl->cdict == NULL);
+ assert(dl->dictSize == 0);
+ return 0;
+ }
+ if (dl->cdict != NULL) {
+ assert(cctx->cdict == dl->cdict);
+ /* Local dictionary already initialized. */
+ return 0;
+ }
+ assert(dl->dictSize > 0);
+ assert(cctx->cdict == NULL);
+ assert(cctx->prefixDict.dict == NULL);
+
+ dl->cdict = ZSTD_createCDict_advanced2(
+ dl->dict,
+ dl->dictSize,
+ ZSTD_dlm_byRef,
+ dl->dictContentType,
+ &cctx->requestedParams,
+ cctx->customMem);
+ RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed");
+ cctx->cdict = dl->cdict;
+ return 0;
+}
+
+size_t ZSTD_CCtx_loadDictionary_advanced(
+ ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't load a dictionary when ctx is not in init stage.");
+ DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
+ ZSTD_clearAllDicts(cctx); /* in case one already exists */
+ if (dict == NULL || dictSize == 0) /* no dictionary mode */
+ return 0;
+ if (dictLoadMethod == ZSTD_dlm_byRef) {
+ cctx->localDict.dict = dict;
+ } else {
+ void* dictBuffer;
+ RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
+ "no malloc for static CCtx");
+ dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
+ RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!");
+ ZSTD_memcpy(dictBuffer, dict, dictSize);
+ cctx->localDict.dictBuffer = dictBuffer;
+ cctx->localDict.dict = dictBuffer;
+ }
+ cctx->localDict.dictSize = dictSize;
+ cctx->localDict.dictContentType = dictContentType;
+ return 0;
+}
+
+size_t ZSTD_CCtx_loadDictionary_byReference(
+ ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_CCtx_loadDictionary_advanced(
+ cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
+}
+
+size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_CCtx_loadDictionary_advanced(
+ cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
+}
+
+
+size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't ref a dict when ctx not in init stage.");
+ /* Free the existing local cdict (if any) to save memory. */
+ ZSTD_clearAllDicts(cctx);
+ cctx->cdict = cdict;
+ return 0;
+}
+
+size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't ref a pool when ctx not in init stage.");
+ cctx->pool = pool;
+ return 0;
+}
+
+size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
+{
+ return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent);
+}
+
+size_t ZSTD_CCtx_refPrefix_advanced(
+ ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't ref a prefix when ctx not in init stage.");
+ ZSTD_clearAllDicts(cctx);
+ if (prefix != NULL && prefixSize > 0) {
+ cctx->prefixDict.dict = prefix;
+ cctx->prefixDict.dictSize = prefixSize;
+ cctx->prefixDict.dictContentType = dictContentType;
+ }
+ return 0;
+}
+
+/*! ZSTD_CCtx_reset() :
+ * Also dumps dictionary */
+size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
+{
+ if ( (reset == ZSTD_reset_session_only)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ cctx->streamStage = zcss_init;
+ cctx->pledgedSrcSizePlusOne = 0;
+ }
+ if ( (reset == ZSTD_reset_parameters)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't reset parameters only when not in init stage.");
+ ZSTD_clearAllDicts(cctx);
+ ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx));
+ return ZSTD_CCtxParams_reset(&cctx->requestedParams);
+ }
+ return 0;
+}
+
+
+/** ZSTD_checkCParams() :
+ control CParam values remain within authorized range.
+ @return : 0, or an error code if one value is beyond authorized range */
+size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
+{
+ BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog);
+ BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog);
+ BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog);
+ BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog);
+ BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch);
+ BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength);
+ BOUNDCHECK(ZSTD_c_strategy, cParams.strategy);
+ return 0;
+}
+
+/** ZSTD_clampCParams() :
+ * make CParam values within valid range.
+ * @return : valid CParams */
+static ZSTD_compressionParameters
+ZSTD_clampCParams(ZSTD_compressionParameters cParams)
+{
+# define CLAMP_TYPE(cParam, val, type) { \
+ ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \
+ if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \
+ else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
+ }
+# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned)
+ CLAMP(ZSTD_c_windowLog, cParams.windowLog);
+ CLAMP(ZSTD_c_chainLog, cParams.chainLog);
+ CLAMP(ZSTD_c_hashLog, cParams.hashLog);
+ CLAMP(ZSTD_c_searchLog, cParams.searchLog);
+ CLAMP(ZSTD_c_minMatch, cParams.minMatch);
+ CLAMP(ZSTD_c_targetLength,cParams.targetLength);
+ CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy);
+ return cParams;
+}
+
+/** ZSTD_cycleLog() :
+ * condition for correct operation : hashLog > 1 */
+U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
+{
+ U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
+ return hashLog - btScale;
+}
+
+/** ZSTD_dictAndWindowLog() :
+ * Returns an adjusted window log that is large enough to fit the source and the dictionary.
+ * The zstd format says that the entire dictionary is valid if one byte of the dictionary
+ * is within the window. So the hashLog and chainLog should be large enough to reference both
+ * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing
+ * the hashLog and windowLog.
+ * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN.
+ */
+static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize)
+{
+ const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX;
+ /* No dictionary ==> No change */
+ if (dictSize == 0) {
+ return windowLog;
+ }
+ assert(windowLog <= ZSTD_WINDOWLOG_MAX);
+ assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */
+ {
+ U64 const windowSize = 1ULL << windowLog;
+ U64 const dictAndWindowSize = dictSize + windowSize;
+ /* If the window size is already large enough to fit both the source and the dictionary
+ * then just use the window size. Otherwise adjust so that it fits the dictionary and
+ * the window.
+ */
+ if (windowSize >= dictSize + srcSize) {
+ return windowLog; /* Window size large enough already */
+ } else if (dictAndWindowSize >= maxWindowSize) {
+ return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */
+ } else {
+ return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1;
+ }
+ }
+}
+
+/** ZSTD_adjustCParams_internal() :
+ * optimize `cPar` for a specified input (`srcSize` and `dictSize`).
+ * mostly downsize to reduce memory consumption and initialization latency.
+ * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known.
+ * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`.
+ * note : `srcSize==0` means 0!
+ * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */
+static ZSTD_compressionParameters
+ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
+ unsigned long long srcSize,
+ size_t dictSize,
+ ZSTD_cParamMode_e mode,
+ ZSTD_paramSwitch_e useRowMatchFinder)
+{
+ const U64 minSrcSize = 513; /* (1<<9) + 1 */
+ const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
+ assert(ZSTD_checkCParams(cPar)==0);
+
+ switch (mode) {
+ case ZSTD_cpm_unknown:
+ case ZSTD_cpm_noAttachDict:
+ /* If we don't know the source size, don't make any
+ * assumptions about it. We will already have selected
+ * smaller parameters if a dictionary is in use.
+ */
+ break;
+ case ZSTD_cpm_createCDict:
+ /* Assume a small source size when creating a dictionary
+ * with an unknown source size.
+ */
+ if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN)
+ srcSize = minSrcSize;
+ break;
+ case ZSTD_cpm_attachDict:
+ /* Dictionary has its own dedicated parameters which have
+ * already been selected. We are selecting parameters
+ * for only the source.
+ */
+ dictSize = 0;
+ break;
+ default:
+ assert(0);
+ break;
+ }
+
+ /* resize windowLog if input is small enough, to use less memory */
+ if ( (srcSize <= maxWindowResize)
+ && (dictSize <= maxWindowResize) ) {
+ U32 const tSize = (U32)(srcSize + dictSize);
+ static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
+ U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
+ ZSTD_highbit32(tSize-1) + 1;
+ if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
+ }
+ if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+ U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize);
+ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
+ if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1;
+ if (cycleLog > dictAndWindowLog)
+ cPar.chainLog -= (cycleLog - dictAndWindowLog);
+ }
+
+ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
+ cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */
+
+ /* We can't use more than 32 bits of hash in total, so that means that we require:
+ * (hashLog + 8) <= 32 && (chainLog + 8) <= 32
+ */
+ if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) {
+ U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS;
+ if (cPar.hashLog > maxShortCacheHashLog) {
+ cPar.hashLog = maxShortCacheHashLog;
+ }
+ if (cPar.chainLog > maxShortCacheHashLog) {
+ cPar.chainLog = maxShortCacheHashLog;
+ }
+ }
+
+
+ /* At this point, we aren't 100% sure if we are using the row match finder.
+ * Unless it is explicitly disabled, conservatively assume that it is enabled.
+ * In this case it will only be disabled for small sources, so shrinking the
+ * hash log a little bit shouldn't result in any ratio loss.
+ */
+ if (useRowMatchFinder == ZSTD_ps_auto)
+ useRowMatchFinder = ZSTD_ps_enable;
+
+ /* We can't hash more than 32-bits in total. So that means that we require:
+ * (hashLog - rowLog + 8) <= 32
+ */
+ if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) {
+ /* Switch to 32-entry rows if searchLog is 5 (or more) */
+ U32 const rowLog = BOUNDED(4, cPar.searchLog, 6);
+ U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS;
+ U32 const maxHashLog = maxRowHashLog + rowLog;
+ assert(cPar.hashLog >= rowLog);
+ if (cPar.hashLog > maxHashLog) {
+ cPar.hashLog = maxHashLog;
+ }
+ }
+
+ return cPar;
+}
+
+ZSTD_compressionParameters
+ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
+ unsigned long long srcSize,
+ size_t dictSize)
+{
+ cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */
+ if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto);
+}
+
+static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
+static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
+
+static void ZSTD_overrideCParams(
+ ZSTD_compressionParameters* cParams,
+ const ZSTD_compressionParameters* overrides)
+{
+ if (overrides->windowLog) cParams->windowLog = overrides->windowLog;
+ if (overrides->hashLog) cParams->hashLog = overrides->hashLog;
+ if (overrides->chainLog) cParams->chainLog = overrides->chainLog;
+ if (overrides->searchLog) cParams->searchLog = overrides->searchLog;
+ if (overrides->minMatch) cParams->minMatch = overrides->minMatch;
+ if (overrides->targetLength) cParams->targetLength = overrides->targetLength;
+ if (overrides->strategy) cParams->strategy = overrides->strategy;
+}
+
+ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
+ const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
+{
+ ZSTD_compressionParameters cParams;
+ if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) {
+ srcSizeHint = CCtxParams->srcSizeHint;
+ }
+ cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode);
+ if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
+ ZSTD_overrideCParams(&cParams, &CCtxParams->cParams);
+ assert(!ZSTD_checkCParams(cParams));
+ /* srcSizeHint == 0 means 0 */
+ return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder);
+}
+
+static size_t
+ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const U32 enableDedicatedDictSearch,
+ const U32 forCCtx)
+{
+ /* chain table size should be 0 for fast or row-hash strategies */
+ size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx)
+ ? ((size_t)1 << cParams->chainLog)
+ : 0;
+ size_t const hSize = ((size_t)1) << cParams->hashLog;
+ U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
+ size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
+ /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't
+ * surrounded by redzones in ASAN. */
+ size_t const tableSpace = chainSize * sizeof(U32)
+ + hSize * sizeof(U32)
+ + h3Size * sizeof(U32);
+ size_t const optPotentialSpace =
+ ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((1<<Litbits) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
+ + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
+ size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
+ ? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16))
+ : 0;
+ size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
+ ? optPotentialSpace
+ : 0;
+ size_t const slackSpace = ZSTD_cwksp_slack_space_required();
+
+ /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */
+ ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4);
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+
+ DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u",
+ (U32)chainSize, (U32)hSize, (U32)h3Size);
+ return tableSpace + optSpace + slackSpace + lazyAdditionalSpace;
+}
+
+/* Helper function for calculating memory requirements.
+ * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */
+static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) {
+ U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4;
+ return blockSize / divider;
+}
+
+static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ const ZSTD_compressionParameters* cParams,
+ const ldmParams_t* ldmParams,
+ const int isStatic,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const size_t buffInSize,
+ const size_t buffOutSize,
+ const U64 pledgedSrcSize,
+ int useSequenceProducer,
+ size_t maxBlockSize)
+{
+ size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
+ size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
+ size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer);
+ size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
+ + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
+ size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE);
+ size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
+ size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
+
+ size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams);
+ size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize);
+ size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ?
+ ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
+
+
+ size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize)
+ + ZSTD_cwksp_alloc_size(buffOutSize);
+
+ size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
+
+ size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+ size_t const externalSeqSpace = useSequenceProducer
+ ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+ : 0;
+
+ size_t const neededSpace =
+ cctxSpace +
+ entropySpace +
+ blockStateSpace +
+ ldmSpace +
+ ldmSeqSpace +
+ matchStateSize +
+ tokenSpace +
+ bufferSpace +
+ externalSeqSpace;
+
+ DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
+ return neededSpace;
+}
+
+size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
+{
+ ZSTD_compressionParameters const cParams =
+ ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
+ ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder,
+ &cParams);
+
+ RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
+ /* estimateCCtxSize is for one-shot compression. So no buffers should
+ * be needed. However, we still allocate two 0-sized buffers, which can
+ * take space under ASAN. */
+ return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
+}
+
+size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
+{
+ ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
+ if (ZSTD_rowMatchFinderSupported(cParams.strategy)) {
+ /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
+ size_t noRowCCtxSize;
+ size_t rowCCtxSize;
+ initialParams.useRowMatchFinder = ZSTD_ps_disable;
+ noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
+ initialParams.useRowMatchFinder = ZSTD_ps_enable;
+ rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
+ return MAX(noRowCCtxSize, rowCCtxSize);
+ } else {
+ return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
+ }
+}
+
+static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
+{
+ int tier = 0;
+ size_t largestSize = 0;
+ static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN};
+ for (; tier < 4; ++tier) {
+ /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict);
+ largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize);
+ }
+ return largestSize;
+}
+
+size_t ZSTD_estimateCCtxSize(int compressionLevel)
+{
+ int level;
+ size_t memBudget = 0;
+ for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
+ /* Ensure monotonically increasing memory usage as compression level increases */
+ size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
+ if (newMB > memBudget) memBudget = newMB;
+ }
+ return memBudget;
+}
+
+size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
+{
+ RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
+ { ZSTD_compressionParameters const cParams =
+ ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
+ size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
+ size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
+ ? ((size_t)1 << cParams.windowLog) + blockSize
+ : 0;
+ size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered)
+ ? ZSTD_compressBound(blockSize) + 1
+ : 0;
+ ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &params->cParams);
+
+ return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
+ ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
+ }
+}
+
+size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams)
+{
+ ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
+ if (ZSTD_rowMatchFinderSupported(cParams.strategy)) {
+ /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
+ size_t noRowCCtxSize;
+ size_t rowCCtxSize;
+ initialParams.useRowMatchFinder = ZSTD_ps_disable;
+ noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
+ initialParams.useRowMatchFinder = ZSTD_ps_enable;
+ rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
+ return MAX(noRowCCtxSize, rowCCtxSize);
+ } else {
+ return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
+ }
+}
+
+static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel)
+{
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
+ return ZSTD_estimateCStreamSize_usingCParams(cParams);
+}
+
+size_t ZSTD_estimateCStreamSize(int compressionLevel)
+{
+ int level;
+ size_t memBudget = 0;
+ for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
+ size_t const newMB = ZSTD_estimateCStreamSize_internal(level);
+ if (newMB > memBudget) memBudget = newMB;
+ }
+ return memBudget;
+}
+
+/* ZSTD_getFrameProgression():
+ * tells how much data has been consumed (input) and produced (output) for current frame.
+ * able to count progression inside worker threads (non-blocking mode).
+ */
+ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers > 0) {
+ return ZSTDMT_getFrameProgression(cctx->mtctx);
+ }
+#endif
+ { ZSTD_frameProgression fp;
+ size_t const buffered = (cctx->inBuff == NULL) ? 0 :
+ cctx->inBuffPos - cctx->inToCompress;
+ if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress);
+ assert(buffered <= ZSTD_BLOCKSIZE_MAX);
+ fp.ingested = cctx->consumedSrcSize + buffered;
+ fp.consumed = cctx->consumedSrcSize;
+ fp.produced = cctx->producedCSize;
+ fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */
+ fp.currentJobID = 0;
+ fp.nbActiveWorkers = 0;
+ return fp;
+} }
+
+/*! ZSTD_toFlushNow()
+ * Only useful for multithreading scenarios currently (nbWorkers >= 1).
+ */
+size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers > 0) {
+ return ZSTDMT_toFlushNow(cctx->mtctx);
+ }
+#endif
+ (void)cctx;
+ return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */
+}
+
+static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1,
+ ZSTD_compressionParameters cParams2)
+{
+ (void)cParams1;
+ (void)cParams2;
+ assert(cParams1.windowLog == cParams2.windowLog);
+ assert(cParams1.chainLog == cParams2.chainLog);
+ assert(cParams1.hashLog == cParams2.hashLog);
+ assert(cParams1.searchLog == cParams2.searchLog);
+ assert(cParams1.minMatch == cParams2.minMatch);
+ assert(cParams1.targetLength == cParams2.targetLength);
+ assert(cParams1.strategy == cParams2.strategy);
+}
+
+void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs)
+{
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; ++i)
+ bs->rep[i] = repStartValue[i];
+ bs->entropy.huf.repeatMode = HUF_repeat_none;
+ bs->entropy.fse.offcode_repeatMode = FSE_repeat_none;
+ bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none;
+ bs->entropy.fse.litlength_repeatMode = FSE_repeat_none;
+}
+
+/*! ZSTD_invalidateMatchState()
+ * Invalidate all the matches in the match finder tables.
+ * Requires nextSrc and base to be set (can be NULL).
+ */
+static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms)
+{
+ ZSTD_window_clear(&ms->window);
+
+ ms->nextToUpdate = ms->window.dictLimit;
+ ms->loadedDictEnd = 0;
+ ms->opt.litLengthSum = 0; /* force reset of btopt stats */
+ ms->dictMatchState = NULL;
+}
+
+/**
+ * Controls, for this matchState reset, whether the tables need to be cleared /
+ * prepared for the coming compression (ZSTDcrp_makeClean), or whether the
+ * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a
+ * subsequent operation will overwrite the table space anyways (e.g., copying
+ * the matchState contents in from a CDict).
+ */
+typedef enum {
+ ZSTDcrp_makeClean,
+ ZSTDcrp_leaveDirty
+} ZSTD_compResetPolicy_e;
+
+/**
+ * Controls, for this matchState reset, whether indexing can continue where it
+ * left off (ZSTDirp_continue), or whether it needs to be restarted from zero
+ * (ZSTDirp_reset).
+ */
+typedef enum {
+ ZSTDirp_continue,
+ ZSTDirp_reset
+} ZSTD_indexResetPolicy_e;
+
+typedef enum {
+ ZSTD_resetTarget_CDict,
+ ZSTD_resetTarget_CCtx
+} ZSTD_resetTarget_e;
+
+
+static size_t
+ZSTD_reset_matchState(ZSTD_matchState_t* ms,
+ ZSTD_cwksp* ws,
+ const ZSTD_compressionParameters* cParams,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const ZSTD_compResetPolicy_e crp,
+ const ZSTD_indexResetPolicy_e forceResetIndex,
+ const ZSTD_resetTarget_e forWho)
+{
+ /* disable chain table allocation for fast or row-based strategies */
+ size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder,
+ ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict))
+ ? ((size_t)1 << cParams->chainLog)
+ : 0;
+ size_t const hSize = ((size_t)1) << cParams->hashLog;
+ U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
+ size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
+
+ DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+ if (forceResetIndex == ZSTDirp_reset) {
+ ZSTD_window_init(&ms->window);
+ ZSTD_cwksp_mark_tables_dirty(ws);
+ }
+
+ ms->hashLog3 = hashLog3;
+
+ ZSTD_invalidateMatchState(ms);
+
+ assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */
+
+ ZSTD_cwksp_clear_tables(ws);
+
+ DEBUGLOG(5, "reserving table space");
+ /* table Space */
+ ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32));
+ ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32));
+ ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32));
+ RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
+ "failed a workspace allocation in ZSTD_reset_matchState");
+
+ DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty);
+ if (crp!=ZSTDcrp_leaveDirty) {
+ /* reset tables only */
+ ZSTD_cwksp_clean_tables(ws);
+ }
+
+ /* opt parser space */
+ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
+ DEBUGLOG(4, "reserving optimal parser space");
+ ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<<Litbits) * sizeof(unsigned));
+ ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned));
+ ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned));
+ ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned));
+ ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t));
+ ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
+ }
+
+ if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
+ { /* Row match finder needs an additional table of hashes ("tags") */
+ size_t const tagTableSize = hSize*sizeof(U16);
+ ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
+ if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize);
+ }
+ { /* Switch to 32-entry rows if searchLog is 5 (or more) */
+ U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
+ assert(cParams->hashLog >= rowLog);
+ ms->rowHashLog = cParams->hashLog - rowLog;
+ }
+ }
+
+ ms->cParams = *cParams;
+
+ RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
+ "failed a workspace allocation in ZSTD_reset_matchState");
+ return 0;
+}
+
+/* ZSTD_indexTooCloseToMax() :
+ * minor optimization : prefer memset() rather than reduceIndex()
+ * which is measurably slow in some circumstances (reported for Visual Studio).
+ * Works when re-using a context for a lot of smallish inputs :
+ * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN,
+ * memset() will be triggered before reduceIndex().
+ */
+#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB)
+static int ZSTD_indexTooCloseToMax(ZSTD_window_t w)
+{
+ return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN);
+}
+
+/** ZSTD_dictTooBig():
+ * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in
+ * one go generically. So we ensure that in that case we reset the tables to zero,
+ * so that we can load as much of the dictionary as possible.
+ */
+static int ZSTD_dictTooBig(size_t const loadedDictSize)
+{
+ return loadedDictSize > ZSTD_CHUNKSIZE_MAX;
+}
+
+/*! ZSTD_resetCCtx_internal() :
+ * @param loadedDictSize The size of the dictionary to be loaded
+ * into the context, if any. If no dictionary is used, or the
+ * dictionary is being attached / copied, then pass 0.
+ * note : `params` are assumed fully validated at this stage.
+ */
+static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
+ ZSTD_CCtx_params const* params,
+ U64 const pledgedSrcSize,
+ size_t const loadedDictSize,
+ ZSTD_compResetPolicy_e const crp,
+ ZSTD_buffered_policy_e const zbuff)
+{
+ ZSTD_cwksp* const ws = &zc->workspace;
+ DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d",
+ (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter);
+ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
+
+ zc->isFirstBlock = 1;
+
+ /* Set applied params early so we can modify them for LDM,
+ * and point params at the applied params.
+ */
+ zc->appliedParams = *params;
+ params = &zc->appliedParams;
+
+ assert(params->useRowMatchFinder != ZSTD_ps_auto);
+ assert(params->useBlockSplitter != ZSTD_ps_auto);
+ assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+ assert(params->maxBlockSize != 0);
+ if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
+ /* Adjust long distance matching parameters */
+ ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
+ assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog);
+ assert(params->ldmParams.hashRateLog < 32);
+ }
+
+ { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
+ size_t const blockSize = MIN(params->maxBlockSize, windowSize);
+ size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useSequenceProducer);
+ size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
+ ? ZSTD_compressBound(blockSize) + 1
+ : 0;
+ size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered)
+ ? windowSize + blockSize
+ : 0;
+ size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize);
+
+ int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window);
+ int const dictTooBig = ZSTD_dictTooBig(loadedDictSize);
+ ZSTD_indexResetPolicy_e needsIndexReset =
+ (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue;
+
+ size_t const neededSpace =
+ ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
+ buffInSize, buffOutSize, pledgedSrcSize, params->useSequenceProducer, params->maxBlockSize);
+ int resizeWorkspace;
+
+ FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
+
+ if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0);
+
+ { /* Check if workspace is large enough, alloc a new one if needed */
+ int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace;
+ int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace);
+ resizeWorkspace = workspaceTooSmall || workspaceWasteful;
+ DEBUGLOG(4, "Need %zu B workspace", neededSpace);
+ DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize);
+
+ if (resizeWorkspace) {
+ DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB",
+ ZSTD_cwksp_sizeof(ws) >> 10,
+ neededSpace >> 10);
+
+ RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize");
+
+ needsIndexReset = ZSTDirp_reset;
+
+ ZSTD_cwksp_free(ws, zc->customMem);
+ FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), "");
+
+ DEBUGLOG(5, "reserving object space");
+ /* Statically sized space.
+ * entropyWorkspace never moves,
+ * though prev/next block swap places */
+ assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t)));
+ zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
+ RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock");
+ zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
+ RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock");
+ zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE);
+ RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace");
+ } }
+
+ ZSTD_cwksp_clear(ws);
+
+ /* init params */
+ zc->blockState.matchState.cParams = params->cParams;
+ zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable;
+ zc->pledgedSrcSizePlusOne = pledgedSrcSize+1;
+ zc->consumedSrcSize = 0;
+ zc->producedCSize = 0;
+ if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)
+ zc->appliedParams.fParams.contentSizeFlag = 0;
+ DEBUGLOG(4, "pledged content size : %u ; flag : %u",
+ (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag);
+ zc->blockSize = blockSize;
+
+ XXH64_reset(&zc->xxhState, 0);
+ zc->stage = ZSTDcs_init;
+ zc->dictID = 0;
+ zc->dictContentSize = 0;
+
+ ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);
+
+ /* ZSTD_wildcopy() is used to copy into the literals buffer,
+ * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes.
+ */
+ zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH);
+ zc->seqStore.maxNbLit = blockSize;
+
+ /* buffers */
+ zc->bufferedPolicy = zbuff;
+ zc->inBuffSize = buffInSize;
+ zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
+ zc->outBuffSize = buffOutSize;
+ zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize);
+
+ /* ldm bucketOffsets table */
+ if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
+ /* TODO: avoid memset? */
+ size_t const numBuckets =
+ ((size_t)1) << (params->ldmParams.hashLog -
+ params->ldmParams.bucketSizeLog);
+ zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets);
+ ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets);
+ }
+
+ /* sequences storage */
+ ZSTD_referenceExternalSequences(zc, NULL, 0);
+ zc->seqStore.maxNbSeq = maxNbSeq;
+ zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
+ zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
+ zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
+ zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
+
+ FORWARD_IF_ERROR(ZSTD_reset_matchState(
+ &zc->blockState.matchState,
+ ws,
+ &params->cParams,
+ params->useRowMatchFinder,
+ crp,
+ needsIndexReset,
+ ZSTD_resetTarget_CCtx), "");
+
+ /* ldm hash table */
+ if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
+ /* TODO: avoid memset? */
+ size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
+ zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
+ ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
+ zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
+ zc->maxNbLdmSequences = maxNbLdmSeq;
+
+ ZSTD_window_init(&zc->ldmState.window);
+ zc->ldmState.loadedDictEnd = 0;
+ }
+
+ /* reserve space for block-level external sequences */
+ if (params->useSequenceProducer) {
+ size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+ zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
+ zc->externalMatchCtx.seqBuffer =
+ (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
+ }
+
+ DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
+ assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
+
+ zc->initialized = 1;
+
+ return 0;
+ }
+}
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
+ int i;
+ for (i=0; i<ZSTD_REP_NUM; i++) cctx->blockState.prevCBlock->rep[i] = 0;
+ assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window));
+}
+
+/* These are the approximate sizes for each strategy past which copying the
+ * dictionary tables into the working context is faster than using them
+ * in-place.
+ */
+static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = {
+ 8 KB, /* unused */
+ 8 KB, /* ZSTD_fast */
+ 16 KB, /* ZSTD_dfast */
+ 32 KB, /* ZSTD_greedy */
+ 32 KB, /* ZSTD_lazy */
+ 32 KB, /* ZSTD_lazy2 */
+ 32 KB, /* ZSTD_btlazy2 */
+ 32 KB, /* ZSTD_btopt */
+ 8 KB, /* ZSTD_btultra */
+ 8 KB /* ZSTD_btultra2 */
+};
+
+static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ U64 pledgedSrcSize)
+{
+ size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy];
+ int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch;
+ return dedicatedDictSearch
+ || ( ( pledgedSrcSize <= cutoff
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || params->attachDictPref == ZSTD_dictForceAttach )
+ && params->attachDictPref != ZSTD_dictForceCopy
+ && !params->forceWindow ); /* dictMatchState isn't correctly
+ * handled in _enforceMaxDist */
+}
+
+static size_t
+ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
+ const ZSTD_CDict* cdict,
+ ZSTD_CCtx_params params,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu",
+ (unsigned long long)pledgedSrcSize);
+ {
+ ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams;
+ unsigned const windowLog = params.cParams.windowLog;
+ assert(windowLog != 0);
+ /* Resize working context table params for input only, since the dict
+ * has its own tables. */
+ /* pledgedSrcSize == 0 means 0! */
+
+ if (cdict->matchState.dedicatedDictSearch) {
+ ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams);
+ }
+
+ params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize,
+ cdict->dictContentSize, ZSTD_cpm_attachDict,
+ params.useRowMatchFinder);
+ params.cParams.windowLog = windowLog;
+ params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */
+ FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
+ /* loadedDictSize */ 0,
+ ZSTDcrp_makeClean, zbuff), "");
+ assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy);
+ }
+
+ { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc
+ - cdict->matchState.window.base);
+ const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit;
+ if (cdictLen == 0) {
+ /* don't even attach dictionaries with no contents */
+ DEBUGLOG(4, "skipping attaching empty dictionary");
+ } else {
+ DEBUGLOG(4, "attaching dictionary into context");
+ cctx->blockState.matchState.dictMatchState = &cdict->matchState;
+
+ /* prep working match state so dict matches never have negative indices
+ * when they are translated to the working context's index space. */
+ if (cctx->blockState.matchState.window.dictLimit < cdictEnd) {
+ cctx->blockState.matchState.window.nextSrc =
+ cctx->blockState.matchState.window.base + cdictEnd;
+ ZSTD_window_clear(&cctx->blockState.matchState.window);
+ }
+ /* loadedDictEnd is expressed within the referential of the active context */
+ cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit;
+ } }
+
+ cctx->dictID = cdict->dictID;
+ cctx->dictContentSize = cdict->dictContentSize;
+
+ /* copy block state */
+ ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
+
+ return 0;
+}
+
+static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize,
+ ZSTD_compressionParameters const* cParams) {
+ if (ZSTD_CDictIndicesAreTagged(cParams)){
+ /* Remove tags from the CDict table if they are present.
+ * See docs on "short cache" in zstd_compress_internal.h for context. */
+ size_t i;
+ for (i = 0; i < tableSize; i++) {
+ U32 const taggedIndex = src[i];
+ U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS;
+ dst[i] = index;
+ }
+ } else {
+ ZSTD_memcpy(dst, src, tableSize * sizeof(U32));
+ }
+}
+
+static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
+ const ZSTD_CDict* cdict,
+ ZSTD_CCtx_params params,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
+
+ assert(!cdict->matchState.dedicatedDictSearch);
+ DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu",
+ (unsigned long long)pledgedSrcSize);
+
+ { unsigned const windowLog = params.cParams.windowLog;
+ assert(windowLog != 0);
+ /* Copy only compression parameters related to tables. */
+ params.cParams = *cdict_cParams;
+ params.cParams.windowLog = windowLog;
+ params.useRowMatchFinder = cdict->useRowMatchFinder;
+ FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
+ /* loadedDictSize */ 0,
+ ZSTDcrp_leaveDirty, zbuff), "");
+ assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
+ assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
+ assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
+ }
+
+ ZSTD_cwksp_mark_tables_dirty(&cctx->workspace);
+ assert(params.useRowMatchFinder != ZSTD_ps_auto);
+
+ /* copy tables */
+ { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */)
+ ? ((size_t)1 << cdict_cParams->chainLog)
+ : 0;
+ size_t const hSize = (size_t)1 << cdict_cParams->hashLog;
+
+ ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable,
+ cdict->matchState.hashTable,
+ hSize, cdict_cParams);
+
+ /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */
+ if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) {
+ ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable,
+ cdict->matchState.chainTable,
+ chainSize, cdict_cParams);
+ }
+ /* copy tag table */
+ if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) {
+ size_t const tagTableSize = hSize*sizeof(U16);
+ ZSTD_memcpy(cctx->blockState.matchState.tagTable,
+ cdict->matchState.tagTable,
+ tagTableSize);
+ }
+ }
+
+ /* Zero the hashTable3, since the cdict never fills it */
+ { int const h3log = cctx->blockState.matchState.hashLog3;
+ size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
+ assert(cdict->matchState.hashLog3 == 0);
+ ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32));
+ }
+
+ ZSTD_cwksp_mark_tables_clean(&cctx->workspace);
+
+ /* copy dictionary offsets */
+ { ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
+ ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
+ dstMatchState->window = srcMatchState->window;
+ dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
+ dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
+ }
+
+ cctx->dictID = cdict->dictID;
+ cctx->dictContentSize = cdict->dictContentSize;
+
+ /* copy block state */
+ ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
+
+ return 0;
+}
+
+/* We have a choice between copying the dictionary context into the working
+ * context, or referencing the dictionary context from the working context
+ * in-place. We decide here which strategy to use. */
+static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+
+ DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)",
+ (unsigned)pledgedSrcSize);
+
+ if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) {
+ return ZSTD_resetCCtx_byAttachingCDict(
+ cctx, cdict, *params, pledgedSrcSize, zbuff);
+ } else {
+ return ZSTD_resetCCtx_byCopyingCDict(
+ cctx, cdict, *params, pledgedSrcSize, zbuff);
+ }
+}
+
+/*! ZSTD_copyCCtx_internal() :
+ * Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
+ * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
+ * The "context", in this case, refers to the hash and chain tables,
+ * entropy tables, and dictionary references.
+ * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx.
+ * @return : 0, or an error code */
+static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
+ const ZSTD_CCtx* srcCCtx,
+ ZSTD_frameParameters fParams,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong,
+ "Can't copy a ctx that's not in init stage.");
+ DEBUGLOG(5, "ZSTD_copyCCtx_internal");
+ ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
+ { ZSTD_CCtx_params params = dstCCtx->requestedParams;
+ /* Copy only compression parameters related to tables. */
+ params.cParams = srcCCtx->appliedParams.cParams;
+ assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto);
+ assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto);
+ assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto);
+ params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder;
+ params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
+ params.ldmParams = srcCCtx->appliedParams.ldmParams;
+ params.fParams = fParams;
+ params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
+ ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
+ /* loadedDictSize */ 0,
+ ZSTDcrp_leaveDirty, zbuff);
+ assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog);
+ assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy);
+ assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog);
+ assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog);
+ assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3);
+ }
+
+ ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace);
+
+ /* copy tables */
+ { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy,
+ srcCCtx->appliedParams.useRowMatchFinder,
+ 0 /* forDDSDict */)
+ ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog)
+ : 0;
+ size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
+ int const h3log = srcCCtx->blockState.matchState.hashLog3;
+ size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
+
+ ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable,
+ srcCCtx->blockState.matchState.hashTable,
+ hSize * sizeof(U32));
+ ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable,
+ srcCCtx->blockState.matchState.chainTable,
+ chainSize * sizeof(U32));
+ ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3,
+ srcCCtx->blockState.matchState.hashTable3,
+ h3Size * sizeof(U32));
+ }
+
+ ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace);
+
+ /* copy dictionary offsets */
+ {
+ const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState;
+ ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState;
+ dstMatchState->window = srcMatchState->window;
+ dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
+ dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
+ }
+ dstCCtx->dictID = srcCCtx->dictID;
+ dstCCtx->dictContentSize = srcCCtx->dictContentSize;
+
+ /* copy block state */
+ ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock));
+
+ return 0;
+}
+
+/*! ZSTD_copyCCtx() :
+ * Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
+ * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
+ * pledgedSrcSize==0 means "unknown".
+* @return : 0, or an error code */
+size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
+{
+ ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+ ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy;
+ ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
+ if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);
+
+ return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx,
+ fParams, pledgedSrcSize,
+ zbuff);
+}
+
+
+#define ZSTD_ROWSIZE 16
+/*! ZSTD_reduceTable() :
+ * reduce table indexes by `reducerValue`, or squash to zero.
+ * PreserveMark preserves "unsorted mark" for btlazy2 strategy.
+ * It must be set to a clear 0/1 value, to remove branch during inlining.
+ * Presume table size is a multiple of ZSTD_ROWSIZE
+ * to help auto-vectorization */
+FORCE_INLINE_TEMPLATE void
+ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark)
+{
+ int const nbRows = (int)size / ZSTD_ROWSIZE;
+ int cellNb = 0;
+ int rowNb;
+ /* Protect special index values < ZSTD_WINDOW_START_INDEX. */
+ U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX;
+ assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */
+ assert(size < (1U<<31)); /* can be casted to int */
+
+#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
+ /* To validate that the table re-use logic is sound, and that we don't
+ * access table space that we haven't cleaned, we re-"poison" the table
+ * space every time we mark it dirty.
+ *
+ * This function however is intended to operate on those dirty tables and
+ * re-clean them. So when this function is used correctly, we can unpoison
+ * the memory it operated on. This introduces a blind spot though, since
+ * if we now try to operate on __actually__ poisoned memory, we will not
+ * detect that. */
+ __msan_unpoison(table, size * sizeof(U32));
+#endif
+
+ for (rowNb=0 ; rowNb < nbRows ; rowNb++) {
+ int column;
+ for (column=0; column<ZSTD_ROWSIZE; column++) {
+ U32 newVal;
+ if (preserveMark && table[cellNb] == ZSTD_DUBT_UNSORTED_MARK) {
+ /* This write is pointless, but is required(?) for the compiler
+ * to auto-vectorize the loop. */
+ newVal = ZSTD_DUBT_UNSORTED_MARK;
+ } else if (table[cellNb] < reducerThreshold) {
+ newVal = 0;
+ } else {
+ newVal = table[cellNb] - reducerValue;
+ }
+ table[cellNb] = newVal;
+ cellNb++;
+ } }
+}
+
+static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue)
+{
+ ZSTD_reduceTable_internal(table, size, reducerValue, 0);
+}
+
+static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue)
+{
+ ZSTD_reduceTable_internal(table, size, reducerValue, 1);
+}
+
+/*! ZSTD_reduceIndex() :
+* rescale all indexes to avoid future overflow (indexes are U32) */
+static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue)
+{
+ { U32 const hSize = (U32)1 << params->cParams.hashLog;
+ ZSTD_reduceTable(ms->hashTable, hSize, reducerValue);
+ }
+
+ if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) {
+ U32 const chainSize = (U32)1 << params->cParams.chainLog;
+ if (params->cParams.strategy == ZSTD_btlazy2)
+ ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue);
+ else
+ ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue);
+ }
+
+ if (ms->hashLog3) {
+ U32 const h3Size = (U32)1 << ms->hashLog3;
+ ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue);
+ }
+}
+
+
+/*-*******************************************************
+* Block entropic compression
+*********************************************************/
+
+/* See doc/zstd_compression_format.md for detailed format description */
+
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
+{
+ const seqDef* const sequences = seqStorePtr->sequencesStart;
+ BYTE* const llCodeTable = seqStorePtr->llCode;
+ BYTE* const ofCodeTable = seqStorePtr->ofCode;
+ BYTE* const mlCodeTable = seqStorePtr->mlCode;
+ U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ U32 u;
+ int longOffsets = 0;
+ assert(nbSeq <= seqStorePtr->maxNbSeq);
+ for (u=0; u<nbSeq; u++) {
+ U32 const llv = sequences[u].litLength;
+ U32 const ofCode = ZSTD_highbit32(sequences[u].offBase);
+ U32 const mlv = sequences[u].mlBase;
+ llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
+ ofCodeTable[u] = (BYTE)ofCode;
+ mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
+ assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN));
+ if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
+ longOffsets = 1;
+ }
+ if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
+ llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
+ if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
+ mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
+ return longOffsets;
+}
+
+/* ZSTD_useTargetCBlockSize():
+ * Returns if target compressed block size param is being used.
+ * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize.
+ * Returns 1 if true, 0 otherwise. */
+static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams)
+{
+ DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize);
+ return (cctxParams->targetCBlockSize != 0);
+}
+
+/* ZSTD_blockSplitterEnabled():
+ * Returns if block splitting param is being used
+ * If used, compression will do best effort to split a block in order to improve compression ratio.
+ * At the time this function is called, the parameter must be finalized.
+ * Returns 1 if true, 0 otherwise. */
+static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams)
+{
+ DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter);
+ assert(cctxParams->useBlockSplitter != ZSTD_ps_auto);
+ return (cctxParams->useBlockSplitter == ZSTD_ps_enable);
+}
+
+/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types
+ * and size of the sequences statistics
+ */
+typedef struct {
+ U32 LLtype;
+ U32 Offtype;
+ U32 MLtype;
+ size_t size;
+ size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
+ int longOffsets;
+} ZSTD_symbolEncodingTypeStats_t;
+
+/* ZSTD_buildSequencesStatistics():
+ * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field.
+ * Modifies `nextEntropy` to have the appropriate values as a side effect.
+ * nbSeq must be greater than 0.
+ *
+ * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
+ */
+static ZSTD_symbolEncodingTypeStats_t
+ZSTD_buildSequencesStatistics(
+ const seqStore_t* seqStorePtr, size_t nbSeq,
+ const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
+ BYTE* dst, const BYTE* const dstEnd,
+ ZSTD_strategy strategy, unsigned* countWorkspace,
+ void* entropyWorkspace, size_t entropyWkspSize)
+{
+ BYTE* const ostart = dst;
+ const BYTE* const oend = dstEnd;
+ BYTE* op = ostart;
+ FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
+ FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
+ FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
+ const BYTE* const ofCodeTable = seqStorePtr->ofCode;
+ const BYTE* const llCodeTable = seqStorePtr->llCode;
+ const BYTE* const mlCodeTable = seqStorePtr->mlCode;
+ ZSTD_symbolEncodingTypeStats_t stats;
+
+ stats.lastCountSize = 0;
+ /* convert length/distances into codes */
+ stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
+ assert(op <= oend);
+ assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
+ /* build CTable for Literal Lengths */
+ { unsigned max = MaxLL;
+ size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
+ DEBUGLOG(5, "Building LL table");
+ nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
+ stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
+ countWorkspace, max, mostFrequent, nbSeq,
+ LLFSELog, prevEntropy->litlengthCTable,
+ LL_defaultNorm, LL_defaultNormLog,
+ ZSTD_defaultAllowed, strategy);
+ assert(set_basic < set_compressed && set_rle < set_compressed);
+ assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
+ { size_t const countSize = ZSTD_buildCTable(
+ op, (size_t)(oend - op),
+ CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype,
+ countWorkspace, max, llCodeTable, nbSeq,
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
+ prevEntropy->litlengthCTable,
+ sizeof(prevEntropy->litlengthCTable),
+ entropyWorkspace, entropyWkspSize);
+ if (ZSTD_isError(countSize)) {
+ DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed");
+ stats.size = countSize;
+ return stats;
+ }
+ if (stats.LLtype == set_compressed)
+ stats.lastCountSize = countSize;
+ op += countSize;
+ assert(op <= oend);
+ } }
+ /* build CTable for Offsets */
+ { unsigned max = MaxOff;
+ size_t const mostFrequent = HIST_countFast_wksp(
+ countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
+ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
+ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
+ DEBUGLOG(5, "Building OF table");
+ nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
+ stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
+ countWorkspace, max, mostFrequent, nbSeq,
+ OffFSELog, prevEntropy->offcodeCTable,
+ OF_defaultNorm, OF_defaultNormLog,
+ defaultPolicy, strategy);
+ assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
+ { size_t const countSize = ZSTD_buildCTable(
+ op, (size_t)(oend - op),
+ CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype,
+ countWorkspace, max, ofCodeTable, nbSeq,
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+ prevEntropy->offcodeCTable,
+ sizeof(prevEntropy->offcodeCTable),
+ entropyWorkspace, entropyWkspSize);
+ if (ZSTD_isError(countSize)) {
+ DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed");
+ stats.size = countSize;
+ return stats;
+ }
+ if (stats.Offtype == set_compressed)
+ stats.lastCountSize = countSize;
+ op += countSize;
+ assert(op <= oend);
+ } }
+ /* build CTable for MatchLengths */
+ { unsigned max = MaxML;
+ size_t const mostFrequent = HIST_countFast_wksp(
+ countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
+ DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
+ nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
+ stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
+ countWorkspace, max, mostFrequent, nbSeq,
+ MLFSELog, prevEntropy->matchlengthCTable,
+ ML_defaultNorm, ML_defaultNormLog,
+ ZSTD_defaultAllowed, strategy);
+ assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
+ { size_t const countSize = ZSTD_buildCTable(
+ op, (size_t)(oend - op),
+ CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype,
+ countWorkspace, max, mlCodeTable, nbSeq,
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
+ prevEntropy->matchlengthCTable,
+ sizeof(prevEntropy->matchlengthCTable),
+ entropyWorkspace, entropyWkspSize);
+ if (ZSTD_isError(countSize)) {
+ DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed");
+ stats.size = countSize;
+ return stats;
+ }
+ if (stats.MLtype == set_compressed)
+ stats.lastCountSize = countSize;
+ op += countSize;
+ assert(op <= oend);
+ } }
+ stats.size = (size_t)(op-ostart);
+ return stats;
+}
+
+/* ZSTD_entropyCompressSeqStore_internal():
+ * compresses both literals and sequences
+ * Returns compressed size of block, or a zstd error.
+ */
+#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
+MEM_STATIC size_t
+ZSTD_entropyCompressSeqStore_internal(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ void* entropyWorkspace, size_t entropyWkspSize,
+ const int bmi2)
+{
+ ZSTD_strategy const strategy = cctxParams->cParams.strategy;
+ unsigned* count = (unsigned*)entropyWorkspace;
+ FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
+ FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
+ FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
+ const seqDef* const sequences = seqStorePtr->sequencesStart;
+ const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ const BYTE* const ofCodeTable = seqStorePtr->ofCode;
+ const BYTE* const llCodeTable = seqStorePtr->llCode;
+ const BYTE* const mlCodeTable = seqStorePtr->mlCode;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart;
+ size_t lastCountSize;
+ int longOffsets = 0;
+
+ entropyWorkspace = count + (MaxSeq + 1);
+ entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
+
+ DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity);
+ ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
+ assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
+
+ /* Compress literals */
+ { const BYTE* const literals = seqStorePtr->litStart;
+ size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+ /* Base suspicion of uncompressibility on ratio of literals to sequences */
+ unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
+ size_t const litSize = (size_t)(seqStorePtr->lit - literals);
+
+ size_t const cSize = ZSTD_compressLiterals(
+ op, dstCapacity,
+ literals, litSize,
+ entropyWorkspace, entropyWkspSize,
+ &prevEntropy->huf, &nextEntropy->huf,
+ cctxParams->cParams.strategy,
+ ZSTD_literalsCompressionIsDisabled(cctxParams),
+ suspectUncompressible, bmi2);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
+ assert(cSize <= dstCapacity);
+ op += cSize;
+ }
+
+ /* Sequences Header */
+ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
+ dstSize_tooSmall, "Can't fit seq hdr in output buf!");
+ if (nbSeq < 128) {
+ *op++ = (BYTE)nbSeq;
+ } else if (nbSeq < LONGNBSEQ) {
+ op[0] = (BYTE)((nbSeq>>8) + 0x80);
+ op[1] = (BYTE)nbSeq;
+ op+=2;
+ } else {
+ op[0]=0xFF;
+ MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ));
+ op+=3;
+ }
+ assert(op <= oend);
+ if (nbSeq==0) {
+ /* Copy the old tables over as if we repeated them */
+ ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
+ return (size_t)(op - ostart);
+ }
+ { BYTE* const seqHead = op++;
+ /* build stats for sequences */
+ const ZSTD_symbolEncodingTypeStats_t stats =
+ ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
+ &prevEntropy->fse, &nextEntropy->fse,
+ op, oend,
+ strategy, count,
+ entropyWorkspace, entropyWkspSize);
+ FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
+ *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
+ lastCountSize = stats.lastCountSize;
+ op += stats.size;
+ longOffsets = stats.longOffsets;
+ }
+
+ { size_t const bitstreamSize = ZSTD_encodeSequences(
+ op, (size_t)(oend - op),
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq,
+ longOffsets, bmi2);
+ FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
+ op += bitstreamSize;
+ assert(op <= oend);
+ /* zstd versions <= 1.3.4 mistakenly report corruption when
+ * FSE_readNCount() receives a buffer < 4 bytes.
+ * Fixed by https://github.com/facebook/zstd/pull/1146.
+ * This can happen when the last set_compressed table present is 2
+ * bytes and the bitstream is only one byte.
+ * In this exceedingly rare case, we will simply emit an uncompressed
+ * block, since it isn't worth optimizing.
+ */
+ if (lastCountSize && (lastCountSize + bitstreamSize) < 4) {
+ /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
+ assert(lastCountSize + bitstreamSize == 3);
+ DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
+ "emitting an uncompressed block.");
+ return 0;
+ }
+ }
+
+ DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart));
+ return (size_t)(op - ostart);
+}
+
+MEM_STATIC size_t
+ZSTD_entropyCompressSeqStore(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ size_t srcSize,
+ void* entropyWorkspace, size_t entropyWkspSize,
+ int bmi2)
+{
+ size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
+ seqStorePtr, prevEntropy, nextEntropy, cctxParams,
+ dst, dstCapacity,
+ entropyWorkspace, entropyWkspSize, bmi2);
+ if (cSize == 0) return 0;
+ /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
+ * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
+ */
+ if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) {
+ DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity);
+ return 0; /* block not compressed */
+ }
+ FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
+
+ /* Check compressibility */
+ { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
+ if (cSize >= maxCSize) return 0; /* block not compressed */
+ }
+ DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
+ /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
+ * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
+ */
+ assert(cSize < ZSTD_BLOCKSIZE_MAX);
+ return cSize;
+}
+
+/* ZSTD_selectBlockCompressor() :
+ * Not static, but internal use only (used by long distance matcher)
+ * assumption : strat is a valid strategy */
+ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
+{
+ static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
+ { ZSTD_compressBlock_fast /* default for 0 */,
+ ZSTD_compressBlock_fast,
+ ZSTD_compressBlock_doubleFast,
+ ZSTD_compressBlock_greedy,
+ ZSTD_compressBlock_lazy,
+ ZSTD_compressBlock_lazy2,
+ ZSTD_compressBlock_btlazy2,
+ ZSTD_compressBlock_btopt,
+ ZSTD_compressBlock_btultra,
+ ZSTD_compressBlock_btultra2 },
+ { ZSTD_compressBlock_fast_extDict /* default for 0 */,
+ ZSTD_compressBlock_fast_extDict,
+ ZSTD_compressBlock_doubleFast_extDict,
+ ZSTD_compressBlock_greedy_extDict,
+ ZSTD_compressBlock_lazy_extDict,
+ ZSTD_compressBlock_lazy2_extDict,
+ ZSTD_compressBlock_btlazy2_extDict,
+ ZSTD_compressBlock_btopt_extDict,
+ ZSTD_compressBlock_btultra_extDict,
+ ZSTD_compressBlock_btultra_extDict },
+ { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */,
+ ZSTD_compressBlock_fast_dictMatchState,
+ ZSTD_compressBlock_doubleFast_dictMatchState,
+ ZSTD_compressBlock_greedy_dictMatchState,
+ ZSTD_compressBlock_lazy_dictMatchState,
+ ZSTD_compressBlock_lazy2_dictMatchState,
+ ZSTD_compressBlock_btlazy2_dictMatchState,
+ ZSTD_compressBlock_btopt_dictMatchState,
+ ZSTD_compressBlock_btultra_dictMatchState,
+ ZSTD_compressBlock_btultra_dictMatchState },
+ { NULL /* default for 0 */,
+ NULL,
+ NULL,
+ ZSTD_compressBlock_greedy_dedicatedDictSearch,
+ ZSTD_compressBlock_lazy_dedicatedDictSearch,
+ ZSTD_compressBlock_lazy2_dedicatedDictSearch,
+ NULL,
+ NULL,
+ NULL,
+ NULL }
+ };
+ ZSTD_blockCompressor selectedCompressor;
+ ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
+
+ assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
+ DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder);
+ if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) {
+ static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = {
+ { ZSTD_compressBlock_greedy_row,
+ ZSTD_compressBlock_lazy_row,
+ ZSTD_compressBlock_lazy2_row },
+ { ZSTD_compressBlock_greedy_extDict_row,
+ ZSTD_compressBlock_lazy_extDict_row,
+ ZSTD_compressBlock_lazy2_extDict_row },
+ { ZSTD_compressBlock_greedy_dictMatchState_row,
+ ZSTD_compressBlock_lazy_dictMatchState_row,
+ ZSTD_compressBlock_lazy2_dictMatchState_row },
+ { ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
+ ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
+ ZSTD_compressBlock_lazy2_dedicatedDictSearch_row }
+ };
+ DEBUGLOG(4, "Selecting a row-based matchfinder");
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+ selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy];
+ } else {
+ selectedCompressor = blockCompressor[(int)dictMode][(int)strat];
+ }
+ assert(selectedCompressor != NULL);
+ return selectedCompressor;
+}
+
+static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr,
+ const BYTE* anchor, size_t lastLLSize)
+{
+ ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+}
+
+void ZSTD_resetSeqStore(seqStore_t* ssPtr)
+{
+ ssPtr->lit = ssPtr->litStart;
+ ssPtr->sequences = ssPtr->sequencesStart;
+ ssPtr->longLengthType = ZSTD_llt_none;
+}
+
+/* ZSTD_postProcessSequenceProducerResult() :
+ * Validates and post-processes sequences obtained through the external matchfinder API:
+ * - Checks whether nbExternalSeqs represents an error condition.
+ * - Appends a block delimiter to outSeqs if one is not already present.
+ * See zstd.h for context regarding block delimiters.
+ * Returns the number of sequences after post-processing, or an error code. */
+static size_t ZSTD_postProcessSequenceProducerResult(
+ ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize
+) {
+ RETURN_ERROR_IF(
+ nbExternalSeqs > outSeqsCapacity,
+ sequenceProducer_failed,
+ "External sequence producer returned error code %lu",
+ (unsigned long)nbExternalSeqs
+ );
+
+ RETURN_ERROR_IF(
+ nbExternalSeqs == 0 && srcSize > 0,
+ sequenceProducer_failed,
+ "Got zero sequences from external sequence producer for a non-empty src buffer!"
+ );
+
+ if (srcSize == 0) {
+ ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence));
+ return 1;
+ }
+
+ {
+ ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1];
+
+ /* We can return early if lastSeq is already a block delimiter. */
+ if (lastSeq.offset == 0 && lastSeq.matchLength == 0) {
+ return nbExternalSeqs;
+ }
+
+ /* This error condition is only possible if the external matchfinder
+ * produced an invalid parse, by definition of ZSTD_sequenceBound(). */
+ RETURN_ERROR_IF(
+ nbExternalSeqs == outSeqsCapacity,
+ sequenceProducer_failed,
+ "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!"
+ );
+
+ /* lastSeq is not a block delimiter, so we need to append one. */
+ ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence));
+ return nbExternalSeqs + 1;
+ }
+}
+
+/* ZSTD_fastSequenceLengthSum() :
+ * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*.
+ * Similar to another function in zstd_compress.c (determine_blockSize),
+ * except it doesn't check for a block delimiter to end summation.
+ * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
+ * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */
+static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) {
+ size_t matchLenSum, litLenSum, i;
+ matchLenSum = 0;
+ litLenSum = 0;
+ for (i = 0; i < seqBufSize; i++) {
+ litLenSum += seqBuf[i].litLength;
+ matchLenSum += seqBuf[i].matchLength;
+ }
+ return litLenSum + matchLenSum;
+}
+
+typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
+
+static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
+{
+ ZSTD_matchState_t* const ms = &zc->blockState.matchState;
+ DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize);
+ assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
+ /* Assert that we have correctly flushed the ctx params into the ms's copy */
+ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams);
+ /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+ * additional 1. We need to revisit and change this logic to be more consistent */
+ if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
+ if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) {
+ ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize);
+ } else {
+ ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch);
+ }
+ return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */
+ }
+ ZSTD_resetSeqStore(&(zc->seqStore));
+ /* required for optimal parser to read stats from dictionary */
+ ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;
+ /* tell the optimal parser how we expect to compress literals */
+ ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode;
+ /* a gap between an attached dict and the current window is not safe,
+ * they must remain adjacent,
+ * and when that stops being the case, the dict must be unset */
+ assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit);
+
+ /* limited update after a very long match */
+ { const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const U32 curr = (U32)(istart-base);
+ if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */
+ if (curr > ms->nextToUpdate + 384)
+ ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384));
+ }
+
+ /* select and store sequences */
+ { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
+ size_t lastLLSize;
+ { int i;
+ for (i = 0; i < ZSTD_REP_NUM; ++i)
+ zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i];
+ }
+ if (zc->externSeqStore.pos < zc->externSeqStore.size) {
+ assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
+
+ /* External matchfinder + LDM is technically possible, just not implemented yet.
+ * We need to revisit soon and implement it. */
+ RETURN_ERROR_IF(
+ zc->appliedParams.useSequenceProducer,
+ parameter_combination_unsupported,
+ "Long-distance matching with external sequence producer enabled is not currently supported."
+ );
+
+ /* Updates ldmSeqStore.pos */
+ lastLLSize =
+ ZSTD_ldm_blockCompress(&zc->externSeqStore,
+ ms, &zc->seqStore,
+ zc->blockState.nextCBlock->rep,
+ zc->appliedParams.useRowMatchFinder,
+ src, srcSize);
+ assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
+ } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
+ rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
+
+ /* External matchfinder + LDM is technically possible, just not implemented yet.
+ * We need to revisit soon and implement it. */
+ RETURN_ERROR_IF(
+ zc->appliedParams.useSequenceProducer,
+ parameter_combination_unsupported,
+ "Long-distance matching with external sequence producer enabled is not currently supported."
+ );
+
+ ldmSeqStore.seq = zc->ldmSequences;
+ ldmSeqStore.capacity = zc->maxNbLdmSequences;
+ /* Updates ldmSeqStore.size */
+ FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore,
+ &zc->appliedParams.ldmParams,
+ src, srcSize), "");
+ /* Updates ldmSeqStore.pos */
+ lastLLSize =
+ ZSTD_ldm_blockCompress(&ldmSeqStore,
+ ms, &zc->seqStore,
+ zc->blockState.nextCBlock->rep,
+ zc->appliedParams.useRowMatchFinder,
+ src, srcSize);
+ assert(ldmSeqStore.pos == ldmSeqStore.size);
+ } else if (zc->appliedParams.useSequenceProducer) {
+ assert(
+ zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize)
+ );
+ assert(zc->externalMatchCtx.mFinder != NULL);
+
+ { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog;
+
+ size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)(
+ zc->externalMatchCtx.mState,
+ zc->externalMatchCtx.seqBuffer,
+ zc->externalMatchCtx.seqBufferCapacity,
+ src, srcSize,
+ NULL, 0, /* dict and dictSize, currently not supported */
+ zc->appliedParams.compressionLevel,
+ windowSize
+ );
+
+ size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult(
+ zc->externalMatchCtx.seqBuffer,
+ nbExternalSeqs,
+ zc->externalMatchCtx.seqBufferCapacity,
+ srcSize
+ );
+
+ /* Return early if there is no error, since we don't need to worry about last literals */
+ if (!ZSTD_isError(nbPostProcessedSeqs)) {
+ ZSTD_sequencePosition seqPos = {0,0,0};
+ size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs);
+ RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!");
+ FORWARD_IF_ERROR(
+ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
+ zc, &seqPos,
+ zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs,
+ src, srcSize,
+ zc->appliedParams.searchForExternalRepcodes
+ ),
+ "Failed to copy external sequences to seqStore!"
+ );
+ ms->ldmSeqStore = NULL;
+ DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs);
+ return ZSTDbss_compress;
+ }
+
+ /* Propagate the error if fallback is disabled */
+ if (!zc->appliedParams.enableMatchFinderFallback) {
+ return nbPostProcessedSeqs;
+ }
+
+ /* Fallback to software matchfinder */
+ { ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+ zc->appliedParams.useRowMatchFinder,
+ dictMode);
+ ms->ldmSeqStore = NULL;
+ DEBUGLOG(
+ 5,
+ "External sequence producer returned error code %lu. Falling back to internal parser.",
+ (unsigned long)nbExternalSeqs
+ );
+ lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+ } }
+ } else { /* not long range mode and no external matchfinder */
+ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+ zc->appliedParams.useRowMatchFinder,
+ dictMode);
+ ms->ldmSeqStore = NULL;
+ lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+ }
+ { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
+ ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
+ } }
+ return ZSTDbss_compress;
+}
+
+static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
+{
+ const seqStore_t* seqStore = ZSTD_getSeqStore(zc);
+ const seqDef* seqStoreSeqs = seqStore->sequencesStart;
+ size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs;
+ size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart);
+ size_t literalsRead = 0;
+ size_t lastLLSize;
+
+ ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex];
+ size_t i;
+ repcodes_t updatedRepcodes;
+
+ assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences);
+ /* Ensure we have enough space for last literals "sequence" */
+ assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1);
+ ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ for (i = 0; i < seqStoreSeqSize; ++i) {
+ U32 rawOffset = seqStoreSeqs[i].offBase - ZSTD_REP_NUM;
+ outSeqs[i].litLength = seqStoreSeqs[i].litLength;
+ outSeqs[i].matchLength = seqStoreSeqs[i].mlBase + MINMATCH;
+ outSeqs[i].rep = 0;
+
+ if (i == seqStore->longLengthPos) {
+ if (seqStore->longLengthType == ZSTD_llt_literalLength) {
+ outSeqs[i].litLength += 0x10000;
+ } else if (seqStore->longLengthType == ZSTD_llt_matchLength) {
+ outSeqs[i].matchLength += 0x10000;
+ }
+ }
+
+ if (seqStoreSeqs[i].offBase <= ZSTD_REP_NUM) {
+ /* Derive the correct offset corresponding to a repcode */
+ outSeqs[i].rep = seqStoreSeqs[i].offBase;
+ if (outSeqs[i].litLength != 0) {
+ rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1];
+ } else {
+ if (outSeqs[i].rep == 3) {
+ rawOffset = updatedRepcodes.rep[0] - 1;
+ } else {
+ rawOffset = updatedRepcodes.rep[outSeqs[i].rep];
+ }
+ }
+ }
+ outSeqs[i].offset = rawOffset;
+ /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode
+ so we provide seqStoreSeqs[i].offset - 1 */
+ ZSTD_updateRep(updatedRepcodes.rep,
+ seqStoreSeqs[i].offBase,
+ seqStoreSeqs[i].litLength == 0);
+ literalsRead += outSeqs[i].litLength;
+ }
+ /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
+ * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
+ * for the block boundary, according to the API.
+ */
+ assert(seqStoreLiteralsSize >= literalsRead);
+ lastLLSize = seqStoreLiteralsSize - literalsRead;
+ outSeqs[i].litLength = (U32)lastLLSize;
+ outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0;
+ seqStoreSeqSize++;
+ zc->seqCollector.seqIndex += seqStoreSeqSize;
+}
+
+size_t ZSTD_sequenceBound(size_t srcSize) {
+ return (srcSize / ZSTD_MINMATCH_MIN) + 1;
+}
+
+size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+ size_t outSeqsSize, const void* src, size_t srcSize)
+{
+ const size_t dstCapacity = ZSTD_compressBound(srcSize);
+ void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem);
+ SeqCollector seqCollector;
+
+ RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!");
+
+ seqCollector.collectSequences = 1;
+ seqCollector.seqStart = outSeqs;
+ seqCollector.seqIndex = 0;
+ seqCollector.maxSequences = outSeqsSize;
+ zc->seqCollector = seqCollector;
+
+ ZSTD_compress2(zc, dst, dstCapacity, src, srcSize);
+ ZSTD_customFree(dst, ZSTD_defaultCMem);
+ return zc->seqCollector.seqIndex;
+}
+
+size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) {
+ size_t in = 0;
+ size_t out = 0;
+ for (; in < seqsSize; ++in) {
+ if (sequences[in].offset == 0 && sequences[in].matchLength == 0) {
+ if (in != seqsSize - 1) {
+ sequences[in+1].litLength += sequences[in].litLength;
+ }
+ } else {
+ sequences[out] = sequences[in];
+ ++out;
+ }
+ }
+ return out;
+}
+
+/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */
+static int ZSTD_isRLE(const BYTE* src, size_t length) {
+ const BYTE* ip = src;
+ const BYTE value = ip[0];
+ const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL);
+ const size_t unrollSize = sizeof(size_t) * 4;
+ const size_t unrollMask = unrollSize - 1;
+ const size_t prefixLength = length & unrollMask;
+ size_t i;
+ if (length == 1) return 1;
+ /* Check if prefix is RLE first before using unrolled loop */
+ if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
+ return 0;
+ }
+ for (i = prefixLength; i != length; i += unrollSize) {
+ size_t u;
+ for (u = 0; u < unrollSize; u += sizeof(size_t)) {
+ if (MEM_readST(ip + i + u) != valueST) {
+ return 0;
+ } } }
+ return 1;
+}
+
+/* Returns true if the given block may be RLE.
+ * This is just a heuristic based on the compressibility.
+ * It may return both false positives and false negatives.
+ */
+static int ZSTD_maybeRLE(seqStore_t const* seqStore)
+{
+ size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
+ size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart);
+
+ return nbSeqs < 4 && nbLits < 10;
+}
+
+static void
+ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
+{
+ ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock;
+ bs->prevCBlock = bs->nextCBlock;
+ bs->nextCBlock = tmp;
+}
+
+/* Writes the block header */
+static void
+writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
+{
+ U32 const cBlockHeader = cSize == 1 ?
+ lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
+ lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
+ MEM_writeLE24(op, cBlockHeader);
+ DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
+}
+
+/** ZSTD_buildBlockEntropyStats_literals() :
+ * Builds entropy for the literals.
+ * Stores literals block type (raw, rle, compressed, repeat) and
+ * huffman description table to hufMetadata.
+ * Requires ENTROPY_WORKSPACE_SIZE workspace
+ * @return : size of huffman description table, or an error code
+ */
+static size_t
+ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
+ const ZSTD_hufCTables_t* prevHuf,
+ ZSTD_hufCTables_t* nextHuf,
+ ZSTD_hufCTablesMetadata_t* hufMetadata,
+ const int literalsCompressionIsDisabled,
+ void* workspace, size_t wkspSize,
+ int hufFlags)
+{
+ BYTE* const wkspStart = (BYTE*)workspace;
+ BYTE* const wkspEnd = wkspStart + wkspSize;
+ BYTE* const countWkspStart = wkspStart;
+ unsigned* const countWksp = (unsigned*)workspace;
+ const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
+ BYTE* const nodeWksp = countWkspStart + countWkspSize;
+ const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp);
+ unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
+ unsigned huffLog = LitHufLog;
+ HUF_repeat repeat = prevHuf->repeatMode;
+ DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize);
+
+ /* Prepare nextEntropy assuming reusing the existing table */
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+
+ if (literalsCompressionIsDisabled) {
+ DEBUGLOG(5, "set_basic - disabled");
+ hufMetadata->hType = set_basic;
+ return 0;
+ }
+
+ /* small ? don't even attempt compression (speed opt) */
+#ifndef COMPRESS_LITERALS_SIZE_MIN
+# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */
+#endif
+ { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
+ if (srcSize <= minLitSize) {
+ DEBUGLOG(5, "set_basic - too small");
+ hufMetadata->hType = set_basic;
+ return 0;
+ } }
+
+ /* Scan input and build symbol stats */
+ { size_t const largest =
+ HIST_count_wksp (countWksp, &maxSymbolValue,
+ (const BYTE*)src, srcSize,
+ workspace, wkspSize);
+ FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
+ if (largest == srcSize) {
+ /* only one literal symbol */
+ DEBUGLOG(5, "set_rle");
+ hufMetadata->hType = set_rle;
+ return 0;
+ }
+ if (largest <= (srcSize >> 7)+4) {
+ /* heuristic: likely not compressible */
+ DEBUGLOG(5, "set_basic - no gain");
+ hufMetadata->hType = set_basic;
+ return 0;
+ } }
+
+ /* Validate the previous Huffman table */
+ if (repeat == HUF_repeat_check
+ && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
+ repeat = HUF_repeat_none;
+ }
+
+ /* Build Huffman Tree */
+ ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags);
+ assert(huffLog <= LitHufLog);
+ { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
+ maxSymbolValue, huffLog,
+ nodeWksp, nodeWkspSize);
+ FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
+ huffLog = (U32)maxBits;
+ }
+ { /* Build and write the CTable */
+ size_t const newCSize = HUF_estimateCompressedSize(
+ (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
+ size_t const hSize = HUF_writeCTable_wksp(
+ hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
+ (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
+ nodeWksp, nodeWkspSize);
+ /* Check against repeating the previous CTable */
+ if (repeat != HUF_repeat_none) {
+ size_t const oldCSize = HUF_estimateCompressedSize(
+ (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
+ if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
+ DEBUGLOG(5, "set_repeat - smaller");
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ hufMetadata->hType = set_repeat;
+ return 0;
+ } }
+ if (newCSize + hSize >= srcSize) {
+ DEBUGLOG(5, "set_basic - no gains");
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ hufMetadata->hType = set_basic;
+ return 0;
+ }
+ DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
+ hufMetadata->hType = set_compressed;
+ nextHuf->repeatMode = HUF_repeat_check;
+ return hSize;
+ }
+}
+
+
+/* ZSTD_buildDummySequencesStatistics():
+ * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic,
+ * and updates nextEntropy to the appropriate repeatMode.
+ */
+static ZSTD_symbolEncodingTypeStats_t
+ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
+{
+ ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
+ nextEntropy->litlength_repeatMode = FSE_repeat_none;
+ nextEntropy->offcode_repeatMode = FSE_repeat_none;
+ nextEntropy->matchlength_repeatMode = FSE_repeat_none;
+ return stats;
+}
+
+/** ZSTD_buildBlockEntropyStats_sequences() :
+ * Builds entropy for the sequences.
+ * Stores symbol compression modes and fse table to fseMetadata.
+ * Requires ENTROPY_WORKSPACE_SIZE wksp.
+ * @return : size of fse tables or error code */
+static size_t
+ZSTD_buildBlockEntropyStats_sequences(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_fseCTables_t* prevEntropy,
+ ZSTD_fseCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_fseCTablesMetadata_t* fseMetadata,
+ void* workspace, size_t wkspSize)
+{
+ ZSTD_strategy const strategy = cctxParams->cParams.strategy;
+ size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ BYTE* const ostart = fseMetadata->fseTablesBuffer;
+ BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
+ BYTE* op = ostart;
+ unsigned* countWorkspace = (unsigned*)workspace;
+ unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1);
+ size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace);
+ ZSTD_symbolEncodingTypeStats_t stats;
+
+ DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq);
+ stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
+ prevEntropy, nextEntropy, op, oend,
+ strategy, countWorkspace,
+ entropyWorkspace, entropyWorkspaceSize)
+ : ZSTD_buildDummySequencesStatistics(nextEntropy);
+ FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
+ fseMetadata->llType = (symbolEncodingType_e) stats.LLtype;
+ fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype;
+ fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype;
+ fseMetadata->lastCountSize = stats.lastCountSize;
+ return stats.size;
+}
+
+
+/** ZSTD_buildBlockEntropyStats() :
+ * Builds entropy for the block.
+ * Requires workspace size ENTROPY_WORKSPACE_SIZE
+ * @return : 0 on success, or an error code
+ * Note : also employed in superblock
+ */
+size_t ZSTD_buildBlockEntropyStats(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize)
+{
+ size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+ int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
+ int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0;
+
+ entropyMetadata->hufMetadata.hufDesSize =
+ ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
+ &prevEntropy->huf, &nextEntropy->huf,
+ &entropyMetadata->hufMetadata,
+ ZSTD_literalsCompressionIsDisabled(cctxParams),
+ workspace, wkspSize, hufFlags);
+
+ FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
+ entropyMetadata->fseMetadata.fseTablesSize =
+ ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
+ &prevEntropy->fse, &nextEntropy->fse,
+ cctxParams,
+ &entropyMetadata->fseMetadata,
+ workspace, wkspSize);
+ FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed");
+ return 0;
+}
+
+/* Returns the size estimate for the literals section (header + content) of a block */
+static size_t
+ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
+ const ZSTD_hufCTables_t* huf,
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
+ size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB);
+ U32 singleStream = litSize < 256;
+
+ if (hufMetadata->hType == set_basic) return litSize;
+ else if (hufMetadata->hType == set_rle) return 1;
+ else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) {
+ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize);
+ if (ZSTD_isError(largest)) return litSize;
+ { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue);
+ if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize;
+ if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */
+ return cLitSizeEstimate + literalSectionHeaderSize;
+ } }
+ assert(0); /* impossible */
+ return 0;
+}
+
+/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
+static size_t
+ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
+ const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
+ const FSE_CTable* fseCTable,
+ const U8* additionalBits,
+ short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ void* workspace, size_t wkspSize)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ const BYTE* ctp = codeTable;
+ const BYTE* const ctStart = ctp;
+ const BYTE* const ctEnd = ctStart + nbSeq;
+ size_t cSymbolTypeSizeEstimateInBits = 0;
+ unsigned max = maxCode;
+
+ HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
+ if (type == set_basic) {
+ /* We selected this encoding type, so it must be valid. */
+ assert(max <= defaultMax);
+ (void)defaultMax;
+ cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max);
+ } else if (type == set_rle) {
+ cSymbolTypeSizeEstimateInBits = 0;
+ } else if (type == set_compressed || type == set_repeat) {
+ cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max);
+ }
+ if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) {
+ return nbSeq * 10;
+ }
+ while (ctp < ctEnd) {
+ if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp];
+ else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */
+ ctp++;
+ }
+ return cSymbolTypeSizeEstimateInBits >> 3;
+}
+
+/* Returns the size estimate for the sequences section (header + content) of a block */
+static size_t
+ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_fseCTables_t* fseTables,
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ);
+ size_t cSeqSizeEstimate = 0;
+ cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff,
+ fseTables->offcodeCTable, NULL,
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL,
+ fseTables->litlengthCTable, LL_bits,
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML,
+ fseTables->matchlengthCTable, ML_bits,
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
+ workspace, wkspSize);
+ if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
+ return cSeqSizeEstimate + sequencesSectionHeaderSize;
+}
+
+/* Returns the size estimate for a given stream of literals, of, ll, ml */
+static size_t
+ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
+ const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_entropyCTables_t* entropy,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize,
+ int writeLitEntropy, int writeSeqEntropy)
+{
+ size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize,
+ &entropy->huf, &entropyMetadata->hufMetadata,
+ workspace, wkspSize, writeLitEntropy);
+ size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
+ nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+ workspace, wkspSize, writeSeqEntropy);
+ return seqSize + literalsSize + ZSTD_blockHeaderSize;
+}
+
+/* Builds entropy statistics and uses them for blocksize estimation.
+ *
+ * @return: estimated compressed size of the seqStore, or a zstd error.
+ */
+static size_t
+ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc)
+{
+ ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
+ DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
+ FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
+ &zc->blockState.prevCBlock->entropy,
+ &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ entropyMetadata,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), "");
+ return ZSTD_estimateBlockSize(
+ seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
+ seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
+ (size_t)(seqStore->sequences - seqStore->sequencesStart),
+ &zc->blockState.nextCBlock->entropy,
+ entropyMetadata,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
+ (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
+}
+
+/* Returns literals bytes represented in a seqStore */
+static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore)
+{
+ size_t literalsBytes = 0;
+ size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
+ size_t i;
+ for (i = 0; i < nbSeqs; ++i) {
+ seqDef const seq = seqStore->sequencesStart[i];
+ literalsBytes += seq.litLength;
+ if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
+ literalsBytes += 0x10000;
+ } }
+ return literalsBytes;
+}
+
+/* Returns match bytes represented in a seqStore */
+static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore)
+{
+ size_t matchBytes = 0;
+ size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
+ size_t i;
+ for (i = 0; i < nbSeqs; ++i) {
+ seqDef seq = seqStore->sequencesStart[i];
+ matchBytes += seq.mlBase + MINMATCH;
+ if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
+ matchBytes += 0x10000;
+ } }
+ return matchBytes;
+}
+
+/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx).
+ * Stores the result in resultSeqStore.
+ */
+static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore,
+ const seqStore_t* originalSeqStore,
+ size_t startIdx, size_t endIdx)
+{
+ *resultSeqStore = *originalSeqStore;
+ if (startIdx > 0) {
+ resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx;
+ resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
+ }
+
+ /* Move longLengthPos into the correct position if necessary */
+ if (originalSeqStore->longLengthType != ZSTD_llt_none) {
+ if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) {
+ resultSeqStore->longLengthType = ZSTD_llt_none;
+ } else {
+ resultSeqStore->longLengthPos -= (U32)startIdx;
+ }
+ }
+ resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx;
+ resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx;
+ if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) {
+ /* This accounts for possible last literals if the derived chunk reaches the end of the block */
+ assert(resultSeqStore->lit == originalSeqStore->lit);
+ } else {
+ size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
+ resultSeqStore->lit = resultSeqStore->litStart + literalsBytes;
+ }
+ resultSeqStore->llCode += startIdx;
+ resultSeqStore->mlCode += startIdx;
+ resultSeqStore->ofCode += startIdx;
+}
+
+/**
+ * Returns the raw offset represented by the combination of offBase, ll0, and repcode history.
+ * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq().
+ */
+static U32
+ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0)
+{
+ U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */
+ assert(OFFBASE_IS_REPCODE(offBase));
+ if (adjustedRepCode == ZSTD_REP_NUM) {
+ assert(ll0);
+ /* litlength == 0 and offCode == 2 implies selection of first repcode - 1
+ * This is only valid if it results in a valid offset value, aka > 0.
+ * Note : it may happen that `rep[0]==1` in exceptional circumstances.
+ * In which case this function will return 0, which is an invalid offset.
+ * It's not an issue though, since this value will be
+ * compared and discarded within ZSTD_seqStore_resolveOffCodes().
+ */
+ return rep[0] - 1;
+ }
+ return rep[adjustedRepCode];
+}
+
+/**
+ * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise
+ * due to emission of RLE/raw blocks that disturb the offset history,
+ * and replaces any repcodes within the seqStore that may be invalid.
+ *
+ * dRepcodes are updated as would be on the decompression side.
+ * cRepcodes are updated exactly in accordance with the seqStore.
+ *
+ * Note : this function assumes seq->offBase respects the following numbering scheme :
+ * 0 : invalid
+ * 1-3 : repcode 1-3
+ * 4+ : real_offset+3
+ */
+static void
+ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
+ const seqStore_t* const seqStore, U32 const nbSeq)
+{
+ U32 idx = 0;
+ for (; idx < nbSeq; ++idx) {
+ seqDef* const seq = seqStore->sequencesStart + idx;
+ U32 const ll0 = (seq->litLength == 0);
+ U32 const offBase = seq->offBase;
+ assert(offBase > 0);
+ if (OFFBASE_IS_REPCODE(offBase)) {
+ U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0);
+ U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0);
+ /* Adjust simulated decompression repcode history if we come across a mismatch. Replace
+ * the repcode with the offset it actually references, determined by the compression
+ * repcode history.
+ */
+ if (dRawOffset != cRawOffset) {
+ seq->offBase = OFFSET_TO_OFFBASE(cRawOffset);
+ }
+ }
+ /* Compression repcode history is always updated with values directly from the unmodified seqStore.
+ * Decompression repcode history may use modified seq->offset value taken from compression repcode history.
+ */
+ ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0);
+ ZSTD_updateRep(cRepcodes->rep, offBase, ll0);
+ }
+}
+
+/* ZSTD_compressSeqStore_singleBlock():
+ * Compresses a seqStore into a block with a block header, into the buffer dst.
+ *
+ * Returns the total size of that block (including header) or a ZSTD error code.
+ */
+static size_t
+ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
+ const seqStore_t* const seqStore,
+ repcodes_t* const dRep, repcodes_t* const cRep,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 lastBlock, U32 isPartition)
+{
+ const U32 rleMaxLength = 25;
+ BYTE* op = (BYTE*)dst;
+ const BYTE* ip = (const BYTE*)src;
+ size_t cSize;
+ size_t cSeqsSize;
+
+ /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */
+ repcodes_t const dRepOriginal = *dRep;
+ DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock");
+ if (isPartition)
+ ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart));
+
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit");
+ cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore,
+ &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize,
+ srcSize,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
+ zc->bmi2);
+ FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!");
+
+ if (!zc->isFirstBlock &&
+ cSeqsSize < rleMaxLength &&
+ ZSTD_isRLE((BYTE const*)src, srcSize)) {
+ /* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ cSeqsSize = 1;
+ }
+
+ if (zc->seqCollector.collectSequences) {
+ ZSTD_copyBlockSequences(zc);
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ return 0;
+ }
+
+ if (cSeqsSize == 0) {
+ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "Nocompress block failed");
+ DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize);
+ *dRep = dRepOriginal; /* reset simulated decompression repcode history */
+ } else if (cSeqsSize == 1) {
+ cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "RLE compress block failed");
+ DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize);
+ *dRep = dRepOriginal; /* reset simulated decompression repcode history */
+ } else {
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ writeBlockHeader(op, cSeqsSize, srcSize, lastBlock);
+ cSize = ZSTD_blockHeaderSize + cSeqsSize;
+ DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize);
+ }
+
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ return cSize;
+}
+
+/* Struct to keep track of where we are in our recursive calls. */
+typedef struct {
+ U32* splitLocations; /* Array of split indices */
+ size_t idx; /* The current index within splitLocations being worked on */
+} seqStoreSplits;
+
+#define MIN_SEQUENCES_BLOCK_SPLITTING 300
+
+/* Helper function to perform the recursive search for block splits.
+ * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half.
+ * If advantageous to split, then we recurse down the two sub-blocks.
+ * If not, or if an error occurred in estimation, then we do not recurse.
+ *
+ * Note: The recursion depth is capped by a heuristic minimum number of sequences,
+ * defined by MIN_SEQUENCES_BLOCK_SPLITTING.
+ * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING).
+ * In practice, recursion depth usually doesn't go beyond 4.
+ *
+ * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS.
+ * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize
+ * maximum of 128 KB, this value is actually impossible to reach.
+ */
+static void
+ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
+ ZSTD_CCtx* zc, const seqStore_t* origSeqStore)
+{
+ seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
+ seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
+ seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
+ size_t estimatedOriginalSize;
+ size_t estimatedFirstHalfSize;
+ size_t estimatedSecondHalfSize;
+ size_t midIdx = (startIdx + endIdx)/2;
+
+ DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
+ assert(endIdx >= startIdx);
+ if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) {
+ DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx);
+ return;
+ }
+ ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
+ ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx);
+ ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx);
+ estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc);
+ estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc);
+ estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc);
+ DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu",
+ estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize);
+ if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) {
+ return;
+ }
+ if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) {
+ DEBUGLOG(5, "split decided at seqNb:%zu", midIdx);
+ ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore);
+ splits->splitLocations[splits->idx] = (U32)midIdx;
+ splits->idx++;
+ ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore);
+ }
+}
+
+/* Base recursive function.
+ * Populates a table with intra-block partition indices that can improve compression ratio.
+ *
+ * @return: number of splits made (which equals the size of the partition table - 1).
+ */
+static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
+{
+ seqStoreSplits splits;
+ splits.splitLocations = partitions;
+ splits.idx = 0;
+ if (nbSeq <= 4) {
+ DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq);
+ /* Refuse to try and split anything with less than 4 sequences */
+ return 0;
+ }
+ ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore);
+ splits.splitLocations[splits.idx] = nbSeq;
+ DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1);
+ return splits.idx;
+}
+
+/* ZSTD_compressBlock_splitBlock():
+ * Attempts to split a given block into multiple blocks to improve compression ratio.
+ *
+ * Returns combined size of all blocks (which includes headers), or a ZSTD error code.
+ */
+static size_t
+ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t blockSize,
+ U32 lastBlock, U32 nbSeq)
+{
+ size_t cSize = 0;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ size_t i = 0;
+ size_t srcBytesTotal = 0;
+ U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
+ seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
+ seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
+ size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
+
+ /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
+ * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
+ * separate repcode histories that simulate repcode history on compression and decompression side,
+ * and use the histories to determine whether we must replace a particular repcode with its raw offset.
+ *
+ * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed
+ * or RLE. This allows us to retrieve the offset value that an invalid repcode references within
+ * a nocompress/RLE block.
+ * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use
+ * the replacement offset value rather than the original repcode to update the repcode history.
+ * dRep also will be the final repcode history sent to the next block.
+ *
+ * See ZSTD_seqStore_resolveOffCodes() for more details.
+ */
+ repcodes_t dRep;
+ repcodes_t cRep;
+ ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+ (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
+ (unsigned)zc->blockState.matchState.nextToUpdate);
+
+ if (numSplits == 0) {
+ size_t cSizeSingleBlock =
+ ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
+ &dRep, &cRep,
+ op, dstCapacity,
+ ip, blockSize,
+ lastBlock, 0 /* isPartition */);
+ FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
+ DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
+ assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX);
+ assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize);
+ return cSizeSingleBlock;
+ }
+
+ ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]);
+ for (i = 0; i <= numSplits; ++i) {
+ size_t cSizeChunk;
+ U32 const lastPartition = (i == numSplits);
+ U32 lastBlockEntireSrc = 0;
+
+ size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore);
+ srcBytesTotal += srcBytes;
+ if (lastPartition) {
+ /* This is the final partition, need to account for possible last literals */
+ srcBytes += blockSize - srcBytesTotal;
+ lastBlockEntireSrc = lastBlock;
+ } else {
+ ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]);
+ }
+
+ cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore,
+ &dRep, &cRep,
+ op, dstCapacity,
+ ip, srcBytes,
+ lastBlockEntireSrc, 1 /* isPartition */);
+ DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size",
+ ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
+ FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
+
+ ip += srcBytes;
+ op += cSizeChunk;
+ dstCapacity -= cSizeChunk;
+ cSize += cSizeChunk;
+ *currSeqStore = *nextSeqStore;
+ assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize);
+ }
+ /* cRep and dRep may have diverged during the compression.
+ * If so, we use the dRep repcodes for the next block.
+ */
+ ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
+ return cSize;
+}
+
+static size_t
+ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, U32 lastBlock)
+{
+ U32 nbSeq;
+ size_t cSize;
+ DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
+ assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable);
+
+ { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
+ FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
+ if (bss == ZSTDbss_noCompress) {
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+ cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
+ DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
+ return cSize;
+ }
+ nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart);
+ }
+
+ cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq);
+ FORWARD_IF_ERROR(cSize, "Splitting blocks failed!");
+ return cSize;
+}
+
+static size_t
+ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, U32 frame)
+{
+ /* This is an estimated upper bound for the length of an rle block.
+ * This isn't the actual upper bound.
+ * Finding the real threshold needs further investigation.
+ */
+ const U32 rleMaxLength = 25;
+ size_t cSize;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+ (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
+ (unsigned)zc->blockState.matchState.nextToUpdate);
+
+ { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
+ FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
+ if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; }
+ }
+
+ if (zc->seqCollector.collectSequences) {
+ ZSTD_copyBlockSequences(zc);
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ return 0;
+ }
+
+ /* encode sequences and literals */
+ cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore,
+ &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ dst, dstCapacity,
+ srcSize,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
+ zc->bmi2);
+
+ if (frame &&
+ /* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ !zc->isFirstBlock &&
+ cSize < rleMaxLength &&
+ ZSTD_isRLE(ip, srcSize))
+ {
+ cSize = 1;
+ op[0] = ip[0];
+ }
+
+out:
+ if (!ZSTD_isError(cSize) && cSize > 1) {
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ }
+ /* We check that dictionaries have offset codes available for the first
+ * block. After the first block, the offcode table might not have large
+ * enough codes to represent the offsets in the data.
+ */
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ return cSize;
+}
+
+static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const size_t bss, U32 lastBlock)
+{
+ DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()");
+ if (bss == ZSTDbss_compress) {
+ if (/* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ !zc->isFirstBlock &&
+ ZSTD_maybeRLE(&zc->seqStore) &&
+ ZSTD_isRLE((BYTE const*)src, srcSize))
+ {
+ return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock);
+ }
+ /* Attempt superblock compression.
+ *
+ * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the
+ * standard ZSTD_compressBound(). This is a problem, because even if we have
+ * space now, taking an extra byte now could cause us to run out of space later
+ * and violate ZSTD_compressBound().
+ *
+ * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize.
+ *
+ * In order to respect ZSTD_compressBound() we must attempt to emit a raw
+ * uncompressed block in these cases:
+ * * cSize == 0: Return code for an uncompressed block.
+ * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize).
+ * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of
+ * output space.
+ * * cSize >= blockBound(srcSize): We have expanded the block too much so
+ * emit an uncompressed block.
+ */
+ { size_t const cSize =
+ ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
+ if (cSize != ERROR(dstSize_tooSmall)) {
+ size_t const maxCSize =
+ srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
+ if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ return cSize;
+ }
+ }
+ }
+ } /* if (bss == ZSTDbss_compress)*/
+
+ DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
+ /* Superblock compression failed, attempt to emit a single no compress block.
+ * The decoder will be able to stream this block since it is uncompressed.
+ */
+ return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
+}
+
+static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 lastBlock)
+{
+ size_t cSize = 0;
+ const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
+ DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)",
+ (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize);
+ FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
+
+ cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed");
+
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ return cSize;
+}
+
+static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
+ ZSTD_cwksp* ws,
+ ZSTD_CCtx_params const* params,
+ void const* ip,
+ void const* iend)
+{
+ U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy);
+ U32 const maxDist = (U32)1 << params->cParams.windowLog;
+ if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) {
+ U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip);
+ ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+ ZSTD_cwksp_mark_tables_dirty(ws);
+ ZSTD_reduceIndex(ms, params, correction);
+ ZSTD_cwksp_mark_tables_clean(ws);
+ if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
+ else ms->nextToUpdate -= correction;
+ /* invalidate dictionaries on overflow correction */
+ ms->loadedDictEnd = 0;
+ ms->dictMatchState = NULL;
+ }
+}
+
+/*! ZSTD_compress_frameChunk() :
+* Compress a chunk of data into one or multiple blocks.
+* All blocks will be terminated, all input will be consumed.
+* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
+* Frame is supposed already started (header already produced)
+* @return : compressed size, or an error code
+*/
+static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 lastFrameChunk)
+{
+ size_t blockSize = cctx->blockSize;
+ size_t remaining = srcSize;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* op = ostart;
+ U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
+
+ assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX);
+
+ DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize);
+ if (cctx->appliedParams.fParams.checksumFlag && srcSize)
+ XXH64_update(&cctx->xxhState, src, srcSize);
+
+ while (remaining) {
+ ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
+ U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
+
+ /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+ * additional 1. We need to revisit and change this logic to be more consistent */
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1,
+ dstSize_tooSmall,
+ "not enough space to store compressed block");
+ if (remaining < blockSize) blockSize = remaining;
+
+ ZSTD_overflowCorrectIfNeeded(
+ ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize);
+ ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+ ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+
+ /* Ensure hash/chain table insertion resumes no sooner than lowlimit */
+ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
+
+ { size_t cSize;
+ if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) {
+ cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed");
+ assert(cSize > 0);
+ assert(cSize <= blockSize + ZSTD_blockHeaderSize);
+ } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) {
+ cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed");
+ assert(cSize > 0 || cctx->seqCollector.collectSequences == 1);
+ } else {
+ cSize = ZSTD_compressBlock_internal(cctx,
+ op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
+ ip, blockSize, 1 /* frame */);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed");
+
+ if (cSize == 0) { /* block is not compressible */
+ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
+ } else {
+ U32 const cBlockHeader = cSize == 1 ?
+ lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
+ lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
+ MEM_writeLE24(op, cBlockHeader);
+ cSize += ZSTD_blockHeaderSize;
+ }
+ } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
+
+
+ ip += blockSize;
+ assert(remaining >= blockSize);
+ remaining -= blockSize;
+ op += cSize;
+ assert(dstCapacity >= cSize);
+ dstCapacity -= cSize;
+ cctx->isFirstBlock = 0;
+ DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u",
+ (unsigned)cSize);
+ } }
+
+ if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
+ return (size_t)(op-ostart);
+}
+
+
+static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
+ const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID)
+{ BYTE* const op = (BYTE*)dst;
+ U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
+ U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
+ U32 const checksumFlag = params->fParams.checksumFlag>0;
+ U32 const windowSize = (U32)1 << params->cParams.windowLog;
+ U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
+ BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
+ U32 const fcsCode = params->fParams.contentSizeFlag ?
+ (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */
+ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
+ size_t pos=0;
+
+ assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
+ RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall,
+ "dst buf is too small to fit worst-case frame header size.");
+ DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
+ !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode);
+ if (params->format == ZSTD_f_zstd1) {
+ MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
+ pos = 4;
+ }
+ op[pos++] = frameHeaderDescriptionByte;
+ if (!singleSegment) op[pos++] = windowLogByte;
+ switch(dictIDSizeCode)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : break;
+ case 1 : op[pos] = (BYTE)(dictID); pos++; break;
+ case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break;
+ case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
+ }
+ switch(fcsCode)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break;
+ case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
+ case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
+ case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
+ }
+ return pos;
+}
+
+/* ZSTD_writeSkippableFrame_advanced() :
+ * Writes out a skippable frame with the specified magic number variant (16 are supported),
+ * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data.
+ *
+ * Returns the total number of bytes written, or a ZSTD error code.
+ */
+size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, unsigned magicVariant) {
+ BYTE* op = (BYTE*)dst;
+ RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */,
+ dstSize_tooSmall, "Not enough room for skippable frame");
+ RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame");
+ RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported");
+
+ MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant));
+ MEM_writeLE32(op+4, (U32)srcSize);
+ ZSTD_memcpy(op+8, src, srcSize);
+ return srcSize + ZSTD_SKIPPABLEHEADERSIZE;
+}
+
+/* ZSTD_writeLastEmptyBlock() :
+ * output an empty Block with end-of-frame mark to complete a frame
+ * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
+ * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
+ */
+size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity)
+{
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall,
+ "dst buf is too small to write frame trailer empty block.");
+ { U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1); /* 0 size */
+ MEM_writeLE24(dst, cBlockHeader24);
+ return ZSTD_blockHeaderSize;
+ }
+}
+
+size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq)
+{
+ RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong,
+ "wrong cctx stage");
+ RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable,
+ parameter_unsupported,
+ "incompatible with ldm");
+ cctx->externSeqStore.seq = seq;
+ cctx->externSeqStore.size = nbSeq;
+ cctx->externSeqStore.capacity = nbSeq;
+ cctx->externSeqStore.pos = 0;
+ cctx->externSeqStore.posInSequence = 0;
+ return 0;
+}
+
+
+static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 frame, U32 lastFrameChunk)
+{
+ ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
+ size_t fhSize = 0;
+
+ DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u",
+ cctx->stage, (unsigned)srcSize);
+ RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong,
+ "missing init (ZSTD_compressBegin)");
+
+ if (frame && (cctx->stage==ZSTDcs_init)) {
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams,
+ cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
+ FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed");
+ assert(fhSize <= dstCapacity);
+ dstCapacity -= fhSize;
+ dst = (char*)dst + fhSize;
+ cctx->stage = ZSTDcs_ongoing;
+ }
+
+ if (!srcSize) return fhSize; /* do not generate an empty block if no input */
+
+ if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) {
+ ms->forceNonContiguous = 0;
+ ms->nextToUpdate = ms->window.dictLimit;
+ }
+ if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
+ ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0);
+ }
+
+ if (!frame) {
+ /* overflow check and correction for block mode */
+ ZSTD_overflowCorrectIfNeeded(
+ ms, &cctx->workspace, &cctx->appliedParams,
+ src, (BYTE const*)src + srcSize);
+ }
+
+ DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize);
+ { size_t const cSize = frame ?
+ ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
+ ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */);
+ FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed");
+ cctx->consumedSrcSize += srcSize;
+ cctx->producedCSize += (cSize + fhSize);
+ assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
+ if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
+ RETURN_ERROR_IF(
+ cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne,
+ srcSize_wrong,
+ "error : pledgedSrcSize = %u, while realSrcSize >= %u",
+ (unsigned)cctx->pledgedSrcSizePlusOne-1,
+ (unsigned)cctx->consumedSrcSize);
+ }
+ return cSize + fhSize;
+ }
+}
+
+size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize);
+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
+}
+
+
+size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
+{
+ ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
+ assert(!ZSTD_checkCParams(cParams));
+ return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
+}
+
+size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize);
+ { size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
+ RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); }
+
+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
+}
+
+/*! ZSTD_loadDictionaryContent() :
+ * @return : 0, or an error code
+ */
+static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
+ ldmState_t* ls,
+ ZSTD_cwksp* ws,
+ ZSTD_CCtx_params const* params,
+ const void* src, size_t srcSize,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp)
+{
+ const BYTE* ip = (const BYTE*) src;
+ const BYTE* const iend = ip + srcSize;
+ int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL;
+
+ /* Assert that the ms params match the params we're being given */
+ ZSTD_assertEqualCParams(params->cParams, ms->cParams);
+
+ { /* Ensure large dictionaries can't cause index overflow */
+
+ /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX.
+ * Dictionaries right at the edge will immediately trigger overflow
+ * correction, but I don't want to insert extra constraints here.
+ */
+ U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX;
+
+ int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(&params->cParams);
+ if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) {
+ /* Some dictionary matchfinders in zstd use "short cache",
+ * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each
+ * CDict hashtable entry as a tag rather than as part of an index.
+ * When short cache is used, we need to truncate the dictionary
+ * so that its indices don't overlap with the tag. */
+ U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX;
+ maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize);
+ assert(!loadLdmDict);
+ }
+
+ /* If the dictionary is too large, only load the suffix of the dictionary. */
+ if (srcSize > maxDictSize) {
+ ip = iend - maxDictSize;
+ src = ip;
+ srcSize = maxDictSize;
+ } }
+
+ if (srcSize > ZSTD_CHUNKSIZE_MAX) {
+ /* We must have cleared our windows when our source is this large. */
+ assert(ZSTD_window_isEmpty(ms->window));
+ if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window));
+ }
+
+ DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
+ ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
+ ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
+ ms->forceNonContiguous = params->deterministicRefPrefix;
+
+ if (loadLdmDict) {
+ ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
+ ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
+ }
+
+ if (srcSize <= HASH_READ_SIZE) return 0;
+
+ ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
+
+ if (loadLdmDict)
+ ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
+
+ switch(params->cParams.strategy)
+ {
+ case ZSTD_fast:
+ ZSTD_fillHashTable(ms, iend, dtlm, tfp);
+ break;
+ case ZSTD_dfast:
+ ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp);
+ break;
+
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ assert(srcSize >= HASH_READ_SIZE);
+ if (ms->dedicatedDictSearch) {
+ assert(ms->chainTable != NULL);
+ ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE);
+ } else {
+ assert(params->useRowMatchFinder != ZSTD_ps_auto);
+ if (params->useRowMatchFinder == ZSTD_ps_enable) {
+ size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
+ ZSTD_memset(ms->tagTable, 0, tagTableSize);
+ ZSTD_row_update(ms, iend-HASH_READ_SIZE);
+ DEBUGLOG(4, "Using row-based hash table for lazy dict");
+ } else {
+ ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
+ DEBUGLOG(4, "Using chain-based hash table for lazy dict");
+ }
+ }
+ break;
+
+ case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ assert(srcSize >= HASH_READ_SIZE);
+ ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
+ break;
+
+ default:
+ assert(0); /* not possible : not a valid strategy id */
+ }
+
+ ms->nextToUpdate = (U32)(iend - ms->window.base);
+ return 0;
+}
+
+
+/* Dictionaries that assign zero probability to symbols that show up causes problems
+ * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check
+ * and only dictionaries with 100% valid symbols can be assumed valid.
+ */
+static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
+{
+ U32 s;
+ if (dictMaxSymbolValue < maxSymbolValue) {
+ return FSE_repeat_check;
+ }
+ for (s = 0; s <= maxSymbolValue; ++s) {
+ if (normalizedCounter[s] == 0) {
+ return FSE_repeat_check;
+ }
+ }
+ return FSE_repeat_valid;
+}
+
+size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
+ const void* const dict, size_t dictSize)
+{
+ short offcodeNCount[MaxOff+1];
+ unsigned offcodeMaxValue = MaxOff;
+ const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */
+ const BYTE* const dictEnd = dictPtr + dictSize;
+ dictPtr += 8;
+ bs->entropy.huf.repeatMode = HUF_repeat_check;
+
+ { unsigned maxSymbolValue = 255;
+ unsigned hasZeroWeights = 1;
+ size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr,
+ dictEnd-dictPtr, &hasZeroWeights);
+
+ /* We only set the loaded table as valid if it contains all non-zero
+ * weights. Otherwise, we set it to check */
+ if (!hasZeroWeights)
+ bs->entropy.huf.repeatMode = HUF_repeat_valid;
+
+ RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, "");
+ dictPtr += hufHeaderSize;
+ }
+
+ { unsigned offcodeLog;
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
+ RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
+ /* fill all offset symbols to avoid garbage at end of table */
+ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+ bs->entropy.fse.offcodeCTable,
+ offcodeNCount, MaxOff, offcodeLog,
+ workspace, HUF_WORKSPACE_SIZE)),
+ dictionary_corrupted, "");
+ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
+ dictPtr += offcodeHeaderSize;
+ }
+
+ { short matchlengthNCount[MaxML+1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
+ RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
+ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+ bs->entropy.fse.matchlengthCTable,
+ matchlengthNCount, matchlengthMaxValue, matchlengthLog,
+ workspace, HUF_WORKSPACE_SIZE)),
+ dictionary_corrupted, "");
+ bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML);
+ dictPtr += matchlengthHeaderSize;
+ }
+
+ { short litlengthNCount[MaxLL+1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
+ RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
+ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+ bs->entropy.fse.litlengthCTable,
+ litlengthNCount, litlengthMaxValue, litlengthLog,
+ workspace, HUF_WORKSPACE_SIZE)),
+ dictionary_corrupted, "");
+ bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL);
+ dictPtr += litlengthHeaderSize;
+ }
+
+ RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
+ bs->rep[0] = MEM_readLE32(dictPtr+0);
+ bs->rep[1] = MEM_readLE32(dictPtr+4);
+ bs->rep[2] = MEM_readLE32(dictPtr+8);
+ dictPtr += 12;
+
+ { size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
+ U32 offcodeMax = MaxOff;
+ if (dictContentSize <= ((U32)-1) - 128 KB) {
+ U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */
+ offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
+ }
+ /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */
+ bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff));
+
+ /* All repCodes must be <= dictContentSize and != 0 */
+ { U32 u;
+ for (u=0; u<3; u++) {
+ RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, "");
+ RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, "");
+ } } }
+
+ return dictPtr - (const BYTE*)dict;
+}
+
+/* Dictionary format :
+ * See :
+ * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format
+ */
+/*! ZSTD_loadZstdDictionary() :
+ * @return : dictID, or an error code
+ * assumptions : magic number supposed already checked
+ * dictSize supposed >= 8
+ */
+static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
+ ZSTD_matchState_t* ms,
+ ZSTD_cwksp* ws,
+ ZSTD_CCtx_params const* params,
+ const void* dict, size_t dictSize,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp,
+ void* workspace)
+{
+ const BYTE* dictPtr = (const BYTE*)dict;
+ const BYTE* const dictEnd = dictPtr + dictSize;
+ size_t dictID;
+ size_t eSize;
+ ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
+ assert(dictSize >= 8);
+ assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY);
+
+ dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ );
+ eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize);
+ FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed");
+ dictPtr += eSize;
+
+ {
+ size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
+ FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(
+ ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), "");
+ }
+ return dictID;
+}
+
+/** ZSTD_compress_insertDictionary() :
+* @return : dictID, or an error code */
+static size_t
+ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
+ ZSTD_matchState_t* ms,
+ ldmState_t* ls,
+ ZSTD_cwksp* ws,
+ const ZSTD_CCtx_params* params,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp,
+ void* workspace)
+{
+ DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
+ if ((dict==NULL) || (dictSize<8)) {
+ RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
+ return 0;
+ }
+
+ ZSTD_reset_compressedBlockState(bs);
+
+ /* dict restricted modes */
+ if (dictContentType == ZSTD_dct_rawContent)
+ return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp);
+
+ if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
+ if (dictContentType == ZSTD_dct_auto) {
+ DEBUGLOG(4, "raw content dictionary detected");
+ return ZSTD_loadDictionaryContent(
+ ms, ls, ws, params, dict, dictSize, dtlm, tfp);
+ }
+ RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
+ assert(0); /* impossible */
+ }
+
+ /* dict as full zstd dictionary */
+ return ZSTD_loadZstdDictionary(
+ bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace);
+}
+
+#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
+#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
+
+/*! ZSTD_compressBegin_internal() :
+ * Assumption : either @dict OR @cdict (or none) is non-NULL, never both
+ * @return : 0, or an error code */
+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params, U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize;
+#if ZSTD_TRACE
+ cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
+#endif
+ DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog);
+ /* params are supposed to be fully validated at this point */
+ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
+ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
+ if ( (cdict)
+ && (cdict->dictContentSize > 0)
+ && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
+ || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || cdict->compressionLevel == 0)
+ && (params->attachDictPref != ZSTD_dictForceLoad) ) {
+ return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
+ }
+
+ FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
+ dictContentSize,
+ ZSTDcrp_makeClean, zbuff) , "");
+ { size_t const dictID = cdict ?
+ ZSTD_compress_insertDictionary(
+ cctx->blockState.prevCBlock, &cctx->blockState.matchState,
+ &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
+ cdict->dictContentSize, cdict->dictContentType, dtlm,
+ ZSTD_tfp_forCCtx, cctx->entropyWorkspace)
+ : ZSTD_compress_insertDictionary(
+ cctx->blockState.prevCBlock, &cctx->blockState.matchState,
+ &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize,
+ dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace);
+ FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
+ assert(dictID <= UINT_MAX);
+ cctx->dictID = (U32)dictID;
+ cctx->dictContentSize = dictContentSize;
+ }
+ return 0;
+}
+
+size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog);
+ /* compression parameters verification and optimization */
+ FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , "");
+ return ZSTD_compressBegin_internal(cctx,
+ dict, dictSize, dictContentType, dtlm,
+ cdict,
+ params, pledgedSrcSize,
+ ZSTDb_not_buffered);
+}
+
+/*! ZSTD_compressBegin_advanced() :
+* @return : 0, or an error code */
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ ZSTD_CCtx_params cctxParams;
+ ZSTD_CCtxParams_init_internal(&cctxParams, &params, ZSTD_NO_CLEVEL);
+ return ZSTD_compressBegin_advanced_internal(cctx,
+ dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast,
+ NULL /*cdict*/,
+ &cctxParams, pledgedSrcSize);
+}
+
+size_t
+ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_CCtx_params cctxParams;
+ { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
+ ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
+ }
+ DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
+ &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
+}
+
+size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
+{
+ return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
+}
+
+
+/*! ZSTD_writeEpilogue() :
+* Ends a frame.
+* @return : nb of bytes written into dst (or an error code) */
+static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* op = ostart;
+ size_t fhSize = 0;
+
+ DEBUGLOG(4, "ZSTD_writeEpilogue");
+ RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing");
+
+ /* special case : empty frame */
+ if (cctx->stage == ZSTDcs_init) {
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0);
+ FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed");
+ dstCapacity -= fhSize;
+ op += fhSize;
+ cctx->stage = ZSTDcs_ongoing;
+ }
+
+ if (cctx->stage != ZSTDcs_ending) {
+ /* write one last empty block, make it the "last" block */
+ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0;
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue");
+ MEM_writeLE32(op, cBlockHeader24);
+ op += ZSTD_blockHeaderSize;
+ dstCapacity -= ZSTD_blockHeaderSize;
+ }
+
+ if (cctx->appliedParams.fParams.checksumFlag) {
+ U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum");
+ DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum);
+ MEM_writeLE32(op, checksum);
+ op += 4;
+ }
+
+ cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
+ return op-ostart;
+}
+
+void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
+{
+#if ZSTD_TRACE
+ if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) {
+ int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0;
+ ZSTD_Trace trace;
+ ZSTD_memset(&trace, 0, sizeof(trace));
+ trace.version = ZSTD_VERSION_NUMBER;
+ trace.streaming = streaming;
+ trace.dictionaryID = cctx->dictID;
+ trace.dictionarySize = cctx->dictContentSize;
+ trace.uncompressedSize = cctx->consumedSrcSize;
+ trace.compressedSize = cctx->producedCSize + extraCSize;
+ trace.params = &cctx->appliedParams;
+ trace.cctx = cctx;
+ ZSTD_trace_compress_end(cctx->traceCtx, &trace);
+ }
+ cctx->traceCtx = 0;
+#else
+ (void)cctx;
+ (void)extraCSize;
+#endif
+}
+
+size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ size_t endResult;
+ size_t const cSize = ZSTD_compressContinue_internal(cctx,
+ dst, dstCapacity, src, srcSize,
+ 1 /* frame mode */, 1 /* last chunk */);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed");
+ endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
+ FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed");
+ assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
+ if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
+ DEBUGLOG(4, "end of frame : controlling src size");
+ RETURN_ERROR_IF(
+ cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1,
+ srcSize_wrong,
+ "error : pledgedSrcSize = %u, while realSrcSize = %u",
+ (unsigned)cctx->pledgedSrcSizePlusOne-1,
+ (unsigned)cctx->consumedSrcSize);
+ }
+ ZSTD_CCtx_trace(cctx, endResult);
+ return cSize + endResult;
+}
+
+size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params)
+{
+ DEBUGLOG(4, "ZSTD_compress_advanced");
+ FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
+ ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, &params, ZSTD_NO_CLEVEL);
+ return ZSTD_compress_advanced_internal(cctx,
+ dst, dstCapacity,
+ src, srcSize,
+ dict, dictSize,
+ &cctx->simpleApiParams);
+}
+
+/* Internal */
+size_t ZSTD_compress_advanced_internal(
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ const ZSTD_CCtx_params* params)
+{
+ DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize);
+ FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
+ dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
+ params, srcSize, ZSTDb_not_buffered) , "");
+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+}
+
+size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ int compressionLevel)
+{
+ {
+ ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
+ assert(params.fParams.contentSizeFlag == 1);
+ ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
+ }
+ DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
+ return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams);
+}
+
+size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel)
+{
+ DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize);
+ assert(cctx != NULL);
+ return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
+}
+
+size_t ZSTD_compress(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel)
+{
+ size_t result;
+#if ZSTD_COMPRESS_HEAPMODE
+ ZSTD_CCtx* cctx = ZSTD_createCCtx();
+ RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed");
+ result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel);
+ ZSTD_freeCCtx(cctx);
+#else
+ ZSTD_CCtx ctxBody;
+ ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem);
+ result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
+ ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */
+#endif
+ return result;
+}
+
+
+/* ===== Dictionary API ===== */
+
+/*! ZSTD_estimateCDictSize_advanced() :
+ * Estimate amount of memory that will be needed to create a dictionary with following arguments */
+size_t ZSTD_estimateCDictSize_advanced(
+ size_t dictSize, ZSTD_compressionParameters cParams,
+ ZSTD_dictLoadMethod_e dictLoadMethod)
+{
+ DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict));
+ return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small
+ * in case we are using DDS with row-hash. */
+ + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams),
+ /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0)
+ + (dictLoadMethod == ZSTD_dlm_byRef ? 0
+ : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
+}
+
+size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
+{
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
+}
+
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
+{
+ if (cdict==NULL) return 0; /* support sizeof on NULL */
+ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict));
+ /* cdict may be in the workspace */
+ return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict))
+ + ZSTD_cwksp_sizeof(&cdict->workspace);
+}
+
+static size_t ZSTD_initCDict_internal(
+ ZSTD_CDict* cdict,
+ const void* dictBuffer, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_CCtx_params params)
+{
+ DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType);
+ assert(!ZSTD_checkCParams(params.cParams));
+ cdict->matchState.cParams = params.cParams;
+ cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch;
+ if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
+ cdict->dictContent = dictBuffer;
+ } else {
+ void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*)));
+ RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!");
+ cdict->dictContent = internalBuffer;
+ ZSTD_memcpy(internalBuffer, dictBuffer, dictSize);
+ }
+ cdict->dictContentSize = dictSize;
+ cdict->dictContentType = dictContentType;
+
+ cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE);
+
+
+ /* Reset the state to no dictionary */
+ ZSTD_reset_compressedBlockState(&cdict->cBlockState);
+ FORWARD_IF_ERROR(ZSTD_reset_matchState(
+ &cdict->matchState,
+ &cdict->workspace,
+ &params.cParams,
+ params.useRowMatchFinder,
+ ZSTDcrp_makeClean,
+ ZSTDirp_reset,
+ ZSTD_resetTarget_CDict), "");
+ /* (Maybe) load the dictionary
+ * Skips loading the dictionary if it is < 8 bytes.
+ */
+ { params.compressionLevel = ZSTD_CLEVEL_DEFAULT;
+ params.fParams.contentSizeFlag = 1;
+ { size_t const dictID = ZSTD_compress_insertDictionary(
+ &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace,
+ &params, cdict->dictContent, cdict->dictContentSize,
+ dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace);
+ FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
+ assert(dictID <= (size_t)(U32)-1);
+ cdict->dictID = (U32)dictID;
+ }
+ }
+
+ return 0;
+}
+
+static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_compressionParameters cParams,
+ ZSTD_paramSwitch_e useRowMatchFinder,
+ U32 enableDedicatedDictSearch,
+ ZSTD_customMem customMem)
+{
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+ { size_t const workspaceSize =
+ ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
+ ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) +
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
+ : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
+ void* const workspace = ZSTD_customMalloc(workspaceSize, customMem);
+ ZSTD_cwksp ws;
+ ZSTD_CDict* cdict;
+
+ if (!workspace) {
+ ZSTD_customFree(workspace, customMem);
+ return NULL;
+ }
+
+ ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc);
+
+ cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
+ assert(cdict != NULL);
+ ZSTD_cwksp_move(&cdict->workspace, &ws);
+ cdict->customMem = customMem;
+ cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
+ cdict->useRowMatchFinder = useRowMatchFinder;
+ return cdict;
+ }
+}
+
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams,
+ ZSTD_customMem customMem)
+{
+ ZSTD_CCtx_params cctxParams;
+ ZSTD_memset(&cctxParams, 0, sizeof(cctxParams));
+ ZSTD_CCtxParams_init(&cctxParams, 0);
+ cctxParams.cParams = cParams;
+ cctxParams.customMem = customMem;
+ return ZSTD_createCDict_advanced2(
+ dictBuffer, dictSize,
+ dictLoadMethod, dictContentType,
+ &cctxParams, customMem);
+}
+
+ZSTD_CDict* ZSTD_createCDict_advanced2(
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ const ZSTD_CCtx_params* originalCctxParams,
+ ZSTD_customMem customMem)
+{
+ ZSTD_CCtx_params cctxParams = *originalCctxParams;
+ ZSTD_compressionParameters cParams;
+ ZSTD_CDict* cdict;
+
+ DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType);
+ if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
+
+ if (cctxParams.enableDedicatedDictSearch) {
+ cParams = ZSTD_dedicatedDictSearch_getCParams(
+ cctxParams.compressionLevel, dictSize);
+ ZSTD_overrideCParams(&cParams, &cctxParams.cParams);
+ } else {
+ cParams = ZSTD_getCParamsFromCCtxParams(
+ &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ }
+
+ if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) {
+ /* Fall back to non-DDSS params */
+ cctxParams.enableDedicatedDictSearch = 0;
+ cParams = ZSTD_getCParamsFromCCtxParams(
+ &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ }
+
+ DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch);
+ cctxParams.cParams = cParams;
+ cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+
+ cdict = ZSTD_createCDict_advanced_internal(dictSize,
+ dictLoadMethod, cctxParams.cParams,
+ cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch,
+ customMem);
+
+ if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+ dict, dictSize,
+ dictLoadMethod, dictContentType,
+ cctxParams) )) {
+ ZSTD_freeCDict(cdict);
+ return NULL;
+ }
+
+ return cdict;
+}
+
+ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
+ ZSTD_dlm_byCopy, ZSTD_dct_auto,
+ cParams, ZSTD_defaultCMem);
+ if (cdict)
+ cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
+ return cdict;
+}
+
+ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
+ ZSTD_dlm_byRef, ZSTD_dct_auto,
+ cParams, ZSTD_defaultCMem);
+ if (cdict)
+ cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
+ return cdict;
+}
+
+size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
+{
+ if (cdict==NULL) return 0; /* support free on NULL */
+ { ZSTD_customMem const cMem = cdict->customMem;
+ int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict);
+ ZSTD_cwksp_free(&cdict->workspace, cMem);
+ if (!cdictInWorkspace) {
+ ZSTD_customFree(cdict, cMem);
+ }
+ return 0;
+ }
+}
+
+/*! ZSTD_initStaticCDict_advanced() :
+ * Generate a digested dictionary in provided memory area.
+ * workspace: The memory area to emplace the dictionary into.
+ * Provided pointer must 8-bytes aligned.
+ * It must outlive dictionary usage.
+ * workspaceSize: Use ZSTD_estimateCDictSize()
+ * to determine how large workspace must be.
+ * cParams : use ZSTD_getCParams() to transform a compression level
+ * into its relevants cParams.
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ * Note : there is no corresponding "free" function.
+ * Since workspace was allocated externally, it must be freed externally.
+ */
+const ZSTD_CDict* ZSTD_initStaticCDict(
+ void* workspace, size_t workspaceSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams)
+{
+ ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
+ /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */
+ size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0);
+ size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ + (dictLoadMethod == ZSTD_dlm_byRef ? 0
+ : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
+ + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ + matchStateSize;
+ ZSTD_CDict* cdict;
+ ZSTD_CCtx_params params;
+
+ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
+
+ {
+ ZSTD_cwksp ws;
+ ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc);
+ cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
+ if (cdict == NULL) return NULL;
+ ZSTD_cwksp_move(&cdict->workspace, &ws);
+ }
+
+ DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
+ (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize));
+ if (workspaceSize < neededSize) return NULL;
+
+ ZSTD_CCtxParams_init(&params, 0);
+ params.cParams = cParams;
+ params.useRowMatchFinder = useRowMatchFinder;
+ cdict->useRowMatchFinder = useRowMatchFinder;
+
+ if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+ dict, dictSize,
+ dictLoadMethod, dictContentType,
+ params) ))
+ return NULL;
+
+ return cdict;
+}
+
+ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict)
+{
+ assert(cdict != NULL);
+ return cdict->matchState.cParams;
+}
+
+/*! ZSTD_getDictID_fromCDict() :
+ * Provides the dictID of the dictionary loaded into `cdict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict)
+{
+ if (cdict==NULL) return 0;
+ return cdict->dictID;
+}
+
+/* ZSTD_compressBegin_usingCDict_internal() :
+ * Implementation of various ZSTD_compressBegin_usingCDict* functions.
+ */
+static size_t ZSTD_compressBegin_usingCDict_internal(
+ ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
+ ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
+{
+ ZSTD_CCtx_params cctxParams;
+ DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal");
+ RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!");
+ /* Initialize the cctxParams from the cdict */
+ {
+ ZSTD_parameters params;
+ params.fParams = fParams;
+ params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
+ || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || cdict->compressionLevel == 0 ) ?
+ ZSTD_getCParamsFromCDict(cdict)
+ : ZSTD_getCParams(cdict->compressionLevel,
+ pledgedSrcSize,
+ cdict->dictContentSize);
+ ZSTD_CCtxParams_init_internal(&cctxParams, &params, cdict->compressionLevel);
+ }
+ /* Increase window log to fit the entire dictionary and source if the
+ * source size is known. Limit the increase to 19, which is the
+ * window log for compression level 1 with the largest source size.
+ */
+ if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+ U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
+ U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
+ cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog);
+ }
+ return ZSTD_compressBegin_internal(cctx,
+ NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast,
+ cdict,
+ &cctxParams, pledgedSrcSize,
+ ZSTDb_not_buffered);
+}
+
+
+/* ZSTD_compressBegin_usingCDict_advanced() :
+ * This function is DEPRECATED.
+ * cdict must be != NULL */
+size_t ZSTD_compressBegin_usingCDict_advanced(
+ ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
+ ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
+{
+ return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize);
+}
+
+/* ZSTD_compressBegin_usingCDict() :
+ * cdict must be != NULL */
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+{
+ ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+ return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
+}
+
+/*! ZSTD_compress_usingCDict_internal():
+ * Implementation of various ZSTD_compress_usingCDict* functions.
+ */
+static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
+{
+ FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */
+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+}
+
+/*! ZSTD_compress_usingCDict_advanced():
+ * This function is DEPRECATED.
+ */
+size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
+{
+ return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
+}
+
+/*! ZSTD_compress_usingCDict() :
+ * Compression using a digested Dictionary.
+ * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+ * Note that compression parameters are decided at CDict creation time
+ * while frame parameters are hardcoded */
+size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict)
+{
+ ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+ return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
+}
+
+
+
+/* ******************************************************************
+* Streaming
+********************************************************************/
+
+ZSTD_CStream* ZSTD_createCStream(void)
+{
+ DEBUGLOG(3, "ZSTD_createCStream");
+ return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
+}
+
+ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize)
+{
+ return ZSTD_initStaticCCtx(workspace, workspaceSize);
+}
+
+ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
+{ /* CStream and CCtx are now same object */
+ return ZSTD_createCCtx_advanced(customMem);
+}
+
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
+{
+ return ZSTD_freeCCtx(zcs); /* same object */
+}
+
+
+
+/*====== Initialization ======*/
+
+size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; }
+
+size_t ZSTD_CStreamOutSize(void)
+{
+ return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
+}
+
+static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize)
+{
+ if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize))
+ return ZSTD_cpm_attachDict;
+ else
+ return ZSTD_cpm_noAttachDict;
+}
+
+/* ZSTD_resetCStream():
+ * pledgedSrcSize == 0 means "unknown" */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
+{
+ /* temporary : 0 interpreted as "unknown" during transition period.
+ * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
+ * 0 will be interpreted as "empty" in the future.
+ */
+ U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+ DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize);
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ return 0;
+}
+
+/*! ZSTD_initCStream_internal() :
+ * Note : for lib/compress only. Used by zstdmt_compress.c.
+ * Assumption 1 : params are valid
+ * Assumption 2 : either dict, or cdict, is defined, not both */
+size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_internal");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
+ zcs->requestedParams = *params;
+ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
+ if (dict) {
+ FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
+ } else {
+ /* Dictionary is cleared if !cdict */
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , "");
+ }
+ return 0;
+}
+
+/* ZSTD_initCStream_usingCDict_advanced() :
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams,
+ unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ zcs->requestedParams.fParams = fParams;
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , "");
+ return 0;
+}
+
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_usingCDict");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , "");
+ return 0;
+}
+
+
+/* ZSTD_initCStream_advanced() :
+ * pledgedSrcSize must be exact.
+ * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pss)
+{
+ /* for compatibility with older programs relying on this behavior.
+ * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN.
+ * This line will be removed in the future.
+ */
+ U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+ DEBUGLOG(4, "ZSTD_initCStream_advanced");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
+ ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, &params);
+ FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
+ return 0;
+}
+
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_usingDict");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
+ return 0;
+}
+
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss)
+{
+ /* temporary : 0 interpreted as "unknown" during transition period.
+ * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
+ * 0 will be interpreted as "empty" in the future.
+ */
+ U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+ DEBUGLOG(4, "ZSTD_initCStream_srcSize");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ return 0;
+}
+
+size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
+{
+ DEBUGLOG(4, "ZSTD_initCStream");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , "");
+ return 0;
+}
+
+/*====== Compression ======*/
+
+static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
+{
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ return cctx->blockSize - cctx->stableIn_notConsumed;
+ }
+ assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered);
+ { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
+ if (hintInSize==0) hintInSize = cctx->blockSize;
+ return hintInSize;
+ }
+}
+
+/** ZSTD_compressStream_generic():
+ * internal function for all *compressStream*() variants
+ * @return : hint size for next input to complete ongoing block */
+static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective const flushMode)
+{
+ const char* const istart = (assert(input != NULL), (const char*)input->src);
+ const char* const iend = (istart != NULL) ? istart + input->size : istart;
+ const char* ip = (istart != NULL) ? istart + input->pos : istart;
+ char* const ostart = (assert(output != NULL), (char*)output->dst);
+ char* const oend = (ostart != NULL) ? ostart + output->size : ostart;
+ char* op = (ostart != NULL) ? ostart + output->pos : ostart;
+ U32 someMoreWork = 1;
+
+ /* check expectations */
+ DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos);
+ assert(zcs != NULL);
+ if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ assert(input->pos >= zcs->stableIn_notConsumed);
+ input->pos -= zcs->stableIn_notConsumed;
+ ip -= zcs->stableIn_notConsumed;
+ zcs->stableIn_notConsumed = 0;
+ }
+ if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
+ assert(zcs->inBuff != NULL);
+ assert(zcs->inBuffSize > 0);
+ }
+ if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) {
+ assert(zcs->outBuff != NULL);
+ assert(zcs->outBuffSize > 0);
+ }
+ if (input->src == NULL) assert(input->size == 0);
+ assert(input->pos <= input->size);
+ if (output->dst == NULL) assert(output->size == 0);
+ assert(output->pos <= output->size);
+ assert((U32)flushMode <= (U32)ZSTD_e_end);
+
+ while (someMoreWork) {
+ switch(zcs->streamStage)
+ {
+ case zcss_init:
+ RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!");
+
+ case zcss_load:
+ if ( (flushMode == ZSTD_e_end)
+ && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */
+ || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */
+ && (zcs->inBuffPos == 0) ) {
+ /* shortcut to compression pass directly into output buffer */
+ size_t const cSize = ZSTD_compressEnd(zcs,
+ op, oend-op, ip, iend-ip);
+ DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
+ ip = iend;
+ op += cSize;
+ zcs->frameEnded = 1;
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ someMoreWork = 0; break;
+ }
+ /* complete loading into inBuffer in buffered mode */
+ if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
+ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
+ size_t const loaded = ZSTD_limitCopy(
+ zcs->inBuff + zcs->inBuffPos, toLoad,
+ ip, iend-ip);
+ zcs->inBuffPos += loaded;
+ if (ip) ip += loaded;
+ if ( (flushMode == ZSTD_e_continue)
+ && (zcs->inBuffPos < zcs->inBuffTarget) ) {
+ /* not enough input to fill full block : stop here */
+ someMoreWork = 0; break;
+ }
+ if ( (flushMode == ZSTD_e_flush)
+ && (zcs->inBuffPos == zcs->inToCompress) ) {
+ /* empty */
+ someMoreWork = 0; break;
+ }
+ } else {
+ assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
+ if ( (flushMode == ZSTD_e_continue)
+ && ( (size_t)(iend - ip) < zcs->blockSize) ) {
+ /* can't compress a full block : stop here */
+ zcs->stableIn_notConsumed = (size_t)(iend - ip);
+ ip = iend; /* pretend to have consumed input */
+ someMoreWork = 0; break;
+ }
+ if ( (flushMode == ZSTD_e_flush)
+ && (ip == iend) ) {
+ /* empty */
+ someMoreWork = 0; break;
+ }
+ }
+ /* compress current block (note : this stage cannot be stopped in the middle) */
+ DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
+ { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered);
+ void* cDst;
+ size_t cSize;
+ size_t oSize = oend-op;
+ size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress
+ : MIN((size_t)(iend - ip), zcs->blockSize);
+ if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
+ cDst = op; /* compress into output buffer, to skip flush stage */
+ else
+ cDst = zcs->outBuff, oSize = zcs->outBuffSize;
+ if (inputBuffered) {
+ unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
+ cSize = lastBlock ?
+ ZSTD_compressEnd(zcs, cDst, oSize,
+ zcs->inBuff + zcs->inToCompress, iSize) :
+ ZSTD_compressContinue(zcs, cDst, oSize,
+ zcs->inBuff + zcs->inToCompress, iSize);
+ FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
+ zcs->frameEnded = lastBlock;
+ /* prepare next block */
+ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
+ if (zcs->inBuffTarget > zcs->inBuffSize)
+ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
+ DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
+ (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize);
+ if (!lastBlock)
+ assert(zcs->inBuffTarget <= zcs->inBuffSize);
+ zcs->inToCompress = zcs->inBuffPos;
+ } else { /* !inputBuffered, hence ZSTD_bm_stable */
+ unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend);
+ cSize = lastBlock ?
+ ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
+ ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
+ /* Consume the input prior to error checking to mirror buffered mode. */
+ if (ip) ip += iSize;
+ FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
+ zcs->frameEnded = lastBlock;
+ if (lastBlock) assert(ip == iend);
+ }
+ if (cDst == op) { /* no need to flush */
+ op += cSize;
+ if (zcs->frameEnded) {
+ DEBUGLOG(5, "Frame completed directly in outBuffer");
+ someMoreWork = 0;
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ }
+ break;
+ }
+ zcs->outBuffContentSize = cSize;
+ zcs->outBuffFlushedSize = 0;
+ zcs->streamStage = zcss_flush; /* pass-through to flush stage */
+ }
+ ZSTD_FALLTHROUGH;
+ case zcss_flush:
+ DEBUGLOG(5, "flush stage");
+ assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered);
+ { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+ size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op),
+ zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+ DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
+ (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
+ if (flushed)
+ op += flushed;
+ zcs->outBuffFlushedSize += flushed;
+ if (toFlush!=flushed) {
+ /* flush not fully completed, presumably because dst is too small */
+ assert(op==oend);
+ someMoreWork = 0;
+ break;
+ }
+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+ if (zcs->frameEnded) {
+ DEBUGLOG(5, "Frame completed on flush");
+ someMoreWork = 0;
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ break;
+ }
+ zcs->streamStage = zcss_load;
+ break;
+ }
+
+ default: /* impossible */
+ assert(0);
+ }
+ }
+
+ input->pos = ip - istart;
+ output->pos = op - ostart;
+ if (zcs->frameEnded) return 0;
+ return ZSTD_nextInputSizeHint(zcs);
+}
+
+static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers >= 1) {
+ assert(cctx->mtctx != NULL);
+ return ZSTDMT_nextInputSizeHint(cctx->mtctx);
+ }
+#endif
+ return ZSTD_nextInputSizeHint(cctx);
+
+}
+
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+ FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , "");
+ return ZSTD_nextInputSizeHint_MTorST(zcs);
+}
+
+/* After a compression call set the expected input/output buffer.
+ * This is validated at the start of the next compression call.
+ */
+static void
+ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input)
+{
+ DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)");
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ cctx->expectedInBuffer = *input;
+ }
+ if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
+ cctx->expectedOutBufferSize = output->size - output->pos;
+ }
+}
+
+/* Validate that the input/output buffers match the expectations set by
+ * ZSTD_setBufferExpectations.
+ */
+static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx,
+ ZSTD_outBuffer const* output,
+ ZSTD_inBuffer const* input,
+ ZSTD_EndDirective endOp)
+{
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ ZSTD_inBuffer const expect = cctx->expectedInBuffer;
+ if (expect.src != input->src || expect.pos != input->pos)
+ RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!");
+ }
+ (void)endOp;
+ if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
+ size_t const outBufferSize = output->size - output->pos;
+ if (cctx->expectedOutBufferSize != outBufferSize)
+ RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!");
+ }
+ return 0;
+}
+
+static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
+ ZSTD_EndDirective endOp,
+ size_t inSize)
+{
+ ZSTD_CCtx_params params = cctx->requestedParams;
+ ZSTD_prefixDict const prefixDict = cctx->prefixDict;
+ FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
+ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
+ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
+ if (cctx->cdict && !cctx->localDict.cdict) {
+ /* Let the cdict's compression level take priority over the requested params.
+ * But do not take the cdict's compression level if the "cdict" is actually a localDict
+ * generated from ZSTD_initLocalDict().
+ */
+ params.compressionLevel = cctx->cdict->compressionLevel;
+ }
+ DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
+ if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */
+
+ { size_t const dictSize = prefixDict.dict
+ ? prefixDict.dictSize
+ : (cctx->cdict ? cctx->cdict->dictContentSize : 0);
+ ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, &params, cctx->pledgedSrcSizePlusOne - 1);
+ params.cParams = ZSTD_getCParamsFromCCtxParams(
+ &params, cctx->pledgedSrcSizePlusOne-1,
+ dictSize, mode);
+ }
+
+ params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, &params.cParams);
+ params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
+ params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
+ params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+ params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
+ params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel);
+
+#ifdef ZSTD_MULTITHREAD
+ /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
+ RETURN_ERROR_IF(
+ params.useSequenceProducer == 1 && params.nbWorkers >= 1,
+ parameter_combination_unsupported,
+ "External sequence producer isn't supported with nbWorkers >= 1"
+ );
+
+ if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
+ params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
+ }
+ if (params.nbWorkers > 0) {
+#if ZSTD_TRACE
+ cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
+#endif
+ /* mt context creation */
+ if (cctx->mtctx == NULL) {
+ DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
+ params.nbWorkers);
+ cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool);
+ RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!");
+ }
+ /* mt compression */
+ DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
+ FORWARD_IF_ERROR( ZSTDMT_initCStream_internal(
+ cctx->mtctx,
+ prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
+ cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
+ cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0;
+ cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize;
+ cctx->consumedSrcSize = 0;
+ cctx->producedCSize = 0;
+ cctx->streamStage = zcss_load;
+ cctx->appliedParams = params;
+ } else
+#endif /* ZSTD_MULTITHREAD */
+ { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1;
+ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+ FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
+ prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast,
+ cctx->cdict,
+ &params, pledgedSrcSize,
+ ZSTDb_buffered) , "");
+ assert(cctx->appliedParams.nbWorkers == 0);
+ cctx->inToCompress = 0;
+ cctx->inBuffPos = 0;
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) {
+ /* for small input: avoid automatic flush on reaching end of block, since
+ * it would require to add a 3-bytes null block to end frame
+ */
+ cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize);
+ } else {
+ cctx->inBuffTarget = 0;
+ }
+ cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0;
+ cctx->streamStage = zcss_load;
+ cctx->frameEnded = 0;
+ }
+ return 0;
+}
+
+/* @return provides a minimum amount of data remaining to be flushed from internal buffers
+ */
+size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective endOp)
+{
+ DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp);
+ /* check conditions */
+ RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer");
+ RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer");
+ RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective");
+ assert(cctx != NULL);
+
+ /* transparent initialization stage */
+ if (cctx->streamStage == zcss_init) {
+ size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */
+ size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed;
+ if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */
+ && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */
+ && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */
+ if (cctx->stableIn_notConsumed) { /* not the first time */
+ /* check stable source guarantees */
+ RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer");
+ RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos");
+ }
+ /* pretend input was consumed, to give a sense forward progress */
+ input->pos = input->size;
+ /* save stable inBuffer, for later control, and flush/end */
+ cctx->expectedInBuffer = *input;
+ /* but actually input wasn't consumed, so keep track of position from where compression shall resume */
+ cctx->stableIn_notConsumed += inputSize;
+ /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */
+ return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */
+ }
+ FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed");
+ ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */
+ }
+ /* end of transparent initialization stage */
+
+ FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers");
+ /* compression stage */
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers > 0) {
+ size_t flushMin;
+ if (cctx->cParamsChanged) {
+ ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
+ cctx->cParamsChanged = 0;
+ }
+ if (cctx->stableIn_notConsumed) {
+ assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable);
+ /* some early data was skipped - make it available for consumption */
+ assert(input->pos >= cctx->stableIn_notConsumed);
+ input->pos -= cctx->stableIn_notConsumed;
+ cctx->stableIn_notConsumed = 0;
+ }
+ for (;;) {
+ size_t const ipos = input->pos;
+ size_t const opos = output->pos;
+ flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
+ cctx->consumedSrcSize += (U64)(input->pos - ipos);
+ cctx->producedCSize += (U64)(output->pos - opos);
+ if ( ZSTD_isError(flushMin)
+ || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
+ if (flushMin == 0)
+ ZSTD_CCtx_trace(cctx, 0);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+ }
+ FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
+
+ if (endOp == ZSTD_e_continue) {
+ /* We only require some progress with ZSTD_e_continue, not maximal progress.
+ * We're done if we've consumed or produced any bytes, or either buffer is
+ * full.
+ */
+ if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size)
+ break;
+ } else {
+ assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end);
+ /* We require maximal progress. We're done when the flush is complete or the
+ * output buffer is full.
+ */
+ if (flushMin == 0 || output->pos == output->size)
+ break;
+ }
+ }
+ DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
+ /* Either we don't require maximum forward progress, we've finished the
+ * flush, or we are out of output space.
+ */
+ assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size);
+ ZSTD_setBufferExpectations(cctx, output, input);
+ return flushMin;
+ }
+#endif /* ZSTD_MULTITHREAD */
+ FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , "");
+ DEBUGLOG(5, "completed ZSTD_compressStream2");
+ ZSTD_setBufferExpectations(cctx, output, input);
+ return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
+}
+
+size_t ZSTD_compressStream2_simpleArgs (
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos,
+ ZSTD_EndDirective endOp)
+{
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ output.dst = dst;
+ output.size = dstCapacity;
+ output.pos = *dstPos;
+ input.src = src;
+ input.size = srcSize;
+ input.pos = *srcPos;
+ /* ZSTD_compressStream2() will check validity of dstPos and srcPos */
+ { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
+ *dstPos = output.pos;
+ *srcPos = input.pos;
+ return cErr;
+ }
+}
+
+size_t ZSTD_compress2(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode;
+ ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode;
+ DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+ /* Enable stable input/output buffers. */
+ cctx->requestedParams.inBufferMode = ZSTD_bm_stable;
+ cctx->requestedParams.outBufferMode = ZSTD_bm_stable;
+ { size_t oPos = 0;
+ size_t iPos = 0;
+ size_t const result = ZSTD_compressStream2_simpleArgs(cctx,
+ dst, dstCapacity, &oPos,
+ src, srcSize, &iPos,
+ ZSTD_e_end);
+ /* Reset to the original values. */
+ cctx->requestedParams.inBufferMode = originalInBufferMode;
+ cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
+ FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
+ if (result != 0) { /* compression not completed, due to lack of output space */
+ assert(oPos == dstCapacity);
+ RETURN_ERROR(dstSize_tooSmall, "");
+ }
+ assert(iPos == srcSize); /* all input is expected consumed */
+ return oPos;
+ }
+}
+
+/* ZSTD_validateSequence() :
+ * @offCode : is presumed to follow format required by ZSTD_storeSeq()
+ * @returns a ZSTD error code if sequence is not valid
+ */
+static size_t
+ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
+ size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer)
+{
+ U32 const windowSize = 1u << windowLog;
+ /* posInSrc represents the amount of data the decoder would decode up to this point.
+ * As long as the amount of data decoded is less than or equal to window size, offsets may be
+ * larger than the total length of output decoded in order to reference the dict, even larger than
+ * window size. After output surpasses windowSize, we're limited to windowSize offsets again.
+ */
+ size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
+ size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4;
+ RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
+ /* Validate maxNbSeq is large enough for the given matchLength and minMatch */
+ RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch");
+ return 0;
+}
+
+/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
+static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0)
+{
+ U32 offBase = OFFSET_TO_OFFBASE(rawOffset);
+
+ if (!ll0 && rawOffset == rep[0]) {
+ offBase = REPCODE1_TO_OFFBASE;
+ } else if (rawOffset == rep[1]) {
+ offBase = REPCODE_TO_OFFBASE(2 - ll0);
+ } else if (rawOffset == rep[2]) {
+ offBase = REPCODE_TO_OFFBASE(3 - ll0);
+ } else if (ll0 && rawOffset == rep[0] - 1) {
+ offBase = REPCODE3_TO_OFFBASE;
+ }
+ return offBase;
+}
+
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+ ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize,
+ ZSTD_paramSwitch_e externalRepSearch)
+{
+ U32 idx = seqPos->idx;
+ U32 const startIdx = idx;
+ BYTE const* ip = (BYTE const*)(src);
+ const BYTE* const iend = ip + blockSize;
+ repcodes_t updatedRepcodes;
+ U32 dictSize;
+
+ DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize);
+
+ if (cctx->cdict) {
+ dictSize = (U32)cctx->cdict->dictContentSize;
+ } else if (cctx->prefixDict.dict) {
+ dictSize = (U32)cctx->prefixDict.dictSize;
+ } else {
+ dictSize = 0;
+ }
+ ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
+ U32 const litLength = inSeqs[idx].litLength;
+ U32 const matchLength = inSeqs[idx].matchLength;
+ U32 offBase;
+
+ if (externalRepSearch == ZSTD_ps_disable) {
+ offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset);
+ } else {
+ U32 const ll0 = (litLength == 0);
+ offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
+ ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+ }
+
+ DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
+ if (cctx->appliedParams.validateSequences) {
+ seqPos->posInSrc += litLength + matchLength;
+ FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+ cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
+ "Sequence validation failed");
+ }
+ RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
+ "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
+ ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
+ ip += matchLength + litLength;
+ }
+
+ /* If we skipped repcode search while parsing, we need to update repcodes now */
+ assert(externalRepSearch != ZSTD_ps_auto);
+ assert(idx >= startIdx);
+ if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
+ U32* const rep = updatedRepcodes.rep;
+ U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */
+
+ if (lastSeqIdx >= startIdx + 2) {
+ rep[2] = inSeqs[lastSeqIdx - 2].offset;
+ rep[1] = inSeqs[lastSeqIdx - 1].offset;
+ rep[0] = inSeqs[lastSeqIdx].offset;
+ } else if (lastSeqIdx == startIdx + 1) {
+ rep[2] = rep[0];
+ rep[1] = inSeqs[lastSeqIdx - 1].offset;
+ rep[0] = inSeqs[lastSeqIdx].offset;
+ } else {
+ assert(lastSeqIdx == startIdx);
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = inSeqs[lastSeqIdx].offset;
+ }
+ }
+
+ ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
+
+ if (inSeqs[idx].litLength) {
+ DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength);
+ ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength);
+ ip += inSeqs[idx].litLength;
+ seqPos->posInSrc += inSeqs[idx].litLength;
+ }
+ RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
+ seqPos->idx = idx+1;
+ return 0;
+}
+
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch)
+{
+ U32 idx = seqPos->idx;
+ U32 startPosInSequence = seqPos->posInSequence;
+ U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize;
+ size_t dictSize;
+ BYTE const* ip = (BYTE const*)(src);
+ BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */
+ repcodes_t updatedRepcodes;
+ U32 bytesAdjustment = 0;
+ U32 finalMatchSplit = 0;
+
+ /* TODO(embg) support fast parsing mode in noBlockDelim mode */
+ (void)externalRepSearch;
+
+ if (cctx->cdict) {
+ dictSize = cctx->cdict->dictContentSize;
+ } else if (cctx->prefixDict.dict) {
+ dictSize = cctx->prefixDict.dictSize;
+ } else {
+ dictSize = 0;
+ }
+ DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
+ DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
+ ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
+ const ZSTD_Sequence currSeq = inSeqs[idx];
+ U32 litLength = currSeq.litLength;
+ U32 matchLength = currSeq.matchLength;
+ U32 const rawOffset = currSeq.offset;
+ U32 offBase;
+
+ /* Modify the sequence depending on where endPosInSequence lies */
+ if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
+ if (startPosInSequence >= litLength) {
+ startPosInSequence -= litLength;
+ litLength = 0;
+ matchLength -= startPosInSequence;
+ } else {
+ litLength -= startPosInSequence;
+ }
+ /* Move to the next sequence */
+ endPosInSequence -= currSeq.litLength + currSeq.matchLength;
+ startPosInSequence = 0;
+ } else {
+ /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
+ does not reach the end of the match. So, we have to split the sequence */
+ DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u",
+ currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence);
+ if (endPosInSequence > litLength) {
+ U32 firstHalfMatchLength;
+ litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence;
+ firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength;
+ if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) {
+ /* Only ever split the match if it is larger than the block size */
+ U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence;
+ if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) {
+ /* Move the endPosInSequence backward so that it creates match of minMatch length */
+ endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
+ bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
+ firstHalfMatchLength -= bytesAdjustment;
+ }
+ matchLength = firstHalfMatchLength;
+ /* Flag that we split the last match - after storing the sequence, exit the loop,
+ but keep the value of endPosInSequence */
+ finalMatchSplit = 1;
+ } else {
+ /* Move the position in sequence backwards so that we don't split match, and break to store
+ * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence
+ * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so
+ * would cause the first half of the match to be too small
+ */
+ bytesAdjustment = endPosInSequence - currSeq.litLength;
+ endPosInSequence = currSeq.litLength;
+ break;
+ }
+ } else {
+ /* This sequence ends inside the literals, break to store the last literals */
+ break;
+ }
+ }
+ /* Check if this offset can be represented with a repcode */
+ { U32 const ll0 = (litLength == 0);
+ offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0);
+ ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+ }
+
+ if (cctx->appliedParams.validateSequences) {
+ seqPos->posInSrc += litLength + matchLength;
+ FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+ cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
+ "Sequence validation failed");
+ }
+ DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
+ RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
+ "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
+ ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
+ ip += matchLength + litLength;
+ if (!finalMatchSplit)
+ idx++; /* Next Sequence */
+ }
+ DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
+ assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
+ seqPos->idx = idx;
+ seqPos->posInSequence = endPosInSequence;
+ ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
+
+ iend -= bytesAdjustment;
+ if (ip != iend) {
+ /* Store any last literals */
+ U32 lastLLSize = (U32)(iend - ip);
+ assert(ip <= iend);
+ DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize);
+ ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize);
+ seqPos->posInSrc += lastLLSize;
+ }
+
+ return bytesAdjustment;
+}
+
+typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
+{
+ ZSTD_sequenceCopier sequenceCopier = NULL;
+ assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode));
+ if (mode == ZSTD_sf_explicitBlockDelimiters) {
+ return ZSTD_copySequencesToSeqStoreExplicitBlockDelim;
+ } else if (mode == ZSTD_sf_noBlockDelimiters) {
+ return ZSTD_copySequencesToSeqStoreNoBlockDelim;
+ }
+ assert(sequenceCopier != NULL);
+ return sequenceCopier;
+}
+
+/* Discover the size of next block by searching for the delimiter.
+ * Note that a block delimiter **must** exist in this mode,
+ * otherwise it's an input error.
+ * The block size retrieved will be later compared to ensure it remains within bounds */
+static size_t
+blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+ int end = 0;
+ size_t blockSize = 0;
+ size_t spos = seqPos.idx;
+ DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize);
+ assert(spos <= inSeqsSize);
+ while (spos < inSeqsSize) {
+ end = (inSeqs[spos].offset == 0);
+ blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength;
+ if (end) {
+ if (inSeqs[spos].matchLength != 0)
+ RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0");
+ break;
+ }
+ spos++;
+ }
+ if (!end)
+ RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter");
+ return blockSize;
+}
+
+/* More a "target" block size */
+static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining)
+{
+ int const lastBlock = (remaining <= blockSize);
+ return lastBlock ? remaining : blockSize;
+}
+
+static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
+ size_t blockSize, size_t remaining,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+ DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining);
+ if (mode == ZSTD_sf_noBlockDelimiters)
+ return blockSize_noDelimiter(blockSize, remaining);
+ { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos);
+ FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters");
+ if (explicitBlockSize > blockSize)
+ RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block");
+ if (explicitBlockSize > remaining)
+ RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source");
+ return explicitBlockSize;
+ }
+}
+
+/* Compress, block-by-block, all of the sequences given.
+ *
+ * Returns the cumulative size of all compressed blocks (including their headers),
+ * otherwise a ZSTD error.
+ */
+static size_t
+ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ const void* src, size_t srcSize)
+{
+ size_t cSize = 0;
+ size_t remaining = srcSize;
+ ZSTD_sequencePosition seqPos = {0, 0, 0};
+
+ BYTE const* ip = (BYTE const*)src;
+ BYTE* op = (BYTE*)dst;
+ ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters);
+
+ DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize);
+ /* Special case: empty frame */
+ if (remaining == 0) {
+ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1);
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header");
+ MEM_writeLE32(op, cBlockHeader24);
+ op += ZSTD_blockHeaderSize;
+ dstCapacity -= ZSTD_blockHeaderSize;
+ cSize += ZSTD_blockHeaderSize;
+ }
+
+ while (remaining) {
+ size_t compressedSeqsSize;
+ size_t cBlockSize;
+ size_t additionalByteAdjustment;
+ size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters,
+ cctx->blockSize, remaining,
+ inSeqs, inSeqsSize, seqPos);
+ U32 const lastBlock = (blockSize == remaining);
+ FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
+ assert(blockSize <= remaining);
+ ZSTD_resetSeqStore(&cctx->seqStore);
+ DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize);
+
+ additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes);
+ FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");
+ blockSize -= additionalByteAdjustment;
+
+ /* If blocks are too small, emit as a nocompress block */
+ /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+ * additional 1. We need to revisit and change this logic to be more consistent */
+ if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
+ cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
+ DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize);
+ cSize += cBlockSize;
+ ip += blockSize;
+ op += cBlockSize;
+ remaining -= blockSize;
+ dstCapacity -= cBlockSize;
+ continue;
+ }
+
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block");
+ compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore,
+ &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
+ &cctx->appliedParams,
+ op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize,
+ blockSize,
+ cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
+ cctx->bmi2);
+ FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed");
+ DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize);
+
+ if (!cctx->isFirstBlock &&
+ ZSTD_maybeRLE(&cctx->seqStore) &&
+ ZSTD_isRLE(ip, blockSize)) {
+ /* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ compressedSeqsSize = 1;
+ }
+
+ if (compressedSeqsSize == 0) {
+ /* ZSTD_noCompressBlock writes the block header as well */
+ cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed");
+ DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize);
+ } else if (compressedSeqsSize == 1) {
+ cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed");
+ DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize);
+ } else {
+ U32 cBlockHeader;
+ /* Error checking and repcodes update */
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState);
+ if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ /* Write block header into beginning of block*/
+ cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3);
+ MEM_writeLE24(op, cBlockHeader);
+ cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
+ DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize);
+ }
+
+ cSize += cBlockSize;
+
+ if (lastBlock) {
+ break;
+ } else {
+ ip += blockSize;
+ op += cBlockSize;
+ remaining -= blockSize;
+ dstCapacity -= cBlockSize;
+ cctx->isFirstBlock = 0;
+ }
+ DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity);
+ }
+
+ DEBUGLOG(4, "cSize final total: %zu", cSize);
+ return cSize;
+}
+
+size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ const void* src, size_t srcSize)
+{
+ BYTE* op = (BYTE*)dst;
+ size_t cSize = 0;
+ size_t compressedBlocksSize = 0;
+ size_t frameHeaderSize = 0;
+
+ /* Transparent initialization stage, same as compressStream2() */
+ DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity);
+ assert(cctx != NULL);
+ FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed");
+ /* Begin writing output, starting with frame header */
+ frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID);
+ op += frameHeaderSize;
+ dstCapacity -= frameHeaderSize;
+ cSize += frameHeaderSize;
+ if (cctx->appliedParams.fParams.checksumFlag && srcSize) {
+ XXH64_update(&cctx->xxhState, src, srcSize);
+ }
+ /* cSize includes block header size and compressed sequences size */
+ compressedBlocksSize = ZSTD_compressSequences_internal(cctx,
+ op, dstCapacity,
+ inSeqs, inSeqsSize,
+ src, srcSize);
+ FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!");
+ cSize += compressedBlocksSize;
+ dstCapacity -= compressedBlocksSize;
+
+ if (cctx->appliedParams.fParams.checksumFlag) {
+ U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum");
+ DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum);
+ MEM_writeLE32((char*)dst + cSize, checksum);
+ cSize += 4;
+ }
+
+ DEBUGLOG(4, "Final compressed size: %zu", cSize);
+ return cSize;
+}
+
+/*====== Finalize ======*/
+
+static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs)
+{
+ const ZSTD_inBuffer nullInput = { NULL, 0, 0 };
+ const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
+ return stableInput ? zcs->expectedInBuffer : nullInput;
+}
+
+/*! ZSTD_flushStream() :
+ * @return : amount of data remaining to flush */
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
+{
+ ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
+ input.size = input.pos; /* do not ingest more input during flush */
+ return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
+}
+
+
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
+{
+ ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
+ size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
+ FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed");
+ if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */
+ /* single thread mode : attempt to calculate remaining to flush more precisely */
+ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
+ size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4);
+ size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize;
+ DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush);
+ return toFlush;
+ }
+}
+
+
+/*-===== Pre-defined compression levels =====-*/
+#include "clevels.h"
+
+int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
+int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; }
+int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; }
+
+static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize)
+{
+ ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict);
+ switch (cParams.strategy) {
+ case ZSTD_fast:
+ case ZSTD_dfast:
+ break;
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG;
+ break;
+ case ZSTD_btlazy2:
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ break;
+ }
+ return cParams;
+}
+
+static int ZSTD_dedicatedDictSearch_isSupported(
+ ZSTD_compressionParameters const* cParams)
+{
+ return (cParams->strategy >= ZSTD_greedy)
+ && (cParams->strategy <= ZSTD_lazy2)
+ && (cParams->hashLog > cParams->chainLog)
+ && (cParams->chainLog <= 24);
+}
+
+/**
+ * Reverses the adjustment applied to cparams when enabling dedicated dict
+ * search. This is used to recover the params set to be used in the working
+ * context. (Otherwise, those tables would also grow.)
+ */
+static void ZSTD_dedicatedDictSearch_revertCParams(
+ ZSTD_compressionParameters* cParams) {
+ switch (cParams->strategy) {
+ case ZSTD_fast:
+ case ZSTD_dfast:
+ break;
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG;
+ if (cParams->hashLog < ZSTD_HASHLOG_MIN) {
+ cParams->hashLog = ZSTD_HASHLOG_MIN;
+ }
+ break;
+ case ZSTD_btlazy2:
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ break;
+ }
+}
+
+static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
+{
+ switch (mode) {
+ case ZSTD_cpm_unknown:
+ case ZSTD_cpm_noAttachDict:
+ case ZSTD_cpm_createCDict:
+ break;
+ case ZSTD_cpm_attachDict:
+ dictSize = 0;
+ break;
+ default:
+ assert(0);
+ break;
+ }
+ { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN;
+ size_t const addedSize = unknown && dictSize > 0 ? 500 : 0;
+ return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize;
+ }
+}
+
+/*! ZSTD_getCParams_internal() :
+ * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize.
+ * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown.
+ * Use dictSize == 0 for unknown or unused.
+ * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */
+static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
+{
+ U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode);
+ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);
+ int row;
+ DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel);
+
+ /* row */
+ if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */
+ else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */
+ else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL;
+ else row = compressionLevel;
+
+ { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];
+ DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy);
+ /* acceleration factor */
+ if (compressionLevel < 0) {
+ int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel);
+ cp.targetLength = (unsigned)(-clampedCompressionLevel);
+ }
+ /* refine parameters based on srcSize & dictSize */
+ return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto);
+ }
+}
+
+/*! ZSTD_getCParams() :
+ * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize.
+ * Size values are optional, provide 0 if not known or unused */
+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
+{
+ if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
+ return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
+}
+
+/*! ZSTD_getParams() :
+ * same idea as ZSTD_getCParams()
+ * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`).
+ * Fields of `ZSTD_frameParameters` are set to default values */
+static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) {
+ ZSTD_parameters params;
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode);
+ DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel);
+ ZSTD_memset(&params, 0, sizeof(params));
+ params.cParams = cParams;
+ params.fParams.contentSizeFlag = 1;
+ return params;
+}
+
+/*! ZSTD_getParams() :
+ * same idea as ZSTD_getCParams()
+ * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`).
+ * Fields of `ZSTD_frameParameters` are set to default values */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) {
+ if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
+ return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
+}
+
+void ZSTD_registerSequenceProducer(
+ ZSTD_CCtx* zc, void* mState,
+ ZSTD_sequenceProducer_F* mFinder
+) {
+ if (mFinder != NULL) {
+ ZSTD_externalMatchCtx emctx;
+ emctx.mState = mState;
+ emctx.mFinder = mFinder;
+ emctx.seqBuffer = NULL;
+ emctx.seqBufferCapacity = 0;
+ zc->externalMatchCtx = emctx;
+ zc->requestedParams.useSequenceProducer = 1;
+ } else {
+ ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+ zc->requestedParams.useSequenceProducer = 0;
+ }
+}
diff --git a/contrib/zstd/zstd_compress.h b/contrib/zstd/zstd_compress.h
new file mode 100644
index 0000000..94606ed
--- /dev/null
+++ b/contrib/zstd/zstd_compress.h
@@ -0,0 +1,307 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_COMPRESS_H
+#define ZSTD_COMPRESS_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_internal.h"
+#ifdef ZSTD_MULTITHREAD
+# include "zstdmt_compress.h"
+#endif
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+* Constants
+***************************************/
+static const U32 g_searchStrength = 8;
+#define HASH_READ_SIZE 8
+
+
+/*-*************************************
+* Context memory management
+***************************************/
+typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
+typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
+
+typedef struct ZSTD_prefixDict_s {
+ const void* dict;
+ size_t dictSize;
+ ZSTD_dictMode_e dictMode;
+} ZSTD_prefixDict;
+
+struct ZSTD_CCtx_s {
+ const BYTE* nextSrc; /* next block here to continue on current prefix */
+ const BYTE* base; /* All regular indexes relative to this position */
+ const BYTE* dictBase; /* extDict indexes relative to this position */
+ U32 dictLimit; /* below that point, need extDict */
+ U32 lowLimit; /* below that point, no more data */
+ U32 nextToUpdate; /* index from which to continue dictionary update */
+ U32 nextToUpdate3; /* index from which to continue dictionary update */
+ U32 hashLog3; /* dispatch table : larger == faster, more memory */
+ U32 loadedDictEnd; /* index of end of dictionary */
+ ZSTD_compressionStage_e stage;
+ U32 dictID;
+ ZSTD_CCtx_params requestedParams;
+ ZSTD_CCtx_params appliedParams;
+ void* workSpace;
+ size_t workSpaceSize;
+ size_t blockSize;
+ U64 pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
+ U64 consumedSrcSize;
+ XXH64_state_t xxhState;
+ ZSTD_customMem customMem;
+ size_t staticSize;
+
+ seqStore_t seqStore; /* sequences storage ptrs */
+ optState_t optState;
+ ldmState_t ldmState; /* long distance matching state */
+ U32* hashTable;
+ U32* hashTable3;
+ U32* chainTable;
+ ZSTD_entropyCTables_t* entropy;
+
+ /* streaming */
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inToCompress;
+ size_t inBuffPos;
+ size_t inBuffTarget;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outBuffContentSize;
+ size_t outBuffFlushedSize;
+ ZSTD_cStreamStage streamStage;
+ U32 frameEnded;
+
+ /* Dictionary */
+ ZSTD_CDict* cdictLocal;
+ const ZSTD_CDict* cdict;
+ ZSTD_prefixDict prefixDict; /* single-usage dictionary */
+
+ /* Multi-threading */
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_CCtx* mtctx;
+#endif
+};
+
+
+static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 16, 17, 17, 18, 18, 19, 19,
+ 20, 20, 20, 20, 21, 21, 21, 21,
+ 22, 22, 22, 22, 22, 22, 22, 22,
+ 23, 23, 23, 23, 23, 23, 23, 23,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24 };
+
+static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+ 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
+ 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
+
+/*! ZSTD_storeSeq() :
+ Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
+ `offsetCode` : distance to match, or 0 == repCode.
+ `matchCode` : matchLength - MINMATCH
+*/
+MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
+{
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
+ static const BYTE* g_start = NULL;
+ U32 const pos = (U32)((const BYTE*)literals - g_start);
+ if (g_start==NULL) g_start = (const BYTE*)literals;
+ if ((pos > 0) && (pos < 1000000000))
+ DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
+ pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+#endif
+ /* copy Literals */
+ assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
+ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
+ seqStorePtr->lit += litLength;
+
+ /* literal Length */
+ if (litLength>0xFFFF) {
+ seqStorePtr->longLengthID = 1;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].litLength = (U16)litLength;
+
+ /* match offset */
+ seqStorePtr->sequences[0].offset = offsetCode + 1;
+
+ /* match Length */
+ if (matchCode>0xFFFF) {
+ seqStorePtr->longLengthID = 2;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].matchLength = (U16)matchCode;
+
+ seqStorePtr->sequences++;
+}
+
+
+/*-*************************************
+* Match length counter
+***************************************/
+static unsigned ZSTD_NbCommonBytes (register size_t val)
+{
+ if (MEM_isLittleEndian()) {
+ if (MEM_64bits()) {
+# if defined(_MSC_VER) && defined(_WIN64)
+ unsigned long r = 0;
+ _BitScanForward64( &r, (U64)val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (__builtin_ctzll((U64)val) >> 3);
+# else
+ static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+ 0, 3, 1, 3, 1, 4, 2, 7,
+ 0, 2, 3, 6, 1, 5, 3, 5,
+ 1, 3, 4, 4, 2, 5, 6, 7,
+ 7, 0, 1, 2, 3, 3, 4, 6,
+ 2, 6, 5, 5, 3, 4, 5, 6,
+ 7, 1, 2, 4, 6, 4, 4, 5,
+ 7, 2, 6, 5, 7, 6, 7, 7 };
+ return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+# endif
+ } else { /* 32 bits */
+# if defined(_MSC_VER)
+ unsigned long r=0;
+ _BitScanForward( &r, (U32)val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3)
+ return (__builtin_ctz((U32)val) >> 3);
+# else
+ static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+ 3, 2, 2, 1, 3, 2, 0, 1,
+ 3, 3, 1, 2, 2, 2, 2, 0,
+ 3, 1, 2, 0, 1, 0, 1, 1 };
+ return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+# endif
+ }
+ } else { /* Big Endian CPU */
+ if (MEM_64bits()) {
+# if defined(_MSC_VER) && defined(_WIN64)
+ unsigned long r = 0;
+ _BitScanReverse64( &r, val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (__builtin_clzll(val) >> 3);
+# else
+ unsigned r;
+ const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
+ if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+ if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+ r += (!val);
+ return r;
+# endif
+ } else { /* 32 bits */
+# if defined(_MSC_VER)
+ unsigned long r = 0;
+ _BitScanReverse( &r, (unsigned long)val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3)
+ return (__builtin_clz((U32)val) >> 3);
+# else
+ unsigned r;
+ if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+ r += (!val);
+ return r;
+# endif
+ } }
+}
+
+
+MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
+{
+ const BYTE* const pStart = pIn;
+ const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
+
+ while (pIn < pInLoopLimit) {
+ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+ if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
+ pIn += ZSTD_NbCommonBytes(diff);
+ return (size_t)(pIn - pStart);
+ }
+ if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+ if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
+ if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+ return (size_t)(pIn - pStart);
+}
+
+/** ZSTD_count_2segments() :
+* can count match length with `ip` & `match` in 2 different segments.
+* convention : on reaching mEnd, match count continue starting from iStart
+*/
+MEM_STATIC size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
+{
+ const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
+ size_t const matchLength = ZSTD_count(ip, match, vEnd);
+ if (match + matchLength != mEnd) return matchLength;
+ return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
+}
+
+
+/*-*************************************
+* Hashes
+***************************************/
+static const U32 prime3bytes = 506832829U;
+static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
+MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
+
+static const U32 prime4bytes = 2654435761U;
+static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
+static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); }
+
+static const U64 prime5bytes = 889523592379ULL;
+static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; }
+static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
+
+static const U64 prime6bytes = 227718039650203ULL;
+static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
+static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
+
+static const U64 prime7bytes = 58295818150454627ULL;
+static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; }
+static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
+
+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
+static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
+static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
+
+MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
+{
+ switch(mls)
+ {
+ default:
+ case 4: return ZSTD_hash4Ptr(p, hBits);
+ case 5: return ZSTD_hash5Ptr(p, hBits);
+ case 6: return ZSTD_hash6Ptr(p, hBits);
+ case 7: return ZSTD_hash7Ptr(p, hBits);
+ case 8: return ZSTD_hash8Ptr(p, hBits);
+ }
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_COMPRESS_H */
diff --git a/contrib/zstd/zstd_compress_internal.h b/contrib/zstd/zstd_compress_internal.h
new file mode 100644
index 0000000..e53b3a5
--- /dev/null
+++ b/contrib/zstd/zstd_compress_internal.h
@@ -0,0 +1,1478 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This header contains definitions
+ * that shall **only** be used by modules within lib/compress.
+ */
+
+#ifndef ZSTD_COMPRESS_H
+#define ZSTD_COMPRESS_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_internal.h"
+#include "zstd_cwksp.h"
+#ifdef ZSTD_MULTITHREAD
+# include "zstdmt_compress.h"
+#endif
+#include "bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+* Constants
+***************************************/
+#define kSearchStrength 8
+#define HASH_READ_SIZE 8
+#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted".
+ It could be confused for a real successor at index "1", if sorted as larger than its predecessor.
+ It's not a big deal though : candidate will just be sorted again.
+ Additionally, candidate position 1 will be lost.
+ But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss.
+ The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy.
+ This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
+
+
+/*-*************************************
+* Context memory management
+***************************************/
+typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
+typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
+
+typedef struct ZSTD_prefixDict_s {
+ const void* dict;
+ size_t dictSize;
+ ZSTD_dictContentType_e dictContentType;
+} ZSTD_prefixDict;
+
+typedef struct {
+ void* dictBuffer;
+ void const* dict;
+ size_t dictSize;
+ ZSTD_dictContentType_e dictContentType;
+ ZSTD_CDict* cdict;
+} ZSTD_localDict;
+
+typedef struct {
+ HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)];
+ HUF_repeat repeatMode;
+} ZSTD_hufCTables_t;
+
+typedef struct {
+ FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+ FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+ FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+ FSE_repeat offcode_repeatMode;
+ FSE_repeat matchlength_repeatMode;
+ FSE_repeat litlength_repeatMode;
+} ZSTD_fseCTables_t;
+
+typedef struct {
+ ZSTD_hufCTables_t huf;
+ ZSTD_fseCTables_t fse;
+} ZSTD_entropyCTables_t;
+
+/***********************************************
+* Entropy buffer statistics structs and funcs *
+***********************************************/
+/** ZSTD_hufCTablesMetadata_t :
+ * Stores Literals Block Type for a super-block in hType, and
+ * huffman tree description in hufDesBuffer.
+ * hufDesSize refers to the size of huffman tree description in bytes.
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
+typedef struct {
+ symbolEncodingType_e hType;
+ BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
+ size_t hufDesSize;
+} ZSTD_hufCTablesMetadata_t;
+
+/** ZSTD_fseCTablesMetadata_t :
+ * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
+ * fse tables in fseTablesBuffer.
+ * fseTablesSize refers to the size of fse tables in bytes.
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
+typedef struct {
+ symbolEncodingType_e llType;
+ symbolEncodingType_e ofType;
+ symbolEncodingType_e mlType;
+ BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
+ size_t fseTablesSize;
+ size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
+} ZSTD_fseCTablesMetadata_t;
+
+typedef struct {
+ ZSTD_hufCTablesMetadata_t hufMetadata;
+ ZSTD_fseCTablesMetadata_t fseMetadata;
+} ZSTD_entropyCTablesMetadata_t;
+
+/** ZSTD_buildBlockEntropyStats() :
+ * Builds entropy for the block.
+ * @return : 0 on success or error code */
+size_t ZSTD_buildBlockEntropyStats(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize);
+
+/*********************************
+* Compression internals structs *
+*********************************/
+
+typedef struct {
+ U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */
+ U32 len; /* Raw length of match */
+} ZSTD_match_t;
+
+typedef struct {
+ U32 offset; /* Offset of sequence */
+ U32 litLength; /* Length of literals prior to match */
+ U32 matchLength; /* Raw length of match */
+} rawSeq;
+
+typedef struct {
+ rawSeq* seq; /* The start of the sequences */
+ size_t pos; /* The index in seq where reading stopped. pos <= size. */
+ size_t posInSequence; /* The position within the sequence at seq[pos] where reading
+ stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */
+ size_t size; /* The number of sequences. <= capacity. */
+ size_t capacity; /* The capacity starting from `seq` pointer */
+} rawSeqStore_t;
+
+typedef struct {
+ U32 idx; /* Index in array of ZSTD_Sequence */
+ U32 posInSequence; /* Position within sequence at idx */
+ size_t posInSrc; /* Number of bytes given by sequences provided so far */
+} ZSTD_sequencePosition;
+
+UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
+
+typedef struct {
+ int price;
+ U32 off;
+ U32 mlen;
+ U32 litlen;
+ U32 rep[ZSTD_REP_NUM];
+} ZSTD_optimal_t;
+
+typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
+
+typedef struct {
+ /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
+ unsigned* litFreq; /* table of literals statistics, of size 256 */
+ unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
+ unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
+ unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
+ ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
+ ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
+
+ U32 litSum; /* nb of literals */
+ U32 litLengthSum; /* nb of litLength codes */
+ U32 matchLengthSum; /* nb of matchLength codes */
+ U32 offCodeSum; /* nb of offset codes */
+ U32 litSumBasePrice; /* to compare to log2(litfreq) */
+ U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */
+ U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */
+ U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
+ ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
+ const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
+ ZSTD_paramSwitch_e literalCompressionMode;
+} optState_t;
+
+typedef struct {
+ ZSTD_entropyCTables_t entropy;
+ U32 rep[ZSTD_REP_NUM];
+} ZSTD_compressedBlockState_t;
+
+typedef struct {
+ BYTE const* nextSrc; /* next block here to continue on current prefix */
+ BYTE const* base; /* All regular indexes relative to this position */
+ BYTE const* dictBase; /* extDict indexes relative to this position */
+ U32 dictLimit; /* below that point, need extDict */
+ U32 lowLimit; /* below that point, no more valid data */
+ U32 nbOverflowCorrections; /* Number of times overflow correction has run since
+ * ZSTD_window_init(). Useful for debugging coredumps
+ * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY.
+ */
+} ZSTD_window_t;
+
+#define ZSTD_WINDOW_START_INDEX 2
+
+typedef struct ZSTD_matchState_t ZSTD_matchState_t;
+
+#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
+
+struct ZSTD_matchState_t {
+ ZSTD_window_t window; /* State for window round buffer management */
+ U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
+ * When loadedDictEnd != 0, a dictionary is in use, and still valid.
+ * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance.
+ * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity().
+ * When dict referential is copied into active context (i.e. not attached),
+ * loadedDictEnd == dictSize, since referential starts from zero.
+ */
+ U32 nextToUpdate; /* index from which to continue table update */
+ U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
+
+ U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
+ U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
+ U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
+
+ U32* hashTable;
+ U32* hashTable3;
+ U32* chainTable;
+
+ U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
+
+ int dedicatedDictSearch; /* Indicates whether this matchState is using the
+ * dedicated dictionary search structure.
+ */
+ optState_t opt; /* optimal parser state */
+ const ZSTD_matchState_t* dictMatchState;
+ ZSTD_compressionParameters cParams;
+ const rawSeqStore_t* ldmSeqStore;
+
+ /* Controls prefetching in some dictMatchState matchfinders.
+ * This behavior is controlled from the cctx ms.
+ * This parameter has no effect in the cdict ms. */
+ int prefetchCDictTables;
+};
+
+typedef struct {
+ ZSTD_compressedBlockState_t* prevCBlock;
+ ZSTD_compressedBlockState_t* nextCBlock;
+ ZSTD_matchState_t matchState;
+} ZSTD_blockState_t;
+
+typedef struct {
+ U32 offset;
+ U32 checksum;
+} ldmEntry_t;
+
+typedef struct {
+ BYTE const* split;
+ U32 hash;
+ U32 checksum;
+ ldmEntry_t* bucket;
+} ldmMatchCandidate_t;
+
+#define LDM_BATCH_SIZE 64
+
+typedef struct {
+ ZSTD_window_t window; /* State for the window round buffer management */
+ ldmEntry_t* hashTable;
+ U32 loadedDictEnd;
+ BYTE* bucketOffsets; /* Next position in bucket to insert entry */
+ size_t splitIndices[LDM_BATCH_SIZE];
+ ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE];
+} ldmState_t;
+
+typedef struct {
+ ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
+ U32 hashLog; /* Log size of hashTable */
+ U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
+ U32 minMatchLength; /* Minimum match length */
+ U32 hashRateLog; /* Log number of entries to skip */
+ U32 windowLog; /* Window log for the LDM */
+} ldmParams_t;
+
+typedef struct {
+ int collectSequences;
+ ZSTD_Sequence* seqStart;
+ size_t seqIndex;
+ size_t maxSequences;
+} SeqCollector;
+
+struct ZSTD_CCtx_params_s {
+ ZSTD_format_e format;
+ ZSTD_compressionParameters cParams;
+ ZSTD_frameParameters fParams;
+
+ int compressionLevel;
+ int forceWindow; /* force back-references to respect limit of
+ * 1<<wLog, even for dictionary */
+ size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize.
+ * No target when targetCBlockSize == 0.
+ * There is no guarantee on compressed block size */
+ int srcSizeHint; /* User's best guess of source size.
+ * Hint is not valid when srcSizeHint == 0.
+ * There is no guarantee that hint is close to actual source size */
+
+ ZSTD_dictAttachPref_e attachDictPref;
+ ZSTD_paramSwitch_e literalCompressionMode;
+
+ /* Multithreading: used to pass parameters to mtctx */
+ int nbWorkers;
+ size_t jobSize;
+ int overlapLog;
+ int rsyncable;
+
+ /* Long distance matching parameters */
+ ldmParams_t ldmParams;
+
+ /* Dedicated dict search algorithm trigger */
+ int enableDedicatedDictSearch;
+
+ /* Input/output buffer modes */
+ ZSTD_bufferMode_e inBufferMode;
+ ZSTD_bufferMode_e outBufferMode;
+
+ /* Sequence compression API */
+ ZSTD_sequenceFormat_e blockDelimiters;
+ int validateSequences;
+
+ /* Block splitting */
+ ZSTD_paramSwitch_e useBlockSplitter;
+
+ /* Param for deciding whether to use row-based matchfinder */
+ ZSTD_paramSwitch_e useRowMatchFinder;
+
+ /* Always load a dictionary in ext-dict mode (not prefix mode)? */
+ int deterministicRefPrefix;
+
+ /* Internal use, for createCCtxParams() and freeCCtxParams() only */
+ ZSTD_customMem customMem;
+
+ /* Controls prefetching in some dictMatchState matchfinders */
+ ZSTD_paramSwitch_e prefetchCDictTables;
+
+ /* Controls whether zstd will fall back to an internal matchfinder
+ * if the external matchfinder returns an error code. */
+ int enableMatchFinderFallback;
+
+ /* Indicates whether an external matchfinder has been referenced.
+ * Users can't set this externally.
+ * It is set internally in ZSTD_registerSequenceProducer(). */
+ int useSequenceProducer;
+
+ /* Adjust the max block size*/
+ size_t maxBlockSize;
+
+ /* Controls repcode search in external sequence parsing */
+ ZSTD_paramSwitch_e searchForExternalRepcodes;
+}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
+
+#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
+#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
+
+/**
+ * Indicates whether this compression proceeds directly from user-provided
+ * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or
+ * whether the context needs to buffer the input/output (ZSTDb_buffered).
+ */
+typedef enum {
+ ZSTDb_not_buffered,
+ ZSTDb_buffered
+} ZSTD_buffered_policy_e;
+
+/**
+ * Struct that contains all elements of block splitter that should be allocated
+ * in a wksp.
+ */
+#define ZSTD_MAX_NB_BLOCK_SPLITS 196
+typedef struct {
+ seqStore_t fullSeqStoreChunk;
+ seqStore_t firstHalfSeqStore;
+ seqStore_t secondHalfSeqStore;
+ seqStore_t currSeqStore;
+ seqStore_t nextSeqStore;
+
+ U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
+ ZSTD_entropyCTablesMetadata_t entropyMetadata;
+} ZSTD_blockSplitCtx;
+
+/* Context for block-level external matchfinder API */
+typedef struct {
+ void* mState;
+ ZSTD_sequenceProducer_F* mFinder;
+ ZSTD_Sequence* seqBuffer;
+ size_t seqBufferCapacity;
+} ZSTD_externalMatchCtx;
+
+struct ZSTD_CCtx_s {
+ ZSTD_compressionStage_e stage;
+ int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
+ int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
+ ZSTD_CCtx_params requestedParams;
+ ZSTD_CCtx_params appliedParams;
+ ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */
+ U32 dictID;
+ size_t dictContentSize;
+
+ ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
+ size_t blockSize;
+ unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
+ unsigned long long consumedSrcSize;
+ unsigned long long producedCSize;
+ XXH64_state_t xxhState;
+ ZSTD_customMem customMem;
+ ZSTD_threadPool* pool;
+ size_t staticSize;
+ SeqCollector seqCollector;
+ int isFirstBlock;
+ int initialized;
+
+ seqStore_t seqStore; /* sequences storage ptrs */
+ ldmState_t ldmState; /* long distance matching state */
+ rawSeq* ldmSequences; /* Storage for the ldm output sequences */
+ size_t maxNbLdmSequences;
+ rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
+ ZSTD_blockState_t blockState;
+ U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
+
+ /* Whether we are streaming or not */
+ ZSTD_buffered_policy_e bufferedPolicy;
+
+ /* streaming */
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inToCompress;
+ size_t inBuffPos;
+ size_t inBuffTarget;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outBuffContentSize;
+ size_t outBuffFlushedSize;
+ ZSTD_cStreamStage streamStage;
+ U32 frameEnded;
+
+ /* Stable in/out buffer verification */
+ ZSTD_inBuffer expectedInBuffer;
+ size_t stableIn_notConsumed; /* nb bytes within stable input buffer that are said to be consumed but are not */
+ size_t expectedOutBufferSize;
+
+ /* Dictionary */
+ ZSTD_localDict localDict;
+ const ZSTD_CDict* cdict;
+ ZSTD_prefixDict prefixDict; /* single-usage dictionary */
+
+ /* Multi-threading */
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_CCtx* mtctx;
+#endif
+
+ /* Tracing */
+#if ZSTD_TRACE
+ ZSTD_TraceCtx traceCtx;
+#endif
+
+ /* Workspace for block splitter */
+ ZSTD_blockSplitCtx blockSplitCtx;
+
+ /* Workspace for external matchfinder */
+ ZSTD_externalMatchCtx externalMatchCtx;
+};
+
+typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
+typedef enum { ZSTD_tfp_forCCtx, ZSTD_tfp_forCDict } ZSTD_tableFillPurpose_e;
+
+typedef enum {
+ ZSTD_noDict = 0,
+ ZSTD_extDict = 1,
+ ZSTD_dictMatchState = 2,
+ ZSTD_dedicatedDictSearch = 3
+} ZSTD_dictMode_e;
+
+typedef enum {
+ ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict.
+ * In this mode we use both the srcSize and the dictSize
+ * when selecting and adjusting parameters.
+ */
+ ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch.
+ * In this mode we only take the srcSize into account when selecting
+ * and adjusting parameters.
+ */
+ ZSTD_cpm_createCDict = 2, /* Creating a CDict.
+ * In this mode we take both the source size and the dictionary size
+ * into account when selecting and adjusting the parameters.
+ */
+ ZSTD_cpm_unknown = 3 /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
+ * We don't know what these parameters are for. We default to the legacy
+ * behavior of taking both the source size and the dict size into account
+ * when selecting and adjusting parameters.
+ */
+} ZSTD_cParamMode_e;
+
+typedef size_t (*ZSTD_blockCompressor) (
+ ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
+
+
+MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
+{
+ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 16, 17, 17, 18, 18, 19, 19,
+ 20, 20, 20, 20, 21, 21, 21, 21,
+ 22, 22, 22, 22, 22, 22, 22, 22,
+ 23, 23, 23, 23, 23, 23, 23, 23,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24 };
+ static const U32 LL_deltaCode = 19;
+ return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
+}
+
+/* ZSTD_MLcode() :
+ * note : mlBase = matchLength - MINMATCH;
+ * because it's the format it's stored in seqStore->sequences */
+MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
+{
+ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+ 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
+ 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
+ static const U32 ML_deltaCode = 36;
+ return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
+}
+
+/* ZSTD_cParam_withinBounds:
+ * @return 1 if value is within cParam bounds,
+ * 0 otherwise */
+MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
+{
+ ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
+ if (ZSTD_isError(bounds.error)) return 0;
+ if (value < bounds.lowerBound) return 0;
+ if (value > bounds.upperBound) return 0;
+ return 1;
+}
+
+/* ZSTD_noCompressBlock() :
+ * Writes uncompressed block to dst buffer from given src.
+ * Returns the size of the block */
+MEM_STATIC size_t
+ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
+{
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
+ DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity);
+ RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
+ dstSize_tooSmall, "dst buf too small for uncompressed block");
+ MEM_writeLE24(dst, cBlockHeader24);
+ ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
+ return ZSTD_blockHeaderSize + srcSize;
+}
+
+MEM_STATIC size_t
+ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
+{
+ BYTE* const op = (BYTE*)dst;
+ U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);
+ RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, "");
+ MEM_writeLE24(op, cBlockHeader);
+ op[3] = src;
+ return 4;
+}
+
+
+/* ZSTD_minGain() :
+ * minimum compression required
+ * to generate a compress block or a compressed literals section.
+ * note : use same formula for both situations */
+MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
+{
+ U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6;
+ ZSTD_STATIC_ASSERT(ZSTD_btultra == 8);
+ assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat));
+ return (srcSize >> minlog) + 2;
+}
+
+MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams)
+{
+ switch (cctxParams->literalCompressionMode) {
+ case ZSTD_ps_enable:
+ return 0;
+ case ZSTD_ps_disable:
+ return 1;
+ default:
+ assert(0 /* impossible: pre-validated */);
+ ZSTD_FALLTHROUGH;
+ case ZSTD_ps_auto:
+ return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
+ }
+}
+
+/*! ZSTD_safecopyLiterals() :
+ * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
+ * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
+ * large copies.
+ */
+static void
+ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w)
+{
+ assert(iend > ilimit_w);
+ if (ip <= ilimit_w) {
+ ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap);
+ op += ilimit_w - ip;
+ ip = ilimit_w;
+ }
+ while (ip < iend) *op++ = *ip++;
+}
+
+
+#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1)
+#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2)
+#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3)
+#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */
+#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM)
+#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM)
+#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM)
+#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM)
+#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */
+
+/*! ZSTD_storeSeq() :
+ * Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t.
+ * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
+ * @matchLength : must be >= MINMATCH
+ * Allowed to over-read literals up to litLimit.
+*/
+HINT_INLINE UNUSED_ATTR void
+ZSTD_storeSeq(seqStore_t* seqStorePtr,
+ size_t litLength, const BYTE* literals, const BYTE* litLimit,
+ U32 offBase,
+ size_t matchLength)
+{
+ BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
+ BYTE const* const litEnd = literals + litLength;
+#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
+ static const BYTE* g_start = NULL;
+ if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
+ { U32 const pos = (U32)((const BYTE*)literals - g_start);
+ DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u",
+ pos, (U32)litLength, (U32)matchLength, (U32)offBase);
+ }
+#endif
+ assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
+ /* copy Literals */
+ assert(seqStorePtr->maxNbLit <= 128 KB);
+ assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
+ assert(literals + litLength <= litLimit);
+ if (litEnd <= litLimit_w) {
+ /* Common case we can use wildcopy.
+ * First copy 16 bytes, because literals are likely short.
+ */
+ ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16);
+ ZSTD_copy16(seqStorePtr->lit, literals);
+ if (litLength > 16) {
+ ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap);
+ }
+ } else {
+ ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w);
+ }
+ seqStorePtr->lit += litLength;
+
+ /* literal Length */
+ if (litLength>0xFFFF) {
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
+ seqStorePtr->longLengthType = ZSTD_llt_literalLength;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].litLength = (U16)litLength;
+
+ /* match offset */
+ seqStorePtr->sequences[0].offBase = offBase;
+
+ /* match Length */
+ assert(matchLength >= MINMATCH);
+ { size_t const mlBase = matchLength - MINMATCH;
+ if (mlBase>0xFFFF) {
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
+ seqStorePtr->longLengthType = ZSTD_llt_matchLength;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].mlBase = (U16)mlBase;
+ }
+
+ seqStorePtr->sequences++;
+}
+
+/* ZSTD_updateRep() :
+ * updates in-place @rep (array of repeat offsets)
+ * @offBase : sum-type, using numeric representation of ZSTD_storeSeq()
+ */
+MEM_STATIC void
+ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
+{
+ if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = OFFBASE_TO_OFFSET(offBase);
+ } else { /* repcode */
+ U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0;
+ if (repCode > 0) { /* note : if repCode==0, no change */
+ U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
+ rep[2] = (repCode >= 2) ? rep[1] : rep[2];
+ rep[1] = rep[0];
+ rep[0] = currentOffset;
+ } else { /* repCode == 0 */
+ /* nothing to do */
+ }
+ }
+}
+
+typedef struct repcodes_s {
+ U32 rep[3];
+} repcodes_t;
+
+MEM_STATIC repcodes_t
+ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
+{
+ repcodes_t newReps;
+ ZSTD_memcpy(&newReps, rep, sizeof(newReps));
+ ZSTD_updateRep(newReps.rep, offBase, ll0);
+ return newReps;
+}
+
+
+/*-*************************************
+* Match length counter
+***************************************/
+MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
+{
+ const BYTE* const pStart = pIn;
+ const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
+
+ if (pIn < pInLoopLimit) {
+ { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+ if (diff) return ZSTD_NbCommonBytes(diff); }
+ pIn+=sizeof(size_t); pMatch+=sizeof(size_t);
+ while (pIn < pInLoopLimit) {
+ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+ if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
+ pIn += ZSTD_NbCommonBytes(diff);
+ return (size_t)(pIn - pStart);
+ } }
+ if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+ if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
+ if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+ return (size_t)(pIn - pStart);
+}
+
+/** ZSTD_count_2segments() :
+ * can count match length with `ip` & `match` in 2 different segments.
+ * convention : on reaching mEnd, match count continue starting from iStart
+ */
+MEM_STATIC size_t
+ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
+ const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
+{
+ const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
+ size_t const matchLength = ZSTD_count(ip, match, vEnd);
+ if (match + matchLength != mEnd) return matchLength;
+ DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength);
+ DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match);
+ DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip);
+ DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart);
+ DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd));
+ return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
+}
+
+
+/*-*************************************
+ * Hashes
+ ***************************************/
+static const U32 prime3bytes = 506832829U;
+static U32 ZSTD_hash3(U32 u, U32 h) { assert(h <= 32); return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
+MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
+
+static const U32 prime4bytes = 2654435761U;
+static U32 ZSTD_hash4(U32 u, U32 h) { assert(h <= 32); return (u * prime4bytes) >> (32-h) ; }
+static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h); }
+
+static const U64 prime5bytes = 889523592379ULL;
+static size_t ZSTD_hash5(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; }
+static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
+
+static const U64 prime6bytes = 227718039650203ULL;
+static size_t ZSTD_hash6(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
+static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
+
+static const U64 prime7bytes = 58295818150454627ULL;
+static size_t ZSTD_hash7(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; }
+static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
+
+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
+static size_t ZSTD_hash8(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
+static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
+
+MEM_STATIC FORCE_INLINE_ATTR
+size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
+{
+ /* Although some of these hashes do support hBits up to 64, some do not.
+ * To be on the safe side, always avoid hBits > 32. */
+ assert(hBits <= 32);
+
+ switch(mls)
+ {
+ default:
+ case 4: return ZSTD_hash4Ptr(p, hBits);
+ case 5: return ZSTD_hash5Ptr(p, hBits);
+ case 6: return ZSTD_hash6Ptr(p, hBits);
+ case 7: return ZSTD_hash7Ptr(p, hBits);
+ case 8: return ZSTD_hash8Ptr(p, hBits);
+ }
+}
+
+/** ZSTD_ipow() :
+ * Return base^exponent.
+ */
+static U64 ZSTD_ipow(U64 base, U64 exponent)
+{
+ U64 power = 1;
+ while (exponent) {
+ if (exponent & 1) power *= base;
+ exponent >>= 1;
+ base *= base;
+ }
+ return power;
+}
+
+#define ZSTD_ROLL_HASH_CHAR_OFFSET 10
+
+/** ZSTD_rollingHash_append() :
+ * Add the buffer to the hash value.
+ */
+static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size)
+{
+ BYTE const* istart = (BYTE const*)buf;
+ size_t pos;
+ for (pos = 0; pos < size; ++pos) {
+ hash *= prime8bytes;
+ hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET;
+ }
+ return hash;
+}
+
+/** ZSTD_rollingHash_compute() :
+ * Compute the rolling hash value of the buffer.
+ */
+MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size)
+{
+ return ZSTD_rollingHash_append(0, buf, size);
+}
+
+/** ZSTD_rollingHash_primePower() :
+ * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash
+ * over a window of length bytes.
+ */
+MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length)
+{
+ return ZSTD_ipow(prime8bytes, length - 1);
+}
+
+/** ZSTD_rollingHash_rotate() :
+ * Rotate the rolling hash by one byte.
+ */
+MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower)
+{
+ hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower;
+ hash *= prime8bytes;
+ hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET;
+ return hash;
+}
+
+/*-*************************************
+* Round buffer management
+***************************************/
+#if (ZSTD_WINDOWLOG_MAX_64 > 31)
+# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX"
+#endif
+/* Max current allowed */
+#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX))
+/* Maximum chunk size before overflow correction needs to be called again */
+#define ZSTD_CHUNKSIZE_MAX \
+ ( ((U32)-1) /* Maximum ending current index */ \
+ - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */
+
+/**
+ * ZSTD_window_clear():
+ * Clears the window containing the history by simply setting it to empty.
+ */
+MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
+{
+ size_t const endT = (size_t)(window->nextSrc - window->base);
+ U32 const end = (U32)endT;
+
+ window->lowLimit = end;
+ window->dictLimit = end;
+}
+
+MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window)
+{
+ return window.dictLimit == ZSTD_WINDOW_START_INDEX &&
+ window.lowLimit == ZSTD_WINDOW_START_INDEX &&
+ (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX;
+}
+
+/**
+ * ZSTD_window_hasExtDict():
+ * Returns non-zero if the window has a non-empty extDict.
+ */
+MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
+{
+ return window.lowLimit < window.dictLimit;
+}
+
+/**
+ * ZSTD_matchState_dictMode():
+ * Inspects the provided matchState and figures out what dictMode should be
+ * passed to the compressor.
+ */
+MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
+{
+ return ZSTD_window_hasExtDict(ms->window) ?
+ ZSTD_extDict :
+ ms->dictMatchState != NULL ?
+ (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) :
+ ZSTD_noDict;
+}
+
+/* Defining this macro to non-zero tells zstd to run the overflow correction
+ * code much more frequently. This is very inefficient, and should only be
+ * used for tests and fuzzers.
+ */
+#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY
+# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1
+# else
+# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0
+# endif
+#endif
+
+/**
+ * ZSTD_window_canOverflowCorrect():
+ * Returns non-zero if the indices are large enough for overflow correction
+ * to work correctly without impacting compression ratio.
+ */
+MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window,
+ U32 cycleLog,
+ U32 maxDist,
+ U32 loadedDictEnd,
+ void const* src)
+{
+ U32 const cycleSize = 1u << cycleLog;
+ U32 const curr = (U32)((BYTE const*)src - window.base);
+ U32 const minIndexToOverflowCorrect = cycleSize
+ + MAX(maxDist, cycleSize)
+ + ZSTD_WINDOW_START_INDEX;
+
+ /* Adjust the min index to backoff the overflow correction frequency,
+ * so we don't waste too much CPU in overflow correction. If this
+ * computation overflows we don't really care, we just need to make
+ * sure it is at least minIndexToOverflowCorrect.
+ */
+ U32 const adjustment = window.nbOverflowCorrections + 1;
+ U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment,
+ minIndexToOverflowCorrect);
+ U32 const indexLargeEnough = curr > adjustedIndex;
+
+ /* Only overflow correct early if the dictionary is invalidated already,
+ * so we don't hurt compression ratio.
+ */
+ U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd;
+
+ return indexLargeEnough && dictionaryInvalidated;
+}
+
+/**
+ * ZSTD_window_needOverflowCorrection():
+ * Returns non-zero if the indices are getting too large and need overflow
+ * protection.
+ */
+MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
+ U32 cycleLog,
+ U32 maxDist,
+ U32 loadedDictEnd,
+ void const* src,
+ void const* srcEnd)
+{
+ U32 const curr = (U32)((BYTE const*)srcEnd - window.base);
+ if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
+ if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) {
+ return 1;
+ }
+ }
+ return curr > ZSTD_CURRENT_MAX;
+}
+
+/**
+ * ZSTD_window_correctOverflow():
+ * Reduces the indices to protect from index overflow.
+ * Returns the correction made to the indices, which must be applied to every
+ * stored index.
+ *
+ * The least significant cycleLog bits of the indices must remain the same,
+ * which may be 0. Every index up to maxDist in the past must be valid.
+ */
+MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
+ U32 maxDist, void const* src)
+{
+ /* preemptive overflow correction:
+ * 1. correction is large enough:
+ * lowLimit > (3<<29) ==> current > 3<<29 + 1<<windowLog
+ * 1<<windowLog <= newCurrent < 1<<chainLog + 1<<windowLog
+ *
+ * current - newCurrent
+ * > (3<<29 + 1<<windowLog) - (1<<windowLog + 1<<chainLog)
+ * > (3<<29) - (1<<chainLog)
+ * > (3<<29) - (1<<30) (NOTE: chainLog <= 30)
+ * > 1<<29
+ *
+ * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow:
+ * After correction, current is less than (1<<chainLog + 1<<windowLog).
+ * In 64-bit mode we are safe, because we have 64-bit ptrdiff_t.
+ * In 32-bit mode we are safe, because (chainLog <= 29), so
+ * ip+ZSTD_CHUNKSIZE_MAX - cctx->base < 1<<32.
+ * 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
+ * windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
+ */
+ U32 const cycleSize = 1u << cycleLog;
+ U32 const cycleMask = cycleSize - 1;
+ U32 const curr = (U32)((BYTE const*)src - window->base);
+ U32 const currentCycle = curr & cycleMask;
+ /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */
+ U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX
+ ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX)
+ : 0;
+ U32 const newCurrent = currentCycle
+ + currentCycleCorrection
+ + MAX(maxDist, cycleSize);
+ U32 const correction = curr - newCurrent;
+ /* maxDist must be a power of two so that:
+ * (newCurrent & cycleMask) == (curr & cycleMask)
+ * This is required to not corrupt the chains / binary tree.
+ */
+ assert((maxDist & (maxDist - 1)) == 0);
+ assert((curr & cycleMask) == (newCurrent & cycleMask));
+ assert(curr > newCurrent);
+ if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
+ /* Loose bound, should be around 1<<29 (see above) */
+ assert(correction > 1<<28);
+ }
+
+ window->base += correction;
+ window->dictBase += correction;
+ if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) {
+ window->lowLimit = ZSTD_WINDOW_START_INDEX;
+ } else {
+ window->lowLimit -= correction;
+ }
+ if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) {
+ window->dictLimit = ZSTD_WINDOW_START_INDEX;
+ } else {
+ window->dictLimit -= correction;
+ }
+
+ /* Ensure we can still reference the full window. */
+ assert(newCurrent >= maxDist);
+ assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX);
+ /* Ensure that lowLimit and dictLimit didn't underflow. */
+ assert(window->lowLimit <= newCurrent);
+ assert(window->dictLimit <= newCurrent);
+
+ ++window->nbOverflowCorrections;
+
+ DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
+ window->lowLimit);
+ return correction;
+}
+
+/**
+ * ZSTD_window_enforceMaxDist():
+ * Updates lowLimit so that:
+ * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd
+ *
+ * It ensures index is valid as long as index >= lowLimit.
+ * This must be called before a block compression call.
+ *
+ * loadedDictEnd is only defined if a dictionary is in use for current compression.
+ * As the name implies, loadedDictEnd represents the index at end of dictionary.
+ * The value lies within context's referential, it can be directly compared to blockEndIdx.
+ *
+ * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0.
+ * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit.
+ * This is because dictionaries are allowed to be referenced fully
+ * as long as the last byte of the dictionary is in the window.
+ * Once input has progressed beyond window size, dictionary cannot be referenced anymore.
+ *
+ * In normal dict mode, the dictionary lies between lowLimit and dictLimit.
+ * In dictMatchState mode, lowLimit and dictLimit are the same,
+ * and the dictionary is below them.
+ * forceWindow and dictMatchState are therefore incompatible.
+ */
+MEM_STATIC void
+ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
+ const void* blockEnd,
+ U32 maxDist,
+ U32* loadedDictEndPtr,
+ const ZSTD_matchState_t** dictMatchStatePtr)
+{
+ U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
+ U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
+ DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
+ (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
+
+ /* - When there is no dictionary : loadedDictEnd == 0.
+ In which case, the test (blockEndIdx > maxDist) is merely to avoid
+ overflowing next operation `newLowLimit = blockEndIdx - maxDist`.
+ - When there is a standard dictionary :
+ Index referential is copied from the dictionary,
+ which means it starts from 0.
+ In which case, loadedDictEnd == dictSize,
+ and it makes sense to compare `blockEndIdx > maxDist + dictSize`
+ since `blockEndIdx` also starts from zero.
+ - When there is an attached dictionary :
+ loadedDictEnd is expressed within the referential of the context,
+ so it can be directly compared against blockEndIdx.
+ */
+ if (blockEndIdx > maxDist + loadedDictEnd) {
+ U32 const newLowLimit = blockEndIdx - maxDist;
+ if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
+ if (window->dictLimit < window->lowLimit) {
+ DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
+ (unsigned)window->dictLimit, (unsigned)window->lowLimit);
+ window->dictLimit = window->lowLimit;
+ }
+ /* On reaching window size, dictionaries are invalidated */
+ if (loadedDictEndPtr) *loadedDictEndPtr = 0;
+ if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
+ }
+}
+
+/* Similar to ZSTD_window_enforceMaxDist(),
+ * but only invalidates dictionary
+ * when input progresses beyond window size.
+ * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL)
+ * loadedDictEnd uses same referential as window->base
+ * maxDist is the window size */
+MEM_STATIC void
+ZSTD_checkDictValidity(const ZSTD_window_t* window,
+ const void* blockEnd,
+ U32 maxDist,
+ U32* loadedDictEndPtr,
+ const ZSTD_matchState_t** dictMatchStatePtr)
+{
+ assert(loadedDictEndPtr != NULL);
+ assert(dictMatchStatePtr != NULL);
+ { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
+ U32 const loadedDictEnd = *loadedDictEndPtr;
+ DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
+ (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
+ assert(blockEndIdx >= loadedDictEnd);
+
+ if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) {
+ /* On reaching window size, dictionaries are invalidated.
+ * For simplification, if window size is reached anywhere within next block,
+ * the dictionary is invalidated for the full block.
+ *
+ * We also have to invalidate the dictionary if ZSTD_window_update() has detected
+ * non-contiguous segments, which means that loadedDictEnd != window->dictLimit.
+ * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use
+ * dictMatchState, so setting it to NULL is not a problem.
+ */
+ DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
+ *loadedDictEndPtr = 0;
+ *dictMatchStatePtr = NULL;
+ } else {
+ if (*loadedDictEndPtr != 0) {
+ DEBUGLOG(6, "dictionary considered valid for current block");
+ } } }
+}
+
+MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
+ ZSTD_memset(window, 0, sizeof(*window));
+ window->base = (BYTE const*)" ";
+ window->dictBase = (BYTE const*)" ";
+ ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */
+ window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */
+ window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */
+ window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */
+ window->nbOverflowCorrections = 0;
+}
+
+/**
+ * ZSTD_window_update():
+ * Updates the window by appending [src, src + srcSize) to the window.
+ * If it is not contiguous, the current prefix becomes the extDict, and we
+ * forget about the extDict. Handles overlap of the prefix and extDict.
+ * Returns non-zero if the segment is contiguous.
+ */
+MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
+ void const* src, size_t srcSize,
+ int forceNonContiguous)
+{
+ BYTE const* const ip = (BYTE const*)src;
+ U32 contiguous = 1;
+ DEBUGLOG(5, "ZSTD_window_update");
+ if (srcSize == 0)
+ return contiguous;
+ assert(window->base != NULL);
+ assert(window->dictBase != NULL);
+ /* Check if blocks follow each other */
+ if (src != window->nextSrc || forceNonContiguous) {
+ /* not contiguous */
+ size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
+ DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
+ window->lowLimit = window->dictLimit;
+ assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
+ window->dictLimit = (U32)distanceFromBase;
+ window->dictBase = window->base;
+ window->base = ip - distanceFromBase;
+ /* ms->nextToUpdate = window->dictLimit; */
+ if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */
+ contiguous = 0;
+ }
+ window->nextSrc = ip + srcSize;
+ /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
+ if ( (ip+srcSize > window->dictBase + window->lowLimit)
+ & (ip < window->dictBase + window->dictLimit)) {
+ ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase;
+ U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
+ window->lowLimit = lowLimitMax;
+ DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit);
+ }
+ return contiguous;
+}
+
+/**
+ * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
+ */
+MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
+{
+ U32 const maxDistance = 1U << windowLog;
+ U32 const lowestValid = ms->window.lowLimit;
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ U32 const isDictionary = (ms->loadedDictEnd != 0);
+ /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary
+ * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't
+ * valid for the entire block. So this check is sufficient to find the lowest valid match index.
+ */
+ U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
+ return matchLowest;
+}
+
+/**
+ * Returns the lowest allowed match index in the prefix.
+ */
+MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
+{
+ U32 const maxDistance = 1U << windowLog;
+ U32 const lowestValid = ms->window.dictLimit;
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ U32 const isDictionary = (ms->loadedDictEnd != 0);
+ /* When computing the lowest prefix index we need to take the dictionary into account to handle
+ * the edge case where the dictionary and the source are contiguous in memory.
+ */
+ U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
+ return matchLowest;
+}
+
+
+
+/* debug functions */
+#if (DEBUGLEVEL>=2)
+
+MEM_STATIC double ZSTD_fWeight(U32 rawStat)
+{
+ U32 const fp_accuracy = 8;
+ U32 const fp_multiplier = (1 << fp_accuracy);
+ U32 const newStat = rawStat + 1;
+ U32 const hb = ZSTD_highbit32(newStat);
+ U32 const BWeight = hb * fp_multiplier;
+ U32 const FWeight = (newStat << fp_accuracy) >> hb;
+ U32 const weight = BWeight + FWeight;
+ assert(hb + fp_accuracy < 31);
+ return (double)weight / fp_multiplier;
+}
+
+/* display a table content,
+ * listing each element, its frequency, and its predicted bit cost */
+MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
+{
+ unsigned u, sum;
+ for (u=0, sum=0; u<=max; u++) sum += table[u];
+ DEBUGLOG(2, "total nb elts: %u", sum);
+ for (u=0; u<=max; u++) {
+ DEBUGLOG(2, "%2u: %5u (%.2f)",
+ u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) );
+ }
+}
+
+#endif
+
+/* Short Cache */
+
+/* Normally, zstd matchfinders follow this flow:
+ * 1. Compute hash at ip
+ * 2. Load index from hashTable[hash]
+ * 3. Check if *ip == *(base + index)
+ * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss.
+ *
+ * Short cache is an optimization which allows us to avoid step 3 most of the time
+ * when the data doesn't actually match. With short cache, the flow becomes:
+ * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip.
+ * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works.
+ * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue.
+ *
+ * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to
+ * dictMatchState matchfinders.
+ */
+#define ZSTD_SHORT_CACHE_TAG_BITS 8
+#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1)
+
+/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable.
+ * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */
+MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) {
+ size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
+ U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK);
+ assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0);
+ hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag;
+}
+
+/* Helper function for short cache matchfinders.
+ * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */
+MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) {
+ U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK;
+ U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK;
+ return tag1 == tag2;
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+/* ===============================================================
+ * Shared internal declarations
+ * These prototypes may be called from sources not in lib/compress
+ * =============================================================== */
+
+/* ZSTD_loadCEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary.
+ * return : size of dictionary header (size of magic number + dict ID + entropy tables)
+ * assumptions : magic number supposed already checked
+ * and dictSize >= 8 */
+size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
+ const void* const dict, size_t dictSize);
+
+void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
+
+/* ==============================================================
+ * Private declarations
+ * These prototypes shall only be called from within lib/compress
+ * ============================================================== */
+
+/* ZSTD_getCParamsFromCCtxParams() :
+ * cParams are built depending on compressionLevel, src size hints,
+ * LDM and manually set compression parameters.
+ * Note: srcSizeHint == 0 means 0!
+ */
+ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
+ const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
+
+/*! ZSTD_initCStream_internal() :
+ * Private use only. Init streaming operation.
+ * expects params to be valid.
+ * must receive dict, or cdict, or none, but not both.
+ * @return : 0, or an error code */
+size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize);
+
+void ZSTD_resetSeqStore(seqStore_t* ssPtr);
+
+/*! ZSTD_getCParamsFromCDict() :
+ * as the name implies */
+ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
+
+/* ZSTD_compressBegin_advanced_internal() :
+ * Private use only. To be called from zstdmt_compress.c. */
+size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ unsigned long long pledgedSrcSize);
+
+/* ZSTD_compress_advanced_internal() :
+ * Private use only. To be called from zstdmt_compress.c. */
+size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ const ZSTD_CCtx_params* params);
+
+
+/* ZSTD_writeLastEmptyBlock() :
+ * output an empty Block with end-of-frame mark to complete a frame
+ * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
+ * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
+ */
+size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
+
+
+/* ZSTD_referenceExternalSequences() :
+ * Must be called before starting a compression operation.
+ * seqs must parse a prefix of the source.
+ * This cannot be used when long range matching is enabled.
+ * Zstd will use these sequences, and pass the literals to a secondary block
+ * compressor.
+ * @return : An error code on failure.
+ * NOTE: seqs are not verified! Invalid sequences can cause out-of-bounds memory
+ * access and data corruption.
+ */
+size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
+
+/** ZSTD_cycleLog() :
+ * condition for correct operation : hashLog > 1 */
+U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
+
+/** ZSTD_CCtx_trace() :
+ * Trace the end of a compression call.
+ */
+void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
+
+/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
+ * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
+ * Note that the block delimiter must include the last literals of the block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+ ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
+/* Returns the number of bytes to move the current read position back by.
+ * Only non-zero if we ended up splitting a sequence.
+ * Otherwise, it may return a ZSTD error if something went wrong.
+ *
+ * This function will attempt to scan through blockSize bytes
+ * represented by the sequences in @inSeqs,
+ * storing any (partial) sequences.
+ *
+ * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
+ * avoid splitting a match, or to avoid splitting a match such that it would produce a match
+ * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
+#endif /* ZSTD_COMPRESS_H */
diff --git a/contrib/zstd/zstd_compress_literals.c b/contrib/zstd/zstd_compress_literals.c
new file mode 100644
index 0000000..bfd4f11
--- /dev/null
+++ b/contrib/zstd/zstd_compress_literals.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+ /*-*************************************
+ * Dependencies
+ ***************************************/
+#include "zstd_compress_literals.h"
+
+
+/* **************************************************************
+* Debug Traces
+****************************************************************/
+#if DEBUGLEVEL >= 2
+
+static size_t showHexa(const void* src, size_t srcSize)
+{
+ const BYTE* const ip = (const BYTE*)src;
+ size_t u;
+ for (u=0; u<srcSize; u++) {
+ RAWLOG(5, " %02X", ip[u]); (void)ip;
+ }
+ RAWLOG(5, " \n");
+ return srcSize;
+}
+
+#endif
+
+
+/* **************************************************************
+* Literals compression - special cases
+****************************************************************/
+size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
+
+ DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity);
+
+ RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
+
+ switch(flSize)
+ {
+ case 1: /* 2 - 1 - 5 */
+ ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3));
+ break;
+ case 2: /* 2 - 2 - 12 */
+ MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4)));
+ break;
+ case 3: /* 2 - 2 - 20 */
+ MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4)));
+ break;
+ default: /* not necessary : flSize is {1,2,3} */
+ assert(0);
+ }
+
+ ZSTD_memcpy(ostart + flSize, src, srcSize);
+ DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
+ return srcSize + flSize;
+}
+
+static int allBytesIdentical(const void* src, size_t srcSize)
+{
+ assert(srcSize >= 1);
+ assert(src != NULL);
+ { const BYTE b = ((const BYTE*)src)[0];
+ size_t p;
+ for (p=1; p<srcSize; p++) {
+ if (((const BYTE*)src)[p] != b) return 0;
+ }
+ return 1;
+ }
+}
+
+size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
+
+ assert(dstCapacity >= 4); (void)dstCapacity;
+ assert(allBytesIdentical(src, srcSize));
+
+ switch(flSize)
+ {
+ case 1: /* 2 - 1 - 5 */
+ ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3));
+ break;
+ case 2: /* 2 - 2 - 12 */
+ MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4)));
+ break;
+ case 3: /* 2 - 2 - 20 */
+ MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4)));
+ break;
+ default: /* not necessary : flSize is {1,2,3} */
+ assert(0);
+ }
+
+ ostart[flSize] = *(const BYTE*)src;
+ DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
+ return flSize+1;
+}
+
+/* ZSTD_minLiteralsToCompress() :
+ * returns minimal amount of literals
+ * for literal compression to even be attempted.
+ * Minimum is made tighter as compression strategy increases.
+ */
+static size_t
+ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
+{
+ assert((int)strategy >= 0);
+ assert((int)strategy <= 9);
+ /* btultra2 : min 8 bytes;
+ * then 2x larger for each successive compression strategy
+ * max threshold 64 bytes */
+ { int const shift = MIN(9-(int)strategy, 3);
+ size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
+ DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
+ return mintc;
+ }
+}
+
+size_t ZSTD_compressLiterals (
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize,
+ const ZSTD_hufCTables_t* prevHuf,
+ ZSTD_hufCTables_t* nextHuf,
+ ZSTD_strategy strategy,
+ int disableLiteralCompression,
+ int suspectUncompressible,
+ int bmi2)
+{
+ size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
+ BYTE* const ostart = (BYTE*)dst;
+ U32 singleStream = srcSize < 256;
+ symbolEncodingType_e hType = set_compressed;
+ size_t cLitSize;
+
+ DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
+ disableLiteralCompression, (U32)srcSize, dstCapacity);
+
+ DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize));
+
+ /* Prepare nextEntropy assuming reusing the existing table */
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+
+ if (disableLiteralCompression)
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+
+ /* if too small, don't even attempt compression (speed opt) */
+ if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+
+ RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
+ { HUF_repeat repeat = prevHuf->repeatMode;
+ int const flags = 0
+ | (bmi2 ? HUF_flags_bmi2 : 0)
+ | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
+ | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
+ | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
+
+ typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
+ huf_compress_f huf_compress;
+ if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
+ huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
+ cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize,
+ src, srcSize,
+ HUF_SYMBOLVALUE_MAX, LitHufLog,
+ entropyWorkspace, entropyWorkspaceSize,
+ (HUF_CElt*)nextHuf->CTable,
+ &repeat, flags);
+ DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
+ if (repeat != HUF_repeat_none) {
+ /* reused the existing table */
+ DEBUGLOG(5, "reusing statistics from previous huffman block");
+ hType = set_repeat;
+ }
+ }
+
+ { size_t const minGain = ZSTD_minGain(srcSize, strategy);
+ if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+ } }
+ if (cLitSize==1) {
+ /* A return value of 1 signals that the alphabet consists of a single symbol.
+ * However, in some rare circumstances, it could be the compressed size (a single byte).
+ * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
+ * (it's also necessary to not generate statistics).
+ * Therefore, in such a case, actively check that all bytes are identical. */
+ if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+ } }
+
+ if (hType == set_compressed) {
+ /* using a newly constructed table */
+ nextHuf->repeatMode = HUF_repeat_check;
+ }
+
+ /* Build header */
+ switch(lhSize)
+ {
+ case 3: /* 2 - 2 - 10 - 10 */
+ if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
+ { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
+ MEM_writeLE24(ostart, lhc);
+ break;
+ }
+ case 4: /* 2 - 2 - 14 - 14 */
+ assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
+ { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
+ MEM_writeLE32(ostart, lhc);
+ break;
+ }
+ case 5: /* 2 - 2 - 18 - 18 */
+ assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
+ { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
+ MEM_writeLE32(ostart, lhc);
+ ostart[4] = (BYTE)(cLitSize >> 10);
+ break;
+ }
+ default: /* not possible : lhSize is {3,4,5} */
+ assert(0);
+ }
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
+ return lhSize+cLitSize;
+}
diff --git a/contrib/zstd/zstd_compress_literals.h b/contrib/zstd/zstd_compress_literals.h
new file mode 100644
index 0000000..b060c8a
--- /dev/null
+++ b/contrib/zstd/zstd_compress_literals.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPRESS_LITERALS_H
+#define ZSTD_COMPRESS_LITERALS_H
+
+#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */
+
+
+size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* ZSTD_compressRleLiteralsBlock() :
+ * Conditions :
+ * - All bytes in @src are identical
+ * - dstCapacity >= 4 */
+size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* ZSTD_compressLiterals():
+ * @entropyWorkspace: must be aligned on 4-bytes boundaries
+ * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
+ * @suspectUncompressible: sampling checks, to potentially skip huffman coding
+ */
+size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize,
+ const ZSTD_hufCTables_t* prevHuf,
+ ZSTD_hufCTables_t* nextHuf,
+ ZSTD_strategy strategy, int disableLiteralCompression,
+ int suspectUncompressible,
+ int bmi2);
+
+#endif /* ZSTD_COMPRESS_LITERALS_H */
diff --git a/contrib/zstd/zstd_compress_sequences.c b/contrib/zstd/zstd_compress_sequences.c
new file mode 100644
index 0000000..8872d4d
--- /dev/null
+++ b/contrib/zstd/zstd_compress_sequences.c
@@ -0,0 +1,442 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+ /*-*************************************
+ * Dependencies
+ ***************************************/
+#include "zstd_compress_sequences.h"
+
+/**
+ * -log2(x / 256) lookup table for x in [0, 256).
+ * If x == 0: Return 0
+ * Else: Return floor(-log2(x / 256) * 256)
+ */
+static unsigned const kInverseProbabilityLog256[256] = {
+ 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162,
+ 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889,
+ 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734,
+ 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626,
+ 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542,
+ 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473,
+ 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415,
+ 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366,
+ 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322,
+ 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282,
+ 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247,
+ 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215,
+ 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185,
+ 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157,
+ 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132,
+ 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108,
+ 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85,
+ 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64,
+ 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44,
+ 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25,
+ 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7,
+ 5, 4, 2, 1,
+};
+
+static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
+ void const* ptr = ctable;
+ U16 const* u16ptr = (U16 const*)ptr;
+ U32 const maxSymbolValue = MEM_read16(u16ptr + 1);
+ return maxSymbolValue;
+}
+
+/**
+ * Returns true if we should use ncount=-1 else we should
+ * use ncount=1 for low probability symbols instead.
+ */
+static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
+{
+ /* Heuristic: This should cover most blocks <= 16K and
+ * start to fade out after 16K to about 32K depending on
+ * compressibility.
+ */
+ return nbSeq >= 2048;
+}
+
+/**
+ * Returns the cost in bytes of encoding the normalized count header.
+ * Returns an error if any of the helper functions return an error.
+ */
+static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
+ size_t const nbSeq, unsigned const FSELog)
+{
+ BYTE wksp[FSE_NCOUNTBOUND];
+ S16 norm[MaxSeq + 1];
+ const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
+ FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), "");
+ return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
+}
+
+/**
+ * Returns the cost in bits of encoding the distribution described by count
+ * using the entropy bound.
+ */
+static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total)
+{
+ unsigned cost = 0;
+ unsigned s;
+
+ assert(total > 0);
+ for (s = 0; s <= max; ++s) {
+ unsigned norm = (unsigned)((256 * count[s]) / total);
+ if (count[s] != 0 && norm == 0)
+ norm = 1;
+ assert(count[s] < total);
+ cost += count[s] * kInverseProbabilityLog256[norm];
+ }
+ return cost >> 8;
+}
+
+/**
+ * Returns the cost in bits of encoding the distribution in count using ctable.
+ * Returns an error if ctable cannot represent all the symbols in count.
+ */
+size_t ZSTD_fseBitCost(
+ FSE_CTable const* ctable,
+ unsigned const* count,
+ unsigned const max)
+{
+ unsigned const kAccuracyLog = 8;
+ size_t cost = 0;
+ unsigned s;
+ FSE_CState_t cstate;
+ FSE_initCState(&cstate, ctable);
+ if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
+ DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
+ ZSTD_getFSEMaxSymbolValue(ctable), max);
+ return ERROR(GENERIC);
+ }
+ for (s = 0; s <= max; ++s) {
+ unsigned const tableLog = cstate.stateLog;
+ unsigned const badCost = (tableLog + 1) << kAccuracyLog;
+ unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
+ if (count[s] == 0)
+ continue;
+ if (bitCost >= badCost) {
+ DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
+ return ERROR(GENERIC);
+ }
+ cost += (size_t)count[s] * bitCost;
+ }
+ return cost >> kAccuracyLog;
+}
+
+/**
+ * Returns the cost in bits of encoding the distribution in count using the
+ * table described by norm. The max symbol support by norm is assumed >= max.
+ * norm must be valid for every symbol with non-zero probability in count.
+ */
+size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
+ unsigned const* count, unsigned const max)
+{
+ unsigned const shift = 8 - accuracyLog;
+ size_t cost = 0;
+ unsigned s;
+ assert(accuracyLog <= 8);
+ for (s = 0; s <= max; ++s) {
+ unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1;
+ unsigned const norm256 = normAcc << shift;
+ assert(norm256 > 0);
+ assert(norm256 < 256);
+ cost += count[s] * kInverseProbabilityLog256[norm256];
+ }
+ return cost >> 8;
+}
+
+symbolEncodingType_e
+ZSTD_selectEncodingType(
+ FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
+ size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
+ FSE_CTable const* prevCTable,
+ short const* defaultNorm, U32 defaultNormLog,
+ ZSTD_defaultPolicy_e const isDefaultAllowed,
+ ZSTD_strategy const strategy)
+{
+ ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0);
+ if (mostFrequent == nbSeq) {
+ *repeatMode = FSE_repeat_none;
+ if (isDefaultAllowed && nbSeq <= 2) {
+ /* Prefer set_basic over set_rle when there are 2 or fewer symbols,
+ * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol.
+ * If basic encoding isn't possible, always choose RLE.
+ */
+ DEBUGLOG(5, "Selected set_basic");
+ return set_basic;
+ }
+ DEBUGLOG(5, "Selected set_rle");
+ return set_rle;
+ }
+ if (strategy < ZSTD_lazy) {
+ if (isDefaultAllowed) {
+ size_t const staticFse_nbSeq_max = 1000;
+ size_t const mult = 10 - strategy;
+ size_t const baseLog = 3;
+ size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */
+ assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */
+ assert(mult <= 9 && mult >= 7);
+ if ( (*repeatMode == FSE_repeat_valid)
+ && (nbSeq < staticFse_nbSeq_max) ) {
+ DEBUGLOG(5, "Selected set_repeat");
+ return set_repeat;
+ }
+ if ( (nbSeq < dynamicFse_nbSeq_min)
+ || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) {
+ DEBUGLOG(5, "Selected set_basic");
+ /* The format allows default tables to be repeated, but it isn't useful.
+ * When using simple heuristics to select encoding type, we don't want
+ * to confuse these tables with dictionaries. When running more careful
+ * analysis, we don't need to waste time checking both repeating tables
+ * and default tables.
+ */
+ *repeatMode = FSE_repeat_none;
+ return set_basic;
+ }
+ }
+ } else {
+ size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC);
+ size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC);
+ size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog);
+ size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq);
+
+ if (isDefaultAllowed) {
+ assert(!ZSTD_isError(basicCost));
+ assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost)));
+ }
+ assert(!ZSTD_isError(NCountCost));
+ assert(compressedCost < ERROR(maxCode));
+ DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u",
+ (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost);
+ if (basicCost <= repeatCost && basicCost <= compressedCost) {
+ DEBUGLOG(5, "Selected set_basic");
+ assert(isDefaultAllowed);
+ *repeatMode = FSE_repeat_none;
+ return set_basic;
+ }
+ if (repeatCost <= compressedCost) {
+ DEBUGLOG(5, "Selected set_repeat");
+ assert(!ZSTD_isError(repeatCost));
+ return set_repeat;
+ }
+ assert(compressedCost < basicCost && compressedCost < repeatCost);
+ }
+ DEBUGLOG(5, "Selected set_compressed");
+ *repeatMode = FSE_repeat_check;
+ return set_compressed;
+}
+
+typedef struct {
+ S16 norm[MaxSeq + 1];
+ U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)];
+} ZSTD_BuildCTableWksp;
+
+size_t
+ZSTD_buildCTable(void* dst, size_t dstCapacity,
+ FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
+ unsigned* count, U32 max,
+ const BYTE* codeTable, size_t nbSeq,
+ const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ const FSE_CTable* prevCTable, size_t prevCTableSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize)
+{
+ BYTE* op = (BYTE*)dst;
+ const BYTE* const oend = op + dstCapacity;
+ DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity);
+
+ switch (type) {
+ case set_rle:
+ FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
+ RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space");
+ *op = codeTable[0];
+ return 1;
+ case set_repeat:
+ ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize);
+ return 0;
+ case set_basic:
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
+ return 0;
+ case set_compressed: {
+ ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace;
+ size_t nbSeq_1 = nbSeq;
+ const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
+ if (count[codeTable[nbSeq-1]] > 1) {
+ count[codeTable[nbSeq-1]]--;
+ nbSeq_1--;
+ }
+ assert(nbSeq_1 > 1);
+ assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
+ (void)entropyWorkspaceSize;
+ FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed");
+ assert(oend >= op);
+ { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */
+ FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed");
+ return NCountSize;
+ }
+ }
+ default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach");
+ }
+}
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_encodeSequences_body(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets)
+{
+ BIT_CStream_t blockStream;
+ FSE_CState_t stateMatchLength;
+ FSE_CState_t stateOffsetBits;
+ FSE_CState_t stateLitLength;
+
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)),
+ dstSize_tooSmall, "not enough space remaining");
+ DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)",
+ (int)(blockStream.endPtr - blockStream.startPtr),
+ (unsigned)dstCapacity);
+
+ /* first symbols */
+ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
+ FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]);
+ FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
+ BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
+ if (MEM_32bits()) BIT_flushBits(&blockStream);
+ BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]);
+ if (MEM_32bits()) BIT_flushBits(&blockStream);
+ if (longOffsets) {
+ U32 const ofBits = ofCodeTable[nbSeq-1];
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+ if (extraBits) {
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits);
+ BIT_flushBits(&blockStream);
+ }
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits,
+ ofBits - extraBits);
+ } else {
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]);
+ }
+ BIT_flushBits(&blockStream);
+
+ { size_t n;
+ for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */
+ BYTE const llCode = llCodeTable[n];
+ BYTE const ofCode = ofCodeTable[n];
+ BYTE const mlCode = mlCodeTable[n];
+ U32 const llBits = LL_bits[llCode];
+ U32 const ofBits = ofCode;
+ U32 const mlBits = ML_bits[mlCode];
+ DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
+ (unsigned)sequences[n].litLength,
+ (unsigned)sequences[n].mlBase + MINMATCH,
+ (unsigned)sequences[n].offBase);
+ /* 32b*/ /* 64b*/
+ /* (7)*/ /* (7)*/
+ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
+ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
+ if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/
+ FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
+ if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog)))
+ BIT_flushBits(&blockStream); /* (7)*/
+ BIT_addBits(&blockStream, sequences[n].litLength, llBits);
+ if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
+ BIT_addBits(&blockStream, sequences[n].mlBase, mlBits);
+ if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
+ if (longOffsets) {
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+ if (extraBits) {
+ BIT_addBits(&blockStream, sequences[n].offBase, extraBits);
+ BIT_flushBits(&blockStream); /* (7)*/
+ }
+ BIT_addBits(&blockStream, sequences[n].offBase >> extraBits,
+ ofBits - extraBits); /* 31 */
+ } else {
+ BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */
+ }
+ BIT_flushBits(&blockStream); /* (7)*/
+ DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr));
+ } }
+
+ DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog);
+ FSE_flushCState(&blockStream, &stateMatchLength);
+ DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog);
+ FSE_flushCState(&blockStream, &stateOffsetBits);
+ DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog);
+ FSE_flushCState(&blockStream, &stateLitLength);
+
+ { size_t const streamSize = BIT_closeCStream(&blockStream);
+ RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space");
+ return streamSize;
+ }
+}
+
+static size_t
+ZSTD_encodeSequences_default(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets)
+{
+ return ZSTD_encodeSequences_body(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+}
+
+
+#if DYNAMIC_BMI2
+
+static BMI2_TARGET_ATTRIBUTE size_t
+ZSTD_encodeSequences_bmi2(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets)
+{
+ return ZSTD_encodeSequences_body(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+}
+
+#endif
+
+size_t ZSTD_encodeSequences(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
+{
+ DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity);
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ return ZSTD_encodeSequences_bmi2(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+ }
+#endif
+ (void)bmi2;
+ return ZSTD_encodeSequences_default(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+}
diff --git a/contrib/zstd/zstd_compress_sequences.h b/contrib/zstd/zstd_compress_sequences.h
new file mode 100644
index 0000000..37848d8
--- /dev/null
+++ b/contrib/zstd/zstd_compress_sequences.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPRESS_SEQUENCES_H
+#define ZSTD_COMPRESS_SEQUENCES_H
+
+#include "fse.h" /* FSE_repeat, FSE_CTable */
+#include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
+
+typedef enum {
+ ZSTD_defaultDisallowed = 0,
+ ZSTD_defaultAllowed = 1
+} ZSTD_defaultPolicy_e;
+
+symbolEncodingType_e
+ZSTD_selectEncodingType(
+ FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
+ size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
+ FSE_CTable const* prevCTable,
+ short const* defaultNorm, U32 defaultNormLog,
+ ZSTD_defaultPolicy_e const isDefaultAllowed,
+ ZSTD_strategy const strategy);
+
+size_t
+ZSTD_buildCTable(void* dst, size_t dstCapacity,
+ FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
+ unsigned* count, U32 max,
+ const BYTE* codeTable, size_t nbSeq,
+ const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ const FSE_CTable* prevCTable, size_t prevCTableSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize);
+
+size_t ZSTD_encodeSequences(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
+
+size_t ZSTD_fseBitCost(
+ FSE_CTable const* ctable,
+ unsigned const* count,
+ unsigned const max);
+
+size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
+ unsigned const* count, unsigned const max);
+#endif /* ZSTD_COMPRESS_SEQUENCES_H */
diff --git a/contrib/zstd/zstd_compress_superblock.c b/contrib/zstd/zstd_compress_superblock.c
new file mode 100644
index 0000000..8349085
--- /dev/null
+++ b/contrib/zstd/zstd_compress_superblock.c
@@ -0,0 +1,577 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+ /*-*************************************
+ * Dependencies
+ ***************************************/
+#include "zstd_compress_superblock.h"
+
+#include "zstd_internal.h" /* ZSTD_getSequenceLength */
+#include "hist.h" /* HIST_countFast_wksp */
+#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
+#include "zstd_compress_sequences.h"
+#include "zstd_compress_literals.h"
+
+/** ZSTD_compressSubBlock_literal() :
+ * Compresses literals section for a sub-block.
+ * When we have to write the Huffman table we will sometimes choose a header
+ * size larger than necessary. This is because we have to pick the header size
+ * before we know the table size + compressed size, so we have a bound on the
+ * table size. If we guessed incorrectly, we fall back to uncompressed literals.
+ *
+ * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
+ * in writing the header, otherwise it is set to 0.
+ *
+ * hufMetadata->hType has literals block type info.
+ * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block.
+ * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block.
+ * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block
+ * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
+ * and the following sub-blocks' literals sections will be Treeless_Literals_Block.
+ * @return : compressed size of literals section of a sub-block
+ * Or 0 if unable to compress.
+ * Or error code */
+static size_t
+ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
+ const BYTE* literals, size_t litSize,
+ void* dst, size_t dstSize,
+ const int bmi2, int writeEntropy, int* entropyWritten)
+{
+ size_t const header = writeEntropy ? 200 : 0;
+ size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header));
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart + lhSize;
+ U32 const singleStream = lhSize == 3;
+ symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
+ size_t cLitSize = 0;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
+
+ *entropyWritten = 0;
+ if (litSize == 0 || hufMetadata->hType == set_basic) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal");
+ return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
+ } else if (hufMetadata->hType == set_rle) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal");
+ return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize);
+ }
+
+ assert(litSize > 0);
+ assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat);
+
+ if (writeEntropy && hufMetadata->hType == set_compressed) {
+ ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
+ op += hufMetadata->hufDesSize;
+ cLitSize += hufMetadata->hufDesSize;
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
+ }
+
+ { int const flags = bmi2 ? HUF_flags_bmi2 : 0;
+ const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags)
+ : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags);
+ op += cSize;
+ cLitSize += cSize;
+ if (cSize == 0 || ERR_isError(cSize)) {
+ DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize));
+ return 0;
+ }
+ /* If we expand and we aren't writing a header then emit uncompressed */
+ if (!writeEntropy && cLitSize >= litSize) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible");
+ return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
+ }
+ /* If we are writing headers then allow expansion that doesn't change our header size. */
+ if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) {
+ assert(cLitSize > litSize);
+ DEBUGLOG(5, "Literals expanded beyond allowed header size");
+ return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
+ }
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize);
+ }
+
+ /* Build header */
+ switch(lhSize)
+ {
+ case 3: /* 2 - 2 - 10 - 10 */
+ { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14);
+ MEM_writeLE24(ostart, lhc);
+ break;
+ }
+ case 4: /* 2 - 2 - 14 - 14 */
+ { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18);
+ MEM_writeLE32(ostart, lhc);
+ break;
+ }
+ case 5: /* 2 - 2 - 18 - 18 */
+ { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22);
+ MEM_writeLE32(ostart, lhc);
+ ostart[4] = (BYTE)(cLitSize >> 10);
+ break;
+ }
+ default: /* not possible : lhSize is {3,4,5} */
+ assert(0);
+ }
+ *entropyWritten = 1;
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart));
+ return op-ostart;
+}
+
+static size_t
+ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
+ const seqDef* sequences, size_t nbSeq,
+ size_t litSize, int lastSequence)
+{
+ const seqDef* const sstart = sequences;
+ const seqDef* const send = sequences + nbSeq;
+ const seqDef* sp = sstart;
+ size_t matchLengthSum = 0;
+ size_t litLengthSum = 0;
+ (void)(litLengthSum); /* suppress unused variable warning on some environments */
+ while (send-sp > 0) {
+ ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
+ litLengthSum += seqLen.litLength;
+ matchLengthSum += seqLen.matchLength;
+ sp++;
+ }
+ assert(litLengthSum <= litSize);
+ if (!lastSequence) {
+ assert(litLengthSum == litSize);
+ }
+ return matchLengthSum + litSize;
+}
+
+/** ZSTD_compressSubBlock_sequences() :
+ * Compresses sequences section for a sub-block.
+ * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have
+ * symbol compression modes for the super-block.
+ * The first successfully compressed block will have these in its header.
+ * We set entropyWritten=1 when we succeed in compressing the sequences.
+ * The following sub-blocks will always have repeat mode.
+ * @return : compressed size of sequences section of a sub-block
+ * Or 0 if it is unable to compress
+ * Or error code. */
+static size_t
+ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
+ const seqDef* sequences, size_t nbSeq,
+ const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ const int bmi2, int writeEntropy, int* entropyWritten)
+{
+ const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart;
+ BYTE* seqHead;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets);
+
+ *entropyWritten = 0;
+ /* Sequences Header */
+ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
+ dstSize_tooSmall, "");
+ if (nbSeq < 0x7F)
+ *op++ = (BYTE)nbSeq;
+ else if (nbSeq < LONGNBSEQ)
+ op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
+ else
+ op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
+ if (nbSeq==0) {
+ return op - ostart;
+ }
+
+ /* seqHead : flags for FSE encoding type */
+ seqHead = op++;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart));
+
+ if (writeEntropy) {
+ const U32 LLtype = fseMetadata->llType;
+ const U32 Offtype = fseMetadata->ofType;
+ const U32 MLtype = fseMetadata->mlType;
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize);
+ *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
+ ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
+ op += fseMetadata->fseTablesSize;
+ } else {
+ const U32 repeat = set_repeat;
+ *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2));
+ }
+
+ { size_t const bitstreamSize = ZSTD_encodeSequences(
+ op, oend - op,
+ fseTables->matchlengthCTable, mlCode,
+ fseTables->offcodeCTable, ofCode,
+ fseTables->litlengthCTable, llCode,
+ sequences, nbSeq,
+ longOffsets, bmi2);
+ FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
+ op += bitstreamSize;
+ /* zstd versions <= 1.3.4 mistakenly report corruption when
+ * FSE_readNCount() receives a buffer < 4 bytes.
+ * Fixed by https://github.com/facebook/zstd/pull/1146.
+ * This can happen when the last set_compressed table present is 2
+ * bytes and the bitstream is only one byte.
+ * In this exceedingly rare case, we will simply emit an uncompressed
+ * block, since it isn't worth optimizing.
+ */
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) {
+ /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
+ assert(fseMetadata->lastCountSize + bitstreamSize == 3);
+ DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
+ "emitting an uncompressed block.");
+ return 0;
+ }
+#endif
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize);
+ }
+
+ /* zstd versions <= 1.4.0 mistakenly report error when
+ * sequences section body size is less than 3 bytes.
+ * Fixed by https://github.com/facebook/zstd/pull/1664.
+ * This can happen when the previous sequences section block is compressed
+ * with rle mode and the current block's sequences section is compressed
+ * with repeat mode where sequences section body size can be 1 byte.
+ */
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ if (op-seqHead < 4) {
+ DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting "
+ "an uncompressed block when sequences are < 4 bytes");
+ return 0;
+ }
+#endif
+
+ *entropyWritten = 1;
+ return op - ostart;
+}
+
+/** ZSTD_compressSubBlock() :
+ * Compresses a single sub-block.
+ * @return : compressed size of the sub-block
+ * Or 0 if it failed to compress. */
+static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ const seqDef* sequences, size_t nbSeq,
+ const BYTE* literals, size_t litSize,
+ const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ const int bmi2,
+ int writeLitEntropy, int writeSeqEntropy,
+ int* litEntropyWritten, int* seqEntropyWritten,
+ U32 lastBlock)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart + ZSTD_blockHeaderSize;
+ DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)",
+ litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock);
+ { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable,
+ &entropyMetadata->hufMetadata, literals, litSize,
+ op, oend-op, bmi2, writeLitEntropy, litEntropyWritten);
+ FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed");
+ if (cLitSize == 0) return 0;
+ op += cLitSize;
+ }
+ { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse,
+ &entropyMetadata->fseMetadata,
+ sequences, nbSeq,
+ llCode, mlCode, ofCode,
+ cctxParams,
+ op, oend-op,
+ bmi2, writeSeqEntropy, seqEntropyWritten);
+ FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed");
+ if (cSeqSize == 0) return 0;
+ op += cSeqSize;
+ }
+ /* Write block header */
+ { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize;
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
+ MEM_writeLE24(ostart, cBlockHeader24);
+ }
+ return op-ostart;
+}
+
+static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize,
+ const ZSTD_hufCTables_t* huf,
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ unsigned maxSymbolValue = 255;
+ size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
+
+ if (hufMetadata->hType == set_basic) return litSize;
+ else if (hufMetadata->hType == set_rle) return 1;
+ else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) {
+ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize);
+ if (ZSTD_isError(largest)) return litSize;
+ { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue);
+ if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize;
+ return cLitSizeEstimate + literalSectionHeaderSize;
+ } }
+ assert(0); /* impossible */
+ return 0;
+}
+
+static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
+ const BYTE* codeTable, unsigned maxCode,
+ size_t nbSeq, const FSE_CTable* fseCTable,
+ const U8* additionalBits,
+ short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ void* workspace, size_t wkspSize)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ const BYTE* ctp = codeTable;
+ const BYTE* const ctStart = ctp;
+ const BYTE* const ctEnd = ctStart + nbSeq;
+ size_t cSymbolTypeSizeEstimateInBits = 0;
+ unsigned max = maxCode;
+
+ HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
+ if (type == set_basic) {
+ /* We selected this encoding type, so it must be valid. */
+ assert(max <= defaultMax);
+ cSymbolTypeSizeEstimateInBits = max <= defaultMax
+ ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max)
+ : ERROR(GENERIC);
+ } else if (type == set_rle) {
+ cSymbolTypeSizeEstimateInBits = 0;
+ } else if (type == set_compressed || type == set_repeat) {
+ cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max);
+ }
+ if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10;
+ while (ctp < ctEnd) {
+ if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp];
+ else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */
+ ctp++;
+ }
+ return cSymbolTypeSizeEstimateInBits / 8;
+}
+
+static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_fseCTables_t* fseTables,
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
+ size_t cSeqSizeEstimate = 0;
+ if (nbSeq == 0) return sequencesSectionHeaderSize;
+ cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff,
+ nbSeq, fseTables->offcodeCTable, NULL,
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL,
+ nbSeq, fseTables->litlengthCTable, LL_bits,
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML,
+ nbSeq, fseTables->matchlengthCTable, ML_bits,
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
+ workspace, wkspSize);
+ if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
+ return cSeqSizeEstimate + sequencesSectionHeaderSize;
+}
+
+static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
+ const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_entropyCTables_t* entropy,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize,
+ int writeLitEntropy, int writeSeqEntropy) {
+ size_t cSizeEstimate = 0;
+ cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize,
+ &entropy->huf, &entropyMetadata->hufMetadata,
+ workspace, wkspSize, writeLitEntropy);
+ cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
+ nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+ workspace, wkspSize, writeSeqEntropy);
+ return cSizeEstimate + ZSTD_blockHeaderSize;
+}
+
+static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata)
+{
+ if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle)
+ return 1;
+ if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle)
+ return 1;
+ if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle)
+ return 1;
+ return 0;
+}
+
+/** ZSTD_compressSubBlock_multi() :
+ * Breaks super-block into multiple sub-blocks and compresses them.
+ * Entropy will be written to the first block.
+ * The following blocks will use repeat mode to compress.
+ * All sub-blocks are compressed blocks (no raw or rle blocks).
+ * @return : compressed size of the super block (which is multiple ZSTD blocks)
+ * Or 0 if it failed to compress. */
+static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
+ const ZSTD_compressedBlockState_t* prevCBlock,
+ ZSTD_compressedBlockState_t* nextCBlock,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const int bmi2, U32 lastBlock,
+ void* workspace, size_t wkspSize)
+{
+ const seqDef* const sstart = seqStorePtr->sequencesStart;
+ const seqDef* const send = seqStorePtr->sequences;
+ const seqDef* sp = sstart;
+ const BYTE* const lstart = seqStorePtr->litStart;
+ const BYTE* const lend = seqStorePtr->lit;
+ const BYTE* lp = lstart;
+ BYTE const* ip = (BYTE const*)src;
+ BYTE const* const iend = ip + srcSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart;
+ const BYTE* llCodePtr = seqStorePtr->llCode;
+ const BYTE* mlCodePtr = seqStorePtr->mlCode;
+ const BYTE* ofCodePtr = seqStorePtr->ofCode;
+ size_t targetCBlockSize = cctxParams->targetCBlockSize;
+ size_t litSize, seqCount;
+ int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed;
+ int writeSeqEntropy = 1;
+ int lastSequence = 0;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)",
+ (unsigned)(lend-lp), (unsigned)(send-sstart));
+
+ litSize = 0;
+ seqCount = 0;
+ do {
+ size_t cBlockSizeEstimate = 0;
+ if (sstart == send) {
+ lastSequence = 1;
+ } else {
+ const seqDef* const sequence = sp + seqCount;
+ lastSequence = sequence == send - 1;
+ litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength;
+ seqCount++;
+ }
+ if (lastSequence) {
+ assert(lp <= lend);
+ assert(litSize <= (size_t)(lend - lp));
+ litSize = (size_t)(lend - lp);
+ }
+ /* I think there is an optimization opportunity here.
+ * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful
+ * since it recalculates estimate from scratch.
+ * For example, it would recount literal distribution and symbol codes every time.
+ */
+ cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount,
+ &nextCBlock->entropy, entropyMetadata,
+ workspace, wkspSize, writeLitEntropy, writeSeqEntropy);
+ if (cBlockSizeEstimate > targetCBlockSize || lastSequence) {
+ int litEntropyWritten = 0;
+ int seqEntropyWritten = 0;
+ const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence);
+ const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
+ sp, seqCount,
+ lp, litSize,
+ llCodePtr, mlCodePtr, ofCodePtr,
+ cctxParams,
+ op, oend-op,
+ bmi2, writeLitEntropy, writeSeqEntropy,
+ &litEntropyWritten, &seqEntropyWritten,
+ lastBlock && lastSequence);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
+ if (cSize > 0 && cSize < decompressedSize) {
+ DEBUGLOG(5, "Committed the sub-block");
+ assert(ip + decompressedSize <= iend);
+ ip += decompressedSize;
+ sp += seqCount;
+ lp += litSize;
+ op += cSize;
+ llCodePtr += seqCount;
+ mlCodePtr += seqCount;
+ ofCodePtr += seqCount;
+ litSize = 0;
+ seqCount = 0;
+ /* Entropy only needs to be written once */
+ if (litEntropyWritten) {
+ writeLitEntropy = 0;
+ }
+ if (seqEntropyWritten) {
+ writeSeqEntropy = 0;
+ }
+ }
+ }
+ } while (!lastSequence);
+ if (writeLitEntropy) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
+ ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
+ }
+ if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
+ /* If we haven't written our entropy tables, then we've violated our contract and
+ * must emit an uncompressed block.
+ */
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten");
+ return 0;
+ }
+ if (ip < iend) {
+ size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock);
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip));
+ FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
+ assert(cSize != 0);
+ op += cSize;
+ /* We have to regenerate the repcodes because we've skipped some sequences */
+ if (sp < send) {
+ seqDef const* seq;
+ repcodes_t rep;
+ ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
+ for (seq = sstart; seq < sp; ++seq) {
+ ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
+ }
+ ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
+ }
+ }
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
+ return op-ostart;
+}
+
+size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ void const* src, size_t srcSize,
+ unsigned lastBlock) {
+ ZSTD_entropyCTablesMetadata_t entropyMetadata;
+
+ FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
+ &zc->blockState.prevCBlock->entropy,
+ &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ &entropyMetadata,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
+
+ return ZSTD_compressSubBlock_multi(&zc->seqStore,
+ zc->blockState.prevCBlock,
+ zc->blockState.nextCBlock,
+ &entropyMetadata,
+ &zc->appliedParams,
+ dst, dstCapacity,
+ src, srcSize,
+ zc->bmi2, lastBlock,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
+}
diff --git a/contrib/zstd/zstd_compress_superblock.h b/contrib/zstd/zstd_compress_superblock.h
new file mode 100644
index 0000000..1ac733b
--- /dev/null
+++ b/contrib/zstd/zstd_compress_superblock.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPRESS_ADVANCED_H
+#define ZSTD_COMPRESS_ADVANCED_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+
+#include "zstd.h" /* ZSTD_CCtx */
+
+/*-*************************************
+* Target Compressed Block Size
+***************************************/
+
+/* ZSTD_compressSuperBlock() :
+ * Used to compress a super block when targetCBlockSize is being used.
+ * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */
+size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ void const* src, size_t srcSize,
+ unsigned lastBlock);
+
+#endif /* ZSTD_COMPRESS_ADVANCED_H */
diff --git a/contrib/zstd/zstd_cwksp.h b/contrib/zstd/zstd_cwksp.h
new file mode 100644
index 0000000..4d007c0
--- /dev/null
+++ b/contrib/zstd/zstd_cwksp.h
@@ -0,0 +1,678 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_CWKSP_H
+#define ZSTD_CWKSP_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_internal.h"
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+* Constants
+***************************************/
+
+/* Since the workspace is effectively its own little malloc implementation /
+ * arena, when we run under ASAN, we should similarly insert redzones between
+ * each internal element of the workspace, so ASAN will catch overruns that
+ * reach outside an object but that stay inside the workspace.
+ *
+ * This defines the size of that redzone.
+ */
+#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE
+#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
+#endif
+
+
+/* Set our tables and aligneds to align by 64 bytes */
+#define ZSTD_CWKSP_ALIGNMENT_BYTES 64
+
+/*-*************************************
+* Structures
+***************************************/
+typedef enum {
+ ZSTD_cwksp_alloc_objects,
+ ZSTD_cwksp_alloc_buffers,
+ ZSTD_cwksp_alloc_aligned
+} ZSTD_cwksp_alloc_phase_e;
+
+/**
+ * Used to describe whether the workspace is statically allocated (and will not
+ * necessarily ever be freed), or if it's dynamically allocated and we can
+ * expect a well-formed caller to free this.
+ */
+typedef enum {
+ ZSTD_cwksp_dynamic_alloc,
+ ZSTD_cwksp_static_alloc
+} ZSTD_cwksp_static_alloc_e;
+
+/**
+ * Zstd fits all its internal datastructures into a single continuous buffer,
+ * so that it only needs to perform a single OS allocation (or so that a buffer
+ * can be provided to it and it can perform no allocations at all). This buffer
+ * is called the workspace.
+ *
+ * Several optimizations complicate that process of allocating memory ranges
+ * from this workspace for each internal datastructure:
+ *
+ * - These different internal datastructures have different setup requirements:
+ *
+ * - The static objects need to be cleared once and can then be trivially
+ * reused for each compression.
+ *
+ * - Various buffers don't need to be initialized at all--they are always
+ * written into before they're read.
+ *
+ * - The matchstate tables have a unique requirement that they don't need
+ * their memory to be totally cleared, but they do need the memory to have
+ * some bound, i.e., a guarantee that all values in the memory they've been
+ * allocated is less than some maximum value (which is the starting value
+ * for the indices that they will then use for compression). When this
+ * guarantee is provided to them, they can use the memory without any setup
+ * work. When it can't, they have to clear the area.
+ *
+ * - These buffers also have different alignment requirements.
+ *
+ * - We would like to reuse the objects in the workspace for multiple
+ * compressions without having to perform any expensive reallocation or
+ * reinitialization work.
+ *
+ * - We would like to be able to efficiently reuse the workspace across
+ * multiple compressions **even when the compression parameters change** and
+ * we need to resize some of the objects (where possible).
+ *
+ * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp
+ * abstraction was created. It works as follows:
+ *
+ * Workspace Layout:
+ *
+ * [ ... workspace ... ]
+ * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers]
+ *
+ * The various objects that live in the workspace are divided into the
+ * following categories, and are allocated separately:
+ *
+ * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict,
+ * so that literally everything fits in a single buffer. Note: if present,
+ * this must be the first object in the workspace, since ZSTD_customFree{CCtx,
+ * CDict}() rely on a pointer comparison to see whether one or two frees are
+ * required.
+ *
+ * - Fixed size objects: these are fixed-size, fixed-count objects that are
+ * nonetheless "dynamically" allocated in the workspace so that we can
+ * control how they're initialized separately from the broader ZSTD_CCtx.
+ * Examples:
+ * - Entropy Workspace
+ * - 2 x ZSTD_compressedBlockState_t
+ * - CDict dictionary contents
+ *
+ * - Tables: these are any of several different datastructures (hash tables,
+ * chain tables, binary trees) that all respect a common format: they are
+ * uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
+ * Their sizes depend on the cparams. These tables are 64-byte aligned.
+ *
+ * - Aligned: these buffers are used for various purposes that require 4 byte
+ * alignment, but don't require any initialization before they're used. These
+ * buffers are each aligned to 64 bytes.
+ *
+ * - Buffers: these buffers are used for various purposes that don't require
+ * any alignment or initialization before they're used. This means they can
+ * be moved around at no cost for a new compression.
+ *
+ * Allocating Memory:
+ *
+ * The various types of objects must be allocated in order, so they can be
+ * correctly packed into the workspace buffer. That order is:
+ *
+ * 1. Objects
+ * 2. Buffers
+ * 3. Aligned/Tables
+ *
+ * Attempts to reserve objects of different types out of order will fail.
+ */
+typedef struct {
+ void* workspace;
+ void* workspaceEnd;
+
+ void* objectEnd;
+ void* tableEnd;
+ void* tableValidEnd;
+ void* allocStart;
+
+ BYTE allocFailed;
+ int workspaceOversizedDuration;
+ ZSTD_cwksp_alloc_phase_e phase;
+ ZSTD_cwksp_static_alloc_e isStatic;
+} ZSTD_cwksp;
+
+/*-*************************************
+* Functions
+***************************************/
+
+MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws);
+
+MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
+ (void)ws;
+ assert(ws->workspace <= ws->objectEnd);
+ assert(ws->objectEnd <= ws->tableEnd);
+ assert(ws->objectEnd <= ws->tableValidEnd);
+ assert(ws->tableEnd <= ws->allocStart);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ assert(ws->allocStart <= ws->workspaceEnd);
+}
+
+/**
+ * Align must be a power of 2.
+ */
+MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
+ size_t const mask = align - 1;
+ assert((align & mask) == 0);
+ return (size + mask) & ~mask;
+}
+
+/**
+ * Use this to determine how much space in the workspace we will consume to
+ * allocate this object. (Normally it should be exactly the size of the object,
+ * but under special conditions, like ASAN, where we pad each object, it might
+ * be larger.)
+ *
+ * Since tables aren't currently redzoned, you don't need to call through this
+ * to figure out how much space you need for the matchState tables. Everything
+ * else is though.
+ *
+ * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size().
+ */
+MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
+ if (size == 0)
+ return 0;
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+#else
+ return size;
+#endif
+}
+
+/**
+ * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes.
+ * Used to determine the number of bytes required for a given "aligned".
+ */
+MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
+ return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES));
+}
+
+/**
+ * Returns the amount of additional space the cwksp must allocate
+ * for internal purposes (currently only alignment).
+ */
+MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
+ /* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes
+ * to align the beginning of tables section, as well as another n_2=[0, 63] bytes
+ * to align the beginning of the aligned section.
+ *
+ * n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and
+ * aligneds being sized in multiples of 64 bytes.
+ */
+ size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES;
+ return slackSpace;
+}
+
+
+/**
+ * Return the number of additional bytes required to align a pointer to the given number of bytes.
+ * alignBytes must be a power of two.
+ */
+MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) {
+ size_t const alignBytesMask = alignBytes - 1;
+ size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
+ assert((alignBytes & alignBytesMask) == 0);
+ assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES);
+ return bytes;
+}
+
+/**
+ * Internal function. Do not use directly.
+ * Reserves the given number of bytes within the aligned/buffer segment of the wksp,
+ * which counts from the end of the wksp (as opposed to the object/table segment).
+ *
+ * Returns a pointer to the beginning of that space.
+ */
+MEM_STATIC void*
+ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes)
+{
+ void* const alloc = (BYTE*)ws->allocStart - bytes;
+ void* const bottom = ws->tableEnd;
+ DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
+ alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ assert(alloc >= bottom);
+ if (alloc < bottom) {
+ DEBUGLOG(4, "cwksp: alloc failed!");
+ ws->allocFailed = 1;
+ return NULL;
+ }
+ /* the area is reserved from the end of wksp.
+ * If it overlaps with tableValidEnd, it voids guarantees on values' range */
+ if (alloc < ws->tableValidEnd) {
+ ws->tableValidEnd = alloc;
+ }
+ ws->allocStart = alloc;
+ return alloc;
+}
+
+/**
+ * Moves the cwksp to the next phase, and does any necessary allocations.
+ * cwksp initialization must necessarily go through each phase in order.
+ * Returns a 0 on success, or zstd error
+ */
+MEM_STATIC size_t
+ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase)
+{
+ assert(phase >= ws->phase);
+ if (phase > ws->phase) {
+ /* Going from allocating objects to allocating buffers */
+ if (ws->phase < ZSTD_cwksp_alloc_buffers &&
+ phase >= ZSTD_cwksp_alloc_buffers) {
+ ws->tableValidEnd = ws->objectEnd;
+ }
+
+ /* Going from allocating buffers to allocating aligneds/tables */
+ if (ws->phase < ZSTD_cwksp_alloc_aligned &&
+ phase >= ZSTD_cwksp_alloc_aligned) {
+ { /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */
+ size_t const bytesToAlign =
+ ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES);
+ DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign);
+ ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */
+ RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign),
+ memory_allocation, "aligned phase - alignment initial allocation failed!");
+ }
+ { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */
+ void* const alloc = ws->objectEnd;
+ size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES);
+ void* const objectEnd = (BYTE*)alloc + bytesToAlign;
+ DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign);
+ RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation,
+ "table phase - alignment initial allocation failed!");
+ ws->objectEnd = objectEnd;
+ ws->tableEnd = objectEnd; /* table area starts being empty */
+ if (ws->tableValidEnd < ws->tableEnd) {
+ ws->tableValidEnd = ws->tableEnd;
+ } } }
+ ws->phase = phase;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ }
+ return 0;
+}
+
+/**
+ * Returns whether this object/buffer/etc was allocated in this workspace.
+ */
+MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr)
+{
+ return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
+}
+
+/**
+ * Internal function. Do not use directly.
+ */
+MEM_STATIC void*
+ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase)
+{
+ void* alloc;
+ if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) {
+ return NULL;
+ }
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* over-reserve space */
+ bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+#endif
+
+ alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes);
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
+ * either size. */
+ if (alloc) {
+ alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ /* We need to keep the redzone poisoned while unpoisoning the bytes that
+ * are actually allocated. */
+ __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE);
+ }
+ }
+#endif
+
+ return alloc;
+}
+
+/**
+ * Reserves and returns unaligned memory.
+ */
+MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes)
+{
+ return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers);
+}
+
+/**
+ * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
+ */
+MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
+{
+ void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
+ ZSTD_cwksp_alloc_aligned);
+ assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
+ return ptr;
+}
+
+/**
+ * Aligned on 64 bytes. These buffers have the special property that
+ * their values remain constrained, allowing us to re-use them without
+ * memset()-ing them.
+ */
+MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
+{
+ const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
+ void* alloc;
+ void* end;
+ void* top;
+
+ if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
+ return NULL;
+ }
+ alloc = ws->tableEnd;
+ end = (BYTE *)alloc + bytes;
+ top = ws->allocStart;
+
+ DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
+ alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
+ assert((bytes & (sizeof(U32)-1)) == 0);
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ assert(end <= top);
+ if (end > top) {
+ DEBUGLOG(4, "cwksp: table alloc failed!");
+ ws->allocFailed = 1;
+ return NULL;
+ }
+ ws->tableEnd = end;
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ __asan_unpoison_memory_region(alloc, bytes);
+ }
+#endif
+
+ assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
+ assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
+ return alloc;
+}
+
+/**
+ * Aligned on sizeof(void*).
+ * Note : should happen only once, at workspace first initialization
+ */
+MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes)
+{
+ size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
+ void* alloc = ws->objectEnd;
+ void* end = (BYTE*)alloc + roundedBytes;
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* over-reserve space */
+ end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+#endif
+
+ DEBUGLOG(4,
+ "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining",
+ alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
+ assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0);
+ assert(bytes % ZSTD_ALIGNOF(void*) == 0);
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ /* we must be in the first phase, no advance is possible */
+ if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) {
+ DEBUGLOG(3, "cwksp: object alloc failed!");
+ ws->allocFailed = 1;
+ return NULL;
+ }
+ ws->objectEnd = end;
+ ws->tableEnd = end;
+ ws->tableValidEnd = end;
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
+ * either size. */
+ alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ __asan_unpoison_memory_region(alloc, bytes);
+ }
+#endif
+
+ return alloc;
+}
+
+MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws)
+{
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty");
+
+#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
+ /* To validate that the table re-use logic is sound, and that we don't
+ * access table space that we haven't cleaned, we re-"poison" the table
+ * space every time we mark it dirty. */
+ {
+ size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
+ assert(__msan_test_shadow(ws->objectEnd, size) == -1);
+ __msan_poison(ws->objectEnd, size);
+ }
+#endif
+
+ assert(ws->tableValidEnd >= ws->objectEnd);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ ws->tableValidEnd = ws->objectEnd;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean");
+ assert(ws->tableValidEnd >= ws->objectEnd);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ if (ws->tableValidEnd < ws->tableEnd) {
+ ws->tableValidEnd = ws->tableEnd;
+ }
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+/**
+ * Zero the part of the allocated tables not already marked clean.
+ */
+MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables");
+ assert(ws->tableValidEnd >= ws->objectEnd);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ if (ws->tableValidEnd < ws->tableEnd) {
+ ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd));
+ }
+ ZSTD_cwksp_mark_tables_clean(ws);
+}
+
+/**
+ * Invalidates table allocations.
+ * All other allocations remain valid.
+ */
+MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: clearing tables!");
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* We don't do this when the workspace is statically allocated, because
+ * when that is the case, we have no capability to hook into the end of the
+ * workspace's lifecycle to unpoison the memory.
+ */
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
+ __asan_poison_memory_region(ws->objectEnd, size);
+ }
+#endif
+
+ ws->tableEnd = ws->objectEnd;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+/**
+ * Invalidates all buffer, aligned, and table allocations.
+ * Object allocations remain valid.
+ */
+MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: clearing!");
+
+#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
+ /* To validate that the context re-use logic is sound, and that we don't
+ * access stuff that this compression hasn't initialized, we re-"poison"
+ * the workspace (or at least the non-static, non-table parts of it)
+ * every time we start a new compression. */
+ {
+ size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd;
+ __msan_poison(ws->tableValidEnd, size);
+ }
+#endif
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* We don't do this when the workspace is statically allocated, because
+ * when that is the case, we have no capability to hook into the end of the
+ * workspace's lifecycle to unpoison the memory.
+ */
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
+ __asan_poison_memory_region(ws->objectEnd, size);
+ }
+#endif
+
+ ws->tableEnd = ws->objectEnd;
+ ws->allocStart = ws->workspaceEnd;
+ ws->allocFailed = 0;
+ if (ws->phase > ZSTD_cwksp_alloc_buffers) {
+ ws->phase = ZSTD_cwksp_alloc_buffers;
+ }
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+/**
+ * The provided workspace takes ownership of the buffer [start, start+size).
+ * Any existing values in the workspace are ignored (the previously managed
+ * buffer, if present, must be separately freed).
+ */
+MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) {
+ DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size);
+ assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */
+ ws->workspace = start;
+ ws->workspaceEnd = (BYTE*)start + size;
+ ws->objectEnd = ws->workspace;
+ ws->tableValidEnd = ws->objectEnd;
+ ws->phase = ZSTD_cwksp_alloc_objects;
+ ws->isStatic = isStatic;
+ ZSTD_cwksp_clear(ws);
+ ws->workspaceOversizedDuration = 0;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) {
+ void* workspace = ZSTD_customMalloc(size, customMem);
+ DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size);
+ RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!");
+ ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc);
+ return 0;
+}
+
+MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
+ void *ptr = ws->workspace;
+ DEBUGLOG(4, "cwksp: freeing workspace");
+ ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp));
+ ZSTD_customFree(ptr, customMem);
+}
+
+/**
+ * Moves the management of a workspace from one cwksp to another. The src cwksp
+ * is left in an invalid state (src must be re-init()'ed before it's used again).
+ */
+MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
+ *dst = *src;
+ ZSTD_memset(src, 0, sizeof(ZSTD_cwksp));
+}
+
+MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
+ return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
+}
+
+MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) {
+ return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace)
+ + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart);
+}
+
+MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
+ return ws->allocFailed;
+}
+
+/*-*************************************
+* Functions Checking Free Space
+***************************************/
+
+/* ZSTD_alignmentSpaceWithinBounds() :
+ * Returns if the estimated space needed for a wksp is within an acceptable limit of the
+ * actual amount of space used.
+ */
+MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws,
+ size_t const estimatedSpace, int resizedWorkspace) {
+ if (resizedWorkspace) {
+ /* Resized/newly allocated wksp should have exact bounds */
+ return ZSTD_cwksp_used(ws) == estimatedSpace;
+ } else {
+ /* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes
+ * than estimatedSpace. See the comments in zstd_cwksp.h for details.
+ */
+ return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63);
+ }
+}
+
+
+MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) {
+ return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd);
+}
+
+MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace;
+}
+
+MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ return ZSTD_cwksp_check_available(
+ ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR);
+}
+
+MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)
+ && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION;
+}
+
+MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
+ ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) {
+ ws->workspaceOversizedDuration++;
+ } else {
+ ws->workspaceOversizedDuration = 0;
+ }
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_CWKSP_H */
diff --git a/contrib/zstd/zstd_ddict.c b/contrib/zstd/zstd_ddict.c
new file mode 100644
index 0000000..06bd996
--- /dev/null
+++ b/contrib/zstd/zstd_ddict.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* zstd_ddict.c :
+ * concentrates all logic that needs to know the internals of ZSTD_DDict object */
+
+/*-*******************************************************
+* Dependencies
+*********************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
+#include "cpu.h" /* bmi2 */
+#include "mem.h" /* low level memory routines */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#include "zstd_decompress_internal.h"
+#include "zstd_ddict.h"
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+# include "../legacy/zstd_legacy.h"
+#endif
+
+
+
+/*-*******************************************************
+* Types
+*********************************************************/
+struct ZSTD_DDict_s {
+ void* dictBuffer;
+ const void* dictContent;
+ size_t dictSize;
+ ZSTD_entropyDTables_t entropy;
+ U32 dictID;
+ U32 entropyPresent;
+ ZSTD_customMem cMem;
+}; /* typedef'd to ZSTD_DDict within "zstd.h" */
+
+const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
+{
+ assert(ddict != NULL);
+ return ddict->dictContent;
+}
+
+size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
+{
+ assert(ddict != NULL);
+ return ddict->dictSize;
+}
+
+void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+ DEBUGLOG(4, "ZSTD_copyDDictParameters");
+ assert(dctx != NULL);
+ assert(ddict != NULL);
+ dctx->dictID = ddict->dictID;
+ dctx->prefixStart = ddict->dictContent;
+ dctx->virtualStart = ddict->dictContent;
+ dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
+ dctx->previousDstEnd = dctx->dictEnd;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ dctx->dictContentBeginForFuzzing = dctx->prefixStart;
+ dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
+#endif
+ if (ddict->entropyPresent) {
+ dctx->litEntropy = 1;
+ dctx->fseEntropy = 1;
+ dctx->LLTptr = ddict->entropy.LLTable;
+ dctx->MLTptr = ddict->entropy.MLTable;
+ dctx->OFTptr = ddict->entropy.OFTable;
+ dctx->HUFptr = ddict->entropy.hufTable;
+ dctx->entropy.rep[0] = ddict->entropy.rep[0];
+ dctx->entropy.rep[1] = ddict->entropy.rep[1];
+ dctx->entropy.rep[2] = ddict->entropy.rep[2];
+ } else {
+ dctx->litEntropy = 0;
+ dctx->fseEntropy = 0;
+ }
+}
+
+
+static size_t
+ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
+ ZSTD_dictContentType_e dictContentType)
+{
+ ddict->dictID = 0;
+ ddict->entropyPresent = 0;
+ if (dictContentType == ZSTD_dct_rawContent) return 0;
+
+ if (ddict->dictSize < 8) {
+ if (dictContentType == ZSTD_dct_fullDict)
+ return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
+ return 0; /* pure content mode */
+ }
+ { U32 const magic = MEM_readLE32(ddict->dictContent);
+ if (magic != ZSTD_MAGIC_DICTIONARY) {
+ if (dictContentType == ZSTD_dct_fullDict)
+ return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
+ return 0; /* pure content mode */
+ }
+ }
+ ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
+
+ /* load entropy tables */
+ RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
+ &ddict->entropy, ddict->dictContent, ddict->dictSize)),
+ dictionary_corrupted, "");
+ ddict->entropyPresent = 1;
+ return 0;
+}
+
+
+static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
+ ddict->dictBuffer = NULL;
+ ddict->dictContent = dict;
+ if (!dict) dictSize = 0;
+ } else {
+ void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
+ ddict->dictBuffer = internalBuffer;
+ ddict->dictContent = internalBuffer;
+ if (!internalBuffer) return ERROR(memory_allocation);
+ ZSTD_memcpy(internalBuffer, dict, dictSize);
+ }
+ ddict->dictSize = dictSize;
+ ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */
+
+ /* parse dictionary content */
+ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
+
+ return 0;
+}
+
+ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_customMem customMem)
+{
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+ { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
+ if (ddict == NULL) return NULL;
+ ddict->cMem = customMem;
+ { size_t const initResult = ZSTD_initDDict_internal(ddict,
+ dict, dictSize,
+ dictLoadMethod, dictContentType);
+ if (ZSTD_isError(initResult)) {
+ ZSTD_freeDDict(ddict);
+ return NULL;
+ } }
+ return ddict;
+ }
+}
+
+/*! ZSTD_createDDict() :
+* Create a digested dictionary, to start decompression without startup delay.
+* `dict` content is copied inside DDict.
+* Consequently, `dict` can be released after `ZSTD_DDict` creation */
+ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
+}
+
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, to start decompression without startup delay.
+ * Dictionary content is simply referenced, it will be accessed during decompression.
+ * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
+ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
+}
+
+
+const ZSTD_DDict* ZSTD_initStaticDDict(
+ void* sBuffer, size_t sBufferSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ size_t const neededSpace = sizeof(ZSTD_DDict)
+ + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
+ ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
+ assert(sBuffer != NULL);
+ assert(dict != NULL);
+ if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
+ if (sBufferSize < neededSpace) return NULL;
+ if (dictLoadMethod == ZSTD_dlm_byCopy) {
+ ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */
+ dict = ddict+1;
+ }
+ if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
+ dict, dictSize,
+ ZSTD_dlm_byRef, dictContentType) ))
+ return NULL;
+ return ddict;
+}
+
+
+size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0; /* support free on NULL */
+ { ZSTD_customMem const cMem = ddict->cMem;
+ ZSTD_customFree(ddict->dictBuffer, cMem);
+ ZSTD_customFree(ddict, cMem);
+ return 0;
+ }
+}
+
+/*! ZSTD_estimateDDictSize() :
+ * Estimate amount of memory that will be needed to create a dictionary for decompression.
+ * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
+size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
+{
+ return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
+}
+
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0; /* support sizeof on NULL */
+ return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
+}
+
+/*! ZSTD_getDictID_fromDDict() :
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0;
+ return ddict->dictID;
+}
diff --git a/contrib/zstd/zstd_ddict.h b/contrib/zstd/zstd_ddict.h
new file mode 100644
index 0000000..f49c570
--- /dev/null
+++ b/contrib/zstd/zstd_ddict.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_DDICT_H
+#define ZSTD_DDICT_H
+
+/*-*******************************************************
+ * Dependencies
+ *********************************************************/
+#include "zstd_deps.h" /* size_t */
+#include "zstd.h" /* ZSTD_DDict, and several public functions */
+
+
+/*-*******************************************************
+ * Interface
+ *********************************************************/
+
+/* note: several prototypes are already published in `zstd.h` :
+ * ZSTD_createDDict()
+ * ZSTD_createDDict_byReference()
+ * ZSTD_createDDict_advanced()
+ * ZSTD_freeDDict()
+ * ZSTD_initStaticDDict()
+ * ZSTD_sizeof_DDict()
+ * ZSTD_estimateDDictSize()
+ * ZSTD_getDictID_fromDict()
+ */
+
+const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
+size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
+
+void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+
+
+#endif /* ZSTD_DDICT_H */
diff --git a/contrib/zstd/zstd_decompress.c b/contrib/zstd/zstd_decompress.c
new file mode 100644
index 0000000..05704ce
--- /dev/null
+++ b/contrib/zstd/zstd_decompress.c
@@ -0,0 +1,2352 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* ***************************************************************
+* Tuning parameters
+*****************************************************************/
+/*!
+ * HEAPMODE :
+ * Select how default decompression function ZSTD_decompress() allocates its context,
+ * on stack (0), or into heap (1, default; requires malloc()).
+ * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
+ */
+#ifndef ZSTD_HEAPMODE
+# define ZSTD_HEAPMODE 1
+#endif
+
+/*!
+* LEGACY_SUPPORT :
+* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+)
+*/
+#ifndef ZSTD_LEGACY_SUPPORT
+# define ZSTD_LEGACY_SUPPORT 0
+#endif
+
+/*!
+ * MAXWINDOWSIZE_DEFAULT :
+ * maximum window size accepted by DStream __by default__.
+ * Frames requiring more memory will be rejected.
+ * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
+ */
+#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
+# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)
+#endif
+
+/*!
+ * NO_FORWARD_PROGRESS_MAX :
+ * maximum allowed nb of calls to ZSTD_decompressStream()
+ * without any forward progress
+ * (defined as: no byte read from input, and no byte flushed to output)
+ * before triggering an error.
+ */
+#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX
+# define ZSTD_NO_FORWARD_PROGRESS_MAX 16
+#endif
+
+
+/*-*******************************************************
+* Dependencies
+*********************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
+#include "mem.h" /* low level memory routines */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#include "xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
+#include "zstd_internal.h" /* blockProperties_t */
+#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
+#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
+#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */
+#include "bits.h" /* ZSTD_highbit32 */
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+# include "../legacy/zstd_legacy.h"
+#endif
+
+
+
+/*************************************
+ * Multiple DDicts Hashset internals *
+ *************************************/
+
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
+ * Currently, that means a 0.75 load factor.
+ * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
+ * the load factor of the ddict hash set.
+ */
+
+#define DDICT_HASHSET_TABLE_BASE_SIZE 64
+#define DDICT_HASHSET_RESIZE_FACTOR 2
+
+/* Hash function to determine starting position of dict insertion within the table
+ * Returns an index between [0, hashSet->ddictPtrTableSize]
+ */
+static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
+ const U64 hash = XXH64(&dictID, sizeof(U32), 0);
+ /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
+ return hash & (hashSet->ddictPtrTableSize - 1);
+}
+
+/* Adds DDict to a hashset without resizing it.
+ * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
+ * Returns 0 if successful, or a zstd error code if something went wrong.
+ */
+static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
+ const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
+ RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
+ while (hashSet->ddictPtrTable[idx] != NULL) {
+ /* Replace existing ddict if inserting ddict with same dictID */
+ if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
+ DEBUGLOG(4, "DictID already exists, replacing rather than adding");
+ hashSet->ddictPtrTable[idx] = ddict;
+ return 0;
+ }
+ idx &= idxRangeMask;
+ idx++;
+ }
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
+ hashSet->ddictPtrTable[idx] = ddict;
+ hashSet->ddictPtrCount++;
+ return 0;
+}
+
+/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
+ * rehashes all values, allocates new table, frees old table.
+ * Returns 0 on success, otherwise a zstd error code.
+ */
+static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+ size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
+ const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
+ const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
+ size_t oldTableSize = hashSet->ddictPtrTableSize;
+ size_t i;
+
+ DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
+ RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
+ hashSet->ddictPtrTable = newTable;
+ hashSet->ddictPtrTableSize = newTableSize;
+ hashSet->ddictPtrCount = 0;
+ for (i = 0; i < oldTableSize; ++i) {
+ if (oldTable[i] != NULL) {
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
+ }
+ }
+ ZSTD_customFree((void*)oldTable, customMem);
+ DEBUGLOG(4, "Finished re-hash");
+ return 0;
+}
+
+/* Fetches a DDict with the given dictID
+ * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
+ */
+static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
+ for (;;) {
+ size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
+ if (currDictID == dictID || currDictID == 0) {
+ /* currDictID == 0 implies a NULL ddict entry */
+ break;
+ } else {
+ idx &= idxRangeMask; /* Goes to start of table when we reach the end */
+ idx++;
+ }
+ }
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
+ return hashSet->ddictPtrTable[idx];
+}
+
+/* Allocates space for and returns a ddict hash set
+ * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
+ * Returns NULL if allocation failed.
+ */
+static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
+ ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
+ DEBUGLOG(4, "Allocating new hash set");
+ if (!ret)
+ return NULL;
+ ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
+ if (!ret->ddictPtrTable) {
+ ZSTD_customFree(ret, customMem);
+ return NULL;
+ }
+ ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
+ ret->ddictPtrCount = 0;
+ return ret;
+}
+
+/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
+ * Note: The ZSTD_DDict* within the table are NOT freed.
+ */
+static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+ DEBUGLOG(4, "Freeing ddict hash set");
+ if (hashSet && hashSet->ddictPtrTable) {
+ ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
+ }
+ if (hashSet) {
+ ZSTD_customFree(hashSet, customMem);
+ }
+}
+
+/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
+ * Returns 0 on success, or a ZSTD error.
+ */
+static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
+ DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
+ if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
+ }
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
+ return 0;
+}
+
+/*-*************************************************************
+* Context management
+***************************************************************/
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
+{
+ if (dctx==NULL) return 0; /* support sizeof NULL */
+ return sizeof(*dctx)
+ + ZSTD_sizeof_DDict(dctx->ddictLocal)
+ + dctx->inBuffSize + dctx->outBuffSize;
+}
+
+size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
+
+
+static size_t ZSTD_startingInputLength(ZSTD_format_e format)
+{
+ size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
+ /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
+ assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
+ return startingInputLength;
+}
+
+static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
+{
+ assert(dctx->streamStage == zdss_init);
+ dctx->format = ZSTD_f_zstd1;
+ dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
+ dctx->outBufferMode = ZSTD_bm_buffered;
+ dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
+ dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
+ dctx->disableHufAsm = 0;
+}
+
+static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
+{
+ dctx->staticSize = 0;
+ dctx->ddict = NULL;
+ dctx->ddictLocal = NULL;
+ dctx->dictEnd = NULL;
+ dctx->ddictIsCold = 0;
+ dctx->dictUses = ZSTD_dont_use;
+ dctx->inBuff = NULL;
+ dctx->inBuffSize = 0;
+ dctx->outBuffSize = 0;
+ dctx->streamStage = zdss_init;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ dctx->legacyContext = NULL;
+ dctx->previousLegacyVersion = 0;
+#endif
+ dctx->noForwardProgress = 0;
+ dctx->oversizedDuration = 0;
+#if DYNAMIC_BMI2
+ dctx->bmi2 = ZSTD_cpuSupportsBmi2();
+#endif
+ dctx->ddictSet = NULL;
+ ZSTD_DCtx_resetParameters(dctx);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ dctx->dictContentEndForFuzzing = NULL;
+#endif
+}
+
+ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
+{
+ ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace;
+
+ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
+ if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */
+
+ ZSTD_initDCtx_internal(dctx);
+ dctx->staticSize = workspaceSize;
+ dctx->inBuff = (char*)(dctx+1);
+ return dctx;
+}
+
+static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) {
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+ { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem);
+ if (!dctx) return NULL;
+ dctx->customMem = customMem;
+ ZSTD_initDCtx_internal(dctx);
+ return dctx;
+ }
+}
+
+ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
+{
+ return ZSTD_createDCtx_internal(customMem);
+}
+
+ZSTD_DCtx* ZSTD_createDCtx(void)
+{
+ DEBUGLOG(3, "ZSTD_createDCtx");
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+}
+
+static void ZSTD_clearDict(ZSTD_DCtx* dctx)
+{
+ ZSTD_freeDDict(dctx->ddictLocal);
+ dctx->ddictLocal = NULL;
+ dctx->ddict = NULL;
+ dctx->dictUses = ZSTD_dont_use;
+}
+
+size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
+{
+ if (dctx==NULL) return 0; /* support free on NULL */
+ RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
+ { ZSTD_customMem const cMem = dctx->customMem;
+ ZSTD_clearDict(dctx);
+ ZSTD_customFree(dctx->inBuff, cMem);
+ dctx->inBuff = NULL;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (dctx->legacyContext)
+ ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
+#endif
+ if (dctx->ddictSet) {
+ ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
+ dctx->ddictSet = NULL;
+ }
+ ZSTD_customFree(dctx, cMem);
+ return 0;
+ }
+}
+
+/* no longer useful */
+void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
+{
+ size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
+ ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
+}
+
+/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
+ * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
+ * accordingly sets the ddict to be used to decompress the frame.
+ *
+ * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
+ *
+ * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
+ */
+static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
+ assert(dctx->refMultipleDDicts && dctx->ddictSet);
+ DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
+ if (dctx->ddict) {
+ const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
+ if (frameDDict) {
+ DEBUGLOG(4, "DDict found!");
+ ZSTD_clearDict(dctx);
+ dctx->dictID = dctx->fParams.dictID;
+ dctx->ddict = frameDDict;
+ dctx->dictUses = ZSTD_use_indefinitely;
+ }
+ }
+}
+
+
+/*-*************************************************************
+ * Frame header decoding
+ ***************************************************************/
+
+/*! ZSTD_isFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ * Note 3 : Skippable Frame Identifiers are considered valid. */
+unsigned ZSTD_isFrame(const void* buffer, size_t size)
+{
+ if (size < ZSTD_FRAMEIDSIZE) return 0;
+ { U32 const magic = MEM_readLE32(buffer);
+ if (magic == ZSTD_MAGICNUMBER) return 1;
+ if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+ }
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(buffer, size)) return 1;
+#endif
+ return 0;
+}
+
+/*! ZSTD_isSkippableFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ */
+unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
+{
+ if (size < ZSTD_FRAMEIDSIZE) return 0;
+ { U32 const magic = MEM_readLE32(buffer);
+ if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+ }
+ return 0;
+}
+
+/** ZSTD_frameHeaderSize_internal() :
+ * srcSize must be large enough to reach header size fields.
+ * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
+ * @return : size of the Frame Header
+ * or an error code, which can be tested with ZSTD_isError() */
+static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
+{
+ size_t const minInputSize = ZSTD_startingInputLength(format);
+ RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");
+
+ { BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
+ U32 const dictID= fhd & 3;
+ U32 const singleSegment = (fhd >> 5) & 1;
+ U32 const fcsId = fhd >> 6;
+ return minInputSize + !singleSegment
+ + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+ + (singleSegment && !fcsId);
+ }
+}
+
+/** ZSTD_frameHeaderSize() :
+ * srcSize must be >= ZSTD_frameHeaderSize_prefix.
+ * @return : size of the Frame Header,
+ * or an error code (if srcSize is too small) */
+size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
+{
+ return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);
+}
+
+
+/** ZSTD_getFrameHeader_advanced() :
+ * decode Frame Header, or require larger `srcSize`.
+ * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
+ * @return : 0, `zfhPtr` is correctly filled,
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
+** or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
+{
+ const BYTE* ip = (const BYTE*)src;
+ size_t const minInputSize = ZSTD_startingInputLength(format);
+
+ DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize);
+
+ if (srcSize > 0) {
+ /* note : technically could be considered an assert(), since it's an invalid entry */
+ RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0");
+ }
+ if (srcSize < minInputSize) {
+ if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) {
+ /* when receiving less than @minInputSize bytes,
+ * control these bytes at least correspond to a supported magic number
+ * in order to error out early if they don't.
+ **/
+ size_t const toCopy = MIN(4, srcSize);
+ unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER);
+ assert(src != NULL);
+ ZSTD_memcpy(hbuf, src, toCopy);
+ if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) {
+ /* not a zstd frame : let's check if it's a skippable frame */
+ MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START);
+ ZSTD_memcpy(hbuf, src, toCopy);
+ if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) {
+ RETURN_ERROR(prefix_unknown,
+ "first bytes don't correspond to any supported magic number");
+ } } }
+ return minInputSize;
+ }
+
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */
+ if ( (format != ZSTD_f_zstd1_magicless)
+ && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
+ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ /* skippable frame */
+ if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
+ return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
+ zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
+ zfhPtr->frameType = ZSTD_skippableFrame;
+ return 0;
+ }
+ RETURN_ERROR(prefix_unknown, "");
+ }
+
+ /* ensure there is enough `srcSize` to fully read/decode frame header */
+ { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format);
+ if (srcSize < fhsize) return fhsize;
+ zfhPtr->headerSize = (U32)fhsize;
+ }
+
+ { BYTE const fhdByte = ip[minInputSize-1];
+ size_t pos = minInputSize;
+ U32 const dictIDSizeCode = fhdByte&3;
+ U32 const checksumFlag = (fhdByte>>2)&1;
+ U32 const singleSegment = (fhdByte>>5)&1;
+ U32 const fcsID = fhdByte>>6;
+ U64 windowSize = 0;
+ U32 dictID = 0;
+ U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported,
+ "reserved bits, must be zero");
+
+ if (!singleSegment) {
+ BYTE const wlByte = ip[pos++];
+ U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, "");
+ windowSize = (1ULL << windowLog);
+ windowSize += (windowSize >> 3) * (wlByte&7);
+ }
+ switch(dictIDSizeCode)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : break;
+ case 1 : dictID = ip[pos]; pos++; break;
+ case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
+ case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
+ }
+ switch(fcsID)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
+ case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
+ case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
+ case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
+ }
+ if (singleSegment) windowSize = frameContentSize;
+
+ zfhPtr->frameType = ZSTD_frame;
+ zfhPtr->frameContentSize = frameContentSize;
+ zfhPtr->windowSize = windowSize;
+ zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+ zfhPtr->dictID = dictID;
+ zfhPtr->checksumFlag = checksumFlag;
+ }
+ return 0;
+}
+
+/** ZSTD_getFrameHeader() :
+ * decode Frame Header, or require larger `srcSize`.
+ * note : this function does not consume input, it only reads it.
+ * @return : 0, `zfhPtr` is correctly filled,
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ * or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
+{
+ return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
+}
+
+/** ZSTD_getFrameContentSize() :
+ * compatible with legacy mode
+ * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
+{
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(src, srcSize)) {
+ unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
+ return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
+ }
+#endif
+ { ZSTD_frameHeader zfh;
+ if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
+ return ZSTD_CONTENTSIZE_ERROR;
+ if (zfh.frameType == ZSTD_skippableFrame) {
+ return 0;
+ } else {
+ return zfh.frameContentSize;
+ } }
+}
+
+static size_t readSkippableFrameSize(void const* src, size_t srcSize)
+{
+ size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
+ U32 sizeU32;
+
+ RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
+
+ sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
+ RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
+ frameParameter_unsupported, "");
+ {
+ size_t const skippableSize = skippableHeaderSize + sizeU32;
+ RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
+ return skippableSize;
+ }
+}
+
+/*! ZSTD_readSkippableFrame() :
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
+ * in the magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
+ const void* src, size_t srcSize)
+{
+ U32 const magicNumber = MEM_readLE32(src);
+ size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
+ size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
+
+ /* check input validity */
+ RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
+ RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
+ RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
+
+ /* deliver payload */
+ if (skippableContentSize > 0 && dst != NULL)
+ ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
+ if (magicVariant != NULL)
+ *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
+ return skippableContentSize;
+}
+
+/** ZSTD_findDecompressedSize() :
+ * compatible with legacy mode
+ * `srcSize` must be the exact length of some number of ZSTD compressed and/or
+ * skippable frames
+ * @return : decompressed size of the frames contained */
+unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
+{
+ unsigned long long totalDstSize = 0;
+
+ while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
+ U32 const magicNumber = MEM_readLE32(src);
+
+ if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ size_t const skippableSize = readSkippableFrameSize(src, srcSize);
+ if (ZSTD_isError(skippableSize)) {
+ return ZSTD_CONTENTSIZE_ERROR;
+ }
+ assert(skippableSize <= srcSize);
+
+ src = (const BYTE *)src + skippableSize;
+ srcSize -= skippableSize;
+ continue;
+ }
+
+ { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+ if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
+
+ /* check for overflow */
+ if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
+ totalDstSize += ret;
+ }
+ { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
+ if (ZSTD_isError(frameSrcSize)) {
+ return ZSTD_CONTENTSIZE_ERROR;
+ }
+
+ src = (const BYTE *)src + frameSrcSize;
+ srcSize -= frameSrcSize;
+ }
+ } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
+
+ if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
+
+ return totalDstSize;
+}
+
+/** ZSTD_getDecompressedSize() :
+ * compatible with legacy mode
+ * @return : decompressed size if known, 0 otherwise
+ note : 0 can mean any of the following :
+ - frame content is empty
+ - decompressed size field is not present in frame header
+ - frame header unknown / not supported
+ - frame header not complete (`srcSize` too small) */
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
+{
+ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);
+ return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;
+}
+
+
+/** ZSTD_decodeFrameHeader() :
+ * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
+ * If multiple DDict references are enabled, also will choose the correct DDict to use.
+ * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
+{
+ size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
+ if (ZSTD_isError(result)) return result; /* invalid header */
+ RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
+
+ /* Reference DDict requested by frame if dctx references multiple ddicts */
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
+ ZSTD_DCtx_selectFrameDDict(dctx);
+ }
+
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ /* Skip the dictID check in fuzzing mode, because it makes the search
+ * harder.
+ */
+ RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
+ dictionary_wrong, "");
+#endif
+ dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
+ if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
+ dctx->processedCSize += headerSize;
+ return 0;
+}
+
+static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
+{
+ ZSTD_frameSizeInfo frameSizeInfo;
+ frameSizeInfo.compressedSize = ret;
+ frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
+ return frameSizeInfo;
+}
+
+static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
+{
+ ZSTD_frameSizeInfo frameSizeInfo;
+ ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(src, srcSize))
+ return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
+#endif
+
+ if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
+ && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
+ assert(ZSTD_isError(frameSizeInfo.compressedSize) ||
+ frameSizeInfo.compressedSize <= srcSize);
+ return frameSizeInfo;
+ } else {
+ const BYTE* ip = (const BYTE*)src;
+ const BYTE* const ipstart = ip;
+ size_t remainingSize = srcSize;
+ size_t nbBlocks = 0;
+ ZSTD_frameHeader zfh;
+
+ /* Extract Frame Header */
+ { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
+ if (ZSTD_isError(ret))
+ return ZSTD_errorFrameSizeInfo(ret);
+ if (ret > 0)
+ return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+ }
+
+ ip += zfh.headerSize;
+ remainingSize -= zfh.headerSize;
+
+ /* Iterate over each block */
+ while (1) {
+ blockProperties_t blockProperties;
+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+ if (ZSTD_isError(cBlockSize))
+ return ZSTD_errorFrameSizeInfo(cBlockSize);
+
+ if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
+ return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+
+ ip += ZSTD_blockHeaderSize + cBlockSize;
+ remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
+ nbBlocks++;
+
+ if (blockProperties.lastBlock) break;
+ }
+
+ /* Final frame content checksum */
+ if (zfh.checksumFlag) {
+ if (remainingSize < 4)
+ return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+ ip += 4;
+ }
+
+ frameSizeInfo.nbBlocks = nbBlocks;
+ frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
+ frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
+ ? zfh.frameContentSize
+ : (unsigned long long)nbBlocks * zfh.blockSizeMax;
+ return frameSizeInfo;
+ }
+}
+
+/** ZSTD_findFrameCompressedSize() :
+ * compatible with legacy mode
+ * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
+ * `srcSize` must be at least as large as the frame contained
+ * @return : the compressed size of the frame starting at `src` */
+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+ ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+ return frameSizeInfo.compressedSize;
+}
+
+/** ZSTD_decompressBound() :
+ * compatible with legacy mode
+ * `src` must point to the start of a ZSTD frame or a skippeable frame
+ * `srcSize` must be at least as large as the frame contained
+ * @return : the maximum decompressed size of the compressed source
+ */
+unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
+{
+ unsigned long long bound = 0;
+ /* Iterate over each frame */
+ while (srcSize > 0) {
+ ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+ size_t const compressedSize = frameSizeInfo.compressedSize;
+ unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+ if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+ return ZSTD_CONTENTSIZE_ERROR;
+ assert(srcSize >= compressedSize);
+ src = (const BYTE*)src + compressedSize;
+ srcSize -= compressedSize;
+ bound += decompressedBound;
+ }
+ return bound;
+}
+
+size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
+{
+ size_t margin = 0;
+ unsigned maxBlockSize = 0;
+
+ /* Iterate over each frame */
+ while (srcSize > 0) {
+ ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+ size_t const compressedSize = frameSizeInfo.compressedSize;
+ unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+ ZSTD_frameHeader zfh;
+
+ FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
+ if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+ return ERROR(corruption_detected);
+
+ if (zfh.frameType == ZSTD_frame) {
+ /* Add the frame header to our margin */
+ margin += zfh.headerSize;
+ /* Add the checksum to our margin */
+ margin += zfh.checksumFlag ? 4 : 0;
+ /* Add 3 bytes per block */
+ margin += 3 * frameSizeInfo.nbBlocks;
+
+ /* Compute the max block size */
+ maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax);
+ } else {
+ assert(zfh.frameType == ZSTD_skippableFrame);
+ /* Add the entire skippable frame size to our margin. */
+ margin += compressedSize;
+ }
+
+ assert(srcSize >= compressedSize);
+ src = (const BYTE*)src + compressedSize;
+ srcSize -= compressedSize;
+ }
+
+ /* Add the max block size back to the margin. */
+ margin += maxBlockSize;
+
+ return margin;
+}
+
+/*-*************************************************************
+ * Frame decoding
+ ***************************************************************/
+
+/** ZSTD_insertBlock() :
+ * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
+size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
+{
+ DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
+ ZSTD_checkContinuity(dctx, blockStart, blockSize);
+ dctx->previousDstEnd = (const char*)blockStart + blockSize;
+ return blockSize;
+}
+
+
+static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_copyRawBlock");
+ RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
+ if (dst == NULL) {
+ if (srcSize == 0) return 0;
+ RETURN_ERROR(dstBuffer_null, "");
+ }
+ ZSTD_memmove(dst, src, srcSize);
+ return srcSize;
+}
+
+static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
+ BYTE b,
+ size_t regenSize)
+{
+ RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
+ if (dst == NULL) {
+ if (regenSize == 0) return 0;
+ RETURN_ERROR(dstBuffer_null, "");
+ }
+ ZSTD_memset(dst, b, regenSize);
+ return regenSize;
+}
+
+static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
+{
+#if ZSTD_TRACE
+ if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
+ ZSTD_Trace trace;
+ ZSTD_memset(&trace, 0, sizeof(trace));
+ trace.version = ZSTD_VERSION_NUMBER;
+ trace.streaming = streaming;
+ if (dctx->ddict) {
+ trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
+ trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
+ trace.dictionaryIsCold = dctx->ddictIsCold;
+ }
+ trace.uncompressedSize = (size_t)uncompressedSize;
+ trace.compressedSize = (size_t)compressedSize;
+ trace.dctx = dctx;
+ ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
+ }
+#else
+ (void)dctx;
+ (void)uncompressedSize;
+ (void)compressedSize;
+ (void)streaming;
+#endif
+}
+
+
+/*! ZSTD_decompressFrame() :
+ * @dctx must be properly initialized
+ * will update *srcPtr and *srcSizePtr,
+ * to make *srcPtr progress by one frame. */
+static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void** srcPtr, size_t *srcSizePtr)
+{
+ const BYTE* const istart = (const BYTE*)(*srcPtr);
+ const BYTE* ip = istart;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
+ BYTE* op = ostart;
+ size_t remainingSrcSize = *srcSizePtr;
+
+ DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr);
+
+ /* check */
+ RETURN_ERROR_IF(
+ remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
+ srcSize_wrong, "");
+
+ /* Frame Header */
+ { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
+ ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
+ if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
+ RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
+ srcSize_wrong, "");
+ FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , "");
+ ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
+ }
+
+ /* Loop on each block */
+ while (1) {
+ BYTE* oBlockEnd = oend;
+ size_t decodedSize;
+ blockProperties_t blockProperties;
+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+ ip += ZSTD_blockHeaderSize;
+ remainingSrcSize -= ZSTD_blockHeaderSize;
+ RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
+
+ if (ip >= op && ip < oBlockEnd) {
+ /* We are decompressing in-place. Limit the output pointer so that we
+ * don't overwrite the block that we are currently reading. This will
+ * fail decompression if the input & output pointers aren't spaced
+ * far enough apart.
+ *
+ * This is important to set, even when the pointers are far enough
+ * apart, because ZSTD_decompressBlock_internal() can decide to store
+ * literals in the output buffer, after the block it is decompressing.
+ * Since we don't want anything to overwrite our input, we have to tell
+ * ZSTD_decompressBlock_internal to never write past ip.
+ *
+ * See ZSTD_allocateLiteralsBuffer() for reference.
+ */
+ oBlockEnd = op + (ip - op);
+ }
+
+ switch(blockProperties.blockType)
+ {
+ case bt_compressed:
+ decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming);
+ break;
+ case bt_raw :
+ /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */
+ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
+ break;
+ case bt_rle :
+ decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize);
+ break;
+ case bt_reserved :
+ default:
+ RETURN_ERROR(corruption_detected, "invalid block type");
+ }
+
+ if (ZSTD_isError(decodedSize)) return decodedSize;
+ if (dctx->validateChecksum)
+ XXH64_update(&dctx->xxhState, op, decodedSize);
+ if (decodedSize != 0)
+ op += decodedSize;
+ assert(ip != NULL);
+ ip += cBlockSize;
+ remainingSrcSize -= cBlockSize;
+ if (blockProperties.lastBlock) break;
+ }
+
+ if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+ RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
+ corruption_detected, "");
+ }
+ if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
+ RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
+ if (!dctx->forceIgnoreChecksum) {
+ U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
+ U32 checkRead;
+ checkRead = MEM_readLE32(ip);
+ RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
+ }
+ ip += 4;
+ remainingSrcSize -= 4;
+ }
+ ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
+ /* Allow caller to get size read */
+ DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr);
+ *srcPtr = ip;
+ *srcSizePtr = remainingSrcSize;
+ return (size_t)(op-ostart);
+}
+
+static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ const ZSTD_DDict* ddict)
+{
+ void* const dststart = dst;
+ int moreThan1Frame = 0;
+
+ DEBUGLOG(5, "ZSTD_decompressMultiFrame");
+ assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */
+
+ if (ddict) {
+ dict = ZSTD_DDict_dictContent(ddict);
+ dictSize = ZSTD_DDict_dictSize(ddict);
+ }
+
+ while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(src, srcSize)) {
+ size_t decodedSize;
+ size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
+ if (ZSTD_isError(frameSize)) return frameSize;
+ RETURN_ERROR_IF(dctx->staticSize, memory_allocation,
+ "legacy support is not compatible with static dctx");
+
+ decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
+ if (ZSTD_isError(decodedSize)) return decodedSize;
+
+ assert(decodedSize <= dstCapacity);
+ dst = (BYTE*)dst + decodedSize;
+ dstCapacity -= decodedSize;
+
+ src = (const BYTE*)src + frameSize;
+ srcSize -= frameSize;
+
+ continue;
+ }
+#endif
+
+ { U32 const magicNumber = MEM_readLE32(src);
+ DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
+ (unsigned)magicNumber, ZSTD_MAGICNUMBER);
+ if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ size_t const skippableSize = readSkippableFrameSize(src, srcSize);
+ FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
+ assert(skippableSize <= srcSize);
+
+ src = (const BYTE *)src + skippableSize;
+ srcSize -= skippableSize;
+ continue;
+ } }
+
+ if (ddict) {
+ /* we were called from ZSTD_decompress_usingDDict */
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), "");
+ } else {
+ /* this will initialize correctly with no dict if dict == NULL, so
+ * use this in all cases but ddict */
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
+ }
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
+
+ { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
+ &src, &srcSize);
+ RETURN_ERROR_IF(
+ (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
+ && (moreThan1Frame==1),
+ srcSize_wrong,
+ "At least one frame successfully completed, "
+ "but following bytes are garbage: "
+ "it's more likely to be a srcSize error, "
+ "specifying more input bytes than size of frame(s). "
+ "Note: one could be unlucky, it might be a corruption error instead, "
+ "happening right at the place where we expect zstd magic bytes. "
+ "But this is _much_ less likely than a srcSize field error.");
+ if (ZSTD_isError(res)) return res;
+ assert(res <= dstCapacity);
+ if (res != 0)
+ dst = (BYTE*)dst + res;
+ dstCapacity -= res;
+ }
+ moreThan1Frame = 1;
+ } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
+
+ RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
+
+ return (size_t)((BYTE*)dst - (BYTE*)dststart);
+}
+
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize)
+{
+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
+}
+
+
+static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
+{
+ switch (dctx->dictUses) {
+ default:
+ assert(0 /* Impossible */);
+ ZSTD_FALLTHROUGH;
+ case ZSTD_dont_use:
+ ZSTD_clearDict(dctx);
+ return NULL;
+ case ZSTD_use_indefinitely:
+ return dctx->ddict;
+ case ZSTD_use_once:
+ dctx->dictUses = ZSTD_dont_use;
+ return dctx->ddict;
+ }
+}
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));
+}
+
+
+size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
+ size_t regenSize;
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+ RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
+ regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
+ ZSTD_freeDCtx(dctx);
+ return regenSize;
+#else /* stack mode */
+ ZSTD_DCtx dctx;
+ ZSTD_initDCtx_internal(&dctx);
+ return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
+#endif
+}
+
+
+/*-**************************************
+* Advanced Streaming Decompression API
+* Bufferless and synchronous
+****************************************/
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
+
+/**
+ * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we
+ * allow taking a partial block as the input. Currently only raw uncompressed blocks can
+ * be streamed.
+ *
+ * For blocks that can be streamed, this allows us to reduce the latency until we produce
+ * output, and avoid copying the input.
+ *
+ * @param inputSize - The total amount of input that the caller currently has.
+ */
+static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {
+ if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock))
+ return dctx->expected;
+ if (dctx->bType != bt_raw)
+ return dctx->expected;
+ return BOUNDED(1, inputSize, dctx->expected);
+}
+
+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
+ switch(dctx->stage)
+ {
+ default: /* should not happen */
+ assert(0);
+ ZSTD_FALLTHROUGH;
+ case ZSTDds_getFrameHeaderSize:
+ ZSTD_FALLTHROUGH;
+ case ZSTDds_decodeFrameHeader:
+ return ZSTDnit_frameHeader;
+ case ZSTDds_decodeBlockHeader:
+ return ZSTDnit_blockHeader;
+ case ZSTDds_decompressBlock:
+ return ZSTDnit_block;
+ case ZSTDds_decompressLastBlock:
+ return ZSTDnit_lastBlock;
+ case ZSTDds_checkChecksum:
+ return ZSTDnit_checksum;
+ case ZSTDds_decodeSkippableHeader:
+ ZSTD_FALLTHROUGH;
+ case ZSTDds_skipFrame:
+ return ZSTDnit_skippableFrame;
+ }
+}
+
+static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
+
+/** ZSTD_decompressContinue() :
+ * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
+ * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
+ * or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
+ /* Sanity check */
+ RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
+
+ dctx->processedCSize += srcSize;
+
+ switch (dctx->stage)
+ {
+ case ZSTDds_getFrameHeaderSize :
+ assert(src != NULL);
+ if (dctx->format == ZSTD_f_zstd1) { /* allows header */
+ assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
+ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
+ dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
+ dctx->stage = ZSTDds_decodeSkippableHeader;
+ return 0;
+ } }
+ dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
+ if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
+ dctx->expected = dctx->headerSize - srcSize;
+ dctx->stage = ZSTDds_decodeFrameHeader;
+ return 0;
+
+ case ZSTDds_decodeFrameHeader:
+ assert(src != NULL);
+ ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
+ FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
+ dctx->expected = ZSTD_blockHeaderSize;
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ return 0;
+
+ case ZSTDds_decodeBlockHeader:
+ { blockProperties_t bp;
+ size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
+ RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum");
+ dctx->expected = cBlockSize;
+ dctx->bType = bp.blockType;
+ dctx->rleSize = bp.origSize;
+ if (cBlockSize) {
+ dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
+ return 0;
+ }
+ /* empty block */
+ if (bp.lastBlock) {
+ if (dctx->fParams.checksumFlag) {
+ dctx->expected = 4;
+ dctx->stage = ZSTDds_checkChecksum;
+ } else {
+ dctx->expected = 0; /* end of frame */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ }
+ } else {
+ dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ }
+ return 0;
+ }
+
+ case ZSTDds_decompressLastBlock:
+ case ZSTDds_decompressBlock:
+ DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock");
+ { size_t rSize;
+ switch(dctx->bType)
+ {
+ case bt_compressed:
+ DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
+ rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
+ dctx->expected = 0; /* Streaming not supported */
+ break;
+ case bt_raw :
+ assert(srcSize <= dctx->expected);
+ rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
+ FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed");
+ assert(rSize == srcSize);
+ dctx->expected -= rSize;
+ break;
+ case bt_rle :
+ rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize);
+ dctx->expected = 0; /* Streaming not supported */
+ break;
+ case bt_reserved : /* should never happen */
+ default:
+ RETURN_ERROR(corruption_detected, "invalid block type");
+ }
+ FORWARD_IF_ERROR(rSize, "");
+ RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
+ DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
+ dctx->decodedSize += rSize;
+ if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
+ dctx->previousDstEnd = (char*)dst + rSize;
+
+ /* Stay on the same stage until we are finished streaming the block. */
+ if (dctx->expected > 0) {
+ return rSize;
+ }
+
+ if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
+ DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
+ RETURN_ERROR_IF(
+ dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+ && dctx->decodedSize != dctx->fParams.frameContentSize,
+ corruption_detected, "");
+ if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
+ dctx->expected = 4;
+ dctx->stage = ZSTDds_checkChecksum;
+ } else {
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
+ dctx->expected = 0; /* ends here */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ }
+ } else {
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ dctx->expected = ZSTD_blockHeaderSize;
+ }
+ return rSize;
+ }
+
+ case ZSTDds_checkChecksum:
+ assert(srcSize == 4); /* guaranteed by dctx->expected */
+ {
+ if (dctx->validateChecksum) {
+ U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
+ U32 const check32 = MEM_readLE32(src);
+ DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
+ RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
+ }
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
+ dctx->expected = 0;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ return 0;
+ }
+
+ case ZSTDds_decodeSkippableHeader:
+ assert(src != NULL);
+ assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
+ ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
+ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
+ dctx->stage = ZSTDds_skipFrame;
+ return 0;
+
+ case ZSTDds_skipFrame:
+ dctx->expected = 0;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ return 0;
+
+ default:
+ assert(0); /* impossible */
+ RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */
+ }
+}
+
+
+static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ dctx->dictEnd = dctx->previousDstEnd;
+ dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
+ dctx->prefixStart = dict;
+ dctx->previousDstEnd = (const char*)dict + dictSize;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ dctx->dictContentBeginForFuzzing = dctx->prefixStart;
+ dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
+#endif
+ return 0;
+}
+
+/*! ZSTD_loadDEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary.
+ * @return : size of entropy tables read */
+size_t
+ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
+ const void* const dict, size_t const dictSize)
+{
+ const BYTE* dictPtr = (const BYTE*)dict;
+ const BYTE* const dictEnd = dictPtr + dictSize;
+
+ RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small");
+ assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
+ dictPtr += 8; /* skip header = magic + dictID */
+
+ ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable));
+ ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable));
+ ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE);
+ { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */
+ size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
+#ifdef HUF_FORCE_DECOMPRESS_X1
+ /* in minimal huffman, we always use X1 variants */
+ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
+ dictPtr, dictEnd - dictPtr,
+ workspace, workspaceSize, /* flags */ 0);
+#else
+ size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
+ dictPtr, (size_t)(dictEnd - dictPtr),
+ workspace, workspaceSize, /* flags */ 0);
+#endif
+ RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
+ dictPtr += hSize;
+ }
+
+ { short offcodeNCount[MaxOff+1];
+ unsigned offcodeMaxValue = MaxOff, offcodeLog;
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
+ RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
+ RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
+ ZSTD_buildFSETable( entropy->OFTable,
+ offcodeNCount, offcodeMaxValue,
+ OF_base, OF_bits,
+ offcodeLog,
+ entropy->workspace, sizeof(entropy->workspace),
+ /* bmi2 */0);
+ dictPtr += offcodeHeaderSize;
+ }
+
+ { short matchlengthNCount[MaxML+1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
+ RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
+ RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
+ ZSTD_buildFSETable( entropy->MLTable,
+ matchlengthNCount, matchlengthMaxValue,
+ ML_base, ML_bits,
+ matchlengthLog,
+ entropy->workspace, sizeof(entropy->workspace),
+ /* bmi2 */ 0);
+ dictPtr += matchlengthHeaderSize;
+ }
+
+ { short litlengthNCount[MaxLL+1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
+ RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
+ RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
+ ZSTD_buildFSETable( entropy->LLTable,
+ litlengthNCount, litlengthMaxValue,
+ LL_base, LL_bits,
+ litlengthLog,
+ entropy->workspace, sizeof(entropy->workspace),
+ /* bmi2 */ 0);
+ dictPtr += litlengthHeaderSize;
+ }
+
+ RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
+ { int i;
+ size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
+ for (i=0; i<3; i++) {
+ U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
+ RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
+ dictionary_corrupted, "");
+ entropy->rep[i] = rep;
+ } }
+
+ return (size_t)(dictPtr - (const BYTE*)dict);
+}
+
+static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
+ { U32 const magic = MEM_readLE32(dict);
+ if (magic != ZSTD_MAGIC_DICTIONARY) {
+ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
+ } }
+ dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
+
+ /* load entropy tables */
+ { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
+ RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, "");
+ dict = (const char*)dict + eSize;
+ dictSize -= eSize;
+ }
+ dctx->litEntropy = dctx->fseEntropy = 1;
+
+ /* reference dictionary content */
+ return ZSTD_refDictContent(dctx, dict, dictSize);
+}
+
+size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
+{
+ assert(dctx != NULL);
+#if ZSTD_TRACE
+ dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
+#endif
+ dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ dctx->processedCSize = 0;
+ dctx->decodedSize = 0;
+ dctx->previousDstEnd = NULL;
+ dctx->prefixStart = NULL;
+ dctx->virtualStart = NULL;
+ dctx->dictEnd = NULL;
+ dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */
+ dctx->litEntropy = dctx->fseEntropy = 0;
+ dctx->dictID = 0;
+ dctx->bType = bt_reserved;
+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
+ ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
+ dctx->LLTptr = dctx->entropy.LLTable;
+ dctx->MLTptr = dctx->entropy.MLTable;
+ dctx->OFTptr = dctx->entropy.OFTable;
+ dctx->HUFptr = dctx->entropy.hufTable;
+ return 0;
+}
+
+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
+ if (dict && dictSize)
+ RETURN_ERROR_IF(
+ ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
+ dictionary_corrupted, "");
+ return 0;
+}
+
+
+/* ====== ZSTD_DDict ====== */
+
+size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+ DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");
+ assert(dctx != NULL);
+ if (ddict) {
+ const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict);
+ size_t const dictSize = ZSTD_DDict_dictSize(ddict);
+ const void* const dictEnd = dictStart + dictSize;
+ dctx->ddictIsCold = (dctx->dictEnd != dictEnd);
+ DEBUGLOG(4, "DDict is %s",
+ dctx->ddictIsCold ? "~cold~" : "hot!");
+ }
+ FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
+ if (ddict) { /* NULL ddict is equivalent to no dictionary */
+ ZSTD_copyDDictParameters(dctx, ddict);
+ }
+ return 0;
+}
+
+/*! ZSTD_getDictID_fromDict() :
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
+{
+ if (dictSize < 8) return 0;
+ if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
+ return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
+}
+
+/*! ZSTD_getDictID_fromFrame() :
+ * Provides the dictID required to decompress frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary (most common case).
+ * - The frame was built with dictID intentionally removed.
+ * Needed dictionary is a hidden piece of information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, frame header could not be decoded.
+ * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to use
+ * ZSTD_getFrameHeader(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
+{
+ ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
+ size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
+ if (ZSTD_isError(hError)) return 0;
+ return zfp.dictID;
+}
+
+
+/*! ZSTD_decompress_usingDDict() :
+* Decompression using a pre-digested Dictionary
+* Use dictionary without significant overhead. */
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_DDict* ddict)
+{
+ /* pass content and size in case legacy frames are encountered */
+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
+ NULL, 0,
+ ddict);
+}
+
+
+/*=====================================
+* Streaming decompression
+*====================================*/
+
+ZSTD_DStream* ZSTD_createDStream(void)
+{
+ DEBUGLOG(3, "ZSTD_createDStream");
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+}
+
+ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
+{
+ return ZSTD_initStaticDCtx(workspace, workspaceSize);
+}
+
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
+{
+ return ZSTD_createDCtx_internal(customMem);
+}
+
+size_t ZSTD_freeDStream(ZSTD_DStream* zds)
+{
+ return ZSTD_freeDCtx(zds);
+}
+
+
+/* *** Initialization *** */
+
+size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
+
+size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ ZSTD_clearDict(dctx);
+ if (dict && dictSize != 0) {
+ dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
+ RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");
+ dctx->ddict = dctx->ddictLocal;
+ dctx->dictUses = ZSTD_use_indefinitely;
+ }
+ return 0;
+}
+
+size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
+}
+
+size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
+}
+
+size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
+{
+ FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");
+ dctx->dictUses = ZSTD_use_once;
+ return 0;
+}
+
+size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
+{
+ return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent);
+}
+
+
+/* ZSTD_initDStream_usingDict() :
+ * return : expected size, aka ZSTD_startingInputLength().
+ * this function cannot fail */
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
+{
+ DEBUGLOG(4, "ZSTD_initDStream_usingDict");
+ FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");
+ return ZSTD_startingInputLength(zds->format);
+}
+
+/* note : this variant can't fail */
+size_t ZSTD_initDStream(ZSTD_DStream* zds)
+{
+ DEBUGLOG(4, "ZSTD_initDStream");
+ FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), "");
+ FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), "");
+ return ZSTD_startingInputLength(zds->format);
+}
+
+/* ZSTD_initDStream_usingDDict() :
+ * ddict will just be referenced, and must outlive decompression session
+ * this function cannot fail */
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
+{
+ DEBUGLOG(4, "ZSTD_initDStream_usingDDict");
+ FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
+ return ZSTD_startingInputLength(dctx->format);
+}
+
+/* ZSTD_resetDStream() :
+ * return : expected size, aka ZSTD_startingInputLength().
+ * this function cannot fail */
+size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
+{
+ DEBUGLOG(4, "ZSTD_resetDStream");
+ FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
+ return ZSTD_startingInputLength(dctx->format);
+}
+
+
+size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ ZSTD_clearDict(dctx);
+ if (ddict) {
+ dctx->ddict = ddict;
+ dctx->dictUses = ZSTD_use_indefinitely;
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
+ if (dctx->ddictSet == NULL) {
+ dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
+ if (!dctx->ddictSet) {
+ RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
+ }
+ }
+ assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
+ }
+ }
+ return 0;
+}
+
+/* ZSTD_DCtx_setMaxWindowSize() :
+ * note : no direct equivalence in ZSTD_DCtx_setParameter,
+ * since this version sets windowSize, and the other sets windowLog */
+size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
+{
+ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
+ size_t const min = (size_t)1 << bounds.lowerBound;
+ size_t const max = (size_t)1 << bounds.upperBound;
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");
+ RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");
+ dctx->maxWindowSize = maxWindowSize;
+ return 0;
+}
+
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
+{
+ return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format);
+}
+
+ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
+{
+ ZSTD_bounds bounds = { 0, 0, 0 };
+ switch(dParam) {
+ case ZSTD_d_windowLogMax:
+ bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ bounds.upperBound = ZSTD_WINDOWLOG_MAX;
+ return bounds;
+ case ZSTD_d_format:
+ bounds.lowerBound = (int)ZSTD_f_zstd1;
+ bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
+ ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
+ return bounds;
+ case ZSTD_d_stableOutBuffer:
+ bounds.lowerBound = (int)ZSTD_bm_buffered;
+ bounds.upperBound = (int)ZSTD_bm_stable;
+ return bounds;
+ case ZSTD_d_forceIgnoreChecksum:
+ bounds.lowerBound = (int)ZSTD_d_validateChecksum;
+ bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
+ return bounds;
+ case ZSTD_d_refMultipleDDicts:
+ bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
+ bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
+ return bounds;
+ case ZSTD_d_disableHuffmanAssembly:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ default:;
+ }
+ bounds.error = ERROR(parameter_unsupported);
+ return bounds;
+}
+
+/* ZSTD_dParam_withinBounds:
+ * @return 1 if value is within dParam bounds,
+ * 0 otherwise */
+static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
+{
+ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);
+ if (ZSTD_isError(bounds.error)) return 0;
+ if (value < bounds.lowerBound) return 0;
+ if (value > bounds.upperBound) return 0;
+ return 1;
+}
+
+#define CHECK_DBOUNDS(p,v) { \
+ RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
+}
+
+size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value)
+{
+ switch (param) {
+ case ZSTD_d_windowLogMax:
+ *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize);
+ return 0;
+ case ZSTD_d_format:
+ *value = (int)dctx->format;
+ return 0;
+ case ZSTD_d_stableOutBuffer:
+ *value = (int)dctx->outBufferMode;
+ return 0;
+ case ZSTD_d_forceIgnoreChecksum:
+ *value = (int)dctx->forceIgnoreChecksum;
+ return 0;
+ case ZSTD_d_refMultipleDDicts:
+ *value = (int)dctx->refMultipleDDicts;
+ return 0;
+ case ZSTD_d_disableHuffmanAssembly:
+ *value = (int)dctx->disableHufAsm;
+ return 0;
+ default:;
+ }
+ RETURN_ERROR(parameter_unsupported, "");
+}
+
+size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
+{
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ switch(dParam) {
+ case ZSTD_d_windowLogMax:
+ if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
+ CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
+ dctx->maxWindowSize = ((size_t)1) << value;
+ return 0;
+ case ZSTD_d_format:
+ CHECK_DBOUNDS(ZSTD_d_format, value);
+ dctx->format = (ZSTD_format_e)value;
+ return 0;
+ case ZSTD_d_stableOutBuffer:
+ CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
+ dctx->outBufferMode = (ZSTD_bufferMode_e)value;
+ return 0;
+ case ZSTD_d_forceIgnoreChecksum:
+ CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
+ dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
+ return 0;
+ case ZSTD_d_refMultipleDDicts:
+ CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
+ if (dctx->staticSize != 0) {
+ RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
+ }
+ dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
+ return 0;
+ case ZSTD_d_disableHuffmanAssembly:
+ CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value);
+ dctx->disableHufAsm = value != 0;
+ return 0;
+ default:;
+ }
+ RETURN_ERROR(parameter_unsupported, "");
+}
+
+size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
+{
+ if ( (reset == ZSTD_reset_session_only)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ dctx->streamStage = zdss_init;
+ dctx->noForwardProgress = 0;
+ }
+ if ( (reset == ZSTD_reset_parameters)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ ZSTD_clearDict(dctx);
+ ZSTD_DCtx_resetParameters(dctx);
+ }
+ return 0;
+}
+
+
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
+{
+ return ZSTD_sizeof_DCtx(dctx);
+}
+
+size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
+{
+ size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+ /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
+ unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
+ unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
+ size_t const minRBSize = (size_t) neededSize;
+ RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
+ frameParameter_windowTooLarge, "");
+ return minRBSize;
+}
+
+size_t ZSTD_estimateDStreamSize(size_t windowSize)
+{
+ size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+ size_t const inBuffSize = blockSize; /* no block can be larger */
+ size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN);
+ return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
+}
+
+size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
+{
+ U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
+ ZSTD_frameHeader zfh;
+ size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
+ if (ZSTD_isError(err)) return err;
+ RETURN_ERROR_IF(err>0, srcSize_wrong, "");
+ RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
+ frameParameter_windowTooLarge, "");
+ return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
+}
+
+
+/* ***** Decompression ***** */
+
+static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
+{
+ return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;
+}
+
+static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
+{
+ if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
+ zds->oversizedDuration++;
+ else
+ zds->oversizedDuration = 0;
+}
+
+static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds)
+{
+ return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;
+}
+
+/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */
+static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)
+{
+ ZSTD_outBuffer const expect = zds->expectedOutBuffer;
+ /* No requirement when ZSTD_obm_stable is not enabled. */
+ if (zds->outBufferMode != ZSTD_bm_stable)
+ return 0;
+ /* Any buffer is allowed in zdss_init, this must be the same for every other call until
+ * the context is reset.
+ */
+ if (zds->streamStage == zdss_init)
+ return 0;
+ /* The buffer must match our expectation exactly. */
+ if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
+ return 0;
+ RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
+}
+
+/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
+ * and updates the stage and the output buffer state. This call is extracted so it can be
+ * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.
+ * NOTE: You must break after calling this function since the streamStage is modified.
+ */
+static size_t ZSTD_decompressContinueStream(
+ ZSTD_DStream* zds, char** op, char* oend,
+ void const* src, size_t srcSize) {
+ int const isSkipFrame = ZSTD_isSkipFrame(zds);
+ if (zds->outBufferMode == ZSTD_bm_buffered) {
+ size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
+ size_t const decodedSize = ZSTD_decompressContinue(zds,
+ zds->outBuff + zds->outStart, dstSize, src, srcSize);
+ FORWARD_IF_ERROR(decodedSize, "");
+ if (!decodedSize && !isSkipFrame) {
+ zds->streamStage = zdss_read;
+ } else {
+ zds->outEnd = zds->outStart + decodedSize;
+ zds->streamStage = zdss_flush;
+ }
+ } else {
+ /* Write directly into the output buffer */
+ size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
+ size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
+ FORWARD_IF_ERROR(decodedSize, "");
+ *op += decodedSize;
+ /* Flushing is not needed. */
+ zds->streamStage = zdss_read;
+ assert(*op <= oend);
+ assert(zds->outBufferMode == ZSTD_bm_stable);
+ }
+ return 0;
+}
+
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+ const char* const src = (const char*)input->src;
+ const char* const istart = input->pos != 0 ? src + input->pos : src;
+ const char* const iend = input->size != 0 ? src + input->size : src;
+ const char* ip = istart;
+ char* const dst = (char*)output->dst;
+ char* const ostart = output->pos != 0 ? dst + output->pos : dst;
+ char* const oend = output->size != 0 ? dst + output->size : dst;
+ char* op = ostart;
+ U32 someMoreWork = 1;
+
+ DEBUGLOG(5, "ZSTD_decompressStream");
+ RETURN_ERROR_IF(
+ input->pos > input->size,
+ srcSize_wrong,
+ "forbidden. in: pos: %u vs size: %u",
+ (U32)input->pos, (U32)input->size);
+ RETURN_ERROR_IF(
+ output->pos > output->size,
+ dstSize_tooSmall,
+ "forbidden. out: pos: %u vs size: %u",
+ (U32)output->pos, (U32)output->size);
+ DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
+ FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), "");
+
+ while (someMoreWork) {
+ switch(zds->streamStage)
+ {
+ case zdss_init :
+ DEBUGLOG(5, "stage zdss_init => transparent reset ");
+ zds->streamStage = zdss_loadHeader;
+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ zds->legacyVersion = 0;
+#endif
+ zds->hostageByte = 0;
+ zds->expectedOutBuffer = *output;
+ ZSTD_FALLTHROUGH;
+
+ case zdss_loadHeader :
+ DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ if (zds->legacyVersion) {
+ RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+ "legacy support is incompatible with static dctx");
+ { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
+ if (hint==0) zds->streamStage = zdss_init;
+ return hint;
+ } }
+#endif
+ { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
+ if (zds->refMultipleDDicts && zds->ddictSet) {
+ ZSTD_DCtx_selectFrameDDict(zds);
+ }
+ if (ZSTD_isError(hSize)) {
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
+ if (legacyVersion) {
+ ZSTD_DDict const* const ddict = ZSTD_getDDict(zds);
+ const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL;
+ size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0;
+ DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
+ RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+ "legacy support is incompatible with static dctx");
+ FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
+ zds->previousLegacyVersion, legacyVersion,
+ dict, dictSize), "");
+ zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
+ { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
+ if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */
+ return hint;
+ } }
+#endif
+ return hSize; /* error */
+ }
+ if (hSize != 0) { /* need more input */
+ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
+ size_t const remainingInput = (size_t)(iend-ip);
+ assert(iend >= ip);
+ if (toLoad > remainingInput) { /* not enough input to load full header */
+ if (remainingInput > 0) {
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
+ zds->lhSize += remainingInput;
+ }
+ input->pos = input->size;
+ /* check first few bytes */
+ FORWARD_IF_ERROR(
+ ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format),
+ "First few bytes detected incorrect" );
+ /* return hint input size */
+ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
+ }
+ assert(ip != NULL);
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
+ break;
+ } }
+
+ /* check for single-pass mode opportunity */
+ if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+ && zds->fParams.frameType != ZSTD_skippableFrame
+ && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
+ size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart));
+ if (cSize <= (size_t)(iend-istart)) {
+ /* shortcut : using single-pass mode */
+ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
+ if (ZSTD_isError(decompressedSize)) return decompressedSize;
+ DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
+ assert(istart != NULL);
+ ip = istart + cSize;
+ op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
+ zds->expected = 0;
+ zds->streamStage = zdss_init;
+ someMoreWork = 0;
+ break;
+ } }
+
+ /* Check output buffer is large enough for ZSTD_odm_stable. */
+ if (zds->outBufferMode == ZSTD_bm_stable
+ && zds->fParams.frameType != ZSTD_skippableFrame
+ && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+ && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
+ RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small");
+ }
+
+ /* Consume header (see ZSTDds_decodeFrameHeader) */
+ DEBUGLOG(4, "Consume header");
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
+
+ if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
+ zds->stage = ZSTDds_skipFrame;
+ } else {
+ FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), "");
+ zds->expected = ZSTD_blockHeaderSize;
+ zds->stage = ZSTDds_decodeBlockHeader;
+ }
+
+ /* control buffer memory usage */
+ DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
+ (U32)(zds->fParams.windowSize >>10),
+ (U32)(zds->maxWindowSize >> 10) );
+ zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
+ RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
+ frameParameter_windowTooLarge, "");
+
+ /* Adapt buffer sizes to frame header instructions */
+ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
+ size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
+ ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
+ : 0;
+
+ ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
+
+ { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
+ int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
+
+ if (tooSmall || tooLarge) {
+ size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
+ DEBUGLOG(4, "inBuff : from %u to %u",
+ (U32)zds->inBuffSize, (U32)neededInBuffSize);
+ DEBUGLOG(4, "outBuff : from %u to %u",
+ (U32)zds->outBuffSize, (U32)neededOutBuffSize);
+ if (zds->staticSize) { /* static DCtx */
+ DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
+ assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
+ RETURN_ERROR_IF(
+ bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
+ memory_allocation, "");
+ } else {
+ ZSTD_customFree(zds->inBuff, zds->customMem);
+ zds->inBuffSize = 0;
+ zds->outBuffSize = 0;
+ zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem);
+ RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
+ }
+ zds->inBuffSize = neededInBuffSize;
+ zds->outBuff = zds->inBuff + zds->inBuffSize;
+ zds->outBuffSize = neededOutBuffSize;
+ } } }
+ zds->streamStage = zdss_read;
+ ZSTD_FALLTHROUGH;
+
+ case zdss_read:
+ DEBUGLOG(5, "stage zdss_read");
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip));
+ DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
+ if (neededInSize==0) { /* end of frame */
+ zds->streamStage = zdss_init;
+ someMoreWork = 0;
+ break;
+ }
+ if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
+ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
+ assert(ip != NULL);
+ ip += neededInSize;
+ /* Function modifies the stage so we must break */
+ break;
+ } }
+ if (ip==iend) { someMoreWork = 0; break; } /* no more input */
+ zds->streamStage = zdss_load;
+ ZSTD_FALLTHROUGH;
+
+ case zdss_load:
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
+ size_t const toLoad = neededInSize - zds->inPos;
+ int const isSkipFrame = ZSTD_isSkipFrame(zds);
+ size_t loadedSize;
+ /* At this point we shouldn't be decompressing a block that we can stream. */
+ assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)));
+ if (isSkipFrame) {
+ loadedSize = MIN(toLoad, (size_t)(iend-ip));
+ } else {
+ RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
+ corruption_detected,
+ "should never happen");
+ loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
+ }
+ if (loadedSize != 0) {
+ /* ip may be NULL */
+ ip += loadedSize;
+ zds->inPos += loadedSize;
+ }
+ if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
+
+ /* decode loaded input */
+ zds->inPos = 0; /* input is consumed */
+ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), "");
+ /* Function modifies the stage so we must break */
+ break;
+ }
+ case zdss_flush:
+ {
+ size_t const toFlushSize = zds->outEnd - zds->outStart;
+ size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
+
+ op = op ? op + flushedSize : op;
+
+ zds->outStart += flushedSize;
+ if (flushedSize == toFlushSize) { /* flush completed */
+ zds->streamStage = zdss_read;
+ if ( (zds->outBuffSize < zds->fParams.frameContentSize)
+ && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
+ DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
+ (int)(zds->outBuffSize - zds->outStart),
+ (U32)zds->fParams.blockSizeMax);
+ zds->outStart = zds->outEnd = 0;
+ }
+ break;
+ } }
+ /* cannot complete flush */
+ someMoreWork = 0;
+ break;
+
+ default:
+ assert(0); /* impossible */
+ RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */
+ } }
+
+ /* result */
+ input->pos = (size_t)(ip - (const char*)(input->src));
+ output->pos = (size_t)(op - (char*)(output->dst));
+
+ /* Update the expected output buffer for ZSTD_obm_stable. */
+ zds->expectedOutBuffer = *output;
+
+ if ((ip==istart) && (op==ostart)) { /* no forward progress */
+ zds->noForwardProgress ++;
+ if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
+ RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, "");
+ RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, "");
+ assert(0);
+ }
+ } else {
+ zds->noForwardProgress = 0;
+ }
+ { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
+ if (!nextSrcSizeHint) { /* frame fully decoded */
+ if (zds->outEnd == zds->outStart) { /* output fully flushed */
+ if (zds->hostageByte) {
+ if (input->pos >= input->size) {
+ /* can't release hostage (not present) */
+ zds->streamStage = zdss_read;
+ return 1;
+ }
+ input->pos++; /* release hostage */
+ } /* zds->hostageByte */
+ return 0;
+ } /* zds->outEnd == zds->outStart */
+ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
+ zds->hostageByte=1;
+ }
+ return 1;
+ } /* nextSrcSizeHint==0 */
+ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
+ assert(zds->inPos <= nextSrcSizeHint);
+ nextSrcSizeHint -= zds->inPos; /* part already loaded*/
+ return nextSrcSizeHint;
+ }
+}
+
+size_t ZSTD_decompressStream_simpleArgs (
+ ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos)
+{
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ output.dst = dst;
+ output.size = dstCapacity;
+ output.pos = *dstPos;
+ input.src = src;
+ input.size = srcSize;
+ input.pos = *srcPos;
+ { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
+ *dstPos = output.pos;
+ *srcPos = input.pos;
+ return cErr;
+ }
+}
diff --git a/contrib/zstd/zstd_decompress_block.c b/contrib/zstd/zstd_decompress_block.c
new file mode 100644
index 0000000..add277e
--- /dev/null
+++ b/contrib/zstd/zstd_decompress_block.c
@@ -0,0 +1,2193 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* zstd_decompress_block :
+ * this module takes care of decompressing _compressed_ block */
+
+/*-*******************************************************
+* Dependencies
+*********************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
+#include "compiler.h" /* prefetch */
+#include "cpu.h" /* bmi2 */
+#include "mem.h" /* low level memory routines */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#include "zstd_internal.h"
+#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
+#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
+#include "zstd_decompress_block.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+/*_*******************************************************
+* Macros
+**********************************************************/
+
+/* These two optional macros force the use one way or another of the two
+ * ZSTD_decompressSequences implementations. You can't force in both directions
+ * at the same time.
+ */
+#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+ defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!"
+#endif
+
+
+/*_*******************************************************
+* Memory operations
+**********************************************************/
+static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
+
+
+/*-*************************************************************
+ * Block decoding
+ ***************************************************************/
+
+/*! ZSTD_getcBlockSize() :
+ * Provides the size of compressed block from block header `src` */
+size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
+ blockProperties_t* bpPtr)
+{
+ RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");
+
+ { U32 const cBlockHeader = MEM_readLE24(src);
+ U32 const cSize = cBlockHeader >> 3;
+ bpPtr->lastBlock = cBlockHeader & 1;
+ bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
+ bpPtr->origSize = cSize; /* only useful for RLE */
+ if (bpPtr->blockType == bt_rle) return 1;
+ RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
+ return cSize;
+ }
+}
+
+/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
+static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
+ const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
+{
+ if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH)
+ {
+ /* room for litbuffer to fit without read faulting */
+ dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH;
+ dctx->litBufferEnd = dctx->litBuffer + litSize;
+ dctx->litBufferLocation = ZSTD_in_dst;
+ }
+ else if (litSize > ZSTD_LITBUFFEREXTRASIZE)
+ {
+ /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
+ if (splitImmediately) {
+ /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
+ dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
+ dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
+ }
+ else {
+ /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */
+ dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
+ dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
+ }
+ dctx->litBufferLocation = ZSTD_split;
+ }
+ else
+ {
+ /* fits entirely within litExtraBuffer, so no split is necessary */
+ dctx->litBuffer = dctx->litExtraBuffer;
+ dctx->litBufferEnd = dctx->litBuffer + litSize;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ }
+}
+
+/* Hidden declaration for fullbench */
+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
+ const void* src, size_t srcSize,
+ void* dst, size_t dstCapacity, const streaming_operation streaming);
+/*! ZSTD_decodeLiteralsBlock() :
+ * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
+ * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current
+ * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
+ * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
+ *
+ * @return : nb of bytes read from src (< srcSize )
+ * note : symbol not declared but exposed for fullbench */
+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
+ const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */
+ void* dst, size_t dstCapacity, const streaming_operation streaming)
+{
+ DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
+ RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
+
+ { const BYTE* const istart = (const BYTE*) src;
+ symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
+
+ switch(litEncType)
+ {
+ case set_repeat:
+ DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
+ RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, "");
+ ZSTD_FALLTHROUGH;
+
+ case set_compressed:
+ RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3");
+ { size_t lhSize, litSize, litCSize;
+ U32 singleStream=0;
+ U32 const lhlCode = (istart[0] >> 2) & 3;
+ U32 const lhc = MEM_readLE32(istart);
+ size_t hufSuccess;
+ size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+ int const flags = 0
+ | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
+ | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
+ switch(lhlCode)
+ {
+ case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
+ /* 2 - 2 - 10 - 10 */
+ singleStream = !lhlCode;
+ lhSize = 3;
+ litSize = (lhc >> 4) & 0x3FF;
+ litCSize = (lhc >> 14) & 0x3FF;
+ break;
+ case 2:
+ /* 2 - 2 - 14 - 14 */
+ lhSize = 4;
+ litSize = (lhc >> 4) & 0x3FFF;
+ litCSize = lhc >> 18;
+ break;
+ case 3:
+ /* 2 - 2 - 18 - 18 */
+ lhSize = 5;
+ litSize = (lhc >> 4) & 0x3FFFF;
+ litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
+ break;
+ }
+ RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+ RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+ if (!singleStream)
+ RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
+ "Not enough literals (%zu) for the 4-streams mode (min %u)",
+ litSize, MIN_LITERALS_FOR_4_STREAMS);
+ RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
+ RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
+ ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
+
+ /* prefetch huffman table if cold */
+ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
+ PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
+ }
+
+ if (litEncType==set_repeat) {
+ if (singleStream) {
+ hufSuccess = HUF_decompress1X_usingDTable(
+ dctx->litBuffer, litSize, istart+lhSize, litCSize,
+ dctx->HUFptr, flags);
+ } else {
+ assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
+ hufSuccess = HUF_decompress4X_usingDTable(
+ dctx->litBuffer, litSize, istart+lhSize, litCSize,
+ dctx->HUFptr, flags);
+ }
+ } else {
+ if (singleStream) {
+#if defined(HUF_FORCE_DECOMPRESS_X2)
+ hufSuccess = HUF_decompress1X_DCtx_wksp(
+ dctx->entropy.hufTable, dctx->litBuffer, litSize,
+ istart+lhSize, litCSize, dctx->workspace,
+ sizeof(dctx->workspace), flags);
+#else
+ hufSuccess = HUF_decompress1X1_DCtx_wksp(
+ dctx->entropy.hufTable, dctx->litBuffer, litSize,
+ istart+lhSize, litCSize, dctx->workspace,
+ sizeof(dctx->workspace), flags);
+#endif
+ } else {
+ hufSuccess = HUF_decompress4X_hufOnly_wksp(
+ dctx->entropy.hufTable, dctx->litBuffer, litSize,
+ istart+lhSize, litCSize, dctx->workspace,
+ sizeof(dctx->workspace), flags);
+ }
+ }
+ if (dctx->litBufferLocation == ZSTD_split)
+ {
+ ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
+ ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
+ dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
+ dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
+ }
+
+ RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
+
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ dctx->litEntropy = 1;
+ if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
+ return litCSize + lhSize;
+ }
+
+ case set_basic:
+ { size_t litSize, lhSize;
+ U32 const lhlCode = ((istart[0]) >> 2) & 3;
+ size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+ switch(lhlCode)
+ {
+ case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
+ lhSize = 1;
+ litSize = istart[0] >> 3;
+ break;
+ case 1:
+ lhSize = 2;
+ litSize = MEM_readLE16(istart) >> 4;
+ break;
+ case 3:
+ lhSize = 3;
+ RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3");
+ litSize = MEM_readLE24(istart) >> 4;
+ break;
+ }
+
+ RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+ RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
+ ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
+ if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
+ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
+ if (dctx->litBufferLocation == ZSTD_split)
+ {
+ ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
+ ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
+ }
+ else
+ {
+ ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
+ }
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ return lhSize+litSize;
+ }
+ /* direct reference into compressed stream */
+ dctx->litPtr = istart+lhSize;
+ dctx->litSize = litSize;
+ dctx->litBufferEnd = dctx->litPtr + litSize;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ return lhSize+litSize;
+ }
+
+ case set_rle:
+ { U32 const lhlCode = ((istart[0]) >> 2) & 3;
+ size_t litSize, lhSize;
+ size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+ switch(lhlCode)
+ {
+ case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
+ lhSize = 1;
+ litSize = istart[0] >> 3;
+ break;
+ case 1:
+ lhSize = 2;
+ RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3");
+ litSize = MEM_readLE16(istart) >> 4;
+ break;
+ case 3:
+ lhSize = 3;
+ RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4");
+ litSize = MEM_readLE24(istart) >> 4;
+ break;
+ }
+ RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+ RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+ RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
+ ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
+ if (dctx->litBufferLocation == ZSTD_split)
+ {
+ ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
+ ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
+ }
+ else
+ {
+ ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
+ }
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ return lhSize+1;
+ }
+ default:
+ RETURN_ERROR(corruption_detected, "impossible");
+ }
+ }
+}
+
+/* Default FSE distribution tables.
+ * These are pre-calculated FSE decoding tables using default distributions as defined in specification :
+ * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
+ * They were generated programmatically with following method :
+ * - start from default distributions, present in /lib/common/zstd_internal.h
+ * - generate tables normally, using ZSTD_buildFSETable()
+ * - printout the content of tables
+ * - pretify output, report below, test with fuzzer to ensure it's correct */
+
+/* Default FSE distribution table for Literal Lengths */
+static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
+ { 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
+ /* nextState, nbAddBits, nbBits, baseVal */
+ { 0, 0, 4, 0}, { 16, 0, 4, 0},
+ { 32, 0, 5, 1}, { 0, 0, 5, 3},
+ { 0, 0, 5, 4}, { 0, 0, 5, 6},
+ { 0, 0, 5, 7}, { 0, 0, 5, 9},
+ { 0, 0, 5, 10}, { 0, 0, 5, 12},
+ { 0, 0, 6, 14}, { 0, 1, 5, 16},
+ { 0, 1, 5, 20}, { 0, 1, 5, 22},
+ { 0, 2, 5, 28}, { 0, 3, 5, 32},
+ { 0, 4, 5, 48}, { 32, 6, 5, 64},
+ { 0, 7, 5, 128}, { 0, 8, 6, 256},
+ { 0, 10, 6, 1024}, { 0, 12, 6, 4096},
+ { 32, 0, 4, 0}, { 0, 0, 4, 1},
+ { 0, 0, 5, 2}, { 32, 0, 5, 4},
+ { 0, 0, 5, 5}, { 32, 0, 5, 7},
+ { 0, 0, 5, 8}, { 32, 0, 5, 10},
+ { 0, 0, 5, 11}, { 0, 0, 6, 13},
+ { 32, 1, 5, 16}, { 0, 1, 5, 18},
+ { 32, 1, 5, 22}, { 0, 2, 5, 24},
+ { 32, 3, 5, 32}, { 0, 3, 5, 40},
+ { 0, 6, 4, 64}, { 16, 6, 4, 64},
+ { 32, 7, 5, 128}, { 0, 9, 6, 512},
+ { 0, 11, 6, 2048}, { 48, 0, 4, 0},
+ { 16, 0, 4, 1}, { 32, 0, 5, 2},
+ { 32, 0, 5, 3}, { 32, 0, 5, 5},
+ { 32, 0, 5, 6}, { 32, 0, 5, 8},
+ { 32, 0, 5, 9}, { 32, 0, 5, 11},
+ { 32, 0, 5, 12}, { 0, 0, 6, 15},
+ { 32, 1, 5, 18}, { 32, 1, 5, 20},
+ { 32, 2, 5, 24}, { 32, 2, 5, 28},
+ { 32, 3, 5, 40}, { 32, 4, 5, 48},
+ { 0, 16, 6,65536}, { 0, 15, 6,32768},
+ { 0, 14, 6,16384}, { 0, 13, 6, 8192},
+}; /* LL_defaultDTable */
+
+/* Default FSE distribution table for Offset Codes */
+static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
+ { 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
+ /* nextState, nbAddBits, nbBits, baseVal */
+ { 0, 0, 5, 0}, { 0, 6, 4, 61},
+ { 0, 9, 5, 509}, { 0, 15, 5,32765},
+ { 0, 21, 5,2097149}, { 0, 3, 5, 5},
+ { 0, 7, 4, 125}, { 0, 12, 5, 4093},
+ { 0, 18, 5,262141}, { 0, 23, 5,8388605},
+ { 0, 5, 5, 29}, { 0, 8, 4, 253},
+ { 0, 14, 5,16381}, { 0, 20, 5,1048573},
+ { 0, 2, 5, 1}, { 16, 7, 4, 125},
+ { 0, 11, 5, 2045}, { 0, 17, 5,131069},
+ { 0, 22, 5,4194301}, { 0, 4, 5, 13},
+ { 16, 8, 4, 253}, { 0, 13, 5, 8189},
+ { 0, 19, 5,524285}, { 0, 1, 5, 1},
+ { 16, 6, 4, 61}, { 0, 10, 5, 1021},
+ { 0, 16, 5,65533}, { 0, 28, 5,268435453},
+ { 0, 27, 5,134217725}, { 0, 26, 5,67108861},
+ { 0, 25, 5,33554429}, { 0, 24, 5,16777213},
+}; /* OF_defaultDTable */
+
+
+/* Default FSE distribution table for Match Lengths */
+static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
+ { 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
+ /* nextState, nbAddBits, nbBits, baseVal */
+ { 0, 0, 6, 3}, { 0, 0, 4, 4},
+ { 32, 0, 5, 5}, { 0, 0, 5, 6},
+ { 0, 0, 5, 8}, { 0, 0, 5, 9},
+ { 0, 0, 5, 11}, { 0, 0, 6, 13},
+ { 0, 0, 6, 16}, { 0, 0, 6, 19},
+ { 0, 0, 6, 22}, { 0, 0, 6, 25},
+ { 0, 0, 6, 28}, { 0, 0, 6, 31},
+ { 0, 0, 6, 34}, { 0, 1, 6, 37},
+ { 0, 1, 6, 41}, { 0, 2, 6, 47},
+ { 0, 3, 6, 59}, { 0, 4, 6, 83},
+ { 0, 7, 6, 131}, { 0, 9, 6, 515},
+ { 16, 0, 4, 4}, { 0, 0, 4, 5},
+ { 32, 0, 5, 6}, { 0, 0, 5, 7},
+ { 32, 0, 5, 9}, { 0, 0, 5, 10},
+ { 0, 0, 6, 12}, { 0, 0, 6, 15},
+ { 0, 0, 6, 18}, { 0, 0, 6, 21},
+ { 0, 0, 6, 24}, { 0, 0, 6, 27},
+ { 0, 0, 6, 30}, { 0, 0, 6, 33},
+ { 0, 1, 6, 35}, { 0, 1, 6, 39},
+ { 0, 2, 6, 43}, { 0, 3, 6, 51},
+ { 0, 4, 6, 67}, { 0, 5, 6, 99},
+ { 0, 8, 6, 259}, { 32, 0, 4, 4},
+ { 48, 0, 4, 4}, { 16, 0, 4, 5},
+ { 32, 0, 5, 7}, { 32, 0, 5, 8},
+ { 32, 0, 5, 10}, { 32, 0, 5, 11},
+ { 0, 0, 6, 14}, { 0, 0, 6, 17},
+ { 0, 0, 6, 20}, { 0, 0, 6, 23},
+ { 0, 0, 6, 26}, { 0, 0, 6, 29},
+ { 0, 0, 6, 32}, { 0, 16, 6,65539},
+ { 0, 15, 6,32771}, { 0, 14, 6,16387},
+ { 0, 13, 6, 8195}, { 0, 12, 6, 4099},
+ { 0, 11, 6, 2051}, { 0, 10, 6, 1027},
+}; /* ML_defaultDTable */
+
+
+static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits)
+{
+ void* ptr = dt;
+ ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;
+ ZSTD_seqSymbol* const cell = dt + 1;
+
+ DTableH->tableLog = 0;
+ DTableH->fastMode = 0;
+
+ cell->nbBits = 0;
+ cell->nextState = 0;
+ assert(nbAddBits < 255);
+ cell->nbAdditionalBits = nbAddBits;
+ cell->baseValue = baseValue;
+}
+
+
+/* ZSTD_buildFSETable() :
+ * generate FSE decoding table for one symbol (ll, ml or off)
+ * cannot fail if input is valid =>
+ * all inputs are presumed validated at this stage */
+FORCE_INLINE_TEMPLATE
+void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize)
+{
+ ZSTD_seqSymbol* const tableDecode = dt+1;
+ U32 const maxSV1 = maxSymbolValue + 1;
+ U32 const tableSize = 1 << tableLog;
+
+ U16* symbolNext = (U16*)wksp;
+ BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1);
+ U32 highThreshold = tableSize - 1;
+
+
+ /* Sanity Checks */
+ assert(maxSymbolValue <= MaxSeq);
+ assert(tableLog <= MaxFSELog);
+ assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE);
+ (void)wkspSize;
+ /* Init, lay down lowprob symbols */
+ { ZSTD_seqSymbol_header DTableH;
+ DTableH.tableLog = tableLog;
+ DTableH.fastMode = 1;
+ { S16 const largeLimit= (S16)(1 << (tableLog-1));
+ U32 s;
+ for (s=0; s<maxSV1; s++) {
+ if (normalizedCounter[s]==-1) {
+ tableDecode[highThreshold--].baseValue = s;
+ symbolNext[s] = 1;
+ } else {
+ if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
+ assert(normalizedCounter[s]>=0);
+ symbolNext[s] = (U16)normalizedCounter[s];
+ } } }
+ ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
+ }
+
+ /* Spread symbols */
+ assert(tableSize <= 512);
+ /* Specialized symbol spreading for the case when there are
+ * no low probability (-1 count) symbols. When compressing
+ * small blocks we avoid low probability symbols to hit this
+ * case, since header decoding speed matters more.
+ */
+ if (highThreshold == tableSize - 1) {
+ size_t const tableMask = tableSize-1;
+ size_t const step = FSE_TABLESTEP(tableSize);
+ /* First lay down the symbols in order.
+ * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
+ * misses since small blocks generally have small table logs, so nearly
+ * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
+ * our buffer to handle the over-write.
+ */
+ {
+ U64 const add = 0x0101010101010101ull;
+ size_t pos = 0;
+ U64 sv = 0;
+ U32 s;
+ for (s=0; s<maxSV1; ++s, sv += add) {
+ int i;
+ int const n = normalizedCounter[s];
+ MEM_write64(spread + pos, sv);
+ for (i = 8; i < n; i += 8) {
+ MEM_write64(spread + pos + i, sv);
+ }
+ assert(n>=0);
+ pos += (size_t)n;
+ }
+ }
+ /* Now we spread those positions across the table.
+ * The benefit of doing it in two stages is that we avoid the
+ * variable size inner loop, which caused lots of branch misses.
+ * Now we can run through all the positions without any branch misses.
+ * We unroll the loop twice, since that is what empirically worked best.
+ */
+ {
+ size_t position = 0;
+ size_t s;
+ size_t const unroll = 2;
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
+ size_t u;
+ for (u = 0; u < unroll; ++u) {
+ size_t const uPosition = (position + (u * step)) & tableMask;
+ tableDecode[uPosition].baseValue = spread[s + u];
+ }
+ position = (position + (unroll * step)) & tableMask;
+ }
+ assert(position == 0);
+ }
+ } else {
+ U32 const tableMask = tableSize-1;
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 s, position = 0;
+ for (s=0; s<maxSV1; s++) {
+ int i;
+ int const n = normalizedCounter[s];
+ for (i=0; i<n; i++) {
+ tableDecode[position].baseValue = s;
+ position = (position + step) & tableMask;
+ while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */
+ } }
+ assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
+ }
+
+ /* Build Decoding table */
+ {
+ U32 u;
+ for (u=0; u<tableSize; u++) {
+ U32 const symbol = tableDecode[u].baseValue;
+ U32 const nextState = symbolNext[symbol]++;
+ tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
+ tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
+ assert(nbAdditionalBits[symbol] < 255);
+ tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
+ tableDecode[u].baseValue = baseValue[symbol];
+ }
+ }
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize)
+{
+ ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize)
+{
+ ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+#endif
+
+void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+ return;
+ }
+#endif
+ (void)bmi2;
+ ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+
+
+/*! ZSTD_buildSeqTable() :
+ * @return : nb bytes read from src,
+ * or an error code if it fails */
+static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
+ symbolEncodingType_e type, unsigned max, U32 maxLog,
+ const void* src, size_t srcSize,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
+ int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize,
+ int bmi2)
+{
+ switch(type)
+ {
+ case set_rle :
+ RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");
+ RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, "");
+ { U32 const symbol = *(const BYTE*)src;
+ U32 const baseline = baseValue[symbol];
+ U8 const nbBits = nbAdditionalBits[symbol];
+ ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);
+ }
+ *DTablePtr = DTableSpace;
+ return 1;
+ case set_basic :
+ *DTablePtr = defaultTable;
+ return 0;
+ case set_repeat:
+ RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");
+ /* prefetch FSE table if used */
+ if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
+ const void* const pStart = *DTablePtr;
+ size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
+ PREFETCH_AREA(pStart, pSize);
+ }
+ return 0;
+ case set_compressed :
+ { unsigned tableLog;
+ S16 norm[MaxSeq+1];
+ size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
+ RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");
+ RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");
+ ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2);
+ *DTablePtr = DTableSpace;
+ return headerSize;
+ }
+ default :
+ assert(0);
+ RETURN_ERROR(GENERIC, "impossible");
+ }
+}
+
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+ const void* src, size_t srcSize)
+{
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* ip = istart;
+ int nbSeq;
+ DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
+
+ /* check */
+ RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, "");
+
+ /* SeqHead */
+ nbSeq = *ip++;
+ if (!nbSeq) {
+ *nbSeqPtr=0;
+ RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, "");
+ return 1;
+ }
+ if (nbSeq > 0x7F) {
+ if (nbSeq == 0xFF) {
+ RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, "");
+ nbSeq = MEM_readLE16(ip) + LONGNBSEQ;
+ ip+=2;
+ } else {
+ RETURN_ERROR_IF(ip >= iend, srcSize_wrong, "");
+ nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
+ }
+ *nbSeqPtr = nbSeq;
+
+ /* FSE table descriptors */
+ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
+ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
+ symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
+ symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
+ ip++;
+
+ /* Build DTables */
+ { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
+ LLtype, MaxLL, LLFSELog,
+ ip, iend-ip,
+ LL_base, LL_bits,
+ LL_defaultDTable, dctx->fseEntropy,
+ dctx->ddictIsCold, nbSeq,
+ dctx->workspace, sizeof(dctx->workspace),
+ ZSTD_DCtx_get_bmi2(dctx));
+ RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+ ip += llhSize;
+ }
+
+ { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
+ OFtype, MaxOff, OffFSELog,
+ ip, iend-ip,
+ OF_base, OF_bits,
+ OF_defaultDTable, dctx->fseEntropy,
+ dctx->ddictIsCold, nbSeq,
+ dctx->workspace, sizeof(dctx->workspace),
+ ZSTD_DCtx_get_bmi2(dctx));
+ RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+ ip += ofhSize;
+ }
+
+ { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
+ MLtype, MaxML, MLFSELog,
+ ip, iend-ip,
+ ML_base, ML_bits,
+ ML_defaultDTable, dctx->fseEntropy,
+ dctx->ddictIsCold, nbSeq,
+ dctx->workspace, sizeof(dctx->workspace),
+ ZSTD_DCtx_get_bmi2(dctx));
+ RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+ ip += mlhSize;
+ }
+ }
+
+ return ip-istart;
+}
+
+
+typedef struct {
+ size_t litLength;
+ size_t matchLength;
+ size_t offset;
+} seq_t;
+
+typedef struct {
+ size_t state;
+ const ZSTD_seqSymbol* table;
+} ZSTD_fseState;
+
+typedef struct {
+ BIT_DStream_t DStream;
+ ZSTD_fseState stateLL;
+ ZSTD_fseState stateOffb;
+ ZSTD_fseState stateML;
+ size_t prevOffset[ZSTD_REP_NUM];
+} seqState_t;
+
+/*! ZSTD_overlapCopy8() :
+ * Copies 8 bytes from ip to op and updates op and ip where ip <= op.
+ * If the offset is < 8 then the offset is spread to at least 8 bytes.
+ *
+ * Precondition: *ip <= *op
+ * Postcondition: *op - *op >= 8
+ */
+HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
+ assert(*ip <= *op);
+ if (offset < 8) {
+ /* close range match, overlap */
+ static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
+ int const sub2 = dec64table[offset];
+ (*op)[0] = (*ip)[0];
+ (*op)[1] = (*ip)[1];
+ (*op)[2] = (*ip)[2];
+ (*op)[3] = (*ip)[3];
+ *ip += dec32table[offset];
+ ZSTD_copy4(*op+4, *ip);
+ *ip -= sub2;
+ } else {
+ ZSTD_copy8(*op, *ip);
+ }
+ *ip += 8;
+ *op += 8;
+ assert(*op - *ip >= 8);
+}
+
+/*! ZSTD_safecopy() :
+ * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
+ * and write up to 16 bytes past oend_w (op >= oend_w is allowed).
+ * This function is only called in the uncommon case where the sequence is near the end of the block. It
+ * should be fast for a single long sequence, but can be slow for several short sequences.
+ *
+ * @param ovtype controls the overlap detection
+ * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
+ * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
+ * The src buffer must be before the dst buffer.
+ */
+static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
+ ptrdiff_t const diff = op - ip;
+ BYTE* const oend = op + length;
+
+ assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
+ (ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
+
+ if (length < 8) {
+ /* Handle short lengths. */
+ while (op < oend) *op++ = *ip++;
+ return;
+ }
+ if (ovtype == ZSTD_overlap_src_before_dst) {
+ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
+ assert(length >= 8);
+ ZSTD_overlapCopy8(&op, &ip, diff);
+ length -= 8;
+ assert(op - ip >= 8);
+ assert(op <= oend);
+ }
+
+ if (oend <= oend_w) {
+ /* No risk of overwrite. */
+ ZSTD_wildcopy(op, ip, length, ovtype);
+ return;
+ }
+ if (op <= oend_w) {
+ /* Wildcopy until we get close to the end. */
+ assert(oend > oend_w);
+ ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
+ ip += oend_w - op;
+ op += oend_w - op;
+ }
+ /* Handle the leftovers. */
+ while (op < oend) *op++ = *ip++;
+}
+
+/* ZSTD_safecopyDstBeforeSrc():
+ * This version allows overlap with dst before src, or handles the non-overlap case with dst after src
+ * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
+static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) {
+ ptrdiff_t const diff = op - ip;
+ BYTE* const oend = op + length;
+
+ if (length < 8 || diff > -8) {
+ /* Handle short lengths, close overlaps, and dst not before src. */
+ while (op < oend) *op++ = *ip++;
+ return;
+ }
+
+ if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
+ ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
+ ip += oend - WILDCOPY_OVERLENGTH - op;
+ op += oend - WILDCOPY_OVERLENGTH - op;
+ }
+
+ /* Handle the leftovers. */
+ while (op < oend) *op++ = *ip++;
+}
+
+/* ZSTD_execSequenceEnd():
+ * This version handles cases that are near the end of the output buffer. It requires
+ * more careful checks to make sure there is no overflow. By separating out these hard
+ * and unlikely cases, we can speed up the common cases.
+ *
+ * NOTE: This function needs to be fast for a single long sequence, but doesn't need
+ * to be optimized for many small sequences, since those fall into ZSTD_execSequence().
+ */
+FORCE_NOINLINE
+size_t ZSTD_execSequenceEnd(BYTE* op,
+ BYTE* const oend, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
+
+ /* bounds checks : careful of address space overflow in 32-bit mode */
+ RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
+ RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
+ assert(op < op + sequenceLength);
+ assert(oLitEnd < op + sequenceLength);
+
+ /* copy literals */
+ ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
+ op = oLitEnd;
+ *litPtr = iLitEnd;
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix */
+ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
+ match = dictEnd - (prefixStart - match);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ }
+ }
+ ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
+ return sequenceLength;
+}
+
+/* ZSTD_execSequenceEndSplitLitBuffer():
+ * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case.
+ */
+FORCE_NOINLINE
+size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
+ BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+
+ /* bounds checks : careful of address space overflow in 32-bit mode */
+ RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
+ RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
+ assert(op < op + sequenceLength);
+ assert(oLitEnd < op + sequenceLength);
+
+ /* copy literals */
+ RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
+ ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
+ op = oLitEnd;
+ *litPtr = iLitEnd;
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix */
+ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
+ match = dictEnd - (prefixStart - match);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ }
+ }
+ ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
+ return sequenceLength;
+}
+
+HINT_INLINE
+size_t ZSTD_execSequence(BYTE* op,
+ BYTE* const oend, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+ assert(op != NULL /* Precondition */);
+ assert(oend_w < oend /* No underflow */);
+
+#if defined(__aarch64__)
+ /* prefetch sequence starting from match that will be used for copy later */
+ PREFETCH_L1(match);
+#endif
+ /* Handle edge cases in a slow path:
+ * - Read beyond end of literals
+ * - Match end is within WILDCOPY_OVERLIMIT of oend
+ * - 32-bit mode and the match length overflows
+ */
+ if (UNLIKELY(
+ iLitEnd > litLimit ||
+ oMatchEnd > oend_w ||
+ (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
+ return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
+
+ /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
+ assert(op <= oLitEnd /* No overflow */);
+ assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
+ assert(oMatchEnd <= oend /* No underflow */);
+ assert(iLitEnd <= litLimit /* Literal length is in bounds */);
+ assert(oLitEnd <= oend_w /* Can wildcopy literals */);
+ assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
+
+ /* Copy Literals:
+ * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
+ * We likely don't need the full 32-byte wildcopy.
+ */
+ assert(WILDCOPY_OVERLENGTH >= 16);
+ ZSTD_copy16(op, (*litPtr));
+ if (UNLIKELY(sequence.litLength > 16)) {
+ ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
+ }
+ op = oLitEnd;
+ *litPtr = iLitEnd; /* update for next sequence */
+
+ /* Copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix -> go into extDict */
+ RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
+ match = dictEnd + (match - prefixStart);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ }
+ }
+ /* Match within prefix of 1 or more bytes */
+ assert(op <= oMatchEnd);
+ assert(oMatchEnd <= oend_w);
+ assert(match >= prefixStart);
+ assert(sequence.matchLength >= 1);
+
+ /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
+ * without overlap checking.
+ */
+ if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
+ /* We bet on a full wildcopy for matches, since we expect matches to be
+ * longer than literals (in general). In silesia, ~10% of matches are longer
+ * than 16 bytes.
+ */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
+ return sequenceLength;
+ }
+ assert(sequence.offset < WILDCOPY_VECLEN);
+
+ /* Copy 8 bytes and spread the offset to be >= 8. */
+ ZSTD_overlapCopy8(&op, &match, sequence.offset);
+
+ /* If the match length is > 8 bytes, then continue with the wildcopy. */
+ if (sequence.matchLength > 8) {
+ assert(op < oMatchEnd);
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
+ }
+ return sequenceLength;
+}
+
+HINT_INLINE
+size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
+ BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+ assert(op != NULL /* Precondition */);
+ assert(oend_w < oend /* No underflow */);
+ /* Handle edge cases in a slow path:
+ * - Read beyond end of literals
+ * - Match end is within WILDCOPY_OVERLIMIT of oend
+ * - 32-bit mode and the match length overflows
+ */
+ if (UNLIKELY(
+ iLitEnd > litLimit ||
+ oMatchEnd > oend_w ||
+ (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
+ return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
+
+ /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
+ assert(op <= oLitEnd /* No overflow */);
+ assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
+ assert(oMatchEnd <= oend /* No underflow */);
+ assert(iLitEnd <= litLimit /* Literal length is in bounds */);
+ assert(oLitEnd <= oend_w /* Can wildcopy literals */);
+ assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
+
+ /* Copy Literals:
+ * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
+ * We likely don't need the full 32-byte wildcopy.
+ */
+ assert(WILDCOPY_OVERLENGTH >= 16);
+ ZSTD_copy16(op, (*litPtr));
+ if (UNLIKELY(sequence.litLength > 16)) {
+ ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
+ }
+ op = oLitEnd;
+ *litPtr = iLitEnd; /* update for next sequence */
+
+ /* Copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix -> go into extDict */
+ RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
+ match = dictEnd + (match - prefixStart);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ } }
+ /* Match within prefix of 1 or more bytes */
+ assert(op <= oMatchEnd);
+ assert(oMatchEnd <= oend_w);
+ assert(match >= prefixStart);
+ assert(sequence.matchLength >= 1);
+
+ /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
+ * without overlap checking.
+ */
+ if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
+ /* We bet on a full wildcopy for matches, since we expect matches to be
+ * longer than literals (in general). In silesia, ~10% of matches are longer
+ * than 16 bytes.
+ */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
+ return sequenceLength;
+ }
+ assert(sequence.offset < WILDCOPY_VECLEN);
+
+ /* Copy 8 bytes and spread the offset to be >= 8. */
+ ZSTD_overlapCopy8(&op, &match, sequence.offset);
+
+ /* If the match length is > 8 bytes, then continue with the wildcopy. */
+ if (sequence.matchLength > 8) {
+ assert(op < oMatchEnd);
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
+ }
+ return sequenceLength;
+}
+
+
+static void
+ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
+{
+ const void* ptr = dt;
+ const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;
+ DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
+ DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",
+ (U32)DStatePtr->state, DTableH->tableLog);
+ BIT_reloadDStream(bitD);
+ DStatePtr->table = dt + 1;
+}
+
+FORCE_INLINE_TEMPLATE void
+ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
+{
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+ DStatePtr->state = nextState + lowBits;
+}
+
+/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
+ * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
+ * bits before reloading. This value is the maximum number of bytes we read
+ * after reloading when we are decoding long offsets.
+ */
+#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \
+ (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \
+ ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
+ : 0)
+
+typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;
+
+FORCE_INLINE_TEMPLATE seq_t
+ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
+{
+ seq_t seq;
+ /*
+ * ZSTD_seqSymbol is a structure with a total of 64 bits wide. So it can be
+ * loaded in one operation and extracted its fields by simply shifting or
+ * bit-extracting on aarch64.
+ * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh
+ * operations that cause performance drop. This can be avoided by using this
+ * ZSTD_memcpy hack.
+ */
+#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__))
+ ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS;
+ ZSTD_seqSymbol* const llDInfo = &llDInfoS;
+ ZSTD_seqSymbol* const mlDInfo = &mlDInfoS;
+ ZSTD_seqSymbol* const ofDInfo = &ofDInfoS;
+ ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol));
+ ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol));
+ ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol));
+#else
+ const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
+ const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
+ const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
+#endif
+ seq.matchLength = mlDInfo->baseValue;
+ seq.litLength = llDInfo->baseValue;
+ { U32 const ofBase = ofDInfo->baseValue;
+ BYTE const llBits = llDInfo->nbAdditionalBits;
+ BYTE const mlBits = mlDInfo->nbAdditionalBits;
+ BYTE const ofBits = ofDInfo->nbAdditionalBits;
+ BYTE const totalBits = llBits+mlBits+ofBits;
+
+ U16 const llNext = llDInfo->nextState;
+ U16 const mlNext = mlDInfo->nextState;
+ U16 const ofNext = ofDInfo->nextState;
+ U32 const llnbBits = llDInfo->nbBits;
+ U32 const mlnbBits = mlDInfo->nbBits;
+ U32 const ofnbBits = ofDInfo->nbBits;
+
+ assert(llBits <= MaxLLBits);
+ assert(mlBits <= MaxMLBits);
+ assert(ofBits <= MaxOff);
+ /*
+ * As gcc has better branch and block analyzers, sometimes it is only
+ * valuable to mark likeliness for clang, it gives around 3-4% of
+ * performance.
+ */
+
+ /* sequence */
+ { size_t offset;
+ #if defined(__clang__)
+ if (LIKELY(ofBits > 1)) {
+ #else
+ if (ofBits > 1) {
+ #endif
+ ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
+ ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
+ ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
+ ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
+ if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
+ /* Always read extra bits, this keeps the logic simple,
+ * avoids branches, and avoids accidentally reading 0 bits.
+ */
+ U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
+ offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
+ BIT_reloadDStream(&seqState->DStream);
+ offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+ } else {
+ offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
+ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+ }
+ seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset;
+ } else {
+ U32 const ll0 = (llDInfo->baseValue == 0);
+ if (LIKELY((ofBits == 0))) {
+ offset = seqState->prevOffset[ll0];
+ seqState->prevOffset[1] = seqState->prevOffset[!ll0];
+ seqState->prevOffset[0] = offset;
+ } else {
+ offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
+ { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
+ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset = temp;
+ } } }
+ seq.offset = offset;
+ }
+
+ #if defined(__clang__)
+ if (UNLIKELY(mlBits > 0))
+ #else
+ if (mlBits > 0)
+ #endif
+ seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
+
+ if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
+ BIT_reloadDStream(&seqState->DStream);
+ if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
+ BIT_reloadDStream(&seqState->DStream);
+ /* Ensure there are enough bits to read the rest of data in 64-bit mode. */
+ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
+
+ #if defined(__clang__)
+ if (UNLIKELY(llBits > 0))
+ #else
+ if (llBits > 0)
+ #endif
+ seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
+
+ if (MEM_32bits())
+ BIT_reloadDStream(&seqState->DStream);
+
+ DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
+ (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
+
+ ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
+ ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
+ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
+ ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
+ }
+
+ return seq;
+}
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd)
+{
+ size_t const windowSize = dctx->fParams.windowSize;
+ /* No dictionary used. */
+ if (dctx->dictContentEndForFuzzing == NULL) return 0;
+ /* Dictionary is our prefix. */
+ if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
+ /* Dictionary is not our ext-dict. */
+ if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
+ /* Dictionary is not within our window size. */
+ if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
+ /* Dictionary is active. */
+ return 1;
+}
+
+MEM_STATIC void ZSTD_assertValidSequence(
+ ZSTD_DCtx const* dctx,
+ BYTE const* op, BYTE const* oend,
+ seq_t const seq,
+ BYTE const* prefixStart, BYTE const* virtualStart)
+{
+#if DEBUGLEVEL >= 1
+ size_t const windowSize = dctx->fParams.windowSize;
+ size_t const sequenceSize = seq.litLength + seq.matchLength;
+ BYTE const* const oLitEnd = op + seq.litLength;
+ DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",
+ (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
+ assert(op <= oend);
+ assert((size_t)(oend - op) >= sequenceSize);
+ assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX);
+ if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {
+ size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing);
+ /* Offset must be within the dictionary. */
+ assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
+ assert(seq.offset <= windowSize + dictSize);
+ } else {
+ /* Offset must be within our window. */
+ assert(seq.offset <= windowSize);
+ }
+#else
+ (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;
+#endif
+}
+#endif
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+
+
+FORCE_INLINE_TEMPLATE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ const BYTE* ip = (const BYTE*)seqStart;
+ const BYTE* const iend = ip + seqSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + maxDstSize;
+ BYTE* op = ostart;
+ const BYTE* litPtr = dctx->litPtr;
+ const BYTE* litBufferEnd = dctx->litBufferEnd;
+ const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
+ const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
+ const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+ DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer");
+ (void)frame;
+
+ /* Regen sequences */
+ if (nbSeq) {
+ seqState_t seqState;
+ dctx->fseEntropy = 1;
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+ corruption_detected, "");
+ ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+ assert(dst != NULL);
+
+ ZSTD_STATIC_ASSERT(
+ BIT_DStream_unfinished < BIT_DStream_completed &&
+ BIT_DStream_endOfBuffer < BIT_DStream_completed &&
+ BIT_DStream_completed < BIT_DStream_overflow);
+
+ /* decompress without overrunning litPtr begins */
+ {
+ seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ /* Align the decompression loop to 32 + 16 bytes.
+ *
+ * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
+ * speed swings based on the alignment of the decompression loop. This
+ * performance swing is caused by parts of the decompression loop falling
+ * out of the DSB. The entire decompression loop should fit in the DSB,
+ * when it can't we get much worse performance. You can measure if you've
+ * hit the good case or the bad case with this perf command for some
+ * compressed file test.zst:
+ *
+ * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
+ * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
+ *
+ * If you see most cycles served out of the MITE you've hit the bad case.
+ * If you see most cycles served out of the DSB you've hit the good case.
+ * If it is pretty even then you may be in an okay case.
+ *
+ * This issue has been reproduced on the following CPUs:
+ * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
+ * Use Instruments->Counters to get DSB/MITE cycles.
+ * I never got performance swings, but I was able to
+ * go from the good case of mostly DSB to half of the
+ * cycles served from MITE.
+ * - Coffeelake: Intel i9-9900k
+ * - Coffeelake: Intel i7-9700k
+ *
+ * I haven't been able to reproduce the instability or DSB misses on any
+ * of the following CPUS:
+ * - Haswell
+ * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
+ * - Skylake
+ *
+ * Alignment is done for each of the three major decompression loops:
+ * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
+ * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
+ * - ZSTD_decompressSequences_body
+ * Alignment choices are made to minimize large swings on bad cases and influence on performance
+ * from changes external to this code, rather than to overoptimize on the current commit.
+ *
+ * If you are seeing performance stability this script can help test.
+ * It tests on 4 commits in zstd where I saw performance change.
+ *
+ * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
+ */
+#if defined(__GNUC__) && defined(__x86_64__)
+ __asm__(".p2align 6");
+# if __GNUC__ >= 7
+ /* good for gcc-7, gcc-9, and gcc-11 */
+ __asm__("nop");
+ __asm__(".p2align 5");
+ __asm__("nop");
+ __asm__(".p2align 4");
+# if __GNUC__ == 8 || __GNUC__ == 10
+ /* good for gcc-8 and gcc-10 */
+ __asm__("nop");
+ __asm__(".p2align 3");
+# endif
+# endif
+#endif
+
+ /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
+ for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) {
+ size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (UNLIKELY(!--nbSeq))
+ break;
+ BIT_reloadDStream(&(seqState.DStream));
+ sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ }
+
+ /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
+ if (nbSeq > 0) {
+ const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+ if (leftoverLit)
+ {
+ RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+ ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+ sequence.litLength -= leftoverLit;
+ op += leftoverLit;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ {
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (--nbSeq)
+ BIT_reloadDStream(&(seqState.DStream));
+ }
+ }
+ }
+
+ if (nbSeq > 0) /* there is remaining lit from extra buffer */
+ {
+
+#if defined(__GNUC__) && defined(__x86_64__)
+ __asm__(".p2align 6");
+ __asm__("nop");
+# if __GNUC__ != 7
+ /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
+ __asm__(".p2align 4");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# elif __GNUC__ >= 11
+ __asm__(".p2align 3");
+# else
+ __asm__(".p2align 5");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# endif
+#endif
+
+ for (; ; ) {
+ seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (UNLIKELY(!--nbSeq))
+ break;
+ BIT_reloadDStream(&(seqState.DStream));
+ }
+ }
+
+ /* check if reached exact end */
+ DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
+ RETURN_ERROR_IF(nbSeq, corruption_detected, "");
+ RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
+ /* save reps for next block */
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+ }
+
+ /* last literal segment */
+ if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */
+ {
+ size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memmove(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ }
+ { size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memcpy(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ }
+
+ return op-ostart;
+}
+
+FORCE_INLINE_TEMPLATE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ const BYTE* ip = (const BYTE*)seqStart;
+ const BYTE* const iend = ip + seqSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer;
+ BYTE* op = ostart;
+ const BYTE* litPtr = dctx->litPtr;
+ const BYTE* const litEnd = litPtr + dctx->litSize;
+ const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
+ const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
+ const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
+ DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
+ (void)frame;
+
+ /* Regen sequences */
+ if (nbSeq) {
+ seqState_t seqState;
+ dctx->fseEntropy = 1;
+ { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
+ corruption_detected, "");
+ ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+ assert(dst != NULL);
+
+ ZSTD_STATIC_ASSERT(
+ BIT_DStream_unfinished < BIT_DStream_completed &&
+ BIT_DStream_endOfBuffer < BIT_DStream_completed &&
+ BIT_DStream_completed < BIT_DStream_overflow);
+
+#if defined(__GNUC__) && defined(__x86_64__)
+ __asm__(".p2align 6");
+ __asm__("nop");
+# if __GNUC__ >= 7
+ __asm__(".p2align 5");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# else
+ __asm__(".p2align 4");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# endif
+#endif
+
+ for ( ; ; ) {
+ seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (UNLIKELY(!--nbSeq))
+ break;
+ BIT_reloadDStream(&(seqState.DStream));
+ }
+
+ /* check if reached exact end */
+ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq);
+ RETURN_ERROR_IF(nbSeq, corruption_detected, "");
+ RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
+ /* save reps for next block */
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+ }
+
+ /* last literal segment */
+ { size_t const lastLLSize = litEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memcpy(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ }
+
+ return op-ostart;
+}
+
+static size_t
+ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+
+static size_t
+ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
+ const BYTE* const prefixStart, const BYTE* const dictEnd)
+{
+ prefetchPos += sequence.litLength;
+ { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
+ const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
+ * No consequence though : memory address is only used for prefetching, not for dereferencing */
+ PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
+ }
+ return prefetchPos + sequence.matchLength;
+}
+
+/* This decoding function employs prefetching
+ * to reduce latency impact of cache misses.
+ * It's generally employed when block contains a significant portion of long-distance matches
+ * or when coupled with a "cold" dictionary */
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_decompressSequencesLong_body(
+ ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ const BYTE* ip = (const BYTE*)seqStart;
+ const BYTE* const iend = ip + seqSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize;
+ BYTE* op = ostart;
+ const BYTE* litPtr = dctx->litPtr;
+ const BYTE* litBufferEnd = dctx->litBufferEnd;
+ const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
+ const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
+ const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+ (void)frame;
+
+ /* Regen sequences */
+ if (nbSeq) {
+#define STORED_SEQS 8
+#define STORED_SEQS_MASK (STORED_SEQS-1)
+#define ADVANCED_SEQS STORED_SEQS
+ seq_t sequences[STORED_SEQS];
+ int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
+ seqState_t seqState;
+ int seqNb;
+ size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
+
+ dctx->fseEntropy = 1;
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+ assert(dst != NULL);
+ assert(iend >= ip);
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+ corruption_detected, "");
+ ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+
+ /* prepare in advance */
+ for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb<seqAdvance); seqNb++) {
+ seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+ sequences[seqNb] = sequence;
+ }
+ RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, "");
+
+ /* decompress without stomping litBuffer */
+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb < nbSeq); seqNb++) {
+ seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ size_t oneSeqSize;
+
+ if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd)
+ {
+ /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
+ const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+ if (leftoverLit)
+ {
+ RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+ ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+ sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
+ op += leftoverLit;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+
+ prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+ sequences[seqNb & STORED_SEQS_MASK] = sequence;
+ op += oneSeqSize;
+ }
+ else
+ {
+ /* lit buffer is either wholly contained in first or second split, or not split at all*/
+ oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
+ ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
+ ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+
+ prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+ sequences[seqNb & STORED_SEQS_MASK] = sequence;
+ op += oneSeqSize;
+ }
+ }
+ RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, "");
+
+ /* finish queue */
+ seqNb -= seqAdvance;
+ for ( ; seqNb<nbSeq ; seqNb++) {
+ seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
+ if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd)
+ {
+ const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+ if (leftoverLit)
+ {
+ RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+ ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+ sequence->litLength -= leftoverLit;
+ op += leftoverLit;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ {
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+ op += oneSeqSize;
+ }
+ }
+ else
+ {
+ size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
+ ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
+ ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+ op += oneSeqSize;
+ }
+ }
+
+ /* save reps for next block */
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+ }
+
+ /* last literal segment */
+ if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */
+ {
+ size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memmove(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ }
+ { size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memmove(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ }
+
+ return op-ostart;
+}
+
+static size_t
+ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+
+
+#if DYNAMIC_BMI2
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+static BMI2_TARGET_ATTRIBUTE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+static BMI2_TARGET_ATTRIBUTE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+static BMI2_TARGET_ATTRIBUTE size_t
+ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+#endif /* DYNAMIC_BMI2 */
+
+typedef size_t (*ZSTD_decompressSequences_t)(
+ ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame);
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+static size_t
+ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ DEBUGLOG(5, "ZSTD_decompressSequences");
+#if DYNAMIC_BMI2
+ if (ZSTD_DCtx_get_bmi2(dctx)) {
+ return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+ }
+#endif
+ return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+static size_t
+ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
+#if DYNAMIC_BMI2
+ if (ZSTD_DCtx_get_bmi2(dctx)) {
+ return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+ }
+#endif
+ return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+/* ZSTD_decompressSequencesLong() :
+ * decompression function triggered when a minimum share of offsets is considered "long",
+ * aka out of cache.
+ * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
+ * This function will try to mitigate main memory latency through the use of prefetching */
+static size_t
+ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ DEBUGLOG(5, "ZSTD_decompressSequencesLong");
+#if DYNAMIC_BMI2
+ if (ZSTD_DCtx_get_bmi2(dctx)) {
+ return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+ }
+#endif
+ return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+
+/**
+ * @returns The total size of the history referencable by zstd, including
+ * both the prefix and the extDict. At @p op any offset larger than this
+ * is invalid.
+ */
+static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
+{
+ return (size_t)(op - virtualStart);
+}
+
+typedef struct {
+ unsigned longOffsetShare;
+ unsigned maxNbAdditionalBits;
+} ZSTD_OffsetInfo;
+
+/* ZSTD_getOffsetInfo() :
+ * condition : offTable must be valid
+ * @return : "share" of long offsets (arbitrarily defined as > (1<<23))
+ * compared to maximum possible of (1<<OffFSELog),
+ * as well as the maximum number additional bits required.
+ */
+static ZSTD_OffsetInfo
+ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq)
+{
+ ZSTD_OffsetInfo info = {0, 0};
+ /* If nbSeq == 0, then the offTable is uninitialized, but we have
+ * no sequences, so both values should be 0.
+ */
+ if (nbSeq != 0) {
+ const void* ptr = offTable;
+ U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
+ const ZSTD_seqSymbol* table = offTable + 1;
+ U32 const max = 1 << tableLog;
+ U32 u;
+ DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
+
+ assert(max <= (1 << OffFSELog)); /* max not too large */
+ for (u=0; u<max; u++) {
+ info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
+ if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
+ }
+
+ assert(tableLog <= OffFSELog);
+ info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */
+ }
+
+ return info;
+}
+
+/**
+ * @returns The maximum offset we can decode in one read of our bitstream, without
+ * reloading more bits in the middle of the offset bits read. Any offsets larger
+ * than this must use the long offset decoder.
+ */
+static size_t ZSTD_maxShortOffset(void)
+{
+ if (MEM_64bits()) {
+ /* We can decode any offset without reloading bits.
+ * This might change if the max window size grows.
+ */
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+ return (size_t)-1;
+ } else {
+ /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
+ * This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
+ * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
+ */
+ size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
+ size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
+ assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
+ return maxOffset;
+ }
+}
+
+size_t
+ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
+{ /* blockType == blockCompressed */
+ const BYTE* ip = (const BYTE*)src;
+ DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
+
+ /* Note : the wording of the specification
+ * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX.
+ * This generally does not happen, as it makes little sense,
+ * since an uncompressed block would feature same size and have no decompression cost.
+ * Also, note that decoder from reference libzstd before < v1.5.4
+ * would consider this edge case as an error.
+ * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX
+ * for broader compatibility with the deployed ecosystem of zstd decoders */
+ RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
+
+ /* Decode literals section */
+ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
+ DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
+ if (ZSTD_isError(litCSize)) return litCSize;
+ ip += litCSize;
+ srcSize -= litCSize;
+ }
+
+ /* Build Decoding Tables */
+ {
+ /* Compute the maximum block size, which must also work when !frame and fParams are unset.
+ * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
+ */
+ size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX));
+ size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart);
+ /* isLongOffset must be true if there are long offsets.
+ * Offsets are long if they are larger than ZSTD_maxShortOffset().
+ * We don't expect that to be the case in 64-bit mode.
+ *
+ * We check here to see if our history is large enough to allow long offsets.
+ * If it isn't, then we can't possible have (valid) long offsets. If the offset
+ * is invalid, then it is okay to read it incorrectly.
+ *
+ * If isLongOffsets is true, then we will later check our decoding table to see
+ * if it is even possible to generate long offsets.
+ */
+ ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
+ /* These macros control at build-time which decompressor implementation
+ * we use. If neither is defined, we do some inspection and dispatch at
+ * runtime.
+ */
+#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+ int usePrefetchDecoder = dctx->ddictIsCold;
+#else
+ /* Set to 1 to avoid computing offset info if we don't need to.
+ * Otherwise this value is ignored.
+ */
+ int usePrefetchDecoder = 1;
+#endif
+ int nbSeq;
+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
+ if (ZSTD_isError(seqHSize)) return seqHSize;
+ ip += seqHSize;
+ srcSize -= seqHSize;
+
+ RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
+
+ /* If we could potentially have long offsets, or we might want to use the prefetch decoder,
+ * compute information about the share of long offsets, and the maximum nbAdditionalBits.
+ * NOTE: could probably use a larger nbSeq limit
+ */
+ if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
+ ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
+ if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
+ /* If isLongOffset, but the maximum number of additional bits that we see in our table is small
+ * enough, then we know it is impossible to have too long an offset in this block, so we can
+ * use the regular offset decoder.
+ */
+ isLongOffset = ZSTD_lo_isRegularOffset;
+ }
+ if (!usePrefetchDecoder) {
+ U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
+ usePrefetchDecoder = (info.longOffsetShare >= minShare);
+ }
+ }
+
+ dctx->ddictIsCold = 0;
+
+#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+ if (usePrefetchDecoder) {
+#else
+ (void)usePrefetchDecoder;
+ {
+#endif
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+ return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
+#endif
+ }
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+ /* else */
+ if (dctx->litBufferLocation == ZSTD_split)
+ return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
+ else
+ return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
+#endif
+ }
+}
+
+
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
+{
+ if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */
+ dctx->dictEnd = dctx->previousDstEnd;
+ dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
+ dctx->prefixStart = dst;
+ dctx->previousDstEnd = dst;
+ }
+}
+
+
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ size_t dSize;
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
+ dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming);
+ dctx->previousDstEnd = (char*)dst + dSize;
+ return dSize;
+}
diff --git a/contrib/zstd/zstd_decompress_block.h b/contrib/zstd/zstd_decompress_block.h
new file mode 100644
index 0000000..418985c
--- /dev/null
+++ b/contrib/zstd/zstd_decompress_block.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_DEC_BLOCK_H
+#define ZSTD_DEC_BLOCK_H
+
+/*-*******************************************************
+ * Dependencies
+ *********************************************************/
+#include "zstd_deps.h" /* size_t */
+#include "zstd.h" /* DCtx, and some public functions */
+#include "zstd_internal.h" /* blockProperties_t, and some public functions */
+#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
+
+
+/* === Prototypes === */
+
+/* note: prototypes already published within `zstd.h` :
+ * ZSTD_decompressBlock()
+ */
+
+/* note: prototypes already published within `zstd_internal.h` :
+ * ZSTD_getcBlockSize()
+ * ZSTD_decodeSeqHeaders()
+ */
+
+
+ /* Streaming state is used to inform allocation of the literal buffer */
+typedef enum {
+ not_streaming = 0,
+ is_streaming = 1
+} streaming_operation;
+
+/* ZSTD_decompressBlock_internal() :
+ * decompress block, starting at `src`,
+ * into destination buffer `dst`.
+ * @return : decompressed block size,
+ * or an error code (which can be tested using ZSTD_isError())
+ */
+size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, const int frame, const streaming_operation streaming);
+
+/* ZSTD_buildFSETable() :
+ * generate FSE decoding table for one symbol (ll, ml or off)
+ * this function must be called with valid parameters only
+ * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
+ * in which case it cannot fail.
+ * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is
+ * defined in zstd_decompress_internal.h.
+ * Internal use only.
+ */
+void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize,
+ int bmi2);
+
+
+#endif /* ZSTD_DEC_BLOCK_H */
diff --git a/contrib/zstd/zstd_decompress_internal.h b/contrib/zstd/zstd_decompress_internal.h
new file mode 100644
index 0000000..a164de3
--- /dev/null
+++ b/contrib/zstd/zstd_decompress_internal.h
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* zstd_decompress_internal:
+ * objects and definitions shared within lib/decompress modules */
+
+ #ifndef ZSTD_DECOMPRESS_INTERNAL_H
+ #define ZSTD_DECOMPRESS_INTERNAL_H
+
+
+/*-*******************************************************
+ * Dependencies
+ *********************************************************/
+#include "mem.h" /* BYTE, U16, U32 */
+#include "zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */
+
+
+
+/*-*******************************************************
+ * Constants
+ *********************************************************/
+static UNUSED_ATTR const U32 LL_base[MaxLL+1] = {
+ 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 18, 20, 22, 24, 28, 32, 40,
+ 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
+ 0x2000, 0x4000, 0x8000, 0x10000 };
+
+static UNUSED_ATTR const U32 OF_base[MaxOff+1] = {
+ 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
+ 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
+ 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
+ 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
+
+static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = {
+ 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31 };
+
+static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
+ 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18,
+ 19, 20, 21, 22, 23, 24, 25, 26,
+ 27, 28, 29, 30, 31, 32, 33, 34,
+ 35, 37, 39, 41, 43, 47, 51, 59,
+ 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
+ 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
+
+
+/*-*******************************************************
+ * Decompression types
+ *********************************************************/
+ typedef struct {
+ U32 fastMode;
+ U32 tableLog;
+ } ZSTD_seqSymbol_header;
+
+ typedef struct {
+ U16 nextState;
+ BYTE nbAdditionalBits;
+ BYTE nbBits;
+ U32 baseValue;
+ } ZSTD_seqSymbol;
+
+ #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
+
+#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
+#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
+#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
+
+typedef struct {
+ ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
+ ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
+ ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
+ HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */
+ U32 rep[ZSTD_REP_NUM];
+ U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
+} ZSTD_entropyDTables_t;
+
+typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
+ ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
+ ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
+ ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
+
+typedef enum { zdss_init=0, zdss_loadHeader,
+ zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
+
+typedef enum {
+ ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */
+ ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */
+ ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
+} ZSTD_dictUses_e;
+
+/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
+typedef struct {
+ const ZSTD_DDict** ddictPtrTable;
+ size_t ddictPtrTableSize;
+ size_t ddictPtrCount;
+} ZSTD_DDictHashSet;
+
+#ifndef ZSTD_DECODER_INTERNAL_BUFFER
+# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16)
+#endif
+
+#define ZSTD_LBMIN 64
+#define ZSTD_LBMAX (128 << 10)
+
+/* extra buffer, compensates when dst is not large enough to store litBuffer */
+#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX)
+
+typedef enum {
+ ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */
+ ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */
+ ZSTD_split = 2 /* Split between litExtraBuffer and dst */
+} ZSTD_litLocation_e;
+
+struct ZSTD_DCtx_s
+{
+ const ZSTD_seqSymbol* LLTptr;
+ const ZSTD_seqSymbol* MLTptr;
+ const ZSTD_seqSymbol* OFTptr;
+ const HUF_DTable* HUFptr;
+ ZSTD_entropyDTables_t entropy;
+ U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
+ const void* previousDstEnd; /* detect continuity */
+ const void* prefixStart; /* start of current segment */
+ const void* virtualStart; /* virtual start of previous segment if it was just before current one */
+ const void* dictEnd; /* end of previous segment */
+ size_t expected;
+ ZSTD_frameHeader fParams;
+ U64 processedCSize;
+ U64 decodedSize;
+ blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
+ ZSTD_dStage stage;
+ U32 litEntropy;
+ U32 fseEntropy;
+ XXH64_state_t xxhState;
+ size_t headerSize;
+ ZSTD_format_e format;
+ ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */
+ U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */
+ const BYTE* litPtr;
+ ZSTD_customMem customMem;
+ size_t litSize;
+ size_t rleSize;
+ size_t staticSize;
+#if DYNAMIC_BMI2 != 0
+ int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
+#endif
+
+ /* dictionary */
+ ZSTD_DDict* ddictLocal;
+ const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
+ U32 dictID;
+ int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
+ ZSTD_dictUses_e dictUses;
+ ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */
+ ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
+ int disableHufAsm;
+
+ /* streaming */
+ ZSTD_dStreamStage streamStage;
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inPos;
+ size_t maxWindowSize;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outStart;
+ size_t outEnd;
+ size_t lhSize;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ void* legacyContext;
+ U32 previousLegacyVersion;
+ U32 legacyVersion;
+#endif
+ U32 hostageByte;
+ int noForwardProgress;
+ ZSTD_bufferMode_e outBufferMode;
+ ZSTD_outBuffer expectedOutBuffer;
+
+ /* workspace */
+ BYTE* litBuffer;
+ const BYTE* litBufferEnd;
+ ZSTD_litLocation_e litBufferLocation;
+ BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+
+ size_t oversizedDuration;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ void const* dictContentBeginForFuzzing;
+ void const* dictContentEndForFuzzing;
+#endif
+
+ /* Tracing */
+#if ZSTD_TRACE
+ ZSTD_TraceCtx traceCtx;
+#endif
+}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
+
+MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
+#if DYNAMIC_BMI2 != 0
+ return dctx->bmi2;
+#else
+ (void)dctx;
+ return 0;
+#endif
+}
+
+/*-*******************************************************
+ * Shared internal functions
+ *********************************************************/
+
+/*! ZSTD_loadDEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary.
+ * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
+size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
+ const void* const dict, size_t const dictSize);
+
+/*! ZSTD_checkContinuity() :
+ * check if next `dst` follows previous position, where decompression ended.
+ * If yes, do nothing (continue on current segment).
+ * If not, classify previous segment as "external dictionary", and start a new segment.
+ * This function cannot fail. */
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
+
+
+#endif /* ZSTD_DECOMPRESS_INTERNAL_H */
diff --git a/contrib/zstd/zstd_deps.h b/contrib/zstd/zstd_deps.h
new file mode 100644
index 0000000..4d767ae
--- /dev/null
+++ b/contrib/zstd/zstd_deps.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides common libc dependencies that zstd requires.
+ * The purpose is to allow replacing this file with a custom implementation
+ * to compile zstd without libc support.
+ */
+
+/* Need:
+ * NULL
+ * INT_MAX
+ * UINT_MAX
+ * ZSTD_memcpy()
+ * ZSTD_memset()
+ * ZSTD_memmove()
+ */
+#ifndef ZSTD_DEPS_COMMON
+#define ZSTD_DEPS_COMMON
+
+#include <limits.h>
+#include <stddef.h>
+#include <string.h>
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
+#else
+# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) memset((p),(v),(l))
+#endif
+
+#endif /* ZSTD_DEPS_COMMON */
+
+/* Need:
+ * ZSTD_malloc()
+ * ZSTD_free()
+ * ZSTD_calloc()
+ */
+#ifdef ZSTD_DEPS_NEED_MALLOC
+#ifndef ZSTD_DEPS_MALLOC
+#define ZSTD_DEPS_MALLOC
+
+#include <stdlib.h>
+
+#define ZSTD_malloc(s) malloc(s)
+#define ZSTD_calloc(n,s) calloc((n), (s))
+#define ZSTD_free(p) free((p))
+
+#endif /* ZSTD_DEPS_MALLOC */
+#endif /* ZSTD_DEPS_NEED_MALLOC */
+
+/*
+ * Provides 64-bit math support.
+ * Need:
+ * U64 ZSTD_div64(U64 dividend, U32 divisor)
+ */
+#ifdef ZSTD_DEPS_NEED_MATH64
+#ifndef ZSTD_DEPS_MATH64
+#define ZSTD_DEPS_MATH64
+
+#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
+
+#endif /* ZSTD_DEPS_MATH64 */
+#endif /* ZSTD_DEPS_NEED_MATH64 */
+
+/* Need:
+ * assert()
+ */
+#ifdef ZSTD_DEPS_NEED_ASSERT
+#ifndef ZSTD_DEPS_ASSERT
+#define ZSTD_DEPS_ASSERT
+
+#include <assert.h>
+
+#endif /* ZSTD_DEPS_ASSERT */
+#endif /* ZSTD_DEPS_NEED_ASSERT */
+
+/* Need:
+ * ZSTD_DEBUG_PRINT()
+ */
+#ifdef ZSTD_DEPS_NEED_IO
+#ifndef ZSTD_DEPS_IO
+#define ZSTD_DEPS_IO
+
+#include <stdio.h>
+#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
+
+#endif /* ZSTD_DEPS_IO */
+#endif /* ZSTD_DEPS_NEED_IO */
+
+/* Only requested when <stdint.h> is known to be present.
+ * Need:
+ * intptr_t
+ */
+#ifdef ZSTD_DEPS_NEED_STDINT
+#ifndef ZSTD_DEPS_STDINT
+#define ZSTD_DEPS_STDINT
+
+#include <stdint.h>
+
+#endif /* ZSTD_DEPS_STDINT */
+#endif /* ZSTD_DEPS_NEED_STDINT */
diff --git a/contrib/zstd/zstd_double_fast.c b/contrib/zstd/zstd_double_fast.c
new file mode 100644
index 0000000..0ad88ff
--- /dev/null
+++ b/contrib/zstd/zstd_double_fast.c
@@ -0,0 +1,758 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "zstd_double_fast.h"
+
+static void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashLarge = ms->hashTable;
+ U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ U32 const mls = cParams->minMatch;
+ U32* const hashSmall = ms->chainTable;
+ U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Always insert every fastHashFillStep position into the hash tables.
+ * Insert the other positions into the large hash table if their entry
+ * is empty.
+ */
+ for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ U32 i;
+ for (i = 0; i < fastHashFillStep; ++i) {
+ size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls);
+ size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8);
+ if (i == 0) {
+ ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i);
+ }
+ if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) {
+ ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i);
+ }
+ /* Only load extra positions for ZSTD_dtlm_full */
+ if (dtlm == ZSTD_dtlm_fast)
+ break;
+ } }
+}
+
+static void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashLarge = ms->hashTable;
+ U32 const hBitsL = cParams->hashLog;
+ U32 const mls = cParams->minMatch;
+ U32* const hashSmall = ms->chainTable;
+ U32 const hBitsS = cParams->chainLog;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Always insert every fastHashFillStep position into the hash tables.
+ * Insert the other positions into the large hash table if their entry
+ * is empty.
+ */
+ for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ U32 i;
+ for (i = 0; i < fastHashFillStep; ++i) {
+ size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls);
+ size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8);
+ if (i == 0)
+ hashSmall[smHash] = curr + i;
+ if (i == 0 || hashLarge[lgHash] == 0)
+ hashLarge[lgHash] = curr + i;
+ /* Only load extra positions for ZSTD_dtlm_full */
+ if (dtlm == ZSTD_dtlm_fast)
+ break;
+ } }
+}
+
+void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp)
+{
+ if (tfp == ZSTD_tfp_forCDict) {
+ ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm);
+ } else {
+ ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm);
+ }
+}
+
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_doubleFast_noDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize, U32 const mls /* template */)
+{
+ ZSTD_compressionParameters const* cParams = &ms->cParams;
+ U32* const hashLong = ms->hashTable;
+ const U32 hBitsL = cParams->hashLog;
+ U32* const hashSmall = ms->chainTable;
+ const U32 hBitsS = cParams->chainLog;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* anchor = istart;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ /* presumes that, if there is a dictionary, it must be using Attach mode */
+ const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
+ const BYTE* const prefixLowest = base + prefixLowestIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1=rep[0], offset_2=rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ size_t mLength;
+ U32 offset;
+ U32 curr;
+
+ /* how many positions to search before increasing step size */
+ const size_t kStepIncr = 1 << kSearchStrength;
+ /* the position at which to increment the step size if no match is found */
+ const BYTE* nextStep;
+ size_t step; /* the current step size */
+
+ size_t hl0; /* the long hash at ip */
+ size_t hl1; /* the long hash at ip1 */
+
+ U32 idxl0; /* the long match index for ip */
+ U32 idxl1; /* the long match index for ip1 */
+
+ const BYTE* matchl0; /* the long match for ip */
+ const BYTE* matchs0; /* the short match for ip */
+ const BYTE* matchl1; /* the long match for ip1 */
+
+ const BYTE* ip = istart; /* the current position */
+ const BYTE* ip1; /* the next position */
+
+ DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
+
+ /* init */
+ ip += ((ip - prefixLowest) == 0);
+ {
+ U32 const current = (U32)(ip - base);
+ U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog);
+ U32 const maxRep = current - windowLow;
+ if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+ if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
+ }
+
+ /* Outer Loop: one iteration per match found and stored */
+ while (1) {
+ step = 1;
+ nextStep = ip + kStepIncr;
+ ip1 = ip + step;
+
+ if (ip1 > ilimit) {
+ goto _cleanup;
+ }
+
+ hl0 = ZSTD_hashPtr(ip, hBitsL, 8);
+ idxl0 = hashLong[hl0];
+ matchl0 = base + idxl0;
+
+ /* Inner Loop: one iteration per search / position */
+ do {
+ const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls);
+ const U32 idxs0 = hashSmall[hs0];
+ curr = (U32)(ip-base);
+ matchs0 = base + idxs0;
+
+ hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */
+
+ /* check noDict repcode */
+ if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
+ mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ goto _match_stored;
+ }
+
+ hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
+
+ if (idxl0 > prefixLowestIndex) {
+ /* check prefix long match */
+ if (MEM_read64(matchl0) == MEM_read64(ip)) {
+ mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
+ offset = (U32)(ip-matchl0);
+ while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ }
+
+ idxl1 = hashLong[hl1];
+ matchl1 = base + idxl1;
+
+ if (idxs0 > prefixLowestIndex) {
+ /* check prefix short match */
+ if (MEM_read32(matchs0) == MEM_read32(ip)) {
+ goto _search_next_long;
+ }
+ }
+
+ if (ip1 >= nextStep) {
+ PREFETCH_L1(ip1 + 64);
+ PREFETCH_L1(ip1 + 128);
+ step++;
+ nextStep += kStepIncr;
+ }
+ ip = ip1;
+ ip1 += step;
+
+ hl0 = hl1;
+ idxl0 = idxl1;
+ matchl0 = matchl1;
+ #if defined(__aarch64__)
+ PREFETCH_L1(ip+256);
+ #endif
+ } while (ip1 <= ilimit);
+
+_cleanup:
+ /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+ * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+ offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = offset_1 ? offset_1 : offsetSaved1;
+ rep[1] = offset_2 ? offset_2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+
+_search_next_long:
+
+ /* check prefix long +1 match */
+ if (idxl1 > prefixLowestIndex) {
+ if (MEM_read64(matchl1) == MEM_read64(ip1)) {
+ ip = ip1;
+ mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8;
+ offset = (U32)(ip-matchl1);
+ while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ }
+
+ /* if no long +1 match, explore the short match we found */
+ mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
+ offset = (U32)(ip - matchs0);
+ while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */
+
+ /* fall-through */
+
+_match_found: /* requires ip, offset, mLength */
+ offset_2 = offset_1;
+ offset_1 = offset;
+
+ if (step < 4) {
+ /* It is unsafe to write this value back to the hashtable when ip1 is
+ * greater than or equal to the new ip we will have after we're done
+ * processing this match. Rather than perform that test directly
+ * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler
+ * more predictable test. The minmatch even if we take a short match is
+ * 4 bytes, so as long as step, the distance between ip and ip1
+ * (initially) is less than 4, we know ip1 < new ip. */
+ hashLong[hl1] = (U32)(ip1 - base);
+ }
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+_match_stored:
+ /* match found */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = curr+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
+
+ /* check immediate repcode */
+ while ( (ip <= ilimit)
+ && ( (offset_2>0)
+ & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
+ /* store sequence */
+ size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
+ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
+ ip += rLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ }
+ }
+}
+
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize,
+ U32 const mls /* template */)
+{
+ ZSTD_compressionParameters const* cParams = &ms->cParams;
+ U32* const hashLong = ms->hashTable;
+ const U32 hBitsL = cParams->hashLog;
+ U32* const hashSmall = ms->chainTable;
+ const U32 hBitsS = cParams->chainLog;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ /* presumes that, if there is a dictionary, it must be using Attach mode */
+ const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
+ const BYTE* const prefixLowest = base + prefixLowestIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1=rep[0], offset_2=rep[1];
+
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
+ const U32* const dictHashLong = dms->hashTable;
+ const U32* const dictHashSmall = dms->chainTable;
+ const U32 dictStartIndex = dms->window.dictLimit;
+ const BYTE* const dictBase = dms->window.base;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const BYTE* const dictEnd = dms->window.nextSrc;
+ const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase);
+ const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic");
+
+ /* if a dictionary is attached, it must be within window range */
+ assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex);
+
+ if (ms->prefetchCDictTables) {
+ size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
+ size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32);
+ PREFETCH_AREA(dictHashLong, hashTableBytes)
+ PREFETCH_AREA(dictHashSmall, chainTableBytes)
+ }
+
+ /* init */
+ ip += (dictAndPrefixLength == 0);
+
+ /* dictMatchState repCode checks don't currently handle repCode == 0
+ * disabling. */
+ assert(offset_1 <= dictAndPrefixLength);
+ assert(offset_2 <= dictAndPrefixLength);
+
+ /* Main Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
+ size_t mLength;
+ U32 offset;
+ size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
+ size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
+ size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8);
+ size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls);
+ U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS];
+ U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS];
+ int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL);
+ int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS);
+ U32 const curr = (U32)(ip-base);
+ U32 const matchIndexL = hashLong[h2];
+ U32 matchIndexS = hashSmall[h];
+ const BYTE* matchLong = base + matchIndexL;
+ const BYTE* match = base + matchIndexS;
+ const U32 repIndex = curr + 1 - offset_1;
+ const BYTE* repMatch = (repIndex < prefixLowestIndex) ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
+
+ /* check repcode */
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ goto _match_stored;
+ }
+
+ if (matchIndexL > prefixLowestIndex) {
+ /* check prefix long match */
+ if (MEM_read64(matchLong) == MEM_read64(ip)) {
+ mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
+ offset = (U32)(ip-matchLong);
+ while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ } else if (dictTagsMatchL) {
+ /* check dictMatchState long match */
+ U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* dictMatchL = dictBase + dictMatchIndexL;
+ assert(dictMatchL < dictEnd);
+
+ if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) {
+ mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8;
+ offset = (U32)(curr - dictMatchIndexL - dictIndexDelta);
+ while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */
+ goto _match_found;
+ } }
+
+ if (matchIndexS > prefixLowestIndex) {
+ /* check prefix short match */
+ if (MEM_read32(match) == MEM_read32(ip)) {
+ goto _search_next_long;
+ }
+ } else if (dictTagsMatchS) {
+ /* check dictMatchState short match */
+ U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS;
+ match = dictBase + dictMatchIndexS;
+ matchIndexS = dictMatchIndexS + dictIndexDelta;
+
+ if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) {
+ goto _search_next_long;
+ } }
+
+ ip += ((ip-anchor) >> kSearchStrength) + 1;
+#if defined(__aarch64__)
+ PREFETCH_L1(ip+256);
+#endif
+ continue;
+
+_search_next_long:
+ { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
+ size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
+ U32 const matchIndexL3 = hashLong[hl3];
+ U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS];
+ int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3);
+ const BYTE* matchL3 = base + matchIndexL3;
+ hashLong[hl3] = curr + 1;
+
+ /* check prefix long +1 match */
+ if (matchIndexL3 > prefixLowestIndex) {
+ if (MEM_read64(matchL3) == MEM_read64(ip+1)) {
+ mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
+ ip++;
+ offset = (U32)(ip-matchL3);
+ while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ } else if (dictTagsMatchL3) {
+ /* check dict long +1 match */
+ U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
+ assert(dictMatchL3 < dictEnd);
+ if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
+ mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8;
+ ip++;
+ offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta);
+ while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */
+ goto _match_found;
+ } } }
+
+ /* if no long +1 match, explore the short match we found */
+ if (matchIndexS < prefixLowestIndex) {
+ mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4;
+ offset = (U32)(curr - matchIndexS);
+ while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+ } else {
+ mLength = ZSTD_count(ip+4, match+4, iend) + 4;
+ offset = (U32)(ip - match);
+ while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+ }
+
+_match_found:
+ offset_2 = offset_1;
+ offset_1 = offset;
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+_match_stored:
+ /* match found */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = curr+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
+
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ U32 const current2 = (U32)(ip-base);
+ U32 const repIndex2 = current2 - offset_2;
+ const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
+ dictBase + repIndex2 - dictIndexDelta :
+ base + repIndex2;
+ if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
+ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
+ const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
+ U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
+ ip += repLength2;
+ anchor = ip;
+ continue;
+ }
+ break;
+ }
+ }
+ } /* while (ip < ilimit) */
+
+ /* save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
+ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
+ void const* src, size_t srcSize) \
+ { \
+ return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
+ }
+
+ZSTD_GEN_DFAST_FN(noDict, 4)
+ZSTD_GEN_DFAST_FN(noDict, 5)
+ZSTD_GEN_DFAST_FN(noDict, 6)
+ZSTD_GEN_DFAST_FN(noDict, 7)
+
+ZSTD_GEN_DFAST_FN(dictMatchState, 4)
+ZSTD_GEN_DFAST_FN(dictMatchState, 5)
+ZSTD_GEN_DFAST_FN(dictMatchState, 6)
+ZSTD_GEN_DFAST_FN(dictMatchState, 7)
+
+
+size_t ZSTD_compressBlock_doubleFast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ const U32 mls = ms->cParams.minMatch;
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);
+ }
+}
+
+
+size_t ZSTD_compressBlock_doubleFast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ const U32 mls = ms->cParams.minMatch;
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
+ }
+}
+
+
+static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize,
+ U32 const mls /* template */)
+{
+ ZSTD_compressionParameters const* cParams = &ms->cParams;
+ U32* const hashLong = ms->hashTable;
+ U32 const hBitsL = cParams->hashLog;
+ U32* const hashSmall = ms->chainTable;
+ U32 const hBitsS = cParams->chainLog;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - 8;
+ const BYTE* const base = ms->window.base;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
+ const U32 dictStartIndex = lowLimit;
+ const U32 dictLimit = ms->window.dictLimit;
+ const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit;
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const BYTE* const dictEnd = dictBase + prefixStartIndex;
+ U32 offset_1=rep[0], offset_2=rep[1];
+
+ DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize);
+
+ /* if extDict is invalidated due to maxDistance, switch to "regular" variant */
+ if (prefixStartIndex == dictStartIndex)
+ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);
+
+ /* Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
+ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
+ const U32 matchIndex = hashSmall[hSmall];
+ const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
+ const BYTE* match = matchBase + matchIndex;
+
+ const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
+ const U32 matchLongIndex = hashLong[hLong];
+ const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base;
+ const BYTE* matchLong = matchLongBase + matchLongIndex;
+
+ const U32 curr = (U32)(ip-base);
+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
+ const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ size_t mLength;
+ hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
+
+ if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
+ & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
+ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ } else {
+ if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
+ const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
+ const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart;
+ U32 offset;
+ mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8;
+ offset = curr - matchLongIndex;
+ while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+ } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
+ size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
+ U32 const matchIndex3 = hashLong[h3];
+ const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base;
+ const BYTE* match3 = match3Base + matchIndex3;
+ U32 offset;
+ hashLong[h3] = curr + 1;
+ if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
+ const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend;
+ const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart;
+ mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8;
+ ip++;
+ offset = curr+1 - matchIndex3;
+ while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
+ } else {
+ const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
+ const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
+ mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
+ offset = curr - matchIndex;
+ while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+ }
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+ } else {
+ ip += ((ip-anchor) >> kSearchStrength) + 1;
+ continue;
+ } }
+
+ /* move to next sequence start */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = curr+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
+
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ U32 const current2 = (U32)(ip-base);
+ U32 const repIndex2 = current2 - offset_2;
+ const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
+ if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
+ & (offset_2 <= current2 - dictStartIndex))
+ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
+ const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+ U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
+ ip += repLength2;
+ anchor = ip;
+ continue;
+ }
+ break;
+ } } }
+
+ /* save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+ZSTD_GEN_DFAST_FN(extDict, 4)
+ZSTD_GEN_DFAST_FN(extDict, 5)
+ZSTD_GEN_DFAST_FN(extDict, 6)
+ZSTD_GEN_DFAST_FN(extDict, 7)
+
+size_t ZSTD_compressBlock_doubleFast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
+ }
+}
diff --git a/contrib/zstd/zstd_double_fast.h b/contrib/zstd/zstd_double_fast.h
new file mode 100644
index 0000000..e8ce20c
--- /dev/null
+++ b/contrib/zstd/zstd_double_fast.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_DOUBLE_FAST_H
+#define ZSTD_DOUBLE_FAST_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "mem.h" /* U32 */
+#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
+
+void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp);
+size_t ZSTD_compressBlock_doubleFast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_doubleFast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_doubleFast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_DOUBLE_FAST_H */
diff --git a/contrib/zstd/zstd_errors.h b/contrib/zstd/zstd_errors.h
new file mode 100644
index 0000000..dc75eee
--- /dev/null
+++ b/contrib/zstd/zstd_errors.h
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*===== dependency =====*/
+#include <stddef.h> /* size_t */
+
+
+/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
+#ifndef ZSTDERRORLIB_VISIBLE
+ /* Backwards compatibility with old macro name */
+# ifdef ZSTDERRORLIB_VISIBILITY
+# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
+# else
+# define ZSTDERRORLIB_VISIBLE
+# endif
+#endif
+
+#ifndef ZSTDERRORLIB_HIDDEN
+# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+# else
+# define ZSTDERRORLIB_HIDDEN
+# endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
+#endif
+
+/*-*********************************************
+ * Error codes list
+ *-*********************************************
+ * Error codes _values_ are pinned down since v1.3.1 only.
+ * Therefore, don't rely on values if you may link to any version < v1.3.1.
+ *
+ * Only values < 100 are considered stable.
+ *
+ * note 1 : this API shall be used with static linking only.
+ * dynamic linking is not yet officially supported.
+ * note 2 : Prefer relying on the enum than on its value whenever possible
+ * This is the only supported way to use the error list < v1.3.1
+ * note 3 : ZSTD_isError() is always correct, whatever the library version.
+ **********************************************/
+typedef enum {
+ ZSTD_error_no_error = 0,
+ ZSTD_error_GENERIC = 1,
+ ZSTD_error_prefix_unknown = 10,
+ ZSTD_error_version_unsupported = 12,
+ ZSTD_error_frameParameter_unsupported = 14,
+ ZSTD_error_frameParameter_windowTooLarge = 16,
+ ZSTD_error_corruption_detected = 20,
+ ZSTD_error_checksum_wrong = 22,
+ ZSTD_error_literals_headerWrong = 24,
+ ZSTD_error_dictionary_corrupted = 30,
+ ZSTD_error_dictionary_wrong = 32,
+ ZSTD_error_dictionaryCreation_failed = 34,
+ ZSTD_error_parameter_unsupported = 40,
+ ZSTD_error_parameter_combination_unsupported = 41,
+ ZSTD_error_parameter_outOfBound = 42,
+ ZSTD_error_tableLog_tooLarge = 44,
+ ZSTD_error_maxSymbolValue_tooLarge = 46,
+ ZSTD_error_maxSymbolValue_tooSmall = 48,
+ ZSTD_error_stabilityCondition_notRespected = 50,
+ ZSTD_error_stage_wrong = 60,
+ ZSTD_error_init_missing = 62,
+ ZSTD_error_memory_allocation = 64,
+ ZSTD_error_workSpace_tooSmall= 66,
+ ZSTD_error_dstSize_tooSmall = 70,
+ ZSTD_error_srcSize_wrong = 72,
+ ZSTD_error_dstBuffer_null = 74,
+ ZSTD_error_noForwardProgress_destFull = 80,
+ ZSTD_error_noForwardProgress_inputEmpty = 82,
+ /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
+ ZSTD_error_frameIndex_tooLarge = 100,
+ ZSTD_error_seekableIO = 102,
+ ZSTD_error_dstBuffer_wrong = 104,
+ ZSTD_error_srcBuffer_wrong = 105,
+ ZSTD_error_sequenceProducer_failed = 106,
+ ZSTD_error_externalSequences_invalid = 107,
+ ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
+} ZSTD_ErrorCode;
+
+/*! ZSTD_getErrorCode() :
+ convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
+ which can be used to compare with enum list published above */
+ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_ERRORS_H_398273423 */
diff --git a/contrib/zstd/zstd_fast.c b/contrib/zstd/zstd_fast.c
new file mode 100644
index 0000000..5f2c6a2
--- /dev/null
+++ b/contrib/zstd/zstd_fast.c
@@ -0,0 +1,960 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */
+#include "zstd_fast.h"
+
+static void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ U32 const mls = cParams->minMatch;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Currently, we always use ZSTD_dtlm_full for filling CDict tables.
+ * Feel free to remove this assert if there's a good reason! */
+ assert(dtlm == ZSTD_dtlm_full);
+
+ /* Always insert every fastHashFillStep position into the hash table.
+ * Insert the other positions if their hash entry is empty.
+ */
+ for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls);
+ ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); }
+
+ if (dtlm == ZSTD_dtlm_fast) continue;
+ /* Only load extra positions for ZSTD_dtlm_full */
+ { U32 p;
+ for (p = 1; p < fastHashFillStep; ++p) {
+ size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls);
+ if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */
+ ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p);
+ } } } }
+}
+
+static void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hBits = cParams->hashLog;
+ U32 const mls = cParams->minMatch;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables.
+ * Feel free to remove this assert if there's a good reason! */
+ assert(dtlm == ZSTD_dtlm_fast);
+
+ /* Always insert every fastHashFillStep position into the hash table.
+ * Insert the other positions if their hash entry is empty.
+ */
+ for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls);
+ hashTable[hash0] = curr;
+ if (dtlm == ZSTD_dtlm_fast) continue;
+ /* Only load extra positions for ZSTD_dtlm_full */
+ { U32 p;
+ for (p = 1; p < fastHashFillStep; ++p) {
+ size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls);
+ if (hashTable[hash] == 0) { /* not yet filled */
+ hashTable[hash] = curr + p;
+ } } } }
+}
+
+void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp)
+{
+ if (tfp == ZSTD_tfp_forCDict) {
+ ZSTD_fillHashTableForCDict(ms, end, dtlm);
+ } else {
+ ZSTD_fillHashTableForCCtx(ms, end, dtlm);
+ }
+}
+
+
+/**
+ * If you squint hard enough (and ignore repcodes), the search operation at any
+ * given position is broken into 4 stages:
+ *
+ * 1. Hash (map position to hash value via input read)
+ * 2. Lookup (map hash val to index via hashtable read)
+ * 3. Load (map index to value at that position via input read)
+ * 4. Compare
+ *
+ * Each of these steps involves a memory read at an address which is computed
+ * from the previous step. This means these steps must be sequenced and their
+ * latencies are cumulative.
+ *
+ * Rather than do 1->2->3->4 sequentially for a single position before moving
+ * onto the next, this implementation interleaves these operations across the
+ * next few positions:
+ *
+ * R = Repcode Read & Compare
+ * H = Hash
+ * T = Table Lookup
+ * M = Match Read & Compare
+ *
+ * Pos | Time -->
+ * ----+-------------------
+ * N | ... M
+ * N+1 | ... TM
+ * N+2 | R H T M
+ * N+3 | H TM
+ * N+4 | R H T M
+ * N+5 | H ...
+ * N+6 | R ...
+ *
+ * This is very much analogous to the pipelining of execution in a CPU. And just
+ * like a CPU, we have to dump the pipeline when we find a match (i.e., take a
+ * branch).
+ *
+ * When this happens, we throw away our current state, and do the following prep
+ * to re-enter the loop:
+ *
+ * Pos | Time -->
+ * ----+-------------------
+ * N | H T
+ * N+1 | H
+ *
+ * This is also the work we do at the beginning to enter the loop initially.
+ */
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_compressBlock_fast_noDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize,
+ U32 const mls, U32 const hasStep)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hlog = cParams->hashLog;
+ /* support stepSize of 0 */
+ size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+
+ const BYTE* anchor = istart;
+ const BYTE* ip0 = istart;
+ const BYTE* ip1;
+ const BYTE* ip2;
+ const BYTE* ip3;
+ U32 current0;
+
+ U32 rep_offset1 = rep[0];
+ U32 rep_offset2 = rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ size_t hash0; /* hash for ip0 */
+ size_t hash1; /* hash for ip1 */
+ U32 idx; /* match idx for ip0 */
+ U32 mval; /* src value at match idx */
+
+ U32 offcode;
+ const BYTE* match0;
+ size_t mLength;
+
+ /* ip0 and ip1 are always adjacent. The targetLength skipping and
+ * uncompressibility acceleration is applied to every other position,
+ * matching the behavior of #1562. step therefore represents the gap
+ * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */
+ size_t step;
+ const BYTE* nextStep;
+ const size_t kStepIncr = (1 << (kSearchStrength - 1));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
+ ip0 += (ip0 == prefixStart);
+ { U32 const curr = (U32)(ip0 - base);
+ U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog);
+ U32 const maxRep = curr - windowLow;
+ if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0;
+ if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0;
+ }
+
+ /* start each op */
+_start: /* Requires: ip0 */
+
+ step = stepSize;
+ nextStep = ip0 + kStepIncr;
+
+ /* calculate positions, ip0 - anchor == 0, so we skip step calc */
+ ip1 = ip0 + 1;
+ ip2 = ip0 + step;
+ ip3 = ip2 + 1;
+
+ if (ip3 >= ilimit) {
+ goto _cleanup;
+ }
+
+ hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+ hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+
+ idx = hashTable[hash0];
+
+ do {
+ /* load repcode match for ip[2]*/
+ const U32 rval = MEM_read32(ip2 - rep_offset1);
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ /* check repcode at ip[2] */
+ if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) {
+ ip0 = ip2;
+ match0 = ip0 - rep_offset1;
+ mLength = ip0[-1] == match0[-1];
+ ip0 -= mLength;
+ match0 -= mLength;
+ offcode = REPCODE1_TO_OFFBASE;
+ mLength += 4;
+
+ /* First write next hash table entry; we've already calculated it.
+ * This write is known to be safe because the ip1 is before the
+ * repcode (ip2). */
+ hashTable[hash1] = (U32)(ip1 - base);
+
+ goto _match;
+ }
+
+ /* load match for ip[0] */
+ if (idx >= prefixStartIndex) {
+ mval = MEM_read32(base + idx);
+ } else {
+ mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
+ }
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+
+ /* First write next hash table entry; we've already calculated it.
+ * This write is known to be safe because the ip1 == ip0 + 1, so
+ * we know we will resume searching after ip1 */
+ hashTable[hash1] = (U32)(ip1 - base);
+
+ goto _offset;
+ }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip3;
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ /* load match for ip[0] */
+ if (idx >= prefixStartIndex) {
+ mval = MEM_read32(base + idx);
+ } else {
+ mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
+ }
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+
+ /* first write next hash table entry; we've already calculated it */
+ if (step <= 4) {
+ /* We need to avoid writing an index into the hash table >= the
+ * position at which we will pick up our searching after we've
+ * taken this match.
+ *
+ * The minimum possible match has length 4, so the earliest ip0
+ * can be after we take this match will be the current ip0 + 4.
+ * ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely
+ * write this position.
+ */
+ hashTable[hash1] = (U32)(ip1 - base);
+ }
+
+ goto _offset;
+ }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip0 + step;
+ ip3 = ip1 + step;
+
+ /* calculate step */
+ if (ip2 >= nextStep) {
+ step++;
+ PREFETCH_L1(ip1 + 64);
+ PREFETCH_L1(ip1 + 128);
+ nextStep += kStepIncr;
+ }
+ } while (ip3 < ilimit);
+
+_cleanup:
+ /* Note that there are probably still a couple positions we could search.
+ * However, it seems to be a meaningful performance hit to try to search
+ * them. So let's not. */
+
+ /* When the repcodes are outside of the prefix, we set them to zero before the loop.
+ * When the offsets are still zero, we need to restore them after the block to have a correct
+ * repcode history. If only one offset was invalid, it is easy. The tricky case is when both
+ * offsets were invalid. We need to figure out which offset to refill with.
+ * - If both offsets are zero they are in the same order.
+ * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`.
+ * - If only one is zero, we need to decide which offset to restore.
+ * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1.
+ * - It is impossible for rep_offset2 to be non-zero.
+ *
+ * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then
+ * set rep[0] = rep_offset1 and rep[1] = offsetSaved1.
+ */
+ offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1;
+ rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+
+_offset: /* Requires: ip0, idx */
+
+ /* Compute the offset code. */
+ match0 = base + idx;
+ rep_offset2 = rep_offset1;
+ rep_offset1 = (U32)(ip0-match0);
+ offcode = OFFSET_TO_OFFBASE(rep_offset1);
+ mLength = 4;
+
+ /* Count the backwards match length. */
+ while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) {
+ ip0--;
+ match0--;
+ mLength++;
+ }
+
+_match: /* Requires: ip0, match0, offcode */
+
+ /* Count the forward length. */
+ mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend);
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
+
+ ip0 += mLength;
+ anchor = ip0;
+
+ /* Fill table and check for immediate repcode. */
+ if (ip0 <= ilimit) {
+ /* Fill Table */
+ assert(base+current0+2 > istart); /* check base overflow */
+ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
+ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
+
+ if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */
+ while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) {
+ /* store sequence */
+ size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4;
+ { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */
+ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
+ ip0 += rLength;
+ ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
+ anchor = ip0;
+ continue; /* faster when present (confirmed on gcc-8) ... (?) */
+ } } }
+
+ goto _start;
+}
+
+#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \
+ static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
+ void const* src, size_t srcSize) \
+ { \
+ return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \
+ }
+
+ZSTD_GEN_FAST_FN(noDict, 4, 1)
+ZSTD_GEN_FAST_FN(noDict, 5, 1)
+ZSTD_GEN_FAST_FN(noDict, 6, 1)
+ZSTD_GEN_FAST_FN(noDict, 7, 1)
+
+ZSTD_GEN_FAST_FN(noDict, 4, 0)
+ZSTD_GEN_FAST_FN(noDict, 5, 0)
+ZSTD_GEN_FAST_FN(noDict, 6, 0)
+ZSTD_GEN_FAST_FN(noDict, 7, 0)
+
+size_t ZSTD_compressBlock_fast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ assert(ms->dictMatchState == NULL);
+ if (ms->cParams.targetLength > 1) {
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize);
+ }
+ } else {
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize);
+ }
+
+ }
+}
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_fast_dictMatchState_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hlog = cParams->hashLog;
+ /* support stepSize of 0 */
+ U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip0 = istart;
+ const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */
+ const BYTE* anchor = istart;
+ const U32 prefixStartIndex = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1=rep[0], offset_2=rep[1];
+
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
+ const U32* const dictHashTable = dms->hashTable;
+ const U32 dictStartIndex = dms->window.dictLimit;
+ const BYTE* const dictBase = dms->window.base;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const BYTE* const dictEnd = dms->window.nextSrc;
+ const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase);
+ const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart);
+ const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+
+ /* if a dictionary is still attached, it necessarily means that
+ * it is within window size. So we just check it. */
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ assert(endIndex - prefixStartIndex <= maxDistance);
+ (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */
+
+ (void)hasStep; /* not currently specialized on whether it's accelerated */
+
+ /* ensure there will be no underflow
+ * when translating a dict index into a local index */
+ assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
+
+ if (ms->prefetchCDictTables) {
+ size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
+ PREFETCH_AREA(dictHashTable, hashTableBytes)
+ }
+
+ /* init */
+ DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic");
+ ip0 += (dictAndPrefixLength == 0);
+ /* dictMatchState repCode checks don't currently handle repCode == 0
+ * disabling. */
+ assert(offset_1 <= dictAndPrefixLength);
+ assert(offset_2 <= dictAndPrefixLength);
+
+ /* Outer search loop */
+ assert(stepSize >= 1);
+ while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */
+ size_t mLength;
+ size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+
+ size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls);
+ U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS];
+ int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0);
+
+ U32 matchIndex = hashTable[hash0];
+ U32 curr = (U32)(ip0 - base);
+ size_t step = stepSize;
+ const size_t kStepIncr = 1 << kSearchStrength;
+ const BYTE* nextStep = ip0 + kStepIncr;
+
+ /* Inner search loop */
+ while (1) {
+ const BYTE* match = base + matchIndex;
+ const U32 repIndex = curr + 1 - offset_1;
+ const BYTE* repMatch = (repIndex < prefixStartIndex) ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+ size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls);
+ hashTable[hash0] = curr; /* update hash table */
+
+ if (((U32) ((prefixStartIndex - 1) - repIndex) >=
+ 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
+ && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) {
+ const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4;
+ ip0++;
+ ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ break;
+ }
+
+ if (dictTagsMatch) {
+ /* Found a possible dict match */
+ const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* dictMatch = dictBase + dictMatchIndex;
+ if (dictMatchIndex > dictStartIndex &&
+ MEM_read32(dictMatch) == MEM_read32(ip0)) {
+ /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */
+ if (matchIndex <= prefixStartIndex) {
+ U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta);
+ mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4;
+ while (((ip0 > anchor) & (dictMatch > dictStart))
+ && (ip0[-1] == dictMatch[-1])) {
+ ip0--;
+ dictMatch--;
+ mLength++;
+ } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+ break;
+ }
+ }
+ }
+
+ if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) {
+ /* found a regular match */
+ U32 const offset = (U32) (ip0 - match);
+ mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4;
+ while (((ip0 > anchor) & (match > prefixStart))
+ && (ip0[-1] == match[-1])) {
+ ip0--;
+ match--;
+ mLength++;
+ } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+ break;
+ }
+
+ /* Prepare for next iteration */
+ dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS];
+ dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1);
+ matchIndex = hashTable[hash1];
+
+ if (ip1 >= nextStep) {
+ step++;
+ nextStep += kStepIncr;
+ }
+ ip0 = ip1;
+ ip1 = ip1 + step;
+ if (ip1 > ilimit) goto _cleanup;
+
+ curr = (U32)(ip0 - base);
+ hash0 = hash1;
+ } /* end inner search loop */
+
+ /* match found */
+ assert(mLength);
+ ip0 += mLength;
+ anchor = ip0;
+
+ if (ip0 <= ilimit) {
+ /* Fill Table */
+ assert(base+curr+2 > istart); /* check base overflow */
+ hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */
+ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
+
+ /* check immediate repcode */
+ while (ip0 <= ilimit) {
+ U32 const current2 = (U32)(ip0-base);
+ U32 const repIndex2 = current2 - offset_2;
+ const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
+ dictBase - dictIndexDelta + repIndex2 :
+ base + repIndex2;
+ if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
+ && (MEM_read32(repMatch2) == MEM_read32(ip0))) {
+ const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+ U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2;
+ ip0 += repLength2;
+ anchor = ip0;
+ continue;
+ }
+ break;
+ }
+ }
+
+ /* Prepare for next iteration */
+ assert(ip0 == anchor);
+ ip1 = ip0 + stepSize;
+ }
+
+_cleanup:
+ /* save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+
+ZSTD_GEN_FAST_FN(dictMatchState, 4, 0)
+ZSTD_GEN_FAST_FN(dictMatchState, 5, 0)
+ZSTD_GEN_FAST_FN(dictMatchState, 6, 0)
+ZSTD_GEN_FAST_FN(dictMatchState, 7, 0)
+
+size_t ZSTD_compressBlock_fast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ assert(ms->dictMatchState != NULL);
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize);
+ }
+}
+
+
+static size_t ZSTD_compressBlock_fast_extDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hlog = cParams->hashLog;
+ /* support stepSize of 0 */
+ size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* anchor = istart;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
+ const U32 dictStartIndex = lowLimit;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const U32 dictLimit = ms->window.dictLimit;
+ const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit;
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const dictEnd = dictBase + prefixStartIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - 8;
+ U32 offset_1=rep[0], offset_2=rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ const BYTE* ip0 = istart;
+ const BYTE* ip1;
+ const BYTE* ip2;
+ const BYTE* ip3;
+ U32 current0;
+
+
+ size_t hash0; /* hash for ip0 */
+ size_t hash1; /* hash for ip1 */
+ U32 idx; /* match idx for ip0 */
+ const BYTE* idxBase; /* base pointer for idx */
+
+ U32 offcode;
+ const BYTE* match0;
+ size_t mLength;
+ const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */
+
+ size_t step;
+ const BYTE* nextStep;
+ const size_t kStepIncr = (1 << (kSearchStrength - 1));
+
+ (void)hasStep; /* not currently specialized on whether it's accelerated */
+
+ DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1);
+
+ /* switch to "regular" variant if extDict is invalidated due to maxDistance */
+ if (prefixStartIndex == dictStartIndex)
+ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
+
+ { U32 const curr = (U32)(ip0 - base);
+ U32 const maxRep = curr - dictStartIndex;
+ if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+ if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0;
+ }
+
+ /* start each op */
+_start: /* Requires: ip0 */
+
+ step = stepSize;
+ nextStep = ip0 + kStepIncr;
+
+ /* calculate positions, ip0 - anchor == 0, so we skip step calc */
+ ip1 = ip0 + 1;
+ ip2 = ip0 + step;
+ ip3 = ip2 + 1;
+
+ if (ip3 >= ilimit) {
+ goto _cleanup;
+ }
+
+ hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+ hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+
+ idx = hashTable[hash0];
+ idxBase = idx < prefixStartIndex ? dictBase : base;
+
+ do {
+ { /* load repcode match for ip[2] */
+ U32 const current2 = (U32)(ip2 - base);
+ U32 const repIndex = current2 - offset_1;
+ const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
+ U32 rval;
+ if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */
+ & (offset_1 > 0) ) {
+ rval = MEM_read32(repBase + repIndex);
+ } else {
+ rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */
+ }
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ /* check repcode at ip[2] */
+ if (MEM_read32(ip2) == rval) {
+ ip0 = ip2;
+ match0 = repBase + repIndex;
+ matchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+ assert((match0 != prefixStart) & (match0 != dictStart));
+ mLength = ip0[-1] == match0[-1];
+ ip0 -= mLength;
+ match0 -= mLength;
+ offcode = REPCODE1_TO_OFFBASE;
+ mLength += 4;
+ goto _match;
+ } }
+
+ { /* load match for ip[0] */
+ U32 const mval = idx >= dictStartIndex ?
+ MEM_read32(idxBase + idx) :
+ MEM_read32(ip0) ^ 1; /* guaranteed not to match */
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+ goto _offset;
+ } }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+ idxBase = idx < prefixStartIndex ? dictBase : base;
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip3;
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ { /* load match for ip[0] */
+ U32 const mval = idx >= dictStartIndex ?
+ MEM_read32(idxBase + idx) :
+ MEM_read32(ip0) ^ 1; /* guaranteed not to match */
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+ goto _offset;
+ } }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+ idxBase = idx < prefixStartIndex ? dictBase : base;
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip0 + step;
+ ip3 = ip1 + step;
+
+ /* calculate step */
+ if (ip2 >= nextStep) {
+ step++;
+ PREFETCH_L1(ip1 + 64);
+ PREFETCH_L1(ip1 + 128);
+ nextStep += kStepIncr;
+ }
+ } while (ip3 < ilimit);
+
+_cleanup:
+ /* Note that there are probably still a couple positions we could search.
+ * However, it seems to be a meaningful performance hit to try to search
+ * them. So let's not. */
+
+ /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+ * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+ offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = offset_1 ? offset_1 : offsetSaved1;
+ rep[1] = offset_2 ? offset_2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+
+_offset: /* Requires: ip0, idx, idxBase */
+
+ /* Compute the offset code. */
+ { U32 const offset = current0 - idx;
+ const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart;
+ matchEnd = idx < prefixStartIndex ? dictEnd : iend;
+ match0 = idxBase + idx;
+ offset_2 = offset_1;
+ offset_1 = offset;
+ offcode = OFFSET_TO_OFFBASE(offset);
+ mLength = 4;
+
+ /* Count the backwards match length. */
+ while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) {
+ ip0--;
+ match0--;
+ mLength++;
+ } }
+
+_match: /* Requires: ip0, match0, offcode, matchEnd */
+
+ /* Count the forward length. */
+ assert(matchEnd != 0);
+ mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart);
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
+
+ ip0 += mLength;
+ anchor = ip0;
+
+ /* write next hash table entry */
+ if (ip1 < ip0) {
+ hashTable[hash1] = (U32)(ip1 - base);
+ }
+
+ /* Fill table and check for immediate repcode. */
+ if (ip0 <= ilimit) {
+ /* Fill Table */
+ assert(base+current0+2 > istart); /* check base overflow */
+ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
+ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
+
+ while (ip0 <= ilimit) {
+ U32 const repIndex2 = (U32)(ip0-base) - offset_2;
+ const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
+ if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */
+ && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) {
+ const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+ { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
+ ip0 += repLength2;
+ anchor = ip0;
+ continue;
+ }
+ break;
+ } }
+
+ goto _start;
+}
+
+ZSTD_GEN_FAST_FN(extDict, 4, 0)
+ZSTD_GEN_FAST_FN(extDict, 5, 0)
+ZSTD_GEN_FAST_FN(extDict, 6, 0)
+ZSTD_GEN_FAST_FN(extDict, 7, 0)
+
+size_t ZSTD_compressBlock_fast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ assert(ms->dictMatchState == NULL);
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize);
+ }
+}
diff --git a/contrib/zstd/zstd_fast.h b/contrib/zstd/zstd_fast.h
new file mode 100644
index 0000000..3e81de4
--- /dev/null
+++ b/contrib/zstd/zstd_fast.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_FAST_H
+#define ZSTD_FAST_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "mem.h" /* U32 */
+#include "zstd_compress_internal.h"
+
+void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp);
+size_t ZSTD_compressBlock_fast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_fast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_fast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_FAST_H */
diff --git a/contrib/zstd/zstd_internal.h b/contrib/zstd/zstd_internal.h
new file mode 100644
index 0000000..8f4f64b
--- /dev/null
+++ b/contrib/zstd/zstd_internal.h
@@ -0,0 +1,397 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_CCOMMON_H_MODULE
+#define ZSTD_CCOMMON_H_MODULE
+
+/* this module contains definitions which must be identical
+ * across compression, decompression and dictBuilder.
+ * It also contains a few functions useful to at least 2 of them
+ * and which benefit from being inlined */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "compiler.h"
+#include "cpu.h"
+#include "mem.h"
+#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
+#include "error_private.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#ifndef XXH_STATIC_LINKING_ONLY
+# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+#endif
+#include "xxhash.h" /* XXH_reset, update, digest */
+#ifndef ZSTD_NO_TRACE
+# include "zstd_trace.h"
+#else
+# define ZSTD_TRACE 0
+#endif
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* ---- static assert (debug) --- */
+#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
+#define ZSTD_isError ERR_isError /* for inlining */
+#define FSE_isError ERR_isError
+#define HUF_isError ERR_isError
+
+
+/*-*************************************
+* shared macros
+***************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+#define BOUNDED(min,val,max) (MAX(min,MIN(val,max)))
+
+
+/*-*************************************
+* Common constants
+***************************************/
+#define ZSTD_OPT_NUM (1<<12)
+
+#define ZSTD_REP_NUM 3 /* number of repcodes */
+static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define BIT7 128
+#define BIT6 64
+#define BIT5 32
+#define BIT4 16
+#define BIT1 2
+#define BIT0 1
+
+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
+static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
+static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
+
+#define ZSTD_FRAMEIDSIZE 4 /* magic number size */
+
+#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
+static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
+typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
+
+#define ZSTD_FRAMECHECKSUMSIZE 4
+
+#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
+#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */
+#define MIN_LITERALS_FOR_4_STREAMS 6
+
+typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
+
+#define LONGNBSEQ 0x7F00
+
+#define MINMATCH 3
+
+#define Litbits 8
+#define LitHufLog 11
+#define MaxLit ((1<<Litbits) - 1)
+#define MaxML 52
+#define MaxLL 35
+#define DefaultMaxOff 28
+#define MaxOff 31
+#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
+#define MLFSELog 9
+#define LLFSELog 9
+#define OffFSELog 8
+#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
+#define MaxMLBits 16
+#define MaxLLBits 16
+
+#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
+/* Each table cannot take more than #symbols * FSELog bits */
+#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8)
+
+static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 2, 2, 3, 3,
+ 4, 6, 7, 8, 9,10,11,12,
+ 13,14,15,16
+};
+static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = {
+ 4, 3, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 1, 1, 1,
+ 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 3, 2, 1, 1, 1, 1, 1,
+ -1,-1,-1,-1
+};
+#define LL_DEFAULTNORMLOG 6 /* for static allocation */
+static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
+
+static UNUSED_ATTR const U8 ML_bits[MaxML+1] = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 2, 2, 3, 3,
+ 4, 4, 5, 7, 8, 9,10,11,
+ 12,13,14,15,16
+};
+static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = {
+ 1, 4, 3, 2, 2, 2, 2, 2,
+ 2, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1,-1,-1,
+ -1,-1,-1,-1,-1
+};
+#define ML_DEFAULTNORMLOG 6 /* for static allocation */
+static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
+
+static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = {
+ 1, 1, 1, 1, 1, 1, 2, 2,
+ 2, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ -1,-1,-1,-1,-1
+};
+#define OF_DEFAULTNORMLOG 5 /* for static allocation */
+static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
+
+
+/*-*******************************************
+* Shared functions to include for inlining
+*********************************************/
+static void ZSTD_copy8(void* dst, const void* src) {
+#if defined(ZSTD_ARCH_ARM_NEON)
+ vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
+#else
+ ZSTD_memcpy(dst, src, 8);
+#endif
+}
+#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
+
+/* Need to use memmove here since the literal buffer can now be located within
+ the dst buffer. In circumstances where the op "catches up" to where the
+ literal buffer is, there can be partial overlaps in this call on the final
+ copy if the literal is being shifted by less than 16 bytes. */
+static void ZSTD_copy16(void* dst, const void* src) {
+#if defined(ZSTD_ARCH_ARM_NEON)
+ vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
+#elif defined(ZSTD_ARCH_X86_SSE2)
+ _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
+#elif defined(__clang__)
+ ZSTD_memmove(dst, src, 16);
+#else
+ /* ZSTD_memmove is not inlined properly by gcc */
+ BYTE copy16_buf[16];
+ ZSTD_memcpy(copy16_buf, src, 16);
+ ZSTD_memcpy(dst, copy16_buf, 16);
+#endif
+}
+#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
+
+#define WILDCOPY_OVERLENGTH 32
+#define WILDCOPY_VECLEN 16
+
+typedef enum {
+ ZSTD_no_overlap,
+ ZSTD_overlap_src_before_dst
+ /* ZSTD_overlap_dst_before_src, */
+} ZSTD_overlap_e;
+
+/*! ZSTD_wildcopy() :
+ * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
+ * @param ovtype controls the overlap detection
+ * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
+ * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
+ * The src buffer must be before the dst buffer.
+ */
+MEM_STATIC FORCE_INLINE_ATTR
+void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
+{
+ ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + length;
+
+ if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
+ /* Handle short offset copies. */
+ do {
+ COPY8(op, ip)
+ } while (op < oend);
+ } else {
+ assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
+ /* Separate out the first COPY16() call because the copy length is
+ * almost certain to be short, so the branches have different
+ * probabilities. Since it is almost certain to be short, only do
+ * one COPY16() in the first call. Then, do two calls per loop since
+ * at that point it is more likely to have a high trip count.
+ */
+ ZSTD_copy16(op, ip);
+ if (16 >= length) return;
+ op += 16;
+ ip += 16;
+ do {
+ COPY16(op, ip);
+ COPY16(op, ip);
+ }
+ while (op < oend);
+ }
+}
+
+MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ size_t const length = MIN(dstCapacity, srcSize);
+ if (length > 0) {
+ ZSTD_memcpy(dst, src, length);
+ }
+ return length;
+}
+
+/* define "workspace is too large" as this number of times larger than needed */
+#define ZSTD_WORKSPACETOOLARGE_FACTOR 3
+
+/* when workspace is continuously too large
+ * during at least this number of times,
+ * context's memory usage is considered wasteful,
+ * because it's sized to handle a worst case scenario which rarely happens.
+ * In which case, resize it down to free some memory */
+#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
+
+/* Controls whether the input/output buffer is buffered or stable. */
+typedef enum {
+ ZSTD_bm_buffered = 0, /* Buffer the input/output */
+ ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
+} ZSTD_bufferMode_e;
+
+
+/*-*******************************************
+* Private declarations
+*********************************************/
+typedef struct seqDef_s {
+ U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
+ U16 litLength;
+ U16 mlBase; /* mlBase == matchLength - MINMATCH */
+} seqDef;
+
+/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
+typedef enum {
+ ZSTD_llt_none = 0, /* no longLengthType */
+ ZSTD_llt_literalLength = 1, /* represents a long literal */
+ ZSTD_llt_matchLength = 2 /* represents a long match */
+} ZSTD_longLengthType_e;
+
+typedef struct {
+ seqDef* sequencesStart;
+ seqDef* sequences; /* ptr to end of sequences */
+ BYTE* litStart;
+ BYTE* lit; /* ptr to end of literals */
+ BYTE* llCode;
+ BYTE* mlCode;
+ BYTE* ofCode;
+ size_t maxNbSeq;
+ size_t maxNbLit;
+
+ /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
+ * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
+ * the existing value of the litLength or matchLength by 0x10000.
+ */
+ ZSTD_longLengthType_e longLengthType;
+ U32 longLengthPos; /* Index of the sequence to apply long length modification to */
+} seqStore_t;
+
+typedef struct {
+ U32 litLength;
+ U32 matchLength;
+} ZSTD_sequenceLength;
+
+/**
+ * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
+ * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
+ */
+MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
+{
+ ZSTD_sequenceLength seqLen;
+ seqLen.litLength = seq->litLength;
+ seqLen.matchLength = seq->mlBase + MINMATCH;
+ if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
+ if (seqStore->longLengthType == ZSTD_llt_literalLength) {
+ seqLen.litLength += 0x10000;
+ }
+ if (seqStore->longLengthType == ZSTD_llt_matchLength) {
+ seqLen.matchLength += 0x10000;
+ }
+ }
+ return seqLen;
+}
+
+/**
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
+ * Note: before using `compressedSize`, check for errors using ZSTD_isError().
+ * similarly, before using `decompressedBound`, check for errors using:
+ * `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
+ */
+typedef struct {
+ size_t nbBlocks;
+ size_t compressedSize;
+ unsigned long long decompressedBound;
+} ZSTD_frameSizeInfo; /* decompress & legacy */
+
+const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
+
+/* custom memory allocation functions */
+void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
+void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
+void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
+
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
+
+
+typedef struct {
+ blockType_e blockType;
+ U32 lastBlock;
+ U32 origSize;
+} blockProperties_t; /* declared here for decompress and fullbench */
+
+/*! ZSTD_getcBlockSize() :
+ * Provides the size of compressed block from block header `src` */
+/* Used by: decompress, fullbench (does not get its definition from here) */
+size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
+ blockProperties_t* bpPtr);
+
+/*! ZSTD_decodeSeqHeaders() :
+ * decode sequence header from src */
+/* Used by: decompress, fullbench (does not get its definition from here) */
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+ const void* src, size_t srcSize);
+
+/**
+ * @returns true iff the CPU supports dynamic BMI2 dispatch.
+ */
+MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
+{
+ ZSTD_cpuid_t cpuid = ZSTD_cpuid();
+ return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_CCOMMON_H_MODULE */
diff --git a/contrib/zstd/zstd_lazy.c b/contrib/zstd/zstd_lazy.c
new file mode 100644
index 0000000..297f520
--- /dev/null
+++ b/contrib/zstd/zstd_lazy.c
@@ -0,0 +1,2127 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "zstd_lazy.h"
+#include "bits.h" /* ZSTD_countTrailingZeros64 */
+
+
+/*-*************************************
+* Binary Tree search
+***************************************/
+
+static void
+ZSTD_updateDUBT(ZSTD_matchState_t* ms,
+ const BYTE* ip, const BYTE* iend,
+ U32 mls)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hashLog = cParams->hashLog;
+
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+
+ const BYTE* const base = ms->window.base;
+ U32 const target = (U32)(ip - base);
+ U32 idx = ms->nextToUpdate;
+
+ if (idx != target)
+ DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)",
+ idx, target, ms->window.dictLimit);
+ assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */
+ (void)iend;
+
+ assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */
+ for ( ; idx < target ; idx++) {
+ size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */
+ U32 const matchIndex = hashTable[h];
+
+ U32* const nextCandidatePtr = bt + 2*(idx&btMask);
+ U32* const sortMarkPtr = nextCandidatePtr + 1;
+
+ DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx);
+ hashTable[h] = idx; /* Update Hash Table */
+ *nextCandidatePtr = matchIndex; /* update BT like a chain */
+ *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK;
+ }
+ ms->nextToUpdate = target;
+}
+
+
+/** ZSTD_insertDUBT1() :
+ * sort one already inserted but unsorted position
+ * assumption : curr >= btlow == (curr - btmask)
+ * doesn't fail */
+static void
+ZSTD_insertDUBT1(const ZSTD_matchState_t* ms,
+ U32 curr, const BYTE* inputEnd,
+ U32 nbCompares, U32 btLow,
+ const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr;
+ const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* match;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = smallerPtr + 1;
+ U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */
+ U32 dummy32; /* to be nullified at the end */
+ U32 const windowValid = ms->window.lowLimit;
+ U32 const maxDistance = 1U << cParams->windowLog;
+ U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid;
+
+
+ DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)",
+ curr, dictLimit, windowLow);
+ assert(curr >= btLow);
+ assert(ip < iend); /* condition for ZSTD_count */
+
+ for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ assert(matchIndex < curr);
+ /* note : all candidates are now supposed sorted,
+ * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK
+ * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */
+
+ if ( (dictMode != ZSTD_extDict)
+ || (matchIndex+matchLength >= dictLimit) /* both in current segment*/
+ || (curr < dictLimit) /* both in extDict */) {
+ const BYTE* const mBase = ( (dictMode != ZSTD_extDict)
+ || (matchIndex+matchLength >= dictLimit)) ?
+ base : dictBase;
+ assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */
+ || (curr < dictLimit) );
+ match = mBase + matchIndex;
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* preparation for next read of match[matchLength] */
+ }
+
+ DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ",
+ curr, matchIndex, (U32)matchLength);
+
+ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
+ }
+
+ if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */
+ /* match is smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u",
+ matchIndex, btLow, nextPtr[1]);
+ smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */
+ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */
+ } else {
+ /* match is larger than current */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u",
+ matchIndex, btLow, nextPtr[0]);
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+}
+
+
+static size_t
+ZSTD_DUBT_findBetterDictMatch (
+ const ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ size_t* offsetPtr,
+ size_t bestLength,
+ U32 nbCompares,
+ U32 const mls,
+ const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_matchState_t * const dms = ms->dictMatchState;
+ const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
+ const U32 * const dictHashTable = dms->hashTable;
+ U32 const hashLog = dmsCParams->hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 dictMatchIndex = dictHashTable[h];
+
+ const BYTE* const base = ms->window.base;
+ const BYTE* const prefixStart = base + ms->window.dictLimit;
+ U32 const curr = (U32)(ip-base);
+ const BYTE* const dictBase = dms->window.base;
+ const BYTE* const dictEnd = dms->window.nextSrc;
+ U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base);
+ U32 const dictLowLimit = dms->window.lowLimit;
+ U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit;
+
+ U32* const dictBt = dms->chainTable;
+ U32 const btLog = dmsCParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;
+
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+
+ (void)dictMode;
+ assert(dictMode == ZSTD_dictMatchState);
+
+ for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) {
+ U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE* match = dictBase + dictMatchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (dictMatchIndex+matchLength >= dictHighLimit)
+ match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */
+
+ if (matchLength > bestLength) {
+ U32 matchIndex = dictMatchIndex + dictIndexDelta;
+ if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
+ DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
+ curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex);
+ bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ }
+ if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ }
+ }
+
+ if (match[matchLength] < ip[matchLength]) {
+ if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ } else {
+ /* match is larger than current */
+ if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */
+ commonLengthLarger = matchLength;
+ dictMatchIndex = nextPtr[0];
+ }
+ }
+
+ if (bestLength >= MINMATCH) {
+ U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex;
+ DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
+ curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
+ }
+ return bestLength;
+
+}
+
+
+static size_t
+ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ size_t* offBasePtr,
+ U32 const mls,
+ const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hashLog = cParams->hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 matchIndex = hashTable[h];
+
+ const BYTE* const base = ms->window.base;
+ U32 const curr = (U32)(ip-base);
+ U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog);
+
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 const btLow = (btMask >= curr) ? 0 : curr - btMask;
+ U32 const unsortLimit = MAX(btLow, windowLow);
+
+ U32* nextCandidate = bt + 2*(matchIndex&btMask);
+ U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1;
+ U32 nbCompares = 1U << cParams->searchLog;
+ U32 nbCandidates = nbCompares;
+ U32 previousCandidate = 0;
+
+ DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr);
+ assert(ip <= iend-8); /* required for h calculation */
+ assert(dictMode != ZSTD_dedicatedDictSearch);
+
+ /* reach end of unsorted candidates list */
+ while ( (matchIndex > unsortLimit)
+ && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK)
+ && (nbCandidates > 1) ) {
+ DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted",
+ matchIndex);
+ *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */
+ previousCandidate = matchIndex;
+ matchIndex = *nextCandidate;
+ nextCandidate = bt + 2*(matchIndex&btMask);
+ unsortedMark = bt + 2*(matchIndex&btMask) + 1;
+ nbCandidates --;
+ }
+
+ /* nullify last candidate if it's still unsorted
+ * simplification, detrimental to compression ratio, beneficial for speed */
+ if ( (matchIndex > unsortLimit)
+ && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) {
+ DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u",
+ matchIndex);
+ *nextCandidate = *unsortedMark = 0;
+ }
+
+ /* batch sort stacked candidates */
+ matchIndex = previousCandidate;
+ while (matchIndex) { /* will end on matchIndex == 0 */
+ U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1;
+ U32 const nextCandidateIdx = *nextCandidateIdxPtr;
+ ZSTD_insertDUBT1(ms, matchIndex, iend,
+ nbCandidates, unsortLimit, dictMode);
+ matchIndex = nextCandidateIdx;
+ nbCandidates++;
+ }
+
+ /* find longest match */
+ { size_t commonLengthSmaller = 0, commonLengthLarger = 0;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = bt + 2*(curr&btMask) + 1;
+ U32 matchEndIdx = curr + 8 + 1;
+ U32 dummy32; /* to be nullified at the end */
+ size_t bestLength = 0;
+
+ matchIndex = hashTable[h];
+ hashTable[h] = curr; /* Update Hash Table */
+
+ for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE* match;
+
+ if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) {
+ match = base + matchIndex;
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
+ }
+
+ if (matchLength > bestLength) {
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) )
+ bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
+ if (dictMode == ZSTD_dictMatchState) {
+ nbCompares = 0; /* in addition to avoiding checking any
+ * further in this loop, make sure we
+ * skip checking in the dictionary. */
+ }
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ }
+ }
+
+ if (match[matchLength] < ip[matchLength]) {
+ /* match is smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ } else {
+ /* match is larger than current */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+
+ assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dictMatchState && nbCompares) {
+ bestLength = ZSTD_DUBT_findBetterDictMatch(
+ ms, ip, iend,
+ offBasePtr, bestLength, nbCompares,
+ mls, dictMode);
+ }
+
+ assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */
+ ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
+ if (bestLength >= MINMATCH) {
+ U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex;
+ DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
+ curr, (U32)bestLength, (U32)*offBasePtr, mIndex);
+ }
+ return bestLength;
+ }
+}
+
+
+/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ size_t* offBasePtr,
+ const U32 mls /* template */,
+ const ZSTD_dictMode_e dictMode)
+{
+ DEBUGLOG(7, "ZSTD_BtFindBestMatch");
+ if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
+ ZSTD_updateDUBT(ms, ip, iLimit, mls);
+ return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode);
+}
+
+/***********************************
+* Dedicated dict search
+***********************************/
+
+void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip)
+{
+ const BYTE* const base = ms->window.base;
+ U32 const target = (U32)(ip - base);
+ U32* const hashTable = ms->hashTable;
+ U32* const chainTable = ms->chainTable;
+ U32 const chainSize = 1 << ms->cParams.chainLog;
+ U32 idx = ms->nextToUpdate;
+ U32 const minChain = chainSize < target - idx ? target - chainSize : idx;
+ U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32 const cacheSize = bucketSize - 1;
+ U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize;
+ U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts;
+
+ /* We know the hashtable is oversized by a factor of `bucketSize`.
+ * We are going to temporarily pretend `bucketSize == 1`, keeping only a
+ * single entry. We will use the rest of the space to construct a temporary
+ * chaintable.
+ */
+ U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32* const tmpHashTable = hashTable;
+ U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog);
+ U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog;
+ U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx;
+ U32 hashIdx;
+
+ assert(ms->cParams.chainLog <= 24);
+ assert(ms->cParams.hashLog > ms->cParams.chainLog);
+ assert(idx != 0);
+ assert(tmpMinChain <= minChain);
+
+ /* fill conventional hash table and conventional chain table */
+ for ( ; idx < target; idx++) {
+ U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch);
+ if (idx >= tmpMinChain) {
+ tmpChainTable[idx - tmpMinChain] = hashTable[h];
+ }
+ tmpHashTable[h] = idx;
+ }
+
+ /* sort chains into ddss chain table */
+ {
+ U32 chainPos = 0;
+ for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) {
+ U32 count;
+ U32 countBeyondMinChain = 0;
+ U32 i = tmpHashTable[hashIdx];
+ for (count = 0; i >= tmpMinChain && count < cacheSize; count++) {
+ /* skip through the chain to the first position that won't be
+ * in the hash cache bucket */
+ if (i < minChain) {
+ countBeyondMinChain++;
+ }
+ i = tmpChainTable[i - tmpMinChain];
+ }
+ if (count == cacheSize) {
+ for (count = 0; count < chainLimit;) {
+ if (i < minChain) {
+ if (!i || ++countBeyondMinChain > cacheSize) {
+ /* only allow pulling `cacheSize` number of entries
+ * into the cache or chainTable beyond `minChain`,
+ * to replace the entries pulled out of the
+ * chainTable into the cache. This lets us reach
+ * back further without increasing the total number
+ * of entries in the chainTable, guaranteeing the
+ * DDSS chain table will fit into the space
+ * allocated for the regular one. */
+ break;
+ }
+ }
+ chainTable[chainPos++] = i;
+ count++;
+ if (i < tmpMinChain) {
+ break;
+ }
+ i = tmpChainTable[i - tmpMinChain];
+ }
+ } else {
+ count = 0;
+ }
+ if (count) {
+ tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count;
+ } else {
+ tmpHashTable[hashIdx] = 0;
+ }
+ }
+ assert(chainPos <= chainSize); /* I believe this is guaranteed... */
+ }
+
+ /* move chain pointers into the last entry of each hash bucket */
+ for (hashIdx = (1 << hashLog); hashIdx; ) {
+ U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32 const chainPackedPointer = tmpHashTable[hashIdx];
+ U32 i;
+ for (i = 0; i < cacheSize; i++) {
+ hashTable[bucketIdx + i] = 0;
+ }
+ hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer;
+ }
+
+ /* fill the buckets of the hash table */
+ for (idx = ms->nextToUpdate; idx < target; idx++) {
+ U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch)
+ << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32 i;
+ /* Shift hash cache down 1. */
+ for (i = cacheSize - 1; i; i--)
+ hashTable[h + i] = hashTable[h + i - 1];
+ hashTable[h] = idx;
+ }
+
+ ms->nextToUpdate = target;
+}
+
+/* Returns the longest match length found in the dedicated dict search structure.
+ * If none are longer than the argument ml, then ml will be returned.
+ */
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts,
+ const ZSTD_matchState_t* const dms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ const BYTE* const prefixStart, const U32 curr,
+ const U32 dictLimit, const size_t ddsIdx) {
+ const U32 ddsLowestIndex = dms->window.dictLimit;
+ const BYTE* const ddsBase = dms->window.base;
+ const BYTE* const ddsEnd = dms->window.nextSrc;
+ const U32 ddsSize = (U32)(ddsEnd - ddsBase);
+ const U32 ddsIndexDelta = dictLimit - ddsSize;
+ const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG);
+ const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1;
+ U32 ddsAttempt;
+ U32 matchIndex;
+
+ for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) {
+ PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]);
+ }
+
+ {
+ U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
+ U32 const chainIndex = chainPackedPointer >> 8;
+
+ PREFETCH_L1(&dms->chainTable[chainIndex]);
+ }
+
+ for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) {
+ size_t currentMl=0;
+ const BYTE* match;
+ matchIndex = dms->hashTable[ddsIdx + ddsAttempt];
+ match = ddsBase + matchIndex;
+
+ if (!matchIndex) {
+ return ml;
+ }
+
+ /* guaranteed by table construction */
+ (void)ddsLowestIndex;
+ assert(matchIndex >= ddsLowestIndex);
+ assert(match+4 <= ddsEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) {
+ /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
+ }
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
+ if (ip+currentMl == iLimit) {
+ /* best possible, avoids read overflow on next attempt */
+ return ml;
+ }
+ }
+ }
+
+ {
+ U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
+ U32 chainIndex = chainPackedPointer >> 8;
+ U32 const chainLength = chainPackedPointer & 0xFF;
+ U32 const chainAttempts = nbAttempts - ddsAttempt;
+ U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts;
+ U32 chainAttempt;
+
+ for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) {
+ PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]);
+ }
+
+ for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) {
+ size_t currentMl=0;
+ const BYTE* match;
+ matchIndex = dms->chainTable[chainIndex];
+ match = ddsBase + matchIndex;
+
+ /* guaranteed by table construction */
+ assert(matchIndex >= ddsLowestIndex);
+ assert(match+4 <= ddsEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) {
+ /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
+ }
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+ }
+ }
+ return ml;
+}
+
+
+/* *********************************
+* Hash Chain
+***********************************/
+#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
+
+/* Update chains up to ip (excluded)
+ Assumption : always within prefix (i.e. not within extDict) */
+FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
+ ZSTD_matchState_t* ms,
+ const ZSTD_compressionParameters* const cParams,
+ const BYTE* ip, U32 const mls)
+{
+ U32* const hashTable = ms->hashTable;
+ const U32 hashLog = cParams->hashLog;
+ U32* const chainTable = ms->chainTable;
+ const U32 chainMask = (1 << cParams->chainLog) - 1;
+ const BYTE* const base = ms->window.base;
+ const U32 target = (U32)(ip - base);
+ U32 idx = ms->nextToUpdate;
+
+ while(idx < target) { /* catch up */
+ size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
+ NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
+ hashTable[h] = idx;
+ idx++;
+ }
+
+ ms->nextToUpdate = target;
+ return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
+}
+
+U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
+}
+
+/* inlining is important to hardwire a hot branch (template emulation) */
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_HcFindBestMatch(
+ ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ size_t* offsetPtr,
+ const U32 mls, const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const chainTable = ms->chainTable;
+ const U32 chainSize = (1 << cParams->chainLog);
+ const U32 chainMask = chainSize-1;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const U32 curr = (U32)(ip-base);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 lowestValid = ms->window.lowLimit;
+ const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ const U32 isDictionary = (ms->loadedDictEnd != 0);
+ const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
+ const U32 minChain = curr > chainSize ? curr - chainSize : 0;
+ U32 nbAttempts = 1U << cParams->searchLog;
+ size_t ml=4-1;
+
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch
+ ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
+ const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch
+ ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
+
+ U32 matchIndex;
+
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ const U32* entry = &dms->hashTable[ddsIdx];
+ PREFETCH_L1(entry);
+ }
+
+ /* HC4 match finder */
+ matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls);
+
+ for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) {
+ size_t currentMl=0;
+ if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
+ const BYTE* const match = base + matchIndex;
+ assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
+ /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+ if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
+ currentMl = ZSTD_count(ip, match, iLimit);
+ } else {
+ const BYTE* const match = dictBase + matchIndex;
+ assert(match+4 <= dictEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
+ }
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+
+ if (matchIndex <= minChain) break;
+ matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
+ }
+
+ assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms,
+ ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
+ } else if (dictMode == ZSTD_dictMatchState) {
+ const U32* const dmsChainTable = dms->chainTable;
+ const U32 dmsChainSize = (1 << dms->cParams.chainLog);
+ const U32 dmsChainMask = dmsChainSize - 1;
+ const U32 dmsLowestIndex = dms->window.dictLimit;
+ const BYTE* const dmsBase = dms->window.base;
+ const BYTE* const dmsEnd = dms->window.nextSrc;
+ const U32 dmsSize = (U32)(dmsEnd - dmsBase);
+ const U32 dmsIndexDelta = dictLimit - dmsSize;
+ const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0;
+
+ matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)];
+
+ for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) {
+ size_t currentMl=0;
+ const BYTE* const match = dmsBase + matchIndex;
+ assert(match+4 <= dmsEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4;
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ assert(curr > matchIndex + dmsIndexDelta);
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+
+ if (matchIndex <= dmsMinChain) break;
+
+ matchIndex = dmsChainTable[matchIndex & dmsChainMask];
+ }
+ }
+
+ return ml;
+}
+
+/* *********************************
+* (SIMD) Row-based matchfinder
+***********************************/
+/* Constants for row-based hash */
+#define ZSTD_ROW_HASH_TAG_OFFSET 16 /* byte offset of hashes in the match state's tagTable from the beginning of a row */
+#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
+#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */
+
+#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1)
+
+typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */
+
+/* ZSTD_VecMask_next():
+ * Starting from the LSB, returns the idx of the next non-zero bit.
+ * Basically counting the nb of trailing zeroes.
+ */
+MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
+ return ZSTD_countTrailingZeros64(val);
+}
+
+/* ZSTD_rotateRight_*():
+ * Rotates a bitfield to the right by "count" bits.
+ * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
+ */
+FORCE_INLINE_TEMPLATE
+U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
+ assert(count < 64);
+ count &= 0x3F; /* for fickle pattern recognition */
+ return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
+}
+
+FORCE_INLINE_TEMPLATE
+U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
+ assert(count < 32);
+ count &= 0x1F; /* for fickle pattern recognition */
+ return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
+}
+
+FORCE_INLINE_TEMPLATE
+U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
+ assert(count < 16);
+ count &= 0x0F; /* for fickle pattern recognition */
+ return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
+}
+
+/* ZSTD_row_nextIndex():
+ * Returns the next index to insert at within a tagTable row, and updates the "head"
+ * value to reflect the update. Essentially cycles backwards from [0, {entries per row})
+ */
+FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) {
+ U32 const next = (*tagRow - 1) & rowMask;
+ *tagRow = (BYTE)next;
+ return next;
+}
+
+/* ZSTD_isAligned():
+ * Checks that a pointer is aligned to "align" bytes which must be a power of 2.
+ */
+MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) {
+ assert((align & (align - 1)) == 0);
+ return (((size_t)ptr) & (align - 1)) == 0;
+}
+
+/* ZSTD_row_prefetch():
+ * Performs prefetching for the hashTable and tagTable at a given row.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) {
+ PREFETCH_L1(hashTable + relRow);
+ if (rowLog >= 5) {
+ PREFETCH_L1(hashTable + relRow + 16);
+ /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */
+ }
+ PREFETCH_L1(tagTable + relRow);
+ if (rowLog == 6) {
+ PREFETCH_L1(tagTable + relRow + 32);
+ }
+ assert(rowLog == 4 || rowLog == 5 || rowLog == 6);
+ assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */
+ assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */
+}
+
+/* ZSTD_row_fillHashCache():
+ * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries,
+ * but not beyond iLimit.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base,
+ U32 const rowLog, U32 const mls,
+ U32 idx, const BYTE* const iLimit)
+{
+ U32 const* const hashTable = ms->hashTable;
+ U16 const* const tagTable = ms->tagTable;
+ U32 const hashLog = ms->rowHashLog;
+ U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1);
+ U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch);
+
+ for (; idx < lim; ++idx) {
+ U32 const hash = (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
+ ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash;
+ }
+
+ DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1],
+ ms->hashCache[2], ms->hashCache[3], ms->hashCache[4],
+ ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]);
+}
+
+/* ZSTD_row_nextCachedHash():
+ * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at
+ * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable.
+ */
+FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
+ U16 const* tagTable, BYTE const* base,
+ U32 idx, U32 const hashLog,
+ U32 const rowLog, U32 const mls)
+{
+ U32 const newHash = (U32)ZSTD_hashPtr(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
+ { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK];
+ cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash;
+ return hash;
+ }
+}
+
+/* ZSTD_row_update_internalImpl():
+ * Updates the hash table with positions starting from updateStartIdx until updateEndIdx.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms,
+ U32 updateStartIdx, U32 const updateEndIdx,
+ U32 const mls, U32 const rowLog,
+ U32 const rowMask, U32 const useCache)
+{
+ U32* const hashTable = ms->hashTable;
+ U16* const tagTable = ms->tagTable;
+ U32 const hashLog = ms->rowHashLog;
+ const BYTE* const base = ms->window.base;
+
+ DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx);
+ for (; updateStartIdx < updateEndIdx; ++updateStartIdx) {
+ U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls)
+ : (U32)ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ U32* const row = hashTable + relRow;
+ BYTE* tagRow = (BYTE*)(tagTable + relRow); /* Though tagTable is laid out as a table of U16, each tag is only 1 byte.
+ Explicit cast allows us to get exact desired position within each row */
+ U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
+
+ assert(hash == ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls));
+ ((BYTE*)tagRow)[pos + ZSTD_ROW_HASH_TAG_OFFSET] = hash & ZSTD_ROW_HASH_TAG_MASK;
+ row[pos] = updateStartIdx;
+ }
+}
+
+/* ZSTD_row_update_internal():
+ * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate.
+ * Skips sections of long matches as is necessary.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip,
+ U32 const mls, U32 const rowLog,
+ U32 const rowMask, U32 const useCache)
+{
+ U32 idx = ms->nextToUpdate;
+ const BYTE* const base = ms->window.base;
+ const U32 target = (U32)(ip - base);
+ const U32 kSkipThreshold = 384;
+ const U32 kMaxMatchStartPositionsToUpdate = 96;
+ const U32 kMaxMatchEndPositionsToUpdate = 32;
+
+ if (useCache) {
+ /* Only skip positions when using hash cache, i.e.
+ * if we are loading a dict, don't skip anything.
+ * If we decide to skip, then we only update a set number
+ * of positions at the beginning and end of the match.
+ */
+ if (UNLIKELY(target - idx > kSkipThreshold)) {
+ U32 const bound = idx + kMaxMatchStartPositionsToUpdate;
+ ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache);
+ idx = target - kMaxMatchEndPositionsToUpdate;
+ ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1);
+ }
+ }
+ assert(target >= idx);
+ ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache);
+ ms->nextToUpdate = target;
+}
+
+/* ZSTD_row_update():
+ * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary
+ * processing.
+ */
+void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
+ const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
+ const U32 rowMask = (1u << rowLog) - 1;
+ const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
+
+ DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog);
+ ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */);
+}
+
+/* Returns the mask width of bits group of which will be set to 1. Given not all
+ * architectures have easy movemask instruction, this helps to iterate over
+ * groups of bits easier and faster.
+ */
+FORCE_INLINE_TEMPLATE U32
+ZSTD_row_matchMaskGroupWidth(const U32 rowEntries)
+{
+ assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+ assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
+ (void)rowEntries;
+#if defined(ZSTD_ARCH_ARM_NEON)
+ /* NEON path only works for little endian */
+ if (!MEM_isLittleEndian()) {
+ return 1;
+ }
+ if (rowEntries == 16) {
+ return 4;
+ }
+ if (rowEntries == 32) {
+ return 2;
+ }
+ if (rowEntries == 64) {
+ return 1;
+ }
+#endif
+ return 1;
+}
+
+#if defined(ZSTD_ARCH_X86_SSE2)
+FORCE_INLINE_TEMPLATE ZSTD_VecMask
+ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head)
+{
+ const __m128i comparisonMask = _mm_set1_epi8((char)tag);
+ int matches[4] = {0};
+ int i;
+ assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4);
+ for (i=0; i<nbChunks; i++) {
+ const __m128i chunk = _mm_loadu_si128((const __m128i*)(const void*)(src + 16*i));
+ const __m128i equalMask = _mm_cmpeq_epi8(chunk, comparisonMask);
+ matches[i] = _mm_movemask_epi8(equalMask);
+ }
+ if (nbChunks == 1) return ZSTD_rotateRight_U16((U16)matches[0], head);
+ if (nbChunks == 2) return ZSTD_rotateRight_U32((U32)matches[1] << 16 | (U32)matches[0], head);
+ assert(nbChunks == 4);
+ return ZSTD_rotateRight_U64((U64)matches[3] << 48 | (U64)matches[2] << 32 | (U64)matches[1] << 16 | (U64)matches[0], head);
+}
+#endif
+
+#if defined(ZSTD_ARCH_ARM_NEON)
+FORCE_INLINE_TEMPLATE ZSTD_VecMask
+ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag, const U32 headGrouped)
+{
+ assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+ if (rowEntries == 16) {
+ /* vshrn_n_u16 shifts by 4 every u16 and narrows to 8 lower bits.
+ * After that groups of 4 bits represent the equalMask. We lower
+ * all bits except the highest in these groups by doing AND with
+ * 0x88 = 0b10001000.
+ */
+ const uint8x16_t chunk = vld1q_u8(src);
+ const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag)));
+ const uint8x8_t res = vshrn_n_u16(equalMask, 4);
+ const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+ return ZSTD_rotateRight_U64(matches, headGrouped) & 0x8888888888888888ull;
+ } else if (rowEntries == 32) {
+ /* Same idea as with rowEntries == 16 but doing AND with
+ * 0x55 = 0b01010101.
+ */
+ const uint16x8x2_t chunk = vld2q_u16((const uint16_t*)(const void*)src);
+ const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]);
+ const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]);
+ const uint8x16_t dup = vdupq_n_u8(tag);
+ const uint8x8_t t0 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk0, dup)), 6);
+ const uint8x8_t t1 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk1, dup)), 6);
+ const uint8x8_t res = vsli_n_u8(t0, t1, 4);
+ const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0) ;
+ return ZSTD_rotateRight_U64(matches, headGrouped) & 0x5555555555555555ull;
+ } else { /* rowEntries == 64 */
+ const uint8x16x4_t chunk = vld4q_u8(src);
+ const uint8x16_t dup = vdupq_n_u8(tag);
+ const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup);
+ const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup);
+ const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup);
+ const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup);
+
+ const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1);
+ const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1);
+ const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2);
+ const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4);
+ const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4);
+ const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0);
+ return ZSTD_rotateRight_U64(matches, headGrouped);
+ }
+}
+#endif
+
+/* Returns a ZSTD_VecMask (U64) that has the nth group (determined by
+ * ZSTD_row_matchMaskGroupWidth) of bits set to 1 if the newly-computed "tag"
+ * matches the hash at the nth position in a row of the tagTable.
+ * Each row is a circular buffer beginning at the value of "headGrouped". So we
+ * must rotate the "matches" bitfield to match up with the actual layout of the
+ * entries within the hashTable */
+FORCE_INLINE_TEMPLATE ZSTD_VecMask
+ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGrouped, const U32 rowEntries)
+{
+ const BYTE* const src = tagRow + ZSTD_ROW_HASH_TAG_OFFSET;
+ assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+ assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
+ assert(ZSTD_row_matchMaskGroupWidth(rowEntries) * rowEntries <= sizeof(ZSTD_VecMask) * 8);
+
+#if defined(ZSTD_ARCH_X86_SSE2)
+
+ return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, headGrouped);
+
+#else /* SW or NEON-LE */
+
+# if defined(ZSTD_ARCH_ARM_NEON)
+ /* This NEON path only works for little endian - otherwise use SWAR below */
+ if (MEM_isLittleEndian()) {
+ return ZSTD_row_getNEONMask(rowEntries, src, tag, headGrouped);
+ }
+# endif /* ZSTD_ARCH_ARM_NEON */
+ /* SWAR */
+ { const int chunkSize = sizeof(size_t);
+ const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
+ const size_t xFF = ~((size_t)0);
+ const size_t x01 = xFF / 0xFF;
+ const size_t x80 = x01 << 7;
+ const size_t splatChar = tag * x01;
+ ZSTD_VecMask matches = 0;
+ int i = rowEntries - chunkSize;
+ assert((sizeof(size_t) == 4) || (sizeof(size_t) == 8));
+ if (MEM_isLittleEndian()) { /* runtime check so have two loops */
+ const size_t extractMagic = (xFF / 0x7F) >> chunkSize;
+ do {
+ size_t chunk = MEM_readST(&src[i]);
+ chunk ^= splatChar;
+ chunk = (((chunk | x80) - x01) | chunk) & x80;
+ matches <<= chunkSize;
+ matches |= (chunk * extractMagic) >> shiftAmount;
+ i -= chunkSize;
+ } while (i >= 0);
+ } else { /* big endian: reverse bits during extraction */
+ const size_t msb = xFF ^ (xFF >> 1);
+ const size_t extractMagic = (msb / 0x1FF) | msb;
+ do {
+ size_t chunk = MEM_readST(&src[i]);
+ chunk ^= splatChar;
+ chunk = (((chunk | x80) - x01) | chunk) & x80;
+ matches <<= chunkSize;
+ matches |= ((chunk >> 7) * extractMagic) >> shiftAmount;
+ i -= chunkSize;
+ } while (i >= 0);
+ }
+ matches = ~matches;
+ if (rowEntries == 16) {
+ return ZSTD_rotateRight_U16((U16)matches, headGrouped);
+ } else if (rowEntries == 32) {
+ return ZSTD_rotateRight_U32((U32)matches, headGrouped);
+ } else {
+ return ZSTD_rotateRight_U64((U64)matches, headGrouped);
+ }
+ }
+#endif
+}
+
+/* The high-level approach of the SIMD row based match finder is as follows:
+ * - Figure out where to insert the new entry:
+ * - Generate a hash from a byte along with an additional 1-byte "short hash". The additional byte is our "tag"
+ * - The hashTable is effectively split into groups or "rows" of 16 or 32 entries of U32, and the hash determines
+ * which row to insert into.
+ * - Determine the correct position within the row to insert the entry into. Each row of 16 or 32 can
+ * be considered as a circular buffer with a "head" index that resides in the tagTable.
+ * - Also insert the "tag" into the equivalent row and position in the tagTable.
+ * - Note: The tagTable has 17 or 33 1-byte entries per row, due to 16 or 32 tags, and 1 "head" entry.
+ * The 17 or 33 entry rows are spaced out to occur every 32 or 64 bytes, respectively,
+ * for alignment/performance reasons, leaving some bytes unused.
+ * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte "short hash" and
+ * generate a bitfield that we can cycle through to check the collisions in the hash table.
+ * - Pick the longest match.
+ */
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_RowFindBestMatch(
+ ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ size_t* offsetPtr,
+ const U32 mls, const ZSTD_dictMode_e dictMode,
+ const U32 rowLog)
+{
+ U32* const hashTable = ms->hashTable;
+ U16* const tagTable = ms->tagTable;
+ U32* const hashCache = ms->hashCache;
+ const U32 hashLog = ms->rowHashLog;
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const U32 curr = (U32)(ip-base);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 lowestValid = ms->window.lowLimit;
+ const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ const U32 isDictionary = (ms->loadedDictEnd != 0);
+ const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
+ const U32 rowEntries = (1U << rowLog);
+ const U32 rowMask = rowEntries - 1;
+ const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */
+ const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries);
+ U32 nbAttempts = 1U << cappedSearchLog;
+ size_t ml=4-1;
+
+ /* DMS/DDS variables that may be referenced laster */
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+
+ /* Initialize the following variables to satisfy static analyzer */
+ size_t ddsIdx = 0;
+ U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */
+ U32 dmsTag = 0;
+ U32* dmsRow = NULL;
+ BYTE* dmsTagRow = NULL;
+
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
+ { /* Prefetch DDS hashtable entry */
+ ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ PREFETCH_L1(&dms->hashTable[ddsIdx]);
+ }
+ ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0;
+ }
+
+ if (dictMode == ZSTD_dictMatchState) {
+ /* Prefetch DMS rows */
+ U32* const dmsHashTable = dms->hashTable;
+ U16* const dmsTagTable = dms->tagTable;
+ U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK;
+ dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow);
+ dmsRow = dmsHashTable + dmsRelRow;
+ ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog);
+ }
+
+ /* Update the hashTable and tagTable up to (but not including) ip */
+ ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
+ { /* Get the hash for ip, compute the appropriate row */
+ U32 const hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls);
+ U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK;
+ U32* const row = hashTable + relRow;
+ BYTE* tagRow = (BYTE*)(tagTable + relRow);
+ U32 const headGrouped = (*tagRow & rowMask) * groupWidth;
+ U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
+ size_t numMatches = 0;
+ size_t currMatch = 0;
+ ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries);
+
+ /* Cycle through the matches and prefetch */
+ for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
+ U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
+ U32 const matchIndex = row[matchPos];
+ assert(numMatches < rowEntries);
+ if (matchIndex < lowLimit)
+ break;
+ if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
+ PREFETCH_L1(base + matchIndex);
+ } else {
+ PREFETCH_L1(dictBase + matchIndex);
+ }
+ matchBuffer[numMatches++] = matchIndex;
+ }
+
+ /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop
+ in ZSTD_row_update_internal() at the next search. */
+ {
+ U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
+ tagRow[pos + ZSTD_ROW_HASH_TAG_OFFSET] = (BYTE)tag;
+ row[pos] = ms->nextToUpdate++;
+ }
+
+ /* Return the longest match */
+ for (; currMatch < numMatches; ++currMatch) {
+ U32 const matchIndex = matchBuffer[currMatch];
+ size_t currentMl=0;
+ assert(matchIndex < curr);
+ assert(matchIndex >= lowLimit);
+
+ if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
+ const BYTE* const match = base + matchIndex;
+ assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
+ /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+ if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
+ currentMl = ZSTD_count(ip, match, iLimit);
+ } else {
+ const BYTE* const match = dictBase + matchIndex;
+ assert(match+4 <= dictEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
+ }
+
+ /* Save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+ }
+ }
+
+ assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms,
+ ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
+ } else if (dictMode == ZSTD_dictMatchState) {
+ /* TODO: Measure and potentially add prefetching to DMS */
+ const U32 dmsLowestIndex = dms->window.dictLimit;
+ const BYTE* const dmsBase = dms->window.base;
+ const BYTE* const dmsEnd = dms->window.nextSrc;
+ const U32 dmsSize = (U32)(dmsEnd - dmsBase);
+ const U32 dmsIndexDelta = dictLimit - dmsSize;
+
+ { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth;
+ U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
+ size_t numMatches = 0;
+ size_t currMatch = 0;
+ ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries);
+
+ for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
+ U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
+ U32 const matchIndex = dmsRow[matchPos];
+ if (matchIndex < dmsLowestIndex)
+ break;
+ PREFETCH_L1(dmsBase + matchIndex);
+ matchBuffer[numMatches++] = matchIndex;
+ }
+
+ /* Return the longest match */
+ for (; currMatch < numMatches; ++currMatch) {
+ U32 const matchIndex = matchBuffer[currMatch];
+ size_t currentMl=0;
+ assert(matchIndex >= dmsLowestIndex);
+ assert(matchIndex < curr);
+
+ { const BYTE* const match = dmsBase + matchIndex;
+ assert(match+4 <= dmsEnd);
+ if (MEM_read32(match) == MEM_read32(ip))
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4;
+ }
+
+ if (currentMl > ml) {
+ ml = currentMl;
+ assert(curr > matchIndex + dmsIndexDelta);
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
+ if (ip+currentMl == iLimit) break;
+ }
+ }
+ }
+ }
+ return ml;
+}
+
+
+/**
+ * Generate search functions templated on (dictMode, mls, rowLog).
+ * These functions are outlined for code size & compilation time.
+ * ZSTD_searchMax() dispatches to the correct implementation function.
+ *
+ * TODO: The start of the search function involves loading and calculating a
+ * bunch of constants from the ZSTD_matchState_t. These computations could be
+ * done in an initialization function, and saved somewhere in the match state.
+ * Then we could pass a pointer to the saved state instead of the match state,
+ * and avoid duplicate computations.
+ *
+ * TODO: Move the match re-winding into searchMax. This improves compression
+ * ratio, and unlocks further simplifications with the next TODO.
+ *
+ * TODO: Try moving the repcode search into searchMax. After the re-winding
+ * and repcode search are in searchMax, there is no more logic in the match
+ * finder loop that requires knowledge about the dictMode. So we should be
+ * able to avoid force inlining it, and we can join the extDict loop with
+ * the single segment loop. It should go in searchMax instead of its own
+ * function to avoid having multiple virtual function calls per search.
+ */
+
+#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls
+#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls
+#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog
+
+#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE
+
+#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \
+ ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \
+ ZSTD_matchState_t* ms, \
+ const BYTE* ip, const BYTE* const iLimit, \
+ size_t* offBasePtr) \
+ { \
+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
+ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \
+ } \
+
+#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \
+ ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \
+ ZSTD_matchState_t* ms, \
+ const BYTE* ip, const BYTE* const iLimit, \
+ size_t* offsetPtr) \
+ { \
+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
+ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \
+ } \
+
+#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \
+ ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \
+ ZSTD_matchState_t* ms, \
+ const BYTE* ip, const BYTE* const iLimit, \
+ size_t* offsetPtr) \
+ { \
+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
+ assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \
+ return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \
+ } \
+
+#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \
+ X(dictMode, mls, 4) \
+ X(dictMode, mls, 5) \
+ X(dictMode, mls, 6)
+
+#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \
+ ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \
+ ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \
+ ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6)
+
+#define ZSTD_FOR_EACH_MLS(X, dictMode) \
+ X(dictMode, 4) \
+ X(dictMode, 5) \
+ X(dictMode, 6)
+
+#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \
+ X(__VA_ARGS__, noDict) \
+ X(__VA_ARGS__, extDict) \
+ X(__VA_ARGS__, dictMatchState) \
+ X(__VA_ARGS__, dedicatedDictSearch)
+
+/* Generate row search fns for each combination of (dictMode, mls, rowLog) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN)
+/* Generate binary Tree search fns for each combination of (dictMode, mls) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN)
+/* Generate hash chain search fns for each combination of (dictMode, mls) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN)
+
+typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e;
+
+#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \
+ case mls: \
+ return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
+#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \
+ case mls: \
+ return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
+#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \
+ case rowLog: \
+ return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr);
+
+#define ZSTD_SWITCH_MLS(X, dictMode) \
+ switch (mls) { \
+ ZSTD_FOR_EACH_MLS(X, dictMode) \
+ }
+
+#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \
+ case mls: \
+ switch (rowLog) { \
+ ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \
+ } \
+ ZSTD_UNREACHABLE; \
+ break;
+
+#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \
+ switch (searchMethod) { \
+ case search_hashChain: \
+ ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \
+ break; \
+ case search_binaryTree: \
+ ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \
+ break; \
+ case search_rowHash: \
+ ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \
+ break; \
+ } \
+ ZSTD_UNREACHABLE;
+
+/**
+ * Searches for the longest match at @p ip.
+ * Dispatches to the correct implementation function based on the
+ * (searchMethod, dictMode, mls, rowLog). We use switch statements
+ * here instead of using an indirect function call through a function
+ * pointer because after Spectre and Meltdown mitigations, indirect
+ * function calls can be very costly, especially in the kernel.
+ *
+ * NOTE: dictMode and searchMethod should be templated, so those switch
+ * statements should be optimized out. Only the mls & rowLog switches
+ * should be left.
+ *
+ * @param ms The match state.
+ * @param ip The position to search at.
+ * @param iend The end of the input data.
+ * @param[out] offsetPtr Stores the match offset into this pointer.
+ * @param mls The minimum search length, in the range [4, 6].
+ * @param rowLog The row log (if applicable), in the range [4, 6].
+ * @param searchMethod The search method to use (templated).
+ * @param dictMode The dictMode (templated).
+ *
+ * @returns The length of the longest match found, or < mls if no match is found.
+ * If a match is found its offset is stored in @p offsetPtr.
+ */
+FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
+ ZSTD_matchState_t* ms,
+ const BYTE* ip,
+ const BYTE* iend,
+ size_t* offsetPtr,
+ U32 const mls,
+ U32 const rowLog,
+ searchMethod_e const searchMethod,
+ ZSTD_dictMode_e const dictMode)
+{
+ if (dictMode == ZSTD_noDict) {
+ ZSTD_SWITCH_SEARCH_METHOD(noDict)
+ } else if (dictMode == ZSTD_extDict) {
+ ZSTD_SWITCH_SEARCH_METHOD(extDict)
+ } else if (dictMode == ZSTD_dictMatchState) {
+ ZSTD_SWITCH_SEARCH_METHOD(dictMatchState)
+ } else if (dictMode == ZSTD_dedicatedDictSearch) {
+ ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch)
+ }
+ ZSTD_UNREACHABLE;
+ return 0;
+}
+
+/* *******************************
+* Common parser - lazy strategy
+*********************************/
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_compressBlock_lazy_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize,
+ const searchMethod_e searchMethod, const U32 depth,
+ ZSTD_dictMode_e const dictMode)
+{
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
+ const BYTE* const base = ms->window.base;
+ const U32 prefixLowestIndex = ms->window.dictLimit;
+ const BYTE* const prefixLowest = base + prefixLowestIndex;
+ const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
+ const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
+
+ U32 offset_1 = rep[0], offset_2 = rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ const int isDMS = dictMode == ZSTD_dictMatchState;
+ const int isDDS = dictMode == ZSTD_dedicatedDictSearch;
+ const int isDxS = isDMS || isDDS;
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0;
+ const BYTE* const dictBase = isDxS ? dms->window.base : NULL;
+ const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL;
+ const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL;
+ const U32 dictIndexDelta = isDxS ?
+ prefixLowestIndex - (U32)(dictEnd - dictBase) :
+ 0;
+ const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod);
+ ip += (dictAndPrefixLength == 0);
+ if (dictMode == ZSTD_noDict) {
+ U32 const curr = (U32)(ip - base);
+ U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog);
+ U32 const maxRep = curr - windowLow;
+ if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+ if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
+ }
+ if (isDxS) {
+ /* dictMatchState repCode checks don't currently handle repCode == 0
+ * disabling. */
+ assert(offset_1 <= dictAndPrefixLength);
+ assert(offset_2 <= dictAndPrefixLength);
+ }
+
+ if (searchMethod == search_rowHash) {
+ ZSTD_row_fillHashCache(ms, base, rowLog,
+ MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
+ ms->nextToUpdate, ilimit);
+ }
+
+ /* Match Loop */
+#if defined(__GNUC__) && defined(__x86_64__)
+ /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the
+ * code alignment is perturbed. To fix the instability align the loop on 32-bytes.
+ */
+ __asm__(".p2align 5");
+#endif
+ while (ip < ilimit) {
+ size_t matchLength=0;
+ size_t offBase = REPCODE1_TO_OFFBASE;
+ const BYTE* start=ip+1;
+ DEBUGLOG(7, "search baseline (depth 0)");
+
+ /* check repCode */
+ if (isDxS) {
+ const U32 repIndex = (U32)(ip - base) + 1 - offset_1;
+ const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch)
+ && repIndex < prefixLowestIndex) ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ if (depth==0) goto _storeSequence;
+ }
+ }
+ if ( dictMode == ZSTD_noDict
+ && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
+ matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
+ if (depth==0) goto _storeSequence;
+ }
+
+ /* first search (depth 0) */
+ { size_t offbaseFound = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode);
+ if (ml2 > matchLength)
+ matchLength = ml2, start = ip, offBase = offbaseFound;
+ }
+
+ if (matchLength < 4) {
+ ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */
+ continue;
+ }
+
+ /* let's try to find a better solution */
+ if (depth>=1)
+ while (ip<ilimit) {
+ DEBUGLOG(7, "search depth 1");
+ ip ++;
+ if ( (dictMode == ZSTD_noDict)
+ && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+ size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
+ int const gain2 = (int)(mlRep * 3);
+ int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ if (isDxS) {
+ const U32 repIndex = (U32)(ip - base) - offset_1;
+ const BYTE* repMatch = repIndex < prefixLowestIndex ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ int const gain2 = (int)(mlRep * 3);
+ int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ }
+ { size_t ofbCandidate=999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue; /* search a better one */
+ } }
+
+ /* let's find an even better one */
+ if ((depth==2) && (ip<ilimit)) {
+ DEBUGLOG(7, "search depth 2");
+ ip ++;
+ if ( (dictMode == ZSTD_noDict)
+ && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+ size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
+ int const gain2 = (int)(mlRep * 4);
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ if (isDxS) {
+ const U32 repIndex = (U32)(ip - base) - offset_1;
+ const BYTE* repMatch = repIndex < prefixLowestIndex ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ int const gain2 = (int)(mlRep * 4);
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ }
+ { size_t ofbCandidate=999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue;
+ } } }
+ break; /* nothing found : store previous solution */
+ }
+
+ /* NOTE:
+ * Pay attention that `start[-value]` can lead to strange undefined behavior
+ * notably if `value` is unsigned, resulting in a large positive `-value`.
+ */
+ /* catch up */
+ if (OFFBASE_IS_OFFSET(offBase)) {
+ if (dictMode == ZSTD_noDict) {
+ while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest))
+ && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */
+ { start--; matchLength++; }
+ }
+ if (isDxS) {
+ U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
+ const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex;
+ const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest;
+ while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
+ }
+ offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
+ }
+ /* store sequence */
+_storeSequence:
+ { size_t const litLength = (size_t)(start - anchor);
+ ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
+ anchor = ip = start + matchLength;
+ }
+
+ /* check immediate repcode */
+ if (isDxS) {
+ while (ip <= ilimit) {
+ U32 const current2 = (U32)(ip-base);
+ U32 const repIndex = current2 - offset_2;
+ const BYTE* repMatch = repIndex < prefixLowestIndex ?
+ dictBase - dictIndexDelta + repIndex :
+ base + repIndex;
+ if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
+ const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
+ offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
+ ip += matchLength;
+ anchor = ip;
+ continue;
+ }
+ break;
+ }
+ }
+
+ if (dictMode == ZSTD_noDict) {
+ while ( ((ip <= ilimit) & (offset_2>0))
+ && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
+ /* store sequence */
+ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
+ offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
+ ip += matchLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ } } }
+
+ /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+ * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+ offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = offset_1 ? offset_1 : offsetSaved1;
+ rep[1] = offset_2 ? offset_2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+
+size_t ZSTD_compressBlock_btlazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_greedy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_btlazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_lazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_lazy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_greedy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState);
+}
+
+
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch);
+}
+
+/* Row-based matchfinder */
+size_t ZSTD_compressBlock_lazy2_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_greedy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_lazy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_greedy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState);
+}
+
+
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch);
+}
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_lazy_extDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize,
+ const searchMethod_e searchMethod, const U32 depth)
+{
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
+ const BYTE* const base = ms->window.base;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const dictStart = dictBase + ms->window.lowLimit;
+ const U32 windowLog = ms->cParams.windowLog;
+ const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
+ const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
+
+ U32 offset_1 = rep[0], offset_2 = rep[1];
+
+ DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod);
+
+ /* init */
+ ip += (ip == prefixStart);
+ if (searchMethod == search_rowHash) {
+ ZSTD_row_fillHashCache(ms, base, rowLog,
+ MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
+ ms->nextToUpdate, ilimit);
+ }
+
+ /* Match Loop */
+#if defined(__GNUC__) && defined(__x86_64__)
+ /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the
+ * code alignment is perturbed. To fix the instability align the loop on 32-bytes.
+ */
+ __asm__(".p2align 5");
+#endif
+ while (ip < ilimit) {
+ size_t matchLength=0;
+ size_t offBase = REPCODE1_TO_OFFBASE;
+ const BYTE* start=ip+1;
+ U32 curr = (U32)(ip-base);
+
+ /* check repCode */
+ { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog);
+ const U32 repIndex = (U32)(curr+1 - offset_1);
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
+ & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */
+ if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
+ /* repcode detected we should take it */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ if (depth==0) goto _storeSequence;
+ } }
+
+ /* first search (depth 0) */
+ { size_t ofbCandidate = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+ if (ml2 > matchLength)
+ matchLength = ml2, start = ip, offBase = ofbCandidate;
+ }
+
+ if (matchLength < 4) {
+ ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */
+ continue;
+ }
+
+ /* let's try to find a better solution */
+ if (depth>=1)
+ while (ip<ilimit) {
+ ip ++;
+ curr++;
+ /* check repCode */
+ if (offBase) {
+ const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
+ const U32 repIndex = (U32)(curr - offset_1);
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
+ /* repcode detected */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ int const gain2 = (int)(repLength * 3);
+ int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((repLength >= 4) && (gain2 > gain1))
+ matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ } }
+
+ /* search match, depth 1 */
+ { size_t ofbCandidate = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue; /* search a better one */
+ } }
+
+ /* let's find an even better one */
+ if ((depth==2) && (ip<ilimit)) {
+ ip ++;
+ curr++;
+ /* check repCode */
+ if (offBase) {
+ const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
+ const U32 repIndex = (U32)(curr - offset_1);
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
+ /* repcode detected */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ int const gain2 = (int)(repLength * 4);
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((repLength >= 4) && (gain2 > gain1))
+ matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ } }
+
+ /* search match, depth 2 */
+ { size_t ofbCandidate = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue;
+ } } }
+ break; /* nothing found : store previous solution */
+ }
+
+ /* catch up */
+ if (OFFBASE_IS_OFFSET(offBase)) {
+ U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
+ const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
+ const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
+ while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
+ offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
+ }
+
+ /* store sequence */
+_storeSequence:
+ { size_t const litLength = (size_t)(start - anchor);
+ ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
+ anchor = ip = start + matchLength;
+ }
+
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ const U32 repCurrent = (U32)(ip-base);
+ const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog);
+ const U32 repIndex = repCurrent - offset_2;
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
+ /* repcode detected we should take it */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
+ ip += matchLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ break;
+ } }
+
+ /* Save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+
+size_t ZSTD_compressBlock_greedy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0);
+}
+
+size_t ZSTD_compressBlock_lazy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1);
+}
+
+size_t ZSTD_compressBlock_lazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2);
+}
+
+size_t ZSTD_compressBlock_btlazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2);
+}
+
+size_t ZSTD_compressBlock_greedy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0);
+}
+
+size_t ZSTD_compressBlock_lazy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1);
+}
+
+size_t ZSTD_compressBlock_lazy2_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
+}
diff --git a/contrib/zstd/zstd_lazy.h b/contrib/zstd/zstd_lazy.h
new file mode 100644
index 0000000..3bde673
--- /dev/null
+++ b/contrib/zstd/zstd_lazy.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_LAZY_H
+#define ZSTD_LAZY_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "zstd_compress_internal.h"
+
+/**
+ * Dedicated Dictionary Search Structure bucket log. In the
+ * ZSTD_dedicatedDictSearch mode, the hashTable has
+ * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just
+ * one.
+ */
+#define ZSTD_LAZY_DDSS_BUCKET_LOG 2
+
+#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */
+
+U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
+void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
+
+void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
+
+void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
+
+size_t ZSTD_compressBlock_btlazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_btlazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_greedy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btlazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_LAZY_H */
diff --git a/contrib/zstd/zstd_ldm.c b/contrib/zstd/zstd_ldm.c
new file mode 100644
index 0000000..b8dcbca
--- /dev/null
+++ b/contrib/zstd/zstd_ldm.c
@@ -0,0 +1,724 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_ldm.h"
+
+#include "debug.h"
+#include "xxhash.h"
+#include "zstd_fast.h" /* ZSTD_fillHashTable() */
+#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
+#include "zstd_ldm_geartab.h"
+
+#define LDM_BUCKET_SIZE_LOG 3
+#define LDM_MIN_MATCH_LENGTH 64
+#define LDM_HASH_RLOG 7
+
+typedef struct {
+ U64 rolling;
+ U64 stopMask;
+} ldmRollingHashState_t;
+
+/** ZSTD_ldm_gear_init():
+ *
+ * Initializes the rolling hash state such that it will honor the
+ * settings in params. */
+static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params)
+{
+ unsigned maxBitsInMask = MIN(params->minMatchLength, 64);
+ unsigned hashRateLog = params->hashRateLog;
+
+ state->rolling = ~(U32)0;
+
+ /* The choice of the splitting criterion is subject to two conditions:
+ * 1. it has to trigger on average every 2^(hashRateLog) bytes;
+ * 2. ideally, it has to depend on a window of minMatchLength bytes.
+ *
+ * In the gear hash algorithm, bit n depends on the last n bytes;
+ * so in order to obtain a good quality splitting criterion it is
+ * preferable to use bits with high weight.
+ *
+ * To match condition 1 we use a mask with hashRateLog bits set
+ * and, because of the previous remark, we make sure these bits
+ * have the highest possible weight while still respecting
+ * condition 2.
+ */
+ if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) {
+ state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog);
+ } else {
+ /* In this degenerate case we simply honor the hash rate. */
+ state->stopMask = ((U64)1 << hashRateLog) - 1;
+ }
+}
+
+/** ZSTD_ldm_gear_reset()
+ * Feeds [data, data + minMatchLength) into the hash without registering any
+ * splits. This effectively resets the hash state. This is used when skipping
+ * over data, either at the beginning of a block, or skipping sections.
+ */
+static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state,
+ BYTE const* data, size_t minMatchLength)
+{
+ U64 hash = state->rolling;
+ size_t n = 0;
+
+#define GEAR_ITER_ONCE() do { \
+ hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
+ n += 1; \
+ } while (0)
+ while (n + 3 < minMatchLength) {
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ }
+ while (n < minMatchLength) {
+ GEAR_ITER_ONCE();
+ }
+#undef GEAR_ITER_ONCE
+}
+
+/** ZSTD_ldm_gear_feed():
+ *
+ * Registers in the splits array all the split points found in the first
+ * size bytes following the data pointer. This function terminates when
+ * either all the data has been processed or LDM_BATCH_SIZE splits are
+ * present in the splits array.
+ *
+ * Precondition: The splits array must not be full.
+ * Returns: The number of bytes processed. */
+static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state,
+ BYTE const* data, size_t size,
+ size_t* splits, unsigned* numSplits)
+{
+ size_t n;
+ U64 hash, mask;
+
+ hash = state->rolling;
+ mask = state->stopMask;
+ n = 0;
+
+#define GEAR_ITER_ONCE() do { \
+ hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
+ n += 1; \
+ if (UNLIKELY((hash & mask) == 0)) { \
+ splits[*numSplits] = n; \
+ *numSplits += 1; \
+ if (*numSplits == LDM_BATCH_SIZE) \
+ goto done; \
+ } \
+ } while (0)
+
+ while (n + 3 < size) {
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ }
+ while (n < size) {
+ GEAR_ITER_ONCE();
+ }
+
+#undef GEAR_ITER_ONCE
+
+done:
+ state->rolling = hash;
+ return n;
+}
+
+void ZSTD_ldm_adjustParameters(ldmParams_t* params,
+ ZSTD_compressionParameters const* cParams)
+{
+ params->windowLog = cParams->windowLog;
+ ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
+ DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
+ if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
+ if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
+ if (params->hashLog == 0) {
+ params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
+ assert(params->hashLog <= ZSTD_HASHLOG_MAX);
+ }
+ if (params->hashRateLog == 0) {
+ params->hashRateLog = params->windowLog < params->hashLog
+ ? 0
+ : params->windowLog - params->hashLog;
+ }
+ params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog);
+}
+
+size_t ZSTD_ldm_getTableSize(ldmParams_t params)
+{
+ size_t const ldmHSize = ((size_t)1) << params.hashLog;
+ size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog);
+ size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
+ size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize)
+ + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t));
+ return params.enableLdm == ZSTD_ps_enable ? totalSize : 0;
+}
+
+size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
+{
+ return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0;
+}
+
+/** ZSTD_ldm_getBucket() :
+ * Returns a pointer to the start of the bucket associated with hash. */
+static ldmEntry_t* ZSTD_ldm_getBucket(
+ ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams)
+{
+ return ldmState->hashTable + (hash << ldmParams.bucketSizeLog);
+}
+
+/** ZSTD_ldm_insertEntry() :
+ * Insert the entry with corresponding hash into the hash table */
+static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
+ size_t const hash, const ldmEntry_t entry,
+ ldmParams_t const ldmParams)
+{
+ BYTE* const pOffset = ldmState->bucketOffsets + hash;
+ unsigned const offset = *pOffset;
+
+ *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
+ *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
+
+}
+
+/** ZSTD_ldm_countBackwardsMatch() :
+ * Returns the number of bytes that match backwards before pIn and pMatch.
+ *
+ * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */
+static size_t ZSTD_ldm_countBackwardsMatch(
+ const BYTE* pIn, const BYTE* pAnchor,
+ const BYTE* pMatch, const BYTE* pMatchBase)
+{
+ size_t matchLength = 0;
+ while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) {
+ pIn--;
+ pMatch--;
+ matchLength++;
+ }
+ return matchLength;
+}
+
+/** ZSTD_ldm_countBackwardsMatch_2segments() :
+ * Returns the number of bytes that match backwards from pMatch,
+ * even with the backwards match spanning 2 different segments.
+ *
+ * On reaching `pMatchBase`, start counting from mEnd */
+static size_t ZSTD_ldm_countBackwardsMatch_2segments(
+ const BYTE* pIn, const BYTE* pAnchor,
+ const BYTE* pMatch, const BYTE* pMatchBase,
+ const BYTE* pExtDictStart, const BYTE* pExtDictEnd)
+{
+ size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase);
+ if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) {
+ /* If backwards match is entirely in the extDict or prefix, immediately return */
+ return matchLength;
+ }
+ DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength);
+ matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart);
+ DEBUGLOG(7, "final backwards match length = %zu", matchLength);
+ return matchLength;
+}
+
+/** ZSTD_ldm_fillFastTables() :
+ *
+ * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies.
+ * This is similar to ZSTD_loadDictionaryContent.
+ *
+ * The tables for the other strategies are filled within their
+ * block compressors. */
+static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
+ void const* end)
+{
+ const BYTE* const iend = (const BYTE*)end;
+
+ switch(ms->cParams.strategy)
+ {
+ case ZSTD_fast:
+ ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
+ break;
+
+ case ZSTD_dfast:
+ ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
+ break;
+
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ case ZSTD_btlazy2:
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ break;
+ default:
+ assert(0); /* not possible : not a valid strategy id */
+ }
+
+ return 0;
+}
+
+void ZSTD_ldm_fillHashTable(
+ ldmState_t* ldmState, const BYTE* ip,
+ const BYTE* iend, ldmParams_t const* params)
+{
+ U32 const minMatchLength = params->minMatchLength;
+ U32 const hBits = params->hashLog - params->bucketSizeLog;
+ BYTE const* const base = ldmState->window.base;
+ BYTE const* const istart = ip;
+ ldmRollingHashState_t hashState;
+ size_t* const splits = ldmState->splitIndices;
+ unsigned numSplits;
+
+ DEBUGLOG(5, "ZSTD_ldm_fillHashTable");
+
+ ZSTD_ldm_gear_init(&hashState, params);
+ while (ip < iend) {
+ size_t hashed;
+ unsigned n;
+
+ numSplits = 0;
+ hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
+
+ for (n = 0; n < numSplits; n++) {
+ if (ip + splits[n] >= istart + minMatchLength) {
+ BYTE const* const split = ip + splits[n] - minMatchLength;
+ U64 const xxhash = XXH64(split, minMatchLength, 0);
+ U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
+ ldmEntry_t entry;
+
+ entry.offset = (U32)(split - base);
+ entry.checksum = (U32)(xxhash >> 32);
+ ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
+ }
+ }
+
+ ip += hashed;
+ }
+}
+
+
+/** ZSTD_ldm_limitTableUpdate() :
+ *
+ * Sets cctx->nextToUpdate to a position corresponding closer to anchor
+ * if it is far way
+ * (after a long match, only update tables a limited amount). */
+static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
+{
+ U32 const curr = (U32)(anchor - ms->window.base);
+ if (curr > ms->nextToUpdate + 1024) {
+ ms->nextToUpdate =
+ curr - MIN(512, curr - ms->nextToUpdate - 1024);
+ }
+}
+
+static size_t ZSTD_ldm_generateSequences_internal(
+ ldmState_t* ldmState, rawSeqStore_t* rawSeqStore,
+ ldmParams_t const* params, void const* src, size_t srcSize)
+{
+ /* LDM parameters */
+ int const extDict = ZSTD_window_hasExtDict(ldmState->window);
+ U32 const minMatchLength = params->minMatchLength;
+ U32 const entsPerBucket = 1U << params->bucketSizeLog;
+ U32 const hBits = params->hashLog - params->bucketSizeLog;
+ /* Prefix and extDict parameters */
+ U32 const dictLimit = ldmState->window.dictLimit;
+ U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
+ BYTE const* const base = ldmState->window.base;
+ BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL;
+ BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL;
+ BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL;
+ BYTE const* const lowPrefixPtr = base + dictLimit;
+ /* Input bounds */
+ BYTE const* const istart = (BYTE const*)src;
+ BYTE const* const iend = istart + srcSize;
+ BYTE const* const ilimit = iend - HASH_READ_SIZE;
+ /* Input positions */
+ BYTE const* anchor = istart;
+ BYTE const* ip = istart;
+ /* Rolling hash state */
+ ldmRollingHashState_t hashState;
+ /* Arrays for staged-processing */
+ size_t* const splits = ldmState->splitIndices;
+ ldmMatchCandidate_t* const candidates = ldmState->matchCandidates;
+ unsigned numSplits;
+
+ if (srcSize < minMatchLength)
+ return iend - anchor;
+
+ /* Initialize the rolling hash state with the first minMatchLength bytes */
+ ZSTD_ldm_gear_init(&hashState, params);
+ ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength);
+ ip += minMatchLength;
+
+ while (ip < ilimit) {
+ size_t hashed;
+ unsigned n;
+
+ numSplits = 0;
+ hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip,
+ splits, &numSplits);
+
+ for (n = 0; n < numSplits; n++) {
+ BYTE const* const split = ip + splits[n] - minMatchLength;
+ U64 const xxhash = XXH64(split, minMatchLength, 0);
+ U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
+
+ candidates[n].split = split;
+ candidates[n].hash = hash;
+ candidates[n].checksum = (U32)(xxhash >> 32);
+ candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
+ PREFETCH_L1(candidates[n].bucket);
+ }
+
+ for (n = 0; n < numSplits; n++) {
+ size_t forwardMatchLength = 0, backwardMatchLength = 0,
+ bestMatchLength = 0, mLength;
+ U32 offset;
+ BYTE const* const split = candidates[n].split;
+ U32 const checksum = candidates[n].checksum;
+ U32 const hash = candidates[n].hash;
+ ldmEntry_t* const bucket = candidates[n].bucket;
+ ldmEntry_t const* cur;
+ ldmEntry_t const* bestEntry = NULL;
+ ldmEntry_t newEntry;
+
+ newEntry.offset = (U32)(split - base);
+ newEntry.checksum = checksum;
+
+ /* If a split point would generate a sequence overlapping with
+ * the previous one, we merely register it in the hash table and
+ * move on */
+ if (split < anchor) {
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+ continue;
+ }
+
+ for (cur = bucket; cur < bucket + entsPerBucket; cur++) {
+ size_t curForwardMatchLength, curBackwardMatchLength,
+ curTotalMatchLength;
+ if (cur->checksum != checksum || cur->offset <= lowestIndex) {
+ continue;
+ }
+ if (extDict) {
+ BYTE const* const curMatchBase =
+ cur->offset < dictLimit ? dictBase : base;
+ BYTE const* const pMatch = curMatchBase + cur->offset;
+ BYTE const* const matchEnd =
+ cur->offset < dictLimit ? dictEnd : iend;
+ BYTE const* const lowMatchPtr =
+ cur->offset < dictLimit ? dictStart : lowPrefixPtr;
+ curForwardMatchLength =
+ ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr);
+ if (curForwardMatchLength < minMatchLength) {
+ continue;
+ }
+ curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments(
+ split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd);
+ } else { /* !extDict */
+ BYTE const* const pMatch = base + cur->offset;
+ curForwardMatchLength = ZSTD_count(split, pMatch, iend);
+ if (curForwardMatchLength < minMatchLength) {
+ continue;
+ }
+ curBackwardMatchLength =
+ ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr);
+ }
+ curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength;
+
+ if (curTotalMatchLength > bestMatchLength) {
+ bestMatchLength = curTotalMatchLength;
+ forwardMatchLength = curForwardMatchLength;
+ backwardMatchLength = curBackwardMatchLength;
+ bestEntry = cur;
+ }
+ }
+
+ /* No match found -- insert an entry into the hash table
+ * and process the next candidate match */
+ if (bestEntry == NULL) {
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+ continue;
+ }
+
+ /* Match found */
+ offset = (U32)(split - base) - bestEntry->offset;
+ mLength = forwardMatchLength + backwardMatchLength;
+ {
+ rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
+
+ /* Out of sequence storage */
+ if (rawSeqStore->size == rawSeqStore->capacity)
+ return ERROR(dstSize_tooSmall);
+ seq->litLength = (U32)(split - backwardMatchLength - anchor);
+ seq->matchLength = (U32)mLength;
+ seq->offset = offset;
+ rawSeqStore->size++;
+ }
+
+ /* Insert the current entry into the hash table --- it must be
+ * done after the previous block to avoid clobbering bestEntry */
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+
+ anchor = split + forwardMatchLength;
+
+ /* If we find a match that ends after the data that we've hashed
+ * then we have a repeating, overlapping, pattern. E.g. all zeros.
+ * If one repetition of the pattern matches our `stopMask` then all
+ * repetitions will. We don't need to insert them all into out table,
+ * only the first one. So skip over overlapping matches.
+ * This is a major speed boost (20x) for compressing a single byte
+ * repeated, when that byte ends up in the table.
+ */
+ if (anchor > ip + hashed) {
+ ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength);
+ /* Continue the outer loop at anchor (ip + hashed == anchor). */
+ ip = anchor - hashed;
+ break;
+ }
+ }
+
+ ip += hashed;
+ }
+
+ return iend - anchor;
+}
+
+/*! ZSTD_ldm_reduceTable() :
+ * reduce table indexes by `reducerValue` */
+static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
+ U32 const reducerValue)
+{
+ U32 u;
+ for (u = 0; u < size; u++) {
+ if (table[u].offset < reducerValue) table[u].offset = 0;
+ else table[u].offset -= reducerValue;
+ }
+}
+
+size_t ZSTD_ldm_generateSequences(
+ ldmState_t* ldmState, rawSeqStore_t* sequences,
+ ldmParams_t const* params, void const* src, size_t srcSize)
+{
+ U32 const maxDist = 1U << params->windowLog;
+ BYTE const* const istart = (BYTE const*)src;
+ BYTE const* const iend = istart + srcSize;
+ size_t const kMaxChunkSize = 1 << 20;
+ size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0);
+ size_t chunk;
+ size_t leftoverSize = 0;
+
+ assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize);
+ /* Check that ZSTD_window_update() has been called for this chunk prior
+ * to passing it to this function.
+ */
+ assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize);
+ /* The input could be very large (in zstdmt), so it must be broken up into
+ * chunks to enforce the maximum distance and handle overflow correction.
+ */
+ assert(sequences->pos <= sequences->size);
+ assert(sequences->size <= sequences->capacity);
+ for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) {
+ BYTE const* const chunkStart = istart + chunk * kMaxChunkSize;
+ size_t const remaining = (size_t)(iend - chunkStart);
+ BYTE const *const chunkEnd =
+ (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize;
+ size_t const chunkSize = chunkEnd - chunkStart;
+ size_t newLeftoverSize;
+ size_t const prevSize = sequences->size;
+
+ assert(chunkStart < iend);
+ /* 1. Perform overflow correction if necessary. */
+ if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) {
+ U32 const ldmHSize = 1U << params->hashLog;
+ U32 const correction = ZSTD_window_correctOverflow(
+ &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart);
+ ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction);
+ /* invalidate dictionaries on overflow correction */
+ ldmState->loadedDictEnd = 0;
+ }
+ /* 2. We enforce the maximum offset allowed.
+ *
+ * kMaxChunkSize should be small enough that we don't lose too much of
+ * the window through early invalidation.
+ * TODO: * Test the chunk size.
+ * * Try invalidation after the sequence generation and test the
+ * offset against maxDist directly.
+ *
+ * NOTE: Because of dictionaries + sequence splitting we MUST make sure
+ * that any offset used is valid at the END of the sequence, since it may
+ * be split into two sequences. This condition holds when using
+ * ZSTD_window_enforceMaxDist(), but if we move to checking offsets
+ * against maxDist directly, we'll have to carefully handle that case.
+ */
+ ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL);
+ /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */
+ newLeftoverSize = ZSTD_ldm_generateSequences_internal(
+ ldmState, sequences, params, chunkStart, chunkSize);
+ if (ZSTD_isError(newLeftoverSize))
+ return newLeftoverSize;
+ /* 4. We add the leftover literals from previous iterations to the first
+ * newly generated sequence, or add the `newLeftoverSize` if none are
+ * generated.
+ */
+ /* Prepend the leftover literals from the last call */
+ if (prevSize < sequences->size) {
+ sequences->seq[prevSize].litLength += (U32)leftoverSize;
+ leftoverSize = newLeftoverSize;
+ } else {
+ assert(newLeftoverSize == chunkSize);
+ leftoverSize += chunkSize;
+ }
+ }
+ return 0;
+}
+
+void
+ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
+{
+ while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
+ rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
+ if (srcSize <= seq->litLength) {
+ /* Skip past srcSize literals */
+ seq->litLength -= (U32)srcSize;
+ return;
+ }
+ srcSize -= seq->litLength;
+ seq->litLength = 0;
+ if (srcSize < seq->matchLength) {
+ /* Skip past the first srcSize of the match */
+ seq->matchLength -= (U32)srcSize;
+ if (seq->matchLength < minMatch) {
+ /* The match is too short, omit it */
+ if (rawSeqStore->pos + 1 < rawSeqStore->size) {
+ seq[1].litLength += seq[0].matchLength;
+ }
+ rawSeqStore->pos++;
+ }
+ return;
+ }
+ srcSize -= seq->matchLength;
+ seq->matchLength = 0;
+ rawSeqStore->pos++;
+ }
+}
+
+/**
+ * If the sequence length is longer than remaining then the sequence is split
+ * between this block and the next.
+ *
+ * Returns the current sequence to handle, or if the rest of the block should
+ * be literals, it returns a sequence with offset == 0.
+ */
+static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
+ U32 const remaining, U32 const minMatch)
+{
+ rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos];
+ assert(sequence.offset > 0);
+ /* Likely: No partial sequence */
+ if (remaining >= sequence.litLength + sequence.matchLength) {
+ rawSeqStore->pos++;
+ return sequence;
+ }
+ /* Cut the sequence short (offset == 0 ==> rest is literals). */
+ if (remaining <= sequence.litLength) {
+ sequence.offset = 0;
+ } else if (remaining < sequence.litLength + sequence.matchLength) {
+ sequence.matchLength = remaining - sequence.litLength;
+ if (sequence.matchLength < minMatch) {
+ sequence.offset = 0;
+ }
+ }
+ /* Skip past `remaining` bytes for the future sequences. */
+ ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch);
+ return sequence;
+}
+
+void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
+ U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
+ while (currPos && rawSeqStore->pos < rawSeqStore->size) {
+ rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
+ if (currPos >= currSeq.litLength + currSeq.matchLength) {
+ currPos -= currSeq.litLength + currSeq.matchLength;
+ rawSeqStore->pos++;
+ } else {
+ rawSeqStore->posInSequence = currPos;
+ break;
+ }
+ }
+ if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
+ rawSeqStore->posInSequence = 0;
+ }
+}
+
+size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ ZSTD_paramSwitch_e useRowMatchFinder,
+ void const* src, size_t srcSize)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ unsigned const minMatch = cParams->minMatch;
+ ZSTD_blockCompressor const blockCompressor =
+ ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
+ /* Input bounds */
+ BYTE const* const istart = (BYTE const*)src;
+ BYTE const* const iend = istart + srcSize;
+ /* Input positions */
+ BYTE const* ip = istart;
+
+ DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
+ /* If using opt parser, use LDMs only as candidates rather than always accepting them */
+ if (cParams->strategy >= ZSTD_btopt) {
+ size_t lastLLSize;
+ ms->ldmSeqStore = rawSeqStore;
+ lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize);
+ ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
+ return lastLLSize;
+ }
+
+ assert(rawSeqStore->pos <= rawSeqStore->size);
+ assert(rawSeqStore->size <= rawSeqStore->capacity);
+ /* Loop through each sequence and apply the block compressor to the literals */
+ while (rawSeqStore->pos < rawSeqStore->size && ip < iend) {
+ /* maybeSplitSequence updates rawSeqStore->pos */
+ rawSeq const sequence = maybeSplitSequence(rawSeqStore,
+ (U32)(iend - ip), minMatch);
+ int i;
+ /* End signal */
+ if (sequence.offset == 0)
+ break;
+
+ assert(ip + sequence.litLength + sequence.matchLength <= iend);
+
+ /* Fill tables for block compressor */
+ ZSTD_ldm_limitTableUpdate(ms, ip);
+ ZSTD_ldm_fillFastTables(ms, ip);
+ /* Run the block compressor */
+ DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength);
+ {
+ size_t const newLitLength =
+ blockCompressor(ms, seqStore, rep, ip, sequence.litLength);
+ ip += sequence.litLength;
+ /* Update the repcodes */
+ for (i = ZSTD_REP_NUM - 1; i > 0; i--)
+ rep[i] = rep[i-1];
+ rep[0] = sequence.offset;
+ /* Store the sequence */
+ ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
+ OFFSET_TO_OFFBASE(sequence.offset),
+ sequence.matchLength);
+ ip += sequence.matchLength;
+ }
+ }
+ /* Fill the tables for the block compressor */
+ ZSTD_ldm_limitTableUpdate(ms, ip);
+ ZSTD_ldm_fillFastTables(ms, ip);
+ /* Compress the last literals */
+ return blockCompressor(ms, seqStore, rep, ip, iend - ip);
+}
diff --git a/contrib/zstd/zstd_ldm.h b/contrib/zstd/zstd_ldm.h
new file mode 100644
index 0000000..38ef9fb
--- /dev/null
+++ b/contrib/zstd/zstd_ldm.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_LDM_H
+#define ZSTD_LDM_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "zstd_compress_internal.h" /* ldmParams_t, U32 */
+#include "zstd.h" /* ZSTD_CCtx, size_t */
+
+/*-*************************************
+* Long distance matching
+***************************************/
+
+#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
+
+void ZSTD_ldm_fillHashTable(
+ ldmState_t* state, const BYTE* ip,
+ const BYTE* iend, ldmParams_t const* params);
+
+/**
+ * ZSTD_ldm_generateSequences():
+ *
+ * Generates the sequences using the long distance match finder.
+ * Generates long range matching sequences in `sequences`, which parse a prefix
+ * of the source. `sequences` must be large enough to store every sequence,
+ * which can be checked with `ZSTD_ldm_getMaxNbSeq()`.
+ * @returns 0 or an error code.
+ *
+ * NOTE: The user must have called ZSTD_window_update() for all of the input
+ * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks.
+ * NOTE: This function returns an error if it runs out of space to store
+ * sequences.
+ */
+size_t ZSTD_ldm_generateSequences(
+ ldmState_t* ldms, rawSeqStore_t* sequences,
+ ldmParams_t const* params, void const* src, size_t srcSize);
+
+/**
+ * ZSTD_ldm_blockCompress():
+ *
+ * Compresses a block using the predefined sequences, along with a secondary
+ * block compressor. The literals section of every sequence is passed to the
+ * secondary block compressor, and those sequences are interspersed with the
+ * predefined sequences. Returns the length of the last literals.
+ * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed.
+ * `rawSeqStore.seq` may also be updated to split the last sequence between two
+ * blocks.
+ * @return The length of the last literals.
+ *
+ * NOTE: The source must be at most the maximum block size, but the predefined
+ * sequences can be any size, and may be longer than the block. In the case that
+ * they are longer than the block, the last sequences may need to be split into
+ * two. We handle that case correctly, and update `rawSeqStore` appropriately.
+ * NOTE: This function does not return any errors.
+ */
+size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ ZSTD_paramSwitch_e useRowMatchFinder,
+ void const* src, size_t srcSize);
+
+/**
+ * ZSTD_ldm_skipSequences():
+ *
+ * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`.
+ * Avoids emitting matches less than `minMatch` bytes.
+ * Must be called for data that is not passed to ZSTD_ldm_blockCompress().
+ */
+void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
+ U32 const minMatch);
+
+/* ZSTD_ldm_skipRawSeqStoreBytes():
+ * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'.
+ * Not to be used in conjunction with ZSTD_ldm_skipSequences().
+ * Must be called for data with is not passed to ZSTD_ldm_blockCompress().
+ */
+void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes);
+
+/** ZSTD_ldm_getTableSize() :
+ * Estimate the space needed for long distance matching tables or 0 if LDM is
+ * disabled.
+ */
+size_t ZSTD_ldm_getTableSize(ldmParams_t params);
+
+/** ZSTD_ldm_getSeqSpace() :
+ * Return an upper bound on the number of sequences that can be produced by
+ * the long distance matcher, or 0 if LDM is disabled.
+ */
+size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
+
+/** ZSTD_ldm_adjustParameters() :
+ * If the params->hashRateLog is not set, set it to its default value based on
+ * windowLog and params->hashLog.
+ *
+ * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to
+ * params->hashLog if it is not).
+ *
+ * Ensures that the minMatchLength >= targetLength during optimal parsing.
+ */
+void ZSTD_ldm_adjustParameters(ldmParams_t* params,
+ ZSTD_compressionParameters const* cParams);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_FAST_H */
diff --git a/contrib/zstd/zstd_ldm_geartab.h b/contrib/zstd/zstd_ldm_geartab.h
new file mode 100644
index 0000000..92455a0
--- /dev/null
+++ b/contrib/zstd/zstd_ldm_geartab.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_LDM_GEARTAB_H
+#define ZSTD_LDM_GEARTAB_H
+
+#include "compiler.h" /* UNUSED_ATTR */
+#include "mem.h" /* U64 */
+
+static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = {
+ 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc,
+ 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05,
+ 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e,
+ 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889,
+ 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e,
+ 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140,
+ 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e,
+ 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f,
+ 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391,
+ 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210,
+ 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be,
+ 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a,
+ 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b,
+ 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4,
+ 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb,
+ 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312,
+ 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01,
+ 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc,
+ 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967,
+ 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553,
+ 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f,
+ 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2,
+ 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d,
+ 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a,
+ 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74,
+ 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3,
+ 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1,
+ 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b,
+ 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568,
+ 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a,
+ 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1,
+ 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9,
+ 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463,
+ 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba,
+ 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9,
+ 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61,
+ 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec,
+ 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6,
+ 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab,
+ 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5,
+ 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59,
+ 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7,
+ 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc,
+ 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb,
+ 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be,
+ 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312,
+ 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1,
+ 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc,
+ 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d,
+ 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445,
+ 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c,
+ 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5,
+ 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5,
+ 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28,
+ 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a,
+ 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9,
+ 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15,
+ 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef,
+ 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2,
+ 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375,
+ 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3,
+ 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595,
+ 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389,
+ 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4,
+ 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756,
+ 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc,
+ 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45,
+ 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea,
+ 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f,
+ 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc,
+ 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c,
+ 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a,
+ 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17,
+ 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3,
+ 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4,
+ 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91,
+ 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40,
+ 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741,
+ 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f,
+ 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4,
+ 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad,
+ 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047,
+ 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2,
+ 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e,
+ 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b,
+ 0x2b4da14f2613d8f4
+};
+
+#endif /* ZSTD_LDM_GEARTAB_H */
diff --git a/contrib/zstd/zstd_opt.c b/contrib/zstd/zstd_opt.c
new file mode 100644
index 0000000..fdd7f9d
--- /dev/null
+++ b/contrib/zstd/zstd_opt.c
@@ -0,0 +1,1470 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "hist.h"
+#include "zstd_opt.h"
+
+
+#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
+#define ZSTD_MAX_PRICE (1<<30)
+
+#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
+
+
+/*-*************************************
+* Price functions for optimal parser
+***************************************/
+
+#if 0 /* approximation at bit level (for tests) */
+# define BITCOST_ACCURACY 0
+# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
+# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat))
+#elif 0 /* fractional bit accuracy (for tests) */
+# define BITCOST_ACCURACY 8
+# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
+# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat))
+#else /* opt==approx, ultra==accurate */
+# define BITCOST_ACCURACY 8
+# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
+# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
+#endif
+
+/* ZSTD_bitWeight() :
+ * provide estimated "cost" of a stat in full bits only */
+MEM_STATIC U32 ZSTD_bitWeight(U32 stat)
+{
+ return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER);
+}
+
+/* ZSTD_fracWeight() :
+ * provide fractional-bit "cost" of a stat,
+ * using linear interpolation approximation */
+MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
+{
+ U32 const stat = rawStat + 1;
+ U32 const hb = ZSTD_highbit32(stat);
+ U32 const BWeight = hb * BITCOST_MULTIPLIER;
+ /* Fweight was meant for "Fractional weight"
+ * but it's effectively a value between 1 and 2
+ * using fixed point arithmetic */
+ U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
+ U32 const weight = BWeight + FWeight;
+ assert(hb + BITCOST_ACCURACY < 31);
+ return weight;
+}
+
+#if (DEBUGLEVEL>=2)
+/* debugging function,
+ * @return price in bytes as fractional value
+ * for debug messages only */
+MEM_STATIC double ZSTD_fCost(int price)
+{
+ return (double)price / (BITCOST_MULTIPLIER*8);
+}
+#endif
+
+static int ZSTD_compressedLiterals(optState_t const* const optPtr)
+{
+ return optPtr->literalCompressionMode != ZSTD_ps_disable;
+}
+
+static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
+{
+ if (ZSTD_compressedLiterals(optPtr))
+ optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel);
+ optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel);
+ optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel);
+ optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel);
+}
+
+
+static U32 sum_u32(const unsigned table[], size_t nbElts)
+{
+ size_t n;
+ U32 total = 0;
+ for (n=0; n<nbElts; n++) {
+ total += table[n];
+ }
+ return total;
+}
+
+typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e;
+
+static U32
+ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1)
+{
+ U32 s, sum=0;
+ DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
+ (unsigned)lastEltIndex+1, (unsigned)shift );
+ assert(shift < 30);
+ for (s=0; s<lastEltIndex+1; s++) {
+ unsigned const base = base1 ? 1 : (table[s]>0);
+ unsigned const newStat = base + (table[s] >> shift);
+ sum += newStat;
+ table[s] = newStat;
+ }
+ return sum;
+}
+
+/* ZSTD_scaleStats() :
+ * reduce all elt frequencies in table if sum too large
+ * return the resulting sum of elements */
+static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
+{
+ U32 const prevsum = sum_u32(table, lastEltIndex+1);
+ U32 const factor = prevsum >> logTarget;
+ DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
+ assert(logTarget < 30);
+ if (factor <= 1) return prevsum;
+ return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed);
+}
+
+/* ZSTD_rescaleFreqs() :
+ * if first block (detected by optPtr->litLengthSum == 0) : init statistics
+ * take hints from dictionary if there is one
+ * and init from zero if there is none,
+ * using src for literals stats, and baseline stats for sequence symbols
+ * otherwise downscale existing stats, to be used as seed for next block.
+ */
+static void
+ZSTD_rescaleFreqs(optState_t* const optPtr,
+ const BYTE* const src, size_t const srcSize,
+ int const optLevel)
+{
+ int const compressedLiterals = ZSTD_compressedLiterals(optPtr);
+ DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
+ optPtr->priceType = zop_dynamic;
+
+ if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */
+
+ /* heuristic: use pre-defined stats for too small inputs */
+ if (srcSize <= ZSTD_PREDEF_THRESHOLD) {
+ DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD);
+ optPtr->priceType = zop_predef;
+ }
+
+ assert(optPtr->symbolCosts != NULL);
+ if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
+
+ /* huffman stats covering the full value set : table presumed generated by dictionary */
+ optPtr->priceType = zop_dynamic;
+
+ if (compressedLiterals) {
+ /* generate literals statistics from huffman table */
+ unsigned lit;
+ assert(optPtr->litFreq != NULL);
+ optPtr->litSum = 0;
+ for (lit=0; lit<=MaxLit; lit++) {
+ U32 const scaleLog = 11; /* scale to 2K */
+ U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit);
+ assert(bitCost <= scaleLog);
+ optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->litSum += optPtr->litFreq[lit];
+ } }
+
+ { unsigned ll;
+ FSE_CState_t llstate;
+ FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable);
+ optPtr->litLengthSum = 0;
+ for (ll=0; ll<=MaxLL; ll++) {
+ U32 const scaleLog = 10; /* scale to 1K */
+ U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll);
+ assert(bitCost < scaleLog);
+ optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->litLengthSum += optPtr->litLengthFreq[ll];
+ } }
+
+ { unsigned ml;
+ FSE_CState_t mlstate;
+ FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable);
+ optPtr->matchLengthSum = 0;
+ for (ml=0; ml<=MaxML; ml++) {
+ U32 const scaleLog = 10;
+ U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml);
+ assert(bitCost < scaleLog);
+ optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->matchLengthSum += optPtr->matchLengthFreq[ml];
+ } }
+
+ { unsigned of;
+ FSE_CState_t ofstate;
+ FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable);
+ optPtr->offCodeSum = 0;
+ for (of=0; of<=MaxOff; of++) {
+ U32 const scaleLog = 10;
+ U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of);
+ assert(bitCost < scaleLog);
+ optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->offCodeSum += optPtr->offCodeFreq[of];
+ } }
+
+ } else { /* first block, no dictionary */
+
+ assert(optPtr->litFreq != NULL);
+ if (compressedLiterals) {
+ /* base initial cost of literals on direct frequency within src */
+ unsigned lit = MaxLit;
+ HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
+ optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
+ }
+
+ { unsigned const baseLLfreqs[MaxLL+1] = {
+ 4, 2, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1
+ };
+ ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs));
+ optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
+ }
+
+ { unsigned ml;
+ for (ml=0; ml<=MaxML; ml++)
+ optPtr->matchLengthFreq[ml] = 1;
+ }
+ optPtr->matchLengthSum = MaxML+1;
+
+ { unsigned const baseOFCfreqs[MaxOff+1] = {
+ 6, 2, 1, 1, 2, 3, 4, 4,
+ 4, 3, 2, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1
+ };
+ ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs));
+ optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
+ }
+
+ }
+
+ } else { /* new block : scale down accumulated statistics */
+
+ if (compressedLiterals)
+ optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
+ optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11);
+ optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11);
+ optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11);
+ }
+
+ ZSTD_setBasePrices(optPtr, optLevel);
+}
+
+/* ZSTD_rawLiteralsCost() :
+ * price of literals (only) in specified segment (which length can be 0).
+ * does not include price of literalLength symbol */
+static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
+ const optState_t* const optPtr,
+ int optLevel)
+{
+ if (litLength == 0) return 0;
+
+ if (!ZSTD_compressedLiterals(optPtr))
+ return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */
+
+ if (optPtr->priceType == zop_predef)
+ return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */
+
+ /* dynamic statistics */
+ { U32 price = optPtr->litSumBasePrice * litLength;
+ U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER;
+ U32 u;
+ assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER);
+ for (u=0; u < litLength; u++) {
+ U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel);
+ if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax;
+ price -= litPrice;
+ }
+ return price;
+ }
+}
+
+/* ZSTD_litLengthPrice() :
+ * cost of literalLength symbol */
+static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel)
+{
+ assert(litLength <= ZSTD_BLOCKSIZE_MAX);
+ if (optPtr->priceType == zop_predef)
+ return WEIGHT(litLength, optLevel);
+
+ /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
+ * because it isn't representable in the zstd format.
+ * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1.
+ * In such a case, the block would be all literals.
+ */
+ if (litLength == ZSTD_BLOCKSIZE_MAX)
+ return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel);
+
+ /* dynamic statistics */
+ { U32 const llCode = ZSTD_LLcode(litLength);
+ return (LL_bits[llCode] * BITCOST_MULTIPLIER)
+ + optPtr->litLengthSumBasePrice
+ - WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
+ }
+}
+
+/* ZSTD_getMatchPrice() :
+ * Provides the cost of the match part (offset + matchLength) of a sequence.
+ * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
+ * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
+ * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
+ */
+FORCE_INLINE_TEMPLATE U32
+ZSTD_getMatchPrice(U32 const offBase,
+ U32 const matchLength,
+ const optState_t* const optPtr,
+ int const optLevel)
+{
+ U32 price;
+ U32 const offCode = ZSTD_highbit32(offBase);
+ U32 const mlBase = matchLength - MINMATCH;
+ assert(matchLength >= MINMATCH);
+
+ if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */
+ return WEIGHT(mlBase, optLevel)
+ + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */
+
+ /* dynamic statistics */
+ price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
+ if ((optLevel<2) /*static*/ && offCode >= 20)
+ price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */
+
+ /* match Length */
+ { U32 const mlCode = ZSTD_MLcode(mlBase);
+ price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel));
+ }
+
+ price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */
+
+ DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price);
+ return price;
+}
+
+/* ZSTD_updateStats() :
+ * assumption : literals + litLength <= iend */
+static void ZSTD_updateStats(optState_t* const optPtr,
+ U32 litLength, const BYTE* literals,
+ U32 offBase, U32 matchLength)
+{
+ /* literals */
+ if (ZSTD_compressedLiterals(optPtr)) {
+ U32 u;
+ for (u=0; u < litLength; u++)
+ optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
+ optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
+ }
+
+ /* literal Length */
+ { U32 const llCode = ZSTD_LLcode(litLength);
+ optPtr->litLengthFreq[llCode]++;
+ optPtr->litLengthSum++;
+ }
+
+ /* offset code : follows storeSeq() numeric representation */
+ { U32 const offCode = ZSTD_highbit32(offBase);
+ assert(offCode <= MaxOff);
+ optPtr->offCodeFreq[offCode]++;
+ optPtr->offCodeSum++;
+ }
+
+ /* match Length */
+ { U32 const mlBase = matchLength - MINMATCH;
+ U32 const mlCode = ZSTD_MLcode(mlBase);
+ optPtr->matchLengthFreq[mlCode]++;
+ optPtr->matchLengthSum++;
+ }
+}
+
+
+/* ZSTD_readMINMATCH() :
+ * function safe only for comparisons
+ * assumption : memPtr must be at least 4 bytes before end of buffer */
+MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
+{
+ switch (length)
+ {
+ default :
+ case 4 : return MEM_read32(memPtr);
+ case 3 : if (MEM_isLittleEndian())
+ return MEM_read32(memPtr)<<8;
+ else
+ return MEM_read32(memPtr)>>8;
+ }
+}
+
+
+/* Update hashTable3 up to ip (excluded)
+ Assumption : always within prefix (i.e. not within extDict) */
+static U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
+ const BYTE* const ip)
+{
+ U32* const hashTable3 = ms->hashTable3;
+ U32 const hashLog3 = ms->hashLog3;
+ const BYTE* const base = ms->window.base;
+ U32 idx = *nextToUpdate3;
+ U32 const target = (U32)(ip - base);
+ size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3);
+ assert(hashLog3 > 0);
+
+ while(idx < target) {
+ hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx;
+ idx++;
+ }
+
+ *nextToUpdate3 = target;
+ return hashTable3[hash3];
+}
+
+
+/*-*************************************
+* Binary Tree search
+***************************************/
+/** ZSTD_insertBt1() : add one or multiple positions to tree.
+ * @param ip assumed <= iend-8 .
+ * @param target The target of ZSTD_updateTree_internal() - we are filling to this position
+ * @return : nb of positions added */
+static U32 ZSTD_insertBt1(
+ const ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ U32 const target,
+ U32 const mls, const int extDict)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hashLog = cParams->hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 matchIndex = hashTable[h];
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* match;
+ const U32 curr = (U32)(ip-base);
+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = smallerPtr + 1;
+ U32 dummy32; /* to be nullified at the end */
+ /* windowLow is based on target because
+ * we only need positions that will be in the window at the end of the tree update.
+ */
+ U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog);
+ U32 matchEndIdx = curr+8+1;
+ size_t bestLength = 8;
+ U32 nbCompares = 1U << cParams->searchLog;
+#ifdef ZSTD_C_PREDICT
+ U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0);
+ U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1);
+ predictedSmall += (predictedSmall>0);
+ predictedLarge += (predictedLarge>0);
+#endif /* ZSTD_C_PREDICT */
+
+ DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr);
+
+ assert(curr <= target);
+ assert(ip <= iend-8); /* required for h calculation */
+ hashTable[h] = curr; /* Update Hash Table */
+
+ assert(windowLow > 0);
+ for (; nbCompares && (matchIndex >= windowLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ assert(matchIndex < curr);
+
+#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */
+ const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
+ if (matchIndex == predictedSmall) {
+ /* no need to check length, result known */
+ *smallerPtr = matchIndex;
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ predictedSmall = predictPtr[1] + (predictPtr[1]>0);
+ continue;
+ }
+ if (matchIndex == predictedLarge) {
+ *largerPtr = matchIndex;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ predictedLarge = predictPtr[0] + (predictPtr[0]>0);
+ continue;
+ }
+#endif
+
+ if (!extDict || (matchIndex+matchLength >= dictLimit)) {
+ assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */
+ match = base + matchIndex;
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
+ }
+
+ if (matchLength > bestLength) {
+ bestLength = matchLength;
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ }
+
+ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
+ }
+
+ if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */
+ /* match is smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */
+ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */
+ } else {
+ /* match is larger than current */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+ { U32 positions = 0;
+ if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */
+ assert(matchEndIdx > curr + 8);
+ return MAX(positions, matchEndIdx - (curr + 8));
+ }
+}
+
+FORCE_INLINE_TEMPLATE
+void ZSTD_updateTree_internal(
+ ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ const U32 mls, const ZSTD_dictMode_e dictMode)
+{
+ const BYTE* const base = ms->window.base;
+ U32 const target = (U32)(ip - base);
+ U32 idx = ms->nextToUpdate;
+ DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
+ idx, target, dictMode);
+
+ while(idx < target) {
+ U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict);
+ assert(idx < (U32)(idx + forward));
+ idx += forward;
+ }
+ assert((size_t)(ip - base) <= (size_t)(U32)(-1));
+ assert((size_t)(iend - base) <= (size_t)(U32)(-1));
+ ms->nextToUpdate = target;
+}
+
+void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
+ ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
+}
+
+FORCE_INLINE_TEMPLATE U32
+ZSTD_insertBtAndGetAllMatches (
+ ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */
+ ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
+ const BYTE* const ip, const BYTE* const iLimit,
+ const ZSTD_dictMode_e dictMode,
+ const U32 rep[ZSTD_REP_NUM],
+ const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
+ const U32 lengthToBeat,
+ const U32 mls /* template */)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
+ const BYTE* const base = ms->window.base;
+ U32 const curr = (U32)(ip-base);
+ U32 const hashLog = cParams->hashLog;
+ U32 const minMatch = (mls==3) ? 3 : 4;
+ U32* const hashTable = ms->hashTable;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 matchIndex = hashTable[h];
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask= (1U << btLog) - 1;
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+ const BYTE* const dictBase = ms->window.dictBase;
+ U32 const dictLimit = ms->window.dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ U32 const btLow = (btMask >= curr) ? 0 : curr - btMask;
+ U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog);
+ U32 const matchLow = windowLow ? windowLow : 1;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = bt + 2*(curr&btMask) + 1;
+ U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */
+ U32 dummy32; /* to be nullified at the end */
+ U32 mnum = 0;
+ U32 nbCompares = 1U << cParams->searchLog;
+
+ const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
+ const ZSTD_compressionParameters* const dmsCParams =
+ dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL;
+ const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL;
+ const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL;
+ U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0;
+ U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0;
+ U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0;
+ U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog;
+ U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog;
+ U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0;
+ U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit;
+
+ size_t bestLength = lengthToBeat-1;
+ DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr);
+
+ /* check repCode */
+ assert(ll0 <= 1); /* necessarily 1 or 0 */
+ { U32 const lastR = ZSTD_REP_NUM + ll0;
+ U32 repCode;
+ for (repCode = ll0; repCode < lastR; repCode++) {
+ U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
+ U32 const repIndex = curr - repOffset;
+ U32 repLen = 0;
+ assert(curr >= dictLimit);
+ if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */
+ /* We must validate the repcode offset because when we're using a dictionary the
+ * valid offset range shrinks when the dictionary goes out of bounds.
+ */
+ if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) {
+ repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch;
+ }
+ } else { /* repIndex < dictLimit || repIndex >= curr */
+ const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ?
+ dmsBase + repIndex - dmsIndexDelta :
+ dictBase + repIndex;
+ assert(curr >= windowLow);
+ if ( dictMode == ZSTD_extDict
+ && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */
+ & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)
+ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
+ repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
+ }
+ if (dictMode == ZSTD_dictMatchState
+ && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */
+ & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
+ repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch;
+ } }
+ /* save longer solution */
+ if (repLen > bestLength) {
+ DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u",
+ repCode, ll0, repOffset, repLen);
+ bestLength = repLen;
+ matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */
+ matches[mnum].len = (U32)repLen;
+ mnum++;
+ if ( (repLen > sufficient_len)
+ | (ip+repLen == iLimit) ) { /* best possible */
+ return mnum;
+ } } } }
+
+ /* HC3 match finder */
+ if ((mls == 3) /*static*/ && (bestLength < mls)) {
+ U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip);
+ if ((matchIndex3 >= matchLow)
+ & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) {
+ size_t mlen;
+ if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) {
+ const BYTE* const match = base + matchIndex3;
+ mlen = ZSTD_count(ip, match, iLimit);
+ } else {
+ const BYTE* const match = dictBase + matchIndex3;
+ mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart);
+ }
+
+ /* save best solution */
+ if (mlen >= mls /* == 3 > bestLength */) {
+ DEBUGLOG(8, "found small match with hlog3, of length %u",
+ (U32)mlen);
+ bestLength = mlen;
+ assert(curr > matchIndex3);
+ assert(mnum==0); /* no prior solution */
+ matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3);
+ matches[0].len = (U32)mlen;
+ mnum = 1;
+ if ( (mlen > sufficient_len) |
+ (ip+mlen == iLimit) ) { /* best possible length */
+ ms->nextToUpdate = curr+1; /* skip insertion */
+ return 1;
+ } } }
+ /* no dictMatchState lookup: dicts don't have a populated HC3 table */
+ } /* if (mls == 3) */
+
+ hashTable[h] = curr; /* Update Hash Table */
+
+ for (; nbCompares && (matchIndex >= matchLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ const BYTE* match;
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ assert(curr > matchIndex);
+
+ if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) {
+ assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */
+ match = base + matchIndex;
+ if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit);
+ } else {
+ match = dictBase + matchIndex;
+ assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* prepare for match[matchLength] read */
+ }
+
+ if (matchLength > bestLength) {
+ DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)",
+ (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
+ assert(matchEndIdx > matchIndex);
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ bestLength = matchLength;
+ matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
+ matches[mnum].len = (U32)matchLength;
+ mnum++;
+ if ( (matchLength > ZSTD_OPT_NUM)
+ | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) {
+ if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */
+ break; /* drop, to preserve bt consistency (miss a little bit of compression) */
+ } }
+
+ if (match[matchLength] < ip[matchLength]) {
+ /* match smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */
+ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */
+ } else {
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+
+ assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dictMatchState && nbCompares) {
+ size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls);
+ U32 dictMatchIndex = dms->hashTable[dmsH];
+ const U32* const dmsBt = dms->chainTable;
+ commonLengthSmaller = commonLengthLarger = 0;
+ for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) {
+ const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE* match = dmsBase + dictMatchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart);
+ if (dictMatchIndex+matchLength >= dmsHighLimit)
+ match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */
+
+ if (matchLength > bestLength) {
+ matchIndex = dictMatchIndex + dmsIndexDelta;
+ DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)",
+ (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ bestLength = matchLength;
+ matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
+ matches[mnum].len = (U32)matchLength;
+ mnum++;
+ if ( (matchLength > ZSTD_OPT_NUM)
+ | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) {
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ } }
+
+ if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */
+ if (match[matchLength] < ip[matchLength]) {
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ } else {
+ /* match is larger than current */
+ commonLengthLarger = matchLength;
+ dictMatchIndex = nextPtr[0];
+ } } } /* if (dictMode == ZSTD_dictMatchState) */
+
+ assert(matchEndIdx > curr+8);
+ ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
+ return mnum;
+}
+
+typedef U32 (*ZSTD_getAllMatchesFn)(
+ ZSTD_match_t*,
+ ZSTD_matchState_t*,
+ U32*,
+ const BYTE*,
+ const BYTE*,
+ const U32 rep[ZSTD_REP_NUM],
+ U32 const ll0,
+ U32 const lengthToBeat);
+
+FORCE_INLINE_TEMPLATE U32 ZSTD_btGetAllMatches_internal(
+ ZSTD_match_t* matches,
+ ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
+ const BYTE* ip,
+ const BYTE* const iHighLimit,
+ const U32 rep[ZSTD_REP_NUM],
+ U32 const ll0,
+ U32 const lengthToBeat,
+ const ZSTD_dictMode_e dictMode,
+ const U32 mls)
+{
+ assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls);
+ DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls);
+ if (ip < ms->window.base + ms->nextToUpdate)
+ return 0; /* skipped area */
+ ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode);
+ return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls);
+}
+
+#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls
+
+#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \
+ static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \
+ ZSTD_match_t* matches, \
+ ZSTD_matchState_t* ms, \
+ U32* nextToUpdate3, \
+ const BYTE* ip, \
+ const BYTE* const iHighLimit, \
+ const U32 rep[ZSTD_REP_NUM], \
+ U32 const ll0, \
+ U32 const lengthToBeat) \
+ { \
+ return ZSTD_btGetAllMatches_internal( \
+ matches, ms, nextToUpdate3, ip, iHighLimit, \
+ rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \
+ }
+
+#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6)
+
+GEN_ZSTD_BT_GET_ALL_MATCHES(noDict)
+GEN_ZSTD_BT_GET_ALL_MATCHES(extDict)
+GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState)
+
+#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \
+ { \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \
+ }
+
+static ZSTD_getAllMatchesFn
+ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode)
+{
+ ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = {
+ ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict),
+ ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict),
+ ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState)
+ };
+ U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6);
+ assert((U32)dictMode < 3);
+ assert(mls - 3 < 4);
+ return getAllMatchesFns[(int)dictMode][mls - 3];
+}
+
+/*************************
+* LDM helper functions *
+*************************/
+
+/* Struct containing info needed to make decision about ldm inclusion */
+typedef struct {
+ rawSeqStore_t seqStore; /* External match candidates store for this block */
+ U32 startPosInBlock; /* Start position of the current match candidate */
+ U32 endPosInBlock; /* End position of the current match candidate */
+ U32 offset; /* Offset of the match candidate */
+} ZSTD_optLdm_t;
+
+/* ZSTD_optLdm_skipRawSeqStoreBytes():
+ * Moves forward in @rawSeqStore by @nbBytes,
+ * which will update the fields 'pos' and 'posInSequence'.
+ */
+static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes)
+{
+ U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
+ while (currPos && rawSeqStore->pos < rawSeqStore->size) {
+ rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
+ if (currPos >= currSeq.litLength + currSeq.matchLength) {
+ currPos -= currSeq.litLength + currSeq.matchLength;
+ rawSeqStore->pos++;
+ } else {
+ rawSeqStore->posInSequence = currPos;
+ break;
+ }
+ }
+ if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
+ rawSeqStore->posInSequence = 0;
+ }
+}
+
+/* ZSTD_opt_getNextMatchAndUpdateSeqStore():
+ * Calculates the beginning and end of the next match in the current block.
+ * Updates 'pos' and 'posInSequence' of the ldmSeqStore.
+ */
+static void
+ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock,
+ U32 blockBytesRemaining)
+{
+ rawSeq currSeq;
+ U32 currBlockEndPos;
+ U32 literalsBytesRemaining;
+ U32 matchBytesRemaining;
+
+ /* Setting match end position to MAX to ensure we never use an LDM during this block */
+ if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
+ optLdm->startPosInBlock = UINT_MAX;
+ optLdm->endPosInBlock = UINT_MAX;
+ return;
+ }
+ /* Calculate appropriate bytes left in matchLength and litLength
+ * after adjusting based on ldmSeqStore->posInSequence */
+ currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos];
+ assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength);
+ currBlockEndPos = currPosInBlock + blockBytesRemaining;
+ literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ?
+ currSeq.litLength - (U32)optLdm->seqStore.posInSequence :
+ 0;
+ matchBytesRemaining = (literalsBytesRemaining == 0) ?
+ currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) :
+ currSeq.matchLength;
+
+ /* If there are more literal bytes than bytes remaining in block, no ldm is possible */
+ if (literalsBytesRemaining >= blockBytesRemaining) {
+ optLdm->startPosInBlock = UINT_MAX;
+ optLdm->endPosInBlock = UINT_MAX;
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining);
+ return;
+ }
+
+ /* Matches may be < MINMATCH by this process. In that case, we will reject them
+ when we are deciding whether or not to add the ldm */
+ optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining;
+ optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining;
+ optLdm->offset = currSeq.offset;
+
+ if (optLdm->endPosInBlock > currBlockEndPos) {
+ /* Match ends after the block ends, we can't use the whole match */
+ optLdm->endPosInBlock = currBlockEndPos;
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock);
+ } else {
+ /* Consume nb of bytes equal to size of sequence left */
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining);
+ }
+}
+
+/* ZSTD_optLdm_maybeAddMatch():
+ * Adds a match if it's long enough,
+ * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock',
+ * into 'matches'. Maintains the correct ordering of 'matches'.
+ */
+static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
+ const ZSTD_optLdm_t* optLdm, U32 currPosInBlock)
+{
+ U32 const posDiff = currPosInBlock - optLdm->startPosInBlock;
+ /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */
+ U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
+
+ /* Ensure that current block position is not outside of the match */
+ if (currPosInBlock < optLdm->startPosInBlock
+ || currPosInBlock >= optLdm->endPosInBlock
+ || candidateMatchLength < MINMATCH) {
+ return;
+ }
+
+ if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) {
+ U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset);
+ DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u",
+ candidateOffBase, candidateMatchLength, currPosInBlock);
+ matches[*nbMatches].len = candidateMatchLength;
+ matches[*nbMatches].off = candidateOffBase;
+ (*nbMatches)++;
+ }
+}
+
+/* ZSTD_optLdm_processMatchCandidate():
+ * Wrapper function to update ldm seq store and call ldm functions as necessary.
+ */
+static void
+ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
+ ZSTD_match_t* matches, U32* nbMatches,
+ U32 currPosInBlock, U32 remainingBytes)
+{
+ if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
+ return;
+ }
+
+ if (currPosInBlock >= optLdm->endPosInBlock) {
+ if (currPosInBlock > optLdm->endPosInBlock) {
+ /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily
+ * at the end of a match from the ldm seq store, and will often be some bytes
+ * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots"
+ */
+ U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock;
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot);
+ }
+ ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes);
+ }
+ ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock);
+}
+
+
+/*-*******************************
+* Optimal parser
+*********************************/
+
+static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
+{
+ return sol.litlen + sol.mlen;
+}
+
+#if 0 /* debug */
+
+static void
+listStats(const U32* table, int lastEltID)
+{
+ int const nbElts = lastEltID + 1;
+ int enb;
+ for (enb=0; enb < nbElts; enb++) {
+ (void)table;
+ /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */
+ RAWLOG(2, "%4i,", table[enb]);
+ }
+ RAWLOG(2, " \n");
+}
+
+#endif
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
+ seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize,
+ const int optLevel,
+ const ZSTD_dictMode_e dictMode)
+{
+ optState_t* const optStatePtr = &ms->opt;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - 8;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const prefixStart = base + ms->window.dictLimit;
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+
+ ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode);
+
+ U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
+ U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
+ U32 nextToUpdate3 = ms->nextToUpdate;
+
+ ZSTD_optimal_t* const opt = optStatePtr->priceTable;
+ ZSTD_match_t* const matches = optStatePtr->matchTable;
+ ZSTD_optimal_t lastSequence;
+ ZSTD_optLdm_t optLdm;
+
+ optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
+ optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
+ ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
+
+ /* init */
+ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
+ (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
+ assert(optLevel <= 2);
+ ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
+ ip += (ip==prefixStart);
+
+ /* Match Loop */
+ while (ip < ilimit) {
+ U32 cur, last_pos = 0;
+
+ /* find first match */
+ { U32 const litlen = (U32)(ip - anchor);
+ U32 const ll0 = !litlen;
+ U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch);
+ ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
+ (U32)(ip-istart), (U32)(iend - ip));
+ if (!nbMatches) { ip++; continue; }
+
+ /* initialize opt[0] */
+ { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
+ opt[0].mlen = 0; /* means is_a_literal */
+ opt[0].litlen = litlen;
+ /* We don't need to include the actual price of the literals because
+ * it is static for the duration of the forward pass, and is included
+ * in every price. We include the literal length to avoid negative
+ * prices when we subtract the previous literal length.
+ */
+ opt[0].price = (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel);
+
+ /* large match -> immediate encoding */
+ { U32 const maxML = matches[nbMatches-1].len;
+ U32 const maxOffBase = matches[nbMatches-1].off;
+ DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series",
+ nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart));
+
+ if (maxML > sufficient_len) {
+ lastSequence.litlen = litlen;
+ lastSequence.mlen = maxML;
+ lastSequence.off = maxOffBase;
+ DEBUGLOG(6, "large match (%u>%u), immediate encoding",
+ maxML, sufficient_len);
+ cur = 0;
+ last_pos = ZSTD_totalLen(lastSequence);
+ goto _shortestPath;
+ } }
+
+ /* set prices for first matches starting position == 0 */
+ assert(opt[0].price >= 0);
+ { U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
+ U32 pos;
+ U32 matchNb;
+ for (pos = 1; pos < minMatch; pos++) {
+ opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */
+ }
+ for (matchNb = 0; matchNb < nbMatches; matchNb++) {
+ U32 const offBase = matches[matchNb].off;
+ U32 const end = matches[matchNb].len;
+ for ( ; pos <= end ; pos++ ) {
+ U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
+ U32 const sequencePrice = literalsPrice + matchPrice;
+ DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
+ pos, ZSTD_fCost((int)sequencePrice));
+ opt[pos].mlen = pos;
+ opt[pos].off = offBase;
+ opt[pos].litlen = litlen;
+ opt[pos].price = (int)sequencePrice;
+ } }
+ last_pos = pos-1;
+ }
+ }
+
+ /* check further positions */
+ for (cur = 1; cur <= last_pos; cur++) {
+ const BYTE* const inr = ip + cur;
+ assert(cur < ZSTD_OPT_NUM);
+ DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur)
+
+ /* Fix current position with one literal if cheaper */
+ { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1;
+ int const price = opt[cur-1].price
+ + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel)
+ + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel)
+ - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel);
+ assert(price < 1000000000); /* overflow check */
+ if (price <= opt[cur].price) {
+ DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
+ inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
+ opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]);
+ opt[cur].mlen = 0;
+ opt[cur].off = 0;
+ opt[cur].litlen = litlen;
+ opt[cur].price = price;
+ } else {
+ DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)",
+ inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price),
+ opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]);
+ }
+ }
+
+ /* Set the repcodes of the current position. We must do it here
+ * because we rely on the repcodes of the 2nd to last sequence being
+ * correct to set the next chunks repcodes during the backward
+ * traversal.
+ */
+ ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
+ assert(cur >= opt[cur].mlen);
+ if (opt[cur].mlen != 0) {
+ U32 const prev = cur - opt[cur].mlen;
+ repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0);
+ ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
+ } else {
+ ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
+ }
+
+ /* last match must start at a minimum distance of 8 from oend */
+ if (inr > ilimit) continue;
+
+ if (cur == last_pos) break;
+
+ if ( (optLevel==0) /*static_test*/
+ && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) {
+ DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1);
+ continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
+ }
+
+ assert(opt[cur].price >= 0);
+ { U32 const ll0 = (opt[cur].mlen != 0);
+ U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
+ U32 const previousPrice = (U32)opt[cur].price;
+ U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
+ U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch);
+ U32 matchNb;
+
+ ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
+ (U32)(inr-istart), (U32)(iend-inr));
+
+ if (!nbMatches) {
+ DEBUGLOG(7, "rPos:%u : no match found", cur);
+ continue;
+ }
+
+ { U32 const maxML = matches[nbMatches-1].len;
+ DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u",
+ inr-istart, cur, nbMatches, maxML);
+
+ if ( (maxML > sufficient_len)
+ || (cur + maxML >= ZSTD_OPT_NUM) ) {
+ lastSequence.mlen = maxML;
+ lastSequence.off = matches[nbMatches-1].off;
+ lastSequence.litlen = litlen;
+ cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */
+ last_pos = cur + ZSTD_totalLen(lastSequence);
+ if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */
+ goto _shortestPath;
+ } }
+
+ /* set prices using matches found at position == cur */
+ for (matchNb = 0; matchNb < nbMatches; matchNb++) {
+ U32 const offset = matches[matchNb].off;
+ U32 const lastML = matches[matchNb].len;
+ U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
+ U32 mlen;
+
+ DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u",
+ matchNb, matches[matchNb].off, lastML, litlen);
+
+ for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */
+ U32 const pos = cur + mlen;
+ int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
+
+ if ((pos > last_pos) || (price < opt[pos].price)) {
+ DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)",
+ pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
+ while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */
+ opt[pos].mlen = mlen;
+ opt[pos].off = offset;
+ opt[pos].litlen = litlen;
+ opt[pos].price = price;
+ } else {
+ DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)",
+ pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
+ if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */
+ }
+ } } }
+ } /* for (cur = 1; cur <= last_pos; cur++) */
+
+ lastSequence = opt[last_pos];
+ cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */
+ assert(cur < ZSTD_OPT_NUM); /* control overflow*/
+
+_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
+ assert(opt[0].mlen == 0);
+
+ /* Set the next chunk's repcodes based on the repcodes of the beginning
+ * of the last match, and the last sequence. This avoids us having to
+ * update them while traversing the sequences.
+ */
+ if (lastSequence.mlen != 0) {
+ repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0);
+ ZSTD_memcpy(rep, &reps, sizeof(reps));
+ } else {
+ ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t));
+ }
+
+ { U32 const storeEnd = cur + 1;
+ U32 storeStart = storeEnd;
+ U32 seqPos = cur;
+
+ DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
+ last_pos, cur); (void)last_pos;
+ assert(storeEnd < ZSTD_OPT_NUM);
+ DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
+ storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
+ opt[storeEnd] = lastSequence;
+ while (seqPos > 0) {
+ U32 const backDist = ZSTD_totalLen(opt[seqPos]);
+ storeStart--;
+ DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
+ seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off);
+ opt[storeStart] = opt[seqPos];
+ seqPos = (seqPos > backDist) ? seqPos - backDist : 0;
+ }
+
+ /* save sequences */
+ DEBUGLOG(6, "sending selected sequences into seqStore")
+ { U32 storePos;
+ for (storePos=storeStart; storePos <= storeEnd; storePos++) {
+ U32 const llen = opt[storePos].litlen;
+ U32 const mlen = opt[storePos].mlen;
+ U32 const offBase = opt[storePos].off;
+ U32 const advance = llen + mlen;
+ DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
+ anchor - istart, (unsigned)llen, (unsigned)mlen);
+
+ if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
+ assert(storePos == storeEnd); /* must be last sequence */
+ ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */
+ continue; /* will finish */
+ }
+
+ assert(anchor + llen <= iend);
+ ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen);
+ ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen);
+ anchor += advance;
+ ip = anchor;
+ } }
+ ZSTD_setBasePrices(optStatePtr, optLevel);
+ }
+ } /* while (ip < ilimit) */
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+static size_t ZSTD_compressBlock_opt0(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
+{
+ return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode);
+}
+
+static size_t ZSTD_compressBlock_opt2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
+{
+ return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode);
+}
+
+size_t ZSTD_compressBlock_btopt(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressBlock_btopt");
+ return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
+}
+
+
+
+
+/* ZSTD_initStats_ultra():
+ * make a first compression pass, just to seed stats with more accurate starting values.
+ * only works on first block, with no dictionary and no ldm.
+ * this function cannot error out, its narrow contract must be respected.
+ */
+static void
+ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
+ seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
+ ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep));
+
+ DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
+ assert(ms->opt.litLengthSum == 0); /* first block */
+ assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
+ assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
+ assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
+
+ ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/
+
+ /* invalidate first scan from history, only keep entropy stats */
+ ZSTD_resetSeqStore(seqStore);
+ ms->window.base -= srcSize;
+ ms->window.dictLimit += (U32)srcSize;
+ ms->window.lowLimit = ms->window.dictLimit;
+ ms->nextToUpdate = ms->window.dictLimit;
+
+}
+
+size_t ZSTD_compressBlock_btultra(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_btultra2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ U32 const curr = (U32)((const BYTE*)src - ms->window.base);
+ DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
+
+ /* 2-passes strategy:
+ * this strategy makes a first pass over first block to collect statistics
+ * in order to seed next round's statistics with it.
+ * After 1st pass, function forgets history, and starts a new block.
+ * Consequently, this can only work if no data has been previously loaded in tables,
+ * aka, no dictionary, no prefix, no ldm preprocessing.
+ * The compression ratio gain is generally small (~0.5% on first block),
+ ** the cost is 2x cpu time on first block. */
+ assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
+ if ( (ms->opt.litLengthSum==0) /* first block */
+ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
+ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
+ && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
+ && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */
+ ) {
+ ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
+ }
+
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_btopt_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_btultra_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_btopt_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
+}
+
+size_t ZSTD_compressBlock_btultra_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
+}
+
+/* note : no btultra2 variant for extDict nor dictMatchState,
+ * because btultra2 is not meant to work with dictionaries
+ * and is only specific for the first block (no prefix) */
diff --git a/contrib/zstd/zstd_opt.h b/contrib/zstd/zstd_opt.h
new file mode 100644
index 0000000..342e5a3
--- /dev/null
+++ b/contrib/zstd/zstd_opt.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_OPT_H
+#define ZSTD_OPT_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "zstd_compress_internal.h"
+
+/* used in ZSTD_loadDictionaryContent() */
+void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend);
+
+size_t ZSTD_compressBlock_btopt(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+
+size_t ZSTD_compressBlock_btopt_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_btopt_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+ /* note : no btultra2 variant for extDict nor dictMatchState,
+ * because btultra2 is not meant to work with dictionaries
+ * and is only specific for the first block (no prefix) */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_OPT_H */
diff --git a/contrib/zstd/zstd_trace.h b/contrib/zstd/zstd_trace.h
new file mode 100644
index 0000000..da20534
--- /dev/null
+++ b/contrib/zstd/zstd_trace.h
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_TRACE_H
+#define ZSTD_TRACE_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include <stddef.h>
+
+/* weak symbol support
+ * For now, enable conservatively:
+ * - Only GNUC
+ * - Only ELF
+ * - Only x86-64, i386 and aarch64
+ * Also, explicitly disable on platforms known not to work so they aren't
+ * forgotten in the future.
+ */
+#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
+ defined(__GNUC__) && defined(__ELF__) && \
+ (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \
+ !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
+ !defined(__CYGWIN__) && !defined(_AIX)
+# define ZSTD_HAVE_WEAK_SYMBOLS 1
+#else
+# define ZSTD_HAVE_WEAK_SYMBOLS 0
+#endif
+#if ZSTD_HAVE_WEAK_SYMBOLS
+# define ZSTD_WEAK_ATTR __attribute__((__weak__))
+#else
+# define ZSTD_WEAK_ATTR
+#endif
+
+/* Only enable tracing when weak symbols are available. */
+#ifndef ZSTD_TRACE
+# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
+#endif
+
+#if ZSTD_TRACE
+
+struct ZSTD_CCtx_s;
+struct ZSTD_DCtx_s;
+struct ZSTD_CCtx_params_s;
+
+typedef struct {
+ /**
+ * ZSTD_VERSION_NUMBER
+ *
+ * This is guaranteed to be the first member of ZSTD_trace.
+ * Otherwise, this struct is not stable between versions. If
+ * the version number does not match your expectation, you
+ * should not interpret the rest of the struct.
+ */
+ unsigned version;
+ /**
+ * Non-zero if streaming (de)compression is used.
+ */
+ unsigned streaming;
+ /**
+ * The dictionary ID.
+ */
+ unsigned dictionaryID;
+ /**
+ * Is the dictionary cold?
+ * Only set on decompression.
+ */
+ unsigned dictionaryIsCold;
+ /**
+ * The dictionary size or zero if no dictionary.
+ */
+ size_t dictionarySize;
+ /**
+ * The uncompressed size of the data.
+ */
+ size_t uncompressedSize;
+ /**
+ * The compressed size of the data.
+ */
+ size_t compressedSize;
+ /**
+ * The fully resolved CCtx parameters (NULL on decompression).
+ */
+ struct ZSTD_CCtx_params_s const* params;
+ /**
+ * The ZSTD_CCtx pointer (NULL on decompression).
+ */
+ struct ZSTD_CCtx_s const* cctx;
+ /**
+ * The ZSTD_DCtx pointer (NULL on compression).
+ */
+ struct ZSTD_DCtx_s const* dctx;
+} ZSTD_Trace;
+
+/**
+ * A tracing context. It must be 0 when tracing is disabled.
+ * Otherwise, any non-zero value returned by a tracing begin()
+ * function is presented to any subsequent calls to end().
+ *
+ * Any non-zero value is treated as tracing is enabled and not
+ * interpreted by the library.
+ *
+ * Two possible uses are:
+ * * A timestamp for when the begin() function was called.
+ * * A unique key identifying the (de)compression, like the
+ * address of the [dc]ctx pointer if you need to track
+ * more information than just a timestamp.
+ */
+typedef unsigned long long ZSTD_TraceCtx;
+
+/**
+ * Trace the beginning of a compression call.
+ * @param cctx The dctx pointer for the compression.
+ * It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled. The return value is
+ * passed to ZSTD_trace_compress_end().
+ */
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
+ struct ZSTD_CCtx_s const* cctx);
+
+/**
+ * Trace the end of a compression call.
+ * @param ctx The return value of ZSTD_trace_compress_begin().
+ * @param trace The zstd tracing info.
+ */
+ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
+ ZSTD_TraceCtx ctx,
+ ZSTD_Trace const* trace);
+
+/**
+ * Trace the beginning of a decompression call.
+ * @param dctx The dctx pointer for the decompression.
+ * It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled. The return value is
+ * passed to ZSTD_trace_compress_end().
+ */
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
+ struct ZSTD_DCtx_s const* dctx);
+
+/**
+ * Trace the end of a decompression call.
+ * @param ctx The return value of ZSTD_trace_decompress_begin().
+ * @param trace The zstd tracing info.
+ */
+ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
+ ZSTD_TraceCtx ctx,
+ ZSTD_Trace const* trace);
+
+#endif /* ZSTD_TRACE */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_TRACE_H */